mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-22 00:45:10 +00:00
fix(mp4): serialize pymp4 parses of bit-field boxes across threads
This commit is contained in:
@@ -0,0 +1,44 @@
|
|||||||
|
"""get_boxes must give each thread its own box when tracks read their init segments at the same time."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from unshackle.core.utilities import get_boxes
|
||||||
|
|
||||||
|
|
||||||
|
def tenc_init(kid: UUID, version: int, crypt_skip: int) -> bytes:
|
||||||
|
body = struct.pack(">B3s", version, b"\0\0\0") + bytes([0, crypt_skip if version else 0]) + b"\x01\x10" + kid.bytes
|
||||||
|
return b"\x00" * 8 + struct.pack(">I", 8 + len(body)) + b"tenc" + body
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_boxes_finds_the_tenc() -> None:
|
||||||
|
kid = uuid4()
|
||||||
|
(box,) = get_boxes(tenc_init(kid, 1, 0x19), b"tenc")
|
||||||
|
assert box.key_ID == kid
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_boxes_is_right_when_tracks_parse_at_the_same_time() -> None:
|
||||||
|
cases = [(uuid4(), version, crypt_skip) for version, crypt_skip in ((0, 0), (1, 0x19), (1, 0x91), (1, 0x55))] * 2
|
||||||
|
failures: list[str] = []
|
||||||
|
|
||||||
|
def parse(kid: UUID, version: int, crypt_skip: int) -> None:
|
||||||
|
data = tenc_init(kid, version, crypt_skip)
|
||||||
|
for _ in range(1500):
|
||||||
|
try:
|
||||||
|
boxes = list(get_boxes(data, b"tenc"))
|
||||||
|
except Exception as e:
|
||||||
|
failures.append(repr(e))
|
||||||
|
continue
|
||||||
|
if [box.key_ID for box in boxes] != [kid]:
|
||||||
|
failures.append("wrong or missing tenc")
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=parse, args=case) for case in cases]
|
||||||
|
for thread in threads:
|
||||||
|
thread.start()
|
||||||
|
for thread in threads:
|
||||||
|
thread.join(timeout=120)
|
||||||
|
assert not failures, failures[:3]
|
||||||
|
assert not any(thread.is_alive() for thread in threads)
|
||||||
@@ -25,7 +25,7 @@ from subtitle_filter import Subtitles
|
|||||||
from unshackle.core import binaries
|
from unshackle.core import binaries
|
||||||
from unshackle.core.config import config
|
from unshackle.core.config import config
|
||||||
from unshackle.core.tracks.track import Track
|
from unshackle.core.tracks.track import Track
|
||||||
from unshackle.core.utilities import try_ensure_utf8, with_script
|
from unshackle.core.utilities import PYMP4_LOCK, try_ensure_utf8, with_script
|
||||||
from unshackle.core.utils.subprocess import log_tool_run
|
from unshackle.core.utils.subprocess import log_tool_run
|
||||||
from unshackle.core.utils.webvtt import merge_segmented_webvtt, rebase_webvtt
|
from unshackle.core.utils.webvtt import merge_segmented_webvtt, rebase_webvtt
|
||||||
|
|
||||||
@@ -737,10 +737,10 @@ class Subtitle(Track):
|
|||||||
caption_set = pycaption.SRTReader().read(text)
|
caption_set = pycaption.SRTReader().read(text)
|
||||||
elif codec == Subtitle.Codec.fTTML:
|
elif codec == Subtitle.Codec.fTTML:
|
||||||
caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList)
|
caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList)
|
||||||
|
with PYMP4_LOCK:
|
||||||
|
boxes = MP4.parse_stream(BytesIO(data))
|
||||||
for segment in (
|
for segment in (
|
||||||
Subtitle.parse(box.data, Subtitle.Codec.TimedTextMarkupLang)
|
Subtitle.parse(box.data, Subtitle.Codec.TimedTextMarkupLang) for box in boxes if box.type == b"mdat"
|
||||||
for box in MP4.parse_stream(BytesIO(data))
|
|
||||||
if box.type == b"mdat"
|
|
||||||
):
|
):
|
||||||
for lang in segment.get_languages():
|
for lang in segment.get_languages():
|
||||||
caption_lists[lang].extend(segment.get_captions(lang))
|
caption_lists[lang].extend(segment.get_captions(lang))
|
||||||
@@ -923,7 +923,9 @@ class Subtitle(Track):
|
|||||||
child["type"] = child.get("type") or child.get("format")
|
child["type"] = child.get("type") or child.get("format")
|
||||||
yield child
|
yield child
|
||||||
|
|
||||||
for box in flatten_boxes(MP4.parse_stream(BytesIO(data))):
|
with PYMP4_LOCK:
|
||||||
|
boxes = MP4.parse_stream(BytesIO(data))
|
||||||
|
for box in flatten_boxes(boxes):
|
||||||
# init
|
# init
|
||||||
if box.type == b"mdhd":
|
if box.type == b"mdhd":
|
||||||
timescale = box.timescale
|
timescale = box.timescale
|
||||||
@@ -963,7 +965,8 @@ class Subtitle(Track):
|
|||||||
if not saw_trun_box:
|
if not saw_trun_box:
|
||||||
raise ValueError("The TRUN box was not found in the Segmented WebVTT.")
|
raise ValueError("The TRUN box was not found in the Segmented WebVTT.")
|
||||||
|
|
||||||
vttc_boxes = MP4.parse_stream(BytesIO(box.data))
|
with PYMP4_LOCK:
|
||||||
|
vttc_boxes = MP4.parse_stream(BytesIO(box.data))
|
||||||
current_time = base_time + period_start
|
current_time = base_time + period_start
|
||||||
|
|
||||||
for sample, vttc_box in zip(samples, vttc_boxes):
|
for sample, vttc_box in zip(samples, vttc_boxes):
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@@ -456,6 +457,11 @@ def resolve_sort_langs(
|
|||||||
return resolved
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
# construct 2.8 gives each bit-field struct one shared stream, so two threads in a pymp4
|
||||||
|
# parse or build of a box with bit fields (tenc, senc, trun, sidx, ...) read each other's bytes
|
||||||
|
PYMP4_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore
|
def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore
|
||||||
"""
|
"""
|
||||||
Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find.
|
Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find.
|
||||||
@@ -506,14 +512,13 @@ def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # t
|
|||||||
box_start = pos - 4
|
box_start = pos - 4
|
||||||
|
|
||||||
try:
|
try:
|
||||||
box = Box.parse(data[box_start:])
|
with PYMP4_LOCK:
|
||||||
if as_bytes:
|
box = Box.parse(data[box_start:])
|
||||||
box = Box.build(box)
|
built = Box.build(box)
|
||||||
|
|
||||||
yield box
|
yield built if as_bytes else box
|
||||||
|
|
||||||
box_size = len(Box.build(box))
|
offset = box_start + len(built)
|
||||||
offset = box_start + box_size
|
|
||||||
|
|
||||||
except IOError:
|
except IOError:
|
||||||
break
|
break
|
||||||
|
|||||||
Reference in New Issue
Block a user