Merge branch 'dev' into feat/downloader-speed

Catch the branch up with dev (45 commits) so testers can run the perf work
against current dev.

Conflict resolutions:
- downloaders/requests.py: keep the branch's racing-gated read1, adaptive
  controller speed reporting and byte-range slice guards; take dev's
  content-encoding checks (raw reads are skipped on an encoded body) and its
  rolling SpeedWindow for the no-controller speed fallback. The single-URL
  ranged-parallel gate now requires no speed limit and no caller Range.
- service.py / session.py: keep the branch's TimeoutHTTPAdapter and retry
  config, raise POOL_MAX_SIZE to 64 so dev's pool-starvation fix survives
  (pool_block=True stalls threads on a pool smaller than the in-flight count).
- docs: dev split configuration.md into per-area pages, so the branch's rnet
  network option rows moved to reference/configuration/network.md.
This commit is contained in:
imSp4rky
2026-08-07 10:45:47 -06:00
104 changed files with 8567 additions and 1375 deletions
+59
View File
@@ -0,0 +1,59 @@
"""URL-backed attachments defer their fetch to download(), which honors no_proxy_download."""
from __future__ import annotations
import requests
import unshackle.core.tracks.track as track_module
from unshackle.core.config import config
from unshackle.core.tracks.attachment import Attachment
URL = "https://cdn.example.com/images/thumb.jpg"
class FakeResponse:
def raise_for_status(self) -> None:
pass
def iter_content(self, chunk_size: int):
yield b"jpegdata"
class FakeSession(requests.Session):
def __init__(self, proxies: dict | None = None) -> None:
super().__init__()
self.proxies = proxies or {}
self.got: list[str] = []
def get(self, url, **kwargs): # type: ignore[override]
self.got.append(url)
return FakeResponse()
def test_url_attachment_defers_download() -> None:
session = FakeSession()
attachment = Attachment.from_url(URL, name="thumbnail", session=session)
assert attachment.path is None
assert not session.got
assert attachment.mime_type == "image/jpeg"
def test_download_without_flag_uses_proxied_session(tmp_path, monkeypatch) -> None:
monkeypatch.setattr(config.directories, "temp", tmp_path)
session = FakeSession(proxies={"https": "http://proxy:8080"})
attachment = Attachment.from_url(URL, name="thumbnail", session=session)
attachment.download(session)
assert session.got == [URL]
assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata"
def test_download_with_flag_bypasses_proxy(tmp_path, monkeypatch) -> None:
monkeypatch.setattr(config.directories, "temp", tmp_path)
direct = FakeSession()
monkeypatch.setattr(track_module, "direct_session", lambda session: direct)
proxied = FakeSession(proxies={"https": "http://proxy:8080"})
attachment = Attachment.from_url(URL, name="thumbnail", session=proxied)
attachment.download(proxied, no_proxy_download=True)
assert not proxied.got
assert direct.got == [URL]
assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata"
@@ -0,0 +1,86 @@
"""Self-check for ``has_encrypted_sample_entry``: a finalized MP4 whose moov still carries
an encv/enca sample entry means the decrypt tool skipped/failed the track (frma was never
applied), so the detector must flag it before the file reaches the muxer. Detection is
scoped to moov, so stray 4CC byte runs in mdat must never trip (or mask) it.
Run: uv run pytest tests/tracks/test_encrypted_sample_entry.py
"""
from __future__ import annotations
import struct
from pathlib import Path
from unshackle.core.tracks.track import has_encrypted_sample_entry
def _box(fmt: bytes, payload: bytes = b"") -> bytes:
return struct.pack(">I", 8 + len(payload)) + fmt + payload
def _sample_entry(fmt: bytes, payload: bytes = b"") -> bytes:
body = b"\x00" * 6 + struct.pack(">H", 1) + payload # reserved + data_reference_index
return struct.pack(">I", 8 + len(body)) + fmt + body
def _stsd(entries: list[bytes]) -> bytes:
body = b"\x00" * 4 + struct.pack(">I", len(entries)) + b"".join(entries) # version/flags + count
return _box(b"stsd", body)
def _moov(*children: bytes) -> bytes:
stbl = _box(b"stbl", b"".join(children))
return _box(b"moov", _box(b"trak", _box(b"mdia", _box(b"minf", stbl))))
FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2")
# stray [size][encv] byte run as compressed A/V payload could randomly contain
STRAY_ENCV = b"\xde\xad" * 4 + struct.pack(">I", 100) + b"encv" + b"\xbe\xef" * 8
def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path:
f = tmp_path / name
f.write_bytes(b"".join(boxes))
return f
def test_dual_stsd_encv_and_avc1_detected(tmp_path: Path) -> None:
sinf = _box(b"sinf", _box(b"frma", b"avc1"))
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
encv = _sample_entry(b"encv", b"\x00" * 70 + sinf)
f = _write(tmp_path, "dual.mp4", FTYP, _moov(_stsd([avc1, encv])), _box(b"mdat", b"\x00" * 32))
assert has_encrypted_sample_entry(f) is True
def test_single_avc1_not_detected(tmp_path: Path) -> None:
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
f = _write(tmp_path, "clear.mp4", FTYP, _moov(_stsd([avc1])), _box(b"mdat", b"\x00" * 32))
assert has_encrypted_sample_entry(f) is False
def test_stray_encv_in_mdat_not_detected(tmp_path: Path) -> None:
# clean moov, mdat happens to contain a plausible [size][encv] run -> must NOT flag
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
f = _write(tmp_path, "stray.mp4", FTYP, _moov(_stsd([avc1])), _box(b"mdat", STRAY_ENCV))
assert has_encrypted_sample_entry(f) is False
def test_leading_mdat_does_not_mask_real_encv(tmp_path: Path) -> None:
# progressive layout (mdat before moov) with a stray run must not hide a genuine encv
enca = _sample_entry(b"enca", b"\x00" * 20 + _box(b"sinf", _box(b"frma", b"mp4a")))
f = _write(tmp_path, "masked.mp4", FTYP, _box(b"mdat", STRAY_ENCV), _moov(_stsd([enca])))
assert has_encrypted_sample_entry(f) is True
def test_text_containing_enca_in_moov_not_detected(tmp_path: Path) -> None:
# "enca" inside a metadata string has no plausible box size before it -> must NOT flag
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
udta = _box(b"udta", b"data is encapsulated here")
f = _write(tmp_path, "text.mp4", FTYP, _box(b"moov", _stsd([avc1]) + udta))
assert has_encrypted_sample_entry(f) is False
def test_missing_or_bogus_file(tmp_path: Path) -> None:
assert has_encrypted_sample_entry(tmp_path / "nope.mp4") is False
bogus = _write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4")
assert has_encrypted_sample_entry(bogus) is False
+167
View File
@@ -0,0 +1,167 @@
"""Self-check for ``assert_fragments_decrypted``: after decryption every moof must have lost
its standard senc. A survivor means the tool silently skipped that fragment (the
tfhd.sample_description_index ceiling bug), which the moov-scoped has_encrypted_sample_entry
structurally cannot see. A leftover PIFF uuid survives a good decrypt too, because content
shipping both boxes keeps it: a decrypter detaches only the atom it consumed.
The guard never reads mdat, so stray byte runs there leave it unmoved.
Run: uv run pytest tests/tracks/test_fragment_decrypted.py
"""
from __future__ import annotations
import struct
from pathlib import Path
import pytest
from unshackle.core.manifests.ism_init import PIFF_SENC_UUID
from unshackle.core.tracks.track import assert_fragments_decrypted
def _box(fmt: bytes, payload: bytes = b"") -> bytes:
return struct.pack(">I", 8 + len(payload)) + fmt + payload
def _senc(sample_count: int = 4) -> bytes:
return _box(b"senc", b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count))
def _piff_senc(sample_count: int = 4) -> bytes:
body = PIFF_SENC_UUID + b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)
return _box(b"uuid", body)
def _moof(*children: bytes, sdi: int = 1) -> bytes:
tfhd = _box(b"tfhd", b"\x00\x00\x00\x02" + struct.pack(">II", 1, sdi)) # sample-description-index-present
return _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _box(b"traf", tfhd + b"".join(children)))
FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2")
MOOV = _box(b"moov", _box(b"mvex", _box(b"trex", b"\x00" * 24)))
MDAT = _box(b"mdat", b"\x00" * 64)
# a compressed payload can contain anything, including plausible-looking box runs
STRAY = _box(b"mdat", struct.pack(">I", 64) + b"senc" + b"\xbe\xef" * 8 + PIFF_SENC_UUID)
def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path:
f = tmp_path / name
f.write_bytes(b"".join(boxes))
return f
def test_all_fragments_decrypted_passes(tmp_path: Path) -> None:
f = _write(tmp_path, "clean.mp4", FTYP, MOOV, _moof(), MDAT, _moof(sdi=2), MDAT)
assert_fragments_decrypted(f)
def test_surviving_senc_raises(tmp_path: Path) -> None:
# the real bug: fragment 1 (index 1) decrypted, the rest (index 2) silently skipped
boxes = [FTYP, MOOV, _moof(), MDAT]
for _ in range(3):
boxes += [_moof(_senc(), sdi=2), MDAT]
f = _write(tmp_path, "skipped.mp4", *boxes)
with pytest.raises(ValueError, match=r"3/4 fragment"):
assert_fragments_decrypted(f)
def test_surviving_cenc_senc_raises(tmp_path: Path) -> None:
f = _write(tmp_path, "cenc.mp4", FTYP, MOOV, _moof(_senc()), MDAT)
with pytest.raises(ValueError, match=r"1/1 fragment"):
assert_fragments_decrypted(f)
def test_leftover_piff_uuid_after_senc_removed_does_not_raise(tmp_path: Path) -> None:
# live DASH shape: the traf shipped senc+saiz+saio plus a duplicate PIFF uuid. mp4decrypt
# consumed senc, detached senc/saiz/saio, and left the uuid on a fully decrypted fragment.
boxes = [FTYP, MOOV]
for _ in range(3):
boxes += [_moof(_piff_senc()), MDAT]
f = _write(tmp_path, "piff_leftover.mp4", *boxes)
assert_fragments_decrypted(f)
def test_stray_senc_bytes_in_mdat_ignored(tmp_path: Path) -> None:
f = _write(tmp_path, "stray.mp4", FTYP, MOOV, _moof(), STRAY, _moof(sdi=2), STRAY)
assert_fragments_decrypted(f)
def test_senc_text_without_plausible_size_ignored(tmp_path: Path) -> None:
# "senc" inside a free-box string has no valid box size before it
f = _write(tmp_path, "text.mp4", FTYP, MOOV, _moof(_box(b"free", b"presence sencoded here")), MDAT)
assert_fragments_decrypted(f)
def test_senc_bytes_inside_trun_payload_ignored(tmp_path: Path) -> None:
# a trun sample entry spelling "senc" after a size-looking duration; only a byte scan bites
trun = _box(b"trun", b"\x00\x00\x03\x01" + struct.pack(">I", 2) + struct.pack(">I", 512) + b"senc")
f = _write(tmp_path, "trun.mp4", FTYP, MOOV, _moof(trun), MDAT)
assert_fragments_decrypted(f)
def test_empty_senc_on_clear_fragment_ignored(tmp_path: Path) -> None:
# sample_count 0 marks a fragment with no protected samples, so the guard ignores it
f = _write(tmp_path, "empty.mp4", FTYP, MOOV, _moof(_senc(sample_count=0)), MDAT)
assert_fragments_decrypted(f)
def _senc_override(sample_count: int) -> bytes:
# flags&1 => AlgorithmID(3) + IV_size(1) + KID(16) precede sample_count
head = b"\x00\x00\x00\x01" + b"\x00\x00\x08" + b"\x08" + b"\xaa" * 16
return _box(b"senc", head + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count))
def test_override_flag_shifts_sample_count_offset(tmp_path: Path) -> None:
clear = _write(tmp_path, "ovr_clear.mp4", FTYP, MOOV, _moof(_senc_override(0)), MDAT)
assert_fragments_decrypted(clear)
f = _write(tmp_path, "ovr.mp4", FTYP, MOOV, _moof(_senc_override(5)), MDAT)
with pytest.raises(ValueError, match=r"1/1 fragment"):
assert_fragments_decrypted(f)
def test_senc_outside_traf_ignored(tmp_path: Path) -> None:
# a decrypted moof whose only "senc" sits in a sibling box outside any traf
f = _write(tmp_path, "sibling.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _senc()), MDAT)
assert_fragments_decrypted(f)
def test_senc_nested_under_non_traf_ignored(tmp_path: Path) -> None:
# senc two levels down under a non-traf container: only traf children count
meta = _box(b"meta", _senc())
f = _write(tmp_path, "nested.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + meta), MDAT)
assert_fragments_decrypted(f)
def test_undersized_largesize_box_does_not_desync_walk(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
# size==1 reads a 16-byte header, so a largesize of 12 clears the 8-byte floor but
# still stalls the cursor onto payload; only a header-aware floor catches it
runt = struct.pack(">I", 1) + b"free" + struct.pack(">Q", 12)
f = _write(tmp_path, "runt.mp4", FTYP, MOOV, runt, _moof(_senc()), MDAT, _moof(_senc()), MDAT)
with caplog.at_level("WARNING"):
assert_fragments_decrypted(f)
assert "malformed box size" in caplog.text
def test_largesize_and_zero_size_boxes_walked(tmp_path: Path) -> None:
frag = _moof(_senc())
large = struct.pack(">I", 1) + b"mdat" + struct.pack(">Q", 16 + 64) + b"\x00" * 64
trailing = struct.pack(">I", 0) + b"mdat" + b"\x00" * 64 # size 0 => runs to EOF
f = _write(tmp_path, "large.mp4", FTYP, MOOV, frag, large, trailing)
with pytest.raises(ValueError, match=r"1/1 fragment"):
assert_fragments_decrypted(f)
def test_malformed_box_size_warns_and_does_not_raise(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
# failing open is deliberate, but the abort must be logged so it cannot pass as clean
overrun = struct.pack(">I", 1 << 30) + b"free" + b"\x00" * 8
f = _write(tmp_path, "overrun.mp4", FTYP, MOOV, overrun, _moof(_senc()), MDAT)
with caplog.at_level("WARNING"):
assert_fragments_decrypted(f)
assert "malformed box size" in caplog.text
def test_missing_or_bogus_file_is_noop(tmp_path: Path) -> None:
assert_fragments_decrypted(tmp_path / "nope.mp4")
assert_fragments_decrypted(_write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4"))
truncated = _write(tmp_path, "cut.mp4", FTYP, MOOV, _moof(_senc())[:20])
assert_fragments_decrypted(truncated)
+159 -2
View File
@@ -3,8 +3,8 @@ import pytest
from unshackle.core.tracks import Subtitle, Tracks
def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) -> Subtitle:
return Subtitle(
def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False, orig: bool = False) -> Subtitle:
track = Subtitle(
url=f"https://example.test/{lang}-{forced}-{sdh}-{cc}",
language=lang,
codec=Subtitle.Codec.SubRip,
@@ -12,6 +12,8 @@ def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) ->
sdh=sdh,
cc=cc,
)
track.is_original_lang = orig
return track
def shape(tracks: Tracks) -> list[tuple[str, str]]:
@@ -66,3 +68,158 @@ def test_cc_counts_as_sdh() -> None:
t.add([sub("en", cc=True), sub("en")])
t.sort_subtitles(by_language=["en"], type_priority=["normal", "sdh"])
assert shape(t) == [("en", "normal"), ("en", "sdh")]
@pytest.fixture
def unprioritised() -> Tracks:
"""Languages that no by_language argument floats to the top."""
t = Tracks()
t.add([sub("fi", sdh=True), sub("fi"), sub("de", sdh=True), sub("de")])
return t
def test_type_grouping_separates_a_language_from_its_sdh(unprioritised: Tracks) -> None:
unprioritised.sort_subtitles()
assert shape(unprioritised) == [
("de", "normal"),
("fi", "normal"),
("de", "sdh"),
("fi", "sdh"),
]
def test_language_grouping_keeps_a_language_with_its_sdh(unprioritised: Tracks) -> None:
unprioritised.sort_subtitles(group_by="language")
assert shape(unprioritised) == [
("de", "normal"),
("de", "sdh"),
("fi", "normal"),
("fi", "sdh"),
]
def test_language_grouping_still_honours_type_priority_and_sections(tracks: Tracks) -> None:
# de sits outside by_language, where type-major sorting would interleave it with en
tracks.add([sub("de"), sub("de", sdh=True)])
tracks.sort_subtitles(by_language=["it"], type_priority=["normal", "sdh", "forced"], group_by="language")
assert shape(tracks) == [
("it", "normal"),
("it", "forced"),
("de", "normal"),
("de", "sdh"),
("en", "normal"),
("en", "sdh"),
("en", "forced"),
]
def test_language_grouping_keeps_regional_variants_adjacent_through_sections() -> None:
# "en" floats en-GB and en-US as one block, so each exact tag keeps its variants
t = Tracks()
t.add(
[
sub("en-US", sdh=True),
sub("fi"),
sub("en-GB"),
sub("en-US"),
sub("fi", sdh=True),
sub("en-GB", forced=True),
sub("en-GB", sdh=True),
]
)
t.sort_subtitles(by_language=["en"], group_by="language")
assert shape(t) == [
("en-GB", "forced"),
("en-GB", "normal"),
("en-GB", "sdh"),
("en-US", "normal"),
("en-US", "sdh"),
("fi", "normal"),
("fi", "sdh"),
]
def langs(tracks: Tracks) -> list[str]:
return [str(s.language) for s in tracks.subtitles]
@pytest.fixture
def many() -> Tracks:
t = Tracks()
t.add([sub(lang, orig=lang == "ja") for lang in ("ar", "bg", "en", "es", "fr", "ja")])
return t
def test_priority_orders_listed_then_rest_alphabetically(many: Tracks) -> None:
many.sort_subtitles(by_language=["en", "es", "fr"], group_by="language")
assert langs(many) == ["en", "es", "fr", "ar", "bg", "ja"]
def test_priority_outranks_the_original_language(many: Tracks) -> None:
# "all" resolves to the original language (ja), which must stay below the named ones
many.sort_subtitles(by_language=["en", "es", "all"], group_by="language")
assert langs(many) == ["en", "es", "ja", "ar", "bg", "fr"]
def test_priority_matches_regional_variants() -> None:
t = Tracks()
t.add([sub("ar"), sub("en-US"), sub("es-419"), sub("ja")])
t.sort_subtitles(by_language=["en", "es"], group_by="language")
assert langs(t) == ["en-US", "es-419", "ar", "ja"]
def test_priority_overlapping_entries_are_harmless() -> None:
t = Tracks()
t.add([sub("ar"), sub("en-GB"), sub("en-US"), sub("es"), sub("ja")])
t.sort_subtitles(by_language=["en", "en-US", "es"], group_by="language")
assert langs(t) == ["en-GB", "en-US", "es", "ar", "ja"]
def test_no_priority_leaves_order_unchanged(many: Tracks) -> None:
many.sort_subtitles(by_language=[], group_by="language")
assert langs(many) == ["ar", "bg", "en", "es", "fr", "ja"]
@pytest.mark.parametrize("token", ["all", "best"])
def test_magic_token_alone_floats_the_original_language(many: Tracks, token: str) -> None:
many.sort_subtitles(by_language=[token], group_by="language")
assert langs(many) == ["ja", "ar", "bg", "en", "es", "fr"]
@pytest.mark.parametrize("token", ["all", "best"])
def test_priority_outranks_either_magic_token(many: Tracks, token: str) -> None:
many.sort_subtitles(by_language=["en", "es", token], group_by="language")
assert langs(many) == ["en", "es", "ja", "ar", "bg", "fr"]
@pytest.fixture
def regionals() -> Tracks:
t = Tracks()
t.add([sub("ar"), sub("en"), sub("en-GB"), sub("en-US"), sub("ja")])
return t
def test_fuzzy_priority_floats_regional_variants(regionals: Tracks) -> None:
regionals.sort_subtitles(by_language=["en"], group_by="language")
assert langs(regionals) == ["en", "en-GB", "en-US", "ar", "ja"]
def test_exact_priority_floats_only_the_named_tag(regionals: Tracks) -> None:
# CLDR rates en/en-US at distance 0, so exact mode needs the RFC 4647 string preference
# to hold en-US back; without it en-US floats and en-GB does not.
regionals.sort_subtitles(by_language=["en"], group_by="language", exact_match=True)
assert langs(regionals) == ["en", "ar", "en-GB", "en-US", "ja"]
def test_exact_priority_still_falls_back_when_no_tag_is_string_equal() -> None:
t = Tracks()
t.add([sub("ar"), sub("cmn"), sub("ja")])
t.sort_subtitles(by_language=["zh"], group_by="language", exact_match=True)
assert langs(t) == ["cmn", "ar", "ja"]
@pytest.mark.parametrize("exact", [False, True])
def test_unparseable_priority_entry_is_ignored(many: Tracks, exact: bool) -> None:
# a config typo in language_priority must not crash the sort
many.sort_subtitles(by_language=["en", "engrish", "f", ""], group_by="language", exact_match=exact)
assert langs(many) == ["en", "ar", "bg", "es", "fr", "ja"]
+22
View File
@@ -0,0 +1,22 @@
from unshackle.core.tracks.subtitle import Subtitle
def test_strips_timestamp_map_keeps_cues():
text = "WEBVTT\nX-TIMESTAMP-MAP=MPEGTS:900000,LOCAL:00:00:00.000\n\n00:00:22.690 --> 00:00:24.710\nHi\n"
out = Subtitle.strip_webvtt_timestamp_map(text)
assert "X-TIMESTAMP-MAP" not in out
assert "00:00:22.690 --> 00:00:24.710" in out
assert out.startswith("WEBVTT\n")
def test_strips_lowercase_timestamp_map():
# SubtitleEdit honours the header case-insensitively, so the strip must too
text = "WEBVTT\nx-timestamp-map=MPEGTS:900000,LOCAL:00:00:00.000\n\n00:00:22.690 --> 00:00:24.710\nHi\n"
out = Subtitle.strip_webvtt_timestamp_map(text)
assert "x-timestamp-map" not in out.lower()
assert "00:00:22.690 --> 00:00:24.710" in out
def test_no_timestamp_map_is_untouched():
text = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nHi\n"
assert Subtitle.strip_webvtt_timestamp_map(text) == text