Merge branch 'dev' into feat/downloader-speed

Catch the branch up with dev (45 commits) so testers can run the perf work
against current dev.

Conflict resolutions:
- downloaders/requests.py: keep the branch's racing-gated read1, adaptive
  controller speed reporting and byte-range slice guards; take dev's
  content-encoding checks (raw reads are skipped on an encoded body) and its
  rolling SpeedWindow for the no-controller speed fallback. The single-URL
  ranged-parallel gate now requires no speed limit and no caller Range.
- service.py / session.py: keep the branch's TimeoutHTTPAdapter and retry
  config, raise POOL_MAX_SIZE to 64 so dev's pool-starvation fix survives
  (pool_block=True stalls threads on a pool smaller than the in-flight count).
- docs: dev split configuration.md into per-area pages, so the branch's rnet
  network option rows moved to reference/configuration/network.md.
This commit is contained in:
imSp4rky
2026-08-07 10:45:47 -06:00
104 changed files with 8567 additions and 1375 deletions
+178
View File
@@ -0,0 +1,178 @@
"""Tests for the HLS AES-128 ClearKey DRM system (``unshackle.core.drm.clearkey``).
Covers the correctness guards on the whole-segment AES-128-CBC path:
- ``ClearKey.decrypt`` passing mislabeled-clear / non-block-aligned segments through
untouched instead of crashing or corrupting (WS1 / F1+F3)
- ``ClearKey.from_m3u_key`` ``data:`` URI parsing and 16-byte key validation (WS5 / F4)
- the RFC 8216 §5.2 sequence-number IV deriving from a segment's absolute media
sequence number rather than the post-filter download index (WS2 / F2)
"""
from __future__ import annotations
import base64
import m3u8
import pytest
from Cryptodome.Cipher import AES
from Cryptodome.Util.Padding import pad
from m3u8.model import Key
from unshackle.core.drm.clearkey import SYNC_BYTE, TS_PACKET_SIZE, ClearKey
KEY = bytes.fromhex("00112233445566778899aabbccddeeff")
IV = bytes.fromhex("0f0e0d0c0b0a09080706050403020100")
def encrypt(plaintext: bytes, key: bytes = KEY, iv: bytes = IV) -> bytes:
return AES.new(key, AES.MODE_CBC, iv).encrypt(pad(plaintext, AES.block_size))
def clear_ts(packets: int = 4) -> bytes:
# Plaintext MPEG-TS: sync byte 0x47 at every 188-byte packet boundary. Default 4 packets so
# the fixture satisfies the hardened 4-boundary sniff (and 752 B is 16-aligned, so it must be
# caught by the clear-TS guard, not the len%16 guard).
return b"".join(bytes([SYNC_BYTE]) + b"\x00" * (TS_PACKET_SIZE - 1) for _ in range(packets))
def test_valid_encrypted_segment_decrypts(tmp_path) -> None:
plaintext = b"the quick brown fox jumps over" * 4
seg = tmp_path / "0.ts"
seg.write_bytes(encrypt(plaintext))
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.read_bytes() == plaintext
def test_clear_ts_passes_through(tmp_path) -> None:
data = clear_ts()
seg = tmp_path / "0.ts"
seg.write_bytes(data)
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.read_bytes() == data
def test_non_block_aligned_passes_through(tmp_path) -> None:
data = b"\x01" * 17 # not TS, and not a 16-byte multiple -> not valid CBC ciphertext
assert len(data) % AES.block_size != 0
seg = tmp_path / "0.ts"
seg.write_bytes(data)
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.read_bytes() == data
def test_empty_segment_does_not_crash(tmp_path) -> None:
seg = tmp_path / "0.ts"
seg.write_bytes(b"")
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.exists()
assert seg.read_bytes() == b""
def test_warns_once_per_instance(tmp_path, caplog) -> None:
drm = ClearKey(key=KEY, iv=IV)
for name in ("0.ts", "1.ts"):
seg = tmp_path / name
seg.write_bytes(clear_ts())
with caplog.at_level("WARNING"):
drm.decrypt(seg)
warnings = [r for r in caplog.records if r.levelname == "WARNING"]
assert len(warnings) == 1
def _data_key(raw: bytes, base64_encoded: bool = True) -> Key:
if base64_encoded:
uri = "data:text/plain;base64," + base64.b64encode(raw).decode("ascii")
else:
uri = "data:text/plain," + raw.decode("latin-1")
return Key(method="AES-128", uri=uri, base_uri="", keyformat=None, keyformatversions=None, iv=None)
def test_data_uri_key_decodes_and_validates_length() -> None:
drm = ClearKey.from_m3u_key(_data_key(KEY))
assert drm.key == KEY
def test_data_uri_key_wrong_length_rejected() -> None:
with pytest.raises(ValueError, match="Unexpected Length"):
ClearKey.from_m3u_key(_data_key(b"\x00" * 8))
def test_raw_data_uri_key_materialises_and_validates() -> None:
# A raw (non-base64) data: payload must be taken as literal key bytes, not hex-parsed.
drm = ClearKey.from_m3u_key(_data_key(KEY, base64_encoded=False))
assert drm.key == KEY
def test_raw_data_uri_key_wrong_length_rejected() -> None:
with pytest.raises(ValueError, match="Unexpected Length"):
ClearKey.from_m3u_key(_data_key(b"\x01" * 8, base64_encoded=False))
def test_data_uri_split_tolerates_commas_in_payload() -> None:
# base64 with padding contains no comma, but the split must only break on the first comma
# so a payload that itself contains a comma is not truncated. Encode a 16-byte key whose
# base64 happens to be split-safe, and assert the mediatype comma is the only split point.
raw = bytes(range(16))
b64 = base64.b64encode(raw).decode("ascii")
key = Key(
method="AES-128",
uri=f"data:application/octet-stream;base64,{b64}",
base_uri="",
keyformat=None,
keyformatversions=None,
iv=None,
)
drm = ClearKey.from_m3u_key(key)
assert drm.key == raw
SEQ_PLAYLIST = """#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:6
#EXT-X-MEDIA-SEQUENCE:100
#EXT-X-KEY:METHOD=AES-128,URI="skd://key"
#EXTINF:6.0,
seg0.ts
#EXTINF:6.0,
seg1.ts
#EXTINF:6.0,
seg2.ts
#EXTINF:6.0,
seg3.ts
#EXT-X-ENDLIST
"""
def test_sequence_number_iv_uses_absolute_media_sequence() -> None:
"""IV must be the segment's absolute media sequence number, not the download index.
Mirrors the derivation in ``HLS.download_track``: filtered segments are dropped, the
surviving segments are named by post-filter download index, and the IV comes from the
wanted segment's ``media_sequence`` (EXT-X-MEDIA-SEQUENCE + absolute position).
"""
master = m3u8.loads(SEQ_PLAYLIST)
# media sequence numbers are the offset (100) plus absolute playlist position.
assert [s.media_sequence for s in master.segments] == [100, 101, 102, 103]
# Drop an early segment (index 1), as an OnSegmentFilter would.
unwanted = {master.segments[1]}
wanted_segments = [s for s in master.segments if s not in unwanted]
# download index 1 is the SECOND surviving segment (absolute seq 102), not 101.
download_i = 1
seq = wanted_segments[download_i].media_sequence
assert seq == 102
iv = int(seq).to_bytes(16, "big")
assert iv == (102).to_bytes(16, "big")
# the old download-index derivation would have produced the wrong IV (100 + 1 = 101)
assert iv != (100 + download_i).to_bytes(16, "big")
+75 -1
View File
@@ -3,10 +3,11 @@ services set via the ``sub_format`` override (must not be rejected as an invalid
from __future__ import annotations
import click
import pytest
from unshackle.core.tracks.subtitle import Subtitle
from unshackle.core.utils.click_types import QUALITY_LIST, SLOW_DELAY_RANGE, SubtitleCodecChoice
from unshackle.core.utils.click_types import QUALITY_LIST, SLOW_DELAY_RANGE, SeasonRange, SubtitleCodecChoice
choice = SubtitleCodecChoice(Subtitle.Codec)
@@ -57,3 +58,76 @@ def test_quality_list_accepts_yaml_native_values(value, expected):
)
def test_slow_delay_range_accepts_bool(value, expected):
assert SLOW_DELAY_RANGE.convert(value, None, None) == expected
# --- SeasonRange ------------------------------------------------------------
def parse(*tokens: str) -> list[str]:
return SeasonRange().parse_tokens(*tokens)
def test_single_episode_token_is_unchanged():
assert parse("S01E01") == ["1x1"]
def test_bare_season_spans_the_whole_episode_range():
"""A bare S01 reads the one match object with a different default per side."""
keys = parse("S01")
assert len(keys) == 1000
assert "1x0" in keys and "1x999" in keys
def test_multi_token_range_is_unchanged():
assert sorted(parse("S02E01", "S02E03-S02E05")) == ["2x1", "2x3", "2x4", "2x5"]
def test_exclusions_are_unchanged():
keys = parse("S01-S05", "-S03", "-S02E01")
assert not any(k.startswith("3x") for k in keys)
assert "2x1" not in keys
assert "2x0" in keys and "2x2" in keys
assert len(keys) == 3999 # 5 seasons less all of S03 less 2x1
@pytest.mark.parametrize("tokens", [("S01E01",), ("S01",), ("S01-S05", "-S03", "-S02E01"), ("S02E03-S02E05",)])
def test_part_less_tokens_never_emit_a_part_or_exclusion_key(tokens):
assert all("." not in k and "!" not in k for k in parse(*tokens))
def test_part_token():
assert parse("S01E01.2") == ["1x1.2"]
def test_part_range_within_one_episode():
assert sorted(parse("S01E01.1-S01E01.3")) == ["1x1.1", "1x1.2", "1x1.3"]
def test_part_exclusion_becomes_a_negative_key():
keys = parse("S01", "-S01E01.2")
assert "!1x1.2" in keys
assert "1x1" in keys # base key stays so the other parts still match
def test_base_exclusion_covers_part_keys():
assert parse("S01E01.1-S01E01.3", "S01E02", "-S01E01") == ["1x2"]
def test_exclusion_removes_duplicate_keys():
# "S01" and "S01E01" both emit 1x1; the exclusion must remove every copy
assert "1x1" not in parse("S01", "S01E01", "-S01E01")
@pytest.mark.parametrize(
"token",
[
"S01E01.0", # parts count from 1
"S01E01.1-S01E02.2", # a part range cannot cross episodes
"S01-S01E01.2", # a part on only one side
"S01E01.3-S01E01.1", # reversed part range
"S01E01.200", # above MAX_PART
],
)
def test_bad_part_tokens_fail(token):
with pytest.raises(click.UsageError):
parse(token)
+74
View File
@@ -0,0 +1,74 @@
"""Unit tests for the code fingerprint behind the banner and the /api/health `code_hash` field.
It runs at import time, so it must always return a string and must never raise. It must change when
the framework source changes and stay unchanged when anything else does."""
from __future__ import annotations
from pathlib import Path
import pytest
import unshackle.core
from unshackle.core import __code_hash__, code_files, code_hash
pytestmark = pytest.mark.unit
def test_is_hex_and_stable() -> None:
first = code_hash()
assert len(first) == 40
assert int(first, 16) >= 0
assert first == code_hash()
def test_exported_value_is_the_short_form() -> None:
assert __code_hash__ == code_hash()[:7]
def test_covers_the_framework_dirs_only() -> None:
rels = code_files()
assert "__main__.py" in rels
assert any(rel.startswith("core/") for rel in rels)
assert not any(rel.startswith("services/") for rel in rels)
assert not any("__pycache__" in rel for rel in rels)
assert all(rel.endswith(".py") for rel in rels)
assert rels == sorted(rels) # order must not depend on the filesystem
def test_content_changes_the_hash() -> None:
rels = ["core/__init__.py", "commands/dl.py"]
blobs = {"core/__init__.py": b"a", "commands/dl.py": b"b"}
before = code_hash(files=rels, read=blobs.__getitem__)
blobs["commands/dl.py"] = b"b "
assert code_hash(files=rels, read=blobs.__getitem__) != before
def test_path_is_part_of_the_hash() -> None:
"""A rename with identical bytes must not look like the same code."""
same_bytes = {"core/a.py": b"x", "core/b.py": b"x"}
assert code_hash(files=["core/a.py"], read=same_bytes.__getitem__) != code_hash(
files=["core/b.py"], read=same_bytes.__getitem__
)
def test_file_order_does_not_matter_to_the_caller() -> None:
"""code_files() sorts, so two orderings of the same set must agree once sorted."""
blobs = {"core/a.py": b"x", "core/b.py": b"y"}
assert code_hash(files=sorted(blobs), read=blobs.__getitem__) == code_hash(
files=sorted(reversed(list(blobs))), read=blobs.__getitem__
)
def test_unreadable_source_yields_empty_string() -> None:
def unreadable(rel: str) -> bytes:
raise PermissionError(rel)
assert code_hash(files=["core/__init__.py"], read=unreadable) == ""
def test_unwalkable_source_yields_empty_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A dir os.walk cannot enter must yield "", not a valid-looking hash of fewer files."""
(tmp_path / "__main__.py").write_bytes(b"")
monkeypatch.setattr(unshackle.core, "_PKG", tmp_path)
assert code_hash() == ""
+120
View File
@@ -0,0 +1,120 @@
"""Tests for {dual}/{multi}/{dubbed} filename variables and dual_multi_mode."""
from __future__ import annotations
import types
import pytest
from unshackle.core.config import config
from unshackle.core.titles.movie import Movie
class DummyService:
pass
def make_audio(language: str) -> types.SimpleNamespace:
"""Audio track stub carrying just the attributes the context builder reads."""
return types.SimpleNamespace(
language=language,
format="AAC",
channel_layout=None,
channellayout_original=None,
channel_s=2,
channels=2,
format_additionalfeatures=None,
joc=None,
)
def make_media_info(languages: list[str]) -> types.SimpleNamespace:
return types.SimpleNamespace(video_tracks=[], audio_tracks=[make_audio(lang) for lang in languages])
def make_movie(language) -> Movie:
return Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language=language)
@pytest.fixture
def strict_mode(monkeypatch):
monkeypatch.setattr(config, "dual_multi_mode", "strict")
@pytest.fixture
def count_mode(monkeypatch):
monkeypatch.setattr(config, "dual_multi_mode", "count")
def ctx(language, languages) -> dict:
return make_movie(language)._build_base_template_context(make_media_info(languages))
def test_strict_dual_with_original(strict_mode):
c = ctx("en", ["en", "es"])
assert c["dual"] == "DUAL"
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_two_dubs_no_original(strict_mode):
c = ctx("en", ["es", "fr"])
assert c["dual"] == ""
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_regional_variants_collapse(strict_mode):
c = ctx("en", ["en-US", "en-GB"])
assert c["dual"] == ""
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_multi(strict_mode):
c = ctx("en", ["es", "fr", "de"])
assert c["multi"] == "MULTi"
assert c["dual"] == ""
assert c["dubbed"] == ""
def test_strict_two_unknown_original(strict_mode):
c = ctx(None, ["en", "es"])
assert c["dual"] == ""
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_dubbed(strict_mode):
c = ctx("en", ["es"])
assert c["dubbed"] == "DUBBED"
assert c["dual"] == ""
assert c["multi"] == ""
def test_strict_single_unknown_original_no_dubbed(strict_mode):
c = ctx(None, ["es"])
assert c["dubbed"] == ""
assert c["dual"] == ""
assert c["multi"] == ""
def test_count_dual_ignores_original(count_mode):
c = ctx(None, ["en", "es"])
assert c["dual"] == "DUAL"
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_count_multi(count_mode):
c = ctx(None, ["es", "fr", "de"])
assert c["multi"] == "MULTi"
assert c["dual"] == ""
assert c["dubbed"] == ""
def test_count_never_dubbed(count_mode):
c = ctx("en", ["es"])
assert c["dubbed"] == ""
assert c["dual"] == ""
assert c["multi"] == ""
+199
View File
@@ -0,0 +1,199 @@
"""Tests for multi-part episodes: Episode.part, its naming, sorting and -w matching."""
from __future__ import annotations
import pytest
from unshackle.core.config import config
from unshackle.core.titles.episode import Episode, Series
from unshackle.core.utils.click_types import SeasonRange
class DummyService:
pass
class StubMediaInfo:
"""Minimal MediaInfo stand-in: the template context only reads track lists."""
video_tracks: list = []
audio_tracks: list = []
def make_episode(**overrides) -> Episode:
kwargs = dict(
id_="ep-0001",
service=DummyService,
title="The Show",
season=1,
number=1,
name="The Reckoning",
)
kwargs.update(overrides)
return Episode(**kwargs)
@pytest.fixture
def reset_template():
saved = (config.output_template, config.folder_templates, config.folder_template, config.tag)
config.output_template = {"series": "{title}.{season_episode}.{quality?}.{source}-{tag}"}
config.folder_templates = {}
config.folder_template = ""
config.tag = "GROUP"
yield config
config.output_template, config.folder_templates, config.folder_template, config.tag = saved
# --- validation -------------------------------------------------------------
def test_part_defaults_to_none():
assert make_episode().part is None
def test_digit_string_part_coerces():
assert make_episode(part="2").part == 2
@pytest.mark.parametrize("bad", [0, -1])
def test_non_positive_part_rejected(bad):
with pytest.raises(ValueError):
make_episode(part=bad)
def test_non_int_part_rejected():
with pytest.raises(TypeError):
make_episode(part="x")
def test_bool_part_rejected():
"""bool is an int subclass; True would render as '.True' in keys and filenames."""
with pytest.raises(TypeError):
make_episode(part=True)
def test_old_cache_entry_reads_as_part_less():
"""jsonpickle restores by populating __dict__; an older entry has no 'part' key."""
restored = Episode.__new__(Episode)
restored.__dict__.update(title="The Show", season=1, number=1, name=None, year=None)
assert restored.part is None
# --- display ----------------------------------------------------------------
def test_str_is_unchanged_without_a_part():
without = make_episode()
assert str(without) == "The Show S01E01 The Reckoning"
def test_str_shows_the_selection_syntax():
assert str(make_episode(part=2)) == "The Show S01E01.2 The Reckoning"
def test_str_tells_dated_parts_apart():
# dated content renders the air date instead of SxxExx, so without the suffix both
# parts print the same string while their filenames differ
assert str(make_episode(part=2, air_date="2024-05-06")) == "The Show 2024.05.06.2 The Reckoning"
assert str(make_episode(air_date="2024-05-06")) == "The Show 2024.05.06 The Reckoning"
def test_tree_label_carries_the_part_on_dated_content():
series = Series([make_episode(part=1, air_date="2024-05-06"), make_episode(part=2, air_date="2024-05-06")])
labels = [node.label for node in series.tree(verbose=True).children[0].children]
assert labels == [
"[bold]2024.05.06.1.[/] [bright_black]The Reckoning",
"[bold]2024.05.06.2.[/] [bright_black]The Reckoning",
]
def test_tree_label_carries_the_part():
series = Series([make_episode(part=1), make_episode(part=2)])
labels = [node.label for node in series.tree(verbose=True).children[0].children]
assert labels == ["[bold]1.1.[/] [bright_black]The Reckoning", "[bold]1.2.[/] [bright_black]The Reckoning"]
def test_series_sorts_parts_within_their_episode():
series = Series(
[
make_episode(number=2),
make_episode(number=1, part=2),
make_episode(number=1),
make_episode(number=1, part=1),
]
)
assert [(e.number, e.part) for e in series] == [(1, None), (1, 1), (1, 2), (2, None)]
# --- naming -----------------------------------------------------------------
def test_filename_is_unchanged_without_a_part(reset_template):
assert make_episode().get_filename(StubMediaInfo()) == "The.Show.S01E01.DummyService-GROUP"
def test_dotted_template_folds_the_part_after_the_episode(reset_template):
name = make_episode(part=2).get_filename(StubMediaInfo())
assert name == "The.Show.S01E01.Part.2.DummyService-GROUP"
assert name.endswith("-GROUP") # group tag stays last
def test_spaced_template_uses_spaces(reset_template):
reset_template.output_template = {"series": "{title} {season_episode} {quality?} {source}-{tag}"}
assert make_episode(part=2).get_filename(StubMediaInfo()) == "The Show S01E01 Part 2 DummyService-GROUP"
def test_template_context_leaves_the_season_token_alone(reset_template):
ctx = make_episode(part=2)._build_template_context(StubMediaInfo())
assert ctx["season"] == "S01"
assert ctx["episode"] == "E01.Part.2"
assert ctx["season_episode"] == "S01E01.Part.2"
assert ctx["part"] == 2
def test_part_token_is_empty_without_a_part(reset_template):
assert make_episode()._build_template_context(StubMediaInfo())["part"] == ""
def test_parts_share_one_season_folder(reset_template):
# {season_episode} in a folder template would otherwise carry the part into the path
reset_template.folder_template = "{title}/{season_episode}"
folders = {make_episode(part=p).get_filename(StubMediaInfo(), folder=True) for p in (1, 2)}
assert folders == {"The.Show/S01E01"}
def test_parts_share_one_derived_season_folder(reset_template):
folders = {make_episode(part=p).get_filename(StubMediaInfo(), folder=True) for p in (1, 2)}
assert len(folders) == 1
assert folders == {make_episode().get_filename(StubMediaInfo(), folder=True)}
# --- matching ---------------------------------------------------------------
# the worked key scheme from the design: (token, part-less E1 selected, E1 part 2 selected)
MATCH_TABLE = [
(("S01",), True, True),
(("S01E01",), True, True),
(("S01E01.2",), False, True),
(("S01E01.1-S01E01.3",), False, True),
(("S01", "-S01E01.2"), True, False),
(("S01", "-S01E01"), False, False),
(("S01E02",), False, False),
]
@pytest.mark.parametrize("tokens,wants_plain,wants_part2", MATCH_TABLE)
def test_matches_wanted_table(tokens, wants_plain, wants_part2):
wanted = SeasonRange().parse_tokens(*tokens)
assert make_episode().matches_wanted(wanted) is wants_plain
assert make_episode(part=2).matches_wanted(wanted) is wants_part2
def test_a_part_exclusion_leaves_the_other_parts_alone():
wanted = SeasonRange().parse_tokens("S01", "-S01E01.2")
assert make_episode(part=1).matches_wanted(wanted) is True
assert make_episode(part=3).matches_wanted(wanted) is True
def test_asking_for_a_part_of_an_unsplit_episode_selects_nothing():
assert make_episode().matches_wanted(SeasonRange().parse_tokens("S01E01.2")) is False
+184
View File
@@ -0,0 +1,184 @@
"""Export/import round trips for the two manifest types that are not DASH.
``get_tracks`` splits on manifest type, and the two halves need opposite things. ISM takes the
re-parse path and needs ``track.data["ism"]["segments"]`` restored, because
``ISM.download_track`` reads it. HLS takes the rebuild-from-dicts path and needs nothing
restored, because ``HLS.download_track`` re-fetches the variant from ``track.url``. That makes
the stored per-track URL load-bearing for HLS in a way it is not for ISM.
These drive the real ``dl.write_export`` rather than hand-written export dicts, so a change to
the export format fails here instead of silently invalidating the fixtures.
"""
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from unshackle.commands.dl import dl
from unshackle.core.import_service import ImportService
from unshackle.core.manifests import HLS, ISM
ISM_URL = "https://cdn.example/x.ism/manifest"
ISM_MANIFEST = """<?xml version="1.0"?>
<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000">
<StreamIndex Type="video" Name="video" Language="en"
Url="QualityLevels({Bitrate})/Fragments(video={start_time})">
<QualityLevel Index="0" Bitrate="3000000" FourCC="H264" MaxWidth="1280" MaxHeight="720"
CodecPrivateData="00000001674D401F"/>
<c t="0" d="20000000" r="3"/>
</StreamIndex>
<StreamIndex Type="audio" Name="audio" Language="en"
Url="QualityLevels({Bitrate})/Fragments(audio={start_time})">
<QualityLevel Index="0" Bitrate="128000" AudioTag="255" SamplingRate="48000"
Channels="2" BitsPerSample="16" CodecPrivateData="1190"/>
<c t="0" d="20000000" r="3"/>
</StreamIndex>
</SmoothStreamingMedia>
"""
HLS_URL = "https://cdn.example/master.m3u8"
HLS_MANIFEST = """#EXTM3U
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",LANGUAGE="en",NAME="English",DEFAULT=YES,URI="audio_en.m3u8"
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="sub",LANGUAGE="en",NAME="English",URI="sub_en.m3u8"
#EXT-X-STREAM-INF:BANDWIDTH=3000000,RESOLUTION=1280x720,CODECS="avc1.4d401f,mp4a.40.2",AUDIO="aud",SUBTITLES="sub"
720.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=6000000,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud",SUBTITLES="sub"
1080.m3u8
"""
def export_tracks(tmp_path: Path, tracks, manifest_url: str | None) -> Path:
"""Write a real export sidecar for a parsed ladder, as a finished download would."""
from unshackle.core.titles import Movie
class StubService:
pass
title = Movie(id_="movie-1", service=StubService, name="Example Movie", year=2024, language="en")
for track in [*tracks.videos, *tracks.audio, *tracks.subtitles]:
title.tracks.add(track)
title.tracks.manifest_url = manifest_url
runner = dl.__new__(dl)
runner.service = "EXAMPLE"
export = tmp_path / "export.json"
for track in [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]:
runner.write_export(export, title, track)
return export
def import_tracks(export: Path):
svc = ImportService(SimpleNamespace(parent=None, params={}), "EXAMPLE", "movie-1", str(export))
return svc, svc.get_tracks(next(iter(svc.get_titles())))
@pytest.fixture
def ism_offline(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_MANIFEST, url))
@pytest.mark.parametrize("manifest_url", [ISM_URL, None], ids=["with-manifest-url", "without-manifest-url"])
def test_ism_round_trip_restores_segment_urls(tmp_path: Path, ism_offline: None, manifest_url: str | None) -> None:
"""ISM.download_track reads data["ism"]["segments"], which to_dict cannot carry across an export.
Without a manifest_url the export's own track URLs are the only way back to the manifest,
which is the case the re-parse branch alone never covered.
"""
original = ISM.from_text(ISM_MANIFEST, ISM_URL).to_tracks(language="en")
expected = {t.id: t.data["ism"]["segments"] for t in [*original.videos, *original.audio]}
_, tracks = import_tracks(export_tracks(tmp_path, original, manifest_url))
rebuilt = {t.id: t.data.get("ism", {}).get("segments") for t in [*tracks.videos, *tracks.audio]}
assert rebuilt == expected, "every ISM track must come back with the fragment URLs it was exported with"
ISM_ENCRYPTED = """<?xml version="1.0"?>
<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000">
<Protection><ProtectionHeader SystemID="{9A04F079-9840-4286-AB92-E65BE0885F95}">dGVzdA==</ProtectionHeader></Protection>
<StreamIndex Type="video" Name="video" Language="en" Url="QualityLevels({Bitrate})/Fragments(video={start_time})">
<QualityLevel Index="0" Bitrate="3000000" FourCC="H264" MaxWidth="1280" MaxHeight="720"
CodecPrivateData="00000001674D401F"/>
<QualityLevel Index="1" Bitrate="6000000" FourCC="H264" MaxWidth="1920" MaxHeight="1080"
CodecPrivateData="00000001674D401F"/>
<c t="0" d="20000000" r="3"/>
</StreamIndex>
</SmoothStreamingMedia>
"""
def test_encrypted_ism_rung_gets_the_exported_keys(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""resolve_server_keys must recognise an ISM manifest's protection, not only a DASH one.
ISM.download_track reads only ``track.drm``, so a rung that misses key injection downloads
encrypted and muxes without error. DASH survives the same omission because it re-derives its
DRM from the manifest elements at download time.
"""
monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_ENCRYPTED, url))
parsed = ISM.from_text(ISM_ENCRYPTED, ISM_URL).to_tracks(language="en")
# export the licensed rung only, as a finished download would
licensed = max(parsed.videos, key=lambda t: t.bitrate or 0)
export = export_tracks(tmp_path, parsed, ISM_URL)
doc = json.loads(export.read_text(encoding="utf8"))
doc["titles"]["movie-1"]["tracks"][licensed.id]["keys"] = {"0" * 32: "ab" * 16}
export.write_text(json.dumps(doc), encoding="utf8")
svc, tracks = import_tracks(export)
movie = next(iter(svc.get_titles()))
movie.tracks = tracks
svc.resolve_server_keys(movie)
other = min(tracks.videos, key=lambda t: t.bitrate or 0)
assert svc.track_is_encrypted(other), "an ISM ProtectionHeader means the rung is encrypted"
assert other.drm and other.drm[0].content_keys, "the unlicensed rung never received the exported key"
def test_hls_round_trip_keeps_every_variant_url(tmp_path: Path) -> None:
"""HLS re-fetches each variant from track.url at download time, so the stored URL has to survive."""
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
expected = {t.id: str(t.url) for t in [*original.videos, *original.audio, *original.subtitles]}
_, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL))
rebuilt = {t.id: str(t.url) for t in [*tracks.videos, *tracks.audio, *tracks.subtitles]}
assert rebuilt == expected, "the whole exported ladder must survive, each with its own variant URL"
assert {t.descriptor.name for t in tracks} == {"HLS"}
def test_hls_import_never_refetches_the_master_playlist(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A master playlist can hand out a fresh token per fetch, so import must not depend on it.
This is why HLS is absent from MANIFEST_DATA_KEYS. Re-fetching would add nothing and give a
long-expired export another way to fail.
"""
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
export = export_tracks(tmp_path, original, HLS_URL)
def explode(*args: object, **kwargs: object) -> None:
raise AssertionError("import re-fetched an HLS manifest")
monkeypatch.setattr(HLS, "from_url", explode)
monkeypatch.setattr(HLS, "from_text", explode)
_, tracks = import_tracks(export)
assert len(list(tracks)) == 4
def test_hls_tracks_tolerate_the_empty_manifest_data(tmp_path: Path) -> None:
"""HLS.download_track assigns into track.data["hls"] without creating it first.
Track.data is a defaultdict, so the assignment auto-vivifies rather than raising. Pin that,
because a plain dict here would turn every HLS import into a KeyError mid-download.
"""
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
_, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL))
track = tracks.videos[0]
assert not track.data.get("hls"), "to_dict cannot carry the m3u8 objects, and does not need to"
track.data["hls"]["segment_durations"] = [1, 2, 3] # hls.py, in download_track
assert track.data["hls"]["segment_durations"] == [1, 2, 3]
+226
View File
@@ -0,0 +1,226 @@
"""Tests for the manifest data an import must rebuild before a DASH track can download.
``Track.to_dict`` cannot serialise manifest XML, so a rebuilt DASH track carries no
``track.data["dash"]``. ``DASH.download_track`` reads that mapping unconditionally, so any
DASH track the import hands back without it raises on the first segment. Two exports reach
that state: one with no top-level ``manifest_url``, and one whose ``manifest_url`` covers
only part of the ladder (a separate MPD per codec).
"""
from __future__ import annotations
import json
from functools import partial
from pathlib import Path
from typing import Any, Optional
import click
import pytest
from unshackle.core.import_service import ImportService
from unshackle.core.manifests import DASH
MPD_TEMPLATE = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="{rep_id}" codecs="{codecs}" bandwidth="{bandwidth}" width="{width}" height="{height}">
<BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/>
</Representation>
</AdaptationSet>
</Period>
</MPD>
"""
AVC_URL = "https://example.invalid/avc/dash/m.mpd"
HEVC_URL = "https://example.invalid/hevc/dash/m.mpd"
MANIFESTS = {
AVC_URL: MPD_TEMPLATE.format(rep_id="v-avc", codecs="avc1.640028", bandwidth=5000000, width=1920, height=1080),
HEVC_URL: MPD_TEMPLATE.format(
rep_id="v-hevc", codecs="hvc1.2.4.L153.B0", bandwidth=6000000, width=3840, height=2160
),
}
def exported_video(track_id: str, url: str, codec: str, width: int, height: int) -> dict[str, Any]:
return {
"type": "Video",
"id": track_id,
"url": url,
"language": "en",
"descriptor": "DASH",
"codec": codec,
"bitrate": 5000000 if codec == "AVC" else 6000000,
"width": width,
"height": height,
}
def make_service(tmp_path: Path, manifest_url: Optional[str], tracks: dict[str, Any]) -> ImportService:
entry: dict[str, Any] = {
"meta": {"type": "movie", "id": "movie-1", "name": "Example Movie", "year": 2024, "language": "en"},
"manifest_type": "DASH",
"tracks": tracks,
}
if manifest_url:
entry["manifest_url"] = manifest_url
export = {"version": 2, "service": "EXAMPLE", "titles": {"movie-1": entry}}
path = tmp_path / "export.json"
path.write_text(json.dumps(export), encoding="utf8")
return ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path))
@pytest.fixture(autouse=True)
def local_manifests(monkeypatch: pytest.MonkeyPatch) -> None:
"""Serve each MPD from memory, so a test never depends on the network."""
monkeypatch.setattr(DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(MANIFESTS[url], url))
def test_export_without_manifest_url_still_rebuilds_dash_manifest_data(tmp_path: Path) -> None:
"""No top-level manifest_url means the export's own track URLs are the only MPDs to fetch."""
service = make_service(tmp_path, None, {"v-avc": exported_video("v-avc", AVC_URL, "AVC", 1920, 1080)})
tracks = service.get_tracks(next(iter(service.get_titles())))
assert [t.id for t in tracks.videos] == ["v-avc"]
# DASH.download_track reads all three of these; a missing one is a KeyError mid-download.
assert set(tracks.videos[0].data.get("dash", {})) >= {"manifest", "adaptation_set", "representation"}
def test_exported_hevc_on_its_own_mpd_survives_the_reparse(tmp_path: Path) -> None:
"""A ladder split across per-codec MPDs loses every codec but manifest_url's if only URL tracks merge back."""
service = make_service(
tmp_path,
AVC_URL,
{"v-hevc": exported_video("v-hevc", HEVC_URL, "HEVC", 3840, 2160)},
)
tracks = service.get_tracks(next(iter(service.get_titles())))
codecs = {t.codec.name for t in tracks.videos if t.codec}
assert "HEVC" in codecs, "the exported HEVC track was dropped, so -v h.265 has nothing to select"
hevc = next(t for t in tracks.videos if t.codec and t.codec.name == "HEVC")
assert set(hevc.data.get("dash", {})) >= {"manifest", "adaptation_set", "representation"}
# bound to its own manifest's representation, not just to any populated mapping
assert hevc.data["dash"]["representation"].get("id") == "v-hevc"
SUB_MPD_URL = "https://example.invalid/subs/de.mpd"
MAIN_WITH_SUB = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="v-avc" codecs="avc1.640028" bandwidth="5000000" width="1920" height="1080">
<BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
<AdaptationSet contentType="text" mimeType="text/vtt" lang="en">
<Representation id="s-en" bandwidth="1000">
<BaseURL>en.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
FOREIGN_SUB_MPD = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="text" mimeType="text/vtt" lang="de">
<Representation id="s-de" bandwidth="1000">
<BaseURL>de.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
def test_foreign_subtitle_adds_to_the_manifests_own(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Only a side-loaded (URL) subtitle replaces the re-parsed set.
A subtitle that lives on its own MPD is an addition. Nothing about it says the service
considered the main manifest's copy the worse one, and clearing on it deletes languages the
export never meant to drop.
"""
manifests = {AVC_URL: MAIN_WITH_SUB, SUB_MPD_URL: FOREIGN_SUB_MPD}
monkeypatch.setattr(DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(manifests[url], url))
foreign = DASH.from_text(FOREIGN_SUB_MPD, SUB_MPD_URL).to_tracks(language="de").subtitles[0]
service = make_service(
tmp_path,
AVC_URL,
{
foreign.id: {
"type": "Subtitle",
"id": foreign.id,
"url": SUB_MPD_URL,
"language": "de",
"descriptor": "DASH",
"codec": "WebVTT",
}
},
)
tracks = service.get_tracks(next(iter(service.get_titles())))
assert sorted(str(t.language) for t in tracks.subtitles) == ["de", "en"], (
"the main manifest's own subtitles were wiped by a subtitle from another MPD"
)
TWO_RUNG_MPD = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="lo" codecs="avc1.640028" bandwidth="3000000" width="1920" height="1080">
<BaseURL>lo.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation>
<Representation id="hi" codecs="avc1.640028" bandwidth="{top}" width="1920" height="1080">
<BaseURL>hi.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
@pytest.mark.parametrize(
"exported_url",
[AVC_URL, AVC_URL + "?token=EXPIRED"],
ids=["same-url", "signed-url"],
)
def test_reencoded_top_rung_does_not_readd_the_exported_one(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, exported_url: str
) -> None:
"""Track ids hash the bitrate, so re-encoding a rung makes the exported id look like a foreign MPD's.
Merging it back on that basis duplicates the rung, and the attribute fallback then binds the
copy to the first representation of the same codec and resolution: the 3 Mbps one. The result
downloads `lo` under a 6 Mbps name.
The signed variant is the same manifest reached by two strings: a service may fetch a signed
URL while registering the bare one as Title.tracks.manifest_url, so comparing them whole
reopens the duplicate.
"""
exported = DASH.from_text(TWO_RUNG_MPD.format(top=6000000), exported_url).to_tracks(language="en")
top = next(t for t in exported.videos if t.bitrate == 6000000)
monkeypatch.setattr(
DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(TWO_RUNG_MPD.format(top=6500000), url)
)
service = make_service(tmp_path, AVC_URL, {top.id: exported_video(top.id, exported_url, "AVC", 1920, 1080)})
tracks = service.get_tracks(next(iter(service.get_titles())))
assert sorted(t.bitrate for t in tracks.videos) == [3000000, 6500000], (
"the re-parsed ladder is the only source for this manifest's rungs"
)
bound = {t.data["dash"]["representation"].get("id") for t in tracks.videos}
assert bound == {"lo", "hi"}, "two tracks were bound to the same representation"
def test_clearkey_license_hook_is_bindable(tmp_path: Path) -> None:
"""dl.py binds service.get_clearkey_license for every track, DRM or not, before downloading.
Bind it the way dl.py does rather than asking hasattr: a non-callable attribute satisfies
hasattr and still fails at the partial with a TypeError.
"""
service = make_service(tmp_path, AVC_URL, {})
bound = partial(service.get_clearkey_license, title=None, track=None)
assert callable(bound)
with pytest.raises(RuntimeError, match="keys come from the export"):
bound(challenge=b"")
+138
View File
@@ -0,0 +1,138 @@
"""Tests for ``ImportService.get_tracks``, which rebuilds tracks from an export.
Regression: an export whose manifest labels its own streams imports even when the service
never set ``Title.language``. When nothing supplies a language the import fails naming the
service rather than blaming the manifest URL. A direct-URL subtitle anywhere in the export
replaces the manifest's whole subtitle set, since the service side-loaded it precisely
because the manifest's copy was the worse one.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Optional
from unittest.mock import patch
import click
import pytest
import requests
from unshackle.core.import_service import ImportService
from unshackle.core.manifests import DASH
MPD_TEMPLATE = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4"{lang}>
<Representation id="{video_id}" codecs="avc1.640028" bandwidth="5000000" width="1920" height="1080">
<BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/>
</Representation>
</AdaptationSet>
<AdaptationSet contentType="text" mimeType="text/vtt"{lang}>
<Representation id="{sub_id}" bandwidth="1000"><BaseURL>s.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
# labelled by @lang; unlabelled but with Unified Streaming ids DASH derives a language from;
# and labelled by neither, which is the only shape that needs the title's language
MPD_WITH_LANG = MPD_TEMPLATE.format(lang=' lang="ja"', video_id="v1", sub_id="s1")
MPD_ID_LANG = MPD_TEMPLATE.format(lang="", video_id="video_eng=5000000", sub_id="text_eng=1000")
MPD_NO_LANG = MPD_TEMPLATE.format(lang="", video_id="v1", sub_id="s1")
URL_SUBTITLE = {
"type": "Subtitle",
"id": "sideloaded-en",
"url": "https://example.invalid/en.srt",
"language": "en",
"descriptor": "URL",
"codec": "SubRip",
}
def make_service(tmp_path: Path, language: Optional[str], tracks: Optional[dict[str, Any]] = None) -> ImportService:
export = {
"version": 2,
"service": "EXAMPLE",
"titles": {
"movie-1": {
"meta": {"type": "movie", "id": "movie-1", "name": "Example Movie", "year": 2024, "language": language},
"manifest_url": "https://example.invalid/m.mpd",
"manifest_type": "DASH",
"tracks": tracks or {},
}
},
}
path = tmp_path / "export.json"
path.write_text(json.dumps(export), encoding="utf8")
return ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path))
def get_tracks(service: ImportService, mpd: str):
with patch.object(DASH, "from_url", side_effect=lambda url, session=None, **kw: DASH.from_text(mpd, url)):
return service.get_tracks(next(iter(service.get_titles())))
@pytest.mark.parametrize("mpd", [MPD_WITH_LANG, MPD_ID_LANG], ids=["lang-attribute", "language-in-rep-id"])
def test_manifest_supplies_language_without_title_language(tmp_path: Path, mpd: str) -> None:
"""A manifest that labels its own streams imports fine with no language in the export."""
tracks = get_tracks(make_service(tmp_path, language=None), mpd)
assert [str(t.language) for t in tracks.videos] == ["ja" if mpd is MPD_WITH_LANG else "en"]
@pytest.mark.parametrize("language", [None, "und"], ids=["missing", "und"])
def test_unlabelled_manifest_without_usable_language_blames_the_service(
tmp_path: Path, language: Optional[str]
) -> None:
"""`und` is truthy and a valid tag, so it must be rejected as firmly as no language at all."""
with pytest.raises(click.ClickException) as error:
get_tracks(make_service(tmp_path, language=language), MPD_NO_LANG)
assert "Title.language" in error.value.message
assert "may have expired" not in error.value.message
def test_exported_url_subtitles_replace_the_manifests(tmp_path: Path) -> None:
"""Services drop the manifest's subtitles and side-load their own; importing both duplicates them."""
tracks = get_tracks(make_service(tmp_path, "ja", {"sideloaded-en": URL_SUBTITLE}), MPD_WITH_LANG)
assert [(t.id, t.descriptor.name) for t in tracks.subtitles] == [("sideloaded-en", "URL")]
assert len(tracks.videos) == 1
def test_direct_url_subtitles_win_over_every_manifest_one(tmp_path: Path) -> None:
"""One side-loaded subtitle condemns the whole re-parsed set, including other languages."""
manifest_sub = {
"type": "Subtitle",
"id": "s1",
"url": "https://example.invalid/m.mpd",
"language": "ja",
"descriptor": "DASH",
"codec": "WebVTT",
}
tracks = get_tracks(
make_service(tmp_path, "ja", {"s1": manifest_sub, "sideloaded-en": URL_SUBTITLE}), MPD_WITH_LANG
)
assert [(t.id, t.descriptor.name) for t in tracks.subtitles] == [("sideloaded-en", "URL")]
def test_unreadable_exported_track_is_skipped(tmp_path: Path) -> None:
"""A track dict this build cannot rebuild must not take the whole import down with it."""
unknown = {**URL_SUBTITLE, "id": "future", "codec": "CODEC_FROM_A_LATER_BUILD"}
tracks = get_tracks(make_service(tmp_path, "ja", {"sideloaded-en": URL_SUBTITLE, "future": unknown}), MPD_WITH_LANG)
assert [t.id for t in tracks.subtitles] == ["sideloaded-en"]
@pytest.mark.parametrize(
"failure",
[requests.ConnectionError("dns"), TypeError("Expected 'MPD' document, but received a 'html' document instead.")],
ids=["connection-error", "expired-token-html-page"],
)
def test_fetch_failure_still_reports_a_possible_expiry(tmp_path: Path, failure: Exception) -> None:
"""An expired signed URL can 200 with an error page, so document failures are expiries too."""
service = make_service(tmp_path, "ja")
title = next(iter(service.get_titles()))
with patch.object(DASH, "from_url", side_effect=failure):
with pytest.raises(click.ClickException) as error:
service.get_tracks(title)
assert "may have expired" in error.value.message
+314
View File
@@ -24,6 +24,7 @@ from unshackle.core.manifests.ism_init import (
parse_codec_private_data_colour,
parse_codec_private_data_vui,
parse_hevc_sps_format,
piff_senc_to_cenc,
read_per_sample_iv_size,
read_track_id,
remove_emulation_prevention,
@@ -111,6 +112,82 @@ def test_hevc_init_structure():
assert b"encv" not in init and b"sinf" not in init
# Boxes whose payload is purely child boxes, so their size must span the children
# exactly. stsd is excluded: its payload starts with a count, and sample entries
# carry a fixed prefix before any child box.
CONTAINER_BOXES = {"moov", "trak", "mdia", "minf", "stbl", "dinf", "mvex", "sinf", "schi"}
def walk_box(data: bytes, start: int, end: int, path: str = "") -> None:
"""Assert every container's declared size spans its children exactly."""
offset = start
while offset < end:
size = struct.unpack_from(">I", data, offset)[0]
name = data[offset + 4 : offset + 8].decode("latin1")
where = f"{path}/{name}"
assert size >= 8, f"{where}: size {size}"
assert offset + size <= end, f"{where}: overruns its parent"
if name in CONTAINER_BOXES:
walk_box(data, offset + 8, offset + size, where)
offset += size
assert offset == end, f"{path or '/'}: children do not span the parent exactly"
def find_stsd(init: bytes) -> int:
"""Offset of the stsd box, found by walking the box tree."""
offset, end = 0, len(init)
for name in ("moov", "trak", "mdia", "minf", "stbl", "stsd"):
cursor = offset
while cursor < end:
size = struct.unpack_from(">I", init, cursor)[0]
if init[cursor + 4 : cursor + 8].decode("latin1") == name:
offset, end = cursor + 8, cursor + size
break
cursor += size
else:
raise AssertionError(f"{name} not found")
return offset - 8
# The two-entry stsd is unconditional, so every stream type and codec the builder
# emits needs covering, encrypted and clear alike.
STSD_CASES = [
("video", "HVC1", VIDEO_HEVC_CPD, KID, b"encv", {"width": 3840, "height": 2160}),
("video", "HVC1", VIDEO_HEVC_CPD, None, b"hvc1", {"width": 3840, "height": 2160}),
("video", "H264", VIDEO_AVC_CPD, None, b"avc1", {"width": 1280, "height": 720}),
("audio", "AACL", AAC_LC_CPD, None, b"mp4a", {"channels": 2}),
("audio", "EC-3", EC3_CPD, KID, b"enca", {"channels": 6}),
("text", "TTML", "", None, b"stpp", {}),
]
@pytest.mark.parametrize("stream_type,fourcc,cpd,kid,expected_4cc,extra", STSD_CASES)
def test_stsd_carries_two_entries_for_sample_description_index_2(stream_type, fourcc, cpd, kid, expected_4cc, extra):
# A single entry leaves tfhd sample_description_index=2 dangling, and mp4decrypt
# skips those fragments without a word: exit 0, empty stderr, still encrypted.
init = build_init_segment(
stream_type=stream_type,
fourcc=fourcc,
codec_private_data=cpd,
timescale=10000000,
kid=kid,
**extra,
)
offset = find_stsd(init)
assert struct.unpack_from(">I", init, offset + 12)[0] == 2, "index 2 would dangle"
entries, cursor = [], offset + 16
for _ in range(2):
size = struct.unpack_from(">I", init, cursor)[0]
entries.append(init[cursor : cursor + size])
cursor += size
assert [e[4:8] for e in entries] == [expected_4cc, expected_4cc]
# stsd must span its header, count and both entries exactly.
assert struct.unpack_from(">I", init, offset)[0] == 16 + sum(len(e) for e in entries)
# The same has to hold for every container box wrapping it.
walk_box(init, 0, len(init))
def test_avc_init_structure():
init = build_init_segment(
stream_type="video",
@@ -502,3 +579,240 @@ def test_hvcc_profile_tier_level_is_nonzero():
level_idc = payload[12]
assert profile_idc != 0
assert level_idc != 0
def _piff_fragment(flags: int = 0x0, override: bool = False) -> bytes:
"""Minimal moof+mdat fragment carrying sample encryption in the PIFF uuid box."""
ivs = b"".join(bytes([i]) * 8 for i in range(1, 3))
senc_payload = struct.pack(">I", 2) + ivs
if override:
flags |= 0x1
senc_payload = b"\x00\x00\x01" + bytes([8]) + KID + senc_payload # AlgorithmID + IV_size + KID
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + flags.to_bytes(3, "big") + senc_payload)
tfhd = full_box(b"tfhd", 0, 0x20000, struct.pack(">I", 1))
trun = full_box(b"trun", 0, 0x1, struct.pack(">II", 2, 0)) # data_offset placeholder
traf = box(b"traf", tfhd + trun + uuid_box)
moof = box(b"moof", full_box(b"mfhd", 0, 0, struct.pack(">I", 1)) + traf)
# patch trun data_offset to the real moof-relative start of the mdat payload
moof = moof.replace(struct.pack(">II", 2, 0), struct.pack(">II", 2, len(moof) + 8), 1)
return moof + box(b"mdat", b"\xab" * 64)
def test_piff_senc_rewritten_to_cenc_senc():
frag = _piff_fragment()
out = piff_senc_to_cenc(frag)
# trun's data_offset is moof-relative; shrinking the box misaims decryption
assert len(out) == len(frag)
assert PIFF_SENC_UUID not in out
assert b"senc" in out and b"free" in out
assert struct.pack(">I", 2) + b"".join(bytes([i]) * 8 for i in range(1, 3)) in out
# moof size unchanged => the recorded data_offset still resolves
assert out[:8] == frag[:8]
assert out[len(out) - 72 :] == frag[len(frag) - 72 :] # mdat untouched
def test_piff_senc_override_header_stripped():
frag = _piff_fragment(override=True)
out = piff_senc_to_cenc(frag)
assert len(out) == len(frag)
assert PIFF_SENC_UUID not in out
senc = out.index(b"senc")
assert out[senc + 4] == 0 # version
assert int.from_bytes(out[senc + 5 : senc + 8], "big") & 0x1 == 0 # override flag cleared
assert out[senc + 8 : senc + 12] == struct.pack(">I", 2) # sample_count directly follows
def test_piff_subsample_flag_survives_override_strip():
# losing 0x2 is silent: no subsample map, so clear NAL headers get encrypted too
frag = _piff_fragment(flags=0x2, override=True)
out = piff_senc_to_cenc(frag)
senc = out.index(b"senc")
flags = int.from_bytes(out[senc + 5 : senc + 8], "big")
assert flags & 0x2, "subsample-encryption flag must survive"
assert not flags & 0x1, "override flag must be cleared"
def test_piff_rewrite_leaves_non_piff_fragments_alone():
plain = box(b"moof", box(b"traf", full_box(b"tfhd", 0, 0, struct.pack(">I", 1)))) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(plain) is plain
assert piff_senc_to_cenc(b"") == b""
def test_piff_rewrite_skips_traf_that_already_has_senc():
# a second senc would leave the decrypter to choose between them
ivs = b"".join(bytes([i]) * 8 for i in range(1, 3))
senc = full_box(b"senc", 0, 0, struct.pack(">I", 2) + ivs)
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0).to_bytes(3, "big") + struct.pack(">I", 2) + ivs)
frag = box(b"moof", box(b"traf", senc + uuid_box)) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(frag) is frag
def test_piff_rewrite_covers_every_moof_and_traf():
frag = _piff_fragment() + _piff_fragment()
out = piff_senc_to_cenc(frag)
assert len(out) == len(frag)
assert PIFF_SENC_UUID not in out
assert out.count(b"senc") == 2
def test_piff_rewrite_refuses_non_ctr_algorithm_id():
# AlgorithmID 2 is AES-CBC but the sample entry says 'cenc'; rewriting yields garbage
ivs = b"".join(bytes([i]) * 8 for i in range(1, 3))
payload = (2).to_bytes(3, "big") + bytes([8]) + KID + struct.pack(">I", 2) + ivs
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload)
frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(frag) is frag
def traf_children_tile_exactly(frag: bytes) -> bool:
"""Every traf child's declared size must chain exactly to the traf end."""
moof_size = struct.unpack(">I", frag[:4])[0]
traf_start = 8
while frag[traf_start + 4 : traf_start + 8] != b"traf":
traf_start += struct.unpack(">I", frag[traf_start : traf_start + 4])[0]
traf_end = traf_start + struct.unpack(">I", frag[traf_start : traf_start + 4])[0]
assert traf_end <= moof_size
offset = traf_start + 8
while offset < traf_end:
size = struct.unpack(">I", frag[offset : offset + 4])[0]
if size < 8 or offset + size > traf_end:
return False
offset += size
return offset == traf_end
def test_piff_rewrite_keeps_traf_children_tiling_exactly():
# a mis-sized senc still "decrypts", just off the wrong bytes, so assert the sizes too
assert traf_children_tile_exactly(_piff_fragment())
for frag in (_piff_fragment(), _piff_fragment(flags=0x2), _piff_fragment(flags=0x2, override=True)):
assert traf_children_tile_exactly(piff_senc_to_cenc(frag)), "rewritten traf no longer tiles"
def test_piff_rewrite_refuses_override_iv_size_mismatch():
# tenc is the only IV width a decrypter consults; a disagreeing override misparses silently
ivs = b"".join(bytes([i]) * 16 for i in range(1, 3))
payload = (1).to_bytes(3, "big") + bytes([16]) + KID + struct.pack(">I", 2) + ivs
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload)
frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(frag, iv_size=8) is frag, "8-byte tenc must reject a 16-byte override"
assert piff_senc_to_cenc(frag, iv_size=16) is not frag, "matching width must still rewrite"
def test_piff_rewrite_never_grows_output_on_malformed_boxes():
# A declared size running past the buffer would extend the output over the mdat.
good = _piff_fragment()
truncations = [good[:n] for n in range(8, len(good), 7)]
oversized = bytearray(good)
struct.pack_into(">I", oversized, 0, len(good) + 4096) # moof claims more than exists
for frag in [*truncations, bytes(oversized), b"", b"\x00" * 12]:
out = piff_senc_to_cenc(frag) # must not raise
assert len(out) == len(frag), f"output grew: {len(frag)} -> {len(out)}"
def _text_fragment(seq: int, begin: str, end: str, text: str, sdi: int) -> bytes:
"""A single-sample fTTML moof+mdat, mirroring what a Smooth text stream serves."""
payload = (
"<?xml version='1.0' encoding='utf-8'?>"
"<tt xmlns='http://www.w3.org/ns/ttml' xml:lang='en'><body><div>"
f"<p begin='{begin}' end='{end}'>{text}</p>"
"</div></body></tt>"
).encode()
mfhd = full_box(b"mfhd", 0, 0, struct.pack(">I", seq))
tfhd = full_box(b"tfhd", 0, 0x020002, struct.pack(">I", 1) + struct.pack(">I", sdi))
tfdt = full_box(b"tfdt", 1, 0, struct.pack(">Q", 0))
trun = full_box(b"trun", 0, 0x000201, struct.pack(">I", 1) + struct.pack(">i", 0) + struct.pack(">I", len(payload)))
return box(b"moof", mfhd + box(b"traf", tfhd + tfdt + trun)) + box(b"mdat", payload)
def test_unity_matrix_is_nine_int32s():
# extras shift every later field (mvhd next_track_ID, tkhd width/height) and desync pymp4
from unshackle.core.manifests.ism_init import UNITY_MATRIX
assert len(UNITY_MATRIX) == 36
@pytest.mark.parametrize("stream_type", ["video", "audio", "text"])
def test_mvhd_tkhd_sizes_match_iso_layout(stream_type):
# a declared size over the spec payload makes parsers under-read and stop at the moov
kwargs = {
"video": dict(fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080),
"audio": dict(fourcc="AACL", codec_private_data=AAC_LC_CPD),
"text": dict(fourcc="TTML", codec_private_data=""),
}[stream_type]
init = build_init_segment(stream_type=stream_type, duration=6_000_000_000, language="eng", **kwargs)
sizes = {}
for name in (b"mvhd", b"tkhd"):
at = init.index(name) - 4
sizes[name] = struct.unpack(">I", init[at : at + 4])[0]
# version-1 mvhd: 8 header + 4 version/flags + 28 time fields + 80 trailer
assert sizes[b"mvhd"] == 120
# version-1 tkhd: 8 header + 4 version/flags + 32 time fields + 60 trailer
assert sizes[b"tkhd"] == 104
def test_tkhd_carries_video_dimensions():
# width/height sit after the matrix, so a mis-sized matrix silently zeroes them
init = build_init_segment(
stream_type="video", fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080
)
at = init.index(b"tkhd") - 4
tkhd = init[at : at + struct.unpack(">I", init[at : at + 4])[0]]
width, height = struct.unpack(">II", tkhd[-8:])
assert (width >> 16, height >> 16) == (1920, 1080)
def test_pymp4_consumes_whole_init_and_reaches_fragments():
# pymp4 must reach the mdat here, or Subtitle.parse finds nothing to yield cues from
from io import BytesIO
from pymp4.parser import MP4
init = build_init_segment(
stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng"
)
data = init + _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1)
types = [b.type for b in MP4.parse_stream(BytesIO(data))]
assert types == [b"ftyp", b"moov", b"moof", b"mdat"]
def test_fttml_subtitle_parse_yields_cues():
from unshackle.core.tracks.subtitle import Subtitle
init = build_init_segment(
stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng"
)
data = (
init
+ _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1)
+ _text_fragment(2, "00:00:03.000", "00:00:04.000", "second", sdi=2)
)
caption_set = Subtitle.parse(data, Subtitle.Codec.fTTML)
cues = [c for lang in caption_set.get_languages() for c in caption_set.get_captions(lang)]
assert len(cues) == 2
assert [c.get_text() for c in cues] == ["first", "second"]
@pytest.mark.parametrize(
"fourcc,expected",
[("HEV1", b"hev1"), ("hev1", b"hev1"), ("HVC1", b"hvc1"), ("hvc1", b"hvc1"), ("HEVC", b"hvc1"), ("H265", b"hvc1")],
)
def test_hevc_sample_entry_honours_manifest_fourcc(fourcc, expected):
# hev1 permits in-band parameter sets where hvc1 requires them in the sample
# entry. Real Smooth manifests ship FourCC="hev1", so relabelling it hvc1
# misdescribes the stream.
init = build_init_segment(
stream_type="video", fourcc=fourcc, codec_private_data=VIDEO_HEVC_CPD, width=3840, height=2160
)
offset = find_stsd(init)
assert init[offset + 20 : offset + 24] == expected
def test_encrypted_hev1_frma_carries_the_same_fourcc():
# frma names the original format the sinf wraps; if it drifts from the sample
# entry, a decrypter restores the wrong sample entry type.
init = build_init_segment(
stream_type="video", fourcc="HEV1", codec_private_data=VIDEO_HEVC_CPD, kid=KID, width=3840, height=2160
)
assert b"encv" in init
assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"hev1"
+223
View File
@@ -0,0 +1,223 @@
"""Tests for the tag_rules engine behind conditional release-group tags."""
from __future__ import annotations
from typing import Any
import pytest
from unshackle.core.utils.tag_rules import evaluate_tag_rules
CONTEXT: dict[str, Any] = {
"tag": "NOGRP",
"title_type": "movie",
"quality": "2160p",
"resolution": "2160",
"hdr": "DV",
"lang_tag": "SUBBED",
"edition": "",
}
def test_first_match_wins() -> None:
rules = [
{"when": {"quality": "2160p"}, "tag": "FIRST"},
{"when": {"title_type": "movie"}, "tag": "SECOND"},
]
assert evaluate_tag_rules(rules, CONTEXT) == "FIRST"
def test_list_matches_any_entry() -> None:
rules = [{"when": {"hdr": ["HDR10P", "DV"]}, "tag": "UHDGROUP"}]
assert evaluate_tag_rules(rules, CONTEXT) == "UHDGROUP"
def test_list_with_no_matching_entry() -> None:
rules = [{"when": {"hdr": ["HDR10P", "HLG"]}, "tag": "UHDGROUP"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_case_insensitive() -> None:
rules = [{"when": {"lang_tag": "subbed", "quality": "2160P"}, "tag": "SUBMOVIES"}]
assert evaluate_tag_rules(rules, CONTEXT) == "SUBMOVIES"
def test_conditions_are_anded() -> None:
rules = [{"when": {"title_type": "movie", "quality": "1080p"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_unknown_key_skips_rule_but_later_rules_match(caplog: pytest.LogCaptureFixture) -> None:
rules = [
{"when": {"bogus": "value"}, "tag": "SKIPPED"},
{"when": {"title_type": "movie"}, "tag": "MATCHED"},
]
assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED"
assert "bogus" in caplog.text
@pytest.mark.parametrize(
"rule",
[
{"when": {"title_type": "movie"}},
{"tag": "NOCONDITIONS"},
{"when": {}, "tag": "EMPTY"},
],
)
def test_malformed_rule_skipped(rule: dict[str, Any]) -> None:
assert evaluate_tag_rules([rule], CONTEXT) is None
def test_no_rules_and_no_match() -> None:
assert evaluate_tag_rules([], CONTEXT) is None
assert evaluate_tag_rules([{"when": {"quality": "720p"}, "tag": "SD"}], CONTEXT) is None
def test_empty_context_value_matches_empty_condition() -> None:
rules = [{"when": {"edition": ""}, "tag": "PLAIN"}]
assert evaluate_tag_rules(rules, CONTEXT) == "PLAIN"
@pytest.mark.parametrize(
("condition", "expected"),
[
(">=2160", True),
(">=4320", False),
("<=2160", True),
("<1080", False),
(">1080", True),
("==2160", True),
("=2160", True),
("!=1080", True),
("!=2160", False),
(">= 2160", True),
],
)
def test_numeric_operators_extract_first_number(condition: str, expected: bool) -> None:
"""The actual value is "2160p", so the suffix must not defeat the comparison."""
rules = [{"when": {"quality": condition}, "tag": "MATCHED"}]
assert (evaluate_tag_rules(rules, CONTEXT) == "MATCHED") is expected
def test_string_inequality() -> None:
assert evaluate_tag_rules([{"when": {"hdr": "!=DV"}, "tag": "NODV"}], CONTEXT) is None
assert evaluate_tag_rules([{"when": {"hdr": "!=HDR10P"}, "tag": "NOHDR10P"}], CONTEXT) == "NOHDR10P"
def test_string_equality_operator_is_case_insensitive() -> None:
assert evaluate_tag_rules([{"when": {"lang_tag": "==subbed"}, "tag": "SUBGRP"}], CONTEXT) == "SUBGRP"
def test_plain_equality_unchanged() -> None:
assert evaluate_tag_rules([{"when": {"quality": "2160p"}, "tag": "UHD"}], CONTEXT) == "UHD"
assert evaluate_tag_rules([{"when": {"quality": "2160"}, "tag": "UHD"}], CONTEXT) is None
def test_numeric_operator_against_value_without_a_number() -> None:
rules = [{"when": {"hdr": ">=1080"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_ordering_operator_with_non_numeric_operand(caplog: pytest.LogCaptureFixture) -> None:
rules = [{"when": {"hdr": ">DV"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
assert "numeric operand" in caplog.text
def test_operators_inside_lists() -> None:
rules = [{"when": {"quality": ["<720", ">=2160"]}, "tag": "MATCHED"}]
assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED"
rules = [{"when": {"quality": ["<720", "1080p"]}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
@pytest.mark.parametrize("rules", ["UHDGROUP", {"when": {"quality": "2160p"}, "tag": "UHD"}, None, 42])
def test_rules_that_are_not_a_list(rules: Any) -> None:
"""A mapping or string here iterates keys/characters, so it must be rejected outright."""
assert evaluate_tag_rules(rules, CONTEXT) is None
@pytest.mark.parametrize("rule", [None, "UHDGROUP", 42, ["when", "tag"]])
def test_rule_entry_that_is_not_a_mapping_skipped(rule: Any) -> None:
assert evaluate_tag_rules([rule], CONTEXT) is None
assert evaluate_tag_rules([rule, {"when": {"title_type": "movie"}, "tag": "MATCHED"}], CONTEXT) == "MATCHED"
@pytest.mark.parametrize("when", [["title_type", "movie"], "title_type", 42])
def test_when_that_is_not_a_mapping_skipped(when: Any) -> None:
assert evaluate_tag_rules([{"when": when, "tag": "NOPE"}], CONTEXT) is None
@pytest.mark.parametrize(
"rule",
[
{"when": {"atmos": True}, "tag": "ATMOS"},
{"when": {"atmos": [True, "Atmos"]}, "tag": "ATMOS"},
{"when": {"title_type": "movie"}, "tag": True},
],
)
def test_yaml_boolean_skipped(rule: dict[str, Any], caplog: pytest.LogCaptureFixture) -> None:
"""`atmos: true` / `tag: yes` are YAML coercions the user meant as text."""
assert evaluate_tag_rules([rule], CONTEXT) is None
assert "boolean" in caplog.text
def test_numbers_stay_valid() -> None:
assert evaluate_tag_rules([{"when": {"resolution": 2160}, "tag": "UHD"}], CONTEXT) == "UHD"
assert evaluate_tag_rules([{"when": {"resolution": 1080}, "tag": "NOPE"}], CONTEXT) is None
def test_numeric_tag_becomes_text() -> None:
assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": 123}], CONTEXT) == "123"
@pytest.mark.parametrize("tag", [["A", "B"], {"x": 1}, ("A",), {"A"}])
def test_non_scalar_tag_skipped(tag: Any, caplog: pytest.LogCaptureFixture) -> None:
"""A list or dict here used to reach the filename as its repr."""
assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": tag}], CONTEXT) is None
assert "must be text or a number" in caplog.text
@pytest.mark.parametrize("tag", ["", 0])
def test_empty_tag_skipped(tag: Any) -> None:
assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": tag}], CONTEXT) is None
def test_skipped_rule_falls_through_to_a_later_one() -> None:
rules = [
{"when": {"atmos": True}, "tag": "SKIPPED"},
{"when": {"title_type": "movie"}, "tag": "MATCHED"},
]
assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED"
def test_several_negations_in_a_list_always_match() -> None:
"""Any-entry matching makes a list of negations useless, which the docs warn about."""
rules = [{"when": {"hdr": ["!=DV", "!=HLG"]}, "tag": "ALWAYS"}]
assert evaluate_tag_rules(rules, CONTEXT) == "ALWAYS"
def test_leading_space_before_an_operator_is_literal() -> None:
"""Current behaviour: the operator is only found at the very start of the value."""
rules = [{"when": {"quality": " >=2160"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_inequality_holds_for_a_value_with_no_number() -> None:
""" "DV" really is not equal to 2160, so != holds while the others fail."""
assert evaluate_tag_rules([{"when": {"hdr": "!=2160"}, "tag": "MATCHED"}], CONTEXT) == "MATCHED"
assert evaluate_tag_rules([{"when": {"hdr": ">=2160"}, "tag": "NOPE"}], CONTEXT) is None
assert evaluate_tag_rules([{"when": {"hdr": "==2160"}, "tag": "NOPE"}], CONTEXT) is None
@pytest.mark.parametrize("condition", ["!=", ">=", "<", "=", "=="])
def test_bare_operator_fails(condition: str, caplog: pytest.LogCaptureFixture) -> None:
assert evaluate_tag_rules([{"when": {"hdr": condition}, "tag": "NOPE"}], CONTEXT) is None
assert "no value to compare against" in caplog.text
def test_composite_hdr_does_not_match_plain_dv() -> None:
"""DV titles carry a base-layer suffix, which the docs tell users to list explicitly."""
context = {**CONTEXT, "hdr": "DV.HDR10P"}
assert evaluate_tag_rules([{"when": {"hdr": "DV"}, "tag": "NOPE"}], context) is None
assert evaluate_tag_rules([{"when": {"hdr": ["DV", "DV.HDR10P"]}, "tag": "DOLBY"}], context) == "DOLBY"
+131
View File
@@ -0,0 +1,131 @@
"""Tests for tag_rules reaching the filename context through Title."""
from __future__ import annotations
import inspect
import types
from typing import Any, Iterator
import pytest
from langcodes import Language
from unshackle.core.config import config
from unshackle.core.titles.movie import Movie
class DummyService:
pass
class TrackStub:
"""Stand-in for Tracks: truthy and iterable, with the two lists lang_tag reads."""
def __init__(self, audio: list[Any], subtitles: list[Any]) -> None:
self.audio = audio
self.subtitles = subtitles
def __iter__(self) -> Iterator[Any]:
return iter([])
def __bool__(self) -> bool:
return True
def make_audio(language: str) -> types.SimpleNamespace:
return types.SimpleNamespace(
language=language,
format="AAC",
channel_layout=None,
channellayout_original=None,
channel_s=2,
channels=2,
format_additionalfeatures=None,
joc=None,
)
def make_media_info(languages: list[str]) -> types.SimpleNamespace:
return types.SimpleNamespace(video_tracks=[], audio_tracks=[make_audio(lang) for lang in languages])
def make_movie() -> Movie:
return Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language="en")
@pytest.fixture(autouse=True)
def clean_config(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "tag", "NOGRP")
monkeypatch.setattr(config, "tag_rules", [])
monkeypatch.setattr(config, "language_tags", {})
def context_for(rules: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch) -> dict:
monkeypatch.setattr(config, "tag_rules", rules)
return make_movie()._build_base_template_context(make_media_info(["en"]))
def test_matching_rule_replaces_the_tag(monkeypatch: pytest.MonkeyPatch) -> None:
context = context_for([{"when": {"title_type": "movie"}, "tag": "MOVIEGRP"}], monkeypatch)
assert context["tag"] == "MOVIEGRP"
def test_no_matching_rule_keeps_the_config_tag(monkeypatch: pytest.MonkeyPatch) -> None:
context = context_for([{"when": {"title_type": "series"}, "tag": "TVGRP"}], monkeypatch)
assert context["tag"] == "NOGRP"
def test_no_rules_keeps_the_config_tag(monkeypatch: pytest.MonkeyPatch) -> None:
context = context_for([], monkeypatch)
assert context["tag"] == "NOGRP"
def test_title_type_is_in_the_context(monkeypatch: pytest.MonkeyPatch) -> None:
assert context_for([], monkeypatch)["title_type"] == "movie"
def test_rules_run_after_lang_tag(monkeypatch: pytest.MonkeyPatch) -> None:
"""A rule matching on lang_tag only works if lang_tag is resolved first."""
monkeypatch.setattr(config, "language_tags", {"rules": [{"audio": "ja", "tag": "SUBBED"}]})
monkeypatch.setattr(config, "tag_rules", [{"when": {"lang_tag": "SUBBED"}, "tag": "SUBGRP"}])
movie = make_movie()
movie.tracks = TrackStub(audio=[types.SimpleNamespace(language=Language.get("ja"))], subtitles=[]) # type: ignore[assignment]
context = movie._build_base_template_context(make_media_info(["ja"]))
assert context["lang_tag"] == "SUBBED"
assert context["tag"] == "SUBGRP"
def test_a_quality_rule_matches_the_built_context(monkeypatch: pytest.MonkeyPatch) -> None:
"""Guards the context keys the docs tell users to match on."""
monkeypatch.setattr(config, "tag_rules", [{"when": {"quality": "2160p", "video": "H.265"}, "tag": "UHDGROUP"}])
video = types.SimpleNamespace(
width=3840,
height=2160,
format="HEVC",
scan_type="Progressive",
other_display_aspect_ratio=["16:9"],
hdr_format_commercial=None,
hdr_format=None,
transfer_characteristics=None,
transfer_characteristics_original=None,
frame_rate="23.976",
)
media_info = types.SimpleNamespace(video_tracks=[video], audio_tracks=[make_audio("en")])
context = make_movie()._build_base_template_context(media_info)
assert context["quality"] == "2160p"
assert context["video"] == "H.265"
assert context["tag"] == "UHDGROUP"
def test_dl_clears_tag_rules_when_tag_is_given() -> None:
"""Source-level pin: the block lives inside the click-driven dl.__init__."""
from unshackle.commands.dl import dl
source = inspect.getsource(dl.__init__)
block = source.split("if tag:")[1].split("\n\n")[0]
assert "config.tag = tag" in block
assert "config.tag_rules = []" in block
+174
View File
@@ -0,0 +1,174 @@
"""Tests for per-task temp directories.
Each run gets a private dir that is removed on every exit path. The sweep reclaims
dirs from hard-killed processes via lock liveness, and `env clear temp` skips the
dir of a task that is still running.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
import pytest
from filelock import FileLock
from unshackle.commands.env import clear_directory
from unshackle.core.config import config
from unshackle.core.temp import LOCK_NAME, TASK_PREFIX, is_stale, sweep_task_dirs, task_temp_dir
pytestmark = pytest.mark.unit
@pytest.fixture
def temp_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
root = tmp_path / "temp"
root.mkdir()
monkeypatch.setattr(config.directories, "temp", root)
return root
def backdate(path: Path) -> None:
stamp = time.time() - 120
os.utime(path, (stamp, stamp))
def make_task_dir(root: Path, name: str = "task_deadbeef", lock: bool = True) -> Path:
task_dir = root / name
task_dir.mkdir()
if lock:
(task_dir / LOCK_NAME).touch()
return task_dir
def test_task_dir_created_repointed_and_restored(temp_root: Path) -> None:
with task_temp_dir() as task_dir:
assert task_dir.parent == temp_root
assert task_dir.name.startswith(TASK_PREFIX)
assert config.directories.temp == task_dir
assert config.directories.temp == temp_root
assert not task_dir.exists()
def test_task_dir_removed_on_exception(temp_root: Path) -> None:
with pytest.raises(ValueError):
with task_temp_dir() as task_dir:
(task_dir / "partial.mp4").write_bytes(b"x")
raise ValueError("boom")
assert not task_dir.exists()
assert config.directories.temp == temp_root
def test_task_dir_removed_on_keyboard_interrupt(temp_root: Path) -> None:
with pytest.raises(KeyboardInterrupt):
with task_temp_dir() as task_dir:
raise KeyboardInterrupt
assert not task_dir.exists()
assert config.directories.temp == temp_root
def test_sweep_removes_unlocked_stale_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
(task_dir / "leftover.mp4").write_bytes(b"x")
backdate(task_dir)
sweep_task_dirs(temp_root)
assert not task_dir.exists()
def test_sweep_skips_locked_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
lock = FileLock(task_dir / LOCK_NAME)
lock.acquire()
backdate(task_dir)
try:
assert is_stale(task_dir) is False
sweep_task_dirs(temp_root)
assert task_dir.exists()
finally:
lock.release()
def test_sweep_skips_young_unlocked_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
assert is_stale(task_dir) is False
sweep_task_dirs(temp_root)
assert task_dir.exists()
def test_sweep_removes_stale_dir_without_lock_file(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root, lock=False)
backdate(task_dir)
assert is_stale(task_dir) is True
sweep_task_dirs(temp_root)
assert not task_dir.exists()
def test_sweep_ignores_non_task_entries(temp_root: Path) -> None:
loose_file = temp_root / "leftover.mp4"
loose_file.write_bytes(b"x")
other_dir = temp_root / "not_a_task"
other_dir.mkdir()
backdate(loose_file)
backdate(other_dir)
sweep_task_dirs(temp_root)
assert loose_file.exists()
assert other_dir.exists()
@pytest.mark.skipif(os.name == "nt", reason="symlink creation needs privileges on Windows")
def test_sweep_never_follows_symlinked_task_dir(temp_root: Path, tmp_path: Path) -> None:
target = tmp_path / "victim"
target.mkdir()
(target / "precious.mkv").write_bytes(b"data")
link = temp_root / "task_evil"
link.symlink_to(target)
backdate(target)
sweep_task_dirs(temp_root)
assert (target / "precious.mkv").exists()
assert not (target / LOCK_NAME).exists()
@pytest.mark.skipif(os.name == "nt", reason="symlink creation needs privileges on Windows")
def test_clear_directory_unlinks_symlinked_dir_without_following(temp_root: Path, tmp_path: Path) -> None:
target = tmp_path / "victim"
target.mkdir()
(target / "precious.mkv").write_bytes(b"data")
link = temp_root / "task_evil"
link.symlink_to(target)
backdate(target)
files_count, _ = clear_directory(temp_root)
assert not link.exists()
assert (target / "precious.mkv").exists()
assert files_count == 1
def test_clear_directory_skips_locked_task_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
(task_dir / "in_flight.mp4").write_bytes(b"xyz")
lock = FileLock(task_dir / LOCK_NAME)
lock.acquire()
backdate(task_dir)
loose_file = temp_root / "leftover.mp4"
loose_file.write_bytes(b"1234")
try:
files_count, freed_bytes = clear_directory(temp_root)
finally:
lock.release()
assert task_dir.exists()
assert (task_dir / "in_flight.mp4").exists()
assert not loose_file.exists()
assert files_count == 1
assert freed_bytes == 4
def test_clear_directory_removes_stale_task_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
(task_dir / "orphan.mp4").write_bytes(b"12345")
backdate(task_dir)
files_count, freed_bytes = clear_directory(temp_root)
assert not task_dir.exists()
assert temp_root.is_dir()
# glob("**/*") never matches dotfiles, so .lock is not counted
assert files_count == 1
assert freed_bytes == 5
+292
View File
@@ -0,0 +1,292 @@
"""Separator cleanup around omitted conditionals in TemplateFormatter.
An omitted `{var?}` must not leave separator debris behind. Trailing spaces are the
dangerous case: the formatter sanitizes each path segment separately, and Windows
drops a trailing space when it creates a directory but not when it looks one up, so
the download fails with FileNotFoundError [WinError 3]. See #107 and #144.
"""
from __future__ import annotations
import pytest
from unshackle.core.config import config
from unshackle.core.titles.episode import Episode
from unshackle.core.titles.movie import Movie
from unshackle.core.titles.music import Song as MusicSong
from unshackle.core.titles.song import Song
from unshackle.core.utilities import sanitize_filename
from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer
@pytest.mark.parametrize(
("template", "context", "expected"),
[
# An omitted conditional inside brackets must take the whole group with it.
("{title} ({year?})", {"title": "Some Movie", "year": ""}, "Some Movie"),
("{title} [{quality?}]", {"title": "Some Movie", "quality": ""}, "Some Movie"),
("{title} ({year?}) [{quality?}]", {"title": "T", "year": "", "quality": ""}, "T"),
# The group survives when the value is present.
("{title} ({year?})", {"title": "Some Movie", "year": "2024"}, "Some Movie (2024)"),
# #107: the dash wins, so the release tag stays attached.
(
"{title}.{year}.{atmos?}-{tag}",
{"title": "Some Movie", "year": "2024", "atmos": "", "tag": "GRP"},
"Some.Movie.2024-GRP",
),
(
"{title}.{year}.{atmos?}-{tag}",
{"title": "Some Movie", "year": "2024", "atmos": "Atmos", "tag": "GRP"},
"Some.Movie.2024.Atmos-GRP",
),
# A conditional between two dashes must not leave a doubled separator.
("{title} - {edition?} - {year}", {"title": "A", "edition": "", "year": "2024"}, "A - 2024"),
# Consecutive omitted conditionals collapse to nothing.
("{a?} {b?} {title}", {"a": "", "b": "", "title": "T"}, "T"),
("{show} S{season}E{ep} {name?}", {"show": "Show", "season": "01", "ep": "02", "name": ""}, "Show S01E02"),
],
)
def test_omitted_conditional_leaves_no_separator_debris(template, context, expected):
assert TemplateFormatter(template).format(context) == expected
@pytest.mark.parametrize(
("template", "context"),
[
("{title} ({year?})", {"title": "Some Movie", "year": ""}),
("{title} [{quality?}]", {"title": "Some Movie", "quality": ""}),
("{title}.{quality?}", {"title": "Some Movie", "quality": ""}),
("{title} - {tag?}", {"title": "Some Movie", "tag": ""}),
],
)
def test_result_is_safe_as_a_windows_path_segment(template, context):
"""Windows strips these when creating a directory but not when looking one up."""
result = TemplateFormatter(template).format(context)
assert result == result.strip(" ."), f"unsafe path segment: {result!r}"
def test_empty_result_falls_back():
assert TemplateFormatter("{title?}").format({"title": ""}) == "untitled"
def test_missing_required_variable_raises():
with pytest.raises(ValueError, match="Missing required template variables"):
TemplateFormatter("{title}.{year}").format({"title": "T"})
@pytest.mark.parametrize(
("raw", "spacer", "expected"),
[
("Some Movie ", " ", "Some Movie"),
(" Some Movie", " ", "Some Movie"),
("Some Movie...", ".", "Some.Movie"),
("S.W.A.T.", ".", "S.W.A.T"),
("...Some Movie...", ".", "Some.Movie"),
],
)
def test_sanitize_filename_strips_unsafe_edges(raw, spacer, expected):
assert sanitize_filename(raw, spacer) == expected
def test_sanitize_filename_keeps_interior_dots():
assert sanitize_filename("Mr. Robot", ".") == "Mr.Robot"
@pytest.mark.parametrize(
("template", "context", "expected"),
[
# Space-separated style.
("{title} ({year})", {"title": "Some Movie", "year": "2024"}, "Some Movie (2024)"),
(
"{title} {year} {quality} {source}",
{"title": "Some Movie", "year": "2024", "quality": "1080p", "source": "WEB-DL"},
"Some Movie 2024 1080p WEB-DL",
),
# Dot-separated style.
(
"{title}.{year}.{quality}.{source}",
{"title": "Some Movie", "year": "2024", "quality": "1080p", "source": "WEB-DL"},
"Some.Movie.2024.1080p.WEB-DL",
),
# Interior dots in the title itself survive both styles.
("{title}.{year}", {"title": "Mr. Robot", "year": "2015"}, "Mr.Robot.2015"),
("{title} ({year})", {"title": "Mr. Robot", "year": "2015"}, "Mr. Robot (2015)"),
("{title}.{year}", {"title": "S.W.A.T.", "year": "2017"}, "S.W.A.T.2017"),
("{title} ({year})", {"title": "S.W.A.T.", "year": "2017"}, "S.W.A.T. (2017)"),
# A decimal inside a codec name is not a separator.
(
"{title}.{year}.{audio}-{tag}",
{"title": "Some Movie", "year": "2024", "audio": "DDP5.1", "tag": "GRP"},
"Some.Movie.2024.DDP5.1-GRP",
),
],
)
def test_separator_styles_keep_interior_spacing(template, context, expected):
assert TemplateFormatter(template).format(context) == expected
class DummyService:
pass
class StubMediaInfo:
"""Minimal MediaInfo stand-in: the template context only reads track lists."""
video_tracks: list = []
audio_tracks: list = []
@pytest.fixture
def folder_config(monkeypatch):
"""Clear the global template config, and restore it after each test."""
monkeypatch.setattr(config, "folder_templates", {})
monkeypatch.setattr(config, "folder_template", "")
monkeypatch.setattr(config, "output_template", {})
return config
def make_movie():
return Movie(id_="movie-0001", service=DummyService, name="Some Movie")
def make_episode(**overrides):
kwargs = dict(id_="episode-0001", service=DummyService, title="Some Show", season=1, number=2, name="Pilot")
kwargs.update(overrides)
return Episode(**kwargs)
def make_song(**overrides):
kwargs = dict(
id_="track-0001",
service=DummyService,
name="Some Track",
artist="Some Artist",
album="Some Album",
track=1,
disc=1,
year=2025,
)
kwargs.update(overrides)
return Song(**kwargs)
def make_album_song(**overrides):
kwargs = dict(
id_="track-0001",
service=DummyService,
name="Some Track",
artist="Some Artist",
album="Some Album",
track=1,
disc=1,
year=2025,
album_artist="Some Artist",
)
kwargs.update(overrides)
return MusicSong(**kwargs)
def assert_safe_path(result: str):
"""No segment may start or end with a space or dot, or Windows loses the directory."""
segments = result.split("/")
assert all(segments), f"empty segment in {result!r}"
assert all(seg == seg.strip(" .") for seg in segments), f"unsafe segment in {result!r}"
@pytest.mark.parametrize(
("kind", "make_title", "template", "expected"),
[
("movies", make_movie, "Movies/{title} ({year?})", "Movies/Some Movie"),
("series", make_episode, "Shows/{title} ({year?})/Season {season}", "Shows/Some Show/Season S01"),
("songs", make_song, "Music/{artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"),
("albums", make_album_song, "Music/{album_artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"),
],
)
def test_folder_templates_keep_path_separators(folder_config, kind, make_title, template, expected):
"""`/` survives because each segment is formatted separately, but edge spaces do not."""
folder_config.folder_templates = {kind: template}
result = make_title().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize(
("kind", "make_title", "template"),
[
("movies", make_movie, "{title} ({year?})"),
("series", make_episode, "{title} ({year?})"),
("songs", make_song, "{artist} - {album} [{genre?}]"),
("albums", make_album_song, "{album_artist} - {album} [{genre?}]"),
],
)
def test_folder_names_are_safe_when_conditionals_are_omitted(folder_config, kind, make_title, template):
folder_config.folder_templates = {kind: template}
assert_safe_path(make_title().get_filename(StubMediaInfo(), folder=True))
@pytest.mark.parametrize(
("template", "expected"),
[
("Shows/{title}.{year?}/Season.{season}", "Shows/Some.Show/Season.S01"),
("Shows/{title} {year?}/Season {season}", "Shows/Some Show/Season S01"),
],
)
def test_series_folder_handles_both_separator_styles(folder_config, template, expected):
folder_config.folder_templates = {"series": template}
result = make_episode().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize(
("template", "expected"),
[
# A segment with one variable has no gap to judge by, so it follows the whole
# path's style instead of falling back to a dot.
("Music/{artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"),
("Music/{artist}/{album}.{year?}", "Music/Some.Artist/Some.Album.2025"),
("{artist}/{album}", "Some.Artist/Some.Album"),
],
)
def test_single_variable_segments_follow_the_whole_path(folder_config, template, expected):
folder_config.folder_templates = {"songs": template}
result = make_song().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize(
("template", "expected"),
[
("{title}.{year}.{quality}", "."),
("{title} {year} {quality}", " "),
("{title} ({year})", " "),
("Movies/{title} ({year?})", " "),
("Shows/{title}.{year?}/Season.{season}", "."),
("{title}", "."), # nothing between two variables, so scene style wins
],
)
def test_detect_spacer(template, expected):
assert detect_spacer(template) == expected
@pytest.mark.parametrize(
("series_template", "expected"),
[
# Deriving a show folder strips the episode tokens, so the template ends on a
# separator once {episode_name?} goes too.
("{title}.{year?}.{season_episode}.{episode_name?}", "Some.Show.S01"),
("{title} {year?} {season_episode} {episode_name?}", "Some Show S01"),
],
)
def test_series_folder_derived_from_output_template(folder_config, series_template, expected):
"""With no folder template set, the show folder comes from the file template."""
folder_config.output_template = {"series": series_template}
result = make_episode().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize("make_title", [make_movie, make_episode, make_song, make_album_song])
def test_folder_fallbacks_need_no_template(folder_config, make_title):
assert_safe_path(make_title().get_filename(StubMediaInfo(), folder=True))
+232
View File
@@ -0,0 +1,232 @@
"""TVDB provider ordering and season-order remapping."""
from __future__ import annotations
import pytest
from unshackle.core import providers
from unshackle.core.config import config
from unshackle.core.providers.tvdb import TVDBProvider, _ids_from_remote, _pick_match
from unshackle.core.utils.tags import _build_tags_from_ids
REMOTE_IDS = [
{"id": "tt0149460", "type": 2, "sourceName": "IMDB"},
{"id": "http://example.com", "type": 4, "sourceName": "Official Website"},
{"id": "615", "type": 12, "sourceName": "TheMovieDB.com"},
]
# Futurama: aired S1 stops at E9, the four held-back episodes open aired S2.
# DVD order folds them back into S1 as E10-E13.
AIRED = [
{"id": 1, "seasonNumber": 1, "number": 9, "name": "Hell Is Other Robots"},
{"id": 2, "seasonNumber": 2, "number": 1, "name": "A Flight to Remember"},
{"id": 3, "seasonNumber": 2, "number": 2, "name": "Mars University"},
{"id": 99, "seasonNumber": 2, "number": 3, "name": "Only In Aired"},
]
DVD = [
{"id": 1, "seasonNumber": 1, "number": 9, "name": "Hell Is Other Robots"},
{"id": 2, "seasonNumber": 1, "number": 10, "name": "A Flight to Remember"},
{"id": 3, "seasonNumber": 1, "number": 11, "name": "Mars University"},
]
def test_ids_from_remote() -> None:
ext = _ids_from_remote(REMOTE_IDS, 73871)
assert ext.imdb_id == "tt0149460"
assert ext.tmdb_id == 615
assert ext.tvdb_id == 73871
def _stub(p: TVDBProvider, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(p, "get_episodes", lambda _id, order: {"official": AIRED, "dvd": DVD}.get(order, []))
def test_order_map_joins_on_episode_id(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
mapping = p.get_order_map(73871, "dvd")
assert mapping[(1, 9)] == (1, 9, "Hell Is Other Robots")
assert mapping[(2, 1)] == (1, 10, "A Flight to Remember")
assert mapping[(2, 2)] == (1, 11, "Mars University")
# an episode absent from the target order is left out, so it keeps its own numbering
assert (2, 3) not in mapping
def test_order_map_reverses_with_source_order(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
mapping = p.get_order_map(73871, "official", source_order="dvd")
assert mapping[(1, 10)] == (2, 1, "A Flight to Remember")
def test_order_map_is_noop_when_orders_match() -> None:
assert TVDBProvider().get_order_map(73871, "official") == {}
def test_order_map_empty_when_order_missing(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
assert p.get_order_map(73871, "alternate") == {}
def test_detect_order_picks_the_order_the_numbering_matches(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
# a service may number a series in a non-aired order, so detection must not assume aired
assert p.detect_order(73871, [(1, 10), (1, 11)]) == "dvd"
assert p.detect_order(73871, [(2, 1), (2, 2), (2, 3)]) == "official"
def test_detect_order_falls_back_to_aired(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
monkeypatch.setattr(p, "get_episodes", lambda _id, order: [])
assert p.detect_order(73871, [(1, 1)]) == "official"
# TVDB holds both Battlestar Galactica series; the 2003 revival is filed under year 2005
BSG = [
{"name": "Battlestar Galactica", "year": "1978", "tvdb_id": "71173"},
{"name": "Battlestar Galactica (2003)", "year": "2005", "tvdb_id": "73545"},
]
def test_pick_match_prefers_the_right_era() -> None:
# exact title match on the 1978 series must not beat the era the caller asked for
best, _, _ = _pick_match(BSG, "Battlestar Galactica", 2004)
assert best["tvdb_id"] == "73545"
best, _, _ = _pick_match(BSG, "Battlestar Galactica", 1978)
assert best["tvdb_id"] == "71173"
def test_pick_match_without_a_year_scores_on_title_alone() -> None:
best, _, _ = _pick_match(BSG, "Battlestar Galactica", None)
assert best["tvdb_id"] == "71173"
def test_pick_match_returns_nothing_when_no_year_fits() -> None:
assert _pick_match(BSG, "Battlestar Galactica", 1995)[0] is None
def test_tvdb2_tag_follows_the_matroska_spec() -> None:
# TVDB2 IDs are prefixed by entity type: matroska.org/technical/tagging.html
from unshackle.core.providers import ExternalIds
assert _build_tags_from_ids(ExternalIds(tvdb_id=73871), "tv")["TVDB2"] == "series/73871"
assert _build_tags_from_ids(ExternalIds(tvdb_id=113), "movie")["TVDB2"] == "movies/113"
def test_provider_order_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "metadata_providers", [])
assert [c.NAME for c in providers.provider_order()] == list(providers.DEFAULT_ORDER)
def test_provider_order_is_configurable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "metadata_providers", ["TVDB", "tmdb", "nope", "tvdb"])
# unknown entries dropped, duplicates collapsed, case ignored, order kept
assert [c.NAME for c in providers.provider_order()] == ["tvdb", "tmdb"]
def test_enrichment_authority_is_fixed(monkeypatch: pytest.MonkeyPatch) -> None:
# search order must not demote TMDB below SIMKL for cross-validation tie-breaks
monkeypatch.setattr(config, "metadata_providers", [])
assert providers._enrichment_providers() == ["tmdb", "simkl", "tvdb"]
# `metadata_providers` filters the set but does not re-rank the trust order
monkeypatch.setattr(config, "metadata_providers", ["tvdb", "simkl"])
assert providers._enrichment_providers() == ["simkl", "tvdb"]
def test_cached_tvdb_result_keeps_enriched_ids() -> None:
cached = {
"tvdb_id": "73871",
"name": "Futurama",
"year": "1999",
"_enriched_ids": {"imdb_id": "tt0149460", "tmdb_id": 615},
}
result = providers._cached_to_result(cached, "tvdb", "tv")
assert result is not None
assert result.external_ids.tvdb_id == 73871
assert result.external_ids.imdb_id == "tt0149460"
assert result.external_ids.tmdb_id == 615
# remote_ids stay authoritative over the enriched copies
cached["remote_ids"] = REMOTE_IDS
cached["_enriched_ids"] = {"imdb_id": "tt9999999", "tmdb_id": 1}
result = providers._cached_to_result(cached, "tvdb", "tv")
assert result.external_ids.imdb_id == "tt0149460"
assert result.external_ids.tmdb_id == 615
def test_get_episodes_fails_closed_on_a_truncated_listing(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
pages: dict[int, object] = {0: {"episodes": [{"id": n} for n in range(500)]}, 1: None}
monkeypatch.setattr(p, "_get", lambda _path, params: pages[params["page"]])
# page 1 fails mid-listing: a partial list would renumber wrongly
assert p.get_episodes(73871, "official") == []
assert p._episodes == {}
def test_get_episodes_does_not_cache_an_empty_listing(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
monkeypatch.setattr(p, "_get", lambda _path, params: None)
assert p.get_episodes(73871, "official") == []
assert p._episodes == {}
# --- apply_tvdb_order over multi-part episodes -------------------------------
class _Svc:
pass
class _FakeProvider:
def __init__(self, mapping: dict) -> None:
self.mapping = mapping
def detect_order(self, _id, _keys) -> str:
return "official"
def get_order_map(self, _id, _order, source_order=None) -> dict:
return self.mapping
def _renumber(monkeypatch: pytest.MonkeyPatch, episodes: list, mapping: dict):
from types import SimpleNamespace
from unshackle.commands.dl import dl
from unshackle.core.titles.episode import Series
monkeypatch.setattr(providers, "get_provider", lambda _name: _FakeProvider(mapping))
errors: list = []
stub = SimpleNamespace(
tvdb_id=73871,
tvdb_order="dvd",
service="STUB",
log=SimpleNamespace(warning=lambda *a, **k: None, info=lambda *a, **k: None, error=lambda *a: errors.append(a)),
)
return dl.apply_tvdb_order(stub, Series(episodes)), errors
def _ep(season: int, number: int, part=None):
from unshackle.core.titles.episode import Episode
return Episode(id_=f"{season}x{number}.{part}", service=_Svc, title="Show", season=season, number=number, part=part)
def test_parts_of_one_episode_renumber_together(monkeypatch: pytest.MonkeyPatch) -> None:
"""Three parts share one (season, number) slot, so they must not read as a collision."""
episodes = [_ep(1, 1, 1), _ep(1, 1, 2), _ep(1, 1, 3), _ep(1, 2)]
mapping = {(1, 1): (2, 1, "Part-ful"), (1, 2): (2, 2, "Whole")}
titles, errors = _renumber(monkeypatch, episodes, mapping)
assert errors == []
assert [(t.season, t.number, t.part) for t in titles] == [(2, 1, 1), (2, 1, 2), (2, 1, 3), (2, 2, None)]
def test_a_real_collision_still_bails(monkeypatch: pytest.MonkeyPatch) -> None:
"""Dedupe must not disable the guard: two distinct episodes on one slot still refuses."""
episodes = [_ep(1, 1), _ep(1, 2)]
mapping = {(1, 2): (1, 1, "Clash")} # (1, 1) is unmapped and keeps the slot (1, 2) moves onto
titles, errors = _renumber(monkeypatch, episodes, mapping)
assert errors # refused, with the "two episodes each" error
assert [(t.season, t.number) for t in titles] == [(1, 1), (1, 2)] # numbering untouched
+110
View File
@@ -0,0 +1,110 @@
import gzip
import socket
import threading
import pytest
import requests as rq
from unshackle.core.downloaders.requests import _is_content_encoded, download
@pytest.mark.parametrize(
("value", "expected"),
[
(None, False),
("", False),
(" ", False),
("identity", False),
("IDENTITY", False),
("identity, identity", False),
("gzip", True),
("GZIP", True),
(" gzip ", True),
("x-gzip", True),
("br", True),
("zstd", True),
("deflate", True),
("compress", True),
("dcb", True),
("gzip, br", True),
("identity, gzip", True),
("gzip, identity", True),
("gzip,,", True),
("some-future-coding", True),
],
)
def test_is_content_encoded(value, expected):
assert _is_content_encoded(value) is expected
PLAIN = b"WEBVTT\n\n" + b"".join(f"{i:05d} cue text\n".encode() for i in range(2000))
class _EncodingServer:
"""Serves one gzip body, honouring Range against the encoded representation."""
def __init__(self):
self.encoded = gzip.compress(PLAIN)
self.sock = socket.socket()
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("127.0.0.1", 0))
self.sock.listen(8)
self.url = f"http://127.0.0.1:{self.sock.getsockname()[1]}/sub.vtt"
threading.Thread(target=self._serve, daemon=True).start()
def _serve(self):
while True:
try:
conn, _ = self.sock.accept()
except OSError:
return
try:
request = conn.recv(65536).decode("latin1")
start = next(
(
int(line.split("=", 1)[1].split("-")[0])
for line in request.split("\r\n")
if line.lower().startswith("range:")
),
None,
)
if start is None:
body = self.encoded
head = f"HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n"
else:
body = self.encoded[start:]
head = (
f"HTTP/1.1 206 Partial Content\r\nContent-Length: {len(body)}\r\n"
f"Content-Range: bytes {start}-{len(self.encoded) - 1}/{len(self.encoded)}\r\n"
)
conn.sendall(
(head + "Content-Encoding: gzip\r\nAccept-Ranges: bytes\r\nConnection: close\r\n\r\n").encode()
+ body
)
except Exception:
pass
finally:
conn.close()
def close(self):
self.sock.close()
@pytest.fixture
def encoding_server():
server = _EncodingServer()
yield server
server.close()
def test_encoded_body_is_decoded_on_disk(tmp_path, encoding_server):
save_path = tmp_path / "sub.vtt"
list(download(url=encoding_server.url, save_path=save_path, session=rq.Session()))
assert save_path.read_bytes() == PLAIN
def test_encoded_body_discards_a_partial_instead_of_range_resuming(tmp_path, encoding_server):
save_path = tmp_path / "sub.vtt"
save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000])
list(download(url=encoding_server.url, save_path=save_path, session=rq.Session()))
assert save_path.read_bytes() == PLAIN
+118
View File
@@ -0,0 +1,118 @@
import importlib
import inspect
import threading
import time
import pytest
from unshackle.core.constants import DOWNLOAD_CANCELLED
from unshackle.core.downloaders.requests import TokenBucket, format_speed, parse_speed_limit, set_speed_limit
requests_downloader = importlib.import_module("unshackle.core.downloaders.requests")
@pytest.mark.parametrize(
("value", "expected"),
[
("500k", 500_000),
("5M", 5_000_000),
("10MB", 10_000_000),
("10mb/s", 10_000_000),
("1.5G", 1_500_000_000),
("1048576", 1_048_576),
(5_000_000, 5_000_000.0),
(None, None),
("off", None),
("none", None),
("unlimited", None),
("0", None),
(0, None),
],
)
def test_parse_speed_limit(value, expected):
assert parse_speed_limit(value) == expected
@pytest.mark.parametrize(
"value",
["fast", "10TB", "10MiB", "-5M", "M", "1..5M", ".", "1.2.3", float("inf"), float("nan"), -5, -1.5],
)
def test_parse_speed_limit_rejects(value):
with pytest.raises(ValueError):
parse_speed_limit(value)
@pytest.mark.parametrize(
("rate", "expected"),
[
(500, "500 bytes/s"),
(512_000, "512.0 kB/s"),
(39 * 1024**2, "40.9 MB/s"),
(1.5 * 1024**3, "1.6 GB/s"),
],
)
def test_format_speed(rate, expected):
assert format_speed(rate) == expected
def test_result_accepts_a_missing_speed_limit():
"""The API/serve path calls dl.result() by keyword without speed_limit."""
from unshackle.commands.dl import dl
assert inspect.signature(dl.result).parameters["speed_limit"].default is None
def test_bucket_holds_aggregate_rate_across_threads():
rate = 10_000_000
bucket = TokenBucket(rate)
chunk = 100_000
totals = [0, 0, 0, 0]
start = time.monotonic()
stop = start + 1.0
def worker(i: int) -> None:
while time.monotonic() < stop:
bucket.consume(chunk)
totals[i] += chunk
threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)]
for t in threads:
t.start()
for t in threads:
t.join()
achieved = sum(totals) / (time.monotonic() - start)
# exceeding the limit is a real failure; undershoot is scheduler noise on a loaded box
assert achieved <= rate * 1.1
assert achieved >= rate * 0.5
def test_bucket_chunk_larger_than_capacity_does_not_deadlock():
bucket = TokenBucket(1_000_000)
start = time.monotonic()
bucket.consume(1_100_000)
assert 1.0 < time.monotonic() - start < 2.0
def test_locked_limit_ignores_unlocked_calls():
"""serve's global_speed_limit must survive the per-job set in dl.result()."""
try:
set_speed_limit(5_000_000, lock=True)
set_speed_limit(None)
set_speed_limit(1_000_000)
limiter = requests_downloader._speed_limiter
assert limiter is not None and limiter.rate == 5_000_000
finally:
requests_downloader._speed_limit_locked = False
requests_downloader._speed_limiter = None
def test_bucket_wait_aborts_on_cancellation():
"""Debt sleeps grow with thread count; cancelling must not wait them out."""
bucket = TokenBucket(1_000)
DOWNLOAD_CANCELLED.set()
try:
start = time.monotonic()
bucket.consume(1_000_000)
assert time.monotonic() - start < 1.0
finally:
DOWNLOAD_CANCELLED.clear()
+43
View File
@@ -0,0 +1,43 @@
import pytest
from unshackle.core.downloaders.requests import SPEED_ROLLING_WINDOW, SpeedWindow
def test_rate_reflects_recent_window_not_full_history():
"""After a slow start (shared bucket), the display must show the current rate."""
win = SpeedWindow(0.0)
total = 0
for t in range(0, 60):
total += 5_000_000
win.rate(float(t), total)
for t in range(60, 120):
total += 10_000_000
rate = win.rate(float(t), total)
assert rate == pytest.approx(10_000_000, rel=0.01)
def test_first_report_is_the_since_start_average():
"""Bytes are credited in lumps; the first lump must not read as a whole window's worth."""
win = SpeedWindow(0.0)
assert win.rate(2.5, 25_000_000) == pytest.approx(10_000_000)
def test_zero_span_returns_none():
win = SpeedWindow(0.0)
assert win.rate(0.0, 1_000_000) is None
def test_stall_returns_none():
"""Once the seed ages out, a total that stops growing has no delta to report."""
win = SpeedWindow(0.0)
for t in range(0, SPEED_ROLLING_WINDOW + 6):
rate = win.rate(float(t), 1_000_000)
assert rate is None
def test_window_drops_old_samples():
win = SpeedWindow(0.0)
for t in range(0, 3 * SPEED_ROLLING_WINDOW):
win.rate(float(t), t * 1_000_000)
assert win.samples[0][0] >= 2 * SPEED_ROLLING_WINDOW - 1
assert len(win.samples) == SPEED_ROLLING_WINDOW + 1
+24
View File
@@ -0,0 +1,24 @@
from __future__ import annotations
from typing import Optional
import pytest
from unshackle.core.utilities import as_requested
@pytest.mark.parametrize(
"tokens,orig_token,expected",
[
# the user passed no -l, so orig resolved to en behind their back; report orig
(["en"], "en", "orig"),
(["en", "ja"], "en", "orig, ja"),
# the user named en, so report en
(["en"], None, "en"),
(["ja"], "en", "ja"),
(["en", "ja"], None, "en, ja"),
([], "en", ""),
],
)
def test_as_requested(tokens: list[str], orig_token: Optional[str], expected: str) -> None:
assert as_requested(tokens, orig_token) == expected
@@ -0,0 +1,25 @@
from __future__ import annotations
import pytest
from unshackle.core.utilities import keep_forced_subtitle
@pytest.mark.parametrize(
"forced,language,forced_s_lang,exact,expected",
[
(False, "en", [], False, True),
(False, "de", ["en"], False, True),
(True, "en", [], False, False),
(True, "en", ["en"], False, True),
(True, "en-US", ["en"], False, True),
(True, "de", ["en"], False, False),
(True, "de", ["en", "de"], False, True),
# CLDR rates paradigm-regional pairs (en/en-US) distance 0, so exact still matches
(True, "en-US", ["en"], True, True),
(True, "en", ["en"], True, True),
(True, "es-419", ["es-ES"], True, False),
],
)
def test_keep_forced_subtitle(forced: bool, language: str, forced_s_lang: list[str], exact: bool, expected: bool):
assert keep_forced_subtitle(forced, language, forced_s_lang, exact) is expected
@@ -0,0 +1,33 @@
from __future__ import annotations
from typing import Sequence
import pytest
from unshackle.core.utilities import matching_languages
POOL = ["ar", "en", "en-GB", "en-US", "ja"]
@pytest.mark.parametrize(
("language", "available", "exact", "expected"),
[
("en", POOL, False, {"en", "en-GB", "en-US"}),
# CLDR rates en/en-US at distance 0, so exact mode leans on the RFC 4647 string preference
("en", POOL, True, {"en"}),
("en-GB", POOL, True, {"en-GB"}),
# no string-equal tag, so exact mode falls back to the distance match
("zh", ["ar", "cmn", "ja"], True, {"cmn"}),
("en", ["ar", "ja"], False, set()),
("en", [], False, set()),
("en", ["ar", None, "en"], True, {"en"}),
# unparseable tags (config typos) must match nothing, never raise
("engrish", POOL, False, set()),
("engrish", POOL, True, set()),
("", POOL, False, set()),
("", POOL, True, set()),
("f", POOL, False, set()),
],
)
def test_matching_languages(language: str, available: Sequence[str], exact: bool, expected: set[str]) -> None:
assert matching_languages(language, available, exact) == expected
@@ -0,0 +1,43 @@
from __future__ import annotations
from typing import Optional
import pytest
from unshackle.core.api.handlers import original_audio_ids
class FakeTrack:
def __init__(self, language: str) -> None:
self.id = language
self.language = language
class FakeTitle:
def __init__(self, language: Optional[str]) -> None:
self.language = language
@pytest.mark.parametrize(
"title_language,track_languages,expected",
[
# paradigm collapse: CLDR calls these the same language, RFC 4647 picks the specific one
(["en"], ["en", "en-US"], ["en"]),
(["en"], ["en-US", "en-GB"], ["en-US"]),
(["pt"], ["pt-BR", "pt-PT"], ["pt-BR"]),
(["es"], ["es-ES", "es-419"], ["es-ES"]),
# non-paradigm regionals: exact mode drops them, so the fuzzy fallback catches
# them and the flag still matches what a default 'orig' download selects
(["es"], ["es-419"], ["es-419"]),
(["fr"], ["fr-CA"], ["fr-CA"]),
# alias
(["zh"], ["cmn"], ["cmn"]),
# nothing in the original language, and no language on the title at all
(["en"], ["ja", "de"], []),
([None], ["en"], []),
],
)
def test_original_audio_ids(title_language: list, track_languages: list[str], expected: list[str]) -> None:
tracks = [FakeTrack(t) for t in track_languages]
ids = original_audio_ids(tracks, FakeTitle(title_language[0]))
assert [t for t in track_languages if t in ids] == expected
@@ -0,0 +1,24 @@
from __future__ import annotations
from typing import Optional, Sequence
import pytest
from langcodes import Language
from unshackle.core.utilities import resolve_sort_langs
@pytest.mark.parametrize(
("tokens", "original", "expected"),
[
(["orig", "en", "es"], "ja", ["ja", "en", "es"]),
(["orig", "en", "es"], None, ["en", "es"]),
(["orig"], Language.get("pt-BR"), ["pt-BR"]),
(["en", "orig"], "en", ["en"]),
(["en", "es", "en", "fr"], None, ["en", "es", "fr"]),
(["all", "best", "en"], "ja", ["all", "best", "en"]),
([], "ja", []),
],
)
def test_resolve_sort_langs(tokens: Sequence[str], original: Optional[str], expected: list[str]) -> None:
assert resolve_sort_langs(tokens, original) == expected
+331
View File
@@ -0,0 +1,331 @@
"""Unit tests for multi-part episode support across the REST/API layer.
Covers serialize_title's conditional `part` key, the list-tracks wanted filter
routing through Episode.matches_wanted, the discrete season/episode/part request
fields, and download_manager's already-internal wanted sniff.
"""
from __future__ import annotations
import asyncio
import json
import re
from typing import Any, Dict, List, Optional
import pytest
from unshackle.core.api import handlers
from unshackle.core.api.errors import APIError
from unshackle.core.api.handlers import serialize_title
from unshackle.core.titles.episode import Episode
pytestmark = pytest.mark.unit
class _FakeSvc:
pass
def _ep(season: int = 1, number: int = 1, part: Optional[int] = None, **overrides: Any) -> Episode:
base: Dict[str, Any] = dict(
id_=f"ep-{season}-{number}-{part or 0}",
service=_FakeSvc,
title="My Show",
season=season,
number=number,
name="Pilot",
year=2024,
language=None,
)
base.update(overrides)
return Episode(part=part, **base)
class _FakeTracks:
videos: List[Any] = []
audio: List[Any] = []
subtitles: List[Any] = []
class _FakeService:
def __init__(self, titles: List[Episode]) -> None:
self._titles = titles
def get_titles(self) -> List[Episode]:
return self._titles
def get_tracks(self, title: Episode) -> _FakeTracks:
return _FakeTracks()
def _run_list_tracks(monkeypatch: pytest.MonkeyPatch, titles: List[Episode], **data: Any) -> Dict[str, Any]:
"""Drive list_tracks_handler end to end with the service layer stubbed out."""
monkeypatch.setattr(handlers, "validate_service", lambda tag, request=None: "FAKE")
monkeypatch.setattr(handlers, "setup_list_service", lambda *a, **kw: _FakeService(titles))
payload = {"service": "FAKE", "title_id": "t1", **data}
try:
response = asyncio.run(handlers.list_tracks_handler(payload))
except APIError as e:
# routes.py turns this into the HTTP error body; keep the fields the tests assert on
return {"status": e.http_status, "error_code": e.error_code, "details": e.details}
return json.loads(response.body)
def _selected(result: Dict[str, Any]) -> List[str]:
"""Selection-syntax keys of whatever the handler returned, single or multi."""
entries = result["episodes"] if "episodes" in result else [{"title": result["title"]}]
keys = []
for entry in entries:
t = entry["title"]
suffix = f".{t['part']}" if "part" in t else ""
keys.append(f"{t['season']}x{t['number']}{suffix}")
return keys
# ---------- serialize_title: the JSON contract ----------
def test_partless_episode_json_has_no_part_key() -> None:
d = serialize_title(_ep(part=None))
assert "part" not in d
assert list(d.keys()) == [
"type",
"name",
"id",
"language",
"description",
"date",
"cover_url",
"year",
"series_title",
"season",
"number",
]
assert d["name"] == "Pilot"
def test_partful_episode_json_carries_part_last() -> None:
d = serialize_title(_ep(part=2))
assert d["part"] == 2
# additive: `part` is appended, every pre-existing key keeps its position
assert list(d.keys())[:11] == list(serialize_title(_ep(part=None)).keys())
assert list(d.keys())[11] == "part"
def test_part_stays_out_of_the_name_field() -> None:
"""`name` is a reconstruction input for _build_title, not a display string."""
assert serialize_title(_ep(part=2))["name"] == "Pilot"
assert serialize_title(_ep(part=3, name=None))["name"] == "Episode 01"
def test_remote_build_title_round_trip() -> None:
from unshackle.core.remote_service import _build_title
rebuilt = _build_title(serialize_title(_ep(1, 4, 2)), "FAKE", "fallback-id")
assert (rebuilt.season, rebuilt.number, rebuilt.part, rebuilt.name) == (1, 4, 2, "Pilot")
partless = _build_title(serialize_title(_ep(1, 4, None)), "FAKE", "fallback-id")
assert partless.part is None
def test_movie_json_unaffected() -> None:
from langcodes import Language
from unshackle.core.titles.movie import Movie
d = serialize_title(Movie(id_="movie-0001", service=_FakeSvc, name="Film", year=2024, language=Language.get("en")))
assert "part" not in d
# ---------- list_tracks_handler: the REST wanted filter ----------
@pytest.fixture
def split_titles() -> List[Episode]:
"""E1 split into three parts, E2 unsplit."""
return [_ep(1, 1, 1), _ep(1, 1, 2), _ep(1, 1, 3), _ep(1, 2, None)]
def test_rest_bare_episode_key_selects_every_part(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E01")
assert _selected(result) == ["1x1.1", "1x1.2", "1x1.3"]
def test_rest_part_key_selects_exactly_one(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E01.2")
assert _selected(result) == ["1x1.2"]
def test_rest_negative_part_key_excludes_only_that_part(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, wanted=["S01", "-S01E01.2"])
assert _selected(result) == ["1x1.1", "1x1.3", "1x2"]
def test_rest_partless_episode_unchanged(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E02")
assert _selected(result) == ["1x2"]
assert "part" not in result["title"]
def test_rest_part_of_unsplit_episode_selects_nothing(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E02.1")
assert result["status"] == 404
# ---------- discrete season/episode/part request fields ----------
def test_discrete_fields_without_part_select_every_part(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1)
assert _selected(result) == ["1x1.1", "1x1.2", "1x1.3"]
def test_discrete_fields_with_part_select_one(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1, part=3)
assert _selected(result) == ["1x1.3"]
def test_discrete_part_accepts_string(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1, part="3")
assert _selected(result) == ["1x1.3"]
def test_discrete_no_match_error_reports_the_part_qualified_key(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1, part=9)
assert result["status"] == 404
assert result["details"]["wanted"] == "1x1.9"
def test_discrete_partless_error_key_unchanged(monkeypatch, split_titles) -> None:
result = _run_list_tracks(monkeypatch, split_titles, season=9, episode=9)
assert result["details"]["wanted"] == "9x9"
def test_part_is_a_transport_key_not_a_service_option() -> None:
"""`part` addresses the title, so it must never reach a service's own --part option."""
assert "part" in handlers.LIST_HANDLER_TRANSPORT_KEYS
assert "part" in handlers.SESSION_TRANSPORT_KEYS
# ---------- ordering ----------
def test_rest_multi_episode_order_is_stable_across_parts(monkeypatch) -> None:
titles = [_ep(1, 2, None), _ep(1, 1, 3), _ep(1, 1, 1), _ep(1, 1, 2)]
result = _run_list_tracks(monkeypatch, titles, wanted="S01")
assert _selected(result) == ["1x1.1", "1x1.2", "1x1.3", "1x2"]
# ---------- download_manager: already-internal wanted sniff ----------
_SNIFF = r"^!?\d+x\d+(\.\d+)?$"
@pytest.mark.parametrize(
"wanted, internal",
[
(["1x1"], True), # today's pre-parsed form
(["1x1", "1x2"], True),
(["1x1.2"], True), # part key
(["!1x1.2"], True), # negative part key
(["1x1", "!1x1.2"], True),
(["S01E01"], False),
(["s1e1.2"], False),
(["S01"], False),
(["1x1", "S01E02"], False),
],
)
def test_download_manager_sniff_regex(wanted: List[str], internal: bool) -> None:
needs_conversion = any(not re.match(_SNIFF, w) for w in wanted)
assert needs_conversion is (not internal)
def test_sniff_regex_matches_the_one_in_download_manager() -> None:
"""Guard the copy above against drift from the production regex."""
from pathlib import Path
import unshackle.core.api.download_manager as dm
source = Path(dm.__file__).read_text(encoding="utf8")
assert f'needs_conversion = any(not re.match(r"{_SNIFF}", w) for w in wanted_raw)' in source
def test_perform_download_skips_reconversion_for_preparsed_part_keys(monkeypatch) -> None:
"""A pre-parsed list with part and negative keys must not reach parse_tokens."""
from unshackle.core.api.download_manager import _perform_download
from unshackle.core.utils.click_types import SeasonRange
calls = []
monkeypatch.setattr(SeasonRange, "parse_tokens", lambda self, *t: calls.append(t) or [])
params: Dict[str, Any] = {"wanted": ["1x1", "1x1.2", "!1x1.3"]}
with pytest.raises(BaseException):
# blows up later on the unknown service; the wanted block runs first
_perform_download("job-1", "NoSuchServiceXYZ", "t1", params)
assert calls == []
assert params["wanted"] == ["1x1", "1x1.2", "!1x1.3"]
def test_perform_download_still_converts_cli_style_tokens(monkeypatch) -> None:
from unshackle.core.api.download_manager import _perform_download
from unshackle.core.utils.click_types import SeasonRange
calls = []
monkeypatch.setattr(SeasonRange, "parse_tokens", lambda self, *t: calls.append(t) or ["1x1"])
params: Dict[str, Any] = {"wanted": "S01E01"}
with pytest.raises(BaseException):
_perform_download("job-2", "NoSuchServiceXYZ", "t1", params)
assert calls == [("S01E01",)]
assert params["wanted"] == ["1x1"]
# ---------- CLI / REST agreement ----------
def test_dl_and_handlers_share_the_one_match_helper() -> None:
"""Both filters must route through Episode.matches_wanted, or they drift."""
from pathlib import Path
import unshackle.commands.dl as dl_module
dl_source = Path(dl_module.__file__).read_text(encoding="utf8")
handlers_source = Path(handlers.__file__).read_text(encoding="utf8")
assert "matches_wanted(wanted)" in dl_source
assert "matches_wanted(wanted)" in handlers_source
# no inline key compare left behind on either side
inline = re.compile(r"f\"\{title\.season\}x\{title\.number\}\"\s*\n?\s*(?:in|not in)\s+wanted")
assert not inline.search(dl_source)
assert not inline.search(handlers_source)
@pytest.mark.parametrize(
"wanted_tokens",
[
("S01E01",),
("S01E01.2",),
("S01",),
("S01", "-S01E01.2"),
("S01", "-S01E01"),
("S01E01.1-S01E01.3",),
("S01E02",),
],
)
def test_rest_filter_agrees_with_cli_filter(monkeypatch, split_titles, wanted_tokens) -> None:
from unshackle.core.utils.click_types import SeasonRange
wanted = SeasonRange().parse_tokens(*wanted_tokens)
cli_selected = [
f"{t.season}x{t.number}" + (f".{t.part}" if t.part is not None else "")
for t in split_titles
if t.matches_wanted(wanted)
]
result = _run_list_tracks(monkeypatch, split_titles, wanted=list(wanted_tokens))
rest_selected = [] if result.get("status") == 404 else _selected(result)
assert sorted(rest_selected) == sorted(cli_selected)
+59
View File
@@ -0,0 +1,59 @@
"""URL-backed attachments defer their fetch to download(), which honors no_proxy_download."""
from __future__ import annotations
import requests
import unshackle.core.tracks.track as track_module
from unshackle.core.config import config
from unshackle.core.tracks.attachment import Attachment
URL = "https://cdn.example.com/images/thumb.jpg"
class FakeResponse:
def raise_for_status(self) -> None:
pass
def iter_content(self, chunk_size: int):
yield b"jpegdata"
class FakeSession(requests.Session):
def __init__(self, proxies: dict | None = None) -> None:
super().__init__()
self.proxies = proxies or {}
self.got: list[str] = []
def get(self, url, **kwargs): # type: ignore[override]
self.got.append(url)
return FakeResponse()
def test_url_attachment_defers_download() -> None:
session = FakeSession()
attachment = Attachment.from_url(URL, name="thumbnail", session=session)
assert attachment.path is None
assert not session.got
assert attachment.mime_type == "image/jpeg"
def test_download_without_flag_uses_proxied_session(tmp_path, monkeypatch) -> None:
monkeypatch.setattr(config.directories, "temp", tmp_path)
session = FakeSession(proxies={"https": "http://proxy:8080"})
attachment = Attachment.from_url(URL, name="thumbnail", session=session)
attachment.download(session)
assert session.got == [URL]
assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata"
def test_download_with_flag_bypasses_proxy(tmp_path, monkeypatch) -> None:
monkeypatch.setattr(config.directories, "temp", tmp_path)
direct = FakeSession()
monkeypatch.setattr(track_module, "direct_session", lambda session: direct)
proxied = FakeSession(proxies={"https": "http://proxy:8080"})
attachment = Attachment.from_url(URL, name="thumbnail", session=proxied)
attachment.download(proxied, no_proxy_download=True)
assert not proxied.got
assert direct.got == [URL]
assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata"
@@ -0,0 +1,86 @@
"""Self-check for ``has_encrypted_sample_entry``: a finalized MP4 whose moov still carries
an encv/enca sample entry means the decrypt tool skipped/failed the track (frma was never
applied), so the detector must flag it before the file reaches the muxer. Detection is
scoped to moov, so stray 4CC byte runs in mdat must never trip (or mask) it.
Run: uv run pytest tests/tracks/test_encrypted_sample_entry.py
"""
from __future__ import annotations
import struct
from pathlib import Path
from unshackle.core.tracks.track import has_encrypted_sample_entry
def _box(fmt: bytes, payload: bytes = b"") -> bytes:
return struct.pack(">I", 8 + len(payload)) + fmt + payload
def _sample_entry(fmt: bytes, payload: bytes = b"") -> bytes:
body = b"\x00" * 6 + struct.pack(">H", 1) + payload # reserved + data_reference_index
return struct.pack(">I", 8 + len(body)) + fmt + body
def _stsd(entries: list[bytes]) -> bytes:
body = b"\x00" * 4 + struct.pack(">I", len(entries)) + b"".join(entries) # version/flags + count
return _box(b"stsd", body)
def _moov(*children: bytes) -> bytes:
stbl = _box(b"stbl", b"".join(children))
return _box(b"moov", _box(b"trak", _box(b"mdia", _box(b"minf", stbl))))
FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2")
# stray [size][encv] byte run as compressed A/V payload could randomly contain
STRAY_ENCV = b"\xde\xad" * 4 + struct.pack(">I", 100) + b"encv" + b"\xbe\xef" * 8
def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path:
f = tmp_path / name
f.write_bytes(b"".join(boxes))
return f
def test_dual_stsd_encv_and_avc1_detected(tmp_path: Path) -> None:
sinf = _box(b"sinf", _box(b"frma", b"avc1"))
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
encv = _sample_entry(b"encv", b"\x00" * 70 + sinf)
f = _write(tmp_path, "dual.mp4", FTYP, _moov(_stsd([avc1, encv])), _box(b"mdat", b"\x00" * 32))
assert has_encrypted_sample_entry(f) is True
def test_single_avc1_not_detected(tmp_path: Path) -> None:
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
f = _write(tmp_path, "clear.mp4", FTYP, _moov(_stsd([avc1])), _box(b"mdat", b"\x00" * 32))
assert has_encrypted_sample_entry(f) is False
def test_stray_encv_in_mdat_not_detected(tmp_path: Path) -> None:
# clean moov, mdat happens to contain a plausible [size][encv] run -> must NOT flag
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
f = _write(tmp_path, "stray.mp4", FTYP, _moov(_stsd([avc1])), _box(b"mdat", STRAY_ENCV))
assert has_encrypted_sample_entry(f) is False
def test_leading_mdat_does_not_mask_real_encv(tmp_path: Path) -> None:
# progressive layout (mdat before moov) with a stray run must not hide a genuine encv
enca = _sample_entry(b"enca", b"\x00" * 20 + _box(b"sinf", _box(b"frma", b"mp4a")))
f = _write(tmp_path, "masked.mp4", FTYP, _box(b"mdat", STRAY_ENCV), _moov(_stsd([enca])))
assert has_encrypted_sample_entry(f) is True
def test_text_containing_enca_in_moov_not_detected(tmp_path: Path) -> None:
# "enca" inside a metadata string has no plausible box size before it -> must NOT flag
avc1 = _sample_entry(b"avc1", b"\x00" * 70)
udta = _box(b"udta", b"data is encapsulated here")
f = _write(tmp_path, "text.mp4", FTYP, _box(b"moov", _stsd([avc1]) + udta))
assert has_encrypted_sample_entry(f) is False
def test_missing_or_bogus_file(tmp_path: Path) -> None:
assert has_encrypted_sample_entry(tmp_path / "nope.mp4") is False
bogus = _write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4")
assert has_encrypted_sample_entry(bogus) is False
+167
View File
@@ -0,0 +1,167 @@
"""Self-check for ``assert_fragments_decrypted``: after decryption every moof must have lost
its standard senc. A survivor means the tool silently skipped that fragment (the
tfhd.sample_description_index ceiling bug), which the moov-scoped has_encrypted_sample_entry
structurally cannot see. A leftover PIFF uuid survives a good decrypt too, because content
shipping both boxes keeps it: a decrypter detaches only the atom it consumed.
The guard never reads mdat, so stray byte runs there leave it unmoved.
Run: uv run pytest tests/tracks/test_fragment_decrypted.py
"""
from __future__ import annotations
import struct
from pathlib import Path
import pytest
from unshackle.core.manifests.ism_init import PIFF_SENC_UUID
from unshackle.core.tracks.track import assert_fragments_decrypted
def _box(fmt: bytes, payload: bytes = b"") -> bytes:
return struct.pack(">I", 8 + len(payload)) + fmt + payload
def _senc(sample_count: int = 4) -> bytes:
return _box(b"senc", b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count))
def _piff_senc(sample_count: int = 4) -> bytes:
body = PIFF_SENC_UUID + b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)
return _box(b"uuid", body)
def _moof(*children: bytes, sdi: int = 1) -> bytes:
tfhd = _box(b"tfhd", b"\x00\x00\x00\x02" + struct.pack(">II", 1, sdi)) # sample-description-index-present
return _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _box(b"traf", tfhd + b"".join(children)))
FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2")
MOOV = _box(b"moov", _box(b"mvex", _box(b"trex", b"\x00" * 24)))
MDAT = _box(b"mdat", b"\x00" * 64)
# a compressed payload can contain anything, including plausible-looking box runs
STRAY = _box(b"mdat", struct.pack(">I", 64) + b"senc" + b"\xbe\xef" * 8 + PIFF_SENC_UUID)
def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path:
f = tmp_path / name
f.write_bytes(b"".join(boxes))
return f
def test_all_fragments_decrypted_passes(tmp_path: Path) -> None:
f = _write(tmp_path, "clean.mp4", FTYP, MOOV, _moof(), MDAT, _moof(sdi=2), MDAT)
assert_fragments_decrypted(f)
def test_surviving_senc_raises(tmp_path: Path) -> None:
# the real bug: fragment 1 (index 1) decrypted, the rest (index 2) silently skipped
boxes = [FTYP, MOOV, _moof(), MDAT]
for _ in range(3):
boxes += [_moof(_senc(), sdi=2), MDAT]
f = _write(tmp_path, "skipped.mp4", *boxes)
with pytest.raises(ValueError, match=r"3/4 fragment"):
assert_fragments_decrypted(f)
def test_surviving_cenc_senc_raises(tmp_path: Path) -> None:
f = _write(tmp_path, "cenc.mp4", FTYP, MOOV, _moof(_senc()), MDAT)
with pytest.raises(ValueError, match=r"1/1 fragment"):
assert_fragments_decrypted(f)
def test_leftover_piff_uuid_after_senc_removed_does_not_raise(tmp_path: Path) -> None:
# live DASH shape: the traf shipped senc+saiz+saio plus a duplicate PIFF uuid. mp4decrypt
# consumed senc, detached senc/saiz/saio, and left the uuid on a fully decrypted fragment.
boxes = [FTYP, MOOV]
for _ in range(3):
boxes += [_moof(_piff_senc()), MDAT]
f = _write(tmp_path, "piff_leftover.mp4", *boxes)
assert_fragments_decrypted(f)
def test_stray_senc_bytes_in_mdat_ignored(tmp_path: Path) -> None:
f = _write(tmp_path, "stray.mp4", FTYP, MOOV, _moof(), STRAY, _moof(sdi=2), STRAY)
assert_fragments_decrypted(f)
def test_senc_text_without_plausible_size_ignored(tmp_path: Path) -> None:
# "senc" inside a free-box string has no valid box size before it
f = _write(tmp_path, "text.mp4", FTYP, MOOV, _moof(_box(b"free", b"presence sencoded here")), MDAT)
assert_fragments_decrypted(f)
def test_senc_bytes_inside_trun_payload_ignored(tmp_path: Path) -> None:
# a trun sample entry spelling "senc" after a size-looking duration; only a byte scan bites
trun = _box(b"trun", b"\x00\x00\x03\x01" + struct.pack(">I", 2) + struct.pack(">I", 512) + b"senc")
f = _write(tmp_path, "trun.mp4", FTYP, MOOV, _moof(trun), MDAT)
assert_fragments_decrypted(f)
def test_empty_senc_on_clear_fragment_ignored(tmp_path: Path) -> None:
# sample_count 0 marks a fragment with no protected samples, so the guard ignores it
f = _write(tmp_path, "empty.mp4", FTYP, MOOV, _moof(_senc(sample_count=0)), MDAT)
assert_fragments_decrypted(f)
def _senc_override(sample_count: int) -> bytes:
# flags&1 => AlgorithmID(3) + IV_size(1) + KID(16) precede sample_count
head = b"\x00\x00\x00\x01" + b"\x00\x00\x08" + b"\x08" + b"\xaa" * 16
return _box(b"senc", head + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count))
def test_override_flag_shifts_sample_count_offset(tmp_path: Path) -> None:
clear = _write(tmp_path, "ovr_clear.mp4", FTYP, MOOV, _moof(_senc_override(0)), MDAT)
assert_fragments_decrypted(clear)
f = _write(tmp_path, "ovr.mp4", FTYP, MOOV, _moof(_senc_override(5)), MDAT)
with pytest.raises(ValueError, match=r"1/1 fragment"):
assert_fragments_decrypted(f)
def test_senc_outside_traf_ignored(tmp_path: Path) -> None:
# a decrypted moof whose only "senc" sits in a sibling box outside any traf
f = _write(tmp_path, "sibling.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _senc()), MDAT)
assert_fragments_decrypted(f)
def test_senc_nested_under_non_traf_ignored(tmp_path: Path) -> None:
# senc two levels down under a non-traf container: only traf children count
meta = _box(b"meta", _senc())
f = _write(tmp_path, "nested.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + meta), MDAT)
assert_fragments_decrypted(f)
def test_undersized_largesize_box_does_not_desync_walk(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
# size==1 reads a 16-byte header, so a largesize of 12 clears the 8-byte floor but
# still stalls the cursor onto payload; only a header-aware floor catches it
runt = struct.pack(">I", 1) + b"free" + struct.pack(">Q", 12)
f = _write(tmp_path, "runt.mp4", FTYP, MOOV, runt, _moof(_senc()), MDAT, _moof(_senc()), MDAT)
with caplog.at_level("WARNING"):
assert_fragments_decrypted(f)
assert "malformed box size" in caplog.text
def test_largesize_and_zero_size_boxes_walked(tmp_path: Path) -> None:
frag = _moof(_senc())
large = struct.pack(">I", 1) + b"mdat" + struct.pack(">Q", 16 + 64) + b"\x00" * 64
trailing = struct.pack(">I", 0) + b"mdat" + b"\x00" * 64 # size 0 => runs to EOF
f = _write(tmp_path, "large.mp4", FTYP, MOOV, frag, large, trailing)
with pytest.raises(ValueError, match=r"1/1 fragment"):
assert_fragments_decrypted(f)
def test_malformed_box_size_warns_and_does_not_raise(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
# failing open is deliberate, but the abort must be logged so it cannot pass as clean
overrun = struct.pack(">I", 1 << 30) + b"free" + b"\x00" * 8
f = _write(tmp_path, "overrun.mp4", FTYP, MOOV, overrun, _moof(_senc()), MDAT)
with caplog.at_level("WARNING"):
assert_fragments_decrypted(f)
assert "malformed box size" in caplog.text
def test_missing_or_bogus_file_is_noop(tmp_path: Path) -> None:
assert_fragments_decrypted(tmp_path / "nope.mp4")
assert_fragments_decrypted(_write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4"))
truncated = _write(tmp_path, "cut.mp4", FTYP, MOOV, _moof(_senc())[:20])
assert_fragments_decrypted(truncated)
+159 -2
View File
@@ -3,8 +3,8 @@ import pytest
from unshackle.core.tracks import Subtitle, Tracks
def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) -> Subtitle:
return Subtitle(
def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False, orig: bool = False) -> Subtitle:
track = Subtitle(
url=f"https://example.test/{lang}-{forced}-{sdh}-{cc}",
language=lang,
codec=Subtitle.Codec.SubRip,
@@ -12,6 +12,8 @@ def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) ->
sdh=sdh,
cc=cc,
)
track.is_original_lang = orig
return track
def shape(tracks: Tracks) -> list[tuple[str, str]]:
@@ -66,3 +68,158 @@ def test_cc_counts_as_sdh() -> None:
t.add([sub("en", cc=True), sub("en")])
t.sort_subtitles(by_language=["en"], type_priority=["normal", "sdh"])
assert shape(t) == [("en", "normal"), ("en", "sdh")]
@pytest.fixture
def unprioritised() -> Tracks:
"""Languages that no by_language argument floats to the top."""
t = Tracks()
t.add([sub("fi", sdh=True), sub("fi"), sub("de", sdh=True), sub("de")])
return t
def test_type_grouping_separates_a_language_from_its_sdh(unprioritised: Tracks) -> None:
unprioritised.sort_subtitles()
assert shape(unprioritised) == [
("de", "normal"),
("fi", "normal"),
("de", "sdh"),
("fi", "sdh"),
]
def test_language_grouping_keeps_a_language_with_its_sdh(unprioritised: Tracks) -> None:
unprioritised.sort_subtitles(group_by="language")
assert shape(unprioritised) == [
("de", "normal"),
("de", "sdh"),
("fi", "normal"),
("fi", "sdh"),
]
def test_language_grouping_still_honours_type_priority_and_sections(tracks: Tracks) -> None:
# de sits outside by_language, where type-major sorting would interleave it with en
tracks.add([sub("de"), sub("de", sdh=True)])
tracks.sort_subtitles(by_language=["it"], type_priority=["normal", "sdh", "forced"], group_by="language")
assert shape(tracks) == [
("it", "normal"),
("it", "forced"),
("de", "normal"),
("de", "sdh"),
("en", "normal"),
("en", "sdh"),
("en", "forced"),
]
def test_language_grouping_keeps_regional_variants_adjacent_through_sections() -> None:
# "en" floats en-GB and en-US as one block, so each exact tag keeps its variants
t = Tracks()
t.add(
[
sub("en-US", sdh=True),
sub("fi"),
sub("en-GB"),
sub("en-US"),
sub("fi", sdh=True),
sub("en-GB", forced=True),
sub("en-GB", sdh=True),
]
)
t.sort_subtitles(by_language=["en"], group_by="language")
assert shape(t) == [
("en-GB", "forced"),
("en-GB", "normal"),
("en-GB", "sdh"),
("en-US", "normal"),
("en-US", "sdh"),
("fi", "normal"),
("fi", "sdh"),
]
def langs(tracks: Tracks) -> list[str]:
return [str(s.language) for s in tracks.subtitles]
@pytest.fixture
def many() -> Tracks:
t = Tracks()
t.add([sub(lang, orig=lang == "ja") for lang in ("ar", "bg", "en", "es", "fr", "ja")])
return t
def test_priority_orders_listed_then_rest_alphabetically(many: Tracks) -> None:
many.sort_subtitles(by_language=["en", "es", "fr"], group_by="language")
assert langs(many) == ["en", "es", "fr", "ar", "bg", "ja"]
def test_priority_outranks_the_original_language(many: Tracks) -> None:
# "all" resolves to the original language (ja), which must stay below the named ones
many.sort_subtitles(by_language=["en", "es", "all"], group_by="language")
assert langs(many) == ["en", "es", "ja", "ar", "bg", "fr"]
def test_priority_matches_regional_variants() -> None:
t = Tracks()
t.add([sub("ar"), sub("en-US"), sub("es-419"), sub("ja")])
t.sort_subtitles(by_language=["en", "es"], group_by="language")
assert langs(t) == ["en-US", "es-419", "ar", "ja"]
def test_priority_overlapping_entries_are_harmless() -> None:
t = Tracks()
t.add([sub("ar"), sub("en-GB"), sub("en-US"), sub("es"), sub("ja")])
t.sort_subtitles(by_language=["en", "en-US", "es"], group_by="language")
assert langs(t) == ["en-GB", "en-US", "es", "ar", "ja"]
def test_no_priority_leaves_order_unchanged(many: Tracks) -> None:
many.sort_subtitles(by_language=[], group_by="language")
assert langs(many) == ["ar", "bg", "en", "es", "fr", "ja"]
@pytest.mark.parametrize("token", ["all", "best"])
def test_magic_token_alone_floats_the_original_language(many: Tracks, token: str) -> None:
many.sort_subtitles(by_language=[token], group_by="language")
assert langs(many) == ["ja", "ar", "bg", "en", "es", "fr"]
@pytest.mark.parametrize("token", ["all", "best"])
def test_priority_outranks_either_magic_token(many: Tracks, token: str) -> None:
many.sort_subtitles(by_language=["en", "es", token], group_by="language")
assert langs(many) == ["en", "es", "ja", "ar", "bg", "fr"]
@pytest.fixture
def regionals() -> Tracks:
t = Tracks()
t.add([sub("ar"), sub("en"), sub("en-GB"), sub("en-US"), sub("ja")])
return t
def test_fuzzy_priority_floats_regional_variants(regionals: Tracks) -> None:
regionals.sort_subtitles(by_language=["en"], group_by="language")
assert langs(regionals) == ["en", "en-GB", "en-US", "ar", "ja"]
def test_exact_priority_floats_only_the_named_tag(regionals: Tracks) -> None:
# CLDR rates en/en-US at distance 0, so exact mode needs the RFC 4647 string preference
# to hold en-US back; without it en-US floats and en-GB does not.
regionals.sort_subtitles(by_language=["en"], group_by="language", exact_match=True)
assert langs(regionals) == ["en", "ar", "en-GB", "en-US", "ja"]
def test_exact_priority_still_falls_back_when_no_tag_is_string_equal() -> None:
t = Tracks()
t.add([sub("ar"), sub("cmn"), sub("ja")])
t.sort_subtitles(by_language=["zh"], group_by="language", exact_match=True)
assert langs(t) == ["cmn", "ar", "ja"]
@pytest.mark.parametrize("exact", [False, True])
def test_unparseable_priority_entry_is_ignored(many: Tracks, exact: bool) -> None:
# a config typo in language_priority must not crash the sort
many.sort_subtitles(by_language=["en", "engrish", "f", ""], group_by="language", exact_match=exact)
assert langs(many) == ["en", "ar", "bg", "es", "fr", "ja"]
+22
View File
@@ -0,0 +1,22 @@
from unshackle.core.tracks.subtitle import Subtitle
def test_strips_timestamp_map_keeps_cues():
text = "WEBVTT\nX-TIMESTAMP-MAP=MPEGTS:900000,LOCAL:00:00:00.000\n\n00:00:22.690 --> 00:00:24.710\nHi\n"
out = Subtitle.strip_webvtt_timestamp_map(text)
assert "X-TIMESTAMP-MAP" not in out
assert "00:00:22.690 --> 00:00:24.710" in out
assert out.startswith("WEBVTT\n")
def test_strips_lowercase_timestamp_map():
# SubtitleEdit honours the header case-insensitively, so the strip must too
text = "WEBVTT\nx-timestamp-map=MPEGTS:900000,LOCAL:00:00:00.000\n\n00:00:22.690 --> 00:00:24.710\nHi\n"
out = Subtitle.strip_webvtt_timestamp_map(text)
assert "x-timestamp-map" not in out.lower()
assert "00:00:22.690 --> 00:00:24.710" in out
def test_no_timestamp_map_is_untouched():
text = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nHi\n"
assert Subtitle.strip_webvtt_timestamp_map(text) == text