Merge branch 'dev' into feat/downloader-speed

Catch the branch up with dev (45 commits) so testers can run the perf work
against current dev.

Conflict resolutions:
- downloaders/requests.py: keep the branch's racing-gated read1, adaptive
  controller speed reporting and byte-range slice guards; take dev's
  content-encoding checks (raw reads are skipped on an encoded body) and its
  rolling SpeedWindow for the no-controller speed fallback. The single-URL
  ranged-parallel gate now requires no speed limit and no caller Range.
- service.py / session.py: keep the branch's TimeoutHTTPAdapter and retry
  config, raise POOL_MAX_SIZE to 64 so dev's pool-starvation fix survives
  (pool_block=True stalls threads on a pool smaller than the in-flight count).
- docs: dev split configuration.md into per-area pages, so the branch's rnet
  network option rows moved to reference/configuration/network.md.
This commit is contained in:
imSp4rky
2026-08-07 10:45:47 -06:00
104 changed files with 8567 additions and 1375 deletions
+178
View File
@@ -0,0 +1,178 @@
"""Tests for the HLS AES-128 ClearKey DRM system (``unshackle.core.drm.clearkey``).
Covers the correctness guards on the whole-segment AES-128-CBC path:
- ``ClearKey.decrypt`` passing mislabeled-clear / non-block-aligned segments through
untouched instead of crashing or corrupting (WS1 / F1+F3)
- ``ClearKey.from_m3u_key`` ``data:`` URI parsing and 16-byte key validation (WS5 / F4)
- the RFC 8216 §5.2 sequence-number IV deriving from a segment's absolute media
sequence number rather than the post-filter download index (WS2 / F2)
"""
from __future__ import annotations
import base64
import m3u8
import pytest
from Cryptodome.Cipher import AES
from Cryptodome.Util.Padding import pad
from m3u8.model import Key
from unshackle.core.drm.clearkey import SYNC_BYTE, TS_PACKET_SIZE, ClearKey
KEY = bytes.fromhex("00112233445566778899aabbccddeeff")
IV = bytes.fromhex("0f0e0d0c0b0a09080706050403020100")
def encrypt(plaintext: bytes, key: bytes = KEY, iv: bytes = IV) -> bytes:
return AES.new(key, AES.MODE_CBC, iv).encrypt(pad(plaintext, AES.block_size))
def clear_ts(packets: int = 4) -> bytes:
# Plaintext MPEG-TS: sync byte 0x47 at every 188-byte packet boundary. Default 4 packets so
# the fixture satisfies the hardened 4-boundary sniff (and 752 B is 16-aligned, so it must be
# caught by the clear-TS guard, not the len%16 guard).
return b"".join(bytes([SYNC_BYTE]) + b"\x00" * (TS_PACKET_SIZE - 1) for _ in range(packets))
def test_valid_encrypted_segment_decrypts(tmp_path) -> None:
plaintext = b"the quick brown fox jumps over" * 4
seg = tmp_path / "0.ts"
seg.write_bytes(encrypt(plaintext))
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.read_bytes() == plaintext
def test_clear_ts_passes_through(tmp_path) -> None:
data = clear_ts()
seg = tmp_path / "0.ts"
seg.write_bytes(data)
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.read_bytes() == data
def test_non_block_aligned_passes_through(tmp_path) -> None:
data = b"\x01" * 17 # not TS, and not a 16-byte multiple -> not valid CBC ciphertext
assert len(data) % AES.block_size != 0
seg = tmp_path / "0.ts"
seg.write_bytes(data)
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.read_bytes() == data
def test_empty_segment_does_not_crash(tmp_path) -> None:
seg = tmp_path / "0.ts"
seg.write_bytes(b"")
ClearKey(key=KEY, iv=IV).decrypt(seg)
assert seg.exists()
assert seg.read_bytes() == b""
def test_warns_once_per_instance(tmp_path, caplog) -> None:
drm = ClearKey(key=KEY, iv=IV)
for name in ("0.ts", "1.ts"):
seg = tmp_path / name
seg.write_bytes(clear_ts())
with caplog.at_level("WARNING"):
drm.decrypt(seg)
warnings = [r for r in caplog.records if r.levelname == "WARNING"]
assert len(warnings) == 1
def _data_key(raw: bytes, base64_encoded: bool = True) -> Key:
if base64_encoded:
uri = "data:text/plain;base64," + base64.b64encode(raw).decode("ascii")
else:
uri = "data:text/plain," + raw.decode("latin-1")
return Key(method="AES-128", uri=uri, base_uri="", keyformat=None, keyformatversions=None, iv=None)
def test_data_uri_key_decodes_and_validates_length() -> None:
drm = ClearKey.from_m3u_key(_data_key(KEY))
assert drm.key == KEY
def test_data_uri_key_wrong_length_rejected() -> None:
with pytest.raises(ValueError, match="Unexpected Length"):
ClearKey.from_m3u_key(_data_key(b"\x00" * 8))
def test_raw_data_uri_key_materialises_and_validates() -> None:
# A raw (non-base64) data: payload must be taken as literal key bytes, not hex-parsed.
drm = ClearKey.from_m3u_key(_data_key(KEY, base64_encoded=False))
assert drm.key == KEY
def test_raw_data_uri_key_wrong_length_rejected() -> None:
with pytest.raises(ValueError, match="Unexpected Length"):
ClearKey.from_m3u_key(_data_key(b"\x01" * 8, base64_encoded=False))
def test_data_uri_split_tolerates_commas_in_payload() -> None:
# base64 with padding contains no comma, but the split must only break on the first comma
# so a payload that itself contains a comma is not truncated. Encode a 16-byte key whose
# base64 happens to be split-safe, and assert the mediatype comma is the only split point.
raw = bytes(range(16))
b64 = base64.b64encode(raw).decode("ascii")
key = Key(
method="AES-128",
uri=f"data:application/octet-stream;base64,{b64}",
base_uri="",
keyformat=None,
keyformatversions=None,
iv=None,
)
drm = ClearKey.from_m3u_key(key)
assert drm.key == raw
SEQ_PLAYLIST = """#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:6
#EXT-X-MEDIA-SEQUENCE:100
#EXT-X-KEY:METHOD=AES-128,URI="skd://key"
#EXTINF:6.0,
seg0.ts
#EXTINF:6.0,
seg1.ts
#EXTINF:6.0,
seg2.ts
#EXTINF:6.0,
seg3.ts
#EXT-X-ENDLIST
"""
def test_sequence_number_iv_uses_absolute_media_sequence() -> None:
"""IV must be the segment's absolute media sequence number, not the download index.
Mirrors the derivation in ``HLS.download_track``: filtered segments are dropped, the
surviving segments are named by post-filter download index, and the IV comes from the
wanted segment's ``media_sequence`` (EXT-X-MEDIA-SEQUENCE + absolute position).
"""
master = m3u8.loads(SEQ_PLAYLIST)
# media sequence numbers are the offset (100) plus absolute playlist position.
assert [s.media_sequence for s in master.segments] == [100, 101, 102, 103]
# Drop an early segment (index 1), as an OnSegmentFilter would.
unwanted = {master.segments[1]}
wanted_segments = [s for s in master.segments if s not in unwanted]
# download index 1 is the SECOND surviving segment (absolute seq 102), not 101.
download_i = 1
seq = wanted_segments[download_i].media_sequence
assert seq == 102
iv = int(seq).to_bytes(16, "big")
assert iv == (102).to_bytes(16, "big")
# the old download-index derivation would have produced the wrong IV (100 + 1 = 101)
assert iv != (100 + download_i).to_bytes(16, "big")
+75 -1
View File
@@ -3,10 +3,11 @@ services set via the ``sub_format`` override (must not be rejected as an invalid
from __future__ import annotations
import click
import pytest
from unshackle.core.tracks.subtitle import Subtitle
from unshackle.core.utils.click_types import QUALITY_LIST, SLOW_DELAY_RANGE, SubtitleCodecChoice
from unshackle.core.utils.click_types import QUALITY_LIST, SLOW_DELAY_RANGE, SeasonRange, SubtitleCodecChoice
choice = SubtitleCodecChoice(Subtitle.Codec)
@@ -57,3 +58,76 @@ def test_quality_list_accepts_yaml_native_values(value, expected):
)
def test_slow_delay_range_accepts_bool(value, expected):
assert SLOW_DELAY_RANGE.convert(value, None, None) == expected
# --- SeasonRange ------------------------------------------------------------
def parse(*tokens: str) -> list[str]:
return SeasonRange().parse_tokens(*tokens)
def test_single_episode_token_is_unchanged():
assert parse("S01E01") == ["1x1"]
def test_bare_season_spans_the_whole_episode_range():
"""A bare S01 reads the one match object with a different default per side."""
keys = parse("S01")
assert len(keys) == 1000
assert "1x0" in keys and "1x999" in keys
def test_multi_token_range_is_unchanged():
assert sorted(parse("S02E01", "S02E03-S02E05")) == ["2x1", "2x3", "2x4", "2x5"]
def test_exclusions_are_unchanged():
keys = parse("S01-S05", "-S03", "-S02E01")
assert not any(k.startswith("3x") for k in keys)
assert "2x1" not in keys
assert "2x0" in keys and "2x2" in keys
assert len(keys) == 3999 # 5 seasons less all of S03 less 2x1
@pytest.mark.parametrize("tokens", [("S01E01",), ("S01",), ("S01-S05", "-S03", "-S02E01"), ("S02E03-S02E05",)])
def test_part_less_tokens_never_emit_a_part_or_exclusion_key(tokens):
assert all("." not in k and "!" not in k for k in parse(*tokens))
def test_part_token():
assert parse("S01E01.2") == ["1x1.2"]
def test_part_range_within_one_episode():
assert sorted(parse("S01E01.1-S01E01.3")) == ["1x1.1", "1x1.2", "1x1.3"]
def test_part_exclusion_becomes_a_negative_key():
keys = parse("S01", "-S01E01.2")
assert "!1x1.2" in keys
assert "1x1" in keys # base key stays so the other parts still match
def test_base_exclusion_covers_part_keys():
assert parse("S01E01.1-S01E01.3", "S01E02", "-S01E01") == ["1x2"]
def test_exclusion_removes_duplicate_keys():
# "S01" and "S01E01" both emit 1x1; the exclusion must remove every copy
assert "1x1" not in parse("S01", "S01E01", "-S01E01")
@pytest.mark.parametrize(
"token",
[
"S01E01.0", # parts count from 1
"S01E01.1-S01E02.2", # a part range cannot cross episodes
"S01-S01E01.2", # a part on only one side
"S01E01.3-S01E01.1", # reversed part range
"S01E01.200", # above MAX_PART
],
)
def test_bad_part_tokens_fail(token):
with pytest.raises(click.UsageError):
parse(token)
+74
View File
@@ -0,0 +1,74 @@
"""Unit tests for the code fingerprint behind the banner and the /api/health `code_hash` field.
It runs at import time, so it must always return a string and must never raise. It must change when
the framework source changes and stay unchanged when anything else does."""
from __future__ import annotations
from pathlib import Path
import pytest
import unshackle.core
from unshackle.core import __code_hash__, code_files, code_hash
pytestmark = pytest.mark.unit
def test_is_hex_and_stable() -> None:
first = code_hash()
assert len(first) == 40
assert int(first, 16) >= 0
assert first == code_hash()
def test_exported_value_is_the_short_form() -> None:
assert __code_hash__ == code_hash()[:7]
def test_covers_the_framework_dirs_only() -> None:
rels = code_files()
assert "__main__.py" in rels
assert any(rel.startswith("core/") for rel in rels)
assert not any(rel.startswith("services/") for rel in rels)
assert not any("__pycache__" in rel for rel in rels)
assert all(rel.endswith(".py") for rel in rels)
assert rels == sorted(rels) # order must not depend on the filesystem
def test_content_changes_the_hash() -> None:
rels = ["core/__init__.py", "commands/dl.py"]
blobs = {"core/__init__.py": b"a", "commands/dl.py": b"b"}
before = code_hash(files=rels, read=blobs.__getitem__)
blobs["commands/dl.py"] = b"b "
assert code_hash(files=rels, read=blobs.__getitem__) != before
def test_path_is_part_of_the_hash() -> None:
"""A rename with identical bytes must not look like the same code."""
same_bytes = {"core/a.py": b"x", "core/b.py": b"x"}
assert code_hash(files=["core/a.py"], read=same_bytes.__getitem__) != code_hash(
files=["core/b.py"], read=same_bytes.__getitem__
)
def test_file_order_does_not_matter_to_the_caller() -> None:
"""code_files() sorts, so two orderings of the same set must agree once sorted."""
blobs = {"core/a.py": b"x", "core/b.py": b"y"}
assert code_hash(files=sorted(blobs), read=blobs.__getitem__) == code_hash(
files=sorted(reversed(list(blobs))), read=blobs.__getitem__
)
def test_unreadable_source_yields_empty_string() -> None:
def unreadable(rel: str) -> bytes:
raise PermissionError(rel)
assert code_hash(files=["core/__init__.py"], read=unreadable) == ""
def test_unwalkable_source_yields_empty_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A dir os.walk cannot enter must yield "", not a valid-looking hash of fewer files."""
(tmp_path / "__main__.py").write_bytes(b"")
monkeypatch.setattr(unshackle.core, "_PKG", tmp_path)
assert code_hash() == ""
+120
View File
@@ -0,0 +1,120 @@
"""Tests for {dual}/{multi}/{dubbed} filename variables and dual_multi_mode."""
from __future__ import annotations
import types
import pytest
from unshackle.core.config import config
from unshackle.core.titles.movie import Movie
class DummyService:
pass
def make_audio(language: str) -> types.SimpleNamespace:
"""Audio track stub carrying just the attributes the context builder reads."""
return types.SimpleNamespace(
language=language,
format="AAC",
channel_layout=None,
channellayout_original=None,
channel_s=2,
channels=2,
format_additionalfeatures=None,
joc=None,
)
def make_media_info(languages: list[str]) -> types.SimpleNamespace:
return types.SimpleNamespace(video_tracks=[], audio_tracks=[make_audio(lang) for lang in languages])
def make_movie(language) -> Movie:
return Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language=language)
@pytest.fixture
def strict_mode(monkeypatch):
monkeypatch.setattr(config, "dual_multi_mode", "strict")
@pytest.fixture
def count_mode(monkeypatch):
monkeypatch.setattr(config, "dual_multi_mode", "count")
def ctx(language, languages) -> dict:
return make_movie(language)._build_base_template_context(make_media_info(languages))
def test_strict_dual_with_original(strict_mode):
c = ctx("en", ["en", "es"])
assert c["dual"] == "DUAL"
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_two_dubs_no_original(strict_mode):
c = ctx("en", ["es", "fr"])
assert c["dual"] == ""
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_regional_variants_collapse(strict_mode):
c = ctx("en", ["en-US", "en-GB"])
assert c["dual"] == ""
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_multi(strict_mode):
c = ctx("en", ["es", "fr", "de"])
assert c["multi"] == "MULTi"
assert c["dual"] == ""
assert c["dubbed"] == ""
def test_strict_two_unknown_original(strict_mode):
c = ctx(None, ["en", "es"])
assert c["dual"] == ""
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_strict_dubbed(strict_mode):
c = ctx("en", ["es"])
assert c["dubbed"] == "DUBBED"
assert c["dual"] == ""
assert c["multi"] == ""
def test_strict_single_unknown_original_no_dubbed(strict_mode):
c = ctx(None, ["es"])
assert c["dubbed"] == ""
assert c["dual"] == ""
assert c["multi"] == ""
def test_count_dual_ignores_original(count_mode):
c = ctx(None, ["en", "es"])
assert c["dual"] == "DUAL"
assert c["multi"] == ""
assert c["dubbed"] == ""
def test_count_multi(count_mode):
c = ctx(None, ["es", "fr", "de"])
assert c["multi"] == "MULTi"
assert c["dual"] == ""
assert c["dubbed"] == ""
def test_count_never_dubbed(count_mode):
c = ctx("en", ["es"])
assert c["dubbed"] == ""
assert c["dual"] == ""
assert c["multi"] == ""
+199
View File
@@ -0,0 +1,199 @@
"""Tests for multi-part episodes: Episode.part, its naming, sorting and -w matching."""
from __future__ import annotations
import pytest
from unshackle.core.config import config
from unshackle.core.titles.episode import Episode, Series
from unshackle.core.utils.click_types import SeasonRange
class DummyService:
pass
class StubMediaInfo:
"""Minimal MediaInfo stand-in: the template context only reads track lists."""
video_tracks: list = []
audio_tracks: list = []
def make_episode(**overrides) -> Episode:
kwargs = dict(
id_="ep-0001",
service=DummyService,
title="The Show",
season=1,
number=1,
name="The Reckoning",
)
kwargs.update(overrides)
return Episode(**kwargs)
@pytest.fixture
def reset_template():
saved = (config.output_template, config.folder_templates, config.folder_template, config.tag)
config.output_template = {"series": "{title}.{season_episode}.{quality?}.{source}-{tag}"}
config.folder_templates = {}
config.folder_template = ""
config.tag = "GROUP"
yield config
config.output_template, config.folder_templates, config.folder_template, config.tag = saved
# --- validation -------------------------------------------------------------
def test_part_defaults_to_none():
assert make_episode().part is None
def test_digit_string_part_coerces():
assert make_episode(part="2").part == 2
@pytest.mark.parametrize("bad", [0, -1])
def test_non_positive_part_rejected(bad):
with pytest.raises(ValueError):
make_episode(part=bad)
def test_non_int_part_rejected():
with pytest.raises(TypeError):
make_episode(part="x")
def test_bool_part_rejected():
"""bool is an int subclass; True would render as '.True' in keys and filenames."""
with pytest.raises(TypeError):
make_episode(part=True)
def test_old_cache_entry_reads_as_part_less():
"""jsonpickle restores by populating __dict__; an older entry has no 'part' key."""
restored = Episode.__new__(Episode)
restored.__dict__.update(title="The Show", season=1, number=1, name=None, year=None)
assert restored.part is None
# --- display ----------------------------------------------------------------
def test_str_is_unchanged_without_a_part():
without = make_episode()
assert str(without) == "The Show S01E01 The Reckoning"
def test_str_shows_the_selection_syntax():
assert str(make_episode(part=2)) == "The Show S01E01.2 The Reckoning"
def test_str_tells_dated_parts_apart():
# dated content renders the air date instead of SxxExx, so without the suffix both
# parts print the same string while their filenames differ
assert str(make_episode(part=2, air_date="2024-05-06")) == "The Show 2024.05.06.2 The Reckoning"
assert str(make_episode(air_date="2024-05-06")) == "The Show 2024.05.06 The Reckoning"
def test_tree_label_carries_the_part_on_dated_content():
series = Series([make_episode(part=1, air_date="2024-05-06"), make_episode(part=2, air_date="2024-05-06")])
labels = [node.label for node in series.tree(verbose=True).children[0].children]
assert labels == [
"[bold]2024.05.06.1.[/] [bright_black]The Reckoning",
"[bold]2024.05.06.2.[/] [bright_black]The Reckoning",
]
def test_tree_label_carries_the_part():
series = Series([make_episode(part=1), make_episode(part=2)])
labels = [node.label for node in series.tree(verbose=True).children[0].children]
assert labels == ["[bold]1.1.[/] [bright_black]The Reckoning", "[bold]1.2.[/] [bright_black]The Reckoning"]
def test_series_sorts_parts_within_their_episode():
series = Series(
[
make_episode(number=2),
make_episode(number=1, part=2),
make_episode(number=1),
make_episode(number=1, part=1),
]
)
assert [(e.number, e.part) for e in series] == [(1, None), (1, 1), (1, 2), (2, None)]
# --- naming -----------------------------------------------------------------
def test_filename_is_unchanged_without_a_part(reset_template):
assert make_episode().get_filename(StubMediaInfo()) == "The.Show.S01E01.DummyService-GROUP"
def test_dotted_template_folds_the_part_after_the_episode(reset_template):
name = make_episode(part=2).get_filename(StubMediaInfo())
assert name == "The.Show.S01E01.Part.2.DummyService-GROUP"
assert name.endswith("-GROUP") # group tag stays last
def test_spaced_template_uses_spaces(reset_template):
reset_template.output_template = {"series": "{title} {season_episode} {quality?} {source}-{tag}"}
assert make_episode(part=2).get_filename(StubMediaInfo()) == "The Show S01E01 Part 2 DummyService-GROUP"
def test_template_context_leaves_the_season_token_alone(reset_template):
ctx = make_episode(part=2)._build_template_context(StubMediaInfo())
assert ctx["season"] == "S01"
assert ctx["episode"] == "E01.Part.2"
assert ctx["season_episode"] == "S01E01.Part.2"
assert ctx["part"] == 2
def test_part_token_is_empty_without_a_part(reset_template):
assert make_episode()._build_template_context(StubMediaInfo())["part"] == ""
def test_parts_share_one_season_folder(reset_template):
# {season_episode} in a folder template would otherwise carry the part into the path
reset_template.folder_template = "{title}/{season_episode}"
folders = {make_episode(part=p).get_filename(StubMediaInfo(), folder=True) for p in (1, 2)}
assert folders == {"The.Show/S01E01"}
def test_parts_share_one_derived_season_folder(reset_template):
folders = {make_episode(part=p).get_filename(StubMediaInfo(), folder=True) for p in (1, 2)}
assert len(folders) == 1
assert folders == {make_episode().get_filename(StubMediaInfo(), folder=True)}
# --- matching ---------------------------------------------------------------
# the worked key scheme from the design: (token, part-less E1 selected, E1 part 2 selected)
MATCH_TABLE = [
(("S01",), True, True),
(("S01E01",), True, True),
(("S01E01.2",), False, True),
(("S01E01.1-S01E01.3",), False, True),
(("S01", "-S01E01.2"), True, False),
(("S01", "-S01E01"), False, False),
(("S01E02",), False, False),
]
@pytest.mark.parametrize("tokens,wants_plain,wants_part2", MATCH_TABLE)
def test_matches_wanted_table(tokens, wants_plain, wants_part2):
wanted = SeasonRange().parse_tokens(*tokens)
assert make_episode().matches_wanted(wanted) is wants_plain
assert make_episode(part=2).matches_wanted(wanted) is wants_part2
def test_a_part_exclusion_leaves_the_other_parts_alone():
wanted = SeasonRange().parse_tokens("S01", "-S01E01.2")
assert make_episode(part=1).matches_wanted(wanted) is True
assert make_episode(part=3).matches_wanted(wanted) is True
def test_asking_for_a_part_of_an_unsplit_episode_selects_nothing():
assert make_episode().matches_wanted(SeasonRange().parse_tokens("S01E01.2")) is False
+184
View File
@@ -0,0 +1,184 @@
"""Export/import round trips for the two manifest types that are not DASH.
``get_tracks`` splits on manifest type, and the two halves need opposite things. ISM takes the
re-parse path and needs ``track.data["ism"]["segments"]`` restored, because
``ISM.download_track`` reads it. HLS takes the rebuild-from-dicts path and needs nothing
restored, because ``HLS.download_track`` re-fetches the variant from ``track.url``. That makes
the stored per-track URL load-bearing for HLS in a way it is not for ISM.
These drive the real ``dl.write_export`` rather than hand-written export dicts, so a change to
the export format fails here instead of silently invalidating the fixtures.
"""
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from unshackle.commands.dl import dl
from unshackle.core.import_service import ImportService
from unshackle.core.manifests import HLS, ISM
ISM_URL = "https://cdn.example/x.ism/manifest"
ISM_MANIFEST = """<?xml version="1.0"?>
<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000">
<StreamIndex Type="video" Name="video" Language="en"
Url="QualityLevels({Bitrate})/Fragments(video={start_time})">
<QualityLevel Index="0" Bitrate="3000000" FourCC="H264" MaxWidth="1280" MaxHeight="720"
CodecPrivateData="00000001674D401F"/>
<c t="0" d="20000000" r="3"/>
</StreamIndex>
<StreamIndex Type="audio" Name="audio" Language="en"
Url="QualityLevels({Bitrate})/Fragments(audio={start_time})">
<QualityLevel Index="0" Bitrate="128000" AudioTag="255" SamplingRate="48000"
Channels="2" BitsPerSample="16" CodecPrivateData="1190"/>
<c t="0" d="20000000" r="3"/>
</StreamIndex>
</SmoothStreamingMedia>
"""
HLS_URL = "https://cdn.example/master.m3u8"
HLS_MANIFEST = """#EXTM3U
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",LANGUAGE="en",NAME="English",DEFAULT=YES,URI="audio_en.m3u8"
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="sub",LANGUAGE="en",NAME="English",URI="sub_en.m3u8"
#EXT-X-STREAM-INF:BANDWIDTH=3000000,RESOLUTION=1280x720,CODECS="avc1.4d401f,mp4a.40.2",AUDIO="aud",SUBTITLES="sub"
720.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=6000000,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud",SUBTITLES="sub"
1080.m3u8
"""
def export_tracks(tmp_path: Path, tracks, manifest_url: str | None) -> Path:
"""Write a real export sidecar for a parsed ladder, as a finished download would."""
from unshackle.core.titles import Movie
class StubService:
pass
title = Movie(id_="movie-1", service=StubService, name="Example Movie", year=2024, language="en")
for track in [*tracks.videos, *tracks.audio, *tracks.subtitles]:
title.tracks.add(track)
title.tracks.manifest_url = manifest_url
runner = dl.__new__(dl)
runner.service = "EXAMPLE"
export = tmp_path / "export.json"
for track in [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]:
runner.write_export(export, title, track)
return export
def import_tracks(export: Path):
svc = ImportService(SimpleNamespace(parent=None, params={}), "EXAMPLE", "movie-1", str(export))
return svc, svc.get_tracks(next(iter(svc.get_titles())))
@pytest.fixture
def ism_offline(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_MANIFEST, url))
@pytest.mark.parametrize("manifest_url", [ISM_URL, None], ids=["with-manifest-url", "without-manifest-url"])
def test_ism_round_trip_restores_segment_urls(tmp_path: Path, ism_offline: None, manifest_url: str | None) -> None:
"""ISM.download_track reads data["ism"]["segments"], which to_dict cannot carry across an export.
Without a manifest_url the export's own track URLs are the only way back to the manifest,
which is the case the re-parse branch alone never covered.
"""
original = ISM.from_text(ISM_MANIFEST, ISM_URL).to_tracks(language="en")
expected = {t.id: t.data["ism"]["segments"] for t in [*original.videos, *original.audio]}
_, tracks = import_tracks(export_tracks(tmp_path, original, manifest_url))
rebuilt = {t.id: t.data.get("ism", {}).get("segments") for t in [*tracks.videos, *tracks.audio]}
assert rebuilt == expected, "every ISM track must come back with the fragment URLs it was exported with"
ISM_ENCRYPTED = """<?xml version="1.0"?>
<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000">
<Protection><ProtectionHeader SystemID="{9A04F079-9840-4286-AB92-E65BE0885F95}">dGVzdA==</ProtectionHeader></Protection>
<StreamIndex Type="video" Name="video" Language="en" Url="QualityLevels({Bitrate})/Fragments(video={start_time})">
<QualityLevel Index="0" Bitrate="3000000" FourCC="H264" MaxWidth="1280" MaxHeight="720"
CodecPrivateData="00000001674D401F"/>
<QualityLevel Index="1" Bitrate="6000000" FourCC="H264" MaxWidth="1920" MaxHeight="1080"
CodecPrivateData="00000001674D401F"/>
<c t="0" d="20000000" r="3"/>
</StreamIndex>
</SmoothStreamingMedia>
"""
def test_encrypted_ism_rung_gets_the_exported_keys(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""resolve_server_keys must recognise an ISM manifest's protection, not only a DASH one.
ISM.download_track reads only ``track.drm``, so a rung that misses key injection downloads
encrypted and muxes without error. DASH survives the same omission because it re-derives its
DRM from the manifest elements at download time.
"""
monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_ENCRYPTED, url))
parsed = ISM.from_text(ISM_ENCRYPTED, ISM_URL).to_tracks(language="en")
# export the licensed rung only, as a finished download would
licensed = max(parsed.videos, key=lambda t: t.bitrate or 0)
export = export_tracks(tmp_path, parsed, ISM_URL)
doc = json.loads(export.read_text(encoding="utf8"))
doc["titles"]["movie-1"]["tracks"][licensed.id]["keys"] = {"0" * 32: "ab" * 16}
export.write_text(json.dumps(doc), encoding="utf8")
svc, tracks = import_tracks(export)
movie = next(iter(svc.get_titles()))
movie.tracks = tracks
svc.resolve_server_keys(movie)
other = min(tracks.videos, key=lambda t: t.bitrate or 0)
assert svc.track_is_encrypted(other), "an ISM ProtectionHeader means the rung is encrypted"
assert other.drm and other.drm[0].content_keys, "the unlicensed rung never received the exported key"
def test_hls_round_trip_keeps_every_variant_url(tmp_path: Path) -> None:
"""HLS re-fetches each variant from track.url at download time, so the stored URL has to survive."""
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
expected = {t.id: str(t.url) for t in [*original.videos, *original.audio, *original.subtitles]}
_, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL))
rebuilt = {t.id: str(t.url) for t in [*tracks.videos, *tracks.audio, *tracks.subtitles]}
assert rebuilt == expected, "the whole exported ladder must survive, each with its own variant URL"
assert {t.descriptor.name for t in tracks} == {"HLS"}
def test_hls_import_never_refetches_the_master_playlist(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A master playlist can hand out a fresh token per fetch, so import must not depend on it.
This is why HLS is absent from MANIFEST_DATA_KEYS. Re-fetching would add nothing and give a
long-expired export another way to fail.
"""
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
export = export_tracks(tmp_path, original, HLS_URL)
def explode(*args: object, **kwargs: object) -> None:
raise AssertionError("import re-fetched an HLS manifest")
monkeypatch.setattr(HLS, "from_url", explode)
monkeypatch.setattr(HLS, "from_text", explode)
_, tracks = import_tracks(export)
assert len(list(tracks)) == 4
def test_hls_tracks_tolerate_the_empty_manifest_data(tmp_path: Path) -> None:
"""HLS.download_track assigns into track.data["hls"] without creating it first.
Track.data is a defaultdict, so the assignment auto-vivifies rather than raising. Pin that,
because a plain dict here would turn every HLS import into a KeyError mid-download.
"""
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
_, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL))
track = tracks.videos[0]
assert not track.data.get("hls"), "to_dict cannot carry the m3u8 objects, and does not need to"
track.data["hls"]["segment_durations"] = [1, 2, 3] # hls.py, in download_track
assert track.data["hls"]["segment_durations"] == [1, 2, 3]
+226
View File
@@ -0,0 +1,226 @@
"""Tests for the manifest data an import must rebuild before a DASH track can download.
``Track.to_dict`` cannot serialise manifest XML, so a rebuilt DASH track carries no
``track.data["dash"]``. ``DASH.download_track`` reads that mapping unconditionally, so any
DASH track the import hands back without it raises on the first segment. Two exports reach
that state: one with no top-level ``manifest_url``, and one whose ``manifest_url`` covers
only part of the ladder (a separate MPD per codec).
"""
from __future__ import annotations
import json
from functools import partial
from pathlib import Path
from typing import Any, Optional
import click
import pytest
from unshackle.core.import_service import ImportService
from unshackle.core.manifests import DASH
MPD_TEMPLATE = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="{rep_id}" codecs="{codecs}" bandwidth="{bandwidth}" width="{width}" height="{height}">
<BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/>
</Representation>
</AdaptationSet>
</Period>
</MPD>
"""
AVC_URL = "https://example.invalid/avc/dash/m.mpd"
HEVC_URL = "https://example.invalid/hevc/dash/m.mpd"
MANIFESTS = {
AVC_URL: MPD_TEMPLATE.format(rep_id="v-avc", codecs="avc1.640028", bandwidth=5000000, width=1920, height=1080),
HEVC_URL: MPD_TEMPLATE.format(
rep_id="v-hevc", codecs="hvc1.2.4.L153.B0", bandwidth=6000000, width=3840, height=2160
),
}
def exported_video(track_id: str, url: str, codec: str, width: int, height: int) -> dict[str, Any]:
return {
"type": "Video",
"id": track_id,
"url": url,
"language": "en",
"descriptor": "DASH",
"codec": codec,
"bitrate": 5000000 if codec == "AVC" else 6000000,
"width": width,
"height": height,
}
def make_service(tmp_path: Path, manifest_url: Optional[str], tracks: dict[str, Any]) -> ImportService:
entry: dict[str, Any] = {
"meta": {"type": "movie", "id": "movie-1", "name": "Example Movie", "year": 2024, "language": "en"},
"manifest_type": "DASH",
"tracks": tracks,
}
if manifest_url:
entry["manifest_url"] = manifest_url
export = {"version": 2, "service": "EXAMPLE", "titles": {"movie-1": entry}}
path = tmp_path / "export.json"
path.write_text(json.dumps(export), encoding="utf8")
return ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path))
@pytest.fixture(autouse=True)
def local_manifests(monkeypatch: pytest.MonkeyPatch) -> None:
"""Serve each MPD from memory, so a test never depends on the network."""
monkeypatch.setattr(DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(MANIFESTS[url], url))
def test_export_without_manifest_url_still_rebuilds_dash_manifest_data(tmp_path: Path) -> None:
"""No top-level manifest_url means the export's own track URLs are the only MPDs to fetch."""
service = make_service(tmp_path, None, {"v-avc": exported_video("v-avc", AVC_URL, "AVC", 1920, 1080)})
tracks = service.get_tracks(next(iter(service.get_titles())))
assert [t.id for t in tracks.videos] == ["v-avc"]
# DASH.download_track reads all three of these; a missing one is a KeyError mid-download.
assert set(tracks.videos[0].data.get("dash", {})) >= {"manifest", "adaptation_set", "representation"}
def test_exported_hevc_on_its_own_mpd_survives_the_reparse(tmp_path: Path) -> None:
"""A ladder split across per-codec MPDs loses every codec but manifest_url's if only URL tracks merge back."""
service = make_service(
tmp_path,
AVC_URL,
{"v-hevc": exported_video("v-hevc", HEVC_URL, "HEVC", 3840, 2160)},
)
tracks = service.get_tracks(next(iter(service.get_titles())))
codecs = {t.codec.name for t in tracks.videos if t.codec}
assert "HEVC" in codecs, "the exported HEVC track was dropped, so -v h.265 has nothing to select"
hevc = next(t for t in tracks.videos if t.codec and t.codec.name == "HEVC")
assert set(hevc.data.get("dash", {})) >= {"manifest", "adaptation_set", "representation"}
# bound to its own manifest's representation, not just to any populated mapping
assert hevc.data["dash"]["representation"].get("id") == "v-hevc"
SUB_MPD_URL = "https://example.invalid/subs/de.mpd"
MAIN_WITH_SUB = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="v-avc" codecs="avc1.640028" bandwidth="5000000" width="1920" height="1080">
<BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
<AdaptationSet contentType="text" mimeType="text/vtt" lang="en">
<Representation id="s-en" bandwidth="1000">
<BaseURL>en.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
FOREIGN_SUB_MPD = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="text" mimeType="text/vtt" lang="de">
<Representation id="s-de" bandwidth="1000">
<BaseURL>de.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
def test_foreign_subtitle_adds_to_the_manifests_own(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Only a side-loaded (URL) subtitle replaces the re-parsed set.
A subtitle that lives on its own MPD is an addition. Nothing about it says the service
considered the main manifest's copy the worse one, and clearing on it deletes languages the
export never meant to drop.
"""
manifests = {AVC_URL: MAIN_WITH_SUB, SUB_MPD_URL: FOREIGN_SUB_MPD}
monkeypatch.setattr(DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(manifests[url], url))
foreign = DASH.from_text(FOREIGN_SUB_MPD, SUB_MPD_URL).to_tracks(language="de").subtitles[0]
service = make_service(
tmp_path,
AVC_URL,
{
foreign.id: {
"type": "Subtitle",
"id": foreign.id,
"url": SUB_MPD_URL,
"language": "de",
"descriptor": "DASH",
"codec": "WebVTT",
}
},
)
tracks = service.get_tracks(next(iter(service.get_titles())))
assert sorted(str(t.language) for t in tracks.subtitles) == ["de", "en"], (
"the main manifest's own subtitles were wiped by a subtitle from another MPD"
)
TWO_RUNG_MPD = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="lo" codecs="avc1.640028" bandwidth="3000000" width="1920" height="1080">
<BaseURL>lo.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation>
<Representation id="hi" codecs="avc1.640028" bandwidth="{top}" width="1920" height="1080">
<BaseURL>hi.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
@pytest.mark.parametrize(
"exported_url",
[AVC_URL, AVC_URL + "?token=EXPIRED"],
ids=["same-url", "signed-url"],
)
def test_reencoded_top_rung_does_not_readd_the_exported_one(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, exported_url: str
) -> None:
"""Track ids hash the bitrate, so re-encoding a rung makes the exported id look like a foreign MPD's.
Merging it back on that basis duplicates the rung, and the attribute fallback then binds the
copy to the first representation of the same codec and resolution: the 3 Mbps one. The result
downloads `lo` under a 6 Mbps name.
The signed variant is the same manifest reached by two strings: a service may fetch a signed
URL while registering the bare one as Title.tracks.manifest_url, so comparing them whole
reopens the duplicate.
"""
exported = DASH.from_text(TWO_RUNG_MPD.format(top=6000000), exported_url).to_tracks(language="en")
top = next(t for t in exported.videos if t.bitrate == 6000000)
monkeypatch.setattr(
DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(TWO_RUNG_MPD.format(top=6500000), url)
)
service = make_service(tmp_path, AVC_URL, {top.id: exported_video(top.id, exported_url, "AVC", 1920, 1080)})
tracks = service.get_tracks(next(iter(service.get_titles())))
assert sorted(t.bitrate for t in tracks.videos) == [3000000, 6500000], (
"the re-parsed ladder is the only source for this manifest's rungs"
)
bound = {t.data["dash"]["representation"].get("id") for t in tracks.videos}
assert bound == {"lo", "hi"}, "two tracks were bound to the same representation"
def test_clearkey_license_hook_is_bindable(tmp_path: Path) -> None:
"""dl.py binds service.get_clearkey_license for every track, DRM or not, before downloading.
Bind it the way dl.py does rather than asking hasattr: a non-callable attribute satisfies
hasattr and still fails at the partial with a TypeError.
"""
service = make_service(tmp_path, AVC_URL, {})
bound = partial(service.get_clearkey_license, title=None, track=None)
assert callable(bound)
with pytest.raises(RuntimeError, match="keys come from the export"):
bound(challenge=b"")
+138
View File
@@ -0,0 +1,138 @@
"""Tests for ``ImportService.get_tracks``, which rebuilds tracks from an export.
Regression: an export whose manifest labels its own streams imports even when the service
never set ``Title.language``. When nothing supplies a language the import fails naming the
service rather than blaming the manifest URL. A direct-URL subtitle anywhere in the export
replaces the manifest's whole subtitle set, since the service side-loaded it precisely
because the manifest's copy was the worse one.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Optional
from unittest.mock import patch
import click
import pytest
import requests
from unshackle.core.import_service import ImportService
from unshackle.core.manifests import DASH
MPD_TEMPLATE = """<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M">
<Period id="1">
<AdaptationSet contentType="video" mimeType="video/mp4"{lang}>
<Representation id="{video_id}" codecs="avc1.640028" bandwidth="5000000" width="1920" height="1080">
<BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/>
</Representation>
</AdaptationSet>
<AdaptationSet contentType="text" mimeType="text/vtt"{lang}>
<Representation id="{sub_id}" bandwidth="1000"><BaseURL>s.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation>
</AdaptationSet>
</Period>
</MPD>
"""
# labelled by @lang; unlabelled but with Unified Streaming ids DASH derives a language from;
# and labelled by neither, which is the only shape that needs the title's language
MPD_WITH_LANG = MPD_TEMPLATE.format(lang=' lang="ja"', video_id="v1", sub_id="s1")
MPD_ID_LANG = MPD_TEMPLATE.format(lang="", video_id="video_eng=5000000", sub_id="text_eng=1000")
MPD_NO_LANG = MPD_TEMPLATE.format(lang="", video_id="v1", sub_id="s1")
URL_SUBTITLE = {
"type": "Subtitle",
"id": "sideloaded-en",
"url": "https://example.invalid/en.srt",
"language": "en",
"descriptor": "URL",
"codec": "SubRip",
}
def make_service(tmp_path: Path, language: Optional[str], tracks: Optional[dict[str, Any]] = None) -> ImportService:
export = {
"version": 2,
"service": "EXAMPLE",
"titles": {
"movie-1": {
"meta": {"type": "movie", "id": "movie-1", "name": "Example Movie", "year": 2024, "language": language},
"manifest_url": "https://example.invalid/m.mpd",
"manifest_type": "DASH",
"tracks": tracks or {},
}
},
}
path = tmp_path / "export.json"
path.write_text(json.dumps(export), encoding="utf8")
return ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path))
def get_tracks(service: ImportService, mpd: str):
with patch.object(DASH, "from_url", side_effect=lambda url, session=None, **kw: DASH.from_text(mpd, url)):
return service.get_tracks(next(iter(service.get_titles())))
@pytest.mark.parametrize("mpd", [MPD_WITH_LANG, MPD_ID_LANG], ids=["lang-attribute", "language-in-rep-id"])
def test_manifest_supplies_language_without_title_language(tmp_path: Path, mpd: str) -> None:
"""A manifest that labels its own streams imports fine with no language in the export."""
tracks = get_tracks(make_service(tmp_path, language=None), mpd)
assert [str(t.language) for t in tracks.videos] == ["ja" if mpd is MPD_WITH_LANG else "en"]
@pytest.mark.parametrize("language", [None, "und"], ids=["missing", "und"])
def test_unlabelled_manifest_without_usable_language_blames_the_service(
tmp_path: Path, language: Optional[str]
) -> None:
"""`und` is truthy and a valid tag, so it must be rejected as firmly as no language at all."""
with pytest.raises(click.ClickException) as error:
get_tracks(make_service(tmp_path, language=language), MPD_NO_LANG)
assert "Title.language" in error.value.message
assert "may have expired" not in error.value.message
def test_exported_url_subtitles_replace_the_manifests(tmp_path: Path) -> None:
"""Services drop the manifest's subtitles and side-load their own; importing both duplicates them."""
tracks = get_tracks(make_service(tmp_path, "ja", {"sideloaded-en": URL_SUBTITLE}), MPD_WITH_LANG)
assert [(t.id, t.descriptor.name) for t in tracks.subtitles] == [("sideloaded-en", "URL")]
assert len(tracks.videos) == 1
def test_direct_url_subtitles_win_over_every_manifest_one(tmp_path: Path) -> None:
"""One side-loaded subtitle condemns the whole re-parsed set, including other languages."""
manifest_sub = {
"type": "Subtitle",
"id": "s1",
"url": "https://example.invalid/m.mpd",
"language": "ja",
"descriptor": "DASH",
"codec": "WebVTT",
}
tracks = get_tracks(
make_service(tmp_path, "ja", {"s1": manifest_sub, "sideloaded-en": URL_SUBTITLE}), MPD_WITH_LANG
)
assert [(t.id, t.descriptor.name) for t in tracks.subtitles] == [("sideloaded-en", "URL")]
def test_unreadable_exported_track_is_skipped(tmp_path: Path) -> None:
"""A track dict this build cannot rebuild must not take the whole import down with it."""
unknown = {**URL_SUBTITLE, "id": "future", "codec": "CODEC_FROM_A_LATER_BUILD"}
tracks = get_tracks(make_service(tmp_path, "ja", {"sideloaded-en": URL_SUBTITLE, "future": unknown}), MPD_WITH_LANG)
assert [t.id for t in tracks.subtitles] == ["sideloaded-en"]
@pytest.mark.parametrize(
"failure",
[requests.ConnectionError("dns"), TypeError("Expected 'MPD' document, but received a 'html' document instead.")],
ids=["connection-error", "expired-token-html-page"],
)
def test_fetch_failure_still_reports_a_possible_expiry(tmp_path: Path, failure: Exception) -> None:
"""An expired signed URL can 200 with an error page, so document failures are expiries too."""
service = make_service(tmp_path, "ja")
title = next(iter(service.get_titles()))
with patch.object(DASH, "from_url", side_effect=failure):
with pytest.raises(click.ClickException) as error:
service.get_tracks(title)
assert "may have expired" in error.value.message
+314
View File
@@ -24,6 +24,7 @@ from unshackle.core.manifests.ism_init import (
parse_codec_private_data_colour,
parse_codec_private_data_vui,
parse_hevc_sps_format,
piff_senc_to_cenc,
read_per_sample_iv_size,
read_track_id,
remove_emulation_prevention,
@@ -111,6 +112,82 @@ def test_hevc_init_structure():
assert b"encv" not in init and b"sinf" not in init
# Boxes whose payload is purely child boxes, so their size must span the children
# exactly. stsd is excluded: its payload starts with a count, and sample entries
# carry a fixed prefix before any child box.
CONTAINER_BOXES = {"moov", "trak", "mdia", "minf", "stbl", "dinf", "mvex", "sinf", "schi"}
def walk_box(data: bytes, start: int, end: int, path: str = "") -> None:
"""Assert every container's declared size spans its children exactly."""
offset = start
while offset < end:
size = struct.unpack_from(">I", data, offset)[0]
name = data[offset + 4 : offset + 8].decode("latin1")
where = f"{path}/{name}"
assert size >= 8, f"{where}: size {size}"
assert offset + size <= end, f"{where}: overruns its parent"
if name in CONTAINER_BOXES:
walk_box(data, offset + 8, offset + size, where)
offset += size
assert offset == end, f"{path or '/'}: children do not span the parent exactly"
def find_stsd(init: bytes) -> int:
"""Offset of the stsd box, found by walking the box tree."""
offset, end = 0, len(init)
for name in ("moov", "trak", "mdia", "minf", "stbl", "stsd"):
cursor = offset
while cursor < end:
size = struct.unpack_from(">I", init, cursor)[0]
if init[cursor + 4 : cursor + 8].decode("latin1") == name:
offset, end = cursor + 8, cursor + size
break
cursor += size
else:
raise AssertionError(f"{name} not found")
return offset - 8
# The two-entry stsd is unconditional, so every stream type and codec the builder
# emits needs covering, encrypted and clear alike.
STSD_CASES = [
("video", "HVC1", VIDEO_HEVC_CPD, KID, b"encv", {"width": 3840, "height": 2160}),
("video", "HVC1", VIDEO_HEVC_CPD, None, b"hvc1", {"width": 3840, "height": 2160}),
("video", "H264", VIDEO_AVC_CPD, None, b"avc1", {"width": 1280, "height": 720}),
("audio", "AACL", AAC_LC_CPD, None, b"mp4a", {"channels": 2}),
("audio", "EC-3", EC3_CPD, KID, b"enca", {"channels": 6}),
("text", "TTML", "", None, b"stpp", {}),
]
@pytest.mark.parametrize("stream_type,fourcc,cpd,kid,expected_4cc,extra", STSD_CASES)
def test_stsd_carries_two_entries_for_sample_description_index_2(stream_type, fourcc, cpd, kid, expected_4cc, extra):
# A single entry leaves tfhd sample_description_index=2 dangling, and mp4decrypt
# skips those fragments without a word: exit 0, empty stderr, still encrypted.
init = build_init_segment(
stream_type=stream_type,
fourcc=fourcc,
codec_private_data=cpd,
timescale=10000000,
kid=kid,
**extra,
)
offset = find_stsd(init)
assert struct.unpack_from(">I", init, offset + 12)[0] == 2, "index 2 would dangle"
entries, cursor = [], offset + 16
for _ in range(2):
size = struct.unpack_from(">I", init, cursor)[0]
entries.append(init[cursor : cursor + size])
cursor += size
assert [e[4:8] for e in entries] == [expected_4cc, expected_4cc]
# stsd must span its header, count and both entries exactly.
assert struct.unpack_from(">I", init, offset)[0] == 16 + sum(len(e) for e in entries)
# The same has to hold for every container box wrapping it.
walk_box(init, 0, len(init))
def test_avc_init_structure():
init = build_init_segment(
stream_type="video",
@@ -502,3 +579,240 @@ def test_hvcc_profile_tier_level_is_nonzero():
level_idc = payload[12]
assert profile_idc != 0
assert level_idc != 0
def _piff_fragment(flags: int = 0x0, override: bool = False) -> bytes:
"""Minimal moof+mdat fragment carrying sample encryption in the PIFF uuid box."""
ivs = b"".join(bytes([i]) * 8 for i in range(1, 3))
senc_payload = struct.pack(">I", 2) + ivs
if override:
flags |= 0x1
senc_payload = b"\x00\x00\x01" + bytes([8]) + KID + senc_payload # AlgorithmID + IV_size + KID
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + flags.to_bytes(3, "big") + senc_payload)
tfhd = full_box(b"tfhd", 0, 0x20000, struct.pack(">I", 1))
trun = full_box(b"trun", 0, 0x1, struct.pack(">II", 2, 0)) # data_offset placeholder
traf = box(b"traf", tfhd + trun + uuid_box)
moof = box(b"moof", full_box(b"mfhd", 0, 0, struct.pack(">I", 1)) + traf)
# patch trun data_offset to the real moof-relative start of the mdat payload
moof = moof.replace(struct.pack(">II", 2, 0), struct.pack(">II", 2, len(moof) + 8), 1)
return moof + box(b"mdat", b"\xab" * 64)
def test_piff_senc_rewritten_to_cenc_senc():
frag = _piff_fragment()
out = piff_senc_to_cenc(frag)
# trun's data_offset is moof-relative; shrinking the box misaims decryption
assert len(out) == len(frag)
assert PIFF_SENC_UUID not in out
assert b"senc" in out and b"free" in out
assert struct.pack(">I", 2) + b"".join(bytes([i]) * 8 for i in range(1, 3)) in out
# moof size unchanged => the recorded data_offset still resolves
assert out[:8] == frag[:8]
assert out[len(out) - 72 :] == frag[len(frag) - 72 :] # mdat untouched
def test_piff_senc_override_header_stripped():
frag = _piff_fragment(override=True)
out = piff_senc_to_cenc(frag)
assert len(out) == len(frag)
assert PIFF_SENC_UUID not in out
senc = out.index(b"senc")
assert out[senc + 4] == 0 # version
assert int.from_bytes(out[senc + 5 : senc + 8], "big") & 0x1 == 0 # override flag cleared
assert out[senc + 8 : senc + 12] == struct.pack(">I", 2) # sample_count directly follows
def test_piff_subsample_flag_survives_override_strip():
# losing 0x2 is silent: no subsample map, so clear NAL headers get encrypted too
frag = _piff_fragment(flags=0x2, override=True)
out = piff_senc_to_cenc(frag)
senc = out.index(b"senc")
flags = int.from_bytes(out[senc + 5 : senc + 8], "big")
assert flags & 0x2, "subsample-encryption flag must survive"
assert not flags & 0x1, "override flag must be cleared"
def test_piff_rewrite_leaves_non_piff_fragments_alone():
plain = box(b"moof", box(b"traf", full_box(b"tfhd", 0, 0, struct.pack(">I", 1)))) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(plain) is plain
assert piff_senc_to_cenc(b"") == b""
def test_piff_rewrite_skips_traf_that_already_has_senc():
# a second senc would leave the decrypter to choose between them
ivs = b"".join(bytes([i]) * 8 for i in range(1, 3))
senc = full_box(b"senc", 0, 0, struct.pack(">I", 2) + ivs)
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0).to_bytes(3, "big") + struct.pack(">I", 2) + ivs)
frag = box(b"moof", box(b"traf", senc + uuid_box)) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(frag) is frag
def test_piff_rewrite_covers_every_moof_and_traf():
frag = _piff_fragment() + _piff_fragment()
out = piff_senc_to_cenc(frag)
assert len(out) == len(frag)
assert PIFF_SENC_UUID not in out
assert out.count(b"senc") == 2
def test_piff_rewrite_refuses_non_ctr_algorithm_id():
# AlgorithmID 2 is AES-CBC but the sample entry says 'cenc'; rewriting yields garbage
ivs = b"".join(bytes([i]) * 8 for i in range(1, 3))
payload = (2).to_bytes(3, "big") + bytes([8]) + KID + struct.pack(">I", 2) + ivs
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload)
frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(frag) is frag
def traf_children_tile_exactly(frag: bytes) -> bool:
"""Every traf child's declared size must chain exactly to the traf end."""
moof_size = struct.unpack(">I", frag[:4])[0]
traf_start = 8
while frag[traf_start + 4 : traf_start + 8] != b"traf":
traf_start += struct.unpack(">I", frag[traf_start : traf_start + 4])[0]
traf_end = traf_start + struct.unpack(">I", frag[traf_start : traf_start + 4])[0]
assert traf_end <= moof_size
offset = traf_start + 8
while offset < traf_end:
size = struct.unpack(">I", frag[offset : offset + 4])[0]
if size < 8 or offset + size > traf_end:
return False
offset += size
return offset == traf_end
def test_piff_rewrite_keeps_traf_children_tiling_exactly():
# a mis-sized senc still "decrypts", just off the wrong bytes, so assert the sizes too
assert traf_children_tile_exactly(_piff_fragment())
for frag in (_piff_fragment(), _piff_fragment(flags=0x2), _piff_fragment(flags=0x2, override=True)):
assert traf_children_tile_exactly(piff_senc_to_cenc(frag)), "rewritten traf no longer tiles"
def test_piff_rewrite_refuses_override_iv_size_mismatch():
# tenc is the only IV width a decrypter consults; a disagreeing override misparses silently
ivs = b"".join(bytes([i]) * 16 for i in range(1, 3))
payload = (1).to_bytes(3, "big") + bytes([16]) + KID + struct.pack(">I", 2) + ivs
uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload)
frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16)
assert piff_senc_to_cenc(frag, iv_size=8) is frag, "8-byte tenc must reject a 16-byte override"
assert piff_senc_to_cenc(frag, iv_size=16) is not frag, "matching width must still rewrite"
def test_piff_rewrite_never_grows_output_on_malformed_boxes():
# A declared size running past the buffer would extend the output over the mdat.
good = _piff_fragment()
truncations = [good[:n] for n in range(8, len(good), 7)]
oversized = bytearray(good)
struct.pack_into(">I", oversized, 0, len(good) + 4096) # moof claims more than exists
for frag in [*truncations, bytes(oversized), b"", b"\x00" * 12]:
out = piff_senc_to_cenc(frag) # must not raise
assert len(out) == len(frag), f"output grew: {len(frag)} -> {len(out)}"
def _text_fragment(seq: int, begin: str, end: str, text: str, sdi: int) -> bytes:
"""A single-sample fTTML moof+mdat, mirroring what a Smooth text stream serves."""
payload = (
"<?xml version='1.0' encoding='utf-8'?>"
"<tt xmlns='http://www.w3.org/ns/ttml' xml:lang='en'><body><div>"
f"<p begin='{begin}' end='{end}'>{text}</p>"
"</div></body></tt>"
).encode()
mfhd = full_box(b"mfhd", 0, 0, struct.pack(">I", seq))
tfhd = full_box(b"tfhd", 0, 0x020002, struct.pack(">I", 1) + struct.pack(">I", sdi))
tfdt = full_box(b"tfdt", 1, 0, struct.pack(">Q", 0))
trun = full_box(b"trun", 0, 0x000201, struct.pack(">I", 1) + struct.pack(">i", 0) + struct.pack(">I", len(payload)))
return box(b"moof", mfhd + box(b"traf", tfhd + tfdt + trun)) + box(b"mdat", payload)
def test_unity_matrix_is_nine_int32s():
# extras shift every later field (mvhd next_track_ID, tkhd width/height) and desync pymp4
from unshackle.core.manifests.ism_init import UNITY_MATRIX
assert len(UNITY_MATRIX) == 36
@pytest.mark.parametrize("stream_type", ["video", "audio", "text"])
def test_mvhd_tkhd_sizes_match_iso_layout(stream_type):
# a declared size over the spec payload makes parsers under-read and stop at the moov
kwargs = {
"video": dict(fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080),
"audio": dict(fourcc="AACL", codec_private_data=AAC_LC_CPD),
"text": dict(fourcc="TTML", codec_private_data=""),
}[stream_type]
init = build_init_segment(stream_type=stream_type, duration=6_000_000_000, language="eng", **kwargs)
sizes = {}
for name in (b"mvhd", b"tkhd"):
at = init.index(name) - 4
sizes[name] = struct.unpack(">I", init[at : at + 4])[0]
# version-1 mvhd: 8 header + 4 version/flags + 28 time fields + 80 trailer
assert sizes[b"mvhd"] == 120
# version-1 tkhd: 8 header + 4 version/flags + 32 time fields + 60 trailer
assert sizes[b"tkhd"] == 104
def test_tkhd_carries_video_dimensions():
# width/height sit after the matrix, so a mis-sized matrix silently zeroes them
init = build_init_segment(
stream_type="video", fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080
)
at = init.index(b"tkhd") - 4
tkhd = init[at : at + struct.unpack(">I", init[at : at + 4])[0]]
width, height = struct.unpack(">II", tkhd[-8:])
assert (width >> 16, height >> 16) == (1920, 1080)
def test_pymp4_consumes_whole_init_and_reaches_fragments():
# pymp4 must reach the mdat here, or Subtitle.parse finds nothing to yield cues from
from io import BytesIO
from pymp4.parser import MP4
init = build_init_segment(
stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng"
)
data = init + _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1)
types = [b.type for b in MP4.parse_stream(BytesIO(data))]
assert types == [b"ftyp", b"moov", b"moof", b"mdat"]
def test_fttml_subtitle_parse_yields_cues():
from unshackle.core.tracks.subtitle import Subtitle
init = build_init_segment(
stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng"
)
data = (
init
+ _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1)
+ _text_fragment(2, "00:00:03.000", "00:00:04.000", "second", sdi=2)
)
caption_set = Subtitle.parse(data, Subtitle.Codec.fTTML)
cues = [c for lang in caption_set.get_languages() for c in caption_set.get_captions(lang)]
assert len(cues) == 2
assert [c.get_text() for c in cues] == ["first", "second"]
@pytest.mark.parametrize(
"fourcc,expected",
[("HEV1", b"hev1"), ("hev1", b"hev1"), ("HVC1", b"hvc1"), ("hvc1", b"hvc1"), ("HEVC", b"hvc1"), ("H265", b"hvc1")],
)
def test_hevc_sample_entry_honours_manifest_fourcc(fourcc, expected):
# hev1 permits in-band parameter sets where hvc1 requires them in the sample
# entry. Real Smooth manifests ship FourCC="hev1", so relabelling it hvc1
# misdescribes the stream.
init = build_init_segment(
stream_type="video", fourcc=fourcc, codec_private_data=VIDEO_HEVC_CPD, width=3840, height=2160
)
offset = find_stsd(init)
assert init[offset + 20 : offset + 24] == expected
def test_encrypted_hev1_frma_carries_the_same_fourcc():
# frma names the original format the sinf wraps; if it drifts from the sample
# entry, a decrypter restores the wrong sample entry type.
init = build_init_segment(
stream_type="video", fourcc="HEV1", codec_private_data=VIDEO_HEVC_CPD, kid=KID, width=3840, height=2160
)
assert b"encv" in init
assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"hev1"
+223
View File
@@ -0,0 +1,223 @@
"""Tests for the tag_rules engine behind conditional release-group tags."""
from __future__ import annotations
from typing import Any
import pytest
from unshackle.core.utils.tag_rules import evaluate_tag_rules
CONTEXT: dict[str, Any] = {
"tag": "NOGRP",
"title_type": "movie",
"quality": "2160p",
"resolution": "2160",
"hdr": "DV",
"lang_tag": "SUBBED",
"edition": "",
}
def test_first_match_wins() -> None:
rules = [
{"when": {"quality": "2160p"}, "tag": "FIRST"},
{"when": {"title_type": "movie"}, "tag": "SECOND"},
]
assert evaluate_tag_rules(rules, CONTEXT) == "FIRST"
def test_list_matches_any_entry() -> None:
rules = [{"when": {"hdr": ["HDR10P", "DV"]}, "tag": "UHDGROUP"}]
assert evaluate_tag_rules(rules, CONTEXT) == "UHDGROUP"
def test_list_with_no_matching_entry() -> None:
rules = [{"when": {"hdr": ["HDR10P", "HLG"]}, "tag": "UHDGROUP"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_case_insensitive() -> None:
rules = [{"when": {"lang_tag": "subbed", "quality": "2160P"}, "tag": "SUBMOVIES"}]
assert evaluate_tag_rules(rules, CONTEXT) == "SUBMOVIES"
def test_conditions_are_anded() -> None:
rules = [{"when": {"title_type": "movie", "quality": "1080p"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_unknown_key_skips_rule_but_later_rules_match(caplog: pytest.LogCaptureFixture) -> None:
rules = [
{"when": {"bogus": "value"}, "tag": "SKIPPED"},
{"when": {"title_type": "movie"}, "tag": "MATCHED"},
]
assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED"
assert "bogus" in caplog.text
@pytest.mark.parametrize(
"rule",
[
{"when": {"title_type": "movie"}},
{"tag": "NOCONDITIONS"},
{"when": {}, "tag": "EMPTY"},
],
)
def test_malformed_rule_skipped(rule: dict[str, Any]) -> None:
assert evaluate_tag_rules([rule], CONTEXT) is None
def test_no_rules_and_no_match() -> None:
assert evaluate_tag_rules([], CONTEXT) is None
assert evaluate_tag_rules([{"when": {"quality": "720p"}, "tag": "SD"}], CONTEXT) is None
def test_empty_context_value_matches_empty_condition() -> None:
rules = [{"when": {"edition": ""}, "tag": "PLAIN"}]
assert evaluate_tag_rules(rules, CONTEXT) == "PLAIN"
@pytest.mark.parametrize(
("condition", "expected"),
[
(">=2160", True),
(">=4320", False),
("<=2160", True),
("<1080", False),
(">1080", True),
("==2160", True),
("=2160", True),
("!=1080", True),
("!=2160", False),
(">= 2160", True),
],
)
def test_numeric_operators_extract_first_number(condition: str, expected: bool) -> None:
"""The actual value is "2160p", so the suffix must not defeat the comparison."""
rules = [{"when": {"quality": condition}, "tag": "MATCHED"}]
assert (evaluate_tag_rules(rules, CONTEXT) == "MATCHED") is expected
def test_string_inequality() -> None:
assert evaluate_tag_rules([{"when": {"hdr": "!=DV"}, "tag": "NODV"}], CONTEXT) is None
assert evaluate_tag_rules([{"when": {"hdr": "!=HDR10P"}, "tag": "NOHDR10P"}], CONTEXT) == "NOHDR10P"
def test_string_equality_operator_is_case_insensitive() -> None:
assert evaluate_tag_rules([{"when": {"lang_tag": "==subbed"}, "tag": "SUBGRP"}], CONTEXT) == "SUBGRP"
def test_plain_equality_unchanged() -> None:
assert evaluate_tag_rules([{"when": {"quality": "2160p"}, "tag": "UHD"}], CONTEXT) == "UHD"
assert evaluate_tag_rules([{"when": {"quality": "2160"}, "tag": "UHD"}], CONTEXT) is None
def test_numeric_operator_against_value_without_a_number() -> None:
rules = [{"when": {"hdr": ">=1080"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_ordering_operator_with_non_numeric_operand(caplog: pytest.LogCaptureFixture) -> None:
rules = [{"when": {"hdr": ">DV"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
assert "numeric operand" in caplog.text
def test_operators_inside_lists() -> None:
rules = [{"when": {"quality": ["<720", ">=2160"]}, "tag": "MATCHED"}]
assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED"
rules = [{"when": {"quality": ["<720", "1080p"]}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
@pytest.mark.parametrize("rules", ["UHDGROUP", {"when": {"quality": "2160p"}, "tag": "UHD"}, None, 42])
def test_rules_that_are_not_a_list(rules: Any) -> None:
"""A mapping or string here iterates keys/characters, so it must be rejected outright."""
assert evaluate_tag_rules(rules, CONTEXT) is None
@pytest.mark.parametrize("rule", [None, "UHDGROUP", 42, ["when", "tag"]])
def test_rule_entry_that_is_not_a_mapping_skipped(rule: Any) -> None:
assert evaluate_tag_rules([rule], CONTEXT) is None
assert evaluate_tag_rules([rule, {"when": {"title_type": "movie"}, "tag": "MATCHED"}], CONTEXT) == "MATCHED"
@pytest.mark.parametrize("when", [["title_type", "movie"], "title_type", 42])
def test_when_that_is_not_a_mapping_skipped(when: Any) -> None:
assert evaluate_tag_rules([{"when": when, "tag": "NOPE"}], CONTEXT) is None
@pytest.mark.parametrize(
"rule",
[
{"when": {"atmos": True}, "tag": "ATMOS"},
{"when": {"atmos": [True, "Atmos"]}, "tag": "ATMOS"},
{"when": {"title_type": "movie"}, "tag": True},
],
)
def test_yaml_boolean_skipped(rule: dict[str, Any], caplog: pytest.LogCaptureFixture) -> None:
"""`atmos: true` / `tag: yes` are YAML coercions the user meant as text."""
assert evaluate_tag_rules([rule], CONTEXT) is None
assert "boolean" in caplog.text
def test_numbers_stay_valid() -> None:
assert evaluate_tag_rules([{"when": {"resolution": 2160}, "tag": "UHD"}], CONTEXT) == "UHD"
assert evaluate_tag_rules([{"when": {"resolution": 1080}, "tag": "NOPE"}], CONTEXT) is None
def test_numeric_tag_becomes_text() -> None:
assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": 123}], CONTEXT) == "123"
@pytest.mark.parametrize("tag", [["A", "B"], {"x": 1}, ("A",), {"A"}])
def test_non_scalar_tag_skipped(tag: Any, caplog: pytest.LogCaptureFixture) -> None:
"""A list or dict here used to reach the filename as its repr."""
assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": tag}], CONTEXT) is None
assert "must be text or a number" in caplog.text
@pytest.mark.parametrize("tag", ["", 0])
def test_empty_tag_skipped(tag: Any) -> None:
assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": tag}], CONTEXT) is None
def test_skipped_rule_falls_through_to_a_later_one() -> None:
rules = [
{"when": {"atmos": True}, "tag": "SKIPPED"},
{"when": {"title_type": "movie"}, "tag": "MATCHED"},
]
assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED"
def test_several_negations_in_a_list_always_match() -> None:
"""Any-entry matching makes a list of negations useless, which the docs warn about."""
rules = [{"when": {"hdr": ["!=DV", "!=HLG"]}, "tag": "ALWAYS"}]
assert evaluate_tag_rules(rules, CONTEXT) == "ALWAYS"
def test_leading_space_before_an_operator_is_literal() -> None:
"""Current behaviour: the operator is only found at the very start of the value."""
rules = [{"when": {"quality": " >=2160"}, "tag": "NOPE"}]
assert evaluate_tag_rules(rules, CONTEXT) is None
def test_inequality_holds_for_a_value_with_no_number() -> None:
""" "DV" really is not equal to 2160, so != holds while the others fail."""
assert evaluate_tag_rules([{"when": {"hdr": "!=2160"}, "tag": "MATCHED"}], CONTEXT) == "MATCHED"
assert evaluate_tag_rules([{"when": {"hdr": ">=2160"}, "tag": "NOPE"}], CONTEXT) is None
assert evaluate_tag_rules([{"when": {"hdr": "==2160"}, "tag": "NOPE"}], CONTEXT) is None
@pytest.mark.parametrize("condition", ["!=", ">=", "<", "=", "=="])
def test_bare_operator_fails(condition: str, caplog: pytest.LogCaptureFixture) -> None:
assert evaluate_tag_rules([{"when": {"hdr": condition}, "tag": "NOPE"}], CONTEXT) is None
assert "no value to compare against" in caplog.text
def test_composite_hdr_does_not_match_plain_dv() -> None:
"""DV titles carry a base-layer suffix, which the docs tell users to list explicitly."""
context = {**CONTEXT, "hdr": "DV.HDR10P"}
assert evaluate_tag_rules([{"when": {"hdr": "DV"}, "tag": "NOPE"}], context) is None
assert evaluate_tag_rules([{"when": {"hdr": ["DV", "DV.HDR10P"]}, "tag": "DOLBY"}], context) == "DOLBY"
+131
View File
@@ -0,0 +1,131 @@
"""Tests for tag_rules reaching the filename context through Title."""
from __future__ import annotations
import inspect
import types
from typing import Any, Iterator
import pytest
from langcodes import Language
from unshackle.core.config import config
from unshackle.core.titles.movie import Movie
class DummyService:
pass
class TrackStub:
"""Stand-in for Tracks: truthy and iterable, with the two lists lang_tag reads."""
def __init__(self, audio: list[Any], subtitles: list[Any]) -> None:
self.audio = audio
self.subtitles = subtitles
def __iter__(self) -> Iterator[Any]:
return iter([])
def __bool__(self) -> bool:
return True
def make_audio(language: str) -> types.SimpleNamespace:
return types.SimpleNamespace(
language=language,
format="AAC",
channel_layout=None,
channellayout_original=None,
channel_s=2,
channels=2,
format_additionalfeatures=None,
joc=None,
)
def make_media_info(languages: list[str]) -> types.SimpleNamespace:
return types.SimpleNamespace(video_tracks=[], audio_tracks=[make_audio(lang) for lang in languages])
def make_movie() -> Movie:
return Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language="en")
@pytest.fixture(autouse=True)
def clean_config(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "tag", "NOGRP")
monkeypatch.setattr(config, "tag_rules", [])
monkeypatch.setattr(config, "language_tags", {})
def context_for(rules: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch) -> dict:
monkeypatch.setattr(config, "tag_rules", rules)
return make_movie()._build_base_template_context(make_media_info(["en"]))
def test_matching_rule_replaces_the_tag(monkeypatch: pytest.MonkeyPatch) -> None:
context = context_for([{"when": {"title_type": "movie"}, "tag": "MOVIEGRP"}], monkeypatch)
assert context["tag"] == "MOVIEGRP"
def test_no_matching_rule_keeps_the_config_tag(monkeypatch: pytest.MonkeyPatch) -> None:
context = context_for([{"when": {"title_type": "series"}, "tag": "TVGRP"}], monkeypatch)
assert context["tag"] == "NOGRP"
def test_no_rules_keeps_the_config_tag(monkeypatch: pytest.MonkeyPatch) -> None:
context = context_for([], monkeypatch)
assert context["tag"] == "NOGRP"
def test_title_type_is_in_the_context(monkeypatch: pytest.MonkeyPatch) -> None:
assert context_for([], monkeypatch)["title_type"] == "movie"
def test_rules_run_after_lang_tag(monkeypatch: pytest.MonkeyPatch) -> None:
"""A rule matching on lang_tag only works if lang_tag is resolved first."""
monkeypatch.setattr(config, "language_tags", {"rules": [{"audio": "ja", "tag": "SUBBED"}]})
monkeypatch.setattr(config, "tag_rules", [{"when": {"lang_tag": "SUBBED"}, "tag": "SUBGRP"}])
movie = make_movie()
movie.tracks = TrackStub(audio=[types.SimpleNamespace(language=Language.get("ja"))], subtitles=[]) # type: ignore[assignment]
context = movie._build_base_template_context(make_media_info(["ja"]))
assert context["lang_tag"] == "SUBBED"
assert context["tag"] == "SUBGRP"
def test_a_quality_rule_matches_the_built_context(monkeypatch: pytest.MonkeyPatch) -> None:
"""Guards the context keys the docs tell users to match on."""
monkeypatch.setattr(config, "tag_rules", [{"when": {"quality": "2160p", "video": "H.265"}, "tag": "UHDGROUP"}])
video = types.SimpleNamespace(
width=3840,
height=2160,
format="HEVC",
scan_type="Progressive",
other_display_aspect_ratio=["16:9"],
hdr_format_commercial=None,
hdr_format=None,
transfer_characteristics=None,
transfer_characteristics_original=None,
frame_rate="23.976",
)
media_info = types.SimpleNamespace(video_tracks=[video], audio_tracks=[make_audio("en")])
context = make_movie()._build_base_template_context(media_info)
assert context["quality"] == "2160p"
assert context["video"] == "H.265"
assert context["tag"] == "UHDGROUP"
def test_dl_clears_tag_rules_when_tag_is_given() -> None:
"""Source-level pin: the block lives inside the click-driven dl.__init__."""
from unshackle.commands.dl import dl
source = inspect.getsource(dl.__init__)
block = source.split("if tag:")[1].split("\n\n")[0]
assert "config.tag = tag" in block
assert "config.tag_rules = []" in block
+174
View File
@@ -0,0 +1,174 @@
"""Tests for per-task temp directories.
Each run gets a private dir that is removed on every exit path. The sweep reclaims
dirs from hard-killed processes via lock liveness, and `env clear temp` skips the
dir of a task that is still running.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
import pytest
from filelock import FileLock
from unshackle.commands.env import clear_directory
from unshackle.core.config import config
from unshackle.core.temp import LOCK_NAME, TASK_PREFIX, is_stale, sweep_task_dirs, task_temp_dir
pytestmark = pytest.mark.unit
@pytest.fixture
def temp_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
root = tmp_path / "temp"
root.mkdir()
monkeypatch.setattr(config.directories, "temp", root)
return root
def backdate(path: Path) -> None:
stamp = time.time() - 120
os.utime(path, (stamp, stamp))
def make_task_dir(root: Path, name: str = "task_deadbeef", lock: bool = True) -> Path:
task_dir = root / name
task_dir.mkdir()
if lock:
(task_dir / LOCK_NAME).touch()
return task_dir
def test_task_dir_created_repointed_and_restored(temp_root: Path) -> None:
with task_temp_dir() as task_dir:
assert task_dir.parent == temp_root
assert task_dir.name.startswith(TASK_PREFIX)
assert config.directories.temp == task_dir
assert config.directories.temp == temp_root
assert not task_dir.exists()
def test_task_dir_removed_on_exception(temp_root: Path) -> None:
with pytest.raises(ValueError):
with task_temp_dir() as task_dir:
(task_dir / "partial.mp4").write_bytes(b"x")
raise ValueError("boom")
assert not task_dir.exists()
assert config.directories.temp == temp_root
def test_task_dir_removed_on_keyboard_interrupt(temp_root: Path) -> None:
with pytest.raises(KeyboardInterrupt):
with task_temp_dir() as task_dir:
raise KeyboardInterrupt
assert not task_dir.exists()
assert config.directories.temp == temp_root
def test_sweep_removes_unlocked_stale_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
(task_dir / "leftover.mp4").write_bytes(b"x")
backdate(task_dir)
sweep_task_dirs(temp_root)
assert not task_dir.exists()
def test_sweep_skips_locked_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
lock = FileLock(task_dir / LOCK_NAME)
lock.acquire()
backdate(task_dir)
try:
assert is_stale(task_dir) is False
sweep_task_dirs(temp_root)
assert task_dir.exists()
finally:
lock.release()
def test_sweep_skips_young_unlocked_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
assert is_stale(task_dir) is False
sweep_task_dirs(temp_root)
assert task_dir.exists()
def test_sweep_removes_stale_dir_without_lock_file(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root, lock=False)
backdate(task_dir)
assert is_stale(task_dir) is True
sweep_task_dirs(temp_root)
assert not task_dir.exists()
def test_sweep_ignores_non_task_entries(temp_root: Path) -> None:
loose_file = temp_root / "leftover.mp4"
loose_file.write_bytes(b"x")
other_dir = temp_root / "not_a_task"
other_dir.mkdir()
backdate(loose_file)
backdate(other_dir)
sweep_task_dirs(temp_root)
assert loose_file.exists()
assert other_dir.exists()
@pytest.mark.skipif(os.name == "nt", reason="symlink creation needs privileges on Windows")
def test_sweep_never_follows_symlinked_task_dir(temp_root: Path, tmp_path: Path) -> None:
target = tmp_path / "victim"
target.mkdir()
(target / "precious.mkv").write_bytes(b"data")
link = temp_root / "task_evil"
link.symlink_to(target)
backdate(target)
sweep_task_dirs(temp_root)
assert (target / "precious.mkv").exists()
assert not (target / LOCK_NAME).exists()
@pytest.mark.skipif(os.name == "nt", reason="symlink creation needs privileges on Windows")
def test_clear_directory_unlinks_symlinked_dir_without_following(temp_root: Path, tmp_path: Path) -> None:
target = tmp_path / "victim"
target.mkdir()
(target / "precious.mkv").write_bytes(b"data")
link = temp_root / "task_evil"
link.symlink_to(target)
backdate(target)
files_count, _ = clear_directory(temp_root)
assert not link.exists()
assert (target / "precious.mkv").exists()
assert files_count == 1
def test_clear_directory_skips_locked_task_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
(task_dir / "in_flight.mp4").write_bytes(b"xyz")
lock = FileLock(task_dir / LOCK_NAME)
lock.acquire()
backdate(task_dir)
loose_file = temp_root / "leftover.mp4"
loose_file.write_bytes(b"1234")
try:
files_count, freed_bytes = clear_directory(temp_root)
finally:
lock.release()
assert task_dir.exists()
assert (task_dir / "in_flight.mp4").exists()
assert not loose_file.exists()
assert files_count == 1
assert freed_bytes == 4
def test_clear_directory_removes_stale_task_dir(temp_root: Path) -> None:
task_dir = make_task_dir(temp_root)
(task_dir / "orphan.mp4").write_bytes(b"12345")
backdate(task_dir)
files_count, freed_bytes = clear_directory(temp_root)
assert not task_dir.exists()
assert temp_root.is_dir()
# glob("**/*") never matches dotfiles, so .lock is not counted
assert files_count == 1
assert freed_bytes == 5
+292
View File
@@ -0,0 +1,292 @@
"""Separator cleanup around omitted conditionals in TemplateFormatter.
An omitted `{var?}` must not leave separator debris behind. Trailing spaces are the
dangerous case: the formatter sanitizes each path segment separately, and Windows
drops a trailing space when it creates a directory but not when it looks one up, so
the download fails with FileNotFoundError [WinError 3]. See #107 and #144.
"""
from __future__ import annotations
import pytest
from unshackle.core.config import config
from unshackle.core.titles.episode import Episode
from unshackle.core.titles.movie import Movie
from unshackle.core.titles.music import Song as MusicSong
from unshackle.core.titles.song import Song
from unshackle.core.utilities import sanitize_filename
from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer
@pytest.mark.parametrize(
("template", "context", "expected"),
[
# An omitted conditional inside brackets must take the whole group with it.
("{title} ({year?})", {"title": "Some Movie", "year": ""}, "Some Movie"),
("{title} [{quality?}]", {"title": "Some Movie", "quality": ""}, "Some Movie"),
("{title} ({year?}) [{quality?}]", {"title": "T", "year": "", "quality": ""}, "T"),
# The group survives when the value is present.
("{title} ({year?})", {"title": "Some Movie", "year": "2024"}, "Some Movie (2024)"),
# #107: the dash wins, so the release tag stays attached.
(
"{title}.{year}.{atmos?}-{tag}",
{"title": "Some Movie", "year": "2024", "atmos": "", "tag": "GRP"},
"Some.Movie.2024-GRP",
),
(
"{title}.{year}.{atmos?}-{tag}",
{"title": "Some Movie", "year": "2024", "atmos": "Atmos", "tag": "GRP"},
"Some.Movie.2024.Atmos-GRP",
),
# A conditional between two dashes must not leave a doubled separator.
("{title} - {edition?} - {year}", {"title": "A", "edition": "", "year": "2024"}, "A - 2024"),
# Consecutive omitted conditionals collapse to nothing.
("{a?} {b?} {title}", {"a": "", "b": "", "title": "T"}, "T"),
("{show} S{season}E{ep} {name?}", {"show": "Show", "season": "01", "ep": "02", "name": ""}, "Show S01E02"),
],
)
def test_omitted_conditional_leaves_no_separator_debris(template, context, expected):
assert TemplateFormatter(template).format(context) == expected
@pytest.mark.parametrize(
("template", "context"),
[
("{title} ({year?})", {"title": "Some Movie", "year": ""}),
("{title} [{quality?}]", {"title": "Some Movie", "quality": ""}),
("{title}.{quality?}", {"title": "Some Movie", "quality": ""}),
("{title} - {tag?}", {"title": "Some Movie", "tag": ""}),
],
)
def test_result_is_safe_as_a_windows_path_segment(template, context):
"""Windows strips these when creating a directory but not when looking one up."""
result = TemplateFormatter(template).format(context)
assert result == result.strip(" ."), f"unsafe path segment: {result!r}"
def test_empty_result_falls_back():
assert TemplateFormatter("{title?}").format({"title": ""}) == "untitled"
def test_missing_required_variable_raises():
with pytest.raises(ValueError, match="Missing required template variables"):
TemplateFormatter("{title}.{year}").format({"title": "T"})
@pytest.mark.parametrize(
("raw", "spacer", "expected"),
[
("Some Movie ", " ", "Some Movie"),
(" Some Movie", " ", "Some Movie"),
("Some Movie...", ".", "Some.Movie"),
("S.W.A.T.", ".", "S.W.A.T"),
("...Some Movie...", ".", "Some.Movie"),
],
)
def test_sanitize_filename_strips_unsafe_edges(raw, spacer, expected):
assert sanitize_filename(raw, spacer) == expected
def test_sanitize_filename_keeps_interior_dots():
assert sanitize_filename("Mr. Robot", ".") == "Mr.Robot"
@pytest.mark.parametrize(
("template", "context", "expected"),
[
# Space-separated style.
("{title} ({year})", {"title": "Some Movie", "year": "2024"}, "Some Movie (2024)"),
(
"{title} {year} {quality} {source}",
{"title": "Some Movie", "year": "2024", "quality": "1080p", "source": "WEB-DL"},
"Some Movie 2024 1080p WEB-DL",
),
# Dot-separated style.
(
"{title}.{year}.{quality}.{source}",
{"title": "Some Movie", "year": "2024", "quality": "1080p", "source": "WEB-DL"},
"Some.Movie.2024.1080p.WEB-DL",
),
# Interior dots in the title itself survive both styles.
("{title}.{year}", {"title": "Mr. Robot", "year": "2015"}, "Mr.Robot.2015"),
("{title} ({year})", {"title": "Mr. Robot", "year": "2015"}, "Mr. Robot (2015)"),
("{title}.{year}", {"title": "S.W.A.T.", "year": "2017"}, "S.W.A.T.2017"),
("{title} ({year})", {"title": "S.W.A.T.", "year": "2017"}, "S.W.A.T. (2017)"),
# A decimal inside a codec name is not a separator.
(
"{title}.{year}.{audio}-{tag}",
{"title": "Some Movie", "year": "2024", "audio": "DDP5.1", "tag": "GRP"},
"Some.Movie.2024.DDP5.1-GRP",
),
],
)
def test_separator_styles_keep_interior_spacing(template, context, expected):
assert TemplateFormatter(template).format(context) == expected
class DummyService:
pass
class StubMediaInfo:
"""Minimal MediaInfo stand-in: the template context only reads track lists."""
video_tracks: list = []
audio_tracks: list = []
@pytest.fixture
def folder_config(monkeypatch):
"""Clear the global template config, and restore it after each test."""
monkeypatch.setattr(config, "folder_templates", {})
monkeypatch.setattr(config, "folder_template", "")
monkeypatch.setattr(config, "output_template", {})
return config
def make_movie():
return Movie(id_="movie-0001", service=DummyService, name="Some Movie")
def make_episode(**overrides):
kwargs = dict(id_="episode-0001", service=DummyService, title="Some Show", season=1, number=2, name="Pilot")
kwargs.update(overrides)
return Episode(**kwargs)
def make_song(**overrides):
kwargs = dict(
id_="track-0001",
service=DummyService,
name="Some Track",
artist="Some Artist",
album="Some Album",
track=1,
disc=1,
year=2025,
)
kwargs.update(overrides)
return Song(**kwargs)
def make_album_song(**overrides):
kwargs = dict(
id_="track-0001",
service=DummyService,
name="Some Track",
artist="Some Artist",
album="Some Album",
track=1,
disc=1,
year=2025,
album_artist="Some Artist",
)
kwargs.update(overrides)
return MusicSong(**kwargs)
def assert_safe_path(result: str):
"""No segment may start or end with a space or dot, or Windows loses the directory."""
segments = result.split("/")
assert all(segments), f"empty segment in {result!r}"
assert all(seg == seg.strip(" .") for seg in segments), f"unsafe segment in {result!r}"
@pytest.mark.parametrize(
("kind", "make_title", "template", "expected"),
[
("movies", make_movie, "Movies/{title} ({year?})", "Movies/Some Movie"),
("series", make_episode, "Shows/{title} ({year?})/Season {season}", "Shows/Some Show/Season S01"),
("songs", make_song, "Music/{artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"),
("albums", make_album_song, "Music/{album_artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"),
],
)
def test_folder_templates_keep_path_separators(folder_config, kind, make_title, template, expected):
"""`/` survives because each segment is formatted separately, but edge spaces do not."""
folder_config.folder_templates = {kind: template}
result = make_title().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize(
("kind", "make_title", "template"),
[
("movies", make_movie, "{title} ({year?})"),
("series", make_episode, "{title} ({year?})"),
("songs", make_song, "{artist} - {album} [{genre?}]"),
("albums", make_album_song, "{album_artist} - {album} [{genre?}]"),
],
)
def test_folder_names_are_safe_when_conditionals_are_omitted(folder_config, kind, make_title, template):
folder_config.folder_templates = {kind: template}
assert_safe_path(make_title().get_filename(StubMediaInfo(), folder=True))
@pytest.mark.parametrize(
("template", "expected"),
[
("Shows/{title}.{year?}/Season.{season}", "Shows/Some.Show/Season.S01"),
("Shows/{title} {year?}/Season {season}", "Shows/Some Show/Season S01"),
],
)
def test_series_folder_handles_both_separator_styles(folder_config, template, expected):
folder_config.folder_templates = {"series": template}
result = make_episode().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize(
("template", "expected"),
[
# A segment with one variable has no gap to judge by, so it follows the whole
# path's style instead of falling back to a dot.
("Music/{artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"),
("Music/{artist}/{album}.{year?}", "Music/Some.Artist/Some.Album.2025"),
("{artist}/{album}", "Some.Artist/Some.Album"),
],
)
def test_single_variable_segments_follow_the_whole_path(folder_config, template, expected):
folder_config.folder_templates = {"songs": template}
result = make_song().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize(
("template", "expected"),
[
("{title}.{year}.{quality}", "."),
("{title} {year} {quality}", " "),
("{title} ({year})", " "),
("Movies/{title} ({year?})", " "),
("Shows/{title}.{year?}/Season.{season}", "."),
("{title}", "."), # nothing between two variables, so scene style wins
],
)
def test_detect_spacer(template, expected):
assert detect_spacer(template) == expected
@pytest.mark.parametrize(
("series_template", "expected"),
[
# Deriving a show folder strips the episode tokens, so the template ends on a
# separator once {episode_name?} goes too.
("{title}.{year?}.{season_episode}.{episode_name?}", "Some.Show.S01"),
("{title} {year?} {season_episode} {episode_name?}", "Some Show S01"),
],
)
def test_series_folder_derived_from_output_template(folder_config, series_template, expected):
"""With no folder template set, the show folder comes from the file template."""
folder_config.output_template = {"series": series_template}
result = make_episode().get_filename(StubMediaInfo(), folder=True)
assert result == expected
assert_safe_path(result)
@pytest.mark.parametrize("make_title", [make_movie, make_episode, make_song, make_album_song])
def test_folder_fallbacks_need_no_template(folder_config, make_title):
assert_safe_path(make_title().get_filename(StubMediaInfo(), folder=True))
+232
View File
@@ -0,0 +1,232 @@
"""TVDB provider ordering and season-order remapping."""
from __future__ import annotations
import pytest
from unshackle.core import providers
from unshackle.core.config import config
from unshackle.core.providers.tvdb import TVDBProvider, _ids_from_remote, _pick_match
from unshackle.core.utils.tags import _build_tags_from_ids
REMOTE_IDS = [
{"id": "tt0149460", "type": 2, "sourceName": "IMDB"},
{"id": "http://example.com", "type": 4, "sourceName": "Official Website"},
{"id": "615", "type": 12, "sourceName": "TheMovieDB.com"},
]
# Futurama: aired S1 stops at E9, the four held-back episodes open aired S2.
# DVD order folds them back into S1 as E10-E13.
AIRED = [
{"id": 1, "seasonNumber": 1, "number": 9, "name": "Hell Is Other Robots"},
{"id": 2, "seasonNumber": 2, "number": 1, "name": "A Flight to Remember"},
{"id": 3, "seasonNumber": 2, "number": 2, "name": "Mars University"},
{"id": 99, "seasonNumber": 2, "number": 3, "name": "Only In Aired"},
]
DVD = [
{"id": 1, "seasonNumber": 1, "number": 9, "name": "Hell Is Other Robots"},
{"id": 2, "seasonNumber": 1, "number": 10, "name": "A Flight to Remember"},
{"id": 3, "seasonNumber": 1, "number": 11, "name": "Mars University"},
]
def test_ids_from_remote() -> None:
ext = _ids_from_remote(REMOTE_IDS, 73871)
assert ext.imdb_id == "tt0149460"
assert ext.tmdb_id == 615
assert ext.tvdb_id == 73871
def _stub(p: TVDBProvider, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(p, "get_episodes", lambda _id, order: {"official": AIRED, "dvd": DVD}.get(order, []))
def test_order_map_joins_on_episode_id(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
mapping = p.get_order_map(73871, "dvd")
assert mapping[(1, 9)] == (1, 9, "Hell Is Other Robots")
assert mapping[(2, 1)] == (1, 10, "A Flight to Remember")
assert mapping[(2, 2)] == (1, 11, "Mars University")
# an episode absent from the target order is left out, so it keeps its own numbering
assert (2, 3) not in mapping
def test_order_map_reverses_with_source_order(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
mapping = p.get_order_map(73871, "official", source_order="dvd")
assert mapping[(1, 10)] == (2, 1, "A Flight to Remember")
def test_order_map_is_noop_when_orders_match() -> None:
assert TVDBProvider().get_order_map(73871, "official") == {}
def test_order_map_empty_when_order_missing(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
assert p.get_order_map(73871, "alternate") == {}
def test_detect_order_picks_the_order_the_numbering_matches(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
_stub(p, monkeypatch)
# a service may number a series in a non-aired order, so detection must not assume aired
assert p.detect_order(73871, [(1, 10), (1, 11)]) == "dvd"
assert p.detect_order(73871, [(2, 1), (2, 2), (2, 3)]) == "official"
def test_detect_order_falls_back_to_aired(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
monkeypatch.setattr(p, "get_episodes", lambda _id, order: [])
assert p.detect_order(73871, [(1, 1)]) == "official"
# TVDB holds both Battlestar Galactica series; the 2003 revival is filed under year 2005
BSG = [
{"name": "Battlestar Galactica", "year": "1978", "tvdb_id": "71173"},
{"name": "Battlestar Galactica (2003)", "year": "2005", "tvdb_id": "73545"},
]
def test_pick_match_prefers_the_right_era() -> None:
# exact title match on the 1978 series must not beat the era the caller asked for
best, _, _ = _pick_match(BSG, "Battlestar Galactica", 2004)
assert best["tvdb_id"] == "73545"
best, _, _ = _pick_match(BSG, "Battlestar Galactica", 1978)
assert best["tvdb_id"] == "71173"
def test_pick_match_without_a_year_scores_on_title_alone() -> None:
best, _, _ = _pick_match(BSG, "Battlestar Galactica", None)
assert best["tvdb_id"] == "71173"
def test_pick_match_returns_nothing_when_no_year_fits() -> None:
assert _pick_match(BSG, "Battlestar Galactica", 1995)[0] is None
def test_tvdb2_tag_follows_the_matroska_spec() -> None:
# TVDB2 IDs are prefixed by entity type: matroska.org/technical/tagging.html
from unshackle.core.providers import ExternalIds
assert _build_tags_from_ids(ExternalIds(tvdb_id=73871), "tv")["TVDB2"] == "series/73871"
assert _build_tags_from_ids(ExternalIds(tvdb_id=113), "movie")["TVDB2"] == "movies/113"
def test_provider_order_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "metadata_providers", [])
assert [c.NAME for c in providers.provider_order()] == list(providers.DEFAULT_ORDER)
def test_provider_order_is_configurable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "metadata_providers", ["TVDB", "tmdb", "nope", "tvdb"])
# unknown entries dropped, duplicates collapsed, case ignored, order kept
assert [c.NAME for c in providers.provider_order()] == ["tvdb", "tmdb"]
def test_enrichment_authority_is_fixed(monkeypatch: pytest.MonkeyPatch) -> None:
# search order must not demote TMDB below SIMKL for cross-validation tie-breaks
monkeypatch.setattr(config, "metadata_providers", [])
assert providers._enrichment_providers() == ["tmdb", "simkl", "tvdb"]
# `metadata_providers` filters the set but does not re-rank the trust order
monkeypatch.setattr(config, "metadata_providers", ["tvdb", "simkl"])
assert providers._enrichment_providers() == ["simkl", "tvdb"]
def test_cached_tvdb_result_keeps_enriched_ids() -> None:
cached = {
"tvdb_id": "73871",
"name": "Futurama",
"year": "1999",
"_enriched_ids": {"imdb_id": "tt0149460", "tmdb_id": 615},
}
result = providers._cached_to_result(cached, "tvdb", "tv")
assert result is not None
assert result.external_ids.tvdb_id == 73871
assert result.external_ids.imdb_id == "tt0149460"
assert result.external_ids.tmdb_id == 615
# remote_ids stay authoritative over the enriched copies
cached["remote_ids"] = REMOTE_IDS
cached["_enriched_ids"] = {"imdb_id": "tt9999999", "tmdb_id": 1}
result = providers._cached_to_result(cached, "tvdb", "tv")
assert result.external_ids.imdb_id == "tt0149460"
assert result.external_ids.tmdb_id == 615
def test_get_episodes_fails_closed_on_a_truncated_listing(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
pages: dict[int, object] = {0: {"episodes": [{"id": n} for n in range(500)]}, 1: None}
monkeypatch.setattr(p, "_get", lambda _path, params: pages[params["page"]])
# page 1 fails mid-listing: a partial list would renumber wrongly
assert p.get_episodes(73871, "official") == []
assert p._episodes == {}
def test_get_episodes_does_not_cache_an_empty_listing(monkeypatch: pytest.MonkeyPatch) -> None:
p = TVDBProvider()
monkeypatch.setattr(p, "_get", lambda _path, params: None)
assert p.get_episodes(73871, "official") == []
assert p._episodes == {}
# --- apply_tvdb_order over multi-part episodes -------------------------------
class _Svc:
pass
class _FakeProvider:
def __init__(self, mapping: dict) -> None:
self.mapping = mapping
def detect_order(self, _id, _keys) -> str:
return "official"
def get_order_map(self, _id, _order, source_order=None) -> dict:
return self.mapping
def _renumber(monkeypatch: pytest.MonkeyPatch, episodes: list, mapping: dict):
from types import SimpleNamespace
from unshackle.commands.dl import dl
from unshackle.core.titles.episode import Series
monkeypatch.setattr(providers, "get_provider", lambda _name: _FakeProvider(mapping))
errors: list = []
stub = SimpleNamespace(
tvdb_id=73871,
tvdb_order="dvd",
service="STUB",
log=SimpleNamespace(warning=lambda *a, **k: None, info=lambda *a, **k: None, error=lambda *a: errors.append(a)),
)
return dl.apply_tvdb_order(stub, Series(episodes)), errors
def _ep(season: int, number: int, part=None):
from unshackle.core.titles.episode import Episode
return Episode(id_=f"{season}x{number}.{part}", service=_Svc, title="Show", season=season, number=number, part=part)
def test_parts_of_one_episode_renumber_together(monkeypatch: pytest.MonkeyPatch) -> None:
"""Three parts share one (season, number) slot, so they must not read as a collision."""
episodes = [_ep(1, 1, 1), _ep(1, 1, 2), _ep(1, 1, 3), _ep(1, 2)]
mapping = {(1, 1): (2, 1, "Part-ful"), (1, 2): (2, 2, "Whole")}
titles, errors = _renumber(monkeypatch, episodes, mapping)
assert errors == []
assert [(t.season, t.number, t.part) for t in titles] == [(2, 1, 1), (2, 1, 2), (2, 1, 3), (2, 2, None)]
def test_a_real_collision_still_bails(monkeypatch: pytest.MonkeyPatch) -> None:
"""Dedupe must not disable the guard: two distinct episodes on one slot still refuses."""
episodes = [_ep(1, 1), _ep(1, 2)]
mapping = {(1, 2): (1, 1, "Clash")} # (1, 1) is unmapped and keeps the slot (1, 2) moves onto
titles, errors = _renumber(monkeypatch, episodes, mapping)
assert errors # refused, with the "two episodes each" error
assert [(t.season, t.number) for t in titles] == [(1, 1), (1, 2)] # numbering untouched