From 2c8a98f337da6b6833f2fc688f62cc820aab684a Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sat, 18 Jul 2026 11:02:02 -0600 Subject: [PATCH 01/40] feat: original-language aware DUAL/MULTi tagging and new DUBBED variable DUAL now requires the original language plus one dub (strict default); dual_multi_mode: count restores the old pure-count behavior. --- docs/guide/output-and-naming.md | 120 +++++++++++++++++++++++++-- docs/reference/configuration.md | 32 ++++++- tests/core/test_dual_multi_naming.py | 120 +++++++++++++++++++++++++++ unshackle/core/config.py | 2 + unshackle/core/titles/title.py | 20 +++-- unshackle/unshackle-example.yaml | 17 ++-- 6 files changed, 288 insertions(+), 23 deletions(-) create mode 100644 tests/core/test_dual_multi_naming.py diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index aece4fc5..8ecd6965 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -107,7 +107,7 @@ Every variable below is valid in both output and folder templates. Values are de | `quality` | Resolution with scan suffix | `1080p`, `2160p`, `576i` | | `resolution` | Resolution number only | `1080` | | `video` | Video codec | `H.264`, `H.265` | -| `hdr` | Dynamic-range label | `HDR`, `HDR10P`, `DV`, `DV.HDR10`, `HLG` | +| `hdr` | Dynamic-range label | `HDR`, `HDR10P`, `DV`, `DV.HDR`, `DV.HDR10P`, `HLG` | | `hfr` | High frame rate marker (frame rate > 30) | `HFR` | | `edition` | Edition label from the track, if any | `Directors Cut` | | `tag` | Release-group tag (from `config.tag` or `--tag`) | `TAG` | @@ -138,8 +138,9 @@ Every variable below is valid in both output and folder templates. Values are de | `audio_channels` | Channel layout | `5.1`, `2.0` | | `audio_full` | Codec + channels combined | `DDP5.1` | | `atmos` | `Atmos` when any track carries JOC | `Atmos` | -| `dual` | `DUAL` for exactly two distinct audio languages | `DUAL` | +| `dual` | `DUAL` for two audio languages (see [`dual_multi_mode`](../reference/configuration.md#dual_multi_mode)) | `DUAL` | | `multi` | `MULTi` for more than two audio languages | `MULTi` | +| `dubbed` | `DUBBED` for a single audio language that is not the original (strict mode only) | `DUBBED` | ### Music-specific @@ -339,6 +340,9 @@ The fallback supports two placeholders: `{i}` (chapter number, starting at 1) an ## Worked examples +Each tab shows a template and what it produces for a few different downloads, so you can see +how the optional (`?`) variables appear and disappear. + === "Movies" ```yaml @@ -347,9 +351,27 @@ The fallback supports two placeholders: `{i}` (chapter number, starting at 1) an folder: movies: "{title} ({year})" ``` + + A 4K Atmos HDR download fills every placeholder: + ```text The Movie (2024)/ - └─ The.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.Atmos.H.265.HDR-TAG.mkv + └─ The.Movie.2024.2160p.EXAMPLE.WEB-DL.DDP5.1.Atmos.H.265.HDR-TAG.mkv + ``` + + The same template on a plain 1080p SDR stereo download, where `{edition?}`, `{atmos?}` + and `{hdr?}` collapse along with their dots: + + ```text + The Movie (2024)/ + └─ The.Movie.2024.1080p.EXAMPLE.WEB-DL.AAC2.0.H.264-TAG.mkv + ``` + + A director's cut fills `{edition?}`: + + ```text + The Movie (2024)/ + └─ The.Movie.2024.Directors.Cut.2160p.EXAMPLE.WEB-DL.DDP5.1.Atmos.H.265.DV.HDR-TAG.mkv ``` === "TV series" @@ -361,10 +383,80 @@ The fallback supports two placeholders: `{i}` (chapter number, starting at 1) an series: "{title} ({year})/Season {season}" ``` ```text - The Show (2023)/Season S02/ + The Show (2023)/Season.S02/ └─ The.Show.S02E04.The.Reckoning.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG.mkv ``` + Each folder segment picks its separator style independently: `{title} ({year})` has + spaces around its variables, but `Season {season}` alone counts as dot style, so its + space becomes a dot. + + When an episode has no on-screen name, `{episode_name?}` disappears cleanly: + + ```text + The Show (2023)/Season.S02/ + └─ The.Show.S02E05.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG.mkv + ``` + +=== "Multi-language" + + Place `{dual?}`, `{multi?}` and `{dubbed?}` next to each other. At most one of them is + ever set (see [`dual_multi_mode`](../reference/configuration.md#dual_multi_mode)), and the + empty ones collapse: + + ```yaml + output_template: + movies: "{title}.{year}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{dubbed?}.{audio_full}.{video}-{tag}" + ``` + + Original English audio plus a French dub (`-l en,fr`): + + ```text + The.Movie.2024.1080p.EXAMPLE.WEB-DL.DUAL.DDP5.1.H.264-TAG.mkv + ``` + + Original plus several dubs (`-l en,fr,de,es`): + + ```text + The.Movie.2024.1080p.EXAMPLE.WEB-DL.MULTi.DDP5.1.H.264-TAG.mkv + ``` + + Only a dub, original audio left out (`-l de` on an English-original title): + + ```text + The.Movie.2024.1080p.EXAMPLE.WEB-DL.DUBBED.DDP5.1.H.264-TAG.mkv + ``` + + Two dialects of one language (`en-US` + `en-GB`) count as a single language, so none of the + three variables is set: + + ```text + The.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv + ``` + + For finer control (e.g. a `SUBBED` tag driven by subtitle languages), use `{lang_tag?}` + with [`language_tags` rules](../reference/configuration.md#language_tags) instead. + +=== "Daily shows" + + Dated content needs no special template. When an episode carries an air date, + `{season_episode}` becomes the date automatically and `{episode}` and `{year}` clear: + + ```yaml + output_template: + series: "{title}.{season_episode}.{episode_name?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}" + ``` + + A normal episode and a dated one, same template: + + ```text + The.Daily.Show.S28E101.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv + The.Daily.Show.2024.06.01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv + ``` + + The date separator follows your template style: dots here, spaces if your `series` + template uses spaces. + === "Music" ```yaml @@ -375,7 +467,25 @@ The fallback supports two placeholders: `{i}` (chapter number, starting at 1) an ``` ```text The Artist - The Album (2024)/ - └─ 01. Opening Track.mka + └─ 01.Opening.Track.mka + ``` + + In `"{track_number}. {title}"` dots and spaces are tied, and ties go to dot style; add + more spaces between variables (as below) if you want space-separated names. + + A multi-disc album with explicit tracks, using more of the music variables. `{disc}` is + empty on disc 1, so `{disc?}` and its separator only appear from disc 2 onward: + + ```yaml + output_template: + songs: "{disc?}-{track_number}. {title} {explicit?}" + folder: + albums: "{album_artist} - {album} ({year}) [{label?}]" + ``` + ```text + The Artist - The Album (2024) [The Label]/ + ├─ 01. Opening Track.mka + └─ 02-03. Closing Track Explicit.mka ``` ## See also diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index af0e422e..681df844 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -459,6 +459,36 @@ language_tags: tag: "SUBBED" ``` +### `dual_multi_mode` + +- **Type:** `str`  ·  **Default:** `strict` + +Rule set for the `{dual}`, `{multi}`, and `{dubbed}` filename variables. + +=== "`strict` (default)" + + | Variable | Set when | + |----------|----------| + | `{dual}` | Exactly two audio languages **and one is the title's original language**. | + | `{multi}` | Three or more audio languages, even without the original. | + | `{dubbed}` | A single audio language that is not the title's original language (requires a known original). | + +=== "`count` (legacy)" + + | Variable | Set when | + |----------|----------| + | `{dual}` | Exactly two audio languages (original language ignored). | + | `{multi}` | Three or more audio languages. | + | `{dubbed}` | Never set. | + +```yaml +dual_multi_mode: strict +``` + +!!! note "Dialects count as one language" + Regional variants of the same base language (e.g. `en-US` and `en-GB`) collapse to a single + language in both modes, so they never trigger `{dual}` on their own. + --- ## Output & naming { #output-naming } @@ -500,7 +530,7 @@ folder-kind keys. `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, `album_artist`, `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, `explicit`, `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, `dual`, - `multi`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`. + `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`. ### Tagging & naming keys diff --git a/tests/core/test_dual_multi_naming.py b/tests/core/test_dual_multi_naming.py new file mode 100644 index 00000000..6e385275 --- /dev/null +++ b/tests/core/test_dual_multi_naming.py @@ -0,0 +1,120 @@ +"""Tests for {dual}/{multi}/{dubbed} filename variables and dual_multi_mode.""" + +from __future__ import annotations + +import types + +import pytest + +from unshackle.core.config import config +from unshackle.core.titles.movie import Movie + + +class DummyService: + pass + + +def make_audio(language: str) -> types.SimpleNamespace: + """Audio track stub carrying just the attributes the context builder reads.""" + return types.SimpleNamespace( + language=language, + format="AAC", + channel_layout=None, + channellayout_original=None, + channel_s=2, + channels=2, + format_additionalfeatures=None, + joc=None, + ) + + +def make_media_info(languages: list[str]) -> types.SimpleNamespace: + return types.SimpleNamespace(video_tracks=[], audio_tracks=[make_audio(lang) for lang in languages]) + + +def make_movie(language) -> Movie: + return Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language=language) + + +@pytest.fixture +def strict_mode(monkeypatch): + monkeypatch.setattr(config, "dual_multi_mode", "strict") + + +@pytest.fixture +def count_mode(monkeypatch): + monkeypatch.setattr(config, "dual_multi_mode", "count") + + +def ctx(language, languages) -> dict: + return make_movie(language)._build_base_template_context(make_media_info(languages)) + + +def test_strict_dual_with_original(strict_mode): + c = ctx("en", ["en", "es"]) + assert c["dual"] == "DUAL" + assert c["multi"] == "" + assert c["dubbed"] == "" + + +def test_strict_two_dubs_no_original(strict_mode): + c = ctx("en", ["es", "fr"]) + assert c["dual"] == "" + assert c["multi"] == "" + assert c["dubbed"] == "" + + +def test_strict_regional_variants_collapse(strict_mode): + c = ctx("en", ["en-US", "en-GB"]) + assert c["dual"] == "" + assert c["multi"] == "" + assert c["dubbed"] == "" + + +def test_strict_multi(strict_mode): + c = ctx("en", ["es", "fr", "de"]) + assert c["multi"] == "MULTi" + assert c["dual"] == "" + assert c["dubbed"] == "" + + +def test_strict_two_unknown_original(strict_mode): + c = ctx(None, ["en", "es"]) + assert c["dual"] == "" + assert c["multi"] == "" + assert c["dubbed"] == "" + + +def test_strict_dubbed(strict_mode): + c = ctx("en", ["es"]) + assert c["dubbed"] == "DUBBED" + assert c["dual"] == "" + assert c["multi"] == "" + + +def test_strict_single_unknown_original_no_dubbed(strict_mode): + c = ctx(None, ["es"]) + assert c["dubbed"] == "" + assert c["dual"] == "" + assert c["multi"] == "" + + +def test_count_dual_ignores_original(count_mode): + c = ctx(None, ["en", "es"]) + assert c["dual"] == "DUAL" + assert c["multi"] == "" + assert c["dubbed"] == "" + + +def test_count_multi(count_mode): + c = ctx(None, ["es", "fr", "de"]) + assert c["multi"] == "MULTi" + assert c["dual"] == "" + assert c["dubbed"] == "" + + +def test_count_never_dubbed(count_mode): + c = ctx("en", ["es"]) + assert c["dubbed"] == "" + assert c["dual"] == "" + assert c["multi"] == "" diff --git a/unshackle/core/config.py b/unshackle/core/config.py index f8cf4973..92ee1b97 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -126,6 +126,7 @@ class Config: self.redact_paths: bool = kwargs.get("redact_paths", True) self.language_tags: dict = kwargs.get("language_tags") or {} + self.dual_multi_mode: str = (kwargs.get("dual_multi_mode") or "strict").lower() self.output_template: dict = kwargs.get("output_template") or {} folder_cfg = self.output_template.pop("folder", "") self.folder_template: str = "" @@ -191,6 +192,7 @@ class Config: "atmos", "dual", "multi", + "dubbed", "video", "hdr", "hfr", diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index 9aeb30c0..b205cce0 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -74,7 +74,8 @@ class Title: ) if primary_audio_track is None: primary_audio_track = next(iter(media_info.audio_tracks), None) - unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) + audio_lang_bases = {x.language.split("-")[0].lower() for x in media_info.audio_tracks if x.language} + unique_audio_languages = len(audio_lang_bases) context: dict[str, Any] = { "source": self.service.__name__ if show_service else "", @@ -88,6 +89,7 @@ class Title: "atmos": "", "dual": "", "multi": "", + "dubbed": "", "video": "", "hdr": "", "hfr": "", @@ -179,15 +181,15 @@ class Title: } ) - if unique_audio_languages == 2: - context["dual"] = "DUAL" - context["multi"] = "" - elif unique_audio_languages > 2: - context["dual"] = "" + strict = config.dual_multi_mode != "count" + if unique_audio_languages > 2: context["multi"] = "MULTi" - else: - context["dual"] = "" - context["multi"] = "" + elif unique_audio_languages == 2: + if not strict or (original_lang_tag and original_lang_tag in audio_lang_bases): + context["dual"] = "DUAL" + elif unique_audio_languages == 1 and strict: + if original_lang_tag and original_lang_tag not in audio_lang_bases: + context["dubbed"] = "DUBBED" lang_tag_rules = config.language_tags.get("rules") if config.language_tags else None if lang_tag_rules and self.tracks: diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 4db3e1a9..e0150f20 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -9,18 +9,19 @@ # Group or username appended to download filenames after a dash. tag: "NOGRP" -# --- Output filename & folder templates --- docs/OUTPUT_CONFIG.md -# Full variable list (35 vars incl. music) and all naming options are -# documented in docs/OUTPUT_CONFIG.md. If omitted, scene-style defaults are -# used (with a warning). Conditional variables take a ? suffix, e.g. {year?}. +# --- Output filename & folder templates --- docs/guide/output-and-naming.md +# The full variable list and all naming options are documented there. +# movies/series have no built-in fallback (define them to download those kinds); +# songs falls back to "{track_number}. {title}". Conditional variables take a +# ? suffix, e.g. {year?}. output_template: # Scene-style naming (dot-separated) - movies: "{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" - series: "{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" + movies: "{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{dubbed?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" + series: "{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{dubbed?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" songs: "{track_number}.{title}.{repack?}.{edition?}.{lang_tag?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}" folder: - movies: "{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" - series: "{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" + movies: "{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{dubbed?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" + series: "{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{dubbed?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}" albums: "{album_artist} - {album} ({year?})" # From 446367f58dbd237ca6f47427d91cbe30b29de302 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sun, 19 Jul 2026 10:21:47 -0600 Subject: [PATCH 02/40] fix: skip decrypt on mislabeled-clear segments --- unshackle/core/drm/clearkey.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/unshackle/core/drm/clearkey.py b/unshackle/core/drm/clearkey.py index f22aa906..43f93aa7 100644 --- a/unshackle/core/drm/clearkey.py +++ b/unshackle/core/drm/clearkey.py @@ -53,7 +53,16 @@ class ClearKey: file=path.name, ) - decrypted = AES.new(self.key, AES.MODE_CBC, self.iv).decrypt(path.read_bytes()) + data = path.read_bytes() + + # Guard mislabeled-clear: EXT-X-KEY says AES-128 but the segment is plaintext MPEG-TS. + # CBC ciphertext is 16-aligned, but so are some clear TS runs (752 bytes = 4 packets), + # so also sniff the 0x47 sync byte at packet 0 and 1. + is_ts = data[:1] == b"\x47" and data[188:189] == b"\x47" + if len(data) % AES.block_size != 0 or is_ts: + return + + decrypted = AES.new(self.key, AES.MODE_CBC, self.iv).decrypt(data) try: decrypted = unpad(decrypted, AES.block_size) From 95a754fa6ca85e10260fa31aca3bdc3149601db1 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sun, 19 Jul 2026 10:49:45 -0600 Subject: [PATCH 03/40] fix: harden ClearKey/CENC decryption correctness --- tests/core/test_clearkey.py | 178 ++++++++++++++++++++++++++++ unshackle/commands/dl.py | 5 +- unshackle/core/drm/clearkey.py | 56 +++++++-- unshackle/core/drm/clearkey_cenc.py | 23 +++- unshackle/core/manifests/hls.py | 9 +- 5 files changed, 255 insertions(+), 16 deletions(-) create mode 100644 tests/core/test_clearkey.py diff --git a/tests/core/test_clearkey.py b/tests/core/test_clearkey.py new file mode 100644 index 00000000..f3df8116 --- /dev/null +++ b/tests/core/test_clearkey.py @@ -0,0 +1,178 @@ +"""Tests for the HLS AES-128 ClearKey DRM system (``unshackle.core.drm.clearkey``). + +Covers the correctness guards on the whole-segment AES-128-CBC path: +- ``ClearKey.decrypt`` passing mislabeled-clear / non-block-aligned segments through + untouched instead of crashing or corrupting (WS1 / F1+F3) +- ``ClearKey.from_m3u_key`` ``data:`` URI parsing and 16-byte key validation (WS5 / F4) +- the RFC 8216 §5.2 sequence-number IV deriving from a segment's absolute media + sequence number rather than the post-filter download index (WS2 / F2) +""" + +from __future__ import annotations + +import base64 + +import m3u8 +import pytest +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import pad +from m3u8.model import Key + +from unshackle.core.drm.clearkey import SYNC_BYTE, TS_PACKET_SIZE, ClearKey + +KEY = bytes.fromhex("00112233445566778899aabbccddeeff") +IV = bytes.fromhex("0f0e0d0c0b0a09080706050403020100") + + +def encrypt(plaintext: bytes, key: bytes = KEY, iv: bytes = IV) -> bytes: + return AES.new(key, AES.MODE_CBC, iv).encrypt(pad(plaintext, AES.block_size)) + + +def clear_ts(packets: int = 4) -> bytes: + # Plaintext MPEG-TS: sync byte 0x47 at every 188-byte packet boundary. Default 4 packets so + # the fixture satisfies the hardened 4-boundary sniff (and 752 B is 16-aligned, so it must be + # caught by the clear-TS guard, not the len%16 guard). + return b"".join(bytes([SYNC_BYTE]) + b"\x00" * (TS_PACKET_SIZE - 1) for _ in range(packets)) + + +def test_valid_encrypted_segment_decrypts(tmp_path) -> None: + plaintext = b"the quick brown fox jumps over" * 4 + seg = tmp_path / "0.ts" + seg.write_bytes(encrypt(plaintext)) + + ClearKey(key=KEY, iv=IV).decrypt(seg) + + assert seg.read_bytes() == plaintext + + +def test_clear_ts_passes_through(tmp_path) -> None: + data = clear_ts() + seg = tmp_path / "0.ts" + seg.write_bytes(data) + + ClearKey(key=KEY, iv=IV).decrypt(seg) + + assert seg.read_bytes() == data + + +def test_non_block_aligned_passes_through(tmp_path) -> None: + data = b"\x01" * 17 # not TS, and not a 16-byte multiple -> not valid CBC ciphertext + assert len(data) % AES.block_size != 0 + seg = tmp_path / "0.ts" + seg.write_bytes(data) + + ClearKey(key=KEY, iv=IV).decrypt(seg) + + assert seg.read_bytes() == data + + +def test_empty_segment_does_not_crash(tmp_path) -> None: + seg = tmp_path / "0.ts" + seg.write_bytes(b"") + + ClearKey(key=KEY, iv=IV).decrypt(seg) + + assert seg.exists() + assert seg.read_bytes() == b"" + + +def test_warns_once_per_instance(tmp_path, caplog) -> None: + drm = ClearKey(key=KEY, iv=IV) + for name in ("0.ts", "1.ts"): + seg = tmp_path / name + seg.write_bytes(clear_ts()) + with caplog.at_level("WARNING"): + drm.decrypt(seg) + + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + + +def _data_key(raw: bytes, base64_encoded: bool = True) -> Key: + if base64_encoded: + uri = "data:text/plain;base64," + base64.b64encode(raw).decode("ascii") + else: + uri = "data:text/plain," + raw.decode("latin-1") + return Key(method="AES-128", uri=uri, base_uri="", keyformat=None, keyformatversions=None, iv=None) + + +def test_data_uri_key_decodes_and_validates_length() -> None: + drm = ClearKey.from_m3u_key(_data_key(KEY)) + assert drm.key == KEY + + +def test_data_uri_key_wrong_length_rejected() -> None: + with pytest.raises(ValueError, match="Unexpected Length"): + ClearKey.from_m3u_key(_data_key(b"\x00" * 8)) + + +def test_raw_data_uri_key_materialises_and_validates() -> None: + # A raw (non-base64) data: payload must be taken as literal key bytes, not hex-parsed. + drm = ClearKey.from_m3u_key(_data_key(KEY, base64_encoded=False)) + assert drm.key == KEY + + +def test_raw_data_uri_key_wrong_length_rejected() -> None: + with pytest.raises(ValueError, match="Unexpected Length"): + ClearKey.from_m3u_key(_data_key(b"\x01" * 8, base64_encoded=False)) + + +def test_data_uri_split_tolerates_commas_in_payload() -> None: + # base64 with padding contains no comma, but the split must only break on the first comma + # so a payload that itself contains a comma is not truncated. Encode a 16-byte key whose + # base64 happens to be split-safe, and assert the mediatype comma is the only split point. + raw = bytes(range(16)) + b64 = base64.b64encode(raw).decode("ascii") + key = Key( + method="AES-128", + uri=f"data:application/octet-stream;base64,{b64}", + base_uri="", + keyformat=None, + keyformatversions=None, + iv=None, + ) + drm = ClearKey.from_m3u_key(key) + assert drm.key == raw + + +SEQ_PLAYLIST = """#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:6 +#EXT-X-MEDIA-SEQUENCE:100 +#EXT-X-KEY:METHOD=AES-128,URI="skd://key" +#EXTINF:6.0, +seg0.ts +#EXTINF:6.0, +seg1.ts +#EXTINF:6.0, +seg2.ts +#EXTINF:6.0, +seg3.ts +#EXT-X-ENDLIST +""" + + +def test_sequence_number_iv_uses_absolute_media_sequence() -> None: + """IV must be the segment's absolute media sequence number, not the download index. + + Mirrors the derivation in ``HLS.download_track``: filtered segments are dropped, the + surviving segments are named by post-filter download index, and the IV comes from the + wanted segment's ``media_sequence`` (EXT-X-MEDIA-SEQUENCE + absolute position). + """ + master = m3u8.loads(SEQ_PLAYLIST) + # media sequence numbers are the offset (100) plus absolute playlist position. + assert [s.media_sequence for s in master.segments] == [100, 101, 102, 103] + + # Drop an early segment (index 1), as an OnSegmentFilter would. + unwanted = {master.segments[1]} + wanted_segments = [s for s in master.segments if s not in unwanted] + + # download index 1 is the SECOND surviving segment (absolute seq 102), not 101. + download_i = 1 + seq = wanted_segments[download_i].media_sequence + assert seq == 102 + + iv = int(seq).to_bytes(16, "big") + assert iv == (102).to_bytes(16, "big") + # the old download-index derivation would have produced the wrong IV (100 + 1 = 101) + assert iv != (100 + download_i).to_bytes(16, "big") diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index a8c71551..708c76c3 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -1851,6 +1851,7 @@ class dl: cdm_only=cdm_only, vaults_only=vaults_only, export=export_path, + service_session=service.session, ), cdm=self.cdm, max_workers=workers, @@ -2921,6 +2922,7 @@ class dl: cdm_only=cdm_only, vaults_only=vaults_only, export=export_path, + service_session=service.session, ), cdm=self.cdm, max_workers=workers, @@ -3645,6 +3647,7 @@ class dl: cdm_only: bool = False, vaults_only: bool = False, export: Optional[Path] = None, + service_session: Optional[Any] = None, ) -> None: """ Prepare the DRM by getting decryption data like KIDs, Keys, and such. @@ -4175,7 +4178,7 @@ class dl: from_vaults = drm.content_keys.copy() try: - drm.get_content_keys(licence=clearkey_licence or (lambda **_: None)) + drm.get_content_keys(licence=clearkey_licence or (lambda **_: None), session=service_session) except Exception as e: if drm.content_keys: self.log.debug(f"License call failed but keys already in content_keys: {e}") diff --git a/unshackle/core/drm/clearkey.py b/unshackle/core/drm/clearkey.py index 43f93aa7..a1732769 100644 --- a/unshackle/core/drm/clearkey.py +++ b/unshackle/core/drm/clearkey.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import logging import shutil from pathlib import Path from typing import Optional, Union @@ -14,6 +15,14 @@ from requests import Session from unshackle.core.session import RnetSession from unshackle.core.utilities import log_event +log = logging.getLogger(__name__) + +# MPEG-TS: 188-byte packets, sync byte 0x47 at every packet boundary (shared with sample_aes). +SYNC_BYTE = 0x47 +TS_PACKET_SIZE = 188 +# Consecutive packet boundaries that must carry the sync byte to call a segment clear MPEG-TS. +TS_SYNC_CHECKS = 4 + class ClearKey: """AES Clear Key DRM System.""" @@ -39,12 +48,38 @@ class ClearKey: self.key: bytes = key self.iv: bytes = iv + self._warned_clear: bool = False + + def _warn_clear(self, reason: str) -> None: + """Warn once per track that a segment was passed through undecrypted.""" + if not self._warned_clear: + log.warning("ClearKey: manifest signaled AES-128 encryption but %s; passing segment through.", reason) + self._warned_clear = True def decrypt(self, path: Path) -> None: """Decrypt a Track with AES Clear Key DRM.""" if not path or not path.exists(): raise ValueError("Tried to decrypt a file that does not exist.") + data = path.read_bytes() + + # Mislabeled-clear guard: plaintext MPEG-TS the manifest wrongly tags AES-128. Require the + # 0x47 sync byte at 4 consecutive packet boundaries plus an exact packet-count length, so + # real CBC ciphertext matching all of these is negligible (~1/256^4). + if ( + len(data) % TS_PACKET_SIZE == 0 + and len(data) >= TS_PACKET_SIZE * TS_SYNC_CHECKS + and all(data[i * TS_PACKET_SIZE] == SYNC_BYTE for i in range(TS_SYNC_CHECKS)) + ): + self._warn_clear("the segment is clear MPEG-TS") + return + + # Valid AES-128-CBC ciphertext is always a 16-byte multiple; anything else is not + # ciphertext (truncated/clear/SAMPLE-AES). Decrypting would raise, so pass it through. + if len(data) % AES.block_size != 0: + self._warn_clear(f"the segment length ({len(data)} bytes) is not a multiple of {AES.block_size}") + return + log_event( "drm_decrypt", level="DEBUG", @@ -53,15 +88,6 @@ class ClearKey: file=path.name, ) - data = path.read_bytes() - - # Guard mislabeled-clear: EXT-X-KEY says AES-128 but the segment is plaintext MPEG-TS. - # CBC ciphertext is 16-aligned, but so are some clear TS runs (752 bytes = 4 packets), - # so also sniff the 0x47 sync byte at packet 0 and 1. - is_ts = data[:1] == b"\x47" and data[188:189] == b"\x47" - if len(data) % AES.block_size != 0 or is_ts: - return - decrypted = AES.new(self.key, AES.MODE_CBC, self.iv).decrypt(data) try: @@ -105,11 +131,17 @@ class ClearKey: session.headers["User-Agent"] = "smartexoplayer/1.1.0 (Linux;Android 8.0.0) ExoPlayerLib/2.13.3" if m3u_key.uri.startswith("data:"): - media_types, data = m3u_key.uri[5:].split(",") + media_types, payload = m3u_key.uri[5:].split(",", 1) media_types = media_types.split(";") + # Materialise the key bytes here so the 16-byte check covers both encodings: base64 + # decodes to bytes, a raw payload is latin-1 bytes. A raw str would otherwise reach + # __init__ and be reinterpreted through fromhex. if "base64" in media_types: - data = base64.b64decode(data) - key = data + key = base64.b64decode(payload) + else: + key = payload.encode("latin-1") + if len(key) != 16: + raise ValueError(f"Unexpected Length of Key ({len(key)} bytes) in M3U data: URI.") else: url = urljoin(m3u_key.base_uri, m3u_key.uri) log_event( diff --git a/unshackle/core/drm/clearkey_cenc.py b/unshackle/core/drm/clearkey_cenc.py index 5e72376d..d0d1c834 100644 --- a/unshackle/core/drm/clearkey_cenc.py +++ b/unshackle/core/drm/clearkey_cenc.py @@ -132,14 +132,33 @@ class ClearKeyCENC: raise ValueError(f"Expected the ClearKey license to be bytes, str, or dict, not {response!r}") for jwk in document.get("keys") or []: - if jwk.get("kty") not in (None, "oct"): + kty = jwk.get("kty") + if kty not in (None, "oct"): continue + if kty is None: + log_event( + "drm_license_warning", + level="WARNING", + message="ClearKey JWK is missing 'kty'; assuming 'oct' per W3C EME", + drm_type="ClearKeyCENC", + ) kid_b64 = jwk.get("kid") key_b64 = jwk.get("k") if not kid_b64 or not key_b64: continue kid = UUID(bytes=b64url_decode(kid_b64)) - self.content_keys[kid] = b64url_decode(key_b64).hex() + key = b64url_decode(key_b64) + if len(key) != 16: + # A CENC content key is always 16 bytes; a wrong-length key would make the + # external tool fail opaquely. Skip it rather than pass it through. + log_event( + "drm_license_warning", + level="WARNING", + message=f"ClearKey JWK for KID {kid.hex} has a {len(key)}-byte key (expected 16); skipping", + drm_type="ClearKeyCENC", + ) + continue + self.content_keys[kid] = key.hex() if not self.content_keys: raise ClearKeyCENC.Exceptions.EmptyLicense("No Content Keys were within the License") diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 0c0a41d6..8a61a26b 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -709,6 +709,9 @@ class HLS: # First segment's Media Sequence Number - used as the AES-128 IV when EXT-X-KEY has # no explicit IV (RFC 8216 §5.2), where each segment's IV is its sequence number. media_sequence_start = getattr(master, "media_sequence", None) or 0 + # Downloaded segment files are named by post-filter index; map that back to the wanted + # segment so the IV uses the absolute media sequence number, not the download index. + wanted_segments = [seg for seg in master.segments if seg not in unwanted_segments] map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None if session_drm: encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (initial_drm_key, session_drm) @@ -802,7 +805,11 @@ class HLS: seq_iv = isinstance(drm, ClearKey) and not (key_obj and key_obj.iv) for file in files: if seq_iv and file.stem.isdigit(): - drm.iv = (media_sequence_start + int(file.stem)).to_bytes(16, "big") + download_i = int(file.stem) + seq = getattr(wanted_segments[download_i], "media_sequence", None) + if seq is None: + seq = media_sequence_start + download_i + drm.iv = int(seq).to_bytes(16, "big") drm.decrypt(file) merge(to=merged_path, via=files, delete=True, include_map_data=True) From 51750a03b0ac4d25db019a9d1a665d242023211e Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Wed, 22 Jul 2026 09:18:14 -0600 Subject: [PATCH 04/40] feat(drm): add PlayReady HRESULT decoder Maps DRM_E_/DRM_S_ result codes to symbol + description so services can turn raw license HRESULTs (e.g. 0x8004C065) into readable errors. --- unshackle/core/drm/playready_errors.py | 865 +++++++++++++++++++++++++ 1 file changed, 865 insertions(+) create mode 100644 unshackle/core/drm/playready_errors.py diff --git a/unshackle/core/drm/playready_errors.py b/unshackle/core/drm/playready_errors.py new file mode 100644 index 00000000..0560eaed --- /dev/null +++ b/unshackle/core/drm/playready_errors.py @@ -0,0 +1,865 @@ +# Generated from PlayReady drmresults.h (DRM_E_/DRM_S_ result codes). + +from __future__ import annotations + +# {unsigned_hresult: (symbol, description)} +ERRORS: dict[int, tuple[str, str]] = { + 0x00000001: ("DRM_S_FALSE", "Operation was successful, but returned a FALSE test condition."), + 0x00000002: ("DRM_S_MORE_DATA", "Operation was successful, but more data is available."), + 0x0004C300: ("DRM_S_TEST_SKIP_FILE", "Skip processing this file, not an eror."), + 0x0004C301: ("DRM_S_TEST_CONVERTED_FILE", "The file was converted to a PlayReady file during the action."), + 0x80000002: ("DRM_E_OUTOFMEMORY", "Insufficient resources exist to complete the request."), + 0x80004001: ("DRM_E_NOTIMPL", "The requested operation is not implemented."), + 0x80004005: ("DRM_E_FAIL", "The requested operation failed."), + 0x80004006: ("DRM_E_HLOS_TAMPERED", "HLOS tampered."), + 0x80004007: ("DRM_E_OPL_BLOCKED", "OPL blocked."), + 0x80004008: ("DRM_E_LOAD_IMG", "Failed on loading playready image."), + 0x80004009: ("DRM_E_VER_MISMATCH", "Version mismatch between HLOS and TZ."), + 0x8000400A: ("DRM_E_SET_BANDWIDTH", "Failed on setting bandwidth."), + 0x8000400B: ("DRM_E_OUT_OF_BOUND", "Out of bound."), + 0x8000400C: ( + "DRM_E_PLAY_ENABLER_BLOCKED", + "WFD is blocked as play enabler object for HDCP doesn't exist in license", + ), + 0x8000400E: ("DRM_E_HDMI_READ_FAIL", "Failed to read HDMI Status"), + 0x80030002: ("DRM_E_FILENOTFOUND", "A requested file could not be found."), + 0x8003006E: ("DRM_E_FILEOPEN", "A request failed due to a file being open."), + 0x80040E80: ("DRM_E_VERIFICATION_FAILURE", "Validation of a Longhorn certificate failed."), + 0x80040E82: ("DRM_E_RSA_SIGNATURE_ERROR", "Error in RSA(PSS) signature."), + 0x80040E86: ("DRM_E_BAD_RSA_EXPONENT", "An incorrect RSA exponent was supplied for a public key."), + 0x80040E87: ("DRM_E_P256_CONVERSION_FAILURE", "An error occurred while converting between P256 types."), + 0x80040E88: ("DRM_E_P256_PKCRYPTO_FAILURE", "An error occurred in an asymmetric P256 cryptographic operation."), + 0x80040E89: ( + "DRM_E_P256_PLAINTEXT_MAPPING_FAILURE", + "An error occurred while attempting to map a plaintext array to a EC Point: There is no conversion for this byte array to a EC Point.", + ), + 0x80040E8A: ( + "DRM_E_P256_INVALID_SIGNATURE", + "The ECDSA signature to be verified was not a valid signature format.", + ), + 0x80040E8B: ( + "DRM_E_P256_ECDSA_VERIFICATION_ERROR", + "The ECDSA verification algorithm encountered an unknown error.", + ), + 0x80040E8C: ("DRM_E_P256_ECDSA_SIGNING_ERROR", "The ECDSA signature algorithm encountered an unknown error."), + 0x80040E8D: ( + "DRM_E_P256_HMAC_KEYGEN_FAILURE", + "Could not generate a valid HMAC key under constraint where CK || HMACK is a valid x coord on the EC (P256).", + ), + 0x80041100: ("DRM_E_CH_BASECODE", ""), + 0x80041103: ("DRM_E_CH_VERSION_MISSING", "Missing content header version."), + 0x80041104: ("DRM_E_CH_KID_MISSING", "Missing KID attribute in content header."), + 0x80041105: ("DRM_E_CH_LAINFO_MISSING", "Missing LAINFO attribute in content header."), + 0x80041106: ("DRM_E_CH_CHECKSUM_MISSING", "Missing content header checksum."), + 0x80041107: ("DRM_E_CH_ATTR_MISSING", "Missing content header attribute."), + 0x80041108: ("DRM_E_CH_INVALID_HEADER", "Invalid content header."), + 0x80041109: ("DRM_E_CH_INVALID_CHECKSUM", "Invalid checksum in the header."), + 0x8004110A: ("DRM_E_CH_UNABLE_TO_VERIFY", "Unable to verify signature of content header."), + 0x8004110B: ("DRM_E_CH_UNSUPPORTED_VERSION", "Unsupported content header version."), + 0x8004110C: ("DRM_E_CH_UNSUPPORTED_HASH_ALGORITHM", "Unsupported hash algorithm."), + 0x8004110D: ("DRM_E_CH_UNSUPPORTED_SIGN_ALGORITHM", "Unsupported signature algorithm."), + 0x8004110E: ("DRM_E_CH_BAD_KEY", "Invalid key."), + 0x8004110F: ("DRM_E_CH_INCOMPATIBLE_HEADER_TYPE", "Incompatible content header type."), + 0x80041110: ("DRM_E_HEADER_ALREADY_SET", "Content header type is already set. Reinitialize is required."), + 0x80041113: ("DRM_E_CH_NOT_SIGNED", "The header was not signed."), + 0x80041116: ("DRM_E_CH_UNKNOWN_ERROR", "Unknown Error."), + 0x80048000: ("DRM_E_LEGACY_BASECODE", ""), + 0x80048007: ("DRM_E_LIC_KEY_DECODE_FAILURE", "Key decode failure."), + 0x80048008: ("DRM_E_LIC_SIGNATURE_FAILURE", "License signature failure."), + 0x80048013: ("DRM_E_LIC_KEY_AND_CERT_MISMATCH", "Key and cert mismatch."), + 0x80048014: ("DRM_E_KEY_MISMATCH", "A public/private keypair is mismatched."), + 0x800480CF: ("DRM_E_INVALID_SIGNATURE", "License signature failure."), + 0x800480D0: ("DRM_E_SYNC_ENTRYNOTFOUND", "An entry was not found in the sync store."), + 0x800480D1: ("DRM_E_STACKTOOSMALL", "The stack supplied to the DRM API was too small."), + 0x800480D2: ( + "DRM_E_CIPHER_NOTINITIALIZED", + "The DRM Cipher routines were not correctly initialized before calling encryption/decryption routines.", + ), + 0x800480D3: ( + "DRM_E_DECRYPT_NOTINITIALIZED", + "The DRM decrypt routines were not correctly initialized before trying to decrypt data.", + ), + 0x800480D4: ( + "DRM_E_SECURESTORE_LOCKNOTOBTAINED", + "Before reading or writing data to securestore in raw mode, first the lock must be obtained using DRM_SST_OpenData.", + ), + 0x800480D5: ("DRM_E_PKCRYPTO_FAILURE", "An error occurred in an asymmetric cryptographic operation."), + 0x800480D6: ("DRM_E_INVALID_DSTSLOTSIZE", "Invalid DST slot size is specified."), + 0x8004A000: ("DRM_E_DEVICE_NOT_REGISTERED", "The DEVICEID does not exist in the device store"), + 0x8004A001: ( + "DRM_E_TOO_MANY_INCLUSION_GUIDS", + "The license contained more than DRM_MAX_INCLUSION_GUIDS entries in its inclusion list", + ), + 0x8004A002: ("DRM_E_REVOCATION_GUID_NOT_RECOGNIZED", "The revocation list type GUID was not recognized"), + 0x8004A003: ("DRM_E_LIC_CHAIN_TOO_DEEP", "The license chained deeper than this implementation can handle"), + 0x8004A004: ( + "DRM_E_DEVICE_SECURITY_LEVEL_TOO_LOW", + "The security level of the remote device is too low to receive the license", + ), + 0x8004A005: ("DRM_E_DST_BLOCK_CACHE_CORRUPT", "The block header cache returned invalid data"), + 0x8004A006: ("DRM_E_CONTRACT_FAILED", "The error code returned by the API is not present in the contract"), + 0x8004A007: ("DRM_E_DST_BLOCK_CACHE_MISS", "The block header cache didn't contain the requested block header"), + 0x8004A013: ("DRM_E_INVALID_METERRESPONSE_SIGNATURE", "Invalid signature in meter response"), + 0x8004A014: ("DRM_E_INVALID_LICENSE_REVOCATION_LIST_SIGNATURE", "Invalid signature in license revocation list."), + 0x8004A015: ("DRM_E_INVALID_METERCERT_SIGNATURE", "Invalid signature in metering certificate"), + 0x8004A016: ("DRM_E_METERSTORE_DATA_NOT_FOUND", "Metering data slot not found due to bad data in response file"), + 0x8004A017: ("DRM_E_NO_LICENSES_TO_SYNC", "No more licenses to sync"), + 0x8004A018: ( + "DRM_E_INVALID_REVOCATION_LIST", + "The revocation list version does not match the current revocation version", + ), + 0x8004A019: ("DRM_E_ENVELOPE_CORRUPT", "The envelope archive or file is corrupt"), + 0x8004A01A: ( + "DRM_E_ENVELOPE_FILE_NOT_COMPATIBLE", + "The envelope file is not compatible with this version of the porting kit", + ), + 0x8004A01B: ( + "DRM_E_EXTENDED_RESTRICTION_NOT_UNDERSTOOD", + "An extensible restriction was not understood by the app, and is mark as being required", + ), + 0x8004A01C: ("DRM_E_INVALID_SLK", "An ILA SLK (symmetric session key) was found, but did not contain valid data"), + 0x8004A01D: ( + "DRM_E_DEVCERT_MODEL_MISMATCH", + "The model string in the certificate does not match the model of the device and so the cert must be re-generated.", + ), + 0x8004C001: ("DRM_E_DEVICENOTINIT", "This device has not been initialized against a DRM init service"), + 0x8004C002: ("DRM_E_DRMNOTINIT", "The app has not call DRM_Init properly"), + 0x8004C003: ("DRM_E_INVALIDRIGHT", "A right in the license in invalid"), + 0x8004C004: ( + "DRM_E_INCOMPATABLELICENSESIZE", + "The size of the license is incompatable. DRM doesn't understand this license", + ), + 0x8004C005: ( + "DRM_E_INVALIDLICENSEFLAGS", + "The flags in the license are invalid. DRM either doesn't understand them or they are conflicting", + ), + 0x8004C006: ("DRM_E_INVALIDLICENSE", "The license is invalid"), + 0x8004C007: ("DRM_E_CONDITIONFAIL", "A condition in the license failed to pass"), + 0x8004C008: ("DRM_E_CONDITIONNOTSUPPORTED", "A condition in the license is not supported by this verison of DRM"), + 0x8004C009: ( + "DRM_E_LICENSEEXPIRED", + "The license has expired either by depleting a play count or via an end time.", + ), + 0x8004C00A: ("DRM_E_LICENSENOTYETVALID", "The license start time had not come to pass yet."), + 0x8004C00B: ("DRM_E_RIGHTSNOTAVAILABLE", "The rights the app has requested are not available in the license"), + 0x8004C00C: ("DRM_E_LICENSEMISMATCH", "The license content id/ sku id doesn't match that requested by the app"), + 0x8004C00D: ("DRM_E_WRONGTOKENTYPE", "The token parameter was of an incompatible type."), + 0x8004C00E: ("DRM_E_NORIGHTSREQUESTED", "The app has not requested any rights before trying to bind"), + 0x8004C00F: ( + "DRM_E_LICENSENOTBOUND", + "A license has not been bound to. Decrypt can not happen without a successful bind call", + ), + 0x8004C010: ("DRM_E_HASHMISMATCH", "A Keyed Hash check failed."), + 0x8004C011: ("DRM_E_INVALIDTIME", "A time structure is invalid."), + 0x8004C012: ("DRM_E_LICENSESTORENOTFOUND", "The external license store was not found."), + 0x8004C013: ("DRM_E_LICENSENOTFOUND", "A license was not found in the license store."), + 0x8004C014: ( + "DRM_E_LICENSEVERSIONNOTSUPPORTED", + "The DRM license version is not supported by the DRM version on the device.", + ), + 0x8004C015: ("DRM_E_INVALIDBINDID", "The bind id is invalid."), + 0x8004C016: ( + "DRM_E_UNSUPPORTEDALGORITHM", + "The encryption algorithm required for this operation is not supported.", + ), + 0x8004C017: ("DRM_E_ALGORITHMNOTSET", "The encryption algorithm required for this operation is not supported."), + 0x8004C018: ( + "DRM_E_LICENSESERVERNEEDSKEY", + "The license server needs a version of the device bind key from the init service.", + ), + 0x8004C019: ( + "DRM_E_INVALIDLICENSESTORE", + "The license store version number is incorrect, or the store is invalid in some other way.", + ), + 0x8004C01A: ("DRM_E_FILEREADERROR", "There was an error reading a file."), + 0x8004C01B: ("DRM_E_FILEWRITEERROR", "There was an error writing a file."), + 0x8004C01C: ( + "DRM_E_CLIENTTIMEINVALID", + "The time/clock on the device is not in sync with the license server within tolerance.", + ), + 0x8004C01D: ("DRM_E_DSTSTOREFULL", "The data store is full."), + 0x8004C01E: ("DRM_E_NOXMLOPENTAG", "XML open tag not found"), + 0x8004C01F: ("DRM_E_NOXMLCLOSETAG", "XML close tag not found"), + 0x8004C020: ("DRM_E_INVALIDXMLTAG", "Invalid XML tag"), + 0x8004C021: ("DRM_E_NOXMLCDATA", "No XML CDATA found"), + 0x8004C022: ("DRM_E_DSTNAMESPACEFULL", "No more room for DST Namespace"), + 0x8004C023: ("DRM_E_DSTNAMESPACENOTFOUND", "No DST Namespace found"), + 0x8004C024: ("DRM_E_DSTSLOTNOTFOUND", "DST Dataslot not found"), + 0x8004C025: ("DRM_E_DSTSLOTEXIST", "DST Dataslot already exists"), + 0x8004C026: ("DRM_E_DSTCORRUPTED", "The data store is corrupted"), + 0x8004C027: ("DRM_E_DSTSEEKERROR", "There was an error attempting to seek in the Data Store"), + 0x8004C028: ("DRM_E_DSTNAMESPACEINUSE", "No DST Namespace in use"), + 0x8004C029: ( + "DRM_E_INVALID_SECURESTORE_PASSWORD", + "The password used to open the secure store key was not able to validate the secure store hash.", + ), + 0x8004C02A: ("DRM_E_SECURESTORE_CORRUPT", "The secure store is corrupt"), + 0x8004C02B: ("DRM_E_SECURESTORE_FULL", "The current secure store key is full. No more data can be added."), + 0x8004C02C: ("DRM_E_NOACTIONINLICENSEREQUEST", "No action(s) added for license request"), + 0x8004C02D: ("DRM_E_DUPLICATEDHEADERATTRIBUTE", "Duplicated attribute in Header"), + 0x8004C02E: ("DRM_E_NOKIDINHEADER", "No KID attribute in Header"), + 0x8004C02F: ("DRM_E_NOLAINFOINHEADER", "No LAINFO attribute in Header"), + 0x8004C030: ("DRM_E_NOCHECKSUMINHEADER", "No Checksum attribute in Header"), + 0x8004C031: ("DRM_E_DSTBLOCKMISMATCH", "DST block mismatch"), + 0x8004C032: ("DRM_E_BACKUP_EXISTS", "Backup file already exist."), + 0x8004C033: ("DRM_E_LICENSE_TOOLONG", "License size is too long"), + 0x8004C034: ("DRM_E_DSTEXISTS", "A DST already exists in the specified location"), + 0x8004C035: ("DRM_E_INVALIDDEVICECERTIFICATE", "The device certificate is invalid."), + 0x8004C036: ("DRM_E_DSTLOCKFAILED", "Locking a segment of the DST failed."), + 0x8004C037: ("DRM_E_FILESEEKERROR", "File Seek Error"), + 0x8004C038: ("DRM_E_DSTNOTLOCKEDEXCLUSIVE", "Existing lock is not exclusive"), + 0x8004C039: ("DRM_E_DSTEXCLUSIVELOCKONLY", "Only exclusive lock is accepted"), + 0x8004C03A: ("DRM_E_DSTRESERVEDKEYDETECTED", "DST reserved key value detected in UniqueKey"), + 0x8004C03B: ("DRM_E_V1_NOT_SUPPORTED", "V1 Lic Acquisition is not supported"), + 0x8004C03C: ("DRM_E_HEADER_NOT_SET", "Content header is not set"), + 0x8004C03D: ("DRM_E_NEEDDEVCERTINDIV", "The device certificate is template. It need Devcert Indiv"), + 0x8004C03E: ("DRM_E_MACHINEIDMISMATCH", "The device has Machine Id different from that in devcert."), + 0x8004C03F: ("DRM_E_CLK_INVALID_RESPONSE", "The secure clock response is invalid."), + 0x8004C040: ("DRM_E_CLK_INVALID_DATE", "The secure clock response is invalid."), + 0x8004C041: ("DRM_E_CLK_UNSUPPORTED_VALUE", "The secure clock response has unsupported value."), + 0x8004C042: ("DRM_E_INVALIDDEVICECERTIFICATETEMPLATE", "The device certificate is invalid."), + 0x8004C043: ("DRM_E_DEVCERTEXCEEDSIZELIMIT", "The device certificate exceeds max size"), + 0x8004C044: ("DRM_E_DEVCERTTEMPLATEEXCEEDSSIZELIMIT", "The device certificate template exceeds max size"), + 0x8004C045: ("DRM_E_DEVCERTREADERROR", "Can't get the device certificate"), + 0x8004C046: ("DRM_E_DEVCERTWRITEERROR", "Can't store the device certificate"), + 0x8004C047: ("DRM_E_PRIVKEYREADERROR", "Can't get device private key"), + 0x8004C048: ("DRM_E_PRIVKEYWRITEERROR", "Can't store device private key"), + 0x8004C049: ("DRM_E_DEVCERTTEMPLATEREADERROR", "Can't get the device certificate template"), + 0x8004C04A: ("DRM_E_CLK_NOT_SUPPORTED", "The secure clock is not supported."), + 0x8004C04B: ("DRM_E_DEVCERTINDIV_NOT_SUPPORTED", "The Devcert Indiv is not supported."), + 0x8004C04C: ("DRM_E_METERING_NOT_SUPPORTED", "The Metering is not supported."), + 0x8004C04D: ("DRM_E_CLK_RESETSTATEREADERROR", "Can not read Secure clock Reset State."), + 0x8004C04E: ("DRM_E_CLK_RESETSTATEWRITEERROR", "Can not write Secure clock Reset State."), + 0x8004C04F: ("DRM_E_XMLNOTFOUND", "a required XML tag was not found"), + 0x8004C050: ("DRM_E_METERING_WRONG_TID", "wrong TID sent on metering response"), + 0x8004C051: ("DRM_E_METERING_INVALID_COMMAND", "wrong command sent on metering response"), + 0x8004C052: ("DRM_E_METERING_STORE_CORRUPT", "The metering store is corrupt"), + 0x8004C053: ("DRM_E_CERTIFICATE_REVOKED", "A certificate given to DRM was revoked."), + 0x8004C054: ("DRM_E_CRYPTO_FAILED", "A cryptographic operation failed."), + 0x8004C055: ("DRM_E_STACK_CORRUPT", "The stack allocator context is corrupt. Likely a buffer overrun problem."), + 0x8004C056: ("DRM_E_UNKNOWN_BINDING_KEY", "A matching binding key could not be found for the license."), + 0x8004C057: ("DRM_E_V1_LICENSE_CHAIN_NOT_SUPPORTED", "License chaining with V1 content is not supported."), + 0x8004C058: ("DRM_E_WRONG_TOKEN_TYPE", "The wrong type of token was used."), + 0x8004C059: ( + "DRM_E_POLICY_METERING_DISABLED", + "Metering code was called but metering is disabled by group or user policy", + ), + 0x8004C05A: ("DRM_E_POLICY_ONLINE_DISABLED", "online communication is disabled by group policy"), + 0x8004C05B: ("DRM_E_CLK_NOT_SET", "License may be there but can not be used as secure clock not set."), + 0x8004C05C: ( + "DRM_E_NO_CLK_SUPPORTED", + "This device does not support any Clock. So time bound licenses can not be played", + ), + 0x8004C05D: ("DRM_E_NO_URL", "Can not find URL info."), + 0x8004C05E: ("DRM_E_UNKNOWN_DEVICE_PROPERTY", "Unknown device property."), + 0x8004C05F: ( + "DRM_E_METERING_MID_MISMATCH", + "The metering ID is not same in Metering Cert and metering response data", + ), + 0x8004C060: ( + "DRM_E_METERING_RESPONSE_DECRYPT_FAILED", + "The encrypted section of metering response can not be decrypted", + ), + 0x8004C063: ("DRM_E_RIV_TOO_SMALL", "RIV on the machine is too small."), + 0x8004C064: ("DRM_E_STACK_ALREADY_INITIALIZED", "DRM_STK_Init called for initialized stack"), + 0x8004C065: ("DRM_E_DEVCERT_REVOKED", "The device certificate given to DRM is revoked."), + 0x8004C066: ("DRM_E_OEM_RSA_DECRYPTION_ERROR", "Error in OEM RSA Decryption."), + 0x8004C067: ("DRM_E_INVALID_DEVSTORE_ATTRIBUTE", "Invalid device attributes in the device store"), + 0x8004C068: ("DRM_E_INVALID_DEVSTORE_ENTRY", "The device store data entry is corrupted"), + 0x8004C069: ("DRM_E_OEM_RSA_ENCRYPTION_ERROR", "Error in OEM RSA Encryption process"), + 0x8004C06A: ("DRM_E_DSTNAMESPACEEXIST", "The DST Namespace already exists."), + 0x8004C06B: ("DRM_E_PERF_SCOPING_ERROR", "Error in performance scope context"), + 0x8004C06C: ("DRM_E_PRECISION_ARITHMETIC_FAIL", "Operation involving multiple precision arithmetic fails"), + 0x8004C06D: ("DRM_E_OEM_RSA_INVALID_PRIVATE_KEY", "Invalid private key."), + 0x8004C06E: ( + "DRM_E_NO_OPL_CALLBACK", + "There is no callback function to process the output restrictions specified in the license", + ), + 0x8004C06F: ("DRM_E_INVALID_PLAYREADY_OBJECT", "Structure of PlayReady object is invalid"), + 0x8004C070: ("DRM_E_DUPLICATE_LICENSE", "There is already a license in the store with the same KID & LID"), + 0x8004C071: ( + "DRM_E_REVOCATION_NOT_SUPPORTED", + "Device does not support revocation, while revocation data was placed into license policy structure.", + ), + 0x8004C072: ("DRM_E_RECORD_NOT_FOUND", "Record with requested type was not found in PlayReady object."), + 0x8004C073: ("DRM_E_BUFFER_BOUNDS_EXCEEDED", "An array is being referenced outside of it's bounds."), + 0x8004C074: ("DRM_E_INVALID_BASE64", "An input string contains invalid Base64 characters."), + 0x8004C075: ("DRM_E_PROTOCOL_VERSION_NOT_SUPPORTED", "The protocol version is not supported."), + 0x8004C076: ( + "DRM_E_INVALID_LICENSE_RESPONSE_SIGNATURE", + "Cannot verify license acquisition's response because signature is invalid.", + ), + 0x8004C077: ( + "DRM_E_INVALID_LICENSE_RESPONSE_RESPONSEID", + "Cannot verify license acquisition's response because response ID is invalid.", + ), + 0x8004C078: ( + "DRM_E_LICENSE_RESPONSE_SIGNATURE_MISSING", + "Cannot verify license acquisition's response because either response ID, license nonce or signature is missing.", + ), + 0x8004C079: ( + "DRM_E_INVALID_DOMAIN_JOIN_RESPONSE_SIGNATURE", + "Cannot verify domain join response because signature is invalid.", + ), + 0x8004C07A: ( + "DRM_E_DOMAIN_JOIN_RESPONSE_SIGNATURE_MISSING", + "Cannot verify domain join response because either signing certificate chain or signature is missing.", + ), + 0x8004C07B: ("DRM_E_ACTIVATION_REQUIRED", "The device must be activated before initialization can succeed."), + 0x8004C07C: ("DRM_E_ACTIVATION_INTERNAL_ERROR", "A server error occurred during device activation."), + 0x8004C07D: ( + "DRM_E_ACTIVATION_GROUP_CERT_REVOKED_ERROR", + "The activation group cert has been revoked and the application must be updated with a new client lib.", + ), + 0x8004C07E: ( + "DRM_E_ACTIVATION_NEW_CLIENT_LIB_REQUIRED_ERROR", + "The client lib used by the application is not supported and must be updated.", + ), + 0x8004C07F: ("DRM_E_ACTIVATION_BAD_REQUEST", "The activation request is invalid"), + 0x8004C080: ("DRM_E_FILEIO_ERROR", "Encountered a system error during file I/O."), + 0x8004C081: ("DRM_E_DISKSPACE_ERROR", "Out of disk space for storing playready files."), + 0x8004C082: ( + "DRM_E_UPLINKLICENSENOTFOUND", + "A license was found in the license store but no license was found for its uplink ID.", + ), + 0x8004C0A0: ("DRM_E_LRB_NOLGPUBKEY", "LRB does not contain a valid LGPUBKEY."), + 0x8004C0A1: ("DRM_E_LRB_INVALIDSIGNATURE", "Signature inside LRB is invalid."), + 0x8004C0A2: ("DRM_E_LRB_LGPUBKEY_MISMATCH", "LRB is signed with a pubkey different from LGPUBKEY"), + 0x8004C0A3: ("DRM_E_LRB_INVALIDLICENSEDATA", "LRB is signed with a pubkey different from LGPUBKEY"), + 0x8004C0C0: ("DRM_E_LICEVAL_LICENSE_NOT_SUPPLIED", "License not supplied in the liceval context"), + 0x8004C0C1: ("DRM_E_LICEVAL_KID_MISMATCH", "Mismatch between KID from header and the one inside license"), + 0x8004C0C2: ("DRM_E_LICEVAL_LICENSE_REVOKED", "License for this content has been revoked"), + 0x8004C0C3: ("DRM_E_LICEVAL_UPDATE_FAILURE", "Failed to update content revocation"), + 0x8004C0C4: ("DRM_E_LICEVAL_REQUIRED_REVOCATION_LIST_NOT_AVAILABLE", "Failed to update content revocation"), + 0x8004C0E0: ("DRM_E_XMR_OBJECT_ALREADY_EXISTS", "XMR builder context already has this object."), + 0x8004C0E1: ("DRM_E_XMR_OBJECT_NOTFOUND", "XMR object was not found."), + 0x8004C0E2: ("DRM_E_XMR_REQUIRED_OBJECT_MISSING", "XMR license doesn't have one or more required objects."), + 0x8004C0E3: ("DRM_E_XMR_INVALID_UNKNOWN_OBJECT", "Invalid unknown object"), + 0x8004C0E4: ("DRM_E_XMR_LICENSE_BINDABLE", "XMR license does not contain the Cannot Bind right"), + 0x8004C0E5: ("DRM_E_XMR_LICENSE_NOT_BINDABLE", "XMR license cannot be bound to because of the Cannot Bind right"), + 0x8004C0E6: ( + "DRM_E_XMR_UNSUPPORTED_XMR_VERSION", + "The version of XMR license is not supported for the current action", + ), + 0x8004C100: ( + "DRM_E_NOT_CRL_BLOB", + "CRL blob provided for parsing does not start with CBLB. It means file is not CRL blob at all.", + ), + 0x8004C101: ( + "DRM_E_BAD_CRL_BLOB", + "The file is structured as CRL blob, but there is some error in file structure or one of CRLs inside is invalid.", + ), + 0x8004C200: ("DRM_E_INVALID_DEVCERT_ATTRIBUTE", "The attributes in the Device certificate are invalid"), + 0x8004C300: ("DRM_E_TEST_PKCRYPTO_FAILURE", "Error in PK encryption/decryption crypto test cases."), + 0x8004C301: ("DRM_E_TEST_PKSIGN_VERIFY_ERROR", "Digital signature verification failed."), + 0x8004C302: ("DRM_E_TEST_ENCRYPT_ERROR", "Error in encryption of cipher text."), + 0x8004C303: ("DRM_E_TEST_RC4KEY_FAILED", "RC4 key failed during crypto operations."), + 0x8004C304: ("DRM_E_TEST_DECRYPT_ERROR", "Error in cipher text decryption."), + 0x8004C305: ("DRM_E_TEST_DESKEY_FAILED", "Decrypted data not equal to original data in a DES operation."), + 0x8004C306: ("DRM_E_TEST_CBC_INVERSEMAC_FAILURE", "Decrypted data not equal to original in Inverse MAC operation."), + 0x8004C307: ("DRM_E_TEST_HMAC_FAILURE", "Error in hashed data in HMAC operation."), + 0x8004C308: ("DRM_E_TEST_INVALIDARG", "Error in the number of arguments or argument data in Test files."), + 0x8004C30A: ( + "DRM_E_TEST_DEVICE_PRIVATE_KEY_INCORRECTLY_STORED", + "DRMManager context should not contain the device private key.", + ), + 0x8004C30B: ("DRM_E_TEST_DRMMANAGER_CONTEXT_NULL", "DRMManager context is NULL."), + 0x8004C30C: ("DRM_E_TEST_UNEXPECTED_REVINFO_RESULT", "Revocation cache result was not as expected."), + 0x8004C30D: ("DRM_E_TEST_RIV_MISMATCH", "Revocation Info Version(RIV) mismatch."), + 0x8004C310: ("DRM_E_TEST_URL_ERROR", "There is an error in the URL from the challenge generated."), + 0x8004C311: ( + "DRM_E_TEST_MID_MISMATCH", + "The MIDs returned from the DRM_MANAGER_CONTEXT does not match the test input.", + ), + 0x8004C312: ( + "DRM_E_TEST_METER_CERTIFICATE_MISMATCH", + "The input data does not match with the Metering certificate returned from the license.", + ), + 0x8004C313: ( + "DRM_E_TEST_LICENSE_STATE_MISMATCH", + "The input data and license state returned from the license do not match.", + ), + 0x8004C316: ( + "DRM_E_TEST_SOURCE_ID_MISMATCH", + "The input data and license state returned from the license do not match.", + ), + 0x8004C317: ( + "DRM_E_TEST_UNEXPECTED_LICENSE_COUNT", + "The input data and the number of license from the KID do not match.", + ), + 0x8004C318: ("DRM_E_TEST_UNEXPECTED_DEVICE_PROPERTY", "Unknown device property."), + 0x8004C319: ("DRM_E_TEST_DRMMANAGER_MISALIGNED_BYTES", "Error due to misalignment of bytes."), + 0x8004C31A: ( + "DRM_E_TEST_LICENSE_RESPONSE_ERROR", + "The license response callbacks did not provide the expected data.", + ), + 0x8004C31B: ( + "DRM_E_TEST_OPL_MISMATCH", + "The minimum levels of the compressed/uncompressed Digital and Analog Video do not match the OPL.", + ), + 0x8004C31C: ("DRM_E_TEST_INVALID_OPL_CALLBACK", "The callback type supplied is not valid."), + 0x8004C31D: ("DRM_E_TEST_INCOMPLETE", "The test function failed to complete."), + 0x8004C31E: ( + "DRM_E_TEST_UNEXPECTED_OUTPUT", + "The output of the function being tested does not match the expected output.", + ), + 0x8004C31F: ( + "DRM_E_TEST_DLA_NO_CONTENT_HEADER", + "Content Header Information was not retrieved correctly in DLA Sync Tests.", + ), + 0x8004C320: ( + "DRM_E_TEST_DLA_CONTENT_HEADER_FOUND", + "Content Header Information was found when it should not have been in DLA Sync Tests.", + ), + 0x8004C321: ("DRM_E_TEST_SYNC_LSD_INCORRECT", "DRM_SNC_GetSyncStoreEntry returned incorrect License State Data."), + 0x8004C322: ("DRM_E_TEST_TOO_SLOW", "The performance test failed because DRM took longer than its maximum time."), + 0x8004C323: ( + "DRM_E_TEST_LICENSESTORE_NOT_OPEN", + "The License Store contexts in the App Manager context are not open.", + ), + 0x8004C324: ("DRM_E_TEST_DEVICE_NOT_INITED", "The device instance has not been initialized prior to use."), + 0x8004C325: ( + "DRM_E_TEST_VARIABLE_NOT_SET", + "A global variable needed for test execution has not been set correctly.", + ), + 0x8004C326: ("DRM_E_TEST_NOMORE", "The same as DRM_E_NOMORE, only explicitly used in test code."), + 0x8004C327: ("DRM_E_TEST_FILE_LOAD_ERROR", "There was an error loading a test data file."), + 0x8004C328: ("DRM_E_TEST_LICENSE_ACQ_FAILED", "The attempt to acquire a license failed."), + 0x8004C329: ( + "DRM_E_TEST_UNSUPPORTED_FILE_FORMAT", + "A file format is being used which is not supported by the test function.", + ), + 0x8004C32A: ("DRM_E_TEST_PARSING_ERROR", "There was an error parsing input parameter."), + 0x8004C32B: ("DRM_E_TEST_NOTIMPL", "The specified test API is not implemented."), + 0x8004C32C: ( + "DRM_E_TEST_VARIABLE_NOTFOUND", + "The specified test varaible was not found in the shared variable table.", + ), + 0x8004C32D: ("DRM_E_TEST_VARIABLE_LISTFULL", "The shared test variable table is full."), + 0x8004C32E: ("DRM_E_TEST_UNEXPECTED_CONTENT_PROPERTY", "Unknown content property."), + 0x8004C32F: ("DRM_E_TEST_PRO_HEADER_NOT_SET", "PlayReady Object Header not set."), + 0x8004C330: ("DRM_E_TEST_NON_PRO_HEADER_TYPE", "Incompatible header - PlayReady Object Header expected."), + 0x8004C331: ("DRM_E_TEST_INVALID_DEVICE_WRAPPER", "The Device Simulator Device Wrapper is not valid."), + 0x8004C332: ("DRM_E_TEST_INVALID_WMDM_WRAPPER", "The Device Simulator WMDM Wrapper is not valid."), + 0x8004C333: ("DRM_E_TEST_INVALID_WPD_WRAPPER", "The Device Simulator WPD Wrapper is not valid."), + 0x8004C334: ("DRM_E_TEST_INVALID_FILE", "The data file given was invalid."), + 0x8004C335: ("DRM_E_TEST_PROPERTY_NOT_FOUND", "The object did not have the property which was queried."), + 0x8004C336: ("DRM_E_TEST_METERING_DATA_INCORRECT", "The metering data reported is incorrect."), + 0x8004C337: ( + "DRM_E_TEST_FILE_ALREADY_OPEN", + "The handle variable for a test file is not NULL. This indicates that a file was opened and not closed properly.", + ), + 0x8004C338: ( + "DRM_E_TEST_FILE_NOT_OPEN", + "The handle variable for a test file is NULL. This indicates that a file was not opened.", + ), + 0x8004C339: ( + "DRM_E_TEST_PICT_COLUMN_TOO_WIDE", + "The PICT input file contains a column which is too wide for the test parser to handle.", + ), + 0x8004C33A: ( + "DRM_E_TEST_PICT_COLUMN_MISMATCH", + "The PICT input file contains a row which doesn't have the same number of columns as the header row.", + ), + 0x8004C33B: ("DRM_E_TEST_TUX_TEST_SKIPPED", "TUX cannot find the speficied test case in target dll. Test Skipped."), + 0x8004C33C: ("DRM_E_TEST_KEYFILE_VERIFICATION_FAILURE", "Verification of the Keyfile context failed."), + 0x8004C33D: ("DRM_E_TEST_DATA_VERIFICATION_FAILURE", "Data does not match expected value and failed verification."), + 0x8004C33E: ("DRM_E_TEST_NET_FAIL", "The Test failed to perform Network I/O."), + 0x8004C3E8: ( + "DRM_E_LOGICERR", + "DRM code has a logic error in it. This result should never be returned. There is an unhandled code path if it is returned.", + ), + 0x8004C3E9: ("DRM_E_INVALIDREVINFO", "The device certificate is invalid."), + 0x8004C3EA: ("DRM_E_SYNCLISTNOTSUPPORTED", "The device does not support synclist."), + 0x8004C3EB: ("DRM_E_REVOCATION_BUFFERTOOSMALL", "The revocation buffer is too small."), + 0x8004C3EC: ( + "DRM_E_DEVICE_ALREADY_REGISTERED", + "There exists already a device in the device store with the same DEVICEID that was given.", + ), + 0x8004C3ED: ("DRM_E_DST_NOT_COMPATIBLE", "The data store version is incompatible with this version of DRM."), + 0x8004C3F0: ("DRM_E_RSA_DECRYPTION_ERROR", "The data block/Encoded message used in OAEP decoding is incorrect."), + 0x8004C3F1: ("DRM_E_OEM_RSA_MESSAGE_TOO_BIG", "The base message buffer is larger than the given modulus."), + 0x8004C3F2: ("DRM_E_METERCERTNOTFOUND", "The metering certificate was not found in the store."), + 0x8004C3F3: ("DRM_E_MODULAR_ARITHMETIC_FAILURE", "A failure occurred in bignum modular arithmetic."), + 0x8004C3F4: ("DRM_E_FEATURE_NOT_SUPPORTED", "The feature is not supported in this release."), + 0x8004C3F5: ("DRM_E_REVOCATION_INVALID_PACKAGE", "The revocation package is invalid"), + 0x8004C3F6: ("DRM_E_HWID_ERROR", "Failed to get the hardware ID."), + 0x8004C500: ("DRM_E_DOMAIN_INVALID_GUID", "Not a correct GUID."), + 0x8004C501: ("DRM_E_DOMAIN_INVALID_CUSTOM_DATA_TYPE", "Not a valid custom data type."), + 0x8004C502: ("DRM_E_DOMAIN_STORE_ADD_DATA", "Failed to add data into the domain store."), + 0x8004C503: ("DRM_E_DOMAIN_STORE_GET_DATA", "Failed to retrieve data from the domain store."), + 0x8004C504: ("DRM_E_DOMAIN_STORE_DELETE_DATA", "Failed to delete data from the domain store."), + 0x8004C505: ("DRM_E_DOMAIN_STORE_OPEN_STORE", "Failed to open the domain store."), + 0x8004C506: ("DRM_E_DOMAIN_STORE_CLOSE_STORE", "Failed to close the domain store."), + 0x8004C507: ("DRM_E_DOMAIN_BIND_LICENSE", "Failed to bind to the domain license."), + 0x8004C508: ("DRM_E_DOMAIN_INVALID_CUSTOM_DATA", "Not a valid custom data."), + 0x8004C509: ("DRM_E_DOMAIN_NOT_FOUND", "No domain information is found."), + 0x8004C50A: ( + "DRM_E_DOMAIN_INVALID_DOMKEYXMR_DATA", + "The domain join response contains invalid domain privkey XMR data.", + ), + 0x8004C50B: ( + "DRM_E_DOMAIN_STORE_INVALID_KEY_RECORD", + "Invalid format of domain private key record read from the domain store.", + ), + 0x8004C580: ( + "DRM_E_DEVICE_DOMAIN_JOIN_REQUIRED", + "This error code communicates to the application that the device is not a member of a domain. The app can uses this error code in turn to decide whether it needs to join the domain or not", + ), + 0x8004C600: ("DRM_E_SERVER_INTERNAL_ERROR", "An internal server error occurred."), + 0x8004C601: ("DRM_E_SERVER_INVALID_MESSAGE", "The message sent to the server was invalid."), + 0x8004C602: ("DRM_E_SERVER_DEVICE_LIMIT_REACHED", "The device limit for the domain has been reached."), + 0x8004C603: ("DRM_E_SERVER_INDIV_REQUIRED", "Individualization of the client is required."), + 0x8004C604: ("DRM_E_SERVER_SERVICE_SPECIFIC", "An error specific to the service has occurred."), + 0x8004C605: ("DRM_E_SERVER_DOMAIN_REQUIRED", "A Domain certificate is required."), + 0x8004C606: ("DRM_E_SERVER_RENEW_DOMAIN", "The Domain certificate needs to be renewed."), + 0x8004C607: ("DRM_E_SERVER_UNKNOWN_METERINGID", "The metering identifier is unknown."), + 0x8004C608: ("DRM_E_SERVER_COMPUTER_LIMIT_REACHED", "The computer limit for the domain has been reached."), + 0x8004C609: ( + "DRM_E_SERVER_PROTOCOL_FALLBACK", + "The client should fallback to the V2 license acquisition protocol.", + ), + 0x8004C60A: ( + "DRM_E_SERVER_NOT_A_MEMBER", + "The client was removed from the domain in an offline fashion and thus still has a domain cert, but not a valid domain membership.", + ), + 0x8004C60B: ( + "DRM_E_SERVER_PROTOCOL_VERSION_MISMATCH", + "The protocol version specified was not supported by the server.", + ), + 0x8004C60C: ("DRM_E_SERVER_UNKNOWN_ACCOUNTID", "The account identifier is unknown."), + 0x8004C60D: ("DRM_E_SERVER_PROTOCOL_REDIRECT", "The protocol has a redirect."), + 0x8004C610: ("DRM_E_SERVER_UNKNOWN_TRANSACTIONID", "The transaction identifier is unknown."), + 0x8004C611: ("DRM_E_SERVER_INVALID_LICENSEID", "The license identifier is invalid."), + 0x8004C612: ( + "DRM_E_SERVER_MAXIMUM_LICENSEID_EXCEEDED", + "The maximum number of license identifiers in the request was exceeded.", + ), + 0x8004C700: ("DRM_E_LICACQ_TOO_MANY_LICENSES", "There are too many licenses in the license response."), + 0x8004C701: ( + "DRM_E_LICACQ_ACK_TRANSACTIONID_TOO_BIG", + "The Transaction ID specified by the server exceeds the allocated buffer.", + ), + 0x8004C702: ( + "DRM_E_LICACQ_ACK_MESSAGE_NOT_CREATED", + "The license acquisition acknowledgement message could not be created.", + ), + 0x8004C780: ("DRM_E_INITIATORS_UNKNOWN_TYPE", "The initiator type is unknown."), + 0x8004C781: ("DRM_E_INITIATORS_INVALID_SERVICEID", "The service ID data is not valid."), + 0x8004C782: ("DRM_E_INITIATORS_INVALID_ACCOUNTID", "The account ID data is not valid."), + 0x8004C783: ("DRM_E_INITIATORS_INVALID_MID", "The account ID data is not valid."), + 0x8004C784: ("DRM_E_INITIATORS_MISSING_DC_URL", "Domain Controller URL is missing."), + 0x8004C785: ("DRM_E_INITIATORS_MISSING_CONTENT_HEADER", "Content header is missing."), + 0x8004C786: ( + "DRM_E_INITIATORS_MISSING_LAURL_IN_CONTENT_HEADER", + "Missing license acquisition URL in content header.", + ), + 0x8004C787: ("DRM_E_INITIATORS_MISSING_METERCERT_URL", "Meter certificate server URL is missing."), + 0x8004C800: ("DRM_E_BCERT_INVALID_SIGNATURE_TYPE", "An invalid signature type was encountered"), + 0x8004C801: ( + "DRM_E_BCERT_CHAIN_TOO_DEEP", + "There are, or there would be, too many certificates in the certificate chain", + ), + 0x8004C802: ("DRM_E_BCERT_INVALID_CERT_TYPE", "An invalid certificate type was encountered"), + 0x8004C803: ("DRM_E_BCERT_INVALID_FEATURE", "An invalid feature entry was encountered"), + 0x8004C804: ("DRM_E_BCERT_INVALID_KEY_USAGE", "An invalid public key usage was encountered"), + 0x8004C805: ("DRM_E_BCERT_INVALID_SECURITY_VERSION", "An invalid Indiv Box security version was encountered"), + 0x8004C806: ("DRM_E_BCERT_INVALID_KEY_TYPE", "An invalid public key type was encountered"), + 0x8004C807: ("DRM_E_BCERT_INVALID_KEY_LENGTH", "An invalid public key length was encountered"), + 0x8004C808: ("DRM_E_BCERT_INVALID_MAX_LICENSE_SIZE", "An invalid maximum license size value was encountered"), + 0x8004C809: ("DRM_E_BCERT_INVALID_MAX_HEADER_SIZE", "An invalid maximum license header size value was encountered"), + 0x8004C80A: ( + "DRM_E_BCERT_INVALID_MAX_LICENSE_CHAIN_DEPTH", + "An invalid maximum license chain depth was encountered", + ), + 0x8004C80B: ("DRM_E_BCERT_INVALID_SECURITY_LEVEL", "An invalid security level was encountered"), + 0x8004C80C: ( + "DRM_E_BCERT_PRIVATE_KEY_NOT_SPECIFIED", + "A private key for signing the certificate was not provided to the builder", + ), + 0x8004C80D: ("DRM_E_BCERT_ISSUER_KEY_NOT_SPECIFIED", "An issuer key was not provided to the builder"), + 0x8004C80E: ("DRM_E_BCERT_ACCOUNT_ID_NOT_SPECIFIED", "An account ID was not provided to the builder"), + 0x8004C80F: ("DRM_E_BCERT_SERVICE_ID_NOT_SPECIFIED", "A service provider ID was not provided to the builder"), + 0x8004C810: ("DRM_E_BCERT_CLIENT_ID_NOT_SPECIFIED", "A client ID was not provided to the builder"), + 0x8004C811: ("DRM_E_BCERT_DOMAIN_URL_NOT_SPECIFIED", "A domain URL was not provided to the builder"), + 0x8004C812: ("DRM_E_BCERT_DOMAIN_URL_TOO_LONG", "The domain URL contains too many ASCII characters"), + 0x8004C813: ("DRM_E_BCERT_HARDWARE_ID_NOT_SPECIFIED", "A hardware ID was not provided to the builder"), + 0x8004C814: ("DRM_E_BCERT_HARDWARE_ID_TOO_LONG", "A hardware ID is longer than the maximum supported bytes"), + 0x8004C815: ("DRM_E_BCERT_SERIAL_NUM_NOT_SPECIFIED", "A device serial number was not provided to the builder"), + 0x8004C816: ("DRM_E_BCERT_CERT_ID_NOT_SPECIFIED", "A certificate ID was not provided to the builder"), + 0x8004C817: ( + "DRM_E_BCERT_PUBLIC_KEY_NOT_SPECIFIED", + "A public key for the certificate was not provided to the builder or not found by the parser", + ), + 0x8004C818: ( + "DRM_E_BCERT_KEY_USAGES_NOT_SPECIFIED", + "The public key usage information was not provided to the builder or not found by the parser", + ), + 0x8004C819: ("DRM_E_BCERT_STRING_NOT_NULL_TERMINATED", "Data string is not null-teminated"), + 0x8004C81A: ("DRM_E_BCERT_OBJECTHEADER_LEN_TOO_BIG", "Object length in object header is too big"), + 0x8004C81B: ("DRM_E_BCERT_INVALID_ISSUERKEY_LENGTH", "IssuerKey Length value is invalid"), + 0x8004C81C: ("DRM_E_BCERT_BASICINFO_CERT_EXPIRED", "Certificate is expired"), + 0x8004C81D: ("DRM_E_BCERT_UNEXPECTED_OBJECT_HEADER", "Object header has unexpected values"), + 0x8004C81E: ( + "DRM_E_BCERT_ISSUERKEY_KEYINFO_MISMATCH", + "The cert's Issuer Key does not match key info in the next cert", + ), + 0x8004C81F: ("DRM_E_BCERT_INVALID_MAX_KEY_USAGES", "Number of key usage entries is invalid"), + 0x8004C820: ("DRM_E_BCERT_INVALID_MAX_FEATURES", "Number of features is invalid"), + 0x8004C821: ("DRM_E_BCERT_INVALID_CHAIN_HEADER_TAG", "Cert chain header tag is invalid"), + 0x8004C822: ("DRM_E_BCERT_INVALID_CHAIN_VERSION", "Cert chain version is invalid"), + 0x8004C823: ("DRM_E_BCERT_INVALID_CHAIN_LENGTH", "Cert chain length value is invalid"), + 0x8004C824: ("DRM_E_BCERT_INVALID_CERT_HEADER_TAG", "Cert header tag is invalid"), + 0x8004C825: ("DRM_E_BCERT_INVALID_CERT_VERSION", "Cert version is invalid"), + 0x8004C826: ("DRM_E_BCERT_INVALID_CERT_LENGTH", "Cert length value is invalid"), + 0x8004C827: ("DRM_E_BCERT_INVALID_SIGNEDCERT_LENGTH", "Length of signed portion of certificate is invalid"), + 0x8004C828: ("DRM_E_BCERT_INVALID_PLATFORM_IDENTIFIER", "An invalid Platform Identifier was specified"), + 0x8004C829: ("DRM_E_BCERT_INVALID_NUMBER_EXTDATARECORDS", "An invalid number of extended data records"), + 0x8004C82A: ("DRM_E_BCERT_INVALID_EXTDATARECORD", "An invalid extended data record"), + 0x8004C82B: ("DRM_E_BCERT_EXTDATA_LENGTH_MUST_PRESENT", "Extended data record length must be present."), + 0x8004C82C: ("DRM_E_BCERT_EXTDATA_PRIVKEY_MUST_PRESENT", "Extended data record length must be present."), + 0x8004C82D: ( + "DRM_E_BCERT_INVALID_EXTDATA_LENGTH", + "Calculated and written extended data object lengths do not match.", + ), + 0x8004C82E: ( + "DRM_E_BCERT_EXTDATA_IS_NOT_PROVIDED", + "Extended data is not provided, the cert builder cannot write it.", + ), + 0x8004C82F: ( + "DRM_E_BCERT_HWIDINFO_IS_MISSING", + "The PC certificate is correct but is not ready to use because has no HWID information", + ), + 0x8004C830: ( + "DRM_E_BCERT_INVALID_EXTDATA_SIGNED_LENGTH", + "Length of signed portion of extended data info is invalid", + ), + 0x8004C831: ("DRM_E_BCERT_INVALID_EXTDATA_RECORD_TYPE", "Extended data record type is invalid"), + 0x8004C832: ( + "DRM_E_BCERT_EXTDATAFLAG_CERT_TYPE_MISMATCH", + "Certificate of this type cannot have extended data flag set", + ), + 0x8004C833: ("DRM_E_BCERT_METERING_ID_NOT_SPECIFIED", "An metering ID was not provided to the builder"), + 0x8004C834: ("DRM_E_BCERT_METERING_URL_NOT_SPECIFIED", "A metering URL was not provided to the builder"), + 0x8004C835: ("DRM_E_BCERT_METERING_URL_TOO_LONG", "The metering URL contains too many ASCII characters"), + 0x8004C836: ("DRM_E_BCERT_VERIFICATION_ERRORS", "Verification errors are found while parsing cert chain"), + 0x8004C837: ("DRM_E_BCERT_REQUIRED_KEYUSAGE_MISSING", "Required key usage is missing"), + 0x8004C838: ( + "DRM_E_BCERT_NO_PUBKEY_WITH_REQUESTED_KEYUSAGE", + "The certificate does not contain a public key with the requested key usage", + ), + 0x8004C839: ("DRM_E_BCERT_MANUFACTURER_STRING_TOO_LONG", "The manufacturer string is too long"), + 0x8004C83A: ("DRM_E_BCERT_TOO_MANY_PUBLIC_KEYS", "There are too many public keys in the certificate"), + 0x8004C83B: ("DRM_E_BCERT_OBJECTHEADER_LEN_TOO_SMALL", "Object length in object header is too small"), + 0x8004C83C: ( + "DRM_E_BCERT_INVALID_WARNING_DAYS", + "An invalid server certificate expiration warning days. Warning days must be greater than zero.", + ), + 0x8004C900: ("DRM_E_XMLSIG_ECDSA_VERIFY_FAILURE", "Error in ECDSA signature verification."), + 0x8004C901: ("DRM_E_XMLSIG_SHA_VERIFY_FAILURE", "Error in SHA verification."), + 0x8004C902: ("DRM_E_XMLSIG_FORMAT", "The format of XML signature or encryption segment is incorrect."), + 0x8004C903: ("DRM_E_XMLSIG_PUBLIC_KEY_ID", "Invalud pre-shared public key ID."), + 0x8004C904: ("DRM_E_XMLSIG_INVALID_KEY_FORMAT", "Invalid type of public/private key format."), + 0x8004C905: ("DRM_E_XMLSIG_SHA_HASH_SIZE", "Size of hash is unexpected."), + 0x8004C906: ("DRM_E_XMLSIG_ECDSA_SIGNATURE_SIZE", "Size of ECDSA signature is unexpected."), + 0x8004CA00: ("DRM_E_UTF_UNEXPECTED_END", "Unexpected end of data in the middle of multibyte character."), + 0x8004CA01: ("DRM_E_UTF_INVALID_CODE", "UTF character maps into a code with invalid value."), + 0x8004CB00: ("DRM_E_SOAPXML_INVALID_STATUS_CODE", "Status code contained in the server error response is invalid."), + 0x8004CB01: ("DRM_E_SOAPXML_XML_FORMAT", "Cannot parse out expected XML node."), + 0x8004CB02: ("DRM_E_SOAPXML_WRONG_MESSAGE_TYPE", "The message type associated with the soap message is wrong."), + 0x8004CB03: ("DRM_E_SOAPXML_SIGNATURE_MISSING", "The message did not have a signature and needed one"), + 0x8004CB04: ( + "DRM_E_SOAPXML_PROTOCOL_NOT_SUPPORTED", + "The requested protocol is not supported by the DRM SOAP parser.", + ), + 0x8004CB05: ("DRM_E_SOAPXML_DATA_NOT_FOUND", "The requested data is not found in the response."), + 0x8004CC00: ( + "DRM_E_CRYPTO_PUBLIC_KEY_NOT_MATCH", + "The public key associated with an encrypted domain private from the server does not match any public key on the device.", + ), + 0x8004CC01: ( + "DRM_E_UNABLE_TO_RESOLVE_LOCATION_TREE", + "Unable to derive the key. Maybe due to blackout or no rights to the service, etc.", + ), + 0x8004CD00: ("DRM_E_SECURE_TRACE_BAD_GLOBAL_DATA_POINTER", "The secure trace global data pointer is NULL"), + 0x8004CD01: ("DRM_E_SECURE_TRACE_INVALID_GLOBAL_DATA", "The secure trace global data structure is invalid"), + 0x8004CD02: ("DRM_E_SECURE_TRACE_FORMATTING_ERROR", "An error occured in formatting the trace message"), + 0x8004CD03: ("DRM_E_SECURE_TRACE_BAD_SCHEME_DATA_POINTER", "A secure trace scheme data pointer is NULL"), + 0x8004CD04: ( + "DRM_E_SECURE_TRACE_BAD_PER_THREAD_AES_DATA_POINTER", + "The secure trace per thread AES data pointer is NULL", + ), + 0x8004CD05: ( + "DRM_E_SECURE_TRACE_BAD_PER_THREAD_AES_BUFFER_POINTER", + "A secure trace per thread AES buffer pointer is NULL", + ), + 0x8004CD06: ("DRM_E_SECURE_TRACE_AES_INSUFFICIENT_BUFFER", "There is no space left in the secure trace AES buffer"), + 0x8004CD07: ("DRM_E_SECURE_TRACE_VERSION_MISMATCH", "All drm dlls do not agree on the same secure trace version"), + 0x8004CD08: ("DRM_E_SECURE_TRACE_UNEXPECTED_ERROR", "An expected error was encountered in secure tracing system"), + 0x8004CE00: ("DRM_E_ND_MUST_REVALIDATE", "The client must be revalidated before executing the intended operation."), + 0x8004CE01: ("DRM_E_ND_INVALID_MESSAGE", "A received message is garbled."), + 0x8004CE02: ("DRM_E_ND_INVALID_MESSAGE_TYPE", "A received message contains an invalid message type."), + 0x8004CE03: ("DRM_E_ND_INVALID_MESSAGE_VERSION", "A received message contains an invalid message version."), + 0x8004CE04: ("DRM_E_ND_INVALID_SESSION", "The requested session is invalid."), + 0x8004CE05: ( + "DRM_E_ND_MEDIA_SESSION_LIMIT_REACHED", + "A new session cannot be opened because the maximum number of sessions has already been opened.", + ), + 0x8004CE06: ( + "DRM_E_ND_UNABLE_TO_VERIFY_PROXIMITY", + "The proximity detection procedure could not confirm that the receiver is near the transmitter in the network.", + ), + 0x8004CE07: ( + "DRM_E_ND_INVALID_PROXIMITY_RESPONSE", + "The response to the proximity detection challenge is invalid.", + ), + 0x8004CE08: ( + "DRM_E_ND_DEVICE_LIMIT_REACHED", + "The maximum number of devices in use has been reached. Unable to open additional devices.", + ), + 0x8004CE09: ("DRM_E_ND_BAD_REQUEST", "The message format is invalid."), + 0x8004CE0A: ("DRM_E_ND_FAILED_SEEK", "It is not possible to seek to the specified mark-in point."), + 0x8004CE0B: ( + "DRM_E_ND_INVALID_CONTEXT", + "Manager context or at least one of it's children is missing (or corrupt).", + ), + 0x8004CF00: ("DRM_E_ASF_BAD_ASF_HEADER", "The ASF file has a bad ASF header."), + 0x8004CF01: ("DRM_E_ASF_BAD_PACKET_HEADER", "The ASF file has a bad packet header."), + 0x8004CF02: ("DRM_E_ASF_BAD_PAYLOAD_HEADER", "The ASF file has a bad payload header."), + 0x8004CF03: ("DRM_E_ASF_BAD_DATA_HEADER", "The ASF file has a bad data header."), + 0x8004CF04: ( + "DRM_E_ASF_INVALID_OPERATION", + "The intended operation is invalid given the current processing state of the ASF file.", + ), + 0x8004CF05: ( + "DRM_E_ASF_AES_PAYLOAD_FOUND", + "ND payload extension system found; the file may be encrypted with AES already.", + ), + 0x8004CF06: ( + "DRM_E_ASF_EXTENDED_STREAM_PROPERTIES_OBJ_NOT_FOUND", + "Extended stream properties object is not found; the file may be in non-supported outdated format.", + ), + 0x8004CF20: ("DRM_E_ASF_INVALID_DATA", "The packet is overstuffed with data."), + 0x8004CF21: ( + "DRM_E_ASF_TOO_MANY_PAYLOADS", + "The number of payloads in the packet is greater than the maximum allowed.", + ), + 0x8004CF22: ("DRM_E_ASF_BANDWIDTH_OVERRUN", "An object is overflowing the leaky bucket."), + 0x8004CF23: ( + "DRM_E_ASF_INVALID_STREAM_NUMBER", + "The stream number is invalid; it is either zero, greater than the maximum value allowed, or has no associated data.", + ), + 0x8004CF24: ( + "DRM_E_ASF_LATE_SAMPLE", + "A sample was encountered with a presentation time outside of the mux's send window.", + ), + 0x8004CF25: ("DRM_E_ASF_NOT_ACCEPTING", "The sample does not fit in the remaining payload space."), + 0x8004CF26: ("DRM_E_ASF_UNEXPECTED", "An unexpected error occurred."), + 0x8004D000: ("DRM_E_NONCE_STORE_TOKEN_NOT_FOUND", "The matching nonce store token is not found."), + 0x8004D001: ("DRM_E_NONCE_STORE_OPEN_STORE", "Fail to open nonce store."), + 0x8004D002: ("DRM_E_NONCE_STORE_CLOSE_STORE", "Fail to close nonce store."), + 0x8004D003: ("DRM_E_NONCE_STORE_ADD_LICENSE", "There is already a license associated with the nonce store token."), + 0x8004D100: ("DRM_E_LICGEN_POLICY_NOT_SUPPORTED", "The license generation policy combination is not supported."), + 0x8004D200: ("DRM_E_POLICYSTATE_NOT_FOUND", "The policy state is not found in the secure store."), + 0x8004D201: ( + "DRM_E_POLICYSTATE_CORRUPTED", + "The policy state is not stored as a valid internal format in the secure store.", + ), + 0x8004D300: ("DRM_E_MOVE_DENIED", "The requested move operation was denied by the service."), + 0x8004D301: ("DRM_E_INVALID_MOVE_RESPONSE", "The move response was incorrectly formed."), + 0x8004D302: ("DRM_E_MOVE_NONCE_MISMATCH", "The nonce in the repsonse did not match the expected value."), + 0x8004D303: ("DRM_E_MOVE_TXID_MISMATCH", "The transaction id in the repsonse did not match the expected value."), + 0x8004D304: ("DRM_E_MOVE_STORE_OPEN_STORE", "Failed to open the move store."), + 0x8004D305: ("DRM_E_MOVE_STORE_CLOSE_STORE", "Failed to close the move store."), + 0x8004D306: ("DRM_E_MOVE_STORE_ADD_DATA", "Failed to add data into the move store."), + 0x8004D307: ("DRM_E_MOVE_STORE_GET_DATA", "Failed to retrieve data from the move store."), + 0x8004D308: ("DRM_E_MOVE_FORMAT_INVALID", "The format of a move page or index is invalid."), + 0x8004D309: ("DRM_E_MOVE_SIGNATURE_INVALID", "The signature of a move index is invalid."), + 0x8004D30A: ("DRM_E_COPY_DENIED", "The requested copy operation was denied by the service."), + 0x8004D400: ("DRM_E_XB_OBJECT_NOTFOUND", "The extensible binary object was not found."), + 0x8004D401: ("DRM_E_XB_INVALID_OBJECT", "The extensible binary object format was invalid."), + 0x8004D402: ( + "DRM_E_XB_OBJECT_ALREADY_EXISTS", + "A single instance extensible binary object was encountered more than once.", + ), + 0x8004D403: ( + "DRM_E_XB_REQUIRED_OBJECT_MISSING", + "A required extensible binary object was not found during building.", + ), + 0x8004D404: ( + "DRM_E_XB_UNKNOWN_ELEMENT_TYPE", + "An extensible binary object description contained an element of an unknown type.", + ), + 0x8004D500: ("DRM_E_KEYFILE_INVALID_PLATFORM", "The keyfile does not support the current platform."), + 0x8004D501: ("DRM_E_KEYFILE_TOO_LARGE", "The keyfile is larger than the maximum supported size."), + 0x8004D502: ("DRM_E_KEYFILE_PRIVATE_KEY_NOT_FOUND", "The private key requested was not found in the keyfile."), + 0x8004D503: ( + "DRM_E_KEYFILE_CERTIFICATE_CHAIN_NOT_FOUND", + "The certificate chain requested was not found in the keyfile.", + ), + 0x8004D504: ("DRM_E_KEYFILE_KEY_NOT_FOUND", "The AES Key ID was not found in the keyfile."), + 0x8004D505: ("DRM_E_KEYFILE_UNKNOWN_DECRYPTION_METHOD", "Unknown keyfile decryption method."), + 0x8004D506: ("DRM_E_KEYFILE_INVALID_SIGNATURE", "The keyfile signature was not valid."), + 0x8004D507: ( + "DRM_E_KEYFILE_INTERNAL_DECRYPTION_BUFFER_TOO_SMALL", + "The internal decryption buffer is too small to hold the encrypted key from the keyfile.", + ), + 0x8004D508: ("DRM_E_KEYFILE_PLATFORMID_MISMATCH", "Platform ID in the certificate does not match expected value."), + 0x8004D509: ( + "DRM_E_KEYFILE_CERTIFICATE_ISSUER_KEY_MISMATCH", + "Issuer key of the device certificate does not match public key of the model certificate.", + ), + 0x8004D50A: ( + "DRM_E_KEYFILE_ROBUSTNESSVERSION_MISMATCH", + "Robustness version in the certificate does not match expected value.", + ), + 0x8004D50B: ( + "DRM_E_KEYFILE_FILE_NOT_CLOSED", + "The KeyFile Close function was not called before trying to unintialize the KeyFile context.", + ), + 0x8004D50C: ("DRM_E_KEYFILE_NOT_INITED", "The KeyFile Context was not initialized before trying to use it."), + 0x8004D50D: ("DRM_E_KEYFILE_FORMAT_INVALID", "The format of the KeyFile was invalid."), + 0x8004D50E: ( + "DRM_E_KEYFILE_UPDATE_NOT_ALLOWED", + "The keyfile of the device is read only, and updates are not permitted.", + ), + 0x8004DFFC: ("DRM_E_OEM_CONSTRAINT_1_FAIL", "Failed while OEM CONSTRAINT-1 check"), + 0x8004DFFD: ("DRM_E_OEM_CONSTRAINT_2_FAIL", "Failed while OEM CONSTRAINT-2 check"), + 0x8004DFFE: ("DRM_E_OEM_CONSTRAINT_3_FAIL", "Failed while OEM CONSTRAINT-3 check"), + 0x8004DFFF: ("DRM_E_HDCP_NON_SECURE", "Attempt to play back through insecure HDMI output path"), + 0x80070002: ("DRM_E_WIN32_FILE_NOT_FOUND", "The system cannot find the file specified."), + 0x80070012: ("DRM_E_WIN32_NO_MORE_FILES", "There are no more files."), + 0x80070057: ("DRM_E_INVALIDARG", "The parameter is incorrect."), + 0x8007007A: ("DRM_E_BUFFERTOOSMALL", "The data area passed to a function is too small."), + 0x80070103: ("DRM_E_NOMORE", "No more data is available."), + 0x80070216: ("DRM_E_ARITHMETIC_OVERFLOW", "Arithmetic result exceeded maximum value."), + 0x80070490: ("DRM_E_NOT_FOUND", "Element not found."), + 0x80070667: ("DRM_E_INVALID_COMMAND_LINE", "Invalid command line argument."), + 0xC00D2712: ("DRM_E_FAILED_TO_STORE_LICENSE", "License storage is not working."), + 0xC00D272F: ("DRM_E_PARAMETERS_MISMATCHED", "A problem has occurred in the Digital Rights Management component."), + 0xC00D275F: ("DRM_E_NOT_ALL_STORED", "Some of the licenses could not be stored."), +} + + +def describe(code: int) -> str | None: + """Human string for a PlayReady/DRM HRESULT, or None if unknown. + + Accepts signed or unsigned 32-bit ints (license servers often return the + signed form, e.g. -2147172251 == 0x8004C065). + """ + if code is None: + return None + key = code & 0xFFFFFFFF + entry = ERRORS.get(key) + if entry is None: + return None + name, text = entry + return f"{name} (0x{key:08X}){f': {text}' if text else ''}" + + +if __name__ == "__main__": + # ponytail: one check, signed and unsigned forms resolve to the same entry + assert describe(0x8004C065) == describe(-2147172251) + assert "DEVCERT_REVOKED" in describe(-2147172251) + assert describe(-1021) is None # non-DRM_E service code + print("ok") From c2f7fd87a314df93beacaeefbdd309c957ddf7eb Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Thu, 23 Jul 2026 08:48:51 -0600 Subject: [PATCH 05/40] feat(drm): detect revoked PlayReady devices from license responses Add is_revocation()/REVOCATION_CODES and centrally raise DeviceRevoked when a license fault carries a device/cert revocation HRESULT, so any service surfaces it without its own decoding. --- unshackle/core/drm/playready.py | 29 +++++++++++++++++++++++++- unshackle/core/drm/playready_errors.py | 20 ++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/unshackle/core/drm/playready.py b/unshackle/core/drm/playready.py index 6fdd46d9..58685361 100644 --- a/unshackle/core/drm/playready.py +++ b/unshackle/core/drm/playready.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import re import shutil import subprocess import textwrap @@ -307,6 +308,7 @@ class PlayReady: challenge = cdm.get_license_challenge(session_id, self.pssh.wrm_headers[0]) if challenge: + license_str = "" try: log_event( "drm_license_request", @@ -339,7 +341,12 @@ class PlayReady: drm_type="PlayReady", response_size=len(license_str), ) - except Exception: + except PlayReady.Exceptions.DeviceRevoked: + raise + except Exception as e: + revoked = self._detect_revocation(license_str) or self._detect_revocation(str(e)) + if revoked: + raise PlayReady.Exceptions.DeviceRevoked(revoked) from e raise keys = self._extract_keys_from_cdm(cdm, session_id) @@ -360,6 +367,23 @@ class PlayReady: if not self.content_keys: raise PlayReady.Exceptions.EmptyLicense("No Content Keys were within the License") + @staticmethod + def _detect_revocation(text: str) -> Optional[str]: + """Return the decoded error string if a revocation HRESULT is present, else None. + + Reads the code from the raw SOAP body (0x8004C065) + or from an exception message that carries it, so any service is covered. + """ + from unshackle.core.drm.playready_errors import describe, is_revocation + + if not text: + return None + for token in re.findall(r"0x[0-9A-Fa-f]{8}|-?\d{7,}", text): + code = int(token, 16) if token.lower().startswith("0x") else int(token) + if is_revocation(code): + return describe(code) or f"0x{code & 0xFFFFFFFF:08X}" + return None + def decrypt(self, path: Path) -> None: """ Decrypt a Track with PlayReady DRM. @@ -548,5 +572,8 @@ class PlayReady: class EmptyLicense(Exception): pass + class DeviceRevoked(Exception): + pass + __all__ = ("PlayReady",) diff --git a/unshackle/core/drm/playready_errors.py b/unshackle/core/drm/playready_errors.py index 0560eaed..9890ea89 100644 --- a/unshackle/core/drm/playready_errors.py +++ b/unshackle/core/drm/playready_errors.py @@ -841,6 +841,20 @@ ERRORS: dict[int, tuple[str, str]] = { } +REVOCATION_CODES = frozenset( + { + 0x8004C065, # DRM_E_DEVCERT_REVOKED + 0x8004C053, # DRM_E_CERTIFICATE_REVOKED + 0x8004C07D, # DRM_E_ACTIVATION_GROUP_CERT_REVOKED_ERROR + } +) + + +def is_revocation(code: int) -> bool: + """True if the HRESULT means a device or certificate was revoked (signed or unsigned).""" + return code is not None and (code & 0xFFFFFFFF) in REVOCATION_CODES + + def describe(code: int) -> str | None: """Human string for a PlayReady/DRM HRESULT, or None if unknown. @@ -862,4 +876,10 @@ if __name__ == "__main__": assert describe(0x8004C065) == describe(-2147172251) assert "DEVCERT_REVOKED" in describe(-2147172251) assert describe(-1021) is None # non-DRM_E service code + assert is_revocation(0x8004C065) and is_revocation(-2147172251) # DEVCERT_REVOKED, both forms + assert not is_revocation(0x8004C006) # INVALIDLICENSE is not revocation + assert is_revocation(0x8004C07D) # activation group cert revoked + assert not is_revocation(0x8004A013) # metering sig, not revocation + assert not is_revocation(0x8004C0C4) # rev-list-not-available, not revocation + assert not is_revocation(0x8004C0C2) # content license revoked, not device print("ok") From ae9ffc773130f004313accd9cb31f43d8da246d4 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Thu, 23 Jul 2026 09:33:14 -0600 Subject: [PATCH 06/40] feat(drm): auto-repack tracks with a residual encrypted sample entry Detect an encv/enca sample entry surviving decryption (moov-scoped 4CC scan) and force a repack so mkvmerge isn't first-FourCC-locked to a generic encrypted codec; warn if one persists after repacking. Closes #142 --- tests/tracks/test_encrypted_sample_entry.py | 86 +++++++++++++++++++++ unshackle/commands/dl.py | 12 +++ unshackle/core/tracks/track.py | 56 ++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 tests/tracks/test_encrypted_sample_entry.py diff --git a/tests/tracks/test_encrypted_sample_entry.py b/tests/tracks/test_encrypted_sample_entry.py new file mode 100644 index 00000000..3874d2ce --- /dev/null +++ b/tests/tracks/test_encrypted_sample_entry.py @@ -0,0 +1,86 @@ +"""Self-check for ``has_encrypted_sample_entry``: a finalized MP4 whose moov still carries +an encv/enca sample entry means the decrypt tool skipped/failed the track (frma was never +applied), so the detector must flag it before the file reaches the muxer. Detection is +scoped to moov, so stray 4CC byte runs in mdat must never trip (or mask) it. + +Run: uv run pytest tests/tracks/test_encrypted_sample_entry.py +""" + +from __future__ import annotations + +import struct +from pathlib import Path + +from unshackle.core.tracks.track import has_encrypted_sample_entry + + +def _box(fmt: bytes, payload: bytes = b"") -> bytes: + return struct.pack(">I", 8 + len(payload)) + fmt + payload + + +def _sample_entry(fmt: bytes, payload: bytes = b"") -> bytes: + body = b"\x00" * 6 + struct.pack(">H", 1) + payload # reserved + data_reference_index + return struct.pack(">I", 8 + len(body)) + fmt + body + + +def _stsd(entries: list[bytes]) -> bytes: + body = b"\x00" * 4 + struct.pack(">I", len(entries)) + b"".join(entries) # version/flags + count + return _box(b"stsd", body) + + +def _moov(*children: bytes) -> bytes: + stbl = _box(b"stbl", b"".join(children)) + return _box(b"moov", _box(b"trak", _box(b"mdia", _box(b"minf", stbl)))) + + +FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2") +# stray [size][encv] byte run as compressed A/V payload could randomly contain +STRAY_ENCV = b"\xde\xad" * 4 + struct.pack(">I", 100) + b"encv" + b"\xbe\xef" * 8 + + +def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path: + f = tmp_path / name + f.write_bytes(b"".join(boxes)) + return f + + +def test_dual_stsd_encv_and_avc1_detected(tmp_path: Path) -> None: + sinf = _box(b"sinf", _box(b"frma", b"avc1")) + avc1 = _sample_entry(b"avc1", b"\x00" * 70) + encv = _sample_entry(b"encv", b"\x00" * 70 + sinf) + f = _write(tmp_path, "dual.mp4", FTYP, _moov(_stsd([avc1, encv])), _box(b"mdat", b"\x00" * 32)) + assert has_encrypted_sample_entry(f) is True + + +def test_single_avc1_not_detected(tmp_path: Path) -> None: + avc1 = _sample_entry(b"avc1", b"\x00" * 70) + f = _write(tmp_path, "clear.mp4", FTYP, _moov(_stsd([avc1])), _box(b"mdat", b"\x00" * 32)) + assert has_encrypted_sample_entry(f) is False + + +def test_stray_encv_in_mdat_not_detected(tmp_path: Path) -> None: + # clean moov, mdat happens to contain a plausible [size][encv] run -> must NOT flag + avc1 = _sample_entry(b"avc1", b"\x00" * 70) + f = _write(tmp_path, "stray.mp4", FTYP, _moov(_stsd([avc1])), _box(b"mdat", STRAY_ENCV)) + assert has_encrypted_sample_entry(f) is False + + +def test_leading_mdat_does_not_mask_real_encv(tmp_path: Path) -> None: + # progressive layout (mdat before moov) with a stray run must not hide a genuine encv + enca = _sample_entry(b"enca", b"\x00" * 20 + _box(b"sinf", _box(b"frma", b"mp4a"))) + f = _write(tmp_path, "masked.mp4", FTYP, _box(b"mdat", STRAY_ENCV), _moov(_stsd([enca]))) + assert has_encrypted_sample_entry(f) is True + + +def test_text_containing_enca_in_moov_not_detected(tmp_path: Path) -> None: + # "enca" inside a metadata string has no plausible box size before it -> must NOT flag + avc1 = _sample_entry(b"avc1", b"\x00" * 70) + udta = _box(b"udta", b"data is encapsulated here") + f = _write(tmp_path, "text.mp4", FTYP, _box(b"moov", _stsd([avc1]) + udta)) + assert has_encrypted_sample_entry(f) is False + + +def test_missing_or_bogus_file(tmp_path: Path) -> None: + assert has_encrypted_sample_entry(tmp_path / "nope.mp4") is False + bogus = _write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4") + assert has_encrypted_sample_entry(bogus) is False diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 708c76c3..6f2482f3 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -67,6 +67,7 @@ from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.dv_fixup import apply_dv_fixup from unshackle.core.tracks.hybrid import Hybrid +from unshackle.core.tracks.track import has_encrypted_sample_entry from unshackle.core.utilities import ( find_font_with_fallbacks, find_missing_langs, @@ -1895,6 +1896,12 @@ class dl: if track.needs_repack: track.repackage() events.emit(events.Types.TRACK_REPACKED, track=track) + # residual encrypted sample entry after repack => never decrypted + if has_encrypted_sample_entry(track.path): + self.log.warning( + f"Track {track.id} still has an encrypted sample entry after repacking, so " + "decryption likely failed and the muxed output may be unplayable." + ) media_info = MediaInfo.parse(track.path) media_infos[id(track)] = media_info @@ -3152,6 +3159,11 @@ class dl: vui_folded.add(id(track)) has_repacked = True events.emit(events.Types.TRACK_REPACKED, track=track) + if has_encrypted_sample_entry(track.path): + self.log.warning( + f"Track {track.id} still has an encrypted sample entry after repacking, so " + "decryption likely failed and the muxed output may be unplayable." + ) if has_repacked: # we don't want to fill up the log with "Repacked x track" self.log.info("Repacked one or more tracks with FFMPEG") diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 30613a3f..53a74a0c 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -48,6 +48,58 @@ def direct_session(session: Union[Session, "RnetSession"]) -> Session: return new +def _read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]: + # seek through top-level box headers; only the wanted box's bytes are read into RAM + file_size = path.stat().st_size + with path.open("rb") as f: + start = 0 + while start + 8 <= file_size: + header = f.read(8) + if len(header) < 8: + return None + size = int.from_bytes(header[:4], "big") + if size == 1: # 64-bit largesize + large = f.read(8) + if len(large) < 8: + return None + size = int.from_bytes(large, "big") + elif size == 0: # box runs to EOF + size = file_size - start + if size < 8: + return None + if header[4:8] == box_type: + f.seek(start) + return f.read(size) + start += size + f.seek(start) + return None + + +def has_encrypted_sample_entry(path: Path) -> bool: + """True if the MP4's moov still carries an encrypted sample entry (encv/enca). + + A faithful decrypt rewrites encv/enca back to the real codec 4CC via frma, so a + survivor means the decrypt tool skipped/failed the track. This is a 4CC scan + scoped to the moov box (where sample entries live), not a structural parse; + scoping avoids chance byte collisions in mdat. Any read error -> False. + """ + try: + moov = _read_top_level_box(path, b"moov") + except Exception: + return False + if not moov: + return False + for tok in (b"encv", b"enca"): + i = moov.find(tok, 4) + while i != -1: + # require a plausible sample-entry box size right before the 4CC + size = int.from_bytes(moov[i - 4 : i], "big") + if 16 <= size <= len(moov) - (i - 4): + return True + i = moov.find(tok, i + 1) + return False + + @dataclass class DownloadContext: """Shared arguments passed to each manifest's ``download_track``.""" @@ -379,6 +431,10 @@ class Track: self.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=self, drm=drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) + # residual encv/enca => decrypt tool skipped/failed; force a repack so the + # muxer isn't first-FourCC-locked to a generic (encrypted) codec. OR-in only. + if has_encrypted_sample_entry(save_path): + self.needs_repack = True if track_type == "Subtitle" and self.codec.name not in ("fVTT", "fTTML"): track_data = self.path.read_bytes() From 7c767bfd31c1e453ce64e92b9f20aea95660d8d9 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Thu, 23 Jul 2026 19:46:24 -0600 Subject: [PATCH 07/40] perf(pipeline): fix pool starvation, unlock vault writes, prep mux Stop remounting adapters on shared sessions (restores dropped 429/5xx retries, ends 16-conn pool starvation, pool sized to 64); defer DRM vault write-backs off the global lock; add mux output_path groundwork for future parallelism. --- unshackle/commands/dl.py | 97 ++++++++++++++++++-------- unshackle/core/downloaders/requests.py | 9 +-- unshackle/core/remote_service.py | 1 + unshackle/core/service.py | 1 + unshackle/core/tracks/tracks.py | 27 ++++--- 5 files changed, 88 insertions(+), 47 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 6f2482f3..852f5815 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -734,6 +734,7 @@ class dl: return dl(ctx, **kwargs) DRM_TABLE_LOCK = Lock() + VAULT_WRITE_LOCK = Lock() EXPORT_LOCK = Lock() LICENSE_KEY_CACHE: dict[UUID, str] = {} @@ -3337,10 +3338,25 @@ class dl: progress_sink( {"phase": "muxing", "progress": 96.0, "status": "downloading", "active_tracks": []} ) + + def unique_mux_output(task_tracks: Tracks, index: int) -> Optional[Path]: + src: Optional[Path] = None + ext = ".muxed.mkv" + if task_tracks.videos: + src = task_tracks.videos[0].path + elif task_tracks.audio: + src, ext = task_tracks.audio[0].path, ".muxed.mka" + elif task_tracks.subtitles: + src, ext = task_tracks.subtitles[0].path, ".muxed.mks" + if src is None: + return None + base = src.with_suffix(ext) + return base.with_name(f"{base.stem}.{index}{base.suffix}") + try: with SyncLive(Padding(progress, (0, 5, 1, 5)), console=console, refresh_per_second=20): - mux_index = 0 - for task_id, task_tracks, audio_codec in multiplex_tasks: + mux_failed = False + for mux_index, (task_id, task_tracks, audio_codec) in enumerate(multiplex_tasks, start=1): progress.start_task(task_id) audio_expected = not video_only and not no_audio muxed_path, return_code, errors = task_tracks.mux( @@ -3350,15 +3366,8 @@ class dl: audio_expected=audio_expected, title_language=title.language, skip_subtitles=skip_subtitle_mux, + output_path=unique_mux_output(task_tracks, mux_index), ) - if muxed_path.exists(): - mux_index += 1 - unique_path = muxed_path.with_name( - f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" - ) - if unique_path != muxed_path: - shutil.move(muxed_path, unique_path) - muxed_path = unique_path muxed_paths.append(muxed_path) muxed_audio_codecs[muxed_path] = audio_codec if return_code >= 2: @@ -3371,7 +3380,10 @@ class dl: else: self.log.warning(line) if return_code >= 2: - sys.exit(1) + mux_failed = True + break + if mux_failed: + sys.exit(1) # Output sidecar subtitles before deleting track files if sidecar_subtitles and not no_mux: @@ -3646,6 +3658,21 @@ class dl: export.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf8") + def flush_vault_writes(self, pending: list[Callable[[], Any]]) -> None: + if not pending: + return + with self.VAULT_WRITE_LOCK: + for write in pending: + write() + pending.clear() + + def cache_keys_to_vaults(self, content_keys: dict[UUID, str]) -> None: + successful_caches = self.vaults.add_keys(content_keys) + self.log.info( + f"Cached {len(content_keys)} Key{'' if len(content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + def prepare_drm( self, drm: DRM_T, @@ -3766,6 +3793,7 @@ class dl: }, ) + pending_vault_writes: list[Callable[[], Any]] = [] with self.DRM_TABLE_LOCK: pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) @@ -3817,7 +3845,9 @@ class dl: label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) - self.vaults.add_key(kid, content_key, excluding=vault_used) + pending_vault_writes.append( + partial(self.vaults.add_key, kid, content_key, excluding=vault_used) + ) self.LICENSE_KEY_CACHE[kid] = content_key if self.debug_logger: @@ -3844,6 +3874,7 @@ class dl: message=msg, context={"kid": kid.hex, "track": str(track)}, ) + self.flush_vault_writes(pending_vault_writes) raise Widevine.Exceptions.CEKNotFound(msg) else: need_license = True @@ -3891,6 +3922,7 @@ class dl: service=self.service, context={"track": str(track), "exception_type": type(e).__name__}, ) + self.flush_vault_writes(pending_vault_writes) raise e log_event( @@ -3922,17 +3954,14 @@ class dl: self.LICENSE_KEY_CACHE.update(drm.content_keys) - successful_caches = self.vaults.add_keys(drm.content_keys) - self.log.info( - f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " - f"{successful_caches}/{len(self.vaults)} Vaults" - ) + pending_vault_writes.append(partial(self.cache_keys_to_vaults, drm.content_keys)) if track_kid and track_kid not in drm.content_keys: msg = f"No Content Key for KID {track_kid.hex} was returned in the License" cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + self.flush_vault_writes(pending_vault_writes) raise Widevine.Exceptions.CEKNotFound(msg) if cek_tree.children and not pre_existing_tree: @@ -3942,6 +3971,8 @@ class dl: if export: self.write_export(export, title, track, drm) + self.flush_vault_writes(pending_vault_writes) + elif isinstance(drm, PlayReady): if self.debug_logger: self.debug_logger.log_drm_operation( @@ -3957,6 +3988,7 @@ class dl: }, ) + pending_vault_writes = [] with self.DRM_TABLE_LOCK: pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") cek_tree = Tree( @@ -4014,7 +4046,9 @@ class dl: label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) - self.vaults.add_key(kid, content_key, excluding=vault_used) + pending_vault_writes.append( + partial(self.vaults.add_key, kid, content_key, excluding=vault_used) + ) self.LICENSE_KEY_CACHE[kid] = content_key if self.debug_logger: @@ -4042,6 +4076,7 @@ class dl: message=msg, context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"}, ) + self.flush_vault_writes(pending_vault_writes) raise PlayReady.Exceptions.CEKNotFound(msg) else: need_license = True @@ -4079,6 +4114,7 @@ class dl: "drm_type": "PlayReady", }, ) + self.flush_vault_writes(pending_vault_writes) raise e for kid_, key in drm.content_keys.items(): @@ -4091,17 +4127,14 @@ class dl: self.LICENSE_KEY_CACHE.update(drm.content_keys) - successful_caches = self.vaults.add_keys(drm.content_keys) - self.log.info( - f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " - f"{successful_caches}/{len(self.vaults)} Vaults" - ) + pending_vault_writes.append(partial(self.cache_keys_to_vaults, drm.content_keys)) if track_kid and track_kid not in drm.content_keys: msg = f"No Content Key for KID {track_kid.hex} was returned in the License" cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + self.flush_vault_writes(pending_vault_writes) raise PlayReady.Exceptions.CEKNotFound(msg) if cek_tree.children and not pre_existing_tree: @@ -4111,7 +4144,10 @@ class dl: if export: self.write_export(export, title, track, drm) + self.flush_vault_writes(pending_vault_writes) + elif isinstance(drm, ClearKeyCENC): + pending_vault_writes = [] with self.DRM_TABLE_LOCK: cek_tree = Tree(Text.assemble(("ClearKey", "cyan"), overflow="fold")) pre_existing_tree = next( @@ -4162,7 +4198,9 @@ class dl: label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) - self.vaults.add_key(kid, content_key, excluding=vault_used) + pending_vault_writes.append( + partial(self.vaults.add_key, kid, content_key, excluding=vault_used) + ) self.LICENSE_KEY_CACHE[kid] = content_key elif vaults_only: msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" @@ -4176,6 +4214,7 @@ class dl: message=msg, context={"kid": kid.hex, "track": str(track), "drm_type": "ClearKeyCENC"}, ) + self.flush_vault_writes(pending_vault_writes) raise ClearKeyCENC.Exceptions.CEKNotFound(msg) else: need_license = True @@ -4215,6 +4254,7 @@ class dl: "drm_type": "ClearKeyCENC", }, ) + self.flush_vault_writes(pending_vault_writes) raise e for kid_, key in drm.content_keys.items(): @@ -4227,17 +4267,14 @@ class dl: self.LICENSE_KEY_CACHE.update(drm.content_keys) - successful_caches = self.vaults.add_keys(drm.content_keys) - self.log.info( - f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " - f"{successful_caches}/{len(self.vaults)} Vaults" - ) + pending_vault_writes.append(partial(self.cache_keys_to_vaults, drm.content_keys)) if track_kid and track_kid not in drm.content_keys: msg = f"No Content Key for KID {track_kid.hex} was returned in the License" cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + self.flush_vault_writes(pending_vault_writes) raise ClearKeyCENC.Exceptions.CEKNotFound(msg) if cek_tree.children and not pre_existing_tree: @@ -4247,6 +4284,8 @@ class dl: if export: self.write_export(export, title, track, drm) + self.flush_vault_writes(pending_vault_writes) + elif isinstance(drm, MonaLisa): with self.DRM_TABLE_LOCK: display_id = drm.content_id or drm.pssh diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index ab49235b..8a13b296 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -504,9 +504,8 @@ def requests( # once per batch; download() skips it for segments output_dir.mkdir(parents=True, exist_ok=True) - # Use provided session or create a new optimized requests.Session - # When a session is provided (e.g., service's RnetSession), don't mutate headers/cookies/proxy — - # they're already set and the session may be shared across tracks. + # Provided sessions may be shared across tracks: don't mutate or remount them. Remounting + # drops the service's 429/5xx Retry config and races get_adapter() in other track threads. if session is None: session = Session() if headers: @@ -516,10 +515,6 @@ def requests( session.cookies.update(cookies) if proxy: session.proxies.update({"all": proxy}) - - # Mount HTTPAdapter with connection pooling sized to worker count. - # Safe to do on any requests.Session — improves connection reuse for parallel downloads. - if _is_requests_session(session): adapter = HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True) session.mount("https://", adapter) session.mount("http://", adapter) diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index ade9643e..2524ee85 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -422,6 +422,7 @@ class RemoteService: "https://", HTTPAdapter( max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + pool_maxsize=64, pool_block=True, ), ) diff --git a/unshackle/core/service.py b/unshackle/core/service.py index 109cd77e..a5d78f3e 100644 --- a/unshackle/core/service.py +++ b/unshackle/core/service.py @@ -328,6 +328,7 @@ class Service(metaclass=ABCMeta): "https://", HTTPAdapter( max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + pool_maxsize=64, pool_block=True, ), ) diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index dcd0cc37..5c4207d8 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -473,6 +473,7 @@ class Tracks: audio_expected: bool = True, title_language: Optional[Language] = None, skip_subtitles: bool = False, + output_path: Optional[Path] = None, ) -> tuple[Path, int, list[str]]: """ Multiplex all the Tracks into a Matroska Container file. @@ -488,6 +489,9 @@ class Tracks: title_language: The title's intended language. Used to select the best video track for audio metadata when multiple video tracks exist. skip_subtitles: Skip muxing subtitle tracks into the container. + output_path: Explicit destination for the muxed container. When None (default) the + path is derived from the first track, so callers muxing several track groups that + share a video list must pass distinct paths to avoid clobbering each other. """ if self.videos and not self.audio and audio_expected: video_track = None @@ -688,17 +692,18 @@ class Tracks: ] ) - output_path = ( - self.videos[0].path.with_suffix(".muxed.mkv") - if self.videos - else self.audio[0].path.with_suffix(".muxed.mka") - if self.audio - else self.subtitles[0].path.with_suffix(".muxed.mks") - if self.subtitles - else chapters_path.with_suffix(".muxed.mkv") - if self.chapters - else None - ) + if output_path is None: + output_path = ( + self.videos[0].path.with_suffix(".muxed.mkv") + if self.videos + else self.audio[0].path.with_suffix(".muxed.mka") + if self.audio + else self.subtitles[0].path.with_suffix(".muxed.mks") + if self.subtitles + else chapters_path.with_suffix(".muxed.mkv") + if self.chapters + else None + ) if not output_path: raise ValueError("No tracks provided, at least one track must be provided.") From 8cf17006843481286117168eddd41fabcf4399f4 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sat, 25 Jul 2026 11:32:17 -0600 Subject: [PATCH 08/40] fix(import): stop losing side-loaded subtitles and blaming the manifest URL Only fall back to Title.language when the manifest labels nothing, and reject und as firmly as a missing tag. Exported direct-URL subtitles now replace the re-parsed set, and fetch failures stay distinct from parse failures. --- docs/guide/downloading.md | 7 ++ tests/core/test_import_tracks.py | 138 +++++++++++++++++++++++++++++++ unshackle/core/import_service.py | 56 ++++++++++--- 3 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 tests/core/test_import_tracks.py diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index c5b3aa20..07b84768 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -499,6 +499,13 @@ unshackle dl --skip-dl --export EXAMPLE 81234567 When `--proxy` is used, the export records the region so an import can reproduce the correct geofence. Without a proxy, no region is stored. +!!! warning "Some DASH and Smooth exports need a title language" + An import re-fetches the DASH or ISM manifest and parses it again. Most manifests label + their own streams, and those import fine. When a manifest labels nothing, the parse falls + back to the title's original language, which comes from `Title.language` on the exporting + service. If a service never sets it, importing that export fails with a message naming + the service. Neither end guesses a language for you, so the fix belongs in the service. + ## Metadata and tagging unshackle looks up metadata automatically, but you can override the identifiers used for diff --git a/tests/core/test_import_tracks.py b/tests/core/test_import_tracks.py new file mode 100644 index 00000000..f503302b --- /dev/null +++ b/tests/core/test_import_tracks.py @@ -0,0 +1,138 @@ +"""Tests for ``ImportService.get_tracks``, which rebuilds tracks from an export. + +Regression: an export whose manifest labels its own streams imports even when the service +never set ``Title.language``. When nothing supplies a language the import fails naming the +service rather than blaming the manifest URL. A direct-URL subtitle anywhere in the export +replaces the manifest's whole subtitle set, since the service side-loaded it precisely +because the manifest's copy was the worse one. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Optional +from unittest.mock import patch + +import click +import pytest +import requests + +from unshackle.core.import_service import ImportService +from unshackle.core.manifests import DASH + +MPD_TEMPLATE = """ + + + + + v.mp4 + + + + s.vtt + + + +""" + +# labelled by @lang; unlabelled but with Unified Streaming ids DASH derives a language from; +# and labelled by neither, which is the only shape that needs the title's language +MPD_WITH_LANG = MPD_TEMPLATE.format(lang=' lang="ja"', video_id="v1", sub_id="s1") +MPD_ID_LANG = MPD_TEMPLATE.format(lang="", video_id="video_eng=5000000", sub_id="text_eng=1000") +MPD_NO_LANG = MPD_TEMPLATE.format(lang="", video_id="v1", sub_id="s1") + +URL_SUBTITLE = { + "type": "Subtitle", + "id": "sideloaded-en", + "url": "https://example.invalid/en.srt", + "language": "en", + "descriptor": "URL", + "codec": "SubRip", +} + + +def make_service(tmp_path: Path, language: Optional[str], tracks: Optional[dict[str, Any]] = None) -> ImportService: + export = { + "version": 2, + "service": "EXAMPLE", + "titles": { + "movie-1": { + "meta": {"type": "movie", "id": "movie-1", "name": "Example Movie", "year": 2024, "language": language}, + "manifest_url": "https://example.invalid/m.mpd", + "manifest_type": "DASH", + "tracks": tracks or {}, + } + }, + } + path = tmp_path / "export.json" + path.write_text(json.dumps(export), encoding="utf8") + return ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path)) + + +def get_tracks(service: ImportService, mpd: str): + with patch.object(DASH, "from_url", side_effect=lambda url, session=None, **kw: DASH.from_text(mpd, url)): + return service.get_tracks(next(iter(service.get_titles()))) + + +@pytest.mark.parametrize("mpd", [MPD_WITH_LANG, MPD_ID_LANG], ids=["lang-attribute", "language-in-rep-id"]) +def test_manifest_supplies_language_without_title_language(tmp_path: Path, mpd: str) -> None: + """A manifest that labels its own streams imports fine with no language in the export.""" + tracks = get_tracks(make_service(tmp_path, language=None), mpd) + assert [str(t.language) for t in tracks.videos] == ["ja" if mpd is MPD_WITH_LANG else "en"] + + +@pytest.mark.parametrize("language", [None, "und"], ids=["missing", "und"]) +def test_unlabelled_manifest_without_usable_language_blames_the_service( + tmp_path: Path, language: Optional[str] +) -> None: + """`und` is truthy and a valid tag, so it must be rejected as firmly as no language at all.""" + with pytest.raises(click.ClickException) as error: + get_tracks(make_service(tmp_path, language=language), MPD_NO_LANG) + assert "Title.language" in error.value.message + assert "may have expired" not in error.value.message + + +def test_exported_url_subtitles_replace_the_manifests(tmp_path: Path) -> None: + """Services drop the manifest's subtitles and side-load their own; importing both duplicates them.""" + tracks = get_tracks(make_service(tmp_path, "ja", {"sideloaded-en": URL_SUBTITLE}), MPD_WITH_LANG) + assert [(t.id, t.descriptor.name) for t in tracks.subtitles] == [("sideloaded-en", "URL")] + assert len(tracks.videos) == 1 + + +def test_direct_url_subtitles_win_over_every_manifest_one(tmp_path: Path) -> None: + """One side-loaded subtitle condemns the whole re-parsed set, including other languages.""" + manifest_sub = { + "type": "Subtitle", + "id": "s1", + "url": "https://example.invalid/m.mpd", + "language": "ja", + "descriptor": "DASH", + "codec": "WebVTT", + } + tracks = get_tracks( + make_service(tmp_path, "ja", {"s1": manifest_sub, "sideloaded-en": URL_SUBTITLE}), MPD_WITH_LANG + ) + assert [(t.id, t.descriptor.name) for t in tracks.subtitles] == [("sideloaded-en", "URL")] + + +def test_unreadable_exported_track_is_skipped(tmp_path: Path) -> None: + """A track dict this build cannot rebuild must not take the whole import down with it.""" + unknown = {**URL_SUBTITLE, "id": "future", "codec": "CODEC_FROM_A_LATER_BUILD"} + tracks = get_tracks(make_service(tmp_path, "ja", {"sideloaded-en": URL_SUBTITLE, "future": unknown}), MPD_WITH_LANG) + assert [t.id for t in tracks.subtitles] == ["sideloaded-en"] + + +@pytest.mark.parametrize( + "failure", + [requests.ConnectionError("dns"), TypeError("Expected 'MPD' document, but received a 'html' document instead.")], + ids=["connection-error", "expired-token-html-page"], +) +def test_fetch_failure_still_reports_a_possible_expiry(tmp_path: Path, failure: Exception) -> None: + """An expired signed URL can 200 with an error page, so document failures are expiries too.""" + service = make_service(tmp_path, "ja") + title = next(iter(service.get_titles())) + with patch.object(DASH, "from_url", side_effect=failure): + with pytest.raises(click.ClickException) as error: + service.get_tracks(title) + assert "may have expired" in error.value.message diff --git a/unshackle/core/import_service.py b/unshackle/core/import_service.py index 3ecaf411..63490245 100644 --- a/unshackle/core/import_service.py +++ b/unshackle/core/import_service.py @@ -9,6 +9,7 @@ from uuid import UUID import click import requests +from langcodes import tag_is_valid from unshackle.core.config import config from unshackle.core.constants import AnyTrack @@ -26,6 +27,18 @@ log = logging.getLogger("import") PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM} +def _fallback_language(language: Any) -> Optional[Any]: + """The title language if the parsers would accept it, else None so they raise. + + DASH drops an und/invalid fallback but ISM does not, and a truthy 'und' there silently + labels every track und instead of failing. + """ + tag = str(language or "").strip() + if not tag or not tag_is_valid(tag) or tag.startswith("und"): + return None + return language + + class ImportService: """Reconstructs a download from an export JSON. @@ -132,9 +145,12 @@ class ImportService: """Reconstruct the title's tracks from the export. DASH/ISM: re-fetch and re-parse the manifest and return the full ladder (the importer - picks quality with normal dl flags; keys are injected by KID later). HLS/URL: rebuild - from the stored per-track dicts, since the variant is re-fetched from track.url at - download time and ATV-style master playlists carry unstable per-fetch tokens. + picks quality with normal dl flags; keys are injected by KID later), then merge in the + exported URL tracks. A service side-loads its own subtitles when the manifest's are the + worse copy, so any direct-URL subtitle in the export means the whole re-parsed subtitle + set is dropped in favour of the exported one. + HLS/URL: rebuild from the stored per-track dicts, since the variant is re-fetched from + track.url at download time and ATV-style master playlists carry unstable per-fetch tokens. """ title_id = str(title.id) if title_id in self.tracks_by_title: @@ -151,21 +167,41 @@ class ImportService: parser = PARSERS.get(manifest_type or "") if manifest_url and parser is not None and manifest_type in ("DASH", "ISM"): try: - parsed = parser.from_url(url=manifest_url, session=self.session).to_tracks(language=title.language) + manifest = parser.from_url(url=manifest_url, session=self.session) except Exception as e: raise click.ClickException( - f"Failed to re-fetch/parse the {manifest_type} manifest for '{title}'. " + f"Failed to fetch the {manifest_type} manifest for '{title}'. " f"The manifest URL may have expired since export. ({e})" ) + try: + parsed = manifest.to_tracks(language=_fallback_language(title.language)) + except ValueError as e: + if "Language information could not be derived" in str(e): + raise click.ClickException( + f"No language for '{title}': the {manifest_type} manifest has none and the " + f"service did not set Title.language." + ) + raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})") + except Exception as e: + raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})") for track in parsed: tracks.add(track) + track_dicts = [t for t in tracks_map.values() if t.get("descriptor", "URL") == "URL"] + if any(t.get("type") == "Subtitle" for t in track_dicts): + tracks.subtitles.clear() else: - for track_dict in tracks_map.values(): + track_dicts = list(tracks_map.values()) + + for track_dict in track_dicts: + try: track = Track.from_dict(track_dict) - drm = self.rebuild_drm(track_dict) - if drm: - track.drm = drm - tracks.add(track) + except Exception as e: + self.log.warning(f"Skipping exported track {track_dict.get('id')!r}: {e}") + continue + drm = self.rebuild_drm(track_dict) + if drm: + track.drm = drm + tracks.add(track, warn_only=True) for attachment in entry.get("attachments") or []: url = attachment.get("url") From b7029c343e4b0a0c6f0294ab36d102c19ae135a2 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sat, 25 Jul 2026 12:59:55 -0600 Subject: [PATCH 09/40] fix(ism): emit two stsd entries so sample_description_index 2 resolves Smooth fragments set tfhd sample_description_index=2 against a server init carrying a second, identical sample entry. A lone entry left that dangling: mp4decrypt skipped every such fragment silently and demuxers stopped after the first, yielding a file that looked fine but was almost entirely encrypted. --- tests/core/test_ism_init.py | 76 ++++++++++++++++++++++++++++ unshackle/core/manifests/ism_init.py | 5 +- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/core/test_ism_init.py b/tests/core/test_ism_init.py index 363bb031..a25e3193 100644 --- a/tests/core/test_ism_init.py +++ b/tests/core/test_ism_init.py @@ -111,6 +111,82 @@ def test_hevc_init_structure(): assert b"encv" not in init and b"sinf" not in init +# Boxes whose payload is purely child boxes, so their size must span the children +# exactly. stsd is excluded: its payload starts with a count, and sample entries +# carry a fixed prefix before any child box. +CONTAINER_BOXES = {"moov", "trak", "mdia", "minf", "stbl", "dinf", "mvex", "sinf", "schi"} + + +def walk_box(data: bytes, start: int, end: int, path: str = "") -> None: + """Assert every container's declared size spans its children exactly.""" + offset = start + while offset < end: + size = struct.unpack_from(">I", data, offset)[0] + name = data[offset + 4 : offset + 8].decode("latin1") + where = f"{path}/{name}" + assert size >= 8, f"{where}: size {size}" + assert offset + size <= end, f"{where}: overruns its parent" + if name in CONTAINER_BOXES: + walk_box(data, offset + 8, offset + size, where) + offset += size + assert offset == end, f"{path or '/'}: children do not span the parent exactly" + + +def find_stsd(init: bytes) -> int: + """Offset of the stsd box, found by walking the box tree.""" + offset, end = 0, len(init) + for name in ("moov", "trak", "mdia", "minf", "stbl", "stsd"): + cursor = offset + while cursor < end: + size = struct.unpack_from(">I", init, cursor)[0] + if init[cursor + 4 : cursor + 8].decode("latin1") == name: + offset, end = cursor + 8, cursor + size + break + cursor += size + else: + raise AssertionError(f"{name} not found") + return offset - 8 + + +# The two-entry stsd is unconditional, so every stream type and codec the builder +# emits needs covering, encrypted and clear alike. +STSD_CASES = [ + ("video", "HVC1", VIDEO_HEVC_CPD, KID, b"encv", {"width": 3840, "height": 2160}), + ("video", "HVC1", VIDEO_HEVC_CPD, None, b"hvc1", {"width": 3840, "height": 2160}), + ("video", "H264", VIDEO_AVC_CPD, None, b"avc1", {"width": 1280, "height": 720}), + ("audio", "AACL", AAC_LC_CPD, None, b"mp4a", {"channels": 2}), + ("audio", "EC-3", EC3_CPD, KID, b"enca", {"channels": 6}), + ("text", "TTML", "", None, b"stpp", {}), +] + + +@pytest.mark.parametrize("stream_type,fourcc,cpd,kid,expected_4cc,extra", STSD_CASES) +def test_stsd_carries_two_entries_for_sample_description_index_2(stream_type, fourcc, cpd, kid, expected_4cc, extra): + # A single entry leaves tfhd sample_description_index=2 dangling, and mp4decrypt + # skips those fragments without a word: exit 0, empty stderr, still encrypted. + init = build_init_segment( + stream_type=stream_type, + fourcc=fourcc, + codec_private_data=cpd, + timescale=10000000, + kid=kid, + **extra, + ) + offset = find_stsd(init) + assert struct.unpack_from(">I", init, offset + 12)[0] == 2, "index 2 would dangle" + + entries, cursor = [], offset + 16 + for _ in range(2): + size = struct.unpack_from(">I", init, cursor)[0] + entries.append(init[cursor : cursor + size]) + cursor += size + assert [e[4:8] for e in entries] == [expected_4cc, expected_4cc] + # stsd must span its header, count and both entries exactly. + assert struct.unpack_from(">I", init, offset)[0] == 16 + sum(len(e) for e in entries) + # The same has to hold for every container box wrapping it. + walk_box(init, 0, len(init)) + + def test_avc_init_structure(): init = build_init_segment( stream_type="video", diff --git a/unshackle/core/manifests/ism_init.py b/unshackle/core/manifests/ism_init.py index fff76966..b3aa96fb 100644 --- a/unshackle/core/manifests/ism_init.py +++ b/unshackle/core/manifests/ism_init.py @@ -773,7 +773,10 @@ def build_init_segment( else: raise NotImplementedError(f"Unsupported text FourCC: {fourcc}") - stsd = full_box(b"stsd", 0, 0, u32.pack(1) + sample_entry_box) + # Fragments may set tfhd sample_description_index=2 (server inits carry a second, + # identical entry). A lone entry leaves that dangling: mp4decrypt skips those + # fragments and demuxers stop after the first. + stsd = full_box(b"stsd", 0, 0, u32.pack(2) + sample_entry_box * 2) # --- empty sample tables (fragmented: real samples live in moof/traf) --- stbl = box( From d60d8abf53fd3422711239ecd8378f0c18d7e72c Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sat, 25 Jul 2026 15:55:31 -0600 Subject: [PATCH 10/40] fix(ism): repair synthesized init and verify fragments decrypted UNITY_MATRIX emitted 11 int32s instead of 9, oversizing mvhd/tkhd so pymp4 desynced and fTTML subtitles yielded no cues. Also rewrites PIFF senc to CENC so shaka can decrypt Smooth, labels HEV1 as hev1, and raises when a fragment survives decryption still carrying a sample-encryption box. --- tests/core/test_ism_init.py | 238 ++++++++++++++++++++++++ tests/tracks/test_fragment_decrypted.py | 155 +++++++++++++++ unshackle/commands/dl.py | 4 +- unshackle/core/manifests/dash.py | 2 + unshackle/core/manifests/hls.py | 2 + unshackle/core/manifests/ism.py | 7 + unshackle/core/manifests/ism_init.py | 108 +++++++++-- unshackle/core/tracks/track.py | 105 +++++++++++ 8 files changed, 609 insertions(+), 12 deletions(-) create mode 100644 tests/tracks/test_fragment_decrypted.py diff --git a/tests/core/test_ism_init.py b/tests/core/test_ism_init.py index a25e3193..0b9378eb 100644 --- a/tests/core/test_ism_init.py +++ b/tests/core/test_ism_init.py @@ -24,6 +24,7 @@ from unshackle.core.manifests.ism_init import ( parse_codec_private_data_colour, parse_codec_private_data_vui, parse_hevc_sps_format, + piff_senc_to_cenc, read_per_sample_iv_size, read_track_id, remove_emulation_prevention, @@ -578,3 +579,240 @@ def test_hvcc_profile_tier_level_is_nonzero(): level_idc = payload[12] assert profile_idc != 0 assert level_idc != 0 + + +def _piff_fragment(flags: int = 0x0, override: bool = False) -> bytes: + """Minimal moof+mdat fragment carrying sample encryption in the PIFF uuid box.""" + ivs = b"".join(bytes([i]) * 8 for i in range(1, 3)) + senc_payload = struct.pack(">I", 2) + ivs + if override: + flags |= 0x1 + senc_payload = b"\x00\x00\x01" + bytes([8]) + KID + senc_payload # AlgorithmID + IV_size + KID + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + flags.to_bytes(3, "big") + senc_payload) + tfhd = full_box(b"tfhd", 0, 0x20000, struct.pack(">I", 1)) + trun = full_box(b"trun", 0, 0x1, struct.pack(">II", 2, 0)) # data_offset placeholder + traf = box(b"traf", tfhd + trun + uuid_box) + moof = box(b"moof", full_box(b"mfhd", 0, 0, struct.pack(">I", 1)) + traf) + # patch trun data_offset to the real moof-relative start of the mdat payload + moof = moof.replace(struct.pack(">II", 2, 0), struct.pack(">II", 2, len(moof) + 8), 1) + return moof + box(b"mdat", b"\xab" * 64) + + +def test_piff_senc_rewritten_to_cenc_senc(): + frag = _piff_fragment() + out = piff_senc_to_cenc(frag) + # trun's data_offset is moof-relative; shrinking the box misaims decryption + assert len(out) == len(frag) + assert PIFF_SENC_UUID not in out + assert b"senc" in out and b"free" in out + assert struct.pack(">I", 2) + b"".join(bytes([i]) * 8 for i in range(1, 3)) in out + # moof size unchanged => the recorded data_offset still resolves + assert out[:8] == frag[:8] + assert out[len(out) - 72 :] == frag[len(frag) - 72 :] # mdat untouched + + +def test_piff_senc_override_header_stripped(): + frag = _piff_fragment(override=True) + out = piff_senc_to_cenc(frag) + assert len(out) == len(frag) + assert PIFF_SENC_UUID not in out + senc = out.index(b"senc") + assert out[senc + 4] == 0 # version + assert int.from_bytes(out[senc + 5 : senc + 8], "big") & 0x1 == 0 # override flag cleared + assert out[senc + 8 : senc + 12] == struct.pack(">I", 2) # sample_count directly follows + + +def test_piff_subsample_flag_survives_override_strip(): + # losing 0x2 is silent: no subsample map, so clear NAL headers get encrypted too + frag = _piff_fragment(flags=0x2, override=True) + out = piff_senc_to_cenc(frag) + senc = out.index(b"senc") + flags = int.from_bytes(out[senc + 5 : senc + 8], "big") + assert flags & 0x2, "subsample-encryption flag must survive" + assert not flags & 0x1, "override flag must be cleared" + + +def test_piff_rewrite_leaves_non_piff_fragments_alone(): + plain = box(b"moof", box(b"traf", full_box(b"tfhd", 0, 0, struct.pack(">I", 1)))) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(plain) is plain + assert piff_senc_to_cenc(b"") == b"" + + +def test_piff_rewrite_skips_traf_that_already_has_senc(): + # a second senc would leave the decrypter to choose between them + ivs = b"".join(bytes([i]) * 8 for i in range(1, 3)) + senc = full_box(b"senc", 0, 0, struct.pack(">I", 2) + ivs) + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0).to_bytes(3, "big") + struct.pack(">I", 2) + ivs) + frag = box(b"moof", box(b"traf", senc + uuid_box)) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(frag) is frag + + +def test_piff_rewrite_covers_every_moof_and_traf(): + frag = _piff_fragment() + _piff_fragment() + out = piff_senc_to_cenc(frag) + assert len(out) == len(frag) + assert PIFF_SENC_UUID not in out + assert out.count(b"senc") == 2 + + +def test_piff_rewrite_refuses_non_ctr_algorithm_id(): + # AlgorithmID 2 is AES-CBC but the sample entry says 'cenc'; rewriting yields garbage + ivs = b"".join(bytes([i]) * 8 for i in range(1, 3)) + payload = (2).to_bytes(3, "big") + bytes([8]) + KID + struct.pack(">I", 2) + ivs + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload) + frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(frag) is frag + + +def traf_children_tile_exactly(frag: bytes) -> bool: + """Every traf child's declared size must chain exactly to the traf end.""" + moof_size = struct.unpack(">I", frag[:4])[0] + traf_start = 8 + while frag[traf_start + 4 : traf_start + 8] != b"traf": + traf_start += struct.unpack(">I", frag[traf_start : traf_start + 4])[0] + traf_end = traf_start + struct.unpack(">I", frag[traf_start : traf_start + 4])[0] + assert traf_end <= moof_size + offset = traf_start + 8 + while offset < traf_end: + size = struct.unpack(">I", frag[offset : offset + 4])[0] + if size < 8 or offset + size > traf_end: + return False + offset += size + return offset == traf_end + + +def test_piff_rewrite_keeps_traf_children_tiling_exactly(): + # a mis-sized senc still "decrypts", just off the wrong bytes, so assert the sizes too + assert traf_children_tile_exactly(_piff_fragment()) + for frag in (_piff_fragment(), _piff_fragment(flags=0x2), _piff_fragment(flags=0x2, override=True)): + assert traf_children_tile_exactly(piff_senc_to_cenc(frag)), "rewritten traf no longer tiles" + + +def test_piff_rewrite_refuses_override_iv_size_mismatch(): + # tenc is the only IV width a decrypter consults; a disagreeing override misparses silently + ivs = b"".join(bytes([i]) * 16 for i in range(1, 3)) + payload = (1).to_bytes(3, "big") + bytes([16]) + KID + struct.pack(">I", 2) + ivs + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload) + frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(frag, iv_size=8) is frag, "8-byte tenc must reject a 16-byte override" + assert piff_senc_to_cenc(frag, iv_size=16) is not frag, "matching width must still rewrite" + + +def test_piff_rewrite_never_grows_output_on_malformed_boxes(): + # A declared size running past the buffer would extend the output over the mdat. + good = _piff_fragment() + truncations = [good[:n] for n in range(8, len(good), 7)] + oversized = bytearray(good) + struct.pack_into(">I", oversized, 0, len(good) + 4096) # moof claims more than exists + for frag in [*truncations, bytes(oversized), b"", b"\x00" * 12]: + out = piff_senc_to_cenc(frag) # must not raise + assert len(out) == len(frag), f"output grew: {len(frag)} -> {len(out)}" + + +def _text_fragment(seq: int, begin: str, end: str, text: str, sdi: int) -> bytes: + """A single-sample fTTML moof+mdat, mirroring what a Smooth text stream serves.""" + payload = ( + "" + "
" + f"

{text}

" + "
" + ).encode() + mfhd = full_box(b"mfhd", 0, 0, struct.pack(">I", seq)) + tfhd = full_box(b"tfhd", 0, 0x020002, struct.pack(">I", 1) + struct.pack(">I", sdi)) + tfdt = full_box(b"tfdt", 1, 0, struct.pack(">Q", 0)) + trun = full_box(b"trun", 0, 0x000201, struct.pack(">I", 1) + struct.pack(">i", 0) + struct.pack(">I", len(payload))) + return box(b"moof", mfhd + box(b"traf", tfhd + tfdt + trun)) + box(b"mdat", payload) + + +def test_unity_matrix_is_nine_int32s(): + # extras shift every later field (mvhd next_track_ID, tkhd width/height) and desync pymp4 + from unshackle.core.manifests.ism_init import UNITY_MATRIX + + assert len(UNITY_MATRIX) == 36 + + +@pytest.mark.parametrize("stream_type", ["video", "audio", "text"]) +def test_mvhd_tkhd_sizes_match_iso_layout(stream_type): + # a declared size over the spec payload makes parsers under-read and stop at the moov + kwargs = { + "video": dict(fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080), + "audio": dict(fourcc="AACL", codec_private_data=AAC_LC_CPD), + "text": dict(fourcc="TTML", codec_private_data=""), + }[stream_type] + init = build_init_segment(stream_type=stream_type, duration=6_000_000_000, language="eng", **kwargs) + + sizes = {} + for name in (b"mvhd", b"tkhd"): + at = init.index(name) - 4 + sizes[name] = struct.unpack(">I", init[at : at + 4])[0] + # version-1 mvhd: 8 header + 4 version/flags + 28 time fields + 80 trailer + assert sizes[b"mvhd"] == 120 + # version-1 tkhd: 8 header + 4 version/flags + 32 time fields + 60 trailer + assert sizes[b"tkhd"] == 104 + + +def test_tkhd_carries_video_dimensions(): + # width/height sit after the matrix, so a mis-sized matrix silently zeroes them + init = build_init_segment( + stream_type="video", fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080 + ) + at = init.index(b"tkhd") - 4 + tkhd = init[at : at + struct.unpack(">I", init[at : at + 4])[0]] + width, height = struct.unpack(">II", tkhd[-8:]) + assert (width >> 16, height >> 16) == (1920, 1080) + + +def test_pymp4_consumes_whole_init_and_reaches_fragments(): + # pymp4 must reach the mdat here, or Subtitle.parse finds nothing to yield cues from + from io import BytesIO + + from pymp4.parser import MP4 + + init = build_init_segment( + stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng" + ) + data = init + _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1) + types = [b.type for b in MP4.parse_stream(BytesIO(data))] + assert types == [b"ftyp", b"moov", b"moof", b"mdat"] + + +def test_fttml_subtitle_parse_yields_cues(): + from unshackle.core.tracks.subtitle import Subtitle + + init = build_init_segment( + stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng" + ) + data = ( + init + + _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1) + + _text_fragment(2, "00:00:03.000", "00:00:04.000", "second", sdi=2) + ) + caption_set = Subtitle.parse(data, Subtitle.Codec.fTTML) + cues = [c for lang in caption_set.get_languages() for c in caption_set.get_captions(lang)] + assert len(cues) == 2 + assert [c.get_text() for c in cues] == ["first", "second"] + + +@pytest.mark.parametrize( + "fourcc,expected", + [("HEV1", b"hev1"), ("hev1", b"hev1"), ("HVC1", b"hvc1"), ("hvc1", b"hvc1"), ("HEVC", b"hvc1"), ("H265", b"hvc1")], +) +def test_hevc_sample_entry_honours_manifest_fourcc(fourcc, expected): + # hev1 permits in-band parameter sets where hvc1 requires them in the sample + # entry. Real Smooth manifests ship FourCC="hev1", so relabelling it hvc1 + # misdescribes the stream. + init = build_init_segment( + stream_type="video", fourcc=fourcc, codec_private_data=VIDEO_HEVC_CPD, width=3840, height=2160 + ) + offset = find_stsd(init) + assert init[offset + 20 : offset + 24] == expected + + +def test_encrypted_hev1_frma_carries_the_same_fourcc(): + # frma names the original format the sinf wraps; if it drifts from the sample + # entry, a decrypter restores the wrong sample entry type. + init = build_init_segment( + stream_type="video", fourcc="HEV1", codec_private_data=VIDEO_HEVC_CPD, kid=KID, width=3840, height=2160 + ) + assert b"encv" in init + assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"hev1" diff --git a/tests/tracks/test_fragment_decrypted.py b/tests/tracks/test_fragment_decrypted.py new file mode 100644 index 00000000..f92cd8ff --- /dev/null +++ b/tests/tracks/test_fragment_decrypted.py @@ -0,0 +1,155 @@ +"""Self-check for ``assert_fragments_decrypted``: after decryption every moof must have lost +its sample-encryption box. A survivor means the tool silently skipped that fragment (the +tfhd.sample_description_index ceiling bug), which the moov-scoped has_encrypted_sample_entry +structurally cannot see. The guard never reads mdat, so stray byte runs there leave it unmoved. + +Run: uv run pytest tests/tracks/test_fragment_decrypted.py +""" + +from __future__ import annotations + +import struct +from pathlib import Path + +import pytest + +from unshackle.core.manifests.ism_init import PIFF_SENC_UUID +from unshackle.core.tracks.track import assert_fragments_decrypted + + +def _box(fmt: bytes, payload: bytes = b"") -> bytes: + return struct.pack(">I", 8 + len(payload)) + fmt + payload + + +def _senc(sample_count: int = 4) -> bytes: + return _box(b"senc", b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)) + + +def _piff_senc(sample_count: int = 4) -> bytes: + body = PIFF_SENC_UUID + b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count) + return _box(b"uuid", body) + + +def _moof(*children: bytes, sdi: int = 1) -> bytes: + tfhd = _box(b"tfhd", b"\x00\x00\x00\x02" + struct.pack(">II", 1, sdi)) # sample-description-index-present + return _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _box(b"traf", tfhd + b"".join(children))) + + +FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2") +MOOV = _box(b"moov", _box(b"mvex", _box(b"trex", b"\x00" * 24))) +MDAT = _box(b"mdat", b"\x00" * 64) +# a compressed payload can contain anything, including plausible-looking box runs +STRAY = _box(b"mdat", struct.pack(">I", 64) + b"senc" + b"\xbe\xef" * 8 + PIFF_SENC_UUID) + + +def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path: + f = tmp_path / name + f.write_bytes(b"".join(boxes)) + return f + + +def test_all_fragments_decrypted_passes(tmp_path: Path) -> None: + f = _write(tmp_path, "clean.mp4", FTYP, MOOV, _moof(), MDAT, _moof(sdi=2), MDAT) + assert_fragments_decrypted(f) + + +def test_surviving_piff_senc_raises(tmp_path: Path) -> None: + # the real bug: fragment 1 (index 1) decrypted, the rest (index 2) silently skipped + boxes = [FTYP, MOOV, _moof(), MDAT] + for _ in range(3): + boxes += [_moof(_piff_senc(), sdi=2), MDAT] + f = _write(tmp_path, "skipped.mp4", *boxes) + with pytest.raises(ValueError, match=r"3/4 fragment"): + assert_fragments_decrypted(f) + + +def test_surviving_cenc_senc_raises(tmp_path: Path) -> None: + f = _write(tmp_path, "cenc.mp4", FTYP, MOOV, _moof(_senc()), MDAT) + with pytest.raises(ValueError, match=r"1/1 fragment"): + assert_fragments_decrypted(f) + + +def test_stray_senc_bytes_in_mdat_ignored(tmp_path: Path) -> None: + f = _write(tmp_path, "stray.mp4", FTYP, MOOV, _moof(), STRAY, _moof(sdi=2), STRAY) + assert_fragments_decrypted(f) + + +def test_senc_text_without_plausible_size_ignored(tmp_path: Path) -> None: + # "senc" inside a free-box string has no valid box size before it + f = _write(tmp_path, "text.mp4", FTYP, MOOV, _moof(_box(b"free", b"presence sencoded here")), MDAT) + assert_fragments_decrypted(f) + + +def test_senc_bytes_inside_trun_payload_ignored(tmp_path: Path) -> None: + # a trun sample entry spelling "senc" after a size-looking duration; only a byte scan bites + trun = _box(b"trun", b"\x00\x00\x03\x01" + struct.pack(">I", 2) + struct.pack(">I", 512) + b"senc") + f = _write(tmp_path, "trun.mp4", FTYP, MOOV, _moof(trun), MDAT) + assert_fragments_decrypted(f) + + +def test_empty_senc_on_clear_fragment_ignored(tmp_path: Path) -> None: + # sample_count 0 marks a fragment with no protected samples, so the guard ignores it + f = _write(tmp_path, "empty.mp4", FTYP, MOOV, _moof(_senc(sample_count=0)), MDAT) + assert_fragments_decrypted(f) + + +def _piff_override(sample_count: int) -> bytes: + # flags&1 => AlgorithmID(3) + IV_size(1) + KID(16) precede sample_count + body = PIFF_SENC_UUID + b"\x00\x00\x00\x01" + b"\x00\x00\x08" + b"\x08" + b"\xaa" * 16 + return _box(b"uuid", body + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)) + + +def test_piff_override_flag_sample_count_offset(tmp_path: Path) -> None: + clear = _write(tmp_path, "ovr_clear.mp4", FTYP, MOOV, _moof(_piff_override(0)), MDAT) + assert_fragments_decrypted(clear) + f = _write(tmp_path, "ovr.mp4", FTYP, MOOV, _moof(_piff_override(5)), MDAT) + with pytest.raises(ValueError, match=r"1/1 fragment"): + assert_fragments_decrypted(f) + + +def test_senc_outside_traf_ignored(tmp_path: Path) -> None: + # a decrypted moof whose only "senc" sits in a sibling box outside any traf + f = _write(tmp_path, "sibling.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _senc()), MDAT) + assert_fragments_decrypted(f) + + +def test_senc_nested_under_non_traf_ignored(tmp_path: Path) -> None: + # senc two levels down under a non-traf container: only traf children count + meta = _box(b"meta", _senc()) + f = _write(tmp_path, "nested.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + meta), MDAT) + assert_fragments_decrypted(f) + + +def test_undersized_largesize_box_does_not_desync_walk(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + # size==1 reads a 16-byte header, so a largesize of 12 clears the 8-byte floor but + # still stalls the cursor onto payload; only a header-aware floor catches it + runt = struct.pack(">I", 1) + b"free" + struct.pack(">Q", 12) + f = _write(tmp_path, "runt.mp4", FTYP, MOOV, runt, _moof(_piff_senc()), MDAT, _moof(_piff_senc()), MDAT) + with caplog.at_level("WARNING"): + assert_fragments_decrypted(f) + assert "malformed box size" in caplog.text + + +def test_largesize_and_zero_size_boxes_walked(tmp_path: Path) -> None: + frag = _moof(_piff_senc()) + large = struct.pack(">I", 1) + b"mdat" + struct.pack(">Q", 16 + 64) + b"\x00" * 64 + trailing = struct.pack(">I", 0) + b"mdat" + b"\x00" * 64 # size 0 => runs to EOF + f = _write(tmp_path, "large.mp4", FTYP, MOOV, frag, large, trailing) + with pytest.raises(ValueError, match=r"1/1 fragment"): + assert_fragments_decrypted(f) + + +def test_malformed_box_size_warns_and_does_not_raise(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + # failing open is deliberate, but the abort must be logged so it cannot pass as clean + overrun = struct.pack(">I", 1 << 30) + b"free" + b"\x00" * 8 + f = _write(tmp_path, "overrun.mp4", FTYP, MOOV, overrun, _moof(_piff_senc()), MDAT) + with caplog.at_level("WARNING"): + assert_fragments_decrypted(f) + assert "malformed box size" in caplog.text + + +def test_missing_or_bogus_file_is_noop(tmp_path: Path) -> None: + assert_fragments_decrypted(tmp_path / "nope.mp4") + assert_fragments_decrypted(_write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4")) + truncated = _write(tmp_path, "cut.mp4", FTYP, MOOV, _moof(_piff_senc())[:20]) + assert_fragments_decrypted(truncated) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 852f5815..68ba3c13 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -67,7 +67,7 @@ from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.dv_fixup import apply_dv_fixup from unshackle.core.tracks.hybrid import Hybrid -from unshackle.core.tracks.track import has_encrypted_sample_entry +from unshackle.core.tracks.track import assert_fragments_decrypted, has_encrypted_sample_entry from unshackle.core.utilities import ( find_font_with_fallbacks, find_missing_langs, @@ -3088,6 +3088,8 @@ class dl: if drm and hasattr(drm, "decrypt"): drm.decrypt(track.path) if not isinstance(drm, MonaLisa): + # MonaLisa decrypts per segment; its decrypt() here is a no-op + assert_fragments_decrypted(track.path) has_decrypted = True events.emit(events.Types.TRACK_REPACKED, track=track) else: diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 5f773760..f41302a3 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -28,6 +28,7 @@ from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine from unshackle.core.events import events from unshackle.core.session import RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video +from unshackle.core.tracks.track import assert_fragments_decrypted from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.xml import load_xml @@ -570,6 +571,7 @@ class DASH: if drm: progress(downloaded="Decrypting", completed=0, total=None) drm.decrypt(save_path) + assert_fragments_decrypted(save_path) track.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 8a61a26b..4ccde219 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -33,6 +33,7 @@ from unshackle.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine from unshackle.core.events import events from unshackle.core.session import RnetResponse, RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video +from unshackle.core.tracks.track import assert_fragments_decrypted from unshackle.core.utilities import get_extension, is_close_match, log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.subprocess import log_tool_run @@ -794,6 +795,7 @@ class HLS: # with widevine we can merge all segments and decrypt once merge(to=merged_path, via=files, delete=True, include_map_data=True) drm.decrypt(merged_path) + assert_fragments_decrypted(merged_path) merged_path.rename(decrypted_path) else: # with other drm we must decrypt separately and then merge them diff --git a/unshackle/core/manifests/ism.py b/unshackle/core/manifests/ism.py index 23854c12..65296012 100644 --- a/unshackle/core/manifests/ism.py +++ b/unshackle/core/manifests/ism.py @@ -22,11 +22,13 @@ from unshackle.core.events import events from unshackle.core.manifests.ism_init import ( build_init_segment, parse_codec_private_data_vui, + piff_senc_to_cenc, read_per_sample_iv_size, read_track_id, ) from unshackle.core.session import RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Track, Tracks, Video +from unshackle.core.tracks.track import assert_fragments_decrypted from unshackle.core.utilities import log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.xml import load_xml @@ -521,6 +523,7 @@ class ISM: init_segment = ISM._init_segment(track, session_drm, first_segment) if init_segment: f.write(init_segment) + iv_size = (read_per_sample_iv_size(first_segment) if session_drm and first_segment else None) or 8 for index, segment_file in enumerate(segments_to_merge): if is_text_subtitle: # first segment was already read for the init synthesis, reuse it @@ -533,6 +536,9 @@ class ISM: .encode("utf8") ) f.write(segment_data) + elif session_drm: + segment_data = first_segment if index == 0 and first_segment else segment_file.read_bytes() + f.write(piff_senc_to_cenc(segment_data, iv_size)) elif index == 0 and first_segment: f.write(first_segment) else: @@ -547,6 +553,7 @@ class ISM: if session_drm: progress(downloaded="Decrypting", completed=0, total=None) session_drm.decrypt(save_path) + assert_fragments_decrypted(save_path) track.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=session_drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) diff --git a/unshackle/core/manifests/ism_init.py b/unshackle/core/manifests/ism_init.py index b3aa96fb..02bb1ac9 100644 --- a/unshackle/core/manifests/ism_init.py +++ b/unshackle/core/manifests/ism_init.py @@ -8,6 +8,10 @@ sends a ``moov``. The init box must be reconstructed from the manifest's or decryptor such as shaka-packager can parse the stream. Ported from yt-dlp's ``write_piff_header`` and N_m3u8DL-RE's ``MSSMoovProcessor`` with HEVC, Dolby Vision, EC-3, TTML and CENC (PIFF) support. + +``piff_senc_to_cenc`` rewrites the fragments themselves, which neither port does: +shaka-packager cannot read the PIFF sample-encryption box, so its payload is +retagged as a standard ``senc``. """ from __future__ import annotations @@ -28,13 +32,14 @@ u1616 = struct.Struct(">Hxx") s32 = struct.Struct(">i") # 3x3 transformation matrix (identity), as stored in tkhd/mvhd. +# Exactly 9 int32s; extras shift every field after it and desync pymp4. UNITY_MATRIX = ( s32.pack(0x10000) - + s32.pack(0) * 3 + + s32.pack(0) * 2 + s32.pack(0) + s32.pack(0x10000) + s32.pack(0) * 2 - + s32.pack(0) * 2 + + s32.pack(0) + s32.pack(0x40000000) ) @@ -308,7 +313,13 @@ def parse_codec_private_data_colour(fourcc: str, codec_private_data: bytes) -> O def iter_boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, Optional[bytes], int, int]]: - """Yield (type, uuid_usertype, payload_start, box_end) for each child box.""" + """Yield (type, uuid_usertype, payload_start, box_end) for each child box. + + Stops silently at the first box whose declared size cannot be walked. A short + result therefore reports how far the walk got, and a caller counting survivors + should treat it as unverified. box_end may exceed end when the declared size + overruns; clamp it before slicing. + """ offset = start while offset + 8 <= end: size = struct.unpack(">I", data[offset : offset + 4])[0] @@ -319,7 +330,7 @@ def iter_boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, Optio header = 16 if size == 0: size = end - offset - if size < 8: # corrupt box header; stop rather than loop forever + if size < header: # size below the consumed header desyncs the walk; stop return usertype = None if box_type == b"uuid" and offset + header + 16 <= end: @@ -426,6 +437,74 @@ def read_per_sample_iv_size(fragment: bytes) -> Optional[int]: return None +def piff_senc_to_cenc(fragment: bytes, iv_size: int = 8) -> bytes: + """Rewrite a fragment's moof/traf PIFF sample-encryption uuid boxes as standard 'senc'. + + shaka-packager does not consume the PIFF uuid form. It finds no per-sample IVs, falls + through to the constant-IV path, and aborts with "IV cannot be empty". mp4decrypt + accepts either form, and the payloads are identical once the optional override header + is stripped, so emitting 'senc' serves both decrypters. + + The rewrite preserves length: bytes reclaimed from the uuid header become a 'free' box. + The moof's byte length must stay the same. trun's data_offset resolves against tfhd's + base_data_offset (the moof start only when that field is absent), so moving the sample + data misaligns decryption and yields plausible garbage with no error. + + iv_size is the default_Per_Sample_IV_Size the init segment's tenc declares, read from + the first fragment by read_per_sample_iv_size. This function skips any fragment whose + override disagrees, since tenc is the only IV width a CENC decrypter consults and a + mismatch misparses every IV. + """ + out: Optional[bytearray] = None + limit = len(fragment) + trafs = [ + (body, min(box_end, limit)) + for moof_type, _, moof_body, moof_end in iter_boxes(fragment, 0, limit) + if moof_type == b"moof" + for box_type, _, body, box_end in iter_boxes(fragment, moof_body, min(moof_end, limit)) + if box_type == b"traf" + ] + for traf_body, traf_end in trafs: + # a second senc would leave the decrypter to choose between them + if any(bt == b"senc" for bt, _, _, _ in iter_boxes(fragment, traf_body, traf_end)): + continue + offset = traf_body + while offset + 8 <= traf_end: + size = struct.unpack(">I", fragment[offset : offset + 4])[0] + if size < 8 or offset + size > traf_end: + break + if ( + size >= 28 + and fragment[offset + 4 : offset + 8] == b"uuid" + and fragment[offset + 8 : offset + 24] == PIFF_SENC_UUID + ): + version = fragment[offset + 24] + flags = int.from_bytes(fragment[offset + 25 : offset + 28], "big") + # flag 0x1 = PIFF override header: AlgorithmID(3) + IV_size(1) + KID(16) + strip = 20 if flags & 0x1 else 0 + # AlgorithmID 2 is AES-CBC and a stray IV width misparses every IV; leave + # either alone so the decrypter fails loudly on a form it cannot handle + if ( + strip + and ( + size < 28 + strip # short-circuits before reading the override fields + or int.from_bytes(fragment[offset + 28 : offset + 31], "big") != 1 + or fragment[offset + 31] != iv_size + ) + ): + offset += size + continue + if offset + 28 + strip <= offset + size: + senc = full_box(b"senc", version, flags & ~0x1, fragment[offset + 28 + strip : offset + size]) + pad = size - len(senc) + if pad >= 8: + if out is None: + out = bytearray(fragment) + out[offset : offset + size] = senc + box(b"free", bytes(pad - 8)) + offset += size + return bytes(out) if out is not None else fragment + + def build_avcc(codec_private_data: bytes, nal_length_size: int = 4) -> bytes: """Build an avcC (AVC decoder config) box from SPS+PPS CodecPrivateData.""" nals = split_nal_units(codec_private_data) @@ -582,7 +661,7 @@ def build_sinf( schm = full_box(b"schm", 0, 0, b"cenc" + u32.pack(0x00010000)) tenc_payload = ( u8.pack(0) # reserved - + u8.pack(0) # default_crypt_byte_block / skip_byte_block (cenc) + + u8.pack(0) # reserved in v0 (crypt/skip nibbles are v1 only) + u8.pack(1) # default_isProtected + u8.pack(0 if constant_iv else iv_size) # default_Per_Sample_IV_Size + kid # default_KID (16 bytes) @@ -616,8 +695,10 @@ def build_init_segment( Build a complete ftyp + moov initialization segment. stream_type: "video" | "audio" | "text". - fourcc: Smooth FourCC ("H264"/"AVC1"/"DAVC", "HVC1"/"HEV1", "DVHE"/"DVH1", - "AACL"/"AACH", "EC-3", "TTML"). + fourcc: Smooth FourCC ("H264"/"AVC1"/"DAVC", "HVC1"/"HEV1"/"HEVC"/"H265", + "DVHE"/"DVH1", "AACL"/"AACH"/"AAC", "EC-3", "TTML"/"STPP"/"DFXP"). + Only "HEV1" yields an hev1 sample entry; the rest of that group yield + hvc1, which requires its parameter sets in the sample entry. codec_private_data: hex string from the manifest QualityLevel. nal_length_size: manifest NALUnitLengthField (bytes per NAL length prefix). kid: 16-byte default key id; when set, the sample entry is wrapped for CENC. @@ -638,6 +719,7 @@ def build_init_segment( ftyp = box(b"ftyp", b"isml" + u32.pack(1) + b"iso5" + b"iso6" + b"piff" + b"msdh") # --- mvhd --- + # version 1: at the 10 MHz ISM timescale a 32-bit duration overflows after ~429s mvhd = full_box( b"mvhd", 1, @@ -724,7 +806,10 @@ def build_init_segment( codec_fourcc = b"avc1" elif fourcc in ("HVC1", "HEV1", "HEVC", "H265"): config_box = build_hvcc(cpd, nal_length_size) - codec_fourcc = b"hvc1" + # hev1 permits in-band parameter sets where hvc1 requires them in the + # sample entry, so relabelling hev1 as hvc1 misdescribes the stream + # (and the frma inside sinf). + codec_fourcc = b"hev1" if fourcc == "HEV1" else b"hvc1" elif fourcc in ("DVHE", "DVH1"): # Dolby Vision over HEVC: same hvcC config, dvh1 sample entry. config_box = build_hvcc(cpd, nal_length_size) @@ -773,9 +858,10 @@ def build_init_segment( else: raise NotImplementedError(f"Unsupported text FourCC: {fourcc}") - # Fragments may set tfhd sample_description_index=2 (server inits carry a second, - # identical entry). A lone entry leaves that dangling: mp4decrypt skips those - # fragments and demuxers stop after the first. + # Fragments may set tfhd sample_description_index=2. A lone entry leaves that + # dangling: mp4decrypt skips those fragments and demuxers stop after the first. + # We duplicate entry 1; a manifest carries one CodecPrivateData, so there is no + # second config to build. The server's larger second entry only repeats its VPS. stsd = full_box(b"stsd", 0, 0, u32.pack(2) + sample_entry_box * 2) # --- empty sample tables (fragmented: real samples live in moof/traf) --- diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 53a74a0c..a2eca307 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -100,6 +100,110 @@ def has_encrypted_sample_entry(path: Path) -> bool: return False +def _senc_protects_samples(buf: bytes, body: int, end: int) -> bool: + """True if a senc/PIFF-uuid box describes protected samples. + + An empty senc (sample_count 0) sits on a genuinely clear fragment, so treating it as + a skipped one would raise on a healthy file. Field order follows + read_per_sample_iv_size. + """ + flags = int.from_bytes(buf[body + 1 : body + 4], "big") + pos = body + 4 + (20 if flags & 0x1 else 0) # PIFF override: AlgorithmID(3)+IV_size(1)+KID(16) + if pos + 4 > end: + return False + return int.from_bytes(buf[pos : pos + 4], "big") > 0 + + +def _moof_still_encrypted(moof: bytes) -> bool: + # structural walk: a 4CC byte scan collides with trun payload + from unshackle.core.manifests.ism_init import PIFF_SENC_UUID, iter_boxes + + body = 16 if int.from_bytes(moof[:4], "big") == 1 else 8 + for box_type, _, traf_body, traf_end in iter_boxes(moof, body, len(moof)): + if box_type != b"traf": + continue + for child, usertype, child_body, child_end in iter_boxes(moof, traf_body, traf_end): + if child == b"senc" or (child == b"uuid" and usertype == PIFF_SENC_UUID): + if _senc_protects_samples(moof, child_body, child_end): + return True + return False + + +def assert_fragments_decrypted(path: Path) -> None: + """Raise if any fragment survived decryption still carrying a sample-encryption box. + + mp4decrypt strips senc from every fragment it processes and leaves it on the ones it + skips, so a survivor marks a silent skip. One cause is a tfhd.sample_description_index + dangling past the stsd entry count, which mp4decrypt reports as success (exit 0, no + stderr) while leaving the payload as ciphertext. Unlike has_encrypted_sample_entry + (moov-scoped) this is fragment-scoped; the moov comes out clean in that failure. + + The guarantee runs in one direction: a survivor proves a skip, while a clean pass is + only as good as the decrypter's own behaviour. shaka-packager (the default + `decryption` backend) remuxes into a single fragment and drops senc whether or not it + decrypted, so this cannot fire on shaka output. The walk is unbuffered and reads only + box headers; it seeks over mdat and never reads it. A malformed box size aborts the + walk with a warning and returns normally, so a file this function cannot parse is + never escalated into a raise. The logged warning is the only signal of that; a caller + sees the same silent return it gets from a verified clean file. + """ + surviving = 0 + total = 0 + first_offset = None + try: + file_size = path.stat().st_size + with path.open("rb", buffering=0) as f: + start = 0 + while start + 8 <= file_size: + f.seek(start) + header = f.read(8) + if len(header) < 8: + break + size = int.from_bytes(header[:4], "big") + box_header = 8 + if size == 1: # 64-bit largesize + large = f.read(8) + if len(large) < 8: + break + size = int.from_bytes(large, "big") + box_header = 16 + elif size == 0: # box runs to EOF + size = file_size - start + # below the consumed header the cursor stalls and desyncs onto payload + if size < box_header or start + size > file_size: + logging.getLogger("track").warning( + f"{path.name}: malformed box size at offset {start}; cannot verify the " + "remaining fragments were decrypted." + ) + break + if header[4:8] == b"moof": + total += 1 + f.seek(start) + if _moof_still_encrypted(f.read(size)): + surviving += 1 + if first_offset is None: + first_offset = start + start += size + except Exception as e: + logging.getLogger("track").warning(f"{path.name}: fragment decryption check did not complete ({e}).") + return + if surviving: + log_event( + "decrypt_fragments_still_encrypted", + level="ERROR", + message=f"{surviving}/{total} fragments still encrypted after decryption", + file=path.name, + surviving=surviving, + total=total, + first_offset=first_offset, + ) + raise ValueError( + f"{path.name}: {surviving}/{total} fragment(s) still encrypted after decryption (first at " + f"byte {first_offset}). The decrypt tool skipped them silently, so check " + f"tfhd.sample_description_index against the stsd entry count in the init segment." + ) + + @dataclass class DownloadContext: """Shared arguments passed to each manifest's ``download_track``.""" @@ -428,6 +532,7 @@ class Track: if drm: progress(downloaded="Decrypting", completed=0, total=None) drm.decrypt(save_path) + assert_fragments_decrypted(save_path) self.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=self, drm=drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) From 41106b0d248f26c3cebbe282a81410187b633d0c Mon Sep 17 00:00:00 2001 From: CodeName393 Date: Sun, 26 Jul 2026 18:16:36 +0900 Subject: [PATCH 11/40] fix(core): strip trailing spaces and dots in template formatting to prevent WinError 3 - Clean up redundant spaces and separators when conditional tags (e.g. {year?}) are omitted. - Strip leading and trailing spaces and dots in sanitize_filename and TemplateFormatter. - Prevent FileNotFoundError [WinError 3] caused by trailing spaces in Windows directory paths. --- unshackle/core/utilities.py | 1 + unshackle/core/utils/template_formatter.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index b86171c0..50a64755 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -142,6 +142,7 @@ def sanitize_filename(filename: str, spacer: str = ".") -> str: filename = re.sub(r"[:; ]", spacer, filename) # structural chars to (spacer) filename = re.sub(r"[\\*!?¿,'\"" "<>|$#~]", "", filename) # not filename safe chars filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s + filename = filename.strip(" .") # strip leading and trailing spaces and dots for OS path safety return filename diff --git a/unshackle/core/utils/template_formatter.py b/unshackle/core/utils/template_formatter.py index 9d0e237e..080f8192 100644 --- a/unshackle/core/utils/template_formatter.py +++ b/unshackle/core/utils/template_formatter.py @@ -96,11 +96,13 @@ class TemplateFormatter: # Clean up multiple consecutive dots/separators and other artifacts result = re.sub(r"\.{2,}", ".", result) # Multiple dots -> single dot result = re.sub(r"\s{2,}", " ", result) # Multiple spaces -> single space - result = re.sub(r"-{2,}", "-", result) # Multiple dashes -> single dash - result = re.sub(r"^[\.\s\-]+|[\.\s\-]+$", "", result) # Remove leading/trailing dots, spaces, dashes + result = re.sub(r"(?:-\s*)+-", "-", result) # Multiple dashes (with optional spaces) -> single dash result = re.sub(r"\.-", "-", result) # Remove dots before dashes (for dot-based templates) result = re.sub(r"[\.\s]+\)", ")", result) # Remove dots/spaces before closing parentheses result = re.sub(r"\(\s*\)", "", result) # Remove empty parentheses (empty conditional) + result = re.sub(r"\s{2,}", " ", result) # Multiple spaces after parenthesis removal -> single space + result = re.sub(r"(?:-\s*)+-", "-", result) # Multiple dashes (with optional spaces) -> single dash + result = re.sub(r"^[\.\s\-]+|[\.\s\-]+$", "", result) # Remove leading/trailing dots, spaces, dashes # Determine the appropriate separator based on template style # Count separator characters between variables (between } and {) @@ -114,6 +116,8 @@ class TemplateFormatter: else: result = sanitize_filename(result, spacer=".") + result = result.strip(" .") + if not result or result.isspace(): log.warning("Template formatting resulted in empty filename, using fallback") return "untitled" From b74b64825ef9da9fd125126c98bdac364a66d99d Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sun, 26 Jul 2026 09:48:58 -0600 Subject: [PATCH 12/40] fix(templates): use one separator style across a whole folder path Detect the dot/space style from the full folder template instead of per segment, so `{artist}/{album} ({year})` no longer yields `Some.Artist/Some Album (2025)`. Also drops empty `[]` groups like the existing `()` handling and removes redundant cleanup passes. --- docs/guide/output-and-naming.md | 14 +- tests/core/test_template_formatter.py | 292 +++++++++++++++++++++ unshackle/core/titles/episode.py | 7 +- unshackle/core/titles/movie.py | 7 +- unshackle/core/titles/music.py | 7 +- unshackle/core/titles/song.py | 7 +- unshackle/core/utils/template_formatter.py | 39 +-- 7 files changed, 339 insertions(+), 34 deletions(-) create mode 100644 tests/core/test_template_formatter.py diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 8ecd6965..1019f124 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -71,7 +71,7 @@ The recognized kinds are: ### Separator style (dots vs. spaces) -unshackle looks at the characters you place *between* variables to decide the filename's separator style. If spaces outnumber dots in your template, it sanitizes with spaces; otherwise it uses dots (the scene-style default). This is also applied to the auto-generated date separators for daily/dated content. +unshackle looks at the characters you place *between* variables to decide the filename's separator style. If spaces outnumber dots in your template, it sanitizes with spaces; otherwise it uses dots (the scene-style default). This is also applied to the auto-generated date separators for daily/dated content, and to every segment of a nested folder template. === "Scene style (dots)" @@ -184,7 +184,7 @@ Folder templates are nested under a special `folder` key inside `output_template albums: "{artist} - {album} ({year})" ``` -The per-kind folder keys are `movies`, `series`, `songs`, and `albums`. Any other key produces a startup warning. Path separators (`/` or `\`) are allowed in folder templates. Each segment is formatted independently, so you can build nested directory structures like `Show/Season 01`. +The per-kind folder keys are `movies`, `series`, `songs`, and `albums`. Any other key produces a startup warning. Path separators (`/` or `\`) are allowed in folder templates. Each segment is formatted independently, so you can build nested directory structures like `Show/Season 01`. unshackle reads the separator style from the whole template, so every segment is spaced alike. **Fallback behavior:** @@ -383,18 +383,18 @@ how the optional (`?`) variables appear and disappear. series: "{title} ({year})/Season {season}" ``` ```text - The Show (2023)/Season.S02/ + The Show (2023)/Season S02/ └─ The.Show.S02E04.The.Reckoning.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG.mkv ``` - Each folder segment picks its separator style independently: `{title} ({year})` has - spaces around its variables, but `Season {season}` alone counts as dot style, so its - space becomes a dot. + unshackle decides the separator style once for the whole folder template, and every + segment then follows it. Spaces outnumber dots in `{title} ({year})/Season {season}`, + so `Season {season}` keeps its space even though it holds only one variable. When an episode has no on-screen name, `{episode_name?}` disappears cleanly: ```text - The Show (2023)/Season.S02/ + The Show (2023)/Season S02/ └─ The.Show.S02E05.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG.mkv ``` diff --git a/tests/core/test_template_formatter.py b/tests/core/test_template_formatter.py new file mode 100644 index 00000000..34716208 --- /dev/null +++ b/tests/core/test_template_formatter.py @@ -0,0 +1,292 @@ +"""Separator cleanup around omitted conditionals in TemplateFormatter. + +An omitted `{var?}` must not leave separator debris behind. Trailing spaces are the +dangerous case: the formatter sanitizes each path segment separately, and Windows +drops a trailing space when it creates a directory but not when it looks one up, so +the download fails with FileNotFoundError [WinError 3]. See #107 and #144. +""" + +from __future__ import annotations + +import pytest + +from unshackle.core.config import config +from unshackle.core.titles.episode import Episode +from unshackle.core.titles.movie import Movie +from unshackle.core.titles.music import Song as MusicSong +from unshackle.core.titles.song import Song +from unshackle.core.utilities import sanitize_filename +from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer + + +@pytest.mark.parametrize( + ("template", "context", "expected"), + [ + # An omitted conditional inside brackets must take the whole group with it. + ("{title} ({year?})", {"title": "Some Movie", "year": ""}, "Some Movie"), + ("{title} [{quality?}]", {"title": "Some Movie", "quality": ""}, "Some Movie"), + ("{title} ({year?}) [{quality?}]", {"title": "T", "year": "", "quality": ""}, "T"), + # The group survives when the value is present. + ("{title} ({year?})", {"title": "Some Movie", "year": "2024"}, "Some Movie (2024)"), + # #107: the dash wins, so the release tag stays attached. + ( + "{title}.{year}.{atmos?}-{tag}", + {"title": "Some Movie", "year": "2024", "atmos": "", "tag": "GRP"}, + "Some.Movie.2024-GRP", + ), + ( + "{title}.{year}.{atmos?}-{tag}", + {"title": "Some Movie", "year": "2024", "atmos": "Atmos", "tag": "GRP"}, + "Some.Movie.2024.Atmos-GRP", + ), + # A conditional between two dashes must not leave a doubled separator. + ("{title} - {edition?} - {year}", {"title": "A", "edition": "", "year": "2024"}, "A - 2024"), + # Consecutive omitted conditionals collapse to nothing. + ("{a?} {b?} {title}", {"a": "", "b": "", "title": "T"}, "T"), + ("{show} S{season}E{ep} {name?}", {"show": "Show", "season": "01", "ep": "02", "name": ""}, "Show S01E02"), + ], +) +def test_omitted_conditional_leaves_no_separator_debris(template, context, expected): + assert TemplateFormatter(template).format(context) == expected + + +@pytest.mark.parametrize( + ("template", "context"), + [ + ("{title} ({year?})", {"title": "Some Movie", "year": ""}), + ("{title} [{quality?}]", {"title": "Some Movie", "quality": ""}), + ("{title}.{quality?}", {"title": "Some Movie", "quality": ""}), + ("{title} - {tag?}", {"title": "Some Movie", "tag": ""}), + ], +) +def test_result_is_safe_as_a_windows_path_segment(template, context): + """Windows strips these when creating a directory but not when looking one up.""" + result = TemplateFormatter(template).format(context) + assert result == result.strip(" ."), f"unsafe path segment: {result!r}" + + +def test_empty_result_falls_back(): + assert TemplateFormatter("{title?}").format({"title": ""}) == "untitled" + + +def test_missing_required_variable_raises(): + with pytest.raises(ValueError, match="Missing required template variables"): + TemplateFormatter("{title}.{year}").format({"title": "T"}) + + +@pytest.mark.parametrize( + ("raw", "spacer", "expected"), + [ + ("Some Movie ", " ", "Some Movie"), + (" Some Movie", " ", "Some Movie"), + ("Some Movie...", ".", "Some.Movie"), + ("S.W.A.T.", ".", "S.W.A.T"), + ("...Some Movie...", ".", "Some.Movie"), + ], +) +def test_sanitize_filename_strips_unsafe_edges(raw, spacer, expected): + assert sanitize_filename(raw, spacer) == expected + + +def test_sanitize_filename_keeps_interior_dots(): + assert sanitize_filename("Mr. Robot", ".") == "Mr.Robot" + + +@pytest.mark.parametrize( + ("template", "context", "expected"), + [ + # Space-separated style. + ("{title} ({year})", {"title": "Some Movie", "year": "2024"}, "Some Movie (2024)"), + ( + "{title} {year} {quality} {source}", + {"title": "Some Movie", "year": "2024", "quality": "1080p", "source": "WEB-DL"}, + "Some Movie 2024 1080p WEB-DL", + ), + # Dot-separated style. + ( + "{title}.{year}.{quality}.{source}", + {"title": "Some Movie", "year": "2024", "quality": "1080p", "source": "WEB-DL"}, + "Some.Movie.2024.1080p.WEB-DL", + ), + # Interior dots in the title itself survive both styles. + ("{title}.{year}", {"title": "Mr. Robot", "year": "2015"}, "Mr.Robot.2015"), + ("{title} ({year})", {"title": "Mr. Robot", "year": "2015"}, "Mr. Robot (2015)"), + ("{title}.{year}", {"title": "S.W.A.T.", "year": "2017"}, "S.W.A.T.2017"), + ("{title} ({year})", {"title": "S.W.A.T.", "year": "2017"}, "S.W.A.T. (2017)"), + # A decimal inside a codec name is not a separator. + ( + "{title}.{year}.{audio}-{tag}", + {"title": "Some Movie", "year": "2024", "audio": "DDP5.1", "tag": "GRP"}, + "Some.Movie.2024.DDP5.1-GRP", + ), + ], +) +def test_separator_styles_keep_interior_spacing(template, context, expected): + assert TemplateFormatter(template).format(context) == expected + + +class DummyService: + pass + + +class StubMediaInfo: + """Minimal MediaInfo stand-in: the template context only reads track lists.""" + + video_tracks: list = [] + audio_tracks: list = [] + + +@pytest.fixture +def folder_config(monkeypatch): + """Clear the global template config, and restore it after each test.""" + monkeypatch.setattr(config, "folder_templates", {}) + monkeypatch.setattr(config, "folder_template", "") + monkeypatch.setattr(config, "output_template", {}) + return config + + +def make_movie(): + return Movie(id_="movie-0001", service=DummyService, name="Some Movie") + + +def make_episode(**overrides): + kwargs = dict(id_="episode-0001", service=DummyService, title="Some Show", season=1, number=2, name="Pilot") + kwargs.update(overrides) + return Episode(**kwargs) + + +def make_song(**overrides): + kwargs = dict( + id_="track-0001", + service=DummyService, + name="Some Track", + artist="Some Artist", + album="Some Album", + track=1, + disc=1, + year=2025, + ) + kwargs.update(overrides) + return Song(**kwargs) + + +def make_album_song(**overrides): + kwargs = dict( + id_="track-0001", + service=DummyService, + name="Some Track", + artist="Some Artist", + album="Some Album", + track=1, + disc=1, + year=2025, + album_artist="Some Artist", + ) + kwargs.update(overrides) + return MusicSong(**kwargs) + + +def assert_safe_path(result: str): + """No segment may start or end with a space or dot, or Windows loses the directory.""" + segments = result.split("/") + assert all(segments), f"empty segment in {result!r}" + assert all(seg == seg.strip(" .") for seg in segments), f"unsafe segment in {result!r}" + + +@pytest.mark.parametrize( + ("kind", "make_title", "template", "expected"), + [ + ("movies", make_movie, "Movies/{title} ({year?})", "Movies/Some Movie"), + ("series", make_episode, "Shows/{title} ({year?})/Season {season}", "Shows/Some Show/Season S01"), + ("songs", make_song, "Music/{artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"), + ("albums", make_album_song, "Music/{album_artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"), + ], +) +def test_folder_templates_keep_path_separators(folder_config, kind, make_title, template, expected): + """`/` survives because each segment is formatted separately, but edge spaces do not.""" + folder_config.folder_templates = {kind: template} + result = make_title().get_filename(StubMediaInfo(), folder=True) + assert result == expected + assert_safe_path(result) + + +@pytest.mark.parametrize( + ("kind", "make_title", "template"), + [ + ("movies", make_movie, "{title} ({year?})"), + ("series", make_episode, "{title} ({year?})"), + ("songs", make_song, "{artist} - {album} [{genre?}]"), + ("albums", make_album_song, "{album_artist} - {album} [{genre?}]"), + ], +) +def test_folder_names_are_safe_when_conditionals_are_omitted(folder_config, kind, make_title, template): + folder_config.folder_templates = {kind: template} + assert_safe_path(make_title().get_filename(StubMediaInfo(), folder=True)) + + +@pytest.mark.parametrize( + ("template", "expected"), + [ + ("Shows/{title}.{year?}/Season.{season}", "Shows/Some.Show/Season.S01"), + ("Shows/{title} {year?}/Season {season}", "Shows/Some Show/Season S01"), + ], +) +def test_series_folder_handles_both_separator_styles(folder_config, template, expected): + folder_config.folder_templates = {"series": template} + result = make_episode().get_filename(StubMediaInfo(), folder=True) + assert result == expected + assert_safe_path(result) + + +@pytest.mark.parametrize( + ("template", "expected"), + [ + # A segment with one variable has no gap to judge by, so it follows the whole + # path's style instead of falling back to a dot. + ("Music/{artist}/{album} ({year?})", "Music/Some Artist/Some Album (2025)"), + ("Music/{artist}/{album}.{year?}", "Music/Some.Artist/Some.Album.2025"), + ("{artist}/{album}", "Some.Artist/Some.Album"), + ], +) +def test_single_variable_segments_follow_the_whole_path(folder_config, template, expected): + folder_config.folder_templates = {"songs": template} + result = make_song().get_filename(StubMediaInfo(), folder=True) + assert result == expected + assert_safe_path(result) + + +@pytest.mark.parametrize( + ("template", "expected"), + [ + ("{title}.{year}.{quality}", "."), + ("{title} {year} {quality}", " "), + ("{title} ({year})", " "), + ("Movies/{title} ({year?})", " "), + ("Shows/{title}.{year?}/Season.{season}", "."), + ("{title}", "."), # nothing between two variables, so scene style wins + ], +) +def test_detect_spacer(template, expected): + assert detect_spacer(template) == expected + + +@pytest.mark.parametrize( + ("series_template", "expected"), + [ + # Deriving a show folder strips the episode tokens, so the template ends on a + # separator once {episode_name?} goes too. + ("{title}.{year?}.{season_episode}.{episode_name?}", "Some.Show.S01"), + ("{title} {year?} {season_episode} {episode_name?}", "Some Show S01"), + ], +) +def test_series_folder_derived_from_output_template(folder_config, series_template, expected): + """With no folder template set, the show folder comes from the file template.""" + folder_config.output_template = {"series": series_template} + result = make_episode().get_filename(StubMediaInfo(), folder=True) + assert result == expected + assert_safe_path(result) + + +@pytest.mark.parametrize("make_title", [make_movie, make_episode, make_song, make_album_song]) +def test_folder_fallbacks_need_no_template(folder_config, make_title): + assert_safe_path(make_title().get_filename(StubMediaInfo(), folder=True)) diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index ce64db69..c3678d88 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -12,7 +12,7 @@ from sortedcontainers import SortedKeyList from unshackle.core.config import config from unshackle.core.titles.title import Title from unshackle.core.utilities import sanitize_filename -from unshackle.core.utils.template_formatter import TemplateFormatter +from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer class Episode(Title): @@ -146,8 +146,11 @@ class Episode(Title): context = self._build_template_context(media_info, show_service) context["season"] = self._folder_season() context["year"] = self.year or "" # folders keep the year + spacer = detect_spacer(template) # one style for the whole path segments = [ - TemplateFormatter(seg).format(context) for seg in re.split(r"[\\/]", template) if seg.strip() + TemplateFormatter(seg, spacer).format(context) + for seg in re.split(r"[\\/]", template) + if seg.strip() ] return "/".join(s for s in segments if s) diff --git a/unshackle/core/titles/movie.py b/unshackle/core/titles/movie.py index 399e9d0c..f02a6223 100644 --- a/unshackle/core/titles/movie.py +++ b/unshackle/core/titles/movie.py @@ -10,7 +10,7 @@ from sortedcontainers import SortedKeyList from unshackle.core.config import config from unshackle.core.titles.title import Title from unshackle.core.utilities import sanitize_filename -from unshackle.core.utils.template_formatter import TemplateFormatter +from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer class Movie(Title): @@ -63,8 +63,11 @@ class Movie(Title): template = config.get_folder_template("movies") if template: context = self._build_template_context(media_info, show_service) + spacer = detect_spacer(template) # one style for the whole path segments = [ - TemplateFormatter(seg).format(context) for seg in re.split(r"[\\/]", template) if seg.strip() + TemplateFormatter(seg, spacer).format(context) + for seg in re.split(r"[\\/]", template) + if seg.strip() ] return "/".join(s for s in segments if s) name = f"{self.name}" diff --git a/unshackle/core/titles/music.py b/unshackle/core/titles/music.py index c3a77882..41aba6ff 100644 --- a/unshackle/core/titles/music.py +++ b/unshackle/core/titles/music.py @@ -9,7 +9,7 @@ from sortedcontainers import SortedKeyList from unshackle.core.config import config from unshackle.core.titles.title import Title from unshackle.core.utilities import sanitize_filename -from unshackle.core.utils.template_formatter import TemplateFormatter +from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer class Song(Title): @@ -171,8 +171,11 @@ class Song(Title): template = config.get_folder_template("albums") or config.get_folder_template("songs") if template: context = self._build_template_context(media_info, show_service) + spacer = detect_spacer(template) # one style for the whole path segments = [ - TemplateFormatter(seg).format(context) for seg in re.split(r"[\\/]", template) if seg.strip() + TemplateFormatter(seg, spacer).format(context) + for seg in re.split(r"[\\/]", template) + if seg.strip() ] return "/".join(s for s in segments if s) name = f"{self.artist} - {self.album}" diff --git a/unshackle/core/titles/song.py b/unshackle/core/titles/song.py index 298f1792..af114c93 100644 --- a/unshackle/core/titles/song.py +++ b/unshackle/core/titles/song.py @@ -10,7 +10,7 @@ from sortedcontainers import SortedKeyList from unshackle.core.config import config from unshackle.core.titles.title import Title from unshackle.core.utilities import sanitize_filename -from unshackle.core.utils.template_formatter import TemplateFormatter +from unshackle.core.utils.template_formatter import TemplateFormatter, detect_spacer class Song(Title): @@ -98,8 +98,11 @@ class Song(Title): template = config.get_folder_template("songs") if template: context = self._build_template_context(media_info, show_service) + spacer = detect_spacer(template) # one style for the whole path segments = [ - TemplateFormatter(seg).format(context) for seg in re.split(r"[\\/]", template) if seg.strip() + TemplateFormatter(seg, spacer).format(context) + for seg in re.split(r"[\\/]", template) + if seg.strip() ] return "/".join(s for s in segments if s) name = f"{self.artist} - {self.album}" diff --git a/unshackle/core/utils/template_formatter.py b/unshackle/core/utils/template_formatter.py index 080f8192..47c777a6 100644 --- a/unshackle/core/utils/template_formatter.py +++ b/unshackle/core/utils/template_formatter.py @@ -9,6 +9,16 @@ from unshackle.core.utilities import sanitize_filename log = logging.getLogger(__name__) +def detect_spacer(template: str) -> str: + """Dot or space, whichever the template uses between its variables. + + Falls back to a dot (scene style) when the template has nothing between two + variables, such as a folder path segment with only one variable. + """ + separator_text = "".join(re.findall(r"\}([^{]*)\{", template)) + return " " if separator_text.count(" ") > separator_text.count(".") else "." + + class TemplateFormatter: """ Template formatter for custom filename patterns. @@ -17,13 +27,17 @@ class TemplateFormatter: Example: '{title}.{year}.{quality?}.{source}-{tag}' """ - def __init__(self, template: str): + def __init__(self, template: str, spacer: str | None = None): """Initialize the template formatter. Args: template: Template string with variables in {variable} format + spacer: Force a separator style instead of detecting it from `template`. + Folder paths pass the style of the whole path, so every segment is + spaced alike even when it holds a single variable. """ self.template = template + self.spacer = spacer or detect_spacer(template) self.variables = self._extract_variables() def _extract_variables(self) -> list[str]: @@ -95,28 +109,15 @@ class TemplateFormatter: # Clean up multiple consecutive dots/separators and other artifacts result = re.sub(r"\.{2,}", ".", result) # Multiple dots -> single dot - result = re.sub(r"\s{2,}", " ", result) # Multiple spaces -> single space - result = re.sub(r"(?:-\s*)+-", "-", result) # Multiple dashes (with optional spaces) -> single dash result = re.sub(r"\.-", "-", result) # Remove dots before dashes (for dot-based templates) - result = re.sub(r"[\.\s]+\)", ")", result) # Remove dots/spaces before closing parentheses - result = re.sub(r"\(\s*\)", "", result) # Remove empty parentheses (empty conditional) - result = re.sub(r"\s{2,}", " ", result) # Multiple spaces after parenthesis removal -> single space + result = re.sub(r"[\.\s]+([)\]])", r"\1", result) # Remove dots/spaces before closing bracket + result = re.sub(r"\(\s*\)|\[\s*\]", "", result) # Remove empty brackets (empty conditional) + # bracket removal is what creates neighbouring separators, so collapse after it + result = re.sub(r"\s{2,}", " ", result) # Multiple spaces -> single space result = re.sub(r"(?:-\s*)+-", "-", result) # Multiple dashes (with optional spaces) -> single dash result = re.sub(r"^[\.\s\-]+|[\.\s\-]+$", "", result) # Remove leading/trailing dots, spaces, dashes - # Determine the appropriate separator based on template style - # Count separator characters between variables (between } and {) - between_vars = re.findall(r"\}([^{]*)\{", self.template) - separator_text = "".join(between_vars) - dot_count = separator_text.count(".") - space_count = separator_text.count(" ") - - if space_count > dot_count: - result = sanitize_filename(result, spacer=" ") - else: - result = sanitize_filename(result, spacer=".") - - result = result.strip(" .") + result = sanitize_filename(result, spacer=self.spacer) if not result or result.isspace(): log.warning("Template formatting resulted in empty filename, using fallback") From 8bacb247ee7ffb9a2c181f740b43a5612fc0a384 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sun, 26 Jul 2026 18:22:46 -0600 Subject: [PATCH 13/40] fix(lang): surface orig in errors and flag the original audio track over the API Error messages named the resolved language for a user who only ever defaulted to orig. Audio tracks now carry is_original, resolved through Tracks.by_language so it agrees with the downloader. --- docs/dev/rest-api/endpoints.md | 5 ++- tests/lang_selection/test_as_requested.py | 24 ++++++++++ .../lang_selection/test_original_audio_ids.py | 43 ++++++++++++++++++ unshackle/commands/dl.py | 39 +++++++++++++--- unshackle/core/api/handlers.py | 44 ++++++++++++++++--- unshackle/core/utilities.py | 5 +++ 6 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 tests/lang_selection/test_as_requested.py create mode 100644 tests/lang_selection/test_original_audio_ids.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 0951e5b7..702dd98a 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -233,6 +233,9 @@ Each serialized title carries `type` (`"episode"`, `"movie"`, or `"other"`), `na List the video, audio, and subtitle tracks for a title. For series, you can scope to specific episodes. +!!! tip "Spotting the original-language audio" + Each audio track carries `is_original`, which marks the track the default `lang` of `orig` selects. Unshackle resolves it with the same language matcher the downloader uses, so it stays correct for regional variants a client cannot separate from tag text alone (`pt` marks `pt-BR` rather than `pt-PT`). Services need not report a title's language, so every track can legitimately come back `false`. + **Request body** | Field | Type | Required | Description | @@ -262,7 +265,7 @@ When both `season` and `episode` are given, they are combined into a `"{season}x "audio": [ { "id": "a-1", "codec": "EC3", "codec_display": "DD+", - "bitrate": 640, "channels": "5.1", "language": "en", + "bitrate": 640, "channels": "5.1", "language": "en", "is_original": true, "atmos": false, "descriptive": false, "drm": [], "descriptor": "DASH" } ], diff --git a/tests/lang_selection/test_as_requested.py b/tests/lang_selection/test_as_requested.py new file mode 100644 index 00000000..36b65abf --- /dev/null +++ b/tests/lang_selection/test_as_requested.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Optional + +import pytest + +from unshackle.core.utilities import as_requested + + +@pytest.mark.parametrize( + "tokens,orig_token,expected", + [ + # the user passed no -l, so orig resolved to en behind their back; report orig + (["en"], "en", "orig"), + (["en", "ja"], "en", "orig, ja"), + # the user named en, so report en + (["en"], None, "en"), + (["ja"], "en", "ja"), + (["en", "ja"], None, "en, ja"), + ([], "en", ""), + ], +) +def test_as_requested(tokens: list[str], orig_token: Optional[str], expected: str) -> None: + assert as_requested(tokens, orig_token) == expected diff --git a/tests/lang_selection/test_original_audio_ids.py b/tests/lang_selection/test_original_audio_ids.py new file mode 100644 index 00000000..74710714 --- /dev/null +++ b/tests/lang_selection/test_original_audio_ids.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Optional + +import pytest + +from unshackle.core.api.handlers import original_audio_ids + + +class FakeTrack: + def __init__(self, language: str) -> None: + self.id = language + self.language = language + + +class FakeTitle: + def __init__(self, language: Optional[str]) -> None: + self.language = language + + +@pytest.mark.parametrize( + "title_language,track_languages,expected", + [ + # paradigm collapse: CLDR calls these the same language, RFC 4647 picks the specific one + (["en"], ["en", "en-US"], ["en"]), + (["en"], ["en-US", "en-GB"], ["en-US"]), + (["pt"], ["pt-BR", "pt-PT"], ["pt-BR"]), + (["es"], ["es-ES", "es-419"], ["es-ES"]), + # non-paradigm regionals: exact mode drops them, so the fuzzy fallback catches + # them and the flag still matches what a default 'orig' download selects + (["es"], ["es-419"], ["es-419"]), + (["fr"], ["fr-CA"], ["fr-CA"]), + # alias + (["zh"], ["cmn"], ["cmn"]), + # nothing in the original language, and no language on the title at all + (["en"], ["ja", "de"], []), + ([None], ["en"], []), + ], +) +def test_original_audio_ids(title_language: list, track_languages: list[str], expected: list[str]) -> None: + tracks = [FakeTrack(t) for t in track_languages] + ids = original_audio_ids(tracks, FakeTitle(title_language[0])) + assert [t for t in track_languages if t in ids] == expected diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 68ba3c13..83ed073d 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -69,6 +69,7 @@ from unshackle.core.tracks.dv_fixup import apply_dv_fixup from unshackle.core.tracks.hybrid import Hybrid from unshackle.core.tracks.track import assert_fragments_decrypted, has_encrypted_sample_entry from unshackle.core.utilities import ( + as_requested, find_font_with_fallbacks, find_missing_langs, get_debug_logger, @@ -1725,22 +1726,30 @@ class dl: audio_languages = a_lang or lang if audio_languages: processed_lang = [] + s_orig_token: Optional[str] = None for language in audio_languages: if language == "orig": if song.language: orig_lang = str(song.language) + s_orig_token = orig_lang if orig_lang not in processed_lang: processed_lang.append(orig_lang) else: self.log.warning("Original language not available for music track, skipping 'orig'") elif language not in processed_lang: processed_lang.append(language) + if s_orig_token in audio_languages: + s_orig_token = None song.tracks.audio = select_best_audio( song.tracks.audio, processed_lang, acodec, audio_description, exact_lang ) if not song.tracks.audio: - self.log.error(f"There's no {processed_lang} Audio Track for {song.name}...") + # empty only when 'orig' was the sole request and did not resolve + self.log.error( + f"There's no {as_requested(processed_lang, s_orig_token) or 'orig'} " + f"Audio Track for {song.name}..." + ) sys.exit(1) self.log.debug("Getting Tracks") @@ -2425,10 +2434,12 @@ class dl: ) if keep_videos and video_languages and "all" not in video_languages: processed_video_lang = [] + v_orig_token: Optional[str] = None for language in video_languages: if language == "orig": if title.language: orig_lang = str(title.language) + v_orig_token = orig_lang if orig_lang not in processed_video_lang: processed_video_lang.append(orig_lang) else: @@ -2438,19 +2449,21 @@ class dl: else: if language not in processed_video_lang: processed_video_lang.append(language) + if v_orig_token in video_languages: + v_orig_token = None missing_v_langs = find_missing_langs( processed_video_lang, [v.language for v in title.tracks.videos], exact=exact_lang, ) if missing_v_langs: - missing_str = ", ".join(missing_v_langs) + missing_str = as_requested(missing_v_langs, v_orig_token) if best_available: remaining_v_langs = [tok for tok in processed_video_lang if tok not in missing_v_langs] if remaining_v_langs: self.log.warning( f"{missing_str} not found in video tracks, " - f"continuing with: {', '.join(remaining_v_langs)}" + f"continuing with: {as_requested(remaining_v_langs, v_orig_token)}" ) processed_video_lang = remaining_v_langs else: @@ -2463,7 +2476,10 @@ class dl: title.tracks.videos, processed_video_lang, exact_match=exact_lang ) if not title.tracks.videos: - self.log.error(f"There's no {processed_video_lang} Video Track...") + # empty only when 'orig' was the sole request and did not resolve + self.log.error( + f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..." + ) sys.exit(1) has_hybrid = any(r == Video.Range.HYBRID for r in range_) @@ -2777,10 +2793,12 @@ class dl: audio_languages = a_lang or lang if audio_languages: processed_lang = [] + a_orig_token: Optional[str] = None for language in audio_languages: if language == "orig": if title.language: orig_lang = str(title.language) + a_orig_token = orig_lang if orig_lang not in processed_lang: processed_lang.append(orig_lang) else: @@ -2790,6 +2808,8 @@ class dl: else: if language not in processed_lang: processed_lang.append(language) + if a_orig_token in audio_languages: + a_orig_token = None if not any(tok in processed_lang for tok in ("best", "all")): missing_a_langs = find_missing_langs( @@ -2798,12 +2818,13 @@ class dl: exact=exact_lang, ) if missing_a_langs: - missing_str = ", ".join(missing_a_langs) + missing_str = as_requested(missing_a_langs, a_orig_token) if best_available: remaining = [tok for tok in processed_lang if tok not in missing_a_langs] if remaining: self.log.warning( - f"{missing_str} not found in audio tracks, continuing with: {', '.join(remaining)}" + f"{missing_str} not found in audio tracks, " + f"continuing with: {as_requested(remaining, a_orig_token)}" ) processed_lang = remaining else: @@ -2819,7 +2840,11 @@ class dl: title.tracks.audio, processed_lang, acodec, audio_description, exact_lang ) if not title.tracks.audio: - self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") + # empty only when 'orig' was the sole request and did not resolve + self.log.error( + f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} " + f"Audio Track, cannot continue..." + ) sys.exit(1) # Reconstruct track set to only include kept tracks diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index a7268cea..39086523 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -17,7 +17,7 @@ from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_C from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy from unshackle.core.services import Services from unshackle.core.titles import Episode, Movie, Title_T -from unshackle.core.tracks import Audio, Subtitle, Video +from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.utils.collections import ci_get from unshackle.core.utils.redact import REDACTED, URL_USERINFO_RE @@ -491,8 +491,31 @@ def serialize_video_track(track: Video, include_url: bool = False) -> Dict[str, return result -def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, Any]: - """Convert audio track to JSON-serializable dict.""" +def original_audio_ids(tracks: List[Audio], title: Title_T) -> set: + """Return the ids of the audio tracks 'orig' resolves to, empty when the title has no language. + + This defers to Tracks.by_language so the flag agrees with the downloader. It asks + exact mode first because CLDR rates a base tag and its paradigm regional variant as + the same language, and only the RFC 4647 preference picks one ('en' over 'en-US' + when both exist, 'pt-BR' over 'pt-PT' for a 'pt' title). The fuzzy fallback then + catches the non-paradigm regionals exact mode drops, such as an 'es' title that + carries only 'es-419'. + """ + language = getattr(title, "language", None) + if not language: + return set() + matches = Tracks.by_language(tracks, [str(language)], exact_match=True) or Tracks.by_language( + tracks, [str(language)] + ) + return {t.id for t in matches} + + +def serialize_audio_track(track: Audio, include_url: bool = False, is_original: bool = False) -> Dict[str, Any]: + """Convert audio track to JSON-serializable dict. + + Resolve is_original with original_audio_ids so the flag always agrees with the + track 'orig' would actually download. + """ codec_name = enum_name(track.codec) result = { @@ -502,6 +525,7 @@ def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, "bitrate": int(track.bitrate / 1000) if track.bitrate else None, "channels": track.channels if track.channels else None, "language": str(track.language) if track.language else None, + "is_original": is_original, "atmos": track.atmos if hasattr(track, "atmos") else False, "descriptive": track.descriptive if hasattr(track, "descriptive") else False, "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, @@ -746,10 +770,13 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + original_ids = original_audio_ids(audio_tracks, title) episode_data = { "title": serialize_title(title), "video": [serialize_video_track(t) for t in video_tracks], - "audio": [serialize_audio_track(t) for t in audio_tracks], + "audio": [ + serialize_audio_track(t, is_original=t.id in original_ids) for t in audio_tracks + ], "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], } episodes_data.append(episode_data) @@ -793,10 +820,11 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + original_ids = original_audio_ids(audio_tracks, first_title) response = { "title": serialize_title(first_title), "video": [serialize_video_track(t) for t in video_tracks], - "audio": [serialize_audio_track(t) for t in audio_tracks], + "audio": [serialize_audio_track(t, is_original=t.id in original_ids) for t in audio_tracks], "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], } @@ -1916,11 +1944,15 @@ async def session_tracks_handler( else: server_cdm_type = "playready" + original_ids = original_audio_ids(audio_tracks, title) + return web.json_response( { "title": serialize_title(title), "video": [serialize_video_track(t, include_url=True) for t in video_tracks], - "audio": [serialize_audio_track(t, include_url=True) for t in audio_tracks], + "audio": [ + serialize_audio_track(t, include_url=True, is_original=t.id in original_ids) for t in audio_tracks + ], "subtitles": [serialize_subtitle_track(t, include_url=True) for t in tracks.subtitles], "chapters": [ {"timestamp": ch.timestamp, "name": ch.name} diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 50a64755..08f7ed2e 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -175,6 +175,11 @@ def find_missing_langs( return [tok for tok in requested if tok not in skip and not match_func(tok, available)] +def as_requested(tokens: Sequence[str], orig_token: Optional[str]) -> str: + """Return language tokens as the user wrote them, so 'orig' reads as 'orig' and not the language it resolved to.""" + return ", ".join("orig" if tok == orig_token else tok for tok in tokens) + + def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore """ Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find. From 40ae213504932593cc7a105944de9aba93a5d60e Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sun, 26 Jul 2026 21:21:01 -0600 Subject: [PATCH 14/40] docs: cleanup and correct documentation --- README.md | 6 +- docs/getting-started/configuration-file.md | 18 ++--- docs/getting-started/installation.md | 8 +-- docs/getting-started/quickstart.md | 82 +++++++--------------- docs/guide/downloading.md | 40 ++++++----- docs/index.md | 18 ++--- docs/reference/configuration.md | 56 +++++++-------- 7 files changed, 102 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index b043aa9a..952d3a78 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,18 @@ unshackle --help ### Requirements -External tools on your `PATH` (recommended versions): +External tools. (recommended versions): - [Python](https://www.python.org/) - 3.11 - 3.14 - [uv](https://docs.astral.sh/uv/) - ≥ 0.5 - [FFmpeg](https://ffmpeg.org/) - ≥ 6.0 - [MKVToolNix](https://mkvtoolnix.download/) - ≥ 80 - [shaka-packager](https://github.com/shaka-project/shaka-packager/releases/tag/v2.6.1) - 2.6.1 -- [Bento4](https://github.com/axiomatic-systems/Bento4) - ≥ 1.6.0-639 -- [dovi_tool](https://github.com/quietvoid/dovi_tool) - ≥ 2.1 Optional: +- [Bento4](https://github.com/axiomatic-systems/Bento4) - ≥ 1.6.0-639 +- [dovi_tool](https://github.com/quietvoid/dovi_tool) - ≥ 2.1 - [SubtitleEdit](https://github.com/SubtitleEdit/subtitleedit/releases) - ≥ 5.0 (`SeConv` CLI) ## License diff --git a/docs/getting-started/configuration-file.md b/docs/getting-started/configuration-file.md index 32d99792..4111bb8e 100644 --- a/docs/getting-started/configuration-file.md +++ b/docs/getting-started/configuration-file.md @@ -105,7 +105,7 @@ turned into underscores (`--best-available` → `best_available`). dl: lang: [en] # -l en quality: [1080] # -q 1080 - vcodec: [H265] # -v H265 + vcodec: [H.265] # -v H.265 sub_format: srt # convert subtitles to SRT downloads: 2 # two tracks at once ``` @@ -159,7 +159,7 @@ The directories unshackle uses: | `fonts` | Bundled fonts | Yes | !!! note "Some directories cannot be moved" - A handful of internal locations are protected: the package root, the core directory, the user-config directory, and the data base. If you list them under `directories`, unshackle silently ignores the override. This is intentional; those paths are tied to where the package is installed. + unshackle protects five internal entries: `app_dirs`, `core_dir`, `namespace_dir`, `user_configs`, and `data`. If you list any of them under `directories`, unshackle ignores the override. This is intentional; those paths are tied to where the package is installed. ### The `services` directory is special @@ -168,12 +168,12 @@ Unlike the other entries, `services` is a **list**, and each entry can be either ```yaml title="unshackle.yaml" directories: services: - - unshackle-dl/services # a GitHub owner/repo shorthand - - https://github.com/me/my-services.git + - you/your-services # a GitHub owner/repo shorthand + - https://example.com/private-services.git - ~/code/local-services # a local folder ``` -Entries are searched in the order listed, and **the first source to define a given service tag wins**, so put local folders last if you want them to act as fallbacks rather than overrides. Remote repositories are cloned and periodically updated for you. See [Services](../dev/creating-a-service.md) for the full details on how service discovery and repositories work. +unshackle searches entries in the order listed, and **the first source to define a given service tag wins**, so put local folders last if you want them to act as fallbacks rather than overrides. unshackle clones remote repositories on first use and refreshes them at most once a day. See [Creating a Service](../dev/creating-a-service.md) for how service discovery and repositories work. ## The `filenames` key @@ -197,10 +197,10 @@ $ unshackle cfg tag MYGRP $ unshackle cfg cdm.default my_device_l3 ``` -Values are parsed as Python literals where possible, so numbers, booleans, and lists work naturally, while bare text is treated as a string: +unshackle parses the value as a Python literal, so write booleans as `True`/`False` and quote the strings inside a list (`"['en']"`). Anything that is not valid Python literal syntax is stored as a plain string: ```console -$ unshackle cfg update_checks false +$ unshackle cfg update_checks False $ unshackle cfg vault_timeout 30 ``` @@ -216,7 +216,7 @@ $ unshackle cfg cdm.default --unset $ unshackle cfg --list ``` -When it writes, `unshackle cfg` targets the config file that was loaded. If none exists yet, it creates `unshackle.yaml` in your OS user-config directory. +When it writes, `unshackle cfg` targets the config file that was loaded. If none exists yet, it creates `unshackle.yaml` inside the `unshackle` package folder (search location 1), not your OS user-config directory. To keep the config outside the package, create the file at the user-config path yourself first, then `unshackle cfg` writes to it. !!! warning "Editing with `cfg` strips comments" Because `unshackle cfg` rewrites the whole file when it saves, any comments in `unshackle.yaml` are removed by a write. If you keep important notes as comments, edit the file by hand instead, or keep those notes elsewhere. @@ -225,4 +225,4 @@ When it writes, `unshackle cfg` targets the config file that was loaded. If none - Browse the [Configuration Reference](../reference/configuration.md) for every key and default. - Set up your first download in [Downloading](../guide/downloading.md). -- Learn how services are discovered and updated in [Services](../dev/creating-a-service.md). +- Learn how services are discovered and updated in [Creating a Service](../dev/creating-a-service.md). diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 1ddeaf5a..af9d8d52 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -88,8 +88,8 @@ uv run unshackle env check ## External tools on your PATH unshackle shells out to a number of external command-line programs for media -processing and DRM work. It looks for each one first in a `binaries/` folder -next to the installed package, and then falls back to your system `PATH`, so +processing and DRM work. It looks for each one first in the `binaries/` folder +inside the installed package, and then falls back to your system `PATH`, so installing these tools system-wide (or dropping them into that folder) both work. @@ -108,7 +108,7 @@ These must be present for core downloading, decryption, and muxing to work: !!! warning "shaka-packager is the default decryptor" unshackle decrypts with shaka-packager unless you explicitly configure a different decryptor. If it is missing, protected downloads will fail at the - decryption step. See the [configuration reference](configuration-file.md) for the + decryption step. See the [configuration reference](../reference/configuration.md#decryption) for the `decryption` key if you want to switch to `mp4decrypt`. ### Recommended and optional tools @@ -147,7 +147,7 @@ it depends on. unshackle --help ``` -This prints the banner and the list of available commands (`dl`, `env`, `kv`, +This prints the list of available commands (`dl`, `env`, `kv`, `serve`, `wvd`, `prd`, `search`, `util`, `cfg`, `import`). If your shell reports that `unshackle` is not found, the `uv tool` bin directory is probably not on your `PATH`. Run `uv tool update-shell` and open a new terminal. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 43881be6..f4b75cf9 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -1,15 +1,21 @@ # Quick Start This page takes you from a fresh install to your first downloaded file. You will -create the one piece of configuration unshackle insists on, add a streaming -service and a CDM so it can actually fetch and decrypt media, run a basic -`unshackle dl` command, and learn where the finished file lands. +create the one piece of configuration unshackle insists on, supply the service +module and CDM it needs to fetch and decrypt media, run a basic `unshackle dl` +command, and learn where the finished file lands. !!! note "Before you start" This guide assumes unshackle is already installed and on your `PATH`. If `unshackle --help` does not print a help screen, work through [Installation](installation.md) first. +!!! warning "unshackle includes no services" + unshackle handles manifests, DRM, downloading, and muxing. It does not include a + module for any streaming platform, so you write that part yourself + ([step 3](#3-write-a-service)). `EXAMPLE` in this guide stands in for the tag you + give yours. + ## 1. Check your environment unshackle drives several external tools (FFmpeg, MKVToolNix, shaka-packager, and @@ -58,12 +64,12 @@ Everything else has a sensible default. A minimal, working config looks like thi ```yaml title="unshackle.yaml" output_template: movies: "{title} ({year}) {quality} {source}" - series: "{title} S{season}E{episode} {episode_name} {quality} {source}" + series: "{title} {season_episode} {episode_name?} {quality} {source}" ``` Each `{variable}` is filled in from the title and the tracks you downloaded. The full set of valid variables, including `resolution`, `video`, `audio`, `hdr`, -`edition`, `tag`, and more, is documented in [Configuration](configuration-file.md). +`edition`, `tag`, and more, is documented in [Output and Naming](../guide/output-and-naming.md). !!! note "Spaces or dots?" unshackle auto-detects your naming style from the template: if the separators @@ -77,51 +83,15 @@ full set of valid variables, including `resolution`, `video`, `audio`, `hdr`, rewrites the file and **removes any comments** it contained. If you keep notes in your config, edit the file by hand instead. -## 3. Add a service +## 3. Write a service -The service tag you pass to `dl` (like `EXAMPLE`) maps to a **service module**, a small -plugin that knows how to talk to one streaming platform. unshackle ships without -any bundled services, so you add the one(s) you want. +The service tag you pass to `dl` (like `EXAMPLE`) maps to a **service module**, the +plugin that talks to one streaming platform. unshackle includes none, so you write +the one you need. [Creating a Service](../dev/creating-a-service.md) covers how. -Services live under the `directories.services` path (run `unshackle env info` to -see where that is). unshackle discovers them by looking for `*/__init__.py`: the -folder name becomes the **service tag**, and the folder must contain a class with -that exact same name. - -=== "Local folder" - - Drop a service folder into your services directory. A service tagged `EXAMPLE` - lives at: - - ``` - /EXAMPLE/__init__.py # defines a class named EXAMPLE - ``` - - Once the folder is in place, the tag `EXAMPLE` is immediately available to `dl`, - `search`, and the other service-aware commands. - -=== "Git repository" - - `directories.services` may also list git repositories. unshackle clones them - on first use and keeps them up to date: - - ```yaml title="unshackle.yaml" - directories: - services: - - owner/unshackle-services # a GitHub owner/repo shorthand - - /path/to/local/services # local folders still work too - ``` - - Force a refresh of every configured service repo at any time with: - - ```shell - unshackle util refresh-services - ``` - -!!! info "Where to get services" - Service modules are kept separate from the core project. Ask in the community - [Discord](https://discord.gg/mHYyPaCbFK) for where to find the service you - need. The core repository deliberately does not host them. +Put the finished module in your `directories.services` folder (`unshackle env info` +shows the path). The tag then works with `dl`, `search`, and the other service +commands. ## 4. Add a CDM for DRM @@ -144,7 +114,7 @@ cdm: ``` PlayReady works the same way with `.prd` files created and managed by the -[`prd`](configuration-file.md) command. If a title is DRM-free, no CDM is needed. +[`prd`](../guide/cli-reference.md#prd) command. If a title is DRM-free, no CDM is needed. !!! tip Run `unshackle wvd parse my_device` to inspect a device's security level and @@ -199,7 +169,7 @@ unshackle dl - **`<SERVICE-TAG>`** picks which service to talk to (e.g. `EXAMPLE`). - **`<TITLE>`** is the URL, ID, or slug the service understands. -A good first command asks for 1080p with English audio and subtitles: +A good first command asks for 1080p with English audio: ```shell title="Your first download" unshackle dl -q 1080 -l en EXAMPLE 81234567 @@ -213,7 +183,7 @@ keys through your CDM (and any key vaults), then decrypt, mux, and tag the resul | Flag | Meaning | |---|---| | `-q`, `--quality` | Target resolution(s), e.g. `-q 1080` or `-q 1080,720`. Defaults to best available. | -| `-l`, `--lang` | Language(s) for video and audio, e.g. `-l en` or `-l orig,en`. `orig` = the title's original language. | +| `-l`, `--lang` | Language(s) for video and audio, e.g. `-l en` or `-l orig,en`. `orig` = the title's original language. Defaults to `orig`. | | `-sl`, `--s-lang` | Subtitle language(s); defaults to `all`. | | `-v`, `--vcodec` | Video codec, e.g. `-v H.265`. Defaults to any. | | `-r`, `--range` | Dynamic range, e.g. `-r HDR10` or `-r DV`. Defaults to `SDR`. | @@ -238,18 +208,18 @@ codec, bitrate, channel-layout, and track-type selection. ## 8. Where the output lands -By default, finished files are written to the `downloads` directory +By default, unshackle writes finished files to the `downloads` directory (`unshackle env info` shows its exact path; the built-in default is a `downloads` -folder alongside the project). Override it per run with `-o`: +folder one level above the installed `unshackle` package). Override it per run with `-o`: ```shell unshackle dl -q 1080 -o /mnt/media/incoming EXAMPLE 81234567 ``` -- **Movies** are written as a single `.mkv` file named from your `movies` template. +- unshackle writes **movies** as a single `.mkv` file named from your `movies` template. - **TV episodes** are grouped into a per-show / per-season folder (from your `series` template) unless you pass `--no-folder`. -- The filename is built from your `output_template`, and IMDb/TMDB IDs are added to +- unshackle builds the filename from your `output_template` and adds IMDb/TMDB IDs to the file's metadata tags when available. To change the default output location permanently, set it in your config: @@ -270,5 +240,5 @@ directories: !!! tip "You are never far from help" `unshackle --help`, `unshackle dl --help`, and `unshackle <command> --help` - list every option with its default. When in doubt, add `--list` to a `dl` + list every option. When in doubt, add `--list` to a `dl` command to preview what it would do before it downloads anything. diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 07b84768..98cc3f95 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -4,7 +4,7 @@ streaming service, selects exactly the tracks you asked for, acquires and applies decryption keys, and muxes everything into a finished Matroska file. The flags below cover quality and codec selection, language and subtitle handling, episode ranges, and -the download flow itself, with examples throughout. +the download flow itself. !!! note "Where the full flag list lives" This page covers the flags you will use most and explains how they interact. It is @@ -64,8 +64,9 @@ helps explain why, for example, `--list` stops early and why keys are fetched ev 9. **Post-process**: extract closed captions, convert subtitles, repack, mux with `mkvmerge`, and move the finished file to your downloads directory. -`--list` and `--list-titles` stop after selection/filtering and print what *would* be -downloaded. `--skip-dl` runs the license step but skips the actual segment download. +`--list` prints the tracks a title exposes and stops before selection. `--list-titles` +prints every title the service returned and stops before `--wanted`/`--latest-episode` +filtering. `--skip-dl` runs the license step but skips the actual segment download. ## Choosing quality @@ -141,9 +142,11 @@ unshackle dl -q 2160 -r HDR10,DV EXAMPLE '...' !!! warning "HYBRID requires dovi_tool" `-r HYBRID` produces a single hybrid stream by injecting the Dolby Vision RPU - metadata onto an HDR10/HDR10+ base layer using **dovi_tool**. It requires the - `dovi_tool` binary on your PATH and needs both an HDR10(+) *and* a DV track to be - available, otherwise the title fails (unless `--best-available` lets it fall back). + metadata onto an HDR10/HDR10+ base layer using **dovi_tool**. + It requires the `dovi_tool` binary, resolved from unshackle's `binaries/` folder or + your `PATH`. The normal case is a DV track plus an HDR10 or HDR10+ base. With HDR10+ + and no DV, unshackle converts the HDR10+ metadata to DV instead, which also needs + `hdr10plus_tool`. With no DV and no HDR10+, the title fails. When HDR10+ is present it is preferred over HDR10 as the base layer. To keep an HDR10+ deliverable *alongside* the hybrid, request both ranges: @@ -171,8 +174,8 @@ Manifest-declared bitrates are sometimes rounded or wrong. Services often advert **peak** or **nominal** bandwidth that is far from the stream's real average. This matters because a track's bitrate drives the track listing, the sort order, *and* the `--vbitrate`/`--vbitrate-range` selection above. A bogus declared value therefore makes -unshackle pick the wrong track. These flags remove that risk by probing the actual media -size to compute a true bitrate for the top renditions, overriding the manifest value: +unshackle pick the wrong track. These flags probe the actual media size to compute a true +bitrate for the top renditions, overriding the manifest value: - `-rvb` / `--real-video-bitrate`: probe real video bitrates (per codec/range). - `-rab` / `--real-audio-bitrate`: probe real audio bitrates (per codec/channels/ @@ -186,12 +189,13 @@ size to compute a true bitrate for the top renditions, overriding the manifest v That is the real *delivered* size, not an over-report or a bug. !!! tip "Why not every rendition is probed" - Probing does not touch every rendition. Only the top rendition of each quality tier is - probed in parallel (video grouped by codec+range, audio by codec+channels+language), - and the pass then reaches downward only as far as it must to keep the ranking correct. - This keeps probing fast even when a service exposes dozens of renditions. Tracks whose - duration can't be determined fall back to `ffprobe`, and a probe failure is non-fatal: - the manifest value is simply left in place. + Probing does not touch every rendition. Only the five highest declared-bitrate + renditions of each quality tier are probed in parallel (video grouped by codec and + range, audio by codec, channels, language, and descriptive flag), and unshackle + extends a group downward while a lower unprobed rendition could still outrank a + probed one. This keeps probing fast even when a service exposes dozens of renditions. + Tracks whose duration can't be determined fall back to `ffprobe`, and a probe failure + is non-fatal: the manifest value is simply left in place. ## Audio codec, channels, and Atmos @@ -347,7 +351,7 @@ with `-` to exclude it. | --- | --- | | `--select-titles` | Interactively pick episodes from a list (Series only). **Cannot combine with `-w`.** | | `--latest-episode` | Download only the single most recent episode. | -| `--list-titles` | List the titles that would be downloaded, then stop. | +| `--list-titles` | List every title the service returned, then stop. `-w`/`--latest-episode` are not applied to this listing. | ```shell title="Grab just the newest episode" unshackle dl --latest-episode EXAMPLE 81234567 @@ -398,8 +402,8 @@ Before committing to a long download, inspect what unshackle *would* do: | Flag | Effect | | --- | --- | -| `--list` | List the available tracks and which would be selected, then stop. No download. | -| `--list-titles` | List the titles (episodes) that would be downloaded, then stop. | +| `--list` | List the tracks the service exposes for each title, then stop. No selection, no download. | +| `--list-titles` | List every title the service returned, then stop. `-w`/`--latest-episode` are not applied to this listing. | | `--skip-dl` | Skip downloading but still acquire the decryption keys. | ```shell title="See the track selection without downloading" @@ -428,7 +432,7 @@ flags change how the output is assembled: | --- | --- | --- | | `--no-mux` | Do not mux; keep the individual track files. | - | | `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) | -| `--merge-video` | Mux all selected video tracks into one file. | config `muxing.merge_video` (off) | +| `--merge-video` | Mux video tracks that share a height, range, and codec into one file, so only language varies inside a file. | config `muxing.merge_video` (off) | ### Naming tags diff --git a/docs/index.md b/docs/index.md index b71f6f46..5eb5fcb3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,18 +5,20 @@ media from streaming manifests, handles the DRM, and muxes everything into clean well-tagged files, all driven from a single command-line tool, or served over a REST API for automation. -It is a fork of [Devine](https://github.com/devine-dl/devine) with first-class -DASH, HLS, and Smooth Streaming (ISM) parsing, both **Widevine** and **PlayReady** -DRM support, and a built-in HTTP server. +It is a fork of [Devine](https://github.com/devine-dl/devine) with DASH, HLS, and +Smooth Streaming (ISM) parsing, both **Widevine** and **PlayReady** DRM support, +and a built-in HTTP server. !!! warning "Use responsibly" - unshackle is licensed under [GPL-3.0](https://github.com/unshackle-dl/unshackle/blob/master/LICENSE). + unshackle is licensed under [GPL-3.0](https://github.com/unshackle-dl/unshackle/blob/main/LICENSE). Do not use it for content you do not have the rights to. Keep the core free and open, keep service code private, and be kind. ## What it does -Point unshackle at a title on a streaming service and it will: +unshackle does everything except talk to the streaming platform. You write that part +as a service module ([Creating a Service](dev/creating-a-service.md)). Point unshackle +at a title and it will: - Fetch the title metadata (movies, full series, seasons, episodes, albums, tracks). - Parse the streaming manifest: **DASH**, **HLS**, or **Smooth Streaming (ISM)**. @@ -40,7 +42,7 @@ Point unshackle at a title on a streaming service and it will: | Subtitles | SRT, WebVTT, ASS/SSA, TTML and more, with optional SDH stripping and format conversion | | Key vaults | SQLite, MySQL, and HTTP/API vaults to store and share content keys | | Proxies | Basic proxies plus NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers | -| Services | Pluggable service modules, loadable from local folders or git repositories | +| Services | A plugin API for the service modules you write. unshackle includes none | | Automation | A REST API with a job queue, live progress, history, and remote-download sessions | ## Installation at a glance @@ -70,7 +72,7 @@ Point unshackle at a title on a streaming service and it will: Every download is three parts: the `dl` command, a **service tag**, and the service's title argument (a URL, ID, or slug). -```shell title="Download a 1080p title with English audio and subtitles" +```shell title="Download a 1080p title with English audio" unshackle dl -q 1080 -l en EXAMPLE 81234567 ``` @@ -84,7 +86,7 @@ install to your first file. - **[Installation](getting-started/installation.md)**. Install unshackle and the external tools it depends on. - **[Quickstart](getting-started/quickstart.md)**. Configure `unshackle.yaml` and run your first download. - **[Downloading](guide/downloading.md)**. The full `dl` command guide: quality, codecs, languages, track selection, and output. -- **[Configuration](getting-started/configuration-file.md)**. Every `unshackle.yaml` key, directories, vaults, proxies, and naming templates. +- **[Configuration](getting-started/configuration-file.md)**. Where `unshackle.yaml` lives, the directory layout, and how to edit values. Full key list in the [Configuration Reference](reference/configuration.md). - **[REST API](dev/rest-api/index.md)**. Run the `serve` HTTP server and drive downloads programmatically. !!! tip "Getting help" diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 681df844..22ab66da 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1,6 +1,6 @@ # Configuration Reference -This page is the exhaustive, authoritative reference for **every** key you can set in +This page documents every key you can set in `unshackle.yaml`. For each key you will find its name, expected type, default value, and what it does. Keys are grouped by area so you can jump straight to the part of unshackle you are configuring. @@ -10,10 +10,12 @@ edit it), start with [The Configuration File](../getting-started/configuration-f This page assumes you already know the basics and want precise details. !!! info "How defaults work" - Every key is optional. If you omit a key, unshackle uses the built-in default listed - here. There is **no schema validation**: unknown keys are silently ignored (they never - become settings), and the file is parsed with `yaml.safe_load`. An empty or missing - file means *all* defaults apply. + Every key is optional except [`output_template`](#output-naming), which `dl` requires. + Without it, `dl` aborts with an error. If you omit any other key, unshackle uses + the built-in default listed here. There is **no schema validation**: unknown keys are + silently ignored (they never become settings), and the file is parsed with + `yaml.safe_load`. An empty or missing file means every other key falls back to its + default. !!! warning "Two keys behave specially on load" - `curl_impersonate` is a **deprecated** alias of [`network`](#network-proxy) and emits a @@ -32,7 +34,7 @@ This page assumes you already know the basics and want precise details. |------|------| | [Directories](#directories) | `directories`, `filenames` | | [Services & authentication](#services-auth) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` | -| [Download & processing](#download-processing) | `dl`, `subtitle`, `audio`, `muxing`, `language_tags` | +| [Download & processing](#download-processing) | `dl`, `subtitle`, `audio`, `muxing`, `language_tags`, `dual_multi_mode` | | [Output & naming](#output-naming) | `output_template`, `tag`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` | | [DRM & CDM](#drm-cdm) | `cdm`, `remote_cdm`, `decryption` | | [Network & proxy](#network-proxy) | `network`, `headers`, `proxy_providers` | @@ -91,7 +93,7 @@ directories: ```yaml directories: services: - - unshackle-dl/community-services # cloned from GitHub + - you/your-services # cloned from GitHub - https://example.com/private.git@stable - ~/my-services # local, lowest priority ``` @@ -238,7 +240,7 @@ is the [REST API](../dev/rest-api/index.md) section; these are the config keys. | `api_secret` | str | *(unset)* | Master secret accepted in the `X-Secret-Key` header. | | `users` | dict | `{}` | Per-user API keys and their allowlists (see below). | | `services` | list | *(unset)* | Global service allowlist. Omit to allow all. | -| `remote_only` | bool | `false` | Run without a local `output_template` (CORS/Cloudflare-friendly). | +| `remote_only` | bool | `false` | Expose only the remote service session endpoints (health, services, search, session) and disable the rest of the REST API. | | `session_ttl` | int (s) | `300` | Lifetime of an interactive auth session. | | `max_sessions` | int | `100` | Maximum concurrent sessions. | | `history_limit` | int | `100` | How many finished jobs to retain in history. | @@ -247,7 +249,6 @@ is the [REST API](../dev/rest-api/index.md) section; these are the config keys. | `allow_job_credentials` | bool | `false` | Permit clients to supply credentials per job. | | `devices` | list | *(auto)* | Widevine devices offered; auto-filled from `directories.wvds`. | | `playready_devices` | list | *(auto)* | PlayReady devices; auto-filled from `directories.prds`. | -| `country` / `locations` | str / dict | *(unset)* | Advertised region metadata. | Each entry under `users` is keyed by that user's API key and may set its own `services`, `devices`, and `playready_devices` allowlists, narrowing the global ones. @@ -266,9 +267,10 @@ serve: ``` !!! tip "Declaring `dl` defaults under `serve`" - You can set any [`dl`](#dl) flag key (`downloads`, `workers`, `best_available`, and so on) - directly inside `serve`. The point is to raise concurrency and behaviour defaults - **server-wide** (every request the server handles picks them up) without having to change + You can set most [`dl`](#dl) flag keys (`downloads`, `workers`, `best_available`, and so on) + directly inside `serve`; the server recognises a fixed subset of download parameters, so a + few CLI-only flags are ignored here. This raises concurrency and behaviour defaults + **server-wide** (every request the server handles picks them up) without changing each client call individually. --- @@ -299,7 +301,7 @@ after the flag's internal destination rather than its visible name; set these ex | `-o` / `--output` | `output_dir` | | `--cdm-only` / `--vaults-only` | `cdm_only` | -**Precedence** (highest first): an explicit CLI flag or environment variable → a per-service +**Precedence** (highest first): an explicit CLI flag → a per-service [`services.<TAG>.dl`](#services-auth) override → this global `dl:` block → the built-in default. In other words, config only fills in options you did not set on the command line. @@ -369,7 +371,7 @@ written as sidecar files. See [Subtitles](../guide/subtitles.md) for the full gu | `convert_before_strip` | bool | `true` | Convert to the working format before stripping SDH. | | `conversion_method` | str | `"auto"` | Subtitle conversion backend. | | `preferred_conversion_method` | str | *(unset)* | Preferred converter when several are available. | -| `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files. | +| `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files; `both` writes sidecars and still muxes. | | `sidecar_format` | str | `"srt"` | Format for sidecar files when `output_mode: sidecar`. | | `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH) that sorts the variants within each language. Types you leave out fall to the end. | @@ -406,9 +408,9 @@ audio: ``` !!! note "`codec_priority` is a soft priority: nothing is dropped" - `codec_priority` only breaks ties **among tracks that share the same bitrate and language**. - Codecs not in the list keep their normal bitrate-based ordering and are simply placed after - all listed codecs, and none are ever removed. Two higher-level rules still override it: **Atmos + `codec_priority` outranks bitrate: any listed codec sorts above every unlisted one, and among + listed codecs the list order decides. Bitrate order survives only within a single codec rank. + Nothing is ever removed. Two higher-level rules still override it: **Atmos tracks always take precedence over `codec_priority`**, and **audio-description tracks are always forced to the end** regardless of it. When `codec_priority` is unset, tracks sort by bitrate alone, with those same Atmos/descriptive rules still applied. @@ -424,7 +426,7 @@ Matroska (MKV) muxing options. | `set_title` | bool | `true` | Write a human-readable title into the MKV container. | | `default_language` | dict | `{}` | Force which language is flagged *default* per track type, e.g. `{audio: en, subtitle: en}`. | | `merge_audio` | bool | `true` | Merge all audio into one file. `--split-audio` on the CLI flips this off. | -| `merge_video` | bool | `false` | Merge all video tracks into one file. `--merge-video` on the CLI flips this on. | +| `merge_video` | bool | `false` | Merge video tracks that share height, range, and codec into one file, so only language varies inside a file. `--merge-video` on the CLI flips this on. | ```yaml muxing: @@ -451,9 +453,6 @@ A rule may test: ```yaml language_tags: rules: - - audio: [en] - subs_contain_all: [en] - tag: "" # native English release, no tag - audio: [ja] subs_contain: [en] tag: "SUBBED" @@ -498,7 +497,7 @@ For the full picture of how templates render, see ### `output_template` -- **Type:** `dict`  ·  **Default:** `{}` +- **Type:** `dict`  ·  **Required** (no usable default; `dl` exits with an error if this is missing) Filename templates keyed by media kind, using `{variable}` placeholders. Recognised kinds are `movies`, `series`, and `songs` (and `albums` for folder templates). A trailing `?` on a @@ -627,7 +626,7 @@ case-insensitively; `"mp4decrypt"` selects Bento4's mp4decrypt, and **anything e ```yaml decryption: default: shaka - SOMESERVICE: mp4decrypt + EXAMPLE1: mp4decrypt ``` --- @@ -758,8 +757,9 @@ key_vaults: !!! tip "Common per-entry options" - `no_push: true` makes a vault read-only (keys are fetched but never written to it). - - A vault of `type: api` whose `name` contains `decrypt_labs` auto-fills its `token` from - [`decrypt_labs_api_key`](#external-api-keys) when not set inline. + - A vault of `type: API` whose `name` contains `decrypt_labs` auto-fills its `token` from + [`decrypt_labs_api_key`](#external-api-keys) when not set inline. Vault `type` values are + case-sensitive module names. - An all-zero content key (32 zeros) is treated as "no key" everywhere and is never stored. @@ -786,9 +786,9 @@ All default to an empty string and enable optional metadata/geolocation features | `ipinfo_api_key` | str | `""` | ipinfo.io API key for IP/region lookups. | !!! tip "SIMKL is a fallback for TMDB" - SIMKL is positioned as an alternative metadata source, and it is most valuable precisely - **when no `tmdb_api_key` is configured**. It improves title-matching and tagging - reliability in that case rather than replacing TMDB where TMDB is available. + SIMKL is an alternative metadata source. Use it when you have no `tmdb_api_key` + configured; it improves title matching and tagging in that case rather than replacing + TMDB. !!! note "`ipinfo_api_key` never touches your service sessions" The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header; it From 3c0fef75a7092837a258584bd010c640799be3db Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Mon, 27 Jul 2026 12:15:14 -0600 Subject: [PATCH 15/40] docs(core): drop service tags from framework comments --- unshackle/core/api/handlers.py | 4 ++-- unshackle/core/import_service.py | 2 +- unshackle/core/manifests/hls.py | 2 +- unshackle/core/services.py | 2 +- unshackle/core/titles/title.py | 2 +- unshackle/core/tracks/hybrid.py | 4 ++-- unshackle/core/utilities.py | 2 +- unshackle/core/utils/bitrate.py | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 39086523..47afc898 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -1011,7 +1011,7 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): for param in service_module.cli.params: if hasattr(param, "name") and param.default is not None and not isinstance(param.default, enum.Enum): - # Store service-specific defaults (e.g., drm_system, hydrate_track, profile for NF) + # Store service-specific defaults (e.g. drm_system, hydrate_track, profile) service_specific_defaults[param.name] = param.default # Get download manager and start workers if needed @@ -2072,7 +2072,7 @@ def _resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional Checks the server's own CDM config (``config.cdm[service]``) to determine the CDM type without loading the full CDM object. This ensures that when ``server_cdm: true`` is used, the server's CDM - determines device selection (e.g. PlayReady vs Widevine for AMZN). + determines device selection (e.g. PlayReady vs Widevine). Falls back to a lightweight stub from *cdm_type* only if no server CDM is configured for the service. diff --git a/unshackle/core/import_service.py b/unshackle/core/import_service.py index 63490245..7082c9b2 100644 --- a/unshackle/core/import_service.py +++ b/unshackle/core/import_service.py @@ -150,7 +150,7 @@ class ImportService: worse copy, so any direct-URL subtitle in the export means the whole re-parsed subtitle set is dropped in favour of the exported one. HLS/URL: rebuild from the stored per-track dicts, since the variant is re-fetched from - track.url at download time and ATV-style master playlists carry unstable per-fetch tokens. + track.url at download time and a master playlist can hand out a new token on every fetch. """ title_id = str(title.id) if title_id in self.tracks_by_title: diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 4ccde219..11caa093 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -208,7 +208,7 @@ class HLS: # DV-composite track: primary codec is plain HEVC but SUPPLEMENTAL-CODECS advertises # a DV codec. Range stays whatever VIDEO-RANGE signaled (HDR10/HLG/SDR); DVFixup will # restore DV signaling post-download. Services that know their encoder embeds HDR10+ - # SEI must override `range` themselves (see services/ATV). + # SEI must override `range` themselves. dv_compatible_bitstream = primary_track_type is Video and not primary_has_dv and supp_dv_codec is not None tracks.add( diff --git a/unshackle/core/services.py b/unshackle/core/services.py index 89815493..aa3ca24d 100644 --- a/unshackle/core/services.py +++ b/unshackle/core/services.py @@ -298,7 +298,7 @@ class Services(click.Group): @staticmethod def get_tag(value: str) -> str: """ - Get the Service Tag (e.g. DSNP, not DisneyPlus/Disney+, etc.) by an Alias. + Get the Service Tag (the name of the service's directory, not one of its aliases) by an Alias. Input value can be of any case-sensitivity. Original input value is returned if it did not match a service tag. """ diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index b205cce0..10456129 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -211,7 +211,7 @@ class Title: media_info: MediaInfo object of the file this name will be used for. folder: This filename will be used as a folder name. Some changes may want to be made if this is the case. - show_service: Show the service tag (e.g., iT, NF) in the filename. + show_service: Show the service tag in the filename. """ diff --git a/unshackle/core/tracks/hybrid.py b/unshackle/core/tracks/hybrid.py index efd1e51f..1f95d403 100644 --- a/unshackle/core/tracks/hybrid.py +++ b/unshackle/core/tracks/hybrid.py @@ -387,8 +387,8 @@ class Hybrid: ) -> tuple[Optional[int], Optional[int], Optional[int], Optional[int]]: """Clamp static L6 values to a valid relationship. - MaxCLL must not exceed the mastering-display peak (some sources, e.g. ATV - HDR10+, ship MaxCLL 10000 on a 1000-nit master), and MaxFALL must not exceed + MaxCLL must not exceed the mastering-display peak (some HDR10+ sources ship + MaxCLL 10000 on a 1000-nit master), and MaxFALL must not exceed MaxCLL. A value of 0 means "unknown" and is preserved as-is. """ if max_mdl and max_cll and max_cll > max_mdl: diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 08f7ed2e..62958a55 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -824,7 +824,7 @@ class DebugLogger: operation: Name of the operation being performed message: Human-readable message context: Additional context information - service: Service name (e.g., DSNP, NF) + service: Service tag error: Exception object if an error occurred request: Request details (URL, method, headers, body) response: Response details (status, headers, body) diff --git a/unshackle/core/utils/bitrate.py b/unshackle/core/utils/bitrate.py index d6a41e44..63ff44cb 100644 --- a/unshackle/core/utils/bitrate.py +++ b/unshackle/core/utils/bitrate.py @@ -71,7 +71,7 @@ def measure_real_bitrate( elif track.descriptor == Track.Descriptor.ISM: segments = extract_ism(track, session) else: - # Descriptor.URL: a single file. Some services (e.g. AMZN) parse a DASH + # Descriptor.URL: a single file. Some services parse a DASH # manifest then collapse each representation to its single BaseURL and # flip the descriptor to URL, leaving the manifest (and its duration) in # track.data — recover the duration from there, else probe the file. @@ -375,7 +375,7 @@ def extract_url(track: "Track", session: Union[Session, RnetSession], *, log: lo duration = duration_ticks / timescale if not duration or duration <= 0: - # Services like AMZN clear the manifest data after collapsing to a single + # Some services clear the manifest data after collapsing to a single # file; fall back to reading the duration straight from the remote file. duration = ffprobe_duration(url, session, log=log) From a24c9b92daeb02c0853a80ee20bdf3157879b161 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Mon, 27 Jul 2026 13:11:13 -0600 Subject: [PATCH 16/40] fix(tracks): ignore leftover PIFF uuid when verifying decryption A decrypter detaches only the atom it consumed, so content shipping both boxes keeps the uuid through a good decrypt and failed on mp4decrypt. --- tests/tracks/test_fragment_decrypted.py | 40 ++++++++++++++++--------- unshackle/core/tracks/track.py | 15 ++++++---- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/tests/tracks/test_fragment_decrypted.py b/tests/tracks/test_fragment_decrypted.py index f92cd8ff..6396b4fa 100644 --- a/tests/tracks/test_fragment_decrypted.py +++ b/tests/tracks/test_fragment_decrypted.py @@ -1,7 +1,9 @@ """Self-check for ``assert_fragments_decrypted``: after decryption every moof must have lost -its sample-encryption box. A survivor means the tool silently skipped that fragment (the +its standard senc. A survivor means the tool silently skipped that fragment (the tfhd.sample_description_index ceiling bug), which the moov-scoped has_encrypted_sample_entry -structurally cannot see. The guard never reads mdat, so stray byte runs there leave it unmoved. +structurally cannot see. A leftover PIFF uuid survives a good decrypt too, because content +shipping both boxes keeps it: a decrypter detaches only the atom it consumed. +The guard never reads mdat, so stray byte runs there leave it unmoved. Run: uv run pytest tests/tracks/test_fragment_decrypted.py """ @@ -53,11 +55,11 @@ def test_all_fragments_decrypted_passes(tmp_path: Path) -> None: assert_fragments_decrypted(f) -def test_surviving_piff_senc_raises(tmp_path: Path) -> None: +def test_surviving_senc_raises(tmp_path: Path) -> None: # the real bug: fragment 1 (index 1) decrypted, the rest (index 2) silently skipped boxes = [FTYP, MOOV, _moof(), MDAT] for _ in range(3): - boxes += [_moof(_piff_senc(), sdi=2), MDAT] + boxes += [_moof(_senc(), sdi=2), MDAT] f = _write(tmp_path, "skipped.mp4", *boxes) with pytest.raises(ValueError, match=r"3/4 fragment"): assert_fragments_decrypted(f) @@ -69,6 +71,16 @@ def test_surviving_cenc_senc_raises(tmp_path: Path) -> None: assert_fragments_decrypted(f) +def test_leftover_piff_uuid_after_senc_removed_does_not_raise(tmp_path: Path) -> None: + # live DASH shape: the traf shipped senc+saiz+saio plus a duplicate PIFF uuid. mp4decrypt + # consumed senc, detached senc/saiz/saio, and left the uuid on a fully decrypted fragment. + boxes = [FTYP, MOOV] + for _ in range(3): + boxes += [_moof(_piff_senc()), MDAT] + f = _write(tmp_path, "piff_leftover.mp4", *boxes) + assert_fragments_decrypted(f) + + def test_stray_senc_bytes_in_mdat_ignored(tmp_path: Path) -> None: f = _write(tmp_path, "stray.mp4", FTYP, MOOV, _moof(), STRAY, _moof(sdi=2), STRAY) assert_fragments_decrypted(f) @@ -93,16 +105,16 @@ def test_empty_senc_on_clear_fragment_ignored(tmp_path: Path) -> None: assert_fragments_decrypted(f) -def _piff_override(sample_count: int) -> bytes: +def _senc_override(sample_count: int) -> bytes: # flags&1 => AlgorithmID(3) + IV_size(1) + KID(16) precede sample_count - body = PIFF_SENC_UUID + b"\x00\x00\x00\x01" + b"\x00\x00\x08" + b"\x08" + b"\xaa" * 16 - return _box(b"uuid", body + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)) + head = b"\x00\x00\x00\x01" + b"\x00\x00\x08" + b"\x08" + b"\xaa" * 16 + return _box(b"senc", head + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)) -def test_piff_override_flag_sample_count_offset(tmp_path: Path) -> None: - clear = _write(tmp_path, "ovr_clear.mp4", FTYP, MOOV, _moof(_piff_override(0)), MDAT) +def test_override_flag_shifts_sample_count_offset(tmp_path: Path) -> None: + clear = _write(tmp_path, "ovr_clear.mp4", FTYP, MOOV, _moof(_senc_override(0)), MDAT) assert_fragments_decrypted(clear) - f = _write(tmp_path, "ovr.mp4", FTYP, MOOV, _moof(_piff_override(5)), MDAT) + f = _write(tmp_path, "ovr.mp4", FTYP, MOOV, _moof(_senc_override(5)), MDAT) with pytest.raises(ValueError, match=r"1/1 fragment"): assert_fragments_decrypted(f) @@ -124,14 +136,14 @@ def test_undersized_largesize_box_does_not_desync_walk(tmp_path: Path, caplog: p # size==1 reads a 16-byte header, so a largesize of 12 clears the 8-byte floor but # still stalls the cursor onto payload; only a header-aware floor catches it runt = struct.pack(">I", 1) + b"free" + struct.pack(">Q", 12) - f = _write(tmp_path, "runt.mp4", FTYP, MOOV, runt, _moof(_piff_senc()), MDAT, _moof(_piff_senc()), MDAT) + f = _write(tmp_path, "runt.mp4", FTYP, MOOV, runt, _moof(_senc()), MDAT, _moof(_senc()), MDAT) with caplog.at_level("WARNING"): assert_fragments_decrypted(f) assert "malformed box size" in caplog.text def test_largesize_and_zero_size_boxes_walked(tmp_path: Path) -> None: - frag = _moof(_piff_senc()) + frag = _moof(_senc()) large = struct.pack(">I", 1) + b"mdat" + struct.pack(">Q", 16 + 64) + b"\x00" * 64 trailing = struct.pack(">I", 0) + b"mdat" + b"\x00" * 64 # size 0 => runs to EOF f = _write(tmp_path, "large.mp4", FTYP, MOOV, frag, large, trailing) @@ -142,7 +154,7 @@ def test_largesize_and_zero_size_boxes_walked(tmp_path: Path) -> None: def test_malformed_box_size_warns_and_does_not_raise(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: # failing open is deliberate, but the abort must be logged so it cannot pass as clean overrun = struct.pack(">I", 1 << 30) + b"free" + b"\x00" * 8 - f = _write(tmp_path, "overrun.mp4", FTYP, MOOV, overrun, _moof(_piff_senc()), MDAT) + f = _write(tmp_path, "overrun.mp4", FTYP, MOOV, overrun, _moof(_senc()), MDAT) with caplog.at_level("WARNING"): assert_fragments_decrypted(f) assert "malformed box size" in caplog.text @@ -151,5 +163,5 @@ def test_malformed_box_size_warns_and_does_not_raise(tmp_path: Path, caplog: pyt def test_missing_or_bogus_file_is_noop(tmp_path: Path) -> None: assert_fragments_decrypted(tmp_path / "nope.mp4") assert_fragments_decrypted(_write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4")) - truncated = _write(tmp_path, "cut.mp4", FTYP, MOOV, _moof(_piff_senc())[:20]) + truncated = _write(tmp_path, "cut.mp4", FTYP, MOOV, _moof(_senc())[:20]) assert_fragments_decrypted(truncated) diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index a2eca307..4f001058 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -115,17 +115,18 @@ def _senc_protects_samples(buf: bytes, body: int, end: int) -> bool: def _moof_still_encrypted(moof: bytes) -> bool: + # senc only: a decrypter detaches just the atom it consumed, so a PIFF uuid can + # outlive a good decrypt. ISM arrives as senc via piff_senc_to_cenc. # structural walk: a 4CC byte scan collides with trun payload - from unshackle.core.manifests.ism_init import PIFF_SENC_UUID, iter_boxes + from unshackle.core.manifests.ism_init import iter_boxes body = 16 if int.from_bytes(moof[:4], "big") == 1 else 8 for box_type, _, traf_body, traf_end in iter_boxes(moof, body, len(moof)): if box_type != b"traf": continue - for child, usertype, child_body, child_end in iter_boxes(moof, traf_body, traf_end): - if child == b"senc" or (child == b"uuid" and usertype == PIFF_SENC_UUID): - if _senc_protects_samples(moof, child_body, child_end): - return True + for child, _usertype, child_body, child_end in iter_boxes(moof, traf_body, traf_end): + if child == b"senc" and _senc_protects_samples(moof, child_body, child_end): + return True return False @@ -138,6 +139,10 @@ def assert_fragments_decrypted(path: Path) -> None: stderr) while leaving the payload as ciphertext. Unlike has_encrypted_sample_entry (moov-scoped) this is fragment-scoped; the moov comes out clean in that failure. + Only a standard senc counts as evidence. Some DASH content ships a PIFF uuid beside it, + and a decrypter detaches just the atom it consumed, so that uuid outlives a good + decrypt; counting it would condemn a healthy file. + The guarantee runs in one direction: a survivor proves a skip, while a clean pass is only as good as the decrypter's own behaviour. shaka-packager (the default `decryption` backend) remuxes into a single fragment and drops senc whether or not it From f7a38732f189202772dce3a6167cfa34efae4565 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Tue, 28 Jul 2026 21:22:14 -0600 Subject: [PATCH 17/40] feat(dl): extend --select-titles to movie listings --- docs/guide/cli-reference.md | 2 +- docs/guide/downloading.md | 4 ++-- unshackle/commands/dl.py | 24 +++++++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index a0571a07..c19c9424 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -106,7 +106,7 @@ The special language tokens `orig`, `all`, and `best` are honoured everywhere a | Flag | Description | |---|---| | `-w`, `--wanted` | Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`. Supports exclusions with a leading `-` (e.g. `-S03`). | -| `--select-titles` | Interactively select titles (series only). **Cannot combine with `-w`.** | +| `--select-titles` | Interactively select what to download: episodes of a series, or films when a title has more than one. **Cannot combine with `-w`.** | | `--latest-episode` | Download only the single most recent episode. | | `--list-titles` | List titles only; do not download. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 98cc3f95..1086e706 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -349,7 +349,7 @@ with `-` to exclude it. | Flag | Behaviour | | --- | --- | -| `--select-titles` | Interactively pick episodes from a list (Series only). **Cannot combine with `-w`.** | +| `--select-titles` | Interactively pick episodes of a series, or films when a title has more than one. **Cannot combine with `-w`.** | | `--latest-episode` | Download only the single most recent episode. | | `--list-titles` | List every title the service returned, then stop. `-w`/`--latest-episode` are not applied to this listing. | @@ -558,7 +558,7 @@ authoritative list. | `--forced-subs` | `-fs` | Include forced subtitles. | | `--sub-format` | | Output subtitle format. | | `--wanted` | `-w` | Episode/season range. | -| `--select-titles` | | Interactively pick episodes. | +| `--select-titles` | | Interactively pick episodes or films. | | `--latest-episode` | | Only the newest episode. | | `--video-only` / `--audio-only` / `--subs-only` | `-V` / `-A` / `-S` | Restrict track types. | | `--no-video` / `--no-audio` / `--no-subs` / `--no-chapters` | `-nv` / `-na` / `-ns` / `-nc` | Skip track types. | diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 83ed073d..3695160f 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -503,7 +503,7 @@ class dl: "--select-titles", is_flag=True, default=False, - help="Interactively select downloads from a list. Only use with Series to select Episodes.", + help="Interactively select downloads from a list: episodes of a series, or films when a title has more than one.", ) @click.option( "-w", @@ -1517,6 +1517,28 @@ class dl: if music_titles and list_: return + if select_titles and isinstance(titles, Movies) and len(titles) > 1: + console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2))) + + selection_start = time.time() + selected_idx = select_multiple( + [str(movie) for movie in titles], + minimal_count=1, + page_size=8, + return_indices=True, + ) + if not selected_idx: + console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5))) + return + start_time += time.time() - selection_start + + keep = set(selected_idx) + for i in range(len(titles) - 1, -1, -1): + if i not in keep: + del titles[i] + + console.print(Padding(f"[text]Total selected: {len(titles)}[/]", (0, 5))) + # Enables manual selection for Series when --select-titles is set if select_titles and isinstance(titles, Series): console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2))) From bece8fb74655aac83231b22b9991099d53e13e9d Mon Sep 17 00:00:00 2001 From: RocketBoy <91538975+ApeDevOne@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:23:02 +0530 Subject: [PATCH 18/40] Implement manifest data resolution for DASH/ISM tracks along with Import Fix for json exports with no "manifest_url" Added a new function to resolve import manifest data for DASH/ISM tracks, enhancing the handling of track data population from export dictionaries. --- unshackle/core/import_service.py | 90 +++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 6 deletions(-) diff --git a/unshackle/core/import_service.py b/unshackle/core/import_service.py index 7082c9b2..322ebb1a 100644 --- a/unshackle/core/import_service.py +++ b/unshackle/core/import_service.py @@ -16,7 +16,7 @@ from unshackle.core.constants import AnyTrack from unshackle.core.credential import Credential from unshackle.core.drm import drm_from_dict from unshackle.core.manifests import DASH, HLS, ISM -from unshackle.core.remote_service import RemoteService, _build_title, _resolve_proxy +from unshackle.core.remote_service import RemoteService, _build_title, _match_track, _resolve_proxy from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles from unshackle.core.tracks import Audio, Chapter, Chapters, Tracks, Video from unshackle.core.tracks.attachment import Attachment @@ -25,6 +25,7 @@ from unshackle.core.tracks.track import Track log = logging.getLogger("import") PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM} +MANIFEST_DATA_KEYS = {"DASH": "dash", "ISM": "ism"} def _fallback_language(language: Any) -> Optional[Any]: @@ -39,6 +40,63 @@ def _fallback_language(language: Any) -> Optional[Any]: return language +def _resolve_import_manifest_data( + tracks: Tracks, + manifest_type: Optional[str], + *, + session: requests.Session, + language: Any, + title: Title_T, +) -> None: + """Populate ``track.data`` for DASH/ISM tracks rebuilt from export dicts. + + Exports may omit ``manifest_url`` (e.g. when the service never set + ``title.tracks.manifest_url``) or carry one MPD per adaptation set. ``Track.from_dict`` + does not serialise manifest XML, so each exported track's ``url`` is re-fetched and + matched to a locally parsed representation before download. + """ + if manifest_type not in MANIFEST_DATA_KEYS: + return + + parser = PARSERS[manifest_type] + data_key = MANIFEST_DATA_KEYS[manifest_type] + pending = [ + track + for track in [*tracks.videos, *tracks.audio, *tracks.subtitles] + if track.descriptor.name == manifest_type and not track.data.get(data_key) + ] + if not pending: + return + + fallback_lang = _fallback_language(language) + for url in {str(track.url) for track in pending if track.url}: + try: + manifest = parser.from_url(url=url, session=session) + parsed = manifest.to_tracks(language=fallback_lang) + except ValueError as e: + if "Language information could not be derived" in str(e): + raise click.ClickException( + f"No language for '{title}': the {manifest_type} manifest has none and the " + f"service did not set Title.language." + ) + raise click.ClickException( + f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})" + ) + except Exception as e: + raise click.ClickException( + f"Failed to fetch the {manifest_type} manifest for '{title}'. " + f"The manifest URL may have expired since export. ({e})" + ) + + local_tracks = [*parsed.videos, *parsed.audio, *parsed.subtitles] + for track in pending: + if track.data.get(data_key) or str(track.url) != url: + continue + matched = _match_track(track, local_tracks) + if matched and matched.data.get(data_key): + track.data.update(matched.data) + + class ImportService: """Reconstructs a download from an export JSON. @@ -144,11 +202,12 @@ class ImportService: def get_tracks(self, title: Title_T) -> Tracks: """Reconstruct the title's tracks from the export. - DASH/ISM: re-fetch and re-parse the manifest and return the full ladder (the importer - picks quality with normal dl flags; keys are injected by KID later), then merge in the - exported URL tracks. A service side-loads its own subtitles when the manifest's are the - worse copy, so any direct-URL subtitle in the export means the whole re-parsed subtitle - set is dropped in favour of the exported one. + DASH/ISM: re-fetch and re-parse ``manifest_url`` for the full ladder on that MPD (the importer + picks quality with normal dl flags; keys are injected by KID later), then merge in exported + DASH/ISM tracks from other MPDs (e.g. a separate HEVC manifest) and direct-URL side-loads. + A service side-loads its own subtitles when the manifest's are the worse copy, so any + direct-URL subtitle in the export means the whole re-parsed subtitle set is dropped in + favour of the exported one. HLS/URL: rebuild from the stored per-track dicts, since the variant is re-fetched from track.url at download time and a master playlist can hand out a new token on every fetch. """ @@ -186,7 +245,13 @@ class ImportService: raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})") for track in parsed: tracks.add(track) + parsed_ids = {str(t.id) for t in tracks} track_dicts = [t for t in tracks_map.values() if t.get("descriptor", "URL") == "URL"] + track_dicts.extend( + t + for t in tracks_map.values() + if t.get("descriptor") == manifest_type and str(t.get("id")) not in parsed_ids + ) if any(t.get("type") == "Subtitle" for t in track_dicts): tracks.subtitles.clear() else: @@ -203,6 +268,14 @@ class ImportService: track.drm = drm tracks.add(track, warn_only=True) + _resolve_import_manifest_data( + tracks, + manifest_type, + session=self.session, + language=title.language, + title=title, + ) + for attachment in entry.get("attachments") or []: url = attachment.get("url") if not url: @@ -318,6 +391,11 @@ class ImportService: ) -> Optional[Union[bytes, str]]: raise RuntimeError("ImportService should not request a license; keys come from the export.") + def get_clearkey_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str, dict]]: + raise RuntimeError("ImportService should not request a license; keys come from the export.") + def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None: pass From bcf0815cf49e356e3bb723621047644982c7b6a8 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Wed, 29 Jul 2026 08:44:27 -0600 Subject: [PATCH 19/40] fix(tracks): size the no-proxy download session's connection pool --- unshackle/core/tracks/track.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 4f001058..3e1b34e6 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -16,6 +16,7 @@ from zlib import crc32 from langcodes import Language from requests import Session +from requests.adapters import HTTPAdapter, Retry from unshackle.core import binaries from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm @@ -30,7 +31,7 @@ from unshackle.core.utils.subprocess import ffprobe def direct_session(session: Union[Session, "RnetSession"]) -> Session: - """Vanilla requests.Session with copied headers/cookies, no proxy.""" + """requests.Session with copied headers/cookies and no proxy.""" new = Session() headers = getattr(session, "headers", None) if headers is not None: @@ -45,6 +46,15 @@ def direct_session(session: Union[Session, "RnetSession"]) -> Session: new.cookies.update(jar if jar is not None else cookies) except Exception: pass + new.mount( + "https://", + HTTPAdapter( + max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + pool_maxsize=64, + pool_block=True, + ), + ) + new.mount("http://", new.adapters["https://"]) return new From 42d7fd83036581dfdb518e5d84c51f29de88943d Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Wed, 29 Jul 2026 10:48:34 -0600 Subject: [PATCH 20/40] fix(import): scope the merge-back and detect ISM protection Only merge back tracks from another manifest, so a drifted id no longer duplicates a rung or wipes the main manifest's own subtitles. track_is_encrypted now recognises an ISM ProtectionHeader. --- tests/core/test_import_hls_ism.py | 184 +++++++++++++++++++ tests/core/test_import_manifest_data.py | 226 ++++++++++++++++++++++++ unshackle/core/import_service.py | 38 ++-- 3 files changed, 438 insertions(+), 10 deletions(-) create mode 100644 tests/core/test_import_hls_ism.py create mode 100644 tests/core/test_import_manifest_data.py diff --git a/tests/core/test_import_hls_ism.py b/tests/core/test_import_hls_ism.py new file mode 100644 index 00000000..f99bbf18 --- /dev/null +++ b/tests/core/test_import_hls_ism.py @@ -0,0 +1,184 @@ +"""Export/import round trips for the two manifest types that are not DASH. + +``get_tracks`` splits on manifest type, and the two halves need opposite things. ISM takes the +re-parse path and needs ``track.data["ism"]["segments"]`` restored, because +``ISM.download_track`` reads it. HLS takes the rebuild-from-dicts path and needs nothing +restored, because ``HLS.download_track`` re-fetches the variant from ``track.url``. That makes +the stored per-track URL load-bearing for HLS in a way it is not for ISM. + +These drive the real ``dl.write_export`` rather than hand-written export dicts, so a change to +the export format fails here instead of silently invalidating the fixtures. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from unshackle.commands.dl import dl +from unshackle.core.import_service import ImportService +from unshackle.core.manifests import HLS, ISM + +ISM_URL = "https://cdn.example/x.ism/manifest" +ISM_MANIFEST = """<?xml version="1.0"?> +<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000"> + <StreamIndex Type="video" Name="video" Language="en" + Url="QualityLevels({Bitrate})/Fragments(video={start_time})"> + <QualityLevel Index="0" Bitrate="3000000" FourCC="H264" MaxWidth="1280" MaxHeight="720" + CodecPrivateData="00000001674D401F"/> + <c t="0" d="20000000" r="3"/> + </StreamIndex> + <StreamIndex Type="audio" Name="audio" Language="en" + Url="QualityLevels({Bitrate})/Fragments(audio={start_time})"> + <QualityLevel Index="0" Bitrate="128000" AudioTag="255" SamplingRate="48000" + Channels="2" BitsPerSample="16" CodecPrivateData="1190"/> + <c t="0" d="20000000" r="3"/> + </StreamIndex> +</SmoothStreamingMedia> +""" + +HLS_URL = "https://cdn.example/master.m3u8" +HLS_MANIFEST = """#EXTM3U +#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",LANGUAGE="en",NAME="English",DEFAULT=YES,URI="audio_en.m3u8" +#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="sub",LANGUAGE="en",NAME="English",URI="sub_en.m3u8" +#EXT-X-STREAM-INF:BANDWIDTH=3000000,RESOLUTION=1280x720,CODECS="avc1.4d401f,mp4a.40.2",AUDIO="aud",SUBTITLES="sub" +720.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=6000000,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud",SUBTITLES="sub" +1080.m3u8 +""" + + +def export_tracks(tmp_path: Path, tracks, manifest_url: str | None) -> Path: + """Write a real export sidecar for a parsed ladder, as a finished download would.""" + from unshackle.core.titles import Movie + + class StubService: + pass + + title = Movie(id_="movie-1", service=StubService, name="Example Movie", year=2024, language="en") + for track in [*tracks.videos, *tracks.audio, *tracks.subtitles]: + title.tracks.add(track) + title.tracks.manifest_url = manifest_url + + runner = dl.__new__(dl) + runner.service = "EXAMPLE" + export = tmp_path / "export.json" + for track in [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]: + runner.write_export(export, title, track) + return export + + +def import_tracks(export: Path): + svc = ImportService(SimpleNamespace(parent=None, params={}), "EXAMPLE", "movie-1", str(export)) + return svc, svc.get_tracks(next(iter(svc.get_titles()))) + + +@pytest.fixture +def ism_offline(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_MANIFEST, url)) + + +@pytest.mark.parametrize("manifest_url", [ISM_URL, None], ids=["with-manifest-url", "without-manifest-url"]) +def test_ism_round_trip_restores_segment_urls(tmp_path: Path, ism_offline: None, manifest_url: str | None) -> None: + """ISM.download_track reads data["ism"]["segments"], which to_dict cannot carry across an export. + + Without a manifest_url the export's own track URLs are the only way back to the manifest, + which is the case the re-parse branch alone never covered. + """ + original = ISM.from_text(ISM_MANIFEST, ISM_URL).to_tracks(language="en") + expected = {t.id: t.data["ism"]["segments"] for t in [*original.videos, *original.audio]} + + _, tracks = import_tracks(export_tracks(tmp_path, original, manifest_url)) + + rebuilt = {t.id: t.data.get("ism", {}).get("segments") for t in [*tracks.videos, *tracks.audio]} + assert rebuilt == expected, "every ISM track must come back with the fragment URLs it was exported with" + + +ISM_ENCRYPTED = """<?xml version="1.0"?> +<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000"> + <Protection><ProtectionHeader SystemID="{9A04F079-9840-4286-AB92-E65BE0885F95}">dGVzdA==</ProtectionHeader></Protection> + <StreamIndex Type="video" Name="video" Language="en" Url="QualityLevels({Bitrate})/Fragments(video={start_time})"> + <QualityLevel Index="0" Bitrate="3000000" FourCC="H264" MaxWidth="1280" MaxHeight="720" + CodecPrivateData="00000001674D401F"/> + <QualityLevel Index="1" Bitrate="6000000" FourCC="H264" MaxWidth="1920" MaxHeight="1080" + CodecPrivateData="00000001674D401F"/> + <c t="0" d="20000000" r="3"/> + </StreamIndex> +</SmoothStreamingMedia> +""" + + +def test_encrypted_ism_rung_gets_the_exported_keys(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """resolve_server_keys must recognise an ISM manifest's protection, not only a DASH one. + + ISM.download_track reads only ``track.drm``, so a rung that misses key injection downloads + encrypted and muxes without error. DASH survives the same omission because it re-derives its + DRM from the manifest elements at download time. + """ + monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_ENCRYPTED, url)) + parsed = ISM.from_text(ISM_ENCRYPTED, ISM_URL).to_tracks(language="en") + + # export the licensed rung only, as a finished download would + licensed = max(parsed.videos, key=lambda t: t.bitrate or 0) + export = export_tracks(tmp_path, parsed, ISM_URL) + doc = json.loads(export.read_text(encoding="utf8")) + doc["titles"]["movie-1"]["tracks"][licensed.id]["keys"] = {"0" * 32: "ab" * 16} + export.write_text(json.dumps(doc), encoding="utf8") + + svc, tracks = import_tracks(export) + movie = next(iter(svc.get_titles())) + movie.tracks = tracks + svc.resolve_server_keys(movie) + + other = min(tracks.videos, key=lambda t: t.bitrate or 0) + assert svc.track_is_encrypted(other), "an ISM ProtectionHeader means the rung is encrypted" + assert other.drm and other.drm[0].content_keys, "the unlicensed rung never received the exported key" + + +def test_hls_round_trip_keeps_every_variant_url(tmp_path: Path) -> None: + """HLS re-fetches each variant from track.url at download time, so the stored URL has to survive.""" + original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en") + expected = {t.id: str(t.url) for t in [*original.videos, *original.audio, *original.subtitles]} + + _, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL)) + + rebuilt = {t.id: str(t.url) for t in [*tracks.videos, *tracks.audio, *tracks.subtitles]} + assert rebuilt == expected, "the whole exported ladder must survive, each with its own variant URL" + assert {t.descriptor.name for t in tracks} == {"HLS"} + + +def test_hls_import_never_refetches_the_master_playlist(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A master playlist can hand out a fresh token per fetch, so import must not depend on it. + + This is why HLS is absent from MANIFEST_DATA_KEYS. Re-fetching would add nothing and give a + long-expired export another way to fail. + """ + original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en") + export = export_tracks(tmp_path, original, HLS_URL) + + def explode(*args: object, **kwargs: object) -> None: + raise AssertionError("import re-fetched an HLS manifest") + + monkeypatch.setattr(HLS, "from_url", explode) + monkeypatch.setattr(HLS, "from_text", explode) + + _, tracks = import_tracks(export) + assert len(list(tracks)) == 4 + + +def test_hls_tracks_tolerate_the_empty_manifest_data(tmp_path: Path) -> None: + """HLS.download_track assigns into track.data["hls"] without creating it first. + + Track.data is a defaultdict, so the assignment auto-vivifies rather than raising. Pin that, + because a plain dict here would turn every HLS import into a KeyError mid-download. + """ + original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en") + _, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL)) + + track = tracks.videos[0] + assert not track.data.get("hls"), "to_dict cannot carry the m3u8 objects, and does not need to" + track.data["hls"]["segment_durations"] = [1, 2, 3] # hls.py, in download_track + assert track.data["hls"]["segment_durations"] == [1, 2, 3] diff --git a/tests/core/test_import_manifest_data.py b/tests/core/test_import_manifest_data.py new file mode 100644 index 00000000..dac53442 --- /dev/null +++ b/tests/core/test_import_manifest_data.py @@ -0,0 +1,226 @@ +"""Tests for the manifest data an import must rebuild before a DASH track can download. + +``Track.to_dict`` cannot serialise manifest XML, so a rebuilt DASH track carries no +``track.data["dash"]``. ``DASH.download_track`` reads that mapping unconditionally, so any +DASH track the import hands back without it raises on the first segment. Two exports reach +that state: one with no top-level ``manifest_url``, and one whose ``manifest_url`` covers +only part of the ladder (a separate MPD per codec). +""" + +from __future__ import annotations + +import json +from functools import partial +from pathlib import Path +from typing import Any, Optional + +import click +import pytest + +from unshackle.core.import_service import ImportService +from unshackle.core.manifests import DASH + +MPD_TEMPLATE = """<?xml version="1.0"?> +<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M"> + <Period id="1"> + <AdaptationSet contentType="video" mimeType="video/mp4" lang="en"> + <Representation id="{rep_id}" codecs="{codecs}" bandwidth="{bandwidth}" width="{width}" height="{height}"> + <BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/> + </Representation> + </AdaptationSet> + </Period> +</MPD> +""" + +AVC_URL = "https://example.invalid/avc/dash/m.mpd" +HEVC_URL = "https://example.invalid/hevc/dash/m.mpd" + +MANIFESTS = { + AVC_URL: MPD_TEMPLATE.format(rep_id="v-avc", codecs="avc1.640028", bandwidth=5000000, width=1920, height=1080), + HEVC_URL: MPD_TEMPLATE.format( + rep_id="v-hevc", codecs="hvc1.2.4.L153.B0", bandwidth=6000000, width=3840, height=2160 + ), +} + + +def exported_video(track_id: str, url: str, codec: str, width: int, height: int) -> dict[str, Any]: + return { + "type": "Video", + "id": track_id, + "url": url, + "language": "en", + "descriptor": "DASH", + "codec": codec, + "bitrate": 5000000 if codec == "AVC" else 6000000, + "width": width, + "height": height, + } + + +def make_service(tmp_path: Path, manifest_url: Optional[str], tracks: dict[str, Any]) -> ImportService: + entry: dict[str, Any] = { + "meta": {"type": "movie", "id": "movie-1", "name": "Example Movie", "year": 2024, "language": "en"}, + "manifest_type": "DASH", + "tracks": tracks, + } + if manifest_url: + entry["manifest_url"] = manifest_url + export = {"version": 2, "service": "EXAMPLE", "titles": {"movie-1": entry}} + path = tmp_path / "export.json" + path.write_text(json.dumps(export), encoding="utf8") + return ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path)) + + +@pytest.fixture(autouse=True) +def local_manifests(monkeypatch: pytest.MonkeyPatch) -> None: + """Serve each MPD from memory, so a test never depends on the network.""" + monkeypatch.setattr(DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(MANIFESTS[url], url)) + + +def test_export_without_manifest_url_still_rebuilds_dash_manifest_data(tmp_path: Path) -> None: + """No top-level manifest_url means the export's own track URLs are the only MPDs to fetch.""" + service = make_service(tmp_path, None, {"v-avc": exported_video("v-avc", AVC_URL, "AVC", 1920, 1080)}) + tracks = service.get_tracks(next(iter(service.get_titles()))) + + assert [t.id for t in tracks.videos] == ["v-avc"] + # DASH.download_track reads all three of these; a missing one is a KeyError mid-download. + assert set(tracks.videos[0].data.get("dash", {})) >= {"manifest", "adaptation_set", "representation"} + + +def test_exported_hevc_on_its_own_mpd_survives_the_reparse(tmp_path: Path) -> None: + """A ladder split across per-codec MPDs loses every codec but manifest_url's if only URL tracks merge back.""" + service = make_service( + tmp_path, + AVC_URL, + {"v-hevc": exported_video("v-hevc", HEVC_URL, "HEVC", 3840, 2160)}, + ) + tracks = service.get_tracks(next(iter(service.get_titles()))) + + codecs = {t.codec.name for t in tracks.videos if t.codec} + assert "HEVC" in codecs, "the exported HEVC track was dropped, so -v h.265 has nothing to select" + hevc = next(t for t in tracks.videos if t.codec and t.codec.name == "HEVC") + assert set(hevc.data.get("dash", {})) >= {"manifest", "adaptation_set", "representation"} + # bound to its own manifest's representation, not just to any populated mapping + assert hevc.data["dash"]["representation"].get("id") == "v-hevc" + + +SUB_MPD_URL = "https://example.invalid/subs/de.mpd" + +MAIN_WITH_SUB = """<?xml version="1.0"?> +<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M"> + <Period id="1"> + <AdaptationSet contentType="video" mimeType="video/mp4" lang="en"> + <Representation id="v-avc" codecs="avc1.640028" bandwidth="5000000" width="1920" height="1080"> + <BaseURL>v.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation> + </AdaptationSet> + <AdaptationSet contentType="text" mimeType="text/vtt" lang="en"> + <Representation id="s-en" bandwidth="1000"> + <BaseURL>en.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation> + </AdaptationSet> + </Period> +</MPD> +""" + +FOREIGN_SUB_MPD = """<?xml version="1.0"?> +<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M"> + <Period id="1"> + <AdaptationSet contentType="text" mimeType="text/vtt" lang="de"> + <Representation id="s-de" bandwidth="1000"> + <BaseURL>de.vtt</BaseURL><SegmentBase indexRange="0-1"/></Representation> + </AdaptationSet> + </Period> +</MPD> +""" + + +def test_foreign_subtitle_adds_to_the_manifests_own(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Only a side-loaded (URL) subtitle replaces the re-parsed set. + + A subtitle that lives on its own MPD is an addition. Nothing about it says the service + considered the main manifest's copy the worse one, and clearing on it deletes languages the + export never meant to drop. + """ + manifests = {AVC_URL: MAIN_WITH_SUB, SUB_MPD_URL: FOREIGN_SUB_MPD} + monkeypatch.setattr(DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(manifests[url], url)) + foreign = DASH.from_text(FOREIGN_SUB_MPD, SUB_MPD_URL).to_tracks(language="de").subtitles[0] + + service = make_service( + tmp_path, + AVC_URL, + { + foreign.id: { + "type": "Subtitle", + "id": foreign.id, + "url": SUB_MPD_URL, + "language": "de", + "descriptor": "DASH", + "codec": "WebVTT", + } + }, + ) + tracks = service.get_tracks(next(iter(service.get_titles()))) + + assert sorted(str(t.language) for t in tracks.subtitles) == ["de", "en"], ( + "the main manifest's own subtitles were wiped by a subtitle from another MPD" + ) + + +TWO_RUNG_MPD = """<?xml version="1.0"?> +<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT10M"> + <Period id="1"> + <AdaptationSet contentType="video" mimeType="video/mp4" lang="en"> + <Representation id="lo" codecs="avc1.640028" bandwidth="3000000" width="1920" height="1080"> + <BaseURL>lo.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation> + <Representation id="hi" codecs="avc1.640028" bandwidth="{top}" width="1920" height="1080"> + <BaseURL>hi.mp4</BaseURL><SegmentBase indexRange="0-1"/></Representation> + </AdaptationSet> + </Period> +</MPD> +""" + + +@pytest.mark.parametrize( + "exported_url", + [AVC_URL, AVC_URL + "?token=EXPIRED"], + ids=["same-url", "signed-url"], +) +def test_reencoded_top_rung_does_not_readd_the_exported_one( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, exported_url: str +) -> None: + """Track ids hash the bitrate, so re-encoding a rung makes the exported id look like a foreign MPD's. + + Merging it back on that basis duplicates the rung, and the attribute fallback then binds the + copy to the first representation of the same codec and resolution: the 3 Mbps one. The result + downloads `lo` under a 6 Mbps name. + + The signed variant is the same manifest reached by two strings: a service may fetch a signed + URL while registering the bare one as Title.tracks.manifest_url, so comparing them whole + reopens the duplicate. + """ + exported = DASH.from_text(TWO_RUNG_MPD.format(top=6000000), exported_url).to_tracks(language="en") + top = next(t for t in exported.videos if t.bitrate == 6000000) + monkeypatch.setattr( + DASH, "from_url", lambda url, session=None, **kw: DASH.from_text(TWO_RUNG_MPD.format(top=6500000), url) + ) + + service = make_service(tmp_path, AVC_URL, {top.id: exported_video(top.id, exported_url, "AVC", 1920, 1080)}) + tracks = service.get_tracks(next(iter(service.get_titles()))) + + assert sorted(t.bitrate for t in tracks.videos) == [3000000, 6500000], ( + "the re-parsed ladder is the only source for this manifest's rungs" + ) + bound = {t.data["dash"]["representation"].get("id") for t in tracks.videos} + assert bound == {"lo", "hi"}, "two tracks were bound to the same representation" + + +def test_clearkey_license_hook_is_bindable(tmp_path: Path) -> None: + """dl.py binds service.get_clearkey_license for every track, DRM or not, before downloading. + + Bind it the way dl.py does rather than asking hasattr: a non-callable attribute satisfies + hasattr and still fails at the partial with a TypeError. + """ + service = make_service(tmp_path, AVC_URL, {}) + bound = partial(service.get_clearkey_license, title=None, track=None) + assert callable(bound) + with pytest.raises(RuntimeError, match="keys come from the export"): + bound(challenge=b"") diff --git a/unshackle/core/import_service.py b/unshackle/core/import_service.py index 322ebb1a..3dedd1f8 100644 --- a/unshackle/core/import_service.py +++ b/unshackle/core/import_service.py @@ -79,9 +79,7 @@ def _resolve_import_manifest_data( f"No language for '{title}': the {manifest_type} manifest has none and the " f"service did not set Title.language." ) - raise click.ClickException( - f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})" - ) + raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})") except Exception as e: raise click.ClickException( f"Failed to fetch the {manifest_type} manifest for '{title}'. " @@ -246,13 +244,22 @@ class ImportService: for track in parsed: tracks.add(track) parsed_ids = {str(t.id) for t in tracks} - track_dicts = [t for t in tracks_map.values() if t.get("descriptor", "URL") == "URL"] - track_dicts.extend( + url_dicts = [t for t in tracks_map.values() if t.get("descriptor", "URL") == "URL"] + # Merge back only tracks from another manifest, such as a separate HEVC one; the + # re-parse already covers manifest_url's own. Query stripped when comparing, since a + # service can sign the URL it fetches and register the bare one for the same manifest. + manifest_base = str(manifest_url).split("?")[0] + foreign_dicts = [ t for t in tracks_map.values() - if t.get("descriptor") == manifest_type and str(t.get("id")) not in parsed_ids - ) - if any(t.get("type") == "Subtitle" for t in track_dicts): + if t.get("descriptor") == manifest_type + and str(t.get("url")).split("?")[0] != manifest_base + and str(t.get("id")) not in parsed_ids + ] + track_dicts = [*url_dicts, *foreign_dicts] + # Only a side-loaded subtitle replaces the re-parsed set, since the service chose its + # own copy over the manifest's. A subtitle on another manifest is an addition. + if any(t.get("type") == "Subtitle" for t in url_dicts): tracks.subtitles.clear() else: track_dicts = list(tracks_map.values()) @@ -355,14 +362,25 @@ class ImportService: @staticmethod def track_is_encrypted(track: Any) -> bool: - """True if the track carries DRM or its DASH manifest declares ContentProtection.""" + """True if the track carries DRM or its manifest declares protection. + + ISM is checked as well as DASH because ISM.download_track reads only ``track.drm``. A rung + that misses key injection here downloads encrypted and muxes without error. DASH re-derives + its DRM from the manifest elements, so it survives the same omission. + """ if track.drm: return True - dash = track.data.get("dash") if getattr(track, "data", None) else None + data = getattr(track, "data", None) or {} + dash = data.get("dash") if dash: for element in (dash.get("representation"), dash.get("adaptation_set")): if element is not None and element.findall("ContentProtection"): return True + ism = data.get("ism") + if ism: + manifest = ism.get("manifest") + if manifest is not None and manifest.findall(".//ProtectionHeader"): + return True return False def exported_drm_system(self) -> str: From 72b8ab7027df555df04a1ad475c629b393708406 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Thu, 30 Jul 2026 13:44:07 -0600 Subject: [PATCH 21/40] feat: show git commit hash alongside the version --- docs/dev/rest-api/endpoints.md | 3 ++ docs/guide/troubleshooting.md | 2 +- tests/core/test_git_commit.py | 57 ++++++++++++++++++++++++++++++++++ unshackle/core/__init__.py | 23 ++++++++++++++ unshackle/core/__main__.py | 5 +-- unshackle/core/api/routes.py | 10 ++++-- unshackle/core/utilities.py | 3 +- 7 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 tests/core/test_git_commit.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 702dd98a..f8894256 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -43,6 +43,7 @@ Health check. **This is the only route exempt from authentication**: you can cal { "status": "ok", "version": "5.3.0", + "commit": "a24c9b9", "update_check": { "update_available": false, "current_version": "5.3.0", @@ -51,6 +52,8 @@ Health check. **This is the only route exempt from authentication**: you can cal } ``` +`commit` is the short git commit of the running checkout. It is `null` when the server cannot read one (for example, an installed package with no `.git` directory, or a host without git). + If the update check fails (for example, no network), `update_available` and `latest_version` are `null` while `current_version` still reports the installed version. | Status | Meaning | diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 383850e3..70cfa4fe 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -74,7 +74,7 @@ For example, `unshackle_debug_EXAMPLE_20260703-142530.jsonl`. The `<service>` ta Every line is a self-contained JSON object. The first line of a session records the environment: ```json -{"timestamp":"2026-07-03T14:25:30.123456+00:00","session_id":"a1b2c3d4","level":"INFO","operation":"session_start","message":"Debug logging session started","context":{"unshackle_version":"5.3.0","python_version":"3.12.4 ...","platform":"Linux-6.18.5-x86_64","platform_system":"Linux","platform_release":"6.18.5"}} +{"timestamp":"2026-07-03T14:25:30.123456+00:00","session_id":"a1b2c3d4","level":"INFO","operation":"session_start","message":"Debug logging session started","context":{"unshackle_version":"5.3.0","unshackle_commit":"a24c9b9","python_version":"3.12.4 ...","platform":"Linux-6.18.5-x86_64","platform_system":"Linux","platform_release":"6.18.5"}} ``` Subsequent lines describe operations: service calls, DRM licence requests, vault lookups, downloader progress, and errors. Common fields include: diff --git a/tests/core/test_git_commit.py b/tests/core/test_git_commit.py new file mode 100644 index 00000000..f375ceab --- /dev/null +++ b/tests/core/test_git_commit.py @@ -0,0 +1,57 @@ +"""Unit tests for the git commit probe behind the banner and the /api/health `commit` field. +The probe runs at import time, so it must always return a string and must never raise, whatever +state git and the checkout are in.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +import unshackle.core +from unshackle.core import _git_commit + +pytestmark = pytest.mark.unit + +# Without a .git the probe returns before it runs git, so the subprocess stubs below prove nothing. +needs_checkout = pytest.mark.skipif( + not (Path(unshackle.core.__file__).parents[2] / ".git").exists(), reason="not running from a git checkout" +) + + +def test_returns_str_in_this_checkout() -> None: + assert isinstance(_git_commit(), str) + + +@pytest.mark.parametrize( + "error", + [ + FileNotFoundError("git"), + PermissionError("git"), + subprocess.TimeoutExpired(cmd="git", timeout=2), + subprocess.SubprocessError("boom"), + ], +) +@needs_checkout +def test_swallows_subprocess_errors(monkeypatch: pytest.MonkeyPatch, error: Exception) -> None: + def raise_it(*args: object, **kwargs: object) -> None: + raise error + + monkeypatch.setattr(subprocess, "run", raise_it) + assert _git_commit() == "" + + +@pytest.mark.parametrize( + ("returncode", "stdout", "expected"), + [ + (0, "a24c9b9\n", "a24c9b9"), + (128, "", ""), # empty repo, corrupt repo, or a stale worktree pointer + (0, "", ""), # no output despite success + ], +) +@needs_checkout +def test_maps_git_result(monkeypatch: pytest.MonkeyPatch, returncode: int, stdout: str, expected: str) -> None: + completed = subprocess.CompletedProcess(args=["git"], returncode=returncode, stdout=stdout, stderr="noise") + monkeypatch.setattr(subprocess, "run", lambda *a, **k: completed) + assert _git_commit() == expected diff --git a/unshackle/core/__init__.py b/unshackle/core/__init__.py index f5752882..7826b872 100644 --- a/unshackle/core/__init__.py +++ b/unshackle/core/__init__.py @@ -1 +1,24 @@ +import subprocess +from pathlib import Path + __version__ = "5.3.0" + + +def _git_commit() -> str: + root = Path(__file__).parents[2] + if not (root / ".git").exists(): + return "" + try: + proc = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd=root, + capture_output=True, + text=True, + timeout=2, + ) + except (OSError, subprocess.SubprocessError): + return "" + return proc.stdout.strip() if proc.returncode == 0 else "" + + +__commit__ = _git_commit() diff --git a/unshackle/core/__main__.py b/unshackle/core/__main__.py index 74744bd3..89f90616 100644 --- a/unshackle/core/__main__.py +++ b/unshackle/core/__main__.py @@ -10,7 +10,7 @@ from rich.padding import Padding from rich.text import Text from urllib3.exceptions import InsecureRequestWarning -from unshackle.core import __version__ +from unshackle.core import __commit__, __version__ from unshackle.core.commands import Commands from unshackle.core.config import config from unshackle.core.console import ComfyRichHandler, console @@ -63,7 +63,8 @@ def main(version: bool, debug: bool) -> None: r" ▀▀▀ ▀▀ █▪ ▀▀▀▀ ▀▀▀ · ▀ ▀ ·▀▀▀ ·▀ ▀.▀▀▀ ▀▀▀ ", style="ascii.art", ), - f"v [repr.number]{__version__}[/] - © 2025-{datetime.now().year} - github.com/unshackle-dl/unshackle", + f"v [repr.number]{__version__}[/]{f' ({__commit__})' if __commit__ else ''}" + f" - © 2025-{datetime.now().year} - github.com/unshackle-dl/unshackle", ), (1, 11, 1, 10), expand=True, diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 44956915..c578170b 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -8,7 +8,7 @@ import click from aiohttp import web from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings -from unshackle.core import __version__ +from unshackle.core import __commit__, __version__ from unshackle.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception from unshackle.core.api.handlers import ( cancel_download_job_handler, @@ -105,6 +105,10 @@ async def health(request: web.Request) -> web.Response: version: type: string example: "2.0.0" + commit: + type: string + nullable: true + example: "a24c9b9" update_check: type: object properties: @@ -128,7 +132,9 @@ async def health(request: web.Request) -> web.Response: log.warning(f"Failed to check for updates: {e}") update_info = {"update_available": None, "current_version": __version__, "latest_version": None} - return web.json_response({"status": "ok", "version": __version__, "update_check": update_info}) + return web.json_response( + {"status": "ok", "version": __version__, "commit": __commit__ or None, "update_check": update_info} + ) @api_handler diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 62958a55..dd100b22 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -787,7 +787,7 @@ class DebugLogger: """Log the start of a new session with environment information.""" import platform - from unshackle.core import __version__ + from unshackle.core import __commit__, __version__ self.log( level="INFO", @@ -795,6 +795,7 @@ class DebugLogger: message="Debug logging session started", context={ "unshackle_version": __version__, + "unshackle_commit": __commit__ or None, "python_version": sys.version, "platform": platform.platform(), "platform_system": platform.system(), From c14277e6dc47d116c553bc8d04c9047405cdda29 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Thu, 30 Jul 2026 16:30:03 -0600 Subject: [PATCH 22/40] feat: identify the running code by source hash --- .gitignore | 3 ++ docs/dev/rest-api/endpoints.md | 4 +- docs/guide/troubleshooting.md | 4 +- tests/core/test_code_hash.py | 74 ++++++++++++++++++++++++++++++++++ tests/core/test_git_commit.py | 57 -------------------------- unshackle/core/__init__.py | 59 ++++++++++++++++++++------- unshackle/core/__main__.py | 4 +- unshackle/core/api/routes.py | 8 ++-- unshackle/core/utilities.py | 4 +- 9 files changed, 134 insertions(+), 83 deletions(-) create mode 100644 tests/core/test_code_hash.py delete mode 100644 tests/core/test_git_commit.py diff --git a/.gitignore b/.gitignore index 745dd310..ee420d65 100644 --- a/.gitignore +++ b/.gitignore @@ -257,3 +257,6 @@ __marimo__/ # Test fixtures with private/local content (services, IDs, credentials) tests/remote/e2e/fixtures/fixtures.yaml + +# Maintainer-side scripts, kept local +tools/ diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index f8894256..4a205505 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -43,7 +43,7 @@ Health check. **This is the only route exempt from authentication**: you can cal { "status": "ok", "version": "5.3.0", - "commit": "a24c9b9", + "code_hash": "1d22a1e", "update_check": { "update_available": false, "current_version": "5.3.0", @@ -52,7 +52,7 @@ Health check. **This is the only route exempt from authentication**: you can cal } ``` -`commit` is the short git commit of the running checkout. It is `null` when the server cannot read one (for example, an installed package with no `.git` directory, or a host without git). +`code_hash` fingerprints the framework source the server is running, so it identifies the code itself rather than the release. It is the same value for a git checkout and for an installed package built from that checkout, and it changes if anything under `unshackle/core`, `unshackle/commands`, `unshackle/utils`, or `unshackle/vaults` is edited. Service modules are not included. The value is `null` only when the source files cannot be read. If the update check fails (for example, no network), `update_available` and `latest_version` are `null` while `current_version` still reports the installed version. diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 70cfa4fe..50cf356f 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -74,9 +74,11 @@ For example, `unshackle_debug_EXAMPLE_20260703-142530.jsonl`. The `<service>` ta Every line is a self-contained JSON object. The first line of a session records the environment: ```json -{"timestamp":"2026-07-03T14:25:30.123456+00:00","session_id":"a1b2c3d4","level":"INFO","operation":"session_start","message":"Debug logging session started","context":{"unshackle_version":"5.3.0","unshackle_commit":"a24c9b9","python_version":"3.12.4 ...","platform":"Linux-6.18.5-x86_64","platform_system":"Linux","platform_release":"6.18.5"}} +{"timestamp":"2026-07-03T14:25:30.123456+00:00","session_id":"a1b2c3d4","level":"INFO","operation":"session_start","message":"Debug logging session started","context":{"unshackle_version":"5.3.0","unshackle_code_hash":"1d22a1e","python_version":"3.12.4 ...","platform":"Linux-6.18.5-x86_64","platform_system":"Linux","platform_release":"6.18.5"}} ``` +Quote `unshackle_code_hash` in a bug report. It fingerprints the framework source you are running, so it tells us the exact code state even when you did not install from git. A hash that matches no release means the source was edited locally. + Subsequent lines describe operations: service calls, DRM licence requests, vault lookups, downloader progress, and errors. Common fields include: | Field | Meaning | diff --git a/tests/core/test_code_hash.py b/tests/core/test_code_hash.py new file mode 100644 index 00000000..9aa10aa3 --- /dev/null +++ b/tests/core/test_code_hash.py @@ -0,0 +1,74 @@ +"""Unit tests for the code fingerprint behind the banner and the /api/health `code_hash` field. +It runs at import time, so it must always return a string and must never raise. It must change when +the framework source changes and stay unchanged when anything else does.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import unshackle.core +from unshackle.core import __code_hash__, code_files, code_hash + +pytestmark = pytest.mark.unit + + +def test_is_hex_and_stable() -> None: + first = code_hash() + assert len(first) == 32 + assert int(first, 16) >= 0 + assert first == code_hash() + + +def test_exported_value_is_the_short_form() -> None: + assert __code_hash__ == code_hash()[:7] + + +def test_covers_the_framework_dirs_only() -> None: + rels = code_files() + assert "__main__.py" in rels + assert any(rel.startswith("core/") for rel in rels) + assert not any(rel.startswith("services/") for rel in rels) + assert not any("__pycache__" in rel for rel in rels) + assert all(rel.endswith(".py") for rel in rels) + assert rels == sorted(rels) # order must not depend on the filesystem + + +def test_content_changes_the_hash() -> None: + rels = ["core/__init__.py", "commands/dl.py"] + blobs = {"core/__init__.py": b"a", "commands/dl.py": b"b"} + before = code_hash(files=rels, read=blobs.__getitem__) + + blobs["commands/dl.py"] = b"b " + assert code_hash(files=rels, read=blobs.__getitem__) != before + + +def test_path_is_part_of_the_hash() -> None: + """A rename with identical bytes must not look like the same code.""" + same_bytes = {"core/a.py": b"x", "core/b.py": b"x"} + assert code_hash(files=["core/a.py"], read=same_bytes.__getitem__) != code_hash( + files=["core/b.py"], read=same_bytes.__getitem__ + ) + + +def test_file_order_does_not_matter_to_the_caller() -> None: + """code_files() sorts, so two orderings of the same set must agree once sorted.""" + blobs = {"core/a.py": b"x", "core/b.py": b"y"} + assert code_hash(files=sorted(blobs), read=blobs.__getitem__) == code_hash( + files=sorted(reversed(list(blobs))), read=blobs.__getitem__ + ) + + +def test_unreadable_source_yields_empty_string() -> None: + def unreadable(rel: str) -> bytes: + raise PermissionError(rel) + + assert code_hash(files=["core/__init__.py"], read=unreadable) == "" + + +def test_unwalkable_source_yields_empty_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A dir os.walk cannot enter must yield "", not a valid-looking hash of fewer files.""" + (tmp_path / "__main__.py").write_bytes(b"") + monkeypatch.setattr(unshackle.core, "_PKG", tmp_path) + assert code_hash() == "" diff --git a/tests/core/test_git_commit.py b/tests/core/test_git_commit.py deleted file mode 100644 index f375ceab..00000000 --- a/tests/core/test_git_commit.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Unit tests for the git commit probe behind the banner and the /api/health `commit` field. -The probe runs at import time, so it must always return a string and must never raise, whatever -state git and the checkout are in.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -import unshackle.core -from unshackle.core import _git_commit - -pytestmark = pytest.mark.unit - -# Without a .git the probe returns before it runs git, so the subprocess stubs below prove nothing. -needs_checkout = pytest.mark.skipif( - not (Path(unshackle.core.__file__).parents[2] / ".git").exists(), reason="not running from a git checkout" -) - - -def test_returns_str_in_this_checkout() -> None: - assert isinstance(_git_commit(), str) - - -@pytest.mark.parametrize( - "error", - [ - FileNotFoundError("git"), - PermissionError("git"), - subprocess.TimeoutExpired(cmd="git", timeout=2), - subprocess.SubprocessError("boom"), - ], -) -@needs_checkout -def test_swallows_subprocess_errors(monkeypatch: pytest.MonkeyPatch, error: Exception) -> None: - def raise_it(*args: object, **kwargs: object) -> None: - raise error - - monkeypatch.setattr(subprocess, "run", raise_it) - assert _git_commit() == "" - - -@pytest.mark.parametrize( - ("returncode", "stdout", "expected"), - [ - (0, "a24c9b9\n", "a24c9b9"), - (128, "", ""), # empty repo, corrupt repo, or a stale worktree pointer - (0, "", ""), # no output despite success - ], -) -@needs_checkout -def test_maps_git_result(monkeypatch: pytest.MonkeyPatch, returncode: int, stdout: str, expected: str) -> None: - completed = subprocess.CompletedProcess(args=["git"], returncode=returncode, stdout=stdout, stderr="noise") - monkeypatch.setattr(subprocess, "run", lambda *a, **k: completed) - assert _git_commit() == expected diff --git a/unshackle/core/__init__.py b/unshackle/core/__init__.py index 7826b872..5005e996 100644 --- a/unshackle/core/__init__.py +++ b/unshackle/core/__init__.py @@ -1,24 +1,53 @@ -import subprocess +import hashlib +import os from pathlib import Path +from typing import Callable, Optional __version__ = "5.3.0" +_PKG = Path(__file__).parent.parent +# Framework code only. Services are user-swappable, so they are not part of the identity. +_CODE_DIRS = ("core", "commands", "utils", "vaults") -def _git_commit() -> str: - root = Path(__file__).parents[2] - if not (root / ".git").exists(): - return "" + +def _raise(error: OSError) -> None: + raise error + + +def code_files() -> list[str]: + """Framework source paths relative to the package root, in a platform-stable order.""" + rels = ["__main__.py"] + for name in _CODE_DIRS: + for root, dirs, files in os.walk(_PKG / name, onerror=_raise): + dirs[:] = [d for d in dirs if d != "__pycache__"] + rel = os.path.relpath(root, _PKG) + rels.extend(os.path.join(rel, f).replace(os.sep, "/") for f in files if f.endswith(".py")) + return sorted(rels) + + +def code_hash( + files: Optional[list[str]] = None, + read: Optional[Callable[[str], bytes]] = None, +) -> str: + """ + md5 of the framework source. Identifies the running code, not the release it claims to be. + + Pass `files` and `read` to digest a different byte source, such as blobs from git history + (see tools/resolve_code_hash.py). Returns "" when the source cannot be read. + """ + if read is None: + + def read(rel: str) -> bytes: + return (_PKG / rel).read_bytes() + + digest = hashlib.md5(usedforsecurity=False) try: - proc = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - cwd=root, - capture_output=True, - text=True, - timeout=2, - ) - except (OSError, subprocess.SubprocessError): + for rel in code_files() if files is None else files: + digest.update(rel.encode()) + digest.update(read(rel)) + except OSError: return "" - return proc.stdout.strip() if proc.returncode == 0 else "" + return digest.hexdigest() -__commit__ = _git_commit() +__code_hash__ = code_hash()[:7] diff --git a/unshackle/core/__main__.py b/unshackle/core/__main__.py index 89f90616..7c30742e 100644 --- a/unshackle/core/__main__.py +++ b/unshackle/core/__main__.py @@ -10,7 +10,7 @@ from rich.padding import Padding from rich.text import Text from urllib3.exceptions import InsecureRequestWarning -from unshackle.core import __commit__, __version__ +from unshackle.core import __code_hash__, __version__ from unshackle.core.commands import Commands from unshackle.core.config import config from unshackle.core.console import ComfyRichHandler, console @@ -63,7 +63,7 @@ def main(version: bool, debug: bool) -> None: r" ▀▀▀ ▀▀ █▪ ▀▀▀▀ ▀▀▀ · ▀ ▀ ·▀▀▀ ·▀ ▀.▀▀▀ ▀▀▀ ", style="ascii.art", ), - f"v [repr.number]{__version__}[/]{f' ({__commit__})' if __commit__ else ''}" + f"v [repr.number]{__version__}[/]{f' ({__code_hash__})' if __code_hash__ else ''}" f" - © 2025-{datetime.now().year} - github.com/unshackle-dl/unshackle", ), (1, 11, 1, 10), diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index c578170b..7605339b 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -8,7 +8,7 @@ import click from aiohttp import web from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings -from unshackle.core import __commit__, __version__ +from unshackle.core import __code_hash__, __version__ from unshackle.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception from unshackle.core.api.handlers import ( cancel_download_job_handler, @@ -105,10 +105,10 @@ async def health(request: web.Request) -> web.Response: version: type: string example: "2.0.0" - commit: + code_hash: type: string nullable: true - example: "a24c9b9" + example: "1d22a1e" update_check: type: object properties: @@ -133,7 +133,7 @@ async def health(request: web.Request) -> web.Response: update_info = {"update_available": None, "current_version": __version__, "latest_version": None} return web.json_response( - {"status": "ok", "version": __version__, "commit": __commit__ or None, "update_check": update_info} + {"status": "ok", "version": __version__, "code_hash": __code_hash__ or None, "update_check": update_info} ) diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index dd100b22..d101ed35 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -787,7 +787,7 @@ class DebugLogger: """Log the start of a new session with environment information.""" import platform - from unshackle.core import __commit__, __version__ + from unshackle.core import __code_hash__, __version__ self.log( level="INFO", @@ -795,7 +795,7 @@ class DebugLogger: message="Debug logging session started", context={ "unshackle_version": __version__, - "unshackle_commit": __commit__ or None, + "unshackle_code_hash": __code_hash__ or None, "python_version": sys.version, "platform": platform.platform(), "platform_system": platform.system(), From e9a40a3af342ed8a53f4b010f6329bbecd06a527 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Thu, 30 Jul 2026 17:11:18 -0600 Subject: [PATCH 23/40] perf: speed up the code hash with sha1 and a plain file reader --- tests/core/test_code_hash.py | 2 +- unshackle/core/__init__.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/core/test_code_hash.py b/tests/core/test_code_hash.py index 9aa10aa3..68ee6d92 100644 --- a/tests/core/test_code_hash.py +++ b/tests/core/test_code_hash.py @@ -16,7 +16,7 @@ pytestmark = pytest.mark.unit def test_is_hex_and_stable() -> None: first = code_hash() - assert len(first) == 32 + assert len(first) == 40 assert int(first, 16) >= 0 assert first == code_hash() diff --git a/unshackle/core/__init__.py b/unshackle/core/__init__.py index 5005e996..130bca20 100644 --- a/unshackle/core/__init__.py +++ b/unshackle/core/__init__.py @@ -16,12 +16,13 @@ def _raise(error: OSError) -> None: def code_files() -> list[str]: """Framework source paths relative to the package root, in a platform-stable order.""" + pkg = str(_PKG) rels = ["__main__.py"] for name in _CODE_DIRS: - for root, dirs, files in os.walk(_PKG / name, onerror=_raise): + for root, dirs, files in os.walk(os.path.join(pkg, name), onerror=_raise): dirs[:] = [d for d in dirs if d != "__pycache__"] - rel = os.path.relpath(root, _PKG) - rels.extend(os.path.join(rel, f).replace(os.sep, "/") for f in files if f.endswith(".py")) + rel = os.path.relpath(root, pkg).replace(os.sep, "/") + rels.extend(f"{rel}/{f}" for f in files if f.endswith(".py")) return sorted(rels) @@ -30,17 +31,19 @@ def code_hash( read: Optional[Callable[[str], bytes]] = None, ) -> str: """ - md5 of the framework source. Identifies the running code, not the release it claims to be. + sha1 of the framework source. Two installs on the same version differ here if their code does. Pass `files` and `read` to digest a different byte source, such as blobs from git history (see tools/resolve_code_hash.py). Returns "" when the source cannot be read. """ if read is None: + pkg = str(_PKG) def read(rel: str) -> bytes: - return (_PKG / rel).read_bytes() + with open(os.path.join(pkg, rel), "rb") as fh: + return fh.read() - digest = hashlib.md5(usedforsecurity=False) + digest = hashlib.sha1(usedforsecurity=False) try: for rel in code_files() if files is None else files: digest.update(rel.encode()) From 734df87badd46169e4163e14afb5ae1ab4fabe22 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Fri, 31 Jul 2026 12:48:46 -0600 Subject: [PATCH 24/40] feat(dl): add --speed-limit and serve global_speed_limit Shared token bucket across all download threads; decimal suffixes (5M = 5.0 MB/s) or plain bytes/sec --- docs/guide/cli-reference.md | 1 + docs/reference/configuration.md | 2 + tests/downloaders/__init__.py | 0 tests/downloaders/test_speed_limit.py | 118 +++++++++++++++++++++++ tests/downloaders/test_speed_window.py | 43 +++++++++ unshackle/commands/dl.py | 21 +++++ unshackle/commands/serve.py | 9 ++ unshackle/core/downloaders/__init__.py | 4 +- unshackle/core/downloaders/requests.py | 125 +++++++++++++++++++++++-- 9 files changed, 313 insertions(+), 10 deletions(-) create mode 100644 tests/downloaders/__init__.py create mode 100644 tests/downloaders/test_speed_limit.py create mode 100644 tests/downloaders/test_speed_window.py diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index c19c9424..ce4d53dd 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -172,6 +172,7 @@ Keep only certain track types, or skip certain track types. Attachments are alwa |---|---|---| | `--workers` | downloader default | Per-track download threads. | | `--downloads` | `1` | Number of tracks downloaded concurrently. | +| `--speed-limit` | unlimited | Cap total download speed across all threads and tracks, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). `off` disables a configured limit. | | `--no-cache` | off | Bypass the title cache. | | `--reset-cache` | off | Clear the title cache. | | `--list` | off | List available/would-be-downloaded tracks; do not download. | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 22ab66da..45a00d26 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -245,6 +245,7 @@ is the [REST API](../dev/rest-api/index.md) section; these are the config keys. | `max_sessions` | int | `100` | Maximum concurrent sessions. | | `history_limit` | int | `100` | How many finished jobs to retain in history. | | `compression_level` | int | `1` | gzip level for responses. | +| `global_speed_limit` | str | *(unlimited)* | Server-wide download speed cap, e.g. `10M`, `1.5G` or plain bytes/sec (same format as `speed_limit`). One shared budget across all concurrent jobs; per-job speed limits are ignored while it is set. | | `cdm_overrides` | dict | *(unset)* | Allowed per-request CDM overrides. | | `allow_job_credentials` | bool | `false` | Permit clients to supply credentials per job. | | `devices` | list | *(auto)* | Widevine devices offered; auto-filled from `directories.wvds`. | @@ -322,6 +323,7 @@ Common keys (a useful subset; every `dl` flag works): | `no_subs` / `no_audio` / `no_chapters` | bool | `false` | Skip that track type. | | `downloads` | int | `1` | Tracks downloaded concurrently. | | `workers` | int | *(downloader default)* | Threads per track. | +| `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). | | `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | | `best_available` | bool | `false` | Fall back to best quality if the request is unavailable. | | `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | diff --git a/tests/downloaders/__init__.py b/tests/downloaders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/downloaders/test_speed_limit.py b/tests/downloaders/test_speed_limit.py new file mode 100644 index 00000000..538447ca --- /dev/null +++ b/tests/downloaders/test_speed_limit.py @@ -0,0 +1,118 @@ +import importlib +import inspect +import threading +import time + +import pytest + +from unshackle.core.constants import DOWNLOAD_CANCELLED +from unshackle.core.downloaders.requests import TokenBucket, format_speed, parse_speed_limit, set_speed_limit + +requests_downloader = importlib.import_module("unshackle.core.downloaders.requests") + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("500k", 500_000), + ("5M", 5_000_000), + ("10MB", 10_000_000), + ("10mb/s", 10_000_000), + ("1.5G", 1_500_000_000), + ("1048576", 1_048_576), + (5_000_000, 5_000_000.0), + (None, None), + ("off", None), + ("none", None), + ("unlimited", None), + ("0", None), + (0, None), + ], +) +def test_parse_speed_limit(value, expected): + assert parse_speed_limit(value) == expected + + +@pytest.mark.parametrize( + "value", + ["fast", "10TB", "10MiB", "-5M", "M", "1..5M", ".", "1.2.3", float("inf"), float("nan"), -5, -1.5], +) +def test_parse_speed_limit_rejects(value): + with pytest.raises(ValueError): + parse_speed_limit(value) + + +@pytest.mark.parametrize( + ("rate", "expected"), + [ + (500, "500 bytes/s"), + (512_000, "512.0 kB/s"), + (39 * 1024**2, "40.9 MB/s"), + (1.5 * 1024**3, "1.6 GB/s"), + ], +) +def test_format_speed(rate, expected): + assert format_speed(rate) == expected + + +def test_result_accepts_a_missing_speed_limit(): + """The API/serve path calls dl.result() by keyword without speed_limit.""" + from unshackle.commands.dl import dl + + assert inspect.signature(dl.result).parameters["speed_limit"].default is None + + +def test_bucket_holds_aggregate_rate_across_threads(): + rate = 10_000_000 + bucket = TokenBucket(rate) + chunk = 100_000 + totals = [0, 0, 0, 0] + start = time.monotonic() + stop = start + 1.0 + + def worker(i: int) -> None: + while time.monotonic() < stop: + bucket.consume(chunk) + totals[i] += chunk + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + achieved = sum(totals) / (time.monotonic() - start) + # exceeding the limit is a real failure; undershoot is scheduler noise on a loaded box + assert achieved <= rate * 1.1 + assert achieved >= rate * 0.5 + + +def test_bucket_chunk_larger_than_capacity_does_not_deadlock(): + bucket = TokenBucket(1_000_000) + start = time.monotonic() + bucket.consume(1_100_000) + assert 1.0 < time.monotonic() - start < 2.0 + + +def test_locked_limit_ignores_unlocked_calls(): + """serve's global_speed_limit must survive the per-job set in dl.result().""" + try: + set_speed_limit(5_000_000, lock=True) + set_speed_limit(None) + set_speed_limit(1_000_000) + limiter = requests_downloader._speed_limiter + assert limiter is not None and limiter.rate == 5_000_000 + finally: + requests_downloader._speed_limit_locked = False + requests_downloader._speed_limiter = None + + +def test_bucket_wait_aborts_on_cancellation(): + """Debt sleeps grow with thread count; cancelling must not wait them out.""" + bucket = TokenBucket(1_000) + DOWNLOAD_CANCELLED.set() + try: + start = time.monotonic() + bucket.consume(1_000_000) + assert time.monotonic() - start < 1.0 + finally: + DOWNLOAD_CANCELLED.clear() diff --git a/tests/downloaders/test_speed_window.py b/tests/downloaders/test_speed_window.py new file mode 100644 index 00000000..46d555b2 --- /dev/null +++ b/tests/downloaders/test_speed_window.py @@ -0,0 +1,43 @@ +import pytest + +from unshackle.core.downloaders.requests import SPEED_ROLLING_WINDOW, SpeedWindow + + +def test_rate_reflects_recent_window_not_full_history(): + """After a slow start (shared bucket), the display must show the current rate.""" + win = SpeedWindow(0.0) + total = 0 + for t in range(0, 60): + total += 5_000_000 + win.rate(float(t), total) + for t in range(60, 120): + total += 10_000_000 + rate = win.rate(float(t), total) + assert rate == pytest.approx(10_000_000, rel=0.01) + + +def test_first_report_is_the_since_start_average(): + """Bytes are credited in lumps; the first lump must not read as a whole window's worth.""" + win = SpeedWindow(0.0) + assert win.rate(2.5, 25_000_000) == pytest.approx(10_000_000) + + +def test_zero_span_returns_none(): + win = SpeedWindow(0.0) + assert win.rate(0.0, 1_000_000) is None + + +def test_stall_returns_none(): + """Once the seed ages out, a total that stops growing has no delta to report.""" + win = SpeedWindow(0.0) + for t in range(0, SPEED_ROLLING_WINDOW + 6): + rate = win.rate(float(t), 1_000_000) + assert rate is None + + +def test_window_drops_old_samples(): + win = SpeedWindow(0.0) + for t in range(0, 3 * SPEED_ROLLING_WINDOW): + win.rate(float(t), t * 1_000_000) + assert win.samples[0][0] >= 2 * SPEED_ROLLING_WINDOW - 1 + assert len(win.samples) == SPEED_ROLLING_WINDOW + 1 diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 3695160f..ed25dd1b 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -45,6 +45,7 @@ from unshackle.core.config import config, resolve_cdm_name, resolve_decryption from unshackle.core.console import GradientPulseBarColumn, SyncLive, console from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings from unshackle.core.credential import Credential +from unshackle.core.downloaders import format_speed, parse_speed_limit, set_speed_limit from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevine from unshackle.core.events import events from unshackle.core.music import ( @@ -691,6 +692,16 @@ class dl: help="Max workers/threads to download with per-track. Default depends on the downloader.", ) @click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.") + @click.option( + "--speed-limit", + "speed_limit", + type=str, + default=None, + metavar="SPEED", + help="Cap total download speed across all downloads combined, e.g. 500k, 5M, 1.5G or plain " + "bytes/sec (5M = 5.0 MB/s; bytes, not bits). " + "'off' disables a limit configured in the dl: config block.", + ) @click.option( "-o", "--output", @@ -1258,6 +1269,7 @@ class dl: downloads: int, worst: bool, best_available: bool, + speed_limit: Optional[str] = None, split_audio: Optional[bool] = None, merge_video: Optional[bool] = None, real_video_bitrate: bool = False, @@ -1275,6 +1287,15 @@ class dl: if skip_dl: DOWNLOAD_LICENCE_ONLY.set() + try: + speed_limit_bps = parse_speed_limit(speed_limit) + except ValueError as e: + self.log.error(str(e)) + sys.exit(1) + set_speed_limit(speed_limit_bps) + if speed_limit_bps: + self.log.info(f"Speed limit: {format_speed(speed_limit_bps)}") + if export: config.directories.exports.mkdir(parents=True, exist_ok=True) export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json" diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index 22ad10e3..65ccbde6 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -10,6 +10,7 @@ from unshackle.core.api import cors_middleware, setup_routes, setup_swagger from unshackle.core.api.compression import compression_middleware from unshackle.core.config import config from unshackle.core.constants import context_settings +from unshackle.core.downloaders import format_speed, parse_speed_limit, set_speed_limit @click.command( @@ -146,6 +147,14 @@ def serve( if remote_only: api_only = True + try: + global_speed_limit = parse_speed_limit(config.serve.get("global_speed_limit")) + except ValueError as e: + raise click.ClickException(f"serve.global_speed_limit: {e}") + if global_speed_limit: + set_speed_limit(global_speed_limit, lock=True) + log.info(f"Global speed limit: {format_speed(global_speed_limit)} (shared by all jobs)") + global_services = config.serve.get("services") if global_services: log.info(f"Global service allowlist: {', '.join(global_services)}") diff --git a/unshackle/core/downloaders/__init__.py b/unshackle/core/downloaders/__init__.py index 66d3b420..bc68b6fa 100644 --- a/unshackle/core/downloaders/__init__.py +++ b/unshackle/core/downloaders/__init__.py @@ -1,3 +1,3 @@ -from .requests import requests +from .requests import format_speed, parse_speed_limit, requests, set_speed_limit -__all__ = ("requests",) +__all__ = ("requests", "format_speed", "parse_speed_limit", "set_speed_limit") diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index 8a13b296..97aa3411 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -1,8 +1,10 @@ import math import os +import re import statistics import threading import time +from collections import deque from concurrent.futures import FIRST_COMPLETED, wait from concurrent.futures.thread import ThreadPoolExecutor from http.cookiejar import CookieJar @@ -42,6 +44,104 @@ RANGE_PARALLEL_MIN_SIZE = 64 * 1024 * 1024 RANGE_PARALLEL_PART_SIZE = 16 * 1024 * 1024 +class SpeedWindow: + """Rolling-window download rate over the last SPEED_ROLLING_WINDOW seconds.""" + + def __init__(self, start: float) -> None: + # zero-byte seed keeps early readings from exceeding the true rate + self.samples: deque[tuple[float, int]] = deque([(start, 0)]) + + def rate(self, now: float, total: int) -> Optional[float]: + self.samples.append((now, total)) + while now - self.samples[0][0] > SPEED_ROLLING_WINDOW: + self.samples.popleft() + span = now - self.samples[0][0] + delta = total - self.samples[0][1] + if span <= 0 or delta <= 0: + return None + return delta / span + + +class TokenBucket: + """Aggregate byte-rate limiter shared by every download thread.""" + + def __init__(self, rate: float) -> None: + self.rate = rate + self.capacity = rate + self.tokens = 0.0 + self.ts = time.monotonic() + self.lock = threading.Lock() + + def consume(self, n: int) -> None: + # tokens may go negative; waiting for n > capacity would deadlock + with self.lock: + now = time.monotonic() + self.tokens = min(self.capacity, self.tokens + (now - self.ts) * self.rate) + self.ts = now + self.tokens -= n + wait = -self.tokens / self.rate if self.tokens < 0 else 0.0 + if wait > 0: + DOWNLOAD_CANCELLED.wait(wait) + + +_speed_limiter: Optional[TokenBucket] = None +_speed_limit_locked = False + +SPEED_UNITS = { + "": 1, + "b": 1, + "k": 1_000, + "kb": 1_000, + "m": 1_000_000, + "mb": 1_000_000, + "g": 1_000_000_000, + "gb": 1_000_000_000, +} + + +def parse_speed_limit(value: Union[str, int, float, None]) -> Optional[float]: + """ + Parse a human-readable speed limit into bytes/sec, or None for unlimited. + + Accepts plain numbers (exact bytes/sec) or a decimal suffix with optional + /s, case-insensitive: "500k", "5M", "10MB/s", "1.5G", 5000000. Suffixes + match the displayed speeds (5M = 5.0 MB/s); values are bytes, not bits. + "", 0, "off", "none" and "unlimited" mean no limit. + """ + if value is None: + return None + if isinstance(value, (int, float)): + if not math.isfinite(value) or value < 0: + raise ValueError(f"Invalid speed limit {value!r}: must be a positive number of bytes/sec.") + return float(value) or None + text = value.strip().lower().removesuffix("/s") + if text in ("", "0", "off", "none", "unlimited"): + return None + match = re.fullmatch(r"(\d*\.?\d+)\s*([kmg]b?|b)?", text) + if not match: + raise ValueError(f"Invalid speed limit {value!r}: use e.g. 500k, 5M, 1.5G, plain bytes/sec, or 'off'.") + return float(match.group(1)) * SPEED_UNITS[match.group(2) or ""] + + +def format_speed(bytes_per_sec: float) -> str: + """Format a byte rate in the same decimal units the progress bars show.""" + return f"{filesize.decimal(int(bytes_per_sec))}/s" + + +def set_speed_limit(bytes_per_sec: Optional[float], lock: bool = False) -> None: + """ + Set (or clear) the global download speed limit shared by all threads. + + A locked limit (serve's global_speed_limit) wins: later unlocked calls, + like the per-job one in dl.result(), become no-ops for the process. + """ + global _speed_limiter, _speed_limit_locked + if _speed_limit_locked and not lock: + return + _speed_limit_locked = lock + _speed_limiter = TokenBucket(bytes_per_sec) if bytes_per_sec else None + + def _adaptive_chunk_size(content_length: int) -> int: """Pick chunk size based on content length. Benchmarked sweet spot: 512KB-4MB.""" if content_length <= 0: @@ -132,7 +232,8 @@ def _dispatch_parts( yield {"total": total_size} total_bytes = 0 - start_time = last_report = time.time() + last_report = time.time() + speed_window = SpeedWindow(last_report) completed = False worker_error = False @@ -153,7 +254,9 @@ def _dispatch_parts( now = time.time() if now - last_report > 0.5 and total_bytes > 0: - yield {"downloaded": f"{filesize.decimal(math.ceil(total_bytes / (now - start_time)))}/s"} + rate = speed_window.rate(now, total_bytes) + if rate: + yield {"downloaded": f"{filesize.decimal(math.ceil(rate))}/s"} last_report = now done, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) @@ -300,7 +403,10 @@ def download( except ValueError: content_length = 0 + limiter = _speed_limiter chunk_size = _adaptive_chunk_size(content_length) + if limiter: + chunk_size = min(chunk_size, max(8192, int(limiter.rate / 4))) total_size = (resume_offset + content_length) if resumed and content_length > 0 else content_length if not segmented and not part_mode: @@ -350,6 +456,8 @@ def download( except Exception: pass return + if limiter is not None: + limiter.consume(len(chunk)) _write(chunk) download_size = len(chunk) written += download_size @@ -543,7 +651,7 @@ def requests( url_item = urls[0] try: ranged_used = False - if max_workers > 1: + if max_workers > 1 and _speed_limiter is None: total_size, supports_ranges = _probe_ranged(url_item["url"], session) if supports_ranges and total_size >= RANGE_PARALLEL_MIN_SIZE: try: @@ -581,6 +689,7 @@ def requests( total_bytes = 0 start_time = time.time() last_speed_report = start_time + speed_window = SpeedWindow(start_time) last_hedge_check = 0.0 hedge_median = 0.0 # cached; recomputed only when seg_durations grows hedge_median_len = 0 @@ -664,10 +773,9 @@ def requests( # Yield speed every 0.5s (throttled to avoid spamming Rich) now = time.time() if now - last_speed_report > 0.5 and total_bytes > 0: - elapsed = now - start_time - if elapsed > 0: - download_speed = math.ceil(total_bytes / elapsed) - yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + rate = speed_window.rate(now, total_bytes) + if rate: + yield dict(downloaded=f"{filesize.decimal(math.ceil(rate))}/s") last_speed_report = now # hedge stuck segments once spare workers exist (pending < max_workers); @@ -675,6 +783,7 @@ def requests( if ( len(pending) < max_workers and seg_durations + and _speed_limiter is None and now - last_hedge_check > 0.5 and not DOWNLOAD_CANCELLED.is_set() ): @@ -768,4 +877,4 @@ def requests( ) -__all__ = ("requests",) +__all__ = ("requests", "format_speed", "parse_speed_limit", "set_speed_limit") From bf85c948b4755bbf919944057d38546df9027747 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Fri, 31 Jul 2026 13:22:18 -0600 Subject: [PATCH 25/40] fix(dl): honour --no-proxy-download for attachments --- docs/guide/cli-reference.md | 2 +- docs/guide/downloading.md | 2 +- docs/guide/proxies-and-vpn.md | 4 +- tests/tracks/test_attachment_download.py | 59 +++++++++++++++++++++ unshackle/commands/dl.py | 8 ++- unshackle/core/api/routes.py | 2 +- unshackle/core/tracks/attachment.py | 66 ++++++++++++++---------- 7 files changed, 109 insertions(+), 34 deletions(-) create mode 100644 tests/tracks/test_attachment_download.py diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index ce4d53dd..c5c995af 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -162,7 +162,7 @@ Keep only certain track types, or skip certain track types. Attachments are alwa |---|---| | `--proxy` | Proxy URI, a 2-letter country code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`). | | `--no-proxy` | Force-disable all proxy use. | -| `--no-proxy-download` | Bypass the proxy for **segment downloads only** (manifest, licence, and auth stay proxied). | +| `--no-proxy-download` | Bypass the proxy for **all downloads** (manifest, licence, and auth stay proxied). | | `--remote` | Use a remote unshackle server. | | `--server` | Name a remote server from the `remote_services` config. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 1086e706..6606a9a3 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -457,7 +457,7 @@ unshackle dl --proxy 'http://user:pass@host:8080' EXAMPLE 81234567 Two related flags: - `--no-proxy`: force-disable all proxy use for this run. -- `--no-proxy-download`: bypass the proxy for **segment downloads only**. The manifest, +- `--no-proxy-download`: bypass the proxy for **all downloads**. The manifest, license, and authentication requests still go through the proxy. This is useful when the proxy is only needed to satisfy geo-checks, not to move the bulk of the data. diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md index 443e29e4..d2020503 100644 --- a/docs/guide/proxies-and-vpn.md +++ b/docs/guide/proxies-and-vpn.md @@ -28,7 +28,7 @@ Proxying is controlled at download time by three options on the `dl` command: |---|---| | `--proxy` | The proxy to use. Either an explicit URI, or a query that unshackle resolves against your configured providers. | | `--no-proxy` | Force **all** proxy use off. No providers are initialised and no proxy query is resolved. | -| `--no-proxy-download` | Use the proxy for the manifest, licence, and authentication, but bypass it for the **segment downloads** themselves. | +| `--no-proxy-download` | Use the proxy for the manifest, licence, and authentication, but bypass it for the **downloads** themselves. | ```shell title="Explicit proxy URI" unshackle dl --proxy http://user:pass@1.2.3.4:8080 EXAMPLE 81234567 @@ -52,7 +52,7 @@ unshackle dl --proxy nordvpn:us EXAMPLE 81234567 Segment downloads are usually the largest share of traffic, and a residential-grade VPN proxy is often the slowest link in the chain. `--no-proxy-download` lets you keep the proxy where it matters (passing the geo-check for the manifest and the licence) while -pulling the actual audio and video segments over your normal connection for full speed. +pulling the files themselves over your normal connection at full speed. ```shell title="Proxy the manifest and licence, download segments direct" unshackle dl --proxy gb --no-proxy-download EXAMPLE 10a1234 diff --git a/tests/tracks/test_attachment_download.py b/tests/tracks/test_attachment_download.py new file mode 100644 index 00000000..caab4ea7 --- /dev/null +++ b/tests/tracks/test_attachment_download.py @@ -0,0 +1,59 @@ +"""URL-backed attachments defer their fetch to download(), which honors no_proxy_download.""" + +from __future__ import annotations + +import requests + +import unshackle.core.tracks.track as track_module +from unshackle.core.config import config +from unshackle.core.tracks.attachment import Attachment + +URL = "https://cdn.example.com/images/thumb.jpg" + + +class FakeResponse: + def raise_for_status(self) -> None: + pass + + def iter_content(self, chunk_size: int): + yield b"jpegdata" + + +class FakeSession(requests.Session): + def __init__(self, proxies: dict | None = None) -> None: + super().__init__() + self.proxies = proxies or {} + self.got: list[str] = [] + + def get(self, url, **kwargs): # type: ignore[override] + self.got.append(url) + return FakeResponse() + + +def test_url_attachment_defers_download() -> None: + session = FakeSession() + attachment = Attachment.from_url(URL, name="thumbnail", session=session) + assert attachment.path is None + assert not session.got + assert attachment.mime_type == "image/jpeg" + + +def test_download_without_flag_uses_proxied_session(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(config.directories, "temp", tmp_path) + session = FakeSession(proxies={"https": "http://proxy:8080"}) + attachment = Attachment.from_url(URL, name="thumbnail", session=session) + attachment.download(session) + assert session.got == [URL] + assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata" + + +def test_download_with_flag_bypasses_proxy(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(config.directories, "temp", tmp_path) + direct = FakeSession() + monkeypatch.setattr(track_module, "direct_session", lambda session: direct) + proxied = FakeSession(proxies={"https": "http://proxy:8080"}) + attachment = Attachment.from_url(URL, name="thumbnail", session=proxied) + attachment.download(proxied, no_proxy_download=True) + assert not proxied.got + assert direct.got == [URL] + assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata" diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index ed25dd1b..8ef9832e 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -678,7 +678,7 @@ class dl: "--no-proxy-download", is_flag=True, default=False, - help="Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy.", + help="Bypass proxy for all downloads. Manifest, license, and auth still use proxy.", ) @click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.") @click.option( @@ -3028,6 +3028,12 @@ class dl: on_subtitle_skipped=on_subtitle_skipped, ) + for attachment in title.tracks.attachments: + attachment.download( + attachment.session or service.session, + no_proxy_download=no_proxy_download, + ) + if ( len(self.skipped_subtitles) > skipped_before and not title.tracks.videos diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 7605339b..8f49811d 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -747,7 +747,7 @@ async def download(request: web.Request) -> web.Response: description: Force disable all proxy use (default - false) no_proxy_download: type: boolean - description: Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy (default - false) + description: Bypass proxy for all downloads. Manifest, license, and auth still use proxy (default - false) tag: type: string description: Set the group tag to be used (default - None) diff --git a/unshackle/core/tracks/attachment.py b/unshackle/core/tracks/attachment.py index d9e427dd..b970768c 100644 --- a/unshackle/core/tracks/attachment.py +++ b/unshackle/core/tracks/attachment.py @@ -12,6 +12,9 @@ import requests from unshackle.core.config import config from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY +from unshackle.core.session import RnetSession + +AnySession = Union[requests.Session, RnetSession] class Attachment: @@ -22,13 +25,13 @@ class Attachment: name: Optional[str] = None, mime_type: Optional[str] = None, description: Optional[str] = None, - session: Optional[requests.Session] = None, + session: Optional[AnySession] = None, ): """ Create a new Attachment. If providing a path, the file must already exist. - If providing a URL, the file will be downloaded to the temp directory. + If providing a URL, download() fetches the file during the download phase. Either path or url must be provided. If name is not provided it will use the file name (without extension). @@ -46,12 +49,13 @@ class Attachment: raise ValueError("Either path or url must be provided.") self.url = url + self.session = session + self.file_name: Optional[str] = None if url: if not isinstance(url, str): raise ValueError("The attachment URL must be a string.") - # If a URL is provided, download the file to the temp directory parsed_url = urlparse(url) file_name = os.path.basename(parsed_url.path) or "attachment" @@ -60,30 +64,7 @@ class Attachment: safe_name = re.sub(r'[<>:"/\\|?*]', "", name).replace(" ", "_") file_name = f"{safe_name}{os.path.splitext(file_name)[1]}" - download_path = config.directories.temp / file_name - - # Download the file unless we're in license-only mode - if DOWNLOAD_LICENCE_ONLY.is_set(): - path = None - else: - try: - if session is None: - with requests.Session() as session: - response = session.get(url, stream=True) - response.raise_for_status() - else: - response = session.get(url, stream=True) - response.raise_for_status() - config.directories.temp.mkdir(parents=True, exist_ok=True) - download_path.parent.mkdir(parents=True, exist_ok=True) - - with open(download_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - - path = download_path - except Exception as e: - raise ValueError(f"Failed to download attachment from URL: {e}") + self.file_name = file_name if path is not None and not isinstance(path, (str, Path)): raise ValueError(f"Invalid attachment path type: expected str or Path, got {type(path).__name__}.") @@ -117,6 +98,35 @@ class Attachment: self.mime_type = mime_type self.description = description + def download( + self, + session: Optional[AnySession] = None, + *, + no_proxy_download: bool = False, + ) -> None: + """Download a URL-backed attachment to the temp directory.""" + if self.path is not None or not self.url or DOWNLOAD_LICENCE_ONLY.is_set(): + return + + from unshackle.core.tracks.track import direct_session + + session = session or self.session or requests.Session() + if no_proxy_download and any(session.proxies.values()): + session = direct_session(session) + + download_path = config.directories.temp / (self.file_name or "attachment") + try: + response = session.get(self.url, stream=True) + response.raise_for_status() + download_path.parent.mkdir(parents=True, exist_ok=True) + with open(download_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + except Exception as e: + raise ValueError(f"Failed to download attachment from URL: {e}") + + self.path = download_path + def __repr__(self) -> str: return "{name}({items})".format( name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) @@ -152,7 +162,7 @@ class Attachment: name: Optional[str] = None, mime_type: Optional[str] = None, description: Optional[str] = None, - session: Optional[requests.Session] = None, + session: Optional[AnySession] = None, ) -> "Attachment": """ Create an attachment from a URL. From 8baf064750510a01d9c16bf3b8b508ae051ffc84 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Sun, 2 Aug 2026 10:45:28 -0600 Subject: [PATCH 26/40] feat(tracks): add subtitle.group_by to keep a language with its variants sort_subtitles applied the language sort before the type sort. Python sorts are stable, so the last pass wins and type became the major key: every forced track, then every normal, then every SDH, each block ordered by language. In a title with many languages this put a track far from its own SDH version, for example Finnish at track 5 and Finnish SDH at track 42. Add a group_by parameter that selects the major key. "type" is the default and keeps the current order byte for byte. "language" swaps the two passes so each language stays next to its own variants. Read it from the new subtitle.group_by config key. Also correct the docs: type_priority was described as sorting the variants within each language, which was never true. It ranks the types across all languages. --- docs/guide/subtitles.md | 13 ++++-- docs/reference/configuration.md | 30 ++++++++++--- tests/tracks/test_subtitle_sort.py | 69 ++++++++++++++++++++++++++++++ unshackle/commands/dl.py | 6 ++- unshackle/core/tracks/tracks.py | 47 +++++++++++++------- 5 files changed, 141 insertions(+), 24 deletions(-) diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index f7c96107..8823be19 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -134,9 +134,16 @@ flags drive selection, sort order, output filenames, and the Matroska track flag often uppercase with `>>>` speaker markers and sound cues. !!! note "SDH and CC are treated together for sorting and stripping" - Within a language, unshackle orders subtitles from fewest to most captions: - **Forced → Normal → SDH/CC**. For the purposes of hearing-impaired handling, SDH and - CC tracks are grouped as the "most captions" variant. + unshackle orders subtitles from fewest to most captions: **Forced → Normal → SDH/CC**. + For the purposes of hearing-impaired handling, SDH and CC tracks are grouped as the + "most captions" variant. + + By default this type order is the primary sort, so all forced tracks come first, then + all normal tracks, then all SDH/CC tracks, each block sorted by language. To keep each + language next to its own variants instead, set + [`subtitle.group_by`](../reference/configuration.md#subtitle) to `language`. Use + [`subtitle.type_priority`](../reference/configuration.md#subtitle) to change the + Forced → Normal → SDH/CC order itself. A single track cannot be both CC and SDH, and a forced track cannot also be flagged CC or SDH. These combinations are rejected as invalid. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 45a00d26..2eab1d3b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -375,7 +375,8 @@ written as sidecar files. See [Subtitles](../guide/subtitles.md) for the full gu | `preferred_conversion_method` | str | *(unset)* | Preferred converter when several are available. | | `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files; `both` writes sidecars and still muxes. | | `sidecar_format` | str | `"srt"` | Format for sidecar files when `output_mode: sidecar`. | -| `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH) that sorts the variants within each language. Types you leave out fall to the end. | +| `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH). Types you leave out fall to the end. | +| `group_by` | str | `"type"` | Which key groups the subtitle tracks. `type` puts all forced tracks together, then all normal, then all SDH. `language` keeps each language next to its own variants. | ```yaml subtitle: @@ -384,11 +385,30 @@ subtitle: sidecar_format: srt ``` +!!! note "`group_by` puts a language next to its own SDH track" + The default `type` order groups by subtitle type first, so every normal track comes + before every SDH track. In a title with many languages, a track ends up far from its + own SDH version, for example Finnish at track 5 and Finnish SDH at track 42. Set + `group_by: language` to keep them together: + + ```yaml + subtitle: + group_by: language + ``` + + | `group_by` | Result | + |---|---| + | `type` | Danish, Finnish, French, … Danish SDH, Finnish SDH, French SDH | + | `language` | Danish, Danish SDH, Finnish, Finnish SDH, French, French SDH | + + `type_priority` still sets the order of the variants, so with `group_by: language` it + controls the order inside each language. + !!! note "`type_priority` also picks the default subtitle" - `type_priority` sorts the tracks within each language, and at mux time the first track - in the default subtitle language gets the `default` flag. The built-in order is `forced`, - `normal`, `sdh`, which means a forced track becomes default whenever one exists. If you - want the full dialogue track as default instead, set: + At mux time the first track in the default subtitle language gets the `default` flag. + The built-in order is `forced`, `normal`, `sdh`, which means a forced track becomes + default whenever one exists. If you want the full dialogue track as default instead, + set: ```yaml subtitle: diff --git a/tests/tracks/test_subtitle_sort.py b/tests/tracks/test_subtitle_sort.py index 5251603e..d114eae0 100644 --- a/tests/tracks/test_subtitle_sort.py +++ b/tests/tracks/test_subtitle_sort.py @@ -66,3 +66,72 @@ def test_cc_counts_as_sdh() -> None: t.add([sub("en", cc=True), sub("en")]) t.sort_subtitles(by_language=["en"], type_priority=["normal", "sdh"]) assert shape(t) == [("en", "normal"), ("en", "sdh")] + + +@pytest.fixture +def unprioritised() -> Tracks: + """Languages that no by_language argument floats to the top.""" + t = Tracks() + t.add([sub("fi", sdh=True), sub("fi"), sub("de", sdh=True), sub("de")]) + return t + + +def test_type_grouping_separates_a_language_from_its_sdh(unprioritised: Tracks) -> None: + unprioritised.sort_subtitles() + assert shape(unprioritised) == [ + ("de", "normal"), + ("fi", "normal"), + ("de", "sdh"), + ("fi", "sdh"), + ] + + +def test_language_grouping_keeps_a_language_with_its_sdh(unprioritised: Tracks) -> None: + unprioritised.sort_subtitles(group_by="language") + assert shape(unprioritised) == [ + ("de", "normal"), + ("de", "sdh"), + ("fi", "normal"), + ("fi", "sdh"), + ] + + +def test_language_grouping_still_honours_type_priority_and_sections(tracks: Tracks) -> None: + # de sits outside by_language, where type-major sorting would interleave it with en + tracks.add([sub("de"), sub("de", sdh=True)]) + tracks.sort_subtitles(by_language=["it"], type_priority=["normal", "sdh", "forced"], group_by="language") + assert shape(tracks) == [ + ("it", "normal"), + ("it", "forced"), + ("de", "normal"), + ("de", "sdh"), + ("en", "normal"), + ("en", "sdh"), + ("en", "forced"), + ] + + +def test_language_grouping_keeps_regional_variants_adjacent_through_sections() -> None: + # "en" floats en-GB and en-US as one block, so each exact tag keeps its variants + t = Tracks() + t.add( + [ + sub("en-US", sdh=True), + sub("fi"), + sub("en-GB"), + sub("en-US"), + sub("fi", sdh=True), + sub("en-GB", forced=True), + sub("en-GB", sdh=True), + ] + ) + t.sort_subtitles(by_language=["en"], group_by="language") + assert shape(t) == [ + ("en-GB", "forced"), + ("en-GB", "normal"), + ("en-GB", "sdh"), + ("en-US", "normal"), + ("en-US", "sdh"), + ("fi", "normal"), + ("fi", "sdh"), + ] diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 8ef9832e..4dd85332 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -2368,7 +2368,11 @@ class dl: by_language=processed_audio_sort_lang, codec_priority=config.audio.get("codec_priority"), ) - title.tracks.sort_subtitles(by_language=s_lang, type_priority=config.subtitle.get("type_priority")) + title.tracks.sort_subtitles( + by_language=s_lang, + type_priority=config.subtitle.get("type_priority"), + group_by=config.subtitle.get("group_by"), + ) if list_: available_tracks, _ = title.tracks.tree() diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 5c4207d8..02bb6d9d 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -289,6 +289,7 @@ class Tracks: self, by_language: Optional[Sequence[Union[str, Language]]] = None, type_priority: Optional[Sequence[str]] = None, + group_by: Optional[str] = None, ) -> None: """ Sort subtitle tracks by various track attributes to a common P2P standard. @@ -299,30 +300,46 @@ class Tracks: - then rest ascending alphabetically after the prioritized groups (Each section ascending alphabetically, but separated) - Language Group Order: + Type Order: - Forced - Normal - Hard of Hearing (SDH/CC) (Least to most captions expected in the subtitle) - type_priority overrides the Language Group Order with an explicit ranking of - "forced", "normal", and "sdh" (cc counts as sdh); unlisted types fall to the end. - The first track after sorting receives the default flag at mux time, so this also - controls which subtitle type becomes default. + type_priority overrides the Type Order with an explicit ranking of "forced", + "normal", and "sdh" (cc counts as sdh); unlisted types fall to the end. + + group_by sets the major key. "type" (default) keeps every forced track together, + then every normal, then every SDH, each block ascending by language. "language" + groups by language instead, so Finnish sits next to Finnish SDH, with the Type + Order applied inside each language. """ if not self.subtitles: return - # language groups - self.subtitles.sort(key=lambda x: str(x.language)) - if type_priority: - rank = {str(t).lower(): i for i, t in enumerate(type_priority)} - default_rank = len(rank) - self.subtitles.sort( - key=lambda x: rank.get("forced" if x.forced else "sdh" if (x.sdh or x.cc) else "normal", default_rank) - ) + + def by_type() -> None: + if type_priority: + rank = {str(t).lower(): i for i, t in enumerate(type_priority)} + default_rank = len(rank) + self.subtitles.sort( + key=lambda x: rank.get( + "forced" if x.forced else "sdh" if (x.sdh or x.cc) else "normal", default_rank + ) + ) + else: + self.subtitles.sort(key=lambda x: x.sdh or x.cc) + self.subtitles.sort(key=lambda x: x.forced, reverse=True) + + def by_lang() -> None: + self.subtitles.sort(key=lambda x: str(x.language)) + + # stable sorts, so the last pass is the major key + if str(group_by or "type").lower() == "language": + by_type() + by_lang() else: - self.subtitles.sort(key=lambda x: x.sdh or x.cc) - self.subtitles.sort(key=lambda x: x.forced, reverse=True) + by_lang() + by_type() # sections for language in reversed(by_language or []): if str(language) == "all": From 4ac34069ecad81977b4ee532aeb75c2a56982d65 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Sun, 2 Aug 2026 10:48:32 -0600 Subject: [PATCH 27/40] feat(providers): add TheTVDB v4 as a metadata provider TVDB covers TV far better than the existing sources, so add it as a provider and let users decide where it sits in the lookup chain. - new TVDB v4 provider: search, get_by_id, get_external_ids and find_by_imdb_id, with a process-level token cache and 401 re-auth - metadata_providers config both orders and filters the providers used, so TVDB can be primary rather than a TMDB fallback. Cross-validation keeps its own fixed trust ranking, unchanged by search order - --tvdb-order renumbers episodes into a TVDB season order. A service does not always use aired order, so the source order is detected rather than assumed, and the renumber is refused when an order that omits episodes would give two of them the same filename - --tvdb sets the series ID directly, skipping the lookup - the Matroska TVDB2 tag now populates on every lookup path, including --imdb, which previously produced no ID tags when IMDxAPI was off Search matches on year as well as title so an identically named remake cannot win on title alone. --- tests/core/test_tvdb.py | 173 +++++++++++++++ unshackle/commands/dl.py | 104 ++++++++- unshackle/core/api/download_manager.py | 4 + unshackle/core/api/handlers.py | 2 + unshackle/core/api/routes.py | 7 + unshackle/core/config.py | 4 + unshackle/core/providers/__init__.py | 83 +++++-- unshackle/core/providers/tvdb.py | 286 +++++++++++++++++++++++++ unshackle/core/utils/tags.py | 30 ++- 9 files changed, 666 insertions(+), 27 deletions(-) create mode 100644 tests/core/test_tvdb.py create mode 100644 unshackle/core/providers/tvdb.py diff --git a/tests/core/test_tvdb.py b/tests/core/test_tvdb.py new file mode 100644 index 00000000..03f82573 --- /dev/null +++ b/tests/core/test_tvdb.py @@ -0,0 +1,173 @@ +"""TVDB provider ordering and season-order remapping.""" + +from __future__ import annotations + +import pytest + +from unshackle.core import providers +from unshackle.core.config import config +from unshackle.core.providers.tvdb import TVDBProvider, _ids_from_remote, _pick_match +from unshackle.core.utils.tags import _build_tags_from_ids + +REMOTE_IDS = [ + {"id": "tt0149460", "type": 2, "sourceName": "IMDB"}, + {"id": "http://example.com", "type": 4, "sourceName": "Official Website"}, + {"id": "615", "type": 12, "sourceName": "TheMovieDB.com"}, +] + +# Futurama: aired S1 stops at E9, the four held-back episodes open aired S2. +# DVD order folds them back into S1 as E10-E13. +AIRED = [ + {"id": 1, "seasonNumber": 1, "number": 9, "name": "Hell Is Other Robots"}, + {"id": 2, "seasonNumber": 2, "number": 1, "name": "A Flight to Remember"}, + {"id": 3, "seasonNumber": 2, "number": 2, "name": "Mars University"}, + {"id": 99, "seasonNumber": 2, "number": 3, "name": "Only In Aired"}, +] +DVD = [ + {"id": 1, "seasonNumber": 1, "number": 9, "name": "Hell Is Other Robots"}, + {"id": 2, "seasonNumber": 1, "number": 10, "name": "A Flight to Remember"}, + {"id": 3, "seasonNumber": 1, "number": 11, "name": "Mars University"}, +] + + +def test_ids_from_remote() -> None: + ext = _ids_from_remote(REMOTE_IDS, 73871) + assert ext.imdb_id == "tt0149460" + assert ext.tmdb_id == 615 + assert ext.tvdb_id == 73871 + + +def _stub(p: TVDBProvider, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(p, "get_episodes", lambda _id, order: {"official": AIRED, "dvd": DVD}.get(order, [])) + + +def test_order_map_joins_on_episode_id(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + _stub(p, monkeypatch) + + mapping = p.get_order_map(73871, "dvd") + assert mapping[(1, 9)] == (1, 9, "Hell Is Other Robots") + assert mapping[(2, 1)] == (1, 10, "A Flight to Remember") + assert mapping[(2, 2)] == (1, 11, "Mars University") + # an episode absent from the target order is left out, so it keeps its own numbering + assert (2, 3) not in mapping + + +def test_order_map_reverses_with_source_order(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + _stub(p, monkeypatch) + + mapping = p.get_order_map(73871, "official", source_order="dvd") + assert mapping[(1, 10)] == (2, 1, "A Flight to Remember") + + +def test_order_map_is_noop_when_orders_match() -> None: + assert TVDBProvider().get_order_map(73871, "official") == {} + + +def test_order_map_empty_when_order_missing(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + _stub(p, monkeypatch) + assert p.get_order_map(73871, "alternate") == {} + + +def test_detect_order_picks_the_order_the_numbering_matches(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + _stub(p, monkeypatch) + # a service may number a series in a non-aired order, so detection must not assume aired + assert p.detect_order(73871, [(1, 10), (1, 11)]) == "dvd" + assert p.detect_order(73871, [(2, 1), (2, 2), (2, 3)]) == "official" + + +def test_detect_order_falls_back_to_aired(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + monkeypatch.setattr(p, "get_episodes", lambda _id, order: []) + assert p.detect_order(73871, [(1, 1)]) == "official" + + +# TVDB holds both Battlestar Galactica series; the 2003 revival is filed under year 2005 +BSG = [ + {"name": "Battlestar Galactica", "year": "1978", "tvdb_id": "71173"}, + {"name": "Battlestar Galactica (2003)", "year": "2005", "tvdb_id": "73545"}, +] + + +def test_pick_match_prefers_the_right_era() -> None: + # exact title match on the 1978 series must not beat the era the caller asked for + best, _, _ = _pick_match(BSG, "Battlestar Galactica", 2004) + assert best["tvdb_id"] == "73545" + best, _, _ = _pick_match(BSG, "Battlestar Galactica", 1978) + assert best["tvdb_id"] == "71173" + + +def test_pick_match_without_a_year_scores_on_title_alone() -> None: + best, _, _ = _pick_match(BSG, "Battlestar Galactica", None) + assert best["tvdb_id"] == "71173" + + +def test_pick_match_returns_nothing_when_no_year_fits() -> None: + assert _pick_match(BSG, "Battlestar Galactica", 1995)[0] is None + + +def test_tvdb2_tag_follows_the_matroska_spec() -> None: + # TVDB2 IDs are prefixed by entity type: matroska.org/technical/tagging.html + from unshackle.core.providers import ExternalIds + + assert _build_tags_from_ids(ExternalIds(tvdb_id=73871), "tv")["TVDB2"] == "series/73871" + assert _build_tags_from_ids(ExternalIds(tvdb_id=113), "movie")["TVDB2"] == "movies/113" + + +def test_provider_order_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(config, "metadata_providers", []) + assert [c.NAME for c in providers.provider_order()] == list(providers.DEFAULT_ORDER) + + +def test_provider_order_is_configurable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(config, "metadata_providers", ["TVDB", "tmdb", "nope", "tvdb"]) + # unknown entries dropped, duplicates collapsed, case ignored, order kept + assert [c.NAME for c in providers.provider_order()] == ["tvdb", "tmdb"] + + +def test_enrichment_authority_is_fixed(monkeypatch: pytest.MonkeyPatch) -> None: + # search order must not demote TMDB below SIMKL for cross-validation tie-breaks + monkeypatch.setattr(config, "metadata_providers", []) + assert providers._enrichment_providers() == ["tmdb", "simkl", "tvdb"] + # `metadata_providers` filters the set but does not re-rank the trust order + monkeypatch.setattr(config, "metadata_providers", ["tvdb", "simkl"]) + assert providers._enrichment_providers() == ["simkl", "tvdb"] + + +def test_cached_tvdb_result_keeps_enriched_ids() -> None: + cached = { + "tvdb_id": "73871", + "name": "Futurama", + "year": "1999", + "_enriched_ids": {"imdb_id": "tt0149460", "tmdb_id": 615}, + } + result = providers._cached_to_result(cached, "tvdb", "tv") + assert result is not None + assert result.external_ids.tvdb_id == 73871 + assert result.external_ids.imdb_id == "tt0149460" + assert result.external_ids.tmdb_id == 615 + # remote_ids stay authoritative over the enriched copies + cached["remote_ids"] = REMOTE_IDS + cached["_enriched_ids"] = {"imdb_id": "tt9999999", "tmdb_id": 1} + result = providers._cached_to_result(cached, "tvdb", "tv") + assert result.external_ids.imdb_id == "tt0149460" + assert result.external_ids.tmdb_id == 615 + + +def test_get_episodes_fails_closed_on_a_truncated_listing(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + pages: dict[int, object] = {0: {"episodes": [{"id": n} for n in range(500)]}, 1: None} + monkeypatch.setattr(p, "_get", lambda _path, params: pages[params["page"]]) + # page 1 fails mid-listing: a partial list would renumber wrongly + assert p.get_episodes(73871, "official") == [] + assert p._episodes == {} + + +def test_get_episodes_does_not_cache_an_empty_listing(monkeypatch: pytest.MonkeyPatch) -> None: + p = TVDBProvider() + monkeypatch.setattr(p, "_get", lambda _path, params: None) + assert p.get_episodes(73871, "official") == [] + assert p._episodes == {} diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 8ef9832e..685ac411 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -11,6 +11,7 @@ import shutil import subprocess import sys import time +from collections import Counter from collections.abc import Iterable from concurrent import futures from concurrent.futures import ThreadPoolExecutor @@ -58,6 +59,7 @@ from unshackle.core.music import ( write_music_manifest, write_music_metadata, ) +from unshackle.core.providers.tvdb import SEASON_TYPES from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN from unshackle.core.service import Service from unshackle.core.services import Services @@ -587,6 +589,20 @@ class dl: default=None, help="Use this TMDB ID for tagging instead of automatic lookup.", ) + @click.option( + "--tvdb", + "tvdb_id", + type=int, + default=None, + help="Use this TVDB ID for --tvdb-order instead of looking the series up automatically.", + ) + @click.option( + "--tvdb-order", + type=click.Choice(SEASON_TYPES, case_sensitive=False), + default=None, + help="Renumber episodes to a TVDB season order. Services normally use 'official' (aired) order; " + "a series' Streaming Order is 'alternate'. Defaults to the tvdb_order config option.", + ) @click.option( "--animeapi", "animeapi_id", @@ -760,6 +776,8 @@ class dl: tag: Optional[str] = None, tmdb_id: Optional[int] = None, imdb_id: Optional[str] = None, + tvdb_id: Optional[int] = None, + tvdb_order: Optional[str] = None, animeapi_id: Optional[str] = None, enrich: bool = False, output_dir: Optional[Path] = None, @@ -798,6 +816,8 @@ class dl: tag = ctx.params.get("tag", tag) tmdb_id = ctx.params.get("tmdb_id", tmdb_id) imdb_id = ctx.params.get("imdb_id", imdb_id) + tvdb_id = ctx.params.get("tvdb_id", tvdb_id) + tvdb_order = ctx.params.get("tvdb_order", tvdb_order) animeapi_id = ctx.params.get("animeapi_id", animeapi_id) enrich = ctx.params.get("enrich", enrich) output_dir = ctx.params.get("output_dir", output_dir) @@ -806,6 +826,10 @@ class dl: self.proxy_requested = bool(proxy) self.tmdb_id = tmdb_id self.imdb_id = imdb_id + self.tvdb_id = tvdb_id + self.tvdb_order = (tvdb_order or config.tvdb_order or "").lower() + if self.tvdb_order and self.tvdb_order not in SEASON_TYPES: + raise click.UsageError(f"tvdb_order must be one of {', '.join(SEASON_TYPES)}, not {self.tvdb_order!r}") self.enrich = enrich self.animeapi_title: Optional[str] = None self.output_dir = output_dir @@ -819,6 +843,8 @@ class dl: self.tmdb_id = anime_ids.tmdb_id if not self.imdb_id and anime_ids.imdb_id: self.imdb_id = anime_ids.imdb_id + if not self.tvdb_id and anime_ids.tvdb_id: + self.tvdb_id = anime_ids.tvdb_id if self.enrich and not (self.tmdb_id or self.imdb_id or self.animeapi_title): raise click.UsageError("--enrich requires --tmdb, --imdb, or --animeapi to provide a metadata source.") @@ -1501,6 +1527,9 @@ class dl: if enrich_year and not titles.year: titles.year = enrich_year + if self.tvdb_order and isinstance(titles, Series): + titles = self.apply_tvdb_order(titles, title_cacher, cache_title_id, cache_region, cache_account_hash) + music_mode = isinstance(titles, Music) music_collection_mode = ( isinstance(titles, list) and bool(titles) and all(isinstance(title, Music) for title in titles) @@ -3622,7 +3651,7 @@ class dl: shutil.move(muxed_path, final_path) used_final_paths.add(final_path) self.completed_files.append(final_path) - tags.tag_file(final_path, title, self.tmdb_id, self.imdb_id) + tags.tag_file(final_path, title, self.tmdb_id, self.imdb_id, self.tvdb_id) title_dl_time = time_elapsed_since(dl_start_time) downloaded_label = "Track" if music_mode and isinstance(title, Song) else "Title" @@ -3648,6 +3677,79 @@ class dl: console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) + def apply_tvdb_order( + self, + titles: Series, + title_cacher: Any = None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, + ) -> Series: + """Renumber episodes from the service's own numbering into a TVDB season order.""" + tvdb = providers.get_provider("tvdb") + if not tvdb: + self.log.warning("--tvdb-order needs a tvdb_api_key in your config, skipping renumber.") + return titles + + tvdb_id = self.tvdb_id + if not tvdb_id: + show = titles[0] + result = providers.search_metadata( + show.title, show.year, "tv", title_cacher, cache_title_id, cache_region, cache_account_hash + ) + tvdb_id = result.external_ids.tvdb_id if result else None + if not tvdb_id: + self.log.warning("Could not resolve a TVDB ID for %r, skipping %s renumber.", str(titles), self.tvdb_order) + return titles + self.tvdb_id = tvdb_id # keep the mux tag on the series the renumbering used + + keys = [(t.season, t.number) for t in titles] + source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined] + if source_order == self.tvdb_order: + self.log.info("%s already lists this series in TVDB %s order.", self.service, self.tvdb_order) + return titles + + mapping = tvdb.get_order_map(tvdb_id, self.tvdb_order, source_order) # type: ignore[attr-defined] + if not mapping: + return titles + + # an unmapped episode keeps its own numbering and can land on a remapped episode's slot + renumbered = [mapping.get(key, (*key, None))[:2] for key in keys] + unmapped = sum(1 for key in keys if key not in mapping) + collisions = [slot for slot, count in Counter(renumbered).items() if count > 1] + if collisions: + self.log.error( + "TVDB's %s order does not cover %d of this series' episodes, and renumbering the rest " + "would give %d season/episode slot(s) two episodes each. Keeping the %s numbering " + "%s uses. Pick an order that covers the whole series.", + self.tvdb_order, + unmapped, + len(collisions), + source_order, + self.service, + ) + return titles + + for title in titles: + entry = mapping.get((title.season, title.number)) + if not entry: + continue + title.season, title.number, tvdb_name = entry + if tvdb_name and not title.name: + title.name = tvdb_name + + if unmapped: + self.log.warning( + "%d episode(s) are not in TVDB's %s order and keep their %s numbering.", + unmapped, + self.tvdb_order, + source_order, + ) + self.log.info("Renumbered episodes from TVDB %s to %s order (ID %s).", source_order, self.tvdb_order, tvdb_id) + + # Series is a SortedKeyList; renumbering in place does not re-sort it + return type(titles)(list(titles)) + @staticmethod def title_to_meta(title: Title_T) -> dict[str, Any]: """Capture the title fields needed to rebuild a Title for output naming on import.""" diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 3e7d3bdf..180ed7e5 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -417,6 +417,8 @@ def _perform_download( "tag": params.get("tag"), "tmdb_id": params.get("tmdb_id"), "imdb_id": params.get("imdb_id"), + "tvdb_id": params.get("tvdb_id"), + "tvdb_order": params.get("tvdb_order"), "animeapi_id": params.get("animeapi_id"), "enrich": params.get("enrich", False), "output_dir": Path(params["output_dir"]) if params.get("output_dir") else None, @@ -447,6 +449,8 @@ def _perform_download( tag=params.get("tag"), tmdb_id=params.get("tmdb_id"), imdb_id=params.get("imdb_id"), + tvdb_id=params.get("tvdb_id"), + tvdb_order=params.get("tvdb_order"), animeapi_id=params.get("animeapi_id"), enrich=params.get("enrich", False), output_dir=Path(params["output_dir"]) if params.get("output_dir") else None, diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 47afc898..f22d9065 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -74,6 +74,8 @@ DEFAULT_DOWNLOAD_PARAMS = { "tag": None, "tmdb_id": None, "imdb_id": None, + "tvdb_id": None, + "tvdb_order": None, "animeapi_id": None, "enrich": False, "output_dir": None, diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 8f49811d..21fffadd 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -787,6 +787,13 @@ async def download(request: web.Request) -> web.Response: imdb_id: type: string description: Use this IMDB ID (e.g. tt1375666) for tagging (default - None) + tvdb_id: + type: integer + description: Use this TVDB ID for tagging and episode ordering (default - None) + tvdb_order: + type: string + enum: [official, dvd, absolute, alternate, regional] + description: Renumber episodes to a TVDB season order (default - the tvdb_order config option) output_dir: type: string description: Override the output directory for this download (default - None) diff --git a/unshackle/core/config.py b/unshackle/core/config.py index 92ee1b97..b543a1e4 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -117,6 +117,10 @@ class Config: self.imdb_api_enabled: bool = kwargs.get("imdb_api_enabled", False) self.omdb_api_key: str = kwargs.get("omdb_api_key") or "" self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or "" + self.tvdb_api_key: str = kwargs.get("tvdb_api_key") or "" + self.tvdb_pin: str = kwargs.get("tvdb_pin") or "" + self.tvdb_order: str = (kwargs.get("tvdb_order") or "").lower() + self.metadata_providers: list = kwargs.get("metadata_providers") or [] self.simkl_client_id: str = kwargs.get("simkl_client_id") or "" self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or "" self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or "" diff --git a/unshackle/core/providers/__init__.py b/unshackle/core/providers/__init__.py index 85bdc667..157196bb 100644 --- a/unshackle/core/providers/__init__.py +++ b/unshackle/core/providers/__init__.py @@ -9,26 +9,42 @@ from unshackle.core.providers.imdbapi import IMDBApiProvider from unshackle.core.providers.omdb import OMDBProvider from unshackle.core.providers.simkl import SimklProvider from unshackle.core.providers.tmdb import TMDBProvider +from unshackle.core.providers.tvdb import TVDBProvider if TYPE_CHECKING: from unshackle.core.title_cacher import TitleCacher -# Ordered by priority: IMDBApi (free), OMDb, SIMKL, TMDB -ALL_PROVIDERS: list[type[MetadataProvider]] = [IMDBApiProvider, OMDBProvider, SimklProvider, TMDBProvider] +REGISTRY: dict[str, type[MetadataProvider]] = { + cls.NAME: cls for cls in (IMDBApiProvider, OMDBProvider, SimklProvider, TMDBProvider, TVDBProvider) +} + +# used when `metadata_providers` is unset +DEFAULT_ORDER: tuple[str, ...] = ("imdbapi", "omdb", "simkl", "tmdb", "tvdb") + + +def provider_order() -> list[type[MetadataProvider]]: + """Provider classes in the order configured by `metadata_providers`.""" + from unshackle.core.config import config + + names = [str(n).lower() for n in (config.metadata_providers or DEFAULT_ORDER)] + unknown = [n for n in names if n not in REGISTRY] + if unknown: + log.warning("Ignoring unknown metadata_providers entries: %s", ", ".join(unknown)) + return [REGISTRY[n] for n in dict.fromkeys(names) if n in REGISTRY] def get_available_providers() -> list[MetadataProvider]: """Return instantiated providers that have valid credentials.""" - return [cls() for cls in ALL_PROVIDERS if cls().is_available()] + return [cls() for cls in provider_order() if cls().is_available()] def get_provider(name: str) -> Optional[MetadataProvider]: """Get a specific provider by name.""" - for cls in ALL_PROVIDERS: - if cls.NAME == name: - p = cls() - return p if p.is_available() else None - return None + cls = REGISTRY.get(name) + if not cls: + return None + p = cls() + return p if p.is_available() else None # -- Public API (replaces tags.py functions) -- @@ -44,9 +60,11 @@ def search_metadata( cache_account_hash: Optional[str] = None, ) -> Optional[MetadataResult]: """Search all available providers for metadata. Returns best match.""" + ordered = provider_order() + # Check cache first if title_cacher and cache_title_id: - for cls in ALL_PROVIDERS: + for cls in ordered: p = cls() if not p.is_available(): continue @@ -58,7 +76,7 @@ def search_metadata( return result # Search providers in priority order - for cls in ALL_PROVIDERS: + for cls in ordered: p = cls() if not p.is_available(): continue @@ -230,9 +248,15 @@ def fetch_external_ids( # -- Internal helpers -- -# Provider authority ranking for tie-breaking (lower index = more authoritative) -_ENRICHMENT_PROVIDERS = ("tmdb", "simkl") -_ENRICHMENT_AUTHORITY: dict[str, int] = {name: i for i, name in enumerate(_ENRICHMENT_PROVIDERS)} +# trust ranking for cross-validating enrichments; `metadata_providers` filters this +# set but sets search order only, not the ranking +_ENRICHMENT_AUTHORITY: tuple[str, ...] = ("tmdb", "simkl", "tvdb") + + +def _enrichment_providers() -> list[str]: + """Names of configured providers that can resolve an IMDB ID, most trusted first.""" + configured = {cls.NAME for cls in provider_order() if hasattr(cls, "find_by_imdb_id")} + return [name for name in _ENRICHMENT_AUTHORITY if name in configured] def enrich_ids(result: MetadataResult) -> None: @@ -251,13 +275,14 @@ def enrich_ids(result: MetadataResult) -> None: kind = result.kind or "movie" # Step 1: Collect enrichment results from all available providers + authority = {name: i for i, name in enumerate(_enrichment_providers())} enrichments: list[tuple[str, ExternalIds]] = [] - for provider_name in _ENRICHMENT_PROVIDERS: + for provider_name in authority: p = get_provider(provider_name) if not p: continue try: - enriched = p.find_by_imdb_id(ids.imdb_id, kind) # type: ignore[union-attr] + enriched = p.find_by_imdb_id(ids.imdb_id, kind) # type: ignore[attr-defined] except Exception as exc: log.debug("Enrichment via %s failed: %s", provider_name, exc) continue @@ -268,7 +293,7 @@ def enrich_ids(result: MetadataResult) -> None: return # Step 2: Cross-validate using tmdb_id as anchor — drop providers that disagree - validated = _validate_enrichments(enrichments) + validated = _validate_enrichments(enrichments, authority) # Step 3: Merge validated data (fill gaps only) for _provider_name, ext in validated: @@ -281,6 +306,7 @@ def enrich_ids(result: MetadataResult) -> None: def _validate_enrichments( enrichments: list[tuple[str, ExternalIds]], + authority: dict[str, int], ) -> list[tuple[str, ExternalIds]]: """Drop providers whose tmdb_id conflicts with the authoritative value. @@ -309,7 +335,7 @@ def _validate_enrichments( # No majority — pick the most authoritative provider best_provider = min( tmdb_votes.keys(), - key=lambda name: _ENRICHMENT_AUTHORITY.get(name, 99), + key=lambda name: authority.get(name, 99), ) anchor_tmdb_id = tmdb_votes[best_provider] @@ -409,6 +435,25 @@ def _cached_to_result(cached: dict, provider_name: str, kind: str) -> Optional[M source="omdb", raw=cached, ) + elif provider_name == "tvdb": + from unshackle.core.providers.tvdb import _ids_from_remote, _parse_int + + tvdb_id = _parse_int(cached.get("tvdb_id") or cached.get("id")) + ext = _ids_from_remote(cached.get("remote_ids") or cached.get("remoteIds"), tvdb_id) + # restore IDs that enrichment filled in beyond the raw remote_ids + enriched = cached.get("_enriched_ids", {}) + ext.imdb_id = ext.imdb_id or enriched.get("imdb_id") + ext.tmdb_id = ext.tmdb_id or enriched.get("tmdb_id") + if ext.tmdb_id: + ext.tmdb_kind = kind + return MetadataResult( + title=cached.get("name"), + year=_parse_int(cached.get("year")), + kind=kind, + external_ids=ext, + source="tvdb", + raw=cached, + ) elif provider_name == "imdbapi": title = cached.get("primaryTitle") or cached.get("originalTitle") year = cached.get("startYear") @@ -432,7 +477,8 @@ def _cached_to_result(cached: dict, provider_name: str, kind: str) -> Optional[M __all__ = [ - "ALL_PROVIDERS", + "DEFAULT_ORDER", + "REGISTRY", "ExternalIds", "MetadataProvider", "MetadataResult", @@ -443,5 +489,6 @@ __all__ = [ "get_provider", "get_title_by_id", "get_year_by_id", + "provider_order", "search_metadata", ] diff --git a/unshackle/core/providers/tvdb.py b/unshackle/core/providers/tvdb.py new file mode 100644 index 00000000..8fa7f097 --- /dev/null +++ b/unshackle/core/providers/tvdb.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +from collections.abc import Collection +from difflib import SequenceMatcher +from typing import Any, Optional, Union + +import requests + +from unshackle.core.config import config +from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, _strip_year + +KIND_TO_TYPE: dict[str, str] = {"movie": "movie", "tv": "series"} +KIND_TO_PATH: dict[str, str] = {"movie": "movies", "tv": "series"} + +# a series' Streaming Order is published as "alternate" +SEASON_TYPES: tuple[str, ...] = ("official", "dvd", "absolute", "alternate", "regional") +AIRED_ORDER = "official" + +# cached per-process; providers are re-instantiated per lookup +_token: Optional[str] = None + + +def _parse_int(value: Optional[str]) -> Optional[int]: + try: + return int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +def _ids_from_remote(remote_ids: Optional[list], tvdb_id: Optional[int]) -> ExternalIds: + """Build ExternalIds from a TVDB remote_ids/remoteIds list.""" + ext = ExternalIds(tvdb_id=tvdb_id) + for entry in remote_ids or []: + source = (entry.get("sourceName") or "").lower() + value = entry.get("id") + if source == "imdb": + ext.imdb_id = value + elif source.startswith("themoviedb"): + ext.tmdb_id = _parse_int(value) + return ext + + +def _pick_match( + results: list[dict], search_title: str, year: Optional[int] +) -> tuple[Optional[dict], Optional[str], float]: + """Best title match among results, ignoring any whose year is more than a year out.""" + best: Optional[dict] = None + best_title: Optional[str] = None + best_ratio = 0.0 + for result in results: + result_year = _parse_int(result.get("year")) + if year and result_year and abs(result_year - year) > 1: + continue + for candidate in [result.get("name"), *(result.get("aliases") or [])]: + if not candidate: + continue + ratio = SequenceMatcher(None, _clean(search_title), _clean(candidate)).ratio() + if ratio > best_ratio: + best, best_title, best_ratio = result, candidate, ratio + return best, best_title, best_ratio + + +class TVDBProvider(MetadataProvider): + """TheTVDB v4 metadata provider. Native IDs are TVDB IDs.""" + + NAME = "tvdb" + REQUIRES_KEY = True + BASE_URL = "https://api4.thetvdb.com/v4" + + def __init__(self) -> None: + super().__init__() + self._episodes: dict[tuple[int, str], list[dict]] = {} + + def is_available(self) -> bool: + return bool(config.tvdb_api_key) + + def _login(self) -> Optional[str]: + global _token + payload: dict[str, str] = {"apikey": config.tvdb_api_key} + if config.tvdb_pin: + payload["pin"] = config.tvdb_pin + try: + r = self.session.post(f"{self.BASE_URL}/login", json=payload, timeout=30) + r.raise_for_status() + _token = r.json()["data"]["token"] + except (requests.RequestException, ValueError, KeyError) as exc: + self.log.warning("Failed to authenticate with TVDB: %s", exc) + _token = None + return _token + + def _get(self, path: str, params: Optional[dict] = None) -> Any: + """The `data` payload of a TVDB response: a dict for entities, a list for searches.""" + global _token + token = _token or self._login() + if not token: + return None + for _ in range(2): + try: + r = self.session.get( + f"{self.BASE_URL}{path}", + params=params, + headers={"Authorization": f"Bearer {token}"}, + timeout=30, + ) + if r.status_code == 401: + _token = None + token = self._login() + if not token: + return None + continue + r.raise_for_status() + return r.json().get("data") + except (requests.RequestException, ValueError) as exc: + self.log.debug("TVDB request %s failed: %s", path, exc) + return None + return None + + def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: + search_title = _strip_year(title) + self.log.debug("Searching TVDB for %r (%s, %s)", search_title, kind, year) + + params: dict[str, str | int] = {"query": search_title, "limit": 10} + entity_type = KIND_TO_TYPE.get(kind) + if entity_type: + params["type"] = entity_type + if year is not None: + params["year"] = year + + results = self._get("/search", params) + if not results and year is not None: + del params["year"] + results = self._get("/search", params) + if not results: + self.log.debug("No TVDB results for %r", search_title) + return None + + # year first, so an identically named remake cannot win on title alone + best, best_title, best_ratio = _pick_match(results, search_title, year) + if best is None: + best, best_title, best_ratio = _pick_match(results, search_title, None) + if best is None: + best, best_title = results[0], results[0].get("name") + + tvdb_id = _parse_int(best.get("tvdb_id")) + ext = _ids_from_remote(best.get("remote_ids"), tvdb_id) + if ext.tmdb_id: + ext.tmdb_kind = kind + + self.log.debug("TVDB -> %r (ID %s, ratio %.2f)", best_title, tvdb_id, best_ratio) + + return MetadataResult( + title=best_title or best.get("name"), + year=_parse_int(best.get("year")), + kind=kind, + external_ids=ext, + source="tvdb", + raw=best, + ) + + def _fetch_extended(self, tvdb_id: int, kind: str) -> Optional[dict]: + path = KIND_TO_PATH.get(kind, "series") + return self._get(f"/{path}/{tvdb_id}/extended", {"short": "true"}) + + def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]: + tvdb_id = int(provider_id) + detail = self._fetch_extended(tvdb_id, kind) + if not detail: + return None + + ext = _ids_from_remote(detail.get("remoteIds"), tvdb_id) + if ext.tmdb_id: + ext.tmdb_kind = kind + + return MetadataResult( + title=detail.get("name"), + year=_parse_int(detail.get("year")), + kind=kind, + external_ids=ext, + source="tvdb", + raw=detail, + ) + + def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds: + tvdb_id = int(provider_id) + detail = self._fetch_extended(tvdb_id, kind) + if not detail: + return ExternalIds(tvdb_id=tvdb_id) + ext = _ids_from_remote(detail.get("remoteIds"), tvdb_id) + if ext.tmdb_id: + ext.tmdb_kind = kind + return ext + + def get_episodes(self, tvdb_id: int, order: str) -> list[dict]: + """All episodes of a series in the given season order.""" + cached = self._episodes.get((tvdb_id, order)) + if cached is not None: + return cached + + episodes: list[dict] = [] + for page in range(20): # ponytail: hard page cap, 500 eps/page covers any real series + data = self._get(f"/series/{tvdb_id}/episodes/{order}", {"page": page}) + if data is None and page: + # a partial listing would renumber episodes wrongly + self.log.warning("TVDB %s order listing for series %s failed at page %d", order, tvdb_id, page) + return [] + batch = (data or {}).get("episodes") or [] + episodes.extend(batch) + if len(batch) < 500: + break + + if episodes: # an empty listing may be a transient failure + self._episodes[(tvdb_id, order)] = episodes + return episodes + + def detect_order(self, tvdb_id: int, keys: Collection[tuple[int, int]]) -> str: + """Guess which TVDB season order a service's (season, episode) numbering follows. + + A service does not always number a series in aired order, so the numbering is + scored against every order rather than assumed. + """ + best, best_score = AIRED_ORDER, -1 + for order in SEASON_TYPES: + episodes = self.get_episodes(tvdb_id, order) + if not episodes: + continue + slots = {(_parse_int(e.get("seasonNumber")), _parse_int(e.get("number"))) for e in episodes} + score = sum(1 for key in keys if key in slots) + self.log.debug("TVDB order %s matches %d/%d of the service's episodes", order, score, len(keys)) + if score > best_score: + best, best_score = order, score + if order == AIRED_ORDER and score == len(keys): + break # aired order already accounts for every episode + return best + + def get_order_map( + self, tvdb_id: int, order: str, source_order: str = AIRED_ORDER + ) -> dict[tuple[int, int], tuple[int, int, Optional[str]]]: + """Map (season, episode) in `source_order` to (season, episode, name) in `order`. + + Every order lists the same episodes under the same TVDB episode ID, which is what + the two listings join on. + """ + if order == source_order: + return {} + + source = self.get_episodes(tvdb_id, source_order) + target = self.get_episodes(tvdb_id, order) + if not source or not target: + self.log.warning("TVDB has no %s order for series %s", order, tvdb_id) + return {} + + by_id = {ep.get("id"): ep for ep in target} + mapping: dict[tuple[int, int], tuple[int, int, Optional[str]]] = {} + for ep in source: + match = by_id.get(ep.get("id")) + if not match: + continue + src_season, src_number = _parse_int(ep.get("seasonNumber")), _parse_int(ep.get("number")) + dst_season, dst_number = _parse_int(match.get("seasonNumber")), _parse_int(match.get("number")) + if src_season is None or src_number is None or dst_season is None or dst_number is None: + continue + mapping[(src_season, src_number)] = (dst_season, dst_number, match.get("name")) + + self.log.debug("TVDB %s -> %s order: mapped %d/%d episodes", source_order, order, len(mapping), len(source)) + return mapping + + def find_by_imdb_id(self, imdb_id: str, kind: str) -> Optional[ExternalIds]: + """Resolve a TVDB ID from an IMDb ID via /search/remoteid.""" + results = self._get(f"/search/remoteid/{imdb_id}") + if not results: + self.log.debug("No TVDB results for IMDB ID %s", imdb_id) + return None + + # each hit wraps a single entity, e.g. {"series": {...}} or {"movie": {...}} + preferred = KIND_TO_TYPE.get(kind, "series") + entity = None + for hit in results: + entity = hit.get(preferred) or entity or next(iter(hit.values()), None) + if hit.get(preferred): + break + if not entity: + return None + + tvdb_id = _parse_int(entity.get("id")) + self.log.debug("TVDB find -> TVDB %s for IMDB %s", tvdb_id, imdb_id) + return ExternalIds(imdb_id=imdb_id, tvdb_id=tvdb_id) diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index c77d0597..1b92e2d5 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -86,6 +86,7 @@ def tag_file( title: Title, tmdb_id: Optional[int] = None, imdb_id: Optional[str] = None, + tvdb_id: Optional[int] = None, ) -> None: log.debug("Tagging file %s with title %r", path, title) custom_tags: dict[str, str] = {} @@ -118,7 +119,7 @@ def tag_file( if config.tag_imdb_tmdb: try: providers = get_available_providers() - if not providers: + if not providers and tvdb_id is None: log.debug("No metadata providers available; skipping tag lookup") apply_tags(path, custom_tags) return @@ -130,9 +131,11 @@ def tag_file( imdbapi = get_provider("imdbapi") if imdbapi: result = imdbapi.get_by_id(imdb_id, kind) - if result: - result.external_ids.imdb_id = imdb_id - enrich_ids(result) + if not result: + # IMDxAPI is off or had no answer; the ID alone still resolves the others + result = MetadataResult(title=name, year=year, kind=kind) + result.external_ids.imdb_id = imdb_id + enrich_ids(result) elif tmdb_id is not None: tmdb = get_provider("tmdb") if tmdb: @@ -140,17 +143,28 @@ def tag_file( if result: ext = tmdb.get_external_ids(tmdb_id, kind) result.external_ids = ext + enrich_ids(result) else: # Search across providers in priority order result = search_metadata(name, year, kind) - # If we got a TMDB ID from search but no full external IDs, fetch them - if result and result.external_ids.tmdb_id and not result.external_ids.imdb_id: + # If we got a TMDB ID from search but not the other IDs, fetch them + if ( + result + and result.external_ids.tmdb_id + and not (result.external_ids.imdb_id and result.external_ids.tvdb_id) + ): ext = fetch_external_ids(result.external_ids.tmdb_id, kind) - if ext.imdb_id: + if ext.imdb_id and not result.external_ids.imdb_id: result.external_ids.imdb_id = ext.imdb_id - if ext.tvdb_id: + if ext.tvdb_id and not result.external_ids.tvdb_id: result.external_ids.tvdb_id = ext.tvdb_id + enrich_ids(result) + + if tvdb_id is not None: + if result is None: + result = MetadataResult(kind=kind) + result.external_ids.tvdb_id = tvdb_id if result and result.external_ids: standard_tags = _build_tags_from_ids(result.external_ids, kind) From 786309ea5c85c778f2414f1bca137cb64524a882 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Sun, 2 Aug 2026 10:48:39 -0600 Subject: [PATCH 28/40] docs: cover the TVDB provider, episode ordering and provider order Document tvdb_api_key, tvdb_pin, tvdb_order and metadata_providers, the --tvdb and --tvdb-order flags with their behaviour when an order does not cover a whole series, the new REST API parameters, and the Matroska TVDB2 tag prefixes. --- docs/dev/rest-api/endpoints.md | 3 ++- docs/getting-started/configuration-file.md | 4 +-- docs/getting-started/quickstart.md | 2 +- docs/guide/cli-reference.md | 16 +++++++++++- docs/guide/downloading.md | 25 ++++++++++++++++++- docs/guide/output-and-naming.md | 6 ++++- docs/reference/configuration.md | 29 ++++++++++++++++++---- 7 files changed, 73 insertions(+), 12 deletions(-) diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 4a205505..206f7532 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -384,7 +384,8 @@ Create a download job. Requires `service` and `title_id`; every other field is a | `best_available` | boolean | `false` | Fall back to best available. | | `repack` | boolean | `false` | Add REPACK tag. | | `tag` | string | `null` | Release group tag. | -| `tmdb_id` / `imdb_id` / `animeapi_id` | - | `null` | External ID overrides for tagging. | +| `tmdb_id` / `imdb_id` / `tvdb_id` / `animeapi_id` | - | `null` | External ID overrides for tagging. | +| `tvdb_order` | `official`, `dvd`, `absolute`, `alternate`, `regional` | `null` | Renumber episodes to a TVDB season order. Falls back to the `tvdb_order` config option. | | `enrich` | boolean | `false` | Override title/year from external source. | | `output_dir` | string | `null` | Override output directory. | | `no_cache` / `reset_cache` | boolean | `false` | Title cache controls. | diff --git a/docs/getting-started/configuration-file.md b/docs/getting-started/configuration-file.md index 4111bb8e..6196aa80 100644 --- a/docs/getting-started/configuration-file.md +++ b/docs/getting-started/configuration-file.md @@ -85,7 +85,7 @@ The top-level keys group loosely into these areas: | Key vaults | `key_vaults`, `vault_timeout` | | Proxies & remote | `proxy_providers`, `remote_services`, `serve`, `services` | | Naming & tagging | `tag`, `output_template`, `chapter_fallback_name` | -| External API keys | `tmdb_api_key`, `simkl_client_id`, `ipinfo_api_key` | +| External API keys | `tmdb_api_key`, `tvdb_api_key`, `simkl_client_id`, `ipinfo_api_key`, `metadata_providers` | | Behavior & logging | `update_checks`, `redact_paths`, `debug`, `unicode_filenames` | | Paths | `directories`, `filenames` | @@ -125,7 +125,7 @@ services: !!! tip "A few keys are named after the flag's internal name" Most keys are obvious, but set these exact ones: `range` (`-r`), `list` (`--list`), - `tmdb_id` (`--tmdb`), `imdb_id` (`--imdb`), `no_atmos` (`--noatmos`), and `output_dir` + `tmdb_id` (`--tmdb`), `imdb_id` (`--imdb`), `tvdb_id` (`--tvdb`), `no_atmos` (`--noatmos`), and `output_dir` (`-o`). The [Configuration Reference](../reference/configuration.md#dl) has the full list and every available key. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index f4b75cf9..c645eeec 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -219,7 +219,7 @@ unshackle dl -q 1080 -o /mnt/media/incoming EXAMPLE 81234567 - unshackle writes **movies** as a single `.mkv` file named from your `movies` template. - **TV episodes** are grouped into a per-show / per-season folder (from your `series` template) unless you pass `--no-folder`. -- unshackle builds the filename from your `output_template` and adds IMDb/TMDB IDs to +- unshackle builds the filename from your `output_template` and adds IMDb/TMDB/TVDB IDs to the file's metadata tags when available. To change the default output location permanently, set it in your config: diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index c5c995af..a5d1daec 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -145,8 +145,22 @@ Keep only certain track types, or skip certain track types. Attachments are alwa |---|---| | `--tmdb` | TMDB ID (integer). | | `--imdb` | IMDb ID, e.g. `tt1375666`. | -| `--animeapi` | AnimeAPI ID, e.g. `mal:12345` or `anilist:98765` (defaults to MAL). Back-fills TMDB/IMDb. | +| `--tvdb` | TVDB ID (integer). Skips the series lookup that `--tvdb-order` would otherwise do. | +| `--animeapi` | AnimeAPI ID, e.g. `mal:12345` or `anilist:98765` (defaults to MAL). Back-fills TMDB/IMDb/TVDB. | | `--enrich` | Override show title/year from the external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | +| `--tvdb-order` | Renumber episodes to a TVDB season order: `official` (aired), `dvd`, `absolute`, `alternate`, or `regional`. Needs `tvdb_api_key`. | + +#### Episode ordering + +A service does not always number a series the way TVDB's aired order does. Some services list +Futurama in TVDB's `alternate` (Streaming) order, for example. `--tvdb-order` works out which +order the service used, then renumbers the episodes into the order you asked for. + +!!! note "Orders that do not cover the whole series" + An order can omit episodes the service carries; TVDB's `dvd` order for Futurama leaves out + the four movies. Those episodes keep their original numbering. If that would give two + episodes the same season/episode slot, and so the same filename, unshackle logs an error + and keeps the service's numbering unchanged. Pick an order that covers the whole series. ### DRM, keys & decryption diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 6606a9a3..11d26828 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -519,13 +519,35 @@ tagging and naming: | --- | --- | --- | | `--tmdb` | `--tmdb 27205` | Use this TMDB ID instead of automatic lookup. | | `--imdb` | `--imdb tt1375666` | Use this IMDb ID. | +| `--tvdb` | `--tvdb 73871` | Use this TVDB ID instead of looking the series up. | | `--animeapi` | `--animeapi mal:12345` | Resolve via AnimeAPI (`mal:`/`anilist:` prefix; defaults to MAL). | | `--enrich` | - | Override the show title and year from an external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | +| `--tvdb-order` | `--tvdb-order dvd` | Renumber episodes to a TVDB season order. Needs `tvdb_api_key`. | ```shell title="Force the right IMDb match and enrich the title" unshackle dl --imdb tt1375666 --enrich EXAMPLE 81234567 ``` +### Episode ordering + +A service does not always number a series the way TVDB's aired order does. Some services list +Futurama in TVDB's `alternate` (Streaming) order, for example, while TVDB's `official` order +holds back four season-one episodes to the start of season two. `--tvdb-order` works out which +order the service used, then renumbers the episodes into the order you asked for: + +```shell title="Download Futurama in DVD order" +unshackle dl --tvdb-order dvd EXAMPLE 81234567 +``` + +Available orders are `official` (aired), `dvd`, `absolute`, `alternate`, and `regional`. Set +`tvdb_order` in your config to apply one by default. + +!!! warning "Orders that do not cover the whole series" + An order can leave out episodes the service carries. TVDB's `dvd` order does not list + Futurama's four movies. Those episodes keep their original numbering. If that would give + two episodes the same season/episode slot, and so the same filename, unshackle logs an + error and leaves the numbering untouched. Pick an order that covers the whole series. + ## Configuration defaults Every flag on this page can be set as a default in your configuration file so you don't @@ -571,4 +593,5 @@ authoritative list. | `--list` / `--list-titles` / `--skip-dl` | | Dry runs. | | `--cdm-only` / `--vaults-only` | | Key source control. | | `--export` | | Export track info and keys to JSON. | -| `--tmdb` / `--imdb` / `--animeapi` / `--enrich` | | Metadata overrides. | +| `--tmdb` / `--imdb` / `--tvdb` / `--animeapi` / `--enrich` | | Metadata overrides. | +| `--tvdb-order` | | Renumber episodes to a TVDB season order. | diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 1019f124..5b8b1647 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -245,7 +245,11 @@ tag_imdb_tmdb: true |---|---|---|---| | `tag` | str | `""` | The release-group tag used for `{tag}`. | | `tag_group_name` | bool | `true` | Write the group name (`config.tag`) into the MKV `Group` metadata tag. | -| `tag_imdb_tmdb` | bool | `true` | Look up and embed IMDb / TMDB / TVDB external-ID tags in the MKV metadata (uses `tmdb_api_key` / `simkl_client_id` when available). | +| `tag_imdb_tmdb` | bool | `true` | Look up and embed IMDb / TMDB / TVDB external-ID tags in the MKV metadata (uses `tmdb_api_key` / `tvdb_api_key` / `simkl_client_id` when available). | + +The IDs are written as the Matroska `IMDB`, `TMDB`, and `TVDB2` tags. `TVDB2` values carry the +entity prefix the [Matroska tagging spec](https://www.matroska.org/technical/tagging.html) +requires: `series/73871` for a show, `movies/113` for a film. You can override the tag per run without editing config: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 45a00d26..4d9015f5 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -39,7 +39,7 @@ This page assumes you already know the basics and want precise details. | [DRM & CDM](#drm-cdm) | `cdm`, `remote_cdm`, `decryption` | | [Network & proxy](#network-proxy) | `network`, `headers`, `proxy_providers` | | [Key vaults](#key-vaults) | `key_vaults`, `vault_timeout` | -| [External API keys](#external-api-keys) | `imdb_api_enabled`, `omdb_api_key`, `tmdb_api_key`, `simkl_client_id`, `decrypt_labs_api_key`, `ipinfo_api_key` | +| [External API keys](#external-api-keys) | `imdb_api_enabled`, `omdb_api_key`, `tmdb_api_key`, `tvdb_api_key`, `tvdb_pin`, `tvdb_order`, `simkl_client_id`, `metadata_providers`, `decrypt_labs_api_key`, `ipinfo_api_key` | | [Caching & updates](#caching-updates) | `title_cache_enabled`, `title_cache_time`, `title_cache_max_retention`, `update_checks`, `update_check_interval` | | [Logging, privacy & debug](#logging-privacy-debug) | `redact_paths`, `debug`, `debug_keys`, `debug_requests`, `set_terminal_bg` | | [Deprecated & removed](#deprecated-removed-keys) | `curl_impersonate`, `downloader`, `scene_naming` | @@ -784,13 +784,31 @@ All default to an empty string and enable optional metadata/geolocation features | `omdb_api_key` | str | `""` | [OMDb](https://www.omdbapi.com/) API key for IMDb metadata lookups; a more reliable alternative to IMDxAPI. Free keys are available on the OMDb site. | | `tmdb_api_key` | str | `""` | TMDB API key for metadata enrichment and external-ID tags. | | `simkl_client_id` | str | `""` | SIMKL client ID for metadata lookups; an alternative/fallback source to TMDB. | +| `tvdb_api_key` | str | `""` | [TheTVDB v4](https://thetvdb.com/api-information) API key; a fallback source to TMDB, strongest on TV series. Free keys are available on the TVDB site. | +| `tvdb_pin` | str | `""` | Subscriber PIN, only needed for a user-supported TVDB key. Leave empty for a normal project key. | +| `tvdb_order` | str | `""` | Default for `--tvdb-order`: renumber episodes to a TVDB season order (`official`, `dvd`, `absolute`, `alternate`, `regional`). Empty keeps the numbering the service gives. | +| `metadata_providers` | list | *(see below)* | Metadata providers to use, in the order they are tried. | | `decrypt_labs_api_key` | str | `""` | Global Decrypt Labs API key (used by remote CDM / vault). | | `ipinfo_api_key` | str | `""` | ipinfo.io API key for IP/region lookups. | -!!! tip "SIMKL is a fallback for TMDB" - SIMKL is an alternative metadata source. Use it when you have no `tmdb_api_key` - configured; it improves title matching and tagging in that case rather than replacing - TMDB. +!!! tip "Choosing and ordering metadata providers" + A lookup goes to each provider in turn and stops at the first good match, so the order + decides which source wins. `metadata_providers` sets that order: + + ```yaml + metadata_providers: [tvdb, tmdb, simkl] + ``` + + Names not in the list are never used, so this both orders and filters. Unknown names are + ignored with a warning. Leave it unset for the default order: + + ```yaml + metadata_providers: [imdbapi, omdb, simkl, tmdb, tvdb] + ``` + + A provider is skipped when its API key is missing, whatever the order. Available names are + `imdbapi`, `omdb`, `simkl`, `tmdb`, and `tvdb`. Put `tvdb` first to make TheTVDB your + primary source; leave it last to use it only as a fallback for TMDB. !!! note "`ipinfo_api_key` never touches your service sessions" The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header; it @@ -907,6 +925,7 @@ proxy_providers: omdb_api_key: "your-omdb-key" tmdb_api_key: "your-tmdb-key" +tvdb_api_key: "your-tvdb-key" title_cache_time: 3600 redact_paths: true ``` From 330e274da67b9ec4dbfea593a593964730a0b6e3 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Sun, 2 Aug 2026 13:28:23 -0600 Subject: [PATCH 29/40] feat(temp): give each download task its own temp directory Each dl run works in temp/task_<id>/, removed on every exit path. Orphans from hard-killed processes are swept via lock liveness, and env clear temp skips directories whose task is still running. --- tests/core/test_task_temp.py | 174 +++++++++++++++++++++++++++++++++++ unshackle/commands/dl.py | 2 + unshackle/commands/env.py | 29 +++++- unshackle/core/temp.py | 79 ++++++++++++++++ 4 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 tests/core/test_task_temp.py create mode 100644 unshackle/core/temp.py diff --git a/tests/core/test_task_temp.py b/tests/core/test_task_temp.py new file mode 100644 index 00000000..69aceaa0 --- /dev/null +++ b/tests/core/test_task_temp.py @@ -0,0 +1,174 @@ +"""Tests for per-task temp directories. + +Each run gets a private dir that is removed on every exit path. The sweep reclaims +dirs from hard-killed processes via lock liveness, and `env clear temp` skips the +dir of a task that is still running. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest +from filelock import FileLock + +from unshackle.commands.env import clear_directory +from unshackle.core.config import config +from unshackle.core.temp import LOCK_NAME, TASK_PREFIX, is_stale, sweep_task_dirs, task_temp_dir + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def temp_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = tmp_path / "temp" + root.mkdir() + monkeypatch.setattr(config.directories, "temp", root) + return root + + +def backdate(path: Path) -> None: + stamp = time.time() - 120 + os.utime(path, (stamp, stamp)) + + +def make_task_dir(root: Path, name: str = "task_deadbeef", lock: bool = True) -> Path: + task_dir = root / name + task_dir.mkdir() + if lock: + (task_dir / LOCK_NAME).touch() + return task_dir + + +def test_task_dir_created_repointed_and_restored(temp_root: Path) -> None: + with task_temp_dir() as task_dir: + assert task_dir.parent == temp_root + assert task_dir.name.startswith(TASK_PREFIX) + assert config.directories.temp == task_dir + assert config.directories.temp == temp_root + assert not task_dir.exists() + + +def test_task_dir_removed_on_exception(temp_root: Path) -> None: + with pytest.raises(ValueError): + with task_temp_dir() as task_dir: + (task_dir / "partial.mp4").write_bytes(b"x") + raise ValueError("boom") + assert not task_dir.exists() + assert config.directories.temp == temp_root + + +def test_task_dir_removed_on_keyboard_interrupt(temp_root: Path) -> None: + with pytest.raises(KeyboardInterrupt): + with task_temp_dir() as task_dir: + raise KeyboardInterrupt + assert not task_dir.exists() + assert config.directories.temp == temp_root + + +def test_sweep_removes_unlocked_stale_dir(temp_root: Path) -> None: + task_dir = make_task_dir(temp_root) + (task_dir / "leftover.mp4").write_bytes(b"x") + backdate(task_dir) + sweep_task_dirs(temp_root) + assert not task_dir.exists() + + +def test_sweep_skips_locked_dir(temp_root: Path) -> None: + task_dir = make_task_dir(temp_root) + lock = FileLock(task_dir / LOCK_NAME) + lock.acquire() + backdate(task_dir) + try: + assert is_stale(task_dir) is False + sweep_task_dirs(temp_root) + assert task_dir.exists() + finally: + lock.release() + + +def test_sweep_skips_young_unlocked_dir(temp_root: Path) -> None: + task_dir = make_task_dir(temp_root) + assert is_stale(task_dir) is False + sweep_task_dirs(temp_root) + assert task_dir.exists() + + +def test_sweep_removes_stale_dir_without_lock_file(temp_root: Path) -> None: + task_dir = make_task_dir(temp_root, lock=False) + backdate(task_dir) + assert is_stale(task_dir) is True + sweep_task_dirs(temp_root) + assert not task_dir.exists() + + +def test_sweep_ignores_non_task_entries(temp_root: Path) -> None: + loose_file = temp_root / "leftover.mp4" + loose_file.write_bytes(b"x") + other_dir = temp_root / "not_a_task" + other_dir.mkdir() + backdate(loose_file) + backdate(other_dir) + sweep_task_dirs(temp_root) + assert loose_file.exists() + assert other_dir.exists() + + +@pytest.mark.skipif(os.name == "nt", reason="symlink creation needs privileges on Windows") +def test_sweep_never_follows_symlinked_task_dir(temp_root: Path, tmp_path: Path) -> None: + target = tmp_path / "victim" + target.mkdir() + (target / "precious.mkv").write_bytes(b"data") + link = temp_root / "task_evil" + link.symlink_to(target) + backdate(target) + sweep_task_dirs(temp_root) + assert (target / "precious.mkv").exists() + assert not (target / LOCK_NAME).exists() + + +@pytest.mark.skipif(os.name == "nt", reason="symlink creation needs privileges on Windows") +def test_clear_directory_unlinks_symlinked_dir_without_following(temp_root: Path, tmp_path: Path) -> None: + target = tmp_path / "victim" + target.mkdir() + (target / "precious.mkv").write_bytes(b"data") + link = temp_root / "task_evil" + link.symlink_to(target) + backdate(target) + files_count, _ = clear_directory(temp_root) + assert not link.exists() + assert (target / "precious.mkv").exists() + assert files_count == 1 + + +def test_clear_directory_skips_locked_task_dir(temp_root: Path) -> None: + task_dir = make_task_dir(temp_root) + (task_dir / "in_flight.mp4").write_bytes(b"xyz") + lock = FileLock(task_dir / LOCK_NAME) + lock.acquire() + backdate(task_dir) + loose_file = temp_root / "leftover.mp4" + loose_file.write_bytes(b"1234") + try: + files_count, freed_bytes = clear_directory(temp_root) + finally: + lock.release() + assert task_dir.exists() + assert (task_dir / "in_flight.mp4").exists() + assert not loose_file.exists() + assert files_count == 1 + assert freed_bytes == 4 + + +def test_clear_directory_removes_stale_task_dir(temp_root: Path) -> None: + task_dir = make_task_dir(temp_root) + (task_dir / "orphan.mp4").write_bytes(b"12345") + backdate(task_dir) + files_count, freed_bytes = clear_directory(temp_root) + assert not task_dir.exists() + assert temp_root.is_dir() + # glob("**/*") never matches dotfiles, so .lock is not counted + assert files_count == 1 + assert freed_bytes == 5 diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index a8615afb..0902518b 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -63,6 +63,7 @@ from unshackle.core.providers.tvdb import SEASON_TYPES from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN from unshackle.core.service import Service from unshackle.core.services import Services +from unshackle.core.temp import with_task_temp from unshackle.core.title_cacher import get_account_hash from unshackle.core.titles import Movie, Movies, Music, Series, Song, Title_T from unshackle.core.titles.episode import Episode @@ -1246,6 +1247,7 @@ class dl: # able to keep `self` as the first positional self.cli._result_callback = self.result + @with_task_temp def result( self, service: Service, diff --git a/unshackle/commands/env.py b/unshackle/commands/env.py index 9e193ac9..f4060677 100644 --- a/unshackle/commands/env.py +++ b/unshackle/commands/env.py @@ -15,6 +15,7 @@ from unshackle.core.config import POSSIBLE_CONFIG_PATHS, config, config_path from unshackle.core.console import console from unshackle.core.constants import context_settings from unshackle.core.services import Services +from unshackle.core.temp import TASK_PREFIX, is_stale def get_dependencies() -> list[dict]: @@ -94,18 +95,36 @@ def get_dependencies() -> list[dict]: def clear_directory(path: Path) -> tuple[int, int]: - """Delete a directory's contents, returning (files_removed, freed_bytes); recreates the dir.""" + """Delete a directory's contents, returning (files_removed, freed_bytes); recreates the dir. + + Skips task directories that belong to a running download. + """ files_count = 0 freed_bytes = 0 if path.exists(): - for entry in path.glob("**/*"): - if entry.is_file(): + for entry in path.iterdir(): + is_real_dir = entry.is_dir() and not entry.is_symlink() + if is_real_dir and entry.name.startswith(TASK_PREFIX) and not is_stale(entry): + continue + if is_real_dir: + for child in entry.glob("**/*"): + if child.is_file(): + files_count += 1 + try: + freed_bytes += child.stat().st_size + except OSError: + pass + shutil.rmtree(entry, ignore_errors=True) + else: files_count += 1 try: - freed_bytes += entry.stat().st_size + freed_bytes += entry.lstat().st_size + except OSError: + pass + try: + entry.unlink(missing_ok=True) except OSError: pass - shutil.rmtree(path, ignore_errors=True) path.mkdir(parents=True, exist_ok=True) return files_count, freed_bytes diff --git a/unshackle/core/temp.py b/unshackle/core/temp.py new file mode 100644 index 00000000..b70d2573 --- /dev/null +++ b/unshackle/core/temp.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import functools +import shutil +import time +import uuid +from contextlib import contextmanager +from pathlib import Path +from typing import Callable, Iterator, Optional, ParamSpec, TypeVar + +from filelock import FileLock, Timeout + +from unshackle.core.config import config + +TASK_PREFIX = "task_" +LOCK_NAME = ".lock" +STALE_GRACE = 60.0 + +P = ParamSpec("P") +R = TypeVar("R") + + +def is_stale(task_dir: Path) -> bool: + """A task dir is stale when nothing holds its lock and it is past the setup grace.""" + try: + if time.time() - task_dir.stat().st_mtime < STALE_GRACE: + return False + except OSError: + return False + lock_path = task_dir / LOCK_NAME + if not lock_path.exists(): + return True + lock = FileLock(lock_path, timeout=0) + try: + lock.acquire() + except (Timeout, OSError): + return False + lock.release() + return True + + +def sweep_task_dirs(root: Path) -> None: + """Remove task dirs left by processes that died without cleaning up.""" + if not root.is_dir(): + return + for entry in root.iterdir(): + if entry.is_dir() and not entry.is_symlink() and entry.name.startswith(TASK_PREFIX) and is_stale(entry): + shutil.rmtree(entry, ignore_errors=True) + + +@contextmanager +def task_temp_dir(task_id: Optional[str] = None) -> Iterator[Path]: + """Point config.directories.temp at a private dir for this run; remove it on any exit.""" + root = config.directories.temp + root.mkdir(parents=True, exist_ok=True) + sweep_task_dirs(root) + + task_dir = root / f"{TASK_PREFIX}{task_id or uuid.uuid4().hex[:12]}" + task_dir.mkdir(parents=True, exist_ok=True) + lock = FileLock(task_dir / LOCK_NAME) + lock.acquire() + config.directories.temp = task_dir + try: + yield task_dir + finally: + config.directories.temp = root + lock.release() + shutil.rmtree(task_dir, ignore_errors=True) + + +def with_task_temp(fn: Callable[P, R]) -> Callable[P, R]: + """Run the wrapped callable inside its own task temp dir.""" + + @functools.wraps(fn) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + with task_temp_dir(): + return fn(*args, **kwargs) + + return wrapper From b580ccdd42b6a8b97f522c5f1339a45f389df5a6 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Sun, 2 Aug 2026 13:41:07 -0600 Subject: [PATCH 30/40] fix(tools): pass -nostdin to all ffmpeg calls When ffmpeg runs with the terminal on stdin it turns off echo for its interactive keys. If several run at once, or one gets killed, echo never comes back and the terminal is left broken after dl exits. --- unshackle/commands/util.py | 16 +++++++++++++++- unshackle/core/manifests/hls.py | 1 + unshackle/core/tracks/track.py | 1 + unshackle/core/tracks/video.py | 3 +++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/unshackle/commands/util.py b/unshackle/commands/util.py index 67d7f4d1..d2322028 100644 --- a/unshackle/commands/util.py +++ b/unshackle/commands/util.py @@ -135,7 +135,19 @@ def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> N ] ffmpeg_call = subprocess.Popen( - [binaries.FFMPEG, "-y", "-i", str(video_path), "-map", "0:v:0", "-c", "copy", "-bsf:v", crop_filter] + [ + binaries.FFMPEG, + "-nostdin", + "-y", + "-i", + str(video_path), + "-map", + "0:v:0", + "-c", + "copy", + "-bsf:v", + crop_filter, + ] + out_path, stdout=subprocess.PIPE, ) @@ -216,6 +228,7 @@ def range_(path: Path, full: bool, preview: bool) -> None: ffmpeg_call = subprocess.Popen( [ binaries.FFMPEG, + "-nostdin", "-y", "-i", str(video_path), @@ -267,6 +280,7 @@ def test(path: Path, map_: str) -> None: p = subprocess.Popen( [ binaries.FFMPEG, + "-nostdin", "-hide_banner", "-benchmark", "-err_detect", diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 11caa093..a84d697c 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -1074,6 +1074,7 @@ class HLS: subprocess.check_call( [ binaries.FFMPEG, + "-nostdin", "-hide_banner", "-loglevel", "error", diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 3e1b34e6..26c10329 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -921,6 +921,7 @@ class Track: def _ffmpeg(extra_args: list[str] = None, bsf: Optional[str] = None): args = [ binaries.FFMPEG, + "-nostdin", "-hide_banner", "-loglevel", "error", diff --git a/unshackle/core/tracks/video.py b/unshackle/core/tracks/video.py index ead971fc..6d2aecd9 100644 --- a/unshackle/core/tracks/video.py +++ b/unshackle/core/tracks/video.py @@ -385,6 +385,7 @@ class Video(Track): subprocess.run( [ binaries.FFMPEG, + "-nostdin", "-hide_banner", "-loglevel", "panic", @@ -493,6 +494,7 @@ class Video(Track): subprocess.run( [ binaries.FFMPEG, + "-nostdin", "-hide_banner", "-loglevel", "error", @@ -667,6 +669,7 @@ class Video(Track): subprocess.run( [ binaries.FFMPEG, + "-nostdin", "-hide_banner", "-loglevel", "panic", From 4e1ccf32303b03c037f47929b2744fd606de5ddc Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Mon, 3 Aug 2026 09:27:40 -0600 Subject: [PATCH 31/40] feat(dl): add -fsl/--forced-s-lang to filter forced subtitles by language --- docs/dev/rest-api/endpoints.md | 1 + docs/guide/cli-reference.md | 1 + docs/guide/downloading.md | 4 ++ docs/guide/subtitles.md | 13 +++++ docs/reference/configuration.md | 1 + .../test_forced_sub_selection.py | 25 +++++++++ unshackle/commands/dl.py | 51 ++++++++++++++++--- unshackle/core/api/download_manager.py | 1 + unshackle/core/api/handlers.py | 1 + unshackle/core/api/routes.py | 5 ++ unshackle/core/utilities.py | 15 ++++++ 11 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 tests/lang_selection/test_forced_sub_selection.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 206f7532..8fc42797 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -366,6 +366,7 @@ Create a download job. Requires `service` and `title_id`; every other field is a | `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. | | `require_subs` | string[] | `[]` | Required subtitle languages. | | `forced_subs` | boolean | `false` | Include forced subtitles. | +| `forced_s_lang` | string[] | `[]` | Forced subtitle language(s); implies `forced_subs`. | | `exact_lang` | boolean | `false` | Exact language matching. | | `sub_format` | string | `null` | Output subtitle format. | | `video_only` / `audio_only` / `subs_only` / `chapters_only` | boolean | `false` | Track-type restriction (at most one). | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index a5d1daec..d3dba5cd 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -96,6 +96,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | `-sl`, `--s-lang` | `all` | Subtitle language(s). | | `--require-subs` | - | Required subtitle langs; keeps **all** subs only if these exist. **Cannot combine with `--s-lang`.** | | `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | +| `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. | | `--exact-lang` | off | Exact matching only: `-l es-419` matches `es-419`, not `es-ES`. | | `--sub-format` | - | Output subtitle format (`SRT`/`srt`, `VTT`/`webvtt`, `ASS`/`ssa`, `TTML`, `SMI`, ...), or `original` to keep the source format. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 11d26828..8959e208 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -289,6 +289,9 @@ download on the presence of a specific subtitle track. - `-fs` / `--forced-subs`: include forced subtitle tracks (signs/foreign dialogue). Without this flag, forced subs are dropped from selection. +- `-fsl` / `--forced-s-lang`: keep forced subtitles only in these languages (implies + `-fs`). Works independently of `--s-lang`, so `-sl all -fsl en` grabs every full + subtitle but only the English forced track. - `--sub-format`: set the output subtitle format, converting only when necessary. Accepts codec names/values and common aliases (`srt`, `vtt`, `ass`, `ssa`, `ttml`, etc.), or the literal `original` to keep the source format. @@ -578,6 +581,7 @@ authoritative list. | `--v-lang` | `-vl` | Video-only language override. | | `--s-lang` | `-sl` | Subtitle language(s); default `all`. | | `--forced-subs` | `-fs` | Include forced subtitles. | +| `--forced-s-lang` | `-fsl` | Forced subtitle language(s); implies `-fs`. | | `--sub-format` | | Output subtitle format. | | `--wanted` | `-w` | Episode/season range. | | `--select-titles` | | Interactively pick episodes or films. | diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index 8823be19..d272445a 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -70,6 +70,7 @@ Subtitle selection happens through `dl` flags. The most important is language se | `-sl`, `--s-lang` | Language(s) wanted for subtitles. Defaults to `all`. | | `--require-subs` | Require these languages to exist; if present, download **all** subtitles. Cannot be combined with `--s-lang`. | | `-fs`, `--forced-subs` | Include forced subtitle tracks (excluded by default). | +| `-fsl`, `--forced-s-lang` | Language(s) wanted for forced subtitles; implies `-fs`. | | `--exact-lang` | Exact language matching, with no regional variants. | | `-S`, `--subs-only` | Download only subtitle tracks. | | `-ns`, `--no-subs` | Do not download subtitle tracks at all. | @@ -121,6 +122,18 @@ default**. Add `-fs` / `--forced-subs` to include them: unshackle dl --s-lang en --forced-subs EXAMPLE 81234567 ``` +`-fs` keeps forced tracks in every selected language. If you only want forced tracks +in certain languages, use `-fsl` / `--forced-s-lang` instead. It implies `-fs`, so +you do not need both: + +```shell title="All full subtitles, but only the English forced track" +unshackle dl --s-lang all --forced-s-lang en EXAMPLE 81234567 +``` + +If you pass both flags, `-fsl` wins and keeps only forced tracks in its languages. +It also understands `orig` (the title's original language) and `all` (every forced +track, same as plain `-fs`). + ## Track types: forced, SDH, and CC Every subtitle track carries flags describing what kind of captions it holds. These diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index eec8fb91..4ededb9c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -320,6 +320,7 @@ Common keys (a useful subset; every `dl` flag works): | `channels` | float | *(unset)* | Audio channels, e.g. `6` for 5.1. | | `sub_format` | str | *(unset)* | Convert subtitles to this format (`srt`, `vtt`, `original`, ...). | | `forced_subs` | bool | `false` | Include forced subtitle tracks. | +| `forced_s_lang` | list | `[]` | Forced subtitle language(s); implies `forced_subs`. | | `no_subs` / `no_audio` / `no_chapters` | bool | `false` | Skip that track type. | | `downloads` | int | `1` | Tracks downloaded concurrently. | | `workers` | int | *(downloader default)* | Threads per track. | diff --git a/tests/lang_selection/test_forced_sub_selection.py b/tests/lang_selection/test_forced_sub_selection.py new file mode 100644 index 00000000..379c9bf3 --- /dev/null +++ b/tests/lang_selection/test_forced_sub_selection.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest + +from unshackle.core.utilities import keep_forced_subtitle + + +@pytest.mark.parametrize( + "forced,language,forced_s_lang,exact,expected", + [ + (False, "en", [], False, True), + (False, "de", ["en"], False, True), + (True, "en", [], False, False), + (True, "en", ["en"], False, True), + (True, "en-US", ["en"], False, True), + (True, "de", ["en"], False, False), + (True, "de", ["en", "de"], False, True), + # CLDR rates paradigm-regional pairs (en/en-US) distance 0, so exact still matches + (True, "en-US", ["en"], True, True), + (True, "en", ["en"], True, True), + (True, "es-419", ["es-ES"], True, False), + ], +) +def test_keep_forced_subtitle(forced: bool, language: str, forced_s_lang: list[str], exact: bool, expected: bool): + assert keep_forced_subtitle(forced, language, forced_s_lang, exact) is expected diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 0902518b..b4b31193 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -80,6 +80,8 @@ from unshackle.core.utilities import ( get_system_fonts, init_debug_logger, is_close_match, + is_exact_match, + keep_forced_subtitle, log_event, suggest_font_packages, time_elapsed_since, @@ -551,6 +553,13 @@ class dl: help="Required subtitle languages. Downloads all subtitles only if these languages exist. Cannot be used with --s-lang.", ) @click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.") + @click.option( + "-fsl", + "--forced-s-lang", + type=LANGUAGE_RANGE, + default=[], + help="Languages wanted for forced subtitles, implies -fs. Keeps forced subs only in these languages.", + ) @click.option( "--exact-lang", is_flag=True, @@ -1270,6 +1279,7 @@ class dl: s_lang: list[str], require_subs: list[str], forced_subs: bool, + forced_s_lang: list[str], exact_lang: bool, sub_format: Optional[Union[Subtitle.Codec, str]], video_only: bool, @@ -1312,6 +1322,15 @@ class dl: self._remote_service = service if self.server_cdm else None start_time = time.time() + if forced_s_lang: + # services read forced_subs from raw ctx params, so the implication must be pushed there too + forced_subs = True + if "all" in forced_s_lang: + forced_s_lang = [] + ctx = getattr(service, "ctx", None) + if ctx and ctx.parent: + ctx.parent.params["forced_subs"] = True + if skip_dl: DOWNLOAD_LICENCE_ONLY.set() @@ -2782,6 +2801,9 @@ class dl: sys.exit(1) # filter subtitle tracks + fsl = [t for t in forced_s_lang if t != "orig"] + if "orig" in forced_s_lang and title.language: + fsl.append(str(title.language)) if keep_subtitles and require_subs: missing_langs = [ lang @@ -2797,8 +2819,6 @@ class dl: f"Required languages found ({', '.join(require_subs)}), downloading all available subtitles" ) elif keep_subtitles and s_lang and "all" not in s_lang: - from unshackle.core.utilities import is_exact_match - match_func = is_exact_match if exact_lang else is_close_match missing_langs = find_missing_langs( @@ -2819,19 +2839,38 @@ class dl: self.log.warning( f"{missing_str} not found in subtitle tracks, continuing without subtitles" ) - title.tracks.subtitles = [] + title.tracks.select_subtitles( + lambda x: ( + x.forced and keep_forced_subtitle(x.forced, x.language, fsl, exact_lang) + ) + ) else: self.log.error(missing_str + " not found in tracks") sys.exit(1) if s_lang and title.tracks.subtitles: - title.tracks.select_subtitles(lambda x: match_func(x.language, s_lang)) + title.tracks.select_subtitles( + lambda x: ( + match_func(x.language, s_lang) + or bool(x.forced and fsl and match_func(x.language, fsl)) + ) + ) if not title.tracks.subtitles and not best_available: self.log.error(f"There's no {s_lang} Subtitle Track...") sys.exit(1) - if keep_subtitles and not forced_subs: - title.tracks.select_subtitles(lambda x: not x.forced) + if keep_subtitles: + # fsl, not forced_s_lang: unresolvable orig must keep all forced, not drop them + if fsl: + title.tracks.select_subtitles( + lambda x: keep_forced_subtitle(x.forced, x.language, fsl, exact_lang) + ) + if not any(x.forced for x in title.tracks.subtitles): + self.log.warning( + f"No forced subtitle tracks matched --forced-s-lang ({', '.join(forced_s_lang)})" + ) + elif not forced_subs: + title.tracks.select_subtitles(lambda x: not x.forced) # filter audio tracks # might have no audio tracks if part of the video, e.g. transport stream hls diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 180ed7e5..56fc0d99 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -536,6 +536,7 @@ def _perform_download( s_lang=params.get("s_lang", ["all"]), require_subs=params.get("require_subs", []), forced_subs=params.get("forced_subs", False), + forced_s_lang=params.get("forced_s_lang", []), exact_lang=params.get("exact_lang", False), sub_format=params.get("sub_format"), video_only=params.get("video_only", False), diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index f22d9065..569dafb2 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -44,6 +44,7 @@ DEFAULT_DOWNLOAD_PARAMS = { "s_lang": ["all"], "require_subs": [], "forced_subs": False, + "forced_s_lang": [], "exact_lang": False, "sub_format": None, "video_only": False, diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 21fffadd..3af50bf4 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -689,6 +689,11 @@ async def download(request: web.Request) -> web.Response: forced_subs: type: boolean description: Include forced subtitle tracks (default - false) + forced_s_lang: + type: array + items: + type: string + description: Languages wanted for forced subtitles, implies forced_subs (default - []) exact_lang: type: boolean description: Use exact language matching (no variants) (default - false) diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index d101ed35..3b008ce3 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -163,6 +163,21 @@ def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_EXACT_DISTANCE +def keep_forced_subtitle( + forced: bool, + language: Union[str, Language], + forced_s_lang: Sequence[str], + exact: bool = False, +) -> bool: + """The -fsl filter: non-forced tracks always pass, forced tracks only if they match forced_s_lang.""" + if not forced: + return True + if not forced_s_lang: + return False + match_func = is_exact_match if exact else is_close_match + return match_func(language, forced_s_lang) + + def find_missing_langs( requested: Sequence[str], available: Sequence[Union[str, Language, None]], From a6997f6bca138b425bec9fea399ffe60023bfe5c Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Mon, 3 Aug 2026 10:15:05 -0600 Subject: [PATCH 32/40] feat(tracks): add language_priority to sort preferred languages first subtitle.language_priority and audio.language_priority prepend to the CLI languages, so the named languages sort first and nothing is filtered. --exact-lang now applies to the sorts too, through matching_languages, which follows the same RFC 4647 rule as Tracks.by_language and ignores invalid tags. --- docs/guide/cli-reference.md | 8 +- docs/guide/downloading.md | 20 +++++ docs/guide/subtitles.md | 2 + docs/reference/configuration.md | 34 +++++++- .../lang_selection/test_matching_languages.py | 33 +++++++ .../lang_selection/test_resolve_sort_langs.py | 24 ++++++ tests/tracks/test_subtitle_sort.py | 85 ++++++++++++++++++- unshackle/commands/dl.py | 43 ++++------ unshackle/core/tracks/tracks.py | 20 +++-- unshackle/core/utilities.py | 44 ++++++++++ 10 files changed, 277 insertions(+), 36 deletions(-) create mode 100644 tests/lang_selection/test_matching_languages.py create mode 100644 tests/lang_selection/test_resolve_sort_langs.py diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index d3dba5cd..09473416 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -97,9 +97,15 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | `--require-subs` | - | Required subtitle langs; keeps **all** subs only if these exist. **Cannot combine with `--s-lang`.** | | `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | | `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. | -| `--exact-lang` | off | Exact matching only: `-l es-419` matches `es-419`, not `es-ES`. | +| `--exact-lang` | off | Exact matching only: `-l es-419` matches `es-419`, not `es-ES`. Applies to selection and to sort order. | | `--sub-format` | - | Output subtitle format (`SRT`/`srt`, `VTT`/`webvtt`, `ASS`/`ssa`, `TTML`, `SMI`, ...), or `original` to keep the source format. | +!!! tip "These flags select languages, they do not order them" + Naming languages with `-l` or `-sl` removes the others. To keep every language but put + your preferred ones first, use `audio.language_priority` and + [`subtitle.language_priority`](../reference/configuration.md#subtitle) in your + configuration file. + The special language tokens `orig`, `all`, and `best` are honoured everywhere a language is expected. ### Title & episode selection diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 8959e208..6686a9d4 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -254,6 +254,26 @@ it is used. You can override each stream type independently: unshackle dl -al en EXAMPLE 81234567 ``` +### Sort order + +`-l` and `-sl` **select** languages: naming some removes the rest. To keep every language +but decide which ones come first, set a priority list in your configuration file: + +```yaml title="unshackle.yaml" +audio: + language_priority: [orig, en] +subtitle: + group_by: language + language_priority: [en, es, fr] +``` + +Subtitles then start with English, Spanish, and French. The languages you leave out follow +alphabetically, after the title's original language. Nothing is removed. Audio works the +same way, but the languages you leave out keep their bitrate and codec order instead of an +alphabetical one. + +See [`subtitle.language_priority`](../reference/configuration.md#subtitle) for the details. + ### Exact vs fuzzy matching By default language matching is fuzzy: `-l en` also accepts `en-US`, `en-GB`, etc. (up diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index d272445a..62d43835 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -376,6 +376,8 @@ subtitle: | `convert_before_strip` | `true` | `true` / `false` | Convert a subtitle to SRT before stripping SDH (for non-SubtitleEdit engines). | | `output_mode` | `mux` | `mux`, `sidecar`, `both` | Whether subtitles are embedded, written as sidecars, or both. | | `sidecar_format` | `srt` | a format name (e.g. `srt`, `vtt`) or `original` | Format used for sidecar files. | +| `group_by` | `type` | `type`, `language` | Whether all tracks of one type stay together, or each language stays next to its own variants. | +| `language_priority` | *(unset)* | a list of languages | Languages to sort to the top. The rest keep the default order. Nothing is removed. | !!! tip "SubtitleEdit is optional but recommended" Several high-fidelity paths (the best-quality conversions and the SubtitleEdit SDH diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 4ededb9c..cab4e4ff 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -378,6 +378,7 @@ written as sidecar files. See [Subtitles](../guide/subtitles.md) for the full gu | `sidecar_format` | str | `"srt"` | Format for sidecar files when `output_mode: sidecar`. | | `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH). Types you leave out fall to the end. | | `group_by` | str | `"type"` | Which key groups the subtitle tracks. `type` puts all forced tracks together, then all normal, then all SDH. `language` keeps each language next to its own variants. | +| `language_priority` | list | *(unset)* | Languages to sort to the top, in the order you give. The rest keep the default order: original language first, then alphabetical. Nothing is removed. | ```yaml subtitle: @@ -405,6 +406,33 @@ subtitle: `type_priority` still sets the order of the variants, so with `group_by: language` it controls the order inside each language. +!!! note "`language_priority` sorts languages, it does not select them" + By default the title's original language comes first and every other language follows + alphabetically. For an English title that gives English, Arabic, Bulgarian, and the + rest. Give `language_priority` the languages you want first: + + ```yaml + subtitle: + group_by: language + language_priority: [en, es, fr] + ``` + + | | Result for an English title | + |---|---| + | Default | English, Arabic, Bulgarian, Spanish, French, Japanese | + | With the list above | English, Spanish, French, Arabic, Bulgarian, Japanese | + + This is only an order. Every other language is still downloaded, which is what makes + it different from `-sl`: `-sl en,es,fr` **removes** all other subtitles. + + Use `orig` in the list for the title's original language. A base tag also covers its + regional forms, so `en` sorts `en-US` and `en-GB`, but `zh` does **not** cover + `zh-Hant`, which you must name. The sort ignores entries that match no track, and it + ignores entries that are not valid language tags. + + `--exact-lang` applies here too: with it, `en` sorts only `en`, and `en-US` falls to + the alphabetical tail with the rest. Name each tag you want when you use that flag. + !!! note "`type_priority` also picks the default subtitle" At mux time the first track in the default subtitle language gets the `default` flag. The built-in order is `forced`, `normal`, `sdh`, which means a forced track becomes @@ -421,13 +449,17 @@ subtitle: - **Type:** `dict`  ·  **Default:** `{}` Audio handling options. The most notable sub-key is **`codec_priority`**, an ordered list of -audio codecs used to break ties in track selection. +audio codecs used to break ties in track selection. **`language_priority`** is an ordered list +of languages that sort to the top of the audio tracks. ```yaml audio: codec_priority: - EAC3 - AC3 + language_priority: + - orig + - en ``` !!! note "`codec_priority` is a soft priority: nothing is dropped" diff --git a/tests/lang_selection/test_matching_languages.py b/tests/lang_selection/test_matching_languages.py new file mode 100644 index 00000000..52a6eb19 --- /dev/null +++ b/tests/lang_selection/test_matching_languages.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Sequence + +import pytest + +from unshackle.core.utilities import matching_languages + +POOL = ["ar", "en", "en-GB", "en-US", "ja"] + + +@pytest.mark.parametrize( + ("language", "available", "exact", "expected"), + [ + ("en", POOL, False, {"en", "en-GB", "en-US"}), + # CLDR rates en/en-US at distance 0, so exact mode leans on the RFC 4647 string preference + ("en", POOL, True, {"en"}), + ("en-GB", POOL, True, {"en-GB"}), + # no string-equal tag, so exact mode falls back to the distance match + ("zh", ["ar", "cmn", "ja"], True, {"cmn"}), + ("en", ["ar", "ja"], False, set()), + ("en", [], False, set()), + ("en", ["ar", None, "en"], True, {"en"}), + # unparseable tags (config typos) must match nothing, never raise + ("engrish", POOL, False, set()), + ("engrish", POOL, True, set()), + ("", POOL, False, set()), + ("", POOL, True, set()), + ("f", POOL, False, set()), + ], +) +def test_matching_languages(language: str, available: Sequence[str], exact: bool, expected: set[str]) -> None: + assert matching_languages(language, available, exact) == expected diff --git a/tests/lang_selection/test_resolve_sort_langs.py b/tests/lang_selection/test_resolve_sort_langs.py new file mode 100644 index 00000000..cb396650 --- /dev/null +++ b/tests/lang_selection/test_resolve_sort_langs.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Optional, Sequence + +import pytest +from langcodes import Language + +from unshackle.core.utilities import resolve_sort_langs + + +@pytest.mark.parametrize( + ("tokens", "original", "expected"), + [ + (["orig", "en", "es"], "ja", ["ja", "en", "es"]), + (["orig", "en", "es"], None, ["en", "es"]), + (["orig"], Language.get("pt-BR"), ["pt-BR"]), + (["en", "orig"], "en", ["en"]), + (["en", "es", "en", "fr"], None, ["en", "es", "fr"]), + (["all", "best", "en"], "ja", ["all", "best", "en"]), + ([], "ja", []), + ], +) +def test_resolve_sort_langs(tokens: Sequence[str], original: Optional[str], expected: list[str]) -> None: + assert resolve_sort_langs(tokens, original) == expected diff --git a/tests/tracks/test_subtitle_sort.py b/tests/tracks/test_subtitle_sort.py index d114eae0..c583b494 100644 --- a/tests/tracks/test_subtitle_sort.py +++ b/tests/tracks/test_subtitle_sort.py @@ -3,8 +3,8 @@ import pytest from unshackle.core.tracks import Subtitle, Tracks -def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) -> Subtitle: - return Subtitle( +def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False, orig: bool = False) -> Subtitle: + track = Subtitle( url=f"https://example.test/{lang}-{forced}-{sdh}-{cc}", language=lang, codec=Subtitle.Codec.SubRip, @@ -12,6 +12,8 @@ def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) -> sdh=sdh, cc=cc, ) + track.is_original_lang = orig + return track def shape(tracks: Tracks) -> list[tuple[str, str]]: @@ -135,3 +137,82 @@ def test_language_grouping_keeps_regional_variants_adjacent_through_sections() - ("fi", "normal"), ("fi", "sdh"), ] + + +def langs(tracks: Tracks) -> list[str]: + return [str(s.language) for s in tracks.subtitles] + + +@pytest.fixture +def many() -> Tracks: + t = Tracks() + t.add([sub(lang, orig=lang == "ja") for lang in ("ar", "bg", "en", "es", "fr", "ja")]) + return t + + +def test_priority_orders_listed_then_rest_alphabetically(many: Tracks) -> None: + many.sort_subtitles(by_language=["en", "es", "fr"], group_by="language") + assert langs(many) == ["en", "es", "fr", "ar", "bg", "ja"] + + +def test_priority_outranks_the_original_language(many: Tracks) -> None: + # "all" resolves to the original language (ja), which must stay below the named ones + many.sort_subtitles(by_language=["en", "es", "all"], group_by="language") + assert langs(many) == ["en", "es", "ja", "ar", "bg", "fr"] + + +def test_priority_matches_regional_variants() -> None: + t = Tracks() + t.add([sub("ar"), sub("en-US"), sub("es-419"), sub("ja")]) + t.sort_subtitles(by_language=["en", "es"], group_by="language") + assert langs(t) == ["en-US", "es-419", "ar", "ja"] + + +def test_priority_overlapping_entries_are_harmless() -> None: + t = Tracks() + t.add([sub("ar"), sub("en-GB"), sub("en-US"), sub("es"), sub("ja")]) + t.sort_subtitles(by_language=["en", "en-US", "es"], group_by="language") + assert langs(t) == ["en-GB", "en-US", "es", "ar", "ja"] + + +def test_no_priority_leaves_order_unchanged(many: Tracks) -> None: + many.sort_subtitles(by_language=[], group_by="language") + assert langs(many) == ["ar", "bg", "en", "es", "fr", "ja"] + + +def test_all_alone_still_floats_the_original_language(many: Tracks) -> None: + many.sort_subtitles(by_language=["all"], group_by="language") + assert langs(many) == ["ja", "ar", "bg", "en", "es", "fr"] + + +@pytest.fixture +def regionals() -> Tracks: + t = Tracks() + t.add([sub("ar"), sub("en"), sub("en-GB"), sub("en-US"), sub("ja")]) + return t + + +def test_fuzzy_priority_floats_regional_variants(regionals: Tracks) -> None: + regionals.sort_subtitles(by_language=["en"], group_by="language") + assert langs(regionals) == ["en", "en-GB", "en-US", "ar", "ja"] + + +def test_exact_priority_floats_only_the_named_tag(regionals: Tracks) -> None: + # CLDR rates en/en-US at distance 0, so exact mode needs the RFC 4647 string preference + # to hold en-US back; without it en-US floats and en-GB does not. + regionals.sort_subtitles(by_language=["en"], group_by="language", exact_match=True) + assert langs(regionals) == ["en", "ar", "en-GB", "en-US", "ja"] + + +def test_exact_priority_still_falls_back_when_no_tag_is_string_equal() -> None: + t = Tracks() + t.add([sub("ar"), sub("cmn"), sub("ja")]) + t.sort_subtitles(by_language=["zh"], group_by="language", exact_match=True) + assert langs(t) == ["cmn", "ar", "ja"] + + +@pytest.mark.parametrize("exact", [False, True]) +def test_unparseable_priority_entry_is_ignored(many: Tracks, exact: bool) -> None: + # a config typo in language_priority must not crash the sort + many.sort_subtitles(by_language=["en", "engrish", "f", ""], group_by="language", exact_match=exact) + assert langs(many) == ["en", "ar", "bg", "es", "fr", "ja"] diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index b4b31193..56b1331b 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -83,6 +83,7 @@ from unshackle.core.utilities import ( is_exact_match, keep_forced_subtitle, log_event, + resolve_sort_langs, suggest_font_packages, time_elapsed_since, ) @@ -1859,8 +1860,12 @@ class dl: song.tracks.add(service.get_tracks(song), warn_only=True) song.tracks.chapters = service.get_chapters(song) song.tracks.sort_audio( - by_language=a_lang or lang, + by_language=resolve_sort_langs( + [*(config.audio.get("language_priority") or []), *(a_lang or lang)], + song.language, + ), codec_priority=config.audio.get("codec_priority"), + exact_match=exact_lang, ) select_music_audio(song) if not song.tracks.audio: @@ -2389,39 +2394,23 @@ class dl: ) with console.status("Sorting tracks by language and bitrate...", spinner="dots"): - video_sort_lang = v_lang or lang - processed_video_sort_lang = [] - for language in video_sort_lang: - if language == "orig": - if title.language: - orig_lang = str(title.language) - if orig_lang not in processed_video_sort_lang: - processed_video_sort_lang.append(orig_lang) - else: - if language not in processed_video_sort_lang: - processed_video_sort_lang.append(language) + audio_priority = config.audio.get("language_priority") or [] + subtitle_priority = config.subtitle.get("language_priority") or [] - audio_sort_lang = a_lang or lang - processed_audio_sort_lang = [] - for language in audio_sort_lang: - if language == "orig": - if title.language: - orig_lang = str(title.language) - if orig_lang not in processed_audio_sort_lang: - processed_audio_sort_lang.append(orig_lang) - else: - if language not in processed_audio_sort_lang: - processed_audio_sort_lang.append(language) - - title.tracks.sort_videos(by_language=processed_video_sort_lang) + title.tracks.sort_videos( + by_language=resolve_sort_langs(v_lang or lang, title.language), + exact_match=exact_lang, + ) title.tracks.sort_audio( - by_language=processed_audio_sort_lang, + by_language=resolve_sort_langs([*audio_priority, *(a_lang or lang)], title.language), codec_priority=config.audio.get("codec_priority"), + exact_match=exact_lang, ) title.tracks.sort_subtitles( - by_language=s_lang, + by_language=resolve_sort_langs([*subtitle_priority, *s_lang], title.language), type_priority=config.subtitle.get("type_priority"), group_by=config.subtitle.get("group_by"), + exact_match=exact_lang, ) if list_: diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 02bb6d9d..268fe3e3 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -23,7 +23,7 @@ from unshackle.core.tracks.chapters import Chapter, Chapters from unshackle.core.tracks.subtitle import Subtitle from unshackle.core.tracks.track import Track from unshackle.core.tracks.video import Video -from unshackle.core.utilities import is_close_match, log_event, sanitize_filename +from unshackle.core.utilities import is_close_match, log_event, matching_languages, sanitize_filename from unshackle.core.utils.collections import as_list, flatten @@ -243,7 +243,9 @@ class Tracks: if duplicates: log.debug(f" - Found and skipped {duplicates} duplicate tracks...") - def sort_videos(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: + def sort_videos( + self, by_language: Optional[Sequence[Union[str, Language]]] = None, exact_match: bool = False + ) -> None: """Sort video tracks by resolution then bitrate, and optionally language.""" if not self.videos: return @@ -255,13 +257,15 @@ class Tracks: language = next((x.language for x in self.videos if x.is_original_lang), "") if not language: continue + wanted = matching_languages(language, [x.language for x in self.videos], exact_match) self.videos.sort(key=lambda x: str(x.language)) - self.videos.sort(key=lambda x: not is_close_match(language, [x.language])) + self.videos.sort(key=lambda x: str(x.language) not in wanted) def sort_audio( self, by_language: Optional[Sequence[Union[str, Language]]] = None, codec_priority: Optional[Sequence[str]] = None, + exact_match: bool = False, ) -> None: """Sort audio tracks by bitrate, codec priority, Atmos, descriptive, and optionally language.""" if not self.audio: @@ -283,13 +287,15 @@ class Tracks: language = next((x.language for x in self.audio if x.is_original_lang), "") if not language: continue - self.audio.sort(key=lambda x: not is_close_match(language, [x.language])) + wanted = matching_languages(language, [x.language for x in self.audio], exact_match) + self.audio.sort(key=lambda x: str(x.language) not in wanted) def sort_subtitles( self, by_language: Optional[Sequence[Union[str, Language]]] = None, type_priority: Optional[Sequence[str]] = None, group_by: Optional[str] = None, + exact_match: bool = False, ) -> None: """ Sort subtitle tracks by various track attributes to a common P2P standard. @@ -313,6 +319,9 @@ class Tracks: then every normal, then every SDH, each block ascending by language. "language" groups by language instead, so Finnish sits next to Finnish SDH, with the Type Order applied inside each language. + + exact_match makes a by_language entry sort only its own tag. By default "en" also + sorts "en-US" and "en-GB". """ if not self.subtitles: return @@ -346,7 +355,8 @@ class Tracks: language = next((x.language for x in self.subtitles if x.is_original_lang), "") if not language: continue - self.subtitles.sort(key=lambda x: is_close_match(language, [x.language]), reverse=True) + wanted = matching_languages(language, [x.language for x in self.subtitles], exact_match) + self.subtitles.sort(key=lambda x: str(x.language) in wanted, reverse=True) def select_video(self, x: Callable[[Video], bool]) -> None: self.videos = list(filter(x, self.videos)) diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 3b008ce3..90c5e500 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -25,6 +25,7 @@ import pycountry from construct import ValidationError from fontTools import ttLib from langcodes import Language, closest_match +from langcodes.tag_parser import LanguageTagError from pymp4.parser import Box from unidecode import unidecode @@ -195,6 +196,49 @@ def as_requested(tokens: Sequence[str], orig_token: Optional[str]) -> str: return ", ".join("orig" if tok == orig_token else tok for tok in tokens) +def matching_languages( + language: Union[str, Language], + available: Sequence[Union[str, Language, None]], + exact: bool = False, +) -> set[str]: + """ + The languages out of `available` that `language` asks for, by the rules Tracks.by_language uses. + + In exact mode CLDR tag distance is not sufficient on its own: it rates a base language and + its "paradigm" regional variant as the same language (distance 0 for en/en-US, pt/pt-BR), so + it cannot separate en-US from en. Follow RFC 4647 Lookup: prefer the string-equal tag, and + fall back to the distance match only when no such tag is present (e.g. zh matches cmn). + """ + tags = {str(x) for x in available if x} + try: + want = Language.get(str(language)).to_tag().casefold() + except LanguageTagError: + # e.g. a config typo in language_priority; an unparseable tag matches nothing + return set() + if not exact: + return {tag for tag in tags if is_close_match(language, [tag])} + string_equal = {tag for tag in tags if Language.get(tag).to_tag().casefold() == want} + return string_equal or {tag for tag in tags if is_exact_match(language, [tag])} + + +def resolve_sort_langs(tokens: Sequence[str], original: Optional[Union[str, Language]] = None) -> list[str]: + """ + Prepare language tokens for a Tracks.sort_* by_language argument. + + "orig" becomes the title language, or is dropped when the title has none. "all" and + "best" pass through, because the sort methods resolve them against the tracks. The + result keeps the first position of each language and drops later repeats. + """ + resolved: list[str] = [] + for token in tokens: + language = str(original) if token == "orig" else str(token) + if token == "orig" and not original: + continue + if language not in resolved: + resolved.append(language) + return resolved + + def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore """ Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find. From a5f5e7d14da884e1a8ef825dfe199b069428cec1 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Mon, 3 Aug 2026 10:28:24 -0600 Subject: [PATCH 33/40] refactor(tracks): honour the best token and share one match rule sort_subtitles resolved only the all token, while the video and audio sorts resolved all and best. by_language now asks matching_languages rather than repeating the RFC 4647 preference, with no behaviour change. --- tests/tracks/test_subtitle_sort.py | 11 +++++++++-- unshackle/core/tracks/tracks.py | 20 +++++--------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/tracks/test_subtitle_sort.py b/tests/tracks/test_subtitle_sort.py index c583b494..02170415 100644 --- a/tests/tracks/test_subtitle_sort.py +++ b/tests/tracks/test_subtitle_sort.py @@ -180,11 +180,18 @@ def test_no_priority_leaves_order_unchanged(many: Tracks) -> None: assert langs(many) == ["ar", "bg", "en", "es", "fr", "ja"] -def test_all_alone_still_floats_the_original_language(many: Tracks) -> None: - many.sort_subtitles(by_language=["all"], group_by="language") +@pytest.mark.parametrize("token", ["all", "best"]) +def test_magic_token_alone_floats_the_original_language(many: Tracks, token: str) -> None: + many.sort_subtitles(by_language=[token], group_by="language") assert langs(many) == ["ja", "ar", "bg", "en", "es", "fr"] +@pytest.mark.parametrize("token", ["all", "best"]) +def test_priority_outranks_either_magic_token(many: Tracks, token: str) -> None: + many.sort_subtitles(by_language=["en", "es", token], group_by="language") + assert langs(many) == ["en", "es", "ja", "ar", "bg", "fr"] + + @pytest.fixture def regionals() -> Tracks: t = Tracks() diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 268fe3e3..faa5ce26 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -7,7 +7,7 @@ from functools import partial from pathlib import Path from typing import Any, Callable, Iterator, Optional, Sequence, Union -from langcodes import Language, closest_supported_match +from langcodes import Language from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn from rich.table import Table from rich.tree import Tree @@ -15,7 +15,7 @@ from rich.tree import Tree from unshackle.core import binaries from unshackle.core.config import config from unshackle.core.console import GradientPulseBarColumn, console -from unshackle.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE, AnyTrack, TrackT +from unshackle.core.constants import AnyTrack, TrackT from unshackle.core.events import events from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.audio import Audio @@ -351,7 +351,7 @@ class Tracks: by_type() # sections for language in reversed(by_language or []): - if str(language) == "all": + if str(language) in ("all", "best"): language = next((x.language for x in self.subtitles if x.is_original_lang), "") if not language: continue @@ -469,21 +469,11 @@ class Tracks: def by_language( tracks: list[TrackT], languages: list[str], per_language: int = 0, exact_match: bool = False ) -> list[TrackT]: - distance = LANGUAGE_EXACT_DISTANCE if exact_match else LANGUAGE_MAX_DISTANCE selected: list[TrackT] = [] seen_ids: set[str] = set() for language in languages: - matches = [x for x in tracks if closest_supported_match(str(x.language), [language], distance)] - if exact_match and len(matches) > 1: - # CLDR tag_distance measures intelligibility, not tag identity: it rates a base - # language and its "paradigm" regional variant as the same language (distance 0 - # for ar/ar-EG, en/en-US, pt/pt-BR), so exact mode alone cannot separate them. - # Follow RFC 4647 Lookup: prefer the most specific (string-equal) tag, and fall - # back to the fuzzy match only when none exists (e.g. zh matches cmn). - want = Language.get(language).to_tag().casefold() - exact_hits = [x for x in matches if Language.get(str(x.language)).to_tag().casefold() == want] - if exact_hits: - matches = exact_hits + wanted = matching_languages(language, [x.language for x in tracks], exact_match) + matches = [x for x in tracks if str(x.language) in wanted] # Overlapping tags can still resolve to the same physical track; dedupe by id so # callers never get duplicate tracks. for track in matches[: per_language or None]: From eb40c67385ddbde0651d29c5b3786b0206e7220e Mon Sep 17 00:00:00 2001 From: hifumi <supersonik@gmx.de> Date: Tue, 4 Aug 2026 16:20:24 +0200 Subject: [PATCH 34/40] feat(vaults): support custom headers on API vault --- docs/guide/vaults.md | 4 ++++ unshackle/vaults/API.py | 44 +++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/docs/guide/vaults.md b/docs/guide/vaults.md index f62926b8..eedeca51 100644 --- a/docs/guide/vaults.md +++ b/docs/guide/vaults.md @@ -208,6 +208,9 @@ key_vaults: name: myapi uri: https://vault.example.com/api token: "your-bearer-token" + headers: + CF-Access-Client-Id: "CLIENT_ID_HERE" + CF-Access-Client-Secret: "CLIENT_SECRET_HERE" ``` | Field | Required | Description | @@ -218,6 +221,7 @@ key_vaults: | `token` | yes | Bearer token, sent as `Authorization: Bearer <token>`. | | `no_push` | no | If `true`, keys are read but never written. | | `timeout` | no | Request timeout in seconds; defaults to `vault_timeout`. | +| `headers` | no | Extra HTTP `headers` sent with every request, for example the Cloudflare Access service-token headers `CF-Access-Client-Id` and `CF-Access-Client-Secret`. | The backend talks to endpoints under `uri` (for example `GET {uri}/{service}/{kid}` to look up a single key) and interprets a numeric `code` field in each JSON response to detect errors such as an invalid token, rate limiting, or an invalid service tag. diff --git a/unshackle/vaults/API.py b/unshackle/vaults/API.py index 257313d3..25dd73b8 100644 --- a/unshackle/vaults/API.py +++ b/unshackle/vaults/API.py @@ -10,7 +10,7 @@ from unshackle.core.vault import Vault class API(Vault): """Key Vault using a simple RESTful HTTP API call.""" - def __init__(self, name: str, uri: str, token: str, no_push: bool = False, timeout: float = 10.0): + def __init__(self, name: str, uri: str, token: str, no_push: bool = False, headers: Optional[dict[str, str]] = None, timeout: float = 10.0): super().__init__(name, no_push) self.uri = uri.rstrip("/") self.timeout = timeout @@ -18,13 +18,21 @@ class API(Vault): self.session.headers.update({"User-Agent": f"unshackle v{__version__}"}) self.session.headers.update({"Authorization": f"Bearer {token}"}) + if headers: + self.session.headers.update(headers) + def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: if isinstance(kid, UUID): kid = kid.hex - data = self.session.get( - url=f"{self.uri}/{service.lower()}/{kid}", headers={"Accept": "application/json"}, timeout=self.timeout - ).json() + response = self.session.get( + url=f"{self.uri}/{service.lower()}/{kid}", + headers={"Accept": "application/json"}, + timeout=self.timeout, + allow_redirects=False + ) + response.raise_for_status() + data = response.json() code = int(data.get("code", 0)) message = data.get("message") @@ -52,12 +60,15 @@ class API(Vault): page = 1 while True: - data = self.session.get( + response = self.session.get( url=f"{self.uri}/{service.lower()}", params={"page": page, "total": 10}, headers={"Accept": "application/json"}, timeout=self.timeout, - ).json() + allow_redirects=False + ) + response.raise_for_status() + data = response.json() code = int(data.get("code", 0)) message = data.get("message") @@ -90,12 +101,15 @@ class API(Vault): if isinstance(kid, UUID): kid = kid.hex - data = self.session.post( + response = self.session.post( url=f"{self.uri}/{service.lower()}/{kid}", json={"content_key": key}, headers={"Accept": "application/json"}, timeout=self.timeout, - ).json() + allow_redirects=False + ) + response.raise_for_status() + data = response.json() code = int(data.get("code", 0)) message = data.get("message") @@ -141,6 +155,7 @@ class API(Vault): json={"content_keys": batch_keys}, headers={"Accept": "application/json"}, timeout=self.timeout, + allow_redirects=False ) # Check for HTTP errors that suggest batch is too large @@ -150,7 +165,8 @@ class API(Vault): else: batch_size = 1 continue - + + response.raise_for_status() data = response.json() except Exception: # JSON decode error or connection issue - try smaller batch @@ -187,7 +203,14 @@ class API(Vault): return total_added def get_services(self) -> Iterator[str]: - data = self.session.post(url=self.uri, headers={"Accept": "application/json"}, timeout=self.timeout).json() + response = self.session.post( + url=self.uri, + headers={"Accept": "application/json"}, + timeout=self.timeout, + allow_redirects=False + ) + response.raise_for_status() + data = response.json() code = int(data.get("code", 0)) message = data.get("message") @@ -227,3 +250,4 @@ class Exceptions: class ContentKeyInvalid(Exception): """The Content Key is invalid.""" + \ No newline at end of file From 051f1e668f62f267b9e443a2cc5afed512473973 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Tue, 4 Aug 2026 08:22:02 -0600 Subject: [PATCH 35/40] style(vaults): fix formatting from #147 --- unshackle/vaults/API.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/unshackle/vaults/API.py b/unshackle/vaults/API.py index 25dd73b8..bb3debae 100644 --- a/unshackle/vaults/API.py +++ b/unshackle/vaults/API.py @@ -10,7 +10,15 @@ from unshackle.core.vault import Vault class API(Vault): """Key Vault using a simple RESTful HTTP API call.""" - def __init__(self, name: str, uri: str, token: str, no_push: bool = False, headers: Optional[dict[str, str]] = None, timeout: float = 10.0): + def __init__( + self, + name: str, + uri: str, + token: str, + no_push: bool = False, + headers: Optional[dict[str, str]] = None, + timeout: float = 10.0, + ): super().__init__(name, no_push) self.uri = uri.rstrip("/") self.timeout = timeout @@ -29,7 +37,7 @@ class API(Vault): url=f"{self.uri}/{service.lower()}/{kid}", headers={"Accept": "application/json"}, timeout=self.timeout, - allow_redirects=False + allow_redirects=False, ) response.raise_for_status() data = response.json() @@ -65,7 +73,7 @@ class API(Vault): params={"page": page, "total": 10}, headers={"Accept": "application/json"}, timeout=self.timeout, - allow_redirects=False + allow_redirects=False, ) response.raise_for_status() data = response.json() @@ -106,7 +114,7 @@ class API(Vault): json={"content_key": key}, headers={"Accept": "application/json"}, timeout=self.timeout, - allow_redirects=False + allow_redirects=False, ) response.raise_for_status() data = response.json() @@ -155,7 +163,7 @@ class API(Vault): json={"content_keys": batch_keys}, headers={"Accept": "application/json"}, timeout=self.timeout, - allow_redirects=False + allow_redirects=False, ) # Check for HTTP errors that suggest batch is too large @@ -165,7 +173,7 @@ class API(Vault): else: batch_size = 1 continue - + response.raise_for_status() data = response.json() except Exception: @@ -204,10 +212,7 @@ class API(Vault): def get_services(self) -> Iterator[str]: response = self.session.post( - url=self.uri, - headers={"Accept": "application/json"}, - timeout=self.timeout, - allow_redirects=False + url=self.uri, headers={"Accept": "application/json"}, timeout=self.timeout, allow_redirects=False ) response.raise_for_status() data = response.json() @@ -250,4 +255,3 @@ class Exceptions: class ContentKeyInvalid(Exception): """The Content Key is invalid.""" - \ No newline at end of file From 3e0d7b5f75e741323a896ce14c6508cdd609fb13 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Tue, 4 Aug 2026 09:42:42 -0600 Subject: [PATCH 36/40] feat(dl): add tag_rules to conditionally swap the group tag --- docs/guide/output-and-naming.md | 5 + docs/reference/configuration.md | 102 ++++++++++- tests/core/test_tag_rules.py | 223 +++++++++++++++++++++++ tests/core/test_tag_rules_integration.py | 131 +++++++++++++ unshackle/commands/dl.py | 6 +- unshackle/core/config.py | 2 + unshackle/core/titles/title.py | 10 + unshackle/core/utils/tag_rules.py | 116 ++++++++++++ unshackle/unshackle-example.yaml | 6 + 9 files changed, 598 insertions(+), 3 deletions(-) create mode 100644 tests/core/test_tag_rules.py create mode 100644 tests/core/test_tag_rules_integration.py create mode 100644 unshackle/core/utils/tag_rules.py diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 5b8b1647..e9c8d110 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -102,6 +102,7 @@ Every variable below is valid in both output and folder templates. Values are de | Variable | Meaning | Example | |---|---|---| | `title` | Title name (movie/show/song name; `$` is rendered as `S`) | `The Show` | +| `title_type` | Media kind of the title | `movie`, `series`, `music` | | `year` | Release year | `2024` | | `source` | Service tag / class name (empty with `--no-source`) | `EXAMPLE` | | `quality` | Resolution with scan suffix | `1080p`, `2160p`, `576i` | @@ -234,6 +235,10 @@ The `--no-mux` flag skips muxing entirely and writes the individual track files, ## Tags & group naming The release-group tag is the `-TAG` portion at the end of scene-style names, produced by the `{tag}` variable. +It can be swapped per release with [`tag_rules`](../reference/configuration.md#tag_rules): ordered rules that +match on the release-attribute variables only (`quality`, `resolution`, `hdr`, `source`, `lang_tag`, +`title_type`, and so on), first match wins. The title's own fields, such as `title` and `season_episode`, +cannot be matched. See the [reference page](../reference/configuration.md#tag_rules) for the exact list. ```yaml title="unshackle.yaml" tag: "MYGROUP" diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index cab4e4ff..51423ae3 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -35,7 +35,7 @@ This page assumes you already know the basics and want precise details. | [Directories](#directories) | `directories`, `filenames` | | [Services & authentication](#services-auth) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` | | [Download & processing](#download-processing) | `dl`, `subtitle`, `audio`, `muxing`, `language_tags`, `dual_multi_mode` | -| [Output & naming](#output-naming) | `output_template`, `tag`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` | +| [Output & naming](#output-naming) | `output_template`, `tag`, `tag_rules`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` | | [DRM & CDM](#drm-cdm) | `cdm`, `remote_cdm`, `decryption` | | [Network & proxy](#network-proxy) | `network`, `headers`, `proxy_providers` | | [Key vaults](#key-vaults) | `key_vaults`, `vault_timeout` | @@ -584,7 +584,8 @@ folder-kind keys. `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, `album_artist`, `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, `explicit`, `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, `dual`, - `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`. + `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`, + `title_type`. ### Tagging & naming keys @@ -596,6 +597,103 @@ folder-kind keys. | `chapter_fallback_name` | str | `""` | Fallback chapter-name template, e.g. `"Chapter {i:02}"`. | | `unicode_filenames` | bool | `false` | Allow Unicode in filenames; when `false`, names are ASCII-sanitised. | +### `tag_rules` + +- **Type:** `list`  ·  **Default:** `[]` + +Conditionally replaces [`tag`](#output-naming) for a given download. Each rule pairs a `when` +dict of conditions with the `tag` to use when they hold. Rules are evaluated in order and the +first match wins; if no rule matches, the global `tag` is kept. + +```yaml title="unshackle.yaml" +tag_rules: + - when: { title_type: movie, lang_tag: SUBBED } + tag: SUBMOVIES + - when: { quality: 2160p, hdr: [DV, DV.HDR, DV.HDR10P, HDR, HDR10P] } + tag: UHDGROUP +``` + +Rule semantics: + +- Every condition in `when` must hold (they are AND-ed). +- A value may be a scalar or a list; a list matches if **any** entry matches. +- Comparison is case-insensitive string equality. +- The keys of `when` are the filename template variables that describe the release itself: + `title_type`, `quality`, `resolution`, `source`, `video`, `hdr`, `hfr`, `audio`, + `audio_channels`, `audio_full`, `atmos`, `dual`, `multi`, `dubbed`, `edition`, `repack`, + and `lang_tag`. Only these are available. The title's own fields (`title`, `year`, `season`, + `episode`, and so on) are added after the rules run and cannot be matched. +- An unknown key in `when` is a mistake: unshackle logs a warning that names the key and skips + that rule. A rule without conditions or without a `tag` is also warned about and skipped. + +Put the more specific rules first, because the first match ends the search. + +!!! warning "Match `hdr` on the exact value" + Matching is exact, and `hdr` is a composite label for Dolby Vision titles: a DV title with + an HDR10 base layer gives `DV.HDR`, and with an HDR10+ base layer `DV.HDR10P`. Plain `DV` + does **not** match those. List every variant you want, as the example above does. Note + also that HDR10 gives `HDR`, not `HDR10`. The full set of values is `HDR`, `HDR10P`, `DV`, + `DV.HDR`, `DV.HDR10P`, and `HLG`, as listed in the + [variable table](../guide/output-and-naming.md#template-variables). + +#### Operators + +A condition value may start with a comparison operator, which is applied to the rest of the +value: + +```yaml title="unshackle.yaml" +tag_rules: + - when: { resolution: "<1080" } + tag: SDGROUP + - when: { quality: ">=2160" } + tag: UHDGROUP + - when: { video: "!=H.265" } + tag: AVCGROUP +``` + +- `>`, `<`, `>=`, `<=` compare numbers. unshackle reads the first number out of the actual + value, so `">=2160"` matches a quality of `2160p`. If the actual value has no number, the + condition fails. +- `=`, `==`, and `!=` compare text, case-insensitively, when the operand is not a number. + With a numeric operand they compare numbers like the operators above. A value with no number + in it is never equal to a number, so `"!=2160"` holds for it and `"==2160"` does not. +- An ordering operator with a text operand (for example `">DV"`) is a mistake: unshackle logs + a warning and the condition fails. So does an operator with nothing after it (`">="`). +- A value with no operator keeps the plain equality behaviour described above. + +!!! warning "Quote operator values in YAML" + Two of these operators break an unquoted YAML value, and both stop the config from + loading: + + - `>` opens a folded block scalar, so `quality: >=2160` fails with a "while scanning a + block scalar" error. + - `!` starts a tag, so `hdr: !=DV` fails with a "could not determine a constructor" + error. + + Quote every operator value: `quality: ">=2160"`, `resolution: "<1080"`, `video: "!=H.265"`. + +!!! note "Keep a negation out of lists" + A list matches if any entry matches, so two or more `!=` entries in one list are always + true. Give a negation as a single value, not as a list entry. + +`title_type` is the media kind of the title: `movie`, `series`, or `music`. It is useful to +give each kind its own group: + +```yaml +tag_rules: + - when: { title_type: movie } + tag: MOVIEGRP + - when: { title_type: series } + tag: TVGRP +``` + +The resolved tag applies to the `{tag}` filename variable only. The `Group` metadata tag +written into MKV and music files always uses [`tag`](#output-naming). + +!!! note "`--tag` wins" + An explicit `dl --tag` on the command line disables the rules completely and uses the + given tag. + --- ## DRM & CDM { #drm-cdm } diff --git a/tests/core/test_tag_rules.py b/tests/core/test_tag_rules.py new file mode 100644 index 00000000..7b8549b1 --- /dev/null +++ b/tests/core/test_tag_rules.py @@ -0,0 +1,223 @@ +"""Tests for the tag_rules engine behind conditional release-group tags.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from unshackle.core.utils.tag_rules import evaluate_tag_rules + +CONTEXT: dict[str, Any] = { + "tag": "NOGRP", + "title_type": "movie", + "quality": "2160p", + "resolution": "2160", + "hdr": "DV", + "lang_tag": "SUBBED", + "edition": "", +} + + +def test_first_match_wins() -> None: + rules = [ + {"when": {"quality": "2160p"}, "tag": "FIRST"}, + {"when": {"title_type": "movie"}, "tag": "SECOND"}, + ] + assert evaluate_tag_rules(rules, CONTEXT) == "FIRST" + + +def test_list_matches_any_entry() -> None: + rules = [{"when": {"hdr": ["HDR10P", "DV"]}, "tag": "UHDGROUP"}] + assert evaluate_tag_rules(rules, CONTEXT) == "UHDGROUP" + + +def test_list_with_no_matching_entry() -> None: + rules = [{"when": {"hdr": ["HDR10P", "HLG"]}, "tag": "UHDGROUP"}] + assert evaluate_tag_rules(rules, CONTEXT) is None + + +def test_case_insensitive() -> None: + rules = [{"when": {"lang_tag": "subbed", "quality": "2160P"}, "tag": "SUBMOVIES"}] + assert evaluate_tag_rules(rules, CONTEXT) == "SUBMOVIES" + + +def test_conditions_are_anded() -> None: + rules = [{"when": {"title_type": "movie", "quality": "1080p"}, "tag": "NOPE"}] + assert evaluate_tag_rules(rules, CONTEXT) is None + + +def test_unknown_key_skips_rule_but_later_rules_match(caplog: pytest.LogCaptureFixture) -> None: + rules = [ + {"when": {"bogus": "value"}, "tag": "SKIPPED"}, + {"when": {"title_type": "movie"}, "tag": "MATCHED"}, + ] + assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED" + assert "bogus" in caplog.text + + +@pytest.mark.parametrize( + "rule", + [ + {"when": {"title_type": "movie"}}, + {"tag": "NOCONDITIONS"}, + {"when": {}, "tag": "EMPTY"}, + ], +) +def test_malformed_rule_skipped(rule: dict[str, Any]) -> None: + assert evaluate_tag_rules([rule], CONTEXT) is None + + +def test_no_rules_and_no_match() -> None: + assert evaluate_tag_rules([], CONTEXT) is None + assert evaluate_tag_rules([{"when": {"quality": "720p"}, "tag": "SD"}], CONTEXT) is None + + +def test_empty_context_value_matches_empty_condition() -> None: + rules = [{"when": {"edition": ""}, "tag": "PLAIN"}] + assert evaluate_tag_rules(rules, CONTEXT) == "PLAIN" + + +@pytest.mark.parametrize( + ("condition", "expected"), + [ + (">=2160", True), + (">=4320", False), + ("<=2160", True), + ("<1080", False), + (">1080", True), + ("==2160", True), + ("=2160", True), + ("!=1080", True), + ("!=2160", False), + (">= 2160", True), + ], +) +def test_numeric_operators_extract_first_number(condition: str, expected: bool) -> None: + """The actual value is "2160p", so the suffix must not defeat the comparison.""" + rules = [{"when": {"quality": condition}, "tag": "MATCHED"}] + assert (evaluate_tag_rules(rules, CONTEXT) == "MATCHED") is expected + + +def test_string_inequality() -> None: + assert evaluate_tag_rules([{"when": {"hdr": "!=DV"}, "tag": "NODV"}], CONTEXT) is None + assert evaluate_tag_rules([{"when": {"hdr": "!=HDR10P"}, "tag": "NOHDR10P"}], CONTEXT) == "NOHDR10P" + + +def test_string_equality_operator_is_case_insensitive() -> None: + assert evaluate_tag_rules([{"when": {"lang_tag": "==subbed"}, "tag": "SUBGRP"}], CONTEXT) == "SUBGRP" + + +def test_plain_equality_unchanged() -> None: + assert evaluate_tag_rules([{"when": {"quality": "2160p"}, "tag": "UHD"}], CONTEXT) == "UHD" + assert evaluate_tag_rules([{"when": {"quality": "2160"}, "tag": "UHD"}], CONTEXT) is None + + +def test_numeric_operator_against_value_without_a_number() -> None: + rules = [{"when": {"hdr": ">=1080"}, "tag": "NOPE"}] + assert evaluate_tag_rules(rules, CONTEXT) is None + + +def test_ordering_operator_with_non_numeric_operand(caplog: pytest.LogCaptureFixture) -> None: + rules = [{"when": {"hdr": ">DV"}, "tag": "NOPE"}] + assert evaluate_tag_rules(rules, CONTEXT) is None + assert "numeric operand" in caplog.text + + +def test_operators_inside_lists() -> None: + rules = [{"when": {"quality": ["<720", ">=2160"]}, "tag": "MATCHED"}] + assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED" + rules = [{"when": {"quality": ["<720", "1080p"]}, "tag": "NOPE"}] + assert evaluate_tag_rules(rules, CONTEXT) is None + + +@pytest.mark.parametrize("rules", ["UHDGROUP", {"when": {"quality": "2160p"}, "tag": "UHD"}, None, 42]) +def test_rules_that_are_not_a_list(rules: Any) -> None: + """A mapping or string here iterates keys/characters, so it must be rejected outright.""" + assert evaluate_tag_rules(rules, CONTEXT) is None + + +@pytest.mark.parametrize("rule", [None, "UHDGROUP", 42, ["when", "tag"]]) +def test_rule_entry_that_is_not_a_mapping_skipped(rule: Any) -> None: + assert evaluate_tag_rules([rule], CONTEXT) is None + assert evaluate_tag_rules([rule, {"when": {"title_type": "movie"}, "tag": "MATCHED"}], CONTEXT) == "MATCHED" + + +@pytest.mark.parametrize("when", [["title_type", "movie"], "title_type", 42]) +def test_when_that_is_not_a_mapping_skipped(when: Any) -> None: + assert evaluate_tag_rules([{"when": when, "tag": "NOPE"}], CONTEXT) is None + + +@pytest.mark.parametrize( + "rule", + [ + {"when": {"atmos": True}, "tag": "ATMOS"}, + {"when": {"atmos": [True, "Atmos"]}, "tag": "ATMOS"}, + {"when": {"title_type": "movie"}, "tag": True}, + ], +) +def test_yaml_boolean_skipped(rule: dict[str, Any], caplog: pytest.LogCaptureFixture) -> None: + """`atmos: true` / `tag: yes` are YAML coercions the user meant as text.""" + assert evaluate_tag_rules([rule], CONTEXT) is None + assert "boolean" in caplog.text + + +def test_numbers_stay_valid() -> None: + assert evaluate_tag_rules([{"when": {"resolution": 2160}, "tag": "UHD"}], CONTEXT) == "UHD" + assert evaluate_tag_rules([{"when": {"resolution": 1080}, "tag": "NOPE"}], CONTEXT) is None + + +def test_numeric_tag_becomes_text() -> None: + assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": 123}], CONTEXT) == "123" + + +@pytest.mark.parametrize("tag", [["A", "B"], {"x": 1}, ("A",), {"A"}]) +def test_non_scalar_tag_skipped(tag: Any, caplog: pytest.LogCaptureFixture) -> None: + """A list or dict here used to reach the filename as its repr.""" + assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": tag}], CONTEXT) is None + assert "must be text or a number" in caplog.text + + +@pytest.mark.parametrize("tag", ["", 0]) +def test_empty_tag_skipped(tag: Any) -> None: + assert evaluate_tag_rules([{"when": {"title_type": "movie"}, "tag": tag}], CONTEXT) is None + + +def test_skipped_rule_falls_through_to_a_later_one() -> None: + rules = [ + {"when": {"atmos": True}, "tag": "SKIPPED"}, + {"when": {"title_type": "movie"}, "tag": "MATCHED"}, + ] + assert evaluate_tag_rules(rules, CONTEXT) == "MATCHED" + + +def test_several_negations_in_a_list_always_match() -> None: + """Any-entry matching makes a list of negations useless, which the docs warn about.""" + rules = [{"when": {"hdr": ["!=DV", "!=HLG"]}, "tag": "ALWAYS"}] + assert evaluate_tag_rules(rules, CONTEXT) == "ALWAYS" + + +def test_leading_space_before_an_operator_is_literal() -> None: + """Current behaviour: the operator is only found at the very start of the value.""" + rules = [{"when": {"quality": " >=2160"}, "tag": "NOPE"}] + assert evaluate_tag_rules(rules, CONTEXT) is None + + +def test_inequality_holds_for_a_value_with_no_number() -> None: + """ "DV" really is not equal to 2160, so != holds while the others fail.""" + assert evaluate_tag_rules([{"when": {"hdr": "!=2160"}, "tag": "MATCHED"}], CONTEXT) == "MATCHED" + assert evaluate_tag_rules([{"when": {"hdr": ">=2160"}, "tag": "NOPE"}], CONTEXT) is None + assert evaluate_tag_rules([{"when": {"hdr": "==2160"}, "tag": "NOPE"}], CONTEXT) is None + + +@pytest.mark.parametrize("condition", ["!=", ">=", "<", "=", "=="]) +def test_bare_operator_fails(condition: str, caplog: pytest.LogCaptureFixture) -> None: + assert evaluate_tag_rules([{"when": {"hdr": condition}, "tag": "NOPE"}], CONTEXT) is None + assert "no value to compare against" in caplog.text + + +def test_composite_hdr_does_not_match_plain_dv() -> None: + """DV titles carry a base-layer suffix, which the docs tell users to list explicitly.""" + context = {**CONTEXT, "hdr": "DV.HDR10P"} + assert evaluate_tag_rules([{"when": {"hdr": "DV"}, "tag": "NOPE"}], context) is None + assert evaluate_tag_rules([{"when": {"hdr": ["DV", "DV.HDR10P"]}, "tag": "DOLBY"}], context) == "DOLBY" diff --git a/tests/core/test_tag_rules_integration.py b/tests/core/test_tag_rules_integration.py new file mode 100644 index 00000000..32e3e5fd --- /dev/null +++ b/tests/core/test_tag_rules_integration.py @@ -0,0 +1,131 @@ +"""Tests for tag_rules reaching the filename context through Title.""" + +from __future__ import annotations + +import inspect +import types +from typing import Any, Iterator + +import pytest +from langcodes import Language + +from unshackle.core.config import config +from unshackle.core.titles.movie import Movie + + +class DummyService: + pass + + +class TrackStub: + """Stand-in for Tracks: truthy and iterable, with the two lists lang_tag reads.""" + + def __init__(self, audio: list[Any], subtitles: list[Any]) -> None: + self.audio = audio + self.subtitles = subtitles + + def __iter__(self) -> Iterator[Any]: + return iter([]) + + def __bool__(self) -> bool: + return True + + +def make_audio(language: str) -> types.SimpleNamespace: + return types.SimpleNamespace( + language=language, + format="AAC", + channel_layout=None, + channellayout_original=None, + channel_s=2, + channels=2, + format_additionalfeatures=None, + joc=None, + ) + + +def make_media_info(languages: list[str]) -> types.SimpleNamespace: + return types.SimpleNamespace(video_tracks=[], audio_tracks=[make_audio(lang) for lang in languages]) + + +def make_movie() -> Movie: + return Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language="en") + + +@pytest.fixture(autouse=True) +def clean_config(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(config, "tag", "NOGRP") + monkeypatch.setattr(config, "tag_rules", []) + monkeypatch.setattr(config, "language_tags", {}) + + +def context_for(rules: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch) -> dict: + monkeypatch.setattr(config, "tag_rules", rules) + return make_movie()._build_base_template_context(make_media_info(["en"])) + + +def test_matching_rule_replaces_the_tag(monkeypatch: pytest.MonkeyPatch) -> None: + context = context_for([{"when": {"title_type": "movie"}, "tag": "MOVIEGRP"}], monkeypatch) + assert context["tag"] == "MOVIEGRP" + + +def test_no_matching_rule_keeps_the_config_tag(monkeypatch: pytest.MonkeyPatch) -> None: + context = context_for([{"when": {"title_type": "series"}, "tag": "TVGRP"}], monkeypatch) + assert context["tag"] == "NOGRP" + + +def test_no_rules_keeps_the_config_tag(monkeypatch: pytest.MonkeyPatch) -> None: + context = context_for([], monkeypatch) + assert context["tag"] == "NOGRP" + + +def test_title_type_is_in_the_context(monkeypatch: pytest.MonkeyPatch) -> None: + assert context_for([], monkeypatch)["title_type"] == "movie" + + +def test_rules_run_after_lang_tag(monkeypatch: pytest.MonkeyPatch) -> None: + """A rule matching on lang_tag only works if lang_tag is resolved first.""" + monkeypatch.setattr(config, "language_tags", {"rules": [{"audio": "ja", "tag": "SUBBED"}]}) + monkeypatch.setattr(config, "tag_rules", [{"when": {"lang_tag": "SUBBED"}, "tag": "SUBGRP"}]) + + movie = make_movie() + movie.tracks = TrackStub(audio=[types.SimpleNamespace(language=Language.get("ja"))], subtitles=[]) # type: ignore[assignment] + context = movie._build_base_template_context(make_media_info(["ja"])) + + assert context["lang_tag"] == "SUBBED" + assert context["tag"] == "SUBGRP" + + +def test_a_quality_rule_matches_the_built_context(monkeypatch: pytest.MonkeyPatch) -> None: + """Guards the context keys the docs tell users to match on.""" + monkeypatch.setattr(config, "tag_rules", [{"when": {"quality": "2160p", "video": "H.265"}, "tag": "UHDGROUP"}]) + video = types.SimpleNamespace( + width=3840, + height=2160, + format="HEVC", + scan_type="Progressive", + other_display_aspect_ratio=["16:9"], + hdr_format_commercial=None, + hdr_format=None, + transfer_characteristics=None, + transfer_characteristics_original=None, + frame_rate="23.976", + ) + media_info = types.SimpleNamespace(video_tracks=[video], audio_tracks=[make_audio("en")]) + + context = make_movie()._build_base_template_context(media_info) + + assert context["quality"] == "2160p" + assert context["video"] == "H.265" + assert context["tag"] == "UHDGROUP" + + +def test_dl_clears_tag_rules_when_tag_is_given() -> None: + """Source-level pin: the block lives inside the click-driven dl.__init__.""" + from unshackle.commands.dl import dl + + source = inspect.getsource(dl.__init__) + block = source.split("if tag:")[1].split("\n\n")[0] + + assert "config.tag = tag" in block + assert "config.tag_rules = []" in block diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 56b1331b..35763bcf 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -574,7 +574,10 @@ class dl: help="Proxy URI to use. If a 2-letter country is provided, it will try to get a proxy from the config.", ) @click.option( - "--tag", type=str, default=None, help="Set the Group Tag to be used, overriding the one in config if any." + "--tag", + type=str, + default=None, + help="Set the Group Tag to be used, overriding the config tag and any tag rules.", ) @click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.") @click.option( @@ -1252,6 +1255,7 @@ class dl: if tag: config.tag = tag + config.tag_rules = [] # needs to be added this way instead of @cli.result_callback to be # able to keep `self` as the first positional diff --git a/unshackle/core/config.py b/unshackle/core/config.py index b543a1e4..9a94e291 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -130,6 +130,7 @@ class Config: self.redact_paths: bool = kwargs.get("redact_paths", True) self.language_tags: dict = kwargs.get("language_tags") or {} + self.tag_rules: list = kwargs.get("tag_rules") or [] self.dual_multi_mode: str = (kwargs.get("dual_multi_mode") or "strict").lower() self.output_template: dict = kwargs.get("output_template") or {} folder_cfg = self.output_template.pop("folder", "") @@ -203,6 +204,7 @@ class Config: "edition", "repack", "lang_tag", + "title_type", } unsafe_chars = r'[<>:"/\\|?*]' diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index 10456129..91224e3a 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -95,6 +95,9 @@ class Title: "hfr": "", "edition": "", "lang_tag": "", + "title_type": {"Movie": "movie", "Episode": "series", "Song": "music"}.get( + type(self).__name__, type(self).__name__.lower() + ), } if self.tracks: @@ -199,6 +202,13 @@ class Title: sub_langs = [s.language for s in self.tracks.subtitles] context["lang_tag"] = evaluate_language_tag(lang_tag_rules, audio_langs, sub_langs) + if config.tag_rules: + from unshackle.core.utils.tag_rules import evaluate_tag_rules + + override = evaluate_tag_rules(config.tag_rules, context) + if override: + context["tag"] = override + return context @abstractmethod diff --git a/unshackle/core/utils/tag_rules.py b/unshackle/core/utils/tag_rules.py new file mode 100644 index 00000000..7e902b1f --- /dev/null +++ b/unshackle/core/utils/tag_rules.py @@ -0,0 +1,116 @@ +"""Release-group tag rule engine for output filename templates.""" + +from __future__ import annotations + +import logging +import operator +import re +from collections.abc import Mapping +from typing import Any, Callable, Optional + +log = logging.getLogger(__name__) + +_OPS: dict[str, Callable[[Any, Any], bool]] = { + ">=": operator.ge, + "<=": operator.le, + "!=": operator.ne, + "==": operator.eq, + ">": operator.gt, + "<": operator.lt, + "=": operator.eq, +} +_ORDERING = (">=", "<=", ">", "<") +_NUMBER = re.compile(r"\d+(?:\.\d+)?") + + +def evaluate_tag_rules(rules: Any, context: dict[str, Any]) -> Optional[str]: + """Evaluate tag rules against a filename template context. + + Rules are evaluated in order; the first matching rule's tag is returned. + Conditions are AND-ed, compared case-insensitively, and a list matches if + any entry matches. A value may lead with a comparison operator, e.g. + ">=2160" or "!=DV". + + Args: + rules: Rules straight from the user's YAML, so any shape is possible. + Each valid rule is a mapping with a ``when`` mapping and a ``tag``. + context: The already-built filename template context. + + Returns: + The tag string from the first matching rule, or ``None`` if none match. + """ + if not isinstance(rules, list): + log.warning("tag_rules must be a list of rules, ignoring: %r", rules) + return None + + for rule in rules: + if not isinstance(rule, Mapping): + log.warning("Tag rule must be a mapping of 'when' and 'tag', skipping: %r", rule) + continue + + when = rule.get("when") + tag = rule.get("tag") + if when is not None and not isinstance(when, Mapping): + log.warning("Tag rule 'when' must be a mapping of conditions, skipping: %s", rule) + continue + if not when or not tag: + log.warning("Tag rule needs both 'when' conditions and a 'tag', skipping: %s", rule) + continue + if isinstance(tag, bool) or any(_has_bool(value) for value in when.values()): + log.warning("Tag rule has a YAML boolean, quote the value to use it as text, skipping: %s", rule) + continue + if not isinstance(tag, (str, int, float)): + log.warning("Tag rule 'tag' must be text or a number, skipping: %s", rule) + continue + + unknown = [key for key in when if key not in context] + if unknown: + log.warning("Tag rule has unknown 'when' key(s) %s, skipping: %s", ", ".join(sorted(unknown)), rule) + continue + + if all(_matches(expected, context[key]) for key, expected in when.items()): + log.debug("Tag rule matched: %s -> %s", rule, tag) + return str(tag) + + return None + + +def _has_bool(expected: Any) -> bool: + """Check for a YAML boolean, which is an unquoted word the user meant as text.""" + values = expected if isinstance(expected, list) else [expected] + return any(isinstance(x, bool) for x in values) + + +def _matches(expected: Any, actual: Any) -> bool: + """Check one condition, where a list matches on any entry.""" + values = expected if isinstance(expected, list) else [expected] + return any(_matches_one(x, actual) for x in values) + + +def _matches_one(expected: Any, actual: Any) -> bool: + """Check a single value, which may carry a leading comparison operator.""" + text = "" if expected is None else str(expected) + for op in sorted(_OPS, key=len, reverse=True): + if text.startswith(op): + return _compare(op, text[len(op) :].strip(), actual) + return str(actual or "").lower() == text.lower() + + +def _compare(op: str, operand: str, actual: Any) -> bool: + """Compare against an operand, numerically when the operand is a number.""" + if not operand: + log.warning("Tag rule condition '%s' has no value to compare against, condition failed", op) + return False + + try: + wanted = float(operand) + except ValueError: + if op in _ORDERING: + log.warning("Tag rule condition '%s%s' needs a numeric operand, condition failed", op, operand) + return False + return _OPS[op](str(actual or "").lower(), operand.lower()) + + found = _NUMBER.search(str(actual or "")) + if not found: + return op == "!=" + return _OPS[op](float(found.group()), wanted) diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index e0150f20..92ae015f 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -9,6 +9,12 @@ # Group or username appended to download filenames after a dash. tag: "NOGRP" +# Conditionally swap the tag above based on filename variables (first match wins). +# See docs/reference/configuration.md#tag_rules for conditions and semantics. +# tag_rules: +# - when: {lang_tag: SUBBED} +# tag: SUBGRP + # --- Output filename & folder templates --- docs/guide/output-and-naming.md # The full variable list and all naming options are documented there. # movies/series have no built-in fallback (define them to download those kinds); From c4b25756ebee1a7bdab86c8511aae2ab723d19e6 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Tue, 4 Aug 2026 10:25:32 -0600 Subject: [PATCH 37/40] docs: split configuration reference into per-area pages --- docs/dev/creating-a-service.md | 2 +- docs/getting-started/configuration-file.md | 10 +- docs/getting-started/installation.md | 2 +- docs/guide/cli-reference.md | 2 +- docs/guide/downloading.md | 2 +- docs/guide/drm-and-cdm.md | 2 +- docs/guide/output-and-naming.md | 11 +- docs/guide/subtitles.md | 4 +- docs/index.md | 2 +- docs/reference/api/config.md | 2 +- docs/reference/configuration.md | 999 +------------------- docs/reference/configuration/directories.md | 88 ++ docs/reference/configuration/download.md | 253 +++++ docs/reference/configuration/drm.md | 157 +++ docs/reference/configuration/example.md | 51 + docs/reference/configuration/index.md | 42 + docs/reference/configuration/misc.md | 109 +++ docs/reference/configuration/network.md | 86 ++ docs/reference/configuration/output.md | 149 +++ docs/reference/configuration/services.md | 137 +++ docs/reference/configuration/vaults.md | 47 + mkdocs.yml | 14 +- 22 files changed, 1169 insertions(+), 1002 deletions(-) create mode 100644 docs/reference/configuration/directories.md create mode 100644 docs/reference/configuration/download.md create mode 100644 docs/reference/configuration/drm.md create mode 100644 docs/reference/configuration/example.md create mode 100644 docs/reference/configuration/index.md create mode 100644 docs/reference/configuration/misc.md create mode 100644 docs/reference/configuration/network.md create mode 100644 docs/reference/configuration/output.md create mode 100644 docs/reference/configuration/services.md create mode 100644 docs/reference/configuration/vaults.md diff --git a/docs/dev/creating-a-service.md b/docs/dev/creating-a-service.md index 242d06da..a2578d89 100644 --- a/docs/dev/creating-a-service.md +++ b/docs/dev/creating-a-service.md @@ -8,7 +8,7 @@ downloading, decryption, muxing, naming) is handled by the unshackle core. !!! info "Who this page is for" This is a **developer guide**. If you only want to *use* services, see [downloading](../guide/downloading.md) and - [configuration](../reference/configuration.md). Nothing here is required to run + [configuration](../reference/configuration/index.md). Nothing here is required to run the CLI. Service code is deliberately kept separate from the core so it can live in its own private repository. diff --git a/docs/getting-started/configuration-file.md b/docs/getting-started/configuration-file.md index 6196aa80..04c6a4ac 100644 --- a/docs/getting-started/configuration-file.md +++ b/docs/getting-started/configuration-file.md @@ -3,7 +3,7 @@ Almost everything about how unshackle behaves is controlled by a single YAML file named `unshackle.yaml`: where it saves downloads, which CDM it uses, your credentials, proxies, filename templates, and more. This page covers where that file lives, how unshackle finds it, the shape of its contents, the directory layout unshackle builds around it, and how to edit values from the command line. !!! note "Where the exhaustive key list lives" - This page is an overview. For the complete, key-by-key breakdown of every setting, its type, and its default, see the [Configuration Reference](../reference/configuration.md). + This page is an overview. For the complete, key-by-key breakdown of every setting, its type, and its default, see the [Configuration Reference](../reference/configuration/index.md). ## The config filename @@ -89,7 +89,7 @@ The top-level keys group loosely into these areas: | Behavior & logging | `update_checks`, `redact_paths`, `debug`, `unicode_filenames` | | Paths | `directories`, `filenames` | -For the full list with types and defaults, see the [Configuration Reference](../reference/configuration.md). +For the full list with types and defaults, see the [Configuration Reference](../reference/configuration/index.md). !!! warning "Unknown keys are silently ignored" unshackle does not validate your config against a schema. If you misspell a key, it is simply skipped and the default is used. You will not get an error. Double-check key names (and their nesting) if a setting does not seem to apply. @@ -126,7 +126,7 @@ services: !!! tip "A few keys are named after the flag's internal name" Most keys are obvious, but set these exact ones: `range` (`-r`), `list` (`--list`), `tmdb_id` (`--tmdb`), `imdb_id` (`--imdb`), `tvdb_id` (`--tvdb`), `no_atmos` (`--noatmos`), and `output_dir` - (`-o`). The [Configuration Reference](../reference/configuration.md#dl) has the full list + (`-o`). The [Configuration Reference](../reference/configuration/download.md#dl) has the full list and every available key. ## The directory layout @@ -177,7 +177,7 @@ unshackle searches entries in the order listed, and **the first source to define ## The `filenames` key -Alongside `directories`, the `filenames` key lets you override the templates unshackle uses when naming its own working and log files (for example the log filename or the temporary chapters file). Most users never need to touch this. The available names and their default templates are listed in the [Configuration Reference](../reference/configuration.md#filenames). +Alongside `directories`, the `filenames` key lets you override the templates unshackle uses when naming its own working and log files (for example the log filename or the temporary chapters file). Most users never need to touch this. The available names and their default templates are listed in the [Configuration Reference](../reference/configuration/directories.md#filenames). ## Editing the config with `unshackle cfg` @@ -223,6 +223,6 @@ When it writes, `unshackle cfg` targets the config file that was loaded. If none ## Next steps -- Browse the [Configuration Reference](../reference/configuration.md) for every key and default. +- Browse the [Configuration Reference](../reference/configuration/index.md) for every key and default. - Set up your first download in [Downloading](../guide/downloading.md). - Learn how services are discovered and updated in [Creating a Service](../dev/creating-a-service.md). diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index af9d8d52..f993ae91 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -108,7 +108,7 @@ These must be present for core downloading, decryption, and muxing to work: !!! warning "shaka-packager is the default decryptor" unshackle decrypts with shaka-packager unless you explicitly configure a different decryptor. If it is missing, protected downloads will fail at the - decryption step. See the [configuration reference](../reference/configuration.md#decryption) for the + decryption step. See the [configuration reference](../reference/configuration/drm.md#decryption) for the `decryption` key if you want to switch to `mp4decrypt`. ### Recommended and optional tools diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 09473416..e6a1c8f9 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -103,7 +103,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] !!! tip "These flags select languages, they do not order them" Naming languages with `-l` or `-sl` removes the others. To keep every language but put your preferred ones first, use `audio.language_priority` and - [`subtitle.language_priority`](../reference/configuration.md#subtitle) in your + [`subtitle.language_priority`](../reference/configuration/download.md#subtitle) in your configuration file. The special language tokens `orig`, `all`, and `best` are honoured everywhere a language is expected. diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 6686a9d4..047b4be1 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -272,7 +272,7 @@ alphabetically, after the title's original language. Nothing is removed. Audio w same way, but the languages you leave out keep their bitrate and codec order instead of an alphabetical one. -See [`subtitle.language_priority`](../reference/configuration.md#subtitle) for the details. +See [`subtitle.language_priority`](../reference/configuration/download.md#subtitle) for the details. ### Exact vs fuzzy matching diff --git a/docs/guide/drm-and-cdm.md b/docs/guide/drm-and-cdm.md index 69c691e5..27824cf8 100644 --- a/docs/guide/drm-and-cdm.md +++ b/docs/guide/drm-and-cdm.md @@ -516,5 +516,5 @@ debug_requests: true - [Downloading](downloading.md): the `dl` command and its DRM-related flags. - [Key vaults](vaults.md): caching content keys across downloads. -- [Configuration](../reference/configuration.md): the full `unshackle.yaml` reference. +- [Configuration](../reference/configuration/index.md): the full `unshackle.yaml` reference. - [Dependencies](../getting-started/installation.md): installing shaka-packager and mp4decrypt. diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 5b8b1647..9b412d05 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -102,6 +102,7 @@ Every variable below is valid in both output and folder templates. Values are de | Variable | Meaning | Example | |---|---|---| | `title` | Title name (movie/show/song name; `$` is rendered as `S`) | `The Show` | +| `title_type` | Media kind of the title | `movie`, `series`, `music` | | `year` | Release year | `2024` | | `source` | Service tag / class name (empty with `--no-source`) | `EXAMPLE` | | `quality` | Resolution with scan suffix | `1080p`, `2160p`, `576i` | @@ -138,7 +139,7 @@ Every variable below is valid in both output and folder templates. Values are de | `audio_channels` | Channel layout | `5.1`, `2.0` | | `audio_full` | Codec + channels combined | `DDP5.1` | | `atmos` | `Atmos` when any track carries JOC | `Atmos` | -| `dual` | `DUAL` for two audio languages (see [`dual_multi_mode`](../reference/configuration.md#dual_multi_mode)) | `DUAL` | +| `dual` | `DUAL` for two audio languages (see [`dual_multi_mode`](../reference/configuration/download.md#dual_multi_mode)) | `DUAL` | | `multi` | `MULTi` for more than two audio languages | `MULTi` | | `dubbed` | `DUBBED` for a single audio language that is not the original (strict mode only) | `DUBBED` | @@ -234,6 +235,10 @@ The `--no-mux` flag skips muxing entirely and writes the individual track files, ## Tags & group naming The release-group tag is the `-TAG` portion at the end of scene-style names, produced by the `{tag}` variable. +It can be swapped per release with [`tag_rules`](../reference/configuration/output.md#tag_rules): ordered rules that +match on the release-attribute variables only (`quality`, `resolution`, `hdr`, `source`, `lang_tag`, +`title_type`, and so on), first match wins. The title's own fields, such as `title` and `season_episode`, +cannot be matched. See the [reference page](../reference/configuration/output.md#tag_rules) for the exact list. ```yaml title="unshackle.yaml" tag: "MYGROUP" @@ -405,7 +410,7 @@ how the optional (`?`) variables appear and disappear. === "Multi-language" Place `{dual?}`, `{multi?}` and `{dubbed?}` next to each other. At most one of them is - ever set (see [`dual_multi_mode`](../reference/configuration.md#dual_multi_mode)), and the + ever set (see [`dual_multi_mode`](../reference/configuration/download.md#dual_multi_mode)), and the empty ones collapse: ```yaml @@ -439,7 +444,7 @@ how the optional (`?`) variables appear and disappear. ``` For finer control (e.g. a `SUBBED` tag driven by subtitle languages), use `{lang_tag?}` - with [`language_tags` rules](../reference/configuration.md#language_tags) instead. + with [`language_tags` rules](../reference/configuration/download.md#language_tags) instead. === "Daily shows" diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index 62d43835..96159d23 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -154,8 +154,8 @@ flags drive selection, sort order, output filenames, and the Matroska track flag By default this type order is the primary sort, so all forced tracks come first, then all normal tracks, then all SDH/CC tracks, each block sorted by language. To keep each language next to its own variants instead, set - [`subtitle.group_by`](../reference/configuration.md#subtitle) to `language`. Use - [`subtitle.type_priority`](../reference/configuration.md#subtitle) to change the + [`subtitle.group_by`](../reference/configuration/download.md#subtitle) to `language`. Use + [`subtitle.type_priority`](../reference/configuration/download.md#subtitle) to change the Forced → Normal → SDH/CC order itself. A single track cannot be both CC and SDH, and a forced track cannot also be flagged diff --git a/docs/index.md b/docs/index.md index 5eb5fcb3..64cdd785 100644 --- a/docs/index.md +++ b/docs/index.md @@ -86,7 +86,7 @@ install to your first file. - **[Installation](getting-started/installation.md)**. Install unshackle and the external tools it depends on. - **[Quickstart](getting-started/quickstart.md)**. Configure `unshackle.yaml` and run your first download. - **[Downloading](guide/downloading.md)**. The full `dl` command guide: quality, codecs, languages, track selection, and output. -- **[Configuration](getting-started/configuration-file.md)**. Where `unshackle.yaml` lives, the directory layout, and how to edit values. Full key list in the [Configuration Reference](reference/configuration.md). +- **[Configuration](getting-started/configuration-file.md)**. Where `unshackle.yaml` lives, the directory layout, and how to edit values. Full key list in the [Configuration Reference](reference/configuration/index.md). - **[REST API](dev/rest-api/index.md)**. Run the `serve` HTTP server and drive downloads programmatically. !!! tip "Getting help" diff --git a/docs/reference/api/config.md b/docs/reference/api/config.md index 3961a558..9e29b85e 100644 --- a/docs/reference/api/config.md +++ b/docs/reference/api/config.md @@ -2,7 +2,7 @@ Auto-generated reference for the [`Config`][unshackle.core.config.Config] object that models the `unshackle` configuration file. For a human-friendly walkthrough of every key, see the -[Configuration Reference](../configuration.md). +[Configuration Reference](../configuration/index.md). ::: unshackle.core.config options: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index cab4e4ff..e6bfa177 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1,987 +1,18 @@ -# Configuration Reference +# Configuration Reference (moved) -This page documents every key you can set in -`unshackle.yaml`. For each key you will find its name, expected type, default value, and -what it does. Keys are grouped by area so you can jump straight to the part of unshackle -you are configuring. +This page has moved. The configuration reference is now split by area under +[`docs/reference/configuration/`](configuration/index.md), or on the docs site at +[docs.unshackle.dev/reference/configuration](https://docs.unshackle.dev/reference/configuration/). -If you are new to the config file (where it lives, how unshackle discovers it, and how to -edit it), start with [The Configuration File](../getting-started/configuration-file.md). -This page assumes you already know the basics and want precise details. - -!!! info "How defaults work" - Every key is optional except [`output_template`](#output-naming), which `dl` requires. - Without it, `dl` aborts with an error. If you omit any other key, unshackle uses - the built-in default listed here. There is **no schema validation**: unknown keys are - silently ignored (they never become settings), and the file is parsed with - `yaml.safe_load`. An empty or missing file means every other key falls back to its - default. - -!!! warning "Two keys behave specially on load" - - `curl_impersonate` is a **deprecated** alias of [`network`](#network-proxy) and emits a - `DeprecationWarning`. - - `scene_naming` has been **removed**. If it is present at all, unshackle exits - immediately with an error telling you to use [`output_template`](#output-naming) - instead. - - See [Deprecated & removed keys](#deprecated-removed-keys) at the end of this page. - ---- - -## Quick index - -| Area | Keys | +| Area | Page | |------|------| -| [Directories](#directories) | `directories`, `filenames` | -| [Services & authentication](#services-auth) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` | -| [Download & processing](#download-processing) | `dl`, `subtitle`, `audio`, `muxing`, `language_tags`, `dual_multi_mode` | -| [Output & naming](#output-naming) | `output_template`, `tag`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` | -| [DRM & CDM](#drm-cdm) | `cdm`, `remote_cdm`, `decryption` | -| [Network & proxy](#network-proxy) | `network`, `headers`, `proxy_providers` | -| [Key vaults](#key-vaults) | `key_vaults`, `vault_timeout` | -| [External API keys](#external-api-keys) | `imdb_api_enabled`, `omdb_api_key`, `tmdb_api_key`, `tvdb_api_key`, `tvdb_pin`, `tvdb_order`, `simkl_client_id`, `metadata_providers`, `decrypt_labs_api_key`, `ipinfo_api_key` | -| [Caching & updates](#caching-updates) | `title_cache_enabled`, `title_cache_time`, `title_cache_max_retention`, `update_checks`, `update_check_interval` | -| [Logging, privacy & debug](#logging-privacy-debug) | `redact_paths`, `debug`, `debug_keys`, `debug_requests`, `set_terminal_bg` | -| [Deprecated & removed](#deprecated-removed-keys) | `curl_impersonate`, `downloader`, `scene_naming` | - ---- - -## Directories { #directories } - -The `directories` key is a **dict** mapping directory names to filesystem paths. Only the -names listed below are honoured; a handful are **protected** and cannot be moved (attempts -to override them are silently skipped). All defaults are computed relative to the installed -package, and every user-settable path is passed through `Path(...).expanduser()`, so `~` -works. - -```yaml title="unshackle.yaml" -directories: - downloads: ~/Media/unshackle - temp: /mnt/fast-scratch/unshackle-temp - wvds: ~/.unshackle/WVDs - prds: ~/.unshackle/PRDs -``` - -| Key | Type | Default | Overridable | Purpose | -|-----|------|---------|:-----------:|---------| -| `downloads` | path | `<repo>/downloads` | ✅ | Default output directory for finished files. | -| `temp` | path | `<repo>/temp` | ✅ | Temporary working files during download/decrypt/mux. | -| `cache` | path | `<data>/cache` | ✅ | Generic cache, title cache, and the update-check store. | -| `cookies` | path | `<data>/cookies` | ✅ | Per-service cookie files (and VPN cookie/token files). | -| `logs` | path | `<data>/logs` | ✅ | Log files. | -| `exports` | path | `<data>/exports` | ✅ | Export JSON files. | -| `wvds` | path | `<data>/WVDs` | ✅ | Widevine device files (`.wvd`). | -| `prds` | path | `<data>/PRDs` | ✅ | PlayReady device files (`.prd`). | -| `dcsl` | path | `<data>/DCSL` | ✅ | DCSL data. | -| `commands` | path | `unshackle/commands` | ✅ | CLI command modules. | -| `services` | list \| path | `[unshackle/services]` | ✅ | Service search paths and/or remote repo specs (see below). | -| `vaults` | path | `unshackle/vaults` | ✅ | Vault backend modules. | -| `fonts` | path | `unshackle/fonts` | ✅ | Bundled fonts. | -| `user_configs` | path | `unshackle/` | ❌ protected | Where `unshackle.yaml` lives. | -| `data` | path | `unshackle/` | ❌ protected | Base for the data subdirectories above. | -| `core_dir` | path | `unshackle/core` | ❌ protected | Package core. | -| `namespace_dir` | path | `unshackle/` | ❌ protected | Package root. | -| `app_dirs` | - | `AppDirs("unshackle", False)` | ❌ protected | Internal AppDirs instance. | - -!!! note "The `services` directory is special" - `services` may be a **list**, and each entry can be either a local directory or a - **repository spec**: a git URL (`https://...`, `ssh://...`, `git@...`, or anything ending in - `.git`) or `owner/repo` shorthand. Repo specs are cloned and updated automatically; plain - paths are used as-is. **List order is priority**: the first source to define a service - tag wins, so put your own local service directories **last** to make them fallbacks. - - ```yaml - directories: - services: - - you/your-services # cloned from GitHub - - https://example.com/private.git@stable - - ~/my-services # local, lowest priority - ``` - -!!! info "How git-backed service repos are handled" - unshackle's use of git is **read-only on the remote**: it only ever performs a shallow - `clone`, `fetch`, `pull`, and a local `reset`; **nothing is ever pushed**. Private repos - are authenticated through your existing git credential helper, and unshackle stores no - tokens of its own. Clones live under `<first-local-services-dir>/_repos/<repo-name>/` (or - the bundled `unshackle/services`), and **nothing is written to the cache directory**. After - the first clone unshackle re-pulls **at most once every 24 h**, so it does not touch the - network on every run. - -!!! warning "The two refresh paths diverge on purpose" - The **automatic** 24 h-TTL refresh that happens during a normal `dl`/`search` run will - deliberately **refuse to refresh and exit** (naming the offending clone) if that clone - has uncommitted changes to tracked files or unpushed local commits, so a background pull can - never clobber work in progress. Untracked files (new service folders, `__pycache__`) never - block it. The **manual** `unshackle util refresh-services` command does the opposite: it - **hard-resets** the clone to upstream, discarding any local edits. Reach for the manual - command only when you actually want to throw local changes away. - -!!! warning "Read-only installs and reinstalls" - If the services directory lives inside the installed package, a reinstall can delete the - `_repos` clones. They are simply re-cloned on next use. On **read-only installs** you must - point `services` at a writable path, or cloning will fail. - -### Filenames { #filenames } - -The `filenames` key is a **dict** of templated file/name patterns. Each value is used -verbatim (no path processing). Braced fields like `{time}` and `{service}` are filled in at -runtime. - -| Key | Type | Default | Notes | -|-----|------|---------|-------| -| `log` | str | `"unshackle_{name}_{time}.log"` | Written under `directories.logs`. | -| `debug_log` | str | `"unshackle_debug_{service}_{time}.jsonl"` | Structured debug log, under `directories.logs`. | -| `config` | str | `"config.yaml"` | Per-service config file, under that service's directory. | -| `root_config` | str | `"unshackle.yaml"` | The main config filename itself. | -| `chapters` | str | `"Chapters_{title}_{random}.txt"` | Under `directories.temp`. | -| `subtitle` | str | `"Subtitle_{id}_{language}.srt"` | Under `directories.temp`. | - ---- - -## Services & authentication { #services-auth } - -### `services` - -- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` - -Per-service configuration, keyed by the canonical service tag (for example `EXAMPLE1`, `EXAMPLE2`, -`EXAMPLE3`). unshackle reads several well-known sub-keys out of each service's block: - -- **`proxy_map`**: a dict mapping `"provider:query"` or `"query"` to a proxy value, used to - auto-select a proxy for that service. -- **`title_map`**: an exact-match rename map applied to fetched titles (source name → - desired name), so a service that names a title differently from your library still matches. - -Individual services may read any additional keys they define. The whole block is handed to -the service as `self.config`. - -!!! tip "Why put secrets here and not in the service's own `config.yaml`?" - A service ships a per-service `config.yaml` inside its own folder, and that file is - typically shared/committed and **not** meant to be edited by you. So anything user- or - device-specific (API keys, account IDs, device attributes) belongs in this `services` - block of `unshackle.yaml` instead, where it is **merged over** the service's `config.yaml` - at load time. That is why the two config locations exist: one for shared service defaults, - one for your private overrides. - -```yaml -services: - EXAMPLE1: - proxy_map: - default: us - title_map: - "My Show: The Movie": "My Show Presents The Movie" -``` - -### `credentials` - -- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` - -Per-service login credentials. Each value is a `username:password[:extra]` string (a list or -dict form is also accepted where a service supports it). These are parsed into `Credential` -objects; the credential's SHA-1 is also used as an account hash for cache keys. - -```yaml -credentials: - EXAMPLE2: user@example.com:hunter2 - EXAMPLE1: - - primary@example.com:pw1 - - secondary@example.com:pw2 -``` - -!!! tip "Cookies vs credentials" - Cookies are stored as files under [`directories.cookies`](#directories), not in this key. - A service's `authenticate()` accepts cookies, credentials, or both. - -### `firefox_cookies` - -- **Type:** `dict`  ·  **Default:** `{}` - -Settings for extracting cookies directly from a local Firefox profile. A service block is -expected to provide `hosts` (a list of cookie hostnames; entries shorter than 3 characters -are ignored to prevent dumping the whole cookie store) and an optional `local_storage` -boolean to also pull matching local-storage entries. Extraction is read-only. - -!!! note "Firefox does not need to be closed" - The extractor copies **both** `cookies.sqlite` **and** its WAL file into a `0700` temp - directory; the WAL copy exists to capture writes Firefox has not yet flushed to the main - DB, so the copy is current even against a running browser. The only real constraint is that - Firefox must not hold an exclusive write lock at the exact instant of the copy. The live - profile is otherwise left untouched. - -!!! tip "When you actually need `local_storage: true`" - Enable it only for services that keep their auth tokens in the browser's **localStorage** - (`webappsstore.sqlite`) rather than in HTTP cookies. If a service authenticates with plain - cookies, leaving it off is correct. - -!!! warning "Extraction fails silently to file cookies" - If extraction yields no cookies or fails for **any** reason, unshackle silently falls back - to the normal file-based cookie path (`cookies/<SERVICE>.txt` or - `cookies/<SERVICE>/<profile>.txt`). This is a safety net, but it also means that if you - expected Firefox cookies and file cookies are being used instead, extraction quietly failed. - Check that first when troubleshooting. - -### `remote_services` - -- **Type:** `dict`  ·  **Default:** `{}` - -Definitions of remote unshackle service servers, used by the `--remote` mode. Each entry -carries a `services` sub-dict of remote service tags, which unshackle turns into synthetic -CLI commands that run against the remote server. - -### `serve` - -- **Type:** `dict`  ·  **Default:** `{}` - -Configuration for the `serve` command (the built-in REST API server). The full server guide -is the [REST API](../dev/rest-api/index.md) section; these are the config keys. - -| Sub-key | Type | Default | Description | -|---------|------|---------|-------------| -| `api_secret` | str | *(unset)* | Master secret accepted in the `X-Secret-Key` header. | -| `users` | dict | `{}` | Per-user API keys and their allowlists (see below). | -| `services` | list | *(unset)* | Global service allowlist. Omit to allow all. | -| `remote_only` | bool | `false` | Expose only the remote service session endpoints (health, services, search, session) and disable the rest of the REST API. | -| `session_ttl` | int (s) | `300` | Lifetime of an interactive auth session. | -| `max_sessions` | int | `100` | Maximum concurrent sessions. | -| `history_limit` | int | `100` | How many finished jobs to retain in history. | -| `compression_level` | int | `1` | gzip level for responses. | -| `global_speed_limit` | str | *(unlimited)* | Server-wide download speed cap, e.g. `10M`, `1.5G` or plain bytes/sec (same format as `speed_limit`). One shared budget across all concurrent jobs; per-job speed limits are ignored while it is set. | -| `cdm_overrides` | dict | *(unset)* | Allowed per-request CDM overrides. | -| `allow_job_credentials` | bool | `false` | Permit clients to supply credentials per job. | -| `devices` | list | *(auto)* | Widevine devices offered; auto-filled from `directories.wvds`. | -| `playready_devices` | list | *(auto)* | PlayReady devices; auto-filled from `directories.prds`. | - -Each entry under `users` is keyed by that user's API key and may set its own `services`, -`devices`, and `playready_devices` allowlists, narrowing the global ones. - -```yaml -serve: - api_secret: change-me - remote_only: true - services: [EXAMPLE1, EXAMPLE2] - # server-wide download defaults (same keys as the `dl:` block) - downloads: 3 - best_available: true - users: - a1b2c3d4: # this user's API key - services: [EXAMPLE1] # may only use EXAMPLE1 -``` - -!!! tip "Declaring `dl` defaults under `serve`" - You can set most [`dl`](#dl) flag keys (`downloads`, `workers`, `best_available`, and so on) - directly inside `serve`; the server recognises a fixed subset of download parameters, so a - few CLI-only flags are ignored here. This raises concurrency and behaviour defaults - **server-wide** (every request the server handles picks them up) without changing - each client call individually. - ---- - -## Download & processing { #download-processing } - -### `dl` - -- **Type:** `dict`  ·  **Default:** `{}` - -Persistent defaults for the `dl` command. This block is used as Click's `default_map`, so -**any** `dl` flag can be given a default here and it applies to every download without you -typing it. See [Downloading](../guide/downloading.md) and the -[CLI Reference](../guide/cli-reference.md) for what each flag does. - -**Key naming.** A key is the flag's long name with dashes replaced by underscores -(`--best-available` → `best_available`, `--sub-format` → `sub_format`). A few keys are named -after the flag's internal destination rather than its visible name; set these exact keys: - -| Flag | Config key | -|------|-----------| -| `-r` / `--range` | `range` | -| `--list` | `list` | -| `--tmdb` | `tmdb_id` | -| `--imdb` | `imdb_id` | -| `--animeapi` | `animeapi_id` | -| `-naa` / `--noatmos` | `no_atmos` | -| `-o` / `--output` | `output_dir` | -| `--cdm-only` / `--vaults-only` | `cdm_only` | - -**Precedence** (highest first): an explicit CLI flag → a per-service -[`services.<TAG>.dl`](#services-auth) override → this global `dl:` block → the built-in -default. In other words, config only fills in options you did not set on the command line. - -Common keys (a useful subset; every `dl` flag works): - -| Key | Type | Default | Sets | -|-----|------|---------|------| -| `lang` | list | `["orig"]` | Video/audio language(s); `orig` = original language. | -| `a_lang` / `v_lang` | list | `[]` | Audio- / video-only language override. | -| `s_lang` | list | `["all"]` | Subtitle language(s). | -| `quality` | list | `[]` (best) | Resolution(s), e.g. `[1080]`. | -| `vcodec` | list | `[]` (any) | Video codec(s), e.g. `[H265]`. | -| `acodec` | list | `[]` (any) | Audio codec(s), e.g. `[EC3]`. | -| `range` | list | `["SDR"]` | Colour range(s): `SDR`, `HDR10`, `HDR10+`, `DV`. | -| `channels` | float | *(unset)* | Audio channels, e.g. `6` for 5.1. | -| `sub_format` | str | *(unset)* | Convert subtitles to this format (`srt`, `vtt`, `original`, ...). | -| `forced_subs` | bool | `false` | Include forced subtitle tracks. | -| `forced_s_lang` | list | `[]` | Forced subtitle language(s); implies `forced_subs`. | -| `no_subs` / `no_audio` / `no_chapters` | bool | `false` | Skip that track type. | -| `downloads` | int | `1` | Tracks downloaded concurrently. | -| `workers` | int | *(downloader default)* | Threads per track. | -| `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). | -| `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | -| `best_available` | bool | `false` | Fall back to best quality if the request is unavailable. | -| `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | -| `no_folder` | bool | `false` | Do not create a per-title folder. | - -```yaml title="Global download defaults" -dl: - lang: [en] - quality: [1080] - range: [HDR10] - vcodec: [H265] - sub_format: srt - downloads: 2 -``` - -!!! tip "Per-service overrides" - Nest a `dl` block under a service in [`services`](#services-auth) to override these defaults - for just that service. It uses the same keys and wins over the global `dl:` block (but still - loses to an explicit CLI flag). - - ```yaml - dl: - lang: [en] # global default - services: - EXAMPLE1: - dl: - lang: [en, ja] # EXAMPLE1 downloads both; everything else stays English - range: [DV] - ``` - -!!! note "`serve` uses these same keys" - The [`serve`](#serve) block accepts the same `dl` option keys at its top level to set - server-wide download defaults for every request it handles. - -### `subtitle` - -- **Type:** `dict`  ·  **Default:** `{}` - -Subtitle handling: SDH stripping, format conversion, and whether subtitles are muxed in or -written as sidecar files. See [Subtitles](../guide/subtitles.md) for the full guide. - -| Sub-key | Type | Default | Description | -|---------|------|---------|-------------| -| `strip_sdh` | bool | `true` | Strip SDH/CC cues (hearing-impaired annotations) into a clean track. | -| `sdh_method` | str | `"auto"` | SDH-stripping backend to use. | -| `preserve_formatting` | bool | `true` | Keep styling/positioning when converting or stripping. | -| `convert_before_strip` | bool | `true` | Convert to the working format before stripping SDH. | -| `conversion_method` | str | `"auto"` | Subtitle conversion backend. | -| `preferred_conversion_method` | str | *(unset)* | Preferred converter when several are available. | -| `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files; `both` writes sidecars and still muxes. | -| `sidecar_format` | str | `"srt"` | Format for sidecar files when `output_mode: sidecar`. | -| `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH). Types you leave out fall to the end. | -| `group_by` | str | `"type"` | Which key groups the subtitle tracks. `type` puts all forced tracks together, then all normal, then all SDH. `language` keeps each language next to its own variants. | -| `language_priority` | list | *(unset)* | Languages to sort to the top, in the order you give. The rest keep the default order: original language first, then alphabetical. Nothing is removed. | - -```yaml -subtitle: - strip_sdh: true - output_mode: sidecar - sidecar_format: srt -``` - -!!! note "`group_by` puts a language next to its own SDH track" - The default `type` order groups by subtitle type first, so every normal track comes - before every SDH track. In a title with many languages, a track ends up far from its - own SDH version, for example Finnish at track 5 and Finnish SDH at track 42. Set - `group_by: language` to keep them together: - - ```yaml - subtitle: - group_by: language - ``` - - | `group_by` | Result | - |---|---| - | `type` | Danish, Finnish, French, … Danish SDH, Finnish SDH, French SDH | - | `language` | Danish, Danish SDH, Finnish, Finnish SDH, French, French SDH | - - `type_priority` still sets the order of the variants, so with `group_by: language` it - controls the order inside each language. - -!!! note "`language_priority` sorts languages, it does not select them" - By default the title's original language comes first and every other language follows - alphabetically. For an English title that gives English, Arabic, Bulgarian, and the - rest. Give `language_priority` the languages you want first: - - ```yaml - subtitle: - group_by: language - language_priority: [en, es, fr] - ``` - - | | Result for an English title | - |---|---| - | Default | English, Arabic, Bulgarian, Spanish, French, Japanese | - | With the list above | English, Spanish, French, Arabic, Bulgarian, Japanese | - - This is only an order. Every other language is still downloaded, which is what makes - it different from `-sl`: `-sl en,es,fr` **removes** all other subtitles. - - Use `orig` in the list for the title's original language. A base tag also covers its - regional forms, so `en` sorts `en-US` and `en-GB`, but `zh` does **not** cover - `zh-Hant`, which you must name. The sort ignores entries that match no track, and it - ignores entries that are not valid language tags. - - `--exact-lang` applies here too: with it, `en` sorts only `en`, and `en-US` falls to - the alphabetical tail with the rest. Name each tag you want when you use that flag. - -!!! note "`type_priority` also picks the default subtitle" - At mux time the first track in the default subtitle language gets the `default` flag. - The built-in order is `forced`, `normal`, `sdh`, which means a forced track becomes - default whenever one exists. If you want the full dialogue track as default instead, - set: - - ```yaml - subtitle: - type_priority: [normal, sdh, forced] - ``` - -### `audio` - -- **Type:** `dict`  ·  **Default:** `{}` - -Audio handling options. The most notable sub-key is **`codec_priority`**, an ordered list of -audio codecs used to break ties in track selection. **`language_priority`** is an ordered list -of languages that sort to the top of the audio tracks. - -```yaml -audio: - codec_priority: - - EAC3 - - AC3 - language_priority: - - orig - - en -``` - -!!! note "`codec_priority` is a soft priority: nothing is dropped" - `codec_priority` outranks bitrate: any listed codec sorts above every unlisted one, and among - listed codecs the list order decides. Bitrate order survives only within a single codec rank. - Nothing is ever removed. Two higher-level rules still override it: **Atmos - tracks always take precedence over `codec_priority`**, and **audio-description tracks are - always forced to the end** regardless of it. When `codec_priority` is unset, tracks sort by - bitrate alone, with those same Atmos/descriptive rules still applied. - -### `muxing` - -- **Type:** `dict`  ·  **Default:** `{}` - -Matroska (MKV) muxing options. - -| Sub-key | Type | Default | Description | -|---------|------|---------|-------------| -| `set_title` | bool | `true` | Write a human-readable title into the MKV container. | -| `default_language` | dict | `{}` | Force which language is flagged *default* per track type, e.g. `{audio: en, subtitle: en}`. | -| `merge_audio` | bool | `true` | Merge all audio into one file. `--split-audio` on the CLI flips this off. | -| `merge_video` | bool | `false` | Merge video tracks that share height, range, and codec into one file, so only language varies inside a file. `--merge-video` on the CLI flips this on. | - -```yaml -muxing: - set_title: true - default_language: - audio: en - subtitle: en -``` - -### `language_tags` - -- **Type:** `dict`  ·  **Default:** `{}` - -Language-tag remapping and the rule engine behind the `{lang_tag}` filename variable. When a -`rules` list is present, each rule is evaluated in order and the first match's `tag` is used. -A rule may test: - -| Condition | Meaning | -|-----------|---------| -| `audio` | Matches against the audio track languages. | -| `subs_contain` | At least one subtitle language matches. | -| `subs_contain_all` | Every listed language must be present (scalar or list). | - -```yaml -language_tags: - rules: - - audio: [ja] - subs_contain: [en] - tag: "SUBBED" -``` - -### `dual_multi_mode` - -- **Type:** `str`  ·  **Default:** `strict` - -Rule set for the `{dual}`, `{multi}`, and `{dubbed}` filename variables. - -=== "`strict` (default)" - - | Variable | Set when | - |----------|----------| - | `{dual}` | Exactly two audio languages **and one is the title's original language**. | - | `{multi}` | Three or more audio languages, even without the original. | - | `{dubbed}` | A single audio language that is not the title's original language (requires a known original). | - -=== "`count` (legacy)" - - | Variable | Set when | - |----------|----------| - | `{dual}` | Exactly two audio languages (original language ignored). | - | `{multi}` | Three or more audio languages. | - | `{dubbed}` | Never set. | - -```yaml -dual_multi_mode: strict -``` - -!!! note "Dialects count as one language" - Regional variants of the same base language (e.g. `en-US` and `en-GB`) collapse to a single - language in both modes, so they never trigger `{dual}` on their own. - ---- - -## Output & naming { #output-naming } - -For the full picture of how templates render, see -[Output & naming](../guide/output-and-naming.md). The keys are summarised here. - -### `output_template` - -- **Type:** `dict`  ·  **Required** (no usable default; `dl` exits with an error if this is missing) - -Filename templates keyed by media kind, using `{variable}` placeholders. Recognised kinds are -`movies`, `series`, and `songs` (and `albums` for folder templates). A trailing `?` on a -variable (e.g. `{quality?}`) marks it optional: if empty, it and one adjacent separator are -dropped. - -```yaml title="unshackle.yaml" -output_template: - movies: "{title}.{year}.{quality?}.{source}-{tag}" - series: "{title}.{season_episode}.{episode_name?}.{quality?}.{source}-{tag}" - songs: "{track_number}. {title}" - folder: # special nested key, see below - movies: "{title} ({year})" - series: "{title} ({year})" -``` - -**The nested `folder` sub-key** controls output subfolders and is popped out of -`output_template`: - -- A **dict** sets per-kind folder templates (kinds: `movies`, `series`, `songs`, `albums`). -- A **str** sets a single folder template for everything. - -unshackle validates templates on load and emits warnings for unknown variables, non-string -values, filesystem-unsafe characters (`< > : " / \ | ? *`), empty templates, and unknown -folder-kind keys. - -??? example "All valid template variables" - `title`, `year`, `season`, `episode`, `season_episode`, `episode_name`, `date`, - `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, `album_artist`, - `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, `explicit`, - `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, `dual`, - `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`. - -### Tagging & naming keys - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| `tag` | str | `""` | Release-group tag appended to filenames and written into MKV tags. | -| `tag_group_name` | bool | `true` | Include the group name (`tag`) in the MKV `Group` tag. | -| `tag_imdb_tmdb` | bool | `true` | Write IMDb/TMDB/TVDB external-ID tags into the MKV (needs metadata providers). | -| `chapter_fallback_name` | str | `""` | Fallback chapter-name template, e.g. `"Chapter {i:02}"`. | -| `unicode_filenames` | bool | `false` | Allow Unicode in filenames; when `false`, names are ASCII-sanitised. | - ---- - -## DRM & CDM { #drm-cdm } - -The full conceptual guide lives at [DRM & CDM](../guide/drm-and-cdm.md). These are the config -keys. Device files themselves live in [`directories.wvds`](#directories) (`.wvd`) and -`directories.prds` (`.prd`). - -### `cdm` - -- **Type:** `dict`  ·  **Default:** `{}` - -Maps a service tag to the CDM (Widevine/PlayReady device) to use, with a `default` fallback. -Resolution is case-insensitive: an explicit CLI override wins, then the per-service entry, -then `default`. - -```yaml -cdm: - default: chromecdm_l3 - EXAMPLE2: android_l1 -``` - -A value may also be a **nested dict** for advanced selection by quality, DRM system, or -profile: - -```yaml -cdm: - EXAMPLE1: - ">=1080": android_l1 # by track height - "<1080": chromecdm_l3 - widevine: android_l1 # by DRM system - playready: sl3000 - default: chromecdm_l3 -``` - -Quality keys support `1080`, `>=1080`, `>720`, `<=576`, `<480` style comparisons; DRM keys -are `widevine` / `playready`. - -### `remote_cdm` - -- **Type:** `list[dict]`  ·  **Default:** `[]` - -A list of remote CDM server definitions. Each entry is matched by its `name` (referenced from -[`cdm`](#cdm)), and its `type` selects the backend: - -| `type` | Backend | Key fields | -|--------|---------|-----------| -| `decrypt_labs` | Decrypt Labs KeyXtractor | `host` (default `https://keyxtractor.decryptlabs.com`), `device_name`, `secret` (falls back to `decrypt_labs_api_key`) | -| `custom_api` | Fully YAML-configurable remote API | `device`, `auth`, `endpoints`, `request_mapping`, `response_mapping`, `caching`, `timeout` | -| *(none)*, `Device Type: PLAYREADY` | pyplayready `RemoteCdm` | `host`, `secret`, `device_name`, `security_level` (default 3000) | -| *(none)*, otherwise | pywidevine `RemoteCdm` | `host`, `secret`, `device_name`, `device_type`, `system_id`, `security_level` (default 3000) | - -```yaml -remote_cdm: - - name: keyxtractor - type: decrypt_labs - device_name: L1 - - name: my_wv_server - host: https://cdm.example.com - secret: s3cr3t - device_name: android_l1 - device_type: ANDROID - system_id: 26830 - security_level: 1 -``` - -### `decryption` - -- **Type:** `str` or `dict`  ·  **Default:** `"shaka"` - -Selects the tool used to physically decrypt CENC content. The value is compared -case-insensitively; `"mp4decrypt"` selects Bento4's mp4decrypt, and **anything else** -(including `"shaka"`) selects shaka-packager. - -=== "Global" - - ```yaml - decryption: shaka - ``` - -=== "Per-service" - - ```yaml - decryption: - default: shaka - EXAMPLE1: mp4decrypt - ``` - ---- - -## Network & proxy { #network-proxy } - -### `network` - -- **Type:** `dict`  ·  **Default:** `{}` - -TLS-fingerprinting and HTTP client settings for the rnet-based session. - -| Sub-key | Type | Default | Description | -|---------|------|---------|-------------| -| `browser` | str | `"Chrome131"` | Impersonation preset. Must be an exact preset name (e.g. `Chrome131`, `Firefox135`, `Edge101`, `Safari18`, `OkHttp4_12`, `OkHttp5`, `Opera118`); unknown names raise an error. | -| `http1_only` | bool | *(unset)* | Force HTTP/1.1. | -| `http2_only` | bool | *(unset)* | Force HTTP/2. | -| `pool_max_idle_per_host` | int | *(unset)* | Connection-pool tuning. | -| `pool_max_size` | int | *(unset)* | Connection-pool tuning. | -| `tcp_nodelay` | bool | *(unset)* | Disable Nagle's algorithm. | - -```yaml -network: - browser: Firefox135 - http2_only: true -``` - -!!! note "When to force `http1_only`" - HTTP/2 is the right default on large CDNs, so leave it on. Force HTTP/1.1 only for a host that - throttles per-connection, or a slow origin stuck behind HTTP/2 flow control. In benchmarks - this won 30 to 50% on such hosts but **cost up to 27% on fast CDNs**, so enable it only when - your own measurements justify it. - -!!! warning "Renamed from `curl_impersonate`" - The old `curl_impersonate` section is a deprecated alias. If you still use it, unshackle - honours it (only when `network` is absent) but emits a `DeprecationWarning`. Rename it to - `network`. - -### `headers` - -- **Type:** `dict`  ·  **Default:** `{}` - -Default HTTP headers merged into every session unshackle creates. - -```yaml -headers: - Accept-Language: en-US,en;q=0.9 -``` - -!!! warning "Don't set `Accept-Encoding` (and similar) here" - Keep this block to sane cross-service defaults such as `Accept-Language` and `User-Agent`. - Compatibility headers like `Accept-Encoding` are set for you by the rnet HTTP backend as - part of its browser-impersonation profile, and overriding them can break the impersonation - fingerprint. Anything useful to only some services belongs in that service's config, not - here. - -### `proxy_providers` - -- **Type:** `dict`  ·  **Default:** `{}` - -Proxy/VPN provider configuration. Each sub-key names a provider, and its block is passed -straight to that provider's constructor. See [Proxies & VPN](../guide/proxies-and-vpn.md) for -the full provider guide. Recognised providers and their exit ports: - -| Provider | Config key | Credentials | Proxy scheme/port | -|----------|-----------|-------------|-------------------| -| Basic (static) | `basic` | country → URI(s) | as specified | -| NordVPN | `nordvpn` | 48-char **service** credentials | `https://...:89` | -| Surfshark | `surfsharkvpn` | 48-char **service** credentials | `https://...:443` | -| Windscribe | `windscribevpn` | service credentials | `https://...:443` | -| ExpressVPN | `expressvpn` | device login (`enable: true`) / token cache | `https://cat:...@...:443` | -| ProtonVPN | `protonvpn` | TV login or exported cookies | `https://...:4443` (or `:443` Secure Core) | -| Gluetun | `gluetun` | per-VPN keys/creds | `http://localhost:{port}` (local Docker) | -| Hola | *(none, auto)* | none | `http://...:{peer}` | - -```yaml -proxy_providers: - basic: - us: http://user:pass@1.2.3.4:8080 - de: - - http://a.example:8080 - - socks5://b.example:1080 - nordvpn: - username: <48-char service username> - password: <48-char service password> -``` - -!!! note "Provider loading differs between CLI and REST server" - The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API / - remote-client path uses a separate resolver that does **not** load `windscribevpn` or - `gluetun`. ExpressVPN and ProtonVPN also auto-load when their cached session exists, and Hola - auto-loads whenever the `hola-proxy` binary is present. - ---- - -## Key vaults { #key-vaults } - -The full guide is at [Vaults](../guide/vaults.md). Two keys configure them. - -### `key_vaults` - -- **Type:** `list[dict]`  ·  **Default:** `[]` - -An ordered list of key-vault backends. unshackle queries them in order and reuses content -keys instead of re-licensing. Each entry needs a `type` (the backend module name) and a -`name`, plus backend-specific keys. - -| `type` | Purpose | Required keys | Notes | -|--------|---------|--------------|-------| -| `SQLite` | Local SQLite database | `name`, `path` | Loaded **critically**; a failure aborts the run. | -| `MySQL` | Remote MySQL database | `name`, `host`, `database`, `username` | Extra keys (e.g. `password`, `port`) forwarded to pymysql. | -| `API` | RESTful JSON API | `name`, `uri`, `token` | Honours `vault_timeout`. | -| `HTTP` | HTTP API with modes | `name`, `host`, one of `password`/`api_key` | `api_mode`: `query` (default), `json`, `decrypt_labs`. | - -```yaml -key_vaults: - - type: SQLite - name: Local - path: ~/.unshackle/keys.db - - type: MySQL - name: Team - host: db.example.com - database: keys - username: unshackle - password: hunter2 - no_push: false -``` - -!!! tip "Common per-entry options" - - `no_push: true` makes a vault read-only (keys are fetched but never written to it). - - A vault of `type: API` whose `name` contains `decrypt_labs` auto-fills its `token` from - [`decrypt_labs_api_key`](#external-api-keys) when not set inline. Vault `type` values are - case-sensitive module names. - - An all-zero content key (32 zeros) is treated as "no key" everywhere and is never - stored. - -### `vault_timeout` - -- **Type:** `float`  ·  **Default:** `10.0` - -Timeout in seconds for vault operations. Injected automatically into any backend whose -constructor accepts a `timeout` parameter (a per-vault `timeout` still wins). - ---- - -## External API keys { #external-api-keys } - -All default to an empty string and enable optional metadata/geolocation features. - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| `imdb_api_enabled` | bool | `false` | Use the free IMDxAPI (api.tiffara.com) metadata provider, which needs no API key. Off by default since the site has been unreliable. | -| `omdb_api_key` | str | `""` | [OMDb](https://www.omdbapi.com/) API key for IMDb metadata lookups; a more reliable alternative to IMDxAPI. Free keys are available on the OMDb site. | -| `tmdb_api_key` | str | `""` | TMDB API key for metadata enrichment and external-ID tags. | -| `simkl_client_id` | str | `""` | SIMKL client ID for metadata lookups; an alternative/fallback source to TMDB. | -| `tvdb_api_key` | str | `""` | [TheTVDB v4](https://thetvdb.com/api-information) API key; a fallback source to TMDB, strongest on TV series. Free keys are available on the TVDB site. | -| `tvdb_pin` | str | `""` | Subscriber PIN, only needed for a user-supported TVDB key. Leave empty for a normal project key. | -| `tvdb_order` | str | `""` | Default for `--tvdb-order`: renumber episodes to a TVDB season order (`official`, `dvd`, `absolute`, `alternate`, `regional`). Empty keeps the numbering the service gives. | -| `metadata_providers` | list | *(see below)* | Metadata providers to use, in the order they are tried. | -| `decrypt_labs_api_key` | str | `""` | Global Decrypt Labs API key (used by remote CDM / vault). | -| `ipinfo_api_key` | str | `""` | ipinfo.io API key for IP/region lookups. | - -!!! tip "Choosing and ordering metadata providers" - A lookup goes to each provider in turn and stops at the first good match, so the order - decides which source wins. `metadata_providers` sets that order: - - ```yaml - metadata_providers: [tvdb, tmdb, simkl] - ``` - - Names not in the list are never used, so this both orders and filters. Unknown names are - ignored with a warning. Leave it unset for the default order: - - ```yaml - metadata_providers: [imdbapi, omdb, simkl, tmdb, tvdb] - ``` - - A provider is skipped when its API key is missing, whatever the order. Available names are - `imdbapi`, `omdb`, `simkl`, `tmdb`, and `tvdb`. Put `tvdb` first to make TheTVDB your - primary source; leave it last to use it only as a fallback for TMDB. - -!!! note "`ipinfo_api_key` never touches your service sessions" - The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header; it - is **never** attached to your service session, so it cannot leak to a streaming provider. - Lookups degrade gracefully through a fallback chain: the authenticated Lite endpoint (higher - rate limits and richer fields) → anonymous ipinfo → `ip-api.in` as a last resort. - ---- - -## Caching & updates { #caching-updates } - -### Title cache - -unshackle caches fetched title metadata (region- and account-aware) to avoid repeat API -calls. - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| `title_cache_enabled` | bool | `true` | Master switch for the title cache. | -| `title_cache_time` | int (seconds) | `1800` (30 min) | Lifetime of fresh cached titles. | -| `title_cache_max_retention` | int (seconds) | `86400` (24 h) | How long stale cached data may still be served as a fallback when a live fetch fails. | - -### Update checks - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| `update_checks` | bool | `true` | Whether to check for new unshackle releases. | -| `update_check_interval` | int (hours) | `24` | Minimum hours between update checks. | - -!!! note - Update checks query the GitHub releases API with a fixed 5-second timeout and cache the - result in `directories.cache/update_check.json`. - ---- - -## Logging, privacy & debug { #logging-privacy-debug } - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| `redact_paths` | bool | `true` | Mask install-root / venv / home prefixes in logged paths (`<unshackle>`, `<venv>`, `~`). Set `false` to show full paths. | -| `debug` | bool | `false` | Global debug mode. | -| `debug_keys` | bool | `false` | Log decryption keys. | -| `debug_requests` | bool | `false` | Log HTTP requests. | -| `set_terminal_bg` | bool | `false` | Append the theme's background colour to output styling. | - -!!! tip "When to enable `set_terminal_bg`" - When on, the theme's background colour is appended to the foreground styles so the full - colour palette renders correctly on terminals whose *default* background differs from - unshackle's theme. Turn it on if the ASCII-art banner or coloured output **looks off** on - your terminal. Otherwise leave it off. - -!!! danger "Key exposure" - Decryption content keys are written to the structured debug log at INFO level during the - DRM handshake. Treat `debug`, `debug_keys`, and the `unshackle_debug_*.jsonl` files as - sensitive, and keep `redact_paths` enabled when sharing logs. - - `debug_keys` affects **only** content-encryption keys (the `content_key`/`key` fields). - Passwords, tokens, cookies, and session tokens are **always** redacted regardless of this - setting, and KIDs, key counts, and other metadata are always logged either way. - ---- - -## Deprecated & removed keys { #deprecated-removed-keys } - -| Key | Status | Behaviour | -|-----|--------|-----------| -| `curl_impersonate` | **Deprecated** → use [`network`](#network-proxy) | Emits a `DeprecationWarning`; still honoured only if `network` is absent. | -| `downloader` | **Deprecated** | Any value other than `"requests"` emits a `DeprecationWarning`; the value is otherwise ignored (the unified requests downloader is always used). | -| `scene_naming` | **Removed** | If present at all, unshackle exits with an error directing you to configure [`output_template`](#output-naming) instead. | - ---- - -## A complete annotated example - -The following config sets a handful of common keys. Every key not shown keeps its default. - -```yaml title="unshackle.yaml" -directories: - downloads: ~/Media/unshackle - temp: /mnt/scratch/unshackle - -network: - browser: Chrome131 - -headers: - Accept-Language: en-US,en;q=0.9 - -cdm: - default: chromecdm_l3 - EXAMPLE2: android_l1 - -decryption: shaka - -credentials: - EXAMPLE2: user@example.com:hunter2 - -output_template: - movies: "{title}.{year}.{quality?}.{source}-{tag}" - series: "{title}.{season_episode}.{quality?}.{source}-{tag}" - folder: - movies: "{title} ({year})" - series: "{title} ({year})" - -tag: MYGRP - -key_vaults: - - type: SQLite - name: Local - path: ~/.unshackle/keys.db - -proxy_providers: - basic: - us: http://user:pass@1.2.3.4:8080 - -omdb_api_key: "your-omdb-key" -tmdb_api_key: "your-tmdb-key" -tvdb_api_key: "your-tvdb-key" -title_cache_time: 3600 -redact_paths: true -``` - -For how these interact at runtime, continue to [Downloading](../guide/downloading.md), -[DRM & CDM](../guide/drm-and-cdm.md), and [Output & naming](../guide/output-and-naming.md). +| Overview & quick index | [configuration/index.md](configuration/index.md) | +| Directories & filenames | [configuration/directories.md](configuration/directories.md) | +| Services & authentication | [configuration/services.md](configuration/services.md) | +| Download & processing | [configuration/download.md](configuration/download.md) | +| Output & naming | [configuration/output.md](configuration/output.md) | +| DRM & CDM | [configuration/drm.md](configuration/drm.md) | +| Network & proxy | [configuration/network.md](configuration/network.md) | +| Key vaults | [configuration/vaults.md](configuration/vaults.md) | +| Other settings | [configuration/misc.md](configuration/misc.md) | +| Complete annotated example | [configuration/example.md](configuration/example.md) | diff --git a/docs/reference/configuration/directories.md b/docs/reference/configuration/directories.md new file mode 100644 index 00000000..363fec81 --- /dev/null +++ b/docs/reference/configuration/directories.md @@ -0,0 +1,88 @@ +# Directories { #directories } + +The `directories` key is a **dict** mapping directory names to filesystem paths. Only the +names listed below are honoured; a handful are **protected** and cannot be moved (attempts +to override them are silently skipped). All defaults are computed relative to the installed +package, and every user-settable path is passed through `Path(...).expanduser()`, so `~` +works. + +```yaml title="unshackle.yaml" +directories: + downloads: ~/Media/unshackle + temp: /mnt/fast-scratch/unshackle-temp + wvds: ~/.unshackle/WVDs + prds: ~/.unshackle/PRDs +``` + +| Key | Type | Default | Overridable | Purpose | +|-----|------|---------|:-----------:|---------| +| `downloads` | path | `<repo>/downloads` | ✅ | Default output directory for finished files. | +| `temp` | path | `<repo>/temp` | ✅ | Temporary working files during download/decrypt/mux. | +| `cache` | path | `<data>/cache` | ✅ | Generic cache, title cache, and the update-check store. | +| `cookies` | path | `<data>/cookies` | ✅ | Per-service cookie files (and VPN cookie files). | +| `logs` | path | `<data>/logs` | ✅ | Log files. | +| `exports` | path | `<data>/exports` | ✅ | Export JSON files. | +| `wvds` | path | `<data>/WVDs` | ✅ | Widevine device files (`.wvd`). | +| `prds` | path | `<data>/PRDs` | ✅ | PlayReady device files (`.prd`). | +| `dcsl` | path | `<data>/DCSL` | ✅ | DCSL data. | +| `commands` | path | `unshackle/commands` | ✅ | CLI command modules. | +| `services` | list \| path | `[unshackle/services]` | ✅ | Service search paths and/or remote repo specs (see below). | +| `vaults` | path | `unshackle/vaults` | ✅ | Vault backend modules. | +| `fonts` | path | `unshackle/fonts` | ✅ | Bundled fonts. | +| `user_configs` | path | `unshackle/` | ❌ protected | Where `unshackle.yaml` lives. | +| `data` | path | `unshackle/` | ❌ protected | Base for the data subdirectories above. | +| `core_dir` | path | `unshackle/core` | ❌ protected | Package core. | +| `namespace_dir` | path | `unshackle/` | ❌ protected | Package root. | +| `app_dirs` | - | `AppDirs("unshackle", False)` | ❌ protected | Internal AppDirs instance. | + +!!! note "The `services` directory is special" + `services` may be a **list**, and each entry can be either a local directory or a + **repository spec**: a git URL (`https://...`, `ssh://...`, `git@...`, or anything ending in + `.git`) or `owner/repo` shorthand. Repo specs are cloned and updated automatically; plain + paths are used as-is. **List order is priority**: the first source to define a service + tag wins. + + ```yaml + directories: + services: + - you/your-services # cloned from GitHub + - https://example.com/private.git@stable + - ~/my-services # local, lowest priority + ``` + +!!! info "How git-backed service repos are handled" + unshackle's use of git is **read-only on the remote**: it only ever performs a shallow + `clone`, `fetch`, `pull`, and a local `reset`; **nothing is ever pushed**. Private repos + are authenticated through your existing git credential helper, and unshackle stores no + tokens of its own. Clones live under `<first-local-services-dir>/_repos/<host>__<owner>__<repo>/` + (or the bundled `unshackle/services`), and **nothing is written to the cache directory**. After + the first clone unshackle re-pulls **at most once every 24 h**, so it does not touch the + network on every run. + +!!! warning "Automatic and manual refresh behave differently" + The **automatic** 24 h-TTL refresh that happens during a normal `dl`/`search` run + **refuses to refresh and exits** (naming the offending clone) if that clone has + uncommitted changes to tracked files or unpushed local commits. Untracked files (new + service folders, `__pycache__`) never block it. The **manual** + `unshackle util refresh-services` command instead **hard-resets** the clone to upstream, + discarding any local edits. + +!!! warning "Read-only installs and reinstalls" + If the services directory lives inside the installed package, a reinstall can delete the + `_repos` clones. They are re-cloned on next use. On **read-only installs** you must + point `services` at a writable path, or cloning will fail. + +## Filenames { #filenames } + +The `filenames` key is a **dict** of templated file/name patterns. Each value is used +verbatim (no path processing). Braced fields like `{time}` and `{service}` are filled in at +runtime. + +| Key | Type | Default | Notes | +|-----|------|---------|-------| +| `log` | str | `"unshackle_{name}_{time}.log"` | Written under `directories.logs`. | +| `debug_log` | str | `"unshackle_debug_{service}_{time}.jsonl"` | Structured debug log, under `directories.logs`. | +| `config` | str | `"config.yaml"` | Per-service config file, under that service's directory. | +| `root_config` | str | `"unshackle.yaml"` | The main config filename itself. | +| `chapters` | str | `"Chapters_{title}_{random}.txt"` | Under `directories.temp`. | +| `subtitle` | str | `"Subtitle_{id}_{language}.srt"` | Under `directories.temp`. | diff --git a/docs/reference/configuration/download.md b/docs/reference/configuration/download.md new file mode 100644 index 00000000..b85b27f3 --- /dev/null +++ b/docs/reference/configuration/download.md @@ -0,0 +1,253 @@ +# Download & processing { #download-processing } + +## `dl` + +- **Type:** `dict`  ·  **Default:** `{}` + +Persistent defaults for the `dl` command. The block is used as Click's `default_map`, so any +`dl` flag can be given a default here. See [Downloading](../../guide/downloading.md) and the +[CLI Reference](../../guide/cli-reference.md) for what each flag does. + +**Key naming.** A key is the flag's long name with dashes replaced by underscores +(`--best-available` → `best_available`, `--sub-format` → `sub_format`). A few keys are named +after the flag's internal destination rather than its visible name; set these exact keys: + +| Flag | Config key | +|------|-----------| +| `-r` / `--range` | `range` | +| `--list` | `list` | +| `--tmdb` | `tmdb_id` | +| `--imdb` | `imdb_id` | +| `--animeapi` | `animeapi_id` | +| `-naa` / `--noatmos` | `no_atmos` | +| `-o` / `--output` | `output_dir` | +| `--cdm-only` / `--vaults-only` | `cdm_only` | + +**Precedence** (highest first): an explicit CLI flag → a per-service +[`services.<TAG>.dl`](services.md) override → this global `dl:` block → the built-in +default. In other words, config only fills in options you did not set on the command line. + +Common keys (a useful subset; every `dl` flag works): + +| Key | Type | Default | Sets | +|-----|------|---------|------| +| `lang` | list | `["orig"]` | Video/audio language(s); `orig` = original language. | +| `a_lang` / `v_lang` | list | `[]` | Audio- / video-only language override. | +| `s_lang` | list | `["all"]` | Subtitle language(s). | +| `quality` | list | `[]` (best) | Resolution(s), e.g. `[1080]`. | +| `vcodec` | list | `[]` (any) | Video codec(s), e.g. `[H265]`. | +| `acodec` | list | `[]` (any) | Audio codec(s), e.g. `[EC3]`. | +| `range` | list | `["SDR"]` | Colour range(s): `SDR`, `HLG`, `HDR10`, `HDR10P`, `DV`, `HYBRID`. | +| `channels` | float | *(unset)* | Audio channels, e.g. `6` for 5.1. | +| `sub_format` | str | *(unset)* | Convert subtitles to this format (`srt`, `vtt`, `original`, ...). | +| `forced_subs` | bool | `false` | Include forced subtitle tracks. | +| `forced_s_lang` | list | `[]` | Forced subtitle language(s); implies `forced_subs`. | +| `no_subs` / `no_audio` / `no_chapters` | bool | `false` | Skip that track type. | +| `downloads` | int | `1` | Tracks downloaded concurrently. | +| `workers` | int | *(downloader default)* | Threads per track. | +| `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). | +| `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | +| `best_available` | bool | `false` | Fall back to best quality if the request is unavailable. | +| `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | +| `no_folder` | bool | `false` | Do not create a per-title folder. | + +```yaml title="Global download defaults" +dl: + lang: [en] + quality: [1080] + range: [HDR10] + vcodec: [H265] + sub_format: srt + downloads: 2 +``` + +!!! note "Per-service overrides" + A `dl` block nested under a service in [`services`](services.md) takes the same keys and + applies to that service only. + + ```yaml + dl: + lang: [en] + services: + EXAMPLE1: + dl: + lang: [en, ja] + range: [DV] + ``` + +!!! note "`serve` uses these same keys" + The [`serve`](services.md#serve) block accepts most of the same `dl` option keys at its top level + to set server-wide download defaults for every request it handles. Keys the API does not know + are ignored there (`speed_limit`, for example; `serve` has its own `global_speed_limit`). + +## `subtitle` + +- **Type:** `dict`  ·  **Default:** `{}` + +Subtitle handling: SDH stripping, format conversion, and whether subtitles are muxed in or +written as sidecar files. See [Subtitles](../../guide/subtitles.md) for the full guide. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `strip_sdh` | bool | `true` | Strip SDH/CC cues (hearing-impaired annotations) into a clean track. | +| `sdh_method` | str | `"auto"` | SDH-stripping backend to use. | +| `preserve_formatting` | bool | `true` | Keep styling/positioning when converting or stripping. | +| `convert_before_strip` | bool | `true` | Convert to the working format before stripping SDH. | +| `conversion_method` | str | `"auto"` | Subtitle conversion backend. | +| `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files; `both` writes sidecars and still muxes. | +| `sidecar_format` | str | `"srt"` | Format for sidecar files when `output_mode` is `sidecar` or `both`. | +| `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH). Types you leave out fall to the end. | +| `group_by` | str | `"type"` | Which key groups the subtitle tracks. `type` puts all forced tracks together, then all normal, then all SDH. `language` keeps each language next to its own variants. | +| `language_priority` | list | *(unset)* | Languages to sort to the top, in the order you give. The rest keep the default order: original language first, then alphabetical. Nothing is removed. | + +```yaml +subtitle: + strip_sdh: true + output_mode: sidecar + sidecar_format: srt +``` + +!!! note "`group_by` track order" + | `group_by` | Result | + |---|---| + | `type` | Danish, Finnish, French, … Danish SDH, Finnish SDH, French SDH | + | `language` | Danish, Danish SDH, Finnish, Finnish SDH, French, French SDH | + + `type_priority` still sets the order of the variants, so with `group_by: language` it + controls the order inside each language. + +!!! note "`language_priority` sorts languages, it does not select them" + ```yaml + subtitle: + group_by: language + language_priority: [en, es, fr] + ``` + + | | Result for an English title | + |---|---| + | Default | English, Arabic, Bulgarian, Spanish, French, Japanese | + | With the list above | English, Spanish, French, Arabic, Bulgarian, Japanese | + + Nothing is removed, unlike `-sl`, which drops every subtitle language it does not name. + + Use `orig` in the list for the title's original language. A base tag also covers its + regional forms, so `en` sorts `en-US` and `en-GB`, but `zh` does **not** cover + `zh-Hant`, which you must name. The sort ignores entries that match no track, and it + ignores entries that are not valid language tags. + + `--exact-lang` applies here too: with it, `en` sorts only `en`, and `en-US` falls to + the alphabetical tail with the rest. Name each tag you want when you use that flag. + +!!! note "`type_priority` picks the default subtitle only with `muxing.default_language`" + At mux time the `default` flag goes to every forced subtitle whose language matches the + first audio track, so a full dialogue track never becomes default on its own. + `type_priority` alone does not change that. Name a subtitle language in + [`muxing.default_language`](#muxing) and the first track in that language gets the flag + instead, which is where `type_priority` decides which track that is: + + ```yaml + muxing: + default_language: + subtitle: en + subtitle: + type_priority: [normal, sdh, forced] + ``` + +## `audio` + +- **Type:** `dict`  ·  **Default:** `{}` + +Audio handling options. The most notable sub-key is **`codec_priority`**, an ordered list of +audio codecs used to break ties in track selection. **`language_priority`** is an ordered list +of languages that sort to the top of the audio tracks. + +```yaml +audio: + codec_priority: + - EAC3 + - AC3 + language_priority: + - orig + - en +``` + +!!! note "`codec_priority` is a soft priority: nothing is dropped" + `codec_priority` outranks bitrate: any listed codec sorts above every unlisted one, and among + listed codecs the list order decides. Bitrate order survives only within a single codec rank. + Nothing is ever removed. Two higher-level rules still override it: **Atmos + tracks always take precedence over `codec_priority`**, and **audio-description tracks are + always forced to the end** regardless of it. When `codec_priority` is unset, tracks sort by + bitrate alone, with those same Atmos/descriptive rules still applied. + +## `muxing` + +- **Type:** `dict`  ·  **Default:** `{}` + +Matroska (MKV) muxing options. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `set_title` | bool | `true` | Write a human-readable title into the MKV container. | +| `default_language` | dict | `{}` | Force which language is flagged *default* per track type, e.g. `{audio: en, subtitle: en}`. | +| `merge_audio` | bool | `true` | Merge all audio into one file. `--split-audio` on the CLI flips this off. | +| `merge_video` | bool | `false` | Merge video tracks that share height, range, and codec into one file, so only language varies inside a file. `--merge-video` on the CLI flips this on. | + +```yaml +muxing: + set_title: true + default_language: + audio: en + subtitle: en +``` + +## `language_tags` + +- **Type:** `dict`  ·  **Default:** `{}` + +Language-tag remapping and the rule engine behind the `{lang_tag}` filename variable. When a +`rules` list is present, each rule is evaluated in order and the first match's `tag` is used. +A rule may test: + +| Condition | Meaning | +|-----------|---------| +| `audio` | Matches against the audio track languages. | +| `subs_contain` | At least one subtitle language matches. | +| `subs_contain_all` | Every listed language must be present (scalar or list). | + +```yaml +language_tags: + rules: + - audio: [ja] + subs_contain: [en] + tag: "SUBBED" +``` + +## `dual_multi_mode` + +- **Type:** `str`  ·  **Default:** `strict` + +Rule set for the `{dual}`, `{multi}`, and `{dubbed}` filename variables. + +=== "`strict` (default)" + + | Variable | Set when | + |----------|----------| + | `{dual}` | Exactly two audio languages **and one is the title's original language**. | + | `{multi}` | Three or more audio languages, even without the original. | + | `{dubbed}` | A single audio language that is not the title's original language (requires a known original). | + +=== "`count` (legacy)" + + | Variable | Set when | + |----------|----------| + | `{dual}` | Exactly two audio languages (original language ignored). | + | `{multi}` | Three or more audio languages. | + | `{dubbed}` | Never set. | + +```yaml +dual_multi_mode: strict +``` + +!!! note "Dialects count as one language" + Regional variants of the same base language (e.g. `en-US` and `en-GB`) collapse to a single + language in both modes, so they never trigger `{dual}` on their own. diff --git a/docs/reference/configuration/drm.md b/docs/reference/configuration/drm.md new file mode 100644 index 00000000..b4c2f0f1 --- /dev/null +++ b/docs/reference/configuration/drm.md @@ -0,0 +1,157 @@ +# DRM & CDM { #drm-cdm } + +These are the config keys. For concepts, device provisioning, and the `wvd`/`prd` commands see +the [DRM & CDM guide](../../guide/drm-and-cdm.md). Device files themselves live in +[`directories.wvds`](directories.md) (`.wvd`) and `directories.prds` (`.prd`). + +## `cdm` + +- **Type:** `dict`  ·  **Default:** `{}` + +Maps a service tag to the CDM (Widevine/PlayReady device) to use, with a `default` fallback. +Resolution is case-insensitive: a per-request API override wins, then the per-service entry, +then `default`. + +```yaml +cdm: + default: chromecdm_l3 + EXAMPLE2: android_l1 +``` + +A value may also be a **nested dict** for advanced selection by quality, DRM system, or +profile: + +```yaml +cdm: + EXAMPLE1: + ">=1080": android_l1 # by track height + "<1080": chromecdm_l3 + widevine: android_l1 # by DRM system + playready: sl3000 + default: chromecdm_l3 +``` + +Quality keys support `1080`, `>=1080`, `>720`, `<=576`, `<480` style comparisons; DRM keys +are `widevine` / `playready`. Quality keys are matched first, but only when a video height is +known; if the value is still a dict after that, `widevine`/`playready` are used when present, +otherwise the credential profile name, then `default`. + +The resolved name is looked up in [`remote_cdm`](#remote_cdm) by `name` first. If no entry +matches, it is loaded as a local device file, in this order: `<name>.prd` in +`directories.prds`, `<name>.prd` in `directories.wvds`, then `<name>.wvd` in +`directories.wvds`. + +## `remote_cdm` + +- **Type:** `list[dict]`  ·  **Default:** `[]` + +A list of remote CDM server definitions. Each entry is matched by its `name` (referenced from +[`cdm`](#cdm)), and its `type` selects the backend: + +| `type` | Backend | Key fields | +|--------|---------|-----------| +| `decrypt_labs` | Decrypt Labs KeyXtractor | `host`, `device_name`, `secret` ([fields](#remote-cdm-decrypt-labs)) | +| `custom_api` | Fully YAML-configurable remote API | `host` (required), `device`, `auth`, `endpoints`, `request_mapping`, `response_mapping`, `caching`, `timeout` ([fields](#remote-cdm-custom-api)) | +| *(none)*, `Device Type: PLAYREADY` | pyplayready `RemoteCdm` | `host`, `secret`, `device_name`, `security_level` (default 3000) | +| *(none)*, otherwise | pywidevine `RemoteCdm` | `host`, `secret`, `device_name`, `device_type`, `system_id`, `security_level` (default 3000) | + +```yaml +remote_cdm: + - name: keyxtractor + type: decrypt_labs + device_name: L1 + - name: my_wv_server + host: https://cdm.example.com + secret: s3cr3t + device_name: android_l1 + device_type: ANDROID + system_id: 26830 + security_level: 1 +``` + +For the two pywidevine/pyplayready backends, field names are read case-insensitively in both +styles: `Device Type`/`device_type`, `System ID`/`system_id`, `Security Level`/`security_level`, +`Host`/`host`, `Secret`/`secret`, `Device Name`/`device_name`. + +!!! warning "PlayReady `host` needs a `/playready` suffix" + pyplayready's `RemoteCdm` treats `host` as a base URL and appends its own endpoint paths, + so a PlayReady entry's `host` must include the trailing `/playready` segment (for example + `https://cdm.example.com/playready`). Without it the server returns `404` rather than a + configuration error. + +### `decrypt_labs` fields { #remote-cdm-decrypt-labs } + +| Field | Default | Notes | +|-------|---------|-------| +| `host` | `https://keyxtractor.decryptlabs.com` | | +| `device_name` | `ChromeCDM` | `ChromeCDM`, `L1`, `L2` (Widevine) or `SL2`, `SL3` (PlayReady) | +| `secret` | from [`decrypt_labs_api_key`](misc.md#external-api-keys) | Sent as the `decrypt-labs-api-key` header. An error is raised if neither is set. | +| `system_id` | `26830` (Widevine), `0` (PlayReady) | | +| `security_level` | Widevine `3`; PlayReady `2000` for `SL2`, else `3000` | | + +PlayReady mode is used when `device_type` is `PLAYREADY` or `device_name` starts with `SL`. + +### `custom_api` fields { #remote-cdm-custom-api } + +| Field | Type | Notes | +|-------|------|-------| +| `host` | str | Required. Base URL of the API. | +| `device` | dict | `name`, `type` (`CHROME`, `ANDROID`, `PLAYREADY`), `system_id`, `security_level` | +| `auth` | dict | `type` (default `header`): `header` uses `header_name` (default `Authorization`) and `key`; `bearer` uses `bearer_token` or `key`; `basic` uses `username` and `password`. `custom_headers` is merged in for any type. | +| `endpoints` | dict | `get_request` and `decrypt_response`, each `{path, method, timeout}` | +| `request_mapping` | dict | Per endpoint: `param_names`, `static_params`, `conditional_params`, `transforms`, `nested_params`, `exclude_params` | +| `response_mapping` | dict | Per endpoint: `fields`, `transforms`, `response_types`, `success_conditions`, `error_fields`, `key_fields` | +| `caching` | dict | `enabled`, `use_vaults`, `check_cached_first` | +| `legacy` | dict | Legacy mode options. | +| `timeout` | int | Default `30`. Request timeout in seconds. | + +Transforms cover base64/hex/JSON encoding and `kid:key` parsing. PlayReady mode is used when +`device.type` is `PLAYREADY` or the device name is `SL2`/`SL3`. + +```yaml +remote_cdm: + - name: my_custom + type: custom_api + host: https://cdm.example.com + device: + name: ChromeCDM + type: CHROME + system_id: 26830 + security_level: 3 + auth: + type: bearer + key: your-token + endpoints: + get_request: + path: /get-challenge + method: POST + decrypt_response: + path: /get-keys + method: POST + timeout: 30 +``` + +## `decryption` + +- **Type:** `str` or `dict`  ·  **Default:** `"shaka"` + +Selects the tool used to physically decrypt CENC content. The value is compared +case-insensitively; `"mp4decrypt"` selects Bento4's mp4decrypt, and **anything else** +(including `"shaka"`) selects shaka-packager. When the value is a dict, the service key is +matched case-insensitively and the `default` entry is the fallback. + +HLS AES-128 ClearKey is decrypted in-process and ignores this setting. + +=== "Global" + + ```yaml + decryption: shaka + ``` + +=== "Per-service" + + ```yaml + decryption: + default: shaka + EXAMPLE1: mp4decrypt + ``` diff --git a/docs/reference/configuration/example.md b/docs/reference/configuration/example.md new file mode 100644 index 00000000..382cca61 --- /dev/null +++ b/docs/reference/configuration/example.md @@ -0,0 +1,51 @@ +# A complete annotated example + +The following config sets a handful of common keys. Every key not shown keeps its default. + +```yaml title="unshackle.yaml" +directories: + downloads: ~/Media/unshackle + temp: /mnt/scratch/unshackle + +network: + browser: Chrome131 + +headers: + Accept-Language: en-US,en;q=0.9 + +cdm: + default: chromecdm_l3 + EXAMPLE2: android_l1 + +decryption: shaka + +credentials: + EXAMPLE2: user@example.com:hunter2 + +output_template: + movies: "{title}.{year}.{quality?}.{source}-{tag}" + series: "{title}.{season_episode}.{quality?}.{source}-{tag}" + folder: + movies: "{title} ({year})" + series: "{title} ({year})" + +tag: MYGRP + +key_vaults: + - type: SQLite + name: Local + path: ~/.unshackle/keys.db + +proxy_providers: + basic: + us: http://user:pass@1.2.3.4:8080 + +omdb_api_key: "your-omdb-key" +tmdb_api_key: "your-tmdb-key" +tvdb_api_key: "your-tvdb-key" +title_cache_time: 3600 +redact_paths: true +``` + +For how these interact at runtime, continue to [Downloading](../../guide/downloading.md), +[DRM & CDM](../../guide/drm-and-cdm.md), and [Output & naming](../../guide/output-and-naming.md). diff --git a/docs/reference/configuration/index.md b/docs/reference/configuration/index.md new file mode 100644 index 00000000..c435059a --- /dev/null +++ b/docs/reference/configuration/index.md @@ -0,0 +1,42 @@ +# Configuration Reference + +Every key you can set in `unshackle.yaml`, with its name, type, default value, and behaviour, +split into one page per area. A worked config is on the [annotated example](example.md) page. + +Where the file lives, how unshackle discovers it, and how to edit it: +[The Configuration File](../../getting-started/configuration-file.md). + +!!! info "How defaults work" + Every key is optional except [`output_template`](output.md), which `dl` requires. + Without it, `dl` aborts with an error. If you omit any other key, unshackle uses + the built-in default listed here. There is **no schema validation**: unknown keys are + silently ignored (they never become settings), and the file is parsed with + `yaml.safe_load`. An empty or missing file means every other key falls back to its + default. + +!!! warning "Two keys behave specially on load" + - `curl_impersonate` is a **deprecated** alias of [`network`](network.md) and emits a + `DeprecationWarning`. + - `scene_naming` has been **removed**. If it is set to any value, unshackle exits + immediately with an error telling you to use [`output_template`](output.md) + instead. + + See [Deprecated & removed keys](misc.md#deprecated-removed-keys). + +--- + +## Quick index + +| Area | Keys | +|------|------| +| [Directories](directories.md) | `directories`, `filenames` | +| [Services & authentication](services.md) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` | +| [Download & processing](download.md) | `dl`, `subtitle`, `audio`, `muxing`, `language_tags`, `dual_multi_mode` | +| [Output & naming](output.md) | `output_template`, `tag`, `tag_rules`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` | +| [DRM & CDM](drm.md) | `cdm`, `remote_cdm`, `decryption` | +| [Network & proxy](network.md) | `network`, `headers`, `proxy_providers` | +| [Key vaults](vaults.md) | `key_vaults`, `vault_timeout` | +| [External API keys](misc.md#external-api-keys) | `imdb_api_enabled`, `omdb_api_key`, `tmdb_api_key`, `tvdb_api_key`, `tvdb_pin`, `tvdb_order`, `simkl_client_id`, `metadata_providers`, `decrypt_labs_api_key`, `ipinfo_api_key` | +| [Caching & updates](misc.md#caching-updates) | `title_cache_enabled`, `title_cache_time`, `title_cache_max_retention`, `update_checks`, `update_check_interval` | +| [Logging, privacy & debug](misc.md#logging-privacy-debug) | `redact_paths`, `debug`, `debug_keys`, `debug_requests`, `set_terminal_bg` | +| [Deprecated & removed](misc.md#deprecated-removed-keys) | `curl_impersonate`, `downloader`, `scene_naming` | diff --git a/docs/reference/configuration/misc.md b/docs/reference/configuration/misc.md new file mode 100644 index 00000000..ce3b5c28 --- /dev/null +++ b/docs/reference/configuration/misc.md @@ -0,0 +1,109 @@ +# Other Settings + +Smaller configuration areas: external metadata API keys, caching and update checks, logging and debug flags, and deprecated keys. + +## External API keys { #external-api-keys } + +All are unset by default and enable optional metadata/geolocation features. + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `imdb_api_enabled` | bool | `false` | Use the free IMDxAPI (api.tiffara.com) metadata provider, which needs no API key. Off by default since the site has been unreliable. | +| `omdb_api_key` | str | `""` | [OMDb](https://www.omdbapi.com/) API key for IMDb metadata lookups; a more reliable alternative to IMDxAPI. Free keys are available on the OMDb site. | +| `tmdb_api_key` | str | `""` | TMDB API key for metadata enrichment and external-ID tags. | +| `simkl_client_id` | str | `""` | SIMKL client ID for metadata lookups; an alternative/fallback source to TMDB. | +| `tvdb_api_key` | str | `""` | [TheTVDB v4](https://thetvdb.com/api-information) API key; a fallback source to TMDB, strongest on TV series. Free keys are available on the TVDB site. | +| `tvdb_pin` | str | `""` | Subscriber PIN, only needed for a user-supported TVDB key. Leave empty for a normal project key. | +| `tvdb_order` | str | `""` | Default for `--tvdb-order`: renumber episodes to a TVDB season order (`official`, `dvd`, `absolute`, `alternate`, `regional`). Empty keeps the numbering the service gives. | +| `metadata_providers` | list | *(see below)* | Metadata providers to use, in the order they are tried. | +| `decrypt_labs_api_key` | str | `""` | Global Decrypt Labs API key (used by remote CDM / vault). | +| `ipinfo_api_key` | str | `""` | ipinfo.io API key for IP/region lookups. | + +!!! note "How `metadata_providers` is applied" + A lookup goes to each provider in turn and stops at the first good match, so the order + decides which source wins. `metadata_providers` sets that order: + + ```yaml + metadata_providers: [tvdb, tmdb, simkl] + ``` + + Names not in the list are never used, so this both orders and filters. Unknown names are + ignored with a warning. Leave it unset for the default order: + + ```yaml + metadata_providers: [imdbapi, omdb, simkl, tmdb, tvdb] + ``` + + Whatever the order, a provider is skipped when its API key is missing (or, for + `imdbapi`, when `imdb_api_enabled` is `false`). Available names are + `imdbapi`, `omdb`, `simkl`, `tmdb`, and `tvdb`. + +!!! note "`ipinfo_api_key` never touches your service sessions" + The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header; it + is **never** attached to your service session, so it cannot leak to a streaming provider. + Lookups degrade gracefully through a fallback chain: the authenticated Lite endpoint (higher + rate limits, and ASN/continent data, but no city or region) → anonymous ipinfo → + `ip-api.in` as a last resort. + +--- + +## Caching & updates { #caching-updates } + +### Title cache + +unshackle caches fetched title metadata (region- and account-aware) to avoid repeat API +calls. + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `title_cache_enabled` | bool | `true` | Master switch for the title cache. | +| `title_cache_time` | int (seconds) | `1800` (30 min) | Lifetime of fresh cached titles. | +| `title_cache_max_retention` | int (seconds) | `86400` (24 h) | Total retention window for a cached title, counted from when it was cached. Past `title_cache_time` the stale copy is only served as a fallback when a live fetch fails. | + +### Update checks + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `update_checks` | bool | `true` | Whether to check for new unshackle releases. | +| `update_check_interval` | int (hours) | `24` | Minimum hours between update checks. | + +!!! note + Update checks query the GitHub releases API with a fixed 5-second timeout and cache the + result in `directories.cache/update_check.json`. + +--- + +## Logging, privacy & debug { #logging-privacy-debug } + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `redact_paths` | bool | `true` | Mask install-root / venv / home prefixes in logged paths (`<unshackle>`, `<venv>`, `~`). Set `false` to show full paths. | +| `debug` | bool | `false` | Global debug mode. | +| `debug_keys` | bool | `false` | Log decryption keys. | +| `debug_requests` | bool | `false` | Log HTTP requests. | +| `set_terminal_bg` | bool | `false` | Append the theme's background colour to output styling. | + +!!! note "Effect of `set_terminal_bg`" + When on, the theme's background colour is appended to the foreground styles, which makes + the full colour palette render correctly on terminals whose *default* background differs + from unshackle's theme. When off, such terminals can render the ASCII-art banner and + coloured output incorrectly. + +!!! danger "Key exposure" + With `debug_keys` on, decryption content keys are written to the structured debug log at + INFO level during the DRM handshake. Treat `debug`, `debug_keys`, and the `unshackle_debug_*.jsonl` files as + sensitive, and keep `redact_paths` enabled when sharing logs. + + `debug_keys` affects **only** content-encryption keys (the `content_key`/`key` fields). + Passwords, tokens, cookies, and session tokens are **always** redacted regardless of this + setting, and KIDs, key counts, and other metadata are always logged either way. + +--- + +## Deprecated & removed keys { #deprecated-removed-keys } + +| Key | Status | Behaviour | +|-----|--------|-----------| +| `curl_impersonate` | **Deprecated** → use [`network`](network.md) | Emits a `DeprecationWarning`; still honoured only if `network` is absent. | +| `downloader` | **Deprecated** | Any value other than `"requests"` emits a `DeprecationWarning`; the value is otherwise ignored (the unified requests downloader is always used). | +| `scene_naming` | **Removed** | If set to any non-null value, unshackle exits with an error directing you to configure [`output_template`](output.md) instead. | diff --git a/docs/reference/configuration/network.md b/docs/reference/configuration/network.md new file mode 100644 index 00000000..652a000e --- /dev/null +++ b/docs/reference/configuration/network.md @@ -0,0 +1,86 @@ +# Network & proxy { #network-proxy } + +## `network` + +- **Type:** `dict`  ·  **Default:** `{}` + +TLS-fingerprinting and HTTP client settings for the rnet-based session. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `browser` | str | `"Chrome131"` | Impersonation preset. Must be an exact preset name (e.g. `Chrome131`, `Firefox135`, `Edge101`, `Safari18`, `OkHttp4_12`, `OkHttp5`, `Opera118`); unknown names raise an error. | +| `http1_only` | bool | *(unset)* | Force HTTP/1.1. | +| `http2_only` | bool | *(unset)* | Force HTTP/2. | +| `pool_max_idle_per_host` | int | *(unset)* | Connection-pool tuning. | +| `pool_max_size` | int | *(unset)* | Connection-pool tuning. | +| `tcp_nodelay` | bool | *(unset)* | Disable Nagle's algorithm. | + +```yaml +network: + browser: Firefox135 + http2_only: true +``` + +!!! note "Measured effect of `http1_only`" + Both `http1_only` and `http2_only` are unset by default. In benchmarks, forcing HTTP/1.1 + gained 30 to 50% on hosts that throttle per-connection or stall behind HTTP/2 flow control, + and **cost up to 27% on fast CDNs**. + +!!! warning "Renamed from `curl_impersonate`" + The old `curl_impersonate` section is a deprecated alias. If you still use it, unshackle + honours it (only when `network` is absent) but emits a `DeprecationWarning`. Rename it to + `network`. + +## `headers` + +- **Type:** `dict`  ·  **Default:** `{}` + +Default HTTP headers merged into every session unshackle creates. + +```yaml +headers: + Accept-Language: en-US,en;q=0.9 +``` + +!!! warning "Don't set `Accept-Encoding` (and similar) here" + Compatibility headers such as `Accept-Encoding` are set by the rnet HTTP backend as part of + its browser-impersonation profile; overriding them breaks the impersonation fingerprint. + This block is for cross-service defaults only (for example `Accept-Language`, + `User-Agent`); per-service headers belong in that service's own config. + +## `proxy_providers` + +- **Type:** `dict`  ·  **Default:** `{}` + +Proxy/VPN provider configuration. Each sub-key names a provider, and its block is passed +straight to that provider's constructor. See [Proxies & VPN](../../guide/proxies-and-vpn.md) for +the full provider guide. Recognised providers and their exit ports: + +| Provider | Config key | Credentials | Proxy scheme/port | +|----------|-----------|-------------|-------------------| +| Basic (static) | `basic` | country → URI(s) | as specified | +| NordVPN | `nordvpn` | **service** credentials (48 chars combined) | `https://...:89` | +| Surfshark | `surfsharkvpn` | **service** credentials (48 chars combined) | `https://...:443` | +| Windscribe | `windscribevpn` | service credentials | `https://...:443` | +| ExpressVPN | `expressvpn` | device login (`enable: true`) / token cache | `https://cat:...@...:443` | +| ProtonVPN | `protonvpn` | TV login or exported cookies | `https://...:4443` (or `:443` Secure Core) | +| Gluetun | `gluetun` | per-VPN keys/creds | `http://localhost:{port}` (local Docker) | +| Hola | *(none, auto)* | none | `http://...:{peer}` | + +```yaml +proxy_providers: + basic: + us: http://user:pass@1.2.3.4:8080 + de: + - http://a.example:8080 + - socks5://b.example:1080 + nordvpn: + username: <service username> + password: <service password> +``` + +!!! note "Provider loading differs between CLI and REST server" + The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API / + remote-client path uses a separate resolver that does **not** load `windscribevpn` or + `gluetun`. ExpressVPN and ProtonVPN also auto-load when their cached session exists, and Hola + auto-loads whenever the `hola-proxy` binary is present. diff --git a/docs/reference/configuration/output.md b/docs/reference/configuration/output.md new file mode 100644 index 00000000..5f696937 --- /dev/null +++ b/docs/reference/configuration/output.md @@ -0,0 +1,149 @@ +# Output & naming { #output-naming } + +Keys that control the output filename, the per-title subfolder, release-group tagging, and +filename character handling. For template cookbook material and worked naming examples, see +the [Output & naming guide](../../guide/output-and-naming.md). + +## `output_template` + +- **Type:** `dict`  ·  **Required** (no usable default; `dl` exits with an error if this is missing) + +Filename templates keyed by media kind, using `{variable}` placeholders. Recognised kinds are +`movies`, `series`, and `songs` (and `albums` for folder templates). A trailing `?` on a +variable (e.g. `{quality?}`) marks it optional: if empty, it and one adjacent separator are +dropped. + +```yaml title="unshackle.yaml" +output_template: + movies: "{title}.{year}.{quality?}.{source}-{tag}" + series: "{title}.{season_episode}.{episode_name?}.{quality?}.{source}-{tag}" + songs: "{track_number}. {title}" + folder: # special nested key, see below + movies: "{title} ({year})" + series: "{title} ({year})" +``` + +**The nested `folder` sub-key** controls output subfolders and is popped out of +`output_template`: + +- A **dict** sets per-kind folder templates (kinds: `movies`, `series`, `songs`, `albums`). +- A **str** sets a single folder template for everything. + +unshackle validates templates on load and emits warnings for unknown variables, non-string +values, filesystem-unsafe characters (`< > : " / \ | ? *`), empty templates, and unknown +folder-kind keys. + +??? example "All valid template variables" + `title`, `year`, `season`, `episode`, `season_episode`, `episode_name`, `date`, + `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, `album_artist`, + `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, `explicit`, + `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, `dual`, + `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`, + `title_type`. + +## Tagging & naming keys + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `tag` | str | `""` | Release-group tag. Fills the `{tag}` template variable and is written into MKV/audio tags. | +| `tag_group_name` | bool | `true` | Include the group name (`tag`) in the MKV `Group` tag. | +| `tag_imdb_tmdb` | bool | `true` | Write IMDb/TMDB/TVDB external-ID tags into the MKV (needs metadata providers). | +| `chapter_fallback_name` | str | `""` | Fallback chapter-name template, e.g. `"Chapter {i:02}"`. | +| `unicode_filenames` | bool | `false` | Allow Unicode in filenames; when `false`, names are ASCII-sanitised. | + +## `tag_rules` + +- **Type:** `list`  ·  **Default:** `[]` + +Conditionally replaces [`tag`](#tagging-naming-keys) for a given download. Each rule pairs a `when` +dict of conditions with the `tag` to use when they hold. Rules are evaluated in order and the +first match wins; if no rule matches, the global `tag` is kept. + +```yaml title="unshackle.yaml" +tag_rules: + - when: { title_type: movie, lang_tag: SUBBED } + tag: SUBMOVIES + - when: { quality: 2160p, hdr: [DV, DV.HDR, DV.HDR10P, HDR, HDR10P] } + tag: UHDGROUP +``` + +Rule semantics: + +- Every condition in `when` must hold (they are AND-ed). +- A value may be a scalar or a list; a list matches if **any** entry matches. +- Comparison is case-insensitive string equality. +- The keys of `when` are the filename template variables that describe the release itself: + `title_type`, `quality`, `resolution`, `source`, `video`, `hdr`, `hfr`, `audio`, + `audio_channels`, `audio_full`, `atmos`, `dual`, `multi`, `dubbed`, `edition`, `repack`, + and `lang_tag`. Only these are available. The title's own fields (`title`, `year`, `season`, + `episode`, and so on) are added after the rules run and cannot be matched. +- An unknown key in `when` is a mistake: unshackle logs a warning that names the key and skips + that rule. A rule without conditions or without a `tag` is also warned about and skipped. + +Put the more specific rules first, because the first match ends the search. + +!!! warning "Match `hdr` on the exact value" + Matching is exact, and `hdr` is a composite label for Dolby Vision titles: a DV title with + an HDR10 base layer gives `DV.HDR`, and with an HDR10+ base layer `DV.HDR10P`. Plain `DV` + does **not** match those. List every variant you want, as the example above does. Note + also that HDR10 gives `HDR`, not `HDR10`. The full set of values is `HDR`, `HDR10P`, `DV`, + `DV.HDR`, `DV.HDR10P`, and `HLG`, as listed in the + [variable table](../../guide/output-and-naming.md#template-variables). + +### Operators + +A condition value may start with a comparison operator, which is applied to the rest of the +value: + +```yaml title="unshackle.yaml" +tag_rules: + - when: { resolution: "<1080" } + tag: SDGROUP + - when: { quality: ">=2160" } + tag: UHDGROUP + - when: { video: "!=H.265" } + tag: AVCGROUP +``` + +- `>`, `<`, `>=`, `<=` compare numbers. unshackle reads the first number out of the actual + value, so `">=2160"` matches a quality of `2160p`. If the actual value has no number, the + condition fails. +- `=`, `==`, and `!=` compare text, case-insensitively, when the operand is not a number. + With a numeric operand they compare numbers like the operators above. A value with no number + in it is never equal to a number, so `"!=2160"` holds for it and `"==2160"` does not. +- An ordering operator with a text operand (for example `">DV"`) is a mistake: unshackle logs + a warning and the condition fails. So does an operator with nothing after it (`">="`). +- A value with no operator keeps the plain equality behaviour described above. + +!!! warning "Quote operator values in YAML" + Two of these operators break an unquoted YAML value, and both stop the config from + loading: + + - `>` opens a folded block scalar, so `quality: >=2160` fails with a "while scanning a + block scalar" error. + - `!` starts a tag, so `hdr: !=DV` fails with a "could not determine a constructor" + error. + + Quote every operator value: `quality: ">=2160"`, `resolution: "<1080"`, `video: "!=H.265"`. + +!!! note "Keep a negation out of lists" + A list matches if any entry matches, so two or more `!=` entries in one list are always + true. Give a negation as a single value, not as a list entry. + +`title_type` is the media kind of the title: `movie`, `series`, or `music`. It is useful to +give each kind its own group: + +```yaml +tag_rules: + - when: { title_type: movie } + tag: MOVIEGRP + - when: { title_type: series } + tag: TVGRP +``` + +The resolved tag applies to the `{tag}` filename variable only. The `Group` metadata tag +written into MKV and music files always uses [`tag`](#tagging-naming-keys). + +!!! note "`--tag` wins" + An explicit `dl --tag` on the command line disables the rules completely and uses the + given tag. diff --git a/docs/reference/configuration/services.md b/docs/reference/configuration/services.md new file mode 100644 index 00000000..3ba1043a --- /dev/null +++ b/docs/reference/configuration/services.md @@ -0,0 +1,137 @@ +# Services & authentication { #services-auth } + +## `services` + +- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` + +Per-service configuration, keyed by the canonical service tag (for example `EXAMPLE1`, `EXAMPLE2`, +`EXAMPLE3`). unshackle reads several well-known sub-keys out of each service's block: + +- **`proxy_map`**: remaps the region query you passed to `-p/--proxy` for this service. A key is + that query, or `"provider:query"` when the flag also named a provider; the value is the query + asked of the proxy provider instead. It has no effect when `-p` was given a full proxy URL. +- **`title_map`**: an exact-match rename map applied to fetched titles (source name → + desired name), so a service that names a title differently from your library still matches. +- **`dl`**: per-service download defaults, using the same keys as the global + [`dl`](download.md#dl) block. + +Individual services may read any additional keys they define. The merged result (the service's +own `config.yaml` with this block layered on top) is handed to the service as `self.config`. + +User- or device-specific values (API keys, account IDs, device attributes) belong in this block +rather than in the service's own `config.yaml`, which holds shared defaults; see +[creating a service](../../dev/creating-a-service.md). + +```yaml +services: + EXAMPLE1: + proxy_map: + us: us-nyc + "nordvpn:ca": ca-toronto + title_map: + "My Show: The Movie": "My Show Presents The Movie" +``` + +## `credentials` + +- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` + +Per-service login credentials. Each value is a `username:password[:extra]` string, the same +data as a `[username, password]` (or `[username, password, extra]`) list, or a dict of profile +name to either of those. With the dict form, `-p/--profile` selects the entry, and the +`default` key is used when no profile was given or the named profile is missing. These are +parsed into `Credential` objects; the credential's SHA-1 is also used as an account hash for +cache keys. + +```yaml +credentials: + EXAMPLE2: user@example.com:hunter2 + EXAMPLE1: + default: primary@example.com:pw1 + second: secondary@example.com:pw2 +``` + +!!! tip "Cookies vs credentials" + Cookies are stored as files under [`directories.cookies`](directories.md), not in this key. + A service's `authenticate()` accepts cookies, credentials, or both. + +## `firefox_cookies` + +- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` + +Settings for extracting cookies directly from a local Firefox profile. A service block is +expected to provide `hosts` (a list of cookie hostnames; entries shorter than 3 characters +are ignored to prevent dumping the whole cookie store) and an optional `local_storage` +boolean to also pull matching entries from `webappsstore.sqlite`, which only services that +keep auth tokens in localStorage rather than in HTTP cookies need. Extraction is read-only. + +!!! note "Firefox does not need to be closed" + The extractor copies **both** `cookies.sqlite` **and** its WAL file into a `0700` temp + directory, so writes Firefox has not yet flushed to the main DB are included. Extraction + fails only if Firefox holds an exclusive write lock at the instant of the copy. The live + profile is not modified. + +!!! warning "Extraction falls back silently to file cookies" + If extraction yields no cookies or fails for **any** reason, unshackle silently falls back + to the normal file-based cookie path (`cookies/<SERVICE>.txt` or + `cookies/<SERVICE>/<profile>.txt`), with no error reported. + +## `remote_services` + +- **Type:** `dict`  ·  **Default:** `{}` + +Definitions of remote unshackle service servers, used by the `--remote` mode. Each entry is +named by you (pick it with `--server`, or omit that flag when only one is configured) and gives +the server's `url` (required), an optional `api_key` sent as `X-Secret-Key`, an optional +`server_cdm` boolean, and an optional `services` sub-dict of per-service local overrides such +as `title_map`. In `--remote` mode unshackle turns the server's service list into synthetic CLI +commands that run against it, falling back to the tags in that `services` sub-dict when the +list cannot be fetched. See [remote sessions](../../dev/rest-api/remote-sessions.md) for the +full setup. + +## `serve` + +- **Type:** `dict`  ·  **Default:** `{}` + +Configuration for the `serve` command (the built-in REST API server). The full server guide +is the [REST API](../../dev/rest-api/index.md) section; these are the config keys. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `api_secret` | str | *(unset)* | Master secret accepted in the `X-Secret-Key` header. Required unless the server is started with `--no-key`. | +| `users` | dict | `{}` | Per-user API keys and their allowlists (see below). | +| `services` | list | *(unset)* | Global service allowlist. Omit to allow all. | +| `remote_only` | bool | `false` | Expose only the remote service session endpoints (health, services, search, session) and disable the rest of the REST API. | +| `session_ttl` | int (s) | `300` | Lifetime of an interactive auth session. | +| `max_sessions` | int | `100` | Maximum concurrent sessions. | +| `history_limit` | int | `100` | How many finished jobs to retain in history. | +| `compression_level` | int | `1` | gzip level for responses. | +| `global_speed_limit` | str | *(unlimited)* | Server-wide download speed cap, e.g. `10M`, `1.5G` or plain bytes/sec (same format as `speed_limit`). One shared budget across all concurrent jobs; per-job speed limits are ignored while it is set. | +| `cdm_overrides` | list or bool | *(unset)* | Allowed per-request CDM overrides: a list of permitted device names, or `true` for any. Unset rejects every override. | +| `allow_job_credentials` | bool | `false` | Permit clients to supply credentials per job. | +| `devices` | list | *(auto)* | Widevine devices offered; auto-filled from `directories.wvds`. | +| `playready_devices` | list | *(auto)* | PlayReady devices; auto-filled from `directories.prds`. | + +Each entry under `users` is keyed by that user's API key and may set its own `services`, +`devices`, and `playready_devices` allowlists, narrowing the global ones, plus an optional +`username` used as the log label for that key (defaults to a truncated form of the key). A +user with no `playready_devices` key gets no PlayReady access at all, not the global list. + +```yaml +serve: + api_secret: change-me + remote_only: true + services: [EXAMPLE1, EXAMPLE2] + # server-wide download defaults (same keys as the `dl:` block) + downloads: 3 + best_available: true + users: + a1b2c3d4: # this user's API key + services: [EXAMPLE1] # may only use EXAMPLE1 +``` + +!!! note "`dl` keys inside `serve`" + Most [`dl`](download.md#dl) flag keys (`downloads`, `workers`, `best_available`, and so on) + can be set directly inside `serve`, where they apply to every request the server handles. + The server recognises a fixed subset of download parameters, so a few CLI-only flags are + ignored here. diff --git a/docs/reference/configuration/vaults.md b/docs/reference/configuration/vaults.md new file mode 100644 index 00000000..2ef9cbd1 --- /dev/null +++ b/docs/reference/configuration/vaults.md @@ -0,0 +1,47 @@ +# Key vaults { #key-vaults } + +The full guide is at [Vaults](../../guide/vaults.md). Two keys configure them. + +## `key_vaults` + +- **Type:** `list[dict]`  ·  **Default:** `[]` + +An ordered list of key-vault backends. unshackle queries them in order and reuses content +keys instead of re-licensing. Each entry needs a `type` (the backend module name) and a +`name`, plus backend-specific keys. + +| `type` | Purpose | Required keys | Notes | +|--------|---------|--------------|-------| +| `SQLite` | Local SQLite database | `name`, `path` | Loaded **critically**; a failure aborts the run. | +| `MySQL` | Remote MySQL database | `name`, `host`, `database`, `username` | Extra keys (e.g. `password`, `port`) forwarded to pymysql. | +| `API` | RESTful JSON API | `name`, `uri`, `token` | Optional `headers` map is sent with every request. Honours `vault_timeout`. | +| `HTTP` | HTTP API with modes | `name`, `host`, one of `password`/`api_key`, and `username` in query mode | `api_mode`: `query` (default), `json`, `decrypt_labs`. Honours `vault_timeout`. | + +```yaml +key_vaults: + - type: SQLite + name: Local + path: ~/.unshackle/keys.db + - type: MySQL + name: Team + host: db.example.com + database: keys + username: unshackle + password: hunter2 + no_push: false +``` + +!!! note "Per-entry options" + - `no_push: true` makes a vault read-only (keys are fetched but never written to it). + - A vault of `type: API` whose `name` contains `decrypt_labs` auto-fills its `token` from + [`decrypt_labs_api_key`](misc.md#external-api-keys) when not set inline. Vault `type` values are + case-sensitive module names. + - An all-zero content key (32 zeros) is treated as "no key" everywhere and is never + stored. + +## `vault_timeout` + +- **Type:** `float`  ·  **Default:** `10.0` + +Timeout in seconds for vault operations. Injected automatically into any backend whose +constructor accepts a `timeout` parameter (a per-vault `timeout` still wins). diff --git a/mkdocs.yml b/mkdocs.yml index c7977bef..8fa8d712 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,6 +5,9 @@ repo_url: https://github.com/unshackle-dl/unshackle repo_name: unshackle-dl/unshackle docs_dir: docs +exclude_docs: | + reference/configuration.md + theme: name: material custom_dir: overrides @@ -117,7 +120,16 @@ nav: - Subtitles: guide/subtitles.md - Troubleshooting: guide/troubleshooting.md - Configuration: - - Reference: reference/configuration.md + - Overview: reference/configuration/index.md + - Directories: reference/configuration/directories.md + - Services & Authentication: reference/configuration/services.md + - Download & Processing: reference/configuration/download.md + - Output & Naming: reference/configuration/output.md + - DRM & CDM: reference/configuration/drm.md + - Network & Proxy: reference/configuration/network.md + - Key Vaults: reference/configuration/vaults.md + - Other Settings: reference/configuration/misc.md + - Annotated Example: reference/configuration/example.md - Developer: - Creating a Service: dev/creating-a-service.md - Service Architecture: dev/service-architecture.md From 041ba5293bb45f6680266bb754d0a27e63093ef2 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Tue, 4 Aug 2026 17:38:30 -0600 Subject: [PATCH 38/40] feat(titles): add optional Episode.part, selected with -w s1e1.2 Opt-in and additive: an episode without a part behaves exactly as before. --- docs/dev/creating-a-service.md | 53 +++- docs/dev/rest-api/endpoints.md | 11 +- docs/guide/cli-reference.md | 2 +- docs/guide/downloading.md | 39 +++ docs/guide/output-and-naming.md | 27 ++ docs/reference/configuration/output.md | 20 +- tests/core/test_click_types.py | 76 ++++- tests/core/test_episode_part.py | 199 +++++++++++++ tests/core/test_tvdb.py | 59 ++++ tests/remote/unit/test_api_episode_part.py | 331 +++++++++++++++++++++ unshackle/commands/dl.py | 14 +- unshackle/core/api/download_manager.py | 3 +- unshackle/core/api/handlers.py | 27 +- unshackle/core/config.py | 1 + unshackle/core/remote_service.py | 1 + unshackle/core/titles/episode.py | 70 ++++- unshackle/core/utils/click_types.py | 57 +++- 17 files changed, 946 insertions(+), 44 deletions(-) create mode 100644 tests/core/test_episode_part.py create mode 100644 tests/remote/unit/test_api_episode_part.py diff --git a/docs/dev/creating-a-service.md b/docs/dev/creating-a-service.md index a2578d89..1c3d5f9b 100644 --- a/docs/dev/creating-a-service.md +++ b/docs/dev/creating-a-service.md @@ -304,11 +304,14 @@ Common constructor fields: ```python Episode(id_, service, title, season, number, name=None, year=None, - language=None, data=None, description=None, air_date=None) + language=None, data=None, description=None, air_date=None, + part=None) ``` Pass `air_date` for daily/sports content to name by date instead of - `SxxExx`. Return a `Series([...])` of episodes. + `SxxExx`. Return a `Series([...])` of episodes. Pass `part` only when the + service splits one episode into several videos, see + [Split episodes](#split-episodes). === "Song" @@ -328,6 +331,52 @@ Common constructor fields: framework path to reuse cached results and gracefully fall back to stale data when the API errors. It also applies the per-service `title_map` config. +#### Split episodes {#split-episodes} + +Some services split one logical episode into several separately playable videos, for +example a variety show that ships episode 1 as *Episode 1 (Part 1)*, *(Part 2)* and +*(Part 3)*. `Episode.number` stays an `int`. The optional `part` carries the index. + +```python +Episode(..., season=1, number=1, part=1, name="The Reckoning") +Episode(..., season=1, number=1, part=2, name="The Reckoning") +``` + +1. `part` counts from 1, contiguous. `part=0` is rejected, and `-w` can only address + parts up to 99. +2. Every part of one episode shares the same `season` and `number`. +3. Each part is a separate `Episode` with its own unique `id_`. +4. Unsplit episode: leave `part` unset. Never pass `part=1` for a whole episode. +5. Do not put the part in `name` or `number`. Pass a clean `name` (or `None`) and set + `part`. +6. `part` is a split *episode*, not a split *season*. A service listing *Season 1 Part 2* + describes a group of episodes. Keep numbering those in sequence, the way the shipped + services do. + +`part` accepts an `int` or a digit string. A value that is not an integer raises +`TypeError`. A value of `0` or less raises `ValueError`. + +!!! warning "The framework never infers a part from the episode name" + Rule 6 is the reason. Shipped services already use the word "Part" to mean a *season* + part, so name-sniffing would misread every one of them. The part has to arrive + structurally, as the `part` argument. + +!!! tip "Clean the part out of the name" + `Episode` discards any `name` that starts with `Episode <number>`, so a raw + `"Episode 1 (Part 2): The Reckoning"` leaves you with no episode name at all. Strip + the prefix before you pass it, or pass `None`. + +Once `part` is set, the filename gains a part token (`S01E01.Part.2`, see +[Output & naming](../guide/output-and-naming.md#split-episodes)), `-w S01E01.2` selects a +single part while a bare `-w S01E01` selects them all, and the `--list-titles` tree labels +the episode `01.2`. The REST API reports it as an extra `part` key on the serialized +title, described in [Endpoints](rest-api/endpoints.md#post-apilist-titles). + +!!! note "Opt-in and additive" + Leaving `part` unset costs you nothing: filenames, sort order, `-w` matching and the + API JSON are identical to a plain episode, so a service that does not split episodes + has nothing to adopt. + ### `get_tracks`: required Given one title, return a `Tracks` object holding `Video`, `Audio`, and diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 8fc42797..05721df4 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -223,6 +223,12 @@ Service-specific CLI options may also be passed as additional keys. Each serialized title carries `type` (`"episode"`, `"movie"`, or `"other"`), `name`, `id`, `language`, `description`, `date`, and `cover_url`. Episodes and movies add `year`; episodes additionally add `series_title`, `season`, and `number`. +!!! info "`part` on a split episode" + A service that splits one episode into several separately playable videos reports each + one as its own episode. Those titles share a `season` and `number` and add a `part` + integer counting from 1. No other title carries the key, so a client that ignores it + is unaffected. See [Split episodes](../creating-a-service.md#split-episodes). + | Status | Error code | Meaning | | --- | --- | --- | | `200` | - | Titles returned. | @@ -248,9 +254,10 @@ List the video, audio, and subtitle tracks for a title. For series, you can scop | `wanted` | string | no | Episode/season range (e.g. `"S01E01-S01E03"`). | | `season` | int/string | no | Season number (combined with `episode`). | | `episode` | int/string | no | Episode number (combined with `season`). | +| `part` | int/string | no | Part index of a split episode (combined with `season` and `episode`). | | `profile`, `proxy`, `no_proxy`, `cdm_type` | - | no | As on `list-titles`. | -When both `season` and `episode` are given, they are combined into a `"{season}x{episode}"` selector. +When both `season` and `episode` are given, they are combined into a `"{season}x{episode}"` selector. Adding `part` narrows that to `"{season}x{episode}.{part}"`, one part of a [split episode](../creating-a-service.md#split-episodes); without it you get every part of the episode. `part` is read only when `season` and `episode` are both given, and `wanted` takes precedence over all three. To scope a part by range, put it in `wanted` instead (`"S01E01.2"`). === "Single title / movie: Response `200`" @@ -361,7 +368,7 @@ Create a download job. Requires `service` and `title_id`; every other field is a | `range` | string[] | `["SDR"]` | Dynamic range(s). | | `channels` | number | `null` | Audio channel count. | | `no_atmos` | boolean | `false` | Exclude Atmos tracks. | -| `wanted` | string[] | `[]` | Episode/season selectors. | +| `wanted` | string[] | `[]` | Episode/season selectors. Accepts the part form, `"S01E01.2"`. | | `latest_episode` | boolean | `false` | Only the newest episode. | | `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. | | `require_subs` | string[] | `[]` | Required subtitle languages. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index e6a1c8f9..4e0ec417 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -112,7 +112,7 @@ The special language tokens `orig`, `all`, and `best` are honoured everywhere a | Flag | Description | |---|---| -| `-w`, `--wanted` | Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`. Supports exclusions with a leading `-` (e.g. `-S03`). | +| `-w`, `--wanted` | Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`. Supports exclusions with a leading `-` (e.g. `-S03`). For a [split episode](downloading.md#split-episodes), `.N` picks one part (`S01E01.2`), a range must stay inside the episode (`S01E01.1-S01E01.3`), and `S01E01` on its own takes every part. | | `--select-titles` | Interactively select what to download: episodes of a series, or films when a title has more than one. **Cannot combine with `-w`.** | | `--latest-episode` | Download only the single most recent episode. | | `--list-titles` | List titles only; do not download. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 047b4be1..a8e013d8 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -368,6 +368,45 @@ with `-` to exclude it. unshackle dl -w S01-S05,-S03 EXAMPLE 81234567 ``` +=== "Split episodes" + + ```shell + # Part 2 of episode 1 only + unshackle dl -w S01E01.2 EXAMPLE 81234567 + + # Parts 1 through 3 of episode 1 + unshackle dl -w S01E01.1-S01E01.3 EXAMPLE 81234567 + + # All of season 1 except episode 1 part 2 + unshackle dl -w S01,-S01E01.2 EXAMPLE 81234567 + ``` + +### Split episodes + +A few services split one episode into several separately playable videos. Where a service +reports that, add `.N` after the episode to pick one part. + +| Token | Selects | +| --- | --- | +| `S01E01` | All parts of episode 1. | +| `S01E01.2` | Part 2 of episode 1, and nothing else. | +| `S01E01.1-S01E01.3` | Parts 1 to 3 of episode 1. | +| `S01,-S01E01.2` | All of season 1 except episode 1 part 2. Parts 1 and 3 are kept. | + +A season token such as `S01` covers every episode and every part, so you only need `.N` +when you want a part on its own. + +!!! warning "A part range stays inside one episode" + `S01E01.1-S01E01.3` is valid. `S01E01.1-S01E02.3` is rejected, because the parser + cannot know how many parts episode 1 has and so cannot work out where the range ends. + To span episodes, list the parts you want as separate tokens: + `-w S01E01.2,S01E02.1`. + +!!! note "A part of an unsplit episode selects nothing" + `-w S01E02.2` on an episode that was never split is deliberately empty: it tells you + the episode has no parts, rather than quietly handing you the whole episode. Run + `--list-titles` to see which episodes carry parts. + ### Other selection flags | Flag | Behaviour | diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 9b412d05..abaed437 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -126,11 +126,38 @@ Every variable below is valid in both output and folder templates. Values are de | `episode` | Zero-padded episode (`E%02d`) | `E05` | | `season_episode` | Combined | `S01E05` | | `episode_name` | Episode title | `Pilot` | +| `part` | Part index of a [split episode](#split-episodes), empty otherwise | `2` | | `date` | ISO air date for daily/dated content | `2024-06-01` | !!! note "Daily & sports content" When an episode has an air date, unshackle switches to date-based naming automatically: `season` and `season_episode` become the formatted air date, `episode` and `year` are cleared, and `{date}` holds the ISO date. The date's internal separator (dots or spaces) follows your `series` template style. +### Split episodes {#split-episodes} + +A few services split one episode into several separately playable videos. Where a service reports that, the part index is folded into `{episode}` and `{season_episode}`, so the stock `series` template names it with no change to your config. + +```text +Show.Name.S01E01.Part.2.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv +``` + +The part token sits immediately after the episode token, before the quality tags, and the group tag stays last. Its separator follows your template's own style, the same way dated content does, so a spaced template gives `S01E01 Part 2` instead. + +The part is left out of the folder name, so every part of an episode lands in the same season folder: + +```text +The.Show.S01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG/ + ├─ The.Show.S01E01.Part.1.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv + ├─ The.Show.S01E01.Part.2.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv + └─ The.Show.S01E02.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv +``` + +The same holds for a folder template of your own: `{season}` never picks up the part, and `{season_episode}` drops it inside a folder template. An episode with no part is named like any other episode. + +!!! note "What media servers do with the part token" + The form above follows scene practice for genuinely split episodes. Kodi stacks the parts back into one playable episode automatically. Jellyfin lists two versions of S01E01, still correct and playable, because its stacking pattern expects the part marker at the end of the name. Plex's scanner is closed-source, so its handling of text after the part token is unverified. + +For custom templates there is also a standalone [`{part}`](../reference/configuration/output.md#output_template) variable. You rarely want it, see the caveat on that page. + ### Audio | Variable | Meaning | Example | diff --git a/docs/reference/configuration/output.md b/docs/reference/configuration/output.md index 5f696937..2a58d909 100644 --- a/docs/reference/configuration/output.md +++ b/docs/reference/configuration/output.md @@ -34,13 +34,23 @@ values, filesystem-unsafe characters (`< > : " / \ | ? *`), empty templates, and folder-kind keys. ??? example "All valid template variables" - `title`, `year`, `season`, `episode`, `season_episode`, `episode_name`, `date`, - `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, `album_artist`, - `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, `explicit`, - `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, `dual`, - `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`, + `title`, `year`, `season`, `episode`, `season_episode`, `episode_name`, `part`, + `date`, `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, + `album_artist`, `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, + `explicit`, `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, + `dual`, `multi`, `dubbed`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`, `title_type`. +!!! warning "`{part?}` is already inside `{season_episode}`" + `part` holds the bare part index of a + [split episode](../../guide/output-and-naming.md#split-episodes) (`2`) and is empty on + every other title. unshackle folds it into `{episode}` and `{season_episode}`, so the + shipped `series` template names split episodes correctly with no edit. Putting + `{part?}` next to either of them renders the part twice (`S01E01.Part.2.2`). Use the + standalone variable only in a template that names neither `{episode}` nor + `{season_episode}`, and note that it is always empty in a folder template, since + folders do not carry the part. + ## Tagging & naming keys | Key | Type | Default | Description | diff --git a/tests/core/test_click_types.py b/tests/core/test_click_types.py index 6b875ec9..4581816a 100644 --- a/tests/core/test_click_types.py +++ b/tests/core/test_click_types.py @@ -3,10 +3,11 @@ services set via the ``sub_format`` override (must not be rejected as an invalid from __future__ import annotations +import click import pytest from unshackle.core.tracks.subtitle import Subtitle -from unshackle.core.utils.click_types import QUALITY_LIST, SLOW_DELAY_RANGE, SubtitleCodecChoice +from unshackle.core.utils.click_types import QUALITY_LIST, SLOW_DELAY_RANGE, SeasonRange, SubtitleCodecChoice choice = SubtitleCodecChoice(Subtitle.Codec) @@ -57,3 +58,76 @@ def test_quality_list_accepts_yaml_native_values(value, expected): ) def test_slow_delay_range_accepts_bool(value, expected): assert SLOW_DELAY_RANGE.convert(value, None, None) == expected + + +# --- SeasonRange ------------------------------------------------------------ + + +def parse(*tokens: str) -> list[str]: + return SeasonRange().parse_tokens(*tokens) + + +def test_single_episode_token_is_unchanged(): + assert parse("S01E01") == ["1x1"] + + +def test_bare_season_spans_the_whole_episode_range(): + """A bare S01 reads the one match object with a different default per side.""" + keys = parse("S01") + assert len(keys) == 1000 + assert "1x0" in keys and "1x999" in keys + + +def test_multi_token_range_is_unchanged(): + assert sorted(parse("S02E01", "S02E03-S02E05")) == ["2x1", "2x3", "2x4", "2x5"] + + +def test_exclusions_are_unchanged(): + keys = parse("S01-S05", "-S03", "-S02E01") + assert not any(k.startswith("3x") for k in keys) + assert "2x1" not in keys + assert "2x0" in keys and "2x2" in keys + assert len(keys) == 3999 # 5 seasons less all of S03 less 2x1 + + +@pytest.mark.parametrize("tokens", [("S01E01",), ("S01",), ("S01-S05", "-S03", "-S02E01"), ("S02E03-S02E05",)]) +def test_part_less_tokens_never_emit_a_part_or_exclusion_key(tokens): + assert all("." not in k and "!" not in k for k in parse(*tokens)) + + +def test_part_token(): + assert parse("S01E01.2") == ["1x1.2"] + + +def test_part_range_within_one_episode(): + assert sorted(parse("S01E01.1-S01E01.3")) == ["1x1.1", "1x1.2", "1x1.3"] + + +def test_part_exclusion_becomes_a_negative_key(): + keys = parse("S01", "-S01E01.2") + assert "!1x1.2" in keys + assert "1x1" in keys # base key stays so the other parts still match + + +def test_base_exclusion_covers_part_keys(): + assert parse("S01E01.1-S01E01.3", "S01E02", "-S01E01") == ["1x2"] + + +def test_exclusion_removes_duplicate_keys(): + # "S01" and "S01E01" both emit 1x1; the exclusion must remove every copy + assert "1x1" not in parse("S01", "S01E01", "-S01E01") + + +@pytest.mark.parametrize( + "token", + [ + "S01E01.0", # parts count from 1 + "S01E01.1-S01E02.2", # a part range cannot cross episodes + "S01-S01E01.2", # a part on only one side + "S01E01.3-S01E01.1", # reversed part range + "S01E01.200", # above MAX_PART + ], +) +def test_bad_part_tokens_fail(token): + with pytest.raises(click.UsageError): + parse(token) diff --git a/tests/core/test_episode_part.py b/tests/core/test_episode_part.py new file mode 100644 index 00000000..b4d1e749 --- /dev/null +++ b/tests/core/test_episode_part.py @@ -0,0 +1,199 @@ +"""Tests for multi-part episodes: Episode.part, its naming, sorting and -w matching.""" + +from __future__ import annotations + +import pytest + +from unshackle.core.config import config +from unshackle.core.titles.episode import Episode, Series +from unshackle.core.utils.click_types import SeasonRange + + +class DummyService: + pass + + +class StubMediaInfo: + """Minimal MediaInfo stand-in: the template context only reads track lists.""" + + video_tracks: list = [] + audio_tracks: list = [] + + +def make_episode(**overrides) -> Episode: + kwargs = dict( + id_="ep-0001", + service=DummyService, + title="The Show", + season=1, + number=1, + name="The Reckoning", + ) + kwargs.update(overrides) + return Episode(**kwargs) + + +@pytest.fixture +def reset_template(): + saved = (config.output_template, config.folder_templates, config.folder_template, config.tag) + config.output_template = {"series": "{title}.{season_episode}.{quality?}.{source}-{tag}"} + config.folder_templates = {} + config.folder_template = "" + config.tag = "GROUP" + yield config + config.output_template, config.folder_templates, config.folder_template, config.tag = saved + + +# --- validation ------------------------------------------------------------- + + +def test_part_defaults_to_none(): + assert make_episode().part is None + + +def test_digit_string_part_coerces(): + assert make_episode(part="2").part == 2 + + +@pytest.mark.parametrize("bad", [0, -1]) +def test_non_positive_part_rejected(bad): + with pytest.raises(ValueError): + make_episode(part=bad) + + +def test_non_int_part_rejected(): + with pytest.raises(TypeError): + make_episode(part="x") + + +def test_bool_part_rejected(): + """bool is an int subclass; True would render as '.True' in keys and filenames.""" + with pytest.raises(TypeError): + make_episode(part=True) + + +def test_old_cache_entry_reads_as_part_less(): + """jsonpickle restores by populating __dict__; an older entry has no 'part' key.""" + restored = Episode.__new__(Episode) + restored.__dict__.update(title="The Show", season=1, number=1, name=None, year=None) + assert restored.part is None + + +# --- display ---------------------------------------------------------------- + + +def test_str_is_unchanged_without_a_part(): + without = make_episode() + assert str(without) == "The Show S01E01 The Reckoning" + + +def test_str_shows_the_selection_syntax(): + assert str(make_episode(part=2)) == "The Show S01E01.2 The Reckoning" + + +def test_str_tells_dated_parts_apart(): + # dated content renders the air date instead of SxxExx, so without the suffix both + # parts print the same string while their filenames differ + assert str(make_episode(part=2, air_date="2024-05-06")) == "The Show 2024.05.06.2 The Reckoning" + assert str(make_episode(air_date="2024-05-06")) == "The Show 2024.05.06 The Reckoning" + + +def test_tree_label_carries_the_part_on_dated_content(): + series = Series([make_episode(part=1, air_date="2024-05-06"), make_episode(part=2, air_date="2024-05-06")]) + labels = [node.label for node in series.tree(verbose=True).children[0].children] + assert labels == [ + "[bold]2024.05.06.1.[/] [bright_black]The Reckoning", + "[bold]2024.05.06.2.[/] [bright_black]The Reckoning", + ] + + +def test_tree_label_carries_the_part(): + series = Series([make_episode(part=1), make_episode(part=2)]) + labels = [node.label for node in series.tree(verbose=True).children[0].children] + assert labels == ["[bold]1.1.[/] [bright_black]The Reckoning", "[bold]1.2.[/] [bright_black]The Reckoning"] + + +def test_series_sorts_parts_within_their_episode(): + series = Series( + [ + make_episode(number=2), + make_episode(number=1, part=2), + make_episode(number=1), + make_episode(number=1, part=1), + ] + ) + assert [(e.number, e.part) for e in series] == [(1, None), (1, 1), (1, 2), (2, None)] + + +# --- naming ----------------------------------------------------------------- + + +def test_filename_is_unchanged_without_a_part(reset_template): + assert make_episode().get_filename(StubMediaInfo()) == "The.Show.S01E01.DummyService-GROUP" + + +def test_dotted_template_folds_the_part_after_the_episode(reset_template): + name = make_episode(part=2).get_filename(StubMediaInfo()) + assert name == "The.Show.S01E01.Part.2.DummyService-GROUP" + assert name.endswith("-GROUP") # group tag stays last + + +def test_spaced_template_uses_spaces(reset_template): + reset_template.output_template = {"series": "{title} {season_episode} {quality?} {source}-{tag}"} + assert make_episode(part=2).get_filename(StubMediaInfo()) == "The Show S01E01 Part 2 DummyService-GROUP" + + +def test_template_context_leaves_the_season_token_alone(reset_template): + ctx = make_episode(part=2)._build_template_context(StubMediaInfo()) + assert ctx["season"] == "S01" + assert ctx["episode"] == "E01.Part.2" + assert ctx["season_episode"] == "S01E01.Part.2" + assert ctx["part"] == 2 + + +def test_part_token_is_empty_without_a_part(reset_template): + assert make_episode()._build_template_context(StubMediaInfo())["part"] == "" + + +def test_parts_share_one_season_folder(reset_template): + # {season_episode} in a folder template would otherwise carry the part into the path + reset_template.folder_template = "{title}/{season_episode}" + folders = {make_episode(part=p).get_filename(StubMediaInfo(), folder=True) for p in (1, 2)} + assert folders == {"The.Show/S01E01"} + + +def test_parts_share_one_derived_season_folder(reset_template): + folders = {make_episode(part=p).get_filename(StubMediaInfo(), folder=True) for p in (1, 2)} + assert len(folders) == 1 + assert folders == {make_episode().get_filename(StubMediaInfo(), folder=True)} + + +# --- matching --------------------------------------------------------------- + +# the worked key scheme from the design: (token, part-less E1 selected, E1 part 2 selected) +MATCH_TABLE = [ + (("S01",), True, True), + (("S01E01",), True, True), + (("S01E01.2",), False, True), + (("S01E01.1-S01E01.3",), False, True), + (("S01", "-S01E01.2"), True, False), + (("S01", "-S01E01"), False, False), + (("S01E02",), False, False), +] + + +@pytest.mark.parametrize("tokens,wants_plain,wants_part2", MATCH_TABLE) +def test_matches_wanted_table(tokens, wants_plain, wants_part2): + wanted = SeasonRange().parse_tokens(*tokens) + assert make_episode().matches_wanted(wanted) is wants_plain + assert make_episode(part=2).matches_wanted(wanted) is wants_part2 + + +def test_a_part_exclusion_leaves_the_other_parts_alone(): + wanted = SeasonRange().parse_tokens("S01", "-S01E01.2") + assert make_episode(part=1).matches_wanted(wanted) is True + assert make_episode(part=3).matches_wanted(wanted) is True + + +def test_asking_for_a_part_of_an_unsplit_episode_selects_nothing(): + assert make_episode().matches_wanted(SeasonRange().parse_tokens("S01E01.2")) is False diff --git a/tests/core/test_tvdb.py b/tests/core/test_tvdb.py index 03f82573..c50711a8 100644 --- a/tests/core/test_tvdb.py +++ b/tests/core/test_tvdb.py @@ -171,3 +171,62 @@ def test_get_episodes_does_not_cache_an_empty_listing(monkeypatch: pytest.Monkey monkeypatch.setattr(p, "_get", lambda _path, params: None) assert p.get_episodes(73871, "official") == [] assert p._episodes == {} + + +# --- apply_tvdb_order over multi-part episodes ------------------------------- + + +class _Svc: + pass + + +class _FakeProvider: + def __init__(self, mapping: dict) -> None: + self.mapping = mapping + + def detect_order(self, _id, _keys) -> str: + return "official" + + def get_order_map(self, _id, _order, source_order=None) -> dict: + return self.mapping + + +def _renumber(monkeypatch: pytest.MonkeyPatch, episodes: list, mapping: dict): + from types import SimpleNamespace + + from unshackle.commands.dl import dl + from unshackle.core.titles.episode import Series + + monkeypatch.setattr(providers, "get_provider", lambda _name: _FakeProvider(mapping)) + errors: list = [] + stub = SimpleNamespace( + tvdb_id=73871, + tvdb_order="dvd", + service="STUB", + log=SimpleNamespace(warning=lambda *a, **k: None, info=lambda *a, **k: None, error=lambda *a: errors.append(a)), + ) + return dl.apply_tvdb_order(stub, Series(episodes)), errors + + +def _ep(season: int, number: int, part=None): + from unshackle.core.titles.episode import Episode + + return Episode(id_=f"{season}x{number}.{part}", service=_Svc, title="Show", season=season, number=number, part=part) + + +def test_parts_of_one_episode_renumber_together(monkeypatch: pytest.MonkeyPatch) -> None: + """Three parts share one (season, number) slot, so they must not read as a collision.""" + episodes = [_ep(1, 1, 1), _ep(1, 1, 2), _ep(1, 1, 3), _ep(1, 2)] + mapping = {(1, 1): (2, 1, "Part-ful"), (1, 2): (2, 2, "Whole")} + titles, errors = _renumber(monkeypatch, episodes, mapping) + assert errors == [] + assert [(t.season, t.number, t.part) for t in titles] == [(2, 1, 1), (2, 1, 2), (2, 1, 3), (2, 2, None)] + + +def test_a_real_collision_still_bails(monkeypatch: pytest.MonkeyPatch) -> None: + """Dedupe must not disable the guard: two distinct episodes on one slot still refuses.""" + episodes = [_ep(1, 1), _ep(1, 2)] + mapping = {(1, 2): (1, 1, "Clash")} # (1, 1) is unmapped and keeps the slot (1, 2) moves onto + titles, errors = _renumber(monkeypatch, episodes, mapping) + assert errors # refused, with the "two episodes each" error + assert [(t.season, t.number) for t in titles] == [(1, 1), (1, 2)] # numbering untouched diff --git a/tests/remote/unit/test_api_episode_part.py b/tests/remote/unit/test_api_episode_part.py new file mode 100644 index 00000000..f1fb8bf9 --- /dev/null +++ b/tests/remote/unit/test_api_episode_part.py @@ -0,0 +1,331 @@ +"""Unit tests for multi-part episode support across the REST/API layer. + +Covers serialize_title's conditional `part` key, the list-tracks wanted filter +routing through Episode.matches_wanted, the discrete season/episode/part request +fields, and download_manager's already-internal wanted sniff. +""" + +from __future__ import annotations + +import asyncio +import json +import re +from typing import Any, Dict, List, Optional + +import pytest + +from unshackle.core.api import handlers +from unshackle.core.api.errors import APIError +from unshackle.core.api.handlers import serialize_title +from unshackle.core.titles.episode import Episode + +pytestmark = pytest.mark.unit + + +class _FakeSvc: + pass + + +def _ep(season: int = 1, number: int = 1, part: Optional[int] = None, **overrides: Any) -> Episode: + base: Dict[str, Any] = dict( + id_=f"ep-{season}-{number}-{part or 0}", + service=_FakeSvc, + title="My Show", + season=season, + number=number, + name="Pilot", + year=2024, + language=None, + ) + base.update(overrides) + return Episode(part=part, **base) + + +class _FakeTracks: + videos: List[Any] = [] + audio: List[Any] = [] + subtitles: List[Any] = [] + + +class _FakeService: + def __init__(self, titles: List[Episode]) -> None: + self._titles = titles + + def get_titles(self) -> List[Episode]: + return self._titles + + def get_tracks(self, title: Episode) -> _FakeTracks: + return _FakeTracks() + + +def _run_list_tracks(monkeypatch: pytest.MonkeyPatch, titles: List[Episode], **data: Any) -> Dict[str, Any]: + """Drive list_tracks_handler end to end with the service layer stubbed out.""" + monkeypatch.setattr(handlers, "validate_service", lambda tag, request=None: "FAKE") + monkeypatch.setattr(handlers, "setup_list_service", lambda *a, **kw: _FakeService(titles)) + payload = {"service": "FAKE", "title_id": "t1", **data} + try: + response = asyncio.run(handlers.list_tracks_handler(payload)) + except APIError as e: + # routes.py turns this into the HTTP error body; keep the fields the tests assert on + return {"status": e.http_status, "error_code": e.error_code, "details": e.details} + return json.loads(response.body) + + +def _selected(result: Dict[str, Any]) -> List[str]: + """Selection-syntax keys of whatever the handler returned, single or multi.""" + entries = result["episodes"] if "episodes" in result else [{"title": result["title"]}] + keys = [] + for entry in entries: + t = entry["title"] + suffix = f".{t['part']}" if "part" in t else "" + keys.append(f"{t['season']}x{t['number']}{suffix}") + return keys + + +# ---------- serialize_title: the JSON contract ---------- + + +def test_partless_episode_json_has_no_part_key() -> None: + d = serialize_title(_ep(part=None)) + assert "part" not in d + assert list(d.keys()) == [ + "type", + "name", + "id", + "language", + "description", + "date", + "cover_url", + "year", + "series_title", + "season", + "number", + ] + assert d["name"] == "Pilot" + + +def test_partful_episode_json_carries_part_last() -> None: + d = serialize_title(_ep(part=2)) + assert d["part"] == 2 + # additive: `part` is appended, every pre-existing key keeps its position + assert list(d.keys())[:11] == list(serialize_title(_ep(part=None)).keys()) + assert list(d.keys())[11] == "part" + + +def test_part_stays_out_of_the_name_field() -> None: + """`name` is a reconstruction input for _build_title, not a display string.""" + assert serialize_title(_ep(part=2))["name"] == "Pilot" + assert serialize_title(_ep(part=3, name=None))["name"] == "Episode 01" + + +def test_remote_build_title_round_trip() -> None: + from unshackle.core.remote_service import _build_title + + rebuilt = _build_title(serialize_title(_ep(1, 4, 2)), "FAKE", "fallback-id") + assert (rebuilt.season, rebuilt.number, rebuilt.part, rebuilt.name) == (1, 4, 2, "Pilot") + + partless = _build_title(serialize_title(_ep(1, 4, None)), "FAKE", "fallback-id") + assert partless.part is None + + +def test_movie_json_unaffected() -> None: + from langcodes import Language + + from unshackle.core.titles.movie import Movie + + d = serialize_title(Movie(id_="movie-0001", service=_FakeSvc, name="Film", year=2024, language=Language.get("en"))) + assert "part" not in d + + +# ---------- list_tracks_handler: the REST wanted filter ---------- + + +@pytest.fixture +def split_titles() -> List[Episode]: + """E1 split into three parts, E2 unsplit.""" + return [_ep(1, 1, 1), _ep(1, 1, 2), _ep(1, 1, 3), _ep(1, 2, None)] + + +def test_rest_bare_episode_key_selects_every_part(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E01") + assert _selected(result) == ["1x1.1", "1x1.2", "1x1.3"] + + +def test_rest_part_key_selects_exactly_one(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E01.2") + assert _selected(result) == ["1x1.2"] + + +def test_rest_negative_part_key_excludes_only_that_part(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, wanted=["S01", "-S01E01.2"]) + assert _selected(result) == ["1x1.1", "1x1.3", "1x2"] + + +def test_rest_partless_episode_unchanged(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E02") + assert _selected(result) == ["1x2"] + assert "part" not in result["title"] + + +def test_rest_part_of_unsplit_episode_selects_nothing(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, wanted="S01E02.1") + assert result["status"] == 404 + + +# ---------- discrete season/episode/part request fields ---------- + + +def test_discrete_fields_without_part_select_every_part(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1) + assert _selected(result) == ["1x1.1", "1x1.2", "1x1.3"] + + +def test_discrete_fields_with_part_select_one(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1, part=3) + assert _selected(result) == ["1x1.3"] + + +def test_discrete_part_accepts_string(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1, part="3") + assert _selected(result) == ["1x1.3"] + + +def test_discrete_no_match_error_reports_the_part_qualified_key(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, season=1, episode=1, part=9) + assert result["status"] == 404 + assert result["details"]["wanted"] == "1x1.9" + + +def test_discrete_partless_error_key_unchanged(monkeypatch, split_titles) -> None: + result = _run_list_tracks(monkeypatch, split_titles, season=9, episode=9) + assert result["details"]["wanted"] == "9x9" + + +def test_part_is_a_transport_key_not_a_service_option() -> None: + """`part` addresses the title, so it must never reach a service's own --part option.""" + assert "part" in handlers.LIST_HANDLER_TRANSPORT_KEYS + assert "part" in handlers.SESSION_TRANSPORT_KEYS + + +# ---------- ordering ---------- + + +def test_rest_multi_episode_order_is_stable_across_parts(monkeypatch) -> None: + titles = [_ep(1, 2, None), _ep(1, 1, 3), _ep(1, 1, 1), _ep(1, 1, 2)] + result = _run_list_tracks(monkeypatch, titles, wanted="S01") + assert _selected(result) == ["1x1.1", "1x1.2", "1x1.3", "1x2"] + + +# ---------- download_manager: already-internal wanted sniff ---------- + + +_SNIFF = r"^!?\d+x\d+(\.\d+)?$" + + +@pytest.mark.parametrize( + "wanted, internal", + [ + (["1x1"], True), # today's pre-parsed form + (["1x1", "1x2"], True), + (["1x1.2"], True), # part key + (["!1x1.2"], True), # negative part key + (["1x1", "!1x1.2"], True), + (["S01E01"], False), + (["s1e1.2"], False), + (["S01"], False), + (["1x1", "S01E02"], False), + ], +) +def test_download_manager_sniff_regex(wanted: List[str], internal: bool) -> None: + needs_conversion = any(not re.match(_SNIFF, w) for w in wanted) + assert needs_conversion is (not internal) + + +def test_sniff_regex_matches_the_one_in_download_manager() -> None: + """Guard the copy above against drift from the production regex.""" + from pathlib import Path + + import unshackle.core.api.download_manager as dm + + source = Path(dm.__file__).read_text(encoding="utf8") + assert f'needs_conversion = any(not re.match(r"{_SNIFF}", w) for w in wanted_raw)' in source + + +def test_perform_download_skips_reconversion_for_preparsed_part_keys(monkeypatch) -> None: + """A pre-parsed list with part and negative keys must not reach parse_tokens.""" + from unshackle.core.api.download_manager import _perform_download + from unshackle.core.utils.click_types import SeasonRange + + calls = [] + monkeypatch.setattr(SeasonRange, "parse_tokens", lambda self, *t: calls.append(t) or []) + + params: Dict[str, Any] = {"wanted": ["1x1", "1x1.2", "!1x1.3"]} + with pytest.raises(BaseException): + # blows up later on the unknown service; the wanted block runs first + _perform_download("job-1", "NoSuchServiceXYZ", "t1", params) + + assert calls == [] + assert params["wanted"] == ["1x1", "1x1.2", "!1x1.3"] + + +def test_perform_download_still_converts_cli_style_tokens(monkeypatch) -> None: + from unshackle.core.api.download_manager import _perform_download + from unshackle.core.utils.click_types import SeasonRange + + calls = [] + monkeypatch.setattr(SeasonRange, "parse_tokens", lambda self, *t: calls.append(t) or ["1x1"]) + + params: Dict[str, Any] = {"wanted": "S01E01"} + with pytest.raises(BaseException): + _perform_download("job-2", "NoSuchServiceXYZ", "t1", params) + + assert calls == [("S01E01",)] + assert params["wanted"] == ["1x1"] + + +# ---------- CLI / REST agreement ---------- + + +def test_dl_and_handlers_share_the_one_match_helper() -> None: + """Both filters must route through Episode.matches_wanted, or they drift.""" + from pathlib import Path + + import unshackle.commands.dl as dl_module + + dl_source = Path(dl_module.__file__).read_text(encoding="utf8") + handlers_source = Path(handlers.__file__).read_text(encoding="utf8") + + assert "matches_wanted(wanted)" in dl_source + assert "matches_wanted(wanted)" in handlers_source + # no inline key compare left behind on either side + inline = re.compile(r"f\"\{title\.season\}x\{title\.number\}\"\s*\n?\s*(?:in|not in)\s+wanted") + assert not inline.search(dl_source) + assert not inline.search(handlers_source) + + +@pytest.mark.parametrize( + "wanted_tokens", + [ + ("S01E01",), + ("S01E01.2",), + ("S01",), + ("S01", "-S01E01.2"), + ("S01", "-S01E01"), + ("S01E01.1-S01E01.3",), + ("S01E02",), + ], +) +def test_rest_filter_agrees_with_cli_filter(monkeypatch, split_titles, wanted_tokens) -> None: + from unshackle.core.utils.click_types import SeasonRange + + wanted = SeasonRange().parse_tokens(*wanted_tokens) + cli_selected = [ + f"{t.season}x{t.number}" + (f".{t.part}" if t.part is not None else "") + for t in split_titles + if t.matches_wanted(wanted) + ] + + result = _run_list_tracks(monkeypatch, split_titles, wanted=list(wanted_tokens)) + rest_selected = [] if result.get("status") == 404 else _selected(result) + + assert sorted(rest_selected) == sorted(cli_selected) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 35763bcf..c26dd713 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -1645,7 +1645,8 @@ class dl: # Apply indentation only for multiple seasons prefix = " " if multiple_seasons else "" - option_text = f"{prefix}{t.number}" + (f". {display_name}" if t.name else "") + part_label = f".{t.part}" if t.part is not None else "" + option_text = f"{prefix}{t.number}{part_label}" + (f". {display_name}" if t.name else "") selection_titles.append(option_text) current_ui_idx = len(selection_titles) - 1 @@ -2217,7 +2218,7 @@ class dl: # If --latest-episode is set, only process the latest episode if f"{title.season}x{title.number}" != latest_episode_id: continue - elif isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted: + elif isinstance(title, Episode) and wanted and not title.matches_wanted(wanted): continue if progress_sink: @@ -2225,7 +2226,8 @@ class dl: progress_sink( { "title": title.title, - "current_title": f"S{title.season or 0:02}E{title.number or 0:02}", + "current_title": f"S{title.season or 0:02}E{title.number or 0:02}" + + (f".{title.part}" if title.part is not None else ""), } ) else: @@ -3741,7 +3743,9 @@ class dl: return titles self.tvdb_id = tvdb_id # keep the mux tag on the series the renumbering used - keys = [(t.season, t.number) for t in titles] + # parts of one episode share a (season, number) slot; without the dedupe the + # collision check below reads them as several episodes clashing on one slot + keys = list(dict.fromkeys((t.season, t.number) for t in titles)) source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined] if source_order == self.tvdb_order: self.log.info("%s already lists this series in TVDB %s order.", self.service, self.tvdb_order) @@ -3809,6 +3813,8 @@ class dl: year=title.year, air_date=title.air_date.isoformat() if isinstance(title.air_date, date) else title.air_date, ) + if title.part is not None: + meta["part"] = title.part elif isinstance(title, Movie): meta.update(type="movie", name=title.name, year=title.year) else: diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 56fc0d99..b250e543 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -387,7 +387,8 @@ def _perform_download( if isinstance(wanted_raw, str): wanted_raw = [wanted_raw] # Only convert if not already in internal "SxE" format - needs_conversion = any(not re.match(r"^\d+x\d+$", w) for w in wanted_raw) + # the !? keeps a pre-parsed part exclusion from being re-fed through parse_tokens + needs_conversion = any(not re.match(r"^!?\d+x\d+(\.\d+)?$", w) for w in wanted_raw) if needs_conversion: season_range = SeasonRange() params["wanted"] = season_range.parse_tokens(*wanted_raw) diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 569dafb2..c6d52d45 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -93,6 +93,7 @@ LIST_HANDLER_TRANSPORT_KEYS = { "profile", "season", "episode", + "part", "wanted", "proxy", "no_proxy", @@ -324,6 +325,11 @@ def require_fields(data: Dict[str, Any], *names: str) -> None: ) +def _part_key_suffix(part: Optional[int]) -> str: + """`.2` selection-syntax suffix for a part-ful episode, empty otherwise.""" + return f".{part}" if part is not None else "" + + def serialize_title(title: Title_T) -> Dict[str, Any]: """Convert a title object to JSON-serializable dict.""" title_language = str(title.language) if hasattr(title, "language") and title.language else None @@ -336,7 +342,10 @@ def serialize_title(title: Title_T) -> Dict[str, Any]: cover_url = _data.get("cover_url") if isinstance(_data, dict) else None is_episode = isinstance(title, Episode) + episode_part = title.part if isinstance(title, Episode) else None if is_episode: + # no part suffix here: remote_service._build_title rebuilds the Episode from this dict, so a + # suffixed name plus the structural `part` below would render the part twice in the filename name = title.name if title.name else f"Episode {title.number:02d}" else: name = str(title.name) if hasattr(title, "name") else str(title) @@ -357,6 +366,8 @@ def serialize_title(title: Title_T) -> Dict[str, Any]: result["series_title"] = str(title.title) result["season"] = title.season result["number"] = title.number + if episode_part is not None: + result["part"] = episode_part # conditional, so part-less JSON is unchanged return result @@ -705,6 +716,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques wanted_param = data.get("wanted") season = data.get("season") episode = data.get("episode") + part = data.get("part") if hasattr(titles, "__iter__") and not isinstance(titles, str): titles_list = list(titles) @@ -729,7 +741,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques details={"wanted": wanted_param, "service": normalized_service}, ) elif season is not None and episode is not None: - wanted = [f"{season}x{episode}"] + wanted = [f"{season}x{episode}{_part_key_suffix(part)}"] if wanted: # Filter titles based on wanted episodes, similar to how dl.py does it @@ -737,8 +749,8 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques log.debug(f"Filtering {len(titles_list)} titles with {len(wanted)} wanted episodes") for title in titles_list: if isinstance(title, Episode): - episode_key = f"{title.season}x{title.number}" - if episode_key in wanted: + episode_key = f"{title.season}x{title.number}{_part_key_suffix(title.part)}" + if title.matches_wanted(wanted): log.debug(f"Episode {episode_key} matches wanted list") matching_titles.append(title) else: @@ -755,7 +767,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques details={ "service": normalized_service, "title_id": title_id, - "wanted": wanted_param or f"{season}x{episode}", + "wanted": wanted_param or wanted[0], }, ) @@ -765,7 +777,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques failed_episodes = [] # Sort matching titles by season and episode number for consistent ordering - sorted_titles = sorted(matching_titles, key=lambda t: (t.season, t.number)) + sorted_titles = sorted(matching_titles, key=lambda t: (t.season, t.number, t.part or 0)) for title in sorted_titles: try: @@ -786,12 +798,12 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques log.debug(f"Successfully got tracks for {title.season}x{title.number}") except SystemExit: # Service calls sys.exit() for unavailable episodes - catch and skip - failed_episodes.append(f"S{title.season}E{title.number:02d}") + failed_episodes.append(f"S{title.season}E{title.number:02d}{_part_key_suffix(title.part)}") log.debug(f"Episode {title.season}x{title.number} not available, skipping") continue except (Exception, SystemExit) as e: # Handle other errors gracefully - failed_episodes.append(f"S{title.season}E{title.number:02d}") + failed_episodes.append(f"S{title.season}E{title.number:02d}{_part_key_suffix(title.part)}") log.debug(f"Error getting tracks for {title.season}x{title.number}: {e}") continue @@ -1586,6 +1598,7 @@ SESSION_TRANSPORT_KEYS = { "title_id", "season", "episode", + "part", "wanted", "proxy", "no_proxy", diff --git a/unshackle/core/config.py b/unshackle/core/config.py index 9a94e291..adc7e9f4 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -173,6 +173,7 @@ class Config: "episode", "season_episode", "episode_name", + "part", "date", "quality", "resolution", diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index 2524ee85..996d4010 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -299,6 +299,7 @@ def _build_title(info: Dict[str, Any], service_tag: str, fallback_id: str) -> Un year=info.get("year"), language=lang, air_date=info.get("air_date"), + part=info.get("part"), ) return Movie( id_=info.get("id", fallback_id), diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index c3678d88..4b1d5d7a 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -2,7 +2,7 @@ import re from abc import ABC from collections import Counter from datetime import date, datetime -from typing import Any, Iterable, Optional, Union +from typing import Any, Collection, Iterable, Optional, Union from langcodes import Language from pymediainfo import MediaInfo @@ -16,6 +16,9 @@ from unshackle.core.utils.template_formatter import TemplateFormatter, detect_sp class Episode(Title): + # class-level default so Episodes restored from an older title cache read as part-less + part: Optional[int] = None + def __init__( self, id_: Any, @@ -29,6 +32,7 @@ class Episode(Title): data: Optional[Any] = None, description: Optional[str] = None, air_date: Optional[Union[date, str]] = None, + part: Optional[Union[int, str]] = None, ) -> None: super().__init__(id_, service, language, data) @@ -51,6 +55,16 @@ class Episode(Title): elif not isinstance(number, int): raise TypeError(f"Expected number to be an int, not {number!r}") + if part is not None: + if isinstance(part, str) and part.isdigit(): + part = int(part) + # bool is an int subclass; True would render as ".True" in keys and filenames + elif isinstance(part, bool) or not isinstance(part, int): + raise TypeError(f"Expected part to be an int, not {part!r}") + # parts are 1-based; a falsy 0 would read as "no part" downstream + if part <= 0: + raise ValueError(f"Episode part cannot be {part}") + if name is not None and not isinstance(name, str): raise TypeError(f"Expected name to be a str, not {name!r}") @@ -89,8 +103,30 @@ class Episode(Title): self.year = year self.description = description self.air_date = air_date + self.part = part - def _build_template_context(self, media_info: MediaInfo, show_service: bool = True) -> dict: + def matches_wanted(self, wanted: Collection[str]) -> bool: + """Whether a parsed ``-w`` key set selects this episode. + + A part-ful episode answers to both its base key and its part key, so ``-w s1e1`` + takes every part. ``!`` keys are part-qualified exclusions resolved here. + """ + base = f"{self.season}x{self.number}" + keys = (base,) if self.part is None else (base, f"{base}.{self.part}") + if any(f"!{k}" in wanted for k in keys): + return False + return any(k in wanted for k in keys) + + def _part_suffix(self) -> str: + """``.Part.2`` / `` Part 2`` in the series template's own separator style.""" + if self.part is None: + return "" + sep = config.get_template_separator("series") if config.output_template.get("series") else "." + return f"{sep}Part{sep}{self.part}" + + def _build_template_context( + self, media_info: MediaInfo, show_service: bool = True, include_part: bool = True + ) -> dict: """Build template context dictionary from MediaInfo.""" context = self._build_base_template_context(media_info, show_service) context["title"] = self.title.replace("$", "S") @@ -100,6 +136,7 @@ class Episode(Title): context["season_episode"] = f"S{self.season:02}E{self.number:02}" context["episode_name"] = self.name or "" context["date"] = "" + context["part"] = self.part if include_part and self.part is not None else "" if self.air_date: # daily/sports: air date replaces SxxExx disp = self._air_date_display() @@ -108,6 +145,12 @@ class Episode(Title): context["season_episode"] = disp context["year"] = "" # air date is the sole date in the file; folders keep the year context["date"] = self.air_date.isoformat() if isinstance(self.air_date, date) else str(self.air_date) + if include_part and self.part is not None: + # folded into the identity tokens, not a token of its own: a standalone {part} is + # absent from every shipped template, so two parts would render the same filename + suffix = self._part_suffix() + context["episode"] = f"{context['episode']}{suffix}" + context["season_episode"] = f"{context['season_episode']}{suffix}" return context def _air_date_display(self) -> str: @@ -123,19 +166,28 @@ class Episode(Title): return f"{self.air_date.year:04}" return f"S{self.season:02}" + def _part_label(self) -> str: + """``.2`` selection-syntax suffix for console output, empty when part-less.""" + return f".{self.part}" if self.part is not None else "" + def __str__(self) -> str: if self.air_date: - return "{title}{year} {date} {name}".format( + # dated content has no SxxExx to hang the part off, but the parts still have to + # be told apart, so the same .N suffix follows the date + return "{title}{year} {date}{part} {name}".format( title=self.title, year=f" {self.year}" if self.year else "", date=self._air_date_display(), + part=self._part_label(), name=self.name or "", ).strip() - return "{title}{year} S{season:02}E{number:02} {name}".format( + # the console shows the -w selection syntax (S01E01.2), not the filename form + return "{title}{year} S{season:02}E{number:02}{part} {name}".format( title=self.title, year=f" {self.year}" if self.year else "", season=self.season, number=self.number, + part=self._part_label(), name=self.name or "", ).strip() @@ -143,7 +195,8 @@ class Episode(Title): if folder: template = config.get_folder_template("series") if template: - context = self._build_template_context(media_info, show_service) + # all parts of an episode land in the same season folder + context = self._build_template_context(media_info, show_service, include_part=False) context["season"] = self._folder_season() context["year"] = self.year or "" # folders keep the year spacer = detect_spacer(template) # one style for the whole path @@ -167,7 +220,7 @@ class Episode(Title): derived_template = re.sub(r"^[\.\s]+|[\.\s]+$", "", derived_template) formatter = TemplateFormatter(derived_template) - context = self._build_template_context(media_info, show_service) + context = self._build_template_context(media_info, show_service, include_part=False) context["season"] = self._folder_season() context["year"] = self.year or "" # folders keep the year @@ -190,7 +243,8 @@ class Episode(Title): class Series(SortedKeyList, ABC): def __init__(self, iterable: Optional[Iterable] = None): - super().__init__(iterable, key=lambda x: (x.season, x.number, x.year or 0)) + # part slots before year so existing ties still resolve by year + super().__init__(iterable, key=lambda x: (x.season, x.number, x.part or 0, x.year or 0)) def __str__(self) -> str: if not self: @@ -218,7 +272,7 @@ class Series(SortedKeyList, ABC): episode._air_date_display() if episode.air_date else str(episode.number).zfill(len(str(episodes))) - ) + ) + episode._part_label() if episode.name: season_tree.add(f"[bold]{label}.[/] [bright_black]{episode.name}") elif episode.air_date: diff --git a/unshackle/core/utils/click_types.py b/unshackle/core/utils/click_types.py index ca4e13aa..9f0f4334 100644 --- a/unshackle/core/utils/click_types.py +++ b/unshackle/core/utils/click_types.py @@ -174,11 +174,18 @@ class SeasonRange(click.ParamType): MIN_EPISODE = 0 MAX_EPISODE = 999 + MIN_PART = 1 + MAX_PART = 99 def parse_tokens(self, *tokens: str) -> list[str]: """ Parse multiple tokens or ranged tokens as '{s}x{e}' strings. + An episode split into separately playable parts is addressed as '{s}x{e}.{p}'. + A part range must stay inside one episode, since how many parts an episode has + is not knowable here. A part-qualified exclusion cannot be removed from the + computed keys (they are base keys), so it becomes a '!' key resolved at match time. + Supports exclusioning by putting a `-` before the token. Example: @@ -189,6 +196,10 @@ class SeasonRange(click.ParamType): ["2x1", "2x3", "2x4", "2x5"] >>> sr.parse_tokens("S01-S05", "-S03", "-S02E01") ["1x0", "1x1", ..., "2x0", (...), "2x2", (...), "4x0", ..., "5x0", ...] + >>> sr.parse_tokens("S01E01.1-S01E01.3") + ["1x1.1", "1x1.2", "1x1.3"] + >>> sr.parse_tokens("S01E01", "-S01E01.2") + ["1x1", "!1x1.2"] """ if len(tokens) == 0: return [] @@ -199,36 +210,56 @@ class SeasonRange(click.ParamType): if exclude: token = token[1:] parsed = [ - re.match(r"^S(?P<season>\d+)(E(?P<episode>\d+))?$", x, re.IGNORECASE) for x in re.split(r"[:-]", token) + re.match(r"^S(?P<season>\d+)(E(?P<episode>\d+)(\.(?P<part>\d+))?)?$", x, re.IGNORECASE) + for x in re.split(r"[:-]", token) ] if len(parsed) > 2: self.fail(f"Invalid token, only a left and right range is acceptable: {token}") if len(parsed) == 1: + # the same match object is read with different per-side defaults, so a bare + # S01 spans from MIN_EPISODE to MAX_EPISODE parsed.append(parsed[0]) if any(x is None for x in parsed): self.fail(f"Invalid token, syntax error occurred: {token}") - from_season, from_episode = [ - int(v) if v is not None else self.MIN_EPISODE - for k, v in parsed[0].groupdict().items() - if parsed[0] # type: ignore[union-attr] - ] - to_season, to_episode = [ - int(v) if v is not None else self.MAX_EPISODE - for k, v in parsed[1].groupdict().items() - if parsed[1] # type: ignore[union-attr] - ] + left, right = parsed[0], parsed[1] + from_season = int(left.group("season")) # type: ignore[union-attr] + from_episode_raw = left.group("episode") # type: ignore[union-attr] + from_episode = int(from_episode_raw) if from_episode_raw is not None else self.MIN_EPISODE + from_part_raw = left.group("part") # type: ignore[union-attr] + from_part = int(from_part_raw) if from_part_raw is not None else None + to_season = int(right.group("season")) # type: ignore[union-attr] + to_episode_raw = right.group("episode") # type: ignore[union-attr] + to_episode = int(to_episode_raw) if to_episode_raw is not None else self.MAX_EPISODE + to_part_raw = right.group("part") # type: ignore[union-attr] + to_part = int(to_part_raw) if to_part_raw is not None else None if from_season > to_season: self.fail(f"Invalid range, left side season cannot be bigger than right side season: {token}") if from_season == to_season and from_episode > to_episode: self.fail(f"Invalid range, left side episode cannot be bigger than right side episode: {token}") + if (from_part is None) != (to_part is None): + self.fail(f"Invalid range, a part must be given on both sides or on neither: {token}") + if from_part is not None and to_part is not None: + if (from_season, from_episode) != (to_season, to_episode): + self.fail(f"Invalid range, a part range must stay within one episode: {token}") + if not all(self.MIN_PART <= p <= self.MAX_PART for p in (from_part, to_part)): + self.fail(f"Invalid part, must be between {self.MIN_PART} and {self.MAX_PART}: {token}") + if from_part > to_part: + self.fail(f"Invalid range, left side part cannot be bigger than right side part: {token}") + for p in range(from_part, to_part + 1): + (computed if not exclude else exclusions).append(f"{from_season}x{from_episode}.{p}") + continue for s in range(from_season, to_season + 1): for e in range( from_episode if s == from_season else 0, (self.MAX_EPISODE if s < to_season else to_episode) + 1 ): (computed if not exclude else exclusions).append(f"{s}x{e}") for exclusion in exclusions: - if exclusion in computed: - computed.remove(exclusion) + if "." in exclusion: + computed.append(f"!{exclusion}") # base key stays; resolved at match time + else: + # a base exclusion must drop that episode's part keys too, or they still match + prefix = f"{exclusion}." + computed = [k for k in computed if k != exclusion and not k.startswith(prefix)] return list(set(computed)) def convert( From 02fd7b93876399fedbf8f86046364c41d855a3b3 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Wed, 5 Aug 2026 11:07:38 -0600 Subject: [PATCH 39/40] fix(subtitles): strip X-TIMESTAMP-MAP so WebVTT cues are not offset on conversion --- tests/tracks/test_webvtt_timestamp_map.py | 22 ++++++++++++++++++++++ unshackle/core/tracks/subtitle.py | 12 ++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/tracks/test_webvtt_timestamp_map.py diff --git a/tests/tracks/test_webvtt_timestamp_map.py b/tests/tracks/test_webvtt_timestamp_map.py new file mode 100644 index 00000000..1f642999 --- /dev/null +++ b/tests/tracks/test_webvtt_timestamp_map.py @@ -0,0 +1,22 @@ +from unshackle.core.tracks.subtitle import Subtitle + + +def test_strips_timestamp_map_keeps_cues(): + text = "WEBVTT\nX-TIMESTAMP-MAP=MPEGTS:900000,LOCAL:00:00:00.000\n\n00:00:22.690 --> 00:00:24.710\nHi\n" + out = Subtitle.strip_webvtt_timestamp_map(text) + assert "X-TIMESTAMP-MAP" not in out + assert "00:00:22.690 --> 00:00:24.710" in out + assert out.startswith("WEBVTT\n") + + +def test_strips_lowercase_timestamp_map(): + # SubtitleEdit honours the header case-insensitively, so the strip must too + text = "WEBVTT\nx-timestamp-map=MPEGTS:900000,LOCAL:00:00:00.000\n\n00:00:22.690 --> 00:00:24.710\nHi\n" + out = Subtitle.strip_webvtt_timestamp_map(text) + assert "x-timestamp-map" not in out.lower() + assert "00:00:22.690 --> 00:00:24.710" in out + + +def test_no_timestamp_map_is_untouched(): + text = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nHi\n" + assert Subtitle.strip_webvtt_timestamp_map(text) == text diff --git a/unshackle/core/tracks/subtitle.py b/unshackle/core/tracks/subtitle.py index 203c271d..b95847c8 100644 --- a/unshackle/core/tracks/subtitle.py +++ b/unshackle/core/tracks/subtitle.py @@ -257,6 +257,7 @@ class Subtitle(Track): self.convert(Subtitle.Codec.WebVTT) elif self.codec == Subtitle.Codec.WebVTT: text = self.path.read_text("utf8") + text = Subtitle.strip_webvtt_timestamp_map(text) if self.descriptor == Track.Descriptor.DASH: if len(self.data["dash"]["segment_durations"]) > 1: text = merge_segmented_webvtt( @@ -303,6 +304,17 @@ class Subtitle(Track): # Keep the sanitized version even if parsing failed self.path.write_text(text, encoding="utf8") + @staticmethod + def strip_webvtt_timestamp_map(text: str) -> str: + """ + Remove X-TIMESTAMP-MAP header lines (RFC 8216 §3.5, HLS only). + + Cue times are already absolute, but SubtitleEdit reads the header and offsets + every cue by it during conversion. + """ + # IGNORECASE: SubtitleEdit matches the header case-insensitively, so must the strip + return re.sub(r"^[ \t]*X-TIMESTAMP-MAP[^\n]*\n?", "", text, flags=re.MULTILINE | re.IGNORECASE) + @staticmethod def sanitize_webvtt_timestamps(text: str) -> str: """ From d448ee8ff9b9d3204af5d714b06d568f6c0f2611 Mon Sep 17 00:00:00 2001 From: imSp4rky <george+slight@slight.me> Date: Wed, 5 Aug 2026 17:56:33 -0600 Subject: [PATCH 40/40] fix(downloader): decode content-encoded bodies instead of writing raw Reading .raw skips urllib3's decoding, so brotli-served subtitles landed compressed and converted to empty files. Also drops a partial rather than Range-resuming an encoded body, since a Range indexes the encoded bytes. --- tests/downloaders/test_content_encoding.py | 110 +++++++++++++++++++++ unshackle/core/downloaders/requests.py | 33 ++++++- unshackle/core/utilities.py | 8 +- 3 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 tests/downloaders/test_content_encoding.py diff --git a/tests/downloaders/test_content_encoding.py b/tests/downloaders/test_content_encoding.py new file mode 100644 index 00000000..fc1d97e9 --- /dev/null +++ b/tests/downloaders/test_content_encoding.py @@ -0,0 +1,110 @@ +import gzip +import socket +import threading + +import pytest +import requests as rq + +from unshackle.core.downloaders.requests import _is_content_encoded, download + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, False), + ("", False), + (" ", False), + ("identity", False), + ("IDENTITY", False), + ("identity, identity", False), + ("gzip", True), + ("GZIP", True), + (" gzip ", True), + ("x-gzip", True), + ("br", True), + ("zstd", True), + ("deflate", True), + ("compress", True), + ("dcb", True), + ("gzip, br", True), + ("identity, gzip", True), + ("gzip, identity", True), + ("gzip,,", True), + ("some-future-coding", True), + ], +) +def test_is_content_encoded(value, expected): + assert _is_content_encoded(value) is expected + + +PLAIN = b"WEBVTT\n\n" + b"".join(f"{i:05d} cue text\n".encode() for i in range(2000)) + + +class _EncodingServer: + """Serves one gzip body, honouring Range against the encoded representation.""" + + def __init__(self): + self.encoded = gzip.compress(PLAIN) + self.sock = socket.socket() + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(8) + self.url = f"http://127.0.0.1:{self.sock.getsockname()[1]}/sub.vtt" + threading.Thread(target=self._serve, daemon=True).start() + + def _serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + try: + request = conn.recv(65536).decode("latin1") + start = next( + ( + int(line.split("=", 1)[1].split("-")[0]) + for line in request.split("\r\n") + if line.lower().startswith("range:") + ), + None, + ) + if start is None: + body = self.encoded + head = f"HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n" + else: + body = self.encoded[start:] + head = ( + f"HTTP/1.1 206 Partial Content\r\nContent-Length: {len(body)}\r\n" + f"Content-Range: bytes {start}-{len(self.encoded) - 1}/{len(self.encoded)}\r\n" + ) + conn.sendall( + (head + "Content-Encoding: gzip\r\nAccept-Ranges: bytes\r\nConnection: close\r\n\r\n").encode() + + body + ) + except Exception: + pass + finally: + conn.close() + + def close(self): + self.sock.close() + + +@pytest.fixture +def encoding_server(): + server = _EncodingServer() + yield server + server.close() + + +def test_encoded_body_is_decoded_on_disk(tmp_path, encoding_server): + save_path = tmp_path / "sub.vtt" + list(download(url=encoding_server.url, save_path=save_path, session=rq.Session())) + assert save_path.read_bytes() == PLAIN + + +def test_encoded_body_discards_a_partial_instead_of_range_resuming(tmp_path, encoding_server): + save_path = tmp_path / "sub.vtt" + save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000]) + list(download(url=encoding_server.url, save_path=save_path, session=rq.Session())) + assert save_path.read_bytes() == PLAIN diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index 97aa3411..8f793b1c 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -161,6 +161,18 @@ def _is_rnet_session(session: Any) -> bool: return isinstance(session, RnetSession) +def _is_content_encoded(value: Optional[str]) -> bool: + """ + Check a Content-Encoding value for any coding other than absent or `identity`. + + Reading .raw skips urllib3's decoding, so an encoded body has to go through + iter_content or the compressed bytes land on disk as the payload. Codings + urllib3 cannot decode still count: an allowlist fails open on whatever token + a server sends next, and the cost of guessing wrong is a corrupt file. + """ + return any(coding.strip() not in ("", "identity") for coding in (value or "").lower().split(",")) + + def _probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]: headers = {**(kwargs.get("headers") or {}), "Range": "bytes=0-0"} rest = {k: v for k, v in kwargs.items() if k != "headers"} @@ -175,8 +187,7 @@ def _probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]: try: if resp.status_code != 206: return 0, False - ce = (resp.headers.get("Content-Encoding") or resp.headers.get("content-encoding") or "").lower() - if ce in ("gzip", "deflate", "br"): + if _is_content_encoded(resp.headers.get("Content-Encoding") or resp.headers.get("content-encoding")): return 0, False content_range = resp.headers.get("Content-Range") or resp.headers.get("content-range") or "" total = content_range.rsplit("/", 1)[-1].strip() @@ -306,7 +317,8 @@ def download( Download a file with optimized I/O. Supports both requests.Session and RnetSession for TLS fingerprinting. - Uses raw socket reads for requests.Session and native rnet streaming for RnetSession. + RnetSession streams natively; requests.Session reads the raw socket, except + on a content-encoded body, which has to go through iter_content to be decoded. Yields the following download status updates while chunks are downloading: @@ -393,16 +405,27 @@ def download( resume_offset = 0 if part_mode and stream.status_code != 206: raise IOError(f"expected 206 for ranged part, got {stream.status_code}") + content_encoded = False if use_rnet: content_length = stream.content_length or 0 else: + content_encoded = _is_content_encoded(stream.headers.get("Content-Encoding")) try: content_length = int(stream.headers.get("Content-Length", "0")) - if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]: + if content_encoded: content_length = 0 except ValueError: content_length = 0 + if resumed and content_encoded: + try: + stream.close() + except Exception: + pass + tmp_file.unlink(missing_ok=True) + resume_offset = 0 + continue + limiter = _speed_limiter chunk_size = _adaptive_chunk_size(content_length) if limiter: @@ -434,7 +457,7 @@ def download( if use_rnet: chunks = stream.stream() - elif use_raw: + elif use_raw and not content_encoded: _read1 = getattr(stream.raw, "read1", None) if claimed is not None and RACER_READ1 else None if _read1 is not None: chunks = iter(lambda: _read1(chunk_size), b"") diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 90c5e500..285a0520 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -454,9 +454,9 @@ def try_ensure_utf8(data: bytes) -> bytes: """ Try to ensure that the given data is encoded in UTF-8. - Automatically decompresses gzip/deflate/zlib data before encoding detection. - This handles cases where HTTP responses are saved with raw Content-Encoding - (e.g., when decode_content=False is used for performance). + Gzip or zlib compressed input is decompressed first, detected by magic bytes. + Callers hand in bytes read from files and from responses they fetched + themselves, so there is no Content-Encoding header left to consult. Parameters: data: Input data that may or may not yet be UTF-8 or another encoding. @@ -464,13 +464,11 @@ def try_ensure_utf8(data: bytes) -> bytes: Returns the input data encoded in UTF-8 if successful. If unable to detect the encoding of the input data, then the original data is returned as-received. """ - # Decompress gzip data (magic bytes: 1f 8b) if data[:2] == b"\x1f\x8b": try: data = gzip.decompress(data) except Exception: pass - # Decompress raw deflate/zlib data (common zlib headers: 78 01, 78 5e, 78 9c, 78 da) elif data[:1] == b"\x78" and len(data) > 1 and data[1:2] in (b"\x01", b"\x5e", b"\x9c", b"\xda"): try: data = zlib.decompress(data)