From 6964f58271dd8aec5bdacbde431b4569c837a74a Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Fri, 13 Nov 2026 18:30:01 -0700 Subject: [PATCH] feat(dl): add --skip-missing to skip titles that miss a required language or track A season run skips such a title and continues; one title, or a run that skips every title, still exits 1 (now also for -ado/-fso). REST jobs report skipped_titles. --- docs/dev/rest-api/endpoints.md | 9 +- docs/dev/rest-api/job-lifecycle.md | 1 + docs/guide/cli-reference.md | 1 + docs/guide/downloading.md | 15 +- docs/guide/subtitles.md | 2 +- docs/reference/configuration/download.md | 1 + docs/reference/configuration/post-scripts.md | 4 + .../test_kind_only_selection.py | 181 +++++++++++++++++- tests/remote/unit/test_download_manager.py | 83 ++++++++ unshackle/commands/dl.py | 162 ++++++++++------ unshackle/core/api/download_manager.py | 17 +- unshackle/core/api/handlers.py | 1 + unshackle/core/api/routes.py | 9 +- 13 files changed, 403 insertions(+), 83 deletions(-) diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index c8036152..84f6e952 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -451,9 +451,10 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `latest_episode` | boolean | `false` | Only the newest episode. | | `latest_episodes` | integer | `null` | Only the N newest episodes. A positive integer. When set, it overrides `latest_episode`. | | `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. A missing `s_lang` language gives a warning and the job continues. | -| `require_audio` | string[] | `[]` | Audio languages that must exist. The job fails if one is missing, even with `best_available`. | -| `require_video` | string[] | `[]` | Video languages that must exist. The job fails if one is missing, even with `best_available`. | -| `require_subs` | string[] | `[]` | Subtitle languages that must exist. The job fails if one is missing. `s_lang` still decides what to keep. | +| `require_audio` | string[] | `[]` | Audio languages that must exist. The job fails if one is missing, even with `best_available`, or skips the title with `skip_missing`. | +| `require_video` | string[] | `[]` | Video languages that must exist. The job fails if one is missing, even with `best_available`, or skips the title with `skip_missing`. | +| `require_subs` | string[] | `[]` | Subtitle languages that must exist. The job fails if one is missing, or skips the title with `skip_missing`. `s_lang` still decides what to keep. | +| `skip_missing` | boolean | `false` | Skip a title that misses a required language or track, and continue with the next title. The job's `skipped_titles` lists each skip. The job fails when it has one title or skips every title. | | `forced_subs` | boolean | `false` | Include forced subtitles. | | `forced_subs_only` | boolean | `false` | Download forced subtitles and no other subtitle; implies `forced_subs`. Not valid with `no_subs`, `video_only`, `audio_only` or `chapters_only`. | | `forced_s_lang` | string[] | `[]` | Forced subtitle language(s); implies `forced_subs`. | @@ -585,6 +586,7 @@ Show download jobs, with optional filtering and sorting. "segments_total": 300, "speed": "5.2 MB/s", "skipped_subtitles": [], + "skipped_titles": [], "input_prompt": null, "notice": null } @@ -853,6 +855,7 @@ Read the persisted history of jobs that reached a terminal state, newest first. "created_time": "2026-07-03T12:00:00+00:00", "completed_time": "2026-07-03T12:08:00+00:00", "output_files": ["/downloads/Example.S01E01.mkv"], + "skipped_titles": [], "error_message": null, "parameters": { "...": "redacted" } } diff --git a/docs/dev/rest-api/job-lifecycle.md b/docs/dev/rest-api/job-lifecycle.md index b441466d..8499bfde 100644 --- a/docs/dev/rest-api/job-lifecycle.md +++ b/docs/dev/rest-api/job-lifecycle.md @@ -162,6 +162,7 @@ full detail: | `segments_total` | number | Total segments for the track in progress. | | `speed` | string | Transfer speed of the track in progress, e.g. `4.20 MB/s`. | | `skipped_subtitles` | list | Subtitles that were skipped as non-fatal failures (when `skip_subtitle_errors` is set). | +| `skipped_titles` | list | Titles that the run skipped because a requested track or language is missing, each `{title, reason}`. | | `input_prompt` | string or `null` | The prompt the service waits on. Answer it with `POST /api/download/jobs/{job_id}/input`. | | `notice` | string or `null` | A message the service shows while it waits for an approval on another device, such as a pair code. It needs no answer. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index f74a64d7..ed3957af 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -102,6 +102,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | `--require-audio` | - | Audio langs that must exist. The title fails even with `--best-available`. Does not select tracks. | | `--require-video` | - | Video langs that must exist. The title fails even with `--best-available`. Does not select tracks. | | `--require-subs` | - | Subtitle langs that must exist. Fails the title if one is missing. `--s-lang` still decides what to keep. | +| `--skip-missing` | off | Skip a title that misses a required language or track, and continue with the next title. A run with one title, or a run that skips every title, still fails. | | `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | | `-fso`, `--forced-subs-only` | off | Keep only forced subtitle tracks; drop the other subtitles. Implies `-fs`. | | `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. A `-` prefix excludes. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 177c8ab1..6928add7 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -531,6 +531,17 @@ The check runs against the full track list, before any language selection or exc `orig` resolves to the title language, and `--exact-lang` applies. Audio carried inside the video track counts. `--no-audio` and `--no-video` drop the matching requirement. +A failed title stops the whole run. To skip that title and come back for it later, add +`--skip-missing`, or set `skip_missing: true` under `dl:` in `unshackle.yaml`. It also skips a +title that has no track for a filter such as `-q`, `-v`, `-r`, `-a` or a language you request. +A run with one title, or a run that skips every title, still fails. A REST API job lists each +skipped title and the reason in `skipped_titles`. + +```bash +# download the episodes that have Korean audio, skip the others +unshackle dl -al ko,en --require-audio ko --skip-missing -w S06 EXAMPLE 0ABC123 +``` + ### Forced subtitles and output format - `-fs` / `--forced-subs`: include forced subtitle tracks (signs/foreign dialogue). @@ -777,7 +788,8 @@ Additional track-type flags: - A flag that drops the same tracks is an error, for example `-ado` with `--no-audio` or `-fso` with `-A`. - A title with no track of that kind stops a run of one title. A run of more than one - title skips that title with a warning and continues. + title skips that title with a warning and continues. The run fails when it skips + every title. - unshackle reads the kind from the manifest or from the service. A track that neither of them marks as descriptive or forced does not match. @@ -1214,6 +1226,7 @@ authoritative list. | `--require-audio` | | Audio languages that must exist. The title fails if one is missing. | | `--require-video` | | Video languages that must exist. The title fails if one is missing. | | `--best-available` | | Degrade gracefully instead of failing. | +| `--skip-missing` | | Skip a title that misses a required language or track, and continue with the next title. | | `--output` | `-o` | Output directory for this run. | | `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. | | `--postscript` / `--no-postscript` | | Run a command after each output file, or run none. | diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index d4888868..63c0e6e5 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -157,7 +157,7 @@ unshackle dl --s-lang en --forced-subs-only EXAMPLE 81234567 `-fso` does not drop video or audio. Add `-S` for a subtitle-only download. A title with no forced track stops a run of one title. A run of more than one title skips that title -with a warning. +with a warning, and fails when it skips every title. ## Track types: forced, SDH, and CC diff --git a/docs/reference/configuration/download.md b/docs/reference/configuration/download.md index 667ee7c8..c1883b18 100644 --- a/docs/reference/configuration/download.md +++ b/docs/reference/configuration/download.md @@ -53,6 +53,7 @@ Common config keys. This is a useful subset, and every `dl` flag works: | `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). | | `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | | `best_available` | bool | `false` | Warn instead of failing when a requested resolution, range, video language, or audio language is missing, or when one DRM system does not license under `--all-drm`. The CLI flag is `--best-available`, also spelled `--warn-only`. | +| `skip_missing` | bool | `false` | Skip a title that misses a required language (`require_audio`, `require_video`, `require_subs`) or a track that a filter asks for, and continue with the next title. A run with one title, or a run that skips every title, still fails. The CLI flag is `--skip-missing`. For REST API jobs, set it in the [`serve`](services.md#serve) block. | | `generate_dv` | bool | `false` | With `-r HYBRID`, make Dolby Vision from the HDR10+ metadata or the decoded video when a title has no DV track. With `-r HDR10` or `-r HDR10P`, make it for that base and ignore a DV track. See [Dolby Vision without a DV track](../../guide/downloading.md#dolby-vision-without-a-dv-track). | | `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | | `no_folder` | bool | `false` | Do not create a per-title folder. | diff --git a/docs/reference/configuration/post-scripts.md b/docs/reference/configuration/post-scripts.md index e987d933..622ae723 100644 --- a/docs/reference/configuration/post-scripts.md +++ b/docs/reference/configuration/post-scripts.md @@ -213,6 +213,10 @@ subs = [p for p in sys.argv[i].removeprefix("--subs=").split("\n") if p] nothing about whether the season is complete on disk. Downloading a single episode fires the post-script for that season as soon as the episode lands. + A title that track selection skips (`--skip-missing`, `-ado`, `-fso`) counts as done. The + `season` post-script still fires for the episodes that landed, so the folder can be missing + the skipped episodes. A run that skips every title fails, and then no `season` post-script fires. + A title that fails stops the whole run. unshackle dispatches the `failure` post-script for that title and then returns. It downloads no further title, and no `season` or `run` post-script fires after that point. This applies to every title type. If one episode of S01 diff --git a/tests/lang_selection/test_kind_only_selection.py b/tests/lang_selection/test_kind_only_selection.py index ff64e1d6..57413131 100644 --- a/tests/lang_selection/test_kind_only_selection.py +++ b/tests/lang_selection/test_kind_only_selection.py @@ -12,9 +12,11 @@ import logging from typing import Any, Optional, cast import pytest +import requests from unshackle.commands.dl import dl from unshackle.core.api.handlers import DEFAULT_DOWNLOAD_PARAMS +from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY from unshackle.core.service import Service from unshackle.core.titles import Episode, Series from unshackle.core.tracks import Audio, Subtitle, Tracks, Video @@ -38,7 +40,7 @@ class Harness(dl): class FakeService: - """One episode per entry; ``ad`` adds a descriptive track, ``forced`` a forced subtitle.""" + """One episode per entry; ``ad`` adds a descriptive track, ``forced`` a forced subtitle, ``ko`` Korean audio.""" def __init__(self, episodes: dict[int, dict[str, bool]]) -> None: self.episodes = episodes @@ -62,6 +64,8 @@ class FakeService: tracks.add( Audio(id_="ad", url="https://x/ad", language="en", codec=Audio.Codec.AAC, bitrate=1, descriptive=True) ) + if kinds.get("ko"): + tracks.add(Audio(id_="ko", url="https://x/ko", language="ko", codec=Audio.Codec.AAC, bitrate=128_000)) tracks.add(Subtitle(id_="s", url="https://x/s", language="en", codec=Subtitle.Codec.WebVTT)) if kinds.get("forced"): tracks.add(Subtitle(id_="sf", url="https://x/sf", language="en", codec=Subtitle.Codec.WebVTT, forced=True)) @@ -71,8 +75,8 @@ class FakeService: return [] def __getattr__(self, name: str) -> Any: - if name.startswith("on_"): - # the event hooks (on_track_downloaded, ...) the loop subscribes + if name.startswith(("on_", "get_")): + # the event hooks (on_track_downloaded, ...) and the DRM callbacks a run past selection probes return lambda *a, **k: None raise AttributeError(name) @@ -82,16 +86,22 @@ def run( caplog: pytest.LogCaptureFixture, episodes: dict, service: Optional[FakeService] = None, + to_end: bool = False, **flags: Any, ) -> Any: - """Return the Selected ids, the exit code, or None for a run that completes.""" - monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self))) + """Return the Selected ids, the exit code, or None for a run that completes; to_end runs past selection.""" + if not to_end: + monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self))) + else: + monkeypatch.setattr("unshackle.commands.dl.download_tracks_in_passes", lambda *a, **k: None) monkeypatch.setattr("unshackle.core.providers.search_metadata", lambda *a, **k: None) cmd = Harness.__new__(Harness) cmd.log = logging.getLogger("test_kind_only") cmd.service = "FAKE" cmd.tmdb_searched = True cmd.completed_files = [] + cmd.skipped_titles = [] + cmd.skipped_subtitles = [] kwargs: dict[str, Any] = {} for name, param in inspect.signature(dl.result).parameters.items(): if name in DEFAULT_DOWNLOAD_PARAMS: @@ -101,12 +111,17 @@ def run( kwargs.update(quality=[], vcodec=[], acodec=[], range_=[], wanted=[], cdm_only=None, downloads=1) kwargs.update(flags) caplog.set_level(logging.WARNING, logger="test_kind_only") + service = service or FakeService(episodes) + if to_end: + service.session = requests.Session() try: - cmd.result(cast(Service, service or FakeService(episodes)), **kwargs) + cmd.result(cast(Service, service), **kwargs) except Selected as sel: return sel.ids except SystemExit as exc: return exc.code + finally: + DOWNLOAD_LICENCE_ONLY.clear() return None @@ -145,8 +160,9 @@ def test_subs_only_with_no_matching_subtitle_exits_a_single_title_and_skips_in_a monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: assert run(monkeypatch, caplog, {1: {}}, subs_only=True, s_lang=["fr"]) == 1 - assert run(monkeypatch, caplog, {1: {}, 2: {}}, subs_only=True, s_lang=["fr"]) is None - assert caplog.text.count("There's no track left for the selected options. Skipping Show S01E") == 2 + assert run(monkeypatch, caplog, {1: {}, 2: {}}, subs_only=True, s_lang=["fr"]) == 1 + assert caplog.text.count("Skipping Show S01E") == 2 + assert "Skipping Show S01E02: There's no track left for the selected options." in caplog.text def test_a_multi_title_run_skips_the_title_and_continues( @@ -157,13 +173,158 @@ def test_a_multi_title_run_skips_the_title_and_continues( assert "Skipping Show S01E01" in caplog.text -def test_a_multi_title_run_with_every_title_skipped_completes( +def test_a_multi_title_run_with_every_title_skipped_fails( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - assert run(monkeypatch, caplog, {1: {}, 2: {}}, forced_subs_only=True) is None + assert run(monkeypatch, caplog, {1: {}, 2: {}}, forced_subs_only=True) == 1 assert caplog.text.count("Skipping Show S01E") == 2 +def test_a_required_language_miss_stops_the_run_by_default( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + sink: list[dict] = [] + assert run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, require_audio=["ko"], progress_sink=sink.append) == 1 + assert "Skipping" not in caplog.text + assert "Show S01E01: Required audio language(s) not found: ko" in caplog.text + assert not any("skipped_titles" in event for event in sink) + + +def test_skip_missing_skips_the_title_and_reports_it( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + sink: list[dict] = [] + ids = run( + monkeypatch, + caplog, + {1: {}, 2: {"ko": True}}, + require_audio=["ko"], + skip_missing=True, + progress_sink=sink.append, + ) + assert ids == {"video": ["v"], "audio": ["a"], "subs": ["s"]} + assert "Skipping Show S01E01: Required audio language(s) not found: ko" in caplog.text + assert {"skipped_titles": [{"title": "Show S01E01", "reason": "Required audio language(s) not found: ko"}]} in sink + + +def test_require_audio_video_and_subs_work_together( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + ids = run(monkeypatch, caplog, {1: {}}, require_audio=["en"], require_video=["en"], require_subs=["en"]) + assert ids == {"video": ["v"], "audio": ["a"], "subs": ["s"]} + assert run(monkeypatch, caplog, {1: {}}, require_audio=["ko"], require_video=["ko"], require_subs=["ko"]) == 1 + assert ( + "Required audio language(s) not found: ko; Required video language(s) not found: ko; " + "Required subtitle language(s) not found: ko" + ) in caplog.text + assert run(monkeypatch, caplog, {1: {}}, require_audio=["en"], require_video=["ko"], require_subs=["en"]) == 1 + assert "Show S01E01: Required video language(s) not found: ko" in caplog.text + + +def test_skip_missing_skips_a_filter_miss_and_continues( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + ids = run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, a_lang=["ko"], skip_missing=True) + assert ids == {"video": ["v"], "audio": ["ko"], "subs": ["s"]} + assert "Skipping Show S01E01" in caplog.text + + +def test_skip_missing_still_fails_a_run_that_skips_every_title( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + assert run(monkeypatch, caplog, {1: {}, 2: {}}, require_audio=["ko"], skip_missing=True) == 1 + assert "Skipped all 2 titles" in caplog.text + assert run(monkeypatch, caplog, {1: {}}, require_audio=["ko"], skip_missing=True) == 1 + + +class LangService(FakeService): + """Each episode names its original language and its audio languages; the video is in the original language.""" + + def get_titles_cached(self) -> Series: + return Series( + Episode(id_=f"show-s01-e{n:03}", service=FakeService, title="Show", season=1, number=n, language=ep["orig"]) + for n, ep in self.episodes.items() + ) + + def get_tracks(self, title: Episode) -> Tracks: + ep = self.episodes[title.number] + tracks = Tracks() + tracks.add(Video(id_="v", url="https://x/v", language=ep["orig"] or "en", codec=Video.Codec.AVC)) + for lang in ep["audio"]: + tracks.add(Audio(id_=lang, url=f"https://x/{lang}", language=lang, codec=Audio.Codec.AAC, bitrate=128_000)) + return tracks + + +JA_EN = {"orig": "ja", "audio": ["ja", "en"]} +FR = {"orig": "ja", "audio": ["fr"]} +EVERY = {"orig": "ja", "audio": ["ja", "ko", "en"]} +ORIG_KO_EN = ["orig", "ko", "en"] + + +@pytest.mark.parametrize( + "first,flags,outcome,logged", + [ + (JA_EN, {}, 1, "Show S01E01: ko not found in audio tracks"), + (JA_EN, {"skip_missing": True}, ["ja", "ko", "en"], "Skipping Show S01E01: ko not found in audio tracks"), + ({"orig": "ja", "audio": ["ko", "en"]}, {"skip_missing": True}, ["ja", "ko", "en"], "S01E01: orig not found"), + (JA_EN, {"best_available": True}, ["ja", "en"], "ko not found in audio tracks, continuing with: orig, en"), + (FR, {"best_available": True}, 1, "orig, ko, en not found in audio tracks and no fallback available"), + (FR, {"best_available": True, "skip_missing": True}, ["ja", "ko", "en"], "Skipping Show S01E01: orig, ko, en"), + (JA_EN, {"best_available": True, "require_audio": ["ko"]}, 1, "Required audio language(s) not found: ko"), + ( + JA_EN, + {"best_available": True, "require_audio": ["ko"], "skip_missing": True}, + ["ja", "ko", "en"], + "Skipping Show S01E01: Required audio language(s) not found: ko", + ), + ], +) +def test_several_audio_languages_skip_an_episode_that_misses_one( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + first: dict, + flags: dict[str, Any], + outcome: Any, + logged: str, +) -> None: + """-al orig,ko,en on a season where only the second episode has all three.""" + episodes = {1: first, 2: EVERY} + result = run(monkeypatch, caplog, episodes, service=LangService(episodes), a_lang=ORIG_KO_EN, **flags) + assert (result if outcome == 1 else result["audio"]) == outcome + assert logged in caplog.text + + +def test_several_audio_languages_fail_a_season_that_misses_one_everywhere( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + episodes = {1: JA_EN, 2: JA_EN} + assert run(monkeypatch, caplog, episodes, service=LangService(episodes), a_lang=ORIG_KO_EN, skip_missing=True) == 1 + assert caplog.text.count("ko not found in audio tracks") == 2 + assert "Skipped all 2 titles" in caplog.text + + +def test_a_partly_skipped_run_completes_and_lists_the_skips( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + episodes = {1: {}, 2: {"ko": True}} + assert run(monkeypatch, caplog, episodes, to_end=True, a_lang=["ko"], skip_missing=True, skip_dl=True) is None + assert "Skipped 1 of 2 titles: Show S01E01" in caplog.text + + +@pytest.mark.parametrize("no_mux,settled", [(False, 2), (True, 1)]) +def test_a_skipped_title_settles_its_post_script_group( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, no_mux: bool, settled: int +) -> None: + """The season event waits on every title of a group; a skipped title counts as settled, once at count time.""" + import unshackle.commands.dl as dl_module + + seen: list[str] = [] + group = dl_module.post_script_group + monkeypatch.setattr(dl_module, "post_script_group", lambda t: seen.append(str(t)) or group(t)) + run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, a_lang=["ko"], skip_missing=True, no_mux=no_mux) + assert seen.count("Show S01E01") == settled + + class LcevcService(FakeService): """A 540p base video and the 1080p LCEVC enhancement layer that decodes on top of it.""" diff --git a/tests/remote/unit/test_download_manager.py b/tests/remote/unit/test_download_manager.py index b963d000..7f199748 100644 --- a/tests/remote/unit/test_download_manager.py +++ b/tests/remote/unit/test_download_manager.py @@ -626,3 +626,86 @@ def test_client_job_view_hides_debug_fields_without_debug_api() -> None: assert client_job_view(job, off) == {"job_id": "j"} assert client_job_view(job, None) == {"job_id": "j"} assert client_job_view(job, on) == job + + +FAKE_SKIPPING_WORKER = """ +import sys +from unshackle.core.api import download_worker + + +def skip_then_stop(*args, progress_callback, **kwargs): + progress_callback({"skipped_titles": [{"title": "Show S01E02", "reason": "missing ko"}]}) + raise RuntimeError("Download failed: Skipped all 2 titles") + + +download_worker.perform_download = skip_then_stop +sys.exit(download_worker.main(["worker"] + sys.argv[1:])) +""" + + +async def test_a_skip_written_just_before_the_worker_exits_reaches_the_job( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, manager: DownloadQueueManager +) -> None: + import asyncio + + from unshackle.core.api import download_manager + + worker = tmp_path / "worker.py" + worker.write_text(FAKE_SKIPPING_WORKER, encoding="utf-8") + spawn = asyncio.create_subprocess_exec + + async def spawn_fake_worker(executable: str, *args: str, **kwargs: Any) -> Any: + return await spawn(executable, str(worker), *args[2:], **kwargs) + + monkeypatch.setattr(download_manager.asyncio, "create_subprocess_exec", spawn_fake_worker) + job = manager.create_job("EXAMPLE", "t") + with pytest.raises(Exception, match="Skipped all 2 titles"): + await asyncio.wait_for(manager.run_download_async(job), timeout=20) + assert job.skipped_titles == [{"title": "Show S01E02", "reason": "missing ko"}] + assert job.to_dict()["skipped_titles"] == job.skipped_titles + + +def test_a_stopped_run_names_its_reason_in_the_job_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import sys + + import click + + import unshackle.commands.dl as dl_module + from unshackle.core.api import download_manager, handlers + from unshackle.core.api.errors import APIError + from unshackle.core.services import Services + + class FakeDl: + cli = dl_module.dl.cli + + def __init__(self, ctx: click.Context, **kwargs: object) -> None: + self.stop_reason = None + + def result(self, **kwargs: object) -> None: + self.stop_reason = "Show S01E01: Required audio language(s) not found: ko" + sys.exit(1) + + class FakeService: + def __init__(self, ctx: click.Context) -> None: + pass + + monkeypatch.setattr(dl_module, "dl", FakeDl) + monkeypatch.setattr(Services, "get_path", staticmethod(lambda s: tmp_path)) + monkeypatch.setattr(Services, "load", staticmethod(lambda s: FakeService)) + monkeypatch.setattr(handlers, "load_full_cdm", lambda *a: None) + + with pytest.raises(APIError, match=r"^Download failed: Show S01E01: Required audio language\(s\) not found: ko$"): + download_manager.perform_download("job-1", "EXAMPLE1", "t1", {}) + + +def test_job_history_keeps_the_skipped_titles(tmp_path: Path, manager: DownloadQueueManager) -> None: + import json + + from unshackle.core.api.download_manager import record_job_history + + job = manager.create_job("EXAMPLE", "t") + job.status = JobStatus.COMPLETED + job.skipped_titles = [{"title": "Show S01E02", "reason": "missing ko"}] + record_job_history(job) + entry = json.loads((tmp_path / "api_history.jsonl").read_text(encoding="utf-8").splitlines()[-1]) + assert entry["skipped_titles"] == job.skipped_titles diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index df0960fa..73c3145f 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -158,6 +158,13 @@ if TYPE_CHECKING: from unshackle.core.remote_service import ServerKeySource +class SkippedTitle(TypedDict): + """A title skipped because a requested track or language is missing. Accumulated as ``dl.skipped_titles``.""" + + title: str + reason: str + + class SkippedSubtitle(TypedDict): """A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``, one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable.""" @@ -979,6 +986,12 @@ class dl: default=[], help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.", ) + @click.option( + "--skip-missing", + is_flag=True, + default=False, + help="Skip a title that misses a required language or track and continue with the next title.", + ) @click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.") @click.option( "-fso", @@ -1383,6 +1396,9 @@ class dl: # Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can # report which weren't available without parsing the console output. See SkippedSubtitle. self.skipped_subtitles: list[SkippedSubtitle] = [] + self.skipped_titles: list[SkippedTitle] = [] + # Why a missing track or language stopped the run, for the API job error. + self.stop_reason: Optional[str] = None # result() catches a download-worker failure, reports it, and returns rather than # re-raising (so the CLI still exits cleanly). Keep it so an embedding caller (the API # worker) can tell the title did not complete instead of reading it as a success. @@ -1765,6 +1781,7 @@ class dl: require_audio: list[str], require_video: list[str], require_subs: list[str], + skip_missing: bool, forced_subs: bool, forced_subs_only: bool, forced_s_lang: list[str], @@ -2348,15 +2365,20 @@ class dl: for folder, context in post_script_last.get(group_key, {}).items(): dispatch("success", "season", season_context(context, folder), postscript) - def skip_title(title: Any, reason: str) -> None: - """Skip a title that has no track of the kind an only flag selects. + def skip_title(title: Any, reason: str, kind_only: bool = False) -> None: + """Skip a title that has no track the selection asks for, and report it to the API job. - A run with one title stops instead, because it has nothing else to download. + Only a kind-only miss (-ado, -fso, no track left) skips without --skip-missing; any other miss + stops the run. A run with one title always stops, because it has nothing else to download. """ - if wanted_total <= 1: - self.log.error(reason) + if wanted_total <= 1 or not (kind_only or skip_missing): + self.stop_reason = f"{title}: {reason}" + self.log.error(self.stop_reason) sys.exit(1) - self.log.warning(f"{reason} Skipping {title}") + self.skipped_titles.append(SkippedTitle(title=str(title), reason=reason)) + if progress_sink: + progress_sink({"skipped_titles": list(self.skipped_titles)}) + self.log.warning(f"Skipping {title}: {reason}") if not no_mux: settle_post_script_group(title) @@ -2641,6 +2663,7 @@ class dl: elif proxy_download: console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}") + missing_required: list[str] = [] for kind, required, available in ( ( "audio", @@ -2659,10 +2682,11 @@ class dl: ], ), ): - missing_required = missing_required_langs(required, available, title.language, exact=exact_lang) - if missing_required: - self.log.error(f"Required {kind} language(s) not found: {', '.join(missing_required)}") - sys.exit(1) + if missing := missing_required_langs(required, available, title.language, exact=exact_lang): + missing_required.append(f"Required {kind} language(s) not found: {', '.join(missing)}") + if missing_required: + skip_title(title, "; ".join(missing_required)) + continue if not keep_videos: title.tracks.videos = [] @@ -2690,16 +2714,18 @@ class dl: for codec in missing_codecs: self.log.warning(f"Skipping {codec.name} video tracks as none are available.") if not title.tracks.videos: - self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...") - sys.exit(1) + skip_title(title, f"There's no {', '.join(c.name for c in vcodec)} Video Track...") + continue elif keep_videos: title.tracks.select_video(lambda x: x.codec != Video.Codec.LCEVC) if keep_videos and generated_dv_bases: title.tracks.videos = keep_generated_dv_bases(title.tracks.videos, generated_dv_bases) if not any(v.range in generated_dv_bases for v in title.tracks.videos): - self.log.error(f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track...") - sys.exit(1) + skip_title( + title, f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track..." + ) + continue if keep_videos and range_: # Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks @@ -2712,8 +2738,8 @@ class dl: if Video.Range.HDR10P in missing_ranges and dv_has_hdr10p: self.log.warning("A DV track carries HDR10+: use -r DV or -r HYBRID.") if not title.tracks.videos: - self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...") - sys.exit(1) + skip_title(title, f"There's no {', '.join(r.name for r in range_)} Video Track...") + continue if keep_videos and vbitrate: if any(r == Video.Range.HYBRID for r in range_): @@ -2723,21 +2749,21 @@ class dl: lambda x: x.range == Video.Range.DV or bool(x.bitrate and x.bitrate // 1000 == vbitrate) ) if not any(x.range != Video.Range.DV for x in title.tracks.videos): - self.log.error(f"There's no {vbitrate}kbps Video Track...") - sys.exit(1) + skip_title(title, f"There's no {vbitrate}kbps Video Track...") + continue else: title.tracks.select_video(lambda x: bool(x.bitrate and x.bitrate // 1000 == vbitrate)) if not title.tracks.videos: - self.log.error(f"There's no {vbitrate}kbps Video Track...") - sys.exit(1) + skip_title(title, f"There's no {vbitrate}kbps Video Track...") + continue if keep_videos and vbitrate_min is not None and vbitrate_max is not None: title.tracks.select_video( lambda x: bool(x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max) ) if not title.tracks.videos: - self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...") - sys.exit(1) + skip_title(title, f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...") + continue if keep_videos and video_excl_r: drop = matched_language_tags( @@ -2745,8 +2771,8 @@ class dl: ) title.tracks.select_video(lambda x: str(x.language) not in drop) if not title.tracks.videos: - self.log.error(f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...") - sys.exit(1) + skip_title(title, f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...") + continue effective_video_lang = v_lang or lang video_languages = [lang for lang in effective_video_lang if lang != "best"] @@ -2776,20 +2802,23 @@ class dl: ) processed_video_lang = remaining_v_langs else: - self.log.error(f"{missing_str} not found in video tracks and no fallback available") - sys.exit(1) + skip_title( + title, f"{missing_str} not found in video tracks and no fallback available" + ) + continue else: - self.log.error(missing_str + " not found in video tracks") - sys.exit(1) + skip_title(title, missing_str + " not found in video tracks") + continue title.tracks.videos = title.tracks.by_language( title.tracks.videos, processed_video_lang, exact_match=exact_lang ) if not title.tracks.videos: # empty only when 'orig' was the sole request and did not resolve - self.log.error( - f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..." + skip_title( + title, + f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track...", ) - sys.exit(1) + continue has_hybrid = any(r == Video.Range.HYBRID for r in range_) non_hybrid_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID] @@ -2854,8 +2883,8 @@ class dl: f"There's no {res_list} Video Track{plural}, continuing with available qualities..." ) else: - self.log.error(f"There's no {res_list} Video Track{plural}...") - sys.exit(1) + skip_title(title, f"There's no {res_list} Video Track{plural}...") + continue pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else [] if keep_videos and has_hybrid: @@ -2968,8 +2997,8 @@ class dl: fallback_selected.append(match) title.tracks.videos = fallback_selected else: - self.log.error(gate_failure) - sys.exit(1) + skip_title(title, gate_failure) + continue fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang) if keep_subtitles and s_excl_r: @@ -3031,27 +3060,31 @@ class dl: if forced_subs_only: title.tracks.select_subtitles(lambda x: x.forced) if not title.tracks.subtitles: - skip_title(title, "There's no forced subtitle track for the selected languages.") + skip_title( + title, + "There's no forced subtitle track for the selected languages.", + kind_only=True, + ) continue if keep_audio and audio_description_only: title.tracks.select_audio(lambda x: x.descriptive) if not title.tracks.audio: - skip_title(title, "There's no audio description track.") + skip_title(title, "There's no audio description track.", kind_only=True) continue # might have no audio tracks if part of the video, e.g. transport stream hls if keep_audio and len(title.tracks.audio) > 0: if not audio_description: title.tracks.select_audio(lambda x: not x.descriptive) if not title.tracks.audio: - self.log.error("Audio description only, use --audio-description...") - sys.exit(1) + skip_title(title, "Audio description only, use --audio-description...") + continue if acodec: title.tracks.select_audio(lambda x: x.codec in acodec) if not title.tracks.audio: codec_names = ", ".join(c.name for c in acodec) - self.log.error(f"No audio tracks matching codecs: {codec_names}") - sys.exit(1) + skip_title(title, f"No audio tracks matching codecs: {codec_names}") + continue if channels: title.tracks.select_audio( lambda x: bool( @@ -3059,33 +3092,33 @@ class dl: ) ) if not title.tracks.audio: - self.log.error(f"There's no {channels} Audio Track...") - sys.exit(1) + skip_title(title, f"There's no {channels} Audio Track...") + continue if no_atmos: title.tracks.audio = [x for x in title.tracks.audio if not x.atmos] if not title.tracks.audio: - self.log.error("No non-Atmos audio tracks available...") - sys.exit(1) + skip_title(title, "No non-Atmos audio tracks available...") + continue if abitrate: title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate) if not title.tracks.audio: - self.log.error(f"There's no {abitrate}kbps Audio Track...") - sys.exit(1) + skip_title(title, f"There's no {abitrate}kbps Audio Track...") + continue if abitrate_min is not None and abitrate_max is not None: title.tracks.select_audio( lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max ) if not title.tracks.audio: - self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...") - sys.exit(1) + skip_title(title, f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...") + continue if audio_excl_r: drop = matched_language_tags( audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True ) title.tracks.select_audio(lambda x: str(x.language) not in drop) if not title.tracks.audio: - self.log.error(f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...") - sys.exit(1) + skip_title(title, f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...") + continue audio_languages = a_lang or lang if audio_languages: processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang) @@ -3115,13 +3148,13 @@ class dl: ) processed_lang = remaining else: - self.log.error( - f"{missing_str} not found in {audio_noun} and no fallback available" + skip_title( + title, f"{missing_str} not found in {audio_noun} and no fallback available" ) - sys.exit(1) + continue else: - self.log.error(f"{missing_str} not found in {audio_noun}") - sys.exit(1) + skip_title(title, f"{missing_str} not found in {audio_noun}") + continue title.tracks.audio = select_best_audio( title.tracks.audio, processed_lang, acodec, audio_description, exact_lang @@ -3129,11 +3162,11 @@ class dl: if not title.tracks.audio and not embedded_langs: # empty when 'orig' was the sole request, did not resolve, and the # video carries no audio of its own to fall back on - self.log.error( - f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} " - f"Audio Track, cannot continue..." + skip_title( + title, + f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} Audio Track...", ) - sys.exit(1) + continue kept_tracks = [] if keep_videos: @@ -3149,7 +3182,7 @@ class dl: title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url) if not (title.tracks.videos or title.tracks.audio or title.tracks.subtitles or title.tracks.chapters): - skip_title(title, "There's no track left for the selected options.") + skip_title(title, "There's no track left for the selected options.", kind_only=True) continue selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) @@ -4063,6 +4096,13 @@ class dl: self.wait_vault_writes() if export_path: self.log_export_summary() + if self.skipped_titles: + if len(self.skipped_titles) == wanted_total: + self.stop_reason = f"Skipped all {wanted_total} titles" + self.log.error(self.stop_reason) + sys.exit(1) + skipped = ", ".join(t["title"] for t in self.skipped_titles) + self.log.warning(f"Skipped {len(self.skipped_titles)} of {wanted_total} titles: {skipped}") dl_time = time_elapsed_since(start_time) console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index d7e55bbb..4aefe9cb 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -137,6 +137,8 @@ class DownloadJob: # Subtitles skipped under skip_subtitle_errors (non-fatal). Each entry is a dl.SkippedSubtitle # dict (id / language / title) so a client can report which weren't available. skipped_subtitles: List[Dict[str, Any]] = field(default_factory=list) + # Titles skipped because a requested track or language is missing; each a dl.SkippedTitle (title / reason). + skipped_titles: List[Dict[str, Any]] = field(default_factory=list) cancel_event: threading.Event = field(default_factory=threading.Event) @@ -168,6 +170,7 @@ class DownloadJob: "segments_total": self.segments_total, "speed": self.speed, "skipped_subtitles": self.skipped_subtitles, + "skipped_titles": self.skipped_titles, "input_prompt": self.input_prompt, "notice": self.notice, } @@ -232,6 +235,7 @@ def record_job_history(job: DownloadJob) -> None: "created_time": job.created_time.isoformat(), "completed_time": job.completed_time.isoformat() if job.completed_time else None, "output_files": job.output_files, + "skipped_titles": job.skipped_titles, "parameters": redact_parameters(job.parameters), "error_message": _redact_text(job.error_message, job.parameters), } @@ -523,6 +527,8 @@ def perform_download( log.error(f"Download exited with code {exc.code}") log.debug("Stdout: %s", stdout_capture.getvalue()) log.debug("Stderr: %s", stderr_capture.getvalue()) + if reason := getattr(dl_instance, "stop_reason", None): + raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed: {reason}") raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed with exit code {exc.code}") except Exception as exc: # noqa: BLE001 - propagate to caller @@ -961,11 +967,8 @@ class DownloadQueueManager: try: while True: done, _ = await asyncio.wait({communicate_task}, timeout=0.5) - if communicate_task in done: - stdout_bytes, stderr_bytes = communicate_task.result() - break - # Check for progress updates (skip re-parse when the file hasn't changed since last tick) + # Read progress before the exit check so the worker's last write lands (skip an unchanged file) try: stat = os.stat(progress_path) stat_key = (stat.st_mtime_ns, stat.st_size) @@ -1003,6 +1006,8 @@ class DownloadQueueManager: job.notice = progress_data["notice"] if progress_data.get("skipped_subtitles"): job.skipped_subtitles = progress_data["skipped_subtitles"] + if progress_data.get("skipped_titles"): + job.skipped_titles = progress_data["skipped_titles"] if "progress" in progress_data: new_progress = float(progress_data["progress"]) if new_progress != job.progress: @@ -1017,6 +1022,10 @@ class DownloadQueueManager: except (OSError, json.JSONDecodeError, ValueError) as e: log.debug(f"Could not read progress for job {job.job_id}: {e}") + if communicate_task in done: + stdout_bytes, stderr_bytes = communicate_task.result() + break + if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED: log.info(f"Cancellation detected for job {job.job_id}, terminating worker process") process.terminate() diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 3439c633..1e82906e 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -88,6 +88,7 @@ DEFAULT_DOWNLOAD_PARAMS = { "require_audio": [], "require_video": [], "require_subs": [], + "skip_missing": False, "forced_subs": False, "forced_subs_only": False, "forced_s_lang": [], diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 086641c9..a2e854d4 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -1008,17 +1008,20 @@ async def download(request: web.Request) -> web.Response: type: array items: type: string - description: Audio languages that must exist, else the job fails (default - []) + description: Audio languages that must exist, else the job fails or, with skip_missing, skips the title (default - []) require_video: type: array items: type: string - description: Video languages that must exist, else the job fails (default - []) + description: Video languages that must exist, else the job fails or, with skip_missing, skips the title (default - []) require_subs: type: array items: type: string - description: Subtitle languages that must exist, else the job fails (default - []) + description: Subtitle languages that must exist, else the job fails or, with skip_missing, skips the title (default - []) + skip_missing: + type: boolean + description: Skip a title that misses a required language or track and continue with the next title; the job fails when it skips every title (default - false) forced_subs: type: boolean description: Include forced subtitle tracks (default - false)