feat(dl): add --skip-missing to skip titles that miss a required language or track

A season run skips such a title and continues; one title, or a run that skips every title, still exits 1 (now also for -ado/-fso). REST jobs report skipped_titles.
This commit is contained in:
imSp4rky
2026-11-13 18:30:01 -07:00
parent 94dfb5d677
commit 6964f58271
13 changed files with 403 additions and 83 deletions
+6 -3
View File
@@ -451,9 +451,10 @@ Make a download job. It requires `service` and `title_id`. Every other field is
| `latest_episode` | boolean | `false` | Only the newest episode. | | `latest_episode` | boolean | `false` | Only the newest episode. |
| `latest_episodes` | integer | `null` | Only the N newest episodes. A positive integer. When set, it overrides `latest_episode`. | | `latest_episodes` | integer | `null` | Only the N newest episodes. A positive integer. When set, it overrides `latest_episode`. |
| `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. A missing `s_lang` language gives a warning and the job continues. | | `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. A missing `s_lang` language gives a warning and the job continues. |
| `require_audio` | string[] | `[]` | Audio languages that must exist. The job fails if one is missing, even with `best_available`. | | `require_audio` | string[] | `[]` | Audio languages that must exist. The job fails if one is missing, even with `best_available`, or skips the title with `skip_missing`. |
| `require_video` | string[] | `[]` | Video languages that must exist. The job fails if one is missing, even with `best_available`. | | `require_video` | string[] | `[]` | Video languages that must exist. The job fails if one is missing, even with `best_available`, or skips the title with `skip_missing`. |
| `require_subs` | string[] | `[]` | Subtitle languages that must exist. The job fails if one is missing. `s_lang` still decides what to keep. | | `require_subs` | string[] | `[]` | Subtitle languages that must exist. The job fails if one is missing, or skips the title with `skip_missing`. `s_lang` still decides what to keep. |
| `skip_missing` | boolean | `false` | Skip a title that misses a required language or track, and continue with the next title. The job's `skipped_titles` lists each skip. The job fails when it has one title or skips every title. |
| `forced_subs` | boolean | `false` | Include forced subtitles. | | `forced_subs` | boolean | `false` | Include forced subtitles. |
| `forced_subs_only` | boolean | `false` | Download forced subtitles and no other subtitle; implies `forced_subs`. Not valid with `no_subs`, `video_only`, `audio_only` or `chapters_only`. | | `forced_subs_only` | boolean | `false` | Download forced subtitles and no other subtitle; implies `forced_subs`. Not valid with `no_subs`, `video_only`, `audio_only` or `chapters_only`. |
| `forced_s_lang` | string[] | `[]` | Forced subtitle language(s); implies `forced_subs`. | | `forced_s_lang` | string[] | `[]` | Forced subtitle language(s); implies `forced_subs`. |
@@ -585,6 +586,7 @@ Show download jobs, with optional filtering and sorting.
"segments_total": 300, "segments_total": 300,
"speed": "5.2 MB/s", "speed": "5.2 MB/s",
"skipped_subtitles": [], "skipped_subtitles": [],
"skipped_titles": [],
"input_prompt": null, "input_prompt": null,
"notice": null "notice": null
} }
@@ -853,6 +855,7 @@ Read the persisted history of jobs that reached a terminal state, newest first.
"created_time": "2026-07-03T12:00:00+00:00", "created_time": "2026-07-03T12:00:00+00:00",
"completed_time": "2026-07-03T12:08:00+00:00", "completed_time": "2026-07-03T12:08:00+00:00",
"output_files": ["/downloads/Example.S01E01.mkv"], "output_files": ["/downloads/Example.S01E01.mkv"],
"skipped_titles": [],
"error_message": null, "error_message": null,
"parameters": { "...": "redacted" } "parameters": { "...": "redacted" }
} }
+1
View File
@@ -162,6 +162,7 @@ full detail:
| `segments_total` | number | Total segments for the track in progress. | | `segments_total` | number | Total segments for the track in progress. |
| `speed` | string | Transfer speed of the track in progress, e.g. `4.20 MB/s`. | | `speed` | string | Transfer speed of the track in progress, e.g. `4.20 MB/s`. |
| `skipped_subtitles` | list | Subtitles that were skipped as non-fatal failures (when `skip_subtitle_errors` is set). | | `skipped_subtitles` | list | Subtitles that were skipped as non-fatal failures (when `skip_subtitle_errors` is set). |
| `skipped_titles` | list | Titles that the run skipped because a requested track or language is missing, each `{title, reason}`. |
| `input_prompt` | string or `null` | The prompt the service waits on. Answer it with `POST /api/download/jobs/{job_id}/input`. | | `input_prompt` | string or `null` | The prompt the service waits on. Answer it with `POST /api/download/jobs/{job_id}/input`. |
| `notice` | string or `null` | A message the service shows while it waits for an approval on another device, such as a pair code. It needs no answer. | | `notice` | string or `null` | A message the service shows while it waits for an approval on another device, such as a pair code. It needs no answer. |
+1
View File
@@ -102,6 +102,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...]
| `--require-audio` | - | Audio langs that must exist. The title fails even with `--best-available`. Does not select tracks. | | `--require-audio` | - | Audio langs that must exist. The title fails even with `--best-available`. Does not select tracks. |
| `--require-video` | - | Video langs that must exist. The title fails even with `--best-available`. Does not select tracks. | | `--require-video` | - | Video langs that must exist. The title fails even with `--best-available`. Does not select tracks. |
| `--require-subs` | - | Subtitle langs that must exist. Fails the title if one is missing. `--s-lang` still decides what to keep. | | `--require-subs` | - | Subtitle langs that must exist. Fails the title if one is missing. `--s-lang` still decides what to keep. |
| `--skip-missing` | off | Skip a title that misses a required language or track, and continue with the next title. A run with one title, or a run that skips every title, still fails. |
| `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | | `-fs`, `--forced-subs` | off | Include forced subtitle tracks. |
| `-fso`, `--forced-subs-only` | off | Keep only forced subtitle tracks; drop the other subtitles. Implies `-fs`. | | `-fso`, `--forced-subs-only` | off | Keep only forced subtitle tracks; drop the other subtitles. Implies `-fs`. |
| `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. A `-` prefix excludes. | | `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. A `-` prefix excludes. |
+14 -1
View File
@@ -531,6 +531,17 @@ The check runs against the full track list, before any language selection or exc
`orig` resolves to the title language, and `--exact-lang` applies. Audio carried inside the `orig` resolves to the title language, and `--exact-lang` applies. Audio carried inside the
video track counts. `--no-audio` and `--no-video` drop the matching requirement. video track counts. `--no-audio` and `--no-video` drop the matching requirement.
A failed title stops the whole run. To skip that title and come back for it later, add
`--skip-missing`, or set `skip_missing: true` under `dl:` in `unshackle.yaml`. It also skips a
title that has no track for a filter such as `-q`, `-v`, `-r`, `-a` or a language you request.
A run with one title, or a run that skips every title, still fails. A REST API job lists each
skipped title and the reason in `skipped_titles`.
```bash
# download the episodes that have Korean audio, skip the others
unshackle dl -al ko,en --require-audio ko --skip-missing -w S06 EXAMPLE 0ABC123
```
### Forced subtitles and output format ### Forced subtitles and output format
- `-fs` / `--forced-subs`: include forced subtitle tracks (signs/foreign dialogue). - `-fs` / `--forced-subs`: include forced subtitle tracks (signs/foreign dialogue).
@@ -777,7 +788,8 @@ Additional track-type flags:
- A flag that drops the same tracks is an error, for example `-ado` with `--no-audio` - A flag that drops the same tracks is an error, for example `-ado` with `--no-audio`
or `-fso` with `-A`. or `-fso` with `-A`.
- A title with no track of that kind stops a run of one title. A run of more than one - A title with no track of that kind stops a run of one title. A run of more than one
title skips that title with a warning and continues. title skips that title with a warning and continues. The run fails when it skips
every title.
- unshackle reads the kind from the manifest or from the service. A track that - unshackle reads the kind from the manifest or from the service. A track that
neither of them marks as descriptive or forced does not match. neither of them marks as descriptive or forced does not match.
@@ -1214,6 +1226,7 @@ authoritative list.
| `--require-audio` | | Audio languages that must exist. The title fails if one is missing. | | `--require-audio` | | Audio languages that must exist. The title fails if one is missing. |
| `--require-video` | | Video languages that must exist. The title fails if one is missing. | | `--require-video` | | Video languages that must exist. The title fails if one is missing. |
| `--best-available` | | Degrade gracefully instead of failing. | | `--best-available` | | Degrade gracefully instead of failing. |
| `--skip-missing` | | Skip a title that misses a required language or track, and continue with the next title. |
| `--output` | `-o` | Output directory for this run. | | `--output` | `-o` | Output directory for this run. |
| `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. | | `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. |
| `--postscript` / `--no-postscript` | | Run a command after each output file, or run none. | | `--postscript` / `--no-postscript` | | Run a command after each output file, or run none. |
+1 -1
View File
@@ -157,7 +157,7 @@ unshackle dl --s-lang en --forced-subs-only EXAMPLE 81234567
`-fso` does not drop video or audio. Add `-S` for a subtitle-only download. A title with `-fso` does not drop video or audio. Add `-S` for a subtitle-only download. A title with
no forced track stops a run of one title. A run of more than one title skips that title no forced track stops a run of one title. A run of more than one title skips that title
with a warning. with a warning, and fails when it skips every title.
## Track types: forced, SDH, and CC ## Track types: forced, SDH, and CC
+1
View File
@@ -53,6 +53,7 @@ Common config keys. This is a useful subset, and every `dl` flag works:
| `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). | | `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). |
| `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | | `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. |
| `best_available` | bool | `false` | Warn instead of failing when a requested resolution, range, video language, or audio language is missing, or when one DRM system does not license under `--all-drm`. The CLI flag is `--best-available`, also spelled `--warn-only`. | | `best_available` | bool | `false` | Warn instead of failing when a requested resolution, range, video language, or audio language is missing, or when one DRM system does not license under `--all-drm`. The CLI flag is `--best-available`, also spelled `--warn-only`. |
| `skip_missing` | bool | `false` | Skip a title that misses a required language (`require_audio`, `require_video`, `require_subs`) or a track that a filter asks for, and continue with the next title. A run with one title, or a run that skips every title, still fails. The CLI flag is `--skip-missing`. For REST API jobs, set it in the [`serve`](services.md#serve) block. |
| `generate_dv` | bool | `false` | With `-r HYBRID`, make Dolby Vision from the HDR10+ metadata or the decoded video when a title has no DV track. With `-r HDR10` or `-r HDR10P`, make it for that base and ignore a DV track. See [Dolby Vision without a DV track](../../guide/downloading.md#dolby-vision-without-a-dv-track). | | `generate_dv` | bool | `false` | With `-r HYBRID`, make Dolby Vision from the HDR10+ metadata or the decoded video when a title has no DV track. With `-r HDR10` or `-r HDR10P`, make it for that base and ignore a DV track. See [Dolby Vision without a DV track](../../guide/downloading.md#dolby-vision-without-a-dv-track). |
| `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | | `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. |
| `no_folder` | bool | `false` | Do not create a per-title folder. | | `no_folder` | bool | `false` | Do not create a per-title folder. |
@@ -213,6 +213,10 @@ subs = [p for p in sys.argv[i].removeprefix("--subs=").split("\n") if p]
nothing about whether the season is complete on disk. Downloading a single episode fires nothing about whether the season is complete on disk. Downloading a single episode fires
the post-script for that season as soon as the episode lands. the post-script for that season as soon as the episode lands.
A title that track selection skips (`--skip-missing`, `-ado`, `-fso`) counts as done. The
`season` post-script still fires for the episodes that landed, so the folder can be missing
the skipped episodes. A run that skips every title fails, and then no `season` post-script fires.
A title that fails stops the whole run. unshackle dispatches the `failure` post-script for A title that fails stops the whole run. unshackle dispatches the `failure` post-script for
that title and then returns. It downloads no further title, and no `season` or `run` that title and then returns. It downloads no further title, and no `season` or `run`
post-script fires after that point. This applies to every title type. If one episode of S01 post-script fires after that point. This applies to every title type. If one episode of S01
+171 -10
View File
@@ -12,9 +12,11 @@ import logging
from typing import Any, Optional, cast from typing import Any, Optional, cast
import pytest import pytest
import requests
from unshackle.commands.dl import dl from unshackle.commands.dl import dl
from unshackle.core.api.handlers import DEFAULT_DOWNLOAD_PARAMS from unshackle.core.api.handlers import DEFAULT_DOWNLOAD_PARAMS
from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY
from unshackle.core.service import Service from unshackle.core.service import Service
from unshackle.core.titles import Episode, Series from unshackle.core.titles import Episode, Series
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
@@ -38,7 +40,7 @@ class Harness(dl):
class FakeService: class FakeService:
"""One episode per entry; ``ad`` adds a descriptive track, ``forced`` a forced subtitle.""" """One episode per entry; ``ad`` adds a descriptive track, ``forced`` a forced subtitle, ``ko`` Korean audio."""
def __init__(self, episodes: dict[int, dict[str, bool]]) -> None: def __init__(self, episodes: dict[int, dict[str, bool]]) -> None:
self.episodes = episodes self.episodes = episodes
@@ -62,6 +64,8 @@ class FakeService:
tracks.add( tracks.add(
Audio(id_="ad", url="https://x/ad", language="en", codec=Audio.Codec.AAC, bitrate=1, descriptive=True) Audio(id_="ad", url="https://x/ad", language="en", codec=Audio.Codec.AAC, bitrate=1, descriptive=True)
) )
if kinds.get("ko"):
tracks.add(Audio(id_="ko", url="https://x/ko", language="ko", codec=Audio.Codec.AAC, bitrate=128_000))
tracks.add(Subtitle(id_="s", url="https://x/s", language="en", codec=Subtitle.Codec.WebVTT)) tracks.add(Subtitle(id_="s", url="https://x/s", language="en", codec=Subtitle.Codec.WebVTT))
if kinds.get("forced"): if kinds.get("forced"):
tracks.add(Subtitle(id_="sf", url="https://x/sf", language="en", codec=Subtitle.Codec.WebVTT, forced=True)) tracks.add(Subtitle(id_="sf", url="https://x/sf", language="en", codec=Subtitle.Codec.WebVTT, forced=True))
@@ -71,8 +75,8 @@ class FakeService:
return [] return []
def __getattr__(self, name: str) -> Any: def __getattr__(self, name: str) -> Any:
if name.startswith("on_"): if name.startswith(("on_", "get_")):
# the event hooks (on_track_downloaded, ...) the loop subscribes # the event hooks (on_track_downloaded, ...) and the DRM callbacks a run past selection probes
return lambda *a, **k: None return lambda *a, **k: None
raise AttributeError(name) raise AttributeError(name)
@@ -82,16 +86,22 @@ def run(
caplog: pytest.LogCaptureFixture, caplog: pytest.LogCaptureFixture,
episodes: dict, episodes: dict,
service: Optional[FakeService] = None, service: Optional[FakeService] = None,
to_end: bool = False,
**flags: Any, **flags: Any,
) -> Any: ) -> Any:
"""Return the Selected ids, the exit code, or None for a run that completes.""" """Return the Selected ids, the exit code, or None for a run that completes; to_end runs past selection."""
monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self))) if not to_end:
monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self)))
else:
monkeypatch.setattr("unshackle.commands.dl.download_tracks_in_passes", lambda *a, **k: None)
monkeypatch.setattr("unshackle.core.providers.search_metadata", lambda *a, **k: None) monkeypatch.setattr("unshackle.core.providers.search_metadata", lambda *a, **k: None)
cmd = Harness.__new__(Harness) cmd = Harness.__new__(Harness)
cmd.log = logging.getLogger("test_kind_only") cmd.log = logging.getLogger("test_kind_only")
cmd.service = "FAKE" cmd.service = "FAKE"
cmd.tmdb_searched = True cmd.tmdb_searched = True
cmd.completed_files = [] cmd.completed_files = []
cmd.skipped_titles = []
cmd.skipped_subtitles = []
kwargs: dict[str, Any] = {} kwargs: dict[str, Any] = {}
for name, param in inspect.signature(dl.result).parameters.items(): for name, param in inspect.signature(dl.result).parameters.items():
if name in DEFAULT_DOWNLOAD_PARAMS: if name in DEFAULT_DOWNLOAD_PARAMS:
@@ -101,12 +111,17 @@ def run(
kwargs.update(quality=[], vcodec=[], acodec=[], range_=[], wanted=[], cdm_only=None, downloads=1) kwargs.update(quality=[], vcodec=[], acodec=[], range_=[], wanted=[], cdm_only=None, downloads=1)
kwargs.update(flags) kwargs.update(flags)
caplog.set_level(logging.WARNING, logger="test_kind_only") caplog.set_level(logging.WARNING, logger="test_kind_only")
service = service or FakeService(episodes)
if to_end:
service.session = requests.Session()
try: try:
cmd.result(cast(Service, service or FakeService(episodes)), **kwargs) cmd.result(cast(Service, service), **kwargs)
except Selected as sel: except Selected as sel:
return sel.ids return sel.ids
except SystemExit as exc: except SystemExit as exc:
return exc.code return exc.code
finally:
DOWNLOAD_LICENCE_ONLY.clear()
return None return None
@@ -145,8 +160,9 @@ def test_subs_only_with_no_matching_subtitle_exits_a_single_title_and_skips_in_a
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
assert run(monkeypatch, caplog, {1: {}}, subs_only=True, s_lang=["fr"]) == 1 assert run(monkeypatch, caplog, {1: {}}, subs_only=True, s_lang=["fr"]) == 1
assert run(monkeypatch, caplog, {1: {}, 2: {}}, subs_only=True, s_lang=["fr"]) is None assert run(monkeypatch, caplog, {1: {}, 2: {}}, subs_only=True, s_lang=["fr"]) == 1
assert caplog.text.count("There's no track left for the selected options. Skipping Show S01E") == 2 assert caplog.text.count("Skipping Show S01E") == 2
assert "Skipping Show S01E02: There's no track left for the selected options." in caplog.text
def test_a_multi_title_run_skips_the_title_and_continues( def test_a_multi_title_run_skips_the_title_and_continues(
@@ -157,13 +173,158 @@ def test_a_multi_title_run_skips_the_title_and_continues(
assert "Skipping Show S01E01" in caplog.text assert "Skipping Show S01E01" in caplog.text
def test_a_multi_title_run_with_every_title_skipped_completes( def test_a_multi_title_run_with_every_title_skipped_fails(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
assert run(monkeypatch, caplog, {1: {}, 2: {}}, forced_subs_only=True) is None assert run(monkeypatch, caplog, {1: {}, 2: {}}, forced_subs_only=True) == 1
assert caplog.text.count("Skipping Show S01E") == 2 assert caplog.text.count("Skipping Show S01E") == 2
def test_a_required_language_miss_stops_the_run_by_default(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
sink: list[dict] = []
assert run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, require_audio=["ko"], progress_sink=sink.append) == 1
assert "Skipping" not in caplog.text
assert "Show S01E01: Required audio language(s) not found: ko" in caplog.text
assert not any("skipped_titles" in event for event in sink)
def test_skip_missing_skips_the_title_and_reports_it(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
sink: list[dict] = []
ids = run(
monkeypatch,
caplog,
{1: {}, 2: {"ko": True}},
require_audio=["ko"],
skip_missing=True,
progress_sink=sink.append,
)
assert ids == {"video": ["v"], "audio": ["a"], "subs": ["s"]}
assert "Skipping Show S01E01: Required audio language(s) not found: ko" in caplog.text
assert {"skipped_titles": [{"title": "Show S01E01", "reason": "Required audio language(s) not found: ko"}]} in sink
def test_require_audio_video_and_subs_work_together(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
ids = run(monkeypatch, caplog, {1: {}}, require_audio=["en"], require_video=["en"], require_subs=["en"])
assert ids == {"video": ["v"], "audio": ["a"], "subs": ["s"]}
assert run(monkeypatch, caplog, {1: {}}, require_audio=["ko"], require_video=["ko"], require_subs=["ko"]) == 1
assert (
"Required audio language(s) not found: ko; Required video language(s) not found: ko; "
"Required subtitle language(s) not found: ko"
) in caplog.text
assert run(monkeypatch, caplog, {1: {}}, require_audio=["en"], require_video=["ko"], require_subs=["en"]) == 1
assert "Show S01E01: Required video language(s) not found: ko" in caplog.text
def test_skip_missing_skips_a_filter_miss_and_continues(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
ids = run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, a_lang=["ko"], skip_missing=True)
assert ids == {"video": ["v"], "audio": ["ko"], "subs": ["s"]}
assert "Skipping Show S01E01" in caplog.text
def test_skip_missing_still_fails_a_run_that_skips_every_title(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
assert run(monkeypatch, caplog, {1: {}, 2: {}}, require_audio=["ko"], skip_missing=True) == 1
assert "Skipped all 2 titles" in caplog.text
assert run(monkeypatch, caplog, {1: {}}, require_audio=["ko"], skip_missing=True) == 1
class LangService(FakeService):
"""Each episode names its original language and its audio languages; the video is in the original language."""
def get_titles_cached(self) -> Series:
return Series(
Episode(id_=f"show-s01-e{n:03}", service=FakeService, title="Show", season=1, number=n, language=ep["orig"])
for n, ep in self.episodes.items()
)
def get_tracks(self, title: Episode) -> Tracks:
ep = self.episodes[title.number]
tracks = Tracks()
tracks.add(Video(id_="v", url="https://x/v", language=ep["orig"] or "en", codec=Video.Codec.AVC))
for lang in ep["audio"]:
tracks.add(Audio(id_=lang, url=f"https://x/{lang}", language=lang, codec=Audio.Codec.AAC, bitrate=128_000))
return tracks
JA_EN = {"orig": "ja", "audio": ["ja", "en"]}
FR = {"orig": "ja", "audio": ["fr"]}
EVERY = {"orig": "ja", "audio": ["ja", "ko", "en"]}
ORIG_KO_EN = ["orig", "ko", "en"]
@pytest.mark.parametrize(
"first,flags,outcome,logged",
[
(JA_EN, {}, 1, "Show S01E01: ko not found in audio tracks"),
(JA_EN, {"skip_missing": True}, ["ja", "ko", "en"], "Skipping Show S01E01: ko not found in audio tracks"),
({"orig": "ja", "audio": ["ko", "en"]}, {"skip_missing": True}, ["ja", "ko", "en"], "S01E01: orig not found"),
(JA_EN, {"best_available": True}, ["ja", "en"], "ko not found in audio tracks, continuing with: orig, en"),
(FR, {"best_available": True}, 1, "orig, ko, en not found in audio tracks and no fallback available"),
(FR, {"best_available": True, "skip_missing": True}, ["ja", "ko", "en"], "Skipping Show S01E01: orig, ko, en"),
(JA_EN, {"best_available": True, "require_audio": ["ko"]}, 1, "Required audio language(s) not found: ko"),
(
JA_EN,
{"best_available": True, "require_audio": ["ko"], "skip_missing": True},
["ja", "ko", "en"],
"Skipping Show S01E01: Required audio language(s) not found: ko",
),
],
)
def test_several_audio_languages_skip_an_episode_that_misses_one(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
first: dict,
flags: dict[str, Any],
outcome: Any,
logged: str,
) -> None:
"""-al orig,ko,en on a season where only the second episode has all three."""
episodes = {1: first, 2: EVERY}
result = run(monkeypatch, caplog, episodes, service=LangService(episodes), a_lang=ORIG_KO_EN, **flags)
assert (result if outcome == 1 else result["audio"]) == outcome
assert logged in caplog.text
def test_several_audio_languages_fail_a_season_that_misses_one_everywhere(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
episodes = {1: JA_EN, 2: JA_EN}
assert run(monkeypatch, caplog, episodes, service=LangService(episodes), a_lang=ORIG_KO_EN, skip_missing=True) == 1
assert caplog.text.count("ko not found in audio tracks") == 2
assert "Skipped all 2 titles" in caplog.text
def test_a_partly_skipped_run_completes_and_lists_the_skips(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
episodes = {1: {}, 2: {"ko": True}}
assert run(monkeypatch, caplog, episodes, to_end=True, a_lang=["ko"], skip_missing=True, skip_dl=True) is None
assert "Skipped 1 of 2 titles: Show S01E01" in caplog.text
@pytest.mark.parametrize("no_mux,settled", [(False, 2), (True, 1)])
def test_a_skipped_title_settles_its_post_script_group(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, no_mux: bool, settled: int
) -> None:
"""The season event waits on every title of a group; a skipped title counts as settled, once at count time."""
import unshackle.commands.dl as dl_module
seen: list[str] = []
group = dl_module.post_script_group
monkeypatch.setattr(dl_module, "post_script_group", lambda t: seen.append(str(t)) or group(t))
run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, a_lang=["ko"], skip_missing=True, no_mux=no_mux)
assert seen.count("Show S01E01") == settled
class LcevcService(FakeService): class LcevcService(FakeService):
"""A 540p base video and the 1080p LCEVC enhancement layer that decodes on top of it.""" """A 540p base video and the 1080p LCEVC enhancement layer that decodes on top of it."""
@@ -626,3 +626,86 @@ def test_client_job_view_hides_debug_fields_without_debug_api() -> None:
assert client_job_view(job, off) == {"job_id": "j"} assert client_job_view(job, off) == {"job_id": "j"}
assert client_job_view(job, None) == {"job_id": "j"} assert client_job_view(job, None) == {"job_id": "j"}
assert client_job_view(job, on) == job assert client_job_view(job, on) == job
FAKE_SKIPPING_WORKER = """
import sys
from unshackle.core.api import download_worker
def skip_then_stop(*args, progress_callback, **kwargs):
progress_callback({"skipped_titles": [{"title": "Show S01E02", "reason": "missing ko"}]})
raise RuntimeError("Download failed: Skipped all 2 titles")
download_worker.perform_download = skip_then_stop
sys.exit(download_worker.main(["worker"] + sys.argv[1:]))
"""
async def test_a_skip_written_just_before_the_worker_exits_reaches_the_job(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, manager: DownloadQueueManager
) -> None:
import asyncio
from unshackle.core.api import download_manager
worker = tmp_path / "worker.py"
worker.write_text(FAKE_SKIPPING_WORKER, encoding="utf-8")
spawn = asyncio.create_subprocess_exec
async def spawn_fake_worker(executable: str, *args: str, **kwargs: Any) -> Any:
return await spawn(executable, str(worker), *args[2:], **kwargs)
monkeypatch.setattr(download_manager.asyncio, "create_subprocess_exec", spawn_fake_worker)
job = manager.create_job("EXAMPLE", "t")
with pytest.raises(Exception, match="Skipped all 2 titles"):
await asyncio.wait_for(manager.run_download_async(job), timeout=20)
assert job.skipped_titles == [{"title": "Show S01E02", "reason": "missing ko"}]
assert job.to_dict()["skipped_titles"] == job.skipped_titles
def test_a_stopped_run_names_its_reason_in_the_job_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
import sys
import click
import unshackle.commands.dl as dl_module
from unshackle.core.api import download_manager, handlers
from unshackle.core.api.errors import APIError
from unshackle.core.services import Services
class FakeDl:
cli = dl_module.dl.cli
def __init__(self, ctx: click.Context, **kwargs: object) -> None:
self.stop_reason = None
def result(self, **kwargs: object) -> None:
self.stop_reason = "Show S01E01: Required audio language(s) not found: ko"
sys.exit(1)
class FakeService:
def __init__(self, ctx: click.Context) -> None:
pass
monkeypatch.setattr(dl_module, "dl", FakeDl)
monkeypatch.setattr(Services, "get_path", staticmethod(lambda s: tmp_path))
monkeypatch.setattr(Services, "load", staticmethod(lambda s: FakeService))
monkeypatch.setattr(handlers, "load_full_cdm", lambda *a: None)
with pytest.raises(APIError, match=r"^Download failed: Show S01E01: Required audio language\(s\) not found: ko$"):
download_manager.perform_download("job-1", "EXAMPLE1", "t1", {})
def test_job_history_keeps_the_skipped_titles(tmp_path: Path, manager: DownloadQueueManager) -> None:
import json
from unshackle.core.api.download_manager import record_job_history
job = manager.create_job("EXAMPLE", "t")
job.status = JobStatus.COMPLETED
job.skipped_titles = [{"title": "Show S01E02", "reason": "missing ko"}]
record_job_history(job)
entry = json.loads((tmp_path / "api_history.jsonl").read_text(encoding="utf-8").splitlines()[-1])
assert entry["skipped_titles"] == job.skipped_titles
+101 -61
View File
@@ -158,6 +158,13 @@ if TYPE_CHECKING:
from unshackle.core.remote_service import ServerKeySource from unshackle.core.remote_service import ServerKeySource
class SkippedTitle(TypedDict):
"""A title skipped because a requested track or language is missing. Accumulated as ``dl.skipped_titles``."""
title: str
reason: str
class SkippedSubtitle(TypedDict): class SkippedSubtitle(TypedDict):
"""A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``, """A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``,
one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable.""" one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable."""
@@ -979,6 +986,12 @@ class dl:
default=[], default=[],
help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.", help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.",
) )
@click.option(
"--skip-missing",
is_flag=True,
default=False,
help="Skip a title that misses a required language or track and continue with the next title.",
)
@click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.") @click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.")
@click.option( @click.option(
"-fso", "-fso",
@@ -1383,6 +1396,9 @@ class dl:
# Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can # Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can
# report which weren't available without parsing the console output. See SkippedSubtitle. # report which weren't available without parsing the console output. See SkippedSubtitle.
self.skipped_subtitles: list[SkippedSubtitle] = [] self.skipped_subtitles: list[SkippedSubtitle] = []
self.skipped_titles: list[SkippedTitle] = []
# Why a missing track or language stopped the run, for the API job error.
self.stop_reason: Optional[str] = None
# result() catches a download-worker failure, reports it, and returns rather than # result() catches a download-worker failure, reports it, and returns rather than
# re-raising (so the CLI still exits cleanly). Keep it so an embedding caller (the API # re-raising (so the CLI still exits cleanly). Keep it so an embedding caller (the API
# worker) can tell the title did not complete instead of reading it as a success. # worker) can tell the title did not complete instead of reading it as a success.
@@ -1765,6 +1781,7 @@ class dl:
require_audio: list[str], require_audio: list[str],
require_video: list[str], require_video: list[str],
require_subs: list[str], require_subs: list[str],
skip_missing: bool,
forced_subs: bool, forced_subs: bool,
forced_subs_only: bool, forced_subs_only: bool,
forced_s_lang: list[str], forced_s_lang: list[str],
@@ -2348,15 +2365,20 @@ class dl:
for folder, context in post_script_last.get(group_key, {}).items(): for folder, context in post_script_last.get(group_key, {}).items():
dispatch("success", "season", season_context(context, folder), postscript) dispatch("success", "season", season_context(context, folder), postscript)
def skip_title(title: Any, reason: str) -> None: def skip_title(title: Any, reason: str, kind_only: bool = False) -> None:
"""Skip a title that has no track of the kind an only flag selects. """Skip a title that has no track the selection asks for, and report it to the API job.
A run with one title stops instead, because it has nothing else to download. Only a kind-only miss (-ado, -fso, no track left) skips without --skip-missing; any other miss
stops the run. A run with one title always stops, because it has nothing else to download.
""" """
if wanted_total <= 1: if wanted_total <= 1 or not (kind_only or skip_missing):
self.log.error(reason) self.stop_reason = f"{title}: {reason}"
self.log.error(self.stop_reason)
sys.exit(1) sys.exit(1)
self.log.warning(f"{reason} Skipping {title}") self.skipped_titles.append(SkippedTitle(title=str(title), reason=reason))
if progress_sink:
progress_sink({"skipped_titles": list(self.skipped_titles)})
self.log.warning(f"Skipping {title}: {reason}")
if not no_mux: if not no_mux:
settle_post_script_group(title) settle_post_script_group(title)
@@ -2641,6 +2663,7 @@ class dl:
elif proxy_download: elif proxy_download:
console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}") console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}")
missing_required: list[str] = []
for kind, required, available in ( for kind, required, available in (
( (
"audio", "audio",
@@ -2659,10 +2682,11 @@ class dl:
], ],
), ),
): ):
missing_required = missing_required_langs(required, available, title.language, exact=exact_lang) if missing := missing_required_langs(required, available, title.language, exact=exact_lang):
if missing_required: missing_required.append(f"Required {kind} language(s) not found: {', '.join(missing)}")
self.log.error(f"Required {kind} language(s) not found: {', '.join(missing_required)}") if missing_required:
sys.exit(1) skip_title(title, "; ".join(missing_required))
continue
if not keep_videos: if not keep_videos:
title.tracks.videos = [] title.tracks.videos = []
@@ -2690,16 +2714,18 @@ class dl:
for codec in missing_codecs: for codec in missing_codecs:
self.log.warning(f"Skipping {codec.name} video tracks as none are available.") self.log.warning(f"Skipping {codec.name} video tracks as none are available.")
if not title.tracks.videos: if not title.tracks.videos:
self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...") skip_title(title, f"There's no {', '.join(c.name for c in vcodec)} Video Track...")
sys.exit(1) continue
elif keep_videos: elif keep_videos:
title.tracks.select_video(lambda x: x.codec != Video.Codec.LCEVC) title.tracks.select_video(lambda x: x.codec != Video.Codec.LCEVC)
if keep_videos and generated_dv_bases: if keep_videos and generated_dv_bases:
title.tracks.videos = keep_generated_dv_bases(title.tracks.videos, generated_dv_bases) title.tracks.videos = keep_generated_dv_bases(title.tracks.videos, generated_dv_bases)
if not any(v.range in generated_dv_bases for v in title.tracks.videos): if not any(v.range in generated_dv_bases for v in title.tracks.videos):
self.log.error(f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track...") skip_title(
sys.exit(1) title, f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track..."
)
continue
if keep_videos and range_: if keep_videos and range_:
# Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks # Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks
@@ -2712,8 +2738,8 @@ class dl:
if Video.Range.HDR10P in missing_ranges and dv_has_hdr10p: if Video.Range.HDR10P in missing_ranges and dv_has_hdr10p:
self.log.warning("A DV track carries HDR10+: use -r DV or -r HYBRID.") self.log.warning("A DV track carries HDR10+: use -r DV or -r HYBRID.")
if not title.tracks.videos: if not title.tracks.videos:
self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...") skip_title(title, f"There's no {', '.join(r.name for r in range_)} Video Track...")
sys.exit(1) continue
if keep_videos and vbitrate: if keep_videos and vbitrate:
if any(r == Video.Range.HYBRID for r in range_): if any(r == Video.Range.HYBRID for r in range_):
@@ -2723,21 +2749,21 @@ class dl:
lambda x: x.range == Video.Range.DV or bool(x.bitrate and x.bitrate // 1000 == vbitrate) lambda x: x.range == Video.Range.DV or bool(x.bitrate and x.bitrate // 1000 == vbitrate)
) )
if not any(x.range != Video.Range.DV for x in title.tracks.videos): if not any(x.range != Video.Range.DV for x in title.tracks.videos):
self.log.error(f"There's no {vbitrate}kbps Video Track...") skip_title(title, f"There's no {vbitrate}kbps Video Track...")
sys.exit(1) continue
else: else:
title.tracks.select_video(lambda x: bool(x.bitrate and x.bitrate // 1000 == vbitrate)) title.tracks.select_video(lambda x: bool(x.bitrate and x.bitrate // 1000 == vbitrate))
if not title.tracks.videos: if not title.tracks.videos:
self.log.error(f"There's no {vbitrate}kbps Video Track...") skip_title(title, f"There's no {vbitrate}kbps Video Track...")
sys.exit(1) continue
if keep_videos and vbitrate_min is not None and vbitrate_max is not None: if keep_videos and vbitrate_min is not None and vbitrate_max is not None:
title.tracks.select_video( title.tracks.select_video(
lambda x: bool(x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max) lambda x: bool(x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max)
) )
if not title.tracks.videos: if not title.tracks.videos:
self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...") skip_title(title, f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...")
sys.exit(1) continue
if keep_videos and video_excl_r: if keep_videos and video_excl_r:
drop = matched_language_tags( drop = matched_language_tags(
@@ -2745,8 +2771,8 @@ class dl:
) )
title.tracks.select_video(lambda x: str(x.language) not in drop) title.tracks.select_video(lambda x: str(x.language) not in drop)
if not title.tracks.videos: if not title.tracks.videos:
self.log.error(f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...") skip_title(title, f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...")
sys.exit(1) continue
effective_video_lang = v_lang or lang effective_video_lang = v_lang or lang
video_languages = [lang for lang in effective_video_lang if lang != "best"] video_languages = [lang for lang in effective_video_lang if lang != "best"]
@@ -2776,20 +2802,23 @@ class dl:
) )
processed_video_lang = remaining_v_langs processed_video_lang = remaining_v_langs
else: else:
self.log.error(f"{missing_str} not found in video tracks and no fallback available") skip_title(
sys.exit(1) title, f"{missing_str} not found in video tracks and no fallback available"
)
continue
else: else:
self.log.error(missing_str + " not found in video tracks") skip_title(title, missing_str + " not found in video tracks")
sys.exit(1) continue
title.tracks.videos = title.tracks.by_language( title.tracks.videos = title.tracks.by_language(
title.tracks.videos, processed_video_lang, exact_match=exact_lang title.tracks.videos, processed_video_lang, exact_match=exact_lang
) )
if not title.tracks.videos: if not title.tracks.videos:
# empty only when 'orig' was the sole request and did not resolve # empty only when 'orig' was the sole request and did not resolve
self.log.error( skip_title(
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..." title,
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track...",
) )
sys.exit(1) continue
has_hybrid = any(r == Video.Range.HYBRID for r in range_) has_hybrid = any(r == Video.Range.HYBRID for r in range_)
non_hybrid_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID] non_hybrid_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID]
@@ -2854,8 +2883,8 @@ class dl:
f"There's no {res_list} Video Track{plural}, continuing with available qualities..." f"There's no {res_list} Video Track{plural}, continuing with available qualities..."
) )
else: else:
self.log.error(f"There's no {res_list} Video Track{plural}...") skip_title(title, f"There's no {res_list} Video Track{plural}...")
sys.exit(1) continue
pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else [] pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else []
if keep_videos and has_hybrid: if keep_videos and has_hybrid:
@@ -2968,8 +2997,8 @@ class dl:
fallback_selected.append(match) fallback_selected.append(match)
title.tracks.videos = fallback_selected title.tracks.videos = fallback_selected
else: else:
self.log.error(gate_failure) skip_title(title, gate_failure)
sys.exit(1) continue
fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang) fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang)
if keep_subtitles and s_excl_r: if keep_subtitles and s_excl_r:
@@ -3031,27 +3060,31 @@ class dl:
if forced_subs_only: if forced_subs_only:
title.tracks.select_subtitles(lambda x: x.forced) title.tracks.select_subtitles(lambda x: x.forced)
if not title.tracks.subtitles: if not title.tracks.subtitles:
skip_title(title, "There's no forced subtitle track for the selected languages.") skip_title(
title,
"There's no forced subtitle track for the selected languages.",
kind_only=True,
)
continue continue
if keep_audio and audio_description_only: if keep_audio and audio_description_only:
title.tracks.select_audio(lambda x: x.descriptive) title.tracks.select_audio(lambda x: x.descriptive)
if not title.tracks.audio: if not title.tracks.audio:
skip_title(title, "There's no audio description track.") skip_title(title, "There's no audio description track.", kind_only=True)
continue continue
# might have no audio tracks if part of the video, e.g. transport stream hls # might have no audio tracks if part of the video, e.g. transport stream hls
if keep_audio and len(title.tracks.audio) > 0: if keep_audio and len(title.tracks.audio) > 0:
if not audio_description: if not audio_description:
title.tracks.select_audio(lambda x: not x.descriptive) title.tracks.select_audio(lambda x: not x.descriptive)
if not title.tracks.audio: if not title.tracks.audio:
self.log.error("Audio description only, use --audio-description...") skip_title(title, "Audio description only, use --audio-description...")
sys.exit(1) continue
if acodec: if acodec:
title.tracks.select_audio(lambda x: x.codec in acodec) title.tracks.select_audio(lambda x: x.codec in acodec)
if not title.tracks.audio: if not title.tracks.audio:
codec_names = ", ".join(c.name for c in acodec) codec_names = ", ".join(c.name for c in acodec)
self.log.error(f"No audio tracks matching codecs: {codec_names}") skip_title(title, f"No audio tracks matching codecs: {codec_names}")
sys.exit(1) continue
if channels: if channels:
title.tracks.select_audio( title.tracks.select_audio(
lambda x: bool( lambda x: bool(
@@ -3059,33 +3092,33 @@ class dl:
) )
) )
if not title.tracks.audio: if not title.tracks.audio:
self.log.error(f"There's no {channels} Audio Track...") skip_title(title, f"There's no {channels} Audio Track...")
sys.exit(1) continue
if no_atmos: if no_atmos:
title.tracks.audio = [x for x in title.tracks.audio if not x.atmos] title.tracks.audio = [x for x in title.tracks.audio if not x.atmos]
if not title.tracks.audio: if not title.tracks.audio:
self.log.error("No non-Atmos audio tracks available...") skip_title(title, "No non-Atmos audio tracks available...")
sys.exit(1) continue
if abitrate: if abitrate:
title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate) title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate)
if not title.tracks.audio: if not title.tracks.audio:
self.log.error(f"There's no {abitrate}kbps Audio Track...") skip_title(title, f"There's no {abitrate}kbps Audio Track...")
sys.exit(1) continue
if abitrate_min is not None and abitrate_max is not None: if abitrate_min is not None and abitrate_max is not None:
title.tracks.select_audio( title.tracks.select_audio(
lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max
) )
if not title.tracks.audio: if not title.tracks.audio:
self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...") skip_title(title, f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...")
sys.exit(1) continue
if audio_excl_r: if audio_excl_r:
drop = matched_language_tags( drop = matched_language_tags(
audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True
) )
title.tracks.select_audio(lambda x: str(x.language) not in drop) title.tracks.select_audio(lambda x: str(x.language) not in drop)
if not title.tracks.audio: if not title.tracks.audio:
self.log.error(f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...") skip_title(title, f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...")
sys.exit(1) continue
audio_languages = a_lang or lang audio_languages = a_lang or lang
if audio_languages: if audio_languages:
processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang) processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang)
@@ -3115,13 +3148,13 @@ class dl:
) )
processed_lang = remaining processed_lang = remaining
else: else:
self.log.error( skip_title(
f"{missing_str} not found in {audio_noun} and no fallback available" title, f"{missing_str} not found in {audio_noun} and no fallback available"
) )
sys.exit(1) continue
else: else:
self.log.error(f"{missing_str} not found in {audio_noun}") skip_title(title, f"{missing_str} not found in {audio_noun}")
sys.exit(1) continue
title.tracks.audio = select_best_audio( title.tracks.audio = select_best_audio(
title.tracks.audio, processed_lang, acodec, audio_description, exact_lang title.tracks.audio, processed_lang, acodec, audio_description, exact_lang
@@ -3129,11 +3162,11 @@ class dl:
if not title.tracks.audio and not embedded_langs: if not title.tracks.audio and not embedded_langs:
# empty when 'orig' was the sole request, did not resolve, and the # empty when 'orig' was the sole request, did not resolve, and the
# video carries no audio of its own to fall back on # video carries no audio of its own to fall back on
self.log.error( skip_title(
f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} " title,
f"Audio Track, cannot continue..." f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} Audio Track...",
) )
sys.exit(1) continue
kept_tracks = [] kept_tracks = []
if keep_videos: if keep_videos:
@@ -3149,7 +3182,7 @@ class dl:
title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url) title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url)
if not (title.tracks.videos or title.tracks.audio or title.tracks.subtitles or title.tracks.chapters): if not (title.tracks.videos or title.tracks.audio or title.tracks.subtitles or title.tracks.chapters):
skip_title(title, "There's no track left for the selected options.") skip_title(title, "There's no track left for the selected options.", kind_only=True)
continue continue
selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True)
@@ -4063,6 +4096,13 @@ class dl:
self.wait_vault_writes() self.wait_vault_writes()
if export_path: if export_path:
self.log_export_summary() self.log_export_summary()
if self.skipped_titles:
if len(self.skipped_titles) == wanted_total:
self.stop_reason = f"Skipped all {wanted_total} titles"
self.log.error(self.stop_reason)
sys.exit(1)
skipped = ", ".join(t["title"] for t in self.skipped_titles)
self.log.warning(f"Skipped {len(self.skipped_titles)} of {wanted_total} titles: {skipped}")
dl_time = time_elapsed_since(start_time) dl_time = time_elapsed_since(start_time)
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
+13 -4
View File
@@ -137,6 +137,8 @@ class DownloadJob:
# Subtitles skipped under skip_subtitle_errors (non-fatal). Each entry is a dl.SkippedSubtitle # Subtitles skipped under skip_subtitle_errors (non-fatal). Each entry is a dl.SkippedSubtitle
# dict (id / language / title) so a client can report which weren't available. # dict (id / language / title) so a client can report which weren't available.
skipped_subtitles: List[Dict[str, Any]] = field(default_factory=list) skipped_subtitles: List[Dict[str, Any]] = field(default_factory=list)
# Titles skipped because a requested track or language is missing; each a dl.SkippedTitle (title / reason).
skipped_titles: List[Dict[str, Any]] = field(default_factory=list)
cancel_event: threading.Event = field(default_factory=threading.Event) cancel_event: threading.Event = field(default_factory=threading.Event)
@@ -168,6 +170,7 @@ class DownloadJob:
"segments_total": self.segments_total, "segments_total": self.segments_total,
"speed": self.speed, "speed": self.speed,
"skipped_subtitles": self.skipped_subtitles, "skipped_subtitles": self.skipped_subtitles,
"skipped_titles": self.skipped_titles,
"input_prompt": self.input_prompt, "input_prompt": self.input_prompt,
"notice": self.notice, "notice": self.notice,
} }
@@ -232,6 +235,7 @@ def record_job_history(job: DownloadJob) -> None:
"created_time": job.created_time.isoformat(), "created_time": job.created_time.isoformat(),
"completed_time": job.completed_time.isoformat() if job.completed_time else None, "completed_time": job.completed_time.isoformat() if job.completed_time else None,
"output_files": job.output_files, "output_files": job.output_files,
"skipped_titles": job.skipped_titles,
"parameters": redact_parameters(job.parameters), "parameters": redact_parameters(job.parameters),
"error_message": _redact_text(job.error_message, job.parameters), "error_message": _redact_text(job.error_message, job.parameters),
} }
@@ -523,6 +527,8 @@ def perform_download(
log.error(f"Download exited with code {exc.code}") log.error(f"Download exited with code {exc.code}")
log.debug("Stdout: %s", stdout_capture.getvalue()) log.debug("Stdout: %s", stdout_capture.getvalue())
log.debug("Stderr: %s", stderr_capture.getvalue()) log.debug("Stderr: %s", stderr_capture.getvalue())
if reason := getattr(dl_instance, "stop_reason", None):
raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed: {reason}")
raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed with exit code {exc.code}") raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed with exit code {exc.code}")
except Exception as exc: # noqa: BLE001 - propagate to caller except Exception as exc: # noqa: BLE001 - propagate to caller
@@ -961,11 +967,8 @@ class DownloadQueueManager:
try: try:
while True: while True:
done, _ = await asyncio.wait({communicate_task}, timeout=0.5) done, _ = await asyncio.wait({communicate_task}, timeout=0.5)
if communicate_task in done:
stdout_bytes, stderr_bytes = communicate_task.result()
break
# Check for progress updates (skip re-parse when the file hasn't changed since last tick) # Read progress before the exit check so the worker's last write lands (skip an unchanged file)
try: try:
stat = os.stat(progress_path) stat = os.stat(progress_path)
stat_key = (stat.st_mtime_ns, stat.st_size) stat_key = (stat.st_mtime_ns, stat.st_size)
@@ -1003,6 +1006,8 @@ class DownloadQueueManager:
job.notice = progress_data["notice"] job.notice = progress_data["notice"]
if progress_data.get("skipped_subtitles"): if progress_data.get("skipped_subtitles"):
job.skipped_subtitles = progress_data["skipped_subtitles"] job.skipped_subtitles = progress_data["skipped_subtitles"]
if progress_data.get("skipped_titles"):
job.skipped_titles = progress_data["skipped_titles"]
if "progress" in progress_data: if "progress" in progress_data:
new_progress = float(progress_data["progress"]) new_progress = float(progress_data["progress"])
if new_progress != job.progress: if new_progress != job.progress:
@@ -1017,6 +1022,10 @@ class DownloadQueueManager:
except (OSError, json.JSONDecodeError, ValueError) as e: except (OSError, json.JSONDecodeError, ValueError) as e:
log.debug(f"Could not read progress for job {job.job_id}: {e}") log.debug(f"Could not read progress for job {job.job_id}: {e}")
if communicate_task in done:
stdout_bytes, stderr_bytes = communicate_task.result()
break
if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED: if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED:
log.info(f"Cancellation detected for job {job.job_id}, terminating worker process") log.info(f"Cancellation detected for job {job.job_id}, terminating worker process")
process.terminate() process.terminate()
+1
View File
@@ -88,6 +88,7 @@ DEFAULT_DOWNLOAD_PARAMS = {
"require_audio": [], "require_audio": [],
"require_video": [], "require_video": [],
"require_subs": [], "require_subs": [],
"skip_missing": False,
"forced_subs": False, "forced_subs": False,
"forced_subs_only": False, "forced_subs_only": False,
"forced_s_lang": [], "forced_s_lang": [],
+6 -3
View File
@@ -1008,17 +1008,20 @@ async def download(request: web.Request) -> web.Response:
type: array type: array
items: items:
type: string type: string
description: Audio languages that must exist, else the job fails (default - []) description: Audio languages that must exist, else the job fails or, with skip_missing, skips the title (default - [])
require_video: require_video:
type: array type: array
items: items:
type: string type: string
description: Video languages that must exist, else the job fails (default - []) description: Video languages that must exist, else the job fails or, with skip_missing, skips the title (default - [])
require_subs: require_subs:
type: array type: array
items: items:
type: string type: string
description: Subtitle languages that must exist, else the job fails (default - []) description: Subtitle languages that must exist, else the job fails or, with skip_missing, skips the title (default - [])
skip_missing:
type: boolean
description: Skip a title that misses a required language or track and continue with the next title; the job fails when it skips every title (default - false)
forced_subs: forced_subs:
type: boolean type: boolean
description: Include forced subtitle tracks (default - false) description: Include forced subtitle tracks (default - false)