mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-14 04:55:07 +00:00
feat(dl): add --skip-missing to skip titles that miss a required language or track
A season run skips such a title and continues; one title, or a run that skips every title, still exits 1 (now also for -ado/-fso). REST jobs report skipped_titles.
This commit is contained in:
@@ -451,9 +451,10 @@ Make a download job. It requires `service` and `title_id`. Every other field is
|
||||
| `latest_episode` | boolean | `false` | Only the newest episode. |
|
||||
| `latest_episodes` | integer | `null` | Only the N newest episodes. A positive integer. When set, it overrides `latest_episode`. |
|
||||
| `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. A missing `s_lang` language gives a warning and the job continues. |
|
||||
| `require_audio` | string[] | `[]` | Audio languages that must exist. The job fails if one is missing, even with `best_available`. |
|
||||
| `require_video` | string[] | `[]` | Video languages that must exist. The job fails if one is missing, even with `best_available`. |
|
||||
| `require_subs` | string[] | `[]` | Subtitle languages that must exist. The job fails if one is missing. `s_lang` still decides what to keep. |
|
||||
| `require_audio` | string[] | `[]` | Audio languages that must exist. The job fails if one is missing, even with `best_available`, or skips the title with `skip_missing`. |
|
||||
| `require_video` | string[] | `[]` | Video languages that must exist. The job fails if one is missing, even with `best_available`, or skips the title with `skip_missing`. |
|
||||
| `require_subs` | string[] | `[]` | Subtitle languages that must exist. The job fails if one is missing, or skips the title with `skip_missing`. `s_lang` still decides what to keep. |
|
||||
| `skip_missing` | boolean | `false` | Skip a title that misses a required language or track, and continue with the next title. The job's `skipped_titles` lists each skip. The job fails when it has one title or skips every title. |
|
||||
| `forced_subs` | boolean | `false` | Include forced subtitles. |
|
||||
| `forced_subs_only` | boolean | `false` | Download forced subtitles and no other subtitle; implies `forced_subs`. Not valid with `no_subs`, `video_only`, `audio_only` or `chapters_only`. |
|
||||
| `forced_s_lang` | string[] | `[]` | Forced subtitle language(s); implies `forced_subs`. |
|
||||
@@ -585,6 +586,7 @@ Show download jobs, with optional filtering and sorting.
|
||||
"segments_total": 300,
|
||||
"speed": "5.2 MB/s",
|
||||
"skipped_subtitles": [],
|
||||
"skipped_titles": [],
|
||||
"input_prompt": null,
|
||||
"notice": null
|
||||
}
|
||||
@@ -853,6 +855,7 @@ Read the persisted history of jobs that reached a terminal state, newest first.
|
||||
"created_time": "2026-07-03T12:00:00+00:00",
|
||||
"completed_time": "2026-07-03T12:08:00+00:00",
|
||||
"output_files": ["/downloads/Example.S01E01.mkv"],
|
||||
"skipped_titles": [],
|
||||
"error_message": null,
|
||||
"parameters": { "...": "redacted" }
|
||||
}
|
||||
|
||||
@@ -162,6 +162,7 @@ full detail:
|
||||
| `segments_total` | number | Total segments for the track in progress. |
|
||||
| `speed` | string | Transfer speed of the track in progress, e.g. `4.20 MB/s`. |
|
||||
| `skipped_subtitles` | list | Subtitles that were skipped as non-fatal failures (when `skip_subtitle_errors` is set). |
|
||||
| `skipped_titles` | list | Titles that the run skipped because a requested track or language is missing, each `{title, reason}`. |
|
||||
| `input_prompt` | string or `null` | The prompt the service waits on. Answer it with `POST /api/download/jobs/{job_id}/input`. |
|
||||
| `notice` | string or `null` | A message the service shows while it waits for an approval on another device, such as a pair code. It needs no answer. |
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...]
|
||||
| `--require-audio` | - | Audio langs that must exist. The title fails even with `--best-available`. Does not select tracks. |
|
||||
| `--require-video` | - | Video langs that must exist. The title fails even with `--best-available`. Does not select tracks. |
|
||||
| `--require-subs` | - | Subtitle langs that must exist. Fails the title if one is missing. `--s-lang` still decides what to keep. |
|
||||
| `--skip-missing` | off | Skip a title that misses a required language or track, and continue with the next title. A run with one title, or a run that skips every title, still fails. |
|
||||
| `-fs`, `--forced-subs` | off | Include forced subtitle tracks. |
|
||||
| `-fso`, `--forced-subs-only` | off | Keep only forced subtitle tracks; drop the other subtitles. Implies `-fs`. |
|
||||
| `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. A `-` prefix excludes. |
|
||||
|
||||
@@ -531,6 +531,17 @@ The check runs against the full track list, before any language selection or exc
|
||||
`orig` resolves to the title language, and `--exact-lang` applies. Audio carried inside the
|
||||
video track counts. `--no-audio` and `--no-video` drop the matching requirement.
|
||||
|
||||
A failed title stops the whole run. To skip that title and come back for it later, add
|
||||
`--skip-missing`, or set `skip_missing: true` under `dl:` in `unshackle.yaml`. It also skips a
|
||||
title that has no track for a filter such as `-q`, `-v`, `-r`, `-a` or a language you request.
|
||||
A run with one title, or a run that skips every title, still fails. A REST API job lists each
|
||||
skipped title and the reason in `skipped_titles`.
|
||||
|
||||
```bash
|
||||
# download the episodes that have Korean audio, skip the others
|
||||
unshackle dl -al ko,en --require-audio ko --skip-missing -w S06 EXAMPLE 0ABC123
|
||||
```
|
||||
|
||||
### Forced subtitles and output format
|
||||
|
||||
- `-fs` / `--forced-subs`: include forced subtitle tracks (signs/foreign dialogue).
|
||||
@@ -777,7 +788,8 @@ Additional track-type flags:
|
||||
- A flag that drops the same tracks is an error, for example `-ado` with `--no-audio`
|
||||
or `-fso` with `-A`.
|
||||
- A title with no track of that kind stops a run of one title. A run of more than one
|
||||
title skips that title with a warning and continues.
|
||||
title skips that title with a warning and continues. The run fails when it skips
|
||||
every title.
|
||||
- unshackle reads the kind from the manifest or from the service. A track that
|
||||
neither of them marks as descriptive or forced does not match.
|
||||
|
||||
@@ -1214,6 +1226,7 @@ authoritative list.
|
||||
| `--require-audio` | | Audio languages that must exist. The title fails if one is missing. |
|
||||
| `--require-video` | | Video languages that must exist. The title fails if one is missing. |
|
||||
| `--best-available` | | Degrade gracefully instead of failing. |
|
||||
| `--skip-missing` | | Skip a title that misses a required language or track, and continue with the next title. |
|
||||
| `--output` | `-o` | Output directory for this run. |
|
||||
| `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. |
|
||||
| `--postscript` / `--no-postscript` | | Run a command after each output file, or run none. |
|
||||
|
||||
@@ -157,7 +157,7 @@ unshackle dl --s-lang en --forced-subs-only EXAMPLE 81234567
|
||||
|
||||
`-fso` does not drop video or audio. Add `-S` for a subtitle-only download. A title with
|
||||
no forced track stops a run of one title. A run of more than one title skips that title
|
||||
with a warning.
|
||||
with a warning, and fails when it skips every title.
|
||||
|
||||
## Track types: forced, SDH, and CC
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ Common config keys. This is a useful subset, and every `dl` flag works:
|
||||
| `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). |
|
||||
| `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. |
|
||||
| `best_available` | bool | `false` | Warn instead of failing when a requested resolution, range, video language, or audio language is missing, or when one DRM system does not license under `--all-drm`. The CLI flag is `--best-available`, also spelled `--warn-only`. |
|
||||
| `skip_missing` | bool | `false` | Skip a title that misses a required language (`require_audio`, `require_video`, `require_subs`) or a track that a filter asks for, and continue with the next title. A run with one title, or a run that skips every title, still fails. The CLI flag is `--skip-missing`. For REST API jobs, set it in the [`serve`](services.md#serve) block. |
|
||||
| `generate_dv` | bool | `false` | With `-r HYBRID`, make Dolby Vision from the HDR10+ metadata or the decoded video when a title has no DV track. With `-r HDR10` or `-r HDR10P`, make it for that base and ignore a DV track. See [Dolby Vision without a DV track](../../guide/downloading.md#dolby-vision-without-a-dv-track). |
|
||||
| `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. |
|
||||
| `no_folder` | bool | `false` | Do not create a per-title folder. |
|
||||
|
||||
@@ -213,6 +213,10 @@ subs = [p for p in sys.argv[i].removeprefix("--subs=").split("\n") if p]
|
||||
nothing about whether the season is complete on disk. Downloading a single episode fires
|
||||
the post-script for that season as soon as the episode lands.
|
||||
|
||||
A title that track selection skips (`--skip-missing`, `-ado`, `-fso`) counts as done. The
|
||||
`season` post-script still fires for the episodes that landed, so the folder can be missing
|
||||
the skipped episodes. A run that skips every title fails, and then no `season` post-script fires.
|
||||
|
||||
A title that fails stops the whole run. unshackle dispatches the `failure` post-script for
|
||||
that title and then returns. It downloads no further title, and no `season` or `run`
|
||||
post-script fires after that point. This applies to every title type. If one episode of S01
|
||||
|
||||
@@ -12,9 +12,11 @@ import logging
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
from unshackle.core.api.handlers import DEFAULT_DOWNLOAD_PARAMS
|
||||
from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY
|
||||
from unshackle.core.service import Service
|
||||
from unshackle.core.titles import Episode, Series
|
||||
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
|
||||
@@ -38,7 +40,7 @@ class Harness(dl):
|
||||
|
||||
|
||||
class FakeService:
|
||||
"""One episode per entry; ``ad`` adds a descriptive track, ``forced`` a forced subtitle."""
|
||||
"""One episode per entry; ``ad`` adds a descriptive track, ``forced`` a forced subtitle, ``ko`` Korean audio."""
|
||||
|
||||
def __init__(self, episodes: dict[int, dict[str, bool]]) -> None:
|
||||
self.episodes = episodes
|
||||
@@ -62,6 +64,8 @@ class FakeService:
|
||||
tracks.add(
|
||||
Audio(id_="ad", url="https://x/ad", language="en", codec=Audio.Codec.AAC, bitrate=1, descriptive=True)
|
||||
)
|
||||
if kinds.get("ko"):
|
||||
tracks.add(Audio(id_="ko", url="https://x/ko", language="ko", codec=Audio.Codec.AAC, bitrate=128_000))
|
||||
tracks.add(Subtitle(id_="s", url="https://x/s", language="en", codec=Subtitle.Codec.WebVTT))
|
||||
if kinds.get("forced"):
|
||||
tracks.add(Subtitle(id_="sf", url="https://x/sf", language="en", codec=Subtitle.Codec.WebVTT, forced=True))
|
||||
@@ -71,8 +75,8 @@ class FakeService:
|
||||
return []
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
if name.startswith("on_"):
|
||||
# the event hooks (on_track_downloaded, ...) the loop subscribes
|
||||
if name.startswith(("on_", "get_")):
|
||||
# the event hooks (on_track_downloaded, ...) and the DRM callbacks a run past selection probes
|
||||
return lambda *a, **k: None
|
||||
raise AttributeError(name)
|
||||
|
||||
@@ -82,16 +86,22 @@ def run(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
episodes: dict,
|
||||
service: Optional[FakeService] = None,
|
||||
to_end: bool = False,
|
||||
**flags: Any,
|
||||
) -> Any:
|
||||
"""Return the Selected ids, the exit code, or None for a run that completes."""
|
||||
monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self)))
|
||||
"""Return the Selected ids, the exit code, or None for a run that completes; to_end runs past selection."""
|
||||
if not to_end:
|
||||
monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self)))
|
||||
else:
|
||||
monkeypatch.setattr("unshackle.commands.dl.download_tracks_in_passes", lambda *a, **k: None)
|
||||
monkeypatch.setattr("unshackle.core.providers.search_metadata", lambda *a, **k: None)
|
||||
cmd = Harness.__new__(Harness)
|
||||
cmd.log = logging.getLogger("test_kind_only")
|
||||
cmd.service = "FAKE"
|
||||
cmd.tmdb_searched = True
|
||||
cmd.completed_files = []
|
||||
cmd.skipped_titles = []
|
||||
cmd.skipped_subtitles = []
|
||||
kwargs: dict[str, Any] = {}
|
||||
for name, param in inspect.signature(dl.result).parameters.items():
|
||||
if name in DEFAULT_DOWNLOAD_PARAMS:
|
||||
@@ -101,12 +111,17 @@ def run(
|
||||
kwargs.update(quality=[], vcodec=[], acodec=[], range_=[], wanted=[], cdm_only=None, downloads=1)
|
||||
kwargs.update(flags)
|
||||
caplog.set_level(logging.WARNING, logger="test_kind_only")
|
||||
service = service or FakeService(episodes)
|
||||
if to_end:
|
||||
service.session = requests.Session()
|
||||
try:
|
||||
cmd.result(cast(Service, service or FakeService(episodes)), **kwargs)
|
||||
cmd.result(cast(Service, service), **kwargs)
|
||||
except Selected as sel:
|
||||
return sel.ids
|
||||
except SystemExit as exc:
|
||||
return exc.code
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
return None
|
||||
|
||||
|
||||
@@ -145,8 +160,9 @@ def test_subs_only_with_no_matching_subtitle_exits_a_single_title_and_skips_in_a
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
assert run(monkeypatch, caplog, {1: {}}, subs_only=True, s_lang=["fr"]) == 1
|
||||
assert run(monkeypatch, caplog, {1: {}, 2: {}}, subs_only=True, s_lang=["fr"]) is None
|
||||
assert caplog.text.count("There's no track left for the selected options. Skipping Show S01E") == 2
|
||||
assert run(monkeypatch, caplog, {1: {}, 2: {}}, subs_only=True, s_lang=["fr"]) == 1
|
||||
assert caplog.text.count("Skipping Show S01E") == 2
|
||||
assert "Skipping Show S01E02: There's no track left for the selected options." in caplog.text
|
||||
|
||||
|
||||
def test_a_multi_title_run_skips_the_title_and_continues(
|
||||
@@ -157,13 +173,158 @@ def test_a_multi_title_run_skips_the_title_and_continues(
|
||||
assert "Skipping Show S01E01" in caplog.text
|
||||
|
||||
|
||||
def test_a_multi_title_run_with_every_title_skipped_completes(
|
||||
def test_a_multi_title_run_with_every_title_skipped_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
assert run(monkeypatch, caplog, {1: {}, 2: {}}, forced_subs_only=True) is None
|
||||
assert run(monkeypatch, caplog, {1: {}, 2: {}}, forced_subs_only=True) == 1
|
||||
assert caplog.text.count("Skipping Show S01E") == 2
|
||||
|
||||
|
||||
def test_a_required_language_miss_stops_the_run_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
sink: list[dict] = []
|
||||
assert run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, require_audio=["ko"], progress_sink=sink.append) == 1
|
||||
assert "Skipping" not in caplog.text
|
||||
assert "Show S01E01: Required audio language(s) not found: ko" in caplog.text
|
||||
assert not any("skipped_titles" in event for event in sink)
|
||||
|
||||
|
||||
def test_skip_missing_skips_the_title_and_reports_it(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
sink: list[dict] = []
|
||||
ids = run(
|
||||
monkeypatch,
|
||||
caplog,
|
||||
{1: {}, 2: {"ko": True}},
|
||||
require_audio=["ko"],
|
||||
skip_missing=True,
|
||||
progress_sink=sink.append,
|
||||
)
|
||||
assert ids == {"video": ["v"], "audio": ["a"], "subs": ["s"]}
|
||||
assert "Skipping Show S01E01: Required audio language(s) not found: ko" in caplog.text
|
||||
assert {"skipped_titles": [{"title": "Show S01E01", "reason": "Required audio language(s) not found: ko"}]} in sink
|
||||
|
||||
|
||||
def test_require_audio_video_and_subs_work_together(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
ids = run(monkeypatch, caplog, {1: {}}, require_audio=["en"], require_video=["en"], require_subs=["en"])
|
||||
assert ids == {"video": ["v"], "audio": ["a"], "subs": ["s"]}
|
||||
assert run(monkeypatch, caplog, {1: {}}, require_audio=["ko"], require_video=["ko"], require_subs=["ko"]) == 1
|
||||
assert (
|
||||
"Required audio language(s) not found: ko; Required video language(s) not found: ko; "
|
||||
"Required subtitle language(s) not found: ko"
|
||||
) in caplog.text
|
||||
assert run(monkeypatch, caplog, {1: {}}, require_audio=["en"], require_video=["ko"], require_subs=["en"]) == 1
|
||||
assert "Show S01E01: Required video language(s) not found: ko" in caplog.text
|
||||
|
||||
|
||||
def test_skip_missing_skips_a_filter_miss_and_continues(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
ids = run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, a_lang=["ko"], skip_missing=True)
|
||||
assert ids == {"video": ["v"], "audio": ["ko"], "subs": ["s"]}
|
||||
assert "Skipping Show S01E01" in caplog.text
|
||||
|
||||
|
||||
def test_skip_missing_still_fails_a_run_that_skips_every_title(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
assert run(monkeypatch, caplog, {1: {}, 2: {}}, require_audio=["ko"], skip_missing=True) == 1
|
||||
assert "Skipped all 2 titles" in caplog.text
|
||||
assert run(monkeypatch, caplog, {1: {}}, require_audio=["ko"], skip_missing=True) == 1
|
||||
|
||||
|
||||
class LangService(FakeService):
|
||||
"""Each episode names its original language and its audio languages; the video is in the original language."""
|
||||
|
||||
def get_titles_cached(self) -> Series:
|
||||
return Series(
|
||||
Episode(id_=f"show-s01-e{n:03}", service=FakeService, title="Show", season=1, number=n, language=ep["orig"])
|
||||
for n, ep in self.episodes.items()
|
||||
)
|
||||
|
||||
def get_tracks(self, title: Episode) -> Tracks:
|
||||
ep = self.episodes[title.number]
|
||||
tracks = Tracks()
|
||||
tracks.add(Video(id_="v", url="https://x/v", language=ep["orig"] or "en", codec=Video.Codec.AVC))
|
||||
for lang in ep["audio"]:
|
||||
tracks.add(Audio(id_=lang, url=f"https://x/{lang}", language=lang, codec=Audio.Codec.AAC, bitrate=128_000))
|
||||
return tracks
|
||||
|
||||
|
||||
JA_EN = {"orig": "ja", "audio": ["ja", "en"]}
|
||||
FR = {"orig": "ja", "audio": ["fr"]}
|
||||
EVERY = {"orig": "ja", "audio": ["ja", "ko", "en"]}
|
||||
ORIG_KO_EN = ["orig", "ko", "en"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"first,flags,outcome,logged",
|
||||
[
|
||||
(JA_EN, {}, 1, "Show S01E01: ko not found in audio tracks"),
|
||||
(JA_EN, {"skip_missing": True}, ["ja", "ko", "en"], "Skipping Show S01E01: ko not found in audio tracks"),
|
||||
({"orig": "ja", "audio": ["ko", "en"]}, {"skip_missing": True}, ["ja", "ko", "en"], "S01E01: orig not found"),
|
||||
(JA_EN, {"best_available": True}, ["ja", "en"], "ko not found in audio tracks, continuing with: orig, en"),
|
||||
(FR, {"best_available": True}, 1, "orig, ko, en not found in audio tracks and no fallback available"),
|
||||
(FR, {"best_available": True, "skip_missing": True}, ["ja", "ko", "en"], "Skipping Show S01E01: orig, ko, en"),
|
||||
(JA_EN, {"best_available": True, "require_audio": ["ko"]}, 1, "Required audio language(s) not found: ko"),
|
||||
(
|
||||
JA_EN,
|
||||
{"best_available": True, "require_audio": ["ko"], "skip_missing": True},
|
||||
["ja", "ko", "en"],
|
||||
"Skipping Show S01E01: Required audio language(s) not found: ko",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_several_audio_languages_skip_an_episode_that_misses_one(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
first: dict,
|
||||
flags: dict[str, Any],
|
||||
outcome: Any,
|
||||
logged: str,
|
||||
) -> None:
|
||||
"""-al orig,ko,en on a season where only the second episode has all three."""
|
||||
episodes = {1: first, 2: EVERY}
|
||||
result = run(monkeypatch, caplog, episodes, service=LangService(episodes), a_lang=ORIG_KO_EN, **flags)
|
||||
assert (result if outcome == 1 else result["audio"]) == outcome
|
||||
assert logged in caplog.text
|
||||
|
||||
|
||||
def test_several_audio_languages_fail_a_season_that_misses_one_everywhere(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
episodes = {1: JA_EN, 2: JA_EN}
|
||||
assert run(monkeypatch, caplog, episodes, service=LangService(episodes), a_lang=ORIG_KO_EN, skip_missing=True) == 1
|
||||
assert caplog.text.count("ko not found in audio tracks") == 2
|
||||
assert "Skipped all 2 titles" in caplog.text
|
||||
|
||||
|
||||
def test_a_partly_skipped_run_completes_and_lists_the_skips(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
episodes = {1: {}, 2: {"ko": True}}
|
||||
assert run(monkeypatch, caplog, episodes, to_end=True, a_lang=["ko"], skip_missing=True, skip_dl=True) is None
|
||||
assert "Skipped 1 of 2 titles: Show S01E01" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_mux,settled", [(False, 2), (True, 1)])
|
||||
def test_a_skipped_title_settles_its_post_script_group(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, no_mux: bool, settled: int
|
||||
) -> None:
|
||||
"""The season event waits on every title of a group; a skipped title counts as settled, once at count time."""
|
||||
import unshackle.commands.dl as dl_module
|
||||
|
||||
seen: list[str] = []
|
||||
group = dl_module.post_script_group
|
||||
monkeypatch.setattr(dl_module, "post_script_group", lambda t: seen.append(str(t)) or group(t))
|
||||
run(monkeypatch, caplog, {1: {}, 2: {"ko": True}}, a_lang=["ko"], skip_missing=True, no_mux=no_mux)
|
||||
assert seen.count("Show S01E01") == settled
|
||||
|
||||
|
||||
class LcevcService(FakeService):
|
||||
"""A 540p base video and the 1080p LCEVC enhancement layer that decodes on top of it."""
|
||||
|
||||
|
||||
@@ -626,3 +626,86 @@ def test_client_job_view_hides_debug_fields_without_debug_api() -> None:
|
||||
assert client_job_view(job, off) == {"job_id": "j"}
|
||||
assert client_job_view(job, None) == {"job_id": "j"}
|
||||
assert client_job_view(job, on) == job
|
||||
|
||||
|
||||
FAKE_SKIPPING_WORKER = """
|
||||
import sys
|
||||
from unshackle.core.api import download_worker
|
||||
|
||||
|
||||
def skip_then_stop(*args, progress_callback, **kwargs):
|
||||
progress_callback({"skipped_titles": [{"title": "Show S01E02", "reason": "missing ko"}]})
|
||||
raise RuntimeError("Download failed: Skipped all 2 titles")
|
||||
|
||||
|
||||
download_worker.perform_download = skip_then_stop
|
||||
sys.exit(download_worker.main(["worker"] + sys.argv[1:]))
|
||||
"""
|
||||
|
||||
|
||||
async def test_a_skip_written_just_before_the_worker_exits_reaches_the_job(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, manager: DownloadQueueManager
|
||||
) -> None:
|
||||
import asyncio
|
||||
|
||||
from unshackle.core.api import download_manager
|
||||
|
||||
worker = tmp_path / "worker.py"
|
||||
worker.write_text(FAKE_SKIPPING_WORKER, encoding="utf-8")
|
||||
spawn = asyncio.create_subprocess_exec
|
||||
|
||||
async def spawn_fake_worker(executable: str, *args: str, **kwargs: Any) -> Any:
|
||||
return await spawn(executable, str(worker), *args[2:], **kwargs)
|
||||
|
||||
monkeypatch.setattr(download_manager.asyncio, "create_subprocess_exec", spawn_fake_worker)
|
||||
job = manager.create_job("EXAMPLE", "t")
|
||||
with pytest.raises(Exception, match="Skipped all 2 titles"):
|
||||
await asyncio.wait_for(manager.run_download_async(job), timeout=20)
|
||||
assert job.skipped_titles == [{"title": "Show S01E02", "reason": "missing ko"}]
|
||||
assert job.to_dict()["skipped_titles"] == job.skipped_titles
|
||||
|
||||
|
||||
def test_a_stopped_run_names_its_reason_in_the_job_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import sys
|
||||
|
||||
import click
|
||||
|
||||
import unshackle.commands.dl as dl_module
|
||||
from unshackle.core.api import download_manager, handlers
|
||||
from unshackle.core.api.errors import APIError
|
||||
from unshackle.core.services import Services
|
||||
|
||||
class FakeDl:
|
||||
cli = dl_module.dl.cli
|
||||
|
||||
def __init__(self, ctx: click.Context, **kwargs: object) -> None:
|
||||
self.stop_reason = None
|
||||
|
||||
def result(self, **kwargs: object) -> None:
|
||||
self.stop_reason = "Show S01E01: Required audio language(s) not found: ko"
|
||||
sys.exit(1)
|
||||
|
||||
class FakeService:
|
||||
def __init__(self, ctx: click.Context) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(dl_module, "dl", FakeDl)
|
||||
monkeypatch.setattr(Services, "get_path", staticmethod(lambda s: tmp_path))
|
||||
monkeypatch.setattr(Services, "load", staticmethod(lambda s: FakeService))
|
||||
monkeypatch.setattr(handlers, "load_full_cdm", lambda *a: None)
|
||||
|
||||
with pytest.raises(APIError, match=r"^Download failed: Show S01E01: Required audio language\(s\) not found: ko$"):
|
||||
download_manager.perform_download("job-1", "EXAMPLE1", "t1", {})
|
||||
|
||||
|
||||
def test_job_history_keeps_the_skipped_titles(tmp_path: Path, manager: DownloadQueueManager) -> None:
|
||||
import json
|
||||
|
||||
from unshackle.core.api.download_manager import record_job_history
|
||||
|
||||
job = manager.create_job("EXAMPLE", "t")
|
||||
job.status = JobStatus.COMPLETED
|
||||
job.skipped_titles = [{"title": "Show S01E02", "reason": "missing ko"}]
|
||||
record_job_history(job)
|
||||
entry = json.loads((tmp_path / "api_history.jsonl").read_text(encoding="utf-8").splitlines()[-1])
|
||||
assert entry["skipped_titles"] == job.skipped_titles
|
||||
|
||||
+101
-61
@@ -158,6 +158,13 @@ if TYPE_CHECKING:
|
||||
from unshackle.core.remote_service import ServerKeySource
|
||||
|
||||
|
||||
class SkippedTitle(TypedDict):
|
||||
"""A title skipped because a requested track or language is missing. Accumulated as ``dl.skipped_titles``."""
|
||||
|
||||
title: str
|
||||
reason: str
|
||||
|
||||
|
||||
class SkippedSubtitle(TypedDict):
|
||||
"""A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``,
|
||||
one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable."""
|
||||
@@ -979,6 +986,12 @@ class dl:
|
||||
default=[],
|
||||
help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.",
|
||||
)
|
||||
@click.option(
|
||||
"--skip-missing",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Skip a title that misses a required language or track and continue with the next title.",
|
||||
)
|
||||
@click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.")
|
||||
@click.option(
|
||||
"-fso",
|
||||
@@ -1383,6 +1396,9 @@ class dl:
|
||||
# Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can
|
||||
# report which weren't available without parsing the console output. See SkippedSubtitle.
|
||||
self.skipped_subtitles: list[SkippedSubtitle] = []
|
||||
self.skipped_titles: list[SkippedTitle] = []
|
||||
# Why a missing track or language stopped the run, for the API job error.
|
||||
self.stop_reason: Optional[str] = None
|
||||
# result() catches a download-worker failure, reports it, and returns rather than
|
||||
# re-raising (so the CLI still exits cleanly). Keep it so an embedding caller (the API
|
||||
# worker) can tell the title did not complete instead of reading it as a success.
|
||||
@@ -1765,6 +1781,7 @@ class dl:
|
||||
require_audio: list[str],
|
||||
require_video: list[str],
|
||||
require_subs: list[str],
|
||||
skip_missing: bool,
|
||||
forced_subs: bool,
|
||||
forced_subs_only: bool,
|
||||
forced_s_lang: list[str],
|
||||
@@ -2348,15 +2365,20 @@ class dl:
|
||||
for folder, context in post_script_last.get(group_key, {}).items():
|
||||
dispatch("success", "season", season_context(context, folder), postscript)
|
||||
|
||||
def skip_title(title: Any, reason: str) -> None:
|
||||
"""Skip a title that has no track of the kind an only flag selects.
|
||||
def skip_title(title: Any, reason: str, kind_only: bool = False) -> None:
|
||||
"""Skip a title that has no track the selection asks for, and report it to the API job.
|
||||
|
||||
A run with one title stops instead, because it has nothing else to download.
|
||||
Only a kind-only miss (-ado, -fso, no track left) skips without --skip-missing; any other miss
|
||||
stops the run. A run with one title always stops, because it has nothing else to download.
|
||||
"""
|
||||
if wanted_total <= 1:
|
||||
self.log.error(reason)
|
||||
if wanted_total <= 1 or not (kind_only or skip_missing):
|
||||
self.stop_reason = f"{title}: {reason}"
|
||||
self.log.error(self.stop_reason)
|
||||
sys.exit(1)
|
||||
self.log.warning(f"{reason} Skipping {title}")
|
||||
self.skipped_titles.append(SkippedTitle(title=str(title), reason=reason))
|
||||
if progress_sink:
|
||||
progress_sink({"skipped_titles": list(self.skipped_titles)})
|
||||
self.log.warning(f"Skipping {title}: {reason}")
|
||||
if not no_mux:
|
||||
settle_post_script_group(title)
|
||||
|
||||
@@ -2641,6 +2663,7 @@ class dl:
|
||||
elif proxy_download:
|
||||
console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}")
|
||||
|
||||
missing_required: list[str] = []
|
||||
for kind, required, available in (
|
||||
(
|
||||
"audio",
|
||||
@@ -2659,10 +2682,11 @@ class dl:
|
||||
],
|
||||
),
|
||||
):
|
||||
missing_required = missing_required_langs(required, available, title.language, exact=exact_lang)
|
||||
if missing_required:
|
||||
self.log.error(f"Required {kind} language(s) not found: {', '.join(missing_required)}")
|
||||
sys.exit(1)
|
||||
if missing := missing_required_langs(required, available, title.language, exact=exact_lang):
|
||||
missing_required.append(f"Required {kind} language(s) not found: {', '.join(missing)}")
|
||||
if missing_required:
|
||||
skip_title(title, "; ".join(missing_required))
|
||||
continue
|
||||
|
||||
if not keep_videos:
|
||||
title.tracks.videos = []
|
||||
@@ -2690,16 +2714,18 @@ class dl:
|
||||
for codec in missing_codecs:
|
||||
self.log.warning(f"Skipping {codec.name} video tracks as none are available.")
|
||||
if not title.tracks.videos:
|
||||
self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {', '.join(c.name for c in vcodec)} Video Track...")
|
||||
continue
|
||||
elif keep_videos:
|
||||
title.tracks.select_video(lambda x: x.codec != Video.Codec.LCEVC)
|
||||
|
||||
if keep_videos and generated_dv_bases:
|
||||
title.tracks.videos = keep_generated_dv_bases(title.tracks.videos, generated_dv_bases)
|
||||
if not any(v.range in generated_dv_bases for v in title.tracks.videos):
|
||||
self.log.error(f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track...")
|
||||
sys.exit(1)
|
||||
skip_title(
|
||||
title, f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track..."
|
||||
)
|
||||
continue
|
||||
|
||||
if keep_videos and range_:
|
||||
# Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks
|
||||
@@ -2712,8 +2738,8 @@ class dl:
|
||||
if Video.Range.HDR10P in missing_ranges and dv_has_hdr10p:
|
||||
self.log.warning("A DV track carries HDR10+: use -r DV or -r HYBRID.")
|
||||
if not title.tracks.videos:
|
||||
self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {', '.join(r.name for r in range_)} Video Track...")
|
||||
continue
|
||||
|
||||
if keep_videos and vbitrate:
|
||||
if any(r == Video.Range.HYBRID for r in range_):
|
||||
@@ -2723,21 +2749,21 @@ class dl:
|
||||
lambda x: x.range == Video.Range.DV or bool(x.bitrate and x.bitrate // 1000 == vbitrate)
|
||||
)
|
||||
if not any(x.range != Video.Range.DV for x in title.tracks.videos):
|
||||
self.log.error(f"There's no {vbitrate}kbps Video Track...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {vbitrate}kbps Video Track...")
|
||||
continue
|
||||
else:
|
||||
title.tracks.select_video(lambda x: bool(x.bitrate and x.bitrate // 1000 == vbitrate))
|
||||
if not title.tracks.videos:
|
||||
self.log.error(f"There's no {vbitrate}kbps Video Track...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {vbitrate}kbps Video Track...")
|
||||
continue
|
||||
|
||||
if keep_videos and vbitrate_min is not None and vbitrate_max is not None:
|
||||
title.tracks.select_video(
|
||||
lambda x: bool(x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max)
|
||||
)
|
||||
if not title.tracks.videos:
|
||||
self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...")
|
||||
continue
|
||||
|
||||
if keep_videos and video_excl_r:
|
||||
drop = matched_language_tags(
|
||||
@@ -2745,8 +2771,8 @@ class dl:
|
||||
)
|
||||
title.tracks.select_video(lambda x: str(x.language) not in drop)
|
||||
if not title.tracks.videos:
|
||||
self.log.error(f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...")
|
||||
continue
|
||||
|
||||
effective_video_lang = v_lang or lang
|
||||
video_languages = [lang for lang in effective_video_lang if lang != "best"]
|
||||
@@ -2776,20 +2802,23 @@ class dl:
|
||||
)
|
||||
processed_video_lang = remaining_v_langs
|
||||
else:
|
||||
self.log.error(f"{missing_str} not found in video tracks and no fallback available")
|
||||
sys.exit(1)
|
||||
skip_title(
|
||||
title, f"{missing_str} not found in video tracks and no fallback available"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
self.log.error(missing_str + " not found in video tracks")
|
||||
sys.exit(1)
|
||||
skip_title(title, missing_str + " not found in video tracks")
|
||||
continue
|
||||
title.tracks.videos = title.tracks.by_language(
|
||||
title.tracks.videos, processed_video_lang, exact_match=exact_lang
|
||||
)
|
||||
if not title.tracks.videos:
|
||||
# empty only when 'orig' was the sole request and did not resolve
|
||||
self.log.error(
|
||||
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..."
|
||||
skip_title(
|
||||
title,
|
||||
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track...",
|
||||
)
|
||||
sys.exit(1)
|
||||
continue
|
||||
|
||||
has_hybrid = any(r == Video.Range.HYBRID for r in range_)
|
||||
non_hybrid_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID]
|
||||
@@ -2854,8 +2883,8 @@ class dl:
|
||||
f"There's no {res_list} Video Track{plural}, continuing with available qualities..."
|
||||
)
|
||||
else:
|
||||
self.log.error(f"There's no {res_list} Video Track{plural}...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {res_list} Video Track{plural}...")
|
||||
continue
|
||||
|
||||
pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else []
|
||||
if keep_videos and has_hybrid:
|
||||
@@ -2968,8 +2997,8 @@ class dl:
|
||||
fallback_selected.append(match)
|
||||
title.tracks.videos = fallback_selected
|
||||
else:
|
||||
self.log.error(gate_failure)
|
||||
sys.exit(1)
|
||||
skip_title(title, gate_failure)
|
||||
continue
|
||||
|
||||
fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang)
|
||||
if keep_subtitles and s_excl_r:
|
||||
@@ -3031,27 +3060,31 @@ class dl:
|
||||
if forced_subs_only:
|
||||
title.tracks.select_subtitles(lambda x: x.forced)
|
||||
if not title.tracks.subtitles:
|
||||
skip_title(title, "There's no forced subtitle track for the selected languages.")
|
||||
skip_title(
|
||||
title,
|
||||
"There's no forced subtitle track for the selected languages.",
|
||||
kind_only=True,
|
||||
)
|
||||
continue
|
||||
|
||||
if keep_audio and audio_description_only:
|
||||
title.tracks.select_audio(lambda x: x.descriptive)
|
||||
if not title.tracks.audio:
|
||||
skip_title(title, "There's no audio description track.")
|
||||
skip_title(title, "There's no audio description track.", kind_only=True)
|
||||
continue
|
||||
# might have no audio tracks if part of the video, e.g. transport stream hls
|
||||
if keep_audio and len(title.tracks.audio) > 0:
|
||||
if not audio_description:
|
||||
title.tracks.select_audio(lambda x: not x.descriptive)
|
||||
if not title.tracks.audio:
|
||||
self.log.error("Audio description only, use --audio-description...")
|
||||
sys.exit(1)
|
||||
skip_title(title, "Audio description only, use --audio-description...")
|
||||
continue
|
||||
if acodec:
|
||||
title.tracks.select_audio(lambda x: x.codec in acodec)
|
||||
if not title.tracks.audio:
|
||||
codec_names = ", ".join(c.name for c in acodec)
|
||||
self.log.error(f"No audio tracks matching codecs: {codec_names}")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"No audio tracks matching codecs: {codec_names}")
|
||||
continue
|
||||
if channels:
|
||||
title.tracks.select_audio(
|
||||
lambda x: bool(
|
||||
@@ -3059,33 +3092,33 @@ class dl:
|
||||
)
|
||||
)
|
||||
if not title.tracks.audio:
|
||||
self.log.error(f"There's no {channels} Audio Track...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {channels} Audio Track...")
|
||||
continue
|
||||
if no_atmos:
|
||||
title.tracks.audio = [x for x in title.tracks.audio if not x.atmos]
|
||||
if not title.tracks.audio:
|
||||
self.log.error("No non-Atmos audio tracks available...")
|
||||
sys.exit(1)
|
||||
skip_title(title, "No non-Atmos audio tracks available...")
|
||||
continue
|
||||
if abitrate:
|
||||
title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate)
|
||||
if not title.tracks.audio:
|
||||
self.log.error(f"There's no {abitrate}kbps Audio Track...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"There's no {abitrate}kbps Audio Track...")
|
||||
continue
|
||||
if abitrate_min is not None and abitrate_max is not None:
|
||||
title.tracks.select_audio(
|
||||
lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max
|
||||
)
|
||||
if not title.tracks.audio:
|
||||
self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...")
|
||||
continue
|
||||
if audio_excl_r:
|
||||
drop = matched_language_tags(
|
||||
audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True
|
||||
)
|
||||
title.tracks.select_audio(lambda x: str(x.language) not in drop)
|
||||
if not title.tracks.audio:
|
||||
self.log.error(f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...")
|
||||
continue
|
||||
audio_languages = a_lang or lang
|
||||
if audio_languages:
|
||||
processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang)
|
||||
@@ -3115,13 +3148,13 @@ class dl:
|
||||
)
|
||||
processed_lang = remaining
|
||||
else:
|
||||
self.log.error(
|
||||
f"{missing_str} not found in {audio_noun} and no fallback available"
|
||||
skip_title(
|
||||
title, f"{missing_str} not found in {audio_noun} and no fallback available"
|
||||
)
|
||||
sys.exit(1)
|
||||
continue
|
||||
else:
|
||||
self.log.error(f"{missing_str} not found in {audio_noun}")
|
||||
sys.exit(1)
|
||||
skip_title(title, f"{missing_str} not found in {audio_noun}")
|
||||
continue
|
||||
|
||||
title.tracks.audio = select_best_audio(
|
||||
title.tracks.audio, processed_lang, acodec, audio_description, exact_lang
|
||||
@@ -3129,11 +3162,11 @@ class dl:
|
||||
if not title.tracks.audio and not embedded_langs:
|
||||
# empty when 'orig' was the sole request, did not resolve, and the
|
||||
# video carries no audio of its own to fall back on
|
||||
self.log.error(
|
||||
f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} "
|
||||
f"Audio Track, cannot continue..."
|
||||
skip_title(
|
||||
title,
|
||||
f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} Audio Track...",
|
||||
)
|
||||
sys.exit(1)
|
||||
continue
|
||||
|
||||
kept_tracks = []
|
||||
if keep_videos:
|
||||
@@ -3149,7 +3182,7 @@ class dl:
|
||||
|
||||
title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url)
|
||||
if not (title.tracks.videos or title.tracks.audio or title.tracks.subtitles or title.tracks.chapters):
|
||||
skip_title(title, "There's no track left for the selected options.")
|
||||
skip_title(title, "There's no track left for the selected options.", kind_only=True)
|
||||
continue
|
||||
|
||||
selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True)
|
||||
@@ -4063,6 +4096,13 @@ class dl:
|
||||
self.wait_vault_writes()
|
||||
if export_path:
|
||||
self.log_export_summary()
|
||||
if self.skipped_titles:
|
||||
if len(self.skipped_titles) == wanted_total:
|
||||
self.stop_reason = f"Skipped all {wanted_total} titles"
|
||||
self.log.error(self.stop_reason)
|
||||
sys.exit(1)
|
||||
skipped = ", ".join(t["title"] for t in self.skipped_titles)
|
||||
self.log.warning(f"Skipped {len(self.skipped_titles)} of {wanted_total} titles: {skipped}")
|
||||
dl_time = time_elapsed_since(start_time)
|
||||
|
||||
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
||||
|
||||
@@ -137,6 +137,8 @@ class DownloadJob:
|
||||
# Subtitles skipped under skip_subtitle_errors (non-fatal). Each entry is a dl.SkippedSubtitle
|
||||
# dict (id / language / title) so a client can report which weren't available.
|
||||
skipped_subtitles: List[Dict[str, Any]] = field(default_factory=list)
|
||||
# Titles skipped because a requested track or language is missing; each a dl.SkippedTitle (title / reason).
|
||||
skipped_titles: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
cancel_event: threading.Event = field(default_factory=threading.Event)
|
||||
|
||||
@@ -168,6 +170,7 @@ class DownloadJob:
|
||||
"segments_total": self.segments_total,
|
||||
"speed": self.speed,
|
||||
"skipped_subtitles": self.skipped_subtitles,
|
||||
"skipped_titles": self.skipped_titles,
|
||||
"input_prompt": self.input_prompt,
|
||||
"notice": self.notice,
|
||||
}
|
||||
@@ -232,6 +235,7 @@ def record_job_history(job: DownloadJob) -> None:
|
||||
"created_time": job.created_time.isoformat(),
|
||||
"completed_time": job.completed_time.isoformat() if job.completed_time else None,
|
||||
"output_files": job.output_files,
|
||||
"skipped_titles": job.skipped_titles,
|
||||
"parameters": redact_parameters(job.parameters),
|
||||
"error_message": _redact_text(job.error_message, job.parameters),
|
||||
}
|
||||
@@ -523,6 +527,8 @@ def perform_download(
|
||||
log.error(f"Download exited with code {exc.code}")
|
||||
log.debug("Stdout: %s", stdout_capture.getvalue())
|
||||
log.debug("Stderr: %s", stderr_capture.getvalue())
|
||||
if reason := getattr(dl_instance, "stop_reason", None):
|
||||
raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed: {reason}")
|
||||
raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed with exit code {exc.code}")
|
||||
|
||||
except Exception as exc: # noqa: BLE001 - propagate to caller
|
||||
@@ -961,11 +967,8 @@ class DownloadQueueManager:
|
||||
try:
|
||||
while True:
|
||||
done, _ = await asyncio.wait({communicate_task}, timeout=0.5)
|
||||
if communicate_task in done:
|
||||
stdout_bytes, stderr_bytes = communicate_task.result()
|
||||
break
|
||||
|
||||
# Check for progress updates (skip re-parse when the file hasn't changed since last tick)
|
||||
# Read progress before the exit check so the worker's last write lands (skip an unchanged file)
|
||||
try:
|
||||
stat = os.stat(progress_path)
|
||||
stat_key = (stat.st_mtime_ns, stat.st_size)
|
||||
@@ -1003,6 +1006,8 @@ class DownloadQueueManager:
|
||||
job.notice = progress_data["notice"]
|
||||
if progress_data.get("skipped_subtitles"):
|
||||
job.skipped_subtitles = progress_data["skipped_subtitles"]
|
||||
if progress_data.get("skipped_titles"):
|
||||
job.skipped_titles = progress_data["skipped_titles"]
|
||||
if "progress" in progress_data:
|
||||
new_progress = float(progress_data["progress"])
|
||||
if new_progress != job.progress:
|
||||
@@ -1017,6 +1022,10 @@ class DownloadQueueManager:
|
||||
except (OSError, json.JSONDecodeError, ValueError) as e:
|
||||
log.debug(f"Could not read progress for job {job.job_id}: {e}")
|
||||
|
||||
if communicate_task in done:
|
||||
stdout_bytes, stderr_bytes = communicate_task.result()
|
||||
break
|
||||
|
||||
if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED:
|
||||
log.info(f"Cancellation detected for job {job.job_id}, terminating worker process")
|
||||
process.terminate()
|
||||
|
||||
@@ -88,6 +88,7 @@ DEFAULT_DOWNLOAD_PARAMS = {
|
||||
"require_audio": [],
|
||||
"require_video": [],
|
||||
"require_subs": [],
|
||||
"skip_missing": False,
|
||||
"forced_subs": False,
|
||||
"forced_subs_only": False,
|
||||
"forced_s_lang": [],
|
||||
|
||||
@@ -1008,17 +1008,20 @@ async def download(request: web.Request) -> web.Response:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Audio languages that must exist, else the job fails (default - [])
|
||||
description: Audio languages that must exist, else the job fails or, with skip_missing, skips the title (default - [])
|
||||
require_video:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Video languages that must exist, else the job fails (default - [])
|
||||
description: Video languages that must exist, else the job fails or, with skip_missing, skips the title (default - [])
|
||||
require_subs:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Subtitle languages that must exist, else the job fails (default - [])
|
||||
description: Subtitle languages that must exist, else the job fails or, with skip_missing, skips the title (default - [])
|
||||
skip_missing:
|
||||
type: boolean
|
||||
description: Skip a title that misses a required language or track and continue with the next title; the job fails when it skips every title (default - false)
|
||||
forced_subs:
|
||||
type: boolean
|
||||
description: Include forced subtitle tracks (default - false)
|
||||
|
||||
Reference in New Issue
Block a user