diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 74ab1139..49f0de14 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -480,7 +480,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. | | `download_processes` | int | `1` | Split a track's segments across this many processes. Only engages for large batches. | | `downloads` | int | `1` | Concurrent tracks. | -| `worst` | boolean | `false` | Pick the lowest-bitrate video at each resolution in `quality`, or at the lowest resolution if `quality` is empty. With `worst`, `HYBRID` in `range` requires `quality`. | +| `worst` | boolean | `false` | Pick the lowest-bitrate video at each resolution in `quality`, or at the lowest resolution if `quality` is empty. | | `best_available` | boolean | `false` | Fall back to best available. With `all_drm`, a DRM system that does not license is a warning. | | `generate_dv` | boolean | `false` | With `HYBRID` in `range`, make Dolby Vision from the HDR10+ metadata or the decoded video when a title has no DV track. With `HDR10` or `HDR10P` in `range` and no `HYBRID`, make it for that base and ignore a DV track. With none of the three in `range`, the field has no effect. | | `repack` | boolean | `false` | Add REPACK tag. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 6018a262..0ad6268d 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -234,7 +234,7 @@ order the service used, then renumbers the episodes into the order you asked for | `--slow` | - | Inter-title delay. Bare `--slow` = 60-120s; `--slow 20-40` = custom range (minimum 20s). | !!! warning "Some flags cannot be combined" - `--select-titles` and `--wanted` are mutually exclusive. With `--worst`, hybrid mode needs `--quality`. `--vbitrate` and `--vbitrate-range` (and the audio equivalents) are mutually exclusive. + `--select-titles` and `--wanted` are mutually exclusive. `--vbitrate` and `--vbitrate-range` (and the audio equivalents) are mutually exclusive. --- diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 676b6493..f2f0c451 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -102,8 +102,8 @@ By default a missing requested resolution is an error. Two flags change that: | `--worst` | Pick the lowest-bitrate video at each requested resolution. With no requested resolution, pick the lowest-bitrate video at the lowest resolution. | A `quality` in the config counts as a requested resolution. When `--best-available` finds none of the -requested resolutions, `--worst` also picks from the lowest resolution. With `--worst`, hybrid mode -requires `-q`, also when `--generate-dv` switches an HDR10 or HDR10+ request to hybrid mode. +requested resolutions, `--worst` also picks from the lowest resolution. In hybrid mode, that is the +lowest resolution that has an HDR10 or HDR10+ base. ```shell title="Never fail on a missing resolution" unshackle dl -q 2160 --best-available EXAMPLE 0ABC123 @@ -266,7 +266,9 @@ measured for each pixel. ## Bitrate selection By default unshackle keeps the highest-bitrate rendition for each selected -resolution/codec/range/language combination. You can constrain this. +resolution/codec/range/language combination. One requested video language is one +language here, also when it matches more than one regional tag (`en` can match +`en-AU` and `en-GB`). You can constrain this. | Flag | Purpose | | --- | --- | diff --git a/tests/core/test_stereo_video.py b/tests/core/test_stereo_video.py index eed1e81d..d54af33e 100644 --- a/tests/core/test_stereo_video.py +++ b/tests/core/test_stereo_video.py @@ -11,7 +11,7 @@ from typing import Any import pytest from langcodes import Language -from unshackle.commands.dl import dl, hybrid_plan, pick_video +from unshackle.commands.dl import dl, hybrid_plan from unshackle.core import binaries from unshackle.core.config import config from unshackle.core.drm.verify import decrypter_for @@ -19,6 +19,7 @@ from unshackle.core.manifests import DASH, HLS from unshackle.core.titles.movie import Movie from unshackle.core.tracks import Audio, Chapter, Chapters, Track, Tracks, Video from unshackle.core.tracks import tracks as tracks_module +from unshackle.core.tracks.tracks import pick_video VIDEO_IDS = itertools.count() diff --git a/tests/core/test_worst_selection_loop.py b/tests/core/test_worst_selection_loop.py index 4ace76ff..51e16142 100644 --- a/tests/core/test_worst_selection_loop.py +++ b/tests/core/test_worst_selection_loop.py @@ -2,33 +2,32 @@ from __future__ import annotations -from pathlib import Path from typing import Any import pytest from tests.lang_selection import test_kind_only_selection as harness -from unshackle.core import binaries from unshackle.core.titles import Episode from unshackle.core.tracks import Tracks, Video LADDER = [ - ("1080-hi", 1920, 1080, 6_000_000), - ("1080-lo", 1920, 1080, 3_000_000), - ("1080-unknown", 1920, 1080, None), - ("540", 960, 540, 900_000), - ("270-lo", 480, 270, 79_000), - ("240-hi", 320, 240, 153_000), - ("240-lo", 320, 240, 140_000), - ("240-unknown", 320, 240, None), + ("1080-hi", Video.Range.SDR, 1920, 1080, 6_000_000), + ("1080-lo", Video.Range.SDR, 1920, 1080, 3_000_000), + ("1080-unknown", Video.Range.SDR, 1920, 1080, None), + ("540", Video.Range.SDR, 960, 540, 900_000), + ("270-lo", Video.Range.SDR, 480, 270, 79_000), + ("240-hi", Video.Range.SDR, 320, 240, 153_000), + ("240-lo", Video.Range.SDR, 320, 240, 140_000), + ("240-unknown", Video.Range.SDR, 320, 240, None), ] class LadderService(harness.FakeService): - """The harness episode with the video ladder above; a track with no bitrate is never the worst.""" + """The harness episode with a video ladder of one codec; a track with no bitrate is never the worst.""" - def __init__(self) -> None: + def __init__(self, ladder: list[tuple[Any, ...]] = LADDER, codec: Video.Codec = Video.Codec.AVC) -> None: super().__init__({1: {}}) + self.ladder, self.codec = ladder, codec def get_tracks(self, title: Episode) -> Tracks: tracks = super().get_tracks(title) @@ -37,13 +36,13 @@ class LadderService(harness.FakeService): id_=id_, url=f"https://x/{id_}", language="en", - codec=Video.Codec.AVC, - range_=Video.Range.SDR, + codec=self.codec, + range_=range_, width=width, height=height, bitrate=bitrate, ) - for id_, width, height, bitrate in LADDER + for id_, range_, width, height, bitrate in self.ladder ] return tracks @@ -64,10 +63,25 @@ def test_worst_picks_the_lowest_bitrate_of_the_quality_or_of_the_lowest_resoluti assert ids["video"] == expected -def test_worst_in_hybrid_mode_needs_a_quality( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +HDR_LADDER = [ + ("hdr-2160", Video.Range.HDR10, 3840, 2160, 15_000_000), + ("hdr-1080-hi", Video.Range.HDR10, 1920, 1080, 6_000_000), + ("hdr-1080-lo", Video.Range.HDR10, 1920, 1080, 4_000_000), + ("hdr-1080-unknown", Video.Range.HDR10, 1920, 1080, None), + ("dv-1080", Video.Range.DV, 1920, 1080, 5_000_000), + ("dv-540", Video.Range.DV, 960, 540, 1_000_000), +] + + +@pytest.mark.parametrize( + "flags", + [{}, {"quality": [1080]}, {"quality": [4320], "best_available": True}], + ids=["no-quality", "quality", "best-available"], +) +def test_worst_in_hybrid_mode_takes_the_lowest_bitrate_base_of_the_lowest_base_height( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, flags: dict[str, Any] ) -> None: - monkeypatch.setattr(binaries, "DoviTool", Path("dovi_tool")) - code = harness.run(monkeypatch, caplog, {1: {}}, service=LadderService(), worst=True, range_=[Video.Range.HYBRID]) - assert code == 1 - assert "--worst requires -q/--quality to be specified in hybrid mode" in caplog.text + harness.stub_hybrid(monkeypatch) + service = LadderService(HDR_LADDER, Video.Codec.HEVC) + ids = harness.run(monkeypatch, caplog, {1: {}}, service=service, worst=True, range_=[Video.Range.HYBRID], **flags) + assert ids["video"] == ["hdr-1080-lo", "dv-540"] diff --git a/tests/lang_selection/test_kind_only_selection.py b/tests/lang_selection/test_kind_only_selection.py index 57413131..54c7630e 100644 --- a/tests/lang_selection/test_kind_only_selection.py +++ b/tests/lang_selection/test_kind_only_selection.py @@ -9,12 +9,14 @@ from __future__ import annotations import inspect import logging +from pathlib import Path from typing import Any, Optional, cast import pytest import requests from unshackle.commands.dl import dl +from unshackle.core import binaries from unshackle.core.api.handlers import DEFAULT_DOWNLOAD_PARAMS from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY from unshackle.core.service import Service @@ -81,6 +83,15 @@ class FakeService: raise AttributeError(name) +def stub_hybrid(monkeypatch: pytest.MonkeyPatch) -> None: + """Let a hybrid run start without dovi_tool, and load the service's tracks with no probes.""" + monkeypatch.setattr(binaries, "DoviTool", Path("/unused/dovi_tool"), raising=False) + monkeypatch.setattr( + "unshackle.commands.dl.load_tracks", + lambda service, title, **_: setattr(title, "tracks", service.get_tracks(title)), + ) + + def run( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, diff --git a/tests/lang_selection/test_regional_selection_loop.py b/tests/lang_selection/test_regional_selection_loop.py index 7812459f..efe5cd8d 100644 --- a/tests/lang_selection/test_regional_selection_loop.py +++ b/tests/lang_selection/test_regional_selection_loop.py @@ -8,6 +8,7 @@ import pytest from langcodes import Language from tests.lang_selection import test_kind_only_selection as harness +from unshackle.commands.dl import hybrid_plan from unshackle.core.titles import Episode from unshackle.core.tracks import Audio, Subtitle, Tracks, Video @@ -197,3 +198,185 @@ def test_the_meant_variant_does_not_depend_on_codec( service = RegionalService("ja", ["ja", "es-ES:EC3", "es-419"], []) ids = run(monkeypatch, caplog, service, a_lang=["es"], acodec=[Audio.Codec.AAC, Audio.Codec.EC3]) assert ids["audio"] == ["es-ES:EC3"] + + +class RegionalVideoService(RegionalService): + """A RegionalService whose one video is replaced by `videos`: SDR AVC tracks given as (id, language, height, kbps).""" + + def __init__(self, title_lang: str, videos: list[tuple[Any, ...]]) -> None: + super().__init__(title_lang, [title_lang], []) + self.videos = videos + + def get_tracks(self, title: Episode) -> Tracks: + tracks = super().get_tracks(title) + tracks.videos = [self.video(*row) for row in self.videos] + return tracks + + def video(self, *row: Any) -> Video: + id_, lang, height, kbps = row + return Video( + id_=id_, + url=f"https://x/{id_}", + language=lang, + codec=Video.Codec.AVC, + range_=Video.Range.SDR, + width=height * 16 // 9, + height=height, + bitrate=kbps * 1000, + ) + + +# en means en-US, and the title has none, so a bare en or orig request takes both regional forms +MIXED = [("au480", "en-AU", 480, 900), ("gb720", "en-GB", 720, 2000)] +SAME_HEIGHT = [("au720", "en-AU", 720, 900), ("gb720", "en-GB", 720, 2000)] + + +@pytest.mark.parametrize("request_", [{"v_lang": ["en"]}, {}], ids=["vl-en", "orig"]) +@pytest.mark.parametrize( + ("videos", "flags", "expected"), + [ + (MIXED, {}, "gb720"), + (MIXED, {"quality": [1], "worst": True, "best_available": True}, "au480"), + (SAME_HEIGHT, {"quality": [720]}, "gb720"), + (SAME_HEIGHT, {"quality": [720], "worst": True}, "au720"), + ], + ids=["best", "lowest-resolution", "best-at-720", "worst-at-720"], +) +def test_the_regional_forms_of_one_request_rank_as_one_language( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + videos: list, + flags: dict, + expected: str, + request_: dict, +) -> None: + ids = run(monkeypatch, caplog, RegionalVideoService("en", videos), **request_, **flags) + assert ids["video"] == [expected] + + +@pytest.mark.parametrize( + ("v_lang", "expected"), + [ + (["en-AU"], ["au480"]), + (["en-GB", "en-AU"], ["gb720", "au480"]), + (["en-AU", "en-GB"], ["au480", "gb720"]), + (["all"], ["gb720", "au480"]), + ], +) +def test_regional_tags_and_several_video_languages_keep_their_selection( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, v_lang: list, expected: list +) -> None: + assert run(monkeypatch, caplog, RegionalVideoService("en", MIXED), v_lang=v_lang)["video"] == expected + + +@pytest.mark.parametrize( + ("flags", "expected"), + [ + ({"v_lang": ["en", "fr"]}, ["gb720", "fr720"]), + ({"v_lang": ["en", "orig"]}, ["gb720"]), + ({"v_lang": ["en", "de"], "best_available": True}, ["gb720"]), + ], + ids=["two-languages", "orig-is-en", "missing-language-dropped"], +) +def test_each_video_request_gives_one_video_across_its_regional_tags( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, flags: dict, expected: list +) -> None: + service = RegionalVideoService("en", [*MIXED, ("fr720", "fr-FR", 720, 1500)]) + assert run(monkeypatch, caplog, service, **flags)["video"] == expected + + +class HdrRegionalService(RegionalVideoService): + """A RegionalVideoService of HEVC videos given as (id, language, range, width, height, kbps).""" + + def video(self, *row: Any) -> Video: + id_, lang, range_, width, height, kbps = row + return Video( + id_=id_, + url=f"https://x/{id_}", + language=lang, + codec=Video.Codec.HEVC, + range_=Video.Range[range_], + width=width, + height=height, + bitrate=kbps * 1000, + ) + + +class SelectedWithStandalone(harness.Selected): + def __init__(self, tracks: Tracks) -> None: + super().__init__(tracks) + self.ids["standalone"] = [t.id for t in tracks.videos if not t.hybrid_base_only] + + +def run_hdr( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, videos: list, ranges: list, **flags: Any +) -> Any: + harness.stub_hybrid(monkeypatch) + monkeypatch.setattr(harness, "Selected", SelectedWithStandalone) + service = HdrRegionalService("en", videos) + range_ = [Video.Range[r] for r in ranges] + return run(monkeypatch, caplog, service, quality=[2160], v_lang=["en"], range_=range_, **flags) + + +def test_hybrid_takes_the_best_dv_source_of_the_lowest_height_across_regional_tags( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + videos = [ + ("hdr", "en-AU", "HDR10", 3840, 2160, 8000), + ("dv-au", "en-AU", "DV", 1920, 1080, 2000), + ("dv-gb", "en-GB", "DV", 1920, 1080, 6000), + ] + assert run_hdr(monkeypatch, caplog, videos, ["HYBRID"])["video"] == ["hdr", "dv-gb"] + + +def test_generated_dv_takes_the_best_base_across_regional_tags( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + videos = [("hdr-au", "en-AU", "HDR10", 3840, 1600, 8000), ("hdr-gb", "en-GB", "HDR10", 3840, 2160, 8000)] + assert run_hdr(monkeypatch, caplog, videos, ["HDR10"], generate_dv=True)["video"] == ["hdr-gb"] + + +@pytest.mark.parametrize( + ("videos", "expected"), + [ + ( + [ + ("hdr-au", "en-AU", "HDR10", 3840, 1600, 8000), + ("hdr-gb", "en-GB", "HDR10", 3840, 2160, 8000), + ("dv", "en-AU", "DV", 1920, 1080, 2000), + ], + ["hdr-gb"], + ), + ( + [ + ("hdr10p", "en-AU", "HDR10P", 3840, 2160, 8000), + ("hdr-crop", "en-AU", "HDR10", 3840, 1600, 9000), + ("dv", "en-AU", "DV", 1920, 1080, 2000), + ], + ["hdr10p"], + ), + ], + ids=["regional-tags", "standalone-at-another-height"], +) +def test_a_standalone_range_beside_hybrid_does_not_add_a_hybrid_base( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, videos: list, expected: list +) -> None: + class Planned(BaseException): + def __init__(self, videos: list[Video], generate_dv: bool) -> None: + self.base_ids = [v.id for v in hybrid_plan(videos, generate_dv)[0]] + + def stop_at_plan(videos: list[Video], generate_dv: bool) -> None: + raise Planned(videos, generate_dv) + + monkeypatch.setattr("unshackle.commands.dl.hybrid_plan", stop_at_plan) + monkeypatch.setattr(Video, "normalize_vui", lambda self: None) + with pytest.raises(Planned) as planned: + run_hdr(monkeypatch, caplog, videos, ["HYBRID", "HDR10"], to_end=True, video_only=True) + assert planned.value.base_ids == expected + + +def test_the_fallback_from_a_failed_hybrid_ranks_regional_tags_as_one_language( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + videos = [("sdr-au", "en-AU", "SDR", 3840, 2160, 3000), ("sdr-gb", "en-GB", "SDR", 3840, 2160, 9000)] + assert run_hdr(monkeypatch, caplog, videos, ["HYBRID", "SDR"], best_available=True)["video"] == ["sdr-gb"] diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index cd50b42f..341d90a2 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -21,7 +21,7 @@ from datetime import date, timedelta from functools import partial from http.cookiejar import CookieJar, MozillaCookieJar from io import StringIO -from itertools import groupby, product +from itertools import groupby from operator import attrgetter from pathlib import Path from threading import Lock @@ -111,7 +111,7 @@ from unshackle.core.tracks.track import ( fix_data_reference_indexes, has_encrypted_sample_entry, ) -from unshackle.core.tracks.tracks import at_resolution +from unshackle.core.tracks.tracks import at_resolution, select_videos from unshackle.core.utilities import ( apply_original_language, as_requested, @@ -376,20 +376,6 @@ def hybrid_plan(videos: Sequence[Video], generate_dv: bool) -> tuple[list[Video] return select_hybrid_bases(base_pool), matching_dv -def pick_video(candidates: list[Video], worst: bool) -> Optional[Video]: - """The best candidate, or with ``worst`` the lowest bitrate of the lowest resolution, preferring 2D. - - A track with no bitrate or no size counts as the highest, and a full tie goes to the later track. - """ - if worst: - return min( - reversed(candidates), - key=lambda v: (v.stereo, standard_resolution(v.width, v.height) or math.inf, v.bitrate or math.inf), - default=None, - ) - return next(iter(candidates), None) - - def hybrid_source_tracks(base: Video, dv: Optional[Video], base_container: Optional[Path] = None) -> list[Video]: """Copies of the base and DV tracks for the hybrid step, with the base on its container file. @@ -1335,7 +1321,7 @@ class dl: is_flag=True, default=False, help="Select the lowest bitrate track within the specified quality, or of the lowest resolution if no " - "quality is set. Hybrid mode requires -q/--quality.", + "quality is set.", ) @click.option( "--best-available", @@ -2054,9 +2040,6 @@ class dl: if not DoviTool: self.log.error("Unable to run hybrid mode: dovi_tool not found") sys.exit(1) - if worst and not quality: - self.log.error("--worst requires -q/--quality to be specified in hybrid mode") - sys.exit(1) if cdm_only is None: vaults_only = None @@ -2724,7 +2707,6 @@ class dl: resolution: Optional[int] color_range: Optional[Video.Range] codec: Optional[Video.Codec] - vlang: Optional[str] if keep_videos and vcodec: title.tracks.select_video(lambda x: x.codec in vcodec) missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)] @@ -2793,9 +2775,7 @@ class dl: effective_video_lang = v_lang or lang video_languages = [lang for lang in effective_video_lang if lang != "best"] - video_multi_lang = ( - "best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1 - ) + processed_video_lang = video_languages if keep_videos and video_languages and "all" not in video_languages: processed_video_lang = resolve_sort_langs(video_languages, title.language, exact=exact_lang) v_orig_token = orig_token(video_languages, title.language, exact_lang) @@ -2836,6 +2816,7 @@ class dl: f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track...", ) continue + video_requests = None if {"best", "all"} & set(effective_video_lang) else processed_video_lang has_hybrid = any(r == Video.Range.HYBRID for r in range_) non_hybrid_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID] @@ -2911,8 +2892,16 @@ class dl: hybrid_quality = effective_quality if not hybrid_quality: - best_resolution = max((v.height for v in hybrid_candidate_tracks if v.height), default=None) - hybrid_quality = [best_resolution] if best_resolution else [] + height_pool = ( + [ + *select_hybrid_bases(hybrid_candidate_tracks), + *(v for v in hybrid_candidate_tracks if v.finished_hybrid), + ] + if worst + else hybrid_candidate_tracks + ) + base_heights = [v.height for v in height_pool if v.height] + hybrid_quality = [(min if worst else max)(base_heights)] if base_heights else [] hybrid_selected = [] if hybrid_quality: hybrid_filter = title.tracks.select_hybrid( @@ -2925,56 +2914,22 @@ class dl: non_hybrid_selected = [] if non_hybrid_ranges and non_hybrid_tracks: - if video_multi_lang: - non_hybrid_langs: list[Optional[str]] = list( - dict.fromkeys(str(v.language) for v in non_hybrid_tracks) - ) - else: - non_hybrid_langs = [None] - for resolution, color_range, codec, vlang in product( - effective_quality or (None,), non_hybrid_ranges, vcodec or (None,), non_hybrid_langs - ): - candidates = [ - t - for t in non_hybrid_tracks - if (not resolution or at_resolution(t, resolution)) - and (not color_range or t.range == color_range) - and (not codec or t.codec == codec) - and (vlang is None or str(t.language) == vlang) - ] - match = pick_video(candidates, worst) - if match and match not in non_hybrid_selected: - non_hybrid_selected.append(match) + non_hybrid_selected = select_videos( + non_hybrid_tracks, + effective_quality, + non_hybrid_ranges, + vcodec, + video_requests, + exact_lang, + worst, + ) title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected) - - # Flag tracks selected only as hybrid ingredients (the HDR base and/or - # the lowest DV) so the standalone mux loop skips them. Tracks also - # picked as explicit deliverables stay unflagged. Tracks.flag_hybrid_ingredients(hybrid_selected, non_hybrid_selected) elif keep_videos: - selected_videos: list[Video] = [] - if video_multi_lang: - unique_video_langs: list[Optional[str]] = list( - dict.fromkeys(str(v.language) for v in title.tracks.videos) - ) - else: - unique_video_langs = [None] - for resolution, color_range, codec, vlang in product( - effective_quality or (None,), range_ or (None,), vcodec or (None,), unique_video_langs - ): - candidates = [ - t - for t in title.tracks.videos - if (not resolution or at_resolution(t, resolution)) - and (not color_range or t.range == color_range) - and (not codec or t.codec == codec) - and (vlang is None or str(t.language) == vlang) - ] - match = pick_video(candidates, worst) - if match and match not in selected_videos: - selected_videos.append(match) - title.tracks.videos = selected_videos + title.tracks.videos = select_videos( + title.tracks.videos, effective_quality, range_, vcodec, video_requests, exact_lang, worst + ) if keep_videos and any(r == Video.Range.HYBRID for r in range_): gate_failure = hybrid_gate_failure(title.tracks.videos, generate_dv) @@ -2990,29 +2945,15 @@ class dl: f"{gate_failure}; continuing with {', '.join(r.name for r in other_ranges)}" ) range_ = other_ranges - fallback_pool = pre_hybrid_videos - if video_multi_lang: - fallback_langs: list[Optional[str]] = list( - dict.fromkeys(str(v.language) for v in fallback_pool) - ) - else: - fallback_langs = [None] - fallback_selected: list[Video] = [] - for resolution, color_range, codec, vlang in product( - effective_quality or (None,), other_ranges, vcodec or (None,), fallback_langs - ): - candidates = [ - t - for t in fallback_pool - if (not resolution or at_resolution(t, resolution)) - and (not color_range or t.range == color_range) - and (not codec or t.codec == codec) - and (vlang is None or str(t.language) == vlang) - ] - match = pick_video(candidates, worst) - if match and match not in fallback_selected: - fallback_selected.append(match) - title.tracks.videos = fallback_selected + title.tracks.videos = select_videos( + pre_hybrid_videos, + effective_quality, + other_ranges, + vcodec, + video_requests, + exact_lang, + worst, + ) else: skip_title(title, gate_failure) continue @@ -3792,7 +3733,9 @@ class dl: standalone_groups: list[list[Optional[Video]]] = [ list(g) for g in group_videos_by_variant(standalone_videos, merge=merge_video) ] - hybrid_bases, matching_dv = hybrid_plan(title.tracks.videos, generate_dv) + hybrid_bases, matching_dv = hybrid_plan( + [v for v in original_videos if v.hybrid_ingredient], generate_dv + ) hybrid_base_ids.update(v.id for v in hybrid_bases) hybrids_made = 0 for video in (v for v in original_videos if v.native_hybrid): diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 58de88af..d655bcaf 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -1178,7 +1178,7 @@ async def download(request: web.Request) -> web.Response: type: boolean description: >- Select the lowest bitrate track within the specified quality, or at the lowest resolution if no - quality is set. HYBRID requires `quality` (default - false) + quality is set (default - false) repack: type: boolean description: Add REPACK tag to the output filename (default - false) diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 7e9fffcd..f8f5232b 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -2,12 +2,14 @@ from __future__ import annotations import json import logging +import math import os import re import subprocess import threading import time from functools import partial +from itertools import product from pathlib import Path from typing import Any, Callable, Iterator, Optional, Sequence, Union @@ -59,6 +61,76 @@ def at_resolution(video: Video, res: int) -> bool: ) +def video_rank(video: Video) -> tuple[int, bool, float, int]: + """The quality key of a video, highest first when sorted in reverse: standard resolution, 2D, bitrate, height.""" + return ( + standard_resolution(video.width, video.height) or 0, + not video.stereo, + float(video.bitrate or 0.0), + video.height or 0, + ) + + +def pick_video(candidates: list[Video], worst: bool) -> Optional[Video]: + """The best candidate by ``video_rank``, or with ``worst`` the lowest bitrate of the lowest resolution. + + ``worst`` prefers 2D and counts a track with no bitrate or no size as the highest. A full tie goes to the + earlier track, or with ``worst`` to the later one. + """ + if worst: + return min( + reversed(candidates), + key=lambda v: (v.stereo, standard_resolution(v.width, v.height) or math.inf, v.bitrate or math.inf), + default=None, + ) + return max(candidates, key=video_rank, default=None) + + +def video_language_groups(videos: list[Video], requests: Optional[list[str]], exact: bool) -> list[Optional[set[str]]]: + """The tag sets to pick one video from each: one per request, or one per tag when ``requests`` is None. + + A request that matches several regional tags (en-AU, en-GB) is one set, so it gives one video. With no + requests, the whole pool is one group, given as None. + """ + if requests is None: + return [{tag} for tag in dict.fromkeys(str(v.language) for v in videos)] + tags = [v.language for v in videos] + return [matching_languages(request, tags, exact) for request in requests] or [None] + + +def select_videos( + pool: list[Video], + qualities: Sequence[int], + ranges: Sequence[Video.Range], + codecs: Sequence[Video.Codec], + requests: Optional[list[str]], + exact: bool, + worst: bool, +) -> list[Video]: + """One ``pick_video`` pick for each quality, range, codec and language group of the pool, without repeats. + + An empty quality, range or codec list puts no limit on that field. The groups come from ``video_language_groups``. + """ + selected: list[Video] = [] + for res, range_, codec, group in product( + qualities or (None,), ranges or (None,), codecs or (None,), video_language_groups(pool, requests, exact) + ): + match = pick_video( + [ + v + for v in pool + if (not res or at_resolution(v, res)) + and (not range_ or v.range == range_) + and (not codec or v.codec == codec) + and (group is None or str(v.language) in group) + ], + worst, + ) + if match and match not in selected: + selected.append(match) + return selected + + def file_track_ids(path: Path) -> dict[str, int]: """Return the mkvmerge ID of the first track of each type in a file. A TS file can hold its audio first.""" res = subprocess.run( @@ -361,15 +433,7 @@ class Tracks: """ if not self.videos: return - self.videos.sort( - key=lambda x: ( - standard_resolution(x.width, x.height) or 0, - not x.stereo, - float(x.bitrate or 0.0), - x.height or 0, - ), - reverse=True, - ) + self.videos.sort(key=video_rank, reverse=True) for language in reversed(by_language or []): if str(language) in ("all", "best"): language = self._original_language(self.videos, exact_match) @@ -527,13 +591,14 @@ class Tracks: @staticmethod def flag_hybrid_ingredients(hybrid_selected: list[Video], non_hybrid_selected: list[Video]) -> None: - """Mark tracks selected only as hybrid ingredients so the standalone mux loop skips them. + """Mark every hybrid ingredient, and the ingredient-only ones that the standalone mux loop skips. A track that the caller also selected as an explicit deliverable (same track in both selections) stays unflagged, and the standalone mux loop muxes it alongside the hybrid. A ``native_hybrid`` track is the hybrid itself, so it stays unflagged too. """ for video in hybrid_selected: + video.hybrid_ingredient = True if video not in non_hybrid_selected and not video.native_hybrid: video.hybrid_base_only = True @@ -544,15 +609,15 @@ class Tracks: A track that already carries DV and an HDR10 or HDR10+ base (DV+HDR10P, HDR10+DV, HDR10P+DV) is a finished hybrid, so its resolution takes that track and no base. One that carries HDR10+ comes first. The other - resolutions get a base only when a DV track can give it an RPU, or with ``generate_dv``. + resolutions get a base only when a DV track can give it an RPU, or with ``generate_dv``. Each pick follows + ``pick_video``, and the DV track is the lowest one, the best of that height. """ - pick = min if worst else max natives = {} for res in quality: candidates = [v for v in tracks if v.finished_hybrid and at_resolution(v, res)] if candidates: with_hdr10plus = [v for v in candidates if v.carries_hdr10plus] - natives[res] = pick(with_hdr10plus or candidates, key=lambda v: v.bitrate or 0) + natives[res] = pick_video(with_hdr10plus or candidates, worst) for video in tracks: video.native_hybrid = any(video is v for v in natives.values()) dv_tracks = [v for v in tracks if v.range == Video.Range.DV] @@ -578,14 +643,14 @@ class Tracks: base_selected = [] for res in quality: - candidates = [v for v in base_tracks if at_resolution(v, res)] - if candidates: - chosen = pick(candidates, key=lambda v: v.bitrate or 0) + chosen = pick_video([v for v in base_tracks if at_resolution(v, res)], worst) + if chosen: base_selected.append(chosen) - lowest_dv = ( - min(dv_tracks, key=lambda v: v.height or 0) if dv_tracks and (base_selected or not natives) else None - ) + lowest_dv = None + if dv_tracks and (base_selected or not natives): + lowest_height = min(v.height or 0 for v in dv_tracks) + lowest_dv = pick_video([v for v in dv_tracks if (v.height or 0) == lowest_height], worst=False) def select(x: Video) -> bool: if x in base_selected or x.native_hybrid: diff --git a/unshackle/core/tracks/video.py b/unshackle/core/tracks/video.py index 8edb3546..0333e18a 100644 --- a/unshackle/core/tracks/video.py +++ b/unshackle/core/tracks/video.py @@ -304,6 +304,7 @@ class Video(Track): self.hdr10_base = hdr10_base self.stereo = stereo self.hybrid_base_only = False + self.hybrid_ingredient = False self.native_hybrid = False def to_dict(self) -> dict[str, Any]: