From a4ce70a53ce414a3e8f19ed0e1d81025ebb3895a Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Thu, 15 Oct 2026 12:26:39 -0600 Subject: [PATCH] feat(tracks)!: match a language tag to the variant it means A tag with a region (en-US, es-419) matches only that variant and reports a miss instead of taking another region. A bare tag takes the variant it means (pt -> pt-BR, es -> es-ES) and falls back to any variant only when that one is missing. `orig` prefers the title's own tag, `-sl orig` resolves, forced and SDH subtitles are picked from their own pools, and the track name no longer strips the region from the cached language. BREAKING CHANGE: -l es-419 no longer selects es-ES, and language_priority, language_tags and muxing.default_language treat a regional tag as exact. --- docs/dev/rest-api/endpoints.md | 2 +- docs/guide/cli-reference.md | 2 +- docs/guide/downloading.md | 31 +++- docs/guide/output-and-naming.md | 2 +- docs/guide/subtitles.md | 26 +-- docs/reference/configuration/download.md | 19 +- tests/core/test_lang_exclusions.py | 22 ++- tests/core/test_language_tag_rules.py | 6 + .../lang_selection/test_find_missing_langs.py | 6 +- .../test_forced_sub_selection.py | 25 --- .../test_kind_only_selection.py | 12 +- .../lang_selection/test_matching_languages.py | 46 ++++- .../lang_selection/test_original_audio_ids.py | 3 + .../test_regional_selection_loop.py | 166 ++++++++++++++++++ tests/lang_selection/test_required_langs.py | 11 +- .../lang_selection/test_resolve_sort_langs.py | 9 +- tests/tracks/test_mux_default_language.py | 26 +++ tests/tracks/test_subtitle_sort.py | 51 +++++- tests/tracks/test_track_name_region.py | 19 ++ unshackle/commands/dl.py | 150 +++++++--------- unshackle/core/api/handlers.py | 15 +- unshackle/core/api/routes.py | 2 +- unshackle/core/tracks/track.py | 4 +- unshackle/core/tracks/tracks.py | 67 ++++--- unshackle/core/utilities.py | 104 +++++++---- unshackle/core/utils/language_tags.py | 11 +- 26 files changed, 584 insertions(+), 253 deletions(-) delete mode 100644 tests/lang_selection/test_forced_sub_selection.py create mode 100644 tests/lang_selection/test_regional_selection_loop.py create mode 100644 tests/tracks/test_mux_default_language.py create mode 100644 tests/tracks/test_track_name_region.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 16cf3186..b401be97 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -448,7 +448,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `forced_subs` | boolean | `false` | Include forced subtitles. | | `forced_subs_only` | boolean | `false` | Download forced subtitles and no other subtitle; implies `forced_subs`. Not valid with `no_subs`, `video_only`, `audio_only` or `chapters_only`. | | `forced_s_lang` | string[] | `[]` | Forced subtitle language(s); implies `forced_subs`. | -| `exact_lang` | boolean | `false` | Exact language matching. | +| `exact_lang` | boolean | `false` | A language tag without a region never falls back to another variant. A tag with a region is always exact. | | `sub_format` | string | `null` | Output subtitle format. | | `video_only` / `audio_only` / `subs_only` / `chapters_only` | boolean | `false` | Track-type restriction (at most one). | | `no_subs` / `no_audio` / `no_chapters` / `no_video` / `no_attachments` | boolean | `false` | Skip a track type. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 436a1cfb..3e79d1eb 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -104,7 +104,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | | `-fso`, `--forced-subs-only` | off | Keep only forced subtitle tracks; drop the other subtitles. Implies `-fs`. | | `-fsl`, `--forced-s-lang` | none | Language(s) wanted for forced subtitles; implies `-fs`. A `-` prefix excludes. | -| `--exact-lang` | off | Exact matching only: `-l es-419` matches `es-419`, not `es-ES`. Applies to selection and to sort order. | +| `--exact-lang` | off | A tag without a region selects only the variant it means and never falls back to another: `-l es` does not select `es-419`. A tag with a region, such as `es-419`, is always exact. Applies to selection and to sort order. | | `--sub-format` | - | Output subtitle format (`SRT`/`srt`, `VTT`/`webvtt`, `ASS`/`ssa`, `TTML`, `SMI`, ...), or `original` to keep the source format. | !!! tip "These flags select languages, they do not order them" diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 2cae867a..0edca1e2 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -315,9 +315,9 @@ unshackle dl -l en,-es -vl ja EXAMPLE 81234567 !!! note "Rules for exclusion tokens" - `orig` becomes the title's own original language, so `-sl -orig` drops the subtitles in that language. - - `--exact-lang` applies to exclusions with the same matching rules as selection: - `-es` removes exactly the tracks that `es` would select. Without it, `-es` also - removes `es-419` and `es-ES`. + - An exclusion without a region removes every regional form: `-es` removes `es-419` + and `es-ES`. An exclusion with a region removes only that variant: `-es-419` removes + only `es-419`. With `--exact-lang`, `-es` removes only the variant `es` means. - Tracks with no language tag are never excluded. - unshackle rejects `-all`. Name the languages you do not want instead. - The comma form is the documented style. `-sl -es` also works, but a shell can read @@ -345,9 +345,28 @@ See [`subtitle.language_priority`](../reference/configuration/download.md#subtit ### Exact vs fuzzy matching -By default language matching is fuzzy: `-l en` also accepts `en-US`, `en-GB`, and the -other regional forms (up to a small distance). Pass `--exact-lang` when only an exact -match is acceptable: with it, `-l es-419` selects only `es-419`, not `es-ES`. +A tag without a region asks for the variant it means: `pt` is Brazilian Portuguese, `es` is +Spanish from Spain, and `en` is US English. With `pt-PT` and `pt-BR` tracks, `-l pt` +selects `pt-BR`. If the title has no such variant, the tag takes every regional form, so +`-l es` on a title with only `es-419` selects `es-419`. + +A tag with a region asks for that variant only: `-l es-419` selects `es-419`, never `es-ES` +or `es-MX`. If the service has no `es-419` track, unshackle reports `es-419 not found` and +stops. With `--best-available`, it continues with the other languages. + +A track without a region matches only the region its language implies. `en` is US English +and `pt` is Brazilian Portuguese, so `-l en-US` accepts a track tagged `en`, but `-l en-GB` +and `-l pt-PT` do not accept `en` or `pt`. To take any English track, write `en`. + +`--exact-lang` turns off that fallback: a tag without a region then selects only the +variant it means, so `-l es` does not select `es-419`. + +`orig` is the title language that the service gives. It selects that exact variant first, +so a `fr-CA` title takes `fr-CA` audio over `fr-FR`. When the title has no track in that +variant, `orig` takes the closest regional form, unless you pass `--exact-lang`. + +An exclusion without a region removes every regional form: `-es` drops `es-ES` and `es-419`. +See the [exclusion rules](#excluding-languages) for `--exact-lang`. !!! tip "The `all` and `best` tokens" For audio, the tokens `all` and `best` bypass the usual one-track-per-language pick diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index ada3dd5c..1d452747 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -116,7 +116,7 @@ Every variable below is valid in both output and folder templates. unshackle tak | `lang_tag` | Result of your `language_tags` rules, if configured | `MULTi` | !!! note "How `language_tags` rules match" - unshackle tries the rules in order, but language comparison is **fuzzy**: a rule condition of `en` matches `en-US`, `en-GB`, and any other `en-*` form. Within a single rule, **all** conditions must match (AND logic) for it to apply. The first rule that fully matches wins. If none match, unshackle removes `{lang_tag?}` from the filename cleanly. + unshackle tries the rules in order, and they compare languages with the same [matching rules](downloading.md#exact-vs-fuzzy-matching) as `-l`: a rule condition of `en` matches `en-US`, `en-GB`, and any other `en-*` form, but `es-419` matches only `es-419`. Within a single rule, **all** conditions must match (AND logic) for it to apply. The first rule that fully matches wins. If none match, unshackle removes `{lang_tag?}` from the filename cleanly. ### Episode-specific diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index 637057c2..90ccfa66 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -76,7 +76,7 @@ Subtitle selection happens through `dl` flags. The most important is language se | `-fs`, `--forced-subs` | Include forced subtitle tracks (excluded by default). | | `-fso`, `--forced-subs-only` | Keep only forced subtitle tracks and drop the other subtitles; implies `-fs`. | | `-fsl`, `--forced-s-lang` | Language(s) wanted for forced subtitles; implies `-fs`. A `-` prefix excludes. | -| `--exact-lang` | Exact language matching, with no regional variants. | +| `--exact-lang` | A tag without a region never falls back to another variant: `es` does not match `es-419`. | | `-S`, `--subs-only` | Download only subtitle tracks. | | `-ns`, `--no-subs` | Do not download subtitle tracks at all. | | `--sub-format` | Output subtitle format / conversion target (see [conversion](#converting-subtitle-formats)). | @@ -91,17 +91,18 @@ To narrow it down, pass a comma-separated list of language tags: unshackle dl --s-lang en,es EXAMPLE 81234567 ``` -Language matching is fuzzy by default: `en` matches `en-US`, `en-GB`, and similar -variants. If a requested language is missing but others were found, unshackle continues -with whatever remains and tells you which languages it dropped. - -For strict matching, where `es-419` matches only `es-419` and never `es-ES`, add -`--exact-lang`: +A tag without a region selects the variant it means, such as `en-US` for `en`, and every +regional form only when the title has no such variant. A tag with a region matches only +that variant: ```shell title="Only Latin-American Spanish, no other Spanish variants" -unshackle dl --s-lang es-419 --exact-lang EXAMPLE 'https://www.example.com/...' +unshackle dl --s-lang es-419 EXAMPLE 'https://www.example.com/...' ``` +If a requested language is missing, unshackle stops with an error. With `--best-available`, +it continues with the languages it found and tells you which ones it dropped. See +[exact vs fuzzy matching](downloading.md#exact-vs-fuzzy-matching) for the full rules. + ### Requiring languages before downloading `--require-subs` is a gate rather than a filter. It downloads **all** available @@ -193,10 +194,11 @@ can make a clean non-SDH version for you by stripping the hearing-impaired cues: sound effects, speaker labels, music notes, and similar. This is on by default and controlled by the [`strip_sdh`](#subtitle-configuration) config key. -The check is deliberately loose: if a close-enough plain track already exists (for -example an `en-GB` non-SDH track when the SDH track is `en-US`), no stripped copy is -created. When a stripped copy *is* created, the original SDH track is kept as well, so -you end up with both. +The check uses the same [matching rules](downloading.md#exact-vs-fuzzy-matching) as +`--s-lang`. A plain track covers an SDH track only when it is the same variant: `en` and +`en-US` cover each other, but a plain `en-GB` track does not stop a stripped `en` or `en-US` +copy. When unshackle creates a stripped +copy, it keeps the original SDH track as well, so you end up with both. The [`sdh_method`](#subtitle-configuration) config key selects the stripping engine itself. The default (`auto`) prefers the `subby` cleaner for SRT, falls back to diff --git a/docs/reference/configuration/download.md b/docs/reference/configuration/download.md index 2267c02e..a7275a70 100644 --- a/docs/reference/configuration/download.md +++ b/docs/reference/configuration/download.md @@ -219,13 +219,15 @@ subtitle: unshackle removes nothing, unlike `-sl`, which drops every subtitle language it does not name. - Use `orig` in the list for the title's original language. A base tag also covers its - regional forms, so `en` sorts `en-US` and `en-GB`, but `zh` does **not** cover - `zh-Hant`, which you must name. The sort ignores entries that match no track, and it - ignores entries that are not valid language tags. + Use `orig` in the list for the title's original language. A base tag puts the variant it + means first and its other regional forms after it, so `en` sorts `en-US` and then + `en-GB`, but `zh` does **not** cover `zh-Hant`, which you must name. A tag with a region sorts only that variant, so + `[en-US]` puts `en-US` before `en-GB`. The sort ignores entries that match no track, and + it ignores entries that are not valid language tags. - `--exact-lang` applies here too: with it, `en` sorts only `en`, and `en-US` falls to - the alphabetical tail with the rest. Name each tag you want when you use that flag. + `--exact-lang` applies here too: with it, `en` sorts only the variant it means, and + `en-GB` falls to the alphabetical tail with the rest. Name each tag you want when you use + that flag. !!! note "`type_priority` picks the default subtitle only with `muxing.default_language`" At mux time the `default` flag goes to every forced subtitle whose language matches the @@ -277,7 +279,7 @@ Matroska (MKV) muxing options. | Sub-key | Type | Default | Description | |---------|------|---------|-------------| | `set_title` | bool | `true` | Write a human-readable title into the MKV container. | -| `default_language` | dict | `{}` | Force which language is flagged *default* per track type, e.g. `{audio: en, subtitle: en}`. | +| `default_language` | dict | `{}` | Force which language is flagged *default* per track type, e.g. `{audio: en, subtitle: en}`. A tag with a region, such as `en-US`, matches only that variant. | | `merge_audio` | bool | `true` | Merge all audio into one file. `--split-audio` on the CLI flips this off. | | `merge_video` | bool | `false` | Merge video tracks that share height, range, and codec into one file, so only language varies inside a file. `--merge-video` on the CLI flips this on. | | `concurrency` | int | `4` | How many mkvmerge processes run at the same time when a title muxes more than one output file (one per video variant or audio codec). Each process uses one CPU core, so this shortens the mux step on an SSD. Set it to `1` on a hard disk, where parallel writes are slower. | @@ -308,7 +310,8 @@ Language-tag remapping and the rule engine behind the `{lang_tag}` filename vari Each language condition takes one language tag or a list of them. A list matches when **any** entry matches, except `subs_contain_all`, which needs them all. An entry that is not a valid -language tag (for example `japanese`) never matches. `dual`, `multi` and `dubbed` take a bare +language tag (for example `japanese`) never matches. A tag with a region matches only that +variant, so `audio: es-419` does not match `es-ES` audio. `dual`, `multi` and `dubbed` take a bare `true` or `false`, not a quoted string. unshackle skips a rule with a condition name it does not know, or with any other value in a boolean condition. It logs a warning in each of these cases and tries the next rule. diff --git a/tests/core/test_lang_exclusions.py b/tests/core/test_lang_exclusions.py index 69f6760f..814b83cf 100644 --- a/tests/core/test_lang_exclusions.py +++ b/tests/core/test_lang_exclusions.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from langcodes import Language -from unshackle.core.utilities import excluded_language_tags, partition_exclusions +from unshackle.core.utilities import matched_language_tags, partition_exclusions from unshackle.core.utils.click_types import LANGUAGE_RANGE @@ -42,34 +42,38 @@ def test_partition_reads_the_tokens_language_range_produces(): def test_close_match_excludes_the_regional_variants(): - assert excluded_language_tags(["es"], ["es-419", "es-ES", "en-US"]) == {"es-419", "es-ES"} + assert matched_language_tags(["es"], ["es-419", "es-ES", "en-US"], every_variant=True) == {"es-419", "es-ES"} + + +def test_selection_takes_only_the_variant_a_bare_tag_means(): + assert matched_language_tags(["es"], ["es-419", "es-ES", "en-US"]) == {"es-ES"} def test_exact_match_prefers_the_string_equal_tag(): # same string preference as selection: an exact 'es' only takes 'es' when it exists - assert excluded_language_tags(["es"], ["es", "es-ES", "es-419"], exact=True) == {"es"} + assert matched_language_tags(["es"], ["es", "es-ES", "es-419"], exact=True) == {"es"} def test_exact_match_falls_back_to_the_paradigm_variant(): # no literal 'es' available, so 'es' resolves the way an exact '-sl es' selection would - assert excluded_language_tags(["es"], ["es-ES", "es-419"], exact=True) == {"es-ES"} + assert matched_language_tags(["es"], ["es-ES", "es-419"], exact=True) == {"es-ES"} def test_exact_match_keeps_an_unnamed_regional_variant(): - assert excluded_language_tags(["es-ES"], ["es-ES", "es-419"], exact=True) == {"es-ES"} + assert matched_language_tags(["es-ES"], ["es-ES", "es-419"], exact=True) == {"es-ES"} def test_language_objects_are_accepted(): - assert excluded_language_tags(["es"], [Language.get("es")]) == {"es"} + assert matched_language_tags(["es"], [Language.get("es")]) == {"es"} def test_an_untagged_track_is_never_excluded(): - assert excluded_language_tags(["es"], [None, "en-US"]) == set() + assert matched_language_tags(["es"], [None, "en-US"]) == set() def test_no_exclusions_excludes_nothing(): - assert excluded_language_tags([], ["es-419", "en-US"]) == set() + assert matched_language_tags([], ["es-419", "en-US"]) == set() def test_an_unmatched_language_is_kept(): - assert excluded_language_tags(["es", "fr"], ["en-US", "ja-JP"]) == set() + assert matched_language_tags(["es", "fr"], ["en-US", "ja-JP"]) == set() diff --git a/tests/core/test_language_tag_rules.py b/tests/core/test_language_tag_rules.py index 65ddfd7c..cba6f003 100644 --- a/tests/core/test_language_tag_rules.py +++ b/tests/core/test_language_tag_rules.py @@ -20,6 +20,12 @@ def test_a_list_matches_any_of_its_entries() -> None: assert evaluate_language_tag(rules, JA, []) == "ASIAN" +def test_a_regional_rule_does_not_match_another_region() -> None: + rules = [{"audio": "es-419", "tag": "LATAM"}] + assert evaluate_language_tag(rules, [Language.get("es-ES")], []) == "" + assert evaluate_language_tag(rules, [Language.get("es-419")], []) == "LATAM" + + def test_subs_contain_all_needs_every_entry() -> None: rules = [{"subs_contain_all": ["en", "fr"], "tag": "MULTI"}] assert evaluate_language_tag(rules, JA, EN_ES) == "" diff --git a/tests/lang_selection/test_find_missing_langs.py b/tests/lang_selection/test_find_missing_langs.py index e31807a7..ef693a6b 100644 --- a/tests/lang_selection/test_find_missing_langs.py +++ b/tests/lang_selection/test_find_missing_langs.py @@ -15,7 +15,9 @@ from unshackle.core.utilities import find_missing_langs (["fil"], ["en", "ja"], ["fil"]), (["fil", "ko"], ["en"], ["fil", "ko"]), (["es"], ["es-419"], []), - (["es-419"], ["es"], []), + (["es-419"], ["es"], ["es-419"]), + (["es-419"], ["es-MX", "es-ES"], ["es-419"]), + (["en-US"], ["en-GB"], ["en-US"]), (["en"], ["en-US"], []), (["all"], [], []), (["best"], [], []), @@ -34,7 +36,7 @@ from unshackle.core.utilities import find_missing_langs (["zh"], ["zh-Hant"], ["zh"]), (["zh-Hans"], ["zh-Hant"], ["zh-Hans"]), (["zh-CN"], ["zh-TW"], ["zh-CN"]), - (["zh-HK"], ["zh-Hant"], []), + (["zh-HK"], ["zh-Hant"], ["zh-HK"]), (["zh"], ["cmn"], []), (["cmn"], ["zh"], []), (["zh"], ["yue"], ["zh"]), diff --git a/tests/lang_selection/test_forced_sub_selection.py b/tests/lang_selection/test_forced_sub_selection.py deleted file mode 100644 index 379c9bf3..00000000 --- a/tests/lang_selection/test_forced_sub_selection.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import annotations - -import pytest - -from unshackle.core.utilities import keep_forced_subtitle - - -@pytest.mark.parametrize( - "forced,language,forced_s_lang,exact,expected", - [ - (False, "en", [], False, True), - (False, "de", ["en"], False, True), - (True, "en", [], False, False), - (True, "en", ["en"], False, True), - (True, "en-US", ["en"], False, True), - (True, "de", ["en"], False, False), - (True, "de", ["en", "de"], False, True), - # CLDR rates paradigm-regional pairs (en/en-US) distance 0, so exact still matches - (True, "en-US", ["en"], True, True), - (True, "en", ["en"], True, True), - (True, "es-419", ["es-ES"], True, False), - ], -) -def test_keep_forced_subtitle(forced: bool, language: str, forced_s_lang: list[str], exact: bool, expected: bool): - assert keep_forced_subtitle(forced, language, forced_s_lang, exact) is expected diff --git a/tests/lang_selection/test_kind_only_selection.py b/tests/lang_selection/test_kind_only_selection.py index e6a1bbaa..631eae95 100644 --- a/tests/lang_selection/test_kind_only_selection.py +++ b/tests/lang_selection/test_kind_only_selection.py @@ -9,7 +9,7 @@ from __future__ import annotations import inspect import logging -from typing import Any, cast +from typing import Any, Optional, cast import pytest @@ -77,7 +77,13 @@ class FakeService: raise AttributeError(name) -def run(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, episodes: dict, **flags: Any) -> Any: +def run( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + episodes: dict, + service: Optional[FakeService] = None, + **flags: Any, +) -> Any: """Return the Selected ids, the exit code, or None for a run that completes.""" monkeypatch.setattr(Tracks, "tree", lambda self, *a, **k: (_ for _ in ()).throw(Selected(self))) monkeypatch.setattr("unshackle.core.providers.search_metadata", lambda *a, **k: None) @@ -96,7 +102,7 @@ def run(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, episo kwargs.update(flags) caplog.set_level(logging.WARNING, logger="test_kind_only") try: - cmd.result(cast(Service, FakeService(episodes)), **kwargs) + cmd.result(cast(Service, service or FakeService(episodes)), **kwargs) except Selected as sel: return sel.ids except SystemExit as exc: diff --git a/tests/lang_selection/test_matching_languages.py b/tests/lang_selection/test_matching_languages.py index 52a6eb19..8261d921 100644 --- a/tests/lang_selection/test_matching_languages.py +++ b/tests/lang_selection/test_matching_languages.py @@ -4,7 +4,7 @@ from typing import Sequence import pytest -from unshackle.core.utilities import matching_languages +from unshackle.core.utilities import ServiceTag, matching_languages POOL = ["ar", "en", "en-GB", "en-US", "ja"] @@ -12,12 +12,31 @@ POOL = ["ar", "en", "en-GB", "en-US", "ja"] @pytest.mark.parametrize( ("language", "available", "exact", "expected"), [ - ("en", POOL, False, {"en", "en-GB", "en-US"}), + # a bare tag takes the variant it means, and every regional form only when that is missing + ("en", POOL, False, {"en"}), + ("en", ["en-US", "en-GB"], False, {"en-US"}), + ("pt", ["pt-PT", "pt-BR"], False, {"pt-BR"}), + ("es", ["es-419", "es-ES"], False, {"es-ES"}), + ("es", ["es-419", "es-MX"], False, {"es-419", "es-MX"}), + ("es", ["es-419", "es-MX"], True, set()), # CLDR rates en/en-US at distance 0, so exact mode leans on the RFC 4647 string preference ("en", POOL, True, {"en"}), ("en-GB", POOL, True, {"en-GB"}), # no string-equal tag, so exact mode falls back to the distance match ("zh", ["ar", "cmn", "ja"], True, {"cmn"}), + # a region asks for that variant in either mode: never another region, a bare tag only + # when it means the same variant + ("en-US", POOL, False, {"en-US"}), + ("en-US", ["en", "en-GB"], False, {"en"}), + ("en-US", ["en-GB", "en-AU"], False, set()), + ("es-419", ["es-419", "es-MX", "es-ES"], False, {"es-419"}), + ("es-419", ["es-MX", "es-ES"], False, set()), + ("es-419", ["es", "es-ES"], False, set()), + ("pt-BR", ["pt", "pt-PT"], False, {"pt"}), + ("pt-PT", ["pt", "pt-BR"], False, set()), + # a script alone is not a region: zh-Hant means zh-TW, else any Traditional Chinese + ("zh-Hant", ["zh-TW", "zh-HK", "zh-Hans"], False, {"zh-TW"}), + ("zh-Hant", ["zh-HK", "zh-Hans"], False, {"zh-HK"}), ("en", ["ar", "ja"], False, set()), ("en", [], False, set()), ("en", ["ar", None, "en"], True, {"en"}), @@ -31,3 +50,26 @@ POOL = ["ar", "en", "en-GB", "en-US", "ja"] ) def test_matching_languages(language: str, available: Sequence[str], exact: bool, expected: set[str]) -> None: assert matching_languages(language, available, exact) == expected + + +@pytest.mark.parametrize( + ("title", "available", "expected"), + [ + # the title's own variant beats the one a bare tag would mean + ("fr-CA", ["fr-FR", "fr-CA"], {"fr-CA"}), + ("en-GB", ["en-US", "en-GB"], {"en-GB"}), + ("zh-HK", ["zh-TW", "zh-HK", "zh-CN"], {"zh-HK"}), + ("sr-ME", ["sr", "sr-ME"], {"sr-ME"}), + # without its own variant, it takes the close regional forms + ("en-US", ["en-GB", "fr"], {"en-GB"}), + ("zh-TW", ["zh-HK", "zh-CN"], {"zh-HK"}), + ], +) +def test_a_service_tag_prefers_its_own_variant(title: str, available: Sequence[str], expected: set[str]) -> None: + assert matching_languages(ServiceTag(title), available) == expected + + +def test_every_variant_returns_each_regional_form_of_a_bare_tag() -> None: + assert matching_languages("es", ["es-419", "es-ES", "en"], every_variant=True) == {"es-419", "es-ES"} + # a region still asks for one variant + assert matching_languages("es-ES", ["es-419", "es-ES"], every_variant=True) == {"es-ES"} diff --git a/tests/lang_selection/test_original_audio_ids.py b/tests/lang_selection/test_original_audio_ids.py index 74710714..2ad02f1e 100644 --- a/tests/lang_selection/test_original_audio_ids.py +++ b/tests/lang_selection/test_original_audio_ids.py @@ -30,6 +30,9 @@ class FakeTitle: # them and the flag still matches what a default 'orig' download selects (["es"], ["es-419"], ["es-419"]), (["fr"], ["fr-CA"], ["fr-CA"]), + # the service chose the title's region, so a different region still counts + (["en-US"], ["en-GB"], ["en-GB"]), + (["zh-HK"], ["zh-Hant", "zh-Hans"], ["zh-Hant"]), # alias (["zh"], ["cmn"], ["cmn"]), # nothing in the original language, and no language on the title at all diff --git a/tests/lang_selection/test_regional_selection_loop.py b/tests/lang_selection/test_regional_selection_loop.py new file mode 100644 index 00000000..faed3cf6 --- /dev/null +++ b/tests/lang_selection/test_regional_selection_loop.py @@ -0,0 +1,166 @@ +"""Run regional language requests through the real dl.result() selection loop.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from langcodes import Language + +from tests.lang_selection import test_kind_only_selection as harness +from unshackle.core.titles import Episode +from unshackle.core.tracks import Audio, Subtitle, Tracks, Video + + +class RegionalService(harness.FakeService): + """One episode in `title_lang`, with one video, one audio track per entry of `audio` and the given subtitles.""" + + def __init__(self, title_lang: str, audio: list[str], subs: list[tuple[str, str]]) -> None: + super().__init__({1: {}}) + self.title_lang, self.audio, self.subs = title_lang, audio, subs + + def get_titles_cached(self) -> Any: + title = super().get_titles_cached() + for episode in title: + episode.language = Language.get(self.title_lang) + return title + + def get_tracks(self, title: Episode) -> Tracks: + tracks = Tracks() + lang = self.audio[0].split(":")[0] + tracks.add(Video(id_="v", url="https://x/v", language=lang, codec=Video.Codec.AVC, range_=Video.Range.SDR)) + for entry in self.audio: + # "es-ES:EC3" gives the track a codec other than AAC + lang, _, codec = entry.partition(":") + tracks.add( + Audio(id_=entry, url=f"https://x/{entry}", language=lang, codec=Audio.Codec[codec or "AAC"], bitrate=1) + ) + for lang, kind in self.subs: + sdh, forced = kind == "sdh", kind == "forced" + sub_id = f"{lang}{'-' + kind if kind else ''}" + tracks.add( + Subtitle( + id_=sub_id, + url=f"https://x/{sub_id}", + language=lang, + codec=Subtitle.Codec.WebVTT, + sdh=sdh, + forced=forced, + ) + ) + return tracks + + +def run( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + service: RegionalService, + **flags: Any, +) -> Any: + return harness.run(monkeypatch, caplog, {1: {}}, service=service, **flags) + + +def test_orig_takes_another_region_of_the_title_language( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + ids = run(monkeypatch, caplog, RegionalService("en-US", ["en-GB", "fr-FR"], [])) + assert ids["audio"] == ["en-GB"] + assert ids["video"] == ["v"] + + +def test_orig_keeps_the_script_the_title_region_implies( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + ids = run(monkeypatch, caplog, RegionalService("zh-TW", ["zh-HK", "zh-CN"], [])) + assert ids["audio"] == ["zh-HK"] + + +def test_a_regional_audio_request_misses_rather_than_take_a_sibling( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + assert run(monkeypatch, caplog, RegionalService("fr-FR", ["fr-CA"], []), a_lang=["fr-FR"]) == 1 + assert "fr-FR not found in audio tracks" in caplog.text + + +def test_a_regional_sdh_track_is_stripped_when_only_a_sibling_is_plain( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("en-US", ["en-US"], [("en-US", "sdh"), ("en-GB", "")]) + ids = run(monkeypatch, caplog, service, s_lang=["en-US"]) + assert sorted(ids["subs"]) == ["en-US-sdh", "en-US-sdh_stripped"] + + +def test_a_forced_track_does_not_hide_the_plain_track_a_region_means( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("ko", ["ko"], [("es", ""), ("es-ES", "forced"), ("fr", "")]) + assert run(monkeypatch, caplog, service, s_lang=["es-ES"])["subs"] == ["es"] + + +def test_a_bare_sdh_track_is_stripped_when_only_another_variant_is_plain( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("en", ["en"], [("en", "sdh"), ("en-GB", "")]) + assert sorted(run(monkeypatch, caplog, service, s_lang=["en"])["subs"]) == ["en-sdh", "en-sdh_stripped"] + + +def test_a_bare_audio_request_takes_the_variant_it_means( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("en", ["en", "pt-PT", "pt-BR"], []) + assert run(monkeypatch, caplog, service, a_lang=["pt"])["audio"] == ["pt-BR"] + + +def test_orig_takes_the_title_variant_over_the_one_a_bare_tag_means( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + assert run(monkeypatch, caplog, RegionalService("fr-CA", ["fr-FR", "fr-CA", "en"], []))["audio"] == ["fr-CA"] + + +@pytest.mark.parametrize( + ("subs", "flags", "expected"), + [ + # -fs keeps a forced track that -sl names, even when no plain track has its language + ([("en", "forced")], {"s_lang": ["en"], "forced_subs": True}, ["en-forced"]), + ([("es", ""), ("es-ES", "forced")], {"s_lang": ["es-ES"], "forced_subs": True}, ["es", "es-ES-forced"]), + # an SDH track cannot take the plain variant's place, and joins it as the same variant + ([("en-US", ""), ("en", "sdh")], {"s_lang": ["en"]}, ["en-US", "en-sdh"]), + # -fsl matches across every forced track, so the result does not depend on -sl + ([("es-ES", "forced"), ("es-419", "forced")], {"forced_s_lang": ["es"]}, ["es-ES-forced"]), + ( + [("fr", ""), ("es-ES", "forced"), ("es-419", "forced")], + {"s_lang": ["fr"], "forced_s_lang": ["es"]}, + ["es-ES-forced", "fr"], + ), + # -sl orig resolves to the title language + ([("en-GB", ""), ("en-US", "")], {"s_lang": ["orig"]}, ["en-GB"]), + ], +) +def test_subtitle_selection( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, subs: list, flags: dict, expected: list +) -> None: + ids = run(monkeypatch, caplog, RegionalService("en-GB", ["ja"], subs), lang=["ja"], **flags) + assert sorted(ids["subs"]) == expected + + +def test_a_forced_only_language_without_fs_is_reported_missing( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("en", ["ja"], [("en", ""), ("es-419", ""), ("es-ES", "forced")]) + assert run(monkeypatch, caplog, service, lang=["ja"], s_lang=["en", "es-ES"]) == 1 + assert "es-ES not found in subtitle tracks" in caplog.text + + +def test_an_exclusion_removes_every_regional_form( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("ja", ["ja", "es-419", "es-ES"], []) + assert run(monkeypatch, caplog, service, a_lang=["all", "-es"])["audio"] == ["ja"] + + +def test_the_meant_variant_does_not_depend_on_codec( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service = RegionalService("ja", ["ja", "es-ES:EC3", "es-419"], []) + ids = run(monkeypatch, caplog, service, a_lang=["es"], acodec=[Audio.Codec.AAC, Audio.Codec.EC3]) + assert ids["audio"] == ["es-ES:EC3"] diff --git a/tests/lang_selection/test_required_langs.py b/tests/lang_selection/test_required_langs.py index a2016305..fe74d943 100644 --- a/tests/lang_selection/test_required_langs.py +++ b/tests/lang_selection/test_required_langs.py @@ -46,10 +46,12 @@ def test_orig_resolves_to_the_title_language(): assert missing_required_langs(["orig"], ["en"], None) == [] -def test_exact_lang_separates_a_regional_variant_from_its_base(): +def test_a_regional_tag_never_matches_another_region(): assert missing_required_langs(["en-US"], ["en-US"], exact=True) == [] assert missing_required_langs(["en-US"], ["en-GB"], exact=True) == ["en-US"] - assert missing_required_langs(["en-US"], ["en-GB"]) == [] + assert missing_required_langs(["en-US"], ["en-GB"]) == ["en-US"] + # 'orig' is the service's tag, so its region does not narrow the match + assert missing_required_langs(["orig"], ["en-GB"], "en-US") == [] def test_all_and_best_tokens_require_nothing(): @@ -150,3 +152,8 @@ def test_all_three_require_flags_share_one_gate(): for kind in ("audio", "video", "subtitle"): assert f'"{kind}",' in source assert "missing_required_langs(required, available, title.language, exact=exact_lang)" in source + + +def test_exact_lang_turns_off_the_bare_tag_fallback(): + assert missing_required_langs(["en"], ["en-GB"]) == [] + assert missing_required_langs(["en"], ["en-GB"], exact=True) == ["en"] diff --git a/tests/lang_selection/test_resolve_sort_langs.py b/tests/lang_selection/test_resolve_sort_langs.py index cb396650..b6139185 100644 --- a/tests/lang_selection/test_resolve_sort_langs.py +++ b/tests/lang_selection/test_resolve_sort_langs.py @@ -5,7 +5,7 @@ from typing import Optional, Sequence import pytest from langcodes import Language -from unshackle.core.utilities import resolve_sort_langs +from unshackle.core.utilities import ServiceTag, resolve_sort_langs @pytest.mark.parametrize( @@ -13,7 +13,9 @@ from unshackle.core.utilities import resolve_sort_langs [ (["orig", "en", "es"], "ja", ["ja", "en", "es"]), (["orig", "en", "es"], None, ["en", "es"]), + # orig keeps the title's full tag; matching_languages handles its fallback (["orig"], Language.get("pt-BR"), ["pt-BR"]), + (["orig"], "zh-Hant-TW", ["zh-Hant-TW"]), (["en", "orig"], "en", ["en"]), (["en", "es", "en", "fr"], None, ["en", "es", "fr"]), (["all", "best", "en"], "ja", ["all", "best", "en"]), @@ -22,3 +24,8 @@ from unshackle.core.utilities import resolve_sort_langs ) def test_resolve_sort_langs(tokens: Sequence[str], original: Optional[str], expected: list[str]) -> None: assert resolve_sort_langs(tokens, original) == expected + + +def test_orig_is_a_service_tag_outside_exact_mode() -> None: + assert isinstance(resolve_sort_langs(["orig"], "pt-BR")[0], ServiceTag) + assert not isinstance(resolve_sort_langs(["orig"], "pt-BR", exact=True)[0], ServiceTag) diff --git a/tests/tracks/test_mux_default_language.py b/tests/tracks/test_mux_default_language.py new file mode 100644 index 00000000..c3a9ddf0 --- /dev/null +++ b/tests/tracks/test_mux_default_language.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Optional + +import pytest + +from unshackle.core.tracks import Audio, Tracks + + +@pytest.mark.parametrize( + ("languages", "preferred", "expected"), + [ + (["en-GB", "en-US"], "en-US", 1), + # matched against every track at once, so the string-equal tag wins over a bare en + (["en", "en-US"], "en-US", 1), + (["en-GB", "fr"], "en-US", None), + (["fr", "en-GB"], "en", 1), + (["en-US"], None, None), + ], +) +def test_first_match_index(languages: list[str], preferred: Optional[str], expected: Optional[int]) -> None: + tracks = [ + Audio(id_=f"a{i}", url=f"https://x/{i}", language=lang, codec=Audio.Codec.AAC, bitrate=1) + for i, lang in enumerate(languages) + ] + assert Tracks.first_match_index(tracks, preferred) == expected diff --git a/tests/tracks/test_subtitle_sort.py b/tests/tracks/test_subtitle_sort.py index 02170415..9cd7b544 100644 --- a/tests/tracks/test_subtitle_sort.py +++ b/tests/tracks/test_subtitle_sort.py @@ -114,7 +114,7 @@ def test_language_grouping_still_honours_type_priority_and_sections(tracks: Trac def test_language_grouping_keeps_regional_variants_adjacent_through_sections() -> None: - # "en" floats en-GB and en-US as one block, so each exact tag keeps its variants + # "en" floats en-US (the variant it means) and then en-GB, and each tag keeps its variants t = Tracks() t.add( [ @@ -129,11 +129,11 @@ def test_language_grouping_keeps_regional_variants_adjacent_through_sections() - ) t.sort_subtitles(by_language=["en"], group_by="language") assert shape(t) == [ + ("en-US", "normal"), + ("en-US", "sdh"), ("en-GB", "forced"), ("en-GB", "normal"), ("en-GB", "sdh"), - ("en-US", "normal"), - ("en-US", "sdh"), ("fi", "normal"), ("fi", "sdh"), ] @@ -172,7 +172,15 @@ def test_priority_overlapping_entries_are_harmless() -> None: t = Tracks() t.add([sub("ar"), sub("en-GB"), sub("en-US"), sub("es"), sub("ja")]) t.sort_subtitles(by_language=["en", "en-US", "es"], group_by="language") - assert langs(t) == ["en-GB", "en-US", "es", "ar", "ja"] + assert langs(t) == ["en-US", "en-GB", "es", "ar", "ja"] + + +@pytest.mark.parametrize("group_by", ["language", "type"]) +def test_a_regional_priority_outranks_its_sibling(group_by: str) -> None: + t = Tracks() + t.add([sub("en-GB"), sub("en-US"), sub("fr")]) + t.sort_subtitles(by_language=["en-US"], group_by=group_by) + assert langs(t) == ["en-US", "en-GB", "fr"] def test_no_priority_leaves_order_unchanged(many: Tracks) -> None: @@ -223,3 +231,38 @@ def test_unparseable_priority_entry_is_ignored(many: Tracks, exact: bool) -> Non # a config typo in language_priority must not crash the sort many.sort_subtitles(by_language=["en", "engrish", "f", ""], group_by="language", exact_match=exact) assert langs(many) == ["en", "ar", "bg", "es", "fr", "ja"] + + +def test_a_bare_audio_priority_sorts_the_meant_variant_then_its_other_forms() -> None: + from unshackle.core.tracks import Audio + + t = Tracks() + t.add( + [ + Audio(id_=lang, url=f"https://x/{lang}", language=lang, codec=Audio.Codec.AAC, bitrate=bitrate) + for lang, bitrate in (("en", 3), ("es-419", 2), ("es-ES", 1)) + ] + ) + t.sort_audio(by_language=["es"]) + assert [str(a.language) for a in t.audio] == ["es-ES", "es-419", "en"] + + +def test_all_sorts_the_original_track_then_its_other_regional_forms() -> None: + from unshackle.core.tracks import Audio + + t = Tracks() + t.add( + [ + Audio( + id_=lang, + url=f"https://x/{lang}", + language=lang, + codec=Audio.Codec.AAC, + bitrate=bitrate, + is_original_lang=orig, + ) + for lang, bitrate, orig in (("en", 3, False), ("fr-FR", 2, False), ("fr-CA", 1, True)) + ] + ) + t.sort_audio(by_language=["all"]) + assert [str(a.language) for a in t.audio] == ["fr-CA", "fr-FR", "en"] diff --git a/tests/tracks/test_track_name_region.py b/tests/tracks/test_track_name_region.py new file mode 100644 index 00000000..0de53d5d --- /dev/null +++ b/tests/tracks/test_track_name_region.py @@ -0,0 +1,19 @@ +from langcodes import Language + +from unshackle.core.tracks import Audio, Tracks + + +def audio(lang: str, bitrate: int, id_: str) -> Audio: + return Audio(url=f"http://x/{id_}", id_=id_, language=lang, codec=Audio.Codec.AAC, bitrate=bitrate, channels=2.0) + + +def test_naming_a_track_keeps_its_region() -> None: + track = audio("fr-FR", 128000, "a1") + assert track.name is None + assert track.language.territory == "FR" + assert Language.get("fr-FR").territory == "FR" + + +def test_a_regional_request_picks_its_own_variant_over_a_higher_bitrate_sibling() -> None: + tracks = [audio("fr-FR", 128000, "a1"), audio("fr-CA", 192000, "a2")] + assert [str(t.language) for t in Tracks.by_language(tracks, ["fr-FR"])] == ["fr-FR"] diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index bbcb0d52..fa85387a 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -85,19 +85,19 @@ from unshackle.core.utilities import ( as_requested, declared_kwargs, embedded_audio_langs, - excluded_language_tags, find_font_with_fallbacks, find_missing_langs, get_debug_logger, get_system_fonts, init_debug_logger, - is_close_match, - is_exact_match, - keep_forced_subtitle, kind_only_conflict, log_event, + matched_language_tags, + matching_languages, missing_required_langs, + original_tag, partition_exclusions, + plain_subtitle_tags, resolve_sort_langs, suggest_font_packages, time_elapsed_since, @@ -825,7 +825,7 @@ class dl: "--exact-lang", is_flag=True, default=False, - help="Use exact language matching (no variants). With this flag, -l es-419 matches ONLY es-419, not es-ES or other variants.", + help="A language tag without a region selects only the variant it means and never falls back to another: -l es does not select es-419. A tag with a region, such as es-419, is always exact.", ) @click.option( "--proxy", @@ -2122,8 +2122,11 @@ class dl: t for t in audio if t.language == language and (codec is None or t.codec == codec) ] else: - pool = [t for t in audio if codec is None or t.codec == codec] - candidates = Tracks.by_language(pool, [language], per_language=0, exact_match=exact_lang) + # match against every codec at once, so the variant a tag means does not depend on codec + tags = matching_languages(language, [t.language for t in audio], exact_lang) + candidates = [ + t for t in audio if str(t.language) in tags and (codec is None or t.codec == codec) + ] if not candidates: continue if audio_description: @@ -2333,16 +2336,13 @@ class dl: level="INFO", operation="get_tracks", service=self.service, context=tracks_info ) - # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available - # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available - # Check if automatic SDH stripping is enabled in config + # a plain track covers an SDH track only when it is the same variant, as -sl selects them, + # so en-US SDH is stripped next to a plain en-GB: -sl en-US would otherwise keep only the SDH if config.subtitle.get("strip_sdh", True): + plain_langs = [x.language for x in title.tracks.subtitles if not x.sdh and not x.forced] for subtitle in title.tracks.subtitles: - if subtitle.sdh and not any( - is_close_match(subtitle.language, [x.language]) - for x in title.tracks.subtitles - if not x.sdh and not x.forced - ): + if subtitle.sdh and not matching_languages(subtitle.language, plain_langs, exact=True): + plain_langs.append(subtitle.language) non_sdh_sub = deepcopy(subtitle) non_sdh_sub.id += "_stripped" non_sdh_sub.sdh = False @@ -2377,16 +2377,20 @@ class dl: subtitle_priority = config.subtitle.get("language_priority") or [] title.tracks.sort_videos( - by_language=resolve_sort_langs(v_lang or lang, title.language), + by_language=resolve_sort_langs(v_lang or lang, title.language, exact=exact_lang), exact_match=exact_lang, ) title.tracks.sort_audio( - by_language=resolve_sort_langs([*audio_priority, *(a_lang or lang)], title.language), + by_language=resolve_sort_langs( + [*audio_priority, *(a_lang or lang)], title.language, exact=exact_lang + ), codec_priority=config.audio.get("codec_priority"), exact_match=exact_lang, ) title.tracks.sort_subtitles( - by_language=resolve_sort_langs([*subtitle_priority, *(s_lang or [])], title.language), + by_language=resolve_sort_langs( + [*subtitle_priority, *(s_lang or [])], title.language, exact=exact_lang + ), type_priority=config.subtitle.get("type_priority"), group_by=config.subtitle.get("group_by"), exact_match=exact_lang, @@ -2468,22 +2472,10 @@ class dl: title.tracks.chapters = [] with console.status("Selecting tracks...", spinner="dots"): - - def resolve_excludes(excludes: list[str], _title: Title_T = title) -> list[str]: - """Change 'orig' to this title's language. An 'orig' with no language excludes nothing.""" - resolved: list[str] = [] - for token in excludes: - value = str(_title.language) if token == "orig" else token - if token == "orig" and not _title.language: - continue - if value not in resolved: - resolved.append(value) - return resolved - - video_excl_r = resolve_excludes(video_excl) - audio_excl_r = resolve_excludes(audio_excl) - s_excl_r = resolve_excludes(s_lang_excl) - fsl_excl_r = resolve_excludes(fsl_excl) + video_excl_r = resolve_sort_langs(video_excl, title.language, exact=exact_lang) + audio_excl_r = resolve_sort_langs(audio_excl, title.language, exact=exact_lang) + s_excl_r = resolve_sort_langs(s_lang_excl, title.language, exact=exact_lang) + fsl_excl_r = resolve_sort_langs(fsl_excl, title.language, exact=exact_lang) if isinstance(title, (Movie, Episode)): if keep_videos and vcodec: @@ -2531,8 +2523,8 @@ class dl: sys.exit(1) if keep_videos and video_excl_r: - drop = excluded_language_tags( - video_excl_r, [t.language for t in title.tracks.videos], exact_lang + drop = matched_language_tags( + video_excl_r, [t.language for t in title.tracks.videos], exact_lang, every_variant=True ) title.tracks.select_video(lambda x: str(x.language) not in drop) if not title.tracks.videos: @@ -2545,22 +2537,15 @@ class dl: "best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1 ) if keep_videos and video_languages and "all" not in video_languages: - processed_video_lang = [] + processed_video_lang = resolve_sort_langs(video_languages, title.language, exact=exact_lang) v_orig_token: Optional[str] = None - for language in video_languages: - if language == "orig": - if title.language: - orig_lang = str(title.language) - v_orig_token = orig_lang - if orig_lang not in processed_video_lang: - processed_video_lang.append(orig_lang) - else: - self.log.warning( - "Original language not available for title, skipping 'orig' selection for video" - ) + if "orig" in video_languages: + if title.language: + v_orig_token = original_tag(title.language, exact_lang) else: - if language not in processed_video_lang: - processed_video_lang.append(language) + self.log.warning( + "Original language not available for title, skipping 'orig' selection for video" + ) if v_orig_token in video_languages: v_orig_token = None missing_v_langs = find_missing_langs( @@ -2810,21 +2795,19 @@ class dl: self.log.error(msg_detail) sys.exit(1) - fsl = [t for t in forced_s_lang if t != "orig"] - if "orig" in forced_s_lang and title.language: - fsl.append(str(title.language)) + fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang) if keep_subtitles and s_excl_r: # applies to forced subs too: an excluded language is unwanted in any form - drop = excluded_language_tags( - s_excl_r, [t.language for t in title.tracks.subtitles], exact_lang + drop = matched_language_tags( + s_excl_r, [t.language for t in title.tracks.subtitles], exact_lang, every_variant=True ) title.tracks.select_subtitles(lambda x: str(x.language) not in drop) if keep_subtitles and s_lang and "all" not in s_lang: - match_func = is_exact_match if exact_lang else is_close_match - + s_lang = resolve_sort_langs(s_lang, title.language, exact=exact_lang) + # the pool selection draws from: forced tracks count only when -fs keeps them missing_langs = find_missing_langs( s_lang, - [sub.language for sub in title.tracks.subtitles], + [sub.language for sub in title.tracks.subtitles if forced_subs or not sub.forced], exact=exact_lang, ) if missing_langs: @@ -2840,21 +2823,21 @@ class dl: self.log.warning( f"{missing_str} not found in subtitle tracks, continuing without subtitles" ) - title.tracks.select_subtitles( - lambda x: ( - x.forced and keep_forced_subtitle(x.forced, x.language, fsl, exact_lang) - ) + fsl_tags = matched_language_tags( + fsl, [x.language for x in title.tracks.subtitles if x.forced], exact_lang ) + title.tracks.select_subtitles(lambda x: x.forced and str(x.language) in fsl_tags) else: self.log.error(missing_str + " not found in subtitle tracks") sys.exit(1) if s_lang and title.tracks.subtitles: + plain_tags = plain_subtitle_tags(s_lang, title.tracks.subtitles, exact_lang) + forced_tags = matched_language_tags( + [*s_lang, *fsl], [x.language for x in title.tracks.subtitles if x.forced], exact_lang + ) title.tracks.select_subtitles( - lambda x: ( - match_func(x.language, s_lang) - or bool(x.forced and fsl and match_func(x.language, fsl)) - ) + lambda x: str(x.language) in (forced_tags if x.forced else plain_tags) ) if not title.tracks.subtitles and not best_available: self.log.error(f"There's no {s_lang} Subtitle Track...") @@ -2863,9 +2846,10 @@ class dl: if keep_subtitles: # fsl, not forced_s_lang: unresolvable orig must keep all forced, not drop them if fsl: - title.tracks.select_subtitles( - lambda x: keep_forced_subtitle(x.forced, x.language, fsl, exact_lang) + fsl_tags = matched_language_tags( + fsl, [x.language for x in title.tracks.subtitles if x.forced], exact_lang ) + title.tracks.select_subtitles(lambda x: not x.forced or str(x.language) in fsl_tags) if not any(x.forced for x in title.tracks.subtitles): self.log.warning( f"No forced subtitle tracks matched --forced-s-lang ({', '.join(forced_s_lang)})" @@ -2873,8 +2857,11 @@ class dl: elif not forced_subs: title.tracks.select_subtitles(lambda x: not x.forced) if fsl_excl_r: - drop = excluded_language_tags( - fsl_excl_r, [t.language for t in title.tracks.subtitles if t.forced], exact_lang + drop = matched_language_tags( + fsl_excl_r, + [t.language for t in title.tracks.subtitles if t.forced], + exact_lang, + every_variant=True, ) title.tracks.select_subtitles(lambda x: not (x.forced and str(x.language) in drop)) if forced_subs_only: @@ -2928,8 +2915,8 @@ class dl: self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...") sys.exit(1) if audio_excl_r: - drop = excluded_language_tags( - audio_excl_r, [t.language for t in title.tracks.audio], exact_lang + drop = matched_language_tags( + audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True ) title.tracks.select_audio(lambda x: str(x.language) not in drop) if not title.tracks.audio: @@ -2937,22 +2924,13 @@ class dl: sys.exit(1) audio_languages = a_lang or lang if audio_languages: - processed_lang = [] + processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang) a_orig_token: Optional[str] = None - for language in audio_languages: - if language == "orig": - if title.language: - orig_lang = str(title.language) - a_orig_token = orig_lang - if orig_lang not in processed_lang: - processed_lang.append(orig_lang) - else: - self.log.warning( - "Original language not available for title, skipping 'orig' selection" - ) + if "orig" in audio_languages: + if title.language: + a_orig_token = original_tag(title.language, exact_lang) else: - if language not in processed_lang: - processed_lang.append(language) + self.log.warning("Original language not available for title, skipping 'orig' selection") if a_orig_token in audio_languages: a_orig_token = None diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 9407b22d..81a458a4 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -36,7 +36,7 @@ from unshackle.core.session import utf8_text from unshackle.core.titles import Episode, Movie, Song, Title_T from unshackle.core.titles.music import SONG_FIELDS from unshackle.core.tracks import Audio, Subtitle, Tracks, Video -from unshackle.core.utilities import declared_kwargs, kind_only_conflict +from unshackle.core.utilities import declared_kwargs, kind_only_conflict, original_tag from unshackle.core.utils.click_types import AUDIO_CODEC_LIST, SUBTITLE_CODEC, VIDEO_CODEC_LIST from unshackle.core.utils.collections import ci_get from unshackle.core.utils.redact import REDACTED, URL_USERINFO_RE, redact_all, redact_secrets, redact_text @@ -1429,20 +1429,13 @@ def serialize_video_track(track: Video, include_url: bool = False) -> Dict[str, def original_audio_ids(tracks: List[Audio], title: Title_T) -> set: """Return the ids of the audio tracks 'orig' resolves to, empty when the title has no language. - This defers to Tracks.by_language so the flag agrees with the downloader. It asks - exact mode first because CLDR rates a base tag and its paradigm regional variant as - the same language, and only the RFC 4647 preference picks one ('en' over 'en-US' - when both exist, 'pt-BR' over 'pt-PT' for a 'pt' title). The fuzzy fallback then - catches the non-paradigm regionals exact mode drops, such as an 'es' title that - carries only 'es-419'. + This defers to Tracks.by_language through original_tag, as 'orig' does, so the flag agrees + with the downloader: the title's own variant first, else its close regional forms. """ language = getattr(title, "language", None) if not language: return set() - matches = Tracks.by_language(tracks, [str(language)], exact_match=True) or Tracks.by_language( - tracks, [str(language)] - ) - return {t.id for t in matches} + return {t.id for t in Tracks.by_language(tracks, [original_tag(language)])} def serialize_audio_track(track: Audio, include_url: bool = False, is_original: bool = False) -> Dict[str, Any]: diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index a4768711..04366eee 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -909,7 +909,7 @@ async def download(request: web.Request) -> web.Response: description: Languages wanted for forced subtitles, implies forced_subs (a '-' prefix excludes) (default - []) exact_lang: type: boolean - description: Use exact language matching (no variants) (default - false) + description: A language tag without a region never falls back to another variant; a tag with a region is always exact (default - false) sub_format: type: string description: Output subtitle format such as SRT or VTT, or "original" to keep the source format (default - None) diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index de5699a8..2e5a2e17 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -508,9 +508,9 @@ class Track: self.extra: Any = extra or {} # allow anything for extra, but default to a dict if self.name is None: - lang = Language.get(self.language) + lang = self.language if (lang.language or "").lower() == (lang.territory or "").lower(): - lang.territory = None + lang = lang.update_dict({"territory": None}) reduced = lang.simplify_script() extra_parts = [] if reduced.script is not None: diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 5b687d0a..5b17c889 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -30,6 +30,7 @@ from unshackle.core.utilities import ( is_close_match, log_event, matching_languages, + original_tag, sanitize_filename, valid_language, ) @@ -270,12 +271,14 @@ class Tracks: self.videos.sort(key=lambda x: (x.height or 0, float(x.bitrate or 0.0)), reverse=True) for language in reversed(by_language or []): if str(language) in ("all", "best"): - language = next((x.language for x in self.videos if x.is_original_lang), "") + language = self._original_language(self.videos, exact_match) if not language: continue - wanted = matching_languages(language, [x.language for x in self.videos], exact_match) + langs = [x.language for x in self.videos] + wanted = matching_languages(language, langs, exact_match) + related = matching_languages(language, langs, exact_match, every_variant=True) self.videos.sort(key=lambda x: str(x.language)) - self.videos.sort(key=lambda x: str(x.language) not in wanted) + self.videos.sort(key=lambda x: (str(x.language) not in related, str(x.language) not in wanted)) def sort_audio( self, @@ -298,11 +301,13 @@ class Tracks: self.audio.sort(key=lambda x: x.descriptive) for language in reversed(by_language or []): if str(language) in ("all", "best"): - language = next((x.language for x in self.audio if x.is_original_lang), "") + language = self._original_language(self.audio, exact_match) if not language: continue - wanted = matching_languages(language, [x.language for x in self.audio], exact_match) - self.audio.sort(key=lambda x: str(x.language) not in wanted) + langs = [x.language for x in self.audio] + wanted = matching_languages(language, langs, exact_match) + related = matching_languages(language, langs, exact_match, every_variant=True) + self.audio.sort(key=lambda x: (str(x.language) not in related, str(x.language) not in wanted)) def sort_subtitles( self, @@ -365,11 +370,13 @@ class Tracks: by_type() for language in reversed(by_language or []): if str(language) in ("all", "best"): - language = next((x.language for x in self.subtitles if x.is_original_lang), "") + language = self._original_language(self.subtitles, exact_match) if not language: continue - wanted = matching_languages(language, [x.language for x in self.subtitles], exact_match) - self.subtitles.sort(key=lambda x: str(x.language) in wanted, reverse=True) + langs = [x.language for x in self.subtitles] + wanted = matching_languages(language, langs, exact_match) + related = matching_languages(language, langs, exact_match, every_variant=True) + self.subtitles.sort(key=lambda x: (str(x.language) not in related, str(x.language) not in wanted)) def select_video(self, x: Callable[[Video], bool]) -> None: self.videos = list(filter(x, self.videos)) @@ -496,6 +503,23 @@ class Tracks: selected.append(track) return selected + @staticmethod + def _original_language(tracks: Sequence[AnyTrack], exact: bool) -> str: + """The language "all" and "best" sort first: the original-language track's, through original_tag.""" + language = next((x.language for x in tracks if x.is_original_lang), None) + return original_tag(language, exact) if language else "" + + @staticmethod + def first_match_index(tracks: Sequence[AnyTrack], language: Optional[str]) -> Optional[int]: + """The index of the first track that `language` asks for, matched against every track at once. + + Matching the whole list lets the RFC 4647 string preference pick en-US over en. + """ + if not language: + return None + wanted = matching_languages(language, [t.language for t in tracks]) + return next((idx for idx, t in enumerate(tracks) if str(t.language) in wanted), None) + @property def sample_base(self) -> float: """The earliest ``sample_start`` of the video and audio tracks, which a sample mux uses as time zero.""" @@ -769,26 +793,11 @@ class Tracks: preferred_audio_lang = valid_language(default_language.get("audio")) preferred_subtitle_lang = valid_language(default_language.get("subtitle")) - preferred_video_idx: Optional[int] = None - if preferred_video_lang: - preferred_video_idx = next( - (idx for idx, v in enumerate(self.videos) if is_close_match(v.language, [preferred_video_lang])), - None, - ) - - preferred_audio_idx: Optional[int] = None - if preferred_audio_lang: - preferred_audio_idx = next( - (idx for idx, a in enumerate(self.audio) if is_close_match(a.language, [preferred_audio_lang])), - None, - ) - - preferred_subtitle_idx: Optional[int] = None - if preferred_subtitle_lang and not skip_subtitles: - preferred_subtitle_idx = next( - (idx for idx, s in enumerate(self.subtitles) if is_close_match(s.language, [preferred_subtitle_lang])), - None, - ) + preferred_video_idx = self.first_match_index(self.videos, preferred_video_lang) + preferred_audio_idx = self.first_match_index(self.audio, preferred_audio_lang) + preferred_subtitle_idx = ( + None if skip_subtitles else self.first_match_index(self.subtitles, preferred_subtitle_lang) + ) for i, vt in enumerate(self.videos): if not vt.path or not vt.path.exists(): diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 03c9ee92..e555fe44 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -205,34 +205,36 @@ def partition_exclusions(tokens: Optional[Sequence[str]]) -> tuple[list[str], li return includes, excludes -def excluded_language_tags( - excludes: Sequence[str], +def matched_language_tags( + tokens: Sequence[str], available: Sequence[Union[str, Language, None]], exact: bool = False, + *, + every_variant: bool = False, ) -> set[str]: """ - The language tags out of `available` that the exclusion tokens remove. + The language tags out of `available` that any of the tokens asks for. - Matches through matching_languages so an exclusion drops exactly the tracks the same - token would select, including its exact-mode string preference. Untagged tracks are - never excluded. + Matches through matching_languages. An exclusion passes every_variant, so "-es" drops + every Spanish track. Untagged tracks never match. """ - return {tag for token in excludes for tag in matching_languages(token, available, exact)} + return {tag for token in tokens for tag in matching_languages(token, available, exact, every_variant=every_variant)} -def keep_forced_subtitle( - forced: bool, - language: Union[str, Language], - forced_s_lang: Sequence[str], - exact: bool = False, -) -> bool: - """The -fsl filter: non-forced tracks always pass, forced tracks only when they match forced_s_lang.""" - if not forced: - return True - if not forced_s_lang: - return False - match_func = is_exact_match if exact else is_close_match - return match_func(language, forced_s_lang) +def plain_subtitle_tags(tokens: Sequence[str], subtitles: Sequence[Any], exact: bool = False) -> set[str]: + """ + The non-forced subtitle tags the -sl tokens select. + + Each token is matched against the plain tracks before the SDH ones, so an SDH track cannot win + the string-equal preference over the plain variant: -sl en with plain en-US and SDH en takes + en-US. An SDH track then joins when it is the same variant as a chosen tag. + """ + plain = [x.language for x in subtitles if not x.forced and not x.sdh] + sdh = [x.language for x in subtitles if not x.forced and x.sdh] + tags: set[str] = set() + for token in tokens: + tags |= matching_languages(token, plain, exact) or matching_languages(token, sdh, exact) + return tags | {str(lang) for lang in sdh if tags and matching_languages(lang, list(tags), exact=True)} def kind_only_conflict(params: Mapping[str, Any]) -> Optional[tuple[str, str]]: @@ -296,9 +298,8 @@ def find_missing_langs( exact: bool = False, ) -> list[str]: """Return the requested language tokens that match no available language.""" - match_func = is_exact_match if exact else is_close_match skip = {"all", "best", "orig"} - return [tok for tok in requested if tok not in skip and not match_func(tok, available)] + return [tok for tok in requested if tok not in skip and not matching_languages(tok, available, exact)] def missing_required_langs( @@ -315,7 +316,7 @@ def missing_required_langs( token ("-fr") asks for a language to be absent, so it is never a required language. """ wanted, _ = partition_exclusions(list(required)) - return find_missing_langs(resolve_sort_langs(wanted, original), available, exact=exact) + return find_missing_langs(resolve_sort_langs(wanted, original, exact=exact), available, exact=exact) def as_requested(tokens: Sequence[str], orig_token: Optional[str]) -> str: @@ -327,40 +328,65 @@ def matching_languages( language: Union[str, Language], available: Sequence[Union[str, Language, None]], exact: bool = False, + *, + every_variant: bool = False, ) -> set[str]: """ The languages out of `available` that `language` asks for, by the rules Tracks.by_language uses. - In exact mode CLDR tag distance is not sufficient on its own: it rates a base language and - its "paradigm" regional variant as the same language (distance 0 for en/en-US, pt/pt-BR), so - it cannot separate en-US from en. Follow RFC 4647 Lookup: prefer the string-equal tag, and - fall back to the distance rating only when no such tag is present (for example, zh matches - cmn). + A tag asks for the variant it means. A tag with a region (en-US, es-419) gets only that + variant. A bare tag gets the variant it implies (pt means pt-BR, es means es-ES) and, when + the title has none, every regional form, unless exact or the caller sets every_variant, + which always returns every form. A ServiceTag falls back the same way, from its own tag. The meant variant follows RFC 4647 Lookup: the string-equal + tag, else CLDR distance 0 (zh matches cmn, en-US matches en). Distance alone cannot do it, + because it rates a base language and its paradigm variant (en/en-US) the same. """ tags = {str(x) for x in available if x} try: - want = Language.get(str(language)).to_tag().casefold() + requested = Language.get(str(language)) except LanguageTagError: # e.g. a config typo in language_priority; an unparseable tag matches nothing return set() - if not exact: - return {tag for tag in tags if is_close_match(language, [tag])} + want = requested.to_tag().casefold() + variants: set[str] = set() + if not exact and (isinstance(language, ServiceTag) or not requested.territory): + variants = {tag for tag in tags if is_close_match(language, [tag])} + if every_variant: + return variants string_equal = {tag for tag in tags if Language.get(tag).to_tag().casefold() == want} - return string_equal or {tag for tag in tags if is_exact_match(language, [tag])} + return string_equal or {tag for tag in tags if is_exact_match(language, [tag])} or variants -def resolve_sort_langs(tokens: Sequence[str], original: Optional[Union[str, Language]] = None) -> list[str]: +class ServiceTag(str): + """A language the service supplied, such as the title language "orig" becomes. + + matching_languages takes its own variant first and falls back to the close regional forms, + because the service, not the user, chose its region. """ - Prepare language tokens for a Tracks.sort_* by_language argument. - "orig" becomes the title language. This function drops it when the title has none. - "all" and "best" pass through, because the sort methods find them among the tracks. The - result keeps the first position of each language and drops later repeats. + +def original_tag(language: Union[str, Language], exact: bool = False) -> str: + """The tag to match a service-supplied language with: a ServiceTag, or the plain tag in exact mode.""" + return str(language) if exact else ServiceTag(str(language)) + + +def resolve_sort_langs( + tokens: Sequence[str], original: Optional[Union[str, Language]] = None, *, exact: bool = False +) -> list[str]: + """ + Resolve language tokens for selection, exclusion and the Tracks.sort_* by_language argument. + + "orig" becomes the title language through original_tag. This function drops it when the + title has none. "all" and "best" pass through, because the sort methods find them among + the tracks. The result keeps the first position of each language and drops later repeats. """ resolved: list[str] = [] for token in tokens: - language = str(original) if token == "orig" else str(token) - if token == "orig" and not original: + if token != "orig": + language = str(token) + elif original: + language = original_tag(original, exact) + else: continue if language not in resolved: resolved.append(language) diff --git a/unshackle/core/utils/language_tags.py b/unshackle/core/utils/language_tags.py index 1881b908..f1c861f1 100644 --- a/unshackle/core/utils/language_tags.py +++ b/unshackle/core/utils/language_tags.py @@ -7,9 +7,8 @@ from collections.abc import Mapping as MappingABC from typing import Any, Mapping, Optional, Sequence from langcodes import Language -from langcodes.tag_parser import LanguageTagError -from unshackle.core.utilities import is_close_match +from unshackle.core.utilities import matching_languages, valid_language log = logging.getLogger(__name__) @@ -116,13 +115,9 @@ def _as_list(value: Any) -> list[Any]: def _matches(lang: Any, languages: Sequence[Language]) -> bool: - """Return True when the rule language closely matches one of the track languages. + """Return True when the rule language matches one of the track languages, as a -l token would. A rule value that is not a valid language tag counts as no match. The filename is built after the mux, so a typo in the config must not discard a finished download. """ - try: - return is_close_match(lang, list(languages)) - except LanguageTagError as e: - log.warning("Language tag rule value %r is not a valid language tag, treating as no match: %s", lang, e) - return False + return bool(valid_language(lang) and matching_languages(lang, languages))