diff --git a/docs/reference/configuration/download.md b/docs/reference/configuration/download.md index b2349163..c685214b 100644 --- a/docs/reference/configuration/download.md +++ b/docs/reference/configuration/download.md @@ -52,6 +52,13 @@ Common config keys. This is a useful subset, and every `dl` flag works: | `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | | `no_folder` | bool | `false` | Do not create a per-title folder. | +!!! note "`no` and `yes` are language tags" + YAML reads a bare `no` as `false` and a bare `yes` as `true`. unshackle puts both back to + the tags `no` (Norwegian) and `yes` (Nyankpa) in every config key that takes a language: `lang`, + `v_lang`, `a_lang`, `s_lang`, `forced_s_lang`, `require_audio`, `require_video`, + `require_subs`, [`language_priority`](#subtitle), [`muxing.default_language`](#muxing) and + the [`language_tags`](#language_tags) rules. Quotes (`lang: "no"`) work as well. + ```yaml title="Global download defaults" dl: lang: [en] @@ -253,9 +260,22 @@ Language-tag remapping and the rule engine behind the `{lang_tag}` filename vari | Condition | Meaning | |-----------|---------| -| `audio` | Matches against the audio track languages. | +| `audio` | At least one audio language matches. | | `subs_contain` | At least one subtitle language matches. | -| `subs_contain_all` | Every listed language must be present (scalar or list). | +| `subs_contain_all` | Every listed language must be present. | +| `dual` / `multi` / `dubbed` | `true` matches when that filename variable is set, `false` when it is not. | + +Each language condition takes one language tag or a list of them. A list matches when **any** +entry matches, except `subs_contain_all`, which needs them all. An entry that is not a valid +language tag (for example `japanese`) never matches. `dual`, `multi` and `dubbed` take a bare +`true` or `false`, not a quoted string. unshackle skips a rule with a condition name it does +not know, or with any other value in a boolean condition. It logs a warning in each of these +cases and tries the next rule. + +`dual`, `multi` and `dubbed` carry the same meaning as the +[filename variables of those names](#dual_multi_mode). Pair one with a language to name a +release yourself, in place of the fixed `DUAL`, `MULTi` and `DUBBED` text. A rule that names +`dubbed: true` and a language tags a localized dub without tagging a native release the same way. ```yaml language_tags: @@ -263,8 +283,18 @@ language_tags: - audio: [ja] subs_contain: [en] tag: "SUBBED" + - audio: th + dubbed: true + tag: "THAIDUB" + - audio: th + dual: true + tag: "THAIDUAL" ``` +!!! note "Put `{lang_tag}` where you want the name" + These rules fill `{lang_tag}`. To use your own name in place of the fixed text, keep + `{lang_tag?}` in the template and leave out `{dual?}`, `{multi?}` and `{dubbed?}`. + ## `dual_multi_mode` - **Type:** `str`  ยท  **Default:** `strict` diff --git a/tests/core/test_dual_multi_naming.py b/tests/core/test_dual_multi_naming.py index b5b8f839..04b11cac 100644 --- a/tests/core/test_dual_multi_naming.py +++ b/tests/core/test_dual_multi_naming.py @@ -118,3 +118,10 @@ def test_count_never_dubbed(count_mode): assert c["dubbed"] == "" assert c["dual"] == "" assert c["multi"] == "" + + +def test_a_missing_template_key_falls_back_instead_of_raising(monkeypatch): + """dl only checks output_template is non-empty, so the title's own key can be absent.""" + monkeypatch.setattr(config, "output_template", {"series": "{title}"}) + name = make_movie("en").get_filename(make_media_info(["en"])) + assert name == "The Film (2024)" diff --git a/tests/core/test_language_tag_rules.py b/tests/core/test_language_tag_rules.py new file mode 100644 index 00000000..65ddfd7c --- /dev/null +++ b/tests/core/test_language_tag_rules.py @@ -0,0 +1,74 @@ +"""Rules accept both the scalar and the list form of every condition.""" + +import pytest +from langcodes import Language + +from unshackle.core.utils.language_tags import evaluate_language_tag + +JA = [Language.get("ja")] +EN_ES = [Language.get("en-US"), Language.get("es")] + + +@pytest.mark.parametrize("audio, subs", [("ja", "en"), (["ja"], ["en"])]) +def test_scalar_and_list_forms_both_match(audio, subs) -> None: + rules = [{"audio": audio, "subs_contain": subs, "tag": "SUBBED"}] + assert evaluate_language_tag(rules, JA, EN_ES) == "SUBBED" + + +def test_a_list_matches_any_of_its_entries() -> None: + rules = [{"audio": ["ko", "ja"], "tag": "ASIAN"}] + assert evaluate_language_tag(rules, JA, []) == "ASIAN" + + +def test_subs_contain_all_needs_every_entry() -> None: + rules = [{"subs_contain_all": ["en", "fr"], "tag": "MULTI"}] + assert evaluate_language_tag(rules, JA, EN_ES) == "" + + +@pytest.mark.parametrize("bad", ["japanese", 1, [["ja"]], {"ja": 1}]) +def test_an_invalid_rule_value_is_no_match_not_a_crash(bad, caplog) -> None: + rules = [{"audio": bad, "tag": "BAD"}, {"audio": "ja", "tag": "OK"}] + assert evaluate_language_tag(rules, JA, EN_ES) == "OK" + assert "not a valid language tag" in caplog.text + + +def test_dubbed_separates_a_localized_dub_from_a_native_release() -> None: + rules = [{"audio": "th", "dubbed": True, "tag": "THAIDUB"}, {"audio": "th", "tag": "THAI"}] + th = [Language.get("th")] + assert evaluate_language_tag(rules, th, [], {"dubbed": True}) == "THAIDUB" + assert evaluate_language_tag(rules, th, [], {"dubbed": False}) == "THAI" + + +@pytest.mark.parametrize("state", ["dual", "multi", "dubbed"]) +def test_every_state_condition_matches_both_ways(state) -> None: + rules = [{state: True, "tag": "ON"}, {state: False, "tag": "OFF"}] + assert evaluate_language_tag(rules, JA, [], {state: True}) == "ON" + assert evaluate_language_tag(rules, JA, [], {state: False}) == "OFF" + assert evaluate_language_tag(rules, JA, [], None) == "OFF" + + +def test_a_state_condition_combines_with_a_language() -> None: + rules = [{"audio": "th", "dual": True, "tag": "THAIDUAL"}, {"dual": True, "tag": "DUAL"}] + states = {"dual": True} + assert evaluate_language_tag(rules, [Language.get("th")], [], states) == "THAIDUAL" + assert evaluate_language_tag(rules, JA, [], states) == "DUAL" + + +@pytest.mark.parametrize("bad", ["yes", "false", 1]) +def test_a_non_boolean_state_value_skips_the_rule(bad, caplog) -> None: + rules = [{"dubbed": bad, "tag": "BAD"}, {"audio": "ja", "tag": "OK"}] + assert evaluate_language_tag(rules, JA, [], {"dubbed": True}) == "OK" + assert evaluate_language_tag(rules, JA, [], {"dubbed": False}) == "OK" + assert "must be true or false" in caplog.text + + +@pytest.mark.parametrize("typo", ["dubed", "subs_contains"]) +def test_a_typo_in_a_condition_name_skips_the_rule(typo, caplog) -> None: + rules = [{"audio": "ja", typo: True, "tag": "BAD"}, {"audio": "ja", "tag": "OK"}] + assert evaluate_language_tag(rules, JA, [], {"dubbed": True}) == "OK" + assert f"unknown key(s) {typo}" in caplog.text + + +@pytest.mark.parametrize("rules", ["notalist", ["foo"], {"audio": "ja"}, [None]]) +def test_a_malformed_rules_block_is_ignored_not_a_crash(rules) -> None: + assert evaluate_language_tag(rules, JA, []) == "" diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index 3c403984..d810f1db 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -258,7 +258,11 @@ class Episode(Title): name += f" {self.folder_season()}" return sanitize_filename(name, " ") - formatter = TemplateFormatter(config.output_template["series"]) + template = config.output_template.get("series") + if not template: + return sanitize_filename(str(self), " ") + + formatter = TemplateFormatter(template) context = self.build_template_context(media_info, show_service) return formatter.format(context) diff --git a/unshackle/core/titles/movie.py b/unshackle/core/titles/movie.py index 429e1ee7..fdb718f7 100644 --- a/unshackle/core/titles/movie.py +++ b/unshackle/core/titles/movie.py @@ -76,7 +76,11 @@ class Movie(Title): name += f" ({self.year})" return sanitize_filename(name, " ") - formatter = TemplateFormatter(config.output_template["movies"]) + template = config.output_template.get("movies") + if not template: + return sanitize_filename(str(self), " ") + + formatter = TemplateFormatter(template) context = self.build_template_context(media_info, show_service) return formatter.format(context) diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index 1d3cec39..d98be289 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -213,13 +213,15 @@ class Title: if original_lang_tag and original_lang_tag not in audio_lang_bases: context["dubbed"] = "DUBBED" - lang_tag_rules = config.language_tags.get("rules") if config.language_tags else None + language_tags = config.language_tags if isinstance(config.language_tags, dict) else {} + lang_tag_rules = language_tags.get("rules") if lang_tag_rules and self.tracks: from unshackle.core.utils.language_tags import evaluate_language_tag audio_langs = [a.language for a in self.tracks.audio] sub_langs = [s.language for s in self.tracks.subtitles] - context["lang_tag"] = evaluate_language_tag(lang_tag_rules, audio_langs, sub_langs) + states = {name: bool(context[name]) for name in ("dual", "multi", "dubbed")} + context["lang_tag"] = evaluate_language_tag(lang_tag_rules, audio_langs, sub_langs, states) if config.tag_rules: from unshackle.core.utils.tag_rules import evaluate_tag_rules diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index a48589b3..4e21550a 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -26,7 +26,13 @@ from unshackle.core.tracks.chapters import Chapter, Chapters from unshackle.core.tracks.subtitle import Subtitle from unshackle.core.tracks.track import Track, has_dts_uhd_sample_entry, strip_duplicate_init_boxes from unshackle.core.tracks.video import Video -from unshackle.core.utilities import is_close_match, log_event, matching_languages, sanitize_filename +from unshackle.core.utilities import ( + is_close_match, + log_event, + matching_languages, + sanitize_filename, + valid_language, +) from unshackle.core.utils.collections import as_list, flatten MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)") @@ -727,9 +733,10 @@ class Tracks: cl.extend(["--title", title]) default_language = config.muxing.get("default_language") or {} - preferred_video_lang = default_language.get("video") - preferred_audio_lang = default_language.get("audio") - preferred_subtitle_lang = default_language.get("subtitle") + # mux() runs after every track is downloaded, so a typo here must not discard the title + preferred_video_lang = valid_language(default_language.get("video")) + preferred_audio_lang = valid_language(default_language.get("audio")) + preferred_subtitle_lang = valid_language(default_language.get("subtitle")) preferred_video_idx: Optional[int] = None if preferred_video_lang: diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index afb98f64..2e316bed 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -153,6 +153,18 @@ def is_close_match(language: Union[str, Language], languages: Sequence[Union[str return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_MAX_DISTANCE +def valid_language(language: Any) -> Optional[str]: + """Return the language tag as text, or None when it is not a valid tag.""" + if not language: + return None + try: + Language.get(str(language)) + except LanguageTagError: + logging.getLogger("utilities").warning("%r is not a valid language tag, ignoring it", language) + return None + return str(language) + + def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: """Examine whether a language exactly matches any of the given languages.""" languages = [x for x in languages if x] diff --git a/unshackle/core/utils/language_tags.py b/unshackle/core/utils/language_tags.py index ac1d6552..1881b908 100644 --- a/unshackle/core/utils/language_tags.py +++ b/unshackle/core/utils/language_tags.py @@ -3,19 +3,26 @@ from __future__ import annotations import logging -from typing import Any, Sequence +from collections.abc import Mapping as MappingABC +from typing import Any, Mapping, Optional, Sequence from langcodes import Language +from langcodes.tag_parser import LanguageTagError from unshackle.core.utilities import is_close_match log = logging.getLogger(__name__) +# Conditions that test a filename variable of the same name rather than a language. +STATE_CONDITIONS = ("dual", "multi", "dubbed") +LANGUAGE_CONDITIONS = ("audio", "subs_contain", "subs_contain_all") + def evaluate_language_tag( rules: list[dict[str, Any]], audio_languages: Sequence[Language], subtitle_languages: Sequence[Language], + states: Optional[Mapping[str, bool]] = None, ) -> str: """Evaluate language tag rules against selected tracks. @@ -26,17 +33,27 @@ def evaluate_language_tag( rules: List of rule dicts from config, each with conditions and a ``tag``. audio_languages: Languages of the selected audio tracks. subtitle_languages: Languages of the selected subtitle tracks. + states: Whether each of ``dual``, ``multi`` and ``dubbed`` is set for this release, + the same state as the filename variable of that name. Returns: The tag string from the first rule that matches, or ``""`` if none match. """ + if not isinstance(rules, list): + log.warning("Language tag rules must be a list of rules, ignoring: %r", rules) + return "" + for rule in rules: + if not isinstance(rule, MappingABC): + log.warning("Language tag rule must be a mapping of conditions and a 'tag', skipping: %r", rule) + continue + tag = rule.get("tag") if not tag: log.warning("Language tag rule missing 'tag' field, skipping: %s", rule) continue - if rule_matches(rule, audio_languages, subtitle_languages): + if rule_matches(rule, audio_languages, subtitle_languages, states): log.debug("Language tag rule matched: %s -> %s", rule, tag) return str(tag) @@ -47,29 +64,43 @@ def rule_matches( rule: dict[str, Any], audio_languages: Sequence[Language], subtitle_languages: Sequence[Language], + states: Optional[Mapping[str, bool]] = None, ) -> bool: """Return True when every condition in the rule matches the tracks.""" + unknown = [key for key in rule if key not in (*LANGUAGE_CONDITIONS, *STATE_CONDITIONS, "tag")] + if unknown: + log.warning("Language tag rule has unknown key(s) %s, skipping: %s", ", ".join(sorted(unknown)), rule) + return False + has_condition = False audio_lang = rule.get("audio") if audio_lang is not None: has_condition = True - if not is_close_match(audio_lang, list(audio_languages)): + if not any(_matches(lang, audio_languages) for lang in _as_list(audio_lang)): return False subs_contain = rule.get("subs_contain") if subs_contain is not None: has_condition = True - if not is_close_match(subs_contain, list(subtitle_languages)): + if not any(_matches(lang, subtitle_languages) for lang in _as_list(subs_contain)): return False subs_contain_all = rule.get("subs_contain_all") if subs_contain_all is not None: has_condition = True - if not isinstance(subs_contain_all, list): - subs_contain_all = [subs_contain_all] - for lang in subs_contain_all: - if not is_close_match(lang, list(subtitle_languages)): + for lang in _as_list(subs_contain_all): + if not _matches(lang, subtitle_languages): + return False + + for name in STATE_CONDITIONS: + wanted = rule.get(name) + if wanted is not None: + has_condition = True + if not isinstance(wanted, bool): + log.warning("Language tag rule condition %r must be true or false, skipping: %s", name, rule) + return False + if wanted is not bool((states or {}).get(name)): return False if not has_condition: @@ -77,3 +108,21 @@ def rule_matches( return False return True + + +def _as_list(value: Any) -> list[Any]: + """Return the rule value as a list, so a scalar and a one-item list behave the same.""" + return value if isinstance(value, list) else [value] + + +def _matches(lang: Any, languages: Sequence[Language]) -> bool: + """Return True when the rule language closely matches one of the track languages. + + A rule value that is not a valid language tag counts as no match. The filename is built + after the mux, so a typo in the config must not discard a finished download. + """ + try: + return is_close_match(lang, list(languages)) + except LanguageTagError as e: + log.warning("Language tag rule value %r is not a valid language tag, treating as no match: %s", lang, e) + return False