mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 03:55:05 +00:00
fix(naming): correct language tag rule matching and add state conditions
Rules took only a scalar, so the documented list syntax raised LanguageTagError after the mux. Normalise every condition, treat an invalid tag as no match, guard the surrounding post-mux config reads, and let a rule match dual, multi and dubbed so a release can carry its own tag.
This commit is contained in:
@@ -52,6 +52,13 @@ Common config keys. This is a useful subset, and every `dl` flag works:
|
||||
| `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. |
|
||||
| `no_folder` | bool | `false` | Do not create a per-title folder. |
|
||||
|
||||
!!! note "`no` and `yes` are language tags"
|
||||
YAML reads a bare `no` as `false` and a bare `yes` as `true`. unshackle puts both back to
|
||||
the tags `no` (Norwegian) and `yes` (Nyankpa) in every config key that takes a language: `lang`,
|
||||
`v_lang`, `a_lang`, `s_lang`, `forced_s_lang`, `require_audio`, `require_video`,
|
||||
`require_subs`, [`language_priority`](#subtitle), [`muxing.default_language`](#muxing) and
|
||||
the [`language_tags`](#language_tags) rules. Quotes (`lang: "no"`) work as well.
|
||||
|
||||
```yaml title="Global download defaults"
|
||||
dl:
|
||||
lang: [en]
|
||||
@@ -253,9 +260,22 @@ Language-tag remapping and the rule engine behind the `{lang_tag}` filename vari
|
||||
|
||||
| Condition | Meaning |
|
||||
|-----------|---------|
|
||||
| `audio` | Matches against the audio track languages. |
|
||||
| `audio` | At least one audio language matches. |
|
||||
| `subs_contain` | At least one subtitle language matches. |
|
||||
| `subs_contain_all` | Every listed language must be present (scalar or list). |
|
||||
| `subs_contain_all` | Every listed language must be present. |
|
||||
| `dual` / `multi` / `dubbed` | `true` matches when that filename variable is set, `false` when it is not. |
|
||||
|
||||
Each language condition takes one language tag or a list of them. A list matches when **any**
|
||||
entry matches, except `subs_contain_all`, which needs them all. An entry that is not a valid
|
||||
language tag (for example `japanese`) never matches. `dual`, `multi` and `dubbed` take a bare
|
||||
`true` or `false`, not a quoted string. unshackle skips a rule with a condition name it does
|
||||
not know, or with any other value in a boolean condition. It logs a warning in each of these
|
||||
cases and tries the next rule.
|
||||
|
||||
`dual`, `multi` and `dubbed` carry the same meaning as the
|
||||
[filename variables of those names](#dual_multi_mode). Pair one with a language to name a
|
||||
release yourself, in place of the fixed `DUAL`, `MULTi` and `DUBBED` text. A rule that names
|
||||
`dubbed: true` and a language tags a localized dub without tagging a native release the same way.
|
||||
|
||||
```yaml
|
||||
language_tags:
|
||||
@@ -263,8 +283,18 @@ language_tags:
|
||||
- audio: [ja]
|
||||
subs_contain: [en]
|
||||
tag: "SUBBED"
|
||||
- audio: th
|
||||
dubbed: true
|
||||
tag: "THAIDUB"
|
||||
- audio: th
|
||||
dual: true
|
||||
tag: "THAIDUAL"
|
||||
```
|
||||
|
||||
!!! note "Put `{lang_tag}` where you want the name"
|
||||
These rules fill `{lang_tag}`. To use your own name in place of the fixed text, keep
|
||||
`{lang_tag?}` in the template and leave out `{dual?}`, `{multi?}` and `{dubbed?}`.
|
||||
|
||||
## `dual_multi_mode`
|
||||
|
||||
- **Type:** `str` · **Default:** `strict`
|
||||
|
||||
@@ -118,3 +118,10 @@ def test_count_never_dubbed(count_mode):
|
||||
assert c["dubbed"] == ""
|
||||
assert c["dual"] == ""
|
||||
assert c["multi"] == ""
|
||||
|
||||
|
||||
def test_a_missing_template_key_falls_back_instead_of_raising(monkeypatch):
|
||||
"""dl only checks output_template is non-empty, so the title's own key can be absent."""
|
||||
monkeypatch.setattr(config, "output_template", {"series": "{title}"})
|
||||
name = make_movie("en").get_filename(make_media_info(["en"]))
|
||||
assert name == "The Film (2024)"
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Rules accept both the scalar and the list form of every condition."""
|
||||
|
||||
import pytest
|
||||
from langcodes import Language
|
||||
|
||||
from unshackle.core.utils.language_tags import evaluate_language_tag
|
||||
|
||||
JA = [Language.get("ja")]
|
||||
EN_ES = [Language.get("en-US"), Language.get("es")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("audio, subs", [("ja", "en"), (["ja"], ["en"])])
|
||||
def test_scalar_and_list_forms_both_match(audio, subs) -> None:
|
||||
rules = [{"audio": audio, "subs_contain": subs, "tag": "SUBBED"}]
|
||||
assert evaluate_language_tag(rules, JA, EN_ES) == "SUBBED"
|
||||
|
||||
|
||||
def test_a_list_matches_any_of_its_entries() -> None:
|
||||
rules = [{"audio": ["ko", "ja"], "tag": "ASIAN"}]
|
||||
assert evaluate_language_tag(rules, JA, []) == "ASIAN"
|
||||
|
||||
|
||||
def test_subs_contain_all_needs_every_entry() -> None:
|
||||
rules = [{"subs_contain_all": ["en", "fr"], "tag": "MULTI"}]
|
||||
assert evaluate_language_tag(rules, JA, EN_ES) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["japanese", 1, [["ja"]], {"ja": 1}])
|
||||
def test_an_invalid_rule_value_is_no_match_not_a_crash(bad, caplog) -> None:
|
||||
rules = [{"audio": bad, "tag": "BAD"}, {"audio": "ja", "tag": "OK"}]
|
||||
assert evaluate_language_tag(rules, JA, EN_ES) == "OK"
|
||||
assert "not a valid language tag" in caplog.text
|
||||
|
||||
|
||||
def test_dubbed_separates_a_localized_dub_from_a_native_release() -> None:
|
||||
rules = [{"audio": "th", "dubbed": True, "tag": "THAIDUB"}, {"audio": "th", "tag": "THAI"}]
|
||||
th = [Language.get("th")]
|
||||
assert evaluate_language_tag(rules, th, [], {"dubbed": True}) == "THAIDUB"
|
||||
assert evaluate_language_tag(rules, th, [], {"dubbed": False}) == "THAI"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["dual", "multi", "dubbed"])
|
||||
def test_every_state_condition_matches_both_ways(state) -> None:
|
||||
rules = [{state: True, "tag": "ON"}, {state: False, "tag": "OFF"}]
|
||||
assert evaluate_language_tag(rules, JA, [], {state: True}) == "ON"
|
||||
assert evaluate_language_tag(rules, JA, [], {state: False}) == "OFF"
|
||||
assert evaluate_language_tag(rules, JA, [], None) == "OFF"
|
||||
|
||||
|
||||
def test_a_state_condition_combines_with_a_language() -> None:
|
||||
rules = [{"audio": "th", "dual": True, "tag": "THAIDUAL"}, {"dual": True, "tag": "DUAL"}]
|
||||
states = {"dual": True}
|
||||
assert evaluate_language_tag(rules, [Language.get("th")], [], states) == "THAIDUAL"
|
||||
assert evaluate_language_tag(rules, JA, [], states) == "DUAL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["yes", "false", 1])
|
||||
def test_a_non_boolean_state_value_skips_the_rule(bad, caplog) -> None:
|
||||
rules = [{"dubbed": bad, "tag": "BAD"}, {"audio": "ja", "tag": "OK"}]
|
||||
assert evaluate_language_tag(rules, JA, [], {"dubbed": True}) == "OK"
|
||||
assert evaluate_language_tag(rules, JA, [], {"dubbed": False}) == "OK"
|
||||
assert "must be true or false" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typo", ["dubed", "subs_contains"])
|
||||
def test_a_typo_in_a_condition_name_skips_the_rule(typo, caplog) -> None:
|
||||
rules = [{"audio": "ja", typo: True, "tag": "BAD"}, {"audio": "ja", "tag": "OK"}]
|
||||
assert evaluate_language_tag(rules, JA, [], {"dubbed": True}) == "OK"
|
||||
assert f"unknown key(s) {typo}" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rules", ["notalist", ["foo"], {"audio": "ja"}, [None]])
|
||||
def test_a_malformed_rules_block_is_ignored_not_a_crash(rules) -> None:
|
||||
assert evaluate_language_tag(rules, JA, []) == ""
|
||||
@@ -258,7 +258,11 @@ class Episode(Title):
|
||||
name += f" {self.folder_season()}"
|
||||
return sanitize_filename(name, " ")
|
||||
|
||||
formatter = TemplateFormatter(config.output_template["series"])
|
||||
template = config.output_template.get("series")
|
||||
if not template:
|
||||
return sanitize_filename(str(self), " ")
|
||||
|
||||
formatter = TemplateFormatter(template)
|
||||
context = self.build_template_context(media_info, show_service)
|
||||
return formatter.format(context)
|
||||
|
||||
|
||||
@@ -76,7 +76,11 @@ class Movie(Title):
|
||||
name += f" ({self.year})"
|
||||
return sanitize_filename(name, " ")
|
||||
|
||||
formatter = TemplateFormatter(config.output_template["movies"])
|
||||
template = config.output_template.get("movies")
|
||||
if not template:
|
||||
return sanitize_filename(str(self), " ")
|
||||
|
||||
formatter = TemplateFormatter(template)
|
||||
context = self.build_template_context(media_info, show_service)
|
||||
return formatter.format(context)
|
||||
|
||||
|
||||
@@ -213,13 +213,15 @@ class Title:
|
||||
if original_lang_tag and original_lang_tag not in audio_lang_bases:
|
||||
context["dubbed"] = "DUBBED"
|
||||
|
||||
lang_tag_rules = config.language_tags.get("rules") if config.language_tags else None
|
||||
language_tags = config.language_tags if isinstance(config.language_tags, dict) else {}
|
||||
lang_tag_rules = language_tags.get("rules")
|
||||
if lang_tag_rules and self.tracks:
|
||||
from unshackle.core.utils.language_tags import evaluate_language_tag
|
||||
|
||||
audio_langs = [a.language for a in self.tracks.audio]
|
||||
sub_langs = [s.language for s in self.tracks.subtitles]
|
||||
context["lang_tag"] = evaluate_language_tag(lang_tag_rules, audio_langs, sub_langs)
|
||||
states = {name: bool(context[name]) for name in ("dual", "multi", "dubbed")}
|
||||
context["lang_tag"] = evaluate_language_tag(lang_tag_rules, audio_langs, sub_langs, states)
|
||||
|
||||
if config.tag_rules:
|
||||
from unshackle.core.utils.tag_rules import evaluate_tag_rules
|
||||
|
||||
@@ -26,7 +26,13 @@ from unshackle.core.tracks.chapters import Chapter, Chapters
|
||||
from unshackle.core.tracks.subtitle import Subtitle
|
||||
from unshackle.core.tracks.track import Track, has_dts_uhd_sample_entry, strip_duplicate_init_boxes
|
||||
from unshackle.core.tracks.video import Video
|
||||
from unshackle.core.utilities import is_close_match, log_event, matching_languages, sanitize_filename
|
||||
from unshackle.core.utilities import (
|
||||
is_close_match,
|
||||
log_event,
|
||||
matching_languages,
|
||||
sanitize_filename,
|
||||
valid_language,
|
||||
)
|
||||
from unshackle.core.utils.collections import as_list, flatten
|
||||
|
||||
MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)")
|
||||
@@ -727,9 +733,10 @@ class Tracks:
|
||||
cl.extend(["--title", title])
|
||||
|
||||
default_language = config.muxing.get("default_language") or {}
|
||||
preferred_video_lang = default_language.get("video")
|
||||
preferred_audio_lang = default_language.get("audio")
|
||||
preferred_subtitle_lang = default_language.get("subtitle")
|
||||
# mux() runs after every track is downloaded, so a typo here must not discard the title
|
||||
preferred_video_lang = valid_language(default_language.get("video"))
|
||||
preferred_audio_lang = valid_language(default_language.get("audio"))
|
||||
preferred_subtitle_lang = valid_language(default_language.get("subtitle"))
|
||||
|
||||
preferred_video_idx: Optional[int] = None
|
||||
if preferred_video_lang:
|
||||
|
||||
@@ -153,6 +153,18 @@ def is_close_match(language: Union[str, Language], languages: Sequence[Union[str
|
||||
return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_MAX_DISTANCE
|
||||
|
||||
|
||||
def valid_language(language: Any) -> Optional[str]:
|
||||
"""Return the language tag as text, or None when it is not a valid tag."""
|
||||
if not language:
|
||||
return None
|
||||
try:
|
||||
Language.get(str(language))
|
||||
except LanguageTagError:
|
||||
logging.getLogger("utilities").warning("%r is not a valid language tag, ignoring it", language)
|
||||
return None
|
||||
return str(language)
|
||||
|
||||
|
||||
def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool:
|
||||
"""Examine whether a language exactly matches any of the given languages."""
|
||||
languages = [x for x in languages if x]
|
||||
|
||||
@@ -3,19 +3,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Sequence
|
||||
from collections.abc import Mapping as MappingABC
|
||||
from typing import Any, Mapping, Optional, Sequence
|
||||
|
||||
from langcodes import Language
|
||||
from langcodes.tag_parser import LanguageTagError
|
||||
|
||||
from unshackle.core.utilities import is_close_match
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Conditions that test a filename variable of the same name rather than a language.
|
||||
STATE_CONDITIONS = ("dual", "multi", "dubbed")
|
||||
LANGUAGE_CONDITIONS = ("audio", "subs_contain", "subs_contain_all")
|
||||
|
||||
|
||||
def evaluate_language_tag(
|
||||
rules: list[dict[str, Any]],
|
||||
audio_languages: Sequence[Language],
|
||||
subtitle_languages: Sequence[Language],
|
||||
states: Optional[Mapping[str, bool]] = None,
|
||||
) -> str:
|
||||
"""Evaluate language tag rules against selected tracks.
|
||||
|
||||
@@ -26,17 +33,27 @@ def evaluate_language_tag(
|
||||
rules: List of rule dicts from config, each with conditions and a ``tag``.
|
||||
audio_languages: Languages of the selected audio tracks.
|
||||
subtitle_languages: Languages of the selected subtitle tracks.
|
||||
states: Whether each of ``dual``, ``multi`` and ``dubbed`` is set for this release,
|
||||
the same state as the filename variable of that name.
|
||||
|
||||
Returns:
|
||||
The tag string from the first rule that matches, or ``""`` if none match.
|
||||
"""
|
||||
if not isinstance(rules, list):
|
||||
log.warning("Language tag rules must be a list of rules, ignoring: %r", rules)
|
||||
return ""
|
||||
|
||||
for rule in rules:
|
||||
if not isinstance(rule, MappingABC):
|
||||
log.warning("Language tag rule must be a mapping of conditions and a 'tag', skipping: %r", rule)
|
||||
continue
|
||||
|
||||
tag = rule.get("tag")
|
||||
if not tag:
|
||||
log.warning("Language tag rule missing 'tag' field, skipping: %s", rule)
|
||||
continue
|
||||
|
||||
if rule_matches(rule, audio_languages, subtitle_languages):
|
||||
if rule_matches(rule, audio_languages, subtitle_languages, states):
|
||||
log.debug("Language tag rule matched: %s -> %s", rule, tag)
|
||||
return str(tag)
|
||||
|
||||
@@ -47,29 +64,43 @@ def rule_matches(
|
||||
rule: dict[str, Any],
|
||||
audio_languages: Sequence[Language],
|
||||
subtitle_languages: Sequence[Language],
|
||||
states: Optional[Mapping[str, bool]] = None,
|
||||
) -> bool:
|
||||
"""Return True when every condition in the rule matches the tracks."""
|
||||
unknown = [key for key in rule if key not in (*LANGUAGE_CONDITIONS, *STATE_CONDITIONS, "tag")]
|
||||
if unknown:
|
||||
log.warning("Language tag rule has unknown key(s) %s, skipping: %s", ", ".join(sorted(unknown)), rule)
|
||||
return False
|
||||
|
||||
has_condition = False
|
||||
|
||||
audio_lang = rule.get("audio")
|
||||
if audio_lang is not None:
|
||||
has_condition = True
|
||||
if not is_close_match(audio_lang, list(audio_languages)):
|
||||
if not any(_matches(lang, audio_languages) for lang in _as_list(audio_lang)):
|
||||
return False
|
||||
|
||||
subs_contain = rule.get("subs_contain")
|
||||
if subs_contain is not None:
|
||||
has_condition = True
|
||||
if not is_close_match(subs_contain, list(subtitle_languages)):
|
||||
if not any(_matches(lang, subtitle_languages) for lang in _as_list(subs_contain)):
|
||||
return False
|
||||
|
||||
subs_contain_all = rule.get("subs_contain_all")
|
||||
if subs_contain_all is not None:
|
||||
has_condition = True
|
||||
if not isinstance(subs_contain_all, list):
|
||||
subs_contain_all = [subs_contain_all]
|
||||
for lang in subs_contain_all:
|
||||
if not is_close_match(lang, list(subtitle_languages)):
|
||||
for lang in _as_list(subs_contain_all):
|
||||
if not _matches(lang, subtitle_languages):
|
||||
return False
|
||||
|
||||
for name in STATE_CONDITIONS:
|
||||
wanted = rule.get(name)
|
||||
if wanted is not None:
|
||||
has_condition = True
|
||||
if not isinstance(wanted, bool):
|
||||
log.warning("Language tag rule condition %r must be true or false, skipping: %s", name, rule)
|
||||
return False
|
||||
if wanted is not bool((states or {}).get(name)):
|
||||
return False
|
||||
|
||||
if not has_condition:
|
||||
@@ -77,3 +108,21 @@ def rule_matches(
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[Any]:
|
||||
"""Return the rule value as a list, so a scalar and a one-item list behave the same."""
|
||||
return value if isinstance(value, list) else [value]
|
||||
|
||||
|
||||
def _matches(lang: Any, languages: Sequence[Language]) -> bool:
|
||||
"""Return True when the rule language closely matches one of the track languages.
|
||||
|
||||
A rule value that is not a valid language tag counts as no match. The filename is built
|
||||
after the mux, so a typo in the config must not discard a finished download.
|
||||
"""
|
||||
try:
|
||||
return is_close_match(lang, list(languages))
|
||||
except LanguageTagError as e:
|
||||
log.warning("Language tag rule value %r is not a valid language tag, treating as no match: %s", lang, e)
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user