diff --git a/tests/lang_selection/test_overlapping_audio_dedup.py b/tests/lang_selection/test_overlapping_audio_dedup.py new file mode 100644 index 00000000..d2341e70 --- /dev/null +++ b/tests/lang_selection/test_overlapping_audio_dedup.py @@ -0,0 +1,123 @@ +"""Regression: exact-lang must distinguish a base language from its regional variant. + +Reproduced with `--exact-lang --lang ar,ar-EG` against a streaming manifest that carries a +distinct `ar` ("Arabic") and `ar-EG` ("Arabic (Egypt)") audio track. CLDR tag_distance measures +intelligibility, not tag identity: it rates a base language and its "paradigm" regional variant +as the same language at distance 0 (ar/ar-EG, en/en-US, pt/pt-BR). Plain distance matching then +resolved both requested tags to the same track, dropping one variant and appending the other +twice, which crashed Tracks() reconstruction. + +In exact mode by_language follows RFC 4647 Lookup: it prefers the most specific (string-equal) +tag, so each tag routes to its own track. It still falls back to the fuzzy match for pure aliases +(zh matches cmn) and the lenient base-to-regional case (en finds en-US when only en-US exists). +Distances above 0 (es/es-419, fr/fr-CA) were never collapsed and stay strict. +""" + +from __future__ import annotations + +import pytest + +from unshackle.core.tracks import Audio +from unshackle.core.tracks.tracks import Tracks +from unshackle.core.utilities import is_exact_match + +# Base language vs regional variant that CLDR rates at distance 0 (paradigm locales). These are +# the pairs that collapse under naive exact matching. +COLLAPSING_PAIRS = [ + ("ar", "ar-EG"), ("en", "en-US"), ("pt", "pt-BR"), ("fr", "fr-FR"), + ("es", "es-ES"), ("de", "de-DE"), ("it", "it-IT"), +] +# Pairs CLDR rates at distance >0, which exact matching already separates. Never collapsed. +# Includes two-regional splits (no bare base) as services more commonly ship them. +DISTINCT_PAIRS = [ + ("es", "es-419"), ("fr", "fr-CA"), ("fr-FR", "fr-CA"), ("pt-PT", "pt-BR"), ("es-ES", "es-419"), + ("en-US", "en-GB"), ("de-DE", "de-AT"), ("it-IT", "it-CH"), ("es-419", "es-AR"), ("no", "nn"), +] + +# A full real-world audio language set for one title, the kind of mix a streaming manifest ships, +# which must resolve 1:1 with no collision or loss. Tags are drawn from published streaming +# language references (Warner Bros. Discovery / Netflix) and include regional and script variants. +REAL_WORLD_AUDIO_LANGS = [ + "ar", "ar-EG", "ar-SA", "bn-IN", "cmn-TW", "cs", "da", "de", "de-AT", "de-CH", "el", "en", + "en-GB", "en-AU", "es-419", "es-AR", "es-ES", "fi", "fr-CA", "fr-FR", "he", "hi", "hu", "id", + "it", "it-CH", "ja", "ko", "nl", "no", "pl", "pt-BR", "pt-PT", "ro", "ru", "sk", "sr", + "sr-Latn", "sv", "ta", "th", "tr", "uk", "vi", "yue", "zh-Hans", "zh-Hant", +] + + +def _mk(*tags): + return [Audio("https://x", language=t, id_=t) for t in tags] + + +def _select(tags, pool, exact=True): + return [(t.id, str(t.language)) for t in Tracks.by_language(pool, tags, per_language=1, exact_match=exact)] + + +@pytest.mark.parametrize("base,reg", COLLAPSING_PAIRS) +def test_collapsing_pairs_are_distance_zero(base, reg): + # Root cause: the regional variant exact-matches the base request (CLDR paradigm data). + assert is_exact_match(base, [reg]) is True + + +@pytest.mark.parametrize("base,reg", DISTINCT_PAIRS) +def test_distinct_pairs_are_not_distance_zero(base, reg): + # Contrast: these never collapse, which is why the bug was language-specific. + assert is_exact_match(base, [reg]) is False + + +@pytest.mark.parametrize("base,reg", COLLAPSING_PAIRS) +def test_exact_lang_selects_distinct_collapsing_variants(base, reg): + # Both tracks present + both tags requested: each tag must route to its own track. + assert _select([base, reg], _mk(base, reg)) == [(base, base), (reg, reg)] + + +@pytest.mark.parametrize("base,reg", DISTINCT_PAIRS) +def test_exact_lang_distinct_regionals_still_work(base, reg): + assert _select([base, reg], _mk(base, reg)) == [(base, base), (reg, reg)] + + +def test_exact_lang_lenient_base_to_regional_fallback(): + # Only the regional present; a base request must still find it when CLDR rates them equal. + assert _select(["en"], _mk("en-US")) == [("en-US", "en-US")] + # Alias with no string-equal track must still resolve (zh -> cmn). + assert _select(["zh"], _mk("cmn")) == [("cmn", "cmn")] + + +def test_exact_lang_stays_strict_for_nonzero_distance(): + # `es` does NOT accept `es-419` in exact mode (distance 5), but fuzzy mode does. + assert _select(["es"], _mk("es-419")) == [] + assert _select(["es"], _mk("es-419"), exact=False) == [("es-419", "es-419")] + + +@pytest.mark.parametrize("base,reg", COLLAPSING_PAIRS) +def test_overlapping_tags_never_duplicate_in_fuzzy_mode(base, reg): + # Non-exact still collapses base/regional, but must dedupe rather than return a track twice. + selected = Tracks.by_language(_mk(base, reg), [base, reg], per_language=1, exact_match=False) + assert len({t.id for t in selected}) == len(selected) + + +def test_exact_lang_three_way_base_and_two_regionals(): + # fr (base, dist 0 to fr-FR) + fr-FR + fr-CA (dist 4): all three must resolve distinctly. + assert _select(["fr", "fr-FR", "fr-CA"], _mk("fr", "fr-FR", "fr-CA")) == [ + ("fr", "fr"), + ("fr-FR", "fr-FR"), + ("fr-CA", "fr-CA"), + ] + # Base request with no bare-tag track falls back to its paradigm regional, without stealing + # the sibling regional (fr -> fr-FR, not fr-CA). + assert _select(["fr", "fr-CA"], _mk("fr-FR", "fr-CA")) == [("fr-FR", "fr-FR"), ("fr-CA", "fr-CA")] + + +@pytest.mark.parametrize("tag", REAL_WORLD_AUDIO_LANGS) +def test_full_language_set_each_tag_resolves_to_itself(tag): + # Against a full real-world language pool, every exact request must return its own track, so + # that no tag captures another's: ar-EG must resolve to ar-EG, never to ar. + pool = _mk(*REAL_WORLD_AUDIO_LANGS) + assert _select([tag], pool) == [(tag, tag)] + + +def test_full_language_set_selects_every_track_once(): + pool = _mk(*REAL_WORLD_AUDIO_LANGS) + selected = Tracks.by_language(pool, REAL_WORLD_AUDIO_LANGS, per_language=1, exact_match=True) + assert len(selected) == len(REAL_WORLD_AUDIO_LANGS) + assert {t.id for t in selected} == set(REAL_WORLD_AUDIO_LANGS) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 081f3470..b1e4f530 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -1622,7 +1622,10 @@ class dl: selected.append(descs[0]) else: selected.append(candidates[0]) - return selected + # Built via per-tag by_language calls, so overlapping tags can append + # the same track across iterations; dedupe by id. + seen_ids: set[str] = set() + return [t for t in selected if not (t.id in seen_ids or seen_ids.add(t.id))] music_group_download = ( bool(music_titles) diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index 4eb68c77..9aeb30c0 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -61,9 +61,7 @@ class Title: returned dict with their specific fields (e.g., season/episode). """ primary_video_track = next(iter(media_info.video_tracks), None) - original_lang_tag = ( - str(self.language).split("-")[0].lower() if self.language else "" - ) + original_lang_tag = str(self.language).split("-")[0].lower() if self.language else "" primary_audio_track = None if original_lang_tag: primary_audio_track = next( @@ -170,9 +168,7 @@ class Title: channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 channels = float(channel_count) - has_atmos = any( - "JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks - ) + has_atmos = any("JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks) context.update( { diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index d3241c14..dcd0cc37 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -443,13 +443,26 @@ class Tracks: tracks: list[TrackT], languages: list[str], per_language: int = 0, exact_match: bool = False ) -> list[TrackT]: distance = LANGUAGE_EXACT_DISTANCE if exact_match else LANGUAGE_MAX_DISTANCE - selected = [] + selected: list[TrackT] = [] + seen_ids: set[str] = set() for language in languages: - selected.extend( - [x for x in tracks if closest_supported_match(str(x.language), [language], distance)][ - : per_language or None - ] - ) + matches = [x for x in tracks if closest_supported_match(str(x.language), [language], distance)] + if exact_match and len(matches) > 1: + # CLDR tag_distance measures intelligibility, not tag identity: it rates a base + # language and its "paradigm" regional variant as the same language (distance 0 + # for ar/ar-EG, en/en-US, pt/pt-BR), so exact mode alone cannot separate them. + # Follow RFC 4647 Lookup: prefer the most specific (string-equal) tag, and fall + # back to the fuzzy match only when none exists (e.g. zh matches cmn). + want = Language.get(language).to_tag().casefold() + exact_hits = [x for x in matches if Language.get(str(x.language)).to_tag().casefold() == want] + if exact_hits: + matches = exact_hits + # Overlapping tags can still resolve to the same physical track; dedupe by id so + # callers never get duplicate tracks. + for track in matches[: per_language or None]: + if track.id not in seen_ids: + seen_ids.add(track.id) + selected.append(track) return selected def mux( @@ -694,7 +707,7 @@ class Tracks: log_event( "mux_start", level="INFO", - message=(f"Muxing {len(self.videos)}V/{len(self.audio)}A/{len(self.subtitles)}S " f"-> {output_path.name}"), + message=(f"Muxing {len(self.videos)}V/{len(self.audio)}A/{len(self.subtitles)}S -> {output_path.name}"), context={ "title": title, "output_path": str(output_path),