diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 0951e5b7..702dd98a 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -233,6 +233,9 @@ Each serialized title carries `type` (`"episode"`, `"movie"`, or `"other"`), `na List the video, audio, and subtitle tracks for a title. For series, you can scope to specific episodes. +!!! tip "Spotting the original-language audio" + Each audio track carries `is_original`, which marks the track the default `lang` of `orig` selects. Unshackle resolves it with the same language matcher the downloader uses, so it stays correct for regional variants a client cannot separate from tag text alone (`pt` marks `pt-BR` rather than `pt-PT`). Services need not report a title's language, so every track can legitimately come back `false`. + **Request body** | Field | Type | Required | Description | @@ -262,7 +265,7 @@ When both `season` and `episode` are given, they are combined into a `"{season}x "audio": [ { "id": "a-1", "codec": "EC3", "codec_display": "DD+", - "bitrate": 640, "channels": "5.1", "language": "en", + "bitrate": 640, "channels": "5.1", "language": "en", "is_original": true, "atmos": false, "descriptive": false, "drm": [], "descriptor": "DASH" } ], diff --git a/tests/lang_selection/test_as_requested.py b/tests/lang_selection/test_as_requested.py new file mode 100644 index 00000000..36b65abf --- /dev/null +++ b/tests/lang_selection/test_as_requested.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Optional + +import pytest + +from unshackle.core.utilities import as_requested + + +@pytest.mark.parametrize( + "tokens,orig_token,expected", + [ + # the user passed no -l, so orig resolved to en behind their back; report orig + (["en"], "en", "orig"), + (["en", "ja"], "en", "orig, ja"), + # the user named en, so report en + (["en"], None, "en"), + (["ja"], "en", "ja"), + (["en", "ja"], None, "en, ja"), + ([], "en", ""), + ], +) +def test_as_requested(tokens: list[str], orig_token: Optional[str], expected: str) -> None: + assert as_requested(tokens, orig_token) == expected diff --git a/tests/lang_selection/test_original_audio_ids.py b/tests/lang_selection/test_original_audio_ids.py new file mode 100644 index 00000000..74710714 --- /dev/null +++ b/tests/lang_selection/test_original_audio_ids.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Optional + +import pytest + +from unshackle.core.api.handlers import original_audio_ids + + +class FakeTrack: + def __init__(self, language: str) -> None: + self.id = language + self.language = language + + +class FakeTitle: + def __init__(self, language: Optional[str]) -> None: + self.language = language + + +@pytest.mark.parametrize( + "title_language,track_languages,expected", + [ + # paradigm collapse: CLDR calls these the same language, RFC 4647 picks the specific one + (["en"], ["en", "en-US"], ["en"]), + (["en"], ["en-US", "en-GB"], ["en-US"]), + (["pt"], ["pt-BR", "pt-PT"], ["pt-BR"]), + (["es"], ["es-ES", "es-419"], ["es-ES"]), + # non-paradigm regionals: exact mode drops them, so the fuzzy fallback catches + # them and the flag still matches what a default 'orig' download selects + (["es"], ["es-419"], ["es-419"]), + (["fr"], ["fr-CA"], ["fr-CA"]), + # alias + (["zh"], ["cmn"], ["cmn"]), + # nothing in the original language, and no language on the title at all + (["en"], ["ja", "de"], []), + ([None], ["en"], []), + ], +) +def test_original_audio_ids(title_language: list, track_languages: list[str], expected: list[str]) -> None: + tracks = [FakeTrack(t) for t in track_languages] + ids = original_audio_ids(tracks, FakeTitle(title_language[0])) + assert [t for t in track_languages if t in ids] == expected diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 68ba3c13..83ed073d 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -69,6 +69,7 @@ from unshackle.core.tracks.dv_fixup import apply_dv_fixup from unshackle.core.tracks.hybrid import Hybrid from unshackle.core.tracks.track import assert_fragments_decrypted, has_encrypted_sample_entry from unshackle.core.utilities import ( + as_requested, find_font_with_fallbacks, find_missing_langs, get_debug_logger, @@ -1725,22 +1726,30 @@ class dl: audio_languages = a_lang or lang if audio_languages: processed_lang = [] + s_orig_token: Optional[str] = None for language in audio_languages: if language == "orig": if song.language: orig_lang = str(song.language) + s_orig_token = orig_lang if orig_lang not in processed_lang: processed_lang.append(orig_lang) else: self.log.warning("Original language not available for music track, skipping 'orig'") elif language not in processed_lang: processed_lang.append(language) + if s_orig_token in audio_languages: + s_orig_token = None song.tracks.audio = select_best_audio( song.tracks.audio, processed_lang, acodec, audio_description, exact_lang ) if not song.tracks.audio: - self.log.error(f"There's no {processed_lang} Audio Track for {song.name}...") + # empty only when 'orig' was the sole request and did not resolve + self.log.error( + f"There's no {as_requested(processed_lang, s_orig_token) or 'orig'} " + f"Audio Track for {song.name}..." + ) sys.exit(1) self.log.debug("Getting Tracks") @@ -2425,10 +2434,12 @@ class dl: ) if keep_videos and video_languages and "all" not in video_languages: processed_video_lang = [] + v_orig_token: Optional[str] = None for language in video_languages: if language == "orig": if title.language: orig_lang = str(title.language) + v_orig_token = orig_lang if orig_lang not in processed_video_lang: processed_video_lang.append(orig_lang) else: @@ -2438,19 +2449,21 @@ class dl: else: if language not in processed_video_lang: processed_video_lang.append(language) + if v_orig_token in video_languages: + v_orig_token = None missing_v_langs = find_missing_langs( processed_video_lang, [v.language for v in title.tracks.videos], exact=exact_lang, ) if missing_v_langs: - missing_str = ", ".join(missing_v_langs) + missing_str = as_requested(missing_v_langs, v_orig_token) if best_available: remaining_v_langs = [tok for tok in processed_video_lang if tok not in missing_v_langs] if remaining_v_langs: self.log.warning( f"{missing_str} not found in video tracks, " - f"continuing with: {', '.join(remaining_v_langs)}" + f"continuing with: {as_requested(remaining_v_langs, v_orig_token)}" ) processed_video_lang = remaining_v_langs else: @@ -2463,7 +2476,10 @@ class dl: title.tracks.videos, processed_video_lang, exact_match=exact_lang ) if not title.tracks.videos: - self.log.error(f"There's no {processed_video_lang} Video Track...") + # empty only when 'orig' was the sole request and did not resolve + self.log.error( + f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..." + ) sys.exit(1) has_hybrid = any(r == Video.Range.HYBRID for r in range_) @@ -2777,10 +2793,12 @@ class dl: audio_languages = a_lang or lang if audio_languages: processed_lang = [] + a_orig_token: Optional[str] = None for language in audio_languages: if language == "orig": if title.language: orig_lang = str(title.language) + a_orig_token = orig_lang if orig_lang not in processed_lang: processed_lang.append(orig_lang) else: @@ -2790,6 +2808,8 @@ class dl: else: if language not in processed_lang: processed_lang.append(language) + if a_orig_token in audio_languages: + a_orig_token = None if not any(tok in processed_lang for tok in ("best", "all")): missing_a_langs = find_missing_langs( @@ -2798,12 +2818,13 @@ class dl: exact=exact_lang, ) if missing_a_langs: - missing_str = ", ".join(missing_a_langs) + missing_str = as_requested(missing_a_langs, a_orig_token) if best_available: remaining = [tok for tok in processed_lang if tok not in missing_a_langs] if remaining: self.log.warning( - f"{missing_str} not found in audio tracks, continuing with: {', '.join(remaining)}" + f"{missing_str} not found in audio tracks, " + f"continuing with: {as_requested(remaining, a_orig_token)}" ) processed_lang = remaining else: @@ -2819,7 +2840,11 @@ class dl: title.tracks.audio, processed_lang, acodec, audio_description, exact_lang ) if not title.tracks.audio: - self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") + # empty only when 'orig' was the sole request and did not resolve + self.log.error( + f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} " + f"Audio Track, cannot continue..." + ) sys.exit(1) # Reconstruct track set to only include kept tracks diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index a7268cea..39086523 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -17,7 +17,7 @@ from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_C from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy from unshackle.core.services import Services from unshackle.core.titles import Episode, Movie, Title_T -from unshackle.core.tracks import Audio, Subtitle, Video +from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.utils.collections import ci_get from unshackle.core.utils.redact import REDACTED, URL_USERINFO_RE @@ -491,8 +491,31 @@ def serialize_video_track(track: Video, include_url: bool = False) -> Dict[str, return result -def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, Any]: - """Convert audio track to JSON-serializable dict.""" +def original_audio_ids(tracks: List[Audio], title: Title_T) -> set: + """Return the ids of the audio tracks 'orig' resolves to, empty when the title has no language. + + This defers to Tracks.by_language so the flag agrees with the downloader. It asks + exact mode first because CLDR rates a base tag and its paradigm regional variant as + the same language, and only the RFC 4647 preference picks one ('en' over 'en-US' + when both exist, 'pt-BR' over 'pt-PT' for a 'pt' title). The fuzzy fallback then + catches the non-paradigm regionals exact mode drops, such as an 'es' title that + carries only 'es-419'. + """ + language = getattr(title, "language", None) + if not language: + return set() + matches = Tracks.by_language(tracks, [str(language)], exact_match=True) or Tracks.by_language( + tracks, [str(language)] + ) + return {t.id for t in matches} + + +def serialize_audio_track(track: Audio, include_url: bool = False, is_original: bool = False) -> Dict[str, Any]: + """Convert audio track to JSON-serializable dict. + + Resolve is_original with original_audio_ids so the flag always agrees with the + track 'orig' would actually download. + """ codec_name = enum_name(track.codec) result = { @@ -502,6 +525,7 @@ def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, "bitrate": int(track.bitrate / 1000) if track.bitrate else None, "channels": track.channels if track.channels else None, "language": str(track.language) if track.language else None, + "is_original": is_original, "atmos": track.atmos if hasattr(track, "atmos") else False, "descriptive": track.descriptive if hasattr(track, "descriptive") else False, "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, @@ -746,10 +770,13 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + original_ids = original_audio_ids(audio_tracks, title) episode_data = { "title": serialize_title(title), "video": [serialize_video_track(t) for t in video_tracks], - "audio": [serialize_audio_track(t) for t in audio_tracks], + "audio": [ + serialize_audio_track(t, is_original=t.id in original_ids) for t in audio_tracks + ], "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], } episodes_data.append(episode_data) @@ -793,10 +820,11 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + original_ids = original_audio_ids(audio_tracks, first_title) response = { "title": serialize_title(first_title), "video": [serialize_video_track(t) for t in video_tracks], - "audio": [serialize_audio_track(t) for t in audio_tracks], + "audio": [serialize_audio_track(t, is_original=t.id in original_ids) for t in audio_tracks], "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], } @@ -1916,11 +1944,15 @@ async def session_tracks_handler( else: server_cdm_type = "playready" + original_ids = original_audio_ids(audio_tracks, title) + return web.json_response( { "title": serialize_title(title), "video": [serialize_video_track(t, include_url=True) for t in video_tracks], - "audio": [serialize_audio_track(t, include_url=True) for t in audio_tracks], + "audio": [ + serialize_audio_track(t, include_url=True, is_original=t.id in original_ids) for t in audio_tracks + ], "subtitles": [serialize_subtitle_track(t, include_url=True) for t in tracks.subtitles], "chapters": [ {"timestamp": ch.timestamp, "name": ch.name} diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 50a64755..08f7ed2e 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -175,6 +175,11 @@ def find_missing_langs( return [tok for tok in requested if tok not in skip and not match_func(tok, available)] +def as_requested(tokens: Sequence[str], orig_token: Optional[str]) -> str: + """Return language tokens as the user wrote them, so 'orig' reads as 'orig' and not the language it resolved to.""" + return ", ".join("orig" if tok == orig_token else tok for tok in tokens) + + def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore """ Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find.