diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 36ffec3c..69e5de78 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -333,7 +333,7 @@ When you give both `season` and `episode`, the server combines them into a `"{se "resolution": "2160p", "fps": "24", "range": "HDR10", "range_display": "HDR10", "scan_type": null, "closed_captions": [], "dv_compatible_bitstream": false, "hdr10plus": false, "language": "en", "is_original_lang": true, - "name": null, "needs_repack": false, "edition": [], "data": null, + "name": null, "track_name": "English", "needs_repack": false, "edition": [], "data": null, "drm": [], "descriptor": "DASH" } ], @@ -342,14 +342,14 @@ When you give both `season` and `episode`, the server combines them into a `"{se "id": "a-1", "codec": "EC3", "codec_display": "DD+", "bitrate": 640, "bitrate_bps": 640000, "channels": "5.1", "language": "en", "is_original": true, "is_original_lang": true, "atmos": false, "joc": 0, - "descriptive": false, "name": null, "needs_repack": false, "edition": [], + "descriptive": false, "name": null, "track_name": "English", "needs_repack": false, "edition": [], "data": null, "drm": [], "descriptor": "DASH" } ], "subtitles": [ { "id": "s-1", "codec": "vtt", "language": "en", "forced": false, "sdh": false, "cc": false, "is_original_lang": false, - "name": null, "needs_repack": false, "edition": [], "data": null, + "name": null, "track_name": "English", "needs_repack": false, "edition": [], "data": null, "drm": [], "descriptor": "DASH" } ] } @@ -377,7 +377,7 @@ The server sorts tracks by bitrate, descending. Episodes the server cannot get g The `drm` array on a track is a list of `{type, pssh?, kids?, content_keys?, license_url?}` objects. -Every track carries the same fields as a local track: `is_original_lang`, `name`, `needs_repack`, `edition`, and `data`. `bitrate` is in kb/s, and `bitrate_bps` gives the exact bits/s. `data` holds the part of the service-set `track.data` that fits in JSON, without the `hls`, `dash`, and `ism` parser keys. Audio keeps `is_original` and `atmos` for clients that read them, and adds the exact `joc` object count. +Every track carries the same fields as a local track: `is_original_lang`, `name`, `needs_repack`, `edition`, and `data`. `name` is the name the service set, or null. `track_name` is the name `--list` shows and the mux writes to an audio or subtitle track: `name` when the service set one, else the English name of the language, with the SDH, CC, Forced or Descriptive label. `bitrate` is in kb/s, and `bitrate_bps` gives the exact bits/s. `data` holds the part of the service-set `track.data` that fits in JSON, without the `hls`, `dash`, and `ism` parser keys. Audio keeps `is_original` and `atmos` for clients that read them, and adds the exact `joc` object count. | Status | Error code | Meaning | | --- | --- | --- | diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 1d452747..74a857c7 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -284,6 +284,27 @@ warning and continues. !!! note "Height channels" A track with height channels is named bed.LFE.heights, such as `5.1.4`. +### Track names and language tags + +When the service gives a track no name, unshackle names it with the English name of its language, +such as `English`, `Portuguese (Portugal)` or `Spanish (Latin America)`. The name leaves out the +region that is the default for the language, so `en-US` is `English` and `pt-BR` is `Portuguese`, +but `en-GB` is `English (United Kingdom)`. A name from the service is kept as is. unshackle adds the SDH, CC, Forced or Descriptive label after either name. + +A subtitle in a language with more than one script gets the script from its region: + +| Service tag | Tag in the file | Track name | +|-------------|-----------------|------------| +| `zh-TW` | `zh-Hant-TW` | Chinese (Traditional) | +| `zh-HK` | `zh-Hant-HK` | Chinese (Traditional, Hong Kong) | +| `zh-CN` | `zh-Hans-CN` | Chinese (Simplified) | +| `sr-ME` | `sr-Latn-ME` | Serbian (Latin, Montenegro) | +| `zh` | `zh` | Chinese | + +A tag without a region, such as `zh`, gets no script because the region is what tells the script. +unshackle writes audio tags as the service gives them, because a script applies only to written text. +Track selection, `--list` tags and file names use the service tag. + ### DTS:X Profile 2 output A title whose audio holds a `dtsx` or `dtsy` sample entry (DTS:X Profile 2, DTS-UHD) muxes diff --git a/tests/core/test_mp4_container_routing.py b/tests/core/test_mp4_container_routing.py index ca1bd9c1..95a6c8fb 100644 --- a/tests/core/test_mp4_container_routing.py +++ b/tests/core/test_mp4_container_routing.py @@ -150,6 +150,30 @@ def test_substation_alpha_subtitles_are_left_out(tmp_path: Path, monkeypatch: py assert "s.vtt:lang=es" in command +def test_mp4_subtitles_get_the_script_their_region_implies(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured: list[FakePopen] = [] + + def fake_popen(args: list[str], **kwargs: object) -> FakePopen: + p = FakePopen(args, **kwargs) + captured.append(p) + Path(args[-1]).write_bytes(b"muxed") + return p + + monkeypatch.setattr(binaries, "MP4Box", Path("MP4Box"), raising=False) + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + tracks = build_tracks(tmp_path, b"dtsx") + vtt = Subtitle(id_="s1", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="zh-TW") + vtt.path = tmp_path / "s.vtt" + vtt.path.write_text("WEBVTT\n") + tracks.add(vtt) + + tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + command = captured[0].args + assert f"{tmp_path / 's.vtt'}:lang=zh-Hant-TW" in command + assert any(arg.endswith("=Chinese (Traditional)") for arg in command) + + def test_mp4box_errors_come_back_to_the_caller(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A failed mux does not raise; the caller inspects the return code and lines.""" diff --git a/tests/tracks/test_track_language_names.py b/tests/tracks/test_track_language_names.py new file mode 100644 index 00000000..ba914e37 --- /dev/null +++ b/tests/tracks/test_track_language_names.py @@ -0,0 +1,143 @@ +from pathlib import Path +from typing import Optional + +import pytest +from langcodes import Language + +from unshackle.core import binaries +from unshackle.core.tracks import Audio, Subtitle, Tracks, Video +from unshackle.core.utilities import language_name, multi_script_languages, with_script + + +def sub(lang: str, name: Optional[str] = None, sdh: bool = False) -> Subtitle: + return Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language=lang, name=name, sdh=sdh) + + +def test_multi_script_languages_come_from_cldr() -> None: + assert {"zh", "yue", "sr"} <= multi_script_languages() + assert not {"en", "pt", "ja"} & multi_script_languages() + + +@pytest.mark.parametrize( + ("tag", "expected"), + [ + ("zh", "zh"), + ("zh-TW", "zh-Hant-TW"), + ("zh-CN", "zh-Hans-CN"), + ("zh-HK", "zh-Hant-HK"), + ("zh-SG", "zh-Hans-SG"), + ("zh-Hant", "zh-Hant"), + ("yue-HK", "yue-Hant-HK"), + ("sr-ME", "sr-Latn-ME"), + ("sr-RS", "sr-Cyrl-RS"), + ("pa-PK", "pa-Arab-PK"), + ("ms-MY", "ms-MY"), + ("en-US", "en-US"), + ("und-TW", "und-TW"), + ("und", "und"), + ], +) +def test_with_script(tag: str, expected: str) -> None: + assert str(with_script(Language.get(tag))) == expected + + +@pytest.mark.parametrize( + ("tag", "expected"), + [ + ("zh-Hant-TW", "Chinese (Traditional)"), + ("zh-Hans-CN", "Chinese (Simplified)"), + ("zh-Hant-HK", "Chinese (Traditional, Hong Kong)"), + ("zh-TW", "Chinese (Taiwan)"), + ("pt-BR", "Portuguese"), + ("pt-PT", "Portuguese (Portugal)"), + ("en-US", "English"), + ("en-GB", "English (United Kingdom)"), + ("ja-JP", "Japanese"), + ("fr-CA", "French (Canada)"), + ("ar-001", "Arabic"), + ("es-419", "Spanish (Latin America)"), + ("en", "English"), + ("de-DE", "German"), + ("en-ZZ", "English"), + ("en-XA", "English"), + ("en-Zzzz", "English"), + ("sq-XK", "Albanian (Kosovo)"), + ("mn-Mong-CN", "Mongolian (China)"), + ("und", None), + ("qaa", None), + ("x-foo", None), + ], +) +def test_language_name(tag: str, expected: Optional[str]) -> None: + assert language_name(Language.get(tag)) == expected + + +def test_subtitle_names_and_tags_the_script_but_keeps_its_language() -> None: + track = sub("zh-TW") + assert track.name is None + assert track.get_track_name() == "Chinese (Traditional)" + assert str(track.mux_language) == "zh-Hant-TW" + assert str(track.language) == "zh-TW" + + +def test_subtitle_flag_follows_the_default_name() -> None: + assert sub("zh-CN", sdh=True).get_track_name() == "Chinese (Simplified) (SDH)" + + +def test_audio_gets_no_script() -> None: + track = Audio(id_="a", url="https://x/a", codec=Audio.Codec.AAC, language="zh-TW", bitrate=1) + assert track.get_track_name() == "Chinese (Taiwan)" + assert str(track.mux_language) == "zh-TW" + + +def test_a_service_name_wins() -> None: + assert sub("zh", name="简体中文").get_track_name() == "简体中文" + + +def test_mkvmerge_gets_the_subtitle_script_and_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + class Stop(Exception): + pass + + captured: list[list[str]] = [] + + def fake_popen(args: list[str], **kwargs: object) -> None: + captured.append(args) + raise Stop + + monkeypatch.setattr("unshackle.core.tracks.tracks.subprocess.Popen", fake_popen) + monkeypatch.setattr(binaries, "MKVToolNix", Path("mkvmerge")) + video = Video(id_="v", url="https://x/v", codec=Video.Codec.AVC, language="zh-TW", width=1920, height=1080) + audio = Audio(id_="a", url="https://x/a", codec=Audio.Codec.AAC, language="zh-TW", bitrate=1) + subtitle = sub("zh-TW") + for track, file in ((video, "v.mp4"), (audio, "a.mp4"), (subtitle, "s.vtt")): + track.path = tmp_path / file + track.path.write_bytes(b"\x00" * 16) + + with pytest.raises(Stop): + Tracks([video, audio, subtitle]).mux("T", delete=False, output_path=tmp_path / "o.mkv") + + def last_option(file: str, flag: str) -> str: + """Return the value of the last flag given before the file, which is the one mkvmerge applies to it.""" + before = captured[0][: captured[0].index(str(tmp_path / file))] + return before[len(before) - before[::-1].index(flag)] + + assert last_option("s.vtt", "--language") == "0:zh-Hant-TW" + assert last_option("s.vtt", "--track-name") == "0:Chinese (Traditional)" + assert last_option("a.mp4", "--language") == "0:zh-TW" + + +def test_the_default_name_follows_a_later_language_change() -> None: + track = sub("yue") + track.language = Language.get("zh-Hant") + assert track.get_track_name() == "Chinese (Traditional)" + + +def test_a_language_assigned_as_text_later_is_parsed() -> None: + track = sub("pt") + track.language = "pt-PT" + assert track.language == Language.get("pt-PT") + assert track.get_track_name() == "Portuguese (Portugal)" + assert str(track) == "SUB | [VTT] | pt-PT | Portuguese (Portugal)" + track.language = None + assert str(track.language) == "und" + assert track.get_track_name() is None diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index fe17970f..533d871d 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -1718,6 +1718,7 @@ def serialize_track_common(track: Any) -> Dict[str, Any]: "language": str(track.language) if track.language else None, "is_original_lang": bool(getattr(track, "is_original_lang", False)), "name": getattr(track, "name", None), + "track_name": track.get_track_name(), "needs_repack": bool(getattr(track, "needs_repack", False)), "edition": list(getattr(track, "edition", None) or []), "descriptor": descriptor_name(track), diff --git a/unshackle/core/tracks/subtitle.py b/unshackle/core/tracks/subtitle.py index 251375ca..3d6858f4 100644 --- a/unshackle/core/tracks/subtitle.py +++ b/unshackle/core/tracks/subtitle.py @@ -15,6 +15,7 @@ import click import pycaption import requests from construct import Container +from langcodes import Language from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader from pycaption.geometry import Layout from pymp4.parser import MP4 @@ -24,7 +25,7 @@ from subtitle_filter import Subtitles from unshackle.core import binaries from unshackle.core.config import config from unshackle.core.tracks.track import Track -from unshackle.core.utilities import try_ensure_utf8 +from unshackle.core.utilities import try_ensure_utf8, with_script from unshackle.core.utils.subprocess import log_tool_run from unshackle.core.utils.webvtt import merge_segmented_webvtt @@ -221,6 +222,11 @@ class Subtitle(Track): forced=data.get("forced", False), ) + @property + def mux_language(self) -> Language: + """The language tag written to the output file, with the script its region implies.""" + return with_script(self.language) + def __str__(self) -> str: return " | ".join( filter( diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 406c48b2..16277a50 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -35,7 +35,7 @@ from unshackle.core.events import events from unshackle.core.session import MAX_BACKOFF, RnetSession, utf8_text from unshackle.core.temp import restore_segments, stash_segments from unshackle.core.tracks import resume -from unshackle.core.utilities import get_boxes, log_event, try_ensure_utf8 +from unshackle.core.utilities import get_boxes, language_name, log_event, try_ensure_utf8 from unshackle.core.utils import run_proxy from unshackle.core.utils.subprocess import ffprobe @@ -467,7 +467,7 @@ class Track: self.path: Optional[Path] = None self.url = url - self.language = Language.get(language) + self.language = language self.is_original_lang = is_original_lang self.descriptor = descriptor self.needs_repack = needs_repack @@ -483,23 +483,6 @@ class Track: self.data = data or {} self.extra: Any = extra or {} # allow anything for extra, but default to a dict - if self.name is None: - lang = self.language - if (lang.language or "").lower() == (lang.territory or "").lower(): - lang = lang.update_dict({"territory": None}) - reduced = lang.simplify_script() - extra_parts = [] - if reduced.script is not None: - script = reduced.script_name(max_distance=25) - if script and script != "Zzzz": - extra_parts.append(script) - if reduced.territory is not None: - territory = reduced.territory_name(max_distance=25) - if territory and territory != "ZZ": - territory = territory.removesuffix(" SAR China") - extra_parts.append(territory) - self.name = ", ".join(extra_parts) or None - if not id_: this = copy(self) this.url = self.url.rsplit("?", maxsplit=1)[0] @@ -588,6 +571,15 @@ class Track: def __eq__(self, other: Any) -> bool: return isinstance(other, Track) and self.id == other.id + @property + def language(self) -> Language: + """The language the service tagged, parsed even when a service assigns a string or None later.""" + return self._language + + @language.setter + def language(self, value: Union[Language, str, None]) -> None: + self._language = Language.get(value or "und") + @property def data(self) -> defaultdict[Any, Any]: """ @@ -1015,9 +1007,14 @@ class Track: raise ValueError(f"Cannot reconstruct unsupported track type: {track_type!r}") return builder.from_dict(data) + @property + def mux_language(self) -> Language: + """The language tag written to the output file.""" + return self.language + def get_track_name(self) -> Optional[str]: - """Get the Track Name.""" - return self.name + """Get the Track Name, or the English name of its language when the service set none.""" + return self.name if self.name is not None else language_name(self.mux_language) def get_drm_for_cdm(self, cdm: Optional[object]) -> Optional[DRM_T]: """Return the DRM matching the provided CDM, if available.""" diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index a50fcfde..ea397d92 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -28,6 +28,7 @@ from unshackle.core.tracks.track import Track, has_dts_uhd_sample_entry, strip_d from unshackle.core.tracks.video import Video from unshackle.core.utilities import ( is_close_match, + language_name, log_event, matching_languages, original_tag, @@ -606,7 +607,7 @@ class Tracks: candidate.unlink() delay = round((track.sample_start - sample_base) * 1000) if config.sample else 0 - cl.extend(["-add", f"{source}:lang={track.language}" + (f":delay={delay}" if delay else "")]) + cl.extend(["-add", f"{source}:lang={track.mux_language}" + (f":delay={delay}" if delay else "")]) name = "" if isinstance(track, Video) else (track.get_track_name() or "") if name: names.extend(["-name", f"{track_id}={name}"]) @@ -765,7 +766,7 @@ class Tracks: video_track = video_track or self.videos[0] if video_track.language.is_valid(): lang_code = str(video_track.language) - lang_name = video_track.language.display_name() + lang_name = language_name(video_track.language) or lang_code for video in self.videos: if video.data.get("audio_language"): @@ -832,7 +833,7 @@ class Tracks: video_args = [ "--language", - f"0:{vt.language}", + f"0:{vt.mux_language}", "--default-track", f"0:{is_default}", "--original-flag", @@ -888,7 +889,7 @@ class Tracks: "--track-name", f"0:{at.get_track_name() or ''}", "--language", - f"0:{at.language}", + f"0:{at.mux_language}", "--default-track", f"0:{audio_default}", "--visual-impaired-flag", @@ -918,7 +919,7 @@ class Tracks: "--track-name", f"0:{st.get_track_name() or ''}", "--language", - f"0:{st.language}", + f"0:{st.mux_language}", "--sub-charset", "0:UTF-8", "--forced-track", diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index e555fe44..edb8a472 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -15,6 +15,7 @@ import unicodedata import zlib from collections import defaultdict from datetime import datetime, timezone +from functools import cache from pathlib import Path from types import ModuleType from typing import Any, Callable, Mapping, Optional, Sequence, Union @@ -26,6 +27,7 @@ import pycountry from construct import ValidationError from fontTools import ttLib from langcodes import Language, closest_match +from langcodes.data_dicts import LIKELY_SUBTAGS from langcodes.tag_parser import LanguageTagError from pymp4.parser import Box from unidecode import unidecode @@ -169,6 +171,53 @@ def valid_language(language: Any) -> Optional[str]: return str(language) +@cache +def multi_script_languages() -> frozenset[str]: + """Return the languages whose likely script changes with the region in CLDR (zh, yue, sr, ...).""" + found = set() + for tag, likely in LIKELY_SUBTAGS.items(): + match = re.fullmatch(r"([a-z]{2,3})-(?:[A-Z]{2}|\d{3})", tag) + if match and match[1] != "und" and Language.get(likely).script != Language.get(match[1]).maximize().script: + found.add(match[1]) + return frozenset(found) + + +def with_script(language: Language) -> Language: + """ + Add the script a region implies to a subtitle tag, e.g. zh-TW to zh-Hant-TW. + + Only a language written in several scripts gets one, and only when the tag has a region: + a bare zh says nothing about the script, so it stays zh. + """ + if language.script or not language.territory or language.language not in multi_script_languages(): + return language + return language.update_dict({"script": language.maximize().script}).simplify_script() + + +def language_name(language: Language) -> Optional[str]: + """ + Return the English name of a track language, or None for an unknown or private-use language. + + It leaves out a script or region that adds nothing: private use (but XK, Kosovo), 001, the CLDR default + region of the language and script (pt-BR, zh-Hant-TW, but not pt-PT), a script named like the language. + """ + code = language.language + if not code or not Language.make(language=code).has_name_data(): + return None + script, region = language.script, language.territory + if script and ( + re.fullmatch(r"Qaa[a-z]|Qab[a-x]|Zzzz", script) or language.script_name() == language.language_name() + ): + script = None + if region and ( + region == "001" + or re.fullmatch(r"AA|Q[M-Z]|X[A-JL-Z]|ZZ", region) + or Language.make(language=code, script=script).maximize().territory == region + ): + region = None + return language.update_dict({"script": script, "territory": region}).display_name().replace(" SAR China", "") + + def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: """Examine whether a language exactly matches any of the given languages.""" languages = [x for x in languages if x] diff --git a/unshackle/services/EXAMPLE/__init__.py b/unshackle/services/EXAMPLE/__init__.py index 086e23e9..e1a6fd49 100644 --- a/unshackle/services/EXAMPLE/__init__.py +++ b/unshackle/services/EXAMPLE/__init__.py @@ -399,10 +399,11 @@ class EXAMPLE(Service): # tells us explicitly we can set it eagerly. if audio.data.get("isAtmos"): audio.joc = 16 - # is_original_lang was set by to_tracks (via the language= arg). Use it to - # label the original audio so it reads "Original" in selection/output. - if audio.is_original_lang: - audio.name = "Original" + # Leave audio.name unset: core names the track from its language and writes + # is_original_lang (set by to_tracks) as the original flag. Set a name only + # for what the language tag cannot say, such as a commentary track. + if "commentary" in label.lower(): + audio.name = "Commentary" # Subtitles built by hand (not via to_tracks) skip the parser's is_original_lang # pass, so determine it ourselves with the same helper the parser uses.