feat(tracks): name unnamed tracks by language and tag subtitle scripts

Unnamed tracks get the English language name (default region left out); subtitle tags gain the script their region implies (zh-TW -> zh-Hant-TW). Refs #169

docs/dev/rest-api/endpoints.md

@@ -333,7 +333,7 @@ When you give both `season` and `episode`, the server combines them into a `"{se
           "resolution": "2160p", "fps": "24", "range": "HDR10",
           "range_display": "HDR10", "scan_type": null, "closed_captions": [],
           "dv_compatible_bitstream": false, "hdr10plus": false, "language": "en", "is_original_lang": true,
-          "name": null, "needs_repack": false, "edition": [], "data": null,
+          "name": null, "track_name": "English", "needs_repack": false, "edition": [], "data": null,
           "drm": [], "descriptor": "DASH"
         }
       ],
@@ -342,14 +342,14 @@ When you give both `season` and `episode`, the server combines them into a `"{se
           "id": "a-1", "codec": "EC3", "codec_display": "DD+",
           "bitrate": 640, "bitrate_bps": 640000, "channels": "5.1", "language": "en",
           "is_original": true, "is_original_lang": true, "atmos": false, "joc": 0,
-          "descriptive": false, "name": null, "needs_repack": false, "edition": [],
+          "descriptive": false, "name": null, "track_name": "English", "needs_repack": false, "edition": [],
           "data": null, "drm": [], "descriptor": "DASH"
         }
       ],
       "subtitles": [
         { "id": "s-1", "codec": "vtt", "language": "en",
           "forced": false, "sdh": false, "cc": false, "is_original_lang": false,
-          "name": null, "needs_repack": false, "edition": [], "data": null,
+          "name": null, "track_name": "English", "needs_repack": false, "edition": [], "data": null,
           "drm": [], "descriptor": "DASH" }
       ]
     }
@@ -377,7 +377,7 @@ The server sorts tracks by bitrate, descending. Episodes the server cannot get g
 
 The `drm` array on a track is a list of `{type, pssh?, kids?, content_keys?, license_url?}` objects.
 
-Every track carries the same fields as a local track: `is_original_lang`, `name`, `needs_repack`, `edition`, and `data`. `bitrate` is in kb/s, and `bitrate_bps` gives the exact bits/s. `data` holds the part of the service-set `track.data` that fits in JSON, without the `hls`, `dash`, and `ism` parser keys. Audio keeps `is_original` and `atmos` for clients that read them, and adds the exact `joc` object count.
+Every track carries the same fields as a local track: `is_original_lang`, `name`, `needs_repack`, `edition`, and `data`. `name` is the name the service set, or null. `track_name` is the name `--list` shows and the mux writes to an audio or subtitle track: `name` when the service set one, else the English name of the language, with the SDH, CC, Forced or Descriptive label. `bitrate` is in kb/s, and `bitrate_bps` gives the exact bits/s. `data` holds the part of the service-set `track.data` that fits in JSON, without the `hls`, `dash`, and `ism` parser keys. Audio keeps `is_original` and `atmos` for clients that read them, and adds the exact `joc` object count.
 
 | Status | Error code | Meaning |
 | --- | --- | --- |

docs/guide/output-and-naming.md

@@ -284,6 +284,27 @@ warning and continues.
 !!! note "Height channels"
     A track with height channels is named bed.LFE.heights, such as `5.1.4`.
 
+### Track names and language tags
+
+When the service gives a track no name, unshackle names it with the English name of its language,
+such as `English`, `Portuguese (Portugal)` or `Spanish (Latin America)`. The name leaves out the
+region that is the default for the language, so `en-US` is `English` and `pt-BR` is `Portuguese`,
+but `en-GB` is `English (United Kingdom)`. A name from the service is kept as is. unshackle adds the SDH, CC, Forced or Descriptive label after either name.
+
+A subtitle in a language with more than one script gets the script from its region:
+
+| Service tag | Tag in the file | Track name |
+|-------------|-----------------|------------|
+| `zh-TW` | `zh-Hant-TW` | Chinese (Traditional) |
+| `zh-HK` | `zh-Hant-HK` | Chinese (Traditional, Hong Kong) |
+| `zh-CN` | `zh-Hans-CN` | Chinese (Simplified) |
+| `sr-ME` | `sr-Latn-ME` | Serbian (Latin, Montenegro) |
+| `zh` | `zh` | Chinese |
+
+A tag without a region, such as `zh`, gets no script because the region is what tells the script.
+unshackle writes audio tags as the service gives them, because a script applies only to written text.
+Track selection, `--list` tags and file names use the service tag.
+
 ### DTS:X Profile 2 output
 
 A title whose audio holds a `dtsx` or `dtsy` sample entry (DTS:X Profile 2, DTS-UHD) muxes

tests/core/test_mp4_container_routing.py

@@ -150,6 +150,30 @@ def test_substation_alpha_subtitles_are_left_out(tmp_path: Path, monkeypatch: py
     assert "s.vtt:lang=es" in command
 
 
+def test_mp4_subtitles_get_the_script_their_region_implies(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    captured: list[FakePopen] = []
+
+    def fake_popen(args: list[str], **kwargs: object) -> FakePopen:
+        p = FakePopen(args, **kwargs)
+        captured.append(p)
+        Path(args[-1]).write_bytes(b"muxed")
+        return p
+
+    monkeypatch.setattr(binaries, "MP4Box", Path("MP4Box"), raising=False)
+    monkeypatch.setattr(subprocess, "Popen", fake_popen)
+
+    tracks = build_tracks(tmp_path, b"dtsx")
+    vtt = Subtitle(id_="s1", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="zh-TW")
+    vtt.path = tmp_path / "s.vtt"
+    vtt.path.write_text("WEBVTT\n")
+    tracks.add(vtt)
+
+    tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv")
+    command = captured[0].args
+    assert f"{tmp_path / 's.vtt'}:lang=zh-Hant-TW" in command
+    assert any(arg.endswith("=Chinese (Traditional)") for arg in command)
+
+
 def test_mp4box_errors_come_back_to_the_caller(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     """A failed mux does not raise; the caller inspects the return code and lines."""
 

tests/tracks/test_track_language_names.py

@@ -0,0 +1,143 @@
+from pathlib import Path
+from typing import Optional
+
+import pytest
+from langcodes import Language
+
+from unshackle.core import binaries
+from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
+from unshackle.core.utilities import language_name, multi_script_languages, with_script
+
+
+def sub(lang: str, name: Optional[str] = None, sdh: bool = False) -> Subtitle:
+    return Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language=lang, name=name, sdh=sdh)
+
+
+def test_multi_script_languages_come_from_cldr() -> None:
+    assert {"zh", "yue", "sr"} <= multi_script_languages()
+    assert not {"en", "pt", "ja"} & multi_script_languages()
+
+
+@pytest.mark.parametrize(
+    ("tag", "expected"),
+    [
+        ("zh", "zh"),
+        ("zh-TW", "zh-Hant-TW"),
+        ("zh-CN", "zh-Hans-CN"),
+        ("zh-HK", "zh-Hant-HK"),
+        ("zh-SG", "zh-Hans-SG"),
+        ("zh-Hant", "zh-Hant"),
+        ("yue-HK", "yue-Hant-HK"),
+        ("sr-ME", "sr-Latn-ME"),
+        ("sr-RS", "sr-Cyrl-RS"),
+        ("pa-PK", "pa-Arab-PK"),
+        ("ms-MY", "ms-MY"),
+        ("en-US", "en-US"),
+        ("und-TW", "und-TW"),
+        ("und", "und"),
+    ],
+)
+def test_with_script(tag: str, expected: str) -> None:
+    assert str(with_script(Language.get(tag))) == expected
+
+
+@pytest.mark.parametrize(
+    ("tag", "expected"),
+    [
+        ("zh-Hant-TW", "Chinese (Traditional)"),
+        ("zh-Hans-CN", "Chinese (Simplified)"),
+        ("zh-Hant-HK", "Chinese (Traditional, Hong Kong)"),
+        ("zh-TW", "Chinese (Taiwan)"),
+        ("pt-BR", "Portuguese"),
+        ("pt-PT", "Portuguese (Portugal)"),
+        ("en-US", "English"),
+        ("en-GB", "English (United Kingdom)"),
+        ("ja-JP", "Japanese"),
+        ("fr-CA", "French (Canada)"),
+        ("ar-001", "Arabic"),
+        ("es-419", "Spanish (Latin America)"),
+        ("en", "English"),
+        ("de-DE", "German"),
+        ("en-ZZ", "English"),
+        ("en-XA", "English"),
+        ("en-Zzzz", "English"),
+        ("sq-XK", "Albanian (Kosovo)"),
+        ("mn-Mong-CN", "Mongolian (China)"),
+        ("und", None),
+        ("qaa", None),
+        ("x-foo", None),
+    ],
+)
+def test_language_name(tag: str, expected: Optional[str]) -> None:
+    assert language_name(Language.get(tag)) == expected
+
+
+def test_subtitle_names_and_tags_the_script_but_keeps_its_language() -> None:
+    track = sub("zh-TW")
+    assert track.name is None
+    assert track.get_track_name() == "Chinese (Traditional)"
+    assert str(track.mux_language) == "zh-Hant-TW"
+    assert str(track.language) == "zh-TW"
+
+
+def test_subtitle_flag_follows_the_default_name() -> None:
+    assert sub("zh-CN", sdh=True).get_track_name() == "Chinese (Simplified) (SDH)"
+
+
+def test_audio_gets_no_script() -> None:
+    track = Audio(id_="a", url="https://x/a", codec=Audio.Codec.AAC, language="zh-TW", bitrate=1)
+    assert track.get_track_name() == "Chinese (Taiwan)"
+    assert str(track.mux_language) == "zh-TW"
+
+
+def test_a_service_name_wins() -> None:
+    assert sub("zh", name="简体中文").get_track_name() == "简体中文"
+
+
+def test_mkvmerge_gets_the_subtitle_script_and_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    class Stop(Exception):
+        pass
+
+    captured: list[list[str]] = []
+
+    def fake_popen(args: list[str], **kwargs: object) -> None:
+        captured.append(args)
+        raise Stop
+
+    monkeypatch.setattr("unshackle.core.tracks.tracks.subprocess.Popen", fake_popen)
+    monkeypatch.setattr(binaries, "MKVToolNix", Path("mkvmerge"))
+    video = Video(id_="v", url="https://x/v", codec=Video.Codec.AVC, language="zh-TW", width=1920, height=1080)
+    audio = Audio(id_="a", url="https://x/a", codec=Audio.Codec.AAC, language="zh-TW", bitrate=1)
+    subtitle = sub("zh-TW")
+    for track, file in ((video, "v.mp4"), (audio, "a.mp4"), (subtitle, "s.vtt")):
+        track.path = tmp_path / file
+        track.path.write_bytes(b"\x00" * 16)
+
+    with pytest.raises(Stop):
+        Tracks([video, audio, subtitle]).mux("T", delete=False, output_path=tmp_path / "o.mkv")
+
+    def last_option(file: str, flag: str) -> str:
+        """Return the value of the last flag given before the file, which is the one mkvmerge applies to it."""
+        before = captured[0][: captured[0].index(str(tmp_path / file))]
+        return before[len(before) - before[::-1].index(flag)]
+
+    assert last_option("s.vtt", "--language") == "0:zh-Hant-TW"
+    assert last_option("s.vtt", "--track-name") == "0:Chinese (Traditional)"
+    assert last_option("a.mp4", "--language") == "0:zh-TW"
+
+
+def test_the_default_name_follows_a_later_language_change() -> None:
+    track = sub("yue")
+    track.language = Language.get("zh-Hant")
+    assert track.get_track_name() == "Chinese (Traditional)"
+
+
+def test_a_language_assigned_as_text_later_is_parsed() -> None:
+    track = sub("pt")
+    track.language = "pt-PT"
+    assert track.language == Language.get("pt-PT")
+    assert track.get_track_name() == "Portuguese (Portugal)"
+    assert str(track) == "SUB | [VTT] | pt-PT | Portuguese (Portugal)"
+    track.language = None
+    assert str(track.language) == "und"
+    assert track.get_track_name() is None

unshackle/core/api/handlers.py

@@ -1718,6 +1718,7 @@ def serialize_track_common(track: Any) -> Dict[str, Any]:
         "language": str(track.language) if track.language else None,
         "is_original_lang": bool(getattr(track, "is_original_lang", False)),
         "name": getattr(track, "name", None),
+        "track_name": track.get_track_name(),
         "needs_repack": bool(getattr(track, "needs_repack", False)),
         "edition": list(getattr(track, "edition", None) or []),
         "descriptor": descriptor_name(track),

unshackle/core/tracks/subtitle.py

@@ -15,6 +15,7 @@ import click
 import pycaption
 import requests
 from construct import Container
+from langcodes import Language
 from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader
 from pycaption.geometry import Layout
 from pymp4.parser import MP4
@@ -24,7 +25,7 @@ from subtitle_filter import Subtitles
 from unshackle.core import binaries
 from unshackle.core.config import config
 from unshackle.core.tracks.track import Track
-from unshackle.core.utilities import try_ensure_utf8
+from unshackle.core.utilities import try_ensure_utf8, with_script
 from unshackle.core.utils.subprocess import log_tool_run
 from unshackle.core.utils.webvtt import merge_segmented_webvtt
 
@@ -221,6 +222,11 @@ class Subtitle(Track):
             forced=data.get("forced", False),
         )
 
+    @property
+    def mux_language(self) -> Language:
+        """The language tag written to the output file, with the script its region implies."""
+        return with_script(self.language)
+
     def __str__(self) -> str:
         return " | ".join(
             filter(

unshackle/core/tracks/track.py

@@ -35,7 +35,7 @@ from unshackle.core.events import events
 from unshackle.core.session import MAX_BACKOFF, RnetSession, utf8_text
 from unshackle.core.temp import restore_segments, stash_segments
 from unshackle.core.tracks import resume
-from unshackle.core.utilities import get_boxes, log_event, try_ensure_utf8
+from unshackle.core.utilities import get_boxes, language_name, log_event, try_ensure_utf8
 from unshackle.core.utils import run_proxy
 from unshackle.core.utils.subprocess import ffprobe
 
@@ -467,7 +467,7 @@ class Track:
 
         self.path: Optional[Path] = None
         self.url = url
-        self.language = Language.get(language)
+        self.language = language
         self.is_original_lang = is_original_lang
         self.descriptor = descriptor
         self.needs_repack = needs_repack
@@ -483,23 +483,6 @@ class Track:
         self.data = data or {}
         self.extra: Any = extra or {}  # allow anything for extra, but default to a dict
 
-        if self.name is None:
-            lang = self.language
-            if (lang.language or "").lower() == (lang.territory or "").lower():
-                lang = lang.update_dict({"territory": None})
-            reduced = lang.simplify_script()
-            extra_parts = []
-            if reduced.script is not None:
-                script = reduced.script_name(max_distance=25)
-                if script and script != "Zzzz":
-                    extra_parts.append(script)
-            if reduced.territory is not None:
-                territory = reduced.territory_name(max_distance=25)
-                if territory and territory != "ZZ":
-                    territory = territory.removesuffix(" SAR China")
-                    extra_parts.append(territory)
-            self.name = ", ".join(extra_parts) or None
-
         if not id_:
             this = copy(self)
             this.url = self.url.rsplit("?", maxsplit=1)[0]
@@ -588,6 +571,15 @@ class Track:
     def __eq__(self, other: Any) -> bool:
         return isinstance(other, Track) and self.id == other.id
 
+    @property
+    def language(self) -> Language:
+        """The language the service tagged, parsed even when a service assigns a string or None later."""
+        return self._language
+
+    @language.setter
+    def language(self, value: Union[Language, str, None]) -> None:
+        self._language = Language.get(value or "und")
+
     @property
     def data(self) -> defaultdict[Any, Any]:
         """
@@ -1015,9 +1007,14 @@ class Track:
             raise ValueError(f"Cannot reconstruct unsupported track type: {track_type!r}")
         return builder.from_dict(data)
 
+    @property
+    def mux_language(self) -> Language:
+        """The language tag written to the output file."""
+        return self.language
+
     def get_track_name(self) -> Optional[str]:
-        """Get the Track Name."""
-        return self.name
+        """Get the Track Name, or the English name of its language when the service set none."""
+        return self.name if self.name is not None else language_name(self.mux_language)
 
     def get_drm_for_cdm(self, cdm: Optional[object]) -> Optional[DRM_T]:
         """Return the DRM matching the provided CDM, if available."""

unshackle/core/tracks/tracks.py

@@ -28,6 +28,7 @@ from unshackle.core.tracks.track import Track, has_dts_uhd_sample_entry, strip_d
 from unshackle.core.tracks.video import Video
 from unshackle.core.utilities import (
     is_close_match,
+    language_name,
     log_event,
     matching_languages,
     original_tag,
@@ -606,7 +607,7 @@ class Tracks:
                 candidate.unlink()
 
             delay = round((track.sample_start - sample_base) * 1000) if config.sample else 0
-            cl.extend(["-add", f"{source}:lang={track.language}" + (f":delay={delay}" if delay else "")])
+            cl.extend(["-add", f"{source}:lang={track.mux_language}" + (f":delay={delay}" if delay else "")])
             name = "" if isinstance(track, Video) else (track.get_track_name() or "")
             if name:
                 names.extend(["-name", f"{track_id}={name}"])
@@ -765,7 +766,7 @@ class Tracks:
             video_track = video_track or self.videos[0]
             if video_track.language.is_valid():
                 lang_code = str(video_track.language)
-                lang_name = video_track.language.display_name()
+                lang_name = language_name(video_track.language) or lang_code
 
                 for video in self.videos:
                     if video.data.get("audio_language"):
@@ -832,7 +833,7 @@ class Tracks:
 
             video_args = [
                 "--language",
-                f"0:{vt.language}",
+                f"0:{vt.mux_language}",
                 "--default-track",
                 f"0:{is_default}",
                 "--original-flag",
@@ -888,7 +889,7 @@ class Tracks:
                     "--track-name",
                     f"0:{at.get_track_name() or ''}",
                     "--language",
-                    f"0:{at.language}",
+                    f"0:{at.mux_language}",
                     "--default-track",
                     f"0:{audio_default}",
                     "--visual-impaired-flag",
@@ -918,7 +919,7 @@ class Tracks:
                         "--track-name",
                         f"0:{st.get_track_name() or ''}",
                         "--language",
-                        f"0:{st.language}",
+                        f"0:{st.mux_language}",
                         "--sub-charset",
                         "0:UTF-8",
                         "--forced-track",

unshackle/core/utilities.py

@@ -15,6 +15,7 @@ import unicodedata
 import zlib
 from collections import defaultdict
 from datetime import datetime, timezone
+from functools import cache
 from pathlib import Path
 from types import ModuleType
 from typing import Any, Callable, Mapping, Optional, Sequence, Union
@@ -26,6 +27,7 @@ import pycountry
 from construct import ValidationError
 from fontTools import ttLib
 from langcodes import Language, closest_match
+from langcodes.data_dicts import LIKELY_SUBTAGS
 from langcodes.tag_parser import LanguageTagError
 from pymp4.parser import Box
 from unidecode import unidecode
@@ -169,6 +171,53 @@ def valid_language(language: Any) -> Optional[str]:
     return str(language)
 
 
+@cache
+def multi_script_languages() -> frozenset[str]:
+    """Return the languages whose likely script changes with the region in CLDR (zh, yue, sr, ...)."""
+    found = set()
+    for tag, likely in LIKELY_SUBTAGS.items():
+        match = re.fullmatch(r"([a-z]{2,3})-(?:[A-Z]{2}|\d{3})", tag)
+        if match and match[1] != "und" and Language.get(likely).script != Language.get(match[1]).maximize().script:
+            found.add(match[1])
+    return frozenset(found)
+
+
+def with_script(language: Language) -> Language:
+    """
+    Add the script a region implies to a subtitle tag, e.g. zh-TW to zh-Hant-TW.
+
+    Only a language written in several scripts gets one, and only when the tag has a region:
+    a bare zh says nothing about the script, so it stays zh.
+    """
+    if language.script or not language.territory or language.language not in multi_script_languages():
+        return language
+    return language.update_dict({"script": language.maximize().script}).simplify_script()
+
+
+def language_name(language: Language) -> Optional[str]:
+    """
+    Return the English name of a track language, or None for an unknown or private-use language.
+
+    It leaves out a script or region that adds nothing: private use (but XK, Kosovo), 001, the CLDR default
+    region of the language and script (pt-BR, zh-Hant-TW, but not pt-PT), a script named like the language.
+    """
+    code = language.language
+    if not code or not Language.make(language=code).has_name_data():
+        return None
+    script, region = language.script, language.territory
+    if script and (
+        re.fullmatch(r"Qaa[a-z]|Qab[a-x]|Zzzz", script) or language.script_name() == language.language_name()
+    ):
+        script = None
+    if region and (
+        region == "001"
+        or re.fullmatch(r"AA|Q[M-Z]|X[A-JL-Z]|ZZ", region)
+        or Language.make(language=code, script=script).maximize().territory == region
+    ):
+        region = None
+    return language.update_dict({"script": script, "territory": region}).display_name().replace(" SAR China", "")
+
+
 def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool:
     """Examine whether a language exactly matches any of the given languages."""
     languages = [x for x in languages if x]

unshackle/services/EXAMPLE/__init__.py

@@ -399,10 +399,11 @@ class EXAMPLE(Service):
             # tells us explicitly we can set it eagerly.
             if audio.data.get("isAtmos"):
                 audio.joc = 16
-            # is_original_lang was set by to_tracks (via the language= arg). Use it to
-            # label the original audio so it reads "Original" in selection/output.
-            if audio.is_original_lang:
-                audio.name = "Original"
+            # Leave audio.name unset: core names the track from its language and writes
+            # is_original_lang (set by to_tracks) as the original flag. Set a name only
+            # for what the language tag cannot say, such as a commentary track.
+            if "commentary" in label.lower():
+                audio.name = "Commentary"
 
         # Subtitles built by hand (not via to_tracks) skip the parser's is_original_lang
         # pass, so determine it ourselves with the same helper the parser uses.