From 235839abc8051b2cb022288f4ec32f4afeb66027 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Fri, 4 Sep 2026 14:19:29 -0600 Subject: [PATCH] feat(audio): support DTS:X Profile 2 with MP4 muxing Route titles with a dtsx/dtsy sample entry to MP4 via MP4Box, since Matroska has no CodecID for DTS-UHD. Add per-rendition HLS audio codec selection, fix merge_segments progress and its cleanup path. --- docs/dev/rest-api/endpoints.md | 2 +- docs/dev/service-architecture.md | 2 +- docs/getting-started/installation.md | 5 + docs/guide/cli-reference.md | 2 +- docs/guide/downloading.md | 1 + docs/guide/output-and-naming.md | 26 ++- docs/index.md | 2 +- tests/core/test_audio_channel_naming.py | 53 +++++ tests/core/test_hls_merge_segments.py | 140 +++++++++++++ tests/core/test_mp4_container_routing.py | 246 +++++++++++++++++++++++ tests/core/test_music_container.py | 3 +- tests/tracks/test_audio_codec_mime.py | 117 +++++++++++ unshackle/core/api/handlers.py | 18 +- unshackle/core/api/routes.py | 2 +- unshackle/core/constants.py | 4 +- unshackle/core/manifests/hls.py | 104 ++++++++-- unshackle/core/titles/title.py | 26 ++- unshackle/core/tracks/audio.py | 5 +- unshackle/core/tracks/track.py | 117 +++++++++++ unshackle/core/tracks/tracks.py | 182 ++++++++++++++++- unshackle/core/utils/tags.py | 3 + 21 files changed, 1020 insertions(+), 40 deletions(-) create mode 100644 tests/core/test_audio_channel_naming.py create mode 100644 tests/core/test_hls_merge_segments.py create mode 100644 tests/core/test_mp4_container_routing.py create mode 100644 tests/tracks/test_audio_codec_mime.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 06c5d61d..6168e07d 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -432,7 +432,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `output_dir` | string | `null` | Override output directory. | | `no_cache` / `reset_cache` | boolean | `false` | Title cache controls. | -**Validation.** Invalid values return `400 INVALID_PARAMETERS`. `vcodec` must be one of H264/H265/H.264/H.265/AVC/HEVC/VC1/VC-1/VP8/VP9/AV1. `acodec` must be one of AAC/AC3/EC3/EAC3/DD/DD+/AC4/OPUS/FLAC/ALAC/VORBIS/OGG/DTS. `range` must be one of SDR/HDR10/HDR10P/DV/HLG/HYBRID, and `HDR10+` is also valid. The bitrate, download worker, and download counts must be positive integers. You may set at most one of the `*_only` flags. You cannot combine `no_subs` with `subs_only`, or `no_audio` with `audio_only`. +**Validation.** Invalid values return `400 INVALID_PARAMETERS`. `vcodec` must be one of H264/H265/H.264/H.265/AVC/HEVC/VC1/VC-1/VP8/VP9/AV1. `acodec` must be one of AAC/AC3/EC3/EAC3/DD/DD+/AC4/OPUS/FLAC/ALAC/VORBIS/OGG/DTS/DTSX/DTS-X. `range` must be one of SDR/HDR10/HDR10P/DV/HLG/HYBRID, and `HDR10+` is also valid. The bitrate, download worker, and download counts must be positive integers. You may set at most one of the `*_only` flags. You cannot combine `no_subs` with `subs_only`, or `no_audio` with `audio_only`. **Service options.** A service's own CLI options travel in the same body. Send them in a nested `service_params` object, or as flat top-level keys. `service_params` wins over a flat key of the diff --git a/docs/dev/service-architecture.md b/docs/dev/service-architecture.md index d4c45bdb..bafe2d7c 100644 --- a/docs/dev/service-architecture.md +++ b/docs/dev/service-architecture.md @@ -282,7 +282,7 @@ closed-caption descriptors. The relevant enums: `Audio` adds `bitrate`, `channels`, `joc` (Joint Object Coding, the Atmos indicator), and `descriptive`. `Audio.Codec` covers `AAC`, `AC3` (`DD`), `EC3` (`DD+`), `AC4`, -`OPUS`, `OGG` (Vorbis), `DTS`, `ALAC`, and `FLAC`. The `atmos` property is `True` when +`OPUS`, `OGG` (Vorbis), `DTS`, `DTSX`, `ALAC`, and `FLAC`. The `atmos` property is `True` when `joc` is set (or when `extra` flags it). ### `Subtitle` diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 8147774e..efbf9e47 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -124,6 +124,7 @@ download. For example, `dovi_tool` only matters if you download Dolby Vision vid | Tool | Binary looked up | Why it is needed | | --- | --- | --- | | Bento4 (`mp4decrypt`) | `mp4decrypt` | Alternative DRM decryptor to shaka-packager | +| MP4Box (GPAC) | `MP4Box` | Muxing DTS:X Profile 2 (DTS-UHD) audio | | dovi_tool | `dovi_tool` | Dolby Vision metadata handling | | HDR10Plus_tool | `hdr10plus_tool` | HDR10+ metadata handling | | SubtitleEdit | `seconv`, `SubtitleEdit` | Subtitle conversion (the `SeConv` CLI from 5.x) | @@ -135,6 +136,10 @@ download. For example, `dovi_tool` only matters if you download Dolby Vision vid | Docker | `docker` | Gluetun VPN proxy support | | Git | `git` | Fetching and updating remote service repositories | +!!! info "MP4Box is only used for DTS:X Profile 2" + Titles with DTS:X Profile 2 (DTS-UHD) audio mux to MP4; everything else muxes to MKV + with `mkvmerge`. Install with `apt install gpac`. + !!! tip "SubtitleEdit specifics" unshackle looks for `seconv` first, the batch CLI shipped with SubtitleEdit 5.x, before falling back to a `SubtitleEdit` (4.x) binary. The diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index bb42b7ef..5684c719 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -70,7 +70,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | `-p`, `--profile` | - | Profile for credentials and cookies. | | `-q`, `--quality` | best | Resolution(s), comma-separated, e.g. `1080,720`. `-q 1080` also matches anamorphic tracks by 16:9 canvas. | | `-v`, `--vcodec` | any | Video codec(s). Accepts names or values: `AVC`/`H.264`, `HEVC`/`H.265`, `VC1`, `VP8`, `VP9`, `AV1`. | -| `-a`, `--acodec` | any | Audio codec(s), comma-separated. Accepts `AAC`, `AC3`/`DD`, `EC3`/`DD+`/`eac3`/`ddp`, `AC4`, `OPUS`, `OGG`/`vorbis`, `DTS`, `ALAC`, `FLAC`. | +| `-a`, `--acodec` | any | Audio codec(s), comma-separated. Accepts `AAC`, `AC3`/`DD`, `EC3`/`DD+`/`eac3`/`ddp`, `AC4`, `OPUS`, `OGG`/`vorbis`, `DTS`, `DTSX`/`DTS-X`, `ALAC`, `FLAC`. | | `-vb`, `--vbitrate` | highest | Exact video bitrate in kbps. | | `-ab`, `--abitrate` | highest | Exact audio bitrate in kbps. | | `-vb-range`, `--vbitrate-range` | - | Video bitrate range in kbps, e.g. `6000-7000`; picks highest within. Mutually exclusive with `--vbitrate`. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 84890e5b..de2c9ef9 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -215,6 +215,7 @@ bitrate for the top renditions, overriding the manifest value: | `OPUS` | `OPUS` | Opus | | `OGG` | `VORB` | Vorbis | | `DTS` | `DTS` | DTS | +| `DTSX` | `DTS-X` | DTS:X Profile 2 (DTS-UHD) | | `ALAC` | `ALAC` | Apple Lossless | | `FLAC` | `FLAC` | FLAC | diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 71f4aa01..d30be8ea 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -15,7 +15,7 @@ When a download finishes, unshackle builds the final path from three pieces: - **Output directory**: `directories.downloads` from your config, or whatever you pass to `-o/--output` on the command line for a single run. - **Folder template**: an optional per-title subfolder. unshackle **always** puts TV episodes and songs in a folder. Movies only get one if you define a `movies` folder template, or a single-string `folder` template that applies to all title kinds (see [Folder templates](#folder-templates)). - **Filename template**: the `output_template` for the title kind (movie, series, or song). -- **Extension**: chosen by the muxer for a movie or an episode: `.mkv` for video, `.mka` for audio-only, `.mks` for subtitle-only. unshackle does not mux a song. It remuxes a song into the container of its codec, such as `.flac`, `.opus`, or `.m4a`. Read [Music output files](#music-output-files). +- **Extension**: chosen by the muxer for a movie or an episode: `.mkv` for video, `.mka` for audio-only, `.mks` for subtitle-only, `.mp4` for [DTS:X Profile 2 audio](#dtsx-profile-2-output). unshackle does not mux a song. It remuxes a song into the container of its codec, such as `.flac`, `.opus`, or `.m4a`. Read [Music output files](#music-output-files). !!! example "A typical episode path" ```text @@ -163,9 +163,9 @@ For custom templates there is also a standalone [`{part}`](../reference/configur | Variable | Meaning | Example | |---|---|---| -| `audio` | Audio codec | `DDP`, `DD`, `AAC` | -| `audio_channels` | Channel layout | `5.1`, `2.0` | -| `audio_full` | Codec + channels combined | `DDP5.1` | +| `audio` | Audio codec | `DDP`, `DD`, `AAC`, `DTS-X` | +| `audio_channels` | Channel layout; bed.LFE.heights when the track has height channels | `5.1`, `2.0`, `5.1.4` | +| `audio_full` | Codec + channels combined | `DDP5.1`, `DTS-X.5.1.4` | | `atmos` | `Atmos` when any track carries JOC | `Atmos` | | `dual` | `DUAL` for two audio languages (see [`dual_multi_mode`](../reference/configuration/download.md#dual_multi_mode)) | `DUAL` | | `multi` | `MULTi` for more than two audio languages | `MULTi` | @@ -277,6 +277,24 @@ warning and continues. ## Muxing options +!!! note "Height channels" + A track with height channels is named bed.LFE.heights, such as `5.1.4`. + +### DTS:X Profile 2 output + +A title whose audio holds a `dtsx` or `dtsy` sample entry (DTS:X Profile 2, DTS-UHD) muxes +to MP4 with [MP4Box](../getting-started/installation.md), because Matroska has no CodecID +for it. Without MP4Box installed the mux stops. + +DTS:X Profile 1 is DTS-HD Master Audio and muxes to MKV as usual. + +An MP4 title loses: + +- the forced, hearing-impaired and original track flags +- attachments, such as subtitle fonts +- SubStation Alpha (`.ass`) subtitles +- container tags (`mkvpropedit`) + You configure muxing (the combination of video, audio, subtitle, chapter and attachment tracks into a single Matroska file with `mkvmerge`) under the `muxing` config key. unshackle muxes only movies and episodes. Read [Music output files](#music-output-files) for what a music download does instead. ```yaml title="unshackle.yaml" diff --git a/docs/index.md b/docs/index.md index 12e9780b..b37b6506 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,7 +38,7 @@ at a title and it will: | Manifests | DASH, HLS, and Smooth Streaming (ISM) | | DRM | Widevine and PlayReady, with ClearKey support | | Video | AVC, HEVC, VC-1, VP8, VP9, AV1; SDR, HLG, HDR10, HDR10+, Dolby Vision, and Dolby Vision **hybrid** merging | -| Audio | AAC, AC-3, E-AC-3, AC-4, Opus, Vorbis, DTS, ALAC, FLAC; Dolby Atmos handling; per-channel-layout selection | +| Audio | AAC, AC-3, E-AC-3, AC-4, Opus, Vorbis, DTS, DTS-X, ALAC, FLAC; Dolby Atmos handling; per-channel-layout selection | | Subtitles | SRT, WebVTT, ASS/SSA, TTML and more, with optional SDH stripping and format conversion | | Key vaults | SQLite, MySQL, and HTTP/API vaults to store and share content keys | | Proxies | Basic proxies plus NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers | diff --git a/tests/core/test_audio_channel_naming.py b/tests/core/test_audio_channel_naming.py new file mode 100644 index 00000000..700146bd --- /dev/null +++ b/tests/core/test_audio_channel_naming.py @@ -0,0 +1,53 @@ +"""Audio codec and channel layout in a filename.""" + +from __future__ import annotations + +import types + +import pytest + +from unshackle.core.titles.movie import Movie + + +class DummyService: + pass + + +def named(fmt: str, layout: str | None, channel_s: int = 6) -> dict: + """Build the naming context from a MediaInfo audio track alone.""" + audio = types.SimpleNamespace( + language="en", + format=fmt, + channel_layout=layout, + channellayout_original=None, + channel_s=channel_s, + channels=channel_s, + format_additionalfeatures=None, + joc=None, + ) + movie = Movie(id_="movie-0001", service=DummyService, name="The Film", year=2024, language="en") + return movie.build_base_template_context(types.SimpleNamespace(video_tracks=[], audio_tracks=[audio])) + + +def test_a_height_layout_is_named_bed_lfe_heights() -> None: + """ + DTS:X and Atmos carry height channels. Folding them into the bed gives 9.1, which names + nine ear-level speakers: a different layout from five plus four overhead. + """ + context = named("DTS-UHD", "L C R LFE Ls Rs Tfl Tfr Tbl Tbr", channel_s=10) + assert context["audio_channels"] == "5.1.4" + assert context["audio"] == "DTS-X", "the filename carries the brand, not the bitstream name" + assert context["audio_full"] == "DTS-X 5.1.4" + + +@pytest.mark.parametrize( + ("layout", "expected"), + [ + ("L R C LFE Ls Rs", "5.1"), + ("L R C LFE Ls Rs Lb Rb", "7.1"), + ("L R", "2.0"), + (None, "6.0"), + ], +) +def test_layouts_without_heights_are_unchanged(layout: str | None, expected: str) -> None: + assert named("E-AC-3", layout)["audio_channels"] == expected diff --git a/tests/core/test_hls_merge_segments.py b/tests/core/test_hls_merge_segments.py new file mode 100644 index 00000000..3ee9aefc --- /dev/null +++ b/tests/core/test_hls_merge_segments.py @@ -0,0 +1,140 @@ +"""HLS.merge_segments must keep the ffmpeg child off the terminal. + +merge_segments runs while the download progress tree is live. A child that inherits the +terminal moves the cursor out from under rich, which leaves an orphaned copy of the tree +on screen. Real trigger: ffmpeg refuses DTS-UHD audio with "Cannot map stream #0:0". +""" + +import os +import subprocess +import tempfile +from pathlib import Path + +import pytest + +from unshackle.core.manifests.hls import HLS + +FFMPEG_ERROR = "[out#0/mp4 @ 0x0] Cannot map stream #0:0 - unsupported type." + + +@pytest.fixture +def segments(tmp_path: Path) -> list[Path]: + save_dir = tmp_path / "Audio_segments" + save_dir.mkdir() + paths = [] + for i in range(3): + path = save_dir / f"{i:05}.mp4" + path.write_bytes(f"segment-{i}".encode()) + paths.append(path) + return paths + + +def _capture_fd(fd: int): + """Redirect a raw file descriptor to a temp file, returning (restore, read).""" + sink = tempfile.TemporaryFile() + saved = os.dup(fd) + os.dup2(sink.fileno(), fd) + + def restore() -> bytes: + os.dup2(saved, fd) + os.close(saved) + sink.seek(0) + data = sink.read() + sink.close() + return data + + return restore + + +def test_failed_ffmpeg_concat_writes_nothing_to_the_terminal( + tmp_path: Path, segments: list[Path], monkeypatch: pytest.MonkeyPatch +) -> None: + def fake_run(args, **kwargs): + # emulate the child: with no capture it writes straight to the inherited terminal + if not kwargs.get("capture_output") and kwargs.get("stderr") is None: + os.write(2, (FFMPEG_ERROR + "\n").encode()) + raise subprocess.CalledProcessError(1, args) + raise subprocess.CalledProcessError(1, args, output="", stderr=FFMPEG_ERROR) + + monkeypatch.setattr("unshackle.core.manifests.hls.binaries.FFMPEG", Path("ffmpeg")) + monkeypatch.setattr("unshackle.core.manifests.hls.subprocess.run", fake_run) + + save_path = tmp_path / "Audio.mp4" + read_stderr = _capture_fd(2) + try: + HLS.merge_segments(segments=segments, save_path=save_path) + finally: + leaked = read_stderr() + + assert leaked == b"", f"ffmpeg output reached the terminal: {leaked!r}" + assert save_path.read_bytes() == b"segment-0segment-1segment-2" + + +def test_failed_ffmpeg_concat_logs_the_reason( + tmp_path: Path, segments: list[Path], monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + def fake_run(args, **kwargs): + raise subprocess.CalledProcessError(1, args, output="", stderr=FFMPEG_ERROR) + + monkeypatch.setattr("unshackle.core.manifests.hls.binaries.FFMPEG", Path("ffmpeg")) + monkeypatch.setattr("unshackle.core.manifests.hls.subprocess.run", fake_run) + + with caplog.at_level("DEBUG", logger="HLS"): + HLS.merge_segments(segments=segments, save_path=tmp_path / "Audio.mp4") + + assert FFMPEG_ERROR in caplog.text + + +def test_binary_concat_advances_progress_per_segment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """FFmpeg concat reports nothing, so the fallback is the only place a count can come from.""" + segments = [] + for i in range(4): + seg = tmp_path / f"seg{i}.mp4" + seg.write_bytes(b"\x00" * 16) + segments.append(seg) + + calls: list[dict] = [] + # no ffmpeg binary means merge_segments takes the binary fallback + monkeypatch.setattr("unshackle.core.manifests.hls.binaries.FFMPEG", None) + HLS.merge_segments( + segments=segments, + save_path=tmp_path / "out.mp4", + progress=lambda **kwargs: calls.append(kwargs), + ) + + assert calls.count({"advance": 1}) == len(segments), f"expected one advance per segment, got {calls}" + assert (tmp_path / "out.mp4").stat().st_size == 4 * 16 + + +def test_merge_segments_without_progress_still_works(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """progress stays optional; other callers pass nothing.""" + seg = tmp_path / "seg0.mp4" + seg.write_bytes(b"\x01" * 8) + monkeypatch.setattr("unshackle.core.manifests.hls.binaries.FFMPEG", None) + size = HLS.merge_segments(segments=[seg], save_path=tmp_path / "out.mp4") + assert size == 8 + + +def test_cleanup_never_climbs_past_the_segment_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """ + A download directory may itself say "_segments". Only the track's own segment store + may be removed, never a parent that merely contains the word. + """ + library = tmp_path / "media_segments" + store = library / "temp" / "task_x" / "Audio_1.mp4_segments" / "segments" + store.mkdir(parents=True) + keep = library / "Some Other Movie.mkv" + keep.write_bytes(b"do not delete me") + + segments = [] + for i in range(2): + seg = store / f"{i:05}.mp4" + seg.write_bytes(b"\x00" * 8) + segments.append(seg) + + monkeypatch.setattr("unshackle.core.manifests.hls.binaries.FFMPEG", None) + HLS.merge_segments(segments=segments, save_path=library / "out.mp4") + + assert keep.exists(), "cleanup deleted an unrelated file in the download directory" + assert library.exists() + assert not (library / "temp" / "task_x" / "Audio_1.mp4_segments").exists(), "the segment store was left behind" diff --git a/tests/core/test_mp4_container_routing.py b/tests/core/test_mp4_container_routing.py new file mode 100644 index 00000000..0f905063 --- /dev/null +++ b/tests/core/test_mp4_container_routing.py @@ -0,0 +1,246 @@ +"""Container selection for audio Matroska cannot name. + +DTS:X ships in two shapes. Profile 2 (DTS-UHD, sample entry dtsx/dtsy) has no Matroska +CodecID, so mkvmerge stores it as an A_QUICKTIME passthrough and no reader can name the +codec; those titles have to go to MP4. Profile 1 rides inside DTS-HD Master Audio +(dtsc/dtse/dtsh/ +dtsl), which Matroska maps to A_DTS and stores properly, so it must stay in Matroska. +""" + +from __future__ import annotations + +import io +import struct +import subprocess +from pathlib import Path + +import pytest + +from unshackle.core import binaries +from unshackle.core.tracks import Audio, Subtitle, Tracks, Video +from unshackle.core.tracks.track import ( + has_dts_uhd_sample_entry, + iter_top_level_boxes, + strip_duplicate_init_boxes, +) + + +def box(box_type: bytes, payload: bytes = b"") -> bytes: + return struct.pack(">I", 8 + len(payload)) + box_type + payload + + +def write_mp4(path: Path, sample_entry: bytes) -> Path: + """An MP4 whose moov holds one audio sample entry of the given 4CC.""" + entry = box(sample_entry, b"\x00" * 20) + path.write_bytes(box(b"ftyp", b"isom") + box(b"moov", box(b"trak", box(b"stsd", entry)))) + return path + + +@pytest.mark.parametrize("fourcc", [b"dtsx", b"dtsy"]) +def test_dts_uhd_sample_entries_are_detected(tmp_path: Path, fourcc: bytes) -> None: + assert has_dts_uhd_sample_entry(write_mp4(tmp_path / "a.mp4", fourcc)) is True + + +@pytest.mark.parametrize("fourcc", [b"dtsc", b"dtse", b"dtsh", b"dtsl", b"ec-3", b"mp4a"]) +def test_other_dts_profiles_stay_in_matroska(tmp_path: Path, fourcc: bytes) -> None: + """ + DTS:X Profile 1 is DTS-HD Master Audio underneath and Matroska stores it as + A_DTS, so it must not be pulled out of Matroska by the DTS:X container rule. + """ + assert has_dts_uhd_sample_entry(write_mp4(tmp_path / "a.mp4", fourcc)) is False + + +def test_a_file_with_no_moov_is_not_detected(tmp_path: Path) -> None: + path = tmp_path / "a.mp4" + path.write_bytes(box(b"ftyp", b"isom") + box(b"mdat", b"dtsx" * 8)) + assert has_dts_uhd_sample_entry(path) is False, "a 4CC in mdat is not a sample entry" + + +def build_tracks(tmp_path: Path, audio_entry: bytes) -> Tracks: + video = Video(id_="v1", url="https://x/v", codec=Video.Codec.AVC, language="en", width=1920, height=1080, fps=24) + audio = Audio(id_="a1", url="https://x/a", codec=Audio.Codec.DTSX, language="en", bitrate=448000) + video.path = write_mp4(tmp_path / "v.mp4", b"avc1") + audio.path = write_mp4(tmp_path / "a.mp4", audio_entry) + return Tracks([video, audio]) + + +class FakePopen: + """Stands in for MP4Box, recording the command it was given.""" + + stdout_text = "" + + def __init__(self, args, **kwargs): + self.args = args + self.kwargs = kwargs + self.stdout = io.StringIO(FakePopen.stdout_text or "ISO File Writing: |== | (50/100)\n") + + def wait(self) -> int: + return 0 + + +def test_dts_uhd_audio_routes_the_mux_to_mp4(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured: list[FakePopen] = [] + + def fake_popen(args, **kwargs): + p = FakePopen(args, **kwargs) + captured.append(p) + Path(args[-1]).write_bytes(b"muxed") + return p + + monkeypatch.setattr(binaries, "MP4Box", Path("MP4Box"), raising=False) + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + tracks = build_tracks(tmp_path, b"dtsx") + out, returncode, errors = tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + + assert out.suffix == ".mp4", "a DTS-UHD title must not be delivered as Matroska" + assert returncode == 0 and not errors + command = captured[0].args + assert command[0] == "MP4Box" + assert command[-2:] == ["-new", str(out)] + assert f"{tmp_path / 'a.mp4'}:lang=en" in command + # the child must not inherit the terminal that the live progress tree is drawing on + assert captured[0].kwargs["stdout"] is subprocess.PIPE + assert captured[0].kwargs["stderr"] is subprocess.STDOUT + + +def test_other_audio_still_muxes_with_mkvmerge(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(binaries, "MP4Box", None, raising=False) + monkeypatch.setattr("unshackle.core.tracks.tracks.binaries.MKVToolNix", None) + + tracks = build_tracks(tmp_path, b"dtsh") + # reaching the mkvmerge requirement proves the DTS-UHD branch was not taken; MP4Box + # is None here too, so the MP4 branch would have raised about MP4Box instead + with pytest.raises(RuntimeError, match="MKVToolNix"): + tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + + +def test_a_missing_mp4box_names_the_install(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(binaries, "MP4Box", None, raising=False) + tracks = build_tracks(tmp_path, b"dtsx") + with pytest.raises(RuntimeError, match="gpac"): + tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + + +def test_substation_alpha_subtitles_are_left_out(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """MP4 has no ASS format and tx3g would drop the styling that picked it.""" + captured: list[FakePopen] = [] + + def fake_popen(args, **kwargs): + p = FakePopen(args, **kwargs) + captured.append(p) + Path(args[-1]).write_bytes(b"muxed") + return p + + monkeypatch.setattr(binaries, "MP4Box", Path("MP4Box"), raising=False) + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + tracks = build_tracks(tmp_path, b"dtsx") + ass = Subtitle(id_="s1", url="https://x/s", codec=Subtitle.Codec.SubStationAlphav4, language="en") + ass.path = tmp_path / "s.ass" + ass.path.write_text("[Script Info]\n") + vtt = Subtitle(id_="s2", url="https://x/s2", codec=Subtitle.Codec.WebVTT, language="es") + vtt.path = tmp_path / "s.vtt" + vtt.path.write_text("WEBVTT\n") + tracks.add([ass, vtt]) + + tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + command = " ".join(captured[0].args) + assert "s.ass" not in command + assert "s.vtt:lang=es" in command + + +def test_mp4box_errors_come_back_to_the_caller(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed mux does not raise; the caller inspects the return code and lines.""" + + class FailingPopen(FakePopen): + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) + self.stdout = io.StringIO("Error importing track: not supported\n") + + def wait(self) -> int: + return 1 + + monkeypatch.setattr(binaries, "MP4Box", Path("MP4Box"), raising=False) + monkeypatch.setattr(subprocess, "Popen", FailingPopen) + + tracks = build_tracks(tmp_path, b"dtsx") + _, returncode, errors = tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + assert returncode == 1 + assert errors == ["Error importing track: not supported"] + + +def init_pair(sample_entry: bytes, stamp: bytes = b"\x00\x00\x00\x01") -> bytes: + """An ftyp+moov pair; the stamp stands in for the fetch time a packager writes.""" + stsd = box(b"stsd", box(sample_entry, b"\x00" * 20)) + return box(b"ftyp", b"isom") + box(b"moov", box(b"mvhd", stamp) + box(b"trak", stsd)) + + +def fragment(payload: bytes) -> bytes: + return box(b"moof", b"\x00" * 8) + box(b"mdat", payload) + + +def test_repeated_inits_are_dropped_when_the_stsd_matches(tmp_path: Path) -> None: + """ + An HLS discontinuity gives each period its own init, so a merged track holds several. + Only the stsd decides how samples decode, and a packager stamps each init with the + fetch time, so copies that differ only in that stamp are safe to drop. + """ + src = tmp_path / "a.mp4" + src.write_bytes( + init_pair(b"dtsx", b"\x00\x00\x00\x01") + + fragment(b"one") + + init_pair(b"dtsx", b"\x00\x00\x00\x99") + + fragment(b"two") + ) + dst = tmp_path / "clean.mp4" + + assert strip_duplicate_init_boxes(src, dst) == 2, "the second ftyp and moov should be dropped" + types = [t for t, _, _ in iter_top_level_boxes(dst)] + assert types == [b"ftyp", b"moov", b"moof", b"mdat", b"moof", b"mdat"] + assert dst.read_bytes().count(b"mdat") == 2, "no sample data may be lost" + + +def test_a_changed_stsd_is_never_dropped(tmp_path: Path) -> None: + """A later init describing a different format is a real change; dropping it corrupts.""" + src = tmp_path / "a.mp4" + src.write_bytes(init_pair(b"dtsx") + fragment(b"one") + init_pair(b"ec-3") + fragment(b"two")) + dst = tmp_path / "clean.mp4" + + assert strip_duplicate_init_boxes(src, dst) == 0 + assert not dst.exists(), "the caller must keep the original when the format changes" + + +def test_a_single_init_is_left_alone(tmp_path: Path) -> None: + src = tmp_path / "a.mp4" + src.write_bytes(init_pair(b"dtsx") + fragment(b"one")) + dst = tmp_path / "clean.mp4" + assert strip_duplicate_init_boxes(src, dst) == 0 + assert not dst.exists() + + +def test_an_unreadable_input_aborts_instead_of_shipping_a_placeholder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + MP4Box answers a file its ISOM reader refused by importing it through ffmpeg, which + writes the samples under a placeholder 4CC. mkvmerge-style return codes treat 1 as a + warning, so that shipped a whole title with an unnameable audio track. + """ + + class RefusingPopen(FakePopen): + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) + self.stdout = io.StringIO("[IsoMedia] error while opening /tmp/a.mp4, error=Invalid IsoMedia File\n") + + def wait(self) -> int: + return 0 + + monkeypatch.setattr(binaries, "MP4Box", Path("MP4Box"), raising=False) + monkeypatch.setattr(subprocess, "Popen", RefusingPopen) + + tracks = build_tracks(tmp_path, b"dtsx") + _, returncode, errors = tracks.mux("Title", delete=False, output_path=tmp_path / "out.muxed.mkv") + + assert returncode >= 2, "a refused input must abort the run, not pass as a warning" + assert errors diff --git a/tests/core/test_music_container.py b/tests/core/test_music_container.py index 6fc82ca6..b4fee39e 100644 --- a/tests/core/test_music_container.py +++ b/tests/core/test_music_container.py @@ -132,7 +132,8 @@ def test_dolby_digital_plus_falls_back_to_the_mp4_muxer(tmp_path: Path) -> None: def test_a_failed_remux_keeps_the_source_and_removes_its_partial_file(tmp_path: Path) -> None: """FFmpeg creates the output before it fails, and nothing else sweeps the temp directory.""" - source = encode(tmp_path / "track.mp4", "pcm_s16le", "-f", "mov") # no MP4 muxer takes PCM + # FFmpeg 9.0 writes PCM to MP4 as ipcm, so PCM no longer fails the remux. + source = encode(tmp_path / "track.mp4", "adpcm_ms", "-f", "mov") # no MP4 muxer takes ADPCM track = Audio(id_="a1", url="https://example.test/a1.mp4", language="en", codec=None) track.path = source diff --git a/tests/tracks/test_audio_codec_mime.py b/tests/tracks/test_audio_codec_mime.py new file mode 100644 index 00000000..f119ee42 --- /dev/null +++ b/tests/tracks/test_audio_codec_mime.py @@ -0,0 +1,117 @@ +import textwrap + +import pytest + +from unshackle.core.constants import SPACED_AUDIO_CODECS +from unshackle.core.manifests.hls import HLS +from unshackle.core.tracks.audio import Audio + +# The codecs one variant declares, in manifest order. One GROUP-ID holds renditions of +# all three, which is why a group-wide codec cannot be right. +DTS_X_GROUP = [ + ("dtsx", Audio.Codec.DTSX), + ("ec-3", Audio.Codec.EC3), + ("mp4a", Audio.Codec.AAC), +] + + +def test_dtsc_maps_to_dts() -> None: + assert Audio.Codec.from_mime("dtsc") == Audio.Codec.DTS + + +@pytest.mark.parametrize("mime", ["dtsx", "dtsy"]) +def test_dts_uhd_sample_entries_map_to_dtsx(mime: str) -> None: + """DTS-UHD is a separate bitstream, so it must not share the DTS name.""" + assert Audio.Codec.from_mime(mime) == Audio.Codec.DTSX + assert Audio.Codec.from_codecs(f"avc1.64001f,{mime},ec-3,mp4a.40.2") == Audio.Codec.DTSX + assert Audio.Codec.DTSX.value == "DTS-X", "':' is not a legal path character on Windows" + + +def test_unknown_mime_still_raises() -> None: + with pytest.raises(ValueError): + Audio.Codec.from_mime("dtsz") + + +@pytest.mark.parametrize( + ("uri", "expected"), + [ + # Rendition URIs that name the codec in one "_" delimited field. + ("r/composite_448k_dtsx_en_PRIMARY_de37f606-8b62-4116-bdec-0de711f2b7d0_x.m3u8", Audio.Codec.DTSX), + ("r/composite_256k_ec-3_cs_PRIMARY_b4556be9-7eb3-47a9-b103-d97263a99ec1_x.m3u8", Audio.Codec.EC3), + ("r/composite_128k_mp4a.40.2_en_NARRATION_3891a283-94c0-4b89-9dc8-a70fcec25661_x.m3u8", Audio.Codec.AAC), + # A UUID can spell "ec-3", so a substring match would call this rendition EC3. + ("r/composite_128k_mp4a.40.2_en_PRIMARY_b4556be9-7eb3-47a9-b103-d97263a9ec-3_x.m3u8", Audio.Codec.AAC), + # Packagers that name the codec as a directory, hyphen or not. + ("https://cdn/x/dtsx-448k/index.m3u8", Audio.Codec.DTSX), + ("https://cdn/x/dtsx/index.m3u8", Audio.Codec.DTSX), + # Nothing names a codec, so the first codec the variant declares wins. + ("r/audio-en.m3u8", Audio.Codec.DTSX), + ], +) +def test_select_audio_codec_matches_delimited_codecs(uri: str, expected: Audio.Codec) -> None: + assert HLS.select_audio_codec(DTS_X_GROUP, uri) == expected + + +def test_select_audio_codec_will_not_read_one_codec_out_of_another() -> None: + """ "ac-3" sits inside the "eac-3" that names an E-AC-3 rendition.""" + candidates = [("ec-3", Audio.Codec.EC3), ("ac-3", Audio.Codec.AC3)] + assert HLS.select_audio_codec(candidates, "https://cdn/x/audio_ac-3_192k.m3u8") == Audio.Codec.AC3 + # No codec matches, so this falls back to the first declared rather than to AC3. + assert HLS.select_audio_codec(candidates, "https://cdn/x/audio_eac-3_768k.m3u8") == Audio.Codec.EC3 + + +def test_select_audio_codec_ignores_the_query_string() -> None: + assert HLS.select_audio_codec(DTS_X_GROUP, "r/composite_448k_dtsx_en.m3u8?fallback=mp4a") == Audio.Codec.DTSX + + +def test_select_audio_codec_holds_a_single_declaration() -> None: + """One declared codec needs no URI, so a group of one is never guessed at.""" + assert HLS.select_audio_codec([("mp4a", Audio.Codec.AAC)], "r/anything_dtsx.m3u8") == Audio.Codec.AAC + assert HLS.select_audio_codec([], "r/x.m3u8") is None + assert HLS.select_audio_codec(None, None) is None + + +def test_spaced_codec_constants_name_real_codecs() -> None: + """The constant holds bare strings, so a rename must not silently stop matching.""" + for value in SPACED_AUDIO_CODECS: + assert value in {c.value for c in Audio.Codec}, f"{value} is not an Audio.Codec value" + + +def test_a_short_ladder_rung_cannot_hide_a_group_codec() -> None: + """ + One GROUP-ID is reached from every variant that references it, and a low rung may + declare fewer codecs than the group holds. The union has to survive the short rung. + """ + manifest = textwrap.dedent("""\ + #EXTM3U + #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="A",NAME="English",LANGUAGE="en",CHANNELS="10",URI="composite_448k_dtsx_en.m3u8" + #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="A",NAME="Czech",LANGUAGE="cs",CHANNELS="6",URI="composite_256k_ec-3_cs.m3u8" + #EXT-X-STREAM-INF:BANDWIDTH=9000000,CODECS="hvc1.2.4.L150.90,dtsx,ec-3",AUDIO="A" + v_high.m3u8 + #EXT-X-STREAM-INF:BANDWIDTH=400000,CODECS="avc1.64001f,ec-3",AUDIO="A" + v_low.m3u8 + """) + tracks = HLS.from_text(manifest, "https://example.test/master.m3u8").to_tracks("en") + by_lang = {str(t.language): t.codec for t in tracks.audio} + assert by_lang["en"] == Audio.Codec.DTSX, "the low rung dropped dtsx from the group" + assert by_lang["cs"] == Audio.Codec.EC3 + + +def test_the_group_fallback_does_not_depend_on_variant_order() -> None: + """ + A rendition URI that names no codec falls back to a codec the group declares. Which + one must not depend on where the packager put its variants: ascending by BANDWIDTH is + as common as descending, and picking the lossy label makes the premium rendition + unreachable through --acodec. + """ + variants = [ + '#EXT-X-STREAM-INF:BANDWIDTH=9000000,CODECS="hvc1.2.4.L150.90,dtsx,ec-3",AUDIO="A"\nv_high.m3u8', + '#EXT-X-STREAM-INF:BANDWIDTH=400000,CODECS="avc1.64001f,ec-3",AUDIO="A"\nv_low.m3u8', + ] + header = ( + '#EXTM3U\n#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="A",NAME="English",LANGUAGE="en",CHANNELS="10",URI="audio_en.m3u8"\n' + ) + for order in (variants, list(reversed(variants))): + manifest = header + "\n".join(order) + "\n" + tracks = HLS.from_text(manifest, "https://example.test/master.m3u8").to_tracks("en") + assert [t.codec for t in tracks.audio] == [Audio.Codec.DTSX], f"variant order decided the codec: {order[0]!r}" diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index ed64f81c..31830dc9 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -1370,7 +1370,23 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques VALID_VCODECS = ["H264", "H265", "H.264", "H.265", "AVC", "HEVC", "VC1", "VC-1", "VP8", "VP9", "AV1"] -VALID_ACODECS = ["AAC", "AC3", "EC3", "EAC3", "DD", "DD+", "AC4", "OPUS", "FLAC", "ALAC", "VORBIS", "OGG", "DTS"] +VALID_ACODECS = [ + "AAC", + "AC3", + "EC3", + "EAC3", + "DD", + "DD+", + "AC4", + "OPUS", + "FLAC", + "ALAC", + "VORBIS", + "OGG", + "DTS", + "DTSX", + "DTS-X", +] def check_codec(value: Any, allowed: List[str], name: str) -> Optional[str]: diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 11d6cb7c..107bed8c 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -673,7 +673,7 @@ async def download(request: web.Request) -> web.Response: - type: array items: type: string - description: Audio codec(s) to download (e.g., "AAC" or ["AAC", "EC3"]) - accepts AAC, AC3, EC3, AC4, OPUS, FLAC, ALAC, DTS, OGG (default - None) + description: Audio codec(s) to download (e.g., "AAC" or ["AAC", "EC3"]) - accepts AAC, AC3, EC3, AC4, OPUS, FLAC, ALAC, DTS, DTSX, DTS-X, OGG (default - None) vbitrate: type: integer description: Video bitrate in kbps (default - None) diff --git a/unshackle/core/constants.py b/unshackle/core/constants.py index 65c66811..0c3a08fe 100644 --- a/unshackle/core/constants.py +++ b/unshackle/core/constants.py @@ -15,7 +15,9 @@ DYNAMIC_RANGE_MAP = { "HDR10 / HDR10+": "HDR10P", "HDR10 / HDR10": "HDR", } -AUDIO_CODEC_MAP = {"E-AC-3": "DDP", "AC-3": "DD"} +AUDIO_CODEC_MAP = {"E-AC-3": "DDP", "AC-3": "DD", "DTS-UHD": "DTS-X"} + +SPACED_AUDIO_CODECS = {"DTS-X"} context_settings = dict( help_option_names=["-?", "-h", "--help"], # default only has --help diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 0e2f42a2..7dc701a5 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -12,7 +12,7 @@ import sys import time from pathlib import Path from typing import Any, Optional, Union -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse from uuid import UUID from zlib import crc32 @@ -140,6 +140,42 @@ class HLS: break return out + @staticmethod + def select_audio_codec( + candidates: Optional[list[tuple[str, Audio.Codec]]], uri: Optional[str] + ) -> Optional[Audio.Codec]: + """ + Choose the Audio Codec of one rendition from the codecs its variant declares. + + EXT-X-STREAM-INF lists every codec of the variant, so a GROUP-ID holding + renditions of more than one codec gives no per-rendition answer. Packagers that do + this name the codec in the rendition URI, so match on that first. + + The codec must sit between delimiters: a plain substring test reads "ec-3" out of + a hex UUID, and "ac-3" out of the "eac-3" that names an E-AC-3 rendition. Longer + codecs match first so one cannot take the place of another it contains, and the + match reads the path only, so a query string cannot decide a codec. + + A rendition that names no codec falls back to the first codec its group declares. + The caller collects those highest-bandwidth variant first, so the fallback is the + top rung's codec, which keeps a premium rendition reachable rather than hiding it + behind a lossy label. + """ + if not candidates: + return None + if len(candidates) > 1: + path = urlparse(uri or "").path.lower() + for mime, codec in sorted(candidates, key=lambda c: len(c[0]), reverse=True): + if re.search(rf"(? Tracks: """ Convert a Variant Playlist M3U(8) document to Video, Audio and Subtitle Track objects. @@ -156,7 +192,23 @@ class HLS: session_drm = HLS.get_all_drm(session_keys) - audio_codecs_by_group_id: dict[str, Audio.Codec] = {} + audio_codecs_by_group_id: dict[str, list[tuple[str, Audio.Codec]]] = {} + for playlist in sorted(self.manifest.playlists, key=lambda p: p.stream_info.bandwidth or 0, reverse=True): + audio_group = playlist.stream_info.audio + if not (audio_group and playlist.stream_info.codecs): + continue + known = audio_codecs_by_group_id.setdefault(audio_group, []) + seen = {mime for mime, _ in known} + for token in playlist.stream_info.codecs.split(","): + mime = token.strip().split(".")[0].lower() + if mime in seen: + continue + try: + known.append((mime, Audio.Codec.from_mime(mime))) + except ValueError: + continue + seen.add(mime) + cc_by_group_id: dict[str, list[dict[str, Any]]] = {} for media in self.manifest.media: if media.type == "CLOSED-CAPTIONS": @@ -174,16 +226,6 @@ class HLS: dv_supp_prefixes = ("dva1", "dvav", "dvhe", "dvh1") for playlist in self.manifest.playlists: - audio_group = playlist.stream_info.audio - audio_codec: Optional[Audio.Codec] = None - if audio_group and playlist.stream_info.codecs: - try: - audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs) - except ValueError: - audio_codec = None - if audio_codec: - audio_codecs_by_group_id[audio_group] = audio_codec - try: # TODO: Any better way to figure out the primary track type? if playlist.stream_info.codecs: @@ -252,7 +294,7 @@ class HLS: joc = 0 if media.type == "AUDIO": track_type = Audio - codec = audio_codecs_by_group_id.get(media.group_id) + codec = HLS.select_audio_codec(audio_codecs_by_group_id.get(media.group_id), media.uri) if media.channels and media.channels.endswith("/JOC"): joc = int(media.channels.split("/JOC")[0]) media.channels = "5.1" @@ -1108,9 +1150,9 @@ class HLS: if len(segments_to_merge) == 1: shutil.move(segments_to_merge[0], save_path) else: - progress(downloaded="Merging", completed=0, total=None) + progress(downloaded="Merging", completed=0, total=len(segments_to_merge)) if isinstance(track, (Video, Audio)): - HLS.merge_segments(segments=segments_to_merge, save_path=save_path) + HLS.merge_segments(segments=segments_to_merge, save_path=save_path, progress=progress) else: with open(save_path, "wb") as f: for discontinuity_file in segments_to_merge: @@ -1136,17 +1178,22 @@ class HLS: events.emit(events.Types.TRACK_DOWNLOADED, track=track) @staticmethod - def merge_segments(segments: list[Path], save_path: Path) -> int: + def merge_segments(segments: list[Path], save_path: Path, progress: Optional[Any] = None) -> int: """ Concatenate Segments using FFmpeg concat with binary fallback. + FFmpeg concat reports nothing per segment, so progress only advances on the binary + fallback, which copies one segment at a time. + Returns the file size of the merged file. """ segment_dirs = set() for segment in segments: current_dir = segment.parent - while current_dir.name and "_segments" in str(current_dir): - segment_dirs.add(current_dir) + while current_dir.name: + if current_dir.name.endswith("_segments"): + segment_dirs.add(current_dir) + break current_dir = current_dir.parent def cleanup_segments_and_dirs(): @@ -1166,7 +1213,7 @@ class HLS: demuxer_file.write_text("\n".join([f"file '{segment.absolute()}'" for segment in segments])) concat_start = time.monotonic() - subprocess.check_call( + subprocess.run( [ binaries.FFMPEG, "-nostdin", @@ -1185,9 +1232,16 @@ class HLS: "copy", save_path, ], - timeout=300, # 5 minute timeout + check=True, + timeout=300, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", ) demuxer_file.unlink(missing_ok=True) + if progress: + progress(completed=len(segments)) cleanup_segments_and_dirs() log_tool_run( "ffmpeg concat segments", @@ -1200,7 +1254,13 @@ class HLS: return save_path.stat().st_size except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e: - logging.getLogger("HLS").debug(f"FFmpeg concat failed ({e}), falling back to binary concatenation") + stderr = getattr(e, "stderr", None) or "" + if isinstance(stderr, bytes): + stderr = stderr.decode("utf-8", "replace") + logging.getLogger("HLS").debug( + f"FFmpeg concat failed ({e}), falling back to binary concatenation" + + (f": {stderr.strip()}" if stderr.strip() else "") + ) demuxer_file.unlink(missing_ok=True) save_path.unlink(missing_ok=True) @@ -1209,6 +1269,8 @@ class HLS: for segment in segments: with open(segment, "rb") as segment_file: shutil.copyfileobj(segment_file, output_file, 1024 * 1024) + if progress: + progress(advance=1) cleanup_segments_and_dirs() return save_path.stat().st_size diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index c43dd5b6..1d3cec39 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -7,7 +7,12 @@ from langcodes import Language from pymediainfo import MediaInfo from unshackle.core.config import config -from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP +from unshackle.core.constants import ( + AUDIO_CODEC_MAP, + DYNAMIC_RANGE_MAP, + SPACED_AUDIO_CODECS, + VIDEO_CODEC_MAP, +) from unshackle.core.tracks import Tracks @@ -170,19 +175,30 @@ class Title: codec = primary_audio_track.format channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original + positions = channel_layout.split(" ") if channel_layout else [] + heights = sum(1 for position in positions if position.upper().startswith("T")) + lfe = sum(1 for position in positions if position.upper().startswith("LFE")) + if channel_layout: - channels = float(sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" "))) + channels = float(sum(0.1 if p.upper().startswith("LFE") else 1 for p in positions)) else: channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 channels = float(channel_count) has_atmos = any("JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks) + if heights: + channels_str = f"{len(positions) - heights - lfe}.{lfe}.{heights}" + else: + channels_str = f"{channels:.1f}" + audio = AUDIO_CODEC_MAP.get(codec, codec) + spacer = " " if audio in SPACED_AUDIO_CODECS else "" + context.update( { - "audio": AUDIO_CODEC_MAP.get(codec, codec), - "audio_channels": f"{channels:.1f}", - "audio_full": f"{AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}", + "audio": audio, + "audio_channels": channels_str, + "audio_full": f"{audio}{spacer}{channels_str}", "atmos": "Atmos" if has_atmos else "", } ) diff --git a/unshackle/core/tracks/audio.py b/unshackle/core/tracks/audio.py index 70f25975..bf02a9e8 100644 --- a/unshackle/core/tracks/audio.py +++ b/unshackle/core/tracks/audio.py @@ -17,7 +17,8 @@ class Audio(Track): AC4 = "AC-4" # https://wikipedia.org/wiki/Dolby_AC-4 OPUS = "OPUS" # https://wikipedia.org/wiki/Opus_(audio_format) OGG = "VORB" # https://wikipedia.org/wiki/Vorbis - DTS = "DTS" # https://en.wikipedia.org/wiki/DTS_(company)#DTS_Digital_Surround + DTS = "DTS" # https://en.wikipedia.org/wiki/DTS,_Inc.#DTS_Digital_Surround + DTSX = "DTS-X" # https://en.wikipedia.org/wiki/DTS,_Inc.#DTS:X ALAC = "ALAC" # https://en.wikipedia.org/wiki/Apple_Lossless_Audio_Codec FLAC = "FLAC" # https://en.wikipedia.org/wiki/FLAC @@ -38,6 +39,8 @@ class Audio(Track): return Audio.Codec.AC4 if mime == "opus": return Audio.Codec.OPUS + if mime in ("dtsx", "dtsy"): + return Audio.Codec.DTSX if mime == "dtsc": return Audio.Codec.DTS if mime == "alac": diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 604af6dc..ff7a9d01 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -98,6 +98,123 @@ def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]: return None +def iter_top_level_boxes(path: Path) -> Iterable[tuple[bytes, int, int]]: + """Yield (box_type, offset, size) for each top-level box, stopping at the first bad header.""" + file_size = path.stat().st_size + with path.open("rb") as f: + offset = 0 + while offset + 8 <= file_size: + f.seek(offset) + header = f.read(8) + if len(header) < 8: + return + size = int.from_bytes(header[:4], "big") + if size == 1: + large = f.read(8) + if len(large) < 8: + return + size = int.from_bytes(large, "big") + elif size == 0: + size = file_size - offset + if size < 8: + return + yield header[4:8], offset, size + offset += size + + +def find_child_box(buf: bytes, box_type: bytes) -> Optional[bytes]: + """Return the first box of the given type nested anywhere in buf, header included.""" + i = buf.find(box_type, 4) + while i != -1: + size = int.from_bytes(buf[i - 4 : i], "big") + if 8 <= size <= len(buf) - (i - 4): + return buf[i - 4 : i - 4 + size] + i = buf.find(box_type, i + 1) + return None + + +def strip_duplicate_init_boxes(src: Path, dst: Path) -> int: + """Write src to dst without its repeated ftyp/moov pairs, returning how many it dropped. + + An HLS track whose playlist carries a discontinuity gets a fresh EXT-X-MAP init for each + period, so the merged file holds one ftyp+moov per period. That plays, and mkvmerge takes + it, but MP4Box rejects the whole file with "Duplicate 'ftyp' detected!". + + A packager stamps each init with the fetch time, so copies that describe the same format + still differ byte for byte. Only the stsd decides how samples decode, so this compares only + the stsd: a later moov with a different stsd is a real format change, and dropping it would + corrupt the output. In that case, and when there is no duplicate at all, this writes nothing, + returns 0, and leaves the caller with the original file. + """ + boxes = list(iter_top_level_boxes(src)) + duplicates = [(t, o, s) for t, o, s in boxes if t in (b"ftyp", b"moov")] + if len(duplicates) <= 2: + return 0 + + with src.open("rb") as f: + stsds = set() + for box_type, offset, size in duplicates: + if box_type != b"moov": + continue + f.seek(offset) + stsd = find_child_box(f.read(size), b"stsd") + if stsd is None: + return 0 + stsds.add(stsd) + if len(stsds) > 1: + return 0 + + dropped = 0 + seen: set[bytes] = set() + with src.open("rb") as f, dst.open("wb") as out: + for box_type, offset, size in boxes: + if box_type in (b"ftyp", b"moov"): + if box_type in seen: + dropped += 1 + continue + seen.add(box_type) + f.seek(offset) + remaining = size + while remaining: + chunk = f.read(min(remaining, 8 * 1024 * 1024)) + if not chunk: + break + out.write(chunk) + remaining -= len(chunk) + return dropped + + +def has_dts_uhd_sample_entry(path: Path) -> bool: + """True if the MP4 holds a DTS-UHD (DTS:X Profile 2) audio sample entry. + + Matroska has no CodecID for DTS-UHD, so mkvmerge stores it as an A_QUICKTIME + passthrough and every reader reports the container fallback instead of the codec. A + file this returns True for has to go into MP4 to keep its codec readable. + + Only "dtsx"/"dtsy" mean DTS-UHD. DTS:X Profile 1 rides inside DTS-HD Master Audio as + "dtsc"/"dtse"/"dtsh"/"dtsl", which Matroska maps to A_DTS and stores properly, so it + must not match here. Reading the entry rather than trusting the manifest keeps a + mislabelled codec from moving a Profile 1 track out of Matroska. + + A 4CC scan scoped to the moov box, not a structural parse; the scoping avoids chance + byte collisions in mdat. Any read error -> False. + """ + try: + moov = read_top_level_box(path, b"moov") + except OSError: + return False + if not moov: + return False + for tok in (b"dtsx", b"dtsy"): + i = moov.find(tok, 4) + while i != -1: + size = int.from_bytes(moov[i - 4 : i], "big") + if 16 <= size <= len(moov) - (i - 4): + return True + i = moov.find(tok, i + 1) + return False + + def has_encrypted_sample_entry(path: Path) -> bool: """True if the MP4's moov still carries an encrypted sample entry (encv/enca). diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 5a767d1a..a48589b3 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import os +import re import subprocess import threading import time @@ -23,11 +24,14 @@ from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.audio import Audio from unshackle.core.tracks.chapters import Chapter, Chapters from unshackle.core.tracks.subtitle import Subtitle -from unshackle.core.tracks.track import Track +from unshackle.core.tracks.track import Track, has_dts_uhd_sample_entry, strip_duplicate_init_boxes from unshackle.core.tracks.video import Video from unshackle.core.utilities import is_close_match, log_event, matching_languages, sanitize_filename from unshackle.core.utils.collections import as_list, flatten +MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)") +MP4BOX_OPEN_FAILED = re.compile(r"error while opening|Error opening file|Invalid IsoMedia File", re.I) + class Tracks: """ @@ -485,6 +489,173 @@ class Tracks: selected.append(track) return selected + def _mux_mp4( + self, + title: str, + delete: bool = True, + progress: Optional[partial] = None, + skip_subtitles: bool = False, + output_path: Optional[Path] = None, + ) -> tuple[Path, int, list[str]]: + """ + Multiplex the Tracks into an MP4 Container with MP4Box. + + Only used for a codec Matroska cannot name, currently DTS-UHD, because MP4 carries + far less: it drops the forced, hearing-impaired and original flags, holds no + attachments at all, and leaves out a subtitle it has no format for rather than + mangling it. Losing those beats shipping audio no player can name. + + Returns the same (path, returncode, errors) shape as the Matroska mux. + """ + mp4box = getattr(binaries, "MP4Box", None) + if not mp4box: + raise RuntimeError( + "MP4Box (GPAC) is required to mux DTS-UHD (DTS:X Profile 2) audio but was not found. " + "Install it with your package manager (e.g. `apt install gpac`), then check it with " + "`unshackle env check`." + ) + + progress = progress or partial(lambda **kwargs: None) + + cl = [str(mp4box)] + names: list[str] = [] + cleaned: list[Path] = [] + track_id = 0 + + for track in [*self.videos, *self.audio, *([] if skip_subtitles else self.subtitles)]: + if not track.path or not track.path.exists(): + raise ValueError(f"{track.__class__.__name__} Track must be downloaded before muxing...") + if isinstance(track, Subtitle) and track.path.suffix.lower() in (".ass", ".ssa"): + log_event( + "mux_mp4_subtitle_skipped", + level="WARNING", + message=f"MP4 cannot store {track.path.suffix.lstrip('.').upper()} subtitles, leaving out {track.language}", + context={"track_id": track.id, "language": str(track.language), "suffix": track.path.suffix}, + ) + continue + events.emit(events.Types.TRACK_MULTIPLEX, track=track) + track_id += 1 + + source = track.path + candidate = config.directories.temp / f"mp4mux_{track.id}_{os.getpid()}_{threading.get_ident()}.mp4" + dropped = strip_duplicate_init_boxes(source, candidate) + if dropped: + cleaned.append(candidate) + source = candidate + log_event( + "mux_mp4_init_deduplicated", + level="DEBUG", + message=f"Dropped {dropped} repeated init box(es) from {track.path.name} for MP4Box", + context={"track_id": track.id, "dropped": dropped}, + ) + elif candidate.exists(): + candidate.unlink() + + cl.extend(["-add", f"{source}:lang={track.language}"]) + name = "" if isinstance(track, Video) else (track.get_track_name() or "") + if name: + names.extend(["-name", f"{track_id}={name}"]) + + if not track_id: + raise ValueError("No tracks provided, at least one track must be provided.") + + cl.extend(names) + + chapters_path = None + if self.chapters: + chapters_path = config.directories.temp / config.filenames.chapters.format( + title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}" + ) + self.chapters.dump(chapters_path, fallback_name=config.chapter_fallback_name) + cl.extend(["-chap", str(chapters_path)]) + + if self.attachments: + log_event( + "mux_mp4_attachments_skipped", + level="WARNING", + message=f"MP4 cannot store attachments, leaving out {len(self.attachments)} file(s)", + context={"count": len(self.attachments)}, + ) + + if output_path is None: + first = self.videos[0].path if self.videos else self.audio[0].path if self.audio else None + if first is None: + raise ValueError("No tracks provided, at least one track must be provided.") + output_path = first.with_suffix(".muxed.mp4") + else: + output_path = output_path.with_suffix(".mp4") + + full_command = [*cl, "-new", str(output_path)] + + log_event( + "mux_start", + level="INFO", + message=f"Muxing {len(self.videos)}V/{len(self.audio)}A/{len(self.subtitles)}S -> {output_path.name}", + context={ + "title": title, + "output_path": str(output_path), + "muxer": str(mp4box), + "command": full_command, + "container": "mp4", + "reason": "DTS-UHD audio has no Matroska CodecID", + }, + ) + + try: + errors = [] + mux_start_time = time.monotonic() + p = subprocess.Popen( + full_command, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + assert p.stdout is not None + for line in iter(p.stdout.readline, ""): + line = line.strip() + if not line: + continue + percent = MP4BOX_PROGRESS.search(line) + if percent: + progress(total=100, completed=int(percent.group(1))) + elif "error" in line.lower() or "not supported" in line.lower(): + errors.append(line) + + returncode = p.wait() + if returncode < 2 and any(MP4BOX_OPEN_FAILED.search(line) for line in errors): + returncode = 2 + mux_duration_ms = round((time.monotonic() - mux_start_time) * 1000, 1) + output_size = output_path.stat().st_size if output_path.exists() else 0 + + log_event( + "mux_failed" if returncode != 0 else "mux_complete", + level="ERROR" if returncode != 0 else "INFO", + message=( + f"MP4Box exited with code {returncode}" + if returncode != 0 + else f"Muxed {output_path.name} ({output_size} bytes) in {mux_duration_ms}ms" + ), + context={ + "output_path": str(output_path), + "output_size": output_size, + "duration_ms": mux_duration_ms, + "returncode": returncode, + "errors": errors, + }, + ) + + return output_path, returncode, errors + finally: + if chapters_path: + chapters_path.unlink() + for path in cleaned: + path.unlink(missing_ok=True) + if delete: + for track in self: + track.delete() + def mux( self, title: str, @@ -535,6 +706,15 @@ class Tracks: video.data["audio_language"] = lang_code video.data["audio_language_name"] = lang_name + if any(at.path and at.path.exists() and has_dts_uhd_sample_entry(at.path) for at in self.audio): + return self._mux_mp4( + title=title, + delete=delete, + progress=progress, + skip_subtitles=skip_subtitles, + output_path=output_path, + ) + if not binaries.MKVToolNix: raise RuntimeError("MKVToolNix (mkvmerge) is required for muxing but was not found") diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index ed0293df..622988e6 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -32,6 +32,9 @@ def apply_tags(path: Path, tags: dict[str, str]) -> None: if not binaries.Mkvpropedit: log.debug("mkvpropedit not found on PATH; skipping tags") return + if Path(path).suffix.lower() not in (".mkv", ".mka", ".mks", ".webm"): + log.debug("Not a Matroska file; skipping tags for %s", path) + return log.debug("Applying tags to %s: %s", path, tags) xml_lines = ['', "", " ", " "] for name, value in tags.items():