From d60d8abf53fd3422711239ecd8378f0c18d7e72c Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sat, 25 Jul 2026 15:55:31 -0600 Subject: [PATCH] fix(ism): repair synthesized init and verify fragments decrypted UNITY_MATRIX emitted 11 int32s instead of 9, oversizing mvhd/tkhd so pymp4 desynced and fTTML subtitles yielded no cues. Also rewrites PIFF senc to CENC so shaka can decrypt Smooth, labels HEV1 as hev1, and raises when a fragment survives decryption still carrying a sample-encryption box. --- tests/core/test_ism_init.py | 238 ++++++++++++++++++++++++ tests/tracks/test_fragment_decrypted.py | 155 +++++++++++++++ unshackle/commands/dl.py | 4 +- unshackle/core/manifests/dash.py | 2 + unshackle/core/manifests/hls.py | 2 + unshackle/core/manifests/ism.py | 7 + unshackle/core/manifests/ism_init.py | 108 +++++++++-- unshackle/core/tracks/track.py | 105 +++++++++++ 8 files changed, 609 insertions(+), 12 deletions(-) create mode 100644 tests/tracks/test_fragment_decrypted.py diff --git a/tests/core/test_ism_init.py b/tests/core/test_ism_init.py index a25e3193..0b9378eb 100644 --- a/tests/core/test_ism_init.py +++ b/tests/core/test_ism_init.py @@ -24,6 +24,7 @@ from unshackle.core.manifests.ism_init import ( parse_codec_private_data_colour, parse_codec_private_data_vui, parse_hevc_sps_format, + piff_senc_to_cenc, read_per_sample_iv_size, read_track_id, remove_emulation_prevention, @@ -578,3 +579,240 @@ def test_hvcc_profile_tier_level_is_nonzero(): level_idc = payload[12] assert profile_idc != 0 assert level_idc != 0 + + +def _piff_fragment(flags: int = 0x0, override: bool = False) -> bytes: + """Minimal moof+mdat fragment carrying sample encryption in the PIFF uuid box.""" + ivs = b"".join(bytes([i]) * 8 for i in range(1, 3)) + senc_payload = struct.pack(">I", 2) + ivs + if override: + flags |= 0x1 + senc_payload = b"\x00\x00\x01" + bytes([8]) + KID + senc_payload # AlgorithmID + IV_size + KID + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + flags.to_bytes(3, "big") + senc_payload) + tfhd = full_box(b"tfhd", 0, 0x20000, struct.pack(">I", 1)) + trun = full_box(b"trun", 0, 0x1, struct.pack(">II", 2, 0)) # data_offset placeholder + traf = box(b"traf", tfhd + trun + uuid_box) + moof = box(b"moof", full_box(b"mfhd", 0, 0, struct.pack(">I", 1)) + traf) + # patch trun data_offset to the real moof-relative start of the mdat payload + moof = moof.replace(struct.pack(">II", 2, 0), struct.pack(">II", 2, len(moof) + 8), 1) + return moof + box(b"mdat", b"\xab" * 64) + + +def test_piff_senc_rewritten_to_cenc_senc(): + frag = _piff_fragment() + out = piff_senc_to_cenc(frag) + # trun's data_offset is moof-relative; shrinking the box misaims decryption + assert len(out) == len(frag) + assert PIFF_SENC_UUID not in out + assert b"senc" in out and b"free" in out + assert struct.pack(">I", 2) + b"".join(bytes([i]) * 8 for i in range(1, 3)) in out + # moof size unchanged => the recorded data_offset still resolves + assert out[:8] == frag[:8] + assert out[len(out) - 72 :] == frag[len(frag) - 72 :] # mdat untouched + + +def test_piff_senc_override_header_stripped(): + frag = _piff_fragment(override=True) + out = piff_senc_to_cenc(frag) + assert len(out) == len(frag) + assert PIFF_SENC_UUID not in out + senc = out.index(b"senc") + assert out[senc + 4] == 0 # version + assert int.from_bytes(out[senc + 5 : senc + 8], "big") & 0x1 == 0 # override flag cleared + assert out[senc + 8 : senc + 12] == struct.pack(">I", 2) # sample_count directly follows + + +def test_piff_subsample_flag_survives_override_strip(): + # losing 0x2 is silent: no subsample map, so clear NAL headers get encrypted too + frag = _piff_fragment(flags=0x2, override=True) + out = piff_senc_to_cenc(frag) + senc = out.index(b"senc") + flags = int.from_bytes(out[senc + 5 : senc + 8], "big") + assert flags & 0x2, "subsample-encryption flag must survive" + assert not flags & 0x1, "override flag must be cleared" + + +def test_piff_rewrite_leaves_non_piff_fragments_alone(): + plain = box(b"moof", box(b"traf", full_box(b"tfhd", 0, 0, struct.pack(">I", 1)))) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(plain) is plain + assert piff_senc_to_cenc(b"") == b"" + + +def test_piff_rewrite_skips_traf_that_already_has_senc(): + # a second senc would leave the decrypter to choose between them + ivs = b"".join(bytes([i]) * 8 for i in range(1, 3)) + senc = full_box(b"senc", 0, 0, struct.pack(">I", 2) + ivs) + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0).to_bytes(3, "big") + struct.pack(">I", 2) + ivs) + frag = box(b"moof", box(b"traf", senc + uuid_box)) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(frag) is frag + + +def test_piff_rewrite_covers_every_moof_and_traf(): + frag = _piff_fragment() + _piff_fragment() + out = piff_senc_to_cenc(frag) + assert len(out) == len(frag) + assert PIFF_SENC_UUID not in out + assert out.count(b"senc") == 2 + + +def test_piff_rewrite_refuses_non_ctr_algorithm_id(): + # AlgorithmID 2 is AES-CBC but the sample entry says 'cenc'; rewriting yields garbage + ivs = b"".join(bytes([i]) * 8 for i in range(1, 3)) + payload = (2).to_bytes(3, "big") + bytes([8]) + KID + struct.pack(">I", 2) + ivs + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload) + frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(frag) is frag + + +def traf_children_tile_exactly(frag: bytes) -> bool: + """Every traf child's declared size must chain exactly to the traf end.""" + moof_size = struct.unpack(">I", frag[:4])[0] + traf_start = 8 + while frag[traf_start + 4 : traf_start + 8] != b"traf": + traf_start += struct.unpack(">I", frag[traf_start : traf_start + 4])[0] + traf_end = traf_start + struct.unpack(">I", frag[traf_start : traf_start + 4])[0] + assert traf_end <= moof_size + offset = traf_start + 8 + while offset < traf_end: + size = struct.unpack(">I", frag[offset : offset + 4])[0] + if size < 8 or offset + size > traf_end: + return False + offset += size + return offset == traf_end + + +def test_piff_rewrite_keeps_traf_children_tiling_exactly(): + # a mis-sized senc still "decrypts", just off the wrong bytes, so assert the sizes too + assert traf_children_tile_exactly(_piff_fragment()) + for frag in (_piff_fragment(), _piff_fragment(flags=0x2), _piff_fragment(flags=0x2, override=True)): + assert traf_children_tile_exactly(piff_senc_to_cenc(frag)), "rewritten traf no longer tiles" + + +def test_piff_rewrite_refuses_override_iv_size_mismatch(): + # tenc is the only IV width a decrypter consults; a disagreeing override misparses silently + ivs = b"".join(bytes([i]) * 16 for i in range(1, 3)) + payload = (1).to_bytes(3, "big") + bytes([16]) + KID + struct.pack(">I", 2) + ivs + uuid_box = box(b"uuid", PIFF_SENC_UUID + bytes([0]) + (0x1).to_bytes(3, "big") + payload) + frag = box(b"moof", box(b"traf", uuid_box)) + box(b"mdat", b"\x00" * 16) + assert piff_senc_to_cenc(frag, iv_size=8) is frag, "8-byte tenc must reject a 16-byte override" + assert piff_senc_to_cenc(frag, iv_size=16) is not frag, "matching width must still rewrite" + + +def test_piff_rewrite_never_grows_output_on_malformed_boxes(): + # A declared size running past the buffer would extend the output over the mdat. + good = _piff_fragment() + truncations = [good[:n] for n in range(8, len(good), 7)] + oversized = bytearray(good) + struct.pack_into(">I", oversized, 0, len(good) + 4096) # moof claims more than exists + for frag in [*truncations, bytes(oversized), b"", b"\x00" * 12]: + out = piff_senc_to_cenc(frag) # must not raise + assert len(out) == len(frag), f"output grew: {len(frag)} -> {len(out)}" + + +def _text_fragment(seq: int, begin: str, end: str, text: str, sdi: int) -> bytes: + """A single-sample fTTML moof+mdat, mirroring what a Smooth text stream serves.""" + payload = ( + "" + "
" + f"

{text}

" + "
" + ).encode() + mfhd = full_box(b"mfhd", 0, 0, struct.pack(">I", seq)) + tfhd = full_box(b"tfhd", 0, 0x020002, struct.pack(">I", 1) + struct.pack(">I", sdi)) + tfdt = full_box(b"tfdt", 1, 0, struct.pack(">Q", 0)) + trun = full_box(b"trun", 0, 0x000201, struct.pack(">I", 1) + struct.pack(">i", 0) + struct.pack(">I", len(payload))) + return box(b"moof", mfhd + box(b"traf", tfhd + tfdt + trun)) + box(b"mdat", payload) + + +def test_unity_matrix_is_nine_int32s(): + # extras shift every later field (mvhd next_track_ID, tkhd width/height) and desync pymp4 + from unshackle.core.manifests.ism_init import UNITY_MATRIX + + assert len(UNITY_MATRIX) == 36 + + +@pytest.mark.parametrize("stream_type", ["video", "audio", "text"]) +def test_mvhd_tkhd_sizes_match_iso_layout(stream_type): + # a declared size over the spec payload makes parsers under-read and stop at the moov + kwargs = { + "video": dict(fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080), + "audio": dict(fourcc="AACL", codec_private_data=AAC_LC_CPD), + "text": dict(fourcc="TTML", codec_private_data=""), + }[stream_type] + init = build_init_segment(stream_type=stream_type, duration=6_000_000_000, language="eng", **kwargs) + + sizes = {} + for name in (b"mvhd", b"tkhd"): + at = init.index(name) - 4 + sizes[name] = struct.unpack(">I", init[at : at + 4])[0] + # version-1 mvhd: 8 header + 4 version/flags + 28 time fields + 80 trailer + assert sizes[b"mvhd"] == 120 + # version-1 tkhd: 8 header + 4 version/flags + 32 time fields + 60 trailer + assert sizes[b"tkhd"] == 104 + + +def test_tkhd_carries_video_dimensions(): + # width/height sit after the matrix, so a mis-sized matrix silently zeroes them + init = build_init_segment( + stream_type="video", fourcc="H264", codec_private_data=VIDEO_AVC_CPD, width=1920, height=1080 + ) + at = init.index(b"tkhd") - 4 + tkhd = init[at : at + struct.unpack(">I", init[at : at + 4])[0]] + width, height = struct.unpack(">II", tkhd[-8:]) + assert (width >> 16, height >> 16) == (1920, 1080) + + +def test_pymp4_consumes_whole_init_and_reaches_fragments(): + # pymp4 must reach the mdat here, or Subtitle.parse finds nothing to yield cues from + from io import BytesIO + + from pymp4.parser import MP4 + + init = build_init_segment( + stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng" + ) + data = init + _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1) + types = [b.type for b in MP4.parse_stream(BytesIO(data))] + assert types == [b"ftyp", b"moov", b"moof", b"mdat"] + + +def test_fttml_subtitle_parse_yields_cues(): + from unshackle.core.tracks.subtitle import Subtitle + + init = build_init_segment( + stream_type="text", fourcc="TTML", codec_private_data="", duration=6_000_000_000, language="eng" + ) + data = ( + init + + _text_fragment(1, "00:00:01.000", "00:00:02.000", "first", sdi=1) + + _text_fragment(2, "00:00:03.000", "00:00:04.000", "second", sdi=2) + ) + caption_set = Subtitle.parse(data, Subtitle.Codec.fTTML) + cues = [c for lang in caption_set.get_languages() for c in caption_set.get_captions(lang)] + assert len(cues) == 2 + assert [c.get_text() for c in cues] == ["first", "second"] + + +@pytest.mark.parametrize( + "fourcc,expected", + [("HEV1", b"hev1"), ("hev1", b"hev1"), ("HVC1", b"hvc1"), ("hvc1", b"hvc1"), ("HEVC", b"hvc1"), ("H265", b"hvc1")], +) +def test_hevc_sample_entry_honours_manifest_fourcc(fourcc, expected): + # hev1 permits in-band parameter sets where hvc1 requires them in the sample + # entry. Real Smooth manifests ship FourCC="hev1", so relabelling it hvc1 + # misdescribes the stream. + init = build_init_segment( + stream_type="video", fourcc=fourcc, codec_private_data=VIDEO_HEVC_CPD, width=3840, height=2160 + ) + offset = find_stsd(init) + assert init[offset + 20 : offset + 24] == expected + + +def test_encrypted_hev1_frma_carries_the_same_fourcc(): + # frma names the original format the sinf wraps; if it drifts from the sample + # entry, a decrypter restores the wrong sample entry type. + init = build_init_segment( + stream_type="video", fourcc="HEV1", codec_private_data=VIDEO_HEVC_CPD, kid=KID, width=3840, height=2160 + ) + assert b"encv" in init + assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"hev1" diff --git a/tests/tracks/test_fragment_decrypted.py b/tests/tracks/test_fragment_decrypted.py new file mode 100644 index 00000000..f92cd8ff --- /dev/null +++ b/tests/tracks/test_fragment_decrypted.py @@ -0,0 +1,155 @@ +"""Self-check for ``assert_fragments_decrypted``: after decryption every moof must have lost +its sample-encryption box. A survivor means the tool silently skipped that fragment (the +tfhd.sample_description_index ceiling bug), which the moov-scoped has_encrypted_sample_entry +structurally cannot see. The guard never reads mdat, so stray byte runs there leave it unmoved. + +Run: uv run pytest tests/tracks/test_fragment_decrypted.py +""" + +from __future__ import annotations + +import struct +from pathlib import Path + +import pytest + +from unshackle.core.manifests.ism_init import PIFF_SENC_UUID +from unshackle.core.tracks.track import assert_fragments_decrypted + + +def _box(fmt: bytes, payload: bytes = b"") -> bytes: + return struct.pack(">I", 8 + len(payload)) + fmt + payload + + +def _senc(sample_count: int = 4) -> bytes: + return _box(b"senc", b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)) + + +def _piff_senc(sample_count: int = 4) -> bytes: + body = PIFF_SENC_UUID + b"\x00" * 4 + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count) + return _box(b"uuid", body) + + +def _moof(*children: bytes, sdi: int = 1) -> bytes: + tfhd = _box(b"tfhd", b"\x00\x00\x00\x02" + struct.pack(">II", 1, sdi)) # sample-description-index-present + return _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _box(b"traf", tfhd + b"".join(children))) + + +FTYP = _box(b"ftyp", b"isom\x00\x00\x02\x00isomiso2") +MOOV = _box(b"moov", _box(b"mvex", _box(b"trex", b"\x00" * 24))) +MDAT = _box(b"mdat", b"\x00" * 64) +# a compressed payload can contain anything, including plausible-looking box runs +STRAY = _box(b"mdat", struct.pack(">I", 64) + b"senc" + b"\xbe\xef" * 8 + PIFF_SENC_UUID) + + +def _write(tmp_path: Path, name: str, *boxes: bytes) -> Path: + f = tmp_path / name + f.write_bytes(b"".join(boxes)) + return f + + +def test_all_fragments_decrypted_passes(tmp_path: Path) -> None: + f = _write(tmp_path, "clean.mp4", FTYP, MOOV, _moof(), MDAT, _moof(sdi=2), MDAT) + assert_fragments_decrypted(f) + + +def test_surviving_piff_senc_raises(tmp_path: Path) -> None: + # the real bug: fragment 1 (index 1) decrypted, the rest (index 2) silently skipped + boxes = [FTYP, MOOV, _moof(), MDAT] + for _ in range(3): + boxes += [_moof(_piff_senc(), sdi=2), MDAT] + f = _write(tmp_path, "skipped.mp4", *boxes) + with pytest.raises(ValueError, match=r"3/4 fragment"): + assert_fragments_decrypted(f) + + +def test_surviving_cenc_senc_raises(tmp_path: Path) -> None: + f = _write(tmp_path, "cenc.mp4", FTYP, MOOV, _moof(_senc()), MDAT) + with pytest.raises(ValueError, match=r"1/1 fragment"): + assert_fragments_decrypted(f) + + +def test_stray_senc_bytes_in_mdat_ignored(tmp_path: Path) -> None: + f = _write(tmp_path, "stray.mp4", FTYP, MOOV, _moof(), STRAY, _moof(sdi=2), STRAY) + assert_fragments_decrypted(f) + + +def test_senc_text_without_plausible_size_ignored(tmp_path: Path) -> None: + # "senc" inside a free-box string has no valid box size before it + f = _write(tmp_path, "text.mp4", FTYP, MOOV, _moof(_box(b"free", b"presence sencoded here")), MDAT) + assert_fragments_decrypted(f) + + +def test_senc_bytes_inside_trun_payload_ignored(tmp_path: Path) -> None: + # a trun sample entry spelling "senc" after a size-looking duration; only a byte scan bites + trun = _box(b"trun", b"\x00\x00\x03\x01" + struct.pack(">I", 2) + struct.pack(">I", 512) + b"senc") + f = _write(tmp_path, "trun.mp4", FTYP, MOOV, _moof(trun), MDAT) + assert_fragments_decrypted(f) + + +def test_empty_senc_on_clear_fragment_ignored(tmp_path: Path) -> None: + # sample_count 0 marks a fragment with no protected samples, so the guard ignores it + f = _write(tmp_path, "empty.mp4", FTYP, MOOV, _moof(_senc(sample_count=0)), MDAT) + assert_fragments_decrypted(f) + + +def _piff_override(sample_count: int) -> bytes: + # flags&1 => AlgorithmID(3) + IV_size(1) + KID(16) precede sample_count + body = PIFF_SENC_UUID + b"\x00\x00\x00\x01" + b"\x00\x00\x08" + b"\x08" + b"\xaa" * 16 + return _box(b"uuid", body + struct.pack(">I", sample_count) + b"\x00" * (8 * sample_count)) + + +def test_piff_override_flag_sample_count_offset(tmp_path: Path) -> None: + clear = _write(tmp_path, "ovr_clear.mp4", FTYP, MOOV, _moof(_piff_override(0)), MDAT) + assert_fragments_decrypted(clear) + f = _write(tmp_path, "ovr.mp4", FTYP, MOOV, _moof(_piff_override(5)), MDAT) + with pytest.raises(ValueError, match=r"1/1 fragment"): + assert_fragments_decrypted(f) + + +def test_senc_outside_traf_ignored(tmp_path: Path) -> None: + # a decrypted moof whose only "senc" sits in a sibling box outside any traf + f = _write(tmp_path, "sibling.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + _senc()), MDAT) + assert_fragments_decrypted(f) + + +def test_senc_nested_under_non_traf_ignored(tmp_path: Path) -> None: + # senc two levels down under a non-traf container: only traf children count + meta = _box(b"meta", _senc()) + f = _write(tmp_path, "nested.mp4", FTYP, MOOV, _box(b"moof", _box(b"mfhd", b"\x00" * 8) + meta), MDAT) + assert_fragments_decrypted(f) + + +def test_undersized_largesize_box_does_not_desync_walk(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + # size==1 reads a 16-byte header, so a largesize of 12 clears the 8-byte floor but + # still stalls the cursor onto payload; only a header-aware floor catches it + runt = struct.pack(">I", 1) + b"free" + struct.pack(">Q", 12) + f = _write(tmp_path, "runt.mp4", FTYP, MOOV, runt, _moof(_piff_senc()), MDAT, _moof(_piff_senc()), MDAT) + with caplog.at_level("WARNING"): + assert_fragments_decrypted(f) + assert "malformed box size" in caplog.text + + +def test_largesize_and_zero_size_boxes_walked(tmp_path: Path) -> None: + frag = _moof(_piff_senc()) + large = struct.pack(">I", 1) + b"mdat" + struct.pack(">Q", 16 + 64) + b"\x00" * 64 + trailing = struct.pack(">I", 0) + b"mdat" + b"\x00" * 64 # size 0 => runs to EOF + f = _write(tmp_path, "large.mp4", FTYP, MOOV, frag, large, trailing) + with pytest.raises(ValueError, match=r"1/1 fragment"): + assert_fragments_decrypted(f) + + +def test_malformed_box_size_warns_and_does_not_raise(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + # failing open is deliberate, but the abort must be logged so it cannot pass as clean + overrun = struct.pack(">I", 1 << 30) + b"free" + b"\x00" * 8 + f = _write(tmp_path, "overrun.mp4", FTYP, MOOV, overrun, _moof(_piff_senc()), MDAT) + with caplog.at_level("WARNING"): + assert_fragments_decrypted(f) + assert "malformed box size" in caplog.text + + +def test_missing_or_bogus_file_is_noop(tmp_path: Path) -> None: + assert_fragments_decrypted(tmp_path / "nope.mp4") + assert_fragments_decrypted(_write(tmp_path, "bogus.bin", b"\x00\x01\x02\x03not-an-mp4")) + truncated = _write(tmp_path, "cut.mp4", FTYP, MOOV, _moof(_piff_senc())[:20]) + assert_fragments_decrypted(truncated) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 852f5815..68ba3c13 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -67,7 +67,7 @@ from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.dv_fixup import apply_dv_fixup from unshackle.core.tracks.hybrid import Hybrid -from unshackle.core.tracks.track import has_encrypted_sample_entry +from unshackle.core.tracks.track import assert_fragments_decrypted, has_encrypted_sample_entry from unshackle.core.utilities import ( find_font_with_fallbacks, find_missing_langs, @@ -3088,6 +3088,8 @@ class dl: if drm and hasattr(drm, "decrypt"): drm.decrypt(track.path) if not isinstance(drm, MonaLisa): + # MonaLisa decrypts per segment; its decrypt() here is a no-op + assert_fragments_decrypted(track.path) has_decrypted = True events.emit(events.Types.TRACK_REPACKED, track=track) else: diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 5f773760..f41302a3 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -28,6 +28,7 @@ from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine from unshackle.core.events import events from unshackle.core.session import RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video +from unshackle.core.tracks.track import assert_fragments_decrypted from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.xml import load_xml @@ -570,6 +571,7 @@ class DASH: if drm: progress(downloaded="Decrypting", completed=0, total=None) drm.decrypt(save_path) + assert_fragments_decrypted(save_path) track.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 8a61a26b..4ccde219 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -33,6 +33,7 @@ from unshackle.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine from unshackle.core.events import events from unshackle.core.session import RnetResponse, RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video +from unshackle.core.tracks.track import assert_fragments_decrypted from unshackle.core.utilities import get_extension, is_close_match, log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.subprocess import log_tool_run @@ -794,6 +795,7 @@ class HLS: # with widevine we can merge all segments and decrypt once merge(to=merged_path, via=files, delete=True, include_map_data=True) drm.decrypt(merged_path) + assert_fragments_decrypted(merged_path) merged_path.rename(decrypted_path) else: # with other drm we must decrypt separately and then merge them diff --git a/unshackle/core/manifests/ism.py b/unshackle/core/manifests/ism.py index 23854c12..65296012 100644 --- a/unshackle/core/manifests/ism.py +++ b/unshackle/core/manifests/ism.py @@ -22,11 +22,13 @@ from unshackle.core.events import events from unshackle.core.manifests.ism_init import ( build_init_segment, parse_codec_private_data_vui, + piff_senc_to_cenc, read_per_sample_iv_size, read_track_id, ) from unshackle.core.session import RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Track, Tracks, Video +from unshackle.core.tracks.track import assert_fragments_decrypted from unshackle.core.utilities import log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.xml import load_xml @@ -521,6 +523,7 @@ class ISM: init_segment = ISM._init_segment(track, session_drm, first_segment) if init_segment: f.write(init_segment) + iv_size = (read_per_sample_iv_size(first_segment) if session_drm and first_segment else None) or 8 for index, segment_file in enumerate(segments_to_merge): if is_text_subtitle: # first segment was already read for the init synthesis, reuse it @@ -533,6 +536,9 @@ class ISM: .encode("utf8") ) f.write(segment_data) + elif session_drm: + segment_data = first_segment if index == 0 and first_segment else segment_file.read_bytes() + f.write(piff_senc_to_cenc(segment_data, iv_size)) elif index == 0 and first_segment: f.write(first_segment) else: @@ -547,6 +553,7 @@ class ISM: if session_drm: progress(downloaded="Decrypting", completed=0, total=None) session_drm.decrypt(save_path) + assert_fragments_decrypted(save_path) track.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=session_drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) diff --git a/unshackle/core/manifests/ism_init.py b/unshackle/core/manifests/ism_init.py index b3aa96fb..02bb1ac9 100644 --- a/unshackle/core/manifests/ism_init.py +++ b/unshackle/core/manifests/ism_init.py @@ -8,6 +8,10 @@ sends a ``moov``. The init box must be reconstructed from the manifest's or decryptor such as shaka-packager can parse the stream. Ported from yt-dlp's ``write_piff_header`` and N_m3u8DL-RE's ``MSSMoovProcessor`` with HEVC, Dolby Vision, EC-3, TTML and CENC (PIFF) support. + +``piff_senc_to_cenc`` rewrites the fragments themselves, which neither port does: +shaka-packager cannot read the PIFF sample-encryption box, so its payload is +retagged as a standard ``senc``. """ from __future__ import annotations @@ -28,13 +32,14 @@ u1616 = struct.Struct(">Hxx") s32 = struct.Struct(">i") # 3x3 transformation matrix (identity), as stored in tkhd/mvhd. +# Exactly 9 int32s; extras shift every field after it and desync pymp4. UNITY_MATRIX = ( s32.pack(0x10000) - + s32.pack(0) * 3 + + s32.pack(0) * 2 + s32.pack(0) + s32.pack(0x10000) + s32.pack(0) * 2 - + s32.pack(0) * 2 + + s32.pack(0) + s32.pack(0x40000000) ) @@ -308,7 +313,13 @@ def parse_codec_private_data_colour(fourcc: str, codec_private_data: bytes) -> O def iter_boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, Optional[bytes], int, int]]: - """Yield (type, uuid_usertype, payload_start, box_end) for each child box.""" + """Yield (type, uuid_usertype, payload_start, box_end) for each child box. + + Stops silently at the first box whose declared size cannot be walked. A short + result therefore reports how far the walk got, and a caller counting survivors + should treat it as unverified. box_end may exceed end when the declared size + overruns; clamp it before slicing. + """ offset = start while offset + 8 <= end: size = struct.unpack(">I", data[offset : offset + 4])[0] @@ -319,7 +330,7 @@ def iter_boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, Optio header = 16 if size == 0: size = end - offset - if size < 8: # corrupt box header; stop rather than loop forever + if size < header: # size below the consumed header desyncs the walk; stop return usertype = None if box_type == b"uuid" and offset + header + 16 <= end: @@ -426,6 +437,74 @@ def read_per_sample_iv_size(fragment: bytes) -> Optional[int]: return None +def piff_senc_to_cenc(fragment: bytes, iv_size: int = 8) -> bytes: + """Rewrite a fragment's moof/traf PIFF sample-encryption uuid boxes as standard 'senc'. + + shaka-packager does not consume the PIFF uuid form. It finds no per-sample IVs, falls + through to the constant-IV path, and aborts with "IV cannot be empty". mp4decrypt + accepts either form, and the payloads are identical once the optional override header + is stripped, so emitting 'senc' serves both decrypters. + + The rewrite preserves length: bytes reclaimed from the uuid header become a 'free' box. + The moof's byte length must stay the same. trun's data_offset resolves against tfhd's + base_data_offset (the moof start only when that field is absent), so moving the sample + data misaligns decryption and yields plausible garbage with no error. + + iv_size is the default_Per_Sample_IV_Size the init segment's tenc declares, read from + the first fragment by read_per_sample_iv_size. This function skips any fragment whose + override disagrees, since tenc is the only IV width a CENC decrypter consults and a + mismatch misparses every IV. + """ + out: Optional[bytearray] = None + limit = len(fragment) + trafs = [ + (body, min(box_end, limit)) + for moof_type, _, moof_body, moof_end in iter_boxes(fragment, 0, limit) + if moof_type == b"moof" + for box_type, _, body, box_end in iter_boxes(fragment, moof_body, min(moof_end, limit)) + if box_type == b"traf" + ] + for traf_body, traf_end in trafs: + # a second senc would leave the decrypter to choose between them + if any(bt == b"senc" for bt, _, _, _ in iter_boxes(fragment, traf_body, traf_end)): + continue + offset = traf_body + while offset + 8 <= traf_end: + size = struct.unpack(">I", fragment[offset : offset + 4])[0] + if size < 8 or offset + size > traf_end: + break + if ( + size >= 28 + and fragment[offset + 4 : offset + 8] == b"uuid" + and fragment[offset + 8 : offset + 24] == PIFF_SENC_UUID + ): + version = fragment[offset + 24] + flags = int.from_bytes(fragment[offset + 25 : offset + 28], "big") + # flag 0x1 = PIFF override header: AlgorithmID(3) + IV_size(1) + KID(16) + strip = 20 if flags & 0x1 else 0 + # AlgorithmID 2 is AES-CBC and a stray IV width misparses every IV; leave + # either alone so the decrypter fails loudly on a form it cannot handle + if ( + strip + and ( + size < 28 + strip # short-circuits before reading the override fields + or int.from_bytes(fragment[offset + 28 : offset + 31], "big") != 1 + or fragment[offset + 31] != iv_size + ) + ): + offset += size + continue + if offset + 28 + strip <= offset + size: + senc = full_box(b"senc", version, flags & ~0x1, fragment[offset + 28 + strip : offset + size]) + pad = size - len(senc) + if pad >= 8: + if out is None: + out = bytearray(fragment) + out[offset : offset + size] = senc + box(b"free", bytes(pad - 8)) + offset += size + return bytes(out) if out is not None else fragment + + def build_avcc(codec_private_data: bytes, nal_length_size: int = 4) -> bytes: """Build an avcC (AVC decoder config) box from SPS+PPS CodecPrivateData.""" nals = split_nal_units(codec_private_data) @@ -582,7 +661,7 @@ def build_sinf( schm = full_box(b"schm", 0, 0, b"cenc" + u32.pack(0x00010000)) tenc_payload = ( u8.pack(0) # reserved - + u8.pack(0) # default_crypt_byte_block / skip_byte_block (cenc) + + u8.pack(0) # reserved in v0 (crypt/skip nibbles are v1 only) + u8.pack(1) # default_isProtected + u8.pack(0 if constant_iv else iv_size) # default_Per_Sample_IV_Size + kid # default_KID (16 bytes) @@ -616,8 +695,10 @@ def build_init_segment( Build a complete ftyp + moov initialization segment. stream_type: "video" | "audio" | "text". - fourcc: Smooth FourCC ("H264"/"AVC1"/"DAVC", "HVC1"/"HEV1", "DVHE"/"DVH1", - "AACL"/"AACH", "EC-3", "TTML"). + fourcc: Smooth FourCC ("H264"/"AVC1"/"DAVC", "HVC1"/"HEV1"/"HEVC"/"H265", + "DVHE"/"DVH1", "AACL"/"AACH"/"AAC", "EC-3", "TTML"/"STPP"/"DFXP"). + Only "HEV1" yields an hev1 sample entry; the rest of that group yield + hvc1, which requires its parameter sets in the sample entry. codec_private_data: hex string from the manifest QualityLevel. nal_length_size: manifest NALUnitLengthField (bytes per NAL length prefix). kid: 16-byte default key id; when set, the sample entry is wrapped for CENC. @@ -638,6 +719,7 @@ def build_init_segment( ftyp = box(b"ftyp", b"isml" + u32.pack(1) + b"iso5" + b"iso6" + b"piff" + b"msdh") # --- mvhd --- + # version 1: at the 10 MHz ISM timescale a 32-bit duration overflows after ~429s mvhd = full_box( b"mvhd", 1, @@ -724,7 +806,10 @@ def build_init_segment( codec_fourcc = b"avc1" elif fourcc in ("HVC1", "HEV1", "HEVC", "H265"): config_box = build_hvcc(cpd, nal_length_size) - codec_fourcc = b"hvc1" + # hev1 permits in-band parameter sets where hvc1 requires them in the + # sample entry, so relabelling hev1 as hvc1 misdescribes the stream + # (and the frma inside sinf). + codec_fourcc = b"hev1" if fourcc == "HEV1" else b"hvc1" elif fourcc in ("DVHE", "DVH1"): # Dolby Vision over HEVC: same hvcC config, dvh1 sample entry. config_box = build_hvcc(cpd, nal_length_size) @@ -773,9 +858,10 @@ def build_init_segment( else: raise NotImplementedError(f"Unsupported text FourCC: {fourcc}") - # Fragments may set tfhd sample_description_index=2 (server inits carry a second, - # identical entry). A lone entry leaves that dangling: mp4decrypt skips those - # fragments and demuxers stop after the first. + # Fragments may set tfhd sample_description_index=2. A lone entry leaves that + # dangling: mp4decrypt skips those fragments and demuxers stop after the first. + # We duplicate entry 1; a manifest carries one CodecPrivateData, so there is no + # second config to build. The server's larger second entry only repeats its VPS. stsd = full_box(b"stsd", 0, 0, u32.pack(2) + sample_entry_box * 2) # --- empty sample tables (fragmented: real samples live in moof/traf) --- diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 53a74a0c..a2eca307 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -100,6 +100,110 @@ def has_encrypted_sample_entry(path: Path) -> bool: return False +def _senc_protects_samples(buf: bytes, body: int, end: int) -> bool: + """True if a senc/PIFF-uuid box describes protected samples. + + An empty senc (sample_count 0) sits on a genuinely clear fragment, so treating it as + a skipped one would raise on a healthy file. Field order follows + read_per_sample_iv_size. + """ + flags = int.from_bytes(buf[body + 1 : body + 4], "big") + pos = body + 4 + (20 if flags & 0x1 else 0) # PIFF override: AlgorithmID(3)+IV_size(1)+KID(16) + if pos + 4 > end: + return False + return int.from_bytes(buf[pos : pos + 4], "big") > 0 + + +def _moof_still_encrypted(moof: bytes) -> bool: + # structural walk: a 4CC byte scan collides with trun payload + from unshackle.core.manifests.ism_init import PIFF_SENC_UUID, iter_boxes + + body = 16 if int.from_bytes(moof[:4], "big") == 1 else 8 + for box_type, _, traf_body, traf_end in iter_boxes(moof, body, len(moof)): + if box_type != b"traf": + continue + for child, usertype, child_body, child_end in iter_boxes(moof, traf_body, traf_end): + if child == b"senc" or (child == b"uuid" and usertype == PIFF_SENC_UUID): + if _senc_protects_samples(moof, child_body, child_end): + return True + return False + + +def assert_fragments_decrypted(path: Path) -> None: + """Raise if any fragment survived decryption still carrying a sample-encryption box. + + mp4decrypt strips senc from every fragment it processes and leaves it on the ones it + skips, so a survivor marks a silent skip. One cause is a tfhd.sample_description_index + dangling past the stsd entry count, which mp4decrypt reports as success (exit 0, no + stderr) while leaving the payload as ciphertext. Unlike has_encrypted_sample_entry + (moov-scoped) this is fragment-scoped; the moov comes out clean in that failure. + + The guarantee runs in one direction: a survivor proves a skip, while a clean pass is + only as good as the decrypter's own behaviour. shaka-packager (the default + `decryption` backend) remuxes into a single fragment and drops senc whether or not it + decrypted, so this cannot fire on shaka output. The walk is unbuffered and reads only + box headers; it seeks over mdat and never reads it. A malformed box size aborts the + walk with a warning and returns normally, so a file this function cannot parse is + never escalated into a raise. The logged warning is the only signal of that; a caller + sees the same silent return it gets from a verified clean file. + """ + surviving = 0 + total = 0 + first_offset = None + try: + file_size = path.stat().st_size + with path.open("rb", buffering=0) as f: + start = 0 + while start + 8 <= file_size: + f.seek(start) + header = f.read(8) + if len(header) < 8: + break + size = int.from_bytes(header[:4], "big") + box_header = 8 + if size == 1: # 64-bit largesize + large = f.read(8) + if len(large) < 8: + break + size = int.from_bytes(large, "big") + box_header = 16 + elif size == 0: # box runs to EOF + size = file_size - start + # below the consumed header the cursor stalls and desyncs onto payload + if size < box_header or start + size > file_size: + logging.getLogger("track").warning( + f"{path.name}: malformed box size at offset {start}; cannot verify the " + "remaining fragments were decrypted." + ) + break + if header[4:8] == b"moof": + total += 1 + f.seek(start) + if _moof_still_encrypted(f.read(size)): + surviving += 1 + if first_offset is None: + first_offset = start + start += size + except Exception as e: + logging.getLogger("track").warning(f"{path.name}: fragment decryption check did not complete ({e}).") + return + if surviving: + log_event( + "decrypt_fragments_still_encrypted", + level="ERROR", + message=f"{surviving}/{total} fragments still encrypted after decryption", + file=path.name, + surviving=surviving, + total=total, + first_offset=first_offset, + ) + raise ValueError( + f"{path.name}: {surviving}/{total} fragment(s) still encrypted after decryption (first at " + f"byte {first_offset}). The decrypt tool skipped them silently, so check " + f"tfhd.sample_description_index against the stsd entry count in the init segment." + ) + + @dataclass class DownloadContext: """Shared arguments passed to each manifest's ``download_track``.""" @@ -428,6 +532,7 @@ class Track: if drm: progress(downloaded="Decrypting", completed=0, total=None) drm.decrypt(save_path) + assert_fragments_decrypted(save_path) self.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=self, drm=drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100)