From b30e6d7cbea7737d454f88def8d4190bbaf5b189 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Fri, 18 Sep 2026 01:10:14 -0600 Subject: [PATCH] fix(vaults): verify vault content keys per KID from the fragment map Map each KID to its fragments from the ciphertext moov/moof (tenc, seig, ISM duration clock) and decode up to three windows per KID that never cross into another KID, so a clear lead cannot hide a wrong key and only the failing KID is flagged. Decode every frame (wrong-key HEVC keyframes decode clean), re-judge mid-GOP starts on keyframes, time out every FFmpeg run, and refuse to store a flagged pair again in SQLite. --- docs/guide/vaults.md | 4 +- docs/reference/configuration/vaults.md | 10 +- tests/core/test_ffmpeg_decodes.py | 143 +++++++++++ tests/core/test_kid_windows.py | 160 ++++++++++++ tests/core/test_vault_bad_keys.py | 160 +++++++++++- unshackle/commands/dl.py | 34 ++- unshackle/core/drm/verify.py | 321 ++++++++++++++++++++++++- unshackle/core/utils/subprocess.py | 114 ++++++--- unshackle/core/vaults.py | 5 +- unshackle/vaults/SQLite.py | 11 + 10 files changed, 921 insertions(+), 41 deletions(-) create mode 100644 tests/core/test_ffmpeg_decodes.py create mode 100644 tests/core/test_kid_windows.py diff --git a/docs/guide/vaults.md b/docs/guide/vaults.md index 52dcff6b..74789771 100644 --- a/docs/guide/vaults.md +++ b/docs/guide/vaults.md @@ -34,11 +34,11 @@ unshackle pushes keys from every source, not only a local CDM. A run against a r ### Wrong keys from a poisoned vault -A public vault can return the right KID with a wrong content key. The decrypters accept a wrong content key and write noise. unshackle therefore trusts a content key from a vault only after FFmpeg decodes the first seconds of the decrypted track with strict error detection. Content keys from a licence skip this check. A file this FFmpeg build cannot judge counts as a pass, never as a wrong content key: a codec it has no decoder for, or a header it cannot read. Without FFmpeg there is no check at all. +A public vault can return the right KID with a wrong content key. The decrypters accept a wrong content key and write noise. unshackle therefore trusts a content key from a vault only after FFmpeg decodes the decrypted track with strict error detection. For a fragmented MP4, unshackle reads from the encrypted file which fragments each KID encrypts, and FFmpeg decodes up to three windows of each KID: from its first, middle and last fragment on through the next fragments of that KID, for up to 10 seconds. A window never runs into a fragment of another KID or a clear fragment. A clear lead or a lead under another KID therefore cannot hide a wrong content key, and only the KID that fails is flagged. A KID that is not in this map, and every KID of any other file, gets one check of the whole file. FFmpeg decodes the first seconds and the last seconds. A failure there flags each of those KIDs. Content keys from a licence skip this check. A file this FFmpeg build cannot judge counts as a pass, never as a wrong content key: a codec it has no decoder for, or a header it cannot read. Without FFmpeg there is no check at all. When the decode fails, unshackle: -1. Flags the KID:KEY in the `bad_keys` table of every local SQLite vault, with the service and the name of the vault it came from. The pair also goes out of the service table, except in a vault marked `no_push`, where only the flag is written. +1. Flags the KID:KEY in the `bad_keys` table of every local SQLite vault, with the service and the name of the vault it came from. The pair also goes out of the service table, except in a vault marked `no_push`, where only the flag is written. A flagged pair cannot go back into a local vault, from a later run or from `kv copy`. An `API` vault that served the pair also gets a report; `HTTP` and `MySQL` vaults have no report route. A pair from a local vault goes to no remote vault, even when a remote vault holds the same pair. 2. Restores the ciphertext from a hard link it made before decryption, then asks for the content key again. Every vault lookup skips a flagged pair, so the request reaches the remaining vaults and then the CDM. A remote answer that arrived after the winning one is reused instead of queried again. 3. Removes the flag when the CDM returns the same content key, because then the decode check was wrong, not the content key. A content key from another vault is checked in its turn, and flagged as well if it also fails. diff --git a/docs/reference/configuration/vaults.md b/docs/reference/configuration/vaults.md index 1e32ddd0..d19cdfac 100644 --- a/docs/reference/configuration/vaults.md +++ b/docs/reference/configuration/vaults.md @@ -40,10 +40,12 @@ key_vaults: stores it. !!! note "Bad content keys" - A content key from a vault is proven by a short FFmpeg decode of the decrypted output. - A content key that fails the decode is written to the `bad_keys` table of every `SQLite` - vault with the name of the vault that supplied it, and every vault lookup skips a flagged - pair from then on. Only the `SQLite` backend stores flags. A `serve` instance with + A content key from a vault is proven by short FFmpeg decodes of the decrypted output, + with windows inside the fragments of each KID in a fragmented MP4. A content key that + fails the decode is written to the `bad_keys` table of every `SQLite` vault with the name + of the vault that supplied it, and every vault lookup skips a flagged pair from then on. A `SQLite` vault also refuses to + store a flagged pair again. Only the `SQLite` backend stores flags; an `API` vault that + supplied the pair gets a report. A `serve` instance with `server_cdm` flags a pair the same way when a remote client reports it, so a server whose `key_vaults` hold no `SQLite` entry cannot remember a bad content key. diff --git a/tests/core/test_ffmpeg_decodes.py b/tests/core/test_ffmpeg_decodes.py new file mode 100644 index 00000000..f39959f8 --- /dev/null +++ b/tests/core/test_ffmpeg_decodes.py @@ -0,0 +1,143 @@ +"""``ffmpeg_decodes`` must pass good media and fail a wrong content key, judged by a real FFmpeg. + +Each test builds its media from FFmpeg lavfi sources, so no media file is kept in the repo. +The encrypted cases use Shaka Packager to encrypt with a clear lead and mp4decrypt to +decrypt with a wrong key, which writes noise in place of the encrypted samples. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from uuid import UUID + +import pytest + +from unshackle.core import binaries +from unshackle.core.drm.verify import KidMap, kid_windows +from unshackle.core.utils.subprocess import ffmpeg_decodes + +pytestmark = pytest.mark.skipif(binaries.FFMPEG is None, reason="requires ffmpeg") + +KID = "00000000098ac6820000000000000000" +KEY = "13c795e9383efc0387c0bad6a10df73f" +WRONG_KEY = "00000000000000000000000000000000" +# 24 fps with a keyframe every 48 frames: one 2 s GOP for each 2 s fragment. +GOP = ["-x264-params", "keyint=48:min-keyint=48:scenecut=0"] + + +def ffmpeg(out: Path, source: str, *codec: str) -> Path: + subprocess.run( + [str(binaries.FFMPEG), "-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", source, *codec, str(out)], + check=True, + ) + return out + + +def video(out: Path, *x264: str, codec: str = "libx264") -> Path: + return ffmpeg( + out, + "testsrc2=size=320x180:rate=24:duration=12", + "-c:v", + codec, + "-preset", + "veryfast", + "-pix_fmt", + "yuv420p", + *x264, + ) + + +def test_audio_only_passes_fallback(tmp_path: Path) -> None: + # -skip_frame on a file with no video decoder makes FFmpeg refuse to open it. + audio = ffmpeg(tmp_path / "a.m4a", "sine=f=440:duration=12", "-c:a", "aac") + assert ffmpeg_decodes(audio) + assert ffmpeg_decodes(audio, video=False) + + +def test_mpegts_video_passes_fallback(tmp_path: Path) -> None: + ts = video(tmp_path / "v.ts", *GOP) + assert ffmpeg_decodes(ts) + assert ffmpeg_decodes(ts, video=True) + + +def test_open_gop_passes_windows(tmp_path: Path) -> None: + # The B-frames before each open-GOP keyframe refer to the GOP before the window. + og = video(tmp_path / "og.mp4", "-x264-params", "open-gop=1:keyint=48:min-keyint=48:scenecut=0:bframes=3") + assert all(ffmpeg_decodes(og, start=start, seconds=2, video=True) for start in (2.0, 4.0, 6.0, 8.0)) + + +def test_open_gop_hevc_passes_windows(tmp_path: Path) -> None: + og = video( + tmp_path / "og.mp4", + "-x265-params", + "open-gop=1:keyint=48:min-keyint=48:scenecut=0:log-level=none", + codec="libx265", + ) + assert all(ffmpeg_decodes(og, start=start, seconds=2, video=True) for start in (2.0, 4.0, 6.0, 8.0)) + + +# An HEVC keyframe decrypted with a wrong key can decode without an error, so the HEVC case +# proves that the check decodes the frames after it too. +@pytest.fixture(scope="module", params=["h264", "hevc"]) +def encrypted(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> tuple[KidMap, Path, Path]: + """The KID map of a clip encrypted after a clear lead, with its right-key and wrong-key decrypts.""" + if not binaries.ShakaPackager or not binaries.Mp4decrypt: + pytest.skip("requires shaka packager and mp4decrypt") + tmp = tmp_path_factory.mktemp("wrong_key") + if request.param == "hevc": + src = video( + tmp / "src.mp4", "-x265-params", "keyint=48:min-keyint=48:scenecut=0:log-level=none", codec="libx265" + ) + else: + src = video(tmp / "src.mp4", *GOP) + enc = tmp / "enc.mp4" + subprocess.run( + [ + str(binaries.ShakaPackager), + f"in={src},stream=video,output={enc}", + "--enable_raw_key_encryption", + "--keys", + f"label=:key_id={KID}:key={KEY}", + "--clear_lead", + "4", + "--fragment_duration", + "2", + "--segment_duration", + "2", + ], + check=True, + capture_output=True, + ) + right, wrong = tmp / "right.mp4", tmp / "wrong.mp4" + subprocess.run([str(binaries.Mp4decrypt), "--key", f"{KID}:{KEY}", str(enc), str(right)], check=True) + subprocess.run([str(binaries.Mp4decrypt), "--key", f"{KID}:{WRONG_KEY}", str(enc), str(wrong)], check=True) + kid_map = kid_windows(enc) + assert kid_map and list(kid_map.windows) == [UUID(KID)] + return kid_map, right, wrong + + +def windows_pass(kid_map: KidMap, path: Path) -> list[bool]: + return [ffmpeg_decodes(path, start=s, seconds=e - s, video=kid_map.video) for s, e in kid_map.windows[UUID(KID)]] + + +def test_right_key_passes_every_window(encrypted: tuple[KidMap, Path, Path]) -> None: + kid_map, right, _ = encrypted + assert all(windows_pass(kid_map, right)) + + +def test_wrong_key_fails_every_window(encrypted: tuple[KidMap, Path, Path]) -> None: + kid_map, _, wrong = encrypted + assert not any(windows_pass(kid_map, wrong)) + + +def test_wrong_key_passes_the_clear_lead(encrypted: tuple[KidMap, Path, Path]) -> None: + # The window ends where the encrypted fragment starts, and B-frames put that fragment's + # keyframe first in decode order: the decoder must not get it. + kid_map, _, wrong = encrypted + lead = kid_map.windows[UUID(KID)][0][0] + assert ffmpeg_decodes(wrong, start=0.0, seconds=lead, video=True) + + +def test_wrong_key_fails_fallback(encrypted: tuple[KidMap, Path, Path]) -> None: + assert not ffmpeg_decodes(encrypted[2]) diff --git a/tests/core/test_kid_windows.py b/tests/core/test_kid_windows.py new file mode 100644 index 00000000..389039ef --- /dev/null +++ b/tests/core/test_kid_windows.py @@ -0,0 +1,160 @@ +import struct +from pathlib import Path +from typing import Optional +from uuid import UUID + +from unshackle.core.drm.verify import PIFF_TENC, PIFF_TFXD, KidMap, kid_windows, pick_windows + +A, B, C = UUID(int=0xA), UUID(int=0xB), UUID(int=0xC) + + +def box(kind: bytes, *parts: bytes) -> bytes: + body = b"".join(parts) + return struct.pack(">I4s", 8 + len(body), kind) + body + + +def full(kind: bytes, payload: bytes, version: int = 0, flags: int = 0) -> bytes: + return box(kind, bytes([version]) + flags.to_bytes(3, "big") + payload) + + +def encv(kid: UUID, piff: bool = False) -> bytes: + tenc_body = b"\x00\x00\x01\x08" + kid.bytes + tenc = box(b"uuid", PIFF_TENC, bytes(4) + tenc_body) if piff else full(b"tenc", tenc_body) + return box(b"encv", bytes(78), box(b"sinf", box(b"frma", b"avc1"), box(b"schi", tenc))) + + +def enca(kid: UUID) -> bytes: + return box(b"enca", bytes(28), box(b"sinf", box(b"schi", full(b"tenc", b"\x00\x00\x01\x08" + kid.bytes)))) + + +def seig(protected: bool, kid: UUID) -> bytes: + return bytes([0, 0, int(protected), 8]) + kid.bytes + + +def sgpd(*entries: bytes, version: int = 1) -> bytes: + head = b"seig" + if version >= 1: + head += struct.pack(">I", 20) + if version >= 2: + head += struct.pack(">I", 0) + return full(b"sgpd", head + struct.pack(">I", len(entries)) + b"".join(entries), version=version) + + +def sbgp(*runs: tuple[int, int]) -> bytes: + return full(b"sbgp", b"seig" + struct.pack(">I", len(runs)) + b"".join(struct.pack(">II", *run) for run in runs)) + + +def movie(*sample_entries: bytes, handler: bytes = b"vide", stbl: bytes = b"") -> bytes: + stsd = full(b"stsd", struct.pack(">I", len(sample_entries)) + b"".join(sample_entries)) + hdlr = full(b"hdlr", bytes(4) + handler + bytes(12)) + trak = box( + b"trak", + full(b"tkhd", bytes(8) + struct.pack(">I", 1)), + box(b"mdia", full(b"mdhd", bytes(8) + struct.pack(">I", 1000)), hdlr, box(b"minf", box(b"stbl", stsd, stbl))), + ) + return box(b"ftyp", b"iso6") + box(b"moov", trak, box(b"mvex", full(b"trex", struct.pack(">III", 1, 1, 0)))) + + +def moof( + entry: int, + time: int, + *groups: bytes, + samples: int = 48, + duration: Optional[int] = None, + tfxd: bool = False, +) -> bytes: + flags = 0x02 | (0x08 if duration is not None else 0) + tfhd = struct.pack(">II", 1, entry) + (struct.pack(">I", duration) if duration is not None else b"") + if tfxd: + timing = box(b"uuid", PIFF_TFXD, bytes([1, 0, 0, 0]) + struct.pack(">QQ", time, 0)) + else: + timing = full(b"tfdt", struct.pack(">Q", time), version=1) + trun = full(b"trun", struct.pack(">I", samples)) + return box(b"moof", box(b"traf", full(b"tfhd", tfhd, flags=flags), timing, trun, *groups)) + box(b"mdat", bytes(64)) + + +def local(kid: UUID, protected: bool = True) -> tuple[bytes, bytes]: + return sgpd(seig(protected, kid)), sbgp((48, 0x10001)) + + +def test_each_kid_maps_to_its_own_fragments(tmp_path: Path) -> None: + path = tmp_path / "enc.mp4" + path.write_bytes( + movie(encv(A), encv(B)) + + moof(1, 5000) # lead under A + + moof(2, 7000) # the rest under B + + moof(2, 9000) + + moof(1, 11000, *local(C)) # a seig group overrides the sample entry's KID + + moof(1, 13000, *local(A, protected=False)) # a clear seig group names no KID + ) + assert kid_windows(path) == KidMap({A: [(0.0, 2.0)], B: [(2.0, 6.0), (4.0, 6.0)], C: [(6.0, 8.0)]}, True) + + +def test_windows_run_through_a_kid_but_stop_at_another_or_after_ten_seconds() -> None: + long_run = [(t * 2.0, t * 2.0 + 2) for t in range(20)] + assert pick_windows(long_run) == [(0.0, 10.0), (20.0, 30.0), (30.0, 40.0)] + # a gap is another KID or a clear stretch: no window crosses it, and the last one ends the last run + assert pick_windows([(0, 2), (2, 4), (10, 12), (12, 14), (14, 16)]) == [(0, 4), (10, 16)] + + +def test_first_middle_and_last_fragment(tmp_path: Path) -> None: + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A)) + b"".join(moof(1, t * 1000) for t in range(5))) + assert kid_windows(path) == KidMap({A: [(0.0, 7.0), (2.0, 7.0)]}, True) + + +def test_last_fragment_ends_after_its_sample_durations(tmp_path: Path) -> None: + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A)) + moof(1, 0, samples=10, duration=250)) + assert kid_windows(path) == KidMap({A: [(0.0, 2.5)]}, True) + + +def test_audio_track_is_not_video(tmp_path: Path) -> None: + path = tmp_path / "enc.m4a" + path.write_bytes(movie(enca(A), handler=b"soun") + moof(1, 0)) + assert kid_windows(path) == KidMap({A: [(0.0, 3.0)]}, False) + + +def test_moov_seig_table_survives_another_grouping(tmp_path: Path) -> None: + roll = full(b"sgpd", b"roll" + struct.pack(">II", 2, 1) + b"\x00\x01", version=1) + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A), stbl=sgpd(seig(True, C)) + roll) + moof(1, 0, sbgp((48, 1)))) + assert kid_windows(path) == KidMap({C: [(0.0, 3.0)]}, True) + + +def test_sgpd_version_2(tmp_path: Path) -> None: + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A)) + moof(1, 0, sgpd(seig(True, C), version=2), sbgp((48, 0x10001)))) + assert kid_windows(path) == KidMap({C: [(0.0, 3.0)]}, True) + + +def test_sgpd_version_0_with_constant_iv(tmp_path: Path) -> None: + constant_iv = bytes([0, 0, 1, 0]) + B.bytes + b"\x08" + bytes(8) + groups = sgpd(constant_iv, seig(True, C), version=0) + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A)) + moof(1, 0, groups, sbgp((24, 0x10001), (24, 0x10002)))) + assert kid_windows(path) == KidMap({B: [(0.0, 3.0)], C: [(0.0, 3.0)]}, True) + + +def test_samples_past_the_sbgp_runs_take_the_sample_entry_kid(tmp_path: Path) -> None: + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A)) + moof(1, 0, sgpd(seig(True, C)), sbgp((10, 0x10001)))) + assert kid_windows(path) == KidMap({A: [(0.0, 3.0)], C: [(0.0, 3.0)]}, True) + + +def test_piff_tenc_and_tfxd(tmp_path: Path) -> None: + path = tmp_path / "enc.ismv" + path.write_bytes(movie(encv(A, piff=True)) + moof(1, 0, tfxd=True) + moof(1, 2000, tfxd=True)) + assert kid_windows(path) == KidMap({A: [(0.0, 5.0), (2.0, 5.0)]}, True) + + +def test_oversized_box(tmp_path: Path) -> None: + path = tmp_path / "enc.mp4" + path.write_bytes(movie(encv(A)) + moof(1, 0) + struct.pack(">I4s", 0x7FFFFFFF, b"moof") + bytes(16)) + assert kid_windows(path) is None + + +def test_not_a_fragmented_mp4(tmp_path: Path) -> None: + path = tmp_path / "video.ts" + path.write_bytes(b"\x47" * 188 * 4) + assert kid_windows(path) is None diff --git a/tests/core/test_vault_bad_keys.py b/tests/core/test_vault_bad_keys.py index ccaf0734..b13b9b64 100644 --- a/tests/core/test_vault_bad_keys.py +++ b/tests/core/test_vault_bad_keys.py @@ -77,7 +77,11 @@ class FakeDRM: def make_cmd(monkeypatch: pytest.MonkeyPatch, vaults: Vaults) -> dl: monkeypatch.setattr(dl_module.binaries, "FFMPEG", "ffmpeg") - monkeypatch.setattr(dl_module, "ffmpeg_decodes", lambda path: path.read_bytes() == b"plain:" + GOOD.encode()) + monkeypatch.setattr( + dl_module, + "ffmpeg_decodes", + lambda path, start=None, seconds=3, video=None: path.read_bytes() == b"plain:" + GOOD.encode(), + ) cmd = dl.__new__(dl) cmd.log = SimpleNamespace(warning=print, info=print, debug=print) cmd.vaults = vaults @@ -248,7 +252,7 @@ def test_cdm_returning_the_flagged_key_clears_the_flag(tmp_path: Path, monkeypat vaults = Vaults("SVC") vaults.vaults = [local, poisoned] cmd = make_cmd(monkeypatch, vaults) - monkeypatch.setattr(dl_module, "ffmpeg_decodes", lambda path: False) + monkeypatch.setattr(dl_module, "ffmpeg_decodes", lambda path, start=None, seconds=3, video=None: False) path = tmp_path / "video.mp4" path.write_bytes(b"cipher") drm = FakeDRM({KID: vaults.get_key(KID)[0]}) @@ -300,6 +304,158 @@ def test_flag_on_a_read_only_vault_keeps_its_rows(tmp_path: Path) -> None: assert list(ro.get_keys("SVC")) == [(KID.hex, BAD)] # but the row itself is untouched +def test_flag_moves_the_row_and_blocks_it_coming_back(tmp_path: Path) -> None: + local = SQLite("local", tmp_path / "local.db") + local.add_key("SVC", KID, BAD) + local.flag_bad_key("SVC", KID, BAD, "poisoned") + assert local.is_bad_key(KID, BAD) + assert list(local.get_keys("SVC")) == [] + assert local.add_key("SVC", KID, BAD) is False + assert local.add_keys("SVC", {KID: BAD}) == 0 + assert list(local.get_keys("SVC")) == [] + assert local.add_key("SVC", KID, GOOD) # the KID itself is not blocked, only the pair + + +def test_bulk_add_refuses_a_flagged_pair_in_any_case_or_kid_form(tmp_path: Path) -> None: + local = SQLite("local", tmp_path / "local.db") + local.flag_bad_key("SVC", KID, BAD, "poisoned") + for kid, key in ((KID.hex.upper(), BAD), (str(KID), BAD.upper()), (KID, BAD)): + assert local.add_keys("SVC", {kid: key}) == 0 + assert list(local.get_keys("SVC")) == [] + + +def test_only_the_kid_whose_window_fails_is_flagged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A lead under one KID and the rest under another: the lead's key is right and stays.""" + lead = UUID(int=7) + local = SQLite("local", tmp_path / "local.db") + poisoned = Remote("poisoned", tmp_path / "poisoned.db") + poisoned.add_key("SVC", lead, BAD) # the same wrong key under both KIDs, as the NF title had + poisoned.add_key("SVC", KID, BAD) + other = Remote("other", tmp_path / "other.db") + other.add_key("SVC", KID, GOOD) + vaults = Vaults("SVC") + vaults.vaults = [local, poisoned] + cmd = make_cmd(monkeypatch, vaults) + drm = FakeDRM({lead: vaults.get_key(lead)[0], KID: vaults.get_key(KID)[0]}) + right = {lead: BAD, KID: GOOD} + monkeypatch.setattr( + dl_module.verify, + "kid_windows", + lambda path: dl_module.verify.KidMap({lead: [(0.0, 2.0)], KID: [(120.0, 122.0)]}, True), + ) + monkeypatch.setattr( + dl_module, + "ffmpeg_decodes", + lambda path, start=None, seconds=3, video=None: ( + drm.decrypted_with[-1][lead if start == 0.0 else KID] == right[lead if start == 0.0 else KID] + ), + ) + path = tmp_path / "video.mp4" + path.write_bytes(b"cipher") + + def licence(drm: FakeDRM, track_kid: UUID) -> None: + vaults.vaults.append(other) + drm.content_keys[KID] = vaults.get_key(KID)[0] + + cmd.decrypt_verified(drm, path, licence, KID) + cmd.wait_vault_writes() + assert drm.content_keys == right + assert local.is_bad_key(KID, BAD) + assert not local.is_bad_key(lead, BAD) + + +def test_a_kid_absent_from_the_map_is_judged_by_the_fallback(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A KID the map does not know (a zero-KID tenc, a licence KID the file does not carry) + shares the whole-file verdict, and a wrong key there is flagged.""" + lead = UUID(int=7) + local = SQLite("local", tmp_path / "local.db") + poisoned = Remote("poisoned", tmp_path / "poisoned.db") + poisoned.add_key("SVC", lead, GOOD) + poisoned.add_key("SVC", KID, BAD) + other = Remote("other", tmp_path / "other.db") + other.add_key("SVC", KID, GOOD) + vaults = Vaults("SVC") + vaults.vaults = [local, poisoned] + cmd = make_cmd(monkeypatch, vaults) + drm = FakeDRM({lead: vaults.get_key(lead)[0], KID: vaults.get_key(KID)[0]}) + calls: list[tuple] = [] + + def decodes(path: Path, start: object = None, seconds: float = 3, video: object = None) -> bool: + calls.append((start, seconds, video)) + return drm.decrypted_with[-1][lead if start is not None else KID] == GOOD + + monkeypatch.setattr( + dl_module.verify, "kid_windows", lambda path: dl_module.verify.KidMap({lead: [(4.0, 6.0)]}, True) + ) + monkeypatch.setattr(dl_module, "ffmpeg_decodes", decodes) + path = tmp_path / "video.mp4" + path.write_bytes(b"cipher") + + def licence(drm: FakeDRM, track_kid: UUID) -> None: + vaults.vaults.append(other) + drm.content_keys[KID] = vaults.get_key(KID)[0] + + cmd.decrypt_verified(drm, path, licence, KID) + cmd.wait_vault_writes() + assert drm.content_keys == {lead: GOOD, KID: GOOD} + assert local.is_bad_key(KID, BAD) + assert not local.is_bad_key(lead, GOOD) + assert (4.0, 2.0, True) in calls # the window is capped at the fragment's end + assert (None, 3, True) in calls # the fallback knows the track is video + + +def test_only_checked_kids_are_copied_to_other_vaults(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Both the mapped KID and the absent one are proven before they reach another vault.""" + lead = UUID(int=7) + local = SQLite("local", tmp_path / "local.db") + remote = Remote("remote", tmp_path / "remote.db") + remote.add_key("SVC", lead, GOOD) + remote.add_key("SVC", KID, GOOD) + vaults = Vaults("SVC") + vaults.vaults = [local, remote] + cmd = make_cmd(monkeypatch, vaults) + drm = FakeDRM({lead: vaults.get_key(lead)[0], KID: vaults.get_key(KID)[0]}) + starts: list = [] + + def decodes(path: Path, start: object = None, seconds: float = 3, video: object = None) -> bool: + starts.append(start) + return True + + monkeypatch.setattr( + dl_module.verify, "kid_windows", lambda path: dl_module.verify.KidMap({lead: [(4.0, 6.0)]}, True) + ) + monkeypatch.setattr(dl_module, "ffmpeg_decodes", decodes) + path = tmp_path / "video.mp4" + path.write_bytes(b"cipher") + + cmd.decrypt_verified(drm, path, None, KID) + cmd.wait_vault_writes() + assert starts.count(None) == 1 # the absent KID went through the fallback before its copy + assert local.get_key(lead, "SVC") == GOOD + assert local.get_key(KID, "SVC") == GOOD + + +def test_flagged_pair_warns_once_per_vault(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + local = SQLite("local", tmp_path / "local.db") + poisoned = Remote("poisoned", tmp_path / "poisoned.db") + poisoned.add_key("SVC", KID, BAD) + local.flag_bad_key("SVC", KID, BAD, "poisoned") + vaults = Vaults("SVC") + vaults.vaults = [local, poisoned] + with caplog.at_level(logging.WARNING, logger="unshackle.core.vaults"): + for _ in range(3): + assert vaults.query_vault(poisoned, KID) is None + assert sum("flagged bad, skipping" in r.getMessage() for r in caplog.records) == 1 + + +def test_add_key_refuses_a_flagged_pair_with_a_dashed_kid(tmp_path: Path) -> None: + local = SQLite("local", tmp_path / "local.db") + local.flag_bad_key("SVC", KID, BAD, "poisoned") + assert local.add_key("SVC", str(KID), BAD) is False + assert local.add_key("SVC", str(KID).upper(), BAD.upper()) is False + assert list(local.get_keys("SVC")) == [] + + def test_a_sibling_verdict_during_the_decrypt_still_triggers_the_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 03eb12c7..5368add0 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -4006,7 +4006,11 @@ class dl: A poisoned vault returns the right KID with a wrong content key, and the decrypters accept it without complaint. A content key from a vault is therefore trusted, and only then copied to the other vaults, once FFmpeg can decode the result. Without FFmpeg - neither happens. On failure the pair is flagged in the local vaults and reported + neither happens. For a fragmented MP4, ``verify.kid_windows`` reads from the ciphertext + which fragments each KID encrypts, and FFmpeg decodes windows inside each KID's + fragments, so only a KID that fails is flagged. A KID that is not in the map, and every KID + of any other file, gets a start and end check of the whole file, and a failure there flags + each of those KIDs. On failure the pair is flagged in the local vaults and reported to the vault that served it, the ciphertext is restored, and the licence runs again. Every vault skips a flagged pair, so each pass burns at most one vault; when the vaults run out the CDM answers, and a CDM key is not checked. A CDM that returns the flagged key clears the @@ -4049,6 +4053,28 @@ class dl: os.link(path, backup) except OSError as e: self.log.debug(f"Cannot keep the ciphertext for a decrypt retry: {e!r}") + kid_map = verify.kid_windows(path) if vault_kids() and decrypt and binaries.FFMPEG else None + windows = kid_map.windows if kid_map else {} + video = kid_map.video if kid_map else None + + def window_decodes(start: float, end: float) -> bool: + return ffmpeg_decodes(path, start=start, seconds=end - start if end > start else 0.5, video=video) + + def failed_kids(kids: dict[UUID, Vault]) -> set[UUID]: + """The vault KIDs whose content key did not decode. + + A KID in the KID map fails when one of its windows fails. The KIDs that are not in the + map share the verdict of one start and end check of the whole file. + """ + failed = {kid for kid in kids if kid in windows and not all(window_decodes(*w) for w in windows[kid])} + absent = {kid for kid in kids if kid not in windows} + if absent and not ffmpeg_decodes(path, video=video): + failed |= absent + self.log.debug( + f"Key check on {path.name}: {len(kids) - len(absent)} KID(s) by fragment window, " + f"{len(absent)} by whole-file check, failed {sorted(k.hex for k in failed)}" + ) + return failed def passes_left(done: int) -> bool: """One pass per source that can still answer: every loaded vault, the server's vaults as @@ -4064,10 +4090,12 @@ class dl: drm.decrypt(path) kids = vault_kids() stale = flagged_kids() + failed: set[UUID] = set() if not stale: if not kids or not binaries.FFMPEG or not path.exists(): return - if ffmpeg_decodes(path): + failed = failed_kids(kids) + if not failed: self.flush_vault_writes( [ partial(self.vaults.add_key, kid, keys[kid], excluding=vault) @@ -4075,7 +4103,7 @@ class dl: ] ) return - bad = {kid: drop(kid) for kid in {*kids, *stale}} + bad = {kid: drop(kid) for kid in {*failed, *stale}} for kid, key in bad.items(): if kid not in kids: continue diff --git a/unshackle/core/drm/verify.py b/unshackle/core/drm/verify.py index f782b9d8..be138bfa 100644 --- a/unshackle/core/drm/verify.py +++ b/unshackle/core/drm/verify.py @@ -1,9 +1,28 @@ +import os +import struct from pathlib import Path -from typing import Any, Callable, Optional +from typing import Any, BinaryIO, Callable, Iterator, NamedTuple, Optional from uuid import UUID DECRYPT_HOOK: Optional[Callable[..., None]] = None +CONTAINERS = {b"moov", b"trak", b"mdia", b"minf", b"stbl", b"mvex", b"moof", b"traf", b"sinf", b"schi"} +PIFF_TENC = bytes.fromhex("8974dbce7be74c5184f97148f9882554") +PIFF_TFXD = bytes.fromhex("6d1d9b0542d544e680e2141daff757b2") +TENC = {b"tenc", b"uuid" + PIFF_TENC} +# the fixed sample entry fields before the child boxes: visual; sound v0 (and ISO v1), QuickTime sound v1 and v2 +ENTRY_FIELDS = {b"encv": (78,), b"enca": (28, 44, 64)} + +Seig = tuple[bool, UUID] +Fragment = tuple[int, int, int, set[UUID]] # track, decode time, duration, KIDs + + +class KidMap(NamedTuple): + """The fragment windows of each KID in a fragmented MP4.""" + + windows: dict[UUID, list[tuple[float, float]]] # KID -> (start, end) seconds of its first, middle and last fragment + video: bool # the encrypted track is video + def decrypt_track( drm: Any, @@ -21,3 +40,303 @@ def decrypt_track( DECRYPT_HOOK(drm, path, licence, track_kid, decrypt) elif decrypt: drm.decrypt(path) + + +def children(data: bytes, start: int = 0, end: Optional[int] = None) -> Iterator[tuple[bytes, int, int]]: + """Yield (type, body start, box end) for each box in ``data[start:end]``.""" + end = len(data) if end is None else end + while start + 8 <= end: + size, kind = struct.unpack_from(">I4s", data, start) + head = 8 + if size == 1: + size, head = struct.unpack_from(">Q", data, start + 8)[0], 16 + elif size == 0: + size = end - start + if size < head or start + size > end: + return + if kind == b"uuid": + kind, head = b"uuid" + data[start + head : start + head + 16], head + 16 + yield kind, start + head, start + size + start += size + + +def walk(data: bytes, start: int = 0, end: Optional[int] = None) -> Iterator[tuple[bytes, int, int]]: + """Yield every box below ``data[start:end]``, descending into the container boxes.""" + for kind, body, box_end in children(data, start, end): + yield kind, body, box_end + if kind in CONTAINERS: + yield from walk(data, body, box_end) + + +def seig_entries(data: bytes, body: int) -> Optional[list[Seig]]: + """Parse an ``sgpd`` box. Returns its seig entries as (protected, KID), or None for another grouping.""" + version = data[body] + if data[body + 4 : body + 8] != b"seig": + return None + pos, default_length = body + 8, 0 + if version >= 1: + default_length = struct.unpack_from(">I", data, pos)[0] + pos += 4 + if version >= 2: + pos += 4 # default_group_description_index + count = struct.unpack_from(">I", data, pos)[0] + pos += 4 + entries = [] + for _ in range(count): + length = default_length + if version >= 1 and not length: + length = struct.unpack_from(">I", data, pos)[0] + pos += 4 + protected, iv_size = data[pos + 2], data[pos + 3] + if version == 0: + # version 0 gives no entry length; a constant IV follows the KID when samples carry no IV + length = 20 + (1 + data[pos + 20] if protected and not iv_size else 0) + entries.append((bool(protected), UUID(bytes=data[pos + 4 : pos + 20]))) + pos += length + return entries + + +def sbgp_runs(data: bytes, body: int) -> Optional[list[tuple[int, int]]]: + """Parse an ``sbgp`` box. Returns its seig runs as (sample count, group index), or None for another grouping.""" + version = data[body] + if data[body + 4 : body + 8] != b"seig": + return None + pos = body + (12 if version == 1 else 8) + count = struct.unpack_from(">I", data, pos)[0] + runs = struct.unpack_from(f">{count * 2}I", data, pos + 4) + return list(zip(runs[::2], runs[1::2])) + + +def entry_kid(data: bytes, kind: bytes, entry: int, end: int) -> Optional[UUID]: + """Return the default KID of an encrypted sample entry, or None for a clear one.""" + for fields in ENTRY_FIELDS.get(kind, ()): + boxes = list(walk(data, entry + fields, end)) + if not any(box_kind == b"sinf" for box_kind, _, _ in boxes): + continue + for box_kind, body, _ in boxes: + if box_kind in TENC: + # CENC isProtected and the last byte of the PIFF AlgorithmID share this offset; both are 0 for clear + return UUID(bytes=data[body + 8 : body + 24]) if data[body + 6] else None + return None + return None + + +def sample_entries(data: bytes, body: int, end: int) -> list[Optional[UUID]]: + """Parse an ``stsd`` box. Returns the default KID of each sample entry, None for a clear one.""" + return [entry_kid(data, kind, entry, entry_end) for kind, entry, entry_end in children(data, body + 8, end)] + + +def read_top_boxes(f: BinaryIO) -> Iterator[tuple[bytes, Optional[bytes]]]: + """Yield each top-level box type, with the body of a moov or moof. Every other body is skipped. + + Raises ValueError when a moov or moof claims more bytes than the file has left. + """ + file_size = os.fstat(f.fileno()).st_size + while True: + head = f.read(8) + if len(head) < 8: + return + size, kind = struct.unpack(">I4s", head) + body_size = size - 8 + if size == 1: + body_size = struct.unpack(">Q", f.read(8))[0] - 16 + elif size == 0: + return + if body_size < 0: + return + if kind in (b"moov", b"moof"): + if body_size > file_size - f.tell(): + raise ValueError(f"{kind!r} box is larger than the rest of the file") + yield kind, f.read(body_size) + else: + f.seek(body_size, 1) + yield kind, None + + +def kid_windows(path: Path) -> Optional[KidMap]: + """Map each KID that encrypts part of a fragmented MP4 to a few fragment time windows, in seconds. + + Reads only the moov and moof boxes of the ciphertext and skips every mdat, so a large + file costs little. A fragment takes the KID of its sample entry's tenc (or PIFF tenc), + unless a seig sample group names another KID or marks the samples clear. For each KID + the result holds up to three (start, end) windows from ``pick_windows``, relative to + the first fragment of the track. A fragment ends where the next fragment of the track + starts. The last fragment ends after its sample durations, or after 3 seconds when + they are not known. Returns None when the file has no encrypted fragment this reader + can place in time, so the caller must use another check. + """ + timescales: dict[int, int] = {} + handlers: dict[int, bytes] = {} + entries: dict[int, list[Optional[UUID]]] = {} + defaults: dict[int, tuple[int, int]] = {} + moov_groups: dict[int, list[Seig]] = {} + fragments: list[Fragment] = [] + clock: dict[int, int] = {} + try: + with path.open("rb") as f: + for kind, box in read_top_boxes(f): + if kind == b"moov" and box: + for trak_kind, trak, trak_end in children(box): + if trak_kind == b"mvex": + for kind_, body, _ in children(box, trak, trak_end): + if kind_ == b"trex": + track, index, duration = struct.unpack_from(">III", box, body + 4) + defaults[track] = (index, duration) + if trak_kind != b"trak": + continue + track = 0 + for kind_, body, box_end in walk(box, trak, trak_end): + if kind_ == b"tkhd": + track = struct.unpack_from(">I", box, body + (20 if box[body] == 1 else 12))[0] + elif kind_ == b"mdhd": + timescale = struct.unpack_from(">I", box, body + (20 if box[body] == 1 else 12))[0] + timescales[track] = timescale + elif kind_ == b"hdlr": + # the mdia handler comes first; a QuickTime minf can hold a data handler after it + handlers.setdefault(track, box[body + 8 : body + 12]) + elif kind_ == b"stsd": + entries[track] = sample_entries(box, body, box_end) + elif kind_ == b"sgpd": + groups = seig_entries(box, body) + if groups is not None: + moov_groups[track] = groups + elif kind == b"moof" and box: + fragments.extend(read_fragment(box, entries, defaults, moov_groups, clock)) + except (OSError, struct.error, IndexError, ValueError, MemoryError, OverflowError): + return None + + tracks: dict[int, list[Fragment]] = {} + for fragment in fragments: + tracks.setdefault(fragment[0], []).append(fragment) + spans: dict[UUID, list[tuple[float, float]]] = {} + video = False + for track, track_fragments in tracks.items(): + timescale = timescales.get(track) + if not timescale: + return None + track_fragments.sort(key=lambda fragment: fragment[1]) + first = track_fragments[0][1] + for i, (_, time, duration, kids) in enumerate(track_fragments): + if not kids: + continue + start = (time - first) / timescale + if i + 1 < len(track_fragments): + end = (track_fragments[i + 1][1] - first) / timescale + else: + end = start + (duration / timescale if duration else 3.0) + video = video or handlers.get(track) == b"vide" + for kid in kids: + spans.setdefault(kid, []).append((start, end)) + if not spans: + return None + return KidMap({kid: pick_windows(fragment_spans) for kid, fragment_spans in spans.items()}, video) + + +def pick_windows(spans: list[tuple[float, float]], longest: float = 10.0) -> list[tuple[float, float]]: + """Pick up to three windows from one KID's fragment (start, end) spans: at its first, middle and last fragment. + + A window starts at a fragment and runs on through the next fragments of the same KID, up + to ``longest`` seconds, but never into a fragment of another KID or a clear one. A short + window can decode noise without an error: the decoder often needs several frames to find it. + The last window ends where the KID's last run of fragments ends. + """ + spans = sorted(spans) + runs: list[list[float]] = [] # [start, end] of each run of back-to-back fragments + for start, end in spans: + if runs and abs(start - runs[-1][1]) <= 1e-3: + runs[-1][1] = end + else: + runs.append([start, end]) + + def run_end(start: float) -> float: + return next(end for run_start, end in runs if run_start <= start < end) + + last_from = max(runs[-1][0], runs[-1][1] - longest) + last = next(i for i, (start, _) in enumerate(spans) if start >= last_from - 1e-3) + return [(spans[i][0], min(run_end(spans[i][0]), spans[i][0] + longest)) for i in sorted({0, len(spans) // 2, last})] + + +def trun_totals(moof: bytes, truns: list[int], default_duration: int) -> tuple[int, int]: + """Return the sample count and the summed sample duration of the ``trun`` boxes at the given body offsets.""" + samples = ticks = 0 + for body in truns: + flags = int.from_bytes(moof[body + 1 : body + 4], "big") + count = struct.unpack_from(">I", moof, body + 4)[0] + samples += count + if flags & 0x100: + pos = body + 8 + (4 if flags & 0x01 else 0) + (4 if flags & 0x04 else 0) + stride = 4 * bin(flags & 0xF00).count("1") + ticks += sum(struct.unpack_from(">I", moof, pos + i * stride)[0] for i in range(count)) + else: + ticks += count * default_duration + return samples, ticks + + +def read_fragment( + moof: bytes, + entries: dict[int, list[Optional[UUID]]], + defaults: dict[int, tuple[int, int]], + moov_groups: dict[int, list[Seig]], + clock: dict[int, int], +) -> Iterator[Fragment]: + """Yield (track, decode time, duration, KIDs that encrypt samples) for each track fragment in a moof. + + A fragment without a tfdt or tfxd starts where the track's previous fragment ended, by + the running sum of sample durations in ``clock``. + """ + for kind, traf, traf_end in children(moof): + if kind != b"traf": + continue + track, time, entry, default_duration = 0, None, 0, 0 + groups: list[Seig] = [] + runs: Optional[list[tuple[int, int]]] = None + truns: list[int] = [] + for kind_, body, _ in children(moof, traf, traf_end): + if kind_ == b"tfhd": + flags = int.from_bytes(moof[body + 1 : body + 4], "big") + track = struct.unpack_from(">I", moof, body + 4)[0] + pos = body + 8 + (8 if flags & 0x01 else 0) + if flags & 0x02: + entry = struct.unpack_from(">I", moof, pos)[0] + pos += 4 + if flags & 0x08: + default_duration = struct.unpack_from(">I", moof, pos)[0] + elif kind_ == b"tfdt": + time = struct.unpack_from(">Q" if moof[body] == 1 else ">I", moof, body + 4)[0] + elif kind_ == b"uuid" + PIFF_TFXD and time is None: + time = struct.unpack_from(">Q" if moof[body] == 1 else ">I", moof, body + 4)[0] + elif kind_ == b"trun": + truns.append(body) + elif kind_ == b"sgpd": + groups = seig_entries(moof, body) or groups + elif kind_ == b"sbgp": + runs = sbgp_runs(moof, body) or runs + if track not in entries: + continue + default_entry, trex_duration = defaults.get(track, (1, 0)) + samples, duration = trun_totals(moof, truns, default_duration or trex_duration) + if time is None: + if not duration: + continue + time = clock.get(track, 0) + clock[track] = time + duration + track_entries = entries[track] + entry = entry or default_entry + base = track_entries[entry - 1] if 0 < entry <= len(track_entries) else None + indexes = [0] + if runs is not None: + indexes = [index for count, index in runs if count] + if samples > sum(count for count, _ in runs): + indexes.append(0) # samples past the last run take the sample entry's tenc + kids: set[UUID] = set() + for index in indexes: + if index == 0: + if base: + kids.add(base) + continue + local = index > 0x10000 + table = groups if local else moov_groups.get(track, []) + position = (index - 0x10001) if local else index - 1 + if 0 <= position < len(table) and table[position][0]: + kids.add(table[position][1]) + yield track, time, duration, kids diff --git a/unshackle/core/utils/subprocess.py b/unshackle/core/utils/subprocess.py index 2e2d454d..e1c28a64 100644 --- a/unshackle/core/utils/subprocess.py +++ b/unshackle/core/utils/subprocess.py @@ -40,40 +40,98 @@ def log_tool_run( ) -def ffmpeg_decodes(path: Path, seconds: int = 3) -> bool: - """Return True when FFmpeg decodes the first seconds of the file without an error. +FFMPEG_INCONCLUSIVE = ( + "not found for", + "no decoder found", + "sub-sample encryption info", + "error reading header", + "not a decoding option", +) + +FFMPEG_MISSING_REFERENCE = ( + "mmco", + "missing reference", + "co located pocs unavailable", + "reference picture missing", + "could not find ref with poc", +) +FFMPEG_CHECK_TIMEOUT = 120 + + +def ffmpeg_decodes(path: Path, start: Optional[float] = None, seconds: float = 3, video: Optional[bool] = None) -> bool: + """Return True when FFmpeg decodes a window of the file without an error. A wrong content key leaves the container intact and the samples as noise, which the decoders reject. ``-xerror`` stops at the first error so the check stays short. - A file this FFmpeg build cannot judge counts as a pass, never as a wrong key. + + With ``start``, the check decodes ``seconds`` from that time. An FFmpeg copy cuts the + window first, so no sample after the window gets to the decoder. Without ``start``, the + check decodes the first seconds and the last 10 seconds, because a title can open with + a clear or separately keyed lead. + + Every frame is decoded, because an HEVC keyframe decrypted with a wrong key can decode + without an error. When a window starts inside a GOP and the only error is a missing + reference frame, the check decodes the keyframes of that window again and uses that + verdict. ``video=False`` skips that second pass. + + A file this FFmpeg build cannot judge counts as a pass, never as a wrong key, and so + does a check that does not finish in time. """ if not binaries.FFMPEG: raise EnvironmentError('FFmpeg executable "ffmpeg" not found but is required.') - args: list[str] = [ - str(binaries.FFMPEG), - "-nostdin", - "-v", - "error", - "-err_detect", - "explode", - "-xerror", - "-i", - str(path), - "-t", - str(seconds), - "-f", - "null", - "-", - ] - ff = subprocess.run(args, capture_output=True, text=True, encoding="utf-8", errors="replace") - if ff.returncode == 0: - return True - err = ff.stderr.lower() - if any( - m in err for m in ("not found for", "no decoder found", "sub-sample encryption info", "error reading header") - ): - return True - return False + ffmpeg = str(binaries.FFMPEG) + + def decode(window: list[str], keyframes: bool) -> bool: + check = [ffmpeg, "-nostdin", "-v", "error", "-err_detect", "explode", "-xerror"] + if keyframes: + check += ["-skip_frame:v", "nokey"] + cut = None + try: + if start is None: + ff = subprocess.run( + [*check, *window, "-i", str(path), "-f", "null", "-"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=FFMPEG_CHECK_TIMEOUT, + ) + else: + cut = subprocess.Popen( + [ffmpeg, "-nostdin", "-v", "error", *window, "-i", str(path)] + + ["-map", "0:v?", "-map", "0:a?", "-c", "copy", "-f", "nut", "pipe:1"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + ff = subprocess.run( + [*check, "-i", "pipe:", "-f", "null", "-"], + stdin=cut.stdout, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=FFMPEG_CHECK_TIMEOUT, + ) + except subprocess.TimeoutExpired: + return True + finally: + if cut: + if cut.stdout: + cut.stdout.close() + cut.kill() + cut.wait() + err = ff.stderr.lower() + if cut and cut.returncode not in (0, -9) and "error opening input" in err: + return True # the cut failed, so no sample got to the decoder + if ff.returncode == 0 or any(m in err for m in FFMPEG_INCONCLUSIVE): + return True + if not keyframes and video is not False and any(m in err for m in FFMPEG_MISSING_REFERENCE): + return decode(window, keyframes=True) + return False + + if start is not None: + return decode(["-ss", f"{start:.6f}", "-t", f"{max(seconds - 0.5, seconds / 2):.6f}"], keyframes=False) + return decode(["-t", str(seconds)], keyframes=False) and decode(["-sseof", "-10", "-t", "10"], keyframes=False) def ffprobe(uri: Union[bytes, Path]) -> dict: diff --git a/unshackle/core/vaults.py b/unshackle/core/vaults.py index d2fd2e06..9e8d5935 100644 --- a/unshackle/core/vaults.py +++ b/unshackle/core/vaults.py @@ -28,6 +28,7 @@ class Vaults: self.sources: dict[Union[UUID, str], tuple[str, Vault]] = {} self.flagged: set[tuple[Union[UUID, str], str]] = set() self.candidates: dict[Union[UUID, str], list[tuple[str, Vault]]] = {} + self.warned: set[tuple[str, Union[UUID, str], str]] = set() def __iter__(self) -> Iterator[Vault]: return iter(self.vaults) @@ -134,8 +135,10 @@ class Vaults: return None found = bool(key and key.count("0") != len(key)) if found and key and self.is_flagged(kid, key): - log.warning(f"{key} from {vault.name} is flagged bad, skipping") found = False + if (vault.name, kid, key) not in self.warned: + self.warned.add((vault.name, kid, key)) + log.warning(f"{key} from {vault.name} is flagged bad, skipping") if dl: dl.log_vault_query( vault.name, diff --git a/unshackle/vaults/SQLite.py b/unshackle/vaults/SQLite.py index 2efcd8a6..2060fb91 100644 --- a/unshackle/vaults/SQLite.py +++ b/unshackle/vaults/SQLite.py @@ -68,6 +68,9 @@ class SQLite(Vault): if isinstance(kid, UUID): kid = kid.hex + if self.is_bad_key(kid.replace("-", ""), key): + return False + conn = self.conn_factory.get() cursor = conn.cursor() @@ -107,6 +110,14 @@ class SQLite(Vault): if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()): kid_keys = {kid.hex if isinstance(kid, UUID) else kid: key_ for kid, key_ in kid_keys.items()} + self.create_bad_keys_table() + bad = set(self.conn_factory.get().execute("SELECT lower(`kid`), lower(`key_`) FROM `bad_keys`").fetchall()) + kid_keys = { + kid: key_ + for kid, key_ in kid_keys.items() + if ((kid.hex if isinstance(kid, UUID) else kid.replace("-", "").lower()), key_.lower()) not in bad + } + if not kid_keys: return 0