mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-22 00:45:10 +00:00
Compare commits
2
Commits
5ffdda3883
...
b3d586a590
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b3d586a590 | ||
|
|
893c81f428 |
@@ -53,7 +53,7 @@ All are `GET`, except the two `drain` routes.
|
||||
| `/api/dashboard/cdm-logs` | `{"seq": N, "records": [...], "sessions": [...]}`: the last 5000 calls to the Widevine and PlayReady CDM device routes, and the open sessions of each device. `?since=<seq>` returns only newer records. See [CDM calls](#cdm-calls). |
|
||||
| `/api/dashboard/sessions/{id}/logs` | One remote session's service log: `{"session_id", "last_seq", "records": [...]}`. Each record has `seq`, `ts`, `level`, `message`. `?since=<seq>` returns only newer records. 404 when the remote session is unknown. |
|
||||
| `/api/dashboard/keys` | Every configured API key: what it may do and what it has done. One row per key in `serve.users`, plus `serve.api_secret` when it is set. |
|
||||
| `/api/dashboard/services` | Every service configured on this server and its load state, including the ones that failed to import. See [Services](#services) for what configured means. |
|
||||
| `/api/dashboard/services` | Every service configured on this server, its load state and its catalogue entry, including the services that failed to import. See [Services](#services) for what configured means. |
|
||||
| `/api/dashboard/health` | Preflight: whether this instance could finish a download. Cached 30 seconds. |
|
||||
| `/api/dashboard/events` | Server-Sent Events stream. With `?since=<seq>` it returns `{"seq", "stats", "events": [...]}` instead: every event after `seq` (the server keeps the last 20000, log records included, so a poll every few seconds is safe) as one JSON burst, so a UI that polls every few seconds still sees a remote session that opened and closed in between. |
|
||||
|
||||
@@ -238,7 +238,10 @@ GET /api/dashboard/services
|
||||
[{"tag": "EXAMPLE", "state": "staged", "error": null,
|
||||
"commit": "9cc230c98f13b83f6c6fe831595b9702d3ba2786", "staged_commit": "5f0aa17...",
|
||||
"staged_since": 1756908900.0, "sessions": 1, "jobs": 1,
|
||||
"aliases": ["EXMPL"], "geofence": ["US"], "geoblock": []}]
|
||||
"aliases": ["EXMPL"], "geofence": ["US"], "geoblock": [],
|
||||
"help": "...", "url": "https://www.service.example", "title_regex": "...", "cli_params": [...],
|
||||
"needs_auth": true, "auth_methods": ["cookies"], "has_search": true, "anonymous_search": false,
|
||||
"has_drm": true, "server_accounts": {"regions": ["gb", "us"], "global": false}}]
|
||||
```
|
||||
|
||||
The list holds only the services configured on this server:
|
||||
@@ -269,6 +272,15 @@ for a service from a plain local directory rather than a git repo.
|
||||
|
||||
Watch the `service` event rather than polling for the swap.
|
||||
|
||||
Each row also carries the service's catalogue entry: the fields that
|
||||
[`GET /api/services`](endpoints.md#get-apiservices) sends, with the same meaning. A service with no loaded
|
||||
module, or whose catalogue entry fails, gets only `help`, `url` and `title_regex`, all `null`.
|
||||
A service whose re-import failed keeps the entry of the module that still runs.
|
||||
|
||||
`server_accounts` shows when `serve.server_accounts` names the service. No API key grant
|
||||
applies to it here: `/api/services` sends it only to an API key with the `server_accounts`
|
||||
grant, but the dashboard key sees it for every service.
|
||||
|
||||
## Health
|
||||
|
||||
```
|
||||
|
||||
@@ -187,6 +187,12 @@ service tag case-insensitively and uses the `default` entry as the fallback.
|
||||
|
||||
unshackle decrypts HLS AES-128 ClearKey in-process and ignores this setting.
|
||||
|
||||
mp4decrypt finds a content key by the KID in the first `sinf` box of the track. When that box
|
||||
has no CENC `tenc` (for example, a PIFF `sinf` before the CENC one), unshackle also gives
|
||||
mp4decrypt the content key of that KID by track ID. When mp4decrypt copies a track through
|
||||
still encrypted, unshackle decrypts it again with shaka-packager and logs a warning. This
|
||||
retry needs shaka-packager installed.
|
||||
|
||||
=== "Global"
|
||||
|
||||
```yaml
|
||||
@@ -213,7 +219,8 @@ This works with `mp4decrypt` only. Set [`decryption`](#decryption) to `mp4decryp
|
||||
service (or as the default); with `shaka` the option has no effect, because shaka-packager
|
||||
cannot decrypt a segment without its init segment. It applies to DASH tracks and to HLS
|
||||
tracks that use one init segment, one `EXT-X-KEY` and no `EXT-X-DISCONTINUITY`. Every other
|
||||
track keeps the whole-file decryption.
|
||||
track keeps the whole-file decryption, and so does a track whose init segment mp4decrypt
|
||||
cannot key by KID when no content key has that KID.
|
||||
|
||||
!!! note "No resume"
|
||||
|
||||
|
||||
@@ -91,3 +91,19 @@ def mux_args(monkeypatch: pytest.MonkeyPatch) -> Callable[..., list[str]]:
|
||||
return captured[0]
|
||||
|
||||
return run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_mp4decrypt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""An mp4decrypt that copies its input to its output and records its argv."""
|
||||
argv_log = tmp_path / "argv.jsonl"
|
||||
script = tmp_path / "mp4decrypt"
|
||||
script.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import json, shutil, sys\n"
|
||||
f"open({str(argv_log)!r}, 'a').write(json.dumps(sys.argv[1:]) + '\\n')\n"
|
||||
"shutil.copyfile(sys.argv[-2], sys.argv[-1])\n"
|
||||
)
|
||||
script.chmod(0o755)
|
||||
monkeypatch.setattr(binaries, "Mp4decrypt", script)
|
||||
return argv_log
|
||||
|
||||
@@ -19,10 +19,14 @@ def full(kind: bytes, payload: bytes, version: int = 0, flags: int = 0) -> bytes
|
||||
return box(kind, bytes([version]) + flags.to_bytes(3, "big") + payload)
|
||||
|
||||
|
||||
def encv(kid: UUID, piff: bool = False) -> bytes:
|
||||
def sinf(kid: UUID, piff: bool = False) -> bytes:
|
||||
tenc_body = b"\x00\x00\x01\x08" + kid.bytes
|
||||
tenc = box(b"uuid", PIFF_TENC, bytes(4) + tenc_body) if piff else full(b"tenc", tenc_body)
|
||||
return box(b"encv", bytes(78), box(b"sinf", box(b"frma", b"avc1"), box(b"schi", tenc)))
|
||||
return box(b"sinf", box(b"frma", b"avc1"), box(b"schi", tenc))
|
||||
|
||||
|
||||
def encv(kid: UUID, piff: bool = False) -> bytes:
|
||||
return box(b"encv", bytes(78), sinf(kid, piff))
|
||||
|
||||
|
||||
def enca(kid: UUID) -> bytes:
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.core.test_kid_windows import A, B, box, clear_audio, encv, moof, movie, sinf
|
||||
from tests.core.test_zero_kid_fallback import _playready, _widevine
|
||||
from tests.test_segment_decrypt import read_argv
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.drm import ClearKeyCENC
|
||||
from unshackle.core.drm.key_args import shaka_noise, track_key_args
|
||||
from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
|
||||
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka
|
||||
|
||||
KEY = "b5df2ae3f63b8dda9049d6e41422e075"
|
||||
OTHER_KEY = "11" * 16
|
||||
POSIX_STUBS = pytest.mark.skipif(sys.platform == "win32", reason="the decrypter stubs are shell scripts")
|
||||
|
||||
|
||||
def two_sinf(kid: UUID, piff_first: bool) -> bytes:
|
||||
"""A sample entry with a PIFF and a CENC sinf for the same KID."""
|
||||
return box(b"encv", bytes(78), sinf(kid, piff_first), sinf(kid, not piff_first))
|
||||
|
||||
|
||||
def track_zero(entry: bytes) -> bytes:
|
||||
"""A movie whose tkhd carries track_ID 0."""
|
||||
data = movie(entry)
|
||||
tkhd = data.index(b"tkhd")
|
||||
return data[: tkhd + 16] + bytes(4) + data[tkhd + 20 :]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("init", "kids"),
|
||||
[
|
||||
pytest.param(movie(two_sinf(A, piff_first=True)), {1: A}, id="piff-then-cenc"),
|
||||
pytest.param(movie(encv(A, piff=True)), {1: A}, id="piff-only"),
|
||||
pytest.param(movie(two_sinf(A, piff_first=False)), {}, id="cenc-then-piff"),
|
||||
pytest.param(movie(encv(A)), {}, id="cenc-only"),
|
||||
pytest.param(clear_audio(), {}, id="clear"),
|
||||
pytest.param(track_zero(two_sinf(A, piff_first=True)), {}, id="track-id-0"),
|
||||
pytest.param(b"\x00\x00\x00\x01moov\x00\x00", {}, id="truncated-large-size"),
|
||||
pytest.param(movie(two_sinf(A, piff_first=True))[:-20], {}, id="cut-short"),
|
||||
],
|
||||
)
|
||||
def test_kidless_track_kids(init: bytes, kids: dict[int, UUID]) -> None:
|
||||
assert kidless_track_kids(init) == kids
|
||||
|
||||
|
||||
def test_kidless_track_kids_survives_deep_nesting() -> None:
|
||||
nested = b""
|
||||
for _ in range(3000):
|
||||
nested = box(b"trak", nested)
|
||||
assert kidless_track_kids(nested) == {}
|
||||
|
||||
|
||||
def test_track_key_needs_the_track_kid() -> None:
|
||||
assert track_key_args({A: KEY, B: OTHER_KEY}, {2: A}) == ["--key", f"2:{KEY}"]
|
||||
assert track_key_args({B: OTHER_KEY}, {2: A}) == []
|
||||
|
||||
|
||||
DRMS: list[Any] = [
|
||||
pytest.param(lambda: _widevine(A), id="widevine"),
|
||||
pytest.param(lambda: _playready(A), id="playready"),
|
||||
pytest.param(lambda: ClearKeyCENC(kids=[A]), id="clearkey"),
|
||||
]
|
||||
|
||||
|
||||
def write_movie(path: Path, entry: bytes, fragment: bytes = b"") -> Path:
|
||||
path.write_bytes(movie(entry) + fragment + box(b"mdat", bytes(16)))
|
||||
return path
|
||||
|
||||
|
||||
@POSIX_STUBS
|
||||
@pytest.mark.parametrize("make", DRMS)
|
||||
def test_whole_file_decrypt_keys_a_piff_first_track_by_id(
|
||||
tmp_path: Path, stub_mp4decrypt: Path, make: Callable[[], Any]
|
||||
) -> None:
|
||||
path = write_movie(tmp_path / "track.mp4", two_sinf(A, piff_first=True))
|
||||
drm = make()
|
||||
drm.content_keys[A] = KEY
|
||||
|
||||
drm.decrypt_with_mp4decrypt(path)
|
||||
|
||||
assert f"1:{KEY}" in read_argv(stub_mp4decrypt)[0]
|
||||
|
||||
|
||||
@POSIX_STUBS
|
||||
@pytest.mark.parametrize("make", DRMS)
|
||||
def test_decrypt_falls_back_to_shaka_on_a_passthrough(
|
||||
tmp_path: Path, stub_mp4decrypt: Path, monkeypatch: pytest.MonkeyPatch, make: Callable[[], Any]
|
||||
) -> None:
|
||||
path = write_movie(tmp_path / "track.mp4", two_sinf(A, piff_first=True), moof(1, 0))
|
||||
drm = make()
|
||||
drm.content_keys[B] = OTHER_KEY
|
||||
shaka: list[Path] = []
|
||||
monkeypatch.setattr(type(drm), "decrypt_with_shaka_packager", lambda self, p: shaka.append(p))
|
||||
monkeypatch.setattr(config, "decryption", "mp4decrypt", raising=False)
|
||||
|
||||
drm.decrypt(path)
|
||||
|
||||
assert shaka == [path]
|
||||
|
||||
|
||||
@POSIX_STUBS
|
||||
def test_segment_decrypter_keys_from_the_init_segment(tmp_path: Path, stub_mp4decrypt: Path) -> None:
|
||||
drm = _widevine(A)
|
||||
drm.content_keys[A] = KEY
|
||||
|
||||
decrypter = SegmentDecrypter(drm, movie(two_sinf(A, piff_first=True)), tmp_path, workers=1)
|
||||
decrypter.pool.shutdown()
|
||||
|
||||
assert f"1:{KEY}" in decrypter.key_args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("key_kid", "usable"), [(A, True), (B, False)], ids=["track-kid-keyed", "other-kid-only"])
|
||||
def test_segment_decrypt_needs_a_key_for_a_kidless_track(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, key_kid: UUID, usable: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(config, "decrypt_segments", True, raising=False)
|
||||
monkeypatch.setattr(binaries, "Mp4decrypt", tmp_path / "mp4decrypt")
|
||||
drm = _widevine(A)
|
||||
drm.content_keys[key_kid] = KEY
|
||||
|
||||
assert can_use(drm, "mp4decrypt", movie(two_sinf(A, piff_first=True))) is usable
|
||||
assert can_use(drm, "mp4decrypt", movie(encv(A))) is True
|
||||
|
||||
|
||||
class StubDRM:
|
||||
"""Records which decrypter ran; mp4decrypt writes ``mp4decrypt_output`` over the file."""
|
||||
|
||||
def __init__(self, mp4decrypt_output: bytes) -> None:
|
||||
self.output = mp4decrypt_output
|
||||
self.calls: list[str] = []
|
||||
|
||||
def decrypt_with_mp4decrypt(self, path: Path) -> None:
|
||||
self.calls.append("mp4decrypt")
|
||||
path.write_bytes(self.output)
|
||||
|
||||
def decrypt_with_shaka_packager(self, path: Path) -> None:
|
||||
self.calls.append("shaka")
|
||||
|
||||
|
||||
ENCRYPTED = movie(encv(A)) + moof(1, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output", "calls"),
|
||||
[
|
||||
pytest.param(ENCRYPTED, ["mp4decrypt", "shaka"], id="copied-through"),
|
||||
pytest.param(movie(encv(A)) + moof(1, 1000), ["mp4decrypt"], id="stale-encrypted-moov"),
|
||||
pytest.param(movie(box(b"avc1", bytes(78))) + moof(1, 1000), ["mp4decrypt"], id="decrypted"),
|
||||
pytest.param(b"not an mp4", ["mp4decrypt"], id="unreadable"),
|
||||
],
|
||||
)
|
||||
def test_mp4decrypt_or_shaka(tmp_path: Path, output: bytes, calls: list[str]) -> None:
|
||||
drm = StubDRM(output)
|
||||
path = tmp_path / "track.mp4"
|
||||
path.write_bytes(ENCRYPTED)
|
||||
|
||||
mp4decrypt_or_shaka(drm, path)
|
||||
|
||||
assert drm.calls == calls
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_packager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Callable[[int], None]:
|
||||
"""A shaka-packager that writes its output, logs an ERROR line and exits with the code given."""
|
||||
|
||||
def install(code: int) -> None:
|
||||
script = tmp_path / "packager"
|
||||
script.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import sys\n"
|
||||
"output = sys.argv[1].split('output=')[1].split(',')[0]\n"
|
||||
"open(output, 'wb').write(b'decrypted')\n"
|
||||
"sys.stderr.write('[1103/153702:ERROR:box_definitions.cc(641)] Not implemented reached\\n')\n"
|
||||
f"sys.exit({code})\n"
|
||||
)
|
||||
script.chmod(0o755)
|
||||
monkeypatch.setattr(binaries, "ShakaPackager", script)
|
||||
|
||||
return install
|
||||
|
||||
|
||||
@POSIX_STUBS
|
||||
@pytest.mark.parametrize("make", DRMS)
|
||||
def test_shaka_error_line_with_exit_0_is_a_decrypt(
|
||||
tmp_path: Path, stub_packager: Callable[[int], None], make: Callable[[], Any]
|
||||
) -> None:
|
||||
stub_packager(0)
|
||||
path = tmp_path / "track.mp4"
|
||||
path.write_bytes(b"encrypted")
|
||||
drm = make()
|
||||
drm.content_keys[A] = KEY
|
||||
|
||||
drm.decrypt_with_shaka_packager(path)
|
||||
|
||||
assert path.read_bytes() == b"decrypted"
|
||||
|
||||
|
||||
@POSIX_STUBS
|
||||
@pytest.mark.parametrize("make", DRMS)
|
||||
def test_shaka_non_zero_exit_raises(
|
||||
tmp_path: Path, stub_packager: Callable[[int], None], make: Callable[[], Any]
|
||||
) -> None:
|
||||
stub_packager(2)
|
||||
path = tmp_path / "track.mp4"
|
||||
path.write_bytes(b"encrypted")
|
||||
drm = make()
|
||||
drm.content_keys[A] = KEY
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
drm.decrypt_with_shaka_packager(path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("line", "noise"),
|
||||
[
|
||||
pytest.param(
|
||||
"I1103 21:41:56.557273 16212 packager_main.cc:591] Excluding 'pssh' boxes", True, id="glog-info"
|
||||
),
|
||||
pytest.param("I0503 01:02:03.000000 1 demuxer.cc:105] Demuxer::Run()", True, id="glog-info-early-month"),
|
||||
pytest.param("W1103 21:41:56.000000 1 x.cc:1] warning", True, id="glog-warning"),
|
||||
pytest.param(
|
||||
"[1103/214121:WARNING:track_run_iterator.cc(699)] Seeing non-zero composition offset", True, id="warning"
|
||||
),
|
||||
pytest.param("[1103/153402:INFO:demuxer.cc(170)] Initialize Demuxer", True, id="info"),
|
||||
pytest.param("[1103/153402:ERROR:box_definitions.cc(641)] Not implemented reached", False, id="error"),
|
||||
pytest.param("E1103 21:41:56.000000 1 x.cc:1] failed", False, id="glog-error"),
|
||||
pytest.param("Packaging completed successfully.", False, id="plain"),
|
||||
],
|
||||
)
|
||||
def test_shaka_noise(line: str, noise: bool) -> None:
|
||||
assert shaka_noise(line) is noise
|
||||
@@ -682,3 +682,68 @@ async def test_health_probe_timeout_reports_what_finished(aiohttp_client, dashbo
|
||||
# The timeout is cached for the full TTL, so callers cannot stack up more stalled threads.
|
||||
assert (await client.get("/api/dashboard/health", headers={"X-Secret-Key": "dash-secret"})).status == 200
|
||||
assert handlers._health_cache["payload"]["status"] == "failing"
|
||||
|
||||
|
||||
async def test_dashboard_services_carry_the_catalogue(aiohttp_client, dashboard_cfg, monkeypatch) -> None:
|
||||
import enum
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from unshackle.core import services as services_module
|
||||
from unshackle.core.api import routes
|
||||
from unshackle.core.service import Service
|
||||
|
||||
class Mode(enum.Enum):
|
||||
FAST = 1
|
||||
|
||||
class Loaded(Service):
|
||||
"""How to use AAA."""
|
||||
|
||||
cli = click.Command(
|
||||
"AAA",
|
||||
params=[
|
||||
click.Argument(["title"]),
|
||||
click.Option(["--mode"], type=click.Choice(Mode)),
|
||||
click.Option(["--dir"], multiple=True, default=[Path("/x")]),
|
||||
],
|
||||
short_help="https://aaa.example",
|
||||
)
|
||||
|
||||
class NoGeofence(Service):
|
||||
"""How to use DDD."""
|
||||
|
||||
GEOFENCE = None
|
||||
|
||||
real_details = routes.service_details
|
||||
|
||||
def details(tag: str) -> dict:
|
||||
if tag == "CCC":
|
||||
raise RuntimeError("bad module")
|
||||
return real_details(tag)
|
||||
|
||||
tags = ("AAA", "BBB", "CCC", "DDD")
|
||||
monkeypatch.setattr(services_module, "SERVICES", [Path(f"/svc/{t}/__init__.py") for t in tags])
|
||||
monkeypatch.setattr(services_module, "MODULES", {"AAA": Loaded, "CCC": Loaded, "DDD": NoGeofence})
|
||||
monkeypatch.setattr(services_module, "LOAD_ERRORS", ["BBB: failed to import - ImportError: boom"])
|
||||
monkeypatch.setattr(services_module, "PENDING", {"DDD"})
|
||||
monkeypatch.setattr(routes, "SERVICE_DETAILS", {})
|
||||
monkeypatch.setattr(routes, "service_details", details)
|
||||
monkeypatch.setitem(config.serve, "server_accounts", {"aaa": {"default": ["us", "gb"]}})
|
||||
|
||||
client = await aiohttp_client(make_app())
|
||||
resp = await client.get("/api/dashboard/services", headers={"X-Secret-Key": "dash-secret"})
|
||||
assert resp.status == 200
|
||||
rows = {r["tag"]: r for r in await resp.json()}
|
||||
|
||||
assert rows["AAA"]["help"] == "How to use AAA." and rows["AAA"]["url"] == "https://aaa.example"
|
||||
title, mode, folder = rows["AAA"]["cli_params"]
|
||||
assert title == {"name": "title", "kind": "argument", "required": True}
|
||||
assert mode["choices"] == ["FAST"] and folder["default"] == ["x"]
|
||||
assert rows["AAA"]["state"] == "loaded"
|
||||
assert rows["AAA"]["server_accounts"] == {"regions": ["gb", "us"], "global": False}
|
||||
for tag in ("BBB", "CCC"):
|
||||
assert rows[tag]["help"] is None and "cli_params" not in rows[tag] and "server_accounts" not in rows[tag]
|
||||
assert rows["BBB"]["state"] == "failed"
|
||||
assert rows["DDD"]["state"] == "staged" and rows["DDD"]["help"] == "How to use DDD."
|
||||
assert rows["DDD"]["geofence"] == []
|
||||
|
||||
@@ -23,26 +23,10 @@ class FakeDRM:
|
||||
def __init__(self) -> None:
|
||||
self.content_keys = {KID: KEY}
|
||||
|
||||
def mp4decrypt_key_args(self) -> list[str]:
|
||||
def mp4decrypt_key_args(self, track_ids: object = ()) -> list[str]:
|
||||
return ["--key", f"{KID.hex}:{KEY}", "--key", f"{ZERO_KID}:{KEY}"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_mp4decrypt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""An mp4decrypt that copies its input to its output and records its argv."""
|
||||
argv_log = tmp_path / "argv.jsonl"
|
||||
script = tmp_path / "mp4decrypt"
|
||||
script.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import json, shutil, sys\n"
|
||||
f"open({str(argv_log)!r}, 'a').write(json.dumps(sys.argv[1:]) + '\\n')\n"
|
||||
"shutil.copyfile(sys.argv[-2], sys.argv[-1])\n"
|
||||
)
|
||||
script.chmod(0o755)
|
||||
monkeypatch.setattr(binaries, "Mp4decrypt", script)
|
||||
return argv_log
|
||||
|
||||
|
||||
def read_argv(argv_log: Path) -> list[list[str]]:
|
||||
return [json.loads(line) for line in argv_log.read_text().splitlines()]
|
||||
|
||||
|
||||
@@ -2821,6 +2821,7 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
|
||||
"""Every service configured on this server, or every installed one with ``?all=1``, and its load state."""
|
||||
from unshackle.core import services as services_module
|
||||
from unshackle.core.api.download_manager import TERMINAL_STATUSES, get_download_manager
|
||||
from unshackle.core.api.routes import SERVICE_DEFAULTS, service_catalogue # lazy: routes imports this module
|
||||
from unshackle.core.api.session_store import get_session_store
|
||||
from unshackle.core.service_repo import head
|
||||
|
||||
@@ -2830,11 +2831,13 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
|
||||
staged_commits: Dict[Any, Optional[str]] = {}
|
||||
|
||||
tags = Services.get_tags() if request.query.get("all") == "1" else configured_service_tags()
|
||||
catalogue = await asyncio.to_thread(service_catalogue, [tag for tag in tags if tag in services_module.MODULES])
|
||||
rows = []
|
||||
for tag in tags:
|
||||
error = errors.get(tag)
|
||||
staged = tag in services_module.PENDING
|
||||
row: Dict[str, Any] = {
|
||||
**catalogue.get(tag, SERVICE_DEFAULTS),
|
||||
"tag": tag,
|
||||
"state": "failed" if error else "staged" if staged else "loaded",
|
||||
"error": error,
|
||||
@@ -2843,9 +2846,6 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
|
||||
"staged_since": services_module.PENDING_SINCE.get(tag),
|
||||
"sessions": sessions.get(tag, 0),
|
||||
"jobs": jobs.get(tag, 0),
|
||||
"aliases": list(services_module.ALIASES.get(tag, ())),
|
||||
"geofence": [],
|
||||
"geoblock": [],
|
||||
}
|
||||
if staged:
|
||||
# Only staged tags need a git call; the working tree already holds the new commit.
|
||||
@@ -2853,10 +2853,8 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
|
||||
if source is not None and source not in staged_commits:
|
||||
staged_commits[source] = head(source) if (source / ".git").exists() else None
|
||||
row["staged_commit"] = staged_commits.get(source)
|
||||
module = services_module.MODULES.get(tag)
|
||||
if module is not None:
|
||||
row["geofence"] = list(getattr(module, "GEOFENCE", ()) or ())
|
||||
row["geoblock"] = list(getattr(module, "GEOBLOCK", ()) or ())
|
||||
if (accounts := server_account_regions(tag)) is not None:
|
||||
row["server_accounts"] = accounts
|
||||
rows.append(row)
|
||||
return web.json_response(rows)
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from unshackle.core.api.handlers import (
|
||||
download_history_handler,
|
||||
download_job_events_handler,
|
||||
download_job_input_handler,
|
||||
enum_name,
|
||||
env_check_handler,
|
||||
get_allowed_services,
|
||||
get_download_job_handler,
|
||||
@@ -296,6 +297,25 @@ SERVICE_DEFAULTS: dict[str, Any] = {
|
||||
SERVICE_DETAILS: dict[str, tuple[Any, dict[str, Any]]] = {}
|
||||
|
||||
|
||||
def json_value(value: Any) -> Any:
|
||||
"""A click default or choice as JSON: a named object such as an enum member by its name, else text."""
|
||||
if value is None or (isinstance(value, (str, int, float, bool)) and not hasattr(value, "name")):
|
||||
return value
|
||||
return enum_name(value)
|
||||
|
||||
|
||||
def service_catalogue(tags: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""The catalogue entry of each tag, or the defaults for one that fails; slow on the first call after a reload."""
|
||||
entries: dict[str, dict[str, Any]] = {}
|
||||
for tag in tags:
|
||||
try:
|
||||
entries[tag] = service_details(tag)
|
||||
except Exception as e:
|
||||
log.warning(f"Could not load details for service {tag}: {e}")
|
||||
entries[tag] = {**SERVICE_DEFAULTS, "tag": tag}
|
||||
return entries
|
||||
|
||||
|
||||
def service_details(tag: str) -> dict[str, Any]:
|
||||
"""The catalogue entry of one service, built again only when its service module reloads."""
|
||||
service_module = Services.load(tag)
|
||||
@@ -304,11 +324,8 @@ def service_details(tag: str) -> dict[str, Any]:
|
||||
return hit[1]
|
||||
service_data: dict[str, Any] = {**SERVICE_DEFAULTS, "tag": tag}
|
||||
|
||||
if hasattr(service_module, "ALIASES"):
|
||||
service_data["aliases"] = list(service_module.ALIASES)
|
||||
|
||||
if hasattr(service_module, "GEOFENCE"):
|
||||
service_data["geofence"] = list(service_module.GEOFENCE)
|
||||
service_data["aliases"] = list(getattr(service_module, "ALIASES", ()) or ())
|
||||
service_data["geofence"] = list(getattr(service_module, "GEOFENCE", ()) or ())
|
||||
service_data["geoblock"] = list(getattr(service_module, "GEOBLOCK", ()) or ())
|
||||
|
||||
if hasattr(service_module, "TITLE_RE"):
|
||||
@@ -342,19 +359,17 @@ def service_details(tag: str) -> dict[str, Any]:
|
||||
param_info["opts"] = list(param.opts) if hasattr(param, "opts") else []
|
||||
param_info["is_flag"] = getattr(param, "is_flag", False)
|
||||
default = param.default
|
||||
if default is None:
|
||||
pass
|
||||
elif callable(default) or type(default).__name__ == "Sentinel":
|
||||
if callable(default) or type(default).__name__ == "Sentinel":
|
||||
default = None
|
||||
elif hasattr(default, "name"):
|
||||
default = default.name
|
||||
elif not isinstance(default, (str, int, float, bool, list)):
|
||||
default = str(default)
|
||||
elif isinstance(default, list):
|
||||
default = [json_value(item) for item in default]
|
||||
else:
|
||||
default = json_value(default)
|
||||
param_info["default"] = default
|
||||
param_info["help"] = getattr(param, "help", None)
|
||||
param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type)
|
||||
if isinstance(param.type, click.Choice):
|
||||
param_info["choices"] = list(param.type.choices)
|
||||
param_info["choices"] = [json_value(choice) for choice in param.type.choices]
|
||||
param_info["multiple"] = getattr(param, "multiple", False)
|
||||
cli_params.append(param_info)
|
||||
service_data["cli_params"] = cli_params
|
||||
@@ -514,13 +529,10 @@ async def services(request: web.Request) -> web.Response:
|
||||
remote_info = await asyncio.to_thread(remote_services_info, remote, allowed)
|
||||
return web.json_response({"services": remote_info, "load_errors": []})
|
||||
services_info = []
|
||||
catalogue = await asyncio.to_thread(service_catalogue, service_tags)
|
||||
|
||||
for tag in service_tags:
|
||||
try:
|
||||
service_data: dict[str, Any] = {**service_details(tag)}
|
||||
except Exception as e:
|
||||
log.warning(f"Could not load details for service {tag}: {e}")
|
||||
service_data = {**SERVICE_DEFAULTS, "tag": tag}
|
||||
service_data: dict[str, Any] = {**catalogue[tag]}
|
||||
if tag in services_module.PENDING:
|
||||
service_data["pending_update"] = True
|
||||
if server_accounts_allowed(request, tag) and (server_accounts := server_account_regions(tag)) is not None:
|
||||
@@ -3102,7 +3114,7 @@ async def dashboard_keys(request: web.Request) -> web.Response:
|
||||
@api_handler
|
||||
async def dashboard_services(request: web.Request) -> web.Response:
|
||||
"""
|
||||
Dashboard: every configured service and its load state.
|
||||
Dashboard: every configured service, its load state and its catalogue entry.
|
||||
---
|
||||
summary: Dashboard services
|
||||
description: >
|
||||
@@ -3169,6 +3181,24 @@ async def dashboard_services(request: web.Request) -> web.Response:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
server_accounts:
|
||||
type: object
|
||||
description: >
|
||||
Regions the server's own accounts cover. Only when `serve.server_accounts` names
|
||||
the service. No API key grant applies here, unlike `/api/services`.
|
||||
properties:
|
||||
regions:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
global:
|
||||
type: boolean
|
||||
additionalProperties:
|
||||
description: >
|
||||
The `/api/services` catalogue fields (help, url, title_regex, cli_params,
|
||||
needs_auth, auth_methods, has_search, anonymous_search, has_drm, client_config).
|
||||
A service with no loaded module, or whose catalogue entry fails, gets only
|
||||
help, url and title_regex, all null.
|
||||
'401':
|
||||
description: Dashboard key missing or invalid
|
||||
"""
|
||||
|
||||
@@ -16,6 +16,8 @@ from rich.text import Text
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka, read_top_level_box
|
||||
from unshackle.core.utilities import log_event
|
||||
from unshackle.core.utils import run_proxy
|
||||
|
||||
@@ -209,7 +211,7 @@ class ClearKeyCENC:
|
||||
|
||||
decrypt_start = time.monotonic()
|
||||
if decrypter == "mp4decrypt":
|
||||
self.decrypt_with_mp4decrypt(path)
|
||||
mp4decrypt_or_shaka(self, path)
|
||||
else:
|
||||
self.decrypt_with_shaka_packager(path)
|
||||
|
||||
@@ -224,12 +226,12 @@ class ClearKeyCENC:
|
||||
output_size=path.stat().st_size if path.exists() else 0,
|
||||
)
|
||||
|
||||
def mp4decrypt_key_args(self) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments for every content key."""
|
||||
def mp4decrypt_key_args(self, track_kids: Optional[dict[int, UUID]] = None) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments for every content key, and by track ID for each ``track_kids`` KID."""
|
||||
key_args = []
|
||||
for kid, key in self.content_keys.items():
|
||||
key_args.extend(["--key", f"{kid.hex}:{key}"])
|
||||
return key_args
|
||||
return key_args + drm_key_args.track_key_args(self.content_keys, track_kids)
|
||||
|
||||
def decrypt_with_mp4decrypt(self, path: Path) -> None:
|
||||
"""Decrypt using mp4decrypt"""
|
||||
@@ -238,7 +240,7 @@ class ClearKeyCENC:
|
||||
|
||||
output_path = path.with_stem(f"{path.stem}_decrypted")
|
||||
|
||||
key_args = self.mp4decrypt_key_args()
|
||||
key_args = self.mp4decrypt_key_args(kidless_track_kids(read_top_level_box(path, b"moov") or b""))
|
||||
|
||||
cmd = [
|
||||
str(binaries.Mp4decrypt),
|
||||
@@ -301,7 +303,6 @@ class ClearKeyCENC:
|
||||
)
|
||||
|
||||
stream_skipped = False
|
||||
had_error = False
|
||||
|
||||
shaka_log_buffer = ""
|
||||
assert p.stderr is not None
|
||||
@@ -312,27 +313,19 @@ class ClearKeyCENC:
|
||||
if "Skip stream" in line:
|
||||
# file/segment was so small that it didn't have any actual data, ignore
|
||||
stream_skipped = True
|
||||
if ":INFO:" in line:
|
||||
continue
|
||||
if "I0" in line or "W0" in line:
|
||||
continue
|
||||
if ":ERROR:" in line:
|
||||
had_error = True
|
||||
if "Insufficient bits in bitstream for given AVC profile" in line:
|
||||
# this is a warning and is something we don't have to worry about
|
||||
if drm_key_args.shaka_noise(line):
|
||||
continue
|
||||
shaka_log_buffer += f"{line.strip()}\n"
|
||||
|
||||
if shaka_log_buffer:
|
||||
# wrap to console width - padding - '[ClearKey]: '
|
||||
shaka_log_buffer = "\n ".join(
|
||||
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
|
||||
)
|
||||
console.log(Text.from_ansi("\n[ClearKey]: " + shaka_log_buffer))
|
||||
|
||||
p.wait()
|
||||
|
||||
if p.returncode != 0 or had_error:
|
||||
if p.returncode != 0:
|
||||
if shaka_log_buffer:
|
||||
# wrap to console width - padding - '[ClearKey]: '
|
||||
shaka_log_buffer = "\n ".join(
|
||||
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
|
||||
)
|
||||
console.log(Text.from_ansi("\n[ClearKey]: " + shaka_log_buffer))
|
||||
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
|
||||
|
||||
path.unlink()
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
"""Decrypter key arguments shared by the Widevine and PlayReady DRM systems."""
|
||||
"""Decrypter key arguments and output filtering shared by the CENC DRM systems."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable, Optional
|
||||
import re
|
||||
from typing import Any, Iterable, Mapping, Optional
|
||||
|
||||
ZERO_KID = "00" * 16
|
||||
SHAKA_NOISE = re.compile(r"^(?:[IWV]\d{4} |\[[^\]]*:(?:INFO|WARNING|VERBOSE\d*):)")
|
||||
|
||||
|
||||
def _hex(value: Any) -> str:
|
||||
@@ -42,15 +44,27 @@ def zero_kid_key(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> Optio
|
||||
return None
|
||||
|
||||
|
||||
def mp4decrypt_key_args(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments: every content key, then the zero-KID fallback."""
|
||||
def track_key_args(content_keys: dict[Any, Any], track_kids: Optional[Mapping[int, Any]]) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments by track ID for each track whose own KID has a content key."""
|
||||
keys = {_hex(kid): _hex(key) for kid, key in content_keys.items()}
|
||||
key_args: list[str] = []
|
||||
for track_id, kid in (track_kids or {}).items():
|
||||
if _hex(kid) in keys:
|
||||
key_args.extend(["--key", f"{track_id}:{keys[_hex(kid)]}"])
|
||||
return key_args
|
||||
|
||||
|
||||
def mp4decrypt_key_args(
|
||||
content_keys: dict[Any, Any], own_kids: Iterable[Any], track_kids: Optional[Mapping[int, Any]] = None
|
||||
) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments: every content key, the zero-KID fallback and the ``track_kids`` keys."""
|
||||
key_args: list[str] = []
|
||||
for kid, key in content_keys.items():
|
||||
key_args.extend(["--key", f"{_hex(kid)}:{_hex(key)}"])
|
||||
fallback = zero_kid_key(content_keys, own_kids)
|
||||
if fallback:
|
||||
key_args.extend(["--key", f"{ZERO_KID}:{fallback}"])
|
||||
return key_args
|
||||
return key_args + track_key_args(content_keys, track_kids)
|
||||
|
||||
|
||||
def shaka_keys(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> str:
|
||||
@@ -60,3 +74,8 @@ def shaka_keys(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> str:
|
||||
if fallback:
|
||||
entries.append(f"label={len(entries)}:key_id={ZERO_KID}:key={fallback}")
|
||||
return ",".join(entries)
|
||||
|
||||
|
||||
def shaka_noise(line: str) -> bool:
|
||||
"""True for a shaka-packager log line below ERROR, in its glog (I1103 ...) or bracket ([1103/...:WARNING:...]) form."""
|
||||
return bool(SHAKA_NOISE.match(line))
|
||||
|
||||
@@ -25,6 +25,7 @@ from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka, read_top_level_box
|
||||
from unshackle.core.session import AnySession, utf8_text
|
||||
from unshackle.core.utilities import call_licence, get_boxes, log_event
|
||||
from unshackle.core.utils import run_proxy
|
||||
@@ -571,7 +572,7 @@ class PlayReady:
|
||||
|
||||
decrypt_start = time.monotonic()
|
||||
if decrypter == "mp4decrypt":
|
||||
self.decrypt_with_mp4decrypt(path)
|
||||
mp4decrypt_or_shaka(self, path)
|
||||
else:
|
||||
self.decrypt_with_shaka_packager(path)
|
||||
|
||||
@@ -596,9 +597,9 @@ class PlayReady:
|
||||
return [self._kid]
|
||||
return self.kids
|
||||
|
||||
def mp4decrypt_key_args(self) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments for every content key and the zero-KID fallback."""
|
||||
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids())
|
||||
def mp4decrypt_key_args(self, track_kids: Optional[dict[int, UUID]] = None) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments for every content key, the zero-KID and the track ID fallbacks."""
|
||||
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids(), track_kids)
|
||||
|
||||
def decrypt_with_mp4decrypt(self, path: Path) -> None:
|
||||
"""Decrypt using mp4decrypt"""
|
||||
@@ -607,7 +608,7 @@ class PlayReady:
|
||||
|
||||
output_path = path.with_stem(f"{path.stem}_decrypted")
|
||||
|
||||
key_args = self.mp4decrypt_key_args()
|
||||
key_args = self.mp4decrypt_key_args(kidless_track_kids(read_top_level_box(path, b"moov") or b""))
|
||||
|
||||
cmd = [
|
||||
str(binaries.Mp4decrypt),
|
||||
@@ -667,7 +668,6 @@ class PlayReady:
|
||||
)
|
||||
|
||||
stream_skipped = False
|
||||
had_error = False
|
||||
shaka_log_buffer = ""
|
||||
assert p.stderr is not None
|
||||
for line in iter(p.stderr.readline, ""):
|
||||
@@ -676,25 +676,18 @@ class PlayReady:
|
||||
continue
|
||||
if "Skip stream" in line:
|
||||
stream_skipped = True
|
||||
if ":INFO:" in line:
|
||||
continue
|
||||
if "I0" in line or "W0" in line:
|
||||
continue
|
||||
if ":ERROR:" in line:
|
||||
had_error = True
|
||||
if "Insufficient bits in bitstream for given AVC profile" in line:
|
||||
if drm_key_args.shaka_noise(line):
|
||||
continue
|
||||
shaka_log_buffer += f"{line.strip()}\n"
|
||||
|
||||
if shaka_log_buffer:
|
||||
shaka_log_buffer = "\n ".join(
|
||||
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
|
||||
)
|
||||
console.log(Text.from_ansi("\n[PlayReady]: " + shaka_log_buffer))
|
||||
|
||||
p.wait()
|
||||
|
||||
if p.returncode != 0 or had_error:
|
||||
if p.returncode != 0:
|
||||
if shaka_log_buffer:
|
||||
shaka_log_buffer = "\n ".join(
|
||||
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
|
||||
)
|
||||
console.log(Text.from_ansi("\n[PlayReady]: " + shaka_log_buffer))
|
||||
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
|
||||
|
||||
path.unlink()
|
||||
|
||||
@@ -12,6 +12,7 @@ from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.drm.clearkey_cenc import ClearKeyCENC
|
||||
from unshackle.core.drm.playready import PlayReady
|
||||
from unshackle.core.drm.verify import kidless_track_kids
|
||||
from unshackle.core.drm.widevine import Widevine
|
||||
|
||||
SEGMENT_DRM_T = Union[Widevine, PlayReady, ClearKeyCENC]
|
||||
@@ -19,13 +20,17 @@ SEGMENT_DRM_T = Union[Widevine, PlayReady, ClearKeyCENC]
|
||||
MAX_WORKERS = 16
|
||||
|
||||
|
||||
def can_use(drm: Any, decryption: str) -> bool:
|
||||
"""Whether unshackle can decrypt a track's segments one by one as they download."""
|
||||
def can_use(drm: Any, decryption: str, init: Optional[bytes] = None) -> bool:
|
||||
"""Whether unshackle can decrypt a track's segments one by one as they download.
|
||||
|
||||
A track whose ``init`` mp4decrypt cannot key goes to the whole-file decrypt, which has the shaka fallback.
|
||||
"""
|
||||
return (
|
||||
isinstance(drm, (Widevine, PlayReady, ClearKeyCENC))
|
||||
and str(decryption).lower() == "mp4decrypt"
|
||||
and config.decrypt_segments
|
||||
and bool(binaries.Mp4decrypt)
|
||||
and not (init and set(kidless_track_kids(init).values()) - set(drm.content_keys))
|
||||
)
|
||||
|
||||
|
||||
@@ -44,7 +49,7 @@ class SegmentDecrypter:
|
||||
raise EnvironmentError("mp4decrypt executable not found but is required.")
|
||||
|
||||
self.drm = drm
|
||||
self.key_args = drm.mp4decrypt_key_args()
|
||||
self.key_args = drm.mp4decrypt_key_args(kidless_track_kids(init_data))
|
||||
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.work_dir = Path(tempfile.mkdtemp(prefix="segdec-", dir=work_dir))
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Callable, Iterator, NamedTuple, Optional
|
||||
from uuid import UUID
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DECRYPT_HOOK: Optional[Callable[..., None]] = None
|
||||
|
||||
CONTAINERS = {b"moov", b"trak", b"mdia", b"minf", b"stbl", b"mvex", b"moof", b"traf", b"sinf", b"schi"}
|
||||
@@ -142,6 +145,94 @@ def has_encrypted_entry(moov: bytes) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]:
|
||||
"""Read a single top-level box from an MP4 file.
|
||||
|
||||
This function walks only the box headers, so the wanted box is the only payload it reads into memory.
|
||||
Returns None if the file has no such top-level box, or if a box header is truncated or declares an
|
||||
impossible size.
|
||||
"""
|
||||
file_size = path.stat().st_size
|
||||
with path.open("rb") as f:
|
||||
start = 0
|
||||
while start + 8 <= file_size:
|
||||
header = f.read(8)
|
||||
if len(header) < 8:
|
||||
return None
|
||||
size = int.from_bytes(header[:4], "big")
|
||||
if size == 1:
|
||||
large = f.read(8)
|
||||
if len(large) < 8:
|
||||
return None
|
||||
size = int.from_bytes(large, "big")
|
||||
elif size == 0:
|
||||
size = file_size - start
|
||||
if size < 8:
|
||||
return None
|
||||
if header[4:8] == box_type:
|
||||
f.seek(start)
|
||||
return f.read(size)
|
||||
start += size
|
||||
f.seek(start)
|
||||
return None
|
||||
|
||||
|
||||
def has_encrypted_sample_entry(path: Path) -> bool:
|
||||
"""True if the MP4's moov still carries an encrypted sample entry (``enc*`` or ``drm*``).
|
||||
|
||||
A faithful decrypt rewrites the entry back to the real codec 4CC through frma, so a
|
||||
survivor means the decrypt tool skipped/failed the track. Only the moov is read, and its
|
||||
stsd boxes are parsed, so byte runs in mdat or in metadata cannot match. Any read or
|
||||
parse error -> False.
|
||||
"""
|
||||
try:
|
||||
moov = read_top_level_box(path, b"moov")
|
||||
return moov is not None and has_encrypted_entry(moov)
|
||||
except (OSError, struct.error, RecursionError):
|
||||
return False
|
||||
|
||||
|
||||
def kidless_kid(data: bytes, kind: bytes, entry: int, end: int) -> Optional[UUID]:
|
||||
"""Return the KID of an encrypted sample entry whose first sinf holds no CENC tenc (a PIFF sinf, say)."""
|
||||
for fields in ENTRY_FIELDS.get(kind, ()):
|
||||
sinfs = [(body, box_end) for box, body, box_end in children(data, entry + fields, end) if box == b"sinf"]
|
||||
if sinfs:
|
||||
if any(box == b"tenc" for box, _, _ in walk(data, *sinfs[0])):
|
||||
return None
|
||||
return entry_kid(data, kind, entry, end)
|
||||
return None
|
||||
|
||||
|
||||
def kidless_track_kids(data: bytes) -> dict[int, UUID]:
|
||||
"""Map each track of a moov or init segment whose first sinf has no tenc for mp4decrypt to read to its KID."""
|
||||
kids: dict[int, UUID] = {}
|
||||
track = 0
|
||||
try:
|
||||
for kind, body, end in walk(data):
|
||||
if kind == b"tkhd":
|
||||
track = struct.unpack_from(">I", data, body + (20 if data[body] == 1 else 12))[0]
|
||||
elif kind == b"stsd" and track:
|
||||
for entry in children(data, body + 8, end):
|
||||
kid = kidless_kid(data, *entry)
|
||||
if kid:
|
||||
kids[track] = kid
|
||||
except (struct.error, IndexError, ValueError, RecursionError):
|
||||
return {}
|
||||
return kids
|
||||
|
||||
|
||||
def mp4decrypt_or_shaka(drm: Any, path: Path) -> None:
|
||||
"""Decrypt ``path`` with mp4decrypt, and again with shaka-packager when mp4decrypt copied it through.
|
||||
|
||||
mp4decrypt copies a track it has no content key for unchanged, so its first moof stays the same.
|
||||
"""
|
||||
first_moof = read_top_level_box(path, b"moof")
|
||||
drm.decrypt_with_mp4decrypt(path)
|
||||
if has_encrypted_sample_entry(path) and read_top_level_box(path, b"moof") == first_moof:
|
||||
log.warning(f"mp4decrypt left {path.name} encrypted, decrypting it with shaka-packager")
|
||||
drm.decrypt_with_shaka_packager(path)
|
||||
|
||||
|
||||
def senc_protects_samples(buf: bytes, body: int, end: int) -> bool:
|
||||
"""True if a senc/PIFF-uuid box describes protected samples.
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka, read_top_level_box
|
||||
from unshackle.core.session import AnySession, utf8_text
|
||||
from unshackle.core.utilities import call_licence, get_boxes, log_event
|
||||
from unshackle.core.utils import run_proxy
|
||||
@@ -300,7 +301,7 @@ class Widevine:
|
||||
|
||||
decrypt_start = time.monotonic()
|
||||
if decrypter == "mp4decrypt":
|
||||
self.decrypt_with_mp4decrypt(path)
|
||||
mp4decrypt_or_shaka(self, path)
|
||||
else:
|
||||
self.decrypt_with_shaka_packager(path)
|
||||
|
||||
@@ -325,9 +326,9 @@ class Widevine:
|
||||
return [self._kid]
|
||||
return self.kids
|
||||
|
||||
def mp4decrypt_key_args(self) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments for every content key and the zero-KID fallback."""
|
||||
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids())
|
||||
def mp4decrypt_key_args(self, track_kids: Optional[dict[int, UUID]] = None) -> list[str]:
|
||||
"""Build the mp4decrypt --key arguments for every content key, the zero-KID and the track ID fallbacks."""
|
||||
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids(), track_kids)
|
||||
|
||||
def decrypt_with_mp4decrypt(self, path: Path) -> None:
|
||||
"""Decrypt using mp4decrypt"""
|
||||
@@ -336,7 +337,7 @@ class Widevine:
|
||||
|
||||
output_path = path.with_stem(f"{path.stem}_decrypted")
|
||||
|
||||
key_args = self.mp4decrypt_key_args()
|
||||
key_args = self.mp4decrypt_key_args(kidless_track_kids(read_top_level_box(path, b"moov") or b""))
|
||||
|
||||
cmd = [
|
||||
str(binaries.Mp4decrypt),
|
||||
@@ -396,7 +397,6 @@ class Widevine:
|
||||
)
|
||||
|
||||
stream_skipped = False
|
||||
had_error = False
|
||||
|
||||
shaka_log_buffer = ""
|
||||
assert p.stderr is not None
|
||||
@@ -407,26 +407,19 @@ class Widevine:
|
||||
if "Skip stream" in line:
|
||||
# file/segment was so small that it didn't have any actual data, ignore
|
||||
stream_skipped = True
|
||||
if ":INFO:" in line:
|
||||
continue
|
||||
if "I0" in line or "W0" in line:
|
||||
continue
|
||||
if ":ERROR:" in line:
|
||||
had_error = True
|
||||
if "Insufficient bits in bitstream for given AVC profile" in line:
|
||||
if drm_key_args.shaka_noise(line):
|
||||
continue
|
||||
shaka_log_buffer += f"{line.strip()}\n"
|
||||
|
||||
if shaka_log_buffer:
|
||||
# wrap to console width - padding - '[Widevine]: '
|
||||
shaka_log_buffer = "\n ".join(
|
||||
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
|
||||
)
|
||||
console.log(Text.from_ansi("\n[Widevine]: " + shaka_log_buffer))
|
||||
|
||||
p.wait()
|
||||
|
||||
if p.returncode != 0 or had_error:
|
||||
if p.returncode != 0:
|
||||
if shaka_log_buffer:
|
||||
# wrap to console width - padding - '[Widevine]: '
|
||||
shaka_log_buffer = "\n ".join(
|
||||
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
|
||||
)
|
||||
console.log(Text.from_ansi("\n[Widevine]: " + shaka_log_buffer))
|
||||
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
|
||||
|
||||
path.unlink()
|
||||
|
||||
@@ -631,7 +631,7 @@ class DASH:
|
||||
collapse_span = None
|
||||
collapse_single_url = collapse_span is not None
|
||||
|
||||
use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption)
|
||||
use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption, init_data)
|
||||
# subtitle segments are tiny, the post-download merge costs nothing there
|
||||
use_rolling_merge = not collapse_single_url and config.merge_segments and not isinstance(track, Subtitle)
|
||||
|
||||
|
||||
@@ -867,6 +867,7 @@ class HLS:
|
||||
init_section = wanted_segments[0].init_section
|
||||
init_content, _ = HLS.fetch_init_section(session, init_section, 0)
|
||||
map_data = (init_section, init_content)
|
||||
use_segment_decrypt = use_segment_decrypt and can_use(session_drm, config.decryption, init_content)
|
||||
if use_segment_decrypt:
|
||||
assert isinstance(session_drm, (Widevine, PlayReady, ClearKeyCENC))
|
||||
segment_decrypter = SegmentDecrypter(session_drm, init_content, save_dir.parent, max_workers)
|
||||
|
||||
@@ -28,7 +28,12 @@ from unshackle.core.config import config
|
||||
from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, DownloadCancelled
|
||||
from unshackle.core.downloaders.requests import requests
|
||||
from unshackle.core.drm import DRM_T, BlowfishStripe, ClearKeyCENC, PlayReady, Widevine
|
||||
from unshackle.core.drm.verify import decrypt_track, has_encrypted_entry, senc_protects_samples
|
||||
from unshackle.core.drm.verify import (
|
||||
decrypt_track,
|
||||
has_encrypted_sample_entry,
|
||||
read_top_level_box,
|
||||
senc_protects_samples,
|
||||
)
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.session import MAX_BACKOFF, AnySession, RnetSession, utf8_text
|
||||
from unshackle.core.temp import restore_segments, stash_segments
|
||||
@@ -85,38 +90,6 @@ def direct_session(session: AnySession, proxy: Optional[str] = None) -> Session:
|
||||
return new
|
||||
|
||||
|
||||
def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]:
|
||||
"""Read a single top-level box from an MP4 file.
|
||||
|
||||
This function walks only the box headers, so the wanted box is the only payload it reads into memory.
|
||||
Returns None if the file has no such top-level box, or if a box header is truncated or declares an
|
||||
impossible size.
|
||||
"""
|
||||
file_size = path.stat().st_size
|
||||
with path.open("rb") as f:
|
||||
start = 0
|
||||
while start + 8 <= file_size:
|
||||
header = f.read(8)
|
||||
if len(header) < 8:
|
||||
return None
|
||||
size = int.from_bytes(header[:4], "big")
|
||||
if size == 1:
|
||||
large = f.read(8)
|
||||
if len(large) < 8:
|
||||
return None
|
||||
size = int.from_bytes(large, "big")
|
||||
elif size == 0:
|
||||
size = file_size - start
|
||||
if size < 8:
|
||||
return None
|
||||
if header[4:8] == box_type:
|
||||
f.seek(start)
|
||||
return f.read(size)
|
||||
start += size
|
||||
f.seek(start)
|
||||
return None
|
||||
|
||||
|
||||
def iter_top_level_boxes(path: Path) -> Iterable[tuple[bytes, int, int]]:
|
||||
"""Yield (box_type, offset, size) for each top-level box, stopping at the first bad header."""
|
||||
file_size = path.stat().st_size
|
||||
@@ -238,21 +211,6 @@ def has_mp4_only_audio_entry(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def has_encrypted_sample_entry(path: Path) -> bool:
|
||||
"""True if the MP4's moov still carries an encrypted sample entry (``enc*`` or ``drm*``).
|
||||
|
||||
A faithful decrypt rewrites the entry back to the real codec 4CC through frma, so a
|
||||
survivor means the decrypt tool skipped/failed the track. Only the moov is read, and its
|
||||
stsd boxes are parsed, so byte runs in mdat or in metadata cannot match. Any read or
|
||||
parse error -> False.
|
||||
"""
|
||||
try:
|
||||
moov = read_top_level_box(path, b"moov")
|
||||
return moov is not None and has_encrypted_entry(moov)
|
||||
except (OSError, struct.error, RecursionError):
|
||||
return False
|
||||
|
||||
|
||||
def moof_still_encrypted(moof: bytes) -> bool:
|
||||
# senc only: a decrypter detaches just the atom it consumed, so a PIFF uuid can
|
||||
# outlive a good decrypt. ISM arrives as senc via piff_senc_to_cenc.
|
||||
|
||||
Reference in New Issue
Block a user