Compare commits

...
2 Commits
Author SHA1 Message Date
imSp4rky b3d586a590 feat(api): carry the service catalogue on /api/dashboard/services
Rows now include the /api/services fields (help, cli_params, auth_methods, ...) and server_accounts. Both routes build the catalogue off the event loop, and an enum choice or a non-JSON list default no longer fails the whole route.
2026-11-03 18:27:49 -07:00
imSp4rky 893c81f428 fix(drm): key PIFF-first tracks by track ID and trust shaka's exit code
mp4decrypt copied a track whose first sinf has no tenc through encrypted; pass its key by track ID and retry with shaka on a passthrough. Shaka fails only on a non-zero exit now.
2026-11-03 17:05:26 -07:00
18 changed files with 577 additions and 168 deletions
+14 -2
View File
@@ -53,7 +53,7 @@ All are `GET`, except the two `drain` routes.
| `/api/dashboard/cdm-logs` | `{"seq": N, "records": [...], "sessions": [...]}`: the last 5000 calls to the Widevine and PlayReady CDM device routes, and the open sessions of each device. `?since=<seq>` returns only newer records. See [CDM calls](#cdm-calls). |
| `/api/dashboard/sessions/{id}/logs` | One remote session's service log: `{"session_id", "last_seq", "records": [...]}`. Each record has `seq`, `ts`, `level`, `message`. `?since=<seq>` returns only newer records. 404 when the remote session is unknown. |
| `/api/dashboard/keys` | Every configured API key: what it may do and what it has done. One row per key in `serve.users`, plus `serve.api_secret` when it is set. |
| `/api/dashboard/services` | Every service configured on this server and its load state, including the ones that failed to import. See [Services](#services) for what configured means. |
| `/api/dashboard/services` | Every service configured on this server, its load state and its catalogue entry, including the services that failed to import. See [Services](#services) for what configured means. |
| `/api/dashboard/health` | Preflight: whether this instance could finish a download. Cached 30 seconds. |
| `/api/dashboard/events` | Server-Sent Events stream. With `?since=<seq>` it returns `{"seq", "stats", "events": [...]}` instead: every event after `seq` (the server keeps the last 20000, log records included, so a poll every few seconds is safe) as one JSON burst, so a UI that polls every few seconds still sees a remote session that opened and closed in between. |
@@ -238,7 +238,10 @@ GET /api/dashboard/services
[{"tag": "EXAMPLE", "state": "staged", "error": null,
"commit": "9cc230c98f13b83f6c6fe831595b9702d3ba2786", "staged_commit": "5f0aa17...",
"staged_since": 1756908900.0, "sessions": 1, "jobs": 1,
"aliases": ["EXMPL"], "geofence": ["US"], "geoblock": []}]
"aliases": ["EXMPL"], "geofence": ["US"], "geoblock": [],
"help": "...", "url": "https://www.service.example", "title_regex": "...", "cli_params": [...],
"needs_auth": true, "auth_methods": ["cookies"], "has_search": true, "anonymous_search": false,
"has_drm": true, "server_accounts": {"regions": ["gb", "us"], "global": false}}]
```
The list holds only the services configured on this server:
@@ -269,6 +272,15 @@ for a service from a plain local directory rather than a git repo.
Watch the `service` event rather than polling for the swap.
Each row also carries the service's catalogue entry: the fields that
[`GET /api/services`](endpoints.md#get-apiservices) sends, with the same meaning. A service with no loaded
module, or whose catalogue entry fails, gets only `help`, `url` and `title_regex`, all `null`.
A service whose re-import failed keeps the entry of the module that still runs.
`server_accounts` shows when `serve.server_accounts` names the service. No API key grant
applies to it here: `/api/services` sends it only to an API key with the `server_accounts`
grant, but the dashboard key sees it for every service.
## Health
```
+8 -1
View File
@@ -187,6 +187,12 @@ service tag case-insensitively and uses the `default` entry as the fallback.
unshackle decrypts HLS AES-128 ClearKey in-process and ignores this setting.
mp4decrypt finds a content key by the KID in the first `sinf` box of the track. When that box
has no CENC `tenc` (for example, a PIFF `sinf` before the CENC one), unshackle also gives
mp4decrypt the content key of that KID by track ID. When mp4decrypt copies a track through
still encrypted, unshackle decrypts it again with shaka-packager and logs a warning. This
retry needs shaka-packager installed.
=== "Global"
```yaml
@@ -213,7 +219,8 @@ This works with `mp4decrypt` only. Set [`decryption`](#decryption) to `mp4decryp
service (or as the default); with `shaka` the option has no effect, because shaka-packager
cannot decrypt a segment without its init segment. It applies to DASH tracks and to HLS
tracks that use one init segment, one `EXT-X-KEY` and no `EXT-X-DISCONTINUITY`. Every other
track keeps the whole-file decryption.
track keeps the whole-file decryption, and so does a track whose init segment mp4decrypt
cannot key by KID when no content key has that KID.
!!! note "No resume"
+16
View File
@@ -91,3 +91,19 @@ def mux_args(monkeypatch: pytest.MonkeyPatch) -> Callable[..., list[str]]:
return captured[0]
return run
@pytest.fixture
def stub_mp4decrypt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""An mp4decrypt that copies its input to its output and records its argv."""
argv_log = tmp_path / "argv.jsonl"
script = tmp_path / "mp4decrypt"
script.write_text(
"#!/usr/bin/env python3\n"
"import json, shutil, sys\n"
f"open({str(argv_log)!r}, 'a').write(json.dumps(sys.argv[1:]) + '\\n')\n"
"shutil.copyfile(sys.argv[-2], sys.argv[-1])\n"
)
script.chmod(0o755)
monkeypatch.setattr(binaries, "Mp4decrypt", script)
return argv_log
+6 -2
View File
@@ -19,10 +19,14 @@ def full(kind: bytes, payload: bytes, version: int = 0, flags: int = 0) -> bytes
return box(kind, bytes([version]) + flags.to_bytes(3, "big") + payload)
def encv(kid: UUID, piff: bool = False) -> bytes:
def sinf(kid: UUID, piff: bool = False) -> bytes:
tenc_body = b"\x00\x00\x01\x08" + kid.bytes
tenc = box(b"uuid", PIFF_TENC, bytes(4) + tenc_body) if piff else full(b"tenc", tenc_body)
return box(b"encv", bytes(78), box(b"sinf", box(b"frma", b"avc1"), box(b"schi", tenc)))
return box(b"sinf", box(b"frma", b"avc1"), box(b"schi", tenc))
def encv(kid: UUID, piff: bool = False) -> bytes:
return box(b"encv", bytes(78), sinf(kid, piff))
def enca(kid: UUID) -> bytes:
+240
View File
@@ -0,0 +1,240 @@
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
from typing import Any, Callable
from uuid import UUID
import pytest
from tests.core.test_kid_windows import A, B, box, clear_audio, encv, moof, movie, sinf
from tests.core.test_zero_kid_fallback import _playready, _widevine
from tests.test_segment_decrypt import read_argv
from unshackle.core import binaries
from unshackle.core.config import config
from unshackle.core.drm import ClearKeyCENC
from unshackle.core.drm.key_args import shaka_noise, track_key_args
from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka
KEY = "b5df2ae3f63b8dda9049d6e41422e075"
OTHER_KEY = "11" * 16
POSIX_STUBS = pytest.mark.skipif(sys.platform == "win32", reason="the decrypter stubs are shell scripts")
def two_sinf(kid: UUID, piff_first: bool) -> bytes:
"""A sample entry with a PIFF and a CENC sinf for the same KID."""
return box(b"encv", bytes(78), sinf(kid, piff_first), sinf(kid, not piff_first))
def track_zero(entry: bytes) -> bytes:
"""A movie whose tkhd carries track_ID 0."""
data = movie(entry)
tkhd = data.index(b"tkhd")
return data[: tkhd + 16] + bytes(4) + data[tkhd + 20 :]
@pytest.mark.parametrize(
("init", "kids"),
[
pytest.param(movie(two_sinf(A, piff_first=True)), {1: A}, id="piff-then-cenc"),
pytest.param(movie(encv(A, piff=True)), {1: A}, id="piff-only"),
pytest.param(movie(two_sinf(A, piff_first=False)), {}, id="cenc-then-piff"),
pytest.param(movie(encv(A)), {}, id="cenc-only"),
pytest.param(clear_audio(), {}, id="clear"),
pytest.param(track_zero(two_sinf(A, piff_first=True)), {}, id="track-id-0"),
pytest.param(b"\x00\x00\x00\x01moov\x00\x00", {}, id="truncated-large-size"),
pytest.param(movie(two_sinf(A, piff_first=True))[:-20], {}, id="cut-short"),
],
)
def test_kidless_track_kids(init: bytes, kids: dict[int, UUID]) -> None:
assert kidless_track_kids(init) == kids
def test_kidless_track_kids_survives_deep_nesting() -> None:
nested = b""
for _ in range(3000):
nested = box(b"trak", nested)
assert kidless_track_kids(nested) == {}
def test_track_key_needs_the_track_kid() -> None:
assert track_key_args({A: KEY, B: OTHER_KEY}, {2: A}) == ["--key", f"2:{KEY}"]
assert track_key_args({B: OTHER_KEY}, {2: A}) == []
DRMS: list[Any] = [
pytest.param(lambda: _widevine(A), id="widevine"),
pytest.param(lambda: _playready(A), id="playready"),
pytest.param(lambda: ClearKeyCENC(kids=[A]), id="clearkey"),
]
def write_movie(path: Path, entry: bytes, fragment: bytes = b"") -> Path:
path.write_bytes(movie(entry) + fragment + box(b"mdat", bytes(16)))
return path
@POSIX_STUBS
@pytest.mark.parametrize("make", DRMS)
def test_whole_file_decrypt_keys_a_piff_first_track_by_id(
tmp_path: Path, stub_mp4decrypt: Path, make: Callable[[], Any]
) -> None:
path = write_movie(tmp_path / "track.mp4", two_sinf(A, piff_first=True))
drm = make()
drm.content_keys[A] = KEY
drm.decrypt_with_mp4decrypt(path)
assert f"1:{KEY}" in read_argv(stub_mp4decrypt)[0]
@POSIX_STUBS
@pytest.mark.parametrize("make", DRMS)
def test_decrypt_falls_back_to_shaka_on_a_passthrough(
tmp_path: Path, stub_mp4decrypt: Path, monkeypatch: pytest.MonkeyPatch, make: Callable[[], Any]
) -> None:
path = write_movie(tmp_path / "track.mp4", two_sinf(A, piff_first=True), moof(1, 0))
drm = make()
drm.content_keys[B] = OTHER_KEY
shaka: list[Path] = []
monkeypatch.setattr(type(drm), "decrypt_with_shaka_packager", lambda self, p: shaka.append(p))
monkeypatch.setattr(config, "decryption", "mp4decrypt", raising=False)
drm.decrypt(path)
assert shaka == [path]
@POSIX_STUBS
def test_segment_decrypter_keys_from_the_init_segment(tmp_path: Path, stub_mp4decrypt: Path) -> None:
drm = _widevine(A)
drm.content_keys[A] = KEY
decrypter = SegmentDecrypter(drm, movie(two_sinf(A, piff_first=True)), tmp_path, workers=1)
decrypter.pool.shutdown()
assert f"1:{KEY}" in decrypter.key_args
@pytest.mark.parametrize(("key_kid", "usable"), [(A, True), (B, False)], ids=["track-kid-keyed", "other-kid-only"])
def test_segment_decrypt_needs_a_key_for_a_kidless_track(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, key_kid: UUID, usable: bool
) -> None:
monkeypatch.setattr(config, "decrypt_segments", True, raising=False)
monkeypatch.setattr(binaries, "Mp4decrypt", tmp_path / "mp4decrypt")
drm = _widevine(A)
drm.content_keys[key_kid] = KEY
assert can_use(drm, "mp4decrypt", movie(two_sinf(A, piff_first=True))) is usable
assert can_use(drm, "mp4decrypt", movie(encv(A))) is True
class StubDRM:
"""Records which decrypter ran; mp4decrypt writes ``mp4decrypt_output`` over the file."""
def __init__(self, mp4decrypt_output: bytes) -> None:
self.output = mp4decrypt_output
self.calls: list[str] = []
def decrypt_with_mp4decrypt(self, path: Path) -> None:
self.calls.append("mp4decrypt")
path.write_bytes(self.output)
def decrypt_with_shaka_packager(self, path: Path) -> None:
self.calls.append("shaka")
ENCRYPTED = movie(encv(A)) + moof(1, 0)
@pytest.mark.parametrize(
("output", "calls"),
[
pytest.param(ENCRYPTED, ["mp4decrypt", "shaka"], id="copied-through"),
pytest.param(movie(encv(A)) + moof(1, 1000), ["mp4decrypt"], id="stale-encrypted-moov"),
pytest.param(movie(box(b"avc1", bytes(78))) + moof(1, 1000), ["mp4decrypt"], id="decrypted"),
pytest.param(b"not an mp4", ["mp4decrypt"], id="unreadable"),
],
)
def test_mp4decrypt_or_shaka(tmp_path: Path, output: bytes, calls: list[str]) -> None:
drm = StubDRM(output)
path = tmp_path / "track.mp4"
path.write_bytes(ENCRYPTED)
mp4decrypt_or_shaka(drm, path)
assert drm.calls == calls
@pytest.fixture
def stub_packager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Callable[[int], None]:
"""A shaka-packager that writes its output, logs an ERROR line and exits with the code given."""
def install(code: int) -> None:
script = tmp_path / "packager"
script.write_text(
"#!/usr/bin/env python3\n"
"import sys\n"
"output = sys.argv[1].split('output=')[1].split(',')[0]\n"
"open(output, 'wb').write(b'decrypted')\n"
"sys.stderr.write('[1103/153702:ERROR:box_definitions.cc(641)] Not implemented reached\\n')\n"
f"sys.exit({code})\n"
)
script.chmod(0o755)
monkeypatch.setattr(binaries, "ShakaPackager", script)
return install
@POSIX_STUBS
@pytest.mark.parametrize("make", DRMS)
def test_shaka_error_line_with_exit_0_is_a_decrypt(
tmp_path: Path, stub_packager: Callable[[int], None], make: Callable[[], Any]
) -> None:
stub_packager(0)
path = tmp_path / "track.mp4"
path.write_bytes(b"encrypted")
drm = make()
drm.content_keys[A] = KEY
drm.decrypt_with_shaka_packager(path)
assert path.read_bytes() == b"decrypted"
@POSIX_STUBS
@pytest.mark.parametrize("make", DRMS)
def test_shaka_non_zero_exit_raises(
tmp_path: Path, stub_packager: Callable[[int], None], make: Callable[[], Any]
) -> None:
stub_packager(2)
path = tmp_path / "track.mp4"
path.write_bytes(b"encrypted")
drm = make()
drm.content_keys[A] = KEY
with pytest.raises(subprocess.CalledProcessError):
drm.decrypt_with_shaka_packager(path)
@pytest.mark.parametrize(
("line", "noise"),
[
pytest.param(
"I1103 21:41:56.557273 16212 packager_main.cc:591] Excluding 'pssh' boxes", True, id="glog-info"
),
pytest.param("I0503 01:02:03.000000 1 demuxer.cc:105] Demuxer::Run()", True, id="glog-info-early-month"),
pytest.param("W1103 21:41:56.000000 1 x.cc:1] warning", True, id="glog-warning"),
pytest.param(
"[1103/214121:WARNING:track_run_iterator.cc(699)] Seeing non-zero composition offset", True, id="warning"
),
pytest.param("[1103/153402:INFO:demuxer.cc(170)] Initialize Demuxer", True, id="info"),
pytest.param("[1103/153402:ERROR:box_definitions.cc(641)] Not implemented reached", False, id="error"),
pytest.param("E1103 21:41:56.000000 1 x.cc:1] failed", False, id="glog-error"),
pytest.param("Packaging completed successfully.", False, id="plain"),
],
)
def test_shaka_noise(line: str, noise: bool) -> None:
assert shaka_noise(line) is noise
+65
View File
@@ -682,3 +682,68 @@ async def test_health_probe_timeout_reports_what_finished(aiohttp_client, dashbo
# The timeout is cached for the full TTL, so callers cannot stack up more stalled threads.
assert (await client.get("/api/dashboard/health", headers={"X-Secret-Key": "dash-secret"})).status == 200
assert handlers._health_cache["payload"]["status"] == "failing"
async def test_dashboard_services_carry_the_catalogue(aiohttp_client, dashboard_cfg, monkeypatch) -> None:
import enum
from pathlib import Path
import click
from unshackle.core import services as services_module
from unshackle.core.api import routes
from unshackle.core.service import Service
class Mode(enum.Enum):
FAST = 1
class Loaded(Service):
"""How to use AAA."""
cli = click.Command(
"AAA",
params=[
click.Argument(["title"]),
click.Option(["--mode"], type=click.Choice(Mode)),
click.Option(["--dir"], multiple=True, default=[Path("/x")]),
],
short_help="https://aaa.example",
)
class NoGeofence(Service):
"""How to use DDD."""
GEOFENCE = None
real_details = routes.service_details
def details(tag: str) -> dict:
if tag == "CCC":
raise RuntimeError("bad module")
return real_details(tag)
tags = ("AAA", "BBB", "CCC", "DDD")
monkeypatch.setattr(services_module, "SERVICES", [Path(f"/svc/{t}/__init__.py") for t in tags])
monkeypatch.setattr(services_module, "MODULES", {"AAA": Loaded, "CCC": Loaded, "DDD": NoGeofence})
monkeypatch.setattr(services_module, "LOAD_ERRORS", ["BBB: failed to import - ImportError: boom"])
monkeypatch.setattr(services_module, "PENDING", {"DDD"})
monkeypatch.setattr(routes, "SERVICE_DETAILS", {})
monkeypatch.setattr(routes, "service_details", details)
monkeypatch.setitem(config.serve, "server_accounts", {"aaa": {"default": ["us", "gb"]}})
client = await aiohttp_client(make_app())
resp = await client.get("/api/dashboard/services", headers={"X-Secret-Key": "dash-secret"})
assert resp.status == 200
rows = {r["tag"]: r for r in await resp.json()}
assert rows["AAA"]["help"] == "How to use AAA." and rows["AAA"]["url"] == "https://aaa.example"
title, mode, folder = rows["AAA"]["cli_params"]
assert title == {"name": "title", "kind": "argument", "required": True}
assert mode["choices"] == ["FAST"] and folder["default"] == ["x"]
assert rows["AAA"]["state"] == "loaded"
assert rows["AAA"]["server_accounts"] == {"regions": ["gb", "us"], "global": False}
for tag in ("BBB", "CCC"):
assert rows[tag]["help"] is None and "cli_params" not in rows[tag] and "server_accounts" not in rows[tag]
assert rows["BBB"]["state"] == "failed"
assert rows["DDD"]["state"] == "staged" and rows["DDD"]["help"] == "How to use DDD."
assert rows["DDD"]["geofence"] == []
+1 -17
View File
@@ -23,26 +23,10 @@ class FakeDRM:
def __init__(self) -> None:
self.content_keys = {KID: KEY}
def mp4decrypt_key_args(self) -> list[str]:
def mp4decrypt_key_args(self, track_ids: object = ()) -> list[str]:
return ["--key", f"{KID.hex}:{KEY}", "--key", f"{ZERO_KID}:{KEY}"]
@pytest.fixture
def stub_mp4decrypt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""An mp4decrypt that copies its input to its output and records its argv."""
argv_log = tmp_path / "argv.jsonl"
script = tmp_path / "mp4decrypt"
script.write_text(
"#!/usr/bin/env python3\n"
"import json, shutil, sys\n"
f"open({str(argv_log)!r}, 'a').write(json.dumps(sys.argv[1:]) + '\\n')\n"
"shutil.copyfile(sys.argv[-2], sys.argv[-1])\n"
)
script.chmod(0o755)
monkeypatch.setattr(binaries, "Mp4decrypt", script)
return argv_log
def read_argv(argv_log: Path) -> list[list[str]]:
return [json.loads(line) for line in argv_log.read_text().splitlines()]
+5 -7
View File
@@ -2821,6 +2821,7 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
"""Every service configured on this server, or every installed one with ``?all=1``, and its load state."""
from unshackle.core import services as services_module
from unshackle.core.api.download_manager import TERMINAL_STATUSES, get_download_manager
from unshackle.core.api.routes import SERVICE_DEFAULTS, service_catalogue # lazy: routes imports this module
from unshackle.core.api.session_store import get_session_store
from unshackle.core.service_repo import head
@@ -2830,11 +2831,13 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
staged_commits: Dict[Any, Optional[str]] = {}
tags = Services.get_tags() if request.query.get("all") == "1" else configured_service_tags()
catalogue = await asyncio.to_thread(service_catalogue, [tag for tag in tags if tag in services_module.MODULES])
rows = []
for tag in tags:
error = errors.get(tag)
staged = tag in services_module.PENDING
row: Dict[str, Any] = {
**catalogue.get(tag, SERVICE_DEFAULTS),
"tag": tag,
"state": "failed" if error else "staged" if staged else "loaded",
"error": error,
@@ -2843,9 +2846,6 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
"staged_since": services_module.PENDING_SINCE.get(tag),
"sessions": sessions.get(tag, 0),
"jobs": jobs.get(tag, 0),
"aliases": list(services_module.ALIASES.get(tag, ())),
"geofence": [],
"geoblock": [],
}
if staged:
# Only staged tags need a git call; the working tree already holds the new commit.
@@ -2853,10 +2853,8 @@ async def dashboard_services_handler(request: web.Request) -> web.Response:
if source is not None and source not in staged_commits:
staged_commits[source] = head(source) if (source / ".git").exists() else None
row["staged_commit"] = staged_commits.get(source)
module = services_module.MODULES.get(tag)
if module is not None:
row["geofence"] = list(getattr(module, "GEOFENCE", ()) or ())
row["geoblock"] = list(getattr(module, "GEOBLOCK", ()) or ())
if (accounts := server_account_regions(tag)) is not None:
row["server_accounts"] = accounts
rows.append(row)
return web.json_response(rows)
+49 -19
View File
@@ -46,6 +46,7 @@ from unshackle.core.api.handlers import (
download_history_handler,
download_job_events_handler,
download_job_input_handler,
enum_name,
env_check_handler,
get_allowed_services,
get_download_job_handler,
@@ -296,6 +297,25 @@ SERVICE_DEFAULTS: dict[str, Any] = {
SERVICE_DETAILS: dict[str, tuple[Any, dict[str, Any]]] = {}
def json_value(value: Any) -> Any:
"""A click default or choice as JSON: a named object such as an enum member by its name, else text."""
if value is None or (isinstance(value, (str, int, float, bool)) and not hasattr(value, "name")):
return value
return enum_name(value)
def service_catalogue(tags: list[str]) -> dict[str, dict[str, Any]]:
"""The catalogue entry of each tag, or the defaults for one that fails; slow on the first call after a reload."""
entries: dict[str, dict[str, Any]] = {}
for tag in tags:
try:
entries[tag] = service_details(tag)
except Exception as e:
log.warning(f"Could not load details for service {tag}: {e}")
entries[tag] = {**SERVICE_DEFAULTS, "tag": tag}
return entries
def service_details(tag: str) -> dict[str, Any]:
"""The catalogue entry of one service, built again only when its service module reloads."""
service_module = Services.load(tag)
@@ -304,11 +324,8 @@ def service_details(tag: str) -> dict[str, Any]:
return hit[1]
service_data: dict[str, Any] = {**SERVICE_DEFAULTS, "tag": tag}
if hasattr(service_module, "ALIASES"):
service_data["aliases"] = list(service_module.ALIASES)
if hasattr(service_module, "GEOFENCE"):
service_data["geofence"] = list(service_module.GEOFENCE)
service_data["aliases"] = list(getattr(service_module, "ALIASES", ()) or ())
service_data["geofence"] = list(getattr(service_module, "GEOFENCE", ()) or ())
service_data["geoblock"] = list(getattr(service_module, "GEOBLOCK", ()) or ())
if hasattr(service_module, "TITLE_RE"):
@@ -342,19 +359,17 @@ def service_details(tag: str) -> dict[str, Any]:
param_info["opts"] = list(param.opts) if hasattr(param, "opts") else []
param_info["is_flag"] = getattr(param, "is_flag", False)
default = param.default
if default is None:
pass
elif callable(default) or type(default).__name__ == "Sentinel":
if callable(default) or type(default).__name__ == "Sentinel":
default = None
elif hasattr(default, "name"):
default = default.name
elif not isinstance(default, (str, int, float, bool, list)):
default = str(default)
elif isinstance(default, list):
default = [json_value(item) for item in default]
else:
default = json_value(default)
param_info["default"] = default
param_info["help"] = getattr(param, "help", None)
param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type)
if isinstance(param.type, click.Choice):
param_info["choices"] = list(param.type.choices)
param_info["choices"] = [json_value(choice) for choice in param.type.choices]
param_info["multiple"] = getattr(param, "multiple", False)
cli_params.append(param_info)
service_data["cli_params"] = cli_params
@@ -514,13 +529,10 @@ async def services(request: web.Request) -> web.Response:
remote_info = await asyncio.to_thread(remote_services_info, remote, allowed)
return web.json_response({"services": remote_info, "load_errors": []})
services_info = []
catalogue = await asyncio.to_thread(service_catalogue, service_tags)
for tag in service_tags:
try:
service_data: dict[str, Any] = {**service_details(tag)}
except Exception as e:
log.warning(f"Could not load details for service {tag}: {e}")
service_data = {**SERVICE_DEFAULTS, "tag": tag}
service_data: dict[str, Any] = {**catalogue[tag]}
if tag in services_module.PENDING:
service_data["pending_update"] = True
if server_accounts_allowed(request, tag) and (server_accounts := server_account_regions(tag)) is not None:
@@ -3102,7 +3114,7 @@ async def dashboard_keys(request: web.Request) -> web.Response:
@api_handler
async def dashboard_services(request: web.Request) -> web.Response:
"""
Dashboard: every configured service and its load state.
Dashboard: every configured service, its load state and its catalogue entry.
---
summary: Dashboard services
description: >
@@ -3169,6 +3181,24 @@ async def dashboard_services(request: web.Request) -> web.Response:
type: array
items:
type: string
server_accounts:
type: object
description: >
Regions the server's own accounts cover. Only when `serve.server_accounts` names
the service. No API key grant applies here, unlike `/api/services`.
properties:
regions:
type: array
items:
type: string
global:
type: boolean
additionalProperties:
description: >
The `/api/services` catalogue fields (help, url, title_regex, cli_params,
needs_auth, auth_methods, has_search, anonymous_search, has_drm, client_config).
A service with no loaded module, or whose catalogue entry fails, gets only
help, url and title_regex, all null.
'401':
description: Dashboard key missing or invalid
"""
+15 -22
View File
@@ -16,6 +16,8 @@ from rich.text import Text
from unshackle.core import binaries
from unshackle.core.config import config
from unshackle.core.console import console
from unshackle.core.drm import key_args as drm_key_args
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka, read_top_level_box
from unshackle.core.utilities import log_event
from unshackle.core.utils import run_proxy
@@ -209,7 +211,7 @@ class ClearKeyCENC:
decrypt_start = time.monotonic()
if decrypter == "mp4decrypt":
self.decrypt_with_mp4decrypt(path)
mp4decrypt_or_shaka(self, path)
else:
self.decrypt_with_shaka_packager(path)
@@ -224,12 +226,12 @@ class ClearKeyCENC:
output_size=path.stat().st_size if path.exists() else 0,
)
def mp4decrypt_key_args(self) -> list[str]:
"""Build the mp4decrypt --key arguments for every content key."""
def mp4decrypt_key_args(self, track_kids: Optional[dict[int, UUID]] = None) -> list[str]:
"""Build the mp4decrypt --key arguments for every content key, and by track ID for each ``track_kids`` KID."""
key_args = []
for kid, key in self.content_keys.items():
key_args.extend(["--key", f"{kid.hex}:{key}"])
return key_args
return key_args + drm_key_args.track_key_args(self.content_keys, track_kids)
def decrypt_with_mp4decrypt(self, path: Path) -> None:
"""Decrypt using mp4decrypt"""
@@ -238,7 +240,7 @@ class ClearKeyCENC:
output_path = path.with_stem(f"{path.stem}_decrypted")
key_args = self.mp4decrypt_key_args()
key_args = self.mp4decrypt_key_args(kidless_track_kids(read_top_level_box(path, b"moov") or b""))
cmd = [
str(binaries.Mp4decrypt),
@@ -301,7 +303,6 @@ class ClearKeyCENC:
)
stream_skipped = False
had_error = False
shaka_log_buffer = ""
assert p.stderr is not None
@@ -312,27 +313,19 @@ class ClearKeyCENC:
if "Skip stream" in line:
# file/segment was so small that it didn't have any actual data, ignore
stream_skipped = True
if ":INFO:" in line:
continue
if "I0" in line or "W0" in line:
continue
if ":ERROR:" in line:
had_error = True
if "Insufficient bits in bitstream for given AVC profile" in line:
# this is a warning and is something we don't have to worry about
if drm_key_args.shaka_noise(line):
continue
shaka_log_buffer += f"{line.strip()}\n"
if shaka_log_buffer:
# wrap to console width - padding - '[ClearKey]: '
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[ClearKey]: " + shaka_log_buffer))
p.wait()
if p.returncode != 0 or had_error:
if p.returncode != 0:
if shaka_log_buffer:
# wrap to console width - padding - '[ClearKey]: '
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[ClearKey]: " + shaka_log_buffer))
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
path.unlink()
+24 -5
View File
@@ -1,10 +1,12 @@
"""Decrypter key arguments shared by the Widevine and PlayReady DRM systems."""
"""Decrypter key arguments and output filtering shared by the CENC DRM systems."""
from __future__ import annotations
from typing import Any, Iterable, Optional
import re
from typing import Any, Iterable, Mapping, Optional
ZERO_KID = "00" * 16
SHAKA_NOISE = re.compile(r"^(?:[IWV]\d{4} |\[[^\]]*:(?:INFO|WARNING|VERBOSE\d*):)")
def _hex(value: Any) -> str:
@@ -42,15 +44,27 @@ def zero_kid_key(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> Optio
return None
def mp4decrypt_key_args(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> list[str]:
"""Build the mp4decrypt --key arguments: every content key, then the zero-KID fallback."""
def track_key_args(content_keys: dict[Any, Any], track_kids: Optional[Mapping[int, Any]]) -> list[str]:
"""Build the mp4decrypt --key arguments by track ID for each track whose own KID has a content key."""
keys = {_hex(kid): _hex(key) for kid, key in content_keys.items()}
key_args: list[str] = []
for track_id, kid in (track_kids or {}).items():
if _hex(kid) in keys:
key_args.extend(["--key", f"{track_id}:{keys[_hex(kid)]}"])
return key_args
def mp4decrypt_key_args(
content_keys: dict[Any, Any], own_kids: Iterable[Any], track_kids: Optional[Mapping[int, Any]] = None
) -> list[str]:
"""Build the mp4decrypt --key arguments: every content key, the zero-KID fallback and the ``track_kids`` keys."""
key_args: list[str] = []
for kid, key in content_keys.items():
key_args.extend(["--key", f"{_hex(kid)}:{_hex(key)}"])
fallback = zero_kid_key(content_keys, own_kids)
if fallback:
key_args.extend(["--key", f"{ZERO_KID}:{fallback}"])
return key_args
return key_args + track_key_args(content_keys, track_kids)
def shaka_keys(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> str:
@@ -60,3 +74,8 @@ def shaka_keys(content_keys: dict[Any, Any], own_kids: Iterable[Any]) -> str:
if fallback:
entries.append(f"label={len(entries)}:key_id={ZERO_KID}:key={fallback}")
return ",".join(entries)
def shaka_noise(line: str) -> bool:
"""True for a shaka-packager log line below ERROR, in its glog (I1103 ...) or bracket ([1103/...:WARNING:...]) form."""
return bool(SHAKA_NOISE.match(line))
+13 -20
View File
@@ -25,6 +25,7 @@ from unshackle.core.config import config
from unshackle.core.console import console
from unshackle.core.constants import AnyTrack
from unshackle.core.drm import key_args as drm_key_args
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka, read_top_level_box
from unshackle.core.session import AnySession, utf8_text
from unshackle.core.utilities import call_licence, get_boxes, log_event
from unshackle.core.utils import run_proxy
@@ -571,7 +572,7 @@ class PlayReady:
decrypt_start = time.monotonic()
if decrypter == "mp4decrypt":
self.decrypt_with_mp4decrypt(path)
mp4decrypt_or_shaka(self, path)
else:
self.decrypt_with_shaka_packager(path)
@@ -596,9 +597,9 @@ class PlayReady:
return [self._kid]
return self.kids
def mp4decrypt_key_args(self) -> list[str]:
"""Build the mp4decrypt --key arguments for every content key and the zero-KID fallback."""
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids())
def mp4decrypt_key_args(self, track_kids: Optional[dict[int, UUID]] = None) -> list[str]:
"""Build the mp4decrypt --key arguments for every content key, the zero-KID and the track ID fallbacks."""
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids(), track_kids)
def decrypt_with_mp4decrypt(self, path: Path) -> None:
"""Decrypt using mp4decrypt"""
@@ -607,7 +608,7 @@ class PlayReady:
output_path = path.with_stem(f"{path.stem}_decrypted")
key_args = self.mp4decrypt_key_args()
key_args = self.mp4decrypt_key_args(kidless_track_kids(read_top_level_box(path, b"moov") or b""))
cmd = [
str(binaries.Mp4decrypt),
@@ -667,7 +668,6 @@ class PlayReady:
)
stream_skipped = False
had_error = False
shaka_log_buffer = ""
assert p.stderr is not None
for line in iter(p.stderr.readline, ""):
@@ -676,25 +676,18 @@ class PlayReady:
continue
if "Skip stream" in line:
stream_skipped = True
if ":INFO:" in line:
continue
if "I0" in line or "W0" in line:
continue
if ":ERROR:" in line:
had_error = True
if "Insufficient bits in bitstream for given AVC profile" in line:
if drm_key_args.shaka_noise(line):
continue
shaka_log_buffer += f"{line.strip()}\n"
if shaka_log_buffer:
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[PlayReady]: " + shaka_log_buffer))
p.wait()
if p.returncode != 0 or had_error:
if p.returncode != 0:
if shaka_log_buffer:
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[PlayReady]: " + shaka_log_buffer))
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
path.unlink()
+8 -3
View File
@@ -12,6 +12,7 @@ from unshackle.core import binaries
from unshackle.core.config import config
from unshackle.core.drm.clearkey_cenc import ClearKeyCENC
from unshackle.core.drm.playready import PlayReady
from unshackle.core.drm.verify import kidless_track_kids
from unshackle.core.drm.widevine import Widevine
SEGMENT_DRM_T = Union[Widevine, PlayReady, ClearKeyCENC]
@@ -19,13 +20,17 @@ SEGMENT_DRM_T = Union[Widevine, PlayReady, ClearKeyCENC]
MAX_WORKERS = 16
def can_use(drm: Any, decryption: str) -> bool:
"""Whether unshackle can decrypt a track's segments one by one as they download."""
def can_use(drm: Any, decryption: str, init: Optional[bytes] = None) -> bool:
"""Whether unshackle can decrypt a track's segments one by one as they download.
A track whose ``init`` mp4decrypt cannot key goes to the whole-file decrypt, which has the shaka fallback.
"""
return (
isinstance(drm, (Widevine, PlayReady, ClearKeyCENC))
and str(decryption).lower() == "mp4decrypt"
and config.decrypt_segments
and bool(binaries.Mp4decrypt)
and not (init and set(kidless_track_kids(init).values()) - set(drm.content_keys))
)
@@ -44,7 +49,7 @@ class SegmentDecrypter:
raise EnvironmentError("mp4decrypt executable not found but is required.")
self.drm = drm
self.key_args = drm.mp4decrypt_key_args()
self.key_args = drm.mp4decrypt_key_args(kidless_track_kids(init_data))
work_dir.mkdir(parents=True, exist_ok=True)
self.work_dir = Path(tempfile.mkdtemp(prefix="segdec-", dir=work_dir))
+91
View File
@@ -1,9 +1,12 @@
import logging
import os
import struct
from pathlib import Path
from typing import Any, BinaryIO, Callable, Iterator, NamedTuple, Optional
from uuid import UUID
log = logging.getLogger(__name__)
DECRYPT_HOOK: Optional[Callable[..., None]] = None
CONTAINERS = {b"moov", b"trak", b"mdia", b"minf", b"stbl", b"mvex", b"moof", b"traf", b"sinf", b"schi"}
@@ -142,6 +145,94 @@ def has_encrypted_entry(moov: bytes) -> bool:
)
def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]:
"""Read a single top-level box from an MP4 file.
This function walks only the box headers, so the wanted box is the only payload it reads into memory.
Returns None if the file has no such top-level box, or if a box header is truncated or declares an
impossible size.
"""
file_size = path.stat().st_size
with path.open("rb") as f:
start = 0
while start + 8 <= file_size:
header = f.read(8)
if len(header) < 8:
return None
size = int.from_bytes(header[:4], "big")
if size == 1:
large = f.read(8)
if len(large) < 8:
return None
size = int.from_bytes(large, "big")
elif size == 0:
size = file_size - start
if size < 8:
return None
if header[4:8] == box_type:
f.seek(start)
return f.read(size)
start += size
f.seek(start)
return None
def has_encrypted_sample_entry(path: Path) -> bool:
"""True if the MP4's moov still carries an encrypted sample entry (``enc*`` or ``drm*``).
A faithful decrypt rewrites the entry back to the real codec 4CC through frma, so a
survivor means the decrypt tool skipped/failed the track. Only the moov is read, and its
stsd boxes are parsed, so byte runs in mdat or in metadata cannot match. Any read or
parse error -> False.
"""
try:
moov = read_top_level_box(path, b"moov")
return moov is not None and has_encrypted_entry(moov)
except (OSError, struct.error, RecursionError):
return False
def kidless_kid(data: bytes, kind: bytes, entry: int, end: int) -> Optional[UUID]:
"""Return the KID of an encrypted sample entry whose first sinf holds no CENC tenc (a PIFF sinf, say)."""
for fields in ENTRY_FIELDS.get(kind, ()):
sinfs = [(body, box_end) for box, body, box_end in children(data, entry + fields, end) if box == b"sinf"]
if sinfs:
if any(box == b"tenc" for box, _, _ in walk(data, *sinfs[0])):
return None
return entry_kid(data, kind, entry, end)
return None
def kidless_track_kids(data: bytes) -> dict[int, UUID]:
"""Map each track of a moov or init segment whose first sinf has no tenc for mp4decrypt to read to its KID."""
kids: dict[int, UUID] = {}
track = 0
try:
for kind, body, end in walk(data):
if kind == b"tkhd":
track = struct.unpack_from(">I", data, body + (20 if data[body] == 1 else 12))[0]
elif kind == b"stsd" and track:
for entry in children(data, body + 8, end):
kid = kidless_kid(data, *entry)
if kid:
kids[track] = kid
except (struct.error, IndexError, ValueError, RecursionError):
return {}
return kids
def mp4decrypt_or_shaka(drm: Any, path: Path) -> None:
"""Decrypt ``path`` with mp4decrypt, and again with shaka-packager when mp4decrypt copied it through.
mp4decrypt copies a track it has no content key for unchanged, so its first moof stays the same.
"""
first_moof = read_top_level_box(path, b"moof")
drm.decrypt_with_mp4decrypt(path)
if has_encrypted_sample_entry(path) and read_top_level_box(path, b"moof") == first_moof:
log.warning(f"mp4decrypt left {path.name} encrypted, decrypting it with shaka-packager")
drm.decrypt_with_shaka_packager(path)
def senc_protects_samples(buf: bytes, body: int, end: int) -> bool:
"""True if a senc/PIFF-uuid box describes protected samples.
+14 -21
View File
@@ -21,6 +21,7 @@ from unshackle.core.config import config
from unshackle.core.console import console
from unshackle.core.constants import AnyTrack
from unshackle.core.drm import key_args as drm_key_args
from unshackle.core.drm.verify import kidless_track_kids, mp4decrypt_or_shaka, read_top_level_box
from unshackle.core.session import AnySession, utf8_text
from unshackle.core.utilities import call_licence, get_boxes, log_event
from unshackle.core.utils import run_proxy
@@ -300,7 +301,7 @@ class Widevine:
decrypt_start = time.monotonic()
if decrypter == "mp4decrypt":
self.decrypt_with_mp4decrypt(path)
mp4decrypt_or_shaka(self, path)
else:
self.decrypt_with_shaka_packager(path)
@@ -325,9 +326,9 @@ class Widevine:
return [self._kid]
return self.kids
def mp4decrypt_key_args(self) -> list[str]:
"""Build the mp4decrypt --key arguments for every content key and the zero-KID fallback."""
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids())
def mp4decrypt_key_args(self, track_kids: Optional[dict[int, UUID]] = None) -> list[str]:
"""Build the mp4decrypt --key arguments for every content key, the zero-KID and the track ID fallbacks."""
return drm_key_args.mp4decrypt_key_args(self.content_keys, self.own_kids(), track_kids)
def decrypt_with_mp4decrypt(self, path: Path) -> None:
"""Decrypt using mp4decrypt"""
@@ -336,7 +337,7 @@ class Widevine:
output_path = path.with_stem(f"{path.stem}_decrypted")
key_args = self.mp4decrypt_key_args()
key_args = self.mp4decrypt_key_args(kidless_track_kids(read_top_level_box(path, b"moov") or b""))
cmd = [
str(binaries.Mp4decrypt),
@@ -396,7 +397,6 @@ class Widevine:
)
stream_skipped = False
had_error = False
shaka_log_buffer = ""
assert p.stderr is not None
@@ -407,26 +407,19 @@ class Widevine:
if "Skip stream" in line:
# file/segment was so small that it didn't have any actual data, ignore
stream_skipped = True
if ":INFO:" in line:
continue
if "I0" in line or "W0" in line:
continue
if ":ERROR:" in line:
had_error = True
if "Insufficient bits in bitstream for given AVC profile" in line:
if drm_key_args.shaka_noise(line):
continue
shaka_log_buffer += f"{line.strip()}\n"
if shaka_log_buffer:
# wrap to console width - padding - '[Widevine]: '
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[Widevine]: " + shaka_log_buffer))
p.wait()
if p.returncode != 0 or had_error:
if p.returncode != 0:
if shaka_log_buffer:
# wrap to console width - padding - '[Widevine]: '
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[Widevine]: " + shaka_log_buffer))
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
path.unlink()
+1 -1
View File
@@ -631,7 +631,7 @@ class DASH:
collapse_span = None
collapse_single_url = collapse_span is not None
use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption)
use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption, init_data)
# subtitle segments are tiny, the post-download merge costs nothing there
use_rolling_merge = not collapse_single_url and config.merge_segments and not isinstance(track, Subtitle)
+1
View File
@@ -867,6 +867,7 @@ class HLS:
init_section = wanted_segments[0].init_section
init_content, _ = HLS.fetch_init_section(session, init_section, 0)
map_data = (init_section, init_content)
use_segment_decrypt = use_segment_decrypt and can_use(session_drm, config.decryption, init_content)
if use_segment_decrypt:
assert isinstance(session_drm, (Widevine, PlayReady, ClearKeyCENC))
segment_decrypter = SegmentDecrypter(session_drm, init_content, save_dir.parent, max_workers)
+6 -48
View File
@@ -28,7 +28,12 @@ from unshackle.core.config import config
from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, DownloadCancelled
from unshackle.core.downloaders.requests import requests
from unshackle.core.drm import DRM_T, BlowfishStripe, ClearKeyCENC, PlayReady, Widevine
from unshackle.core.drm.verify import decrypt_track, has_encrypted_entry, senc_protects_samples
from unshackle.core.drm.verify import (
decrypt_track,
has_encrypted_sample_entry,
read_top_level_box,
senc_protects_samples,
)
from unshackle.core.events import events
from unshackle.core.session import MAX_BACKOFF, AnySession, RnetSession, utf8_text
from unshackle.core.temp import restore_segments, stash_segments
@@ -85,38 +90,6 @@ def direct_session(session: AnySession, proxy: Optional[str] = None) -> Session:
return new
def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]:
"""Read a single top-level box from an MP4 file.
This function walks only the box headers, so the wanted box is the only payload it reads into memory.
Returns None if the file has no such top-level box, or if a box header is truncated or declares an
impossible size.
"""
file_size = path.stat().st_size
with path.open("rb") as f:
start = 0
while start + 8 <= file_size:
header = f.read(8)
if len(header) < 8:
return None
size = int.from_bytes(header[:4], "big")
if size == 1:
large = f.read(8)
if len(large) < 8:
return None
size = int.from_bytes(large, "big")
elif size == 0:
size = file_size - start
if size < 8:
return None
if header[4:8] == box_type:
f.seek(start)
return f.read(size)
start += size
f.seek(start)
return None
def iter_top_level_boxes(path: Path) -> Iterable[tuple[bytes, int, int]]:
"""Yield (box_type, offset, size) for each top-level box, stopping at the first bad header."""
file_size = path.stat().st_size
@@ -238,21 +211,6 @@ def has_mp4_only_audio_entry(path: Path) -> bool:
return False
def has_encrypted_sample_entry(path: Path) -> bool:
"""True if the MP4's moov still carries an encrypted sample entry (``enc*`` or ``drm*``).
A faithful decrypt rewrites the entry back to the real codec 4CC through frma, so a
survivor means the decrypt tool skipped/failed the track. Only the moov is read, and its
stsd boxes are parsed, so byte runs in mdat or in metadata cannot match. Any read or
parse error -> False.
"""
try:
moov = read_top_level_box(path, b"moov")
return moov is not None and has_encrypted_entry(moov)
except (OSError, struct.error, RecursionError):
return False
def moof_still_encrypted(moof: bytes) -> bool:
# senc only: a decrypter detaches just the atom it consumed, so a PIFF uuid can
# outlive a good decrypt. ISM arrives as senc via piff_senc_to_cenc.