From ee38477b116beebb5ab986f56a73731a327832fd Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Mon, 5 Oct 2026 15:22:01 -0600 Subject: [PATCH] feat(dl): add --all-drm to license Widevine and PlayReady per track Each DRM system sends one challenge per KID set in a run, also when a vault has the content keys. Needs a CDM for each system in the cdm config. --- docs/dev/rest-api/endpoints.md | 3 +- docs/guide/cli-reference.md | 3 +- docs/guide/downloading.md | 10 +- docs/guide/drm-and-cdm.md | 36 +++ docs/reference/configuration/download.md | 2 +- docs/reference/configuration/drm.md | 17 ++ tests/core/test_all_drm.py | 270 +++++++++++++++++++++++ unshackle/commands/dl.py | 156 ++++++++++++- unshackle/core/api/download_manager.py | 2 + unshackle/core/api/handlers.py | 6 + unshackle/core/api/routes.py | 3 + unshackle/core/constants.py | 1 + unshackle/core/manifests/dash.py | 11 +- unshackle/core/manifests/hls.py | 26 ++- unshackle/core/tracks/track.py | 35 ++- 15 files changed, 563 insertions(+), 18 deletions(-) create mode 100644 tests/core/test_all_drm.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 6806c4c1..5e5c1038 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -444,6 +444,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `slow` | boolean/string | `null` | Randomized delay between downloads. | | `split_audio` | boolean | `null` | Separate files per audio codec. | | `skip_dl` | boolean | `false` | Only fetch keys, do not download. | +| `all_drm` | boolean | `false` | License each track with both Widevine and PlayReady. Needs a CDM for each DRM system. The server refuses it with `cdm_only: false` and with `cdm`, which selects one device. | | `export` | boolean | `false` | Export manifest/keys/subs to JSON. | | `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. | | `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. | @@ -453,7 +454,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `download_processes` | int | `1` | Split a track's segments across this many processes. Only engages for large batches. | | `downloads` | int | `1` | Concurrent tracks. | | `worst` | boolean | `false` | Lowest bitrate within quality. | -| `best_available` | boolean | `false` | Fall back to best available. | +| `best_available` | boolean | `false` | Fall back to best available. With `all_drm`, a DRM system that does not license is a warning. | | `repack` | boolean | `false` | Add REPACK tag. | | `tag` | string | `null` | Release group tag. | | `tmdb_id` / `imdb_id` / `tvdb_id` / `anilist_id` | - | `null` | External ID overrides. Each resolves its metadata directly instead of by a title search, and is used for tagging. Set `enrich` to also take the title, year and original language. Give at most one of `tmdb_id`, `imdb_id` and `tvdb_id`, since unshackle resolves the others from it. Sending two returns `400`. `anilist_id` still combines with one of them. `tmdb_id` and `tvdb_id` must be positive integers, `imdb_id` must look like `tt1375666`, and `anilist_id` must be a positive integer or a string like `mal:12345`, or the request returns `400`. An ID whose provider is unconfigured, such as `tmdb_id` with no `tmdb_api_key`, fails the job rather than returning `400`. `anilist_id` needs no API key. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index ad5cd38d..8fb3962e 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -83,7 +83,7 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | `-c`, `--channels` | - | Audio channels; matches sub-layouts (5.1 ≈ 6.0). | | `-naa`, `--noatmos` | off | Exclude Dolby Atmos audio tracks. | | `--worst` | off | Pick the lowest bitrate within the requested quality. **Requires `-q`.** | -| `--best-available`, `--warn-only` | off | Warn and continue when a requested resolution, range, or language is absent, instead of failing. | +| `--best-available`, `--warn-only` | off | Warn and continue when a requested resolution, range, or language is absent, instead of failing. With `--all-drm`, also when one DRM system does not license. | | `-rvb`, `--real-video-bitrate` | off | Probe real media size for true video bitrates, overriding the manifest. | | `-rab`, `--real-audio-bitrate` | off | Same for audio (slower). | @@ -194,6 +194,7 @@ order the service used, then renumbers the episodes into the order you asked for | `--cdm-only` / `--vaults-only` | Use only the CDM, or only Key Vaults, for key acquisition. | | `--cdm ` | Use the named CDM device from the `cdm` config mapping for this run, ignoring the service/default mapping. Over `--remote`, that device licenses the remote session, not the server CDM. | | `--skip-dl` | Skip the download but still retrieve keys. | +| `--all-drm` | License each track with both Widevine and PlayReady. Needs a CDM for each DRM system. Not for use with `--vaults-only` or `--cdm`. | | `--export` | Export track info and keys to a `mediaexport` JSON file in the exports directory. | ### Network & proxy diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index f9dd6172..9e937162 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -97,7 +97,7 @@ By default a missing requested resolution is an error. Two flags change that: | Flag | Behaviour | | --- | --- | -| `--best-available` (`--warn-only`) | If the requested resolution(s) are not present, continue with the best that *is* available instead of failing. Also softens missing video/audio/subtitle languages and hybrid fallbacks. | +| `--best-available` (`--warn-only`) | If the requested resolution(s) are not present, continue with the best that *is* available instead of failing. Also softens missing video/audio/subtitle languages and hybrid fallbacks, and with `--all-drm` a DRM system that does not license. | | `--worst` | Within the specified quality, pick the **lowest** bitrate rendition. **Requires `-q/--quality`.** | ```shell title="Never fail on a missing resolution" @@ -662,6 +662,7 @@ Before committing to a long download, inspect what unshackle *would* do: | `--list` | List the tracks the service exposes for each title, then stop. No selection, no download. | | `--list-titles` | List every title the service returned, then stop. `-w`, `--latest-episode` and `--latest-episodes` are not applied to this listing. | | `--skip-dl` | Skip downloading but still acquire the decryption keys. | +| `--all-drm` | License each track with both Widevine and PlayReady. See [Widevine and PlayReady on the same title](drm-and-cdm.md#widevine-and-playready-on-the-same-title). | ```shell title="See the track selection without downloading" unshackle dl -q 1080 -v H.265 -r HDR10 --list EXAMPLE 81234567 @@ -795,6 +796,13 @@ older unshackle `version: 2` files and unidl's own export files. unshackle dl --skip-dl --export EXAMPLE 81234567 ``` +Add `--all-drm` to send a challenge for each DRM system the title offers. The export then +holds the DRM init data of both. + +```shell title="License with Widevine and PlayReady, no download" +unshackle dl --skip-dl --all-drm --export EXAMPLE 81234567 +``` + The file name tells what the file holds, in the form `{title}.{scope}.{resolutions}.{codecs}.{ranges}.{manifests}.{audio}-{SERVICE}.json`: diff --git a/docs/guide/drm-and-cdm.md b/docs/guide/drm-and-cdm.md index 5c4f64c4..1d3c839e 100644 --- a/docs/guide/drm-and-cdm.md +++ b/docs/guide/drm-and-cdm.md @@ -482,6 +482,42 @@ Use the `cdm` mapping's [DRM-based selection](#advanced-selection-by-quality-drm to configure a device for each system. unshackle selects the right one per track automatically. +### License both systems with `--all-drm` + +By default a track licenses with one DRM system. `--all-drm` sends a challenge for +Widevine and for PlayReady on each track that offers both. Use it with `--skip-dl` and +`--export` to keep the content keys and the DRM init data of both systems. The two +systems can return different sets of KIDs, and unshackle keeps the content keys of both. + +```shell +unshackle dl --skip-dl --all-drm --export EXAMPLE 81234567 +``` + +- The `cdm` mapping must name a device for each system. A single device name, or `--cdm`, + stops the run with an error that names the missing system. +- Each DRM system sends one challenge for a set of KIDs in a run. A content key in a + Key Vault does not stop the challenge, so `--vaults-only` is refused. +- A service with a Widevine licence function only gets the PlayReady challenge through that + function. If its licence server refuses the challenge, that DRM system does not license. +- If one DRM system does not license, the title fails. With `--best-available` unshackle + shows a warning and continues, if the other system gave the content key of the track. + unshackle does not send the challenge again for the same KIDs in that run. +- A track gets its DRM systems from the manifest. For a DASH track or a direct URL, + unshackle also reads the init segment for a system that the manifest does not carry. +- The service gets a challenge from each DRM system for each set of KIDs. A licence server + that limits the request rate can answer `429`, and unshackle then tries again. + +!!! note "Where `--all-drm` licenses one system only" + - A service that gets a different manifest for each DRM system. unshackle shows a + warning one time when a track offers one system only. + - A service that fetched its playback for one DRM system and refuses the other. That + system does not license, so the title fails unless you use `--best-available`. + - An HLS playlist that changes its `EXT-X-KEY` entries part-way: only the first set + licenses with the second system. + - A remote session that the server CDM licenses, and `unshackle import`. unshackle + shows a warning and ignores the flag. + - A remote CDM that stores content keys can answer without a challenge to the service. + ## Keys, vaults, and caching Fetching a license is comparatively slow and, for some services, rate-limited. diff --git a/docs/reference/configuration/download.md b/docs/reference/configuration/download.md index 4ebc4b7a..e5dc2bcb 100644 --- a/docs/reference/configuration/download.md +++ b/docs/reference/configuration/download.md @@ -48,7 +48,7 @@ Common config keys. This is a useful subset, and every `dl` flag works: | `workers` | int | *(downloader default)* | Threads per track. | | `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). | | `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | -| `best_available` | bool | `false` | Warn instead of failing when a requested resolution, range, or language is missing. The CLI flag is `--best-available`, also spelled `--warn-only`. | +| `best_available` | bool | `false` | Warn instead of failing when a requested resolution, range, or language is missing, or when one DRM system does not license under `--all-drm`. The CLI flag is `--best-available`, also spelled `--warn-only`. | | `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | | `no_folder` | bool | `false` | Do not create a per-title folder. | diff --git a/docs/reference/configuration/drm.md b/docs/reference/configuration/drm.md index 02ace59b..deef6625 100644 --- a/docs/reference/configuration/drm.md +++ b/docs/reference/configuration/drm.md @@ -45,6 +45,23 @@ are `widevine` / `playready`. unshackle matches the quality keys first, but only height is known. If the value is still a dict after that, unshackle uses `widevine`/`playready` when present, or else the credential profile name, then `default`. +A quality key can also hold a dict of `widevine` / `playready` devices, and so can the +top-level `default` entry: + +```yaml +cdm: + default: + widevine: chromecdm_l3 + playready: sl2000 + EXAMPLE1: + ">=1080": { widevine: android_l1, playready: sl3000 } + "<1080": { widevine: chromecdm_l3, playready: sl2000 } +``` + +A quality key that holds one device name applies to both DRM systems. When a track needs the +other system, unshackle uses the `widevine` / `playready` config key of the entry. `dl --all-drm` +needs a device for each system from one of these shapes. + unshackle finds the resolved name in [`remote_cdm`](#remote_cdm) by `name` first. If no entry matches, unshackle loads it as a local device file, in this order: `.prd` in `directories.prds`, `.prd` in `directories.wvds`, then `.wvd` in diff --git a/tests/core/test_all_drm.py b/tests/core/test_all_drm.py new file mode 100644 index 00000000..f80df887 --- /dev/null +++ b/tests/core/test_all_drm.py @@ -0,0 +1,270 @@ +"""--all-drm: each track licenses with Widevine and with PlayReady, through one wrapper in dl.""" + +from __future__ import annotations + +import base64 +from collections import defaultdict +from functools import partial +from types import SimpleNamespace +from typing import Any, Optional +from uuid import UUID + +import m3u8 +import pytest +from pyplayready.system.pssh import PSSH as PR_PSSH +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.pssh import PSSH +from rich.table import Table + +from unshackle.commands.dl import dl +from unshackle.core.api.handlers import validate_download_parameters +from unshackle.core.cdm.detect import cdm_type_stub, is_widevine_cdm +from unshackle.core.drm import PlayReady, Widevine +from unshackle.core.manifests import HLS + +KID = UUID("00112233-4455-6677-8899-aabbccddeeff") +KEY = "00000000000000000000000000000001" +VAULT_KEY = "000000000000000000000000000000aa" +OTHER_KID = UUID("11112233-4455-6677-8899-aabbccddeeff") +OTHER_KEY = "00000000000000000000000000000002" + + +def playready_object(kid: UUID) -> str: + kid_b64 = base64.b64encode(kid.bytes_le).decode() + xml = ( + '' + f'' + "" + ).encode("utf-16-le") + pro = PR_PSSH.PlayreadyHeader.build( + {"length": 10 + len(xml), "records": [{"type": 1, "length": len(xml), "data": xml}]} + ) + return base64.b64encode(pro).decode() + + +def widevine() -> Widevine: + return Widevine(pssh=PSSH.new(system_id=PSSH.SystemId.Widevine, key_ids=[KID])) + + +def playready(kid: UUID = KID) -> PlayReady: + pro = playready_object(kid) + return PlayReady(pssh=PR_PSSH(base64.b64decode(pro)), pssh_b64=pro) + + +class Prepare: + """Stands in for prepare_drm: records each call and gives the content key, or fails for one DRM system.""" + + def __init__(self, fail: Optional[type] = None) -> None: + self.fail = fail + self.calls: list[tuple[str, bool, dict[UUID, str]]] = [] + + def __call__(self, drm: Any, track_kid: Optional[UUID] = None, **kwargs: Any) -> None: + forced = bool(kwargs.get("force") and kwargs.get("cdm_only")) + self.calls.append((type(drm).__name__, forced, dict(drm.content_keys))) + if type(drm) is self.fail: + raise ValueError("licence refused") + drm.content_keys[KID] = KEY + + def forced(self) -> list[str]: + return [name for name, forced, _ in self.calls if forced] + + +@pytest.fixture +def cmd() -> dl: + cmd = dl.__new__(dl) + cmd.log = SimpleNamespace(warning=lambda *a: None, debug=lambda *a: None) + cmd.cdm = object() + cmd.all_drm_attempted = defaultdict(set) + cmd.all_drm_warned = set() + dl.DRM_LOCKS.clear() + return cmd + + +def test_each_drm_system_licenses_although_a_content_key_is_held(cmd: dl) -> None: + wv, pr = widevine(), playready() + wv.content_keys[KID] = VAULT_KEY + prepare = Prepare() + + cmd.licence_all_drm(prepare, SimpleNamespace(drm=[wv, pr]), False, wv, track_kid=KID) + + assert prepare.forced() == ["Widevine", "PlayReady"] + assert all(held == {} for _, _, held in prepare.calls) + assert wv.content_keys == {KID: KEY} and pr.content_keys == {KID: KEY} + + +def test_a_second_track_with_the_same_kids_sends_no_request(cmd: dl) -> None: + prepare = Prepare(fail=PlayReady) + for _ in range(2): + wv, pr = widevine(), playready() + cmd.licence_all_drm(prepare, SimpleNamespace(drm=[wv, pr]), True, wv, track_kid=KID) + + assert prepare.forced() == ["Widevine", "PlayReady"] + assert len(prepare.calls) == 4 + + +def test_a_drm_system_that_fails_stops_the_title(cmd: dl) -> None: + wv, pr = widevine(), playready() + wv.content_keys[KID] = VAULT_KEY + + with pytest.raises(ValueError, match="licence refused"): + cmd.licence_all_drm(Prepare(fail=Widevine), SimpleNamespace(drm=[wv, pr]), False, wv, track_kid=KID) + + assert wv.content_keys == {KID: VAULT_KEY} + + +def test_with_tolerance_the_other_drm_system_gives_the_content_key(cmd: dl) -> None: + wv, pr = widevine(), playready() + loaded = cmd.cdm + + cmd.licence_all_drm(Prepare(fail=Widevine), SimpleNamespace(drm=[wv, pr]), True, wv, track_kid=KID) + + assert wv.content_keys == {KID: KEY} + assert cmd.cdm is loaded + + +def test_with_tolerance_a_track_with_no_content_key_still_fails(cmd: dl) -> None: + wv = widevine() + + with pytest.raises(ValueError, match="licence refused"): + cmd.licence_all_drm(Prepare(fail=Widevine), SimpleNamespace(drm=[wv]), True, wv, track_kid=KID) + + +def test_a_drm_type_with_no_cdm_goes_straight_through(cmd: dl) -> None: + prepare = Prepare() + other = SimpleNamespace(content_keys={}) + + cmd.licence_all_drm(prepare, SimpleNamespace(drm=[other, playready()]), False, other) + + assert prepare.calls == [("SimpleNamespace", False, {})] + + +def hls_keys(*lines: str) -> list[Any]: + playlist = "#EXTM3U\n#EXT-X-TARGETDURATION:6\n" + "".join(lines) + "#EXTINF:6.0,\nseg0.m4s\n#EXT-X-ENDLIST\n" + return [key for key in m3u8.loads(playlist).keys if key] + + +def test_hls_builds_the_other_drm_system_from_its_key_entry() -> None: + wv_pssh = PSSH.new(system_id=PSSH.SystemId.Widevine, key_ids=[KID]).dumps() + keys = hls_keys( + f'#EXT-X-KEY:METHOD=SAMPLE-AES,KEYFORMAT="{WidevineCdm.urn}",URI="data:text/plain;base64,{wv_pssh}"\n', + '#EXT-X-KEY:METHOD=SAMPLE-AES,KEYFORMAT="com.microsoft.playready",' + f'URI="data:text/plain;charset=UTF-16;base64,{playready_object(KID)}"\n', + ) + + assert [type(d) for d in HLS.alternate_drm(keys, widevine())] == [PlayReady] + assert [type(d) for d in HLS.alternate_drm(keys, playready())] == [Widevine] + + +def test_hls_requests_no_key_uri_for_the_other_drm_system() -> None: + keys = hls_keys('#EXT-X-KEY:METHOD=AES-128,URI="https://example.invalid/key.bin"\n') + + assert HLS.alternate_drm(keys, widevine()) == [] + + +def test_a_playready_track_with_two_widevine_objects_licenses(cmd: dl) -> None: + pr = playready() + prepare = Prepare() + + cmd.licence_all_drm(prepare, SimpleNamespace(drm=[pr, widevine(), widevine()]), False, pr, track_kid=KID) + + assert prepare.forced() == ["PlayReady", "Widevine"] + + +class RealPrepare: + """A dl whose real prepare_drm runs, with the licence of each DRM object replaced by fixed content keys.""" + + def __init__(self, vault: Optional[dict[UUID, str]] = None) -> None: + self.requests: list[str] = [] + self.vault_queries: list[UUID] = [] + cmd = dl.__new__(dl) + cmd.log = SimpleNamespace(warning=lambda *a: None, debug=lambda *a: None, info=lambda *a: None) + cmd.debug_logger = None + cmd.service = "EXAMPLE" + cmd.profile = None + cmd.cdm = cdm_type_stub("widevine") + cmd.all_drm_attempted = defaultdict(set) + cmd.all_drm_warned = set() + cmd.vaults = SimpleNamespace(get_key=self.get_key(vault or {}), sources={}, flagged=set()) + cmd.flush_vault_writes = lambda writes: None + cmd.cdm_for = lambda system, quality=None: cdm_type_stub(system) + dl.DRM_LOCKS.clear() + dl.LICENSE_KEY_CACHE.clear() + self.cmd = cmd + + def get_key(self, vault: dict[UUID, str]) -> Any: + def get_key(kid: UUID) -> tuple[Optional[str], Optional[str]]: + self.vault_queries.append(kid) + return vault.get(kid), "vault" if kid in vault else None + + return get_key + + def drm(self, drm: Any, keys: dict[UUID, str]) -> Any: + def get_content_keys(cdm: Any, certificate: Any = None, licence: Any = None) -> None: + self.requests.append(type(drm).__name__) + drm.content_keys.update(keys) + + drm.get_content_keys = get_content_keys + return drm + + def run(self, wv: Any, pr: Any) -> None: + table = Table.grid() + table.add_row("") + track = SimpleNamespace(drm=[wv, pr], id="track", height=None) + prepare = partial( + self.cmd.prepare_drm, + table=table, + track=track, + title=None, + certificate=None, + licence=lambda **kwargs: b"", + cdm_only=False, + vaults_only=False, + ) + self.cmd.licence_all_drm(prepare, track, False, wv, track_kid=KID) + + +def test_prepare_drm_licenses_each_drm_system_and_does_not_read_the_vault() -> None: + real = RealPrepare(vault={KID: VAULT_KEY}) + wv, pr = real.drm(widevine(), {KID: KEY}), real.drm(playready(), {KID: KEY}) + + real.run(wv, pr) + + assert real.requests == ["Widevine", "PlayReady"] + assert real.vault_queries == [] + assert wv.content_keys == {KID: KEY} and pr.content_keys == {KID: KEY} + assert is_widevine_cdm(real.cmd.cdm) + + real.run(real.drm(widevine(), {KID: KEY}), real.drm(playready(), {KID: KEY})) + + assert real.requests == ["Widevine", "PlayReady"] + + +def test_the_other_drm_system_can_return_a_different_set_of_kids() -> None: + real = RealPrepare() + wv = real.drm(widevine(), {KID: KEY}) + pr = real.drm(playready(OTHER_KID), {OTHER_KID: OTHER_KEY}) + + real.run(wv, pr) + + assert wv.content_keys == {KID: KEY, OTHER_KID: OTHER_KEY} + + +def test_the_api_refuses_all_drm_with_vaults_only_or_one_device() -> None: + assert "cdm_only" in (validate_download_parameters({"all_drm": True, "cdm_only": False}) or "") + assert "cdm" in (validate_download_parameters({"all_drm": True, "cdm": "device"}) or "") + assert validate_download_parameters({"all_drm": True, "cdm_only": True}) is None + + +def test_the_init_segment_adds_the_drm_system_the_track_does_not_have(monkeypatch: pytest.MonkeyPatch) -> None: + from unshackle.core.tracks import Track + + wv, pr = widevine(), playready() + monkeypatch.setattr(PlayReady, "from_init_data", classmethod(lambda cls, data: pr)) + monkeypatch.setattr( + Widevine, "from_init_data", classmethod(lambda cls, data: pytest.fail("the track has Widevine already")) + ) + track = SimpleNamespace(drm=[wv], prefers_playready=lambda cdm: True) + + Track.add_init_drm(track, b"init", None) + + assert track.drm == [wv, pr] diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 310649e1..045eb659 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -10,7 +10,7 @@ import shutil import subprocess import sys import time -from collections import Counter +from collections import Counter, defaultdict from collections.abc import Iterable from concurrent import futures from concurrent.futures import ThreadPoolExecutor @@ -42,10 +42,16 @@ from rich.tree import Tree from unshackle.core import __version__, binaries, providers from unshackle.core.cdm import DecryptLabsRemoteCDM -from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_widevine_cdm +from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote_cdm, is_widevine_cdm from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel -from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings +from unshackle.core.constants import ( + DOWNLOAD_ALL_DRM, + DOWNLOAD_CANCELLED, + DOWNLOAD_LICENCE_ONLY, + AnyTrack, + context_settings, +) from unshackle.core.credential import Credential from unshackle.core.downloaders import default_max_workers, format_speed, parse_speed_limit, set_speed_limit from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevine, own_kids, verify @@ -875,6 +881,12 @@ class dl: @click.option( "--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys." ) + @click.option( + "--all-drm", + is_flag=True, + default=False, + help="License each track with both Widevine and PlayReady. Needs a CDM for each DRM system.", + ) @click.option( "--export", is_flag=True, @@ -1030,6 +1042,7 @@ class dl: VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer") EXPORT_LOCK = Lock() LICENSE_KEY_CACHE: dict[UUID, str] = {} + ALL_DRM_LOCK = Lock() def __init__( self, @@ -1526,9 +1539,16 @@ class dl: progress_sink: Optional[Callable[[dict[str, Any]], None]] = None, postscript: Sequence[str] = (), no_postscript: bool = False, + all_drm: bool = False, *_: Any, **__: Any, ) -> None: + if all_drm: + DOWNLOAD_ALL_DRM.set() + else: + DOWNLOAD_ALL_DRM.clear() + self.all_drm_attempted: defaultdict[type, set[UUID]] = defaultdict(set) + self.all_drm_warned: set[str] = set() if no_postscript: postscript = NO_POST_SCRIPTS if continue_downloads: @@ -1697,6 +1717,8 @@ class dl: vaults_only = None else: vaults_only = not cdm_only + if all_drm and vaults_only: + raise click.ClickException("--all-drm sends licence requests, so it cannot be used with --vaults-only.") config.decryption = resolve_decryption( config.decryption_map, config.decryption, service.__class__.__name__.upper() @@ -2863,7 +2885,7 @@ class dl: download_table.add_row(selected_tracks) def prepare_drm_for(track: AnyTrack) -> Callable: - return partial( + prepare = partial( partial(self.prepare_drm, table=download_table), track=track, title=title, @@ -2875,6 +2897,9 @@ class dl: export=export_path, service_session=service.session, ) + if title_all_drm: + return partial(self.licence_all_drm, prepare, track, best_available) + return prepare server_cdm_type = None if getattr(self._remote_service, "_server_cdm", False): @@ -2910,6 +2935,28 @@ class dl: ) self.cdm = quality_based_cdm + title_all_drm = all_drm and not server_cdm_type + if all_drm and server_cdm_type and DOWNLOAD_ALL_DRM.is_set(): + DOWNLOAD_ALL_DRM.clear() + self.log.warning("--all-drm is ignored: the server CDM licenses this session with one DRM system") + if title_all_drm: + heights = {track.height for track in title.tracks.videos if track.height} or {None} + for system, matches in (("Widevine", is_widevine_cdm), ("PlayReady", is_playready_cdm)): + system_cdms = [self.cdm_for(system.lower(), height) for height in heights] + if not all(system_cdm and matches(system_cdm) for system_cdm in system_cdms): + hint = ( + "--cdm selects one device, so remove it." + if self.cdm_override + else f"Set cdm.{self.service}.{system.lower()} in the configuration." + ) + raise click.ClickException(f"--all-drm needs a {system} CDM for {self.service}. {hint}") + if is_remote_cdm(system_cdms[0]) and system not in self.all_drm_warned: + self.all_drm_warned.add(system) + self.log.warning( + f"The {system} CDM is remote. It can answer from its own key cache " + "and send no licence request." + ) + if hasattr(service, "resolve_server_keys"): service.resolve_server_keys(title) if not cdm_only: @@ -4381,6 +4428,96 @@ class dl: fn = service.get_playready_license if drm_system == "playready" else service.get_widevine_license return fn(**declared_kwargs(fn, kwargs)) + def cdm_for(self, system: str, quality: Optional[int] = None) -> Optional[object]: + """Get the CDM for one DRM system, ``widevine`` or ``playready``. + + A quality key that names one device applies to both DRM systems. When that device is for + the other system, the ``widevine`` or ``playready`` config key selects the device. + """ + matches = is_playready_cdm if system == "playready" else is_widevine_cdm + cdm = self.get_cdm(self.service, self.profile, drm=system, quality=quality) + if quality and not (cdm and matches(cdm)): + cdm = self.get_cdm(self.service, self.profile, drm=system) + return cdm + + def licence_one_drm(self, prepare: Callable, drm: DRM_T, track_kid: Optional[UUID]) -> None: + """Send one challenge for the KIDs of this DRM, the first time a run sees them for its DRM system. + + prepare_drm asks the CDM only for a KID with no content key, so the content keys the DRM + holds are removed for the call and put back after it. + """ + kids = {*drm.kids, *([track_kid] if track_kid else [])} + attempted = self.all_drm_attempted[type(drm)] + if kids <= attempted: + prepare(drm, track_kid=track_kid) + return + with self.drm_lock(drm): + held = dict(drm.content_keys) + drm.content_keys.clear() + try: + prepare(drm, track_kid=track_kid, cdm_only=True, force=True) + finally: + attempted |= kids + with self.drm_lock(drm): + for held_kid, key in held.items(): + drm.content_keys.setdefault(held_kid, key) + + def licence_all_drm( + self, + prepare: Callable, + track: AnyTrack, + tolerate: bool, + drm: DRM_T, + track_kid: Optional[UUID] = None, + ) -> None: + """License the track with each DRM system it offers, not only the one the downloader selected. + + Each DRM system sends one challenge for a set of KIDs in a run, also when a vault + has the content keys. A DRM system that fails stops the title. With ``tolerate`` it is a + warning, if the other DRM system gave the content key of the track. + """ + if type(drm) not in (Widevine, PlayReady): + prepare(drm, track_kid=track_kid) + return + + other_type = PlayReady if type(drm) is Widevine else Widevine + others = [d for d in track.drm or [] if type(d) is other_type] + if other_type is PlayReady and len(others) > 1: + others[0].absorb(*others[1:]) + if not others and other_type.__name__ not in self.all_drm_warned: + self.all_drm_warned.add(other_type.__name__) + self.log.warning( + f"A track does not offer {other_type.__name__}, so it licenses with " + f"{type(drm).__name__} only. This warning shows one time: {track}" + ) + + targets = [drm, *others[:1]] + error: Optional[Exception] = None + with self.ALL_DRM_LOCK: + loaded_cdm = self.cdm + try: + for target in targets: + kid = track_kid if target is drm or track_kid in target.kids else None + try: + self.licence_one_drm(prepare, target, kid) + except Exception as e: + if not tolerate: + raise + self.log.warning(f"{type(target).__name__} licence failed for {track}: {e}") + error = error or e + + keys = {kid: key for target in targets for kid, key in target.content_keys.items()} + for target in targets: + with self.drm_lock(target): + for kid, key in keys.items(): + target.content_keys.setdefault(kid, key) + finally: + self.cdm = loaded_cdm + + track_keys = getattr(drm, "content_keys", {}) + if error and (not track_keys or (track_kid and track_kid not in track_keys)): + raise error + def prepare_drm( self, drm: DRM_T, @@ -4395,10 +4532,13 @@ class dl: vaults_only: bool = False, export: Optional[Path] = None, service_session: Optional[Any] = None, + force: bool = False, ) -> None: """ Prepare the DRM by getting decryption data like KIDs, Keys, and such. The DRM object should be ready for decryption once this function ends. + ``force`` skips the content keys of earlier tracks in this run. The CDM then licenses + again for each KID the DRM holds no content key for. """ if not drm: return @@ -4548,7 +4688,7 @@ class dl: if not server_cdm: if isinstance(drm, Widevine): if not is_widevine_cdm(self.cdm): - widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) + widevine_cdm = self.cdm_for("widevine", track_quality) if widevine_cdm and is_widevine_cdm(widevine_cdm): if track_quality: self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") @@ -4561,7 +4701,7 @@ class dl: elif isinstance(drm, PlayReady): if not is_playready_cdm(self.cdm): - playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) + playready_cdm = self.cdm_for("playready", track_quality) if playready_cdm and is_playready_cdm(playready_cdm): if track_quality: self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") @@ -4613,7 +4753,7 @@ class dl: is_track_kid = ["", "*"][kid == track_kid] - cached_key = self.LICENSE_KEY_CACHE.get(kid) + cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid) if cached_key: drm.content_keys[kid] = cached_key label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache" @@ -4812,7 +4952,7 @@ class dl: is_track_kid = ["", "*"][kid == track_kid] - cached_key = self.LICENSE_KEY_CACHE.get(kid) + cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid) if cached_key: drm.content_keys[kid] = cached_key label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache" diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 03a30b1a..fc04a574 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -464,6 +464,7 @@ def perform_download( "proxy_download": params.get("proxy_download"), "profile": params.get("profile"), "cdm_name": params.get("cdm"), + "all_drm": params.get("all_drm", False), "repack": params.get("repack", False), "tag": params.get("tag"), "tmdb_id": params.get("tmdb_id"), @@ -618,6 +619,7 @@ def perform_download( list_=False, list_titles=False, skip_dl=params.get("skip_dl", False), + all_drm=params.get("all_drm", False), export=params.get("export"), cdm_only=params.get("cdm_only"), no_proxy=params.get("no_proxy", False), diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 01acf78d..ced00c78 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -80,6 +80,7 @@ DEFAULT_DOWNLOAD_PARAMS = { "slow": None, "split_audio": None, "skip_dl": False, + "all_drm": False, "export": False, "cdm_only": None, "proxy": None, @@ -1834,6 +1835,11 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: return "output_dir must be a path under the server's downloads directory." data["output_dir"] = str(target) + if data.get("all_drm") and data.get("cdm_only") is False: + return "all_drm sends licence requests, so it cannot be used with cdm_only set to false." + if data.get("all_drm") and data.get("cdm"): + return "all_drm needs a CDM for each DRM system, so it cannot be used with cdm, which selects one device." + if "vcodec" in data and data["vcodec"]: err = check_codec(data["vcodec"], VALID_VCODECS, "vcodec") if err: diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 760f6151..4a3d1642 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -947,6 +947,9 @@ async def download(request: web.Request) -> web.Response: skip_dl: type: boolean description: Skip downloading, only retrieve decryption keys (default - false) + all_drm: + type: boolean + description: License each track with both Widevine and PlayReady. Needs a CDM for each DRM system and cannot be used with `cdm` or with `cdm_only` set to false (default - false) export: type: boolean description: Export manifest, DRM init data, keys, and track info to a JSON file in the exports directory (default - false) diff --git a/unshackle/core/constants.py b/unshackle/core/constants.py index 51ad0f3b..ffd84427 100644 --- a/unshackle/core/constants.py +++ b/unshackle/core/constants.py @@ -3,6 +3,7 @@ from typing import TypeVar, Union DOWNLOAD_CANCELLED = Event() DOWNLOAD_LICENCE_ONLY = Event() +DOWNLOAD_ALL_DRM = Event() class DownloadCancelled(Exception): diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 3b59965f..415776eb 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -27,7 +27,7 @@ from requests import Session from unshackle.core import binaries from unshackle.core.config import config -from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack +from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use from unshackle.core.drm.verify import decrypt_track @@ -395,8 +395,13 @@ class DASH: track.drm = manifest_drm else: track.drm = existing_drm + if DOWNLOAD_ALL_DRM.is_set(): + track.drm = [ + *existing_drm, + *(d for d in manifest_drm if not any(type(d) is type(e) for e in existing_drm)), + ] - if track.drm and track.drm_preference: + if track.drm and track.drm_preference and not DOWNLOAD_ALL_DRM.is_set(): wanted = DRM_PREFERENCE_TYPES[track.drm_preference] preferred = [drm_obj for drm_obj in track.drm if isinstance(drm_obj, wanted)] if preferred: @@ -478,6 +483,8 @@ class DASH: if not track.drm and init_data and isinstance(track, (Video, Audio)) and not binaries.FFProbe: log.warning("FFprobe was not found, so the init segment was not probed for a PSSH.") + elif DOWNLOAD_ALL_DRM.is_set() and init_data and binaries.FFProbe and isinstance(track, (Video, Audio)): + track.add_init_drm(init_data, cdm) elif not track.drm and init_data and isinstance(track, (Video, Audio)): prefers_playready = track.prefers_playready(cdm) if prefers_playready: diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index f3e86e12..118fb6d5 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -28,9 +28,9 @@ from pywidevine.pssh import PSSH as WV_PSSH from requests import Session from unshackle.core import binaries -from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm +from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_widevine_cdm from unshackle.core.config import config -from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack +from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from unshackle.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use from unshackle.core.drm.verify import decrypt_track @@ -697,6 +697,8 @@ class HLS: if hasattr(existing_drm, "content_keys") and existing_drm.content_keys: media_drm.content_keys.update(existing_drm.content_keys) track.drm = [media_drm] + if DOWNLOAD_ALL_DRM.is_set(): + track.drm += HLS.alternate_drm(media_keys, media_drm, session) try: if not license_widevine: raise ValueError("license_widevine func must be supplied to use DRM") @@ -1435,6 +1437,26 @@ class HLS: return [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats] return keys + @staticmethod + def alternate_drm( + keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], + chosen: DRM_T, + session: Optional[Union[Session, RnetSession]] = None, + ) -> list[DRM_T]: + """Build the Widevine or PlayReady DRM that the EXT-X-KEY entries offer next to ``chosen``. + + Only an entry with the KEYFORMAT of the other DRM system counts, so unshackle requests no URI. + """ + system = "widevine" if isinstance(chosen, PlayReady) else "playready" + other_keys = [k for k in HLS.filter_keys_for_cdm(keys, cdm_type_stub(system)) if k.method != "NONE"] + if not other_keys: + return [] + try: + return [HLS.get_drm(other_keys[0], session)] + except Exception as e: + logging.getLogger("HLS").warning(f"Could not read the {system} EXT-X-KEY entry: {e}") + return [] + @staticmethod def get_track_kid_from_init( master: M3U8, diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index a5014367..fa804fcb 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -21,7 +21,7 @@ from requests.adapters import HTTPAdapter, Retry from unshackle.core import binaries from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm from unshackle.core.config import config -from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, DownloadCancelled +from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, DownloadCancelled from unshackle.core.downloaders import requests from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine from unshackle.core.drm.verify import decrypt_track @@ -563,6 +563,31 @@ class Track: ) self._drm_preference = value.lower() + def add_init_drm(self, init_data: Optional[bytes], cdm: Optional[object]) -> None: + """Add the Widevine and PlayReady DRM that the init segment carries and the track does not have. + + A track with a different DRM type stays as it is. The DRM system that agrees with the CDM comes first. + """ + log = logging.getLogger("track") + held: list[DRM_T] = list(self.drm or []) + if not init_data or any(type(d) not in (Widevine, PlayReady) for d in held): + return + for system in (PlayReady, Widevine) if self.prefers_playready(cdm) else (Widevine, PlayReady): + if any(type(d) is system for d in held): + continue + try: + held.append(system.from_init_data(init_data)) + except (PlayReady.Exceptions.PSSHNotFound, Widevine.Exceptions.PSSHNotFound): + continue + except Exception as e: + if not held: + raise + log.debug(f"Could not read the {system.__name__} PSSH of the init segment: {e}") + if held: + self.drm = held + else: + log.debug("No PlayReady or Widevine PSSH was found for this track, is it DRM free?") + def prefers_playready(self, cdm: Optional[object]) -> bool: """Whether to try PlayReady before Widevine. The track's preference wins over the loaded CDM.""" if self._drm_preference: @@ -665,7 +690,13 @@ class Track: manifest_parsers[self.descriptor].download_track(track=self, ctx=ctx) elif self.descriptor == self.Descriptor.URL: try: - if not self.drm and track_type in ("Video", "Audio"): + if ( + DOWNLOAD_ALL_DRM.is_set() + and track_type in ("Video", "Audio") + and not {Widevine, PlayReady} <= {type(d) for d in self.drm or []} + ): + self.add_init_drm(self.get_init_segment(session=session), cdm) + elif not self.drm and track_type in ("Video", "Audio"): if self.prefers_playready(cdm): try: self.drm = [PlayReady.from_track(self, session)]