Merge branch 'feat/new-export' into dev

This commit is contained in:
imSp4rky
2026-09-23 11:12:19 -06:00
9 changed files with 390 additions and 111 deletions
+16 -3
View File
@@ -776,14 +776,27 @@ a content key when the vault misses. You can force one side or the other:
unshackle dl --skip-dl EXAMPLE 81234567 unshackle dl --skip-dl EXAMPLE 81234567
``` ```
`--export` writes a JSON file, into the configured exports directory, containing track `--export` writes a `mediaexport` JSON file into the configured exports directory. It holds
info and the acquired content keys for each title. This is the format consumed by the manifest, the DRM init data, the content keys and the title metadata for each title.
`unshackle import` to reconstruct a download later. `unshackle import` reads it to reconstruct a download later. The importer also reads the
older unshackle `version: 2` files and unidl's own export files.
```shell title="Export track info and keys" ```shell title="Export track info and keys"
unshackle dl --skip-dl --export EXAMPLE 81234567 unshackle dl --skip-dl --export EXAMPLE 81234567
``` ```
The export holds a content key for each track your flags selected and no others. To make a
file another person can use, select generously: for example `-a AAC,EC3` and every language
you want them to have. An import can only pick tracks whose key is in the file. A subtitle
that another tool exported as a direct URL is added next to the manifest's own.
!!! warning "The export file is a secret"
It holds the content keys and usually a signed manifest URL. Hand it over on purpose,
not by accident.
The [mediaexport](https://github.com/unshackle-dl/mediaexport) package specifies the
`mediaexport` format and is the reader and writer unshackle uses.
!!! note "Region is recorded only with a proxy" !!! note "Region is recorded only with a proxy"
When you use `--proxy`, the export records the region so an import can reproduce the When you use `--proxy`, the export records the region so an import can reproduce the
correct geofence. Without a proxy, the export stores no region. correct geofence. Without a proxy, the export stores no region.
+2
View File
@@ -70,6 +70,7 @@ dependencies = [
"mutagen>=1.47.0,<2", "mutagen>=1.47.0,<2",
"defusedxml>=0.7.1", "defusedxml>=0.7.1",
"rich-click>=1.9.8", "rich-click>=1.9.8",
"mediaexport",
] ]
[project.urls] [project.urls]
@@ -172,3 +173,4 @@ skips = [
[tool.uv.sources] [tool.uv.sources]
unshackle = { workspace = true } unshackle = { workspace = true }
mediaexport = { git = "https://github.com/unshackle-dl/mediaexport", branch = "main" }
+72 -22
View File
@@ -1,4 +1,4 @@
"""Tests for ``dl.write_export``, the ``--export`` JSON sidecar. """Tests for ``dl.write_export``, the ``--export`` mediaexport sidecar.
Regression: DRM-free tracks never pass through ``prepare_drm``, so ``write_export`` Regression: DRM-free tracks never pass through ``prepare_drm``, so ``write_export``
must accept ``drm=None`` (and DRM systems without ``to_dict``/``content_keys`` such must accept ``drm=None`` (and DRM systems without ``to_dict``/``content_keys`` such
@@ -83,6 +83,10 @@ def read_export(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf8")) return json.loads(path.read_text(encoding="utf8"))
def entry(path: Path) -> dict:
return read_export(path)["titles"][0]
def test_drm_free_track_exports(tmp_path: Path) -> None: def test_drm_free_track_exports(tmp_path: Path) -> None:
"""The reported bug: DRM-free downloads produced no usable export.""" """The reported bug: DRM-free downloads produced no usable export."""
export = tmp_path / "export.json" export = tmp_path / "export.json"
@@ -92,13 +96,14 @@ def test_drm_free_track_exports(tmp_path: Path) -> None:
make_dl().write_export(export, title, video) make_dl().write_export(export, title, video)
doc = read_export(export) doc = read_export(export)
assert doc["version"] == 2 assert doc["kind"] == "mediaexport" and doc["version"] == 1
assert doc["service"] == "EXAMPLE" assert doc["service"]["tag"] == "EXAMPLE"
tinfo = doc["titles"]["movie-1"] tinfo = doc["titles"][0]
assert tinfo["meta"]["name"] == "Example Movie" assert tinfo["title"] == "Example Movie" and tinfo["kind"] == "movie"
assert set(tinfo["tracks"]) == {"v1", "a1", "s1"} assert {row["id"] for row in tinfo["tracks"]} == {"v1", "a1", "s1"}
assert "keys" not in tinfo["tracks"]["v1"] assert tinfo["x-unshackle"]["meta"]["name"] == "Example Movie"
assert "drm" not in tinfo["tracks"]["v1"] assert set(tinfo["x-unshackle"]["tracks"]) == {"v1"}
assert "keys" not in tinfo and "drm" not in tinfo
def test_clearkey_drm_exports_track_without_keys(tmp_path: Path) -> None: def test_clearkey_drm_exports_track_without_keys(tmp_path: Path) -> None:
@@ -109,10 +114,9 @@ def test_clearkey_drm_exports_track_without_keys(tmp_path: Path) -> None:
make_dl().write_export(export, title, video, ClearKey(key="bb" * 16)) make_dl().write_export(export, title, video, ClearKey(key="bb" * 16))
doc = read_export(export) tinfo = entry(export)
track = doc["titles"]["movie-1"]["tracks"]["v1"] assert "keys" not in tinfo and "drm" not in tinfo
assert "keys" not in track assert "v1" in tinfo["x-unshackle"]["tracks"]
assert "drm" not in track
def test_post_download_write_keeps_licensed_keys(tmp_path: Path) -> None: def test_post_download_write_keeps_licensed_keys(tmp_path: Path) -> None:
@@ -125,9 +129,10 @@ def test_post_download_write_keeps_licensed_keys(tmp_path: Path) -> None:
runner.write_export(export, title, video, StubDRM()) # prepare_drm runner.write_export(export, title, video, StubDRM()) # prepare_drm
runner.write_export(export, title, video) # post-download hook runner.write_export(export, title, video) # post-download hook
track = read_export(export)["titles"]["movie-1"]["tracks"]["v1"] tinfo = entry(export)
assert track["drm"] == [{"system": "Widevine", "pssh_b64": "AAAA"}] assert tinfo["drm"] == [{"system": "widevine", "pssh": "AAAA"}]
assert track["keys"] == {KID.hex: "aa" * 16} assert tinfo["keys"] == {KID.hex: "aa" * 16}
assert tinfo["x-unshackle"]["drm"] == [{"system": "Widevine", "pssh_b64": "AAAA"}]
def test_drm_free_export_roundtrips_through_import_service(tmp_path: Path) -> None: def test_drm_free_export_roundtrips_through_import_service(tmp_path: Path) -> None:
@@ -174,11 +179,13 @@ def test_clearkey_cenc_exports_drm_and_keys(tmp_path: Path) -> None:
make_dl().write_export(export, title, video, drm) make_dl().write_export(export, title, video, drm)
track = read_export(export)["titles"]["movie-1"]["tracks"]["v1"] tinfo = entry(export)
assert track["drm"] == [{"system": "ClearKeyCENC", "kids": [KID.hex], "laurl": "https://license.example.test/ck"}] assert tinfo["drm"] == [{"system": "clearkey"}]
assert track["keys"] == {KID.hex: "cc" * 16} assert tinfo["keys"] == {KID.hex: "cc" * 16}
own = tinfo["x-unshackle"]["drm"]
assert own == [{"system": "ClearKeyCENC", "kids": [KID.hex], "laurl": "https://license.example.test/ck"}]
rebuilt = drm_from_dict({**track["drm"][0], "content_keys": track["keys"]}) rebuilt = drm_from_dict({**own[0], "content_keys": tinfo["keys"]})
assert isinstance(rebuilt, ClearKeyCENC) assert isinstance(rebuilt, ClearKeyCENC)
assert rebuilt.content_keys == {KID: "cc" * 16} assert rebuilt.content_keys == {KID: "cc" * 16}
@@ -193,6 +200,49 @@ def test_keyless_content_keys_writes_no_keys_entry(tmp_path: Path) -> None:
make_dl().write_export(export, title, video, drm) make_dl().write_export(export, title, video, drm)
track = read_export(export)["titles"]["movie-1"]["tracks"]["v1"] tinfo = entry(export)
assert track["drm"] == [{"system": "Widevine", "pssh_b64": "AAAA"}] assert tinfo["drm"] == [{"system": "widevine", "pssh": "AAAA"}]
assert "keys" not in track assert "keys" not in tinfo
def test_unidl_export_imports_without_track_dicts(tmp_path: Path) -> None:
"""A file from another tool has no x-unshackle block: titles, keys and chapters still rebuild."""
export = tmp_path / "unidl.json"
export.write_text(
json.dumps(
{
"kind": "unidl-export",
"version": 1,
"service": "example",
"titles": [
{
"save_name": "Show.S01E02",
"title": {
"id": "ep-2",
"kind": "episode",
"name": "Show",
"episode_name": "Two",
"season": 1,
"episode": 2,
"year": "2024",
},
"keys": [f"{KID.hex}:{'dd' * 16}"],
"manifest_url": "https://example.test/m.mpd",
"headers": {"User-Agent": "unidl-ua"},
"drm": {"system": "widevine", "pssh": "AAAA"},
"chapters": [{"start_ms": 5000, "title": "Recap"}],
}
],
}
),
encoding="utf8",
)
svc = ImportService(SimpleNamespace(parent=None, params={}), "EXAMPLE", "ep-2", str(export))
ep = next(iter(svc.get_titles()))
assert (ep.title, ep.season, ep.number) == ("Show", 1, 2)
assert svc.session.headers["User-Agent"] == "unidl-ua"
assert svc.key_pool() == {KID: "dd" * 16}
assert svc.exported_drm_system() == "widevine"
assert [c.name for c in svc.get_chapters(ep)] == [None, "Recap"]
assert svc.titles_data["ep-2"]["manifest_type"] == "DASH"
+2 -3
View File
@@ -121,11 +121,10 @@ def test_encrypted_ism_rung_gets_the_exported_keys(tmp_path: Path, monkeypatch:
monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_ENCRYPTED, url)) monkeypatch.setattr(ISM, "from_url", lambda url, session=None, **kw: ISM.from_text(ISM_ENCRYPTED, url))
parsed = ISM.from_text(ISM_ENCRYPTED, ISM_URL).to_tracks(language="en") parsed = ISM.from_text(ISM_ENCRYPTED, ISM_URL).to_tracks(language="en")
# export the licensed rung only, as a finished download would
licensed = max(parsed.videos, key=lambda t: t.bitrate or 0)
export = export_tracks(tmp_path, parsed, ISM_URL) export = export_tracks(tmp_path, parsed, ISM_URL)
doc = json.loads(export.read_text(encoding="utf8")) doc = json.loads(export.read_text(encoding="utf8"))
doc["titles"]["movie-1"]["tracks"][licensed.id]["keys"] = {"0" * 32: "ab" * 16} doc["titles"][0]["keys"] = {"0" * 32: "ab" * 16}
doc["titles"][0]["drm"] = [{"system": "playready"}]
export.write_text(json.dumps(doc), encoding="utf8") export.write_text(json.dumps(doc), encoding="utf8")
svc, tracks = import_tracks(export) svc, tracks = import_tracks(export)
+41
View File
@@ -136,3 +136,44 @@ def test_fetch_failure_still_reports_a_possible_expiry(tmp_path: Path, failure:
with pytest.raises(click.ClickException) as error: with pytest.raises(click.ClickException) as error:
service.get_tracks(title) service.get_tracks(title)
assert "may have expired" in error.value.message assert "may have expired" in error.value.message
def test_a_subtitle_side_load_from_another_tool_is_added(tmp_path: Path) -> None:
"""A foreign export has no track dicts, so a ``tracks[]`` row with a url is the side-load."""
export = {
"kind": "mediaexport",
"version": 1,
"service": {"tag": "EXAMPLE"},
"titles": [
{
"id": "movie-1",
"kind": "clip",
"title": "Example Movie",
"language": "ja",
"manifests": [{"url": "https://example.invalid/m.mpd", "type": "dash"}],
"tracks": [
{"type": "video", "id": "v1", "codec": "avc"},
{
"type": "subtitle",
"id": "sub-en",
"url": "https://example.invalid/en.srt",
"codec": "srt",
"language": "en",
"sdh": True,
},
{"type": "subtitle", "id": "sub-x", "url": "https://example.invalid/x.srt"},
],
}
],
}
path = tmp_path / "export.json"
path.write_text(json.dumps(export), encoding="utf8")
service = ImportService(click.Context(click.Command("dl")), "EXAMPLE", "movie-1", str(path))
tracks = get_tracks(service, MPD_WITH_LANG)
assert [(t.descriptor.name, str(t.language)) for t in tracks.subtitles] == [
("DASH", "ja"),
("URL", "en"),
("URL", "ja"),
]
assert [t.id for t in tracks.subtitles[1:]] == ["sub-en", "sub-x"]
assert tracks.subtitles[1].sdh and tracks.subtitles[1].codec is not None
+115 -47
View File
@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
import html import html
import json
import logging import logging
import math import math
import os import os
@@ -27,6 +26,7 @@ from typing import Any, Callable, Collection, Optional, Sequence, TypedDict, Uni
from uuid import UUID from uuid import UUID
import click import click
import mediaexport
import yaml import yaml
from click.core import ParameterSource from click.core import ParameterSource
from langcodes import Language, tag_is_valid from langcodes import Language, tag_is_valid
@@ -40,7 +40,7 @@ from rich.table import Table
from rich.text import Text from rich.text import Text
from rich.tree import Tree from rich.tree import Tree
from unshackle.core import binaries, providers from unshackle.core import __version__, binaries, providers
from unshackle.core.cdm import DecryptLabsRemoteCDM from unshackle.core.cdm import DecryptLabsRemoteCDM
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_widevine_cdm from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_widevine_cdm
from unshackle.core.config import config, resolve_cdm_name, resolve_decryption from unshackle.core.config import config, resolve_cdm_name, resolve_decryption
@@ -3936,63 +3936,131 @@ class dl:
return meta return meta
def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any = None) -> None: def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any = None) -> None:
"""Write a shareable v2 export usable by ``unshackle import``. """Write a shareable mediaexport file usable by ``unshackle import`` and unidl.
Carries no HTTP session, cookies, or dl-flags. The export records the region (country Carries no HTTP session or cookies. The export records the region (country code)
code) only when the export used ``--proxy``, as an import geofence. Each track records only when the export used ``--proxy``, as an import geofence. DRM init data and
only the licensed DRM system. Content keys live once under the track's ``keys``. ``drm`` may be None content keys live once per title. ``drm`` may be None (DRM-free track) or a DRM
(DRM-free track) or a DRM system without ``to_dict``/``content_keys`` (e.g. ClearKey) - system without ``to_dict``/``content_keys`` (e.g. ClearKey) - the export still
the export still records the track, manifest, chapter and attachment info. records the track, manifest, chapter and attachment info. unshackle's full track
dicts and title meta go under ``x-unshackle`` for its own importer.
""" """
with self.EXPORT_LOCK: with self.EXPORT_LOCK:
doc: dict[str, Any] = {} try:
if export.is_file(): doc = mediaexport.read(export) if export.is_file() else mediaexport.Document(service_tag=self.service)
doc = json.loads(export.read_text(encoding="utf8")) or {} except mediaexport.ExportError as e:
# a download mid-licensing must not die over the file it exports to
self.log.warning(f"Not exporting {title.id}: {export} is not a usable export ({e})")
return
doc.generator.setdefault("app", "unshackle")
doc.generator.setdefault("version", __version__)
if not doc.region and getattr(self, "proxy_requested", False):
doc.region = getattr(getattr(self, "export_service", None), "current_region", None) or ""
doc.setdefault("version", 2) entry = doc.get(str(title.id))
doc.setdefault("service", self.service) if entry is None:
if "region" not in doc and getattr(self, "proxy_requested", False): entry = self.export_entry(title)
region = getattr(getattr(self, "export_service", None), "current_region", None) doc.add(entry)
if region:
doc["region"] = region
titles = doc.setdefault("titles", {}) tracks_map = entry.ext("unshackle").setdefault("tracks", {})
tinfo = titles.setdefault(str(title.id), {}) tracks_map.setdefault(str(track.id), track.to_dict())
tinfo.setdefault("meta", self.title_to_meta(title)) for row in entry.tracks:
if row.get("id") == str(track.id):
row["selected"] = True
if title.tracks.manifest_url:
tinfo.setdefault("manifest_url", title.tracks.manifest_url)
all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]
if "manifest_type" not in tinfo:
tinfo["manifest_type"] = next(
(t.descriptor.name for t in all_tracks if t.descriptor != Video.Descriptor.URL), None
)
tracks_map = tinfo.setdefault("tracks", {})
if not tracks_map:
for t in all_tracks:
tracks_map[str(t.id)] = t.to_dict()
track_data = tracks_map.setdefault(str(track.id), track.to_dict())
if drm is not None: if drm is not None:
if hasattr(drm, "to_dict"): if hasattr(drm, "to_dict"):
track_data["drm"] = [drm.to_dict()] d = drm.to_dict()
own = entry.ext("unshackle").setdefault("drm", [])
if d not in own:
own.append(d)
system = str(d.get("system", "")).lower().replace("cenc", "")
pssh = str(d.get("pssh_b64") or "")
if system and all(x.pssh != pssh or x.system != system for x in entry.drm):
entry.drm.append(mediaexport.Drm(system, pssh))
content_keys = getattr(drm, "content_keys", None) or {} content_keys = getattr(drm, "content_keys", None) or {}
if content_keys: pool = doc.key_pool()
keys = track_data.setdefault("keys", {}) for kid, key in content_keys.items():
for kid, key in content_keys.items(): # a KID that already holds a different key must not be overwritten:
keys[kid.hex] = key # the reader rejects a file that disagrees with itself
if pool.setdefault(kid.hex, key) != key:
self.log.warning(
f"KID {kid.hex} already has a different key in {export}, keeping the existing one"
)
else:
entry.keys[kid.hex] = key
if "chapters" not in tinfo: mediaexport.write(export, doc)
tinfo["chapters"] = [
{"timestamp": chapter.timestamp, "name": chapter.name} for chapter in (title.tracks.chapters or [])
]
if "attachments" not in tinfo: def export_entry(self, title: Title_T) -> mediaexport.Entry:
tinfo["attachments"] = [a.to_dict() for a in (title.tracks.attachments or []) if a.url] """A mediaexport entry for ``title``: manifests, chapters, a small track list, no keys yet."""
meta = self.title_to_meta(title)
all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]
primary_url = title.tracks.manifest_url or next(
(str(t.url) for t in all_tracks if t.descriptor != Video.Descriptor.URL), ""
)
manifest_type = next(
(t.descriptor.name.lower() for t in all_tracks if t.descriptor != Video.Descriptor.URL), ""
)
session = getattr(getattr(self, "export_service", None), "session", None)
# mediaexport drops Cookie and Authorization itself, on write and on read
headers = dict(getattr(session, "headers", None) or {})
manifests = [mediaexport.Manifest(primary_url, manifest_type, headers, role="primary")] if primary_url else []
# the whole URL identifies a manifest: two profiles on one endpoint are two manifests
seen = {primary_url}
for t in all_tracks:
url = str(t.url)
if t.descriptor in (Video.Descriptor.DASH, Video.Descriptor.ISM) and url not in seen:
seen.add(url)
manifests.append(mediaexport.Manifest(url, t.descriptor.name.lower(), role="extra"))
export.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf8") rows = []
for t in all_tracks:
codec = getattr(t, "codec", None)
row: dict[str, Any] = {
"id": str(t.id),
"type": t.__class__.__name__.lower(),
# a subtitle side-load names its file format so another tool can fetch it
"codec": (codec.value if isinstance(t, Subtitle) else codec.name).lower() if codec else "",
"language": str(t.language) if t.language else "",
"bitrate": int(bitrate) if (bitrate := getattr(t, "bitrate", None)) else None,
}
if isinstance(t, Video):
row.update(width=t.width, height=t.height, range=t.range.name.lower() if t.range else "")
elif isinstance(t, Audio):
row.update(channels=str(t.channels) if t.channels else "", atmos=bool(t.joc), descriptive=t.descriptive)
elif isinstance(t, Subtitle):
row.update(sdh=t.sdh, forced=t.forced, cc=t.cc)
if t.descriptor == Video.Descriptor.URL:
row["url"] = str(t.url)
rows.append({k: v for k, v in row.items() if v not in (None, "")})
return mediaexport.Entry(
id=str(title.id),
kind=meta.get("type", "movie"),
title=meta.get("name") or "",
series=meta.get("series_title") or "",
season=meta.get("season"),
episode=meta.get("number"),
year=meta.get("year"),
language=meta.get("language") or "",
artist=meta.get("artist") or "",
album=meta.get("album") or "",
track_number=meta.get("track"),
manifests=manifests,
chapters=[
{"start_ms": mediaexport.ts_ms(c.timestamp), "title": c.name or ""}
for c in (title.tracks.chapters or [])
],
tracks=rows,
extensions={
"x-unshackle": {
"meta": meta,
"tracks": {},
"attachments": [a.to_dict() for a in (title.tracks.attachments or []) if a.url],
}
},
)
def decrypt_verified( def decrypt_verified(
self, self,
+1 -1
View File
@@ -889,7 +889,7 @@ async def download(request: web.Request) -> web.Response:
description: Skip downloading, only retrieve decryption keys (default - false) description: Skip downloading, only retrieve decryption keys (default - false)
export: export:
type: boolean type: boolean
description: Export manifest, track URLs, keys, and subtitles to JSON in the exports directory (default - false) description: Export manifest, DRM init data, keys, and track info to a JSON file in the exports directory (default - false)
cdm_only: cdm_only:
type: boolean type: boolean
description: Only use CDM for content key retrieval (true) or only vaults (false) (default - None) description: Only use CDM for content key retrieval (true) or only vaults (false) (default - None)
+134 -35
View File
@@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
import json
import logging import logging
from http.cookiejar import CookieJar from http.cookiejar import CookieJar
from pathlib import Path from pathlib import Path
@@ -8,6 +7,7 @@ from typing import Any, Optional, Union
from uuid import UUID from uuid import UUID
import click import click
import mediaexport
import requests import requests
from langcodes import tag_is_valid from langcodes import tag_is_valid
@@ -18,7 +18,7 @@ from unshackle.core.drm import drm_from_dict
from unshackle.core.manifests import DASH, HLS, ISM from unshackle.core.manifests import DASH, HLS, ISM
from unshackle.core.remote_service import RemoteService, build_title, match_track, resolve_proxy_arg from unshackle.core.remote_service import RemoteService, build_title, match_track, resolve_proxy_arg
from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
from unshackle.core.tracks import Audio, Chapter, Chapters, Tracks, Video from unshackle.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video
from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.attachment import Attachment
from unshackle.core.tracks.track import Track from unshackle.core.tracks.track import Track
@@ -125,33 +125,72 @@ class ImportService:
if not export_path.is_file(): if not export_path.is_file():
raise click.ClickException(f"Export file not found: {export_path}") raise click.ClickException(f"Export file not found: {export_path}")
self.data: dict[str, Any] = json.loads(export_path.read_text(encoding="utf8")) try:
version = self.data.get("version") self.doc = mediaexport.read(export_path)
if version != 2: except mediaexport.ExportError as e:
raise click.ClickException( raise click.ClickException(f"Cannot import {export_path.name}: {e}")
f"Unsupported export version {version!r}. Re-create the export with a current build."
)
self.titles_data: dict[str, Any] = self.data.get("titles", {}) self.titles_data: dict[str, Any] = {e.id: self.legacy_entry(e) for e in self.doc.titles}
self.region: Optional[str] = self.data.get("region") self.region: Optional[str] = self.doc.region or None
self.titles: Optional[Titles_T] = None self.titles: Optional[Titles_T] = None
self.tracks_by_title: dict[str, Tracks] = {} self.tracks_by_title: dict[str, Tracks] = {}
self._server_cdm = True self._server_cdm = True
self._server_cdm_type = "widevine" self._server_cdm_type = "widevine"
self.session = self.build_session(ctx, self.region) first = self.doc.titles[0].primary if self.doc.titles else None
headers = first.headers if first else {}
self.session = self.build_session(ctx, self.region, headers)
@staticmethod @staticmethod
def build_session(ctx: click.Context, region: Optional[str] = None) -> requests.Session: def legacy_entry(entry: mediaexport.Entry) -> dict[str, Any]:
"""The per-title dict the rest of this class reads, from a mediaexport entry.
unshackle's own exports carry the full title meta and track dicts under
``x-unshackle``; a file from another tool has only the shared fields, so the meta is
rebuilt from those and the track map stays empty (the manifest is parsed instead).
"""
x = entry.extensions.get("x-unshackle") or {}
meta = x.get("meta") or {
"type": entry.kind if entry.kind in ("movie", "episode", "song") else "movie",
"id": entry.id,
"name": entry.title,
"series_title": entry.series,
"season": entry.season,
"number": entry.episode,
"year": entry.year,
"language": entry.language or None,
"artist": entry.artist,
"album": entry.album,
"track": entry.track_number,
}
primary = entry.primary
manifest_type = x.get("manifest_type") or (
(primary.type or mediaexport.guess_type(primary.url)).upper() if primary else ""
)
return {
"meta": meta,
"manifest_url": primary.url if primary else None,
"manifest_type": manifest_type or None,
"tracks": x.get("tracks") or {},
"chapters": entry.chapters,
"attachments": x.get("attachments") or [],
}
@staticmethod
def build_session(
ctx: click.Context, region: Optional[str] = None, headers: Optional[dict[str, str]] = None
) -> requests.Session:
"""HTTP session for re-fetching the manifest. """HTTP session for re-fetching the manifest.
Honours the importer's ``--proxy``. Without one, it falls back to the export region Exported request headers go over the config defaults, since a CDN can gate the
as a geofence. An explicit proxy that unshackle cannot find raises an error. A region manifest on the User-Agent the exporter used. Honours the importer's ``--proxy``.
fallback gives a warning. Without one, it falls back to the export region as a geofence. An explicit proxy
that unshackle cannot find raises an error. A region fallback gives a warning.
""" """
session = requests.Session() session = requests.Session()
session.headers.update(config.headers) session.headers.update(config.headers)
session.headers.update(headers or {})
params = ctx.parent.params if ctx.parent else {} params = ctx.parent.params if ctx.parent else {}
if params.get("no_proxy"): if params.get("no_proxy"):
@@ -225,7 +264,8 @@ class ImportService:
tracks.manifest_url = manifest_url tracks.manifest_url = manifest_url
parser = PARSERS.get(manifest_type or "") parser = PARSERS.get(manifest_type or "")
if manifest_url and parser is not None and manifest_type in ("DASH", "ISM"): # An export from another tool has no track dicts, so HLS is parsed from the manifest too
if manifest_url and parser is not None and (manifest_type in ("DASH", "ISM") or not tracks_map):
try: try:
manifest = parser.from_url(url=manifest_url, session=self.session) manifest = parser.from_url(url=manifest_url, session=self.session)
except Exception as e: except Exception as e:
@@ -267,13 +307,17 @@ class ImportService:
else: else:
track_dicts = list(tracks_map.values()) track_dicts = list(tracks_map.values())
# a side-load from another tool is only a tracks[] row with a url, not a track dict
for sub in self.sideloaded_subtitles(title_id, skip=set(tracks_map)):
tracks.add(sub, warn_only=True)
for track_dict in track_dicts: for track_dict in track_dicts:
try: try:
track = Track.from_dict(track_dict) track = Track.from_dict(track_dict)
except Exception as e: except Exception as e:
self.log.warning(f"Skipping exported track {track_dict.get('id')!r}: {e}") self.log.warning(f"Skipping exported track {track_dict.get('id')!r}: {e}")
continue continue
drm = self.rebuild_drm(track_dict) drm = self.rebuild_drm(track_dict, title_id)
if drm: if drm:
track.drm = drm track.drm = drm
tracks.add(track, warn_only=True) tracks.add(track, warn_only=True)
@@ -306,20 +350,72 @@ class ImportService:
self.tracks_by_title[title_id] = tracks self.tracks_by_title[title_id] = tracks
return tracks return tracks
def sideloaded_subtitles(self, title_id: str, skip: set[str]) -> list[Subtitle]:
"""Subtitle tracks for the ``tracks[]`` rows with a ``url``, except the ids in ``skip``.
The row names the file format as its codec, or nothing, in which case the download
finds it with MediaInfo. A row with no language cannot be muxed, so it is skipped.
"""
entry = self.doc.get(title_id)
out = []
for row in entry.tracks if entry else []:
if row.get("type") != "subtitle" or not row.get("url") or str(row.get("id")) in skip:
continue
language = row.get("language") or (entry.language if entry else "")
if not language:
self.log.warning(f"Skipping exported subtitle {row.get('id')!r}: it has no language")
continue
codec = None
try:
codec = Subtitle.Codec.from_mime(str(row.get("codec") or ""))
except ValueError:
pass
out.append(
Subtitle(
id_=str(row.get("id") or ""),
url=str(row["url"]),
language=str(language),
codec=codec,
sdh=bool(row.get("sdh")),
forced=bool(row.get("forced")),
cc=bool(row.get("cc")),
)
)
return out
def key_pool(self, title_id: Optional[str] = None) -> dict[UUID, str]: def key_pool(self, title_id: Optional[str] = None) -> dict[UUID, str]:
"""Exported KID:KEY pairs as {UUID: key_hex}, for one title or across every title.""" """Exported KID:KEY pairs as {UUID: key_hex}, for one title or across every title."""
pool: dict[UUID, str] = {} entry = self.doc.get(title_id) if title_id else None
entries = [self.titles_data.get(title_id, {})] if title_id else self.titles_data.values() keys = entry.keys if entry else self.doc.key_pool()
for entry in entries: return {UUID(hex=kid): key for kid, key in keys.items()}
for track_dict in (entry.get("tracks") or {}).values():
for kid_hex, key in (track_dict.get("keys") or {}).items():
pool[UUID(hex=kid_hex)] = key
return pool
def rebuild_drm(self, track_dict: dict[str, Any]) -> Optional[list[Any]]: def title_drm_dicts(self, title_id: str) -> list[dict[str, Any]]:
"""Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys.""" """The title's DRM entries in ``drm_from_dict`` form.
keys = track_dict.get("keys") or {}
drm_dicts = track_dict.get("drm") or [] unshackle's own export keeps the exact ``to_dict`` output under ``x-unshackle``, so a
ClearKeyCENC licence URL or a PlayReady PSSH comes back byte for byte.
"""
entry = self.doc.get(title_id)
if entry is None:
return []
own = (entry.extensions.get("x-unshackle") or {}).get("drm")
if own:
return list(own)
out = []
for d in entry.drm:
system = {"widevine": "Widevine", "playready": "PlayReady", "clearkey": "ClearKeyCENC"}.get(d.system)
if system:
out.append({"system": system, "pssh_b64": d.pssh, "kids": list(entry.keys)})
return out
def rebuild_drm(self, track_dict: dict[str, Any], title_id: str) -> Optional[list[Any]]:
"""Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys.
Keys and DRM are title-level in the shared format; a legacy v2 track dict that still
carries its own is honoured first.
"""
keys = track_dict.get("keys") or {kid.hex: key for kid, key in self.key_pool(title_id).items()}
drm_dicts = track_dict.get("drm") or self.title_drm_dicts(title_id)
if not drm_dicts and not keys: if not drm_dicts and not keys:
return None return None
@@ -396,17 +492,20 @@ class ImportService:
def exported_drm_system(self) -> str: def exported_drm_system(self) -> str:
"""The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine.""" """The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine."""
for entry in self.titles_data.values(): for entry in self.doc.titles:
for track_dict in (entry.get("tracks") or {}).values(): for d in entry.drm:
for drm_dict in track_dict.get("drm") or []: if d.system:
if drm_dict.get("system"): return d.system
return drm_dict["system"].lower()
return "widevine" return "widevine"
def get_chapters(self, title: Title_T) -> Chapters: def get_chapters(self, title: Title_T) -> Chapters:
entry = self.titles_data.get(str(title.id), {}) entry = self.doc.get(str(title.id))
return Chapters( return Chapters(
[Chapter(ch["timestamp"], ch.get("name")) for ch in (entry.get("chapters") or []) if ch.get("timestamp")] [
Chapter(int(ch["start_ms"]), ch.get("title") or None)
for ch in (entry.chapters if entry else [])
if ch.get("start_ms") is not None
]
) )
def get_widevine_service_certificate(self, **_: Any) -> Optional[str]: def get_widevine_service_certificate(self, **_: Any) -> Optional[str]:
Generated
+7
View File
@@ -1074,6 +1074,11 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
] ]
[[package]]
name = "mediaexport"
version = "0.1.0"
source = { git = "https://github.com/unshackle-dl/mediaexport?branch=main#07893237199fc865bd5c63b6848235e4f6ef81c8" }
[[package]] [[package]]
name = "more-itertools" name = "more-itertools"
version = "10.8.0" version = "10.8.0"
@@ -2155,6 +2160,7 @@ dependencies = [
{ name = "langcodes" }, { name = "langcodes" },
{ name = "language-data" }, { name = "language-data" },
{ name = "lxml" }, { name = "lxml" },
{ name = "mediaexport" },
{ name = "mutagen" }, { name = "mutagen" },
{ name = "protobuf" }, { name = "protobuf" },
{ name = "pycaption" }, { name = "pycaption" },
@@ -2223,6 +2229,7 @@ requires-dist = [
{ name = "langcodes", specifier = ">=3.4.0,<4" }, { name = "langcodes", specifier = ">=3.4.0,<4" },
{ name = "language-data", specifier = ">=1.4.0" }, { name = "language-data", specifier = ">=1.4.0" },
{ name = "lxml", specifier = ">=5.2.1,<7" }, { name = "lxml", specifier = ">=5.2.1,<7" },
{ name = "mediaexport", git = "https://github.com/unshackle-dl/mediaexport?branch=main" },
{ name = "mutagen", specifier = ">=1.47.0,<2" }, { name = "mutagen", specifier = ">=1.47.0,<2" },
{ name = "protobuf", specifier = ">=4.25.3,<7" }, { name = "protobuf", specifier = ">=4.25.3,<7" },
{ name = "pycaption", specifier = ">=2.2.6,<2.2.27" }, { name = "pycaption", specifier = ">=2.2.6,<2.2.27" },