mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-17 22:45:06 +00:00
551 lines
23 KiB
Python
551 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from http.cookiejar import CookieJar
|
|
from pathlib import Path
|
|
from typing import Any, Optional, Union
|
|
from uuid import UUID
|
|
|
|
import click
|
|
import mediaexport
|
|
import requests
|
|
from langcodes import tag_is_valid
|
|
|
|
from unshackle.core.config import config
|
|
from unshackle.core.constants import AnyTrack
|
|
from unshackle.core.credential import Credential
|
|
from unshackle.core.drm import drm_from_dict
|
|
from unshackle.core.manifests import DASH, HLS, ISM
|
|
from unshackle.core.remote_service import RemoteService, build_title, match_track, resolve_proxy_arg
|
|
from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
|
|
from unshackle.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video
|
|
from unshackle.core.tracks.attachment import Attachment
|
|
from unshackle.core.tracks.track import Track
|
|
|
|
log = logging.getLogger("import")
|
|
|
|
PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM}
|
|
MANIFEST_DATA_KEYS = {"DASH": "dash", "ISM": "ism"}
|
|
|
|
|
|
def fallback_language(language: Any) -> Optional[Any]:
|
|
"""The title language if the parsers would accept it, else None so they raise.
|
|
|
|
DASH drops an und/invalid fallback but ISM does not, and a truthy 'und' there silently
|
|
labels every track und instead of failing.
|
|
"""
|
|
tag = str(language or "").strip()
|
|
if not tag or not tag_is_valid(tag) or tag.startswith("und"):
|
|
return None
|
|
return language
|
|
|
|
|
|
def resolve_import_manifest_data(
|
|
tracks: Tracks,
|
|
manifest_type: Optional[str],
|
|
*,
|
|
session: requests.Session,
|
|
language: Any,
|
|
title: Title_T,
|
|
) -> None:
|
|
"""Fill ``track.data`` for DASH/ISM tracks rebuilt from export dicts.
|
|
|
|
Exports can be without ``manifest_url`` (for example, when the service never set
|
|
``title.tracks.manifest_url``) or carry one MPD per adaptation set. ``Track.from_dict``
|
|
does not serialise manifest XML, so each exported track's ``url`` is re-fetched and
|
|
matched to a locally parsed representation before download.
|
|
"""
|
|
if manifest_type not in MANIFEST_DATA_KEYS:
|
|
return
|
|
|
|
parser = PARSERS[manifest_type]
|
|
data_key = MANIFEST_DATA_KEYS[manifest_type]
|
|
pending = [
|
|
track
|
|
for track in [*tracks.videos, *tracks.audio, *tracks.subtitles]
|
|
if track.descriptor.name == manifest_type and not track.data.get(data_key)
|
|
]
|
|
if not pending:
|
|
return
|
|
|
|
fallback_lang = fallback_language(language)
|
|
for url in {str(track.url) for track in pending if track.url}:
|
|
try:
|
|
manifest = parser.from_url(url=url, session=session)
|
|
parsed = manifest.to_tracks(language=fallback_lang)
|
|
except ValueError as e:
|
|
if "Language information could not be derived" in str(e):
|
|
raise click.ClickException(
|
|
f"No language for '{title}': the {manifest_type} manifest has none and the "
|
|
f"service did not set Title.language."
|
|
)
|
|
raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})")
|
|
except Exception as e:
|
|
raise click.ClickException(
|
|
f"Failed to fetch the {manifest_type} manifest for '{title}'. "
|
|
f"The manifest URL may have expired since export. ({e})"
|
|
)
|
|
|
|
local_tracks = [*parsed.videos, *parsed.audio, *parsed.subtitles]
|
|
for track in pending:
|
|
if track.data.get(data_key) or str(track.url) != url:
|
|
continue
|
|
matched = match_track(track, local_tracks)
|
|
if matched and matched.data.get(data_key):
|
|
track.data.update(matched.data)
|
|
|
|
|
|
class ImportService:
|
|
"""Reconstructs a download from an export JSON.
|
|
|
|
ImportService does not authenticate and does not license. It rebuilds the tracks from
|
|
the export and injects the keys directly. ``_server_cdm``/``_server_cdm_type`` keep their
|
|
underscores: dl.py reads them through getattr as the server-CDM contract that skips
|
|
client licensing.
|
|
"""
|
|
|
|
ALIASES: tuple[str, ...] = ()
|
|
GEOFENCE: tuple[str, ...] = ()
|
|
GEOBLOCK: tuple[str, ...] = ()
|
|
NO_SUBTITLES: bool = False
|
|
|
|
def __init__(self, ctx: click.Context, service_tag: str, title: str, import_file: Optional[str]) -> None:
|
|
self.__class__.__name__ = service_tag
|
|
self.service_tag = service_tag
|
|
self.title_id = title
|
|
self.ctx = ctx
|
|
self.log = logging.getLogger(service_tag)
|
|
self.credential: Optional[Credential] = None
|
|
self.current_region: Optional[str] = None
|
|
self.title_cache = None
|
|
|
|
if not import_file:
|
|
raise click.ClickException("No export file was provided to import from.")
|
|
export_path = Path(import_file)
|
|
if not export_path.is_file():
|
|
raise click.ClickException(f"Export file not found: {export_path}")
|
|
|
|
try:
|
|
self.doc = mediaexport.read(export_path)
|
|
except mediaexport.ExportError as e:
|
|
raise click.ClickException(f"Cannot import {export_path.name}: {e}")
|
|
for refused in self.doc.refused:
|
|
self.log.warning(f"Skipping exported title {refused.raw.get('id')!r}: {refused.reason}")
|
|
if not self.doc.titles:
|
|
raise click.ClickException(f"Cannot import {export_path.name}: it has no title unshackle can use")
|
|
|
|
self.titles_data: dict[str, Any] = {e.id: self.legacy_entry(e) for e in self.doc.titles}
|
|
self.region: Optional[str] = self.doc.region or None
|
|
self.titles: Optional[Titles_T] = None
|
|
self.tracks_by_title: dict[str, Tracks] = {}
|
|
|
|
self._server_cdm = True
|
|
self._server_cdm_type = "widevine"
|
|
|
|
first = self.doc.titles[0].primary if self.doc.titles else None
|
|
headers = first.headers if first else {}
|
|
self.session = self.build_session(ctx, self.region, headers)
|
|
|
|
@staticmethod
|
|
def legacy_entry(entry: mediaexport.Entry) -> dict[str, Any]:
|
|
"""The per-title dict the rest of this class reads, from a mediaexport entry.
|
|
|
|
unshackle's own exports carry the full title meta and track dicts under
|
|
``x-unshackle``; a file from another tool has only the shared fields, so the meta is
|
|
rebuilt from those and the track map stays empty (the manifest is parsed instead).
|
|
"""
|
|
x = entry.extensions.get("x-unshackle") or {}
|
|
meta = x.get("meta") or {
|
|
"type": entry.kind if entry.kind in ("movie", "episode", "song") else "movie",
|
|
"id": entry.id,
|
|
"name": entry.title,
|
|
"series_title": entry.series,
|
|
"season": entry.season,
|
|
"number": entry.episode,
|
|
"year": entry.year,
|
|
"language": entry.language or None,
|
|
"artist": entry.artist,
|
|
"album": entry.album,
|
|
"track": entry.track_number,
|
|
}
|
|
primary = entry.primary
|
|
manifest_type = x.get("manifest_type") or (
|
|
(primary.type or mediaexport.guess_type(primary.url)).upper() if primary else ""
|
|
)
|
|
return {
|
|
"meta": meta,
|
|
"manifest_url": primary.url if primary else None,
|
|
"manifest_type": manifest_type or None,
|
|
"tracks": x.get("tracks") or {},
|
|
"chapters": entry.chapters,
|
|
"attachments": x.get("attachments") or [],
|
|
}
|
|
|
|
@staticmethod
|
|
def build_session(
|
|
ctx: click.Context, region: Optional[str] = None, headers: Optional[dict[str, str]] = None
|
|
) -> requests.Session:
|
|
"""HTTP session for re-fetching the manifest.
|
|
|
|
Exported request headers go over the config defaults, since a CDN can gate the
|
|
manifest on the User-Agent the exporter used. Honours the importer's ``--proxy``.
|
|
Without one, it falls back to the export region as a geofence. An explicit proxy
|
|
that unshackle cannot find raises an error. A region fallback gives a warning.
|
|
"""
|
|
session = requests.Session()
|
|
session.headers.update(config.headers)
|
|
session.headers.update(headers or {})
|
|
|
|
params = ctx.parent.params if ctx.parent else {}
|
|
if params.get("no_proxy"):
|
|
return session
|
|
|
|
explicit = params.get("proxy")
|
|
proxy_query = explicit or region
|
|
if not proxy_query:
|
|
return session
|
|
|
|
try:
|
|
proxy = resolve_proxy_arg(proxy_query)
|
|
except Exception as e:
|
|
if explicit:
|
|
raise click.ClickException(f"Failed to resolve proxy '{proxy_query}': {e}")
|
|
log.warning(f"Could not auto-select a proxy for export region '{region}': {e}. Continuing without proxy.")
|
|
proxy = None
|
|
|
|
if proxy:
|
|
session.proxies.update({"all": proxy})
|
|
if not explicit:
|
|
log.info(f"No --proxy given; using export region '{region}' via your proxy provider.")
|
|
return session
|
|
|
|
@property
|
|
def title(self) -> str:
|
|
return self.title_id
|
|
|
|
def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None:
|
|
self.credential = credential
|
|
|
|
def get_titles(self) -> Titles_T:
|
|
if self.titles is not None:
|
|
return self.titles
|
|
titles_list = [
|
|
build_title(entry.get("meta", {}), self.service_tag, fallback_id=title_id)
|
|
for title_id, entry in self.titles_data.items()
|
|
]
|
|
self.titles = (
|
|
Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list)
|
|
)
|
|
return self.titles
|
|
|
|
def get_titles_cached(self, title_id: Optional[str] = None) -> Titles_T:
|
|
"""Apply the service's title_map to titles reconstructed from the export sidecar."""
|
|
title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {}
|
|
return remap_titles(self.get_titles(), title_map)
|
|
|
|
def get_tracks(self, title: Title_T) -> Tracks:
|
|
"""Reconstruct the title's tracks from the export.
|
|
|
|
DASH/ISM: re-fetch and re-parse ``manifest_url`` for the full ladder on that MPD (the importer
|
|
picks quality with normal dl flags, and injects the keys by KID later), then merge in exported
|
|
DASH/ISM tracks from other MPDs (e.g. a separate HEVC manifest) and direct-URL side-loads.
|
|
A service side-loads its own subtitles when the manifest's are the worse copy, so any
|
|
direct-URL subtitle in the export means unshackle drops the whole re-parsed subtitle
|
|
set in favour of the exported one.
|
|
HLS/URL: rebuild from the stored per-track dicts, since the variant is re-fetched from
|
|
track.url at download time and a master playlist can hand out a new token on every fetch.
|
|
"""
|
|
title_id = str(title.id)
|
|
if title_id in self.tracks_by_title:
|
|
return self.tracks_by_title[title_id]
|
|
|
|
entry = self.titles_data.get(title_id, {})
|
|
tracks_map: dict[str, Any] = entry.get("tracks") or {}
|
|
manifest_url = entry.get("manifest_url")
|
|
manifest_type = entry.get("manifest_type")
|
|
|
|
tracks = Tracks()
|
|
tracks.manifest_url = manifest_url
|
|
|
|
parser = PARSERS.get(manifest_type or "")
|
|
# An export from another tool has no track dicts, so HLS is parsed from the manifest too
|
|
if manifest_url and parser is not None and (manifest_type in ("DASH", "ISM") or not tracks_map):
|
|
try:
|
|
manifest = parser.from_url(url=manifest_url, session=self.session)
|
|
except Exception as e:
|
|
raise click.ClickException(
|
|
f"Failed to fetch the {manifest_type} manifest for '{title}'. "
|
|
f"The manifest URL may have expired since export. ({e})"
|
|
)
|
|
try:
|
|
parsed = manifest.to_tracks(language=fallback_language(title.language))
|
|
except ValueError as e:
|
|
if "Language information could not be derived" in str(e):
|
|
raise click.ClickException(
|
|
f"No language for '{title}': the {manifest_type} manifest has none and the "
|
|
f"service did not set Title.language."
|
|
)
|
|
raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})")
|
|
except Exception as e:
|
|
raise click.ClickException(f"Failed to parse the {manifest_type} manifest for '{title}'. ({e})")
|
|
for track in parsed:
|
|
tracks.add(track)
|
|
parsed_ids = {str(t.id) for t in tracks}
|
|
url_dicts = [t for t in tracks_map.values() if t.get("descriptor", "URL") == "URL"]
|
|
# Merge back only tracks from another manifest, such as a separate HEVC one; the
|
|
# re-parse already covers manifest_url's own. Query stripped when comparing, since a
|
|
# service can sign the URL it fetches and register the bare one for the same manifest.
|
|
manifest_base = str(manifest_url).split("?")[0]
|
|
foreign_dicts = [
|
|
t
|
|
for t in tracks_map.values()
|
|
if t.get("descriptor") == manifest_type
|
|
and str(t.get("url")).split("?")[0] != manifest_base
|
|
and str(t.get("id")) not in parsed_ids
|
|
]
|
|
track_dicts = [*url_dicts, *foreign_dicts]
|
|
# Only a side-loaded subtitle replaces the re-parsed set, since the service chose its
|
|
# own copy over the manifest's. A subtitle on another manifest is an addition.
|
|
if any(t.get("type") == "Subtitle" for t in url_dicts):
|
|
tracks.subtitles.clear()
|
|
else:
|
|
track_dicts = list(tracks_map.values())
|
|
|
|
# a side-load from another tool is only a tracks[] row with a url, not a track dict
|
|
for sub in self.sideloaded_subtitles(title_id, skip=set(tracks_map)):
|
|
tracks.add(sub, warn_only=True)
|
|
|
|
for track_dict in track_dicts:
|
|
try:
|
|
track = Track.from_dict(track_dict)
|
|
except Exception as e:
|
|
self.log.warning(f"Skipping exported track {track_dict.get('id')!r}: {e}")
|
|
continue
|
|
drm = self.rebuild_drm(track_dict, title_id)
|
|
if drm:
|
|
track.drm = drm
|
|
tracks.add(track, warn_only=True)
|
|
|
|
resolve_import_manifest_data(
|
|
tracks,
|
|
manifest_type,
|
|
session=self.session,
|
|
language=title.language,
|
|
title=title,
|
|
)
|
|
|
|
for attachment in entry.get("attachments") or []:
|
|
url = attachment.get("url")
|
|
if not url:
|
|
continue
|
|
try:
|
|
tracks.attachments.append(
|
|
Attachment.from_url(
|
|
url,
|
|
name=attachment.get("name"),
|
|
mime_type=attachment.get("mime_type"),
|
|
description=attachment.get("description"),
|
|
session=self.session,
|
|
)
|
|
)
|
|
except Exception as e:
|
|
self.log.warning(f"Skipping attachment '{attachment.get('name')}': {e}")
|
|
|
|
self.tracks_by_title[title_id] = tracks
|
|
return tracks
|
|
|
|
def sideloaded_subtitles(self, title_id: str, skip: set[str]) -> list[Subtitle]:
|
|
"""Subtitle tracks for the ``tracks[]`` rows with a ``url``, except the ids in ``skip``.
|
|
|
|
The row names the file format as its codec, or nothing, in which case the download
|
|
finds it with MediaInfo. A row with no language cannot be muxed, so it is skipped.
|
|
"""
|
|
entry = self.doc.get(title_id)
|
|
out = []
|
|
for row in entry.tracks if entry else []:
|
|
if row.get("type") != "subtitle" or not row.get("url") or str(row.get("id")) in skip:
|
|
continue
|
|
language = row.get("language") or (entry.language if entry else "")
|
|
if not language:
|
|
self.log.warning(f"Skipping exported subtitle {row.get('id')!r}: it has no language")
|
|
continue
|
|
codec = None
|
|
try:
|
|
codec = Subtitle.Codec.from_mime(str(row.get("codec") or ""))
|
|
except ValueError:
|
|
pass
|
|
out.append(
|
|
Subtitle(
|
|
id_=str(row.get("id") or ""),
|
|
url=str(row["url"]),
|
|
language=str(language),
|
|
codec=codec,
|
|
sdh=bool(row.get("sdh")),
|
|
forced=bool(row.get("forced")),
|
|
cc=bool(row.get("cc")),
|
|
)
|
|
)
|
|
return out
|
|
|
|
def key_pool(self, title_id: Optional[str] = None) -> dict[UUID, str]:
|
|
"""Exported KID:KEY pairs as {UUID: key_hex}, for one title or across every title."""
|
|
entry = self.doc.get(title_id) if title_id else None
|
|
keys = entry.keys if entry else self.doc.key_pool()
|
|
return {UUID(hex=kid): key for kid, key in keys.items()}
|
|
|
|
def title_drm_dicts(self, title_id: str) -> list[dict[str, Any]]:
|
|
"""The title's DRM entries in ``drm_from_dict`` form.
|
|
|
|
unshackle's own export keeps the exact ``to_dict`` output under ``x-unshackle``, so a
|
|
ClearKeyCENC licence URL or a PlayReady PSSH comes back byte for byte.
|
|
"""
|
|
entry = self.doc.get(title_id)
|
|
if entry is None:
|
|
return []
|
|
own = (entry.extensions.get("x-unshackle") or {}).get("drm")
|
|
if own:
|
|
return list(own)
|
|
out = []
|
|
for d in entry.drm:
|
|
system = {"widevine": "Widevine", "playready": "PlayReady", "clearkey": "ClearKeyCENC"}.get(d.system)
|
|
if system:
|
|
out.append({"system": system, "pssh_b64": d.pssh, "kids": list(entry.keys)})
|
|
return out
|
|
|
|
def rebuild_drm(self, track_dict: dict[str, Any], title_id: str) -> Optional[list[Any]]:
|
|
"""Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys.
|
|
|
|
Keys and DRM are title-level in the shared format; a legacy v2 track dict that still
|
|
carries its own is honoured first.
|
|
"""
|
|
keys = track_dict.get("keys") or {kid.hex: key for kid, key in self.key_pool(title_id).items()}
|
|
drm_dicts = track_dict.get("drm") or self.title_drm_dicts(title_id)
|
|
if not drm_dicts and not keys:
|
|
return None
|
|
|
|
drm_obj = None
|
|
if drm_dicts:
|
|
try:
|
|
drm_obj = drm_from_dict(drm_dicts[0])
|
|
except Exception as e:
|
|
self.log.debug(f"Falling back to DRM stub (PSSH rebuild failed: {e})")
|
|
|
|
if drm_obj is None and keys:
|
|
drm_type = (drm_dicts[0].get("system", "Widevine").lower()) if drm_dicts else "widevine"
|
|
drm_obj = RemoteService.create_drm_stub(drm_type, list(keys.keys()))
|
|
|
|
if drm_obj is None:
|
|
return None
|
|
|
|
for kid_hex, key in keys.items():
|
|
drm_obj.content_keys[UUID(hex=kid_hex)] = key
|
|
return [drm_obj]
|
|
|
|
def resolve_server_keys(self, title: Title_T) -> None:
|
|
"""Inject exported keys into the selected encrypted tracks by KID (no network).
|
|
|
|
dl.py calls this method after selection. It touches only encrypted video and audio
|
|
tracks. Encrypted DASH tracks (no DRM at parse time) get a stub holding the keys,
|
|
which DASH.download_track preserves. decrypt() applies the content key whose KID
|
|
matches the media.
|
|
"""
|
|
pool = self.key_pool(str(title.id)) or self.key_pool()
|
|
if not pool:
|
|
return
|
|
|
|
system = self.exported_drm_system()
|
|
kid_hexes = [kid.hex for kid in pool]
|
|
|
|
for track in title.tracks:
|
|
if not isinstance(track, (Video, Audio)) or not self.track_is_encrypted(track, self.session):
|
|
continue
|
|
drm_obj = track.drm[0] if track.drm else RemoteService.create_drm_stub(system, kid_hexes)
|
|
for kid, key in pool.items():
|
|
drm_obj.content_keys[kid] = key
|
|
track.drm = [drm_obj]
|
|
self._server_cdm_type = drm_obj.__class__.__name__.lower()
|
|
|
|
@staticmethod
|
|
def track_is_encrypted(track: Any, session: Optional[requests.Session] = None) -> bool:
|
|
"""True if the track carries DRM, its manifest declares protection, or its init segment does.
|
|
|
|
This method examines ISM as well as DASH, because ISM.download_track reads only
|
|
``track.drm``. A rung that misses content key injection here downloads encrypted and
|
|
muxes without error. A DASH manifest with no ContentProtection can still describe an
|
|
encrypted track, with the PSSH only in the init segment, so that case probes the init
|
|
segment with `session`.
|
|
"""
|
|
if track.drm:
|
|
return True
|
|
data = getattr(track, "data", None) or {}
|
|
dash = data.get("dash")
|
|
if dash:
|
|
for element in (dash.get("representation"), dash.get("adaptation_set")):
|
|
if element is not None and element.findall("ContentProtection"):
|
|
return True
|
|
from unshackle.core.api.handlers import drm_from_init_segment
|
|
|
|
if drm_from_init_segment(track, session):
|
|
return True
|
|
ism = data.get("ism")
|
|
if ism:
|
|
manifest = ism.get("manifest")
|
|
if manifest is not None and manifest.findall(".//ProtectionHeader"):
|
|
return True
|
|
return False
|
|
|
|
def exported_drm_system(self) -> str:
|
|
"""The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine."""
|
|
for entry in self.doc.titles:
|
|
for d in entry.drm:
|
|
if d.system:
|
|
return d.system
|
|
return "widevine"
|
|
|
|
def get_chapters(self, title: Title_T) -> Chapters:
|
|
entry = self.doc.get(str(title.id))
|
|
return Chapters(
|
|
[
|
|
Chapter(int(ch["start_ms"]), ch.get("title") or None)
|
|
for ch in (entry.chapters if entry else [])
|
|
if ch.get("start_ms") is not None
|
|
]
|
|
)
|
|
|
|
def get_widevine_service_certificate(self, **_: Any) -> Optional[str]:
|
|
return None
|
|
|
|
def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
|
|
raise RuntimeError("ImportService should not request a license; keys come from the export.")
|
|
|
|
def get_playready_license(
|
|
self, *, challenge: bytes, title: Title_T, track: AnyTrack
|
|
) -> Optional[Union[bytes, str]]:
|
|
raise RuntimeError("ImportService should not request a license; keys come from the export.")
|
|
|
|
def get_clearkey_license(
|
|
self, *, challenge: bytes, title: Title_T, track: AnyTrack
|
|
) -> Optional[Union[bytes, str, dict]]:
|
|
raise RuntimeError("ImportService should not request a license; keys come from the export.")
|
|
|
|
def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None:
|
|
pass
|
|
|
|
def on_track_downloaded(self, track: AnyTrack) -> None:
|
|
pass
|
|
|
|
def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None:
|
|
pass
|
|
|
|
def on_track_repacked(self, track: AnyTrack) -> None:
|
|
pass
|
|
|
|
def on_track_multiplex(self, track: AnyTrack) -> None:
|
|
pass
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
self.session.close()
|
|
except Exception:
|
|
pass
|