mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-14 21:15:06 +00:00
1456 lines
63 KiB
Python
1456 lines
63 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import html
|
|
import logging
|
|
import math
|
|
import re
|
|
import shutil
|
|
import sys
|
|
from concurrent.futures import Future
|
|
from contextlib import ExitStack, closing
|
|
from copy import deepcopy
|
|
from functools import lru_cache, partial
|
|
from pathlib import Path
|
|
from typing import Any, BinaryIO, Callable, Optional, Union
|
|
from urllib.parse import urljoin, urlparse
|
|
from uuid import UUID
|
|
from zlib import crc32
|
|
|
|
import requests
|
|
from langcodes import Language, tag_is_valid
|
|
from lxml.etree import Element, ElementTree, XMLSyntaxError
|
|
from pyplayready.system.pssh import PSSH as PR_PSSH
|
|
from pywidevine.cdm import Cdm as WidevineCdm
|
|
from pywidevine.pssh import PSSH
|
|
from requests import Session
|
|
|
|
from unshackle.core import binaries
|
|
from unshackle.core.config import config
|
|
from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
|
|
from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine
|
|
from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
|
|
from unshackle.core.drm.verify import decrypt_track
|
|
from unshackle.core.events import events
|
|
from unshackle.core.session import RnetSession
|
|
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
|
from unshackle.core.tracks.track import (
|
|
DRM_PREFERENCE_TYPES,
|
|
assert_fragments_decrypted,
|
|
parse_sidx,
|
|
)
|
|
from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8
|
|
from unshackle.core.utils.redact import safe_display_url
|
|
from unshackle.core.utils.xml import load_xml
|
|
|
|
|
|
def append_segment(dst: BinaryIO, segment_file: Path, is_text_subtitle: bool) -> None:
|
|
"""Append one segment file to the output, with the text subtitle fixups when they apply."""
|
|
if is_text_subtitle:
|
|
segment_data = try_ensure_utf8(segment_file.read_bytes())
|
|
segment_data = (
|
|
segment_data.decode("utf8")
|
|
.replace("‎", html.unescape("‎"))
|
|
.replace("‏", html.unescape("‏"))
|
|
.encode("utf8")
|
|
)
|
|
dst.write(segment_data)
|
|
else:
|
|
with open(segment_file, "rb") as src:
|
|
shutil.copyfileobj(src, dst, 1024 * 1024)
|
|
|
|
|
|
class RollingMerge:
|
|
"""Appends segments to the output in index order as they arrive, out of order, and unlinks them.
|
|
|
|
A cursor holds the next expected index; every `add` records one ready segment and then drains
|
|
every contiguous ready segment from the cursor onward. A segment that carries a future (per-segment
|
|
decryption) waits for that future before the append, so the output never holds raw bytes.
|
|
|
|
`append(dst, segment_file)` writes one segment; the default is a plain byte copy. A parser
|
|
whose merge needs a per-segment transform, or a head that only exists once segment 0 has
|
|
landed (ISM), passes its own.
|
|
"""
|
|
|
|
def __init__(self, dst: BinaryIO, append: Optional[Callable[[BinaryIO, Path], None]] = None) -> None:
|
|
self.dst = dst
|
|
self.append = append or (lambda dst, path: append_segment(dst, path, is_text_subtitle=False))
|
|
self.cursor = 0
|
|
self.ready: dict[int, tuple[Path, Optional[Future]]] = {}
|
|
|
|
def add(self, index: int, segment_file: Path, future: Optional[Future] = None) -> None:
|
|
# runs on the downloader's own loop, so a decrypt wait or a catch-up burst
|
|
# delays new submissions; move the drain to a writer thread if throughput measurably drops
|
|
self.ready[index] = (segment_file, future)
|
|
while self.cursor in self.ready:
|
|
path, pending = self.ready.pop(self.cursor)
|
|
if pending is not None:
|
|
pending.result()
|
|
self.append(self.dst, path)
|
|
self.cursor += 1
|
|
# the bytes are already in the output; a handle another process still holds on
|
|
# the just-replaced file (Windows) must not fail the track, the segment dir sweep
|
|
# after the download removes any leftover
|
|
try:
|
|
path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
@property
|
|
def merged(self) -> int:
|
|
return self.cursor
|
|
|
|
|
|
class DASH:
|
|
def __init__(self, manifest, url: str):
|
|
if manifest is None:
|
|
raise ValueError("DASH manifest must be provided.")
|
|
if manifest.tag != "MPD":
|
|
raise TypeError(f"Expected 'MPD' document, but received a '{manifest.tag}' document instead.")
|
|
|
|
if not url:
|
|
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
|
|
if not isinstance(url, str):
|
|
raise TypeError(f"Expected url to be a {str}, not {url!r}")
|
|
|
|
self.manifest = manifest
|
|
self.url = url
|
|
|
|
@classmethod
|
|
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH:
|
|
"""Fetch and parse an MPD. The XML declaration gives the encoding, then the HTTP charset."""
|
|
if not url:
|
|
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
|
|
if not isinstance(url, str):
|
|
raise TypeError(f"Expected url to be a {str}, not {url!r}")
|
|
|
|
if not session:
|
|
session = Session()
|
|
elif not isinstance(session, (Session, RnetSession)):
|
|
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
|
|
|
|
res = session.get(url, **args)
|
|
if res.url != url:
|
|
url = res.url
|
|
|
|
if not res.ok:
|
|
raise requests.ConnectionError("Failed to request the MPD document.", response=res)
|
|
|
|
log_event(
|
|
"manifest_dash_fetch",
|
|
level="DEBUG",
|
|
message=f"Fetched DASH manifest ({len(res.content)} bytes)",
|
|
context={"url": safe_display_url(url), "size": len(res.content)},
|
|
)
|
|
|
|
try:
|
|
return cls(load_xml(res.content), url)
|
|
except XMLSyntaxError:
|
|
# no declaration and not UTF-8: only the HTTP charset can name the encoding
|
|
return cls.from_text(res.text, url)
|
|
|
|
@classmethod
|
|
def from_text(cls, text: str, url: str) -> DASH:
|
|
if not text:
|
|
raise ValueError("DASH manifest Text must be provided.")
|
|
if not isinstance(text, str):
|
|
raise TypeError(f"Expected text to be a {str}, not {text!r}")
|
|
|
|
if not url:
|
|
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
|
|
if not isinstance(url, str):
|
|
raise TypeError(f"Expected url to be a {str}, not {url!r}")
|
|
|
|
manifest = load_xml(text)
|
|
|
|
return cls(manifest, url)
|
|
|
|
def to_tracks(
|
|
self, language: Optional[Union[str, Language]] = None, period_filter: Optional[Callable] = None
|
|
) -> Tracks:
|
|
"""
|
|
Convert an MPEG-DASH document to Video, Audio and Subtitle Track objects.
|
|
|
|
Parameters:
|
|
language: The Title's Original Recorded Language. unshackle also uses it as a
|
|
fallback track language value when the manifest gives no language information.
|
|
period_filter: Filter out period's within the manifest.
|
|
|
|
Only the first main-content period becomes tracks. Later periods do not add tracks of their
|
|
own, but download_track stitches the segments of every content period into each returned
|
|
track.
|
|
|
|
All Track URLs will be a list of segment URLs.
|
|
"""
|
|
tracks = Tracks()
|
|
|
|
filtered_period_ids: list[str] = []
|
|
|
|
for period in self.manifest.findall("Period"):
|
|
if callable(period_filter) and period_filter(period):
|
|
if period_id := period.get("id"):
|
|
filtered_period_ids.append(period_id)
|
|
continue
|
|
if not DASH.is_content_period(period, []):
|
|
if period_id := period.get("id"):
|
|
filtered_period_ids.append(period_id)
|
|
continue
|
|
|
|
for adaptation_set in period.findall("AdaptationSet"):
|
|
if self.is_trick_mode(adaptation_set):
|
|
# we don't want trick mode streams (they are only used for fast-forward/rewind)
|
|
continue
|
|
|
|
for rep in adaptation_set.findall("Representation"):
|
|
get = partial(self.get_attr, adaptation_set=adaptation_set, representation=rep)
|
|
findall = partial(self.find_elements, adaptation_set=adaptation_set, representation=rep, both=True)
|
|
segment_base = rep.find("SegmentBase")
|
|
|
|
codecs = get("codecs")
|
|
content_type = get("contentType")
|
|
mime_type = get("mimeType")
|
|
|
|
if not content_type and mime_type:
|
|
content_type = mime_type.split("/")[0]
|
|
if not content_type and not mime_type:
|
|
raise ValueError("Unable to determine the format of a Representation, cannot continue...")
|
|
|
|
if mime_type == "application/mp4" or content_type == "application":
|
|
# likely mp4-boxed subtitles
|
|
# TODO: It may not actually be subtitles
|
|
try:
|
|
real_codec = Subtitle.Codec.from_mime(codecs)
|
|
content_type = "text"
|
|
mime_type = f"application/mp4; codecs='{real_codec.value.lower()}'"
|
|
except ValueError:
|
|
raise ValueError(f"Unsupported content type '{content_type}' with codecs of '{codecs}'")
|
|
|
|
if content_type == "text" and mime_type and "/mp4" not in mime_type:
|
|
# mimeType likely specifies the subtitle codec better than `codecs`
|
|
codecs = mime_type.split("/")[1]
|
|
|
|
if content_type == "video":
|
|
track_type = Video
|
|
track_codec = Video.Codec.from_codecs(codecs)
|
|
track_fps = get("frameRate")
|
|
if not track_fps and segment_base is not None:
|
|
track_fps = segment_base.get("timescale")
|
|
|
|
scan_type = None
|
|
scan_type_str = get("scanType")
|
|
if scan_type_str and scan_type_str.lower() == "interlaced":
|
|
scan_type = Video.ScanType.INTERLACED
|
|
|
|
track_args = dict(
|
|
range_=self.get_video_range(
|
|
codecs, findall("SupplementalProperty"), findall("EssentialProperty")
|
|
),
|
|
bitrate=get("bandwidth") or None,
|
|
width=get("width") or 0,
|
|
height=get("height") or 0,
|
|
fps=track_fps or None,
|
|
scan_type=scan_type,
|
|
)
|
|
elif content_type == "audio":
|
|
track_type = Audio
|
|
track_codec = Audio.Codec.from_codecs(codecs)
|
|
track_args = dict(
|
|
bitrate=get("bandwidth") or None,
|
|
channels=next(
|
|
iter(
|
|
rep.xpath("AudioChannelConfiguration/@value")
|
|
or adaptation_set.xpath("AudioChannelConfiguration/@value")
|
|
),
|
|
None,
|
|
),
|
|
joc=self.get_ddp_complexity_index(adaptation_set, rep),
|
|
descriptive=self.is_descriptive(adaptation_set),
|
|
)
|
|
elif content_type == "text":
|
|
track_type = Subtitle
|
|
track_codec = Subtitle.Codec.from_codecs(codecs or "vtt")
|
|
track_args = dict(
|
|
cc=self.is_closed_caption(adaptation_set),
|
|
sdh=self.is_sdh(adaptation_set),
|
|
forced=self.is_forced(adaptation_set),
|
|
)
|
|
elif content_type == "image":
|
|
# we don't want what's likely thumbnails for the seekbar
|
|
continue
|
|
else:
|
|
raise ValueError(f"Unknown Track Type '{content_type}'")
|
|
|
|
track_lang = self.get_language(adaptation_set, rep, fallback=language)
|
|
if not track_lang:
|
|
msg = "Language information could not be derived from a Representation."
|
|
if language is None:
|
|
msg += " No fallback language was provided when calling DASH.to_tracks()."
|
|
elif not tag_is_valid((str(language) or "").strip()) or str(language).startswith("und"):
|
|
msg += f" The fallback language provided is also invalid: {language}"
|
|
raise ValueError(msg)
|
|
|
|
# for some reason it's incredibly common for services to not provide
|
|
# a good and actually unique track ID, sometimes because of the lang
|
|
# dialect not being represented in the id, or the bitrate, or such.
|
|
# this combines all of them as one and hashes it to keep it small(ish).
|
|
track_id = hex(
|
|
crc32(
|
|
"{codec}-{lang}-{bitrate}-{base_url}-{ids}-{track_args}".format(
|
|
codec=codecs,
|
|
lang=track_lang,
|
|
bitrate=get("bitrate"),
|
|
base_url=(rep.findtext("BaseURL") or "").split("?")[0],
|
|
ids=[get("audioTrackId"), get("id"), period.get("id")],
|
|
track_args=track_args,
|
|
).encode()
|
|
)
|
|
)[2:]
|
|
|
|
tracks.add(
|
|
track_type(
|
|
id_=track_id,
|
|
url=self.url,
|
|
codec=track_codec,
|
|
language=track_lang,
|
|
is_original_lang=bool(language and is_close_match(track_lang, [language])),
|
|
descriptor=Video.Descriptor.DASH,
|
|
data={
|
|
"dash": {
|
|
"manifest": self.manifest,
|
|
"period": period,
|
|
"adaptation_set": adaptation_set,
|
|
"representation": rep,
|
|
"representation_id": rep.get("id"),
|
|
"filtered_period_ids": filtered_period_ids,
|
|
}
|
|
},
|
|
**track_args,
|
|
)
|
|
)
|
|
|
|
break
|
|
|
|
tracks.manifest_url = self.url
|
|
|
|
log_event(
|
|
"manifest_dash_parse",
|
|
level="INFO",
|
|
message=(
|
|
f"Parsed DASH manifest: {len(tracks.videos)} video, "
|
|
f"{len(tracks.audio)} audio, {len(tracks.subtitles)} subtitle track(s)"
|
|
),
|
|
context={
|
|
"videos": len(tracks.videos),
|
|
"audio": len(tracks.audio),
|
|
"subtitles": len(tracks.subtitles),
|
|
"ranges": sorted({str(v.range) for v in tracks.videos}),
|
|
"vcodecs": sorted({str(v.codec) for v in tracks.videos}),
|
|
},
|
|
)
|
|
return tracks
|
|
|
|
@staticmethod
|
|
def download_track(track: AnyTrack, ctx: DownloadContext) -> None:
|
|
session = ctx.ensure_session()
|
|
save_path = ctx.save_path
|
|
save_dir = ctx.save_dir
|
|
progress = ctx.progress
|
|
proxy = ctx.proxy
|
|
max_workers = ctx.max_workers
|
|
adaptive = ctx.adaptive_workers
|
|
processes = ctx.download_processes
|
|
license_widevine = ctx.license_widevine
|
|
cdm = ctx.cdm
|
|
|
|
if proxy:
|
|
session.proxies.update({"all": proxy})
|
|
|
|
log = logging.getLogger("DASH")
|
|
|
|
manifest: ElementTree = track.data["dash"]["manifest"]
|
|
adaptation_set: Element = track.data["dash"]["adaptation_set"]
|
|
representation: Element = track.data["dash"]["representation"]
|
|
rep_id: Optional[str] = track.data["dash"].get("representation_id") or representation.get("id")
|
|
filtered_period_ids: list[str] = track.data["dash"].get("filtered_period_ids", [])
|
|
|
|
# Preserve existing DRM if it was set by the service, especially when service set Widevine
|
|
# but manifest only contains PlayReady protection (common scenario for some services)
|
|
existing_drm = track.drm
|
|
manifest_drm = DASH.get_drm(
|
|
representation.findall("ContentProtection") + adaptation_set.findall("ContentProtection")
|
|
)
|
|
|
|
# Only override existing DRM if:
|
|
# 1. No existing DRM was set, OR
|
|
# 2. Existing DRM contains same type as manifest DRM, OR
|
|
# 3. Existing DRM is not Widevine (preserve Widevine when service explicitly set it)
|
|
should_override_drm = (
|
|
not existing_drm
|
|
or (
|
|
existing_drm
|
|
and manifest_drm
|
|
and any(isinstance(existing, type(manifest)) for existing in existing_drm for manifest in manifest_drm)
|
|
)
|
|
or (existing_drm and not any(isinstance(drm, Widevine) for drm in existing_drm))
|
|
)
|
|
|
|
pre_existing_keys = {}
|
|
if existing_drm:
|
|
for drm_obj in existing_drm:
|
|
if hasattr(drm_obj, "content_keys") and drm_obj.content_keys:
|
|
pre_existing_keys.update(drm_obj.content_keys)
|
|
|
|
if should_override_drm:
|
|
track.drm = manifest_drm
|
|
else:
|
|
track.drm = existing_drm
|
|
if DOWNLOAD_ALL_DRM.is_set():
|
|
track.drm = [
|
|
*existing_drm,
|
|
*(d for d in manifest_drm if not any(type(d) is type(e) for e in existing_drm)),
|
|
]
|
|
|
|
if track.drm and track.drm_preference and not DOWNLOAD_ALL_DRM.is_set():
|
|
wanted = DRM_PREFERENCE_TYPES[track.drm_preference]
|
|
preferred = [drm_obj for drm_obj in track.drm if isinstance(drm_obj, wanted)]
|
|
if preferred:
|
|
track.drm = preferred
|
|
else:
|
|
log.warning(
|
|
f"Track wants {track.drm_preference} DRM but this manifest does not offer it, "
|
|
"using the manifest's DRM instead"
|
|
)
|
|
|
|
if pre_existing_keys and track.drm:
|
|
for drm_obj in track.drm:
|
|
if hasattr(drm_obj, "content_keys"):
|
|
for kid, key in pre_existing_keys.items():
|
|
if kid not in drm_obj.content_keys:
|
|
drm_obj.content_keys[kid] = key
|
|
|
|
all_periods = manifest.findall("Period")
|
|
segments: list[tuple[str, Optional[str]]] = []
|
|
seen_segments: set[tuple[str, Optional[str]]] = set()
|
|
segment_durations: list[int] = []
|
|
segment_timescale: float = 0
|
|
init_data: Optional[bytes] = None
|
|
track_kid: Optional[UUID] = None
|
|
|
|
content_periods = [p for p in all_periods if DASH.is_content_period(p, filtered_period_ids)]
|
|
period_count = len(content_periods)
|
|
|
|
if period_count > 1:
|
|
log.debug(f"Multi-period manifest detected with {period_count} content periods")
|
|
|
|
stored_period: Optional[Element] = track.data["dash"].get("period")
|
|
|
|
for period_idx, content_period in enumerate(content_periods):
|
|
if content_period is stored_period:
|
|
matched_as, matched_rep = adaptation_set, representation
|
|
else:
|
|
matched_as, matched_rep = DASH.resolve_representation(content_period, rep_id, adaptation_set) or (
|
|
None,
|
|
None,
|
|
)
|
|
|
|
if matched_rep is None or matched_as is None:
|
|
period_id = content_period.get("id", period_idx)
|
|
log.warning(f"Representation '{rep_id}' not found in period '{period_id}', skipping")
|
|
continue
|
|
|
|
p_init, p_segments, p_timescale, p_durations, p_kid = DASH.get_period_segments(
|
|
period=content_period,
|
|
adaptation_set=matched_as,
|
|
representation=matched_rep,
|
|
manifest=manifest,
|
|
track=track,
|
|
track_url=track.url,
|
|
session=session,
|
|
probe_kid=period_idx == 0,
|
|
window=config.sample,
|
|
)
|
|
|
|
if period_idx == 0:
|
|
# First period: use its init data and KID for DRM licensing
|
|
init_data = p_init
|
|
track_kid = p_kid
|
|
segment_timescale = p_timescale
|
|
|
|
for seg in p_segments:
|
|
if seg not in seen_segments:
|
|
seen_segments.add(seg)
|
|
segments.append(seg)
|
|
segment_durations.extend(p_durations)
|
|
# ponytail: a sample comes from the first content period only; map the window across
|
|
# periods if a manifest with a short first period needs it
|
|
if config.sample and segments:
|
|
break
|
|
|
|
if not segments:
|
|
log.error("Could not find a way to get segments from this MPD manifest.")
|
|
log.debug(track.url)
|
|
sys.exit(1)
|
|
|
|
if config.sample and not segment_durations and not isinstance(track, Subtitle):
|
|
log.error("--sample cannot cut this track: the manifest gives it as one file with no segment index.")
|
|
sys.exit(1)
|
|
|
|
# TODO: Should we floor/ceil/round, or is int() ok?
|
|
track.data["dash"]["timescale"] = int(segment_timescale)
|
|
track.data["dash"]["segment_durations"] = segment_durations
|
|
|
|
if not track.drm and init_data and isinstance(track, (Video, Audio)) and not binaries.FFProbe:
|
|
log.warning("FFprobe was not found, so the init segment was not probed for a PSSH.")
|
|
elif DOWNLOAD_ALL_DRM.is_set() and init_data and binaries.FFProbe and isinstance(track, (Video, Audio)):
|
|
track.add_init_drm(init_data, cdm)
|
|
elif not track.drm and init_data and isinstance(track, (Video, Audio)):
|
|
prefers_playready = track.prefers_playready(cdm)
|
|
if prefers_playready:
|
|
try:
|
|
track.drm = [PlayReady.from_init_data(init_data)]
|
|
except PlayReady.Exceptions.PSSHNotFound:
|
|
try:
|
|
track.drm = [Widevine.from_init_data(init_data)]
|
|
except Widevine.Exceptions.PSSHNotFound:
|
|
log.debug("No PlayReady or Widevine PSSH was found for this track, is it DRM free?")
|
|
else:
|
|
try:
|
|
track.drm = [Widevine.from_init_data(init_data)]
|
|
except Widevine.Exceptions.PSSHNotFound:
|
|
try:
|
|
track.drm = [PlayReady.from_init_data(init_data)]
|
|
except PlayReady.Exceptions.PSSHNotFound:
|
|
log.debug("No Widevine or PlayReady PSSH was found for this track, is it DRM free?")
|
|
|
|
if track.drm:
|
|
track_kid = track_kid or track.get_key_id(url=segments[0][0], session=session)
|
|
drm = track.get_drm_for_cdm(cdm)
|
|
if isinstance(drm, (Widevine, PlayReady, ClearKeyCENC)):
|
|
# license and grab content keys (ClearKeyCENC uses no CDM)
|
|
try:
|
|
if not license_widevine:
|
|
raise ValueError("license_widevine func must be supplied to use DRM")
|
|
progress(downloaded="LICENSING")
|
|
license_widevine(drm, track_kid=track_kid)
|
|
progress(downloaded="[yellow]LICENSED")
|
|
except Exception: # noqa
|
|
DOWNLOAD_CANCELLED.set() # skip pending track downloads
|
|
progress(downloaded="[red]FAILED")
|
|
raise
|
|
else:
|
|
drm = None
|
|
|
|
if DOWNLOAD_LICENCE_ONLY.is_set():
|
|
progress(downloaded="[yellow]SKIPPED")
|
|
return
|
|
|
|
progress(total=len(segments))
|
|
|
|
downloader = track.downloader
|
|
|
|
# When every segment is a byte range of one parent resource and init is its [0, len)
|
|
# prefix, the whole resource is itself a valid MP4, so one direct download to save_path
|
|
# replaces the per-segment files, init write and merge pass.
|
|
collapse_single_url = not config.sample and DASH.collapsible_single_url(
|
|
isinstance(track, Subtitle),
|
|
segments,
|
|
len(init_data) if init_data is not None else None,
|
|
)
|
|
|
|
use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption)
|
|
# subtitle segments are tiny, the post-download merge costs nothing there
|
|
use_rolling_merge = not collapse_single_url and config.merge_segments and not isinstance(track, Subtitle)
|
|
|
|
if not collapse_single_url:
|
|
# per-segment decryption rewrites each segment in place, so a segment kept from an
|
|
# earlier run would either be decrypted twice or never at all; a rolling merge
|
|
# unlinks each segment as it is appended, so there is nothing left to reuse
|
|
can_resume = config.continue_downloads and not use_segment_decrypt and not use_rolling_merge
|
|
# reuse a prior run's segments only when the fingerprint proves the segmentation unchanged
|
|
digest = resume.fingerprint(track.url, segments)
|
|
if not (can_resume and resume.reusable(save_dir, digest)):
|
|
shutil.rmtree(save_dir, ignore_errors=True)
|
|
save_dir.mkdir(parents=True, exist_ok=True)
|
|
if can_resume:
|
|
resume.write_sidecar(save_dir, digest)
|
|
|
|
if collapse_single_url:
|
|
# must carry no Range header, or the downloader would skip its ranged-parallel path
|
|
downloader_args = dict(
|
|
urls=[{"url": segments[0][0]}],
|
|
output_dir=save_path.parent,
|
|
filename=save_path.name,
|
|
headers=session.headers,
|
|
cookies=session.cookies,
|
|
proxy=proxy,
|
|
max_workers=max_workers,
|
|
session=session,
|
|
adaptive=adaptive,
|
|
processes=processes,
|
|
)
|
|
else:
|
|
downloader_args = dict(
|
|
urls=[
|
|
{"url": url, "headers": {"Range": f"bytes={bytes_range}"} if bytes_range else {}}
|
|
for url, bytes_range in segments
|
|
],
|
|
output_dir=save_dir,
|
|
filename="{i:0%d}.mp4" % (len(str(len(segments)))),
|
|
headers=session.headers,
|
|
cookies=session.cookies,
|
|
proxy=proxy,
|
|
max_workers=max_workers,
|
|
session=session,
|
|
adaptive=adaptive,
|
|
processes=processes,
|
|
)
|
|
|
|
log_event(
|
|
"manifest_dash_download_start",
|
|
level="DEBUG",
|
|
message="Starting DASH manifest download",
|
|
context={
|
|
"track_id": getattr(track, "id", None),
|
|
"track_type": track.__class__.__name__,
|
|
"total_segments": len(segments),
|
|
"has_drm": bool(track.drm),
|
|
"drm_types": [drm.__class__.__name__ for drm in (track.drm or [])],
|
|
"save_path": str(save_path),
|
|
"has_init_data": bool(init_data),
|
|
},
|
|
)
|
|
|
|
is_text_subtitle = (
|
|
not drm and isinstance(track, Subtitle) and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML)
|
|
)
|
|
|
|
decrypter: Optional[SegmentDecrypter] = None
|
|
if use_segment_decrypt and init_data:
|
|
decrypter = SegmentDecrypter(drm, init_data, save_dir.parent, max_workers)
|
|
|
|
merger: Optional[RollingMerge] = None
|
|
try:
|
|
with ExitStack() as stack:
|
|
if use_rolling_merge:
|
|
output = stack.enter_context(open(save_path, "wb"))
|
|
head = decrypter.init_bytes() if decrypter else init_data
|
|
if head:
|
|
output.write(head)
|
|
merger = RollingMerge(output)
|
|
|
|
# an exception raised in this body (a decrypt failure re-raised by the merge)
|
|
# must close the generator now, so its finally aborts the workers
|
|
stream = stack.enter_context(closing(downloader(**downloader_args)))
|
|
for status_update in stream:
|
|
file_downloaded = status_update.get("file_downloaded")
|
|
if file_downloaded:
|
|
future = decrypter.submit(file_downloaded) if decrypter else None
|
|
# listeners get the segment while it still exists; the merge unlinks it
|
|
events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
|
|
if merger:
|
|
merger.add(int(file_downloaded.stem), file_downloaded, future)
|
|
else:
|
|
downloaded = status_update.get("downloaded")
|
|
if downloaded and downloaded.endswith("/s"):
|
|
status_update["downloaded"] = f"DASH {downloaded}"
|
|
progress(**status_update)
|
|
|
|
if decrypter:
|
|
init_data = decrypter.finish()
|
|
except BaseException:
|
|
if decrypter:
|
|
decrypter.close()
|
|
if merger:
|
|
save_path.unlink(missing_ok=True)
|
|
raise
|
|
|
|
if collapse_single_url:
|
|
with open(save_path, "r+b") as collapsed:
|
|
collapsed.truncate(int(segments[-1][1].split("-")[1]) + 1)
|
|
elif merger:
|
|
for control_file in save_dir.glob("*.!dev"):
|
|
control_file.unlink(missing_ok=True)
|
|
if merger.merged != len(segments):
|
|
error_msg = f"Rolling merge appended {merger.merged} of {len(segments)} segments: {save_dir}"
|
|
log_event(
|
|
"manifest_dash_rolling_merge_incomplete",
|
|
level="ERROR",
|
|
message=error_msg,
|
|
context={
|
|
"track_id": getattr(track, "id", None),
|
|
"track_type": track.__class__.__name__,
|
|
"save_dir": str(save_dir),
|
|
"merged": merger.merged,
|
|
"total_segments": len(segments),
|
|
"downloader": "requests",
|
|
},
|
|
)
|
|
raise FileNotFoundError(error_msg)
|
|
else:
|
|
if not save_dir.exists():
|
|
error_msg = f"Output directory does not exist: {save_dir}"
|
|
log_event(
|
|
"manifest_dash_download_output_missing",
|
|
level="ERROR",
|
|
message=error_msg,
|
|
context={
|
|
"track_id": getattr(track, "id", None),
|
|
"track_type": track.__class__.__name__,
|
|
"save_dir": str(save_dir),
|
|
"save_path": str(save_path),
|
|
"downloader": "requests",
|
|
},
|
|
)
|
|
raise FileNotFoundError(error_msg)
|
|
|
|
for control_file in save_dir.glob("*.!dev"):
|
|
control_file.unlink(missing_ok=True)
|
|
|
|
segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()]
|
|
|
|
log_event(
|
|
"manifest_dash_download_complete",
|
|
level="DEBUG",
|
|
message="DASH download complete, preparing to merge",
|
|
context={
|
|
"track_id": getattr(track, "id", None),
|
|
"track_type": track.__class__.__name__,
|
|
"save_dir": str(save_dir),
|
|
"save_dir_exists": save_dir.exists(),
|
|
"segments_found": len(segments_to_merge),
|
|
"segment_files": [f.name for f in segments_to_merge[:10]],
|
|
"downloader": "requests",
|
|
},
|
|
)
|
|
|
|
if not segments_to_merge:
|
|
error_msg = f"No segment files found in output directory: {save_dir}"
|
|
all_contents = list(save_dir.iterdir()) if save_dir.exists() else []
|
|
log_event(
|
|
"manifest_dash_download_no_segments",
|
|
level="ERROR",
|
|
message=error_msg,
|
|
context={
|
|
"track_id": getattr(track, "id", None),
|
|
"track_type": track.__class__.__name__,
|
|
"save_dir": str(save_dir),
|
|
"directory_contents": [str(p) for p in all_contents],
|
|
"downloader": "requests",
|
|
},
|
|
)
|
|
raise FileNotFoundError(error_msg)
|
|
|
|
with open(save_path, "wb") as f:
|
|
if init_data:
|
|
f.write(init_data)
|
|
if len(segments_to_merge) > 1:
|
|
progress(downloaded="Merging", completed=0, total=len(segments_to_merge))
|
|
for segment_file in segments_to_merge:
|
|
append_segment(f, segment_file, is_text_subtitle)
|
|
segment_file.unlink()
|
|
progress(advance=1)
|
|
|
|
track.path = save_path
|
|
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
|
|
|
|
if drm:
|
|
progress(downloaded="Decrypting", completed=0, total=None)
|
|
decrypt_track(drm, save_path, license_widevine, decrypt=not decrypter)
|
|
assert_fragments_decrypted(save_path)
|
|
track.drm = None
|
|
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None)
|
|
progress(downloaded="Decrypted", completed=100, total=100)
|
|
|
|
if save_dir.exists() and save_dir.name.endswith("_segments"):
|
|
try:
|
|
save_dir.rmdir()
|
|
except OSError:
|
|
# Directory might not be empty, try removing recursively
|
|
shutil.rmtree(save_dir, ignore_errors=True)
|
|
resume.clear_sidecar(save_dir)
|
|
|
|
progress(downloaded="Downloaded")
|
|
|
|
@staticmethod
|
|
def find_segment_element(name: str, adaptation_set: Element, representation: Element) -> Optional[Element]:
|
|
"""Find a segment description element on the representation, or else on its adaptation set."""
|
|
element = representation.find(name)
|
|
return element if element is not None else adaptation_set.find(name)
|
|
|
|
@staticmethod
|
|
def has_segment_timing(track: AnyTrack) -> bool:
|
|
"""Return False when the manifest gives the track as one file with no index, so a sample cannot cut it."""
|
|
dash = track.data.get("dash") or {}
|
|
representation, adaptation_set = dash.get("representation"), dash.get("adaptation_set")
|
|
if representation is None or adaptation_set is None:
|
|
return True
|
|
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
|
return (
|
|
DASH.merge_segment_templates(adaptation_set, representation) is not None
|
|
or DASH.find_segment_element("SegmentList", adaptation_set, representation) is not None
|
|
or (segment_base is not None and bool(segment_base.get("indexRange")))
|
|
)
|
|
|
|
@staticmethod
|
|
def collapsible_single_url(
|
|
is_subtitle: bool,
|
|
segments: list[tuple[str, Optional[str]]],
|
|
init_len: Optional[int],
|
|
) -> bool:
|
|
"""
|
|
Whether every segment is a byte range of one parent resource whose [0, init_len)
|
|
prefix is the init segment, so unshackle can download the whole resource in one pass.
|
|
|
|
Returns False for subtitle tracks, an unknown init length, mixed URLs, malformed
|
|
ranges, and anything but one gapless run in document order starting at the init
|
|
prefix. A gap would pull bytes the segmented merge path excludes (a period the
|
|
manifest filters out) into the collapsed file, so it fails closed.
|
|
"""
|
|
if is_subtitle or init_len is None or not segments:
|
|
return False
|
|
|
|
first_url = segments[0][0]
|
|
ranges: list[tuple[int, int]] = []
|
|
for url, bytes_range in segments:
|
|
if url != first_url or not bytes_range:
|
|
return False
|
|
parts = bytes_range.split("-")
|
|
if len(parts) != 2 or not parts[0] or not parts[1]:
|
|
return False
|
|
try:
|
|
start, end = int(parts[0]), int(parts[1])
|
|
except ValueError:
|
|
return False
|
|
if end < start:
|
|
return False
|
|
ranges.append((start, end))
|
|
|
|
prev_end = init_len - 1
|
|
for start, end in ranges:
|
|
if start != prev_end + 1:
|
|
return False
|
|
prev_end = end
|
|
return True
|
|
|
|
@staticmethod
|
|
def is_content_period(period: Element, filtered_period_ids: list[str]) -> bool:
|
|
"""Whether a period is a valid content period (not an ad, not filtered, not trick mode)."""
|
|
period_id = period.get("id")
|
|
if period_id and period_id in filtered_period_ids:
|
|
return False
|
|
if next(iter(period.xpath("SegmentType/@value")), "content") != "content":
|
|
return False
|
|
if "urn:amazon:primevideo:cachingBreadth" in [
|
|
x.get("schemeIdUri") for x in period.findall("SupplementalProperty")
|
|
]:
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def merge_segment_templates(adaptation_set: Element, representation: Element) -> Optional[Element]:
|
|
"""
|
|
Assemble the effective SegmentTemplate for a Representation by cascading the
|
|
AdaptationSet > Representation levels (ISO/IEC 23009-1 5.3.9.1).
|
|
|
|
The Representation-level node, when present, is the base. It takes the attributes
|
|
and the SegmentTimeline child that it does not declare from the AdaptationSet-level
|
|
node. Returns None if no SegmentTemplate exists at either level.
|
|
"""
|
|
levels = [node.find("SegmentTemplate") for node in (adaptation_set, representation)]
|
|
present = [node for node in levels if node is not None]
|
|
if not present:
|
|
return None
|
|
|
|
merged = deepcopy(present[-1])
|
|
for ancestor in reversed(present[:-1]):
|
|
for attr, value in ancestor.attrib.items():
|
|
if merged.get(attr) is None:
|
|
merged.set(attr, value)
|
|
if merged.find("SegmentTimeline") is None:
|
|
timeline = ancestor.find("SegmentTimeline")
|
|
if timeline is not None:
|
|
merged.append(deepcopy(timeline))
|
|
return merged
|
|
|
|
@staticmethod
|
|
def get_period_segments(
|
|
period: Element,
|
|
adaptation_set: Element,
|
|
representation: Element,
|
|
manifest: ElementTree,
|
|
track: AnyTrack,
|
|
track_url: str,
|
|
session: Union[Session, RnetSession],
|
|
probe_kid: bool = True,
|
|
window: Optional[tuple[int, int]] = None,
|
|
) -> tuple[
|
|
Optional[bytes],
|
|
list[tuple[str, Optional[str]]],
|
|
float,
|
|
list[int],
|
|
Optional[UUID],
|
|
]:
|
|
"""
|
|
Extract segments from a single period's representation.
|
|
|
|
Parameters:
|
|
probe_kid: Probe the initialization segment for the Key ID. Set it to False when
|
|
the caller needs neither the init data nor the Key ID of this period. False
|
|
drops the FFprobe call, and also the init request where the rest of the parse
|
|
does not need those bytes.
|
|
|
|
Returns:
|
|
A tuple of (init_data, segments, segment_timescale, segment_durations, track_kid).
|
|
"""
|
|
manifest_base_url = manifest.findtext("BaseURL")
|
|
if not manifest_base_url:
|
|
manifest_base_url = track_url
|
|
elif not re.match("^https?://", manifest_base_url, re.IGNORECASE):
|
|
manifest_base_url = urljoin(track_url, f"./{manifest_base_url}")
|
|
period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL") or "")
|
|
adaptation_set_base_url = urljoin(period_base_url, adaptation_set.findtext("BaseURL") or "")
|
|
rep_base_url = urljoin(adaptation_set_base_url, representation.findtext("BaseURL") or "")
|
|
|
|
period_duration = period.get("duration") or manifest.get("mediaPresentationDuration")
|
|
init_data: Optional[bytes] = None
|
|
|
|
segment_template = DASH.merge_segment_templates(adaptation_set, representation)
|
|
|
|
segment_list = DASH.find_segment_element("SegmentList", adaptation_set, representation)
|
|
|
|
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
|
|
|
segments: list[tuple[str, Optional[str]]] = []
|
|
segment_timescale: float = 0
|
|
segment_durations: list[int] = []
|
|
track_kid: Optional[UUID] = None
|
|
|
|
if segment_template is not None:
|
|
start_number = int(segment_template.get("startNumber") or 1)
|
|
end_number = int(segment_template.get("endNumber") or 0) or None
|
|
segment_timeline = segment_template.find("SegmentTimeline")
|
|
segment_timescale = float(segment_template.get("timescale") or 1)
|
|
|
|
for item in ("initialization", "media"):
|
|
value = segment_template.get(item)
|
|
if not value:
|
|
continue
|
|
if not re.match("^https?://", value, re.IGNORECASE):
|
|
if not rep_base_url:
|
|
raise ValueError("Resolved Segment URL is not absolute, and no Base URL is available.")
|
|
value = urljoin(rep_base_url, value)
|
|
if not urlparse(value).query:
|
|
manifest_url_query = urlparse(track_url).query
|
|
if manifest_url_query:
|
|
value += f"?{manifest_url_query}"
|
|
segment_template.set(item, value)
|
|
|
|
init_url = segment_template.get("initialization")
|
|
if init_url and probe_kid:
|
|
res = session.get(
|
|
DASH.replace_fields(
|
|
init_url, Bandwidth=representation.get("bandwidth"), RepresentationID=representation.get("id")
|
|
)
|
|
)
|
|
res.raise_for_status()
|
|
init_data = res.content
|
|
track_kid = track.get_key_id(init_data)
|
|
|
|
if segment_timeline is not None:
|
|
current_time = 0
|
|
for s in segment_timeline.findall("S"):
|
|
if s.get("t"):
|
|
current_time = int(s.get("t"))
|
|
for _ in range(1 + (int(s.get("r") or 0))):
|
|
segment_durations.append(current_time)
|
|
current_time += int(s.get("d"))
|
|
|
|
keep = track.sample_slice(segment_durations, segment_timescale, window) if window else slice(None)
|
|
start_number += keep.start or 0
|
|
segment_durations = segment_durations[keep]
|
|
|
|
if not end_number:
|
|
end_number = start_number + len(segment_durations) - 1
|
|
|
|
for t, n in zip(segment_durations, range(start_number, end_number + 1)):
|
|
segments.append(
|
|
(
|
|
DASH.replace_fields(
|
|
segment_template.get("media"),
|
|
Bandwidth=representation.get("bandwidth"),
|
|
Number=n,
|
|
RepresentationID=representation.get("id"),
|
|
Time=t,
|
|
),
|
|
None,
|
|
)
|
|
)
|
|
else:
|
|
if not period_duration:
|
|
raise ValueError("Duration of the Period was unable to be determined.")
|
|
period_duration = DASH.pt_to_sec(period_duration)
|
|
segment_duration = float(segment_template.get("duration")) or 1
|
|
|
|
if not end_number:
|
|
segment_count = math.ceil(period_duration / (segment_duration / segment_timescale))
|
|
end_number = start_number + segment_count - 1
|
|
|
|
numbers = range(start_number, end_number + 1)
|
|
if window:
|
|
starts = [n * segment_duration for n in range(len(numbers))]
|
|
numbers = numbers[track.sample_slice(starts, segment_timescale, window)]
|
|
for s in numbers:
|
|
segments.append(
|
|
(
|
|
DASH.replace_fields(
|
|
segment_template.get("media"),
|
|
Bandwidth=representation.get("bandwidth"),
|
|
Number=s,
|
|
RepresentationID=representation.get("id"),
|
|
Time=s,
|
|
),
|
|
None,
|
|
)
|
|
)
|
|
# TODO: Should we floor/ceil/round, or is int() ok?
|
|
segment_durations.append(int(segment_duration))
|
|
elif segment_list is not None:
|
|
segment_timescale = float(segment_list.get("timescale") or 1)
|
|
|
|
init_data = None
|
|
initialization = segment_list.find("Initialization")
|
|
if initialization is not None and probe_kid:
|
|
source_url = initialization.get("sourceURL")
|
|
if not source_url:
|
|
source_url = rep_base_url
|
|
elif not re.match("^https?://", source_url, re.IGNORECASE):
|
|
source_url = urljoin(rep_base_url, f"./{source_url}")
|
|
|
|
if initialization.get("range"):
|
|
init_range_header = {"Range": f"bytes={initialization.get('range')}"}
|
|
else:
|
|
init_range_header = None
|
|
|
|
res = session.get(url=source_url, headers=init_range_header)
|
|
res.raise_for_status()
|
|
init_data = res.content
|
|
track_kid = track.get_key_id(init_data)
|
|
|
|
segment_urls = segment_list.findall("SegmentURL")
|
|
list_duration = float(segment_list.get("duration") or 0)
|
|
if window and list_duration:
|
|
starts = [n * list_duration for n in range(len(segment_urls))]
|
|
segment_urls = segment_urls[track.sample_slice(starts, segment_timescale, window)]
|
|
for segment_url in segment_urls:
|
|
media_url = segment_url.get("media")
|
|
if not media_url:
|
|
media_url = rep_base_url
|
|
elif not re.match("^https?://", media_url, re.IGNORECASE):
|
|
media_url = urljoin(rep_base_url, f"./{media_url}")
|
|
|
|
segments.append((media_url, segment_url.get("mediaRange")))
|
|
segment_durations.append(int(segment_url.get("duration") or 1))
|
|
elif segment_base is not None:
|
|
media_range = None
|
|
init_data = None
|
|
initialization = segment_base.find("Initialization")
|
|
if initialization is not None:
|
|
if initialization.get("range"):
|
|
init_range_header = {"Range": f"bytes={initialization.get('range')}"}
|
|
else:
|
|
init_range_header = None
|
|
|
|
res = session.get(url=rep_base_url, headers=init_range_header)
|
|
res.raise_for_status()
|
|
init_data = res.content
|
|
if probe_kid:
|
|
track_kid = track.get_key_id(init_data)
|
|
total_size = res.headers.get("Content-Range", "").split("/")[-1]
|
|
if total_size:
|
|
media_range = f"{len(init_data)}-{total_size}"
|
|
|
|
subsegments: list[tuple[int, int, int]] = []
|
|
index_range = segment_base.get("indexRange")
|
|
if window and index_range and not isinstance(track, Subtitle):
|
|
res = session.get(url=rep_base_url, headers={"Range": f"bytes={index_range}"})
|
|
res.raise_for_status()
|
|
segment_timescale, subsegments = parse_sidx(res.content, int(index_range.split("-")[0]))
|
|
|
|
if subsegments:
|
|
segment_durations = [time for time, _, _ in subsegments]
|
|
keep = track.sample_slice(segment_durations, segment_timescale, window)
|
|
segment_durations = segment_durations[keep]
|
|
segments.extend((rep_base_url, f"{first}-{last}") for _, first, last in subsegments[keep])
|
|
else:
|
|
segments.append((rep_base_url, media_range))
|
|
elif rep_base_url:
|
|
segments.append((rep_base_url, None))
|
|
|
|
return init_data, segments, segment_timescale, segment_durations, track_kid
|
|
|
|
@staticmethod
|
|
def get_attr(item: str, adaptation_set: Element, representation: Optional[Element] = None) -> Optional[Any]:
|
|
"""Helper to get a requested item from the Representation, otherwise from the AdaptationSet."""
|
|
adaptation_set_item = adaptation_set.get(item)
|
|
if representation is None:
|
|
return adaptation_set_item
|
|
|
|
representation_item = representation.get(item)
|
|
if representation_item is not None:
|
|
return representation_item
|
|
|
|
return adaptation_set_item
|
|
|
|
@staticmethod
|
|
def find_elements(
|
|
item: str, adaptation_set: Element, representation: Optional[Element] = None, both: bool = False
|
|
) -> list[Any]:
|
|
"""
|
|
Helper to get all requested items from the Representation, otherwise from the AdaptationSet.
|
|
Optionally, you may pass both=True to keep both values (where available).
|
|
"""
|
|
adaptation_set_items = adaptation_set.findall(item)
|
|
if representation is None:
|
|
return adaptation_set_items
|
|
|
|
representation_items = representation.findall(item)
|
|
|
|
if both:
|
|
return representation_items + adaptation_set_items
|
|
|
|
if representation_items:
|
|
return representation_items
|
|
|
|
return adaptation_set_items
|
|
|
|
@staticmethod
|
|
def get_language(
|
|
adaptation_set: Element,
|
|
representation: Optional[Element] = None,
|
|
fallback: Optional[Union[str, Language]] = None,
|
|
) -> Optional[Language]:
|
|
"""
|
|
Get Language (if any) from the AdaptationSet or Representation.
|
|
|
|
The caller can give a fallback language for when the manifest holds no
|
|
language information.
|
|
"""
|
|
options = []
|
|
|
|
if representation is not None:
|
|
options.append(representation.get("lang"))
|
|
# derive language from somewhat common id string format
|
|
# the format is typically "{rep_id}_{lang}={bitrate}" or similar
|
|
rep_id = representation.get("id")
|
|
if rep_id:
|
|
m = re.match(r"\w+_(\w+)=\d+", rep_id)
|
|
if m:
|
|
options.append(m.group(1))
|
|
|
|
options.append(adaptation_set.get("lang"))
|
|
|
|
if fallback:
|
|
options.append(fallback)
|
|
|
|
for option in options:
|
|
option = (str(option) or "").strip()
|
|
if not tag_is_valid(option) or option.startswith("und"):
|
|
continue
|
|
return Language.get(option)
|
|
|
|
@staticmethod
|
|
def get_video_range(
|
|
codecs: str, all_supplemental_props: list[Element], all_essential_props: list[Element]
|
|
) -> Video.Range:
|
|
# TODO: Detect Dolby Vision composite streams in DASH manifests (DV RPU embedded but
|
|
# primary codec is plain hvc1, signaled via a separate AdaptationSet/Representation
|
|
# with DV codec strings or DolbyVisionConfigurationBox). When found, mark the track
|
|
# with dv_compatible_bitstream=True so DVFixup runs pre-mux. No DASH samples seen yet.
|
|
if codecs.startswith(("dva1", "dvav", "dvhe", "dvh1")):
|
|
return Video.Range.DV
|
|
|
|
return Video.Range.from_cicp(
|
|
primaries=next(
|
|
(
|
|
int(x.get("value"))
|
|
for x in all_supplemental_props + all_essential_props
|
|
if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:ColourPrimaries"
|
|
),
|
|
0,
|
|
),
|
|
transfer=next(
|
|
(
|
|
int(x.get("value"))
|
|
for x in all_supplemental_props + all_essential_props
|
|
if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics"
|
|
),
|
|
0,
|
|
),
|
|
matrix=next(
|
|
(
|
|
int(x.get("value"))
|
|
for x in all_supplemental_props + all_essential_props
|
|
if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:MatrixCoefficients"
|
|
),
|
|
0,
|
|
),
|
|
)
|
|
|
|
@staticmethod
|
|
def is_trick_mode(adaptation_set: Element) -> bool:
|
|
"""Whether the Adaptation Set carries Trick-Mode tracks."""
|
|
essential_props = adaptation_set.findall("EssentialProperty")
|
|
supplemental_props = adaptation_set.findall("SupplementalProperty")
|
|
|
|
return any(
|
|
prop.get("schemeIdUri") == "http://dashif.org/guidelines/trickmode"
|
|
for prop in essential_props + supplemental_props
|
|
)
|
|
|
|
@staticmethod
|
|
def adaptation_set_key(adaptation_set: Element) -> tuple[Optional[str], ...]:
|
|
"""Identity of an AdaptationSet, used to re-find its counterpart in another Period."""
|
|
return tuple(
|
|
adaptation_set.get(attr)
|
|
for attr in ("contentType", "mimeType", "lang", "audioTrackId", "audioTrackSubtype", "id")
|
|
)
|
|
|
|
@staticmethod
|
|
def resolve_representation(
|
|
period: Element, rep_id: Optional[str], adaptation_set: Element
|
|
) -> Optional[tuple[Element, Element]]:
|
|
"""
|
|
Find the (AdaptationSet, Representation) in another Period matching rep_id.
|
|
|
|
Representation ids are only unique within an AdaptationSet, so the set matching
|
|
`adaptation_set`'s identity wins over an earlier sibling that reuses the same id
|
|
for a different track (for example, audio description vs dialog).
|
|
"""
|
|
as_key = DASH.adaptation_set_key(adaptation_set)
|
|
|
|
def candidates(within: Element) -> list[tuple[Element, Element]]:
|
|
found = []
|
|
for as_ in within.findall("AdaptationSet"):
|
|
if DASH.is_trick_mode(as_) or DASH.adaptation_set_key(as_) != as_key:
|
|
continue
|
|
for rep in as_.findall("Representation"):
|
|
if rep.get("id") == rep_id:
|
|
found.append((as_, rep))
|
|
break
|
|
return found
|
|
|
|
matches = candidates(period)
|
|
if matches:
|
|
parent = adaptation_set.getparent()
|
|
siblings = [as_ for as_, _ in candidates(parent)] if parent is not None else []
|
|
ordinal = siblings.index(adaptation_set) if adaptation_set in siblings else 0
|
|
return matches[min(ordinal, len(matches) - 1)]
|
|
|
|
for as_ in period.findall("AdaptationSet"):
|
|
if DASH.is_trick_mode(as_):
|
|
continue
|
|
for rep in as_.findall("Representation"):
|
|
if rep.get("id") == rep_id:
|
|
return as_, rep
|
|
|
|
return None
|
|
|
|
@staticmethod
|
|
def is_descriptive(adaptation_set: Element) -> bool:
|
|
"""Whether the Adaptation Set is Descriptive."""
|
|
return any(
|
|
(x.get("schemeIdUri"), x.get("value"))
|
|
in (
|
|
("urn:mpeg:dash:role:2011", "description"),
|
|
("urn:mpeg:dash:role:2011", "descriptive"),
|
|
("urn:tva:metadata:cs:AudioPurposeCS:2007", "1"),
|
|
)
|
|
for x in (*adaptation_set.findall("Accessibility"), *adaptation_set.findall("Role"))
|
|
)
|
|
|
|
@staticmethod
|
|
def is_forced(adaptation_set: Element) -> bool:
|
|
"""Whether the Adaptation Set is a Forced Subtitle."""
|
|
return any(
|
|
x.get("schemeIdUri") == "urn:mpeg:dash:role:2011"
|
|
and x.get("value") in ("forced-subtitle", "forced_subtitle")
|
|
for x in adaptation_set.findall("Role")
|
|
)
|
|
|
|
@staticmethod
|
|
def is_sdh(adaptation_set: Element) -> bool:
|
|
"""Whether the Adaptation Set is SDH."""
|
|
return any(
|
|
(x.get("schemeIdUri"), x.get("value")) == ("urn:tva:metadata:cs:AudioPurposeCS:2007", "2")
|
|
for x in adaptation_set.findall("Accessibility")
|
|
)
|
|
|
|
@staticmethod
|
|
def is_closed_caption(adaptation_set: Element) -> bool:
|
|
"""Whether the Adaptation Set is a Closed Caption Subtitle."""
|
|
return any(
|
|
(x.get("schemeIdUri"), x.get("value")) == ("urn:mpeg:dash:role:2011", "caption")
|
|
for x in adaptation_set.findall("Role")
|
|
)
|
|
|
|
@staticmethod
|
|
def get_ddp_complexity_index(adaptation_set: Element, representation: Optional[Element]) -> Optional[int]:
|
|
"""Get the DD+ Complexity Index (if any) from the AdaptationSet or Representation."""
|
|
return next(
|
|
(
|
|
int(x.get("value"))
|
|
for x in DASH.find_elements("SupplementalProperty", adaptation_set, representation, both=True)
|
|
if x.get("schemeIdUri") == "tag:dolby.com,2018:dash:EC3_ExtensionComplexityIndex:2018"
|
|
),
|
|
None,
|
|
)
|
|
|
|
@staticmethod
|
|
def get_drm(protections: list[Element]) -> list[DRM_T]:
|
|
drm: list[DRM_T] = []
|
|
PLACEHOLDER_KIDS = {
|
|
UUID("00000000-0000-0000-0000-000000000000"), # All zeros (key rotation default)
|
|
UUID("00010203-0405-0607-0809-0a0b0c0d0e0f"), # Sequential 0x00-0x0f
|
|
UUID("00010203-0405-0607-0809-101112131415"), # Shaka Packager test pattern
|
|
}
|
|
|
|
for protection in protections:
|
|
urn = (protection.get("schemeIdUri") or "").lower()
|
|
|
|
if urn == WidevineCdm.urn:
|
|
pssh_text = protection.findtext("pssh") or protection.findtext("{urn:mpeg:cenc:2013}pssh")
|
|
if not pssh_text:
|
|
continue
|
|
pssh = PSSH(pssh_text)
|
|
kid_attr = protection.get("kid") or protection.get("{urn:mpeg:cenc:2013}kid")
|
|
kid = UUID(bytes=base64.b64decode(kid_attr)) if kid_attr else None
|
|
|
|
if not kid:
|
|
default_kid_attr = protection.get("default_KID") or protection.get(
|
|
"{urn:mpeg:cenc:2013}default_KID"
|
|
)
|
|
kid = UUID(default_kid_attr) if default_kid_attr else None
|
|
|
|
if not kid:
|
|
kid = next(
|
|
(
|
|
UUID(p.get("default_KID") or p.get("{urn:mpeg:cenc:2013}default_KID"))
|
|
for p in protections
|
|
if p.get("default_KID") or p.get("{urn:mpeg:cenc:2013}default_KID")
|
|
),
|
|
None,
|
|
)
|
|
|
|
if kid and pssh.key_ids and all(k.int == 0 or k in PLACEHOLDER_KIDS for k in pssh.key_ids):
|
|
pssh.set_key_ids([kid])
|
|
|
|
drm.append(Widevine(pssh=pssh, kid=kid))
|
|
|
|
elif urn in ("urn:uuid:9a04f079-9840-4286-ab92-e65be0885f95", "urn:microsoft:playready"):
|
|
pr_pssh_b64 = (
|
|
protection.findtext("pssh")
|
|
or protection.findtext("{urn:mpeg:cenc:2013}pssh")
|
|
or protection.findtext("pro")
|
|
or protection.findtext("{urn:microsoft:playready}pro")
|
|
)
|
|
if not pr_pssh_b64:
|
|
continue
|
|
pr_pssh = PR_PSSH(pr_pssh_b64)
|
|
kid_b64 = protection.findtext("kid")
|
|
kid = None
|
|
if kid_b64:
|
|
try:
|
|
kid = UUID(bytes=base64.b64decode(kid_b64))
|
|
except ValueError:
|
|
kid = None
|
|
|
|
drm.append(PlayReady(pssh=pr_pssh, kid=kid, pssh_b64=pr_pssh_b64))
|
|
|
|
elif urn == ClearKeyCENC.urn:
|
|
# W3C EME ClearKey (org.w3.clearkey): match the scheme UUID alone,
|
|
# value="ClearKey1.0" is spec'd (DASH-IF CCP) but not required in the wild
|
|
kid_attr = protection.get("default_KID") or protection.get("{urn:mpeg:cenc:2013}default_KID")
|
|
kid = None
|
|
if kid_attr:
|
|
try:
|
|
kid = UUID(kid_attr)
|
|
except ValueError:
|
|
try:
|
|
kid = UUID(bytes=base64.b64decode(kid_attr))
|
|
except ValueError:
|
|
kid = None
|
|
|
|
if not kid:
|
|
# DASH-IF puts default_KID on the sibling mp4protection element
|
|
kid = next(
|
|
(
|
|
UUID(p.get("default_KID") or p.get("{urn:mpeg:cenc:2013}default_KID"))
|
|
for p in protections
|
|
if p.get("default_KID") or p.get("{urn:mpeg:cenc:2013}default_KID")
|
|
),
|
|
None,
|
|
)
|
|
|
|
if not kid or kid in PLACEHOLDER_KIDS:
|
|
continue
|
|
|
|
# license URL appears under several namespaces/casings in the wild
|
|
laurl = next(
|
|
(
|
|
text.strip()
|
|
for name in (
|
|
"{https://dashif.org/CPS}Laurl",
|
|
"{https://dashif.org/CPS}laurl",
|
|
"{http://dashif.org/guidelines/clearKey}Laurl",
|
|
"{http://dashif.org/guidelines/clearKey}laurl",
|
|
"Laurl",
|
|
"laurl",
|
|
)
|
|
for text in [protection.findtext(name)]
|
|
if text and text.strip()
|
|
),
|
|
None,
|
|
)
|
|
|
|
drm.append(ClearKeyCENC(kids=[kid], laurl=laurl))
|
|
|
|
return drm
|
|
|
|
@staticmethod
|
|
def pt_to_sec(d: Union[str, float]) -> float:
|
|
if isinstance(d, float):
|
|
return d
|
|
has_ymd = d[0:8] == "P0Y0M0DT"
|
|
if d[0:2] != "PT" and not has_ymd:
|
|
raise ValueError("Input data is not a valid time string.")
|
|
if has_ymd:
|
|
d = d[6:].upper()
|
|
else:
|
|
d = d[2:].upper()
|
|
m = re.findall(r"([\d.]+.)", d)
|
|
return sum(float(x[0:-1]) * {"H": 60 * 60, "M": 60, "S": 1}[x[-1].upper()] for x in m)
|
|
|
|
@staticmethod
|
|
@lru_cache(maxsize=None)
|
|
def field_format_pattern(field: str) -> re.Pattern:
|
|
# printf-style `$Field%fmt$` matcher, compiled once per field name
|
|
return re.compile(rf"\${re.escape(field)}%([a-z0-9]+)\$", flags=re.I)
|
|
|
|
@staticmethod
|
|
def replace_fields(url: str, **kwargs: Any) -> str:
|
|
if "$" not in url:
|
|
return url
|
|
# printf-style `$Field%fmt$` padding always contains a literal '%'; skip the
|
|
# per-field regex entirely for the common case (plain `$Field$` templates)
|
|
has_fmt = "%" in url
|
|
for field, value in kwargs.items():
|
|
url = url.replace(f"${field}$", str(value))
|
|
if has_fmt:
|
|
m = DASH.field_format_pattern(field).search(url)
|
|
if m:
|
|
url = url.replace(m.group(), f"{value:{m.group(1)}}")
|
|
return url
|
|
|
|
|
|
__all__ = ("DASH",)
|