Files
unshackle/unshackle/core/manifests/hls.py
T
imSp4rky bfda31ce4a fix(proxy): keep service traffic on --proxy and warn about requests that bypass it
--proxy now overrides HTTP(S)_PROXY and the Windows system proxy while keeping the env CA bundle. Core fallback sessions use the run's proxy, and a warn-only audit hook names service code that sends a request or opens a raw socket outside the proxy; a session with proxy_exempt = True opts out.
2026-10-16 16:12:42 -06:00

1671 lines
73 KiB
Python

from __future__ import annotations
import base64
import html
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import time
from contextlib import ExitStack, closing
from itertools import accumulate
from pathlib import Path
from typing import Any, Optional, Union
from urllib.parse import urljoin, urlparse
from uuid import UUID
from zlib import crc32
import m3u8
import requests
from langcodes import Language, tag_is_valid
from m3u8 import M3U8
from pyplayready.cdm import Cdm as PlayReadyCdm
from pyplayready.system.pssh import PSSH as PR_PSSH
from pywidevine.cdm import Cdm as WidevineCdm
from pywidevine.pssh import PSSH as WV_PSSH
from requests import Session
from unshackle.core import binaries
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_widevine_cdm
from unshackle.core.config import config
from unshackle.core.constants import DOWNLOAD_ALL_DRM, DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
from unshackle.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine
from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
from unshackle.core.drm.verify import decrypt_track
from unshackle.core.events import events
from unshackle.core.manifests.dash import RollingMerge
from unshackle.core.session import RnetResponse, RnetSession, utf8_text
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
from unshackle.core.utilities import get_extension, is_close_match, log_event, try_ensure_utf8
from unshackle.core.utils import run_proxy
from unshackle.core.utils.redact import safe_display_url
from unshackle.core.utils.subprocess import log_tool_run
class HLS:
SUPP_CODECS_RE = re.compile(r'SUPPLEMENTAL-CODECS="([^"]+)"', re.IGNORECASE)
def __init__(
self,
manifest: M3U8,
session: Optional[Union[Session, RnetSession]] = None,
url: Optional[str] = None,
raw_text: Optional[str] = None,
):
if not manifest:
raise ValueError("HLS manifest must be provided.")
if not isinstance(manifest, M3U8):
raise TypeError(f"Expected manifest to be a {M3U8}, not {manifest!r}")
if not manifest.is_variant:
raise ValueError("Expected the M3U(8) manifest to be a Variant Playlist.")
self.manifest = manifest
self.session = session or run_proxy.fallback_session()
self.url = url
self.raw_text = raw_text
@classmethod
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> HLS:
if not url:
raise requests.URLRequired("HLS manifest URL must be provided.")
if not isinstance(url, str):
raise TypeError(f"Expected url to be a {str}, not {url!r}")
if not session:
session = run_proxy.fallback_session()
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
res = session.get(url, **args)
if not isinstance(res, (requests.Response, RnetResponse)):
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}")
if not res.ok:
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
content = utf8_text(res)
master = m3u8.loads(content, uri=url)
log_event(
"manifest_hls_fetch",
level="DEBUG",
message=f"Fetched HLS manifest ({len(content)} bytes)",
context={"url": safe_display_url(url), "size": len(content)},
)
return cls(master, session, url=url, raw_text=content)
@classmethod
def from_text(cls, text: str, url: str) -> HLS:
if not text:
raise ValueError("HLS manifest Text must be provided.")
if not isinstance(text, str):
raise TypeError(f"Expected text to be a {str}, not {text!r}")
if not url:
raise requests.URLRequired("HLS manifest URL must be provided for relative path computations.")
if not isinstance(url, str):
raise TypeError(f"Expected url to be a {str}, not {url!r}")
master = m3u8.loads(text, uri=url)
return cls(master, raw_text=text)
def supplemental_codecs_by_uri(self) -> dict[str, str]:
"""Map each variant URI to its SUPPLEMENTAL-CODECS value.
python-m3u8 drops this attribute, so we re-parse the raw text to recover it for
Dolby Vision composite detection (dvh1.08.x advertised only in SUPPLEMENTAL-CODECS
while primary CODECS stays plain hvc1).
"""
if not self.raw_text:
return {}
out: dict[str, str] = {}
lines = self.raw_text.splitlines()
for i, line in enumerate(lines):
if not line.startswith("#EXT-X-STREAM-INF"):
continue
supp_match = self.SUPP_CODECS_RE.search(line)
if not supp_match:
continue
for j in range(i + 1, len(lines)):
uri = lines[j].strip()
if uri and not uri.startswith("#"):
out[uri] = supp_match.group(1)
break
return out
@staticmethod
def select_audio_codec(
candidates: Optional[list[tuple[str, Audio.Codec]]], uri: Optional[str]
) -> Optional[Audio.Codec]:
"""
Choose the Audio Codec of one rendition from the codecs its variant declares.
EXT-X-STREAM-INF lists every codec of the variant, so a GROUP-ID holding
renditions of more than one codec gives no per-rendition answer. Packagers that do
this name the codec in the rendition URI, so match on that first.
The codec must sit between delimiters: a plain substring test reads "ec-3" out of
a hex UUID, and "ac-3" out of the "eac-3" that names an E-AC-3 rendition. Longer
codecs match first so one cannot take the place of another it contains, and the
match reads the path only, so a query string cannot decide a codec.
A rendition that names no codec falls back to the first codec its group declares.
The caller collects those highest-bandwidth variant first, so the fallback is the
top rung's codec, which keeps a premium rendition reachable rather than hiding it
behind a lossy label.
"""
if not candidates:
return None
if len(candidates) > 1:
path = urlparse(uri or "").path.lower()
for mime, codec in sorted(candidates, key=lambda c: len(c[0]), reverse=True):
if re.search(rf"(?<![a-z0-9]){re.escape(mime)}(?![a-z0-9])", path):
return codec
log_event(
"manifest_hls_audio_codec_guess",
level="DEBUG",
message=f"No codec named in the rendition path, assuming {candidates[0][1].value}",
context={"declared": [mime for mime, _ in candidates], "uri": safe_display_url(uri or "")},
)
return candidates[0][1]
def to_tracks(self, language: Union[str, Language]) -> Tracks:
"""
Convert a Variant Playlist M3U(8) document to Video, Audio and Subtitle Track objects.
Parameters:
language: Language you expect the Primary Track to be in.
All Track objects' URL will be to another M3U(8) document. However, these documents
will be Invariant Playlists and contain the list of segments URIs among other metadata.
"""
session_keys = list(self.manifest.session_keys or [])
if not session_keys:
session_keys = HLS.parse_session_data_keys(self.manifest, self.session)
session_drm = HLS.get_all_drm(session_keys)
audio_codecs_by_group_id: dict[str, list[tuple[str, Audio.Codec]]] = {}
for playlist in sorted(self.manifest.playlists, key=lambda p: p.stream_info.bandwidth or 0, reverse=True):
audio_group = playlist.stream_info.audio
if not (audio_group and playlist.stream_info.codecs):
continue
known = audio_codecs_by_group_id.setdefault(audio_group, [])
seen = {mime for mime, _ in known}
for token in playlist.stream_info.codecs.split(","):
mime = token.strip().split(".")[0].lower()
if mime in seen:
continue
try:
known.append((mime, Audio.Codec.from_mime(mime)))
except ValueError:
continue
seen.add(mime)
cc_by_group_id: dict[str, list[dict[str, Any]]] = {}
for media in self.manifest.media:
if media.type == "CLOSED-CAPTIONS":
cc_by_group_id.setdefault(media.group_id, []).append(
{
"language": media.language,
"name": media.name,
"instream_id": media.instream_id,
"characteristics": media.characteristics,
}
)
tracks = Tracks()
supplemental_codecs = self.supplemental_codecs_by_uri()
dv_supp_prefixes = ("dva1", "dvav", "dvhe", "dvh1")
for playlist in self.manifest.playlists:
try:
# TODO: Any better way to figure out the primary track type?
if playlist.stream_info.codecs:
Video.Codec.from_codecs(playlist.stream_info.codecs)
except ValueError:
primary_track_type = Audio
else:
primary_track_type = Video
primary_codecs = (playlist.stream_info.codecs or "").lower()
primary_has_dv = any(codec.split(".")[0] in dv_supp_prefixes for codec in primary_codecs.split(","))
supp_codecs_str = supplemental_codecs.get(playlist.uri, "")
supp_dv_codec: Optional[str] = None
for codec in supp_codecs_str.lower().split(","):
token = codec.strip().split("/")[0]
if token.split(".")[0] in dv_supp_prefixes:
supp_dv_codec = token
break
video_range = (
Video.Range.DV if primary_has_dv else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range)
)
# DV-composite track: primary codec is plain HEVC but SUPPLEMENTAL-CODECS advertises
# a DV codec. Range stays whatever VIDEO-RANGE signaled (HDR10/HLG/SDR); DVFixup will
# restore DV signaling post-download. Services that know their encoder embeds HDR10+
# SEI must override `range` themselves.
dv_compatible_bitstream = primary_track_type is Video and not primary_has_dv and supp_dv_codec is not None
tracks.add(
primary_track_type(
id_=hex(crc32(str(playlist).encode()))[2:],
url=urljoin(playlist.base_uri, playlist.uri),
codec=(
primary_track_type.Codec.from_codecs(playlist.stream_info.codecs)
if playlist.stream_info.codecs
else None
),
language=language, # HLS manifests do not seem to have language info
is_original_lang=True, # TODO: All we can do is assume Yes
bitrate=playlist.stream_info.average_bandwidth or playlist.stream_info.bandwidth,
descriptor=Video.Descriptor.HLS,
drm=session_drm,
data={"hls": {"playlist": playlist}},
**(
dict(
range_=video_range,
width=playlist.stream_info.resolution[0] if playlist.stream_info.resolution else None,
height=playlist.stream_info.resolution[1] if playlist.stream_info.resolution else None,
fps=playlist.stream_info.frame_rate,
closed_captions=cc_by_group_id.get(
(playlist.stream_info.closed_captions or "").strip('"'), []
),
dv_compatible_bitstream=dv_compatible_bitstream,
)
if primary_track_type is Video
else {}
),
)
)
for media in self.manifest.media:
if not media.uri:
continue
joc = 0
if media.type == "AUDIO":
track_type = Audio
codec = HLS.select_audio_codec(audio_codecs_by_group_id.get(media.group_id), media.uri)
if media.channels and media.channels.endswith("/JOC"):
joc = int(media.channels.split("/JOC")[0])
media.channels = "5.1"
else:
track_type = Subtitle
codec = Subtitle.Codec.WebVTT # assuming WebVTT, codec info isn't shown
track_lang = next(
(
Language.get(option)
for x in (media.language, language)
for option in [(str(x) or "").strip()]
if tag_is_valid(option) and not option.startswith("und")
),
None,
)
if not track_lang:
msg = "Language information could not be derived for a media."
if language is None:
msg += " No fallback language was provided when calling HLS.to_tracks()."
elif not tag_is_valid((str(language) or "").strip()) or str(language).startswith("und"):
msg += f" The fallback language provided is also invalid: {language}"
raise ValueError(msg)
tracks.add(
track_type(
id_=hex(crc32(str(media).encode()))[2:],
url=urljoin(media.base_uri, media.uri),
codec=codec,
language=track_lang, # HLS media may not have language info, fallback if needed
is_original_lang=bool(language and is_close_match(track_lang, [language])),
descriptor=Audio.Descriptor.HLS,
drm=session_drm if media.type == "AUDIO" else None,
data={"hls": {"media": media}},
**(
dict(
bitrate=0, # TODO: M3U doesn't seem to state bitrate?
channels=media.channels,
joc=joc,
descriptive="public.accessibility.describes-video" in (media.characteristics or ""),
)
if track_type is Audio
else dict(
forced=media.forced == "YES",
sdh="public.accessibility.describes-music-and-sound" in (media.characteristics or ""),
)
if track_type is Subtitle
else {}
),
)
)
for video in tracks.videos:
has_resolution = video.width and video.height
has_codec = video.codec is not None
if has_resolution and has_codec:
continue
try:
probe = HLS.probe_ts_info(video.url, self.session)
if probe:
width, height, codec = probe
if not has_resolution:
video.width, video.height = width, height
if not has_codec:
video.codec = codec
# best-effort probe over network + ffprobe; the track stays usable without it
except Exception as e:
logging.getLogger("HLS").debug(f"TS probe failed for {video.url}: {e!r}")
if self.url:
tracks.manifest_url = self.url
log_event(
"manifest_hls_parse",
level="INFO",
message=(
f"Parsed HLS manifest: {len(tracks.videos)} video, "
f"{len(tracks.audio)} audio, {len(tracks.subtitles)} subtitle track(s)"
),
context={
"videos": len(tracks.videos),
"audio": len(tracks.audio),
"subtitles": len(tracks.subtitles),
"ranges": sorted({str(v.range) for v in tracks.videos}),
"vcodecs": sorted({str(v.codec) for v in tracks.videos}),
},
)
return tracks
@staticmethod
def probe_ts_info(
variant_url: str, session: Optional[Union[Session, RnetSession]] = None
) -> Optional[tuple[int, int, Video.Codec]]:
"""Probe the first TS segment of a variant playlist to extract resolution and codec."""
if not session:
session = run_proxy.fallback_session()
res = session.get(variant_url)
variant = m3u8.loads(utf8_text(res), uri=variant_url)
if not variant.segments:
return None
seg_uri = urljoin(variant_url, variant.segments[0].uri)
# Download only the first 8KB: SPS is always near the start of the first TS packet
res = session.get(seg_uri, headers={"Range": "bytes=0-8191"})
data = res.content
return HLS.parse_ts_video_info(data)
@staticmethod
def parse_ts_video_info(data: bytes) -> Optional[tuple[int, int, Video.Codec]]:
"""Parse H.264/H.265 NAL units from TS segment data to extract resolution and codec."""
class _BitReader:
def __init__(self, buf: bytes) -> None:
self.data = buf
self.pos = 0
def bits(self, n: int) -> int:
val = 0
for _ in range(n):
val = (val << 1) | ((self.data[self.pos >> 3] >> (7 - (self.pos & 7))) & 1)
self.pos += 1
return val
def ue(self) -> int:
zeros = 0
while self.bits(1) == 0:
zeros += 1
return (1 << zeros) - 1 + self.bits(zeros) if zeros else 0
def se(self) -> int:
val = self.ue()
return (val + 1) // 2 if val & 1 else -(val // 2)
# H.264: NAL type 7 (SPS), identified by byte & 0x1F == 7
# H.265: NAL type 33 (SPS), identified by (byte >> 1) & 0x3F == 33
# Find SPS NAL unit via start code
for i in range(len(data) - 4):
start3 = data[i : i + 3] == b"\x00\x00\x01"
start4 = data[i : i + 4] == b"\x00\x00\x00\x01"
if not start3 and not start4:
continue
offset = i + (4 if start4 else 3)
if offset >= len(data):
continue
nal_byte = data[offset]
h264_type = nal_byte & 0x1F
h265_type = (nal_byte >> 1) & 0x3F
# H.264 SPS (NAL type 7)
if h264_type == 7:
sps = data[offset : offset + 64]
if len(sps) < 5:
continue
try:
r = _BitReader(sps[1:]) # skip NAL header byte
profile = r.bits(8)
r.bits(8) # constraint flags
r.bits(8) # level
r.ue() # sps_id
if profile in (100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134):
chroma = r.ue()
if chroma == 3:
r.bits(1)
r.ue() # bit_depth_luma
r.ue() # bit_depth_chroma
r.bits(1) # qpprime_y_zero_transform_bypass
if r.bits(1): # scaling_matrix_present
for j in range(6 if chroma != 3 else 12):
if r.bits(1):
last = 8
for _ in range(16 if j < 6 else 64):
if last != 0:
last = (last + r.se()) & 0xFF
r.ue() # log2_max_frame_num
poc_type = r.ue()
if poc_type == 0:
r.ue()
elif poc_type == 1:
r.bits(1)
r.se()
r.se()
for _ in range(r.ue()):
r.se()
r.ue() # max_num_ref_frames
r.bits(1) # gaps_in_frame_num
w_mbs = r.ue() + 1
h_map = r.ue() + 1
frame_mbs_only = r.bits(1)
if not frame_mbs_only:
r.bits(1)
r.bits(1) # direct_8x8_inference
cl = cr = ct = cb = 0
if r.bits(1): # crop
cl, cr, ct, cb = r.ue(), r.ue(), r.ue(), r.ue()
width = w_mbs * 16 - (cl + cr) * 2
height = (2 - frame_mbs_only) * h_map * 16 - (ct + cb) * 2
return (width, height, Video.Codec.AVC)
except (IndexError, ValueError):
continue
# H.265 SPS (NAL type 33)
elif h265_type == 33:
sps = data[offset : offset + 128]
if len(sps) < 10:
continue
try:
r = _BitReader(sps[2:]) # skip 2-byte NAL header
r.bits(4) # sps_video_parameter_set_id
max_sub_layers = r.bits(3)
r.bits(1) # sps_temporal_id_nesting
# profile_tier_level
r.bits(2) # general_profile_space
r.bits(1) # general_tier
r.bits(5) # general_profile_idc
r.bits(32) # general_profile_compatibility_flags
r.bits(48) # general_constraint_indicator_flags
r.bits(8) # general_level_idc
sub_layer_flags = []
for _ in range(max_sub_layers - 1):
sub_layer_flags.append((r.bits(1), r.bits(1)))
if max_sub_layers - 1 > 0:
for _ in range(8 - (max_sub_layers - 1)):
r.bits(2)
for profile_present, level_present in sub_layer_flags:
if profile_present:
r.bits(2 + 1 + 5 + 32 + 48 + 8)
if level_present:
r.bits(8)
r.ue() # sps_seq_parameter_set_id
chroma = r.ue()
if chroma == 3:
r.bits(1)
width = r.ue()
height = r.ue()
if r.bits(1): # conformance_window
cl = r.ue()
cr = r.ue()
ct = r.ue()
cb = r.ue()
sub_w = 2 if chroma in (1, 2) else 1
sub_h = 2 if chroma == 1 else 1
width -= (cl + cr) * sub_w
height -= (ct + cb) * sub_h
return (width, height, Video.Codec.HEVC)
except (IndexError, ValueError):
continue
return None
@staticmethod
def resolve_segment_key(
segment_keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]],
cdm: object,
drm_preference: Optional[str] = None,
) -> Optional[m3u8.Key]:
"""Pick the EXT-X-KEY for a segment, preferring one the CDM can handle."""
if cdm:
cdm_segment_keys = HLS.filter_keys_for_cdm(segment_keys, cdm, drm_preference)
if cdm_segment_keys:
return HLS.get_supported_key(cdm_segment_keys)
return HLS.get_supported_key(segment_keys)
@staticmethod
def fetch_init_section(
session: Union[Session, RnetSession],
init_section: m3u8.model.InitializationSection,
range_offset: int,
) -> tuple[bytes, int]:
"""Download an EXT-X-MAP init segment. Returns its bytes and the next range offset."""
if init_section.byterange:
init_byte_range = HLS.calculate_byte_range(init_section.byterange, range_offset)
range_offset = int(init_byte_range.split("-")[0])
init_range_header = {"Range": f"bytes={init_byte_range}"}
else:
init_range_header = {}
res = session.get(
url=urljoin(init_section.base_uri, init_section.uri),
headers=init_range_header,
)
if not isinstance(res, (requests.Response, RnetResponse)):
raise TypeError(f"Expected response to be requests.Response or rnet.Response, not {type(res)}")
res.raise_for_status()
return res.content, range_offset
@staticmethod
def single_init_no_rotation(
wanted_segments: list[Any],
initial_key: Optional[m3u8.Key],
cdm: object,
drm_preference: Optional[str] = None,
) -> bool:
"""Whether every wanted segment shares one init segment and one EXT-X-KEY, with no discontinuity.
Only then does the track merge into a single file with one init segment and one set of
content keys, which segment-by-segment decryption needs.
"""
if not wanted_segments:
return False
first = wanted_segments[0]
if not first.init_section:
return False
first_keys = list(getattr(first, "keys", None) or [])
for segment in wanted_segments[1:]:
if segment.discontinuity or segment.init_section != first.init_section:
return False
# the merge loop treats a segment without its own EXT-X-KEY as "key unchanged"
segment_keys = list(getattr(segment, "keys", None) or [])
if segment_keys and segment_keys != first_keys:
return False
if not first_keys:
return True
# the merge loop re-resolves the content key per segment; a pick other than the one already
# licensed would re-key mid-track, which one pass of segment decryption cannot follow
return initial_key is not None and HLS.resolve_segment_key(first_keys, cdm, drm_preference) == initial_key
@staticmethod
def download_track(track: AnyTrack, ctx: DownloadContext) -> None:
session = ctx.ensure_session()
save_path = ctx.save_path
save_dir = ctx.save_dir
progress = ctx.progress
proxy = ctx.proxy
max_workers = ctx.max_workers
adaptive = ctx.adaptive_workers
processes = ctx.download_processes
license_widevine = ctx.license_widevine
cdm = ctx.cdm
if proxy:
if isinstance(session, Session):
session.proxies.update({"all": proxy})
log = logging.getLogger("HLS")
if track.from_file:
master = m3u8.load(str(track.from_file))
else:
response = session.get(track.url)
if not isinstance(response, (requests.Response, RnetResponse)):
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}")
if not response.ok:
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
sys.exit(1)
master = m3u8.loads(utf8_text(response), uri=track.url)
if not master.segments:
log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.")
sys.exit(1)
unwanted_segments = [
segment for segment in master.segments if callable(track.OnSegmentFilter) and track.OnSegmentFilter(segment)
]
hls_data = track.data.setdefault("hls", {})
if hls_data.get("unwanted_segments") and not unwanted_segments:
log.warning(
f"None of the {len(hls_data['unwanted_segments'])} segments that the export drops is in the "
f"playlist of {track.id}. The download keeps every segment."
)
hls_data["unwanted_segments"] = [s.absolute_uri for s in unwanted_segments]
if config.sample:
content = [segment for segment in master.segments if segment not in unwanted_segments]
starts = list(accumulate((segment.duration or 0 for segment in content[:-1]), initial=0.0))
del content[track.sample_slice(starts, window=config.sample)]
unwanted_segments += content
# Get session DRM as fallback but prefer media playlist keys for accurate KID matching
if track.drm:
session_drm = track.get_drm_for_cdm(cdm)
else:
session_drm = None
initial_drm_licensed = False
initial_drm_key = None # Track the EXT-X-KEY used for initial licensing
media_keys = [k for k in (master.keys or []) if k is not None]
if media_keys:
cdm_media_keys = HLS.filter_keys_for_cdm(media_keys, cdm, track.drm_preference)
media_playlist_key = HLS.get_supported_key(cdm_media_keys) if cdm_media_keys else None
if media_playlist_key:
init_kid = (
None
if media_playlist_key.method == "AES-128"
else HLS.get_track_kid_from_init(master, track, session)
)
media_drm = HLS.get_drm(media_playlist_key, session, **({"kid": init_kid} if init_kid else {}))
if isinstance(media_drm, (Widevine, PlayReady)):
track_kid = init_kid or media_drm.kid
# Preserve pre-existing keys (e.g. from server_cdm)
if track.drm:
for existing_drm in track.drm:
if hasattr(existing_drm, "content_keys") and existing_drm.content_keys:
media_drm.content_keys.update(existing_drm.content_keys)
track.drm = [media_drm]
if DOWNLOAD_ALL_DRM.is_set():
track.drm += HLS.alternate_drm(media_keys, media_drm, session, init_kid)
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
progress(downloaded="LICENSING")
license_widevine(media_drm, track_kid=track_kid)
progress(downloaded="[yellow]LICENSED")
initial_drm_licensed = True
initial_drm_key = media_playlist_key
session_drm = media_drm
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
elif isinstance(media_drm, ClearKey):
# AES-128 (ClearKey) needs no license server - the key is already fetched.
track.drm = [media_drm]
session_drm = media_drm
initial_drm_key = media_playlist_key
initial_drm_licensed = True
if not initial_drm_licensed and session_drm and isinstance(session_drm, (Widevine, PlayReady)):
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
track_kid = HLS.get_track_kid_from_init(master, track, session) or session_drm.kid
progress(downloaded="LICENSING")
license_widevine(session_drm, track_kid=track_kid)
progress(downloaded="[yellow]LICENSED")
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
if not initial_drm_licensed and session_drm and isinstance(session_drm, MonaLisa):
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
progress(downloaded="LICENSING")
license_widevine(session_drm)
progress(downloaded="[yellow]LICENSED")
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
if DOWNLOAD_LICENCE_ONLY.is_set():
# a playlist can change its key part-way, and only the download loop licenses the later keys
licensed_keys = [initial_drm_key]
for segment in master.segments:
if not segment.keys or segment in unwanted_segments:
continue
key = HLS.resolve_segment_key(segment.keys, cdm, track.drm_preference)
if key is None or key.method == "AES-128" or key in licensed_keys:
continue
licensed_keys.append(key)
later_drm = HLS.get_drm_or_probe(key, master, track, session, segment.init_section)
if isinstance(later_drm, (Widevine, PlayReady)) and license_widevine:
license_widevine(later_drm, track_kid=later_drm.kid)
progress(downloaded="[yellow]SKIPPED")
return
# Downloaded segment files are named by post-filter index; map that back to the wanted
# segment so the IV uses the absolute media sequence number, not the download index.
wanted_segments = [seg for seg in master.segments if seg not in unwanted_segments]
single_init = HLS.single_init_no_rotation(wanted_segments, initial_drm_key, cdm, track.drm_preference)
use_segment_decrypt = (
isinstance(session_drm, (Widevine, PlayReady)) and can_use(session_drm, config.decryption) and single_init
)
use_rolling_merge = (
config.merge_segments
and single_init
and not isinstance(track, Subtitle)
and (session_drm is None or isinstance(session_drm, (Widevine, PlayReady)))
)
total_segments = len(master.segments) - len(unwanted_segments)
progress(total=total_segments)
downloader = track.downloader
urls: list[dict[str, Any]] = []
segment_durations: list[int] = []
range_offset = 0
for segment in master.segments:
if segment.byterange:
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
range_offset = int(byte_range.split("-")[1]) + 1
else:
byte_range = None
if segment in unwanted_segments:
continue
segment_durations.append(int(segment.duration))
urls.append(
{
"url": urljoin(segment.base_uri, segment.uri),
"headers": {"Range": f"bytes={byte_range}"} if byte_range else {},
}
)
track.data["hls"]["segment_durations"] = segment_durations
segment_save_dir = save_dir / "segments"
# no resume for AES-128/ClearKey: in-place decryption at merge time makes numbered files ambiguous
# per-segment decryption rewrites each segment in place, so a segment kept from an
# earlier run would either be decrypted twice or never at all
resumable_drm = (
not use_segment_decrypt
and not use_rolling_merge
and (session_drm is None or isinstance(session_drm, (Widevine, PlayReady)))
and not any(key and key.method == "AES-128" for key in (master.keys or []))
)
# media_sequence feeds AES IVs; total_segments pins the OnSegmentFilter verdict (shifts every index)
digest = resume.fingerprint(
track.url,
[(url["url"], url["headers"].get("Range")) for url in urls],
extra=[str(getattr(master, "media_sequence", None) or 0), str(total_segments)],
)
if not (config.continue_downloads and resumable_drm and resume.reusable(save_dir, digest)):
shutil.rmtree(save_dir, ignore_errors=True)
segment_save_dir.mkdir(parents=True, exist_ok=True)
if config.continue_downloads and resumable_drm:
resume.write_sidecar(save_dir, digest)
downloader_args = dict(
urls=urls,
output_dir=segment_save_dir,
filename="{i:0%d}{ext}" % len(str(len(urls))),
headers=session.headers,
cookies=session.cookies,
proxy=proxy,
max_workers=max_workers,
session=session,
adaptive=adaptive,
processes=processes,
)
log_event(
"manifest_hls_download_start",
level="DEBUG",
message="Starting HLS manifest download",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"total_segments": total_segments,
"has_drm": bool(session_drm),
"drm_type": session_drm.__class__.__name__ if session_drm else None,
"save_path": str(save_path),
},
)
map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None
segment_decrypter: Optional[SegmentDecrypter] = None
if use_segment_decrypt or use_rolling_merge:
init_section = wanted_segments[0].init_section
init_content, _ = HLS.fetch_init_section(session, init_section, 0)
map_data = (init_section, init_content)
if use_segment_decrypt:
segment_decrypter = SegmentDecrypter(session_drm, init_content, save_dir.parent, max_workers)
merger: Optional[RollingMerge] = None
try:
with ExitStack() as stack:
if use_rolling_merge:
output = stack.enter_context(open(save_path, "wb"))
output.write(segment_decrypter.init_bytes() if segment_decrypter else init_content)
merger = RollingMerge(output)
stream = stack.enter_context(closing(downloader(**downloader_args)))
for status_update in stream:
file_downloaded = status_update.get("file_downloaded")
if file_downloaded:
future = segment_decrypter.submit(file_downloaded) if segment_decrypter else None
events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
if merger:
merger.add(int(file_downloaded.stem), file_downloaded, future)
else:
downloaded = status_update.get("downloaded")
if downloaded and downloaded.endswith("/s"):
status_update["downloaded"] = f"HLS {downloaded}"
progress(**status_update)
except BaseException:
if segment_decrypter:
segment_decrypter.close()
if merger:
save_path.unlink(missing_ok=True)
raise
for control_file in segment_save_dir.glob("*.!dev"):
control_file.unlink(missing_ok=True)
# merge mutates segment files in place from here on; withdraw the reuse proof
resume.clear_sidecar(save_dir)
if merger:
if segment_decrypter:
segment_decrypter.finish()
if merger.merged != total_segments:
error_msg = f"Rolling merge appended {merger.merged} of {total_segments} segments: {save_dir}"
log_event(
"manifest_hls_rolling_merge_incomplete",
level="ERROR",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"merged": merger.merged,
"total_segments": total_segments,
"downloader": "requests",
},
)
save_path.unlink(missing_ok=True)
raise FileNotFoundError(error_msg)
if save_dir.name.endswith("_segments"):
shutil.rmtree(save_dir, ignore_errors=True)
log_event(
"manifest_hls_download_complete",
level="DEBUG",
message="HLS download complete, segments appended during the download",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"segments_found": merger.merged,
"rolling_merge": True,
"downloader": "requests",
},
)
if session_drm:
progress(downloaded="Decrypting", completed=0, total=None)
decrypt_track(session_drm, save_path, license_widevine, decrypt=not segment_decrypter)
assert_fragments_decrypted(save_path)
track.drm = None
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=session_drm, segment=None)
progress(downloaded="Downloaded", completed=100, total=100)
track.path = save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
return
progress(total=total_segments, completed=0, downloaded="Merging")
name_len = len(str(total_segments))
discon_i = 0
range_offset = 0
# First segment's Media Sequence Number - used as the AES-128 IV when EXT-X-KEY has
# no explicit IV (RFC 8216 §5.2), where each segment's IV is its sequence number.
media_sequence_start = getattr(master, "media_sequence", None) or 0
decrypted_init: Optional[bytes] = None
if session_drm:
encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (initial_drm_key, session_drm)
else:
encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None
i = -1
for real_i, segment in enumerate(master.segments):
if segment not in unwanted_segments:
i += 1
is_last_segment = (real_i + 1) == len(master.segments)
def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False):
"""
Merge all files to a given path, optionally including map data.
Parameters:
to: The output file with all merged data.
via: List of files to merge, in sequence.
delete: Delete each source file after this function merges it.
include_map_data: Whether to include the init map data.
"""
prepend = include_map_data and map_data and map_data[1]
if len(via) == 1 and not prepend and delete:
try:
os.replace(via[0], to)
return
except OSError:
pass
with open(to, "wb") as x:
if prepend:
x.write(map_data[1])
for file in via:
with open(file, "rb") as segment:
shutil.copyfileobj(segment, x, 1024 * 1024)
x.flush()
if delete:
file.unlink()
def decrypt(include_this_segment: bool) -> Path:
"""
Decrypt all segments that uses the currently set DRM.
This function merges every segment of this DRM together in sequence, adds
the init data (if any) at the start, and then deletes the segment files. It
then decrypts the merged file. The merged and decrypted file names state the
range of segments the file holds.
Parameters:
include_this_segment: Whether to include the current segment in the
list of segments to merge and decrypt. This should be False if
decrypting on EXT-X-KEY changes, or True when decrypting on the
last segment.
Returns the decrypted path.
"""
nonlocal map_data, decrypted_init
drm = encryption_data[1]
first_segment_i = next(
(int(file.stem) for file in sorted(segment_save_dir.iterdir()) if file.stem.isdigit()), None
)
last_segment_i = max(0, i - int(not include_this_segment))
if first_segment_i is None or first_segment_i > last_segment_i:
return None
range_len = (last_segment_i - first_segment_i) + 1
segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}"
ext = get_extension(wanted_segments[last_segment_i].uri) or ""
merged_path = segment_save_dir / f"{segment_range}{ext}"
decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}"
files = [
file
for file in sorted(segment_save_dir.iterdir())
if file.stem.isdigit() and first_segment_i <= int(file.stem) <= last_segment_i
]
if not files:
raise ValueError(f"None of the segment files for {segment_range} exist...")
elif len(files) != range_len:
raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...")
if isinstance(drm, (Widevine, PlayReady)):
if segment_decrypter:
if not map_data:
raise ValueError("Segment decryption needs an EXT-X-MAP init segment.")
if decrypted_init is None:
decrypted_init = segment_decrypter.finish()
map_data = (map_data[0], decrypted_init)
merge(to=merged_path, via=files, delete=True, include_map_data=True)
decrypt_track(drm, merged_path, license_widevine, decrypt=not segment_decrypter)
assert_fragments_decrypted(merged_path)
merged_path.rename(decrypted_path)
else:
# AES segments must be decrypted individually before merging: each segment
# likely carries its own 16-byte padding
key_obj = encryption_data[0]
# AES-128 with no explicit IV: each segment's IV is its media sequence
# number (RFC 8216 §5.2). Without this the engine used a zero IV and the
# output decrypted to garbage.
seq_iv = isinstance(drm, ClearKey) and not (key_obj and key_obj.iv)
for file in files:
if seq_iv and file.stem.isdigit():
download_i = int(file.stem)
seq = getattr(wanted_segments[download_i], "media_sequence", None)
if seq is None:
seq = media_sequence_start + download_i
drm.iv = int(seq).to_bytes(16, "big")
drm.decrypt(file)
merge(to=merged_path, via=files, delete=True, include_map_data=True)
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=decrypted_path)
return decrypted_path
def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True):
"""
Merge all segments of the discontinuity.
The caller must download every segment file of this discontinuity first.
The caller must also decrypt them, when they need decryption.
Parameters:
include_this_segment: Whether to include the current segment in the
list of segments to merge and decrypt. This should be False if
decrypting on EXT-X-KEY changes, or True when decrypting on the
last segment.
include_map_data: Whether to prepend the init map data before the
segment files when merging.
"""
last_segment_i = max(0, i - int(not include_this_segment))
files = [
file
for file in sorted(segment_save_dir.iterdir())
if int(file.stem.replace("_decrypted", "").split("-")[-1]) <= last_segment_i
]
if files:
to_dir = segment_save_dir.parent
to_path = to_dir / f"{str(discon_i).zfill(name_len)}{files[-1].suffix}"
merge(to=to_path, via=files, delete=True, include_map_data=include_map_data)
if segment not in unwanted_segments:
if isinstance(track, Subtitle):
segment_file_ext = get_extension(segment.uri) or ""
segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}"
original_data = segment_file_path.read_bytes()
segment_data = try_ensure_utf8(original_data)
if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML):
segment_data = (
segment_data.decode("utf8")
.replace("&lrm;", html.unescape("&lrm;"))
.replace("&rlm;", html.unescape("&rlm;"))
.encode("utf8")
)
if segment_data != original_data:
segment_file_path.write_bytes(segment_data)
if segment.discontinuity and i != 0:
if encryption_data:
decrypt(include_this_segment=False)
merge_discontinuity(
include_this_segment=False, include_map_data=not encryption_data or not encryption_data[1]
)
discon_i += 1
range_offset = 0 # TODO: Should this be reset or not?
map_data = None
if segment.init_section and (not map_data or segment.init_section != map_data[0]):
init_content, range_offset = HLS.fetch_init_section(session, segment.init_section, range_offset)
map_data = (segment.init_section, init_content)
segment_keys = getattr(segment, "keys", None)
if segment_keys and segment not in unwanted_segments:
key = HLS.resolve_segment_key(segment_keys, cdm, track.drm_preference)
if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments:
decrypt(include_this_segment=False)
if key is None:
encryption_data = None
elif not encryption_data or encryption_data[0] != key:
try:
init_kid = track.get_key_id(map_data[1]) if map_data and key.method != "AES-128" else None
drm = HLS.get_drm(key, session, **({"kid": init_kid} if init_kid else {}))
if isinstance(drm, (Widevine, PlayReady)):
track_kid = init_kid or drm.kid
progress(downloaded="LICENSING")
license_widevine(drm, track_kid=track_kid)
progress(downloaded="[yellow]LICENSED")
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
if (
encryption_data
and isinstance(drm, (Widevine, PlayReady))
and isinstance(encryption_data[1], type(drm))
and getattr(encryption_data[1], "content_keys", None)
):
for prev_kid, prev_key in encryption_data[1].content_keys.items():
drm.content_keys.setdefault(prev_kid, prev_key)
encryption_data = (key, drm)
if is_last_segment:
# required as it won't end with EXT-X-DISCONTINUITY nor a new key
if encryption_data:
decrypt(include_this_segment=True)
merge_discontinuity(
include_this_segment=True, include_map_data=not encryption_data or not encryption_data[1]
)
progress(advance=1)
def find_segments_recursively(directory: Path) -> list[Path]:
"""Find all segment files recursively in any directory structure created by downloaders."""
segments = []
if directory.exists():
segments.extend([x for x in directory.iterdir() if x.is_file()])
if not segments:
for subdir in directory.iterdir():
if subdir.is_dir():
segments.extend(find_segments_recursively(subdir))
return sorted(segments)
segments_to_merge = find_segments_recursively(save_dir)
log_event(
"manifest_hls_download_complete",
level="DEBUG",
message="HLS download complete, preparing to merge",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_dir_exists": save_dir.exists(),
"segments_found": len(segments_to_merge),
"segment_files": [f.name for f in segments_to_merge[:10]],
"downloader": "requests",
},
)
if not segments_to_merge:
error_msg = f"No segment files found in output directory: {save_dir}"
all_contents = list(save_dir.iterdir()) if save_dir.exists() else []
log_event(
"manifest_hls_download_no_segments",
level="ERROR",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_dir_exists": save_dir.exists(),
"directory_contents": [str(p) for p in all_contents],
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
if len(segments_to_merge) == 1:
shutil.move(segments_to_merge[0], save_path)
else:
progress(downloaded="Merging", completed=0, total=len(segments_to_merge))
if isinstance(track, (Video, Audio)):
HLS.merge_segments(segments=segments_to_merge, save_path=save_path, progress=progress)
else:
with open(save_path, "wb") as f:
for discontinuity_file in segments_to_merge:
with open(discontinuity_file, "rb") as src:
shutil.copyfileobj(src, f, 1024 * 1024)
f.flush()
os.fsync(f.fileno())
discontinuity_file.unlink()
if save_dir.exists() and save_dir.name.endswith("_segments"):
try:
save_dir.rmdir()
except OSError:
shutil.rmtree(save_dir, ignore_errors=True)
progress(downloaded="Downloaded", completed=100, total=100)
track.path = save_path
if session_drm:
track.drm = None
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
@staticmethod
def merge_segments(segments: list[Path], save_path: Path, progress: Optional[Any] = None) -> int:
"""
Concatenate Segments using FFmpeg concat with binary fallback.
FFmpeg concat reports nothing per segment, so progress only advances on the binary
fallback, which copies one segment at a time.
Returns the file size of the merged file.
"""
segment_dirs = set()
for segment in segments:
current_dir = segment.parent
while current_dir.name:
if current_dir.name.endswith("_segments"):
segment_dirs.add(current_dir)
break
current_dir = current_dir.parent
def cleanup_segments_and_dirs():
"""Clean up segments and directories after successful merge."""
for segment in segments:
segment.unlink(missing_ok=True)
for segment_dir in segment_dirs:
if segment_dir.exists():
try:
shutil.rmtree(segment_dir)
except OSError:
pass # Directory cleanup failed, but merge succeeded
if binaries.FFMPEG:
try:
demuxer_file = save_path.parent / f"ffmpeg_concat_demuxer_{save_path.stem}.txt"
demuxer_file.write_text(
"\n".join([f"file '{segment.absolute()}'" for segment in segments]),
encoding="utf-8",
newline="",
)
concat_start = time.monotonic()
subprocess.run(
[
binaries.FFMPEG,
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-f",
"concat",
"-safe",
"0",
"-i",
demuxer_file,
"-map",
"0",
"-c",
"copy",
save_path,
],
check=True,
timeout=300,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
demuxer_file.unlink(missing_ok=True)
if progress:
progress(completed=len(segments))
cleanup_segments_and_dirs()
log_tool_run(
"ffmpeg concat segments",
"ffmpeg",
0,
duration_ms=round((time.monotonic() - concat_start) * 1000, 1),
segments=len(segments),
output_size=save_path.stat().st_size if save_path.exists() else 0,
)
return save_path.stat().st_size
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e:
stderr = getattr(e, "stderr", None) or ""
if isinstance(stderr, bytes):
stderr = stderr.decode("utf-8", "replace")
logging.getLogger("HLS").debug(
f"FFmpeg concat failed ({e}), falling back to binary concatenation"
+ (f": {stderr.strip()}" if stderr.strip() else "")
)
demuxer_file.unlink(missing_ok=True)
save_path.unlink(missing_ok=True)
logging.getLogger("HLS").debug(f"Using binary concatenation for {len(segments)} segments")
with open(save_path, "wb") as output_file:
for segment in segments:
with open(segment, "rb") as segment_file:
shutil.copyfileobj(segment_file, output_file, 1024 * 1024)
if progress:
progress(advance=1)
cleanup_segments_and_dirs()
return save_path.stat().st_size
@staticmethod
def parse_session_data_keys(
manifest: M3U8, session: Optional[Union[Session, RnetSession]] = None
) -> list[m3u8.model.Key]:
"""Parse the `com.apple.hls.keys` `EXT-X-SESSION-DATA` entry into `Key` objects."""
keys: list[m3u8.model.Key] = []
for data in getattr(manifest, "session_data", []) or []:
if getattr(data, "data_id", None) != "com.apple.hls.keys":
continue
value = getattr(data, "value", None)
if not value and data.uri:
if not session:
session = run_proxy.fallback_session()
value = utf8_text(session.get(urljoin(manifest.base_uri or "", data.uri)))
if not value:
continue
try:
decoded = base64.b64decode(value).decode()
except ValueError:
decoded = value
try:
items = json.loads(decoded)
except ValueError:
continue
for item in items if isinstance(items, list) else []:
if not isinstance(item, dict):
continue
key = m3u8.model.Key(
method=item.get("method"),
base_uri=manifest.base_uri or "",
uri=item.get("uri"),
keyformat=item.get("keyformat"),
keyformatversions=",".join(item.get("keyformatversion") or item.get("keyformatversions") or []),
)
if key.method in {"AES-128", "ISO-23001-7"} or (
key.keyformat
and key.keyformat.lower()
in {
WidevineCdm.urn,
PlayReadyCdm,
"com.microsoft.playready",
}
):
keys.append(key)
return keys
@staticmethod
def filter_keys_for_cdm(
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]],
cdm: object,
drm_preference: Optional[str] = None,
) -> list[Union[m3u8.model.SessionKey, m3u8.model.Key]]:
"""
Filter EXT-X-KEY entries to only include those matching the CDM type.
This makes sure that we select the correct DRM system (Widevine or PlayReady)
for the CDM in the configuration, so the challenge does not fail.
A track's drm_preference wins over the CDM. If the playlist has no EXT-X-KEY entry
for it, the CDM decides as usual.
"""
playready_urn = f"urn:uuid:{PR_PSSH.SYSTEM_ID}"
playready_keyformats = {playready_urn, "com.microsoft.playready"}
if drm_preference:
if DRM_PREFERENCE_TYPES[drm_preference] is PlayReady:
preferred = [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats]
else:
preferred = [k for k in keys if k.keyformat and k.keyformat.lower() == WidevineCdm.urn]
if preferred:
return preferred
logging.getLogger("HLS").warning(
f"Track wants {drm_preference} DRM but this playlist has no EXT-X-KEY entry for it, "
"using the CDM's DRM instead"
)
if is_widevine_cdm(cdm):
return [k for k in keys if k.keyformat and k.keyformat.lower() == WidevineCdm.urn]
elif is_playready_cdm(cdm):
return [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats]
return keys
@staticmethod
def alternate_drm(
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]],
chosen: DRM_T,
session: Optional[Union[Session, RnetSession]] = None,
kid: Optional[UUID] = None,
) -> list[DRM_T]:
"""Build the Widevine or PlayReady DRM that the EXT-X-KEY entries offer next to ``chosen``.
Only an entry with the KEYFORMAT of the other DRM system counts, so unshackle requests no URI.
"""
system = "widevine" if isinstance(chosen, PlayReady) else "playready"
other_keys = [k for k in HLS.filter_keys_for_cdm(keys, cdm_type_stub(system)) if k.method != "NONE"]
if not other_keys:
return []
try:
return [HLS.get_drm(other_keys[0], session, **({"kid": kid} if kid else {}))]
except Exception as e:
logging.getLogger("HLS").warning(f"Could not read the {system} EXT-X-KEY entry: {e}")
return []
@staticmethod
def get_track_kid_from_init(
master: M3U8,
track: AnyTrack,
session: Union[Session, RnetSession],
init_section: Optional[m3u8.model.InitializationSection] = None,
) -> Optional[UUID]:
"""
Extract the track's Key ID from an init segment (EXT-X-MAP), by default the first in the playlist.
Returns None if no init segment exists or KID extraction fails.
The caller should fall back to drm.kid from the PSSH if this returns None.
"""
map_section = init_section or next((seg.init_section for seg in master.segments if seg.init_section), None)
if not map_section:
return None
map_uri = urljoin(map_section.base_uri or master.base_uri or "", map_section.uri)
try:
if map_section.byterange:
byte_range = HLS.calculate_byte_range(map_section.byterange, 0)
headers = {"Range": f"bytes={byte_range}"}
else:
headers = {}
map_res = session.get(url=map_uri, headers=headers)
if map_res.ok:
return track.get_key_id(map_res.content)
logging.getLogger("HLS").warning(
f"Could not read the Key ID from the init segment {safe_display_url(map_uri)}: "
f"HTTP {map_res.status_code}"
)
# best-effort key-id probe over network + init parse; caller falls back without it
except Exception as e:
logging.getLogger("HLS").debug(f"Init map key-id probe failed for {map_uri}: {e!r}")
return None
@staticmethod
def get_supported_key(keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]]) -> Optional[m3u8.Key]:
"""
Get the first DRM system unshackle can use from the given `EXT-X-KEY` entries.
Note that unshackle chooses the DRM systems in an opinionated order.
Returns None if one of the DRM systems is method=NONE. Then every segment from
that point on is plain text, until another DRM system that is not method=NONE
comes in the playlist.
Raises NotImplementedError when unshackle can use none of the DRM systems.
"""
if any(key.method == "NONE" for key in keys):
return None
unsupported_systems = []
for key in keys:
if not key:
continue
# TODO: Add a way to specify which supported key system to use
# TODO: Add support for 'SAMPLE-AES', 'AES-CTR', 'AES-CBC', 'ClearKey'
elif key.method == "AES-128":
return key
elif key.method == "ISO-23001-7":
return key
elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn:
return key
elif key.keyformat and key.keyformat.lower() in {
f"urn:uuid:{PR_PSSH.SYSTEM_ID}",
"com.microsoft.playready",
}:
return key
else:
unsupported_systems.append(key.method + (f" ({key.keyformat})" if key.keyformat else ""))
else:
raise NotImplementedError(f"None of the key systems are supported: {', '.join(unsupported_systems)}")
@staticmethod
def get_drm(
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
session: Optional[Union[Session, RnetSession]] = None,
kid: Optional[UUID] = None,
) -> DRM_T:
"""
Convert HLS EXT-X-KEY data to an initialized DRM object.
Parameters:
key: The m3u8 `EXT-X-KEY` object.
session: Optional HTTP session used to request AES-128 URIs.
Useful to set headers, proxies, cookies, and so forth.
kid: The Key ID from the init segment. Widevine uses it only when the tag
and the PSSH both carry no real Key ID.
Raises a NotImplementedError when unshackle cannot use the DRM system.
"""
if not isinstance(session, (Session, RnetSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}")
if not session:
session = run_proxy.fallback_session()
# TODO: Add support for 'SAMPLE-AES', 'AES-CTR', 'AES-CBC', 'ClearKey'
if key.method == "AES-128":
drm = ClearKey.from_m3u_key(key, session)
elif key.method == "ISO-23001-7":
drm = Widevine(pssh=WV_PSSH.new(key_ids=[key.uri.split(",")[-1]], system_id=WV_PSSH.SystemId.Widevine))
elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn:
extra_params = dict(key._extra_params) # noqa
pssh = WV_PSSH(key.uri.split(",")[-1])
if kid in Widevine.PLACEHOLDER_KIDS or any(k not in Widevine.PLACEHOLDER_KIDS for k in pssh.key_ids):
kid = None
# a v0 PSSH can carry no KID: the tag's own KEYID is the first source, the init segment the last
tag_kid = extra_params.pop("keyid", None) or extra_params.pop("kid", None)
if isinstance(tag_kid, str):
tag_kid = tag_kid.removeprefix("0x").removeprefix("0X")
drm = Widevine(
pssh=pssh,
kid=tag_kid or kid,
**extra_params,
)
elif key.keyformat and key.keyformat.lower() in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}:
drm = PlayReady(
pssh=PR_PSSH(key.uri.split(",")[-1]),
pssh_b64=key.uri.split(",")[-1],
)
else:
raise NotImplementedError(f"The key system is not supported: {key}")
return drm
@staticmethod
def get_drm_or_probe(
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
master: M3U8,
track: AnyTrack,
session: Optional[Union[Session, RnetSession]] = None,
init_section: Optional[m3u8.model.InitializationSection] = None,
probe_session: Any = None,
) -> DRM_T:
"""
Convert HLS EXT-X-KEY data to a DRM object, with the Key ID of the init segment when the key carries none.
The init segment request uses `probe_session` when the caller has no HTTP session that `get_drm` accepts.
"""
try:
return HLS.get_drm(key, session)
except Widevine.Exceptions.KIDNotFound:
kid = HLS.get_track_kid_from_init(master, track, probe_session or session or run_proxy.fallback_session(), init_section)
if not kid:
raise
return HLS.get_drm(key, session, kid=kid)
@staticmethod
def get_all_drm(
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], proxy: Optional[str] = None
) -> list[DRM_T]:
"""
Convert HLS EXT-X-KEY data to initialized DRM objects.
Parameters:
keys: The m3u8 `EXT-X-KEY` objects.
proxy: Optional proxy string used for requesting AES-128 URIs.
Raises a NotImplementedError when unshackle can use none of the DRM systems.
"""
unsupported_keys: list[m3u8.Key] = []
drm_objects: list[DRM_T] = []
if any(key.method == "NONE" for key in keys):
return []
for key in keys:
try:
drm = HLS.get_drm(key, proxy)
drm_objects.append(drm)
except NotImplementedError:
unsupported_keys.append(key)
except Widevine.Exceptions.KIDNotFound:
logging.getLogger("HLS").warning(
"Skipped an EXT-X-SESSION-KEY entry that carries no Key ID. The media playlist must supply the key."
)
if not drm_objects and unsupported_keys:
logging.debug(
"Ignoring unsupported key systems: %s",
", ".join([str(k.keyformat or k.method) for k in unsupported_keys]),
)
return []
return drm_objects
@staticmethod
def calculate_byte_range(m3u_range: str, fallback_offset: int = 0) -> str:
"""
Convert a HLS EXT-X-BYTERANGE value to a more traditional range value.
E.g., '1433@0' -> '0-1432', '357392@1433' -> '1433-358824'.
"""
parts = [int(x) for x in m3u_range.split("@")]
if len(parts) != 2:
parts.append(fallback_offset)
length, offset = parts
return f"{offset}-{offset + length - 1}"
__all__ = ("HLS",)