kenzuya/unshackle
1
0
Fork 2
mirror of https://github.com/unshackle-dl/unshackle.git synced 2026-11-14 21:15:06 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
d8663b0dadb9ddbb92376c08856da3b4ab47c3b3
unshackle/unshackle/commands/dl.py
T
imSp4rky 9f054269db fix(vaults): copy no vault key from a clear track 
kid_windows returns an empty map for an MP4 with no encrypted sample entry or protecting senc, so the key check judges, flags and copies nothing for it.
2026-10-23 16:59:40 -06:00

5814 lines
276 KiB
Python
Raw Blame History

from __future__ import annotations
import html
import logging
import math
import os
import random
import re
import shutil
import subprocess
import sys
import time
from collections import Counter, defaultdict
from collections.abc import Iterable
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from datetime import date, timedelta
from functools import partial
from http.cookiejar import CookieJar, MozillaCookieJar
from io import StringIO
from itertools import product
from pathlib import Path
from threading import Lock
from typing import Any, Callable, Collection, Optional, Sequence, TypedDict, Union
from uuid import UUID
import click
import mediaexport
from click.core import ParameterSource
from langcodes import Language, tag_is_valid
from pymediainfo import MediaInfo
from rich.console import Group
from rich.padding import Padding
from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn
from rich.rule import Rule
from rich.spinner import Spinner
from rich.table import Table
from rich.text import Text
from rich.tree import Tree
from unshackle.core import __version__, binaries, providers
from unshackle.core.cdm import DecryptLabsRemoteCDM
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote_cdm, is_widevine_cdm
from unshackle.core.config import cdm_entry_names_device, config, load_yaml, resolve_cdm_name, resolve_decryption
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
from unshackle.core.constants import (
DEFAULT_SAMPLE,
DOWNLOAD_ALL_DRM,
DOWNLOAD_CANCELLED,
DOWNLOAD_LICENCE_ONLY,
AnyTrack,
context_settings,
)
from unshackle.core.credential import Credential
from unshackle.core.downloaders import default_max_workers, format_speed, parse_speed_limit, set_speed_limit
from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevine, cbcs, own_kids, real_kids, verify
from unshackle.core.events import events
from unshackle.core.export_name import ExportNamer, move_export, plural, season_episodes, title_label
from unshackle.core.providers.anilist import parse_anilist_ref
from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int
from unshackle.core.proxies.resolve import (
REGION,
describe_explicit_proxy,
describe_proxy,
is_loopback,
load_proxy_providers,
pick_proxy,
require_alone_local_proxy,
resolve_proxy,
split_proxy_query,
)
from unshackle.core.service import Service, grow_session_pool
from unshackle.core.services import ROUTING_PARAMS, Services
from unshackle.core.temp import with_task_temp
from unshackle.core.title_cacher import get_account_hash
from unshackle.core.titles import Movie, Movies, Series, Song, Title_T
from unshackle.core.titles.episode import Episode
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
from unshackle.core.tracks.attachment import Attachment
from unshackle.core.tracks.dv_fixup import apply_dv_fixup
from unshackle.core.tracks.hybrid import Hybrid
from unshackle.core.tracks.track import assert_fragments_decrypted, has_encrypted_sample_entry
from unshackle.core.tracks.tracks import at_resolution
from unshackle.core.utilities import (
apply_original_language,
as_requested,
declared_kwargs,
embedded_audio_langs,
find_font_with_fallbacks,
find_missing_langs,
get_debug_logger,
get_system_fonts,
init_debug_logger,
kind_only_conflict,
log_event,
matched_language_tags,
matching_languages,
missing_required_langs,
original_tag,
partition_exclusions,
plain_subtitle_tags,
resolve_sort_langs,
suggest_font_packages,
time_elapsed_since,
)
from unshackle.core.utils import tags
from unshackle.core.utils.bitrate import apply_real_bitrates
from unshackle.core.utils.click_types import (
AUDIO_CODEC_LIST,
LANGUAGE_RANGE,
QUALITY_LIST,
SAMPLE_RANGE,
SEASON_RANGE,
SLOW_DELAY_RANGE,
SUBTITLE_CODEC,
VIDEO_CODEC_LIST,
ContextData,
MultipleChoice,
)
from unshackle.core.utils.collections import ci_get, merge_dict
from unshackle.core.utils.hdr10plus import probed_tracks
from unshackle.core.utils.post_scripts import NO_POST_SCRIPTS, build_context, dispatch, season_context
from unshackle.core.utils.redact import mask_proxy
from unshackle.core.utils.selector import select_multiple
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe, run_step
from unshackle.core.vault import Vault
from unshackle.core.vaults import Vaults
class SkippedSubtitle(TypedDict):
"""A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``,
one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable."""
id: str
language: str
title: str
# Config keys accepted as natural names for params whose Python name dodges a builtin.
DL_OPTION_ALIASES = {"range": "range_", "list": "list_"}
def parse_language(tag: Optional[str]) -> Optional[Language]:
"""A provider's original-language tag as a Language, or None if it is absent or not a real tag."""
tag = (tag or "").strip()
if not tag or not tag_is_valid(tag) or tag.startswith("und"):
return None
return Language.get(tag)
def normalize_dl_config(dl_config: dict[str, Any]) -> dict[str, Any]:
"""Map aliased config keys (e.g. ``range``) to their Click parameter names (``range_``)."""
return {DL_OPTION_ALIASES.get(key, key): value for key, value in dl_config.items()}
ID_PROVIDER_KEYS = {"tmdb": "tmdb_api_key", "tvdb": "tvdb_api_key", "omdb": "omdb_api_key"}
def validate_metadata_ids(
tmdb_id: Optional[int],
imdb_id: Optional[str],
tvdb_id: Optional[int],
anilist_id: Optional[Union[int, str]] = None,
) -> None:
"""Reject ID flags that conflict, or that no configured metadata provider could ever find.
--anilist stays outside the mutual exclusion: AniList knows no western IDs, so pairing
it with one of the other three is how an anime title gets both.
"""
if anilist_id and not parse_anilist_ref(anilist_id):
raise click.UsageError("--anilist takes an AniList ID like 21, or a MyAnimeList ID like mal:21.")
supplied = [
(flag, value) for flag, value in (("--tmdb", tmdb_id), ("--imdb", imdb_id), ("--tvdb", tvdb_id)) if value
]
if len(supplied) > 1:
raise click.UsageError(
f"{', '.join(flag for flag, _ in supplied)} cannot be used together. "
"Give one ID and unshackle resolves the others from it."
)
if not supplied:
return
flag, _ = supplied[0]
id_kind = flag.lstrip("-")
ordered = {cls.NAME: cls for kind in ("tv", "movie") for cls in providers.provider_order(kind)}
consumers = [cls for cls in ordered.values() if cls.ID_KIND == id_kind]
if any(cls().is_available() for cls in consumers):
return
if consumers:
keys = sorted({ID_PROVIDER_KEYS[cls.NAME] for cls in consumers if cls.NAME in ID_PROVIDER_KEYS})
raise click.UsageError(
f"{flag} needs the {consumers[0].NAME} provider. Set {' or '.join(keys)} in your config."
)
names = sorted({cls.NAME for cls in providers.REGISTRY.values() if cls.ID_KIND == id_kind})
raise click.UsageError(
f"{flag} needs the {' or '.join(names)} provider, which your metadata_providers config leaves out."
)
def group_videos_by_variant(videos: list[Video], *, merge: bool) -> list[list[Video]]:
"""Group video tracks for muxing.
When ``merge`` is True, this function groups tracks that share ``(height, range, codec)``
into one file, so only language varies within a group. Different resolutions, ranges and
codecs stay in separate groups (separate files). When False, each track is its own group
(one file per track, the default behaviour). Group order follows first-seen track order.
"""
if not merge:
return [[video] for video in videos]
groups: dict[tuple[Any, ...], list[Video]] = {}
for video in videos:
groups.setdefault((video.height, video.range, video.codec), []).append(video)
return list(groups.values())
def select_hybrid_bases(videos: Sequence[Video]) -> list[Video]:
"""The HDR10 or HDR10+ track that is the hybrid base for each height, tallest first.
Only HEVC tracks qualify, because dovi_tool injects the RPU into an HEVC stream.
"""
bases: dict[int, Video] = {}
for video in sorted(
(
v
for v in videos
if v.range in (Video.Range.HDR10P, Video.Range.HDR10) and v.codec in (Video.Codec.HEVC, None)
),
key=lambda v: v.height or 0,
reverse=True,
):
bases.setdefault(video.height or 0, video)
return list(bases.values())
def hybrid_gate_failure(videos: Sequence[Video], generate_dv: bool) -> Optional[tuple[str, str]]:
"""The (message, detail) that stops HYBRID for these selected videos, or None when it can go on."""
if any(v.native_hybrid for v in videos):
return None
hdr_tracks = [v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)]
base_tracks = [v for v in hdr_tracks if v.codec in (Video.Codec.HEVC, None)]
dv_tracks = [v for v in videos if v.range == Video.Range.DV]
has_hdr10p = any(v.range == Video.Range.HDR10P for v in base_tracks)
available_ranges = sorted(set(v.range.name for v in videos))
if not hdr_tracks and not dv_tracks:
return (
"HYBRID mode requires both HDR10/HDR10+ and DV tracks, but neither is available",
f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}",
)
if not hdr_tracks:
return (
"HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only DV is available",
f"Available ranges: {', '.join(available_ranges)}",
)
if not base_tracks:
codecs = sorted(set(v.codec.name for v in hdr_tracks if v.codec))
return (
"HYBRID mode requires an HEVC HDR10/HDR10+ base, but none is available",
f"Available HDR10/HDR10+ codecs: {', '.join(codecs)}",
)
if not dv_tracks and not (generate_dv and has_hdr10p):
msg_detail = f"Available ranges: {', '.join(available_ranges)}"
if generate_dv:
msg_detail += ". --generate-dv needs an HEVC HDR10+ track, and the service returned none."
elif has_hdr10p:
msg_detail += ". Add --generate-dv to generate Dolby Vision from HDR10+."
return "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only HDR10 is available", msg_detail
return None
def hybrid_plan(videos: Sequence[Video], generate_dv: bool) -> tuple[list[Video], Optional[Video]]:
"""The hybrid bases and the DV track to pair with them.
With no DV track, only HDR10+ bases qualify, and only with ``--generate-dv``. A resolution with a
finished hybrid (``native_hybrid``) gets no base: that track is its hybrid.
"""
dv_tracks = [v for v in videos if v.range == Video.Range.DV]
native_heights = {v.height for v in videos if v.native_hybrid}
base_pool = [
v
for v in videos
if v.height not in native_heights and (dv_tracks or (generate_dv and v.range == Video.Range.HDR10P))
]
matching_dv = min(dv_tracks, key=lambda v: v.height or 0) if dv_tracks else None
return select_hybrid_bases(base_pool), matching_dv
def hybrid_source_tracks(base: Video, dv: Optional[Video], base_container: Optional[Path] = None) -> list[Video]:
"""Copies of the base and DV tracks for the hybrid step, with the base on its container file.
The hybrid step reads the duration and seeks, and the raw stream from the DV fixup
permits neither. With no DV track, the hybrid step makes Dolby Vision from the base's HDR10+.
"""
tracks = [deepcopy(track) for track in (base, dv) if track]
tracks[0].path = base_container or base.path
for track in tracks:
track.needs_duration_fix = True
return tracks
def hybrid_step_error(tracks: list[Video], source: str, output_name: str) -> Optional[str]:
"""Run the hybrid step for one resolution; the error that failed it, or None.
A failure here is this resolution's alone: the title goes on with its other resolutions.
"""
try:
Hybrid(tracks, source, output_name=output_name)
except (ValueError, KeyError, OSError, RuntimeError) as e:
return str(e) or type(e).__name__
return None
def missing_hybrid_error(bases: Sequence[Video], made: int) -> Optional[str]:
"""Why the title has no hybrid file, or None when at least one was made."""
if made:
return None
if not bases:
return "No HEVC HDR10 or HDR10+ track to make a hybrid from"
return f"Every hybrid failed ({', '.join(f'{v.height or 0}p' for v in bases)}), so the title has no hybrid file"
def title_wanted(candidate: Any, wanted: Collection[str]) -> bool:
"""Whether ``-w`` selects this title. A title type without selection keys is always kept.
The download loop and the post-script counter both read this, so the counter cannot
fall out of step with the titles the loop skips.
"""
if not wanted or not isinstance(candidate, (Episode, Song)):
return True
return bool(candidate.matches_wanted(wanted))
def latest_episode_keys(titles: Any, latest_episode: bool, latest_episodes: Optional[int]) -> list[tuple[int, int]]:
"""The (season, number) of each episode ``--latest-episode(s)`` keeps, oldest first.
Counts distinct episodes, not Series entries, so a split episode takes one slot and all its parts download.
"""
count = latest_episodes or (1 if latest_episode else 0)
if not count or not isinstance(titles, Series):
return []
# a Series keeps itself sorted, so the newest episodes are last
return list(dict.fromkeys((ep.season, ep.number) for ep in titles))[-count:]
SAMPLE_DIR = "Sample"
def template_type(title: Any) -> str:
"""Return the output template key of a title: ``series``, ``songs`` or ``movies``."""
return "series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
def sample_name(name: str, title: Any) -> str:
"""Return ``name`` with a SAMPLE tag, joined with the separator of the output template of the title."""
return f"{name.rstrip()}{config.get_template_separator(template_type(title))}SAMPLE"
def apply_enrich(title: Title_T, name: Optional[str], year: Optional[int], language: Optional[Language]) -> None:
"""Set the --enrich name, year and original language on a title. A song takes only the language.
A service can set the title again in get_tracks, so dl applies them after that call too.
"""
if isinstance(title, Episode):
title.title = name or title.title
elif isinstance(title, Movie):
title.name = name or title.name
if isinstance(title, (Episode, Movie)):
title.year = year or title.year
if language:
apply_original_language(title, language)
def load_tracks(
service: Service,
title: Title_T,
*,
name: Optional[str] = None,
year: Optional[int] = None,
language: Optional[Language] = None,
workers: Optional[int] = None,
on_probe: Optional[Callable[[], None]] = None,
) -> None:
"""Load the tracks and chapters of a title, then apply the --enrich name, year and language again."""
title.tracks.add(probed_tracks(service, title, workers=workers, on_probe=on_probe), warn_only=True)
title.tracks.chapters = service.get_chapters(title)
apply_enrich(title, name, year, language)
def server_url(server_name: Optional[str]) -> str:
"""The configured URL of a remote server, or an empty string when the config cannot be read."""
from unshackle.core.remote_service import resolve_server
try:
return resolve_server(server_name)[0]
except Exception:
return ""
def post_script_group(candidate: Any) -> Any:
"""Group a title by the folder its outputs share, for post-scripts in season mode."""
if isinstance(candidate, Episode):
return ("episode", candidate.title, candidate.season)
if isinstance(candidate, Song):
artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "")
return ("album", artist, getattr(candidate, "album", ""))
return ("title", id(candidate))
def remux_music_tracks(title: Any, log: logging.Logger) -> None:
"""Remux each audio track of a Song into the container of its codec.
A Song never reaches the muxer, so each downloaded audio track is itself a delivered
file. A failure warns, keeps that downloaded track, and moves on to the next one: the
user still gets the audio, in a container a player cuts short.
"""
if not isinstance(title, Song):
return
for track in title.tracks.audio:
try:
track.to_music_container()
except Exception as e: # noqa: BLE001
log.warning(f"Could not remux audio track {track.id} for music: {e}")
def cut_music_sample(title: Any, window: tuple[int, int], log: logging.Logger) -> None:
"""Cut each audio track of a Song to the sample window. A Song has no mux step that cuts it."""
if not isinstance(title, Song) or not binaries.FFMPEG:
return
for track in title.tracks.audio:
if not track.path or not track.path.exists():
continue
cut_path = track.path.with_stem(f"{track.path.stem}_cut")
try:
run_step(
[
binaries.FFMPEG,
"-v",
"error",
"-y",
"-ss",
f"{max(window[0] - track.sample_start, 0):.3f}",
"-i",
track.path,
"-t",
str(window[1] - window[0]),
"-map",
"0",
"-c",
"copy",
cut_path,
],
output=cut_path,
label="music sample cut",
)
os.replace(cut_path, track.path)
except RuntimeError as e:
log.warning(f"Could not cut audio track {track.id} to the sample window, so it holds whole segments: {e}")
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
The config replaces defaults and global ``dl:`` default_map values.
``remote`` and ``server`` are skipped: they pick the code that runs, so ``Services.resolve_command`` sets
them before the service loads, and an API job takes them from its request."""
params_by_name = {param.name: param for param in ctx.command.params if param.name}
for name, value in normalize_dl_config(service_dl_config).items():
if name in ROUTING_PARAMS:
continue
param = params_by_name.get(name)
if param is None:
log.warning(f"Ignoring unknown dl option '{name}' in service config")
continue
if name not in ctx.params:
log.debug(f"Skipping service dl override '{name}': not present in this context")
continue
if value is None:
continue
source = ctx.get_parameter_source(name)
if source not in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP):
continue
try:
ctx.params[name] = param.type_cast_value(ctx, value)
log.debug(f"Applied service dl override '{name}': {ctx.params[name]}")
except Exception as e:
log.warning(
f"Failed to apply service dl override '{name}'={value!r}: {e}. "
f"Check that the value is valid for this parameter."
)
def download_tracks_in_passes(
tracks: Iterable[AnyTrack],
max_concurrent: int,
run_one: Callable[[AnyTrack, int], None],
*,
skip_subtitle_errors: bool,
on_subtitle_skipped: Callable[[Subtitle], None],
) -> None:
"""Download a title's tracks so a skippable subtitle failure cannot corrupt the rest.
A failed track sets the process-global ``DOWNLOAD_CANCELLED`` event, making other in-flight
tracks early-return without raising. With ``skip_subtitle_errors`` set, video/audio download
concurrently first. The subtitles then download one at a time (a concurrent pass would let one
failure's cancel silently drop the others, unrecorded), with the event cleared before each.
Video/audio failures stay fatal. This function clears the event on entry (stale cancel from a
prior title) and in ``finally`` (never leave it set for later code).
``run_one(track, index)`` downloads a single track. ``on_subtitle_skipped(track)`` records a
subtitle whose download raised (handled here).
"""
DOWNLOAD_CANCELLED.clear()
indexed = list(enumerate(tracks))
if skip_subtitle_errors:
primary = [(i, t) for i, t in indexed if not isinstance(t, Subtitle)]
skippable = [(i, t) for i, t in indexed if isinstance(t, Subtitle)]
else:
primary, skippable = indexed, []
try:
with ThreadPoolExecutor(max_concurrent) as pool:
future_to_track = {pool.submit(run_one, track, i): track for i, track in primary}
for download in futures.as_completed(future_to_track):
download.result() # a video/audio failure is fatal
for i, track in skippable:
DOWNLOAD_CANCELLED.clear()
try:
run_one(track, i)
except Exception:
on_subtitle_skipped(track)
finally:
DOWNLOAD_CANCELLED.clear() # never leave a failed track's cancel set for later code
class dl:
@staticmethod
def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str:
"""Truncate PSSH string for display when not in debug mode."""
if logging.root.level == logging.DEBUG or not pssh_string:
return pssh_string
max_width = console.width - len(drm_type) - 12
if len(pssh_string) <= max_width:
return pssh_string
return pssh_string[: max_width - 3] + "..."
def find_custom_font(self, font_name: str) -> Optional[Path]:
"""
Find font in custom fonts directory.
Args:
font_name: Font family name to find
Returns:
Path to font file, or None if not found
"""
family_dir = Path(config.directories.fonts, font_name)
if family_dir.exists():
fonts = list(family_dir.glob("*.*tf"))
return fonts[0] if fonts else None
return None
def prepare_temp_font(self, font_name: str, matched_font: Path, system_fonts: dict[str, Path]) -> Path:
"""
Copy system font to temp and log if using fallback.
Args:
font_name: Requested font name
matched_font: Path to matched system font
system_fonts: Dictionary of available system fonts
Returns:
Path to temp font file
"""
matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None)
if matched_name and matched_name.lower() != font_name.lower():
self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'")
safe_name = font_name.replace(" ", "_").replace("/", "_")
temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}"
if not temp_path.exists():
shutil.copy2(matched_font, temp_path)
return temp_path
def attach_subtitle_fonts(self, font_names: list[str], title: Title_T) -> tuple[int, list[str]]:
"""
Attach fonts for subtitle rendering.
Args:
font_names: List of font names requested by subtitles
title: Title object to attach fonts to
Returns:
Tuple of (fonts_attached_count, missing_fonts_list)
"""
system_fonts = get_system_fonts()
font_count = 0
missing_fonts = []
for font_name in set(font_names):
if custom_font := self.find_custom_font(font_name):
title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})"))
font_count += 1
continue
if system_font := find_font_with_fallbacks(font_name, system_fonts):
temp_path = self.prepare_temp_font(font_name, system_font, system_fonts)
title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})"))
font_count += 1
else:
self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found")
missing_fonts.append(font_name)
return font_count, missing_fonts
def suggest_missing_fonts(self, missing_fonts: list[str]) -> None:
"""
Show package installation suggestions for missing fonts.
Args:
missing_fonts: List of font names that could not be found
"""
if suggestions := suggest_font_packages(missing_fonts):
self.log.info("Install font packages to improve subtitle rendering:")
for package_cmd, fonts in suggestions.items():
self.log.info(f" $ sudo apt install {package_cmd}")
self.log.info(f" → Provides: {', '.join(fonts)}")
def generate_sidecar_subtitle_path(
self,
subtitle: Subtitle,
base_filename: str,
output_dir: Path,
target_codec: Optional[Subtitle.Codec] = None,
source_path: Optional[Path] = None,
) -> Path:
"""Make the sidecar path: {base}.{lang}[.forced][.sdh].{ext}"""
lang_suffix = str(subtitle.language) if subtitle.language else "und"
forced_suffix = ".forced" if subtitle.forced else ""
sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else ""
extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension
if not target_codec and not subtitle.codec and source_path and source_path.suffix:
extension = source_path.suffix.lstrip(".")
filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}"
return output_dir / filename
def output_subtitle_sidecars(
self,
subtitles: list[Subtitle],
base_filename: str,
output_dir: Path,
sidecar_format: str,
original_paths: Optional[dict[str, Path]] = None,
) -> list[Path]:
"""
Output subtitles as sidecar files, converting if needed.
Conversion runs on a temp copy, so each subtitle's own file stays untouched and mkvmerge can still mux it.
"""
created_paths: list[Path] = []
config.directories.temp.mkdir(parents=True, exist_ok=True)
for subtitle in subtitles:
source_path = subtitle.path
if sidecar_format == "original" and original_paths and subtitle.id in original_paths:
source_path = original_paths[subtitle.id]
if not source_path or not source_path.exists():
continue
if sidecar_format == "original":
target_codec = None
if source_path.suffix:
try:
target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip("."))
except ValueError:
target_codec = None
else:
target_codec = Subtitle.Codec.from_mime(sidecar_format)
sidecar_path = self.generate_sidecar_subtitle_path(
subtitle, base_filename, output_dir, target_codec, source_path=source_path
)
if not target_codec or subtitle.codec == target_codec:
shutil.copy2(source_path, sidecar_path)
else:
temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}"
shutil.copy2(source_path, temp_path)
temp_sub = Subtitle(
subtitle.url,
subtitle.language,
is_original_lang=subtitle.is_original_lang,
descriptor=subtitle.descriptor,
codec=subtitle.codec,
forced=subtitle.forced,
sdh=subtitle.sdh,
cc=subtitle.cc,
id_=f"{subtitle.id}_sc",
)
temp_sub.path = temp_path
try:
temp_sub.convert(target_codec, forced=True)
if temp_sub.path and temp_sub.path.exists():
shutil.copy2(temp_sub.path, sidecar_path)
finally:
if temp_sub.path and temp_sub.path.exists():
temp_sub.path.unlink(missing_ok=True)
temp_path.unlink(missing_ok=True)
created_paths.append(sidecar_path)
return created_paths
def post_script_ids(self) -> dict[str, Any]:
"""Tagging IDs as they stand right now.
Read at each post-script call, never snapshotted: the search inside the title loop
finds an Episode's TMDB ID, so a snapshot taken before it is always empty.
"""
return {"tmdb": self.tmdb_id, "imdb": self.imdb_id, "tvdb": self.tvdb_id}
@click.command(
short_help="Download, Decrypt, and Mux tracks for titles from a Service.",
cls=Services,
context_settings=dict(**context_settings, default_map=normalize_dl_config(config.dl)),
)
@click.option(
"-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)."
)
@click.option(
"-q",
"--quality",
type=QUALITY_LIST,
default=[],
help="Download Resolution(s), defaults to the best available resolution.",
)
@click.option(
"-v",
"--vcodec",
type=VIDEO_CODEC_LIST,
default=[],
help="Video Codec(s) to download, defaults to any codec.",
)
@click.option(
"-a",
"--acodec",
type=AUDIO_CODEC_LIST,
default=[],
help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.",
)
@click.option(
"-vb",
"--vbitrate",
type=int,
default=None,
help="Video Bitrate to download (in kbps), defaults to highest available.",
)
@click.option(
"-ab",
"--abitrate",
type=int,
default=None,
help="Audio Bitrate to download (in kbps), defaults to highest available.",
)
@click.option(
"-vb-range",
"--vbitrate-range",
type=str,
default=None,
help="Video Bitrate range in kbps (e.g., '6000-7000'). Selects the highest bitrate within the range.",
)
@click.option(
"-ab-range",
"--abitrate-range",
type=str,
default=None,
help="Audio Bitrate range in kbps (e.g., '128-256'). Selects the highest bitrate within the range.",
)
@click.option(
"-r",
"--range",
"range_",
type=MultipleChoice(Video.Range, case_sensitive=False),
default=[Video.Range.SDR],
help="Video Color Range(s) to download, defaults to SDR.",
)
@click.option(
"-c",
"--channels",
type=float,
default=None,
help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.",
)
@click.option(
"-naa",
"--noatmos",
"no_atmos",
is_flag=True,
default=False,
help="Exclude Dolby Atmos audio tracks when selecting audio.",
)
@click.option(
"--split-audio",
"split_audio",
is_flag=True,
default=None,
help="Create separate output files per audio codec instead of merging all audio.",
)
@click.option(
"--merge-video",
"merge_video",
is_flag=True,
default=None,
help="Mux all selected video tracks into a single file instead of one file per track.",
)
@click.option(
"--select-titles",
is_flag=True,
default=False,
help="Interactively select downloads from a list: episodes of a series, or films when a title has more than one.",
)
@click.option(
"-w",
"--wanted",
type=SEASON_RANGE,
default=None,
help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, etc. Music uses track numbers, e.g. `1-5`, `1,3,7`, or `2x3` for disc 2 track 3. Defaults to all.",
)
@click.option(
"-l",
"--lang",
type=LANGUAGE_RANGE,
default="orig",
help="Language(s) wanted for Video and Audio (comma-separated). Use 'orig' to select the original language, e.g. 'orig,en' for both original and English. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
)
@click.option(
"--latest-episode",
is_flag=True,
default=False,
help="Download only the single most recent episode available.",
)
@click.option(
"--latest-episodes",
type=click.IntRange(min=1),
default=None,
help="Download only the N most recent episodes available.",
)
@click.option(
"-vl",
"--v-lang",
type=LANGUAGE_RANGE,
default=[],
help="Language wanted for Video. You would use this if the video language doesn't match the audio. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
)
@click.option(
"-al",
"--a-lang",
type=LANGUAGE_RANGE,
default=[],
help="Language wanted for Audio, overrides -l/--lang for audio tracks. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
)
@click.option(
"--require-audio",
type=LANGUAGE_RANGE,
default=[],
help="Audio languages that must exist. The title fails even with --best-available.",
)
@click.option(
"--require-video",
type=LANGUAGE_RANGE,
default=[],
help="Video languages that must exist. The title fails even with --best-available.",
)
@click.option(
"-sl",
"--s-lang",
type=LANGUAGE_RANGE,
default=["all"],
help="Language wanted for Subtitles. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
)
@click.option(
"--require-subs",
type=LANGUAGE_RANGE,
default=[],
help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.",
)
@click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.")
@click.option(
"-fso",
"--forced-subs-only",
is_flag=True,
default=False,
help="Download forced subtitle tracks and no other subtitle tracks.",
)
@click.option(
"-fsl",
"--forced-s-lang",
type=LANGUAGE_RANGE,
default=[],
help="Languages wanted for forced subtitles, implies -fs. Keeps forced subs only in these languages. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
)
@click.option(
"--exact-lang",
is_flag=True,
default=False,
help="A language tag without a region selects only the variant it means and never falls back to another: -l es does not select es-419. A tag with a region, such as es-419, is always exact.",
)
@click.option(
"--proxy",
type=str,
default=None,
help="Proxy URI to use. If a 2-letter country is provided, it will try to get a proxy from the config.",
)
@click.option(
"--local-proxy",
is_flag=True,
default=False,
help="With --remote, the server sends its service requests out through this machine's own connection (and any VPN on it). Works with no other proxy option.",
)
@click.option(
"--tag",
type=str,
default=None,
help="Set the Group Tag to be used, overriding the config tag and any tag rules.",
)
@click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.")
@click.option(
"-rvb",
"--real-video-bitrate",
is_flag=True,
default=False,
help="Probe actual media size to compute true video bitrates (top renditions per codec/range), "
"overriding the manifest's declared bitrate.",
)
@click.option(
"-rab",
"--real-audio-bitrate",
is_flag=True,
default=False,
help="Probe actual media size to compute true audio bitrates (top renditions per codec/channels/language), "
"overriding the manifest's declared bitrate. Slower than --real-video-bitrate (more renditions).",
)
@click.option(
"--tmdb",
"tmdb_id",
type=int,
default=None,
help="Use this TMDB ID for tagging instead of an automatic search. Add --enrich to also take "
"its title, year and original language.",
)
@click.option(
"--tvdb",
"tvdb_id",
type=int,
default=None,
help="Use this TVDB ID for --tvdb-order and tagging instead of looking the series up "
"automatically. Add --enrich to also take its title, year and original language.",
)
@click.option(
"--tvdb-order",
type=click.Choice(SEASON_TYPES, case_sensitive=False),
default=None,
help="Renumber episodes to a TVDB season order. Services normally use 'official' (aired) order; "
"a series' Streaming Order is 'alternate'. Defaults to the tvdb_order config option.",
)
@click.option(
"--anilist",
"anilist_id",
type=str,
default=None,
help="Use this AniList ID for tagging instead of an automatic search, or a MyAnimeList ID as mal:12345. "
"Add --enrich to also take its title, year and original language. Combines with one of "
"--tmdb, --imdb or --tvdb, which AniList does not know.",
)
@click.option(
"--enrich",
is_flag=True,
default=False,
help=(
"Overwrite the show title, year and original language with the external source's. "
"Requires --tmdb, --imdb, --tvdb, or --anilist."
),
)
@click.option(
"--daily",
is_flag=True,
default=False,
help="Treat the title as daily/date-based content and fill missing air dates from TVDB during --enrich.",
)
@click.option(
"--imdb",
"imdb_id",
type=str,
default=None,
help="Use this IMDB ID (e.g. tt1375666) for tagging instead of an automatic search. Add --enrich "
"to also take its title, year and original language.",
)
@click.option(
"--sub-format",
type=SUBTITLE_CODEC,
default=None,
help="Set Output Subtitle Format, only converting if necessary. Use 'original' to keep source format.",
)
@click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.")
@click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.")
@click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.")
@click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapter markers.")
@click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.")
@click.option(
"--skip-subtitle-errors",
is_flag=True,
default=False,
help="If a subtitle track fails to download, skip it and continue instead of "
"aborting the whole title (video/audio failures stay fatal).",
)
@click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.")
@click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapter markers.")
@click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.")
@click.option(
"-nt",
"--no-attachments",
is_flag=True,
default=False,
help="Do not download or mux attachments (cover art, subtitle fonts, and files the service attaches).",
)
@click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.")
@click.option(
"-ado",
"--audio-description-only",
is_flag=True,
default=False,
help="Download audio description tracks and no standard audio tracks.",
)
@click.option(
"--slow",
type=SLOW_DELAY_RANGE,
default=None,
help="Add a delay between each Title download to act more like a real device. "
"Use --slow for a 60-120s delay, or --slow MIN-MAX (e.g., --slow 20-40) for a custom range. "
"Minimum delay is 20 seconds.",
)
@click.option(
"--list",
"list_",
is_flag=True,
default=False,
help="Skip downloading and list available tracks and what tracks would have been downloaded.",
)
@click.option(
"--list-titles",
is_flag=True,
default=False,
help="Skip downloading, only list available titles that would have been downloaded.",
)
@click.option(
"--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys."
)
@click.option(
"--all-drm",
is_flag=True,
default=False,
help="License each track with both Widevine and PlayReady. Needs a CDM for each DRM system.",
)
@click.option(
"--export",
is_flag=True,
default=False,
help="Export track info and decryption keys to a JSON file in the exports directory.",
)
@click.option(
"--import",
"import_file",
type=str,
default=None,
hidden=True,
help="Internal: path to an export JSON to reconstruct a download from (used by 'unshackle import').",
)
@click.option(
"--cdm-only/--vaults-only",
is_flag=True,
default=None,
help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.",
)
@click.option(
"--cdm",
"cdm_name",
type=str,
default=None,
help="Use this CDM device for the run, overriding the cdm config mapping and, over --remote, the server CDM.",
)
@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.")
@click.option(
"--no-proxy-download",
is_flag=True,
default=False,
help="Bypass proxy for all downloads. Manifest, license, and auth still use proxy.",
)
@click.option(
"--proxy-download",
type=str,
default=None,
help="Proxy for the downloads only, in the same form as --proxy. Manifest, license, and auth use --proxy.",
)
@click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.")
@click.option(
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
)
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
@click.option(
"--sample",
type=SAMPLE_RANGE,
default=None,
help="Download a short sample and write it to a Sample folder. "
"Use --sample for 60 seconds from 2 minutes in, or --sample START-END (e.g., --sample 0-60) for a custom window.",
)
@click.option(
"--include-sample",
is_flag=True,
default=False,
help="Also cut a 60 second sample from each muxed file into a Sample folder next to it.",
)
@click.option(
"--postscript",
"postscript",
type=str,
multiple=True,
metavar="COMMAND",
help=(
"Run COMMAND after each downloaded file, with {filepath}, {title}, {season} and other "
"variables substituted. Repeatable. Replaces the post_scripts config for this run."
),
)
@click.option(
"--no-postscript",
is_flag=True,
default=False,
help="Do not run any post-script for this run, configured or --postscript.",
)
@click.option(
"--workers",
type=int,
default=None,
help="Max workers/threads to download with per-track. Default depends on the downloader.",
)
@click.option(
"--adaptive-workers",
is_flag=True,
default=False,
help=(
"Dynamically scale per-track segment workers up to the --workers cap (or the built-in cap "
"when --workers is unset) based on measured CDN throughput and errors. Off by default."
),
)
@click.option(
"--download-processes",
type=int,
default=1,
help=(
"Split a track's segment downloads across this many processes to beat the single-interpreter "
"throughput cap. Only engages for large segment batches. Default 1 (single process)."
),
)
@click.option(
"--continue-downloads",
is_flag=True,
default=False,
help=(
"Keep completed segment files across runs and resume a previously failed download. "
"One-off enable of the continue_downloads config option."
),
)
@click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.")
@click.option(
"--speed-limit",
"speed_limit",
type=str,
default=None,
metavar="SPEED",
help="Cap total download speed across all downloads combined, e.g. 500k, 5M, 1.5G or plain "
"bytes/sec (5M = 5.0 MB/s; bytes, not bits). "
"'off' disables a limit configured in the dl: config block.",
)
@click.option(
"-o",
"--output",
"output_dir",
type=Path,
default=None,
help="Override the output directory for this download, instead of the one in config.",
)
@click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.")
@click.option(
"--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching."
)
@click.option(
"--worst",
is_flag=True,
default=False,
help="Select the lowest bitrate track within the specified quality. Requires -q/--quality.",
)
@click.option(
"--best-available",
"--warn-only",
"best_available",
is_flag=True,
default=False,
help="Warn instead of failing when a requested resolution, range, or language is missing, and continue with what is available.",
)
@click.option(
"--generate-dv",
is_flag=True,
default=False,
help="With -r HYBRID, make Dolby Vision from the HDR10+ metadata when a title has no Dolby Vision track.",
)
@click.option(
"--remote/--local",
default=False,
is_eager=True,
help="Use a remote unshackle server instead of local service code. --local overrides a remote: true config.",
)
@click.option(
"--server",
type=str,
default=None,
is_eager=True,
help="Name of the remote server from remote_services config (if multiple configured).",
)
@click.pass_context
def cli(ctx: click.Context, **kwargs: Any) -> dl:
return dl(ctx, **kwargs)
DRM_LOCKS: dict[str, Lock] = {}
DRM_LOCKS_GUARD = Lock()
VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer")
EXPORT_LOCK = Lock()
LICENSE_KEY_CACHE: dict[UUID, str] = {}
ALL_DRM_LOCK = Lock()
def __init__(
self,
ctx: click.Context,
no_proxy: bool,
profile: Optional[str] = None,
proxy: Optional[str] = None,
repack: bool = False,
tag: Optional[str] = None,
tmdb_id: Optional[int] = None,
imdb_id: Optional[str] = None,
tvdb_id: Optional[int] = None,
tvdb_order: Optional[str] = None,
anilist_id: Optional[Union[int, str]] = None,
enrich: bool = False,
daily: bool = False,
output_dir: Optional[Path] = None,
proxy_providers: Optional[list[Any]] = None,
*_: Any,
**__: Any,
):
if not ctx.invoked_subcommand:
raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.")
self.log = logging.getLogger("download")
self.completed_files: list[Path] = []
# Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can
# report which weren't available without parsing the console output. See SkippedSubtitle.
self.skipped_subtitles: list[SkippedSubtitle] = []
# result() catches a download-worker failure, reports it, and returns rather than
# re-raising (so the CLI still exits cleanly). Flag it so an embedding caller (the API
# worker) can tell the title did not complete instead of reading it as a success.
self.download_failed: bool = False
if not config.output_template:
raise click.ClickException(
"No 'output_template' configured in your unshackle.yaml.\n"
"Please add an 'output_template' section with movies, series, and songs/music templates.\n"
"See unshackle-example.yaml for examples."
)
self.service = Services.get_tag(ctx.invoked_subcommand)
self.vault_service = Services.get_vault_tag(self.service)
apply_service_dl_overrides(ctx, (ci_get(config.services, self.service) or {}).get("dl") or {}, self.log)
# Refresh locals Click bound before the overrides ran.
no_proxy = ctx.params.get("no_proxy", no_proxy)
profile = ctx.params.get("profile", profile)
proxy = ctx.params.get("proxy", proxy)
repack = ctx.params.get("repack", repack)
tag = ctx.params.get("tag", tag)
tmdb_id = ctx.params.get("tmdb_id", tmdb_id)
imdb_id = ctx.params.get("imdb_id", imdb_id)
tvdb_id = ctx.params.get("tvdb_id", tvdb_id)
tvdb_order = ctx.params.get("tvdb_order", tvdb_order)
anilist_id = ctx.params.get("anilist_id", anilist_id)
enrich = ctx.params.get("enrich", enrich)
daily = ctx.params.get("daily", daily)
output_dir = ctx.params.get("output_dir", output_dir)
self.profile = profile
self.proxy_requested = bool(proxy)
validate_metadata_ids(tmdb_id, imdb_id, tvdb_id, anilist_id)
self.tmdb_id = tmdb_id
self.imdb_id = imdb_id
self.tvdb_id = tvdb_id
self.anilist_id = anilist_id
self.tvdb_order = (tvdb_order or config.tvdb_order or "").lower()
if self.tvdb_order and self.tvdb_order not in SEASON_TYPES:
raise click.UsageError(f"tvdb_order must be one of {', '.join(SEASON_TYPES)}, not {self.tvdb_order!r}")
self.enrich = enrich
self.daily = daily
self.output_dir = output_dir
self.service_anime = False
self.service_daily = False
if self.enrich and not (self.tmdb_id or self.imdb_id or self.tvdb_id or self.anilist_id):
raise click.UsageError(
"--enrich requires --tmdb, --imdb, --tvdb, or --anilist to provide a metadata source."
)
if self.daily and not self.enrich:
self.log.warning("--daily fills air dates from TVDB only with --enrich. A service can set them itself.")
# Initialize debug logger with service name if debug logging is enabled
if config.debug or logging.root.level == logging.DEBUG:
from collections import defaultdict
from datetime import datetime
debug_log_path = config.directories.logs / config.filenames.debug_log.format_map(
defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S"))
)
init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys)
self.debug_logger = get_debug_logger()
if self.debug_logger:
self.debug_logger.log(
level="INFO",
operation="download_init",
message=f"Download command initialized for service {self.service}",
service=self.service,
context={
"profile": profile,
"proxy": proxy,
"tag": tag,
"tmdb_id": self.tmdb_id,
"imdb_id": self.imdb_id,
"anilist_id": anilist_id,
"enrich": enrich,
"cli_params": {
k: v
for k, v in ctx.params.items()
if k
not in [
"profile",
"proxy",
"tag",
"tmdb_id",
"imdb_id",
"anilist_id",
"enrich",
]
},
},
)
# Log binary versions for diagnostics
binary_versions = {}
for name, binary in [
("shaka_packager", binaries.ShakaPackager),
("mp4decrypt", binaries.Mp4decrypt),
("mkvmerge", binaries.MKVToolNix),
("ffmpeg", binaries.FFMPEG),
("ffprobe", binaries.FFProbe),
]:
if binary:
version = None
try:
if name == "shaka_packager":
r = subprocess.run(
[str(binary), "--version"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=5,
)
version = (r.stdout or r.stderr or "").strip()
elif name in ("ffmpeg", "ffprobe"):
r = subprocess.run(
[str(binary), "-version"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=5,
)
version = (r.stdout or "").split("\n")[0].strip()
elif name == "mkvmerge":
r = subprocess.run(
[str(binary), "--version"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=5,
)
version = (r.stdout or "").strip()
elif name == "mp4decrypt":
r = subprocess.run(
[str(binary)],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=5,
)
output = (r.stdout or "") + (r.stderr or "")
lines = [line.strip() for line in output.split("\n") if line.strip()]
version = " | ".join(lines[:2]) if lines else None
except Exception:
version = "<error getting version>"
binary_versions[name] = {"path": str(binary), "version": version}
else:
binary_versions[name] = None
self.debug_logger.log(
level="DEBUG",
operation="binary_versions",
message="Binary tool versions",
context=binary_versions,
)
else:
self.debug_logger = None
if self.profile:
self.log.info(f"Using profile: '{self.profile}'")
self.is_remote = bool(ctx.params.get("remote"))
with console.status("Loading Service Config...", spinner="dots"):
self.service_config = {}
if not self.is_remote:
try:
service_config_path = Services.get_path(self.service) / config.filenames.config
if service_config_path.exists():
self.service_config = load_yaml(service_config_path)
self.log.info("Service Config loaded")
# log key names only: the full config carries service certificates,
# device fingerprints and endpoints that bloat the log and may be sensitive
log_event(
"load_service_config",
level="DEBUG",
service=self.service,
context={
"config_path": str(service_config_path),
"config_keys": sorted(self.service_config or []),
},
)
except KeyError:
pass
merge_dict(ci_get(config.services, self.service), self.service_config)
service_config = ci_get(config.services, self.service, {})
if service_config:
reserved_keys = {
"profiles",
"api_key",
"certificate",
"api_endpoint",
"region",
"device",
"endpoints",
"client",
"dl",
}
for config_key, override_value in service_config.items():
if config_key in reserved_keys or not isinstance(override_value, dict):
continue
if hasattr(config, config_key):
current_config = getattr(config, config_key, {})
if isinstance(current_config, dict):
merged_config = deepcopy(current_config)
merge_dict(override_value, merged_config)
setattr(config, config_key, merged_config)
self.log.debug(
f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}"
)
self.cdm_override = ctx.params.get("cdm_name")
cdm_only = ctx.params.get("cdm_only")
self.vault_cache_tally: Optional[tuple[set[UUID], int]] = None
if cdm_only:
self.vaults = Vaults(self.vault_service)
verify.DECRYPT_HOOK = self.decrypt_verified
self.log.info("CDM-only mode: Skipping vault loading")
log_event(
"vault_loading_skipped",
level="INFO",
service=self.service,
context={"reason": "cdm_only flag set"},
)
else:
with console.status("Loading Key Vaults...", spinner="dots"):
self.vaults = Vaults(self.vault_service)
verify.DECRYPT_HOOK = self.decrypt_verified
total_vaults = len(config.key_vaults)
failed_vaults = []
for vault in config.key_vaults:
vault_type = vault["type"]
vault_name = vault.get("name", vault_type)
vault_copy = vault.copy()
del vault_copy["type"]
if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower():
if "token" not in vault_copy or not vault_copy["token"]:
if config.decrypt_labs_api_key:
vault_copy["token"] = config.decrypt_labs_api_key
else:
self.log.warning(
f"No token provided for DecryptLabs vault '{vault_name}' and no global "
"decrypt_labs_api_key configured"
)
if vault_type.lower() == "sqlite":
try:
self.vaults.load_critical(vault_type, **vault_copy)
self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})")
except Exception as e:
self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}")
raise
else:
# Other vaults (MySQL, HTTP, API) - soft fail
if not self.vaults.load(vault_type, **vault_copy):
failed_vaults.append(vault_name)
self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})")
else:
self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})")
loaded_count = len(self.vaults)
if failed_vaults:
self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}")
self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults")
# Debug: Show detailed vault status
if loaded_count > 0:
vault_names = [vault.name for vault in self.vaults]
self.log.debug(f"Active vaults: {', '.join(vault_names)}")
else:
self.log.debug("No vaults are currently active")
with console.status("Loading DRM CDM...", spinner="dots"):
try:
self.cdm = self.get_cdm(self.service, self.profile)
except ValueError as e:
self.log.error(f"Failed to load CDM, {e}")
if self.debug_logger:
self.debug_logger.log_error("load_cdm", e, service=self.service)
sys.exit(1)
if self.cdm:
cdm_info = {}
if isinstance(self.cdm, DecryptLabsRemoteCDM):
drm_type = "PlayReady" if self.cdm.is_playready else "Widevine"
self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})")
cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level}
elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]:
self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})")
cdm_info = {
"type": "Widevine",
"system_id": self.cdm.system_id,
"security_level": self.cdm.security_level,
"device_type": self.cdm.device_type.name,
}
else:
# Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None)
is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name")
if is_remote:
cdm_name = self.cdm.device_name
self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})")
else:
cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown"
self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})")
cdm_info = {
"type": "PlayReady",
"certificate": cdm_name,
"security_level": self.cdm.security_level,
}
if cdm_info:
log_event("load_cdm", level="INFO", service=self.service, context={"cdm": cdm_info})
if ctx.params.get("local_proxy"):
require_alone_local_proxy(ctx)
proxy, no_proxy = ctx.params.get("proxy"), bool(ctx.params.get("no_proxy"))
# A server on this machine can reach a local proxy, so only a server elsewhere is guarded
self.remote_needs_public_proxy = self.is_remote and not is_loopback(server_url(ctx.params.get("server")))
self.proxy_providers = []
if no_proxy:
ctx.params["proxy"] = None
else:
if self.remote_needs_public_proxy and proxy and proxy.lower().startswith("gluetun:"):
raise click.UsageError("Gluetun runs on your machine, so --remote cannot use it.")
if proxy_providers is not None:
self.proxy_providers = list(proxy_providers)
else:
with console.status("Loading Proxy Providers...", spinner="dots"):
self.proxy_providers = load_proxy_providers()
ctx.params["proxy_arg"] = proxy
if proxy and proxy.startswith("controld://"):
proxy = ctx.params["proxy"] = resolve_proxy(proxy, [])
if proxy:
try:
requested_provider, query = split_proxy_query(proxy, self.proxy_providers)
except ValueError as e:
self.log.error(str(e))
sys.exit(1)
# Match region codes (us, ca, uk1, us:ny, yul) or provider:region (nordvpn:ca, protonvpn:us:ny).
if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", query, re.IGNORECASE):
# Kept apart from ctx.params["proxy"], which becomes the URI: a service proxy_map keys on it.
proxy_query = query.lower()
status_msg = (
f"Connecting to VPN ({proxy_query})..."
if requested_provider == "gluetun"
else f"Getting a Proxy to {proxy_query}..."
)
with console.status(status_msg, spinner="dots"):
try:
proxy_provider, proxy_uri = pick_proxy(
self.proxy_providers, proxy_query, requested_provider
)
except ValueError as e:
self.log.error(str(e))
sys.exit(1)
ctx.params["proxy"] = proxy_uri
self.log.info(describe_proxy(proxy_provider, proxy_uri, proxy_query))
ctx.params["proxy_query"] = proxy_query
ctx.params["proxy_provider"] = requested_provider
else:
self.log.info(describe_explicit_proxy(ctx.params["proxy"]))
ctx.params["proxy_query"] = None
ctx.params["proxy_provider"] = None
ctx.obj = ContextData(
config=self.service_config,
cdm=self.cdm,
proxy_providers=self.proxy_providers,
profile=self.profile,
cdm_entry=self.cdm_from_service_entry(),
)
if repack:
config.repack = True
if tag:
config.tag = tag
config.tag_rules = []
# needs to be added this way instead of @cli.result_callback to be
# able to keep `self` as the first positional
self.cli._result_callback = self.result
@with_task_temp
def result(
self,
service: Service,
quality: list[int],
vcodec: list[Video.Codec],
acodec: list[Audio.Codec],
vbitrate: int,
abitrate: int,
vbitrate_range: Optional[str],
abitrate_range: Optional[str],
range_: list[Video.Range],
channels: float,
no_atmos: bool,
select_titles: bool,
wanted: list[str],
latest_episode: bool,
latest_episodes: Optional[int],
lang: list[str],
v_lang: list[str],
a_lang: list[str],
s_lang: list[str],
require_audio: list[str],
require_video: list[str],
require_subs: list[str],
forced_subs: bool,
forced_subs_only: bool,
forced_s_lang: list[str],
exact_lang: bool,
sub_format: Optional[Union[Subtitle.Codec, str]],
video_only: bool,
audio_only: bool,
subs_only: bool,
chapters_only: bool,
no_subs: bool,
skip_subtitle_errors: bool,
no_audio: bool,
no_chapters: bool,
no_video: bool,
no_attachments: bool,
audio_description: bool,
audio_description_only: bool,
slow: Optional[tuple[int, int]],
list_: bool,
list_titles: bool,
skip_dl: bool,
export: bool,
cdm_only: Optional[bool],
no_proxy: bool,
no_proxy_download: bool,
proxy_download: Optional[str],
no_folder: bool,
no_source: bool,
no_mux: bool,
sample: Optional[tuple[int, int]],
include_sample: bool,
workers: Optional[int],
adaptive_workers: bool,
download_processes: int,
continue_downloads: bool,
downloads: int,
worst: bool,
best_available: bool,
speed_limit: Optional[str] = None,
split_audio: Optional[bool] = None,
merge_video: Optional[bool] = None,
real_video_bitrate: bool = False,
real_audio_bitrate: bool = False,
progress_sink: Optional[Callable[[dict[str, Any]], None]] = None,
postscript: Sequence[str] = (),
no_postscript: bool = False,
all_drm: bool = False,
generate_dv: bool = False,
*_: Any,
**__: Any,
) -> None:
if all_drm:
DOWNLOAD_ALL_DRM.set()
else:
DOWNLOAD_ALL_DRM.clear()
self.all_drm_attempted: defaultdict[type, set[UUID]] = defaultdict(set)
self.all_drm_warned: set[str] = set()
if no_postscript:
postscript = NO_POST_SCRIPTS
if continue_downloads:
config.continue_downloads = True
config.sample = sample
if sample and include_sample:
raise click.ClickException("--sample and --include-sample cannot be used together.")
sample_dir = SAMPLE_DIR if sample else ""
self.tmdb_searched = False
self.search_source = None
self.service_anime = bool(getattr(service, "ANIME", False))
self.service_daily = bool(getattr(service, "DAILY", False))
self.server_cdm = getattr(service, "_server_cdm", False)
from unshackle.core.import_service import ImportService
# a remote session sends --cdm to the server, which can hand it the tracks above its cap
if self.cdm_override and isinstance(service, ImportService):
self.log.warning("--cdm is ignored: an import uses the keys in the export")
self._remote_service = service if hasattr(service, "_server_cdm") else None
start_time = time.time()
lang, lang_excl = partition_exclusions(lang)
v_lang, v_lang_excl = partition_exclusions(v_lang)
a_lang, a_lang_excl = partition_exclusions(a_lang)
s_lang, s_lang_excl = partition_exclusions(s_lang)
forced_s_lang, fsl_excl = partition_exclusions(forced_s_lang)
for flag, excludes in (
("-l/--lang", lang_excl),
("-vl/--v-lang", v_lang_excl),
("-al/--a-lang", a_lang_excl),
("-sl/--s-lang", s_lang_excl),
("-fsl/--forced-s-lang", fsl_excl),
):
if any(token.lower() == "all" for token in excludes):
self.log.error(f"{flag}: 'all' cannot be excluded. Name the languages to exclude instead.")
sys.exit(1)
# exclusion-only use falls back to the flag's own default; -vl/-al have none, they cascade to -l
if lang_excl and not lang:
lang = ["orig"]
if s_lang_excl and not s_lang:
s_lang = ["all"]
# an override that names languages replaces -l entirely, so it must not inherit its exclusions
video_excl = list(dict.fromkeys(v_lang_excl + ([] if v_lang else lang_excl)))
audio_excl = list(dict.fromkeys(a_lang_excl + ([] if a_lang else lang_excl)))
for flag, tokens, excludes in (
("--require-audio", require_audio, audio_excl),
("--require-video", require_video, video_excl),
("--require-subs", require_subs, s_lang_excl),
):
required, own_excl = partition_exclusions(tokens)
clash = own_excl + [t for t in required if any(t.lower() == e.lower() for e in excludes)]
if clash:
self.log.error(f"{flag}: {clash[0]} cannot be required and excluded at the same time.")
sys.exit(1)
ctx = getattr(service, "ctx", None)
parent_params = ctx.parent.params if ctx and ctx.parent else None
if parent_params is not None:
# services read the raw ctx params, they must never see a '-xx' token
for name, includes, excludes in (
("lang", lang, lang_excl),
("v_lang", v_lang, v_lang_excl),
("a_lang", a_lang, a_lang_excl),
("s_lang", s_lang, s_lang_excl),
("forced_s_lang", forced_s_lang, fsl_excl),
):
if excludes and name in parent_params:
parent_params[name] = includes
conflict = kind_only_conflict(
{
"audio_description_only": audio_description_only,
"forced_subs_only": forced_subs_only,
"no_audio": no_audio,
"no_subs": no_subs,
"video_only": video_only,
"audio_only": audio_only,
"subs_only": subs_only,
"chapters_only": chapters_only,
}
)
if conflict:
first, second = (f"--{name.replace('_', '-')}" for name in conflict)
self.log.error(f"{first} cannot be used with {second}: {second} drops the tracks that {first} selects.")
sys.exit(1)
if audio_description_only:
audio_description = True
if forced_s_lang or fsl_excl or forced_subs_only:
# services read forced_subs from raw ctx params, so the implication must be pushed there too
forced_subs = True
if "all" in forced_s_lang:
forced_s_lang = []
if parent_params is not None:
parent_params["forced_subs"] = True
if skip_dl:
DOWNLOAD_LICENCE_ONLY.set()
try:
speed_limit_bps = parse_speed_limit(speed_limit)
except ValueError as e:
self.log.error(str(e))
sys.exit(1)
set_speed_limit(speed_limit_bps)
if speed_limit_bps:
self.log.info(f"Speed limit: {format_speed(speed_limit_bps)}")
if no_proxy or no_proxy_download:
proxy_download = None
elif proxy_download and (
proxy_download.startswith("controld://")
or re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", proxy_download, re.I)
):
# same shapes --proxy resolves against providers (two prefixes for gluetun:nordvpn:ca); else an explicit URI
try:
proxy_download = resolve_proxy(proxy_download.lower(), self.proxy_providers)
except ValueError as e:
self.log.error(f"--proxy-download: {e}")
sys.exit(1)
if export:
config.directories.exports.mkdir(parents=True, exist_ok=True)
# a working name until the first write, which renames it after what it holds
export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json"
self.export_service = service
self.export_namer = ExportNamer(self.service)
else:
export_path = None
vbitrate_min, vbitrate_max = None, None
if vbitrate_range:
if vbitrate and vbitrate_range:
self.log.error("Cannot use both --vbitrate and --vbitrate-range at the same time.")
sys.exit(1)
try:
parts = vbitrate_range.split("-")
if len(parts) != 2:
raise ValueError
vbitrate_min, vbitrate_max = int(parts[0]), int(parts[1])
if vbitrate_min > vbitrate_max:
vbitrate_min, vbitrate_max = vbitrate_max, vbitrate_min
except (ValueError, IndexError):
self.log.error("Invalid --vbitrate-range format. Use 'MIN-MAX' (e.g., '6000-7000').")
sys.exit(1)
abitrate_min, abitrate_max = None, None
if abitrate_range:
if abitrate and abitrate_range:
self.log.error("Cannot use both --abitrate and --abitrate-range at the same time.")
sys.exit(1)
try:
parts = abitrate_range.split("-")
if len(parts) != 2:
raise ValueError
abitrate_min, abitrate_max = int(parts[0]), int(parts[1])
if abitrate_min > abitrate_max:
abitrate_min, abitrate_max = abitrate_max, abitrate_min
except (ValueError, IndexError):
self.log.error("Invalid --abitrate-range format. Use 'MIN-MAX' (e.g., '128-256').")
sys.exit(1)
if not acodec:
acodec = []
elif isinstance(acodec, Audio.Codec):
acodec = [acodec]
elif isinstance(acodec, str) or (
isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec)
):
acodec = AUDIO_CODEC_LIST.convert(acodec)
if worst and not quality:
self.log.error("--worst requires -q/--quality to be specified")
sys.exit(1)
if select_titles and wanted:
self.log.error("--select-titles and -w/--wanted cannot be used together")
sys.exit(1)
if any(r == Video.Range.HYBRID for r in range_):
from unshackle.core.binaries import DoviTool
if not DoviTool:
self.log.error("Unable to run hybrid mode: dovi_tool not detected")
self.log.error("Please install dovi_tool from https://github.com/quietvoid/dovi_tool")
sys.exit(1)
if generate_dv:
from unshackle.core.binaries import HDR10PlusTool
if not HDR10PlusTool:
self.log.error("Unable to run --generate-dv: hdr10plus_tool not detected")
self.log.error("Please install hdr10plus_tool from https://github.com/quietvoid/hdr10plus_tool")
sys.exit(1)
if cdm_only is None:
vaults_only = None
else:
vaults_only = not cdm_only
config.decryption = resolve_decryption(
config.decryption_map, config.decryption, service.__class__.__name__.upper()
)
log_event(
"drm_mode_config",
level="DEBUG",
service=self.service,
context={
"cdm_only": cdm_only,
"vaults_only": vaults_only,
"mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"),
"decryption": config.decryption,
},
)
with console.status(
"Preparing Remote Service Session..." if self.is_remote else "Preparing Service Session...",
spinner="dots",
):
try:
cookies = self.get_cookie_jar(self.service, self.profile)
credential = self.get_credentials(self.service, self.profile)
service.authenticate(cookies, credential)
if cookies or credential:
self.log.info("Authenticated with Service")
log_event(
"authenticate",
level="INFO",
service=self.service,
context={
"has_cookies": bool(cookies),
"has_credentials": bool(credential),
"profile": self.profile,
},
)
except Exception as e:
if self.debug_logger:
self.debug_logger.log_error(
"authenticate", e, service=self.service, context={"profile": self.profile}
)
raise
with console.status(
"Fetching Remote Title Metadata..." if self.is_remote else "Fetching Title Metadata...", spinner="dots"
):
try:
titles = service.get_titles_cached()
if not titles:
self.log.error("No titles returned, nothing to download...")
log_event(
"get_titles",
level="ERROR",
service=self.service,
message="No titles returned from service",
success=False,
)
sys.exit(1)
except Exception as e:
if self.debug_logger:
self.debug_logger.log_error("get_titles", e, service=self.service)
raise
if self.debug_logger:
titles_info = {
"type": titles.__class__.__name__,
"count": len(titles) if hasattr(titles, "__len__") else 1,
"title": str(titles),
}
if hasattr(titles, "seasons"):
titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0
self.debug_logger.log(
level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info}
)
title_cacher = service.title_cache if hasattr(service, "title_cache") else None
cache_title_id = None
if hasattr(service, "title"):
cache_title_id = service.title
elif hasattr(service, "title_id"):
cache_title_id = service.title_id
cache_region = service.current_region if hasattr(service, "current_region") else None
cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None
enrich_lang: Optional[Language] = None
enrich_title: Optional[str] = None
enrich_year: Optional[int] = None
if self.enrich:
sample_title = titles[0] if hasattr(titles, "__getitem__") else titles
kind = "tv" if isinstance(sample_title, Episode) else "movie"
enrich_result = providers.resolve_by_ids(
self.tmdb_id,
self.imdb_id,
self.tvdb_id,
self.anilist_id,
kind=kind,
title_cacher=title_cacher,
cache_title_id=cache_title_id,
cache_region=cache_region,
cache_account_hash=cache_account_hash,
anime=self.anime_hint(sample_title),
)
if enrich_result:
enrich_title = enrich_result.title
enrich_year = enrich_result.year
enrich_lang = parse_language(enrich_result.original_language)
if enrich_lang:
self.log.info(f"Original language from metadata: {enrich_lang}")
if not (enrich_title or enrich_year or enrich_lang):
self.log.warning("--enrich found no metadata. Is the provider's API key configured?")
elif missing := [
name
for name, value in (("title", enrich_title), ("year", enrich_year), ("language", enrich_lang))
if not value
]:
self.log.warning(f"--enrich source gave no {', '.join(missing)}. Those fields are unchanged.")
if isinstance(titles, (Series, Movies)):
for t in titles:
apply_enrich(t, enrich_title, enrich_year, enrich_lang)
if isinstance(titles, Series):
enrich_tvdb_id = self.tvdb_id or (enrich_result.external_ids.tvdb_id if enrich_result else None)
self.fill_absolute_numbers(titles, enrich_tvdb_id)
if any(self.daily_hint(t) for t in titles):
self.fill_air_dates(titles, enrich_tvdb_id)
if self.tvdb_order and isinstance(titles, Series):
titles = self.apply_tvdb_order(titles, title_cacher, cache_title_id, cache_region, cache_account_hash)
console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2)))
console.print(Padding(titles.tree(verbose=list_titles), (0, 5)))
if list_titles:
return
if export_path and isinstance(titles, Series):
# before --select-titles and -w drop titles, so a whole season can be told from part of one
self.export_namer.seasons = season_episodes(titles)
if select_titles and isinstance(titles, Movies) and len(titles) > 1:
console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2)))
selection_start = time.time()
selected_idx = select_multiple(
[str(movie) for movie in titles],
minimal_count=1,
page_size=8,
return_indices=True,
)
if not selected_idx:
console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5)))
return
start_time += time.time() - selection_start
keep = set(selected_idx)
for i in range(len(titles) - 1, -1, -1):
if i not in keep:
del titles[i]
console.print(Padding(f"[text]Total selected: {len(titles)}[/]", (0, 5)))
# Enables manual selection for Series when --select-titles is set
if select_titles and isinstance(titles, Series):
console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2)))
selection_titles = []
dependencies = {}
original_indices = {}
current_season = None
current_season_header_idx = -1
unique_seasons = {t.season for t in titles}
multiple_seasons = len(unique_seasons) > 1
# Build selection options
for i, t in enumerate(titles):
# Insert season header only if multiple seasons exist
if multiple_seasons and t.season != current_season:
current_season = t.season
header_text = f"Season {t.season}"
selection_titles.append(header_text)
current_season_header_idx = len(selection_titles) - 1
dependencies[current_season_header_idx] = []
display_name = ((t.name[:30].rstrip() + "...") if len(t.name) > 30 else t.name) if t.name else None
prefix = " " if multiple_seasons else ""
part_label = f".{t.part}" if t.part is not None else ""
option_text = f"{prefix}{t.number}{part_label}" + (f". {display_name}" if t.name else "")
selection_titles.append(option_text)
current_ui_idx = len(selection_titles) - 1
original_indices[current_ui_idx] = i
if current_season_header_idx != -1:
dependencies[current_season_header_idx].append(current_ui_idx)
selection_start = time.time()
selected_ui_idx = select_multiple(
selection_titles,
minimal_count=1,
page_size=8,
return_indices=True,
dependencies=dependencies,
collapse_on_start=multiple_seasons,
)
if not selected_ui_idx:
console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5)))
return
selection_end = time.time()
start_time += selection_end - selection_start
selected_idx = []
for idx in selected_ui_idx:
if idx in original_indices:
selected_idx.append(original_indices[idx])
# Ensure indices are unique and ordered
selected_idx = sorted(set(selected_idx))
keep = set(selected_idx)
for i in range(len(titles) - 1, -1, -1):
if i not in keep:
del titles[i]
if titles:
count = len(titles)
console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5)))
latest_keys = latest_episode_keys(titles, latest_episode, latest_episodes)
if latest_keys:
picked = ", ".join(f"S{season:02}E{number:02}" for season, number in latest_keys)
self.log.info(f"Latest episode mode: Selecting {picked}")
# replaces -w; a base SxE key selects every part of a split episode
wanted = [f"{season}x{number}" for season, number in latest_keys]
def select_best_audio(
audio: list[Audio],
languages: list[str],
acodec: list[Audio.Codec],
audio_description: bool,
exact_lang: bool,
) -> list[Audio]:
"""Pick the best audio per (language, codec) group from a sort_audio-ordered list.
"Best" is the first track of each order-preserving candidate filter, so the
sort_audio ranking (descriptive-last, Atmos, codec_priority, bitrate) decides the
winner. With audio_description both the best standard and best descriptive track per
group are kept. "best"/"all" tokens select every language present in sorted order.
"""
select_all = any(tok in ("best", "all") for tok in languages)
if select_all:
target_langs: list[Any] = list(dict.fromkeys(t.language for t in audio))
else:
target_langs = languages
codecs: list[Optional[Audio.Codec]] = list(acodec) if len(acodec) > 1 else [None]
selected: list[Audio] = []
for language in target_langs:
for codec in codecs:
if select_all:
candidates = [
t for t in audio if t.language == language and (codec is None or t.codec == codec)
]
else:
# match against every codec at once, so the variant a tag means does not depend on codec
tags = matching_languages(language, [t.language for t in audio], exact_lang)
candidates = [
t for t in audio if str(t.language) in tags and (codec is None or t.codec == codec)
]
if not candidates:
continue
if audio_description:
standards = [t for t in candidates if not t.descriptive]
if standards:
selected.append(standards[0])
descs = [t for t in candidates if t.descriptive]
if descs:
selected.append(descs[0])
else:
selected.append(candidates[0])
# Built via per-tag by_language calls, so overlapping tags can append
# the same track across iterations; dedupe by id.
seen_ids: set[str] = set()
return [t for t in selected if not (t.id in seen_ids or seen_ids.add(t.id))]
base_selection = (v_lang, a_lang, s_lang, range_)
post_script_pending: dict[Any, int] = {}
for candidate in titles:
if title_wanted(candidate, wanted):
key = post_script_group(candidate)
post_script_pending[key] = post_script_pending.get(key, 0) + 1
post_script_last: dict[Any, dict[Path, dict[str, str]]] = {}
post_script_folders: list[Path] = []
post_script_sample: dict[str, str] = {}
wanted_total = sum(post_script_pending.values())
def settle_post_script_group(title: Any) -> None:
"""Count one title of its group as done; send the season event after the last one."""
group_key = post_script_group(title)
post_script_pending[group_key] = post_script_pending.get(group_key, 1) - 1
if post_script_pending[group_key] <= 0:
for folder, context in post_script_last.get(group_key, {}).items():
dispatch("success", "season", season_context(context, folder), postscript)
def skip_title(title: Any, reason: str) -> None:
"""Skip a title that has no track of the kind an only flag selects.
A run with one title stops instead, because it has nothing else to download.
"""
if wanted_total <= 1:
self.log.error(reason)
sys.exit(1)
self.log.warning(f"{reason} Skipping {title}")
if not no_mux:
settle_post_script_group(title)
for i, title in enumerate(titles):
v_lang, a_lang, s_lang, range_ = base_selection
if not title_wanted(title, wanted):
continue
if progress_sink:
if isinstance(title, Episode):
progress_sink(
{
"title": title.title,
"current_title": f"S{title.season or 0:02}E{title.number or 0:02}"
+ (f".{title.part}" if title.part is not None else ""),
}
)
else:
progress_sink({"title": getattr(title, "name", None) or str(title)})
title_rule = f"Track {title.track:02}: {title.name}" if isinstance(title, Song) else str(title)
console.print(Padding(Rule(f"[rule.text]{title_rule}"), (1, 2)))
if isinstance(title, Episode) and not self.tmdb_searched:
kind = "tv"
tmdb_title: Optional[str] = None
has_ids = bool(self.tmdb_id or self.imdb_id or self.tvdb_id or self.anilist_id)
result = providers.resolve_by_ids(
self.tmdb_id,
self.imdb_id,
self.tvdb_id,
self.anilist_id,
title=title.title,
year=title.year,
kind=kind,
title_cacher=title_cacher,
cache_title_id=cache_title_id,
cache_region=cache_region,
cache_account_hash=cache_account_hash,
anime=self.anime_hint(title),
)
if result:
if has_ids:
tmdb_title = result.title
if not self.tmdb_id:
self.tmdb_id = result.external_ids.tmdb_id
elif result.title and providers.fuzzy_match(result.title, title.title):
self.tmdb_id = result.external_ids.tmdb_id
tmdb_title = result.title
self.search_source = result.source
if list_ or list_titles:
if self.tmdb_id:
console.print(
Padding(
f"Search -> {tmdb_title or '?'} [bright_black](ID {self.tmdb_id})",
(0, 5),
)
)
else:
console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5)))
self.tmdb_searched = True
if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id:
movie_result = providers.resolve_by_ids(
None,
self.imdb_id,
self.tvdb_id,
self.anilist_id,
title=title.name,
year=title.year,
kind="movie",
title_cacher=title_cacher,
cache_title_id=cache_title_id,
cache_region=cache_region,
cache_account_hash=cache_account_hash,
anime=self.anime_hint(title),
)
if movie_result and movie_result.external_ids.tmdb_id:
console.print(
Padding(
f"Search -> {movie_result.title or '?'} "
f"[bright_black](ID {movie_result.external_ids.tmdb_id})",
(0, 5),
)
)
else:
console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5)))
if self.tmdb_id and getattr(self, "search_source", None) not in ("simkl", "imdb"):
kind = "tv" if isinstance(title, Episode) else "movie"
providers.fetch_external_ids(
self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash
)
if slow and i != 0:
delay = random.randint(slow[0], slow[1])
spinner = Spinner("dots", text=f"Delaying by {delay} seconds...")
with SyncLive(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True):
for remaining in range(delay, 0, -1):
spinner.update(text=f"Delaying by {remaining} seconds...")
time.sleep(1)
with console.status("Subscribing to events...", spinner="dots"):
events.reset()
events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded)
events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded)
events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted)
events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked)
events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex)
tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..."
tracks_spinner = Spinner("dots", text=tracks_label)
with SyncLive(Padding(tracks_spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True):
try:
load_tracks(
service,
title,
name=enrich_title,
year=enrich_year,
language=enrich_lang,
workers=workers,
on_probe=lambda: tracks_spinner.update(text="Probing HDR10 tracks for HDR10+..."),
)
except Exception as e:
if self.debug_logger:
self.debug_logger.log_error(
"get_tracks", e, service=self.service, context={"title": str(title)}
)
raise
if self.debug_logger:
tracks_info = {
"title": str(title),
"video_tracks": len(title.tracks.videos),
"audio_tracks": len(title.tracks.audio),
"subtitle_tracks": len(title.tracks.subtitles),
"has_chapters": bool(title.tracks.chapters),
"videos": [
{
"codec": str(v.codec),
"resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown",
"bitrate": v.bitrate,
"range": str(v.range),
"language": str(v.language) if v.language else None,
"drm": [str(type(d).__name__) for d in v.drm] if v.drm else [],
}
for v in title.tracks.videos
],
"audio": [
{
"codec": str(a.codec),
"bitrate": a.bitrate,
"channels": a.channels,
"language": str(a.language) if a.language else None,
"descriptive": a.descriptive,
"drm": [str(type(d).__name__) for d in a.drm] if a.drm else [],
}
for a in title.tracks.audio
],
"subtitles": [
{
"codec": str(s.codec),
"language": str(s.language) if s.language else None,
"forced": s.forced,
"sdh": s.sdh,
}
for s in title.tracks.subtitles
],
}
self.debug_logger.log(
level="INFO", operation="get_tracks", service=self.service, context=tracks_info
)
# a plain track covers an SDH track only when it is the same variant, as -sl selects them,
# so en-US SDH is stripped next to a plain en-GB: -sl en-US would otherwise keep only the SDH
if config.subtitle.get("strip_sdh", True):
plain_langs = [x.language for x in title.tracks.subtitles if not x.sdh and not x.forced]
for subtitle in title.tracks.subtitles:
if subtitle.sdh and not matching_languages(subtitle.language, plain_langs, exact=True):
plain_langs.append(subtitle.language)
non_sdh_sub = deepcopy(subtitle)
non_sdh_sub.id += "_stripped"
non_sdh_sub.sdh = False
title.tracks.add(non_sdh_sub)
events.subscribe(
events.Types.TRACK_MULTIPLEX,
lambda track, sub_id=non_sdh_sub.id: (
(track.strip_hearing_impaired()) if track.id == sub_id else None
),
)
if real_video_bitrate:
with console.status("Probing real video bitrates...", spinner="dots"):
apply_real_bitrates(
title.tracks.videos,
service.session,
log=self.log,
group_key=lambda t: (t.codec, t.range),
)
if real_audio_bitrate:
with console.status("Probing real audio bitrates...", spinner="dots"):
apply_real_bitrates(
title.tracks.audio,
service.session,
log=self.log,
group_key=lambda t: (t.codec, t.channels, str(t.language), t.descriptive),
)
with console.status("Sorting tracks by language and bitrate...", spinner="dots"):
audio_priority = config.audio.get("language_priority") or []
subtitle_priority = config.subtitle.get("language_priority") or []
title.tracks.sort_videos(
by_language=resolve_sort_langs(v_lang or lang, title.language, exact=exact_lang),
exact_match=exact_lang,
)
title.tracks.sort_audio(
by_language=resolve_sort_langs(
[*audio_priority, *(a_lang or lang)], title.language, exact=exact_lang
),
codec_priority=config.audio.get("codec_priority"),
exact_match=exact_lang,
)
title.tracks.sort_subtitles(
by_language=resolve_sort_langs(
[*subtitle_priority, *(s_lang or [])], title.language, exact=exact_lang
),
type_priority=config.subtitle.get("type_priority"),
group_by=config.subtitle.get("group_by"),
exact_match=exact_lang,
)
if list_:
available_tracks, _ = title.tracks.tree()
console.print(Padding(listing_panel(available_tracks, "Available Tracks"), (0, 5)))
continue
keep_videos = True
keep_audio = True
keep_subtitles = True
keep_chapters = True
if video_only or audio_only or subs_only or chapters_only:
keep_videos = video_only
keep_audio = audio_only
keep_subtitles = subs_only
keep_chapters = chapters_only
if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES:
console.log("Skipping subtitles - service does not support subtitle downloads")
keep_subtitles = False
s_lang = None
elif no_subs:
console.log("Skipped subtitles as --no-subs was used...")
keep_subtitles = False
s_lang = None
if no_video:
console.log("Skipped video as --no-video was used...")
keep_videos = False
v_lang = None
if no_audio:
console.log("Skipped audio as --no-audio was used...")
keep_audio = False
a_lang = None
if no_chapters:
console.log("Skipped chapters as --no-chapters was used...")
keep_chapters = False
if no_attachments:
console.log("Skipped attachments as --no-attachments was used...")
if no_proxy_download and any(service.session.proxies.values()):
console.log("Bypassing proxy for downloads as --no-proxy-download was used...")
elif proxy_download:
console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}")
for kind, required, available in (
(
"audio",
require_audio if keep_audio else [],
[a.language for a in title.tracks.audio if a.descriptive or not audio_description_only]
+ embedded_audio_langs(title.tracks.videos, keep_videos and not audio_description_only),
),
("video", require_video if keep_videos else [], [v.language for v in title.tracks.videos]),
(
"subtitle",
require_subs if keep_subtitles else [],
[t.language for t in title.tracks.subtitles if t.forced or not forced_subs_only],
),
):
missing_required = missing_required_langs(required, available, title.language, exact=exact_lang)
if missing_required:
self.log.error(f"Required {kind} language(s) not found: {', '.join(missing_required)}")
sys.exit(1)
if not keep_videos:
title.tracks.videos = []
if not keep_audio:
title.tracks.audio = []
if not keep_subtitles:
title.tracks.subtitles = []
if not keep_chapters:
title.tracks.chapters = []
with console.status("Selecting tracks...", spinner="dots"):
video_excl_r = resolve_sort_langs(video_excl, title.language, exact=exact_lang)
audio_excl_r = resolve_sort_langs(audio_excl, title.language, exact=exact_lang)
s_excl_r = resolve_sort_langs(s_lang_excl, title.language, exact=exact_lang)
fsl_excl_r = resolve_sort_langs(fsl_excl, title.language, exact=exact_lang)
if isinstance(title, (Movie, Episode)):
if keep_videos and vcodec:
title.tracks.select_video(lambda x: x.codec in vcodec)
missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)]
for codec in missing_codecs:
self.log.warning(f"Skipping {codec.name} video tracks as none are available.")
if not title.tracks.videos:
self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...")
sys.exit(1)
if keep_videos and range_:
# Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks
if Video.Range.HYBRID not in range_:
dv_has_hdr10p = any(x.hdr10plus for x in title.tracks.videos)
title.tracks.select_video(lambda x: x.range in range_)
missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)]
for color_range in missing_ranges:
self.log.warning(f"Skipping {color_range.name} video tracks as none are available.")
if Video.Range.HDR10P in missing_ranges and dv_has_hdr10p:
self.log.warning("A DV track carries HDR10+: use -r DV or -r HYBRID.")
if not title.tracks.videos:
self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...")
sys.exit(1)
if keep_videos and vbitrate:
if any(r == Video.Range.HYBRID for r in range_):
# In HYBRID mode, only apply bitrate filter to non-DV tracks
# DV tracks are kept regardless since they're only used for RPU metadata
title.tracks.select_video(
lambda x: x.range == Video.Range.DV or (x.bitrate and x.bitrate // 1000 == vbitrate)
)
if not any(x.range != Video.Range.DV for x in title.tracks.videos):
self.log.error(f"There's no {vbitrate}kbps Video Track...")
sys.exit(1)
else:
title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate)
if not title.tracks.videos:
self.log.error(f"There's no {vbitrate}kbps Video Track...")
sys.exit(1)
if keep_videos and vbitrate_min is not None and vbitrate_max is not None:
title.tracks.select_video(
lambda x: x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max
)
if not title.tracks.videos:
self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...")
sys.exit(1)
if keep_videos and video_excl_r:
drop = matched_language_tags(
video_excl_r, [t.language for t in title.tracks.videos], exact_lang, every_variant=True
)
title.tracks.select_video(lambda x: str(x.language) not in drop)
if not title.tracks.videos:
self.log.error(f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...")
sys.exit(1)
effective_video_lang = v_lang or lang
video_languages = [lang for lang in effective_video_lang if lang != "best"]
video_multi_lang = (
"best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1
)
if keep_videos and video_languages and "all" not in video_languages:
processed_video_lang = resolve_sort_langs(video_languages, title.language, exact=exact_lang)
v_orig_token: Optional[str] = None
if "orig" in video_languages:
if title.language:
v_orig_token = original_tag(title.language, exact_lang)
else:
self.log.warning(
"Original language not available for title, skipping 'orig' selection for video"
)
if v_orig_token in video_languages:
v_orig_token = None
missing_v_langs = find_missing_langs(
processed_video_lang,
[v.language for v in title.tracks.videos],
exact=exact_lang,
)
if missing_v_langs:
missing_str = as_requested(missing_v_langs, v_orig_token)
if best_available:
remaining_v_langs = [tok for tok in processed_video_lang if tok not in missing_v_langs]
if remaining_v_langs:
self.log.warning(
f"{missing_str} not found in video tracks, "
f"continuing with: {as_requested(remaining_v_langs, v_orig_token)}"
)
processed_video_lang = remaining_v_langs
else:
self.log.error(f"{missing_str} not found in video tracks and no fallback available")
sys.exit(1)
else:
self.log.error(missing_str + " not found in video tracks")
sys.exit(1)
title.tracks.videos = title.tracks.by_language(
title.tracks.videos, processed_video_lang, exact_match=exact_lang
)
if not title.tracks.videos:
# empty only when 'orig' was the sole request and did not resolve
self.log.error(
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..."
)
sys.exit(1)
has_hybrid = any(r == Video.Range.HYBRID for r in range_)
non_hybrid_ranges = [r for r in range_ if r != Video.Range.HYBRID]
effective_quality = quality
if keep_videos and quality:
missing_resolutions = []
if has_hybrid:
hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos(
title.tracks.videos, non_hybrid_ranges
)
# only ranges that can actually be delivered count towards "present"
deliverable_pool = hybrid_candidate_tracks + [
v for v in non_hybrid_tracks if v.range in non_hybrid_ranges
]
for resolution in quality:
if any(v.height == resolution for v in deliverable_pool):
continue
if any(v.width and int(v.width * 9 / 16) == resolution for v in deliverable_pool):
continue
missing_resolutions.append(resolution)
if len(missing_resolutions) == len(quality) and best_available:
# every requested resolution missing: leave the pool untouched so the
# best-available hybrid pass below selects the top resolution instead
effective_quality = []
else:
hybrid_filter = title.tracks.select_hybrid(
hybrid_candidate_tracks, quality, worst=worst, generate_dv=generate_dv
)
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
if non_hybrid_ranges and non_hybrid_tracks:
non_hybrid_selected = [
v
for v in non_hybrid_tracks
if any(
v.height == res or (v.width and int(v.width * (9 / 16)) == res)
for res in quality
)
]
title.tracks.videos = Tracks.merge_video_selections(
hybrid_selected, non_hybrid_selected
)
else:
title.tracks.videos = hybrid_selected
else:
pre_quality_videos = list(title.tracks.videos)
title.tracks.by_resolutions(quality)
for resolution in quality:
if any(v.height == resolution for v in title.tracks.videos):
continue
if any(v.width and int(v.width * 9 / 16) == resolution for v in title.tracks.videos):
continue
missing_resolutions.append(resolution)
if len(missing_resolutions) == len(quality) and best_available:
# every requested resolution missing: fall back to the best available
title.tracks.videos = pre_quality_videos
effective_quality = []
if missing_resolutions:
res_list = ""
if len(missing_resolutions) > 1:
res_list = ", ".join([f"{x}p" for x in missing_resolutions[:-1]]) + " or "
res_list = f"{res_list}{missing_resolutions[-1]}p"
plural = "s" if len(missing_resolutions) > 1 else ""
if best_available:
self.log.warning(
f"There's no {res_list} Video Track{plural}, continuing with available qualities..."
)
else:
self.log.error(f"There's no {res_list} Video Track{plural}...")
sys.exit(1)
pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else []
if keep_videos and has_hybrid:
hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos(
title.tracks.videos, non_hybrid_ranges
)
hybrid_quality = effective_quality
if not hybrid_quality:
best_resolution = max((v.height for v in hybrid_candidate_tracks), default=None)
hybrid_quality = [best_resolution] if best_resolution else []
hybrid_selected = []
if hybrid_quality:
hybrid_filter = title.tracks.select_hybrid(
hybrid_candidate_tracks, hybrid_quality, worst=worst, generate_dv=generate_dv
)
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
for res in hybrid_quality:
if not any(at_resolution(v, res) for v in hybrid_selected):
self.log.warning(f"Skipping {res}p: nothing at that resolution can make a hybrid")
non_hybrid_selected: list[Video] = []
if non_hybrid_ranges and non_hybrid_tracks:
if video_multi_lang:
non_hybrid_langs = list(dict.fromkeys(str(v.language) for v in non_hybrid_tracks))
else:
non_hybrid_langs = [None]
for resolution, color_range, codec, vlang in product(
effective_quality or [None], non_hybrid_ranges, vcodec or [None], non_hybrid_langs
):
candidates = [
t
for t in non_hybrid_tracks
if (
not resolution
or t.height == resolution
or (t.width and int(t.width * (9 / 16)) == resolution)
)
and (not color_range or t.range == color_range)
and (not codec or t.codec == codec)
and (vlang is None or str(t.language) == vlang)
]
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
if match and match not in non_hybrid_selected:
non_hybrid_selected.append(match)
title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected)
# Flag tracks selected only as hybrid ingredients (the HDR base and/or
# the lowest DV) so the standalone mux loop skips them. Tracks also
# picked as explicit deliverables stay unflagged.
Tracks.flag_hybrid_ingredients(hybrid_selected, non_hybrid_selected)
elif keep_videos:
selected_videos: list[Video] = []
if video_multi_lang:
unique_video_langs = list(dict.fromkeys(str(v.language) for v in title.tracks.videos))
else:
unique_video_langs = [None]
for resolution, color_range, codec, vlang in product(
effective_quality or [None], range_ or [None], vcodec or [None], unique_video_langs
):
candidates = [
t
for t in title.tracks.videos
if (
not resolution
or t.height == resolution
or (t.width and int(t.width * (9 / 16)) == resolution)
)
and (not color_range or t.range == color_range)
and (not codec or t.codec == codec)
and (vlang is None or str(t.language) == vlang)
]
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
if match and match not in selected_videos:
selected_videos.append(match)
title.tracks.videos = selected_videos
if keep_videos and any(r == Video.Range.HYBRID for r in range_):
gate_failure = hybrid_gate_failure(title.tracks.videos, generate_dv)
if not gate_failure and not any(
v.range == Video.Range.DV or v.native_hybrid for v in title.tracks.videos
):
self.log.info("No Dolby Vision track: Dolby Vision will be generated from HDR10+")
if gate_failure:
msg, msg_detail = gate_failure
other_ranges = [r for r in range_ if r != Video.Range.HYBRID]
if best_available and other_ranges:
self.log.warning(msg)
self.log.warning(
f"Continuing with remaining range(s): {', '.join(r.name for r in other_ranges)}"
)
range_ = other_ranges
fallback_pool = pre_hybrid_videos
if video_multi_lang:
fallback_langs = list(dict.fromkeys(str(v.language) for v in fallback_pool))
else:
fallback_langs = [None]
fallback_selected: list[Video] = []
for resolution, color_range, codec, vlang in product(
effective_quality or [None], other_ranges, vcodec or [None], fallback_langs
):
candidates = [
t
for t in fallback_pool
if (
not resolution
or t.height == resolution
or (t.width and int(t.width * (9 / 16)) == resolution)
)
and (not color_range or t.range == color_range)
and (not codec or t.codec == codec)
and (vlang is None or str(t.language) == vlang)
]
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
if match and match not in fallback_selected:
fallback_selected.append(match)
title.tracks.videos = fallback_selected
else:
self.log.error(msg)
self.log.error(msg_detail)
sys.exit(1)
fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang)
if keep_subtitles and s_excl_r:
# applies to forced subs too: an excluded language is unwanted in any form
drop = matched_language_tags(
s_excl_r, [t.language for t in title.tracks.subtitles], exact_lang, every_variant=True
)
title.tracks.select_subtitles(lambda x: str(x.language) not in drop)
if keep_subtitles and s_lang and "all" not in s_lang:
s_lang = resolve_sort_langs(s_lang, title.language, exact=exact_lang)
# the pool selection draws from: forced tracks count only when -fs keeps them
missing_langs = find_missing_langs(
s_lang,
[sub.language for sub in title.tracks.subtitles if forced_subs or not sub.forced],
exact=exact_lang,
)
if missing_langs:
missing_str = ", ".join(missing_langs)
if best_available:
remaining = [tok for tok in s_lang if tok not in missing_langs]
if remaining:
self.log.warning(
f"{missing_str} not found in subtitle tracks, continuing with: {', '.join(remaining)}"
)
s_lang = remaining
else:
self.log.warning(
f"{missing_str} not found in subtitle tracks, continuing without subtitles"
)
fsl_tags = matched_language_tags(
fsl, [x.language for x in title.tracks.subtitles if x.forced], exact_lang
)
title.tracks.select_subtitles(lambda x: x.forced and str(x.language) in fsl_tags)
else:
self.log.error(missing_str + " not found in subtitle tracks")
sys.exit(1)
if s_lang and title.tracks.subtitles:
plain_tags = plain_subtitle_tags(s_lang, title.tracks.subtitles, exact_lang)
forced_tags = matched_language_tags(
[*s_lang, *fsl], [x.language for x in title.tracks.subtitles if x.forced], exact_lang
)
title.tracks.select_subtitles(
lambda x: str(x.language) in (forced_tags if x.forced else plain_tags)
)
if not title.tracks.subtitles and not best_available:
self.log.error(f"There's no {s_lang} Subtitle Track...")
sys.exit(1)
if keep_subtitles:
# fsl, not forced_s_lang: unresolvable orig must keep all forced, not drop them
if fsl:
fsl_tags = matched_language_tags(
fsl, [x.language for x in title.tracks.subtitles if x.forced], exact_lang
)
title.tracks.select_subtitles(lambda x: not x.forced or str(x.language) in fsl_tags)
if not any(x.forced for x in title.tracks.subtitles):
self.log.warning(
f"No forced subtitle tracks matched --forced-s-lang ({', '.join(forced_s_lang)})"
)
elif not forced_subs:
title.tracks.select_subtitles(lambda x: not x.forced)
if fsl_excl_r:
drop = matched_language_tags(
fsl_excl_r,
[t.language for t in title.tracks.subtitles if t.forced],
exact_lang,
every_variant=True,
)
title.tracks.select_subtitles(lambda x: not (x.forced and str(x.language) in drop))
if forced_subs_only:
title.tracks.select_subtitles(lambda x: x.forced)
if not title.tracks.subtitles:
skip_title(title, "There's no forced subtitle track for the selected languages.")
continue
if keep_audio and audio_description_only:
title.tracks.select_audio(lambda x: x.descriptive)
if not title.tracks.audio:
skip_title(title, "There's no audio description track.")
continue
# might have no audio tracks if part of the video, e.g. transport stream hls
if keep_audio and len(title.tracks.audio) > 0:
if not audio_description:
title.tracks.select_audio(lambda x: not x.descriptive)
if not title.tracks.audio:
self.log.error("Audio description only, use --audio-description...")
sys.exit(1)
if acodec:
title.tracks.select_audio(lambda x: x.codec in acodec)
if not title.tracks.audio:
codec_names = ", ".join(c.name for c in acodec)
self.log.error(f"No audio tracks matching codecs: {codec_names}")
sys.exit(1)
if channels:
title.tracks.select_audio(
lambda x: bool(
x.channels and math.ceil(Audio.channel_total(x.channels)) == math.ceil(channels)
)
)
if not title.tracks.audio:
self.log.error(f"There's no {channels} Audio Track...")
sys.exit(1)
if no_atmos:
title.tracks.audio = [x for x in title.tracks.audio if not x.atmos]
if not title.tracks.audio:
self.log.error("No non-Atmos audio tracks available...")
sys.exit(1)
if abitrate:
title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate)
if not title.tracks.audio:
self.log.error(f"There's no {abitrate}kbps Audio Track...")
sys.exit(1)
if abitrate_min is not None and abitrate_max is not None:
title.tracks.select_audio(
lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max
)
if not title.tracks.audio:
self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...")
sys.exit(1)
if audio_excl_r:
drop = matched_language_tags(
audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True
)
title.tracks.select_audio(lambda x: str(x.language) not in drop)
if not title.tracks.audio:
self.log.error(f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...")
sys.exit(1)
audio_languages = a_lang or lang
if audio_languages:
processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang)
a_orig_token: Optional[str] = None
if "orig" in audio_languages:
if title.language:
a_orig_token = original_tag(title.language, exact_lang)
else:
self.log.warning("Original language not available for title, skipping 'orig' selection")
if a_orig_token in audio_languages:
a_orig_token = None
embedded_langs = embedded_audio_langs(
title.tracks.videos, keep_videos and not audio_description_only
)
audio_noun = "audio description tracks" if audio_description_only else "audio tracks"
if not any(tok in processed_lang for tok in ("best", "all")):
missing_a_langs = find_missing_langs(
processed_lang,
[a.language for a in title.tracks.audio] + embedded_langs,
exact=exact_lang,
)
if missing_a_langs:
missing_str = as_requested(missing_a_langs, a_orig_token)
if best_available:
remaining = [tok for tok in processed_lang if tok not in missing_a_langs]
if remaining:
self.log.warning(
f"{missing_str} not found in {audio_noun}, "
f"continuing with: {as_requested(remaining, a_orig_token)}"
)
processed_lang = remaining
else:
self.log.error(
f"{missing_str} not found in {audio_noun} and no fallback available"
)
sys.exit(1)
else:
self.log.error(f"{missing_str} not found in {audio_noun}")
sys.exit(1)
title.tracks.audio = select_best_audio(
title.tracks.audio, processed_lang, acodec, audio_description, exact_lang
)
if not title.tracks.audio and not embedded_langs:
# empty when 'orig' was the sole request, did not resolve, and the
# video carries no audio of its own to fall back on
self.log.error(
f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} "
f"Audio Track, cannot continue..."
)
sys.exit(1)
kept_tracks = []
if keep_videos:
kept_tracks.extend(title.tracks.videos)
if keep_audio:
kept_tracks.extend(title.tracks.audio)
if keep_subtitles:
kept_tracks.extend(title.tracks.subtitles)
if keep_chapters:
kept_tracks.extend(title.tracks.chapters)
if not no_attachments:
kept_tracks.extend(title.tracks.attachments)
title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url)
selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True)
log_event(
"track_select",
level="INFO",
message=(
f"Selected {len(title.tracks.videos)} video, {len(title.tracks.audio)} audio, "
f"{len(title.tracks.subtitles)} subtitle track(s)"
),
context={
"videos": [
f"{v.codec} {v.range} {v.height}p {v.bitrate}bps {v.language}" for v in title.tracks.videos
],
"audio": [
f"{a.codec} {a.language} {getattr(a, 'channels', None)}ch {a.bitrate}bps"
for a in title.tracks.audio
],
"subtitles": [f"{s.codec} {s.language}" for s in title.tracks.subtitles],
},
)
if progress_sink is not None:
from unshackle.core.api.progress import build_job_progress_callables
tracks_progress_callables = build_job_progress_callables(
list(title.tracks), tracks_progress_callables, progress_sink
)
drm_pending = [track for track in title.tracks if getattr(track, "needs_drm_loading", False)]
if drm_pending:
with ThreadPoolExecutor(min(16, len(drm_pending))) as drm_pool:
list(drm_pool.map(lambda track: track.load_drm_if_needed(service), drm_pending))
download_table = Table.grid()
download_table.add_row(selected_tracks)
def prepare_drm_for(track: AnyTrack) -> Callable:
prepare = partial(
partial(self.prepare_drm, table=download_table),
track=track,
title=title,
certificate=partial(service.get_widevine_service_certificate, title=title, track=track),
licence=partial(self.service_licence, service, title=title, track=track),
clearkey_licence=partial(service.get_clearkey_license, title=title, track=track),
cdm_only=cdm_only,
vaults_only=vaults_only,
export=export_path,
service_session=service.session,
)
if title_all_drm:
return partial(self.licence_all_drm, prepare, track, best_available, force=bool(cdm_only))
return prepare
server_cdm_type = None
if getattr(self._remote_service, "_server_cdm", False):
server_cdm_type = getattr(self._remote_service, "_server_cdm_type", "widevine")
self.cdm = cdm_type_stub(server_cdm_type)
from unshackle.core.import_service import ImportService
if isinstance(self._remote_service, ImportService):
self.log.info("Using keys from the export; no CDM required")
else:
self.log.info(f"Using server CDM ({server_cdm_type.title()}); no local CDM required")
video_tracks = title.tracks.videos
if video_tracks and not server_cdm_type:
highest_quality = max((track.height for track in video_tracks if track.height), default=0)
if highest_quality > 0:
if is_widevine_cdm(self.cdm):
quality_based_cdm = self.get_cdm(
self.service, self.profile, drm="widevine", quality=highest_quality
)
if quality_based_cdm and quality_based_cdm != self.cdm:
self.log.debug(
f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks"
)
self.cdm = quality_based_cdm
elif is_playready_cdm(self.cdm):
quality_based_cdm = self.get_cdm(
self.service, self.profile, drm="playready", quality=highest_quality
)
if quality_based_cdm and quality_based_cdm != self.cdm:
self.log.debug(
f"Pre-selecting PlayReady CDM based on highest quality {highest_quality}p across all video tracks"
)
self.cdm = quality_based_cdm
title_all_drm = all_drm and not server_cdm_type
if all_drm and server_cdm_type and DOWNLOAD_ALL_DRM.is_set():
DOWNLOAD_ALL_DRM.clear()
self.log.warning("--all-drm is ignored: the server CDM licenses this session with one DRM system")
if title_all_drm:
heights = {track.height for track in title.tracks.videos if track.height} or {None}
for system, matches in (("Widevine", is_widevine_cdm), ("PlayReady", is_playready_cdm)):
system_cdms = [self.cdm_for(system.lower(), height) for height in heights]
if not all(system_cdm and matches(system_cdm) for system_cdm in system_cdms):
hint = (
"--cdm selects one device, so remove it."
if self.cdm_override
else f"Set cdm.{self.service}.{system.lower()} in the configuration."
)
raise click.ClickException(f"--all-drm needs a {system} CDM for {self.service}. {hint}")
if is_remote_cdm(system_cdms[0]) and system not in self.all_drm_warned:
self.all_drm_warned.add(system)
self.log.warning(
f"The {system} CDM is remote. It can answer from its own key cache "
"and send no licence request."
)
if hasattr(service, "resolve_server_keys"):
service.resolve_server_keys(title)
if not cdm_only:
self.prefer_vault_keys(title)
self.cache_resolved_keys(title)
if sample and not skip_dl:
whole_files = [
track
for track in (*title.tracks.videos, *title.tracks.audio)
if not track.can_sample(track.session or service.session)
]
if whole_files:
raise click.ClickException(
f"--sample cannot cut {len(whole_files)} track(s) of {title}: "
f"{self.service} gives each one as one file with no segment index."
)
dl_start_time = time.time()
try:
with SyncLive(Padding(download_table, (1, 5)), console=console, refresh_per_second=20):
def download_track(track: AnyTrack, i: int) -> None:
track.download(
session=track.session or service.session,
no_proxy_download=no_proxy_download,
proxy_download=proxy_download,
prepare_drm=prepare_drm_for(track),
cdm=self.cdm,
max_workers=workers,
adaptive_workers=adaptive_workers,
download_processes=download_processes,
progress=tracks_progress_callables[i],
)
if (
track.path
and service.decrypts_samples(track)
and not DOWNLOAD_CANCELLED.is_set()
and not DOWNLOAD_LICENCE_ONLY.is_set()
):
progress = tracks_progress_callables[i]
progress(downloaded="Decrypting", completed=0, total=None)
cbcs.decrypt_file(
track.path, lambda _l, entry: service.get_sample_decrypter(track, entry.index)
)
assert_fragments_decrypted(track.path)
progress(downloaded="Decrypted", completed=100, total=100)
# DRM-free and HLS-ClearKey tracks never reach prepare_drm, so export here.
# drm=None on purpose: licensed tracks already recorded their DRM/keys
# in prepare_drm, and write_export merges via setdefault.
if export_path:
self.write_export(export_path, title, track)
def on_subtitle_skipped(track: Subtitle) -> None:
lang = str(track.language)
self.log.warning(f"Subtitle {lang} failed to download, skipping it.")
self.skipped_subtitles.append(SkippedSubtitle(id=track.id, language=lang, title=str(title)))
try:
title.tracks.subtitles.remove(track)
except ValueError:
self.log.debug(f"Skipped subtitle {track.id} was already absent from the track list.")
skipped_before = len(self.skipped_subtitles)
grow_session_pool(service.session, downloads * (workers or default_max_workers()))
download_tracks_in_passes(
title.tracks,
downloads,
download_track,
skip_subtitle_errors=skip_subtitle_errors,
on_subtitle_skipped=on_subtitle_skipped,
)
for attachment in title.tracks.attachments:
attachment.download(
attachment.session or service.session,
no_proxy_download=no_proxy_download,
proxy_download=proxy_download,
)
if (
len(self.skipped_subtitles) > skipped_before
and not title.tracks.videos
and not title.tracks.audio
and not title.tracks.subtitles
):
self.log.warning(
f"{title}: all subtitles were skipped and no video or audio was "
"downloaded - nothing was produced for this title."
)
except KeyboardInterrupt:
console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5)))
log_event(
"download_tracks",
level="WARNING",
service=self.service,
message="Download cancelled by user",
context={"title": str(title)},
)
self.wait_vault_writes()
if export_path:
self.log_export_summary()
return
except Exception as e: # noqa
# Reported and swallowed (no re-raise) so the CLI exits cleanly; flag it so the
# API worker sees the title failed rather than completing with no output.
self.download_failed = True
console.print(Padding(Group(*self.failure_lines(e)), (1, 5)))
if not isinstance(e, click.ClickException):
console.print_exception()
if self.debug_logger:
self.debug_logger.log_error(
"download_tracks",
e,
service=self.service,
context={
"title": str(title),
"error_type": type(e).__name__,
"tracks_count": len(title.tracks),
"returncode": getattr(e, "returncode", None),
},
)
dispatch(
"failure",
"file",
build_context(
title,
service=self.service,
ids=self.post_script_ids(),
error=f"{type(e).__name__}: {e}",
),
postscript,
)
self.wait_vault_writes()
if export_path:
self.log_export_summary()
return
self.wait_vault_writes()
if export_path:
self.log_export_title(title)
if skip_dl:
console.log("Skipped downloads as --skip-dl was used...")
else:
dl_time = time_elapsed_since(dl_start_time)
console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5)))
subtitle_output_mode = config.subtitle.get("output_mode", "mux")
sidecar_format = config.subtitle.get("sidecar_format", "srt")
skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio)
sidecar_subtitles: list[Subtitle] = []
sidecar_original_paths: dict[str, Path] = {}
sidecar_files: dict[Optional[Path], list[Path]] = {}
if subtitle_output_mode in ("sidecar", "both") and not no_mux:
sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()]
if sidecar_format == "original":
config.directories.temp.mkdir(parents=True, exist_ok=True)
for subtitle in sidecar_subtitles:
original_path = (
config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}"
)
shutil.copy2(subtitle.path, original_path)
sidecar_original_paths[subtitle.id] = original_path
with console.status("Converting Subtitles..."):
sub_conversions: dict[tuple[str, str], int] = {}
for subtitle in title.tracks.subtitles:
if sub_format == "original":
continue
if sub_format:
if subtitle.codec != sub_format:
src = getattr(subtitle.codec, "name", str(subtitle.codec))
dst = getattr(sub_format, "name", str(sub_format))
subtitle.convert(sub_format, forced=True)
sub_conversions[(src, dst)] = sub_conversions.get((src, dst), 0) + 1
elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang:
# MKV does not support TTML, VTT is the next best option
src = getattr(subtitle.codec, "name", str(subtitle.codec))
subtitle.convert(Subtitle.Codec.WebVTT)
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
)
if sample:
for subtitle in title.tracks.subtitles:
if subtitle.path and subtitle.path.exists():
subtitle.trim(*sample)
for (src, dst), count in sub_conversions.items():
log_event(
"subtitle_convert",
level="INFO",
message=f"Converted {src}->{dst} x{count}",
context={"from": src, "to": dst, "count": count},
)
with console.status("Checking Subtitles for Fonts..."):
font_names: list[str] = []
for subtitle in [] if no_attachments else title.tracks.subtitles:
if subtitle.codec in (Subtitle.Codec.SubStationAlpha, Subtitle.Codec.SubStationAlphav4):
font_names.extend(Subtitle.extract_fonts(subtitle.path.read_text("utf8")))
font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title)
if font_count:
self.log.info(f"Attached {font_count} fonts for the Subtitles")
if missing_fonts and sys.platform != "win32":
self.suggest_missing_fonts(missing_fonts)
# Handle DRM decryption BEFORE repacking (must decrypt first!)
service_name = service.__class__.__name__.upper()
decryption_method = resolve_decryption(config.decryption_map, config.decryption, service_name)
decrypt_tool = "mp4decrypt" if decryption_method.lower() == "mp4decrypt" else "Shaka Packager"
drm_tracks = [track for track in title.tracks if track.drm]
if drm_tracks:
with console.status(f"Decrypting tracks with {decrypt_tool}..."):
has_decrypted = False
for track in drm_tracks:
drm = track.get_drm_for_cdm(self.cdm)
if drm and hasattr(drm, "decrypt"):
self.decrypt_verified(drm, track.path, prepare_drm_for(track))
if not isinstance(drm, MonaLisa):
# MonaLisa decrypts per segment; its decrypt() here is a no-op
assert_fragments_decrypted(track.path)
has_decrypted = True
events.emit(events.Types.TRACK_REPACKED, track=track)
else:
self.log.warning(
f"No matching DRM found for track {track} with CDM type {type(self.cdm).__name__}"
)
if has_decrypted:
self.log.info(f"Decrypted tracks with {decrypt_tool}")
if (
not no_subs
and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES)
and not video_only
and not no_video
):
for video_track_n, video_track in enumerate(title.tracks.videos):
has_manifest_cc = bool(getattr(video_track, "closed_captions", None))
has_eia_cc = (
not has_manifest_cc
and not title.tracks.subtitles
and any(
x.get("codec_name", "").startswith("eia_")
for x in ffprobe(video_track.path).get("streams", [])
)
)
if not has_manifest_cc and not has_eia_cc:
continue
with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."):
try:
cc_lang = (
Language.get(video_track.closed_captions[0]["language"])
if has_manifest_cc and video_track.closed_captions[0].get("language")
else title.language or video_track.language
)
track_id = f"ccextractor-{video_track.id}"
cc = video_track.ccextractor(
track_id=track_id,
out_path=config.directories.temp
/ config.filenames.subtitle.format(id=track_id, language=cc_lang),
language=cc_lang,
original=False,
)
if cc:
cc.cc = True
title.tracks.add(cc)
self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
else:
self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
except EnvironmentError:
self.log.error(
"Cannot extract Closed Captions as the ccextractor executable was not found..."
)
break
# Now repack the decrypted tracks
if progress_sink and any(getattr(t, "needs_repack", False) for t in title.tracks):
progress_sink(
{"phase": "repackaging", "progress": 92.0, "status": "downloading", "active_tracks": []}
)
with console.status("Repackaging tracks with FFMPEG..."):
has_repacked = False
# Video tracks whose VUI was folded into the repack pass; skipped in the VUI loop below.
vui_folded = set()
for track in title.tracks:
if track.needs_repack:
bsf = track.vui_bsf() if isinstance(track, Video) else None
if track.repackage(bsf_v=bsf):
vui_folded.add(id(track))
has_repacked = True
events.emit(events.Types.TRACK_REPACKED, track=track)
if has_encrypted_sample_entry(track.path):
self.log.warning(
f"Track {track.id} still has an encrypted sample entry after repacking, so "
"decryption likely failed and the muxed output may be unplayable."
)
if has_repacked:
self.log.info("Repacked one or more tracks with FFMPEG")
with console.status("Normalizing video VUI..."):
for track in title.tracks.videos:
if id(track) in vui_folded:
continue
try:
track.normalize_vui()
except Exception as e: # noqa: BLE001
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
remux_music_tracks(title, self.log)
if sample:
cut_music_sample(title, sample, self.log)
muxed_paths = []
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
tagged_paths: set[Path] = set()
append_audio_codec_suffix = True
if no_mux:
for track in title.tracks:
if track.path and track.path.exists():
muxed_paths.append(track.path)
elif isinstance(title, (Movie, Episode)):
progress = Progress(
TextColumn("[progress.description]{task.description}"),
SpinnerColumn(finished_text=""),
GradientPulseBarColumn(),
"•",
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
"•",
TextColumn("{task.fields[downloaded]}"),
console=console,
)
merge_audio = (
(not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True)
)
# When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec.
append_audio_codec_suffix = not merge_audio
merge_video = merge_video if merge_video is not None else config.muxing.get("merge_video", False)
multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = []
mux_futures: list[futures.Future[tuple[Path, int, list[str]]]] = []
# Track hybrid-processing outputs explicitly so we can always clean them up,
# even if muxing fails early (e.g. SystemExit) before the normal delete loop.
hybrid_temp_paths: list[Path] = []
hybrid_base_ids: set[str] = set()
container_paths: dict[str, Path] = {}
def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks:
task_tracks = Tracks()
task_tracks.videos = list(base_tracks.videos)
task_tracks.audio = audio_tracks
task_tracks.subtitles = list(base_tracks.subtitles)
task_tracks.chapters = base_tracks.chapters
task_tracks.attachments = list(base_tracks.attachments)
return task_tracks
audio_expected = not video_only and not no_audio
def unique_mux_output(task_tracks: Tracks, index: int) -> Optional[Path]:
src: Optional[Path] = None
ext = ".muxed.mkv"
if task_tracks.videos:
src = task_tracks.videos[0].path
elif task_tracks.audio:
src, ext = task_tracks.audio[0].path, ".muxed.mka"
elif task_tracks.subtitles:
src, ext = task_tracks.subtitles[0].path, ".muxed.mks"
if src is None:
return None
base = src.with_suffix(ext)
return base.with_name(f"{base.stem}.{index}{base.suffix}")
def mux_one(mux_index: int, task_id: TaskID, task_tracks: Tracks) -> tuple[Path, int, list[str]]:
progress.start_task(task_id)
muxed_path, return_code, errors = task_tracks.mux(
str(title),
progress=partial(progress.update, task_id=task_id),
delete=False,
audio_expected=audio_expected,
title_language=title.language,
skip_subtitles=skip_subtitle_mux,
output_path=unique_mux_output(task_tracks, mux_index),
)
if return_code < 2:
tags.tag_file(
muxed_path,
title,
self.tmdb_id,
self.imdb_id,
self.tvdb_id,
self.anilist_id,
self.anime_hint(title),
session=service.session,
)
tagged_paths.add(muxed_path)
return muxed_path, return_code, errors
def enqueue_mux_tasks(task_description: str, base_tracks: Tracks, hybrid: bool = False) -> None:
pool = hybrid_mux_pool if hybrid else mux_pool
if merge_audio or not base_tracks.audio:
task_id = progress.add_task(
f"{task_description}...", total=None, start=False, downloaded=""
)
multiplex_tasks.append((task_id, base_tracks, None))
mux_futures.append(pool.submit(mux_one, len(multiplex_tasks), task_id, base_tracks))
return
audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {}
for audio_track in base_tracks.audio:
audio_by_codec.setdefault(audio_track.codec, []).append(audio_track)
for audio_codec, codec_audio_tracks in audio_by_codec.items():
description = task_description
if audio_codec:
description = f"{task_description} {audio_codec.name}"
task_id = progress.add_task(f"{description}...", total=None, start=False, downloaded="")
task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks)
multiplex_tasks.append((task_id, task_tracks, audio_codec))
mux_futures.append(pool.submit(mux_one, len(multiplex_tasks), task_id, task_tracks))
def mux_video_group(video_tracks: list[Optional[Video]]) -> None:
for video_track in video_tracks:
if video_track and video_track.dv_compatible_bitstream:
container = apply_dv_fixup(video_track, keep_source=video_track.id in hybrid_base_ids)
if container:
container_paths[video_track.id] = container
hybrid_temp_paths.append(container)
task_description = "Multiplexing"
# All tracks in a merged group share height/range/codec, so describe from the first.
head = next((v for v in video_tracks if v), None)
if head:
if len(quality) > 1:
task_description += f" {head.height}p"
if len(range_) > 1:
task_description += f" {head.range.name}"
if len(vcodec) > 1:
task_description += f" {head.codec.name}"
task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
if head:
task_tracks.videos = [v for v in video_tracks if v]
enqueue_mux_tasks(task_description, task_tracks)
def group_file_size(video_tracks: Sequence[Optional[Video]]) -> int:
return sum(v.path.stat().st_size for v in video_tracks if v and v.path and v.path.exists())
if progress_sink:
progress_sink(
{"phase": "muxing", "progress": 96.0, "status": "downloading", "active_tracks": []}
)
try:
with SyncLive(Padding(progress, (0, 5, 1, 5)), console=console, refresh_per_second=20):
mux_failed = False
mux_concurrency = max(1, int(config.muxing.get("concurrency", 4)))
with (
ThreadPoolExecutor(mux_concurrency) as mux_pool,
ThreadPoolExecutor(mux_concurrency) as extra_mux_pool,
):
hybrid_mux_pool = mux_pool if mux_concurrency == 1 else extra_mux_pool
if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos:
self.log.info("Processing Hybrid HDR10+DV tracks...")
original_videos = list(title.tracks.videos)
standalone_videos = [v for v in original_videos if not v.hybrid_base_only]
standalone_groups: list[list[Optional[Video]]] = [
list(g) for g in group_videos_by_variant(standalone_videos, merge=merge_video)
]
hybrid_bases, matching_dv = hybrid_plan(title.tracks.videos, generate_dv)
hybrid_base_ids.update(v.id for v in hybrid_bases)
hybrids_made = 0
for video in (v for v in original_videos if v.native_hybrid):
self.log.info(
f"The {video.height}p {video.range_label} track is already the hybrid"
)
hybrids_made += 1
for group in sorted(standalone_groups, key=group_file_size, reverse=True):
mux_video_group(group)
for hdr10_track in hybrid_bases:
resolution = hdr10_track.height or 0
container = container_paths.get(hdr10_track.id)
resolution_tracks = hybrid_source_tracks(hdr10_track, matching_dv, container)
hybrid_filename = f"HDR10-DV-{resolution}p.hevc"
hybrid_output_path = config.directories.temp / hybrid_filename
hybrid_temp_paths.append(hybrid_output_path)
if len({track.sample_start for track in resolution_tracks}) > 1:
raise click.ClickException(
"--sample cut the HDR10 and Dolby Vision tracks at different "
"points, so a hybrid of them would not line up."
)
step_error = hybrid_step_error(resolution_tracks, self.service, hybrid_filename)
if step_error:
# the other files of the title are already queued for muxing
self.log.error(f"Hybrid {resolution}p failed, skipping it: {step_error}")
continue
hybrids_made += 1
if container:
try:
container.unlink(missing_ok=True)
except PermissionError:
pass # still in hybrid_temp_paths, so the final cleanup tries again
task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p"
task_tracks = (
Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
)
hybrid_track = deepcopy(hdr10_track)
hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}"
hybrid_track.path = hybrid_output_path
hybrid_track.range = Video.Range.DV
hybrid_track.needs_duration_fix = True
title.tracks.add(hybrid_track)
task_tracks.videos = [hybrid_track]
enqueue_mux_tasks(task_description, task_tracks, hybrid=True)
no_hybrid = missing_hybrid_error(hybrid_bases, hybrids_made)
if no_hybrid:
raise click.ClickException(no_hybrid)
console.print()
else:
groups: list[list[Optional[Video]]] = [
list(g) for g in group_videos_by_variant(title.tracks.videos, merge=merge_video)
] or [[None]]
for group in sorted(groups, key=group_file_size, reverse=True):
mux_video_group(group)
mux_results = [f.result() for f in mux_futures]
for (task_id, task_tracks, audio_codec), (muxed_path, return_code, errors) in zip(
multiplex_tasks, mux_results
):
muxed_paths.append(muxed_path)
muxed_audio_codecs[muxed_path] = audio_codec
if return_code >= 2:
self.log.error(f"Failed to Mux video to Matroska file ({return_code}):")
elif return_code == 1 or errors:
self.log.warning("mkvmerge had at least one warning or error, continuing anyway...")
for line in errors:
if line.startswith("#GUI#error"):
self.log.error(line)
else:
self.log.warning(line)
if return_code >= 2:
mux_failed = True
if mux_failed:
sys.exit(1)
if sidecar_subtitles and not no_mux:
sidecar_targets: list[tuple[Optional[Path], str, Path]] = []
for muxed_path in muxed_paths or [None]:
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
if media_info:
base_filename = title.get_filename(media_info, show_service=not no_source)
if sample:
base_filename = sample_name(base_filename, title)
else:
base_filename = str(title)
sidecar_dir = self.output_dir or config.directories.downloads
if (
not no_folder
and media_info
and (
isinstance(title, (Episode, Song))
or (isinstance(title, Movie) and config.get_folder_template("movies"))
)
):
sidecar_dir /= title.get_filename(
media_info, show_service=not no_source, folder=True
)
sidecar_dir /= sample_dir
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
with console.status("Saving subtitle sidecar files..."):
first_path, first_base, first_dir = sidecar_targets[0]
first_dir.mkdir(parents=True, exist_ok=True)
created = self.output_subtitle_sidecars(
sidecar_subtitles,
first_base,
first_dir,
sidecar_format,
original_paths=sidecar_original_paths or None,
)
if created:
sidecar_files[first_path] = list(created)
self.log.info(f"Saved {len(created)} sidecar subtitle files")
for muxed_path, base_filename, sidecar_dir in sidecar_targets[1:]:
sidecar_dir.mkdir(parents=True, exist_ok=True)
copies = []
for source in created:
target = sidecar_dir / f"{base_filename}{source.name[len(first_base) :]}"
if target != source:
shutil.copy2(source, target)
copies.append(target)
if copies:
sidecar_files[muxed_path] = copies
self.log.info(f"Saved {len(copies)} sidecar subtitle files")
for track in title.tracks:
track.delete()
for attachment in title.tracks.attachments:
attachment.delete()
for temp_path in sidecar_original_paths.values():
temp_path.unlink(missing_ok=True)
finally:
# Hybrid() writes a temp HEVC per resolution, so delete each one here.
for temp_path in hybrid_temp_paths:
try:
temp_path.unlink(missing_ok=True)
except PermissionError:
self.log.warning(f"Failed to delete temp file (in use?): {temp_path}")
for leftover in (
"HDR10-DV.hevc",
"HDR10.hevc",
"DV.hevc",
"RPU.bin",
"RPU_UNT.bin",
"RPU_L5.bin",
"RPU_L6.bin",
"L5.json",
"HDR10Plus.json",
):
leftover_path = config.directories.temp / leftover
try:
leftover_path.unlink(missing_ok=True)
except PermissionError:
self.log.warning(f"Failed to delete temp file (in use?): {leftover_path}")
else:
muxed_paths.extend(track.path for track in title.tracks.audio if track.path and track.path.exists())
if no_mux:
final_dir = self.output_dir or config.directories.downloads
if not no_folder and (
isinstance(title, (Episode, Song))
or (isinstance(title, Movie) and config.get_folder_template("movies"))
):
sample_track = (
title.tracks.videos[0]
if title.tracks.videos
else (
title.tracks.audio[0]
if title.tracks.audio
else (title.tracks.subtitles[0] if title.tracks.subtitles else None)
)
)
if sample_track and sample_track.path:
media_info = MediaInfo.parse(sample_track.path)
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
final_dir /= sample_dir
final_dir.mkdir(parents=True, exist_ok=True)
for track_path in muxed_paths:
media_info = MediaInfo.parse(track_path)
base_filename = title.get_filename(media_info, show_service=not no_source)
if sample:
base_filename = sample_name(base_filename, title)
track = next((t for t in title.tracks if t.path == track_path), None)
if track:
if isinstance(track, Video):
track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}"
elif isinstance(track, Audio):
lang_suffix = f".{track.language}" if track.language else ""
track_suffix = (
f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}"
)
elif isinstance(track, Subtitle):
lang_suffix = f".{track.language}" if track.language else ""
forced_suffix = ".forced" if track.forced else ""
sdh_suffix = ".sdh" if track.sdh else ""
track_suffix = f"{lang_suffix}{forced_suffix}{sdh_suffix}"
else:
track_suffix = ""
final_path = final_dir / f"{base_filename}{track_suffix}{track_path.suffix}"
else:
final_path = final_dir / f"{base_filename}{track_path.suffix}"
shutil.move(track_path, final_path)
self.completed_files.append(final_path)
self.log.debug(f"Saved: {final_path.name}")
else:
used_final_paths: set[Path] = set()
for muxed_path in muxed_paths:
media_info = MediaInfo.parse(muxed_path)
final_dir = self.output_dir or config.directories.downloads
final_filename = title.get_filename(media_info, show_service=not no_source)
if sample:
final_filename = sample_name(final_filename, title)
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
if not no_folder and (
isinstance(title, (Episode, Song))
or (isinstance(title, Movie) and config.get_folder_template("movies"))
):
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
final_dir /= sample_dir
final_dir.mkdir(parents=True, exist_ok=True)
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
sep = config.get_template_separator(template_type(title))
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
if final_path in used_final_paths:
i = 2
while final_path in used_final_paths:
final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}"
i += 1
try:
os.replace(muxed_path, final_path)
except OSError:
if final_path.exists():
final_path.unlink()
shutil.move(muxed_path, final_path)
used_final_paths.add(final_path)
self.completed_files.append(final_path)
if muxed_path not in tagged_paths:
tags.tag_file(
final_path,
title,
self.tmdb_id,
self.imdb_id,
self.tvdb_id,
self.anilist_id,
self.anime_hint(title),
session=service.session,
)
if include_sample:
self.write_sample(final_path, title, media_info)
if isinstance(title, Song) and title.synced_lyrics and title.lyrics_suffix:
lyrics_path = final_path.with_suffix(title.lyrics_suffix)
lyrics_path.write_text(f"{title.synced_lyrics}\n", encoding="utf-8")
sidecar_files.setdefault(muxed_path, []).append(lyrics_path)
self.log.info(f"Saved lyrics to {lyrics_path.name}")
post_script_context = build_context(
title,
media_info,
filepath=final_path,
sidecars=sidecar_files.get(muxed_path, []),
service=self.service,
ids=self.post_script_ids(),
)
dispatch("success", "file", post_script_context, postscript)
post_script_last.setdefault(post_script_group(title), {})[final_path.parent] = (
post_script_context
)
post_script_sample = post_script_context
if final_path.parent not in post_script_folders:
post_script_folders.append(final_path.parent)
if not no_mux:
settle_post_script_group(title)
title_dl_time = time_elapsed_since(dl_start_time)
downloaded_label = "Track" if isinstance(title, Song) else "Title"
console.print(
Padding(
f":tada: {downloaded_label} downloaded in [progress.elapsed]{title_dl_time}[/]!",
(0, 5, 1, 5),
)
)
if progress_sink:
progress_sink({"output_files": [str(p) for p in self.completed_files]})
if not hasattr(service, "close"):
cookie_file = self.get_cookie_path(self.service, self.profile)
if cookie_file:
self.save_cookies(cookie_file, service.session.cookies)
if hasattr(service, "close"):
service.close()
if post_script_folders and post_script_sample:
run_context = dict.fromkeys(post_script_sample, "")
for folder in post_script_folders:
run_context["folder"] = str(folder)
dispatch("success", "run", run_context, postscript)
self.wait_vault_writes()
if export_path:
self.log_export_summary()
dl_time = time_elapsed_since(start_time)
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
def write_sample(self, source: Path, title: Title_T, media_info: MediaInfo) -> None:
"""Cut a 60 second sample from a finished file into a ``Sample`` folder next to it.
The sample starts 2 minutes in. A file shorter than 3 minutes gives its middle 60 seconds.
"""
duration = float(media_info.general_tracks[0].duration or 0) / 1000
start, end = DEFAULT_SAMPLE
length = end - start
if duration < end:
start = max(0.0, (duration - length) / 2)
output = source.parent / SAMPLE_DIR / f"{sample_name(source.stem, title)}{source.suffix}"
output.parent.mkdir(exist_ok=True)
returncode = subprocess.run(
[
str(binaries.MKVToolNix),
"-q",
"-o",
str(output),
"--split",
f"parts:{start:.3f}s-{start + length:.3f}s",
source,
]
).returncode
if returncode > 1 or not output.exists():
self.log.warning(f"mkvmerge wrote no sample for {source.name}")
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
"""Whether metadata lookups for this title should prefer AniList."""
per_title = getattr(title, "anime", None)
return self.service_anime if per_title is None else bool(per_title)
def daily_hint(self, title: Optional[Title_T] = None) -> bool:
"""Whether this title is daily content named by air date."""
per_title = getattr(title, "daily", None)
return self.daily or (self.service_daily if per_title is None else bool(per_title))
def fill_air_dates(self, titles: Series, tvdb_id: Optional[int] = None) -> None:
"""Fill in missing episode air dates from TVDB.
Additive only: an air date the service already set is kept.
"""
tvdb = providers.get_provider("tvdb")
if not tvdb:
self.log.debug("Air dates need a tvdb_api_key in your config, skipping.")
return
if not tvdb_id:
self.log.debug("Could not resolve a TVDB ID, skipping air dates.")
return
wanted = [t for t in titles if self.daily_hint(t)]
if not wanted:
return
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
episodes = tvdb.get_episodes(tvdb_id, source_order) # type: ignore[attr-defined]
if not episodes:
self.log.debug("TVDB listed no episodes for series %s, skipping air dates.", tvdb_id)
return
aired_by_key: dict[tuple[int, int], str] = {}
for episode in episodes:
season, number = parse_int(episode.get("seasonNumber")), parse_int(episode.get("number"))
aired = episode.get("aired")
if season is None or number is None or not aired:
continue
aired_by_key[(season, number)] = str(aired)
# TVDB carries placeholder schedule dates for unaired episodes. Stamping one would
# name the file with a date the episode never aired on.
limit = date.today() + timedelta(days=1)
filled = 0
for title in wanted:
if title.air_date is not None:
continue
aired = aired_by_key.get((title.season, title.number))
if not aired:
continue
try:
air_date = date.fromisoformat(aired[:10])
except ValueError:
continue
if air_date.year < 1970 or air_date > limit:
continue
title.air_date = air_date
filled += 1
if filled:
self.log.info("Filled air dates for %d episode(s) from TVDB (ID %s).", filled, tvdb_id)
def fill_absolute_numbers(self, titles: Series, tvdb_id: Optional[int] = None) -> None:
"""Fill in missing absolute episode numbers from TVDB's absolute order.
Additive only: season and number are never written, and an absolute number the
service already set is kept.
"""
tvdb = providers.get_provider("tvdb")
if not tvdb:
self.log.debug("Absolute episode numbers need a tvdb_api_key in your config, skipping.")
return
if not tvdb_id:
self.log.debug("Could not resolve a TVDB ID, skipping absolute episode numbers.")
return
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
if source_order == "absolute":
# the service already numbers in absolute order; get_order_map would return the
# empty same-order map, but here the episode numbers are their own fill
mapping = {key: (*key, None) for key in keys}
else:
mapping = tvdb.get_order_map(tvdb_id, "absolute", source_order) # type: ignore[attr-defined]
if not mapping:
self.log.debug("TVDB has no absolute order for series %s, skipping absolute episode numbers.", tvdb_id)
return
filled = 0
for title in titles:
if title.absolute is not None:
continue
entry = mapping.get((title.season, title.number))
if not entry:
continue
title.absolute = entry[1]
filled += 1
if filled:
self.log.info("Filled absolute numbers for %d episode(s) from TVDB (ID %s).", filled, tvdb_id)
def apply_tvdb_order(
self,
titles: Series,
title_cacher: Any = None,
cache_title_id: Optional[str] = None,
cache_region: Optional[str] = None,
cache_account_hash: Optional[str] = None,
) -> Series:
"""Renumber episodes from the service's own numbering into a TVDB season order."""
tvdb = providers.get_provider("tvdb")
if not tvdb:
self.log.warning("--tvdb-order needs a tvdb_api_key in your config, skipping renumber.")
return titles
tvdb_id = self.tvdb_id
if not tvdb_id:
show = titles[0]
result = providers.resolve_by_ids(
self.tmdb_id,
self.imdb_id,
None,
title=show.title,
year=show.year,
kind="tv",
title_cacher=title_cacher,
cache_title_id=cache_title_id,
cache_region=cache_region,
cache_account_hash=cache_account_hash,
)
tvdb_id = result.external_ids.tvdb_id if result else None
if not tvdb_id:
self.log.warning("Could not resolve a TVDB ID for %r, skipping %s renumber.", str(titles), self.tvdb_order)
return titles
self.tvdb_id = tvdb_id # keep the mux tag on the series the renumbering used
# parts of one episode share a (season, number) slot; without the dedupe the
# collision check below reads them as several episodes clashing on one slot
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
if source_order == self.tvdb_order:
self.log.info("%s already lists this series in TVDB %s order.", self.service, self.tvdb_order)
return titles
mapping = tvdb.get_order_map(tvdb_id, self.tvdb_order, source_order) # type: ignore[attr-defined]
if not mapping:
return titles
# an unmapped episode keeps its own numbering and can land on a remapped episode's slot
renumbered = [mapping.get(key, (*key, None))[:2] for key in keys]
unmapped = sum(1 for key in keys if key not in mapping)
collisions = [slot for slot, count in Counter(renumbered).items() if count > 1]
if collisions:
self.log.error(
"TVDB's %s order does not cover %d of this series' episodes, and renumbering the rest "
"would give %d season/episode slot(s) two episodes each. Keeping the %s numbering "
"%s uses. Pick an order that covers the whole series.",
self.tvdb_order,
unmapped,
len(collisions),
source_order,
self.service,
)
return titles
for title in titles:
entry = mapping.get((title.season, title.number))
if not entry:
continue
title.season, title.number, tvdb_name = entry
if tvdb_name and not title.name:
title.name = tvdb_name
if unmapped:
self.log.warning(
"%d episode(s) are not in TVDB's %s order and keep their %s numbering.",
unmapped,
self.tvdb_order,
source_order,
)
self.log.info("Renumbered episodes from TVDB %s to %s order (ID %s).", source_order, self.tvdb_order, tvdb_id)
# Series is a SortedKeyList; renumbering in place does not re-sort it
return type(titles)(list(titles))
@staticmethod
def title_to_meta(title: Title_T) -> dict[str, Any]:
"""Capture the title fields needed to rebuild a Title for output naming on import."""
from datetime import date
from unshackle.core.titles import Episode, Movie
meta: dict[str, Any] = {
"id": str(title.id),
"language": str(title.language) if getattr(title, "language", None) else None,
}
if isinstance(title, Episode):
meta.update(
type="episode",
series_title=title.title,
season=title.season,
number=title.number,
name=title.name,
year=title.year,
air_date=title.air_date.isoformat() if isinstance(title.air_date, date) else title.air_date,
)
# conditional, so meta for a title without them is unchanged
if title.part is not None:
meta["part"] = title.part
if title.absolute is not None:
meta["absolute"] = title.absolute
if getattr(title, "daily", None) is not None:
meta["daily"] = title.daily
elif isinstance(title, Movie):
meta.update(type="movie", name=title.name, year=title.year)
else:
meta.update(type="movie", name=str(title))
if getattr(title, "anime", None) is not None:
meta["anime"] = title.anime
return meta
def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any = None) -> None:
"""Write a shareable mediaexport file usable by ``unshackle import`` and unidl.
Carries no HTTP session or cookies. The export records the region (country code)
only when the export used ``--proxy``, as an import geofence. DRM init data and
content keys live once per title. ``drm`` may be None (DRM-free track) or a DRM
system without ``to_dict``/``content_keys`` (e.g. ClearKey) - the export still
records the track, manifest, chapter and attachment info. unshackle's full track
dicts and title meta go under ``x-unshackle`` for its own importer.
"""
namer: Optional[ExportNamer] = getattr(self, "export_namer", None)
with self.EXPORT_LOCK:
if namer and namer.path:
export = namer.path
try:
doc = mediaexport.read(export) if export.is_file() else mediaexport.Document(service_tag=self.service)
except mediaexport.ExportError as e:
# a download mid-licensing must not die over the file it exports to
self.log.warning(f"Not exporting {title.id}: {export} is not a usable export ({e})")
return
doc.generator.setdefault("app", "unshackle")
doc.generator.setdefault("version", __version__)
if not doc.region and getattr(self, "proxy_requested", False):
doc.region = getattr(getattr(self, "export_service", None), "current_region", None) or ""
entry = doc.get(str(title.id))
if entry is None:
entry = self.export_entry(title)
doc.add(entry)
tracks_map = entry.ext("unshackle").setdefault("tracks", {})
track_dict = tracks_map.setdefault(str(track.id), track.to_dict())
unwanted = (track.data.get("hls") or {}).get("unwanted_segments")
if unwanted:
track_dict["unwanted_segments"] = unwanted
elif unwanted is not None:
track_dict.pop("unwanted_segments", None)
# the row names the track's KIDs, so an import gives the track only their keys
track_kids = [kid.hex for kid in own_kids(drm)] if drm is not None else []
for row in entry.tracks:
if row.get("id") == str(track.id):
row["selected"] = True
if track_kids:
row["kids"] = list(dict.fromkeys([*(row.get("kids") or []), *track_kids]))
if drm is not None:
if hasattr(drm, "to_dict"):
d = drm.to_dict()
own = entry.ext("unshackle").setdefault("drm", [])
if d not in own:
own.append(d)
system = str(d.get("system", "")).lower().replace("cenc", "")
pssh = str(d.get("pssh_b64") or "")
if system and all(x.pssh != pssh or x.system != system for x in entry.drm):
entry.drm.append(mediaexport.Drm(system, pssh))
content_keys = getattr(drm, "content_keys", None) or {}
pool = doc.key_pool()
for kid, key in content_keys.items():
# a KID that already holds a different key must not be overwritten:
# the reader rejects a file that disagrees with itself
if pool.setdefault(kid.hex, key.lower()) != key.lower():
self.log.warning(
f"KID {kid.hex} already has a different key in {export}, keeping the existing one"
)
continue
try:
entry.add_key(kid.hex, key)
except mediaexport.ExportError as e:
# a key the format cannot hold (not 16 bytes of hex) costs the export that key,
# never the download
self.log.warning(f"Not exporting the key for KID {kid.hex}: {e}")
try:
mediaexport.write(export, doc)
except (mediaexport.ExportError, OSError) as e:
self.log.warning(f"Could not write export {export}: {e}")
return
if namer:
self.rename_export(namer, export, title, track)
def rename_export(self, namer: ExportNamer, export: Path, title: Title_T, track: AnyTrack) -> None:
"""Give the run's export a name that tells what it holds, after each write to it.
The file on disk is complete after each write, so a run that stops early still leaves
a usable file with a name that matches what it holds.
"""
first = namer.path is None
namer.add(title, track)
try:
namer.path = move_export(export, namer.name())
except OSError as e:
self.log.warning(f"Could not rename export {export}: {e}")
namer.path = export
if first:
self.log.info(f"Exporting to {namer.path.name}")
elif namer.path != export:
self.log.debug(f"Renamed the export to {namer.path.name}")
def log_export_title(self, title: Title_T) -> None:
"""Log what the export holds for one title: its track counts and its keys."""
namer: Optional[ExportNamer] = getattr(self, "export_namer", None)
entry = None
if namer and namer.path:
try:
entry = mediaexport.read(namer.path).get(str(title.id))
except (mediaexport.ExportError, OSError) as e:
self.log.warning(f"Could not read export {namer.path}: {e}")
return
if entry is None:
self.log.warning(f"Exported nothing for {title_label(title)}")
return
selected = [r.get("type") for r in entry.tracks if r.get("selected")]
self.log.info(
f"Exported {title_label(title)}: {selected.count('video')} video, {selected.count('audio')} audio, "
f"{plural(selected.count('subtitle'), 'subtitle')}, {plural(len(entry.keys), 'key')}"
)
def log_export_summary(self) -> None:
"""Log where the run's export is and what it holds, and that it must stay private."""
namer: Optional[ExportNamer] = getattr(self, "export_namer", None)
if not namer or not namer.path:
self.log.warning("Nothing was exported")
return
try:
doc = mediaexport.read(namer.path)
except (mediaexport.ExportError, OSError) as e:
self.log.warning(f"Could not read export {namer.path}: {e}")
return
keys = len(doc.key_pool())
self.log.info(f"Saved the export to {namer.path}: {plural(len(doc.titles), 'title')}, {plural(keys, 'key')}")
if keys:
self.log.warning("The export holds content keys. Keep it private.")
def export_entry(self, title: Title_T) -> mediaexport.Entry:
"""A mediaexport entry for ``title``: manifests, chapters, a small track list, no keys yet."""
meta = self.title_to_meta(title)
all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]
primary_url = title.tracks.manifest_url or next(
(str(t.url) for t in all_tracks if t.descriptor != Video.Descriptor.URL), ""
)
manifest_type = next(
(t.descriptor.name.lower() for t in all_tracks if t.descriptor != Video.Descriptor.URL), ""
)
session = getattr(getattr(self, "export_service", None), "session", None)
# mediaexport drops Cookie and Authorization itself, on write and on read
headers = dict(getattr(session, "headers", None) or {})
manifests = [mediaexport.Manifest(primary_url, manifest_type, headers, role="primary")] if primary_url else []
# the whole URL identifies a manifest: two profiles on one endpoint are two manifests
seen = {primary_url}
for t in all_tracks:
url = str(t.url)
if t.descriptor in (Video.Descriptor.DASH, Video.Descriptor.ISM) and url not in seen:
seen.add(url)
manifests.append(mediaexport.Manifest(url, t.descriptor.name.lower(), role="extra"))
rows = []
for t in all_tracks:
codec = getattr(t, "codec", None)
row: dict[str, Any] = {
"id": str(t.id),
"type": t.__class__.__name__.lower(),
# a subtitle side-load names its file format so another tool can fetch it
"codec": (codec.value if isinstance(t, Subtitle) else codec.name).lower() if codec else "",
"language": str(t.language) if t.language else "",
"bitrate": int(bitrate) if (bitrate := getattr(t, "bitrate", None)) else None,
}
if isinstance(t, Video):
row.update(width=t.width, height=t.height, range=t.range.name.lower() if t.range else "")
elif isinstance(t, Audio):
row.update(channels=str(t.channels) if t.channels else "", atmos=bool(t.joc), descriptive=t.descriptive)
elif isinstance(t, Subtitle):
row.update(sdh=t.sdh, forced=t.forced, cc=t.cc)
if t.descriptor == Video.Descriptor.URL:
row["url"] = str(t.url)
rows.append({k: v for k, v in row.items() if v not in (None, "")})
return mediaexport.Entry(
id=str(title.id),
kind=meta.get("type", "movie"),
title=meta.get("name") or "",
series=meta.get("series_title") or "",
season=meta.get("season"),
episode=meta.get("number"),
year=meta.get("year"),
language=meta.get("language") or "",
artist=meta.get("artist") or "",
album=meta.get("album") or "",
track_number=meta.get("track"),
manifests=manifests,
chapters=[
{"start_ms": mediaexport.ts_ms(c.timestamp), "title": c.name or ""}
for c in (title.tracks.chapters or [])
],
tracks=rows,
extensions={
"x-unshackle": {
"meta": meta,
"tracks": {},
"attachments": [a.to_dict() for a in (title.tracks.attachments or []) if a.url],
}
},
)
@staticmethod
def failure_lines(e: BaseException) -> list[str]:
"""Return the lines that report a failed download.
A ClickException is a deliberate stop with a message for the user, so it gets no
"unexpected error" line, and the caller prints no traceback for it.
"""
if isinstance(e, click.ClickException):
return [":x: Download Failed...", f" {e.format_message()}"]
lines = [":x: Download Failed...", f" {type(e).__name__}: {e}"]
if (returncode := getattr(e, "returncode", None)) is not None:
lines.append(f" Binary call failed, Process exit code: {returncode}")
lines.append(" An unexpected error occurred in one of the download workers.")
return lines
def decrypt_verified(
self,
drm: DRM_T,
path: Path,
licence: Optional[Callable] = None,
track_kid: Optional[UUID] = None,
decrypt: bool = True,
) -> None:
"""Decrypt the file, and when a vault supplied a content key, prove the output decodes.
A poisoned vault returns the right KID with a wrong content key, and the decrypters
accept it without complaint. A content key from a vault is therefore trusted, and only
then copied to the other vaults, once FFmpeg can decode the result. Without FFmpeg
neither happens. For a fragmented MP4, ``verify.kid_windows`` reads from the ciphertext
which fragments each KID encrypts, and FFmpeg decodes windows inside each KID's
fragments, so only a KID that fails is flagged. A complete map does not judge a KID it does
not hold, so a clear file judges and copies no KID. A KID that is not in an incomplete map,
and every KID of any other file, gets a start and end check of the whole file, and a
failure there flags each of those KIDs. On failure the pair is flagged in the local
vaults, the ciphertext is restored, and the licence runs again; ``Vaults`` tells the
vault that served it. Every vault skips a flagged pair, so each pass burns at most one
vault; when the vaults run out the CDM answers, and a CDM key is not checked. A CDM
that returns the flagged key clears the flag and stores the content key again, because
then the decode check was wrong.
With ``decrypt=False`` the downloader already decrypted the segments in place, so a
failure can only flag and raise; the next run skips the flagged pair.
"""
keys = getattr(drm, "content_keys", None)
if keys is None:
if decrypt:
drm.decrypt(path)
return
def flagged_kids() -> list[UUID]:
"""The KIDs on this track whose content key another track already proved wrong."""
return [kid for kid, key in keys.items() if (kid, key) in self.vaults.flagged]
def drop(kid: UUID) -> str:
key = keys.pop(kid)
if self.LICENSE_KEY_CACHE.get(kid) == key:
self.LICENSE_KEY_CACHE.pop(kid)
return key
def relicense(dropped: set[UUID]) -> None:
"""Licence again, then ask for each dropped KID that is still without a content key.
prepare_drm licenses only for a KID it is told to need, so without the second call a
track that still holds other content keys would decrypt without the dropped one.
"""
if not licence:
return
licence(drm, track_kid=track_kid)
for kid in sorted(dropped):
# prepare_drm can replace the dict, so read it from the DRM each time
if kid != track_kid and kid not in getattr(drm, "content_keys", {}):
licence(drm, track_kid=kid)
known_bad = flagged_kids()
if known_bad and licence:
for kid in known_bad:
drop(kid)
relicense(set(known_bad))
def vault_kids() -> dict[UUID, Vault]:
"""The KIDs whose key on the track came from a vault, with that vault."""
found = {kid: self.vaults.sources.get(kid) for kid in keys}
return {kid: src[1] for kid, src in found.items() if src and src[0] == keys[kid]}
backup = path.with_name(path.name + ".enc")
backup.unlink(missing_ok=True) # a killed run leaves one behind; its bytes may not match
checking = bool(vault_kids() and decrypt and binaries.FFMPEG)
if checking:
try:
os.link(path, backup)
except OSError as e:
self.log.debug(f"Cannot keep the ciphertext for a decrypt retry: {e!r}")
kid_map = verify.kid_windows(path) if checking else None
windows = kid_map.windows if kid_map else {}
video = kid_map.video if kid_map else None
def window_decodes(start: float, end: float) -> bool:
return ffmpeg_decodes(path, start=start, seconds=end - start if end > start else 0.5, video=video)
def judged_kids() -> dict[UUID, Vault]:
"""The vault KIDs this file can judge. A map whose KIDs all have keys skips the KIDs it lacks."""
kids = vault_kids()
if kid_map is not None and all(kid in keys for kid in windows):
return {kid: vault for kid, vault in kids.items() if kid in windows}
return kids
def failed_kids(kids: dict[UUID, Vault]) -> set[UUID]:
"""The vault KIDs whose content key did not decode.
A KID in the KID map fails when one of its windows fails. The KIDs that are not in the
map share the verdict of one start and end check of the whole file, taken only when
every mapped KID passed: a wrong mapped key fails that check too and would flag them
for noise that is not theirs. The retry judges them again with the replaced key.
"""
failed = {kid for kid in kids if kid in windows and not all(window_decodes(*w) for w in windows[kid])}
absent = {kid for kid in kids if kid not in windows}
if absent and not failed and not ffmpeg_decodes(path, video=video):
failed |= absent
self.log.debug(
f"Key check on {path.name}: {len(kids) - len(absent)} KID(s) by fragment window, "
f"{len(absent)} by whole-file check, failed {sorted(k.hex for k in failed)}"
)
return failed
def passes_left(done: int) -> bool:
"""One pass per source that can still answer: every loaded vault, the server's vaults as
one source that walks them itself over --remote, and one for the CDM."""
remote = 1 if getattr(getattr(self, "_remote_service", None), "_server_cdm", False) else 0
return done < len(self.vaults) + remote + 1
try:
done = 0
while passes_left(done):
done += 1
if decrypt:
drm.decrypt(path)
kids = judged_kids()
stale = flagged_kids()
failed: set[UUID] = set()
if not stale:
checked = bool(kids and binaries.FFMPEG and path.exists())
failed = failed_kids(kids) if checked else set()
if not failed:
if checked:
self.flush_vault_writes(
[
partial(self.vaults.add_key, kid, keys[kid], excluding=vault)
for kid, vault in kids.items()
]
)
self.vaults.report_replaced(keys)
return
bad = {kid: drop(kid) for kid in {*failed, *stale}}
for kid, key in bad.items():
if kid not in kids:
continue
self.log.warning(f"{key} from {kids[kid].name} was bad, trying other vaults")
self.vaults.flag_bad_key(kid, key)
if not backup.exists() or not licence:
raise ValueError("The content key from the vault did not decrypt the track; run again")
path.unlink()
os.link(backup, path)
relicense(set(bad))
for kid, key in bad.items():
new = keys.get(kid)
if new is None:
continue
source = self.vaults.sources.get(kid)
if source is None:
if new == key:
self.vaults.unflag_bad_key(kid, key)
self.flush_vault_writes([partial(self.vaults.add_key, kid, key)])
elif not self.vaults.held_locally(kid, new):
self.flush_vault_writes([partial(self.vaults.add_key, kid, new)])
elif new == key and kid in kids and source[1] is kids[kid]:
raise ValueError(f"{key} from {kids[kid].name} was bad and no other source has the key")
raise ValueError("No vault or CDM produced a content key that decrypts the track")
finally:
backup.unlink(missing_ok=True)
def flush_vault_writes(self, pending: list[Callable[[], Any]]) -> None:
"""Hand the queued vault writes to the background writer so the track does not wait on them."""
for write in pending:
self.VAULT_WRITER.submit(write).add_done_callback(self.report_vault_write)
pending.clear()
def report_vault_write(self, future: futures.Future[Any]) -> None:
if exc := future.exception():
self.log.error(f"Vault write failed: {exc!r}")
def wait_vault_writes(self) -> None:
"""Block until every queued vault write has run, then log the total once."""
self.VAULT_WRITER.submit(lambda: None).result()
if not (self.vault_cache_tally and self.vault_cache_tally[0]):
self.vault_cache_tally = None
return
kids, successful_caches = self.vault_cache_tally
keys = len(kids)
self.vault_cache_tally = None
self.log.info(f"Cached {keys} Key{'' if keys == 1 else 's'} to {successful_caches}/{len(self.vaults)} Vaults")
def prefer_vault_keys(self, title: Title_T) -> None:
"""Let the client vaults answer before a content key the server batch licence returned.
The client only learns the KIDs from the batch response, so the server has already
answered. A vault key still goes first, so ``decrypt_verified`` proves it and flags a
poisoned row; the server key waits as the next candidate. A server CDM key is trusted
like a local CDM key: it goes to the run cache and the vaults, and a retry after a
flagged vault key takes it before any vault. A server vault key is unproven, so it
waits in ``Vaults.candidates`` and gets the same check when its turn comes. Only the KIDs
the DRM names are looked up, each once, so sibling keys add no remote vault requests.
"""
server_vault_keys = getattr(self._remote_service, "server_vault_keys", {})
looked_up: dict[UUID, tuple[Optional[str], Any]] = {}
for track in title.tracks:
for drm in getattr(track, "drm", None) or []:
content_keys = getattr(drm, "content_keys", {})
named = set(real_kids(drm) or content_keys)
for kid, server_key in list(content_keys.items()):
if kid not in named:
continue
if kid not in looked_up:
looked_up[kid] = self.vaults.get_key(kid)
vault_key, vault = looked_up[kid]
if not vault_key or vault_key == server_key:
continue
drm.content_keys[kid] = vault_key
if server_vault_keys.get(kid) == server_key:
source = self._remote_service.server_vault
self.vaults.candidates.setdefault(kid, []).append((server_key, source))
elif kid not in self.LICENSE_KEY_CACHE:
self.LICENSE_KEY_CACHE[kid] = server_key
self.flush_vault_writes([partial(self.cache_keys_to_vaults, {kid: server_key})])
self.log.debug(
f"{vault.name} holds a different key for {kid.hex} than the server, testing it first"
)
def cache_resolved_keys(self, title: Title_T) -> None:
"""Cache the keys a batch licence or an import put on the tracks.
Both fill track.drm before any prepare_drm call, so this is the only point where those
keys can still reach the local vaults. A content key a local vault already holds is not pushed again.
"""
server_vault_keys = getattr(getattr(self, "_remote_service", None), "server_vault_keys", {})
resolved = {
kid: key
for track in title.tracks
for drm in (getattr(track, "drm", None) or [])
for kid, key in getattr(drm, "content_keys", {}).items()
if kid not in self.LICENSE_KEY_CACHE
and server_vault_keys.get(kid) != key
and self.vaults.sources.get(kid, (None,))[0] != key
}
keys = {kid: key for kid, key in resolved.items() if not self.vaults.held_locally(kid, key)}
if keys:
self.LICENSE_KEY_CACHE.update(keys)
self.flush_vault_writes([partial(self.cache_keys_to_vaults, keys)])
def cache_keys_to_vaults(self, content_keys: dict[UUID, str]) -> None:
"""Store licence keys in every vault. A licence key clears a flag on its pair first: the
flag came from a decode check that was wrong, and a local vault refuses a flagged pair."""
if not content_keys or not self.vaults:
return
self.vaults.clear_licensed_flags(content_keys)
successful_caches = self.vaults.add_keys(content_keys)
kids, caches = self.vault_cache_tally or (set(), successful_caches)
self.vault_cache_tally = (kids | content_keys.keys(), min(caches, successful_caches))
@classmethod
def drm_lock(cls, drm: DRM_T) -> Lock:
"""Return the lock for one DRM's set of content keys.
Tracks that share KIDs still take turns, so the second one finds the content keys in
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs send a challenge
and query the key vaults at the same time.
Callers take the lock after get_drm_for_cdm(), which folds a track's sibling PlayReady
objects into the returned one, so the KID set is complete for that track's DRM list
before the lock is chosen. This holds because every manifest parser gives the tracks
that share a PlayReady object the same DRM list (ISM attaches one list to all tracks;
DASH and HLS build a list per track). Two tracks that share one object but list
different siblings would get different locks.
"""
kids = getattr(drm, "kids", None) or []
key = ",".join(sorted(getattr(k, "hex", str(k)) for k in kids)) or getattr(drm, "content_id", None)
with cls.DRM_LOCKS_GUARD:
return cls.DRM_LOCKS.setdefault(key or type(drm).__name__, Lock())
def service_licence(self, service: Service, drm_system: Optional[str] = None, **kwargs: Any) -> Any:
"""Call the service's licence function for the track's DRM system.
Arguments go through declared_kwargs, so a service receives only the ones its own
licence function declares.
prepare_drm passes drm_system so the choice follows the track, not the loaded CDM:
a track downloading at the same time can swap self.cdm.
"""
if drm_system is None:
drm_system = "playready" if is_playready_cdm(self.cdm) else "widevine"
fn = service.get_playready_license if drm_system == "playready" else service.get_widevine_license
return fn(**declared_kwargs(fn, kwargs))
def cdm_for(self, system: str, quality: Optional[int] = None) -> Optional[object]:
"""Get the CDM for one DRM system, ``widevine`` or ``playready``.
A quality key that names one device applies to both DRM systems. When that device is for
the other system, the ``widevine`` or ``playready`` config key selects the device.
"""
matches = is_playready_cdm if system == "playready" else is_widevine_cdm
cdm = self.get_cdm(self.service, self.profile, drm=system, quality=quality)
if quality and not (cdm and matches(cdm)):
cdm = self.get_cdm(self.service, self.profile, drm=system)
return cdm
def licence_one_drm(self, prepare: Callable, drm: DRM_T, track_kid: Optional[UUID]) -> None:
"""Send one challenge for the KIDs of this DRM, also when the DRM or a vault has the content keys.
This is the ``--cdm-only`` path. The challenge goes out the first time a run sees the KIDs
for the DRM system.
prepare_drm asks the CDM only for a KID with no content key, so the content keys the DRM
holds are removed for the call and put back after it.
"""
kids = {*drm.kids, *([track_kid] if track_kid else [])}
attempted = self.all_drm_attempted[type(drm)]
if kids <= attempted:
prepare(drm, track_kid=track_kid)
return
with self.drm_lock(drm):
held = dict(drm.content_keys)
drm.content_keys.clear()
try:
prepare(drm, track_kid=track_kid, cdm_only=True, force=True)
finally:
attempted |= kids
with self.drm_lock(drm):
for held_kid, key in held.items():
drm.content_keys.setdefault(held_kid, key)
def licence_all_drm(
self,
prepare: Callable,
track: AnyTrack,
tolerate: bool,
drm: DRM_T,
track_kid: Optional[UUID] = None,
force: bool = False,
) -> None:
"""License the track with each DRM system it offers, not only the one the downloader selected.
A DRM system sends a challenge only for a KID that no vault and no earlier track gave a
content key for. With ``force``, each DRM system sends one challenge for a set of KIDs in
a run. A DRM system that fails stops the title. With ``tolerate`` it is a warning, if the
other DRM system gave the content key of the track.
"""
if type(drm) not in (Widevine, PlayReady):
prepare(drm, track_kid=track_kid)
return
other_type = PlayReady if type(drm) is Widevine else Widevine
others = [d for d in track.drm or [] if type(d) is other_type]
if other_type is PlayReady and len(others) > 1:
others[0].absorb(*others[1:])
if not others and other_type.__name__ not in self.all_drm_warned:
self.all_drm_warned.add(other_type.__name__)
self.log.warning(
f"A track does not offer {other_type.__name__}, so it licenses with "
f"{type(drm).__name__} only. This warning shows one time: {track}"
)
targets = [drm, *others[:1]]
error: Optional[Exception] = None
with self.ALL_DRM_LOCK:
loaded_cdm = self.cdm
try:
for target in targets:
kid = track_kid if target is drm or track_kid in target.kids else None
try:
if force:
self.licence_one_drm(prepare, target, kid)
else:
prepare(target, track_kid=kid)
except Exception as e:
if not tolerate:
raise
self.log.warning(f"{type(target).__name__} licence failed for {track}: {e}")
error = error or e
keys = {kid: key for target in targets for kid, key in target.content_keys.items()}
for target in targets:
with self.drm_lock(target):
for kid, key in keys.items():
target.content_keys.setdefault(kid, key)
finally:
self.cdm = loaded_cdm
track_keys = getattr(drm, "content_keys", {})
if error and (not track_keys or (track_kid and track_kid not in track_keys)):
raise error
def prepare_drm(
self,
drm: DRM_T,
track: AnyTrack,
title: Title_T,
certificate: Callable,
licence: Callable,
clearkey_licence: Optional[Callable] = None,
track_kid: Optional[UUID] = None,
table: Table = None,
cdm_only: bool = False,
vaults_only: bool = False,
export: Optional[Path] = None,
service_session: Optional[Any] = None,
force: bool = False,
) -> None:
"""
Prepare the DRM by getting decryption data like KIDs, Keys, and such.
The DRM object should be ready for decryption once this function ends.
``force`` skips the content keys of earlier tracks in this run. The CDM then licenses
again for each KID the DRM holds no content key for.
"""
if not drm:
return
svc_for_cdm = getattr(self, "_remote_service", None)
server_cdm = getattr(svc_for_cdm, "_server_cdm", getattr(self, "server_cdm", False))
client_licensed = getattr(svc_for_cdm, "client_licensed", ())
licensed_locally = bool(client_licensed) and str(track.id) in client_licensed
if server_cdm and not licensed_locally:
with self.drm_lock(drm):
pending_vault_writes: list[Callable[[], Any]] = []
vault_kids = list(getattr(drm, "kids", None) or [])
if track_kid and track_kid not in vault_kids:
vault_kids.append(track_kid)
for kid in vault_kids:
if kid in drm.content_keys:
continue
content_key = self.LICENSE_KEY_CACHE.get(kid)
if not content_key and not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
self.LICENSE_KEY_CACHE[kid] = content_key
known_keys = set(drm.content_keys)
def missing_track_key() -> bool:
return not drm.content_keys or bool(track_kid and track_kid not in drm.content_keys)
if missing_track_key():
from unshackle.core.remote_service import RemoteError
try:
licence(
drm_system="playready" if drm.__class__.__name__ == "PlayReady" else "widevine",
challenge=b"",
)
except click.ClickException:
raise
except (Exception, RemoteError) as e:
self.log.debug(f"Server CDM licence with an empty challenge failed: {e!r}")
server_vault_keys = {} if cdm_only else getattr(svc_for_cdm, "server_vault_keys", {})
for kid, key in drm.content_keys.items():
if server_vault_keys.get(kid) == key:
self.vaults.sources[kid] = (key, svc_for_cdm.server_vault)
new_keys = {
kid: key
for kid, key in drm.content_keys.items()
if kid not in known_keys and server_vault_keys.get(kid) != key
}
if new_keys:
self.LICENSE_KEY_CACHE.update(new_keys)
pending_vault_writes.append(partial(self.cache_keys_to_vaults, new_keys))
self.flush_vault_writes(pending_vault_writes)
# the server can refuse the licence above its cap and hand the track to this machine
licensed_locally = bool(client_licensed) and str(track.id) in client_licensed
if server_cdm and not licensed_locally:
if not drm.content_keys:
self.log.warning("Server CDM did not resolve any keys for this track")
return
if track_kid and track_kid not in drm.content_keys:
msg = f"No Content Key for KID {track_kid.hex} was returned by the server CDM"
if isinstance(drm, PlayReady):
raise PlayReady.Exceptions.CEKNotFound(msg)
raise Widevine.Exceptions.CEKNotFound(msg)
svc = getattr(self, "_remote_service", None)
server_drm_type = getattr(svc, "_server_cdm_type", None) if svc else None
drm_name = {"widevine": "Widevine", "playready": "PlayReady"}.get(
server_drm_type or "", drm.__class__.__name__
)
log_event(
"drm_content_keys",
level="INFO",
message=f"Server CDM resolved {len(drm.content_keys)} {drm_name} content key(s)",
service=self.service,
drm_type=drm_name,
key_count=len(drm.content_keys),
keys=[{"kid": k.hex, "key": v} for k, v in drm.content_keys.items()],
remote=True,
)
with self.drm_lock(drm):
pssh_str = ""
expected_class = "PlayReady" if server_drm_type == "playready" else "Widevine"
matching_drm = next(
(d for d in (track.drm or []) if d.__class__.__name__ == expected_class),
drm,
)
if hasattr(matching_drm, "pssh") and matching_drm.pssh:
if hasattr(matching_drm.pssh, "dumps"):
pssh_str = self.truncate_pssh_for_display(matching_drm.pssh.dumps(), drm_name)
elif hasattr(matching_drm, "data") and matching_drm.data.get("pssh_b64"):
pssh_str = self.truncate_pssh_for_display(matching_drm.data["pssh_b64"], drm_name)
if pssh_str:
cek_tree = Tree(Text.assemble((drm_name, "cyan"), (f"({pssh_str})", "text"), overflow="fold"))
else:
cek_tree = Tree(Text.assemble((drm_name, "cyan"), overflow="fold"))
all_kids = list(getattr(drm, "kids", []))
if track_kid and track_kid not in all_kids:
all_kids.append(track_kid)
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
cek_tree.add(f"[text2]{kid.hex}:{key}{is_track_kid}")
for kid, key in drm.content_keys.items():
if kid not in all_kids:
cek_tree.add(f"[text2]{kid.hex}:{key}")
if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells):
table.add_row(cek_tree)
if export:
self.write_export(export, title, track, drm)
return
track_quality = None
if isinstance(track, Video) and track.height:
track_quality = track.height
elif title is not None:
track_quality = max((v.height for v in title.tracks.videos if v.height), default=None)
track_cdm = self.cdm
if licensed_locally:
track_cdm = getattr(svc_for_cdm, "local_cdm", None)
local_drm = track.get_drm_for_cdm(track_cdm)
if local_drm is not None and local_drm is not drm:
# a mid-download handover leaves drm on the server's DRM system, and the caller decrypts with it
self.prepare_drm(
local_drm,
track,
title,
certificate,
licence,
clearkey_licence,
track_kid,
table,
cdm_only,
vaults_only,
export,
service_session,
)
with self.drm_lock(drm):
drm.content_keys.update(local_drm.content_keys)
return
drm_system = "playready" if isinstance(drm, PlayReady) else "widevine"
licence = partial(licence, drm_system=drm_system)
if not server_cdm and isinstance(drm, (Widevine, PlayReady)):
matches = is_playready_cdm if drm_system == "playready" else is_widevine_cdm
if not matches(self.cdm):
name, other = ("PlayReady", "Widevine") if drm_system == "playready" else ("Widevine", "PlayReady")
system_cdm = self.cdm_for(drm_system, track_quality)
if system_cdm and matches(system_cdm):
self.log.info(f"Switching to the {name} CDM")
self.cdm = system_cdm
track_cdm = system_cdm
elif self.cdm is None:
raise click.ClickException(f"No CDM configured for {self.service} and no default CDM")
else:
raise ValueError(f"Title needs a {name} CDM but {self.service} is configured with {other}.")
if isinstance(drm, Widevine):
if self.debug_logger:
self.debug_logger.log_drm_operation(
drm_type="Widevine",
operation="prepare_drm",
service=self.service,
context={
"track": str(track),
"title": str(title),
"pssh": drm.pssh.dumps() if drm.pssh else None,
"kids": [k.hex for k in drm.kids],
"track_kid": track_kid.hex if track_kid else None,
},
)
pending_vault_writes: list[Callable[[], Any]] = []
with self.drm_lock(drm):
pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine")
cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
pre_existing_tree = next(
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
)
if pre_existing_tree:
cek_tree = pre_existing_tree
need_license = False
all_kids = list(drm.kids)
if track_kid and track_kid not in all_kids:
all_kids.append(track_kid)
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
continue
is_track_kid = ["", "*"][kid == track_kid]
cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid)
if cached_key:
drm.content_keys[kid] = cached_key
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={
"kid": kid.hex,
"content_key": cached_key,
"track": str(track),
"drm_type": "Widevine",
},
)
continue
if not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
self.LICENSE_KEY_CACHE[kid] = content_key
if self.debug_logger:
self.debug_logger.log_vault_query(
vault_name=vault_used,
operation="get_key_success",
service=self.service,
context={
"kid": kid.hex,
"content_key": content_key,
"track": str(track),
"from_cache": True,
},
)
elif vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
log_event(
"vault_key_not_found",
level="ERROR",
service=self.service,
message=msg,
context={"kid": kid.hex, "track": str(track)},
)
self.flush_vault_writes(pending_vault_writes)
raise Widevine.Exceptions.CEKNotFound(msg)
else:
need_license = True
if kid not in drm.content_keys and cdm_only:
need_license = True
if need_license and all(kid in drm.content_keys for kid in all_kids):
need_license = False
if need_license and not vaults_only:
from_vaults = drm.content_keys.copy()
log_event(
"get_license",
level="INFO",
service=self.service,
message="Requesting Widevine license from service",
context={
"track": str(track),
"kids_needed": [k.hex for k in all_kids if k not in drm.content_keys],
},
)
try:
drm.get_content_keys(cdm=track_cdm, licence=licence, certificate=certificate)
except Exception as e:
if drm.content_keys:
self.log.debug(f"License call failed but keys already in content_keys: {e}")
else:
if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)):
msg = str(e)
else:
msg = f"An exception occurred in the Service's license function: {e}"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
if self.debug_logger:
self.debug_logger.log_error(
"get_license",
e,
service=self.service,
context={"track": str(track), "exception_type": type(e).__name__},
)
self.flush_vault_writes(pending_vault_writes)
raise e
log_event(
"license_keys_retrieved",
level="INFO",
service=self.service,
context={
"track": str(track),
"keys_count": len(drm.content_keys),
"kids": [k.hex for k in drm.content_keys.keys()],
},
)
for kid_, key in drm.content_keys.items():
if key == "0" * 32:
key = f"[red]{key}[/]"
is_track_kid_marker = ["", "*"][kid_ == track_kid]
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
drm.content_keys = {
kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key)
}
# The CDM keys may have returned blank content keys for KIDs we got from vaults.
# So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data.
drm.content_keys.update(from_vaults)
self.LICENSE_KEY_CACHE.update(drm.content_keys)
pending_vault_writes.append(
partial(
self.cache_keys_to_vaults,
{k: v for k, v in drm.content_keys.items() if k not in from_vaults},
)
)
if track_kid and track_kid not in drm.content_keys:
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
self.flush_vault_writes(pending_vault_writes)
raise Widevine.Exceptions.CEKNotFound(msg)
if cek_tree.children and not pre_existing_tree:
table.add_row()
table.add_row(cek_tree)
self.flush_vault_writes(pending_vault_writes)
if export:
self.write_export(export, title, track, drm)
elif isinstance(drm, PlayReady):
if self.debug_logger:
self.debug_logger.log_drm_operation(
drm_type="PlayReady",
operation="prepare_drm",
service=self.service,
context={
"track": str(track),
"title": str(title),
"pssh": drm.pssh_b64 or "",
"kids": [k.hex for k in drm.kids],
"track_kid": track_kid.hex if track_kid else None,
},
)
pending_vault_writes = []
with self.drm_lock(drm):
pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady")
cek_tree = Tree(
Text.assemble(
("PlayReady", "cyan"),
(f"({pssh_display})", "text"),
overflow="fold",
)
)
pre_existing_tree = next(
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
)
if pre_existing_tree:
cek_tree = pre_existing_tree
need_license = False
all_kids = list(drm.kids)
if track_kid and track_kid not in all_kids:
all_kids.append(track_kid)
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
continue
is_track_kid = ["", "*"][kid == track_kid]
cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid)
if cached_key:
drm.content_keys[kid] = cached_key
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={
"kid": kid.hex,
"content_key": cached_key,
"track": str(track),
"drm_type": "PlayReady",
},
)
continue
if not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
self.LICENSE_KEY_CACHE[kid] = content_key
if self.debug_logger:
self.debug_logger.log_vault_query(
vault_name=vault_used,
operation="get_key_success",
service=self.service,
context={
"kid": kid.hex,
"content_key": content_key,
"track": str(track),
"from_cache": True,
"drm_type": "PlayReady",
},
)
elif vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
log_event(
"vault_key_not_found",
level="ERROR",
service=self.service,
message=msg,
context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"},
)
self.flush_vault_writes(pending_vault_writes)
raise PlayReady.Exceptions.CEKNotFound(msg)
else:
need_license = True
if kid not in drm.content_keys and cdm_only:
need_license = True
if need_license and all(kid in drm.content_keys for kid in all_kids):
need_license = False
if need_license and not vaults_only:
from_vaults = drm.content_keys.copy()
try:
drm.get_content_keys(cdm=track_cdm, licence=licence, certificate=certificate)
except Exception as e:
if drm.content_keys:
self.log.debug(f"License call failed but keys already in content_keys: {e}")
else:
if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)):
msg = str(e)
else:
msg = f"An exception occurred in the Service's license function: {e}"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
if self.debug_logger:
self.debug_logger.log_error(
"get_license_playready",
e,
service=self.service,
context={
"track": str(track),
"exception_type": type(e).__name__,
"drm_type": "PlayReady",
},
)
self.flush_vault_writes(pending_vault_writes)
raise e
for kid_, key in drm.content_keys.items():
is_track_kid_marker = ["", "*"][kid_ == track_kid]
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
drm.content_keys.update(from_vaults)
self.LICENSE_KEY_CACHE.update(drm.content_keys)
pending_vault_writes.append(
partial(
self.cache_keys_to_vaults,
{k: v for k, v in drm.content_keys.items() if k not in from_vaults},
)
)
if track_kid and track_kid not in drm.content_keys:
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
self.flush_vault_writes(pending_vault_writes)
raise PlayReady.Exceptions.CEKNotFound(msg)
if cek_tree.children and not pre_existing_tree:
table.add_row()
table.add_row(cek_tree)
self.flush_vault_writes(pending_vault_writes)
if export:
self.write_export(export, title, track, drm)
elif isinstance(drm, ClearKeyCENC):
pending_vault_writes = []
with self.drm_lock(drm):
cek_tree = Tree(Text.assemble(("ClearKey", "cyan"), overflow="fold"))
pre_existing_tree = next(
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
)
if pre_existing_tree:
cek_tree = pre_existing_tree
need_license = False
all_kids = list(drm.kids)
if track_kid and track_kid not in all_kids:
all_kids.append(track_kid)
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
continue
is_track_kid = ["", "*"][kid == track_kid]
cached_key = self.LICENSE_KEY_CACHE.get(kid)
if cached_key:
drm.content_keys[kid] = cached_key
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={
"kid": kid.hex,
"content_key": cached_key,
"track": str(track),
"drm_type": "ClearKeyCENC",
},
)
continue
if not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
self.LICENSE_KEY_CACHE[kid] = content_key
elif vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
log_event(
"vault_key_not_found",
level="ERROR",
service=self.service,
message=msg,
context={"kid": kid.hex, "track": str(track), "drm_type": "ClearKeyCENC"},
)
self.flush_vault_writes(pending_vault_writes)
raise ClearKeyCENC.Exceptions.CEKNotFound(msg)
else:
need_license = True
if kid not in drm.content_keys and cdm_only:
need_license = True
if need_license and all(kid in drm.content_keys for kid in all_kids):
need_license = False
if need_license and not vaults_only:
from_vaults = drm.content_keys.copy()
try:
drm.get_content_keys(licence=clearkey_licence or (lambda **_: None), session=service_session)
except Exception as e:
if drm.content_keys:
self.log.debug(f"License call failed but keys already in content_keys: {e}")
else:
if isinstance(
e, (ClearKeyCENC.Exceptions.EmptyLicense, ClearKeyCENC.Exceptions.CEKNotFound)
):
msg = str(e)
else:
msg = f"An exception occurred in the Service's license function: {e}"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
if self.debug_logger:
self.debug_logger.log_error(
"get_license_clearkey",
e,
service=self.service,
context={
"track": str(track),
"exception_type": type(e).__name__,
"drm_type": "ClearKeyCENC",
},
)
self.flush_vault_writes(pending_vault_writes)
raise e
for kid_, key in drm.content_keys.items():
is_track_kid_marker = ["", "*"][kid_ == track_kid]
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
drm.content_keys.update(from_vaults)
self.LICENSE_KEY_CACHE.update(drm.content_keys)
pending_vault_writes.append(
partial(
self.cache_keys_to_vaults,
{k: v for k, v in drm.content_keys.items() if k not in from_vaults},
)
)
if track_kid and track_kid not in drm.content_keys:
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
table.add_row(cek_tree)
self.flush_vault_writes(pending_vault_writes)
raise ClearKeyCENC.Exceptions.CEKNotFound(msg)
if cek_tree.children and not pre_existing_tree:
table.add_row()
table.add_row(cek_tree)
self.flush_vault_writes(pending_vault_writes)
if export:
self.write_export(export, title, track, drm)
elif isinstance(drm, MonaLisa):
with self.drm_lock(drm):
display_id = drm.content_id or drm.pssh
pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa")
cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
pre_existing_tree = next(
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
)
if pre_existing_tree:
cek_tree = pre_existing_tree
for kid_, key in drm.content_keys.items():
label = f"[text2]{kid_.hex}:{key}*"
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
cek_tree.add(label)
if cek_tree.children and not pre_existing_tree:
table.add_row()
table.add_row(cek_tree)
if export:
self.write_export(export, title, track, drm)
@staticmethod
def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]:
"""Get Service Cookie File Path for Profile."""
direct_cookie_file = config.directories.cookies / f"{service}.txt"
profile_cookie_file = config.directories.cookies / service / f"{profile}.txt"
default_cookie_file = config.directories.cookies / service / "default.txt"
if direct_cookie_file.exists():
return direct_cookie_file
elif profile_cookie_file.exists():
return profile_cookie_file
elif default_cookie_file.exists():
return default_cookie_file
@staticmethod
def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[CookieJar]:
"""
Get Service Cookies for Profile.
If the config has firefox_cookies for the service, cookies come from Firefox first. If that extraction
raises or returns nothing, unshackle uses the profile's cookie file instead and does not log the failure.
"""
ff_settings = getattr(config, "firefox_cookies", {}).get(service)
if ff_settings:
try:
from unshackle.core.utils.firefox_cookie_extractor import get_firefox_cookies
extracted_jar = get_firefox_cookies(ff_settings)
if extracted_jar:
return extracted_jar
except Exception:
pass
cookie_file = dl.get_cookie_path(service, profile)
if cookie_file:
return dl.load_cookie_file(cookie_file)
return None
@staticmethod
def load_cookie_file(cookie_file: Path) -> MozillaCookieJar:
"""Load a Netscape cookie file with every expiry blanked, so no layer drops an expired cookie."""
cookie_jar = dl.parse_cookie_text(cookie_file.read_text("utf-8-sig"), str(cookie_file))
cookie_jar.filename = str(cookie_file)
return cookie_jar
@staticmethod
def parse_cookie_text(text: str, source: str) -> MozillaCookieJar:
"""Parse Netscape cookie text the way :meth:`load_cookie_file` does; ``source`` names it in errors."""
cookie_jar = MozillaCookieJar()
cookie_data = html.unescape(text).splitlines(keepends=False)
head = next((line.lstrip() for line in cookie_data if line.strip()), "")
if head[:1] in ("[", "{"):
raise ValueError(f"{source} is a JSON export; export it again in Netscape format")
for i, line in enumerate(cookie_data):
body = line.lstrip()
prefix = "#HttpOnly_" if body.startswith("#HttpOnly_") else ""
body = body[len(prefix) :]
if not body or body.startswith("#"):
continue
line_data = body.split("\t") if "\t" in body else body.split(None, 6)
if len(line_data) == 6:
line_data.append("")
if len(line_data) != 7:
raise ValueError(f"{source} line {i + 1} is not a Netscape cookie row: {line!r}")
line_data[4] = ""
line_data[1] = str(line_data[0].startswith(".")).upper()
cookie_data[i] = prefix + "\t".join(line_data)
cookie_data.insert(0, "# Netscape HTTP Cookie File")
cookie_jar._really_load(StringIO("\n".join(cookie_data)), source, True, True) # type: ignore[attr-defined]
if not len(cookie_jar):
raise ValueError(f"{source} holds no cookies; export it again in Netscape format")
return cookie_jar
@staticmethod
def save_cookies(path: Path, cookies: CookieJar):
if hasattr(cookies, "jar"):
cookies = cookies.jar
cookie_jar = dl.load_cookie_file(path) if path.exists() else MozillaCookieJar(path)
for cookie in cookies:
if not cookie.name or not cookie.domain:
continue
cookie_jar.set_cookie(cookie)
cookie_jar.save(ignore_discard=True, ignore_expires=True)
@staticmethod
def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]:
"""Get Service Credentials for Profile."""
credentials = config.credentials.get(service)
if credentials:
if isinstance(credentials, dict):
if profile:
credentials = credentials.get(profile) or credentials.get("default")
else:
credentials = credentials.get("default")
if credentials:
if isinstance(credentials, list):
return Credential(*credentials)
return Credential.loads(credentials) # type: ignore
def cdm_from_service_entry(self) -> bool:
"""Whether the service's own ``cdm`` entry named the loaded device, not ``--cdm`` or the ``default`` entry."""
return (
self.cdm is not None
and not self.cdm_override
and cdm_entry_names_device(ci_get(config.cdm, self.service), self.profile)
)
def get_cdm(
self,
service: str,
profile: Optional[str] = None,
drm: Optional[str] = None,
quality: Optional[int] = None,
) -> Optional[object]:
"""
Get CDM for a specified service (either Local or Remote CDM).
Now supports quality-based selection when the caller gives a quality.
With no ``drm``, an entry gives the DRM system that it lists first.
Raises a ValueError if there is a problem getting a CDM.
"""
# A per-request override (set by the REST API per job) takes precedence over the
# global config, so a job can select a specific CDM device without mutating shared state.
cdm_name = resolve_cdm_name(config.cdm, service, getattr(self, "cdm_override", None))
if not cdm_name:
return None
if isinstance(cdm_name, dict):
if quality:
quality_match = None
quality_keys = []
for key in cdm_name.keys():
if (
isinstance(key, str)
and any(op in key for op in [">=", ">", "<=", "<"])
or (isinstance(key, str) and key.isdigit())
):
quality_keys.append(key)
def sort_quality_key(key):
if key.isdigit():
return (0, int(key)) # Exact matches first
elif key.startswith(">="):
return (1, -int(key[2:])) # >= descending
elif key.startswith(">"):
return (1, -int(key[1:])) # > descending
elif key.startswith("<="):
return (2, int(key[2:])) # <= ascending
elif key.startswith("<"):
return (2, int(key[1:])) # < ascending
return (3, 0) # Other keys last
quality_keys.sort(key=sort_quality_key)
for key in quality_keys:
if key.isdigit() and quality == int(key):
quality_match = cdm_name[key]
self.log.debug(f"Selected CDM based on exact quality match {quality}p: {quality_match}")
break
elif key.startswith(">="):
threshold = int(key[2:])
if quality >= threshold:
quality_match = cdm_name[key]
self.log.debug(f"Selected CDM based on quality {quality}p >= {threshold}p: {quality_match}")
break
elif key.startswith(">"):
threshold = int(key[1:])
if quality > threshold:
quality_match = cdm_name[key]
self.log.debug(f"Selected CDM based on quality {quality}p > {threshold}p: {quality_match}")
break
elif key.startswith("<="):
threshold = int(key[2:])
if quality <= threshold:
quality_match = cdm_name[key]
self.log.debug(f"Selected CDM based on quality {quality}p <= {threshold}p: {quality_match}")
break
elif key.startswith("<"):
threshold = int(key[1:])
if quality < threshold:
quality_match = cdm_name[key]
self.log.debug(f"Selected CDM based on quality {quality}p < {threshold}p: {quality_match}")
break
if quality_match:
cdm_name = quality_match
if isinstance(cdm_name, dict):
lower_keys = {k.lower(): v for k, v in cdm_name.items()}
if {"widevine", "playready"} & lower_keys.keys():
drm_key = None
if drm:
drm_key = {
"wv": "widevine",
"widevine": "widevine",
"pr": "playready",
"playready": "playready",
}.get(drm.lower())
first = next(key for key in lower_keys if key in ("widevine", "playready"))
cdm_name = lower_keys.get(drm_key or first) or lower_keys.get("playready")
else:
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(config.cdm, "default")
if not cdm_name:
return None
from unshackle.core.cdm import load_cdm
if not isinstance(cdm_name, str):
return load_cdm(cdm_name, service_name=service, vaults=self.vaults)
cdm_cache: dict[tuple[str, Optional[str], str], Any] = getattr(self, "_cdm_cache", {})
self._cdm_cache = cdm_cache
cache_key = (service, profile, cdm_name)
if cache_key not in cdm_cache:
cdm_cache[cache_key] = load_cdm(cdm_name, service_name=service, vaults=self.vaults)
return cdm_cache[cache_key]
Reference in New Issue View Git Blame Copy Permalink