forked from kenzuya/unshackle
Gluetun and loopback proxy URIs only listen on this machine. Stop with an error instead of forwarding them, unless the server is on this machine too.
4883 lines
233 KiB
Python
4883 lines
233 KiB
Python
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import math
|
|
import os
|
|
import random
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import Counter
|
|
from collections.abc import Iterable
|
|
from concurrent import futures
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from copy import deepcopy
|
|
from datetime import date, timedelta
|
|
from functools import partial
|
|
from http.cookiejar import CookieJar, MozillaCookieJar
|
|
from io import StringIO
|
|
from itertools import product
|
|
from pathlib import Path
|
|
from threading import Lock
|
|
from typing import Any, Callable, Collection, Optional, Sequence, TypedDict, Union
|
|
from uuid import UUID
|
|
|
|
import click
|
|
import yaml
|
|
from click.core import ParameterSource
|
|
from langcodes import Language, tag_is_valid
|
|
from pymediainfo import MediaInfo
|
|
from rich.console import Group
|
|
from rich.padding import Padding
|
|
from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn
|
|
from rich.rule import Rule
|
|
from rich.spinner import Spinner
|
|
from rich.table import Table
|
|
from rich.text import Text
|
|
from rich.tree import Tree
|
|
|
|
from unshackle.core import binaries, providers
|
|
from unshackle.core.cdm import DecryptLabsRemoteCDM
|
|
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_widevine_cdm
|
|
from unshackle.core.config import config, resolve_cdm_name, resolve_decryption
|
|
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
|
|
from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings
|
|
from unshackle.core.credential import Credential
|
|
from unshackle.core.downloaders import default_max_workers, format_speed, parse_speed_limit, set_speed_limit
|
|
from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevine
|
|
from unshackle.core.events import events
|
|
from unshackle.core.providers.anilist import parse_anilist_ref
|
|
from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int
|
|
from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN
|
|
from unshackle.core.proxies.resolve import is_loopback
|
|
from unshackle.core.service import Service, grow_session_pool
|
|
from unshackle.core.services import Services
|
|
from unshackle.core.temp import with_task_temp
|
|
from unshackle.core.title_cacher import get_account_hash
|
|
from unshackle.core.titles import Movie, Movies, Series, Song, Title_T
|
|
from unshackle.core.titles.episode import Episode
|
|
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
|
|
from unshackle.core.tracks.attachment import Attachment
|
|
from unshackle.core.tracks.dv_fixup import apply_dv_fixup
|
|
from unshackle.core.tracks.hybrid import Hybrid
|
|
from unshackle.core.tracks.track import assert_fragments_decrypted, has_encrypted_sample_entry
|
|
from unshackle.core.utilities import (
|
|
as_requested,
|
|
embedded_audio_langs,
|
|
excluded_language_tags,
|
|
find_font_with_fallbacks,
|
|
find_missing_langs,
|
|
get_debug_logger,
|
|
get_system_fonts,
|
|
init_debug_logger,
|
|
is_close_match,
|
|
is_exact_match,
|
|
keep_forced_subtitle,
|
|
log_event,
|
|
missing_required_langs,
|
|
partition_exclusions,
|
|
resolve_sort_langs,
|
|
suggest_font_packages,
|
|
time_elapsed_since,
|
|
)
|
|
from unshackle.core.utils import tags
|
|
from unshackle.core.utils.bitrate import apply_real_bitrates
|
|
from unshackle.core.utils.click_types import (
|
|
AUDIO_CODEC_LIST,
|
|
LANGUAGE_RANGE,
|
|
QUALITY_LIST,
|
|
SEASON_RANGE,
|
|
SLOW_DELAY_RANGE,
|
|
ContextData,
|
|
MultipleChoice,
|
|
MultipleVideoCodecChoice,
|
|
SubtitleCodecChoice,
|
|
)
|
|
from unshackle.core.utils.collections import ci_get, merge_dict
|
|
from unshackle.core.utils.post_scripts import build_context, dispatch, season_context
|
|
from unshackle.core.utils.redact import mask_proxy
|
|
from unshackle.core.utils.selector import select_multiple
|
|
from unshackle.core.utils.subprocess import ffprobe
|
|
from unshackle.core.vaults import Vaults
|
|
|
|
|
|
class SkippedSubtitle(TypedDict):
|
|
"""A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``,
|
|
one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable."""
|
|
|
|
id: str
|
|
language: str
|
|
title: str
|
|
|
|
|
|
# Config keys accepted as natural names for params whose Python name dodges a builtin.
|
|
DL_OPTION_ALIASES = {"range": "range_", "list": "list_"}
|
|
|
|
|
|
def parse_language(tag: Optional[str]) -> Optional[Language]:
|
|
"""A provider's original-language tag as a Language, or None if it is absent or not a real tag."""
|
|
tag = (tag or "").strip()
|
|
if not tag or not tag_is_valid(tag) or tag.startswith("und"):
|
|
return None
|
|
return Language.get(tag)
|
|
|
|
|
|
def normalize_dl_config(dl_config: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map aliased config keys (e.g. ``range``) to their Click parameter names (``range_``)."""
|
|
return {DL_OPTION_ALIASES.get(key, key): value for key, value in dl_config.items()}
|
|
|
|
|
|
ID_PROVIDER_KEYS = {"tmdb": "tmdb_api_key", "tvdb": "tvdb_api_key", "omdb": "omdb_api_key"}
|
|
|
|
|
|
def validate_metadata_ids(
|
|
tmdb_id: Optional[int],
|
|
imdb_id: Optional[str],
|
|
tvdb_id: Optional[int],
|
|
anilist_id: Optional[Union[int, str]] = None,
|
|
) -> None:
|
|
"""Reject ID flags that conflict, or that no configured metadata provider could ever find.
|
|
|
|
--anilist stays outside the mutual exclusion: AniList knows no western IDs, so pairing
|
|
it with one of the other three is how an anime title gets both.
|
|
"""
|
|
if anilist_id and not parse_anilist_ref(anilist_id):
|
|
raise click.UsageError("--anilist takes an AniList ID like 21, or a MyAnimeList ID like mal:21.")
|
|
|
|
supplied = [
|
|
(flag, value) for flag, value in (("--tmdb", tmdb_id), ("--imdb", imdb_id), ("--tvdb", tvdb_id)) if value
|
|
]
|
|
if len(supplied) > 1:
|
|
raise click.UsageError(
|
|
f"{', '.join(flag for flag, _ in supplied)} cannot be used together. "
|
|
"Give one ID and unshackle resolves the others from it."
|
|
)
|
|
if not supplied:
|
|
return
|
|
|
|
flag, _ = supplied[0]
|
|
id_kind = flag.lstrip("-")
|
|
ordered = {cls.NAME: cls for kind in ("tv", "movie") for cls in providers.provider_order(kind)}
|
|
consumers = [cls for cls in ordered.values() if cls.ID_KIND == id_kind]
|
|
if any(cls().is_available() for cls in consumers):
|
|
return
|
|
|
|
if consumers:
|
|
keys = sorted({ID_PROVIDER_KEYS[cls.NAME] for cls in consumers if cls.NAME in ID_PROVIDER_KEYS})
|
|
raise click.UsageError(
|
|
f"{flag} needs the {consumers[0].NAME} provider. Set {' or '.join(keys)} in your config."
|
|
)
|
|
names = sorted({cls.NAME for cls in providers.REGISTRY.values() if cls.ID_KIND == id_kind})
|
|
raise click.UsageError(
|
|
f"{flag} needs the {' or '.join(names)} provider, which your metadata_providers config leaves out."
|
|
)
|
|
|
|
|
|
def group_videos_by_variant(videos: list[Video], *, merge: bool) -> list[list[Video]]:
|
|
"""Group video tracks for muxing.
|
|
|
|
When ``merge`` is True, this function groups tracks that share ``(height, range, codec)``
|
|
into one file, so only language varies within a group. Different resolutions, ranges and
|
|
codecs stay in separate groups (separate files). When False, each track is its own group
|
|
(one file per track, the default behaviour). Group order follows first-seen track order.
|
|
"""
|
|
if not merge:
|
|
return [[video] for video in videos]
|
|
groups: dict[tuple[Any, ...], list[Video]] = {}
|
|
for video in videos:
|
|
groups.setdefault((video.height, video.range, video.codec), []).append(video)
|
|
return list(groups.values())
|
|
|
|
|
|
def title_wanted(candidate: Any, wanted: Collection[str]) -> bool:
|
|
"""Whether ``-w`` selects this title. A title type without selection keys is always kept.
|
|
|
|
The download loop and the post-script counter both read this, so the counter cannot
|
|
fall out of step with the titles the loop skips.
|
|
"""
|
|
if not wanted or not isinstance(candidate, (Episode, Song)):
|
|
return True
|
|
return bool(candidate.matches_wanted(wanted))
|
|
|
|
|
|
def server_url(server_name: Optional[str]) -> str:
|
|
"""The configured URL of a remote server, or an empty string when the config cannot be read."""
|
|
from unshackle.core.remote_service import resolve_server
|
|
|
|
try:
|
|
return resolve_server(server_name)[0]
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def post_script_group(candidate: Any) -> Any:
|
|
"""Group a title by the folder its outputs share, for post-scripts in season mode."""
|
|
if isinstance(candidate, Episode):
|
|
return ("episode", candidate.title, candidate.season)
|
|
if isinstance(candidate, Song):
|
|
artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "")
|
|
return ("album", artist, getattr(candidate, "album", ""))
|
|
return ("title", id(candidate))
|
|
|
|
|
|
def remux_music_tracks(title: Any, log: logging.Logger) -> None:
|
|
"""Remux each audio track of a Song into the container of its codec.
|
|
|
|
A Song never reaches the muxer, so each downloaded audio track is itself a delivered
|
|
file. A failure warns, keeps that downloaded track, and moves on to the next one: the
|
|
user still gets the audio, in a container a player cuts short.
|
|
"""
|
|
if not isinstance(title, Song):
|
|
return
|
|
for track in title.tracks.audio:
|
|
try:
|
|
track.to_music_container()
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning(f"Could not remux audio track {track.id} for music: {e}")
|
|
|
|
|
|
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
|
|
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
|
|
The config replaces defaults and global ``dl:`` default_map values."""
|
|
params_by_name = {param.name: param for param in ctx.command.params if param.name}
|
|
for name, value in normalize_dl_config(service_dl_config).items():
|
|
param = params_by_name.get(name)
|
|
if param is None:
|
|
log.warning(f"Ignoring unknown dl option '{name}' in service config")
|
|
continue
|
|
if name not in ctx.params:
|
|
log.debug(f"Skipping service dl override '{name}': not present in this context")
|
|
continue
|
|
if value is None:
|
|
continue
|
|
source = ctx.get_parameter_source(name)
|
|
if source not in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP):
|
|
continue
|
|
try:
|
|
ctx.params[name] = param.type_cast_value(ctx, value)
|
|
log.debug(f"Applied service dl override '{name}': {ctx.params[name]}")
|
|
except Exception as e:
|
|
log.warning(
|
|
f"Failed to apply service dl override '{name}'={value!r}: {e}. "
|
|
f"Check that the value is valid for this parameter."
|
|
)
|
|
|
|
|
|
def download_tracks_in_passes(
|
|
tracks: Iterable[AnyTrack],
|
|
max_concurrent: int,
|
|
run_one: Callable[[AnyTrack, int], None],
|
|
*,
|
|
skip_subtitle_errors: bool,
|
|
on_subtitle_skipped: Callable[[Subtitle], None],
|
|
) -> None:
|
|
"""Download a title's tracks so a skippable subtitle failure cannot corrupt the rest.
|
|
|
|
A failed track sets the process-global ``DOWNLOAD_CANCELLED`` event, making other in-flight
|
|
tracks early-return without raising. With ``skip_subtitle_errors`` set, video/audio download
|
|
concurrently first. The subtitles then download one at a time (a concurrent pass would let one
|
|
failure's cancel silently drop the others, unrecorded), with the event cleared before each.
|
|
Video/audio failures stay fatal. This function clears the event on entry (stale cancel from a
|
|
prior title) and in ``finally`` (never leave it set for later code).
|
|
|
|
``run_one(track, index)`` downloads a single track. ``on_subtitle_skipped(track)`` records a
|
|
subtitle whose download raised (handled here).
|
|
"""
|
|
DOWNLOAD_CANCELLED.clear()
|
|
indexed = list(enumerate(tracks))
|
|
if skip_subtitle_errors:
|
|
primary = [(i, t) for i, t in indexed if not isinstance(t, Subtitle)]
|
|
skippable = [(i, t) for i, t in indexed if isinstance(t, Subtitle)]
|
|
else:
|
|
primary, skippable = indexed, []
|
|
|
|
try:
|
|
with ThreadPoolExecutor(max_concurrent) as pool:
|
|
future_to_track = {pool.submit(run_one, track, i): track for i, track in primary}
|
|
for download in futures.as_completed(future_to_track):
|
|
download.result() # a video/audio failure is fatal
|
|
|
|
for i, track in skippable:
|
|
DOWNLOAD_CANCELLED.clear()
|
|
try:
|
|
run_one(track, i)
|
|
except Exception:
|
|
on_subtitle_skipped(track)
|
|
finally:
|
|
DOWNLOAD_CANCELLED.clear() # never leave a failed track's cancel set for later code
|
|
|
|
|
|
class dl:
|
|
@staticmethod
|
|
def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str:
|
|
"""Truncate PSSH string for display when not in debug mode."""
|
|
if logging.root.level == logging.DEBUG or not pssh_string:
|
|
return pssh_string
|
|
|
|
max_width = console.width - len(drm_type) - 12
|
|
if len(pssh_string) <= max_width:
|
|
return pssh_string
|
|
|
|
return pssh_string[: max_width - 3] + "..."
|
|
|
|
def find_custom_font(self, font_name: str) -> Optional[Path]:
|
|
"""
|
|
Find font in custom fonts directory.
|
|
|
|
Args:
|
|
font_name: Font family name to find
|
|
|
|
Returns:
|
|
Path to font file, or None if not found
|
|
"""
|
|
family_dir = Path(config.directories.fonts, font_name)
|
|
if family_dir.exists():
|
|
fonts = list(family_dir.glob("*.*tf"))
|
|
return fonts[0] if fonts else None
|
|
return None
|
|
|
|
def prepare_temp_font(
|
|
self, font_name: str, matched_font: Path, system_fonts: dict[str, Path], temp_font_files: list[Path]
|
|
) -> Path:
|
|
"""
|
|
Copy system font to temp and log if using fallback.
|
|
|
|
Args:
|
|
font_name: Requested font name
|
|
matched_font: Path to matched system font
|
|
system_fonts: Dictionary of available system fonts
|
|
temp_font_files: List to track temp files for cleanup
|
|
|
|
Returns:
|
|
Path to temp font file
|
|
"""
|
|
matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None)
|
|
|
|
if matched_name and matched_name.lower() != font_name.lower():
|
|
self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'")
|
|
|
|
safe_name = font_name.replace(" ", "_").replace("/", "_")
|
|
temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}"
|
|
|
|
if not temp_path.exists():
|
|
shutil.copy2(matched_font, temp_path)
|
|
temp_font_files.append(temp_path)
|
|
|
|
return temp_path
|
|
|
|
def attach_subtitle_fonts(
|
|
self, font_names: list[str], title: Title_T, temp_font_files: list[Path]
|
|
) -> tuple[int, list[str]]:
|
|
"""
|
|
Attach fonts for subtitle rendering.
|
|
|
|
Args:
|
|
font_names: List of font names requested by subtitles
|
|
title: Title object to attach fonts to
|
|
temp_font_files: List to track temp files for cleanup
|
|
|
|
Returns:
|
|
Tuple of (fonts_attached_count, missing_fonts_list)
|
|
"""
|
|
system_fonts = get_system_fonts()
|
|
|
|
font_count = 0
|
|
missing_fonts = []
|
|
|
|
for font_name in set(font_names):
|
|
if custom_font := self.find_custom_font(font_name):
|
|
title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})"))
|
|
font_count += 1
|
|
continue
|
|
|
|
if system_font := find_font_with_fallbacks(font_name, system_fonts):
|
|
temp_path = self.prepare_temp_font(font_name, system_font, system_fonts, temp_font_files)
|
|
title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})"))
|
|
font_count += 1
|
|
else:
|
|
self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found")
|
|
missing_fonts.append(font_name)
|
|
|
|
return font_count, missing_fonts
|
|
|
|
def suggest_missing_fonts(self, missing_fonts: list[str]) -> None:
|
|
"""
|
|
Show package installation suggestions for missing fonts.
|
|
|
|
Args:
|
|
missing_fonts: List of font names that could not be found
|
|
"""
|
|
if suggestions := suggest_font_packages(missing_fonts):
|
|
self.log.info("Install font packages to improve subtitle rendering:")
|
|
for package_cmd, fonts in suggestions.items():
|
|
self.log.info(f" $ sudo apt install {package_cmd}")
|
|
self.log.info(f" → Provides: {', '.join(fonts)}")
|
|
|
|
def generate_sidecar_subtitle_path(
|
|
self,
|
|
subtitle: Subtitle,
|
|
base_filename: str,
|
|
output_dir: Path,
|
|
target_codec: Optional[Subtitle.Codec] = None,
|
|
source_path: Optional[Path] = None,
|
|
) -> Path:
|
|
"""Make the sidecar path: {base}.{lang}[.forced][.sdh].{ext}"""
|
|
lang_suffix = str(subtitle.language) if subtitle.language else "und"
|
|
forced_suffix = ".forced" if subtitle.forced else ""
|
|
sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else ""
|
|
|
|
extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension
|
|
if not target_codec and not subtitle.codec and source_path and source_path.suffix:
|
|
extension = source_path.suffix.lstrip(".")
|
|
|
|
filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}"
|
|
return output_dir / filename
|
|
|
|
def output_subtitle_sidecars(
|
|
self,
|
|
subtitles: list[Subtitle],
|
|
base_filename: str,
|
|
output_dir: Path,
|
|
sidecar_format: str,
|
|
original_paths: Optional[dict[str, Path]] = None,
|
|
) -> list[Path]:
|
|
"""
|
|
Output subtitles as sidecar files, converting if needed.
|
|
|
|
Conversion runs on a temp copy, so each subtitle's own file stays untouched and mkvmerge can still mux it.
|
|
"""
|
|
created_paths: list[Path] = []
|
|
config.directories.temp.mkdir(parents=True, exist_ok=True)
|
|
|
|
for subtitle in subtitles:
|
|
source_path = subtitle.path
|
|
if sidecar_format == "original" and original_paths and subtitle.id in original_paths:
|
|
source_path = original_paths[subtitle.id]
|
|
|
|
if not source_path or not source_path.exists():
|
|
continue
|
|
|
|
if sidecar_format == "original":
|
|
target_codec = None
|
|
if source_path.suffix:
|
|
try:
|
|
target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip("."))
|
|
except ValueError:
|
|
target_codec = None
|
|
else:
|
|
target_codec = Subtitle.Codec.from_mime(sidecar_format)
|
|
|
|
sidecar_path = self.generate_sidecar_subtitle_path(
|
|
subtitle, base_filename, output_dir, target_codec, source_path=source_path
|
|
)
|
|
|
|
if not target_codec or subtitle.codec == target_codec:
|
|
shutil.copy2(source_path, sidecar_path)
|
|
else:
|
|
temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}"
|
|
shutil.copy2(source_path, temp_path)
|
|
|
|
temp_sub = Subtitle(
|
|
subtitle.url,
|
|
subtitle.language,
|
|
is_original_lang=subtitle.is_original_lang,
|
|
descriptor=subtitle.descriptor,
|
|
codec=subtitle.codec,
|
|
forced=subtitle.forced,
|
|
sdh=subtitle.sdh,
|
|
cc=subtitle.cc,
|
|
id_=f"{subtitle.id}_sc",
|
|
)
|
|
temp_sub.path = temp_path
|
|
try:
|
|
temp_sub.convert(target_codec, forced=True)
|
|
if temp_sub.path and temp_sub.path.exists():
|
|
shutil.copy2(temp_sub.path, sidecar_path)
|
|
finally:
|
|
if temp_sub.path and temp_sub.path.exists():
|
|
temp_sub.path.unlink(missing_ok=True)
|
|
temp_path.unlink(missing_ok=True)
|
|
|
|
created_paths.append(sidecar_path)
|
|
|
|
return created_paths
|
|
|
|
def post_script_ids(self) -> dict[str, Any]:
|
|
"""Tagging IDs as they stand right now.
|
|
|
|
Read at each post-script call, never snapshotted: the search inside the title loop
|
|
finds an Episode's TMDB ID, so a snapshot taken before it is always empty.
|
|
"""
|
|
return {"tmdb": self.tmdb_id, "imdb": self.imdb_id, "tvdb": self.tvdb_id}
|
|
|
|
@click.command(
|
|
short_help="Download, Decrypt, and Mux tracks for titles from a Service.",
|
|
cls=Services,
|
|
context_settings=dict(**context_settings, default_map=normalize_dl_config(config.dl)),
|
|
)
|
|
@click.option(
|
|
"-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)."
|
|
)
|
|
@click.option(
|
|
"-q",
|
|
"--quality",
|
|
type=QUALITY_LIST,
|
|
default=[],
|
|
help="Download Resolution(s), defaults to the best available resolution.",
|
|
)
|
|
@click.option(
|
|
"-v",
|
|
"--vcodec",
|
|
type=MultipleVideoCodecChoice(Video.Codec),
|
|
default=[],
|
|
help="Video Codec(s) to download, defaults to any codec.",
|
|
)
|
|
@click.option(
|
|
"-a",
|
|
"--acodec",
|
|
type=AUDIO_CODEC_LIST,
|
|
default=[],
|
|
help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.",
|
|
)
|
|
@click.option(
|
|
"-vb",
|
|
"--vbitrate",
|
|
type=int,
|
|
default=None,
|
|
help="Video Bitrate to download (in kbps), defaults to highest available.",
|
|
)
|
|
@click.option(
|
|
"-ab",
|
|
"--abitrate",
|
|
type=int,
|
|
default=None,
|
|
help="Audio Bitrate to download (in kbps), defaults to highest available.",
|
|
)
|
|
@click.option(
|
|
"-vb-range",
|
|
"--vbitrate-range",
|
|
type=str,
|
|
default=None,
|
|
help="Video Bitrate range in kbps (e.g., '6000-7000'). Selects the highest bitrate within the range.",
|
|
)
|
|
@click.option(
|
|
"-ab-range",
|
|
"--abitrate-range",
|
|
type=str,
|
|
default=None,
|
|
help="Audio Bitrate range in kbps (e.g., '128-256'). Selects the highest bitrate within the range.",
|
|
)
|
|
@click.option(
|
|
"-r",
|
|
"--range",
|
|
"range_",
|
|
type=MultipleChoice(Video.Range, case_sensitive=False),
|
|
default=[Video.Range.SDR],
|
|
help="Video Color Range(s) to download, defaults to SDR.",
|
|
)
|
|
@click.option(
|
|
"-c",
|
|
"--channels",
|
|
type=float,
|
|
default=None,
|
|
help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.",
|
|
)
|
|
@click.option(
|
|
"-naa",
|
|
"--noatmos",
|
|
"no_atmos",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Exclude Dolby Atmos audio tracks when selecting audio.",
|
|
)
|
|
@click.option(
|
|
"--split-audio",
|
|
"split_audio",
|
|
is_flag=True,
|
|
default=None,
|
|
help="Create separate output files per audio codec instead of merging all audio.",
|
|
)
|
|
@click.option(
|
|
"--merge-video",
|
|
"merge_video",
|
|
is_flag=True,
|
|
default=None,
|
|
help="Mux all selected video tracks into a single file instead of one file per track.",
|
|
)
|
|
@click.option(
|
|
"--select-titles",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Interactively select downloads from a list: episodes of a series, or films when a title has more than one.",
|
|
)
|
|
@click.option(
|
|
"-w",
|
|
"--wanted",
|
|
type=SEASON_RANGE,
|
|
default=None,
|
|
help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, etc. Music uses track numbers, e.g. `1-5`, `1,3,7`, or `2x3` for disc 2 track 3. Defaults to all.",
|
|
)
|
|
@click.option(
|
|
"-l",
|
|
"--lang",
|
|
type=LANGUAGE_RANGE,
|
|
default="orig",
|
|
help="Language(s) wanted for Video and Audio (comma-separated). Use 'orig' to select the original language, e.g. 'orig,en' for both original and English. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--latest-episode",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Download only the single most recent episode available.",
|
|
)
|
|
@click.option(
|
|
"-vl",
|
|
"--v-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Language wanted for Video. You would use this if the video language doesn't match the audio. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"-al",
|
|
"--a-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Language wanted for Audio, overrides -l/--lang for audio tracks. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--require-audio",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Audio languages that must exist. The title fails even with --best-available.",
|
|
)
|
|
@click.option(
|
|
"--require-video",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Video languages that must exist. The title fails even with --best-available.",
|
|
)
|
|
@click.option(
|
|
"-sl",
|
|
"--s-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=["all"],
|
|
help="Language wanted for Subtitles. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--require-subs",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.",
|
|
)
|
|
@click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.")
|
|
@click.option(
|
|
"-fsl",
|
|
"--forced-s-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Languages wanted for forced subtitles, implies -fs. Keeps forced subs only in these languages. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--exact-lang",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Use exact language matching (no variants). With this flag, -l es-419 matches ONLY es-419, not es-ES or other variants.",
|
|
)
|
|
@click.option(
|
|
"--proxy",
|
|
type=str,
|
|
default=None,
|
|
help="Proxy URI to use. If a 2-letter country is provided, it will try to get a proxy from the config.",
|
|
)
|
|
@click.option(
|
|
"--tag",
|
|
type=str,
|
|
default=None,
|
|
help="Set the Group Tag to be used, overriding the config tag and any tag rules.",
|
|
)
|
|
@click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.")
|
|
@click.option(
|
|
"-rvb",
|
|
"--real-video-bitrate",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Probe actual media size to compute true video bitrates (top renditions per codec/range), "
|
|
"overriding the manifest's declared bitrate.",
|
|
)
|
|
@click.option(
|
|
"-rab",
|
|
"--real-audio-bitrate",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Probe actual media size to compute true audio bitrates (top renditions per codec/channels/language), "
|
|
"overriding the manifest's declared bitrate. Slower than --real-video-bitrate (more renditions).",
|
|
)
|
|
@click.option(
|
|
"--tmdb",
|
|
"tmdb_id",
|
|
type=int,
|
|
default=None,
|
|
help="Use this TMDB ID for tagging instead of an automatic search. Add --enrich to also take "
|
|
"its title, year and original language.",
|
|
)
|
|
@click.option(
|
|
"--tvdb",
|
|
"tvdb_id",
|
|
type=int,
|
|
default=None,
|
|
help="Use this TVDB ID for --tvdb-order and tagging instead of looking the series up "
|
|
"automatically. Add --enrich to also take its title, year and original language.",
|
|
)
|
|
@click.option(
|
|
"--tvdb-order",
|
|
type=click.Choice(SEASON_TYPES, case_sensitive=False),
|
|
default=None,
|
|
help="Renumber episodes to a TVDB season order. Services normally use 'official' (aired) order; "
|
|
"a series' Streaming Order is 'alternate'. Defaults to the tvdb_order config option.",
|
|
)
|
|
@click.option(
|
|
"--anilist",
|
|
"anilist_id",
|
|
type=str,
|
|
default=None,
|
|
help="Use this AniList ID for tagging instead of an automatic search, or a MyAnimeList ID as mal:12345. "
|
|
"Add --enrich to also take its title, year and original language. Combines with one of "
|
|
"--tmdb, --imdb or --tvdb, which AniList does not know.",
|
|
)
|
|
@click.option(
|
|
"--enrich",
|
|
is_flag=True,
|
|
default=False,
|
|
help=(
|
|
"Overwrite the show title, year and original language with the external source's. "
|
|
"Requires --tmdb, --imdb, --tvdb, or --anilist."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--daily",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Treat the title as daily/date-based content and fill missing air dates from TVDB during --enrich.",
|
|
)
|
|
@click.option(
|
|
"--imdb",
|
|
"imdb_id",
|
|
type=str,
|
|
default=None,
|
|
help="Use this IMDB ID (e.g. tt1375666) for tagging instead of an automatic search. Add --enrich "
|
|
"to also take its title, year and original language.",
|
|
)
|
|
@click.option(
|
|
"--sub-format",
|
|
type=SubtitleCodecChoice(Subtitle.Codec),
|
|
default=None,
|
|
help="Set Output Subtitle Format, only converting if necessary. Use 'original' to keep source format.",
|
|
)
|
|
@click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.")
|
|
@click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.")
|
|
@click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.")
|
|
@click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapter markers.")
|
|
@click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.")
|
|
@click.option(
|
|
"--skip-subtitle-errors",
|
|
is_flag=True,
|
|
default=False,
|
|
help="If a subtitle track fails to download, skip it and continue instead of "
|
|
"aborting the whole title (video/audio failures stay fatal).",
|
|
)
|
|
@click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.")
|
|
@click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapter markers.")
|
|
@click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.")
|
|
@click.option(
|
|
"-nt",
|
|
"--no-attachments",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Do not download or mux attachments (cover art, subtitle fonts, and files the service attaches).",
|
|
)
|
|
@click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.")
|
|
@click.option(
|
|
"--slow",
|
|
type=SLOW_DELAY_RANGE,
|
|
default=None,
|
|
help="Add a delay between each Title download to act more like a real device. "
|
|
"Use --slow for a 60-120s delay, or --slow MIN-MAX (e.g., --slow 20-40) for a custom range. "
|
|
"Minimum delay is 20 seconds.",
|
|
)
|
|
@click.option(
|
|
"--list",
|
|
"list_",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip downloading and list available tracks and what tracks would have been downloaded.",
|
|
)
|
|
@click.option(
|
|
"--list-titles",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip downloading, only list available titles that would have been downloaded.",
|
|
)
|
|
@click.option(
|
|
"--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys."
|
|
)
|
|
@click.option(
|
|
"--export",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Export track info and decryption keys to a JSON file in the exports directory.",
|
|
)
|
|
@click.option(
|
|
"--import",
|
|
"import_file",
|
|
type=str,
|
|
default=None,
|
|
hidden=True,
|
|
help="Internal: path to an export JSON to reconstruct a download from (used by 'unshackle import').",
|
|
)
|
|
@click.option(
|
|
"--cdm-only/--vaults-only",
|
|
is_flag=True,
|
|
default=None,
|
|
help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.",
|
|
)
|
|
@click.option(
|
|
"--cdm",
|
|
"cdm_name",
|
|
type=str,
|
|
default=None,
|
|
help="Use this CDM device for the run, overriding the cdm config mapping.",
|
|
)
|
|
@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.")
|
|
@click.option(
|
|
"--no-proxy-download",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Bypass proxy for all downloads. Manifest, license, and auth still use proxy.",
|
|
)
|
|
@click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.")
|
|
@click.option(
|
|
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
|
|
)
|
|
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
|
|
@click.option(
|
|
"--postscript",
|
|
"postscript",
|
|
type=str,
|
|
multiple=True,
|
|
metavar="COMMAND",
|
|
help=(
|
|
"Run COMMAND after each downloaded file, with {filepath}, {title}, {season} and other "
|
|
"variables substituted. Repeatable. Replaces the post_scripts config for this run."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--workers",
|
|
type=int,
|
|
default=None,
|
|
help="Max workers/threads to download with per-track. Default depends on the downloader.",
|
|
)
|
|
@click.option(
|
|
"--adaptive-workers",
|
|
is_flag=True,
|
|
default=False,
|
|
help=(
|
|
"Dynamically scale per-track segment workers up to the --workers cap (or the built-in cap "
|
|
"when --workers is unset) based on measured CDN throughput and errors. Off by default."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--download-processes",
|
|
type=int,
|
|
default=1,
|
|
help=(
|
|
"Split a track's segment downloads across this many processes to beat the single-interpreter "
|
|
"throughput cap. Only engages for large segment batches. Default 1 (single process)."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--continue-downloads",
|
|
is_flag=True,
|
|
default=False,
|
|
help=(
|
|
"Keep completed segment files across runs and resume a previously failed download. "
|
|
"One-off enable of the continue_downloads config option."
|
|
),
|
|
)
|
|
@click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.")
|
|
@click.option(
|
|
"--speed-limit",
|
|
"speed_limit",
|
|
type=str,
|
|
default=None,
|
|
metavar="SPEED",
|
|
help="Cap total download speed across all downloads combined, e.g. 500k, 5M, 1.5G or plain "
|
|
"bytes/sec (5M = 5.0 MB/s; bytes, not bits). "
|
|
"'off' disables a limit configured in the dl: config block.",
|
|
)
|
|
@click.option(
|
|
"-o",
|
|
"--output",
|
|
"output_dir",
|
|
type=Path,
|
|
default=None,
|
|
help="Override the output directory for this download, instead of the one in config.",
|
|
)
|
|
@click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.")
|
|
@click.option(
|
|
"--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching."
|
|
)
|
|
@click.option(
|
|
"--worst",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Select the lowest bitrate track within the specified quality. Requires -q/--quality.",
|
|
)
|
|
@click.option(
|
|
"--best-available",
|
|
"--warn-only",
|
|
"best_available",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Warn instead of failing when a requested resolution, range, or language is missing, and continue with what is available.",
|
|
)
|
|
@click.option(
|
|
"--remote",
|
|
is_flag=True,
|
|
default=False,
|
|
is_eager=True,
|
|
help="Use a remote unshackle server instead of local service code.",
|
|
)
|
|
@click.option(
|
|
"--server",
|
|
type=str,
|
|
default=None,
|
|
is_eager=True,
|
|
help="Name of the remote server from remote_services config (if multiple configured).",
|
|
)
|
|
@click.pass_context
|
|
def cli(ctx: click.Context, **kwargs: Any) -> dl:
|
|
return dl(ctx, **kwargs)
|
|
|
|
DRM_LOCKS: dict[str, Lock] = {}
|
|
DRM_LOCKS_GUARD = Lock()
|
|
VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer")
|
|
EXPORT_LOCK = Lock()
|
|
LICENSE_KEY_CACHE: dict[UUID, str] = {}
|
|
|
|
def __init__(
|
|
self,
|
|
ctx: click.Context,
|
|
no_proxy: bool,
|
|
profile: Optional[str] = None,
|
|
proxy: Optional[str] = None,
|
|
repack: bool = False,
|
|
tag: Optional[str] = None,
|
|
tmdb_id: Optional[int] = None,
|
|
imdb_id: Optional[str] = None,
|
|
tvdb_id: Optional[int] = None,
|
|
tvdb_order: Optional[str] = None,
|
|
anilist_id: Optional[Union[int, str]] = None,
|
|
enrich: bool = False,
|
|
daily: bool = False,
|
|
output_dir: Optional[Path] = None,
|
|
proxy_providers: Optional[list[Any]] = None,
|
|
*_: Any,
|
|
**__: Any,
|
|
):
|
|
if not ctx.invoked_subcommand:
|
|
raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.")
|
|
|
|
self.log = logging.getLogger("download")
|
|
self.completed_files: list[Path] = []
|
|
# Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can
|
|
# report which weren't available without parsing the console output. See SkippedSubtitle.
|
|
self.skipped_subtitles: list[SkippedSubtitle] = []
|
|
# result() catches a download-worker failure, reports it, and returns rather than
|
|
# re-raising (so the CLI still exits cleanly). Flag it so an embedding caller (the API
|
|
# worker) can tell the title did not complete instead of reading it as a success.
|
|
self.download_failed: bool = False
|
|
|
|
if not config.output_template:
|
|
raise click.ClickException(
|
|
"No 'output_template' configured in your unshackle.yaml.\n"
|
|
"Please add an 'output_template' section with movies, series, and songs/music templates.\n"
|
|
"See unshackle-example.yaml for examples."
|
|
)
|
|
|
|
self.service = Services.get_tag(ctx.invoked_subcommand)
|
|
self.vault_service = Services.get_vault_tag(self.service)
|
|
apply_service_dl_overrides(ctx, config.services.get(self.service, {}).get("dl", {}), self.log)
|
|
|
|
# Refresh locals Click bound before the overrides ran.
|
|
no_proxy = ctx.params.get("no_proxy", no_proxy)
|
|
profile = ctx.params.get("profile", profile)
|
|
proxy = ctx.params.get("proxy", proxy)
|
|
repack = ctx.params.get("repack", repack)
|
|
tag = ctx.params.get("tag", tag)
|
|
tmdb_id = ctx.params.get("tmdb_id", tmdb_id)
|
|
imdb_id = ctx.params.get("imdb_id", imdb_id)
|
|
tvdb_id = ctx.params.get("tvdb_id", tvdb_id)
|
|
tvdb_order = ctx.params.get("tvdb_order", tvdb_order)
|
|
anilist_id = ctx.params.get("anilist_id", anilist_id)
|
|
enrich = ctx.params.get("enrich", enrich)
|
|
daily = ctx.params.get("daily", daily)
|
|
output_dir = ctx.params.get("output_dir", output_dir)
|
|
|
|
self.profile = profile
|
|
self.proxy_requested = bool(proxy)
|
|
validate_metadata_ids(tmdb_id, imdb_id, tvdb_id, anilist_id)
|
|
self.tmdb_id = tmdb_id
|
|
self.imdb_id = imdb_id
|
|
self.tvdb_id = tvdb_id
|
|
self.anilist_id = anilist_id
|
|
self.tvdb_order = (tvdb_order or config.tvdb_order or "").lower()
|
|
if self.tvdb_order and self.tvdb_order not in SEASON_TYPES:
|
|
raise click.UsageError(f"tvdb_order must be one of {', '.join(SEASON_TYPES)}, not {self.tvdb_order!r}")
|
|
self.enrich = enrich
|
|
self.daily = daily
|
|
self.output_dir = output_dir
|
|
self.service_anime = False
|
|
self.service_daily = False
|
|
|
|
if self.enrich and not (self.tmdb_id or self.imdb_id or self.tvdb_id or self.anilist_id):
|
|
raise click.UsageError(
|
|
"--enrich requires --tmdb, --imdb, --tvdb, or --anilist to provide a metadata source."
|
|
)
|
|
|
|
if self.daily and not self.enrich:
|
|
self.log.warning("--daily fills air dates from TVDB only with --enrich. A service can set them itself.")
|
|
|
|
# Initialize debug logger with service name if debug logging is enabled
|
|
if config.debug or logging.root.level == logging.DEBUG:
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
|
|
debug_log_path = config.directories.logs / config.filenames.debug_log.format_map(
|
|
defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S"))
|
|
)
|
|
init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys)
|
|
self.debug_logger = get_debug_logger()
|
|
|
|
if self.debug_logger:
|
|
self.debug_logger.log(
|
|
level="INFO",
|
|
operation="download_init",
|
|
message=f"Download command initialized for service {self.service}",
|
|
service=self.service,
|
|
context={
|
|
"profile": profile,
|
|
"proxy": proxy,
|
|
"tag": tag,
|
|
"tmdb_id": self.tmdb_id,
|
|
"imdb_id": self.imdb_id,
|
|
"anilist_id": anilist_id,
|
|
"enrich": enrich,
|
|
"cli_params": {
|
|
k: v
|
|
for k, v in ctx.params.items()
|
|
if k
|
|
not in [
|
|
"profile",
|
|
"proxy",
|
|
"tag",
|
|
"tmdb_id",
|
|
"imdb_id",
|
|
"anilist_id",
|
|
"enrich",
|
|
]
|
|
},
|
|
},
|
|
)
|
|
|
|
# Log binary versions for diagnostics
|
|
binary_versions = {}
|
|
for name, binary in [
|
|
("shaka_packager", binaries.ShakaPackager),
|
|
("mp4decrypt", binaries.Mp4decrypt),
|
|
("mkvmerge", binaries.MKVToolNix),
|
|
("ffmpeg", binaries.FFMPEG),
|
|
("ffprobe", binaries.FFProbe),
|
|
]:
|
|
if binary:
|
|
version = None
|
|
try:
|
|
if name == "shaka_packager":
|
|
r = subprocess.run(
|
|
[str(binary), "--version"],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=5,
|
|
)
|
|
version = (r.stdout or r.stderr or "").strip()
|
|
elif name in ("ffmpeg", "ffprobe"):
|
|
r = subprocess.run(
|
|
[str(binary), "-version"],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=5,
|
|
)
|
|
version = (r.stdout or "").split("\n")[0].strip()
|
|
elif name == "mkvmerge":
|
|
r = subprocess.run(
|
|
[str(binary), "--version"],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=5,
|
|
)
|
|
version = (r.stdout or "").strip()
|
|
elif name == "mp4decrypt":
|
|
r = subprocess.run(
|
|
[str(binary)],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=5,
|
|
)
|
|
output = (r.stdout or "") + (r.stderr or "")
|
|
lines = [line.strip() for line in output.split("\n") if line.strip()]
|
|
version = " | ".join(lines[:2]) if lines else None
|
|
except Exception:
|
|
version = "<error getting version>"
|
|
binary_versions[name] = {"path": str(binary), "version": version}
|
|
else:
|
|
binary_versions[name] = None
|
|
|
|
self.debug_logger.log(
|
|
level="DEBUG",
|
|
operation="binary_versions",
|
|
message="Binary tool versions",
|
|
context=binary_versions,
|
|
)
|
|
else:
|
|
self.debug_logger = None
|
|
|
|
if self.profile:
|
|
self.log.info(f"Using profile: '{self.profile}'")
|
|
|
|
self.is_remote = bool(ctx.params.get("remote"))
|
|
|
|
with console.status("Loading Service Config...", spinner="dots"):
|
|
self.service_config = {}
|
|
if not self.is_remote:
|
|
try:
|
|
service_config_path = Services.get_path(self.service) / config.filenames.config
|
|
if service_config_path.exists():
|
|
self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
|
|
self.log.info("Service Config loaded")
|
|
# log key names only: the full config carries service certificates,
|
|
# device fingerprints and endpoints that bloat the log and may be sensitive
|
|
log_event(
|
|
"load_service_config",
|
|
level="DEBUG",
|
|
service=self.service,
|
|
context={
|
|
"config_path": str(service_config_path),
|
|
"config_keys": sorted(self.service_config or []),
|
|
},
|
|
)
|
|
except KeyError:
|
|
pass
|
|
merge_dict(config.services.get(self.service), self.service_config)
|
|
|
|
service_config = config.services.get(self.service, {})
|
|
if service_config:
|
|
reserved_keys = {
|
|
"profiles",
|
|
"api_key",
|
|
"certificate",
|
|
"api_endpoint",
|
|
"region",
|
|
"device",
|
|
"endpoints",
|
|
"client",
|
|
"dl",
|
|
}
|
|
|
|
for config_key, override_value in service_config.items():
|
|
if config_key in reserved_keys or not isinstance(override_value, dict):
|
|
continue
|
|
|
|
if hasattr(config, config_key):
|
|
current_config = getattr(config, config_key, {})
|
|
|
|
if isinstance(current_config, dict):
|
|
merged_config = deepcopy(current_config)
|
|
merge_dict(override_value, merged_config)
|
|
setattr(config, config_key, merged_config)
|
|
|
|
self.log.debug(
|
|
f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}"
|
|
)
|
|
|
|
self.cdm_override = ctx.params.get("cdm_name")
|
|
cdm_only = ctx.params.get("cdm_only")
|
|
self.vault_cache_tally: Optional[tuple[int, int]] = None
|
|
|
|
if cdm_only:
|
|
self.vaults = Vaults(self.vault_service)
|
|
self.log.info("CDM-only mode: Skipping vault loading")
|
|
log_event(
|
|
"vault_loading_skipped",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={"reason": "cdm_only flag set"},
|
|
)
|
|
else:
|
|
with console.status("Loading Key Vaults...", spinner="dots"):
|
|
self.vaults = Vaults(self.vault_service)
|
|
total_vaults = len(config.key_vaults)
|
|
failed_vaults = []
|
|
|
|
for vault in config.key_vaults:
|
|
vault_type = vault["type"]
|
|
vault_name = vault.get("name", vault_type)
|
|
vault_copy = vault.copy()
|
|
del vault_copy["type"]
|
|
|
|
if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower():
|
|
if "token" not in vault_copy or not vault_copy["token"]:
|
|
if config.decrypt_labs_api_key:
|
|
vault_copy["token"] = config.decrypt_labs_api_key
|
|
else:
|
|
self.log.warning(
|
|
f"No token provided for DecryptLabs vault '{vault_name}' and no global "
|
|
"decrypt_labs_api_key configured"
|
|
)
|
|
|
|
if vault_type.lower() == "sqlite":
|
|
try:
|
|
self.vaults.load_critical(vault_type, **vault_copy)
|
|
self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})")
|
|
except Exception as e:
|
|
self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}")
|
|
raise
|
|
else:
|
|
# Other vaults (MySQL, HTTP, API) - soft fail
|
|
if not self.vaults.load(vault_type, **vault_copy):
|
|
failed_vaults.append(vault_name)
|
|
self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})")
|
|
else:
|
|
self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})")
|
|
|
|
loaded_count = len(self.vaults)
|
|
if failed_vaults:
|
|
self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}")
|
|
self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults")
|
|
|
|
# Debug: Show detailed vault status
|
|
if loaded_count > 0:
|
|
vault_names = [vault.name for vault in self.vaults]
|
|
self.log.debug(f"Active vaults: {', '.join(vault_names)}")
|
|
else:
|
|
self.log.debug("No vaults are currently active")
|
|
|
|
with console.status("Loading DRM CDM...", spinner="dots"):
|
|
try:
|
|
self.cdm = self.get_cdm(self.service, self.profile)
|
|
except ValueError as e:
|
|
self.log.error(f"Failed to load CDM, {e}")
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error("load_cdm", e, service=self.service)
|
|
sys.exit(1)
|
|
|
|
if self.cdm:
|
|
cdm_info = {}
|
|
if isinstance(self.cdm, DecryptLabsRemoteCDM):
|
|
drm_type = "PlayReady" if self.cdm.is_playready else "Widevine"
|
|
self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})")
|
|
cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level}
|
|
elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]:
|
|
self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})")
|
|
cdm_info = {
|
|
"type": "Widevine",
|
|
"system_id": self.cdm.system_id,
|
|
"security_level": self.cdm.security_level,
|
|
"device_type": self.cdm.device_type.name,
|
|
}
|
|
else:
|
|
# Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None)
|
|
is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name")
|
|
if is_remote:
|
|
cdm_name = self.cdm.device_name
|
|
self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})")
|
|
else:
|
|
cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown"
|
|
self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})")
|
|
cdm_info = {
|
|
"type": "PlayReady",
|
|
"certificate": cdm_name,
|
|
"security_level": self.cdm.security_level,
|
|
}
|
|
|
|
if cdm_info:
|
|
log_event("load_cdm", level="INFO", service=self.service, context={"cdm": cdm_info})
|
|
|
|
# A server on this machine can reach a local proxy, so only a server elsewhere is guarded
|
|
self.remote_needs_public_proxy = self.is_remote and not is_loopback(server_url(ctx.params.get("server")))
|
|
|
|
self.proxy_providers = []
|
|
if no_proxy:
|
|
ctx.params["proxy"] = None
|
|
else:
|
|
if self.remote_needs_public_proxy and proxy and proxy.lower().startswith("gluetun:"):
|
|
raise click.UsageError("Gluetun runs on your machine, so --remote cannot use it.")
|
|
if proxy_providers is not None:
|
|
self.proxy_providers = list(proxy_providers)
|
|
else:
|
|
with console.status("Loading Proxy Providers...", spinner="dots"):
|
|
if config.proxy_providers.get("basic"):
|
|
self.proxy_providers.append(Basic(**config.proxy_providers["basic"]))
|
|
# ExpressVPN/ProtonVPN auto-load when their default cookie file exists (no yaml needed)
|
|
expressvpn = ExpressVPN(**(config.proxy_providers.get("expressvpn") or {}))
|
|
if config.proxy_providers.get("expressvpn") or expressvpn.cache_path.is_file():
|
|
self.proxy_providers.append(expressvpn)
|
|
if config.proxy_providers.get("nordvpn"):
|
|
self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"]))
|
|
proton = ProtonVPN(**(config.proxy_providers.get("protonvpn") or {}))
|
|
if config.proxy_providers.get("protonvpn") or proton.cookie_path.is_file():
|
|
self.proxy_providers.append(proton)
|
|
if config.proxy_providers.get("surfsharkvpn"):
|
|
self.proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"]))
|
|
if config.proxy_providers.get("windscribevpn"):
|
|
self.proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"]))
|
|
if config.proxy_providers.get("gluetun"):
|
|
self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"]))
|
|
if binaries.HolaProxy:
|
|
self.proxy_providers.append(Hola())
|
|
for proxy_provider in self.proxy_providers:
|
|
self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}")
|
|
|
|
if proxy:
|
|
|
|
def log_proxy_used(provider: object, uri: str) -> None:
|
|
self.log.info(
|
|
f"Using {provider.__class__.__name__} Proxy: {mask_proxy(uri, isinstance(provider, Basic))}"
|
|
)
|
|
|
|
requested_provider = None
|
|
if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE):
|
|
# requesting proxy from a specific proxy provider
|
|
requested_provider, proxy = proxy.split(":", maxsplit=1)
|
|
# Match simple region codes (us, ca, uk1, us:ny) or provider:region (nordvpn:ca, protonvpn:us:ny).
|
|
# ':' is allowed as a city separator (e.g. nordvpn:us:seattle, protonvpn:de:berlin).
|
|
if re.match(r"^[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE) or re.match(
|
|
r"^[a-z]+:[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE
|
|
):
|
|
proxy = proxy.lower()
|
|
# Preserve the original user query (region code) for service-specific proxy_map overrides.
|
|
# NOTE: `proxy` may be overwritten with the resolved proxy URI later.
|
|
proxy_query = proxy
|
|
status_msg = (
|
|
f"Connecting to VPN ({proxy})..."
|
|
if requested_provider == "gluetun"
|
|
else f"Getting a Proxy to {proxy}..."
|
|
)
|
|
with console.status(status_msg, spinner="dots"):
|
|
if requested_provider:
|
|
proxy_provider = next(
|
|
(
|
|
x
|
|
for x in self.proxy_providers
|
|
if x.__class__.__name__.lower() == requested_provider.lower()
|
|
),
|
|
None,
|
|
)
|
|
if not proxy_provider:
|
|
self.log.error(f"The proxy provider '{requested_provider}' was not recognised.")
|
|
sys.exit(1)
|
|
proxy_uri = proxy_provider.get_proxy(proxy)
|
|
if not proxy_uri:
|
|
self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}")
|
|
sys.exit(1)
|
|
proxy = ctx.params["proxy"] = proxy_uri
|
|
# Show connection info for Gluetun (IP, location) instead of proxy URL
|
|
if hasattr(proxy_provider, "get_connection_info"):
|
|
conn_info = proxy_provider.get_connection_info(proxy_query)
|
|
if conn_info and conn_info.get("public_ip"):
|
|
location_parts = [conn_info.get("city"), conn_info.get("country")]
|
|
location = ", ".join(p for p in location_parts if p)
|
|
self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})")
|
|
else:
|
|
log_proxy_used(proxy_provider, proxy)
|
|
else:
|
|
display = None
|
|
if hasattr(proxy_provider, "last_connection_display"):
|
|
display = proxy_provider.last_connection_display()
|
|
if display:
|
|
self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy {display}")
|
|
else:
|
|
log_proxy_used(proxy_provider, proxy)
|
|
else:
|
|
for proxy_provider in self.proxy_providers:
|
|
proxy_uri = proxy_provider.get_proxy(proxy)
|
|
if proxy_uri:
|
|
proxy = ctx.params["proxy"] = proxy_uri
|
|
# Show connection info for Gluetun (IP, location) instead of proxy URL
|
|
if hasattr(proxy_provider, "get_connection_info"):
|
|
conn_info = proxy_provider.get_connection_info(proxy_query)
|
|
if conn_info and conn_info.get("public_ip"):
|
|
location_parts = [conn_info.get("city"), conn_info.get("country")]
|
|
location = ", ".join(p for p in location_parts if p)
|
|
self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})")
|
|
else:
|
|
log_proxy_used(proxy_provider, proxy)
|
|
else:
|
|
display = None
|
|
if hasattr(proxy_provider, "last_connection_display"):
|
|
display = proxy_provider.last_connection_display()
|
|
if display:
|
|
self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy {display}")
|
|
else:
|
|
log_proxy_used(proxy_provider, proxy)
|
|
break
|
|
# Store proxy query info for service-specific overrides
|
|
ctx.params["proxy_query"] = proxy_query
|
|
ctx.params["proxy_provider"] = requested_provider
|
|
else:
|
|
self.log.info(f"Using explicit Proxy: {mask_proxy(ctx.params['proxy'])}")
|
|
# For explicit proxies, store None for query/provider
|
|
ctx.params["proxy_query"] = None
|
|
ctx.params["proxy_provider"] = None
|
|
|
|
if self.remote_needs_public_proxy and ctx.params.get("proxy") and is_loopback(ctx.params["proxy"]):
|
|
raise click.UsageError("That proxy is on your machine, so --remote cannot use it.")
|
|
|
|
ctx.obj = ContextData(
|
|
config=self.service_config, cdm=self.cdm, proxy_providers=self.proxy_providers, profile=self.profile
|
|
)
|
|
|
|
if repack:
|
|
config.repack = True
|
|
|
|
if tag:
|
|
config.tag = tag
|
|
config.tag_rules = []
|
|
|
|
# needs to be added this way instead of @cli.result_callback to be
|
|
# able to keep `self` as the first positional
|
|
self.cli._result_callback = self.result
|
|
|
|
@with_task_temp
|
|
def result(
|
|
self,
|
|
service: Service,
|
|
quality: list[int],
|
|
vcodec: list[Video.Codec],
|
|
acodec: list[Audio.Codec],
|
|
vbitrate: int,
|
|
abitrate: int,
|
|
vbitrate_range: Optional[str],
|
|
abitrate_range: Optional[str],
|
|
range_: list[Video.Range],
|
|
channels: float,
|
|
no_atmos: bool,
|
|
select_titles: bool,
|
|
wanted: list[str],
|
|
latest_episode: bool,
|
|
lang: list[str],
|
|
v_lang: list[str],
|
|
a_lang: list[str],
|
|
s_lang: list[str],
|
|
require_audio: list[str],
|
|
require_video: list[str],
|
|
require_subs: list[str],
|
|
forced_subs: bool,
|
|
forced_s_lang: list[str],
|
|
exact_lang: bool,
|
|
sub_format: Optional[Union[Subtitle.Codec, str]],
|
|
video_only: bool,
|
|
audio_only: bool,
|
|
subs_only: bool,
|
|
chapters_only: bool,
|
|
no_subs: bool,
|
|
skip_subtitle_errors: bool,
|
|
no_audio: bool,
|
|
no_chapters: bool,
|
|
no_video: bool,
|
|
no_attachments: bool,
|
|
audio_description: bool,
|
|
slow: Optional[tuple[int, int]],
|
|
list_: bool,
|
|
list_titles: bool,
|
|
skip_dl: bool,
|
|
export: bool,
|
|
cdm_only: Optional[bool],
|
|
no_proxy: bool,
|
|
no_proxy_download: bool,
|
|
no_folder: bool,
|
|
no_source: bool,
|
|
no_mux: bool,
|
|
workers: Optional[int],
|
|
adaptive_workers: bool,
|
|
download_processes: int,
|
|
continue_downloads: bool,
|
|
downloads: int,
|
|
worst: bool,
|
|
best_available: bool,
|
|
speed_limit: Optional[str] = None,
|
|
split_audio: Optional[bool] = None,
|
|
merge_video: Optional[bool] = None,
|
|
real_video_bitrate: bool = False,
|
|
real_audio_bitrate: bool = False,
|
|
progress_sink: Optional[Callable[[dict[str, Any]], None]] = None,
|
|
postscript: Sequence[str] = (),
|
|
*_: Any,
|
|
**__: Any,
|
|
) -> None:
|
|
if continue_downloads:
|
|
config.continue_downloads = True
|
|
self.tmdb_searched = False
|
|
self.search_source = None
|
|
self.service_anime = bool(getattr(service, "ANIME", False))
|
|
self.service_daily = bool(getattr(service, "DAILY", False))
|
|
self.server_cdm = getattr(service, "_server_cdm", False)
|
|
if self.server_cdm and self.cdm_override:
|
|
self.log.warning("--cdm is ignored: this remote service licenses with the server CDM")
|
|
self._remote_service = service if hasattr(service, "_server_cdm") else None
|
|
start_time = time.time()
|
|
|
|
lang, lang_excl = partition_exclusions(lang)
|
|
v_lang, v_lang_excl = partition_exclusions(v_lang)
|
|
a_lang, a_lang_excl = partition_exclusions(a_lang)
|
|
s_lang, s_lang_excl = partition_exclusions(s_lang)
|
|
forced_s_lang, fsl_excl = partition_exclusions(forced_s_lang)
|
|
for flag, excludes in (
|
|
("-l/--lang", lang_excl),
|
|
("-vl/--v-lang", v_lang_excl),
|
|
("-al/--a-lang", a_lang_excl),
|
|
("-sl/--s-lang", s_lang_excl),
|
|
("-fsl/--forced-s-lang", fsl_excl),
|
|
):
|
|
if any(token.lower() == "all" for token in excludes):
|
|
self.log.error(f"{flag}: 'all' cannot be excluded. Name the languages to exclude instead.")
|
|
sys.exit(1)
|
|
# exclusion-only use falls back to the flag's own default; -vl/-al have none, they cascade to -l
|
|
if lang_excl and not lang:
|
|
lang = ["orig"]
|
|
if s_lang_excl and not s_lang:
|
|
s_lang = ["all"]
|
|
# an override that names languages replaces -l entirely, so it must not inherit its exclusions
|
|
video_excl = list(dict.fromkeys(v_lang_excl + ([] if v_lang else lang_excl)))
|
|
audio_excl = list(dict.fromkeys(a_lang_excl + ([] if a_lang else lang_excl)))
|
|
for flag, tokens, excludes in (
|
|
("--require-audio", require_audio, audio_excl),
|
|
("--require-video", require_video, video_excl),
|
|
("--require-subs", require_subs, s_lang_excl),
|
|
):
|
|
required, own_excl = partition_exclusions(tokens)
|
|
clash = own_excl + [t for t in required if any(t.lower() == e.lower() for e in excludes)]
|
|
if clash:
|
|
self.log.error(f"{flag}: {clash[0]} cannot be required and excluded at the same time.")
|
|
sys.exit(1)
|
|
|
|
ctx = getattr(service, "ctx", None)
|
|
parent_params = ctx.parent.params if ctx and ctx.parent else None
|
|
if parent_params is not None:
|
|
# services read the raw ctx params, they must never see a '-xx' token
|
|
for name, includes, excludes in (
|
|
("lang", lang, lang_excl),
|
|
("v_lang", v_lang, v_lang_excl),
|
|
("a_lang", a_lang, a_lang_excl),
|
|
("s_lang", s_lang, s_lang_excl),
|
|
("forced_s_lang", forced_s_lang, fsl_excl),
|
|
):
|
|
if excludes and name in parent_params:
|
|
parent_params[name] = includes
|
|
|
|
if forced_s_lang or fsl_excl:
|
|
# services read forced_subs from raw ctx params, so the implication must be pushed there too
|
|
forced_subs = True
|
|
if "all" in forced_s_lang:
|
|
forced_s_lang = []
|
|
if parent_params is not None:
|
|
parent_params["forced_subs"] = True
|
|
|
|
if skip_dl:
|
|
DOWNLOAD_LICENCE_ONLY.set()
|
|
|
|
try:
|
|
speed_limit_bps = parse_speed_limit(speed_limit)
|
|
except ValueError as e:
|
|
self.log.error(str(e))
|
|
sys.exit(1)
|
|
set_speed_limit(speed_limit_bps)
|
|
if speed_limit_bps:
|
|
self.log.info(f"Speed limit: {format_speed(speed_limit_bps)}")
|
|
|
|
if export:
|
|
config.directories.exports.mkdir(parents=True, exist_ok=True)
|
|
export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json"
|
|
self.export_service = service
|
|
else:
|
|
export_path = None
|
|
|
|
vbitrate_min, vbitrate_max = None, None
|
|
if vbitrate_range:
|
|
if vbitrate and vbitrate_range:
|
|
self.log.error("Cannot use both --vbitrate and --vbitrate-range at the same time.")
|
|
sys.exit(1)
|
|
try:
|
|
parts = vbitrate_range.split("-")
|
|
if len(parts) != 2:
|
|
raise ValueError
|
|
vbitrate_min, vbitrate_max = int(parts[0]), int(parts[1])
|
|
if vbitrate_min > vbitrate_max:
|
|
vbitrate_min, vbitrate_max = vbitrate_max, vbitrate_min
|
|
except (ValueError, IndexError):
|
|
self.log.error("Invalid --vbitrate-range format. Use 'MIN-MAX' (e.g., '6000-7000').")
|
|
sys.exit(1)
|
|
|
|
abitrate_min, abitrate_max = None, None
|
|
if abitrate_range:
|
|
if abitrate and abitrate_range:
|
|
self.log.error("Cannot use both --abitrate and --abitrate-range at the same time.")
|
|
sys.exit(1)
|
|
try:
|
|
parts = abitrate_range.split("-")
|
|
if len(parts) != 2:
|
|
raise ValueError
|
|
abitrate_min, abitrate_max = int(parts[0]), int(parts[1])
|
|
if abitrate_min > abitrate_max:
|
|
abitrate_min, abitrate_max = abitrate_max, abitrate_min
|
|
except (ValueError, IndexError):
|
|
self.log.error("Invalid --abitrate-range format. Use 'MIN-MAX' (e.g., '128-256').")
|
|
sys.exit(1)
|
|
|
|
if not acodec:
|
|
acodec = []
|
|
elif isinstance(acodec, Audio.Codec):
|
|
acodec = [acodec]
|
|
elif isinstance(acodec, str) or (
|
|
isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec)
|
|
):
|
|
acodec = AUDIO_CODEC_LIST.convert(acodec)
|
|
|
|
if worst and not quality:
|
|
self.log.error("--worst requires -q/--quality to be specified")
|
|
sys.exit(1)
|
|
|
|
if select_titles and wanted:
|
|
self.log.error("--select-titles and -w/--wanted cannot be used together")
|
|
sys.exit(1)
|
|
|
|
if any(r == Video.Range.HYBRID for r in range_):
|
|
from unshackle.core.binaries import DoviTool
|
|
|
|
if not DoviTool:
|
|
self.log.error("Unable to run hybrid mode: dovi_tool not detected")
|
|
self.log.error("Please install dovi_tool from https://github.com/quietvoid/dovi_tool")
|
|
sys.exit(1)
|
|
|
|
if cdm_only is None:
|
|
vaults_only = None
|
|
else:
|
|
vaults_only = not cdm_only
|
|
|
|
config.decryption = resolve_decryption(
|
|
config.decryption_map, config.decryption, service.__class__.__name__.upper()
|
|
)
|
|
|
|
log_event(
|
|
"drm_mode_config",
|
|
level="DEBUG",
|
|
service=self.service,
|
|
context={
|
|
"cdm_only": cdm_only,
|
|
"vaults_only": vaults_only,
|
|
"mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"),
|
|
"decryption": config.decryption,
|
|
},
|
|
)
|
|
|
|
with console.status(
|
|
"Preparing Remote Service Session..." if self.is_remote else "Preparing Service Session...",
|
|
spinner="dots",
|
|
):
|
|
try:
|
|
cookies = self.get_cookie_jar(self.service, self.profile)
|
|
credential = self.get_credentials(self.service, self.profile)
|
|
service.authenticate(cookies, credential)
|
|
if cookies or credential:
|
|
self.log.info("Authenticated with Service")
|
|
log_event(
|
|
"authenticate",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"has_cookies": bool(cookies),
|
|
"has_credentials": bool(credential),
|
|
"profile": self.profile,
|
|
},
|
|
)
|
|
except Exception as e:
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"authenticate", e, service=self.service, context={"profile": self.profile}
|
|
)
|
|
raise
|
|
|
|
with console.status(
|
|
"Fetching Remote Title Metadata..." if self.is_remote else "Fetching Title Metadata...", spinner="dots"
|
|
):
|
|
try:
|
|
titles = service.get_titles_cached()
|
|
if not titles:
|
|
self.log.error("No titles returned, nothing to download...")
|
|
log_event(
|
|
"get_titles",
|
|
level="ERROR",
|
|
service=self.service,
|
|
message="No titles returned from service",
|
|
success=False,
|
|
)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error("get_titles", e, service=self.service)
|
|
raise
|
|
|
|
if self.debug_logger:
|
|
titles_info = {
|
|
"type": titles.__class__.__name__,
|
|
"count": len(titles) if hasattr(titles, "__len__") else 1,
|
|
"title": str(titles),
|
|
}
|
|
if hasattr(titles, "seasons"):
|
|
titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0
|
|
self.debug_logger.log(
|
|
level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info}
|
|
)
|
|
|
|
title_cacher = service.title_cache if hasattr(service, "title_cache") else None
|
|
cache_title_id = None
|
|
if hasattr(service, "title"):
|
|
cache_title_id = service.title
|
|
elif hasattr(service, "title_id"):
|
|
cache_title_id = service.title_id
|
|
cache_region = service.current_region if hasattr(service, "current_region") else None
|
|
cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None
|
|
|
|
if self.enrich:
|
|
sample_title = titles[0] if hasattr(titles, "__getitem__") else titles
|
|
kind = "tv" if isinstance(sample_title, Episode) else "movie"
|
|
|
|
enrich_title: Optional[str] = None
|
|
enrich_year: Optional[int] = None
|
|
enrich_lang: Optional[Language] = None
|
|
|
|
enrich_result = providers.resolve_by_ids(
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
kind=kind,
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
anime=self.anime_hint(sample_title),
|
|
)
|
|
if enrich_result:
|
|
enrich_title = enrich_result.title
|
|
enrich_year = enrich_result.year
|
|
enrich_lang = parse_language(enrich_result.original_language)
|
|
|
|
if enrich_lang:
|
|
self.log.info(f"Original language from metadata: {enrich_lang}")
|
|
if not (enrich_title or enrich_year or enrich_lang):
|
|
self.log.warning("--enrich found no metadata. Is the provider's API key configured?")
|
|
elif missing := [
|
|
name
|
|
for name, value in (("title", enrich_title), ("year", enrich_year), ("language", enrich_lang))
|
|
if not value
|
|
]:
|
|
self.log.warning(f"--enrich source gave no {', '.join(missing)}. Those fields are unchanged.")
|
|
|
|
if enrich_title or enrich_year or enrich_lang:
|
|
for t in titles if isinstance(titles, (Series, Movies)) else [titles]:
|
|
if enrich_title:
|
|
if isinstance(t, Episode):
|
|
t.title = enrich_title
|
|
else:
|
|
t.name = enrich_title
|
|
if enrich_year:
|
|
t.year = enrich_year
|
|
if enrich_lang:
|
|
t.language = enrich_lang
|
|
|
|
if isinstance(titles, Series):
|
|
enrich_tvdb_id = self.tvdb_id or (enrich_result.external_ids.tvdb_id if enrich_result else None)
|
|
self.fill_absolute_numbers(titles, enrich_tvdb_id)
|
|
if any(self.daily_hint(t) for t in titles):
|
|
self.fill_air_dates(titles, enrich_tvdb_id)
|
|
|
|
if self.tvdb_order and isinstance(titles, Series):
|
|
titles = self.apply_tvdb_order(titles, title_cacher, cache_title_id, cache_region, cache_account_hash)
|
|
|
|
console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2)))
|
|
console.print(Padding(titles.tree(verbose=list_titles), (0, 5)))
|
|
|
|
if list_titles:
|
|
return
|
|
|
|
if select_titles and isinstance(titles, Movies) and len(titles) > 1:
|
|
console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2)))
|
|
|
|
selection_start = time.time()
|
|
selected_idx = select_multiple(
|
|
[str(movie) for movie in titles],
|
|
minimal_count=1,
|
|
page_size=8,
|
|
return_indices=True,
|
|
)
|
|
if not selected_idx:
|
|
console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5)))
|
|
return
|
|
start_time += time.time() - selection_start
|
|
|
|
keep = set(selected_idx)
|
|
for i in range(len(titles) - 1, -1, -1):
|
|
if i not in keep:
|
|
del titles[i]
|
|
|
|
console.print(Padding(f"[text]Total selected: {len(titles)}[/]", (0, 5)))
|
|
|
|
# Enables manual selection for Series when --select-titles is set
|
|
if select_titles and isinstance(titles, Series):
|
|
console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2)))
|
|
|
|
selection_titles = []
|
|
dependencies = {}
|
|
original_indices = {}
|
|
|
|
current_season = None
|
|
current_season_header_idx = -1
|
|
|
|
unique_seasons = {t.season for t in titles}
|
|
multiple_seasons = len(unique_seasons) > 1
|
|
|
|
# Build selection options
|
|
for i, t in enumerate(titles):
|
|
# Insert season header only if multiple seasons exist
|
|
if multiple_seasons and t.season != current_season:
|
|
current_season = t.season
|
|
header_text = f"Season {t.season}"
|
|
selection_titles.append(header_text)
|
|
current_season_header_idx = len(selection_titles) - 1
|
|
dependencies[current_season_header_idx] = []
|
|
|
|
display_name = ((t.name[:30].rstrip() + "...") if len(t.name) > 30 else t.name) if t.name else None
|
|
|
|
prefix = " " if multiple_seasons else ""
|
|
part_label = f".{t.part}" if t.part is not None else ""
|
|
option_text = f"{prefix}{t.number}{part_label}" + (f". {display_name}" if t.name else "")
|
|
|
|
selection_titles.append(option_text)
|
|
current_ui_idx = len(selection_titles) - 1
|
|
|
|
original_indices[current_ui_idx] = i
|
|
|
|
if current_season_header_idx != -1:
|
|
dependencies[current_season_header_idx].append(current_ui_idx)
|
|
|
|
selection_start = time.time()
|
|
|
|
selected_ui_idx = select_multiple(
|
|
selection_titles,
|
|
minimal_count=1,
|
|
page_size=8,
|
|
return_indices=True,
|
|
dependencies=dependencies,
|
|
collapse_on_start=multiple_seasons,
|
|
)
|
|
|
|
if not selected_ui_idx:
|
|
console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5)))
|
|
return
|
|
|
|
selection_end = time.time()
|
|
start_time += selection_end - selection_start
|
|
|
|
selected_idx = []
|
|
for idx in selected_ui_idx:
|
|
if idx in original_indices:
|
|
selected_idx.append(original_indices[idx])
|
|
|
|
# Ensure indices are unique and ordered
|
|
selected_idx = sorted(set(selected_idx))
|
|
keep = set(selected_idx)
|
|
|
|
for i in range(len(titles) - 1, -1, -1):
|
|
if i not in keep:
|
|
del titles[i]
|
|
|
|
if titles:
|
|
count = len(titles)
|
|
console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5)))
|
|
|
|
latest_episode_id = None
|
|
if latest_episode and isinstance(titles, Series) and len(titles) > 0:
|
|
# Series is already sorted by (season, number, year)
|
|
# The last episode in the sorted list is the latest
|
|
latest_ep = titles[-1]
|
|
latest_episode_id = f"{latest_ep.season}x{latest_ep.number}"
|
|
self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}")
|
|
|
|
def select_best_audio(
|
|
audio: list[Audio],
|
|
languages: list[str],
|
|
acodec: list[Audio.Codec],
|
|
audio_description: bool,
|
|
exact_lang: bool,
|
|
) -> list[Audio]:
|
|
"""Pick the best audio per (language, codec) group from a sort_audio-ordered list.
|
|
|
|
"Best" is the first track of each order-preserving candidate filter, so the
|
|
sort_audio ranking (descriptive-last, Atmos, codec_priority, bitrate) decides the
|
|
winner. With audio_description both the best standard and best descriptive track per
|
|
group are kept. "best"/"all" tokens select every language present in sorted order.
|
|
"""
|
|
select_all = any(tok in ("best", "all") for tok in languages)
|
|
if select_all:
|
|
target_langs: list[Any] = list(dict.fromkeys(t.language for t in audio))
|
|
else:
|
|
target_langs = languages
|
|
codecs: list[Optional[Audio.Codec]] = list(acodec) if len(acodec) > 1 else [None]
|
|
|
|
selected: list[Audio] = []
|
|
for language in target_langs:
|
|
for codec in codecs:
|
|
if select_all:
|
|
candidates = [
|
|
t for t in audio if t.language == language and (codec is None or t.codec == codec)
|
|
]
|
|
else:
|
|
pool = [t for t in audio if codec is None or t.codec == codec]
|
|
candidates = Tracks.by_language(pool, [language], per_language=0, exact_match=exact_lang)
|
|
if not candidates:
|
|
continue
|
|
if audio_description:
|
|
standards = [t for t in candidates if not t.descriptive]
|
|
if standards:
|
|
selected.append(standards[0])
|
|
descs = [t for t in candidates if t.descriptive]
|
|
if descs:
|
|
selected.append(descs[0])
|
|
else:
|
|
selected.append(candidates[0])
|
|
# Built via per-tag by_language calls, so overlapping tags can append
|
|
# the same track across iterations; dedupe by id.
|
|
seen_ids: set[str] = set()
|
|
return [t for t in selected if not (t.id in seen_ids or seen_ids.add(t.id))]
|
|
|
|
base_selection = (v_lang, a_lang, s_lang, range_)
|
|
|
|
def post_script_queued(candidate: Any) -> bool:
|
|
"""Mirror of the filters the loop below applies, so the season counter matches it."""
|
|
if isinstance(candidate, Episode) and latest_episode and latest_episode_id:
|
|
return f"{candidate.season}x{candidate.number}" == latest_episode_id
|
|
return title_wanted(candidate, wanted)
|
|
|
|
post_script_pending: dict[Any, int] = {}
|
|
for candidate in titles:
|
|
if post_script_queued(candidate):
|
|
key = post_script_group(candidate)
|
|
post_script_pending[key] = post_script_pending.get(key, 0) + 1
|
|
post_script_last: dict[Any, dict[Path, dict[str, str]]] = {}
|
|
post_script_folders: list[Path] = []
|
|
post_script_sample: dict[str, str] = {}
|
|
|
|
for i, title in enumerate(titles):
|
|
v_lang, a_lang, s_lang, range_ = base_selection
|
|
if isinstance(title, Episode) and latest_episode and latest_episode_id:
|
|
# If --latest-episode is set, only process the latest episode
|
|
if f"{title.season}x{title.number}" != latest_episode_id:
|
|
continue
|
|
elif not title_wanted(title, wanted):
|
|
continue
|
|
|
|
if progress_sink:
|
|
if isinstance(title, Episode):
|
|
progress_sink(
|
|
{
|
|
"title": title.title,
|
|
"current_title": f"S{title.season or 0:02}E{title.number or 0:02}"
|
|
+ (f".{title.part}" if title.part is not None else ""),
|
|
}
|
|
)
|
|
else:
|
|
progress_sink({"title": getattr(title, "name", None) or str(title)})
|
|
|
|
title_rule = f"Track {title.track:02}: {title.name}" if isinstance(title, Song) else str(title)
|
|
console.print(Padding(Rule(f"[rule.text]{title_rule}"), (1, 2)))
|
|
temp_font_files = []
|
|
|
|
if isinstance(title, Episode) and not self.tmdb_searched:
|
|
kind = "tv"
|
|
tmdb_title: Optional[str] = None
|
|
has_ids = bool(self.tmdb_id or self.imdb_id or self.tvdb_id or self.anilist_id)
|
|
result = providers.resolve_by_ids(
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
title=title.title,
|
|
year=title.year,
|
|
kind=kind,
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
anime=self.anime_hint(title),
|
|
)
|
|
if result:
|
|
if has_ids:
|
|
tmdb_title = result.title
|
|
if not self.tmdb_id:
|
|
self.tmdb_id = result.external_ids.tmdb_id
|
|
elif result.title and providers.fuzzy_match(result.title, title.title):
|
|
self.tmdb_id = result.external_ids.tmdb_id
|
|
tmdb_title = result.title
|
|
self.search_source = result.source
|
|
if list_ or list_titles:
|
|
if self.tmdb_id:
|
|
console.print(
|
|
Padding(
|
|
f"Search -> {tmdb_title or '?'} [bright_black](ID {self.tmdb_id})",
|
|
(0, 5),
|
|
)
|
|
)
|
|
else:
|
|
console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5)))
|
|
self.tmdb_searched = True
|
|
|
|
if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id:
|
|
movie_result = providers.resolve_by_ids(
|
|
None,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
title=title.name,
|
|
year=title.year,
|
|
kind="movie",
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
anime=self.anime_hint(title),
|
|
)
|
|
if movie_result and movie_result.external_ids.tmdb_id:
|
|
console.print(
|
|
Padding(
|
|
f"Search -> {movie_result.title or '?'} "
|
|
f"[bright_black](ID {movie_result.external_ids.tmdb_id})",
|
|
(0, 5),
|
|
)
|
|
)
|
|
else:
|
|
console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5)))
|
|
|
|
if self.tmdb_id and getattr(self, "search_source", None) not in ("simkl", "imdb"):
|
|
kind = "tv" if isinstance(title, Episode) else "movie"
|
|
providers.fetch_external_ids(
|
|
self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash
|
|
)
|
|
|
|
if slow and i != 0:
|
|
delay = random.randint(slow[0], slow[1])
|
|
spinner = Spinner("dots", text=f"Delaying by {delay} seconds...")
|
|
with SyncLive(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True):
|
|
for remaining in range(delay, 0, -1):
|
|
spinner.update(text=f"Delaying by {remaining} seconds...")
|
|
time.sleep(1)
|
|
|
|
with console.status("Subscribing to events...", spinner="dots"):
|
|
events.reset()
|
|
events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded)
|
|
events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded)
|
|
events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted)
|
|
events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked)
|
|
events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex)
|
|
|
|
tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..."
|
|
with console.status(tracks_label, spinner="dots"):
|
|
try:
|
|
title.tracks.add(service.get_tracks(title), warn_only=True)
|
|
title.tracks.chapters = service.get_chapters(title)
|
|
except Exception as e:
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"get_tracks", e, service=self.service, context={"title": str(title)}
|
|
)
|
|
raise
|
|
|
|
if self.debug_logger:
|
|
tracks_info = {
|
|
"title": str(title),
|
|
"video_tracks": len(title.tracks.videos),
|
|
"audio_tracks": len(title.tracks.audio),
|
|
"subtitle_tracks": len(title.tracks.subtitles),
|
|
"has_chapters": bool(title.tracks.chapters),
|
|
"videos": [
|
|
{
|
|
"codec": str(v.codec),
|
|
"resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown",
|
|
"bitrate": v.bitrate,
|
|
"range": str(v.range),
|
|
"language": str(v.language) if v.language else None,
|
|
"drm": [str(type(d).__name__) for d in v.drm] if v.drm else [],
|
|
}
|
|
for v in title.tracks.videos
|
|
],
|
|
"audio": [
|
|
{
|
|
"codec": str(a.codec),
|
|
"bitrate": a.bitrate,
|
|
"channels": a.channels,
|
|
"language": str(a.language) if a.language else None,
|
|
"descriptive": a.descriptive,
|
|
"drm": [str(type(d).__name__) for d in a.drm] if a.drm else [],
|
|
}
|
|
for a in title.tracks.audio
|
|
],
|
|
"subtitles": [
|
|
{
|
|
"codec": str(s.codec),
|
|
"language": str(s.language) if s.language else None,
|
|
"forced": s.forced,
|
|
"sdh": s.sdh,
|
|
}
|
|
for s in title.tracks.subtitles
|
|
],
|
|
}
|
|
self.debug_logger.log(
|
|
level="INFO", operation="get_tracks", service=self.service, context=tracks_info
|
|
)
|
|
|
|
# strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available
|
|
# uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available
|
|
# Check if automatic SDH stripping is enabled in config
|
|
if config.subtitle.get("strip_sdh", True):
|
|
for subtitle in title.tracks.subtitles:
|
|
if subtitle.sdh and not any(
|
|
is_close_match(subtitle.language, [x.language])
|
|
for x in title.tracks.subtitles
|
|
if not x.sdh and not x.forced
|
|
):
|
|
non_sdh_sub = deepcopy(subtitle)
|
|
non_sdh_sub.id += "_stripped"
|
|
non_sdh_sub.sdh = False
|
|
title.tracks.add(non_sdh_sub)
|
|
events.subscribe(
|
|
events.Types.TRACK_MULTIPLEX,
|
|
lambda track, sub_id=non_sdh_sub.id: (
|
|
(track.strip_hearing_impaired()) if track.id == sub_id else None
|
|
),
|
|
)
|
|
|
|
if real_video_bitrate:
|
|
with console.status("Probing real video bitrates...", spinner="dots"):
|
|
apply_real_bitrates(
|
|
title.tracks.videos,
|
|
service.session,
|
|
log=self.log,
|
|
group_key=lambda t: (t.codec, t.range),
|
|
)
|
|
|
|
if real_audio_bitrate:
|
|
with console.status("Probing real audio bitrates...", spinner="dots"):
|
|
apply_real_bitrates(
|
|
title.tracks.audio,
|
|
service.session,
|
|
log=self.log,
|
|
group_key=lambda t: (t.codec, t.channels, str(t.language), t.descriptive),
|
|
)
|
|
|
|
with console.status("Sorting tracks by language and bitrate...", spinner="dots"):
|
|
audio_priority = config.audio.get("language_priority") or []
|
|
subtitle_priority = config.subtitle.get("language_priority") or []
|
|
|
|
title.tracks.sort_videos(
|
|
by_language=resolve_sort_langs(v_lang or lang, title.language),
|
|
exact_match=exact_lang,
|
|
)
|
|
title.tracks.sort_audio(
|
|
by_language=resolve_sort_langs([*audio_priority, *(a_lang or lang)], title.language),
|
|
codec_priority=config.audio.get("codec_priority"),
|
|
exact_match=exact_lang,
|
|
)
|
|
title.tracks.sort_subtitles(
|
|
by_language=resolve_sort_langs([*subtitle_priority, *(s_lang or [])], title.language),
|
|
type_priority=config.subtitle.get("type_priority"),
|
|
group_by=config.subtitle.get("group_by"),
|
|
exact_match=exact_lang,
|
|
)
|
|
|
|
if list_:
|
|
available_tracks, _ = title.tracks.tree()
|
|
console.print(Padding(listing_panel(available_tracks, "Available Tracks"), (0, 5)))
|
|
continue
|
|
|
|
keep_videos = True
|
|
keep_audio = True
|
|
keep_subtitles = True
|
|
keep_chapters = True
|
|
|
|
if video_only or audio_only or subs_only or chapters_only:
|
|
keep_videos = video_only
|
|
keep_audio = audio_only
|
|
keep_subtitles = subs_only
|
|
keep_chapters = chapters_only
|
|
|
|
if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES:
|
|
console.log("Skipping subtitles - service does not support subtitle downloads")
|
|
keep_subtitles = False
|
|
s_lang = None
|
|
elif no_subs:
|
|
console.log("Skipped subtitles as --no-subs was used...")
|
|
keep_subtitles = False
|
|
s_lang = None
|
|
|
|
if no_video:
|
|
console.log("Skipped video as --no-video was used...")
|
|
keep_videos = False
|
|
v_lang = None
|
|
|
|
if no_audio:
|
|
console.log("Skipped audio as --no-audio was used...")
|
|
keep_audio = False
|
|
a_lang = None
|
|
|
|
if no_chapters:
|
|
console.log("Skipped chapters as --no-chapters was used...")
|
|
keep_chapters = False
|
|
|
|
if no_attachments:
|
|
console.log("Skipped attachments as --no-attachments was used...")
|
|
|
|
if no_proxy_download and any(service.session.proxies.values()):
|
|
console.log("Bypassing proxy for downloads as --no-proxy-download was used...")
|
|
|
|
for kind, required, available in (
|
|
(
|
|
"audio",
|
|
require_audio if keep_audio else [],
|
|
[a.language for a in title.tracks.audio] + embedded_audio_langs(title.tracks.videos, keep_videos),
|
|
),
|
|
("video", require_video if keep_videos else [], [v.language for v in title.tracks.videos]),
|
|
(
|
|
"subtitle",
|
|
require_subs if keep_subtitles else [],
|
|
[t.language for t in title.tracks.subtitles],
|
|
),
|
|
):
|
|
missing_required = missing_required_langs(required, available, title.language, exact=exact_lang)
|
|
if missing_required:
|
|
self.log.error(f"Required {kind} language(s) not found: {', '.join(missing_required)}")
|
|
sys.exit(1)
|
|
|
|
if not keep_videos:
|
|
title.tracks.videos = []
|
|
if not keep_audio:
|
|
title.tracks.audio = []
|
|
if not keep_subtitles:
|
|
title.tracks.subtitles = []
|
|
if not keep_chapters:
|
|
title.tracks.chapters = []
|
|
|
|
with console.status("Selecting tracks...", spinner="dots"):
|
|
|
|
def resolve_excludes(excludes: list[str], _title: Title_T = title) -> list[str]:
|
|
"""Change 'orig' to this title's language. An 'orig' with no language excludes nothing."""
|
|
resolved: list[str] = []
|
|
for token in excludes:
|
|
value = str(_title.language) if token == "orig" else token
|
|
if token == "orig" and not _title.language:
|
|
continue
|
|
if value not in resolved:
|
|
resolved.append(value)
|
|
return resolved
|
|
|
|
video_excl_r = resolve_excludes(video_excl)
|
|
audio_excl_r = resolve_excludes(audio_excl)
|
|
s_excl_r = resolve_excludes(s_lang_excl)
|
|
fsl_excl_r = resolve_excludes(fsl_excl)
|
|
|
|
if isinstance(title, (Movie, Episode)):
|
|
if keep_videos and vcodec:
|
|
title.tracks.select_video(lambda x: x.codec in vcodec)
|
|
missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)]
|
|
for codec in missing_codecs:
|
|
self.log.warning(f"Skipping {codec.name} video tracks as none are available.")
|
|
if not title.tracks.videos:
|
|
self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...")
|
|
sys.exit(1)
|
|
|
|
if keep_videos and range_:
|
|
# Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks
|
|
if Video.Range.HYBRID not in range_:
|
|
title.tracks.select_video(lambda x: x.range in range_)
|
|
missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)]
|
|
for color_range in missing_ranges:
|
|
self.log.warning(f"Skipping {color_range.name} video tracks as none are available.")
|
|
if not title.tracks.videos:
|
|
self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...")
|
|
sys.exit(1)
|
|
|
|
if keep_videos and vbitrate:
|
|
if any(r == Video.Range.HYBRID for r in range_):
|
|
# In HYBRID mode, only apply bitrate filter to non-DV tracks
|
|
# DV tracks are kept regardless since they're only used for RPU metadata
|
|
title.tracks.select_video(
|
|
lambda x: x.range == Video.Range.DV or (x.bitrate and x.bitrate // 1000 == vbitrate)
|
|
)
|
|
if not any(x.range != Video.Range.DV for x in title.tracks.videos):
|
|
self.log.error(f"There's no {vbitrate}kbps Video Track...")
|
|
sys.exit(1)
|
|
else:
|
|
title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate)
|
|
if not title.tracks.videos:
|
|
self.log.error(f"There's no {vbitrate}kbps Video Track...")
|
|
sys.exit(1)
|
|
|
|
if keep_videos and vbitrate_min is not None and vbitrate_max is not None:
|
|
title.tracks.select_video(
|
|
lambda x: x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max
|
|
)
|
|
if not title.tracks.videos:
|
|
self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...")
|
|
sys.exit(1)
|
|
|
|
if keep_videos and video_excl_r:
|
|
drop = excluded_language_tags(
|
|
video_excl_r, [t.language for t in title.tracks.videos], exact_lang
|
|
)
|
|
title.tracks.select_video(lambda x: str(x.language) not in drop)
|
|
if not title.tracks.videos:
|
|
self.log.error(f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...")
|
|
sys.exit(1)
|
|
|
|
effective_video_lang = v_lang or lang
|
|
video_languages = [lang for lang in effective_video_lang if lang != "best"]
|
|
video_multi_lang = (
|
|
"best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1
|
|
)
|
|
if keep_videos and video_languages and "all" not in video_languages:
|
|
processed_video_lang = []
|
|
v_orig_token: Optional[str] = None
|
|
for language in video_languages:
|
|
if language == "orig":
|
|
if title.language:
|
|
orig_lang = str(title.language)
|
|
v_orig_token = orig_lang
|
|
if orig_lang not in processed_video_lang:
|
|
processed_video_lang.append(orig_lang)
|
|
else:
|
|
self.log.warning(
|
|
"Original language not available for title, skipping 'orig' selection for video"
|
|
)
|
|
else:
|
|
if language not in processed_video_lang:
|
|
processed_video_lang.append(language)
|
|
if v_orig_token in video_languages:
|
|
v_orig_token = None
|
|
missing_v_langs = find_missing_langs(
|
|
processed_video_lang,
|
|
[v.language for v in title.tracks.videos],
|
|
exact=exact_lang,
|
|
)
|
|
if missing_v_langs:
|
|
missing_str = as_requested(missing_v_langs, v_orig_token)
|
|
if best_available:
|
|
remaining_v_langs = [tok for tok in processed_video_lang if tok not in missing_v_langs]
|
|
if remaining_v_langs:
|
|
self.log.warning(
|
|
f"{missing_str} not found in video tracks, "
|
|
f"continuing with: {as_requested(remaining_v_langs, v_orig_token)}"
|
|
)
|
|
processed_video_lang = remaining_v_langs
|
|
else:
|
|
self.log.error(f"{missing_str} not found in video tracks and no fallback available")
|
|
sys.exit(1)
|
|
else:
|
|
self.log.error(missing_str + " not found in video tracks")
|
|
sys.exit(1)
|
|
title.tracks.videos = title.tracks.by_language(
|
|
title.tracks.videos, processed_video_lang, exact_match=exact_lang
|
|
)
|
|
if not title.tracks.videos:
|
|
# empty only when 'orig' was the sole request and did not resolve
|
|
self.log.error(
|
|
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track..."
|
|
)
|
|
sys.exit(1)
|
|
|
|
has_hybrid = any(r == Video.Range.HYBRID for r in range_)
|
|
non_hybrid_ranges = [r for r in range_ if r != Video.Range.HYBRID]
|
|
effective_quality = quality
|
|
if keep_videos and quality:
|
|
missing_resolutions = []
|
|
if has_hybrid:
|
|
hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos(
|
|
title.tracks.videos, non_hybrid_ranges
|
|
)
|
|
# only ranges that can actually be delivered count towards "present"
|
|
deliverable_pool = hybrid_candidate_tracks + [
|
|
v for v in non_hybrid_tracks if v.range in non_hybrid_ranges
|
|
]
|
|
for resolution in quality:
|
|
if any(v.height == resolution for v in deliverable_pool):
|
|
continue
|
|
if any(v.width and int(v.width * 9 / 16) == resolution for v in deliverable_pool):
|
|
continue
|
|
missing_resolutions.append(resolution)
|
|
|
|
if len(missing_resolutions) == len(quality) and best_available:
|
|
# every requested resolution missing: leave the pool untouched so the
|
|
# best-available hybrid pass below selects the top resolution instead
|
|
effective_quality = []
|
|
else:
|
|
hybrid_filter = title.tracks.select_hybrid(
|
|
hybrid_candidate_tracks, quality, worst=worst
|
|
)
|
|
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
|
|
|
|
if non_hybrid_ranges and non_hybrid_tracks:
|
|
non_hybrid_selected = [
|
|
v
|
|
for v in non_hybrid_tracks
|
|
if any(
|
|
v.height == res or (v.width and int(v.width * (9 / 16)) == res)
|
|
for res in quality
|
|
)
|
|
]
|
|
title.tracks.videos = Tracks.merge_video_selections(
|
|
hybrid_selected, non_hybrid_selected
|
|
)
|
|
else:
|
|
title.tracks.videos = hybrid_selected
|
|
else:
|
|
pre_quality_videos = list(title.tracks.videos)
|
|
title.tracks.by_resolutions(quality)
|
|
|
|
for resolution in quality:
|
|
if any(v.height == resolution for v in title.tracks.videos):
|
|
continue
|
|
if any(v.width and int(v.width * 9 / 16) == resolution for v in title.tracks.videos):
|
|
continue
|
|
missing_resolutions.append(resolution)
|
|
|
|
if len(missing_resolutions) == len(quality) and best_available:
|
|
# every requested resolution missing: fall back to the best available
|
|
title.tracks.videos = pre_quality_videos
|
|
effective_quality = []
|
|
|
|
if missing_resolutions:
|
|
res_list = ""
|
|
if len(missing_resolutions) > 1:
|
|
res_list = ", ".join([f"{x}p" for x in missing_resolutions[:-1]]) + " or "
|
|
res_list = f"{res_list}{missing_resolutions[-1]}p"
|
|
plural = "s" if len(missing_resolutions) > 1 else ""
|
|
|
|
if best_available:
|
|
self.log.warning(
|
|
f"There's no {res_list} Video Track{plural}, continuing with available qualities..."
|
|
)
|
|
else:
|
|
self.log.error(f"There's no {res_list} Video Track{plural}...")
|
|
sys.exit(1)
|
|
|
|
pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else []
|
|
if keep_videos and has_hybrid:
|
|
hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos(
|
|
title.tracks.videos, non_hybrid_ranges
|
|
)
|
|
|
|
if not effective_quality:
|
|
best_resolution = max((v.height for v in hybrid_candidate_tracks), default=None)
|
|
if best_resolution:
|
|
hybrid_filter = title.tracks.select_hybrid(
|
|
hybrid_candidate_tracks, [best_resolution], worst=worst
|
|
)
|
|
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
|
|
else:
|
|
hybrid_selected = []
|
|
else:
|
|
hybrid_filter = title.tracks.select_hybrid(
|
|
hybrid_candidate_tracks, effective_quality, worst=worst
|
|
)
|
|
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
|
|
|
|
non_hybrid_selected: list[Video] = []
|
|
if non_hybrid_ranges and non_hybrid_tracks:
|
|
if video_multi_lang:
|
|
non_hybrid_langs = list(dict.fromkeys(str(v.language) for v in non_hybrid_tracks))
|
|
else:
|
|
non_hybrid_langs = [None]
|
|
for resolution, color_range, codec, vlang in product(
|
|
effective_quality or [None], non_hybrid_ranges, vcodec or [None], non_hybrid_langs
|
|
):
|
|
candidates = [
|
|
t
|
|
for t in non_hybrid_tracks
|
|
if (
|
|
not resolution
|
|
or t.height == resolution
|
|
or (t.width and int(t.width * (9 / 16)) == resolution)
|
|
)
|
|
and (not color_range or t.range == color_range)
|
|
and (not codec or t.codec == codec)
|
|
and (vlang is None or str(t.language) == vlang)
|
|
]
|
|
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
|
|
if match and match not in non_hybrid_selected:
|
|
non_hybrid_selected.append(match)
|
|
|
|
title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected)
|
|
|
|
# Flag tracks selected only as hybrid ingredients (the HDR base and/or
|
|
# the lowest DV) so the standalone mux loop skips them. Tracks also
|
|
# picked as explicit deliverables stay unflagged.
|
|
Tracks.flag_hybrid_ingredients(hybrid_selected, non_hybrid_selected)
|
|
elif keep_videos:
|
|
selected_videos: list[Video] = []
|
|
if video_multi_lang:
|
|
unique_video_langs = list(dict.fromkeys(str(v.language) for v in title.tracks.videos))
|
|
else:
|
|
unique_video_langs = [None]
|
|
for resolution, color_range, codec, vlang in product(
|
|
effective_quality or [None], range_ or [None], vcodec or [None], unique_video_langs
|
|
):
|
|
candidates = [
|
|
t
|
|
for t in title.tracks.videos
|
|
if (
|
|
not resolution
|
|
or t.height == resolution
|
|
or (t.width and int(t.width * (9 / 16)) == resolution)
|
|
)
|
|
and (not color_range or t.range == color_range)
|
|
and (not codec or t.codec == codec)
|
|
and (vlang is None or str(t.language) == vlang)
|
|
]
|
|
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
|
|
if match and match not in selected_videos:
|
|
selected_videos.append(match)
|
|
title.tracks.videos = selected_videos
|
|
|
|
if keep_videos and any(r == Video.Range.HYBRID for r in range_):
|
|
base_tracks = [
|
|
v for v in title.tracks.videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)
|
|
]
|
|
dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV]
|
|
|
|
hybrid_failed = False
|
|
if not base_tracks and not dv_tracks:
|
|
available_ranges = sorted(set(v.range.name for v in title.tracks.videos))
|
|
msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but neither is available"
|
|
msg_detail = (
|
|
f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}"
|
|
)
|
|
hybrid_failed = True
|
|
elif not base_tracks:
|
|
available_ranges = sorted(set(v.range.name for v in title.tracks.videos))
|
|
msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only DV is available"
|
|
msg_detail = f"Available ranges: {', '.join(available_ranges)}"
|
|
hybrid_failed = True
|
|
elif not dv_tracks:
|
|
available_ranges = sorted(set(v.range.name for v in title.tracks.videos))
|
|
msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only HDR10 is available"
|
|
msg_detail = f"Available ranges: {', '.join(available_ranges)}"
|
|
hybrid_failed = True
|
|
|
|
if hybrid_failed:
|
|
other_ranges = [r for r in range_ if r != Video.Range.HYBRID]
|
|
if best_available and other_ranges:
|
|
self.log.warning(msg)
|
|
self.log.warning(
|
|
f"Continuing with remaining range(s): {', '.join(r.name for r in other_ranges)}"
|
|
)
|
|
range_ = other_ranges
|
|
fallback_pool = pre_hybrid_videos
|
|
if video_multi_lang:
|
|
fallback_langs = list(dict.fromkeys(str(v.language) for v in fallback_pool))
|
|
else:
|
|
fallback_langs = [None]
|
|
fallback_selected: list[Video] = []
|
|
for resolution, color_range, codec, vlang in product(
|
|
effective_quality or [None], other_ranges, vcodec or [None], fallback_langs
|
|
):
|
|
candidates = [
|
|
t
|
|
for t in fallback_pool
|
|
if (
|
|
not resolution
|
|
or t.height == resolution
|
|
or (t.width and int(t.width * (9 / 16)) == resolution)
|
|
)
|
|
and (not color_range or t.range == color_range)
|
|
and (not codec or t.codec == codec)
|
|
and (vlang is None or str(t.language) == vlang)
|
|
]
|
|
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
|
|
if match and match not in fallback_selected:
|
|
fallback_selected.append(match)
|
|
title.tracks.videos = fallback_selected
|
|
else:
|
|
self.log.error(msg)
|
|
self.log.error(msg_detail)
|
|
sys.exit(1)
|
|
|
|
fsl = [t for t in forced_s_lang if t != "orig"]
|
|
if "orig" in forced_s_lang and title.language:
|
|
fsl.append(str(title.language))
|
|
if keep_subtitles and s_excl_r:
|
|
# applies to forced subs too: an excluded language is unwanted in any form
|
|
drop = excluded_language_tags(
|
|
s_excl_r, [t.language for t in title.tracks.subtitles], exact_lang
|
|
)
|
|
title.tracks.select_subtitles(lambda x: str(x.language) not in drop)
|
|
if keep_subtitles and s_lang and "all" not in s_lang:
|
|
match_func = is_exact_match if exact_lang else is_close_match
|
|
|
|
missing_langs = find_missing_langs(
|
|
s_lang,
|
|
[sub.language for sub in title.tracks.subtitles],
|
|
exact=exact_lang,
|
|
)
|
|
if missing_langs:
|
|
missing_str = ", ".join(missing_langs)
|
|
if best_available:
|
|
remaining = [tok for tok in s_lang if tok not in missing_langs]
|
|
if remaining:
|
|
self.log.warning(
|
|
f"{missing_str} not found in subtitle tracks, continuing with: {', '.join(remaining)}"
|
|
)
|
|
s_lang = remaining
|
|
else:
|
|
self.log.warning(
|
|
f"{missing_str} not found in subtitle tracks, continuing without subtitles"
|
|
)
|
|
title.tracks.select_subtitles(
|
|
lambda x: (
|
|
x.forced and keep_forced_subtitle(x.forced, x.language, fsl, exact_lang)
|
|
)
|
|
)
|
|
else:
|
|
self.log.error(missing_str + " not found in subtitle tracks")
|
|
sys.exit(1)
|
|
|
|
if s_lang and title.tracks.subtitles:
|
|
title.tracks.select_subtitles(
|
|
lambda x: (
|
|
match_func(x.language, s_lang)
|
|
or bool(x.forced and fsl and match_func(x.language, fsl))
|
|
)
|
|
)
|
|
if not title.tracks.subtitles and not best_available:
|
|
self.log.error(f"There's no {s_lang} Subtitle Track...")
|
|
sys.exit(1)
|
|
|
|
if keep_subtitles:
|
|
# fsl, not forced_s_lang: unresolvable orig must keep all forced, not drop them
|
|
if fsl:
|
|
title.tracks.select_subtitles(
|
|
lambda x: keep_forced_subtitle(x.forced, x.language, fsl, exact_lang)
|
|
)
|
|
if not any(x.forced for x in title.tracks.subtitles):
|
|
self.log.warning(
|
|
f"No forced subtitle tracks matched --forced-s-lang ({', '.join(forced_s_lang)})"
|
|
)
|
|
elif not forced_subs:
|
|
title.tracks.select_subtitles(lambda x: not x.forced)
|
|
if fsl_excl_r:
|
|
drop = excluded_language_tags(
|
|
fsl_excl_r, [t.language for t in title.tracks.subtitles if t.forced], exact_lang
|
|
)
|
|
title.tracks.select_subtitles(lambda x: not (x.forced and str(x.language) in drop))
|
|
|
|
# might have no audio tracks if part of the video, e.g. transport stream hls
|
|
if keep_audio and len(title.tracks.audio) > 0:
|
|
if not audio_description:
|
|
title.tracks.select_audio(lambda x: not x.descriptive)
|
|
if not title.tracks.audio:
|
|
self.log.error("Audio description only, use --audio-description...")
|
|
sys.exit(1)
|
|
if acodec:
|
|
title.tracks.select_audio(lambda x: x.codec in acodec)
|
|
if not title.tracks.audio:
|
|
codec_names = ", ".join(c.name for c in acodec)
|
|
self.log.error(f"No audio tracks matching codecs: {codec_names}")
|
|
sys.exit(1)
|
|
if channels:
|
|
title.tracks.select_audio(
|
|
lambda x: bool(
|
|
x.channels and math.ceil(Audio.channel_total(x.channels)) == math.ceil(channels)
|
|
)
|
|
)
|
|
if not title.tracks.audio:
|
|
self.log.error(f"There's no {channels} Audio Track...")
|
|
sys.exit(1)
|
|
if no_atmos:
|
|
title.tracks.audio = [x for x in title.tracks.audio if not x.atmos]
|
|
if not title.tracks.audio:
|
|
self.log.error("No non-Atmos audio tracks available...")
|
|
sys.exit(1)
|
|
if abitrate:
|
|
title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate)
|
|
if not title.tracks.audio:
|
|
self.log.error(f"There's no {abitrate}kbps Audio Track...")
|
|
sys.exit(1)
|
|
if abitrate_min is not None and abitrate_max is not None:
|
|
title.tracks.select_audio(
|
|
lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max
|
|
)
|
|
if not title.tracks.audio:
|
|
self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...")
|
|
sys.exit(1)
|
|
if audio_excl_r:
|
|
drop = excluded_language_tags(
|
|
audio_excl_r, [t.language for t in title.tracks.audio], exact_lang
|
|
)
|
|
title.tracks.select_audio(lambda x: str(x.language) not in drop)
|
|
if not title.tracks.audio:
|
|
self.log.error(f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...")
|
|
sys.exit(1)
|
|
audio_languages = a_lang or lang
|
|
if audio_languages:
|
|
processed_lang = []
|
|
a_orig_token: Optional[str] = None
|
|
for language in audio_languages:
|
|
if language == "orig":
|
|
if title.language:
|
|
orig_lang = str(title.language)
|
|
a_orig_token = orig_lang
|
|
if orig_lang not in processed_lang:
|
|
processed_lang.append(orig_lang)
|
|
else:
|
|
self.log.warning(
|
|
"Original language not available for title, skipping 'orig' selection"
|
|
)
|
|
else:
|
|
if language not in processed_lang:
|
|
processed_lang.append(language)
|
|
if a_orig_token in audio_languages:
|
|
a_orig_token = None
|
|
|
|
embedded_langs = embedded_audio_langs(title.tracks.videos, keep_videos)
|
|
|
|
if not any(tok in processed_lang for tok in ("best", "all")):
|
|
missing_a_langs = find_missing_langs(
|
|
processed_lang,
|
|
[a.language for a in title.tracks.audio] + embedded_langs,
|
|
exact=exact_lang,
|
|
)
|
|
if missing_a_langs:
|
|
missing_str = as_requested(missing_a_langs, a_orig_token)
|
|
if best_available:
|
|
remaining = [tok for tok in processed_lang if tok not in missing_a_langs]
|
|
if remaining:
|
|
self.log.warning(
|
|
f"{missing_str} not found in audio tracks, "
|
|
f"continuing with: {as_requested(remaining, a_orig_token)}"
|
|
)
|
|
processed_lang = remaining
|
|
else:
|
|
self.log.error(
|
|
f"{missing_str} not found in audio tracks and no fallback available"
|
|
)
|
|
sys.exit(1)
|
|
else:
|
|
self.log.error(missing_str + " not found in audio tracks")
|
|
sys.exit(1)
|
|
|
|
title.tracks.audio = select_best_audio(
|
|
title.tracks.audio, processed_lang, acodec, audio_description, exact_lang
|
|
)
|
|
if not title.tracks.audio and not embedded_langs:
|
|
# empty when 'orig' was the sole request, did not resolve, and the
|
|
# video carries no audio of its own to fall back on
|
|
self.log.error(
|
|
f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} "
|
|
f"Audio Track, cannot continue..."
|
|
)
|
|
sys.exit(1)
|
|
|
|
kept_tracks = []
|
|
if keep_videos:
|
|
kept_tracks.extend(title.tracks.videos)
|
|
if keep_audio:
|
|
kept_tracks.extend(title.tracks.audio)
|
|
if keep_subtitles:
|
|
kept_tracks.extend(title.tracks.subtitles)
|
|
if keep_chapters:
|
|
kept_tracks.extend(title.tracks.chapters)
|
|
if not no_attachments:
|
|
kept_tracks.extend(title.tracks.attachments)
|
|
|
|
title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url)
|
|
|
|
selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True)
|
|
|
|
log_event(
|
|
"track_select",
|
|
level="INFO",
|
|
message=(
|
|
f"Selected {len(title.tracks.videos)} video, {len(title.tracks.audio)} audio, "
|
|
f"{len(title.tracks.subtitles)} subtitle track(s)"
|
|
),
|
|
context={
|
|
"videos": [
|
|
f"{v.codec} {v.range} {v.height}p {v.bitrate}bps {v.language}" for v in title.tracks.videos
|
|
],
|
|
"audio": [
|
|
f"{a.codec} {a.language} {getattr(a, 'channels', None)}ch {a.bitrate}bps"
|
|
for a in title.tracks.audio
|
|
],
|
|
"subtitles": [f"{s.codec} {s.language}" for s in title.tracks.subtitles],
|
|
},
|
|
)
|
|
|
|
if progress_sink is not None:
|
|
from unshackle.core.api.progress import build_job_progress_callables
|
|
|
|
tracks_progress_callables = build_job_progress_callables(
|
|
list(title.tracks), tracks_progress_callables, progress_sink
|
|
)
|
|
|
|
drm_pending = [track for track in title.tracks if getattr(track, "needs_drm_loading", False)]
|
|
if drm_pending:
|
|
with ThreadPoolExecutor(min(16, len(drm_pending))) as drm_pool:
|
|
list(drm_pool.map(lambda track: track.load_drm_if_needed(service), drm_pending))
|
|
|
|
download_table = Table.grid()
|
|
download_table.add_row(selected_tracks)
|
|
|
|
if getattr(self._remote_service, "_server_cdm", False) and self.cdm is None:
|
|
cdm_type = getattr(self._remote_service, "_server_cdm_type", "widevine")
|
|
self.cdm = cdm_type_stub(cdm_type)
|
|
self.log.info(f"Using server CDM ({cdm_type.title()}); no local CDM required")
|
|
|
|
video_tracks = title.tracks.videos
|
|
if video_tracks:
|
|
highest_quality = max((track.height for track in video_tracks if track.height), default=0)
|
|
if highest_quality > 0:
|
|
if is_widevine_cdm(self.cdm):
|
|
quality_based_cdm = self.get_cdm(
|
|
self.service, self.profile, drm="widevine", quality=highest_quality
|
|
)
|
|
if quality_based_cdm and quality_based_cdm != self.cdm:
|
|
self.log.debug(
|
|
f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks"
|
|
)
|
|
self.cdm = quality_based_cdm
|
|
elif is_playready_cdm(self.cdm):
|
|
quality_based_cdm = self.get_cdm(
|
|
self.service, self.profile, drm="playready", quality=highest_quality
|
|
)
|
|
if quality_based_cdm and quality_based_cdm != self.cdm:
|
|
self.log.debug(
|
|
f"Pre-selecting PlayReady CDM based on highest quality {highest_quality}p across all video tracks"
|
|
)
|
|
self.cdm = quality_based_cdm
|
|
|
|
if hasattr(service, "resolve_server_keys"):
|
|
service.resolve_server_keys(title)
|
|
|
|
dl_start_time = time.time()
|
|
|
|
try:
|
|
with SyncLive(Padding(download_table, (1, 5)), console=console, refresh_per_second=20):
|
|
|
|
def download_track(track: AnyTrack, i: int) -> None:
|
|
track.download(
|
|
session=track.session or service.session,
|
|
no_proxy_download=no_proxy_download,
|
|
prepare_drm=partial(
|
|
partial(self.prepare_drm, table=download_table),
|
|
track=track,
|
|
title=title,
|
|
certificate=partial(
|
|
service.get_widevine_service_certificate,
|
|
title=title,
|
|
track=track,
|
|
),
|
|
licence=partial(
|
|
self.service_licence,
|
|
service,
|
|
title=title,
|
|
track=track,
|
|
),
|
|
clearkey_licence=partial(
|
|
service.get_clearkey_license,
|
|
title=title,
|
|
track=track,
|
|
),
|
|
cdm_only=cdm_only,
|
|
vaults_only=vaults_only,
|
|
export=export_path,
|
|
service_session=service.session,
|
|
),
|
|
cdm=self.cdm,
|
|
max_workers=workers,
|
|
adaptive_workers=adaptive_workers,
|
|
download_processes=download_processes,
|
|
progress=tracks_progress_callables[i],
|
|
)
|
|
# DRM-free and HLS-ClearKey tracks never reach prepare_drm, so export here.
|
|
# drm=None on purpose: licensed tracks already recorded their DRM/keys
|
|
# in prepare_drm, and write_export merges via setdefault.
|
|
if export_path:
|
|
self.write_export(export_path, title, track)
|
|
|
|
def on_subtitle_skipped(track: Subtitle) -> None:
|
|
lang = str(track.language)
|
|
self.log.warning(f"Subtitle {lang} failed to download, skipping it.")
|
|
self.skipped_subtitles.append(SkippedSubtitle(id=track.id, language=lang, title=str(title)))
|
|
try:
|
|
title.tracks.subtitles.remove(track)
|
|
except ValueError:
|
|
self.log.debug(f"Skipped subtitle {track.id} was already absent from the track list.")
|
|
|
|
skipped_before = len(self.skipped_subtitles)
|
|
grow_session_pool(service.session, downloads * (workers or default_max_workers()))
|
|
download_tracks_in_passes(
|
|
title.tracks,
|
|
downloads,
|
|
download_track,
|
|
skip_subtitle_errors=skip_subtitle_errors,
|
|
on_subtitle_skipped=on_subtitle_skipped,
|
|
)
|
|
|
|
for attachment in title.tracks.attachments:
|
|
attachment.download(
|
|
attachment.session or service.session,
|
|
no_proxy_download=no_proxy_download,
|
|
)
|
|
|
|
if (
|
|
len(self.skipped_subtitles) > skipped_before
|
|
and not title.tracks.videos
|
|
and not title.tracks.audio
|
|
and not title.tracks.subtitles
|
|
):
|
|
self.log.warning(
|
|
f"{title}: all subtitles were skipped and no video or audio was "
|
|
"downloaded - nothing was produced for this title."
|
|
)
|
|
|
|
except KeyboardInterrupt:
|
|
console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5)))
|
|
log_event(
|
|
"download_tracks",
|
|
level="WARNING",
|
|
service=self.service,
|
|
message="Download cancelled by user",
|
|
context={"title": str(title)},
|
|
)
|
|
self.wait_vault_writes()
|
|
return
|
|
except Exception as e: # noqa
|
|
# Reported and swallowed (no re-raise) so the CLI exits cleanly; flag it so the
|
|
# API worker sees the title failed rather than completing with no output.
|
|
self.download_failed = True
|
|
error_messages = [
|
|
":x: Download Failed...",
|
|
f" {type(e).__name__}: {e}",
|
|
]
|
|
if hasattr(e, "returncode"):
|
|
error_messages.append(f" Binary call failed, Process exit code: {e.returncode}")
|
|
error_messages.append(
|
|
" An unexpected error occurred in one of the download workers.",
|
|
)
|
|
console.print(Padding(Group(*error_messages), (1, 5)))
|
|
console.print_exception()
|
|
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"download_tracks",
|
|
e,
|
|
service=self.service,
|
|
context={
|
|
"title": str(title),
|
|
"error_type": type(e).__name__,
|
|
"tracks_count": len(title.tracks),
|
|
"returncode": getattr(e, "returncode", None),
|
|
},
|
|
)
|
|
|
|
dispatch(
|
|
"failure",
|
|
"file",
|
|
build_context(
|
|
title,
|
|
service=self.service,
|
|
ids=self.post_script_ids(),
|
|
error=f"{type(e).__name__}: {e}",
|
|
),
|
|
postscript,
|
|
)
|
|
self.wait_vault_writes()
|
|
return
|
|
|
|
self.wait_vault_writes()
|
|
|
|
if skip_dl:
|
|
console.log("Skipped downloads as --skip-dl was used...")
|
|
else:
|
|
dl_time = time_elapsed_since(dl_start_time)
|
|
console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5)))
|
|
|
|
subtitle_output_mode = config.subtitle.get("output_mode", "mux")
|
|
sidecar_format = config.subtitle.get("sidecar_format", "srt")
|
|
skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio)
|
|
sidecar_subtitles: list[Subtitle] = []
|
|
sidecar_original_paths: dict[str, Path] = {}
|
|
sidecar_files: dict[Optional[Path], list[Path]] = {}
|
|
if subtitle_output_mode in ("sidecar", "both") and not no_mux:
|
|
sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()]
|
|
if sidecar_format == "original":
|
|
config.directories.temp.mkdir(parents=True, exist_ok=True)
|
|
for subtitle in sidecar_subtitles:
|
|
original_path = (
|
|
config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}"
|
|
)
|
|
shutil.copy2(subtitle.path, original_path)
|
|
sidecar_original_paths[subtitle.id] = original_path
|
|
|
|
with console.status("Converting Subtitles..."):
|
|
sub_conversions: dict[tuple[str, str], int] = {}
|
|
for subtitle in title.tracks.subtitles:
|
|
if sub_format == "original":
|
|
continue
|
|
if sub_format:
|
|
if subtitle.codec != sub_format:
|
|
src = getattr(subtitle.codec, "name", str(subtitle.codec))
|
|
dst = getattr(sub_format, "name", str(sub_format))
|
|
subtitle.convert(sub_format, forced=True)
|
|
sub_conversions[(src, dst)] = sub_conversions.get((src, dst), 0) + 1
|
|
elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang:
|
|
# MKV does not support TTML, VTT is the next best option
|
|
src = getattr(subtitle.codec, "name", str(subtitle.codec))
|
|
subtitle.convert(Subtitle.Codec.WebVTT)
|
|
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
|
|
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
|
|
)
|
|
for (src, dst), count in sub_conversions.items():
|
|
log_event(
|
|
"subtitle_convert",
|
|
level="INFO",
|
|
message=f"Converted {src}->{dst} x{count}",
|
|
context={"from": src, "to": dst, "count": count},
|
|
)
|
|
|
|
with console.status("Checking Subtitles for Fonts..."):
|
|
font_names: list[str] = []
|
|
for subtitle in [] if no_attachments else title.tracks.subtitles:
|
|
if subtitle.codec in (Subtitle.Codec.SubStationAlpha, Subtitle.Codec.SubStationAlphav4):
|
|
font_names.extend(Subtitle.extract_fonts(subtitle.path.read_text("utf8")))
|
|
|
|
font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title, temp_font_files)
|
|
|
|
if font_count:
|
|
self.log.info(f"Attached {font_count} fonts for the Subtitles")
|
|
|
|
if missing_fonts and sys.platform != "win32":
|
|
self.suggest_missing_fonts(missing_fonts)
|
|
|
|
# Handle DRM decryption BEFORE repacking (must decrypt first!)
|
|
service_name = service.__class__.__name__.upper()
|
|
decryption_method = resolve_decryption(config.decryption_map, config.decryption, service_name)
|
|
decrypt_tool = "mp4decrypt" if decryption_method.lower() == "mp4decrypt" else "Shaka Packager"
|
|
|
|
drm_tracks = [track for track in title.tracks if track.drm]
|
|
if drm_tracks:
|
|
with console.status(f"Decrypting tracks with {decrypt_tool}..."):
|
|
has_decrypted = False
|
|
for track in drm_tracks:
|
|
drm = track.get_drm_for_cdm(self.cdm)
|
|
if drm and hasattr(drm, "decrypt"):
|
|
drm.decrypt(track.path)
|
|
if not isinstance(drm, MonaLisa):
|
|
# MonaLisa decrypts per segment; its decrypt() here is a no-op
|
|
assert_fragments_decrypted(track.path)
|
|
has_decrypted = True
|
|
events.emit(events.Types.TRACK_REPACKED, track=track)
|
|
else:
|
|
self.log.warning(
|
|
f"No matching DRM found for track {track} with CDM type {type(self.cdm).__name__}"
|
|
)
|
|
if has_decrypted:
|
|
self.log.info(f"Decrypted tracks with {decrypt_tool}")
|
|
|
|
if (
|
|
not no_subs
|
|
and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES)
|
|
and not video_only
|
|
and not no_video
|
|
):
|
|
for video_track_n, video_track in enumerate(title.tracks.videos):
|
|
has_manifest_cc = bool(getattr(video_track, "closed_captions", None))
|
|
has_eia_cc = (
|
|
not has_manifest_cc
|
|
and not title.tracks.subtitles
|
|
and any(
|
|
x.get("codec_name", "").startswith("eia_")
|
|
for x in ffprobe(video_track.path).get("streams", [])
|
|
)
|
|
)
|
|
if not has_manifest_cc and not has_eia_cc:
|
|
continue
|
|
|
|
with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."):
|
|
try:
|
|
cc_lang = (
|
|
Language.get(video_track.closed_captions[0]["language"])
|
|
if has_manifest_cc and video_track.closed_captions[0].get("language")
|
|
else title.language or video_track.language
|
|
)
|
|
track_id = f"ccextractor-{video_track.id}"
|
|
cc = video_track.ccextractor(
|
|
track_id=track_id,
|
|
out_path=config.directories.temp
|
|
/ config.filenames.subtitle.format(id=track_id, language=cc_lang),
|
|
language=cc_lang,
|
|
original=False,
|
|
)
|
|
if cc:
|
|
cc.cc = True
|
|
title.tracks.add(cc)
|
|
self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
|
|
else:
|
|
self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
|
|
except EnvironmentError:
|
|
self.log.error(
|
|
"Cannot extract Closed Captions as the ccextractor executable was not found..."
|
|
)
|
|
break
|
|
|
|
# Now repack the decrypted tracks
|
|
if progress_sink and any(getattr(t, "needs_repack", False) for t in title.tracks):
|
|
progress_sink(
|
|
{"phase": "repackaging", "progress": 92.0, "status": "downloading", "active_tracks": []}
|
|
)
|
|
with console.status("Repackaging tracks with FFMPEG..."):
|
|
has_repacked = False
|
|
# Video tracks whose VUI was folded into the repack pass; skipped in the VUI loop below.
|
|
vui_folded = set()
|
|
for track in title.tracks:
|
|
if track.needs_repack:
|
|
bsf = track.vui_bsf() if isinstance(track, Video) else None
|
|
if track.repackage(bsf_v=bsf):
|
|
vui_folded.add(id(track))
|
|
has_repacked = True
|
|
events.emit(events.Types.TRACK_REPACKED, track=track)
|
|
if has_encrypted_sample_entry(track.path):
|
|
self.log.warning(
|
|
f"Track {track.id} still has an encrypted sample entry after repacking, so "
|
|
"decryption likely failed and the muxed output may be unplayable."
|
|
)
|
|
if has_repacked:
|
|
self.log.info("Repacked one or more tracks with FFMPEG")
|
|
|
|
with console.status("Normalizing video VUI..."):
|
|
for track in title.tracks.videos:
|
|
if id(track) in vui_folded:
|
|
continue
|
|
try:
|
|
track.normalize_vui()
|
|
except Exception as e: # noqa: BLE001
|
|
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
|
|
|
|
remux_music_tracks(title, self.log)
|
|
|
|
muxed_paths = []
|
|
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
|
|
tagged_paths: set[Path] = set()
|
|
append_audio_codec_suffix = True
|
|
|
|
if no_mux:
|
|
for track in title.tracks:
|
|
if track.path and track.path.exists():
|
|
muxed_paths.append(track.path)
|
|
elif isinstance(title, (Movie, Episode)):
|
|
progress = Progress(
|
|
TextColumn("[progress.description]{task.description}"),
|
|
SpinnerColumn(finished_text=""),
|
|
GradientPulseBarColumn(),
|
|
"•",
|
|
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
|
|
"•",
|
|
TextColumn("{task.fields[downloaded]}"),
|
|
console=console,
|
|
)
|
|
|
|
merge_audio = (
|
|
(not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True)
|
|
)
|
|
# When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec.
|
|
append_audio_codec_suffix = not merge_audio
|
|
|
|
merge_video = merge_video if merge_video is not None else config.muxing.get("merge_video", False)
|
|
|
|
multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = []
|
|
mux_futures: list[futures.Future[tuple[Path, int, list[str]]]] = []
|
|
# Track hybrid-processing outputs explicitly so we can always clean them up,
|
|
# even if muxing fails early (e.g. SystemExit) before the normal delete loop.
|
|
hybrid_temp_paths: list[Path] = []
|
|
|
|
def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks:
|
|
task_tracks = Tracks()
|
|
task_tracks.videos = list(base_tracks.videos)
|
|
task_tracks.audio = audio_tracks
|
|
task_tracks.subtitles = list(base_tracks.subtitles)
|
|
task_tracks.chapters = base_tracks.chapters
|
|
task_tracks.attachments = list(base_tracks.attachments)
|
|
return task_tracks
|
|
|
|
audio_expected = not video_only and not no_audio
|
|
|
|
def unique_mux_output(task_tracks: Tracks, index: int) -> Optional[Path]:
|
|
src: Optional[Path] = None
|
|
ext = ".muxed.mkv"
|
|
if task_tracks.videos:
|
|
src = task_tracks.videos[0].path
|
|
elif task_tracks.audio:
|
|
src, ext = task_tracks.audio[0].path, ".muxed.mka"
|
|
elif task_tracks.subtitles:
|
|
src, ext = task_tracks.subtitles[0].path, ".muxed.mks"
|
|
if src is None:
|
|
return None
|
|
base = src.with_suffix(ext)
|
|
return base.with_name(f"{base.stem}.{index}{base.suffix}")
|
|
|
|
def mux_one(mux_index: int, task_id: TaskID, task_tracks: Tracks) -> tuple[Path, int, list[str]]:
|
|
progress.start_task(task_id)
|
|
muxed_path, return_code, errors = task_tracks.mux(
|
|
str(title),
|
|
progress=partial(progress.update, task_id=task_id),
|
|
delete=False,
|
|
audio_expected=audio_expected,
|
|
title_language=title.language,
|
|
skip_subtitles=skip_subtitle_mux,
|
|
output_path=unique_mux_output(task_tracks, mux_index),
|
|
)
|
|
if return_code < 2:
|
|
tags.tag_file(
|
|
muxed_path,
|
|
title,
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
self.anime_hint(title),
|
|
session=service.session,
|
|
)
|
|
tagged_paths.add(muxed_path)
|
|
return muxed_path, return_code, errors
|
|
|
|
def enqueue_mux_tasks(task_description: str, base_tracks: Tracks, hybrid: bool = False) -> None:
|
|
pool = hybrid_mux_pool if hybrid else mux_pool
|
|
if merge_audio or not base_tracks.audio:
|
|
task_id = progress.add_task(
|
|
f"{task_description}...", total=None, start=False, downloaded=""
|
|
)
|
|
multiplex_tasks.append((task_id, base_tracks, None))
|
|
mux_futures.append(pool.submit(mux_one, len(multiplex_tasks), task_id, base_tracks))
|
|
return
|
|
|
|
audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {}
|
|
for audio_track in base_tracks.audio:
|
|
audio_by_codec.setdefault(audio_track.codec, []).append(audio_track)
|
|
|
|
for audio_codec, codec_audio_tracks in audio_by_codec.items():
|
|
description = task_description
|
|
if audio_codec:
|
|
description = f"{task_description} {audio_codec.name}"
|
|
|
|
task_id = progress.add_task(f"{description}...", total=None, start=False, downloaded="")
|
|
task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks)
|
|
multiplex_tasks.append((task_id, task_tracks, audio_codec))
|
|
mux_futures.append(pool.submit(mux_one, len(multiplex_tasks), task_id, task_tracks))
|
|
|
|
def mux_video_group(video_tracks: list[Optional[Video]]) -> None:
|
|
for video_track in video_tracks:
|
|
if video_track and video_track.dv_compatible_bitstream:
|
|
apply_dv_fixup(video_track)
|
|
|
|
task_description = "Multiplexing"
|
|
# All tracks in a merged group share height/range/codec, so describe from the first.
|
|
head = next((v for v in video_tracks if v), None)
|
|
if head:
|
|
if len(quality) > 1:
|
|
task_description += f" {head.height}p"
|
|
if len(range_) > 1:
|
|
task_description += f" {head.range.name}"
|
|
if len(vcodec) > 1:
|
|
task_description += f" {head.codec.name}"
|
|
|
|
task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
|
|
if head:
|
|
task_tracks.videos = [v for v in video_tracks if v]
|
|
|
|
enqueue_mux_tasks(task_description, task_tracks)
|
|
|
|
def group_file_size(video_tracks: Sequence[Optional[Video]]) -> int:
|
|
return sum(v.path.stat().st_size for v in video_tracks if v and v.path and v.path.exists())
|
|
|
|
if progress_sink:
|
|
progress_sink(
|
|
{"phase": "muxing", "progress": 96.0, "status": "downloading", "active_tracks": []}
|
|
)
|
|
|
|
try:
|
|
with SyncLive(Padding(progress, (0, 5, 1, 5)), console=console, refresh_per_second=20):
|
|
mux_failed = False
|
|
mux_concurrency = max(1, int(config.muxing.get("concurrency", 4)))
|
|
|
|
with (
|
|
ThreadPoolExecutor(mux_concurrency) as mux_pool,
|
|
ThreadPoolExecutor(mux_concurrency) as extra_mux_pool,
|
|
):
|
|
hybrid_mux_pool = mux_pool if mux_concurrency == 1 else extra_mux_pool
|
|
|
|
if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos:
|
|
self.log.info("Processing Hybrid HDR10+DV tracks...")
|
|
|
|
original_videos = list(title.tracks.videos)
|
|
|
|
standalone_videos = [v for v in original_videos if not v.hybrid_base_only]
|
|
standalone_groups: list[list[Optional[Video]]] = [
|
|
list(g) for g in group_videos_by_variant(standalone_videos, merge=merge_video)
|
|
]
|
|
for group in sorted(standalone_groups, key=group_file_size, reverse=True):
|
|
mux_video_group(group)
|
|
|
|
resolutions_processed = set()
|
|
base_tracks_list = sorted(
|
|
(
|
|
v
|
|
for v in title.tracks.videos
|
|
if v.range in (Video.Range.HDR10P, Video.Range.HDR10)
|
|
),
|
|
key=lambda v: v.height,
|
|
reverse=True,
|
|
)
|
|
dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV]
|
|
|
|
for hdr10_track in base_tracks_list:
|
|
resolution = hdr10_track.height
|
|
if resolution in resolutions_processed:
|
|
continue
|
|
|
|
matching_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None
|
|
|
|
if matching_dv:
|
|
resolutions_processed.add(resolution)
|
|
|
|
resolution_tracks = [deepcopy(hdr10_track), deepcopy(matching_dv)]
|
|
for track in resolution_tracks:
|
|
track.needs_duration_fix = True
|
|
|
|
hybrid_filename = f"HDR10-DV-{resolution}p.hevc"
|
|
hybrid_output_path = config.directories.temp / hybrid_filename
|
|
hybrid_temp_paths.append(hybrid_output_path)
|
|
|
|
Hybrid(resolution_tracks, self.service, output_name=hybrid_filename)
|
|
|
|
task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p"
|
|
task_tracks = (
|
|
Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
|
|
)
|
|
|
|
hybrid_track = deepcopy(hdr10_track)
|
|
hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}"
|
|
hybrid_track.path = hybrid_output_path
|
|
hybrid_track.range = Video.Range.DV
|
|
hybrid_track.needs_duration_fix = True
|
|
title.tracks.add(hybrid_track)
|
|
task_tracks.videos = [hybrid_track]
|
|
|
|
enqueue_mux_tasks(task_description, task_tracks, hybrid=True)
|
|
|
|
console.print()
|
|
else:
|
|
groups: list[list[Optional[Video]]] = [
|
|
list(g) for g in group_videos_by_variant(title.tracks.videos, merge=merge_video)
|
|
] or [[None]]
|
|
for group in sorted(groups, key=group_file_size, reverse=True):
|
|
mux_video_group(group)
|
|
|
|
mux_results = [f.result() for f in mux_futures]
|
|
for (task_id, task_tracks, audio_codec), (muxed_path, return_code, errors) in zip(
|
|
multiplex_tasks, mux_results
|
|
):
|
|
muxed_paths.append(muxed_path)
|
|
muxed_audio_codecs[muxed_path] = audio_codec
|
|
if return_code >= 2:
|
|
self.log.error(f"Failed to Mux video to Matroska file ({return_code}):")
|
|
elif return_code == 1 or errors:
|
|
self.log.warning("mkvmerge had at least one warning or error, continuing anyway...")
|
|
for line in errors:
|
|
if line.startswith("#GUI#error"):
|
|
self.log.error(line)
|
|
else:
|
|
self.log.warning(line)
|
|
if return_code >= 2:
|
|
mux_failed = True
|
|
if mux_failed:
|
|
sys.exit(1)
|
|
|
|
if sidecar_subtitles and not no_mux:
|
|
sidecar_targets: list[tuple[Optional[Path], str, Path]] = []
|
|
for muxed_path in muxed_paths or [None]:
|
|
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
|
|
if media_info:
|
|
base_filename = title.get_filename(media_info, show_service=not no_source)
|
|
else:
|
|
base_filename = str(title)
|
|
|
|
sidecar_dir = self.output_dir or config.directories.downloads
|
|
if (
|
|
not no_folder
|
|
and media_info
|
|
and (
|
|
isinstance(title, (Episode, Song))
|
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
|
)
|
|
):
|
|
sidecar_dir /= title.get_filename(
|
|
media_info, show_service=not no_source, folder=True
|
|
)
|
|
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
|
|
|
|
with console.status("Saving subtitle sidecar files..."):
|
|
first_path, first_base, first_dir = sidecar_targets[0]
|
|
first_dir.mkdir(parents=True, exist_ok=True)
|
|
created = self.output_subtitle_sidecars(
|
|
sidecar_subtitles,
|
|
first_base,
|
|
first_dir,
|
|
sidecar_format,
|
|
original_paths=sidecar_original_paths or None,
|
|
)
|
|
if created:
|
|
sidecar_files[first_path] = list(created)
|
|
self.log.info(f"Saved {len(created)} sidecar subtitle files")
|
|
|
|
for muxed_path, base_filename, sidecar_dir in sidecar_targets[1:]:
|
|
sidecar_dir.mkdir(parents=True, exist_ok=True)
|
|
copies = []
|
|
for source in created:
|
|
target = sidecar_dir / f"{base_filename}{source.name[len(first_base) :]}"
|
|
if target != source:
|
|
shutil.copy2(source, target)
|
|
copies.append(target)
|
|
if copies:
|
|
sidecar_files[muxed_path] = copies
|
|
self.log.info(f"Saved {len(copies)} sidecar subtitle files")
|
|
|
|
for track in title.tracks:
|
|
track.delete()
|
|
|
|
for attachment in title.tracks.attachments:
|
|
if attachment.path and attachment.path in temp_font_files:
|
|
attachment.path = None
|
|
else:
|
|
attachment.delete()
|
|
|
|
for temp_path in temp_font_files:
|
|
temp_path.unlink(missing_ok=True)
|
|
for temp_path in sidecar_original_paths.values():
|
|
temp_path.unlink(missing_ok=True)
|
|
finally:
|
|
# Hybrid() writes a temp HEVC per resolution, so delete each one here.
|
|
for temp_path in hybrid_temp_paths:
|
|
try:
|
|
temp_path.unlink(missing_ok=True)
|
|
except PermissionError:
|
|
self.log.warning(f"Failed to delete temp file (in use?): {temp_path}")
|
|
for leftover in (
|
|
"HDR10-DV.hevc",
|
|
"HDR10.hevc",
|
|
"DV.hevc",
|
|
"RPU.bin",
|
|
"RPU_UNT.bin",
|
|
"RPU_L5.bin",
|
|
"RPU_L6.bin",
|
|
"L5.json",
|
|
):
|
|
leftover_path = config.directories.temp / leftover
|
|
try:
|
|
leftover_path.unlink(missing_ok=True)
|
|
except PermissionError:
|
|
self.log.warning(f"Failed to delete temp file (in use?): {leftover_path}")
|
|
|
|
else:
|
|
muxed_paths.extend(track.path for track in title.tracks.audio if track.path and track.path.exists())
|
|
|
|
if no_mux:
|
|
final_dir = self.output_dir or config.directories.downloads
|
|
if not no_folder and (
|
|
isinstance(title, (Episode, Song))
|
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
|
):
|
|
sample_track = (
|
|
title.tracks.videos[0]
|
|
if title.tracks.videos
|
|
else (
|
|
title.tracks.audio[0]
|
|
if title.tracks.audio
|
|
else (title.tracks.subtitles[0] if title.tracks.subtitles else None)
|
|
)
|
|
)
|
|
if sample_track and sample_track.path:
|
|
media_info = MediaInfo.parse(sample_track.path)
|
|
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
|
|
|
final_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
for track_path in muxed_paths:
|
|
media_info = MediaInfo.parse(track_path)
|
|
base_filename = title.get_filename(media_info, show_service=not no_source)
|
|
|
|
track = next((t for t in title.tracks if t.path == track_path), None)
|
|
if track:
|
|
if isinstance(track, Video):
|
|
track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}"
|
|
elif isinstance(track, Audio):
|
|
lang_suffix = f".{track.language}" if track.language else ""
|
|
track_suffix = (
|
|
f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}"
|
|
)
|
|
elif isinstance(track, Subtitle):
|
|
lang_suffix = f".{track.language}" if track.language else ""
|
|
forced_suffix = ".forced" if track.forced else ""
|
|
sdh_suffix = ".sdh" if track.sdh else ""
|
|
track_suffix = f"{lang_suffix}{forced_suffix}{sdh_suffix}"
|
|
else:
|
|
track_suffix = ""
|
|
|
|
final_path = final_dir / f"{base_filename}{track_suffix}{track_path.suffix}"
|
|
else:
|
|
final_path = final_dir / f"{base_filename}{track_path.suffix}"
|
|
|
|
shutil.move(track_path, final_path)
|
|
self.completed_files.append(final_path)
|
|
self.log.debug(f"Saved: {final_path.name}")
|
|
else:
|
|
used_final_paths: set[Path] = set()
|
|
for muxed_path in muxed_paths:
|
|
media_info = MediaInfo.parse(muxed_path)
|
|
final_dir = self.output_dir or config.directories.downloads
|
|
final_filename = title.get_filename(media_info, show_service=not no_source)
|
|
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
|
|
|
|
if not no_folder and (
|
|
isinstance(title, (Episode, Song))
|
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
|
):
|
|
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
|
|
|
final_dir.mkdir(parents=True, exist_ok=True)
|
|
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
|
template_type = (
|
|
"series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
|
)
|
|
sep = config.get_template_separator(template_type)
|
|
|
|
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
|
|
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
|
|
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
|
|
|
if final_path in used_final_paths:
|
|
i = 2
|
|
while final_path in used_final_paths:
|
|
final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}"
|
|
i += 1
|
|
|
|
try:
|
|
os.replace(muxed_path, final_path)
|
|
except OSError:
|
|
if final_path.exists():
|
|
final_path.unlink()
|
|
shutil.move(muxed_path, final_path)
|
|
used_final_paths.add(final_path)
|
|
self.completed_files.append(final_path)
|
|
if muxed_path not in tagged_paths:
|
|
tags.tag_file(
|
|
final_path,
|
|
title,
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
self.anime_hint(title),
|
|
session=service.session,
|
|
)
|
|
|
|
post_script_context = build_context(
|
|
title,
|
|
media_info,
|
|
filepath=final_path,
|
|
sidecars=sidecar_files.get(muxed_path, []),
|
|
service=self.service,
|
|
ids=self.post_script_ids(),
|
|
)
|
|
dispatch("success", "file", post_script_context, postscript)
|
|
post_script_last.setdefault(post_script_group(title), {})[final_path.parent] = (
|
|
post_script_context
|
|
)
|
|
post_script_sample = post_script_context
|
|
if final_path.parent not in post_script_folders:
|
|
post_script_folders.append(final_path.parent)
|
|
|
|
if not no_mux:
|
|
group_key = post_script_group(title)
|
|
post_script_pending[group_key] = post_script_pending.get(group_key, 1) - 1
|
|
if post_script_pending[group_key] <= 0:
|
|
for folder, context in post_script_last.get(group_key, {}).items():
|
|
dispatch("success", "season", season_context(context, folder), postscript)
|
|
|
|
title_dl_time = time_elapsed_since(dl_start_time)
|
|
downloaded_label = "Track" if isinstance(title, Song) else "Title"
|
|
console.print(
|
|
Padding(
|
|
f":tada: {downloaded_label} downloaded in [progress.elapsed]{title_dl_time}[/]!",
|
|
(0, 5, 1, 5),
|
|
)
|
|
)
|
|
|
|
if progress_sink:
|
|
progress_sink({"output_files": [str(p) for p in self.completed_files]})
|
|
|
|
if not hasattr(service, "close"):
|
|
cookie_file = self.get_cookie_path(self.service, self.profile)
|
|
if cookie_file:
|
|
self.save_cookies(cookie_file, service.session.cookies)
|
|
|
|
if hasattr(service, "close"):
|
|
service.close()
|
|
|
|
if post_script_folders and post_script_sample:
|
|
run_context = dict.fromkeys(post_script_sample, "")
|
|
for folder in post_script_folders:
|
|
run_context["folder"] = str(folder)
|
|
dispatch("success", "run", run_context, postscript)
|
|
|
|
self.wait_vault_writes()
|
|
dl_time = time_elapsed_since(start_time)
|
|
|
|
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
|
|
|
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
|
|
"""Whether metadata lookups for this title should prefer AniList."""
|
|
per_title = getattr(title, "anime", None)
|
|
return self.service_anime if per_title is None else bool(per_title)
|
|
|
|
def daily_hint(self, title: Optional[Title_T] = None) -> bool:
|
|
"""Whether this title is daily content named by air date."""
|
|
per_title = getattr(title, "daily", None)
|
|
return self.daily or (self.service_daily if per_title is None else bool(per_title))
|
|
|
|
def fill_air_dates(self, titles: Series, tvdb_id: Optional[int] = None) -> None:
|
|
"""Fill in missing episode air dates from TVDB.
|
|
|
|
Additive only: an air date the service already set is kept.
|
|
"""
|
|
tvdb = providers.get_provider("tvdb")
|
|
if not tvdb:
|
|
self.log.debug("Air dates need a tvdb_api_key in your config, skipping.")
|
|
return
|
|
|
|
if not tvdb_id:
|
|
self.log.debug("Could not resolve a TVDB ID, skipping air dates.")
|
|
return
|
|
|
|
wanted = [t for t in titles if self.daily_hint(t)]
|
|
if not wanted:
|
|
return
|
|
|
|
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
|
|
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
|
|
episodes = tvdb.get_episodes(tvdb_id, source_order) # type: ignore[attr-defined]
|
|
if not episodes:
|
|
self.log.debug("TVDB listed no episodes for series %s, skipping air dates.", tvdb_id)
|
|
return
|
|
|
|
aired_by_key: dict[tuple[int, int], str] = {}
|
|
for episode in episodes:
|
|
season, number = parse_int(episode.get("seasonNumber")), parse_int(episode.get("number"))
|
|
aired = episode.get("aired")
|
|
if season is None or number is None or not aired:
|
|
continue
|
|
aired_by_key[(season, number)] = str(aired)
|
|
|
|
# TVDB carries placeholder schedule dates for unaired episodes. Stamping one would
|
|
# name the file with a date the episode never aired on.
|
|
limit = date.today() + timedelta(days=1)
|
|
|
|
filled = 0
|
|
for title in wanted:
|
|
if title.air_date is not None:
|
|
continue
|
|
aired = aired_by_key.get((title.season, title.number))
|
|
if not aired:
|
|
continue
|
|
try:
|
|
air_date = date.fromisoformat(aired[:10])
|
|
except ValueError:
|
|
continue
|
|
if air_date.year < 1970 or air_date > limit:
|
|
continue
|
|
title.air_date = air_date
|
|
filled += 1
|
|
|
|
if filled:
|
|
self.log.info("Filled air dates for %d episode(s) from TVDB (ID %s).", filled, tvdb_id)
|
|
|
|
def fill_absolute_numbers(self, titles: Series, tvdb_id: Optional[int] = None) -> None:
|
|
"""Fill in missing absolute episode numbers from TVDB's absolute order.
|
|
|
|
Additive only: season and number are never written, and an absolute number the
|
|
service already set is kept.
|
|
"""
|
|
tvdb = providers.get_provider("tvdb")
|
|
if not tvdb:
|
|
self.log.debug("Absolute episode numbers need a tvdb_api_key in your config, skipping.")
|
|
return
|
|
|
|
if not tvdb_id:
|
|
self.log.debug("Could not resolve a TVDB ID, skipping absolute episode numbers.")
|
|
return
|
|
|
|
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
|
|
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
|
|
if source_order == "absolute":
|
|
# the service already numbers in absolute order; get_order_map would return the
|
|
# empty same-order map, but here the episode numbers are their own fill
|
|
mapping = {key: (*key, None) for key in keys}
|
|
else:
|
|
mapping = tvdb.get_order_map(tvdb_id, "absolute", source_order) # type: ignore[attr-defined]
|
|
if not mapping:
|
|
self.log.debug("TVDB has no absolute order for series %s, skipping absolute episode numbers.", tvdb_id)
|
|
return
|
|
|
|
filled = 0
|
|
for title in titles:
|
|
if title.absolute is not None:
|
|
continue
|
|
entry = mapping.get((title.season, title.number))
|
|
if not entry:
|
|
continue
|
|
title.absolute = entry[1]
|
|
filled += 1
|
|
|
|
if filled:
|
|
self.log.info("Filled absolute numbers for %d episode(s) from TVDB (ID %s).", filled, tvdb_id)
|
|
|
|
def apply_tvdb_order(
|
|
self,
|
|
titles: Series,
|
|
title_cacher: Any = None,
|
|
cache_title_id: Optional[str] = None,
|
|
cache_region: Optional[str] = None,
|
|
cache_account_hash: Optional[str] = None,
|
|
) -> Series:
|
|
"""Renumber episodes from the service's own numbering into a TVDB season order."""
|
|
tvdb = providers.get_provider("tvdb")
|
|
if not tvdb:
|
|
self.log.warning("--tvdb-order needs a tvdb_api_key in your config, skipping renumber.")
|
|
return titles
|
|
|
|
tvdb_id = self.tvdb_id
|
|
if not tvdb_id:
|
|
show = titles[0]
|
|
result = providers.resolve_by_ids(
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
None,
|
|
title=show.title,
|
|
year=show.year,
|
|
kind="tv",
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
)
|
|
tvdb_id = result.external_ids.tvdb_id if result else None
|
|
if not tvdb_id:
|
|
self.log.warning("Could not resolve a TVDB ID for %r, skipping %s renumber.", str(titles), self.tvdb_order)
|
|
return titles
|
|
self.tvdb_id = tvdb_id # keep the mux tag on the series the renumbering used
|
|
|
|
# parts of one episode share a (season, number) slot; without the dedupe the
|
|
# collision check below reads them as several episodes clashing on one slot
|
|
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
|
|
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
|
|
if source_order == self.tvdb_order:
|
|
self.log.info("%s already lists this series in TVDB %s order.", self.service, self.tvdb_order)
|
|
return titles
|
|
|
|
mapping = tvdb.get_order_map(tvdb_id, self.tvdb_order, source_order) # type: ignore[attr-defined]
|
|
if not mapping:
|
|
return titles
|
|
|
|
# an unmapped episode keeps its own numbering and can land on a remapped episode's slot
|
|
renumbered = [mapping.get(key, (*key, None))[:2] for key in keys]
|
|
unmapped = sum(1 for key in keys if key not in mapping)
|
|
collisions = [slot for slot, count in Counter(renumbered).items() if count > 1]
|
|
if collisions:
|
|
self.log.error(
|
|
"TVDB's %s order does not cover %d of this series' episodes, and renumbering the rest "
|
|
"would give %d season/episode slot(s) two episodes each. Keeping the %s numbering "
|
|
"%s uses. Pick an order that covers the whole series.",
|
|
self.tvdb_order,
|
|
unmapped,
|
|
len(collisions),
|
|
source_order,
|
|
self.service,
|
|
)
|
|
return titles
|
|
|
|
for title in titles:
|
|
entry = mapping.get((title.season, title.number))
|
|
if not entry:
|
|
continue
|
|
title.season, title.number, tvdb_name = entry
|
|
if tvdb_name and not title.name:
|
|
title.name = tvdb_name
|
|
|
|
if unmapped:
|
|
self.log.warning(
|
|
"%d episode(s) are not in TVDB's %s order and keep their %s numbering.",
|
|
unmapped,
|
|
self.tvdb_order,
|
|
source_order,
|
|
)
|
|
self.log.info("Renumbered episodes from TVDB %s to %s order (ID %s).", source_order, self.tvdb_order, tvdb_id)
|
|
|
|
# Series is a SortedKeyList; renumbering in place does not re-sort it
|
|
return type(titles)(list(titles))
|
|
|
|
@staticmethod
|
|
def title_to_meta(title: Title_T) -> dict[str, Any]:
|
|
"""Capture the title fields needed to rebuild a Title for output naming on import."""
|
|
from datetime import date
|
|
|
|
from unshackle.core.titles import Episode, Movie
|
|
|
|
meta: dict[str, Any] = {
|
|
"id": str(title.id),
|
|
"language": str(title.language) if getattr(title, "language", None) else None,
|
|
}
|
|
if isinstance(title, Episode):
|
|
meta.update(
|
|
type="episode",
|
|
series_title=title.title,
|
|
season=title.season,
|
|
number=title.number,
|
|
name=title.name,
|
|
year=title.year,
|
|
air_date=title.air_date.isoformat() if isinstance(title.air_date, date) else title.air_date,
|
|
)
|
|
# conditional, so meta for a title without them is unchanged
|
|
if title.part is not None:
|
|
meta["part"] = title.part
|
|
if title.absolute is not None:
|
|
meta["absolute"] = title.absolute
|
|
if getattr(title, "daily", None) is not None:
|
|
meta["daily"] = title.daily
|
|
elif isinstance(title, Movie):
|
|
meta.update(type="movie", name=title.name, year=title.year)
|
|
else:
|
|
meta.update(type="movie", name=str(title))
|
|
if getattr(title, "anime", None) is not None:
|
|
meta["anime"] = title.anime
|
|
return meta
|
|
|
|
def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any = None) -> None:
|
|
"""Write a shareable v2 export usable by ``unshackle import``.
|
|
|
|
Carries no HTTP session, cookies, or dl-flags. The export records the region (country
|
|
code) only when the export used ``--proxy``, as an import geofence. Each track records
|
|
only the licensed DRM system. Content keys live once under the track's ``keys``. ``drm`` may be None
|
|
(DRM-free track) or a DRM system without ``to_dict``/``content_keys`` (e.g. ClearKey) -
|
|
the export still records the track, manifest, chapter and attachment info.
|
|
"""
|
|
with self.EXPORT_LOCK:
|
|
doc: dict[str, Any] = {}
|
|
if export.is_file():
|
|
doc = json.loads(export.read_text(encoding="utf8")) or {}
|
|
|
|
doc.setdefault("version", 2)
|
|
doc.setdefault("service", self.service)
|
|
if "region" not in doc and getattr(self, "proxy_requested", False):
|
|
region = getattr(getattr(self, "export_service", None), "current_region", None)
|
|
if region:
|
|
doc["region"] = region
|
|
|
|
titles = doc.setdefault("titles", {})
|
|
tinfo = titles.setdefault(str(title.id), {})
|
|
tinfo.setdefault("meta", self.title_to_meta(title))
|
|
|
|
if title.tracks.manifest_url:
|
|
tinfo.setdefault("manifest_url", title.tracks.manifest_url)
|
|
|
|
all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]
|
|
if "manifest_type" not in tinfo:
|
|
tinfo["manifest_type"] = next(
|
|
(t.descriptor.name for t in all_tracks if t.descriptor != Video.Descriptor.URL), None
|
|
)
|
|
|
|
tracks_map = tinfo.setdefault("tracks", {})
|
|
if not tracks_map:
|
|
for t in all_tracks:
|
|
tracks_map[str(t.id)] = t.to_dict()
|
|
|
|
track_data = tracks_map.setdefault(str(track.id), track.to_dict())
|
|
if drm is not None:
|
|
if hasattr(drm, "to_dict"):
|
|
track_data["drm"] = [drm.to_dict()]
|
|
content_keys = getattr(drm, "content_keys", None) or {}
|
|
if content_keys:
|
|
keys = track_data.setdefault("keys", {})
|
|
for kid, key in content_keys.items():
|
|
keys[kid.hex] = key
|
|
|
|
if "chapters" not in tinfo:
|
|
tinfo["chapters"] = [
|
|
{"timestamp": chapter.timestamp, "name": chapter.name} for chapter in (title.tracks.chapters or [])
|
|
]
|
|
|
|
if "attachments" not in tinfo:
|
|
tinfo["attachments"] = [a.to_dict() for a in (title.tracks.attachments or []) if a.url]
|
|
|
|
export.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf8")
|
|
|
|
def flush_vault_writes(self, pending: list[Callable[[], Any]]) -> None:
|
|
"""Hand the queued vault writes to the background writer so the track does not wait on them."""
|
|
for write in pending:
|
|
self.VAULT_WRITER.submit(write).add_done_callback(self.report_vault_write)
|
|
pending.clear()
|
|
|
|
def report_vault_write(self, future: futures.Future[Any]) -> None:
|
|
if exc := future.exception():
|
|
self.log.error(f"Vault write failed: {exc!r}")
|
|
|
|
def wait_vault_writes(self) -> None:
|
|
"""Block until every queued vault write has run, then log the total once."""
|
|
self.VAULT_WRITER.submit(lambda: None).result()
|
|
if not (self.vault_cache_tally and self.vault_cache_tally[0]):
|
|
self.vault_cache_tally = None
|
|
return
|
|
keys, successful_caches = self.vault_cache_tally
|
|
self.vault_cache_tally = None
|
|
self.log.info(f"Cached {keys} Key{'' if keys == 1 else 's'} to {successful_caches}/{len(self.vaults)} Vaults")
|
|
|
|
def cache_keys_to_vaults(self, content_keys: dict[UUID, str]) -> None:
|
|
successful_caches = self.vaults.add_keys(content_keys)
|
|
keys, caches = self.vault_cache_tally or (0, successful_caches)
|
|
self.vault_cache_tally = (keys + len(content_keys), min(caches, successful_caches))
|
|
|
|
@classmethod
|
|
def drm_lock(cls, drm: DRM_T) -> Lock:
|
|
"""Return the lock for one DRM's set of content keys.
|
|
|
|
Tracks that share KIDs still take turns, so the second one finds the content keys in
|
|
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs send a challenge
|
|
and query the key vaults at the same time.
|
|
"""
|
|
kids = getattr(drm, "kids", None) or []
|
|
key = ",".join(sorted(getattr(k, "hex", str(k)) for k in kids)) or getattr(drm, "content_id", None)
|
|
with cls.DRM_LOCKS_GUARD:
|
|
return cls.DRM_LOCKS.setdefault(key or type(drm).__name__, Lock())
|
|
|
|
def service_licence(self, service: Service, drm_system: Optional[str] = None, **kwargs: Any) -> Any:
|
|
"""Call the service's licence function for the track's DRM system.
|
|
|
|
prepare_drm passes drm_system so the choice follows the track, not the loaded CDM:
|
|
a track downloading at the same time can swap self.cdm.
|
|
"""
|
|
if drm_system is None:
|
|
drm_system = "playready" if is_playready_cdm(self.cdm) else "widevine"
|
|
if drm_system == "playready":
|
|
return service.get_playready_license(**kwargs)
|
|
return service.get_widevine_license(**kwargs)
|
|
|
|
def prepare_drm(
|
|
self,
|
|
drm: DRM_T,
|
|
track: AnyTrack,
|
|
title: Title_T,
|
|
certificate: Callable,
|
|
licence: Callable,
|
|
clearkey_licence: Optional[Callable] = None,
|
|
track_kid: Optional[UUID] = None,
|
|
table: Table = None,
|
|
cdm_only: bool = False,
|
|
vaults_only: bool = False,
|
|
export: Optional[Path] = None,
|
|
service_session: Optional[Any] = None,
|
|
) -> None:
|
|
"""
|
|
Prepare the DRM by getting decryption data like KIDs, Keys, and such.
|
|
The DRM object should be ready for decryption once this function ends.
|
|
"""
|
|
if not drm:
|
|
return
|
|
|
|
svc_for_cdm = getattr(self, "_remote_service", None)
|
|
server_cdm = getattr(svc_for_cdm, "_server_cdm", getattr(self, "server_cdm", False))
|
|
|
|
if server_cdm:
|
|
if not cdm_only:
|
|
vault_kids = list(getattr(drm, "kids", None) or [])
|
|
if track_kid and track_kid not in vault_kids:
|
|
vault_kids.append(track_kid)
|
|
for kid in vault_kids:
|
|
if kid in drm.content_keys:
|
|
continue
|
|
content_key = self.LICENSE_KEY_CACHE.get(kid) or self.vaults.get_key(kid)[0]
|
|
if content_key:
|
|
drm.content_keys[kid] = content_key
|
|
self.LICENSE_KEY_CACHE[kid] = content_key
|
|
|
|
def missing_track_key() -> bool:
|
|
return not drm.content_keys or bool(track_kid and track_kid not in drm.content_keys)
|
|
|
|
if missing_track_key():
|
|
try:
|
|
licence(
|
|
drm_system="playready" if drm.__class__.__name__ == "PlayReady" else "widevine",
|
|
challenge=b"",
|
|
)
|
|
except Exception as e:
|
|
self.log.debug(f"Server CDM licence with an empty challenge failed: {e!r}")
|
|
|
|
if not drm.content_keys:
|
|
self.log.warning("Server CDM did not resolve any keys for this track")
|
|
return
|
|
if track_kid and track_kid not in drm.content_keys:
|
|
msg = f"No Content Key for KID {track_kid.hex} was returned by the server CDM"
|
|
if isinstance(drm, PlayReady):
|
|
raise PlayReady.Exceptions.CEKNotFound(msg)
|
|
raise Widevine.Exceptions.CEKNotFound(msg)
|
|
svc = getattr(self, "_remote_service", None)
|
|
server_drm_type = getattr(svc, "_server_cdm_type", None) if svc else None
|
|
drm_name = {"widevine": "Widevine", "playready": "PlayReady"}.get(
|
|
server_drm_type or "", drm.__class__.__name__
|
|
)
|
|
log_event(
|
|
"drm_content_keys",
|
|
level="INFO",
|
|
message=f"Server CDM resolved {len(drm.content_keys)} {drm_name} content key(s)",
|
|
service=self.service,
|
|
drm_type=drm_name,
|
|
key_count=len(drm.content_keys),
|
|
keys=[{"kid": k.hex, "key": v} for k, v in drm.content_keys.items()],
|
|
remote=True,
|
|
)
|
|
with self.drm_lock(drm):
|
|
pssh_str = ""
|
|
expected_class = "PlayReady" if server_drm_type == "playready" else "Widevine"
|
|
matching_drm = next(
|
|
(d for d in (track.drm or []) if d.__class__.__name__ == expected_class),
|
|
drm,
|
|
)
|
|
if hasattr(matching_drm, "pssh") and matching_drm.pssh:
|
|
if hasattr(matching_drm.pssh, "dumps"):
|
|
pssh_str = self.truncate_pssh_for_display(matching_drm.pssh.dumps(), drm_name)
|
|
elif hasattr(matching_drm, "data") and matching_drm.data.get("pssh_b64"):
|
|
pssh_str = self.truncate_pssh_for_display(matching_drm.data["pssh_b64"], drm_name)
|
|
if pssh_str:
|
|
cek_tree = Tree(Text.assemble((drm_name, "cyan"), (f"({pssh_str})", "text"), overflow="fold"))
|
|
else:
|
|
cek_tree = Tree(Text.assemble((drm_name, "cyan"), overflow="fold"))
|
|
all_kids = list(getattr(drm, "kids", []))
|
|
if track_kid and track_kid not in all_kids:
|
|
all_kids.append(track_kid)
|
|
for kid in all_kids:
|
|
if kid in drm.content_keys:
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
key = drm.content_keys[kid]
|
|
cek_tree.add(f"[text2]{kid.hex}:{key}{is_track_kid}")
|
|
for kid, key in drm.content_keys.items():
|
|
if kid not in all_kids:
|
|
cek_tree.add(f"[text2]{kid.hex}:{key}")
|
|
if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells):
|
|
table.add_row(cek_tree)
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
return
|
|
|
|
track_quality = None
|
|
if isinstance(track, Video) and track.height:
|
|
track_quality = track.height
|
|
elif title is not None:
|
|
track_quality = max((v.height for v in title.tracks.videos if v.height), default=None)
|
|
|
|
track_cdm = self.cdm
|
|
|
|
licence = partial(licence, drm_system="playready" if isinstance(drm, PlayReady) else "widevine")
|
|
|
|
if not server_cdm:
|
|
if isinstance(drm, Widevine):
|
|
if not is_widevine_cdm(self.cdm):
|
|
widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality)
|
|
if widevine_cdm and is_widevine_cdm(widevine_cdm):
|
|
if track_quality:
|
|
self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content")
|
|
else:
|
|
self.log.info("Switching to Widevine CDM for Widevine content")
|
|
self.cdm = widevine_cdm
|
|
track_cdm = widevine_cdm
|
|
else:
|
|
raise ValueError(f"Title needs a Widevine CDM but {self.service} is configured with PlayReady.")
|
|
|
|
elif isinstance(drm, PlayReady):
|
|
if not is_playready_cdm(self.cdm):
|
|
playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality)
|
|
if playready_cdm and is_playready_cdm(playready_cdm):
|
|
if track_quality:
|
|
self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content")
|
|
else:
|
|
self.log.info("Switching to PlayReady CDM for PlayReady content")
|
|
self.cdm = playready_cdm
|
|
track_cdm = playready_cdm
|
|
else:
|
|
raise ValueError(f"Title needs a PlayReady CDM but {self.service} is configured with Widevine.")
|
|
|
|
if isinstance(drm, Widevine):
|
|
if self.debug_logger:
|
|
self.debug_logger.log_drm_operation(
|
|
drm_type="Widevine",
|
|
operation="prepare_drm",
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"title": str(title),
|
|
"pssh": drm.pssh.dumps() if drm.pssh else None,
|
|
"kids": [k.hex for k in drm.kids],
|
|
"track_kid": track_kid.hex if track_kid else None,
|
|
},
|
|
)
|
|
|
|
pending_vault_writes: list[Callable[[], Any]] = []
|
|
with self.drm_lock(drm):
|
|
pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine")
|
|
cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
|
|
pre_existing_tree = next(
|
|
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
|
)
|
|
if pre_existing_tree:
|
|
cek_tree = pre_existing_tree
|
|
|
|
need_license = False
|
|
all_kids = list(drm.kids)
|
|
if track_kid and track_kid not in all_kids:
|
|
all_kids.append(track_kid)
|
|
|
|
for kid in all_kids:
|
|
if kid in drm.content_keys:
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
key = drm.content_keys[kid]
|
|
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
|
|
if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
continue
|
|
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
|
|
cached_key = self.LICENSE_KEY_CACHE.get(kid)
|
|
if cached_key:
|
|
drm.content_keys[kid] = cached_key
|
|
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
|
|
if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
log_event(
|
|
"license_cache_hit",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"kid": kid.hex,
|
|
"content_key": cached_key,
|
|
"track": str(track),
|
|
"drm_type": "Widevine",
|
|
},
|
|
)
|
|
continue
|
|
|
|
if not cdm_only:
|
|
content_key, vault_used = self.vaults.get_key(kid)
|
|
if content_key:
|
|
drm.content_keys[kid] = content_key
|
|
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
|
|
if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
pending_vault_writes.append(
|
|
partial(self.vaults.add_key, kid, content_key, excluding=vault_used)
|
|
)
|
|
self.LICENSE_KEY_CACHE[kid] = content_key
|
|
|
|
if self.debug_logger:
|
|
self.debug_logger.log_vault_query(
|
|
vault_name=vault_used,
|
|
operation="get_key_success",
|
|
service=self.service,
|
|
context={
|
|
"kid": kid.hex,
|
|
"content_key": content_key,
|
|
"track": str(track),
|
|
"from_cache": True,
|
|
},
|
|
)
|
|
elif vaults_only:
|
|
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
log_event(
|
|
"vault_key_not_found",
|
|
level="ERROR",
|
|
service=self.service,
|
|
message=msg,
|
|
context={"kid": kid.hex, "track": str(track)},
|
|
)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise Widevine.Exceptions.CEKNotFound(msg)
|
|
else:
|
|
need_license = True
|
|
|
|
if kid not in drm.content_keys and cdm_only:
|
|
need_license = True
|
|
|
|
if need_license and all(kid in drm.content_keys for kid in all_kids):
|
|
need_license = False
|
|
|
|
if need_license and not vaults_only:
|
|
from_vaults = drm.content_keys.copy()
|
|
|
|
log_event(
|
|
"get_license",
|
|
level="INFO",
|
|
service=self.service,
|
|
message="Requesting Widevine license from service",
|
|
context={
|
|
"track": str(track),
|
|
"kids_needed": [k.hex for k in all_kids if k not in drm.content_keys],
|
|
},
|
|
)
|
|
|
|
try:
|
|
if self.service == "NF":
|
|
drm.get_NF_content_keys(cdm=track_cdm, licence=licence, certificate=certificate)
|
|
else:
|
|
drm.get_content_keys(cdm=track_cdm, licence=licence, certificate=certificate)
|
|
except Exception as e:
|
|
if drm.content_keys:
|
|
self.log.debug(f"License call failed but keys already in content_keys: {e}")
|
|
else:
|
|
if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)):
|
|
msg = str(e)
|
|
else:
|
|
msg = f"An exception occurred in the Service's license function: {e}"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"get_license",
|
|
e,
|
|
service=self.service,
|
|
context={"track": str(track), "exception_type": type(e).__name__},
|
|
)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise e
|
|
|
|
log_event(
|
|
"license_keys_retrieved",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"keys_count": len(drm.content_keys),
|
|
"kids": [k.hex for k in drm.content_keys.keys()],
|
|
},
|
|
)
|
|
|
|
for kid_, key in drm.content_keys.items():
|
|
if key == "0" * 32:
|
|
key = f"[red]{key}[/]"
|
|
is_track_kid_marker = ["", "*"][kid_ == track_kid]
|
|
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
|
|
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
|
|
drm.content_keys = {
|
|
kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key)
|
|
}
|
|
|
|
# The CDM keys may have returned blank content keys for KIDs we got from vaults.
|
|
# So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data.
|
|
drm.content_keys.update(from_vaults)
|
|
|
|
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
|
|
|
pending_vault_writes.append(partial(self.cache_keys_to_vaults, dict(drm.content_keys)))
|
|
|
|
if track_kid and track_kid not in drm.content_keys:
|
|
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise Widevine.Exceptions.CEKNotFound(msg)
|
|
|
|
if cek_tree.children and not pre_existing_tree:
|
|
table.add_row()
|
|
table.add_row(cek_tree)
|
|
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
|
|
elif isinstance(drm, PlayReady):
|
|
if self.debug_logger:
|
|
self.debug_logger.log_drm_operation(
|
|
drm_type="PlayReady",
|
|
operation="prepare_drm",
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"title": str(title),
|
|
"pssh": drm.pssh_b64 or "",
|
|
"kids": [k.hex for k in drm.kids],
|
|
"track_kid": track_kid.hex if track_kid else None,
|
|
},
|
|
)
|
|
|
|
pending_vault_writes = []
|
|
with self.drm_lock(drm):
|
|
pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady")
|
|
cek_tree = Tree(
|
|
Text.assemble(
|
|
("PlayReady", "cyan"),
|
|
(f"({pssh_display})", "text"),
|
|
overflow="fold",
|
|
)
|
|
)
|
|
pre_existing_tree = next(
|
|
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
|
)
|
|
if pre_existing_tree:
|
|
cek_tree = pre_existing_tree
|
|
|
|
need_license = False
|
|
all_kids = list(drm.kids)
|
|
if track_kid and track_kid not in all_kids:
|
|
all_kids.append(track_kid)
|
|
|
|
for kid in all_kids:
|
|
if kid in drm.content_keys:
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
key = drm.content_keys[kid]
|
|
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
|
|
if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
continue
|
|
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
|
|
cached_key = self.LICENSE_KEY_CACHE.get(kid)
|
|
if cached_key:
|
|
drm.content_keys[kid] = cached_key
|
|
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
|
|
if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
log_event(
|
|
"license_cache_hit",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"kid": kid.hex,
|
|
"content_key": cached_key,
|
|
"track": str(track),
|
|
"drm_type": "PlayReady",
|
|
},
|
|
)
|
|
continue
|
|
|
|
if not cdm_only:
|
|
content_key, vault_used = self.vaults.get_key(kid)
|
|
if content_key:
|
|
drm.content_keys[kid] = content_key
|
|
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
|
|
if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
pending_vault_writes.append(
|
|
partial(self.vaults.add_key, kid, content_key, excluding=vault_used)
|
|
)
|
|
self.LICENSE_KEY_CACHE[kid] = content_key
|
|
|
|
if self.debug_logger:
|
|
self.debug_logger.log_vault_query(
|
|
vault_name=vault_used,
|
|
operation="get_key_success",
|
|
service=self.service,
|
|
context={
|
|
"kid": kid.hex,
|
|
"content_key": content_key,
|
|
"track": str(track),
|
|
"from_cache": True,
|
|
"drm_type": "PlayReady",
|
|
},
|
|
)
|
|
elif vaults_only:
|
|
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
log_event(
|
|
"vault_key_not_found",
|
|
level="ERROR",
|
|
service=self.service,
|
|
message=msg,
|
|
context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"},
|
|
)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise PlayReady.Exceptions.CEKNotFound(msg)
|
|
else:
|
|
need_license = True
|
|
|
|
if kid not in drm.content_keys and cdm_only:
|
|
need_license = True
|
|
|
|
if need_license and all(kid in drm.content_keys for kid in all_kids):
|
|
need_license = False
|
|
|
|
if need_license and not vaults_only:
|
|
from_vaults = drm.content_keys.copy()
|
|
|
|
try:
|
|
drm.get_content_keys(cdm=track_cdm, licence=licence, certificate=certificate)
|
|
except Exception as e:
|
|
if drm.content_keys:
|
|
self.log.debug(f"License call failed but keys already in content_keys: {e}")
|
|
else:
|
|
if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)):
|
|
msg = str(e)
|
|
else:
|
|
msg = f"An exception occurred in the Service's license function: {e}"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"get_license_playready",
|
|
e,
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"exception_type": type(e).__name__,
|
|
"drm_type": "PlayReady",
|
|
},
|
|
)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise e
|
|
|
|
for kid_, key in drm.content_keys.items():
|
|
is_track_kid_marker = ["", "*"][kid_ == track_kid]
|
|
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
|
|
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
|
|
drm.content_keys.update(from_vaults)
|
|
|
|
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
|
|
|
pending_vault_writes.append(partial(self.cache_keys_to_vaults, dict(drm.content_keys)))
|
|
|
|
if track_kid and track_kid not in drm.content_keys:
|
|
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise PlayReady.Exceptions.CEKNotFound(msg)
|
|
|
|
if cek_tree.children and not pre_existing_tree:
|
|
table.add_row()
|
|
table.add_row(cek_tree)
|
|
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
|
|
elif isinstance(drm, ClearKeyCENC):
|
|
pending_vault_writes = []
|
|
with self.drm_lock(drm):
|
|
cek_tree = Tree(Text.assemble(("ClearKey", "cyan"), overflow="fold"))
|
|
pre_existing_tree = next(
|
|
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
|
)
|
|
if pre_existing_tree:
|
|
cek_tree = pre_existing_tree
|
|
|
|
need_license = False
|
|
all_kids = list(drm.kids)
|
|
if track_kid and track_kid not in all_kids:
|
|
all_kids.append(track_kid)
|
|
|
|
for kid in all_kids:
|
|
if kid in drm.content_keys:
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
key = drm.content_keys[kid]
|
|
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
|
|
if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
continue
|
|
|
|
is_track_kid = ["", "*"][kid == track_kid]
|
|
|
|
cached_key = self.LICENSE_KEY_CACHE.get(kid)
|
|
if cached_key:
|
|
drm.content_keys[kid] = cached_key
|
|
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
|
|
if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
log_event(
|
|
"license_cache_hit",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"kid": kid.hex,
|
|
"content_key": cached_key,
|
|
"track": str(track),
|
|
"drm_type": "ClearKeyCENC",
|
|
},
|
|
)
|
|
continue
|
|
|
|
if not cdm_only:
|
|
content_key, vault_used = self.vaults.get_key(kid)
|
|
if content_key:
|
|
drm.content_keys[kid] = content_key
|
|
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
|
|
if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
pending_vault_writes.append(
|
|
partial(self.vaults.add_key, kid, content_key, excluding=vault_used)
|
|
)
|
|
self.LICENSE_KEY_CACHE[kid] = content_key
|
|
elif vaults_only:
|
|
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
log_event(
|
|
"vault_key_not_found",
|
|
level="ERROR",
|
|
service=self.service,
|
|
message=msg,
|
|
context={"kid": kid.hex, "track": str(track), "drm_type": "ClearKeyCENC"},
|
|
)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise ClearKeyCENC.Exceptions.CEKNotFound(msg)
|
|
else:
|
|
need_license = True
|
|
|
|
if kid not in drm.content_keys and cdm_only:
|
|
need_license = True
|
|
|
|
if need_license and all(kid in drm.content_keys for kid in all_kids):
|
|
need_license = False
|
|
|
|
if need_license and not vaults_only:
|
|
from_vaults = drm.content_keys.copy()
|
|
|
|
try:
|
|
drm.get_content_keys(licence=clearkey_licence or (lambda **_: None), session=service_session)
|
|
except Exception as e:
|
|
if drm.content_keys:
|
|
self.log.debug(f"License call failed but keys already in content_keys: {e}")
|
|
else:
|
|
if isinstance(
|
|
e, (ClearKeyCENC.Exceptions.EmptyLicense, ClearKeyCENC.Exceptions.CEKNotFound)
|
|
):
|
|
msg = str(e)
|
|
else:
|
|
msg = f"An exception occurred in the Service's license function: {e}"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"get_license_clearkey",
|
|
e,
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"exception_type": type(e).__name__,
|
|
"drm_type": "ClearKeyCENC",
|
|
},
|
|
)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise e
|
|
|
|
for kid_, key in drm.content_keys.items():
|
|
is_track_kid_marker = ["", "*"][kid_ == track_kid]
|
|
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
|
|
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
|
|
drm.content_keys.update(from_vaults)
|
|
|
|
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
|
|
|
pending_vault_writes.append(partial(self.cache_keys_to_vaults, dict(drm.content_keys)))
|
|
|
|
if track_kid and track_kid not in drm.content_keys:
|
|
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
raise ClearKeyCENC.Exceptions.CEKNotFound(msg)
|
|
|
|
if cek_tree.children and not pre_existing_tree:
|
|
table.add_row()
|
|
table.add_row(cek_tree)
|
|
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
|
|
elif isinstance(drm, MonaLisa):
|
|
with self.drm_lock(drm):
|
|
display_id = drm.content_id or drm.pssh
|
|
pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa")
|
|
cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
|
|
pre_existing_tree = next(
|
|
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
|
)
|
|
if pre_existing_tree:
|
|
cek_tree = pre_existing_tree
|
|
|
|
for kid_, key in drm.content_keys.items():
|
|
label = f"[text2]{kid_.hex}:{key}*"
|
|
if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children):
|
|
cek_tree.add(label)
|
|
|
|
if cek_tree.children and not pre_existing_tree:
|
|
table.add_row()
|
|
table.add_row(cek_tree)
|
|
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
|
|
@staticmethod
|
|
def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]:
|
|
"""Get Service Cookie File Path for Profile."""
|
|
direct_cookie_file = config.directories.cookies / f"{service}.txt"
|
|
profile_cookie_file = config.directories.cookies / service / f"{profile}.txt"
|
|
default_cookie_file = config.directories.cookies / service / "default.txt"
|
|
|
|
if direct_cookie_file.exists():
|
|
return direct_cookie_file
|
|
elif profile_cookie_file.exists():
|
|
return profile_cookie_file
|
|
elif default_cookie_file.exists():
|
|
return default_cookie_file
|
|
|
|
@staticmethod
|
|
def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[CookieJar]:
|
|
"""
|
|
Get Service Cookies for Profile.
|
|
|
|
If the config has firefox_cookies for the service, cookies come from Firefox first. If that extraction
|
|
raises or returns nothing, unshackle uses the profile's cookie file instead and does not log the failure.
|
|
"""
|
|
ff_settings = getattr(config, "firefox_cookies", {}).get(service)
|
|
if ff_settings:
|
|
try:
|
|
from unshackle.core.utils.firefox_cookie_extractor import get_firefox_cookies
|
|
|
|
extracted_jar = get_firefox_cookies(ff_settings)
|
|
if extracted_jar:
|
|
return extracted_jar
|
|
except Exception:
|
|
pass
|
|
|
|
cookie_file = dl.get_cookie_path(service, profile)
|
|
if cookie_file:
|
|
return dl.load_cookie_file(cookie_file)
|
|
|
|
return None
|
|
|
|
@staticmethod
|
|
def load_cookie_file(cookie_file: Path) -> MozillaCookieJar:
|
|
"""Load a Netscape cookie file with every expiry blanked, so no layer drops an expired cookie."""
|
|
cookie_jar = MozillaCookieJar(cookie_file)
|
|
cookie_data = html.unescape(cookie_file.read_text("utf-8-sig")).splitlines(keepends=False)
|
|
head = next((line.lstrip() for line in cookie_data if line.strip()), "")
|
|
if head[:1] in ("[", "{"):
|
|
raise ValueError(f"{cookie_file} is a JSON export; export it again in Netscape format")
|
|
for i, line in enumerate(cookie_data):
|
|
body = line.lstrip()
|
|
prefix = "#HttpOnly_" if body.startswith("#HttpOnly_") else ""
|
|
body = body[len(prefix) :]
|
|
if not body or body.startswith("#"):
|
|
continue
|
|
line_data = body.split("\t") if "\t" in body else body.split(None, 6)
|
|
if len(line_data) == 6:
|
|
line_data.append("")
|
|
if len(line_data) != 7:
|
|
raise ValueError(f"{cookie_file} line {i + 1} is not a Netscape cookie row: {line!r}")
|
|
line_data[4] = ""
|
|
line_data[1] = str(line_data[0].startswith(".")).upper()
|
|
cookie_data[i] = prefix + "\t".join(line_data)
|
|
cookie_data.insert(0, "# Netscape HTTP Cookie File")
|
|
cookie_jar._really_load(StringIO("\n".join(cookie_data)), str(cookie_file), True, True) # type: ignore[attr-defined]
|
|
if not len(cookie_jar):
|
|
raise ValueError(f"{cookie_file} holds no cookies; export it again in Netscape format")
|
|
return cookie_jar
|
|
|
|
@staticmethod
|
|
def save_cookies(path: Path, cookies: CookieJar):
|
|
if hasattr(cookies, "jar"):
|
|
cookies = cookies.jar
|
|
|
|
cookie_jar = dl.load_cookie_file(path) if path.exists() else MozillaCookieJar(path)
|
|
for cookie in cookies:
|
|
if not cookie.name or not cookie.domain:
|
|
continue
|
|
cookie_jar.set_cookie(cookie)
|
|
cookie_jar.save(ignore_discard=True, ignore_expires=True)
|
|
|
|
@staticmethod
|
|
def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]:
|
|
"""Get Service Credentials for Profile."""
|
|
credentials = config.credentials.get(service)
|
|
if credentials:
|
|
if isinstance(credentials, dict):
|
|
if profile:
|
|
credentials = credentials.get(profile) or credentials.get("default")
|
|
else:
|
|
credentials = credentials.get("default")
|
|
if credentials:
|
|
if isinstance(credentials, list):
|
|
return Credential(*credentials)
|
|
return Credential.loads(credentials) # type: ignore
|
|
|
|
def get_cdm(
|
|
self,
|
|
service: str,
|
|
profile: Optional[str] = None,
|
|
drm: Optional[str] = None,
|
|
quality: Optional[int] = None,
|
|
) -> Optional[object]:
|
|
"""
|
|
Get CDM for a specified service (either Local or Remote CDM).
|
|
Now supports quality-based selection when the caller gives a quality.
|
|
Raises a ValueError if there is a problem getting a CDM.
|
|
"""
|
|
# A per-request override (set by the REST API per job) takes precedence over the
|
|
# global config, so a job can select a specific CDM device without mutating shared state.
|
|
cdm_name = resolve_cdm_name(config.cdm, service, getattr(self, "cdm_override", None))
|
|
if not cdm_name:
|
|
return None
|
|
|
|
if isinstance(cdm_name, dict):
|
|
if quality:
|
|
quality_match = None
|
|
quality_keys = []
|
|
|
|
for key in cdm_name.keys():
|
|
if (
|
|
isinstance(key, str)
|
|
and any(op in key for op in [">=", ">", "<=", "<"])
|
|
or (isinstance(key, str) and key.isdigit())
|
|
):
|
|
quality_keys.append(key)
|
|
|
|
def sort_quality_key(key):
|
|
if key.isdigit():
|
|
return (0, int(key)) # Exact matches first
|
|
elif key.startswith(">="):
|
|
return (1, -int(key[2:])) # >= descending
|
|
elif key.startswith(">"):
|
|
return (1, -int(key[1:])) # > descending
|
|
elif key.startswith("<="):
|
|
return (2, int(key[2:])) # <= ascending
|
|
elif key.startswith("<"):
|
|
return (2, int(key[1:])) # < ascending
|
|
return (3, 0) # Other keys last
|
|
|
|
quality_keys.sort(key=sort_quality_key)
|
|
|
|
for key in quality_keys:
|
|
if key.isdigit() and quality == int(key):
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on exact quality match {quality}p: {quality_match}")
|
|
break
|
|
elif key.startswith(">="):
|
|
threshold = int(key[2:])
|
|
if quality >= threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p >= {threshold}p: {quality_match}")
|
|
break
|
|
elif key.startswith(">"):
|
|
threshold = int(key[1:])
|
|
if quality > threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p > {threshold}p: {quality_match}")
|
|
break
|
|
elif key.startswith("<="):
|
|
threshold = int(key[2:])
|
|
if quality <= threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p <= {threshold}p: {quality_match}")
|
|
break
|
|
elif key.startswith("<"):
|
|
threshold = int(key[1:])
|
|
if quality < threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p < {threshold}p: {quality_match}")
|
|
break
|
|
|
|
if quality_match:
|
|
cdm_name = quality_match
|
|
|
|
if isinstance(cdm_name, dict):
|
|
lower_keys = {k.lower(): v for k, v in cdm_name.items()}
|
|
if {"widevine", "playready"} & lower_keys.keys():
|
|
drm_key = None
|
|
if drm:
|
|
drm_key = {
|
|
"wv": "widevine",
|
|
"widevine": "widevine",
|
|
"pr": "playready",
|
|
"playready": "playready",
|
|
}.get(drm.lower())
|
|
cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready")
|
|
else:
|
|
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(config.cdm, "default")
|
|
if not cdm_name:
|
|
return None
|
|
|
|
from unshackle.core.cdm import load_cdm
|
|
|
|
return load_cdm(cdm_name, service_name=service, vaults=self.vaults)
|