mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-17 14:35:06 +00:00
One video language request that matches several regional tags now gives one video, the best of them; hybrid bases and the DV source use the same pick, and hybrid --worst no longer needs -q. Thanks to reel.deal for the report and the first patch.
5729 lines
267 KiB
Python
5729 lines
267 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import html
|
|
import logging
|
|
import math
|
|
import os
|
|
import random
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import Counter, defaultdict
|
|
from collections.abc import Iterable
|
|
from concurrent import futures
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import ExitStack
|
|
from copy import deepcopy
|
|
from datetime import date, timedelta
|
|
from functools import partial
|
|
from http.cookiejar import CookieJar, MozillaCookieJar
|
|
from io import StringIO
|
|
from itertools import groupby
|
|
from operator import attrgetter
|
|
from pathlib import Path
|
|
from threading import Lock
|
|
from typing import TYPE_CHECKING, Any, Callable, Collection, Optional, Sequence, TypedDict, Union, cast
|
|
from uuid import UUID
|
|
|
|
import click
|
|
import mediaexport
|
|
from click.core import ParameterSource
|
|
from langcodes import Language, tag_is_valid
|
|
from pymediainfo import MediaInfo
|
|
from rich.console import Group
|
|
from rich.padding import Padding
|
|
from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn
|
|
from rich.rule import Rule
|
|
from rich.spinner import Spinner
|
|
from rich.table import Table
|
|
from rich.text import Text
|
|
from rich.tree import Tree
|
|
|
|
from unshackle.core import __version__, binaries, providers
|
|
from unshackle.core.cdm import DecryptLabsRemoteCDM
|
|
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote_cdm, is_widevine_cdm
|
|
from unshackle.core.config import cdm_entry_names_device, config, load_yaml, resolve_cdm_name, resolve_decryption
|
|
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel, print_listing
|
|
from unshackle.core.constants import (
|
|
DEFAULT_SAMPLE,
|
|
DOWNLOAD_ALL_DRM,
|
|
DOWNLOAD_CANCELLED,
|
|
DOWNLOAD_LICENCE_ONLY,
|
|
AnyTrack,
|
|
DownloadCancelled,
|
|
context_settings,
|
|
)
|
|
from unshackle.core.credential import Credential
|
|
from unshackle.core.downloaders import (
|
|
default_max_workers,
|
|
format_speed,
|
|
parse_speed_limit,
|
|
retries_exhausted,
|
|
set_speed_limit,
|
|
)
|
|
from unshackle.core.drm import (
|
|
DRM_T,
|
|
ClearKey,
|
|
ClearKeyCENC,
|
|
MonaLisa,
|
|
PlayReady,
|
|
Widevine,
|
|
cbcs,
|
|
drop_null_keys,
|
|
is_null_key,
|
|
own_kids,
|
|
real_kids,
|
|
verify,
|
|
)
|
|
from unshackle.core.events import events
|
|
from unshackle.core.export_name import ExportNamer, move_export, plural, season_episodes, title_label
|
|
from unshackle.core.manifests.snapshot import capture
|
|
from unshackle.core.providers.anilist import parse_anilist_ref
|
|
from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int
|
|
from unshackle.core.proxies.resolve import (
|
|
REGION,
|
|
describe_explicit_proxy,
|
|
describe_proxy,
|
|
is_loopback,
|
|
load_proxy_providers,
|
|
pick_proxy,
|
|
require_alone_local_proxy,
|
|
resolve_proxy,
|
|
split_proxy_query,
|
|
)
|
|
from unshackle.core.service import Service, grow_session_pool
|
|
from unshackle.core.services import ROUTING_PARAMS, Services
|
|
from unshackle.core.temp import with_task_temp
|
|
from unshackle.core.title_cacher import get_account_hash
|
|
from unshackle.core.titles import Movie, Movies, Series, Song, Title_T
|
|
from unshackle.core.titles.episode import Episode
|
|
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
|
|
from unshackle.core.tracks.attachment import Attachment
|
|
from unshackle.core.tracks.dv_fixup import apply_dv_fixup
|
|
from unshackle.core.tracks.hybrid import Hybrid
|
|
from unshackle.core.tracks.subtitle_convert import PYCAPTION_WRITERS, SUBSTATION, build_ass_style
|
|
from unshackle.core.tracks.track import (
|
|
Track,
|
|
assert_fragments_decrypted,
|
|
fix_data_reference_indexes,
|
|
has_encrypted_sample_entry,
|
|
)
|
|
from unshackle.core.tracks.tracks import at_resolution, select_videos
|
|
from unshackle.core.utilities import (
|
|
apply_original_language,
|
|
as_requested,
|
|
declared_kwargs,
|
|
embedded_audio_langs,
|
|
find_font_with_fallbacks,
|
|
find_missing_langs,
|
|
get_debug_logger,
|
|
get_system_fonts,
|
|
init_debug_logger,
|
|
kind_only_conflict,
|
|
log_event,
|
|
matched_language_tags,
|
|
matching_languages,
|
|
missing_required_langs,
|
|
orig_token,
|
|
partition_exclusions,
|
|
plain_subtitle_tags,
|
|
resolve_sort_langs,
|
|
standard_resolution,
|
|
suggest_font_packages,
|
|
time_elapsed_since,
|
|
)
|
|
from unshackle.core.utils import tags
|
|
from unshackle.core.utils.audio_vivid import decode_audio_vivid
|
|
from unshackle.core.utils.bitrate import apply_real_bitrates
|
|
from unshackle.core.utils.click_types import (
|
|
AUDIO_CODEC_LIST,
|
|
LANGUAGE_RANGE,
|
|
QUALITY_LIST,
|
|
SAMPLE_RANGE,
|
|
SEASON_RANGE,
|
|
SLOW_DELAY_RANGE,
|
|
SUBTITLE_CODEC,
|
|
VIDEO_CODEC_LIST,
|
|
ContextData,
|
|
MultipleChoice,
|
|
)
|
|
from unshackle.core.utils.collections import ci_get, merge_dict
|
|
from unshackle.core.utils.hdr10plus import probed_tracks
|
|
from unshackle.core.utils.post_scripts import NO_POST_SCRIPTS, build_context, dispatch, season_context
|
|
from unshackle.core.utils.redact import mask_proxy, redact_secrets
|
|
from unshackle.core.utils.selector import select_multiple
|
|
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe
|
|
from unshackle.core.vault import Vault
|
|
from unshackle.core.vaults import Vaults
|
|
|
|
if TYPE_CHECKING:
|
|
import pysubs2
|
|
|
|
from unshackle.core.remote_service import ServerKeySource
|
|
|
|
|
|
class SkippedTitle(TypedDict):
|
|
"""A title skipped because a requested track or language is missing. Accumulated as ``dl.skipped_titles``."""
|
|
|
|
title: str
|
|
reason: str
|
|
|
|
|
|
class SkippedSubtitle(TypedDict):
|
|
"""A subtitle skipped under ``--skip-subtitle-errors``. Accumulated as ``dl.skipped_subtitles``,
|
|
one entry per track. ``id`` and ``title`` identify which subtitle of which title was unavailable."""
|
|
|
|
id: str
|
|
language: str
|
|
title: str
|
|
|
|
|
|
# Config keys accepted as natural names for params whose Python name dodges a builtin.
|
|
DL_OPTION_ALIASES = {"range": "range_", "list": "list_"}
|
|
|
|
|
|
def parse_language(tag: Optional[str]) -> Optional[Language]:
|
|
"""A provider's original-language tag as a Language, or None if it is absent or not a real tag."""
|
|
tag = (tag or "").strip()
|
|
if not tag or not tag_is_valid(tag) or tag.startswith("und"):
|
|
return None
|
|
return Language.get(tag)
|
|
|
|
|
|
def normalize_dl_config(dl_config: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map aliased config keys (e.g. ``range``) to their Click parameter names (``range_``)."""
|
|
return {DL_OPTION_ALIASES.get(key, key): value for key, value in dl_config.items()}
|
|
|
|
|
|
ID_PROVIDER_KEYS = {"tmdb": "tmdb_api_key", "tvdb": "tvdb_api_key", "omdb": "omdb_api_key"}
|
|
|
|
|
|
def validate_metadata_ids(
|
|
tmdb_id: Optional[int],
|
|
imdb_id: Optional[str],
|
|
tvdb_id: Optional[int],
|
|
anilist_id: Optional[Union[int, str]] = None,
|
|
) -> None:
|
|
"""Reject ID flags that conflict, or that no configured metadata provider could ever find.
|
|
|
|
--anilist stays outside the mutual exclusion: AniList knows no western IDs, so pairing
|
|
it with one of the other three is how an anime title gets both.
|
|
"""
|
|
if anilist_id and not parse_anilist_ref(anilist_id):
|
|
raise click.UsageError("--anilist takes an AniList ID like 21, or a MyAnimeList ID like mal:21.")
|
|
|
|
supplied = [
|
|
(flag, value) for flag, value in (("--tmdb", tmdb_id), ("--imdb", imdb_id), ("--tvdb", tvdb_id)) if value
|
|
]
|
|
if len(supplied) > 1:
|
|
raise click.UsageError(f"{', '.join(flag for flag, _ in supplied)} cannot be used together; give one ID")
|
|
if not supplied:
|
|
return
|
|
|
|
flag, _ = supplied[0]
|
|
id_kind = flag.lstrip("-")
|
|
ordered = {cls.NAME: cls for kind in ("tv", "movie") for cls in providers.provider_order(kind)}
|
|
consumers = [cls for cls in ordered.values() if cls.ID_KIND == id_kind]
|
|
if any(cls().is_available() for cls in consumers):
|
|
return
|
|
|
|
if consumers:
|
|
keys = sorted({ID_PROVIDER_KEYS[cls.NAME] for cls in consumers if cls.NAME in ID_PROVIDER_KEYS})
|
|
raise click.UsageError(f"{flag} needs the {consumers[0].NAME} provider: set {' or '.join(keys)} in your config")
|
|
names = sorted({cls.NAME for cls in providers.REGISTRY.values() if cls.ID_KIND == id_kind})
|
|
raise click.UsageError(
|
|
f"{flag} needs the {' or '.join(names)} provider, which your metadata_providers config leaves out."
|
|
)
|
|
|
|
|
|
def group_videos_by_variant(videos: list[Video], *, merge: bool) -> list[list[Video]]:
|
|
"""Group video tracks for muxing.
|
|
|
|
When ``merge`` is True, this function groups tracks that share ``(height, range, codec)``
|
|
into one file, so only language varies within a group. Different resolutions, ranges and
|
|
codecs stay in separate groups (separate files). When False, each track is its own group
|
|
(one file per track, the default behaviour). Group order follows first-seen track order.
|
|
"""
|
|
if not merge:
|
|
return [[video] for video in videos]
|
|
groups: dict[tuple[Any, ...], list[Video]] = {}
|
|
for video in videos:
|
|
groups.setdefault((video.height, video.range, video.codec), []).append(video)
|
|
return list(groups.values())
|
|
|
|
|
|
HYBRID_BASE_RANGES = (Video.Range.HDR10P, Video.Range.HDR10)
|
|
|
|
|
|
def convert_scc(subtitle: Subtitle, sub_format: Optional[Union[Subtitle.Codec, str]]) -> Subtitle.Codec:
|
|
"""
|
|
Convert an SCC subtitle to sub_format when pycaption can write it, else to SRT, and return the new codec.
|
|
Only pycaption reads SCC, and nothing later in the pipeline (styling, SDH strip, mux) can.
|
|
"""
|
|
target = Subtitle.Codec.SubRip
|
|
if isinstance(sub_format, Subtitle.Codec) and sub_format in PYCAPTION_WRITERS:
|
|
target = sub_format
|
|
subtitle.convert(target)
|
|
if subtitle.path and not subtitle.path.read_text("utf8").strip():
|
|
logging.getLogger("Subtitle").warning(f"SCC subtitle {subtitle.language} has no CC1 captions")
|
|
return target
|
|
|
|
|
|
def select_hybrid_bases(videos: Sequence[Video]) -> list[Video]:
|
|
"""The HDR10 or HDR10+ track that is the hybrid base for each height, tallest first.
|
|
|
|
Only HEVC tracks qualify, because dovi_tool injects the RPU into an HEVC stream. HDR10+ comes before
|
|
HDR10 at one height.
|
|
"""
|
|
bases: dict[int, Video] = {}
|
|
for video in sorted(
|
|
(v for v in videos if v.range in HYBRID_BASE_RANGES and v.codec in (Video.Codec.HEVC, None)),
|
|
key=lambda v: (v.height or 0, v.range == Video.Range.HDR10P),
|
|
reverse=True,
|
|
):
|
|
bases.setdefault(video.height or 0, video)
|
|
return list(bases.values())
|
|
|
|
|
|
def generated_dv_ranges(
|
|
range_: Sequence[Video.Range], generate_dv: bool
|
|
) -> tuple[list[Video.Range], list[Video.Range]]:
|
|
"""Turn ``-r HDR10 --generate-dv`` (or HDR10P) into a HYBRID run on that base alone.
|
|
|
|
Returns the ranges to run with, and the base ranges the user named. The second list is empty when
|
|
the flag is off, HYBRID or DV is in the ranges, or no base range is. The service still gets the
|
|
ranges the user gave, so it fetches no DV track.
|
|
"""
|
|
bases = [r for r in range_ if r in HYBRID_BASE_RANGES]
|
|
if not generate_dv or not bases or Video.Range.HYBRID in range_ or Video.Range.DV in range_:
|
|
return list(range_), []
|
|
return [Video.Range.HYBRID, *(r for r in range_ if r not in bases)], bases
|
|
|
|
|
|
def keep_generated_dv_bases(videos: Sequence[Video], bases: Sequence[Video.Range]) -> list[Video]:
|
|
"""The videos without the DV tracks and without a hybrid base range the user did not name."""
|
|
return [v for v in videos if v.range != Video.Range.DV and (v.range not in HYBRID_BASE_RANGES or v.range in bases)]
|
|
|
|
|
|
def subtitle_look(
|
|
sub_format: Optional[Union[Subtitle.Codec, str]],
|
|
sub_font: Optional[str],
|
|
style_fields: Any,
|
|
*,
|
|
typed_font: bool = True,
|
|
typed_format: bool = True,
|
|
) -> tuple[Optional[Union[Subtitle.Codec, str]], Optional[pysubs2.SSAStyle]]:
|
|
"""The subtitle format and ASS style for --sub-font and the subtitle.ass_style config.
|
|
|
|
A style makes ASS the format when none is set. Another format leaves the style unused. A typed --sub-font
|
|
overrides a format from config, refuses a typed one, and a font from config is ignored like the config style.
|
|
"""
|
|
if typed_font and sub_font and sub_format is not None and sub_format not in SUBSTATION:
|
|
if typed_format:
|
|
raise ValueError(f"--sub-font needs ASS subtitles, not {getattr(sub_format, 'name', sub_format)}")
|
|
sub_format = None
|
|
if not (style_fields or sub_font) or sub_format not in (None, *SUBSTATION):
|
|
return sub_format, None
|
|
if not isinstance(style_fields, (dict, type(None))):
|
|
raise ValueError("subtitle.ass_style must be a mapping of style fields")
|
|
style = build_ass_style({**(style_fields or {}), **({"fontname": sub_font} if sub_font else {})})
|
|
return sub_format or Subtitle.Codec.SubStationAlphav4, style
|
|
|
|
|
|
def set_subtitle_look(
|
|
subtitles: Iterable[Subtitle], style: Optional[pysubs2.SSAStyle], videos: Iterable[Video]
|
|
) -> None:
|
|
"""Give each subtitle that is not already SSA/ASS the style, and every subtitle the size of the first video."""
|
|
size = next(((v.width, v.height) for v in videos if v.width and v.height), None)
|
|
for subtitle in subtitles:
|
|
subtitle.ass_style = None if subtitle.codec in SUBSTATION else style
|
|
subtitle.ass_video_size = size
|
|
|
|
|
|
def hybrid_gate_failure(videos: Sequence[Video], generate_dv: bool) -> Optional[str]:
|
|
"""The message that stops HYBRID for these selected videos, or None when it can go on."""
|
|
if any(v.native_hybrid for v in videos):
|
|
return None
|
|
hdr_tracks = [v for v in videos if v.range in HYBRID_BASE_RANGES]
|
|
base_tracks = [v for v in hdr_tracks if v.codec in (Video.Codec.HEVC, None)]
|
|
dv_tracks = [v for v in videos if v.range == Video.Range.DV]
|
|
ranges = ", ".join(sorted(set(v.range.name for v in videos))) or "none"
|
|
missing = f"HYBRID needs HDR10/HDR10+ and DV tracks, available ranges: {ranges}"
|
|
|
|
if not hdr_tracks:
|
|
return missing
|
|
if not base_tracks:
|
|
codecs = sorted(set(v.codec.name for v in hdr_tracks if v.codec))
|
|
return f"HYBRID needs an HEVC HDR10/HDR10+ base, available codecs: {', '.join(codecs)}"
|
|
if not dv_tracks and not generate_dv:
|
|
return f"{missing}; add --generate-dv to make DV"
|
|
return None
|
|
|
|
|
|
def hybrid_plan(videos: Sequence[Video], generate_dv: bool) -> tuple[list[Video], Optional[Video]]:
|
|
"""The hybrid bases and the 2D DV track to pair with them.
|
|
|
|
With no DV track, a base qualifies only with ``--generate-dv``. A resolution with a
|
|
finished hybrid (``native_hybrid``) gets no base: that track is its hybrid.
|
|
"""
|
|
dv_tracks = [v for v in videos if v.range == Video.Range.DV and not v.stereo]
|
|
native_heights = {v.height for v in videos if v.native_hybrid}
|
|
base_pool = [v for v in videos if v.height not in native_heights and (dv_tracks or generate_dv)]
|
|
matching_dv = min(dv_tracks, key=lambda v: v.height or 0) if dv_tracks else None
|
|
return select_hybrid_bases(base_pool), matching_dv
|
|
|
|
|
|
def hybrid_source_tracks(base: Video, dv: Optional[Video], base_container: Optional[Path] = None) -> list[Video]:
|
|
"""Copies of the base and DV tracks for the hybrid step, with the base on its container file.
|
|
|
|
The hybrid step reads the duration and seeks, and the raw stream from the DV fixup
|
|
permits neither. With no DV track, the hybrid step makes Dolby Vision for the base.
|
|
"""
|
|
tracks = [deepcopy(track) for track in (base, dv) if track]
|
|
tracks[0].path = base_container or base.path
|
|
return tracks
|
|
|
|
|
|
def hybrid_step_error(tracks: list[Video], source: str, output_name: str) -> Optional[str]:
|
|
"""Run the hybrid step for one resolution; the error that failed it, or None.
|
|
|
|
A failure here is this resolution's alone: the title goes on with its other resolutions.
|
|
"""
|
|
try:
|
|
Hybrid(tracks, source, output_name=output_name)
|
|
except (ValueError, KeyError, OSError, RuntimeError) as e:
|
|
return str(e) or type(e).__name__
|
|
return None
|
|
|
|
|
|
def missing_hybrid_error(bases: Sequence[Video], made: int) -> Optional[str]:
|
|
"""Why the title has no hybrid file, or None when at least one was made."""
|
|
if made:
|
|
return None
|
|
if not bases:
|
|
return "No HEVC HDR10 or HDR10+ track to make a hybrid from"
|
|
return f"Every hybrid failed ({', '.join(f'{v.height or 0}p' for v in bases)}), so the title has no hybrid file"
|
|
|
|
|
|
def title_wanted(candidate: Any, wanted: Collection[str]) -> bool:
|
|
"""Whether ``-w`` selects this title. A title type without selection keys is always kept.
|
|
|
|
The download loop and the post-script counter both read this, so the counter cannot
|
|
fall out of step with the titles the loop skips.
|
|
"""
|
|
if not wanted or not isinstance(candidate, (Episode, Song)):
|
|
return True
|
|
return bool(candidate.matches_wanted(wanted))
|
|
|
|
|
|
def latest_episode_keys(titles: Any, latest_episode: bool, latest_episodes: Optional[int]) -> list[tuple[int, int]]:
|
|
"""The (season, number) of each episode ``--latest-episode(s)`` keeps, oldest first.
|
|
|
|
Counts distinct episodes, not Series entries, so a split episode takes one slot and all its parts download.
|
|
"""
|
|
count = latest_episodes or (1 if latest_episode else 0)
|
|
if not count or not isinstance(titles, Series):
|
|
return []
|
|
# a Series keeps itself sorted, so the newest episodes are last
|
|
return list(dict.fromkeys((ep.season, ep.number) for ep in titles))[-count:]
|
|
|
|
|
|
SAMPLE_DIR = "Sample"
|
|
|
|
|
|
def template_type(title: Any) -> str:
|
|
"""Return the output template key of a title: ``series``, ``songs`` or ``movies``."""
|
|
return "series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
|
|
|
|
|
def sample_name(name: str, title: Any) -> str:
|
|
"""Return ``name`` with a SAMPLE tag, joined with the separator of the output template of the title."""
|
|
return f"{name.rstrip()}{config.get_template_separator(template_type(title))}SAMPLE"
|
|
|
|
|
|
def apply_enrich(title: Title_T, name: Optional[str], year: Optional[int], language: Optional[Language]) -> None:
|
|
"""Set the --enrich name, year and original language on a title. A song takes only the language.
|
|
|
|
A service can set the title again in get_tracks, so dl applies them after that call too.
|
|
"""
|
|
if isinstance(title, Episode):
|
|
title.title = name or title.title
|
|
elif isinstance(title, Movie):
|
|
title.name = name or title.name
|
|
if isinstance(title, (Episode, Movie)):
|
|
title.year = year or title.year
|
|
if language:
|
|
apply_original_language(title, language)
|
|
|
|
|
|
def load_tracks(
|
|
service: Service,
|
|
title: Title_T,
|
|
*,
|
|
name: Optional[str] = None,
|
|
year: Optional[int] = None,
|
|
language: Optional[Language] = None,
|
|
workers: Optional[int] = None,
|
|
on_probe: Optional[Callable[[], None]] = None,
|
|
) -> None:
|
|
"""Load the tracks and chapters of a title, then apply the --enrich name, year and language again."""
|
|
title.tracks.add(probed_tracks(service, title, workers=workers, on_probe=on_probe), warn_only=True)
|
|
title.tracks.chapters = service.get_chapters(title)
|
|
apply_enrich(title, name, year, language)
|
|
|
|
|
|
def server_url(server_name: Optional[str]) -> str:
|
|
"""The configured URL of a remote server, or an empty string when the config cannot be read."""
|
|
from unshackle.core.remote_service import resolve_server
|
|
|
|
try:
|
|
return resolve_server(server_name)[0]
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def post_script_group(candidate: Any) -> Any:
|
|
"""Group a title by the folder its outputs share, for post-scripts in season mode."""
|
|
if isinstance(candidate, Episode):
|
|
return ("episode", candidate.title, candidate.season)
|
|
if isinstance(candidate, Song):
|
|
artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "")
|
|
return ("album", artist, getattr(candidate, "album", ""))
|
|
return ("title", id(candidate))
|
|
|
|
|
|
def remux_music_tracks(title: Any, log: logging.Logger, sample: Optional[tuple[int, int]] = None) -> None:
|
|
"""Remux each audio track of a Song into the container of its codec, cut to ``sample`` when given.
|
|
|
|
A Song never reaches the muxer, so each downloaded audio track is itself a delivered
|
|
file. A failure warns, keeps that downloaded track, and moves on to the next one: the
|
|
user still gets the audio, in a container a player cuts short.
|
|
"""
|
|
if not isinstance(title, Song):
|
|
return
|
|
for track in title.tracks.audio:
|
|
window = (max(sample[0] - track.sample_start, 0), sample[1] - sample[0]) if sample else (0.0, None)
|
|
try:
|
|
track.to_music_container(*window)
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning(f"Could not remux audio track {track.id} for music: {e}")
|
|
|
|
|
|
def mux_skipped(no_mux: bool, title: Any) -> bool:
|
|
"""A Song has no mux step, so --no-mux never applies to it."""
|
|
return no_mux and not isinstance(title, Song)
|
|
|
|
|
|
def take_unmuxable_audio(tracks: Tracks, log: logging.Logger) -> list[Audio]:
|
|
"""Remove and return the Audio Vivid tracks, which no muxer carries, to be saved beside the output.
|
|
|
|
With the user's decoder, each one also goes back into the mux as a WAV copy.
|
|
"""
|
|
kept = [a for a in tracks.audio if a.codec == Audio.Codec.AV3A]
|
|
tracks.audio = [a for a in tracks.audio if a.codec != Audio.Codec.AV3A]
|
|
for track in kept:
|
|
wav = decode_audio_vivid(track.path, log) if track.path else None
|
|
if wav:
|
|
decoded = copy.copy(track)
|
|
decoded.path = wav
|
|
decoded.codec = Audio.Codec.PCM
|
|
tracks.audio.append(decoded)
|
|
return kept
|
|
|
|
|
|
def track_file_suffix(track: Any) -> str:
|
|
"""The name part a track file saved on its own adds after the title: language and codec, or subtitle flags."""
|
|
if isinstance(track, Video):
|
|
return f".{track.codec.name if track.codec is not None else 'video'}"
|
|
lang_suffix = f".{track.language}" if track.language else ""
|
|
if isinstance(track, Audio):
|
|
return f"{lang_suffix}.{track.codec.name if track.codec is not None else 'audio'}"
|
|
if isinstance(track, Subtitle):
|
|
return f"{lang_suffix}{'.forced' if track.forced else ''}{'.sdh' if track.sdh else ''}"
|
|
return ""
|
|
|
|
|
|
def no_mux_extension(track: Any, path: Path, media_info: MediaInfo) -> str:
|
|
"""The extension of a --no-mux track file: an audio track in an MPEG-4 file is saved as .m4a."""
|
|
container = media_info.general_tracks[0].format if media_info.general_tracks else None
|
|
return ".m4a" if isinstance(track, Audio) and path.suffix == ".mp4" and container == "MPEG-4" else path.suffix
|
|
|
|
|
|
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
|
|
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
|
|
The config replaces defaults and global ``dl:`` default_map values.
|
|
``remote`` and ``server`` are skipped: they pick the code that runs, so ``Services.resolve_command`` sets
|
|
them before the service loads, and an API job takes them from its request."""
|
|
params_by_name = {param.name: param for param in ctx.command.params if param.name}
|
|
for name, value in normalize_dl_config(service_dl_config).items():
|
|
if name in ROUTING_PARAMS:
|
|
continue
|
|
param = params_by_name.get(name)
|
|
if param is None:
|
|
log.warning(f"Ignoring unknown dl option '{name}' in service config")
|
|
continue
|
|
if name not in ctx.params:
|
|
log.debug(f"Skipping service dl override '{name}': not present in this context")
|
|
continue
|
|
if value is None:
|
|
continue
|
|
source = ctx.get_parameter_source(name)
|
|
if source not in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP):
|
|
continue
|
|
try:
|
|
ctx.params[name] = param.type_cast_value(ctx, value)
|
|
log.debug(f"Applied service dl override '{name}': {ctx.params[name]}")
|
|
except Exception as e:
|
|
log.warning(f"Failed to apply service dl override '{name}'={value!r}: {e}")
|
|
|
|
|
|
def download_tracks_in_passes(
|
|
tracks: Iterable[AnyTrack],
|
|
max_concurrent: int,
|
|
run_one: Callable[[AnyTrack, int], None],
|
|
*,
|
|
skip_subtitle_errors: bool,
|
|
on_subtitle_skipped: Callable[[Subtitle], None],
|
|
) -> None:
|
|
"""Download a title's tracks so a skippable subtitle failure cannot corrupt the rest.
|
|
|
|
A failed track sets the process-global ``DOWNLOAD_CANCELLED`` event, making other in-flight
|
|
tracks early-return without raising. With ``skip_subtitle_errors`` set, video/audio download
|
|
concurrently first. The subtitles then download one at a time (a concurrent pass would let one
|
|
failure's cancel silently drop the others, unrecorded), with the event cleared before each.
|
|
Video/audio failures stay fatal. This function clears the event on entry (stale cancel from a
|
|
prior title) and in ``finally`` (never leave it set for later code).
|
|
|
|
``run_one(track, index)`` downloads a single track. ``on_subtitle_skipped(track)`` records a
|
|
subtitle whose download raised (handled here).
|
|
"""
|
|
DOWNLOAD_CANCELLED.clear()
|
|
indexed = list(enumerate(tracks))
|
|
if skip_subtitle_errors:
|
|
primary: list[tuple[int, AnyTrack]] = [(i, t) for i, t in indexed if not isinstance(t, Subtitle)]
|
|
skippable = [(i, t) for i, t in indexed if isinstance(t, Subtitle)]
|
|
else:
|
|
primary, skippable = indexed, []
|
|
|
|
try:
|
|
with ThreadPoolExecutor(max_concurrent) as pool:
|
|
future_to_track = {pool.submit(run_one, track, i): track for i, track in primary}
|
|
for download in futures.as_completed(future_to_track):
|
|
download.result() # a video/audio failure is fatal
|
|
|
|
for i, track in skippable:
|
|
DOWNLOAD_CANCELLED.clear()
|
|
try:
|
|
run_one(track, i)
|
|
except Exception:
|
|
on_subtitle_skipped(track)
|
|
finally:
|
|
DOWNLOAD_CANCELLED.clear() # never leave a failed track's cancel set for later code
|
|
|
|
|
|
class dl:
|
|
@staticmethod
|
|
def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str:
|
|
"""Truncate PSSH string for display when not in debug mode."""
|
|
if logging.root.level == logging.DEBUG or not pssh_string:
|
|
return pssh_string
|
|
|
|
max_width = console.width - len(drm_type) - 12
|
|
if len(pssh_string) <= max_width:
|
|
return pssh_string
|
|
|
|
return pssh_string[: max_width - 3] + "..."
|
|
|
|
def find_custom_font(self, font_name: str) -> Optional[Path]:
|
|
"""
|
|
Find font in custom fonts directory.
|
|
|
|
Args:
|
|
font_name: Font family name to find
|
|
|
|
Returns:
|
|
Path to font file, or None if not found. A name that is a path, not one folder name, finds nothing.
|
|
"""
|
|
if Path(font_name).name != font_name or font_name in ("", ".."):
|
|
return None
|
|
family_dir = Path(config.directories.fonts, font_name)
|
|
if family_dir.exists():
|
|
fonts = list(family_dir.glob("*.*tf"))
|
|
return fonts[0] if fonts else None
|
|
return None
|
|
|
|
def prepare_temp_font(self, font_name: str, matched_font: Path, system_fonts: dict[str, Path]) -> Path:
|
|
"""
|
|
Copy system font to temp and log if using fallback.
|
|
|
|
Args:
|
|
font_name: Requested font name
|
|
matched_font: Path to matched system font
|
|
system_fonts: Dictionary of available system fonts
|
|
|
|
Returns:
|
|
Path to temp font file
|
|
"""
|
|
matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None)
|
|
|
|
if matched_name and matched_name.lower() != font_name.lower():
|
|
self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'")
|
|
|
|
safe_name = font_name.replace(" ", "_").replace("/", "_")
|
|
temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}"
|
|
|
|
if not temp_path.exists():
|
|
shutil.copy2(matched_font, temp_path)
|
|
|
|
return temp_path
|
|
|
|
def attach_subtitle_fonts(self, font_names: list[str], title: Title_T) -> tuple[int, list[str]]:
|
|
"""
|
|
Attach fonts for subtitle rendering.
|
|
|
|
Args:
|
|
font_names: List of font names requested by subtitles
|
|
title: Title object to attach fonts to
|
|
|
|
Returns:
|
|
Tuple of (fonts_attached_count, missing_fonts_list)
|
|
"""
|
|
if not font_names:
|
|
return 0, []
|
|
system_fonts = get_system_fonts()
|
|
|
|
font_count = 0
|
|
missing_fonts = []
|
|
|
|
for font_name in set(font_names):
|
|
if custom_font := self.find_custom_font(font_name):
|
|
title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})"))
|
|
font_count += 1
|
|
continue
|
|
|
|
if system_font := find_font_with_fallbacks(font_name, system_fonts):
|
|
temp_path = self.prepare_temp_font(font_name, system_font, system_fonts)
|
|
title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})"))
|
|
font_count += 1
|
|
else:
|
|
self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found")
|
|
missing_fonts.append(font_name)
|
|
|
|
return font_count, missing_fonts
|
|
|
|
def suggest_missing_fonts(self, missing_fonts: list[str]) -> None:
|
|
"""
|
|
Show package installation suggestions for missing fonts.
|
|
|
|
Args:
|
|
missing_fonts: List of font names that could not be found
|
|
"""
|
|
if suggestions := suggest_font_packages(missing_fonts):
|
|
self.log.info("Install font packages to improve subtitle rendering:")
|
|
for package_cmd, fonts in suggestions.items():
|
|
self.log.info(f" $ sudo apt install {package_cmd}")
|
|
self.log.info(f" → Provides: {', '.join(fonts)}")
|
|
|
|
def generate_sidecar_subtitle_path(
|
|
self,
|
|
subtitle: Subtitle,
|
|
base_filename: str,
|
|
output_dir: Path,
|
|
target_codec: Optional[Subtitle.Codec] = None,
|
|
source_path: Optional[Path] = None,
|
|
) -> Path:
|
|
"""Make the sidecar path: {base}.{lang}[.forced][.sdh].{ext}"""
|
|
lang_suffix = str(subtitle.language) if subtitle.language else "und"
|
|
forced_suffix = ".forced" if subtitle.forced else ""
|
|
sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else ""
|
|
|
|
extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension
|
|
if not target_codec and not subtitle.codec and source_path and source_path.suffix:
|
|
extension = source_path.suffix.lstrip(".")
|
|
|
|
filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}"
|
|
return output_dir / filename
|
|
|
|
def output_subtitle_sidecars(
|
|
self,
|
|
subtitles: list[Subtitle],
|
|
base_filename: str,
|
|
output_dir: Path,
|
|
sidecar_format: str,
|
|
original_paths: Optional[dict[str, Path]] = None,
|
|
) -> list[Path]:
|
|
"""
|
|
Output subtitles as sidecar files, converting if needed.
|
|
|
|
Conversion runs on a temp copy, so each subtitle's own file stays untouched and mkvmerge can still mux it.
|
|
"""
|
|
created_paths: list[Path] = []
|
|
config.directories.temp.mkdir(parents=True, exist_ok=True)
|
|
|
|
for subtitle in subtitles:
|
|
source_path = subtitle.path
|
|
if sidecar_format == "original" and original_paths and subtitle.id in original_paths:
|
|
source_path = original_paths[subtitle.id]
|
|
|
|
if not source_path or not source_path.exists():
|
|
continue
|
|
|
|
if sidecar_format == "original":
|
|
target_codec = None
|
|
if source_path.suffix:
|
|
try:
|
|
target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip("."))
|
|
except ValueError:
|
|
target_codec = None
|
|
else:
|
|
target_codec = Subtitle.Codec.from_mime(sidecar_format)
|
|
|
|
sidecar_path = self.generate_sidecar_subtitle_path(
|
|
subtitle, base_filename, output_dir, target_codec, source_path=source_path
|
|
)
|
|
|
|
if not target_codec or subtitle.codec == target_codec:
|
|
shutil.copy2(source_path, sidecar_path)
|
|
else:
|
|
temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}"
|
|
shutil.copy2(source_path, temp_path)
|
|
|
|
temp_sub = Subtitle(
|
|
subtitle.url,
|
|
subtitle.language,
|
|
is_original_lang=subtitle.is_original_lang,
|
|
descriptor=subtitle.descriptor,
|
|
codec=subtitle.codec,
|
|
forced=subtitle.forced,
|
|
sdh=subtitle.sdh,
|
|
cc=subtitle.cc,
|
|
id_=f"{subtitle.id}_sc",
|
|
)
|
|
temp_sub.path = temp_path
|
|
try:
|
|
temp_sub.convert(target_codec, forced=True)
|
|
if temp_sub.path and temp_sub.path.exists():
|
|
shutil.copy2(temp_sub.path, sidecar_path)
|
|
finally:
|
|
if temp_sub.path and temp_sub.path.exists():
|
|
temp_sub.path.unlink(missing_ok=True)
|
|
temp_path.unlink(missing_ok=True)
|
|
|
|
created_paths.append(sidecar_path)
|
|
|
|
return created_paths
|
|
|
|
def post_script_ids(self) -> dict[str, Any]:
|
|
"""Tagging IDs as they stand right now.
|
|
|
|
Read at each post-script call, never snapshotted: the search inside the title loop
|
|
finds an Episode's TMDB ID, so a snapshot taken before it is always empty.
|
|
"""
|
|
return {"tmdb": self.tmdb_id, "imdb": self.imdb_id, "tvdb": self.tvdb_id}
|
|
|
|
@click.command(
|
|
short_help="Download, Decrypt, and Mux tracks for titles from a Service.",
|
|
cls=Services,
|
|
context_settings=dict(**context_settings, default_map=normalize_dl_config(config.dl)),
|
|
)
|
|
@click.option(
|
|
"-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)."
|
|
)
|
|
@click.option(
|
|
"-q",
|
|
"--quality",
|
|
type=QUALITY_LIST,
|
|
default=[],
|
|
help="Download Resolution(s), defaults to the best available resolution.",
|
|
)
|
|
@click.option(
|
|
"-v",
|
|
"--vcodec",
|
|
type=VIDEO_CODEC_LIST,
|
|
default=[],
|
|
help="Video Codec(s) to download, defaults to any codec.",
|
|
)
|
|
@click.option(
|
|
"-a",
|
|
"--acodec",
|
|
type=AUDIO_CODEC_LIST,
|
|
default=[],
|
|
help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.",
|
|
)
|
|
@click.option(
|
|
"-vb",
|
|
"--vbitrate",
|
|
type=int,
|
|
default=None,
|
|
help="Video Bitrate to download (in kbps), defaults to highest available.",
|
|
)
|
|
@click.option(
|
|
"-ab",
|
|
"--abitrate",
|
|
type=int,
|
|
default=None,
|
|
help="Audio Bitrate to download (in kbps), defaults to highest available.",
|
|
)
|
|
@click.option(
|
|
"-vb-range",
|
|
"--vbitrate-range",
|
|
type=str,
|
|
default=None,
|
|
help="Video Bitrate range in kbps (e.g., '6000-7000'). Selects the highest bitrate within the range.",
|
|
)
|
|
@click.option(
|
|
"-ab-range",
|
|
"--abitrate-range",
|
|
type=str,
|
|
default=None,
|
|
help="Audio Bitrate range in kbps (e.g., '128-256'). Selects the highest bitrate within the range.",
|
|
)
|
|
@click.option(
|
|
"-r",
|
|
"--range",
|
|
"range_",
|
|
type=MultipleChoice(Video.Range, case_sensitive=False),
|
|
default=[Video.Range.SDR],
|
|
help="Video Color Range(s) to download, defaults to SDR.",
|
|
)
|
|
@click.option(
|
|
"-c",
|
|
"--channels",
|
|
type=float,
|
|
default=None,
|
|
help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.",
|
|
)
|
|
@click.option(
|
|
"-naa",
|
|
"--noatmos",
|
|
"no_atmos",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Exclude Dolby Atmos audio tracks when selecting audio.",
|
|
)
|
|
@click.option(
|
|
"--split-audio",
|
|
"split_audio",
|
|
is_flag=True,
|
|
default=None,
|
|
help="Create separate output files per audio codec instead of merging all audio.",
|
|
)
|
|
@click.option(
|
|
"--merge-video",
|
|
"merge_video",
|
|
is_flag=True,
|
|
default=None,
|
|
help="Mux all selected video tracks into a single file instead of one file per track.",
|
|
)
|
|
@click.option(
|
|
"--select-titles",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Interactively select downloads from a list: episodes of a series, or films when a title has more than one.",
|
|
)
|
|
@click.option(
|
|
"-w",
|
|
"--wanted",
|
|
type=SEASON_RANGE,
|
|
default=None,
|
|
help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, etc. Music uses track numbers, e.g. `1-5`, `1,3,7`, or `2x3` for disc 2 track 3. Defaults to all.",
|
|
)
|
|
@click.option(
|
|
"-l",
|
|
"--lang",
|
|
type=LANGUAGE_RANGE,
|
|
default="orig",
|
|
help="Language(s) wanted for Video and Audio (comma-separated). Use 'orig' to select the original language, e.g. 'orig,en' for both original and English. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--latest-episode",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Download only the single most recent episode available.",
|
|
)
|
|
@click.option(
|
|
"--latest-episodes",
|
|
type=click.IntRange(min=1),
|
|
default=None,
|
|
help="Download only the N most recent episodes available.",
|
|
)
|
|
@click.option(
|
|
"-vl",
|
|
"--v-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Language wanted for Video. You would use this if the video language doesn't match the audio. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"-al",
|
|
"--a-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Language wanted for Audio, overrides -l/--lang for audio tracks. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--require-audio",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Audio languages that must exist. The title fails even with --best-available.",
|
|
)
|
|
@click.option(
|
|
"--require-video",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Video languages that must exist. The title fails even with --best-available.",
|
|
)
|
|
@click.option(
|
|
"-sl",
|
|
"--s-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=["all"],
|
|
help="Language wanted for Subtitles. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--require-subs",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Subtitle languages that must exist. The title fails if one is missing. -sl still decides what to keep.",
|
|
)
|
|
@click.option(
|
|
"--skip-missing",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip a title that misses a required language or track and continue with the next title.",
|
|
)
|
|
@click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.")
|
|
@click.option(
|
|
"-fso",
|
|
"--forced-subs-only",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Download forced subtitle tracks and no other subtitle tracks.",
|
|
)
|
|
@click.option(
|
|
"-fsl",
|
|
"--forced-s-lang",
|
|
type=LANGUAGE_RANGE,
|
|
default=[],
|
|
help="Languages wanted for forced subtitles, implies -fs. Keeps forced subs only in these languages. Prefix a value with '-' to exclude it, e.g. 'all,-es'.",
|
|
)
|
|
@click.option(
|
|
"--exact-lang",
|
|
is_flag=True,
|
|
default=False,
|
|
help="A language tag without a region selects only the variant it means and never falls back to another: -l es does not select es-419. A tag with a region, such as es-419, is always exact.",
|
|
)
|
|
@click.option(
|
|
"--proxy",
|
|
type=str,
|
|
default=None,
|
|
help="Proxy URI to use. If a 2-letter country is provided, it will try to get a proxy from the config.",
|
|
)
|
|
@click.option(
|
|
"--local-proxy",
|
|
is_flag=True,
|
|
default=False,
|
|
help="With --remote, the server sends its service requests out through this machine's own connection (and any VPN on it), or through a --proxy dialled from this machine.",
|
|
)
|
|
@click.option(
|
|
"--tag",
|
|
type=str,
|
|
default=None,
|
|
help="Set the Group Tag to be used, overriding the config tag and any tag rules.",
|
|
)
|
|
@click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.")
|
|
@click.option(
|
|
"-rvb",
|
|
"--real-video-bitrate",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Probe actual media size to compute true video bitrates (top renditions per codec/range), "
|
|
"overriding the manifest's declared bitrate.",
|
|
)
|
|
@click.option(
|
|
"-rab",
|
|
"--real-audio-bitrate",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Probe actual media size to compute true audio bitrates (top renditions per codec/channels/language), "
|
|
"overriding the manifest's declared bitrate. Slower than --real-video-bitrate (more renditions).",
|
|
)
|
|
@click.option(
|
|
"--tmdb",
|
|
"tmdb_id",
|
|
type=int,
|
|
default=None,
|
|
help="Use this TMDB ID for tagging instead of an automatic search. Add --enrich to also take "
|
|
"its title, year and original language.",
|
|
)
|
|
@click.option(
|
|
"--tvdb",
|
|
"tvdb_id",
|
|
type=int,
|
|
default=None,
|
|
help="Use this TVDB ID for --tvdb-order and tagging instead of looking the series up "
|
|
"automatically. Add --enrich to also take its title, year and original language.",
|
|
)
|
|
@click.option(
|
|
"--tvdb-order",
|
|
type=click.Choice(SEASON_TYPES, case_sensitive=False),
|
|
default=None,
|
|
help="Renumber episodes to a TVDB season order. Services normally use 'official' (aired) order; "
|
|
"a series' Streaming Order is 'alternate'. Defaults to the tvdb_order config option.",
|
|
)
|
|
@click.option(
|
|
"-ce",
|
|
"--continuous-episodes",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Number each season on from the season before it, e.g. S02E05 after a four-episode season one.",
|
|
)
|
|
@click.option(
|
|
"--anilist",
|
|
"anilist_id",
|
|
type=str,
|
|
default=None,
|
|
help="Use this AniList ID for tagging instead of an automatic search, or a MyAnimeList ID as mal:12345. "
|
|
"Add --enrich to also take its title, year and original language. Combines with one of "
|
|
"--tmdb, --imdb or --tvdb, which AniList does not know.",
|
|
)
|
|
@click.option(
|
|
"--enrich",
|
|
is_flag=True,
|
|
default=False,
|
|
help=(
|
|
"Overwrite the show title, year and original language with the external source's. "
|
|
"Requires --tmdb, --imdb, --tvdb, or --anilist."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--daily",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Treat the title as daily/date-based content and fill missing air dates from TVDB during --enrich.",
|
|
)
|
|
@click.option(
|
|
"--imdb",
|
|
"imdb_id",
|
|
type=str,
|
|
default=None,
|
|
help="Use this IMDB ID (e.g. tt1375666) for tagging instead of an automatic search. Add --enrich "
|
|
"to also take its title, year and original language.",
|
|
)
|
|
@click.option(
|
|
"--sub-format",
|
|
type=SUBTITLE_CODEC,
|
|
default=None,
|
|
help="Set Output Subtitle Format, only converting if necessary. Use 'original' to keep source format.",
|
|
)
|
|
@click.option(
|
|
"--sub-font",
|
|
type=str,
|
|
default=None,
|
|
help="Font for subtitles converted to ASS, with the subtitle.ass_style look. Implies --sub-format ASS.",
|
|
)
|
|
@click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.")
|
|
@click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.")
|
|
@click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.")
|
|
@click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapter markers.")
|
|
@click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.")
|
|
@click.option(
|
|
"--skip-subtitle-errors",
|
|
is_flag=True,
|
|
default=False,
|
|
help="If a subtitle track fails to download, skip it and continue instead of "
|
|
"aborting the whole title (video/audio failures stay fatal).",
|
|
)
|
|
@click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.")
|
|
@click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapter markers.")
|
|
@click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.")
|
|
@click.option(
|
|
"-nt",
|
|
"--no-attachments",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Do not download or mux attachments (cover art, subtitle fonts, and files the service attaches).",
|
|
)
|
|
@click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.")
|
|
@click.option(
|
|
"-ado",
|
|
"--audio-description-only",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Download audio description tracks and no standard audio tracks.",
|
|
)
|
|
@click.option(
|
|
"--slow",
|
|
type=SLOW_DELAY_RANGE,
|
|
default=None,
|
|
help="Add a delay between each Title download to act more like a real device. "
|
|
"Use --slow for a 60-120s delay, or --slow MIN-MAX (e.g., --slow 20-40) for a custom range. "
|
|
"Minimum delay is 20 seconds.",
|
|
)
|
|
@click.option(
|
|
"--list",
|
|
"list_",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip downloading and list available tracks and what tracks would have been downloaded.",
|
|
)
|
|
@click.option(
|
|
"--list-titles",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip downloading, only list available titles that would have been downloaded.",
|
|
)
|
|
@click.option(
|
|
"--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys."
|
|
)
|
|
@click.option(
|
|
"--all-drm",
|
|
is_flag=True,
|
|
default=False,
|
|
help="License each track with both Widevine and PlayReady. Needs a CDM for each DRM system.",
|
|
)
|
|
@click.option(
|
|
"--export",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Export track info and decryption keys to a JSON file in the exports directory.",
|
|
)
|
|
@click.option(
|
|
"--import",
|
|
"import_file",
|
|
type=str,
|
|
default=None,
|
|
hidden=True,
|
|
help="Internal: path to an export JSON to reconstruct a download from (used by 'unshackle import').",
|
|
)
|
|
@click.option(
|
|
"--cdm-only/--vaults-only",
|
|
is_flag=True,
|
|
default=None,
|
|
help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.",
|
|
)
|
|
@click.option(
|
|
"--cdm",
|
|
"cdm_name",
|
|
type=str,
|
|
default=None,
|
|
help="Use this CDM device for the run, overriding the cdm config mapping and, over --remote, the server CDM.",
|
|
)
|
|
@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.")
|
|
@click.option(
|
|
"--no-proxy-download",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Bypass proxy for all downloads. Manifest, license, and auth still use proxy.",
|
|
)
|
|
@click.option(
|
|
"--proxy-download",
|
|
type=str,
|
|
default=None,
|
|
help="Proxy for the downloads only, in the same form as --proxy. Manifest, license, and auth use --proxy.",
|
|
)
|
|
@click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.")
|
|
@click.option(
|
|
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
|
|
)
|
|
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
|
|
@click.option(
|
|
"--sample",
|
|
type=SAMPLE_RANGE,
|
|
default=None,
|
|
help="Download a short sample and write it to a Sample folder. "
|
|
"Use --sample for 60 seconds from 2 minutes in, or --sample START-END (e.g., --sample 0-60) for a custom window.",
|
|
)
|
|
@click.option(
|
|
"--include-sample",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Also cut a 60 second sample from each muxed file into a Sample folder next to it.",
|
|
)
|
|
@click.option(
|
|
"--postscript",
|
|
"postscript",
|
|
type=str,
|
|
multiple=True,
|
|
metavar="COMMAND",
|
|
help=(
|
|
"Run COMMAND after each downloaded file, with {filepath}, {title}, {season} and other "
|
|
"variables substituted. Repeatable. Replaces the post_scripts config for this run."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--no-postscript",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Do not run any post-script for this run, configured or --postscript.",
|
|
)
|
|
@click.option(
|
|
"--workers",
|
|
type=int,
|
|
default=None,
|
|
help="Max workers/threads to download with per-track. Default depends on the downloader.",
|
|
)
|
|
@click.option(
|
|
"--adaptive-workers",
|
|
is_flag=True,
|
|
default=False,
|
|
help=(
|
|
"Dynamically scale per-track segment workers up to the --workers cap (or the built-in cap "
|
|
"when --workers is unset) based on measured CDN throughput and errors. Off by default."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--download-processes",
|
|
type=int,
|
|
default=1,
|
|
help=(
|
|
"Split a track's segment downloads across this many processes to beat the single-interpreter "
|
|
"throughput cap. Only engages for large segment batches. Default 1 (single process)."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--continue-downloads",
|
|
is_flag=True,
|
|
default=False,
|
|
help=(
|
|
"Keep completed segment files across runs and resume a previously failed download. "
|
|
"One-off enable of the continue_downloads config option."
|
|
),
|
|
)
|
|
@click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.")
|
|
@click.option(
|
|
"--speed-limit",
|
|
"speed_limit",
|
|
type=str,
|
|
default=None,
|
|
metavar="SPEED",
|
|
help="Cap total download speed across all downloads combined, e.g. 500k, 5M, 1.5G or plain "
|
|
"bytes/sec (5M = 5.0 MB/s; bytes, not bits). "
|
|
"'off' disables a limit configured in the dl: config block.",
|
|
)
|
|
@click.option(
|
|
"-o",
|
|
"--output",
|
|
"output_dir",
|
|
type=Path,
|
|
default=None,
|
|
help="Override the output directory for this download, instead of the one in config.",
|
|
)
|
|
@click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.")
|
|
@click.option(
|
|
"--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching."
|
|
)
|
|
@click.option(
|
|
"--worst",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Select the lowest bitrate track within the specified quality, or of the lowest resolution if no "
|
|
"quality is set.",
|
|
)
|
|
@click.option(
|
|
"--best-available",
|
|
"--warn-only",
|
|
"best_available",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Warn instead of failing when a requested resolution, range, video language, or audio language is missing.",
|
|
)
|
|
@click.option(
|
|
"--generate-dv",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Make Dolby Vision from HDR10+ or the decoded video: with -r HYBRID when a title has no DV track, "
|
|
"with -r HDR10 or HDR10P for that base alone.",
|
|
)
|
|
@click.option(
|
|
"--remote/--local",
|
|
default=False,
|
|
is_eager=True,
|
|
help="Use a remote unshackle server instead of local service code. --local overrides a remote: true config.",
|
|
)
|
|
@click.option(
|
|
"--server",
|
|
type=str,
|
|
default=None,
|
|
is_eager=True,
|
|
help="Name of the remote server from remote_services config (if multiple configured).",
|
|
)
|
|
@click.pass_context
|
|
def cli(ctx: click.Context, **kwargs: Any) -> dl:
|
|
return dl(ctx, **kwargs)
|
|
|
|
DRM_LOCKS: dict[str, Lock] = {}
|
|
DRM_LOCKS_GUARD = Lock()
|
|
LICENCE_LOCK = Lock()
|
|
VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer")
|
|
EXPORT_LOCK = Lock()
|
|
export_snapshots: dict[str, Optional[dict[str, Any]]]
|
|
LICENSE_KEY_CACHE: dict[UUID, str] = {}
|
|
ALL_DRM_LOCK = Lock()
|
|
_remote_service: Optional["ServerKeySource"] = None
|
|
typed_params: frozenset[str] = frozenset()
|
|
|
|
def __init__(
|
|
self,
|
|
ctx: click.Context,
|
|
no_proxy: bool,
|
|
profile: Optional[str] = None,
|
|
proxy: Optional[str] = None,
|
|
repack: bool = False,
|
|
tag: Optional[str] = None,
|
|
tmdb_id: Optional[int] = None,
|
|
imdb_id: Optional[str] = None,
|
|
tvdb_id: Optional[int] = None,
|
|
tvdb_order: Optional[str] = None,
|
|
continuous_episodes: bool = False,
|
|
anilist_id: Optional[Union[int, str]] = None,
|
|
enrich: bool = False,
|
|
daily: bool = False,
|
|
output_dir: Optional[Path] = None,
|
|
proxy_providers: Optional[list[Any]] = None,
|
|
*_: Any,
|
|
**__: Any,
|
|
) -> None:
|
|
if not ctx.invoked_subcommand:
|
|
raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.")
|
|
|
|
self.log = logging.getLogger("download")
|
|
self.typed_params = frozenset(
|
|
name
|
|
for name in ("sub_font", "sub_format")
|
|
if ctx.get_parameter_source(name) not in (None, ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP)
|
|
)
|
|
self.completed_files: list[Path] = []
|
|
# Subtitles skipped under --skip-subtitle-errors, recorded so an embedding caller can
|
|
# report which weren't available without parsing the console output. See SkippedSubtitle.
|
|
self.skipped_subtitles: list[SkippedSubtitle] = []
|
|
self.skipped_titles: list[SkippedTitle] = []
|
|
# Why a missing track or language stopped the run, for the API job error.
|
|
self.stop_reason: Optional[str] = None
|
|
# result() catches a download-worker failure, reports it, and returns rather than
|
|
# re-raising (so the CLI still exits cleanly). Keep it so an embedding caller (the API
|
|
# worker) can tell the title did not complete instead of reading it as a success.
|
|
self.download_error: Optional[BaseException] = None
|
|
|
|
if not config.output_template:
|
|
raise click.ClickException("No 'output_template' in your unshackle.yaml")
|
|
|
|
self.service = Services.get_tag(ctx.invoked_subcommand)
|
|
self.vault_service = Services.get_vault_tag(self.service)
|
|
apply_service_dl_overrides(ctx, (ci_get(config.services, self.service) or {}).get("dl") or {}, self.log)
|
|
|
|
# Refresh locals Click bound before the overrides ran.
|
|
no_proxy = ctx.params.get("no_proxy", no_proxy)
|
|
profile = ctx.params.get("profile", profile)
|
|
proxy = ctx.params.get("proxy", proxy)
|
|
repack = ctx.params.get("repack", repack)
|
|
tag = ctx.params.get("tag", tag)
|
|
tmdb_id = ctx.params.get("tmdb_id", tmdb_id)
|
|
imdb_id = ctx.params.get("imdb_id", imdb_id)
|
|
tvdb_id = ctx.params.get("tvdb_id", tvdb_id)
|
|
tvdb_order = ctx.params.get("tvdb_order", tvdb_order)
|
|
continuous_episodes = ctx.params.get("continuous_episodes", continuous_episodes)
|
|
anilist_id = ctx.params.get("anilist_id", anilist_id)
|
|
enrich = ctx.params.get("enrich", enrich)
|
|
daily = ctx.params.get("daily", daily)
|
|
output_dir = ctx.params.get("output_dir", output_dir)
|
|
|
|
self.profile = profile
|
|
self.proxy_requested = bool(proxy)
|
|
validate_metadata_ids(tmdb_id, imdb_id, tvdb_id, anilist_id)
|
|
self.tmdb_id = tmdb_id
|
|
self.imdb_id = imdb_id
|
|
self.tvdb_id = tvdb_id
|
|
self.anilist_id = anilist_id
|
|
self.tvdb_order = (tvdb_order or config.tvdb_order or "").lower()
|
|
if self.tvdb_order and self.tvdb_order not in SEASON_TYPES:
|
|
raise click.UsageError(f"tvdb_order must be one of {', '.join(SEASON_TYPES)}, not {self.tvdb_order!r}")
|
|
self.continuous_episodes = continuous_episodes
|
|
self.enrich = enrich
|
|
self.daily = daily
|
|
self.output_dir = output_dir
|
|
self.service_anime = False
|
|
self.service_daily = False
|
|
|
|
if self.enrich and not (self.tmdb_id or self.imdb_id or self.tvdb_id or self.anilist_id):
|
|
raise click.UsageError(
|
|
"--enrich requires --tmdb, --imdb, --tvdb, or --anilist to provide a metadata source."
|
|
)
|
|
|
|
if self.daily and not self.enrich:
|
|
self.log.warning("--daily takes TVDB air dates only with --enrich")
|
|
|
|
# Initialize debug logger with service name if debug logging is enabled
|
|
if config.debug or logging.root.level == logging.DEBUG:
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
|
|
debug_log_path = config.directories.logs / config.filenames.debug_log.format_map(
|
|
defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S"))
|
|
)
|
|
init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys)
|
|
self.debug_logger = get_debug_logger()
|
|
|
|
if self.debug_logger:
|
|
self.debug_logger.log(
|
|
level="INFO",
|
|
operation="download_init",
|
|
message=f"Download command initialized for service {self.service}",
|
|
service=self.service,
|
|
context={
|
|
"profile": profile,
|
|
"proxy": proxy,
|
|
"tag": tag,
|
|
"tmdb_id": self.tmdb_id,
|
|
"imdb_id": self.imdb_id,
|
|
"anilist_id": anilist_id,
|
|
"enrich": enrich,
|
|
"cli_params": {
|
|
k: v
|
|
for k, v in ctx.params.items()
|
|
if k
|
|
not in [
|
|
"profile",
|
|
"proxy",
|
|
"tag",
|
|
"tmdb_id",
|
|
"imdb_id",
|
|
"anilist_id",
|
|
"enrich",
|
|
]
|
|
},
|
|
},
|
|
)
|
|
|
|
binary_versions: dict[str, Optional[dict[str, Optional[str]]]] = {}
|
|
for name, binary in [
|
|
("shaka_packager", binaries.ShakaPackager),
|
|
("mp4decrypt", binaries.Mp4decrypt),
|
|
("mkvmerge", binaries.MKVToolNix),
|
|
("ffmpeg", binaries.FFMPEG),
|
|
("ffprobe", binaries.FFProbe),
|
|
]:
|
|
binary_versions[name] = (
|
|
{"path": str(binary), "version": binaries.binary_version(binary)} if binary else None
|
|
)
|
|
|
|
self.debug_logger.log(
|
|
level="DEBUG",
|
|
operation="binary_versions",
|
|
message="Binary tool versions",
|
|
context=binary_versions,
|
|
)
|
|
else:
|
|
self.debug_logger = None
|
|
|
|
if self.profile:
|
|
self.log.info(f"Using profile: '{self.profile}'")
|
|
|
|
self.is_remote = bool(ctx.params.get("remote"))
|
|
|
|
with console.status("Loading Service Config...", spinner="dots"):
|
|
self.service_config = {}
|
|
if not self.is_remote:
|
|
try:
|
|
service_config_path = Services.get_path(self.service) / config.filenames.config
|
|
if service_config_path.exists():
|
|
self.service_config = load_yaml(service_config_path)
|
|
self.log.info("Service Config loaded")
|
|
# log key names only: the full config carries service certificates,
|
|
# device fingerprints and endpoints that bloat the log and may be sensitive
|
|
log_event(
|
|
"load_service_config",
|
|
level="DEBUG",
|
|
service=self.service,
|
|
context={
|
|
"config_path": str(service_config_path),
|
|
"config_keys": sorted(self.service_config or []),
|
|
},
|
|
)
|
|
except KeyError:
|
|
pass
|
|
merge_dict(ci_get(config.services, self.service), self.service_config)
|
|
|
|
service_config = ci_get(config.services, self.service, {})
|
|
if service_config:
|
|
reserved_keys = {
|
|
"profiles",
|
|
"api_key",
|
|
"certificate",
|
|
"api_endpoint",
|
|
"region",
|
|
"device",
|
|
"endpoints",
|
|
"client",
|
|
"dl",
|
|
}
|
|
|
|
for config_key, override_value in service_config.items():
|
|
if config_key in reserved_keys or not isinstance(override_value, dict):
|
|
continue
|
|
|
|
if hasattr(config, config_key):
|
|
current_config = getattr(config, config_key, {})
|
|
|
|
if isinstance(current_config, dict):
|
|
merged_config = deepcopy(current_config)
|
|
merge_dict(override_value, merged_config)
|
|
setattr(config, config_key, merged_config)
|
|
|
|
self.log.debug(
|
|
f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}"
|
|
)
|
|
|
|
self.cdm_override = ctx.params.get("cdm_name")
|
|
cdm_only = ctx.params.get("cdm_only")
|
|
self.vault_cache_tally: Optional[tuple[set[UUID], int]] = None
|
|
|
|
if cdm_only:
|
|
self.vaults = Vaults(self.vault_service)
|
|
verify.DECRYPT_HOOK = self.decrypt_verified
|
|
self.log.info("CDM-only mode: Skipping vault loading")
|
|
log_event(
|
|
"vault_loading_skipped",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={"reason": "cdm_only flag set"},
|
|
)
|
|
else:
|
|
with console.status("Loading Key Vaults...", spinner="dots"):
|
|
self.vaults = Vaults(self.vault_service)
|
|
verify.DECRYPT_HOOK = self.decrypt_verified
|
|
total_vaults = len(config.key_vaults)
|
|
failed_vaults = []
|
|
|
|
for vault in config.key_vaults:
|
|
vault_type = vault["type"]
|
|
vault_name = vault.get("name", vault_type)
|
|
vault_copy = vault.copy()
|
|
del vault_copy["type"]
|
|
|
|
if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower():
|
|
if "token" not in vault_copy or not vault_copy["token"]:
|
|
if config.decrypt_labs_api_key:
|
|
vault_copy["token"] = config.decrypt_labs_api_key
|
|
else:
|
|
self.log.warning(
|
|
f"No token provided for DecryptLabs vault '{vault_name}' and no global "
|
|
"decrypt_labs_api_key configured"
|
|
)
|
|
|
|
if vault_type.lower() == "sqlite":
|
|
try:
|
|
self.vaults.load_critical(vault_type, **vault_copy)
|
|
self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})")
|
|
except Exception as e:
|
|
self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}")
|
|
raise
|
|
else:
|
|
# Other vaults (MySQL, HTTP, API) - soft fail
|
|
if not self.vaults.load(vault_type, **vault_copy):
|
|
failed_vaults.append(vault_name)
|
|
self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})")
|
|
else:
|
|
self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})")
|
|
|
|
loaded_count = len(self.vaults)
|
|
if failed_vaults:
|
|
self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}")
|
|
self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults")
|
|
|
|
# Debug: Show detailed vault status
|
|
if loaded_count > 0:
|
|
vault_names = [vault.name for vault in self.vaults]
|
|
self.log.debug(f"Active vaults: {', '.join(vault_names)}")
|
|
else:
|
|
self.log.debug("No vaults are currently active")
|
|
|
|
with console.status("Loading DRM CDM...", spinner="dots"):
|
|
try:
|
|
self.cdm = self.get_cdm(self.service, self.profile)
|
|
except ValueError as e:
|
|
self.log.error(f"Failed to load CDM, {e}")
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error("load_cdm", e, service=self.service)
|
|
sys.exit(1)
|
|
|
|
if self.cdm:
|
|
cdm_info = {}
|
|
if isinstance(self.cdm, DecryptLabsRemoteCDM):
|
|
drm_type = "PlayReady" if self.cdm.is_playready else "Widevine"
|
|
self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})")
|
|
cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level}
|
|
elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]:
|
|
self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})")
|
|
cdm_info = {
|
|
"type": "Widevine",
|
|
"system_id": self.cdm.system_id,
|
|
"security_level": self.cdm.security_level,
|
|
"device_type": self.cdm.device_type.name,
|
|
}
|
|
else:
|
|
# Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None)
|
|
is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name")
|
|
if is_remote:
|
|
cdm_name = self.cdm.device_name
|
|
self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})")
|
|
else:
|
|
cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown"
|
|
self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})")
|
|
cdm_info = {
|
|
"type": "PlayReady",
|
|
"certificate": cdm_name,
|
|
"security_level": self.cdm.security_level,
|
|
}
|
|
|
|
if cdm_info:
|
|
log_event("load_cdm", level="INFO", service=self.service, context={"cdm": cdm_info})
|
|
|
|
if ctx.params.get("local_proxy"):
|
|
require_alone_local_proxy(ctx)
|
|
proxy, no_proxy = ctx.params.get("proxy"), bool(ctx.params.get("no_proxy"))
|
|
|
|
# A server on this machine can reach a local proxy, so only a server elsewhere is guarded
|
|
self.remote_needs_public_proxy = (
|
|
self.is_remote
|
|
and not ctx.params.get("local_proxy")
|
|
and not is_loopback(server_url(ctx.params.get("server")))
|
|
)
|
|
|
|
self.proxy_providers = []
|
|
if no_proxy:
|
|
ctx.params["proxy"] = None
|
|
else:
|
|
if self.remote_needs_public_proxy and proxy and proxy.lower().startswith("gluetun:"):
|
|
raise click.UsageError("Gluetun runs on your machine, so --remote needs --local-proxy to use it.")
|
|
if proxy_providers is not None:
|
|
self.proxy_providers = list(proxy_providers)
|
|
else:
|
|
with console.status("Loading Proxy Providers...", spinner="dots"):
|
|
self.proxy_providers = load_proxy_providers()
|
|
|
|
ctx.params["proxy_arg"] = proxy
|
|
if proxy and proxy.startswith("controld://"):
|
|
proxy = ctx.params["proxy"] = resolve_proxy(proxy, [])
|
|
if proxy:
|
|
try:
|
|
requested_provider, query = split_proxy_query(proxy, self.proxy_providers)
|
|
except ValueError as e:
|
|
self.log.error(str(e))
|
|
sys.exit(1)
|
|
# Match region codes (us, ca, uk1, us:ny, yul) or provider:region (nordvpn:ca, protonvpn:us:ny).
|
|
if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", query, re.IGNORECASE):
|
|
# Kept apart from ctx.params["proxy"], which becomes the URI: a service proxy_map keys on it.
|
|
proxy_query = query.lower()
|
|
status_msg = (
|
|
f"Connecting to VPN ({proxy_query})..."
|
|
if requested_provider == "gluetun"
|
|
else f"Getting a Proxy to {proxy_query}..."
|
|
)
|
|
with console.status(status_msg, spinner="dots"):
|
|
try:
|
|
proxy_provider, proxy_uri = pick_proxy(
|
|
self.proxy_providers, proxy_query, requested_provider
|
|
)
|
|
except ValueError as e:
|
|
self.log.error(str(e))
|
|
sys.exit(1)
|
|
ctx.params["proxy"] = proxy_uri
|
|
self.log.info(describe_proxy(proxy_provider, proxy_uri, proxy_query))
|
|
ctx.params["proxy_query"] = proxy_query
|
|
ctx.params["proxy_provider"] = requested_provider
|
|
else:
|
|
self.log.info(describe_explicit_proxy(ctx.params["proxy"]))
|
|
ctx.params["proxy_query"] = None
|
|
ctx.params["proxy_provider"] = None
|
|
|
|
ctx.obj = ContextData(
|
|
config=self.service_config,
|
|
cdm=self.cdm,
|
|
proxy_providers=self.proxy_providers,
|
|
profile=self.profile,
|
|
cdm_entry=self.cdm_from_service_entry(),
|
|
cdm_for=self.cdm_for,
|
|
)
|
|
|
|
if repack:
|
|
config.repack = True
|
|
|
|
if tag:
|
|
config.tag = tag
|
|
config.tag_rules = []
|
|
|
|
# needs to be added this way instead of @cli.result_callback to be
|
|
# able to keep `self` as the first positional
|
|
self.cli._result_callback = self.result
|
|
|
|
@with_task_temp
|
|
def result(
|
|
self,
|
|
service: Service,
|
|
quality: list[int],
|
|
vcodec: list[Video.Codec],
|
|
acodec: list[Audio.Codec],
|
|
vbitrate: Optional[int],
|
|
abitrate: Optional[int],
|
|
vbitrate_range: Optional[str],
|
|
abitrate_range: Optional[str],
|
|
range_: list[Video.Range],
|
|
channels: Optional[float],
|
|
no_atmos: bool,
|
|
select_titles: bool,
|
|
wanted: list[str],
|
|
latest_episode: bool,
|
|
latest_episodes: Optional[int],
|
|
lang: list[str],
|
|
v_lang: Optional[list[str]],
|
|
a_lang: Optional[list[str]],
|
|
s_lang: Optional[list[str]],
|
|
require_audio: list[str],
|
|
require_video: list[str],
|
|
require_subs: list[str],
|
|
skip_missing: bool,
|
|
forced_subs: bool,
|
|
forced_subs_only: bool,
|
|
forced_s_lang: list[str],
|
|
exact_lang: bool,
|
|
sub_format: Optional[Union[Subtitle.Codec, str]],
|
|
sub_font: Optional[str],
|
|
video_only: bool,
|
|
audio_only: bool,
|
|
subs_only: bool,
|
|
chapters_only: bool,
|
|
no_subs: bool,
|
|
skip_subtitle_errors: bool,
|
|
no_audio: bool,
|
|
no_chapters: bool,
|
|
no_video: bool,
|
|
no_attachments: bool,
|
|
audio_description: bool,
|
|
audio_description_only: bool,
|
|
slow: Optional[tuple[int, int]],
|
|
list_: bool,
|
|
list_titles: bool,
|
|
skip_dl: bool,
|
|
export: Optional[bool],
|
|
cdm_only: Optional[bool],
|
|
no_proxy: bool,
|
|
no_proxy_download: bool,
|
|
proxy_download: Optional[str],
|
|
no_folder: bool,
|
|
no_source: bool,
|
|
no_mux: bool,
|
|
sample: Optional[tuple[int, int]],
|
|
include_sample: bool,
|
|
workers: Optional[int],
|
|
adaptive_workers: bool,
|
|
download_processes: int,
|
|
continue_downloads: bool,
|
|
downloads: int,
|
|
worst: bool,
|
|
best_available: bool,
|
|
speed_limit: Optional[str] = None,
|
|
split_audio: Optional[bool] = None,
|
|
merge_video: Optional[bool] = None,
|
|
real_video_bitrate: bool = False,
|
|
real_audio_bitrate: bool = False,
|
|
progress_sink: Optional[Callable[[dict[str, Any]], None]] = None,
|
|
postscript: Sequence[str] = (),
|
|
no_postscript: bool = False,
|
|
all_drm: bool = False,
|
|
generate_dv: bool = False,
|
|
*_: Any,
|
|
**__: Any,
|
|
) -> None:
|
|
if all_drm:
|
|
DOWNLOAD_ALL_DRM.set()
|
|
else:
|
|
DOWNLOAD_ALL_DRM.clear()
|
|
self.all_drm_attempted: defaultdict[type, set[UUID]] = defaultdict(set)
|
|
self.all_drm_warned: set[str] = set()
|
|
if no_postscript:
|
|
postscript = NO_POST_SCRIPTS
|
|
if continue_downloads:
|
|
config.continue_downloads = True
|
|
config.sample = sample
|
|
if sample and include_sample:
|
|
raise click.ClickException("--sample and --include-sample cannot be used together.")
|
|
sample_dir = SAMPLE_DIR if sample else ""
|
|
self.tmdb_searched = False
|
|
self.search_source = None
|
|
self.service_anime = bool(getattr(service, "ANIME", False))
|
|
self.service_daily = bool(getattr(service, "DAILY", False))
|
|
from unshackle.core.import_service import SERVER_KEY_SOURCES, ImportService
|
|
|
|
# a remote session sends --cdm to the server, which can hand it the tracks above its cap
|
|
if self.cdm_override and isinstance(service, ImportService):
|
|
self.log.warning("--cdm is ignored: an import uses the keys in the export")
|
|
self._remote_service = service if isinstance(service, SERVER_KEY_SOURCES) else None
|
|
start_time = time.time()
|
|
|
|
lang, lang_excl = partition_exclusions(lang)
|
|
v_lang, v_lang_excl = partition_exclusions(v_lang)
|
|
a_lang, a_lang_excl = partition_exclusions(a_lang)
|
|
s_lang, s_lang_excl = partition_exclusions(s_lang)
|
|
forced_s_lang, fsl_excl = partition_exclusions(forced_s_lang)
|
|
for flag, excludes in (
|
|
("-l/--lang", lang_excl),
|
|
("-vl/--v-lang", v_lang_excl),
|
|
("-al/--a-lang", a_lang_excl),
|
|
("-sl/--s-lang", s_lang_excl),
|
|
("-fsl/--forced-s-lang", fsl_excl),
|
|
):
|
|
if any(token.lower() == "all" for token in excludes):
|
|
self.log.error(f"{flag}: 'all' cannot be excluded")
|
|
sys.exit(1)
|
|
# exclusion-only use falls back to the flag's own default; -vl/-al have none, they cascade to -l
|
|
if lang_excl and not lang:
|
|
lang = ["orig"]
|
|
if s_lang_excl and not s_lang:
|
|
s_lang = ["all"]
|
|
# an override that names languages replaces -l entirely, so it must not inherit its exclusions
|
|
video_excl = list(dict.fromkeys(v_lang_excl + ([] if v_lang else lang_excl)))
|
|
audio_excl = list(dict.fromkeys(a_lang_excl + ([] if a_lang else lang_excl)))
|
|
for flag, tokens, excludes in (
|
|
("--require-audio", require_audio, audio_excl),
|
|
("--require-video", require_video, video_excl),
|
|
("--require-subs", require_subs, s_lang_excl),
|
|
):
|
|
required, own_excl = partition_exclusions(tokens)
|
|
clash = own_excl + [t for t in required if any(t.lower() == e.lower() for e in excludes)]
|
|
if clash:
|
|
self.log.error(f"{flag}: {clash[0]} cannot be required and excluded at the same time.")
|
|
sys.exit(1)
|
|
|
|
ctx = getattr(service, "ctx", None)
|
|
parent_params = ctx.parent.params if ctx and ctx.parent else None
|
|
if parent_params is not None:
|
|
# services read the raw ctx params, they must never see a '-xx' token
|
|
for name, includes, excludes in (
|
|
("lang", lang, lang_excl),
|
|
("v_lang", v_lang, v_lang_excl),
|
|
("a_lang", a_lang, a_lang_excl),
|
|
("s_lang", s_lang, s_lang_excl),
|
|
("forced_s_lang", forced_s_lang, fsl_excl),
|
|
):
|
|
if excludes and name in parent_params:
|
|
parent_params[name] = includes
|
|
|
|
conflict = kind_only_conflict(
|
|
{
|
|
"audio_description_only": audio_description_only,
|
|
"forced_subs_only": forced_subs_only,
|
|
"no_audio": no_audio,
|
|
"no_subs": no_subs,
|
|
"video_only": video_only,
|
|
"audio_only": audio_only,
|
|
"subs_only": subs_only,
|
|
"chapters_only": chapters_only,
|
|
}
|
|
)
|
|
if conflict:
|
|
first, second = (f"--{name.replace('_', '-')}" for name in conflict)
|
|
self.log.error(f"{first} cannot be used with {second}: {second} drops the tracks that {first} selects.")
|
|
sys.exit(1)
|
|
|
|
if audio_description_only:
|
|
audio_description = True
|
|
|
|
if forced_s_lang or fsl_excl or forced_subs_only:
|
|
# services read forced_subs from raw ctx params, so the implication must be pushed there too
|
|
forced_subs = True
|
|
if "all" in forced_s_lang:
|
|
forced_s_lang = []
|
|
if parent_params is not None:
|
|
parent_params["forced_subs"] = True
|
|
|
|
if skip_dl:
|
|
DOWNLOAD_LICENCE_ONLY.set()
|
|
|
|
try:
|
|
speed_limit_bps = parse_speed_limit(speed_limit)
|
|
except ValueError as e:
|
|
self.log.error(str(e))
|
|
sys.exit(1)
|
|
set_speed_limit(speed_limit_bps)
|
|
if speed_limit_bps:
|
|
self.log.info(f"Speed limit: {format_speed(speed_limit_bps)}")
|
|
|
|
if no_proxy or no_proxy_download:
|
|
proxy_download = None
|
|
elif proxy_download and (
|
|
proxy_download.startswith("controld://")
|
|
or re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", proxy_download, re.I)
|
|
):
|
|
# same shapes --proxy resolves against providers (two prefixes for gluetun:nordvpn:ca); else an explicit URI
|
|
try:
|
|
proxy_download = resolve_proxy(proxy_download.lower(), self.proxy_providers)
|
|
except ValueError as e:
|
|
self.log.error(f"--proxy-download: {e}")
|
|
sys.exit(1)
|
|
|
|
if export:
|
|
config.directories.exports.mkdir(parents=True, exist_ok=True)
|
|
# a working name until the first write, which renames it after what it holds
|
|
export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json"
|
|
self.export_service = service
|
|
self.export_namer = ExportNamer(self.service)
|
|
else:
|
|
export_path = None
|
|
|
|
vbitrate_min, vbitrate_max = None, None
|
|
if vbitrate_range:
|
|
if vbitrate and vbitrate_range:
|
|
self.log.error("Cannot use both --vbitrate and --vbitrate-range at the same time.")
|
|
sys.exit(1)
|
|
try:
|
|
parts = vbitrate_range.split("-")
|
|
if len(parts) != 2:
|
|
raise ValueError
|
|
vbitrate_min, vbitrate_max = int(parts[0]), int(parts[1])
|
|
if vbitrate_min > vbitrate_max:
|
|
vbitrate_min, vbitrate_max = vbitrate_max, vbitrate_min
|
|
except (ValueError, IndexError):
|
|
self.log.error("Invalid --vbitrate-range: expected MIN-MAX, such as 6000-7000")
|
|
sys.exit(1)
|
|
|
|
abitrate_min, abitrate_max = None, None
|
|
if abitrate_range:
|
|
if abitrate and abitrate_range:
|
|
self.log.error("Cannot use both --abitrate and --abitrate-range at the same time.")
|
|
sys.exit(1)
|
|
try:
|
|
parts = abitrate_range.split("-")
|
|
if len(parts) != 2:
|
|
raise ValueError
|
|
abitrate_min, abitrate_max = int(parts[0]), int(parts[1])
|
|
if abitrate_min > abitrate_max:
|
|
abitrate_min, abitrate_max = abitrate_max, abitrate_min
|
|
except (ValueError, IndexError):
|
|
self.log.error("Invalid --abitrate-range: expected MIN-MAX, such as 128-256")
|
|
sys.exit(1)
|
|
|
|
if not acodec:
|
|
acodec = []
|
|
elif isinstance(acodec, Audio.Codec):
|
|
acodec = [acodec]
|
|
elif isinstance(acodec, str) or (
|
|
isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec)
|
|
):
|
|
acodec = AUDIO_CODEC_LIST.convert(acodec)
|
|
|
|
if select_titles and wanted:
|
|
self.log.error("--select-titles and -w/--wanted cannot be used together")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
sub_format, ass_style = subtitle_look(
|
|
sub_format,
|
|
sub_font,
|
|
config.subtitle.get("ass_style"),
|
|
typed_font="sub_font" in self.typed_params,
|
|
typed_format="sub_format" in self.typed_params,
|
|
)
|
|
except ValueError as e:
|
|
self.log.error(str(e))
|
|
sys.exit(1)
|
|
|
|
if generate_dv and Video.Range.DV in range_ and Video.Range.HYBRID not in range_:
|
|
self.log.warning("--generate-dv ignored: DV is in the selected ranges")
|
|
range_, generated_dv_bases = generated_dv_ranges(range_, generate_dv)
|
|
if any(r == Video.Range.HYBRID for r in range_):
|
|
from unshackle.core.binaries import DoviTool
|
|
|
|
if not DoviTool:
|
|
self.log.error("Unable to run hybrid mode: dovi_tool not found")
|
|
sys.exit(1)
|
|
|
|
if cdm_only is None:
|
|
vaults_only = None
|
|
else:
|
|
vaults_only = not cdm_only
|
|
|
|
config.decryption = resolve_decryption(
|
|
config.decryption_map, config.decryption, service.__class__.__name__.upper()
|
|
)
|
|
|
|
log_event(
|
|
"drm_mode_config",
|
|
level="DEBUG",
|
|
service=self.service,
|
|
context={
|
|
"cdm_only": cdm_only,
|
|
"vaults_only": vaults_only,
|
|
"mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"),
|
|
"decryption": config.decryption,
|
|
},
|
|
)
|
|
|
|
with console.status(
|
|
"Preparing Remote Service Session..." if self.is_remote else "Preparing Service Session...",
|
|
spinner="dots",
|
|
):
|
|
try:
|
|
cookies = self.get_cookie_jar(self.service, self.profile)
|
|
credential = self.get_credentials(self.service, self.profile)
|
|
service.authenticate(cookies, credential)
|
|
if cookies or credential:
|
|
self.log.info("Authenticated with Service")
|
|
log_event(
|
|
"authenticate",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"has_cookies": bool(cookies),
|
|
"has_credentials": bool(credential),
|
|
"profile": self.profile,
|
|
},
|
|
)
|
|
except Exception as e:
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"authenticate", e, service=self.service, context={"profile": self.profile}
|
|
)
|
|
raise
|
|
|
|
with console.status(
|
|
"Fetching Remote Title Metadata..." if self.is_remote else "Fetching Title Metadata...", spinner="dots"
|
|
):
|
|
try:
|
|
titles = service.get_titles_cached()
|
|
if not titles:
|
|
self.log.error("No titles returned, nothing to download...")
|
|
log_event(
|
|
"get_titles",
|
|
level="ERROR",
|
|
service=self.service,
|
|
message="No titles returned from service",
|
|
success=False,
|
|
)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error("get_titles", e, service=self.service)
|
|
raise
|
|
|
|
if self.debug_logger:
|
|
titles_info = {
|
|
"type": titles.__class__.__name__,
|
|
"count": len(titles) if hasattr(titles, "__len__") else 1,
|
|
"title": str(titles),
|
|
}
|
|
if hasattr(titles, "seasons"):
|
|
titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0
|
|
self.debug_logger.log(
|
|
level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info}
|
|
)
|
|
|
|
title_cacher = service.title_cache if hasattr(service, "title_cache") else None
|
|
cache_title_id = None
|
|
if hasattr(service, "title"):
|
|
cache_title_id = service.title
|
|
elif hasattr(service, "title_id"):
|
|
cache_title_id = service.title_id
|
|
cache_region = service.current_region if hasattr(service, "current_region") else None
|
|
cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None
|
|
|
|
enrich_lang: Optional[Language] = None
|
|
enrich_title: Optional[str] = None
|
|
enrich_year: Optional[int] = None
|
|
if self.enrich:
|
|
sample_title = titles[0] if hasattr(titles, "__getitem__") else titles
|
|
kind = "tv" if isinstance(sample_title, Episode) else "movie"
|
|
|
|
enrich_result = providers.resolve_by_ids(
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
kind=kind,
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
anime=self.anime_hint(sample_title),
|
|
)
|
|
if enrich_result:
|
|
enrich_title = enrich_result.title
|
|
enrich_year = enrich_result.year
|
|
enrich_lang = parse_language(enrich_result.original_language)
|
|
|
|
if enrich_lang:
|
|
self.log.info(f"Original language from metadata: {enrich_lang}")
|
|
if not (enrich_title or enrich_year or enrich_lang):
|
|
self.log.warning("--enrich found no metadata")
|
|
elif missing := [
|
|
name
|
|
for name, value in (("title", enrich_title), ("year", enrich_year), ("language", enrich_lang))
|
|
if not value
|
|
]:
|
|
self.log.warning(f"--enrich source gave no {', '.join(missing)}, kept the service's values")
|
|
|
|
if isinstance(titles, (Series, Movies)):
|
|
for t in titles:
|
|
apply_enrich(t, enrich_title, enrich_year, enrich_lang)
|
|
|
|
if isinstance(titles, Series):
|
|
enrich_tvdb_id = self.tvdb_id or (enrich_result.external_ids.tvdb_id if enrich_result else None)
|
|
self.fill_absolute_numbers(titles, enrich_tvdb_id)
|
|
if any(self.daily_hint(t) for t in titles):
|
|
self.fill_air_dates(titles, enrich_tvdb_id)
|
|
|
|
if self.tvdb_order and isinstance(titles, Series):
|
|
titles = self.apply_tvdb_order(titles, title_cacher, cache_title_id, cache_region, cache_account_hash)
|
|
|
|
if self.continuous_episodes and isinstance(titles, Series):
|
|
titles = self.apply_continuous_episodes(titles)
|
|
|
|
console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2)))
|
|
print_listing(titles.tree(verbose=list_titles))
|
|
|
|
if list_titles:
|
|
return
|
|
|
|
if export_path and isinstance(titles, Series):
|
|
# before --select-titles and -w drop titles, so a whole season can be told from part of one
|
|
self.export_namer.seasons = season_episodes(titles)
|
|
|
|
if select_titles and isinstance(titles, Movies) and len(titles) > 1:
|
|
console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2)))
|
|
|
|
selection_start = time.time()
|
|
selected_idx = select_multiple(
|
|
[str(movie) for movie in titles],
|
|
minimal_count=1,
|
|
page_size=8,
|
|
return_indices=True,
|
|
)
|
|
if not selected_idx:
|
|
console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5)))
|
|
return
|
|
start_time += time.time() - selection_start
|
|
|
|
keep = set(selected_idx)
|
|
for i in range(len(titles) - 1, -1, -1):
|
|
if i not in keep:
|
|
del titles[i]
|
|
|
|
console.print(Padding(f"[text]Total selected: {len(titles)}[/]", (0, 5)))
|
|
|
|
# Enables manual selection for Series when --select-titles is set
|
|
if select_titles and isinstance(titles, Series):
|
|
console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2)))
|
|
|
|
selection_titles = []
|
|
dependencies: dict[int, list[int]] = {}
|
|
original_indices = {}
|
|
|
|
current_season = None
|
|
current_season_header_idx = -1
|
|
|
|
unique_seasons = {t.season for t in titles}
|
|
multiple_seasons = len(unique_seasons) > 1
|
|
|
|
# Build selection options
|
|
for i, t in enumerate(titles):
|
|
# Insert season header only if multiple seasons exist
|
|
if multiple_seasons and t.season != current_season:
|
|
current_season = t.season
|
|
header_text = f"Season {t.season}"
|
|
selection_titles.append(header_text)
|
|
current_season_header_idx = len(selection_titles) - 1
|
|
dependencies[current_season_header_idx] = []
|
|
|
|
display_name = ((t.name[:30].rstrip() + "...") if len(t.name) > 30 else t.name) if t.name else None
|
|
|
|
prefix = " " if multiple_seasons else ""
|
|
part_label = f".{t.part}" if t.part is not None else ""
|
|
option_text = f"{prefix}{t.number}{part_label}" + (f". {display_name}" if t.name else "")
|
|
|
|
selection_titles.append(option_text)
|
|
current_ui_idx = len(selection_titles) - 1
|
|
|
|
original_indices[current_ui_idx] = i
|
|
|
|
if current_season_header_idx != -1:
|
|
dependencies[current_season_header_idx].append(current_ui_idx)
|
|
|
|
selection_start = time.time()
|
|
|
|
selected_ui_idx = select_multiple(
|
|
selection_titles,
|
|
minimal_count=1,
|
|
page_size=8,
|
|
return_indices=True,
|
|
dependencies=dependencies,
|
|
collapse_on_start=multiple_seasons,
|
|
)
|
|
|
|
if not selected_ui_idx:
|
|
console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5)))
|
|
return
|
|
|
|
selection_end = time.time()
|
|
start_time += selection_end - selection_start
|
|
|
|
selected_idx = []
|
|
for idx in selected_ui_idx:
|
|
if idx in original_indices:
|
|
selected_idx.append(original_indices[idx])
|
|
|
|
# Ensure indices are unique and ordered
|
|
selected_idx = sorted(set(selected_idx))
|
|
keep = set(selected_idx)
|
|
|
|
for i in range(len(titles) - 1, -1, -1):
|
|
if i not in keep:
|
|
del titles[i]
|
|
|
|
if titles:
|
|
count = len(titles)
|
|
console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5)))
|
|
|
|
latest_keys = latest_episode_keys(titles, latest_episode, latest_episodes)
|
|
if latest_keys:
|
|
picked = ", ".join(f"S{season:02}E{number:02}" for season, number in latest_keys)
|
|
self.log.info(f"Latest episode mode: Selecting {picked}")
|
|
# replaces -w; a base SxE key selects every part of a split episode
|
|
wanted = [f"{season}x{number}" for season, number in latest_keys]
|
|
|
|
def select_best_audio(
|
|
audio: list[Audio],
|
|
languages: list[str],
|
|
acodec: list[Audio.Codec],
|
|
audio_description: bool,
|
|
exact_lang: bool,
|
|
) -> list[Audio]:
|
|
"""Pick the best audio per (language, codec) group from a sort_audio-ordered list.
|
|
|
|
"Best" is the first track of each order-preserving candidate filter, so the
|
|
sort_audio ranking (descriptive-last, Atmos, codec_priority, bitrate) decides the
|
|
winner. With audio_description both the best standard and best descriptive track per
|
|
group are kept. "best"/"all" tokens select every language present in sorted order.
|
|
"""
|
|
select_all = any(tok in ("best", "all") for tok in languages)
|
|
if select_all:
|
|
target_langs: list[Any] = list(dict.fromkeys(t.language for t in audio))
|
|
else:
|
|
target_langs = languages
|
|
codecs: list[Optional[Audio.Codec]] = list(acodec) if len(acodec) > 1 else [None]
|
|
|
|
selected: list[Audio] = []
|
|
for language in target_langs:
|
|
for codec in codecs:
|
|
if select_all:
|
|
candidates = [
|
|
t for t in audio if t.language == language and (codec is None or t.codec == codec)
|
|
]
|
|
else:
|
|
# match against every codec at once, so the variant a tag means does not depend on codec
|
|
tags = matching_languages(language, [t.language for t in audio], exact_lang)
|
|
candidates = [
|
|
t for t in audio if str(t.language) in tags and (codec is None or t.codec == codec)
|
|
]
|
|
if not candidates:
|
|
continue
|
|
if audio_description:
|
|
standards = [t for t in candidates if not t.descriptive]
|
|
if standards:
|
|
selected.append(standards[0])
|
|
descs = [t for t in candidates if t.descriptive]
|
|
if descs:
|
|
selected.append(descs[0])
|
|
else:
|
|
selected.append(candidates[0])
|
|
# Built via per-tag by_language calls, so overlapping tags can append
|
|
# the same track across iterations; dedupe by id.
|
|
unique: dict[str, Audio] = {}
|
|
for t in selected:
|
|
unique.setdefault(t.id, t)
|
|
return list(unique.values())
|
|
|
|
base_selection = (v_lang, a_lang, s_lang, range_)
|
|
|
|
post_script_pending: dict[Any, int] = {}
|
|
for candidate in titles:
|
|
if title_wanted(candidate, wanted):
|
|
key = post_script_group(candidate)
|
|
post_script_pending[key] = post_script_pending.get(key, 0) + 1
|
|
post_script_last: dict[Any, dict[Path, dict[str, str]]] = {}
|
|
post_script_folders: list[Path] = []
|
|
post_script_sample: dict[str, str] = {}
|
|
wanted_total = sum(post_script_pending.values())
|
|
|
|
def settle_post_script_group(title: Any) -> None:
|
|
"""Count one title of its group as done; send the season event after the last one."""
|
|
group_key = post_script_group(title)
|
|
post_script_pending[group_key] = post_script_pending.get(group_key, 1) - 1
|
|
if post_script_pending[group_key] <= 0:
|
|
for folder, context in post_script_last.get(group_key, {}).items():
|
|
dispatch("success", "season", season_context(context, folder), postscript)
|
|
|
|
def skip_title(title: Any, reason: str, kind_only: bool = False) -> None:
|
|
"""Skip a title that has no track the selection asks for, and report it to the API job.
|
|
|
|
Only a kind-only miss (-ado, -fso, no track left) skips without --skip-missing; any other miss
|
|
stops the run. A run with one title always stops, because it has nothing else to download.
|
|
"""
|
|
if wanted_total <= 1 or not (kind_only or skip_missing):
|
|
self.stop_reason = f"{title}: {reason}"
|
|
self.log.error(self.stop_reason)
|
|
sys.exit(1)
|
|
self.skipped_titles.append(SkippedTitle(title=str(title), reason=reason))
|
|
if progress_sink:
|
|
progress_sink({"skipped_titles": list(self.skipped_titles)})
|
|
self.log.warning(f"Skipping {title}: {reason}")
|
|
if not mux_skipped(no_mux, title):
|
|
settle_post_script_group(title)
|
|
|
|
for i, title in enumerate(titles):
|
|
v_lang, a_lang, s_lang, range_ = base_selection
|
|
if not title_wanted(title, wanted):
|
|
continue
|
|
|
|
if progress_sink:
|
|
if isinstance(title, Episode):
|
|
progress_sink(
|
|
{
|
|
"title": title.title,
|
|
"current_title": f"S{title.season or 0:02}E{title.number or 0:02}"
|
|
+ (f".{title.part}" if title.part is not None else ""),
|
|
}
|
|
)
|
|
else:
|
|
progress_sink({"title": getattr(title, "name", None) or str(title)})
|
|
|
|
title_rule = f"Track {title.track:02}: {title.name}" if isinstance(title, Song) else str(title)
|
|
console.print(Padding(Rule(f"[rule.text]{title_rule}"), (1, 2)))
|
|
|
|
if isinstance(title, Episode) and not self.tmdb_searched:
|
|
kind = "tv"
|
|
tmdb_title: Optional[str] = None
|
|
has_ids = bool(self.tmdb_id or self.imdb_id or self.tvdb_id or self.anilist_id)
|
|
result = providers.resolve_by_ids(
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
title=title.title,
|
|
year=title.year,
|
|
kind=kind,
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
anime=self.anime_hint(title),
|
|
)
|
|
if result:
|
|
if has_ids:
|
|
tmdb_title = result.title
|
|
if not self.tmdb_id:
|
|
self.tmdb_id = result.external_ids.tmdb_id
|
|
elif result.title and providers.fuzzy_match(result.title, title.title):
|
|
self.tmdb_id = result.external_ids.tmdb_id
|
|
tmdb_title = result.title
|
|
self.search_source = result.source
|
|
if list_ or list_titles:
|
|
if self.tmdb_id:
|
|
console.print(
|
|
Padding(
|
|
f"Search -> {tmdb_title or '?'} [bright_black](ID {self.tmdb_id})",
|
|
(0, 5),
|
|
)
|
|
)
|
|
else:
|
|
console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5)))
|
|
self.tmdb_searched = True
|
|
|
|
if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id:
|
|
movie_result = providers.resolve_by_ids(
|
|
None,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
title=title.name,
|
|
year=title.year,
|
|
kind="movie",
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
anime=self.anime_hint(title),
|
|
)
|
|
if movie_result and movie_result.external_ids.tmdb_id:
|
|
console.print(
|
|
Padding(
|
|
f"Search -> {movie_result.title or '?'} "
|
|
f"[bright_black](ID {movie_result.external_ids.tmdb_id})",
|
|
(0, 5),
|
|
)
|
|
)
|
|
else:
|
|
console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5)))
|
|
|
|
if self.tmdb_id and getattr(self, "search_source", None) not in ("simkl", "imdb"):
|
|
kind = "tv" if isinstance(title, Episode) else "movie"
|
|
providers.fetch_external_ids(
|
|
self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash
|
|
)
|
|
|
|
if slow and i != 0:
|
|
delay = random.randint(slow[0], slow[1])
|
|
spinner = Spinner("dots", text=f"Delaying by {delay} seconds...")
|
|
with SyncLive(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True):
|
|
for seconds_left in range(delay, 0, -1):
|
|
spinner.update(text=f"Delaying by {seconds_left} seconds...")
|
|
time.sleep(1)
|
|
|
|
with console.status("Subscribing to events...", spinner="dots"):
|
|
events.reset()
|
|
events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded)
|
|
events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded)
|
|
events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted)
|
|
events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked)
|
|
events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex)
|
|
|
|
tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..."
|
|
tracks_spinner = Spinner("dots", text=tracks_label)
|
|
with SyncLive(Padding(tracks_spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True):
|
|
try:
|
|
load_tracks(
|
|
service,
|
|
title,
|
|
name=enrich_title,
|
|
year=enrich_year,
|
|
language=enrich_lang,
|
|
workers=workers,
|
|
on_probe=lambda: tracks_spinner.update(text="Probing HDR10 tracks for HDR10+..."),
|
|
)
|
|
except Exception as e:
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"get_tracks", e, service=self.service, context={"title": str(title)}
|
|
)
|
|
raise
|
|
|
|
if self.debug_logger:
|
|
tracks_info = {
|
|
"title": str(title),
|
|
"video_tracks": len(title.tracks.videos),
|
|
"audio_tracks": len(title.tracks.audio),
|
|
"subtitle_tracks": len(title.tracks.subtitles),
|
|
"has_chapters": bool(title.tracks.chapters),
|
|
"videos": [
|
|
{
|
|
"codec": str(v.codec),
|
|
"resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown",
|
|
"bitrate": v.bitrate,
|
|
"range": str(v.range),
|
|
"language": str(v.language) if v.language else None,
|
|
"drm": [str(type(d).__name__) for d in v.drm] if v.drm else [],
|
|
}
|
|
for v in title.tracks.videos
|
|
],
|
|
"audio": [
|
|
{
|
|
"codec": str(a.codec),
|
|
"bitrate": a.bitrate,
|
|
"channels": a.channels,
|
|
"language": str(a.language) if a.language else None,
|
|
"descriptive": a.descriptive,
|
|
"drm": [str(type(d).__name__) for d in a.drm] if a.drm else [],
|
|
}
|
|
for a in title.tracks.audio
|
|
],
|
|
"subtitles": [
|
|
{
|
|
"codec": str(s.codec),
|
|
"language": str(s.language) if s.language else None,
|
|
"forced": s.forced,
|
|
"sdh": s.sdh,
|
|
}
|
|
for s in title.tracks.subtitles
|
|
],
|
|
}
|
|
self.debug_logger.log(
|
|
level="INFO", operation="get_tracks", service=self.service, context=tracks_info
|
|
)
|
|
|
|
if dedupe_codecs := config.subtitle.get("dedupe_codecs"):
|
|
codecs = map(SUBTITLE_CODEC.convert, dedupe_codecs)
|
|
title.tracks.dedupe_subtitle_codecs([c for c in codecs if isinstance(c, Subtitle.Codec)])
|
|
|
|
# a plain track covers an SDH track only when it is the same variant, as -sl selects them,
|
|
# so en-US SDH is stripped next to a plain en-GB: -sl en-US would otherwise keep only the SDH
|
|
if config.subtitle.get("strip_sdh", True):
|
|
plain_langs = [x.language for x in title.tracks.subtitles if not x.sdh and not x.forced]
|
|
for subtitle in title.tracks.subtitles:
|
|
if subtitle.sdh and not matching_languages(subtitle.language, plain_langs, exact=True):
|
|
plain_langs.append(subtitle.language)
|
|
non_sdh_sub = deepcopy(subtitle)
|
|
non_sdh_sub.id += "_stripped"
|
|
non_sdh_sub.sdh = False
|
|
title.tracks.add(non_sdh_sub)
|
|
events.subscribe(
|
|
events.Types.TRACK_MULTIPLEX,
|
|
lambda track, sub_id=non_sdh_sub.id: (
|
|
(track.strip_hearing_impaired()) if track.id == sub_id else None
|
|
),
|
|
)
|
|
|
|
if real_video_bitrate:
|
|
with console.status("Probing real video bitrates...", spinner="dots"):
|
|
apply_real_bitrates(
|
|
title.tracks.videos,
|
|
service.session,
|
|
log=self.log,
|
|
group_key=lambda t: (t.codec, t.range),
|
|
)
|
|
|
|
if real_audio_bitrate:
|
|
with console.status("Probing real audio bitrates...", spinner="dots"):
|
|
apply_real_bitrates(
|
|
title.tracks.audio,
|
|
service.session,
|
|
log=self.log,
|
|
group_key=lambda t: (t.codec, t.channels, str(t.language), t.descriptive),
|
|
)
|
|
|
|
with console.status("Sorting tracks by language and bitrate...", spinner="dots"):
|
|
audio_priority = config.audio.get("language_priority") or []
|
|
subtitle_priority = config.subtitle.get("language_priority") or []
|
|
|
|
title.tracks.sort_videos(
|
|
by_language=resolve_sort_langs(v_lang or lang, title.language, exact=exact_lang),
|
|
exact_match=exact_lang,
|
|
)
|
|
title.tracks.sort_audio(
|
|
by_language=resolve_sort_langs(
|
|
[*audio_priority, *(a_lang or lang)], title.language, exact=exact_lang
|
|
),
|
|
codec_priority=config.audio.get("codec_priority"),
|
|
exact_match=exact_lang,
|
|
)
|
|
title.tracks.sort_subtitles(
|
|
by_language=resolve_sort_langs(
|
|
[*subtitle_priority, *(s_lang or [])], title.language, exact=exact_lang
|
|
),
|
|
type_priority=config.subtitle.get("type_priority"),
|
|
group_by=config.subtitle.get("group_by"),
|
|
exact_match=exact_lang,
|
|
)
|
|
|
|
if list_:
|
|
available_tracks, _ = title.tracks.tree()
|
|
print_listing(listing_panel(available_tracks, "Available Tracks"))
|
|
continue
|
|
|
|
keep_videos = True
|
|
keep_audio = True
|
|
keep_subtitles = True
|
|
keep_chapters = True
|
|
|
|
if video_only or audio_only or subs_only or chapters_only:
|
|
keep_videos = video_only
|
|
keep_audio = audio_only
|
|
keep_subtitles = subs_only
|
|
keep_chapters = chapters_only
|
|
|
|
if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES:
|
|
console.log("Skipping subtitles - service does not support subtitle downloads")
|
|
keep_subtitles = False
|
|
s_lang = None
|
|
elif no_subs:
|
|
console.log("Skipped subtitles as --no-subs was used...")
|
|
keep_subtitles = False
|
|
s_lang = None
|
|
|
|
if no_video:
|
|
console.log("Skipped video as --no-video was used...")
|
|
keep_videos = False
|
|
v_lang = None
|
|
|
|
if no_audio:
|
|
console.log("Skipped audio as --no-audio was used...")
|
|
keep_audio = False
|
|
a_lang = None
|
|
|
|
if no_chapters:
|
|
console.log("Skipped chapters as --no-chapters was used...")
|
|
keep_chapters = False
|
|
|
|
if no_attachments:
|
|
console.log("Skipped attachments as --no-attachments was used...")
|
|
|
|
if no_proxy_download and any(service.session.proxies.values()):
|
|
console.log("Bypassing proxy for downloads as --no-proxy-download was used...")
|
|
elif proxy_download:
|
|
console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}")
|
|
|
|
missing_required: list[str] = []
|
|
for kind, required, available in (
|
|
(
|
|
"audio",
|
|
require_audio if keep_audio else [],
|
|
[a.language for a in title.tracks.audio if a.descriptive or not audio_description_only]
|
|
+ embedded_audio_langs(title.tracks.videos, keep_videos and not audio_description_only),
|
|
),
|
|
("video", require_video if keep_videos else [], [v.language for v in title.tracks.videos]),
|
|
(
|
|
"subtitle",
|
|
require_subs if keep_subtitles else [],
|
|
[
|
|
t.language
|
|
for t in title.tracks.subtitles
|
|
if (t.forced if forced_subs_only else forced_subs or not t.forced)
|
|
],
|
|
),
|
|
):
|
|
if missing := missing_required_langs(required, available, title.language, exact=exact_lang):
|
|
missing_required.append(f"Required {kind} language(s) not found: {', '.join(missing)}")
|
|
if missing_required:
|
|
skip_title(title, "; ".join(missing_required))
|
|
continue
|
|
|
|
if not keep_videos:
|
|
title.tracks.videos = []
|
|
if not keep_audio:
|
|
title.tracks.audio = []
|
|
if not keep_subtitles:
|
|
title.tracks.subtitles = []
|
|
if not keep_chapters:
|
|
title.tracks.chapters = []
|
|
|
|
with console.status("Selecting tracks...", spinner="dots"):
|
|
video_excl_r = resolve_sort_langs(video_excl, title.language, exact=exact_lang)
|
|
audio_excl_r = resolve_sort_langs(audio_excl, title.language, exact=exact_lang)
|
|
s_excl_r = resolve_sort_langs(s_lang_excl, title.language, exact=exact_lang)
|
|
fsl_excl_r = resolve_sort_langs(fsl_excl, title.language, exact=exact_lang)
|
|
|
|
if isinstance(title, (Movie, Episode)):
|
|
resolution: Optional[int]
|
|
color_range: Optional[Video.Range]
|
|
codec: Optional[Video.Codec]
|
|
if keep_videos and vcodec:
|
|
title.tracks.select_video(lambda x: x.codec in vcodec)
|
|
missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)]
|
|
for codec in missing_codecs:
|
|
self.log.warning(f"Skipping {codec.name} video tracks as none are available.")
|
|
if not title.tracks.videos:
|
|
skip_title(title, f"There's no {', '.join(c.name for c in vcodec)} Video Track...")
|
|
continue
|
|
elif keep_videos:
|
|
title.tracks.select_video(lambda x: x.codec != Video.Codec.LCEVC)
|
|
|
|
if keep_videos and generated_dv_bases:
|
|
title.tracks.videos = keep_generated_dv_bases(title.tracks.videos, generated_dv_bases)
|
|
if not any(v.range in generated_dv_bases for v in title.tracks.videos):
|
|
skip_title(
|
|
title, f"There's no {', '.join(r.name for r in generated_dv_bases)} Video Track..."
|
|
)
|
|
continue
|
|
|
|
if keep_videos and range_:
|
|
# Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks
|
|
if Video.Range.HYBRID not in range_:
|
|
dv_has_hdr10p = any(x.hdr10plus for x in title.tracks.videos)
|
|
title.tracks.select_video(lambda x: x.range in range_)
|
|
missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)]
|
|
for color_range in missing_ranges:
|
|
self.log.warning(f"Skipping {color_range.name} video tracks as none are available.")
|
|
if Video.Range.HDR10P in missing_ranges and dv_has_hdr10p:
|
|
self.log.warning("A DV track carries HDR10+: use -r DV or -r HYBRID.")
|
|
if not title.tracks.videos:
|
|
skip_title(title, f"There's no {', '.join(r.name for r in range_)} Video Track...")
|
|
continue
|
|
|
|
if keep_videos and vbitrate:
|
|
if any(r == Video.Range.HYBRID for r in range_):
|
|
# In HYBRID mode, only apply bitrate filter to non-DV tracks
|
|
# DV tracks are kept regardless since they're only used for RPU metadata
|
|
title.tracks.select_video(
|
|
lambda x: x.range == Video.Range.DV or bool(x.bitrate and x.bitrate // 1000 == vbitrate)
|
|
)
|
|
if not any(x.range != Video.Range.DV for x in title.tracks.videos):
|
|
skip_title(title, f"There's no {vbitrate}kbps Video Track...")
|
|
continue
|
|
else:
|
|
title.tracks.select_video(lambda x: bool(x.bitrate and x.bitrate // 1000 == vbitrate))
|
|
if not title.tracks.videos:
|
|
skip_title(title, f"There's no {vbitrate}kbps Video Track...")
|
|
continue
|
|
|
|
if keep_videos and vbitrate_min is not None and vbitrate_max is not None:
|
|
title.tracks.select_video(
|
|
lambda x: bool(x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max)
|
|
)
|
|
if not title.tracks.videos:
|
|
skip_title(title, f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...")
|
|
continue
|
|
|
|
if keep_videos and video_excl_r:
|
|
drop = matched_language_tags(
|
|
video_excl_r, [t.language for t in title.tracks.videos], exact_lang, every_variant=True
|
|
)
|
|
title.tracks.select_video(lambda x: str(x.language) not in drop)
|
|
if not title.tracks.videos:
|
|
skip_title(title, f"Every Video Track was excluded by -{', -'.join(video_excl_r)}...")
|
|
continue
|
|
|
|
effective_video_lang = v_lang or lang
|
|
video_languages = [lang for lang in effective_video_lang if lang != "best"]
|
|
processed_video_lang = video_languages
|
|
if keep_videos and video_languages and "all" not in video_languages:
|
|
processed_video_lang = resolve_sort_langs(video_languages, title.language, exact=exact_lang)
|
|
v_orig_token = orig_token(video_languages, title.language, exact_lang)
|
|
if "orig" in video_languages and not title.language:
|
|
self.log.warning(
|
|
"Original language not available for title, skipping 'orig' selection for video"
|
|
)
|
|
missing_v_langs = find_missing_langs(
|
|
processed_video_lang,
|
|
[v.language for v in title.tracks.videos],
|
|
exact=exact_lang,
|
|
)
|
|
if missing_v_langs:
|
|
missing_str = as_requested(missing_v_langs, v_orig_token)
|
|
if best_available:
|
|
remaining_v_langs = [tok for tok in processed_video_lang if tok not in missing_v_langs]
|
|
if remaining_v_langs:
|
|
self.log.warning(
|
|
f"{missing_str} not found in video tracks, "
|
|
f"continuing with: {as_requested(remaining_v_langs, v_orig_token)}"
|
|
)
|
|
processed_video_lang = remaining_v_langs
|
|
else:
|
|
skip_title(
|
|
title, f"{missing_str} not found in video tracks and no fallback available"
|
|
)
|
|
continue
|
|
else:
|
|
skip_title(title, missing_str + " not found in video tracks")
|
|
continue
|
|
title.tracks.videos = title.tracks.by_language(
|
|
title.tracks.videos, processed_video_lang, exact_match=exact_lang
|
|
)
|
|
if not title.tracks.videos:
|
|
# empty only when 'orig' was the sole request and did not resolve
|
|
skip_title(
|
|
title,
|
|
f"There's no {as_requested(processed_video_lang, v_orig_token) or 'orig'} Video Track...",
|
|
)
|
|
continue
|
|
video_requests = None if {"best", "all"} & set(effective_video_lang) else processed_video_lang
|
|
|
|
has_hybrid = any(r == Video.Range.HYBRID for r in range_)
|
|
non_hybrid_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID]
|
|
effective_quality = quality
|
|
if keep_videos and quality:
|
|
missing_resolutions = []
|
|
if has_hybrid:
|
|
hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos(
|
|
title.tracks.videos, non_hybrid_ranges
|
|
)
|
|
# only ranges that can actually be delivered count towards "present"
|
|
deliverable_pool = hybrid_candidate_tracks + [
|
|
v for v in non_hybrid_tracks if v.range in non_hybrid_ranges
|
|
]
|
|
for resolution in quality:
|
|
if any(at_resolution(v, resolution) for v in deliverable_pool):
|
|
continue
|
|
missing_resolutions.append(resolution)
|
|
|
|
if len(missing_resolutions) == len(quality) and best_available:
|
|
# every requested resolution missing: leave the pool untouched so the
|
|
# best-available hybrid pass below selects the top resolution instead
|
|
effective_quality = []
|
|
else:
|
|
hybrid_filter = title.tracks.select_hybrid(
|
|
hybrid_candidate_tracks, quality, worst=worst, generate_dv=generate_dv
|
|
)
|
|
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
|
|
|
|
if non_hybrid_ranges and non_hybrid_tracks:
|
|
non_hybrid_selected = [
|
|
v for v in non_hybrid_tracks if any(at_resolution(v, res) for res in quality)
|
|
]
|
|
title.tracks.videos = Tracks.merge_video_selections(
|
|
hybrid_selected, non_hybrid_selected
|
|
)
|
|
else:
|
|
title.tracks.videos = hybrid_selected
|
|
else:
|
|
pre_quality_videos = list(title.tracks.videos)
|
|
title.tracks.by_resolutions(quality)
|
|
|
|
for resolution in quality:
|
|
if any(at_resolution(v, resolution) for v in title.tracks.videos):
|
|
continue
|
|
missing_resolutions.append(resolution)
|
|
|
|
if len(missing_resolutions) == len(quality) and best_available:
|
|
# every requested resolution missing: fall back to the best available
|
|
title.tracks.videos = pre_quality_videos
|
|
effective_quality = []
|
|
|
|
if missing_resolutions:
|
|
res_list = ""
|
|
if len(missing_resolutions) > 1:
|
|
res_list = ", ".join([f"{x}p" for x in missing_resolutions[:-1]]) + " or "
|
|
res_list = f"{res_list}{missing_resolutions[-1]}p"
|
|
plural = "s" if len(missing_resolutions) > 1 else ""
|
|
|
|
if best_available:
|
|
self.log.warning(
|
|
f"There's no {res_list} Video Track{plural}, continuing with available qualities..."
|
|
)
|
|
else:
|
|
skip_title(title, f"There's no {res_list} Video Track{plural}...")
|
|
continue
|
|
|
|
pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else []
|
|
if keep_videos and has_hybrid:
|
|
hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos(
|
|
title.tracks.videos, non_hybrid_ranges
|
|
)
|
|
|
|
hybrid_quality = effective_quality
|
|
if not hybrid_quality:
|
|
height_pool = (
|
|
[
|
|
*select_hybrid_bases(hybrid_candidate_tracks),
|
|
*(v for v in hybrid_candidate_tracks if v.finished_hybrid),
|
|
]
|
|
if worst
|
|
else hybrid_candidate_tracks
|
|
)
|
|
base_heights = [v.height for v in height_pool if v.height]
|
|
hybrid_quality = [(min if worst else max)(base_heights)] if base_heights else []
|
|
hybrid_selected = []
|
|
if hybrid_quality:
|
|
hybrid_filter = title.tracks.select_hybrid(
|
|
hybrid_candidate_tracks, hybrid_quality, worst=worst, generate_dv=generate_dv
|
|
)
|
|
hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks))
|
|
for res in hybrid_quality:
|
|
if not any(at_resolution(v, res) for v in hybrid_selected):
|
|
self.log.warning(f"Skipping {res}p: nothing at that resolution can make a hybrid")
|
|
|
|
non_hybrid_selected = []
|
|
if non_hybrid_ranges and non_hybrid_tracks:
|
|
non_hybrid_selected = select_videos(
|
|
non_hybrid_tracks,
|
|
effective_quality,
|
|
non_hybrid_ranges,
|
|
vcodec,
|
|
video_requests,
|
|
exact_lang,
|
|
worst,
|
|
)
|
|
|
|
title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected)
|
|
Tracks.flag_hybrid_ingredients(hybrid_selected, non_hybrid_selected)
|
|
elif keep_videos:
|
|
title.tracks.videos = select_videos(
|
|
title.tracks.videos, effective_quality, range_, vcodec, video_requests, exact_lang, worst
|
|
)
|
|
|
|
if keep_videos and any(r == Video.Range.HYBRID for r in range_):
|
|
gate_failure = hybrid_gate_failure(title.tracks.videos, generate_dv)
|
|
if not gate_failure and not any(
|
|
v.range == Video.Range.DV or v.native_hybrid for v in title.tracks.videos
|
|
):
|
|
self.log.info("No Dolby Vision track: Dolby Vision will be generated")
|
|
|
|
if gate_failure:
|
|
other_ranges: list[Video.Range] = [r for r in range_ if r != Video.Range.HYBRID]
|
|
if best_available and other_ranges:
|
|
self.log.warning(
|
|
f"{gate_failure}; continuing with {', '.join(r.name for r in other_ranges)}"
|
|
)
|
|
range_ = other_ranges
|
|
title.tracks.videos = select_videos(
|
|
pre_hybrid_videos,
|
|
effective_quality,
|
|
other_ranges,
|
|
vcodec,
|
|
video_requests,
|
|
exact_lang,
|
|
worst,
|
|
)
|
|
else:
|
|
skip_title(title, gate_failure)
|
|
continue
|
|
|
|
fsl = resolve_sort_langs(forced_s_lang, title.language, exact=exact_lang)
|
|
if keep_subtitles and s_excl_r:
|
|
# applies to forced subs too: an excluded language is unwanted in any form
|
|
drop = matched_language_tags(
|
|
s_excl_r, [t.language for t in title.tracks.subtitles], exact_lang, every_variant=True
|
|
)
|
|
title.tracks.select_subtitles(lambda x: str(x.language) not in drop)
|
|
if keep_subtitles and s_lang and "all" not in s_lang:
|
|
s_orig_token = orig_token(s_lang, title.language, exact_lang)
|
|
s_lang = resolve_sort_langs(s_lang, title.language, exact=exact_lang)
|
|
# the pool selection draws from: forced tracks count only when -fs keeps them
|
|
missing_langs = find_missing_langs(
|
|
s_lang,
|
|
[sub.language for sub in title.tracks.subtitles if forced_subs or not sub.forced],
|
|
exact=exact_lang,
|
|
)
|
|
if missing_langs:
|
|
remaining = [tok for tok in s_lang if tok not in missing_langs]
|
|
tail = (
|
|
f"continuing with: {as_requested(remaining, s_orig_token)}"
|
|
if remaining
|
|
else "continuing without subtitles"
|
|
)
|
|
self.log.warning(
|
|
f"{as_requested(missing_langs, s_orig_token)} not found in subtitle tracks, {tail}"
|
|
)
|
|
|
|
if s_lang and title.tracks.subtitles:
|
|
plain_tags = plain_subtitle_tags(s_lang, title.tracks.subtitles, exact_lang)
|
|
forced_tags = matched_language_tags(
|
|
[*s_lang, *fsl], [x.language for x in title.tracks.subtitles if x.forced], exact_lang
|
|
)
|
|
title.tracks.select_subtitles(
|
|
lambda x: str(x.language) in (forced_tags if x.forced else plain_tags)
|
|
)
|
|
|
|
if keep_subtitles:
|
|
# fsl, not forced_s_lang: unresolvable orig must keep all forced, not drop them
|
|
if fsl:
|
|
fsl_tags = matched_language_tags(
|
|
fsl, [x.language for x in title.tracks.subtitles if x.forced], exact_lang
|
|
)
|
|
title.tracks.select_subtitles(lambda x: not x.forced or str(x.language) in fsl_tags)
|
|
if not any(x.forced for x in title.tracks.subtitles):
|
|
self.log.warning(
|
|
f"No forced subtitle tracks matched --forced-s-lang ({', '.join(forced_s_lang)})"
|
|
)
|
|
elif not forced_subs:
|
|
title.tracks.select_subtitles(lambda x: not x.forced)
|
|
if fsl_excl_r:
|
|
drop = matched_language_tags(
|
|
fsl_excl_r,
|
|
[t.language for t in title.tracks.subtitles if t.forced],
|
|
exact_lang,
|
|
every_variant=True,
|
|
)
|
|
title.tracks.select_subtitles(lambda x: not (x.forced and str(x.language) in drop))
|
|
if forced_subs_only:
|
|
title.tracks.select_subtitles(lambda x: x.forced)
|
|
if not title.tracks.subtitles:
|
|
skip_title(
|
|
title,
|
|
"There's no forced subtitle track for the selected languages.",
|
|
kind_only=True,
|
|
)
|
|
continue
|
|
|
|
if keep_audio and audio_description_only:
|
|
title.tracks.select_audio(lambda x: x.descriptive)
|
|
if not title.tracks.audio:
|
|
skip_title(title, "There's no audio description track.", kind_only=True)
|
|
continue
|
|
# might have no audio tracks if part of the video, e.g. transport stream hls
|
|
if keep_audio and len(title.tracks.audio) > 0:
|
|
if not audio_description:
|
|
title.tracks.select_audio(lambda x: not x.descriptive)
|
|
if not title.tracks.audio:
|
|
skip_title(title, "Audio description only, use --audio-description...")
|
|
continue
|
|
if acodec:
|
|
title.tracks.select_audio(lambda x: x.codec in acodec)
|
|
if not title.tracks.audio:
|
|
codec_names = ", ".join(c.name for c in acodec)
|
|
skip_title(title, f"No audio tracks matching codecs: {codec_names}")
|
|
continue
|
|
if channels:
|
|
title.tracks.select_audio(
|
|
lambda x: bool(
|
|
x.channels and math.ceil(Audio.channel_total(x.channels)) == math.ceil(channels)
|
|
)
|
|
)
|
|
if not title.tracks.audio:
|
|
skip_title(title, f"There's no {channels} Audio Track...")
|
|
continue
|
|
if no_atmos:
|
|
title.tracks.audio = [x for x in title.tracks.audio if not x.atmos]
|
|
if not title.tracks.audio:
|
|
skip_title(title, "No non-Atmos audio tracks available...")
|
|
continue
|
|
if abitrate:
|
|
title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate)
|
|
if not title.tracks.audio:
|
|
skip_title(title, f"There's no {abitrate}kbps Audio Track...")
|
|
continue
|
|
if abitrate_min is not None and abitrate_max is not None:
|
|
title.tracks.select_audio(
|
|
lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max
|
|
)
|
|
if not title.tracks.audio:
|
|
skip_title(title, f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...")
|
|
continue
|
|
if audio_excl_r:
|
|
drop = matched_language_tags(
|
|
audio_excl_r, [t.language for t in title.tracks.audio], exact_lang, every_variant=True
|
|
)
|
|
title.tracks.select_audio(lambda x: str(x.language) not in drop)
|
|
if not title.tracks.audio:
|
|
skip_title(title, f"Every Audio Track was excluded by -{', -'.join(audio_excl_r)}...")
|
|
continue
|
|
audio_languages = a_lang or lang
|
|
if audio_languages:
|
|
processed_lang = resolve_sort_langs(audio_languages, title.language, exact=exact_lang)
|
|
a_orig_token = orig_token(audio_languages, title.language, exact_lang)
|
|
if "orig" in audio_languages and not title.language:
|
|
self.log.warning("Original language not available for title, skipping 'orig' selection")
|
|
|
|
embedded_langs = embedded_audio_langs(
|
|
title.tracks.videos, keep_videos and not audio_description_only
|
|
)
|
|
audio_noun = "audio description tracks" if audio_description_only else "audio tracks"
|
|
|
|
if not any(tok in processed_lang for tok in ("best", "all")):
|
|
missing_a_langs = find_missing_langs(
|
|
processed_lang,
|
|
[a.language for a in title.tracks.audio] + embedded_langs,
|
|
exact=exact_lang,
|
|
)
|
|
if missing_a_langs:
|
|
missing_str = as_requested(missing_a_langs, a_orig_token)
|
|
if best_available:
|
|
remaining = [tok for tok in processed_lang if tok not in missing_a_langs]
|
|
if remaining:
|
|
self.log.warning(
|
|
f"{missing_str} not found in {audio_noun}, "
|
|
f"continuing with: {as_requested(remaining, a_orig_token)}"
|
|
)
|
|
processed_lang = remaining
|
|
else:
|
|
skip_title(
|
|
title, f"{missing_str} not found in {audio_noun} and no fallback available"
|
|
)
|
|
continue
|
|
else:
|
|
skip_title(title, f"{missing_str} not found in {audio_noun}")
|
|
continue
|
|
|
|
title.tracks.audio = select_best_audio(
|
|
title.tracks.audio, processed_lang, acodec, audio_description, exact_lang
|
|
)
|
|
if not title.tracks.audio and not embedded_langs:
|
|
# empty when 'orig' was the sole request, did not resolve, and the
|
|
# video carries no audio of its own to fall back on
|
|
skip_title(
|
|
title,
|
|
f"There's no {as_requested(processed_lang, a_orig_token) or 'orig'} Audio Track...",
|
|
)
|
|
continue
|
|
|
|
kept_tracks = []
|
|
if keep_videos:
|
|
kept_tracks.extend(title.tracks.videos)
|
|
if keep_audio:
|
|
kept_tracks.extend(title.tracks.audio)
|
|
if keep_subtitles:
|
|
kept_tracks.extend(title.tracks.subtitles)
|
|
if keep_chapters:
|
|
kept_tracks.extend(title.tracks.chapters)
|
|
if not no_attachments:
|
|
kept_tracks.extend(title.tracks.attachments)
|
|
|
|
title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url)
|
|
if not (title.tracks.videos or title.tracks.audio or title.tracks.subtitles or title.tracks.chapters):
|
|
skip_title(title, "There's no track left for the selected options.", kind_only=True)
|
|
continue
|
|
|
|
selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True)
|
|
|
|
log_event(
|
|
"track_select",
|
|
level="INFO",
|
|
message=(
|
|
f"Selected {len(title.tracks.videos)} video, {len(title.tracks.audio)} audio, "
|
|
f"{len(title.tracks.subtitles)} subtitle track(s)"
|
|
),
|
|
context={
|
|
"videos": [
|
|
f"{v.codec} {v.range} {v.height}p {v.bitrate}bps {v.language}" for v in title.tracks.videos
|
|
],
|
|
"audio": [
|
|
f"{a.codec} {a.language} {getattr(a, 'channels', None)}ch {a.bitrate}bps"
|
|
for a in title.tracks.audio
|
|
],
|
|
"subtitles": [f"{s.codec} {s.language}" for s in title.tracks.subtitles],
|
|
},
|
|
)
|
|
|
|
if progress_sink is not None:
|
|
from unshackle.core.api.progress import build_job_progress_callables
|
|
|
|
tracks_progress_callables = build_job_progress_callables(
|
|
list(title.tracks), tracks_progress_callables, progress_sink
|
|
)
|
|
|
|
drm_pending = [track for track in title.tracks if getattr(track, "needs_drm_loading", False)]
|
|
if drm_pending:
|
|
with ThreadPoolExecutor(min(16, len(drm_pending))) as drm_pool:
|
|
list(drm_pool.map(lambda track: track.load_drm_if_needed(service), drm_pending))
|
|
|
|
download_table = Table.grid()
|
|
download_table.add_row(selected_tracks)
|
|
|
|
def prepare_drm_for(track: AnyTrack) -> partial:
|
|
prepare = partial(
|
|
partial(self.prepare_drm, table=download_table),
|
|
track=track,
|
|
title=title,
|
|
certificate=partial(service.get_widevine_service_certificate, title=title, track=track),
|
|
licence=partial(self.service_licence, service, title=title, track=track),
|
|
clearkey_licence=partial(service.get_clearkey_license, title=title, track=track),
|
|
cdm_only=cdm_only,
|
|
vaults_only=vaults_only,
|
|
export=export_path,
|
|
service_session=service.session,
|
|
)
|
|
if title_all_drm:
|
|
return partial(self.licence_all_drm, prepare, track, best_available, force=bool(cdm_only))
|
|
return prepare
|
|
|
|
server_cdm_type = None
|
|
if self._remote_service and self._remote_service.server_cdm:
|
|
server_cdm_type = self._remote_service.server_cdm_type
|
|
self.cdm = cdm_type_stub(server_cdm_type)
|
|
from unshackle.core.import_service import ImportService
|
|
|
|
if isinstance(self._remote_service, ImportService):
|
|
self.log.info("Using keys from the export; no CDM required")
|
|
else:
|
|
self.log.info(f"Using server CDM ({server_cdm_type.title()}); no local CDM required")
|
|
|
|
video_tracks = title.tracks.videos
|
|
if video_tracks and not server_cdm_type:
|
|
highest_quality = max((track.height for track in video_tracks if track.height), default=0)
|
|
if highest_quality > 0:
|
|
if is_widevine_cdm(self.cdm):
|
|
quality_based_cdm = self.get_cdm(
|
|
self.service, self.profile, drm="widevine", quality=highest_quality
|
|
)
|
|
if quality_based_cdm and quality_based_cdm != self.cdm:
|
|
self.log.debug(
|
|
f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks"
|
|
)
|
|
self.cdm = quality_based_cdm
|
|
elif is_playready_cdm(self.cdm):
|
|
quality_based_cdm = self.get_cdm(
|
|
self.service, self.profile, drm="playready", quality=highest_quality
|
|
)
|
|
if quality_based_cdm and quality_based_cdm != self.cdm:
|
|
self.log.debug(
|
|
f"Pre-selecting PlayReady CDM based on highest quality {highest_quality}p across all video tracks"
|
|
)
|
|
self.cdm = quality_based_cdm
|
|
|
|
title_all_drm = all_drm and not server_cdm_type
|
|
if all_drm and server_cdm_type and DOWNLOAD_ALL_DRM.is_set():
|
|
DOWNLOAD_ALL_DRM.clear()
|
|
self.log.warning("--all-drm is ignored: the server CDM licenses this session with one DRM system")
|
|
if title_all_drm:
|
|
heights = {track.height for track in title.tracks.videos if track.height} or {None}
|
|
for system, matches in (("Widevine", is_widevine_cdm), ("PlayReady", is_playready_cdm)):
|
|
system_cdms = [self.cdm_for(system.lower(), height) for height in heights]
|
|
if not all(system_cdm and matches(system_cdm) for system_cdm in system_cdms):
|
|
hint = (
|
|
"drop --cdm, which selects one device"
|
|
if self.cdm_override
|
|
else f"set cdm.{self.service}.{system.lower()}"
|
|
)
|
|
raise click.ClickException(f"--all-drm needs a {system} CDM for {self.service}: {hint}")
|
|
if is_remote_cdm(system_cdms[0]) and system not in self.all_drm_warned:
|
|
self.all_drm_warned.add(system)
|
|
self.log.warning(f"The {system} CDM is remote and can answer with cached content keys")
|
|
|
|
if self._remote_service:
|
|
self.take_server_keys(self._remote_service, title, bool(cdm_only))
|
|
|
|
if sample and not skip_dl:
|
|
whole_files = [
|
|
track
|
|
for track in (*title.tracks.videos, *title.tracks.audio)
|
|
if not track.can_sample(track.session or service.session)
|
|
]
|
|
if whole_files:
|
|
raise click.ClickException(
|
|
f"--sample cannot cut {len(whole_files)} track(s) of {title}: "
|
|
f"{self.service} gives each one as one file with no segment index."
|
|
)
|
|
|
|
dl_start_time = time.time()
|
|
|
|
try:
|
|
with SyncLive(Padding(download_table, (1, 5)), console=console, refresh_per_second=20):
|
|
|
|
def download_track(track: AnyTrack, i: int) -> None:
|
|
track.download(
|
|
session=track.session or service.session,
|
|
no_proxy_download=no_proxy_download,
|
|
proxy_download=proxy_download,
|
|
prepare_drm=prepare_drm_for(track),
|
|
cdm=self.cdm,
|
|
max_workers=workers,
|
|
adaptive_workers=adaptive_workers,
|
|
download_processes=download_processes,
|
|
progress=tracks_progress_callables[i],
|
|
)
|
|
if (
|
|
track.path
|
|
and service.decrypts_samples(track)
|
|
and not DOWNLOAD_CANCELLED.is_set()
|
|
and not DOWNLOAD_LICENCE_ONLY.is_set()
|
|
):
|
|
progress = tracks_progress_callables[i]
|
|
progress(downloaded="Decrypting", completed=0, total=None)
|
|
cbcs.decrypt_file(
|
|
track.path, lambda _l, entry: service.get_sample_decrypter(track, entry.index)
|
|
)
|
|
assert_fragments_decrypted(track.path)
|
|
progress(downloaded="Decrypted", completed=100, total=100)
|
|
# DRM-free and HLS-ClearKey tracks never reach prepare_drm, so export here.
|
|
# drm=None on purpose: licensed tracks already recorded their DRM/keys
|
|
# in prepare_drm, and write_export merges via setdefault.
|
|
if export_path:
|
|
self.write_export(export_path, title, track)
|
|
|
|
def on_subtitle_skipped(track: Subtitle) -> None:
|
|
lang = str(track.language)
|
|
self.log.warning(f"Subtitle {lang} failed to download, skipping it.")
|
|
self.skipped_subtitles.append(SkippedSubtitle(id=track.id, language=lang, title=str(title)))
|
|
try:
|
|
title.tracks.subtitles.remove(track)
|
|
except ValueError:
|
|
self.log.debug(f"Skipped subtitle {track.id} was already absent from the track list.")
|
|
|
|
skipped_before = len(self.skipped_subtitles)
|
|
grow_session_pool(service.session, downloads * (workers or default_max_workers()))
|
|
download_tracks_in_passes(
|
|
title.tracks,
|
|
downloads,
|
|
download_track,
|
|
skip_subtitle_errors=skip_subtitle_errors,
|
|
on_subtitle_skipped=on_subtitle_skipped,
|
|
)
|
|
|
|
for attachment in title.tracks.attachments:
|
|
attachment.download(
|
|
attachment.session or service.session,
|
|
no_proxy_download=no_proxy_download,
|
|
proxy_download=proxy_download,
|
|
)
|
|
|
|
if (
|
|
len(self.skipped_subtitles) > skipped_before
|
|
and not title.tracks.videos
|
|
and not title.tracks.audio
|
|
and not title.tracks.subtitles
|
|
):
|
|
self.log.warning(
|
|
f"{title}: all subtitles were skipped and no video or audio was "
|
|
"downloaded - nothing was produced for this title."
|
|
)
|
|
|
|
except KeyboardInterrupt:
|
|
console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5)))
|
|
log_event(
|
|
"download_tracks",
|
|
level="WARNING",
|
|
service=self.service,
|
|
message="Download cancelled by user",
|
|
context={"title": str(title)},
|
|
)
|
|
self.wait_vault_writes()
|
|
if export_path:
|
|
self.log_export_summary()
|
|
return
|
|
except Exception as e: # noqa
|
|
# Reported and swallowed (no re-raise) so the CLI exits cleanly; keep it so the
|
|
# API worker sees the title failed rather than completing with no output.
|
|
self.download_error = e
|
|
console.print(Padding(Group(*self.failure_lines(e)), (1, 5)))
|
|
if not isinstance(e, click.ClickException) and not retries_exhausted(e):
|
|
console.print_exception()
|
|
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"download_tracks",
|
|
e,
|
|
service=self.service,
|
|
context={
|
|
"title": str(title),
|
|
"error_type": type(e).__name__,
|
|
"tracks_count": len(title.tracks),
|
|
"returncode": getattr(e, "returncode", None),
|
|
},
|
|
)
|
|
|
|
dispatch(
|
|
"failure",
|
|
"file",
|
|
build_context(
|
|
title,
|
|
service=self.service,
|
|
ids=self.post_script_ids(),
|
|
error=f"{type(e).__name__}: {e}",
|
|
),
|
|
postscript,
|
|
)
|
|
self.wait_vault_writes()
|
|
if export_path:
|
|
self.log_export_summary()
|
|
return
|
|
|
|
self.wait_vault_writes()
|
|
if export_path:
|
|
self.log_export_title(title)
|
|
|
|
if skip_dl:
|
|
console.log("Skipped downloads as --skip-dl was used...")
|
|
else:
|
|
dl_time = time_elapsed_since(dl_start_time)
|
|
console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5)))
|
|
|
|
subtitle_output_mode = config.subtitle.get("output_mode", "mux")
|
|
sidecar_format = config.subtitle.get("sidecar_format", "srt")
|
|
skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio)
|
|
sidecar_subtitles: list[Subtitle] = []
|
|
sidecar_original_paths: dict[str, Path] = {}
|
|
sidecar_files: dict[Optional[Path], list[Path]] = {}
|
|
if subtitle_output_mode in ("sidecar", "both") and not no_mux:
|
|
sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()]
|
|
if sidecar_format == "original":
|
|
config.directories.temp.mkdir(parents=True, exist_ok=True)
|
|
for subtitle in sidecar_subtitles:
|
|
assert subtitle.path is not None
|
|
original_path = (
|
|
config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}"
|
|
)
|
|
shutil.copy2(subtitle.path, original_path)
|
|
sidecar_original_paths[subtitle.id] = original_path
|
|
|
|
set_subtitle_look(title.tracks.subtitles, ass_style, title.tracks.videos)
|
|
with console.status("Converting Subtitles..."):
|
|
sub_conversions: dict[tuple[str, str], int] = {}
|
|
for subtitle in title.tracks.subtitles:
|
|
if subtitle.codec == Subtitle.Codec.SCC:
|
|
scc_dst = convert_scc(subtitle, sub_format).name
|
|
sub_conversions[("SCC", scc_dst)] = sub_conversions.get(("SCC", scc_dst), 0) + 1
|
|
if sub_format == "original":
|
|
continue
|
|
if sub_format:
|
|
if subtitle.codec != sub_format:
|
|
src = getattr(subtitle.codec, "name", str(subtitle.codec))
|
|
dst = getattr(sub_format, "name", str(sub_format))
|
|
subtitle.convert(sub_format, forced=True)
|
|
sub_conversions[(src, dst)] = sub_conversions.get((src, dst), 0) + 1
|
|
elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang:
|
|
# MKV does not support TTML, VTT is the next best option
|
|
src = getattr(subtitle.codec, "name", str(subtitle.codec))
|
|
subtitle.convert(Subtitle.Codec.WebVTT)
|
|
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
|
|
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
|
|
)
|
|
if sample:
|
|
for subtitle in title.tracks.subtitles:
|
|
if subtitle.path and subtitle.path.exists():
|
|
subtitle.trim(*sample)
|
|
for (src, dst), count in sub_conversions.items():
|
|
log_event(
|
|
"subtitle_convert",
|
|
level="INFO",
|
|
message=f"Converted {src}->{dst} x{count}",
|
|
context={"from": src, "to": dst, "count": count},
|
|
)
|
|
|
|
with console.status("Checking Subtitles for Fonts..."):
|
|
font_names: list[str] = []
|
|
for subtitle in [] if no_attachments else title.tracks.subtitles:
|
|
if subtitle.codec in (Subtitle.Codec.SubStationAlpha, Subtitle.Codec.SubStationAlphav4):
|
|
font_names.extend(Subtitle.extract_fonts(subtitle.path.read_text("utf8")))
|
|
|
|
font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title)
|
|
|
|
if font_count:
|
|
self.log.info(f"Attached {font_count} fonts for the Subtitles")
|
|
|
|
if missing_fonts and sys.platform != "win32":
|
|
self.suggest_missing_fonts(missing_fonts)
|
|
|
|
# Handle DRM decryption BEFORE repacking (must decrypt first!)
|
|
drm_tracks = [track for track in title.tracks if track.drm]
|
|
if drm_tracks:
|
|
with console.status("Decrypting tracks..."):
|
|
has_decrypted = False
|
|
for track in drm_tracks:
|
|
drm = track.get_drm_for_cdm(self.cdm)
|
|
if drm and hasattr(drm, "decrypt"):
|
|
self.decrypt_verified(drm, track.path, prepare_drm_for(track))
|
|
if not isinstance(drm, MonaLisa):
|
|
# MonaLisa decrypts per segment; its decrypt() here is a no-op
|
|
assert_fragments_decrypted(track.path)
|
|
has_decrypted = True
|
|
events.emit(events.Types.TRACK_REPACKED, track=track)
|
|
else:
|
|
self.log.warning(
|
|
f"No matching DRM found for track {track} with CDM type {type(self.cdm).__name__}"
|
|
)
|
|
if has_decrypted:
|
|
self.log.info("Decrypted tracks")
|
|
|
|
if (
|
|
not no_subs
|
|
and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES)
|
|
and not video_only
|
|
and not no_video
|
|
):
|
|
for video_track_n, video_track in enumerate(title.tracks.videos):
|
|
has_manifest_cc = bool(getattr(video_track, "closed_captions", None))
|
|
has_eia_cc = (
|
|
not has_manifest_cc
|
|
and not title.tracks.subtitles
|
|
and any(
|
|
x.get("codec_name", "").startswith("eia_")
|
|
for x in ffprobe(video_track.path).get("streams", [])
|
|
)
|
|
)
|
|
if not has_manifest_cc and not has_eia_cc:
|
|
continue
|
|
|
|
with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."):
|
|
try:
|
|
cc_lang = (
|
|
Language.get(video_track.closed_captions[0]["language"])
|
|
if has_manifest_cc and video_track.closed_captions[0].get("language")
|
|
else title.language or video_track.language
|
|
)
|
|
track_id = f"ccextractor-{video_track.id}"
|
|
cc = video_track.ccextractor(
|
|
track_id=track_id,
|
|
out_path=config.directories.temp
|
|
/ config.filenames.subtitle.format(id=track_id, language=cc_lang),
|
|
language=cc_lang,
|
|
original=False,
|
|
)
|
|
if cc:
|
|
cc.cc = True
|
|
title.tracks.add(cc)
|
|
self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
|
|
else:
|
|
self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
|
|
except EnvironmentError:
|
|
self.log.error(
|
|
"Cannot extract Closed Captions as the ccextractor executable was not found..."
|
|
)
|
|
break
|
|
|
|
# Now repack the decrypted tracks
|
|
if progress_sink and any(getattr(t, "needs_repack", False) for t in title.tracks):
|
|
progress_sink(
|
|
{"phase": "repackaging", "progress": 92.0, "status": "downloading", "active_tracks": []}
|
|
)
|
|
with console.status("Repackaging tracks with FFMPEG..."):
|
|
has_repacked = False
|
|
# Video tracks whose VUI was folded into the repack pass; skipped in the VUI loop below.
|
|
vui_folded = set()
|
|
for track in title.tracks:
|
|
if track.needs_repack:
|
|
bsf = track.vui_bsf() if isinstance(track, Video) else None
|
|
if track.repackage(bsf_v=bsf):
|
|
vui_folded.add(id(track))
|
|
has_repacked = True
|
|
events.emit(events.Types.TRACK_REPACKED, track=track)
|
|
if has_encrypted_sample_entry(track.path):
|
|
self.log.warning(f"Track {track.id} still has an encrypted sample entry after repack")
|
|
if has_repacked:
|
|
self.log.info("Repacked one or more tracks with FFMPEG")
|
|
|
|
with console.status("Normalizing video VUI..."):
|
|
for track in title.tracks.videos:
|
|
if id(track) in vui_folded:
|
|
continue
|
|
try:
|
|
track.normalize_vui()
|
|
except Exception as e: # noqa: BLE001
|
|
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
|
|
|
|
remux_music_tracks(title, self.log, sample)
|
|
|
|
muxed_paths = []
|
|
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
|
|
tagged_paths: set[Path] = set()
|
|
append_audio_codec_suffix = True
|
|
skip_mux = mux_skipped(no_mux, title)
|
|
kept_audio = [] if skip_mux else take_unmuxable_audio(title.tracks, self.log)
|
|
if kept_audio and not title.tracks:
|
|
title.tracks.audio = kept_audio
|
|
kept_audio = []
|
|
skip_mux = True
|
|
|
|
if skip_mux:
|
|
for track in title.tracks:
|
|
if track.path and track.path.exists():
|
|
muxed_paths.append(track.path)
|
|
elif isinstance(title, (Movie, Episode)):
|
|
progress = Progress(
|
|
TextColumn("[progress.description]{task.description}"),
|
|
SpinnerColumn(finished_text=""),
|
|
GradientPulseBarColumn(),
|
|
"•",
|
|
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
|
|
"•",
|
|
TextColumn("{task.fields[downloaded]}"),
|
|
console=console,
|
|
)
|
|
|
|
merge_audio = (
|
|
(not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True)
|
|
)
|
|
# When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec.
|
|
append_audio_codec_suffix = not merge_audio
|
|
|
|
merge_video = merge_video if merge_video is not None else config.muxing.get("merge_video", False)
|
|
|
|
multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = []
|
|
mux_futures: list[futures.Future[tuple[Path, int, list[str]]]] = []
|
|
# Track hybrid-processing outputs explicitly so we can always clean them up,
|
|
# even if muxing fails early (e.g. SystemExit) before the normal delete loop.
|
|
hybrid_temp_paths: list[Path] = []
|
|
hybrid_base_ids: set[str] = set()
|
|
container_paths: dict[str, Path] = {}
|
|
|
|
def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks:
|
|
task_tracks = Tracks()
|
|
task_tracks.videos = list(base_tracks.videos)
|
|
task_tracks.audio = audio_tracks
|
|
task_tracks.subtitles = list(base_tracks.subtitles)
|
|
task_tracks.chapters = base_tracks.chapters
|
|
task_tracks.attachments = list(base_tracks.attachments)
|
|
return task_tracks
|
|
|
|
audio_expected = not video_only and not no_audio
|
|
|
|
def unique_mux_output(task_tracks: Tracks, index: int) -> Optional[Path]:
|
|
src: Optional[Path] = None
|
|
ext = ".muxed.mkv"
|
|
if task_tracks.videos:
|
|
src = task_tracks.videos[0].path
|
|
elif task_tracks.audio:
|
|
src, ext = task_tracks.audio[0].path, ".muxed.mka"
|
|
elif task_tracks.subtitles:
|
|
src, ext = task_tracks.subtitles[0].path, ".muxed.mks"
|
|
if src is None:
|
|
return None
|
|
base = src.with_suffix(ext)
|
|
return base.with_name(f"{base.stem}.{index}{base.suffix}")
|
|
|
|
def mux_one(mux_index: int, task_id: TaskID, task_tracks: Tracks) -> tuple[Path, int, list[str]]:
|
|
progress.start_task(task_id)
|
|
muxed_path, return_code, errors = task_tracks.mux(
|
|
str(title),
|
|
progress=partial(progress.update, task_id=task_id),
|
|
delete=False,
|
|
audio_expected=audio_expected,
|
|
title_language=title.language,
|
|
skip_subtitles=skip_subtitle_mux,
|
|
output_path=unique_mux_output(task_tracks, mux_index),
|
|
)
|
|
if return_code < 2:
|
|
tags.tag_file(
|
|
muxed_path,
|
|
title,
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
self.anime_hint(title),
|
|
session=service.session,
|
|
)
|
|
tagged_paths.add(muxed_path)
|
|
return muxed_path, return_code, errors
|
|
|
|
def enqueue_mux_tasks(task_description: str, base_tracks: Tracks, hybrid: bool = False) -> None:
|
|
pool = hybrid_mux_pool if hybrid else mux_pool
|
|
if merge_audio or not base_tracks.audio:
|
|
task_id = progress.add_task(
|
|
f"{task_description}...", total=None, start=False, downloaded=""
|
|
)
|
|
multiplex_tasks.append((task_id, base_tracks, None))
|
|
mux_futures.append(pool.submit(mux_one, len(multiplex_tasks), task_id, base_tracks))
|
|
return
|
|
|
|
audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {}
|
|
for audio_track in base_tracks.audio:
|
|
audio_by_codec.setdefault(audio_track.codec, []).append(audio_track)
|
|
|
|
for audio_codec, codec_audio_tracks in audio_by_codec.items():
|
|
description = task_description
|
|
if audio_codec:
|
|
description = f"{task_description} {audio_codec.name}"
|
|
|
|
task_id = progress.add_task(f"{description}...", total=None, start=False, downloaded="")
|
|
task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks)
|
|
multiplex_tasks.append((task_id, task_tracks, audio_codec))
|
|
mux_futures.append(pool.submit(mux_one, len(multiplex_tasks), task_id, task_tracks))
|
|
|
|
def mux_video_group(video_tracks: list[Optional[Video]]) -> None:
|
|
for video_track in video_tracks:
|
|
if video_track and video_track.dv_compatible_bitstream:
|
|
container = apply_dv_fixup(video_track, keep_source=video_track.id in hybrid_base_ids)
|
|
if container:
|
|
container_paths[video_track.id] = container
|
|
hybrid_temp_paths.append(container)
|
|
|
|
task_description = "Multiplexing"
|
|
# All tracks in a merged group share height/range/codec, so describe from the first.
|
|
head = next((v for v in video_tracks if v), None)
|
|
if head:
|
|
if len(quality) > 1:
|
|
task_description += f" {standard_resolution(head.width, head.height)}p"
|
|
if len(range_) > 1:
|
|
task_description += f" {head.range.name}"
|
|
if len(vcodec) > 1:
|
|
assert head.codec is not None
|
|
task_description += f" {head.codec.name}"
|
|
|
|
task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
|
|
if head:
|
|
task_tracks.videos = [v for v in video_tracks if v]
|
|
|
|
enqueue_mux_tasks(task_description, task_tracks)
|
|
|
|
def group_file_size(video_tracks: Sequence[Optional[Video]]) -> int:
|
|
return sum(v.path.stat().st_size for v in video_tracks if v and v.path and v.path.exists())
|
|
|
|
if progress_sink:
|
|
progress_sink(
|
|
{"phase": "muxing", "progress": 96.0, "status": "downloading", "active_tracks": []}
|
|
)
|
|
|
|
try:
|
|
with SyncLive(Padding(progress, (0, 5, 1, 5)), console=console, refresh_per_second=20):
|
|
mux_failed = False
|
|
mux_concurrency = max(1, int(config.muxing.get("concurrency", 4)))
|
|
|
|
with (
|
|
ThreadPoolExecutor(mux_concurrency) as mux_pool,
|
|
ThreadPoolExecutor(mux_concurrency) as extra_mux_pool,
|
|
):
|
|
hybrid_mux_pool = mux_pool if mux_concurrency == 1 else extra_mux_pool
|
|
|
|
if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos:
|
|
self.log.info("Processing Hybrid HDR10+DV tracks...")
|
|
|
|
original_videos = list(title.tracks.videos)
|
|
|
|
standalone_videos = [v for v in original_videos if not v.hybrid_base_only]
|
|
standalone_groups: list[list[Optional[Video]]] = [
|
|
list(g) for g in group_videos_by_variant(standalone_videos, merge=merge_video)
|
|
]
|
|
hybrid_bases, matching_dv = hybrid_plan(
|
|
[v for v in original_videos if v.hybrid_ingredient], generate_dv
|
|
)
|
|
hybrid_base_ids.update(v.id for v in hybrid_bases)
|
|
hybrids_made = 0
|
|
for video in (v for v in original_videos if v.native_hybrid):
|
|
self.log.info(
|
|
f"The {video.height}p {video.range_label} track is already the hybrid"
|
|
)
|
|
hybrids_made += 1
|
|
|
|
for group in sorted(standalone_groups, key=group_file_size, reverse=True):
|
|
mux_video_group(group)
|
|
|
|
for hdr10_track in hybrid_bases:
|
|
resolution = hdr10_track.height or 0
|
|
container = container_paths.get(hdr10_track.id)
|
|
resolution_tracks = hybrid_source_tracks(hdr10_track, matching_dv, container)
|
|
|
|
hybrid_filename = f"HDR10-DV-{resolution}p.hevc"
|
|
hybrid_output_path = config.directories.temp / hybrid_filename
|
|
hybrid_temp_paths.append(hybrid_output_path)
|
|
|
|
step_error = hybrid_step_error(resolution_tracks, self.service, hybrid_filename)
|
|
if step_error:
|
|
# the other files of the title are already queued for muxing
|
|
self.log.error(f"Hybrid {resolution}p failed, skipping it: {step_error}")
|
|
continue
|
|
hybrids_made += 1
|
|
|
|
if container:
|
|
try:
|
|
container.unlink(missing_ok=True)
|
|
except PermissionError:
|
|
pass # still in hybrid_temp_paths, so the final cleanup tries again
|
|
|
|
task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p"
|
|
task_tracks = (
|
|
Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
|
|
)
|
|
|
|
hybrid_track = deepcopy(hdr10_track)
|
|
hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}"
|
|
hybrid_track.path = hybrid_output_path
|
|
hybrid_track.range = Video.Range.DV
|
|
hybrid_track.needs_duration_fix = True
|
|
title.tracks.add(hybrid_track)
|
|
task_tracks.videos = [hybrid_track]
|
|
|
|
enqueue_mux_tasks(task_description, task_tracks, hybrid=True)
|
|
|
|
no_hybrid = missing_hybrid_error(hybrid_bases, hybrids_made)
|
|
if no_hybrid:
|
|
raise click.ClickException(no_hybrid)
|
|
|
|
console.print()
|
|
else:
|
|
groups: list[list[Optional[Video]]] = [
|
|
list(g) for g in group_videos_by_variant(title.tracks.videos, merge=merge_video)
|
|
] or [[None]]
|
|
for group in sorted(groups, key=group_file_size, reverse=True):
|
|
mux_video_group(group)
|
|
|
|
mux_results = [f.result() for f in mux_futures]
|
|
for (task_id, task_tracks, audio_codec), (muxed_path, return_code, errors) in zip(
|
|
multiplex_tasks, mux_results
|
|
):
|
|
muxed_paths.append(muxed_path)
|
|
muxed_audio_codecs[muxed_path] = audio_codec
|
|
if return_code >= 2:
|
|
self.log.error(f"Failed to Mux video to Matroska file ({return_code}):")
|
|
elif return_code == 1 or errors:
|
|
self.log.warning("mkvmerge had at least one warning or error, continuing anyway...")
|
|
for line in errors:
|
|
if line.startswith("#GUI#error"):
|
|
self.log.error(line)
|
|
else:
|
|
self.log.warning(line)
|
|
if return_code >= 2:
|
|
mux_failed = True
|
|
if mux_failed:
|
|
sys.exit(1)
|
|
|
|
if sidecar_subtitles and not no_mux:
|
|
sidecar_targets: list[tuple[Optional[Path], str, Path]] = []
|
|
for muxed_path in muxed_paths or [None]:
|
|
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
|
|
if media_info:
|
|
base_filename = title.get_filename(media_info, show_service=not no_source)
|
|
if sample:
|
|
base_filename = sample_name(base_filename, title)
|
|
else:
|
|
base_filename = str(title)
|
|
|
|
sidecar_dir = self.output_dir or config.directories.downloads
|
|
if (
|
|
not no_folder
|
|
and media_info
|
|
and (
|
|
isinstance(title, (Episode, Song))
|
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
|
)
|
|
):
|
|
sidecar_dir /= title.get_filename(
|
|
media_info, show_service=not no_source, folder=True
|
|
)
|
|
sidecar_dir /= sample_dir
|
|
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
|
|
|
|
with console.status("Saving subtitle sidecar files..."):
|
|
first_path, first_base, first_dir = sidecar_targets[0]
|
|
first_dir.mkdir(parents=True, exist_ok=True)
|
|
created = self.output_subtitle_sidecars(
|
|
sidecar_subtitles,
|
|
first_base,
|
|
first_dir,
|
|
sidecar_format,
|
|
original_paths=sidecar_original_paths or None,
|
|
)
|
|
if created:
|
|
sidecar_files[first_path] = list(created)
|
|
self.log.info(f"Saved {len(created)} sidecar subtitle files")
|
|
|
|
for sidecar_path, base_filename, sidecar_dir in sidecar_targets[1:]:
|
|
sidecar_dir.mkdir(parents=True, exist_ok=True)
|
|
copies = []
|
|
for source in created:
|
|
target = sidecar_dir / f"{base_filename}{source.name[len(first_base) :]}"
|
|
if target != source:
|
|
shutil.copy2(source, target)
|
|
copies.append(target)
|
|
if copies:
|
|
sidecar_files[sidecar_path] = copies
|
|
self.log.info(f"Saved {len(copies)} sidecar subtitle files")
|
|
|
|
for track in title.tracks:
|
|
track.delete()
|
|
|
|
for attachment in title.tracks.attachments:
|
|
attachment.delete()
|
|
for temp_path in sidecar_original_paths.values():
|
|
temp_path.unlink(missing_ok=True)
|
|
finally:
|
|
# Hybrid() writes a temp HEVC per resolution, so delete each one here.
|
|
for temp_path in hybrid_temp_paths:
|
|
try:
|
|
temp_path.unlink(missing_ok=True)
|
|
except PermissionError:
|
|
self.log.warning(f"Failed to delete temp file (in use?): {temp_path}")
|
|
for leftover in (
|
|
"HDR10-DV.hevc",
|
|
"HDR10.hevc",
|
|
"DV.hevc",
|
|
"RPU.bin",
|
|
"RPU_UNT.bin",
|
|
"RPU_L5.bin",
|
|
"RPU_L6.bin",
|
|
"L5.json",
|
|
"HDR10Plus.json",
|
|
):
|
|
leftover_path = config.directories.temp / leftover
|
|
try:
|
|
leftover_path.unlink(missing_ok=True)
|
|
except PermissionError:
|
|
self.log.warning(f"Failed to delete temp file (in use?): {leftover_path}")
|
|
|
|
else:
|
|
muxed_paths.extend(track.path for track in title.tracks.audio if track.path and track.path.exists())
|
|
|
|
if skip_mux:
|
|
final_dir = self.output_dir or config.directories.downloads
|
|
if not no_folder and (
|
|
isinstance(title, (Episode, Song))
|
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
|
):
|
|
sample_track = (
|
|
title.tracks.videos[0]
|
|
if title.tracks.videos
|
|
else (
|
|
title.tracks.audio[0]
|
|
if title.tracks.audio
|
|
else (title.tracks.subtitles[0] if title.tracks.subtitles else None)
|
|
)
|
|
)
|
|
if sample_track and sample_track.path:
|
|
media_info = MediaInfo.parse(sample_track.path)
|
|
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
|
|
|
final_dir /= sample_dir
|
|
final_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
for track_path in muxed_paths:
|
|
fix_data_reference_indexes(track_path)
|
|
media_info = MediaInfo.parse(track_path)
|
|
base_filename = title.get_filename(media_info, show_service=not no_source)
|
|
if sample:
|
|
base_filename = sample_name(base_filename, title)
|
|
|
|
track = next((t for t in title.tracks if t.path == track_path), None)
|
|
if track:
|
|
ext = no_mux_extension(track, track_path, media_info)
|
|
final_path = final_dir / f"{base_filename}{track_file_suffix(track)}{ext}"
|
|
else:
|
|
final_path = final_dir / f"{base_filename}{track_path.suffix}"
|
|
|
|
shutil.move(track_path, final_path)
|
|
self.completed_files.append(final_path)
|
|
self.log.debug(f"Saved: {final_path.name}")
|
|
else:
|
|
used_final_paths: set[Path] = set()
|
|
for muxed_path in muxed_paths:
|
|
media_info = MediaInfo.parse(muxed_path)
|
|
final_dir = self.output_dir or config.directories.downloads
|
|
final_filename = title.get_filename(media_info, show_service=not no_source)
|
|
if sample:
|
|
final_filename = sample_name(final_filename, title)
|
|
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
|
|
|
|
if not no_folder and (
|
|
isinstance(title, (Episode, Song))
|
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
|
):
|
|
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
|
final_dir /= sample_dir
|
|
|
|
final_dir.mkdir(parents=True, exist_ok=True)
|
|
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
|
sep = config.get_template_separator(template_type(title))
|
|
|
|
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
|
|
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
|
|
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
|
|
|
if final_path in used_final_paths:
|
|
i = 2
|
|
while final_path in used_final_paths:
|
|
final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}"
|
|
i += 1
|
|
|
|
try:
|
|
os.replace(muxed_path, final_path)
|
|
except OSError:
|
|
if final_path.exists():
|
|
final_path.unlink()
|
|
shutil.move(muxed_path, final_path)
|
|
used_final_paths.add(final_path)
|
|
self.completed_files.append(final_path)
|
|
for kept in kept_audio:
|
|
assert kept.path is not None
|
|
kept_path = final_path.with_name(
|
|
f"{final_path.stem}{track_file_suffix(kept)}{kept.path.suffix}"
|
|
)
|
|
shutil.move(kept.path, kept_path)
|
|
self.completed_files.append(kept_path)
|
|
sidecar_files.setdefault(muxed_path, []).append(kept_path)
|
|
self.log.info(f"Saved {kept_path.name}: no muxer carries Audio Vivid")
|
|
kept_audio = []
|
|
if muxed_path not in tagged_paths:
|
|
tags.tag_file(
|
|
final_path,
|
|
title,
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
self.tvdb_id,
|
|
self.anilist_id,
|
|
self.anime_hint(title),
|
|
session=service.session,
|
|
)
|
|
if include_sample:
|
|
self.write_sample(final_path, title, media_info)
|
|
if isinstance(title, Song) and title.synced_lyrics and title.lyrics_suffix:
|
|
lyrics_path = final_path.with_suffix(title.lyrics_suffix)
|
|
lyrics_path.write_text(f"{title.synced_lyrics}\n", encoding="utf-8")
|
|
sidecar_files.setdefault(muxed_path, []).append(lyrics_path)
|
|
self.log.info(f"Saved lyrics to {lyrics_path.name}")
|
|
|
|
post_script_context = build_context(
|
|
title,
|
|
media_info,
|
|
filepath=final_path,
|
|
sidecars=sidecar_files.get(muxed_path, []),
|
|
service=self.service,
|
|
ids=self.post_script_ids(),
|
|
)
|
|
dispatch("success", "file", post_script_context, postscript)
|
|
post_script_last.setdefault(post_script_group(title), {})[final_path.parent] = (
|
|
post_script_context
|
|
)
|
|
post_script_sample = post_script_context
|
|
if final_path.parent not in post_script_folders:
|
|
post_script_folders.append(final_path.parent)
|
|
|
|
if not skip_mux:
|
|
settle_post_script_group(title)
|
|
|
|
title_dl_time = time_elapsed_since(dl_start_time)
|
|
downloaded_label = "Track" if isinstance(title, Song) else "Title"
|
|
console.print(
|
|
Padding(
|
|
f":tada: {downloaded_label} downloaded in [progress.elapsed]{title_dl_time}[/]!",
|
|
(0, 5, 1, 5),
|
|
)
|
|
)
|
|
|
|
if progress_sink:
|
|
progress_sink({"output_files": [str(p) for p in self.completed_files]})
|
|
|
|
if not hasattr(service, "close"):
|
|
cookie_file = self.get_cookie_path(self.service, self.profile)
|
|
if cookie_file:
|
|
self.save_cookies(cookie_file, service.session.cookies)
|
|
|
|
if hasattr(service, "close"):
|
|
service.close()
|
|
|
|
if post_script_folders and post_script_sample:
|
|
run_context = dict.fromkeys(post_script_sample, "")
|
|
for folder in post_script_folders:
|
|
run_context["folder"] = str(folder)
|
|
dispatch("success", "run", run_context, postscript)
|
|
|
|
self.wait_vault_writes()
|
|
if export_path:
|
|
self.log_export_summary()
|
|
if self.skipped_titles:
|
|
if len(self.skipped_titles) == wanted_total:
|
|
self.stop_reason = f"Skipped all {wanted_total} titles"
|
|
self.log.error(self.stop_reason)
|
|
sys.exit(1)
|
|
skipped = ", ".join(t["title"] for t in self.skipped_titles)
|
|
self.log.warning(f"Skipped {len(self.skipped_titles)} of {wanted_total} titles: {skipped}")
|
|
dl_time = time_elapsed_since(start_time)
|
|
|
|
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
|
|
|
def write_sample(self, source: Path, title: Title_T, media_info: MediaInfo) -> None:
|
|
"""Cut a 60 second sample from a finished file into a ``Sample`` folder next to it.
|
|
|
|
The sample starts 2 minutes in. A file shorter than 3 minutes gives its middle 60 seconds.
|
|
mkvmerge cuts only Matroska: an MP4 (3D video, or audio Matroska cannot carry) gets no sample.
|
|
"""
|
|
if source.suffix.lower() != ".mkv":
|
|
self.log.warning(f"No sample for {source.name}: mkvmerge cuts only Matroska files")
|
|
return
|
|
duration = float(media_info.general_tracks[0].duration or 0) / 1000
|
|
start: float
|
|
start, end = DEFAULT_SAMPLE
|
|
length = end - start
|
|
if duration < end:
|
|
start = max(0.0, (duration - length) / 2)
|
|
output = source.parent / SAMPLE_DIR / f"{sample_name(source.stem, title)}{source.suffix}"
|
|
output.parent.mkdir(exist_ok=True)
|
|
returncode = subprocess.run(
|
|
[
|
|
str(binaries.MKVToolNix),
|
|
"-q",
|
|
"-o",
|
|
str(output),
|
|
"--split",
|
|
f"parts:{start:.3f}s-{start + length:.3f}s",
|
|
source,
|
|
]
|
|
).returncode
|
|
if returncode > 1 or not output.exists():
|
|
self.log.warning(f"mkvmerge wrote no sample for {source.name}")
|
|
|
|
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
|
|
"""Whether metadata lookups for this title should prefer AniList."""
|
|
per_title = getattr(title, "anime", None)
|
|
return self.service_anime if per_title is None else bool(per_title)
|
|
|
|
def daily_hint(self, title: Optional[Title_T] = None) -> bool:
|
|
"""Whether this title is daily content named by air date."""
|
|
per_title = getattr(title, "daily", None)
|
|
return self.daily or (self.service_daily if per_title is None else bool(per_title))
|
|
|
|
def fill_air_dates(self, titles: Series, tvdb_id: Optional[int] = None) -> None:
|
|
"""Fill in missing episode air dates from TVDB.
|
|
|
|
Additive only: an air date the service already set is kept.
|
|
"""
|
|
tvdb = providers.get_provider("tvdb")
|
|
if not tvdb:
|
|
self.log.debug("Air dates need a tvdb_api_key in your config, skipping.")
|
|
return
|
|
|
|
if not tvdb_id:
|
|
self.log.debug("Could not resolve a TVDB ID, skipping air dates.")
|
|
return
|
|
|
|
wanted = [t for t in titles if self.daily_hint(t)]
|
|
if not wanted:
|
|
return
|
|
|
|
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
|
|
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
|
|
episodes = tvdb.get_episodes(tvdb_id, source_order) # type: ignore[attr-defined]
|
|
if not episodes:
|
|
self.log.debug("TVDB listed no episodes for series %s, skipping air dates.", tvdb_id)
|
|
return
|
|
|
|
aired_by_key: dict[tuple[int, int], str] = {}
|
|
for episode in episodes:
|
|
season, number = parse_int(episode.get("seasonNumber")), parse_int(episode.get("number"))
|
|
aired = episode.get("aired")
|
|
if season is None or number is None or not aired:
|
|
continue
|
|
aired_by_key[(season, number)] = str(aired)
|
|
|
|
# TVDB carries placeholder schedule dates for unaired episodes. Stamping one would
|
|
# name the file with a date the episode never aired on.
|
|
limit = date.today() + timedelta(days=1)
|
|
|
|
filled = 0
|
|
for title in wanted:
|
|
if title.air_date is not None:
|
|
continue
|
|
aired = aired_by_key.get((title.season, title.number))
|
|
if not aired:
|
|
continue
|
|
try:
|
|
air_date = date.fromisoformat(aired[:10])
|
|
except ValueError:
|
|
continue
|
|
if air_date.year < 1970 or air_date > limit:
|
|
continue
|
|
title.air_date = air_date
|
|
filled += 1
|
|
|
|
if filled:
|
|
self.log.info("Filled air dates for %d episode(s) from TVDB (ID %s).", filled, tvdb_id)
|
|
|
|
def fill_absolute_numbers(self, titles: Series, tvdb_id: Optional[int] = None) -> None:
|
|
"""Fill in missing absolute episode numbers from TVDB's absolute order.
|
|
|
|
Additive only: season and number are never written, and an absolute number the
|
|
service already set is kept.
|
|
"""
|
|
tvdb = providers.get_provider("tvdb")
|
|
if not tvdb:
|
|
self.log.debug("Absolute episode numbers need a tvdb_api_key in your config, skipping.")
|
|
return
|
|
|
|
if not tvdb_id:
|
|
self.log.debug("Could not resolve a TVDB ID, skipping absolute episode numbers.")
|
|
return
|
|
|
|
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
|
|
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
|
|
if source_order == "absolute":
|
|
# the service already numbers in absolute order; get_order_map would return the
|
|
# empty same-order map, but here the episode numbers are their own fill
|
|
mapping = {key: (*key, None) for key in keys}
|
|
else:
|
|
mapping = tvdb.get_order_map(tvdb_id, "absolute", source_order) # type: ignore[attr-defined]
|
|
if not mapping:
|
|
self.log.debug("TVDB has no absolute order for series %s, skipping absolute episode numbers.", tvdb_id)
|
|
return
|
|
|
|
filled = 0
|
|
for title in titles:
|
|
if title.absolute is not None:
|
|
continue
|
|
entry = mapping.get((title.season, title.number))
|
|
if not entry:
|
|
continue
|
|
title.absolute = entry[1]
|
|
filled += 1
|
|
|
|
if filled:
|
|
self.log.info("Filled absolute numbers for %d episode(s) from TVDB (ID %s).", filled, tvdb_id)
|
|
|
|
def apply_continuous_episodes(self, titles: Series) -> Series:
|
|
"""Number each season on from the last episode of the season before it.
|
|
|
|
Season 0 and episode 0 keep their numbers, and a season that already starts past the season before it is kept.
|
|
The Series is rebuilt, as it does not re-sort an episode whose number changed.
|
|
"""
|
|
last = 0
|
|
renumbered = 0
|
|
for season, group in groupby(titles, key=attrgetter("season")):
|
|
episodes = [t for t in group if t.number]
|
|
if not season or not episodes:
|
|
continue
|
|
if min(t.number for t in episodes) <= last:
|
|
for t in episodes:
|
|
t.number += last
|
|
renumbered += len(episodes)
|
|
last = max(t.number for t in episodes)
|
|
self.log.info("Numbered %d episode(s) on from the season before.", renumbered)
|
|
return type(titles)(list(titles))
|
|
|
|
def apply_tvdb_order(
|
|
self,
|
|
titles: Series,
|
|
title_cacher: Any = None,
|
|
cache_title_id: Optional[str] = None,
|
|
cache_region: Optional[str] = None,
|
|
cache_account_hash: Optional[str] = None,
|
|
) -> Series:
|
|
"""Renumber episodes from the service's own numbering into a TVDB season order."""
|
|
tvdb = providers.get_provider("tvdb")
|
|
if not tvdb:
|
|
self.log.warning("--tvdb-order needs a tvdb_api_key in your config, skipping renumber.")
|
|
return titles
|
|
|
|
tvdb_id = self.tvdb_id
|
|
if not tvdb_id:
|
|
show = titles[0]
|
|
result = providers.resolve_by_ids(
|
|
self.tmdb_id,
|
|
self.imdb_id,
|
|
None,
|
|
title=show.title,
|
|
year=show.year,
|
|
kind="tv",
|
|
title_cacher=title_cacher,
|
|
cache_title_id=cache_title_id,
|
|
cache_region=cache_region,
|
|
cache_account_hash=cache_account_hash,
|
|
)
|
|
tvdb_id = result.external_ids.tvdb_id if result else None
|
|
if not tvdb_id:
|
|
self.log.warning("Could not resolve a TVDB ID for %r, skipping %s renumber.", str(titles), self.tvdb_order)
|
|
return titles
|
|
self.tvdb_id = tvdb_id # keep the mux tag on the series the renumbering used
|
|
|
|
# parts of one episode share a (season, number) slot; without the dedupe the
|
|
# collision check below reads them as several episodes clashing on one slot
|
|
keys = list(dict.fromkeys((t.season, t.number) for t in titles))
|
|
source_order = tvdb.detect_order(tvdb_id, keys) # type: ignore[attr-defined]
|
|
if source_order == self.tvdb_order:
|
|
self.log.info("%s already lists this series in TVDB %s order.", self.service, self.tvdb_order)
|
|
return titles
|
|
|
|
mapping = tvdb.get_order_map(tvdb_id, self.tvdb_order, source_order) # type: ignore[attr-defined]
|
|
if not mapping:
|
|
return titles
|
|
|
|
# an unmapped episode keeps its own numbering and can land on a remapped episode's slot
|
|
renumbered = [mapping.get(key, (*key, None))[:2] for key in keys]
|
|
unmapped = sum(1 for key in keys if key not in mapping)
|
|
collisions = [slot for slot, count in Counter(renumbered).items() if count > 1]
|
|
if collisions:
|
|
self.log.error(
|
|
"TVDB's %s order misses %d episode(s) and puts two in %d slot(s), keeping the %s numbering of %s",
|
|
self.tvdb_order,
|
|
unmapped,
|
|
len(collisions),
|
|
source_order,
|
|
self.service,
|
|
)
|
|
return titles
|
|
|
|
for title in titles:
|
|
entry = mapping.get((title.season, title.number))
|
|
if not entry:
|
|
continue
|
|
title.season, title.number, tvdb_name = entry
|
|
if tvdb_name and not title.name:
|
|
title.name = tvdb_name
|
|
|
|
if unmapped:
|
|
self.log.warning(
|
|
"%d episode(s) are not in TVDB's %s order and keep their %s numbering.",
|
|
unmapped,
|
|
self.tvdb_order,
|
|
source_order,
|
|
)
|
|
self.log.info("Renumbered episodes from TVDB %s to %s order (ID %s).", source_order, self.tvdb_order, tvdb_id)
|
|
|
|
# Series is a SortedKeyList; renumbering in place does not re-sort it
|
|
return type(titles)(list(titles))
|
|
|
|
@staticmethod
|
|
def title_to_meta(title: Title_T) -> dict[str, Any]:
|
|
"""Capture the title fields needed to rebuild a Title for output naming on import."""
|
|
from datetime import date
|
|
|
|
from unshackle.core.titles import Episode, Movie
|
|
|
|
meta: dict[str, Any] = {
|
|
"id": str(title.id),
|
|
"language": str(title.language) if getattr(title, "language", None) else None,
|
|
}
|
|
if isinstance(title, Episode):
|
|
meta.update(
|
|
type="episode",
|
|
series_title=title.title,
|
|
season=title.season,
|
|
number=title.number,
|
|
name=title.name,
|
|
year=title.year,
|
|
air_date=title.air_date.isoformat() if isinstance(title.air_date, date) else title.air_date,
|
|
)
|
|
# conditional, so meta for a title without them is unchanged
|
|
if title.part is not None:
|
|
meta["part"] = title.part
|
|
if title.absolute is not None:
|
|
meta["absolute"] = title.absolute
|
|
if getattr(title, "daily", None) is not None:
|
|
meta["daily"] = title.daily
|
|
if title.sport:
|
|
meta["sport"] = True
|
|
elif isinstance(title, Movie):
|
|
meta.update(type="movie", name=title.name, year=title.year)
|
|
else:
|
|
meta.update(type="movie", name=str(title))
|
|
if getattr(title, "anime", None) is not None:
|
|
meta["anime"] = title.anime
|
|
return meta
|
|
|
|
def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any = None) -> None:
|
|
"""Write a shareable mediaexport file usable by ``unshackle import`` and unidl.
|
|
|
|
Carries no HTTP session or cookies. The export records the region (country code)
|
|
only when the export used ``--proxy``, as an import geofence. DRM init data and
|
|
content keys live once per title. ``drm`` may be None (DRM-free track) or a DRM
|
|
system without ``to_dict``/``content_keys`` (e.g. ClearKey) - the export still
|
|
records the track, manifest, chapter and attachment info. unshackle's full track
|
|
dicts, title meta and manifest snapshot go under ``x-unshackle`` for its own importer.
|
|
"""
|
|
namer: Optional[ExportNamer] = getattr(self, "export_namer", None)
|
|
snapshots = getattr(self, "export_snapshots", None)
|
|
if snapshots is None:
|
|
snapshots = self.export_snapshots = {}
|
|
if str(title.id) not in snapshots:
|
|
try:
|
|
fresh = capture(title.tracks)
|
|
snapshots[str(title.id)] = fresh if any(fresh.values()) else None
|
|
except Exception as e:
|
|
self.log.warning(f"Not saving the manifest of {title.id} in {export}: {e!r}")
|
|
snapshots[str(title.id)] = None
|
|
with self.EXPORT_LOCK:
|
|
if namer and namer.path:
|
|
export = namer.path
|
|
try:
|
|
doc = mediaexport.read(export) if export.is_file() else mediaexport.Document(service_tag=self.service)
|
|
except mediaexport.ExportError as e:
|
|
# a download mid-licensing must not die over the file it exports to
|
|
self.log.warning(f"Not exporting {title.id}: {export} is not a usable export ({e})")
|
|
return
|
|
doc.generator.setdefault("app", "unshackle")
|
|
doc.generator.setdefault("version", __version__)
|
|
if not doc.region and getattr(self, "proxy_requested", False):
|
|
doc.region = getattr(getattr(self, "export_service", None), "current_region", None) or ""
|
|
|
|
entry = doc.get(str(title.id))
|
|
if entry is None:
|
|
entry = self.export_entry(title)
|
|
doc.add(entry)
|
|
|
|
tracks_map = entry.ext("unshackle").setdefault("tracks", {})
|
|
track_dict = tracks_map.setdefault(str(track.id), track.to_dict())
|
|
unwanted = (track.data.get("hls") or {}).get("unwanted_segments")
|
|
if unwanted:
|
|
track_dict["unwanted_segments"] = unwanted
|
|
elif unwanted is not None:
|
|
track_dict.pop("unwanted_segments", None)
|
|
if saved := snapshots[str(title.id)]:
|
|
entry.ext("unshackle")["snapshot"] = saved
|
|
# the row names the track's KIDs, so an import gives the track only their keys
|
|
track_kids = [kid.hex for kid in own_kids(drm)] if drm is not None else []
|
|
for row in entry.tracks:
|
|
if row.get("id") == str(track.id):
|
|
row["selected"] = True
|
|
if track_kids:
|
|
row["kids"] = list(dict.fromkeys([*(row.get("kids") or []), *track_kids]))
|
|
|
|
if drm is not None:
|
|
if hasattr(drm, "to_dict"):
|
|
d = drm.to_dict()
|
|
own = entry.ext("unshackle").setdefault("drm", [])
|
|
if d not in own:
|
|
own.append(d)
|
|
system = str(d.get("system", "")).lower().replace("cenc", "")
|
|
pssh = str(d.get("pssh_b64") or "")
|
|
if system and all(x.pssh != pssh or x.system != system for x in entry.drm):
|
|
entry.drm.append(mediaexport.Drm(system, pssh))
|
|
content_keys = getattr(drm, "content_keys", None) or {}
|
|
pool = doc.key_pool()
|
|
for kid, key in content_keys.items():
|
|
# a KID that already holds a different key must not be overwritten:
|
|
# the reader rejects a file that disagrees with itself
|
|
if pool.setdefault(kid.hex, key.lower()) != key.lower():
|
|
self.log.warning(
|
|
f"KID {kid.hex} already has a different key in {export}, keeping the existing one"
|
|
)
|
|
continue
|
|
try:
|
|
entry.add_key(kid.hex, key)
|
|
except mediaexport.ExportError as e:
|
|
# a key the format cannot hold (not 16 bytes of hex) costs the export that key,
|
|
# never the download
|
|
self.log.warning(f"Not exporting the key for KID {kid.hex}: {e}")
|
|
|
|
try:
|
|
mediaexport.write(export, doc)
|
|
except (mediaexport.ExportError, OSError) as e:
|
|
self.log.warning(f"Could not write export {export}: {e}")
|
|
return
|
|
if namer:
|
|
self.rename_export(namer, export, title, track)
|
|
|
|
def rename_export(self, namer: ExportNamer, export: Path, title: Title_T, track: AnyTrack) -> None:
|
|
"""Give the run's export a name that tells what it holds, after each write to it.
|
|
|
|
The file on disk is complete after each write, so a run that stops early still leaves
|
|
a usable file with a name that matches what it holds.
|
|
"""
|
|
first = namer.path is None
|
|
namer.add(title, track)
|
|
try:
|
|
namer.path = move_export(export, namer.name())
|
|
except OSError as e:
|
|
self.log.warning(f"Could not rename export {export}: {e}")
|
|
namer.path = export
|
|
if first:
|
|
self.log.info(f"Exporting to {namer.path.name}")
|
|
elif namer.path != export:
|
|
self.log.debug(f"Renamed the export to {namer.path.name}")
|
|
|
|
def log_export_title(self, title: Title_T) -> None:
|
|
"""Log what the export holds for one title: its track counts and its keys."""
|
|
namer: Optional[ExportNamer] = getattr(self, "export_namer", None)
|
|
entry = None
|
|
if namer and namer.path:
|
|
try:
|
|
entry = mediaexport.read(namer.path).get(str(title.id))
|
|
except (mediaexport.ExportError, OSError) as e:
|
|
self.log.warning(f"Could not read export {namer.path}: {e}")
|
|
return
|
|
if entry is None:
|
|
self.log.warning(f"Exported nothing for {title_label(title)}")
|
|
return
|
|
selected = [r.get("type") for r in entry.tracks if r.get("selected")]
|
|
self.log.info(
|
|
f"Exported {title_label(title)}: {selected.count('video')} video, {selected.count('audio')} audio, "
|
|
f"{plural(selected.count('subtitle'), 'subtitle')}, {plural(len(entry.keys), 'key')}"
|
|
)
|
|
|
|
def log_export_summary(self) -> None:
|
|
"""Log where the run's export is and what it holds, and that it must stay private."""
|
|
namer: Optional[ExportNamer] = getattr(self, "export_namer", None)
|
|
if not namer or not namer.path:
|
|
self.log.warning("Nothing was exported")
|
|
return
|
|
try:
|
|
doc = mediaexport.read(namer.path)
|
|
except (mediaexport.ExportError, OSError) as e:
|
|
self.log.warning(f"Could not read export {namer.path}: {e}")
|
|
return
|
|
keys = len(doc.key_pool())
|
|
self.log.info(f"Saved the export to {namer.path}: {plural(len(doc.titles), 'title')}, {plural(keys, 'key')}")
|
|
if keys:
|
|
self.log.warning("The export holds content keys: keep it private")
|
|
|
|
def export_entry(self, title: Title_T) -> mediaexport.Entry:
|
|
"""A mediaexport entry for ``title``: manifests, chapters, a small track list, no keys yet."""
|
|
meta = self.title_to_meta(title)
|
|
all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles]
|
|
primary_url = title.tracks.manifest_url or next(
|
|
(str(t.url) for t in all_tracks if t.descriptor != Video.Descriptor.URL), ""
|
|
)
|
|
manifest_type = next(
|
|
(t.descriptor.name.lower() for t in all_tracks if t.descriptor != Video.Descriptor.URL), ""
|
|
)
|
|
session = getattr(getattr(self, "export_service", None), "session", None)
|
|
# mediaexport drops Cookie and Authorization itself, on write and on read
|
|
headers = dict(getattr(session, "headers", None) or {})
|
|
manifests = (
|
|
[
|
|
mediaexport.Manifest(
|
|
primary_url,
|
|
manifest_type,
|
|
headers,
|
|
role="primary",
|
|
content_periods=self.content_periods(all_tracks, primary_url),
|
|
)
|
|
]
|
|
if primary_url
|
|
else []
|
|
)
|
|
# the whole URL identifies a manifest: two profiles on one endpoint are two manifests
|
|
seen = {primary_url}
|
|
for t in all_tracks:
|
|
url = str(t.url)
|
|
if t.descriptor in (Video.Descriptor.DASH, Video.Descriptor.ISM) and url not in seen:
|
|
seen.add(url)
|
|
manifests.append(
|
|
mediaexport.Manifest(
|
|
url,
|
|
t.descriptor.name.lower(),
|
|
role="extra",
|
|
content_periods=self.content_periods(all_tracks, url),
|
|
)
|
|
)
|
|
|
|
rows = []
|
|
for t in all_tracks:
|
|
codec = getattr(t, "codec", None)
|
|
row: dict[str, Any] = {
|
|
"id": str(t.id),
|
|
"type": t.__class__.__name__.lower(),
|
|
# a subtitle side-load names its file format so another tool can fetch it
|
|
"codec": (codec.value if isinstance(t, Subtitle) else codec.name).lower() if codec else "",
|
|
"language": str(t.language) if t.language else "",
|
|
"bitrate": int(bitrate) if (bitrate := getattr(t, "bitrate", None)) else None,
|
|
}
|
|
if isinstance(t, Video):
|
|
row.update(width=t.width, height=t.height, range=t.range.name.lower() if t.range else "")
|
|
if t.stereo:
|
|
row["stereo"] = True
|
|
elif isinstance(t, Audio):
|
|
row.update(channels=str(t.channels) if t.channels else "", atmos=bool(t.joc), descriptive=t.descriptive)
|
|
elif isinstance(t, Subtitle):
|
|
row.update(sdh=t.sdh, forced=t.forced, cc=t.cc)
|
|
if t.descriptor == Video.Descriptor.URL:
|
|
row["url"] = str(t.url)
|
|
rows.append({k: v for k, v in row.items() if v not in (None, "")})
|
|
|
|
return mediaexport.Entry(
|
|
id=str(title.id),
|
|
kind=meta.get("type", "movie"),
|
|
title=meta.get("name") or "",
|
|
series=meta.get("series_title") or "",
|
|
season=meta.get("season"),
|
|
episode=meta.get("number"),
|
|
year=meta.get("year"),
|
|
language=meta.get("language") or "",
|
|
artist=meta.get("artist") or "",
|
|
album=meta.get("album") or "",
|
|
track_number=meta.get("track"),
|
|
manifests=manifests,
|
|
chapters=[
|
|
{"start_ms": mediaexport.ts_ms(c.timestamp), "title": c.name or ""}
|
|
for c in (title.tracks.chapters or [])
|
|
],
|
|
tracks=rows,
|
|
extensions={
|
|
"x-unshackle": {
|
|
"meta": meta,
|
|
"tracks": {},
|
|
"attachments": [a.to_dict() for a in (title.tracks.attachments or []) if a.url],
|
|
}
|
|
},
|
|
)
|
|
|
|
@staticmethod
|
|
def content_periods(tracks: Sequence[Track], url: str) -> list[str]:
|
|
"""The ids of the DASH periods a service kept in the manifest at ``url``, empty when it filtered none.
|
|
|
|
An import re-fetches the manifest without the service, so its period filter must travel in the export.
|
|
Empty also when a kept period has no id, since an id allow-list cannot name it.
|
|
"""
|
|
dash = next(
|
|
(d for t in tracks if (d := t.data.get("dash") or {}).get("filtered_period_ids") and str(t.url) == url),
|
|
None,
|
|
)
|
|
if dash is None:
|
|
return []
|
|
kept = [
|
|
p.get("id") for p in dash["manifest"].findall("Period") if p.get("id") not in dash["filtered_period_ids"]
|
|
]
|
|
return kept if all(kept) else []
|
|
|
|
@staticmethod
|
|
def failure_lines(e: BaseException) -> list[str]:
|
|
"""Return the header and the one line that report a failed download.
|
|
|
|
A ClickException is a deliberate stop with a message for the user, so it is reported
|
|
without its type, and the caller prints no traceback for it. Neither does a request
|
|
that failed on every retry: its line ends with its retry notes.
|
|
"""
|
|
if isinstance(e, click.ClickException):
|
|
line = e.format_message()
|
|
else:
|
|
line = f"{type(e).__name__}: {e}"
|
|
if retries_exhausted(e):
|
|
line += f", {'; '.join(e.__notes__)}"
|
|
return [":x: Download Failed...", f" {line}"]
|
|
|
|
def decrypt_verified(
|
|
self,
|
|
drm: DRM_T,
|
|
path: Path,
|
|
licence: Optional[Callable] = None,
|
|
track_kid: Optional[UUID] = None,
|
|
decrypt: bool = True,
|
|
) -> None:
|
|
"""Decrypt the file, and when a vault supplied a content key, prove the output decodes.
|
|
|
|
A poisoned vault returns the right KID with a wrong content key, and the decrypters
|
|
accept it without complaint. A content key from a vault is therefore trusted, and only
|
|
then copied to the other vaults, once FFmpeg can decode the result. Without FFmpeg
|
|
neither happens. For a fragmented MP4, ``verify.kid_windows`` reads from the ciphertext
|
|
which fragments each KID encrypts, and FFmpeg decodes windows inside each KID's
|
|
fragments, so only a KID that fails is flagged. A complete map does not judge a KID it does
|
|
not hold, so a clear file judges and copies no KID. A KID that is not in an incomplete map,
|
|
and every KID of any other file, gets a start and end check of the whole file, and a
|
|
failure there flags each of those KIDs. On failure the pair is flagged in the local
|
|
vaults, the ciphertext is restored, and the licence runs again; ``Vaults`` tells the
|
|
vault that served it. Every vault skips a flagged pair, so each pass burns at most one
|
|
vault; when the vaults run out the CDM answers, and a CDM key is not checked. A CDM
|
|
that returns the flagged key clears the flag and stores the content key again, because
|
|
then the decode check was wrong.
|
|
|
|
With ``decrypt=False`` the downloader already decrypted the segments in place, so a
|
|
failure can only flag and raise; the next run skips the flagged pair.
|
|
|
|
Before the decrypt, each KID the DRM names without a content key is licensed when the file
|
|
uses it, or when the file cannot be mapped, because ``prepare_drm`` licensed only for the track's KID.
|
|
"""
|
|
keys: Any = getattr(drm, "content_keys", None)
|
|
if keys is None:
|
|
if decrypt:
|
|
drm.decrypt(path)
|
|
return
|
|
|
|
def flagged_kids() -> list[UUID]:
|
|
"""The KIDs on this track whose content key another track already proved wrong."""
|
|
return [kid for kid, key in keys.items() if (kid, key) in self.vaults.flagged]
|
|
|
|
def drop(kid: UUID) -> str:
|
|
key = keys.pop(kid)
|
|
if self.LICENSE_KEY_CACHE.get(kid) == key:
|
|
self.LICENSE_KEY_CACHE.pop(kid)
|
|
return key
|
|
|
|
def relicense(bad: dict[UUID, str], kids: dict[UUID, Vault]) -> None:
|
|
"""Licence again for each dropped KID, then settle each dropped pair against the new key.
|
|
|
|
prepare_drm licenses only for a KID it is told to need, so without the second call a
|
|
track that still holds other content keys would decrypt without the dropped one. A
|
|
licence can replace the DRM's dict, so ``keys`` is read from the DRM again afterwards.
|
|
A licence key equal to the dropped one proves the decode check wrong and clears its flag.
|
|
"""
|
|
nonlocal keys
|
|
if not licence:
|
|
return
|
|
licence(drm, track_kid=track_kid)
|
|
for kid in sorted(bad):
|
|
if kid != track_kid and kid not in getattr(drm, "content_keys", {}):
|
|
licence(drm, track_kid=kid)
|
|
keys = getattr(drm, "content_keys", keys)
|
|
for kid, key in bad.items():
|
|
new = keys.get(kid)
|
|
if new is None:
|
|
continue
|
|
source = self.vaults.sources.get(kid)
|
|
if source is None:
|
|
if new == key:
|
|
self.vaults.unflag_bad_key(kid, key)
|
|
self.flush_vault_writes([partial(self.vaults.add_key, kid, key)])
|
|
elif not self.vaults.held_locally(kid, new):
|
|
self.flush_vault_writes([partial(self.vaults.add_key, kid, new)])
|
|
elif new == key and kid in kids and source[1] is kids[kid]:
|
|
raise ValueError(f"{key} from {kids[kid].name} was bad and no other source has the key")
|
|
|
|
known_bad = flagged_kids()
|
|
if known_bad and licence:
|
|
relicense({kid: drop(kid) for kid in known_bad}, {})
|
|
|
|
unkeyed = [kid for kid in getattr(drm, "kids", None) or [] if kid not in keys]
|
|
mapped = bool(unkeyed and decrypt and licence)
|
|
kid_map = verify.kid_windows(path) if mapped else None
|
|
if mapped and licence:
|
|
for kid in unkeyed:
|
|
if kid_map is None or kid in kid_map.windows:
|
|
licence(drm, track_kid=kid)
|
|
keys = getattr(drm, "content_keys", keys)
|
|
|
|
def vault_kids() -> dict[UUID, Vault]:
|
|
"""The KIDs whose key on the track came from a vault, with that vault."""
|
|
found = {kid: self.vaults.sources.get(kid) for kid in keys}
|
|
return {kid: src[1] for kid, src in found.items() if src and src[0] == keys[kid]}
|
|
|
|
backup = path.with_name(path.name + ".enc")
|
|
backup.unlink(missing_ok=True) # a killed run leaves one behind; its bytes may not match
|
|
checking = bool(vault_kids() and decrypt and binaries.FFMPEG)
|
|
if checking:
|
|
try:
|
|
os.link(path, backup)
|
|
except OSError as e:
|
|
self.log.debug(f"Cannot keep the ciphertext for a decrypt retry: {e!r}")
|
|
kid_map = (kid_map if mapped else verify.kid_windows(path)) if checking else None
|
|
windows = kid_map.windows if kid_map else {}
|
|
video = kid_map.video if kid_map else None
|
|
|
|
def window_decodes(start: float, end: float) -> bool:
|
|
return ffmpeg_decodes(path, start=start, seconds=end - start if end > start else 0.5, video=video)
|
|
|
|
def judged_kids() -> dict[UUID, Vault]:
|
|
"""The vault KIDs this file can judge. A map whose KIDs all have keys skips the KIDs it lacks."""
|
|
kids = vault_kids()
|
|
if kid_map is not None and all(kid in keys for kid in windows):
|
|
return {kid: vault for kid, vault in kids.items() if kid in windows}
|
|
return kids
|
|
|
|
def failed_kids(kids: dict[UUID, Vault]) -> set[UUID]:
|
|
"""The vault KIDs whose content key did not decode.
|
|
|
|
A KID in the KID map fails when one of its windows fails. The KIDs that are not in the
|
|
map share the verdict of one start and end check of the whole file, taken only when
|
|
every mapped KID passed: a wrong mapped key fails that check too and would flag them
|
|
for noise that is not theirs. The retry judges them again with the replaced key.
|
|
"""
|
|
failed = {kid for kid in kids if kid in windows and not all(window_decodes(*w) for w in windows[kid])}
|
|
absent = {kid for kid in kids if kid not in windows}
|
|
if absent and not failed and not ffmpeg_decodes(path, video=video):
|
|
failed |= absent
|
|
self.log.debug(
|
|
f"Key check on {path.name}: {len(kids) - len(absent)} KID(s) by fragment window, "
|
|
f"{len(absent)} by whole-file check, failed {sorted(k.hex for k in failed)}"
|
|
)
|
|
return failed
|
|
|
|
def passes_left(done: int) -> bool:
|
|
"""One pass per source that can still answer: every loaded vault, the server's vaults as
|
|
one source that walks them itself over --remote, and one for the CDM."""
|
|
remote = 1 if self._remote_service and self._remote_service.server_cdm else 0
|
|
return done < len(self.vaults) + remote + 1
|
|
|
|
try:
|
|
done = 0
|
|
while passes_left(done):
|
|
done += 1
|
|
if decrypt:
|
|
drm.decrypt(path)
|
|
kids = judged_kids()
|
|
stale = flagged_kids()
|
|
failed: set[UUID] = set()
|
|
if not stale:
|
|
checked = bool(kids and binaries.FFMPEG and path.exists())
|
|
failed = failed_kids(kids) if checked else set()
|
|
if not failed:
|
|
if checked:
|
|
self.flush_vault_writes(
|
|
[
|
|
partial(self.vaults.add_key, kid, keys[kid], excluding=vault)
|
|
for kid, vault in kids.items()
|
|
]
|
|
)
|
|
self.vaults.report_replaced(keys)
|
|
return
|
|
bad = {kid: drop(kid) for kid in {*failed, *stale}}
|
|
for kid, key in bad.items():
|
|
if kid not in kids:
|
|
continue
|
|
self.log.warning(f"{key} from {kids[kid].name} was bad, trying other vaults")
|
|
self.vaults.flag_bad_key(kid, key)
|
|
if not backup.exists() or not licence:
|
|
raise ValueError("The content key from the vault did not decrypt the track; run again")
|
|
path.unlink()
|
|
os.link(backup, path)
|
|
relicense(bad, kids)
|
|
raise ValueError("No vault or CDM produced a content key that decrypts the track")
|
|
finally:
|
|
backup.unlink(missing_ok=True)
|
|
|
|
def flush_vault_writes(self, pending: list[Callable[[], Any]]) -> None:
|
|
"""Hand the queued vault writes to the background writer so the track does not wait on them."""
|
|
for write in pending:
|
|
self.VAULT_WRITER.submit(write).add_done_callback(self.report_vault_write)
|
|
pending.clear()
|
|
|
|
def report_vault_write(self, future: futures.Future[Any]) -> None:
|
|
if exc := future.exception():
|
|
self.log.error(f"Vault write failed: {exc!r}")
|
|
|
|
def wait_vault_writes(self) -> None:
|
|
"""Block until every queued vault write has run, then log the total once."""
|
|
self.VAULT_WRITER.submit(lambda: None).result()
|
|
if not (self.vault_cache_tally and self.vault_cache_tally[0]):
|
|
self.vault_cache_tally = None
|
|
return
|
|
kids, successful_caches = self.vault_cache_tally
|
|
keys = len(kids)
|
|
self.vault_cache_tally = None
|
|
self.log.info(f"Cached {keys} Key{'' if keys == 1 else 's'} to {successful_caches}/{len(self.vaults)} Vaults")
|
|
|
|
def take_server_keys(self, source: "ServerKeySource", title: Title_T, cdm_only: bool) -> None:
|
|
"""Put the content keys of ``source`` on the tracks of ``title``, drop the null keys, then cache the rest."""
|
|
source.resolve_server_keys(title)
|
|
for track in title.tracks:
|
|
for drm in track.drm or []:
|
|
drop_null_keys(drm)
|
|
if not cdm_only:
|
|
self.prefer_vault_keys(source, title)
|
|
self.cache_resolved_keys(source, title)
|
|
|
|
def prefer_vault_keys(self, source: "ServerKeySource", title: Title_T) -> None:
|
|
"""Let the client vaults answer before a content key the server batch licence returned.
|
|
|
|
The client only learns the KIDs from the batch response, so the server has already
|
|
answered. A vault key still goes first, so ``decrypt_verified`` proves it and flags a
|
|
poisoned row; the server key waits as the next candidate. A server CDM key is trusted
|
|
like a local CDM key: it goes to the run cache and the vaults, and a retry after a
|
|
flagged vault key takes it before any vault. A server vault key is unproven, so it
|
|
waits in ``Vaults.candidates`` and gets the same check when its turn comes. Only the KIDs
|
|
the DRM names are looked up, each once, so sibling keys add no remote vault requests.
|
|
"""
|
|
looked_up: dict[UUID, tuple[Optional[str], Any]] = {}
|
|
for track in title.tracks:
|
|
for drm in getattr(track, "drm", None) or []:
|
|
content_keys = getattr(drm, "content_keys", {})
|
|
named = set(real_kids(drm) or content_keys)
|
|
for kid, server_key in list(content_keys.items()):
|
|
if kid not in named:
|
|
continue
|
|
if kid not in looked_up:
|
|
looked_up[kid] = self.vaults.get_key(kid)
|
|
vault_key, vault = looked_up[kid]
|
|
if not vault_key or vault_key == server_key:
|
|
continue
|
|
drm.content_keys[kid] = vault_key
|
|
server_vault = source.vault_source(kid, server_key)
|
|
if server_vault is not None:
|
|
self.vaults.candidates.setdefault(kid, []).append((server_key, server_vault))
|
|
elif kid not in self.LICENSE_KEY_CACHE:
|
|
self.LICENSE_KEY_CACHE[kid] = server_key
|
|
self.flush_vault_writes([partial(self.cache_keys_to_vaults, {kid: server_key})])
|
|
self.log.debug(
|
|
f"{vault.name} holds a different key for {kid.hex} than the server, testing it first"
|
|
)
|
|
|
|
def cache_resolved_keys(self, source: "ServerKeySource", title: Title_T) -> None:
|
|
"""Cache the keys a batch licence or an import put on the tracks.
|
|
|
|
Both fill track.drm before any prepare_drm call, so this is the only point where those
|
|
keys can still reach the local vaults. A content key a local vault already holds is not pushed again.
|
|
"""
|
|
resolved = {
|
|
kid: key
|
|
for track in title.tracks
|
|
for drm in (getattr(track, "drm", None) or [])
|
|
for kid, key in getattr(drm, "content_keys", {}).items()
|
|
if kid not in self.LICENSE_KEY_CACHE
|
|
and source.vault_source(kid, key) is None
|
|
and self.vaults.sources.get(kid, (None,))[0] != key
|
|
}
|
|
keys = {kid: key for kid, key in resolved.items() if not self.vaults.held_locally(kid, key)}
|
|
if keys:
|
|
self.LICENSE_KEY_CACHE.update(keys)
|
|
self.flush_vault_writes([partial(self.cache_keys_to_vaults, keys)])
|
|
|
|
def cache_keys_to_vaults(self, content_keys: dict[UUID, str]) -> None:
|
|
"""Store licence keys in every vault. A licence key clears a flag on its pair first: the
|
|
flag came from a decode check that was wrong, and a local vault refuses a flagged pair."""
|
|
if not content_keys or not self.vaults:
|
|
return
|
|
self.vaults.clear_licensed_flags(content_keys)
|
|
successful_caches = self.vaults.add_keys(cast(dict[Union[UUID, str], str], content_keys))
|
|
kids, caches = self.vault_cache_tally or (set(), successful_caches)
|
|
self.vault_cache_tally = (kids | content_keys.keys(), min(caches, successful_caches))
|
|
|
|
@classmethod
|
|
def drm_lock(cls, drm: DRM_T) -> Lock:
|
|
"""Return the lock for one DRM's set of content keys.
|
|
|
|
Tracks that share KIDs still take turns, so the second one finds the content keys in
|
|
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs query the key vaults
|
|
at the same time; their licences take turns under ``take_licence_turn``.
|
|
|
|
Callers take the lock after get_drm_for_cdm(), which folds a track's sibling PlayReady
|
|
objects into the returned one, so the KID set is complete for that track's DRM list
|
|
before the lock is chosen. This holds because every manifest parser gives the tracks
|
|
that share a PlayReady object the same DRM list (ISM attaches one list to all tracks;
|
|
DASH and HLS build a list per track). Two tracks that share one object but list
|
|
different siblings would get different locks.
|
|
"""
|
|
kids = getattr(drm, "kids", None) or []
|
|
key = ",".join(sorted(getattr(k, "hex", str(k)) for k in kids)) or getattr(drm, "content_id", None)
|
|
with cls.DRM_LOCKS_GUARD:
|
|
return cls.DRM_LOCKS.setdefault(key or type(drm).__name__, Lock())
|
|
|
|
def take_licence_turn(
|
|
self,
|
|
turn: ExitStack,
|
|
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
|
kids: list[UUID],
|
|
track: AnyTrack,
|
|
track_kid: Optional[UUID],
|
|
cek_tree: Optional[Tree],
|
|
) -> None:
|
|
"""Hold LICENCE_LOCK until ``turn`` closes, then take the content keys of ``kids`` an earlier licence cached.
|
|
|
|
One licence can carry the keys of a DRM with other KIDs, so licences take turns. The caller
|
|
holds its ``drm_lock`` first, so the vault lookups before this still run at the same time.
|
|
"""
|
|
turn.enter_context(self.LICENCE_LOCK)
|
|
if DOWNLOAD_CANCELLED.is_set():
|
|
raise DownloadCancelled()
|
|
for kid in kids:
|
|
if kid not in drm.content_keys:
|
|
self.take_cached_key(drm, kid, track, track_kid, cek_tree)
|
|
|
|
def take_cached_key(
|
|
self,
|
|
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
|
kid: UUID,
|
|
track: AnyTrack,
|
|
track_kid: Optional[UUID],
|
|
cek_tree: Optional[Tree],
|
|
) -> bool:
|
|
"""Give the DRM the content key LICENSE_KEY_CACHE holds for ``kid``, and return whether it held one."""
|
|
key = self.LICENSE_KEY_CACHE.get(kid)
|
|
if not key:
|
|
return False
|
|
drm.content_keys[kid] = key
|
|
if cek_tree is not None:
|
|
self.add_key_label(cek_tree, kid, key, track_kid, "from cache")
|
|
log_event(
|
|
"license_cache_hit",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={"kid": kid.hex, "content_key": key, "track": str(track), "drm_type": type(drm).__name__},
|
|
)
|
|
return True
|
|
|
|
def take_vault_key(
|
|
self,
|
|
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
|
kid: UUID,
|
|
track: AnyTrack,
|
|
track_kid: Optional[UUID],
|
|
cek_tree: Optional[Tree],
|
|
) -> bool:
|
|
"""Give the DRM the content key a key vault holds for ``kid``, cache it for the run, and return whether one did."""
|
|
key, vault = self.vaults.get_key(kid)
|
|
if not key:
|
|
return False
|
|
drm.content_keys[kid] = key
|
|
self.LICENSE_KEY_CACHE[kid] = key
|
|
if cek_tree is not None:
|
|
self.add_key_label(cek_tree, kid, key, track_kid, f"from {vault}")
|
|
if self.debug_logger:
|
|
self.debug_logger.log_vault_query(
|
|
vault_name=str(vault),
|
|
operation="get_key_success",
|
|
service=self.service,
|
|
context={
|
|
"kid": kid.hex,
|
|
"content_key": key,
|
|
"track": str(track),
|
|
"from_cache": True,
|
|
"drm_type": type(drm).__name__,
|
|
},
|
|
)
|
|
return True
|
|
|
|
@staticmethod
|
|
def with_track_kid(drm: Any, track_kid: Optional[UUID]) -> list[UUID]:
|
|
"""Return the KIDs the DRM names, plus the track's own KID when the DRM leaves it out."""
|
|
kids = list(getattr(drm, "kids", None) or [])
|
|
if track_kid and track_kid not in kids:
|
|
kids.append(track_kid)
|
|
return kids
|
|
|
|
def take_known_keys(
|
|
self,
|
|
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
|
kids: list[UUID],
|
|
track: AnyTrack,
|
|
track_kid: Optional[UUID],
|
|
cek_tree: Optional[Tree],
|
|
use_cache: bool = True,
|
|
use_vaults: bool = True,
|
|
stop_at_miss: bool = False,
|
|
) -> list[UUID]:
|
|
"""Give the DRM the content keys of ``kids`` the run cache or a key vault holds, and return the KIDs still missing.
|
|
|
|
``stop_at_miss`` returns at the first miss, so no more vaults are asked for a run that fails anyway.
|
|
"""
|
|
missing = []
|
|
for kid in kids:
|
|
if kid in drm.content_keys:
|
|
if cek_tree is not None:
|
|
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
|
|
continue
|
|
if use_cache and self.take_cached_key(drm, kid, track, track_kid, cek_tree):
|
|
continue
|
|
if not (use_vaults and self.take_vault_key(drm, kid, track, track_kid, cek_tree)):
|
|
missing.append(kid)
|
|
if stop_at_miss:
|
|
break
|
|
return missing
|
|
|
|
def license_drm(
|
|
self,
|
|
drm: Union[ClearKeyCENC, Widevine, PlayReady],
|
|
get_keys: Callable[[], None],
|
|
track: AnyTrack,
|
|
title: Title_T,
|
|
track_kid: Optional[UUID],
|
|
table: Table,
|
|
cdm_only: bool,
|
|
vaults_only: bool,
|
|
force: bool,
|
|
) -> None:
|
|
"""Give a Widevine, PlayReady or ClearKey DRM its content keys: run cache, then key vaults, then ``get_keys``.
|
|
|
|
A held key for ``track_kid`` skips the licence, as the file is encrypted with that KID;
|
|
``decrypt_verified`` licenses any other KID the file turns out to use.
|
|
``force`` skips the run cache, so the licence runs again for each KID no vault answers.
|
|
A licence can answer a KID with a blank or null key; a vault key for that KID wins.
|
|
"""
|
|
pssh: Optional[str] = None
|
|
if isinstance(drm, ClearKeyCENC):
|
|
name = "ClearKey"
|
|
else:
|
|
name = type(drm).__name__
|
|
pssh = drm.pssh.dumps() if isinstance(drm, Widevine) else drm.pssh_b64 or ""
|
|
if pssh is not None and self.debug_logger:
|
|
self.debug_logger.log_drm_operation(
|
|
drm_type=name,
|
|
operation="prepare_drm",
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"title": str(title),
|
|
"pssh": pssh,
|
|
"kids": [k.hex for k in drm.kids],
|
|
"track_kid": track_kid.hex if track_kid else None,
|
|
},
|
|
)
|
|
pending_vault_writes: list[Callable[[], Any]] = []
|
|
with self.drm_lock(drm), ExitStack() as licence_turn:
|
|
label = [(f"({self.truncate_pssh_for_display(pssh, name)})", "text")] if pssh is not None else []
|
|
cek_tree = Tree(Text.assemble((name, "cyan"), *label, overflow="fold"))
|
|
pre_existing_tree = next(
|
|
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
|
)
|
|
if pre_existing_tree:
|
|
cek_tree = pre_existing_tree
|
|
|
|
def show_error(msg: str) -> None:
|
|
cek_tree.add(f"[logging.level.error]{msg}")
|
|
if not pre_existing_tree:
|
|
table.add_row(cek_tree)
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
|
|
all_kids = self.with_track_kid(drm, track_kid)
|
|
stop_at_miss = vaults_only and not cdm_only
|
|
missing = self.take_known_keys(
|
|
drm, all_kids, track, track_kid, cek_tree, not force, not cdm_only, stop_at_miss
|
|
)
|
|
if missing and stop_at_miss:
|
|
msg = f"No Vault has a Key for {missing[0].hex} and --vaults-only was used"
|
|
log_event(
|
|
"vault_key_not_found",
|
|
level="ERROR",
|
|
service=self.service,
|
|
message=msg,
|
|
context={"kid": missing[0].hex, "track": str(track), "drm_type": type(drm).__name__},
|
|
)
|
|
show_error(msg)
|
|
raise drm.Exceptions.CEKNotFound(msg)
|
|
|
|
need_license = bool(missing) and (force or not track_kid or track_kid not in drm.content_keys)
|
|
if need_license:
|
|
# force licenses again, so the turn takes no cached keys
|
|
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
|
|
need_license = not all(kid in drm.content_keys for kid in all_kids)
|
|
|
|
if need_license and not vaults_only:
|
|
from_vaults = drm.content_keys.copy()
|
|
log_event(
|
|
"get_license",
|
|
level="INFO",
|
|
service=self.service,
|
|
message=f"Requesting {name} license from service",
|
|
context={
|
|
"track": str(track),
|
|
"drm_type": type(drm).__name__,
|
|
"kids_needed": [k.hex for k in all_kids if k not in drm.content_keys],
|
|
},
|
|
)
|
|
try:
|
|
get_keys()
|
|
except Exception as e:
|
|
if drm.content_keys and (not track_kid or track_kid in drm.content_keys):
|
|
self.log.warning(
|
|
f"Licence failed, using the {len(drm.content_keys)} known key(s): {redact_secrets(str(e))}"
|
|
)
|
|
else:
|
|
if isinstance(e, (drm.Exceptions.EmptyLicense, drm.Exceptions.CEKNotFound)):
|
|
msg = str(e)
|
|
else:
|
|
msg = f"An exception occurred in the Service's license function: {e}"
|
|
if self.debug_logger:
|
|
self.debug_logger.log_error(
|
|
"get_license" if name == "Widevine" else f"get_license_{name.lower()}",
|
|
e,
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"exception_type": type(e).__name__,
|
|
"drm_type": type(drm).__name__,
|
|
},
|
|
)
|
|
show_error(msg)
|
|
raise
|
|
else:
|
|
log_event(
|
|
"license_keys_retrieved",
|
|
level="INFO",
|
|
service=self.service,
|
|
context={
|
|
"track": str(track),
|
|
"drm_type": type(drm).__name__,
|
|
"keys_count": len(drm.content_keys),
|
|
"kids": [k.hex for k in drm.content_keys.keys()],
|
|
},
|
|
)
|
|
|
|
for kid, key in drm.content_keys.items():
|
|
self.add_key_label(cek_tree, kid, f"[red]{key}[/]" if is_null_key(key) else key, track_kid)
|
|
drop_null_keys(drm)
|
|
drm.content_keys.update(from_vaults)
|
|
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
|
pending_vault_writes.append(
|
|
partial(
|
|
self.cache_keys_to_vaults,
|
|
{k: v for k, v in drm.content_keys.items() if k not in from_vaults},
|
|
)
|
|
)
|
|
|
|
if track_kid and track_kid not in drm.content_keys:
|
|
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
|
show_error(msg)
|
|
raise drm.Exceptions.CEKNotFound(msg)
|
|
|
|
if cek_tree.children and not pre_existing_tree:
|
|
table.add_row()
|
|
table.add_row(cek_tree)
|
|
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
|
|
@staticmethod
|
|
def add_key_label(tree: Tree, kid: UUID, key: str, track_kid: Optional[UUID] = None, source: str = "") -> None:
|
|
"""Add ``kid:key`` to a content key tree once, with ``*`` on the track's own KID and ``source`` after it."""
|
|
if any(f"{kid.hex}:{key}" in str(x.label) for x in tree.children):
|
|
return
|
|
label = f"[text2]{kid.hex}:{key}"
|
|
if kid == track_kid:
|
|
label += "*"
|
|
if source:
|
|
label += f" {source}"
|
|
tree.add(label)
|
|
|
|
def service_licence(self, service: Service, drm_system: Optional[str] = None, **kwargs: Any) -> Any:
|
|
"""Call the service's licence function for the track's DRM system.
|
|
|
|
Arguments go through declared_kwargs, so a service receives only the ones its own
|
|
licence function declares.
|
|
|
|
prepare_drm passes drm_system so the choice follows the track, not the loaded CDM:
|
|
a track downloading at the same time can swap self.cdm.
|
|
"""
|
|
if drm_system is None:
|
|
drm_system = "playready" if is_playready_cdm(self.cdm) else "widevine"
|
|
fn = service.get_playready_license if drm_system == "playready" else service.get_widevine_license
|
|
return fn(**declared_kwargs(fn, kwargs))
|
|
|
|
def cdm_for(self, system: str, quality: Optional[int] = None) -> Optional[object]:
|
|
"""Get the CDM for one DRM system, ``widevine`` or ``playready``.
|
|
|
|
A quality key that names one device applies to both DRM systems. When that device is for
|
|
the other system, the ``widevine`` or ``playready`` config key selects the device.
|
|
"""
|
|
matches = is_playready_cdm if system == "playready" else is_widevine_cdm
|
|
cdm = self.get_cdm(self.service, self.profile, drm=system, quality=quality)
|
|
if quality and not (cdm and matches(cdm)):
|
|
cdm = self.get_cdm(self.service, self.profile, drm=system)
|
|
return cdm
|
|
|
|
def licence_one_drm(self, prepare: Callable, drm: Union[Widevine, PlayReady], track_kid: Optional[UUID]) -> None:
|
|
"""Send one challenge for the KIDs of this DRM, also when the DRM or a vault has the content keys.
|
|
|
|
This is the ``--cdm-only`` path. The challenge goes out the first time a run sees the KIDs
|
|
for the DRM system.
|
|
|
|
prepare_drm asks the CDM only for a KID with no content key, so the content keys the DRM
|
|
holds are removed for the call and put back after it.
|
|
"""
|
|
kids = {*drm.kids, *([track_kid] if track_kid else [])}
|
|
attempted = self.all_drm_attempted[type(drm)]
|
|
if kids <= attempted:
|
|
prepare(drm, track_kid=track_kid)
|
|
return
|
|
with self.drm_lock(drm):
|
|
held = dict(drm.content_keys)
|
|
drm.content_keys.clear()
|
|
try:
|
|
prepare(drm, track_kid=track_kid, cdm_only=True, force=True)
|
|
finally:
|
|
attempted |= kids
|
|
with self.drm_lock(drm):
|
|
for held_kid, key in held.items():
|
|
drm.content_keys.setdefault(held_kid, key)
|
|
|
|
def licence_all_drm(
|
|
self,
|
|
prepare: Callable,
|
|
track: AnyTrack,
|
|
tolerate: bool,
|
|
drm: DRM_T,
|
|
track_kid: Optional[UUID] = None,
|
|
force: bool = False,
|
|
) -> None:
|
|
"""License the track with each DRM system it offers, not only the one the downloader selected.
|
|
|
|
A DRM system sends a challenge only for a KID that no vault and no earlier track gave a
|
|
content key for. With ``force``, each DRM system sends one challenge for a set of KIDs in
|
|
a run. A DRM system that fails stops the title. With ``tolerate`` it is a warning, if the
|
|
other DRM system gave the content key of the track.
|
|
"""
|
|
if not isinstance(drm, (Widevine, PlayReady)):
|
|
prepare(drm, track_kid=track_kid)
|
|
return
|
|
|
|
other_type = PlayReady if type(drm) is Widevine else Widevine
|
|
others = [d for d in track.drm or [] if isinstance(d, (Widevine, PlayReady)) and type(d) is other_type]
|
|
if other_type is PlayReady and len(others) > 1:
|
|
playready = [d for d in others if isinstance(d, PlayReady)]
|
|
playready[0].absorb(*playready[1:])
|
|
if not others and other_type.__name__ not in self.all_drm_warned:
|
|
self.all_drm_warned.add(other_type.__name__)
|
|
self.log.warning(
|
|
f"Track {track} has no {other_type.__name__} DRM, licensing with {type(drm).__name__} only"
|
|
)
|
|
|
|
targets = [drm, *others[:1]]
|
|
error: Optional[Exception] = None
|
|
with self.ALL_DRM_LOCK:
|
|
loaded_cdm = self.cdm
|
|
try:
|
|
for target in targets:
|
|
kid = track_kid if target is drm or track_kid in target.kids else None
|
|
try:
|
|
if force:
|
|
self.licence_one_drm(prepare, target, kid)
|
|
else:
|
|
prepare(target, track_kid=kid)
|
|
except Exception as e:
|
|
if not tolerate:
|
|
raise
|
|
self.log.warning(f"{type(target).__name__} licence failed for {track}: {e}")
|
|
error = error or e
|
|
|
|
keys = {kid: key for target in targets for kid, key in target.content_keys.items()}
|
|
for target in targets:
|
|
with self.drm_lock(target):
|
|
for kid, key in keys.items():
|
|
target.content_keys.setdefault(kid, key)
|
|
finally:
|
|
self.cdm = loaded_cdm
|
|
|
|
track_keys = getattr(drm, "content_keys", {})
|
|
if error and (not track_keys or (track_kid and track_kid not in track_keys)):
|
|
raise error
|
|
|
|
def prepare_drm(
|
|
self,
|
|
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
|
track: AnyTrack,
|
|
title: Title_T,
|
|
certificate: Callable,
|
|
licence: Callable,
|
|
clearkey_licence: Optional[Callable] = None,
|
|
track_kid: Optional[UUID] = None,
|
|
table: Optional[Table] = None,
|
|
cdm_only: bool = False,
|
|
vaults_only: bool = False,
|
|
export: Optional[Path] = None,
|
|
service_session: Optional[Any] = None,
|
|
force: bool = False,
|
|
) -> None:
|
|
"""
|
|
Prepare the DRM by getting decryption data like KIDs, Keys, and such.
|
|
The DRM object should be ready for decryption once this function ends.
|
|
``force`` skips the content keys of earlier tracks in this run. The CDM then licenses
|
|
again for each KID the DRM holds no content key for.
|
|
"""
|
|
if not drm:
|
|
return
|
|
assert table is not None
|
|
|
|
remote = self._remote_service
|
|
server_cdm = bool(remote and remote.server_cdm)
|
|
licensed_locally = bool(remote and remote.licenses_locally(track))
|
|
|
|
if server_cdm and not licensed_locally:
|
|
assert remote is not None
|
|
with self.drm_lock(drm), ExitStack() as licence_turn:
|
|
pending_vault_writes: list[Callable[[], Any]] = []
|
|
vault_kids = self.with_track_kid(drm, track_kid)
|
|
self.take_known_keys(drm, vault_kids, track, track_kid, None, use_vaults=not cdm_only)
|
|
|
|
def missing_track_key() -> bool:
|
|
return not drm.content_keys or bool(track_kid and track_kid not in drm.content_keys)
|
|
|
|
if missing_track_key():
|
|
self.take_licence_turn(licence_turn, drm, vault_kids, track, track_kid, None)
|
|
known_keys = set(drm.content_keys)
|
|
|
|
if missing_track_key():
|
|
from unshackle.core.remote_service import RemoteError
|
|
|
|
try:
|
|
licence(
|
|
drm_system="playready" if drm.__class__.__name__ == "PlayReady" else "widevine",
|
|
challenge=b"",
|
|
kid=track_kid,
|
|
)
|
|
except click.ClickException:
|
|
raise
|
|
except (Exception, RemoteError) as e:
|
|
self.log.debug(f"Server CDM licence with an empty challenge failed: {e!r}")
|
|
drop_null_keys(drm)
|
|
|
|
new_keys = {}
|
|
for kid, key in drm.content_keys.items():
|
|
source = None if cdm_only else remote.vault_source(kid, key)
|
|
if source is not None:
|
|
self.vaults.sources[kid] = (key, source)
|
|
elif kid not in known_keys:
|
|
new_keys[kid] = key
|
|
if new_keys:
|
|
self.LICENSE_KEY_CACHE.update(new_keys)
|
|
pending_vault_writes.append(partial(self.cache_keys_to_vaults, new_keys))
|
|
self.flush_vault_writes(pending_vault_writes)
|
|
|
|
# the server can refuse the licence above its cap and hand the track to this machine
|
|
licensed_locally = bool(remote and remote.licenses_locally(track))
|
|
if server_cdm and not licensed_locally:
|
|
if not drm.content_keys:
|
|
self.log.warning("Server CDM did not resolve any keys for this track")
|
|
return
|
|
if track_kid and track_kid not in drm.content_keys:
|
|
msg = f"No Content Key for KID {track_kid.hex} was returned by the server CDM"
|
|
if isinstance(drm, PlayReady):
|
|
raise PlayReady.Exceptions.CEKNotFound(msg)
|
|
raise Widevine.Exceptions.CEKNotFound(msg)
|
|
assert remote is not None
|
|
server_drm_type = remote.server_cdm_type
|
|
drm_name = {"widevine": "Widevine", "playready": "PlayReady"}.get(server_drm_type, drm.__class__.__name__)
|
|
log_event(
|
|
"drm_content_keys",
|
|
level="INFO",
|
|
message=f"Server CDM resolved {len(drm.content_keys)} {drm_name} content key(s)",
|
|
service=self.service,
|
|
drm_type=drm_name,
|
|
key_count=len(drm.content_keys),
|
|
keys=[{"kid": k.hex, "key": v} for k, v in drm.content_keys.items()],
|
|
remote=True,
|
|
)
|
|
with self.drm_lock(drm):
|
|
pssh_str = ""
|
|
expected_class = "PlayReady" if server_drm_type == "playready" else "Widevine"
|
|
matching_drm = next(
|
|
(d for d in (track.drm or []) if d.__class__.__name__ == expected_class),
|
|
drm,
|
|
)
|
|
if hasattr(matching_drm, "pssh") and matching_drm.pssh:
|
|
if hasattr(matching_drm.pssh, "dumps"):
|
|
pssh_str = self.truncate_pssh_for_display(matching_drm.pssh.dumps(), drm_name)
|
|
elif hasattr(matching_drm, "data") and matching_drm.data.get("pssh_b64"):
|
|
pssh_str = self.truncate_pssh_for_display(matching_drm.data["pssh_b64"], drm_name)
|
|
if pssh_str:
|
|
cek_tree = Tree(Text.assemble((drm_name, "cyan"), (f"({pssh_str})", "text"), overflow="fold"))
|
|
else:
|
|
cek_tree = Tree(Text.assemble((drm_name, "cyan"), overflow="fold"))
|
|
all_kids = self.with_track_kid(drm, track_kid)
|
|
for kid in all_kids:
|
|
if kid in drm.content_keys:
|
|
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
|
|
for kid, key in drm.content_keys.items():
|
|
if kid not in all_kids:
|
|
self.add_key_label(cek_tree, kid, key)
|
|
if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells):
|
|
table.add_row(cek_tree)
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
return
|
|
|
|
track_quality = None
|
|
if isinstance(track, Video) and track.height:
|
|
track_quality = track.height
|
|
elif title is not None:
|
|
track_quality = max((v.height for v in title.tracks.videos if v.height), default=None)
|
|
|
|
track_cdm = self.cdm
|
|
if licensed_locally:
|
|
assert remote is not None
|
|
track_cdm = remote.local_cdm
|
|
local_drm = track.get_drm_for_cdm(track_cdm)
|
|
if isinstance(local_drm, (Widevine, PlayReady)) and isinstance(local_drm, PlayReady) != is_playready_cdm(
|
|
track_cdm
|
|
):
|
|
track_cdm = self.cdm_for("playready" if isinstance(local_drm, PlayReady) else "widevine", track_quality)
|
|
if local_drm is not None and local_drm is not drm:
|
|
assert not isinstance(local_drm, ClearKey)
|
|
# a mid-download handover leaves drm on the server's DRM system, and the caller decrypts with it
|
|
self.prepare_drm(
|
|
local_drm,
|
|
track,
|
|
title,
|
|
certificate,
|
|
licence,
|
|
clearkey_licence,
|
|
track_kid,
|
|
table,
|
|
cdm_only,
|
|
vaults_only,
|
|
export,
|
|
service_session,
|
|
)
|
|
with self.drm_lock(drm):
|
|
drm.content_keys.update(local_drm.content_keys)
|
|
return
|
|
|
|
drm_system = "playready" if isinstance(drm, PlayReady) else "widevine"
|
|
licence = partial(licence, drm_system=drm_system)
|
|
|
|
if not server_cdm and isinstance(drm, (Widevine, PlayReady)):
|
|
matches = is_playready_cdm if drm_system == "playready" else is_widevine_cdm
|
|
if not matches(self.cdm):
|
|
name, other = ("PlayReady", "Widevine") if drm_system == "playready" else ("Widevine", "PlayReady")
|
|
system_cdm = self.cdm_for(drm_system, track_quality)
|
|
if system_cdm and matches(system_cdm):
|
|
self.log.info(f"Switching to the {name} CDM")
|
|
self.cdm = system_cdm
|
|
track_cdm = system_cdm
|
|
elif self.cdm is None:
|
|
raise click.ClickException(f"No CDM configured for {self.service} and no default CDM")
|
|
else:
|
|
raise ValueError(f"Title needs a {name} CDM but {self.service} is configured with {other}.")
|
|
|
|
if isinstance(drm, (Widevine, PlayReady, ClearKeyCENC)):
|
|
if isinstance(drm, ClearKeyCENC):
|
|
get_keys = partial(
|
|
drm.get_content_keys, licence=clearkey_licence or (lambda **_: None), session=service_session
|
|
)
|
|
else:
|
|
get_keys = partial(drm.get_content_keys, cdm=track_cdm, licence=licence, certificate=certificate)
|
|
self.license_drm(drm, get_keys, track, title, track_kid, table, cdm_only, vaults_only, force)
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
|
|
elif isinstance(drm, MonaLisa):
|
|
with self.drm_lock(drm):
|
|
display_id = drm.content_id or drm.pssh
|
|
pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa")
|
|
cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
|
|
pre_existing_tree = next(
|
|
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
|
)
|
|
if pre_existing_tree:
|
|
cek_tree = pre_existing_tree
|
|
|
|
for kid_, key in drm.content_keys.items():
|
|
self.add_key_label(cek_tree, kid_, key, track_kid=kid_) # every MonaLisa key is the track's own
|
|
|
|
if cek_tree.children and not pre_existing_tree:
|
|
table.add_row()
|
|
table.add_row(cek_tree)
|
|
|
|
if export:
|
|
self.write_export(export, title, track, drm)
|
|
|
|
@staticmethod
|
|
def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]:
|
|
"""Get Service Cookie File Path for Profile."""
|
|
direct_cookie_file = config.directories.cookies / f"{service}.txt"
|
|
profile_cookie_file = config.directories.cookies / service / f"{profile}.txt"
|
|
default_cookie_file = config.directories.cookies / service / "default.txt"
|
|
|
|
if direct_cookie_file.exists():
|
|
return direct_cookie_file
|
|
elif profile_cookie_file.exists():
|
|
return profile_cookie_file
|
|
elif default_cookie_file.exists():
|
|
return default_cookie_file
|
|
return None
|
|
|
|
@staticmethod
|
|
def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[CookieJar]:
|
|
"""
|
|
Get Service Cookies for Profile.
|
|
|
|
If the config has firefox_cookies for the service, cookies come from Firefox first. If that extraction
|
|
raises or returns nothing, unshackle uses the profile's cookie file instead and does not log the failure.
|
|
"""
|
|
ff_settings = getattr(config, "firefox_cookies", {}).get(service)
|
|
if ff_settings:
|
|
try:
|
|
from unshackle.core.utils.firefox_cookie_extractor import get_firefox_cookies
|
|
|
|
extracted_jar = get_firefox_cookies(ff_settings)
|
|
if extracted_jar:
|
|
return extracted_jar
|
|
except Exception:
|
|
pass
|
|
|
|
cookie_file = dl.get_cookie_path(service, profile)
|
|
if cookie_file:
|
|
return dl.load_cookie_file(cookie_file)
|
|
|
|
return None
|
|
|
|
@staticmethod
|
|
def load_cookie_file(cookie_file: Path) -> MozillaCookieJar:
|
|
"""Load a Netscape cookie file with every expiry blanked, so no layer drops an expired cookie."""
|
|
cookie_jar = dl.parse_cookie_text(cookie_file.read_text("utf-8-sig"), str(cookie_file))
|
|
cookie_jar.filename = str(cookie_file)
|
|
return cookie_jar
|
|
|
|
@staticmethod
|
|
def parse_cookie_text(text: str, source: str) -> MozillaCookieJar:
|
|
"""Parse Netscape cookie text the way :meth:`load_cookie_file` does; ``source`` names it in errors."""
|
|
cookie_jar = MozillaCookieJar()
|
|
cookie_data = html.unescape(text).splitlines(keepends=False)
|
|
head = next((line.lstrip() for line in cookie_data if line.strip()), "")
|
|
if head[:1] in ("[", "{"):
|
|
raise ValueError(f"{source} is a JSON export; export it again in Netscape format")
|
|
for i, line in enumerate(cookie_data):
|
|
body = line.lstrip()
|
|
prefix = "#HttpOnly_" if body.startswith("#HttpOnly_") else ""
|
|
body = body[len(prefix) :]
|
|
if not body or body.startswith("#"):
|
|
continue
|
|
line_data = body.split("\t") if "\t" in body else body.split(None, 6)
|
|
if len(line_data) == 6:
|
|
line_data.append("")
|
|
if len(line_data) != 7:
|
|
raise ValueError(f"{source} line {i + 1} is not a Netscape cookie row: {line!r}")
|
|
line_data[4] = ""
|
|
line_data[1] = str(line_data[0].startswith(".")).upper()
|
|
cookie_data[i] = prefix + "\t".join(line_data)
|
|
cookie_data.insert(0, "# Netscape HTTP Cookie File")
|
|
cookie_jar._really_load(StringIO("\n".join(cookie_data)), source, True, True) # type: ignore[attr-defined]
|
|
if not len(cookie_jar):
|
|
raise ValueError(f"{source} holds no cookies; export it again in Netscape format")
|
|
return cookie_jar
|
|
|
|
@staticmethod
|
|
def save_cookies(path: Path, cookies: CookieJar) -> None:
|
|
if hasattr(cookies, "jar"):
|
|
cookies = cookies.jar
|
|
|
|
cookie_jar = dl.load_cookie_file(path) if path.exists() else MozillaCookieJar(path)
|
|
for cookie in cookies:
|
|
if not cookie.name or not cookie.domain:
|
|
continue
|
|
cookie_jar.set_cookie(cookie)
|
|
cookie_jar.save(ignore_discard=True, ignore_expires=True)
|
|
|
|
@staticmethod
|
|
def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]:
|
|
"""Get Service Credentials for Profile."""
|
|
credentials = config.credentials.get(service)
|
|
if credentials:
|
|
if isinstance(credentials, dict):
|
|
if profile:
|
|
credentials = credentials.get(profile) or credentials.get("default")
|
|
else:
|
|
credentials = credentials.get("default")
|
|
if credentials:
|
|
if isinstance(credentials, list):
|
|
return Credential(*credentials)
|
|
return Credential.loads(credentials) # type: ignore
|
|
return None
|
|
|
|
def cdm_from_service_entry(self) -> bool:
|
|
"""Whether the service's own ``cdm`` entry named the loaded device, not ``--cdm`` or the ``default`` entry."""
|
|
return (
|
|
self.cdm is not None
|
|
and not self.cdm_override
|
|
and cdm_entry_names_device(ci_get(config.cdm, self.service), self.profile)
|
|
)
|
|
|
|
def get_cdm(
|
|
self,
|
|
service: str,
|
|
profile: Optional[str] = None,
|
|
drm: Optional[str] = None,
|
|
quality: Optional[int] = None,
|
|
) -> Any:
|
|
"""
|
|
Get CDM for a specified service (either Local or Remote CDM).
|
|
Now supports quality-based selection when the caller gives a quality.
|
|
With no ``drm``, an entry gives the DRM system that it lists first.
|
|
Raises a ValueError if there is a problem getting a CDM.
|
|
"""
|
|
# A per-request override (set by the REST API per job) takes precedence over the
|
|
# global config, so a job can select a specific CDM device without mutating shared state.
|
|
cdm_name = resolve_cdm_name(config.cdm, service, getattr(self, "cdm_override", None))
|
|
if not cdm_name:
|
|
return None
|
|
|
|
if isinstance(cdm_name, dict):
|
|
if quality:
|
|
quality_match = None
|
|
quality_keys = []
|
|
|
|
for key in cdm_name.keys():
|
|
if (
|
|
isinstance(key, str)
|
|
and any(op in key for op in [">=", ">", "<=", "<"])
|
|
or (isinstance(key, str) and key.isdigit())
|
|
):
|
|
quality_keys.append(key)
|
|
|
|
def sort_quality_key(key: str) -> tuple[int, int]:
|
|
if key.isdigit():
|
|
return (0, int(key)) # Exact matches first
|
|
elif key.startswith(">="):
|
|
return (1, -int(key[2:])) # >= descending
|
|
elif key.startswith(">"):
|
|
return (1, -int(key[1:])) # > descending
|
|
elif key.startswith("<="):
|
|
return (2, int(key[2:])) # <= ascending
|
|
elif key.startswith("<"):
|
|
return (2, int(key[1:])) # < ascending
|
|
return (3, 0) # Other keys last
|
|
|
|
quality_keys.sort(key=sort_quality_key)
|
|
|
|
for key in quality_keys:
|
|
if key.isdigit() and quality == int(key):
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on exact quality match {quality}p: {quality_match}")
|
|
break
|
|
elif key.startswith(">="):
|
|
threshold = int(key[2:])
|
|
if quality >= threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p >= {threshold}p: {quality_match}")
|
|
break
|
|
elif key.startswith(">"):
|
|
threshold = int(key[1:])
|
|
if quality > threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p > {threshold}p: {quality_match}")
|
|
break
|
|
elif key.startswith("<="):
|
|
threshold = int(key[2:])
|
|
if quality <= threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p <= {threshold}p: {quality_match}")
|
|
break
|
|
elif key.startswith("<"):
|
|
threshold = int(key[1:])
|
|
if quality < threshold:
|
|
quality_match = cdm_name[key]
|
|
self.log.debug(f"Selected CDM based on quality {quality}p < {threshold}p: {quality_match}")
|
|
break
|
|
|
|
if quality_match:
|
|
cdm_name = quality_match
|
|
|
|
if isinstance(cdm_name, dict):
|
|
lower_keys = {k.lower(): v for k, v in cdm_name.items()}
|
|
if {"widevine", "playready"} & lower_keys.keys():
|
|
drm_key = None
|
|
if drm:
|
|
drm_key = {
|
|
"wv": "widevine",
|
|
"widevine": "widevine",
|
|
"pr": "playready",
|
|
"playready": "playready",
|
|
}.get(drm.lower())
|
|
first = next(key for key in lower_keys if key in ("widevine", "playready"))
|
|
cdm_name = lower_keys.get(drm_key or first) or lower_keys.get("playready")
|
|
else:
|
|
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(config.cdm, "default")
|
|
if not cdm_name:
|
|
return None
|
|
|
|
from unshackle.core.cdm import load_cdm
|
|
|
|
if not isinstance(cdm_name, str):
|
|
return load_cdm(cdm_name, service_name=service, vaults=self.vaults)
|
|
|
|
cdm_cache: dict[tuple[str, Optional[str], str], Any] = getattr(self, "_cdm_cache", {})
|
|
self._cdm_cache = cdm_cache
|
|
|
|
cache_key = (service, profile, cdm_name)
|
|
if cache_key not in cdm_cache:
|
|
cdm_cache[cache_key] = load_cdm(cdm_name, service_name=service, vaults=self.vaults)
|
|
return cdm_cache[cache_key]
|