3 Commits

Author SHA1 Message Date
Andy
9921690339 feat: add service-specific configuration overrides
Add support for per-service configuration overrides allowing fine-tuned control of downloader and command options on a service-by-service basis.

Fixes #13
2025-10-18 07:32:17 +00:00
Andy
3dd12b0cbe chore(api): fix import ordering in download_manager and handlers 2025-10-18 07:05:05 +00:00
Andy
ed1314572b feat(dl): add --latest-episode option to download only the most recent episode
Adds a new CLI option `-le, --latest-episode` that automatically selects and downloads only the single most recent episode from a series, regardless of which season it's in.

Fixes #28
2025-10-18 07:04:11 +00:00
4 changed files with 283 additions and 184 deletions

View File

@@ -152,6 +152,13 @@ class dl:
default=None,
help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.",
)
@click.option(
"-le",
"--latest-episode",
is_flag=True,
default=False,
help="Download only the single most recent episode available.",
)
@click.option(
"-l",
"--lang",
@@ -322,11 +329,7 @@ class dl:
debug_log_path = config.directories.logs / config.filenames.debug_log.format_map(
defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S"))
)
init_debug_logger(
log_path=debug_log_path,
enabled=True,
log_keys=config.debug_keys
)
init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys)
self.debug_logger = get_debug_logger()
if self.debug_logger:
@@ -342,8 +345,12 @@ class dl:
"tmdb_id": tmdb_id,
"tmdb_name": tmdb_name,
"tmdb_year": tmdb_year,
"cli_params": {k: v for k, v in ctx.params.items() if k not in ['profile', 'proxy', 'tag', 'tmdb_id', 'tmdb_name', 'tmdb_year']}
}
"cli_params": {
k: v
for k, v in ctx.params.items()
if k not in ["profile", "proxy", "tag", "tmdb_id", "tmdb_name", "tmdb_year"]
},
},
)
else:
self.debug_logger = None
@@ -361,7 +368,7 @@ class dl:
level="DEBUG",
operation="load_service_config",
service=self.service,
context={"config_path": str(service_config_path), "config": self.service_config}
context={"config_path": str(service_config_path), "config": self.service_config},
)
else:
self.service_config = {}
@@ -373,6 +380,33 @@ class dl:
if getattr(config, "decryption_map", None):
config.decryption = config.decryption_map.get(self.service, config.decryption)
service_config = config.services.get(self.service, {})
reserved_keys = {
"profiles",
"api_key",
"certificate",
"api_endpoint",
"region",
"device",
"endpoints",
"client",
}
for config_key, override_value in service_config.items():
if config_key in reserved_keys:
continue
if isinstance(override_value, dict) and hasattr(config, config_key):
current_config = getattr(config, config_key, {})
if isinstance(current_config, dict):
merged_config = {**current_config, **override_value}
setattr(config, config_key, merged_config)
self.log.debug(
f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}"
)
with console.status("Loading Key Vaults...", spinner="dots"):
self.vaults = Vaults(self.service)
total_vaults = len(config.key_vaults)
@@ -438,19 +472,25 @@ class dl:
cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level}
elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]:
self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})")
cdm_info = {"type": "Widevine", "system_id": self.cdm.system_id, "security_level": self.cdm.security_level, "device_type": self.cdm.device_type.name}
cdm_info = {
"type": "Widevine",
"system_id": self.cdm.system_id,
"security_level": self.cdm.security_level,
"device_type": self.cdm.device_type.name,
}
else:
self.log.info(
f"Loaded PlayReady CDM: {self.cdm.certificate_chain.get_name()} (L{self.cdm.security_level})"
)
cdm_info = {"type": "PlayReady", "certificate": self.cdm.certificate_chain.get_name(), "security_level": self.cdm.security_level}
cdm_info = {
"type": "PlayReady",
"certificate": self.cdm.certificate_chain.get_name(),
"security_level": self.cdm.security_level,
}
if self.debug_logger and cdm_info:
self.debug_logger.log(
level="INFO",
operation="load_cdm",
service=self.service,
context={"cdm": cdm_info}
level="INFO", operation="load_cdm", service=self.service, context={"cdm": cdm_info}
)
self.proxy_providers = []
@@ -526,6 +566,7 @@ class dl:
channels: float,
no_atmos: bool,
wanted: list[str],
latest_episode: bool,
lang: list[str],
v_lang: list[str],
a_lang: list[str],
@@ -587,8 +628,8 @@ class dl:
context={
"cdm_only": cdm_only,
"vaults_only": vaults_only,
"mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults")
}
"mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"),
},
)
with console.status("Authenticating with Service...", spinner="dots"):
@@ -606,16 +647,13 @@ class dl:
context={
"has_cookies": bool(cookies),
"has_credentials": bool(credential),
"profile": self.profile
}
"profile": self.profile,
},
)
except Exception as e:
if self.debug_logger:
self.debug_logger.log_error(
"authenticate",
e,
service=self.service,
context={"profile": self.profile}
"authenticate", e, service=self.service, context={"profile": self.profile}
)
raise
@@ -630,31 +668,24 @@ class dl:
operation="get_titles",
service=self.service,
message="No titles returned from service",
success=False
success=False,
)
sys.exit(1)
except Exception as e:
if self.debug_logger:
self.debug_logger.log_error(
"get_titles",
e,
service=self.service
)
self.debug_logger.log_error("get_titles", e, service=self.service)
raise
if self.debug_logger:
titles_info = {
"type": titles.__class__.__name__,
"count": len(titles) if hasattr(titles, "__len__") else 1,
"title": str(titles)
"title": str(titles),
}
if hasattr(titles, "seasons"):
titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0
self.debug_logger.log(
level="INFO",
operation="get_titles",
service=self.service,
context={"titles": titles_info}
level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info}
)
if self.tmdb_year and self.tmdb_id:
@@ -674,8 +705,21 @@ class dl:
if list_titles:
return
# Determine the latest episode if --latest-episode is set
latest_episode_id = None
if latest_episode and isinstance(titles, Series) and len(titles) > 0:
# Series is already sorted by (season, number, year)
# The last episode in the sorted list is the latest
latest_ep = titles[-1]
latest_episode_id = f"{latest_ep.season}x{latest_ep.number}"
self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}")
for i, title in enumerate(titles):
if isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted:
if isinstance(title, Episode) and latest_episode and latest_episode_id:
# If --latest-episode is set, only process the latest episode
if f"{title.season}x{title.number}" != latest_episode_id:
continue
elif isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted:
continue
console.print(Padding(Rule(f"[rule.text]{title}"), (1, 2)))
@@ -750,10 +794,7 @@ class dl:
except Exception as e:
if self.debug_logger:
self.debug_logger.log_error(
"get_tracks",
e,
service=self.service,
context={"title": str(title)}
"get_tracks", e, service=self.service, context={"title": str(title)}
)
raise
@@ -764,34 +805,40 @@ class dl:
"audio_tracks": len(title.tracks.audio),
"subtitle_tracks": len(title.tracks.subtitles),
"has_chapters": bool(title.tracks.chapters),
"videos": [{
"codec": str(v.codec),
"resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown",
"bitrate": v.bitrate,
"range": str(v.range),
"language": str(v.language) if v.language else None,
"drm": [str(type(d).__name__) for d in v.drm] if v.drm else []
} for v in title.tracks.videos],
"audio": [{
"codec": str(a.codec),
"bitrate": a.bitrate,
"channels": a.channels,
"language": str(a.language) if a.language else None,
"descriptive": a.descriptive,
"drm": [str(type(d).__name__) for d in a.drm] if a.drm else []
} for a in title.tracks.audio],
"subtitles": [{
"codec": str(s.codec),
"language": str(s.language) if s.language else None,
"forced": s.forced,
"sdh": s.sdh
} for s in title.tracks.subtitles]
"videos": [
{
"codec": str(v.codec),
"resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown",
"bitrate": v.bitrate,
"range": str(v.range),
"language": str(v.language) if v.language else None,
"drm": [str(type(d).__name__) for d in v.drm] if v.drm else [],
}
for v in title.tracks.videos
],
"audio": [
{
"codec": str(a.codec),
"bitrate": a.bitrate,
"channels": a.channels,
"language": str(a.language) if a.language else None,
"descriptive": a.descriptive,
"drm": [str(type(d).__name__) for d in a.drm] if a.drm else [],
}
for a in title.tracks.audio
],
"subtitles": [
{
"codec": str(s.codec),
"language": str(s.language) if s.language else None,
"forced": s.forced,
"sdh": s.sdh,
}
for s in title.tracks.subtitles
],
}
self.debug_logger.log(
level="INFO",
operation="get_tracks",
service=self.service,
context=tracks_info
level="INFO", operation="get_tracks", service=self.service, context=tracks_info
)
# strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available
@@ -1185,7 +1232,7 @@ class dl:
operation="download_tracks",
service=self.service,
message="Download cancelled by user",
context={"title": str(title)}
context={"title": str(title)},
)
return
except Exception as e: # noqa
@@ -1219,8 +1266,8 @@ class dl:
"title": str(title),
"error_type": type(e).__name__,
"tracks_count": len(title.tracks),
"returncode": getattr(e, "returncode", None)
}
"returncode": getattr(e, "returncode", None),
},
)
return
@@ -1475,9 +1522,13 @@ class dl:
if not no_folder and isinstance(title, (Episode, Song)):
# Create folder based on title
# Use first available track for filename generation
sample_track = title.tracks.videos[0] if title.tracks.videos else (
title.tracks.audio[0] if title.tracks.audio else (
title.tracks.subtitles[0] if title.tracks.subtitles else None
sample_track = (
title.tracks.videos[0]
if title.tracks.videos
else (
title.tracks.audio[0]
if title.tracks.audio
else (title.tracks.subtitles[0] if title.tracks.subtitles else None)
)
)
if sample_track and sample_track.path:
@@ -1498,7 +1549,9 @@ class dl:
track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}"
elif isinstance(track, Audio):
lang_suffix = f".{track.language}" if track.language else ""
track_suffix = f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}"
track_suffix = (
f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}"
)
elif isinstance(track, Subtitle):
lang_suffix = f".{track.language}" if track.language else ""
forced_suffix = ".forced" if track.forced else ""
@@ -1595,8 +1648,8 @@ class dl:
"title": str(title),
"pssh": drm.pssh.dumps() if drm.pssh else None,
"kids": [k.hex for k in drm.kids],
"track_kid": track_kid.hex if track_kid else None
}
"track_kid": track_kid.hex if track_kid else None,
},
)
with self.DRM_TABLE_LOCK:
@@ -1637,8 +1690,8 @@ class dl:
"kid": kid.hex,
"content_key": content_key,
"track": str(track),
"from_cache": True
}
"from_cache": True,
},
)
elif vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
@@ -1651,7 +1704,7 @@ class dl:
operation="vault_key_not_found",
service=self.service,
message=msg,
context={"kid": kid.hex, "track": str(track)}
context={"kid": kid.hex, "track": str(track)},
)
raise Widevine.Exceptions.CEKNotFound(msg)
else:
@@ -1671,8 +1724,8 @@ class dl:
message="Requesting Widevine license from service",
context={
"track": str(track),
"kids_needed": [k.hex for k in all_kids if k not in drm.content_keys]
}
"kids_needed": [k.hex for k in all_kids if k not in drm.content_keys],
},
)
try:
@@ -1693,10 +1746,7 @@ class dl:
"get_license",
e,
service=self.service,
context={
"track": str(track),
"exception_type": type(e).__name__
}
context={"track": str(track), "exception_type": type(e).__name__},
)
raise e
@@ -1708,8 +1758,8 @@ class dl:
context={
"track": str(track),
"keys_count": len(drm.content_keys),
"kids": [k.hex for k in drm.content_keys.keys()]
}
"kids": [k.hex for k in drm.content_keys.keys()],
},
)
for kid_, key in drm.content_keys.items():
@@ -1767,8 +1817,8 @@ class dl:
"title": str(title),
"pssh": drm.pssh_b64 or "",
"kids": [k.hex for k in drm.kids],
"track_kid": track_kid.hex if track_kid else None
}
"track_kid": track_kid.hex if track_kid else None,
},
)
with self.DRM_TABLE_LOCK:
@@ -1816,8 +1866,8 @@ class dl:
"content_key": content_key,
"track": str(track),
"from_cache": True,
"drm_type": "PlayReady"
}
"drm_type": "PlayReady",
},
)
elif vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
@@ -1830,7 +1880,7 @@ class dl:
operation="vault_key_not_found",
service=self.service,
message=msg,
context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"}
context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"},
)
raise PlayReady.Exceptions.CEKNotFound(msg)
else:
@@ -1860,8 +1910,8 @@ class dl:
context={
"track": str(track),
"exception_type": type(e).__name__,
"drm_type": "PlayReady"
}
"drm_type": "PlayReady",
},
)
raise e
@@ -1937,7 +1987,7 @@ class dl:
@staticmethod
def save_cookies(path: Path, cookies: CookieJar):
if hasattr(cookies, 'jar'):
if hasattr(cookies, "jar"):
cookies = cookies.jar
cookie_jar = MozillaCookieJar(path)
@@ -2084,12 +2134,12 @@ class dl:
else:
return RemoteCdm(
device_type=cdm_api['Device Type'],
system_id=cdm_api['System ID'],
security_level=cdm_api['Security Level'],
host=cdm_api['Host'],
secret=cdm_api['Secret'],
device_name=cdm_api['Device Name'],
device_type=cdm_api["Device Type"],
system_id=cdm_api["System ID"],
security_level=cdm_api["Security Level"],
host=cdm_api["Host"],
secret=cdm_api["Secret"],
device_name=cdm_api["Device Name"],
)
prd_path = config.directories.prds / f"{cdm_name}.prd"

View File

@@ -6,11 +6,11 @@ import sys
import tempfile
import threading
import uuid
from contextlib import suppress
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Callable, Dict, List, Optional
from datetime import datetime, timedelta
from contextlib import suppress
log = logging.getLogger("download_manager")
@@ -87,14 +87,15 @@ def _perform_download(
if cancel_event and cancel_event.is_set():
raise Exception(f"Job was cancelled {stage}")
from contextlib import redirect_stderr, redirect_stdout
from io import StringIO
from contextlib import redirect_stdout, redirect_stderr
_check_cancel("before execution started")
# Import dl.py components lazily to avoid circular deps during module import
import click
import yaml
from unshackle.commands.dl import dl
from unshackle.core.config import config
from unshackle.core.services import Services

View File

@@ -20,7 +20,6 @@ def initialize_proxy_providers() -> List[Any]:
proxy_providers = []
try:
from unshackle.core import binaries
# Load the main unshackle config to get proxy provider settings
from unshackle.core.config import config as main_config

View File

@@ -34,21 +34,23 @@ title_cache_max_retention: 86400 # Maximum cache retention for fallback when API
# Debug logging configuration
# Comprehensive JSON-based debug logging for troubleshooting and service development
debug: false # Enable structured JSON debug logging (default: false)
# When enabled with --debug flag or set to true:
# - Creates JSON Lines (.jsonl) log files with complete debugging context
# - Logs: session info, CLI params, service config, CDM details, authentication,
# titles, tracks metadata, DRM operations, vault queries, errors with stack traces
# - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl
# - Also creates text log: logs/unshackle_root_{timestamp}.log
debug:
false # Enable structured JSON debug logging (default: false)
# When enabled with --debug flag or set to true:
# - Creates JSON Lines (.jsonl) log files with complete debugging context
# - Logs: session info, CLI params, service config, CDM details, authentication,
# titles, tracks metadata, DRM operations, vault queries, errors with stack traces
# - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl
# - Also creates text log: logs/unshackle_root_{timestamp}.log
debug_keys: false # Log decryption keys in debug logs (default: false)
# Set to true to include actual decryption keys in logs
# Useful for debugging key retrieval and decryption issues
# SECURITY NOTE: Passwords, tokens, cookies, and session tokens
# are ALWAYS redacted regardless of this setting
# Only affects: content_key, key fields (the actual CEKs)
# Never affects: kid, keys_count, key_id (metadata is always logged)
debug_keys:
false # Log decryption keys in debug logs (default: false)
# Set to true to include actual decryption keys in logs
# Useful for debugging key retrieval and decryption issues
# SECURITY NOTE: Passwords, tokens, cookies, and session tokens
# are ALWAYS redacted regardless of this setting
# Only affects: content_key, key fields (the actual CEKs)
# Never affects: kid, keys_count, key_id (metadata is always logged)
# Muxing configuration
muxing:
@@ -128,72 +130,72 @@ cdm:
# Use pywidevine Serve-compliant Remote CDMs
# Example: Custom CDM API Configuration
# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format
# - name: "chrome"
# type: "custom_api"
# host: "http://remotecdm.test/"
# timeout: 30
# device:
# name: "ChromeCDM"
# type: "CHROME"
# system_id: 34312
# security_level: 3
# auth:
# type: "header"
# header_name: "x-api-key"
# key: "YOUR_API_KEY_HERE"
# custom_headers:
# User-Agent: "Unshackle/2.0.0"
# endpoints:
# get_request:
# path: "/get-challenge"
# method: "POST"
# timeout: 30
# decrypt_response:
# path: "/get-keys"
# method: "POST"
# timeout: 30
# request_mapping:
# get_request:
# param_names:
# scheme: "device"
# init_data: "init_data"
# static_params:
# scheme: "Widevine"
# decrypt_response:
# param_names:
# scheme: "device"
# license_request: "license_request"
# license_response: "license_response"
# static_params:
# scheme: "Widevine"
# response_mapping:
# get_request:
# fields:
# challenge: "challenge"
# session_id: "session_id"
# message: "message"
# message_type: "message_type"
# response_types:
# - condition: "message_type == 'license-request'"
# type: "license_request"
# success_conditions:
# - "message == 'success'"
# decrypt_response:
# fields:
# keys: "keys"
# message: "message"
# key_fields:
# kid: "kid"
# key: "key"
# type: "type"
# success_conditions:
# - "message == 'success'"
# caching:
# enabled: true
# use_vaults: true
# check_cached_first: true
# Example: Custom CDM API Configuration
# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format
# - name: "chrome"
# type: "custom_api"
# host: "http://remotecdm.test/"
# timeout: 30
# device:
# name: "ChromeCDM"
# type: "CHROME"
# system_id: 34312
# security_level: 3
# auth:
# type: "header"
# header_name: "x-api-key"
# key: "YOUR_API_KEY_HERE"
# custom_headers:
# User-Agent: "Unshackle/2.0.0"
# endpoints:
# get_request:
# path: "/get-challenge"
# method: "POST"
# timeout: 30
# decrypt_response:
# path: "/get-keys"
# method: "POST"
# timeout: 30
# request_mapping:
# get_request:
# param_names:
# scheme: "device"
# init_data: "init_data"
# static_params:
# scheme: "Widevine"
# decrypt_response:
# param_names:
# scheme: "device"
# license_request: "license_request"
# license_response: "license_response"
# static_params:
# scheme: "Widevine"
# response_mapping:
# get_request:
# fields:
# challenge: "challenge"
# session_id: "session_id"
# message: "message"
# message_type: "message_type"
# response_types:
# - condition: "message_type == 'license-request'"
# type: "license_request"
# success_conditions:
# - "message == 'success'"
# decrypt_response:
# fields:
# keys: "keys"
# message: "message"
# key_fields:
# kid: "kid"
# key: "key"
# type: "type"
# success_conditions:
# - "message == 'success'"
# caching:
# enabled: true
# use_vaults: true
# check_cached_first: true
remote_cdm:
- name: "chrome"
@@ -360,9 +362,13 @@ services:
# Service-specific configuration goes here
# Profile-specific configurations can be nested under service names
# Example: with profile-specific device configs
# You can override ANY global configuration option on a per-service basis
# This allows fine-tuned control for services with special requirements
# Supported overrides: dl, aria2c, n_m3u8dl_re, curl_impersonate, subtitle, muxing, headers, etc.
# Example: Comprehensive service configuration showing all features
EXAMPLE:
# Global service config
# Standard service config
api_key: "service_api_key"
# Service certificate for Widevine L1/L2 (base64 encoded)
@@ -383,6 +389,42 @@ services:
app_name: "AIV"
device_model: "Fire TV Stick 4K"
# NEW: Configuration overrides (can be combined with profiles and certificates)
# Override dl command defaults for this service
dl:
downloads: 4 # Limit concurrent track downloads (global default: 6)
workers: 8 # Reduce workers per track (global default: 16)
lang: ["en", "es-419"] # Different language priority for this service
sub_format: srt # Force SRT subtitle format
# Override n_m3u8dl_re downloader settings
n_m3u8dl_re:
thread_count: 8 # Lower thread count for rate-limited service (global default: 16)
use_proxy: true # Force proxy usage for this service
retry_count: 10 # More retries for unstable connections
ad_keyword: "advertisement" # Service-specific ad filtering
# Override aria2c downloader settings
aria2c:
max_concurrent_downloads: 2 # Limit concurrent downloads (global default: 4)
max_connection_per_server: 1 # Single connection per server
split: 3 # Fewer splits (global default: 5)
file_allocation: none # Faster allocation for this service
# Override subtitle processing for this service
subtitle:
conversion_method: pycaption # Use specific subtitle converter
sdh_method: auto
# Service-specific headers
headers:
User-Agent: "Service-specific user agent string"
Accept-Language: "en-US,en;q=0.9"
# Override muxing options
muxing:
set_title: true
# Example: Service with different regions per profile
SERVICE_NAME:
profiles:
@@ -393,6 +435,13 @@ services:
region: "GB"
api_endpoint: "https://api.uk.service.com"
# Notes on service-specific overrides:
# - Overrides are merged with global config, not replaced
# - Only specified keys are overridden, others use global defaults
# - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides
# - Any dict-type config option can be overridden (dl, aria2c, n_m3u8dl_re, etc.)
# - Use --debug flag to see which overrides are applied during downloads
# External proxy provider services
proxy_providers:
nordvpn: