From 1d8cec6d2b61df8291374ae174f1231169f69636 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Fri, 3 Jul 2026 19:24:11 -0600 Subject: [PATCH] docs: restructure documentation into MkDocs Material site --- .github/workflows/docs.yml | 2 +- docs/ADVANCED_CONFIG.md | 257 ------ docs/API.md | 14 - docs/CREATING_A_SERVICE.md | 488 ---------- docs/DEBUG_LOGGING.md | 79 -- docs/DOWNLOAD_CONFIG.md | 326 ------- docs/DRM_CONFIG.md | 598 ------------- docs/GLUETUN.md | 322 ------- docs/NETWORK_CONFIG.md | 284 ------ docs/OUTPUT_CONFIG.md | 446 ---------- docs/README.md | 19 - docs/SERVICE_CONFIG.md | 302 ------- docs/SUBTITLE_CONFIG.md | 101 --- docs/api/authentication.md | 75 -- docs/api/client-configuration.md | 64 -- docs/api/download-job-lifecycle.md | 16 - docs/api/endpoint-map.md | 26 - docs/api/endpoints.md | 519 ----------- docs/api/error-responses.md | 28 - docs/api/quick-start.md | 37 - docs/api/remote-service-sessions.md | 329 ------- docs/assets/favicon.svg | 1 + docs/dev/creating-a-service.md | 634 +++++++++++++ docs/dev/rest-api/authentication.md | 311 +++++++ docs/dev/rest-api/endpoints.md | 980 +++++++++++++++++++++ docs/dev/rest-api/errors.md | 254 ++++++ docs/dev/rest-api/index.md | 237 +++++ docs/dev/rest-api/job-lifecycle.md | 412 +++++++++ docs/dev/rest-api/quickstart.md | 437 +++++++++ docs/dev/rest-api/remote-sessions.md | 431 +++++++++ docs/dev/service-architecture.md | 452 ++++++++++ docs/getting-started/configuration-file.md | 193 ++++ docs/getting-started/installation.md | 196 +++++ docs/getting-started/quickstart.md | 274 ++++++ docs/guide/cli-reference.md | 550 ++++++++++++ docs/guide/downloading.md | 563 ++++++++++++ docs/guide/drm-and-cdm.md | 524 +++++++++++ docs/guide/output-and-naming.md | 384 ++++++++ docs/guide/proxies-and-vpn.md | 639 ++++++++++++++ docs/guide/subtitles.md | 356 ++++++++ docs/guide/troubleshooting.md | 303 +++++++ docs/guide/vaults.md | 318 +++++++ docs/index.md | 92 ++ docs/reference/api/config.md | 12 + docs/reference/api/drm.md | 13 + docs/reference/api/service.md | 12 + docs/reference/api/titles.md | 14 + docs/reference/api/tracks.md | 14 + docs/reference/configuration.md | 730 +++++++++++++++ docs/requirements-docs.txt | 4 + mkdocs.yml | 82 +- 51 files changed, 9398 insertions(+), 4356 deletions(-) delete mode 100644 docs/ADVANCED_CONFIG.md delete mode 100644 docs/API.md delete mode 100644 docs/CREATING_A_SERVICE.md delete mode 100644 docs/DEBUG_LOGGING.md delete mode 100644 docs/DOWNLOAD_CONFIG.md delete mode 100644 docs/DRM_CONFIG.md delete mode 100644 docs/GLUETUN.md delete mode 100644 docs/NETWORK_CONFIG.md delete mode 100644 docs/OUTPUT_CONFIG.md delete mode 100644 docs/README.md delete mode 100644 docs/SERVICE_CONFIG.md delete mode 100644 docs/SUBTITLE_CONFIG.md delete mode 100644 docs/api/authentication.md delete mode 100644 docs/api/client-configuration.md delete mode 100644 docs/api/download-job-lifecycle.md delete mode 100644 docs/api/endpoint-map.md delete mode 100644 docs/api/endpoints.md delete mode 100644 docs/api/error-responses.md delete mode 100644 docs/api/quick-start.md delete mode 100644 docs/api/remote-service-sessions.md create mode 100644 docs/assets/favicon.svg create mode 100644 docs/dev/creating-a-service.md create mode 100644 docs/dev/rest-api/authentication.md create mode 100644 docs/dev/rest-api/endpoints.md create mode 100644 docs/dev/rest-api/errors.md create mode 100644 docs/dev/rest-api/index.md create mode 100644 docs/dev/rest-api/job-lifecycle.md create mode 100644 docs/dev/rest-api/quickstart.md create mode 100644 docs/dev/rest-api/remote-sessions.md create mode 100644 docs/dev/service-architecture.md create mode 100644 docs/getting-started/configuration-file.md create mode 100644 docs/getting-started/installation.md create mode 100644 docs/getting-started/quickstart.md create mode 100644 docs/guide/cli-reference.md create mode 100644 docs/guide/downloading.md create mode 100644 docs/guide/drm-and-cdm.md create mode 100644 docs/guide/output-and-naming.md create mode 100644 docs/guide/proxies-and-vpn.md create mode 100644 docs/guide/subtitles.md create mode 100644 docs/guide/troubleshooting.md create mode 100644 docs/guide/vaults.md create mode 100644 docs/index.md create mode 100644 docs/reference/api/config.md create mode 100644 docs/reference/api/drm.md create mode 100644 docs/reference/api/service.md create mode 100644 docs/reference/api/titles.md create mode 100644 docs/reference/api/tracks.md create mode 100644 docs/reference/configuration.md create mode 100644 docs/requirements-docs.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 25bfca01..e56a8db1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,5 +23,5 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.x" - - run: pip install mkdocs-material + - run: pip install -e . -r docs/requirements-docs.txt - run: mkdocs gh-deploy --remote-branch docs diff --git a/docs/ADVANCED_CONFIG.md b/docs/ADVANCED_CONFIG.md deleted file mode 100644 index daa427ff..00000000 --- a/docs/ADVANCED_CONFIG.md +++ /dev/null @@ -1,257 +0,0 @@ -# Advanced & System Configuration - -This document covers advanced features, debugging, and system-level configuration options. - -## serve (dict) - -Configuration for the integrated server that provides CDM endpoints (Widevine/PlayReady) and a REST API for remote downloading. - -Start the server with: - -```bash -unshackle serve # Default: localhost:8786 -unshackle serve -h 0.0.0.0 -p 8888 # Listen on all interfaces -unshackle serve --no-key # Disable authentication -unshackle serve --api-only # REST API only, no CDM endpoints -unshackle serve --remote-only # Only expose remote service session endpoints -``` - -### CLI Options - -| Option | Default | Description | -| --- | --- | --- | -| `-h, --host` | `127.0.0.1` | Host to serve from | -| `-p, --port` | `8786` | Port to serve from | -| `--caddy` | `false` | Also serve with Caddy reverse-proxy for HTTPS | -| `--api-only` | `false` | Serve only the REST API, disable CDM endpoints | -| `--no-widevine` | `false` | Disable Widevine CDM endpoints | -| `--no-playready` | `false` | Disable PlayReady CDM endpoints | -| `--no-key` | `false` | Disable API key authentication (allows all requests) | -| `--debug-api` | `false` | Include tracebacks and stderr in API error responses | -| `--debug` | `false` | Enable debug logging for API operations | -| `--remote-only` | `false` | Only expose remote service session endpoints (health, services, search, session) | - -### Configuration - -- `api_secret` - Secret key for REST API authentication. Required unless `--no-key` is used. All API requests must include this key via the `X-Secret-Key` header. -- `compression_level` - Compression level for API payloads (manifests, cache, cookies). `0`=off, `1`=fast, `6`=balanced, `9`=max. Default: `1`. -- `session_ttl` - Session inactivity timeout in seconds. Each request resets the timer. Default: `300`. -- `max_sessions` - Maximum concurrent sessions before the oldest is evicted. Default: `100`. -- `services` - Optional global service allowlist. Only these service tags are exposed. If omitted, all services are available. -- `devices` - List of Widevine device files (.wvd). If not specified, auto-populated from the WVDs directory. -- `playready_devices` - List of PlayReady device files (.prd). If not specified, auto-populated from the PRDs directory. -- `users` - Dictionary mapping user secret keys to their access configuration: - - `devices` - List of Widevine devices this user can access - - `playready_devices` - List of PlayReady devices this user can access - - `username` - Internal logging name for the user (not visible to users) - - `services` - Optional per-user service allowlist. Effective access is the intersection of global and per-user allowlists. - -#### Server-side `dl` defaults - -Any key accepted by `/api/download` (see `docs/API.md`) can also be declared directly under `serve:` and the REST API will treat it as a default. Per-request bodies still win. Use this to raise concurrency, force `best_available`, etc. without each client repeating the values: - -```yaml -serve: - api_secret: "..." - users: { ... } - downloads: 4 # parallel tracks per job - workers: 16 # threads per track - best_available: true - no_proxy_download: false -``` - -Layering: built-in defaults < `serve.*` overrides < service-specific defaults < request body. - -For example, - -```yaml -serve: - api_secret: "your-secret-key-here" - compression_level: 1 - session_ttl: 300 - max_sessions: 100 - # services: # global allowlist (optional) - # - EXAMPLE1 - # - EXAMPLE2 - users: - secret_key_for_jane: # 32bit hex recommended, case-sensitive - devices: # list of allowed Widevine devices for this user - - generic_nexus_4464_l3 - playready_devices: # list of allowed PlayReady devices for this user - - my_playready_device - username: jane # only for internal logging, users will not see this name - # services: # per-user allowlist (optional) - # - EXAMPLE1 - secret_key_for_james: - devices: - - generic_nexus_4464_l3 - username: james - # devices can be manually specified by path if you don't want to add it to - # unshackle's WVDs directory for whatever reason - # devices: - # - 'C:\Users\john\Devices\test_devices_001.wvd' - # playready_devices: - # - '/path/to/device.prd' -``` - -### REST API - -When the server is running, interactive API documentation is available at: - -- **Swagger UI**: `http://localhost:8786/api/docs/` - -See [API.md](API.md) for full REST API documentation with endpoints, parameters, and examples. - ---- - -## max_concurrent_downloads (int) - -Maximum number of `/api/download` jobs the serve queue manager will execute in parallel. Each job runs the full `dl` pipeline (authenticate, fetch tracks, decrypt, mux) in its own worker subprocess. This is independent of `serve.downloads`, which controls parallel tracks **inside** a single job. Default: `2`. - -```yaml -max_concurrent_downloads: 4 -``` - ---- - -## download_job_retention_hours (int) - -How long completed, failed, or cancelled download jobs remain queryable via `/api/download/jobs/{job_id}` before the periodic cleanup loop drops them. Default: `24`. - -```yaml -download_job_retention_hours: 48 -``` - ---- - -## serve.history_limit (int) - -How many terminal jobs to keep in the persistent history file (`api_history.jsonl`, stored under the cache directory so it stays out of the config/data tree). Older entries are dropped as new jobs finish. Set to `0` for unlimited. Default: `100`. - -```yaml -serve: - history_limit: 250 -``` - ---- - -## debug (bool) - -Enables comprehensive debug logging. Default: `false` - -When enabled (either via config or the `-d`/`--debug` CLI flag): -- Sets console log level to DEBUG for verbose output -- Creates JSON Lines (`.jsonl`) debug log files with structured logging -- Logs detailed information about sessions, service configuration, DRM operations, and errors with full stack traces - -For example, - -```yaml -debug: true -``` - ---- - -## debug_keys (bool) - -Controls whether actual decryption keys (CEKs) are included in debug logs. Default: `false` - -When enabled: -- Content encryption keys are logged in debug output -- Only affects `content_key` and `key` fields (the actual CEKs) -- Key metadata (`kid`, `keys_count`, `key_id`) is always logged regardless of this setting -- Passwords, tokens, cookies, and session tokens remain redacted even when enabled - -For example, - -```yaml -debug_keys: true -``` - ---- - -## debug_requests (bool) - -Controls whether HTTP-library logs are shown during `--debug` sessions. Default: `false` - -When `--debug` is active and this is `false`, unshackle silences the noisy per-request logs emitted by `urllib3`, `requests`, `rnet`, `httpx`, and related libraries (e.g. `GET ...seg-001.m4s 200`) so that unshackle's own structured DEBUG output stays readable. Set to `true` to see every individual HTTP request, including per-segment downloads. - -Has no effect when `--debug` is not active. - -```yaml -debug_requests: true -``` - ---- - -## set_terminal_bg (bool) - -Controls whether unshackle applies its theme background color to the terminal. Default: `false` - -When enabled, the Rich console theme's background color is appended to the foreground styles for both the general color scheme and the ASCII art banner, so the full color palette renders correctly on terminals whose default background differs from the theme. Set to `true` if the banner or colored output looks off on your terminal. - -```yaml -set_terminal_bg: true -``` - ---- - -## update_checks (bool) - -Check for updates from the GitHub repository on startup. Default: `true`. - ---- - -## update_check_interval (int) - -How often to check for updates, in hours. Default: `24`. - ---- - -## title_cache_enabled (bool) - -Enable or disable title metadata caching globally. Default: `true`. - ---- - -## title_cache_time (int) - -Title cache duration in seconds. Default: `1800` (30 minutes). - ---- - -## title_cache_max_retention (int) - -Maximum cache retention in seconds, used as fallback when the upstream API fails. Default: `86400` (24 hours). - ---- - -## unicode_filenames (bool) - -When `false`, replaces non-ASCII characters in output filenames with ASCII equivalents. Default: `false`. - ---- - -## ipinfo_api_key (str) - -Optional ipinfo.io token. When set, unshackle uses the ipinfo.io Lite endpoint for IP/geolocation lookups instead of the unauthenticated fallback. - ---- - -## tmdb_api_key (str) - -Optional TMDB API key, used for metadata enrichment and IMDb/TMDb tagging. - ---- - -## simkl_client_id (str) - -Optional Simkl client ID for metadata lookups. - ---- - -## decrypt_labs_api_key (str) - -Optional Decrypt Labs API key, used by services that integrate with the service. - ---- diff --git a/docs/API.md b/docs/API.md deleted file mode 100644 index 7bc891d6..00000000 --- a/docs/API.md +++ /dev/null @@ -1,14 +0,0 @@ -# REST API Documentation - -The unshackle REST API allows you to control downloads, search services, drive remote downloads from a thin client, and (optionally) co-host the pywidevine/pyplayready CDM. Start the server with `unshackle serve` and access the interactive Swagger UI at `http://localhost:8786/api/docs/`. - -The server is built on **aiohttp** (not FastAPI). Implementation lives in `unshackle/commands/serve.py` and `unshackle/core/api/` (`routes.py`, `handlers.py`, `session_store.py`, `input_bridge.py`, `download_manager.py`, `download_worker.py`). - -- [Quick Start](api/quick-start.md) -- [Authentication](api/authentication.md) -- [Endpoint Map](api/endpoint-map.md) -- [Endpoints](api/endpoints.md) -- [Remote Service Sessions](api/remote-service-sessions.md) -- [Error Responses](api/error-responses.md) -- [Download Job Lifecycle](api/download-job-lifecycle.md) -- [Client Configuration: `remote_services`](api/client-configuration.md) diff --git a/docs/CREATING_A_SERVICE.md b/docs/CREATING_A_SERVICE.md deleted file mode 100644 index ee934373..00000000 --- a/docs/CREATING_A_SERVICE.md +++ /dev/null @@ -1,488 +0,0 @@ -# Creating a Service - -This guide walks through writing a new service adapter, section by section, using the bundled -reference implementation at -[`unshackle/services/EXAMPLE/__init__.py`](https://github.com/unshackle-dl/unshackle/blob/main/unshackle/services/EXAMPLE/__init__.py) -as the canonical example. `EXAMPLE` is deliberately exhaustive - it shows every framework feature -in one file so you have one place to copy from. It does not run against a real API. - -## File layout - -A service is a directory under `unshackle/services/` named by its **tag** (the short code used on -the CLI - `EXAMPLE`, or its aliases `EX`/`DOMAIN`, in the reference service). The directory needs: - -``` -unshackle/services/TAG/ -├── __init__.py # required - the Service subclass -└── config.yaml # optional - endpoints, headers, client profiles -``` - -The class inside `__init__.py` **must** have the same name as the directory (`class TAG(Service):` -in `services/TAG/__init__.py`). `unshackle/core/services.py` discovers services by globbing -`services/*/__init__.py` and looking up a class named after the directory; a mismatch raises -`no class named '' found ... the class name must match the directory name` at load time, so -the failure is loud, not silent. - -## Lifecycle at a glance - -Before the method-by-method breakdown, here's the order the framework calls your service during a -download - your methods are the hooks, the framework drives the loop: - -``` -unshackle dl TAG - └─ cli(ctx, title=...) → constructs TAG(ctx, ...) - └─ __init__ → super().__init__(ctx) builds session/cache/track_request, resolves geofence proxy - 1. authenticate(cookies, credential) login / token exchange (always called; base just loads cookies/credential) - 2. get_titles() → Movies / Series / Album (wrapped by get_titles_cached + title_map) - · framework applies episode/season filters - 3. per title: - a. get_tracks(title) → Tracks (parse manifest, post-process, store license data) - b. get_chapters(title) → Chapters - c. framework selects tracks resolution / range / codec / language (best_available softens misses) - d. per track: download → license callback → decrypt → repackage → mux → move to final filename -``` - -Two things follow from this order: - -- **`get_tracks()` runs before any track selection**, so it must return *everything*; the framework - filters afterward in step (c). -- **The license callbacks** (`get_widevine_license` etc.) fire in step (d), long after - `get_tracks()` returned - they only receive `challenge`, `title`, and `track` (nothing else from - `get_tracks()`), so anything they need (license URL, session token) must be stashed during - `get_tracks()`. - -## Class docstring and class variables - -```python -class EXAMPLE(Service): - """ - Reference service for domain.com - ... - - Version: 2.0.0 - Author: sp4rk.y - Date: 2026-06-28 - Authorization: Cookies + Credentials - Geofence: US, UK - Robustness: - Widevine: - L1: 2160p, HDR10, HDR10+, DV - L3: 1080p, SDR - PlayReady: - SL3000: 2160p - ClearKey: 1080p (DRM-free fallback) - - Tips: - - Input may be a full URL or a bare ID/slug - """ - - ALIASES = ("EX", "DOMAIN") - GEOFENCE = ("US", "UK") - TITLE_RE = r"^(?:https?://(?:www\.)?domain\.com/details/)?(?P<title_id>[^/?#]+)" - NO_SUBTITLES = False - VAULT_TAG = "DIFFERENT_NAME" -``` - -The docstring **is** the `unshackle dl TAG --help` output, so it follows Click's conventions: - -- Keep `Version`, `Author`, `date`, `Authorization`, `Geofence`, and `Robustness` - they're the fields - users and other contributors check first when deciding whether a service still works. - -Class variables, all optional except where noted: - -| Variable | Purpose | -|---|---| -| `ALIASES: tuple[str, ...]` | Extra CLI tags that resolve to this service, e.g. `dl EX ...` | -| `GEOFENCE: tuple[str, ...]` | ISO country codes required; the framework warns/auto-proxies on mismatch | -| `TITLE_RE: str` | Regex with named groups (e.g. `(?P<title_id>...)`) used in `get_titles()` to accept either a full URL or a bare ID | -| `NO_SUBTITLES: bool` | Convention checked via `hasattr` by `dl.py`, not declared on the base class - set `True` if the service has no subtitle tracks, to skip subtitle handling | -| `VAULT_TAG: Optional[str]` | Store/read keys under a different vault namespace than this service's own tag - lets sibling services (e.g. regional variants of the same backend) share one key vault | -| `AUTH_METHODS: tuple[str, ...]` | Declares which of `"cookies"`/`"credentials"` the service accepts. Optional - when unset, `unshackle serve`'s `/services` endpoint infers it by inspecting `authenticate()`'s source. Set it explicitly if that inference would guess wrong | - -See [Service Integration & Authentication](SERVICE_CONFIG.md#service-class-conventions) for the -full list of `self.*` helpers available once `super().__init__(ctx)` has run. - -## `cli()` - the Click entry point - -```python -@staticmethod -@click.command(name="EXAMPLE", short_help="https://domain.com", help=__doc__) -@click.argument("title", type=str) -@click.option("-m", "--movie", is_flag=True, default=False, help="Treat the title as a movie.") -@click.option( - "-d", "--device", - type=click.Choice(["android_tv", "web", "ios"], case_sensitive=False), - default="android_tv", - help="Client profile block to use from config.yaml.", -) -@click.pass_context -def cli(ctx: click.Context, **kwargs: Any) -> EXAMPLE: - return EXAMPLE(ctx, **kwargs) -``` - -Keep `name="EXAMPLE"` in `@click.command` matching the class/directory name - resolution of -`unshackle dl EXAMPLE ...` is actually driven by the directory name matching the class name (see -File layout above), but a mismatched `name=` produces confusing `--help`/usage output. -`help=__doc__` reuses the class docstring above as the `--help` text. Add `@click.option` entries here for anything a user should be able to toggle per -download (device/client profile, movie-vs-series disambiguation, region, etc.) - they arrive as -keyword arguments into `__init__`. - -## `__init__()` - -```python -def __init__(self, ctx: click.Context, title: str, movie: bool, device: str): - self.title = title - self.movie = movie - self.device = device - - super().__init__(ctx) - - self.cdm = ctx.obj.cdm - self.is_playready = is_playready_cdm(self.cdm) - self.is_widevine = is_widevine_cdm(self.cdm) - - if self.is_playready: - self.device = "playready_tv" - elif self.is_widevine: - self.log.info(" + Widevine CDM detected") - - # The active -p/--profile, used below to key per-profile token caches. - self.profile = (ctx.parent.params.get("profile") if ctx.parent else None) or "default" - - if any(r != Video.Range.SDR for r in self.track_request.ranges): - self.track_request.codecs = [Video.Codec.HEVC] - - if self.is_widevine and getattr(self.cdm, "security_level", None) == 3: - self.track_request.codecs = [Video.Codec.AVC] - self.track_request.ranges = [Video.Range.SDR] - - if self.config is None: - raise EnvironmentError("config.yaml is missing for this service.") -``` - -Order matters here: - -1. Store CLI args **before** `super().__init__(ctx)` only if base-class init needs them; otherwise - it doesn't matter, but storing first is the convention every service follows. -2. `super().__init__(ctx)` wires up `self.config`, `self.log`, `self.session`, `self.cache`, - `self.title_cache`, `self.current_region`, and builds `self.track_request` from the global `dl` - flags. Nothing below this line works without it. (`self.request_input` is a base-class method, - always available - it isn't assigned here.) -3. After that, inspect `ctx.obj.cdm` and classify it with `is_playready_cdm`/`is_widevine_cdm` - (`unshackle.core.cdm.detect`) - never hand-roll `isinstance` checks, these helpers also handle - remote/wrapper CDMs. -4. Read or rewrite `self.track_request` for service-level constraints: e.g. force HEVC when any - non-SDR range was requested, or clamp a Widevine L3 box down to AVC/SDR. - -`self.track_request` fields: `codecs: list[Video.Codec]` (empty = no preference), -`ranges: list[Video.Range]` (defaults to `[SDR]`), `best_available: bool`. Read codec/range -preferences from `self.track_request`, not from `ctx.parent.params` directly - it's the same data, -but `track_request` is what `_get_tracks_for_variants()` and the rest of the framework consult. - -## `self.session` and `get_session()` - -`super().__init__(ctx)` sets `self.session = self.get_session()`. The **base** `get_session()` is a -plain `requests.Session` with `config.headers` applied and a retry adapter mounted (5 retries, -backoff on `429/500/502/503/504`) - it does **not** use rnet/TLS-impersonation by default. If a -service needs TLS fingerprinting to get past bot detection, override `get_session()`: - -```python -from unshackle.core.session import session - -@staticmethod -def get_session() -> RnetSession: - return session("Chrome131") # any rnet.Impersonate preset name, e.g. OkHttp4_12, Edge101, Safari18 -``` - -`RnetSession` is a drop-in `requests.Session` replacement (same `.get`/`.post`/cookie jar -interface), so nothing else in the service needs to change. For custom SSL cipher requirements -instead, mount `unshackle.core.utils.sslciphers.SSLCiphers` on a plain `requests.Session` the same -way. - -## `authenticate()` - -```python -def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: - super().authenticate(cookies, credential) - self.session.headers.update({"user-agent": self.config["client"][self.device]["user_agent"]}) - - cache = self.cache.get(f"tokens_{self.device}_{self.profile}") - if cache and cache.data.get("expires_in", 0) > int(datetime.now().timestamp()): - self.log.info(" + Using cached tokens") - elif cache and cache.data.get("refresh_token"): - refresh = self.session.post(url=self.config["endpoints"]["refresh"], data=...).json() - cache.set(data=refresh, expiration=refresh.get("expires_in")) - else: - if cookies: - ... # decode a JWT cookie - elif credential: - otp = self.request_input("Enter the OTP sent to your device: ") - ... - else: - raise EnvironmentError("Service requires either Cookies or Credentials.") - token = self.session.post(url=self.config["endpoints"]["login"], data=body).json() - cache.set(data=token, expiration=token.get("expires_in")) - - self.token = cache.data["token"] -``` - -Skip this method entirely for services that need no auth at all. Otherwise: - -- **Always call `super().authenticate(cookies, credential)` first** - it loads the cookie jar into - `self.session` and stores `self.credential`. -- Cache tokens with `self.cache.get(key)` / `cache.set(data, expiration=seconds)`. Key the cache - by whatever varies per session (device, profile, or `credential.sha1`) so multiple profiles - don't collide. -- Use `self.request_input(prompt)` instead of bare `input()` for anything interactive (OTP, 2FA - codes) - locally it prompts via the shared Rich console (`console.input`, so the prompt renders - correctly alongside progress/log output), but under `serve` the attached `InputBridge` relays the - prompt to the remote client instead of hanging the server. -- Raise `EnvironmentError` if auth is required but neither cookies nor a credential were given. -- Don't hold onto the raw `Credential` object past this method - store the derived token instead. - -## `search()` - -```python -def search(self) -> Generator[SearchResult, None, None]: - results = self.session.get(url=self.config["endpoints"]["search"], params={"q": self.title}).json() - for result in results["entries"]: - yield SearchResult( - id_=result["id"], - title=result["title"], - description=result.get("description"), - label="SERIES" if result["programType"] == "series" else result["programType"].upper(), - url=result.get("url"), - ) -``` - -Optional - implement it to support `unshackle search TAG <query>`. A generator yielding -`SearchResult` objects; nothing else consumes its return value, so don't collect to a list first. - -## `get_titles()` - -```python -def get_titles(self) -> Titles_T: - match = re.match(self.TITLE_RE, self.title) - if not match: - raise ValueError("Could not parse a title ID - is the URL/ID correct?") - title_id = match.group("title_id") - - metadata = self.session.get(url=self.config["endpoints"]["metadata"].format(title_id=title_id)).json() - original_lang = Language.find(metadata["languages"][0]) - - if self.movie or metadata["programType"] == "movie": - return Movies([ - Movie( - id_=metadata["id"], - service=self.__class__, - name=metadata["title"], - year=metadata.get("releaseYear") or None, - language=original_lang, - data=metadata, - ) - ]) - - episodes = [...] # flatten seasons into Episode(...) the same way - return Series(episodes) -``` - -Required. Returns `Movies`, `Series`, or (for music) `Album` - never a raw list. - -- Parse `self.title` with `TITLE_RE` first; raise `ValueError` with a clear message if it doesn't - match. -- `language=` on each `Movie`/`Episode`/`Song` must be the title's **original** recorded language, - not the user's preferred language - it's the source of truth `to_tracks()` uses later to flag - `is_original_lang` on every track, which drives `-l best/all` selection and the "Original" audio - label. -- `service=self.__class__`, not `service=self` - a common copy-paste mistake. -- Pass the full API response through `data=` on the title. `get_tracks()` and `get_chapters()` - read it back as `title.data` - this is the standard way to avoid a second request later. - -## `get_tracks()` - -Two shapes, depending on what the service's API gives you: - -**Pattern A - separate manifest per codec/range.** Use the `_get_tracks_for_variants()` helper: - -```python -def get_tracks(self, title: Title_T) -> Tracks: - def _fetch_variant(title, codec, range_): - vcodec_str = "H265" if codec == Video.Codec.HEVC else "H264" - return self._fetch_dash_manifest(title, vcodec=vcodec_str, range_=range_) - return self._get_tracks_for_variants(title, _fetch_variant) -``` - -The helper walks every `codec x range` combination from `self.track_request`, handles `HYBRID` -(fetches HDR10 and DV separately and merges them), and - when `--best-available` is set - skips -combos the service can't deliver instead of raising. - -**Pattern B - one manifest contains every variant.** Just parse and return; `dl.py` filters by -the user's selection: - -```python -def get_tracks(self, title: Title_T) -> Tracks: - playback = self.session.get(url=self.config["endpoints"]["playback"].format(title_id=title.id)).json() - return HLS.from_url(url=playback["manifest_url"], session=self.session).to_tracks(language=title.language) -``` - -`ISM` (Microsoft Smooth Streaming) follows the same shape as `HLS` here. - -Whichever pattern you use, the manifest fetch is also where you post-process tracks before -returning them - see `_fetch_dash_manifest` in `EXAMPLE` for the full set of fixups: - -- **Store DRM/license data** on `self` (e.g. `self.license_data = {...}`) or on `title.data` - - `get_widevine_license()` only receives `title` and `track`, not the manifest response, so this - is the only place to capture license URLs/tokens. -- **Stamp `video.range`** - manifests can't always be trusted to signal range correctly; set it - explicitly to what you asked for. -- **Flip HDR10 → HDR10+** when the service is known to embed HDR10+ SEI but only labels HDR10 in - the manifest - this is a bitstream feature the HLS parser intentionally does not infer. -- **Set `video.dv_compatible_bitstream = True`** for HEVC tracks carrying DV RPU NALs in a - container that only signals plain HEVC - `DVFixup` then round-trips the bitstream through - `dovi_tool` before mux so the output is recognised as Dolby Vision. -- **Drop decoy renditions** (e.g. unencrypted "clear" copies some APIs return alongside the real - ones). -- **Mark descriptive audio** from the manifest's `Role`/label data (`audio.descriptive = True`). -- **Set `audio.joc`** when the API tells you a track is Dolby Atmos explicitly (the framework - also detects JOC itself from the bitstream). -- **Build subtitles by hand** when the manifest's own subtitle list is wrong or incomplete - in - that case compute `is_original_lang` yourself with `is_close_match(sub_lang, [title.language])`, - the same helper `to_tracks()` uses internally. -- **Add cover art** as an `Attachment` if the API provides one. - -```python -def get_tracks(self, title: Title_T) -> Tracks: - # ... never filter by resolution/codec/language here - return everything, - # the framework applies the user's selection after this returns. -``` - -This is the single most common service bug: filtering inside `get_tracks()`. Don't. - -## `get_chapters()` - -```python -def get_chapters(self, title: Title_T) -> Chapters: - chapters = [] - seen = set() - for marker in title.data.get("markers", []): - ts = marker["start"] # seconds, ms, or "HH:MM:SS.mmm" - if ts <= 0 or ts in seen: # skip a marker at 0 and any duplicate timestamp - continue - seen.add(ts) - chapters.append(Chapter(timestamp=ts, name=marker.get("label"))) - return Chapters(chapters) -``` - -Return `Chapters()` (or `[]`) if the service has no chapter data - don't skip implementing the -method. Timestamps accept float seconds, int milliseconds, or `"HH:MM:SS.mmm"` strings. Prefer -descriptive names ("Intro", "End Credits") over generic ones ("Chapter 01"). - -**The opening chapter at `00:00:00.000` is created for you - never add your own.** The moment you -add the first `Chapter`, `Chapters.add()` injects a `Chapter(0)` at the start if one isn't already -present, so every file gets an opening chapter automatically. Two consequences you must handle: - -- **Don't add a "Chapter 1" at timestamp 0 yourself** - it's redundant with the auto-inserted one. -- **Guard against duplicate timestamps.** `Chapters.add()` raises `ValueError` if a chapter already - exists at that exact timestamp, so a marker the API reports at `0` (an intro/recap that starts at - the very beginning) collides with the auto-inserted opening chapter, and two markers at the same - time collide with each other. Skip any marker at `0` and de-duplicate timestamps before adding, - as the `seen` set above does. - -A `Chapter` with no `name` is valid - it inserts an unnamed chapter break at that timestamp, which -is the standard way to close out a named range (e.g. mark the end of an "Intro" so the next chapter -boundary lands there). - -## DRM license methods - -```python -def get_widevine_service_certificate(self, **_: Any) -> Optional[str]: - return self.config.get("certificate") # or None to use the common privacy cert - -def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: - license_url = self.license_data.get("url") or self.config["endpoints"].get("widevine_license") - response = self.session.post(url=license_url, data=challenge, ...) - response.raise_for_status() - return response.content - -def get_playready_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: - response = self.session.post(url=self.config["endpoints"]["playready_license"], data=challenge) - response.raise_for_status() - return response.content - -def get_clearkey_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str, dict]]: - license_url = self.config["endpoints"].get("clearkey_license") - if not license_url: - return None # fall back to the manifest-provided Laurl, if any - response = self.session.post(url=license_url, data=challenge) - response.raise_for_status() - return response.json() -``` - -- `get_widevine_service_certificate()` - return the service's privacy cert string to enable - privacy-mode license requests, or `None` to use the common cert. -- `get_widevine_license()` - POST `challenge` (raw CDM challenge bytes) to the license endpoint, - return the raw license (bytes), or JSON-unwrap it first if the service wraps it - (`response.json()["license"]`). `track` is passed in case the service rotates keys per segment - and you need its per-segment PSSH. -- `get_playready_license()` - same shape as Widevine. If omitted, the base class delegates to - `get_widevine_license()`, so only implement it when the service has a genuinely separate - PlayReady endpoint. -- `get_clearkey_license()` - only for DASH `org.w3.clearkey` (W3C EME). `challenge` is the W3C - JSON license request; return the JWK Set as dict, str, or bytes. **Three integration levels, - simplest first:** - 1. Manifest carries a `<Laurl>` - implement nothing; the framework POSTs there itself. - 2. Custom endpoint/headers - override this method as shown above. - 3. Obfuscated/bespoke key delivery - fetch and unwrap the key yourself, then pre-populate - `drm.content_keys[kid] = key_hex` on the track's `ClearKeyCENC` during `get_tracks()`; the - framework skips the license round-trip entirely when every KID is already keyed. -- HLS AES-128 ClearKey and fully unencrypted content have no license callback at all - the key - comes straight from the manifest or a side endpoint and is attached to the track's DRM object - directly. - -## Other extension points - -A handful of base-class hooks aren't exercised by `EXAMPLE` because it has nothing interesting to -do in them, but are there when you need them: - -- **Event hooks** - no-ops on the base class, fire during the download pipeline if overridden: - `on_segment_downloaded(track, segment: Path)`, `on_track_downloaded(track)`, - `on_track_decrypted(track, drm, segment=None)`, `on_track_repacked(track)`, - `on_track_multiplex(track)`. Useful for progress reporting or service-specific cleanup tied to - pipeline stages. -- **`resolve_server_keys(title)`** - not declared on `Service` at all; `dl.py` checks for it with - `hasattr` and calls it if present. Use it for services where decryption keys need to be resolved - server-side ahead of the normal license flow, separate from the per-track license callbacks. -- **`get_titles_cached()` / `apply_title_map()`** - base-class wrappers around your `get_titles()` - that add the title-metadata cache and `title_map` config rewriting (see - [`title_map`](SERVICE_CONFIG.md#title_map-dict)). `dl.py` calls `get_titles_cached()`, not - `get_titles()` directly - implement `get_titles()`, don't override the cached wrapper. - -## `config.yaml` - -```yaml -endpoints: - base_url: "https://api.example.com" - content: "https://api.example.com/content" - search: "https://api.example.com/search" - license: "https://license.example.com/v1" -headers: - User-Agent: "..." -client: - id: "..." -device: - platform_name: "..." -``` - -No enforced schema - `self.config` is just whatever YAML you put here, merged with any -`services.<TAG>` overrides from the user's `unshackle.yaml` (see -[Service Integration & Authentication](SERVICE_CONFIG.md#services-dict)). Conventions: - -- **All URLs live here, never hardcoded in `__init__.py`.** Use `{}` or `{name}` placeholders for - dynamic segments and `.format(...)` at the call site. -- Treat this file as shared/checked-in - don't put per-user secrets in it. Sensitive, - user-specific values (API keys, device IDs) belong in the user's `unshackle.yaml` under - `services.<TAG>`, which gets merged into `self.config` at runtime. -- `client:`/`device:` blocks are a common idiom for services with multiple device profiles - (web/android_tv/ios) - index into them with whatever `-d/--device` option you exposed in `cli()`. - -Use `-d`/`--debug` (before the subcommand) for structured JSONL logging while iterating - see -[Structured Debug Logging](DEBUG_LOGGING.md). diff --git a/docs/DEBUG_LOGGING.md b/docs/DEBUG_LOGGING.md deleted file mode 100644 index 34a52283..00000000 --- a/docs/DEBUG_LOGGING.md +++ /dev/null @@ -1,79 +0,0 @@ -# Structured debug logging - -unshackle emits structured JSON Lines (JSONL) when `-d`/`--debug` (a **global** flag, before the -subcommand: `unshackle -d dl ...`) or `config.debug` is set. Output lands at -`config.directories.logs / unshackle_debug_<service>_<time>.jsonl`. The logger is built for -**developers troubleshooting pipeline flow** — maximum signal, minimum noise — not end users. - -## Adding logging to a new feature - -Use the two primitives from `unshackle.core.utilities`. Both are **no-ops when debug logging is -disabled**, so they are always safe to call unguarded — no `if debug_logger:` needed. - -```python -from unshackle.core.utilities import log_event, timed_operation - -# One-shot event: -log_event("myfeature_done", message="Did the thing", context={"count": n}) - -# Time a block (logs once at the end with duration_ms; ERROR + exception if it raises): -with timed_operation("myfeature_run", context={"input": str(path)}): - do_the_work() -``` - -That's it. Do **not** write `if dl := get_debug_logger(): dl.log(...)` in new code — `log_event` -replaces that boilerplate. - -### External tools (ffmpeg, mkvmerge, dovi_tool, …) - -Route binary calls through the helpers in `unshackle.core.utils.subprocess`: - -- `run_step(args, *, label=..., output=...)` — runs a CLI step and **auto-logs** a `tool_run` - entry (label, tool, returncode, duration). Prefer this for new tool calls. -- `ffprobe(uri)` — auto-logs its `tool_run`. -- `log_tool_run(label, tool, returncode, *, duration_ms=..., **ctx)` — for a direct - `subprocess.run` you can't route through `run_step`; call it right after the process returns. - -## Conventions - -- **Operation names**: `<area>_<event>` lowercase, e.g. `manifest_dash_parse`, `drm_decrypt`, - `mux_complete`, `vault_get_key`, `tool_run`. Names are plain inline strings (no central registry). -- **Levels = the flow skeleton.** One **INFO** milestone per stage (a dev runs - `jq 'select(.level=="INFO")'` to read the end-to-end flow); internals at **DEBUG**; failures at - **ERROR**. Keep INFO sparse. -- **Every entry carries a one-sentence `message`** that reads on its own; structured data - (`context`, `duration_ms`, counts, ids) lives in fields, not prose. -- **No raw dumps.** Counts, ids, sizes, and `safe_display_url(url)` only — never a full `Tracks`, - MPD/manifest body, or response payload. -- **Secrets, URLs & paths.** Every logged string passes through `redact_all` = - `redact_text` (mask password/token/secret/auth/cookie keys + URL userinfo/secret query params) - → `redact_url` (collapse any http(s) URL to `redacted[.ext]`, hiding CDN/content/manifest/api - locations while keeping the extension, e.g. `redacted.mpd`) → `redact_path` (strip local base - dirs: install root → `<unshackle>`, venv → `<venv>`, home → `~`). `key` fields are also redacted - unless `config.debug_keys`. Pass user data via `context=`/`request=`/kwargs so it is sanitized. - Net effect: a shared JSONL leaks no account URLs, machine paths, or usernames. - Set `redact_paths: false` in `unshackle.yaml` to show full absolute paths instead of `<unshackle>`/ - `<venv>`/`~` tokens (config-only, no CLI flag). Affects `redact_path` everywhere it is used — - the debug log and the service load-summary detail. Secret/URL redaction is unaffected. -- **Service calls are intentionally not logged** (no per-request POST/GET to services). Manifest - parsing (`manifest_*_parse`) is the seam for request-level visibility. - -## Reading the output - -```bash -# Flow skeleton (one line per milestone): -jq -r 'select(.level=="INFO") | "\(.operation)\t\(.message)"' unshackle/logs/unshackle_debug_*.jsonl - -# Everything for one correlated operation: -jq 'select(.operation_id=="abc12345")' <log> -``` - -## Primitives reference - -| Symbol | Module | Purpose | -|---|---|---| -| `log_event(op, *, level, message, **ctx)` | `core.utilities` | one-shot structured entry | -| `timed_operation(op, *, level, message, **ctx)` | `core.utilities` | context manager; logs once at end with `duration_ms` (ERROR on raise) | -| `DebugLogger.log_drm_operation / log_vault_query / log_service_call` | `core.utilities` | typed convenience wrappers (accept `message=`/`level=` overrides) | -| `run_step` / `ffprobe` / `log_tool_run` | `core.utils.subprocess` | external-tool `tool_run` logging | -| `get_debug_logger()` | `core.utilities` | low-level accessor (rarely needed directly) | diff --git a/docs/DOWNLOAD_CONFIG.md b/docs/DOWNLOAD_CONFIG.md deleted file mode 100644 index 72144b2e..00000000 --- a/docs/DOWNLOAD_CONFIG.md +++ /dev/null @@ -1,326 +0,0 @@ -# Download & Processing Configuration - -This document covers configuration options related to downloading and processing media content. - -## downloader - -unshackle ships a single unified downloader at `unshackle/core/downloaders/requests.py`. The legacy -`aria2c`, `curl_impersonate`, and `n_m3u8dl_re` backends have been removed; their config blocks no -longer have any effect. - -The unified downloader: - -- Works with both a standard `requests.Session` and `RnetSession` (rnet/BoringSSL TLS impersonation, - which replaces the previous `curl_cffi` backend). When a service exposes its own session via - `self.session`, TLS fingerprinting is preserved on every segment. -- Uses adaptive chunk sizing between **512 KB and 4 MB**, picked from the response `Content-Length`. -- Spawns **up to `min(16, cpu_count + 4)` worker threads** by default for segmented downloads - (override via `--workers` / `dl.workers`). -- Resumes interrupted downloads via HTTP `Range` requests (a sibling `<file>.!dev` control file - marks an in-progress download). -- Has a single-URL fast path: if the server supports byte ranges and the file is at least 64 MB, - the file is split into 16 MB parts and downloaded in parallel into a pre-allocated file. -- Is selected per-track via `track.downloader`, which defaults to this unified `requests` downloader. - -There is no `downloader:` config key to set anymore. Setting one to a legacy value will emit a -`DeprecationWarning` and otherwise be ignored. - ---- - -## dl (dict) - -Pre-define default options and switches of the `dl` command. -The values will be ignored if explicitly set in the CLI call. - -The Key is the option name with dashes as underscores (`--v-lang` -> `v_lang`). `range` and `list` -work as-is; their internal Python names (`range_`, `list_`, suffixed to avoid the builtins) are -also accepted. - -For example to set the default primary language to download to German, - -```yaml -lang: de -``` - -You can also set multiple preferred languages using a list, e.g., - -```yaml -lang: - - en - - fr -``` - -to set how many tracks to download concurrently to 4 and download threads to 16, - -```yaml -downloads: 4 -workers: 16 -``` - -to set `--bitrate=CVBR` for a specific service, - -```yaml -lang: de -EXAMPLE: - bitrate: CVBR -``` - -or to change the output subtitle format from the default (original format) to WebVTT, - -```yaml -sub_format: vtt -``` - -### All Available `dl` Keys - -Below is a comprehensive list of keys that can be pre-defined in the `dl` section. Each corresponds -to a CLI option on the `dl` command. CLI arguments always take priority over config values. - -**Quality and codec:** - -| Key | Type | Default | Description | -| --- | --- | --- | --- | -| `quality` | int or list | best | Resolution(s) to download (e.g., `1080`, `[1080, 2160]`) | -| `vcodec` | str or list | any | Video codec(s): `H264`, `H265`, `VP9`, `AV1`, `VC1` | -| `acodec` | str or list | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS` | -| `vbitrate` | int | highest | Video bitrate in kbps | -| `abitrate` | int | highest | Audio bitrate in kbps | -| `vbitrate_range` | str | none | Video bitrate window in kbps, format `MIN-MAX` (e.g., `6000-7000`) | -| `abitrate_range` | str | none | Audio bitrate window in kbps, format `MIN-MAX` | -| `real_video_bitrate` | bool | `false` | Probe actual media size to compute true video bitrates, overriding the manifest's declared value (`-rvb`). See [Real bitrate probing](#real-bitrate-probing) | -| `real_audio_bitrate` | bool | `false` | Same as above for audio tracks (`-rab`). Slower than video (more renditions) | -| `range` (or `range_`) | str or list | `SDR` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` | -| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) | -| `worst` | bool | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` | -| `best_available` | bool | `false` | Continue if requested quality is unavailable | - -**Language:** - -| Key | Type | Default | Description | -| --- | --- | --- | --- | -| `lang` | str or list | `orig` | Language for video and audio (`orig` = original language) | -| `v_lang` | list | `[]` | Language override for video tracks only | -| `a_lang` | list | `[]` | Language override for audio tracks only | -| `s_lang` | list | `["all"]` | Language for subtitles | -| `require_subs` | list | `[]` | Required subtitle languages (skip title if missing) | -| `forced_subs` | bool | `false` | Include forced subtitle tracks | -| `exact_lang` | bool | `false` | Exact language matching (no regional variants) | -| `latest_episode` | bool | `false` | Download only the single most recent episode of a series | - -**Track selection:** - -| Key | Type | Default | Description | -| --- | --- | --- | --- | -| `video_only` | bool | `false` | Only download video tracks | -| `audio_only` | bool | `false` | Only download audio tracks | -| `subs_only` | bool | `false` | Only download subtitle tracks | -| `chapters_only` | bool | `false` | Only download chapters | -| `no_video` | bool | `false` | Skip video tracks | -| `no_audio` | bool | `false` | Skip audio tracks | -| `no_subs` | bool | `false` | Skip subtitle tracks | -| `no_chapters` | bool | `false` | Skip chapters | -| `no_atmos` | bool | `false` | Exclude Dolby Atmos audio tracks | -| `audio_description` | bool | `false` | Include audio description tracks | - -**Output and tagging:** - -| Key | Type | Default | Description | -| --- | --- | --- | --- | -| `tag` | str | config default | Override group tag | -| `repack` | bool | `false` | Add REPACK tag to output filename | -| `sub_format` | str | original | Output subtitle format: `srt`, `vtt`, `ass`, `ssa`, `ttml` | -| `no_folder` | bool | `false` | Disable folder creation for TV shows | -| `no_source` | bool | `false` | Remove source tag from filename | -| `no_mux` | bool | `false` | Do not mux tracks into a container file | -| `split_audio` | bool | `false` | Create separate output files per audio codec | -| `export` | bool | `false` | Write a JSON sidecar with manifest URLs, subtitles, per-track KID:KEY, codec/track info | - -**Metadata enrichment:** - -| Key | Type | Default | Description | -| --- | --- | --- | --- | -| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging | -| `imdb_id` | str | `null` | Use specific IMDB ID (e.g., `tt1375666`) | -| `animeapi_id` | str | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`, `anilist:98765`) | -| `enrich` | bool | `false` | Override show title and year from external source. Requires `tmdb_id`, `imdb_id`, or `animeapi_id` | - -**Download behavior:** - -| Key | Type | Default | Description | -| --- | --- | --- | --- | -| `downloads` | int | `1` | Concurrent track downloads | -| `workers` | int | `min(16, cpu_count + 4)` | Max threads per track download (segments / ranged parts) | -| `slow` | bool or `MIN-MAX` | `false` | Randomized delay between titles. `true` uses 60-120s; pass `MIN-MAX` (e.g., `20-40`) for a custom range | -| `no_proxy_download` | bool | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy | -| `skip_dl` | bool | `false` | Skip download, only get decryption keys | -| `cdm_only` | bool | `null` | Only use CDM (`true`) or only vaults (`false`) | - -### Real bitrate probing - -Some services declare inaccurate `bandwidth`/`BANDWIDTH` in their manifests — often -a peak or nominal figure that is far from the real average. Because `track.bitrate` -drives the track listing, sorting, and `--vbitrate` / `--vbitrate-range` selection, -a wrong value picks the wrong track. - -`-rvb` / `--real-video-bitrate` (and `-rab` / `--real-audio-bitrate` for audio) -probe the actual media size and overwrite `track.bitrate` with the measured value -(`bytes * 8 / duration`) before listing and selection. So `-rvb --list` shows the -true numbers, and `-rvb --vbitrate-range 6000-7000` selects against them. Without -the flag, behaviour is unchanged (the manifest value is used). - -How it works: - -- **Single-file tracks** (one whole file per rendition — e.g. DASH `SegmentBase` - or services that collapse to a `BaseURL`) are measured **exactly**: the whole - file size over the track duration. -- **Multi-segment tracks** (most HLS) are a **sampled estimate** — a spread of - segments is probed and extrapolated, typically within a few percent. Segment - bytes include container overhead, so MPEG-TS HLS reads a few percent above the - demuxed stream (this is the real *delivered* size). -- Only the top renditions per quality tier are probed (video grouped by - codec + range, audio by codec + channels + language), in parallel, then extended - downward only as far as needed to keep ranking correct. This keeps the pass fast - even when a service exposes dozens of renditions. -- Tracks whose duration cannot be determined fall back to `ffprobe`; probe failures - are non-fatal and leave the manifest bitrate in place. - -Per-track before→after values are logged at debug level (run with `-d`); the -corrected values always appear in the Available Tracks panel. - -You can also set per-service `dl` overrides under the `services` section (see -[Service Integration & Authentication Configuration](SERVICE_CONFIG.md)). Per-service values beat the -global `dl:` section; explicit CLI arguments beat both: - -```yaml -dl: - lang: en - downloads: 4 - workers: 16 - -services: - EXAMPLE: - dl: - v_lang: [en] - quality: 1080 -``` - ---- - -## audio (dict) - -Configuration for audio track selection. - -- `codec_priority` - Optional list of audio codec names defining the preferred order when multiple audio - tracks share the same bitrate and language. Listed codecs are ranked in the order given. - Codecs not in the list retain their bitrate-based ordering and are placed after all - listed codecs (i.e. soft priority — nothing is dropped). - - Atmos tracks still take precedence over codec priority, and audio description tracks - are still moved to the end. - - Valid codec names: `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `OGG`, `DTS`, `ALAC`, `FLAC`. - -For example, - -```yaml -audio: - codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG] -``` - -Or to only prefer a subset (e.g. surround codecs first, everything else falls back to -bitrate order): - -```yaml -audio: - codec_priority: [EC3, DTS, AC3, AAC] -``` - -When unset, audio tracks are sorted by bitrate alone (with Atmos/descriptive rules still -applied). - ---- - -## subtitle (dict) - -Configuration for subtitle processing and conversion. - -- `conversion_method` - Method to use for converting subtitles between formats. Default: `"auto"` - - `"auto"` — Smart routing: uses subby for WebVTT/SAMI, pycaption for others. - - `"subby"` — Always use subby with advanced processing. - - `"pycaption"` — Use only pycaption library (no SubtitleEdit, no subby). - - `"subtitleedit"` — Prefer SubtitleEdit when available, fall back to pycaption. - - `"pysubs2"` — Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP). -- `sdh_method` - Method to use for SDH (hearing impaired) stripping. Default: `"auto"` - - `"auto"` — Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter. - - `"subby"` — Use subby library (SRT only). - - `"subtitleedit"` — Use SubtitleEdit tool (Windows only, falls back to subtitle-filter). - - `"filter-subs"` — Use subtitle-filter library directly. -- `strip_sdh` - Automatically create stripped (non-SDH) versions of SDH subtitles. Default: `true` -- `convert_before_strip` - Auto-convert VTT/other formats to SRT before using subtitle-filter for SDH stripping. - Ensures compatibility when subtitle-filter is used as fallback. Default: `true` -- `preserve_formatting` - Preserve original subtitle formatting (tags, positioning, styling). - When `true`, skips pycaption processing for WebVTT files to keep tags like `<i>`, `<b>`, - positioning intact. Combined with no `sub_format` setting, ensures subtitles remain in - their original format. Default: `true` -- `output_mode` - Output mode for subtitles. Default: `"mux"` - - `"mux"` — Embed subtitles in MKV container only. - - `"sidecar"` — Save subtitles as separate files only. - - `"both"` — Embed in MKV and save as sidecar files. -- `sidecar_format` - Format for sidecar subtitle files when `output_mode` is `"sidecar"` or `"both"`. Default: `"srt"` - Options: `srt`, `vtt`, `ass`, `original` (keep current format). - -For example, - -```yaml -subtitle: - conversion_method: auto - sdh_method: auto - strip_sdh: true - convert_before_strip: true - preserve_formatting: true - output_mode: mux - sidecar_format: srt -``` - ---- - -## decryption (str | dict) - -Choose what software to use to decrypt DRM-protected content throughout unshackle where needed. -You may provide a single decryption method globally or a mapping of service tags to -decryption methods. - -Options: - -- `shaka` (default) - Shaka Packager - <https://github.com/shaka-project/shaka-packager> -- `mp4decrypt` - mp4decrypt from Bento4 - <https://github.com/axiomatic-systems/Bento4> - -Note that Shaka Packager is the traditional method and works with most services. mp4decrypt -is an alternative that may work better with certain services that have specific encryption formats. - -Example mapping: - -```yaml -decryption: - EXAMPLE: mp4decrypt - EXAMPLE2: shaka - default: shaka -``` - -The `default` entry is optional. If omitted, `shaka` will be used for services not listed. - -Simple configuration (single method for all services): - -```yaml -decryption: mp4decrypt -``` - ---- diff --git a/docs/DRM_CONFIG.md b/docs/DRM_CONFIG.md deleted file mode 100644 index 6e6c7185..00000000 --- a/docs/DRM_CONFIG.md +++ /dev/null @@ -1,598 +0,0 @@ -# DRM & CDM Configuration - -This document covers Digital Rights Management (DRM) and Content Decryption Module (CDM) configuration options. - -## cdm (dict) - -Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-insensitive). -The value should be a WVD or PRD filename without the file extension. When -loading the device, unshackle will look in both the `WVDs` and `PRDs` directories -for a matching file. - -For example, - -```yaml -EXAMPLE: chromecdm_903_l3 -EXAMPLE2: nexus_6_l1 -``` - -You may also specify this device based on the profile used. - -For example, - -```yaml -EXAMPLE: chromecdm_903_l3 -EXAMPLE2: nexus_6_l1 -EXAMPLE3: - john_sd: chromecdm_903_l3 - jane_uhd: nexus_5_l1 -``` - -You can also specify a fallback value to predefine if a match was not made. -This can be done using `default` key. This can help reduce redundancy in your specifications. - -For example, the following has the same result as the previous example, as well as all other -services and profiles being pre-defined to use `chromecdm_903_l3`. - -```yaml -EXAMPLE2: nexus_6_l1 -EXAMPLE3: - jane_uhd: nexus_5_l1 -default: chromecdm_903_l3 -``` - -You can also select CDMs based on video resolution using comparison operators (`>=`, `>`, `<=`, `<`) -or exact match on the resolution height. - -For example, - -```yaml -EXAMPLE: - "<=1080": generic_android_l3 # Use L3 for 1080p and below - ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) - default: generic_android_l3 # Fallback if no quality match -``` - -You can mix profiles and quality thresholds in the same service: - -```yaml -EXAMPLE: - john: example_l3_profile # Profile-based selection - "<=720": example_mobile_l3 # Quality-based selection - "1080": example_standard_l3 # Exact match for 1080p - ">=1440": example_premium_l1 # Quality-based selection - default: example_standard_l3 # Fallback -``` - ---- - -## remote_cdm (list\[dict]) - -Configure remote CDM (Content Decryption Module) APIs to use for decrypting DRM-protected content. -Remote CDMs allow you to use high-security CDMs (L1/L2 for Widevine, SL2000/SL3000 for PlayReady) without -having the physical device files locally. - -unshackle supports multiple types of remote CDM providers: - -1. **DecryptLabs CDM** - Official DecryptLabs KeyXtractor API with intelligent caching -2. **Custom API CDM** - Highly configurable adapter for any third-party CDM API -3. **Legacy PyWidevine Serve** - Standard pywidevine serve-compliant APIs - -The name of each defined remote CDM can be referenced in the `cdm` configuration as if it was a local device file. - -### DecryptLabs Remote CDM - -DecryptLabs provides a professional CDM API service with support for multiple device types and intelligent key caching. - -**Supported Devices:** -- **Widevine**: `ChromeCDM` (L3), `L1` (Security Level 1), `L2` (Security Level 2) -- **PlayReady**: `SL2` (SL2000), `SL3` (SL3000) - -**Configuration:** - -```yaml -remote_cdm: - # Widevine L1 Device - - name: decrypt_labs_l1 - type: decrypt_labs # Required: identifies as DecryptLabs CDM - device_name: L1 # Required: must match exactly (L1, L2, ChromeCDM, SL2, SL3) - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY # Your DecryptLabs API key - - # Widevine L2 Device - - name: decrypt_labs_l2 - type: decrypt_labs - device_name: L2 - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY - - # Chrome CDM (L3) - - name: decrypt_labs_chrome - type: decrypt_labs - device_name: ChromeCDM - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY - - # PlayReady SL2000 - - name: decrypt_labs_playready_sl2 - type: decrypt_labs - device_name: SL2 - device_type: PLAYREADY # Required for PlayReady - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY - - # PlayReady SL3000 - - name: decrypt_labs_playready_sl3 - type: decrypt_labs - device_name: SL3 - device_type: PLAYREADY - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY -``` - -**Features:** -- Intelligent key caching system (reduces API calls) -- Automatic integration with unshackle's vault system -- Support for both Widevine and PlayReady -- Multiple security levels (L1, L2, L3, SL2000, SL3000) - -!!! note - - The `device_type` field determines whether the CDM operates in PlayReady or Widevine mode. - Setting `device_type: PLAYREADY` (or using `device_name: SL2` / `SL3`) activates PlayReady mode. - The `security_level` field is auto-computed from `device_name` when not specified (e.g., SL2 defaults - to 2000, SL3 to 3000, and Widevine devices default to 3). You can override these if needed. - -### Custom API Remote CDM - -A highly configurable CDM adapter that can work with virtually any third-party CDM API through YAML configuration. -This allows you to integrate custom CDM services without writing code. - -**Basic Example:** - -```yaml -remote_cdm: - - name: custom_chrome_cdm - type: custom_api # Required: identifies as Custom API CDM - host: https://your-cdm-api.com - timeout: 30 # Optional: request timeout in seconds - - device: - name: ChromeCDM - type: CHROME # CHROME, ANDROID, PLAYREADY - system_id: 27175 - security_level: 3 - - auth: - type: bearer # bearer, header, basic, body - key: YOUR_API_TOKEN - - endpoints: - get_request: - path: /get-challenge - method: POST - decrypt_response: - path: /get-keys - method: POST - - caching: - enabled: true # Enable key caching - use_vaults: true # Integrate with vault system -``` - -**Full field reference:** - -```yaml -remote_cdm: - - name: advanced_custom_api - type: custom_api - host: https://api.example.com - timeout: 30 # Default request timeout (seconds); overridden per-endpoint - - device: - name: L1 # Passed as 'scheme' in base request params - type: ANDROID # CHROME, ANDROID, or PLAYREADY - system_id: 26830 # Widevine system ID (informational; not sent) - security_level: 1 # 1–3 for Widevine; 2000/3000 for PlayReady - - # Authentication — applied to every request via session headers or Basic auth - auth: - type: header # bearer | header | basic | body | query - header_name: X-API-Key # used when type=header (default: Authorization) - key: YOUR_SECRET_KEY # API key / token / password - # bearer_token: TOKEN # explicit token for type=bearer (falls back to 'key') - # username: user # required for type=basic - # password: pass # required for type=basic - custom_headers: # merged into every request - User-Agent: Unshackle/3.1.0 - X-Client-Version: "1.0" - - # Endpoints — both keys are required - endpoints: - get_request: # first call: obtain challenge or cached keys - path: /v2/challenge - method: POST - timeout: 30 # overrides top-level timeout for this endpoint - decrypt_response: # second call: exchange license response for keys - path: /v2/decrypt - method: POST - timeout: 30 - - # request_mapping — transform outgoing parameters before each request - # Base params sent to get_request: scheme, init_data, [service], [service_certificate] - # Base params sent to decrypt_response: scheme, session_id, init_data, license_request, license_response - request_mapping: - get_request: - param_names: # rename base param keys - init_data: pssh - scheme: device_type - static_params: # always-added fixed values - api_version: "2.0" - conditional_params: # added only when condition is true - - condition: "is_playready == True" - params: - drm_type: playready - transforms: # applied after renaming; type values listed below - - param: pssh - type: base64_encode - nested_params: # group named params under a parent key - payload: - - pssh - - device_type - exclude_params: # remove unwanted params before sending - - service_certificate - decrypt_response: - param_names: - license_request: challenge - license_response: license - - # response_mapping — extract and validate fields from API responses - response_mapping: - get_request: - fields: # map standard names to dot-path locations in response JSON - challenge: data.challenge - session_id: session.id - message: status.message - response_types: # classify response so CDM can choose the right path - - condition: "message_type == 'license-request'" - type: license_request # CDM expects challenge + session_id - - condition: "message_type == 'cached-keys'" - type: cached_keys # CDM skips license round-trip - success_conditions: # all must be true; failure raises RequestException - - "message == 'success'" - error_fields: # inspected when success_conditions fail (default: error, message, details) - - error - - details - transforms: # applied to extracted fields - - field: challenge - type: base64_decode - decrypt_response: - fields: - keys: data.keys - message: status.message - key_fields: # map key-object property names to standard names - kid: key_id # default: kid - key: content_key # default: key - type: key_type # default: type (value used as-is; CONTENT treated as content key) - success_conditions: - - "message == 'success'" - - # caching — vault and API-level key caching - caching: - enabled: true # master switch (default: true) - use_vaults: true # check/write unshackle key vaults (default: true) - check_cached_first: true # check vaults before making any API call (default: true) -``` - -**Supported `transforms` types** (for both `request_mapping` and `response_mapping`): -`base64_encode`, `base64_decode`, `hex_encode`, `hex_decode`, `json_stringify`, `json_parse`, `parse_key_string` - -**Supported authentication types:** -- `bearer` — `Authorization: Bearer <key>` (or `bearer_token` field) -- `header` — arbitrary header; set `header_name` (default `Authorization`) -- `basic` — HTTP Basic auth via `username` / `password` -- `body` — credentials injected into request body (handled by request_mapping) -- `query` — credentials appended to query string (handled by request_mapping) - -### PyPlayReady Remote CDM - -Connects to a running `unshackle serve` instance for PlayReady licensing. Uses -`pyplayready.remote.remotecdm.RemoteCdm` under the hood. No `type` field is needed — -the entry is routed to PlayReady when `device_type: PLAYREADY` is set (same branching -logic as the legacy Widevine path). - -```yaml -remote_cdm: - - name: playready_remote - device_name: my_prd_device # Device name registered on the serve instance - device_type: PLAYREADY # Required to select the PlayReady code path - system_id: 0 # Not used by pyplayready RemoteCdm; set to 0 - security_level: 3000 # 2000 = SL2000, 3000 = SL3000 - host: http://127.0.0.1:8786/playready # Must include the /playready path suffix - secret: your-api-secret-key # X-Secret-Key sent to the serve instance -``` - -!!! note - - The `/playready` path suffix in `host` is required — `pyplayready.RemoteCdm` - appends its own endpoint paths on top of this base, so omitting the suffix will result - in 404 errors. - -### Legacy PyWidevine Serve Format - -Standard [pywidevine] serve-compliant remote CDM configuration (backwards compatibility). - -```yaml -remote_cdm: - - name: legacy_chrome_cdm - device_name: chrome - device_type: CHROME - system_id: 27175 - security_level: 3 - host: https://domain.com/api - secret: secret_key -``` - -!!! note - - If the `type` field is not specified, the entry is treated as a legacy pywidevine serve CDM. - -[pywidevine]: https://github.com/rlaphoenix/pywidevine - ---- - -## decrypt_labs_api_key (str) - -API key for DecryptLabs CDM service integration. - -When set, enables the use of DecryptLabs remote CDM services in your `remote_cdm` configuration. -This is used specifically for `type: "decrypt_labs"` entries in the remote CDM list. - -For example, - -```yaml -decrypt_labs_api_key: "your_api_key_here" -``` - -!!! note - - This is different from the per-CDM `secret` field in `remote_cdm` entries. This provides a global - API key that can be referenced across multiple DecryptLabs CDM configurations. If a `remote_cdm` entry with - `type: "decrypt_labs"` does not have a `secret` field specified, the global `decrypt_labs_api_key` will be - used as a fallback. - ---- - -## decryption (str|dict) - -Configure which decryption tool to use for DRM-protected content. Default: `shaka`. - -Supported values: -- `shaka` - Shaka Packager (default) -- `mp4decrypt` - Bento4 mp4decrypt - -You can specify a single decrypter for all services: - -```yaml -decryption: shaka -``` - -Or configure per-service with a `DEFAULT` fallback: - -```yaml -decryption: - DEFAULT: shaka - EXAMPLE: mp4decrypt - EXAMPLE2: shaka -``` - -Service keys are case-insensitive (normalized to uppercase internally). - ---- - -## MonaLisa DRM - -MonaLisa is a WASM-based DRM system that uses local key extraction and two-stage segment decryption. -Unlike Widevine and PlayReady, MonaLisa does not use a challenge/response flow with a license server. -Instead, the PSSH value (ticket) is provided directly by the service API, and keys are extracted -locally via a WASM module. - -### Requirements - -- **ML-Worker binary**: Must be available on your system `PATH` (discovered via `binaries.ML_Worker`). - This is the binary that performs stage-1 decryption. - -### Decryption stages - -1. **ML-Worker binary**: Removes MonaLisa encryption layer (bbts -> ents). The key is passed via command-line argument. -2. **AES-ECB decryption**: Final decryption with service-provided key. - -MonaLisa uses per-segment decryption during download (not post-download like Widevine/PlayReady), -so segments are decrypted as they are downloaded. - -!!! note - - MonaLisa is configured per-service rather than through global config options. Services - that use MonaLisa handle ticket/key retrieval and CDM initialization internally. - ---- - -## ClearKey DRM - -Two distinct ClearKey mechanisms are supported; neither needs a CDM device or any DRM config: - -### HLS AES-128 ClearKey - -The key is fetched from (or near) the M3U8 `EXT-X-KEY` URI and segments are decrypted with -pure-Python AES-CBC. Fully automatic — nothing to configure. - -### DASH ClearKey (`org.w3.clearkey`) - -W3C EME ClearKey for DASH CENC content. The DASH parser recognises the clearkey -ContentProtection scheme (`urn:uuid:e2719d58-a985-b3c9-781a-b030af78d30e`), takes the KID from -`cenc:default_KID`, and reads the license server URL from the manifest's `<Laurl>` element when -present. - -License flow: the W3C JSON license request (`{"kids": [...], "type": "temporary"}`) is POSTed to -the license server, which returns the content key as a JWK Set. Keys land in the same vault and -`--export` paths as Widevine/PlayReady, and decryption uses the same shaka-packager/mp4decrypt -CENC backends (`decryption` config option applies). - -Service integration (simplest first): -1. Manifest carries a `<Laurl>` — works with zero service code. -2. Custom endpoint/headers — service overrides `get_clearkey_license`. -3. Bespoke key delivery — service pre-populates the DRM object's keys in `get_tracks`. - ---- - -## key_vaults (list\[dict]) - -Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service. - -This can help reduce unnecessary License calls even during the first download. This is because a Service may -provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates. - -You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on -Teams as sharing CEKs immediately can help reduce License calls drastically. - -Four types of Vaults are in the Core codebase: API, SQLite, MySQL, and HTTP. API and HTTP make HTTP requests to a RESTful API, -whereas SQLite and MySQL directly connect to an SQLite or MySQL Database. - -Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such. -Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be -accessible outside their hosting platform. - -Additional behavior: - -- `no_push` (bool): Optional per-vault flag. When `true`, the vault will not receive pushed keys (writes) but - will still be queried and can provide keys for lookups. Useful for read-only/backup vaults. -- `timeout` (float): Optional per-vault network timeout in seconds for the remote `API` and `HTTP` vaults. - Defaults to the global `vault_timeout` (10 seconds) so an unreachable vault host fails fast instead of - hanging the download. Set it on a vault to override the global, e.g. `timeout: 5.0`. Has no effect on - SQLite (local) or MySQL (uses its driver's own connect timeout). - -Set a global default for all remote vaults at the top level of the config: - -```yaml -vault_timeout: 10.0 # seconds; per-vault `timeout:` overrides this -``` - -### Using an API Vault - -API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may -not work in unshackle. The API format can be seen in the [API Vault Code](https://github.com/unshackle-dl/unshackle/blob/dev/unshackle/vaults/API.py). - -```yaml -- type: API - name: "John#0001's Vault" # arbitrary vault name - uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port) - # uri: "127.0.0.1:80/key-vault" - # uri: "https://api.example.com/key-vault" - token: "random secret key" # authorization token - # no_push: true # optional; make this API vault read-only (lookups only) - # timeout: 5.0 # optional; per-vault network timeout (seconds), overrides vault_timeout -``` - -### Using a MySQL Vault - -MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB. -A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults. - -```yaml -- type: MySQL - name: "John#0001's Vault" # arbitrary vault name - host: "127.0.0.1" # host/ip - # port: 3306 # port (defaults to 3306) - database: vault # database used for unshackle - username: jane11 - password: Doe123 - # no_push: false # optional; defaults to false -``` - -I recommend giving only a trustable user (or yourself) CREATE permission and then use unshackle to cache at least one CEK -per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to -make tables yourself. - -- Use a password on all user accounts. -- Never use the root account with unshackle (even if it's you). -- Do not give multiple users the same username and/or password. -- Only give users access to the database used for unshackle. -- You may give trusted users CREATE permission so unshackle can create tables if needed. -- Other uses should only be given SELECT and INSERT permissions. - -### Using an SQLite Vault - -SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage -drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in -case something happens to your MySQL Vault. - -```yaml -- type: SQLite - name: "My Local Vault" # arbitrary vault name - path: "C:/Users/Jane11/Documents/unshackle/data/key_vault.db" - # no_push: true # optional; commonly true for local backup vaults -``` - -!!! note - - You do not need to create the file at the specified path. - SQLite will create a new SQLite database at that path if one does not exist. - Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end - up with multiple databases. - -If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault -together. - -### Using an HTTP Vault - -HTTP Vaults provide flexible HTTP-based key storage with support for multiple API modes. This vault type -is useful for integrating with various third-party key vault APIs. - -```yaml -- type: HTTP - name: "My HTTP Vault" - host: "https://vault-api.example.com" - api_key: "your_api_key" # or use 'password' field - api_mode: "json" # query, json, or decrypt_labs - # username: "user" # required for query mode only - # no_push: false # optional; defaults to false - # timeout: 5.0 # optional; per-vault network timeout (seconds), overrides vault_timeout -``` - -**Supported API Modes:** - -- `query` - Uses GET requests with query parameters. Requires `username` field. -- `json` - Uses POST requests with JSON payloads. Token-based authentication. -- `decrypt_labs` - DecryptLabs API format. Read-only mode (`no_push` is forced to `true`). - -**Example configurations:** - -```yaml -# Query mode (requires username) -- type: HTTP - name: "Query Vault" - host: "https://api.example.com/keys" - username: "myuser" - password: "mypassword" - api_mode: "query" - -# JSON mode -- type: HTTP - name: "JSON Vault" - host: "https://api.example.com/vault" - api_key: "secret_token" - api_mode: "json" - -# DecryptLabs mode (read-only) -- type: HTTP - name: "DecryptLabs Cache" - host: "https://keyxtractor.decryptlabs.com/cache" - api_key: "your_decrypt_labs_api_key" - api_mode: "decrypt_labs" -``` - -!!! note - - The `decrypt_labs` mode is always read-only and cannot receive pushed keys. - ---- diff --git a/docs/GLUETUN.md b/docs/GLUETUN.md deleted file mode 100644 index ab36284d..00000000 --- a/docs/GLUETUN.md +++ /dev/null @@ -1,322 +0,0 @@ -# Gluetun VPN Proxy - -Gluetun provides Docker-managed VPN proxies supporting 50+ VPN providers. - -## Prerequisites - -**Docker must be installed and running.** - -```bash -# Linux -curl -fsSL https://get.docker.com | sh -sudo usermod -aG docker $USER # Then log out/in - -# Windows/Mac -# Install Docker Desktop: https://www.docker.com/products/docker-desktop/ -``` - -## Quick Start - -### 1. Configuration - -Add to `~/.config/unshackle/unshackle.yaml`: - -```yaml -proxy_providers: - gluetun: - providers: - windscribe: - vpn_type: openvpn - credentials: - username: "YOUR_OPENVPN_USERNAME" - password: "YOUR_OPENVPN_PASSWORD" -``` - -### 2. Usage - -Use 2-letter country codes directly: - -```bash -unshackle dl SERVICE CONTENT --proxy gluetun:windscribe:us -unshackle dl SERVICE CONTENT --proxy gluetun:windscribe:uk -``` - -Format: `gluetun:provider:region` - -## Provider Credential Requirements - -**OpenVPN (Recommended)**: Most providers support OpenVPN with just `username` and `password` - the simplest setup. - -**WireGuard**: Requires private keys and varies by provider. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific requirements. Note that `vpn_type` defaults to `wireguard` if not specified. - -## Getting Your Credentials - -### Windscribe (OpenVPN) - -1. Go to [windscribe.com/getconfig/openvpn](https://windscribe.com/getconfig/openvpn) -2. Log in with your Windscribe account -3. Select any location and click "Get Config" -4. Copy the username and password shown - -### NordVPN (OpenVPN) - -1. Go to [NordVPN Service Credentials](https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/service-credentials/) -2. Log in with your NordVPN account -3. Generate or view your service credentials -4. Copy the username and password - -!!! note - Use service credentials, NOT your account email/password. - -### WireGuard Credentials (Advanced) - -WireGuard requires private keys instead of username/password. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific WireGuard setup. - -## Configuration Examples - -**OpenVPN (Recommended)** - -Most providers support OpenVPN with just username and password: - -```yaml -providers: - windscribe: - vpn_type: openvpn - credentials: - username: YOUR_OPENVPN_USERNAME - password: YOUR_OPENVPN_PASSWORD - - nordvpn: - vpn_type: openvpn - credentials: - username: YOUR_SERVICE_USERNAME - password: YOUR_SERVICE_PASSWORD -``` - -**WireGuard (Advanced)** - -WireGuard can be faster but requires more complex credential setup: - -```yaml -# NordVPN/ProtonVPN (only private_key needed) -providers: - nordvpn: - vpn_type: wireguard - credentials: - private_key: YOUR_PRIVATE_KEY - -# Surfshark/Mullvad/IVPN (private_key AND addresses required) - surfshark: - vpn_type: wireguard - credentials: - private_key: YOUR_PRIVATE_KEY - addresses: 10.x.x.x/32 - -# Windscribe (all three credentials required) - windscribe: - vpn_type: wireguard - credentials: - private_key: YOUR_PRIVATE_KEY - addresses: 10.x.x.x/32 - preshared_key: YOUR_PRESHARED_KEY -``` - -## Server Selection - -Most providers use `SERVER_COUNTRIES`, but some use `SERVER_REGIONS`: - -| Variable | Providers | -|----------|-----------| -| `SERVER_COUNTRIES` | NordVPN, ProtonVPN, Surfshark, Mullvad, ExpressVPN, and most others | -| `SERVER_REGIONS` | Windscribe, VyprVPN, VPN Secure | - -Unshackle handles this automatically - just use 2-letter country codes. - -### Per-Provider Server Mapping - -You can explicitly map region codes to country names, cities, or hostnames per provider: - -```yaml -providers: - nordvpn: - vpn_type: openvpn - credentials: - username: YOUR_USERNAME - password: YOUR_PASSWORD - server_countries: - us: "United States" - uk: "United Kingdom" - server_cities: - us: "New York" - server_hostnames: - us: "us1239.nordvpn.com" -``` - -### Specific Server Selection - -Use a `<country><number>` region (e.g. `us1239`) to target a specific server. Unshackle builds the -hostname automatically per provider: - -| Provider | Hostname format | -|----------|-----------------| -| NordVPN | `us1239.nordvpn.com` | -| Surfshark | `us-1239.prod.surfshark.com` | -| ExpressVPN | `us-1239.expressvpn.com` | -| CyberGhost | `us-s1239.cg-dialup.net` | -| Other | `us1239` (passed as-is to `SERVER_HOSTNAMES`) | - -### Extra Environment Variables - -You can pass additional Gluetun environment variables per provider using `extra_env`: - -```yaml -providers: - nordvpn: - vpn_type: openvpn - credentials: - username: YOUR_USERNAME - password: YOUR_PASSWORD - extra_env: - LOG_LEVEL: debug -``` - -## Global Settings - -```yaml -proxy_providers: - gluetun: - providers: {...} - base_port: 8888 # Starting port (default: 8888) - auto_cleanup: true # Remove containers on exit (default: true) - verify_ip: true # Verify IP matches region (default: true) - container_prefix: "unshackle-gluetun" # Docker container name prefix (default: "unshackle-gluetun") - auth_user: username # Proxy auth (optional) - auth_password: password # Proxy auth (optional) -``` - -## Features - -- **Container Reuse**: First request takes 10-30s; subsequent requests are instant. Containers created by other unshackle processes are auto-detected via `docker inspect` and reused. -- **Ready Detection**: Waits up to 60s for both the HTTP proxy to listen (`[http proxy] listening`) and the VPN tunnel to come up (`initialization sequence completed` or `public ip address is`) before returning the proxy URI. Bails early on `fatal` or `invalid credentials` log lines. -- **IP Verification**: When `verify_ip: true` (default), looks up the exit IP via `ipinfo.io` through the proxy and compares country code to the requested region. Retries 3 times with exponential backoff (1s, 2s, 4s). -- **Concurrent Sessions**: Multiple downloads share the same container; ports are allocated thread-safely starting at `base_port`. -- **Specific Servers**: Use `--proxy gluetun:nordvpn:us1239` for specific server selection (see table above). -- **Automatic Image Pull**: The Gluetun Docker image (`qmcgaw/gluetun:latest`) is pulled automatically on first use (5 min timeout). -- **Secure Credentials**: Credentials are passed via temporary env files (mode 0600), then zero-overwritten and unlinked after `docker run`. They never appear in process listings. -- **Auto Cleanup**: Containers are removed via `atexit` (Ctrl+C still works normally). Disable with `auto_cleanup: false` to leave them stopped instead. - -## Container Management - -```bash -# View containers -docker ps | grep unshackle-gluetun - -# Check logs -docker logs unshackle-gluetun-nordvpn-us - -# Remove all containers -docker ps -a | grep unshackle-gluetun | awk '{print $1}' | xargs docker rm -f -``` - -## Troubleshooting - -### Docker Permission Denied (Linux) -```bash -sudo usermod -aG docker $USER -# Then log out and log back in -``` - -### VPN Connection Failed -Check container logs for specific errors: -```bash -docker logs unshackle-gluetun-nordvpn-us -``` - -Common issues: -- Invalid/missing credentials -- Windscribe WireGuard requires `preshared_key` (can be empty string, but must be set in credentials) -- VPN provider server issues -- Container startup timeout (default 60 seconds) - -## Resources - -- [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki) - Official provider documentation -- [Gluetun GitHub](https://github.com/qdm12/gluetun) - ---- - -## ExpressVPN HTTPS Proxy - -ExpressVPN is a standalone HTTPS proxy provider — it does **not** use Docker or the gluetun sub-provider system. It authenticates via a browser-extension OAuth flow (Keycloak/PKCE) and returns an authenticated `https://cat:<token>@<host>:443` proxy URL. - -### Prerequisites - -An active ExpressVPN subscription is required. The provider resolves proxy-capable locations through ExpressVPN's own API, so no local VPN client or Docker container is needed. - -### Authentication - -The provider uses a token cascade (tried in order): - -1. **Cached tokens** — `cache/global/expressvpn_tokens.json` (auto-written after first auth; access and refresh tokens are refreshed automatically when they expire). -2. **Exported browser cookies** — `cookies/vpn/expressvpn.txt` (also accepted at `cookies/vpns/expressvpn.txt`). Must contain the `KEYCLOAK_IDENTITY` and `KEYCLOAK_SESSION` cookies from `auth.expressvpn.com`. Used for a one-time PKCE bootstrap that populates the token cache. -3. **Desktop `account.json`** — configured via the `account_json` key (see below). - -Export cookies from a browser that is logged in to `auth.expressvpn.com`. Any Netscape-format, JSON array, or `name=value` file is accepted. - -### Configuration - -Add to `~/.config/unshackle/unshackle.yaml` under `proxy_providers`: - -```yaml -proxy_providers: - expressvpn: - region_map: - us: "ny" # --proxy expressvpn:us → New York (default city) - gb: "london" # --proxy expressvpn:gb → London - de: "frankfurt" # --proxy expressvpn:de → Frankfurt - es: "madrid" # --proxy expressvpn:es → Madrid - mx: # --proxy expressvpn:mx → smart/random MX location -``` - -`region_map` maps a country code to a default city preset. An empty (null) value enables smart connection (a random location in that country). The preset is only applied when the CLI query contains no city; an explicit city in the query always takes precedence. - -#### Optional keys - -| Key | Default | Description | -|-----|---------|-------------| -| `region_map` | `{}` | Country → default city presets (see above) | -| `server_map` | `{}` | Alias → ExpressVPN location slug or direct hostname | -| `cookie_path` | `cookies/vpn/expressvpn.txt` | Path to exported browser cookies | -| `cache_path` | `cache/global/expressvpn_tokens.json` | Token cache location | -| `account_json` | *(none)* | Path to ExpressVPN desktop `account.json` | -| `refresh_token` | *(none)* | OAuth refresh token (manual override) | -| `timeout` | `10.0` | HTTP request timeout in seconds | - -### Usage - -```bash -unshackle dl SERVICE CONTENT --proxy expressvpn:us # random/smart US location -unshackle dl SERVICE CONTENT --proxy expressvpn:us-ny # specific city (New York) -unshackle dl SERVICE CONTENT --proxy expressvpn:us-ny-2 # pinned server #2 in New York -unshackle dl SERVICE CONTENT --proxy expressvpn:mx # random/smart Mexico location -unshackle dl SERVICE CONTENT --proxy expressvpn:es-madrid # specific city (Madrid) -``` - -Query format: `expressvpn:<country>[-<city>[-<N>]]` - -- `country` — 2-letter country code (e.g. `us`, `gb`, `de`). -- `city` — optional city abbreviation or slug matched against location names using first-letter abbreviation (`ny` → "New York"), exact slug, prefix, then substring strategies. -- `N` — optional 1-based server index (e.g. `-2` selects the second server in the matched city). If `N` exceeds the available server count, a random server is selected. - -### City matching - -The city part of the query is matched against ExpressVPN location names using these strategies (in priority order): - -1. First-letter abbreviation — `ny` matches "New York" -2. Exact slug — `miami` matches "Miami" -3. Prefix match — `mia` matches "Miami" -4. Substring match — `york` matches "New York" - -### Token cache - -After the first successful authentication the token cache is written to `cache/global/expressvpn_tokens.json` (mode 0600). Subsequent runs refresh expired access/SRT/connection tokens automatically without re-reading cookies. diff --git a/docs/NETWORK_CONFIG.md b/docs/NETWORK_CONFIG.md deleted file mode 100644 index 61eb2bc1..00000000 --- a/docs/NETWORK_CONFIG.md +++ /dev/null @@ -1,284 +0,0 @@ -# Network & Proxy Configuration - -This document covers network and proxy configuration options for bypassing geofencing and managing connections. - -## proxy_providers (dict) - -Enable external proxy provider services. These proxies will be used automatically where needed as defined by the -Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider -to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers -support specific query formats for selecting a country/server. - -### basic (dict[str, str|list]) - -Define a mapping of country to proxy to use where required. -The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`. -Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`. - -Do note that each key's value can be a list of strings, or a string. For example, - -```yaml -us: - - "http://john%40email.tld:password123@proxy-us.domain.tld:8080" - - "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080" -de: "https://127.0.0.1:8080" -``` - -Note that if multiple proxies are defined for a region, then by default one will be randomly chosen. -You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the -second proxy of the US list. - -### nordvpn (dict) - -Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy -system where required. - -You can also specify specific servers to use per-region with the `server_map` key. -Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. - -You can also select servers by city using the format `--proxy nordvpn:us:seattle` or `--proxy nordvpn:ca:calgary`. - -For example, - -```yaml -username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format -password: wXVHmht22hhRKUEQ32PQVjCZ -server_map: - us: 12 # force US server #12 for US proxies -``` - -The username and password should NOT be your normal NordVPN Account Credentials. -They should be the `Service credentials` which can be found on your Nord Account Dashboard. - -Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy nordvpn:gb` or such. -You can even set a specific server number this way, e.g., `--proxy nordvpn:gb2366`. - -Note that `gb` is used instead of `uk` to be more consistent across regional systems. - -### protonvpn (dict) - -Enable Proton VPN as a proxy provider. Proton mints short-lived HTTPS proxy credentials per session, so -authentication reuses a Proton account session (obtained via TV login or an exported browser session) rather than -static credentials. **Free Proton accounts work**, limited to the free-tier exit countries. - -There are two ways to obtain that session — TV login or cookie export. You only need one. They can also coexist: a -cached or cookie session is always tried first, and TV login is only used as a fallback when no session is present. - -#### TV login (self-sustaining, recommended) - -Set `enable: true` and run any command that uses the proxy. On first use, when no session is cached, unshackle -prints a code and pauses: - -1. Open `https://account.proton.me/vpn/tv/code` in a browser where you are signed in to Proton. -2. Enter the code shown in the terminal and approve it. -3. Return to the terminal and **press Enter** to continue. - -The minted refresh token is cached at `<cache>/vpn/protonvpn.json` and **refreshes on its own forever after**, so -this is genuinely a one-time step — later runs load straight from the cache without prompting. The prompt is **off -by default** (`enable: false`) so nobody is prompted unexpectedly, and it is skipped automatically when there is no -interactive terminal (e.g. under `serve`, cron, or piped input) — in those contexts, use a cookie export or run an -interactive command once to seed the cache first. - -#### Cookie export (easiest to collect, but lapses) - -Export the cookies for `account.proton.me` **or** `account.protonvpn.com` (Netscape `cookies.txt` or a JSON cookie -list) and save them at `<cookies>/vpn/protonvpn.txt`. The provider auto-loads when that file exists (no YAML -required); cookies for other Proton apps (e.g. `mail.proton.me`) are ignored automatically. - -- The export **must include the `AUTH-<UID>` cookie**. It is **HttpOnly** — many one-click cookie-export - extensions skip HttpOnly cookies by default, which is the most common reason an export "doesn't work". Use an - exporter that includes them. -- A browser session **cannot be refreshed headlessly** — Proton rotates browser refresh tokens and ties them to - the live browser. unshackle therefore uses the cookie's access token only (it never tries to refresh it) and the - session works until that token expires (Proton's TTL, typically ~24h); after that you must re-export. For a - set-and-forget setup, prefer TV login. -- Cookies are read fresh from the file on every run and are **never written to the cache**. If a TV-login cache - also exists, it is preferred unless the cookie file is newer — so to force the cookie session, just re-save - (re-export) the cookie file. - -#### Resetting - -To start clean, delete both `<cookies>/vpn/protonvpn.txt` and `<cache>/vpn/protonvpn.json`. - -#### Troubleshooting - -- **`Proton: no session available …`** — no cookies and no cache were found. Either export cookies, or set - `enable: true` and run from an interactive terminal (the prompt is skipped on non-tty stdin). -- **`Proton: session expired; re-export … cookies`** — a cookie session's access token lapsed and cannot be - refreshed (expected for cookie auth). Re-export the cookies, or switch to TV login. - -#### Configuration - -Configuration is optional — only add a block to enable TV login or override defaults: - -```yaml -proxy_providers: - protonvpn: - enable: true # prompt the one-time TV login when no session is cached (default false) - cookie_path: /path/to/protonvpn.txt # optional; defaults to <cookies>/vpn/protonvpn.txt - cache_path: /path/to/protonvpn.json # optional; defaults to <cache>/vpn/protonvpn.json - timeout: 10 # optional; per-request timeout in seconds (default 10) - server_map: - stream-us: us:ny # optional alias -> country / country:city / countryNN -``` - -Query formats (after the provider prefix). City uses the NordVPN-style colon, like `nordvpn:us:seattle`: - -- `--proxy protonvpn:us` — random server in the country (the chosen server name is logged) -- `--proxy protonvpn:us:ny` / `protonvpn:ca:vancouver` — by city (name or initials) -- `--proxy protonvpn:de203` — pin Proton server **#203** (the number shown in the log, e.g. `DE#203`) - -Use `uk` (Proton's code for the United Kingdom); `gb` is accepted and mapped to `uk` automatically. Proton's -server numbers are not sequential (a country may have `#203`, `#813`, …), so pick the number from a previous -run's log line rather than guessing. - -Selection only offers servers your plan can use (paid accounts are detected automatically) and **prefers paid -servers over the free pool**, falling back to free only when no paid server is available — a pinned `#NN` always -wins. **Tor and Secure Core servers are excluded** (both are slow for a plain proxy). Secure Core servers use -port 443, all others 4443 — handled automatically. - -### surfsharkvpn (dict) - -Enable Surfshark VPN proxy service using Surfshark Service credentials (not your login password). -You may pin specific server IDs per region using `server_map`. - -You can also select servers by city using the format `--proxy surfsharkvpn:us:seattle`. - -```yaml -username: your_surfshark_service_username # https://my.surfshark.com/vpn/manual-setup/main/openvpn -password: your_surfshark_service_password # service credentials, not account password -server_map: - us: 3844 # force US server #3844 - gb: 2697 # force GB server #2697 - au: 4621 # force AU server #4621 -``` - -### hola - -Enable Hola VPN proxy service. Requires the `hola-proxy` binary to be installed and available in your PATH. -No configuration is needed under `proxy_providers`. Hola is loaded automatically when the `hola-proxy` binary -is detected. - -Once available, use `--proxy hola:us` or similar to connect through Hola. - -### windscribevpn (dict) - -Enable Windscribe VPN proxy service using static OpenVPN service credentials. - -Use the service credentials from https://windscribe.com/getconfig/openvpn (not your account login credentials). - -```yaml -proxy_providers: - windscribevpn: - username: openvpn_username # From https://windscribe.com/getconfig/openvpn - password: openvpn_password # Service credentials, NOT your account password -``` - -#### Server Mapping - -You can optionally pin specific servers using `server_map`: - -```yaml -proxy_providers: - windscribevpn: - username: openvpn_username - password: openvpn_password - server_map: - us: us-central-096.totallyacdn.com # Force specific US server - gb: uk-london-001.totallyacdn.com # Force specific UK server -``` - -Once configured, use `--proxy windscribevpn:us` or `--proxy windscribevpn:gb` etc. to connect through Windscribe. - -You can also select specific servers by number (e.g., `--proxy windscribevpn:sg007`) or filter by city -(e.g., `--proxy windscribevpn:ca:toronto`). - -### gluetun (dict) - -Docker-managed VPN proxy supporting 50+ VPN providers via Gluetun. See [GLUETUN.md](GLUETUN.md) for full -configuration and usage details. - -```yaml -proxy_providers: - gluetun: - providers: - windscribe: - vpn_type: openvpn - credentials: - username: "YOUR_OPENVPN_USERNAME" - password: "YOUR_OPENVPN_PASSWORD" -``` - -Usage: `--proxy gluetun:windscribe:us` - ---- - -## headers (dict) - -Case-Insensitive dictionary of headers that all Services begin their Request Session state with. -All requests will use these unless changed explicitly or implicitly via a Server response. -These should be sane defaults and anything that would only be useful for some Services should not -be put here. - -Avoid headers like 'Accept-Encoding' as that would be a compatibility header that the underlying -HTTP backend (rnet) will set for you as part of its browser impersonation profile. - -I recommend using, - -```yaml -Accept-Language: "en-US,en;q=0.8" -User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" -``` - ---- - -## HTTP Session Backend - -unshackle uses [`rnet`](https://github.com/0x676e67/rnet) (Rust + BoringSSL) for HTTP with TLS -fingerprinting. `RnetSession` is a drop-in `requests.Session` replacement and is what -`self.session` exposes to services. It supports: - -- Browser/app impersonation via named `rnet.Impersonate` presets (Chrome, Edge, Firefox, Safari, - OkHttp, etc.) — picks JA3, ALPN, HTTP/2 SETTINGS and header order to match the chosen client. -- Native rnet proxy support (HTTP, HTTPS, SOCKS5) — used by all proxy providers below. -- Cookie-jar and `requests`-style `data=` / `json=` / `headers=` kwargs for compatibility. - -The legacy `curl_cffi` backend has been removed. - -### network (dict) - -HTTP session settings: which client to impersonate and how the underlying rnet -client behaves. - -```yaml -network: - browser: Chrome131 # exact rnet.Impersonate preset name -``` - -`browser` must be an exact `rnet.Impersonate` preset name (e.g. `Chrome131`, `Chrome124`, -`Edge101`, `Firefox133`, `Safari18`, `OkHttp4_12`). See the rnet README for the full list. -Default when unset: `Chrome131`. - -The same section accepts optional rnet client tuning keys, forwarded verbatim: - -```yaml -network: - browser: Chrome131 - http1_only: true # force HTTP/1.1 instead of negotiated HTTP/2 - # http2_only: true - # pool_max_size: 64 # max total connections in the pool - # pool_max_idle_per_host: 32 - # tcp_nodelay: true -``` - -!!! note "Renamed from `curl_impersonate`" - This section was previously named `curl_impersonate` (after the removed backend). - The old spelling still works but is deprecated; rename it to `network`. - -!!! tip "When to force HTTP/1.1" - HTTP/2 is the right default on large CDNs. Force `http1_only: true` when a host - throttles per connection or has a slow origin behind HTTP/2 flow control — in - benchmarks this gained 30–50% on such hosts, while costing up to 27% on fast - CDNs. Set it only when measurements support it. - ---- diff --git a/docs/OUTPUT_CONFIG.md b/docs/OUTPUT_CONFIG.md deleted file mode 100644 index 685f73a4..00000000 --- a/docs/OUTPUT_CONFIG.md +++ /dev/null @@ -1,446 +0,0 @@ -# Output & Naming Configuration - -This document covers output file organization and naming configuration options. - -## filenames (dict) - -Override the default filenames used across unshackle. -The filenames use various variables that are replaced during runtime. - -The following filenames are available and may be overridden: - -- `log` - Log filenames. Uses `{name}` and `{time}` variables. -- `debug_log` - Debug log filenames. Uses `{service}` and `{time}` variables. -- `config` - Service configuration filenames. -- `root_config` - Root configuration filename. -- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables. -- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables. - -For example, - -```yaml -filenames: - log: "unshackle_{name}_{time}.log" - debug_log: "unshackle_debug_{service}_{time}.jsonl" - config: "config.yaml" - root_config: "unshackle.yaml" - chapters: "Chapters_{title}_{random}.txt" - subtitle: "Subtitle_{id}_{language}.srt" -``` - ---- - -## output_template (dict) - -Configure custom output filename templates for movies, series, songs, and albums. -This is **required** in your `unshackle.yaml` - a warning is shown if not configured. - -Add `?` suffix to make a variable conditional (omitted when empty): `{year?}`, `{hdr?}`, `{repack?}` - -### General variables - -| Variable | Meaning | -|---|---| -| `title` | Title of the movie, episode, or track | -| `year` | Release year | -| `source` | Service source tag (the uppercase service directory name) | -| `tag` | Group/release tag configured via `tag:` | -| `edition` | Edition label (e.g. `Extended`, `Theatrical`) | -| `repack` | `REPACK` when the release is a repack, otherwise empty | -| `lang_tag` | Language tag produced by `language_tags` rules (e.g. `NORDiC`) | - -### Series / episode variables - -| Variable | Meaning | -|---|---| -| `season` | Season number (e.g. `1`) | -| `episode` | Episode number (e.g. `1`) | -| `season_episode` | Combined season/episode (e.g. `S01E01`) | -| `episode_name` | Episode title | -| `date` | Air date in ISO form (e.g. `2024-06-30`), only for daily/sports content | - -!!! note "Daily / sports content" - When a service supplies an air date (daily talk shows, news, sports events), - `{season_episode}` automatically renders the date using your template's separator - (e.g. `2024.06.30` for a dotted template, `2024 06 30` for a spaced one). The - standalone `{year}` is dropped from the filename — the air date is the sole date — - so every episode of a show names consistently. Folders still keep the year (e.g. - `Show Name (2010)/2024/`). No template change is needed. - - Use the `{date}` variable instead if you want a fixed `YYYY-MM-DD` form regardless - of separator. Folders for dated episodes group by air year (e.g. `Show Name/2024/`). - -### Video / audio variables - -| Variable | Meaning | -|---|---| -| `quality` | Resolution label (e.g. `1080p`, `2160p`) | -| `resolution` | Raw resolution string | -| `video` | Video codec label (e.g. `H.264`, `H.265`) | -| `hdr` | HDR format label (e.g. `HDR`, `DV`) | -| `hfr` | HFR label when frame rate exceeds normal threshold | -| `audio` | Audio codec label (e.g. `AAC`, `EAC3`) | -| `audio_channels` | Channel count (e.g. `2.0`, `5.1`) | -| `audio_full` | Combined codec + channels (e.g. `DDP5.1`) | -| `atmos` | `Atmos` when Dolby Atmos is present, otherwise empty | -| `dual` | `DUAL` when two audio languages are present, otherwise empty | -| `multi` | `MULTi` when three or more audio languages are present, otherwise empty | - -### Music variables - -| Variable | Meaning | -|---|---| -| `track_number` | Track number within album | -| `disc` | Disc number | -| `track_total` | Total tracks on disc | -| `disc_total` | Total discs in release | -| `artist` | Track artist | -| `album_artist` | Album-level artist (may differ from track artist) | -| `album` | Album title | -| `release_type` | Release type (e.g. `Album`, `Single`, `EP`) | -| `genre` | Genre string | -| `explicit` | `Explicit` when track has explicit content flag, otherwise empty | -| `isrc` | International Standard Recording Code | -| `upc` | Universal Product Code (album barcode) | -| `label` | Record label name | - -```yaml -output_template: - # Scene-style (dot-separated) - movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' - series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' - songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' - - # Plex-friendly (space-separated) - # movies: '{title} ({year}) {quality}' - # series: '{title} {season_episode} {episode_name?}' - # songs: '{track_number}. {title}' -``` - -Example outputs: -- Scene movies: `Example.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` -- Scene movies (REPACK): `Example.Movie.2024.REPACK.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG` -- Scene series: `Example.Show.2024.S01E01.Pilot.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` -- Plex movies: `Example Movie (2024) 1080p` - -### folder (optional) - -Controls the folder name for downloaded content. Uses the same template variables as the file templates above. - -If not configured, the default folder naming is used: -- Movies: `Title (Year)` -- Series: Derived from the `series` template with episode-specific variables removed -- Songs: `Artist - Album (Year)` - -`folder` accepts either a single string (applies to all title kinds) or a mapping with per-kind -templates keyed by `movies`, `series`, `songs`, and/or `albums`. Unknown keys are warned about and ignored. - -Use `/` in a folder template to create nested directories - each segment is sanitized -independently and joined as real path separators. For example `'{source}/Series/{title}.{year?}'` -produces `EXAMPLE/Series/The.Show.2024/`. Note `{source}` is the service **tag** (e.g. `EXAMPLE`, -`EXAMPLE2`), not the display name; it is blank when `--no-source` is used. - -Movies only get their own folder when a `movies` folder template is set; without one the movie file -is written directly into the downloads directory (unchanged default behaviour). Series and songs -always get a folder unless `--no-folder` is passed. - -Useful music variables for album folders: `album_artist`, `album`, `artist`, `year`, `genre`, `label`, `release_type`, `track_total`, `disc_total`. - -```yaml -output_template: - movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' - series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' - songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' - - # Scene-style folder (single template, applies to all kinds) - folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' - - # Plex-friendly folder - # folder: '{title} ({year?})' - - # Per-kind folder templates - # folder: - # movies: '{title} ({year})' - # series: '{title} ({year?})' - # songs: '{artist} - {album} ({year?})' - # albums: '{album_artist} - {album} ({year?})' - - # Nested per-service layout (EXAMPLE/Series/Title.Year/...) - # folder: - # series: '{source}/Series/{title}.{year?}' - # movies: '{source}/Movies/{title}.{year?}' - # songs: '{source}/Music/{album_artist}/{album}.{year?}' -``` - -Example outputs: -- Scene folder: `Example.Show.2024.S01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG/` -- Plex folder: `Example Show (2024)/` -- Nested folder: `EXAMPLE/Series/Example.Show.2024/` - ---- - ---- - -## language_tags (dict) - -Automatically adds language-based identifiers (e.g., `DANiSH`, `NORDiC`, `DKsubs`) to output filenames -based on audio and subtitle track languages. Use `{lang_tag?}` in your `output_template` to place the tag. - -Rules are evaluated in order; the first matching rule wins. All conditions within a single rule -must match (AND logic). If no rules match, `{lang_tag?}` is cleanly removed from the filename. - -### Conditions - -| Condition | Type | Description | -|-----------|------|-------------| -| `audio` | string | Matches if any selected audio track has this language | -| `subs_contain` | string | Matches if any selected subtitle has this language | -| `subs_contain_all` | list | Matches if subtitles include ALL listed languages | - -Language matching uses fuzzy matching (e.g., `en` matches `en-US`, `en-GB`). - -### Example: Nordic tagging - -```yaml -language_tags: - rules: - - audio: da - tag: DANiSH - - audio: sv - tag: SWEDiSH - - audio: nb - tag: NORWEGiAN - - audio: en - subs_contain_all: [da, sv, nb] - tag: NORDiC - - audio: en - subs_contain: da - tag: DKsubs - -output_template: - movies: '{title}.{year?}.{lang_tag?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}' -``` - -Example outputs: -- Danish audio: `Example.Show.S01E01.DANiSH.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` -- English audio + multiple Nordic subs: `Example.Show.S01E01.NORDiC.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` -- English audio + Danish subs only: `Example.Show.S01E01.DKsubs.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` -- No matching languages: `Example.Show.S01E01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` - -### Example: Other regional tags - -```yaml -language_tags: - rules: - - audio: nl - tag: DUTCH - - audio: de - tag: GERMAN - - audio: fr - subs_contain: en - tag: ENGFR - - audio: fr - tag: FRENCH -``` - ---- - -## unicode_filenames (bool) - -Allow Unicode characters in output filenames. When `false`, Unicode characters are transliterated -to ASCII equivalents. Default: `false`. - ---- - -## tag (str) - -Group or Username to postfix to the end of download filenames following a dash. -Use `{tag}` in your output template to include it. -For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames. - ---- - -## tag_group_name (bool) - -Enable/disable tagging downloads with your group name when `tag` is set. Default: `true`. - ---- - -## tag_imdb_tmdb (bool) - -Enable/disable tagging downloaded files with IMDB/TMDB/TVDB identifiers (when available). Default: `true`. - ---- - -## muxing (dict) - -- `set_title` - Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true` -- `merge_audio` - Merge all audio tracks into each output file. Default: `true` - - `true`: All selected audio tracks are muxed into one MKV per quality. - - `false`: Separate MKV per (quality, audio_codec) combination. - For example: `Title.1080p.AAC.mkv`, `Title.1080p.EC3.mkv`. - - Note: The `--split-audio` CLI flag overrides this setting. When `--split-audio` is passed, - `merge_audio` is effectively set to `false` for that run. - -- `merge_video` - Merge video **language variants** into one file. Default: `false` - - `false`: One MKV per video track (the default behaviour). - - `true`: Group the selected video tracks by `(resolution, range, codec)` and merge - each group into one MKV, so only language varies within a file. The player switches - between the language tracks. No re-encode, no concatenation. - - Only the language dimension is collapsed. Different **resolutions**, **ranges** - (SDR/HDR10/HDR10+/DV/HYBRID) and **codecs** (H264/H265) always stay in separate files. - For example, `-r HYBRID,DV,HDR10,SDR --merge-video` produces one file per range (never a - single combined file), while a title offering English + French video of the same - resolution/range/codec produces one file containing both video tracks. - - Note: The `--merge-video` CLI flag overrides this setting. Can be set per service under - `services.<TAG>.muxing.merge_video`. Change `group_videos_by_variant` in - `unshackle/commands/dl.py` to adjust the grouping. - -- `default_language` (dict) - Override which track is flagged as the default in the muxed MKV, regardless - of the title's original language. Useful when you always want your player to - open on a specific language (e.g. always default to Polish audio even on - English originals). Only affects the MKV `--default-track` flag - track - selection (`-l`, `--alang`, etc.) is unchanged. All keys are optional; each - track type falls back to its previous default rule when the configured - language isn't present in the manifest. - - - `audio`: BCP-47 tag (e.g. `pl`, `en`, `pt-BR`). Wins over `is_original_lang`. - The `--original-flag` continues to mark the true original-audio track. - - `video`: BCP-47 tag. Wins over the title-language / first-track rule. - - `subtitle`: BCP-47 tag. Wins over the "forced sub matching audio" rule. - - Languages are matched with the same close-match logic used elsewhere - (`pt` matches `pt-BR`, etc.). Supports per-service overrides like the rest - of `muxing`. - - ```yaml - muxing: - default_language: - audio: pl - video: pl - subtitle: pl - ``` - ---- - -## chapter_fallback_name (str) - -The Chapter Name to use when exporting a Chapter without a Name. -The default is no fallback name at all and no Chapter name will be set. - -The fallback name can use the following variables in f-string style: - -- `{i}`: The Chapter number starting at 1. - E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3". -- `{j}`: A number starting at 1 that increments any time a Chapter has no title. - E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2". - -These are formatted with f-strings, directives are supported. -For example, `"Chapter {i:02}"` will result in `"Chapter 01"`. - ---- - -## directories (dict) - -Override the default directories used across unshackle. -The directories are set to common values by default. - -The following directories are available and may be overridden, - -- `commands` - CLI Command Classes. -- `services` - Service Classes. -- `vaults` - Vault Classes. -- `fonts` - Font files (ttf or otf). -- `downloads` - Downloads. -- `temp` - Temporary files or conversions during download. -- `cache` - Expiring data like Authorization tokens, or other misc data. -- `cookies` - Expiring Cookie data. -- `logs` - Logs. -- `exports` - JSON sidecar exports written when `--export` is used on `dl`. -- `wvds` - Widevine Devices. -- `prds` - PlayReady Devices. -- `dcsl` - Device Certificate Status List. - -Notes: - -- `services` accepts either a single directory or a list of directories to search for service modules. - Entries may also be **git repo specs** to load service packs hosted in a repo (see below). - -For example, - -```yaml -directories: - downloads: "D:/Downloads/unshackle" - temp: "D:/Temp/unshackle" -``` - -There are directories not listed that cannot be modified as they are crucial to the operation of unshackle. - -### Loading services from a git repo - -A `services` entry may be a git repo instead of a local path, letting you host service packs on -GitHub or any git host (GitLab, Gitea, self-hosted). Local paths and repo specs can be mixed: - -```yaml -directories: - services: - - https://github.com/you/your-services # https repo (highest priority - listed first) - - git@gitlab.com:you/your-services.git # ssh repo (private, via your git auth) - - you/your-services@main # owner/repo shorthand + optional @branch - - ~/my-local-services # local path (fallback - listed last) -``` - -How it works: - -- On first use the repo is cloned (shallow) to `<your-services-dir>/_repos/<repo-name>/` - the first - local `services` entry, or the bundled `unshackle/services` if you configured none. **Nothing is - written to the `cache` directory.** -- After that, unshackle does **not** hit the network on every run. It re-pulls at most once every 24h, - or immediately when you run `unshackle util refresh-services`. -- Requires `git` on your PATH. Private repos use your existing git credential helper - unshackle - stores no tokens. Git use is **read-only on the remote** - only `clone`, `fetch`, `pull`, and a - local `reset` are run; nothing is ever pushed. -- **Local edits and refresh.** You can edit a clone under `_repos/` directly. The two refresh paths - treat your edits differently: - - *Automatic* (the 24h TTL during a normal `dl`/`search` run): if the clone has uncommitted - changes to tracked files or unpushed local commits, unshackle **refuses to refresh and exits**, - naming the clone, so a background pull never clobbers work in progress. Commit and push it - upstream (or revert), then it refreshes normally. - - *Manual* (`unshackle util refresh-services`): an explicit "get upstream's latest" - it - **hard-resets the clone to upstream, discarding local changes**. Run it only when you want to - throw away local edits. - - Untracked files (new service folders, `__pycache__`) never block the automatic path - a - fast-forward pull doesn't touch them. -- The repo's **top level** must contain `<TAG>/__init__.py` service dirs (same layout as - `unshackle/services/`). -- **Priority is list order.** The first source to define a tag is the one that loads; if a later - source (repo or local) has the same tag, that copy is treated as a duplicate and ignored. So list - the sources you trust most first - e.g. repos first and local last to make local a fallback, or - local first to let your local tweaks override a repo. -- **What you see on load.** A one-line summary is logged each run, e.g.: - - ```text - Loaded 103 services (36 duplicate(s) ignored) - ``` - - The full per-duplicate detail is logged only at debug verbosity (`unshackle -d ...`), one line per - duplicate, naming the path that loaded and the path that was ignored: - - ```text - EXAMPLE: using <unshackle>/services/EXAMPLE/__init__.py, ignoring duplicate <unshackle>/services/_repos/your-repo/EXAMPLE/__init__.py - ``` - - Paths are shortened to `<unshackle>`/`<venv>`/`~` tokens; set `redact_paths: false` to show full - absolute paths (see [Debug Logging](DEBUG_LOGGING.md)). - -If `<your-services-dir>` is inside the installed package, a reinstall may remove the clones; they are -simply re-cloned on next use. On read-only installs, point `services` at a writable path. - ---- diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index f5ba502b..00000000 --- a/docs/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# unshackle Documentation - -This is the **canonical configuration reference** for unshackle. `unshackle/unshackle-example.yaml` is a minimal starter config only - it intentionally omits most options. Come here for the full picture. - -## Contents - -| Document | Description | -|---|---| -| [Creating a Service](CREATING_A_SERVICE.md) | Section-by-section guide to writing a new service adapter, based on the `EXAMPLE` service | -| [Advanced & System Configuration](ADVANCED_CONFIG.md) | `serve`, debugging, and system-level options not covered elsewhere | -| [REST API Documentation](API.md) | HTTP API exposed by `unshackle serve`: endpoints, auth, remote downloads, CDM hosting | -| [Structured Debug Logging](DEBUG_LOGGING.md) | `-d`/`--debug` JSONL output: schema, redaction rules, `log_event`/`timed_operation` conventions | -| [Download & Processing Configuration](DOWNLOAD_CONFIG.md) | Downloader settings, concurrency, retries, and post-processing behaviour | -| [DRM & CDM Configuration](DRM_CONFIG.md) | Widevine, PlayReady, and ClearKey CDM setup; local and remote CDMs; key vaults | -| [Gluetun VPN Proxy](GLUETUN.md) | Docker-managed VPN proxies via Gluetun; setup, WireGuard ready-detection | -| [Network & Proxy Configuration](NETWORK_CONFIG.md) | Proxy providers, geofence bypass, per-service proxy overrides | -| [Output & Naming Configuration](OUTPUT_CONFIG.md) | Output directory layout, filename templates, folder naming | -| [Service Integration & Authentication](SERVICE_CONFIG.md) | Per-service config blocks, cookies, credentials, and metadata sources | -| [Subtitle Processing Configuration](SUBTITLE_CONFIG.md) | `subtitle:` key options: format conversion, styling, forced-sub rules | diff --git a/docs/SERVICE_CONFIG.md b/docs/SERVICE_CONFIG.md deleted file mode 100644 index 591f20a4..00000000 --- a/docs/SERVICE_CONFIG.md +++ /dev/null @@ -1,302 +0,0 @@ -# Service Integration & Authentication Configuration - -This document covers service-specific configuration, authentication, and metadata integration options. - -## services (dict) - -Configuration data for each Service. The Service will have the data within this section merged into the per-service -`config.yaml` (located in the service's directory) before being provided to the Service class. - -Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs, -device attributes, and so on. A per-service `config.yaml` file is typically shared and not meant to be modified, -so use this for any sensitive configuration data. - -The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or -a dictionary. - -For example, - -```yaml -EXAMPLE: - client: - auth_scheme: MESSO - # ... more sensitive data -``` - -### Per-Service Configuration Overrides - -You can override many global configuration options on a per-service basis by nesting them under the -service tag in the `services` section. Supported override keys include: `dl`, `subtitle`, `muxing`, -`headers`, `proxy_map`, `title_map`, and more. - -Overrides are merged with global config (not replaced) -- only specified keys are overridden, others -use global defaults. - -Any `dl` command option can be overridden per service. Use the option name with dashes as underscores -(`--v-lang` -> `v_lang`). `range` and `list` work as-is; their internal Python names (`range_`, -`list_`, suffixed to avoid the builtins) are also accepted. - -Precedence (highest first): - -1. Explicit CLI arguments / environment variables (e.g. `--v-lang en`) -2. Per-service config (`services.<TAG>.dl`) -3. Global `dl:` config -4. Built-in option defaults - -For example, - -```yaml -services: - RATE_LIMITED_SERVICE: - dl: - downloads: 2 # Limit concurrent track downloads - workers: 4 # Reduce workers to avoid rate limits - headers: - User-Agent: "..." # Service-specific UA override -``` - -Note: unshackle uses a single unified `requests`-based downloader. The legacy `aria2c`, -`n_m3u8dl_re`, and `curl_impersonate` override sections have been removed. - -### title_map (dict) - -Rewrites service-provided titles before naming and output. Some services name a title differently -from how you want it stored, which can break library matching (e.g. a regional variant reusing the -international name). Keys are the exact title string the service returns; values are the desired -output title. - -```yaml -services: - EXAMPLE: - title_map: - Service Title: Desired Title -``` - -Episodes are matched on their show title, Movies and Songs on their name. The remap is applied -after the title cache (so edits take effect without a cache reset) and before any `--enrich` -override (so an explicit enrich still wins). - -It applies on the local `dl` path, the `import` command, and the remote client (`dl --remote`). -For remote services the **client's** `title_map` is applied to the titles returned by the server, -so you can rename titles for services you don't have installed locally. The server sends raw -titles and does not remap, leaving the final name fully under the client's control. - -### Service Class Conventions - -Each service directory under `unshackle/services/` exports a class extending -`unshackle.core.service.Service`. The class name must match the directory name (the service tag). - -Key class variables (defined on `Service` or by service-level idiom): - -- `ALIASES: tuple[str, ...]` — alternative tags accepted on the CLI. Empty by default. -- `GEOFENCE: tuple[str, ...]` — ISO country codes the service is available in. Empty == no geofence. -- `TITLE_RE: str` — regex (with named groups, e.g. `(?P<id>...)`, `(?P<type>...)`) used by the - service to parse the CLI title argument. Service-level idiom, not declared on the base class. -- `NO_SUBTITLES: bool` — service-level idiom indicating the service has no subtitle tracks. - -`self.*` helpers available after `super().__init__(ctx)`: - -- `self.session` — pre-configured HTTP session (`requests.Session`, or `RnetSession` when TLS - impersonation is active). Cookies, headers, proxies pre-applied. -- `self.config` — merged service config (per-service `config.yaml` plus the `services.<TAG>` block - from `unshackle.yaml`). -- `self.log` — `logging.Logger` named for the service class. -- `self.cache` — generic `Cacher` for arbitrary key/value persistence. -- `self.title_cache` — specialized `TitleCacher` for title metadata. -- `self.track_request` — `TrackRequest` built from CLI flags. Fields: `codecs: list[Video.Codec]`, - `ranges: list[Video.Range]` (defaults to `[SDR]`), `best_available: bool`. Services may - read or rewrite these (e.g. force HEVC for HDR ranges). -- `self.credential` — set during `authenticate()`; `None` if cookies-only. -- `self.current_region` — lowercase ISO country code from proxy/geolocation, or `None`. -- `self.request_input(prompt: str) -> str` — interactive prompt. Falls through to `input()` - locally; under `serve`, the attached `InputBridge` relays the prompt to the remote client. - -Driving CLI flags (parsed into `self.track_request`): - -- `-v` / `--vcodec` — comma-separated `Video.Codec` list (e.g. `H264,H265`). -- `-a` / `--acodec` — comma-separated audio codec list. -- `-r` / `--range` — comma-separated `Video.Range` list (`SDR`, `HDR10`, `HDR10+`, `DV`, - `HYBRID`). Defaults to `[SDR]`. -- `-q` / `--quality` — resolution list. -- `--vbitrate-range` / `--abitrate-range` — `MIN-MAX` kbps windows. - ---- - -## credentials (dict[str, str|list|dict]) - -Specify login credentials to use for each Service, and optionally per-profile. - -For example, - -```yaml -EXAMPLE: jane@example.tld:LoremIpsum100 # directly -EXAMPLE2: # or per-profile, optionally with a default - default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used - james: james@example.tld:TheFriend97 - john: john@example.tld:LoremIpsum98 -EXAMPLE3: # the `default` key is not necessary, but no credential will be used by default - john: john@example.tld:SecretPassword123 -``` - -The value should be in string form, i.e. `john@example.tld:password123` or `john:password123`. -Any arbitrary values can be used on the left (username/password/phone) and right (password/secret). -You can also specify these in list form, i.e., `["john@example.tld", ":PasswordWithAColon"]`. - -If you specify multiple credentials with keys like the `EXAMPLE2` and `EXAMPLE3` example above, then you should -use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You -do not have to use a `default` key at all. - -Please be aware that this information is sensitive and to keep it safe. Do not share your config. - ---- - -## firefox_cookies (dict) - -Automatically extract cookies from your running Firefox installation instead of maintaining a -`cookies/<SERVICE>.txt` file. Configured as a top-level dict keyed by service tag; each value is -a settings dict for that service. - -```yaml -firefox_cookies: - EXAMPLE: - hosts: - - example.com - - api.example.com - local_storage: false # optional, default false -``` - -**Fields:** - -- `hosts` — list of host patterns to match against Firefox's cookie store. Each pattern must be at - least 3 characters. A pattern matches the exact host **and** any dot-suffix subdomain - (e.g. `example.com` also matches `sub.example.com` and `.example.com`). -- `local_storage` — when `true`, also harvest `localStorage` entries for the listed hosts from - Firefox's `webappsstore.sqlite`. Default: `false`. - -At download time unshackle picks the **most recently modified** valid Firefox profile (the one -whose `cookies.sqlite` has the latest mtime), copies the database to a restricted temporary -directory (`0700`), and reads cookies from it without touching the live profile. If extraction -returns no cookies or fails for any reason, it falls back silently to the normal file-based cookie -(`cookies/<SERVICE>.txt` or `cookies/<SERVICE>/<profile>.txt`). - -**Notes:** - -- Firefox must not have an exclusive write lock on the database at the moment of extraction (i.e. - Firefox may be open — the extractor copies both `cookies.sqlite` and its WAL file to capture any - unflushed writes). -- `local_storage: true` is only needed for services that store auth tokens in the browser's - localStorage rather than in HTTP cookies. -- The feature is implemented in `core/utils/firefox_cookie_extractor.py`; the fallback logic lives - in `dl.get_cookie_jar` (`commands/dl.py`). - ---- - -## tmdb_api_key (str) - -API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB, -IMDB and TVDB identifiers. Leave empty to disable automatic lookups. - -To obtain a TMDB API key: - -1. Create an account at <https://www.themoviedb.org/> -2. Go to <https://www.themoviedb.org/settings/api> to register for API access -3. Fill out the API application form with your project details -4. Once approved, you'll receive your API key - -For example, - -```yaml -tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key -``` - -!!! note - Keep your API key secure and do not share it publicly. This key is used by the `core/providers/tmdb.py` metadata provider to fetch metadata from TMDB for proper file tagging and ID enrichment. - ---- - -## simkl_client_id (str) - -Client ID for SIMKL API integration. SIMKL is used as a metadata source for improved title matching and tagging, -especially when a TMDB API key is not configured. - -To obtain a SIMKL Client ID: - -1. Create an account at <https://simkl.com/> -2. Go to <https://simkl.com/settings/developer/> -3. Register a new application to receive your Client ID - -For example, - -```yaml -simkl_client_id: "your_client_id_here" -``` - -!!! note - While optional, having a SIMKL Client ID improves metadata lookup reliability. SIMKL serves as an alternative or fallback metadata source to TMDB. This is used by the `core/providers/simkl.py` metadata provider. - ---- - -## ipinfo_api_key (str) - -Optional API token for [ipinfo.io](https://ipinfo.io). When set, unshackle uses the free authenticated **Lite** endpoint (`https://api.ipinfo.io/lite/me`), which has substantially higher rate limits than the anonymous endpoint and returns richer fields (ASN, organization name, continent). Leave empty to use the anonymous ipinfo.io endpoint, with [ip-api.in](https://ip-api.in) as a final fallback. - -To obtain an ipinfo.io token: - -1. Sign up for a free account at <https://ipinfo.io/signup> -2. Copy the token from your dashboard - -For example, - -```yaml -ipinfo_api_key: "12a3b45cd678ef" # Not a real key -``` - -!!! note - The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header — it is never attached to your session for service requests. Used by `core/utils/ip_info.py` for region detection and proxy verification. - ---- - -## title_cache_enabled (bool) - -Enable/disable caching of title metadata to reduce redundant API calls. Default: `true`. - ---- - -## title_cache_time (int) - -Cache duration in seconds for title metadata. Default: `1800` (30 minutes). - ---- - -## title_cache_max_retention (int) - -Maximum retention time in seconds for serving slightly stale cached title metadata when API calls fail. -Default: `86400` (24 hours). Effective retention is `min(title_cache_time + grace, title_cache_max_retention)`. - ---- - -## debug (bool) - -Enable structured JSON debug logging for troubleshooting and service development. Default: `false`. - -When enabled (via config or the `--debug` CLI flag): -- Creates JSON Lines (`.jsonl`) log files with complete debugging context -- Logs: session info, CLI params, service config, CDM details, authentication, titles, tracks metadata, - DRM operations, vault queries, errors with stack traces -- File location: `logs/unshackle_debug_{service}_{timestamp}.jsonl` - ---- - -## debug_keys (bool) - -Log decryption keys in debug logs. Default: `false`. - -When `true`, actual content encryption keys (CEKs) are included in debug log output. Useful for -debugging key retrieval and decryption issues. - -!!! note "Security note" - Passwords, tokens, cookies, and session tokens are always redacted regardless - of this setting. Only content keys (`content_key`, `key` fields) are affected. Key IDs (`kid`), - key counts, and other metadata are always logged. - ---- diff --git a/docs/SUBTITLE_CONFIG.md b/docs/SUBTITLE_CONFIG.md deleted file mode 100644 index cb6c1617..00000000 --- a/docs/SUBTITLE_CONFIG.md +++ /dev/null @@ -1,101 +0,0 @@ -# Subtitle Processing Configuration - -This document covers subtitle processing and formatting options under the top-level `subtitle:` key in `unshackle.yaml`. - -This document is the canonical reference for subtitle configuration. `unshackle/unshackle-example.yaml` is a minimal starter only. - -## subtitle (dict) - -Control subtitle conversion, SDH (hearing-impaired) stripping, formatting preservation, and output behavior. - -- `conversion_method`: Which backend to convert subtitles with. Default: `auto`. - - Routing is data-driven (`unshackle/core/tracks/subtitle_convert.py`): a registry of backends each - declares the source→target codec pairs it supports plus a preference rank. For a conversion, the - available backends that support the pair are tried in rank order — a real fallback chain. A - non-`auto` value **pins** that backend first, then still falls back through the chain if it can't - handle the pair or errors (pin-then-fallback). A service may also set `preferred_conversion_method` - on its tracks; an explicit `conversion_method` in config always wins. - - - `auto`: Best available backend by rank — SubtitleEdit (if installed) for highest fidelity; - otherwise subby for WebVTT/fVTT/SAMI→SRT (adds `CommonIssuesFixer` cleanup), pysubs2 for SSA/ASS - and the broad format set, pycaption as last resort. - - `subby`: Prefer subby (`CommonIssuesFixer`); reads WebVTT/fVTT/SAMI, writes SRT (and TTML/VTT via - an SRT intermediate). - - `subtitleedit`: Prefer SubtitleEdit / `seconv`. Highest fidelity — preserves positioning/italics. - - `pycaption`: Prefer pycaption. **Flattens positioning/italics**, writes only SRT/TTML/WebVTT. - - `pysubs2`: Prefer pysubs2 (SRT, SSA, ASS, WebVTT, TTML, SAMI, MicroDVD, MPL2, TMP). The only - pure-Python backend that reads ASS/SSA, so it is the default for styled SubStation sources. - - **Styled-subtitle protection**: ASS/SSA are never *automatically* downconverted to SRT (the - conversion is skipped and the original kept) — SRT cannot carry their positioning/colours/styling. - This applies to the default muxed track only; explicit requests still convert: a per-download - `--sub-format srt` for the muxed track, or `sidecar_format: srt` for sidecars. To keep raw styled - sidecars, set `sidecar_format: original`. - - **Segmented subtitles** (`fVTT`/WVTT and `fTTML`/STPP from DASH/HLS) are read directly - from the fragmented MP4: fVTT via subby's `WVTTConverter`, fTTML via pycaption's box parsing. They - can be converted *from* but not *to*. - - **SubtitleEdit on Linux/macOS**: install the SubtitleEdit 5+ CLI (`SeConv` / `seconv`, the - self-contained cross-platform build from the SubtitleEdit releases) onto `PATH` or into - `unshackle/binaries/`. unshackle targets the SubtitleEdit **5+** command syntax. The Windows - `SubtitleEdit.exe` is the GUI app — use the `SeConv` CLI binary for headless conversion. - -- `sdh_method`: How to strip SDH cues. Default: `auto`. - - `auto`: Try subby for SRT first, then SubtitleEdit (when `conversion_method` is `auto`/`subtitleedit` and the binary is available), then subtitle-filter as the final fallback. - - `subby`: Use subby's `SDHStripper`. **Only operates on SRT**; for other codecs the call returns without stripping. - - `subtitleedit`: Use SubtitleEdit's `--remove-text-for-hi` (SE5 CLI) when the binary is available; otherwise falls through to subtitle-filter. - - `filter-subs`: Use the `subtitle-filter` library directly (`rm_fonts`, `rm_ast`, `rm_music`, `rm_effects`, `rm_names`, `rm_author`). - -- `strip_sdh`: Enable/disable automatic SDH stripping for tracks flagged as SDH. Default: `true`. - -- `convert_before_strip`: When falling through to the subtitle-filter path, auto-convert non-SRT subtitles to SRT first for better compatibility. Default: `true`. Has no effect when SubtitleEdit handles stripping directly. - -- `preserve_formatting`: Keep original subtitle tags and positioning during WebVTT processing. When `true`, sanitized WebVTT is written back without round-tripping through pycaption, preserving tags like `<i>`, `<b>`, and `line:` positioning. Default: `true`. - -- `output_mode`: Controls how subtitles are included in the output. Default: `mux`. - - `mux`: Embed subtitles in the MKV container only. - - `sidecar`: Save subtitles as separate files only (not muxed). - - `both`: Embed in the MKV container and save as sidecar files. - -- `sidecar_format`: Format for sidecar subtitle files (used when `output_mode` is `sidecar` or `both`). Default: `srt`. - - `srt`: SubRip. - - `vtt`: WebVTT. - - `ass`: Advanced SubStation Alpha. - - `original`: Keep the subtitle in its current format without conversion. - -Example: - -```yaml -subtitle: - conversion_method: auto - sdh_method: auto - strip_sdh: true - convert_before_strip: true - preserve_formatting: true - output_mode: mux - sidecar_format: srt -``` - -## WebVTT Sanitization (automatic, not configurable) - -After download, WebVTT and segmented WebVTT (`fVTT`/`WVTT`) tracks pass through a fixed sanitization pipeline before any conversion or muxing: - -1. **Segment merge** — segmented DASH/HLS WebVTT is stitched via `merge_segmented_webvtt` (uses pysubs2 for lenient parsing when `conversion_method` is `auto` or `pysubs2`, otherwise pycaption directly). -2. **Negative timestamps** — `sanitize_webvtt_timestamps` rewrites `-HH:MM:SS.mmm` cues to `00:00:00.000`. -3. **Cue identifiers** — `sanitize_webvtt_cue_identifiers` strips letter+digit IDs (e.g. `Q0`, `S12`) on their own line before a timing line, which otherwise confuse parsers like pysubs2. -4. **Overlapping cues** — `merge_overlapping_webvtt_cues` collapses cues with start times within 50 ms and matching end times into a single multi-line cue, ordered by `line:` percentage (lower % = higher on screen = first line). -5. **Fallback hardening** — when `preserve_formatting` is `false` and the first pycaption parse fails, `sanitize_webvtt` retries with a `WEBVTT` header guard, hour-padded timings, and another negative-timestamp pass; if that still fails, the sanitized text is written as-is. - -`sanitize_broken_webvtt` and `space_webvtt_headers` additionally run inside `Subtitle.parse()` to drop malformed `-->` lines and reflow merged-segment headers. `merge_same_cues` and `filter_unwanted_cues` (drops ` `/whitespace-only cues) run only on the pycaption path. - -These behaviors are intentional and have no config knobs — they apply to every WebVTT track regardless of `conversion_method`. - -## Related - -- Filename sanitization (e.g. parenthesis handling, unidecode bracket artifacts from PR #105) lives in `unshackle/core/utilities.py::sanitize_filename` and is governed by `output_template`, not the `subtitle:` config block. -- Subtitle codec support is defined in `unshackle/core/tracks/subtitle.py`; the conversion backend - registry, capability matrix, and ranks live in `unshackle/core/tracks/subtitle_convert.py`. - ---- diff --git a/docs/api/authentication.md b/docs/api/authentication.md deleted file mode 100644 index 6801895e..00000000 --- a/docs/api/authentication.md +++ /dev/null @@ -1,75 +0,0 @@ -# Authentication - -When `api_secret` is set in `unshackle.yaml`, all API requests require the **`X-Secret-Key`** header. There is no query-parameter fallback. `/api/health` is always reachable without authentication. `--no-key` disables auth entirely (not recommended for public-facing servers). - -```yaml -# unshackle.yaml -serve: - api_secret: "your-master-secret" # falls back to global users map below - remote_only: false # also toggleable via --remote-only - services: ["EXAMPLE1", "EXAMPLE2"] # optional global service allowlist - users: - user-secret-1: - username: alice - devices: ["my_widevine_l3"] # Widevine WVD names this user may use - playready_devices: ["my_pr_sl2000"] # PlayReady PRD names; defaults to [] (no access) - services: ["EXAMPLE1"] # optional per-user allowlist (intersected with global) - user-secret-2: - username: bob - devices: [] - playready_devices: [] -``` - -## Service allowlists - -`config.serve.services` is the global allowlist; `users.<key>.services` further narrows it per key. The effective set is the intersection. Endpoints affected: `/api/services`, `/api/search`, `/api/list-titles`, `/api/list-tracks`, `/api/download`, and all `/api/session/*` routes. - -## CDM access (server-side decryption) - -There is no separate "tier" flag. Whether the server can return KID:KEY for a session-mode download depends solely on the device lists configured for the calling user key: - -- Empty `devices` and `playready_devices` -> server can only proxy CDM challenges; the client must run its own CDM and parse the license. -- Populated lists -> the client may set `mode: "server_cdm"` on `/api/session/{id}/license` and receive `{ "keys": { "<track_id>": { "<KID>": "<KEY>" } } }` instead of raw license bytes. - -Per-service CDM type can be pinned via `config.cdm` (`widevine`/`playready`) or per-service `cdm_type`; otherwise the server picks the type the user has devices for. - -## Server-side `dl` defaults - -Any flag accepted by `/api/download` (see the table below) can be declared under `serve:` in `unshackle.yaml` and the API will apply it as a default. Request-body values still win. Useful for raising concurrency without changing every client call: - -```yaml -serve: - api_secret: "..." - users: { ... } - downloads: 4 # parallel tracks per download job - workers: 16 # threads per track segment fetch - best_available: true - no_proxy_download: false -``` - -Layering order: built-in defaults < `serve.*` overrides < service-specific click defaults < request body. - -## Per-request override gates - -Two security gates under `serve:` guard capabilities that `/api/download` clients can request. Both default OFF; an unset or `false` value causes the server to reject the request with `403 FORBIDDEN`. - -**`cdm_overrides`** — a `/api/download` job body may include a `cdm` field to select a specific server-side device. When this gate is off, every such request is rejected. Set to `true` to allow any device the server has, or to a list of device names to restrict to an explicit subset: - -```yaml -serve: - # Allow any server-side device: - cdm_overrides: true - - # Or restrict to specific devices: - cdm_overrides: - - generic_nexus_4464_l3 -``` - -**`allow_job_credentials`** — a `/api/download` job body may include a `credential` (single) or `credentials` (map) field to authenticate the job with client-supplied secrets instead of the server's configured credentials. When this gate is off, any job containing either field is rejected. Each distinct credential gets its own isolated token cache on the server. Set to `true` to allow: - -```yaml -serve: - allow_job_credentials: true -``` - -Both gates are independent and can be combined. A typical locked-down deployment leaves both unset; a trusted single-client deployment might enable both. diff --git a/docs/api/client-configuration.md b/docs/api/client-configuration.md deleted file mode 100644 index c12d5054..00000000 --- a/docs/api/client-configuration.md +++ /dev/null @@ -1,64 +0,0 @@ -# Client Configuration: `remote_services` - -`remote_services` is a top-level block in `unshackle.yaml` on the **client** machine. It tells `dl` how to reach a remote `unshackle serve` instance instead of running service code locally. The client handles track selection, download, decrypt, and mux; the server handles auth, manifest parsing, and (optionally) DRM licensing. - -## CLI flags - -| Flag | Description | -| --- | --- | -| `--remote` | Use a remote server instead of local service code. Reads `remote_services` from config. | -| `--server <name>` | Select a named server when multiple are configured. If only one entry exists it is picked automatically. | - -```bash -unshackle dl --remote EXAMPLE1 "https://example.com/show/abc123" -unshackle dl --remote --server us-server EXAMPLE1 "https://example.com/show/abc123" -``` - -## Config structure - -```yaml -remote_services: - my-server: - url: "http://192.168.1.100:8786" # base URL of the remote serve instance - api_key: "your-secret-key-here" # sent as X-Secret-Key; omit or set "" for --no-key servers - - # server_cdm: the server runs its own CDM and returns KID:KEY pairs (mode: server_cdm). - # false (default): client runs its own CDM; license challenges are proxied through the server. - server_cdm: false - - # Per-service overrides applied on the client side after the remote session is set up. - # Supported keys: decryption, downloader, cdm (and any free-form service config keys). - services: - EXAMPLE1: - decryption: mp4decrypt # override decryption tool for this service (shaka or mp4decrypt) -``` - -**`server_cdm`** — when `false` (default) the client builds CDM challenges locally and calls `POST /api/session/{id}/license` with `mode: "proxy"`; the server forwards the challenge to the service's license endpoint and returns raw license bytes. When `true` the client skips local CDM setup and calls with `mode: "server_cdm"`; the server performs the full CDM flow and returns `{ "keys": { "<KID>": "<KEY>" } }`. The server-side user key must have a matching device configured (`devices` / `playready_devices`) for `server_cdm` to work. - -## Multiple servers - -When more than one server is configured, `--server <name>` is required: - -```yaml -remote_services: - us-server: - url: "https://us.example.com:8786" - api_key: "us-key" - server_cdm: true - services: - EXAMPLE1: - decryption: mp4decrypt - - eu-server: - url: "https://eu.example.com:8786" - api_key: "eu-key" - server_cdm: false -``` - -```bash -unshackle dl --remote --server us-server EXAMPLE1 "https://example.com/show/abc123" -``` - -## Cache forwarding - -On session open the client sends any local `*.json` cache files for the service (tokens, etc.) to the server so it can skip interactive re-auth. On session close the server returns updated cache files; the client writes them back locally so the next remote session can forward them again. diff --git a/docs/api/download-job-lifecycle.md b/docs/api/download-job-lifecycle.md deleted file mode 100644 index ce3c1965..00000000 --- a/docs/api/download-job-lifecycle.md +++ /dev/null @@ -1,16 +0,0 @@ -# Download Job Lifecycle - -```text -queued -> downloading -> completed - \-> failed -queued -> cancelled -downloading -> cancelled -``` - -## Progress reporting - -A multi-episode request (`wanted: ["S01E01", ...]`) is a single job that processes episodes sequentially. During a run the job status updates live: `current_title` names the episode downloading now, `output_files` grows as each episode finishes muxing, and `track_progress` lists per-track progress when `downloads` > 1 runs tracks concurrently. `progress` and track counts reset as each episode starts; use `output_files` against `parameters.wanted` to derive overall completion. - -Jobs are retained for 24 hours after completion (override via top-level `download_job_retention_hours` in `unshackle.yaml`). The server runs up to 2 concurrent download jobs by default; override via top-level `max_concurrent_downloads`. This is independent of `serve.downloads`, which controls parallel tracks **within** a single job. - -Remote sessions are managed by `SessionStore` (`unshackle/core/api/session_store.py`); idle sessions and their `InputBridge` instances are cleaned up by a background loop started/stopped with the app lifecycle. diff --git a/docs/api/endpoint-map.md b/docs/api/endpoint-map.md deleted file mode 100644 index 2cbd0c7e..00000000 --- a/docs/api/endpoint-map.md +++ /dev/null @@ -1,26 +0,0 @@ -# Endpoint Map - -Standard endpoints (suppressed in `--remote-only` mode are marked R): - -| Method | Path | R | -| --- | --- | :-: | -| GET | `/api/health` | ok | -| GET | `/api/services` | ok | -| POST | `/api/search` | ok | -| POST | `/api/list-titles` | hidden | -| POST | `/api/list-tracks` | hidden | -| POST | `/api/download` | hidden | -| GET | `/api/download/jobs` | hidden | -| GET | `/api/download/jobs/{job_id}` | hidden | -| DELETE | `/api/download/jobs/{job_id}` | hidden | -| POST | `/api/session/create` | ok | -| GET | `/api/session/{session_id}` | ok | -| DELETE | `/api/session/{session_id}` | ok | -| GET | `/api/session/{session_id}/titles` | ok | -| POST | `/api/session/{session_id}/tracks` | ok | -| POST | `/api/session/{session_id}/segments` | ok | -| POST | `/api/session/{session_id}/license` | ok | -| GET | `/api/session/{session_id}/prompt` | ok | -| POST | `/api/session/{session_id}/prompt` | ok | - -CDM endpoints (`/{wvd}/...`, `/playready/{prd}/...`) are exposed unless `--api-only` / `--remote-only` / `--no-widevine` / `--no-playready` is set, and use pywidevine / pyplayready's own auth scheme. diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md deleted file mode 100644 index 3a91c706..00000000 --- a/docs/api/endpoints.md +++ /dev/null @@ -1,519 +0,0 @@ -# Endpoints - -## GET /api/health - -Health check with version and update information. Always reachable without auth. - -```bash -curl http://localhost:8786/api/health -``` - -```json -{ - "status": "ok", - "version": "4.0.0", - "update_check": { - "update_available": false, - "current_version": "4.0.0", - "latest_version": null - } -} -``` - ---- - -## GET /api/services - -List all available streaming services (filtered by the effective allowlist for the caller). - -```bash -curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/services -``` - -Returns `{"services": [...]}`. Each entry has `tag`, `aliases`, `geofence`, `title_regex`, `url` (from `cli.short_help`), `help` (full docstring), and `cli_params` describing the service-level Click parameters. - ---- - -## POST /api/search - -Search for titles from a streaming service. - -**Required parameters:** -| Parameter | Type | Description | -| --- | --- | --- | -| `service` | string | Service tag | -| `query` | string | Search query | - -**Optional parameters:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `profile` | string | `null` | Profile for credentials/cookies | -| `proxy` | string | `null` | Proxy URI or country code | -| `no_proxy` | boolean | `false` | Disable all proxy use | - -```bash -curl -X POST http://localhost:8786/api/search \ - -H "X-Secret-Key: $KEY" \ - -H "Content-Type: application/json" \ - -d '{"service": "EXAMPLE1", "query": "example show"}' -``` - -```json -{ - "results": [ - { - "id": "abc123def456", - "title": "Example Show", - "description": null, - "label": "TV Show", - "url": "https://example.com/show/abc123def456" - } - ], - "count": 1 -} -``` - ---- - -## POST /api/list-titles - -Get available titles (seasons/episodes/movies) for a service and title ID. Disabled in `--remote-only` mode. - -**Required parameters:** -| Parameter | Type | Description | -| --- | --- | --- | -| `service` | string | Service tag | -| `title_id` | string | Title ID or URL | - -```bash -curl -X POST http://localhost:8786/api/list-titles \ - -H "X-Secret-Key: $KEY" \ - -H "Content-Type: application/json" \ - -d '{"service": "EXAMPLE1", "title_id": "abc123def456"}' -``` - ---- - -## POST /api/list-tracks - -Get video, audio, and subtitle tracks for a title. Disabled in `--remote-only` mode. - -**Required parameters:** -| Parameter | Type | Description | -| --- | --- | --- | -| `service` | string | Service tag | -| `title_id` | string | Title ID or URL | - -**Optional parameters:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `wanted` | array | all | Episode filter (e.g., `["S01E01"]`) | -| `profile` | string | `null` | Profile for credentials/cookies | -| `proxy` | string | `null` | Proxy URI or country code | -| `no_proxy` | boolean | `false` | Disable all proxy use | - -Returns video, audio, and subtitle tracks with codec, bitrate, resolution, language, and DRM information. - ---- - -## POST /api/download - -Start a download job. Returns immediately with a job ID (HTTP 202). Disabled in `--remote-only` mode. - -**Required parameters:** -| Parameter | Type | Description | -| --- | --- | --- | -| `service` | string | Service tag | -| `title_id` | string | Title ID or URL | - -**Quality and codec parameters:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `quality` | array[int] | best | Resolution(s) (e.g., `[1080, 2160]`) | -| `vcodec` | string or array | any | Video codec(s): `H264`, `H265`/`HEVC`, `VP9`, `AV1`, `VC1`, `VP8` | -| `acodec` | string or array | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS`, `OGG` | -| `vbitrate` | int | highest | Video bitrate in kbps | -| `abitrate` | int | highest | Audio bitrate in kbps | -| `range` | array[string] | `["SDR"]` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` | -| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) | -| `no_atmos` | boolean | `false` | Exclude Dolby Atmos tracks | -| `split_audio` | boolean | `null` | Create separate output per audio codec | -| `sub_format` | string | `null` | Output subtitle format: `SRT`, `VTT`, `ASS`, `SSA`, `TTML` | - -**Episode selection:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `wanted` | array[string] | all | Episodes (e.g., `["S01E01", "S01E02-S01E05"]`) | -| `latest_episode` | boolean | `false` | Download only the most recent episode | - -**Language parameters:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `lang` | array[string] | `["orig"]` | Language for video and audio (`orig` = original) | -| `v_lang` | array[string] | `[]` | Language override for video tracks only | -| `a_lang` | array[string] | `[]` | Language override for audio tracks only | -| `s_lang` | array[string] | `["all"]` | Language for subtitles | -| `require_subs` | array[string] | `[]` | Required subtitle languages (skip if missing) | -| `forced_subs` | boolean | `false` | Include forced subtitle tracks | -| `exact_lang` | boolean | `false` | Exact language matching (no variants) | - -**Track selection:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `video_only` | boolean | `false` | Only download video tracks | -| `audio_only` | boolean | `false` | Only download audio tracks | -| `subs_only` | boolean | `false` | Only download subtitle tracks | -| `chapters_only` | boolean | `false` | Only download chapters | -| `no_video` | boolean | `false` | Skip video tracks | -| `no_audio` | boolean | `false` | Skip audio tracks | -| `no_subs` | boolean | `false` | Skip subtitle tracks | -| `no_chapters` | boolean | `false` | Skip chapters | -| `audio_description` | boolean | `false` | Include audio description tracks | - -**Output and tagging:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `tag` | string | `null` | Override group tag | -| `repack` | boolean | `false` | Add REPACK tag to filename | -| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging | -| `imdb_id` | string | `null` | Use specific IMDB ID (e.g., `tt1375666`) | -| `animeapi_id` | string | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`) | -| `enrich` | boolean | `false` | Override show title and year from external source | -| `no_folder` | boolean | `false` | Disable folder creation for TV shows | -| `no_source` | boolean | `false` | Remove source tag from filename | -| `no_mux` | boolean | `false` | Do not mux tracks into container | -| `output_dir` | string | `null` | Override output directory | - -**Download behavior:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `profile` | string | `null` | Profile for credentials/cookies | -| `proxy` | string | `null` | Proxy URI or country code | -| `no_proxy` | boolean | `false` | Disable all proxy use | -| `no_proxy_download` | boolean | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy | -| `workers` | int | `null` | Max threads per track download | -| `downloads` | int | `1` | Concurrent track downloads | -| `slow` | boolean or string | `null` | Add randomized delay between titles. `true` = 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 | -| `best_available` | boolean | `false` | Continue if requested quality unavailable | -| `worst` | boolean | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` | -| `skip_dl` | boolean | `false` | Skip download, only get decryption keys | -| `export` | boolean | `false` | Export manifest, track URLs, keys, and subtitles to JSON in the exports directory | -| `cdm_only` | boolean | `null` | Only use CDM (`true`) or only vaults (`false`) | -| `no_cache` | boolean | `false` | Bypass title cache | -| `reset_cache` | boolean | `false` | Clear title cache before fetching | - -**Example:** - -```bash -curl -X POST http://localhost:8786/api/download \ - -H "X-Secret-Key: $KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "service": "EXAMPLE1", - "title_id": "abc123def456", - "wanted": ["S01E01"], - "quality": [1080, 2160], - "vcodec": ["H265"], - "acodec": ["AAC", "EC3"], - "range": ["HDR10", "SDR"], - "split_audio": true, - "lang": ["en"] - }' -``` - -```json -{ - "job_id": "504db959-80b0-446c-a764-7924b761d613", - "status": "queued", - "created_time": "2026-02-27T18:00:00.000000" -} -``` - ---- - -## GET /api/download/jobs - -List all download jobs with optional filtering and sorting. Disabled in `--remote-only` mode. - -**Query parameters:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `status` | string | all | Filter by status: `queued`, `downloading`, `completed`, `failed`, `cancelled` | -| `service` | string | all | Filter by service tag | -| `sort_by` | string | `created_time` | Sort field: `created_time`, `started_time`, `completed_time`, `progress`, `status`, `service` | -| `sort_order` | string | `desc` | Sort order: `asc`, `desc` | -| `full` | string | `false` | `true` to include full job details per job (parameters, timestamps, `output_files`, error fields) | - -```bash -curl -H "X-Secret-Key: $KEY" "http://localhost:8786/api/download/jobs?status=completed" -``` - ---- - -## GET /api/download/jobs/{job_id} - -Get detailed information about a specific download job including progress, parameters, and error details. - -```json -{ - "job_id": "504db959-80b0-446c-a764-7924b761d613", - "status": "downloading", - "created_time": "2026-02-27T18:00:00.000000", - "service": "EXAMPLE1", - "title_id": "abc123def456", - "title": "Example Show", - "progress": 74.9, - "phase": "downloading video 1080p SDR", - "current_title": "S01E04", - "completed_tracks": 4, - "total_tracks": 13, - "active_tracks": ["video 1080p SDR"], - "track_progress": [ - { "label": "video 1080p SDR", "progress": 74.9, "speed": "DASH 265.5 MB/s" } - ], - "segments_done": 75.0, - "segments_total": 100.0, - "speed": "DASH 265.5 MB/s", - "parameters": { }, - "started_time": "2026-02-27T18:00:01.000000", - "completed_time": null, - "output_files": [ - "/downloads/Example.Show.S01E01.mkv", - "/downloads/Example.Show.S01E02.mkv", - "/downloads/Example.Show.S01E03.mkv" - ], - "error_message": null, - "error_details": null -} -``` - -**Progress fields:** - -| Field | Type | Description | -| --- | --- | --- | -| `progress` | number | Overall progress for the title currently downloading, 0–100. Resets per episode on multi-episode jobs | -| `phase` | string | Human-readable current step, e.g. `downloading video 1080p SDR`, `muxing` | -| `title` | string | Human-readable show/movie name, available once the download starts | -| `current_title` | string | `SxxEyy` code of the episode downloading now (episodic jobs only) | -| `completed_tracks` / `total_tracks` | int | Track counts for the current title | -| `active_tracks` | array[string] | Labels of tracks downloading now | -| `track_progress` | array | Per-track `{label, progress, speed}` for each concurrently active track | -| `segments_done` / `segments_total` | number | Segment counts of the track downloading now | -| `speed` | string | Transfer speed of the track downloading now | -| `output_files` | array[string] | Final file paths, appended as each title finishes muxing | - -The list endpoint (`GET /api/download/jobs`) returns the same progress fields per job but omits `parameters`, timestamps, `output_files`, and error details. - ---- - -## DELETE /api/download/jobs/{job_id} - -Cancel a queued or running download job, or remove a finished one. - -- `queued` / `downloading` job: cancelled, returns `200` with `{"status": "success", "message": "Job cancelled"}`. -- `completed` / `failed` / `cancelled` job: removed from the manager entirely, returns `204` (no body). -- Unknown job: `404`. - ---- - -## POST /api/download/jobs/clear-finished - -Remove all `completed`, `failed`, and `cancelled` jobs from the manager. Disabled in `--remote-only` mode. - -```json -{ "removed": 3 } -``` - ---- - -## POST /api/download/jobs/{job_id}/retry - -Enqueue a new job reusing a finished job's service, title, and original (unredacted) parameters. Only allowed on `completed`, `failed`, or `cancelled` jobs — returns `409` otherwise, `404` if unknown. The original job is left in place. - -Creation-time authorization is re-applied at retry time: the job's service must pass the caller's service allowlist (otherwise `400` invalid service), and jobs whose original parameters carried a `cdm` override or per-job credentials return `403` if `serve.cdm_overrides` / `serve.allow_job_credentials` no longer permit them. - -Returns `202` with the same shape as `POST /api/download`: - -```json -{ - "job_id": "b7e9c1f2-...", - "status": "queued", - "created_time": "2026-02-27T18:00:00.000000" -} -``` - ---- - -## POST /api/download/jobs/{job_id}/priority - -Move a `queued` job to the front of the download queue so a worker picks it up next. Returns `409` if the job is not queued, `404` if unknown. - -```json -{ "job_id": "504db959-...", "position": "front" } -``` - ---- - -## GET /api/profiles - -List named credential profiles per service, usable as the `profile` parameter on search/list/download. Only services whose credentials are configured as a `profile-name: credential` mapping appear (a `default` key is included if present); a service with a single plain (unnamed) credential is omitted entirely. Filtered by the caller's service allowlist. Disabled in `--remote-only` mode. - -```bash -curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/profiles -``` - -```json -{ "profiles": { "EXAMPLE1": ["default", "uk", "us"], "EXAMPLE2": ["alice"] } } -``` - ---- - -## GET /api/config - -Read-only, redacted view of the effective server configuration for a UI settings page. Secrets (`api_secret`, `users`, credentials, tokens) are never included; secret-looking keys inside `dl` are masked with `***`. Disabled in `--remote-only` mode. - -```bash -curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/config -``` - -```json -{ - "config": { - "dl": { "lang": "orig", "sub_format": "srt", "downloads": 12 }, - "serve": { - "max_concurrent_downloads": 2, - "job_retention_hours": 24, - "history_limit": 100, - "services": null, - "remote_only": false, - "cdm_overrides": null, - "allow_job_credentials": false - }, - "directories": { - "downloads": "/srv/downloads", - "temp": "/srv/temp", - "cache": "/srv/cache" - }, - "services": ["EXAMPLE1", "EXAMPLE2"] - } -} -``` - -| Field | Description | -| --- | --- | -| `dl` | Default `dl` parameters from the config file (redacted) | -| `serve.services` | Global service allowlist, or `null` when all services are allowed | -| `serve.cdm_overrides` | Permitted per-request CDM device names (list), `true`, or `null` | -| `services` | Available service tags, filtered by the caller's allowlist | - ---- - -## GET /api/history - -Persistent download history. Every job that reaches a terminal state (`completed`, `failed`, `cancelled`) is appended as one JSON line to `api_history.jsonl` in the cache directory, surviving restarts and job cleanup. Only the newest `serve.history_limit` entries are retained (default `100`, `0` for unlimited; see Advanced Config). Newest first. Corrupt lines are skipped; a missing file yields an empty list. Entries are filtered by the caller's service allowlist. Disabled in `--remote-only` mode. - -**Query parameters:** -| Parameter | Type | Default | Description | -| --- | --- | --- | --- | -| `limit` | int | `100` | Maximum entries returned (must be >= 1) | -| `service` | string | all | Filter by service tag (case-insensitive) | - -```bash -curl -H "X-Secret-Key: $KEY" "http://localhost:8786/api/history?limit=50&service=EXAMPLE1" -``` - -```json -{ - "history": [ - { - "job_id": "504db959-80b0-446c-a764-7924b761d613", - "service": "EXAMPLE1", - "title_id": "abc123def456", - "title": "Example Show", - "status": "completed", - "created_time": "2026-02-27T18:00:00.000000", - "completed_time": "2026-02-27T18:12:34.000000", - "output_files": ["/downloads/Example.Show.S01E01.mkv"], - "parameters": { "wanted": ["S01E01"], "quality": [1080], "skip_dl": false }, - "error_message": null - } - ], - "count": 1 -} -``` - -`error_message` is redacted with the same secret-scrubbing applied to job responses. `parameters` is the redacted download selection the job ran with (quality/vcodec/range/wanted/skip_dl, etc.). `completed_time` is `null` for jobs cancelled while still queued. - ---- - -## DELETE /api/history/{job_id} - -Remove a single history entry by `job_id` (the file is rewritten without it). Returns `204` on success, `404` if no matching entry exists or it falls outside the caller's service allowlist. Corrupt lines in the history file are preserved. - -```bash -curl -X DELETE -H "X-Secret-Key: $KEY" "http://localhost:8786/api/history/504db959-80b0-446c-a764-7924b761d613" -``` - ---- - -## POST /api/maintenance/clear-cache - -Delete the contents of the cache directory (the directory is recreated empty). `freed_bytes` is best effort. This also removes `api_history.jsonl`, so persistent download history is cleared. Returns `409` with the active job ids if any job is currently downloading, since in-flight jobs may be reading the cache. Disabled in `--remote-only` mode. - -```bash -curl -X POST -H "X-Secret-Key: $KEY" http://localhost:8786/api/maintenance/clear-cache -``` - -```json -{ "cleared": true, "freed_bytes": 1234567 } -``` - ---- - -## POST /api/maintenance/clear-temp - -Same as clear-cache, for the temp directory. Also refuses with `409` while any job is downloading — active workers write segment files under temp. - -```json -{ "cleared": true, "freed_bytes": 0 } -``` - ---- - -## POST /api/maintenance/refresh-services - -Force-sync (git pull, hard reset to upstream) every service repo configured in `directories.services`. `refreshed` is `true` when all repos synced (or none are configured). Disabled in `--remote-only` mode. - -```bash -curl -X POST -H "X-Secret-Key: $KEY" http://localhost:8786/api/maintenance/refresh-services -``` - -```json -{ - "refreshed": true, - "repos": [ - { "spec": "https://github.com/example/services", "updated": true, "changes": ["~ EXAMPLE1 (modified)"] } - ] -} -``` - -An empty `changes` array means the repo was already up to date. A repo that failed to sync has `updated: false` (and flips `refreshed` to `false`). - ---- - -## GET /api/env/check - -Install status of the binaries `unshackle env check` inspects (FFmpeg, MKVToolNix, shaka-packager, etc.). `version` is a best-effort probe and may be `null` even when installed. Disabled in `--remote-only` mode. - -```bash -curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/env/check -``` - -```json -{ - "checks": [ - { "name": "FFmpeg", "installed": true, "version": "7.1.0", "required": true }, - { "name": "mp4decrypt", "installed": false, "version": null, "required": false } - ] -} -``` diff --git a/docs/api/error-responses.md b/docs/api/error-responses.md deleted file mode 100644 index f2b49e33..00000000 --- a/docs/api/error-responses.md +++ /dev/null @@ -1,28 +0,0 @@ -# Error Responses - -All endpoints return consistent error responses: - -```json -{ - "status": "error", - "error_code": "INVALID_PARAMETERS", - "message": "Invalid vcodec: XYZ. Must be one of: H264, H265, VP9, AV1, VC1, VP8", - "timestamp": "2026-02-27T18:00:00.000000+00:00", - "details": { } -} -``` - -Common error codes: - -- `INVALID_INPUT` -- malformed request body -- `INVALID_PARAMETERS` -- invalid parameter values -- `MISSING_SERVICE` -- service tag not provided -- `INVALID_SERVICE` -- service not found or not in the caller's allowlist -- `SERVICE_ERROR` -- service initialization or runtime error -- `AUTH_FAILED` -- authentication failure -- `NOT_FOUND` / `TRACK_NOT_FOUND` / session not found -- job/session/track/title missing -- `INTERNAL_ERROR` -- unexpected server error - -When `--debug-api` is enabled, error responses include additional `debug_info` with tracebacks and stderr output. - -Authentication errors from the auth middleware are returned as `{"status": 401, "message": "..."}` (not the standard error envelope). diff --git a/docs/api/quick-start.md b/docs/api/quick-start.md deleted file mode 100644 index 2a4f3679..00000000 --- a/docs/api/quick-start.md +++ /dev/null @@ -1,37 +0,0 @@ -# Quick Start - -```bash -# Start the server (no authentication) -unshackle serve --no-key - -# Start with authentication (api_secret in unshackle.yaml) -unshackle serve - -# Serve only the REST API (no pywidevine/pyplayready CDM) -unshackle serve --api-only - -# Serve only the remote-dl session endpoints (CORS/Cloudflare friendly) -unshackle serve --remote-only - -# Disable just one CDM -unshackle serve --no-widevine -unshackle serve --no-playready - -# Verbose error responses (tracebacks/stderr in JSON) -unshackle serve --debug-api -``` - -`serve` flags: - -| Flag | Description | -| --- | --- | -| `-h, --host` | Bind host (default `127.0.0.1`) | -| `-p, --port` | Bind port (default `8786`) | -| `--caddy` | Also launch Caddy using `Caddyfile` next to the unshackle config | -| `--api-only` | REST API only; skip the bundled pywidevine/pyplayready CDM endpoints | -| `--no-widevine` | Disable Widevine CDM endpoints | -| `--no-playready` | Disable PlayReady CDM endpoints | -| `--no-key` | Disable API key authentication entirely | -| `--debug-api` | Include tracebacks/stderr in error responses | -| `--debug` | Enable DEBUG-level logging for API operations | -| `--remote-only` | Expose only `/api/health`, `/api/services`, `/api/search`, and `/api/session/*` (implies `--api-only`) | diff --git a/docs/api/remote-service-sessions.md b/docs/api/remote-service-sessions.md deleted file mode 100644 index 143ff5ba..00000000 --- a/docs/api/remote-service-sessions.md +++ /dev/null @@ -1,329 +0,0 @@ -# Remote Service Sessions - -These endpoints back the `RemoteService` adapter in `unshackle/core/remote_service.py`. They let a thin `dl` client (or any consumer) authenticate against a service on the server, fetch titles/tracks/manifests, and either proxy CDM challenges or have the server resolve KID:KEY directly. The `dl` command's `RemoteService` adapter replaces the old `remote_dl` command. These endpoints are the only `/api/*` routes available in `--remote-only` mode (in addition to `health`, `services`, and `search`). - -## POST /api/session/create - -Authenticate against a service and open a session. Body fields: - -| Field | Type | Description | -| --- | --- | --- | -| `service` | string | Service tag (required) | -| `title_id` | string | Title ID/URL (required) | -| `credentials` | object | Auth credentials forwarded to `Service.authenticate` | -| `cookies` | string | Cookie blob (Netscape or JSON) | -| `proxy` | string | Proxy URI or country code | -| `no_proxy` | bool | Force-disable proxies | -| `profile` | string | Profile name | -| `cache` | object | Optional pre-warmed title cache payload | - -If the service requires interactive input during authentication, poll `GET /api/session/{id}/prompt` and submit responses via `POST /api/session/{id}/prompt` until status is `authenticated`. - -**Request:** - -```json -{ - "service": "EXAMPLE1", - "title_id": "abc123def456", - "credentials": {"username": "alice", "password": "hunter2"}, - "cookies": "# Netscape HTTP Cookie File\n...", - "proxy": "us", - "no_proxy": false, - "profile": "default", - "cache": {} -} -``` - -**Response (202-style; auth runs asynchronously):** - -```json -{ - "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", - "service": "EXAMPLE1", - "status": "authenticating" -} -``` - -## GET /api/session/{session_id} - -Returns session metadata. 404 if expired or unknown. - -```json -{ - "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", - "service": "EXAMPLE1", - "valid": true, - "expires_in": 3600, - "track_count": 0, - "title_count": 0 -} -``` - -## DELETE /api/session/{session_id} - -Tears down the session, cancels any pending prompts, and returns any updated per-session cache files (base64-encoded, zlib-compressed) so the client can re-warm next time. - -```json -{ - "status": "ok", - "cache": { - "tokens": "eJzLSM3JyVcozy/KSVGo5AIAGgQEvQ==" - } -} -``` - -## GET /api/session/{session_id}/titles - -Returns the resolved titles list. - -```json -{ - "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", - "titles": [ - { - "type": "episode", - "name": "Pilot", - "series_title": "Example Show", - "season": 1, - "number": 1, - "year": 2024, - "id": "ep-0001", - "language": "en" - }, - { - "type": "movie", - "name": "Example Movie", - "year": 2024, - "id": "mov-0001", - "language": "en" - } - ] -} -``` - -## POST /api/session/{session_id}/tracks - -**Request:** - -```json -{"title_id": "ep-0001"} -``` - -**Response:** - -```json -{ - "title": { - "type": "episode", - "name": "Pilot", - "series_title": "Example Show", - "season": 1, - "number": 1, - "year": 2024, - "id": "ep-0001", - "language": "en" - }, - "video": [ - { - "id": "v-1080p-h264", - "codec": "H264", - "codec_display": "H.264", - "bitrate": 6000, - "width": 1920, - "height": 1080, - "resolution": "1920x1080", - "fps": "23.976", - "range": "SDR", - "range_display": "SDR", - "language": "en", - "drm": [ - { - "type": "widevine", - "pssh": "AAAAW3Bzc2gAAAAA7e+...", - "kids": ["abcdef0123456789abcdef0123456789"], - "license_url": "https://license.example.com/widevine" - } - ], - "descriptor": "DASH", - "url": "https://cdn.example.com/manifest.mpd" - } - ], - "audio": [ - { - "id": "a-en-eac3", - "codec": "EC3", - "codec_display": "Dolby Digital Plus", - "bitrate": 640, - "channels": "5.1", - "language": "en", - "atmos": false, - "descriptive": false, - "drm": null, - "descriptor": "DASH", - "url": "https://cdn.example.com/manifest.mpd" - } - ], - "subtitles": [ - { - "id": "s-en-vtt", - "codec": "WebVTT", - "language": "en", - "forced": false, - "sdh": false, - "cc": false, - "descriptor": "DASH", - "url": "https://cdn.example.com/subs/en.vtt" - } - ], - "chapters": [ - {"timestamp": "00:00:00.000", "name": "Chapter 1"} - ], - "attachments": [], - "manifests": [ - { - "type": "dash", - "url": "https://cdn.example.com/manifest.mpd", - "data": "eJzNVk1v2zAM/Ss..." - } - ], - "session_headers": { - "User-Agent": "Mozilla/5.0 ..." - }, - "session_cookies": { - "session": "abc123" - }, - "server_cdm_type": "widevine" -} -``` - -## POST /api/session/{session_id}/segments - -**Request:** - -```json -{"track_ids": ["v-1080p-h264", "a-en-eac3"]} -``` - -**Response:** - -```json -{ - "tracks": { - "v-1080p-h264": { - "descriptor": "DASH", - "url": "https://cdn.example.com/manifest.mpd", - "drm": [ - { - "type": "widevine", - "pssh": "AAAAW3Bzc2gAAAAA7e+...", - "kids": ["abcdef0123456789abcdef0123456789"], - "license_url": "https://license.example.com/widevine" - } - ], - "headers": {"User-Agent": "Mozilla/5.0 ..."}, - "cookies": {"session": "abc123"}, - "data": {} - }, - "a-en-eac3": { - "descriptor": "DASH", - "url": "https://cdn.example.com/manifest.mpd", - "drm": null, - "headers": {"User-Agent": "Mozilla/5.0 ..."}, - "cookies": {"session": "abc123"}, - "data": {} - } - } -} -``` - -## POST /api/session/{session_id}/license - -Two modes, selected by the `mode` field. - -**`mode: "proxy"` (default)** -- forward a client-built CDM challenge to the service's license endpoint. - -Request: - -```json -{ - "mode": "proxy", - "track_id": "v-1080p-h264", - "challenge": "CAESxQEK...", - "drm_type": "widevine", - "pssh": "AAAAW3Bzc2gAAAAA7e+..." -} -``` - -Response: - -```json -{"license": "CAIS3wIK..."} -``` - -**`mode: "server_cdm"`** -- the server uses its own CDM to license the track and extract keys. Single-track form takes `track_id`; batch form takes `track_ids`. Requires the calling user key to have a matching device (`devices` for Widevine, `playready_devices` for PlayReady) in `unshackle.yaml`. - -Request (batch): - -```json -{ - "mode": "server_cdm", - "track_ids": ["v-1080p-h264", "a-en-eac3"], - "drm_type": "widevine" -} -``` - -Response: - -```json -{ - "keys": { - "v-1080p-h264": { - "abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff" - }, - "a-en-eac3": { - "abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff" - } - }, - "drm_type": "widevine" -} -``` - -## GET /api/session/{session_id}/prompt - -Polled by the client during interactive authentication (OTP, PIN, device codes). Backed by the `InputBridge` in `unshackle/core/api/input_bridge.py`; `Service.request_input()` blocks server-side until the client posts a response. - -Pending input: - -```json -{"status": "pending_input", "prompt": "Enter OTP code: "} -``` - -Other states: - -```json -{"status": "authenticating"} -``` - -```json -{"status": "authenticated"} -``` - -```json -{"status": "failed", "error": "Invalid credentials"} -``` - -## POST /api/session/{session_id}/prompt - -Unblocks the server-side `request_input()` call. - -Request: - -```json -{"response": "123456"} -``` - -Response: - -```json -{"status": "accepted"} -``` diff --git a/docs/assets/favicon.svg b/docs/assets/favicon.svg new file mode 100644 index 00000000..b2834eab --- /dev/null +++ b/docs/assets/favicon.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" height="24" viewBox="0 -960 960 960" width="24" fill="#26a69a"><path d="m800-274-80-80v-206H514l-80-80h166v-80q0-50-34.5-85T481-840q-50 0-84 34.5T363-720v9l-73-73q22-61 75-98.5T481-920q83 0 141 58.5T680-720v80h40q33 0 56.5 23.5T800-560v286Zm20 246-62-62q-11 5-20 7.5T720-80H240q-33 0-56.5-23.5T160-160v-400q0-25 14.5-46t37.5-30L28-820l56-56L876-84l-56 56ZM686-160 539-309q-11 11-25.5 17t-31.5 6q-33 0-56.5-23.5T402-366q0-17 6-31.5t17-25.5L286-560h-46v400h446ZM486-360Zm131-97Z"/></svg> diff --git a/docs/dev/creating-a-service.md b/docs/dev/creating-a-service.md new file mode 100644 index 00000000..23472d24 --- /dev/null +++ b/docs/dev/creating-a-service.md @@ -0,0 +1,634 @@ +# Creating a Service + +A **service** is a plugin that teaches unshackle how to talk to one streaming +provider: how to log in, how to look up a title, what video/audio/subtitle +tracks exist, and how to license any DRM. Everything else (track selection, +downloading, decryption, muxing, naming) is handled by the unshackle core. + +!!! info "Who this page is for" + This is a **developer guide**. If you only want to *use* services, see + [downloading](../guide/downloading.md) and + [configuration](../reference/configuration.md). Nothing here is required to run + the CLI. Service code is deliberately kept separate from the core so it can + live in its own private repository. + +Unshackle ships one reference service, **EXAMPLE**, at +`unshackle/services/EXAMPLE/`. It is a deliberately exhaustive, non-runnable +showcase of every framework feature in a single file. Read it alongside this +page; most snippets below are drawn from it. + +--- + +## How a service is discovered + +At startup unshackle scans every path in the `directories.services` config key +(a **list**). Each entry is either a local directory of services or a remote +repo spec (a git URL or `owner/repo` shorthand, which is cloned automatically). +The default is the bundled `unshackle/services` directory. + +Within a services directory, **every subfolder that contains an `__init__.py` +is a service**, and the folder name is the service **tag**: + +```text +services/ +└── EXAMPLE/ + ├── __init__.py # required — defines the service class + └── config.yaml # optional — per-service configuration +``` + +Two rules the loader enforces: + +1. **The class name must exactly match the folder/tag name.** Folder `AMZN` must + define `class AMZN`. A mismatch raises a `RuntimeError`. +2. **List order is priority.** The first services path to define a given tag + wins; later duplicates are shadowed. Put your own local overrides *last* to + use them as fallbacks, or *first* to override a repo. + +Once installed, a service is invoked as a subcommand of `dl`: + +```bash +unshackle dl EXAMPLE 20914 +unshackle dl EX 20914 # via an ALIAS +``` + +!!! tip "Config lives next to the code" + A service's own settings go in `config.yaml` **inside the service folder** + (`directories.services/<TAG>/config.yaml`). This is separate from the global + `unshackle.yaml`. Access it in code as `self.config[...]`. Never hardcode + URLs, user agents, or certificates; put them here. + +!!! warning "`config.yaml` is shared: keep secrets out of it" + Treat `config.yaml` as a checked-in file of **defaults**: it belongs in the + service repo and is shared by everyone who runs the service. Per-user secrets + (API keys, device IDs, account tokens) must **not** live here. Those go in the + user's own `unshackle.yaml` under `services.<TAG>`, which is **merged into + `self.config` at runtime**. That checked-in-defaults / per-user-overrides + split is exactly why every URL lives in `config.yaml` with `{}`/`{name}` + placeholders while the secrets that fill them stay out of it. Both halves + read back through the same `self.config[...]`. + +--- + +## The Service base class + +Every service subclasses `unshackle.core.service.Service`. The minimum is a +Click `cli` entry point plus three abstract methods. + +### Class variables + +Declared at class level to configure framework behaviour: + +| Variable | Type | Purpose | +|---|---|---| +| `ALIASES` | `tuple[str, ...]` | Extra tags that resolve to this service (e.g. `("EX", "DOMAIN")`). Default `()`. | +| `GEOFENCE` | `tuple[str, ...]` | IP region codes the service requires. Empty = no geofence. The **first** entry is treated as the main region for auto-proxy. | +| `VAULT_TAG` | `Optional[str]` | Overrides the key-vault namespace so sibling services can share one vault. Default `None` (use the service's own tag). | +| `AUTH_METHODS` | `Optional[tuple[str, ...]]` | Auth methods accepted (`"cookies"` / `"credentials"`). When `None`, the REST `/services` endpoint infers them from `authenticate()`. | +| `NO_SUBTITLES` | `bool` | Set `True` on a service with no subtitle tracks to skip subtitle handling entirely. | + +!!! note "`NO_SUBTITLES` is a convention, not a base-class attribute" + `NO_SUBTITLES` is checked by `dl.py` via `hasattr` and is deliberately **not** + declared on the `Service` base class, so it only exists if your service + defines it. Setting `NO_SUBTITLES = True` is what makes the pipeline skip + subtitle work for services that never carry subtitles; leave it off otherwise. + +`GEOFENCE` drives an automatic proxy: on startup, if you did not pass an +explicit `--proxy`, the base class does a live IP check and, if your region is +blocked, fetches a proxy to `GEOFENCE[0]` from your configured proxy providers. + +```python +class EXAMPLE(Service): + ALIASES = ("EX", "DOMAIN") + GEOFENCE = ("US", "UK") + VAULT_TAG = "DIFFERENT_NAME" +``` + +### What `Service.__init__` wires up + +When you call `super().__init__(ctx)`, the base class sets up the attributes you +will use throughout the service: + +| Attribute | What it is | +|---|---| +| `self.config` | This service's `config.yaml` contents (a dict), or `None` if absent. | +| `self.log` | A `logging.Logger` named after your class. | +| `self.session` | A prepared `requests.Session` with config headers and a retry adapter (5 retries, backoff 0.2, retries on 429/500/502/503/504). | +| `self.cache` | A `Cacher` for arbitrary key/value data (tokens, etc.). | +| `self.title_cache` | A region/account-aware `TitleCacher` used by `get_titles_cached`. | +| `self.cache_dir` | `config.directories.cache / <ClassName>`. | +| `self.track_request` | A `TrackRequest` built from the CLI `--vcodec` / `--range` / `--best-available` flags. | +| `self.credential` | `None` until `authenticate()` runs. | +| `self.current_region` | The two-letter region resolved from proxy/IP. | + +`self.track_request` is a small dataclass you can read *and rewrite* before +fetching tracks: + +```python +@dataclass +class TrackRequest: + codecs: list[Video.Codec] = [] # empty == accept any codec + ranges: list[Video.Range] = [Video.Range.SDR] + best_available: bool = False +``` + +!!! note "Read the request, don't second-guess the user" + Services may narrow `track_request` for hard technical constraints (e.g. + *"HDR on this service is only delivered as HEVC"*), but must **not** filter + the tracks they return by resolution, bitrate, or language. All selection is + done by the core after `get_tracks` returns. + +!!! tip "Override `get_session()` to defeat TLS-fingerprint bot detection" + The base `get_session()` returns a plain `requests.Session` (config headers + + retry adapter) and **deliberately does not** do TLS impersonation. When a + provider blocks you on TLS fingerprint alone, override it to return + `session("Chrome131")` (any `rnet.Impersonate` preset). `RnetSession` is a + drop-in `requests.Session` replacement (same `.get`/`.post`/cookie-jar + interface), so nothing else in the service changes. If instead you need + specific SSL *cipher* behaviour, mount + `unshackle.core.utils.sslciphers.SSLCiphers` onto a plain `requests.Session`. + +--- + +## The methods you implement + +Methods run in the order shown. Only the three abstract ones are mandatory. + +```mermaid +graph LR + A[cli] --> B[__init__] --> C[authenticate] --> D[get_titles] --> E[get_tracks] --> F[get_chapters] +``` + +### `cli`: the Click entry point + +A static method decorated as a Click command. It defines the CLI arguments and +returns an **instance** of your service. Keep the command `name=` matching the +tag. + +!!! note "`name=` is cosmetic: the directory name is what resolves the command" + `unshackle dl EXAMPLE` is dispatched by the **folder/class name**, not by the + `name=` in `@click.command`. A mismatched `name=` still loads and runs; it + just produces confusing `--help`/usage output that names the wrong command. + That confusing output is the reason to keep it in sync, not any dispatch + requirement. + +```python +@staticmethod +@click.command(name="EXAMPLE", short_help="https://domain.com", help=__doc__) +@click.argument("title", type=str) +@click.option("-m", "--movie", is_flag=True, default=False, help="Treat the title as a movie.") +@click.option("-d", "--device", type=click.Choice(["android_tv", "web", "ios"]), default="android_tv") +@click.pass_context +def cli(ctx: click.Context, **kwargs: Any) -> EXAMPLE: + return EXAMPLE(ctx, **kwargs) + +def __init__(self, ctx: click.Context, title: str, movie: bool, device: str): + self.title = title + self.movie = movie + self.device = device + super().__init__(ctx) # wires up config, log, session, caches, track_request... + self.cdm = ctx.obj.cdm +``` + +Your docstring becomes the CLI `--help` text (`help=__doc__`), so document the +accepted URL/ID formats and options there. + +!!! tip "CDM-aware behaviour" + The resolved CDM is on `ctx.obj.cdm` (may be `None` for DRM-free runs). Use + `is_widevine_cdm()` / `is_playready_cdm()` from `unshackle.core.cdm.detect` + to classify it. These correctly handle local *and* remote CDMs, so never + hand-roll an `isinstance` check. Services often pick a device profile or + manifest endpoint based on which DRM the CDM speaks. + +### `authenticate`: optional login + +Override to log in with cookies and/or credentials. **Call `super().authenticate()` +first**: the base implementation loads the cookie jar into `self.session.cookies` +and stores `self.credential`. Do all token fetching here; it runs before +`get_titles`. + +```python +def authenticate(self, cookies=None, credential=None) -> None: + super().authenticate(cookies, credential) + self.session.headers.update({"user-agent": self.config["client"][self.device]["user_agent"]}) + + cache = self.cache.get(f"tokens_{self.device}_{self.profile}") + if cache and cache.data.get("expires_in", 0) > int(datetime.now().timestamp()): + self.log.info(" + Using cached tokens") + else: + token = self.session.post(self.config["endpoints"]["login"], data=body).json() + cache.set(data=token, expiration=token.get("expires_in")) + self.token = cache.data["token"] +``` + +Cookies come from files under `directories.cookies`; credentials from the +`credentials` map in `unshackle.yaml`. For interactive prompts (OTP, captcha) +use `self.request_input(prompt)`, never a bare `input()`. Under `serve` mode +there is no local terminal, so a bare `input()` would hang the server waiting on +stdin that never arrives; `request_input` instead relays the prompt to the +remote client through the attached `InputBridge`. Locally it routes through the +shared Rich console (`console.input`) so the prompt renders correctly alongside +progress and log output. + +!!! warning "Key token caches by whatever varies per session" + The cache key above is `tokens_{device}_{profile}` on purpose. A token cache + must include every dimension that changes the token (device, profile, or + `credential.sha1`), or two profiles/devices will share one cache entry and + stomp each other's tokens. This isn't obvious from the caching API, which + happily lets you use a single flat key. + +!!! warning "Handle the credential carefully" + Use the `Credential` only inside `authenticate()` to obtain tokens; don't + stash it or the raw cookie jar elsewhere. The base class already caches + identity for you. + +### `get_titles`: required + +Return a titles collection for the given ID. The return type depends on content: + +| Return | Contains | Use for | +|---|---|---| +| `Movies` | `Movie` objects | Films | +| `Series` | `Episode` objects | Shows (flatten seasons into episodes) | +| `Album` | `Song` objects | Music | + +Every title carries a `language` (its **original recorded language**) and an +arbitrary `data` dict you can use to stash metadata for later methods. At least +one title must be returned, or the ID is treated as invalid. + +```python +def get_titles(self) -> Titles_T: + match = re.match(self.TITLE_RE, self.title) + if not match: + raise ValueError("Could not parse a title ID - is the URL/ID correct?") + metadata = self.session.get( + self.config["endpoints"]["metadata"].format(title_id=match.group("title_id")), + params={"token": self.token}, + ).json() + original_lang = Language.find(metadata["languages"][0]) + + return Movies([ + Movie( + id_=metadata["id"], + service=self.__class__, + name=metadata["title"], + year=metadata.get("releaseYear") or None, + language=original_lang, # ORIGINAL audio language, not the -l preference + data=metadata, # read later as title.data + ) + ]) +``` + +Common constructor fields: + +=== "Movie" + + ```python + Movie(id_, service, name, year=None, language=None, data=None, description=None) + ``` + +=== "Episode" + + ```python + Episode(id_, service, title, season, number, name=None, year=None, + language=None, data=None, description=None, air_date=None) + ``` + + Pass `air_date` for daily/sports content to name by date instead of + `SxxExx`. Return a `Series([...])` of episodes. + +=== "Song" + + ```python + Song(id_, service, name, artist, album, track, disc, year, + language=None, data=None) + ``` + + Return an `Album([...])` of songs. + +!!! note "The ID must be unique and stable" + `id_` must be truthy and, if it has a length, at least 4 characters, to + avoid clashes. Use the service's own stable IDs. + +!!! tip "Title caching" + You can call `self.get_titles_cached()` instead of `get_titles()` from the + framework path to reuse cached results and gracefully fall back to stale data + when the API errors. It also applies the per-service `title_map` config. + +### `get_tracks`: required + +Given one title, return a `Tracks` object holding `Video`, `Audio`, and +`Subtitle` tracks. In almost all cases you build these by parsing a manifest, +not by hand. + +The manifest parsers live in `unshackle.core.manifests`: + +```python +from unshackle.core.manifests import DASH, HLS, ISM +``` + +Each exposes `from_url(url, session=...)` (or `from_text(text, url)`) and then +`.to_tracks(language=...)`: + +=== "DASH (.mpd)" + + ```python + def get_tracks(self, title: Title_T) -> Tracks: + return DASH.from_url(manifest_url, session=self.session) \ + .to_tracks(language=title.language) + ``` + +=== "HLS (.m3u8)" + + ```python + def get_tracks(self, title: Title_T) -> Tracks: + return HLS.from_url(manifest_url, session=self.session) \ + .to_tracks(language=title.language) + ``` + + The URL must be a **variant (master) playlist**, not a media playlist. + +=== "ISM (Smooth Streaming)" + + ```python + def get_tracks(self, title: Title_T) -> Tracks: + return ISM.from_url(manifest_url, session=self.session) \ + .to_tracks(language=title.language) + ``` + +!!! warning "Always pass `language=`" + Pass the title's original language to `to_tracks(language=...)`. For DASH and + HLS this is **required as a fallback**. If a track's language can't be + derived and no valid fallback was given, `to_tracks` raises `ValueError`. + It's also what lets the parser flag `is_original_lang` on each track (via + `is_close_match`), driving `-l best/all` selection and the filename language + token. + +Some services deliver a **separate manifest per codec/range**. The base class +provides `_get_tracks_for_variants(title, fetch_fn)` to fan out over every +codec×range in the `TrackRequest`, including `HYBRID` (fetch HDR10 + DV and +merge) and `--best-available` skip-on-error handling: + +```python +def get_tracks(self, title: Title_T) -> Tracks: + def _fetch_variant(title, codec, range_) -> Tracks: + vcodec = "H265" if codec == Video.Codec.HEVC else "H264" + return self._fetch_dash_manifest(title, vcodec=vcodec, range_=range_) + return self._get_tracks_for_variants(title, _fetch_variant) +``` + +After parsing, you may **correct** track metadata the manifest gets wrong: +stamp the real `video.range`, fix odd audio channel counts, mark descriptive +audio, add hand-built subtitles or an `Attachment` (e.g. cover art). Do *not* +filter for resolution/bitrate. See `_fetch_dash_manifest` in the EXAMPLE service +for a thorough demonstration. + +!!! note "Flip HDR10 → HDR10+ by hand when you know the platform embeds it" + HDR10+ is a **bitstream (SEI)** feature. The HLS parser intentionally does + not sniff it from the stream, so a manifest that embeds HDR10+ SEI but labels + the variant plain HDR10 will parse as HDR10. The label is the only signal + the parser has, and it's wrong. Only the service author knows the platform + embeds HDR10+, so it's on you to correct `video.range` to HDR10+ here for + those services. + +!!! warning "`get_tracks` is your only chance to capture license inputs" + The license callbacks (`get_widevine_license`, etc.) fire **much later**, in + the download/decrypt step, long after `get_tracks` has returned, and they + receive **only** `challenge`, `title`, and `track`. Nothing else from the + manifest response reaches them. So anything a license call needs (the license + URL, `dt-custom-data`, a session token) has to be stashed **now**, during + `get_tracks`, onto `self.license_data` or `title.data`. This is a + lifecycle-timing consequence you can't infer from the callback signatures. + +!!! tip "Provide the KID if you cheaply can" + If you can obtain a track's Key ID (32-char hex) without downloading stream + data, set `track.kid`; it speeds up the decryption/key-lookup path. And be + sure encrypted tracks carry the correct `drm` so the core licenses them. + +### `get_chapters`: required + +Return a `Chapters` object (0 or more `Chapter`s). You don't need to number or +sort them; `Chapters` does that automatically and inserts a `00:00:00.000` +marker if missing. A `Chapter` timestamp accepts `"HH:MM:SS[.mmm]"`, an int in +milliseconds, or a float in seconds. + +```python +def get_chapters(self, title: Title_T) -> Chapters: + chapters = Chapters() + seen: set[int] = set() + for ch in title.data.get("chapters", []): + chapter = Chapter(timestamp=ch["start"], name=ch.get("name")) + if chapter.timestamp == 0 or chapter.timestamp in seen: + continue # see the warning below + seen.add(chapter.timestamp) + chapters.add(chapter) + return chapters +``` + +!!! warning "Skip markers at 0 and de-duplicate timestamps before `add()`" + `Chapters.add()` auto-injects a `Chapter(0)` at `00:00:00.000` the first time + you add *any* chapter, **and** it raises `ValueError` if a chapter already + exists at an exact timestamp. So an API marker reported at `0` (an intro or + recap that starts at the very beginning) collides with that auto-inserted + opening chapter, and two markers sharing a timestamp collide with each other. + Guard both cases: drop any marker at timestamp `0` and de-duplicate timestamps + before adding, as above. + +!!! note "An unnamed chapter is the idiom for closing a named range" + A `Chapter` with no name is perfectly valid. Inserting an unnamed marker is + the standard way to *close out* a named range: e.g. add `Chapter(name="Intro")` + at the start and an unnamed `Chapter` where the intro ends, so the "Intro" + label applies only to that span. + +!!! warning "Don't invent chapter names" + Never name chapters `"Chapter 1"` yourself. Leave unnamed markers unnamed; + users who want generic names set `chapter_fallback_name` (e.g. + `"Chapter {i:02}"`) in their config. + +--- + +## Handling DRM + +The core drives licensing. Your job is to answer the license challenges the CDM +produces. Mark encrypted tracks with the right DRM in `get_tracks` (the manifest +parsers do this automatically for PSSH/`EXT-X-KEY` found in the manifest), then +implement the license callbacks you need. Each receives the challenge plus the +current `title` and `track`: + +| Method | For | Returns | +|---|---|---| +| `get_widevine_service_certificate` | Widevine privacy mode | The service certificate (bytes or base64 str), or `None`. | +| `get_widevine_license` | Widevine | The raw license response (bytes or base64 str), **unmodified**. | +| `get_playready_license` | PlayReady | The license response. Defaults to delegating to `get_widevine_license`. | +| `get_clearkey_license` | DASH `org.w3.clearkey` | The JWK Set (dict/JSON/bytes), or `None` to let the framework POST the challenge to the manifest's Laurl. | + +```python +def get_widevine_license(self, *, challenge: bytes, title, track): + response = self.session.post( + self.license_data["url"], + data=challenge, # POST the challenge as-is + headers={"dt-custom-data": self.license_data["data"]}, + ) + response.raise_for_status() + try: + return response.json()["license"] # some services wrap it in JSON + except (ValueError, KeyError): + return response.content # others return raw bytes +``` + +!!! warning "Return the license untouched" + Do **not** base64-encode or decode the challenge or response. Pass them + through verbatim. A malformed license request can get your CDM/device + flagged, banned, or downgraded. + +!!! tip "`ClearKeyCENC` (`org.w3.clearkey`) has three escalating integration levels" + Pick the lowest level that works for the platform: + + 1. **The manifest carries a `<Laurl>`**: implement nothing. The framework + POSTs the challenge to that URL itself. + 2. **A custom endpoint or extra headers are needed**: override + `get_clearkey_license` to do the POST yourself. + 3. **Keys arrive obfuscated or through a bespoke channel**: fetch and unwrap + the key in `get_tracks` and pre-populate + `drm.content_keys[kid] = key_hex` on the track's `ClearKeyCENC`. The + non-obvious payoff: when every KID is already keyed, the framework skips + the license round-trip entirely. + +Things you don't handle: + +- **HLS AES-128 (`ClearKey`)** and **DRM-free** content have no license + callback. The key comes from the manifest and is applied directly. +- **Key vaults** cache `KID:KEY` pairs across runs, so repeat downloads skip the + license round-trip entirely. +- **Decryption tooling** (shaka-packager or mp4decrypt) and the choice of local + vs remote CDM are user configuration, not service concerns. + +See the [DRM & CDM reference](../guide/drm-and-cdm.md) for the full picture. + +--- + +## Optional event hooks + +Override any of these to react to pipeline stages (all no-ops by default): +`on_segment_downloaded`, `on_track_downloaded`, `on_track_decrypted`, +`on_track_repacked`, `on_track_multiplex`. Also override `search()` to yield +`SearchResult` objects for `unshackle search`. + +```python +def search(self) -> Generator[SearchResult, None, None]: + results = self.session.get(self.config["endpoints"]["search"], + params={"q": self.title}).json() + for r in results["entries"]: + yield SearchResult(id_=r["id"], title=r["title"], + description=r.get("description"), url=r.get("url")) +``` + +--- + +## Minimal service skeleton + +A complete, minimal single-manifest service. Save as +`services/MYSVC/__init__.py` (class name `MYSVC`), with an optional +`services/MYSVC/config.yaml` beside it. + +```python title="services/MYSVC/__init__.py" +from __future__ import annotations + +import re +from typing import Any, Optional + +import click + +from unshackle.core.manifests import DASH +from unshackle.core.service import Service +from unshackle.core.titles import Movie, Movies, Title_T, Titles_T +from unshackle.core.tracks import Chapters, Tracks + + +class MYSVC(Service): + """ + MyService — https://myservice.com + + \b + Usage: unshackle dl MYSVC <title-id-or-url> + """ + + ALIASES = ("MY",) + GEOFENCE = ("US",) + TITLE_RE = r"^(?:https?://myservice\.com/watch/)?(?P<id>[\w-]+)" + + @staticmethod + @click.command(name="MYSVC", short_help="https://myservice.com", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: click.Context, **kwargs: Any) -> "MYSVC": + return MYSVC(ctx, **kwargs) + + def __init__(self, ctx: click.Context, title: str): + self.title = title + super().__init__(ctx) + + def authenticate(self, cookies=None, credential=None) -> None: + super().authenticate(cookies, credential) + # obtain any tokens here, e.g. self.token = ... + + def get_titles(self) -> Titles_T: + match = re.match(self.TITLE_RE, self.title) + if not match: + raise ValueError("Could not parse a title ID.") + meta = self.session.get( + self.config["endpoints"]["metadata"].format(id=match.group("id")) + ).json() + return Movies([ + Movie( + id_=meta["id"], + service=self.__class__, + name=meta["title"], + year=meta.get("year") or None, + language=meta.get("originalLanguage"), + data=meta, + ) + ]) + + def get_tracks(self, title: Title_T) -> Tracks: + playback = self.session.get( + self.config["endpoints"]["playback"].format(id=title.id) + ).json() + return DASH.from_url( + url=playback["manifest_url"], session=self.session + ).to_tracks(language=title.language) + + def get_chapters(self, title: Title_T) -> Chapters: + return Chapters() + + def get_widevine_license(self, *, challenge: bytes, title, track): + res = self.session.post(self.config["endpoints"]["license"], data=challenge) + res.raise_for_status() + return res.content +``` + +```yaml title="services/MYSVC/config.yaml" +endpoints: + metadata: https://api.myservice.com/v1/metadata/{id} + playback: https://api.myservice.com/v1/playback/{id} + license: https://api.myservice.com/v1/license/widevine +``` + +Test it with: + +```bash +unshackle dl MYSVC https://myservice.com/watch/abc123 +``` + +--- + +## Checklist + +- [ ] Folder name, class name, and Click `name=` all match the tag exactly. +- [ ] `cli` is a `@staticmethod` returning an instance; `__init__` calls `super().__init__(ctx)`. +- [ ] `get_titles`, `get_tracks`, `get_chapters` implemented; titles carry the original `language`. +- [ ] Tracks come from a manifest parser with `to_tracks(language=title.language)`; no filtering by quality/language. +- [ ] Encrypted tracks are marked with DRM; the license callbacks you need return responses **unmodified**. +- [ ] URLs, user agents, and certificates live in `config.yaml`, read via `self.config[...]`. + +Read `unshackle/services/EXAMPLE/` end to end; it annotates every feature +touched above in one place. diff --git a/docs/dev/rest-api/authentication.md b/docs/dev/rest-api/authentication.md new file mode 100644 index 00000000..83a678dd --- /dev/null +++ b/docs/dev/rest-api/authentication.md @@ -0,0 +1,311 @@ +# REST API Authentication + +The unshackle REST API protects every endpoint (with one exception) behind a single API key. Clients present that key in a request header, and the server checks it against the keys defined in your configuration. This page explains how that check works, how to configure keys, how clients send them, and the extra controls that build on top of the key: per-key service allowlists, per-key CDM device access, and session IP binding. + +The API is started with the [`unshackle serve`](../../guide/cli-reference.md) command. Authentication is on by default: if you start the server without an API key configured, it refuses to launch. + +!!! note "Which server am I running?" + `unshackle serve` can run in two shapes, and each uses a slightly different piece of code to check your key: + + - The **integrated server** (the default) exposes the REST API *and* the pywidevine / pyplayready CDM HTTP endpoints. + - The **API-only server** (`--api-only`, also implied by `--remote-only`) exposes only the REST API. + + The header, the config, and the behaviour you see as a client are the same in both. The differences are internal and are called out in the developer notes below. + +## The short version + +1. Add an `api_secret` to the `serve` section of your `unshackle.yaml`. +2. Start the server with `unshackle serve`. +3. Send that secret in the `X-Secret-Key` header on every request except `GET /api/health`. + +```bash title="A minimal authenticated request" +curl -H "X-Secret-Key: your-secret-key-here" \ + http://127.0.0.1:8786/api/services +``` + +## Configuring authentication + +All authentication settings live under the `serve` section of your unshackle config file (`unshackle.yaml`). + +### A single API key + +The simplest setup defines one shared secret with `api_secret`: + +```yaml title="unshackle.yaml" +serve: + api_secret: "a-long-random-secret-string" +``` + +This value becomes the one and only key that the server accepts. Requests that present it in the `X-Secret-Key` header are allowed; everything else is rejected. + +!!! warning "The server will not start without a key" + If you start `unshackle serve` with no `api_secret` configured, and you have *not* passed `--no-key`, the command fails immediately with: + + ``` + API secret key is not configured. Please add 'api_secret' to the 'serve' section in your config. + ``` + + Choose a long, unpredictable value. The key is the only thing standing between the internet and your CDM and download endpoints. + +### Multiple keys with `users` + +For finer control you can define named users, each with their own key. A user entry can also grant CDM device access and restrict which services that key may touch: + +```yaml title="unshackle.yaml" +serve: + api_secret: "the-primary-admin-key" + users: + a-secret-key-for-alice: + username: alice + devices: ["my_widevine_device"] # Widevine (.wvd) devices this key may use + playready_devices: ["my_playready_prd"] # PlayReady (.prd) devices this key may use + services: ["AMZN", "NF"] # optional per-key service allowlist + a-secret-key-for-bob: + username: bob + devices: ["my_widevine_device"] +``` + +The key of each entry under `users` (for example `a-secret-key-for-alice`) *is* the secret that client sends in `X-Secret-Key`. The `username` is a human-readable label used in logs and in job/history metadata. + +!!! info "How `api_secret` and `users` combine (integrated server)" + On the integrated server, the configured `api_secret` is automatically added as an extra accepted key alongside everyone in `users`. It is granted access to all loaded Widevine and PlayReady devices and is labelled `api_user`. So `api_secret` behaves as an admin-level key in addition to any named users you define. + + On the API-only server, only `api_secret` is accepted: the single-key model. (See the developer notes for the exact difference.) + +### CDM device access per key + +Under the integrated server, a key can only run DRM licensing through a CDM device that its user entry lists: + +| Config key | Grants access to | +|---|---| +| `devices` | Widevine `.wvd` devices, by name | +| `playready_devices` | PlayReady `.prd` devices, by name | + +!!! warning "PlayReady access is opt-in per user" + If a user entry omits `playready_devices`, that key is given **no** PlayReady access, and the server logs a warning at startup. You must list PlayReady devices explicitly for any key that needs them. Widevine and PlayReady device files themselves are auto-loaded from your WVDs and PRDs directories. + +!!! note "Device lists *are* the server-side-decryption switch" + There is no separate tier, capability flag, or permission toggle for whether the server will hand back keys (`KID:KEY`) for a session download. The presence of devices on the calling key decides it. With **empty** `devices` and `playready_devices`, the server can only proxy CDM challenges, so the client must run its own CDM. Once those lists are **populated**, the client may request `mode:server_cdm` and receive keys back. Don't go looking for a tier setting; configuring devices is what enables server-side decryption. + +### Disabling authentication + +The `--no-key` flag turns authentication off entirely. Every request is allowed, `X-Secret-Key` is ignored, and no key needs to be configured. + +```bash +unshackle serve --no-key +``` + +!!! danger "Only use `--no-key` on a trusted, isolated network" + With `--no-key` there is nothing stopping any client that can reach the port from listing your services, starting downloads, or using your CDM devices. Never expose a `--no-key` server to an untrusted network. + +## How clients present credentials + +Clients authenticate by sending the secret in a single HTTP request header on every call: + +``` +X-Secret-Key: <your-secret> +``` + +The value must exactly match one of the configured keys (`api_secret`, or one of the keys under `users`). There is no login step, no session cookie, and no token exchange for API access. The same static key is sent on each request. + +=== "curl" + + ```bash + curl -H "X-Secret-Key: your-secret-key-here" \ + http://127.0.0.1:8786/api/services + ``` + +=== "Python (requests)" + + ```python + import requests + + BASE = "http://127.0.0.1:8786" + HEADERS = {"X-Secret-Key": "your-secret-key-here"} + + resp = requests.get(f"{BASE}/api/services", headers=HEADERS, timeout=30) + resp.raise_for_status() + print(resp.json()) + ``` + +=== "JavaScript (fetch)" + + ```javascript + const resp = await fetch("http://127.0.0.1:8786/api/services", { + headers: { "X-Secret-Key": "your-secret-key-here" }, + }); + if (!resp.ok) throw new Error(`Server returned ${resp.status}`); + console.log(await resp.json()); + ``` + +!!! tip "The reference client sets it for you" + unshackle's own [remote download](../../guide/downloading.md) client (`RemoteService`) sends `X-Secret-Key` automatically, reading the key from your `remote_services` config. It also sends a `User-Agent` of `unshackle/<version>`. You only deal with the header directly when writing your own client. + +### The health check is always open + +One endpoint is exempt from authentication: `GET /api/health`. It never requires a key, so you can use it for liveness and version probes without embedding a secret in your monitoring: + +```bash title="No key needed" +curl http://127.0.0.1:8786/api/health +``` + +```json title="Response" +{ + "status": "ok", + "version": "5.3.0", + "update_check": { + "update_available": null, + "current_version": "5.3.0", + "latest_version": null + } +} +``` + +Every other endpoint (including the Swagger UI at `/api/docs/` on the API-only server) requires a valid key. + +## Authentication failures + +When a request is rejected for authentication reasons, the response has an HTTP `401 Unauthorized` status and a small JSON body. + +=== "Missing header" + + Sending a request with no `X-Secret-Key` header: + + ```json + { + "status": 401, + "message": "Secret Key is Empty." + } + ``` + +=== "Wrong key" + + Sending a key that is not in your config: + + ```json + { + "status": 401, + "message": "Secret Key is Invalid." + } + ``` + +!!! warning "Auth errors use a different JSON shape than other errors (developer note)" + The rest of the API reports errors with a structured body containing `error_code`, `message`, `timestamp`, and other fields (see [Errors](errors.md)). The authentication middleware is different: it returns `{"status": 401, "message": "..."}`, where `status` is an **integer**, and there is no `error_code` or `timestamp`. + + A robust client should treat any `401` as an authentication problem and be prepared to parse *both* shapes rather than assuming the standard error body is always present. + +## Building on the key: access controls + +Once a request is authenticated, several additional controls decide what that specific key is allowed to do. + +### Per-key service allowlists + +You can restrict which services a key may use. Two layers combine: + +- **Global allowlist**: `serve.services`, applied to every key. +- **Per-key allowlist**: `services` inside that key's `users` entry. + +The effective allowlist for a request is: + +| Global set? | Per-key set? | Result | +|---|---|---| +| Yes | Yes | The **intersection** of the two | +| Yes | No | The global list | +| No | Yes | The per-key list | +| No | No | No restriction, all services allowed | + +```yaml title="Restricting services" +serve: + api_secret: "admin-key" + services: ["AMZN", "NF", "DSNP"] # global ceiling for all keys + users: + limited-key: + username: limited + services: ["AMZN"] # this key ends up allowed only AMZN +``` + +Endpoints that name a service (listing services, searching, listing titles/tracks, downloading, profiles, history, and remote sessions) all filter against this allowlist. A request for a service outside the key's allowlist is rejected as an invalid service. Service tags are matched after normalization, so casing and aliases resolve consistently. + +### CDM overrides and per-job credentials + +Two download-time capabilities are gated behind server config and, unless enabled, are refused with `403 Forbidden` even for an authenticated key: + +| Config key | Default | Controls | +|---|---|---| +| `serve.cdm_overrides` | off | Whether a request may pick a specific CDM device via the `cdm` field. Set to `true` to allow any, or to a list/set of device names to allow only those. | +| `serve.allow_job_credentials` | off | Whether a request may pass its own `credential` / `credentials` login details for a download job. | + +```yaml title="Opting in to per-request CDM and credentials" +serve: + api_secret: "admin-key" + cdm_overrides: ["my_widevine_device"] # or true to allow any configured device + allow_job_credentials: true +``` + +!!! tip "When to enable these gates" + The two gates are fully independent and can be combined. The intended posture is that a locked-down or public-facing deployment leaves **both** unset (they default off and reject with `403`) so client requests can never steer the server's CDM or feed it their own logins. Enabling them only makes sense for a trusted single-client deployment, where the caller and the operator are effectively the same party. + +!!! note "Client-supplied credentials get isolated token caches" + When `allow_job_credentials` is enabled and a download job supplies its own `credential` / `credentials`, each distinct credential is given its **own** isolated token cache on the server. Client-supplied logins are never shared or merged with the server's configured-credential caches, so they cannot cross-contaminate or read the server's own token state, and they don't clobber each other. + +See [Downloads](endpoints.md) for how these fields are used on the download endpoint. + +### Session IP binding (remote sessions) + +The remote-download session endpoints (`/api/session/*`) add an extra check on top of the key. When a session is created, the server records the client's IP address. Any later request for that session must come from the **same** IP, or it is rejected with `403 Forbidden`. This means a session is bound both to the key that created it and to the address it was created from. + +## Transport security (HTTPS) + +The API server speaks plain HTTP. Because your key is sent on every request, you should not expose it over an unencrypted connection on an untrusted network. To serve over HTTPS, run a reverse proxy in front of unshackle. The `serve` command can launch Caddy for you: + +```bash +unshackle serve --caddy +``` + +With `--caddy`, unshackle also starts `caddy run` using the `Caddyfile` located next to your unshackle config, letting you terminate TLS and reverse-proxy to the API. Configuring Caddy itself (certificates, domains, upstream) is done in that `Caddyfile`. + +## Quick reference + +| Item | Value | +|---|---| +| Auth header | `X-Secret-Key` | +| Default bind address | `127.0.0.1:8786` | +| Exempt endpoint | `GET /api/health` | +| Failure status | `401 Unauthorized` | +| Missing-header body | `{"status": 401, "message": "Secret Key is Empty."}` | +| Invalid-key body | `{"status": 401, "message": "Secret Key is Invalid."}` | +| Disable auth | `--no-key` (all requests allowed) | +| Config location | `serve.api_secret` and `serve.users` in `unshackle.yaml` | + +--- + +## Developer notes: how the check is implemented + +!!! note "This section is for people writing or debugging the server, not for API consumers." + +### Two middlewares, one header + +The header and config are identical between modes, but the code that enforces them differs: + +- **API-only server** (`--api-only`, or `--remote-only` which forces API-only). An `api_key_authentication` middleware runs on every request. It lets `GET /api/health` through unconditionally, then reads `X-Secret-Key`. An empty header returns the *"Secret Key is Empty."* body; a header that is not a configured user returns *"Secret Key is Invalid."*. In this mode the accepted-keys map contains exactly one entry: your `api_secret` (labelled `api_user`, with no devices). With `--no-key`, this middleware is simply not installed and the users map is empty. + +- **Integrated server** (default). A `serve_authentication` middleware wraps pywidevine's authentication (and pyplayready's for `/playready*` paths), and the REST routes live in the same application, guarded by the same check. Here the accepted-keys map is your full `serve` config: `api_secret` (auto-added with access to all loaded devices) plus every entry under `users`, each with its declared `devices` / `playready_devices`. + +Because the API-only middleware guards *all* non-health paths, the Swagger UI at `/api/docs/` is also behind `X-Secret-Key` there. In `--remote-only` mode the Swagger UI is not mounted at all. + +### CORS and the `Authorization` header + +A `cors_middleware` runs on every response in both modes (and answers `OPTIONS` preflight requests with an empty response). It emits: + +``` +Access-Control-Allow-Origin: * +Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS +Access-Control-Allow-Headers: Content-Type, X-Secret-Key, Authorization +Access-Control-Max-Age: 3600 +``` + +The REST API authenticates with `X-Secret-Key`; `Authorization` is advertised because the co-hosted pywidevine / pyplayready CDM endpoints use it. `PUT` appears in the allowed methods list but no REST route uses it; only `GET`, `POST`, and `DELETE` are wired up. + +### Where device access is read + +The map of accepted keys lives at `request.app["config"]["users"]`. For DRM operations, a key's permitted Widevine devices come from its `devices` list and PlayReady devices from `playready_devices`; the first configured device is used when a request does not (or may not) override it. A server-wide `config.cdm[<service>]` mapping, if present, takes precedence over the key's device list when resolving which device to use. diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md new file mode 100644 index 00000000..f752ef5b --- /dev/null +++ b/docs/dev/rest-api/endpoints.md @@ -0,0 +1,980 @@ +# REST API Endpoints + +This is the full endpoint reference for the unshackle REST API. It covers every route: its method and path, the request body it accepts, the response shape it returns, and the status codes it can produce. Routes are grouped into general, download jobs, server management, and remote-dl sessions. + +Start the server with [`unshackle serve`](../../guide/cli-reference.md). By default it binds to `127.0.0.1:8786`, and every route lives under the `/api/` prefix. Authentication, server modes, and the `X-Secret-Key` header are covered in detail on the [Authentication](authentication.md) page. Read that first if you have not already. For an end-to-end walkthrough, see the [Quickstart](quickstart.md). + +!!! note "Base URL" + All examples below assume a base URL of `http://127.0.0.1:8786`. Replace the host and port to match your own `unshackle serve -h <host> -p <port>` invocation. + +!!! tip "Interactive docs" + A live Swagger UI is mounted at **`/api/docs/`** whenever the server is not running in `--remote-only` mode. Its title is "Unshackle REST API" and its version tracks the installed package version. In API-only mode with a key configured, the Swagger UI itself sits behind `X-Secret-Key`; only `/api/health` is exempt. + +## Conventions + +- **Authentication.** Every request except `GET /api/health` requires the `X-Secret-Key` header when a key is configured. Starting the server with `--no-key` disables the check entirely. See [Authentication](authentication.md). +- **Content type.** Request bodies are JSON (`Content-Type: application/json`); all success responses are JSON unless a `204 No Content` is documented. +- **CORS.** Every response carries permissive CORS headers, and `OPTIONS` preflight requests are answered automatically. +- **Compression.** JSON responses of 256 bytes or more are gzip-compressed when the client sends `Accept-Encoding: gzip`. +- **Service allowlist.** Service-facing endpoints are filtered by the effective allowlist for your API key (the intersection of the global `serve.services` list and your per-key list). Services you are not allowed to use are treated as unknown. + +!!! warning "Error responses have two shapes" + Structured application errors (raised inside a handler) return `{"status": "error", "error_code": "...", "message": "...", "timestamp": "..."}`. The authentication middleware, however, returns a **different** shape: `{"status": 401, "message": "..."}` with an integer `status` and no `error_code`. Clients must handle both. The full error model is documented in the [Error responses](#error-responses) section at the bottom of this page. + +--- + +## General endpoints + +These endpoints report server state and enumerate what is available. `GET /api/health`, `GET /api/services`, and `POST /api/search` are also exposed in `--remote-only` mode. + +### `GET /api/health` + +Health check. **This is the only route exempt from authentication**: you can call it without an `X-Secret-Key` header. It reports the running version and a best-effort update check. + +=== "Request" + + ```bash + curl http://127.0.0.1:8786/api/health + ``` + +=== "Response `200`" + + ```json + { + "status": "ok", + "version": "5.3.0", + "update_check": { + "update_available": false, + "current_version": "5.3.0", + "latest_version": "5.3.0" + } + } + ``` + +If the update check fails (for example, no network), `update_available` and `latest_version` are `null` while `current_version` still reports the installed version. + +| Status | Meaning | +| --- | --- | +| `200` | Server is healthy. | + +### `GET /api/services` + +List the streaming services available on this server, filtered by your allowlist. Each entry describes the service's tag, matching rules, capability flags, and its CLI parameters (useful for building a UI that accepts service-specific options). + +=== "Request" + + ```bash + curl -H "X-Secret-Key: $KEY" http://127.0.0.1:8786/api/services + ``` + +=== "Response `200`" + + ```json + { + "services": [ + { + "tag": "EXAMPLE", + "aliases": ["EX"], + "geofence": ["US"], + "title_regex": "^https?://example\\.com/.+", + "url": "https://example.com", + "help": "EXAMPLE\nAuthorization: Cookies\nSecurity: FHD@L3", + "cli_params": [ + {"name": "title", "kind": "argument", "required": true}, + { + "name": "movie", + "kind": "option", + "opts": ["-m", "--movie"], + "is_flag": true, + "default": false, + "help": "Title is a movie.", + "type": "boolean", + "multiple": false + } + ], + "needs_auth": true, + "has_search": true, + "has_drm": true, + "auth_methods": ["cookies"] + } + ] + } + ``` + +Field notes: + +| Field | Description | +| --- | --- | +| `tag` | Canonical service tag; use this as the `service` value on other endpoints. | +| `aliases` | Alternate tags the service also answers to. | +| `geofence` | Region codes the service is geofenced to (may be empty). | +| `title_regex` | URL/ID pattern the service matches, as a string or list of strings (or `null`). | +| `url` | Service URL, taken from the service's CLI short help. | +| `help` | The service module's docstring. | +| `cli_params` | Array of parameter descriptors. Arguments have `{name, kind: "argument", required}`; options have `{name, kind: "option", opts, is_flag, default, help, type, multiple}` and, for choice options, a `choices` list. | +| `needs_auth` | Whether the service overrides `authenticate()`. | +| `has_search` | Whether the service supports `POST /api/search`. | +| `has_drm` | Whether the service overrides a Widevine or PlayReady license hook. | +| `auth_methods` | Declared or inferred auth methods, e.g. `["cookies"]`, `["credentials"]`. | + +| Status | Meaning | +| --- | --- | +| `200` | Service list returned. | +| `500` | Unexpected server error (`INTERNAL_ERROR`). | + +### `POST /api/search` + +Search a service for titles by query string. The service must implement `search()` (see `has_search` above). + +**Request body** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `service` | string | yes | — | Service tag. | +| `query` | string | yes | — | Search query. | +| `profile` | string | no | `null` | Credential/cookie profile to use. | +| `proxy` | string | no | `null` | Proxy URI or country code. | +| `no_proxy` | boolean | no | `false` | Force-disable all proxy use. | + +=== "Request" + + ```bash + curl -X POST http://127.0.0.1:8786/api/search \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"service": "EXAMPLE", "query": "planet earth"}' + ``` + +=== "Response `200`" + + ```json + { + "results": [ + { + "id": "12345", + "title": "Planet Earth", + "description": "A nature documentary series.", + "label": "Available", + "url": "https://example.com/title/12345" + } + ], + "count": 1 + } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Search completed. | +| `400` | `INVALID_PARAMETERS` | Missing `query`. | +| `400` | `INVALID_SERVICE` | Unknown or disallowed service. | +| `400` | `INVALID_PROXY` | Proxy could not be resolved. | +| `502` | `SERVICE_ERROR` | Service failed to initialize, or does not support search. | + +### `POST /api/list-titles` + +Resolve a title ID into its list of titles (episodes, movie, etc.) without downloading anything. + +**Request body** + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `service` | string | yes | Service tag. | +| `title_id` | string | yes | Title identifier. | +| `profile` | string | no | Credential/cookie profile. | +| `proxy` | string | no | Proxy URI or country code. | +| `no_proxy` | boolean | no | Force-disable proxy. | +| `cdm_type` | string | no | Preferred CDM type. | + +Service-specific CLI options may also be passed as additional keys. + +=== "Request" + + ```bash + curl -X POST http://127.0.0.1:8786/api/list-titles \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"service": "EXAMPLE", "title_id": "12345"}' + ``` + +=== "Response `200`" + + ```json + { + "titles": [ + { + "type": "episode", + "name": "Pilot", + "id": "ep-1", + "language": "en", + "description": "The first episode.", + "date": "2020-01-01", + "cover_url": "https://example.com/cover.jpg", + "year": 2020, + "series_title": "Example Show", + "season": 1, + "number": 1 + } + ] + } + ``` + +Each serialized title carries `type` (`"episode"`, `"movie"`, or `"other"`), `name`, `id`, `language`, `description`, `date`, and `cover_url`. Episodes and movies add `year`; episodes additionally add `series_title`, `season`, and `number`. + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Titles returned. | +| `400` | `INVALID_INPUT` | Missing `service` or `title_id`. | +| `400` | `INVALID_SERVICE` | Unknown or disallowed service. | +| `401` | `AUTH_FAILED` | Authentication with the service failed. | +| `404` | `NOT_FOUND` | Title not found. | +| `500` | `INTERNAL_ERROR` | Unexpected error. | + +### `POST /api/list-tracks` + +List the video, audio, and subtitle tracks for a title. For series, you can scope to specific episodes. + +**Request body** + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `service` | string | yes | Service tag. | +| `title_id` | string | yes | Title identifier. | +| `wanted` | string | no | Episode/season range (e.g. `"S01E01-S01E03"`). | +| `season` | int/string | no | Season number (combined with `episode`). | +| `episode` | int/string | no | Episode number (combined with `season`). | +| `profile`, `proxy`, `no_proxy`, `cdm_type` | — | no | As on `list-titles`. | + +When both `season` and `episode` are given, they are combined into a `"{season}x{episode}"` selector. + +=== "Single title / movie: Response `200`" + + ```json + { + "title": { "type": "movie", "name": "Example Movie", "id": "12345", "year": 2021 }, + "video": [ + { + "id": "v-1", "codec": "H265", "codec_display": "HEVC", + "bitrate": 8000, "width": 3840, "height": 2160, + "resolution": "2160p", "fps": "24", "range": "HDR10", + "range_display": "HDR10", "language": "en", "drm": [], "descriptor": "DASH" + } + ], + "audio": [ + { + "id": "a-1", "codec": "EC3", "codec_display": "DD+", + "bitrate": 640, "channels": "5.1", "language": "en", + "atmos": false, "descriptive": false, "drm": [], "descriptor": "DASH" + } + ], + "subtitles": [ + { "id": "s-1", "codec": "vtt", "language": "en", + "forced": false, "sdh": false, "cc": false, "descriptor": "DASH" } + ] + } + ``` + +=== "Multiple episodes: Response `200`" + + ```json + { + "episodes": [ + { + "title": { "type": "episode", "name": "Pilot", "id": "ep-1", "season": 1, "number": 1 }, + "video": [ ... ], + "audio": [ ... ], + "subtitles": [ ... ] + } + ], + "unavailable_episodes": [ + { "title": { "id": "ep-2" }, "error": "..." } + ] + } + ``` + +Tracks are sorted by bitrate, descending. Episodes that fail to resolve are collected into `unavailable_episodes` rather than failing the whole request. Track objects here do **not** include a download `url`; segment URLs are only exposed through the remote-dl session endpoints. + +The `drm` array on a track is a list of `{type, pssh?, kids?, content_keys?, license_url?}` objects. + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Tracks returned. | +| `400` | `INVALID_INPUT` | Missing `service` or `title_id`. | +| `400` | `INVALID_PARAMETERS` | Invalid `wanted` range. | +| `400` | `INVALID_SERVICE` | Unknown or disallowed service. | +| `404` | `NO_CONTENT` | No matching episodes, or all matches failed. | + +--- + +## Download jobs + +Downloads are asynchronous. `POST /api/download` enqueues a job and returns immediately with a `job_id`; the server runs each job in a background worker subprocess. You then poll the job endpoints for progress and results. + +!!! info "Job states" + A job moves through `queued` → `downloading` → one of the terminal states `completed`, `failed`, or `cancelled`. Only queued jobs can be prioritized; only terminal jobs can be retried or removed. + +### `POST /api/download` + +Create a download job. Requires `service` and `title_id`; every other field is an optional download parameter with a server-side default. On success this returns **`202 Accepted`**, not `200`. + +=== "Request" + + ```bash + curl -X POST http://127.0.0.1:8786/api/download \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "service": "EXAMPLE", + "title_id": "12345", + "quality": [2160], + "vcodec": "H265", + "range": ["HDR10"], + "wanted": ["S01E01"] + }' + ``` + +=== "Response `202`" + + ```json + { + "job_id": "b0f7c8e2-4a1d-4e9c-9b7e-2f3a1c0d5e6f", + "status": "queued", + "created_time": "2026-07-03T12:00:00+00:00" + } + ``` + +**Common parameters** (all optional). These mirror the [`dl` command](../../guide/downloading.md) flags: + +| Field | Type | Default | Notes | +| --- | --- | --- | --- | +| `profile` | string | `null` | Credential/cookie profile. | +| `quality` | int[] | `[]` | Target resolution(s); best available if empty. | +| `vcodec` | string/string[] | `null` | Video codec filter. | +| `acodec` | string/string[] | `null` | Audio codec filter. | +| `vbitrate` / `abitrate` | int | `null` | Video / audio bitrate in kbps (positive). | +| `vbitrate_range` / `abitrate_range` | string | `null` | `"MIN-MAX"` bitrate window. | +| `range` | string[] | `["SDR"]` | Dynamic range(s). | +| `channels` | number | `null` | Audio channel count. | +| `no_atmos` | boolean | `false` | Exclude Atmos tracks. | +| `wanted` | string[] | `[]` | Episode/season selectors. | +| `latest_episode` | boolean | `false` | Only the newest episode. | +| `lang` / `v_lang` / `a_lang` / `s_lang` | string[] | `["orig"]` / `[]` / `[]` / `["all"]` | Language filters. | +| `require_subs` | string[] | `[]` | Required subtitle languages. | +| `forced_subs` | boolean | `false` | Include forced subtitles. | +| `exact_lang` | boolean | `false` | Exact language matching. | +| `sub_format` | string | `null` | Output subtitle format. | +| `video_only` / `audio_only` / `subs_only` / `chapters_only` | boolean | `false` | Track-type restriction (at most one). | +| `no_subs` / `no_audio` / `no_chapters` / `no_video` | boolean | `false` | Skip a track type. | +| `audio_description` | boolean | `false` | Download audio description. | +| `slow` | boolean/string | `null` | Randomized delay between downloads. | +| `split_audio` | boolean | `null` | Separate files per audio codec. | +| `skip_dl` | boolean | `false` | Only fetch keys, do not download. | +| `export` | boolean | `false` | Export manifest/keys/subs to JSON. | +| `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. | +| `proxy` / `no_proxy` / `no_proxy_download` | string / bool / bool | `null` / `false` / `false` | Proxy controls. | +| `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. | +| `workers` | int | `null` | Threads per track. | +| `downloads` | int | `1` | Concurrent tracks. | +| `worst` | boolean | `false` | Lowest bitrate within quality. | +| `best_available` | boolean | `false` | Fall back to best available. | +| `repack` | boolean | `false` | Add REPACK tag. | +| `tag` | string | `null` | Release group tag. | +| `tmdb_id` / `imdb_id` / `animeapi_id` | — | `null` | External ID overrides for tagging. | +| `enrich` | boolean | `false` | Override title/year from external source. | +| `output_dir` | string | `null` | Override output directory. | +| `no_cache` / `reset_cache` | boolean | `false` | Title cache controls. | + +**Validation.** Invalid values return `400 INVALID_PARAMETERS`. Notably: `vcodec` must be one of H264/H265/H.264/H.265/AVC/HEVC/VC1/VC-1/VP8/VP9/AV1; `acodec` one of AAC/AC3/EC3/EAC3/DD/DD+/AC4/OPUS/FLAC/ALAC/VORBIS/OGG/DTS; `range` one of SDR/HDR10/HDR10P/DV/HLG/HYBRID (`HDR10+` is accepted); bitrate/worker/download counts must be positive integers; at most one of the `*_only` flags may be set; `no_subs` cannot be combined with `subs_only`, `no_audio` with `audio_only`, or `s_lang` with `require_subs`. + +!!! warning "Gated parameters (developer)" + A few keys are accepted but only permitted when the server config opts in, otherwise the request is rejected with `403 FORBIDDEN`: + + - `cdm` (per-request CDM device name): allowed only if `serve.cdm_overrides` is `true` or a list containing that device name. + - `credential` (`"user:pass"`) and `credentials` (a map): allowed only if `serve.allow_job_credentials` is truthy (off by default). + + Server-side, effective parameters are merged as defaults → `serve:` config overrides → service-specific defaults → your request body. + +| Status | Error code | Meaning | +| --- | --- | --- | +| `202` | — | Job queued. | +| `400` | `INVALID_INPUT` | Missing `service`/`title_id` or malformed JSON. | +| `400` | `INVALID_PARAMETERS` | A parameter failed validation. | +| `400` | `INVALID_SERVICE` | Unknown or disallowed service. | +| `403` | `FORBIDDEN` | A gated parameter is not permitted. | + +### `GET /api/download/jobs` + +List download jobs, with optional filtering and sorting. + +**Query parameters** + +| Param | Values | Default | Description | +| --- | --- | --- | --- | +| `status` | `queued`, `downloading`, `completed`, `failed`, `cancelled` | — | Filter by status. | +| `service` | service tag | — | Filter by service. | +| `sort_by` | `created_time`, `started_time`, `completed_time`, `progress`, `status`, `service` | `created_time` | Sort field. | +| `sort_order` | `asc`, `desc` | `desc` | Sort direction. | +| `full` | `true`, `false` | `false` | Include full per-job details. | + +=== "Request" + + ```bash + curl -H "X-Secret-Key: $KEY" \ + "http://127.0.0.1:8786/api/download/jobs?status=downloading&sort_by=progress&sort_order=desc" + ``` + +=== "Response `200`" + + ```json + { + "jobs": [ + { + "job_id": "b0f7c8e2-...", + "status": "downloading", + "created_time": "2026-07-03T12:00:00+00:00", + "service": "EXAMPLE", + "title_id": "12345", + "title": "Example Show S01E01", + "progress": 42.5, + "phase": "downloading", + "current_title": "Episode 1", + "completed_tracks": 1, + "total_tracks": 3, + "active_tracks": ["v-1"], + "track_progress": [{"label": "Video", "progress": 42.5, "speed": "5.2 MB/s"}], + "segments_done": 120, + "segments_total": 300, + "speed": "5.2 MB/s", + "skipped_subtitles": [] + } + ] + } + ``` + +With `full=true`, each job additionally includes `parameters` (with secrets redacted), `started_time`, `completed_time`, `output_files`, `error_message`, `error_details`, `error_code`, `error_traceback`, and `worker_stderr`. + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Jobs returned. | +| `400` | `INVALID_PARAMETERS` | Invalid `sort_by` or `sort_order`. | + +### `GET /api/download/jobs/{job_id}` + +Fetch a single job with full details (equivalent to a `full=true` list entry). + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Job returned. | +| `404` | `JOB_NOT_FOUND` | No such job. | + +### `DELETE /api/download/jobs/{job_id}` + +Cancel or remove a job. The behaviour depends on the job's current state: + +- **Terminal job** (completed/failed/cancelled): removed from the manager, returning **`204 No Content`** with an empty body. +- **Queued or downloading job**: cancelled, returning `200` `{"status": "success", "message": "Job cancelled"}`. + +!!! warning + A `204` response has no body. Do not attempt to parse JSON from it. + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Job cancelled. | +| `204` | — | Terminal job removed (empty body). | +| `400` | `INVALID_PARAMETERS` | Job is in a state that cannot be cancelled. | +| `404` | `JOB_NOT_FOUND` | No such job. | + +### `POST /api/download/jobs/clear-finished` + +Remove every terminal (completed/failed/cancelled) job from the manager. No request body. + +=== "Response `200`" + + ```json + { "removed": 3 } + ``` + +### `POST /api/download/jobs/{job_id}/retry` + +Enqueue a **new** job that reuses a terminal job's service, title, and parameters. The service allowlist and parameter gates are re-checked. Returns `202` with the new job's ID. + +!!! warning "Retry reuses the original, unredacted parameters" + Retry replays the finished job's stored parameters exactly as they were submitted, including any secrets such as `credential`/`credentials`, not the redacted `parameters` view that `GET /api/download/jobs` and `GET /api/history` expose. This is deliberate: it lets a retried job reproduce a credentialed download that the redacted job record alone never could. It also means anyone who can retry a job can re-run it with its original credentials, even though those credentials are masked everywhere else in the API. + +=== "Response `202`" + + ```json + { + "job_id": "9c1e2f3a-...", + "status": "queued", + "created_time": "2026-07-03T12:05:00+00:00" + } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `202` | — | New job queued. | +| `404` | `JOB_NOT_FOUND` | No such job. | +| `409` | `CONFLICT` | Job is not in a terminal state. | +| `400` | `INVALID_SERVICE` | Service is no longer allowed. | +| `403` | `FORBIDDEN` | A gated parameter is no longer permitted. | + +### `POST /api/download/jobs/{job_id}/priority` + +Move a queued job to the front of the download queue. + +=== "Response `200`" + + ```json + { "job_id": "b0f7c8e2-...", "position": "front" } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Job moved to front. | +| `404` | `JOB_NOT_FOUND` | No such job. | +| `409` | `CONFLICT` | Job is not queued. | + +--- + +## Server management + +Endpoints for profiles, effective configuration, download history, environment checks, and maintenance actions. + +### `GET /api/profiles` + +List the named credential profiles configured per service (usable as the `profile` parameter elsewhere). Only services whose credentials are a mapping of profile name to credential are listed; services with a single unnamed credential are omitted. Filtered by your allowlist. + +=== "Response `200`" + + ```json + { + "profiles": { + "EXAMPLE": ["default", "premium"], + "OTHER": ["us-account"] + } + } + ``` + +### `GET /api/config` + +Return a read-only, redacted view of the effective server configuration, suitable for a settings page in a UI. Secrets are never emitted: keys matching `secret`, `password`, `token`, `api_key`, or `credential`, and any userinfo in proxy URLs, are masked. + +=== "Response `200`" + + ```json + { + "config": { + "dl": { "...": "redacted default dl parameters" }, + "serve": { + "max_concurrent_downloads": 2, + "job_retention_hours": 24, + "history_limit": 100, + "services": null, + "remote_only": false, + "cdm_overrides": null, + "allow_job_credentials": false + }, + "directories": { + "downloads": "/home/user/Downloads/unshackle", + "temp": "/home/user/.unshackle/temp", + "cache": "/home/user/.unshackle/cache" + }, + "services": ["EXAMPLE", "OTHER"] + } + } + ``` + +### `GET /api/history` + +Read the persisted history of jobs that reached a terminal state, newest first. History is stored in `api_history.jsonl` in the cache directory; corrupt lines are skipped and a missing file yields an empty list. + +**Query parameters** + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| `limit` | int (≥ 1) | `100` | Maximum entries to return. | +| `service` | string | — | Case-insensitive service filter. | + +=== "Response `200`" + + ```json + { + "history": [ + { + "job_id": "b0f7c8e2-...", + "service": "EXAMPLE", + "title_id": "12345", + "title": "Example Show S01E01", + "status": "completed", + "created_time": "2026-07-03T12:00:00+00:00", + "completed_time": "2026-07-03T12:08:00+00:00", + "output_files": ["/downloads/Example.S01E01.mkv"], + "error_message": null, + "parameters": { "...": "redacted" } + } + ], + "count": 1 + } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | History returned. | +| `400` | `INVALID_PARAMETERS` | Invalid `limit`. | + +### `DELETE /api/history/{job_id}` + +Remove a single persisted history entry. Returns **`204 No Content`** (empty body) on success. + +| Status | Error code | Meaning | +| --- | --- | --- | +| `204` | — | Entry removed (empty body). | +| `404` | `NOT_FOUND` | No such entry (or outside your allowlist). | + +### `POST /api/maintenance/clear-cache` and `POST /api/maintenance/clear-temp` + +Delete and recreate the cache or temp directory. Neither takes a request body. Both are blocked with `409 CONFLICT` if any job is currently `downloading` (the `details` include the offending `active_jobs`). + +The `409` guard is about active file I/O, not an arbitrary lock: an in-flight job may still be *reading* cached title data under the cache directory, and active download workers are *writing* segment files under temp. Removing either directory mid-download would corrupt those running jobs, so the endpoints refuse until nothing is downloading. + +!!! warning "clear-cache also wipes download history" + The persistent history read by [`GET /api/history`](#get-apihistory) lives in `api_history.jsonl` **inside the cache directory**. Because clear-cache deletes and recreates that whole directory, calling it also erases your entire download history, not just cached title data. If you want to keep history, do not clear the cache; remove individual entries with [`DELETE /api/history/{job_id}`](#delete-apihistoryjob_id) instead. + +=== "Response `200`" + + ```json + { "cleared": true, "freed_bytes": 1048576 } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Directory cleared. | +| `409` | `CONFLICT` | A download is currently active. | + +### `POST /api/maintenance/refresh-services` + +git-pull every service repository configured under `directories.services`. `refreshed` is `true` when all repos updated successfully (or none are configured). + +=== "Response `200`" + + ```json + { + "refreshed": true, + "repos": [ + { "spec": "github.com/example/services", "updated": true, "changes": ["a1b2c3d Update EXAMPLE"] } + ] + } + ``` + +### `GET /api/env/check` + +Report the install status of the external binaries unshackle depends on (the same set `unshackle env check` inspects), with best-effort version detection. + +=== "Response `200`" + + ```json + { + "checks": [ + { "name": "ffmpeg", "installed": true, "version": "6.1.1", "required": true }, + { "name": "ccextractor", "installed": false, "version": null, "required": false } + ] + } + ``` + +--- + +## Remote-dl sessions + +!!! note "Developer feature" + The `/api/session/*` endpoints power unshackle's **remote download** mode, where a thin local client authenticates once against a remote server, then fetches titles, tracks, and segments and proxies DRM licensing back through it. These endpoints are exposed in `--remote-only` mode, and they are what the built-in `RemoteService` client drives. Most end users never call them directly. + +Sessions are held in memory. They expire after `serve.session_ttl` (default 300s), except while awaiting interactive authentication input, which uses a 600s window. A session is bound to the IP that created it, so a request from a different IP is rejected with `403 FORBIDDEN`. + +### Session lifecycle overview + +```text +POST /api/session/create → returns session_id, status "authenticating" +GET /api/session/{id}/prompt → poll until authenticated (submit answers via POST) +GET /api/session/{id}/titles → list titles +POST /api/session/{id}/tracks → tracks + chapters (with URLs) for one title +POST /api/session/{id}/segments → resolve segment URLs / DRM / headers per track +POST /api/session/{id}/license → proxy or server-CDM DRM licensing +DELETE /api/session/{id} → tear down, harvest updated cache +``` + +### `POST /api/session/create` + +Create a session for a service and title. Authentication runs asynchronously in the background. This call returns immediately with `status: "authenticating"`, and you then poll the prompt endpoint. The body accepts `service` and `title_id` (both required) plus a broad set of optional keys (`additionalProperties` is allowed): `credentials` (`{username, password, extra?}`), `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, `no_proxy`, `profile`, `cache` (map of forwarded cache files), `client_region`, `cdm_type`, `range_`, `vcodec`, `quality`, `best_available`, and any service CLI options. + +=== "Response `200`" + + ```json + { + "session_id": "d4e5f6a7-...", + "service": "EXAMPLE", + "status": "authenticating" + } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Session created; authentication in progress. | +| `400` | `INVALID_INPUT` | Missing `service` or `title_id`. | +| `400` | `INVALID_SERVICE` | Unknown or disallowed service. | + +### `GET /api/session/{session_id}/prompt` + +Poll for the authentication status and any pending interactive prompt (OTP, PIN, device code). Response variants: + +=== "Authenticating" + + ```json + { "status": "authenticating" } + ``` + +=== "Pending input" + + ```json + { "status": "pending_input", "prompt": "Enter the code sent to your email:" } + ``` + +=== "Authenticated" + + ```json + { "status": "authenticated" } + ``` + +=== "Failed" + + ```json + { "status": "failed", "error": "Invalid credentials" } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Status returned. | +| `403` | `FORBIDDEN` | Request IP differs from the session creator. | +| `404` | `SESSION_NOT_FOUND` | No such session. | + +### `POST /api/session/{session_id}/prompt` + +Submit a response to a pending prompt. Body: `{ "response": "..." }` (required). + +=== "Response `200`" + + ```json + { "status": "accepted" } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Response accepted. | +| `400` | `INVALID_INPUT` | Missing `response`, or no prompt is pending. | +| `404` | `SESSION_NOT_FOUND` | No such session. | + +### `GET /api/session/{session_id}/titles` + +Fetch the titles for an authenticated session. + +=== "Response `200`" + + ```json + { + "session_id": "d4e5f6a7-...", + "titles": [ { "type": "episode", "name": "Pilot", "id": "ep-1" } ] + } + ``` + +Requires an authenticated session; `404 SESSION_NOT_FOUND` otherwise. + +### `POST /api/session/{session_id}/tracks` + +Get tracks and chapters for a specific title within the session. Body: `{ "title_id": "..." }` (required). Unlike `list-tracks`, track objects here **include** download URLs, along with session headers/cookies and manifest data the client needs to download directly. + +=== "Response `200`" + + ```json + { + "title": { "type": "episode", "name": "Pilot", "id": "ep-1" }, + "video": [ { "id": "v-1", "url": "https://...", "drm": [ ... ] } ], + "audio": [ { "id": "a-1", "url": "https://..." } ], + "subtitles": [ { "id": "s-1", "url": "https://..." } ], + "chapters": [ { "timestamp": "00:00:00.000", "name": "Intro" } ], + "attachments": [ { "url": "https://...", "name": "font.ttf", "mime_type": "font/ttf", "description": "" } ], + "manifests": [ { "type": "dash", "url": "https://...", "data": "<base64 zlib xml>" } ], + "session_headers": { "User-Agent": "..." }, + "session_cookies": { "sid": "..." }, + "server_cdm_type": "widevine" + } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Tracks returned. | +| `400` | `INVALID_INPUT` | Missing or unknown `title_id`. | +| `404` | `SESSION_NOT_FOUND` | No such session. | + +### `POST /api/session/{session_id}/segments` + +Resolve the download URL, DRM info, headers, cookies, and raw track data for selected tracks. Body: `{ "track_ids": ["v-1", "a-1"] }` (required). + +=== "Response `200`" + + ```json + { + "tracks": { + "v-1": { "descriptor": "DASH", "url": "https://...", "drm": [ ... ], "headers": { ... }, "cookies": { ... }, "data": { ... } } + } + } + ``` + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | Segment info returned. | +| `400` | `INVALID_INPUT` | Missing `track_ids`. | +| `404` | `TRACK_NOT_FOUND` | An unknown track ID was requested. | + +### `POST /api/session/{session_id}/license` + +Acquire DRM keys. Two modes are supported via the `mode` field. + +**Proxy mode** (`mode: "proxy"`, the default) forwards the client's CDM `challenge` to the service's license endpoint and returns the raw license bytes: + +=== "Request" + + ```json + { "track_id": "v-1", "challenge": "<base64 CDM challenge>", "drm_type": "widevine" } + ``` + +=== "Response `200`" + + ```json + { "license": "<base64 license bytes>" } + ``` + +**Server-CDM mode** (`mode: "server_cdm"`) runs the full CDM flow on the server using the device configured for your API key, returning content keys directly. Use `track_id` for a single track or `track_ids` for a batch: + +=== "Single: Response `200`" + + ```json + { "keys": { "<kid_hex>": "<key_hex>" } } + ``` + +=== "Batch: Response `200`" + + ```json + { "keys": { "v-1": { "<kid_hex>": "<key_hex>" } }, "drm_type": "widevine" } + ``` + +| Field | Description | +| --- | --- | +| `track_id` / `track_ids` | Single track, or a batch of tracks. | +| `challenge` | Base64 CDM challenge (proxy mode). | +| `pssh` | Base64 PSSH (server-CDM mode). | +| `drm_type` | `widevine` (default) or `playready`. | +| `mode` | `proxy` (default) or `server_cdm`. | + +| Status | Error code | Meaning | +| --- | --- | --- | +| `200` | — | License/keys returned. | +| `400` | `INVALID_INPUT` | Missing challenge/PSSH, or no CDM device configured for the key. | +| `400` | `INVALID_PARAMETERS` | Unsupported `drm_type`. | +| `404` | `TRACK_NOT_FOUND` | Unknown track ID. | +| `404` | `NO_CONTENT` | Server CDM produced no keys. | +| `502` | `SERVICE_ERROR` | The service aborted during licensing. | + +### `GET /api/session/{session_id}` + +Return session metadata and validity. + +=== "Response `200`" + + ```json + { + "session_id": "d4e5f6a7-...", + "service": "EXAMPLE", + "valid": true, + "expires_in": 300, + "track_count": 4, + "title_count": 10 + } + ``` + +`404 SESSION_NOT_FOUND` if the session does not exist or has expired. + +### `DELETE /api/session/{session_id}` + +Tear down a session. Its input bridge is cancelled, any updated cache files are harvested, and the session plus its cache directory are deleted. + +=== "Response `200`" + + ```json + { + "status": "ok", + "cache": { "tokens.json": "<base64 zlib bytes>" } + } + ``` + +The `cache` field is present only if the session produced cache files. `404 SESSION_NOT_FOUND` if the session does not exist. + +--- + +## Error responses + +Every structured error raised inside a handler is serialized to the same JSON envelope: + +```json +{ + "status": "error", + "error_code": "INVALID_PARAMETERS", + "message": "Invalid video codec: XYZ", + "timestamp": "2026-07-03T12:00:00+00:00", + "details": { "field": "vcodec" }, + "retryable": true, + "debug_info": { "exception_type": "APIError", "traceback": "..." } +} +``` + +- `details` appears only when non-empty. +- `retryable` appears only when the error is retryable (e.g. network errors, rate limiting). +- `debug_info` appears only when the server was started with `--debug-api`. + +!!! warning "Authentication errors differ" + Errors raised by the API-key middleware use a distinct shape (`{"status": 401, "message": "Secret Key is Empty."}` or `{"status": 401, "message": "Secret Key is Invalid."}`) with an integer `status` and no `error_code`, `timestamp`, or `details`. A malformed JSON body on any POST, by contrast, returns the standard envelope with `error_code: "INVALID_INPUT"`. + +### Error codes and default status + +| Error code | HTTP status | +| --- | --- | +| `INVALID_INPUT` | 400 | +| `INVALID_SERVICE` | 400 | +| `INVALID_PROXY` | 400 | +| `INVALID_PARAMETERS` | 400 | +| `AUTH_FAILED` | 401 | +| `FORBIDDEN` | 403 | +| `GEOFENCE` | 403 | +| `NOT_FOUND` | 404 | +| `NO_CONTENT` | 404 | +| `JOB_NOT_FOUND` | 404 | +| `SESSION_NOT_FOUND` | 404 | +| `TRACK_NOT_FOUND` | 404 | +| `CONFLICT` | 409 | +| `RATE_LIMITED` | 429 | +| `INTERNAL_ERROR` | 500 | +| `DOWNLOAD_ERROR` | 500 | +| `WORKER_ERROR` | 500 | +| `SERVICE_ERROR` | 502 | +| `DRM_ERROR` | 502 | +| `NETWORK_ERROR` | 503 | +| `SERVICE_UNAVAILABLE` | 503 | + +When a generic (uncategorized) exception reaches a handler, unshackle inspects its message and type to pick a code. For example, messages mentioning "connection" or "timeout" become `NETWORK_ERROR` (503, retryable), "rate limit" or "429" become `RATE_LIMITED` (429, retryable), and anything unrecognized collapses to `INTERNAL_ERROR` (500). + +--- + +## See also + +- [REST API overview](index.md). What the server is and when to use it. +- [Authentication](authentication.md). Keys, server modes, and the `X-Secret-Key` header. +- [Quickstart](quickstart.md). A first end-to-end request walkthrough. +- [Downloading](../../guide/downloading.md). The `dl` flags mirrored by `POST /api/download`. diff --git a/docs/dev/rest-api/errors.md b/docs/dev/rest-api/errors.md new file mode 100644 index 00000000..32b8ceee --- /dev/null +++ b/docs/dev/rest-api/errors.md @@ -0,0 +1,254 @@ +# Error Responses + +Every endpoint of the unshackle REST API reports failures in a single, predictable +shape. Whether a request was rejected because a field was missing, a service refused +to authenticate, or the download worker crashed, the server answers with a JSON body +that carries a machine-readable **error code**, a human-readable **message**, and a +matching **HTTP status**. This page documents that format and catalogues every error +code the server can emit. + +If you are building a client against the API, read this page alongside the endpoint +reference. Each endpoint lists the specific codes it raises, and the meanings of +those codes are defined here. + +!!! note "Where this comes from" + The error format is defined in `unshackle/core/api/errors.py`, and the wrapper + that turns raised errors into responses lives in `unshackle/core/api/routes.py`. + Nothing on this page is invented; it reflects the actual server behaviour. + +## The standard error shape + +A structured error is always a JSON object with `status` set to the literal string +`"error"`. The full set of fields: + +```json title="Error response body" +{ + "status": "error", + "error_code": "INVALID_PARAMETERS", + "message": "vcodec must be one of: H264, H265, ...", + "timestamp": "2026-07-03T12:34:56.789012+00:00", + "details": { "field": "vcodec" }, + "retryable": true, + "debug_info": { + "exception_type": "APIError", + "traceback": "..." + } +} +``` + +| Field | Type | Always present? | Meaning | +|---|---|---|---| +| `status` | string | Yes | Always the literal `"error"`. Lets a client distinguish an error body from a success body at a glance. | +| `error_code` | string | Yes | A stable, machine-readable code from the [catalogue below](#error-code-catalogue). Match on this, not on `message`. | +| `message` | string | Yes | A human-readable description of what went wrong. Wording may change between versions, so do not parse it. | +| `timestamp` | string | Yes | ISO 8601 timestamp (UTC) of when the error response was built. | +| `details` | object | Only when non-empty | Structured, error-specific context (for example the invalid field, a list of available titles, or the set of currently active jobs). | +| `retryable` | boolean | Only when `true` | Present and set to `true` when retrying the same request may succeed later (network blips, rate limiting, transient outages). Absent otherwise. | +| `debug_info` | object | Only with `--debug-api` | Technical diagnostics; see [Debug information](#debug-information). | + +!!! tip "Match on `error_code`, branch on HTTP status" + Treat `error_code` as the primary signal your code reacts to, and the HTTP status + as a coarse fallback for anything you do not recognise. The two are consistent: + every code maps to a default status (see the catalogue), and the pairing is + stable across releases. + +### The `details` object + +When present, `details` carries extra context specific to the error. It is only +included when it is non-empty, so a client must treat a missing `details` as "no +extra information". Some concrete examples you may encounter: + +- A validation failure may include the offending field or the list of valid values. +- A "title not found" error on a remote-dl session includes `available_titles`. +- A maintenance conflict includes `active_jobs`, the jobs blocking the operation. +- Categorised exceptions add a `reason` tag, such as `"network_connectivity"` or + `"geofence_restriction"`. + +### The `retryable` hint + +The `retryable` field appears **only when it is `true`**. It is set for conditions +the server believes are transient: network errors, rate limiting, and temporary +service unavailability. When you see it, backing off and retrying is reasonable; when +it is absent, assume the request will keep failing until you change something. + +!!! example "A retryable network error" + ```json + { + "status": "error", + "error_code": "NETWORK_ERROR", + "message": "Network error occurred: Connection timed out", + "timestamp": "2026-07-03T12:00:00+00:00", + "details": { "reason": "network_connectivity" }, + "retryable": true + } + ``` + +## Error code catalogue + +Every code below is a member of the `APIErrorCode` enumeration. The **Status** column +is the default HTTP status the server returns for that code. A handler can override +the status in special cases (for example, a network error is always forced to `503`), +but the defaults hold for the overwhelming majority of responses. + +### Client errors (4xx) + +| Code | Status | Meaning | +|---|---|---| +| `INVALID_INPUT` | 400 | A required field is missing, or the request body could not be parsed. Also used as the generic "your input was malformed" code. | +| `INVALID_SERVICE` | 400 | The named service is unknown, or is not permitted for your API key. | +| `INVALID_PROXY` | 400 | The supplied proxy specification could not be resolved or is malformed. | +| `INVALID_PARAMETERS` | 400 | One or more download or query parameters failed validation (bad codec, bitrate, sort field, and so on). | +| `AUTH_FAILED` | 401 | Authentication with the streaming service failed (bad credentials or cookies). | +| `FORBIDDEN` | 403 | The action is not allowed. Raised by server-side gates (per-key restrictions on CDM or credential overrides) and by session IP binding. | +| `GEOFENCE` | 403 | The content is not available in the applicable region. | +| `NOT_FOUND` | 404 | A requested resource (title, history entry, and so on) does not exist. | +| `NO_CONTENT` | 404 | The request was valid but produced nothing: no matching titles, tracks, episodes, or keys. | +| `JOB_NOT_FOUND` | 404 | The referenced download job does not exist. | +| `SESSION_NOT_FOUND` | 404 | The remote-dl session does not exist or has expired. | +| `TRACK_NOT_FOUND` | 404 | A referenced track ID is not present in the session. | +| `CONFLICT` | 409 | The target resource is in a state that disallows the action (retrying a running job, prioritising a non-queued job, clearing cache during an active download). | +| `RATE_LIMITED` | 429 | The streaming service is rate-limiting requests. Retryable. | + +### Server errors (5xx) + +| Code | Status | Meaning | +|---|---|---| +| `INTERNAL_ERROR` | 500 | An unexpected server-side error. This is the fallback for any exception the server could not categorise. | +| `DOWNLOAD_ERROR` | 500 | The download process itself failed. | +| `WORKER_ERROR` | 500 | The download worker subprocess errored out. | +| `SERVICE_ERROR` | 502 | The streaming service's API returned an error, or the service does not support the requested operation. | +| `DRM_ERROR` | 502 | DRM or license acquisition failed. Retryable. | +| `NETWORK_ERROR` | 503 | A network connectivity problem (connection, timeout, DNS, TLS). Retryable. | +| `SERVICE_UNAVAILABLE` | 503 | The service is temporarily unavailable or under maintenance. Retryable. | + +!!! note "Fallback status" + If a code is ever emitted without a mapped status, the server falls back to + `500`. In practice every code above has an explicit mapping. + +## How errors are produced + +Understanding where an error comes from helps interpret it. + +**Explicitly raised errors.** Most endpoints raise a structured `APIError` directly. +For example, a missing `service` field yields `INVALID_INPUT`, and a disallowed +service yields `INVALID_SERVICE`. These carry a precise code, message, and often a +populated `details` object. + +**Categorised exceptions.** When an endpoint calls into a streaming service and an +uncategorised exception bubbles up, the server runs it through a classifier that +inspects the exception type and message and maps it to the most appropriate code. +The classification is keyword-based, checked in order: + +| If the error mentions… | It becomes | Retryable | +|---|---|---| +| auth, login, credential, unauthorized, forbidden, token | `AUTH_FAILED` (401) | no | +| connection, timeout, network, unreachable, socket, dns, resolve (or `ConnectionError`, `TimeoutError`, `URLError`, `SSLError`) | `NETWORK_ERROR` (503) | yes | +| geofence, region, "not available in", territory | `GEOFENCE` (403) | no | +| "not found", 404, "does not exist", "invalid id" | `NOT_FOUND` (404) | no | +| rate limit, too many requests, 429, throttle | `RATE_LIMITED` (429) | yes | +| drm, license, widevine, playready, decrypt | `DRM_ERROR` (502) | no | +| "service unavailable", 503, maintenance, "temporarily unavailable" | `SERVICE_UNAVAILABLE` (503) | yes | +| invalid, malformed, validation (or `ValueError`, `ValidationError`) | `INVALID_INPUT` (400) | no | +| anything else | `INTERNAL_ERROR` (500) | no | + +Categorised errors include a `reason` tag inside `details` (such as +`"network_connectivity"` or `"drm_failure"`) so a client can react without parsing +the message text. + +**Invalid JSON bodies.** If a `POST` endpoint receives a body that is not valid JSON, +the server returns `INVALID_INPUT` (400) with the parse error in +`details.error`: + +```json title="Malformed request body" +{ + "status": "error", + "error_code": "INVALID_INPUT", + "message": "Invalid JSON request body", + "timestamp": "2026-07-03T12:00:00+00:00", + "details": { "error": "Expecting value: line 1 column 1 (char 0)" } +} +``` + +## The authentication error exception + +There is one place where the response does **not** follow the standard shape: the API +key authentication middleware. When you send a request to a protected endpoint +without a valid `X-Secret-Key` header, the middleware short-circuits with a legacy +body that uses an integer `status` and no `error_code` or `timestamp`. + +=== "Missing key" + + ```json + { "status": 401, "message": "Secret Key is Empty." } + ``` + +=== "Invalid key" + + ```json + { "status": 401, "message": "Secret Key is Invalid." } + ``` + +!!! warning "Handle both shapes" + A robust client must expect two error formats: the standard structured error + (string `status` of `"error"`, with `error_code`), and this authentication error + (integer `status`, no `error_code`). The `GET /api/health` endpoint is exempt + from authentication, so it never returns the key-error shape. + +## Responses with no body + +Not every failure or completion has a JSON body. Some endpoints answer with +**`204 No Content`** and an empty body, notably deleting a terminal download job and +deleting a history entry. + +!!! warning "Do not parse `204` bodies" + A `204` response has no body. Clients must check the status code before attempting + to decode JSON, or the decode will fail on an empty string. + +## Debug information + +By default, error responses never include tracebacks or internal diagnostics. If the +server was started with the `--debug-api` flag, every structured error gains a +`debug_info` object: + +```json title="Error with --debug-api enabled" +{ + "status": "error", + "error_code": "INTERNAL_ERROR", + "message": "An unexpected error occurred: ...", + "timestamp": "2026-07-03T12:00:00+00:00", + "debug_info": { + "exception_type": "KeyError", + "traceback": "Traceback (most recent call last): ..." + } +} +``` + +The `debug_info` object always carries `exception_type` and, for exception-derived +errors, a formatted `traceback`. Some endpoints add extra diagnostics such as +captured worker stderr. + +!!! warning "Developer use only" + `--debug-api` exposes internal tracebacks in responses. Enable it only for local + debugging, never on a server reachable by untrusted clients. + +## Client handling recommendations + +For anyone writing a consumer of the API, a durable strategy is: + +1. **Read the HTTP status first.** It tells you the broad class of outcome and lets + you handle the `204`/no-body and authentication-error cases before touching the + body. +2. **Branch on `error_code`.** For structured errors, this is the stable identifier + to switch on. Do not match on `message`. +3. **Respect `retryable`.** When it is `true`, back off and retry; the reference + client retries `429`, `500`, `502`, `503`, and `504` with exponential backoff. When + absent, fix the request rather than repeating it. +4. **Fall back gracefully.** Unrecognised codes should be surfaced using the + `message` plus the HTTP status. + +!!! note "Reference consumer" + unshackle's own remote-download client treats any status of `400` or above as + fatal: it logs `Server error [<error_code>]: <message>` and exits. It also wraps + requests in a retry adapter (five attempts, backing off on `429`, `500`, `502`, + `503`, and `504`). This is a reasonable template for your own client's error + handling. diff --git a/docs/dev/rest-api/index.md b/docs/dev/rest-api/index.md new file mode 100644 index 00000000..0f5dee19 --- /dev/null +++ b/docs/dev/rest-api/index.md @@ -0,0 +1,237 @@ +# REST API Overview + +unshackle ships with a built-in HTTP server. Alongside the command-line tool, you +can run `unshackle serve` and drive the same download engine over a REST API: +searching services, listing titles and tracks, queueing downloads, watching live +progress, and browsing history, all as JSON over HTTP. + +This page explains what the API is for, how to start the server, where it listens, +and how the endpoints are organised. Later pages cover each area in depth. + +!!! info "Who this section is for" + The REST API is aimed at people building automation, dashboards, or remote + clients on top of unshackle. If you only download the occasional title, the + [command-line tool](../../guide/cli-reference.md) does everything the API does + and is simpler to use. Reach for the API when you want a long-running service, a + job queue, or a web front-end. + +## What the API is for + +The server exposes the same capabilities as the `dl` command, but as discrete HTTP +calls you can script against: + +- **Discovery**: list the services available on the server, run a service's search, + and enumerate the titles and tracks behind a title ID. +- **Downloading**: submit a download as a background *job*, then poll its status and + live progress, cancel it, retry it, or bump it to the front of the queue. +- **History and housekeeping**: read the log of finished jobs, inspect the server's + effective (redacted) configuration, check external tool versions, and clear caches + or temp files. +- **Remote-download sessions**: a thin local client can authenticate against a + service *on the server*, then pull titles, tracks, and segment info back, and proxy + DRM licensing. This is how one machine with the credentials and CDMs can serve + downloads to another. See [Remote Sessions](remote-sessions.md). + +Because every download runs as a queued job in a separate worker process, the API is +well suited to running unattended: submit work, come back later, and read the result. + +## Running the server + +Start the server with the `serve` command: + +```shell title="Start the REST API on the default host and port" +unshackle serve +``` + +By default the server binds to `127.0.0.1:8786` and requires an API key on every +request. It also serves the local Widevine and PlayReady CDM endpoints (for +pywidevine / pyplayready clients) in addition to the REST API. + +!!! warning "An API key is required by default" + Unless you pass `--no-key`, the server needs an API secret configured under the + `serve` section of your config. If none is set, `unshackle serve` refuses to + start. See [Authentication](authentication.md) for how to set `api_secret` and + how the `X-Secret-Key` header works. + +### Common options + +The `serve` command accepts the following flags: + +| Option | Default | Effect | +|---|---|---| +| `-h`, `--host` | `127.0.0.1` | Address to bind to. Use `0.0.0.0` to accept connections from other machines. | +| `-p`, `--port` | `8786` | Port to bind to. | +| `--api-only` | off | Serve only the REST API, without the Widevine/PlayReady CDM HTTP endpoints. | +| `--no-widevine` | off | Disable the Widevine CDM endpoints. Cannot be combined with `--api-only`. | +| `--no-playready` | off | Disable the PlayReady CDM endpoints. Cannot be combined with `--api-only`. | +| `--no-key` | off | Disable API-key authentication entirely; every request is allowed. | +| `--remote-only` | off | Expose only the remote-session subset of routes. Implies `--api-only`. | +| `--debug-api` | off | Include tracebacks and worker stderr in error responses. | +| `--debug` | off | Enable debug-level logging. | +| `--caddy` | off | Also launch [Caddy](https://caddyserver.com/) as a reverse proxy, using the `Caddyfile` in your user-config directory. | + +```shell title="Serve the REST API only, on all interfaces" +unshackle serve --api-only --host 0.0.0.0 --port 8786 +``` + +!!! danger "Binding to a public interface" + `--no-key` disables authentication completely. Anyone who can reach the port can + search, download, and read your configuration. Only combine it with a host like + `127.0.0.1`, or keep the default key-based auth when binding to `0.0.0.0`. Put a + TLS-terminating reverse proxy (Caddy, nginx) in front for anything internet-facing. + +### Server modes + +The set of routes that is actually mounted depends on how you start the server: + +=== "Default (CDM + REST)" + + Both the pywidevine/pyplayready CDM endpoints and the full REST API are mounted. + This is the mode used when you run `unshackle serve` with no mode flags. + +=== "`--api-only`" + + Only the REST API is mounted; the CDM HTTP endpoints are omitted. Authentication + uses the single configured API key (or none, with `--no-key`). + +=== "`--remote-only`" + + Only the remote-session subset of routes is exposed (health, services, search, + and everything under `/api/session/*`). This mode implies `--api-only`, and the + interactive Swagger UI is **not** mounted. + +## Base URL + +All REST endpoints live under the `/api` prefix. With the default bind address the +base URL is: + +``` +http://127.0.0.1:8786/api +``` + +Every path in this documentation is relative to the host and port. For example, the +health check is `GET http://127.0.0.1:8786/api/health`. + +!!! tip "Interactive API docs" + Unless you started the server with `--remote-only`, an interactive Swagger UI is + available at [`/api/docs/`](http://127.0.0.1:8786/api/docs/). It is generated from + the same route table the server actually serves, so it stays in sync with the + running build. Note that in `--api-only` mode the docs page is itself behind the + API key; only `/api/health` is exempt from authentication. + +A quick smoke test that needs no key: + +```shell title="Confirm the server is up" +curl http://127.0.0.1:8786/api/health +``` + +```json title="Response" +{ + "status": "ok", + "version": "5.3.0", + "update_check": { + "update_available": false, + "current_version": "5.3.0", + "latest_version": "5.3.0" + } +} +``` + +## Endpoint map + +Every REST path uses JSON request and response bodies and lives under `/api`. The +table below is a map of the surface; the [Endpoints](endpoints.md) page documents each +request and response in full. + +### Discovery + +| Method | Path | Purpose | +|---|---|---| +| `GET` | `/api/health` | Liveness and version check. Never requires a key. | +| `GET` | `/api/services` | List the services available on this server. | +| `POST` | `/api/search` | Run a service's search for a query. | +| `POST` | `/api/list-titles` | List the titles behind a service title ID. | +| `POST` | `/api/list-tracks` | List the video, audio, and subtitle tracks for a title. | + +### Downloads and jobs + +| Method | Path | Purpose | +|---|---|---| +| `POST` | `/api/download` | Queue a download; returns a `job_id`. | +| `GET` | `/api/download/jobs` | List jobs, with filtering and sorting. | +| `POST` | `/api/download/jobs/clear-finished` | Remove all finished jobs. | +| `GET` | `/api/download/jobs/{job_id}` | Full detail for one job. | +| `DELETE` | `/api/download/jobs/{job_id}` | Cancel a running job, or remove a finished one. | +| `POST` | `/api/download/jobs/{job_id}/retry` | Re-queue a finished job's parameters as a new job. | +| `POST` | `/api/download/jobs/{job_id}/priority` | Move a queued job to the front. | + +### History and server info + +| Method | Path | Purpose | +|---|---|---| +| `GET` | `/api/history` | Read the persisted log of finished jobs. | +| `DELETE` | `/api/history/{job_id}` | Delete a history entry. | +| `GET` | `/api/profiles` | Named credential profiles per service. | +| `GET` | `/api/config` | The server's effective, redacted configuration. | +| `GET` | `/api/env/check` | Presence and versions of external tools. | + +### Maintenance + +| Method | Path | Purpose | +|---|---|---| +| `POST` | `/api/maintenance/clear-cache` | Clear the cache directory. | +| `POST` | `/api/maintenance/clear-temp` | Clear the temp directory. | +| `POST` | `/api/maintenance/refresh-services` | `git pull` each configured service repository. | + +### Remote-download sessions + +| Method | Path | Purpose | +|---|---|---| +| `POST` | `/api/session/create` | Start a server-side, authenticated service session. | +| `GET` | `/api/session/{session_id}` | Session info and time-to-live. | +| `DELETE` | `/api/session/{session_id}` | Tear down a session. | +| `GET` | `/api/session/{session_id}/titles` | Titles for the session's title ID. | +| `POST` | `/api/session/{session_id}/tracks` | Tracks (with URLs) for a chosen title. | +| `POST` | `/api/session/{session_id}/segments` | Per-track segment/download details. | +| `POST` | `/api/session/{session_id}/license` | Proxy or server-side DRM licensing. | +| `GET` | `/api/session/{session_id}/prompt` | Poll for interactive auth status/prompt. | +| `POST` | `/api/session/{session_id}/prompt` | Submit an answer to an auth prompt. | + +!!! note "Remote-only routes" + The health, services, search, and `/api/session/*` routes are the ones exposed + when the server runs with `--remote-only`. Everything else (downloads, jobs, + history, config, maintenance) is only available in the full or `--api-only` modes. + +## Authentication in one line + +With the default configuration, send your API secret in the `X-Secret-Key` header on +every request except `/api/health`: + +```shell title="An authenticated request" +curl -X POST http://127.0.0.1:8786/api/search \ + -H "X-Secret-Key: your-api-secret" \ + -H "Content-Type: application/json" \ + -d '{"service": "NF", "query": "example"}' +``` + +A missing or unknown key returns `401`. The full model, including per-key service +allowlists, is covered in [Authentication](authentication.md). + +## Where to go next + +<div class="grid cards" markdown> + +- **[Quick Start](quickstart.md)**: start the server, set a key, and run your first + end-to-end download over the API. +- **[Authentication](authentication.md)**: the `X-Secret-Key` header, `--no-key`, + per-key service allowlists, and CORS. +- **[Endpoints](endpoints.md)**: every request body, query parameter, and response + shape, endpoint by endpoint. +- **[Job Lifecycle](job-lifecycle.md)**: how a download moves from `queued` to a + terminal state, live progress, cancellation, and retries. +- **[Remote Sessions](remote-sessions.md)**: the `/api/session/*` flow for driving a + service from a thin remote client. +- **[Error Responses](errors.md)**: the standard error envelope, error codes, and + their HTTP statuses. + +</div> diff --git a/docs/dev/rest-api/job-lifecycle.md b/docs/dev/rest-api/job-lifecycle.md new file mode 100644 index 00000000..f300996c --- /dev/null +++ b/docs/dev/rest-api/job-lifecycle.md @@ -0,0 +1,412 @@ +# Download Job Lifecycle + +When you submit a download over the REST API, the server does not block the request +until the file is written. Instead it creates a **job**, hands it back to you with an +ID, and works through it in the background. Everything you do afterwards (watching +progress, cancelling, retrying, reading the result) is done by referring to that job +ID. + +This page explains the states a job moves through, how live progress is reported, how +to cancel or retry, and, in clearly marked developer sections, how the worker and +manager behind the queue actually run the download. + +!!! info "Who this page is for" + The first half is for anyone driving the API: the states, the fields you can read, + and the calls that move a job around. The **Under the hood** sections near the end + describe the internal worker/manager model and are aimed at contributors and people + debugging the server. You do not need them to use the API. + +For the full request and response shape of each endpoint, see +[Endpoints](endpoints.md). For how errors are reported, see +[Error Responses](errors.md). + +## The five states + +A job is always in exactly one of five states, exposed as the `status` field: + +| Status | Meaning | +|---|---| +| `queued` | Created and waiting for a free worker. No download has started yet. | +| `downloading` | A worker has picked the job up and the download is in progress. | +| `completed` | The download finished successfully; `output_files` lists the results. | +| `failed` | The download stopped with an error; `error_message` and `error_code` explain why. | +| `cancelled` | The job was cancelled before it could finish, from either the queued or downloading state. | + +The last three (`completed`, `failed`, and `cancelled`) are **terminal**: once a +job reaches one of them it never changes state again. Terminal jobs can be removed, +retried, or read from history, but they will not resume. + +## How a job moves between states + +```mermaid +stateDiagram-v2 + [*] --> queued: POST /api/download + queued --> downloading: a worker picks it up + queued --> cancelled: DELETE (while queued) + downloading --> completed: download succeeds + downloading --> failed: download errors + downloading --> cancelled: DELETE (while downloading) + completed --> [*] + failed --> [*] + cancelled --> [*] + + completed --> queued: POST .../retry (as a NEW job) + failed --> queued: POST .../retry (as a NEW job) + cancelled --> queued: POST .../retry (as a NEW job) +``` + +The transitions in prose: + +- **`queued` → `downloading`** happens automatically. As soon as one of the server's + workers is free, it takes the next job off the queue and starts it. You do not + trigger this; you only observe it. +- **`downloading` → `completed`** is the happy path: the download engine wrote all the + requested files and reported success. +- **`downloading` → `failed`** is any error during the download: a service failure, a + network problem, a DRM error, and so on. The job records what went wrong. +- **`queued`/`downloading` → `cancelled`** happens when you send a `DELETE` for a job + that has not finished (see [Cancelling a job](#cancelling-a-job)). +- **Retry** does *not* move a terminal job back to `queued`. Instead it creates a + brand-new job that reuses the old job's service, title, and parameters. The original + job stays in its terminal state (see [Retrying a job](#retrying-a-job)). + +!!! note "Priority does not change status" + Moving a job to the front of the queue with the priority endpoint reorders *when* + it will start, but it stays `queued` until a worker actually picks it up. It is not + a state change. + +## Creating a job + +A download is queued with `POST /api/download`. The request needs at least a `service` +and a `title_id`, plus any download options you want to set. On success the server +replies with **`202 Accepted`** and the new job's identity: + +```shell title="Queue a download" +curl -X POST http://127.0.0.1:8786/api/download \ + -H "X-Secret-Key: your-api-secret" \ + -H "Content-Type: application/json" \ + -d '{ + "service": "NF", + "title_id": "81234567", + "quality": ["1080"], + "range": ["HDR10"] + }' +``` + +```json title="202 Accepted" +{ + "job_id": "b1e6c7de-6d0a-4f2f-9d1e-3a2b4c5d6e7f", + "status": "queued", + "created_time": "2026-07-03T12:00:00.000000" +} +``` + +The job starts in `queued`. Workers auto-start on the first job submitted, so there is +nothing else to switch on. Keep the `job_id`; every later call is addressed to it. + +!!! tip "The full parameter set" + `POST /api/download` accepts the same download options as the `dl` command + (quality, codecs, language selection, `wanted` episode ranges, and many more), with + server-side defaults filling in anything you omit. The complete list, their + defaults, and validation rules are documented on the [Endpoints](endpoints.md) page. + +## Reading a job's status and progress + +There are two ways to look at jobs. + +=== "One job in full" + + `GET /api/download/jobs/{job_id}` returns the complete detail for a single job, + including parameters (with secrets redacted), timing, output files, and error + information if it failed. + + ```shell + curl http://127.0.0.1:8786/api/download/jobs/b1e6c7de-6d0a-4f2f-9d1e-3a2b4c5d6e7f \ + -H "X-Secret-Key: your-api-secret" + ``` + +=== "All jobs (summaries)" + + `GET /api/download/jobs` lists every job the manager currently holds. By default + each entry is a **summary**. Pass `full=true` to get the same full detail as the + single-job endpoint for every job. + + ```shell + curl "http://127.0.0.1:8786/api/download/jobs?status=downloading&sort_by=progress&sort_order=desc" \ + -H "X-Secret-Key: your-api-secret" + ``` + + The list can be filtered by `status` and `service`, and ordered with `sort_by` + (`created_time`, `started_time`, `completed_time`, `progress`, `status`, or + `service`) and `sort_order` (`asc` / `desc`). See [Endpoints](endpoints.md) for the + query-parameter reference. + +### The progress fields + +While a job is `downloading`, the following fields update roughly twice a second so a +client can poll and render live progress. They are present on both the summary and the +full detail: + +| Field | Type | Meaning | +|---|---|---| +| `progress` | number `0` to `100` | Overall percent complete. | +| `phase` | string | Human-readable phase, e.g. `downloading video 1080p HDR10, audio en 5.1`. | +| `title` | string | Title being processed. | +| `current_title` | string | The specific episode/movie in progress (for a batch). | +| `completed_tracks` | int | Number of tracks finished so far. | +| `total_tracks` | int | Total tracks selected for the download. | +| `active_tracks` | list of strings | Labels of the tracks downloading right now. | +| `track_progress` | list | Per-active-track `{label, progress, speed}` entries. | +| `segments_done` | number | Segments downloaded for the track in progress. | +| `segments_total` | number | Total segments for the track in progress. | +| `speed` | string | Transfer speed of the track in progress, e.g. `4.20 MB/s`. | +| `skipped_subtitles` | list | Subtitles that were skipped as non-fatal failures (when `skip_subtitle_errors` is set). | + +The full detail adds `started_time`, `completed_time`, `output_files`, the redacted +`parameters`, and, for failed jobs, `error_message`, `error_details`, `error_code`, +`error_traceback`, and `worker_stderr`. + +!!! example "A typical polling loop" + ```shell + JOB=b1e6c7de-6d0a-4f2f-9d1e-3a2b4c5d6e7f + while true; do + curl -s "http://127.0.0.1:8786/api/download/jobs/$JOB" \ + -H "X-Secret-Key: your-api-secret" \ + | python -c 'import sys,json; d=json.load(sys.stdin); print(d["status"], d["progress"], d.get("phase"))' + sleep 1 + done + ``` + +!!! note "What the percentage means" + `progress` climbs from `0` toward `90` while tracks download, then the final stretch + to `100` covers post-download work such as decryption cleanup, repackaging, and + muxing. A job that has finished downloading but is still muxing can sit near `90` + for a moment before completing. On success the manager sets it to exactly `100`. + +!!! warning "For a batch, `progress` is per-episode, not whole-batch" + A multi-episode request (a `wanted` range) is a **single job** that processes its + episodes sequentially. As each new episode begins, `progress` and the track counts + (`completed_tracks` / `total_tracks`) **reset to zero** and count up again for that + episode alone. The top-level bar therefore tracks the *current* episode, not the + batch as a whole. To derive true overall completion, compare the growing + `output_files` array against `parameters.wanted` rather than trusting `progress`. + +## Terminal states in detail + +### Completed + +A `completed` job has its `output_files` populated with the paths of the muxed +results, and `progress` is `100`. This is the state you are waiting for. + +### Failed + +A `failed` job carries the error. `error_message` is the short human message, +`error_code` is one of the API's structured error codes (for example `NETWORK_ERROR`, +`DRM_ERROR`, `AUTH_FAILED`, or `DOWNLOAD_ERROR`), and `error_details` gives more +context. When the server was started with `--debug-api`, `error_traceback` and +`worker_stderr` are also filled in for diagnosis. + +!!! tip "Secrets are scrubbed from error text" + A service can echo a credential or a proxy URL into its output. The server masks + any known secret values out of `error_message`, `error_details`, `error_traceback`, + and `worker_stderr` before returning them, so it is safe to log these fields. + +The error-code catalogue and how each maps to an HTTP status is on the +[Error Responses](errors.md) page. + +### Cancelled + +A `cancelled` job was stopped on request before finishing. It may have partial output, +but it is not considered complete. See below. + +## Cancelling a job + +Send `DELETE /api/download/jobs/{job_id}`. The behaviour depends on the job's current +state, and this is important because the response differs: + +| Job state when you DELETE | What happens | Response | +|---|---|---| +| `queued` or `downloading` | The job is cancelled. A running download's worker process is terminated. | `200` with `{"status": "success", "message": "Job cancelled"}` | +| `completed`, `failed`, or `cancelled` (terminal) | The job is **removed** from the manager entirely. | `204 No Content`, empty body | +| Not found | — | `404` with error code `JOB_NOT_FOUND` | + +In other words, `DELETE` on a live job *cancels* it, while `DELETE` on an already +finished job *deletes it from the list*. This dual meaning is deliberate: one verb +both stops in-flight work and tidies up afterwards. + +!!! warning "Do not parse the body of a 204" + Removing a terminal job returns `204 No Content` with an empty body. A client must + not try to decode JSON from it; check the status code first. + +```shell title="Cancel a running job" +curl -X DELETE http://127.0.0.1:8786/api/download/jobs/b1e6c7de-6d0a-4f2f-9d1e-3a2b4c5d6e7f \ + -H "X-Secret-Key: your-api-secret" +``` + +To remove *all* terminal jobs at once, use `POST /api/download/jobs/clear-finished`, +which returns `{"removed": <count>}`. + +## Retrying a job + +`POST /api/download/jobs/{job_id}/retry` re-runs a finished job by creating a **new** +job with the same service, title, and parameters. The original job is untouched and +stays in its terminal state; the response gives you the new job's ID with `202`. + +- Only terminal jobs can be retried. Retrying a `queued` or `downloading` job returns + `409` with error code `CONFLICT`. +- A missing job returns `404` (`JOB_NOT_FOUND`). +- The service allowlist and any download gates are re-checked for the new job, exactly + as they were for the original submission, so a retry can be rejected with + `INVALID_SERVICE` or `FORBIDDEN` if permissions changed. + +```json title="202 Accepted, a fresh job" +{ + "job_id": "c2f7d8ef-7e1b-5a3a-ae2f-4b3c5d6e7f80", + "status": "queued", + "created_time": "2026-07-03T12:30:00.000000" +} +``` + +## Prioritising a job + +`POST /api/download/jobs/{job_id}/priority` moves a **queued** job to the front of the +line, so it will be the next one a free worker picks up. + +- The job must be `queued`. Prioritising a job in any other state returns `409` + (`CONFLICT`); a missing job returns `404`. +- On success the response is `{"job_id": "<id>", "position": "front"}`. + +This only reorders the queue; the job's status stays `queued` until a worker starts +it. + +## History and retention + +Every job that reaches a terminal state is also written once to a persistent history +log, so it survives even after the job is removed from the live list. + +- Read it with `GET /api/history` (newest first; supports `limit` and `service` + filters). +- Delete a single entry with `DELETE /api/history/{job_id}` (returns `204`). + +The live in-memory jobs are separate from this on-disk history. A background cleanup +task periodically drops old terminal jobs from the live list based on a retention +period; the history file keeps them for later reference. See [Endpoints](endpoints.md) +for the history response shape. + +!!! note "Why the history file lives under the cache directory" + `api_history.jsonl` is placed under the cache directory on purpose, so it stays out + of the config/data tree. It is disposable runtime state, not user configuration or + curated data, which is exactly why clearing the cache is a safe operation against + it. + +--- + +## Under the hood: the worker/manager model + +!!! info "Developer section" + Everything below describes server internals. It is useful for contributors and for + anyone debugging the server, but it is not needed to *use* the API. + +### The manager and its queue + +A single process-wide **download queue manager** owns all jobs. It holds them in +memory keyed by ID, backs the wait list with a FIFO queue, and keeps a separate +priority deque for jobs that have been bumped to the front. On the first job, it starts +a fixed pool of worker tasks plus one cleanup task; starting is idempotent, so later +jobs reuse the same pool. + +- **Concurrency** is capped by `max_concurrent_downloads` (default **2**). That many + jobs can be in `downloading` at once; the rest wait in `queued`. +- **Retention** is governed by `job_retention_hours` (default **24**). An hourly + cleanup task removes terminal jobs whose completion time (or, if never started, + creation time) is older than the retention window. + +!!! note "Two concurrency knobs that are easy to conflate" + `max_concurrent_downloads` caps how many separate download **jobs** run at once. + The separate `serve.downloads` setting controls how many **tracks** download in + parallel *within* a single job. They are independent, and raising one does not affect + the other. + +!!! note "Concurrency and retention config keys" + The manager reads these as top-level config attributes + (`config.max_concurrent_downloads` and `config.download_job_retention_hours`). + Those keys are not part of the standard config schema, so in practice the built-in + defaults of **2** and **24** apply. The live values are surfaced by + [`GET /api/config`](endpoints.md). The related `serve.history_limit` (default + **100**) instead controls how many entries the on-disk history file keeps. + +Each worker loop drains the priority deque first, then the FIFO queue. A job that was +promoted has its stale FIFO copy skipped so it does not run twice. When a worker takes +a job it flips it to `downloading`, stamps `started_time`, runs it, and on finish +stamps `completed_time` and records history. + +### Each job runs in its own subprocess + +The manager does **not** run the download in the server's event loop or even in the +server process. For each job it spawns a separate Python subprocess: + +```text +python -m unshackle.core.api.download_worker <payload.json> <result.json> <progress.json> +``` + +Three temporary files bridge the parent and the worker: + +- **`payload.json`**: written by the manager before launch; contains the job ID, + service, title ID, and merged parameters. +- **`progress.json`**: written continuously by the worker as the download proceeds. +- **`result.json`**: written by the worker at the end. + +Running each download out-of-process isolates the server from crashes, memory growth, +and blocking work inside the download engine, and makes hard cancellation (killing the +process) reliable. + +The worker's exit codes are `0` for success, `1` for a download error, and `2` for bad +arguments. Its `result.json` is either `{"status": "success", "output_files": [...]}` +or `{"status": "error", "message", "error_details", "error_code", "traceback"}`. If the +process exits non-zero, or the result is missing or not a success, the parent marks the +job `failed` and adopts the worker's `error_code` (or derives one by categorising the +exception). + +### How progress reaches the API + +While the worker runs, the manager polls `progress.json` every **0.5 s** and copies the +fields it finds onto the live `DownloadJob`. That is what the `GET` endpoints return. + +Inside the worker, progress is aggregated across all selected tracks. Each track's +completion fraction is weighted by its bitrate, so video and audio dominate the bar and +subtitles barely move it (subtitles get a small fixed weight). Track fractions are kept +**monotonic**: the percentage only ever climbs. The download portion is scaled to fill +`0` to `90`; the remaining `90` to `100` is driven by the download engine as it repackages +and muxes. A track counts as done when it reaches a terminal state such as +`Downloaded`, `Decrypted`, or a skipped subtitle. + +### How cancellation actually works + +Each job carries a cancellation event. When you `DELETE` a live job: + +- If it is **`queued`**, it flips straight to `cancelled`, its history is recorded + immediately (it never reaches a worker), and the worker loop skips it if it later + pops it off the queue. +- If it is **`downloading`**, the manager sets the cancellation event and flips the job + to `cancelled`. The polling loop notices the event, `terminate()`s the worker + subprocess (escalating to `kill()` if it does not exit within 5 seconds), and raises + a cancellation so the task unwinds. The subprocess model is what makes this a clean, + immediate stop rather than a cooperative one. + +### Secret handling in serialized jobs + +Job parameters may carry secrets: a raw `user:pass` credential, or a proxy URL with +embedded userinfo. Whenever the manager serializes a job (for an API response or the +history file), it masks the sensitive keys (`credential`, `credentials`, `password`, +`token`, `api_key`) and strips userinfo from proxy URLs. Free-text fields that a +service might have leaked a secret into (`error_message`, `error_details`, +`error_traceback`, `worker_stderr`, and worker stdout/stderr in the logs) are scrubbed +of the job's known secret values, longest-match first, before they are ever returned or +logged. + +## See also + +- [Endpoints](endpoints.md): the full request/response reference for every job endpoint. +- [Error Responses](errors.md): the error envelope, codes, and HTTP statuses a failed job reports. +- [Authentication](authentication.md): the `X-Secret-Key` header and per-key service allowlists that gate job creation and retry. +- [Downloading](../../guide/downloading.md): the equivalent options on the `dl` command line. diff --git a/docs/dev/rest-api/quickstart.md b/docs/dev/rest-api/quickstart.md new file mode 100644 index 00000000..f0f02323 --- /dev/null +++ b/docs/dev/rest-api/quickstart.md @@ -0,0 +1,437 @@ +# REST API Quick Start + +Unshackle ships with a built-in REST API that exposes the same download engine you drive from the command line. It is served by the `unshackle serve` command and lets you queue downloads, poll their progress, browse services, and manage history over HTTP instead of the terminal. + +The full loop runs end to end: start the server, authenticate, submit a download job, and poll it to completion, using real `curl` commands built directly from the API routes. + +!!! note "Who this page is for" + This is a **developer** guide. It assumes you are comfortable with HTTP, JSON, and a tool like `curl`. If you just want to download something, the [command line](../../index.md) is simpler. The REST API is meant for building tools, UIs, and automation on top of unshackle. + +--- + +## Before you begin + +You need a working unshackle install with services and CDM devices already configured, exactly as you would for a normal command-line download. If `unshackle dl` works for you, the API will too, since it runs the identical download pipeline. + +=== "Installed as a tool" + + ```shell + uv tool install git+https://github.com/unshackle-dl/unshackle.git + unshackle --help + ``` + +=== "From a clone" + + ```shell + uv run unshackle --help + ``` + +--- + +## Step 1: Configure an API secret + +Every request (except the health check) is authenticated with a secret key sent in the `X-Secret-Key` header. The server reads this key from the `serve` section of your `unshackle.yaml` config. + +At minimum you must set `api_secret`: + +```yaml title="unshackle.yaml" +serve: + api_secret: "choose-a-long-random-secret" +``` + +If you want the API to be able to license and download DRM-protected content, you also need to grant CDM devices to the key. Configure that under `serve.users`, keyed by the secret: + +```yaml title="unshackle.yaml" +serve: + api_secret: "choose-a-long-random-secret" + users: + choose-a-long-random-secret: + devices: ["your_widevine_device"] # Widevine .wvd stems + playready_devices: ["your_playready_device"] # PlayReady .prd stems + username: api_user +``` + +!!! warning "The secret must exist" + If `api_secret` is not set and you do **not** pass `--no-key`, `unshackle serve` refuses to start and raises an error. Treat the secret like a password. Anyone who has it can queue downloads on your machine. + +!!! tip "Disabling auth for local testing" + Passing `--no-key` to `serve` turns authentication off entirely: every request is allowed and the `X-Secret-Key` header is ignored. This is convenient for a throwaway local test, but never expose a `--no-key` server to a network you do not fully trust. + +--- + +## Step 2: Start the server + +Start the REST API with the `serve` command: + +```shell +unshackle serve --api-only +``` + +By default the server binds to `127.0.0.1:8786`. The `--api-only` flag serves just the REST API and omits the Widevine/PlayReady CDM HTTP endpoints; drop it if you also want to expose those. + +Common options: + +| Option | Default | Effect | +| --- | --- | --- | +| `-h, --host` | `127.0.0.1` | Address to bind to | +| `-p, --port` | `8786` | Port to bind to | +| `--api-only` | off | Serve only the REST API, no CDM endpoints | +| `--no-key` | off | Disable API-key authentication (allow all requests) | +| `--debug-api` | off | Include tracebacks and stderr in error responses | +| `--debug` | off | Enable DEBUG-level logging | +| `--remote-only` | off | Expose only the remote-session subset of routes | +| `--caddy` | off | Also launch Caddy as a reverse proxy (for HTTPS) | + +Once it is up you will see the API mounted under `/api/` and, unless you started in remote-only mode, an interactive Swagger UI: + +```text +REST API endpoints available at http://127.0.0.1:8786/api/ +Swagger UI available at http://127.0.0.1:8786/api/docs/ +``` + +!!! note "Swagger UI is also behind auth" + In `--api-only` mode (without `--no-key`), the Swagger UI at `/api/docs/` requires the `X-Secret-Key` header just like every other route. Only `/api/health` is exempt from authentication. + +For the rest of this page we will assume the server is reachable at `http://127.0.0.1:8786` and store the secret in a shell variable: + +```shell +export SECRET="choose-a-long-random-secret" +export BASE="http://127.0.0.1:8786" +``` + +--- + +## Step 3: Confirm the server is alive + +The health endpoint is the one route that never requires a key. Use it to confirm the server is running and to read its version: + +```shell +curl "$BASE/api/health" +``` + +```json title="200 OK" +{ + "status": "ok", + "version": "5.3.0", + "update_check": { + "update_available": false, + "current_version": "5.3.0", + "latest_version": "5.3.0" + } +} +``` + +The `update_check` block is best effort. If unshackle cannot reach the update source, `update_available` and `latest_version` come back as `null` rather than failing the request. + +--- + +## Step 4: Authenticate a real request + +Every other endpoint needs the `X-Secret-Key` header. A quick way to prove your key works is to list the services the server can see: + +```shell +curl "$BASE/api/services" \ + -H "X-Secret-Key: $SECRET" +``` + +```json title="200 OK (truncated)" +{ + "services": [ + { + "tag": "EXAMPLE", + "aliases": ["EX"], + "geofence": [], + "title_regex": null, + "url": "https://example.com", + "help": "EXAMPLE service documentation...", + "needs_auth": true, + "has_search": true, + "has_drm": true, + "auth_methods": ["cookies", "credentials"], + "cli_params": [] + } + ] +} +``` + +If the key is missing or wrong, you get a `401` with a plain body: + +```json title="401 Unauthorized" +{ "status": 401, "message": "Secret Key is Invalid." } +``` + +!!! warning "Two error shapes exist" + The authentication middleware returns the short `{"status": 401, "message": "..."}` shape shown above. Every **other** error the API produces uses the richer structured shape (`error_code`, `timestamp`, `details`, …) described in [Error handling](#error-handling). A robust client should handle both. + +The `tag` value for the service you want is the identifier you pass as `service` in the calls below. + +--- + +## Step 5: Submit a download job + +Downloads are asynchronous. You `POST` a job to `/api/download`, the server immediately hands back a `job_id`, and the actual download runs in the background so your request never blocks. + +The only required fields are `service` and `title_id`. Everything else mirrors the flags of the `dl` command and has a sensible default, so a minimal job looks like this: + +```shell +curl -X POST "$BASE/api/download" \ + -H "X-Secret-Key: $SECRET" \ + -H "Content-Type: application/json" \ + -d '{ + "service": "EXAMPLE", + "title_id": "12345" + }' +``` + +A more realistic job pins the quality and codecs and asks for specific episodes: + +```shell +curl -X POST "$BASE/api/download" \ + -H "X-Secret-Key: $SECRET" \ + -H "Content-Type: application/json" \ + -d '{ + "service": "EXAMPLE", + "title_id": "12345", + "quality": [1080], + "vcodec": "H265", + "range": ["SDR"], + "wanted": ["S01E01", "S01E02"] + }' +``` + +On success the server responds with **`202 Accepted`** and the identifier you will poll: + +```json title="202 Accepted" +{ + "job_id": "3f9c1e0a-8d2b-4e6f-9a1c-77b0f2a1c4d5", + "status": "queued", + "created_time": "2026-07-03T12:00:00.000000+00:00" +} +``` + +Capture the `job_id` for the next step: + +```shell +JOB=$(curl -s -X POST "$BASE/api/download" \ + -H "X-Secret-Key: $SECRET" \ + -H "Content-Type: application/json" \ + -d '{"service":"EXAMPLE","title_id":"12345","quality":[1080]}' \ + | python -c "import sys, json; print(json.load(sys.stdin)['job_id'])") +``` + +### Frequently used download fields + +The request body accepts the full download parameter set. A few of the most useful: + +| Field | Default | Meaning | +| --- | --- | --- | +| `service` | — | Service tag (required) | +| `title_id` | — | Title identifier (required) | +| `profile` | `null` | Named credential/cookie profile to use | +| `quality` | `[]` | Resolution(s), e.g. `[1080]`; empty means best available | +| `vcodec` | `null` | Video codec, e.g. `"H265"` | +| `acodec` | `null` | Audio codec, e.g. `"EC3"` | +| `range` | `["SDR"]` | Dynamic range: `SDR`, `HDR10`, `HDR10+`, `DV`, `HLG`, `HYBRID` | +| `wanted` | `[]` | Episodes to fetch, e.g. `["S01E01"]`; empty means all | +| `lang` | `["orig"]` | Video/audio language(s) | +| `s_lang` | `["all"]` | Subtitle language(s) | +| `downloads` | `1` | Number of tracks to download concurrently | +| `proxy` | `null` | Proxy URI or country code | + +Validation is strict: invalid codecs, subtitle formats, dynamic-range names, non-positive counts, or mutually exclusive flags (for example both `video_only` and `audio_only`) come back as `400` with error code `INVALID_PARAMETERS`. See the full reference on the [download reference page](endpoints.md) for every accepted field. + +!!! note "Gated fields" + Passing a per-request CDM (`cdm`) or inline credentials (`credential` / `credentials`) is refused with `403 FORBIDDEN` unless the server operator has explicitly opted in via `serve.cdm_overrides` and `serve.allow_job_credentials`. Both are off by default. + +--- + +## Step 6: Poll the job to completion + +A job moves through these states: + +```text +queued → downloading → completed | failed | cancelled +``` + +The last three are **terminal**. Poll the job detail endpoint until you reach one of them. This endpoint always returns the full job record: + +```shell +curl "$BASE/api/download/jobs/$JOB" \ + -H "X-Secret-Key: $SECRET" +``` + +While the download is running you will see `status: "downloading"` and a `progress` value climbing from `0` toward `100`: + +```json title="200 OK (in progress)" +{ + "job_id": "3f9c1e0a-8d2b-4e6f-9a1c-77b0f2a1c4d5", + "status": "downloading", + "progress": 42.7, + "phase": "downloading", + "service": "EXAMPLE", + "title_id": "12345", + "title": "Example Show S01E01", + "current_title": "Example Show S01E01", + "completed_tracks": 1, + "total_tracks": 3, + "segments_done": 512, + "segments_total": 1200, + "speed": "8.4 MiB/s", + "created_time": "2026-07-03T12:00:00.000000+00:00", + "started_time": "2026-07-03T12:00:03.000000+00:00" +} +``` + +When it finishes you will see `status: "completed"` and the produced files under `output_files`: + +```json title="200 OK (completed)" +{ + "job_id": "3f9c1e0a-8d2b-4e6f-9a1c-77b0f2a1c4d5", + "status": "completed", + "progress": 100.0, + "output_files": ["/path/to/downloads/Example Show S01E01.mkv"], + "completed_time": "2026-07-03T12:04:31.000000+00:00" +} +``` + +A simple polling loop in shell: + +```shell title="poll.sh" +while true; do + STATUS=$(curl -s "$BASE/api/download/jobs/$JOB" \ + -H "X-Secret-Key: $SECRET" \ + | python -c "import sys, json; print(json.load(sys.stdin)['status'])") + echo "job status: $STATUS" + case "$STATUS" in + completed|failed|cancelled) break ;; + esac + sleep 3 +done +``` + +!!! tip "Polling cadence" + A few seconds between polls is plenty. The server writes progress updates roughly twice a second internally, but there is no push/streaming channel; you read the current snapshot each time you ask. + +If a job ends in `failed`, the full record includes `error_message`, `error_code`, and (when the server was started with `--debug-api`) `error_traceback` and `worker_stderr` to help you diagnose it. Sensitive values in these fields are scrubbed before they are returned. + +--- + +## Step 7: Clean up + +Once you have collected the output you can remove the finished job. `DELETE` on a job has two behaviors depending on its state: + +```shell +curl -X DELETE "$BASE/api/download/jobs/$JOB" \ + -H "X-Secret-Key: $SECRET" +``` + +- If the job is **terminal** (completed/failed/cancelled) it is removed from the manager and you get **`204 No Content`** with an empty body. +- If the job is still **queued or downloading** it is cancelled instead, and you get `200` with `{"status": "success", "message": "Job cancelled"}`. + +!!! warning "Do not parse JSON on 204" + A `204` response has no body. This applies to terminal-job removal here and to deleting history entries; check the status code before attempting to decode JSON. + +To clear every finished job at once instead of one at a time: + +```shell +curl -X POST "$BASE/api/download/jobs/clear-finished" \ + -H "X-Secret-Key: $SECRET" +``` + +```json title="200 OK" +{ "removed": 3 } +``` + +--- + +## Managing the queue + +A handful of endpoints round out job control. All require the `X-Secret-Key` header. + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/api/download/jobs` | List all jobs, with optional filtering and sorting | +| `GET` | `/api/download/jobs/{job_id}` | Full detail for one job | +| `DELETE` | `/api/download/jobs/{job_id}` | Cancel a running job, or remove a terminal one | +| `POST` | `/api/download/jobs/{job_id}/retry` | Re-queue a new job from a terminal one | +| `POST` | `/api/download/jobs/{job_id}/priority` | Move a queued job to the front of the queue | +| `POST` | `/api/download/jobs/clear-finished` | Remove all terminal jobs | + +Listing supports query parameters for filtering by `status` or `service`, sorting via `sort_by` (`created_time`, `started_time`, `completed_time`, `progress`, `status`, `service`) and `sort_order` (`asc` / `desc`), and requesting full per-job detail with `full=true`: + +```shell +curl "$BASE/api/download/jobs?status=downloading&sort_by=progress&sort_order=desc" \ + -H "X-Secret-Key: $SECRET" +``` + +```json title="200 OK" +{ + "jobs": [ + { + "job_id": "3f9c1e0a-8d2b-4e6f-9a1c-77b0f2a1c4d5", + "status": "downloading", + "service": "EXAMPLE", + "title_id": "12345", + "title": "Example Show S01E01", + "progress": 42.7, + "phase": "downloading", + "completed_tracks": 1, + "total_tracks": 3 + } + ] +} +``` + +Retrying reuses the original job's service, title, and parameters to enqueue a brand new job (returning `202` with a fresh `job_id`); it only works on terminal jobs and returns `409 CONFLICT` otherwise. Prioritizing only works on jobs still in the `queued` state. + +--- + +## Beyond a single download + +The download loop above is the core of the API, but the same server exposes more: + +- **Discovery**: `POST /api/search` finds titles by query, and `POST /api/list-titles` / `POST /api/list-tracks` inspect what a title offers before you commit to downloading it. See [list titles and tracks](endpoints.md). +- **History**: `GET /api/history` reads a persisted log of jobs that reached a terminal state (newest first), and `DELETE /api/history/{job_id}` removes a single entry. See [history](endpoints.md). +- **Introspection**: `GET /api/profiles` lists configured credential profiles per service, `GET /api/config` returns a redacted view of the effective server config, and `GET /api/env/check` reports which external binaries are installed. +- **Maintenance**: `POST /api/maintenance/clear-cache`, `.../clear-temp`, and `.../refresh-services` perform housekeeping (the cache/temp clears are blocked with `409` while a download is active). +- **Remote sessions**: the `/api/session/*` endpoints drive the thin-client "remote download" flow, where the server authenticates and licenses on behalf of a local client. See [remote sessions](remote-sessions.md). + +The interactive **Swagger UI at `/api/docs/`** documents every route and lets you try them in the browser (send your `X-Secret-Key` there too). + +--- + +## Error handling + +Apart from the auth middleware's short form, every error the API returns uses a single structured envelope: + +```json +{ + "status": "error", + "error_code": "INVALID_PARAMETERS", + "message": "Human-readable description", + "timestamp": "2026-07-03T12:00:00Z", + "details": { } +} +``` + +`details` appears only when there is something to add, `retryable: true` appears only for transient failures, and a `debug_info` block (exception type and traceback) is included only when the server runs with `--debug-api`. + +The `error_code` maps to an HTTP status. The ones you will meet most often: + +| Error code | HTTP | Typical cause | +| --- | --- | --- | +| `INVALID_INPUT` | 400 | Malformed JSON body or missing required field | +| `INVALID_SERVICE` | 400 | Unknown or disallowed service tag | +| `INVALID_PARAMETERS` | 400 | A download field failed validation | +| `AUTH_FAILED` | 401 | Service login/credential failure | +| `FORBIDDEN` | 403 | A gated field was used without server opt-in | +| `NOT_FOUND` | 404 | Title or resource does not exist | +| `JOB_NOT_FOUND` | 404 | No job with that `job_id` | +| `CONFLICT` | 409 | Illegal state transition (e.g. retry a running job) | +| `RATE_LIMITED` | 429 | Upstream throttling (retryable) | +| `INTERNAL_ERROR` | 500 | Unexpected server-side failure | +| `SERVICE_ERROR` | 502 | The service raised an error | +| `NETWORK_ERROR` | 503 | Connection/timeout reaching the service (retryable) | + +!!! tip "Retry politely" + Codes marked retryable (`NETWORK_ERROR`, `RATE_LIMITED`, `SERVICE_UNAVAILABLE`) reflect transient conditions. Back off and retry rather than hammering the server. diff --git a/docs/dev/rest-api/remote-sessions.md b/docs/dev/rest-api/remote-sessions.md new file mode 100644 index 00000000..a65905bf --- /dev/null +++ b/docs/dev/rest-api/remote-sessions.md @@ -0,0 +1,431 @@ +# Remote Service Sessions + +A **remote service session** lets one machine do the hard, service-specific work +(authenticating to a streaming service, listing titles, resolving tracks and +manifests, and, optionally, DRM licensing) while another machine runs the actual +download, decryption, and muxing locally. + +This is the mechanism behind unshackle's client/server "remote download" workflow. +A thin local client (`RemoteService`) talks to an unshackle `serve` instance over +HTTP. The server keeps an authenticated service instance alive between requests so +the client can authenticate once and then make many follow-up calls against the +same session. + +!!! note "Why you'd use this" + - The server has service accounts, cookies, or a CDM you don't want to copy to + every machine. + - You want to run downloads on a fast/local machine but keep credentials and + licensing centralized. + - A service is region-locked to where the server lives, and the client isn't. + +--- + +## The big picture + +``` +┌──────────────────────────┐ HTTP (X-Secret-Key) ┌──────────────────────────┐ +│ Local machine (client) │ ───────────────────────────────▶ │ unshackle serve │ +│ │ │ │ +│ unshackle dl --server │ POST /api/session/create │ authenticated Service │ +│ │ GET /api/session/{id}/titles │ instance, kept alive in │ +│ • track selection │ POST /api/session/{id}/tracks │ an in-memory session │ +│ • download / decrypt │ POST /api/session/{id}/segments │ store (TTL-based) │ +│ • mux │ POST /api/session/{id}/license │ │ +│ │ DELETE /api/session/{id} │ │ +└──────────────────────────┘ └──────────────────────────┘ +``` + +The client implements the same interface `dl` expects from a normal service, but +every service-facing method proxies to the server. Everything after track +selection (downloading segments, decrypting, and muxing) runs locally. + +!!! note "Migration: `remote_dl` is gone" + The `RemoteService` adapter lives inside the ordinary `dl` command and replaces + the old standalone `remote_dl` command. If you have prior scripts or docs that + invoke `remote_dl`, use `dl` with a configured remote server instead. The + functionality now rides on `dl` via `RemoteService`. + +For the auth-facing endpoints, header format, and error shapes referenced below, +see [Authentication](authentication.md) and the endpoint reference in +[the REST API index](index.md). + +--- + +## Configuring a remote server (client side) + +On the **client**, add a `remote_services` block to your `unshackle.yaml`. Each +entry names a server and provides its URL and API key. + +```yaml title="unshackle.yaml (client)" +remote_services: + my_server: + url: "https://my-box.example:8786" + api_key: "your-api-key" + # Optional: let the server's CDM do the licensing instead of a local CDM + server_cdm: false + # Optional: per-service config overrides applied locally (title_map, cdm, etc.) + services: + AMZN: + title_map: + "0ABCDEF": "The Show (Renamed Locally)" +``` + +| Key | Required | Meaning | +|---|---|---| +| `url` | yes | Base URL of the `unshackle serve` instance (no trailing slash needed) | +| `api_key` | no | Sent as the `X-Secret-Key` header on every request | +| `server_cdm` | no | When `true`, DRM keys are resolved by the server's CDM rather than a local device (default `false`) | +| `services` | no | Per-service local config overrides, keyed by service tag | + +!!! warning "Port default" + The `unshackle serve` default port is **`8786`** (not `8080`). Use the port + your server actually binds. Some older example snippets show `8080`, which is + not the default. + +### Selecting a server + +If exactly one server is configured, unshackle uses it implicitly: + +```bash +unshackle dl AMZN 0ABCDEF +``` + +If **more than one** server is configured, you must pick one with `--server`: + +```bash +unshackle dl --server my_server AMZN 0ABCDEF +``` + +With no `remote_services` configured at all, unshackle raises a clear error telling +you to add the block shown above. + +--- + +## What happens during a download + +From your point of view the command looks like an ordinary download. Under the +hood the client walks a session through its lifecycle. + +=== "1. Create + authenticate" + + The client calls `POST /api/session/create`, forwarding whatever it can so the + server doesn't have to prompt: + + - Local **credentials** for the service/profile (`{username, password, extra?}`) + - Local **cookies**, compressed and base64-encoded + - A resolved **proxy**, or, if you didn't set one, your detected + **client region** so the server can auto-proxy to match + - Track-selection hints (`range_`, `vcodec`, `quality`, `best_available`) so + the server fetches the right manifests + - Your local per-service **cache files** (e.g. refreshed tokens) + + The server responds immediately with a session ID and a status. Authentication + runs in the background on the server. + +=== "2. Interactive prompts (if needed)" + + Some services need a one-time code, PIN, or device confirmation. When the + server's auth flow asks for input, the session enters `pending_input`. The + client polls, displays the prompt to you locally, collects your answer, and + sends it back. The server's auth thread resumes with your response. + + ```text + Enter the 6-digit code sent to your email: 483920 + ``` + +=== "3. Titles + tracks" + + Once authenticated, the client fetches the title list, then the tracks for the + chosen title. The tracks come back with playback URLs, and DASH/ISM manifests + are shipped as compressed XML so the client can re-parse them locally for + downloading. Any `session_headers` / `session_cookies` the server used are + merged into the client's local HTTP session. + +=== "4. Licensing" + + For DRM content the client either **proxies** its CDM challenge through the + server, or, if `server_cdm: true`, asks the server to run the full CDM flow + and return the content keys directly. See + [Server-side vs. proxied CDM](#server-side-vs-proxied-cdm) below. + +=== "5. Download + close" + + The client downloads, decrypts, and muxes locally. On completion it deletes + the session. If the server has updated cache files (e.g. a refreshed token), + they're returned on delete and saved locally so the **next** session can skip + interactive auth. + +!!! tip "Renaming remote titles locally" + You can rename titles for a remote service you don't have installed locally by + adding a `title_map` under that service in your client `remote_services.<name>.services` + config. The server sends raw titles; your local map wins. + + This is deliberate: the server does **no** `title_map` remapping of its own and + sends titles exactly as the service returns them. All remapping happens on the + client, applied to the titles the server sends back. It keeps the final output + name fully under the client's control and lets you rename titles even for + services you have no local install of. + +--- + +## Server-side vs. proxied CDM + +There are two ways DRM keys get resolved, chosen by the client's `server_cdm` flag. + +=== "Proxied CDM (default)" + + Your **local CDM** builds the license challenge. The client sends that challenge + to `POST /api/session/{id}/license`, the server forwards it to the service, and + the raw license bytes come back for your local CDM to parse. + + - Keeps your CDM local; the server just relays the license request. + - Used when `server_cdm` is `false` (the default). + +=== "Server-side CDM (server_cdm: true)" + + The **server's CDM** does everything. The client sends track IDs (or a PSSH), + the server checks its key vaults, loads the device configured for your API key, + runs the CDM flow, and returns `KID:KEY` pairs directly. No CDM is needed on the + client. + + - Enable with `server_cdm: true` in the server entry. + - The server tells the client which DRM type it actually used + (`widevine` or `playready`). + +--- + +## Session lifecycle and expiry + +Sessions live in an **in-memory store on the server** and expire on a timer. You +generally never touch this directly, but it explains behavior you might observe. + +| Behavior | Value | Notes | +|---|---|---| +| Idle session TTL | **300s** (5 min) default | Refreshed on every request to the session | +| Max concurrent sessions | **100** default | Oldest (least recently used) is evicted when full | +| Auth/input timeout | **600s** (10 min) | Sessions still authenticating or awaiting a prompt use this longer window instead of the TTL | +| Cleanup sweep | every 60s | Expired sessions are removed and their input prompts cancelled | + +!!! note "Auth is never rushed" + A session that is still `authenticating` or waiting on `pending_input` is **not** + subject to the short 300s TTL; it gets the full 600s auth window. This gives you + time to enter an interactive code without the session vanishing mid-prompt. + +The TTL and max-session limits are read from server config: + +```yaml title="unshackle.yaml (server)" +serve: + session_ttl: 300 # seconds a session may sit idle + max_sessions: 100 # cap on concurrent sessions +``` + +--- + +## Security properties worth knowing + +- **API key required.** Every session request carries `X-Secret-Key`; requests + without a valid key are rejected (health check aside). See + [Authentication](authentication.md). +- **IP binding.** A session records the creator's IP at create time. If a later + request for the same session comes from a different IP, the server returns + `403 FORBIDDEN`. Sessions are not portable between hosts. +- **Namespaced, isolated cache.** Each session gets its own cache directory, + namespaced by a hash of the API key and the session ID, so sessions can't read + each other's cached tokens. The directory is deleted when the session ends. +- **Secrets are scrubbed from logs.** Session IDs, service tags, and other user + values are sanitized before logging. + +--- + +## Developer reference + +!!! info "This section is for developers" + The material below documents the server-side implementation and the HTTP + contract. End users configuring a client can stop at the sections above. + +### Session endpoints + +All routes are exposed even in `--remote-only` server mode. Paths use the +`session_id` returned by create. + +!!! note "Why `--remote-only` exists" + `--remote-only` narrows the server to just the health, services, search, and + session subset (and emits CORS headers) precisely so it is safe to sit behind + Cloudflare or serve cross-origin browser clients. That trimmed surface, rather + than the full `--api-only` mode, is what makes a CORS/Cloudflare-fronted + deployment practical: reach for it when the server is public-facing or accessed + from a browser origin, not when you simply want the local HTTP API. + +| Method | Path | Purpose | +|---|---|---| +| `POST` | `/api/session/create` | Build + authenticate a service instance; returns a session ID immediately | +| `GET` | `/api/session/{id}/titles` | Fetch and cache the title list | +| `POST` | `/api/session/{id}/tracks` | Resolve tracks, manifests, chapters, headers/cookies for a title | +| `POST` | `/api/session/{id}/segments` | Resolve per-segment/track download descriptors | +| `POST` | `/api/session/{id}/license` | DRM licensing (proxy or server CDM) | +| `GET` | `/api/session/{id}/prompt` | Poll interactive auth status / pending prompt | +| `POST` | `/api/session/{id}/prompt` | Submit an answer to a pending prompt | +| `GET` | `/api/session/{id}` | Session info (validity, TTL, counts) | +| `DELETE` | `/api/session/{id}` | Close the session, return updated cache, clean up | + +### Create: request and response + +`POST /api/session/create` requires `service` and `title_id`. It also accepts +`credentials`, `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, +`no_proxy`, `profile`, `cache` (a map of `filename → base64(zlib(bytes))`), +`client_region`, `cdm_type`, and the track-selection hints `range_`, `vcodec`, +`quality`, `best_available`, plus arbitrary service CLI options +(`additionalProperties: true`). + +The response returns **before** authentication finishes: + +```json +{ "session_id": "…uuid4…", "service": "AMZN", "status": "authenticating" } +``` + +Authentication runs on a background thread (`asyncio.to_thread(authenticate, …)`). +The session starts in `AUTHENTICATING`; the client must poll the prompt endpoint +until it reaches `authenticated` (or `failed`). + +### Interactive auth: the `InputBridge` + +When the service calls its input function during `authenticate()` on the server +thread, an `InputBridge` pauses that thread and exposes the prompt to the HTTP +layer. + +- `AuthStatus` values: `authenticating`, `pending_input`, `authenticated`, `failed`. +- `InputBridge.request_input(prompt, timeout=600)` blocks the sync auth thread on a + `threading.Event` until `submit_response()` or `cancel()` fires. +- A timeout raises `TimeoutError` and marks the session `FAILED`. +- `AUTH_INPUT_TIMEOUT = 600.0` seconds. This is also the TTL granted to + `AUTHENTICATING` / `PENDING_INPUT` sessions in the store. + +`GET /api/session/{id}/prompt` returns one of: + +```json +{ "status": "authenticated" } +{ "status": "authenticating" } +{ "status": "pending_input", "prompt": "Enter code: " } +{ "status": "failed", "error": "…message…" } +``` + +A missing session returns `404 SESSION_NOT_FOUND`; an IP mismatch returns +`403 FORBIDDEN`. `POST /api/session/{id}/prompt` takes `{ "response": "…" }` and +returns `{ "status": "accepted" }`; posting with no pending prompt is an +`INVALID_INPUT` error. + +### `SessionStore` internals + +Source: `unshackle/core/api/session_store.py`. A singleton obtained via +`get_session_store()`. + +- **Config-driven limits.** `serve.session_ttl` (default `300`) and + `serve.max_sessions` (default `100`) are read as properties, so config changes + take effect without recreating the store. +- **`create()`** evicts the least-recently-accessed session when at capacity, + then stores a new `SessionEntry` (defaulting `auth_status` to `AUTHENTICATED`; + the create handler overrides it to `AUTHENTICATING`). +- **`get()`** refreshes `last_accessed` via `touch()`. It returns `None` (and + deletes the entry) if an authenticated session has been idle longer than the TTL. + Sessions in `AUTHENTICATING` / `PENDING_INPUT` are exempt from TTL expiry. +- **`cleanup_expired()`** runs every 60s: authenticated sessions expire at `ttl`, + in-flight-auth sessions expire at `AUTH_INPUT_TIMEOUT`. Removing a session + cancels its `InputBridge` and deletes its cache directory (pruning empty parent + dirs up to, but not including, the cache root). +- **`cancel_all_bridges()`** is called on server shutdown to unblock any waiting + auth threads. + +`SessionEntry` fields: + +| Field | Description | +|---|---| +| `session_id` | UUID4 string | +| `service_tag` | Normalized service tag | +| `service_instance` | The authenticated service object kept alive between calls | +| `titles` / `title_map` | Result of `get_titles()` and a `title_id → Title` map | +| `tracks` / `tracks_by_title` / `chapters_by_title` | Cached resolved tracks and chapters | +| `creator_ip` | IP recorded at create time for IP-binding checks | +| `cache_tag` | Per-session cache directory tag | +| `input_bridge` | `InputBridge` for interactive auth, if any | +| `auth_status` / `auth_error` | Current `AuthStatus` and last error message | +| `created_at` / `last_accessed` | Timestamps; `last_accessed` drives TTL and LRU eviction | + +### Per-session cache namespacing + +The create handler builds a `Cacher` namespaced as: + +```text +_sessions/<pbkdf2_hmac(sha256, X-Secret-Key, "unshackle-session-ns", 100000)[:12]>/<session_id>/<service> +``` + +Forwarded `cache` files are written into that directory before authentication. +On `DELETE`, the handler harvests updated cache files (compressing each with zlib +and base64-encoding, **excluding** `titles_*` files) and returns them under a +`cache` key so the client can persist refreshed tokens: + +```json +{ "status": "ok", "cache": { "tokens": "…base64(zlib(bytes))…" } } +``` + +### Session info response + +`GET /api/session/{id}`: + +```json +{ + "session_id": "…", + "service": "AMZN", + "valid": true, + "expires_in": 300, + "track_count": 12, + "title_count": 40 +} +``` + +!!! warning "`expires_in` is the configured TTL, not time remaining" + The `expires_in` value reports the store's configured `session_ttl`, not the + seconds left before this specific session expires. + +### Reference client: `RemoteService` + +Source: `unshackle/core/remote_service.py`. This is the canonical consumer of the +session API and a good template for any client. + +- **`RemoteClient._request`** sets `X-Secret-Key` and `User-Agent: unshackle/<version>`, + uses a 120s timeout for `POST` and 30s for `GET`/`DELETE`, and treats any + `status_code >= 400` as fatal: it logs `Server error [<error_code>]: <message>` + and raises `SystemExit(1)`. +- **Retries.** The download-side HTTP session mounts an adapter with + `Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504])`. +- **Flow.** `authenticate()` → `create` (+ poll `prompt` every 2s up to a 600s + deadline, answering `pending_input` prompts) → `get_titles()` → `get_tracks()` + (merging returned `session_headers`/`session_cookies`, re-parsing `manifests`) → + license via proxy or `server_cdm` → `close()` (`DELETE`, saving any returned + `cache`). +- **Server resolution.** `resolve_server()` reads `config.remote_services.<name>` + into `{url, api_key, services, server_cdm}`, injecting the `server_cdm` flag into + the services map as `_server_cdm`. + +--- + +## Troubleshooting + +!!! example "\"Could not connect to remote server … Is it running?\"" + The client couldn't reach the URL. Confirm `unshackle serve` is running on the + server, the `url`/port in `remote_services` are correct (default port `8786`), + and any firewall or reverse proxy allows the connection. + +!!! example "`403 FORBIDDEN` mid-download" + The request came from a different IP than the one that created the session. + Sessions are IP-bound. Don't move between networks (or NAT egress IPs) during a + remote download. + +!!! example "Auth times out or the prompt never resolves" + Interactive auth allows up to 600s. If you miss that window the session is + marked `failed` and cleaned up; just re-run the download. If the server never + prompts you, check that credentials/cookies forwarded from the client are valid + for the service. + +!!! example "\"Multiple remote services configured. Use --server …\"" + You have more than one entry under `remote_services`. Pass `--server <name>` + to pick one. diff --git a/docs/dev/service-architecture.md b/docs/dev/service-architecture.md new file mode 100644 index 00000000..40f645cf --- /dev/null +++ b/docs/dev/service-architecture.md @@ -0,0 +1,452 @@ +# Service Architecture + +This page is the conceptual map of the internal object model that a **service author** +works against. It ties together the moving parts: the `Service` base class, the +`Title` hierarchy a service returns, the `Track` model those titles carry, the +manifest parsers that produce tracks, the DRM hooks that license and decrypt them, +and the request/session objects wired up for you at construction time. + +!!! info "Who this page is for" + This is a **developer reference**. If you only want to *run* downloads, you never + touch these classes directly; see [Downloading](../guide/downloading.md) and the + [CLI reference](../guide/cli-reference.md) instead. If you are writing or reading a + service plugin, this page explains what each object is and how the pieces connect. + For a guided, step-by-step build, see [Creating a Service](creating-a-service.md); + for exact signatures, see the auto-generated [Service API](../reference/api/service.md), + [Titles API](../reference/api/titles.md), and [Tracks API](../reference/api/tracks.md). + +## The shape of a download + +A service is a translator between one streaming platform and unshackle's generic +pipeline. It never downloads or decrypts anything itself; it just answers questions. +The `dl` command drives the whole flow and only ever calls three required methods on +your service, in order: + +```text +Service.get_titles → which movies / episodes / songs exist + │ +Service.get_tracks → fetch a manifest for one title … + │ +manifest.to_tracks → … parsed into Video / Audio / Subtitle tracks + │ +Track.download → fetch + merge segments (core-driven) + │ +DRM license + decrypt → service license hooks → CDM → decrypt tool + │ +Tracks.mux → mkvmerge → tagged .mkv +``` + +1. **`get_titles()`** returns a `Titles` collection: the movies, episodes, or songs + the platform exposes for the requested ID. +2. **`get_tracks(title)`** returns a `Tracks` object for a single chosen title. In + practice this means fetching a manifest and handing it to a parser, which builds + the `Video`/`Audio`/`Subtitle` objects. +3. **`get_chapters(title)`** returns any chapter markers. + +Everything after that (track selection, licensing, decryption, repacking, subtitle +conversion, and muxing) is handled by the core using the objects your service +returned. Your job is to produce accurate `Title` and `Track` objects; the framework +does the rest. + +## The `Service` base class + +Every service subclasses `unshackle.core.service.Service`, an abstract base class. +Two kinds of members matter: **class variables** that describe the service, and +**methods** that the pipeline calls. + +### Class variables + +Declared at the top of your service class, these are static descriptors: + +| Class variable | Type | Purpose | +|---|---|---| +| `ALIASES` | `tuple[str, ...]` | Alternate tags the service answers to (resolved by `Services.get_tag`). | +| `GEOFENCE` | `tuple[str, ...]` | ISO region codes the service requires; empty means no geofence. The first entry is treated as the "main region" for automatic proxy selection. | +| `VAULT_TAG` | `Optional[str]` | Overrides the key-vault namespace; defaults to the service's own tag. | +| `AUTH_METHODS` | `Optional[tuple[str, ...]]` | Accepted auth methods (`"cookies"` / `"credentials"`). When `None`, the REST `/services` endpoint infers them from `authenticate()`. | + +### Required methods + +These are abstract; a service will not load unless it implements all three: + +- `get_titles() -> Titles_T`: resolve the requested ID into a title collection. +- `get_tracks(title) -> Tracks`: build the track set for one title. +- `get_chapters(title) -> Chapters`: return chapter markers (may be empty). + +### Optional methods and hooks + +- **`authenticate(cookies=None, credential=None)`**: the base implementation loads a + `CookieJar` into `self.session.cookies` and stores `self.credential`. Override it to + perform a real login handshake. +- **`search()`**: raises `NotImplementedError` by default; implement it to support the + `search` command. +- **DRM license hooks**: `get_widevine_service_certificate`, `get_widevine_license`, + `get_playready_license` (delegates to Widevine by default), and + `get_clearkey_license` (returns `None` by default). See [DRM hooks](#drm-hooks) below. +- **`request_input(prompt)`**: prompts the user; routed through the serve-mode input + bridge or the rich console. +- **Event callbacks**: `on_segment_downloaded`, `on_track_downloaded`, + `on_track_decrypted`, `on_track_repacked`, `on_track_multiplex` let a service react to + pipeline events (for example, MonaLisa decrypts each segment in `on_segment_downloaded`). +- **`resolve_server_keys(title)`**: an escape hatch for services whose decryption keys are + resolved server-side *ahead of* the normal per-track license flow. Unlike everything else + here it is **not declared on the `Service` base class at all**: `dl.py` probes for it with + `hasattr` and calls it (once, after track selection) only if your service defines it. Use + it when keys must be fetched or injected in bulk before download, separate from the + per-track license callbacks below. + +### What `__init__` wires up for you + +You rarely override `__init__`; the base constructor takes the Click context and sets +up the objects your methods rely on: + +| Attribute | What it is | +|---|---| +| `self.config` | The per-service config slice (see [Request and session objects](#request-and-session-objects)). | +| `self.log` | A `logging.Logger` named after your class. | +| `self.session` | A ready-to-use HTTP session (headers, retries, and proxy applied). | +| `self.cache` | A generic key/value `Cacher` namespaced to the service. | +| `self.title_cache` | A region- and account-aware `TitleCacher`. | +| `self.cache_dir` | `config.directories.cache / <ServiceName>`. | +| `self.credential` | The active `Credential`, or `None` until authenticated. | +| `self.track_request` | A `TrackRequest` describing the requested codecs, ranges, and best-available flag (see below). | + +The constructor also resolves proxies: it reads `--proxy` / `--proxy-query` / +`--proxy-provider`, consults the per-service `proxy_map`, and, if `GEOFENCE` is set and +no explicit proxy was given, does a live IP check and auto-fetches a proxy to +`GEOFENCE[0]` when it detects a geoblock. See [Proxies and VPN](../guide/proxies-and-vpn.md). + +!!! note "`TrackRequest`" + `self.track_request` is a small dataclass built from the CLI: `codecs` + (from `--vcodec`, empty means "accept any"), `ranges` (from `--range`, default + `[Video.Range.SDR]`), and `best_available`. Use it in `get_tracks` to decide which + variants to fetch. + +## Titles and the `Title` hierarchy + +`get_titles()` returns a **title collection**, one of the `Titles_T` types. Each +collection holds one or more `Title` objects (`Title_T`). There are three parallel +families, one per media kind: + +| Media kind | Item (`Title_T`) | Collection (`Titles_T`) | +|---|---|---| +| Movies | `Movie` | `Movies` | +| TV | `Episode` | `Series` | +| Music | `Song` | `Album`, and `Music` (the base collection) | + +All items subclass `Title`, which carries the shared identity: a **unique `id`**, the +originating `service` class, an optional original `language`, arbitrary `data`, and a +freshly-constructed `self.tracks = Tracks()` that `get_tracks` will fill. Collections +are sorted lists, so titles stay in a stable order. + +!!! warning "The unique ID is validated" + `Title.__init__` rejects a falsy `id_`, and rejects any ID shorter than 4 characters + (`"The unique ID is not large enough, clash likely."`). Pick a stable, sufficiently + unique identifier from the platform. + +### Constructors at a glance + +Each subclass adds its own required fields on top of `id_` and `service`: + +=== "Movie" + + ```python + Movie(id_, service, name, year=None, language=None, data=None, description=None) + ``` + + `name` is required; `year` is optional but must be a positive integer if given. + +=== "Episode" + + ```python + Episode(id_, service, title, season, number, + name=None, year=None, language=None, data=None, + description=None, air_date=None) + ``` + + `title` is the **show** name; `season` and `number` are required (0 is allowed). + An `air_date` is normalized to a `date` so the `{date}` naming token stays clean, + and a placeholder episode `name` like `"Episode 4"` is discarded automatically. + +=== "Song" + + ```python + Song(id_, service, name, artist, album, track, disc, year, + language=None, data=None) + ``` + + `name`, `artist`, `album`, `track`, `disc`, and `year` are all required and validated. + +### Title remapping + +After titles are fetched (and after any cache), the framework applies +`remap_titles(titles, title_map)` using the per-service `title_map` config. This does an +**exact-string** rename (`Episode`s are matched on their show `title`, `Movie`s and +`Song`s on their `name`), so a platform's odd naming can be normalized to what your +library expects. Titles are also cache-wrapped via `get_titles_cached()`, which is region- +and account-aware; see the [TitleCacher](#request-and-session-objects) notes. + +The ordering here is deliberate on both sides. `title_map` is applied **after** the title +cache so that editing the map takes effect immediately, without forcing a cache reset, and +**before** metadata enrichment (`--enrich`) so that an explicit enrich override still wins +over a mapped name. + +## Tracks and the `Track` model + +A `Tracks` object is the container `get_tracks` returns. It has typed buckets +(`videos`, `audio`, `subtitles`, `chapters`, and `attachments`) plus a `manifest_url`. +Iterating a `Tracks` yields **only** video, audio, and subtitle tracks (not chapters or +attachments), and the canonical display/mux order is Video → Audio → Subtitle → Chapter +→ Attachment. + +!!! warning "Track IDs must be deterministic" + `Tracks.add` refuses duplicate IDs. Track IDs are CRC32-derived from the track's + identity (URL, codec, language, bitrate) and **must be stable across requests**, never + random. The manifest parsers already do this correctly; if you build tracks by hand, + keep IDs deterministic or `add` will raise. + +### The base `Track` + +`Track` is the base for `Video`, `Audio`, and `Subtitle`. (`Chapter`, `Chapters`, and +`Attachment` are standalone classes, **not** `Track` subclasses.) Every track knows how +it is delivered, via `Track.Descriptor`: + +| Descriptor | Delivery | +|---|---| +| `URL` | A single direct URL. | +| `HLS` | HTTP Live Streaming. | +| `DASH` | Dynamic Adaptive Streaming over HTTP. | +| `ISM` | Smooth Streaming. | + +Key attributes include `url` (a string or list of strings), `language` (a +`langcodes.Language`), `drm` (an optional iterable of DRM objects), `needs_repack`, and +descriptor-specific state stashed in `track.data["hls"]` / `track.data["dash"]`. When +`name` is omitted, a human-readable name is derived from the language automatically. + +### `Video` + +`Video` adds `codec`, `range`, `bitrate`, `width`, `height`, `fps`, `scan_type`, and +closed-caption descriptors. The relevant enums: + +- **`Video.Codec`**: `AVC` (H.264), `HEVC` (H.265), `VC1`, `VP8`, `VP9`, `AV1`. +- **`Video.Range`**: `SDR`, `HLG`, `HDR10`, `HDR10P` (HDR10+), `DV` (Dolby Vision), and + `HYBRID`. `HYBRID` is special: it tells the pipeline to select both an HDR10(+) base and + a DV track and merge the DV RPU onto the HDR10 stream with `dovi_tool` (see + [Dolby Vision hybrid](#dolby-vision-and-hybrid)). + +!!! note "Bitrate units" + Both `Video` and `Audio` store `bitrate` in **bytes per second** (rounded up), even + though the string form displays kb/s. + +### `Audio` + +`Audio` adds `bitrate`, `channels`, `joc` (Joint Object Coding, the Atmos indicator), +and `descriptive`. `Audio.Codec` covers `AAC`, `AC3` (`DD`), `EC3` (`DD+`), `AC4`, +`OPUS`, `OGG` (Vorbis), `DTS`, `ALAC`, and `FLAC`. The `atmos` property is `True` when +`joc` is set (or when `extra` flags it). + +### `Subtitle` + +`Subtitle` adds three mutually-constrained flags: `cc` (closed captions), `sdh` +(subtitles for the deaf and hard-of-hearing), and `forced`. A track cannot be both `cc` +and `sdh`, and `forced` cannot combine with either. `Subtitle.Codec` includes `SubRip` +(SRT), `WebVTT` (VTT), `SubStationAlpha`/`SubStationAlphav4` (SSA/ASS), +`TimedTextMarkupLang` (TTML), `SAMI`, and the box-encapsulated DASH formats `fTTML` +(STPP) and `fVTT` (WVTT). Subtitles receive post-download normalization +(segment merging, WebVTT sanitization, format conversion, and optional SDH stripping), +all documented for users in [Subtitles](../guide/subtitles.md). + +### Chapters and attachments + +- **`Chapter(timestamp, name=None)`**: a single marker; the timestamp accepts + `HH:MM:SS[.mmm]`, integer milliseconds, or float seconds. +- **`Chapters`**: a sorted collection that auto-inserts a `00:00:00.000` chapter and + reads/writes OGM Simple format. +- **`Attachment`**: a font or image attached to the mux (e.g. fonts an ASS subtitle + references), built from a local `path` or a `url`. + +### Selection, sorting, and muxing + +`Tracks` is also the selection engine the `dl` command uses. It offers +`sort_videos` / `sort_audio` / `sort_subtitles` (P2P-standard ordering), predicate-based +`select_*` filters, `by_resolutions`, and `by_language`. When downloads finish, the same +object multiplexes everything: **`Tracks.mux(title, ...)`** builds and runs an +`mkvmerge` command, applying language, default-track, forced, and hearing-impaired flags, +attaching chapters and fonts, and writing a `.muxed.mkv` (or `.mka` / `.mks` for +audio- or subtitle-only sets). As a service author you never call `mux` yourself; you +just return correct tracks. + +## Manifest parsers + +Most services do not build tracks by hand; they hand a manifest to one of three parsers +in `unshackle.core.manifests`. Each parser has the same three-part contract: + +| Format | Class | Notes | +|---|---|---| +| MPEG-DASH (`.mpd`) | `DASH` | Parses `<MPD>`; enumerates tracks from the first content period. | +| HLS (`.m3u8`) | `HLS` | Requires a **variant (master) playlist**, not a media playlist. | +| Smooth Streaming (`.ism`) | `ISM` | Synthesizes init segments from `CodecPrivateData`. | + +### The parser contract + +1. **Construct**: `DASH.from_url(url, session=...)` or `DASH.from_text(text, url)` + (and likewise for `HLS` / `ISM`). A `url` is always required, even for `from_text`, + because relative segment paths are resolved against it. +2. **`to_tracks(...)`**: parses the manifest into a `Tracks` container of `Video`, + `Audio`, and `Subtitle` objects, each carrying descriptor-specific state. +3. **`download_track(...)`**: a staticmethod the core calls to fetch, decrypt, and merge + segments. You do not call it; you just return the tracks it will operate on. + +A typical `get_tracks` therefore looks like fetching the manifest text with +`self.session` and returning `DASH.from_text(mpd, url).to_tracks(language=title.language)`. + +!!! tip "Pass the original language" + DASH and HLS `to_tracks` raise `ValueError` if a track's language cannot be derived + and no valid fallback was supplied. Always pass the title's original recorded + language. For HLS this fallback is used directly for the primary track, since HLS + variants carry no language. + +### Segment-filtering hooks + +- **HLS**: set `track.OnSegmentFilter(segment)` to drop unwanted segments (ads/bumpers). +- **DASH**: pass a `period_filter` callable to `to_tracks` to drop whole periods. + Amazon caching-breadth periods and non-`content` segment types are dropped automatically. + +### Format gotchas worth knowing + +- **DASH** enumerates tracks from only the *first* content period, but downloads segments + from *all* content periods matching the representation. `SegmentTemplate`, + `SegmentList`, and `SegmentBase` addressing and `$Number%0Nd$` padding are handled. +- **HLS** re-fetches the media playlist at download time, recovers `SUPPLEMENTAL-CODECS` + from raw text (python-m3u8 drops it) to detect Dolby Vision composite streams, and uses + the media-sequence-number IV for AES-128 keys that omit an explicit IV. +- **ISM** has no segment list and no init segments: segment URLs are synthesized from the + `<c>` fragment timeline, and a `moov`/`ftyp` init box is rebuilt from `CodecPrivateData` + (supported for AVC, HEVC, DV-over-HEVC, AAC, EC-3, and TTML). + +## DRM hooks + +If a track carries DRM, the framework licenses and decrypts it, but it needs your +service to talk to the platform's license server. unshackle recognizes five DRM systems: + +| DRM class | System | License flow | +|---|---|---| +| `Widevine` | Widevine | CDM challenge → license server → content keys (`.wvd` device) | +| `PlayReady` | PlayReady | CDM challenge (WRM header) → license server → content keys (`.prd` device) | +| `ClearKeyCENC` | W3C `org.w3.clearkey` over CENC | JSON KID request → JWK set; no CDM | +| `ClearKey` | HLS AES-128 | key fetched directly from a URI; no CDM, no challenge | +| `MonaLisa` | Proprietary WASM DRM | keys extracted locally; no license server | + +!!! warning "`ClearKey` vs `ClearKeyCENC`" + Despite the shared prefix these are entirely different systems. `ClearKey` is HLS + AES-128, decrypted in-process. `ClearKeyCENC` is the W3C EME JWK flow over MPEG-CENC, + decrypted with shaka-packager or mp4decrypt like Widevine. + +### The service-side hooks + +The framework supplies the CDM and the decryption tool; your service provides the network +calls that the CDM's challenge must be sent through. Override the hooks that your platform +needs: + +| Hook | Return | Purpose | +|---|---|---| +| `get_widevine_service_certificate(title, track, challenge)` | cert bytes | Optional privacy-mode service certificate. | +| `get_widevine_license(title, track, challenge)` | license bytes | POST the Widevine challenge to the license server, return the response. | +| `get_playready_license(title, track, challenge)` | license | Same for PlayReady; defaults to delegating to Widevine. | +| `get_clearkey_license(title, track, challenge)` | dict / str / bytes / `None` | For `ClearKeyCENC`; return `None` to let the framework POST to the manifest `Laurl`. | + +You do not open CDM sessions or run shaka-packager yourself. The `Widevine`/`PlayReady` +classes drive the CDM handshake and invoke the decryption tool. Your hook is just the +"send this challenge, give me back the license" step. + +### How a CDM is chosen and keys are found + +CDM selection is config-driven: the `cdm` map resolves a service tag (and optionally the +track quality or DRM type) to a device name, which `load_cdm` instantiates as a local +`.wvd`/`.prd` or a remote CDM. Before ever requesting a license, the framework tries an +in-process key cache and the configured **key vaults**; only unresolved KIDs trigger an +actual license request, and newly licensed keys are written back to every vault. This is +why the same key is rarely re-licensed twice. Users configure all of this in +[DRM and CDM](../guide/drm-and-cdm.md) and [Key vaults](../guide/vaults.md); the API +surface is documented in the [DRM API reference](../reference/api/drm.md). + +### Dolby Vision and hybrid + +Two DV mechanisms sit on top of the track model. **DV fixup** re-exposes DV signaling on +HLS streams that carry a valid DV RPU but advertise plain HEVC (`dv_compatible_bitstream`). +**Hybrid** processing (`Video.Range.HYBRID`) selects an HDR10(+) base and a DV donor, +extracts the RPU with `dovi_tool`, and injects it onto the base layer, producing a single +stream with both HDR10 and DV metadata. Both are automatic once the right tracks are +selected; a service only needs to expose accurate ranges. + +## Request and session objects + +Everything a service does over the network flows through objects the base constructor +prepared, all derived from the Click context (`ctx`) that `dl` passes in. + +- **`self.config`**: your service's slice of `unshackle.yaml`, i.e. + `config.services[<ServiceName>]`. This is a plain dict; use it for per-service settings + like base URLs, an API key, or a `title_map`. See + [Configuration](../getting-started/configuration-file.md). +- **`self.session`**: a `requests.Session` with `config.headers` applied, a retry policy + (`total=5`, backoff, retry on 429/5xx), and any resolved proxy already mounted. Pass this + session into the manifest parsers and DRM calls so headers, cookies, and proxies flow + everywhere. (unshackle also ships an rnet-based TLS-fingerprinting session used by the + newer `session()` factory and configured via the `network` config key.) +- **`self.credential`**: the active `Credential` (`username:password[:extra]`), populated + by `authenticate`. Its `sha1` is used as the account hash for cache keys. +- **`self.cache`**: a generic `Cacher`: JSON key/value storage under the service's cache + dir, with CRC32 integrity checks, versioning, and optional expiry (auto-derived from a + JWT `exp` claim when present). Good for auth tokens and API responses. +- **`self.title_cache`**: a `TitleCacher` that wraps `get_titles` results, keyed by title + ID, region, and account hash. It serves fresh data within `title_cache_time`, falls back + to stale data within `title_cache_max_retention` when an API call fails, and is bypassed + by `--no-cache`. +- **`self.track_request`**: the `TrackRequest` describing what the user asked for + (codecs, ranges, best-available), for use inside `get_tracks`. + +!!! example "A minimal service skeleton" + ```python title="services/EXMP/__init__.py" + from unshackle.core.service import Service + from unshackle.core.manifests import DASH + from unshackle.core.titles import Movie, Movies + + + class EXMP(Service): + ALIASES = ("example",) + GEOFENCE = ("us",) + + def get_titles(self) -> Movies: + data = self.session.get(f"https://api.example.com/title/{self.title}").json() + return Movies([ + Movie( + id_=data["id"], + service=self.__class__, + name=data["name"], + year=data.get("year"), + language=data.get("language"), + ) + ]) + + def get_tracks(self, title) -> "Tracks": + mpd_url = self.session.get(...).json()["manifest"] + mpd = self.session.get(mpd_url).text + return DASH.from_text(mpd, mpd_url).to_tracks(language=title.language) + + def get_chapters(self, title): + return [] + + def get_widevine_license(self, *, challenge, title, track): + return self.session.post("https://lic.example.com/wv", data=challenge).content + ``` + + The directory name (`EXMP`) must exactly match the class name; that stem is the + service tag. Put your service in one of the `directories.services` paths. + +## Where to go next + +- **[Creating a Service](creating-a-service.md)**: a guided walkthrough that builds on this model. +- **[Service API](../reference/api/service.md)**: exact signatures for the `Service` base class. +- **[Titles API](../reference/api/titles.md)** and **[Tracks API](../reference/api/tracks.md)**: the full object reference. +- **[DRM API](../reference/api/drm.md)**: the DRM and CDM classes in detail. +- **[Downloading](../guide/downloading.md)**: how the flags a user passes map onto track selection. diff --git a/docs/getting-started/configuration-file.md b/docs/getting-started/configuration-file.md new file mode 100644 index 00000000..2ab2d7d3 --- /dev/null +++ b/docs/getting-started/configuration-file.md @@ -0,0 +1,193 @@ +# The Configuration File + +Almost everything about how unshackle behaves is controlled by a single YAML file named `unshackle.yaml`: where it saves downloads, which CDM it uses, your credentials, proxies, filename templates, and more. This page covers where that file lives, how unshackle finds it, the shape of its contents, the directory layout unshackle builds around it, and how to edit values from the command line. + +!!! note "Where the exhaustive key list lives" + This page is an overview. For the complete, key-by-key breakdown of every setting, its type, and its default, see the [Configuration Reference](../reference/configuration.md). + +## The config filename + +The main configuration file is always named `unshackle.yaml`. It is a plain YAML file. You can open and edit it in any text editor, or manage individual values with the [`unshackle cfg`](#editing-the-config-with-unshackle-cfg) command. + +If you have never created one, that is fine: unshackle runs entirely on built-in defaults when no config file exists. The file only needs to contain the keys you want to change from their defaults. + +## Where the config file lives + +When unshackle starts, it searches a fixed list of locations and uses the **first one that exists**. In order of priority: + +| # | Location | Typical path | +|---|----------|--------------| +| 1 | The unshackle package folder | `.../site-packages/unshackle/unshackle.yaml` | +| 2 | The parent of the package folder | `.../site-packages/unshackle.yaml` | +| 3 | Your OS user-config directory | see the table below | + +The third location, your per-user config directory, is the recommended place for most installations, because it lives outside the package and survives reinstalls and upgrades. Its exact path depends on your operating system: + +=== "Linux" + + ```text + ~/.config/unshackle/unshackle.yaml + ``` + +=== "Windows" + + ```text + %LOCALAPPDATA%\unshackle\unshackle.yaml + ``` + +=== "macOS" + + ```text + ~/Library/Application Support/unshackle/unshackle.yaml + ``` + +!!! tip "Not sure which file is being used?" + Run `unshackle env info`. It prints the path the config was loaded from (or tells you none was found), along with every directory unshackle is currently using. + +The config is read exactly once, when unshackle starts. If you edit the file, the change takes effect on the next command you run. + +## Top-level structure + +`unshackle.yaml` is a flat map of top-level keys. Each key configures one area of the program. You only include the keys you want to change; everything else falls back to its default. + +Here is a small, realistic example that sets a download location, a release tag, a default CDM, and one service's credentials: + +```yaml title="unshackle.yaml" +# Where finished downloads are written +directories: + downloads: ~/Videos/unshackle + +# Release group tag appended to filenames +tag: MYGRP + +# Default Widevine/PlayReady device to use, with per-service overrides +cdm: + default: my_device_l3 + AMZN: my_device_l1 + +# Per-service login details (service tag -> "username:password") +credentials: + AMZN: my_email@example.com:hunter2 + +# Default HTTP headers merged into every request +headers: + Accept-Language: en-US,en;q=0.9 +``` + +The top-level keys group loosely into these areas: + +| Area | Example keys | +|------|--------------| +| Downloading & DRM | `dl`, `cdm`, `remote_cdm`, `decryption` | +| Networking | `network`, `headers` | +| Credentials & cookies | `credentials`, `firefox_cookies` | +| Tracks & muxing | `subtitle`, `audio`, `muxing`, `language_tags` | +| Key vaults | `key_vaults`, `vault_timeout` | +| Proxies & remote | `proxy_providers`, `remote_services`, `serve`, `services` | +| Naming & tagging | `tag`, `output_template`, `chapter_fallback_name` | +| External API keys | `tmdb_api_key`, `simkl_client_id`, `ipinfo_api_key` | +| Behavior & logging | `update_checks`, `redact_paths`, `debug`, `unicode_filenames` | +| Paths | `directories`, `filenames` | + +For the full list with types and defaults, see the [Configuration Reference](../reference/configuration.md). + +!!! warning "Unknown keys are silently ignored" + unshackle does not validate your config against a schema. If you misspell a key, it is simply skipped and the default is used. You will not get an error. Double-check key names (and their nesting) if a setting does not seem to apply. + +## The directory layout + +The `directories` key controls where unshackle reads and writes its various files. Each directory has a sensible default, and you can override most of them by giving a new path. Paths support `~` for your home directory. + +```yaml title="unshackle.yaml" +directories: + downloads: ~/Videos/unshackle + temp: /mnt/fast/unshackle-temp + cache: ~/.cache/unshackle +``` + +The directories unshackle uses: + +| Name | Purpose | Overridable | +|------|---------|-------------| +| `downloads` | Default output folder for finished downloads | Yes | +| `temp` | Temporary working files during a download | Yes | +| `cache` | Cache store (title cache, update checks, service caches) | Yes | +| `cookies` | Per-service cookie files | Yes | +| `logs` | Log files | Yes | +| `exports` | Export JSONs | Yes | +| `wvds` | Widevine devices (`.wvd` files) | Yes | +| `prds` | PlayReady devices (`.prd` files) | Yes | +| `dcsl` | DCSL data | Yes | +| `services` | Search paths for service code (see below) | Yes | +| `commands` | CLI command modules | Yes | +| `vaults` | Vault modules | Yes | +| `fonts` | Bundled fonts | Yes | + +!!! note "Some directories cannot be moved" + A handful of internal locations are protected: the package root, the core directory, the user-config directory, and the data base. If you list them under `directories`, unshackle silently ignores the override. This is intentional; those paths are tied to where the package is installed. + +### The `services` directory is special + +Unlike the other entries, `services` is a **list**, and each entry can be either a local folder or a remote repository. This lets you mix your own local service code with services pulled from Git: + +```yaml title="unshackle.yaml" +directories: + services: + - unshackle-dl/services # a GitHub owner/repo shorthand + - https://github.com/me/my-services.git + - ~/code/local-services # a local folder +``` + +Entries are searched in the order listed, and **the first source to define a given service tag wins**, so put local folders last if you want them to act as fallbacks rather than overrides. Remote repositories are cloned and periodically updated for you. See [Services](../dev/creating-a-service.md) for the full details on how service discovery and repositories work. + +## The `filenames` key + +Alongside `directories`, the `filenames` key lets you override the templates unshackle uses when naming its own working and log files (for example the log filename or the temporary chapters file). Most users never need to touch this. The available names and their default templates are listed in the [Configuration Reference](../reference/configuration.md#filenames). + +## Editing the config with `unshackle cfg` + +You do not have to edit `unshackle.yaml` by hand. The `unshackle cfg` command reads and writes individual values for you, creating the config file (and its parent directory) if it does not exist yet. + +**Read a single value** by passing its key. Nested keys use dot notation: + +```console +$ unshackle cfg tag +$ unshackle cfg cdm.default +``` + +**Set a value** by passing a key and a value: + +```console +$ unshackle cfg tag MYGRP +$ unshackle cfg cdm.default my_device_l3 +``` + +Values are parsed as Python literals where possible, so numbers, booleans, and lists work naturally, while bare text is treated as a string: + +```console +$ unshackle cfg update_checks false +$ unshackle cfg vault_timeout 30 +``` + +**Remove a value** with `--unset`: + +```console +$ unshackle cfg cdm.default --unset +``` + +**List everything** currently set with `--list`: + +```console +$ unshackle cfg --list +``` + +When it writes, `unshackle cfg` targets the config file that was loaded. If none exists yet, it creates `unshackle.yaml` in your OS user-config directory. + +!!! warning "Editing with `cfg` strips comments" + Because `unshackle cfg` rewrites the whole file when it saves, any comments in `unshackle.yaml` are removed by a write. If you keep important notes as comments, edit the file by hand instead, or keep those notes elsewhere. + +## Next steps + +- Browse the [Configuration Reference](../reference/configuration.md) for every key and default. +- Set up your first download in [Downloading](../guide/downloading.md). +- Learn how services are discovered and updated in [Services](../dev/creating-a-service.md). diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 00000000..1ddeaf5a --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,196 @@ +# Installation + +unshackle is a modular archival tool for movies, TV, and music. It ships as a +single Python package named `unshackle` that installs one console command, also +called `unshackle`. This page covers installing that command, the Python +version it needs, and the external command-line tools it expects to find on your +`PATH`. + +!!! note "What you get" + Installing the package registers a single entry point, the `unshackle` + command. It covers everything: downloading, key-vault management, device + provisioning, the REST server, and the helper utilities. + +## Requirements at a glance + +| Requirement | Recommended version | Needed for | +| --- | --- | --- | +| [Python](https://www.python.org/) | 3.11 to 3.14 | Running unshackle itself | +| [uv](https://docs.astral.sh/uv/) | ≥ 0.5 | Installing and running unshackle | +| [FFmpeg](https://ffmpeg.org/) | ≥ 6.0 | Media processing and analysis | +| [MKVToolNix](https://mkvtoolnix.download/) | ≥ 80 | MKV muxing and metadata editing | +| [shaka-packager](https://github.com/shaka-project/shaka-packager) | 2.6.1 | DRM decryption | +| [Bento4](https://github.com/axiomatic-systems/Bento4) | ≥ 1.6.0-639 | Alternate DRM decryption (`mp4decrypt`) | +| [dovi_tool](https://github.com/quietvoid/dovi_tool) | ≥ 2.1 | Dolby Vision handling | +| [SubtitleEdit](https://github.com/SubtitleEdit/subtitleedit/releases) | ≥ 5.0 | Subtitle conversion (optional) | + +The sections below explain each of these and how to confirm your setup is +complete. + +## Python version support + +unshackle requires **Python 3.11 or newer, up to and including 3.14** +(`requires-python = ">=3.11,<3.15"`). Anything older than 3.11 or 3.15 and +later is unsupported and the installer will refuse to build the environment. + +You do not usually need to install a matching Python interpreter by hand, since +`uv` can download and manage one for you. If you are working inside an existing +virtual environment, make sure it is on a supported version. + +## Installing with uv + +The recommended way to install unshackle is as a `uv` tool. This creates an +isolated environment for the package and puts the `unshackle` command on your +`PATH` without touching your system Python. + +```shell title="Install as a uv tool" +uv tool install git+https://github.com/unshackle-dl/unshackle.git +``` + +Once it finishes, the `unshackle` command is available globally: + +```shell +unshackle --help +``` + +!!! tip "Upgrading and removing" + Because it is installed from a git URL, re-run the same + `uv tool install` command (uv will update in place) to pull a newer build, + and use `uv tool uninstall unshackle` to remove it. + +### Running from a clone + +If you would rather work from a checkout of the repository (for example to try +an in-development branch or to keep local service code alongside unshackle), use +`uv run` inside the clone. This keeps the project's virtual environment active +for the duration of the command, so you do not have to activate it yourself. + +```shell title="Run from a git clone" +git clone https://github.com/unshackle-dl/unshackle.git +cd unshackle +uv run unshackle --help +``` + +Every command shown elsewhere in these docs works the same way from a clone, +just prefix it with `uv run`: + +```shell +uv run unshackle env check +``` + +!!! note "Developer note" + Contributors will also want the development and test dependency groups. The + project defines `dev` (linters, type checkers) and `test` (pytest and + friends) groups in `pyproject.toml`; install them with + `uv sync --group dev --group test` inside the clone. End users can ignore + these entirely. + +## External tools on your PATH + +unshackle shells out to a number of external command-line programs for media +processing and DRM work. It looks for each one first in a `binaries/` folder +next to the installed package, and then falls back to your system `PATH`, so +installing these tools system-wide (or dropping them into that folder) both +work. + +### Required tools + +These must be present for core downloading, decryption, and muxing to work: + +| Tool | Binary looked up | Why it is needed | +| --- | --- | --- | +| FFmpeg | `ffmpeg` | Media processing: remuxing, conversion, stream handling | +| FFprobe | `ffprobe` | Media analysis: inspecting tracks and container details | +| MKVToolNix | `mkvmerge` | Muxing downloaded tracks into a final MKV | +| mkvpropedit | `mkvpropedit` | Editing MKV metadata after muxing | +| shaka-packager | `shaka-packager` / `packager` | Decrypting DRM-protected content (the default decryptor) | + +!!! warning "shaka-packager is the default decryptor" + unshackle decrypts with shaka-packager unless you explicitly configure a + different decryptor. If it is missing, protected downloads will fail at the + decryption step. See the [configuration reference](configuration-file.md) for the + `decryption` key if you want to switch to `mp4decrypt`. + +### Recommended and optional tools + +These extend unshackle's capabilities. Install the ones relevant to what you +download. For example, `dovi_tool` only matters if you handle Dolby Vision. + +| Tool | Binary looked up | Why it is needed | +| --- | --- | --- | +| Bento4 (`mp4decrypt`) | `mp4decrypt` | Alternative DRM decryptor to shaka-packager | +| dovi_tool | `dovi_tool` | Dolby Vision metadata handling | +| HDR10Plus_tool | `hdr10plus_tool` | HDR10+ metadata handling | +| SubtitleEdit | `seconv`, `SubtitleEdit` | Subtitle conversion (the `SeConv` CLI from 5.x) | +| CCExtractor | `ccextractor` | Extracting closed captions | +| FFplay | `ffplay` | Simple preview player | +| MPV | `mpv` | Advanced preview player (used by `util crop`/`util range` previews) | +| HolaProxy | `hola-proxy` | Hola proxy provider support | +| Caddy | `caddy` | Optional reverse proxy for `unshackle serve --caddy` | +| Docker | `docker` | Gluetun VPN proxy support | +| Git | `git` | Fetching and updating remote service repositories | + +!!! tip "SubtitleEdit specifics" + unshackle looks for `seconv` first, the batch CLI shipped with + SubtitleEdit 5.x, before falling back to a `SubtitleEdit` (4.x) binary. The + 5.0 GUI has no batch mode, so the `SeConv` CLI is what you want for + automated subtitle conversion. + +## Verifying your installation + +After installing, confirm the command runs and that unshackle can see the tools +it depends on. + +### Confirm the command is available + +```shell +unshackle --help +``` + +This prints the banner and the list of available commands (`dl`, `env`, `kv`, +`serve`, `wvd`, `prd`, `search`, `util`, `cfg`, `import`). If your shell reports +that `unshackle` is not found, the `uv tool` bin directory is probably not on +your `PATH`. Run `uv tool update-shell` and open a new terminal. + +### Check external dependencies + +The `env check` subcommand inspects your environment and reports which tools it +found: + +```shell +unshackle env check +``` + +=== "Output" + + It prints a table grouped by category (Core, DRM, HDR, Subtitle, Player, + Network) with a status column: a green check for tools that resolved and a + red cross for those it could not find, plus a summary line such as + `All required tools installed ✓` or `Missing required: <names>`. + +=== "From a clone" + + ```shell + uv run unshackle env check + ``` + +If a required tool shows as missing, install it and put it on your `PATH` (or in +the package's `binaries/` folder), then re-run the check. + +### Inspect environment paths + +To see where unshackle will read its configuration from and where it stores +downloads, caches, cookies, and devices, use: + +```shell +unshackle env info +``` + +If no configuration file exists yet, this command lists the locations unshackle +searches for one, so you know where to create it. See the +[configuration guide](configuration-file.md) for the details of that file. + +## Next steps + +- Set up your [configuration](configuration-file.md) file (`unshackle.yaml`). +- Start [downloading](../guide/downloading.md) content once your tools check out. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 00000000..861c8766 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,274 @@ +# Quick Start + +This page takes you from a fresh install to your first downloaded file. You will +create the one piece of configuration unshackle insists on, add a streaming +service and a CDM so it can actually fetch and decrypt media, run a basic +`unshackle dl` command, and learn where the finished file lands. + +!!! note "Before you start" + This guide assumes unshackle is already installed and on your `PATH`. If + `unshackle --help` does not print a help screen, work through + [Installation](installation.md) first. + +## 1. Check your environment + +unshackle drives several external tools (FFmpeg, MKVToolNix, shaka-packager, and +others) to decrypt, repack, and mux media. Confirm they are visible before you +download anything: + +```shell +unshackle env check +``` + +This prints a dependency table. The **required** tools (FFmpeg, FFprobe, +MKVToolNix, mkvpropedit, and shaka-packager) must show a green check. Optional +tools such as `dovi_tool` (Dolby Vision) and CCExtractor (closed captions) only +matter for the features that use them. + +!!! tip + The summary line at the bottom reports `installed/total` and lists anything + required that is still missing, so you know exactly what to install next. + +## 2. Create a minimal config + +unshackle reads a single YAML file named `unshackle.yaml`. To see where it looks +for that file, and where it *would* accept one if you have not made it yet, run: + +```shell +unshackle env info +``` + +If no config exists, this prints the candidate locations. In search order they are: + +1. `unshackle.yaml` inside the unshackle package folder. +2. `unshackle.yaml` in that folder's parent. +3. `unshackle.yaml` in your OS user-config directory + (`~/.config/unshackle/` on Linux, `%LOCALAPPDATA%\unshackle\` on Windows, + `~/Library/Application Support/unshackle/` on macOS). + +The **first** file that exists wins. Create `unshackle.yaml` in one of those +locations. + +### The one key you must set + +unshackle refuses to start a download unless it knows how to name the output file. +That means `output_template` is the one setting a first run genuinely requires. +Everything else has a sensible default. A minimal, working config looks like this: + +```yaml title="unshackle.yaml" +output_template: + movies: "{title} ({year}) {quality} {source}" + series: "{title} S{season}E{episode} {episode_name} {quality} {source}" +``` + +Each `{variable}` is filled in from the title and the tracks you downloaded. The +full set of valid variables, including `resolution`, `video`, `audio`, `hdr`, +`edition`, `tag`, and more, is documented in [Configuration](configuration-file.md). + +!!! note "Spaces or dots?" + unshackle auto-detects your naming style from the template: if the separators + between variables are mostly spaces, it uses spaces; if they are mostly dots, + it produces scene-style `Title.S01E01.1080p` names. Write the template in the + style you want the filenames to look. + +!!! warning "Editing config from the CLI strips comments" + You can read and set keys with `unshackle cfg` (for example + `unshackle cfg tag MYGROUP` or `unshackle cfg --list`), but writing a value + rewrites the file and **removes any comments** it contained. If you keep notes + in your config, edit the file by hand instead. + +## 3. Add a service + +The service tag you pass to `dl` (like `NF`) maps to a **service module**, a small +plugin that knows how to talk to one streaming platform. unshackle ships without +any bundled services, so you add the one(s) you want. + +Services live under the `directories.services` path (run `unshackle env info` to +see where that is). unshackle discovers them by looking for `*/__init__.py`: the +folder name becomes the **service tag**, and the folder must contain a class with +that exact same name. + +=== "Local folder" + + Drop a service folder into your services directory. A service tagged `NF` + lives at: + + ``` + <services>/NF/__init__.py # defines a class named NF + ``` + + Once the folder is in place, the tag `NF` is immediately available to `dl`, + `search`, and the other service-aware commands. + +=== "Git repository" + + `directories.services` may also list git repositories. unshackle clones them + on first use and keeps them up to date: + + ```yaml title="unshackle.yaml" + directories: + services: + - owner/unshackle-services # a GitHub owner/repo shorthand + - /path/to/local/services # local folders still work too + ``` + + Force a refresh of every configured service repo at any time with: + + ```shell + unshackle util refresh-services + ``` + +!!! info "Where to get services" + Service modules are kept separate from the core project. Ask in the community + [Discord](https://discord.gg/mHYyPaCbFK) for where to find the service you + need. The core repository deliberately does not host them. + +## 4. Add a CDM for DRM + +Most streaming content is encrypted. To fetch decryption keys, unshackle needs a +**CDM**, a Widevine device (`.wvd`) or a PlayReady device (`.prd`). Register a +Widevine device you already have with: + +```shell +unshackle wvd add /path/to/device.wvd +``` + +This validates the file and moves it into your WVDs directory. Then point services +at it in your config. The `cdm` map is keyed by service tag, with a `default` that +covers everything else: + +```yaml title="unshackle.yaml" +cdm: + default: my_device # the .wvd file's name, without the extension + NF: my_other_device # override for a specific service +``` + +PlayReady works the same way with `.prd` files created and managed by the +[`prd`](configuration-file.md) command. If a title is DRM-free, no CDM is needed. + +!!! tip + Run `unshackle wvd parse my_device` to inspect a device's security level and + contents, and `unshackle env info` to confirm where WVDs and PRDs are stored. + +## 5. Provide authentication + +Services that require a login read either **cookies** or **credentials**. + +- **Cookies**: export the service's cookies to a Netscape-format text file and + place it in your cookies directory. unshackle looks for, in order: + `cookies/{SERVICE}.txt`, then `cookies/{SERVICE}/{profile}.txt`, then + `cookies/{SERVICE}/default.txt`. So a file at `cookies/NF.txt` is picked up + automatically for the `NF` service. + +- **Credentials**: store a username and password per service in your config: + + ```yaml title="unshackle.yaml" + credentials: + NF: "email@example.com:your-password" + ``` + +Use the `-p/--profile` flag to switch between multiple accounts for the same +service. Whether a given service needs cookies, credentials, or nothing at all +depends on the service module. + +## 6. Find a title (optional) + +If you have a URL or ID already, skip this. Otherwise, search the service for a +title and note the `id` it prints, since that is what you feed to `dl`: + +```shell +unshackle search NF "The Office" +``` + +You can also list what a service exposes for a given title without downloading: + +```shell +unshackle dl --list-titles NF 81234567 # show seasons/episodes +unshackle dl --list NF 81234567 # show available tracks +``` + +## 7. Run your first download + +A download always has three parts: + +``` +unshackle dl <FLAGS> <SERVICE-TAG> <TITLE> +``` + +- **`dl`** carries every quality, language, track, and output flag. +- **`<SERVICE-TAG>`** picks which service to talk to (`NF`, `DSNP`, …). +- **`<TITLE>`** is the URL, ID, or slug the service understands. + +A good first command asks for 1080p with English audio and subtitles: + +```shell title="Your first download" +unshackle dl -q 1080 -l en NF 81234567 +``` + +unshackle will fetch the title, select the tracks matching your flags, acquire +keys through your CDM (and any key vaults), then decrypt, mux, and tag the result. + +### Handy first flags + +| Flag | Meaning | +|---|---| +| `-q`, `--quality` | Target resolution(s), e.g. `-q 1080` or `-q 1080,720`. Defaults to best available. | +| `-l`, `--lang` | Language(s) for video and audio, e.g. `-l en` or `-l orig,en`. `orig` = the title's original language. | +| `-sl`, `--s-lang` | Subtitle language(s); defaults to `all`. | +| `-v`, `--vcodec` | Video codec, e.g. `-v H.265`. Defaults to any. | +| `-r`, `--range` | Dynamic range, e.g. `-r HDR10` or `-r DV`. Defaults to `SDR`. | +| `-w`, `--wanted` | Which episodes, e.g. `-w S01` or `-w S01E01-S01E03`. | +| `-o`, `--output` | Override the output directory for this run. | +| `--list` | List the tracks that would be downloaded, then stop. | + +!!! example "A few realistic variations" + ```shell + # A whole first season in the best available quality + unshackle dl -w S01 NF 81234567 + + # 2160p HDR10 with the original-language audio plus English subtitles + unshackle dl -q 2160 -r HDR10 -l orig -sl en NF 81234567 + + # Just the newest episode of an ongoing show + unshackle dl --latest-episode NF 81234567 + ``` + +See [Downloading](../guide/downloading.md) for the complete flag reference, including +codec, bitrate, channel-layout, and track-type selection. + +## 8. Where the output lands + +By default, finished files are written to the `downloads` directory +(`unshackle env info` shows its exact path; the built-in default is a `downloads` +folder alongside the project). Override it per run with `-o`: + +```shell +unshackle dl -q 1080 -o /mnt/media/incoming NF 81234567 +``` + +- **Movies** are written as a single `.mkv` file named from your `movies` template. +- **TV episodes** are grouped into a per-show / per-season folder (from your + `series` template) unless you pass `--no-folder`. +- The filename is built from your `output_template`, and IMDb/TMDB IDs are added to + the file's metadata tags when available. + +To change the default output location permanently, set it in your config: + +```yaml title="unshackle.yaml" +directories: + downloads: /mnt/media/incoming +``` + +## Where to go next + +- **[Downloading](../guide/downloading.md)**. The full `dl` command: quality, codecs, + languages, track selection, hybrid Dolby Vision, and output control. +- **[Configuration](configuration-file.md)**. Every `unshackle.yaml` key, plus + directories, key vaults, proxies, and naming templates. +- **[REST API](../dev/rest-api/index.md)**. Run the `serve` server to drive downloads over + HTTP. + +!!! tip "You are never far from help" + `unshackle --help`, `unshackle dl --help`, and `unshackle <command> --help` + list every option with its default. When in doubt, add `--list` to a `dl` + command to preview what it would do before it downloads anything. diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md new file mode 100644 index 00000000..741a9e2f --- /dev/null +++ b/docs/guide/cli-reference.md @@ -0,0 +1,550 @@ +# CLI Reference + +A complete reference for every `unshackle` command, subcommand, and flag. Commands are grouped by purpose: + +- **[`dl`](#dl)**: the download pipeline (the command you use most). +- **[`search`](#search)**: find titles on a service. +- **[`import`](#import)**: rebuild a download from an export file. +- **[`cfg`](#cfg)**, **[`env`](#env)**: manage configuration and the environment. +- **[`kv`](#kv)**: Key Vault operations. +- **[`wvd`](#wvd)**, **[`prd`](#prd)**: Widevine / PlayReady device management. +- **[`serve`](#serve)**: run the local CDM + REST API server. +- **[`util`](#util)**: helper media utilities. + +!!! tip "Getting help on any command" + Every command accepts `-h`, `--help`, or `-?`. For example `unshackle dl --help` or `unshackle wvd new --help`. Service-specific arguments for `dl` and `search` (such as a title ID or search query) are defined by each service, so check `unshackle dl SERVICE --help` for those. + +--- + +## Root command + +``` +unshackle [OPTIONS] COMMAND [ARGS]... +``` + +> unshackle: Modular Movie, TV, and Music Archival Software. + +Every invocation prints a banner and, when `update_checks` is enabled in your config, checks for a newer release. + +| Option | Description | +|---|---| +| `-v`, `--version` | Print version information and exit. | +| `-d`, `--debug` | Enable `DEBUG`-level logs and JSON debug logging. Silences noisy HTTP libraries unless `debug_requests` is set in config. | +| `-h`, `--help`, `-?` | Show help and exit. | + +Commands are auto-discovered, so third-party command modules dropped into the commands directory appear automatically. + +--- + +## `dl` + +The download command. It is itself a group whose **subcommands are the installed streaming services**, so an invocation always has three layers: + +``` +unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] +``` + +- `OPTIONS`: every flag below, parsed at the `dl` level. +- `SERVICE`: a service tag (e.g. `NF`, `DSNP`). Tags are case-insensitive and honour each service's aliases (`disney+`, `DSNP`, etc. all resolve to the same tag). +- `SERVICE ARGS`: the title, URL, or ID, plus any service-specific options. These belong to the service, not to `dl`. + +!!! example "Typical downloads" + ```shell + # Best available 1080p SDR of a Netflix title + unshackle dl -q 1080 NF 81234567 + + # 4K HDR10, English audio + subs, from a URL + unshackle dl -q 2160 -r HDR10 -l en DSNP "https://www.disneyplus.com/…" + + # A season range, HEVC video, EC3 audio + unshackle dl -w S01-S03 -v H.265 -a EC3 SERVICE TITLE-ID + ``` + +!!! note "Config-driven defaults" + Any `dl` flag can be given a default under the `dl:` section of `unshackle.yaml`, and per-service under `services.<TAG>.dl`. Explicit command-line values (and environment values) always win over both; service defaults only fill in options you did not set. See the [configuration file](../getting-started/configuration-file.md) guide. + +### Quality, codec, bitrate & range + +| Flag | Default | Description | +|---|---|---| +| `-p`, `--profile` | — | Profile for credentials and cookies. | +| `-q`, `--quality` | best | Resolution(s), comma-separated, e.g. `1080,720`. `-q 1080` also matches anamorphic tracks by 16:9 canvas. | +| `-v`, `--vcodec` | any | Video codec(s). Accepts names or values: `AVC`/`H.264`, `HEVC`/`H.265`, `VC1`, `VP8`, `VP9`, `AV1`. | +| `-a`, `--acodec` | any | Audio codec(s), comma-separated. Accepts `AAC`, `AC3`/`DD`, `EC3`/`DD+`/`eac3`/`ddp`, `AC4`, `OPUS`, `OGG`/`vorbis`, `DTS`, `ALAC`, `FLAC`. | +| `-vb`, `--vbitrate` | highest | Exact video bitrate in kbps. | +| `-ab`, `--abitrate` | highest | Exact audio bitrate in kbps. | +| `-vb-range`, `--vbitrate-range` | — | Video bitrate range in kbps, e.g. `6000-7000`; picks highest within. Mutually exclusive with `--vbitrate`. | +| `-ab-range`, `--abitrate-range` | — | Audio bitrate range in kbps, e.g. `128-256`. Mutually exclusive with `--abitrate`. | +| `-r`, `--range` | `SDR` | Colour range(s): `SDR`, `HLG`, `HDR10`, `HDR10P` (HDR10+), `DV`, `HYBRID`. | +| `-c`, `--channels` | — | Audio channels; matches sub-layouts (5.1 ≈ 6.0). | +| `-naa`, `--noatmos` | off | Exclude Dolby Atmos audio tracks. | +| `--worst` | off | Pick the lowest bitrate within the requested quality. **Requires `-q`.** | +| `--best-available` | off | Continue with the best available if a requested resolution/language is absent, instead of failing. | +| `-rvb`, `--real-video-bitrate` | off | Probe real media size for true video bitrates, overriding the manifest. | +| `-rab`, `--real-audio-bitrate` | off | Same for audio (slower). | + +!!! note "`HYBRID` range" + `-r HYBRID` fetches both HDR10/HDR10+ and Dolby Vision and merges them with `dovi_tool`. It requires the `dovi_tool` binary on your `PATH`. + +### Language & subtitles + +| Flag | Default | Description | +|---|---|---| +| `-l`, `--lang` | `orig` | Language(s) for **both** video and audio. `orig` = the title's original language; e.g. `orig,en`. | +| `-vl`, `--v-lang` | — | Video-only language (overrides `-l` for video). | +| `-al`, `--a-lang` | — | Audio-only language (overrides `-l` for audio). | +| `-sl`, `--s-lang` | `all` | Subtitle language(s). | +| `--require-subs` | — | Required subtitle langs; keeps **all** subs only if these exist. **Cannot combine with `--s-lang`.** | +| `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | +| `--exact-lang` | off | Exact matching only: `-l es-419` matches `es-419`, not `es-ES`. | +| `--sub-format` | — | Output subtitle format (`SRT`/`srt`, `VTT`/`webvtt`, `ASS`/`ssa`, `TTML`, `SMI`, …), or `original` to keep the source format. | + +The special language tokens `orig`, `all`, and `best` are honoured everywhere a language is expected. + +### Title & episode selection + +| Flag | Description | +|---|---| +| `-w`, `--wanted` | Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`. Supports exclusions with a leading `-` (e.g. `-S03`). | +| `--select-titles` | Interactively select titles (series only). **Cannot combine with `-w`.** | +| `--latest-episode` | Download only the single most recent episode. | +| `--list-titles` | List titles only; do not download. | + +### Track-type inclusion / exclusion + +Keep only certain track types, or skip certain track types. Attachments are always kept. + +| Include-only | Skip | +|---|---| +| `-V`, `--video-only` | `-nv`, `--no-video` | +| `-A`, `--audio-only` | `-na`, `--no-audio` | +| `-S`, `--subs-only` | `-ns`, `--no-subs` | +| `-C`, `--chapters-only` | `-nc`, `--no-chapters` | + +| Flag | Description | +|---|---| +| `-ad`, `--audio-description` | Include descriptive (audio-description) tracks. | +| `--skip-subtitle-errors` | Skip a failed subtitle instead of aborting the title. Video/audio failures remain fatal. | + +### Output, muxing & files + +| Flag | Description | +|---|---| +| `--split-audio` | Write a separate output file per audio codec instead of merging. Defaults to config `muxing.merge_audio`. | +| `--merge-video` | Mux all selected video tracks into one file. Defaults to config `muxing.merge_video`. | +| `-o`, `--output` | Override the output directory for this run. | +| `--no-folder` | Disable folder creation for TV shows. | +| `--no-source` | Remove the source tag from the filename/path. | +| `--no-mux` | Do not mux; keep individual track files. | +| `--tag` | Group tag override. | +| `--repack` | Add a `REPACK` tag to the filename. | + +### Metadata & tagging + +| Flag | Description | +|---|---| +| `--tmdb` | TMDB ID (integer). | +| `--imdb` | IMDb ID, e.g. `tt1375666`. | +| `--animeapi` | AnimeAPI ID, e.g. `mal:12345` or `anilist:98765` (defaults to MAL). Back-fills TMDB/IMDb. | +| `--enrich` | Override show title/year from the external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | + +### DRM, keys & decryption + +| Flag | Description | +|---|---| +| `--cdm-only` / `--vaults-only` | Use only the CDM, or only Key Vaults, for key acquisition. | +| `--skip-dl` | Skip the download but still retrieve keys. | +| `--export` | Export track info and keys to a JSON file in the exports directory. | + +### Network & proxy + +| Flag | Description | +|---|---| +| `--proxy` | Proxy URI, a 2-letter country code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`). | +| `--no-proxy` | Force-disable all proxy use. | +| `--no-proxy-download` | Bypass the proxy for **segment downloads only** (manifest, licence, and auth stay proxied). | +| `--remote` | Use a remote unshackle server. | +| `--server` | Name a remote server from the `remote_services` config. | + +### Concurrency, caching & listing + +| Flag | Default | Description | +|---|---|---| +| `--workers` | downloader default | Per-track download threads. | +| `--downloads` | `1` | Number of tracks downloaded concurrently. | +| `--no-cache` | off | Bypass the title cache. | +| `--reset-cache` | off | Clear the title cache. | +| `--list` | off | List available/would-be-downloaded tracks; do not download. | +| `--slow` | — | Inter-title delay. Bare `--slow` = 60-120s; `--slow 20-40` = custom range (minimum 20s). | + +!!! warning "Some flags cannot be combined" + `--require-subs` and `--s-lang`; `--select-titles` and `--wanted`; `--worst` requires `--quality`; `--vbitrate` and `--vbitrate-range` (and the audio equivalents) are mutually exclusive. + +--- + +## `search` + +Search a service for titles. Like `dl`, `search` is a group whose subcommands are the installed services, and it reuses `dl`'s authentication, cookie, and proxy machinery. + +``` +unshackle search [OPTIONS] SERVICE [QUERY...] +``` + +The query syntax is defined per service. Results are printed as a tree of titles with their service IDs. Feed an ID straight into `dl`. + +| Option | Description | +|---|---| +| `-p`, `--profile` | Profile for credentials and cookies. | +| `--proxy` | Proxy URI, 2-letter country code, or `provider:region`. | +| `--no-proxy` | Force-disable all proxy use. | + +!!! example + ```shell + unshackle search NF "The Office" + unshackle search -p myprofile NF "breaking bad" + unshackle search --proxy nordvpn:ca SERVICE "query" + ``` + +--- + +## `import` + +Reconstruct a download (download → decrypt → mux) from an `--export` JSON file without re-contacting the service. It re-fetches the manifest, injects the stored keys, and then runs the normal `dl` pipeline. The service tag is read from the export file, not passed by you. + +``` +unshackle import EXPORT_FILE [DL_ARGS...] +``` + +Any `dl` options placed after the file are forwarded verbatim, so you can override quality, range, proxy, and so on. + +!!! example + ```shell + unshackle import export.json + unshackle import export.json -r HDR10 --proxy US + ``` + +The export file must be a valid v2 export (created by a current build of unshackle via `dl --export`) and must contain a `service` tag. + +--- + +## `cfg` + +Read, set, delete, or list configuration values in `unshackle.yaml` without hand-editing YAML. + +``` +unshackle cfg [KEY] [VALUE] [OPTIONS] +``` + +| Argument / Option | Description | +|---|---| +| `KEY` | Dotted path into the config, e.g. `tag`, `serve.api_secret`, `directories.downloads`. | +| `VALUE` | Value to set. Parsed as a Python literal when possible, so `true`, `123`, `['a','b']`, and `{'k':1}` become real types; bare words stay strings. | +| `--unset` | Remove the configuration value. | +| `--list` | List all set configuration values. | + +!!! warning "Comments are stripped" + Writing a value through `cfg` rewrites the YAML file and **removes all comments** from it. If your config relies on comments, edit it by hand instead. Setting a value and using `--unset` together is an error. + +!!! example + ```shell + unshackle cfg --list # print entire config + unshackle cfg serve.api_secret # read a nested key + unshackle cfg tag MYGROUP # set a string + unshackle cfg update_checks false # literal → boolean False + unshackle cfg directories.downloads /mnt/dl # set a nested path + unshackle cfg tag --unset # remove a key + ``` + +--- + +## `env` + +Inspect and manage the project environment. + +``` +unshackle env COMMAND [ARGS]... +``` + +### `env check` + +``` +unshackle env check +``` + +Prints a dependency table (Category / Tool / Status / Required / Purpose) showing which external tools resolve on your `PATH`, and a summary of how many required tools are installed. + +| Category | Tools | +|---|---| +| Core | FFmpeg*, FFprobe*, MKVToolNix*, mkvpropedit* | +| DRM | shaka-packager*, mp4decrypt, ML-Worker | +| HDR | dovi_tool, HDR10Plus_tool | +| Subtitle | SubtitleEdit, CCExtractor | +| Player | FFplay, MPV | +| Network | HolaProxy, Caddy, Docker, git | + +<small>* required; all others are optional.</small> + +### `env info` + +``` +unshackle env info +``` + +Shows where the config was loaded from (or lists the candidate config locations if none was found) and prints a table of every configured directory (downloads, temp, cache, cookies, logs, exports, WVDs, PRDs, services, and more). + +### `env clear` + +Clear an environment directory. The directory is emptied and recreated, and the number of files and bytes freed is reported. + +| Command | Description | +|---|---| +| `unshackle env clear cache [SERVICE]` | Clear the cache directory, or just one service's cache subdirectory. | +| `unshackle env clear temp` | Clear the temp directory. | + +!!! example + ```shell + unshackle env check + unshackle env info + unshackle env clear cache + unshackle env clear cache NF # one service only + unshackle env clear temp + ``` + +--- + +## `kv` + +Manage Key Vaults. Vaults are configured under `key_vaults` in `unshackle.yaml`, each with a `name`, a `type` (`sqlite`, `mysql`, `http`, `api`), and type-specific options. Service tags are normalised automatically. + +``` +unshackle kv COMMAND [ARGS]... +``` + +### `kv copy TO_VAULT FROM_VAULT...` + +Copy keys from one or more source vaults into a destination vault. Rows with matching KIDs are skipped unless the existing row has no key; existing data is never altered or deleted. + +| Option | Description | +|---|---| +| `-s`, `--service` | Only copy data for a specific service. | +| `-l`, `--local-only` | Only copy data for services installed locally. Mutually exclusive with `--service`. | + +### `kv sync VAULT...` + +Ensure two or more vaults hold the same set of keys, essentially a chained bidirectional copy. Requires more than one vault. Accepts the same `--service` / `--local-only` options as `copy`. + +### `kv add FILE SERVICE VAULT...` + +Add content keys to one or more vaults for a service. `FILE` contains one `KID:KEY` pair per line (32 hex : 32 hex, UTF-8). Lines that don't match are skipped. + +### `kv search KID` + +Search configured vaults for a KID (32 hex characters, no dashes) and report any matching key. + +| Option | Description | +|---|---| +| `-s`, `--service` | Limit the search to a specific service tag. | +| `-v`, `--vault` | Limit the search to a specific configured vault by name. | + +!!! note + Remote vaults cannot be enumerated without a service, so pass `--service` when searching them. + +### `kv prepare VAULT...` + +Create service tables on vaults that use tables, for every installed service, where they don't already exist. + +!!! example + ```shell + unshackle kv copy main backup1 backup2 + unshackle kv copy main backup -s NF + unshackle kv sync main mysql_vault + unshackle kv add keys.txt NF main backup + unshackle kv search 0123456789abcdef0123456789abcdef -s NF -v main + unshackle kv prepare main mysql_vault + ``` + +--- + +## `wvd` + +Manage Widevine Device (`.wvd`) files. Devices live in the configured WVDs directory. + +``` +unshackle wvd COMMAND [ARGS]... +``` + +| Command | Description | +|---|---| +| `wvd add PATHS...` | Validate and move one or more `.wvd` files into the WVDs directory. | +| `wvd delete NAMES...` | Delete `.wvd` files by name (without extension). Prompts for confirmation. | +| `wvd parse PATH` | Parse a `.wvd` and print its System ID, security level, type, flags, and client info. Relative paths resolve against the WVDs directory. | +| `wvd dump WVD_PATHS... OUT_DIR` | Extract a device's contents (metadata, private key, client ID, VMP) into `OUT_DIR/<name>/`. With no paths, dumps every WVD in the WVDs directory. | +| `wvd new NAME PRIVATE_KEY CLIENT_ID [FILE_HASHES]` | Create a new `.wvd` from a PEM private key and a `ClientIdentification` blob, optionally with a VMP (`FileHashes`) blob. | + +`wvd new` options: + +| Option | Default | Description | +|---|---|---| +| `-t`, `--type` | `Android` | Device type. | +| `-l`, `--level` | `1` | Security level (1-3). | +| `-o`, `--output` | WVDs dir | Output directory. | + +!!! example + ```shell + unshackle wvd add device.wvd + unshackle wvd parse nexus_6p_1234_l1 + unshackle wvd dump device.wvd ./out_dir + unshackle wvd new "Nexus 6P" private_key.pem client_id.bin vmp.bin -t ANDROID -l 3 + ``` + +--- + +## `prd` + +Manage PlayReady Device (`.prd`) files. Devices live in the configured PRDs directory. Built on `pyplayready`. + +``` +unshackle prd COMMAND [ARGS]... +``` + +### `prd new PATHS...` + +Create a new `.prd`. Provide either a single folder containing `zgpriv.dat` (group key) and `bgroupcert.dat` (group certificate), or two file paths (group key, then group certificate). + +| Option | Description | +|---|---| +| `-e`, `--encryption_key` | Optional device ECC private encryption key (generated if omitted). | +| `-s`, `--signing_key` | Optional device ECC private signing key (generated if omitted). | +| `-o`, `--output` | Output directory or `.prd` file path. | + +`prd new` will not overwrite an existing file. + +### `prd reprovision PRD_PATH` + +Reprovision an existing device by replacing its leaf certificate and keys. The device must support reprovisioning (version 3 or higher). Accepts the same `-e` / `-s` / `-o` options; defaults to overwriting the device in place. + +### `prd test DEVICE` + +Test a device against the Microsoft PlayReady demo server and print the returned keys. + +| Option | Default | Description | +|---|---|---| +| `-c`, `--ckt` | `aesctr` | Content key encryption type: `aesctr` or `aescbc`. | +| `-sl`, `--security-level` | `2000` | Minimum security level: `150`, `2000`, or `3000`. | + +!!! example + ```shell + unshackle prd new ./device_folder/ + unshackle prd new zgpriv.dat bgroupcert.dat -o /out/dir + unshackle prd reprovision mydevice.prd + unshackle prd test mydevice.prd -sl 3000 -c aescbc + ``` + +--- + +## `serve` + +Serve your local Widevine/PlayReady devices and the REST API for remote access. Built on `aiohttp`. + +``` +unshackle serve [OPTIONS] +``` + +| Option | Default | Description | +|---|---|---| +| `-h`, `--host` | `127.0.0.1` | Host to serve from. | +| `-p`, `--port` | `8786` | Port to serve from. | +| `--caddy` | off | Also serve through Caddy (requires the `caddy` binary and a `Caddyfile`). | +| `--api-only` | off | Serve only the REST API, not the CDM endpoints. Incompatible with `--no-widevine`/`--no-playready`. | +| `--no-widevine` | off | Disable the Widevine CDM endpoints. | +| `--no-playready` | off | Disable the PlayReady CDM endpoints. | +| `--no-key` | off | Disable API-key authentication (allows all requests). | +| `--debug-api` | off | Include tracebacks/stderr in API error responses. | +| `--debug` | off | Enable debug logging for API operations. | +| `--remote-only` | off | Expose only the remote service session endpoints (health, services, search, session). Implies `--api-only`. | + +WVD files in the WVDs directory and PRD files in the PRDs directory are auto-loaded. The REST API lives under `http://<host>:<port>/api/`, with Swagger UI at `/api/docs/` and a health check at `/api/health` (exempt from auth). + +!!! warning "Configure `api_secret` first" + Unless you pass `--no-key`, `serve.api_secret` must be set in your config. Requests authenticate with the `X-Secret-Key` header. Running with `--no-key` disables authentication entirely and should only be used on a trusted, private network. + + ```yaml title="unshackle.yaml" + serve: + api_secret: "your-api-secret" + users: + your-secret-key: + devices: ["device_name"] # Widevine devices + playready_devices: ["device_name"] # PlayReady devices + username: user + ``` + +!!! example + ```shell + unshackle serve + unshackle serve -h 0.0.0.0 -p 8080 + unshackle serve --api-only + unshackle serve --remote-only + unshackle serve --caddy + ``` + +--- + +## `util` + +Various helper media utilities. When a command is given a directory, it processes every `.mkv`/`.mp4` inside it in natural (S01E01 before S01E10) order. + +``` +unshackle util COMMAND [ARGS]... +``` + +### `util refresh-services` + +``` +unshackle util refresh-services +``` + +Force a refresh (git pull / hard reset) of all service repos configured under `directories.services`. + +### `util crop PATH ASPECT` + +Losslessly crop H.264/H.265 video at the bitstream level. `ASPECT` is a `W:H` target such as `2.39:1`. + +| Option | Default | Description | +|---|---|---| +| `--letter` / `--pillar` | `--letter` | Crop top/bottom (`--letter`) or the sides (`--pillar`). | +| `-o`, `--offset` | `0` | Fine-tune the computed crop area if not perfectly centred. | +| `-p`, `--preview` | off | Preview the crop in MPV (or FFplay) instead of writing a file. | + +### `util range PATH` + +Losslessly set the video range flag to full or limited at the bitstream level. + +| Option | Default | Description | +|---|---|---| +| `--full` / `--limited` | — | Full (0-255) or limited (16-235) range. | +| `-p`, `--preview` | off | Preview instead of writing a file. | + +### `util test PATH` + +Decode an entire video with FFmpeg and report any corruptions or errors. All streams are tested by default; subtitles cannot be tested. + +| Option | Default | Description | +|---|---|---| +| `-m`, `--map` | `0` | Test specific streams via FFmpeg's `-map`, e.g. `0:v:0` or `0:a`. | + +!!! example + ```shell + unshackle util refresh-services + unshackle util crop video.mkv 2.39:1 + unshackle util crop ./season/ 16:9 --pillar -o 10 + unshackle util range video.mkv --full + unshackle util test video.mkv -m 0:v:0 + ``` + +--- + +## See also + +- [Installation](../getting-started/installation.md): install unshackle and its dependencies. +- [Quickstart](../getting-started/quickstart.md): your first download. +- [Configuration file](../getting-started/configuration-file.md): set defaults for any `dl` flag and configure vaults, proxies, and directories. diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md new file mode 100644 index 00000000..1fd25e67 --- /dev/null +++ b/docs/guide/downloading.md @@ -0,0 +1,563 @@ +# Downloading + +`unshackle dl` is the main download command: it fetches a title's manifest from a +streaming service, selects exactly the tracks you asked for, acquires and applies +decryption keys, and muxes everything into a finished Matroska file. The flags below +cover quality and codec selection, language and subtitle handling, episode ranges, and +the download flow itself, with examples throughout. + +!!! note "Where the full flag list lives" + This page covers the flags you will use most and explains how they interact. It is + not an exhaustive enumeration. For the complete, always-current list of every option + and its exact wording, run `unshackle dl --help`. Every flag shown here can also be + given a default in your configuration file. See + [Configuration file](../getting-started/configuration-file.md). + +## Command shape + +A download is always three layers: the root CLI, the `dl` command (which is where all +the flags below are parsed), and a **service tag** subcommand that carries the title +argument. + +```shell +unshackle dl [OPTIONS] SERVICE [SERVICE ARGUMENTS] +``` + +- `unshackle`: the root command. +- `dl`: the download command. **All of the options on this page are parsed here**, so + they go *before* the service tag. +- `SERVICE`: a service tag such as `NF`, `DSNP`, `AMZN`. Service tags are + case-insensitive and honour each service's aliases (for example `disney+` resolves to + its real tag). +- `SERVICE ARGUMENTS`: usually a title ID or URL. This positional argument belongs to + the **service**, not to `dl`, so it comes *after* the tag. + +```shell title="A minimal download" +unshackle dl NF 81234567 +``` + +```shell title="Flags belong to dl, the title belongs to the service" +unshackle dl -q 1080 -v H.265 -r HDR10 --lang en DSNP 'https://www.disneyplus.com/…' +``` + +!!! tip "See what a service accepts" + Because the positional argument is defined by each service, run + `unshackle dl SERVICE --help` (e.g. `unshackle dl NF --help`) to see that service's + own argument and any service-specific options. + +## The download flow + +When you run a download, `dl` moves through the pipeline below. Understanding the order +helps explain why, for example, `--list` stops early and why keys are fetched even with +`--skip-dl`. + +1. **Setup**: load the DRM CDM, key vaults, proxy providers, cookies, and credentials. +2. **Authenticate**: sign in to the service with your profile's cookies/credentials. +3. **Fetch titles**: retrieve the movie, episode list, or album (cached unless + `--no-cache`/`--reset-cache`). +4. **Filter titles**: apply `--wanted`, `--latest-episode`, or `--select-titles`. +5. **Get tracks**: parse the manifest into video, audio, subtitle, and chapter tracks. +6. **Select tracks**: narrow each type down using your quality, codec, range, language, + and bitrate flags. +7. **Download**: pull segments for the selected tracks (concurrently per `--downloads`). +8. **License & decrypt**: acquire content keys (vault → CDM), decrypt each track. +9. **Post-process**: extract closed captions, convert subtitles, repack, mux with + `mkvmerge`, and move the finished file to your downloads directory. + +`--list` and `--list-titles` stop after selection/filtering and print what *would* be +downloaded. `--skip-dl` runs the license step but skips the actual segment download. + +## Choosing quality + +### Resolution + +`-q` / `--quality` takes one or more target resolutions (heights) as a comma-separated +list. Without it, unshackle picks the **best available** resolution. + +```shell title="Single resolution" +unshackle dl -q 1080 NF 81234567 +``` + +```shell title="Multiple resolutions in one run" +unshackle dl -q 2160,1080,720 NF 81234567 +``` + +!!! note "16:9 canvas matching" + Resolution matching is by track **height** first. If no track has an exact matching + height, unshackle falls back to a 16:9-canvas match, so `-q 1080` will also match an + anamorphic `1920×804` track (computed as `int(width × 9 / 16)`). + +### Best-available and worst + +By default a missing requested resolution is an error. Two flags change that: + +| Flag | Behaviour | +| --- | --- | +| `--best-available` | If the requested resolution(s) aren't present, continue with the best that *is* available instead of failing. Also softens missing audio/subtitle languages and hybrid fallbacks. | +| `--worst` | Within the specified quality, pick the **lowest** bitrate rendition. **Requires `-q/--quality`.** | + +```shell title="Never fail on a missing resolution" +unshackle dl -q 2160 --best-available AMZN 0ABC123 +``` + +## Video codec and color range + +### Codec + +`-v` / `--vcodec` selects one or more video codecs; the default is any codec. It accepts +either enum **names** or their **values**, comma-separated. + +| Name | Value | +| --- | --- | +| `AVC` | `H.264` | +| `HEVC` | `H.265` | +| `VC1` | `VC-1` | +| `VP8` | `VP8` | +| `VP9` | `VP9` | +| `AV1` | `AV1` | + +```shell title="Either spelling works" +unshackle dl -v HEVC NF 81234567 +unshackle dl -v H.265 NF 81234567 +unshackle dl -v hevc,avc NF 81234567 +``` + +### Color range + +`-r` / `--range` selects one or more color ranges; the default is `SDR`. + +| Range | Meaning | +| --- | --- | +| `SDR` | Standard dynamic range (default) | +| `HLG` | Hybrid Log-Gamma | +| `HDR10` | HDR10 | +| `HDR10P` | HDR10+ | +| `DV` | Dolby Vision | +| `HYBRID` | Fetch both an HDR10/HDR10+ base and a DV track, then merge them | + +```shell title="Grab HDR10 and Dolby Vision in one run" +unshackle dl -q 2160 -r HDR10,DV DSNP '…' +``` + +!!! warning "HYBRID requires dovi_tool" + `-r HYBRID` produces a single hybrid stream by injecting the Dolby Vision RPU + metadata onto an HDR10/HDR10+ base layer using **dovi_tool**. It requires the + `dovi_tool` binary on your PATH and needs both an HDR10(+) *and* a DV track to be + available, otherwise the title fails (unless `--best-available` lets it fall back). + When HDR10+ is present it is preferred over HDR10 as the base layer. + + To keep an HDR10+ deliverable *alongside* the hybrid, request both ranges: + `-r HYBRID,HDR10P`. `-r HYBRID` on its own muxes only the merged hybrid. + +## Bitrate selection + +By default unshackle keeps the highest-bitrate rendition for each selected +resolution/codec/range/language combination. You can constrain this. + +| Flag | Purpose | +| --- | --- | +| `-vb` / `--vbitrate` | Exact video bitrate in kbps. | +| `-ab` / `--abitrate` | Exact audio bitrate in kbps. | +| `-vb-range` / `--vbitrate-range` | Video bitrate range in kbps, e.g. `6000-7000`; picks the highest within it. | +| `-ab-range` / `--abitrate-range` | Audio bitrate range in kbps, e.g. `128-256`. | + +!!! warning "Exact vs range are mutually exclusive" + `--vbitrate` cannot be combined with `--vbitrate-range`, and likewise for the audio + pair. Pick one form. + +### Real bitrate probing + +Manifest-declared bitrates are sometimes rounded or wrong. Services often advertise a +**peak** or **nominal** bandwidth that is far from the stream's real average. This matters +because a track's bitrate drives the track listing, the sort order, *and* the +`--vbitrate`/`--vbitrate-range` selection above. A bogus declared value therefore makes +unshackle pick the wrong track. These flags remove that risk by probing the actual media +size to compute a true bitrate for the top renditions, overriding the manifest value: + +- `-rvb` / `--real-video-bitrate`: probe real video bitrates (per codec/range). +- `-rab` / `--real-audio-bitrate`: probe real audio bitrates (per codec/channels/ + language). Slower than the video variant because there are more renditions to probe. + +!!! note "Reading the probed numbers" + A single-file track (DASH `SegmentBase`/`BaseURL`) is measured **exactly**. A + multi-segment track (most HLS) is a **sampled estimate**, normally within a few percent + of the true value. For MPEG-TS HLS the probed figure also reads a few percent *above* + the demuxed elementary stream, because the segment bytes include container overhead. + That is the real *delivered* size, not an over-report or a bug. + +!!! tip "Why not every rendition is probed" + Probing does not touch every rendition. Only the top rendition of each quality tier is + probed in parallel (video grouped by codec+range, audio by codec+channels+language), + and the pass then reaches downward only as far as it must to keep the ranking correct. + This keeps probing fast even when a service exposes dozens of renditions. Tracks whose + duration can't be determined fall back to `ffprobe`, and a probe failure is non-fatal: + the manifest value is simply left in place. + +## Audio codec, channels, and Atmos + +### Codec + +`-a` / `--acodec` selects one or more audio codecs (comma-separated); the default is any. + +| Name | Value | Codec | +| --- | --- | --- | +| `AAC` | `AAC` | Advanced Audio Coding | +| `AC3` | `DD` | Dolby Digital | +| `EC3` | `DD+` | Dolby Digital Plus | +| `AC4` | `AC-4` | Dolby AC-4 | +| `OPUS` | `OPUS` | Opus | +| `OGG` | `VORB` | Vorbis | +| `DTS` | `DTS` | DTS | +| `ALAC` | `ALAC` | Apple Lossless | +| `FLAC` | `FLAC` | FLAC | + +Names, values, and a few aliases are accepted: `eac3` and `ddp` both resolve to `EC3`, +and `vorbis` resolves to `OGG`. + +```shell title="Prefer Dolby Digital Plus, fall back to AAC" +unshackle dl -a EC3,AAC NF 81234567 +``` + +### Channels and Atmos + +- `-c` / `--channels`: desired channel layout, e.g. `5.1` or `2.0`. Matching is by + ceiling, so `5.1` implicitly matches a `6.0`-reported layout. +- `-naa` / `--noatmos`: exclude Dolby Atmos audio tracks from selection. + +```shell title="5.1 audio, no Atmos" +unshackle dl -c 5.1 --noatmos AMZN 0ABC123 +``` + +## Languages + +### Video and audio language + +`-l` / `--lang` sets the wanted language(s) for **both video and audio**. The default is +`orig`, the title's original language. + +```shell title="Original language plus English" +unshackle dl -l orig,en NF 81234567 +``` + +The special token `orig` is resolved to the title's actual original language everywhere +it is used. You can override each stream type independently: + +- `-vl` / `--v-lang`: language for **video only** (overrides `-l` for video). Useful + when the burned-in video language differs from the audio you want. +- `-al` / `--a-lang`: language for **audio only** (overrides `-l` for audio). + +```shell title="English audio over the original video" +unshackle dl -al en NF 81234567 +``` + +### Exact vs fuzzy matching + +By default language matching is fuzzy: `-l en` also accepts `en-US`, `en-GB`, etc. (up +to a small distance). Pass `--exact-lang` to require an exact match: with it, `-l es-419` +matches only `es-419`, not `es-ES`. + +!!! tip "The `all` and `best` tokens" + For audio, the tokens `all` and `best` bypass the usual one-track-per-language pick + and instead select the best track for **each** language present. For subtitles, `all` + keeps every subtitle language. + +## Subtitles + +### Selecting subtitle languages + +`-sl` / `--s-lang` sets the wanted subtitle language(s). The default is `all`, meaning +every available subtitle language is downloaded. + +```shell title="Only English and Spanish subtitles" +unshackle dl -sl en,es NF 81234567 +``` + +### Requiring subtitles + +`--require-subs` takes a list of languages that **must** exist. If they all exist, *all* +subtitles are downloaded; if any is missing, the title fails. This is useful for gating a +download on the presence of a specific subtitle track. + +!!! warning "Cannot combine with `--s-lang`" + `--require-subs` and `--s-lang` are mutually exclusive. Use one or the other. + +### Forced subtitles and output format + +- `-fs` / `--forced-subs`: include forced subtitle tracks (signs/foreign dialogue). + Without this flag, forced subs are dropped from selection. +- `--sub-format`: set the output subtitle format, converting only when necessary. + Accepts codec names/values and common aliases (`srt`, `vtt`, `ass`, `ssa`, `ttml`, + etc.), or the literal `original` to keep the source format. + +| Value | Format | +| --- | --- | +| `SRT` / `srt` | SubRip | +| `VTT` / `vtt` | WebVTT | +| `ASS` / `ass` | Advanced SubStation Alpha | +| `SSA` / `ssa` | SubStation Alpha | +| `TTML` / `ttml` | Timed Text Markup | +| `original` | Keep the source format, no conversion | + +```shell title="Convert subtitles to SRT" +unshackle dl --sub-format srt NF 81234567 +``` + +!!! note "SDH stripping happens by default" + When a subtitle track is marked SDH (for the deaf/hard-of-hearing) and there is no + plain same-language subtitle, unshackle produces a stripped, non-SDH variant + automatically. This behaviour is governed by the `subtitle.strip_sdh` config option + (default on). + +## Selecting episodes + +For series, several flags control which episodes are downloaded. By default, **all** +episodes are downloaded. + +### Wanted ranges + +`-w` / `--wanted` accepts flexible season/episode ranges, comma-separated. Prefix a token +with `-` to exclude it. + +=== "Whole seasons" + + ```shell + unshackle dl -w S01 NF 81234567 + unshackle dl -w S01-S03 NF 81234567 + unshackle dl -w S01,S03,S05 NF 81234567 + ``` + +=== "Specific episodes" + + ```shell + unshackle dl -w S01E01 NF 81234567 + unshackle dl -w S01E01-S01E05 NF 81234567 + unshackle dl -w S01E01-S02E03 NF 81234567 + ``` + +=== "Exclusions" + + ```shell + # Seasons 1 through 5, but not season 3 + unshackle dl -w S01-S05,-S03 NF 81234567 + ``` + +### Other selection flags + +| Flag | Behaviour | +| --- | --- | +| `--select-titles` | Interactively pick episodes from a list (Series only). **Cannot combine with `-w`.** | +| `--latest-episode` | Download only the single most recent episode. | +| `--list-titles` | List the titles that would be downloaded, then stop. | + +```shell title="Grab just the newest episode" +unshackle dl --latest-episode NF 81234567 +``` + +## Including and excluding track types + +You can restrict a download to certain track categories, either positively (only these) +or negatively (everything but these). + +**Only these types** (`*-only` flags): + +- `-V` / `--video-only` +- `-A` / `--audio-only` +- `-S` / `--subs-only` +- `-C` / `--chapters-only` + +**Skip these types** (`no-*` flags): + +- `-nv` / `--no-video` +- `-na` / `--no-audio` +- `-ns` / `--no-subs` +- `-nc` / `--no-chapters` + +Additional track-type flags: + +- `-ad` / `--audio-description`: include descriptive (audio-description) tracks, which + are dropped by default. +- `--skip-subtitle-errors`: if a subtitle fails to download, skip it and continue rather + than aborting the whole title. Video and audio failures remain fatal. + +```shell title="Subtitles only" +unshackle dl -S -sl en NF 81234567 +``` + +```shell title="Everything except chapters" +unshackle dl -nc NF 81234567 +``` + +!!! note + You can combine an `*-only` flag with `no-*` flags to fine-tune: `*-only` chooses the + starting set of categories, then `no-*` subtracts from it. Attachments (e.g. fonts) + are always kept. + +## Listing and dry runs + +Before committing to a long download, inspect what unshackle *would* do: + +| Flag | Effect | +| --- | --- | +| `--list` | List the available tracks and which would be selected, then stop. No download. | +| `--list-titles` | List the titles (episodes) that would be downloaded, then stop. | +| `--skip-dl` | Skip downloading but still acquire the decryption keys. | + +```shell title="See the track selection without downloading" +unshackle dl -q 1080 -v H.265 -r HDR10 --list NF 81234567 +``` + +## Output and muxing + +### Output location and folders + +- `-o` / `--output`: override the output directory for this run (otherwise the + configured downloads directory is used). +- `--no-folder`: do not create a per-show folder for TV downloads. +- `--no-source`: remove the service source tag from the filename and path. + +```shell title="Send this download somewhere specific" +unshackle dl -o ~/Videos/incoming NF 81234567 +``` + +### Muxing behaviour + +By default, tracks are muxed into a single Matroska (`.mkv`) file with `mkvmerge`. These +flags change how the output is assembled: + +| Flag | Behaviour | Default source | +| --- | --- | --- | +| `--no-mux` | Do not mux; keep the individual track files. | — | +| `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) | +| `--merge-video` | Mux all selected video tracks into one file. | config `muxing.merge_video` (off) | + +### Naming tags + +- `--tag`: set the release group tag (overrides the configured tag). +- `--repack`: add a `REPACK` tag to the output filename. + +```shell title="Custom group tag and a REPACK label" +unshackle dl --tag MYGRP --repack NF 81234567 +``` + +## Proxies + +`--proxy` accepts a full proxy URI, a 2-letter country code (resolved through your +configured proxy providers), or a `provider:region` form. + +```shell title="Proxy forms" +unshackle dl --proxy us NF 81234567 +unshackle dl --proxy nordvpn:ca NF 81234567 +unshackle dl --proxy 'http://user:pass@host:8080' NF 81234567 +``` + +Two related flags: + +- `--no-proxy`: force-disable all proxy use for this run. +- `--no-proxy-download`: bypass the proxy for **segment downloads only**. The manifest, + license, and authentication requests still go through the proxy. This is useful when the + proxy is only needed to satisfy geo-checks, not to move the bulk of the data. + +## Performance and caching + +| Flag | Purpose | +| --- | --- | +| `--workers N` | Threads used per track for segment downloads. Default depends on the downloader. | +| `--downloads N` | Number of tracks downloaded concurrently. Default `1`. | +| `--slow [MIN-MAX]` | Add a delay between titles to look more like a real device. `--slow` alone means 60-120s; `--slow 20-40` sets a custom range. Minimum 20s. | +| `--no-cache` | Bypass the title cache for this download. | +| `--reset-cache` | Clear the title cache before fetching. | + +```shell title="Two tracks at a time, eight threads each" +unshackle dl --downloads 2 --workers 8 NF 81234567 +``` + +```shell title="Space out a season download" +unshackle dl -w S01 --slow 30-60 NF 81234567 +``` + +## Keys, vaults, and export + +By default, unshackle checks your **key vaults** first and only asks a **CDM** to license +a key when the vault misses. You can force one side or the other: + +- `--cdm-only`: only use the CDM (skip vaults). +- `--vaults-only`: only use key vaults (never license via the CDM); a missing key fails. + +```shell title="Fetch keys only, no download" +unshackle dl --skip-dl NF 81234567 +``` + +`--export` writes a JSON file, into the configured exports directory, containing track +info and the acquired content keys for each title. This is the format consumed by +`unshackle import` to reconstruct a download later. + +```shell title="Export track info and keys" +unshackle dl --skip-dl --export NF 81234567 +``` + +!!! note "Region is recorded only with a proxy" + When `--proxy` is used, the export records the region so an import can reproduce the + correct geofence. Without a proxy, no region is stored. + +## Metadata and tagging + +unshackle looks up metadata automatically, but you can override the identifiers used for +tagging and naming: + +| Flag | Example | Purpose | +| --- | --- | --- | +| `--tmdb` | `--tmdb 27205` | Use this TMDB ID instead of automatic lookup. | +| `--imdb` | `--imdb tt1375666` | Use this IMDb ID. | +| `--animeapi` | `--animeapi mal:12345` | Resolve via AnimeAPI (`mal:`/`anilist:` prefix; defaults to MAL). | +| `--enrich` | — | Override the show title and year from an external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | + +```shell title="Force the right IMDb match and enrich the title" +unshackle dl --imdb tt1375666 --enrich NF 81234567 +``` + +## Configuration defaults + +Every flag on this page can be set as a default in your configuration file so you don't +have to type it each time, and defaults can be scoped per service. See +[Configuration file](../getting-started/configuration-file.md) for the `dl:` block and +per-service overrides. + +!!! tip "Explicit flags always win" + A value you pass on the command line (or via an environment variable) always takes + precedence over both the global `dl:` defaults and any per-service `dl:` overrides. + Config only fills in options you didn't set explicitly. + +## Quick reference + +A condensed lookup of the most-used options. Run `unshackle dl --help` for the complete, +authoritative list. + +| Flag | Short | Meaning | +| --- | --- | --- | +| `--quality` | `-q` | Target resolution(s), e.g. `1080,720`. | +| `--vcodec` | `-v` | Video codec(s). | +| `--acodec` | `-a` | Audio codec(s). | +| `--range` | `-r` | Color range(s); default `SDR`. | +| `--channels` | `-c` | Audio channel layout. | +| `--noatmos` | `-naa` | Exclude Atmos audio. | +| `--lang` | `-l` | Video + audio language(s); default `orig`. | +| `--a-lang` | `-al` | Audio-only language override. | +| `--v-lang` | `-vl` | Video-only language override. | +| `--s-lang` | `-sl` | Subtitle language(s); default `all`. | +| `--forced-subs` | `-fs` | Include forced subtitles. | +| `--sub-format` | | Output subtitle format. | +| `--wanted` | `-w` | Episode/season range. | +| `--select-titles` | | Interactively pick episodes. | +| `--latest-episode` | | Only the newest episode. | +| `--video-only` / `--audio-only` / `--subs-only` | `-V` / `-A` / `-S` | Restrict track types. | +| `--no-video` / `--no-audio` / `--no-subs` / `--no-chapters` | `-nv` / `-na` / `-ns` / `-nc` | Skip track types. | +| `--worst` | | Lowest bitrate within `-q`. | +| `--best-available` | | Degrade gracefully instead of failing. | +| `--output` | `-o` | Output directory for this run. | +| `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. | +| `--proxy` / `--no-proxy` / `--no-proxy-download` | | Proxy control. | +| `--workers` / `--downloads` / `--slow` | | Concurrency and pacing. | +| `--list` / `--list-titles` / `--skip-dl` | | Dry runs. | +| `--cdm-only` / `--vaults-only` | | Key source control. | +| `--export` | | Export track info and keys to JSON. | +| `--tmdb` / `--imdb` / `--animeapi` / `--enrich` | | Metadata overrides. | diff --git a/docs/guide/drm-and-cdm.md b/docs/guide/drm-and-cdm.md new file mode 100644 index 00000000..4b8438b3 --- /dev/null +++ b/docs/guide/drm-and-cdm.md @@ -0,0 +1,524 @@ +# DRM & CDM Setup + +Most commercial streaming content is protected by a Digital Rights Management (DRM) +system. To download and play it back, unshackle needs to obtain the **content keys** +for the encrypted tracks and then decrypt them. This page explains the DRM systems +unshackle supports, how to provide a device so it can talk to a license server, the +difference between local and remote CDMs, and how to choose a decryption backend. + +!!! note "Who this page is for" + This is a practical configuration guide for end users of the unshackle CLI. + A few sections are marked **Developer**. They describe internal behaviour that is + useful when writing a service, but you do not need them to download protected + content. + +## Key concepts + +Before configuring anything, it helps to know the vocabulary. unshackle draws a firm +line between two things: + +- A **CDM** (Content Decryption Module) is the trusted "black box" that turns a + *license challenge* into *content keys*. It is backed by a **device**, a file + containing the cryptographic identity of a real client (a phone, a browser, etc.). +- A **DRM system** is the wrapper around a CDM that parses the protection metadata, + drives the challenge → license → keys handshake, and finally invokes an external + tool to physically decrypt the downloaded file. + +A handful of other terms show up throughout: + +| Term | Meaning | +|------|---------| +| **PSSH** | Protection System Specific Header. The per-system init data extracted from the file's init segment or the manifest. | +| **KID** | Key ID: a 16-byte UUID identifying *which* key a track needs. | +| **CEK** | Content Encryption Key: the actual AES key, mapped from its KID. | +| **Challenge** | The license request the CDM produces, sent to the service's license server. | +| **License** | The server's response, which the CDM parses to recover the content keys. | + +For a Widevine or PlayReady title, the flow is: unshackle reads the PSSH and KIDs +from the track, asks the CDM for a license challenge, sends that challenge to the +service's license server, hands the response back to the CDM to extract the keys, and +then decrypts the file with those keys. + +## Supported DRM systems + +unshackle implements five DRM systems. Only the first two need a device file and a +license-server handshake; the rest are special cases. + +| System | Device file | License flow | Decryption | +|--------|-------------|--------------|------------| +| **Widevine** (Google) | `.wvd` | CDM challenge → license server → CDM parses → keys | shaka-packager / mp4decrypt | +| **PlayReady** (Microsoft) | `.prd` | CDM challenge (WRM header) → license server → CDM parses → keys | shaka-packager / mp4decrypt | +| **ClearKey (CENC)**, W3C EME `org.w3.clearkey` | none | JSON `{"kids":[…],"type":"temporary"}` → license server returns a JWK Set. No CDM. | shaka-packager / mp4decrypt | +| **ClearKey (HLS)**, AES-128 `EXT-X-KEY` | none | Key fetched directly from a URI or `data:` URL. No CDM, no challenge. | in-process AES-CBC | +| **MonaLisa** (proprietary, WASM) | `.mld` | Ticket from the service API; keys extracted locally via WASM. No license server. | ML-Worker binary + AES-ECB, per segment | + +!!! warning "Two different ClearKeys" + "ClearKey (HLS)" and "ClearKey (CENC)" are entirely different systems that happen + to share a name. The HLS variant is the AES-128 key referenced by an `EXT-X-KEY` + tag and is decrypted in-process. The CENC variant is the W3C EME + `org.w3.clearkey` scheme, delivers keys as a JWK Set, and is decrypted through + shaka-packager or mp4decrypt just like Widevine. Neither involves a device file. + +### Widevine vs PlayReady + +These are the two "real" DRMs, and in unshackle they behave almost identically: the +same PSSH/KID handling, the same decryption backends, and the same key-caching flow. +The differences that matter to you are: + +- **Device file**: Widevine uses a `.wvd`; PlayReady uses a `.prd`. +- **Availability**: some services offer only one system, some offer both. When a + manifest carries both, unshackle picks the appropriate CDM for each track (see + [Widevine and PlayReady on the same title](#widevine-and-playready-on-the-same-title)). + +To download a Widevine-protected title you need a `.wvd` device; for PlayReady you +need a `.prd` device. You can hold both and let unshackle choose, or force one with +the `--drm` flag. + +!!! info "Obtaining device files" + A `.wvd` or `.prd` encodes the private keys of a real client device. unshackle + does **not** ship with one and cannot generate the underlying secret material for + you. You must supply your own. The `wvd` and `prd` commands below manage and + provision device files once you have the raw key material. + +## Providing a device with the `wvd` and `prd` commands + +Device files live in dedicated directories, by default: + +| Kind | Default directory | +|------|-------------------| +| Widevine `.wvd` | `data/WVDs` | +| PlayReady `.prd` | `data/PRDs` | + +You can move these with the `directories` section of your `unshackle.yaml`: + +```yaml title="unshackle.yaml" +directories: + wvds: /home/george/unshackle-data/WVDs + prds: /home/george/unshackle-data/PRDs +``` + +### Widevine devices: the `wvd` command + +The `wvd` command group manages `.wvd` files. + +=== "Add an existing device" + + ```console + $ unshackle wvd add ./nexus_6p.wvd + ``` + + The file is validated and moved into the WVDs directory. You can add several at + once by passing multiple paths. + +=== "Inspect a device" + + ```console + $ unshackle wvd parse nexus_6p + ``` + + Prints the System ID, Security Level, Type, Flags, and whether a Private Key, + Client ID, and VMP are present. A bare name (no extension) is resolved inside the + WVDs directory; a path with an extension is used as-is. + +=== "Create from raw material" + + ```console + $ unshackle wvd new "Nexus 6P" ./private_key.pem ./client_id.bin ./vmp.bin -t ANDROID -l 3 + ``` + + Builds a `.wvd` from a PEM private key, a Client ID blob, and an optional + FileHashes (VMP) blob. `-t/--type` is the device type (default `Android`) and + `-l/--level` is the security level 1 to 3 (default `1`). The VMP argument is + optional. + +=== "Delete / dump" + + ```console + $ unshackle wvd delete nexus_6p + $ unshackle wvd dump nexus_6p ./out + ``` + + `delete` removes a device (with a confirmation prompt). `dump` extracts a + device's contents (metadata, private key, Client ID, VMP) into a folder. + +!!! warning "v1 WVD files must be migrated" + If loading a device fails with a message about a "v1 WVD file", it was made with + an older format. Migrate it with pywidevine before use: + + ```console + $ pywidevine migrate --help + ``` + +### PlayReady devices: the `prd` command + +The `prd` command group creates and maintains `.prd` files. + +=== "Create a new device" + + ```console + $ unshackle prd new ./zgpriv.dat ./bgroupcert.dat + ``` + + Provision a new device from a group key and group certificate. You can instead + point at a single folder that contains `zgpriv.dat` and `bgroupcert.dat`: + + ```console + $ unshackle prd new ./device_folder/ + ``` + + A fresh leaf certificate is generated and the device is written to the PRDs + directory (or to `-o/--output`). Use `-e/--encryption_key` and `-s/--signing_key` + to supply your own ECC keys instead of generating them. + +=== "Reprovision" + + ```console + $ unshackle prd reprovision ./data/PRDs/mydevice.prd + ``` + + Regenerates the leaf certificate and encryption/signing keys of an existing + device in place. The device must support reprovisioning (a group key must be + present, i.e. a version 3 or higher device). + +=== "Test against Microsoft's demo server" + + ```console + $ unshackle prd test ./data/PRDs/mydevice.prd -sl 2000 -c aesctr + ``` + + Runs a full challenge/license/keys cycle against Microsoft's public PlayReady + test server and prints the returned keys, confirming the device works. + `-sl/--security-level` accepts `150`, `2000`, or `3000` (default `2000`); + `-c/--ckt` accepts `aesctr` or `aescbc`. + +## Selecting which device to use: the `cdm` config + +Having a device on disk is not enough; you must tell unshackle which device to use +for which service. That is the job of the `cdm` section, which maps a service tag to +a device name (the file's name without extension). A `default` entry acts as the +fallback: + +```yaml title="unshackle.yaml" +cdm: + default: nexus_6p_26830_l3 + AMZN: chromecdm_l3 + NF: sm_g935f_l1 +``` + +Lookups are case-insensitive. On the command line you can override the configured +device for a single run: + +```console +$ unshackle dl -w chromecdm_l3 AMZN B0ABCDEF +``` + +!!! tip "The device name is the file name" + A device named `chromecdm_l3` in the config resolves to `chromecdm_l3.wvd` in the + WVDs directory (or `chromecdm_l3.prd` in the PRDs directory). It may also be the + `name` of a remote CDM defined in `remote_cdm`. See + [Remote CDMs](#remote-cdms) below. + +### Advanced selection: by quality, DRM, or profile + +Instead of a plain name, a `cdm` entry can be a nested mapping, letting unshackle pick +a different device based on the track quality, the DRM system, or the active service +profile. + +=== "By quality" + + Keys are matched against the video height. You can use exact heights or + comparisons (`>=1080`, `>720`, `<=576`, `<480`). More specific matches win. + + ```yaml + cdm: + SOMESERVICE: + ">=1080": sm_g935f_l1 + default: chromecdm_l3 + ``` + +=== "By DRM system" + + Keys `widevine` and `playready` select a device per system. This pairs with the + `--drm` flag, which accepts `wv`/`widevine` or `pr`/`playready`. + + ```yaml + cdm: + SOMESERVICE: + widevine: sm_g935f_l1 + playready: mydevice_sl3 + ``` + +=== "By profile" + + Keys are matched against the active profile, with a `default` fallback. + + ```yaml + cdm: + SOMESERVICE: + premium: sm_g935f_l1 + default: chromecdm_l3 + ``` + +## Local vs remote CDMs + +A CDM can run in two places: + +- A **local CDM** loads a `.wvd` or `.prd` device from your machine and performs the + challenge/license/keys handshake locally. This is what the `cdm` mapping above + resolves to by default. +- A **remote CDM** delegates the CDM operations to an HTTP API. Your device (and its + secrets) live on the remote server; unshackle sends it PSSH/challenge data and + receives keys back. This is useful for sharing a high-security-level device, using + a hosted key service, or keeping device secrets off your download machine. + +The device name in your `cdm` mapping resolves to a remote CDM whenever it matches the +`name` of an entry in the `remote_cdm` list. Otherwise it falls back to a local +`.prd`/`.wvd` file of that name. + +## Remote CDMs + +Remote CDMs are declared as a list under `remote_cdm`. Each entry has a `name` (which +you then reference from the `cdm` mapping) and a shape that depends on its `type`. +There are four kinds. + +### Widevine remote CDM (pywidevine serve) + +With no `type` field and a non-PlayReady device type, the entry is treated as a +standard pywidevine "serve" endpoint: + +```yaml title="unshackle.yaml" +remote_cdm: + - name: my_wv_remote + device_type: ANDROID + system_id: 26830 + security_level: 3 + host: https://cdm.example.com + secret: your-api-secret + device_name: sm_g935f_l1 + +cdm: + default: my_wv_remote +``` + +`security_level` defaults to `3000` if omitted. Field names are read +case-insensitively, so `Device Type`/`device_type`, `System ID`/`system_id`, etc. are +all accepted. + +### PlayReady remote CDM + +If the device type is `PLAYREADY`, the entry is routed to the pyplayready remote CDM: + +```yaml title="unshackle.yaml" +remote_cdm: + - name: my_pr_remote + device_type: PLAYREADY + security_level: 3000 + host: https://cdm.example.com + secret: your-api-secret + device_name: mydevice_sl3 +``` + +!!! warning "The `host` must end in `/playready`" + For a PlayReady remote CDM the `host` URL has to include the trailing + `/playready` path segment (e.g. `https://cdm.example.com/playready`). pyplayready's + `RemoteCdm` treats `host` as a base and appends its own endpoint paths on top of it, + so a host without the suffix points the requests at the wrong location. The symptom + is `404` responses from the server rather than an obvious configuration error. + +### Decrypt Labs (hosted KeyXtractor API) + +Set `type: decrypt_labs` to use Decrypt Labs' hosted KeyXtractor service, which +includes an intelligent key-caching layer: + +```yaml title="unshackle.yaml" +remote_cdm: + - name: decryptlabs + type: decrypt_labs + device_name: L1 # ChromeCDM, L1, L2 (Widevine) or SL2, SL3 (PlayReady) + # host: https://keyxtractor.decryptlabs.com # optional, this is the default + # secret: ... # optional if decrypt_labs_api_key is set globally + +decrypt_labs_api_key: your-decrypt-labs-key +``` + +The API key is sent as the `decrypt-labs-api-key` header. If you omit `secret` on the +entry, unshackle falls back to the global `decrypt_labs_api_key` value; if neither is +present it raises an error. The `host` defaults to +`https://keyxtractor.decryptlabs.com`. PlayReady devices (`SL2`/`SL3`) default to +security level `2000`/`3000` respectively; Widevine devices default to +`system_id: 26830`, `security_level: 3`. + +### Custom API (`custom_api`) + +For an arbitrary CDM HTTP API, `type: custom_api` provides a fully configurable client +that adapts to almost any request/response shape through YAML alone. A minimal example: + +```yaml title="unshackle.yaml" +remote_cdm: + - name: my_custom + type: custom_api + device: + name: ChromeCDM + type: CHROME # CHROME, ANDROID, or PLAYREADY + system_id: 26830 + security_level: 3 + auth: + type: bearer # header, bearer, or basic + key: your-token + endpoints: + get_request: + path: /get-challenge + method: POST + decrypt_response: + path: /get-keys + method: POST + timeout: 30 +``` + +The `custom_api` type supports much more: request/response field remapping, +transforms (base64/hex/JSON encoding, `kid:key` parsing), conditional parameters, +success/error detection, and vault-backed caching. Its full schema is aimed at +developers integrating a bespoke key server; see the developer note below. + +!!! note "Developer: full `custom_api` schema" + A `custom_api` entry accepts these sub-objects: `device` (`name`, `type`, + `system_id`, `security_level`); `auth` (`type` of `header`/`bearer`/`basic` with + the matching credential fields, plus `custom_headers`); `endpoints` + (`get_request` and `decrypt_response`, each `{path, method, timeout}`); + per-endpoint `request_mapping` (`param_names`, `static_params`, + `conditional_params`, `transforms`, `nested_params`, `exclude_params`) and + `response_mapping` (`fields`, `transforms`, `response_types`, + `success_conditions`, `error_fields`, `key_fields`); `caching` (`enabled`, + `use_vaults`, `check_cached_first`); and `legacy`/`timeout`. PlayReady mode is + inferred when `device.type == PLAYREADY` or the device name is `SL2`/`SL3`. + +## Decryption backends + +Once the content keys are known, the encrypted file (for Widevine, PlayReady, and +ClearKey-CENC) is decrypted by one of two external tools: + +- **shaka-packager** is the default. +- **mp4decrypt** is part of the Bento4 tools. + +Both must be discoverable in unshackle's bundled `binaries/` directory or on your +`PATH`. If the selected tool's binary is missing at decrypt time, unshackle raises an +"executable not found but is required" error. See [dependencies](../getting-started/installation.md) for +installation. + +You choose the backend with the `decryption` key. It can be a single value applied +globally, or a per-service mapping: + +=== "Global default" + + ```yaml title="unshackle.yaml" + decryption: shaka + ``` + +=== "Per service" + + ```yaml title="unshackle.yaml" + decryption: + default: shaka + SOMESERVICE: mp4decrypt + ``` + +!!! info "How the value is interpreted" + The default is `shaka`. Only the exact value `mp4decrypt` selects mp4decrypt; + **anything else** (including `shaka`) selects shaka-packager. Per-service lookups + are case-insensitive, with the `default`/`DEFAULT` entry used as the fallback. + +The two backends produce equivalent output for standard CENC content. shaka-packager +is generally preferred; switch a specific service to `mp4decrypt` if you hit a track +that shaka-packager cannot handle. + +!!! note "ClearKey (HLS) does not use these tools" + HLS AES-128 ClearKey content is decrypted in-process using AES-CBC and ignores the + `decryption` setting entirely. There is nothing to configure. + +## ClearKey + +### ClearKey over HLS (AES-128) + +When an HLS playlist references a key with an `EXT-X-KEY` tag, unshackle fetches the +key directly (from the given URI or an inline `data:` URL) and decrypts each segment +in-process with AES-CBC. There is no device, no CDM, and no license challenge, so +there is nothing to set up. Such tracks are handled automatically during download. + +### ClearKey over CENC (W3C EME `org.w3.clearkey`) + +The CENC ClearKey scheme is a proper license flow, but the "license server" simply +returns the keys as a JWK Set rather than a DRM-wrapped blob. unshackle builds a W3C +EME request of the form: + +```json +{"kids": ["<base64url-kid>", …], "type": "temporary"} +``` + +It sends this to the manifest's license URL (or to whatever a service's ClearKey +license hook returns) and reads the JSON Web Key set back. No device file is needed. +The decrypted content still goes through shaka-packager or mp4decrypt like any other +CENC track, so the `decryption` setting applies. + +## Widevine and PlayReady on the same title + +A single manifest can advertise both Widevine and PlayReady. unshackle picks the CDM +per track: if a track needs a system that your loaded CDM does not provide, it +transparently switches to the appropriate device for that track using your `cdm` +mapping. If it cannot find a matching device, it fails with a clear message such as: + +``` +Title needs a Widevine CDM but SOMESERVICE is configured with PlayReady. +``` + +Use the `cdm` mapping's [DRM-based selection](#advanced-selection-by-quality-drm-or-profile) +to configure a device for each system, or force one system with `--drm` on the +command line. + +## Keys, vaults, and caching + +Fetching a license is comparatively slow and, for some services, rate-limited. +unshackle therefore resolves each key ID in this order before ever contacting a +license server: + +1. Keys already known for the current DRM object. +2. An in-process license-key cache (for the running session). +3. Your configured [key vaults](vaults.md). +4. Only if still missing, a live license request. + +After a successful license request, the recovered keys are written back to all +configured vaults so future downloads can skip the license server. Two flags let you +control this trade-off on the `dl` command: + +| Flag | Effect | +|------|--------| +| `--vaults-only` | Never send a license request; use only cached/vault keys. Fails if a key is missing. | +| `--cdm-only` | Ignore vaults for sourcing keys and always perform a fresh license request. | + +See [key vaults](vaults.md) for how to configure vault backends. + +## Debugging DRM issues + +A few configuration toggles help when a title will not decrypt: + +| Config key | Default | Effect | +|------------|---------|--------| +| `debug` | `false` | Global debug output. | +| `debug_keys` | `false` | Log decryption keys. | +| `debug_requests` | `false` | Log HTTP requests. | + +```yaml title="unshackle.yaml" +debug_keys: true +debug_requests: true +``` + +!!! warning "Keys and paths in logs" + When key logging is enabled, content keys (KID:key pairs) are written to the logs. + Treat those logs as sensitive. The unrelated `redact_paths` option (default + `true`) masks local directory paths in log output. + +## See also + +- [Downloading](downloading.md): the `dl` command and its DRM-related flags. +- [Key vaults](vaults.md): caching content keys across downloads. +- [Configuration](../reference/configuration.md): the full `unshackle.yaml` reference. +- [Dependencies](../getting-started/installation.md): installing shaka-packager and mp4decrypt. diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md new file mode 100644 index 00000000..3c8665d1 --- /dev/null +++ b/docs/guide/output-and-naming.md @@ -0,0 +1,384 @@ +# Output & Naming + +This page explains where unshackle writes finished downloads and how it names them. You control naming with **output templates** (the filename) and **folder templates** (the directory the file lands in), using a set of `{variable}` placeholders that unshackle fills in from the media it just downloaded. It also covers muxing options, release-group tagging, Unicode filenames, and how to relocate unshackle's working directories. + +If you have not configured `unshackle.yaml` yet, read [Configuration File](../getting-started/configuration-file.md) first. Everything below lives in that file. + +## How an output path is assembled + +When a download finishes muxing, unshackle builds the final path from three pieces: + +```text +<output directory> / <folder template> / <filename template><extension> +``` + +- **Output directory**: `directories.downloads` from your config, or whatever you pass to `-o/--output` on the command line for a single run. +- **Folder template**: an optional per-title subfolder. TV episodes and music tracks are **always** placed in a folder; movies only get one if you define a `movies` folder template (see [Folder templates](#folder-templates)). +- **Filename template**: the `output_template` for the title kind (movie, series, or song). +- **Extension**: chosen by the muxer: `.mkv` for video, `.mka` for audio-only, `.mks` for subtitle-only. + +!!! example "A typical episode path" + ```text + /home/user/Downloads/ + └─ Show Name S01 1080p AMZN WEB-DL DDP5.1 H.264/ + └─ Show.Name.S01E01.Pilot.1080p.AMZN.WEB-DL.DDP5.1.H.264-TAG.mkv + ``` + The folder name comes from the `series` folder template and the file from the `series` output template. + +Two command-line flags change this layout for a single run: + +| Flag | Effect | +|---|---| +| `-o`, `--output` | Use this directory instead of `directories.downloads`. | +| `--no-folder` | Skip the per-title subfolder; write the file directly into the output directory. | +| `--no-source` | Omit the service source tag from both the filename and folder (the `{source}` variable resolves to empty). | + +## Output templates + +Filename templates live under the `output_template` key, keyed by title kind. Each value is a format string built from `{variable}` placeholders. + +```yaml title="unshackle.yaml" +output_template: + movies: "{title}.{year}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}" + series: "{title}.{season_episode}.{episode_name?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}" + songs: "{track_number}. {title}" +``` + +The recognized kinds are: + +| Kind | Applies to | Default when unset | +|---|---|---| +| `movies` | Movies | *(required for movie downloads to be named, no built-in default)* | +| `series` | TV episodes | *(required for episode downloads to be named)* | +| `songs` | Music tracks | `"{track_number}. {title}"` | +| `albums` | Album **folders** only (see below) | — | + +!!! note + `movies` and `series` have no hardcoded filename fallback. If you download those title types, define their templates. `songs` falls back to `"{track_number}. {title}"` when omitted. + +### Required vs. optional variables + +- `{variable}`: **required**. If it is missing from the download's context, formatting raises an error. (Note that most variables are always present in context but resolve to an *empty string* when not applicable, so being "present but empty" is different from "missing".) +- `{variable?}`: **optional**. A trailing `?` marks the variable conditional. When it resolves to empty, unshackle removes the placeholder *and* one adjacent separator (`.`, `-`, or space), so the surrounding text collapses cleanly. + +!!! tip "Use `?` for anything that isn't always there" + Episode names, editions, tags, HDR labels and audio extras aren't present on every title. Mark them optional so you never get doubled dots or dangling separators: + + ```yaml + series: "{title}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}-{tag?}" + ``` + For an episode with no on-screen name, `{episode_name?}` and its adjacent dot simply disappear. + +### Separator style (dots vs. spaces) + +unshackle looks at the characters you place *between* variables to decide the filename's separator style. If spaces outnumber dots in your template, it sanitizes with spaces; otherwise it uses dots (the scene-style default). This is also applied to the auto-generated date separators for daily/dated content. + +=== "Scene style (dots)" + + ```yaml + output_template: + movies: "{title}.{year}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}" + ``` + ```text + The.Movie.2024.1080p.NF.WEB-DL.DDP5.1.H.264-TAG.mkv + ``` + +=== "Readable style (spaces)" + + ```yaml + output_template: + movies: "{title} ({year}) {quality} {source} WEB-DL {audio_full} {video}" + ``` + ```text + The Movie (2024) 1080p NF WEB-DL DDP5.1 H.264.mkv + ``` + +## Template variables + +Every variable below is valid in both output and folder templates. Values are derived from the downloaded media (via MediaInfo), from the title metadata, or from your config. Variables that don't apply to a given title resolve to an empty string. + +### Common / video + +| Variable | Meaning | Example | +|---|---|---| +| `title` | Title name (movie/show/song name; `$` is rendered as `S`) | `The Show` | +| `year` | Release year | `2024` | +| `source` | Service tag / class name (empty with `--no-source`) | `NF` | +| `quality` | Resolution with scan suffix | `1080p`, `2160p`, `576i` | +| `resolution` | Resolution number only | `1080` | +| `video` | Video codec | `H.264`, `H.265` | +| `hdr` | Dynamic-range label | `HDR`, `HDR10P`, `DV`, `DV.HDR10`, `HLG` | +| `hfr` | High frame rate marker (frame rate > 30) | `HFR` | +| `edition` | Edition label from the track, if any | `Directors Cut` | +| `tag` | Release-group tag (from `config.tag` or `--tag`) | `TAG` | +| `repack` | `REPACK` when the `--repack` flag is used | `REPACK` | +| `lang_tag` | Result of your `language_tags` rules, if configured | `MULTi` | + +!!! note "How `language_tags` rules match" + Rules are tried in order, but language comparison is **fuzzy**: a rule condition of `en` matches `en-US`, `en-GB`, and any other `en-*` variant. Within a single rule, **all** conditions must match (AND logic) for it to apply. The first rule that fully matches wins; if none match, `{lang_tag?}` is removed from the filename cleanly. + +### Episode-specific + +| Variable | Meaning | Example | +|---|---|---| +| `season` | Zero-padded season (`S%02d`) | `S01` | +| `episode` | Zero-padded episode (`E%02d`) | `E05` | +| `season_episode` | Combined | `S01E05` | +| `episode_name` | Episode title | `Pilot` | +| `date` | ISO air date for daily/dated content | `2024-06-01` | + +!!! note "Daily & sports content" + When an episode has an air date, unshackle switches to date-based naming automatically: `season` and `season_episode` become the formatted air date, `episode` and `year` are cleared, and `{date}` holds the ISO date. The date's internal separator (dots or spaces) follows your `series` template style. + +### Audio + +| Variable | Meaning | Example | +|---|---|---| +| `audio` | Audio codec | `DDP`, `DD`, `AAC` | +| `audio_channels` | Channel layout | `5.1`, `2.0` | +| `audio_full` | Codec + channels combined | `DDP5.1` | +| `atmos` | `Atmos` when any track carries JOC | `Atmos` | +| `dual` | `DUAL` for exactly two distinct audio languages | `DUAL` | +| `multi` | `MULTi` for more than two audio languages | `MULTi` | + +### Music-specific + +| Variable | Meaning | +|---|---| +| `track_number` | Zero-padded track number | +| `artist` / `album_artist` | Artist / album artist (falls back to artist) | +| `album` | Album name | +| `disc` | Disc number (only shown when greater than 1) | +| `track_total` / `disc_total` | Totals | +| `release_type` | `album`, `single`, `ep`, etc. | +| `genre` | Genre | +| `explicit` | `Explicit` when flagged | +| `isrc` / `upc` / `label` | ISRC, UPC and record label | + +!!! warning "Unknown variables are ignored, not errors" + If you use a variable that isn't in the list above, unshackle emits a warning at startup but continues. Double-check spelling. A typo'd `{quailty}` will simply be left in the filename literally or dropped, not filled in. + +## Folder templates + +Folder templates are nested under a special `folder` key inside `output_template`. This key is handled separately from the filename kinds. You can give it a single string (used for all title kinds) or a per-kind map. + +=== "Single folder template" + + ```yaml title="unshackle.yaml" + output_template: + movies: "{title}.{year}.{quality}.{source}-{tag}" + series: "{title}.{season_episode}.{quality}.{source}-{tag}" + folder: "{title}.{year}" + ``` + +=== "Per-kind folder templates" + + ```yaml title="unshackle.yaml" + output_template: + movies: "{title}.{year}.{quality}.{source}-{tag}" + series: "{title}.{season_episode}.{quality}.{source}-{tag}" + folder: + movies: "{title} ({year})" + series: "{title} ({year})/Season {season}" + albums: "{artist} - {album} ({year})" + ``` + +The per-kind folder keys are `movies`, `series`, `songs`, and `albums`. Any other key produces a startup warning. Path separators (`/` or `\`) are allowed in folder templates. Each segment is formatted independently, so you can build nested directory structures like `Show/Season 01`. + +**Fallback behavior:** + +- A per-kind folder template wins if present; otherwise the single `folder` string is used; otherwise unshackle falls back to a built-in default. +- Built-in defaults when no folder template is set: movies get no folder at all unless a `movies` folder template exists; series fall back to a folder *derived* from the `series` output template (stripping `{episode}`, `{episode_name}`, and collapsing `{season_episode}` down to `{season}`); music albums fall back to `{artist} - {album} ({year})`. + +!!! note "Movies are flat by default" + Episodes and songs are always foldered. Movies are written directly into the output directory *unless* you define a `movies` folder template. Set one if you want each movie in its own directory. + +## Muxing options + +Muxing (combining video, audio, subtitle, chapter and attachment tracks into a single Matroska file with `mkvmerge`) is configured under the `muxing` key. + +```yaml title="unshackle.yaml" +muxing: + set_title: true + merge_video: true + merge_audio: true + default_language: + audio: en + subtitle: en +``` + +| Key | Type | Default | Effect | +|---|---|---|---| +| `set_title` | bool | `true` | Write the title name into the MKV container title with `--title`. Set to `false` to omit it. | +| `merge_video` | bool | `true` | Group video tracks that share the same resolution, range, and codec into one file so only language varies inside it. | +| `merge_audio` | bool | `true` | Merge audio tracks of the same kind so multiple languages sit in one file. | +| `default_language` | map | *(unset)* | Preferred language per track type (`video` / `audio` / `subtitle`). A track in the preferred language is flagged as the default track. | + +When no preferred language is matched, unshackle applies sensible defaults: the video default falls back to the title language / original-language track / first track; the audio default is the original-language track; and the subtitle default is a forced track matching the first audio's language. + +!!! note "`default_language` only sets the default-track flag" + `default_language` controls **which track carries the MKV `--default-track` flag**. Nothing else. It does not change track *selection* (that stays with `-l`/`--alang` and friends), and it does not touch which track is marked as original: `--original-flag` still tags the *true* original-audio track. That is the point: you can make your player default to, say, Polish audio on an English-original title without altering the original marker. When the configured language isn't present in the manifest, each track type falls back to its normal default rule described above. + +!!! warning "`merge_video` collapses only the language dimension" + `merge_video` groups video tracks by `(resolution, range, codec)` and merges them so that only **language** differs within a single file, with no re-encode and no concatenation. Different resolutions, ranges (SDR / HDR10 / HDR10+ / DV / HYBRID), and codecs (H.264 / H.265) **always** land in separate files. So `-r HYBRID,DV,HDR10,SDR --merge-video` produces one file *per range* (never a single fused file), whereas English + French video of identical resolution, range, and codec yields one file containing both video tracks. `merge_audio` works the same way for audio languages. + +A few muxing behaviors are automatic and not configurable: + +- `--no-date` is always passed for privacy (no timestamps embedded). +- Descriptive audio gets the visually-impaired flag; SDH subtitles get the hearing-impaired flag; forced subtitles get the forced flag. +- Chapters are written from the title's chapter list (see [Chapters](#chapters-naming) below). + +The `--no-mux` flag skips muxing entirely and writes the individual track files, each with a track-type suffix, into the same output/folder structure. + +## Tags & group naming + +The release-group tag is the `-TAG` portion at the end of scene-style names, produced by the `{tag}` variable. + +```yaml title="unshackle.yaml" +tag: "MYGROUP" +tag_group_name: true +tag_imdb_tmdb: true +``` + +| Key | Type | Default | Effect | +|---|---|---|---| +| `tag` | str | `""` | The release-group tag used for `{tag}`. | +| `tag_group_name` | bool | `true` | Write the group name (`config.tag`) into the MKV `Group` metadata tag. | +| `tag_imdb_tmdb` | bool | `true` | Look up and embed IMDb / TMDB / TVDB external-ID tags in the MKV metadata (uses `tmdb_api_key` / `simkl_client_id` when available). | + +You can override the tag per run without editing config: + +```bash +unshackle dl --tag OTHERGROUP AMZN "..." +``` + +The `--repack` flag adds the `REPACK` marker (surfaced through the `{repack}` variable) to the filename for that run: + +```bash +unshackle dl --repack NF "..." +``` + +!!! note "Empty tags collapse cleanly" + If `config.tag` is empty and your template ends with `-{tag?}`, the trailing `-` and the placeholder are both removed. Use the optional form `{tag?}` if you sometimes download without a tag set. + +## Unicode filenames + +By default unshackle transliterates non-ASCII characters (Korean, Japanese, Chinese, accented Latin, etc.) into ASCII equivalents, which is safest for older tools, DDL, and P2P sharing. Set `unicode_filenames` to keep the native characters. + +```yaml title="unshackle.yaml" +unicode_filenames: true +``` + +| Value | Result | +|---|---| +| `false` (default) | `기생충` → transliterated ASCII; accents stripped. | +| `true` | Native characters preserved in filenames and folders. | + +Regardless of this setting, a set of filesystem-unsafe and structural characters is always removed or replaced during sanitization (for example `/` and `;` become ` & `, and characters like `\ * ! ? , ' " < > | $ # ~` are stripped). This is why you should never put path-unsafe characters directly in a template. unshackle warns about templates containing `< > : " / \ | ? *` at startup (folder templates are checked per path segment, so `/` is allowed there as a directory separator). + +## Directory & filename configuration + +The `directories` key relocates unshackle's working folders. Only the folders listed here can be moved; core package paths are protected and silently ignored if you try to override them. + +```yaml title="unshackle.yaml" +directories: + downloads: "~/Media/unshackle" + temp: "/mnt/fast/unshackle-temp" + cache: "~/.cache/unshackle" + cookies: "~/.config/unshackle/cookies" + logs: "~/.config/unshackle/logs" +``` + +| Name | Purpose | +|---|---| +| `downloads` | Default output directory for finished files. | +| `temp` | Working directory for in-progress downloads, muxing, and intermediate files. | +| `cache` | Title/HTTP cache and update-check state. | +| `cookies` | Per-service cookie files. | +| `logs` | Log files. | +| `exports` | Export JSONs. | +| `services`, `vaults`, `fonts`, `commands` | Search paths for services, key-vault backends, bundled fonts, and CLI commands. | +| `wvds`, `prds`, `dcsl` | Widevine devices, PlayReady devices, and DCSL data. | + +Paths support `~` expansion. The names `app_dirs`, `core_dir`, `namespace_dir`, `user_configs`, and `data` are protected and cannot be changed. + +### Filename patterns + +The `filenames` key overrides the naming patterns for a handful of internal files. These are **not** your media output names. They are logs, temp files, and per-service configs. + +```yaml title="unshackle.yaml" +filenames: + log: "unshackle_{name}_{time}.log" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" +``` + +| Name | Default | Where | +|---|---|---| +| `log` | `unshackle_{name}_{time}.log` | `directories.logs` | +| `debug_log` | `unshackle_debug_{service}_{time}.jsonl` | `directories.logs` | +| `config` | `config.yaml` | per-service directory | +| `root_config` | `unshackle.yaml` | main config filename | +| `chapters` | `Chapters_{title}_{random}.txt` | `directories.temp` | +| `subtitle` | `Subtitle_{id}_{language}.srt` | `directories.temp` | + +### Chapters naming {#chapters-naming} + +When a title has chapters, they are written to an OGM chapters file (`filenames.chapters`) and passed to the muxer. Unnamed chapters use the `chapter_fallback_name` template: + +```yaml title="unshackle.yaml" +chapter_fallback_name: "Chapter {i:02}" +``` + +The fallback supports two placeholders: `{i}` (chapter number, starting at 1) and `{j}` (which increments only for unnamed chapters). Standard format directives like `{i:02}` are allowed. + +## Removed & migrated options + +!!! warning "`scene_naming` was removed" + The old `scene_naming` option no longer exists. If it is present in your config, unshackle exits immediately with an error asking you to configure `output_template` instead. Migrate any scene-naming setup to the templates described on this page. + +## Worked examples + +=== "Movies" + + ```yaml + output_template: + movies: "{title}.{year}.{edition?}.{quality}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}.{hdr?}-{tag?}" + folder: + movies: "{title} ({year})" + ``` + ```text + The Movie (2024)/ + └─ The.Movie.2024.1080p.NF.WEB-DL.DDP5.1.Atmos.H.265.HDR-TAG.mkv + ``` + +=== "TV series" + + ```yaml + output_template: + series: "{title}.{season_episode}.{episode_name?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}" + folder: + series: "{title} ({year})/Season {season}" + ``` + ```text + The Show (2023)/Season S02/ + └─ The.Show.S02E04.The.Reckoning.2160p.AMZN.WEB-DL.DDP5.1.H.265-TAG.mkv + ``` + +=== "Music" + + ```yaml + output_template: + songs: "{track_number}. {title}" + folder: + albums: "{album_artist} - {album} ({year})" + ``` + ```text + The Artist - The Album (2024)/ + └─ 01. Opening Track.mka + ``` + +## See also + +- [Configuration File](../getting-started/configuration-file.md): the full `unshackle.yaml` structure. +- [Downloading](downloading.md): the `dl` command and per-run flags like `-o`, `--no-folder`, `--tag`, and `--repack`. diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md new file mode 100644 index 00000000..fd5505ec --- /dev/null +++ b/docs/guide/proxies-and-vpn.md @@ -0,0 +1,639 @@ +# Proxies & VPN + +Most streaming services are geo-restricted, region-priced, or simply refuse to hand +over a manifest from the "wrong" country. unshackle routes its traffic through a proxy +so the service sees the location you choose instead of your own. You can point it at a +single proxy URL, or configure one of several **proxy providers** (commercial VPNs and +self-hosted tunnels), then ask for a country by its two-letter code and let +unshackle pick a working server for you. + +The `--proxy` flag controls proxying, and a query is resolved to a real proxy URL by the +providers unshackle supports: [basic static proxies](#basic-static-proxies), +[Gluetun](#gluetun) (documented in the most detail, since it is the most flexible), +[ExpressVPN](#expressvpn), [NordVPN](#nordvpn), [Proton VPN](#proton-vpn), +[Hola](#hola), [Surfshark](#surfshark), and [Windscribe](#windscribe). + +!!! note "Where proxies are configured" + All provider configuration lives under the top-level `proxy_providers:` key in your + `unshackle.yaml`. Each provider has its own sub-key, and the keys inside it map + directly onto that provider's settings. See the + [Configuration file](../getting-started/configuration-file.md) page for where the + file lives and how it is loaded. + +## The `--proxy` flag + +Proxying is controlled at download time by three options on the `dl` command: + +| Option | Effect | +|---|---| +| `--proxy` | The proxy to use. Either an explicit URI, or a query that unshackle resolves against your configured providers. | +| `--no-proxy` | Force **all** proxy use off. No providers are initialised and no proxy query is resolved. | +| `--no-proxy-download` | Use the proxy for the manifest, licence, and authentication, but bypass it for the **segment downloads** themselves. | + +```shell title="Explicit proxy URI" +unshackle dl --proxy http://user:pass@1.2.3.4:8080 NF 81234567 +``` + +```shell title="Resolve a US server from your configured providers" +unshackle dl --proxy us NF 81234567 +``` + +```shell title="Target a specific provider" +unshackle dl --proxy nordvpn:us NF 81234567 +``` + +!!! tip "Set a default proxy in config" + Like every `dl` flag, `--proxy` can be given a default in your configuration so you + do not have to pass it on every command. See + [Downloading](downloading.md) for how `dl` options and config defaults interact. + +### `--no-proxy-download` + +Segment downloads are usually the largest share of traffic, and a residential-grade VPN +proxy is often the slowest link in the chain. `--no-proxy-download` lets you keep the +proxy where it matters (passing the geo-check for the manifest and the licence) while +pulling the actual audio and video segments over your normal connection for full speed. + +```shell title="Proxy the manifest and licence, download segments direct" +unshackle dl --proxy gb --no-proxy-download ITV 10a1234 +``` + +!!! warning "When this is unsafe" + Some services tie segment delivery (CDN tokens, per-segment geo-checks) to the same + region as the manifest. On those services, bypassing the proxy for downloads will + cause segment fetches to fail or return the wrong region. If downloads break with + `--no-proxy-download`, remove it. + +## How proxy resolution works + +The value you pass to `--proxy` is interpreted in one of three ways. unshackle checks +them in order. + +```mermaid +flowchart TD + A["--proxy value"] --> B{"Looks like a URI?<br/>http:// https:// socks"} + B -- yes --> C["Use it verbatim"] + B -- no --> D{"Has a provider prefix?<br/>provider:query"} + D -- yes --> E["Ask that one provider<br/>for the query"] + D -- no --> F["Ask every provider in order,<br/>first match wins"] +``` + +1. **An explicit URI.** Anything shaped like `http://…`, `https://…`, or a `socks…` + URI is used exactly as given. unshackle logs `Using explicit Proxy: …` and does no + lookup. +2. **A provider-prefixed query**: `provider:query`, for example `nordvpn:us` or + `gluetun:windscribe:us`. unshackle finds the provider whose name matches the prefix + (case-insensitive) and asks *only* that provider to resolve the remainder. If no such + provider is loaded, or it returns no proxy, the download errors. +3. **A bare query**: a region like `us`, `gb`, `us:seattle`, or `us1234`. unshackle + asks each loaded provider **in order** and uses the first proxy any of them returns. + +The query is matched against a region grammar (a two-letter country code, optionally +followed by a server number and/or a `:city` or `-city` part), lowercased, and then +handed to the provider. The exact forms each provider accepts are documented in its +section below. + +### Provider load order + +When you run a download without `--no-proxy`, unshackle instantiates every configured +provider once, in this fixed order, and that same order is used for bare-query +resolution: + +**Basic → ExpressVPN → NordVPN → Proton VPN → Surfshark → Windscribe → Gluetun → Hola** + +So if you have both Basic and NordVPN configured and run `--proxy us`, a `us` entry in +your Basic config wins because Basic is tried first. To skip ahead to a specific +provider, prefix the query (`--proxy nordvpn:us`). + +!!! note "Auto-loading providers" + Most providers only load when you configure them under `proxy_providers:`. Three are + special: **ExpressVPN** and **Proton VPN** also load automatically if their cookie + file already exists on disk (even with no YAML), and **Hola** loads automatically + whenever the `hola-proxy` binary is found on your `PATH`. + +### Connection feedback + +After a provider resolves a query, unshackle prints a line so you can confirm +where you came out: + +- **Gluetun** reports the verified exit IP, country, and city of the container. +- **ExpressVPN** and **Proton VPN** print a location summary such as + `(USA - New York, #3 of 5): .214`. +- Other providers log the resolved proxy URL. + +## Basic (static proxies) + +The `Basic` provider is pure static configuration, with no accounts and no network calls. You +list proxy URLs per country code, and unshackle serves them back when you query that +country. This is the right choice when you already have proxy endpoints (from a proxy +seller, your own servers, or a corporate gateway). + +Under each two-letter country code you can put either a single proxy string or a list of +them: + +```yaml title="unshackle.yaml" +proxy_providers: + basic: + us: http://user:pass@1.2.3.4:8080 + de: + - http://a.example:8080 + - socks5://b.example:1080 +``` + +**Query forms:** + +| Query | Meaning | +|---|---| +| `us` | A proxy for the US. If several are listed, one is chosen at random. | +| `us2` | The **2nd** entry in the US list (1-based). | + +- If a country has a single proxy string, that proxy is always returned. +- If it has a list and you give a plain country code, one is picked at random; append an + index (`de1`, `de2`) to pin a specific entry. +- Each URL is normalised (a missing scheme defaults to `http`) and validated, so a + malformed URI is rejected up front rather than failing mid-download. + +```shell +unshackle dl --proxy de2 NF 81234567 +``` + +## Gluetun + +Gluetun is unshackle's most flexible proxy backend, and the one to reach for if you want +one configuration to cover many VPN providers. Instead of returning a remote proxy URL, +Gluetun **launches a local Docker container** running the +[`qmcgaw/gluetun`](https://github.com/qdm12/gluetun) image. That container establishes a +WireGuard or OpenVPN tunnel to your chosen VPN provider and exposes a plain HTTP proxy on +`localhost`, which unshackle then routes traffic through. When the download finishes, the +container is torn down automatically. + +This means you get access to the **50+ VPN providers** that Gluetun supports, using your own +VPN subscription, through a single uniform interface, with no per-provider integration +in unshackle itself. + +!!! warning "Docker is required" + Gluetun needs a working [Docker](https://docs.docker.com/get-docker/) installation on + your `PATH`. If Docker is not found, the provider fails to load with an error linking + to the install docs. The container also needs the `NET_ADMIN` capability and access + to `/dev/net/tun` to bring up the tunnel. unshackle requests both automatically. + +### Query format + +Gluetun uses a nested selector: the provider prefix picks Gluetun, and the rest names the +**VPN provider** and the **region**. + +``` +gluetun:<vpn-provider>:<region> +``` + +```shell title="A Windscribe US exit via Gluetun" +unshackle dl --proxy gluetun:windscribe:us NF 81234567 +``` + +```shell title="A NordVPN Germany exit via Gluetun" +unshackle dl --proxy gluetun:nordvpn:de NF 81234567 +``` + +After unshackle strips the leading `gluetun:`, the provider receives exactly +`provider:region` (two colon-separated parts; anything else is an error). The +`<vpn-provider>` must be one you have configured under `gluetun.providers`, and friendly +names are normalised to Gluetun's identifiers (for example `pia` and +`privateinternetaccess` both map to `private internet access`). + +### Configuration + +Each VPN provider you want to use gets an entry under `providers:`, carrying its VPN type, +credentials, and how your region aliases map onto Gluetun's server selectors. + +```yaml title="unshackle.yaml" +proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: wireguard + credentials: + private_key: YOUR_WIREGUARD_PRIVATE_KEY + addresses: YOUR_WIREGUARD_ADDRESS + preshared_key: YOUR_PRESHARED_KEY + server_countries: + us: US + uk: GB + server_cities: {} + server_hostnames: {} + extra_env: {} + nordvpn: + vpn_type: wireguard + credentials: + private_key: YOUR_WIREGUARD_PRIVATE_KEY + server_countries: + us: US + de: DE + + # Global settings (all optional) + base_port: 8888 + auto_cleanup: true + container_prefix: unshackle-gluetun + auth_user: null + auth_password: null + verify_ip: true +``` + +**Per-provider keys:** + +| Key | Purpose | +|---|---| +| `vpn_type` | `wireguard` (default) or `openvpn`. | +| `credentials` | The secrets for the tunnel; see the credential rules below. | +| `server_countries` | Maps your region alias (e.g. `us`) to Gluetun's country value (e.g. `US`). | +| `server_cities` | Optional. Maps an alias to a Gluetun city. | +| `server_hostnames` | Optional. Maps an alias directly to a specific server hostname. | +| `extra_env` | Optional. Raw environment variables merged into the container last, for any Gluetun setting not covered above. | + +**Credential rules by VPN type:** + +=== "WireGuard" + + - `private_key` is **always** required. + - `surfshark`, `mullvad`, and `ivpn` also require `addresses`. + - `windscribe` additionally requires **both** `addresses` and `preshared_key` + (`preshared_key` may be an empty string, but the key must be present). + - NordVPN and Proton VPN need only `private_key`. + +=== "OpenVPN" + + - Requires `username` and `password`. + +!!! tip "Which `vpn_type` should I pick?" + WireGuard is the default because it **can be faster**, but it needs the fiddlier + credential setup above (private keys, and for some providers `addresses` and a + preshared key). OpenVPN is the **simplest** to configure (just a username and + password) at the cost of some speed. If you are getting started or hitting credential + trouble, OpenVPN is the path of least resistance; switch to WireGuard when you want the + extra throughput. + +**Global keys:** + +| Key | Default | Purpose | +|---|---|---| +| `base_port` | `8888` | The first local port to expose proxies on. Additional containers take the next free ports. | +| `auto_cleanup` | `true` | Remove containers on exit. If `false`, containers are only stopped, not deleted. | +| `container_prefix` | `unshackle-gluetun` | Prefix for Docker container names. | +| `auth_user` / `auth_password` | `null` | Optional HTTP-proxy credentials. If set, the local proxy requires auth and the credentials are embedded in the proxy URL. | +| `verify_ip` | `true` | After connecting, verify the container's real exit IP and region. | + +### Region selection + +For a given query region, Gluetun resolves the server like this: + +1. An explicit entry in `server_hostnames`, `server_cities`, or `server_countries` for + that alias always wins. +2. A `us1239`-style query (country code + number) with no explicit mapping is turned + into a provider-specific server hostname (for example `us1239.nordvpn.com` for + NordVPN, `us-1239.prod.surfshark.com` for Surfshark). +3. A bare two-letter code is expanded to the full country name Gluetun expects. + +A few providers (Windscribe, VyprVPN, VPN Secure) select by **region** rather than +country, and Windscribe region names are translated automatically. For instance `us` +becomes `US East` and `uk`/`gb` become `United Kingdom`. + +### Container lifecycle + +Understanding what Gluetun does behind the scenes helps when something goes wrong: + +1. **Image check.** `qmcgaw/gluetun:latest` is pulled if it is not already present. +2. **Reuse.** If a container for this exact `provider:region` is already running (from a + concurrent unshackle session), it is adopted instead of starting a new one. Stopped or + dead containers with the same name are pruned first. + +!!! note "Expect a slow first request, then instant reuse" + Bringing up a fresh tunnel takes roughly **10 to 30 seconds** for the first request to a + given `provider:region`. This is the VPN handshake and readiness wait, not a hang. + Every subsequent request that reuses that same container is effectively **instant**. A + slow first run is normal. +3. **Port allocation.** A free local port is chosen starting from `base_port`. +4. **Start.** The container is launched bound to `127.0.0.1:{port}`. Credentials are + passed through a temporary, `0600`-permission env-file (never as `-e` flags), which is + overwritten and deleted immediately after. +5. **Readiness wait.** unshackle polls the container logs (up to 60 seconds) until both + the HTTP proxy is listening **and** the VPN reports a completed connection. Fatal + errors like `authentication failed` or `invalid credentials` abort early, and recent + logs are attached to the error to help you diagnose it. +6. **IP verification.** When `verify_ip` is on, unshackle queries the container's exit IP + and checks the country matches what you asked for, raising a *region mismatch* error if + it does not. +7. **Proxy handed back.** You get `http://localhost:{port}` (or with auth credentials if + configured). + +!!! note "Cleanup and Ctrl+C" + Containers are cleaned up via an `atexit` handler on normal exit. Gluetun deliberately + does **not** install its own signal handlers, so `Ctrl+C` continues to work as you + expect. If a run is killed hard (e.g. `kill -9`) a container can be left behind; with + `auto_cleanup: true` the next run adopting that name will still work, or you can remove + it manually with `docker rm -f`. + +### Troubleshooting + +!!! example "Common Gluetun issues" + - **`RuntimeError` about Docker not found.** Install Docker and confirm `docker` is + on your `PATH`. + - **`invalid credentials` / `authentication failed` in the readiness wait.** Your + WireGuard/OpenVPN credentials are wrong or in the wrong fields. Double-check + `private_key`, `addresses`, and (for Windscribe) `preshared_key`. + - **Region mismatch error.** The VPN connected but exited in a different country than + requested. Check your `server_countries` mapping, or set `verify_ip: false` if you + are intentionally using a nearby region. + - **Timeout after 60 seconds.** The tunnel never came up. Inspect the container logs + (`docker logs <container>`) for the underlying VPN error. + +## ExpressVPN + +ExpressVPN emulates the flow used by the ExpressVPN browser extension: it authenticates +with cookies (or a cached token), obtains proxy-capable locations from ExpressVPN's API, +and returns an authenticated HTTPS proxy. Because it relies on browser cookies rather than +service credentials, setup is slightly more involved than the credential-based providers. + +ExpressVPN **auto-loads** if its cookie file exists, so once the cookie is in place you do +not strictly need a YAML block, but you can add one to pin regions or servers. + +**Default file locations:** + +| File | Default path | +|---|---| +| Browser cookies | `{cookies}/vpn/expressvpn.txt` | +| Token cache | `{cache}/global/expressvpn_tokens.json` | + +Where `{cookies}` and `{cache}` are your configured cookie and cache directories. The +cookie file may be a Netscape `cookies.txt`, a JSON cookie array, or a JSON dict. Export +your ExpressVPN session cookies from a logged-in browser into one of those forms. + +```yaml title="unshackle.yaml (optional)" +proxy_providers: + expressvpn: + region_map: + us: ny-02 # prefer New York, server 02 for the US + gb: # empty = smart random location + server_map: + myserver: usny-newyork-2 +``` + +**Query forms:** + +| Query | Meaning | +|---|---| +| `us` | A smart, random US location. | +| `us-ny` | US, New York. | +| `us-ny-2` / `us-ny2` | US, New York, the 2nd server. | +| `usny-newyork-2` | A full location slug. | +| `hostname.expressprovider.com` | A direct endpoint hostname. | + +The resolved proxy is an HTTPS proxy on port `443`. On success, unshackle prints a +location summary such as `(USA - New York, #3 of 5): .214`. + +!!! note "Cache safety" + ExpressVPN's token cache is written with `0600` permissions so the refresh token is + never briefly world-readable. + +## NordVPN + +NordVPN uses your **Service Credentials**, not your normal NordVPN login. Generate them +from the [NordVPN dashboard](https://my.nordaccount.com/dashboard/nordvpn/) under manual +setup; they are a long random username/password pair, not your email and password. + +```yaml title="unshackle.yaml" +proxy_providers: + nordvpn: + username: YOUR_SERVICE_USERNAME + password: YOUR_SERVICE_PASSWORD + server_map: # optional: pin specific server IDs + us: 1234 + us:seattle: 5678 +``` + +**Required keys:** `username`, `password`. unshackle validates them up front: the +combined username and password must be exactly 48 lowercase alphanumeric characters, and +the username must not contain an `@`. If validation fails, you have almost certainly used +your login credentials instead of your service credentials. + +`server_map` is optional and maps a region (or `code:city`) to a pinned NordVPN server ID. +Its main use is tuning: for a given streaming service one specific server often performs +noticeably better than NordVPN's recommended pick, so hard-coding that server for a day or +two is a handy workaround until it degrades and you swap in another. + +!!! note "Use `gb`, not `uk`, for the United Kingdom" + Query the UK as `gb`. unshackle uses the Alpha-2 country code `gb` deliberately to stay + consistent with its other regional/country-code systems and avoid the language-vs- + country-code confusion that `uk` invites. + +**Query forms:** + +| Query | Meaning | +|---|---| +| `us` | A recommended US server. | +| `us1234` | The specific server `us1234`. | +| `us:seattle` | A recommended server in Seattle. | +| `228` | A NordVPN numeric country ID. | + +The returned proxy is HTTPS on **port 89** (NordVPN disabled its plain-HTTP proxies on +port 80 in 2021). + +```shell +unshackle dl --proxy nordvpn:us:seattle NF 81234567 +``` + +## Proton VPN + +Proton VPN supports two authentication methods, and unshackle can work with either. Note +that free Proton accounts work but are limited to Proton's free-tier exit countries. + +=== "TV login (recommended)" + + A self-sustaining, refreshable session obtained through Proton's TV device-login flow. + This is the only method that can refresh itself headlessly, so it keeps working without + re-exporting anything. Enable it and run a download interactively once: + + ```yaml title="unshackle.yaml" + proxy_providers: + protonvpn: + enable: true + ``` + + With `enable: true` and an interactive terminal, unshackle prints a code and asks you + to enter it at [account.proton.me/vpn/tv/code](https://account.proton.me/vpn/tv/code). + Once approved, the session is cached and reused automatically. + +=== "Exported cookies" + + Export your `account.proton.me` session cookies (the `AUTH-<UID>` cookie) to the Proton + cookie file. This method is **access-only**: it cannot refresh, so you must re-export + when it expires. + + !!! warning "The `AUTH-<UID>` cookie is HttpOnly" + This is the single most common cause of a failed Proton cookie import: the required + `AUTH-<UID>` cookie is **HttpOnly**, and many one-click cookie-export extensions + silently skip HttpOnly cookies by default. If your import "doesn't work," use an + exporter that explicitly includes HttpOnly cookies. + + !!! note "Why this session can't refresh, and lapses after ~24h" + Proton rotates browser refresh tokens and ties them to the live browser, so a + headless refresh is impossible. unshackle therefore deliberately uses only the + cookie's short-lived **access token** and never attempts a refresh, which is why the + session lapses at Proton's ~24h TTL and must be re-exported. This is exactly why TV + login is preferred for set-and-forget use. + +**Default file locations:** + +| File | Default path | +|---|---| +| Cookies | `{cookies}/vpn/protonvpn.txt` | +| Session cache | `{cache}/vpn/protonvpn.json` | + +Proton VPN **auto-loads** if its cookie file exists. Like ExpressVPN, its cached session +is written with `0600` permissions. + +!!! note "Cache vs cookie precedence" + If both a TV-login cache and a cookie file exist, the **cache is preferred** unless the + cookie file is newer. Cookies are read fresh every run and never written back to the + cache. So to deliberately force use of the cookie session, just re-save (re-export) the + cookie file so its modification time is newer than the cache. + +**Query forms (after the `protonvpn:` prefix):** + +| Query | Meaning | +|---|---| +| `us` | A random US server (paid tiers preferred over free). | +| `us12` | Proton US server #12. | +| `us:ny` | A server in New York. | + +The returned proxy is HTTPS on **port 4443** (or **port 443** for Secure Core servers). +Secure Core and Tor servers are excluded from selection because they are too slow for a +plain proxy. On success, unshackle prints a summary such as `(Name - City): server-host`. + +!!! tip "Proton server numbers aren't sequential" + When pinning a server with `protonvpn:deNN`, don't guess a low number. A country may + expose non-contiguous IDs like `#203`, `#813`, and so on. Copy the number from a + previous run's log line (e.g. `DE#203`) instead. + +## Hola + +Hola requires no configuration at all. It uses the +[`hola-proxy`](https://github.com/Snawoot/hola-proxy) binary, and **auto-loads** whenever +that binary is found on your `PATH`. If you query Hola but the binary is missing, the +provider raises an error telling you to install it. + +```shell title="Install hola-proxy, then just query a country" +unshackle dl --proxy hola:us NF 81234567 +``` + +**Query forms:** a two-letter country code (e.g. `us`, `gb`). unshackle asks `hola-proxy` +for available proxies in that country and picks one at random, returning an HTTP proxy. + +!!! warning "Temporary bans" + Hola's free tier can rate-limit or temporarily ban you if queried too aggressively. If + you see a *temporary ban detected* error, wait before retrying. Hola currently uses + only datacenter proxies. + +## Surfshark + +Surfshark, like NordVPN, uses **Service Credentials** rather than your login. Generate +them from the [Surfshark manual-setup page](https://my.surfshark.com/vpn/manual-setup/main/openvpn). + +```yaml title="unshackle.yaml" +proxy_providers: + surfsharkvpn: + username: YOUR_SERVICE_USERNAME + password: YOUR_SERVICE_PASSWORD + server_map: # optional + us: 1234 +``` + +**Required keys:** `username`, `password`, validated with the same 48-character rule as +NordVPN (combined username+password must be 48 lowercase alphanumeric characters, no `@`). +`server_map` optionally pins server IDs per region. + +**Query forms:** + +| Query | Meaning | +|---|---| +| `us` | A random US server. | +| `us-bos` | A specific server. | +| `us:seattle` | A random server in Seattle. | + +The returned proxy is HTTPS on **port 443**. + +## Windscribe + +Windscribe uses **Service Credentials** from its +[OpenVPN config page](https://windscribe.com/getconfig/openvpn). Only the presence of a +username and password is checked (there is no 48-character rule). + +```yaml title="unshackle.yaml" +proxy_providers: + windscribevpn: + username: YOUR_SERVICE_USERNAME + password: YOUR_SERVICE_PASSWORD + server_map: # optional: alias -> hostname + us: us-east.totallyacdn.com + us:seattle: us-seattle.totallyacdn.com +``` + +**Required keys:** `username`, `password`. `server_map` optionally maps a region (or +`region:city`) directly to a hostname. + +**Query forms:** + +| Query | Meaning | +|---|---| +| `us` | A random US server. | +| `us150` / `sg007` | A specific numbered server. | +| `us:seattle` | A random server in Seattle. | + +The returned proxy is HTTPS on **port 443**. + +!!! warning "Windscribe is not loaded by the REST API resolver" + The standalone Windscribe provider (and Gluetun) are loaded by the `dl` command, but + **not** by the proxy resolver used by the REST API and remote-download client. If you + drive downloads through the API and need Windscribe + or Gluetun exits, resolve the proxy on the CLI side or use a different provider for API + jobs. See the developer note below. + +## Developer notes + +!!! note "Two resolution paths (developers)" + unshackle has **two** proxy-resolution code paths that are intentionally not + identical: + + - `unshackle/commands/dl.py` inlines its own logic and loads **all eight** providers, + including Windscribe and Gluetun. + - `unshackle/core/proxies/resolve.py::initialize_proxy_providers()` is the shared + resolver used by the REST API handlers and the remote-service client. It **omits** + Windscribe and Gluetun. + + Both paths accept the same query grammar (a direct URI, a `provider:query`, or a bare + country query tried against providers in order), but the set of available providers + differs. Keep this in mind when adding a provider: wiring it into `dl.py` does not make + it available over the API, and vice versa. + +!!! note "Implementing a new provider (developers)" + Every provider subclasses `unshackle.core.proxies.proxy.Proxy` and implements three + methods: + + - `__init__(self, **kwargs)` receives its `proxy_providers` sub-dict splatted as + keyword arguments, so YAML keys map 1:1 to constructor parameters. Do authentication + and catalog pre-fetching here. + - `__repr__(self)` is a human summary, conventionally `"{n} Countries ({m} Servers)"`. + - `get_proxy(self, query)` returns a Requests-compatible proxy URI, or `None` only + when the query was valid but no proxy exists. Raise on genuine errors or bad queries. + + Optionally expose `get_connection_info(query)` (as Gluetun does) or + `last_connection_display()` (as ExpressVPN and Proton VPN do) to have `dl` print a + friendly connection line instead of the raw proxy URL. + +## See also + +- [Downloading](downloading.md): the full `dl` command guide, including where proxy + flags sit in the download flow. +- [Configuration file](../getting-started/configuration-file.md): the `unshackle.yaml` + reference, including directories for cookies and cache used by the cookie-based + providers. +- [Configuration reference](../reference/api/config.md): the developer-facing + configuration API, including the `proxy_providers` structure. diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md new file mode 100644 index 00000000..ab39249a --- /dev/null +++ b/docs/guide/subtitles.md @@ -0,0 +1,356 @@ +# Subtitles + +Subtitles are first-class tracks in unshackle. When you download a title, subtitle +tracks are parsed from the manifest alongside video and audio, downloaded, cleaned up, +optionally converted to the format you want, and then either muxed into the final +Matroska file or written out as sidecar files next to it. The sections below cover the +supported formats, how tracks are normalized after download, how to select and filter +subtitles, and every subtitle-related configuration option. + +Most of this is controlled with a handful of `dl` flags and a small `subtitle` block in +your [configuration file](../getting-started/configuration-file.md). See +[Downloading](downloading.md) for how subtitle flags fit into a full download command. + +## Supported formats + +unshackle understands the following subtitle codecs. The **Format** column is the value +shown in track listings; the **Extension** column is the file extension used on disk. + +| Codec (internal name) | Format | Extension | Notes | +| --- | --- | --- | --- | +| `SubRip` | SRT | `srt` | SubRip, the most widely compatible plain-text format | +| `SubStationAlpha` | SSA | `ssa` | SubStation Alpha (styled) | +| `SubStationAlphav4` | ASS | `ass` | Advanced SubStation Alpha (styled) | +| `TimedTextMarkupLang` | TTML | `ttml` | Timed Text Markup Language (IMSC/DFXP) | +| `WebVTT` | VTT | `vtt` | WebVTT | +| `SAMI` | SMI | `smi` | Synchronized Accessible Media Interchange | +| `MicroDVD` | SUB | `sub` | MicroDVD | +| `MPL2` | MPL2 | `mpl2` | MPL2 | +| `TMP` | TMP | `tmp` | TMP | +| `fTTML` | STPP | `stpp` | MPEG-DASH box-encapsulated TTML (IMSC1) | +| `fVTT` | WVTT | `wvtt` | MPEG-DASH box-encapsulated WebVTT | + +!!! note "The two boxed formats convert automatically" + `fTTML` (STPP) and `fVTT` (WVTT) are the fragmented, MP4-box-encapsulated forms that + DASH and HLS deliver. You never keep these as-is: right after download they are + unwrapped into their plain counterparts (`fTTML` becomes TTML and `fVTT` becomes + WebVTT), so you always end up with an editable text subtitle. Note that this is a + one-way, source-only relationship: you can convert *from* these boxed formats, but no + backend can produce them as an output target, so they can never be a conversion + destination. + +## What happens to a subtitle after download + +A subtitle download does more than save a file. Once the bytes are on disk, unshackle runs +format-appropriate clean-up to produce a single, valid file: + +- **Encoding**: non-boxed subtitles are forced to UTF-8, and stray `‎` / `‏` + bidirectional entities are unescaped. +- **Segment merging**: when a WebVTT or boxed subtitle arrives as many DASH/HLS + segments, the segments are stitched back into one continuous document with correct + timing. +- **WebVTT sanitization**: merged WebVTT runs through a repair pipeline that clamps + negative timestamps to `00:00:00.000`, removes stray cue identifiers (like `Q0`, `Q1`) + that confuse downstream parsers, and merges multi-line cues that a service split into + separate overlapping cues. By default the original styling and layout are preserved + (see [`preserve_formatting`](#subtitle-configuration)). +- **TTML for muxing**: Matroska cannot carry TTML. If you do not request a specific + output format, any plain TTML track is automatically converted to WebVTT before + muxing (the closest lossless equivalent). Boxed `fTTML` is likewise unwrapped to TTML + first. + +You normally do not need to think about any of this. It just produces a clean file. + +## Selecting and filtering subtitles + +Subtitle selection happens through `dl` flags. The most important is language selection. + +| Flag | Purpose | +| --- | --- | +| `-sl`, `--s-lang` | Language(s) wanted for subtitles. Defaults to `all`. | +| `--require-subs` | Require these languages to exist; if present, download **all** subtitles. Cannot be combined with `--s-lang`. | +| `-fs`, `--forced-subs` | Include forced subtitle tracks (excluded by default). | +| `--exact-lang` | Exact language matching, with no regional variants. | +| `-S`, `--subs-only` | Download only subtitle tracks. | +| `-ns`, `--no-subs` | Do not download subtitle tracks at all. | +| `--sub-format` | Output subtitle format / conversion target (see [conversion](#converting-subtitle-formats)). | +| `--skip-subtitle-errors` | Skip a subtitle that fails to download instead of aborting the whole title. | + +### Choosing languages + +By default (`--s-lang all`) unshackle keeps every subtitle language the service offers. +To narrow it down, pass a comma-separated list of language tags: + +```shell title="English and Spanish subtitles only" +unshackle dl --s-lang en,es NF 81234567 +``` + +Language matching is fuzzy by default: `en` will match `en-US`, `en-GB`, and similar +variants. If a requested language is missing but others were found, unshackle continues +with whatever remains and tells you which languages it dropped. + +For strict matching, where `es-419` matches only `es-419` and never `es-ES`, add +`--exact-lang`: + +```shell title="Only Latin-American Spanish, no other Spanish variants" +unshackle dl --s-lang es-419 --exact-lang DSNP 'https://www.disneyplus.com/…' +``` + +### Requiring languages before downloading + +`--require-subs` is a gate rather than a filter. It downloads **all** available +subtitles, but only if every language you list is present; if any is missing the title +errors out. This is useful for batch jobs where a title without your must-have language +should fail loudly rather than silently produce a partial result. + +```shell title="Fail unless both English and French subs exist, then grab everything" +unshackle dl --require-subs en,fr AMZN B0ABCDEFGH +``` + +!!! warning "`--require-subs` and `--s-lang` are mutually exclusive" + Because `--require-subs` forces downloading of all subtitle languages, it cannot be + combined with `--s-lang`. Choose one or the other. + +### Forced subtitles + +Forced tracks carry only the important on-screen signs and foreign-dialogue lines meant +to display even when you are watching in your own language. They are **excluded by +default**. Add `-fs` / `--forced-subs` to include them: + +```shell title="Include forced tracks alongside full subtitles" +unshackle dl --s-lang en --forced-subs NF 81234567 +``` + +## Track types: forced, SDH, and CC + +Every subtitle track carries flags describing what kind of captions it holds. These +flags drive selection, sort order, output filenames, and the Matroska track flags. + +- **Forced**: only signs and foreign dialogue; meant to play when the audio language + matches. Off by default; enable with `--forced-subs`. +- **SDH** (Subtitles for the Deaf and Hard-of-Hearing; also called HOH): a full + transcript that includes both dialogue and non-speech sound cues. +- **CC** (Closed Captions): captions originating from an EIA-608/708 broadcast stream, + often uppercase with `>>>` speaker markers and sound cues. + +!!! note "SDH and CC are treated together for sorting and stripping" + Within a language, unshackle orders subtitles from fewest to most captions: + **Forced → Normal → SDH/CC**. For the purposes of hearing-impaired handling, SDH and + CC tracks are grouped as the "most captions" variant. + +A single track cannot be both CC and SDH, and a forced track cannot also be flagged +CC or SDH. These combinations are rejected as invalid. + +### Closed captions embedded in video + +Some services do not ship subtitles as separate tracks but instead embed EIA-608/708 +captions inside the video stream. After a video track is downloaded and decrypted, +unshackle probes it for these captions and, when found, extracts them into a SubRip +(`SRT`) subtitle flagged as CC. This runs automatically unless you disabled subtitles +(`--no-subs`) or video (`--no-video` / `--video-only` semantics), and it requires the +CCExtractor tool on your `PATH`. + +## Automatic SDH stripping + +When a language has an SDH track but **no** plain (non-SDH, non-forced) track, unshackle +can generate a clean non-SDH version for you by stripping the hearing-impaired cues: +sound effects, speaker labels, music notes, and similar. This is on by default and +controlled by the [`strip_sdh`](#subtitle-configuration) config key. + +The check is deliberately loose: if a close-enough plain track already exists (for +example an `en-GB` non-SDH track when the SDH track is `en-US`), no stripped copy is +created. When a stripped copy *is* created, the original SDH track is kept as well, so +you end up with both. + +The stripping engine itself is chosen by the [`sdh_method`](#subtitle-configuration) +config key. The default (`auto`) prefers the `subby` cleaner for SRT, falls back to +SubtitleEdit when it is installed, and otherwise uses the built-in `filter-subs` +approach. + +!!! note "Why `convert_before_strip` exists" + `subby`'s SDH stripper only operates on SRT. Handed any other codec it returns + silently without stripping anything, not an error but a no-op that leaves the SDH + cues in place. This is the whole reason for [`convert_before_strip`](#subtitle-configuration): + non-SubtitleEdit engines need the subtitle converted to SRT first, or the + hearing-impaired cues survive the strip untouched. + +## Converting subtitle formats + +By default unshackle keeps each subtitle in its downloaded format (only auto-converting +TTML to WebVTT for muxing, as noted above). To force a specific output format, use +`--sub-format`: + +```shell title="Convert every subtitle to SRT" +unshackle dl --s-lang en,es --sub-format srt NF 81234567 +``` + +`--sub-format` accepts the common format names and aliases, including `srt`, `vtt`, +`ssa`, `ass`, and `ttml`. Pass `original` to explicitly keep the source format and skip +all conversion: + +```shell title="Keep whatever the service delivered" +unshackle dl --sub-format original DSNP 'https://www.disneyplus.com/…' +``` + +Conversion only runs when the source format differs from the target, so +`--sub-format srt` on a track that is already SRT is a no-op. + +!!! warning "Down-converting styled subtitles is lossy" + Converting styled SSA/ASS subtitles to SRT throws away positioning, colours, and + italics. unshackle never does this automatically. It only happens when you + explicitly ask for it via `--sub-format`. If you want to preserve styling, convert + to a format that supports it (or keep `original`). + + Note that this automatic protection guards **only the default muxed track**: there the + conversion is skipped and the styled original is kept. It does **not** cover sidecars. + A `sidecar_format: srt` will still lossily flatten styled subtitles, dropping + positioning, colours, and styling, and a per-download `--sub-format srt` forces the + muxed track to convert as well. To keep raw styled sidecars, set + [`sidecar_format: original`](#subtitle-configuration). + +### How the conversion backend is chosen + +Under the hood unshackle picks the highest-fidelity converter that supports the +source→target pair. Which converter runs depends on the +[`conversion_method`](#subtitle-configuration) config key (default `auto`) and the +formats involved: + +| Backend | Best at | Requires | +| --- | --- | --- | +| `subtitleedit` | Highest fidelity; preserves positioning and italics | SubtitleEdit installed | +| `subby` | Native SRT output with clean-up | — | +| `pysubs2` | Best fidelity for styled SSA/ASS | — (pure Python, always available) | +| `pycaption` | Last-resort fallback; flattens styling | — | + +With `conversion_method: auto`, unshackle ranks these automatically per conversion. +Setting it to a specific value pins that backend as the first choice, falling back to +others only if the pin cannot handle the pair. + +!!! note "Config wins over a service's preference" + A service may set a `preferred_conversion_method` on its own tracks (for example when + it ships subtitles that a particular backend handles best). An explicit + `conversion_method` in `unshackle.yaml` always overrides that per-track preference, so + your config takes precedence. This only matters when a service ships a non-default + preference; leave `conversion_method` at `auto` to let the service's hint stand. + +## Muxed vs. sidecar output + +By default subtitles are muxed **into** the final `.mkv`. You can instead (or +additionally) write them as separate sidecar files next to the video, controlled by the +[`output_mode`](#subtitle-configuration) config key: + +=== "mux (default)" + + ```yaml title="unshackle.yaml" + subtitle: + output_mode: mux + ``` + + Subtitles are embedded in the Matroska file. Nothing is written separately. + +=== "sidecar" + + ```yaml title="unshackle.yaml" + subtitle: + output_mode: sidecar + sidecar_format: srt + ``` + + Subtitles are written as standalone files and **not** muxed into the video (when + there is a video or audio track to sit beside). + +=== "both" + + ```yaml title="unshackle.yaml" + subtitle: + output_mode: both + sidecar_format: srt + ``` + + Subtitles are muxed into the video **and** written as sidecar files. + +Sidecar files are named after the output file with the language and flags encoded in the +name, so players and media managers can match them automatically: + +```text +Show.S01E01.1080p.WEB.en.srt +Show.S01E01.1080p.WEB.en.forced.srt +Show.S01E01.1080p.WEB.en.sdh.srt +``` + +The pattern is `{base}.{language}[.forced][.sdh].{extension}`. The sidecar file format +is set by [`sidecar_format`](#subtitle-configuration); use a format name like `srt` or +`vtt`, or `original` to keep each subtitle in its downloaded format. + +## Fonts for styled subtitles + +Styled SSA/ASS subtitles reference specific fonts by name. When unshackle muxes such a +subtitle it scans it for the fonts it uses and attaches the matching font files to the +Matroska output so the styling renders correctly on playback. If a referenced font is +not available locally, unshackle logs the missing font names and suggests installing the +relevant font packages. + +## Subtitle configuration + +All of the behaviour above can be defaulted in the `subtitle` block of your +[configuration file](../getting-started/configuration-file.md): + +```yaml title="unshackle.yaml" +subtitle: + strip_sdh: true + preserve_formatting: true + conversion_method: auto + sdh_method: auto + convert_before_strip: true + output_mode: mux + sidecar_format: srt +``` + +| Key | Default | Values | Purpose | +| --- | --- | --- | --- | +| `strip_sdh` | `true` | `true` / `false` | Auto-generate a stripped non-SDH copy when a language has only an SDH track. | +| `preserve_formatting` | `true` | `true` / `false` | Keep original WebVTT styling and layout during sanitization. When `false`, WebVTT is re-encoded more aggressively (merging identical cues, dropping empty ones). | +| `conversion_method` | `auto` | `auto`, `subby`, `pysubs2`, `subtitleedit`, `pycaption` | Which conversion backend to prefer. `auto` ranks them per conversion. | +| `sdh_method` | `auto` | `auto`, `subby`, `subtitleedit`, `filter-subs` | Engine used to strip hearing-impaired cues. | +| `convert_before_strip` | `true` | `true` / `false` | Convert a subtitle to SRT before stripping SDH (for non-SubtitleEdit engines). | +| `output_mode` | `mux` | `mux`, `sidecar`, `both` | Whether subtitles are embedded, written as sidecars, or both. | +| `sidecar_format` | `srt` | a format name (e.g. `srt`, `vtt`) or `original` | Format used for sidecar files. | + +!!! tip "SubtitleEdit is optional but recommended" + Several high-fidelity paths (the best-quality conversions and the SubtitleEdit SDH + stripper) only activate when the SubtitleEdit CLI is installed and on your `PATH`. + See [Installation](../getting-started/installation.md) for how to add it. Without it, + unshackle falls back to its built-in Python backends. + +--- + +## Developer reference + +!!! note "Audience" + This section is for people extending unshackle or writing services. Everyday users + can stop here. + +The subtitle model lives in `unshackle.core.tracks.Subtitle`. A few internals worth +knowing when working with subtitle tracks in code: + +- **Flags and validation**: `Subtitle(codec=..., cc=..., sdh=..., forced=...)`. A track + cannot be both `cc` and `sdh`, and `forced` cannot be combined with either; both raise + `ValueError` at construction. +- **`convert(codec, *, forced=False)`**: converts the downloaded file in place through + the backend chain in `subtitle_convert.run_conversion`. `forced=True` (what + `--sub-format` sets) is the only way lossy styled down-converts such as + ASS→SRT are permitted. Raises `NotImplementedError` if no backend supports the pair and + `RuntimeError` if all attempts fail. +- **`strip_hearing_impaired()`**: removes SDH cues using the engine selected by + `config.subtitle["sdh_method"]`. +- **`reverse_rtl()`**: fixes right-to-left sentence-ending positioning via SubtitleEdit + (requires the binary). +- **`parse(data, codec)`**: the central parser, returning a `pycaption.CaptionSet`; it + handles the boxed DASH formats, SAMI, and the WebVTT repair helpers. +- **`extract_fonts(text)`**: returns the set of font names referenced by an ASS/SSA + subtitle, used to decide which fonts to attach at mux time. + +The conversion backends (`SubtitleEditBackend`, `SubbyBackend`, `Pysubs2Backend`, +`PycaptionBackend`) each implement a small protocol (`is_available()`, `can_convert()`, +`rank()`, `convert()`) and are tried in rank order, lowest rank first. See +[the Tracks API reference](../reference/api/tracks.md) for the full model. diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md new file mode 100644 index 00000000..6f79df58 --- /dev/null +++ b/docs/guide/troubleshooting.md @@ -0,0 +1,303 @@ +# Troubleshooting & Debugging + +When a download fails, a service misbehaves, or a key won't resolve, unshackle gives you three tools to find out why: + +- **Debug logging**: verbose `DEBUG`-level console output plus a structured, machine-readable log file. +- **[`unshackle env`](#checking-your-environment)**: verify that the external tools unshackle depends on are installed, and see exactly which directories and config file are in use. +- **Targeted toggles**: `debug_keys` and `debug_requests` for the two areas (DRM keys and raw HTTP traffic) that are deliberately quiet by default. + +The sections below cover how to enable each of them, how to read what they produce, and which diagnostic fits which failure mode. + +--- + +## Turning on debug logging + +There are two independent ways to enable debug logging, and they do slightly different things. + +=== "The `-d` / `--debug` flag" + + The root-level flag turns on verbose logging for a single invocation: + + ```console + unshackle -d dl SERVICE TITLE_ID + ``` + + It must come **before** the subcommand, because it is an option on the root `unshackle` command, not on `dl`. + + `-d` does three things: + + 1. Sets the console log level to `DEBUG`, so every module logs its full detail. + 2. Enables the structured JSON debug log file (see [below](#the-structured-json-debug-log)). + 3. Adds source-path annotations to console log lines (which file and line each message came from). + +=== "The `debug` config key" + + To make debug logging the default for every run without typing `-d` each time, set it in your `unshackle.yaml`: + + ```yaml + debug: true + ``` + + or via the [`cfg`](cli-reference.md#cfg) command: + + ```console + unshackle cfg debug true + ``` + + !!! note "`debug: true` is not identical to `-d`" + The `debug` config key enables the **JSON debug log file**, but it does **not** raise the console to `DEBUG` level and does not add source-path annotations. Those only happen when you pass the `-d` flag. Use the flag when you want to *watch* the verbose output live; use the config key when you just want the log file written every time. Passing `-d` is the most complete option, and you can combine both. + +!!! warning "Debug logging can be noisy and revealing" + Debug output is verbose and includes request metadata, service internals, and stack traces. Secrets, URLs, and local paths are automatically redacted (see [Redaction](#what-gets-redacted)), but you should still glance over a log before sharing it publicly. + +--- + +## The structured JSON debug log + +Beyond the console output, unshackle writes a structured log in [JSON Lines](https://jsonlines.org/) format: one complete JSON object per line. This file is designed to be filtered, searched, and analysed with standard tooling (`jq`, `grep`, a text editor), and it is safe to attach to a bug report because it is redacted. + +### Where it is written + +The debug log is created during a `dl` run (and by [`import`](cli-reference.md#import), which wraps `dl`). The path is: + +``` +<logs directory>/unshackle_debug_<service>_<timestamp>.jsonl +``` + +For example, `unshackle_debug_NETFLIX_20260703-142530.jsonl`. The `<service>` tag and the timestamp mean each download gets its own file, so runs never overwrite each other. + +!!! tip "Find your logs directory" + The logs directory defaults to a `logs` folder inside the package data directory, but it can be relocated with the `directories.logs` config key. To see the exact resolved path on your machine, run [`unshackle env info`](#env-info) and read the **Directories** table. + +### What a log entry looks like + +Every line is a self-contained JSON object. The first line of a session records the environment: + +```json +{"timestamp":"2026-07-03T14:25:30.123456+00:00","session_id":"a1b2c3d4","level":"INFO","operation":"session_start","message":"Debug logging session started","context":{"unshackle_version":"5.3.0","python_version":"3.12.4 ...","platform":"Linux-6.18.5-x86_64","platform_system":"Linux","platform_release":"6.18.5"}} +``` + +Subsequent lines describe operations: service calls, DRM licence requests, vault lookups, downloader progress, and errors. Common fields include: + +| Field | Meaning | +|---|---| +| `timestamp` | UTC ISO-8601 time of the entry. | +| `session_id` | Short random ID shared by every entry in one run, handy for correlating lines. | +| `level` | `DEBUG`, `INFO`, `WARNING`, or `ERROR`. | +| `operation` | The action being logged, e.g. `service_call`, `download_init`, `drm_get_widevine_license`, `vault_get_key`, `session_end`. | +| `message` | Human-readable description (redacted). | +| `service` | Service tag the entry belongs to, when applicable. | +| `context` | Arbitrary structured detail for the operation. | +| `request` / `response` | HTTP method/URL and response metadata for network operations. | +| `duration_ms` | How long a timed operation took. | +| `success` | Whether the operation succeeded. | +| `error` | On failures: the exception `type`, `message`, and a full `traceback` array (all redacted). | + +### Reading and filtering the log + +Because each line is independent JSON, `jq` is the natural tool: + +```console +# Show only error-level entries +jq 'select(.level == "ERROR")' unshackle_debug_NETFLIX_20260703-142530.jsonl + +# List every operation and whether it succeeded +jq -c '{operation, success}' unshackle_debug_NETFLIX_20260703-142530.jsonl + +# Pull the traceback out of the first error +jq -r 'select(.error) | .error.traceback[]' unshackle_debug_NETFLIX_20260703-142530.jsonl + +# Show only DRM / licensing operations +jq 'select(.operation | startswith("drm_"))' unshackle_debug_NETFLIX_20260703-142530.jsonl +``` + +The session's binary-tool versions (Shaka Packager, mp4decrypt, mkvmerge, FFmpeg, FFprobe) are logged near the start of a `dl` run, which is often the fastest way to confirm a muxing or decryption failure is really a wrong-version problem. + +--- + +## Debugging DRM keys with `debug_keys` + +By default, content-decryption **keys are redacted** in the JSON debug log. Any field that looks like a key is written as `[REDACTED]`. This keeps logs shareable. When you are specifically debugging a key-retrieval or decryption problem and need to see the actual KID/KEY values that were fetched, enable `debug_keys`: + +```yaml +debug_keys: true +``` + +```console +unshackle cfg debug_keys true +``` + +With `debug_keys: true`, the debug log stops masking key fields, so you can confirm which keys came back from a licence server or vault and compare them against what decryption expected. + +!!! warning "Keys are sensitive, turn this back off" + `debug_keys` writes real decryption keys into the log file in cleartext. Only enable it while actively diagnosing a key issue, never share a log produced with it on, and set it back to `false` when you're done. Note that `debug_keys` only affects the **JSON debug log file**. It does not change what the console prints. + +Fields that are *always* redacted regardless of `debug_keys` (anything whose name contains `password`, `token`, `secret`, `auth`, or `cookie`) stay masked, so credentials never leak even in a keys-on log. + +--- + +## Debugging HTTP traffic with `debug_requests` + +When you pass `-d`, unshackle raises its own modules to `DEBUG` but deliberately **silences** the noisy low-level HTTP libraries (`urllib3`, `requests`, `rnet`, `httpx`, `httpcore`, `hpack`, `h2`) back down to `WARNING`. Without this, a single download would bury the useful output under thousands of connection-pool and HTTP/2 frame messages. + +If you need that low-level traffic (for example when a request is failing at the TLS or connection layer and you need to see the underlying library's own logs), enable `debug_requests`: + +```yaml +debug_requests: true +``` + +```console +unshackle cfg debug_requests true +``` + +With `debug_requests: true`, those libraries are left at `DEBUG` level when you run with `-d`, so their request/response chatter appears in the console. + +!!! note "`debug_requests` needs `-d` to have any effect" + The `debug_requests` toggle only decides whether the HTTP libraries are silenced when the `-d` flag is active. If you are not running with `-d` (verbose console logging), it changes nothing. Combine them: `unshackle -d dl ...` with `debug_requests: true` in config. + +--- + +## What gets redacted + +unshackle redacts logged strings so that debug output is safe to share. Three passes are applied to every logged message, error, and structured value: + +- **Secrets**: passwords, tokens, API secrets, auth values, and cookies are replaced with `[REDACTED]`. +- **URLs**: every `http(s)` URL is collapsed to `redacted` (keeping only the file extension, so a manifest still shows as `redacted.mpd` and a segment as `redacted.m4s`). This hides content, CDN, manifest, and licence-server locations while preserving the *type* of resource. +- **Local paths**: install-root, virtualenv, and home-directory prefixes are replaced with tokens, so your username and machine layout don't appear in logs. + +Path redaction is controlled by the `redact_paths` config key, which is **on by default**: + +```yaml +redact_paths: true # default — mask local base directories in logged paths +``` + +Set `redact_paths: false` if you are debugging a path problem locally and want to see full, unmasked paths in the console and logs. Secret and URL redaction in the JSON debug log are always applied and are not affected by this toggle. + +--- + +## Checking your environment + +Many "download failed" problems are really missing external tools or a config file that isn't where you think it is. The [`unshackle env`](cli-reference.md#env) command group answers both questions before you dig into logs. + +### `env check` + +`unshackle env check` verifies every external binary unshackle can use and prints a table of what's installed: + +```console +unshackle env check +``` + +It reports each tool's status (`✓` installed / `✗` missing), whether it is **required** or optional, and what it's for. A summary line tells you the installed-vs-total count and, if anything mandatory is absent, exactly which required tools are missing. + +The required tools (unshackle will not get far without them) are: + +| Tool | Purpose | +|---|---| +| **FFmpeg** | Media processing | +| **FFprobe** | Media analysis | +| **MKVToolNix** (`mkvmerge`) | MKV muxing | +| **mkvpropedit** | MKV metadata | +| **Shaka Packager** | DRM decryption | + +Optional tools unlock extra capabilities. For example, `mp4decrypt` (alternative decryptor), `dovi_tool` / `HDR10Plus_tool` (Dolby Vision / HDR10+), `SubtitleEdit` / `CCExtractor` (subtitle conversion and CC extraction), `MPV` / `FFplay` (playback preview), and `git` (service repositories) or `docker` (Gluetun VPN, see [Proxies & VPN](proxies-and-vpn.md)). + +!!! tip "First thing to run when something breaks" + If a decryption step errors, a mux fails, or a Dolby Vision title won't process, run `unshackle env check` first. A red `✗` next to the relevant tool is the answer far more often than a bug in unshackle. + +### `env info` + +`unshackle env info` shows where unshackle is reading configuration from and where every working directory lives: + +```console +unshackle env info +``` + +- If a config file was found, it prints the path it was **loaded from**. If none was found, it lists every location it *would* accept one (see [Config file discovery](#why-is-my-config-not-being-picked-up) below), so you can create the file in the right place. +- It then prints a **Directories** table for every configured path: `downloads`, `temp`, `cache`, `cookies`, `logs`, `exports`, `wvds`, `prds`, the `services` search paths, and more. + +This is the authoritative way to answer "where are my logs / downloads / cookies actually going?" The defaults are relative to the installed package, and any `directories:` overrides in your config are reflected here. + +### `env clear` + +Stale cache or leftover temp files can cause confusing behaviour (an old cached title list, a half-written temp file). `env clear` wipes those directories safely: + +```console +unshackle env clear cache # clear the whole cache directory +unshackle env clear cache NETFLIX # clear just one service's cache subdirectory +unshackle env clear temp # clear the temp working directory +``` + +Each command reports how many files it removed and how much space it freed, then recreates the empty directory. + +!!! tip "Cache-related weirdness" + If a service keeps returning stale titles or metadata even after the source changed, clear that service's cache with `unshackle env clear cache <SERVICE>`. See [Vaults](vaults.md) for the separate, persistent key-vault store, which `env clear` does **not** touch. + +--- + +## Common failure modes and how to inspect them + +!!! example "A required tool is missing" + **Symptom:** decryption, muxing, or analysis fails early, sometimes with a "binary not found" style error. + + **Inspect:** run `unshackle env check` and look for a red `✗` on a **required** row. Install the missing tool and re-run. + +!!! example "Decryption produces garbage or fails" + **Symptom:** the download completes but the file won't play, or the decrypt step errors. + + **Inspect:** enable `debug_keys` and run with `-d`, then check the JSON debug log's `drm_*` and `vault_*` operations to confirm which keys were fetched and from where. Also confirm Shaka Packager (or `mp4decrypt`) shows a sane version in the `download_init` binary-versions entry. See [DRM & CDM](drm-and-cdm.md). + +!!! example "A network request fails at the connection or TLS layer" + **Symptom:** timeouts, connection resets, or handshake errors that unshackle's own logs don't fully explain. + + **Inspect:** set `debug_requests: true` and run with `-d` to surface the underlying `urllib3` / `rnet` / `httpx` logs. In the JSON debug log, filter for `service_call` operations to see which request failed. If a proxy is involved, cross-check [Proxies & VPN](proxies-and-vpn.md). + +!!! example "The wrong titles/metadata keep coming back" + **Symptom:** a service returns outdated titles even after the source changed. + + **Inspect:** the title cache is likely serving stale data. Clear it with `unshackle env clear cache <SERVICE>`, or bypass it for one run using the `dl` no-cache / reset-cache options (see [Downloading](downloading.md) and the [CLI Reference](cli-reference.md#dl)). + +!!! example "A config value doesn't seem to apply" + **Symptom:** a setting you edited has no effect. + + **Inspect:** run `unshackle env info` to confirm **which** config file is actually loaded, and `unshackle cfg --list` to dump the effective values. It's common to edit a config file in a location unshackle isn't reading from. See below. + +--- + +## Why is my config not being picked up? + +unshackle loads the **first** `unshackle.yaml` it finds, in this order: + +1. `unshackle.yaml` inside the package's namespace directory. +2. `unshackle.yaml` in the parent of that directory. +3. `unshackle.yaml` in your OS user-config directory: on Linux `~/.config/unshackle/unshackle.yaml`, on Windows `%LOCALAPPDATA%\unshackle\unshackle.yaml`, on macOS `~/Library/Application Support/unshackle/unshackle.yaml`. + +If you have config in more than one of these, an earlier one wins and your edits to a later file are silently ignored. `unshackle env info` prints the path that was actually loaded (or, if none exists, lists all three candidate locations so you can create the file correctly). See [Configuration](cli-reference.md#cfg) for editing values with the `cfg` command. + +--- + +!!! note "Developer note: structured logging in code" + Service and command code emits entries through the shared `log_event(...)` helper (and the `timed_operation(...)` context manager for timing a block) from `unshackle.core.utilities`. These are no-ops when debug logging is disabled, so they are always safe to leave in place. When adding a new feature, call `log_event("my_feature_event", message="...", context={...})` rather than reimplementing the "is the logger enabled?" guard. The redaction and JSON-serialisation are handled for you, and any field whose name looks like a key/secret/token is redacted automatically (keys only appear when the run has `debug_keys` enabled). + +### Logging conventions when adding entries + +The debug log is tuned for **developers troubleshooting pipeline flow** (maximum signal, minimum noise), not for end users. A few conventions keep it that way. They are not enforced by the `log_event` signature, so they are easy to miss and worth stating explicitly. + +!!! tip "Log levels are a flow skeleton, not just severity" + Treat `level` as structure, not decoration. Emit exactly **one `INFO` milestone per pipeline stage** so that `jq 'select(.level=="INFO")'` reads back the whole end-to-end flow of a run. Everything internal goes to `DEBUG`; failures go to `ERROR`. Keeping `INFO` sparse is the whole point. It is the line a developer skims first to see where a run got to before it broke. + + ```console + # Read the pipeline's high-level flow, one line per stage + jq 'select(.level == "INFO")' unshackle_debug_NETFLIX_20260703-142530.jsonl + ``` + +!!! warning "No raw dumps" + Never log a full `Tracks` object, an MPD/manifest body, or a response payload. Log only **counts, ids, sizes, and `safe_display_url(url)`**. This keeps the log low-noise and prevents leaking content or manifest data into the file *before* redaction even runs. Redaction is a backstop, not a licence to dump raw objects. + +!!! note "Operation names: `<area>_<event>`, no registry" + Operation names are plain lowercase strings following an `<area>_<event>` convention: `manifest_dash_parse`, `drm_decrypt`, `vault_get_key`, `tool_run`. They are written inline at each call site on purpose: there is **no central registry or enum**, and you should not go hunting for one or create one. The absence is a deliberate design choice, not an omission. + +!!! note "Message reads alone; data lives in fields" + Give every entry a single-sentence `message` that stands on its own, and put all structured data (`context`, counts, ids, `duration_ms`) in **fields** rather than baking it into the prose. This separation is what lets the log be both human-scannable and machine-filterable; a message like `"Parsed DASH manifest"` with `context={"video": 4, "audio": 2}` beats `"Parsed manifest with 4 video and 2 audio tracks"`. + +!!! tip "Let tool runs log themselves" + For external-tool calls (FFmpeg, mkvmerge, `dovi_tool`, and friends), route through `run_step` / `ffprobe`, which auto-emit a `tool_run` entry capturing the binary's version, return code, and duration consistently. Only fall back to calling `log_tool_run` manually for a direct `subprocess.run` you genuinely can't route through `run_step`. Auto-logging is the preferred path precisely because it captures those fields the same way every time. diff --git a/docs/guide/vaults.md b/docs/guide/vaults.md new file mode 100644 index 00000000..a78a7c1a --- /dev/null +++ b/docs/guide/vaults.md @@ -0,0 +1,318 @@ +# Key Vaults + +A **key vault** is a store of DRM **content keys** that unshackle remembers between downloads. Once a title's keys have been recovered, unshackle saves them to your configured vaults. The next time you download something that reuses those same keys, unshackle finds them in a vault instead of performing another license exchange. The download starts faster and puts less load on the service. + +Vaults are entirely optional, but if you download regularly they are one of the biggest quality-of-life improvements you can configure. + +!!! tip "Vaults save license calls even on the very first download" + The benefit is not limited to repeat downloads. Within a single run, a service often issues the *same* Key ID and content key for both the video and audio tracks, and for several resolutions or bitrates of the same title. Once unshackle has recovered a key it writes it to your vaults immediately, so the moment the next track presents a KID it has already seen, the key is served from the vault instead of triggering another license exchange. A single fetched key can therefore satisfy many tracks in one run. + +## What a vault stores + +Every key is stored as a **KID → KEY** pair, grouped by **service**: + +- **KID** (Key ID): a 16-byte identifier for a specific encrypted stream, stored as 32 lowercase hex characters with no dashes. +- **KEY**: the 16-byte content key that decrypts that stream, also 32 hex characters. +- **Service**: the canonical service tag (for example `DSNP`, not `DisneyPlus`) that namespaces the key. Each service gets its own logical table. + +!!! note "Why keys are matched by KID" + Vaults match keys by KID, never by PSSH/CENC header. A KID does not change unless the underlying video file itself changes, whereas the PSSH box can vary between requests for the very same content. That makes the KID a far more reliable match key. + +!!! info "Null keys are ignored everywhere" + A key made entirely of zeroes (`00000000000000000000000000000000`) is treated as "no key". Vault lookups skip it, and every backend raises an error if you try to add one. This prevents a placeholder key from masking a real one. + +## How lookups and caching work + +When unshackle needs a key during a download it asks your vaults **in the order they are listed in your config**. It returns the first non-null key it finds and stops searching. If a vault raises a permission error or is unreachable, that vault is skipped and the search continues with the next one. + +When new keys are recovered from a license, unshackle pushes them to **all** configured vaults, so every vault gradually converges on the same set of keys. Two rules apply to pushing: + +- Keys that already exist in a vault are skipped (existing data is never overwritten or deleted). +- A vault marked `no_push: true` receives lookups but is never written to. This is useful for read-only or shared upstream vaults you do not own. + +!!! tip "A good two-vault setup" + A common arrangement is a fast local **SQLite** vault plus a shared remote vault (**MySQL**, **HTTP**, or **API**). The local vault answers instantly and works offline; the remote vault lets you share keys across machines or with a group. + +## Configuring vaults + +Vaults are declared under the `key_vaults` list in your unshackle config file. Each entry is a mapping with two required fields, `type` (which backend to use) and `name` (a label you choose), plus whatever settings that backend needs. + +```yaml title="unshackle.yaml" +key_vaults: + - type: SQLite + name: local + path: ~/.local/share/unshackle/vaults/local.db + - type: MySQL + name: shared + host: vault.example.com + database: unshackle + username: reader + password: "s3cret" +``` + +The `type` value is matched to a vault backend by name. The four backends that ship with unshackle are `SQLite`, `MySQL`, `HTTP`, and `API`, described below. + +!!! warning "SQLite vaults are loaded critically" + During a download, a **SQLite** vault that fails to load will **abort the run**. The assumption is that a local database you configured should always be available. `MySQL`, `HTTP`, and `API` vaults are loaded leniently: if one fails to load it is logged and skipped, and the download continues with the remaining vaults. + +### `vault_timeout` + +Network-backed vaults (`HTTP` and `API`) accept an optional per-vault `timeout` in seconds. If you omit it, unshackle injects the global default: + +```yaml title="unshackle.yaml" +vault_timeout: 10.0 # default, in seconds +``` + +A `timeout` set inline on an individual vault always wins over this global value. The `SQLite` and `MySQL` backends do not take a `timeout` field. + +--- + +## SQLite backend + +A locally-accessed SQLite database file. This is the simplest and fastest option and needs no server. + +```yaml title="unshackle.yaml" +key_vaults: + - type: SQLite + name: local + path: ~/.local/share/unshackle/vaults/local.db +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `type` | yes | Must be `SQLite`. | +| `name` | yes | A label for this vault. | +| `path` | yes | Path to the database file. `~` is expanded. The file (and its tables) are created automatically on first use. | +| `no_push` | no | If `true`, keys are read but never written. Defaults to `false`. | + +Each service gets its own table (named after the service tag), created on demand with `kid` and `key_` columns. Connections use WAL journaling with a 30-second busy timeout, so the same database file can be used safely from multiple threads. + +!!! warning "Moving the .db file silently forks your vault" + Because the file is created on demand, if you relocate an existing `.db` without updating `path` to match, unshackle will simply create a fresh empty database at the old location, leaving you with two divergent vaults and no error to warn you. Always update the config path whenever you move the file. + +!!! tip "Keep a local SQLite vault even when you share a MySQL one" + The SQLite vault is best thought of as an offline-only backup should anything ever happen to a shared `MySQL` vault. Each member of a group should keep their own local SQLite vault alongside the shared database, so no one loses their keys if the shared vault goes down or away. + +--- + +## MySQL backend + +A remotely-accessed MySQL/MariaDB database, ideal for sharing keys across several machines. It connects via `pymysql`. MariaDB is recommended over MySQL where you have the choice. + +!!! warning "The vault connects directly to the database server" + A `MySQL` vault (like `SQLite`) speaks the database wire protocol directly to the host or IP you configure. It cannot sit behind a PHP API or any other application layer. A common real-world snag: many hosting providers firewall their MySQL server to their own internal network, so a database on shared hosting is frequently unreachable from the outside world even when your credentials are perfectly correct. If connections time out, confirm the server accepts remote connections from your address before suspecting the config. + +```yaml title="unshackle.yaml" +key_vaults: + - type: MySQL + name: shared + host: vault.example.com + database: unshackle + username: unshackle + password: "your-password" + port: 3306 +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `type` | yes | Must be `MySQL`. | +| `name` | yes | A label for this vault. | +| `host` | yes | Database server hostname or IP. | +| `database` | yes | Database (schema) name. | +| `username` | yes | Login user. | +| `no_push` | no | If `true`, keys are read but never written. | + +Any additional fields you provide (such as `password`, `port`, or `ssl`) are passed straight through to `pymysql.connect`, so you can use the full range of its connection options. + +!!! warning "Account security is on you, not on unshackle" + unshackle enforces none of this. It is operational policy for a database that may be shared with a group. Never connect with the `root` account, not even for yourself. Give every account a password, and never let two users share a username/password. Restrict each account's access to only the `unshackle` database, and grant normal users just `SELECT` and `INSERT`. Reserve the `CREATE` grant for a single trusted user who bootstraps the service tables (see [`kv prepare`](#kv-prepare-pre-create-service-tables)); everyone else can insert new keys but should not be creating tables. + +!!! note "Grants determine what the vault can do" + On connection the backend reads its own `SHOW GRANTS`. It requires at least `SELECT` to load at all (a missing `SELECT` grant makes the vault fail to load). Writing keys additionally needs `INSERT`, and creating a new service table needs `CREATE`. A read-only account still works fine as a lookup source. It simply cannot push new keys. Because `MySQL` is loaded leniently during downloads, a permission problem will not abort the run. + +--- + +## HTTP backend + +A flexible HTTP vault that speaks one of three wire protocols, selected with `api_mode`. Use this to integrate with a community or self-hosted vault service. + +| `api_mode` | Transport | Authentication | Notes | +|------------|-----------|----------------|-------| +| `query` (default) | `GET` with query parameters | `username` + `password` | Full read and write. | +| `json` | `POST` with a JSON payload | `password` used as a token | Read and write per key; cannot bulk-enumerate keys. | +| `decrypt_labs` | `POST` in DecryptLabs format | `api_key` sent as a header | **Read-only**. `no_push` is forced on. | + +=== "query mode" + + ```yaml title="unshackle.yaml" + key_vaults: + - type: HTTP + name: community + host: https://vault.example.com/api + api_mode: query + username: myuser + password: "your-password" + ``` + + `username` is required in query mode. Both the username and password are sent as query parameters on each request. + +=== "json mode" + + ```yaml title="unshackle.yaml" + key_vaults: + - type: HTTP + name: jsonvault + host: https://vault.example.com/rpc + api_mode: json + password: "your-api-token" + ``` + + Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling. + +=== "decrypt_labs mode" + + ```yaml title="unshackle.yaml" + key_vaults: + - type: HTTP + name: decrypt_labs + host: https://api.decryptlabs.example/ + api_mode: decrypt_labs + api_key: "your-decrypt-labs-key" + ``` + + A read-only lookup source. The key is sent in a `decrypt-labs-api-key` header and the vault refuses all writes. + +| Field | Required | Description | +|-------|----------|-------------| +| `type` | yes | Must be `HTTP`. | +| `name` | yes | A label for this vault. | +| `host` | yes | Base URL of the vault API. | +| `api_mode` | no | `query` (default), `json`, or `decrypt_labs`. | +| `username` | for `query` | Required in query mode; ignored otherwise. | +| `password` | one of these | Password (query mode) or token (json mode). | +| `api_key` | one of these | Alternative to `password`; used for `decrypt_labs`. If both are given, `api_key` wins. | +| `no_push` | no | Read-only when `true` (always `true` for `decrypt_labs`). | +| `timeout` | no | Request timeout in seconds; defaults to `vault_timeout`. | + +You must supply either `password` or `api_key`, or the vault will fail to load. + +--- + +## API backend + +A simple RESTful HTTP vault with a fixed, well-defined protocol. Authentication is a bearer token. + +```yaml title="unshackle.yaml" +key_vaults: + - type: API + name: myapi + uri: https://vault.example.com/api + token: "your-bearer-token" +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `type` | yes | Must be `API`. | +| `name` | yes | A label for this vault. | +| `uri` | yes | Base URI of the vault API (a trailing slash is trimmed). | +| `token` | yes | Bearer token, sent as `Authorization: Bearer <token>`. | +| `no_push` | no | If `true`, keys are read but never written. | +| `timeout` | no | Request timeout in seconds; defaults to `vault_timeout`. | + +The backend talks to endpoints under `uri` (for example `GET {uri}/{service}/{kid}` to look up a single key) and interprets a numeric `code` field in each JSON response to detect errors such as an invalid token, rate limiting, or an invalid service tag. + +!!! warning "This is unshackle's own protocol, not a generic vault client" + The `API` backend expects unshackle's specific HTTP request and response shape. An API or HTTP key-vault endpoint built for a different project or service will generally *not* be compatible. You cannot simply point this at an arbitrary third-party vault API and expect it to work. To bridge another service, put an adapter in front of it that speaks this format, or use the `HTTP` backend if one of its wire protocols matches. When pushing many keys at once it batches requests and automatically shrinks the batch size if the server rejects an over-large request. + +!!! tip "DecryptLabs shortcut" + An `API` vault whose `name` contains `decrypt_labs` will have its `token` filled in automatically from a global `decrypt_labs_api_key` in your config if you leave `token` unset. This lets you keep the key in one place. + +--- + +## Managing vaults with `unshackle kv` + +The `kv` command group manages the contents of your configured vaults. Every subcommand refers to vaults **by the `name` you gave them** in `key_vaults`; an unknown name is an error. + +```console +$ unshackle kv --help +``` + +### `kv add`: import keys from a file + +Add `KID:KEY` pairs to one or more vaults for a given service. The file must contain one pair per line as `HEX:HEX` (32 hex characters each); lines that do not match are ignored. + +```console +$ unshackle kv add keys.txt DSNP local shared +``` + +```text title="keys.txt" +9a04f07998404286ab92e65be0885f95:b1c2d3e4f5a6978012345678deadbeef +0e7cc1a4e2f9411db0f24a3fbb8f8b21:00112233445566778899aabbccddeeff +``` + +The service argument (`DSNP` above) is normalized to its canonical tag automatically. Keys that already exist in a vault are reported as skipped. + +### `kv search`: find a key by KID + +Search your vaults for a KID and print any matching key. + +```console +$ unshackle kv search 9a04f07998404286ab92e65be0885f95 +``` + +The KID must be 32 hex characters (dashes are stripped for you). By default every configured vault and every service table is scanned; narrow the search with options: + +| Option | Description | +|--------|-------------| +| `-s`, `--service` | Limit the search to a single service tag. | +| `-v`, `--vault` | Limit the search to one configured vault by name. | + +!!! note + Remote vaults that cannot enumerate their service tables (for example an `HTTP` vault in `json` or `decrypt_labs` mode) can only be searched when you also pass `--service`. Without it, they are skipped. + +### `kv copy`: copy keys between vaults + +Copy every key from one or more source vaults into a single destination vault. Existing rows are never altered; only genuinely new KIDs are added, and null keys are skipped. + +```console +$ unshackle kv copy shared local +``` + +The first argument is the destination; the rest are sources. You can supply several sources at once: + +```console +$ unshackle kv copy local shared community +``` + +| Option | Description | +|--------|-------------| +| `-s`, `--service` | Only copy the given service. | +| `-l`, `--local-only` | Only copy services you have installed locally, skipping tables for unknown services. | + +`--service` and `--local-only` are mutually exclusive. + +### `kv sync`: mirror vaults both ways + +Ensure two or more vaults each end up with every key the others have. This is effectively a two-way `copy` between each pair, so afterwards all listed vaults hold the same set of keys. + +```console +$ unshackle kv sync local shared +``` + +It accepts the same `--service` and `--local-only` options as `copy`, and requires at least two vaults. + +### `kv prepare`: pre-create service tables + +Create the per-service tables on table-based vaults (`SQLite`, `MySQL`) ahead of time, for every service you have installed. Vaults that do not use tables (the network-backed `HTTP` and `API` backends) are skipped. + +```console +$ unshackle kv prepare shared +``` + +This is mainly useful with a `MySQL` vault where you want the tables created once by an account that has the `CREATE` grant. + +--- + +## See also + +- [Downloading](downloading.md): how keys are recovered and used during a download. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..2ab8380a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,92 @@ +# unshackle + +**unshackle** is a modular archival tool for **movies, TV, and music**. It fetches +media from streaming manifests, handles the DRM, and muxes everything into clean, +well-tagged files, all driven from a single command-line tool, or served over a +REST API for automation. + +It is a fork of [Devine](https://github.com/devine-dl/devine) with first-class +DASH, HLS, and Smooth Streaming (ISM) parsing, both **Widevine** and **PlayReady** +DRM support, and a built-in HTTP server. + +!!! warning "Use responsibly" + unshackle is licensed under [GPL-3.0](https://github.com/unshackle-dl/unshackle/blob/master/LICENSE). + Do not use it for content you do not have the rights to. Keep the core free and + open, keep service code private, and be kind. + +## What it does + +Point unshackle at a title on a streaming service and it will: + +- Fetch the title metadata (movies, full series, seasons, episodes, albums, tracks). +- Parse the streaming manifest: **DASH**, **HLS**, or **Smooth Streaming (ISM)**. +- Select the tracks you asked for by resolution, codec, bitrate, range, language, + and channel layout. +- Acquire decryption keys via a **Widevine** or **PlayReady** CDM, with optional + **key vaults** so keys are reused instead of re-licensed. +- Decrypt, extract closed captions, repack, and **mux** the result into a Matroska + (`.mkv`) file with correct language, forced, and default flags. +- Tag and name the output using configurable filename templates. + +## Feature highlights + +| Area | What you get | +|---|---| +| Media types | Movies, TV series/seasons/episodes, and music albums/tracks | +| Manifests | DASH, HLS, and Smooth Streaming (ISM) | +| DRM | Widevine and PlayReady, with ClearKey support | +| Video | AVC, HEVC, VC-1, VP8, VP9, AV1; SDR, HLG, HDR10, HDR10+, Dolby Vision, and Dolby Vision **hybrid** merging | +| Audio | AAC, AC-3, E-AC-3, AC-4, Opus, Vorbis, DTS, ALAC, FLAC; Dolby Atmos handling; per-channel-layout selection | +| Subtitles | SRT, WebVTT, ASS/SSA, TTML and more, with optional SDH stripping and format conversion | +| Key vaults | SQLite, MySQL, and HTTP/API vaults to store and share content keys | +| Proxies | Basic proxies plus NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers | +| Services | Pluggable service modules, loadable from local folders or git repositories | +| Automation | A REST API with a job queue, live progress, history, and remote-download sessions | + +## Installation at a glance + +=== "Install as a tool" + + ```shell + uv tool install git+https://github.com/unshackle-dl/unshackle.git + unshackle --help + ``` + +=== "Run from a clone" + + ```shell + git clone https://github.com/unshackle-dl/unshackle.git + cd unshackle + uv run unshackle --help + ``` + +!!! note "External tools" + unshackle shells out to FFmpeg, MKVToolNix, shaka-packager, Bento4, and + `dovi_tool` for decryption, repacking, muxing, and Dolby Vision work. See + [Installation](getting-started/installation.md) for the full list and recommended versions. + +## A first download + +Every download is three parts: the `dl` command, a **service tag**, and the +service's title argument (a URL, ID, or slug). + +```shell title="Download a 1080p title with English audio and subtitles" +unshackle dl -q 1080 -l en NF 81234567 +``` + +The `dl` command carries all the quality, language, and output flags; the service +tag (`NF` above) selects which streaming service to talk to, and the value after it +is passed to that service. See the [Quickstart](getting-started/quickstart.md) to go from a fresh +install to your first file. + +## Where to go next + +- **[Installation](getting-started/installation.md)**. Install unshackle and the external tools it depends on. +- **[Quickstart](getting-started/quickstart.md)**. Configure `unshackle.yaml` and run your first download. +- **[Downloading](guide/downloading.md)**. The full `dl` command guide: quality, codecs, languages, track selection, and output. +- **[Configuration](getting-started/configuration-file.md)**. Every `unshackle.yaml` key, directories, vaults, proxies, and naming templates. +- **[REST API](dev/rest-api/index.md)**. Run the `serve` HTTP server and drive downloads programmatically. + +!!! tip "Getting help" + Run `unshackle --help` or `unshackle dl --help` at any time. The community + [Discord](https://discord.gg/mHYyPaCbFK) is the fastest place to ask questions. diff --git a/docs/reference/api/config.md b/docs/reference/api/config.md new file mode 100644 index 00000000..3961a558 --- /dev/null +++ b/docs/reference/api/config.md @@ -0,0 +1,12 @@ +# Config API + +Auto-generated reference for the [`Config`][unshackle.core.config.Config] object that models the +`unshackle` configuration file. For a human-friendly walkthrough of every key, see the +[Configuration Reference](../configuration.md). + +::: unshackle.core.config + options: + show_root_heading: false + show_source: false + members_order: source + heading_level: 2 diff --git a/docs/reference/api/drm.md b/docs/reference/api/drm.md new file mode 100644 index 00000000..c56becd0 --- /dev/null +++ b/docs/reference/api/drm.md @@ -0,0 +1,13 @@ +# DRM API + +Auto-generated reference for the DRM systems: Widevine, PlayReady, ClearKey, and related helpers. +For configuring DRM as a user, see [DRM & CDM Setup](../../guide/drm-and-cdm.md). + +::: unshackle.core.drm + options: + show_root_heading: false + show_source: false + members_order: source + heading_level: 2 + filters: + - "!^_" diff --git a/docs/reference/api/service.md b/docs/reference/api/service.md new file mode 100644 index 00000000..fa9bbee0 --- /dev/null +++ b/docs/reference/api/service.md @@ -0,0 +1,12 @@ +# Service API + +Auto-generated reference for the [`Service`][unshackle.core.service.Service] base class that every +service plugin subclasses. See [Creating a Service](../../dev/creating-a-service.md) for a guided +walkthrough and [Service Architecture](../../dev/service-architecture.md) for the wider object model. + +::: unshackle.core.service + options: + show_root_heading: false + show_source: true + members_order: source + heading_level: 2 diff --git a/docs/reference/api/titles.md b/docs/reference/api/titles.md new file mode 100644 index 00000000..503be49a --- /dev/null +++ b/docs/reference/api/titles.md @@ -0,0 +1,14 @@ +# Titles API + +Auto-generated reference for the title model: Movies, Series/Episodes, and Music/Albums/Songs. +A service returns one of these collections from its title lookup. See +[Creating a Service](../../dev/creating-a-service.md). + +::: unshackle.core.titles + options: + show_root_heading: false + show_source: false + members_order: source + heading_level: 2 + filters: + - "!^_" diff --git a/docs/reference/api/tracks.md b/docs/reference/api/tracks.md new file mode 100644 index 00000000..af67bb65 --- /dev/null +++ b/docs/reference/api/tracks.md @@ -0,0 +1,14 @@ +# Tracks API + +Auto-generated reference for the track model: [`Tracks`][unshackle.core.tracks.Tracks], +[`Track`][unshackle.core.tracks.Track], and the media-specific subclasses. See +[Service Architecture](../../dev/service-architecture.md) for how these fit together. + +::: unshackle.core.tracks + options: + show_root_heading: false + show_source: false + members_order: source + heading_level: 2 + filters: + - "!^_" diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md new file mode 100644 index 00000000..4f34e023 --- /dev/null +++ b/docs/reference/configuration.md @@ -0,0 +1,730 @@ +# Configuration Reference + +This page is the exhaustive, authoritative reference for **every** key you can set in +`unshackle.yaml`. For each key you will find its name, expected type, default value, and +what it does. Keys are grouped by area so you can jump straight to the part of unshackle +you are configuring. + +If you are new to the config file (where it lives, how unshackle discovers it, and how to +edit it), start with [The Configuration File](../getting-started/configuration-file.md). +This page assumes you already know the basics and want precise details. + +!!! info "How defaults work" + Every key is optional. If you omit a key, unshackle uses the built-in default listed + here. There is **no schema validation**: unknown keys are silently ignored (they never + become settings), and the file is parsed with `yaml.safe_load`. An empty or missing + file means *all* defaults apply. + +!!! warning "Two keys behave specially on load" + - `curl_impersonate` is a **deprecated** alias of [`network`](#network-proxy) and emits a + `DeprecationWarning`. + - `scene_naming` has been **removed**. If it is present at all, unshackle exits + immediately with an error telling you to use [`output_template`](#output-naming) + instead. + + See [Deprecated & removed keys](#deprecated-removed-keys) at the end of this page. + +--- + +## Quick index + +| Area | Keys | +|------|------| +| [Directories](#directories) | `directories`, `filenames` | +| [Services & authentication](#services-auth) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` | +| [Download & processing](#download-processing) | `dl`, `subtitle`, `audio`, `muxing`, `language_tags` | +| [Output & naming](#output-naming) | `output_template`, `tag`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` | +| [DRM & CDM](#drm-cdm) | `cdm`, `remote_cdm`, `decryption` | +| [Network & proxy](#network-proxy) | `network`, `headers`, `proxy_providers` | +| [Key vaults](#key-vaults) | `key_vaults`, `vault_timeout` | +| [External API keys](#external-api-keys) | `tmdb_api_key`, `simkl_client_id`, `decrypt_labs_api_key`, `ipinfo_api_key` | +| [Caching & updates](#caching-updates) | `title_cache_enabled`, `title_cache_time`, `title_cache_max_retention`, `update_checks`, `update_check_interval` | +| [Logging, privacy & debug](#logging-privacy-debug) | `redact_paths`, `debug`, `debug_keys`, `debug_requests`, `set_terminal_bg` | +| [Deprecated & removed](#deprecated-removed-keys) | `curl_impersonate`, `downloader`, `scene_naming` | + +--- + +## Directories { #directories } + +The `directories` key is a **dict** mapping directory names to filesystem paths. Only the +names listed below are honoured; a handful are **protected** and cannot be moved (attempts +to override them are silently skipped). All defaults are computed relative to the installed +package, and every user-settable path is passed through `Path(...).expanduser()`, so `~` +works. + +```yaml title="unshackle.yaml" +directories: + downloads: ~/Media/unshackle + temp: /mnt/fast-scratch/unshackle-temp + wvds: ~/.unshackle/WVDs + prds: ~/.unshackle/PRDs +``` + +| Key | Type | Default | Overridable | Purpose | +|-----|------|---------|:-----------:|---------| +| `downloads` | path | `<repo>/downloads` | ✅ | Default output directory for finished files. | +| `temp` | path | `<repo>/temp` | ✅ | Temporary working files during download/decrypt/mux. | +| `cache` | path | `<data>/cache` | ✅ | Generic cache, title cache, and the update-check store. | +| `cookies` | path | `<data>/cookies` | ✅ | Per-service cookie files (and VPN cookie/token files). | +| `logs` | path | `<data>/logs` | ✅ | Log files. | +| `exports` | path | `<data>/exports` | ✅ | Export JSON files. | +| `wvds` | path | `<data>/WVDs` | ✅ | Widevine device files (`.wvd`). | +| `prds` | path | `<data>/PRDs` | ✅ | PlayReady device files (`.prd`). | +| `dcsl` | path | `<data>/DCSL` | ✅ | DCSL data. | +| `commands` | path | `unshackle/commands` | ✅ | CLI command modules. | +| `services` | list \| path | `[unshackle/services]` | ✅ | Service search paths and/or remote repo specs (see below). | +| `vaults` | path | `unshackle/vaults` | ✅ | Vault backend modules. | +| `fonts` | path | `unshackle/fonts` | ✅ | Bundled fonts. | +| `user_configs` | path | `unshackle/` | ❌ protected | Where `unshackle.yaml` lives. | +| `data` | path | `unshackle/` | ❌ protected | Base for the data subdirectories above. | +| `core_dir` | path | `unshackle/core` | ❌ protected | Package core. | +| `namespace_dir` | path | `unshackle/` | ❌ protected | Package root. | +| `app_dirs` | — | `AppDirs("unshackle", False)` | ❌ protected | Internal AppDirs instance. | + +!!! note "The `services` directory is special" + `services` may be a **list**, and each entry can be either a local directory or a + **repository spec**: a git URL (`https://…`, `ssh://…`, `git@…`, or anything ending in + `.git`) or `owner/repo` shorthand. Repo specs are cloned and updated automatically; plain + paths are used as-is. **List order is priority**: the first source to define a service + tag wins, so put your own local service directories **last** to make them fallbacks. + + ```yaml + directories: + services: + - unshackle-dl/community-services # cloned from GitHub + - https://example.com/private.git@stable + - ~/my-services # local, lowest priority + ``` + +!!! info "How git-backed service repos are handled" + unshackle's use of git is **read-only on the remote**: it only ever performs a shallow + `clone`, `fetch`, `pull`, and a local `reset`; **nothing is ever pushed**. Private repos + are authenticated through your existing git credential helper, and unshackle stores no + tokens of its own. Clones live under `<first-local-services-dir>/_repos/<repo-name>/` (or + the bundled `unshackle/services`), and **nothing is written to the cache directory**. After + the first clone unshackle re-pulls **at most once every 24 h**, so it does not touch the + network on every run. + +!!! warning "The two refresh paths diverge on purpose" + The **automatic** 24 h-TTL refresh that happens during a normal `dl`/`search` run will + deliberately **refuse to refresh and exit** (naming the offending clone) if that clone + has uncommitted changes to tracked files or unpushed local commits, so a background pull can + never clobber work in progress. Untracked files (new service folders, `__pycache__`) never + block it. The **manual** `unshackle util refresh-services` command does the opposite: it + **hard-resets** the clone to upstream, discarding any local edits. Reach for the manual + command only when you actually want to throw local changes away. + +!!! warning "Read-only installs and reinstalls" + If the services directory lives inside the installed package, a reinstall can delete the + `_repos` clones. They are simply re-cloned on next use. On **read-only installs** you must + point `services` at a writable path, or cloning will fail. + +### Filenames { #filenames } + +The `filenames` key is a **dict** of templated file/name patterns. Each value is used +verbatim (no path processing). Braced fields like `{time}` and `{service}` are filled in at +runtime. + +| Key | Type | Default | Notes | +|-----|------|---------|-------| +| `log` | str | `"unshackle_{name}_{time}.log"` | Written under `directories.logs`. | +| `debug_log` | str | `"unshackle_debug_{service}_{time}.jsonl"` | Structured debug log, under `directories.logs`. | +| `config` | str | `"config.yaml"` | Per-service config file, under that service's directory. | +| `root_config` | str | `"unshackle.yaml"` | The main config filename itself. | +| `chapters` | str | `"Chapters_{title}_{random}.txt"` | Under `directories.temp`. | +| `subtitle` | str | `"Subtitle_{id}_{language}.srt"` | Under `directories.temp`. | + +--- + +## Services & authentication { #services-auth } + +### `services` + +- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` + +Per-service configuration, keyed by the canonical service tag (for example `AMZN`, `NF`, +`DSNP`). unshackle reads several well-known sub-keys out of each service's block: + +- **`proxy_map`**: a dict mapping `"provider:query"` or `"query"` to a proxy value, used to + auto-select a proxy for that service. +- **`title_map`**: an exact-match rename map applied to fetched titles (source name → + desired name), so a service that names a title differently from your library still matches. + +Individual services may read any additional keys they define. The whole block is handed to +the service as `self.config`. + +!!! tip "Why put secrets here and not in the service's own `config.yaml`?" + A service ships a per-service `config.yaml` inside its own folder, and that file is + typically shared/committed and **not** meant to be edited by you. So anything user- or + device-specific (API keys, account IDs, device attributes) belongs in this `services` + block of `unshackle.yaml` instead, where it is **merged over** the service's `config.yaml` + at load time. That is why the two config locations exist: one for shared service defaults, + one for your private overrides. + +```yaml +services: + AMZN: + proxy_map: + default: us + title_map: + "The Grand Tour: The Movie": "The Grand Tour Presents The Movie" +``` + +### `credentials` + +- **Type:** `dict` keyed by service tag  ·  **Default:** `{}` + +Per-service login credentials. Each value is a `username:password[:extra]` string (a list or +dict form is also accepted where a service supports it). These are parsed into `Credential` +objects; the credential's SHA-1 is also used as an account hash for cache keys. + +```yaml +credentials: + NF: user@example.com:hunter2 + AMZN: + - primary@example.com:pw1 + - secondary@example.com:pw2 +``` + +!!! tip "Cookies vs credentials" + Cookies are stored as files under [`directories.cookies`](#directories), not in this key. + A service's `authenticate()` accepts cookies, credentials, or both. + +### `firefox_cookies` + +- **Type:** `dict`  ·  **Default:** `{}` + +Settings for extracting cookies directly from a local Firefox profile. A service block is +expected to provide `hosts` (a list of cookie hostnames; entries shorter than 3 characters +are ignored to prevent dumping the whole cookie store) and an optional `local_storage` +boolean to also pull matching local-storage entries. Extraction is read-only. + +!!! note "Firefox does not need to be closed" + The extractor copies **both** `cookies.sqlite` **and** its WAL file into a `0700` temp + directory; the WAL copy exists to capture writes Firefox has not yet flushed to the main + DB, so the copy is current even against a running browser. The only real constraint is that + Firefox must not hold an exclusive write lock at the exact instant of the copy. The live + profile is otherwise left untouched. + +!!! tip "When you actually need `local_storage: true`" + Enable it only for services that keep their auth tokens in the browser's **localStorage** + (`webappsstore.sqlite`) rather than in HTTP cookies. If a service authenticates with plain + cookies, leaving it off is correct. + +!!! warning "Extraction fails silently to file cookies" + If extraction yields no cookies or fails for **any** reason, unshackle silently falls back + to the normal file-based cookie path (`cookies/<SERVICE>.txt` or + `cookies/<SERVICE>/<profile>.txt`). This is a safety net, but it also means that if you + expected Firefox cookies and file cookies are being used instead, extraction quietly failed. + Check that first when troubleshooting. + +### `remote_services` + +- **Type:** `dict`  ·  **Default:** `{}` + +Definitions of remote unshackle service servers, used by the `--remote` mode. Each entry +carries a `services` sub-dict of remote service tags, which unshackle turns into synthetic +CLI commands that run against the remote server. + +### `serve` + +- **Type:** `dict`  ·  **Default:** `{}` + +Configuration for the `serve` command (the built-in REST API server). + +!!! tip "Declaring `dl` defaults under `serve`" + You can nest `dl` flags (`downloads`, `workers`, `best_available`, and so on) inside + `serve`. The point is to raise concurrency and behaviour defaults **server-wide** (every + request the server handles picks them up) without having to change each client call + individually. + +--- + +## Download & processing { #download-processing } + +### `dl` + +- **Type:** `dict`  ·  **Default:** `{}` + +Default options forwarded to the `dl` command. Anything you would otherwise pass as a `dl` +flag can be given a persistent default here. See [Downloading](../guide/downloading.md) for +the flags themselves. + +### `subtitle` + +- **Type:** `dict`  ·  **Default:** `{}` + +Subtitle handling options. See [Subtitles](../guide/subtitles.md). + +### `audio` + +- **Type:** `dict`  ·  **Default:** `{}` + +Audio handling options. The most notable sub-key is **`codec_priority`**, an ordered list of +audio codecs used to break ties in track selection. + +```yaml +audio: + codec_priority: + - EAC3 + - AC3 +``` + +!!! note "`codec_priority` is a soft priority: nothing is dropped" + `codec_priority` only breaks ties **among tracks that share the same bitrate and language**. + Codecs not in the list keep their normal bitrate-based ordering and are simply placed after + all listed codecs, and none are ever removed. Two higher-level rules still override it: **Atmos + tracks always take precedence over `codec_priority`**, and **audio-description tracks are + always forced to the end** regardless of it. When `codec_priority` is unset, tracks sort by + bitrate alone, with those same Atmos/descriptive rules still applied. + +### `muxing` + +- **Type:** `dict`  ·  **Default:** `{}` + +Matroska (MKV) muxing options. + +### `language_tags` + +- **Type:** `dict`  ·  **Default:** `{}` + +Language-tag remapping and the rule engine behind the `{lang_tag}` filename variable. When a +`rules` list is present, each rule is evaluated in order and the first match's `tag` is used. +A rule may test: + +| Condition | Meaning | +|-----------|---------| +| `audio` | Matches against the audio track languages. | +| `subs_contain` | At least one subtitle language matches. | +| `subs_contain_all` | Every listed language must be present (scalar or list). | + +```yaml +language_tags: + rules: + - audio: [en] + subs_contain_all: [en] + tag: "" # native English release, no tag + - audio: [ja] + subs_contain: [en] + tag: "SUBBED" +``` + +--- + +## Output & naming { #output-naming } + +For the full picture of how templates render, see +[Output & naming](../guide/output-and-naming.md). The keys are summarised here. + +### `output_template` + +- **Type:** `dict`  ·  **Default:** `{}` + +Filename templates keyed by media kind, using `{variable}` placeholders. Recognised kinds are +`movies`, `series`, and `songs` (and `albums` for folder templates). A trailing `?` on a +variable (e.g. `{quality?}`) marks it optional: if empty, it and one adjacent separator are +dropped. + +```yaml title="unshackle.yaml" +output_template: + movies: "{title}.{year}.{quality?}.{source}-{tag}" + series: "{title}.{season_episode}.{episode_name?}.{quality?}.{source}-{tag}" + songs: "{track_number}. {title}" + folder: # special nested key, see below + movies: "{title} ({year})" + series: "{title} ({year})" +``` + +**The nested `folder` sub-key** controls output subfolders and is popped out of +`output_template`: + +- A **dict** sets per-kind folder templates (kinds: `movies`, `series`, `songs`, `albums`). +- A **str** sets a single folder template for everything. + +unshackle validates templates on load and emits warnings for unknown variables, non-string +values, filesystem-unsafe characters (`< > : " / \ | ? *`), empty templates, and unknown +folder-kind keys. + +??? example "All valid template variables" + `title`, `year`, `season`, `episode`, `season_episode`, `episode_name`, `date`, + `quality`, `resolution`, `source`, `tag`, `track_number`, `artist`, `album_artist`, + `album`, `disc`, `track_total`, `disc_total`, `release_type`, `genre`, `explicit`, + `isrc`, `upc`, `label`, `audio`, `audio_channels`, `audio_full`, `atmos`, `dual`, + `multi`, `video`, `hdr`, `hfr`, `edition`, `repack`, `lang_tag`. + +### Tagging & naming keys + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `tag` | str | `""` | Release-group tag appended to filenames and written into MKV tags. | +| `tag_group_name` | bool | `true` | Include the group name (`tag`) in the MKV `Group` tag. | +| `tag_imdb_tmdb` | bool | `true` | Write IMDb/TMDB/TVDB external-ID tags into the MKV (needs metadata providers). | +| `chapter_fallback_name` | str | `""` | Fallback chapter-name template, e.g. `"Chapter {i:02}"`. | +| `unicode_filenames` | bool | `false` | Allow Unicode in filenames; when `false`, names are ASCII-sanitised. | + +--- + +## DRM & CDM { #drm-cdm } + +The full conceptual guide lives at [DRM & CDM](../guide/drm-and-cdm.md). These are the config +keys. Device files themselves live in [`directories.wvds`](#directories) (`.wvd`) and +`directories.prds` (`.prd`). + +### `cdm` + +- **Type:** `dict`  ·  **Default:** `{}` + +Maps a service tag to the CDM (Widevine/PlayReady device) to use, with a `default` fallback. +Resolution is case-insensitive: an explicit CLI override wins, then the per-service entry, +then `default`. + +```yaml +cdm: + default: chromecdm_l3 + NF: android_l1 +``` + +A value may also be a **nested dict** for advanced selection by quality, DRM system, or +profile: + +```yaml +cdm: + AMZN: + ">=1080": android_l1 # by track height + "<1080": chromecdm_l3 + widevine: android_l1 # by DRM system + playready: sl3000 + default: chromecdm_l3 +``` + +Quality keys support `1080`, `>=1080`, `>720`, `<=576`, `<480` style comparisons; DRM keys +are `widevine` / `playready`. + +### `remote_cdm` + +- **Type:** `list[dict]`  ·  **Default:** `[]` + +A list of remote CDM server definitions. Each entry is matched by its `name` (referenced from +[`cdm`](#cdm)), and its `type` selects the backend: + +| `type` | Backend | Key fields | +|--------|---------|-----------| +| `decrypt_labs` | Decrypt Labs KeyXtractor | `host` (default `https://keyxtractor.decryptlabs.com`), `device_name`, `secret` (falls back to `decrypt_labs_api_key`) | +| `custom_api` | Fully YAML-configurable remote API | `device`, `auth`, `endpoints`, `request_mapping`, `response_mapping`, `caching`, `timeout` | +| *(none)*, `Device Type: PLAYREADY` | pyplayready `RemoteCdm` | `host`, `secret`, `device_name`, `security_level` (default 3000) | +| *(none)*, otherwise | pywidevine `RemoteCdm` | `host`, `secret`, `device_name`, `device_type`, `system_id`, `security_level` (default 3000) | + +```yaml +remote_cdm: + - name: keyxtractor + type: decrypt_labs + device_name: L1 + - name: my_wv_server + host: https://cdm.example.com + secret: s3cr3t + device_name: android_l1 + device_type: ANDROID + system_id: 26830 + security_level: 1 +``` + +### `decryption` + +- **Type:** `str` or `dict`  ·  **Default:** `"shaka"` + +Selects the tool used to physically decrypt CENC content. The value is compared +case-insensitively; `"mp4decrypt"` selects Bento4's mp4decrypt, and **anything else** +(including `"shaka"`) selects shaka-packager. + +=== "Global" + + ```yaml + decryption: shaka + ``` + +=== "Per-service" + + ```yaml + decryption: + default: shaka + SOMESERVICE: mp4decrypt + ``` + +--- + +## Network & proxy { #network-proxy } + +### `network` + +- **Type:** `dict`  ·  **Default:** `{}` + +TLS-fingerprinting and HTTP client settings for the rnet-based session. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `browser` | str | `"Chrome131"` | Impersonation preset. Must be an exact preset name (e.g. `Chrome131`, `Firefox135`, `Edge101`, `Safari18`, `OkHttp4_12`, `OkHttp5`, `Opera118`); unknown names raise an error. | +| `http1_only` | bool | *(unset)* | Force HTTP/1.1. | +| `http2_only` | bool | *(unset)* | Force HTTP/2. | +| `pool_max_idle_per_host` | int | *(unset)* | Connection-pool tuning. | +| `pool_max_size` | int | *(unset)* | Connection-pool tuning. | +| `tcp_nodelay` | bool | *(unset)* | Disable Nagle's algorithm. | + +```yaml +network: + browser: Firefox135 + http2_only: true +``` + +!!! note "When to force `http1_only`" + HTTP/2 is the right default on large CDNs, so leave it on. Force HTTP/1.1 only for a host that + throttles per-connection, or a slow origin stuck behind HTTP/2 flow control. In benchmarks + this won 30 to 50% on such hosts but **cost up to 27% on fast CDNs**, so enable it only when + your own measurements justify it. + +!!! warning "Renamed from `curl_impersonate`" + The old `curl_impersonate` section is a deprecated alias. If you still use it, unshackle + honours it (only when `network` is absent) but emits a `DeprecationWarning`. Rename it to + `network`. + +### `headers` + +- **Type:** `dict`  ·  **Default:** `{}` + +Default HTTP headers merged into every session unshackle creates. + +```yaml +headers: + Accept-Language: en-US,en;q=0.9 +``` + +!!! warning "Don't set `Accept-Encoding` (and similar) here" + Keep this block to sane cross-service defaults such as `Accept-Language` and `User-Agent`. + Compatibility headers like `Accept-Encoding` are set for you by the rnet HTTP backend as + part of its browser-impersonation profile, and overriding them can break the impersonation + fingerprint. Anything useful to only some services belongs in that service's config, not + here. + +### `proxy_providers` + +- **Type:** `dict`  ·  **Default:** `{}` + +Proxy/VPN provider configuration. Each sub-key names a provider, and its block is passed +straight to that provider's constructor. See [Proxies & VPN](../guide/proxies-and-vpn.md) for +the full provider guide. Recognised providers and their exit ports: + +| Provider | Config key | Credentials | Proxy scheme/port | +|----------|-----------|-------------|-------------------| +| Basic (static) | `basic` | country → URI(s) | as specified | +| NordVPN | `nordvpn` | 48-char **service** credentials | `https://…:89` | +| Surfshark | `surfsharkvpn` | 48-char **service** credentials | `https://…:443` | +| Windscribe | `windscribevpn` | service credentials | `https://…:443` | +| ExpressVPN | `expressvpn` | browser cookies / token cache | `https://cat:…@…:443` | +| ProtonVPN | `protonvpn` | TV login or exported cookies | `https://…:4443` (or `:443` Secure Core) | +| Gluetun | `gluetun` | per-VPN keys/creds | `http://localhost:{port}` (local Docker) | +| Hola | *(none, auto)* | none | `http://…:{peer}` | + +```yaml +proxy_providers: + basic: + us: http://user:pass@1.2.3.4:8080 + de: + - http://a.example:8080 + - socks5://b.example:1080 + nordvpn: + username: <48-char service username> + password: <48-char service password> +``` + +!!! note "Provider loading differs between CLI and REST server" + The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API / + remote-client path uses a separate resolver that does **not** load `windscribevpn` or + `gluetun`. ExpressVPN and ProtonVPN also auto-load when their cookie file exists, and Hola + auto-loads whenever the `hola-proxy` binary is present. + +--- + +## Key vaults { #key-vaults } + +The full guide is at [Vaults](../guide/vaults.md). Two keys configure them. + +### `key_vaults` + +- **Type:** `list[dict]`  ·  **Default:** `[]` + +An ordered list of key-vault backends. unshackle queries them in order and reuses content +keys instead of re-licensing. Each entry needs a `type` (the backend module name) and a +`name`, plus backend-specific keys. + +| `type` | Purpose | Required keys | Notes | +|--------|---------|--------------|-------| +| `SQLite` | Local SQLite database | `name`, `path` | Loaded **critically**; a failure aborts the run. | +| `MySQL` | Remote MySQL database | `name`, `host`, `database`, `username` | Extra keys (e.g. `password`, `port`) forwarded to pymysql. | +| `API` | RESTful JSON API | `name`, `uri`, `token` | Honours `vault_timeout`. | +| `HTTP` | HTTP API with modes | `name`, `host`, one of `password`/`api_key` | `api_mode`: `query` (default), `json`, `decrypt_labs`. | + +```yaml +key_vaults: + - type: SQLite + name: Local + path: ~/.unshackle/keys.db + - type: MySQL + name: Team + host: db.example.com + database: keys + username: unshackle + password: hunter2 + no_push: false +``` + +!!! tip "Common per-entry options" + - `no_push: true` makes a vault read-only (keys are fetched but never written to it). + - A vault of `type: api` whose `name` contains `decrypt_labs` auto-fills its `token` from + [`decrypt_labs_api_key`](#external-api-keys) when not set inline. + - An all-zero content key (32 zeros) is treated as "no key" everywhere and is never + stored. + +### `vault_timeout` + +- **Type:** `float`  ·  **Default:** `10.0` + +Timeout in seconds for vault operations. Injected automatically into any backend whose +constructor accepts a `timeout` parameter (a per-vault `timeout` still wins). + +--- + +## External API keys { #external-api-keys } + +All default to an empty string and enable optional metadata/geolocation features. + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `tmdb_api_key` | str | `""` | TMDB API key for metadata enrichment and external-ID tags. | +| `simkl_client_id` | str | `""` | SIMKL client ID for metadata lookups; an alternative/fallback source to TMDB. | +| `decrypt_labs_api_key` | str | `""` | Global Decrypt Labs API key (used by remote CDM / vault). | +| `ipinfo_api_key` | str | `""` | ipinfo.io API key for IP/region lookups. | + +!!! tip "SIMKL is a fallback for TMDB" + SIMKL is positioned as an alternative metadata source, and it is most valuable precisely + **when no `tmdb_api_key` is configured**. It improves title-matching and tagging + reliability in that case rather than replacing TMDB where TMDB is available. + +!!! note "`ipinfo_api_key` never touches your service sessions" + The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header; it + is **never** attached to your service session, so it cannot leak to a streaming provider. + Lookups degrade gracefully through a fallback chain: the authenticated Lite endpoint (higher + rate limits and richer fields) → anonymous ipinfo → `ip-api.in` as a last resort. + +--- + +## Caching & updates { #caching-updates } + +### Title cache + +unshackle caches fetched title metadata (region- and account-aware) to avoid repeat API +calls. + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `title_cache_enabled` | bool | `true` | Master switch for the title cache. | +| `title_cache_time` | int (seconds) | `1800` (30 min) | Lifetime of fresh cached titles. | +| `title_cache_max_retention` | int (seconds) | `86400` (24 h) | How long stale cached data may still be served as a fallback when a live fetch fails. | + +### Update checks + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `update_checks` | bool | `true` | Whether to check for new unshackle releases. | +| `update_check_interval` | int (hours) | `24` | Minimum hours between update checks. | + +!!! note + Update checks query the GitHub releases API with a fixed 5-second timeout and cache the + result in `directories.cache/update_check.json`. + +--- + +## Logging, privacy & debug { #logging-privacy-debug } + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `redact_paths` | bool | `true` | Mask install-root / venv / home prefixes in logged paths (`<unshackle>`, `<venv>`, `~`). Set `false` to show full paths. | +| `debug` | bool | `false` | Global debug mode. | +| `debug_keys` | bool | `false` | Log decryption keys. | +| `debug_requests` | bool | `false` | Log HTTP requests. | +| `set_terminal_bg` | bool | `false` | Append the theme's background colour to output styling. | + +!!! tip "When to enable `set_terminal_bg`" + When on, the theme's background colour is appended to the foreground styles so the full + colour palette renders correctly on terminals whose *default* background differs from + unshackle's theme. Turn it on if the ASCII-art banner or coloured output **looks off** on + your terminal. Otherwise leave it off. + +!!! danger "Key exposure" + Decryption content keys are written to the structured debug log at INFO level during the + DRM handshake. Treat `debug`, `debug_keys`, and the `unshackle_debug_*.jsonl` files as + sensitive, and keep `redact_paths` enabled when sharing logs. + + `debug_keys` affects **only** content-encryption keys (the `content_key`/`key` fields). + Passwords, tokens, cookies, and session tokens are **always** redacted regardless of this + setting, and KIDs, key counts, and other metadata are always logged either way. + +--- + +## Deprecated & removed keys { #deprecated-removed-keys } + +| Key | Status | Behaviour | +|-----|--------|-----------| +| `curl_impersonate` | **Deprecated** → use [`network`](#network-proxy) | Emits a `DeprecationWarning`; still honoured only if `network` is absent. | +| `downloader` | **Deprecated** | Any value other than `"requests"` emits a `DeprecationWarning`; the value is otherwise ignored (the unified requests downloader is always used). | +| `scene_naming` | **Removed** | If present at all, unshackle exits with an error directing you to configure [`output_template`](#output-naming) instead. | + +--- + +## A complete annotated example + +The following config sets a handful of common keys. Every key not shown keeps its default. + +```yaml title="unshackle.yaml" +directories: + downloads: ~/Media/unshackle + temp: /mnt/scratch/unshackle + +network: + browser: Chrome131 + +headers: + Accept-Language: en-US,en;q=0.9 + +cdm: + default: chromecdm_l3 + NF: android_l1 + +decryption: shaka + +credentials: + NF: user@example.com:hunter2 + +output_template: + movies: "{title}.{year}.{quality?}.{source}-{tag}" + series: "{title}.{season_episode}.{quality?}.{source}-{tag}" + folder: + movies: "{title} ({year})" + series: "{title} ({year})" + +tag: MYGRP + +key_vaults: + - type: SQLite + name: Local + path: ~/.unshackle/keys.db + +proxy_providers: + basic: + us: http://user:pass@1.2.3.4:8080 + +tmdb_api_key: "your-tmdb-key" +title_cache_time: 3600 +redact_paths: true +``` + +For how these interact at runtime, continue to [Downloading](../guide/downloading.md), +[DRM & CDM](../guide/drm-and-cdm.md), and [Output & naming](../guide/output-and-naming.md). diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 00000000..acf10184 --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,4 @@ +mkdocs-material>=9.5 +mkdocstrings[python]>=0.26 +# ruff (or black) lets mkdocstrings format rendered signatures +ruff>=0.3 diff --git a/mkdocs.yml b/mkdocs.yml index 6f269ee8..c7977bef 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,8 @@ site_name: unshackle -site_description: Configuration reference for unshackle +site_description: Movie, TV, and Music archival software — user guide and REST API reference site_url: https://docs.unshackle.dev/ repo_url: https://github.com/unshackle-dl/unshackle +repo_name: unshackle-dl/unshackle docs_dir: docs theme: @@ -12,12 +13,13 @@ theme: logo: custom/no_encryption palette: - media: "(prefers-color-scheme)" + primary: black toggle: icon: material/link name: Switch to light mode - media: "(prefers-color-scheme: light)" scheme: default - primary: indigo + primary: black accent: indigo toggle: icon: material/toggle-switch @@ -35,6 +37,7 @@ theme: features: - content.code.annotate - content.code.copy + - content.tabs.link - content.tooltips - navigation.footer - navigation.indexes @@ -61,6 +64,9 @@ markdown_extensions: smart_enable: all - pymdownx.caret - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: anchor_linenums: true line_spans: __span @@ -69,7 +75,11 @@ markdown_extensions: - pymdownx.keys - pymdownx.mark - pymdownx.smartsymbols - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true - pymdownx.tasklist: @@ -78,28 +88,50 @@ markdown_extensions: plugins: - search + - mkdocstrings: + handlers: + python: + paths: [.] + options: + docstring_style: google + show_root_heading: true + show_symbol_type_heading: true + show_symbol_type_toc: true + separate_signature: true + signature_crossrefs: true + merge_init_into_class: true nav: - - Home: README.md - - Creating a Service: CREATING_A_SERVICE.md + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - Quick Start: getting-started/quickstart.md + - The Configuration File: getting-started/configuration-file.md + - User Guide: + - Downloading: guide/downloading.md + - CLI Reference: guide/cli-reference.md + - DRM & CDM Setup: guide/drm-and-cdm.md + - Proxies & VPN: guide/proxies-and-vpn.md + - Key Vaults: guide/vaults.md + - Output & Naming: guide/output-and-naming.md + - Subtitles: guide/subtitles.md + - Troubleshooting: guide/troubleshooting.md - Configuration: - - Service Integration & Auth: SERVICE_CONFIG.md - - Download & Processing: DOWNLOAD_CONFIG.md - - Output & Naming: OUTPUT_CONFIG.md - - Subtitles: SUBTITLE_CONFIG.md - - DRM & CDM: DRM_CONFIG.md - - Network & Proxy: NETWORK_CONFIG.md - - Gluetun VPN: GLUETUN.md - - Advanced & System: ADVANCED_CONFIG.md - - REST API: - - Overview: API.md - - Quick Start: api/quick-start.md - - Authentication: api/authentication.md - - Endpoints: api/endpoints.md - - Endpoint Map: api/endpoint-map.md - - Download Job Lifecycle: api/download-job-lifecycle.md - - Remote Service Sessions: api/remote-service-sessions.md - - Client Configuration: api/client-configuration.md - - Error Responses: api/error-responses.md - - Reference: - - Debug Logging: DEBUG_LOGGING.md + - Reference: reference/configuration.md + - Developer: + - Creating a Service: dev/creating-a-service.md + - Service Architecture: dev/service-architecture.md + - REST API: + - Overview: dev/rest-api/index.md + - Quick Start: dev/rest-api/quickstart.md + - Authentication: dev/rest-api/authentication.md + - Endpoints: dev/rest-api/endpoints.md + - Job Lifecycle: dev/rest-api/job-lifecycle.md + - Remote Sessions: dev/rest-api/remote-sessions.md + - Error Responses: dev/rest-api/errors.md + - API Reference: + - Service: reference/api/service.md + - Config: reference/api/config.md + - Tracks: reference/api/tracks.md + - Titles: reference/api/titles.md + - DRM: reference/api/drm.md