diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..123c1279 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: ci + +on: + push: + branches: [dev, main] + paths-ignore: + - "docs/**" + - "overrides/**" + - "mkdocs.yml" + - "**.md" + pull_request: + paths-ignore: + - "docs/**" + - "overrides/**" + - "mkdocs.yml" + - "**.md" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8.3.2 + with: + enable-cache: true + - run: uv sync --locked + - name: Ruff lint + run: uv run ruff check . + - name: Ruff format + run: uv run ruff format --check . + - name: Import order + run: uv run isort --check-only unshackle tests --skip-glob "*/services/*" + + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.11", "3.14"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8.3.2 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + - name: Install + run: uv sync --locked --group test + - name: Import check + run: uv run python -c "import unshackle; print(unshackle.__name__, 'imports OK')" + - name: CLI loads + run: uv run unshackle --help + - name: Tests + run: uv run pytest -m "not live and not slow" + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8.3.2 + with: + enable-cache: true + cache-suffix: build + - name: Build sdist and wheel + run: uv build + - name: Smoke-test wheel + run: uv run --isolated --no-project --with dist/*.whl unshackle --help + + dependency-review: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/dependency-review-action@v5.0.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e56a8db1..b2e603a0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,19 +8,41 @@ on: - "overrides/**" - "mkdocs.yml" - ".github/workflows/docs.yml" + pull_request: + paths: + - "docs/**" + - "overrides/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" workflow_dispatch: +concurrency: + group: docs-deploy + cancel-in-progress: false + permissions: contents: write jobs: - deploy: + check: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.x" + - run: pip install -e . -r docs/requirements-docs.txt + - run: mkdocs build --strict + + deploy: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.x" - run: pip install -e . -r docs/requirements-docs.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51370595..f941862c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,8 +10,9 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.12 hooks: - - id: ruff + - id: ruff-check args: [--fix] + - id: ruff-format - repo: https://github.com/pycqa/isort rev: 9.0.0a3 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index cc236d31..50224540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,55 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm This changelog is automatically generated using [git-cliff](https://git-cliff.org). -## [5.3.0] - 2026-06-30 +## [Unreleased] + +### Features + +- *api*: Report per-episode and per-track progress on download jobs +- *api*: Add job retry/priority/clear, history, config and maintenance endpoints +- *api*: Expose choice values and multiple flag in service cli_params +- *core*: Add debug logging for rnet requests +- *tracks*: Add subtitle.type_priority config for variant ordering +- *proxies*: [**breaking**] Rewrite ExpressVPN provider to Android device login +- *dl*: Animate merge/decrypt/mux stages with gradient pulse bars +- *tags*: Add imdb_api_enabled toggle, off by default +- *tags*: Add OMDb metadata provider with omdb_api_key config + +### Bug Fixes + +- *api*: Use PBKDF2 instead of SHA256 for cache-namespace hashing +- *api*: Pass track-selection params to service context +- *remote*: Add missing get_clearkey_license to RemoteService +- *remote*: Harden serve API access control and input handling +- *core*: Decode subprocess output as UTF-8 with replacement +- *tracks*: Treat ccextractor signal-crash as non-fatal +- *downloader*: Release hedge loser file handles before merge sweep +- *dl*: Restore real progress bars where progression is trackable +- *dl*: Select only best descriptive audio track per language +- *dl*: [**breaking**] Make audio/video track selection consistent and crash-safe +- *tags*: Point IMDbApi provider at api.tiffara.com (IMDxAPI) +- *gitignore*: Add memory directory to .gitignore +- *dl*: Prevent missing tracks error when excluding track types +- *subs*: Stop segmented WebVTT merge from stripping formatting + +### Documentation + +- Restructure documentation into MkDocs Material site +- Expand config reference and genericize service names + +### Performance Improvements + +- *downloader*: Cut per-segment overhead, bound stalled connections + +### Changes + +- *core*: [**breaking**] Bundle download_track args into DownloadContext + +### Builds + +- *deps*: Cap pycaption below 2.2.27 + +## [5.3.0] - 2026-07-01 ### Features @@ -770,6 +818,8 @@ This changelog is automatically generated using [git-cliff](https://git-cliff.or - Reorganize Planned Features section in README for clarity - Improve track selection logic in dl.py +[unreleased]: https://github.com/unshackle-dl/unshackle/compare/5.3.0..HEAD +[5.3.0]: https://github.com/unshackle-dl/unshackle/compare/5.2.0..5.3.0 [5.2.0]: https://github.com/unshackle-dl/unshackle/compare/5.1.0..5.2.0 [5.1.0]: https://github.com/unshackle-dl/unshackle/compare/5.0.0..5.1.0 [5.0.0]: https://github.com/unshackle-dl/unshackle/compare/4.0.0..5.0.0 diff --git a/docs/dev/creating-a-service.md b/docs/dev/creating-a-service.md index 23472d24..242d06da 100644 --- a/docs/dev/creating-a-service.md +++ b/docs/dev/creating-a-service.md @@ -32,14 +32,14 @@ is a service**, and the folder name is the service **tag**: ```text services/ └── EXAMPLE/ - ├── __init__.py # required — defines the service class - └── config.yaml # optional — per-service configuration + ├── __init__.py # required - defines the service class + └── config.yaml # optional - per-service configuration ``` Two rules the loader enforces: -1. **The class name must exactly match the folder/tag name.** Folder `AMZN` must - define `class AMZN`. A mismatch raises a `RuntimeError`. +1. **The class name must exactly match the folder/tag name.** Folder `EXAMPLE` must + define `class EXAMPLE`. A mismatch raises a `RuntimeError`. 2. **List order is priority.** The first services path to define a given tag wins; later duplicates are shadowed. Put your own local overrides *last* to use them as fallbacks, or *first* to override a repo. @@ -67,6 +67,20 @@ unshackle dl EX 20914 # via an ALIAS placeholders while the secrets that fill them stay out of it. Both halves read back through the same `self.config[...]`. +!!! warning "Split across files? Use relative imports" + If your service grows beyond a single `__init__.py`, import your own + modules **relatively**: + + ```python + from .helpers import parse_manifest # correct + from unshackle.services.TAG.helpers import parse_manifest # breaks from a repo + ``` + + A service loaded from a remote repo lives outside the `unshackle` package, + so the absolute `unshackle.services..*` path does not resolve and the + import fails with `no known parent package`. Relative imports work both + locally and from a cloned repo. + --- ## The Service base class @@ -546,7 +560,7 @@ from unshackle.core.tracks import Chapters, Tracks class MYSVC(Service): """ - MyService — https://myservice.com + MyService - https://myservice.com \b Usage: unshackle dl MYSVC @@ -629,6 +643,7 @@ unshackle dl MYSVC https://myservice.com/watch/abc123 - [ ] Tracks come from a manifest parser with `to_tracks(language=title.language)`; no filtering by quality/language. - [ ] Encrypted tracks are marked with DRM; the license callbacks you need return responses **unmodified**. - [ ] URLs, user agents, and certificates live in `config.yaml`, read via `self.config[...]`. +- [ ] Multi-file services import their own modules relatively (`from .helpers import x`), so they work when loaded from a repo. Read `unshackle/services/EXAMPLE/` end to end; it annotates every feature touched above in one place. diff --git a/docs/dev/rest-api/authentication.md b/docs/dev/rest-api/authentication.md index 83a678dd..6cf5b644 100644 --- a/docs/dev/rest-api/authentication.md +++ b/docs/dev/rest-api/authentication.md @@ -59,7 +59,7 @@ serve: username: alice devices: ["my_widevine_device"] # Widevine (.wvd) devices this key may use playready_devices: ["my_playready_prd"] # PlayReady (.prd) devices this key may use - services: ["AMZN", "NF"] # optional per-key service allowlist + services: ["EXAMPLE1", "EXAMPLE2"] # optional per-key service allowlist a-secret-key-for-bob: username: bob devices: ["my_widevine_device"] @@ -217,11 +217,11 @@ The effective allowlist for a request is: ```yaml title="Restricting services" serve: api_secret: "admin-key" - services: ["AMZN", "NF", "DSNP"] # global ceiling for all keys + services: ["EXAMPLE1", "EXAMPLE2", "EXAMPLE3"] # global ceiling for all keys users: limited-key: username: limited - services: ["AMZN"] # this key ends up allowed only AMZN + services: ["EXAMPLE1"] # this key ends up allowed only EXAMPLE1 ``` Endpoints that name a service (listing services, searching, listing titles/tracks, downloading, profiles, history, and remote sessions) all filter against this allowlist. A request for a service outside the key's allowlist is rejected as an invalid service. Service tags are matched after normalization, so casing and aliases resolve consistently. diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index f752ef5b..0951e5b7 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -130,8 +130,8 @@ Search a service for titles by query string. The service must implement `search( | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `service` | string | yes | — | Service tag. | -| `query` | string | yes | — | Search query. | +| `service` | string | yes | - | Service tag. | +| `query` | string | yes | - | Search query. | | `profile` | string | no | `null` | Credential/cookie profile to use. | | `proxy` | string | no | `null` | Proxy URI or country code. | | `no_proxy` | boolean | no | `false` | Force-disable all proxy use. | @@ -164,7 +164,7 @@ Search a service for titles by query string. The service must implement `search( | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Search completed. | +| `200` | - | Search completed. | | `400` | `INVALID_PARAMETERS` | Missing `query`. | | `400` | `INVALID_SERVICE` | Unknown or disallowed service. | | `400` | `INVALID_PROXY` | Proxy could not be resolved. | @@ -222,7 +222,7 @@ Each serialized title carries `type` (`"episode"`, `"movie"`, or `"other"`), `na | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Titles returned. | +| `200` | - | Titles returned. | | `400` | `INVALID_INPUT` | Missing `service` or `title_id`. | | `400` | `INVALID_SERVICE` | Unknown or disallowed service. | | `401` | `AUTH_FAILED` | Authentication with the service failed. | @@ -242,7 +242,7 @@ List the video, audio, and subtitle tracks for a title. For series, you can scop | `wanted` | string | no | Episode/season range (e.g. `"S01E01-S01E03"`). | | `season` | int/string | no | Season number (combined with `episode`). | | `episode` | int/string | no | Episode number (combined with `season`). | -| `profile`, `proxy`, `no_proxy`, `cdm_type` | — | no | As on `list-titles`. | +| `profile`, `proxy`, `no_proxy`, `cdm_type` | - | no | As on `list-titles`. | When both `season` and `episode` are given, they are combined into a `"{season}x{episode}"` selector. @@ -297,7 +297,7 @@ The `drm` array on a track is a list of `{type, pssh?, kids?, content_keys?, lic | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Tracks returned. | +| `200` | - | Tracks returned. | | `400` | `INVALID_INPUT` | Missing `service` or `title_id`. | | `400` | `INVALID_PARAMETERS` | Invalid `wanted` range. | | `400` | `INVALID_SERVICE` | Unknown or disallowed service. | @@ -378,7 +378,7 @@ Create a download job. Requires `service` and `title_id`; every other field is a | `best_available` | boolean | `false` | Fall back to best available. | | `repack` | boolean | `false` | Add REPACK tag. | | `tag` | string | `null` | Release group tag. | -| `tmdb_id` / `imdb_id` / `animeapi_id` | — | `null` | External ID overrides for tagging. | +| `tmdb_id` / `imdb_id` / `animeapi_id` | - | `null` | External ID overrides for tagging. | | `enrich` | boolean | `false` | Override title/year from external source. | | `output_dir` | string | `null` | Override output directory. | | `no_cache` / `reset_cache` | boolean | `false` | Title cache controls. | @@ -395,7 +395,7 @@ Create a download job. Requires `service` and `title_id`; every other field is a | Status | Error code | Meaning | | --- | --- | --- | -| `202` | — | Job queued. | +| `202` | - | Job queued. | | `400` | `INVALID_INPUT` | Missing `service`/`title_id` or malformed JSON. | | `400` | `INVALID_PARAMETERS` | A parameter failed validation. | | `400` | `INVALID_SERVICE` | Unknown or disallowed service. | @@ -409,8 +409,8 @@ List download jobs, with optional filtering and sorting. | Param | Values | Default | Description | | --- | --- | --- | --- | -| `status` | `queued`, `downloading`, `completed`, `failed`, `cancelled` | — | Filter by status. | -| `service` | service tag | — | Filter by service. | +| `status` | `queued`, `downloading`, `completed`, `failed`, `cancelled` | - | Filter by status. | +| `service` | service tag | - | Filter by service. | | `sort_by` | `created_time`, `started_time`, `completed_time`, `progress`, `status`, `service` | `created_time` | Sort field. | | `sort_order` | `asc`, `desc` | `desc` | Sort direction. | | `full` | `true`, `false` | `false` | Include full per-job details. | @@ -454,7 +454,7 @@ With `full=true`, each job additionally includes `parameters` (with secrets reda | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Jobs returned. | +| `200` | - | Jobs returned. | | `400` | `INVALID_PARAMETERS` | Invalid `sort_by` or `sort_order`. | ### `GET /api/download/jobs/{job_id}` @@ -463,7 +463,7 @@ Fetch a single job with full details (equivalent to a `full=true` list entry). | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Job returned. | +| `200` | - | Job returned. | | `404` | `JOB_NOT_FOUND` | No such job. | ### `DELETE /api/download/jobs/{job_id}` @@ -478,8 +478,8 @@ Cancel or remove a job. The behaviour depends on the job's current state: | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Job cancelled. | -| `204` | — | Terminal job removed (empty body). | +| `200` | - | Job cancelled. | +| `204` | - | Terminal job removed (empty body). | | `400` | `INVALID_PARAMETERS` | Job is in a state that cannot be cancelled. | | `404` | `JOB_NOT_FOUND` | No such job. | @@ -512,7 +512,7 @@ Enqueue a **new** job that reuses a terminal job's service, title, and parameter | Status | Error code | Meaning | | --- | --- | --- | -| `202` | — | New job queued. | +| `202` | - | New job queued. | | `404` | `JOB_NOT_FOUND` | No such job. | | `409` | `CONFLICT` | Job is not in a terminal state. | | `400` | `INVALID_SERVICE` | Service is no longer allowed. | @@ -530,7 +530,7 @@ Move a queued job to the front of the download queue. | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Job moved to front. | +| `200` | - | Job moved to front. | | `404` | `JOB_NOT_FOUND` | No such job. | | `409` | `CONFLICT` | Job is not queued. | @@ -593,7 +593,7 @@ Read the persisted history of jobs that reached a terminal state, newest first. | Param | Type | Default | Description | | --- | --- | --- | --- | | `limit` | int (≥ 1) | `100` | Maximum entries to return. | -| `service` | string | — | Case-insensitive service filter. | +| `service` | string | - | Case-insensitive service filter. | === "Response `200`" @@ -619,7 +619,7 @@ Read the persisted history of jobs that reached a terminal state, newest first. | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | History returned. | +| `200` | - | History returned. | | `400` | `INVALID_PARAMETERS` | Invalid `limit`. | ### `DELETE /api/history/{job_id}` @@ -628,7 +628,7 @@ Remove a single persisted history entry. Returns **`204 No Content`** (empty bod | Status | Error code | Meaning | | --- | --- | --- | -| `204` | — | Entry removed (empty body). | +| `204` | - | Entry removed (empty body). | | `404` | `NOT_FOUND` | No such entry (or outside your allowlist). | ### `POST /api/maintenance/clear-cache` and `POST /api/maintenance/clear-temp` @@ -648,7 +648,7 @@ The `409` guard is about active file I/O, not an arbitrary lock: an in-flight jo | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Directory cleared. | +| `200` | - | Directory cleared. | | `409` | `CONFLICT` | A download is currently active. | ### `POST /api/maintenance/refresh-services` @@ -718,7 +718,7 @@ Create a session for a service and title. Authentication runs asynchronously in | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Session created; authentication in progress. | +| `200` | - | Session created; authentication in progress. | | `400` | `INVALID_INPUT` | Missing `service` or `title_id`. | | `400` | `INVALID_SERVICE` | Unknown or disallowed service. | @@ -752,7 +752,7 @@ Poll for the authentication status and any pending interactive prompt (OTP, PIN, | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Status returned. | +| `200` | - | Status returned. | | `403` | `FORBIDDEN` | Request IP differs from the session creator. | | `404` | `SESSION_NOT_FOUND` | No such session. | @@ -768,7 +768,7 @@ Submit a response to a pending prompt. Body: `{ "response": "..." }` (required). | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Response accepted. | +| `200` | - | Response accepted. | | `400` | `INVALID_INPUT` | Missing `response`, or no prompt is pending. | | `404` | `SESSION_NOT_FOUND` | No such session. | @@ -810,7 +810,7 @@ Get tracks and chapters for a specific title within the session. Body: `{ "title | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Tracks returned. | +| `200` | - | Tracks returned. | | `400` | `INVALID_INPUT` | Missing or unknown `title_id`. | | `404` | `SESSION_NOT_FOUND` | No such session. | @@ -830,7 +830,7 @@ Resolve the download URL, DRM info, headers, cookies, and raw track data for sel | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | Segment info returned. | +| `200` | - | Segment info returned. | | `400` | `INVALID_INPUT` | Missing `track_ids`. | | `404` | `TRACK_NOT_FOUND` | An unknown track ID was requested. | @@ -876,7 +876,7 @@ Acquire DRM keys. Two modes are supported via the `mode` field. | Status | Error code | Meaning | | --- | --- | --- | -| `200` | — | License/keys returned. | +| `200` | - | License/keys returned. | | `400` | `INVALID_INPUT` | Missing challenge/PSSH, or no CDM device configured for the key. | | `400` | `INVALID_PARAMETERS` | Unsupported `drm_type`. | | `404` | `TRACK_NOT_FOUND` | Unknown track ID. | diff --git a/docs/dev/rest-api/errors.md b/docs/dev/rest-api/errors.md index 32b8ceee..8f708793 100644 --- a/docs/dev/rest-api/errors.md +++ b/docs/dev/rest-api/errors.md @@ -139,7 +139,7 @@ uncategorised exception bubbles up, the server runs it through a classifier that inspects the exception type and message and maps it to the most appropriate code. The classification is keyword-based, checked in order: -| If the error mentions… | It becomes | Retryable | +| If the error mentions... | It becomes | Retryable | |---|---|---| | auth, login, credential, unauthorized, forbidden, token | `AUTH_FAILED` (401) | no | | connection, timeout, network, unreachable, socket, dns, resolve (or `ConnectionError`, `TimeoutError`, `URLError`, `SSLError`) | `NETWORK_ERROR` (503) | yes | diff --git a/docs/dev/rest-api/index.md b/docs/dev/rest-api/index.md index 0f5dee19..d6f9c40c 100644 --- a/docs/dev/rest-api/index.md +++ b/docs/dev/rest-api/index.md @@ -211,7 +211,7 @@ every request except `/api/health`: curl -X POST http://127.0.0.1:8786/api/search \ -H "X-Secret-Key: your-api-secret" \ -H "Content-Type: application/json" \ - -d '{"service": "NF", "query": "example"}' + -d '{"service": "EXAMPLE", "query": "example"}' ``` A missing or unknown key returns `401`. The full model, including per-key service diff --git a/docs/dev/rest-api/job-lifecycle.md b/docs/dev/rest-api/job-lifecycle.md index f300996c..56b9c761 100644 --- a/docs/dev/rest-api/job-lifecycle.md +++ b/docs/dev/rest-api/job-lifecycle.md @@ -86,7 +86,7 @@ curl -X POST http://127.0.0.1:8786/api/download \ -H "X-Secret-Key: your-api-secret" \ -H "Content-Type: application/json" \ -d '{ - "service": "NF", + "service": "EXAMPLE", "title_id": "81234567", "quality": ["1080"], "range": ["HDR10"] @@ -228,7 +228,7 @@ state, and this is important because the response differs: |---|---|---| | `queued` or `downloading` | The job is cancelled. A running download's worker process is terminated. | `200` with `{"status": "success", "message": "Job cancelled"}` | | `completed`, `failed`, or `cancelled` (terminal) | The job is **removed** from the manager entirely. | `204 No Content`, empty body | -| Not found | — | `404` with error code `JOB_NOT_FOUND` | +| Not found | - | `404` with error code `JOB_NOT_FOUND` | In other words, `DELETE` on a live job *cancels* it, while `DELETE` on an already finished job *deletes it from the list*. This dual meaning is deliberate: one verb diff --git a/docs/dev/rest-api/quickstart.md b/docs/dev/rest-api/quickstart.md index f0f02323..da02a045 100644 --- a/docs/dev/rest-api/quickstart.md +++ b/docs/dev/rest-api/quickstart.md @@ -161,7 +161,7 @@ If the key is missing or wrong, you get a `401` with a plain body: ``` !!! warning "Two error shapes exist" - The authentication middleware returns the short `{"status": 401, "message": "..."}` shape shown above. Every **other** error the API produces uses the richer structured shape (`error_code`, `timestamp`, `details`, …) described in [Error handling](#error-handling). A robust client should handle both. + The authentication middleware returns the short `{"status": 401, "message": "..."}` shape shown above. Every **other** error the API produces uses the richer structured shape (`error_code`, `timestamp`, `details`, ...) described in [Error handling](#error-handling). A robust client should handle both. The `tag` value for the service you want is the identifier you pass as `service` in the calls below. @@ -225,8 +225,8 @@ The request body accepts the full download parameter set. A few of the most usef | Field | Default | Meaning | | --- | --- | --- | -| `service` | — | Service tag (required) | -| `title_id` | — | Title identifier (required) | +| `service` | - | Service tag (required) | +| `title_id` | - | Title identifier (required) | | `profile` | `null` | Named credential/cookie profile to use | | `quality` | `[]` | Resolution(s), e.g. `[1080]`; empty means best available | | `vcodec` | `null` | Video codec, e.g. `"H265"` | diff --git a/docs/dev/rest-api/remote-sessions.md b/docs/dev/rest-api/remote-sessions.md index a65905bf..56a89ecf 100644 --- a/docs/dev/rest-api/remote-sessions.md +++ b/docs/dev/rest-api/remote-sessions.md @@ -65,7 +65,7 @@ remote_services: server_cdm: false # Optional: per-service config overrides applied locally (title_map, cdm, etc.) services: - AMZN: + EXAMPLE: title_map: "0ABCDEF": "The Show (Renamed Locally)" ``` @@ -87,13 +87,13 @@ remote_services: If exactly one server is configured, unshackle uses it implicitly: ```bash -unshackle dl AMZN 0ABCDEF +unshackle dl EXAMPLE 0ABCDEF ``` If **more than one** server is configured, you must pick one with `--server`: ```bash -unshackle dl --server my_server AMZN 0ABCDEF +unshackle dl --server my_server EXAMPLE 0ABCDEF ``` With no `remote_services` configured at all, unshackle raises a clear error telling @@ -280,10 +280,10 @@ All routes are exposed even in `--remote-only` server mode. Paths use the The response returns **before** authentication finishes: ```json -{ "session_id": "…uuid4…", "service": "AMZN", "status": "authenticating" } +{ "session_id": "...uuid4...", "service": "EXAMPLE", "status": "authenticating" } ``` -Authentication runs on a background thread (`asyncio.to_thread(authenticate, …)`). +Authentication runs on a background thread (`asyncio.to_thread(authenticate, ...)`). The session starts in `AUTHENTICATING`; the client must poll the prompt endpoint until it reaches `authenticated` (or `failed`). @@ -306,11 +306,11 @@ layer. { "status": "authenticated" } { "status": "authenticating" } { "status": "pending_input", "prompt": "Enter code: " } -{ "status": "failed", "error": "…message…" } +{ "status": "failed", "error": "...message..." } ``` A missing session returns `404 SESSION_NOT_FOUND`; an IP mismatch returns -`403 FORBIDDEN`. `POST /api/session/{id}/prompt` takes `{ "response": "…" }` and +`403 FORBIDDEN`. `POST /api/session/{id}/prompt` takes `{ "response": "..." }` and returns `{ "status": "accepted" }`; posting with no pending prompt is an `INVALID_INPUT` error. @@ -364,7 +364,7 @@ and base64-encoding, **excluding** `titles_*` files) and returns them under a `cache` key so the client can persist refreshed tokens: ```json -{ "status": "ok", "cache": { "tokens": "…base64(zlib(bytes))…" } } +{ "status": "ok", "cache": { "tokens": "...base64(zlib(bytes))..." } } ``` ### Session info response @@ -373,8 +373,8 @@ and base64-encoding, **excluding** `titles_*` files) and returns them under a ```json { - "session_id": "…", - "service": "AMZN", + "session_id": "...", + "service": "EXAMPLE", "valid": true, "expires_in": 300, "track_count": 12, @@ -410,7 +410,7 @@ session API and a good template for any client. ## Troubleshooting -!!! example "\"Could not connect to remote server … Is it running?\"" +!!! example "\"Could not connect to remote server ... Is it running?\"" The client couldn't reach the URL. Confirm `unshackle serve` is running on the server, the `url`/port in `remote_services` are correct (default port `8786`), and any firewall or reverse proxy allows the connection. @@ -426,6 +426,6 @@ session API and a good template for any client. prompts you, check that credentials/cookies forwarded from the client are valid for the service. -!!! example "\"Multiple remote services configured. Use --server …\"" +!!! example "\"Multiple remote services configured. Use --server ...\"" You have more than one entry under `remote_services`. Pass `--server ` to pick one. diff --git a/docs/dev/service-architecture.md b/docs/dev/service-architecture.md index 40f645cf..59598417 100644 --- a/docs/dev/service-architecture.md +++ b/docs/dev/service-architecture.md @@ -25,9 +25,9 @@ your service, in order: ```text Service.get_titles → which movies / episodes / songs exist │ -Service.get_tracks → fetch a manifest for one title … +Service.get_tracks → fetch a manifest for one title ... │ -manifest.to_tracks → … parsed into Video / Audio / Subtitle tracks +manifest.to_tracks → ... parsed into Video / Audio / Subtitle tracks │ Track.download → fetch + merge segments (core-driven) │ @@ -310,7 +310,7 @@ A typical `get_tracks` therefore looks like fetching the manifest text with - **HLS**: set `track.OnSegmentFilter(segment)` to drop unwanted segments (ads/bumpers). - **DASH**: pass a `period_filter` callable to `to_tracks` to drop whole periods. - Amazon caching-breadth periods and non-`content` segment types are dropped automatically. + Caching-breadth periods some services emit and non-`content` segment types are dropped automatically. ### Format gotchas worth knowing diff --git a/docs/getting-started/configuration-file.md b/docs/getting-started/configuration-file.md index 2ab2d7d3..32d99792 100644 --- a/docs/getting-started/configuration-file.md +++ b/docs/getting-started/configuration-file.md @@ -63,11 +63,11 @@ tag: MYGRP # Default Widevine/PlayReady device to use, with per-service overrides cdm: default: my_device_l3 - AMZN: my_device_l1 + EXAMPLE: my_device_l1 # Per-service login details (service tag -> "username:password") credentials: - AMZN: my_email@example.com:hunter2 + EXAMPLE: my_email@example.com:hunter2 # Default HTTP headers merged into every request headers: @@ -94,6 +94,41 @@ For the full list with types and defaults, see the [Configuration Reference](../ !!! warning "Unknown keys are silently ignored" unshackle does not validate your config against a schema. If you misspell a key, it is simply skipped and the default is used. You will not get an error. Double-check key names (and their nesting) if a setting does not seem to apply. +## Setting download defaults (`dl:`) + +If you always pass the same `dl` flags (a language, a resolution, a codec), put them under a +`dl:` key once and unshackle applies them to every download. Any flag from +[Downloading](../guide/downloading.md) works here; the key is the flag's long name with dashes +turned into underscores (`--best-available` → `best_available`). + +```yaml title="unshackle.yaml" +dl: + lang: [en] # -l en + quality: [1080] # -q 1080 + vcodec: [H265] # -v H265 + sub_format: srt # convert subtitles to SRT + downloads: 2 # two tracks at once +``` + +You can still override any of these on the command line for a one-off; an explicit flag +always beats the config default. You can also scope defaults to a single service by nesting a +`dl:` block under it: + +```yaml title="Per-service defaults" +dl: + lang: [en] # default for everything +services: + EXAMPLE: + dl: + lang: [en, ja] # Example downloads English + Japanese +``` + +!!! tip "A few keys are named after the flag's internal name" + Most keys are obvious, but set these exact ones: `range` (`-r`), `list` (`--list`), + `tmdb_id` (`--tmdb`), `imdb_id` (`--imdb`), `no_atmos` (`--noatmos`), and `output_dir` + (`-o`). The [Configuration Reference](../reference/configuration.md#dl) has the full list + and every available key. + ## The directory layout The `directories` key controls where unshackle reads and writes its various files. Each directory has a sensible default, and you can override most of them by giving a new path. Paths support `~` for your home directory. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 861c8766..43881be6 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -79,7 +79,7 @@ full set of valid variables, including `resolution`, `video`, `audio`, `hdr`, ## 3. Add a service -The service tag you pass to `dl` (like `NF`) maps to a **service module**, a small +The service tag you pass to `dl` (like `EXAMPLE`) maps to a **service module**, a small plugin that knows how to talk to one streaming platform. unshackle ships without any bundled services, so you add the one(s) you want. @@ -90,14 +90,14 @@ that exact same name. === "Local folder" - Drop a service folder into your services directory. A service tagged `NF` + Drop a service folder into your services directory. A service tagged `EXAMPLE` lives at: ``` - /NF/__init__.py # defines a class named NF + /EXAMPLE/__init__.py # defines a class named EXAMPLE ``` - Once the folder is in place, the tag `NF` is immediately available to `dl`, + Once the folder is in place, the tag `EXAMPLE` is immediately available to `dl`, `search`, and the other service-aware commands. === "Git repository" @@ -140,7 +140,7 @@ covers everything else: ```yaml title="unshackle.yaml" cdm: default: my_device # the .wvd file's name, without the extension - NF: my_other_device # override for a specific service + EXAMPLE: my_other_device # override for a specific service ``` PlayReady works the same way with `.prd` files created and managed by the @@ -157,14 +157,14 @@ Services that require a login read either **cookies** or **credentials**. - **Cookies**: export the service's cookies to a Netscape-format text file and place it in your cookies directory. unshackle looks for, in order: `cookies/{SERVICE}.txt`, then `cookies/{SERVICE}/{profile}.txt`, then - `cookies/{SERVICE}/default.txt`. So a file at `cookies/NF.txt` is picked up - automatically for the `NF` service. + `cookies/{SERVICE}/default.txt`. So a file at `cookies/EXAMPLE.txt` is picked up + automatically for the `EXAMPLE` service. - **Credentials**: store a username and password per service in your config: ```yaml title="unshackle.yaml" credentials: - NF: "email@example.com:your-password" + EXAMPLE: "email@example.com:your-password" ``` Use the `-p/--profile` flag to switch between multiple accounts for the same @@ -177,14 +177,14 @@ If you have a URL or ID already, skip this. Otherwise, search the service for a title and note the `id` it prints, since that is what you feed to `dl`: ```shell -unshackle search NF "The Office" +unshackle search EXAMPLE "My Show" ``` You can also list what a service exposes for a given title without downloading: ```shell -unshackle dl --list-titles NF 81234567 # show seasons/episodes -unshackle dl --list NF 81234567 # show available tracks +unshackle dl --list-titles EXAMPLE 81234567 # show seasons/episodes +unshackle dl --list EXAMPLE 81234567 # show available tracks ``` ## 7. Run your first download @@ -196,13 +196,13 @@ unshackle dl ``` - **`dl`** carries every quality, language, track, and output flag. -- **`<SERVICE-TAG>`** picks which service to talk to (`NF`, `DSNP`, …). +- **`<SERVICE-TAG>`** picks which service to talk to (e.g. `EXAMPLE`). - **`<TITLE>`** is the URL, ID, or slug the service understands. A good first command asks for 1080p with English audio and subtitles: ```shell title="Your first download" -unshackle dl -q 1080 -l en NF 81234567 +unshackle dl -q 1080 -l en EXAMPLE 81234567 ``` unshackle will fetch the title, select the tracks matching your flags, acquire @@ -224,13 +224,13 @@ keys through your CDM (and any key vaults), then decrypt, mux, and tag the resul !!! example "A few realistic variations" ```shell # A whole first season in the best available quality - unshackle dl -w S01 NF 81234567 + unshackle dl -w S01 EXAMPLE 81234567 # 2160p HDR10 with the original-language audio plus English subtitles - unshackle dl -q 2160 -r HDR10 -l orig -sl en NF 81234567 + unshackle dl -q 2160 -r HDR10 -l orig -sl en EXAMPLE 81234567 # Just the newest episode of an ongoing show - unshackle dl --latest-episode NF 81234567 + unshackle dl --latest-episode EXAMPLE 81234567 ``` See [Downloading](../guide/downloading.md) for the complete flag reference, including @@ -243,7 +243,7 @@ By default, finished files are written to the `downloads` directory folder alongside the project). Override it per run with `-o`: ```shell -unshackle dl -q 1080 -o /mnt/media/incoming NF 81234567 +unshackle dl -q 1080 -o /mnt/media/incoming EXAMPLE 81234567 ``` - **Movies** are written as a single `.mkv` file named from your `movies` template. diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 360e41ee..f2968c24 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -45,16 +45,16 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] ``` - `OPTIONS`: every flag below, parsed at the `dl` level. -- `SERVICE`: a service tag (e.g. `NF`, `DSNP`). Tags are case-insensitive and honour each service's aliases (`disney+`, `DSNP`, etc. all resolve to the same tag). +- `SERVICE`: a service tag (e.g. `EXAMPLE1`, `EXAMPLE2`). Tags are case-insensitive and honour each service's aliases (`example+`, `EXAMPLE2`, etc. all resolve to the same tag). - `SERVICE ARGS`: the title, URL, or ID, plus any service-specific options. These belong to the service, not to `dl`. !!! example "Typical downloads" ```shell - # Best available 1080p SDR of a Netflix title - unshackle dl -q 1080 NF 81234567 + # Best available 1080p SDR of a title + unshackle dl -q 1080 EXAMPLE 81234567 # 4K HDR10, English audio + subs, from a URL - unshackle dl -q 2160 -r HDR10 -l en DSNP "https://www.disneyplus.com/…" + unshackle dl -q 2160 -r HDR10 -l en EXAMPLE "https://www.example.com/..." # A season range, HEVC video, EC3 audio unshackle dl -w S01-S03 -v H.265 -a EC3 SERVICE TITLE-ID @@ -67,16 +67,16 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | Flag | Default | Description | |---|---|---| -| `-p`, `--profile` | — | Profile for credentials and cookies. | +| `-p`, `--profile` | - | Profile for credentials and cookies. | | `-q`, `--quality` | best | Resolution(s), comma-separated, e.g. `1080,720`. `-q 1080` also matches anamorphic tracks by 16:9 canvas. | | `-v`, `--vcodec` | any | Video codec(s). Accepts names or values: `AVC`/`H.264`, `HEVC`/`H.265`, `VC1`, `VP8`, `VP9`, `AV1`. | | `-a`, `--acodec` | any | Audio codec(s), comma-separated. Accepts `AAC`, `AC3`/`DD`, `EC3`/`DD+`/`eac3`/`ddp`, `AC4`, `OPUS`, `OGG`/`vorbis`, `DTS`, `ALAC`, `FLAC`. | | `-vb`, `--vbitrate` | highest | Exact video bitrate in kbps. | | `-ab`, `--abitrate` | highest | Exact audio bitrate in kbps. | -| `-vb-range`, `--vbitrate-range` | — | Video bitrate range in kbps, e.g. `6000-7000`; picks highest within. Mutually exclusive with `--vbitrate`. | -| `-ab-range`, `--abitrate-range` | — | Audio bitrate range in kbps, e.g. `128-256`. Mutually exclusive with `--abitrate`. | +| `-vb-range`, `--vbitrate-range` | - | Video bitrate range in kbps, e.g. `6000-7000`; picks highest within. Mutually exclusive with `--vbitrate`. | +| `-ab-range`, `--abitrate-range` | - | Audio bitrate range in kbps, e.g. `128-256`. Mutually exclusive with `--abitrate`. | | `-r`, `--range` | `SDR` | Colour range(s): `SDR`, `HLG`, `HDR10`, `HDR10P` (HDR10+), `DV`, `HYBRID`. | -| `-c`, `--channels` | — | Audio channels; matches sub-layouts (5.1 ≈ 6.0). | +| `-c`, `--channels` | - | Audio channels; matches sub-layouts (5.1 ≈ 6.0). | | `-naa`, `--noatmos` | off | Exclude Dolby Atmos audio tracks. | | `--worst` | off | Pick the lowest bitrate within the requested quality. **Requires `-q`.** | | `--best-available` | off | Continue with the best available if a requested resolution/language is absent, instead of failing. | @@ -91,13 +91,13 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGS...] | Flag | Default | Description | |---|---|---| | `-l`, `--lang` | `orig` | Language(s) for **both** video and audio. `orig` = the title's original language; e.g. `orig,en`. | -| `-vl`, `--v-lang` | — | Video-only language (overrides `-l` for video). | -| `-al`, `--a-lang` | — | Audio-only language (overrides `-l` for audio). | +| `-vl`, `--v-lang` | - | Video-only language (overrides `-l` for video). | +| `-al`, `--a-lang` | - | Audio-only language (overrides `-l` for audio). | | `-sl`, `--s-lang` | `all` | Subtitle language(s). | -| `--require-subs` | — | Required subtitle langs; keeps **all** subs only if these exist. **Cannot combine with `--s-lang`.** | +| `--require-subs` | - | Required subtitle langs; keeps **all** subs only if these exist. **Cannot combine with `--s-lang`.** | | `-fs`, `--forced-subs` | off | Include forced subtitle tracks. | | `--exact-lang` | off | Exact matching only: `-l es-419` matches `es-419`, not `es-ES`. | -| `--sub-format` | — | Output subtitle format (`SRT`/`srt`, `VTT`/`webvtt`, `ASS`/`ssa`, `TTML`, `SMI`, …), or `original` to keep the source format. | +| `--sub-format` | - | Output subtitle format (`SRT`/`srt`, `VTT`/`webvtt`, `ASS`/`ssa`, `TTML`, `SMI`, ...), or `original` to keep the source format. | The special language tokens `orig`, `all`, and `best` are honoured everywhere a language is expected. @@ -177,7 +177,7 @@ Keep only certain track types, or skip certain track types. Attachments are alwa | `--no-cache` | off | Bypass the title cache. | | `--reset-cache` | off | Clear the title cache. | | `--list` | off | List available/would-be-downloaded tracks; do not download. | -| `--slow` | — | Inter-title delay. Bare `--slow` = 60-120s; `--slow 20-40` = custom range (minimum 20s). | +| `--slow` | - | Inter-title delay. Bare `--slow` = 60-120s; `--slow 20-40` = custom range (minimum 20s). | !!! warning "Some flags cannot be combined" `--require-subs` and `--s-lang`; `--select-titles` and `--wanted`; `--worst` requires `--quality`; `--vbitrate` and `--vbitrate-range` (and the audio equivalents) are mutually exclusive. @@ -202,8 +202,8 @@ The query syntax is defined per service. Results are printed as a tree of titles !!! example ```shell - unshackle search NF "The Office" - unshackle search -p myprofile NF "breaking bad" + unshackle search EXAMPLE "My Show" + unshackle search -p myprofile EXAMPLE "Another Show" unshackle search --proxy nordvpn:ca SERVICE "query" ``` @@ -308,7 +308,7 @@ Clear an environment directory. The directory is emptied and recreated, and the unshackle env check unshackle env info unshackle env clear cache - unshackle env clear cache NF # one service only + unshackle env clear cache EXAMPLE # one service only unshackle env clear temp ``` @@ -358,10 +358,10 @@ Create service tables on vaults that use tables, for every installed service, wh !!! example ```shell unshackle kv copy main backup1 backup2 - unshackle kv copy main backup -s NF + unshackle kv copy main backup -s EXAMPLE unshackle kv sync main mysql_vault - unshackle kv add keys.txt NF main backup - unshackle kv search 0123456789abcdef0123456789abcdef -s NF -v main + unshackle kv add keys.txt EXAMPLE main backup + unshackle kv search 0123456789abcdef0123456789abcdef -s EXAMPLE -v main unshackle kv prepare main mysql_vault ``` @@ -523,7 +523,7 @@ Losslessly set the video range flag to full or limited at the bitstream level. | Option | Default | Description | |---|---|---| -| `--full` / `--limited` | — | Full (0-255) or limited (16-235) range. | +| `--full` / `--limited` | - | Full (0-255) or limited (16-235) range. | | `-p`, `--preview` | off | Preview instead of writing a file. | ### `util test PATH` diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 19ea4484..d51cd124 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -26,23 +26,23 @@ unshackle dl [OPTIONS] SERVICE [SERVICE ARGUMENTS] - `unshackle`: the root command. - `dl`: the download command. **All of the options on this page are parsed here**, so they go *before* the service tag. -- `SERVICE`: a service tag such as `NF`, `DSNP`, `AMZN`. Service tags are - case-insensitive and honour each service's aliases (for example `disney+` resolves to +- `SERVICE`: a service tag such as `EXAMPLE1`, `EXAMPLE2`, `EXAMPLE3`. Service tags are + case-insensitive and honour each service's aliases (for example `example+` resolves to its real tag). - `SERVICE ARGUMENTS`: usually a title ID or URL. This positional argument belongs to the **service**, not to `dl`, so it comes *after* the tag. ```shell title="A minimal download" -unshackle dl NF 81234567 +unshackle dl EXAMPLE 81234567 ``` ```shell title="Flags belong to dl, the title belongs to the service" -unshackle dl -q 1080 -v H.265 -r HDR10 --lang en DSNP 'https://www.disneyplus.com/…' +unshackle dl -q 1080 -v H.265 -r HDR10 --lang en EXAMPLE 'https://www.example.com/...' ``` !!! tip "See what a service accepts" Because the positional argument is defined by each service, run - `unshackle dl SERVICE --help` (e.g. `unshackle dl NF --help`) to see that service's + `unshackle dl SERVICE --help` (e.g. `unshackle dl EXAMPLE --help`) to see that service's own argument and any service-specific options. ## The download flow @@ -75,11 +75,11 @@ downloaded. `--skip-dl` runs the license step but skips the actual segment downl list. Without it, unshackle picks the **best available** resolution. ```shell title="Single resolution" -unshackle dl -q 1080 NF 81234567 +unshackle dl -q 1080 EXAMPLE 81234567 ``` ```shell title="Multiple resolutions in one run" -unshackle dl -q 2160,1080,720 NF 81234567 +unshackle dl -q 2160,1080,720 EXAMPLE 81234567 ``` !!! note "16:9 canvas matching" @@ -93,11 +93,11 @@ By default a missing requested resolution is an error. Two flags change that: | Flag | Behaviour | | --- | --- | -| `--best-available` | If the requested resolution(s) aren't present, continue with the best that *is* available instead of failing. Also softens missing audio/subtitle languages and hybrid fallbacks. | +| `--best-available` | If the requested resolution(s) aren't present, continue with the best that *is* available instead of failing. Also softens missing video/audio/subtitle languages and hybrid fallbacks. | | `--worst` | Within the specified quality, pick the **lowest** bitrate rendition. **Requires `-q/--quality`.** | ```shell title="Never fail on a missing resolution" -unshackle dl -q 2160 --best-available AMZN 0ABC123 +unshackle dl -q 2160 --best-available EXAMPLE 0ABC123 ``` ## Video codec and color range @@ -117,9 +117,9 @@ either enum **names** or their **values**, comma-separated. | `AV1` | `AV1` | ```shell title="Either spelling works" -unshackle dl -v HEVC NF 81234567 -unshackle dl -v H.265 NF 81234567 -unshackle dl -v hevc,avc NF 81234567 +unshackle dl -v HEVC EXAMPLE 81234567 +unshackle dl -v H.265 EXAMPLE 81234567 +unshackle dl -v hevc,avc EXAMPLE 81234567 ``` ### Color range @@ -136,7 +136,7 @@ unshackle dl -v hevc,avc NF 81234567 | `HYBRID` | Fetch both an HDR10/HDR10+ base and a DV track, then merge them | ```shell title="Grab HDR10 and Dolby Vision in one run" -unshackle dl -q 2160 -r HDR10,DV DSNP '…' +unshackle dl -q 2160 -r HDR10,DV EXAMPLE '...' ``` !!! warning "HYBRID requires dovi_tool" @@ -215,7 +215,7 @@ Names, values, and a few aliases are accepted: `eac3` and `ddp` both resolve to and `vorbis` resolves to `OGG`. ```shell title="Prefer Dolby Digital Plus, fall back to AAC" -unshackle dl -a EC3,AAC NF 81234567 +unshackle dl -a EC3,AAC EXAMPLE 81234567 ``` ### Channels and Atmos @@ -225,7 +225,7 @@ unshackle dl -a EC3,AAC NF 81234567 - `-naa` / `--noatmos`: exclude Dolby Atmos audio tracks from selection. ```shell title="5.1 audio, no Atmos" -unshackle dl -c 5.1 --noatmos AMZN 0ABC123 +unshackle dl -c 5.1 --noatmos EXAMPLE 0ABC123 ``` ## Languages @@ -236,7 +236,7 @@ unshackle dl -c 5.1 --noatmos AMZN 0ABC123 `orig`, the title's original language. ```shell title="Original language plus English" -unshackle dl -l orig,en NF 81234567 +unshackle dl -l orig,en EXAMPLE 81234567 ``` The special token `orig` is resolved to the title's actual original language everywhere @@ -247,7 +247,7 @@ it is used. You can override each stream type independently: - `-al` / `--a-lang`: language for **audio only** (overrides `-l` for audio). ```shell title="English audio over the original video" -unshackle dl -al en NF 81234567 +unshackle dl -al en EXAMPLE 81234567 ``` ### Exact vs fuzzy matching @@ -269,7 +269,7 @@ matches only `es-419`, not `es-ES`. every available subtitle language is downloaded. ```shell title="Only English and Spanish subtitles" -unshackle dl -sl en,es NF 81234567 +unshackle dl -sl en,es EXAMPLE 81234567 ``` ### Requiring subtitles @@ -299,7 +299,7 @@ download on the presence of a specific subtitle track. | `original` | Keep the source format, no conversion | ```shell title="Convert subtitles to SRT" -unshackle dl --sub-format srt NF 81234567 +unshackle dl --sub-format srt EXAMPLE 81234567 ``` !!! note "SDH stripping happens by default" @@ -321,24 +321,24 @@ with `-` to exclude it. === "Whole seasons" ```shell - unshackle dl -w S01 NF 81234567 - unshackle dl -w S01-S03 NF 81234567 - unshackle dl -w S01,S03,S05 NF 81234567 + unshackle dl -w S01 EXAMPLE 81234567 + unshackle dl -w S01-S03 EXAMPLE 81234567 + unshackle dl -w S01,S03,S05 EXAMPLE 81234567 ``` === "Specific episodes" ```shell - unshackle dl -w S01E01 NF 81234567 - unshackle dl -w S01E01-S01E05 NF 81234567 - unshackle dl -w S01E01-S02E03 NF 81234567 + unshackle dl -w S01E01 EXAMPLE 81234567 + unshackle dl -w S01E01-S01E05 EXAMPLE 81234567 + unshackle dl -w S01E01-S02E03 EXAMPLE 81234567 ``` === "Exclusions" ```shell # Seasons 1 through 5, but not season 3 - unshackle dl -w S01-S05,-S03 NF 81234567 + unshackle dl -w S01-S05,-S03 EXAMPLE 81234567 ``` ### Other selection flags @@ -350,7 +350,7 @@ with `-` to exclude it. | `--list-titles` | List the titles that would be downloaded, then stop. | ```shell title="Grab just the newest episode" -unshackle dl --latest-episode NF 81234567 +unshackle dl --latest-episode EXAMPLE 81234567 ``` ## Including and excluding track types @@ -380,11 +380,11 @@ Additional track-type flags: than aborting the whole title. Video and audio failures remain fatal. ```shell title="Subtitles only" -unshackle dl -S -sl en NF 81234567 +unshackle dl -S -sl en EXAMPLE 81234567 ``` ```shell title="Everything except chapters" -unshackle dl -nc NF 81234567 +unshackle dl -nc EXAMPLE 81234567 ``` !!! note @@ -403,7 +403,7 @@ Before committing to a long download, inspect what unshackle *would* do: | `--skip-dl` | Skip downloading but still acquire the decryption keys. | ```shell title="See the track selection without downloading" -unshackle dl -q 1080 -v H.265 -r HDR10 --list NF 81234567 +unshackle dl -q 1080 -v H.265 -r HDR10 --list EXAMPLE 81234567 ``` ## Output and muxing @@ -416,7 +416,7 @@ unshackle dl -q 1080 -v H.265 -r HDR10 --list NF 81234567 - `--no-source`: remove the service source tag from the filename and path. ```shell title="Send this download somewhere specific" -unshackle dl -o ~/Videos/incoming NF 81234567 +unshackle dl -o ~/Videos/incoming EXAMPLE 81234567 ``` ### Muxing behaviour @@ -426,7 +426,7 @@ flags change how the output is assembled: | Flag | Behaviour | Default source | | --- | --- | --- | -| `--no-mux` | Do not mux; keep the individual track files. | — | +| `--no-mux` | Do not mux; keep the individual track files. | - | | `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) | | `--merge-video` | Mux all selected video tracks into one file. | config `muxing.merge_video` (off) | @@ -436,7 +436,7 @@ flags change how the output is assembled: - `--repack`: add a `REPACK` tag to the output filename. ```shell title="Custom group tag and a REPACK label" -unshackle dl --tag MYGRP --repack NF 81234567 +unshackle dl --tag MYGRP --repack EXAMPLE 81234567 ``` ## Proxies @@ -445,9 +445,9 @@ unshackle dl --tag MYGRP --repack NF 81234567 configured proxy providers), or a `provider:region` form. ```shell title="Proxy forms" -unshackle dl --proxy us NF 81234567 -unshackle dl --proxy nordvpn:ca NF 81234567 -unshackle dl --proxy 'http://user:pass@host:8080' NF 81234567 +unshackle dl --proxy us EXAMPLE 81234567 +unshackle dl --proxy nordvpn:ca EXAMPLE 81234567 +unshackle dl --proxy 'http://user:pass@host:8080' EXAMPLE 81234567 ``` Two related flags: @@ -470,11 +470,11 @@ Two related flags: | `--reset-cache` | Clear the title cache before fetching. | ```shell title="Two tracks at a time, eight threads each" -unshackle dl --downloads 2 --workers 8 NF 81234567 +unshackle dl --downloads 2 --workers 8 EXAMPLE 81234567 ``` ```shell title="Space out a season download" -unshackle dl -w S01 --slow 30-60 NF 81234567 +unshackle dl -w S01 --slow 30-60 EXAMPLE 81234567 ``` ## Keys, vaults, and export @@ -486,7 +486,7 @@ a key when the vault misses. You can force one side or the other: - `--vaults-only`: only use key vaults (never license via the CDM); a missing key fails. ```shell title="Fetch keys only, no download" -unshackle dl --skip-dl NF 81234567 +unshackle dl --skip-dl EXAMPLE 81234567 ``` `--export` writes a JSON file, into the configured exports directory, containing track @@ -494,7 +494,7 @@ info and the acquired content keys for each title. This is the format consumed b `unshackle import` to reconstruct a download later. ```shell title="Export track info and keys" -unshackle dl --skip-dl --export NF 81234567 +unshackle dl --skip-dl --export EXAMPLE 81234567 ``` !!! note "Region is recorded only with a proxy" @@ -511,10 +511,10 @@ tagging and naming: | `--tmdb` | `--tmdb 27205` | Use this TMDB ID instead of automatic lookup. | | `--imdb` | `--imdb tt1375666` | Use this IMDb ID. | | `--animeapi` | `--animeapi mal:12345` | Resolve via AnimeAPI (`mal:`/`anilist:` prefix; defaults to MAL). | -| `--enrich` | — | Override the show title and year from an external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | +| `--enrich` | - | Override the show title and year from an external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | ```shell title="Force the right IMDb match and enrich the title" -unshackle dl --imdb tt1375666 --enrich NF 81234567 +unshackle dl --imdb tt1375666 --enrich EXAMPLE 81234567 ``` ## Configuration defaults diff --git a/docs/guide/drm-and-cdm.md b/docs/guide/drm-and-cdm.md index 4b8438b3..69c691e5 100644 --- a/docs/guide/drm-and-cdm.md +++ b/docs/guide/drm-and-cdm.md @@ -48,7 +48,7 @@ license-server handshake; the rest are special cases. |--------|-------------|--------------|------------| | **Widevine** (Google) | `.wvd` | CDM challenge → license server → CDM parses → keys | shaka-packager / mp4decrypt | | **PlayReady** (Microsoft) | `.prd` | CDM challenge (WRM header) → license server → CDM parses → keys | shaka-packager / mp4decrypt | -| **ClearKey (CENC)**, W3C EME `org.w3.clearkey` | none | JSON `{"kids":[…],"type":"temporary"}` → license server returns a JWK Set. No CDM. | shaka-packager / mp4decrypt | +| **ClearKey (CENC)**, W3C EME `org.w3.clearkey` | none | JSON `{"kids":[...],"type":"temporary"}` → license server returns a JWK Set. No CDM. | shaka-packager / mp4decrypt | | **ClearKey (HLS)**, AES-128 `EXT-X-KEY` | none | Key fetched directly from a URI or `data:` URL. No CDM, no challenge. | in-process AES-CBC | | **MonaLisa** (proprietary, WASM) | `.mld` | Ticket from the service API; keys extracted locally via WASM. No license server. | ML-Worker binary + AES-ECB, per segment | @@ -71,8 +71,8 @@ The differences that matter to you are: [Widevine and PlayReady on the same title](#widevine-and-playready-on-the-same-title)). To download a Widevine-protected title you need a `.wvd` device; for PlayReady you -need a `.prd` device. You can hold both and let unshackle choose, or force one with -the `--drm` flag. +need a `.prd` device. You can hold both, and unshackle automatically selects the +appropriate one for each track. !!! info "Obtaining device files" A `.wvd` or `.prd` encodes the private keys of a real client device. unshackle @@ -201,16 +201,11 @@ fallback: ```yaml title="unshackle.yaml" cdm: default: nexus_6p_26830_l3 - AMZN: chromecdm_l3 - NF: sm_g935f_l1 + EXAMPLE1: chromecdm_l3 + EXAMPLE2: sm_g935f_l1 ``` -Lookups are case-insensitive. On the command line you can override the configured -device for a single run: - -```console -$ unshackle dl -w chromecdm_l3 AMZN B0ABCDEF -``` +Lookups are case-insensitive. !!! tip "The device name is the file name" A device named `chromecdm_l3` in the config resolves to `chromecdm_l3.wvd` in the @@ -238,8 +233,9 @@ profile. === "By DRM system" - Keys `widevine` and `playready` select a device per system. This pairs with the - `--drm` flag, which accepts `wv`/`widevine` or `pr`/`playready`. + Keys `widevine` and `playready` select a device per system. unshackle applies + these automatically, using the `widevine` device for Widevine tracks and the + `playready` device for PlayReady tracks. ```yaml cdm: @@ -451,7 +447,7 @@ returns the keys as a JWK Set rather than a DRM-wrapped blob. unshackle builds a EME request of the form: ```json -{"kids": ["<base64url-kid>", …], "type": "temporary"} +{"kids": ["<base64url-kid>", ...], "type": "temporary"} ``` It sends this to the manifest's license URL (or to whatever a service's ClearKey @@ -471,8 +467,8 @@ Title needs a Widevine CDM but SOMESERVICE is configured with PlayReady. ``` Use the `cdm` mapping's [DRM-based selection](#advanced-selection-by-quality-drm-or-profile) -to configure a device for each system, or force one system with `--drm` on the -command line. +to configure a device for each system; unshackle selects the right one per track +automatically. ## Keys, vaults, and caching diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index 3c8665d1..aece4fc5 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -13,15 +13,15 @@ When a download finishes muxing, unshackle builds the final path from three piec ``` - **Output directory**: `directories.downloads` from your config, or whatever you pass to `-o/--output` on the command line for a single run. -- **Folder template**: an optional per-title subfolder. TV episodes and music tracks are **always** placed in a folder; movies only get one if you define a `movies` folder template (see [Folder templates](#folder-templates)). +- **Folder template**: an optional per-title subfolder. TV episodes and music tracks are **always** placed in a folder; movies only get one if you define a `movies` folder template, or a single-string `folder` template that applies to all title kinds (see [Folder templates](#folder-templates)). - **Filename template**: the `output_template` for the title kind (movie, series, or song). - **Extension**: chosen by the muxer: `.mkv` for video, `.mka` for audio-only, `.mks` for subtitle-only. !!! example "A typical episode path" ```text /home/user/Downloads/ - └─ Show Name S01 1080p AMZN WEB-DL DDP5.1 H.264/ - └─ Show.Name.S01E01.Pilot.1080p.AMZN.WEB-DL.DDP5.1.H.264-TAG.mkv + └─ Show Name S01 1080p EXAMPLE WEB-DL DDP5.1 H.264/ + └─ Show.Name.S01E01.Pilot.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv ``` The folder name comes from the `series` folder template and the file from the `series` output template. @@ -51,7 +51,7 @@ The recognized kinds are: | `movies` | Movies | *(required for movie downloads to be named, no built-in default)* | | `series` | TV episodes | *(required for episode downloads to be named)* | | `songs` | Music tracks | `"{track_number}. {title}"` | -| `albums` | Album **folders** only (see below) | — | +| `albums` | Album **folders** only (see below) | - | !!! note `movies` and `series` have no hardcoded filename fallback. If you download those title types, define their templates. `songs` falls back to `"{track_number}. {title}"` when omitted. @@ -80,7 +80,7 @@ unshackle looks at the characters you place *between* variables to decide the fi movies: "{title}.{year}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}" ``` ```text - The.Movie.2024.1080p.NF.WEB-DL.DDP5.1.H.264-TAG.mkv + The.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG.mkv ``` === "Readable style (spaces)" @@ -90,7 +90,7 @@ unshackle looks at the characters you place *between* variables to decide the fi movies: "{title} ({year}) {quality} {source} WEB-DL {audio_full} {video}" ``` ```text - The Movie (2024) 1080p NF WEB-DL DDP5.1 H.264.mkv + The Movie (2024) 1080p EXAMPLE WEB-DL DDP5.1 H.264.mkv ``` ## Template variables @@ -103,7 +103,7 @@ Every variable below is valid in both output and folder templates. Values are de |---|---|---| | `title` | Title name (movie/show/song name; `$` is rendered as `S`) | `The Show` | | `year` | Release year | `2024` | -| `source` | Service tag / class name (empty with `--no-source`) | `NF` | +| `source` | Service tag / class name (empty with `--no-source`) | `EXAMPLE` | | `quality` | Resolution with scan suffix | `1080p`, `2160p`, `576i` | | `resolution` | Resolution number only | `1080` | | `video` | Video codec | `H.264`, `H.265` | @@ -188,10 +188,10 @@ The per-kind folder keys are `movies`, `series`, `songs`, and `albums`. Any othe **Fallback behavior:** - A per-kind folder template wins if present; otherwise the single `folder` string is used; otherwise unshackle falls back to a built-in default. -- Built-in defaults when no folder template is set: movies get no folder at all unless a `movies` folder template exists; series fall back to a folder *derived* from the `series` output template (stripping `{episode}`, `{episode_name}`, and collapsing `{season_episode}` down to `{season}`); music albums fall back to `{artist} - {album} ({year})`. +- Built-in defaults when no folder template is set: movies get no folder at all unless a `movies` folder template (or a single-string `folder` template) exists; series fall back to a folder *derived* from the `series` output template (stripping `{episode}`, `{episode_name}`, and collapsing `{season_episode}` down to `{season}`); music albums fall back to `{artist} - {album} ({year})`. !!! note "Movies are flat by default" - Episodes and songs are always foldered. Movies are written directly into the output directory *unless* you define a `movies` folder template. Set one if you want each movie in its own directory. + Episodes and songs are always foldered. Movies are written directly into the output directory *unless* you define a `movies` folder template (or a single-string `folder` template, which folders every kind). Set one if you want each movie in its own directory. ## Muxing options @@ -200,7 +200,7 @@ Muxing (combining video, audio, subtitle, chapter and attachment tracks into a s ```yaml title="unshackle.yaml" muxing: set_title: true - merge_video: true + merge_video: false merge_audio: true default_language: audio: en @@ -210,7 +210,7 @@ muxing: | Key | Type | Default | Effect | |---|---|---|---| | `set_title` | bool | `true` | Write the title name into the MKV container title with `--title`. Set to `false` to omit it. | -| `merge_video` | bool | `true` | Group video tracks that share the same resolution, range, and codec into one file so only language varies inside it. | +| `merge_video` | bool | `false` | Group video tracks that share the same resolution, range, and codec into one file so only language varies inside it. | | `merge_audio` | bool | `true` | Merge audio tracks of the same kind so multiple languages sit in one file. | | `default_language` | map | *(unset)* | Preferred language per track type (`video` / `audio` / `subtitle`). A track in the preferred language is flagged as the default track. | @@ -249,13 +249,13 @@ tag_imdb_tmdb: true You can override the tag per run without editing config: ```bash -unshackle dl --tag OTHERGROUP AMZN "..." +unshackle dl --tag OTHERGROUP EXAMPLE "..." ``` The `--repack` flag adds the `REPACK` marker (surfaced through the `{repack}` variable) to the filename for that run: ```bash -unshackle dl --repack NF "..." +unshackle dl --repack EXAMPLE "..." ``` !!! note "Empty tags collapse cleanly" @@ -349,7 +349,7 @@ The fallback supports two placeholders: `{i}` (chapter number, starting at 1) an ``` ```text The Movie (2024)/ - └─ The.Movie.2024.1080p.NF.WEB-DL.DDP5.1.Atmos.H.265.HDR-TAG.mkv + └─ The.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.Atmos.H.265.HDR-TAG.mkv ``` === "TV series" @@ -362,7 +362,7 @@ The fallback supports two placeholders: `{i}` (chapter number, starting at 1) an ``` ```text The Show (2023)/Season S02/ - └─ The.Show.S02E04.The.Reckoning.2160p.AMZN.WEB-DL.DDP5.1.H.265-TAG.mkv + └─ The.Show.S02E04.The.Reckoning.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG.mkv ``` === "Music" diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md index fd5505ec..443e29e4 100644 --- a/docs/guide/proxies-and-vpn.md +++ b/docs/guide/proxies-and-vpn.md @@ -31,15 +31,15 @@ Proxying is controlled at download time by three options on the `dl` command: | `--no-proxy-download` | Use the proxy for the manifest, licence, and authentication, but bypass it for the **segment downloads** themselves. | ```shell title="Explicit proxy URI" -unshackle dl --proxy http://user:pass@1.2.3.4:8080 NF 81234567 +unshackle dl --proxy http://user:pass@1.2.3.4:8080 EXAMPLE 81234567 ``` ```shell title="Resolve a US server from your configured providers" -unshackle dl --proxy us NF 81234567 +unshackle dl --proxy us EXAMPLE 81234567 ``` ```shell title="Target a specific provider" -unshackle dl --proxy nordvpn:us NF 81234567 +unshackle dl --proxy nordvpn:us EXAMPLE 81234567 ``` !!! tip "Set a default proxy in config" @@ -55,7 +55,7 @@ proxy where it matters (passing the geo-check for the manifest and the licence) pulling the actual audio and video segments over your normal connection for full speed. ```shell title="Proxy the manifest and licence, download segments direct" -unshackle dl --proxy gb --no-proxy-download ITV 10a1234 +unshackle dl --proxy gb --no-proxy-download EXAMPLE 10a1234 ``` !!! warning "When this is unsafe" @@ -78,8 +78,8 @@ flowchart TD D -- no --> F["Ask every provider in order,<br/>first match wins"] ``` -1. **An explicit URI.** Anything shaped like `http://…`, `https://…`, or a `socks…` - URI is used exactly as given. unshackle logs `Using explicit Proxy: …` and does no +1. **An explicit URI.** Anything shaped like `http://...`, `https://...`, or a `socks...` + URI is used exactly as given. unshackle logs `Using explicit Proxy: ...` and does no lookup. 2. **A provider-prefixed query**: `provider:query`, for example `nordvpn:us` or `gluetun:windscribe:us`. unshackle finds the provider whose name matches the prefix @@ -107,8 +107,8 @@ provider, prefix the query (`--proxy nordvpn:us`). !!! note "Auto-loading providers" Most providers only load when you configure them under `proxy_providers:`. Three are - special: **ExpressVPN** and **Proton VPN** also load automatically if their cookie - file already exists on disk (even with no YAML), and **Hola** loads automatically + special: **ExpressVPN** and **Proton VPN** also load automatically once their cached + session file exists on disk (even with no YAML), and **Hola** loads automatically whenever the `hola-proxy` binary is found on your `PATH`. ### Connection feedback @@ -154,7 +154,7 @@ proxy_providers: malformed URI is rejected up front rather than failing mid-download. ```shell -unshackle dl --proxy de2 NF 81234567 +unshackle dl --proxy de2 EXAMPLE 81234567 ``` ## Gluetun @@ -187,11 +187,11 @@ gluetun:<vpn-provider>:<region> ``` ```shell title="A Windscribe US exit via Gluetun" -unshackle dl --proxy gluetun:windscribe:us NF 81234567 +unshackle dl --proxy gluetun:windscribe:us EXAMPLE 81234567 ``` ```shell title="A NordVPN Germany exit via Gluetun" -unshackle dl --proxy gluetun:nordvpn:de NF 81234567 +unshackle dl --proxy gluetun:nordvpn:de EXAMPLE 81234567 ``` After unshackle strips the leading `gluetun:`, the provider receives exactly @@ -347,24 +347,24 @@ Understanding what Gluetun does behind the scenes helps when something goes wron ## ExpressVPN -ExpressVPN emulates the flow used by the ExpressVPN browser extension: it authenticates -with cookies (or a cached token), obtains proxy-capable locations from ExpressVPN's API, -and returns an authenticated HTTPS proxy. Because it relies on browser cookies rather than -service credentials, setup is slightly more involved than the credential-based providers. +ExpressVPN mirrors the ExpressVPN Android TV app: it signs in through Express's OAuth 2.0 +device-login flow, obtains proxy-capable locations from Express's API, and returns an +authenticated HTTPS proxy. The device login runs once; after that the cached refresh token +keeps the session alive headlessly. -ExpressVPN **auto-loads** if its cookie file exists, so once the cookie is in place you do -not strictly need a YAML block, but you can add one to pin regions or servers. +Enable it and run a download interactively once: -**Default file locations:** +```yaml title="unshackle.yaml" +proxy_providers: + expressvpn: + enable: true +``` -| File | Default path | -|---|---| -| Browser cookies | `{cookies}/vpn/expressvpn.txt` | -| Token cache | `{cache}/global/expressvpn_tokens.json` | - -Where `{cookies}` and `{cache}` are your configured cookie and cache directories. The -cookie file may be a Netscape `cookies.txt`, a JSON cookie array, or a JSON dict. Export -your ExpressVPN session cookies from a logged-in browser into one of those forms. +With `enable: true` and an interactive terminal, unshackle prints a code and a URL to +enter it at. Once approved, the tokens are cached and reused automatically, so later runs +need no terminal. ExpressVPN also **auto-loads** once its token cache exists, so you can +drop `enable` after the first login, but you can keep a YAML block to pin regions or +servers: ```yaml title="unshackle.yaml (optional)" proxy_providers: @@ -376,6 +376,14 @@ proxy_providers: myserver: usny-newyork-2 ``` +**Default file location:** + +| File | Default path | +|---|---| +| Token cache | `{cache}/vpn/expressvpn_tokens.json` | + +Where `{cache}` is your configured cache directory. + **Query forms:** | Query | Meaning | @@ -410,7 +418,7 @@ proxy_providers: ``` **Required keys:** `username`, `password`. unshackle validates them up front: the -combined username and password must be exactly 48 lowercase alphanumeric characters, and +combined username and password must be exactly 48 alphanumeric characters (case-insensitive), and the username must not contain an `@`. If validation fails, you have almost certainly used your login credentials instead of your service credentials. @@ -437,7 +445,7 @@ The returned proxy is HTTPS on **port 89** (NordVPN disabled its plain-HTTP prox port 80 in 2021). ```shell -unshackle dl --proxy nordvpn:us:seattle NF 81234567 +unshackle dl --proxy nordvpn:us:seattle EXAMPLE 81234567 ``` ## Proton VPN @@ -521,7 +529,7 @@ that binary is found on your `PATH`. If you query Hola but the binary is missing provider raises an error telling you to install it. ```shell title="Install hola-proxy, then just query a country" -unshackle dl --proxy hola:us NF 81234567 +unshackle dl --proxy hola:us EXAMPLE 81234567 ``` **Query forms:** a two-letter country code (e.g. `us`, `gb`). unshackle asks `hola-proxy` @@ -547,7 +555,7 @@ proxy_providers: ``` **Required keys:** `username`, `password`, validated with the same 48-character rule as -NordVPN (combined username+password must be 48 lowercase alphanumeric characters, no `@`). +NordVPN (combined username+password must be 48 alphanumeric characters, case-insensitive, no `@`). `server_map` optionally pins server IDs per region. **Query forms:** diff --git a/docs/guide/subtitles.md b/docs/guide/subtitles.md index ab39249a..f7c96107 100644 --- a/docs/guide/subtitles.md +++ b/docs/guide/subtitles.md @@ -82,7 +82,7 @@ By default (`--s-lang all`) unshackle keeps every subtitle language the service To narrow it down, pass a comma-separated list of language tags: ```shell title="English and Spanish subtitles only" -unshackle dl --s-lang en,es NF 81234567 +unshackle dl --s-lang en,es EXAMPLE 81234567 ``` Language matching is fuzzy by default: `en` will match `en-US`, `en-GB`, and similar @@ -93,7 +93,7 @@ For strict matching, where `es-419` matches only `es-419` and never `es-ES`, add `--exact-lang`: ```shell title="Only Latin-American Spanish, no other Spanish variants" -unshackle dl --s-lang es-419 --exact-lang DSNP 'https://www.disneyplus.com/…' +unshackle dl --s-lang es-419 --exact-lang EXAMPLE 'https://www.example.com/...' ``` ### Requiring languages before downloading @@ -104,7 +104,7 @@ errors out. This is useful for batch jobs where a title without your must-have l should fail loudly rather than silently produce a partial result. ```shell title="Fail unless both English and French subs exist, then grab everything" -unshackle dl --require-subs en,fr AMZN B0ABCDEFGH +unshackle dl --require-subs en,fr EXAMPLE B0ABCDEFGH ``` !!! warning "`--require-subs` and `--s-lang` are mutually exclusive" @@ -118,7 +118,7 @@ to display even when you are watching in your own language. They are **excluded default**. Add `-fs` / `--forced-subs` to include them: ```shell title="Include forced tracks alongside full subtitles" -unshackle dl --s-lang en --forced-subs NF 81234567 +unshackle dl --s-lang en --forced-subs EXAMPLE 81234567 ``` ## Track types: forced, SDH, and CC @@ -181,7 +181,7 @@ TTML to WebVTT for muxing, as noted above). To force a specific output format, u `--sub-format`: ```shell title="Convert every subtitle to SRT" -unshackle dl --s-lang en,es --sub-format srt NF 81234567 +unshackle dl --s-lang en,es --sub-format srt EXAMPLE 81234567 ``` `--sub-format` accepts the common format names and aliases, including `srt`, `vtt`, @@ -189,7 +189,7 @@ unshackle dl --s-lang en,es --sub-format srt NF 81234567 all conversion: ```shell title="Keep whatever the service delivered" -unshackle dl --sub-format original DSNP 'https://www.disneyplus.com/…' +unshackle dl --sub-format original EXAMPLE 'https://www.example.com/...' ``` Conversion only runs when the source format differs from the target, so @@ -218,14 +218,55 @@ formats involved: | Backend | Best at | Requires | | --- | --- | --- | | `subtitleedit` | Highest fidelity; preserves positioning and italics | SubtitleEdit installed | -| `subby` | Native SRT output with clean-up | — | -| `pysubs2` | Best fidelity for styled SSA/ASS | — (pure Python, always available) | -| `pycaption` | Last-resort fallback; flattens styling | — | +| `subby` | Native SRT output with clean-up | - | +| `pysubs2` | Best fidelity for styled SSA/ASS | - (pure Python, always available) | +| `pycaption` | Last-resort fallback; flattens styling | - | With `conversion_method: auto`, unshackle ranks these automatically per conversion. Setting it to a specific value pins that backend as the first choice, falling back to others only if the pin cannot handle the pair. +#### What each backend actually preserves + +The table below comes from round-tripping a subtitle that carries italics, bold, +underline, positioning, and colour through every backend. Each cell lists the +styling that survives; `—` means the backend cannot handle that pair. + +| Conversion | `subtitleedit` | `pysubs2` | `subby` | `pycaption` | +| --- | --- | --- | --- | --- | +| WebVTT → SRT | italic, bold, underline, position | italic, underline | italic, position | *(strips all)* | +| WebVTT → ASS | italic, bold, underline, position | italic, bold, underline | — | — | +| WebVTT → TTML | italic, bold, underline, position | italic, bold, underline | position | *(strips all)* | +| ASS/SSA → SRT | all (+ colour) | italic, underline | — | — | +| ASS/SSA → TTML | position only | italic, bold, underline | — | — | +| ASS/SSA → VTT | position only | italic, underline | — | — | + +Reading the table: + +- **Keep the original**: leaving `--sub-format` unset (or set to `original`) never + round-trips the file, so every style survives. Only convert when your player cannot + read the source format. +- **`subtitleedit`** (SubtitleEdit / `seconv`): the best choice for anything → SRT. It + is the only backend that carries colour, and it embeds `{\an8}` tags to preserve + positioning. When writing TTML or WebVTT from ASS it flattens inline styling to plain + text and keeps only positioning, so avoid it there. +- **`pysubs2`**: keeps inline italic/underline on every pair, and bold except when + writing SRT or WebVTT. It never carries positioning or colour. SSA/ASS is its native + model, which makes it the best pick for SSA↔ASS. +- **`subby`**: reads only WebVTT/fVTT/SAMI (never ASS) and is tuned for → SRT, where it + uniquely converts WebVTT cue settings into `{\an8}` positioning. Its + `CommonIssuesFixer` may also drop near-duplicate cues. +- **`pycaption`**: strips all styling; last-resort fallback only. + +!!! tip "What `auto` picks" + `auto` prefers `subtitleedit` first when it is installed, then falls back to `subby` + for → SRT and `pysubs2` otherwise. The one exception comes from the table above: + for ASS/SSA → TTML or WebVTT it picks `pysubs2` even when SubtitleEdit is + installed, since SubtitleEdit flattens inline styling on those pairs. Most installs + do not ship SubtitleEdit, so in practice `auto` means `subby` (→ SRT) or `pysubs2`. + Install [SubtitleEdit / `seconv`](#subtitle-configuration) if you need colour or the + highest → SRT fidelity. + !!! note "Config wins over a service's preference" A service may set a `preferred_conversion_method` on its own tracks (for example when it ships subtitles that a particular backend handles best). An explicit diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 6f79df58..383850e3 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -64,7 +64,7 @@ The debug log is created during a `dl` run (and by [`import`](cli-reference.md#i <logs directory>/unshackle_debug_<service>_<timestamp>.jsonl ``` -For example, `unshackle_debug_NETFLIX_20260703-142530.jsonl`. The `<service>` tag and the timestamp mean each download gets its own file, so runs never overwrite each other. +For example, `unshackle_debug_EXAMPLE_20260703-142530.jsonl`. The `<service>` tag and the timestamp mean each download gets its own file, so runs never overwrite each other. !!! tip "Find your logs directory" The logs directory defaults to a `logs` folder inside the package data directory, but it can be relocated with the `directories.logs` config key. To see the exact resolved path on your machine, run [`unshackle env info`](#env-info) and read the **Directories** table. @@ -99,16 +99,16 @@ Because each line is independent JSON, `jq` is the natural tool: ```console # Show only error-level entries -jq 'select(.level == "ERROR")' unshackle_debug_NETFLIX_20260703-142530.jsonl +jq 'select(.level == "ERROR")' unshackle_debug_EXAMPLE_20260703-142530.jsonl # List every operation and whether it succeeded -jq -c '{operation, success}' unshackle_debug_NETFLIX_20260703-142530.jsonl +jq -c '{operation, success}' unshackle_debug_EXAMPLE_20260703-142530.jsonl # Pull the traceback out of the first error -jq -r 'select(.error) | .error.traceback[]' unshackle_debug_NETFLIX_20260703-142530.jsonl +jq -r 'select(.error) | .error.traceback[]' unshackle_debug_EXAMPLE_20260703-142530.jsonl # Show only DRM / licensing operations -jq 'select(.operation | startswith("drm_"))' unshackle_debug_NETFLIX_20260703-142530.jsonl +jq 'select(.operation | startswith("drm_"))' unshackle_debug_EXAMPLE_20260703-142530.jsonl ``` The session's binary-tool versions (Shaka Packager, mp4decrypt, mkvmerge, FFmpeg, FFprobe) are logged near the start of a `dl` run, which is often the fastest way to confirm a muxing or decryption failure is really a wrong-version problem. @@ -168,7 +168,7 @@ unshackle redacts logged strings so that debug output is safe to share. Three pa Path redaction is controlled by the `redact_paths` config key, which is **on by default**: ```yaml -redact_paths: true # default — mask local base directories in logged paths +redact_paths: true # default - mask local base directories in logged paths ``` Set `redact_paths: false` if you are debugging a path problem locally and want to see full, unmasked paths in the console and logs. Secret and URL redaction in the JSON debug log are always applied and are not affected by this toggle. @@ -223,7 +223,7 @@ Stale cache or leftover temp files can cause confusing behaviour (an old cached ```console unshackle env clear cache # clear the whole cache directory -unshackle env clear cache NETFLIX # clear just one service's cache subdirectory +unshackle env clear cache EXAMPLE # clear just one service's cache subdirectory unshackle env clear temp # clear the temp working directory ``` @@ -287,7 +287,7 @@ The debug log is tuned for **developers troubleshooting pipeline flow** (maximum ```console # Read the pipeline's high-level flow, one line per stage - jq 'select(.level == "INFO")' unshackle_debug_NETFLIX_20260703-142530.jsonl + jq 'select(.level == "INFO")' unshackle_debug_EXAMPLE_20260703-142530.jsonl ``` !!! warning "No raw dumps" diff --git a/docs/guide/vaults.md b/docs/guide/vaults.md index a78a7c1a..f62926b8 100644 --- a/docs/guide/vaults.md +++ b/docs/guide/vaults.md @@ -13,7 +13,7 @@ Every key is stored as a **KID → KEY** pair, grouped by **service**: - **KID** (Key ID): a 16-byte identifier for a specific encrypted stream, stored as 32 lowercase hex characters with no dashes. - **KEY**: the 16-byte content key that decrypts that stream, also 32 hex characters. -- **Service**: the canonical service tag (for example `DSNP`, not `DisneyPlus`) that namespaces the key. Each service gets its own logical table. +- **Service**: the canonical service tag (for example `EXAMPLE`, not `ExampleService`) that namespaces the key. Each service gets its own logical table. !!! note "Why keys are matched by KID" Vaults match keys by KID, never by PSSH/CENC header. A KID does not change unless the underlying video file itself changes, whereas the PSSH box can vary between requests for the very same content. That makes the KID a far more reliable match key. @@ -242,7 +242,7 @@ $ unshackle kv --help Add `KID:KEY` pairs to one or more vaults for a given service. The file must contain one pair per line as `HEX:HEX` (32 hex characters each); lines that do not match are ignored. ```console -$ unshackle kv add keys.txt DSNP local shared +$ unshackle kv add keys.txt EXAMPLE local shared ``` ```text title="keys.txt" @@ -250,7 +250,7 @@ $ unshackle kv add keys.txt DSNP local shared 0e7cc1a4e2f9411db0f24a3fbb8f8b21:00112233445566778899aabbccddeeff ``` -The service argument (`DSNP` above) is normalized to its canonical tag automatically. Keys that already exist in a vault are reported as skipped. +The service argument (`EXAMPLE` above) is normalized to its canonical tag automatically. Keys that already exist in a vault are reported as skipped. ### `kv search`: find a key by KID diff --git a/docs/index.md b/docs/index.md index 2ab8380a..b71f6f46 100644 --- a/docs/index.md +++ b/docs/index.md @@ -71,11 +71,11 @@ Every download is three parts: the `dl` command, a **service tag**, and the service's title argument (a URL, ID, or slug). ```shell title="Download a 1080p title with English audio and subtitles" -unshackle dl -q 1080 -l en NF 81234567 +unshackle dl -q 1080 -l en EXAMPLE 81234567 ``` The `dl` command carries all the quality, language, and output flags; the service -tag (`NF` above) selects which streaming service to talk to, and the value after it +tag (`EXAMPLE` above) selects which streaming service to talk to, and the value after it is passed to that service. See the [Quickstart](getting-started/quickstart.md) to go from a fresh install to your first file. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 3d22f2c6..633a1350 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -37,7 +37,7 @@ This page assumes you already know the basics and want precise details. | [DRM & CDM](#drm-cdm) | `cdm`, `remote_cdm`, `decryption` | | [Network & proxy](#network-proxy) | `network`, `headers`, `proxy_providers` | | [Key vaults](#key-vaults) | `key_vaults`, `vault_timeout` | -| [External API keys](#external-api-keys) | `tmdb_api_key`, `simkl_client_id`, `decrypt_labs_api_key`, `ipinfo_api_key` | +| [External API keys](#external-api-keys) | `imdb_api_enabled`, `omdb_api_key`, `tmdb_api_key`, `simkl_client_id`, `decrypt_labs_api_key`, `ipinfo_api_key` | | [Caching & updates](#caching-updates) | `title_cache_enabled`, `title_cache_time`, `title_cache_max_retention`, `update_checks`, `update_check_interval` | | [Logging, privacy & debug](#logging-privacy-debug) | `redact_paths`, `debug`, `debug_keys`, `debug_requests`, `set_terminal_bg` | | [Deprecated & removed](#deprecated-removed-keys) | `curl_impersonate`, `downloader`, `scene_naming` | @@ -79,11 +79,11 @@ directories: | `data` | path | `unshackle/` | ❌ protected | Base for the data subdirectories above. | | `core_dir` | path | `unshackle/core` | ❌ protected | Package core. | | `namespace_dir` | path | `unshackle/` | ❌ protected | Package root. | -| `app_dirs` | — | `AppDirs("unshackle", False)` | ❌ protected | Internal AppDirs instance. | +| `app_dirs` | - | `AppDirs("unshackle", False)` | ❌ protected | Internal AppDirs instance. | !!! note "The `services` directory is special" `services` may be a **list**, and each entry can be either a local directory or a - **repository spec**: a git URL (`https://…`, `ssh://…`, `git@…`, or anything ending in + **repository spec**: a git URL (`https://...`, `ssh://...`, `git@...`, or anything ending in `.git`) or `owner/repo` shorthand. Repo specs are cloned and updated automatically; plain paths are used as-is. **List order is priority**: the first source to define a service tag wins, so put your own local service directories **last** to make them fallbacks. @@ -142,8 +142,8 @@ runtime. - **Type:** `dict` keyed by service tag  ·  **Default:** `{}` -Per-service configuration, keyed by the canonical service tag (for example `AMZN`, `NF`, -`DSNP`). unshackle reads several well-known sub-keys out of each service's block: +Per-service configuration, keyed by the canonical service tag (for example `EXAMPLE1`, `EXAMPLE2`, +`EXAMPLE3`). unshackle reads several well-known sub-keys out of each service's block: - **`proxy_map`**: a dict mapping `"provider:query"` or `"query"` to a proxy value, used to auto-select a proxy for that service. @@ -163,11 +163,11 @@ the service as `self.config`. ```yaml services: - AMZN: + EXAMPLE1: proxy_map: default: us title_map: - "The Grand Tour: The Movie": "The Grand Tour Presents The Movie" + "My Show: The Movie": "My Show Presents The Movie" ``` ### `credentials` @@ -180,8 +180,8 @@ objects; the credential's SHA-1 is also used as an account hash for cache keys. ```yaml credentials: - NF: user@example.com:hunter2 - AMZN: + EXAMPLE2: user@example.com:hunter2 + EXAMPLE1: - primary@example.com:pw1 - secondary@example.com:pw2 ``` @@ -230,13 +230,46 @@ CLI commands that run against the remote server. - **Type:** `dict`  ·  **Default:** `{}` -Configuration for the `serve` command (the built-in REST API server). +Configuration for the `serve` command (the built-in REST API server). The full server guide +is the [REST API](../dev/rest-api/index.md) section; these are the config keys. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `api_secret` | str | *(unset)* | Master secret accepted in the `X-Secret-Key` header. | +| `users` | dict | `{}` | Per-user API keys and their allowlists (see below). | +| `services` | list | *(unset)* | Global service allowlist. Omit to allow all. | +| `remote_only` | bool | `false` | Run without a local `output_template` (CORS/Cloudflare-friendly). | +| `session_ttl` | int (s) | `300` | Lifetime of an interactive auth session. | +| `max_sessions` | int | `100` | Maximum concurrent sessions. | +| `history_limit` | int | `100` | How many finished jobs to retain in history. | +| `compression_level` | int | `1` | gzip level for responses. | +| `cdm_overrides` | dict | *(unset)* | Allowed per-request CDM overrides. | +| `allow_job_credentials` | bool | `false` | Permit clients to supply credentials per job. | +| `devices` | list | *(auto)* | Widevine devices offered; auto-filled from `directories.wvds`. | +| `playready_devices` | list | *(auto)* | PlayReady devices; auto-filled from `directories.prds`. | +| `country` / `locations` | str / dict | *(unset)* | Advertised region metadata. | + +Each entry under `users` is keyed by that user's API key and may set its own `services`, +`devices`, and `playready_devices` allowlists, narrowing the global ones. + +```yaml +serve: + api_secret: change-me + remote_only: true + services: [EXAMPLE1, EXAMPLE2] + # server-wide download defaults (same keys as the `dl:` block) + downloads: 3 + best_available: true + users: + a1b2c3d4: # this user's API key + services: [EXAMPLE1] # may only use EXAMPLE1 +``` !!! tip "Declaring `dl` defaults under `serve`" - You can nest `dl` flags (`downloads`, `workers`, `best_available`, and so on) inside - `serve`. The point is to raise concurrency and behaviour defaults **server-wide** (every - request the server handles picks them up) without having to change each client call - individually. + You can set any [`dl`](#dl) flag key (`downloads`, `workers`, `best_available`, and so on) + directly inside `serve`. The point is to raise concurrency and behaviour defaults + **server-wide** (every request the server handles picks them up) without having to change + each client call individually. --- @@ -246,15 +279,117 @@ Configuration for the `serve` command (the built-in REST API server). - **Type:** `dict`  ·  **Default:** `{}` -Default options forwarded to the `dl` command. Anything you would otherwise pass as a `dl` -flag can be given a persistent default here. See [Downloading](../guide/downloading.md) for -the flags themselves. +Persistent defaults for the `dl` command. This block is used as Click's `default_map`, so +**any** `dl` flag can be given a default here and it applies to every download without you +typing it. See [Downloading](../guide/downloading.md) and the +[CLI Reference](../guide/cli-reference.md) for what each flag does. + +**Key naming.** A key is the flag's long name with dashes replaced by underscores +(`--best-available` → `best_available`, `--sub-format` → `sub_format`). A few keys are named +after the flag's internal destination rather than its visible name; set these exact keys: + +| Flag | Config key | +|------|-----------| +| `-r` / `--range` | `range` | +| `--list` | `list` | +| `--tmdb` | `tmdb_id` | +| `--imdb` | `imdb_id` | +| `--animeapi` | `animeapi_id` | +| `-naa` / `--noatmos` | `no_atmos` | +| `-o` / `--output` | `output_dir` | +| `--cdm-only` / `--vaults-only` | `cdm_only` | + +**Precedence** (highest first): an explicit CLI flag or environment variable → a per-service +[`services.<TAG>.dl`](#services-auth) override → this global `dl:` block → the built-in +default. In other words, config only fills in options you did not set on the command line. + +Common keys (a useful subset; every `dl` flag works): + +| Key | Type | Default | Sets | +|-----|------|---------|------| +| `lang` | list | `["orig"]` | Video/audio language(s); `orig` = original language. | +| `a_lang` / `v_lang` | list | `[]` | Audio- / video-only language override. | +| `s_lang` | list | `["all"]` | Subtitle language(s). | +| `quality` | list | `[]` (best) | Resolution(s), e.g. `[1080]`. | +| `vcodec` | list | `[]` (any) | Video codec(s), e.g. `[H265]`. | +| `acodec` | list | `[]` (any) | Audio codec(s), e.g. `[EC3]`. | +| `range` | list | `["SDR"]` | Colour range(s): `SDR`, `HDR10`, `HDR10+`, `DV`. | +| `channels` | float | *(unset)* | Audio channels, e.g. `6` for 5.1. | +| `sub_format` | str | *(unset)* | Convert subtitles to this format (`srt`, `vtt`, `original`, ...). | +| `forced_subs` | bool | `false` | Include forced subtitle tracks. | +| `no_subs` / `no_audio` / `no_chapters` | bool | `false` | Skip that track type. | +| `downloads` | int | `1` | Tracks downloaded concurrently. | +| `workers` | int | *(downloader default)* | Threads per track. | +| `slow` | str | *(unset)* | Inter-title delay, e.g. `"20-40"`. | +| `best_available` | bool | `false` | Fall back to best quality if the request is unavailable. | +| `proxy` | str | *(unset)* | Default proxy URI or 2-letter country. | +| `no_folder` | bool | `false` | Do not create a per-title folder. | + +```yaml title="Global download defaults" +dl: + lang: [en] + quality: [1080] + range: [HDR10] + vcodec: [H265] + sub_format: srt + downloads: 2 +``` + +!!! tip "Per-service overrides" + Nest a `dl` block under a service in [`services`](#services-auth) to override these defaults + for just that service. It uses the same keys and wins over the global `dl:` block (but still + loses to an explicit CLI flag). + + ```yaml + dl: + lang: [en] # global default + services: + EXAMPLE1: + dl: + lang: [en, ja] # EXAMPLE1 downloads both; everything else stays English + range: [DV] + ``` + +!!! note "`serve` uses these same keys" + The [`serve`](#serve) block accepts the same `dl` option keys at its top level to set + server-wide download defaults for every request it handles. ### `subtitle` - **Type:** `dict`  ·  **Default:** `{}` -Subtitle handling options. See [Subtitles](../guide/subtitles.md). +Subtitle handling: SDH stripping, format conversion, and whether subtitles are muxed in or +written as sidecar files. See [Subtitles](../guide/subtitles.md) for the full guide. + +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `strip_sdh` | bool | `true` | Strip SDH/CC cues (hearing-impaired annotations) into a clean track. | +| `sdh_method` | str | `"auto"` | SDH-stripping backend to use. | +| `preserve_formatting` | bool | `true` | Keep styling/positioning when converting or stripping. | +| `convert_before_strip` | bool | `true` | Convert to the working format before stripping SDH. | +| `conversion_method` | str | `"auto"` | Subtitle conversion backend. | +| `preferred_conversion_method` | str | *(unset)* | Preferred converter when several are available. | +| `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files. | +| `sidecar_format` | str | `"srt"` | Format for sidecar files when `output_mode: sidecar`. | +| `type_priority` | list | *(unset)* | Ordered ranking of subtitle types (`forced`, `normal`, `sdh`; CC counts as SDH) that sorts the variants within each language. Types you leave out fall to the end. | + +```yaml +subtitle: + strip_sdh: true + output_mode: sidecar + sidecar_format: srt +``` + +!!! note "`type_priority` also picks the default subtitle" + `type_priority` sorts the tracks within each language, and at mux time the first track + in the default subtitle language gets the `default` flag. The built-in order is `forced`, + `normal`, `sdh`, which means a forced track becomes default whenever one exists. If you + want the full dialogue track as default instead, set: + + ```yaml + subtitle: + type_priority: [normal, sdh, forced] + ``` ### `audio` @@ -284,6 +419,21 @@ audio: Matroska (MKV) muxing options. +| Sub-key | Type | Default | Description | +|---------|------|---------|-------------| +| `set_title` | bool | `true` | Write a human-readable title into the MKV container. | +| `default_language` | dict | `{}` | Force which language is flagged *default* per track type, e.g. `{audio: en, subtitle: en}`. | +| `merge_audio` | bool | `true` | Merge all audio into one file. `--split-audio` on the CLI flips this off. | +| `merge_video` | bool | `false` | Merge all video tracks into one file. `--merge-video` on the CLI flips this on. | + +```yaml +muxing: + set_title: true + default_language: + audio: en + subtitle: en +``` + ### `language_tags` - **Type:** `dict`  ·  **Default:** `{}` @@ -381,7 +531,7 @@ then `default`. ```yaml cdm: default: chromecdm_l3 - NF: android_l1 + EXAMPLE2: android_l1 ``` A value may also be a **nested dict** for advanced selection by quality, DRM system, or @@ -389,7 +539,7 @@ profile: ```yaml cdm: - AMZN: + EXAMPLE1: ">=1080": android_l1 # by track height "<1080": chromecdm_l3 widevine: android_l1 # by DRM system @@ -523,13 +673,13 @@ the full provider guide. Recognised providers and their exit ports: | Provider | Config key | Credentials | Proxy scheme/port | |----------|-----------|-------------|-------------------| | Basic (static) | `basic` | country → URI(s) | as specified | -| NordVPN | `nordvpn` | 48-char **service** credentials | `https://…:89` | -| Surfshark | `surfsharkvpn` | 48-char **service** credentials | `https://…:443` | -| Windscribe | `windscribevpn` | service credentials | `https://…:443` | -| ExpressVPN | `expressvpn` | browser cookies / token cache | `https://cat:…@…:443` | -| ProtonVPN | `protonvpn` | TV login or exported cookies | `https://…:4443` (or `:443` Secure Core) | +| NordVPN | `nordvpn` | 48-char **service** credentials | `https://...:89` | +| Surfshark | `surfsharkvpn` | 48-char **service** credentials | `https://...:443` | +| Windscribe | `windscribevpn` | service credentials | `https://...:443` | +| ExpressVPN | `expressvpn` | device login (`enable: true`) / token cache | `https://cat:...@...:443` | +| ProtonVPN | `protonvpn` | TV login or exported cookies | `https://...:4443` (or `:443` Secure Core) | | Gluetun | `gluetun` | per-VPN keys/creds | `http://localhost:{port}` (local Docker) | -| Hola | *(none, auto)* | none | `http://…:{peer}` | +| Hola | *(none, auto)* | none | `http://...:{peer}` | ```yaml proxy_providers: @@ -546,7 +696,7 @@ proxy_providers: !!! note "Provider loading differs between CLI and REST server" The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API / remote-client path uses a separate resolver that does **not** load `windscribevpn` or - `gluetun`. ExpressVPN and ProtonVPN also auto-load when their cookie file exists, and Hola + `gluetun`. ExpressVPN and ProtonVPN also auto-load when their cached session exists, and Hola auto-loads whenever the `hola-proxy` binary is present. --- @@ -606,6 +756,8 @@ All default to an empty string and enable optional metadata/geolocation features | Key | Type | Default | Description | |-----|------|---------|-------------| +| `imdb_api_enabled` | bool | `false` | Use the free IMDxAPI (api.tiffara.com) metadata provider, which needs no API key. Off by default since the site has been unreliable. | +| `omdb_api_key` | str | `""` | [OMDb](https://www.omdbapi.com/) API key for IMDb metadata lookups; a more reliable alternative to IMDxAPI. Free keys are available on the OMDb site. | | `tmdb_api_key` | str | `""` | TMDB API key for metadata enrichment and external-ID tags. | | `simkl_client_id` | str | `""` | SIMKL client ID for metadata lookups; an alternative/fallback source to TMDB. | | `decrypt_labs_api_key` | str | `""` | Global Decrypt Labs API key (used by remote CDM / vault). | @@ -704,12 +856,12 @@ headers: cdm: default: chromecdm_l3 - NF: android_l1 + EXAMPLE2: android_l1 decryption: shaka credentials: - NF: user@example.com:hunter2 + EXAMPLE2: user@example.com:hunter2 output_template: movies: "{title}.{year}.{quality?}.{source}-{tag}" @@ -729,6 +881,7 @@ proxy_providers: basic: us: http://user:pass@1.2.3.4:8080 +omdb_api_key: "your-omdb-key" tmdb_api_key: "your-tmdb-key" title_cache_time: 3600 redact_paths: true diff --git a/pyproject.toml b/pyproject.toml index 27a146fb..a78d0fc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "langcodes>=3.4.0,<4", "lxml>=5.2.1,<7", "protobuf>=4.25.3,<7", - "pycaption>=2.2.6,<3", + "pycaption>=2.2.6,<2.2.27", "pycryptodomex>=3.20.0,<4", "pyjwt>=2.12.0,<3", "pymediainfo>=6.1.0,<8", @@ -56,7 +56,7 @@ dependencies = [ "chardet>=5.2.0,<6", "pyplayready>=0.8.3,<0.9", "cryptography>=45.0.0,<47", - "subby", + "subby @ git+https://github.com/vevv/subby.git@a057280f31acd3a0ba5c6dc69eaefbb88aea974e", "aiohttp>=3.14.0,<4", "aiohttp-swagger3>=0.9.0,<1", "pysubs2>=1.7.0,<2", @@ -92,7 +92,7 @@ dev = [ "types-PyMySQL>=1.1.0.1,<2", "types-requests>=2.31.0.20240406,<3", "isort>=5.13.2,<8", - "ruff>=0.3.7,<0.15", + "ruff>=0.15.12,<0.16", ] test = [ "pytest>=8.3.0,<9", @@ -120,11 +120,15 @@ filterwarnings = [ ] addopts = "-ra --strict-markers" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build.targets.wheel] packages = ["unshackle"] [tool.hatch.build.targets.sdist] include = [ + "unshackle", "CHANGELOG.md", "README.md", "LICENSE", @@ -137,12 +141,19 @@ line-length = 120 [tool.ruff.lint] select = ["E4", "E7", "E9", "F", "W"] +[tool.ruff.lint.per-file-ignores] +# binaries are resolved lazily via module-level __getattr__ (PEP 562), so the +# names in __all__ are intentionally not defined statically. +"unshackle/core/binaries.py" = ["F822"] + [tool.isort] +profile = "black" line_length = 118 [tool.mypy] check_untyped_defs = true disallow_incomplete_defs = true +explicit_package_bases = true disallow_untyped_defs = true follow_imports = "silent" ignore_missing_imports = true @@ -160,4 +171,3 @@ skips = [ [tool.uv.sources] unshackle = { workspace = true } -subby = { git = "https://github.com/vevv/subby.git", rev = "a057280f31acd3a0ba5c6dc69eaefbb88aea974e" } diff --git a/tests/core/test_cdm_detect.py b/tests/core/test_cdm_detect.py new file mode 100644 index 00000000..612f2a1c --- /dev/null +++ b/tests/core/test_cdm_detect.py @@ -0,0 +1,208 @@ +"""CDM detection and load-time DRM stamping for local and remote CDMs. + +Covers the predicates in ``unshackle.core.cdm.detect`` (playready/widevine, local/remote, +and the remote combinations) and the loader stamping that sets a remote CDM's DRM type at +load time instead of guessing it on each call. Services such as AMZN route to a device +profile off that type, so a wrong classification here is what produces a Downgrade.Hd +license denial. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from unshackle.core.cdm import loader as loader_mod +from unshackle.core.cdm.detect import ( + cdm_location, + is_local_cdm, + is_playready_cdm, + is_remote_cdm, + is_remote_playready_cdm, + is_remote_widevine_cdm, + is_widevine_cdm, +) +from unshackle.core.cdm.loader import _stamp_remote + +pytestmark = pytest.mark.unit + +REPO = Path(__file__).resolve().parents[2] +PRD_DIR = REPO / "unshackle" / "PRDs" + + +def test_stamped_remote_playready(): + cdm = SimpleNamespace(drm="playready", is_remote_cdm=True) + assert is_playready_cdm(cdm) + assert not is_widevine_cdm(cdm) + assert is_remote_cdm(cdm) + assert is_remote_playready_cdm(cdm) + assert not is_remote_widevine_cdm(cdm) + assert cdm_location(cdm) == "remote" + + +def test_stamped_remote_widevine(): + cdm = SimpleNamespace(drm="widevine", is_remote_cdm=True) + assert is_widevine_cdm(cdm) + assert not is_playready_cdm(cdm) + assert is_remote_widevine_cdm(cdm) + assert not is_remote_playready_cdm(cdm) + assert cdm_location(cdm) == "remote" + + +def test_drm_stamp_beats_conflicting_is_playready_attr(): + # The stamped drm takes priority over the wrapper's own is_playready guess. + cdm = SimpleNamespace(drm="playready", is_playready=False, is_remote_cdm=True) + assert is_playready_cdm(cdm) + assert not is_widevine_cdm(cdm) + + +def test_drm_stamp_case_insensitive(): + cdm = SimpleNamespace(drm="PlayReady", is_remote_cdm=True) + assert is_playready_cdm(cdm) + + +def test_none_classifies_as_nothing(): + for fn in ( + is_playready_cdm, + is_widevine_cdm, + is_remote_cdm, + is_local_cdm, + is_remote_playready_cdm, + is_remote_widevine_cdm, + ): + assert fn(None) is False + assert cdm_location(None) == "unknown" + + +@pytest.mark.parametrize("device_name", ["SL2", "SL3", "SL2000", "SL3000", "sl3000"]) +def test_decryptlabs_sl_names_are_playready(device_name): + from unshackle.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + cdm = DecryptLabsRemoteCDM(secret="x", device_name=device_name) + assert cdm.is_playready + assert is_playready_cdm(cdm) + assert is_remote_cdm(cdm) + assert is_remote_playready_cdm(cdm) + assert not is_local_cdm(cdm) + + +@pytest.mark.parametrize("device_name", ["ChromeCDM", "L1", "L2"]) +def test_decryptlabs_widevine_names(device_name): + from unshackle.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + cdm = DecryptLabsRemoteCDM(secret="x", device_name=device_name) + assert not cdm.is_playready + assert is_widevine_cdm(cdm) + assert is_remote_widevine_cdm(cdm) + + +def test_decryptlabs_device_type_forces_playready(): + from unshackle.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + cdm = DecryptLabsRemoteCDM(secret="x", device_name="ChromeCDM", device_type="PLAYREADY") + assert cdm.is_playready + + +@pytest.mark.parametrize( + ("device", "expect_pr"), + [ + ({"name": "SL3000"}, True), + ({"name": "SL2"}, True), + ({"name": "ChromeCDM"}, False), + ({"type": "PLAYREADY", "name": "ChromeCDM"}, True), + ], +) +def test_custom_remote_cdm_classification(device, expect_pr): + from unshackle.core.cdm.custom_remote_cdm import CustomRemoteCDM + + cdm = CustomRemoteCDM(host="http://example.invalid", device=device) + assert cdm.is_playready is expect_pr + assert is_playready_cdm(cdm) is expect_pr + assert is_remote_cdm(cdm) + + +def test_stamp_remote_sets_attrs_and_returns_same_object(): + obj = SimpleNamespace() + out = _stamp_remote(obj, "playready") + assert out is obj + assert obj.drm == "playready" + assert obj.is_remote_cdm is True + + +def test_stamp_remote_is_best_effort_on_unsettable_object(): + class Slotted: + __slots__ = () + + obj = Slotted() + out = _stamp_remote(obj, "widevine") # must not raise + assert out is obj + assert not hasattr(obj, "drm") + + +def test_load_remote_decryptlabs_playready_is_stamped(): + cdm_api = {"name": "dl-pr", "type": "decrypt_labs", "secret": "x", "device_name": "SL3000"} + cdm = loader_mod._load_remote_cdm(dict(cdm_api), "dl-pr", "AMZN", None) + assert cdm.drm == "playready" + assert cdm.is_remote_cdm is True + assert is_remote_playready_cdm(cdm) + + +def test_load_remote_decryptlabs_widevine_is_stamped(): + cdm_api = {"name": "dl-wv", "type": "decrypt_labs", "secret": "x", "device_name": "ChromeCDM"} + cdm = loader_mod._load_remote_cdm(dict(cdm_api), "dl-wv", "AMZN", None) + assert cdm.drm == "widevine" + assert is_remote_widevine_cdm(cdm) + + +def test_load_remote_native_playready_is_stamped(monkeypatch): + import pyplayready.remote.remotecdm as prmod + + class FakePlayReadyRemote: + def __init__(self, **kwargs): + self.kwargs = kwargs + + monkeypatch.setattr(prmod, "RemoteCdm", FakePlayReadyRemote) + cdm_api = {"name": "pr", "Device Type": "PLAYREADY", "host": "h", "secret": "s", "device_name": "d"} + cdm = loader_mod._load_remote_cdm(dict(cdm_api), "pr", "AMZN", None) + assert isinstance(cdm, FakePlayReadyRemote) + assert cdm.drm == "playready" + assert cdm.is_remote_cdm is True + assert is_remote_playready_cdm(cdm) + + +def test_load_remote_native_widevine_is_stamped(monkeypatch): + import pywidevine.remotecdm as wvmod + + class FakeWidevineRemote: + def __init__(self, **kwargs): + self.kwargs = kwargs + + monkeypatch.setattr(wvmod, "RemoteCdm", FakeWidevineRemote) + cdm_api = {"name": "wv", "Device Type": "ANDROID", "host": "h", "secret": "s", "device_name": "d"} + cdm = loader_mod._load_remote_cdm(dict(cdm_api), "wv", "AMZN", None) + assert isinstance(cdm, FakeWidevineRemote) + assert cdm.drm == "widevine" + assert is_remote_widevine_cdm(cdm) + + +@pytest.fixture +def local_playready_cdm(): + prds = sorted(PRD_DIR.glob("*.prd")) + if not prds: + pytest.skip(f"no local .prd present in {PRD_DIR}") + from pyplayready.cdm import Cdm + from pyplayready.device import Device + + return Cdm.from_device(Device.load(str(prds[0]))) + + +def test_local_playready_is_playready_and_local(local_playready_cdm): + cdm = local_playready_cdm + assert is_playready_cdm(cdm) + assert not is_widevine_cdm(cdm) + assert is_local_cdm(cdm) + assert not is_remote_cdm(cdm) + assert not is_remote_playready_cdm(cdm) + assert cdm_location(cdm) == "local" diff --git a/tests/core/test_clearkey_cenc.py b/tests/core/test_clearkey_cenc.py index 6fbb3198..e19b59f2 100644 --- a/tests/core/test_clearkey_cenc.py +++ b/tests/core/test_clearkey_cenc.py @@ -16,6 +16,7 @@ from typing import Any, Optional from uuid import UUID import pytest + # lxml.etree: XML parser used to build ContentProtection fixtures for DASH.get_drm from lxml import etree diff --git a/tests/core/test_debug_logging.py b/tests/core/test_debug_logging.py index b52bfa5f..7041f160 100644 --- a/tests/core/test_debug_logging.py +++ b/tests/core/test_debug_logging.py @@ -9,8 +9,13 @@ from pathlib import Path import pytest -from unshackle.core.utilities import (close_debug_logger, get_debug_logger, init_debug_logger, log_event, - timed_operation) +from unshackle.core.utilities import ( + close_debug_logger, + get_debug_logger, + init_debug_logger, + log_event, + timed_operation, +) from unshackle.core.utils.redact import redact_all, redact_path, redact_text, redact_url, safe_display_url from unshackle.core.utils.subprocess import log_tool_run diff --git a/tests/core/test_ism_init.py b/tests/core/test_ism_init.py index 94e9855b..363bb031 100644 --- a/tests/core/test_ism_init.py +++ b/tests/core/test_ism_init.py @@ -12,11 +12,24 @@ import struct import pytest -from unshackle.core.manifests.ism_init import (NAL_START_CODE, PIFF_SENC_UUID, box, build_avcc, build_dec3, - build_hvcc, build_init_segment, full_box, - parse_codec_private_data_colour, parse_hevc_sps_format, - read_per_sample_iv_size, read_track_id, remove_emulation_prevention, - split_nal_units, synthesize_aac_codec_private_data) +from unshackle.core.manifests.ism_init import ( + NAL_START_CODE, + PIFF_SENC_UUID, + box, + build_avcc, + build_dec3, + build_hvcc, + build_init_segment, + full_box, + parse_codec_private_data_colour, + parse_codec_private_data_vui, + parse_hevc_sps_format, + read_per_sample_iv_size, + read_track_id, + remove_emulation_prevention, + split_nal_units, + synthesize_aac_codec_private_data, +) # Real CodecPrivateData taken from a Smooth Streaming manifest. VIDEO_HEVC_CPD = ( @@ -472,6 +485,15 @@ def test_parse_colour_absent_or_unknown_returns_none(): assert parse_codec_private_data_colour("HVC1", b"\x00\x00\x00\x01\x42") is None +def test_parse_vui_fps(): + assert parse_codec_private_data_vui("HVC1", bytes.fromhex(VIDEO_HEVC_CPD))[1] == pytest.approx(24000 / 1001) + assert parse_codec_private_data_vui("HVC1", bytes.fromhex(VIDEO_HEVC_PQ_CPD))[1] == 25.0 + # VUI timing can be present even when colour info is absent. + assert parse_codec_private_data_vui("HVC1", bytes.fromhex(VIDEO_HEVC10_CPD)) == (None, 25.0) + # AVC is deliberately untrusted for fps (field-based VUI timing). + assert parse_codec_private_data_vui("H264", bytes.fromhex(VIDEO_AVC_CPD)) == (None, None) + + def test_hvcc_profile_tier_level_is_nonzero(): # De-emulated PTL must yield real profile/level, not the off-by-one garbage. hvcc = build_hvcc(bytes.fromhex(VIDEO_HEVC_CPD)) diff --git a/tests/core/test_ism_parse.py b/tests/core/test_ism_parse.py new file mode 100644 index 00000000..0f0aa28b --- /dev/null +++ b/tests/core/test_ism_parse.py @@ -0,0 +1,43 @@ +import pytest + +from unshackle.core.manifests.ism import ISM +from unshackle.core.tracks import Audio + +MANIFEST = """<?xml version="1.0"?> +<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" Duration="60000000" TimeScale="10000000"> + <StreamIndex Type="audio" Name="audio" Language="en" + Url="QualityLevels({Bitrate})/Fragments(audio={start_time})"> + <QualityLevel Index="0" Bitrate="128000" AudioTag="255" SamplingRate="48000" + Channels="2" BitsPerSample="16" CodecPrivateData="1190"/> + <c t="0" d="20000000" r="2"/> + <c/> + </StreamIndex> +</SmoothStreamingMedia> +""" + + +def test_placeholder_spellings_repeat_and_audiotag_fallback() -> None: + tracks = ISM.from_text(MANIFEST, "https://cdn.example/x.ism/manifest").to_tracks() + (audio,) = tracks.audio + segments = audio.data["ism"]["segments"] + assert segments == [ + "https://cdn.example/x.ism/QualityLevels(128000)/Fragments(audio=0)", + "https://cdn.example/x.ism/QualityLevels(128000)/Fragments(audio=20000000)", + "https://cdn.example/x.ism/QualityLevels(128000)/Fragments(audio=40000000)", + ] + assert audio.codec == Audio.Codec.AAC # AudioTag 255 means AACL + + +def test_missing_language_without_fallback_raises() -> None: + # MS-SSTR Language is optional; video StreamIndexes commonly omit it. + no_lang = MANIFEST.replace(' Language="en"', "") + with pytest.raises(ValueError, match="fallback language"): + ISM.from_text(no_lang, "https://cdn.example/x.ism/manifest").to_tracks() + tracks = ISM.from_text(no_lang, "https://cdn.example/x.ism/manifest").to_tracks(language="en") + assert str(tracks.audio[0].language) == "en" + + +def test_live_manifest_rejected() -> None: + live = MANIFEST.replace('Duration="60000000"', 'Duration="60000000" IsLive="TRUE"', 1) + with pytest.raises(ValueError, match="Live"): + ISM.from_text(live, "https://cdn.example/x.ism/manifest").to_tracks() diff --git a/tests/core/test_ism_range.py b/tests/core/test_ism_range.py index 1cb88eb9..59728409 100644 --- a/tests/core/test_ism_range.py +++ b/tests/core/test_ism_range.py @@ -4,11 +4,11 @@ undecidable from the manifest; the post-mux bitstream probe names it.""" from __future__ import annotations +from tests.core.test_ism_init import VIDEO_HEVC10_CPD, VIDEO_HEVC_DV_CPD, VIDEO_HEVC_PQ_CPD, VIDEO_HEVC_SDR_CPD from unshackle.core.manifests import ISM from unshackle.core.tracks import Video -from tests.core.test_ism_init import (VIDEO_HEVC10_CPD, VIDEO_HEVC_DV_CPD, VIDEO_HEVC_HLG_CPD, # isort: skip - VIDEO_HEVC_PQ_CPD, VIDEO_HEVC_SDR_CPD) +from tests.core.test_ism_init import VIDEO_HEVC_HLG_CPD # isort: skip def manifest_xml(cpd: str, fourcc: str = "HVC1") -> str: diff --git a/tests/core/test_music_display.py b/tests/core/test_music_display.py index ba5c98b7..0992086d 100644 --- a/tests/core/test_music_display.py +++ b/tests/core/test_music_display.py @@ -12,8 +12,14 @@ from typing import Any, Optional from rich.console import Console, RenderableType from rich.panel import Panel -from unshackle.core.music.display import (MusicHeaderInfo, TrackRow, format_track_detail_quality, render_album_header, - render_artwork_preview, render_track_panel) +from unshackle.core.music.display import ( + MusicHeaderInfo, + TrackRow, + format_track_detail_quality, + render_album_header, + render_artwork_preview, + render_track_panel, +) from unshackle.core.titles.music import Song @@ -97,8 +103,20 @@ def test_format_track_detail_quality_passthrough() -> None: def test_render_track_panel_returns_panel() -> None: rows = [ - TrackRow(song=make_song(track=1, name="First"), quality_label="FLAC 16-bit/44.1kHz", layout="Stereo", duration_str="3:00", cd=True), - TrackRow(song=make_song(track=2, name="Second"), quality_label="OGG 320", layout="Stereo", duration_str="4:00", hires=True), + TrackRow( + song=make_song(track=1, name="First"), + quality_label="FLAC 16-bit/44.1kHz", + layout="Stereo", + duration_str="3:00", + cd=True, + ), + TrackRow( + song=make_song(track=2, name="Second"), + quality_label="OGG 320", + layout="Stereo", + duration_str="4:00", + hires=True, + ), ] panel = render_track_panel(rows, total=len(rows)) assert isinstance(panel, Panel) @@ -112,7 +130,9 @@ def test_render_track_panel_returns_panel() -> None: def test_render_track_panel_singular_label() -> None: - rows = [TrackRow(song=make_song(name="Only"), quality_label="FLAC 16-bit/44.1kHz", layout="Stereo", duration_str="3:00")] + rows = [ + TrackRow(song=make_song(name="Only"), quality_label="FLAC 16-bit/44.1kHz", layout="Stereo", duration_str="3:00") + ] text = render_to_text(render_track_panel(rows, total=1)) # The count node reads "1 Track" (singular); the panel title is always # "Available Tracks", so we only check the count node phrasing here. diff --git a/tests/core/test_music_extract.py b/tests/core/test_music_extract.py index 2702781b..3b7ca080 100644 --- a/tests/core/test_music_extract.py +++ b/tests/core/test_music_extract.py @@ -6,9 +6,17 @@ from typing import Any, Optional import pytest -from unshackle.core.music.extract import (build_music_from_songs, classify_release_kind, dedupe_track_options, - duration_seconds, first_number, first_text, format_duration, format_names, - year_from_value) +from unshackle.core.music.extract import ( + build_music_from_songs, + classify_release_kind, + dedupe_track_options, + duration_seconds, + first_number, + first_text, + format_duration, + format_names, + year_from_value, +) from unshackle.core.music.models import MusicTrackOption from unshackle.core.titles.music import Music, Song diff --git a/tests/core/test_service_dl_overrides.py b/tests/core/test_service_dl_overrides.py index 704f1f64..25321aa5 100644 --- a/tests/core/test_service_dl_overrides.py +++ b/tests/core/test_service_dl_overrides.py @@ -115,9 +115,7 @@ def hand_built_ctx(body: dict[str, Any]) -> click.Context: ctx = click.Context(parsed.command) ctx.params = {"repack": body.get("repack", False)} for name in ctx.params: - ctx.set_parameter_source( - name, ParameterSource.COMMANDLINE if name in body else ParameterSource.DEFAULT - ) + ctx.set_parameter_source(name, ParameterSource.COMMANDLINE if name in body else ParameterSource.DEFAULT) return ctx diff --git a/tests/core/test_service_repo.py b/tests/core/test_service_repo.py index 17b12782..05ee84f1 100644 --- a/tests/core/test_service_repo.py +++ b/tests/core/test_service_repo.py @@ -240,9 +240,7 @@ def test_refresh_reports_discarded_local_edits(monkeypatch, tmp_path): def test_changed_services_up_to_date(monkeypatch): monkeypatch.setattr(service_repo.binaries, "Git", "git") # same HEAD before/after → no diff computed at all - monkeypatch.setattr( - service_repo.subprocess, "run", lambda *a, **k: pytest.fail("no git diff when HEAD unchanged") - ) + monkeypatch.setattr(service_repo.subprocess, "run", lambda *a, **k: pytest.fail("no git diff when HEAD unchanged")) assert service_repo._changed_services(Path("/x"), "same", "same") == [] diff --git a/tests/lang_selection/test_overlapping_audio_dedup.py b/tests/lang_selection/test_overlapping_audio_dedup.py new file mode 100644 index 00000000..7de66f77 --- /dev/null +++ b/tests/lang_selection/test_overlapping_audio_dedup.py @@ -0,0 +1,179 @@ +"""Regression: exact-lang must distinguish a base language from its regional variant. + +Reproduced with `--exact-lang --lang ar,ar-EG` against a streaming manifest that carries a +distinct `ar` ("Arabic") and `ar-EG` ("Arabic (Egypt)") audio track. CLDR tag_distance measures +intelligibility, not tag identity: it rates a base language and its "paradigm" regional variant +as the same language at distance 0 (ar/ar-EG, en/en-US, pt/pt-BR). Plain distance matching then +resolved both requested tags to the same track, dropping one variant and appending the other +twice, which crashed Tracks() reconstruction. + +In exact mode by_language follows RFC 4647 Lookup: it prefers the most specific (string-equal) +tag, so each tag routes to its own track. It still falls back to the fuzzy match for pure aliases +(zh matches cmn) and the lenient base-to-regional case (en finds en-US when only en-US exists). +Distances above 0 (es/es-419, fr/fr-CA) were never collapsed and stay strict. +""" + +from __future__ import annotations + +import pytest + +from unshackle.core.tracks import Audio +from unshackle.core.tracks.tracks import Tracks +from unshackle.core.utilities import is_exact_match + +# Base language vs regional variant that CLDR rates at distance 0 (paradigm locales). These are +# the pairs that collapse under naive exact matching. +COLLAPSING_PAIRS = [ + ("ar", "ar-EG"), + ("en", "en-US"), + ("pt", "pt-BR"), + ("fr", "fr-FR"), + ("es", "es-ES"), + ("de", "de-DE"), + ("it", "it-IT"), +] +# Pairs CLDR rates at distance >0, which exact matching already separates. Never collapsed. +# Includes two-regional splits (no bare base) as services more commonly ship them. +DISTINCT_PAIRS = [ + ("es", "es-419"), + ("fr", "fr-CA"), + ("fr-FR", "fr-CA"), + ("pt-PT", "pt-BR"), + ("es-ES", "es-419"), + ("en-US", "en-GB"), + ("de-DE", "de-AT"), + ("it-IT", "it-CH"), + ("es-419", "es-AR"), + ("no", "nn"), +] + +# A full real-world audio language set for one title, the kind of mix a streaming manifest ships, +# which must resolve 1:1 with no collision or loss. Tags are drawn from published streaming +# language references (Warner Bros. Discovery / Netflix) and include regional and script variants. +REAL_WORLD_AUDIO_LANGS = [ + "ar", + "ar-EG", + "ar-SA", + "bn-IN", + "cmn-TW", + "cs", + "da", + "de", + "de-AT", + "de-CH", + "el", + "en", + "en-GB", + "en-AU", + "es-419", + "es-AR", + "es-ES", + "fi", + "fr-CA", + "fr-FR", + "he", + "hi", + "hu", + "id", + "it", + "it-CH", + "ja", + "ko", + "nl", + "no", + "pl", + "pt-BR", + "pt-PT", + "ro", + "ru", + "sk", + "sr", + "sr-Latn", + "sv", + "ta", + "th", + "tr", + "uk", + "vi", + "yue", + "zh-Hans", + "zh-Hant", +] + + +def _mk(*tags): + return [Audio("https://x", language=t, id_=t) for t in tags] + + +def _select(tags, pool, exact=True): + return [(t.id, str(t.language)) for t in Tracks.by_language(pool, tags, per_language=1, exact_match=exact)] + + +@pytest.mark.parametrize("base,reg", COLLAPSING_PAIRS) +def test_collapsing_pairs_are_distance_zero(base, reg): + # Root cause: the regional variant exact-matches the base request (CLDR paradigm data). + assert is_exact_match(base, [reg]) is True + + +@pytest.mark.parametrize("base,reg", DISTINCT_PAIRS) +def test_distinct_pairs_are_not_distance_zero(base, reg): + # Contrast: these never collapse, which is why the bug was language-specific. + assert is_exact_match(base, [reg]) is False + + +@pytest.mark.parametrize("base,reg", COLLAPSING_PAIRS) +def test_exact_lang_selects_distinct_collapsing_variants(base, reg): + # Both tracks present + both tags requested: each tag must route to its own track. + assert _select([base, reg], _mk(base, reg)) == [(base, base), (reg, reg)] + + +@pytest.mark.parametrize("base,reg", DISTINCT_PAIRS) +def test_exact_lang_distinct_regionals_still_work(base, reg): + assert _select([base, reg], _mk(base, reg)) == [(base, base), (reg, reg)] + + +def test_exact_lang_lenient_base_to_regional_fallback(): + # Only the regional present; a base request must still find it when CLDR rates them equal. + assert _select(["en"], _mk("en-US")) == [("en-US", "en-US")] + # Alias with no string-equal track must still resolve (zh -> cmn). + assert _select(["zh"], _mk("cmn")) == [("cmn", "cmn")] + + +def test_exact_lang_stays_strict_for_nonzero_distance(): + # `es` does NOT accept `es-419` in exact mode (distance 5), but fuzzy mode does. + assert _select(["es"], _mk("es-419")) == [] + assert _select(["es"], _mk("es-419"), exact=False) == [("es-419", "es-419")] + + +@pytest.mark.parametrize("base,reg", COLLAPSING_PAIRS) +def test_overlapping_tags_never_duplicate_in_fuzzy_mode(base, reg): + # Non-exact still collapses base/regional, but must dedupe rather than return a track twice. + selected = Tracks.by_language(_mk(base, reg), [base, reg], per_language=1, exact_match=False) + assert len({t.id for t in selected}) == len(selected) + + +def test_exact_lang_three_way_base_and_two_regionals(): + # fr (base, dist 0 to fr-FR) + fr-FR + fr-CA (dist 4): all three must resolve distinctly. + assert _select(["fr", "fr-FR", "fr-CA"], _mk("fr", "fr-FR", "fr-CA")) == [ + ("fr", "fr"), + ("fr-FR", "fr-FR"), + ("fr-CA", "fr-CA"), + ] + # Base request with no bare-tag track falls back to its paradigm regional, without stealing + # the sibling regional (fr -> fr-FR, not fr-CA). + assert _select(["fr", "fr-CA"], _mk("fr-FR", "fr-CA")) == [("fr-FR", "fr-FR"), ("fr-CA", "fr-CA")] + + +@pytest.mark.parametrize("tag", REAL_WORLD_AUDIO_LANGS) +def test_full_language_set_each_tag_resolves_to_itself(tag): + # Against a full real-world language pool, every exact request must return its own track, so + # that no tag captures another's: ar-EG must resolve to ar-EG, never to ar. + pool = _mk(*REAL_WORLD_AUDIO_LANGS) + assert _select([tag], pool) == [(tag, tag)] + + +def test_full_language_set_selects_every_track_once(): + pool = _mk(*REAL_WORLD_AUDIO_LANGS) + selected = Tracks.by_language(pool, REAL_WORLD_AUDIO_LANGS, per_language=1, exact_match=True) + assert len(selected) == len(REAL_WORLD_AUDIO_LANGS) + assert {t.id for t in selected} == set(REAL_WORLD_AUDIO_LANGS) diff --git a/tests/remote/conftest.py b/tests/remote/conftest.py index 9b6b0785..fc90669b 100644 --- a/tests/remote/conftest.py +++ b/tests/remote/conftest.py @@ -86,6 +86,14 @@ def _wait_for_health(url: str, timeout: float = 60.0) -> bool: return False +@pytest.fixture(autouse=True) +def _isolated_job_history(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Redirect job history writes to tmp_path so tests don't append to the server's api_history.jsonl.""" + from unshackle.core.api import download_manager + + monkeypatch.setattr(download_manager, "_history_path", lambda: tmp_path / "api_history.jsonl") + + @pytest.fixture(scope="session") def secret_key(request: pytest.FixtureRequest) -> str: return str(request.config.getoption("--secret-key")) diff --git a/tests/remote/unit/test_download_gates_redaction.py b/tests/remote/unit/test_download_gates_redaction.py index 3d361389..15658475 100644 --- a/tests/remote/unit/test_download_gates_redaction.py +++ b/tests/remote/unit/test_download_gates_redaction.py @@ -9,8 +9,13 @@ import pytest from aiohttp import web from unshackle.core.api import handlers -from unshackle.core.api.download_manager import (DownloadJob, JobStatus, _redact_parameters, _redact_text, - _secret_values) +from unshackle.core.api.download_manager import ( + DownloadJob, + JobStatus, + _redact_parameters, + _redact_text, + _secret_values, +) from unshackle.core.api.errors import APIError, APIErrorCode pytestmark = pytest.mark.unit @@ -116,9 +121,7 @@ async def test_cdm_override_allowed_when_enabled(stub_handler): async def test_cdm_override_allowlist_permits_only_named_device(stub_handler): stub_handler.setattr(handlers.config, "serve", {"cdm_overrides": ["good"]}) - assert isinstance( - await handlers.download_handler({"service": "ATV", "title_id": "t", "cdm": "good"}), web.Response - ) + assert isinstance(await handlers.download_handler({"service": "ATV", "title_id": "t", "cdm": "good"}), web.Response) with pytest.raises(APIError) as ei: await handlers.download_handler({"service": "ATV", "title_id": "t", "cdm": "other"}) assert ei.value.error_code == APIErrorCode.FORBIDDEN diff --git a/tests/remote/unit/test_errors.py b/tests/remote/unit/test_errors.py index 51f37534..e0de7dfc 100644 --- a/tests/remote/unit/test_errors.py +++ b/tests/remote/unit/test_errors.py @@ -6,8 +6,13 @@ import json import pytest -from unshackle.core.api.errors import (APIError, APIErrorCode, build_error_response, categorize_exception, - handle_api_exception) +from unshackle.core.api.errors import ( + APIError, + APIErrorCode, + build_error_response, + categorize_exception, + handle_api_exception, +) pytestmark = pytest.mark.unit diff --git a/tests/remote/unit/test_handlers_serialize.py b/tests/remote/unit/test_handlers_serialize.py index 473b0fe9..7924d886 100644 --- a/tests/remote/unit/test_handlers_serialize.py +++ b/tests/remote/unit/test_handlers_serialize.py @@ -5,9 +5,16 @@ from __future__ import annotations import pytest from langcodes import Language -from unshackle.core.api.handlers import (sanitize_log, serialize_audio_track, serialize_drm, serialize_subtitle_track, - serialize_title, serialize_video_track, validate_download_parameters, - validate_service) +from unshackle.core.api.handlers import ( + sanitize_log, + serialize_audio_track, + serialize_drm, + serialize_subtitle_track, + serialize_title, + serialize_video_track, + validate_download_parameters, + validate_service, +) from unshackle.core.titles.episode import Episode from unshackle.core.titles.movie import Movie from unshackle.core.tracks import Audio, Subtitle, Video diff --git a/tests/remote/unit/test_progress_sink.py b/tests/remote/unit/test_progress_sink.py index 934832e3..7a7e7313 100644 --- a/tests/remote/unit/test_progress_sink.py +++ b/tests/remote/unit/test_progress_sink.py @@ -9,8 +9,12 @@ from __future__ import annotations import pytest -from unshackle.core.api.progress import (DOWNLOAD_PROGRESS_CEILING, build_job_progress_callables, - track_progress_label, track_progress_weight) +from unshackle.core.api.progress import ( + DOWNLOAD_PROGRESS_CEILING, + build_job_progress_callables, + track_progress_label, + track_progress_weight, +) pytestmark = pytest.mark.unit @@ -76,9 +80,7 @@ def test_weighting_makes_video_dominate_progress(): def test_active_tracks_labels_reported_and_cleared_on_done(): updates: list[dict] = [] - cbs = build_job_progress_callables( - [Video(2160, "DV"), Audio("en-US", "2.0")], [_noop, _noop], updates.append - ) + cbs = build_job_progress_callables([Video(2160, "DV"), Audio("en-US", "2.0")], [_noop, _noop], updates.append) cbs[0](total=100, completed=10) # video downloading assert updates[-1]["active_tracks"] == ["video 2160p DV"] diff --git a/tests/remote/unit/test_remote_service_helpers.py b/tests/remote/unit/test_remote_service_helpers.py index 032759f5..81665c37 100644 --- a/tests/remote/unit/test_remote_service_helpers.py +++ b/tests/remote/unit/test_remote_service_helpers.py @@ -6,8 +6,16 @@ from enum import Enum import pytest -from unshackle.core.remote_service import (_build_title, _build_tracks, _deserialize_audio, _deserialize_subtitle, - _deserialize_video, _enum_get, _match_track, _reconstruct_drm) +from unshackle.core.remote_service import ( + _build_title, + _build_tracks, + _deserialize_audio, + _deserialize_subtitle, + _deserialize_video, + _enum_get, + _match_track, + _reconstruct_drm, +) from unshackle.core.titles.episode import Episode from unshackle.core.titles.movie import Movie from unshackle.core.tracks import Audio, Subtitle, Video diff --git a/tests/tracks/test_subtitle_convert.py b/tests/tracks/test_subtitle_convert.py index 1a65f4ff..1629d246 100644 --- a/tests/tracks/test_subtitle_convert.py +++ b/tests/tracks/test_subtitle_convert.py @@ -87,6 +87,18 @@ def test_subtitleedit_ranks_first_when_available(monkeypatch): assert chain[0] == "subtitleedit" +def test_styled_ass_to_ttml_vtt_prefers_pysubs2_over_subtitleedit(monkeypatch): + # Measured: SE flattens ASS inline styling to TTML/VTT; pysubs2 keeps i/b/u. + monkeypatch.setattr(binaries, "SubtitleEdit", "/usr/bin/seconv") + for target in (Codec.TimedTextMarkupLang, Codec.WebVTT): + chain = [b.name for b in sc.resolve_backends(Codec.SubStationAlphav4, target)] + assert chain[0] == "pysubs2", f"expected pysubs2 first for ASS->{target.name}, got {chain}" + assert "subtitleedit" in chain # still a fallback + # ASS->SRT keeps SE first (only backend that keeps colour there). + chain = [b.name for b in sc.resolve_backends(Codec.SubStationAlphav4, Codec.SubRip)] + assert chain[0] == "subtitleedit" + + def test_pin_then_fallback_orders_pin_first(): chain = [b.name for b in sc.resolve_backends(Codec.WebVTT, Codec.SubRip, pin="pysubs2")] assert chain[0] == "pysubs2" diff --git a/tests/tracks/test_subtitle_skip.py b/tests/tracks/test_subtitle_skip.py index daf4d408..02c0c868 100644 --- a/tests/tracks/test_subtitle_skip.py +++ b/tests/tracks/test_subtitle_skip.py @@ -90,8 +90,11 @@ def test_failed_subtitle_does_not_truncate_video_or_audio(): h = Harness(fail_ids={"s-he"}) download_tracks_in_passes( - [video, audio, sub], 4, h.run_one, - skip_subtitle_errors=True, on_subtitle_skipped=h.on_subtitle_skipped, + [video, audio, sub], + 4, + h.run_one, + skip_subtitle_errors=True, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert set(h.completed) == {"v", "a"} # both fatal tracks fully downloaded @@ -105,8 +108,11 @@ def test_good_subtitle_kept_bad_subtitle_skipped(): h = Harness(fail_ids={"s-fr"}) download_tracks_in_passes( - [video, good, bad], 4, h.run_one, - skip_subtitle_errors=True, on_subtitle_skipped=h.on_subtitle_skipped, + [video, good, bad], + 4, + h.run_one, + skip_subtitle_errors=True, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert "s-en" in h.completed # the available subtitle still downloaded @@ -121,8 +127,11 @@ def test_subtitle_failure_stays_fatal_without_flag(): with pytest.raises(RuntimeError): download_tracks_in_passes( - [video, sub], 4, h.run_one, - skip_subtitle_errors=False, on_subtitle_skipped=h.on_subtitle_skipped, + [video, sub], + 4, + h.run_one, + skip_subtitle_errors=False, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert not DOWNLOAD_CANCELLED.is_set() # the finally clears the event even on the fatal path @@ -135,8 +144,11 @@ def test_cancel_event_is_reset_between_titles(): h = Harness(fail_ids=set()) download_tracks_in_passes( - [video, audio], 4, h.run_one, - skip_subtitle_errors=True, on_subtitle_skipped=h.on_subtitle_skipped, + [video, audio], + 4, + h.run_one, + skip_subtitle_errors=True, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert set(h.completed) == {"v", "a"} @@ -150,8 +162,11 @@ def test_cancel_event_cleared_after_failed_final_subtitle(): h = Harness(fail_ids={"s-he"}) download_tracks_in_passes( - [video, sub], 4, h.run_one, - skip_subtitle_errors=True, on_subtitle_skipped=h.on_subtitle_skipped, + [video, sub], + 4, + h.run_one, + skip_subtitle_errors=True, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert not DOWNLOAD_CANCELLED.is_set() # the helper clears it on exit for any later code @@ -164,8 +179,11 @@ def test_all_subtitles_skipped_video_audio_kept(): h = Harness(fail_ids={"s-en", "s-he"}) download_tracks_in_passes( - [video, audio, s1, s2], 4, h.run_one, - skip_subtitle_errors=True, on_subtitle_skipped=h.on_subtitle_skipped, + [video, audio, s1, s2], + 4, + h.run_one, + skip_subtitle_errors=True, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert set(h.completed) == {"v", "a"} # both fatal tracks survived @@ -179,8 +197,11 @@ def test_duplicate_language_subtitles_distinguished_by_id(): h = Harness(fail_ids={"en-forced", "en-sdh"}) download_tracks_in_passes( - [make_video(), forced, sdh], 4, h.run_one, - skip_subtitle_errors=True, on_subtitle_skipped=h.on_subtitle_skipped, + [make_video(), forced, sdh], + 4, + h.run_one, + skip_subtitle_errors=True, + on_subtitle_skipped=h.on_subtitle_skipped, ) assert [t.id for t in h.skipped] == ["en-forced", "en-sdh"] # same language, distinct ids diff --git a/tests/tracks/test_subtitle_sort.py b/tests/tracks/test_subtitle_sort.py new file mode 100644 index 00000000..5251603e --- /dev/null +++ b/tests/tracks/test_subtitle_sort.py @@ -0,0 +1,68 @@ +import pytest + +from unshackle.core.tracks import Subtitle, Tracks + + +def sub(lang: str, forced: bool = False, sdh: bool = False, cc: bool = False) -> Subtitle: + return Subtitle( + url=f"https://example.test/{lang}-{forced}-{sdh}-{cc}", + language=lang, + codec=Subtitle.Codec.SubRip, + forced=forced, + sdh=sdh, + cc=cc, + ) + + +def shape(tracks: Tracks) -> list[tuple[str, str]]: + return [ + (str(s.language), "forced" if s.forced else "sdh" if (s.sdh or s.cc) else "normal") for s in tracks.subtitles + ] + + +@pytest.fixture +def tracks() -> Tracks: + t = Tracks() + t.add([sub("en", sdh=True), sub("en", forced=True), sub("en"), sub("it", forced=True), sub("it")]) + return t + + +def test_default_order_is_forced_normal_sdh(tracks: Tracks) -> None: + tracks.sort_subtitles(by_language=["en"]) + assert shape(tracks) == [ + ("en", "forced"), + ("en", "normal"), + ("en", "sdh"), + ("it", "forced"), + ("it", "normal"), + ] + + +def test_type_priority_overrides_group_order(tracks: Tracks) -> None: + tracks.sort_subtitles(by_language=["en"], type_priority=["normal", "sdh", "forced"]) + assert shape(tracks) == [ + ("en", "normal"), + ("en", "sdh"), + ("en", "forced"), + ("it", "normal"), + ("it", "forced"), + ] + + +def test_type_priority_unlisted_types_fall_to_end(tracks: Tracks) -> None: + tracks.sort_subtitles(by_language=["en"], type_priority=["normal"]) + # unlisted types keep their pre-sort relative order (stable sort) + assert shape(tracks) == [ + ("en", "normal"), + ("en", "sdh"), + ("en", "forced"), + ("it", "normal"), + ("it", "forced"), + ] + + +def test_cc_counts_as_sdh() -> None: + t = Tracks() + t.add([sub("en", cc=True), sub("en")]) + t.sort_subtitles(by_language=["en"], type_priority=["normal", "sdh"]) + assert shape(t) == [("en", "normal"), ("en", "sdh")] diff --git a/tests/tracks/test_track_selection_fixes.py b/tests/tracks/test_track_selection_fixes.py new file mode 100644 index 00000000..f99ad9de --- /dev/null +++ b/tests/tracks/test_track_selection_fixes.py @@ -0,0 +1,138 @@ +"""Unit tests for the audio/video track-selection primitives. + +Cover the pure ``Tracks`` helpers behind the ``dl`` selection layer: + +- ``Tracks.by_resolutions``: the 16:9 canvas fallback must tolerate tracks with + no known width instead of crashing on ``int(None * 9 / 16)``. +- ``Tracks.sort_videos``: resolution outranks bitrate, and an unknown-bitrate + track can still win on resolution. +- ``Tracks.sort_audio``: descriptive-last, Atmos, codec_priority then bitrate; + the first track after sorting is the "best" pick the dl layer selects. +""" + +from __future__ import annotations + +from unshackle.core.tracks import Audio, Tracks, Video + + +def make_video( + track_id: str, + *, + height: int, + bitrate: int | None, + width: int | None = None, + codec: Video.Codec = Video.Codec.HEVC, +) -> Video: + return Video( + id_=track_id, + url=f"https://example.test/{track_id}.m3u8", + language="en", + codec=codec, + range_=Video.Range.SDR, + width=width, + height=height, + bitrate=bitrate, + ) + + +def make_audio( + track_id: str, + *, + codec: Audio.Codec, + bitrate: int, + joc: int | None = None, + descriptive: bool = False, + language: str = "en", +) -> Audio: + return Audio( + id_=track_id, + url=f"https://example.test/{track_id}.m4a", + language=language, + codec=codec, + bitrate=bitrate, + joc=joc, + descriptive=descriptive, + ) + + +# --------------------------------------------------------------------------- +# by_resolutions: width=None must not crash the canvas fallback +# --------------------------------------------------------------------------- + + +def test_by_resolutions_tolerates_width_none_track() -> None: + tracks = Tracks() + tracks.videos = [ + make_video("h1080", height=1080, width=1920, bitrate=8_000_000), # no exact/canvas 720 match + make_video("no-width", height=480, width=None, bitrate=1_000_000), # width-less, must be skipped + make_video("canvas-720", height=406, width=1280, bitrate=3_000_000), # 1280*9/16 == 720 + ] + + tracks.by_resolutions([720]) + + assert [t.id for t in tracks.videos] == ["canvas-720"] + + +# --------------------------------------------------------------------------- +# sort_videos: resolution before bitrate +# --------------------------------------------------------------------------- + + +def test_sort_videos_resolution_outranks_bitrate() -> None: + tracks = Tracks() + tracks.videos = [ + make_video("hi-bitrate-720", height=720, bitrate=10_000_000, width=1280), + make_video("lo-bitrate-2160", height=2160, bitrate=5_000_000, width=3840), + ] + + tracks.sort_videos() + + assert [t.id for t in tracks.videos] == ["lo-bitrate-2160", "hi-bitrate-720"] + + +def test_sort_videos_unknown_bitrate_still_wins_on_resolution() -> None: + tracks = Tracks() + tracks.videos = [ + make_video("known-720", height=720, bitrate=10_000_000, width=1280), + make_video("unknown-1080", height=1080, bitrate=None, width=1920), + ] + + tracks.sort_videos() + + assert [t.id for t in tracks.videos] == ["unknown-1080", "known-720"] + + +# --------------------------------------------------------------------------- +# sort_audio: first track after sort is the "best" the dl layer picks +# --------------------------------------------------------------------------- + + +def test_sort_audio_atmos_ranks_first() -> None: + tracks = Tracks() + tracks.audio = [ + make_audio("ec3-320", codec=Audio.Codec.EC3, bitrate=320_000), + make_audio("ac3-640", codec=Audio.Codec.AC3, bitrate=640_000), + make_audio("ec3-atmos", codec=Audio.Codec.EC3, bitrate=768_000, joc=16), + make_audio("ec3-desc", codec=Audio.Codec.EC3, bitrate=768_000, descriptive=True), + ] + + tracks.sort_audio(codec_priority=["EC3", "AC3"]) + + best = Tracks.by_language(tracks.audio, ["en"])[0] + assert best.id == "ec3-atmos" + # descriptive always sorts last + assert tracks.audio[-1].id == "ec3-desc" + + +def test_sort_audio_codec_priority_beats_bitrate() -> None: + tracks = Tracks() + tracks.audio = [ + make_audio("ac3-640", codec=Audio.Codec.AC3, bitrate=640_000), + make_audio("ec3-320", codec=Audio.Codec.EC3, bitrate=320_000), + ] + + tracks.sort_audio(codec_priority=["EC3", "AC3"]) + + best = Tracks.by_language(tracks.audio, ["en"])[0] + # EC3 is prioritized even though the AC3 track has the higher bitrate + assert best.id == "ec3-320" diff --git a/tests/tracks/test_webvtt_merge.py b/tests/tracks/test_webvtt_merge.py new file mode 100644 index 00000000..07258e97 --- /dev/null +++ b/tests/tracks/test_webvtt_merge.py @@ -0,0 +1,109 @@ +"""Regression tests for segmented-WebVTT merging (``core/utils/webvtt.py``). + +The merge used to round-trip cues through pycaption's reader/writer (and a pysubs2 +pre-normalization step), both of which silently dropped inline formatting despite +``preserve_formatting=True``. It is now a pure string-level splice: cue +timing lines, settings, and payload are kept verbatim, so span tags +(``<i>``/``<u>``/``<b>``), ASS-style overrides (``{\\an8}``), and entities all +survive, while duplicate headers collapse and boundary-duplicate cues dedupe. +""" + +from __future__ import annotations + +from unshackle.core.utils.webvtt import merge_segmented_webvtt + +# Two segments, each a full WebVTT document with its own X-TIMESTAMP-MAP header, +# as they arrive concatenated from a DASH/HLS subtitle stream. The first cue carries +# span tags, an {\an8} override, a cue setting, and a literal < entity. +SEGMENTED_VTT = ( + "WEBVTT\n" + "X-TIMESTAMP-MAP=MPEGTS:900000,LOCAL:00:00:00.000\n\n" + "00:00:01.000 --> 00:00:03.000 line:10% align:center\n" + "<i>Hello</i> {\\an8}world <u>und</u> <b>bold</b> 5 < 6\n\n" + "WEBVTT\n" + "X-TIMESTAMP-MAP=MPEGTS:1800000,LOCAL:00:00:00.000\n\n" + "00:00:04.000 --> 00:00:06.000\n" + "<i>Second</i> cue\n" +) + + +def test_merge_preserves_inline_formatting(): + out = merge_segmented_webvtt(SEGMENTED_VTT, segment_durations=[900000, 900000], timescale=1) + for tag in ("<i>", "</i>", "<u>", "</u>", "<b>", "</b>", "{\\an8}"): + assert tag in out, f"{tag!r} was stripped from merged output:\n{out}" + + +def test_merge_preserves_cue_settings_and_entities(): + out = merge_segmented_webvtt(SEGMENTED_VTT, segment_durations=[900000, 900000], timescale=1) + assert "line:10% align:center" in out # cue positioning survives + assert "<" in out # a literal entity is not turned into a stray tag + + +def test_merge_collapses_headers_into_single_document(): + out = merge_segmented_webvtt(SEGMENTED_VTT, segment_durations=[900000, 900000], timescale=1) + assert out.count("WEBVTT") == 1 + assert "X-TIMESTAMP-MAP" not in out + + +def test_merge_dedupes_boundary_duplicate_cues(): + # A cue that spans a segment boundary is emitted in both segments; the merge + # should collapse the adjacent identical pair into one. + dup = SEGMENTED_VTT + ( + "\nWEBVTT\n" + "X-TIMESTAMP-MAP=MPEGTS:2700000,LOCAL:00:00:00.000\n\n" + "00:00:04.000 --> 00:00:06.000\n" + "<i>Second</i> cue\n" + ) + out = merge_segmented_webvtt(dup, segment_durations=[900000, 900000, 900000], timescale=1) + assert out.count("<i>Second</i> cue") == 1, f"boundary duplicate not deduped:\n{out}" + + +def test_merge_keeps_timing_lines_verbatim(): + # String-level merge must not re-render timestamps (pycaption shortened + # 00:00:01.000 -> 00:01.000) and must keep the settings tail untouched. + out = merge_segmented_webvtt(SEGMENTED_VTT, segment_durations=[900000, 900000], timescale=1) + assert "00:00:01.000 --> 00:00:03.000 line:10% align:center" in out + assert "00:00:04.000 --> 00:00:06.000" in out + + +def test_merge_splices_triple_duplicate_into_one(): + # A cue spanning two boundaries appears in three consecutive segments; the + # kept cue must be extended each time (latent bug: extending the dropped one). + vtt = ( + "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nSame cue\n\n" + "WEBVTT\n\n00:00:02.000 --> 00:00:03.000\nSame cue\n\n" + "WEBVTT\n\n00:00:03.000 --> 00:00:04.000\nSame cue\n" + ) + out = merge_segmented_webvtt(vtt) + assert out.count("Same cue") == 1 + assert "00:00:01.000 --> 00:00:04.000" in out + + +def test_merge_drops_cue_ids_and_notes_keeps_style(): + vtt = ( + "WEBVTT\n\n" + "STYLE\n::cue { color: red }\n\n" + "NOTE a comment\n\n" + "some-cue-id-42\n00:00:01.000 --> 00:00:02.000\nHello\n" + ) + out = merge_segmented_webvtt(vtt) + assert "Hello" in out + assert "some-cue-id-42" not in out + assert "NOTE" not in out + assert "STYLE\n::cue { color: red }" in out # kept; players without STYLE support ignore it + + +def test_merge_dedupes_repeated_style_blocks_before_cues(): + # Every segment repeats the same STYLE block; keep exactly one, before the first cue. + style = "WEBVTT\n\nSTYLE\n::cue { color: red }\n\n" + vtt = style + "00:00:01.000 --> 00:00:02.000\nOne\n\n" + style + "00:00:03.000 --> 00:00:04.000\nTwo\n" + out = merge_segmented_webvtt(vtt) + assert out.count("STYLE") == 1 + assert out.index("STYLE") < out.index("-->") + + +def test_merge_skips_malformed_timing_line(): + vtt = "WEBVTT\n\ngarbage --> not-a-time\nBroken\n\n00:00:01.000 --> 00:00:02.000\nGood\n" + out = merge_segmented_webvtt(vtt) + assert "Good" in out + assert "Broken" not in out diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index bbfb64ff..f47c5ddf 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -29,10 +29,9 @@ from click.core import ParameterSource from langcodes import Language from pymediainfo import MediaInfo from rich.console import Group -from rich.live import Live from rich.padding import Padding from rich.panel import Panel -from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn +from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn from rich.rule import Rule from rich.spinner import Spinner from rich.table import Table @@ -43,13 +42,21 @@ from unshackle.core import binaries, providers from unshackle.core.cdm import DecryptLabsRemoteCDM from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm from unshackle.core.config import config, resolve_cdm_name, resolve_decryption -from unshackle.core.console import console +from unshackle.core.console import GradientPulseBarColumn, SyncLive, console from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings from unshackle.core.credential import Credential from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevine from unshackle.core.events import events -from unshackle.core.music import (MusicAudioIntegrityError, MusicMetadataResult, MusicPlanner, MusicRenderer, - file_md5, verify_music_audio, write_music_manifest, write_music_metadata) +from unshackle.core.music import ( + MusicAudioIntegrityError, + MusicMetadataResult, + MusicPlanner, + MusicRenderer, + file_md5, + verify_music_audio, + write_music_manifest, + write_music_metadata, +) from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN from unshackle.core.service import Service from unshackle.core.services import Services @@ -60,14 +67,30 @@ from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.dv_fixup import apply_dv_fixup from unshackle.core.tracks.hybrid import Hybrid -from unshackle.core.utilities import (find_font_with_fallbacks, find_missing_langs, get_debug_logger, - get_system_fonts, init_debug_logger, is_close_match, log_event, - suggest_font_packages, time_elapsed_since) +from unshackle.core.utilities import ( + find_font_with_fallbacks, + find_missing_langs, + get_debug_logger, + get_system_fonts, + init_debug_logger, + is_close_match, + log_event, + suggest_font_packages, + time_elapsed_since, +) from unshackle.core.utils import tags from unshackle.core.utils.bitrate import apply_real_bitrates -from unshackle.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, - SLOW_DELAY_RANGE, ContextData, MultipleChoice, MultipleVideoCodecChoice, - SubtitleCodecChoice) +from unshackle.core.utils.click_types import ( + AUDIO_CODEC_LIST, + LANGUAGE_RANGE, + QUALITY_LIST, + SEASON_RANGE, + SLOW_DELAY_RANGE, + ContextData, + MultipleChoice, + MultipleVideoCodecChoice, + SubtitleCodecChoice, +) from unshackle.core.utils.collections import ci_get, merge_dict from unshackle.core.utils.selector import select_multiple from unshackle.core.utils.subprocess import ffprobe @@ -860,19 +883,43 @@ class dl: try: if name == "shaka_packager": r = subprocess.run( - [str(binary), "--version"], capture_output=True, text=True, timeout=5 + [str(binary), "--version"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, ) version = (r.stdout or r.stderr or "").strip() elif name in ("ffmpeg", "ffprobe"): - r = subprocess.run([str(binary), "-version"], capture_output=True, text=True, timeout=5) + r = subprocess.run( + [str(binary), "-version"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + ) version = (r.stdout or "").split("\n")[0].strip() elif name == "mkvmerge": r = subprocess.run( - [str(binary), "--version"], capture_output=True, text=True, timeout=5 + [str(binary), "--version"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, ) version = (r.stdout or "").strip() elif name == "mp4decrypt": - r = subprocess.run([str(binary)], capture_output=True, text=True, timeout=5) + r = subprocess.run( + [str(binary)], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + ) output = (r.stdout or "") + (r.stderr or "") lines = [line.strip() for line in output.split("\n") if line.strip()] version = " | ".join(lines[:2]) if lines else None @@ -1059,7 +1106,7 @@ class dl: self.proxy_providers.append(Basic(**config.proxy_providers["basic"])) # ExpressVPN/ProtonVPN auto-load when their default cookie file exists (no yaml needed) expressvpn = ExpressVPN(**(config.proxy_providers.get("expressvpn") or {})) - if config.proxy_providers.get("expressvpn") or expressvpn.cookie_path.is_file(): + if config.proxy_providers.get("expressvpn") or expressvpn.cache_path.is_file(): self.proxy_providers.append(expressvpn) if config.proxy_providers.get("nordvpn"): self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) @@ -1099,7 +1146,11 @@ class dl: with console.status(status_msg, spinner="dots"): if requested_provider: proxy_provider = next( - (x for x in self.proxy_providers if x.__class__.__name__.lower() == requested_provider), + ( + x + for x in self.proxy_providers + if x.__class__.__name__.lower() == requested_provider.lower() + ), None, ) if not proxy_provider: @@ -1573,6 +1624,53 @@ class dl: latest_episode_id = f"{latest_ep.season}x{latest_ep.number}" self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}") + def select_best_audio( + audio: list[Audio], + languages: list[str], + acodec: list[Audio.Codec], + audio_description: bool, + exact_lang: bool, + ) -> list[Audio]: + """Pick the best audio per (language, codec) group from a sort_audio-ordered list. + + "Best" is the first track of each order-preserving candidate filter, so the + sort_audio ranking (descriptive-last, atmos, codec_priority, bitrate) decides the + winner. With audio_description both the best standard and best descriptive track per + group are kept. "best"/"all" tokens select every language present in sorted order. + """ + select_all = any(tok in ("best", "all") for tok in languages) + if select_all: + target_langs: list[Any] = list(dict.fromkeys(t.language for t in audio)) + else: + target_langs = languages + codecs: list[Optional[Audio.Codec]] = list(acodec) if len(acodec) > 1 else [None] + + selected: list[Audio] = [] + for language in target_langs: + for codec in codecs: + if select_all: + candidates = [ + t for t in audio if t.language == language and (codec is None or t.codec == codec) + ] + else: + pool = [t for t in audio if codec is None or t.codec == codec] + candidates = Tracks.by_language(pool, [language], per_language=0, exact_match=exact_lang) + if not candidates: + continue + if audio_description: + standards = [t for t in candidates if not t.descriptive] + if standards: + selected.append(standards[0]) + descs = [t for t in candidates if t.descriptive] + if descs: + selected.append(descs[0]) + else: + selected.append(candidates[0]) + # Built via per-tag by_language calls, so overlapping tags can append + # the same track across iterations; dedupe by id. + seen_ids: set[str] = set() + return [t for t in selected if not (t.id in seen_ids or seen_ids.add(t.id))] + music_group_download = ( bool(music_titles) and getattr(service, "GROUP_AUDIO_DOWNLOADS", False) @@ -1616,7 +1714,9 @@ class dl: self.log.error(f"No audio tracks matching codecs for {song.name}: {codec_names}") sys.exit(1) if channels: - song.tracks.select_audio(lambda x: math.ceil(x.channels) == math.ceil(channels)) + song.tracks.select_audio( + lambda x: bool(x.channels and math.ceil(x.channels) == math.ceil(channels)) + ) if not song.tracks.audio: self.log.error(f"There's no {channels} Audio Track for {song.name}...") sys.exit(1) @@ -1646,9 +1746,7 @@ class dl: for language in audio_languages: if language == "orig": if song.language: - orig_lang = ( - str(song.language) if hasattr(song.language, "__str__") else song.language - ) + orig_lang = str(song.language) if orig_lang not in processed_lang: processed_lang.append(orig_lang) else: @@ -1656,16 +1754,12 @@ class dl: elif language not in processed_lang: processed_lang.append(language) - if "best" not in processed_lang and "all" not in processed_lang: - song.tracks.audio = song.tracks.by_language( - song.tracks.audio, - processed_lang, - per_language=1, - exact_match=exact_lang, - ) - if not song.tracks.audio: - self.log.error(f"There's no {processed_lang} Audio Track for {song.name}...") - sys.exit(1) + song.tracks.audio = select_best_audio( + song.tracks.audio, processed_lang, acodec, audio_description, exact_lang + ) + if not song.tracks.audio: + self.log.error(f"There's no {processed_lang} Audio Track for {song.name}...") + sys.exit(1) self.log.debug("Getting Tracks") tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..." @@ -1680,11 +1774,20 @@ class dl: song.tracks.add(service.get_tracks(song), warn_only=True) song.tracks.chapters = service.get_chapters(song) - song.tracks.sort_audio(by_language=a_lang or lang) + song.tracks.sort_audio( + by_language=a_lang or lang, + codec_priority=config.audio.get("codec_priority"), + ) select_music_audio(song) if not song.tracks.audio: self.log.error(f"No audio tracks returned for {song.name}.") sys.exit(1) + if len(song.tracks.audio) > 1: + # group audio mode is one-track-per-song by design + self.log.warning( + f"Group audio downloads take one track per song, keeping best for {song.name}" + ) + song.tracks.audio = song.tracks.audio[:1] music_tree = Tree( f"[repr.number]{len(titles)}[/] {'Track' if len(titles) == 1 else 'Tracks'}", @@ -1694,7 +1797,7 @@ class dl: track = song.tracks.audio[0] progress = Progress( SpinnerColumn(finished_text=""), - BarColumn(), + GradientPulseBarColumn(), " | ", TimeRemainingColumn(compact=True, elapsed_when_finished=True), " | ", @@ -1706,17 +1809,25 @@ class dl: state = {"total": 100.0} def update_track_progress( - task_id: int = task, + task_id: TaskID = task, _state: dict[str, float] = state, _progress: Progress = progress, - **kwargs, + **kwargs: Any, ) -> None: - if "total" in kwargs and kwargs["total"] is not None: - _state["total"] = kwargs["total"] + if "total" in kwargs: + if kwargs["total"] is None: + # Progress.update() ignores total=None; an un-started task pulses + del kwargs["total"] + _progress.reset(task_id, start=False) + else: + _state["total"] = kwargs["total"] + _progress.start_task(task_id) downloaded_state = kwargs.get("downloaded") if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}: kwargs["completed"] = _state["total"] + kwargs["total"] = _state["total"] + _progress.start_task(task_id) _progress.update(task_id=task_id, **kwargs) track_table = Table.grid() @@ -1734,7 +1845,7 @@ class dl: download_table.add_row(music_tree) try: - with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): + with SyncLive(Padding(download_table, (1, 5)), console=console, refresh_per_second=20): with ThreadPoolExecutor(downloads) as pool: download_futures = [ pool.submit( @@ -2081,7 +2192,7 @@ class dl: if slow and i != 0: delay = random.randint(slow[0], slow[1]) spinner = Spinner("dots", text=f"Delaying by {delay} seconds...") - with Live(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True): + with SyncLive(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True): for remaining in range(delay, 0, -1): spinner.update(text=f"Delaying by {remaining} seconds...") time.sleep(1) @@ -2094,33 +2205,6 @@ class dl: events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked) events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex) - if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES: - console.log("Skipping subtitles - service does not support subtitle downloads") - no_subs = True - s_lang = None - title.tracks.subtitles = [] - elif no_subs: - console.log("Skipped subtitles as --no-subs was used...") - s_lang = None - title.tracks.subtitles = [] - - if no_video: - console.log("Skipped video as --no-video was used...") - v_lang = None - title.tracks.videos = [] - - if no_audio: - console.log("Skipped audio as --no-audio was used...") - a_lang = None - title.tracks.audio = [] - - if no_chapters: - console.log("Skipped chapters as --no-chapters was used...") - title.tracks.chapters = [] - - if no_proxy_download and any(service.session.proxies.values()): - console.log("Bypassing proxy for downloads as --no-proxy-download was used...") - tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..." with console.status(tracks_label, spinner="dots"): try: @@ -2221,7 +2305,7 @@ class dl: for language in video_sort_lang: if language == "orig": if title.language: - orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + orig_lang = str(title.language) if orig_lang not in processed_video_sort_lang: processed_video_sort_lang.append(orig_lang) else: @@ -2233,7 +2317,7 @@ class dl: for language in audio_sort_lang: if language == "orig": if title.language: - orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + orig_lang = str(title.language) if orig_lang not in processed_audio_sort_lang: processed_audio_sort_lang.append(orig_lang) else: @@ -2245,17 +2329,65 @@ class dl: by_language=processed_audio_sort_lang, codec_priority=config.audio.get("codec_priority"), ) - title.tracks.sort_subtitles(by_language=s_lang) + title.tracks.sort_subtitles(by_language=s_lang, type_priority=config.subtitle.get("type_priority")) if list_: available_tracks, _ = title.tracks.tree() console.print(Padding(Panel(available_tracks, title="Available Tracks"), (0, 5))) continue + # Determine which tracks to keep + keep_videos = True + keep_audio = True + keep_subtitles = True + keep_chapters = True + + if video_only or audio_only or subs_only or chapters_only: + keep_videos = video_only + keep_audio = audio_only + keep_subtitles = subs_only + keep_chapters = chapters_only + + if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES: + console.log("Skipping subtitles - service does not support subtitle downloads") + keep_subtitles = False + s_lang = None + elif no_subs: + console.log("Skipped subtitles as --no-subs was used...") + keep_subtitles = False + s_lang = None + + if no_video: + console.log("Skipped video as --no-video was used...") + keep_videos = False + v_lang = None + + if no_audio: + console.log("Skipped audio as --no-audio was used...") + keep_audio = False + a_lang = None + + if no_chapters: + console.log("Skipped chapters as --no-chapters was used...") + keep_chapters = False + + if no_proxy_download and any(service.session.proxies.values()): + console.log("Bypassing proxy for downloads as --no-proxy-download was used...") + + # Clear unwanted tracks + if not keep_videos: + title.tracks.videos = [] + if not keep_audio: + title.tracks.audio = [] + if not keep_subtitles: + title.tracks.subtitles = [] + if not keep_chapters: + title.tracks.chapters = [] + with console.status("Selecting tracks...", spinner="dots"): if isinstance(title, (Movie, Episode)): # filter video tracks - if vcodec: + if keep_videos and vcodec: title.tracks.select_video(lambda x: x.codec in vcodec) missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)] for codec in missing_codecs: @@ -2264,7 +2396,7 @@ class dl: self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...") sys.exit(1) - if range_: + if keep_videos and range_: # Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks if Video.Range.HYBRID not in range_: title.tracks.select_video(lambda x: x.range in range_) @@ -2275,7 +2407,7 @@ class dl: self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...") sys.exit(1) - if vbitrate: + if keep_videos and vbitrate: if any(r == Video.Range.HYBRID for r in range_): # In HYBRID mode, only apply bitrate filter to non-DV tracks # DV tracks are kept regardless since they're only used for RPU metadata @@ -2291,7 +2423,7 @@ class dl: self.log.error(f"There's no {vbitrate}kbps Video Track...") sys.exit(1) - if vbitrate_min is not None and vbitrate_max is not None: + if keep_videos and vbitrate_min is not None and vbitrate_max is not None: title.tracks.select_video( lambda x: x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max ) @@ -2304,14 +2436,12 @@ class dl: video_multi_lang = ( "best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1 ) - if video_languages and "all" not in video_languages: + if keep_videos and video_languages and "all" not in video_languages: processed_video_lang = [] for language in video_languages: if language == "orig": if title.language: - orig_lang = ( - str(title.language) if hasattr(title.language, "__str__") else title.language - ) + orig_lang = str(title.language) if orig_lang not in processed_video_lang: processed_video_lang.append(orig_lang) else: @@ -2321,6 +2451,27 @@ class dl: else: if language not in processed_video_lang: processed_video_lang.append(language) + missing_v_langs = find_missing_langs( + processed_video_lang, + [v.language for v in title.tracks.videos], + exact=exact_lang, + ) + if missing_v_langs: + missing_str = ", ".join(missing_v_langs) + if best_available: + remaining_v_langs = [tok for tok in processed_video_lang if tok not in missing_v_langs] + if remaining_v_langs: + self.log.warning( + f"{missing_str} not found in video tracks, " + f"continuing with: {', '.join(remaining_v_langs)}" + ) + processed_video_lang = remaining_v_langs + else: + self.log.error(f"{missing_str} not found in video tracks and no fallback available") + sys.exit(1) + else: + self.log.error(missing_str + " not found in video tracks") + sys.exit(1) title.tracks.videos = title.tracks.by_language( title.tracks.videos, processed_video_lang, exact_match=exact_lang ) @@ -2330,37 +2481,64 @@ class dl: has_hybrid = any(r == Video.Range.HYBRID for r in range_) non_hybrid_ranges = [r for r in range_ if r != Video.Range.HYBRID] - if quality: + effective_quality = quality + if keep_videos and quality: missing_resolutions = [] if has_hybrid: hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos( title.tracks.videos, non_hybrid_ranges ) + # only ranges that can actually be delivered count towards "present" + deliverable_pool = hybrid_candidate_tracks + [ + v for v in non_hybrid_tracks if v.range in non_hybrid_ranges + ] + for resolution in quality: + if any(v.height == resolution for v in deliverable_pool): + continue + if any(v.width and int(v.width * 9 / 16) == resolution for v in deliverable_pool): + continue + missing_resolutions.append(resolution) - hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality, worst=worst) - hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) - - if non_hybrid_ranges and non_hybrid_tracks: - non_hybrid_selected = [ - v - for v in non_hybrid_tracks - if any(v.height == res or int(v.width * (9 / 16)) == res for res in quality) - ] - title.tracks.videos = Tracks.merge_video_selections( - hybrid_selected, non_hybrid_selected - ) + if len(missing_resolutions) == len(quality) and best_available: + # every requested resolution missing: leave the pool untouched so the + # best-available hybrid pass below selects the top resolution instead + effective_quality = [] else: - title.tracks.videos = hybrid_selected + hybrid_filter = title.tracks.select_hybrid( + hybrid_candidate_tracks, quality, worst=worst + ) + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + + if non_hybrid_ranges and non_hybrid_tracks: + non_hybrid_selected = [ + v + for v in non_hybrid_tracks + if any( + v.height == res or (v.width and int(v.width * (9 / 16)) == res) + for res in quality + ) + ] + title.tracks.videos = Tracks.merge_video_selections( + hybrid_selected, non_hybrid_selected + ) + else: + title.tracks.videos = hybrid_selected else: + pre_quality_videos = list(title.tracks.videos) title.tracks.by_resolutions(quality) for resolution in quality: if any(v.height == resolution for v in title.tracks.videos): continue - if any(int(v.width * 9 / 16) == resolution for v in title.tracks.videos): + if any(v.width and int(v.width * 9 / 16) == resolution for v in title.tracks.videos): continue missing_resolutions.append(resolution) + if len(missing_resolutions) == len(quality) and best_available: + # every requested resolution missing: fall back to the best available + title.tracks.videos = pre_quality_videos + effective_quality = [] + if missing_resolutions: res_list = "" if len(missing_resolutions) > 1: @@ -2378,13 +2556,13 @@ class dl: # choose best track by range and quality pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else [] - if has_hybrid: + if keep_videos and has_hybrid: # Apply hybrid selection for HYBRID tracks hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos( title.tracks.videos, non_hybrid_ranges ) - if not quality: + if not effective_quality: best_resolution = max((v.height for v in hybrid_candidate_tracks), default=None) if best_resolution: hybrid_filter = title.tracks.select_hybrid( @@ -2394,7 +2572,9 @@ class dl: else: hybrid_selected = [] else: - hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality, worst=worst) + hybrid_filter = title.tracks.select_hybrid( + hybrid_candidate_tracks, effective_quality, worst=worst + ) hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) # For non-hybrid ranges, apply Cartesian product selection @@ -2406,7 +2586,7 @@ class dl: else: non_hybrid_langs = [None] for resolution, color_range, codec, vlang in product( - quality or [None], non_hybrid_ranges, vcodec or [None], non_hybrid_langs + effective_quality or [None], non_hybrid_ranges, vcodec or [None], non_hybrid_langs ): candidates = [ t @@ -2414,7 +2594,7 @@ class dl: if ( not resolution or t.height == resolution - or int(t.width * (9 / 16)) == resolution + or (t.width and int(t.width * (9 / 16)) == resolution) ) and (not color_range or t.range == color_range) and (not codec or t.codec == codec) @@ -2430,19 +2610,23 @@ class dl: # the lowest DV) so the standalone mux loop skips them. Tracks also # picked as explicit deliverables stay unflagged. Tracks.flag_hybrid_ingredients(hybrid_selected, non_hybrid_selected) - else: + elif keep_videos: selected_videos: list[Video] = [] if video_multi_lang: unique_video_langs = list(dict.fromkeys(str(v.language) for v in title.tracks.videos)) else: unique_video_langs = [None] for resolution, color_range, codec, vlang in product( - quality or [None], range_ or [None], vcodec or [None], unique_video_langs + effective_quality or [None], range_ or [None], vcodec or [None], unique_video_langs ): candidates = [ t for t in title.tracks.videos - if (not resolution or t.height == resolution or int(t.width * (9 / 16)) == resolution) + if ( + not resolution + or t.height == resolution + or (t.width and int(t.width * (9 / 16)) == resolution) + ) and (not color_range or t.range == color_range) and (not codec or t.codec == codec) and (vlang is None or str(t.language) == vlang) @@ -2453,7 +2637,7 @@ class dl: title.tracks.videos = selected_videos # validate hybrid mode requirements - if any(r == Video.Range.HYBRID for r in range_): + if keep_videos and any(r == Video.Range.HYBRID for r in range_): base_tracks = [ v for v in title.tracks.videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P) ] @@ -2493,7 +2677,7 @@ class dl: fallback_langs = [None] fallback_selected: list[Video] = [] for resolution, color_range, codec, vlang in product( - quality or [None], other_ranges, vcodec or [None], fallback_langs + effective_quality or [None], other_ranges, vcodec or [None], fallback_langs ): candidates = [ t @@ -2501,7 +2685,7 @@ class dl: if ( not resolution or t.height == resolution - or int(t.width * (9 / 16)) == resolution + or (t.width and int(t.width * (9 / 16)) == resolution) ) and (not color_range or t.range == color_range) and (not codec or t.codec == codec) @@ -2517,7 +2701,7 @@ class dl: sys.exit(1) # filter subtitle tracks - if require_subs: + if keep_subtitles and require_subs: missing_langs = [ lang for lang in require_subs @@ -2531,7 +2715,7 @@ class dl: self.log.info( f"Required languages found ({', '.join(require_subs)}), downloading all available subtitles" ) - elif s_lang and "all" not in s_lang: + elif keep_subtitles and s_lang and "all" not in s_lang: from unshackle.core.utilities import is_exact_match match_func = is_exact_match if exact_lang else is_close_match @@ -2565,12 +2749,12 @@ class dl: self.log.error(f"There's no {s_lang} Subtitle Track...") sys.exit(1) - if not forced_subs: + if keep_subtitles and not forced_subs: title.tracks.select_subtitles(lambda x: not x.forced) # filter audio tracks # might have no audio tracks if part of the video, e.g. transport stream hls - if len(title.tracks.audio) > 0: + if keep_audio and len(title.tracks.audio) > 0: if not audio_description: title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio if acodec: @@ -2580,7 +2764,9 @@ class dl: self.log.error(f"No audio tracks matching codecs: {codec_names}") sys.exit(1) if channels: - title.tracks.select_audio(lambda x: math.ceil(x.channels) == math.ceil(channels)) + title.tracks.select_audio( + lambda x: bool(x.channels and math.ceil(x.channels) == math.ceil(channels)) + ) if not title.tracks.audio: self.log.error(f"There's no {channels} Audio Track...") sys.exit(1) @@ -2607,9 +2793,7 @@ class dl: for language in audio_languages: if language == "orig": if title.language: - orig_lang = ( - str(title.language) if hasattr(title.language, "__str__") else title.language - ) + orig_lang = str(title.language) if orig_lang not in processed_lang: processed_lang.append(orig_lang) else: @@ -2644,138 +2828,26 @@ class dl: self.log.error(missing_str + " not found in audio tracks") sys.exit(1) - if "best" in processed_lang or "all" in processed_lang: - unique_languages = {track.language for track in title.tracks.audio} - selected_audio = [] - best_key = lambda x: (bool(x.atmos), x.bitrate or 0) # noqa: E731 - for language in unique_languages: - codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] - for codec in codecs_to_check: - base_candidates = [ - t - for t in title.tracks.audio - if t.language == language and (codec is None or t.codec == codec) - ] - if not base_candidates: - continue - if audio_description: - standards = [t for t in base_candidates if not t.descriptive] - if standards: - selected_audio.append(max(standards, key=best_key)) - descs = [t for t in base_candidates if t.descriptive] - if descs: - selected_audio.append(max(descs, key=best_key)) - else: - selected_audio.append(max(base_candidates, key=best_key)) - title.tracks.audio = selected_audio - else: - # If multiple codecs were explicitly requested, pick the best track per codec per - # requested language instead of selecting *all* bitrate variants of a codec. - if acodec and len(acodec) > 1: - selected_audio: list[Audio] = [] - best_key = lambda x: (bool(x.atmos), x.bitrate or 0) # noqa: E731 + title.tracks.audio = select_best_audio( + title.tracks.audio, processed_lang, acodec, audio_description, exact_lang + ) + if not title.tracks.audio: + self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") + sys.exit(1) - for language in processed_lang: - for codec in acodec: - codec_tracks = [a for a in title.tracks.audio if a.codec == codec] - if not codec_tracks: - continue + # Reconstruct track set to only include kept tracks + kept_tracks = [] + if keep_videos: + kept_tracks.extend(title.tracks.videos) + if keep_audio: + kept_tracks.extend(title.tracks.audio) + if keep_subtitles: + kept_tracks.extend(title.tracks.subtitles) + if keep_chapters: + kept_tracks.extend(title.tracks.chapters) + kept_tracks.extend(title.tracks.attachments) - candidates = title.tracks.by_language( - codec_tracks, [language], per_language=0, exact_match=exact_lang - ) - if not candidates: - continue - - if audio_description: - standards = [t for t in candidates if not t.descriptive] - if standards: - selected_audio.append(max(standards, key=best_key)) - descs = [t for t in candidates if t.descriptive] - if descs: - selected_audio.append(max(descs, key=best_key)) - else: - selected_audio.append(max(candidates, key=best_key)) - - title.tracks.audio = selected_audio - else: - per_language = 1 - if audio_description: - standard_audio = [a for a in title.tracks.audio if not a.descriptive] - selected_standards = title.tracks.by_language( - standard_audio, - processed_lang, - per_language=per_language, - exact_match=exact_lang, - ) - desc_audio = [a for a in title.tracks.audio if a.descriptive] - # Include all descriptive tracks for the requested languages. - selected_descs = title.tracks.by_language( - desc_audio, processed_lang, per_language=0, exact_match=exact_lang - ) - title.tracks.audio = selected_standards + selected_descs - else: - title.tracks.audio = title.tracks.by_language( - title.tracks.audio, - processed_lang, - per_language=per_language, - exact_match=exact_lang, - ) - if not title.tracks.audio: - self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") - sys.exit(1) - - if ( - video_only - or audio_only - or subs_only - or chapters_only - or no_subs - or no_audio - or no_chapters - or no_video - ): - keep_videos = False - keep_audio = False - keep_subtitles = False - keep_chapters = False - - if video_only or audio_only or subs_only or chapters_only: - if video_only: - keep_videos = True - if audio_only: - keep_audio = True - if subs_only: - keep_subtitles = True - if chapters_only: - keep_chapters = True - else: - keep_videos = True - keep_audio = True - keep_subtitles = True - keep_chapters = True - - if no_subs: - keep_subtitles = False - if no_audio: - keep_audio = False - if no_chapters: - keep_chapters = False - if no_video: - keep_videos = False - - kept_tracks = [] - if keep_videos: - kept_tracks.extend(title.tracks.videos) - if keep_audio: - kept_tracks.extend(title.tracks.audio) - if keep_subtitles: - kept_tracks.extend(title.tracks.subtitles) - if keep_chapters: - kept_tracks.extend(title.tracks.chapters) - kept_tracks.extend(title.tracks.attachments) - - title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url) + title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url) selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) @@ -2841,7 +2913,7 @@ class dl: dl_start_time = time.time() try: - with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): + with SyncLive(Padding(download_table, (1, 5)), console=console, refresh_per_second=20): def download_track(track: AnyTrack, i: int) -> None: track.download( @@ -3093,9 +3165,13 @@ class dl: ) with console.status("Repackaging tracks with FFMPEG..."): has_repacked = False + # Video tracks whose VUI was folded into the repack pass; skipped in the VUI loop below. + vui_folded = set() for track in title.tracks: if track.needs_repack: - track.repackage() + bsf = track.vui_bsf() if isinstance(track, Video) else None + if track.repackage(bsf_v=bsf): + vui_folded.add(id(track)) has_repacked = True events.emit(events.Types.TRACK_REPACKED, track=track) if has_repacked: @@ -3104,6 +3180,8 @@ class dl: with console.status("Normalizing video VUI..."): for track in title.tracks.videos: + if id(track) in vui_folded: + continue try: track.normalize_vui() except Exception as e: # noqa: BLE001 @@ -3122,7 +3200,7 @@ class dl: progress = Progress( TextColumn("[progress.description]{task.description}"), SpinnerColumn(finished_text=""), - BarColumn(), + GradientPulseBarColumn(), "•", TimeRemainingColumn(compact=True, elapsed_when_finished=True), "•", @@ -3270,10 +3348,10 @@ class dl: {"phase": "muxing", "progress": 96.0, "status": "downloading", "active_tracks": []} ) try: - with Live(Padding(progress, (0, 5, 1, 5)), console=console): + with SyncLive(Padding(progress, (0, 5, 1, 5)), console=console, refresh_per_second=20): mux_index = 0 for task_id, task_tracks, audio_codec in multiplex_tasks: - progress.start_task(task_id) # TODO: Needed? + progress.start_task(task_id) audio_expected = not video_only and not no_audio muxed_path, return_code, errors = task_tracks.mux( str(title), diff --git a/unshackle/commands/search.py b/unshackle/commands/search.py index 76fe5938..3985b4d5 100644 --- a/unshackle/commands/search.py +++ b/unshackle/commands/search.py @@ -68,7 +68,7 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr if config.proxy_providers.get("basic"): proxy_providers.append(Basic(**config.proxy_providers["basic"])) expressvpn = ExpressVPN(**(config.proxy_providers.get("expressvpn") or {})) - if config.proxy_providers.get("expressvpn") or expressvpn.cookie_path.is_file(): + if config.proxy_providers.get("expressvpn") or expressvpn.cache_path.is_file(): proxy_providers.append(expressvpn) if config.proxy_providers.get("nordvpn"): proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index 57417593..22ad10e3 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -1,3 +1,4 @@ +import hmac import logging import subprocess @@ -135,7 +136,9 @@ def serve( secret_key = request.headers.get("X-Secret-Key") if not secret_key: return web.json_response({"status": 401, "message": "Secret Key is Empty."}, status=401) - if secret_key not in request.app["config"]["users"]: + # Constant-time compare against every configured key so a valid key can't be + # recovered byte-by-byte from response timing. + if not any(hmac.compare_digest(secret_key, k) for k in request.app["config"]["users"]): return web.json_response({"status": 401, "message": "Secret Key is Invalid."}, status=401) return await handler(request) diff --git a/unshackle/commands/util.py b/unshackle/commands/util.py index 7a3c42c0..67d7f4d1 100644 --- a/unshackle/commands/util.py +++ b/unshackle/commands/util.py @@ -282,6 +282,8 @@ def test(path: Path, map_: str) -> None: ], stderr=subprocess.PIPE, universal_newlines=True, + encoding="utf-8", + errors="replace", ) reached_output = False errors = 0 diff --git a/unshackle/core/api/compression.py b/unshackle/core/api/compression.py index 104751b8..8b05a607 100644 --- a/unshackle/core/api/compression.py +++ b/unshackle/core/api/compression.py @@ -3,9 +3,26 @@ from __future__ import annotations import gzip +import zlib from aiohttp import web +# Cap the inflated size of a peer-supplied compressed blob (cookies, session cache, +# serialized manifests) so a tiny payload cannot expand without bound (zip-bomb DoS). +MAX_INFLATE_BYTES = 64 * 1024 * 1024 # 64 MiB + + +def safe_inflate(data: bytes, max_size: int = MAX_INFLATE_BYTES) -> bytes: + """zlib-decompress *data*, raising ValueError if the result would exceed *max_size*.""" + decompressor = zlib.decompressobj() + out = decompressor.decompress(data, max_size + 1) + if len(out) > max_size or decompressor.unconsumed_tail: + raise ValueError("compressed payload expands beyond allowed size") + out += decompressor.flush() + if len(out) > max_size: + raise ValueError("compressed payload expands beyond allowed size") + return out + @web.middleware async def compression_middleware(request: web.Request, handler) -> web.Response: diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index aa4aac64..3e7d3bdf 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -127,6 +127,9 @@ class DownloadJob: # Guards against writing the same job to the persistent history file twice. history_recorded: bool = False + # X-Secret-Key that created the job; scopes per-caller access, never serialized into to_dict(). + owner_key: Optional[str] = None + def to_dict(self, include_full_details: bool = False) -> Dict[str, Any]: """Convert job to dictionary for JSON response.""" result = { @@ -617,7 +620,7 @@ class DownloadQueueManager: f"Initialized download queue manager: max_concurrent={max_concurrent_downloads}, retention_hours={job_retention_hours}" ) - def create_job(self, service: str, title_id: str, **parameters) -> DownloadJob: + def create_job(self, service: str, title_id: str, owner_key: Optional[str] = None, **parameters) -> DownloadJob: """Create a new download job and add it to the queue.""" job_id = str(uuid.uuid4()) job = DownloadJob( @@ -627,6 +630,7 @@ class DownloadQueueManager: service=service, title_id=title_id, parameters=parameters, + owner_key=owner_key, ) self._jobs[job_id] = job diff --git a/unshackle/core/api/download_worker.py b/unshackle/core/api/download_worker.py index 148483f5..81cf0072 100644 --- a/unshackle/core/api/download_worker.py +++ b/unshackle/core/api/download_worker.py @@ -63,9 +63,7 @@ def main(argv: list[str]) -> int: except Exception as e: log.error(f"Failed to write progress update: {e}") - output_files = _perform_download( - job_id, service, title_id, params, progress_callback=progress_callback - ) + output_files = _perform_download(job_id, service, title_id, params, progress_callback=progress_callback) result = {"status": "success", "output_files": output_files} diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 1b9ad0a7..a7268cea 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -8,9 +8,10 @@ from typing import Any, Dict, List, Optional from aiohttp import web +from unshackle.core.api.compression import safe_inflate from unshackle.core.api.errors import APIError, APIErrorCode, handle_api_exception from unshackle.core.api.input_bridge import AuthStatus, InputBridge -from unshackle.core.api.sanitize import sanitize_log +from unshackle.core.api.sanitize import safe_cache_key, sanitize_log from unshackle.core.config import config from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy @@ -275,6 +276,25 @@ def get_allowed_services(request: Optional[web.Request] = None) -> Optional[List return list(result) +def caller_key(request: Optional[web.Request] = None) -> str: + """The authenticating X-Secret-Key for a request, or 'anonymous' when unauthenticated.""" + return request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + + +def owns_job(job: Any, request: Optional[web.Request] = None) -> bool: + """Whether the calling key owns this job. + + Jobs created without an owner (no-key mode / legacy) stay shared; otherwise a job is + only visible to the key that created it (constant-time compare). + """ + import hmac + + owner = getattr(job, "owner_key", None) + if owner is None: + return True + return hmac.compare_digest(owner, caller_key(request)) + + def validate_service(service_tag: str, request: Optional[web.Request] = None) -> Optional[str]: """Validate, normalize, and check allowlist for service tag.""" try: @@ -909,9 +929,7 @@ def enforce_download_gates(params: Dict[str, Any]) -> None: requested_cdm = params.get("cdm") if requested_cdm: allowed = (config.serve or {}).get("cdm_overrides") - permitted = allowed is True or ( - isinstance(allowed, (list, tuple, set)) and requested_cdm in allowed - ) + permitted = allowed is True or (isinstance(allowed, (list, tuple, set)) and requested_cdm in allowed) if not permitted: raise APIError( APIErrorCode.FORBIDDEN, @@ -985,7 +1003,7 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] **service_specific_defaults, **filtered_params, } - job = manager.create_job(normalized_service, title_id, **params_with_defaults) + job = manager.create_job(normalized_service, title_id, owner_key=caller_key(request), **params_with_defaults) return web.json_response( {"job_id": job.job_id, "status": job.status.value, "created_time": job.created_time.isoformat()}, status=202 @@ -1009,7 +1027,7 @@ async def list_download_jobs_handler(data: Dict[str, Any], request: Optional[web try: manager = get_download_manager() - jobs = manager.list_jobs() + jobs = [job for job in manager.list_jobs() if owns_job(job, request)] status_filter = data.get("status") if status_filter: @@ -1080,7 +1098,7 @@ async def get_download_job_handler(job_id: str, request: Optional[web.Request] = manager = get_download_manager() job = manager.get_job(job_id) - if not job: + if not job or not owns_job(job, request): raise APIError( APIErrorCode.JOB_NOT_FOUND, "Job not found", @@ -1109,7 +1127,7 @@ async def cancel_download_job_handler(job_id: str, request: Optional[web.Request manager = get_download_manager() job = manager.get_job(job_id) - if not job: + if not job or not owns_job(job, request): raise APIError( APIErrorCode.JOB_NOT_FOUND, "Job not found", @@ -1173,7 +1191,7 @@ async def retry_download_job_handler(job_id: str, request: Optional[web.Request] manager = get_download_manager() job = manager.get_job(job_id) - if not job: + if not job or not owns_job(job, request): raise APIError( APIErrorCode.JOB_NOT_FOUND, "Job not found", @@ -1200,10 +1218,14 @@ async def retry_download_job_handler(job_id: str, request: Optional[web.Request] await manager.start_workers() # Reuse the raw in-memory parameters; redaction only ever applies to serialized copies. - new_job = manager.create_job(job.service, job.title_id, **job.parameters) + new_job = manager.create_job(job.service, job.title_id, owner_key=caller_key(request), **job.parameters) return web.json_response( - {"job_id": new_job.job_id, "status": new_job.status.value, "created_time": new_job.created_time.isoformat()}, + { + "job_id": new_job.job_id, + "status": new_job.status.value, + "created_time": new_job.created_time.isoformat(), + }, status=202, ) @@ -1227,7 +1249,7 @@ async def prioritize_download_job_handler(job_id: str, request: Optional[web.Req manager = get_download_manager() job = manager.get_job(job_id) - if not job: + if not job or not owns_job(job, request): raise APIError( APIErrorCode.JOB_NOT_FOUND, "Job not found", @@ -1482,7 +1504,9 @@ def _binary_version(path: Any) -> Optional[str]: for flag in ("--version", "-version"): try: - proc = subprocess.run([str(path), flag], capture_output=True, text=True, timeout=5) + proc = subprocess.run( + [str(path), flag], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=5 + ) except (OSError, subprocess.SubprocessError, ValueError): return None match = _VERSION_RE.search(proc.stdout or "") or _VERSION_RE.search(proc.stderr or "") @@ -1626,10 +1650,9 @@ def _create_service_instance( if cookie_text and isinstance(cookie_text, str): import base64 import tempfile - import zlib from http.cookiejar import MozillaCookieJar - cookie_str = zlib.decompress(base64.b64decode(cookie_text)).decode("utf-8") + cookie_str = safe_inflate(base64.b64decode(cookie_text)).decode("utf-8") with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f: f.write(cookie_str) tmp_path = f.name @@ -1700,13 +1723,16 @@ async def session_create_handler(data: Dict[str, Any], request: Optional[web.Req cache_data = data.get("cache", {}) if cache_data: import base64 - import zlib cache_dir = app_config.directories.cache / session_cache_tag cache_dir.mkdir(parents=True, exist_ok=True) for key, content in cache_data.items(): - decompressed = zlib.decompress(base64.b64decode(content)).decode("utf-8") - (cache_dir / key).with_suffix(".json").write_text(decompressed, encoding="utf-8") + safe_name = safe_cache_key(key) + if not safe_name: + log.warning(f"Rejecting unsafe session cache key: {sanitize_log(key)}") + continue + decompressed = safe_inflate(base64.b64decode(content)).decode("utf-8") + (cache_dir / safe_name).with_suffix(".json").write_text(decompressed, encoding="utf-8") bridge = InputBridge() service_instance._input_bridge = bridge @@ -1718,6 +1744,7 @@ async def session_create_handler(data: Dict[str, Any], request: Optional[web.Req session_id=session_id, ) session.creator_ip = request.remote if request else None + session.owner_key = api_key session.cache_tag = session_cache_tag session.input_bridge = bridge session.auth_status = AuthStatus.AUTHENTICATING @@ -2138,7 +2165,14 @@ async def session_prompt_post_handler( async def _get_validated_session(session_id: str, request: Optional[web.Request]) -> Any: - """Fetch a session and verify the requesting IP matches the creator.""" + """Fetch a session and verify the caller owns it. + + Ownership is bound to the authenticating X-Secret-Key rather than the source IP: + behind a reverse proxy every caller shares the proxy's address, so the IP check + (kept as defence in depth) cannot distinguish users on its own. + """ + import hmac + from unshackle.core.api.session_store import get_session_store store = get_session_store() @@ -2149,6 +2183,13 @@ async def _get_validated_session(session_id: str, request: Optional[web.Request] f"Session not found or expired: {session_id}", details={"session_id": session_id}, ) + if session.owner_key is not None and request is not None: + caller_key = request.headers.get("X-Secret-Key", "anonymous") + if not hmac.compare_digest(caller_key, session.owner_key): + raise APIError( + APIErrorCode.FORBIDDEN, + "Session access denied", + ) if session.creator_ip and request and request.remote != session.creator_ip: raise APIError( APIErrorCode.FORBIDDEN, @@ -2407,11 +2448,14 @@ def _handle_single_server_cdm( pr_pssh = PlayReadyPSSH(base64.b64decode(pssh_b64)) pr_drm = PlayReady(pssh=pr_pssh, pssh_b64=pssh_b64) + # Gate on the caller's CDM device first so a caller with no device cannot + # harvest server-side keys from the vault fallback below. + device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) + vault_keys = _check_vaults(pr_drm.kids, service.__class__.__name__) if vault_keys: return vault_keys - device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) cdm = load_cdm(device_name, service_name=service.__class__.__name__) pr_drm.get_content_keys( cdm=cdm, @@ -2427,11 +2471,14 @@ def _handle_single_server_cdm( wv_pssh = WvPSSH(pssh_b64) wv_drm = Widevine(pssh=wv_pssh) + # Gate on the caller's CDM device first so a caller with no device cannot + # harvest server-side keys from the vault fallback below. + device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) + vault_keys = _check_vaults(wv_drm.kids, service.__class__.__name__) if vault_keys: return vault_keys - device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) cdm = load_cdm(device_name, service_name=service.__class__.__name__) wv_drm.get_content_keys( cdm=cdm, diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 693b2a1f..44956915 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -10,17 +10,36 @@ from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings from unshackle.core import __version__ from unshackle.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception -from unshackle.core.api.handlers import (cancel_download_job_handler, clear_cache_handler, - clear_finished_download_jobs_handler, clear_temp_handler, - delete_history_handler, download_handler, download_history_handler, - env_check_handler, get_allowed_services, get_download_job_handler, - list_download_jobs_handler, list_titles_handler, list_tracks_handler, - prioritize_download_job_handler, profiles_handler, refresh_services_handler, - retry_download_job_handler, search_handler, server_config_handler, - session_create_handler, session_delete_handler, session_info_handler, - session_license_handler, session_prompt_get_handler, - session_prompt_post_handler, session_segments_handler, - session_titles_handler, session_tracks_handler) +from unshackle.core.api.handlers import ( + cancel_download_job_handler, + clear_cache_handler, + clear_finished_download_jobs_handler, + clear_temp_handler, + delete_history_handler, + download_handler, + download_history_handler, + env_check_handler, + get_allowed_services, + get_download_job_handler, + list_download_jobs_handler, + list_titles_handler, + list_tracks_handler, + prioritize_download_job_handler, + profiles_handler, + refresh_services_handler, + retry_download_job_handler, + search_handler, + server_config_handler, + session_create_handler, + session_delete_handler, + session_info_handler, + session_license_handler, + session_prompt_get_handler, + session_prompt_post_handler, + session_segments_handler, + session_titles_handler, + session_tracks_handler, +) from unshackle.core.services import Services from unshackle.core.update_checker import UpdateChecker diff --git a/unshackle/core/api/sanitize.py b/unshackle/core/api/sanitize.py index 81e4a0e1..f62bcf33 100644 --- a/unshackle/core/api/sanitize.py +++ b/unshackle/core/api/sanitize.py @@ -1,8 +1,25 @@ -"""Helpers for safely logging user-provided values.""" +"""Helpers for safely logging and handling user-provided values.""" from __future__ import annotations +from pathlib import Path +from typing import Optional + def sanitize_log(value: object) -> str: """Sanitize a value for safe logging by removing newlines and control characters.""" return str(value).replace("\n", "").replace("\r", "").replace("\x00", "") + + +def safe_cache_key(key: object) -> Optional[str]: + """Return a bare filename for a peer-supplied cache key, or None if it escapes its directory. + + Cache keys are written as ``<key>.json`` inside a fixed directory. A key containing a + path separator, ``..``, or an absolute path would let the peer write outside that + directory, so anything that is not already a plain filename is rejected. + """ + text = str(key) + name = Path(text).name + if not name or name != text or name in {".", ".."}: + return None + return name diff --git a/unshackle/core/api/session_store.py b/unshackle/core/api/session_store.py index 0ba1a0ec..5d990b60 100644 --- a/unshackle/core/api/session_store.py +++ b/unshackle/core/api/session_store.py @@ -35,6 +35,7 @@ class SessionEntry: tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) # title_key -> {track_id -> Track} chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) # title_key -> [Chapter] creator_ip: Optional[str] = None + owner_key: Optional[str] = None # X-Secret-Key that owns this session cache_tag: Optional[str] = None # per-session cache directory tag input_bridge: Optional[InputBridge] = None auth_status: AuthStatus = AuthStatus.AUTHENTICATED @@ -98,9 +99,7 @@ class SessionStore: if entry.auth_status not in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): elapsed = (datetime.now(timezone.utc) - entry.last_accessed).total_seconds() if elapsed > self._ttl: - log.info( - f"Session {sanitize_log(session_id)} expired (elapsed={elapsed:.0f}s, ttl={self._ttl}s)" - ) + log.info(f"Session {sanitize_log(session_id)} expired (elapsed={elapsed:.0f}s, ttl={self._ttl}s)") del self._sessions[session_id] return None diff --git a/unshackle/core/binaries.py b/unshackle/core/binaries.py index 5ddf3a02..daa77506 100644 --- a/unshackle/core/binaries.py +++ b/unshackle/core/binaries.py @@ -34,30 +34,44 @@ def find(*names: str) -> Optional[Path]: return None -FFMPEG = find("ffmpeg") -FFProbe = find("ffprobe") -FFPlay = find("ffplay") -# seconv = SubtitleEdit 5+ CLI (the 5.0 GUI has no batch mode); SubtitleEdit = 4.x -SubtitleEdit = find("seconv", "SubtitleEdit") -ShakaPackager = find( - "shaka-packager", - "packager", - f"packager-{__shaka_platform}", - f"packager-{__shaka_platform}-arm64", - f"packager-{__shaka_platform}-x64", -) -CCExtractor = find("ccextractor", "ccextractorwin", "ccextractorwinfull") -HolaProxy = find("hola-proxy") -MPV = find("mpv") -Caddy = find("caddy") -MKVToolNix = find("mkvmerge") -Mkvpropedit = find("mkvpropedit") -DoviTool = find("dovi_tool") -HDR10PlusTool = find("hdr10plus_tool", "HDR10Plus_tool") -Mp4decrypt = find("mp4decrypt") -Docker = find("docker") -ML_Worker = find("ML-Worker") -Git = find("git") +# Binary attribute -> candidate names passed to find(). Resolved lazily on first +# attribute access (see __getattr__) to avoid ~500ms of shutil.which PATH scans at import. +__binaries = { + "FFMPEG": ("ffmpeg",), + "FFProbe": ("ffprobe",), + "FFPlay": ("ffplay",), + # seconv = SubtitleEdit 5+ CLI (the 5.0 GUI has no batch mode); SubtitleEdit = 4.x + "SubtitleEdit": ("seconv", "SubtitleEdit"), + "ShakaPackager": ( + "shaka-packager", + "packager", + f"packager-{__shaka_platform}", + f"packager-{__shaka_platform}-arm64", + f"packager-{__shaka_platform}-x64", + ), + "CCExtractor": ("ccextractor", "ccextractorwin", "ccextractorwinfull"), + "HolaProxy": ("hola-proxy",), + "MPV": ("mpv",), + "Caddy": ("caddy",), + "MKVToolNix": ("mkvmerge",), + "Mkvpropedit": ("mkvpropedit",), + "DoviTool": ("dovi_tool",), + "HDR10PlusTool": ("hdr10plus_tool", "HDR10Plus_tool"), + "Mp4decrypt": ("mp4decrypt",), + "Docker": ("docker",), + "ML_Worker": ("ML-Worker",), + "Git": ("git",), +} + + +def __getattr__(name: str) -> Optional[Path]: + candidates = __binaries.get(name) + if candidates is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + # Cache the result (including None for an absent binary) so later access is a plain attr hit. + resolved = find(*candidates) + globals()[name] = resolved + return resolved __all__ = ( diff --git a/unshackle/core/cdm/custom_remote_cdm.py b/unshackle/core/cdm/custom_remote_cdm.py index cd4c5591..5e7562c7 100644 --- a/unshackle/core/cdm/custom_remote_cdm.py +++ b/unshackle/core/cdm/custom_remote_cdm.py @@ -152,7 +152,9 @@ class CustomRemoteCDM: self.security_level = device.get("security_level", 3) # Determine if this is a PlayReady CDM - self._is_playready = self.device_type_str.upper() == "PLAYREADY" or self.device_name in ["SL2", "SL3"] + self._is_playready = self.device_type_str.upper() == "PLAYREADY" or ( + bool(self.device_name) and self.device_name.upper().startswith("SL") + ) # Get device type enum for compatibility if self.device_type_str: diff --git a/unshackle/core/cdm/decrypt_labs_remote_cdm.py b/unshackle/core/cdm/decrypt_labs_remote_cdm.py index 78e383fc..39cf7d33 100644 --- a/unshackle/core/cdm/decrypt_labs_remote_cdm.py +++ b/unshackle/core/cdm/decrypt_labs_remote_cdm.py @@ -134,7 +134,9 @@ class DecryptLabsRemoteCDM: if device_type: self.device_type = self._get_device_type_enum(device_type) - self._is_playready = (device_type and device_type.upper() == "PLAYREADY") or (device_name in ["SL2", "SL3"]) + self._is_playready = (device_type and device_type.upper() == "PLAYREADY") or ( + bool(device_name) and device_name.upper().startswith("SL") + ) if self._is_playready: self.system_id = system_id or 0 diff --git a/unshackle/core/cdm/detect.py b/unshackle/core/cdm/detect.py index 767ba1ee..162413f4 100644 --- a/unshackle/core/cdm/detect.py +++ b/unshackle/core/cdm/detect.py @@ -112,6 +112,10 @@ def is_playready_cdm(cdm: Any) -> bool: if cdm is None: return False + drm = getattr(cdm, "drm", None) + if drm: + return str(drm).lower() == "playready" + if hasattr(cdm, "is_playready"): try: return bool(getattr(cdm, "is_playready")) @@ -144,6 +148,18 @@ def is_playready_cdm(cdm: Any) -> bool: return "pyplayready" in mod +def is_remote_playready_cdm(cdm: Any) -> bool: + """Return True if the CDM is a remote PlayReady CDM (as opposed to a local .prd).""" + + return is_playready_cdm(cdm) and is_remote_cdm(cdm) + + +def is_remote_widevine_cdm(cdm: Any) -> bool: + """Return True if the CDM is a remote Widevine CDM (as opposed to a local .wvd).""" + + return is_widevine_cdm(cdm) and is_remote_cdm(cdm) + + def is_widevine_cdm(cdm: Any) -> bool: """ Return True if the given CDM should be treated as Widevine. @@ -155,6 +171,10 @@ def is_widevine_cdm(cdm: Any) -> bool: if cdm is None: return False + drm = getattr(cdm, "drm", None) + if drm: + return str(drm).lower() == "widevine" + if hasattr(cdm, "is_playready"): try: return not bool(getattr(cdm, "is_playready")) diff --git a/unshackle/core/cdm/loader.py b/unshackle/core/cdm/loader.py index 7f0a7f4f..be66c428 100644 --- a/unshackle/core/cdm/loader.py +++ b/unshackle/core/cdm/loader.py @@ -13,6 +13,22 @@ from typing import Any, Optional log = logging.getLogger("cdm") +def _stamp_remote(cdm: Any, drm: str) -> Any: + """Record a remote CDM's DRM type and mark it as remote. + + The loader knows the DRM type from config, so it sets it here as the value + `detect.py` reads before it falls back to class or module checks. Setting the + attributes is best effort: a third-party CDM object that rejects them still + works, since detect.py keeps its existing checks. + """ + try: + cdm.drm = drm + cdm.is_remote_cdm = True + except (AttributeError, TypeError): + pass + return cdm + + def load_cdm( cdm_name: str, *, @@ -63,35 +79,43 @@ def _load_remote_cdm( f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global decrypt_labs_api_key configured" ) - return DecryptLabsRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + cdm: Any = DecryptLabsRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + return _stamp_remote(cdm, "playready" if cdm.is_playready else "widevine") if cdm_type == "custom_api": from unshackle.core.cdm.custom_remote_cdm import CustomRemoteCDM del cdm_api["name"] del cdm_api["type"] - return CustomRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + cdm = CustomRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + return _stamp_remote(cdm, "playready" if cdm.is_playready else "widevine") device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) if str(device_type).upper() == "PLAYREADY": from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm - return PlayReadyRemoteCdm( - security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), - host=cdm_api.get("Host", cdm_api.get("host")), - secret=cdm_api.get("Secret", cdm_api.get("secret")), - device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + return _stamp_remote( + PlayReadyRemoteCdm( + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ), + "playready", ) from pywidevine.remotecdm import RemoteCdm - return RemoteCdm( - device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), - system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), - security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), - host=cdm_api.get("Host", cdm_api.get("host")), - secret=cdm_api.get("Secret", cdm_api.get("secret")), - device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + return _stamp_remote( + RemoteCdm( + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ), + "widevine", ) diff --git a/unshackle/core/config.py b/unshackle/core/config.py index 9aa160b8..f8cf4973 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -114,6 +114,8 @@ class Config: self.tag: str = kwargs.get("tag") or "" self.tag_group_name: bool = kwargs.get("tag_group_name", True) self.tag_imdb_tmdb: bool = kwargs.get("tag_imdb_tmdb", True) + self.imdb_api_enabled: bool = kwargs.get("imdb_api_enabled", False) + self.omdb_api_key: str = kwargs.get("omdb_api_key") or "" self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or "" self.simkl_client_id: str = kwargs.get("simkl_client_id") or "" self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or "" diff --git a/unshackle/core/console.py b/unshackle/core/console.py index c5af3686..52c0415f 100644 --- a/unshackle/core/console.py +++ b/unshackle/core/console.py @@ -1,17 +1,21 @@ import logging +import math from datetime import datetime from types import ModuleType -from typing import IO, Callable, Iterable, List, Literal, Mapping, Optional, TextIO, Union +from typing import IO, Any, Callable, Iterable, List, Literal, Mapping, Optional, TextIO, Union from rich._log_render import FormatTimeCallable, LogRender -from rich.console import Console, ConsoleRenderable, HighlighterType, RenderableType +from rich.color import Color, blend_rgb +from rich.console import Console, ConsoleOptions, ConsoleRenderable, HighlighterType, RenderableType, RenderResult from rich.emoji import EmojiVariant from rich.highlighter import Highlighter, ReprHighlighter from rich.live import Live from rich.logging import RichHandler +from rich.measure import Measurement from rich.padding import Padding, PaddingDimensions +from rich.progress import BarColumn, Task from rich.status import Status -from rich.style import StyleType +from rich.style import Style, StyleType from rich.table import Table from rich.text import Text, TextType from rich.theme import Theme @@ -279,7 +283,7 @@ class ComfyConsole(Console): padding = Padding(status_renderable, (top, right, bottom, left)) - return Live(padding, console=self, transient=True) + return SyncLive(padding, console=self, transient=True) return status_renderable @@ -349,6 +353,60 @@ if config.set_terminal_bg: custom_colors["ascii.art"] += f" on {primary_scheme['bg']}" +class SyncLive(Live): + """Live that wraps each repaint in DEC 2026 begin/end-sync so the terminal draws it as one atomic frame.""" + + def refresh(self) -> None: + if not self.console.is_terminal: + super().refresh() + return + self.console.file.write("\x1b[?2026h") + try: + super().refresh() + finally: + self.console.file.write("\x1b[?2026l") + self.console.file.flush() + + +class GradientPulseBarColumn(BarColumn): + _LUT_SIZE = 32 + + def __init__(self, *args: Any, pulse_period: float = 40.0, pulse_speed: float = 15.0, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.pulse_period = pulse_period + self.pulse_speed = pulse_speed + lo = Color.parse(primary_scheme["dark_gray"]).triplet + hi = Color.parse(primary_scheme["pink"]).triplet + assert lo and hi + self._lut = [ + Style(color=Color.from_triplet(blend_rgb(lo, hi, i / (self._LUT_SIZE - 1)))) for i in range(self._LUT_SIZE) + ] + + def render(self, task: Task) -> RenderableType: # type: ignore[override] + if task.started and task.total is not None: + return super().render(task) + return _GradientPulse(self._lut, task.get_time() * self.pulse_speed, self.pulse_period, self.bar_width) + + +class _GradientPulse: + def __init__(self, lut: list[Style], offset: float, period: float, width: Optional[int]) -> None: + self.lut = lut + self.offset = offset + self.period = period + self.width = width + + def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderResult: + width = min(self.width or options.max_width, options.max_width) + bar = Text() + for i in range(width): + fade = (math.cos((i - self.offset) * math.tau / self.period) + 1) / 2 + bar.append("━", self.lut[int(fade * (len(self.lut) - 1))]) + yield bar + + def __rich_measure__(self, console: Console, options: ConsoleOptions) -> Measurement: + return Measurement(4, self.width or options.max_width) + + console = ComfyConsole( log_time=False, log_path=False, @@ -358,7 +416,7 @@ console = ComfyConsole( "bar.back": primary_scheme["dark_gray"], "bar.complete": primary_scheme["pink"], "bar.finished": primary_scheme["green"], - "bar.pulse": primary_scheme["bright_black"], + "bar.pulse": primary_scheme["pink"], "black": primary_scheme["black"], "inspect.async_def": f"italic {primary_scheme['cyan']}", "progress.data.speed": "dark_orange", @@ -377,4 +435,4 @@ console = ComfyConsole( ) -__all__ = ("ComfyLogRenderer", "ComfyRichHandler", "ComfyConsole", "console") +__all__ = ("ComfyLogRenderer", "ComfyRichHandler", "ComfyConsole", "GradientPulseBarColumn", "SyncLive", "console") diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index 1b29f901..9f4084b3 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -1085,7 +1085,7 @@ def requests( start_time = time.time() last_speed_report = start_time last_hedge_check = 0.0 - hedge_median = 0.0 + hedge_median = 0.0 # cached; recomputed only when seg_durations grows hedge_median_len = 0 pool = ThreadPoolExecutor(max_workers=max_workers) @@ -1364,7 +1364,8 @@ def requests( yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") last_speed_report = now - # hedge stuck segments once spare workers exist within the active target + # hedge stuck segments once spare workers exist within the active target; + # throttled to ~0.5s and median recomputed only when seg_durations grows if ( len(pending) < target and seg_durations @@ -1372,7 +1373,7 @@ def requests( and not DOWNLOAD_CANCELLED.is_set() ): last_hedge_check = now - cur_len = len(seg_durations) # len() is safe here even though workers append under seg_lock + cur_len = len(seg_durations) # atomic read; appended under seg_lock by workers if cur_len != hedge_median_len: hedge_median = statistics.median(seg_durations) hedge_median_len = cur_len diff --git a/unshackle/core/drm/clearkey_cenc.py b/unshackle/core/drm/clearkey_cenc.py index 5c07434a..5e72376d 100644 --- a/unshackle/core/drm/clearkey_cenc.py +++ b/unshackle/core/drm/clearkey_cenc.py @@ -223,7 +223,15 @@ class ClearKeyCENC: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") + subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) @@ -262,6 +270,8 @@ class ClearKeyCENC: stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True, + encoding="utf-8", + errors="replace", ) stream_skipped = False diff --git a/unshackle/core/drm/monalisa.py b/unshackle/core/drm/monalisa.py index 3b6f29a6..284a19bf 100644 --- a/unshackle/core/drm/monalisa.py +++ b/unshackle/core/drm/monalisa.py @@ -247,6 +247,8 @@ class MonaLisa: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", + errors="replace", startupinfo=startupinfo, timeout=worker_timeout_s, ) @@ -294,6 +296,6 @@ class MonaLisa: the content has already been decrypted and muxed into a container. Args: - path: Path to the file (ignored). + _path: Path to the file (ignored). """ pass diff --git a/unshackle/core/drm/playready.py b/unshackle/core/drm/playready.py index bfcfb8e5..6fdd46d9 100644 --- a/unshackle/core/drm/playready.py +++ b/unshackle/core/drm/playready.py @@ -438,7 +438,15 @@ class PlayReady: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") + subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) diff --git a/unshackle/core/drm/widevine.py b/unshackle/core/drm/widevine.py index b48a930e..2b036235 100644 --- a/unshackle/core/drm/widevine.py +++ b/unshackle/core/drm/widevine.py @@ -80,8 +80,8 @@ class Widevine: headers and more. Raises: - PSSHNotFound - If the PSSH was not found within the data. - KIDNotFound - If the KID was not found within the data or PSSH. + PSSHNotFound: If the PSSH was not found within the data. + KIDNotFound: If the KID was not found within the data or PSSH. """ if not session: session = Session() @@ -131,8 +131,8 @@ class Widevine: It is *rare* to need to use this. Raises: - PSSHNotFound - If the PSSH was not found within the data. - KIDNotFound - If the KID was not found within the data or PSSH. + PSSHNotFound: If the PSSH was not found within the data. + KIDNotFound: If the KID was not found within the data or PSSH. """ if not init_data: raise ValueError("Init data should be provided.") @@ -378,7 +378,15 @@ class Widevine: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") + subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) @@ -426,6 +434,8 @@ class Widevine: stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True, + encoding="utf-8", + errors="replace", ) stream_skipped = False diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 992f65ee..796d3bbf 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -599,11 +599,11 @@ class DASH: events.emit(events.Types.TRACK_DOWNLOADED, track=track) if drm: - progress(downloaded="Decrypting", completed=0, total=100) + progress(downloaded="Decrypting", completed=0, total=None) drm.decrypt(save_path) track.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) - progress(downloaded="Decrypting", advance=100) + progress(downloaded="Decrypted", completed=100, total=100) # Clean up empty segment directory if save_dir.exists() and save_dir.name.endswith("_segments"): @@ -1180,7 +1180,7 @@ class DASH: @staticmethod @lru_cache(maxsize=None) def _field_format_pattern(field: str) -> re.Pattern: - # matches printf-style `$Field%fmt$` format tokens + # printf-style `$Field%fmt$` matcher, compiled once per field name return re.compile(rf"\${re.escape(field)}%([a-z0-9]+)\$", flags=re.I) @staticmethod diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 90110cfe..a55d5a06 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -1003,7 +1003,7 @@ class HLS: if len(segments_to_merge) == 1: shutil.move(segments_to_merge[0], save_path) else: - progress(downloaded="Merging") + progress(downloaded="Merging", completed=0, total=None) if isinstance(track, (Video, Audio)): HLS.merge_segments(segments=segments_to_merge, save_path=save_path) else: @@ -1023,7 +1023,7 @@ class HLS: # Directory might not be empty, try removing recursively shutil.rmtree(save_dir, ignore_errors=True) - progress(downloaded="Downloaded") + progress(downloaded="Downloaded", completed=100, total=100) track.path = save_path diff --git a/unshackle/core/manifests/ism.py b/unshackle/core/manifests/ism.py index 85dd9649..840d1672 100644 --- a/unshackle/core/manifests/ism.py +++ b/unshackle/core/manifests/ism.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 import hashlib import html +import re import shutil import struct import urllib.parse @@ -18,14 +19,24 @@ from requests import Session from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from unshackle.core.drm import DRM_T, PlayReady, Widevine from unshackle.core.events import events -from unshackle.core.manifests.ism_init import (build_init_segment, parse_codec_private_data_colour, - read_per_sample_iv_size, read_track_id) +from unshackle.core.manifests.ism_init import ( + build_init_segment, + parse_codec_private_data_vui, + read_per_sample_iv_size, + read_track_id, +) from unshackle.core.session import RnetSession from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Track, Tracks, Video from unshackle.core.utilities import log_event, try_ensure_utf8 from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.xml import load_xml +# MS-SSTR: FourCC may be absent; AudioTag carries the WAVE format tag instead. +AUDIO_TAG_FOURCC = {"255": "AACL", "65534": "EC-3"} + +# Smooth FourCCs that Codec.from_mime (RFC 6381 names) doesn't know directly. +FOURCC_MIME = {"H264": "avc1", "H265": "hvc1", "HEVC": "hvc1", "AACL": "mp4a", "AACH": "mp4a", "AACP": "mp4a"} + class ISM: def __init__(self, manifest: Element, url: str) -> None: @@ -88,20 +99,25 @@ class ISM: return drm @staticmethod - def get_video_range(fourcc: str, codec_private_data: str) -> Video.Range: - """Derive colour range from the SPS VUI in CodecPrivateData — Smooth - manifests carry no range attributes. Soft-fails to SDR.""" + def get_video_range_and_fps(fourcc: str, codec_private_data: str) -> tuple[Video.Range, Optional[float]]: + """Derive colour range and fps from the SPS VUI in CodecPrivateData, + since Smooth manifests carry neither as attributes. Range soft-fails to + SDR; fps is None for non-HEVC codecs and VUIs without timing info.""" fourcc = (fourcc or "").upper() - if fourcc in ("DVHE", "DVH1"): - return Video.Range.DV try: cpd = bytes.fromhex(codec_private_data or "") except ValueError: - return Video.Range.SDR - cicp = parse_codec_private_data_colour(fourcc, cpd) + cpd = b"" + cicp, fps = parse_codec_private_data_vui(fourcc, cpd) + if fourcc in ("DVHE", "DVH1"): + return Video.Range.DV, fps if not cicp: - return Video.Range.SDR - return Video.Range.from_cicp(*cicp) + return Video.Range.SDR, fps + return Video.Range.from_cicp(*cicp), fps + + @staticmethod + def get_video_range(fourcc: str, codec_private_data: str) -> Video.Range: + return ISM.get_video_range_and_fps(fourcc, codec_private_data)[0] @staticmethod def _init_segment( @@ -121,7 +137,7 @@ class ISM: # CodecPrivateData may legitimately be empty (AAC config is synthesized, # EC-3 decoders sync from the frames); the builder handles each case. cpd = quality_level.get("CodecPrivateData") or "" - fourcc = quality_level.get("FourCC") or "" + fourcc = quality_level.get("FourCC") or AUDIO_TAG_FOURCC.get(quality_level.get("AudioTag") or "") or "" root_timescale = manifest.get("TimeScale") if manifest is not None else None timescale = int(stream_index.get("TimeScale") or root_timescale or 10000000) @@ -202,6 +218,8 @@ class ISM: return None def to_tracks(self, language: Optional[Union[str, Language]] = None) -> Tracks: + if (self.manifest.get("IsLive") or "").upper() == "TRUE": + raise ValueError("Live Smooth Streaming manifests are not supported") tracks = Tracks() base_url = self.url duration = int(self.manifest.get("Duration") or 0) @@ -212,17 +230,30 @@ class ISM: if not content_type: raise ValueError("No content type value could be found") for ql in stream_index.findall("QualityLevel"): - codec = ql.get("FourCC") + codec = ql.get("FourCC") or AUDIO_TAG_FOURCC.get(ql.get("AudioTag") or "") if codec == "TTML": codec = "STPP" track_lang = None lang = (stream_index.get("Language") or "").strip() if lang and tag_is_valid(lang) and not lang.startswith("und"): track_lang = Language.get(lang) + if not track_lang and not language: + # Language is optional in MS-SSTR; video streams commonly omit it. + raise ValueError( + "Language information could not be derived from the manifest and no fallback " + "language was provided when calling ISM.to_tracks()." + ) track_urls: list[str] = [] fragment_time = 0 fragments = stream_index.findall("c") + # MS-SSTR UrlPattern; regex over str.format so {Bitrate}/{start_time} + # spellings work and unknown placeholders like {CustomAttributes} + # or stray braces in query strings don't raise. + url_template = urllib.parse.urljoin( + base_url, + re.sub(r"\{[Bb]itrate\}", str(ql.get("Bitrate") or 0), stream_index.get("Url") or ""), + ) # Some manifests omit the first fragment in the <c> list but # still expect a request for start time 0 which contains the # initialization segment. If the first declared fragment is not @@ -230,17 +261,7 @@ class ISM: if fragments: first_time = int(fragments[0].get("t") or 0) if first_time != 0: - track_urls.append( - urllib.parse.urljoin( - base_url, - stream_index.get("Url").format_map( - { - "bitrate": ql.get("Bitrate"), - "start time": "0", - } - ), - ) - ) + track_urls.append(re.sub(r"\{start[ _]time\}", "0", url_template)) for idx, frag in enumerate(fragments): fragment_time = int(frag.get("t", fragment_time)) @@ -251,19 +272,11 @@ class ISM: next_time = int(fragments[idx + 1].get("t")) except (IndexError, AttributeError): next_time = duration - duration_frag = (next_time - fragment_time) / repeat + # floor division: float times would corrupt segment URLs; + # any drift is reset by the next fragment's explicit t. + duration_frag = (next_time - fragment_time) // repeat for _ in range(repeat): - track_urls.append( - urllib.parse.urljoin( - base_url, - stream_index.get("Url").format_map( - { - "bitrate": ql.get("Bitrate"), - "start time": str(fragment_time), - } - ), - ) - ) + track_urls.append(re.sub(r"\{start[ _]time\}", str(fragment_time), url_template)) fragment_time += duration_frag track_id = hashlib.md5( @@ -288,20 +301,25 @@ class ISM: if content_type == "video": try: - vcodec = Video.Codec.from_mime(codec) if codec else None + vcodec = Video.Codec.from_mime(FOURCC_MIME.get(codec.upper(), codec)) if codec else None except ValueError: vcodec = None + range_, fps = self.get_video_range_and_fps(codec or "", ql.get("CodecPrivateData") or "") tracks.add( Video( id_=track_id, url=self.url, codec=vcodec, - range_=self.get_video_range(codec or "", ql.get("CodecPrivateData") or ""), + range_=range_, + fps=fps, language=track_lang or language, is_original_lang=bool(language and track_lang and str(track_lang) == str(language)), bitrate=ql.get("Bitrate"), - width=int(ql.get("MaxWidth") or 0) or int(stream_index.get("MaxWidth") or 0), - height=int(ql.get("MaxHeight") or 0) or int(stream_index.get("MaxHeight") or 0), + # Width/Height are non-spec but common when Max* are absent + width=int(ql.get("MaxWidth") or ql.get("Width") or 0) + or int(stream_index.get("MaxWidth") or stream_index.get("Width") or 0), + height=int(ql.get("MaxHeight") or ql.get("Height") or 0) + or int(stream_index.get("MaxHeight") or stream_index.get("Height") or 0), descriptor=Video.Descriptor.ISM, drm=drm, data=data, @@ -309,7 +327,7 @@ class ISM: ) elif content_type == "audio": try: - acodec = Audio.Codec.from_mime(codec) if codec else None + acodec = Audio.Codec.from_mime(FOURCC_MIME.get(codec.upper(), codec)) if codec else None except ValueError: acodec = None tracks.add( @@ -501,6 +519,7 @@ class ISM: and isinstance(track, Subtitle) and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) ) + progress(downloaded="Merging", completed=0, total=len(segments_to_merge)) with open(save_path, "wb") as f: first_segment = segments_to_merge[0].read_bytes() if segments_to_merge else None init_segment = ISM._init_segment(track, session_drm, first_segment) @@ -530,11 +549,11 @@ class ISM: events.emit(events.Types.TRACK_DOWNLOADED, track=track) if session_drm: - progress(downloaded="Decrypting", completed=0, total=100) + progress(downloaded="Decrypting", completed=0, total=None) session_drm.decrypt(save_path) track.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=session_drm, segment=None) - progress(downloaded="Decrypting", advance=100) + progress(downloaded="Decrypted", completed=100, total=100) try: save_dir.rmdir() diff --git a/unshackle/core/manifests/ism_init.py b/unshackle/core/manifests/ism_init.py index a762dcb2..fff76966 100644 --- a/unshackle/core/manifests/ism_init.py +++ b/unshackle/core/manifests/ism_init.py @@ -29,9 +29,13 @@ s32 = struct.Struct(">i") # 3x3 transformation matrix (identity), as stored in tkhd/mvhd. UNITY_MATRIX = ( - s32.pack(0x10000) + s32.pack(0) * 3 - + s32.pack(0) + s32.pack(0x10000) + s32.pack(0) * 2 - + s32.pack(0) * 2 + s32.pack(0x40000000) + s32.pack(0x10000) + + s32.pack(0) * 3 + + s32.pack(0) + + s32.pack(0x10000) + + s32.pack(0) * 2 + + s32.pack(0) * 2 + + s32.pack(0x40000000) ) TRACK_ENABLED = 0x1 @@ -215,9 +219,9 @@ def skip_hevc_st_ref_pic_set(r: BitReader, idx: int, num_delta_pocs: list[int]) return num_negative + num_positive -def parse_hevc_sps_colour(sps_rbsp: bytes) -> Optional[tuple[int, int, int]]: - """VUI (colour_primaries, transfer_characteristics, matrix_coeffs) from a - de-emulated HEVC SPS, or None when no colour description is present.""" +def parse_hevc_sps_vui(sps_rbsp: bytes) -> tuple[Optional[tuple[int, int, int]], Optional[float]]: + """((colour_primaries, transfer_characteristics, matrix_coeffs), fps) from a + de-emulated HEVC SPS VUI; either element is None when absent.""" r = BitReader(sps_rbsp) _, _, _, max_sub_layers_minus1 = read_hevc_sps_to_bit_depth(r) log2_max_poc_lsb_minus4 = r.read_ue() @@ -245,37 +249,62 @@ def parse_hevc_sps_colour(sps_rbsp: bytes) -> Optional[tuple[int, int, int]]: r.read_bits(1) # used_by_curr_pic_lt_sps_flag r.read_bits(2) # sps_temporal_mvp_enabled + strong_intra_smoothing_enabled if not r.read_bits(1): # vui_parameters_present_flag - return None + return None, None if r.read_bits(1): # aspect_ratio_info_present_flag if r.read_bits(8) == 255: # aspect_ratio_idc == EXTENDED_SAR r.read_bits(32) # sar_width + sar_height if r.read_bits(1): # overscan_info_present_flag r.read_bits(1) # overscan_appropriate_flag - if not r.read_bits(1): # video_signal_type_present_flag - return None - r.read_bits(3) # video_format - r.read_bits(1) # video_full_range_flag - if not r.read_bits(1): # colour_description_present_flag - return None - return r.read_bits(8), r.read_bits(8), r.read_bits(8) + colour: Optional[tuple[int, int, int]] = None + if r.read_bits(1): # video_signal_type_present_flag + r.read_bits(4) # video_format + video_full_range_flag + if r.read_bits(1): # colour_description_present_flag + colour = (r.read_bits(8), r.read_bits(8), r.read_bits(8)) + fps: Optional[float] = None + try: + if r.read_bits(1): # chroma_loc_info_present_flag + r.read_ue() # chroma_sample_loc_type_top_field + r.read_ue() # chroma_sample_loc_type_bottom_field + r.read_bits(3) # neutral_chroma_indication + field_seq + frame_field_info flags + if r.read_bits(1): # default_display_window_flag + for _ in range(4): + r.read_ue() # def_disp_win offsets + if r.read_bits(1): # vui_timing_info_present_flag + num_units_in_tick = r.read_bits(32) + time_scale = r.read_bits(32) + if num_units_in_tick: + fps = time_scale / num_units_in_tick + except (IndexError, ValueError): + pass # truncated VUI: keep whatever colour info was already read + return colour, fps HEVC_FOURCCS = frozenset(("HVC1", "HEV1", "HEVC", "H265", "DVHE", "DVH1")) -def parse_codec_private_data_colour(fourcc: str, codec_private_data: bytes) -> Optional[tuple[int, int, int]]: - """SPS VUI colour triple from HEVC CodecPrivateData; None when the codec - is unsupported, no colour description, or malformed data.""" +def parse_codec_private_data_vui( + fourcc: str, codec_private_data: bytes +) -> tuple[Optional[tuple[int, int, int]], Optional[float]]: + """(colour triple, fps) from the SPS VUI in HEVC CodecPrivateData; (None, None) + when the codec is unsupported or the data is malformed. H.264 is excluded + on purpose: its VUI timing is field-based and often misdeclared, so fps + read from it can't be trusted.""" if (fourcc or "").upper() not in HEVC_FOURCCS: - return None + return None, None try: nals = split_nal_units(codec_private_data) sps = next((n for n in nals if (n[0] >> 1) & 0x3F == 33), None) if sps is None: - return None - return parse_hevc_sps_colour(remove_emulation_prevention(sps)) + return None, None + return parse_hevc_sps_vui(remove_emulation_prevention(sps)) except (IndexError, ValueError): - return None + return None, None + + +def parse_codec_private_data_colour(fourcc: str, codec_private_data: bytes) -> Optional[tuple[int, int, int]]: + """SPS VUI colour triple from HEVC CodecPrivateData; None when the codec + is unsupported, no colour description, or malformed data.""" + return parse_codec_private_data_vui(fourcc, codec_private_data)[0] def iter_boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, Optional[bytes], int, int]]: @@ -610,27 +639,54 @@ def build_init_segment( # --- mvhd --- mvhd = full_box( - b"mvhd", 1, 0, - u64.pack(EPOCH) + u64.pack(EPOCH) + u32.pack(timescale) + u64.pack(duration) - + s1616.pack(1) + s88.pack(1) + u16.pack(0) + u32.pack(0) * 2 - + UNITY_MATRIX + u32.pack(0) * 6 + u32.pack(0xFFFFFFFF), + b"mvhd", + 1, + 0, + u64.pack(EPOCH) + + u64.pack(EPOCH) + + u32.pack(timescale) + + u64.pack(duration) + + s1616.pack(1) + + s88.pack(1) + + u16.pack(0) + + u32.pack(0) * 2 + + UNITY_MATRIX + + u32.pack(0) * 6 + + u32.pack(0xFFFFFFFF), ) # --- tkhd --- tkhd = full_box( - b"tkhd", 1, TRACK_ENABLED | TRACK_IN_MOVIE | TRACK_IN_PREVIEW, - u64.pack(EPOCH) + u64.pack(EPOCH) + u32.pack(track_id) + u32.pack(0) - + u64.pack(duration) + u32.pack(0) * 2 + s16.pack(0) + s16.pack(0) - + s88.pack(1 if stream_type == "audio" else 0) + u16.pack(0) + UNITY_MATRIX - + u1616.pack(width) + u1616.pack(height), + b"tkhd", + 1, + TRACK_ENABLED | TRACK_IN_MOVIE | TRACK_IN_PREVIEW, + u64.pack(EPOCH) + + u64.pack(EPOCH) + + u32.pack(track_id) + + u32.pack(0) + + u64.pack(duration) + + u32.pack(0) * 2 + + s16.pack(0) + + s16.pack(0) + + s88.pack(1 if stream_type == "audio" else 0) + + u16.pack(0) + + UNITY_MATRIX + + u1616.pack(width) + + u1616.pack(height), ) # --- mdhd + hdlr --- packed_lang = ((ord(lang[0]) - 0x60) << 10) | ((ord(lang[1]) - 0x60) << 5) | (ord(lang[2]) - 0x60) mdhd = full_box( - b"mdhd", 1, 0, - u64.pack(EPOCH) + u64.pack(EPOCH) + u32.pack(timescale) + u64.pack(duration) - + u16.pack(packed_lang) + u16.pack(0), + b"mdhd", + 1, + 0, + u64.pack(EPOCH) + + u64.pack(EPOCH) + + u32.pack(timescale) + + u64.pack(duration) + + u16.pack(packed_lang) + + u16.pack(0), ) if stream_type == "audio": hdlr = full_box(b"hdlr", 0, 0, u32.pack(0) + b"soun" + u32.pack(0) * 3 + b"SoundHandler\0") @@ -650,10 +706,18 @@ def build_init_segment( sample_entry_payload = u8.pack(0) * 6 + u16.pack(1) # reserved + data reference index if stream_type == "video": sample_entry_payload += ( - u16.pack(0) + u16.pack(0) + u32.pack(0) * 3 - + u16.pack(width) + u16.pack(height) - + u1616.pack(0x48) + u1616.pack(0x48) + u32.pack(0) + u16.pack(1) - + u8.pack(0) * 32 + u16.pack(0x18) + s16.pack(-1) + u16.pack(0) + + u16.pack(0) + + u32.pack(0) * 3 + + u16.pack(width) + + u16.pack(height) + + u1616.pack(0x48) + + u1616.pack(0x48) + + u32.pack(0) + + u16.pack(1) + + u8.pack(0) * 32 + + u16.pack(0x18) + + s16.pack(-1) ) if fourcc in ("H264", "AVC1", "DAVC"): config_box = build_avcc(cpd, nal_length_size) @@ -677,8 +741,12 @@ def build_init_segment( # samplerate is 16.16 fixed-point; rates above 65535 Hz are written as 0 # (decoders read the real rate from the codec config), matching ffmpeg. sample_entry_payload += ( - u32.pack(0) * 2 + u16.pack(channels) + u16.pack(bits_per_sample) - + u16.pack(0) + u16.pack(0) + u32.pack((sampling_rate if sampling_rate <= 0xFFFF else 0) << 16) + u32.pack(0) * 2 + + u16.pack(channels) + + u16.pack(bits_per_sample) + + u16.pack(0) + + u16.pack(0) + + u32.pack((sampling_rate if sampling_rate <= 0xFFFF else 0) << 16) ) if fourcc in ("AACL", "AACH", "AAC"): if not cpd: @@ -724,7 +792,9 @@ def build_init_segment( # --- mvex (mehd + trex) signals a fragmented file --- mehd = full_box(b"mehd", 1, 0, u64.pack(duration)) trex = full_box( - b"trex", 0, 0, + b"trex", + 0, + 0, u32.pack(track_id) + u32.pack(1) + u32.pack(0) + u32.pack(0) + u32.pack(0), ) mvex = box(b"mvex", mehd + trex) diff --git a/unshackle/core/music/__init__.py b/unshackle/core/music/__init__.py index df6779e1..87189990 100644 --- a/unshackle/core/music/__init__.py +++ b/unshackle/core/music/__init__.py @@ -1,6 +1,15 @@ from .display import MusicHeaderInfo, TrackRow, render_album_header, render_artwork_preview, render_track_panel -from .extract import (build_music_from_songs, classify_release_kind, dedupe_track_options, duration_seconds, - first_number, first_text, format_duration, format_names, year_from_value) +from .extract import ( + build_music_from_songs, + classify_release_kind, + dedupe_track_options, + duration_seconds, + first_number, + first_text, + format_duration, + format_names, + year_from_value, +) from .hasher import file_md5 from .integrity import MusicAudioIntegrityError, MusicAudioIntegrityResult, verify_music_audio from .manifest import write_music_manifest diff --git a/unshackle/core/music/integrity.py b/unshackle/core/music/integrity.py index 8ade62fd..c1706406 100644 --- a/unshackle/core/music/integrity.py +++ b/unshackle/core/music/integrity.py @@ -40,7 +40,9 @@ class MusicAudioIntegrityResult: } -def verify_music_audio(path: Path, *, song: Any = None, track: Any = None, media_info: Optional[MediaInfo] = None) -> MusicAudioIntegrityResult: +def verify_music_audio( + path: Path, *, song: Any = None, track: Any = None, media_info: Optional[MediaInfo] = None +) -> MusicAudioIntegrityResult: """Verify a downloaded music audio file after it is playable/decrypted.""" path = Path(path) if not path.exists(): diff --git a/unshackle/core/music/planner.py b/unshackle/core/music/planner.py index 9f6797f1..2ac0bcde 100644 --- a/unshackle/core/music/planner.py +++ b/unshackle/core/music/planner.py @@ -21,7 +21,8 @@ class MusicPlanner: plan = MusicDownloadPlan( kind=getattr(music, "kind", "music"), title=getattr(music, "title", None) or (first_song.album if first_song else ""), - artist=getattr(music, "artist", None) or (first_song.album_artist or first_song.artist if first_song else ""), + artist=getattr(music, "artist", None) + or (first_song.album_artist or first_song.artist if first_song else ""), album_artist=(first_song.album_artist if first_song else "") or "", year=getattr(music, "year", None) or (first_song.year if first_song else None), released=self._first_text( @@ -180,9 +181,14 @@ class MusicPlanner: if any(option.atmos for option in options): return "Dolby Atmos" - if any(option.hires and (option.lossless or str(option.codec or "").upper() in {"FLAC", "ALAC", "WAV", "AIFF"}) for option in options): + if any( + option.hires and (option.lossless or str(option.codec or "").upper() in {"FLAC", "ALAC", "WAV", "AIFF"}) + for option in options + ): return "Hi-Res Lossless" - if any(option.lossless or str(option.codec or "").upper() in {"FLAC", "ALAC", "WAV", "AIFF"} for option in options): + if any( + option.lossless or str(option.codec or "").upper() in {"FLAC", "ALAC", "WAV", "AIFF"} for option in options + ): return "Lossless" if "AAC" in codecs: return "AAC" diff --git a/unshackle/core/music/renderer.py b/unshackle/core/music/renderer.py index 6bfaf44c..65674aa0 100644 --- a/unshackle/core/music/renderer.py +++ b/unshackle/core/music/renderer.py @@ -57,9 +57,7 @@ class MusicRenderer: metadata.get("released_at"), ) ) - length = self._format_total_duration( - getattr(music, "total_duration", None) or self._sum_duration(music) - ) + length = self._format_total_duration(getattr(music, "total_duration", None) or self._sum_duration(music)) quality = self._quality_summary( self._first_text( getattr(music, "quality", None), diff --git a/unshackle/core/music/tagger.py b/unshackle/core/music/tagger.py index 42172361..f756f15b 100644 --- a/unshackle/core/music/tagger.py +++ b/unshackle/core/music/tagger.py @@ -15,8 +15,23 @@ except ImportError: # pragma: no cover - optional artwork enhancement try: from mutagen.flac import FLAC, Picture - from mutagen.id3 import (APIC, COMM, TALB, TCOM, TCON, TCOP, TDRC, TIT2, TPE1, TPE2, TPOS, TRCK, TSRC, TXXX, - ID3NoHeaderError) + from mutagen.id3 import ( + APIC, + COMM, + TALB, + TCOM, + TCON, + TCOP, + TDRC, + TIT2, + TPE1, + TPE2, + TPOS, + TRCK, + TSRC, + TXXX, + ID3NoHeaderError, + ) from mutagen.mp3 import MP3 from mutagen.mp4 import MP4, MP4Cover except ImportError: # pragma: no cover - optional tagging dependency diff --git a/unshackle/core/providers/__init__.py b/unshackle/core/providers/__init__.py index 5ec73435..85bdc667 100644 --- a/unshackle/core/providers/__init__.py +++ b/unshackle/core/providers/__init__.py @@ -6,14 +6,15 @@ import requests from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, fuzzy_match, log from unshackle.core.providers.imdbapi import IMDBApiProvider +from unshackle.core.providers.omdb import OMDBProvider from unshackle.core.providers.simkl import SimklProvider from unshackle.core.providers.tmdb import TMDBProvider if TYPE_CHECKING: from unshackle.core.title_cacher import TitleCacher -# Ordered by priority: IMDBApi (free), SIMKL, TMDB -ALL_PROVIDERS: list[type[MetadataProvider]] = [IMDBApiProvider, SimklProvider, TMDBProvider] +# Ordered by priority: IMDBApi (free), OMDb, SIMKL, TMDB +ALL_PROVIDERS: list[type[MetadataProvider]] = [IMDBApiProvider, OMDBProvider, SimklProvider, TMDBProvider] def get_available_providers() -> list[MetadataProvider]: @@ -390,6 +391,24 @@ def _cached_to_result(cached: dict, provider_name: str, kind: str) -> Optional[M source="simkl", raw=cached, ) + elif provider_name == "omdb": + title = cached.get("Title") + year_str = cached.get("Year") + year = int(year_str[:4]) if year_str and len(year_str) >= 4 and year_str[:4].isdigit() else None + enriched = cached.get("_enriched_ids", {}) + return MetadataResult( + title=title, + year=year, + kind=kind, + external_ids=ExternalIds( + imdb_id=cached.get("imdbID"), + tmdb_id=enriched.get("tmdb_id"), + tmdb_kind=enriched.get("tmdb_kind"), + tvdb_id=enriched.get("tvdb_id"), + ), + source="omdb", + raw=cached, + ) elif provider_name == "imdbapi": title = cached.get("primaryTitle") or cached.get("originalTitle") year = cached.get("startYear") diff --git a/unshackle/core/providers/imdbapi.py b/unshackle/core/providers/imdbapi.py index b8189277..b57bbd95 100644 --- a/unshackle/core/providers/imdbapi.py +++ b/unshackle/core/providers/imdbapi.py @@ -5,9 +5,10 @@ from typing import Optional, Union import requests +from unshackle.core.config import config from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, fuzzy_match -# Mapping from our kind ("movie"/"tv") to imdbapi.dev title types +# Mapping from our kind ("movie"/"tv") to IMDxAPI title types KIND_TO_TYPES: dict[str, list[str]] = { "movie": ["movie"], "tv": ["tvSeries", "tvMiniSeries"], @@ -15,14 +16,14 @@ KIND_TO_TYPES: dict[str, list[str]] = { class IMDBApiProvider(MetadataProvider): - """IMDb metadata provider using imdbapi.dev (free, no API key).""" + """IMDb metadata provider using IMDxAPI (api.tiffara.com, free, no API key).""" NAME = "imdbapi" REQUIRES_KEY = False - BASE_URL = "https://api.imdbapi.dev" + BASE_URL = "https://api.tiffara.com" def is_available(self) -> bool: - return True # no key needed + return config.imdb_api_enabled def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: self.log.debug("Searching IMDBApi for %r (%s, %s)", title, kind, year) diff --git a/unshackle/core/providers/omdb.py b/unshackle/core/providers/omdb.py new file mode 100644 index 00000000..8f450f84 --- /dev/null +++ b/unshackle/core/providers/omdb.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from difflib import SequenceMatcher +from typing import Optional, Union + +import requests + +from unshackle.core.config import config +from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, fuzzy_match + +# Mapping from our kind ("movie"/"tv") to OMDb title types +KIND_TO_TYPE: dict[str, str] = { + "movie": "movie", + "tv": "series", +} + + +def _parse_year(value: Optional[str]) -> Optional[int]: + # OMDb years look like "2017", "2008–2013" or "2023–" + if value and len(value) >= 4 and value[:4].isdigit(): + return int(value[:4]) + return None + + +class OMDBProvider(MetadataProvider): + """OMDb metadata provider (omdbapi.com). Native IDs are IMDb IDs.""" + + NAME = "omdb" + REQUIRES_KEY = True + BASE_URL = "https://www.omdbapi.com/" + + def is_available(self) -> bool: + return bool(config.omdb_api_key) + + @property + def _api_key(self) -> str: + return config.omdb_api_key + + def _get(self, params: dict[str, str]) -> Optional[dict]: + try: + r = self.session.get(self.BASE_URL, params={"apikey": self._api_key, **params}, timeout=30) + r.raise_for_status() + data = r.json() + except (requests.RequestException, ValueError) as exc: + self.log.debug("OMDb request failed: %s", exc) + return None + if data.get("Response") != "True": + self.log.debug("OMDb error: %s", data.get("Error")) + return None + return data + + def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: + self.log.debug("Searching OMDb for %r (%s, %s)", title, kind, year) + + params: dict[str, str] = {"s": title} + type_filter = KIND_TO_TYPE.get(kind) + if type_filter: + params["type"] = type_filter + if year is not None: + params["y"] = str(year) + + data = self._get(params) + if not data and year is not None: + # OMDb's year filter is exact; retry without it for off-by-one metadata + del params["y"] + data = self._get(params) + if not data: + return None + + results = data.get("Search") or [] + + best_match: Optional[dict] = None + best_ratio = 0.0 + for candidate in results: + name = candidate.get("Title") or "" + if not name: + continue + ratio = SequenceMatcher(None, _clean(title), _clean(name)).ratio() + if ratio > best_ratio: + candidate_year = _parse_year(candidate.get("Year")) + if year and candidate_year and abs(year - candidate_year) > 1: + continue + best_ratio = ratio + best_match = candidate + + if not best_match: + self.log.debug("No matching result found in OMDb for %r", title) + return None + + result_title = best_match.get("Title") + if not result_title or not fuzzy_match(result_title, title): + self.log.debug("OMDb title mismatch: searched %r, got %r", title, result_title) + return None + + imdb_id = best_match.get("imdbID") + self.log.debug("OMDb -> %s (ID %s)", result_title, imdb_id) + + # Fetch full detail so raw carries ratings, genre, rating cert, etc. + detail = self._get({"i": imdb_id}) if imdb_id else None + + return MetadataResult( + title=result_title, + year=_parse_year(best_match.get("Year")), + kind=kind, + external_ids=ExternalIds(imdb_id=imdb_id), + source="omdb", + raw=detail or best_match, + ) + + def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]: + """Fetch metadata by IMDb ID (e.g. 'tt1375666').""" + imdb_id = str(provider_id) + self.log.debug("Fetching OMDb title %s", imdb_id) + + data = self._get({"i": imdb_id}) + if not data: + return None + + return MetadataResult( + title=data.get("Title"), + year=_parse_year(data.get("Year")), + kind=kind, + external_ids=ExternalIds(imdb_id=data.get("imdbID")), + source="omdb", + raw=data, + ) + + def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds: + """Return external IDs. For OMDb, the provider_id IS the IMDb ID.""" + return ExternalIds(imdb_id=str(provider_id)) diff --git a/unshackle/core/proxies/expressvpn.py b/unshackle/core/proxies/expressvpn.py index 43061a6b..a26d1529 100644 --- a/unshackle/core/proxies/expressvpn.py +++ b/unshackle/core/proxies/expressvpn.py @@ -1,17 +1,15 @@ from __future__ import annotations import base64 -import hashlib import json import logging import os import random import re -import secrets +import sys import time from pathlib import Path -from typing import Optional -from urllib.parse import parse_qs, urlparse +from typing import Any, Optional import requests @@ -25,27 +23,25 @@ class ExpressVPN(Proxy): """ ExpressVPN HTTPS proxy provider. - This provider follows the browser extension proxy flow: - browser cookies or a cached refresh token are exchanged for API tokens, - those tokens are used to resolve proxy-capable locations, and get_proxy() - returns an authenticated HTTPS proxy URL. + Mirrors the ExpressVPN Android TV app: an OAuth 2.0 device authorization grant obtains + API tokens (interactive on first run, silent via cached refresh token afterwards), a + subscription receipt (SRT) resolves proxy-capable locations, and get_proxy() returns an + authenticated HTTPS proxy URL (https://cat:<token>@server:443). - Query format: - country -- smart connection (random location) - country-city -- specific city - country-city-N / cityN -- pinned server by position - full-slug -- ExpressVPN location slug - hostname.expressprovider.com -- direct hostname + Query format (after the provider prefix, e.g. "expressvpn:ca"): + ca random location in the country (smart connection) + us-ny city + us-ny-2 / usny2 pinned server by position + usa-new-york full ExpressVPN location slug + host.expressprovider.com direct hostname """ - CLIENT_ID = "f457fed092a54b9e9f1e2113782d74a2" - AUTH_BASE = "https://auth.expressvpn.com/realms/xvpn/protocol/openid-connect" + CLIENT_ID = "8V18PnFYlrYnnYvRlnxKwifxxhYKfjIG" + AUTH_BASE = "https://auth.expressvpn.com/oauth" API_BASE = "https://cp.expressapisv2.net" - EXTENSION_REDIRECT_URI = "chrome-extension://fgddmllnllkalaagkghckoinaemmogpe/src/html/auth-callback.html" - USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - ) + SCOPE = "openid profile email offline_access" + DEVICE_POLL_TIMEOUT = 600 + USER_AGENT = "okhttp/5.3.2" def __init__( self, @@ -55,436 +51,246 @@ class ExpressVPN(Proxy): access_token: Optional[str] = None, connection_token: Optional[str] = None, account_json: Optional[str] = None, - cookie_path: Optional[str] = None, cache_path: Optional[str] = None, timeout: float = 10.0, + enable: bool = False, ): """ - Proxy Service using ExpressVPN browser-extension proxy credentials. + Proxy Service using ExpressVPN app proxy credentials. - Args: - region_map: Optional country-to-preset mapping. Keys are country - codes (e.g. ``"us"``), values are optional city/server presets - (e.g. ``"ny-02"``). When a key is used without a city in the - CLI query, the preset is applied. Empty/null values enable - smart connection (random location in that country). - server_map: Optional aliases using the same convention as other - proxy providers. Values may be ExpressVPN location slugs or - concrete .expressprovider.com hosts. - refresh_token: Optional OAuth refresh token. If omitted, cached - tokens or browser cookies are used. - access_token: Optional OAuth access token for advanced/manual use. - connection_token: Optional cached connection authorization token. - account_json: Optional path to ExpressVPN desktop account.json. - cookie_path: Optional path to exported ExpressVPN browser cookies. - cache_path: Optional path for cached ExpressVPN tokens. - timeout: Request timeout in seconds. + region_map maps a country code to an optional city/server preset (e.g. "us": "ny-02") + applied when the CLI query names only the country; empty values mean smart connection. + server_map maps aliases to location slugs or concrete .expressprovider.com hosts. + Tokens are optional: without them, cached tokens are used, else account_json is read. + Set enable: true to run the one-time interactive device login when no session exists. """ if region_map is not None and not isinstance(region_map, dict): raise TypeError(f"Expected region_map to be a dict mapping aliases to locations, not '{region_map!r}'.") if server_map is not None and not isinstance(server_map, dict): raise TypeError(f"Expected server_map to be a dict mapping aliases to locations, not '{server_map!r}'.") - # region_map: country code → optional preset (city[-server]) self.region_map = { - str(k).lower().strip(): (str(v).lower().strip() if v else None) - for k, v in (region_map or {}).items() - } - # server_map: legacy alias → location slug or direct hostname - self.server_map = { - str(k).lower().strip(): str(v).lower().strip() - for k, v in (server_map or {}).items() + str(k).lower().strip(): (str(v).lower().strip() if v else None) for k, v in (region_map or {}).items() } + self.server_map = {str(k).lower().strip(): str(v).lower().strip() for k, v in (server_map or {}).items()} self.refresh_token = refresh_token or None self.access_token = access_token or None self.connection_token = connection_token or None self.account_json = Path(account_json).expanduser() if account_json else None self.timeout = timeout - self._tokens: Optional[dict] = None - self._srt: Optional[str] = None - self._locations: Optional[list[dict]] = None + self.enable = enable - # Display info set during _resolve_endpoint for log messages - self._last_location_name: Optional[str] = None - self._last_server_index: Optional[int] = None - self._last_server_total: Optional[int] = None - self._last_endpoint_host: Optional[str] = None + self.tokens: Optional[dict] = None + self.srt: Optional[str] = None + self.locations: Optional[list[dict]] = None + self.last_name: Optional[str] = None + self.last_index: Optional[int] = None + self.last_total: Optional[int] = None + self.last_host: Optional[str] = None - self.cookie_path = Path(cookie_path).expanduser() if cookie_path else self._default_cookie_path() - self.cache_path = Path(cache_path).expanduser() if cache_path else self._default_cache_path() + self.cache_path = Path(cache_path).expanduser() if cache_path else self.default_cache_path() def __repr__(self) -> str: - if self._locations is not None: - locations = self._locations - countries = len({ - str(loc.get("country_code") or "").upper() - for loc in locations if loc.get("country_code") - }) - servers = len(locations) + if self.locations is None and self.has_silent_session(): + try: + self.get_locations() + except (requests.RequestException, ValueError) as error: + log.debug("ExpressVPN: could not fetch locations for status line: %s", error) + if self.locations: + countries = len({str(x.get("country_code") or "").upper() for x in self.locations if x.get("country_code")}) + locations = len(self.locations) return ( f"{countries} Countr{'ies' if countries != 1 else 'y'} " - f"({servers} Server{'s' if servers != 1 else ''})" + f"({locations} Location{'s' if locations != 1 else ''})" ) - alias_count = len(self.region_map) + len(self.server_map) - if alias_count: - return f"{alias_count} Region Alias{'es' if alias_count != 1 else ''} (ExpressVPN HTTPS Proxy)" + aliases = len(self.region_map) + len(self.server_map) + if aliases: + return f"{aliases} Region Alias{'es' if aliases != 1 else ''} (ExpressVPN HTTPS Proxy)" return "ExpressVPN HTTPS Proxy" + def has_silent_session(self) -> bool: + """Whether tokens can be obtained without the interactive device login.""" + return bool( + self.tokens + or self.access_token + or self.refresh_token + or self.connection_token + or (self.account_json and self.account_json.is_file()) + or self.load_cache() + ) + def get_proxy(self, query: str) -> Optional[str]: - query = query.strip().lower() - endpoint = self._resolve_endpoint(query) + endpoint = self.resolve_endpoint(query.strip().lower()) if not endpoint: return None - connection_token = self._get_connection_token() + connection_token = self.get_connection_token() if not connection_token: log.error("ExpressVPN: connection token was not available") return None - display = self.last_connection_display() - log.debug("ExpressVPN proxy ready: %s", display or f"https://cat:***@{endpoint}:443") + log.debug("ExpressVPN proxy ready: %s", self.last_connection_display() or endpoint) return f"https://cat:{connection_token}@{endpoint}:443" def last_connection_display(self) -> Optional[str]: - """Return a human-readable string describing the last resolved connection. - - Used by the download/search commands to display a friendly log message - instead of a raw sanitized proxy URL. - - Returns: - A string like ``"(USA - New York, #3 of 5): .214"`` or None if - no connection has been resolved yet. - """ - if not self._last_location_name or not self._last_endpoint_host: + if not self.last_name or not self.last_host: return None + parts = [self.last_name] + if self.last_index is not None and self.last_total is not None: + parts.append(f"#{self.last_index} of {self.last_total}") + return f"({', '.join(parts)}): {self.last_host.replace('.expressprovider.com', '')}" - # Strip the .expressprovider.com suffix for readability - short_host = self._last_endpoint_host.replace(".expressprovider.com", "") + def resolve_endpoint(self, query: str) -> Optional[str]: + self.last_name = self.last_index = self.last_total = self.last_host = None + query = self.server_map.get(query) or query - parts = [self._last_location_name] - if self._last_server_index is not None and self._last_server_total is not None: - parts.append(f"#{self._last_server_index} of {self._last_server_total}") - - return f"({', '.join(parts)}): {short_host}" - - # ------------------------------------------------------------------ - # Endpoint Resolution - # ------------------------------------------------------------------ - - def _resolve_endpoint(self, query: str) -> Optional[str]: - # Reset display info - self._last_location_name = None - self._last_server_index = None - self._last_server_total = None - self._last_endpoint_host = None - - # 1) Check legacy server_map first (direct hostname / slug aliases) - mapped = self.server_map.get(query) - if mapped: - query = mapped - - # 2) Direct hostname pass-through if "expressprovider" in query: - host = query if query.endswith(".expressprovider.com") else f"{query}.expressprovider.com" - self._last_endpoint_host = host - return host + self.last_host = query if query.endswith(".expressprovider.com") else f"{query}.expressprovider.com" + return self.last_host - # 3) Parse the query into (country, city, server_num) - country, city, server_num = self._parse_query(query) + country, city, server_num = self.parse_query(query) + if country and not city and (preset := self.region_map.get(country)): + _, city, server_num = self.parse_query(f"{country}-{preset}") - # 4) If the query matched a country in region_map and no city was - # specified in the CLI, apply the preset from region_map - if country and not city: - preset = self.region_map.get(country) - if preset: - # Preset is like "ny-02" or "ny" or "ny2" - _, preset_city, preset_server = self._parse_query(f"{country}-{preset}") - city = preset_city - server_num = preset_server - - # 5) Resolve the location if country: - location = self._resolve_location_by_country(country, city) + location = self.resolve_location(country, city) else: - # Fallback: try the full query as a location slug - location = self._resolve_location_by_slug(query) - + location = self.resolve_slug(query) if not location: log.warning("ExpressVPN: no location matched query '%s'", query) return None - self._last_location_name = location.get("name") - - # 6) Get endpoints and pick server - endpoints = self._get_endpoints_for_location(location) + self.last_name = location.get("name") + endpoints = self.get_endpoints(location) if not endpoints: log.error("ExpressVPN: no proxy endpoints returned for %s", location.get("name") or query) return None + return self.pick_endpoint(endpoints, server_num) - host = self._pick_endpoint(endpoints, server_num) - return host - - def _parse_query(self, query: str) -> tuple[Optional[str], Optional[str], Optional[int]]: - """Parse a query string into (country_code, city_query, server_number). - - Supported formats:: - - us -> ("us", None, None) - us-ny -> ("us", "ny", None) - us-ny-2 -> ("us", "ny", 2) - us-ny2 -> ("us", "ny", 2) - us-ny-02 -> ("us", "ny", 2) - usa-new-york -> (None, None, None) -- treated as slug - mx -> ("mx", None, None) - - When the query doesn't look like a country[-city] pattern, all - fields are returned as None and the caller falls back to slug - resolution. - """ - # Try to extract a trailing server number: "us-ny-2" or "us-ny2" + def parse_query(self, query: str) -> tuple[Optional[str], Optional[str], Optional[int]]: + """Parse "us", "us-ny", "us-ny-2"/"us-ny2" into (country, city, server number).""" server_num = None - base_query = query - - # Match trailing -N or trailing digits glued to alpha num_match = re.match(r"^(.+?)[-]?(\d+)$", query) - if num_match: - candidate_base = num_match.group(1).rstrip("-") - candidate_num = int(num_match.group(2)) - # Only treat as server number if the base contains alpha characters - # and the number is reasonable (1-99) - if re.search(r"[a-z]", candidate_base) and 1 <= candidate_num <= 99: - base_query = candidate_base - server_num = candidate_num + if num_match and re.search(r"[a-z]", num_match.group(1).rstrip("-")) and 1 <= int(num_match.group(2)) <= 99: + query = num_match.group(1).rstrip("-") + server_num = int(num_match.group(2)) - # Try to split as country-city - # Country codes are 2 letters; city is the rest - country_match = re.match(r"^([a-z]{2})(?:-(.+))?$", base_query) - if country_match: - country = country_match.group(1) - city = country_match.group(2) # may be None + match = re.match(r"^([a-z]{2})(?:-(.+))?$", query) + if match: + valid = {str(x.get("country_code") or "").lower() for x in self.get_locations()} + if match.group(1) in valid: + return match.group(1), match.group(2) or None, server_num - # Verify this is a valid country code by checking locations - locations = self._get_locations() - valid_countries = {str(loc.get("country_code") or "").lower() for loc in locations} - if country in valid_countries: - return country, city or None, server_num - - # Not a country-city pattern; return None to trigger slug fallback + # not a country[-city] pattern; caller falls back to slug resolution return None, None, None - def _resolve_location_by_country( - self, country_code: str, city_query: Optional[str] = None - ) -> Optional[dict]: - """Find a location by country code and optional city query. - - If *city_query* is None, a random location in the country is selected - (smart connection). Otherwise, the city is matched against location - names using abbreviation, prefix, and substring strategies. - """ - locations = self._get_locations() - if not locations: - return None - - # Filter locations for this country - country_locations = [ - loc for loc in locations - if str(loc.get("country_code") or "").lower() == country_code - ] - if not country_locations: + def resolve_location(self, country_code: str, city_query: Optional[str] = None) -> Optional[dict]: + pool = [x for x in self.get_locations() if str(x.get("country_code") or "").lower() == country_code] + if not pool: log.warning("ExpressVPN: no locations found for country '%s'", country_code.upper()) return None - if not city_query: - # Smart connection: random location in the country - return random.choice(country_locations) + return random.choice(pool) + return self.match_city(pool, city_query) - # Match city query against location names - return self._match_city(country_locations, city_query) - - def _match_city(self, locations: list[dict], city_query: str) -> Optional[dict]: - """Match a short city query against a list of locations. - - Matching strategies (in priority order): - 1. First-letter abbreviation: ``"ny"`` matches ``"New York"`` - 2. Exact slug match: ``"miami"`` matches ``"miami"`` - 3. Prefix match on slug: ``"mia"`` matches ``"miami"`` - 4. Substring match on slug: ``"york"`` matches ``"new-york"`` - - Returns the first match from the highest-priority non-empty bucket. - """ + def match_city(self, locations: list[dict], city_query: str) -> Optional[dict]: + """Match a city query by abbreviation ("ny"), exact slug, slug prefix, then substring.""" city_query = city_query.strip().lower() - abbreviation_hits: list[dict] = [] - exact_hits: list[dict] = [] - prefix_hits: list[dict] = [] - substring_hits: list[dict] = [] - + abbrev, exact, prefix, substring = [], [], [], [] for loc in locations: - full_name = str(loc.get("name") or "") - # Extract city part: strip country prefix like "USA - " - city_part = re.sub(r"^[A-Z]{2,}(?:\s*-\s*)", "", full_name, count=1).strip() - if not city_part: - city_part = full_name - - city_slug = _slugify(city_part) - - # Strategy 1: first-letter abbreviation - words = re.findall(r"[a-zA-Z]+", city_part) - abbreviation = "".join(w[0] for w in words).lower() if words else "" + # location names look like "USA - New York"; keep the city part + city = re.sub(r"^[A-Z]{2,}(?:\s*-\s*)", "", str(loc.get("name") or ""), count=1).strip() + slug = slugify(city) + abbreviation = "".join(w[0] for w in re.findall(r"[a-zA-Z]+", city)).lower() if city_query == abbreviation: - abbreviation_hits.append(loc) - continue + abbrev.append(loc) + elif city_query == slug: + exact.append(loc) + elif slug.startswith(city_query): + prefix.append(loc) + elif city_query in slug: + substring.append(loc) - # Strategy 2: exact slug match - if city_query == city_slug: - exact_hits.append(loc) - continue - - # Strategy 3: prefix match on slug - if city_slug.startswith(city_query): - prefix_hits.append(loc) - continue - - # Strategy 4: substring match on slug - if city_query in city_slug: - substring_hits.append(loc) - - # Return the first match from the highest-priority non-empty bucket - candidates = abbreviation_hits or exact_hits or prefix_hits or substring_hits + candidates = abbrev or exact or prefix or substring if not candidates: log.warning("ExpressVPN: no city matched '%s' in available locations", city_query) return None - if len(candidates) > 1: - names = [c.get("name") for c in candidates] - log.debug("ExpressVPN: city '%s' matched %d locations: %s", city_query, len(candidates), names) - + log.debug("ExpressVPN: city '%s' matched %d locations", city_query, len(candidates)) return candidates[0] - def _resolve_location_by_slug(self, query: str) -> Optional[dict]: - """Resolve a location by matching the full query against slugified names, IDs, or country codes.""" - target = query.strip().lower() - locations = self._get_locations() - if not locations: - return None - - for location in locations: - location_id = str(location.get("id") or "").lower() - location_name = str(location.get("name") or "").lower() - country_code = str(location.get("country_code") or "").lower() - slug = _slugify(location_name) - if target in (location_id, location_name, country_code, slug): - return location + def resolve_slug(self, query: str) -> Optional[dict]: + for loc in self.get_locations(): + name = str(loc.get("name") or "").lower() + keys = (str(loc.get("id") or "").lower(), name, str(loc.get("country_code") or "").lower(), slugify(name)) + if query in keys: + return loc return None - def _pick_endpoint(self, endpoints: list[dict], server_num: Optional[int] = None) -> Optional[str]: - """Select an endpoint host from the list, optionally by position index. - - Args: - endpoints: List of endpoint dicts with ``"host"`` keys. - server_num: 1-based server position index, or None for random. - - Returns: - The selected hostname, or None if no endpoints are available. - """ - hosts = [str(ep.get("host") or "").lower() for ep in endpoints if ep.get("host")] + def pick_endpoint(self, endpoints: list[dict], server_num: Optional[int] = None) -> Optional[str]: + hosts = [str(x.get("host") or "").lower() for x in endpoints if x.get("host")] if not hosts: return None - total = len(hosts) - self._last_server_total = total + self.last_total = total = len(hosts) + if server_num is not None and not 1 <= server_num <= total: + log.warning( + "ExpressVPN: server #%d not available (%d server%s), picking random", + server_num, + total, + "s" if total != 1 else "", + ) + server_num = None + self.last_index = server_num if server_num is not None else random.randint(1, total) + self.last_host = hosts[self.last_index - 1] + return self.last_host - if server_num is not None: - if server_num < 1 or server_num > total: - log.warning( - "ExpressVPN: server #%d is not available (only %d server%s), selecting random", - server_num, total, "s" if total != 1 else "", - ) - chosen = random.choice(hosts) - self._last_server_index = hosts.index(chosen) + 1 - else: - chosen = hosts[server_num - 1] - self._last_server_index = server_num - else: - chosen = random.choice(hosts) - self._last_server_index = hosts.index(chosen) + 1 + def get_locations(self) -> list[dict]: + if self.locations is None: + srt = self.get_srt() + if not srt: + return [] + response = self.request( + "GET", f"{self.API_BASE}/ids2/locations", headers=self.api_headers(srt), params={"protocols": "proxy"} + ) + self.locations = response.json().get("locations", []) + return self.locations - self._last_endpoint_host = chosen - return chosen - - # ------------------------------------------------------------------ - # Locations & Endpoints API - # ------------------------------------------------------------------ - - def _get_locations(self) -> list[dict]: - if self._locations is not None: - return self._locations - - srt = self._get_srt() + def get_endpoints(self, location: dict) -> list[dict]: + srt = self.get_srt() if not srt: return [] - - response = self._request( - "GET", - f"{self.API_BASE}/ids2/locations", - headers=self._api_headers(srt), - params={"protocols": "proxy"}, - ) - self._locations = response.json().get("locations", []) - return self._locations - - def _get_endpoints_for_location(self, location: dict) -> list[dict]: - srt = self._get_srt() - if not srt: - return [] - - location_id = location.get("id") - response = self._request( + response = self.request( "POST", - f"{self.API_BASE}/ids2/locations/{location_id}/instances", - headers=self._api_headers(srt), + f"{self.API_BASE}/ids2/locations/{location.get('id')}/instances", + headers=self.api_headers(srt), params={"protocols": "proxy"}, json={}, ) return response.json().get("endpoints", []) - # ------------------------------------------------------------------ - # Token Management - # ------------------------------------------------------------------ + def get_tokens(self) -> dict: + if self.tokens and not jwt_expired(self.tokens.get("access_token")): + return self.tokens - def _get_tokens(self) -> dict: - # Return in-memory cache if access_token is still valid - if self._tokens: - access_token = self._tokens.get("access_token") - if access_token and not self._is_jwt_expired(access_token): - return self._tokens - - cached = self._load_cached_tokens() - tokens = { - **cached, - **{k: v for k, v in { - "access_token": self.access_token, - "refresh_token": self.refresh_token, - "connection_token": self.connection_token, - }.items() if v}, + overrides = { + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "connection_token": self.connection_token, } + tokens = {**self.load_cache(), **{k: v for k, v in overrides.items() if v}} - access_token = tokens.get("access_token") - if access_token and not self._is_jwt_expired(access_token): - self._tokens = tokens + if tokens.get("access_token") and not jwt_expired(tokens["access_token"]): + self.tokens = tokens return tokens - refresh_token = tokens.get("refresh_token") - if refresh_token: - refreshed = self._refresh_access_token(refresh_token) + if refresh_token := tokens.get("refresh_token"): + refreshed = self.refresh_access_token(refresh_token) if refreshed: - tokens.update( - { - "access_token": refreshed.get("access_token"), - "refresh_token": refreshed.get("refresh_token") or refresh_token, - } - ) - self._save_cached_tokens(tokens) - self._tokens = tokens + # refresh tokens rotate; keep the new one, fall back to the old if omitted + tokens["access_token"] = refreshed.get("access_token") + tokens["refresh_token"] = refreshed.get("refresh_token") or refresh_token + self.save_cache(tokens) return tokens log.warning("ExpressVPN: refresh token failed or session expired") @@ -495,43 +301,34 @@ class ExpressVPN(Proxy): log.error("ExpressVPN: failed to read account_json %s: %s", self.account_json, error) else: tokens.update( - { - "access_token": data.get("accessToken"), - "connection_token": data.get("connectionToken"), - "subscription_id": data.get("subscriptionId"), - } + access_token=data.get("accessToken"), + connection_token=data.get("connectionToken"), + subscription_id=data.get("subscriptionId"), ) - tokens = {k: v for k, v in tokens.items() if v} - self._tokens = tokens + self.tokens = {k: v for k, v in tokens.items() if v} + return self.tokens + + if self.enable and sys.stdin.isatty(): + if bootstrapped := self.device_login(): + tokens["access_token"] = bootstrapped.get("access_token") + tokens["refresh_token"] = bootstrapped.get("refresh_token") or tokens.get("refresh_token") + self.save_cache(tokens) return tokens + elif not tokens.get("access_token"): + log.error( + "ExpressVPN: no session available; set proxy_providers.expressvpn.enable: true " + "for the one-time device login" + ) - if self.cookie_path.is_file(): - bootstrapped = self._run_pkce_bootstrap() - if bootstrapped: - tokens.update( - { - "access_token": bootstrapped.get("access_token"), - "refresh_token": bootstrapped.get("refresh_token"), - } - ) - self._save_cached_tokens(tokens) - self._tokens = tokens - return tokens + self.tokens = {k: v for k, v in tokens.items() if v} + return self.tokens - tokens = {k: v for k, v in tokens.items() if v} - self._tokens = tokens - return tokens - - def _refresh_access_token(self, refresh_token: str) -> Optional[dict]: - response = self._request( + def refresh_access_token(self, refresh_token: str) -> Optional[dict]: + response = self.request( "POST", f"{self.AUTH_BASE}/token", - data={ - "grant_type": "refresh_token", - "client_id": self.CLIENT_ID, - "refresh_token": refresh_token, - }, - headers=self._form_headers(), + data={"grant_type": "refresh_token", "client_id": self.CLIENT_ID, "refresh_token": refresh_token}, + headers=self.form_headers(), allow_error=True, ) if not response.ok: @@ -539,165 +336,111 @@ class ExpressVPN(Proxy): return None return response.json() - def _run_pkce_bootstrap(self) -> Optional[dict]: - cookies = self._load_cookies() - keycloak_identity = cookies.get("KEYCLOAK_IDENTITY") - keycloak_session = cookies.get("KEYCLOAK_SESSION") - auth_session_id = cookies.get("AUTH_SESSION_ID") - if not keycloak_identity or not keycloak_session: - log.error("ExpressVPN: KEYCLOAK_IDENTITY or KEYCLOAK_SESSION is missing from %s", self.cookie_path) - return None - - verifier = base64.urlsafe_b64encode( - secrets.token_bytes(32) - ).decode("utf-8").rstrip("=") - challenge = base64.urlsafe_b64encode( - hashlib.sha256(verifier.encode("utf-8")).digest() - ).decode("utf-8").rstrip("=") - auth_cookies = requests.cookies.RequestsCookieJar() - auth_cookies.set("KEYCLOAK_IDENTITY", keycloak_identity, domain="auth.expressvpn.com") - auth_cookies.set("KEYCLOAK_SESSION", keycloak_session, domain="auth.expressvpn.com") - if auth_session_id: - auth_cookies.set("AUTH_SESSION_ID", auth_session_id, domain="auth.expressvpn.com") - - response = self._request( - "GET", - f"{self.AUTH_BASE}/auth", - params={ - "client_id": self.CLIENT_ID, - "response_type": "code", - "redirect_uri": self.EXTENSION_REDIRECT_URI, - "scope": "profile offline_access", - "code_challenge": challenge, - "code_challenge_method": "S256", - "ui_locales": "en", - }, - headers=self._browser_headers(), - cookies=auth_cookies, - allow_redirects=False, - allow_error=True, - ) - - redirect_url = response.headers.get("Location", "") - code = parse_qs(urlparse(redirect_url).query).get("code", [None])[0] - if not code: - log.error("ExpressVPN: OAuth bootstrap did not return an authorization code") - return None - - token_response = self._request( + def device_login(self) -> Optional[dict]: + """One-time interactive bootstrap via the OAuth 2.0 device authorization grant (RFC 8628).""" + response = self.request( "POST", - f"{self.AUTH_BASE}/token", - data={ - "grant_type": "authorization_code", - "client_id": self.CLIENT_ID, - "code": code, - "redirect_uri": self.EXTENSION_REDIRECT_URI, - "code_verifier": verifier, - }, - headers=self._form_headers(), + f"{self.AUTH_BASE}/device/code", + data={"client_id": self.CLIENT_ID, "scope": self.SCOPE, "audience": ""}, + headers=self.form_headers(), allow_error=True, ) - if not token_response.ok: - log.error("ExpressVPN: OAuth token exchange failed with HTTP %s", token_response.status_code) + if not response.ok: + log.error("ExpressVPN: device code request failed with HTTP %s", response.status_code) return None - return token_response.json() + data = response.json() - def _get_srt(self) -> Optional[str]: - if self._srt and not self._is_jwt_expired(self._srt): - return self._srt + log.info( + "ExpressVPN: sign in to authorize this device: open %s and enter code %s", + data.get("verification_uri", f"{self.AUTH_BASE}/../realms/xvpn/device"), + data.get("user_code"), + ) - tokens = self._get_tokens() - cached_srt = tokens.get("srt") - if cached_srt and not self._is_jwt_expired(cached_srt): - self._srt = cached_srt - return self._srt + interval = int(data.get("interval") or 5) + deadline = time.time() + int(data.get("expires_in") or self.DEVICE_POLL_TIMEOUT) + while time.time() < deadline: + time.sleep(interval) + poll = self.request( + "POST", + f"{self.AUTH_BASE}/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "client_id": self.CLIENT_ID, + "device_code": data.get("device_code"), + }, + headers=self.form_headers(), + allow_error=True, + ) + if poll.ok: + return poll.json() + error = poll.json().get("error") + if error == "slow_down": + interval += 5 + elif error != "authorization_pending": + log.error("ExpressVPN: device authorization failed: %s", error) + return None + log.error("ExpressVPN: device authorization timed out before approval") + return None + + def get_srt(self) -> Optional[str]: + if self.srt and not jwt_expired(self.srt): + return self.srt + + tokens = self.get_tokens() + if (cached := tokens.get("srt")) and not jwt_expired(cached): + self.srt = cached + return cached access_token = tokens.get("access_token") if not access_token: log.error("ExpressVPN: access token was not available") return None - response = self._request( - "POST", - f"{self.API_BASE}/srs2/subscription_receipts", - headers=self._api_headers(access_token), - json={}, + response = self.request( + "POST", f"{self.API_BASE}/srs2/subscription_receipts", headers=self.api_headers(access_token), json={} ) - receipts = response.json().get("srts", []) - active_subscription_id = tokens.get("subscription_id") - srt = self._select_srt(receipts, active_subscription_id) + srt = self.select_srt(response.json().get("srts", []), tokens.get("subscription_id")) if not srt: log.error("ExpressVPN: no active subscription receipt was found") return None - self._srt = srt - tokens["srt"] = srt - self._save_cached_tokens(tokens) + self.srt = tokens["srt"] = srt + self.save_cache(tokens) return srt - def _get_connection_token(self) -> Optional[str]: - tokens = self._get_tokens() + def get_connection_token(self) -> Optional[str]: + tokens = self.get_tokens() connection_token = tokens.get("connection_token") - if connection_token and not self._is_jwt_expired(connection_token): + if connection_token and not jwt_expired(connection_token): return connection_token - auth_token = self._get_srt() or tokens.get("access_token") + auth_token = self.get_srt() or tokens.get("access_token") if not auth_token: return connection_token - response = self._request( - "POST", - f"{self.API_BASE}/srs2/connection_token", - headers=self._api_headers(auth_token), - json={}, + response = self.request( + "POST", f"{self.API_BASE}/srs2/connection_token", headers=self.api_headers(auth_token), json={} ) body = response.json() - connection_token = body.get("connection_token") or body.get("token") + # the API returns the token under "cat" + connection_token = body.get("cat") or body.get("connection_token") or body.get("token") if connection_token: tokens["connection_token"] = connection_token - self._save_cached_tokens(tokens) + self.save_cache(tokens) return connection_token - # ------------------------------------------------------------------ - # Cookies & Cache - # ------------------------------------------------------------------ + def select_srt(self, receipts: list[dict], subscription_id: Optional[str]) -> Optional[str]: + for receipt in receipts: + srt = receipt.get("srt") + if "xv.vpn" in jwt_payload(srt).get("entitlements", {}): + return srt + if subscription_id: + for receipt in receipts: + if receipt.get("subscription_id") == subscription_id: + return receipt.get("srt") + return receipts[0].get("srt") if receipts else None - def _load_cookies(self) -> dict[str, str]: - if not self.cookie_path.is_file(): - return {} - - try: - content = self.cookie_path.read_text(encoding="utf-8").strip() - except OSError as error: - log.error("ExpressVPN: failed to read cookies file %s: %s", self.cookie_path, error) - return {} - - cookies: dict[str, str] = {} - try: - data = json.loads(content) - except json.JSONDecodeError: - for line in content.splitlines(): - if not line.strip() or line.startswith("#"): - continue - parts = line.rstrip("\n").split("\t") - if len(parts) >= 7 and "expressvpn.com" in parts[0]: - cookies[parts[5]] = parts[6] - elif "=" in line: - name, value = line.split("=", maxsplit=1) - cookies[name.strip()] = value.strip() - else: - if isinstance(data, list): - for cookie in data: - if isinstance(cookie, dict) and "expressvpn.com" in str(cookie.get("domain", "")): - name = cookie.get("name") - value = cookie.get("value") - if name and value is not None: - cookies[str(name)] = str(value) - elif isinstance(data, dict): - cookies.update({str(k): str(v) for k, v in data.items()}) - return cookies - - def _load_cached_tokens(self) -> dict: + def load_cache(self) -> dict: if not self.cache_path.is_file(): return {} try: @@ -707,129 +450,72 @@ class ExpressVPN(Proxy): return {} return data if isinstance(data, dict) else {} - def _save_cached_tokens(self, tokens: dict) -> None: - # Invalidate in-memory cache so next _get_tokens() re-reads - self._tokens = tokens.copy() + def save_cache(self, tokens: dict) -> None: + self.tokens = tokens.copy() try: self.cache_path.parent.mkdir(parents=True, exist_ok=True) - payload = json.dumps({k: v for k, v in tokens.items() if v}, indent=2) - _write_private(self.cache_path, payload) + write_private(self.cache_path, json.dumps({k: v for k, v in tokens.items() if v}, indent=2)) except OSError as error: log.error("ExpressVPN: failed to save token cache %s: %s", self.cache_path, error) - # ------------------------------------------------------------------ - # HTTP & Helpers - # ------------------------------------------------------------------ - - def _request(self, method: str, url: str, allow_error: bool = False, **kwargs) -> requests.Response: + def request(self, method: str, url: str, allow_error: bool = False, **kwargs: Any) -> requests.Response: kwargs.setdefault("timeout", self.timeout) response = requests.request(method, url, **kwargs) if response.ok or allow_error: return response raise ValueError(f"ExpressVPN request failed with HTTP {response.status_code}: {url}") - def _api_headers(self, token: str) -> dict[str, str]: + def api_headers(self, token: str) -> dict[str, str]: return { "Authorization": f"Bearer {token}", "Accept": "application/json", "Content-Type": "application/json", "User-Agent": self.USER_AGENT, - "X-Client-App-Version": "12.0.0", - "X-Client-OS": "Windows", - "X-Client-Device-Model": "Browser Extension", + "X-Client-App-Version": "12.69.0", + "X-Client-OS": "Android", + "X-Client-Device-Model": "SHIELD Android TV", } - def _browser_headers(self) -> dict[str, str]: - return { - "User-Agent": self.USER_AGENT, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - } - - def _form_headers(self) -> dict[str, str]: + def form_headers(self) -> dict[str, str]: return { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", "User-Agent": self.USER_AGENT, } - def _select_srt(self, receipts: list[dict], active_subscription_id: Optional[str]) -> Optional[str]: - for receipt in receipts: - srt = receipt.get("srt") - if "xv.vpn" in self._decode_jwt_payload(srt).get("entitlements", {}): - return srt - - if active_subscription_id: - for receipt in receipts: - if receipt.get("subscription_id") == active_subscription_id: - return receipt.get("srt") - - if receipts: - return receipts[0].get("srt") - return None - - def _decode_jwt_payload(self, token: Optional[str]) -> dict: - if not token: - return {} - try: - parts = token.split(".") - if len(parts) < 2: - return {} - payload = parts[1] + "=" * ((4 - len(parts[1]) % 4) % 4) - return json.loads(base64.urlsafe_b64decode(payload).decode("utf-8")) - except (ValueError, json.JSONDecodeError): - return {} - - def _is_jwt_expired(self, token: str) -> bool: - expires_at = self._decode_jwt_payload(token).get("exp") - if not expires_at: - # Tokens without ``exp`` are treated as non-expiring. Keycloak - # access/refresh tokens always carry ``exp``; proprietary SRT/ - # connection tokens may omit it — they are refreshed via the - # normal SRT → connection_token flow anyway. - return False - return time.time() >= (int(expires_at) - 300) - - def _default_cookie_path(self) -> Path: - cookies_dir = Path(config.directories.cookies) - for folder in ("vpn", "vpns"): - candidate = cookies_dir / folder / "expressvpn.txt" - if candidate.is_file(): - return candidate - return cookies_dir / "vpn" / "expressvpn.txt" - - def _default_cache_path(self) -> Path: - cache_dir = Path(config.directories.cache) - for folder in ("vpn", "vpns"): - candidate = cache_dir / "global" / f"{folder}_expressvpn_tokens.json" - if candidate.is_file(): - return candidate - candidate = cache_dir / "global" / "expressvpn_tokens.json" - if candidate.is_file(): - return candidate - return cache_dir / "global" / "expressvpn_tokens.json" - - def close(self) -> None: - pass - - def __enter__(self) -> ExpressVPN: - return self - - def __exit__(self, *_: object) -> None: - self.close() + def default_cache_path(self) -> Path: + return Path(config.directories.cache) / "vpn" / "expressvpn_tokens.json" -def _slugify(value: str) -> str: +def slugify(value: str) -> str: return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") -def _write_private(path: Path, content: str) -> None: - """Write *content* to *path* with owner-only permissions (0600). +def jwt_payload(token: Optional[str]) -> dict: + if not token: + return {} + try: + parts = token.split(".") + if len(parts) < 2: + return {} + payload = parts[1] + "=" * ((4 - len(parts[1]) % 4) % 4) + return json.loads(base64.urlsafe_b64decode(payload).decode("utf-8")) + except (ValueError, json.JSONDecodeError): + return {} - Opens with ``os.open`` so the file is **created** at 0600 — no window - where it is world-readable. On Windows ``os.fchmod`` is unavailable, - so we fall back to ``path.chmod`` after close. - """ + +def jwt_expired(token: Optional[str]) -> bool: + """Treat tokens without an exp claim (proprietary SRT/connection tokens) as non-expiring.""" + if not token: + return True + expires_at = jwt_payload(token).get("exp") + if not expires_at: + return False + return time.time() >= (int(expires_at) - 300) + + +def write_private(path: Path, content: str) -> None: + """Write content to path created with owner-only (0600) permissions.""" if hasattr(os, "fchmod"): fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) try: diff --git a/unshackle/core/proxies/gluetun.py b/unshackle/core/proxies/gluetun.py index 5dc40b04..68ea4a2e 100644 --- a/unshackle/core/proxies/gluetun.py +++ b/unshackle/core/proxies/gluetun.py @@ -544,6 +544,8 @@ class Gluetun(Proxy): ["docker", "ps", "--format", "{{.Ports}}"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=5, ) if result.returncode == 0: @@ -892,6 +894,8 @@ class Gluetun(Proxy): ], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=5, ) if result.returncode != 0: @@ -926,6 +930,8 @@ class Gluetun(Proxy): ["docker", "inspect", container_name, "--format", "{{.NetworkSettings.Ports}}"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=5, ) @@ -1109,6 +1115,8 @@ class Gluetun(Proxy): ["docker", "inspect", container_name, "--format", "{{.State.ExitCode}}:{{.State.Error}}"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=5, ) if result.returncode == 0: @@ -1310,6 +1318,8 @@ class Gluetun(Proxy): ["docker", "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}:{{.Status}}"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=5, ) @@ -1328,6 +1338,8 @@ class Gluetun(Proxy): ["docker", "rm", "-f", container_name], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=10, ) return True @@ -1346,6 +1358,8 @@ class Gluetun(Proxy): ["docker", "rm", "-f", container_name], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=10, ) else: @@ -1354,6 +1368,8 @@ class Gluetun(Proxy): ["docker", "stop", container_name], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=10, ) except subprocess.TimeoutExpired: @@ -1363,6 +1379,8 @@ class Gluetun(Proxy): ["docker", "rm", "-f", container_name], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=5, ) except subprocess.TimeoutExpired: diff --git a/unshackle/core/proxies/hola.py b/unshackle/core/proxies/hola.py index 5f886e45..292871cd 100644 --- a/unshackle/core/proxies/hola.py +++ b/unshackle/core/proxies/hola.py @@ -33,9 +33,9 @@ class Hola(Proxy): """ query = query.lower() - p = subprocess.check_output( - [self.binary, "-country", query, "-list-proxies"], stderr=subprocess.STDOUT - ).decode() + p = subprocess.check_output([self.binary, "-country", query, "-list-proxies"], stderr=subprocess.STDOUT).decode( + "utf-8", errors="replace" + ) if "Transaction error: temporary ban detected." in p: raise ConnectionError("Hola banned your IP temporarily from it's services. Try change your IP.") @@ -55,6 +55,6 @@ class Hola(Proxy): def get_countries(self) -> list[dict[str, str]]: """Get a list of available Countries.""" - p = subprocess.check_output([self.binary, "-list-countries"]).decode("utf8") + p = subprocess.check_output([self.binary, "-list-countries"]).decode("utf-8", errors="replace") return [{code: name} for country in p.splitlines() for (code, name) in [country.split(" - ", maxsplit=1)]] diff --git a/unshackle/core/proxies/resolve.py b/unshackle/core/proxies/resolve.py index 1c1df39f..a5290935 100644 --- a/unshackle/core/proxies/resolve.py +++ b/unshackle/core/proxies/resolve.py @@ -31,7 +31,7 @@ def initialize_proxy_providers() -> List[Any]: proxy_providers.append(Basic(**proxy_config["basic"])) # ExpressVPN/ProtonVPN auto-load when their default cookie file exists (no yaml needed) expressvpn = ExpressVPN(**(proxy_config.get("expressvpn") or {})) - if proxy_config.get("expressvpn") or expressvpn.cookie_path.is_file(): + if proxy_config.get("expressvpn") or expressvpn.cache_path.is_file(): proxy_providers.append(expressvpn) if proxy_config.get("nordvpn"): proxy_providers.append(NordVPN(**proxy_config["nordvpn"])) diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index a1805130..ade9643e 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -207,7 +207,8 @@ def _resolve_manifest_data(tracks: Tracks, manifests: list) -> None: to copy track.data. HLS is skipped as it re-fetches from track.url. """ import base64 as b64 - import zlib + + from unshackle.core.api.compression import safe_inflate if not manifests: return @@ -223,7 +224,7 @@ def _resolve_manifest_data(tracks: Tracks, manifests: list) -> None: continue try: - raw = zlib.decompress(b64.b64decode(m_data)) + raw = safe_inflate(b64.b64decode(m_data)) if m_type == "dash": from lxml import etree @@ -836,17 +837,22 @@ class RemoteService: if not cache_data: return - import zlib + from unshackle.core.api.compression import safe_inflate + from unshackle.core.api.sanitize import safe_cache_key cache_dir = config.directories.cache / self.service_tag cache_dir.mkdir(parents=True, exist_ok=True) for key, content in cache_data.items(): + safe_name = safe_cache_key(key) + if not safe_name: + self.log.warning(f"Rejecting unsafe cache filename from server: {key!r}") + continue try: - decompressed = zlib.decompress(base64.b64decode(content)) - (cache_dir / key).with_suffix(".json").write_bytes(decompressed) + decompressed = safe_inflate(base64.b64decode(content)) + (cache_dir / safe_name).with_suffix(".json").write_bytes(decompressed) except Exception as e: - self.log.warning(f"Failed to save returned cache file '{key}': {e}") + self.log.warning(f"Failed to save returned cache file '{safe_name}': {e}") self.log.info(f"Saved {len(cache_data)} cache file(s) from server") diff --git a/unshackle/core/service_repo.py b/unshackle/core/service_repo.py index dcd96d0f..92649bfb 100644 --- a/unshackle/core/service_repo.py +++ b/unshackle/core/service_repo.py @@ -20,6 +20,7 @@ class DirtyServiceRepo(Exception): self.path = path super().__init__(str(path)) + DEFAULT_TTL = 24 * 60 * 60 # refresh at most once a day (mirrors UpdateChecker) STAMP_SUFFIX = ".fetch" # last-fetch unix time, kept beside (not inside) the clone @@ -28,6 +29,7 @@ def stamp_for(dest: Path) -> Path: """Fetch-timestamp file path, kept OUTSIDE the clone so it never shows up in `git status`.""" return dest.parent / f"{dest.name}{STAMP_SUFFIX}" + # owner/repo shorthand: exactly two path-like segments, no scheme SHORTHAND_RE = re.compile(r"^[\w.-]+/[\w.-]+$") @@ -125,9 +127,7 @@ def _pull(dest: Path, stamp: Path) -> None: if _is_dirty(dest): raise DirtyServiceRepo(dest) try: - subprocess.run( - [str(binaries.Git), "-C", str(dest), "pull", "--ff-only"], check=True, capture_output=True - ) # nosec B603 + subprocess.run([str(binaries.Git), "-C", str(dest), "pull", "--ff-only"], check=True, capture_output=True) # nosec B603 except subprocess.CalledProcessError as e: log.warning("could not update service repo at %s, using existing copy: %s", dest, _stderr(e)) _write_stamp(stamp) # stamp regardless so a flaky remote can't trigger a fetch every run (force overrides) @@ -172,9 +172,7 @@ def _changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> lis if not old or not new or old == new: return [] git = str(binaries.Git) - name_status = subprocess.run( - [git, "-C", str(dest), "diff", "--name-status", f"{old}..{new}"], capture_output=True - ) # nosec B603 + name_status = subprocess.run([git, "-C", str(dest), "diff", "--name-status", f"{old}..{new}"], capture_output=True) # nosec B603 by_dir: dict[str, set[str]] = {} if name_status.returncode == 0: for raw in name_status.stdout.decode(errors="ignore").splitlines(): @@ -223,9 +221,7 @@ def _is_dirty(dest: Path) -> bool: if status.returncode == 0 and status.stdout.strip(): return True # local commits ahead of upstream (committed but not pushed); no upstream → treat as clean - ahead = subprocess.run( - [git, "-C", str(dest), "rev-list", "--count", "@{u}..HEAD"], capture_output=True - ) # nosec B603 + ahead = subprocess.run([git, "-C", str(dest), "rev-list", "--count", "@{u}..HEAD"], capture_output=True) # nosec B603 if ahead.returncode == 0: try: return int(ahead.stdout.strip() or b"0") > 0 diff --git a/unshackle/core/services.py b/unshackle/core/services.py index b4dda122..89815493 100644 --- a/unshackle/core/services.py +++ b/unshackle/core/services.py @@ -41,7 +41,9 @@ for _dir in _service_dirs: for _path in _dir.glob("*/__init__.py"): tag = _path.parent.stem if tag in _seen: - SHADOWED.append(f"{tag}: using {redact_path(str(_seen[tag]))}, ignoring duplicate {redact_path(str(_path))}") + SHADOWED.append( + f"{tag}: using {redact_path(str(_seen[tag]))}, ignoring duplicate {redact_path(str(_path))}" + ) else: _seen[tag] = _path _SERVICES = sorted(_seen.values(), key=lambda x: x.parent.stem) diff --git a/unshackle/core/titles/title.py b/unshackle/core/titles/title.py index 4eb68c77..9aeb30c0 100644 --- a/unshackle/core/titles/title.py +++ b/unshackle/core/titles/title.py @@ -61,9 +61,7 @@ class Title: returned dict with their specific fields (e.g., season/episode). """ primary_video_track = next(iter(media_info.video_tracks), None) - original_lang_tag = ( - str(self.language).split("-")[0].lower() if self.language else "" - ) + original_lang_tag = str(self.language).split("-")[0].lower() if self.language else "" primary_audio_track = None if original_lang_tag: primary_audio_track = next( @@ -170,9 +168,7 @@ class Title: channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 channels = float(channel_count) - has_atmos = any( - "JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks - ) + has_atmos = any("JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks) context.update( { diff --git a/unshackle/core/tracks/hybrid.py b/unshackle/core/tracks/hybrid.py index 3f67121e..efd1e51f 100644 --- a/unshackle/core/tracks/hybrid.py +++ b/unshackle/core/tracks/hybrid.py @@ -225,6 +225,8 @@ class Hybrid: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", + errors="replace", ) if result_duration.returncode != 0: @@ -257,6 +259,8 @@ class Hybrid: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", + errors="replace", ) if result_streams.returncode != 0: @@ -300,6 +304,8 @@ class Hybrid: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", + errors="replace", ) # cropdetect outputs crop=w:h:x:y @@ -332,7 +338,7 @@ class Hybrid: for crop in crop_results: crop_counts[crop] = crop_counts.get(crop, 0) + 1 most_common = max(crop_counts, key=crop_counts.get) - left, top, right, bottom = most_common # frame instead of leaving phantom bars from the source. + left, top, right, bottom = most_common # frame instead of leaving phantom bars from the source. l5_json = { "active_area": { @@ -603,6 +609,8 @@ class Hybrid: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", + errors="replace", ) max_mdl = 1000 diff --git a/unshackle/core/tracks/subtitle_convert.py b/unshackle/core/tracks/subtitle_convert.py index 478f91f8..964ee5d7 100644 --- a/unshackle/core/tracks/subtitle_convert.py +++ b/unshackle/core/tracks/subtitle_convert.py @@ -154,6 +154,11 @@ class SubtitleEditBackend: return source in self.reads and target in self.writes def rank(self, source: Codec, target: Codec) -> int: + if source in (Codec.SubStationAlpha, Codec.SubStationAlphav4) and target in ( + Codec.TimedTextMarkupLang, + Codec.WebVTT, + ): + return 3 return 0 def convert(self, source: Codec, src: Path, target: Codec, out: Path) -> None: diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 1c2d5e8c..2700863c 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -382,11 +382,11 @@ class Track: events.emit(events.Types.TRACK_DOWNLOADED, track=self) if drm: - progress(downloaded="Decrypting", completed=0, total=100) + progress(downloaded="Decrypting", completed=0, total=None) drm.decrypt(save_path) self.drm = None events.emit(events.Types.TRACK_DECRYPTED, track=self, drm=drm, segment=None) - progress(downloaded="Decrypted", completed=100) + progress(downloaded="Decrypted", completed=100, total=100) if track_type == "Subtitle" and self.codec.name not in ("fVTT", "fTTML"): track_data = self.path.read_bytes() @@ -576,7 +576,7 @@ class Track: Load DRM information for this track if it was deferred during parsing. Args: - service: Service instance that can fetch track-specific DRM info + service (Service | None): Service instance that can fetch track-specific DRM info Returns: True if DRM was loaded or already present, False if failed @@ -731,7 +731,16 @@ class Track: return init_data - def repackage(self) -> None: + def repackage(self, bsf_v: Optional[str] = None) -> bool: + """Remux the track with ffmpeg ``-c copy``. + + When ``bsf_v`` is given it is folded into the same pass as ``-bsf:v`` (used to + normalize video VUI colour metadata without a second full-file remux). Repackaging + is mandatory; the bitstream filter is best-effort. If the combined run fails (and it + isn't the AAC-retry case) it is retried once without ``bsf_v`` so the remux still + succeeds. Returns True if the requested ``bsf_v`` was applied (always False when + ``bsf_v`` is None, since nothing was requested). + """ if not self.path or not self.path.exists(): raise ValueError("Cannot repackage a Track that has not been downloaded.") @@ -741,7 +750,7 @@ class Track: original_path = self.path output_path = original_path.with_stem(f"{original_path.stem}_repack") - def _ffmpeg(extra_args: list[str] = None): + def _ffmpeg(extra_args: list[str] = None, bsf: Optional[str] = None): args = [ binaries.FFMPEG, "-hide_banner", @@ -775,9 +784,11 @@ class Track: "bitexact", # only have minimal tag data, reproducible mux "-codec", "copy", - str(output_path), ] ) + if bsf: + args.extend(["-bsf:v", bsf]) + args.append(str(output_path)) subprocess.run( args, @@ -793,12 +804,20 @@ class Track: context={"track_type": self.__class__.__name__, "id": self.id, "file": original_path.name}, ) + bsf_applied = False try: - _ffmpeg() + _ffmpeg(bsf=bsf_v) + bsf_applied = bsf_v is not None except subprocess.CalledProcessError as e: if b"Malformed AAC bitstream detected" in e.stderr: # e.g., TruTV's dodgy encodes - _ffmpeg(["-y", "-bsf:a", "aac_adtstoasc"]) + _ffmpeg(["-y", "-bsf:a", "aac_adtstoasc"], bsf=bsf_v) + bsf_applied = bsf_v is not None + elif bsf_v is not None: + # Repack is mandatory, the VUI bitstream filter is best-effort: retry + # without it so the remux still succeeds; caller falls back to normalize_vui. + output_path.unlink(missing_ok=True) + _ffmpeg() else: raise @@ -816,6 +835,7 @@ class Track: "output_size": output_path.stat().st_size if output_path.exists() else 0, }, ) + return bsf_applied __all__ = ("Track", "DownloadContext") diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 7907cd8b..dcd0cc37 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -5,16 +5,16 @@ import subprocess import time from functools import partial from pathlib import Path -from typing import Callable, Iterator, Optional, Sequence, Union +from typing import Any, Callable, Iterator, Optional, Sequence, Union from langcodes import Language, closest_supported_match -from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeRemainingColumn +from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn from rich.table import Table from rich.tree import Tree from unshackle.core import binaries from unshackle.core.config import config -from unshackle.core.console import console +from unshackle.core.console import GradientPulseBarColumn, console from unshackle.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE, AnyTrack, TrackT from unshackle.core.events import events from unshackle.core.tracks.attachment import Attachment @@ -114,7 +114,7 @@ class Tracks: if add_progress and track_type not in (Chapter, Attachment): progress = Progress( SpinnerColumn(finished_text=""), - BarColumn(), + GradientPulseBarColumn(), "•", TimeRemainingColumn(compact=True, elapsed_when_finished=True), "•", @@ -126,10 +126,10 @@ class Tracks: state = {"total": 100.0} def update_track_progress( - task_id: int = task, + task_id: TaskID = task, _state: dict[str, float] = state, _progress: Progress = progress, - **kwargs, + **kwargs: Any, ) -> None: """ Ensure terminal status states render as a fully completed bar. @@ -137,12 +137,20 @@ class Tracks: Some downloaders can report completed slightly below total before emitting the final "Downloaded" state. """ - if "total" in kwargs and kwargs["total"] is not None: - _state["total"] = kwargs["total"] + if "total" in kwargs: + if kwargs["total"] is None: + # Progress.update() ignores total=None; an un-started task pulses + del kwargs["total"] + _progress.reset(task_id, start=False) + else: + _state["total"] = kwargs["total"] + _progress.start_task(task_id) downloaded_state = kwargs.get("downloaded") if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}: kwargs["completed"] = _state["total"] + kwargs["total"] = _state["total"] + _progress.start_task(task_id) _progress.update(task_id=task_id, **kwargs) progress_callables.append(update_track_progress) @@ -236,11 +244,11 @@ class Tracks: log.debug(f" - Found and skipped {duplicates} duplicate tracks...") def sort_videos(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: - """Sort video tracks by bitrate, and optionally language.""" + """Sort video tracks by resolution then bitrate, and optionally language.""" if not self.videos: return - # bitrate - self.videos.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + # resolution first, then bitrate (unknown-bitrate tracks still rank by resolution) + self.videos.sort(key=lambda x: (x.height or 0, float(x.bitrate or 0.0)), reverse=True) # language for language in reversed(by_language or []): if str(language) in ("all", "best"): @@ -277,7 +285,11 @@ class Tracks: continue self.audio.sort(key=lambda x: not is_close_match(language, [x.language])) - def sort_subtitles(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: + def sort_subtitles( + self, + by_language: Optional[Sequence[Union[str, Language]]] = None, + type_priority: Optional[Sequence[str]] = None, + ) -> None: """ Sort subtitle tracks by various track attributes to a common P2P standard. You may optionally provide a sequence of languages to prioritize to the top. @@ -292,13 +304,25 @@ class Tracks: - Normal - Hard of Hearing (SDH/CC) (Least to most captions expected in the subtitle) + + type_priority overrides the Language Group Order with an explicit ranking of + "forced", "normal", and "sdh" (cc counts as sdh); unlisted types fall to the end. + The first track after sorting receives the default flag at mux time, so this also + controls which subtitle type becomes default. """ if not self.subtitles: return # language groups self.subtitles.sort(key=lambda x: str(x.language)) - self.subtitles.sort(key=lambda x: x.sdh or x.cc) - self.subtitles.sort(key=lambda x: x.forced, reverse=True) + if type_priority: + rank = {str(t).lower(): i for i, t in enumerate(type_priority)} + default_rank = len(rank) + self.subtitles.sort( + key=lambda x: rank.get("forced" if x.forced else "sdh" if (x.sdh or x.cc) else "normal", default_rank) + ) + else: + self.subtitles.sort(key=lambda x: x.sdh or x.cc) + self.subtitles.sort(key=lambda x: x.forced, reverse=True) # sections for language in reversed(by_language or []): if str(language) == "all": @@ -372,7 +396,9 @@ class Tracks: base_tracks = [] for range_type in base_ranges: base_tracks = [ - v for v in tracks if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality) + v + for v in tracks + if v.range == range_type and (v.height in quality or (v.width and int(v.width * 9 / 16) in quality)) ] if base_tracks: break @@ -380,7 +406,7 @@ class Tracks: pick = min if worst else max base_selected = [] for res in quality: - candidates = [v for v in base_tracks if v.height == res or int(v.width * 9 / 16) == res] + candidates = [v for v in base_tracks if v.height == res or (v.width and int(v.width * 9 / 16) == res)] if candidates: chosen = pick(candidates, key=lambda v: v.bitrate) base_selected.append(chosen) @@ -407,7 +433,7 @@ class Tracks: ] if not matches: matches = [ # 16:9 canvas matches - x for x in self.videos if int(x.width * (9 / 16)) == resolution + x for x in self.videos if x.width and int(x.width * (9 / 16)) == resolution ] selected.extend(matches[: per_resolution or None]) self.videos = selected @@ -417,13 +443,26 @@ class Tracks: tracks: list[TrackT], languages: list[str], per_language: int = 0, exact_match: bool = False ) -> list[TrackT]: distance = LANGUAGE_EXACT_DISTANCE if exact_match else LANGUAGE_MAX_DISTANCE - selected = [] + selected: list[TrackT] = [] + seen_ids: set[str] = set() for language in languages: - selected.extend( - [x for x in tracks if closest_supported_match(str(x.language), [language], distance)][ - : per_language or None - ] - ) + matches = [x for x in tracks if closest_supported_match(str(x.language), [language], distance)] + if exact_match and len(matches) > 1: + # CLDR tag_distance measures intelligibility, not tag identity: it rates a base + # language and its "paradigm" regional variant as the same language (distance 0 + # for ar/ar-EG, en/en-US, pt/pt-BR), so exact mode alone cannot separate them. + # Follow RFC 4647 Lookup: prefer the most specific (string-equal) tag, and fall + # back to the fuzzy match only when none exists (e.g. zh matches cmn). + want = Language.get(language).to_tag().casefold() + exact_hits = [x for x in matches if Language.get(str(x.language)).to_tag().casefold() == want] + if exact_hits: + matches = exact_hits + # Overlapping tags can still resolve to the same physical track; dedupe by id so + # callers never get duplicate tracks. + for track in matches[: per_language or None]: + if track.id not in seen_ids: + seen_ids.add(track.id) + selected.append(track) return selected def mux( @@ -668,7 +707,7 @@ class Tracks: log_event( "mux_start", level="INFO", - message=(f"Muxing {len(self.videos)}V/{len(self.audio)}A/{len(self.subtitles)}S " f"-> {output_path.name}"), + message=(f"Muxing {len(self.videos)}V/{len(self.audio)}A/{len(self.subtitles)}S -> {output_path.name}"), context={ "title": title, "output_path": str(output_path), @@ -697,7 +736,7 @@ class Tracks: errors = [] warnings = [] mux_start_time = time.monotonic() - p = subprocess.Popen(full_command, text=True, stdout=subprocess.PIPE) + p = subprocess.Popen(full_command, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE) for line in iter(p.stdout.readline, ""): if line.startswith("#GUI#error") or line.startswith("#GUI#warning"): errors.append(line) diff --git a/unshackle/core/tracks/video.py b/unshackle/core/tracks/video.py index 7082aac9..ead971fc 100644 --- a/unshackle/core/tracks/video.py +++ b/unshackle/core/tracks/video.py @@ -410,20 +410,19 @@ class Video(Track): self.path = output_path original_path.unlink() - def normalize_vui(self) -> bool: - """Rewrite SPS VUI colour metadata to match ``self.range``. + def vui_bsf(self) -> Optional[str]: + """Return the ``-bsf:v`` value that rewrites SPS VUI colour metadata to match ``self.range``. - Some services ship HDR10/HLG bitstreams with stale BT.709 VUI, which makes - downstream tools mis-classify the file. The manifest-derived range is the - source of truth. Skips SDR, DV, and HYBRID. Returns True if the bitstream - was rewritten. + Returns None when no rewrite is needed: SDR/DV/HYBRID ranges, non-AVC/HEVC codecs, a + missing file, or when the bitstream already ships the correct colour metadata. Otherwise + returns a ``{h264,hevc}_metadata=...`` string suitable for ffmpeg ``-bsf:v``. """ if not self.path or not self.path.exists(): - return False + return None if self.codec not in (Video.Codec.AVC, Video.Codec.HEVC): - return False + return None if self.range in (Video.Range.SDR, Video.Range.DV, Video.Range.HYBRID): - return False + return None vui = { Video.Range.HDR10: (9, 16, 9), @@ -431,7 +430,7 @@ class Video(Track): Video.Range.HLG: (9, 18, 9), }.get(self.range) if not vui: - return False + return None # Skip the rewrite when the bitstream VUI already matches the target — avoids a full # bitstream pass on services that already ship correct colour metadata @@ -456,23 +455,38 @@ class Video(Track): ], capture_output=True, text=True, + encoding="utf-8", + errors="replace", ) vals = dict(line.split("=", 1) for line in probe.stdout.splitlines() if "=" in line) current = (vals.get("color_primaries"), vals.get("color_transfer"), vals.get("color_space")) if current == expected: - return False - - if not binaries.FFMPEG: - raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + return None primaries, transfer, matrix = vui filter_key = {Video.Codec.AVC: "h264_metadata", Video.Codec.HEVC: "hevc_metadata"}[self.codec] - bsf = ( + return ( f"{filter_key}=colour_primaries={primaries}" f":transfer_characteristics={transfer}" f":matrix_coefficients={matrix}" ) + def normalize_vui(self) -> bool: + """Rewrite SPS VUI colour metadata to match ``self.range``. + + Some services ship HDR10/HLG bitstreams with stale BT.709 VUI, which makes + downstream tools mis-classify the file. The manifest-derived range is the + source of truth. Skips SDR, DV, and HYBRID. Returns True if the bitstream + was rewritten. + """ + bsf = self.vui_bsf() + if bsf is None: + return False + assert self.path is not None # vui_bsf() returns None when path is unset + + if not binaries.FFMPEG: + raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + original_path = self.path output_path = original_path.with_stem(f"{original_path.stem}_vui") try: @@ -506,7 +520,7 @@ class Video(Track): context={ "codec": str(self.codec), "range": str(self.range), - "vui": {"primaries": primaries, "transfer": transfer, "matrix": matrix}, + "vui": bsf, "id": self.id, }, ) @@ -541,6 +555,11 @@ class Video(Track): e.returncode, duration_ms=round((time.monotonic() - cc_start) * 1000, 1), ) + if e.returncode < 0: + logging.getLogger("Video").warning( + f"ccextractor crashed (signal {-e.returncode}) on {self.path.name}; skipping CC extraction" + ) + return out_path.exists() if e.returncode != 10: # 10 = No captions found raise return out_path.exists() diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 572bfd0d..b86171c0 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -111,6 +111,7 @@ def import_module_by_path(path: Path) -> ModuleType: spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + sys.modules[name] = module spec.loader.exec_module(module) return module @@ -981,7 +982,9 @@ class DebugLogger: """ message = kwargs.pop("message", "") level = kwargs.pop("level", "DEBUG") - self.log(level=level, operation="service_call", message=message, request={"method": method, "url": url, **kwargs}) + self.log( + level=level, operation="service_call", message=message, request={"method": method, "url": url, **kwargs} + ) def log_drm_operation(self, drm_type: str, operation: str, **kwargs): """ diff --git a/unshackle/core/utils/dovi.py b/unshackle/core/utils/dovi.py index 3bd002ee..805120f7 100644 --- a/unshackle/core/utils/dovi.py +++ b/unshackle/core/utils/dovi.py @@ -84,7 +84,9 @@ def info_summary(rpu: Path) -> str: """Return the textual summary (`dovi_tool info -i ... -s`) for an RPU file.""" tool = _require_dovi_tool() info_start = time.monotonic() - p = subprocess.run([tool, "info", "-i", str(rpu), "-s"], capture_output=True, text=True) + p = subprocess.run( + [tool, "info", "-i", str(rpu), "-s"], capture_output=True, text=True, encoding="utf-8", errors="replace" + ) log_tool_run( "dovi_tool info", "dovi_tool", diff --git a/unshackle/core/utils/firefox_cookie_extractor.py b/unshackle/core/utils/firefox_cookie_extractor.py index f9ab6ffe..75c00fe7 100644 --- a/unshackle/core/utils/firefox_cookie_extractor.py +++ b/unshackle/core/utils/firefox_cookie_extractor.py @@ -17,15 +17,15 @@ def get_firefox_root() -> Path: """Locates the operating system specific root directory for Firefox profiles.""" system = platform.system() home = Path.home() - if system == 'Windows': - return Path(os.getenv('APPDATA', '')) / 'Mozilla' / 'Firefox' - elif system == 'Darwin': - return home / 'Library' / 'Application Support' / 'Firefox' - elif system == 'Linux': + if system == "Windows": + return Path(os.getenv("APPDATA", "")) / "Mozilla" / "Firefox" + elif system == "Darwin": + return home / "Library" / "Application Support" / "Firefox" + elif system == "Linux": paths = [ - home / '.mozilla' / 'firefox', - home / 'snap' / 'firefox' / 'common' / '.mozilla' / 'firefox', - home / '.var' / 'app' / 'org.mozilla.firefox' / '.mozilla' / 'firefox' + home / ".mozilla" / "firefox", + home / "snap" / "firefox" / "common" / ".mozilla" / "firefox", + home / ".var" / "app" / "org.mozilla.firefox" / ".mozilla" / "firefox", ] for p in paths: if p.exists(): @@ -36,16 +36,16 @@ def get_firefox_root() -> Path: def get_latest_profile_path(ff_root: Path) -> Path: """Finds the most recently modified valid Firefox profile subdirectory.""" - search_dirs = [ff_root / 'Profiles', ff_root] + search_dirs = [ff_root / "Profiles", ff_root] valid_profiles = [] for base in search_dirs: if base.exists(): - valid_profiles.extend([p for p in base.iterdir() if p.is_dir() and (p / 'cookies.sqlite').exists()]) + valid_profiles.extend([p for p in base.iterdir() if p.is_dir() and (p / "cookies.sqlite").exists()]) if not valid_profiles: raise FileNotFoundError("No active Firefox profiles detected.") - return max(valid_profiles, key=lambda p: (p / 'cookies.sqlite').stat().st_mtime) + return max(valid_profiles, key=lambda p: (p / "cookies.sqlite").stat().st_mtime) def get_local_storage_data(profile_path: Path, hosts: List[str], tmp_dir_path: Path) -> Dict[str, str]: @@ -53,7 +53,7 @@ def get_local_storage_data(profile_path: Path, hosts: List[str], tmp_dir_path: P Extracts LocalStorage tokens from webappsstore.sqlite with strict origin matching. Operates within a secure temporary directory. """ - ls_db = profile_path / 'webappsstore.sqlite' + ls_db = profile_path / "webappsstore.sqlite" if not ls_db.exists(): return {} @@ -87,10 +87,10 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: Extracts cookies and optionally localStorage from Firefox based on provided host patterns. Implements security hardening and data consistency (WAL support) as per PR review. """ - raw_hosts = service_settings.get('hosts', []) + raw_hosts = service_settings.get("hosts", []) # Filter empty or dangerously short hosts to prevent full store dumps (Finding #3) priority_hosts = [h.strip().lower() for h in raw_hosts if h and len(h.strip()) >= 3] - use_local_storage = service_settings.get('local_storage', False) + use_local_storage = service_settings.get("local_storage", False) if not priority_hosts: return None @@ -112,8 +112,8 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: try: # Copy main DB and its WAL log to ensure fresh data capture (Finding #5) - shutil.copy2(profile_path / 'cookies.sqlite', temp_db) - wal_path = profile_path / 'cookies.sqlite-wal' + shutil.copy2(profile_path / "cookies.sqlite", temp_db) + wal_path = profile_path / "cookies.sqlite-wal" if wal_path.exists(): shutil.copy2(wal_path, temp_wal) @@ -134,11 +134,23 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: # Final validation to ensure proper domain suffix if h == host or h.endswith(f".{host}"): c = Cookie( - version=0, name=n, value=v, port=None, port_specified=False, - domain=h, domain_specified=True, domain_initial_dot=h.startswith('.'), - path=p, path_specified=True, secure=bool(secure), expires=e, - discard=False, comment=None, comment_url=None, - rest={'HttpOnly': str(bool(httponly))}, rfc2109=False + version=0, + name=n, + value=v, + port=None, + port_specified=False, + domain=h, + domain_specified=True, + domain_initial_dot=h.startswith("."), + path=p, + path_specified=True, + secure=bool(secure), + expires=e, + discard=False, + comment=None, + comment_url=None, + rest={"HttpOnly": str(bool(httponly))}, + rfc2109=False, ) cookie_jar.set_cookie(c) conn.close() @@ -148,10 +160,23 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: storage_data = get_local_storage_data(profile_path, priority_hosts, tmp_dir_path) for key, val in storage_data.items(): c = Cookie( - version=0, name=key, value=val, port=None, port_specified=False, - domain=".localstorage", domain_specified=True, domain_initial_dot=False, - path='/', path_specified=True, secure=False, expires=None, - discard=True, comment=None, comment_url=None, rest={}, rfc2109=False + version=0, + name=key, + value=val, + port=None, + port_specified=False, + domain=".localstorage", + domain_specified=True, + domain_initial_dot=False, + path="/", + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={}, + rfc2109=False, ) cookie_jar.set_cookie(c) diff --git a/unshackle/core/utils/subprocess.py b/unshackle/core/utils/subprocess.py index 4189dc62..1eb0c3e8 100644 --- a/unshackle/core/utils/subprocess.py +++ b/unshackle/core/utils/subprocess.py @@ -66,7 +66,7 @@ def ffprobe(uri: Union[bytes, Path]) -> dict: context={"tool": "ffprobe", "duration_ms": round((time.monotonic() - start) * 1000, 1)}, ) return {} - result = json.loads(ff.stdout.decode("utf8")) + result = json.loads(ff.stdout.decode("utf-8", errors="replace")) if dl: dl.log( level="DEBUG", diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index d756f80a..c77d0597 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -10,8 +10,16 @@ from xml.sax.saxutils import escape from unshackle.core import binaries from unshackle.core.config import config -from unshackle.core.providers import (ExternalIds, MetadataResult, enrich_ids, fetch_external_ids, fuzzy_match, - get_available_providers, get_provider, search_metadata) +from unshackle.core.providers import ( + ExternalIds, + MetadataResult, + enrich_ids, + fetch_external_ids, + fuzzy_match, + get_available_providers, + get_provider, + search_metadata, +) from unshackle.core.titles.episode import Episode from unshackle.core.titles.movie import Movie from unshackle.core.titles.title import Title @@ -41,6 +49,8 @@ def apply_tags(path: Path, tags: dict[str, str]) -> None: check=False, capture_output=True, text=True, + encoding="utf-8", + errors="replace", ) log_tool_run( "mkvpropedit tags", diff --git a/unshackle/core/utils/webvtt.py b/unshackle/core/utils/webvtt.py index 68cc52dc..11b3f67b 100644 --- a/unshackle/core/utils/webvtt.py +++ b/unshackle/core/utils/webvtt.py @@ -1,225 +1,74 @@ import re -import sys -import typing from typing import Optional -import pysubs2 -from pycaption import Caption, CaptionList, CaptionNode, CaptionReadError, WebVTTReader, WebVTTWriter - -from unshackle.core.config import config +# Timing line: optional hours, verbatim settings tail. +_TIMING = re.compile(r"^((?:\d+:)?\d{2}:\d{2}\.\d{3})[ \t]+-->[ \t]+((?:\d+:)?\d{2}:\d{2}\.\d{3})(.*)$") -class CaptionListExt(CaptionList): - @typing.no_type_check - def __init__(self, iterable=None, layout_info=None): - self.first_segment_mpegts = 0 - super().__init__(iterable, layout_info) - - -class CaptionExt(Caption): - @typing.no_type_check - def __init__(self, start, end, nodes, style=None, layout_info=None, segment_index=0, mpegts=0, cue_time=0.0): - style = style or {} - self.segment_index: int = segment_index - self.mpegts: float = mpegts - self.cue_time: float = cue_time - super().__init__(start, end, nodes, style, layout_info) - - -class WebVTTReaderExt(WebVTTReader): - # HLS extension support <https://datatracker.ietf.org/doc/html/rfc8216#section-3.5> - RE_TIMESTAMP_MAP = re.compile(r"X-TIMESTAMP-MAP.*") - RE_MPEGTS = re.compile(r"MPEGTS:(\d+)") - RE_LOCAL = re.compile(r"LOCAL:((?:(\d{1,}):)?(\d{2}):(\d{2})\.(\d{3}))") - - def _parse(self, lines: list[str]) -> CaptionList: - captions = CaptionListExt() - start = None - end = None - nodes: list[CaptionNode] = [] - layout_info = None - found_timing = False - segment_index = -1 - mpegts = 0 - cue_time = 0.0 - - # The first segment MPEGTS is needed to calculate the rest. It is possible that - # the first segment contains no cue and is ignored by pycaption, this acts as a fallback. - captions.first_segment_mpegts = 0 - - for i, line in enumerate(lines): - if "-->" in line: - found_timing = True - timing_line = i - last_start_time = captions[-1].start if captions else 0 - try: - start, end, layout_info = self._parse_timing_line(line, last_start_time) - except CaptionReadError as e: - new_msg = f"{e.args[0]} (line {timing_line})" - tb = sys.exc_info()[2] - raise type(e)(new_msg).with_traceback(tb) from None - - elif "" == line: - if found_timing and nodes: - found_timing = False - caption = CaptionExt( - start, - end, - nodes, - layout_info=layout_info, - segment_index=segment_index, - mpegts=mpegts, - cue_time=cue_time, - ) - captions.append(caption) - nodes = [] - - elif "WEBVTT" in line: - # Merged segmented VTT doesn't have index information, track manually. - segment_index += 1 - mpegts = 0 - cue_time = 0.0 - elif m := self.RE_TIMESTAMP_MAP.match(line): - if r := self.RE_MPEGTS.search(m.group()): - mpegts = int(r.group(1)) - - cue_time = self._parse_local(m.group()) - - # Early assignment in case the first segment contains no cue. - if segment_index == 0: - captions.first_segment_mpegts = mpegts - - else: - if found_timing: - if nodes: - nodes.append(CaptionNode.create_break()) - nodes.append(CaptionNode.create_text(self._decode(line))) - else: - # it's a comment or some metadata; ignore it - pass - - # Add a last caption if there are remaining nodes - if nodes: - caption = CaptionExt(start, end, nodes, layout_info=layout_info, segment_index=segment_index, mpegts=mpegts) - captions.append(caption) - - return captions - - @staticmethod - def _parse_local(string: str) -> float: - """ - Parse WebVTT LOCAL time and convert it to seconds. - """ - m = WebVTTReaderExt.RE_LOCAL.search(string) - if not m: - return 0 - - parsed = m.groups() - if not parsed: - return 0 - hours = int(parsed[1]) - minutes = int(parsed[2]) - seconds = int(parsed[3]) - milliseconds = int(parsed[4]) - return (milliseconds / 1000) + seconds + (minutes * 60) + (hours * 3600) +def _timestamp_ms(ts: str) -> int: + hms, ms = ts.rsplit(".", 1) + parts = [int(p) for p in hms.split(":")] + while len(parts) < 3: + parts.insert(0, 0) + hours, minutes, seconds = parts + return ((hours * 60 + minutes) * 60 + seconds) * 1000 + int(ms) def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] = None, timescale: int = 1) -> str: """ - Merge Segmented WebVTT data. + Merge concatenated Segmented WebVTT documents into a single document. - Parameters: - vtt_raw: The concatenated WebVTT files to merge. All WebVTT headers must be - appropriately spaced apart, or it may produce unwanted effects like - considering headers as captions, timestamp lines, etc. - segment_durations: A list of each segment's duration. If not provided it will try - to get it from the X-TIMESTAMP-MAP headers, specifically the MPEGTS number. - timescale: The number of time units per second. + Strips the per-segment WEBVTT and X-TIMESTAMP-MAP headers and splices cues + repeated across a segment boundary (identical payload, <=1ms gap or overlap). + Timing lines, cue settings and payload are kept verbatim, so inline formatting + (span tags, entities, ASS-style overrides) survives untouched. - This parses the X-TIMESTAMP-MAP data to compute new absolute timestamps, replacing - the old start and end timestamp values. All X-TIMESTAMP-MAP header information will - be removed from the output as they are no longer of concern. Consider this function - the opposite of a WebVTT Segmenter, a WebVTT Joiner of sorts. + No timestamp offset is applied: segments are expected to carry absolute cue + times already (same guard as shaka-player and N_m3u8DL-RE; re-offsetting + double-shifts such streams). segment_durations/timescale are accepted for + call-site compatibility and only matter if offsetting is ever reintroduced. - Algorithm borrowed from N_m3u8DL-RE and shaka-player. + Unique STYLE/REGION blocks are kept (players that don't support them ignore + them); cue identifiers and NOTE blocks are dropped; cues with a malformed + timing line are skipped (downstream sanitizers catch worse). """ - MPEG_TIMESCALE = 90_000 + # each kept cue: [start, end, settings, payload, normalized payload for dedupe] + cues: list[list[str]] = [] + headers: list[str] = [] # unique STYLE/REGION blocks, first-seen order + prev: Optional[list[str]] = None + prev_end_ms = 0 - # Check config for conversion method preference - conversion_method = config.subtitle.get("conversion_method", "auto") - use_pysubs2 = conversion_method in ("pysubs2", "auto") + for block in re.split(r"\n[ \t]*\n", vtt_raw.replace("\r\n", "\n").replace("\r", "\n")): + lines = [ + line + for line in block.split("\n") + if not (line.startswith("WEBVTT") or line.lstrip().startswith("X-TIMESTAMP-MAP")) + ] + timing_i = next((i for i, line in enumerate(lines) if "-->" in line), None) + if timing_i is None: # NOTE or other metadata block + first = next((line.strip() for line in lines if line.strip()), "") + if first.startswith(("STYLE", "REGION")): + header = "\n".join(lines).strip("\n") + if header not in headers: # segments repeat the same block + headers.append(header) + continue + timing = _TIMING.match(lines[timing_i].strip()) + if not timing: + continue + start, end, settings = timing.groups() + payload = [line for line in lines[timing_i + 1 :] if line.strip()] + if not payload: + continue - if use_pysubs2: - # Try using pysubs2 first for more lenient parsing - try: - # Use pysubs2 to parse and normalize the VTT - subs = pysubs2.SSAFile.from_string(vtt_raw) - # Convert back to WebVTT string for pycaption processing - normalized_vtt = subs.to_string("vtt") - vtt = WebVTTReaderExt().read(normalized_vtt) - except Exception: - # Fall back to direct pycaption parsing - vtt = WebVTTReaderExt().read(vtt_raw) - else: - # Use pycaption directly - vtt = WebVTTReaderExt().read(vtt_raw) - for lang in vtt.get_languages(): - prev_caption = None - duplicate_index: list[int] = [] - captions = vtt.get_captions(lang) + normalized = "\n".join(line.strip() for line in payload) + if prev is not None and _timestamp_ms(start) - prev_end_ms <= 1 and normalized == prev[4]: + prev[1] = end # splice: extend the kept cue, drop the duplicate + prev_end_ms = _timestamp_ms(end) + continue - # Some providers can produce "segment_index" values that are - # outside the provided segment_durations list after normalization/merge. - # This used to crash with IndexError and abort the entire download. - if segment_durations and captions: - max_idx = max(getattr(c, "segment_index", 0) for c in captions) - if max_idx >= len(segment_durations): - # Pad with the last known duration (or 0 if empty) so indexing is safe. - pad_val = segment_durations[-1] if segment_durations else 0 - segment_durations = segment_durations + [pad_val] * (max_idx - len(segment_durations) + 1) + prev = [start, end, settings, "\n".join(payload), normalized] + prev_end_ms = _timestamp_ms(end) + cues.append(prev) - if captions[0].segment_index == 0: - first_segment_mpegts = captions[0].mpegts - else: - first_segment_mpegts = segment_durations[0] if segment_durations else captions.first_segment_mpegts - - caption: CaptionExt - for i, caption in enumerate(captions): - # DASH WebVTT doesn't have MPEGTS timestamp like HLS. Instead, - # calculate the timestamp from SegmentTemplate/SegmentList duration. - likely_dash = first_segment_mpegts == 0 and caption.mpegts == 0 - if likely_dash and segment_durations: - # Defensive: segment_index can still be out of range if captions are malformed. - if caption.segment_index < 0 or caption.segment_index >= len(segment_durations): - continue - duration = segment_durations[caption.segment_index] - caption.mpegts = MPEG_TIMESCALE * (duration / timescale) - - if caption.mpegts == 0: - continue - - # Commeted to fix DSNP subs being out of sync and mistimed. - # seconds = (caption.mpegts - first_segment_mpegts) / MPEG_TIMESCALE - caption.cue_time - # offset = seconds * 1_000_000 # pycaption use microseconds - - # if caption.start < offset: - # caption.start += offset - # caption.end += offset - - # If the difference between current and previous captions is <=1ms - # and the payload is equal then splice. - if ( - prev_caption - and not caption.is_empty() - and (caption.start - prev_caption.end) <= 1000 # 1ms in microseconds - and caption.get_text() == prev_caption.get_text() - ): - prev_caption.end = caption.end - duplicate_index.append(i) - - prev_caption = caption - - # Remove duplicate - captions[:] = [c for c_index, c in enumerate(captions) if c_index not in set(duplicate_index)] - - return WebVTTWriter().write(vtt) + blocks = headers + [f"{c[0]} --> {c[1]}{c[2]}\n{c[3]}" for c in cues] + return "WEBVTT\n\n" + "\n\n".join(blocks) + "\n" diff --git a/uv.lock b/uv.lock index 1cbfa82d..d7674e55 100644 --- a/uv.lock +++ b/uv.lock @@ -1998,28 +1998,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.14.14" +version = "0.15.21" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/06/f71e3a86b2df0dfa2d2f72195941cd09b44f87711cb7fa5193732cb9a5fc/ruff-0.14.14.tar.gz", hash = "sha256:2d0f819c9a90205f3a867dbbd0be083bee9912e170fd7d9704cc8ae45824896b", size = 4515732, upload-time = "2026-01-22T22:30:17.527Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/89/20a12e97bc6b9f9f68343952da08a8099c57237aef953a56b82711d55edd/ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed", size = 10467650, upload-time = "2026-01-22T22:30:08.578Z" }, - { url = "https://files.pythonhosted.org/packages/a3/b1/c5de3fd2d5a831fcae21beda5e3589c0ba67eec8202e992388e4b17a6040/ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c", size = 10883245, upload-time = "2026-01-22T22:30:04.155Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7c/3c1db59a10e7490f8f6f8559d1db8636cbb13dccebf18686f4e3c9d7c772/ruff-0.14.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:026c1d25996818f0bf498636686199d9bd0d9d6341c9c2c3b62e2a0198b758de", size = 10231273, upload-time = "2026-01-22T22:30:34.642Z" }, - { url = "https://files.pythonhosted.org/packages/a1/6e/5e0e0d9674be0f8581d1f5e0f0a04761203affce3232c1a1189d0e3b4dad/ruff-0.14.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e", size = 10585753, upload-time = "2026-01-22T22:30:31.781Z" }, - { url = "https://files.pythonhosted.org/packages/23/09/754ab09f46ff1884d422dc26d59ba18b4e5d355be147721bb2518aa2a014/ruff-0.14.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c0f18b922c6d2ff9a5e6c3ee16259adc513ca775bcf82c67ebab7cbd9da5bc8", size = 10286052, upload-time = "2026-01-22T22:30:24.827Z" }, - { url = "https://files.pythonhosted.org/packages/c8/cc/e71f88dd2a12afb5f50733851729d6b571a7c3a35bfdb16c3035132675a0/ruff-0.14.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1629e67489c2dea43e8658c3dba659edbfd87361624b4040d1df04c9740ae906", size = 11043637, upload-time = "2026-01-22T22:30:13.239Z" }, - { url = "https://files.pythonhosted.org/packages/67/b2/397245026352494497dac935d7f00f1468c03a23a0c5db6ad8fc49ca3fb2/ruff-0.14.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:27493a2131ea0f899057d49d303e4292b2cae2bb57253c1ed1f256fbcd1da480", size = 12194761, upload-time = "2026-01-22T22:30:22.542Z" }, - { url = "https://files.pythonhosted.org/packages/5b/06/06ef271459f778323112c51b7587ce85230785cd64e91772034ddb88f200/ruff-0.14.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01ff589aab3f5b539e35db38425da31a57521efd1e4ad1ae08fc34dbe30bd7df", size = 12005701, upload-time = "2026-01-22T22:30:20.499Z" }, - { url = "https://files.pythonhosted.org/packages/41/d6/99364514541cf811ccc5ac44362f88df66373e9fec1b9d1c4cc830593fe7/ruff-0.14.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc12d74eef0f29f51775f5b755913eb523546b88e2d733e1d701fe65144e89b", size = 11282455, upload-time = "2026-01-22T22:29:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/ca/71/37daa46f89475f8582b7762ecd2722492df26421714a33e72ccc9a84d7a5/ruff-0.14.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb8481604b7a9e75eff53772496201690ce2687067e038b3cc31aaf16aa0b974", size = 11215882, upload-time = "2026-01-22T22:29:57.032Z" }, - { url = "https://files.pythonhosted.org/packages/2c/10/a31f86169ec91c0705e618443ee74ede0bdd94da0a57b28e72db68b2dbac/ruff-0.14.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:14649acb1cf7b5d2d283ebd2f58d56b75836ed8c6f329664fa91cdea19e76e66", size = 11180549, upload-time = "2026-01-22T22:30:27.175Z" }, - { url = "https://files.pythonhosted.org/packages/fd/1e/c723f20536b5163adf79bdd10c5f093414293cdf567eed9bdb7b83940f3f/ruff-0.14.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13", size = 10543416, upload-time = "2026-01-22T22:30:01.964Z" }, - { url = "https://files.pythonhosted.org/packages/3e/34/8a84cea7e42c2d94ba5bde1d7a4fae164d6318f13f933d92da6d7c2041ff/ruff-0.14.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412", size = 10285491, upload-time = "2026-01-22T22:30:29.51Z" }, - { url = "https://files.pythonhosted.org/packages/55/ef/b7c5ea0be82518906c978e365e56a77f8de7678c8bb6651ccfbdc178c29f/ruff-0.14.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:cc8b22da8d9d6fdd844a68ae937e2a0adf9b16514e9a97cc60355e2d4b219fc3", size = 10733525, upload-time = "2026-01-22T22:30:06.499Z" }, - { url = "https://files.pythonhosted.org/packages/6a/5b/aaf1dfbcc53a2811f6cc0a1759de24e4b03e02ba8762daabd9b6bd8c59e3/ruff-0.14.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:16bc890fb4cc9781bb05beb5ab4cd51be9e7cb376bf1dd3580512b24eb3fda2b", size = 11315626, upload-time = "2026-01-22T22:30:36.848Z" }, - { url = "https://files.pythonhosted.org/packages/2c/aa/9f89c719c467dfaf8ad799b9bae0df494513fb21d31a6059cb5870e57e74/ruff-0.14.14-py3-none-win32.whl", hash = "sha256:b530c191970b143375b6a68e6f743800b2b786bbcf03a7965b06c4bf04568167", size = 10502442, upload-time = "2026-01-22T22:30:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/87/44/90fa543014c45560cae1fffc63ea059fb3575ee6e1cb654562197e5d16fb/ruff-0.14.14-py3-none-win_amd64.whl", hash = "sha256:3dde1435e6b6fe5b66506c1dff67a421d0b7f6488d466f651c07f4cab3bf20fd", size = 11630486, upload-time = "2026-01-22T22:30:10.852Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" }, + { url = "https://files.pythonhosted.org/packages/28/9d/d825b07ee7ea9e2d61df92a860033c94e06e7300d50a1c2653aac27d24fe/ruff-0.15.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd", size = 11139539, upload-time = "2026-07-09T20:00:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/f5/de/3b107712e642f063c7a9e0887c427b22cb44097de5aab36c05f2e280670c/ruff-0.15.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646", size = 10595437, upload-time = "2026-07-09T20:01:00.006Z" }, + { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" }, + { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" }, + { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" }, + { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" }, + { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" }, + { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3c/37d0ecb729a7cc2d393ea7dce316fc585680f35d93b8d62139d7d0a3700c/ruff-0.15.21-py3-none-win32.whl", hash = "sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd", size = 10896555, upload-time = "2026-07-09T20:01:26.941Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b8/e43466b2a6067ce91e669068f6e28d6c719a920f014b070d5c8731725de3/ruff-0.15.21-py3-none-win_amd64.whl", hash = "sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3", size = 12038772, upload-time = "2026-07-09T20:01:29.497Z" }, + { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" }, ] [[package]] @@ -2236,7 +2235,7 @@ requires-dist = [ { name = "lxml", specifier = ">=5.2.1,<7" }, { name = "mutagen", specifier = ">=1.47.0,<2" }, { name = "protobuf", specifier = ">=4.25.3,<7" }, - { name = "pycaption", specifier = ">=2.2.6,<3" }, + { name = "pycaption", specifier = ">=2.2.6,<2.2.27" }, { name = "pycountry", specifier = ">=24.6.1" }, { name = "pycryptodomex", specifier = ">=3.20.0,<4" }, { name = "pyexecjs", specifier = ">=1.5.1,<2" }, @@ -2267,7 +2266,7 @@ dev = [ { name = "mypy", specifier = ">=1.9.0,<2" }, { name = "mypy-protobuf", specifier = ">=3.6.0,<4" }, { name = "pre-commit", specifier = ">=3.7.0,<5" }, - { name = "ruff", specifier = ">=0.3.7,<0.15" }, + { name = "ruff", specifier = ">=0.15.12,<0.16" }, { name = "types-protobuf", specifier = ">=4.24.0.20240408,<7" }, { name = "types-pymysql", specifier = ">=1.1.0.1,<2" }, { name = "types-requests", specifier = ">=2.31.0.20240406,<3" },