mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-15 13:35:05 +00:00
Compare commits
15
Commits
f86359de5d
...
6.0.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
83eef01fad | ||
|
|
e7bb825def | ||
|
|
4fbaf2a61a | ||
|
|
3e64a7d684 | ||
|
|
2ba442f50e | ||
|
|
af4e6d5d47 | ||
|
|
b4136e1cf5 | ||
|
|
4c2e74de80 | ||
|
|
ecdfb667c9 | ||
|
|
0a7c76561c | ||
|
|
0cded6427f | ||
|
|
355df2e54f | ||
|
|
09cae2ec3f | ||
|
|
5b0369e63a | ||
|
|
3fd69e8039 |
+261
@@ -6,6 +6,267 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||
|
||||
This changelog is automatically generated using [git-cliff](https://git-cliff.org).
|
||||
|
||||
## [6.0.0] - 2026-10-13
|
||||
|
||||
### Features
|
||||
|
||||
- *binary*: Add custom binary registration for services
|
||||
- *cdm*: Send license_type and privacy_mode with the request to the server
|
||||
- *click*: Change the minimum time for slow
|
||||
- *downloader*: Adaptive worker scaling, tail boost, session parity, resume fixes
|
||||
- *downloader*: Fan large segment batches across worker processes
|
||||
- *downloader*: Resume failed downloads from completed segments across runs
|
||||
- *dl*: Run user post-download scripts with metadata variables
|
||||
- *dl*: Add --require-audio/--require-video and --warn-only alias
|
||||
- *drm*: Let a track pin the DRM system it licenses with
|
||||
- *dl*: Add --no-attachments to skip cover art and subtitle fonts
|
||||
- *dl*: Add --cdm to pin one CDM device for a run
|
||||
- *post-scripts*: Add wait option to block until the script exits
|
||||
- *post-scripts*: Show a spinner while waiting for a script
|
||||
- *services*: Add services_repo_force and compact refresh output
|
||||
- *serve*: Hot-reload service repos on a timer without a restart
|
||||
- *serve*: Gate server_cdm per service and let clients follow it
|
||||
- *kv*: Show a progress bar while adding keys to a vault
|
||||
- *serve*: [**breaking**] Gate the server's own proxies behind server_proxy
|
||||
- *vaults*: Batch HTTP json vault writes with InsertKeys
|
||||
- *remote*: Forward HLS segment filters to remote clients
|
||||
- *serve*: Add --quiet mode and developer dashboard API
|
||||
- *serve*: [**breaking**] Add server-held accounts for remote sessions
|
||||
- *serve*: Forward per-session service logs to remote-dl clients
|
||||
- *proxy*: Mask proxy credentials in console and API output
|
||||
- *remote*: Spoof a type-only CDM so server_cdm clients need no local .wvd/.prd
|
||||
- *serve*: Add operator dashboard endpoints and per-key rate limits
|
||||
- *audio*: Support DTS:X Profile 2 with MP4 muxing
|
||||
- *dl*: Add --no-postscript to skip post-scripts for one run
|
||||
- *api*: Report the live title and the client's command line to the dashboard
|
||||
- *vaults*: Flag and skip wrong content keys from poisoned vaults
|
||||
- *remote*: Verify server-CDM vault keys on the client and report bad pairs
|
||||
- *dl*: Add --proxy-download for a separate download proxy
|
||||
- *vaults*: Report bad keys to the vault that served them
|
||||
- *service*: Add GEOBLOCK negative geofence
|
||||
- *dash*: Add merge_segments to append segments during the download
|
||||
- *manifests*: Extend merge_segments rolling merge to HLS and ISM
|
||||
- *export*: Write mediaexport files and read legacy exports through the package
|
||||
- *proxies*: Add Control D proxy provider
|
||||
- *export*: Record each track's KIDs
|
||||
- *import*: Build tracks from direct-file export rows
|
||||
- *export*: Name export files after their titles and tracks
|
||||
- *export*: Separate export file name parts with dots
|
||||
- *remote*: Keep slow remote session requests alive with a heartbeat
|
||||
- *remote*: Cap server CDM licensing by height per API key and service
|
||||
- *remote*: List only configured services on the dashboard
|
||||
- *serve*: Relay service prompts from download jobs to the REST client
|
||||
- *remote*: Let a remote client license with its own device identity
|
||||
- *dl*: Add --latest-episodes N to download the N newest episodes
|
||||
- *setup*: Add unshackle setup command and cross-platform install scripts
|
||||
- *remote*: License with the client's device when the user picked one
|
||||
- *dl*: Add --all-drm to license Widevine and PlayReady per track
|
||||
- *dl*: Add --audio-description-only and --forced-subs-only
|
||||
- *serve*: Log CDM calls for the dashboard
|
||||
- *serve*: Reload unshackle.yaml on SIGHUP without a restart
|
||||
- *serve*: Add --cdm-only for a CDM backend with no REST API
|
||||
- *service*: Add wait_for_approval and return the cache for a login with no prompt
|
||||
- *serve*: Relay approval messages to jobs and proxy remote downloads
|
||||
- *search*: Add ANONYMOUS_SEARCH to skip the login for a service
|
||||
- *dl*: Add --sample and --include-sample
|
||||
- *dl*: Load the DRM system a cdm entry lists first
|
||||
- *api*: Forward exact_lang to the service context
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- *downloader*: Release hedge loser file handles before merge sweep
|
||||
- *downloader*: Validate 206 on byte-range segments and force identity encoding
|
||||
- *downloader*: Stop leaked racers holding segment handles after a failed batch
|
||||
- *dash*: License ClearKeyCENC before decrypt
|
||||
- *subtitle*: Convert ASS to SRT in the SDH-strip fallback
|
||||
- *api*: Keep sub_format "original" working through the API
|
||||
- *webvtt*: Don't shorten cues when splicing boundary duplicates
|
||||
- *api*: Reject backslash traversal in safe_cache_key
|
||||
- *downloader*: Honor Retry-After with cancel-safe backoff, add fault-injection bench and tests
|
||||
- *downloader*: Keep speed-limited batches in a single process
|
||||
- *downloader*: Fail closed on rnet content codings outside the old allowlist
|
||||
- *downloader*: Accept a 200 whose body is exactly the requested byte-range slice
|
||||
- *downloader*: Report rolling speed on the multiprocess progress bar
|
||||
- *downloaders*: Submit segments in a window so progress can drain
|
||||
- *api*: Pass adaptive_workers and download_processes to dl.result
|
||||
- *downloader*: Validate Content-Range before placing 206 bytes
|
||||
- *downloader*: Harden ranged reads, teardown and multiprocess sessions
|
||||
- *titles*: Drop folder {year?} when it repeats the dated season label
|
||||
- *cli*: Work around rich-click's patched click classes
|
||||
- *vaults*: Look tables up by the name the vault stores
|
||||
- *hls*: Keep ad-filtered segments out of DRM key state
|
||||
- *music*: Remux songs into the container of their codec
|
||||
- *api*: Survive Windows errno 13 on the job progress file
|
||||
- *api*: Guard two more Windows errno 13 paths in the serve layer
|
||||
- *dl*: Accept a clamped 206 and report audio-description-only titles
|
||||
- *services*: Match service aliases without case
|
||||
- *downloader*: Keep retrying while a cut connection makes progress
|
||||
- *downloader*: Bound requests after a cut and stop retry nesting
|
||||
- *subtitle*: Keep WebVTT-only cue markup out of converted subtitles
|
||||
- *proxies*: Resolve windscribevpn hostname prefixes and uk alias
|
||||
- *mux*: Give each parallel mux its own chapters temp file
|
||||
- *remote*: Carry the full title and track state across the remote API
|
||||
- *HTTP*: Simplify key retrieval condition in get_key method
|
||||
- *tracks*: Match resolutions per range and codec
|
||||
- *remote*: Resolve service names against the remote server
|
||||
- *remote*: Send only the active profile's cache files to the server
|
||||
- *utils*: Preserve _ssl_context when copying SSLCiphers adapter
|
||||
- *util*: Remove :
|
||||
- *cli*: Show banner when dl -q is used
|
||||
- *serve*: License multi-key PlayReady through server CDM instead of vault subset
|
||||
- *remote*: Raise GET timeout to 120s so large catalogs load
|
||||
- *cdm*: Scope remote CDM licence state to its session
|
||||
- *remote*: Pass fallback language to ISM re-parse
|
||||
- *ism*: Signal Dolby Vision and Atmos from Smooth manifests
|
||||
- *dash*: Number SegmentTimeline segments from startNumber
|
||||
- *remote*: Match re-parsed manifest tracks by ID before attributes
|
||||
- *remote*: Send per-track MPD fragments for service-built AdaptationSets
|
||||
- *dl*: Repair damaged cookie files without rewriting them
|
||||
- *kv*: Keep copy log rows single-spaced
|
||||
- *dl*: Fall back to local vaults when the server CDM returns no key
|
||||
- *cli*: Stop service tags from renaming CLI options
|
||||
- *dl*: Log vault writes under their own title
|
||||
- *dl*: Specify 'subtitle tracks' in error log message
|
||||
- *remote*: Keep option choices and optional title in remote -h
|
||||
- *audio*: Accept immersive channel layouts like 5.1.4
|
||||
- *serve*: Resolve DRM keys for manifests with no ContentProtection PSSH
|
||||
- *dl*: Reject local proxies when --remote targets another machine
|
||||
- *dl*: End a cancelled track by raising instead of returning short
|
||||
- *services*: Name the build a compiled service is missing
|
||||
- *serve*: Lock shared session state and throttle progress writes
|
||||
- *downloader*: Correct truncation, ranged sizing, and session carry
|
||||
- *dl*: Cache server-CDM keys to the local vaults
|
||||
- *drm*: License every PlayReady header until all KIDs are covered
|
||||
- *dash*: Treat a missing FFprobe as no PSSH instead of crashing
|
||||
- *api*: License every PlayReady header in the server CDM path
|
||||
- *api*: Reach every PlayReady header on all server CDM routes
|
||||
- *api*: Let the server CDM see service-supplied keys and HLS track DRM
|
||||
- *api*: Tell the service that the client sent a cookie jar
|
||||
- *api*: Stop serve leaking server credentials, paths, and cross-tenant data
|
||||
- *api*: Keep error URLs readable and mask host paths in dashboard logs
|
||||
- *api*: Reject a NUL byte in output_dir before resolving it
|
||||
- *manifests*: Flag DASH audio using the standard description role
|
||||
- *api*: Serialize the job progress replace to stop Windows errno 13
|
||||
- *api*: Decode challenge bytes to UTF-8 for PlayReady license request
|
||||
- *proxy*: Check the proxy exit IP before any service request
|
||||
- *cookies*: Find Firefox profiles in the XDG config directory
|
||||
- *naming*: Correct language tag rule matching and add state conditions
|
||||
- *config*: Keep no and yes as language tags
|
||||
- *import*: Inject only the imported title's keys into each track
|
||||
- *main*: Remove unwanted -
|
||||
- *serve*: Defer service hot reload while a remote session uses it
|
||||
- *remote*: Return session cache for device-code and cache-only logins
|
||||
- *remote*: Honor proxy client cdm_type for DRM system selection
|
||||
- *drm*: Log missing Widevine/PlayReady PSSH at debug level
|
||||
- *cli*: Make the click param types the one source of codec spellings
|
||||
- *vaults*: Verify vault content keys per KID from the fragment map
|
||||
- *vaults*: Harden the KID window check and clear a stale flag on licence
|
||||
- *remote*: Forward nested service cache files to and from the server
|
||||
- *remote*: Stop warning about missing keys for tracks that carry no DRM
|
||||
- *remote*: Keep the server's own CDM when the tier licenses with server_cdm
|
||||
- *post-scripts*: Expose {part} and all title-owned variables on failure
|
||||
- *serve*: Apply staged service reloads when sessions end, with a max staging age
|
||||
- *remote*: Let the server's CDM mapping decide the DRM system under server_cdm
|
||||
- *remote*: Forward dl track selection and profile to remote sessions
|
||||
- *dl*: Skip empty vault pushes and count cached keys once
|
||||
- *serve*: Stop repeating server vault pushes and retry failed ones
|
||||
- *export*: Import side-loaded subtitles and keep conflicting keys apart
|
||||
- *export*: Strip session headers from exports and normalise merged keys
|
||||
- *import*: Read export files through mediaexport
|
||||
- *export*: Skip bad keys and failed writes without stopping the download
|
||||
- *import*: Report refused titles and fail when none is usable
|
||||
- *drm*: Give the zero-KID fallback only the track's own key
|
||||
- *import*: Give each track only the keys it declares
|
||||
- *import*: Stop clearly when no export or vault key exists
|
||||
- *export*: Name the file from the first write
|
||||
- *import*: Say an import uses the export's keys, not a server CDM
|
||||
- *import*: Give no title keys to a track that is clear
|
||||
- *import*: Read an HLS export that names no title language or master key
|
||||
- *remote*: Report a missing title list and a refused track URI clearly
|
||||
- *tracks*: Keep the Dolby Vision record when remuxing video with ffmpeg
|
||||
- *proxies*: Match surfshark's current server api
|
||||
- *proxies*: Skip a proxy provider that fails to load
|
||||
- *proxies*: Return None for unservable queries and repair live providers
|
||||
- *proxies*: Link to the expressvpn device login with the code prefilled
|
||||
- *cdm*: Skip the server header probe for pywidevine and pyplayready remote CDMs
|
||||
- *api*: Stop dashboard event and log reads racing worker threads
|
||||
- *api*: Keep the worker's error on failed jobs, gate tracebacks behind --debug-api
|
||||
- *sanitize*: Keep nonspacing marks on letters in unicode filenames
|
||||
- *sanitize*: Normalise filenames to NFC and keep combining marks
|
||||
- *console*: Write UTF-8 to redirected streams on Windows
|
||||
- *dl*: Keep the --enrich original language after get_tracks
|
||||
- *attachment*: Keep custom fonts after mux cleanup
|
||||
- *dl*: Keep the --enrich title and year after get_tracks
|
||||
- *docs*: Hide the release version in the header repo widget
|
||||
- *dl*: Read the vaults first under --all-drm
|
||||
- *hybrid*: Read the container file for a base that had the DV fixup
|
||||
- *cdm*: Send a PlayReady licence without an XML declaration as XML
|
||||
- *import*: Carry the HLS segment filter in an export
|
||||
- *cdm*: Simplify cached keys retrieval in CustomRemoteCDM
|
||||
- *hls*: Read the key id from the init segment when the PSSH has none
|
||||
- *api*: Follow the cdm config key order for the server CDM
|
||||
|
||||
### Documentation
|
||||
|
||||
- *session*: Correct stale pool-size comment
|
||||
- Strip non-essential comments, fold key facts into docstrings
|
||||
- *services*: Add MUSIC_EXAMPLE reference service
|
||||
- Apply simplified technical english across docs and docstrings.
|
||||
- Phase 2 of the prose style across docs and docstrings
|
||||
- Widen supported shaka-packager range to 2.6.1 or >= 3.10.0
|
||||
- *dev*: Correct get_chapters guidance on markers at timestamp 0
|
||||
- *example*: Show drm_preference pinning for dual-PSSH manifests
|
||||
- *post-scripts*: Use post-script as the one term for hooks
|
||||
- *api*: Describe the dashboard session payload in the OpenAPI yaml
|
||||
- *cli*: Correct --merge-video scope in cli reference
|
||||
- *export*: Describe the export file name and its messages
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
- *downloader*: Slow-start ramp for the adaptive worker controller
|
||||
- *downloader*: Use per-arrival reads only while a segment is hedge-raced
|
||||
- *dash*: Collapse single-URL byte-range segments into one direct download
|
||||
- *downloader*: Stride multiprocess chunk assignment across children
|
||||
- *api*: Skip progress re-parse when the file is unchanged
|
||||
- *api*: Append job history instead of rewriting the file
|
||||
- *downloader*: Unblock hedge-loser reads at teardown
|
||||
- *dash*: Speed up segment collection and URL templating
|
||||
- *dl*: Parallelise mux, licensing, and vault lookups; size the shared pool
|
||||
- *dl*: Overlap hybrid builds with muxing, tag in the mux thread, preload DRM in parallel
|
||||
- *hybrid*: Reuse DV-side files across resolutions, trim the RPU chain, fix L6 fallback
|
||||
- *dl*: Push vault key writes to a background thread
|
||||
- *drm*: Decrypt fMP4 segments during download with mp4decrypt
|
||||
- *kv*: Push whole services to SQL vaults in one call
|
||||
- *kv*: Push keys to vaults concurrently, log one summary per title
|
||||
- *sample*: Download only the subtitle segments around the window
|
||||
- *manifests*: Decode manifests without charset detection
|
||||
|
||||
### Changes
|
||||
|
||||
- *monalisa*: Remove ML-Worker and improve MonaLisa DRM
|
||||
- *downloader*: Extract shared byte-range split helper
|
||||
- *console*: Route interactive prompts through a themed helper
|
||||
- *core*: [**breaking**] Drop leading underscores from internal definitions
|
||||
- *tests*: Drop leading underscores from test helper functions
|
||||
- *music*: [**breaking**] Make Song an ordinary Title on the generic path
|
||||
- *console*: Style env listings like the rich-click help screens
|
||||
- *drm*: Drop the hardcoded service branch in the license flow
|
||||
- *proxies*: Share proxy dispatch between dl, search and resolve_proxy
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Restore memory/ to .gitignore from dev
|
||||
- *scripts*: Expand bench suite for cross-machine runs
|
||||
- *scripts*: Harden bench suite report capture
|
||||
- *scripts*: Add attempt-1-only tail-slow fault to downloader bench
|
||||
- Bump dev version to 5.5.0
|
||||
- Mark install.sh executable
|
||||
|
||||
### Builds
|
||||
|
||||
- *deps*: Pull mediaexport from github in the package metadata
|
||||
- *deps*: Lock mediaexport at fa9357e
|
||||
|
||||
## [5.4.0] - 2026-08-15
|
||||
|
||||
### Features
|
||||
|
||||
@@ -100,6 +100,7 @@ Declared at class level to configure framework behaviour:
|
||||
| `GEOBLOCK` | `tuple[str, ...]` | IP region codes where the service refuses to work. Every other region is allowed. Default `()`. |
|
||||
| `VAULT_TAG` | `Optional[str]` | Overrides the key-vault namespace so sibling services can share one vault. Default `None` (use the service's own tag). |
|
||||
| `AUTH_METHODS` | `Optional[tuple[str, ...]]` | Auth methods accepted (`"cookies"` / `"credentials"`). When `None`, the REST `/services` endpoint infers them from `authenticate()`. |
|
||||
| `ANONYMOUS_SEARCH` | `bool` | Set `True` when `search()` needs no login and no state from `authenticate()`. A search then skips the cookie load, `authenticate()` and the cookie save. Default `False`. |
|
||||
| `NO_SUBTITLES` | `bool` | Set `True` on a service with no subtitle tracks to skip subtitle handling entirely. |
|
||||
| `ANIME` | `bool` | Set `True` when the catalogue is anime, so metadata lookups prefer AniList. A title's own `anime` flag overrides it. |
|
||||
| `DAILY` | `bool` | Set `True` when the catalogue is daily/date-based (talk shows, news, sports), so unshackle names episodes by air date. A title's own `daily` flag overrides it. |
|
||||
@@ -256,6 +257,28 @@ REST download job shows it as `input_prompt`. Locally it routes through the
|
||||
shared Rich console (`prompt_user`) so the prompt renders correctly alongside
|
||||
progress and log output.
|
||||
|
||||
For a login that the user approves on another device (a pair code, a QR code),
|
||||
use `self.wait_for_approval(message, check, timeout)`, never a hand-written
|
||||
loop. It shows the message, then calls `check` at each interval until `check`
|
||||
returns a value. `check` returns `None` while the approval is pending.
|
||||
|
||||
```python
|
||||
token = self.wait_for_approval(
|
||||
f"Open {self.config['endpoints']['pair']} and enter code {code}",
|
||||
check=lambda: self.poll_pair_token(code),
|
||||
timeout=expires_in,
|
||||
)
|
||||
```
|
||||
|
||||
It raises `TimeoutError` after `timeout` seconds. Under `serve` mode it also stops
|
||||
when the client deletes the remote session, so the loop does not continue on the
|
||||
server with no client. The message goes out as a log line, so a remote
|
||||
session shows it. A REST download job shows it in its `notice` field until the wait ends.
|
||||
|
||||
The wait sees a cancel only between two `check` calls, so `check` must return within a few
|
||||
seconds. A `check` that reads a socket keeps the socket open across calls and returns `None`
|
||||
after each short idle wait.
|
||||
|
||||
!!! warning "Make each token cache entry unique"
|
||||
The cache above uses `tokens_{device}_{profile}` on purpose. A token cache
|
||||
must include every dimension that changes the token (device, profile, or
|
||||
@@ -587,6 +610,12 @@ Override any of these to react to pipeline stages (all no-ops by default):
|
||||
`on_track_repacked`, `on_track_multiplex`. Also override `search()` to yield
|
||||
`SearchResult` objects for `unshackle search`.
|
||||
|
||||
!!! warning "Set `ANONYMOUS_SEARCH` only for a search that needs no account"
|
||||
With `ANONYMOUS_SEARCH = True`, `search()` operates on an HTTP session with no cookies, and
|
||||
`authenticate()` does not run. Set it only when the search results do not depend on the
|
||||
account, and when `search()` does its own setup. A `search()` that reads a token, a header
|
||||
or a region from `authenticate()` fails or gives results for the wrong region.
|
||||
|
||||
```python
|
||||
def search(self) -> Generator[SearchResult, None, None]:
|
||||
results = self.session.get(self.config["endpoints"]["search"],
|
||||
|
||||
@@ -98,6 +98,7 @@ Show the streaming services available on this server, filtered by your allowlist
|
||||
],
|
||||
"needs_auth": true,
|
||||
"has_search": true,
|
||||
"anonymous_search": false,
|
||||
"has_drm": true,
|
||||
"auth_methods": ["cookies"]
|
||||
}
|
||||
@@ -120,6 +121,7 @@ Field notes:
|
||||
| `cli_params` | Array of parameter descriptors. Arguments have `{name, kind: "argument", required}`; options have `{name, kind: "option", opts, is_flag, default, help, type, multiple}` and, for choice options, a `choices` list. |
|
||||
| `needs_auth` | Whether the service overrides `authenticate()`. |
|
||||
| `has_search` | Whether the service supports `POST /api/search`. |
|
||||
| `anonymous_search` | Whether `POST /api/search` operates with no login for this service. The server then ignores `credentials` and `cookies` on a search. |
|
||||
| `has_drm` | Whether the service overrides a Widevine or PlayReady license hook. |
|
||||
| `pending_update` | Set to `true` while a service repository refresh waits for this service's jobs to finish before it swaps in the new code. Absent otherwise. |
|
||||
| `server_accounts` | Present when the server lends its own accounts for this service: `{regions: ["ca", "gb"], global: bool}`, the regions those accounts cover. A client sends no credentials, cookies, or cache for such a service. |
|
||||
@@ -134,6 +136,8 @@ Field notes:
|
||||
|
||||
Find titles in a service by query string. The service must have a `search()` method (see `has_search` above).
|
||||
|
||||
For a service with `anonymous_search`, the server does not log in and ignores `credentials` and `cookies`. The server account check still applies.
|
||||
|
||||
**Request body**
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
@@ -197,6 +201,13 @@ Get the list of titles behind a title ID, for example episodes or a movie, witho
|
||||
|
||||
You can also pass service-specific CLI options as additional keys.
|
||||
|
||||
#### Logins that need the user
|
||||
|
||||
`search`, `list-titles` and `list-tracks` have no channel to the user. A login that asks for
|
||||
input or waits for an approval on another device fails at once with `AUTH_FAILED` on these
|
||||
endpoints. Complete that login in a download job or a remote session, then send the saved
|
||||
login with the request.
|
||||
|
||||
#### Client cache on a remote-only server
|
||||
|
||||
A `--remote-only` server keeps nothing that a client sends. When the request
|
||||
@@ -305,7 +316,7 @@ Show the video, audio, and subtitle tracks for a title. For series, you can scop
|
||||
| `part` | int/string | no | Part index of a split episode (combined with `season` and `episode`). |
|
||||
| `profile`, `proxy`, `no_proxy`, `cdm_type` | - | no | As on `list-titles`. |
|
||||
| `credentials`, `cookies`, `cache` | - | no | As on `list-titles`. |
|
||||
| `dl_params` | object | no | The `dl` track selection that the service reads from `ctx.parent.params`: `lang`, `v_lang`, `a_lang`, `acodec` and `forced_subs`, as on [`POST /api/session/create`](#post-apisessioncreate). Some services select the manifests they fetch from these values. |
|
||||
| `dl_params` | object | no | The `dl` track selection that the service reads from `ctx.parent.params`: `lang`, `v_lang`, `a_lang`, `acodec`, `forced_subs` and `exact_lang`, as on [`POST /api/session/create`](#post-apisessioncreate). Some services select the manifests they fetch from these values. |
|
||||
|
||||
When you give both `season` and `episode`, the server combines them into a `"{season}x{episode}"` selector. `part` narrows that to `"{season}x{episode}.{part}"`, one part of a [split episode](../creating-a-service.md#split-episodes). Without `part` you get every part of the episode. The server reads `part` only when you give both `season` and `episode`, and `wanted` takes precedence over all three. To scope a part by range, put it in `wanted` instead (`"S01E01.2"`).
|
||||
|
||||
@@ -451,6 +462,8 @@ Make a download job. It requires `service` and `title_id`. Every other field is
|
||||
| `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. |
|
||||
| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://<resolver>@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. |
|
||||
| `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. |
|
||||
| `sample` | boolean/string | `null` | Download a short sample and write it with a `SAMPLE` tag on the filename in a `Sample` folder inside the output folder. `true` is 60 seconds from 2 minutes in; `"0-60"` is a window in seconds. |
|
||||
| `include_sample` | boolean | `false` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder next to it. |
|
||||
| `workers` | int | `null` | Threads per track. |
|
||||
| `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. |
|
||||
| `download_processes` | int | `1` | Split a track's segments across this many processes. Only engages for large batches. |
|
||||
@@ -556,7 +569,8 @@ Show download jobs, with optional filtering and sorting.
|
||||
"segments_total": 300,
|
||||
"speed": "5.2 MB/s",
|
||||
"skipped_subtitles": [],
|
||||
"input_prompt": null
|
||||
"input_prompt": null,
|
||||
"notice": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -721,6 +735,10 @@ example an OTP code, a PIN, or a device-code confirmation. The job then shows th
|
||||
`input_prompt` field and in its `progress` events. Show the prompt to the user and send the
|
||||
answer here.
|
||||
|
||||
A job that waits for an approval on another device, such as a pair code login, shows the
|
||||
message in its `notice` field instead. A notice needs no answer: show it to the user and keep
|
||||
polling.
|
||||
|
||||
=== "Request"
|
||||
|
||||
```json
|
||||
@@ -940,7 +958,7 @@ the newlines must go out at once.
|
||||
|
||||
Make a remote session for a service and title. Authentication runs asynchronously in the background. This call returns immediately with `status: "authenticating"`, and you then poll the prompt endpoint. The body accepts `service` and `title_id` (both required). It also accepts a broad set of optional keys, because the body allows `additionalProperties`. These are `credentials` (`{username, password, extra?}`), `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, `no_proxy`, `profile`, `cache` (map of forwarded cache files, keyed by the file path relative to the service cache directory with `/` separators and no `.json` suffix, so a nested key such as `session_web/<sha1>` lands in a subdirectory), `client_region`, `proxy_region`, `cdm_type`, `range_`, `vcodec`, `quality`, `best_available`, `dl_params`, `client`, and any service CLI options. `proxy_region` is the country code the client resolved `proxy` from; the server matches it against its own accounts. `client` is a freeform object the dashboard shows as sent (the CLI puts `version`, `code_hash`, `platform` and a redacted `argv` in it); the server ignores it above 4096 bytes of JSON.
|
||||
|
||||
`dl_params` is an object with the `dl` track selection: `lang`, `v_lang` and `a_lang` (arrays of language strings), `acodec` (an array of audio codec names or `dl -a` aliases such as `ddp`) and `forced_subs` (boolean). The service reads these values from `ctx.parent.params`. The server uses the `dl` default for each absent or malformed value, and drops an unknown codec name.
|
||||
`dl_params` is an object with the `dl` track selection: `lang`, `v_lang` and `a_lang` (arrays of language strings), `acodec` (an array of audio codec names or `dl -a` aliases such as `ddp`), and `forced_subs` and `exact_lang` (booleans). The service reads these values from `ctx.parent.params`. The server uses the `dl` default for each absent or malformed value, and drops an unknown codec name.
|
||||
|
||||
Service CLI options also travel in a nested `service_params` object, which wins over a flat key with the same name. `profile` at the top level always means the credentials profile, never a service's own `--profile` option.
|
||||
|
||||
@@ -957,7 +975,7 @@ The `proxy` value must be a full proxy URI, unless the operator gives your API k
|
||||
}
|
||||
```
|
||||
|
||||
`server_account` is `true` when the server authenticated with one of its own accounts (`serve.server_accounts`). Otherwise the server uses only what the client sent, and never falls back to its own credentials. Only a client that sent its own cookies, credentials, or cache files gets cookies, auth headers, and the remote session cache back. A client that answered a login prompt (a device code or an OTP) gets them too. This applies only when the login then succeeds and the remote session does not use a server account. For any other remote session (a server account, or an anonymous login through the server's proxy) the tracks and segments responses carry no cookies and no auth headers. The server also drops secret-looking keys from track data, and the cache stays on the server. A server-account service also refuses a client proxy URI, because the server's login would pass through it.
|
||||
`server_account` is `true` when the server authenticated with one of its own accounts (`serve.server_accounts`). Otherwise the server uses only what the client sent, and never falls back to its own credentials. When the login succeeds and the remote session does not use a server account, the login belongs to the client, and the client gets cookies, auth headers, and the remote session cache back. A remote session on the server's device is the exception: the client gets its cookies back, but no auth headers and no cache. For a server-account remote session the tracks and segments responses carry no cookies and no auth headers. The server also drops secret-looking keys from track data, and the cache stays on the server. A server-account service also refuses a client proxy URI, because the server's login would pass through it.
|
||||
|
||||
| Status | Error code | Meaning |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -63,6 +63,7 @@ The `serve` command accepts the following flags:
|
||||
| `-h`, `--host` | `127.0.0.1` | Address to bind to. Use `0.0.0.0` to accept connections from other machines. |
|
||||
| `-p`, `--port` | `8786` | Port to bind to. |
|
||||
| `--api-only` | off | Serve only the REST API, without the Widevine/PlayReady CDM HTTP endpoints. |
|
||||
| `--cdm-only` | off | Serve only the Widevine/PlayReady CDM HTTP endpoints, plus `/api/health` and the dashboard. No REST API, no remote sessions, no Swagger UI. Cannot be combined with `--api-only` or `--remote-only`. |
|
||||
| `--no-widevine` | off | Disable the Widevine CDM endpoints. Cannot be combined with `--api-only`. |
|
||||
| `--no-playready` | off | Disable the PlayReady CDM endpoints. Cannot be combined with `--api-only`. |
|
||||
| `--no-key` | off | Disable API-key authentication entirely; every request is allowed. |
|
||||
@@ -101,6 +102,13 @@ The routes the server mounts depend on how you start the server:
|
||||
and everything under `/api/session/*`). This mode implies `--api-only`, and the
|
||||
interactive Swagger UI is **not** mounted.
|
||||
|
||||
=== "`--cdm-only`"
|
||||
|
||||
The server mounts only the pywidevine and pyplayready CDM endpoints. Under `/api/` it
|
||||
answers `/api/health` and, when `serve.dashboard.key` is set, the dashboard. It does not
|
||||
start the remote session store, the service refresh loop or the Swagger UI. Use it for a
|
||||
CDM backend that a proxy reaches on its own hostname.
|
||||
|
||||
## Base URL
|
||||
|
||||
All REST endpoints live under the `/api` prefix. With the default bind address the
|
||||
|
||||
@@ -163,6 +163,7 @@ full detail:
|
||||
| `speed` | string | Transfer speed of the track in progress, e.g. `4.20 MB/s`. |
|
||||
| `skipped_subtitles` | list | Subtitles that were skipped as non-fatal failures (when `skip_subtitle_errors` is set). |
|
||||
| `input_prompt` | string or `null` | The prompt the service waits on. Answer it with `POST /api/download/jobs/{job_id}/input`. |
|
||||
| `notice` | string or `null` | A message the service shows while it waits for an approval on another device, such as a pair code. It needs no answer. |
|
||||
|
||||
The full detail adds `started_time`, `completed_time`, `output_files`, the redacted
|
||||
`parameters`, and, for failed jobs, `error_message`, `error_details`, and `error_code`.
|
||||
@@ -368,6 +369,8 @@ Three temporary files bridge the parent and the job worker:
|
||||
The job worker has no terminal. When a service asks the user for input, the job worker writes
|
||||
the prompt to `progress.json` as `input_prompt` and waits. The answer from
|
||||
`POST /api/download/jobs/{job_id}/input` goes to the job worker on its stdin.
|
||||
When a service waits for an approval on another device, the job worker writes the message as
|
||||
`notice` and sets it back to `null` when the wait ends.
|
||||
|
||||
Running each download out-of-process isolates the server from crashes, memory growth,
|
||||
and blocking work inside the download engine, and makes hard cancellation (killing the
|
||||
|
||||
@@ -150,6 +150,7 @@ curl "$BASE/api/services" \
|
||||
"help": "EXAMPLE service documentation...",
|
||||
"needs_auth": true,
|
||||
"has_search": true,
|
||||
"anonymous_search": false,
|
||||
"has_drm": true,
|
||||
"auth_methods": ["cookies", "credentials"],
|
||||
"cli_params": []
|
||||
|
||||
@@ -127,6 +127,9 @@ hood the client walks a remote session through its lifecycle.
|
||||
URI and runs its own forwarder for it. A bare region skips Control D, because its
|
||||
proxy is a forwarder on the client. See
|
||||
[Control D](../../guide/proxies-and-vpn.md#control-d)
|
||||
The client also downloads playlists and segments through the same proxy,
|
||||
as a local run does. `--no-proxy-download` and `--proxy-download` change
|
||||
that for downloads only.
|
||||
- Track-selection hints (`range_`, `vcodec`, `quality`, `best_available`) so
|
||||
the server fetches the right manifests
|
||||
- Your language and audio codec selection (`-l`, `-vl`, `-al`, `-a`, `-fs`)
|
||||
@@ -213,9 +216,9 @@ hood the client walks a remote session through its lifecycle.
|
||||
deletes the remote session. If the login belongs to the client, the `DELETE`
|
||||
request returns the updated cache files (for example a refreshed token) and the
|
||||
client saves them locally, so the **next** remote session can skip interactive
|
||||
authentication. The login belongs to the client when the client sent
|
||||
cookies, credentials, or cache files, or answered a login prompt (a device code
|
||||
or an OTP) that led to a successful login. A server-account login never does,
|
||||
authentication. The login belongs to the client when it succeeds and the
|
||||
remote session does not use a server account. This includes a pair code that
|
||||
the user approves on another device. A server-account login never does,
|
||||
and neither does a remote session on the server's device (see below).
|
||||
The client saves only the returned files that pass the same profile check it
|
||||
applies before it sends them, so a file for another profile cannot overwrite
|
||||
@@ -429,11 +432,11 @@ object the dashboard shows as sent), and the track-selection hints `range_`, `vc
|
||||
|
||||
`dl_params` holds the `dl` track selection: `lang`, `v_lang` and `a_lang`
|
||||
(arrays of language strings), `acodec` (an array of audio codec names, such as
|
||||
`EC3`), and `forced_subs` (boolean). The server gives these values to the
|
||||
`EC3`), and `forced_subs` and `exact_lang` (booleans). The server gives these values to the
|
||||
service in `ctx.parent.params`, never as service options, so a service option
|
||||
with the same name, such as an own `--lang`, does not get them. The server uses
|
||||
the `dl` default for each absent or malformed value: `lang` is `["orig"]`,
|
||||
`v_lang`, `a_lang` and `acodec` are `[]`, and `forced_subs` is `false`. The
|
||||
`v_lang`, `a_lang` and `acodec` are `[]`, and `forced_subs` and `exact_lang` are `false`. The
|
||||
server drops an unknown codec name. An empty array is a valid value and stays
|
||||
empty.
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ Declared at the top of your service class, these are static descriptors:
|
||||
| `GEOBLOCK` | `tuple[str, ...]` | ISO region codes where the service refuses to work; every other region is allowed. Use it for a service that works worldwide except in its home market. |
|
||||
| `VAULT_TAG` | `Optional[str]` | Overrides the key-vault namespace; defaults to the service's own tag. |
|
||||
| `AUTH_METHODS` | `Optional[tuple[str, ...]]` | Accepted auth methods (`"cookies"` / `"credentials"`). When `None`, the REST `/services` endpoint infers them from `authenticate()`. |
|
||||
| `ANONYMOUS_SEARCH` | `bool` | `search()` needs no login and no state from `authenticate()`. A search then skips the cookie load, `authenticate()` and the cookie save. Default `False`. |
|
||||
| `ANIME` | `bool` | The catalogue is anime, so metadata lookups prefer AniList. A title's own `anime` flag overrides it. |
|
||||
| `DAILY` | `bool` | The catalogue is daily/date-based, so unshackle names episodes by air date. A title's own `daily` flag overrides it. |
|
||||
|
||||
@@ -83,12 +84,17 @@ These are abstract. A service will not load unless it defines all three:
|
||||
`CookieJar` into `self.session.cookies` and stores `self.credential`. Override it to
|
||||
do a real login handshake.
|
||||
- **`search()`**: raises `NotImplementedError` by default. Write it so that the
|
||||
`search` command can operate.
|
||||
`search` command can operate. unshackle calls `authenticate()` first, unless the
|
||||
service sets `ANONYMOUS_SEARCH`.
|
||||
- **DRM license hooks**: `get_widevine_service_certificate`, `get_widevine_license`,
|
||||
`get_playready_license` (delegates to Widevine by default), and
|
||||
`get_clearkey_license` (returns `None` by default). See [DRM hooks](#drm-hooks) below.
|
||||
- **`request_input(prompt)`**: prompts the user. It goes through the remote-dl input
|
||||
bridge, the REST download job relay, or the rich console.
|
||||
- **`wait_for_approval(message, check, timeout, interval=3.0)`**: shows a message, then
|
||||
calls `check` until it returns a value. Use it for a login that the user approves on
|
||||
another device, such as a pair code. A remote-dl session shows the message as a log
|
||||
line, and a REST download job shows it as the job's `notice`.
|
||||
- **Event callbacks**: `on_segment_downloaded`, `on_track_downloaded`,
|
||||
`on_track_decrypted`, `on_track_repacked`, `on_track_multiplex` let a service react to
|
||||
pipeline events (for example, MonaLisa decrypts each segment in `on_segment_downloaded`).
|
||||
|
||||
@@ -153,6 +153,8 @@ Keep only certain track types, or skip certain track types.
|
||||
| `--no-folder` | Disable folder creation for TV shows. |
|
||||
| `--no-source` | Remove the source tag from the filename/path. |
|
||||
| `--no-mux` | Do not mux; keep individual track files. |
|
||||
| `--sample` | Download a short sample and write it with a `SAMPLE` tag on the filename in a `Sample` folder inside the output folder. Bare `--sample` = 60 seconds from 2 minutes in; `--sample 0-60` = a custom window in seconds. |
|
||||
| `--include-sample` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder next to it. |
|
||||
| `--tag` | Group tag override. |
|
||||
| `--repack` | Add a `REPACK` tag to the filename. |
|
||||
| `--postscript` | Run a command after each output file, with `{variable}` placeholders substituted. Repeatable. Replaces the `post_scripts` config for this run. See [Post-scripts](../reference/configuration/post-scripts.md). |
|
||||
@@ -240,6 +242,8 @@ unshackle search [OPTIONS] SERVICE [QUERY...]
|
||||
|
||||
Each service defines its own query syntax. unshackle prints the results as a tree of titles with their service IDs. Feed an ID straight into `dl`.
|
||||
|
||||
unshackle logs in to the service before the search. A service that declares a search with no login is the exception: unshackle then reads no cookies and no credentials, and `--profile` has no effect.
|
||||
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `-p`, `--profile` | Profile for credentials and cookies. |
|
||||
|
||||
@@ -711,9 +711,24 @@ By default, `mkvmerge` muxes the tracks of a movie or an episode into a single M
|
||||
| Flag | Behaviour | Default source |
|
||||
| --- | --- | --- |
|
||||
| `--no-mux` | Do not mux; keep the individual track files. | - |
|
||||
| `--sample` | Download a short sample: 60 seconds from 2 minutes in, or `--sample START-END` for a window in seconds. | - |
|
||||
| `--include-sample` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder. | - |
|
||||
| `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) |
|
||||
| `--merge-video` | Mux video tracks that share a height, range, and codec into one file, so only language varies inside a file. | config `muxing.merge_video` (off) |
|
||||
|
||||
!!! note "`--sample` needs a segmented track"
|
||||
`--sample` downloads only the segments that hold the window, and `mkvmerge` cuts the output
|
||||
to the window. The cut starts at the first video keyframe in the window, so a sample can be
|
||||
some seconds short. unshackle downloads each subtitle track in full and then keeps only the
|
||||
cues of the window.
|
||||
|
||||
The run stops before the download starts in two conditions: a service gives a video or
|
||||
audio track as one file with no segments and no segment index, or the title ends before the
|
||||
window starts.
|
||||
|
||||
A sidecar subtitle file with `sidecar_format: original` is not trimmed. It holds the segments around the
|
||||
window, or the whole file when the track is one file.
|
||||
|
||||
!!! note "Music is never muxed"
|
||||
A song skips the muxer. unshackle keeps the container that the service
|
||||
delivered, such as `.flac`, `.m4a`, or `.mp3`, and writes the metadata into that file.
|
||||
|
||||
@@ -495,6 +495,9 @@ unshackle dl --skip-dl --all-drm --export EXAMPLE 81234567
|
||||
|
||||
- The `cdm` mapping must name a device for each system. A single device name, or `--cdm`,
|
||||
stops the run with an error that names the missing system.
|
||||
- The order of the `widevine` and `playready` config keys in the `cdm` entry sets the order
|
||||
of the two systems, as in a run without the flag. Each track licenses first with the
|
||||
system of the first config key.
|
||||
- A Key Vault is read first, as in a run without the flag. A DRM system sends a challenge
|
||||
only for a KID that no Key Vault and no earlier track of the run gave a content key
|
||||
for. When the first system returns all the KIDs of the second, the second sends no
|
||||
|
||||
@@ -330,6 +330,22 @@ A few muxing behaviors are automatic and not configurable:
|
||||
|
||||
The `--no-mux` flag skips muxing entirely and writes the individual track files, each with a track-type suffix, into the same output/folder structure.
|
||||
|
||||
## Sample files
|
||||
|
||||
[`--sample`](cli-reference.md#output-muxing-files) and `--include-sample` write a sample into a
|
||||
`Sample` folder inside the output folder of the title. The sample keeps the filename of the
|
||||
title, with a `SAMPLE` tag at the end. The tag uses the separator of the output template.
|
||||
|
||||
```text title="A movie with --include-sample"
|
||||
Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG/
|
||||
├── Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG.mkv
|
||||
└── Sample/
|
||||
└── Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG.SAMPLE.mkv
|
||||
```
|
||||
|
||||
With `--sample`, the `Sample` folder holds the only output file. If the run makes no title
|
||||
folder, the `Sample` folder is in the downloads directory.
|
||||
|
||||
## Tags & group naming
|
||||
|
||||
The release-group tag is the `-TAG` portion at the end of scene-style names, which the `{tag}` variable gives.
|
||||
|
||||
@@ -45,6 +45,11 @@ are `widevine` / `playready`. unshackle matches the quality keys first, but only
|
||||
height is known. If the value is still a dict after that, unshackle uses
|
||||
`widevine`/`playready` when present, or else the credential profile name, then `default`.
|
||||
|
||||
When an entry holds `widevine` and `playready`, their order sets which device unshackle loads
|
||||
first. A track that offers the two DRM systems licenses with that system. A track that offers
|
||||
only the other system uses the other device. A `widevine` / `playready` dict in a quality key
|
||||
does not set the order. The server CDM of a remote session follows the same order.
|
||||
|
||||
A quality key can also hold a dict of `widevine` / `playready` devices, and so can the
|
||||
top-level `default` entry:
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "unshackle"
|
||||
version = "5.5.0"
|
||||
version = "6.0.0"
|
||||
description = "Modular Movie, TV, and Music Archival Software."
|
||||
authors = [{ name = "unshackle team" }]
|
||||
requires-python = ">=3.11,<3.15"
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""The search command skips the login for a service that sets ANONYMOUS_SEARCH."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.commands import search as search_command
|
||||
from unshackle.core.search_result import SearchResult
|
||||
|
||||
|
||||
class _SearchService:
|
||||
ANONYMOUS_SEARCH = False
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
self.session = SimpleNamespace(cookies=object())
|
||||
|
||||
def authenticate(self, cookies, credential):
|
||||
self.calls.append("authenticate")
|
||||
|
||||
def search(self):
|
||||
yield SearchResult(id_="abc", title="A Title")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def login_calls(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
|
||||
def record(name, value):
|
||||
def call(*args):
|
||||
calls.append(name)
|
||||
return value
|
||||
|
||||
return call
|
||||
|
||||
dl = search_command.dl
|
||||
monkeypatch.setattr(dl, "get_cookie_jar", record("get_cookie_jar", None))
|
||||
monkeypatch.setattr(dl, "get_credentials", record("get_credentials", None))
|
||||
monkeypatch.setattr(dl, "get_cookie_path", record("get_cookie_path", tmp_path / "cookies.txt"))
|
||||
monkeypatch.setattr(dl, "save_cookies", record("save_cookies", None))
|
||||
return calls
|
||||
|
||||
|
||||
def test_search_authenticates_and_saves_cookies_by_default(login_calls):
|
||||
service = _SearchService()
|
||||
search_command.result(service)
|
||||
|
||||
assert service.calls == ["authenticate"]
|
||||
assert login_calls == ["get_cookie_jar", "get_credentials", "get_cookie_path", "save_cookies"]
|
||||
|
||||
|
||||
def test_anonymous_search_touches_no_login_material_and_no_cookie_file(login_calls):
|
||||
service = _SearchService()
|
||||
service.ANONYMOUS_SEARCH = True
|
||||
search_command.result(service)
|
||||
|
||||
assert service.calls == []
|
||||
assert login_calls == []
|
||||
@@ -17,7 +17,7 @@ from pywidevine.pssh import PSSH
|
||||
from rich.table import Table
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
from unshackle.core.api.handlers import validate_download_parameters
|
||||
from unshackle.core.api.handlers import load_full_cdm, resolve_server_cdm, validate_download_parameters
|
||||
from unshackle.core.cdm.detect import cdm_type_stub, is_widevine_cdm
|
||||
from unshackle.core.drm import PlayReady, Widevine
|
||||
from unshackle.core.manifests import HLS
|
||||
@@ -286,3 +286,42 @@ def test_the_init_segment_adds_the_drm_system_the_track_does_not_have(monkeypatc
|
||||
Track.add_init_drm(track, b"init", None)
|
||||
|
||||
assert track.drm == [wv, pr]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("entry", "expected"),
|
||||
[
|
||||
({"playready": "pr_device", "widevine": "wv_device"}, "pr_device"),
|
||||
({"widevine": "wv_device", "playready": "pr_device"}, "wv_device"),
|
||||
],
|
||||
)
|
||||
def test_the_first_drm_system_of_the_config_entry_loads_first(
|
||||
cmd: dl, monkeypatch: pytest.MonkeyPatch, entry: dict, expected: str
|
||||
) -> None:
|
||||
monkeypatch.setattr("unshackle.commands.dl.config.cdm", {"EXAMPLE": entry})
|
||||
monkeypatch.setattr("unshackle.core.cdm.load_cdm", lambda name, **_: name)
|
||||
monkeypatch.setattr("unshackle.core.config.config.cdm", {"EXAMPLE": entry})
|
||||
cmd.vaults = None
|
||||
|
||||
assert cmd.get_cdm("EXAMPLE") == expected
|
||||
assert load_full_cdm("EXAMPLE", None) == expected
|
||||
assert cmd.get_cdm("EXAMPLE", drm="widevine") == "wv_device"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("entry", "expected"),
|
||||
[
|
||||
({"playready": "pr_device", "widevine": "wv_device"}, "playready"),
|
||||
({"widevine": "wv_device", "playready": "pr_device"}, "widevine"),
|
||||
],
|
||||
)
|
||||
def test_the_server_cdm_stub_is_the_first_drm_system_of_the_config_entry(
|
||||
monkeypatch: pytest.MonkeyPatch, entry: dict, expected: str
|
||||
) -> None:
|
||||
monkeypatch.setattr("unshackle.core.config.config.cdm", {"EXAMPLE": entry})
|
||||
monkeypatch.setattr(
|
||||
"unshackle.core.api.handlers.detect_cdm_type",
|
||||
lambda name, _: "playready" if name == "pr_device" else "widevine",
|
||||
)
|
||||
|
||||
assert is_widevine_cdm(resolve_server_cdm("EXAMPLE", None, None)) is (expected == "widevine")
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""An HLS Widevine key whose PSSH holds a content id and no Key ID takes the Key ID of its init segment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import m3u8
|
||||
from pywidevine.license_protocol_pb2 import WidevinePsshData
|
||||
from pywidevine.pssh import PSSH
|
||||
|
||||
from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY
|
||||
from unshackle.core.drm import PlayReady, Widevine
|
||||
from unshackle.core.import_service import hls_playlist_kids
|
||||
from unshackle.core.manifests.hls import HLS
|
||||
from unshackle.core.tracks import Video
|
||||
from unshackle.core.tracks.track import DownloadContext, Track
|
||||
|
||||
INIT_KID = UUID("11111111-1111-1111-1111-111111111111")
|
||||
PSSH_KID = UUID("22222222-2222-2222-2222-222222222222")
|
||||
TAG_KID = UUID("33333333-3333-3333-3333-333333333333")
|
||||
|
||||
ZERO_KID = UUID(int=0)
|
||||
|
||||
|
||||
def no_kid_pssh(content_id: bytes = b"content-id-only") -> str:
|
||||
return PSSH.new(
|
||||
system_id=PSSH.SystemId.Widevine,
|
||||
init_data=WidevinePsshData(content_id=content_id).SerializeToString(),
|
||||
).dumps()
|
||||
|
||||
|
||||
NO_KID_PSSH = no_kid_pssh()
|
||||
KEY_LINE = (
|
||||
'#EXT-X-KEY:METHOD=SAMPLE-AES-CTR,URI="data:text/plain;base64,{pssh}",'
|
||||
'KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed",KEYFORMATVERSIONS="1"'
|
||||
)
|
||||
|
||||
|
||||
def keyed_segment(pssh: str, extra: str = "", init: str = "init.mp4") -> str:
|
||||
return f'#EXT-X-MAP:URI="{init}"\n{KEY_LINE.format(pssh=pssh)}{extra}\n#EXTINF:6.0,\n{init}.m4s\n'
|
||||
|
||||
|
||||
def playlist(*segments: str) -> str:
|
||||
body = "#EXT-X-DISCONTINUITY\n".join(segments)
|
||||
return f"#EXTM3U\n#EXT-X-VERSION:7\n#EXT-X-TARGETDURATION:6\n{body}#EXT-X-ENDLIST\n"
|
||||
|
||||
|
||||
def key_of(pssh: str, extra: str = "") -> m3u8.Key:
|
||||
return m3u8.loads(playlist(keyed_segment(pssh, extra))).keys[0]
|
||||
|
||||
|
||||
def licence_only(text: str, tmp_path: Path) -> list[tuple[list[UUID], Any]]:
|
||||
"""Run a licence-only download of the playlist and return the Key IDs of each licence call."""
|
||||
media = tmp_path / "media.m3u8"
|
||||
media.write_text(text, encoding="utf-8")
|
||||
track = Video(
|
||||
url="https://example.test/media.m3u8",
|
||||
language="en",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
codec=Video.Codec.AVC,
|
||||
)
|
||||
track.from_file = media
|
||||
licensed: list[tuple[list[UUID], Any]] = []
|
||||
ctx = DownloadContext(
|
||||
save_path=tmp_path / "video.mp4",
|
||||
save_dir=tmp_path / "video_segments",
|
||||
progress=lambda **_: None,
|
||||
license_widevine=lambda drm, track_kid=None: licensed.append((drm.kids, track_kid)),
|
||||
cdm=None,
|
||||
)
|
||||
DOWNLOAD_LICENCE_ONLY.set()
|
||||
try:
|
||||
HLS.download_track(track=track, ctx=ctx)
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
return licensed
|
||||
|
||||
|
||||
def test_download_track_licenses_with_the_init_segment_kid(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
# the probe itself needs the network and FFprobe; the order of the calls is under test
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(lambda *args, **kwargs: INIT_KID))
|
||||
assert licence_only(playlist(keyed_segment(NO_KID_PSSH)), tmp_path) == [([INIT_KID], INIT_KID)]
|
||||
|
||||
|
||||
def test_licence_only_reads_each_later_key_from_its_own_init_segment(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
def probe(master: Any, track: Any, session: Any, init_section: Any = None) -> UUID:
|
||||
return PSSH_KID if init_section is not None and init_section.uri == "init2.mp4" else INIT_KID
|
||||
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(probe))
|
||||
text = playlist(keyed_segment(no_kid_pssh(b"first")), keyed_segment(no_kid_pssh(b"later"), init="init2.mp4"))
|
||||
assert licence_only(text, tmp_path) == [([INIT_KID], INIT_KID), ([PSSH_KID], PSSH_KID)]
|
||||
|
||||
|
||||
def test_get_drm_keeps_a_pssh_that_has_its_own_kid() -> None:
|
||||
pssh = PSSH.new(key_ids=[PSSH_KID], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
drm = HLS.get_drm(key_of(pssh), kid=INIT_KID)
|
||||
assert drm.kids == [PSSH_KID]
|
||||
assert drm.pssh.dumps() == pssh
|
||||
|
||||
|
||||
def test_get_drm_prefers_the_tag_keyid_over_the_init_segment_kid() -> None:
|
||||
drm = HLS.get_drm(key_of(NO_KID_PSSH, f",KEYID=0x{TAG_KID.hex}"), kid=INIT_KID)
|
||||
assert drm.kids == [TAG_KID]
|
||||
|
||||
|
||||
def test_get_drm_replaces_a_placeholder_pssh_kid_and_ignores_a_placeholder_init_kid() -> None:
|
||||
placeholder = PSSH.new(key_ids=[ZERO_KID], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
assert HLS.get_drm(key_of(placeholder), kid=INIT_KID).kids == [INIT_KID]
|
||||
assert HLS.get_drm(key_of(placeholder), kid=ZERO_KID).kids == [ZERO_KID]
|
||||
|
||||
|
||||
def test_session_key_without_a_kid_does_not_stop_the_parse() -> None:
|
||||
with_kid = PSSH.new(key_ids=[PSSH_KID], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
drm = HLS.get_all_drm([key_of(NO_KID_PSSH), key_of(with_kid)])
|
||||
assert [d.kids for d in drm] == [[PSSH_KID]]
|
||||
|
||||
|
||||
def test_alternate_drm_builds_the_other_system_with_the_init_segment_kid() -> None:
|
||||
chosen = object.__new__(PlayReady)
|
||||
alternate = HLS.alternate_drm([key_of(NO_KID_PSSH)], chosen, kid=INIT_KID)
|
||||
assert [type(d) for d in alternate] == [Widevine] and alternate[0].kids == [INIT_KID]
|
||||
|
||||
|
||||
def test_import_reads_the_kid_of_a_playlist_whose_key_has_none() -> None:
|
||||
def get(url: str, headers: Any = None) -> Any:
|
||||
return SimpleNamespace(text=playlist(keyed_segment(NO_KID_PSSH)), ok=True, content=b"init")
|
||||
|
||||
track = SimpleNamespace(id="vid", url="https://example.test/media.m3u8", get_key_id=lambda init: INIT_KID)
|
||||
assert hls_playlist_kids(track, SimpleNamespace(get=get)) == {INIT_KID}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Regression test for a licence-only HLS run on a playlist that changes its key part-way.
|
||||
|
||||
``HLS.download_track`` licensed only the first key before it returned, so an export made
|
||||
with no download held no content key for the later KID, and the import failed there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from pywidevine.pssh import PSSH
|
||||
|
||||
from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY
|
||||
from unshackle.core.manifests.hls import HLS
|
||||
from unshackle.core.remote_service import segment_uri_filter
|
||||
from unshackle.core.tracks import Video
|
||||
from unshackle.core.tracks.track import DownloadContext, Track
|
||||
|
||||
FIRST_KID = UUID("11111111-1111-1111-1111-111111111111")
|
||||
LATER_KID = UUID("22222222-2222-2222-2222-222222222222")
|
||||
|
||||
|
||||
def key_tag(kid: UUID) -> str:
|
||||
pssh = PSSH.new(key_ids=[kid], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
return (
|
||||
'#EXT-X-KEY:METHOD=SAMPLE-AES,URI="data:text/plain;base64,'
|
||||
f'{pssh}",KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed",KEYFORMATVERSIONS="1"'
|
||||
)
|
||||
|
||||
|
||||
PLAYLIST = f"""#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-TARGETDURATION:6
|
||||
{key_tag(FIRST_KID)}
|
||||
#EXTINF:6.0,
|
||||
seg0.m4s
|
||||
#EXT-X-DISCONTINUITY
|
||||
{key_tag(LATER_KID)}
|
||||
#EXTINF:6.0,
|
||||
seg1.m4s
|
||||
#EXT-X-ENDLIST
|
||||
"""
|
||||
|
||||
|
||||
def test_licence_only_licenses_a_key_that_starts_part_way(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
playlist = tmp_path / "media.m3u8"
|
||||
playlist.write_text(PLAYLIST, encoding="utf-8")
|
||||
track = Video(
|
||||
url="https://example.test/media.m3u8",
|
||||
language="en",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
codec=Video.Codec.HEVC,
|
||||
)
|
||||
track.from_file = playlist
|
||||
# the KID normally comes from the init segment; this test is offline
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(lambda *args, **kwargs: None))
|
||||
|
||||
licensed: list[UUID] = []
|
||||
ctx = DownloadContext(
|
||||
save_path=tmp_path / "video.mp4",
|
||||
save_dir=tmp_path / "video_segments",
|
||||
progress=lambda **_: None,
|
||||
license_widevine=lambda drm, track_kid=None: licensed.append(track_kid),
|
||||
cdm=None,
|
||||
)
|
||||
|
||||
DOWNLOAD_LICENCE_ONLY.set()
|
||||
try:
|
||||
HLS.download_track(track=track, ctx=ctx)
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
|
||||
assert licensed == [FIRST_KID, LATER_KID]
|
||||
assert track.data["hls"]["unwanted_segments"] == []
|
||||
|
||||
|
||||
def test_licence_only_skips_the_key_of_a_filtered_segment(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
"""The filtered segment URIs go to the track data, and an import filter built from them drops the same segment."""
|
||||
playlist = tmp_path / "media.m3u8"
|
||||
playlist.write_text(PLAYLIST, encoding="utf-8")
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(lambda *args, **kwargs: None))
|
||||
|
||||
def licence_only(segment_filter: Any) -> tuple[list[UUID], list[str]]:
|
||||
track = Video(
|
||||
url="https://example.test/media.m3u8",
|
||||
language="en",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
codec=Video.Codec.HEVC,
|
||||
)
|
||||
track.from_file = playlist
|
||||
track.OnSegmentFilter = segment_filter
|
||||
licensed: list[UUID] = []
|
||||
ctx = DownloadContext(
|
||||
save_path=tmp_path / "video.mp4",
|
||||
save_dir=tmp_path / "video_segments",
|
||||
progress=lambda **_: None,
|
||||
license_widevine=lambda drm, track_kid=None: licensed.append(track_kid),
|
||||
cdm=None,
|
||||
)
|
||||
DOWNLOAD_LICENCE_ONLY.set()
|
||||
try:
|
||||
HLS.download_track(track=track, ctx=ctx)
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
return licensed, track.data["hls"]["unwanted_segments"]
|
||||
|
||||
licensed, unwanted = licence_only(lambda segment: segment.uri == "seg1.m4s")
|
||||
assert licensed == [FIRST_KID]
|
||||
assert len(unwanted) == 1 and unwanted[0].endswith("seg1.m4s")
|
||||
|
||||
assert licence_only(segment_uri_filter(unwanted)) == (licensed, unwanted)
|
||||
|
||||
|
||||
def test_licence_only_warns_when_no_dropped_segment_is_in_the_playlist(
|
||||
tmp_path: Path, monkeypatch: Any, caplog: Any
|
||||
) -> None:
|
||||
"""A changed host or path makes an exported list match nothing, and the download then keeps every segment."""
|
||||
playlist = tmp_path / "media.m3u8"
|
||||
playlist.write_text(PLAYLIST, encoding="utf-8")
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(lambda *args, **kwargs: None))
|
||||
track = Video(
|
||||
url="https://example.test/media.m3u8",
|
||||
language="en",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
codec=Video.Codec.HEVC,
|
||||
)
|
||||
track.from_file = playlist
|
||||
stale = ["https://other.example/moved.m4s"]
|
||||
track.OnSegmentFilter = segment_uri_filter(stale)
|
||||
track.data["hls"]["unwanted_segments"] = stale
|
||||
ctx = DownloadContext(
|
||||
save_path=tmp_path / "video.mp4",
|
||||
save_dir=tmp_path / "video_segments",
|
||||
progress=lambda **_: None,
|
||||
license_widevine=lambda drm, track_kid=None: None,
|
||||
cdm=None,
|
||||
)
|
||||
|
||||
DOWNLOAD_LICENCE_ONLY.set()
|
||||
try:
|
||||
with caplog.at_level("WARNING", logger="HLS"):
|
||||
HLS.download_track(track=track, ctx=ctx)
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
|
||||
assert "keeps every segment" in caplog.text
|
||||
|
||||
|
||||
def test_licence_only_fetches_no_aes_128_key(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
"""An AES-128 key with an IV on each segment is a new key each time, and licence-only must not fetch them."""
|
||||
playlist = tmp_path / "media.m3u8"
|
||||
playlist.write_text(
|
||||
"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:6\n"
|
||||
+ "".join(
|
||||
f'#EXT-X-KEY:METHOD=AES-128,URI="https://example.test/key",IV=0x{i:032x}\n#EXTINF:6.0,\nseg{i}.ts\n'
|
||||
for i in range(1, 4)
|
||||
)
|
||||
+ "#EXT-X-ENDLIST\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
track = Video(
|
||||
url="https://example.test/media.m3u8",
|
||||
language="en",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
codec=Video.Codec.AVC,
|
||||
)
|
||||
track.from_file = playlist
|
||||
fetched: list[Any] = []
|
||||
monkeypatch.setattr(HLS, "get_drm", staticmethod(lambda key, session=None: fetched.append(key)))
|
||||
ctx = DownloadContext(
|
||||
save_path=tmp_path / "video.mp4",
|
||||
save_dir=tmp_path / "video_segments",
|
||||
progress=lambda **_: None,
|
||||
license_widevine=lambda drm, track_kid=None: None,
|
||||
cdm=None,
|
||||
)
|
||||
|
||||
DOWNLOAD_LICENCE_ONLY.set()
|
||||
try:
|
||||
HLS.download_track(track=track, ctx=ctx)
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
|
||||
assert len(fetched) <= 1, "only the first key of the playlist, as before"
|
||||
@@ -181,3 +181,24 @@ def test_hls_tracks_tolerate_the_empty_manifest_data(tmp_path: Path) -> None:
|
||||
assert not track.data.get("hls"), "to_dict cannot carry the m3u8 objects, and does not need to"
|
||||
track.data["hls"]["segment_durations"] = [1, 2, 3] # hls.py, in download_track
|
||||
assert track.data["hls"]["segment_durations"] == [1, 2, 3]
|
||||
|
||||
|
||||
def test_hls_round_trip_keeps_the_dropped_segments(tmp_path: Path) -> None:
|
||||
"""An import has no service to run the segment filter, so the export carries the dropped URIs."""
|
||||
original = HLS.from_text(HLS_MANIFEST, HLS_URL).to_tracks(language="en")
|
||||
video = original.videos[0]
|
||||
dropped = "https://cdn.example/bumper_1.m4s?token=a"
|
||||
video.data["hls"]["unwanted_segments"] = [dropped] # hls.py, in download_track
|
||||
|
||||
export = export_tracks(tmp_path, original, HLS_URL)
|
||||
_, tracks = import_tracks(export)
|
||||
|
||||
rebuilt = next(t for t in tracks.videos if t.id == video.id)
|
||||
assert rebuilt.OnSegmentFilter(SimpleNamespace(absolute_uri="https://cdn.example/bumper_1.m4s?token=b"))
|
||||
assert not rebuilt.OnSegmentFilter(SimpleNamespace(absolute_uri="https://cdn.example/main_1.m4s"))
|
||||
assert rebuilt.data["hls"]["unwanted_segments"] == [dropped]
|
||||
assert all(t.OnSegmentFilter is None for t in tracks if t.id != video.id)
|
||||
|
||||
video.data["hls"]["unwanted_segments"] = []
|
||||
_, tracks = import_tracks(export_tracks(tmp_path, original, HLS_URL))
|
||||
assert next(t for t in tracks.videos if t.id == video.id).OnSegmentFilter is None, "a stale list must go"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Unit tests for M3U8 response decoding. Reading `.text` on a response with no encoding
|
||||
makes requests run chardet over the whole body, so both playlist fetch paths set utf-8
|
||||
first. A charset the server did send must survive."""
|
||||
"""Unit tests for manifest response decoding. Reading `.text` on a response with no encoding
|
||||
makes requests run chardet over the whole body, so the playlist fetch paths set utf-8
|
||||
first. A charset the server did send must survive. An MPD is parsed from bytes, so the
|
||||
XML declaration gives the encoding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,7 +10,9 @@ from typing import Any, Optional
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from unshackle.core.manifests.dash import DASH
|
||||
from unshackle.core.manifests.hls import HLS
|
||||
from unshackle.core.session import RnetResponse, utf8_text
|
||||
from unshackle.core.tracks import Video
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
@@ -24,6 +27,8 @@ def make_response(body: bytes, url: str, encoding: Optional[str]) -> requests.Re
|
||||
res._content = body
|
||||
res.url = url
|
||||
res.encoding = encoding
|
||||
if encoding:
|
||||
res.headers["Content-Type"] = f"application/vnd.apple.mpegurl; charset={encoding}"
|
||||
return res
|
||||
|
||||
|
||||
@@ -74,3 +79,61 @@ def test_load_drm_from_playlist_keeps_a_charset_the_server_sent() -> None:
|
||||
|
||||
make_track(res).load_drm_from_playlist()
|
||||
assert res.encoding == "iso-8859-1"
|
||||
|
||||
|
||||
MPD_LATIN1 = '<?xml version="1.0" encoding="ISO-8859-1"?><MPD><Title>café</Title></MPD>'.encode("iso-8859-1")
|
||||
|
||||
|
||||
class StubRnetResponse:
|
||||
"""The part of an rnet response that `RnetResponse` reads."""
|
||||
|
||||
ok = True
|
||||
status_code = 200
|
||||
encoding = None
|
||||
|
||||
def __init__(self, body: bytes, url: str) -> None:
|
||||
self.body = body
|
||||
self.url = url
|
||||
|
||||
def bytes(self) -> bytes:
|
||||
return self.body
|
||||
|
||||
|
||||
def test_dash_from_url_follows_the_xml_declaration() -> None:
|
||||
res = make_response(MPD_LATIN1, "https://example.com/manifest.mpd", None)
|
||||
dash = DASH.from_url("https://example.com/manifest.mpd", FakeSession(res))
|
||||
|
||||
assert dash.manifest.findtext("Title") == "café"
|
||||
|
||||
|
||||
def test_dash_from_url_reads_an_rnet_response() -> None:
|
||||
res = RnetResponse(StubRnetResponse(MPD_LATIN1, "https://example.com/manifest.mpd"))
|
||||
dash = DASH.from_url("https://example.com/manifest.mpd", FakeSession(res)) # type: ignore[arg-type]
|
||||
|
||||
assert dash.manifest.findtext("Title") == "café"
|
||||
|
||||
|
||||
def test_utf8_text_reads_an_rnet_response() -> None:
|
||||
assert utf8_text(RnetResponse(StubRnetResponse(MEDIA, "https://example.com/video/720.m3u8"))) == MEDIA.decode()
|
||||
|
||||
|
||||
def test_utf8_text_ignores_the_requests_default_for_a_text_type() -> None:
|
||||
res = make_response('#EXTM3U\n#EXT-X-MEDIA:TYPE=AUDIO,NAME="中文"\n'.encode(), "https://example.com/a.m3u8", None)
|
||||
res.headers["Content-Type"] = "text/plain"
|
||||
res.encoding = "ISO-8859-1"
|
||||
|
||||
assert "中文" in utf8_text(res)
|
||||
|
||||
|
||||
def test_dash_from_url_uses_the_http_charset_without_an_xml_declaration() -> None:
|
||||
res = make_response("<MPD><Title>中文</Title></MPD>".encode("gbk"), "https://example.com/manifest.mpd", "gbk")
|
||||
dash = DASH.from_url("https://example.com/manifest.mpd", FakeSession(res))
|
||||
|
||||
assert dash.manifest.findtext("Title") == "中文"
|
||||
|
||||
|
||||
def test_dash_from_url_rejects_an_empty_body() -> None:
|
||||
res = make_response(b"", "https://example.com/manifest.mpd", None)
|
||||
|
||||
with pytest.raises(ValueError, match="must be provided"):
|
||||
DASH.from_url("https://example.com/manifest.mpd", FakeSession(res))
|
||||
|
||||
@@ -21,6 +21,7 @@ from Cryptodome.Cipher import AES
|
||||
from Cryptodome.Util.Padding import pad
|
||||
from pywidevine.pssh import PSSH
|
||||
|
||||
from tests.core.test_hls_kid_from_init import KEY_LINE, no_kid_pssh
|
||||
from tests.core.test_ism_init import piff_fragment
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
@@ -264,3 +265,34 @@ def test_hls_aes128_keeps_post_merge(server, tmp_path, monkeypatch):
|
||||
|
||||
plain, rolling = download_both_ways(HLS, make, tmp_path, monkeypatch, False)
|
||||
assert rolling == plain == INIT + b"".join(PARTS)
|
||||
|
||||
|
||||
def test_hls_rekey_takes_the_kid_from_the_init_segment_of_the_new_key(server, tmp_path, monkeypatch):
|
||||
"""Each key carries a PSSH with a content id and no Key ID, so only the init segment names the KID."""
|
||||
later_kid = UUID(hex="22222222222222222222222222222222")
|
||||
later_init = bytes(range(100, 200))
|
||||
server.routes["/init2.mp4"] = later_init
|
||||
no_decrypt(monkeypatch, hls_module)
|
||||
monkeypatch.setattr(config, "merge_segments", False)
|
||||
|
||||
def key_line(content_id: bytes) -> str:
|
||||
return KEY_LINE.format(pssh=no_kid_pssh(content_id))
|
||||
|
||||
track = hls_track(server, False)
|
||||
lines = ["#EXTM3U", "#EXT-X-VERSION:7", "#EXT-X-TARGETDURATION:4", '#EXT-X-MAP:URI="init.mp4"', key_line(b"a")]
|
||||
for i in range(len(PARTS)):
|
||||
if i == 2:
|
||||
lines += ["#EXT-X-DISCONTINUITY", '#EXT-X-MAP:URI="init2.mp4"', key_line(b"b")]
|
||||
lines += ["#EXTINF:4.0,", f"part{i}.m4s"]
|
||||
server.routes["/media.m3u8"] = "\n".join(lines + ["#EXT-X-ENDLIST"]).encode()
|
||||
# the real probe needs FFprobe and a tenc box; the KID follows the init bytes the parser hands over
|
||||
monkeypatch.setattr(
|
||||
type(track), "get_key_id", lambda self, init_data=None, **kw: {later_init: later_kid}.get(init_data, KID)
|
||||
)
|
||||
|
||||
licensed: list[UUID] = []
|
||||
ctx = make_ctx(tmp_path, "rekey.mp4")
|
||||
ctx.license_widevine = lambda drm, track_kid=None: licensed.append(track_kid)
|
||||
HLS.download_track(track, ctx)
|
||||
|
||||
assert licensed == [KID, later_kid]
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core import console
|
||||
from unshackle.core.api.input_bridge import NO_USER_MESSAGE, InputBridge, NoUserBridge
|
||||
from unshackle.core.service import Service
|
||||
|
||||
|
||||
def _service(bridge=None):
|
||||
return SimpleNamespace(log=logging.getLogger("test"), _input_bridge=bridge)
|
||||
|
||||
|
||||
def test_returns_the_first_value_check_gives():
|
||||
answers = iter([None, None, "token"])
|
||||
assert Service.wait_for_approval(_service(), "enter code", lambda: next(answers), timeout=5, interval=0) == "token"
|
||||
|
||||
|
||||
def test_times_out_when_check_never_gives_a_value():
|
||||
with pytest.raises(TimeoutError):
|
||||
Service.wait_for_approval(_service(), "enter code", lambda: None, timeout=0.05, interval=0.01)
|
||||
|
||||
|
||||
def test_stops_when_the_remote_session_is_cancelled():
|
||||
bridge = InputBridge()
|
||||
calls = []
|
||||
|
||||
def check():
|
||||
calls.append(1)
|
||||
bridge.cancel()
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
Service.wait_for_approval(_service(bridge), "enter code", check, timeout=5, interval=0)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def _raise_value_error():
|
||||
raise ValueError("denied")
|
||||
|
||||
|
||||
def test_logs_the_message_for_a_remote_session(caplog):
|
||||
"""A remote session reads the message only from the service log."""
|
||||
with caplog.at_level(logging.INFO, logger="test"):
|
||||
Service.wait_for_approval(_service(), "enter code", lambda: "token", timeout=5, interval=0)
|
||||
assert "enter code" in caplog.messages
|
||||
|
||||
|
||||
def test_a_request_with_no_user_fails_at_once():
|
||||
calls = []
|
||||
with pytest.raises(RuntimeError, match="needs the user"):
|
||||
Service.wait_for_approval(_service(NoUserBridge()), "enter code", lambda: calls.append(1), timeout=5)
|
||||
assert not calls
|
||||
with pytest.raises(RuntimeError, match="needs the user"):
|
||||
Service.request_input(_service(NoUserBridge()), "Enter OTP")
|
||||
|
||||
|
||||
def test_no_user_message_is_an_authentication_error():
|
||||
from unshackle.core.api.errors import APIErrorCode, categorize_exception
|
||||
|
||||
assert categorize_exception(RuntimeError(NO_USER_MESSAGE)).error_code == APIErrorCode.AUTH_FAILED
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"check, expectation",
|
||||
[
|
||||
(lambda: "token", nullcontext()),
|
||||
(lambda: None, pytest.raises(TimeoutError)),
|
||||
(_raise_value_error, pytest.raises(ValueError)),
|
||||
],
|
||||
)
|
||||
def test_shows_the_message_as_a_notice_and_clears_it(monkeypatch, check, expectation):
|
||||
notices = []
|
||||
monkeypatch.setattr(console, "_notice_handler", notices.append)
|
||||
with expectation:
|
||||
Service.wait_for_approval(_service(), "enter code", check, timeout=0.05, interval=0.01)
|
||||
assert notices == ["enter code", None]
|
||||
@@ -101,7 +101,7 @@ def test_the_client_refuses_a_challenge_for_the_other_drm_system() -> None:
|
||||
assert "not a playready device" in answer["error"]
|
||||
|
||||
|
||||
def test_a_cdm_call_after_authentication_leaves_the_status_and_client_auth_alone() -> None:
|
||||
def test_a_cdm_call_after_authentication_leaves_the_status_alone() -> None:
|
||||
bridge = InputBridge()
|
||||
bridge.status = AuthStatus.AUTHENTICATED
|
||||
result: list[dict] = []
|
||||
@@ -118,7 +118,6 @@ def test_a_cdm_call_after_authentication_leaves_the_status_and_client_auth_alone
|
||||
|
||||
assert result == [{"session": "s", "challenge": "Yw=="}]
|
||||
assert bridge.status is AuthStatus.AUTHENTICATED
|
||||
assert bridge.answered is False
|
||||
assert bridge.get_pending_cdm_call() is None
|
||||
|
||||
|
||||
|
||||
@@ -192,10 +192,17 @@ class _CapturedCtx(Exception):
|
||||
@pytest.mark.parametrize(
|
||||
("params", "expected"),
|
||||
[
|
||||
({}, (["orig"], [], [], [], False)),
|
||||
({}, (["orig"], [], [], [], False, False)),
|
||||
(
|
||||
{"lang": ["es-419"], "v_lang": ["en"], "a_lang": ["ja"], "acodec": "AAC,EC3", "forced_subs": True},
|
||||
(["es-419"], ["en"], ["ja"], ["AAC", "EC3"], True),
|
||||
{
|
||||
"lang": ["es-419"],
|
||||
"v_lang": ["en"],
|
||||
"a_lang": ["ja"],
|
||||
"acodec": "AAC,EC3",
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
},
|
||||
(["es-419"], ["en"], ["ja"], ["AAC", "EC3"], True, True),
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -224,7 +231,8 @@ def test_perform_download_puts_the_lang_selection_in_the_service_ctx(
|
||||
|
||||
ctx = exc.value.args[0]
|
||||
p = ctx.params
|
||||
assert (p["lang"], p["v_lang"], p["a_lang"], [c.name for c in p["acodec"]], p["forced_subs"]) == expected
|
||||
selection = (p["lang"], p["v_lang"], p["a_lang"], [c.name for c in p["acodec"]], p["forced_subs"], p["exact_lang"])
|
||||
assert selection == expected
|
||||
|
||||
|
||||
def test_perform_download_passes_latest_episodes_to_dl_result(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@@ -286,6 +294,30 @@ def test_worker_relays_a_prompt_and_reads_the_answer(monkeypatch: pytest.MonkeyP
|
||||
assert updates == [{"input_prompt": "Enter PIN"}, {"input_prompt": None}]
|
||||
|
||||
|
||||
def test_worker_relays_a_notice_as_progress(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A show_notice() message reaches the parent as ``notice`` progress, and None clears it."""
|
||||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
from unshackle.core.api import download_worker
|
||||
from unshackle.core.console import set_notice_handler, set_prompt_handler, show_notice
|
||||
|
||||
read_fd, write_fd = os.pipe()
|
||||
monkeypatch.setattr(sys, "stdin", SimpleNamespace(fileno=lambda: read_fd))
|
||||
updates: list[dict] = []
|
||||
|
||||
download_worker.relay_prompts(updates.append)
|
||||
try:
|
||||
show_notice("Enter pair code ABCD")
|
||||
show_notice(None)
|
||||
finally:
|
||||
set_prompt_handler(None)
|
||||
set_notice_handler(None)
|
||||
os.close(write_fd)
|
||||
assert updates == [{"notice": "Enter pair code ABCD"}, {"notice": None}]
|
||||
|
||||
|
||||
def test_a_prompt_discards_an_answer_sent_before_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A late answer to an earlier prompt must not answer the next one; the timeout is not retryable."""
|
||||
import sys
|
||||
@@ -420,6 +452,46 @@ async def test_a_repeated_prompt_reaches_an_event_listener(
|
||||
assert job.input_prompt is None
|
||||
|
||||
|
||||
async def test_a_notice_reaches_the_job_and_an_event_listener(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, manager: DownloadQueueManager
|
||||
) -> None:
|
||||
"""The job carries the worker's notice while the worker waits, and drops it at the end."""
|
||||
import asyncio
|
||||
|
||||
from unshackle.core.api import download_manager
|
||||
|
||||
worker = tmp_path / "worker.py"
|
||||
# The prompt holds the worker open until the test has seen the notice.
|
||||
source = FAKE_PROMPT_WORKER.replace("import prompt_user", "import prompt_user, show_notice").replace(
|
||||
'answers = [prompt_user("Enter OTP"), prompt_user("Enter OTP")]',
|
||||
'show_notice("Enter pair code ABCD")\nanswers = [prompt_user("Continue")]',
|
||||
)
|
||||
assert source.count("show_notice") == 2, "FAKE_PROMPT_WORKER changed; update the replacements"
|
||||
worker.write_text(source, encoding="utf-8")
|
||||
spawn = asyncio.create_subprocess_exec
|
||||
|
||||
async def spawn_fake_worker(executable: str, *args: str, **kwargs: Any) -> Any:
|
||||
return await spawn(executable, str(worker), *args[2:], **kwargs)
|
||||
|
||||
monkeypatch.setattr(download_manager.asyncio, "create_subprocess_exec", spawn_fake_worker)
|
||||
job = manager.create_job("EXAMPLE", "t")
|
||||
events = manager.subscribe(job.job_id)
|
||||
run = asyncio.create_task(manager.run_download_async(job))
|
||||
|
||||
async def answer_after_notice() -> None:
|
||||
while True:
|
||||
event = await events.get()
|
||||
if event and event["data"]["input_prompt"]:
|
||||
assert event["data"]["notice"] == "Enter pair code ABCD"
|
||||
assert job.to_dict()["notice"] == "Enter pair code ABCD"
|
||||
assert manager.submit_input(job, "ok") is True
|
||||
return
|
||||
|
||||
await asyncio.wait_for(answer_after_notice(), timeout=20)
|
||||
assert await asyncio.wait_for(run, timeout=20) == ["ok"]
|
||||
assert job.notice is None
|
||||
|
||||
|
||||
async def test_input_handler_rejects_bad_requests(
|
||||
monkeypatch: pytest.MonkeyPatch, manager: DownloadQueueManager
|
||||
) -> None:
|
||||
|
||||
@@ -13,6 +13,7 @@ from uuid import UUID
|
||||
import pytest
|
||||
from pywidevine.pssh import PSSH as WvPSSH
|
||||
|
||||
from tests.core.test_hls_kid_from_init import NO_KID_PSSH
|
||||
from unshackle.core.api import handlers
|
||||
from unshackle.core.drm import Widevine
|
||||
from unshackle.core.tracks import Track
|
||||
@@ -22,13 +23,14 @@ pytestmark = pytest.mark.unit
|
||||
KID = UUID("161219ec3df64e0eadbb3827c0ccc46d")
|
||||
|
||||
|
||||
def _media_playlist() -> str:
|
||||
pssh = WvPSSH.new(key_ids=[KID], system_id=WvPSSH.SystemId.Widevine).dumps()
|
||||
def _media_playlist(pssh: str | None = None) -> str:
|
||||
pssh = pssh or WvPSSH.new(key_ids=[KID], system_id=WvPSSH.SystemId.Widevine).dumps()
|
||||
return "\n".join(
|
||||
[
|
||||
"#EXTM3U",
|
||||
"#EXT-X-VERSION:6",
|
||||
"#EXT-X-TARGETDURATION:6",
|
||||
'#EXT-X-MAP:URI="init.mp4"',
|
||||
f'#EXT-X-KEY:METHOD=SAMPLE-AES,URI="data:text/plain;base64,{pssh}",'
|
||||
'KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed",KEYFORMATVERSIONS="1"',
|
||||
"#EXTINF:6.0,",
|
||||
@@ -49,3 +51,21 @@ def test_hls_descriptor_alone_reaches_the_media_playlist_keys():
|
||||
assert track.drm and isinstance(track.drm[0], Widevine), "media playlist DRM not read"
|
||||
assert track.drm[0].kids == [KID]
|
||||
assert base64.b64decode(handlers.extract_pssh_from_track(track, "widevine"))
|
||||
|
||||
|
||||
def test_key_without_a_kid_takes_it_from_the_init_segment():
|
||||
"""A PSSH that holds a content id and no Key ID is DRM, and the server must not call the track clear."""
|
||||
response = SimpleNamespace(text=_media_playlist(NO_KID_PSSH), ok=True, content=b"init")
|
||||
session = SimpleNamespace(get=lambda url, headers=None: response)
|
||||
track = SimpleNamespace(
|
||||
id="vid",
|
||||
url="https://cdn.example.invalid/media.m3u8",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
data={},
|
||||
drm=None,
|
||||
get_key_id=lambda init: KID,
|
||||
)
|
||||
|
||||
handlers.ensure_track_drm(track, session)
|
||||
|
||||
assert track.drm and track.drm[0].kids == [KID]
|
||||
|
||||
@@ -96,19 +96,6 @@ def test_status_setter() -> None:
|
||||
assert bridge.status is AuthStatus.AUTHENTICATED
|
||||
|
||||
|
||||
def test_answered_only_after_an_accepted_response() -> None:
|
||||
bridge = InputBridge()
|
||||
assert bridge.submit_response("early") is False
|
||||
assert bridge.answered is False
|
||||
worker = threading.Thread(target=bridge.request_input, args=("code?",))
|
||||
worker.start()
|
||||
while bridge.status != AuthStatus.PENDING_INPUT:
|
||||
time.sleep(0.01)
|
||||
assert bridge.submit_response("CODE") is True
|
||||
worker.join(timeout=2)
|
||||
assert bridge.answered is True
|
||||
|
||||
|
||||
def test_request_input_after_authentication_fails_fast() -> None:
|
||||
"""The client stops polling for prompts once authenticated, so a later prompt must not wait for an answer."""
|
||||
bridge = InputBridge()
|
||||
|
||||
@@ -7,6 +7,7 @@ from enum import Enum
|
||||
import pytest
|
||||
|
||||
from unshackle.core.remote_service import (
|
||||
RemoteService,
|
||||
build_title,
|
||||
build_tracks,
|
||||
deserialize_audio,
|
||||
@@ -563,11 +564,12 @@ class _SentCreate(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _sent_create_data(parent_params: dict, service_params: dict) -> dict:
|
||||
def _remote_service(parent_params: dict, service_params: dict) -> RemoteService:
|
||||
"""A RemoteService whose session create raises _SentCreate with the request body."""
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
from unshackle.core.remote_service import RemoteService
|
||||
import requests
|
||||
|
||||
def post(path: str, data: dict) -> dict:
|
||||
raise _SentCreate(data)
|
||||
@@ -579,11 +581,32 @@ def _sent_create_data(parent_params: dict, service_params: dict) -> dict:
|
||||
svc._server_accounts = {"global": True}
|
||||
svc._server_cdm = True
|
||||
svc._service_params = service_params
|
||||
svc._session = requests.Session()
|
||||
svc.log = logging.getLogger("test")
|
||||
svc.client = SimpleNamespace(post=post)
|
||||
svc.client = SimpleNamespace(post=post, server_url="https://server.example")
|
||||
return svc
|
||||
|
||||
|
||||
def _sent_create_data(parent_params: dict, service_params: dict) -> dict:
|
||||
with pytest.raises(_SentCreate) as exc:
|
||||
_remote_service(parent_params, service_params).authenticate()
|
||||
return exc.value.args[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_proxy, expected", [(False, {"all": "http://proxy.example:8080"}), (True, {})])
|
||||
def test_the_client_session_downloads_through_the_proxy_it_sends(
|
||||
monkeypatch: pytest.MonkeyPatch, no_proxy: bool, expected: dict
|
||||
) -> None:
|
||||
"""The client fetches playlists and segments itself, so --proxy must reach its own session too."""
|
||||
from unshackle.core import remote_service
|
||||
|
||||
monkeypatch.setattr(remote_service, "resolve_remote_proxy_arg", lambda proxy: proxy)
|
||||
monkeypatch.setattr("unshackle.core.utils.ip_info.get_ip_info", lambda *args, **kwargs: None)
|
||||
svc = _remote_service({"no_proxy": no_proxy, "proxy": "http://proxy.example:8080"}, {})
|
||||
with pytest.raises(_SentCreate) as exc:
|
||||
svc.authenticate()
|
||||
return exc.value.args[0]
|
||||
assert dict(svc._session.proxies) == expected
|
||||
assert exc.value.args[0].get("proxy") == (None if no_proxy else "http://proxy.example:8080")
|
||||
|
||||
|
||||
def test_dl_selection_travels_nested_and_a_service_option_cannot_override_it() -> None:
|
||||
@@ -593,6 +616,7 @@ def test_dl_selection_travels_nested_and_a_service_option_cannot_override_it() -
|
||||
"a_lang": [],
|
||||
"acodec": [Audio.Codec.EC3],
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
}
|
||||
data = _sent_create_data(parent_params, {"lang": 3, "a_lang": "x"})
|
||||
assert data["dl_params"] == {
|
||||
@@ -601,6 +625,7 @@ def test_dl_selection_travels_nested_and_a_service_option_cannot_override_it() -
|
||||
"a_lang": [],
|
||||
"acodec": ["EC3"],
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
}
|
||||
assert data["lang"] == 3
|
||||
assert data["service_params"] == {"lang": 3, "a_lang": "x"}
|
||||
|
||||
@@ -144,6 +144,27 @@ async def test_server_account_uses_its_own_cache_and_returns_none(seen, tmp_path
|
||||
assert not (tmp_path / "_requests").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_search_skips_the_login_and_keeps_the_account_cache(seen, tmp_path, monkeypatch):
|
||||
def search(self):
|
||||
seen["search_cache_dir"] = self.cache.get("tokens").path.parent
|
||||
return iter(())
|
||||
|
||||
def no_login_material(*args):
|
||||
raise AssertionError("an anonymous search must not read login material")
|
||||
|
||||
handlers.Services.load("EXAMPLE").ANONYMOUS_SEARCH = True
|
||||
monkeypatch.setattr(_CacheService, "search", search)
|
||||
monkeypatch.setattr(handlers, "resolve_server_account", lambda *args: "acct")
|
||||
monkeypatch.setattr(handlers, "server_login_material", no_login_material)
|
||||
response = await handlers.search_handler({"service": "EXAMPLE", "query": "q", **CREDENTIALS})
|
||||
|
||||
assert json.loads(response.body)["results"] == []
|
||||
assert "seeded" not in seen
|
||||
assert seen["parent_params"]["cookies_supplied"] is False
|
||||
assert seen["search_cache_dir"] == tmp_path / "_accounts" / "EXAMPLE" / "acct"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_mode_keeps_the_server_cache(seen, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(stats, "mode", "full")
|
||||
|
||||
@@ -286,3 +286,13 @@ async def test_auth_middleware_accepts_known_key(make_app, aiohttp_client) -> No
|
||||
resp = await client.get("/api/session/nonexistent", headers={"X-Secret-Key": "good-key"})
|
||||
# Auth passed; handler then 404s the session; anything other than 401 is fine here.
|
||||
assert resp.status != 401
|
||||
|
||||
|
||||
def test_setup_routes_cdm_only_keeps_health_and_dashboard_only() -> None:
|
||||
"""A CDM-only server answers health and the dashboard under /api/, nothing else."""
|
||||
app = web.Application()
|
||||
setup_routes(app, dashboard=True, cdm_only=True)
|
||||
paths = {path for _, path in collect_paths(app)}
|
||||
assert "/api/health" in paths
|
||||
assert any(path.startswith("/api/dashboard/") for path in paths)
|
||||
assert all(path == "/api/health" or path.startswith("/api/dashboard/") for path in paths)
|
||||
|
||||
@@ -181,3 +181,11 @@ def test_good_rate_limit_config_starts(runner: CliRunner, monkeypatch: pytest.Mo
|
||||
{"tiers": {"bot": {"rate_limit": 600}}, "users": {"k": {"username": "bot", "tier": "bot"}}},
|
||||
)
|
||||
assert started, output
|
||||
|
||||
|
||||
def test_serve_cdm_only_with_remote_only_rejected(runner: CliRunner) -> None:
|
||||
"""`--cdm-only` is mutually exclusive with `--api-only` and `--remote-only`; the check runs before config."""
|
||||
result = runner.invoke(serve, ["--cdm-only", "--remote-only"])
|
||||
assert result.exit_code != 0
|
||||
output = re.sub(r"\x1b\[[0-9;]*m", "", result.output or str(result.exception))
|
||||
assert "Cannot use --cdm-only" in output
|
||||
|
||||
@@ -162,6 +162,9 @@ def _create_with_real_ctx(monkeypatch, data):
|
||||
return svc
|
||||
|
||||
|
||||
DL_DEFAULTS = {"lang": ["orig"], "v_lang": [], "a_lang": [], "acodec": [], "forced_subs": False, "exact_lang": False}
|
||||
|
||||
|
||||
def test_nested_dl_params_reach_the_parent_ctx(monkeypatch):
|
||||
from unshackle.core.tracks import Audio
|
||||
|
||||
@@ -172,6 +175,7 @@ def test_nested_dl_params_reach_the_parent_ctx(monkeypatch):
|
||||
"a_lang": ["es-419"],
|
||||
"acodec": ["EC3", "NOPE", "AAC"],
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
}
|
||||
}
|
||||
svc = _create_with_real_ctx(monkeypatch, data)
|
||||
@@ -180,12 +184,13 @@ def test_nested_dl_params_reach_the_parent_ctx(monkeypatch):
|
||||
assert svc.parent_params["a_lang"] == ["es-419"]
|
||||
assert svc.parent_params["acodec"] == [Audio.Codec.EC3, Audio.Codec.AAC]
|
||||
assert svc.parent_params["forced_subs"] is True
|
||||
assert svc.parent_params["exact_lang"] is True
|
||||
|
||||
|
||||
def test_absent_dl_params_fall_back_to_the_dl_defaults(monkeypatch):
|
||||
svc = _create_with_real_ctx(monkeypatch, {})
|
||||
selection = {k: svc.parent_params[k] for k in ("lang", "v_lang", "a_lang", "acodec", "forced_subs")}
|
||||
assert selection == {"lang": ["orig"], "v_lang": [], "a_lang": [], "acodec": [], "forced_subs": False}
|
||||
selection = {k: svc.parent_params[k] for k in (*DL_DEFAULTS,)}
|
||||
assert selection == DL_DEFAULTS
|
||||
|
||||
|
||||
def test_an_empty_lang_list_is_kept(monkeypatch):
|
||||
@@ -204,14 +209,14 @@ def test_a_service_lang_option_never_gets_the_dl_lang(monkeypatch):
|
||||
"dl_params",
|
||||
[
|
||||
"en",
|
||||
{"lang": "en", "v_lang": "ja", "a_lang": {"x": 1}, "acodec": "EC3", "forced_subs": "yes"},
|
||||
{"lang": [1], "v_lang": [None], "a_lang": ["en", None], "acodec": [3], "forced_subs": 1},
|
||||
{"lang": "en", "v_lang": "ja", "a_lang": {"x": 1}, "acodec": "EC3", "forced_subs": "yes", "exact_lang": "yes"},
|
||||
{"lang": [1], "v_lang": [None], "a_lang": ["en", None], "acodec": [3], "forced_subs": 1, "exact_lang": 1},
|
||||
],
|
||||
)
|
||||
def test_malformed_dl_params_are_ignored(monkeypatch, dl_params):
|
||||
svc = _create_with_real_ctx(monkeypatch, {"dl_params": dl_params})
|
||||
selection = {k: svc.parent_params[k] for k in ("lang", "v_lang", "a_lang", "acodec", "forced_subs")}
|
||||
assert selection == {"lang": ["orig"], "v_lang": [], "a_lang": [], "acodec": [], "forced_subs": False}
|
||||
selection = {k: svc.parent_params[k] for k in (*DL_DEFAULTS,)}
|
||||
assert selection == DL_DEFAULTS
|
||||
|
||||
|
||||
def test_profile_reaches_the_parent_ctx_params(monkeypatch):
|
||||
|
||||
@@ -221,11 +221,12 @@ def _run_session(monkeypatch, tmp_path, data, answer, server_account=False, fail
|
||||
@pytest.mark.parametrize(
|
||||
"data, answer, server_account, fail, expected",
|
||||
[
|
||||
({}, None, False, False, False), # anonymous login through the server: stays the server's
|
||||
({}, None, False, False, True), # no prompt: a pair code shown in the log, approved on another device
|
||||
({}, "CODE", False, False, True), # device code the client approved
|
||||
({}, "CODE", False, True, False), # answered, but the login failed
|
||||
({}, "CODE", True, False, False), # server account OTP answered by the client
|
||||
({"cache": "tokens"}, None, False, False, True), # client uploaded its own earlier login
|
||||
({}, None, True, False, False), # server account
|
||||
],
|
||||
)
|
||||
def test_client_auth_follows_login_provenance(monkeypatch, tmp_path, data, answer, server_account, fail, expected):
|
||||
|
||||
@@ -268,7 +268,7 @@ def test_season_context_blanks_every_episode_identity_field():
|
||||
|
||||
def test_every_documented_variable_is_one_build_context_produces():
|
||||
"""The variable tables in docs/reference/configuration/post-scripts.md must not drift from the code."""
|
||||
from unshackle.core.titles.song import Song
|
||||
from unshackle.core.titles.music import Song
|
||||
|
||||
doc = Path(__file__).resolve().parents[1] / "docs" / "reference" / "configuration" / "post-scripts.md"
|
||||
documented = {
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
"""A ``--sample`` run downloads only the segments of its window and cuts the mux to the window."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import http.server
|
||||
import struct
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import click
|
||||
import pytest
|
||||
from requests import Session
|
||||
|
||||
from unshackle.commands.dl import sample_name
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.api.handlers import validate_download_parameters
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.manifests.dash import DASH
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.titles import Episode, Movie
|
||||
from unshackle.core.tracks import Audio, Chapter, Subtitle, Tracks, Video
|
||||
from unshackle.core.tracks.track import parse_sidx
|
||||
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||
from unshackle.core.utils.xml import load_xml
|
||||
|
||||
STARTS = [1000 + i * 4000 for i in range(100)] # 4 s segments at timescale 1000
|
||||
|
||||
|
||||
def make_video() -> Video:
|
||||
return Video(id_="v", url="https://x/v", codec=Video.Codec.AVC, language="en", width=1920, height=1080, fps=24)
|
||||
|
||||
|
||||
def make_audio(url: str = "https://x/a") -> Audio:
|
||||
return Audio(id_="a", url=url, codec=Audio.Codec.AAC, language="en", bitrate=128000)
|
||||
|
||||
|
||||
def test_no_window_keeps_every_segment() -> None:
|
||||
assert make_audio().sample_slice(STARTS, 1000) == slice(None)
|
||||
|
||||
|
||||
def test_the_window_is_measured_from_the_first_segment() -> None:
|
||||
track = make_audio()
|
||||
assert track.sample_slice(STARTS, 1000, (62, 120)) == slice(15, 30)
|
||||
assert track.sample_start == 60
|
||||
|
||||
|
||||
def test_a_video_keeps_one_segment_after_the_window() -> None:
|
||||
"""mkvmerge ends the cut only at a video keyframe, so the video must reach past the window."""
|
||||
assert make_video().sample_slice(STARTS, 1000, (0, 60)) == slice(0, 16)
|
||||
assert make_audio().sample_slice(STARTS, 1000, (0, 60)) == slice(0, 15)
|
||||
|
||||
|
||||
def test_a_subtitle_keeps_one_segment_on_each_side_of_the_window() -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="en")
|
||||
video = make_video()
|
||||
video.sample_tail = 0
|
||||
keep = video.sample_slice(STARTS, 1000, (62, 120))
|
||||
assert subtitle.sample_slice(STARTS, 1000, (62, 120)) == slice(keep.start - 1, keep.stop + 1)
|
||||
assert subtitle.sample_start == 0, "cue times stay absolute, so the mux does not shift the track"
|
||||
assert subtitle.sample_slice(STARTS, 1000, None) == slice(None)
|
||||
|
||||
|
||||
def test_a_subtitle_that_ends_before_the_window_keeps_its_last_segment() -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="en")
|
||||
assert subtitle.sample_slice([0, 4000], 1000, (120, 180)) == slice(0, None)
|
||||
|
||||
|
||||
def test_a_window_after_the_end_stops_the_run() -> None:
|
||||
with pytest.raises(click.ClickException, match="ends before the sample window"):
|
||||
make_audio().sample_slice(STARTS[:10], 1000, (120, 180))
|
||||
|
||||
|
||||
TIMELINE_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<SegmentTemplate timescale="1" media="a/$Number$-$Time$.mp4" startNumber="5">
|
||||
<SegmentTimeline><S t="1000" d="4" r="99"/></SegmentTimeline>
|
||||
</SegmentTemplate>
|
||||
<Representation id="a" bandwidth="1"/>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
DURATION_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S" duration="PT400S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<SegmentTemplate timescale="25" duration="100" media="a/$Number$.mp4" startNumber="5"/>
|
||||
<Representation id="a" bandwidth="1"/>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
LIST_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<Representation id="a" bandwidth="1">
|
||||
<BaseURL>https://example.com/a/</BaseURL>
|
||||
<SegmentList timescale="1" duration="4">{urls}</SegmentList>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
""".format(urls="".join(f'<SegmentURL media="{n}.mp4"/>' for n in range(100)))
|
||||
|
||||
BASE_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<Representation id="a" bandwidth="1"><BaseURL>https://example.com/a.mp4</BaseURL></Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
|
||||
WINDOW = (62, 120)
|
||||
|
||||
|
||||
def dash_names(mpd: str, window: Optional[tuple[int, int]]) -> tuple[list[str], float]:
|
||||
tree = load_xml(mpd.encode())
|
||||
period = tree.find("Period")
|
||||
aset = period.find("AdaptationSet")
|
||||
rep = aset.find("Representation")
|
||||
track = make_audio()
|
||||
_, segments, _, _, _ = DASH.get_period_segments(
|
||||
period, aset, rep, tree, track, "https://example.com/main.mpd", MagicMock(), window=window
|
||||
)
|
||||
return [url.rsplit("/", 1)[1].split(".")[0] for url, _ in segments], track.sample_start
|
||||
|
||||
|
||||
def test_dash_timeline_keeps_the_numbers_and_times_of_the_window() -> None:
|
||||
names, start = dash_names(TIMELINE_MPD, WINDOW)
|
||||
assert names == [f"{5 + n}-{1000 + n * 4}" for n in range(15, 30)]
|
||||
assert start == 60
|
||||
|
||||
|
||||
def test_dash_fixed_duration_keeps_the_numbers_of_the_window() -> None:
|
||||
names, start = dash_names(DURATION_MPD, WINDOW)
|
||||
assert names == [str(5 + n) for n in range(15, 30)]
|
||||
assert start == 60
|
||||
|
||||
|
||||
def test_dash_segment_list_keeps_the_urls_of_the_window() -> None:
|
||||
names, start = dash_names(LIST_MPD, WINDOW)
|
||||
assert names == [str(n) for n in range(15, 30)]
|
||||
assert start == 60
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mpd", [TIMELINE_MPD, DURATION_MPD, LIST_MPD])
|
||||
def test_a_caller_that_passes_no_window_gets_every_segment(mpd: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The bitrate probe divides by the full duration, so a sample run must not cut its segment list."""
|
||||
monkeypatch.setattr(config, "sample", WINDOW)
|
||||
assert len(dash_names(mpd, None)[0]) == 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("mpd", "timed"), [(TIMELINE_MPD, True), (LIST_MPD, True), (BASE_MPD, False)])
|
||||
def test_a_track_given_as_one_file_has_no_segment_timing(mpd: str, timed: bool) -> None:
|
||||
aset = load_xml(mpd.encode()).find("Period").find("AdaptationSet")
|
||||
track = SimpleNamespace(data={"dash": {"adaptation_set": aset, "representation": aset.find("Representation")}})
|
||||
assert DASH.has_segment_timing(track) is timed
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def served(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str:
|
||||
handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(tmp_path))
|
||||
handler.log_message = lambda *args: None # type: ignore[attr-defined]
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
(tmp_path / "temp").mkdir()
|
||||
monkeypatch.setattr(config.directories, "temp", tmp_path / "temp")
|
||||
yield f"http://127.0.0.1:{server.server_port}"
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_an_hls_subtitle_downloads_the_segments_around_the_window(
|
||||
tmp_path: Path, served: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
for n in range(8):
|
||||
(tmp_path / f"s{n}.vtt").write_text(
|
||||
f"WEBVTT\n\n00:00:{n * 10 + 1:02}.000 --> 00:00:{n * 10 + 2:02}.000\ncue {n}\n"
|
||||
)
|
||||
(tmp_path / "s.m3u8").write_text(
|
||||
"#EXTM3U\n#EXT-X-TARGETDURATION:10\n"
|
||||
+ "".join(f"#EXTINF:10.0,\ns{n}.vtt\n" for n in range(8))
|
||||
+ "#EXT-X-ENDLIST\n"
|
||||
)
|
||||
monkeypatch.setattr(config, "sample", (32, 38))
|
||||
subtitle = Subtitle(
|
||||
id_="s", url=f"{served}/s.m3u8", codec=Subtitle.Codec.WebVTT, language="en", descriptor=Subtitle.Descriptor.HLS
|
||||
)
|
||||
|
||||
subtitle.download(session=Session(), prepare_drm=lambda *a, **k: None, progress=lambda **k: None)
|
||||
|
||||
text = subtitle.path.read_text()
|
||||
assert [n for n in range(8) if f"cue {n}" in text] == [2, 3, 4]
|
||||
assert "00:00:31.000 --> 00:00:32.000" in text, "cue times stay absolute"
|
||||
|
||||
|
||||
def test_subtitle_trim_keeps_the_cues_of_the_window_from_zero(tmp_path: Path) -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.SubRip, language="en")
|
||||
subtitle.path = tmp_path / "s.srt"
|
||||
subtitle.path.write_text(
|
||||
"1\n00:00:05,000 --> 00:00:06,000\nbefore\n\n"
|
||||
"2\n00:02:10,000 --> 00:02:12,000\ninside\n\n"
|
||||
"3\n00:10:00,000 --> 00:10:01,000\nafter\n"
|
||||
)
|
||||
|
||||
assert subtitle.trim(120, 180) is True
|
||||
|
||||
text = subtitle.path.read_text()
|
||||
assert "inside" in text and "before" not in text and "after" not in text
|
||||
assert "00:00:10,000 --> 00:00:12,000" in text
|
||||
assert subtitle.sample_start == 120
|
||||
|
||||
|
||||
def test_subtitle_trim_leaves_an_unreadable_file_whole(tmp_path: Path) -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.fTTML, language="en")
|
||||
subtitle.path = tmp_path / "s.stpp"
|
||||
subtitle.path.write_bytes(b"\x00\x01not a subtitle")
|
||||
|
||||
assert subtitle.trim(120, 180) is False
|
||||
assert subtitle.path.read_bytes() == b"\x00\x01not a subtitle"
|
||||
|
||||
|
||||
def mux_command(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
class Stop(Exception):
|
||||
pass
|
||||
|
||||
captured: list[list[str]] = []
|
||||
|
||||
def fake_popen(args: list[str], **kwargs: object) -> None:
|
||||
captured.append(args)
|
||||
raise Stop
|
||||
|
||||
monkeypatch.setattr("unshackle.core.tracks.tracks.subprocess.Popen", fake_popen)
|
||||
monkeypatch.setattr(binaries, "MKVToolNix", Path("mkvmerge"))
|
||||
|
||||
video, audio = make_video(), make_audio()
|
||||
trimmed = Subtitle(id_="s1", url="https://x/s1", codec=Subtitle.Codec.SubRip, language="en")
|
||||
whole = Subtitle(id_="s2", url="https://x/s2", codec=Subtitle.Codec.SubRip, language="fr")
|
||||
for track, name in ((video, "v.mp4"), (audio, "a.mp4"), (trimmed, "s1.srt"), (whole, "s2.srt")):
|
||||
track.path = tmp_path / name
|
||||
track.path.write_bytes(b"\x00" * 16)
|
||||
video.sample_start, audio.sample_start, trimmed.sample_start = 120.0, 119.5, 121.0
|
||||
tracks = Tracks([video, audio, trimmed, whole])
|
||||
tracks.chapters.add(Chapter("00:02:05.000", "Scene"))
|
||||
|
||||
with pytest.raises(Stop):
|
||||
tracks.mux("Title", delete=False, progress=lambda **k: None, output_path=tmp_path / "out.mkv")
|
||||
return captured[0]
|
||||
|
||||
|
||||
def test_the_mux_shifts_each_track_and_cuts_to_the_window(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "sample", (121, 181))
|
||||
command = mux_command(tmp_path, monkeypatch)
|
||||
|
||||
def sync_before(name: str) -> str:
|
||||
file_at = command.index(str(tmp_path / name))
|
||||
options = command[:file_at]
|
||||
sync_at = len(options) - 1 - options[::-1].index("--sync")
|
||||
assert all(not arg.endswith((".mp4", ".srt")) for arg in command[sync_at:file_at]), (
|
||||
"the --sync is another file's"
|
||||
)
|
||||
return command[sync_at + 1]
|
||||
|
||||
assert command[command.index("--split") + 1] == "parts:1.500s-61.500s"
|
||||
assert sync_before("v.mp4") == "0:500"
|
||||
assert "--sync" not in command[: command.index(str(tmp_path / "a.mp4"))][-12:], "the earliest track is not moved"
|
||||
assert sync_before("s1.srt") == "0:1500"
|
||||
assert sync_before("s2.srt") == "0:-119500"
|
||||
assert command[command.index("--chapter-sync") + 1] == "-119500"
|
||||
assert command.index("--chapter-sync") < command.index("--chapters")
|
||||
|
||||
|
||||
def test_a_full_download_adds_no_sample_options(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "sample", None)
|
||||
command = mux_command(tmp_path, monkeypatch)
|
||||
assert not {"--split", "--sync", "--chapter-sync"} & set(command)
|
||||
|
||||
|
||||
def test_sample_range_reads_a_window() -> None:
|
||||
assert SAMPLE_RANGE.convert("60-120", None, None) == (60, 120)
|
||||
assert SAMPLE_RANGE.convert(True, None, None) == (120, 180)
|
||||
assert SAMPLE_RANGE.convert(False, None, None) is None
|
||||
for bad in ("120-60", "60-60", "abc", "-5-10", "60"):
|
||||
with pytest.raises(click.BadParameter):
|
||||
SAMPLE_RANGE.convert(bad, None, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample", [True, "60-120", [60, 120]])
|
||||
def test_the_api_accepts_a_sample_window(sample: object) -> None:
|
||||
assert validate_download_parameters({"sample": sample}) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample", ["abc", "120-60", [1, 2, 3], ["a", "b"], 5, {"start": 1}])
|
||||
def test_the_api_refuses_a_bad_sample_window(sample: object) -> None:
|
||||
assert "sample" in (validate_download_parameters({"sample": sample}) or "")
|
||||
|
||||
|
||||
def test_the_api_refuses_sample_with_include_sample() -> None:
|
||||
assert "include_sample" in (validate_download_parameters({"sample": True, "include_sample": True}) or "")
|
||||
|
||||
|
||||
def test_sample_name_uses_the_separator_of_the_template(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
separators = {"movies": ".", "series": " "}
|
||||
monkeypatch.setattr(config, "get_template_separator", lambda kind="movies": separators[kind])
|
||||
movie = Movie.__new__(Movie)
|
||||
episode = Episode.__new__(Episode)
|
||||
assert sample_name("Some.Movie.2026.1080p-TAG", movie) == "Some.Movie.2026.1080p-TAG.SAMPLE"
|
||||
assert sample_name("Some Show S01E01 1080p-TAG", episode) == "Some Show S01E01 1080p-TAG SAMPLE"
|
||||
|
||||
|
||||
SIDX_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<Representation id="a" bandwidth="1">
|
||||
<BaseURL>https://example.com/a.mp4</BaseURL>
|
||||
<SegmentBase indexRange="800-{last}"><Initialization range="0-799"/></SegmentBase>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
|
||||
def sidx_box(count: int = 100, size: int = 5000, duration: int = 4000, timescale: int = 1000) -> bytes:
|
||||
body = struct.pack(">IIIII", 0, 1, timescale, 0, 0) + struct.pack(">HH", 0, count)
|
||||
body += b"".join(struct.pack(">III", size, duration, 0x90000000) for _ in range(count))
|
||||
return struct.pack(">I", 8 + len(body)) + b"sidx" + body
|
||||
|
||||
|
||||
def test_parse_sidx_gives_the_time_and_byte_range_of_each_subsegment() -> None:
|
||||
box = sidx_box(count=3)
|
||||
end = 800 + len(box)
|
||||
assert parse_sidx(box, 800) == (
|
||||
1000,
|
||||
[(0, end, end + 4999), (4000, end + 5000, end + 9999), (8000, end + 10000, end + 14999)],
|
||||
)
|
||||
|
||||
|
||||
def test_parse_sidx_refuses_what_it_cannot_cut() -> None:
|
||||
index_of_indexes = bytearray(sidx_box(count=2))
|
||||
index_of_indexes[32] |= 0x80
|
||||
assert parse_sidx(bytes(index_of_indexes), 0) == (0, [])
|
||||
assert parse_sidx(b"\x00\x00\x00\x20moof" + b"\x00" * 24, 0) == (0, [])
|
||||
assert parse_sidx(sidx_box(count=3)[:-4], 0) == (0, [])
|
||||
|
||||
|
||||
def test_a_single_file_dash_track_is_cut_by_its_segment_index(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
box = sidx_box()
|
||||
requested: list[str] = []
|
||||
|
||||
def get(url: str, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
byte_range = (headers or {}).get("Range", "")
|
||||
requested.append(byte_range)
|
||||
content = box if byte_range == f"bytes=800-{800 + len(box) - 1}" else b"\x00" * 800
|
||||
return MagicMock(content=content, headers={"Content-Range": "bytes 0-799/900000"})
|
||||
|
||||
tree = load_xml(SIDX_MPD.format(last=800 + len(box) - 1).encode())
|
||||
period = tree.find("Period")
|
||||
aset = period.find("AdaptationSet")
|
||||
rep = aset.find("Representation")
|
||||
track = make_audio()
|
||||
track.data["dash"] = {"adaptation_set": aset, "representation": rep}
|
||||
session = MagicMock(get=get)
|
||||
first_media_byte = 800 + len(box)
|
||||
|
||||
_, segments, timescale, starts, _ = DASH.get_period_segments(
|
||||
period, aset, rep, tree, track, "https://example.com/main.mpd", session, probe_kid=False, window=WINDOW
|
||||
)
|
||||
|
||||
assert DASH.has_segment_timing(track) is True
|
||||
assert timescale == 1000 and starts == [n * 4000 for n in range(15, 30)]
|
||||
assert segments[0] == (
|
||||
"https://example.com/a.mp4",
|
||||
f"{first_media_byte + 15 * 5000}-{first_media_byte + 16 * 5000 - 1}",
|
||||
)
|
||||
assert len(segments) == 15 and track.sample_start == 60
|
||||
|
||||
requested.clear()
|
||||
_, whole, _, _, _ = DASH.get_period_segments(
|
||||
period, aset, rep, tree, track, "https://example.com/main.mpd", session, probe_kid=False
|
||||
)
|
||||
assert len(whole) == 1 and not any("800-" in r for r in requested), "a full download does not read the index"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "window"),
|
||||
[
|
||||
(["--sample", "TAG", "-t", "x", "id"], (120, 180)),
|
||||
(["--sample", "60-120", "TAG", "id"], (60, 120)),
|
||||
(["TAG", "id"], None),
|
||||
],
|
||||
)
|
||||
def test_a_bare_sample_option_does_not_take_the_service_tag(args: list[str], window: object) -> None:
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
class Group(Services):
|
||||
def list_commands(self, ctx: click.Context) -> list[str]:
|
||||
return ["TAG"]
|
||||
|
||||
def get_command(self, ctx: click.Context, name: str) -> click.Command:
|
||||
return click.Command(
|
||||
name, params=[click.Option(["-t"]), click.Argument(["title"])], callback=lambda **k: None
|
||||
)
|
||||
|
||||
@click.group(cls=Group)
|
||||
@click.option("--sample", type=SAMPLE_RANGE, default=None)
|
||||
def group(sample: object) -> None:
|
||||
seen["sample"] = sample
|
||||
|
||||
group(args, standalone_mode=False)
|
||||
assert seen["sample"] == window
|
||||
|
||||
|
||||
class RangeSession:
|
||||
"""Serves byte ranges of one in-memory file and records each range it was asked for."""
|
||||
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
self.ranges: list[str] = []
|
||||
|
||||
def get(self, url: str, headers: dict[str, str]) -> SimpleNamespace:
|
||||
self.ranges.append(headers["Range"])
|
||||
first, last = (int(n) for n in headers["Range"].removeprefix("bytes=").split("-"))
|
||||
return SimpleNamespace(ok=True, content=self.data[first : last + 1])
|
||||
|
||||
|
||||
def one_file(moov_size: int, index: bytes) -> bytes:
|
||||
ftyp = struct.pack(">I4s", 16, b"ftyp") + b"isom" + b"\x00" * 4
|
||||
moov = struct.pack(">I4s", moov_size, b"moov") + b"\x00" * (moov_size - 8)
|
||||
return ftyp + moov + index + b"\x00" * 600_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moov_size", [700, 200_000])
|
||||
def test_a_one_file_track_is_cut_by_the_index_in_the_file(moov_size: int, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A large moov puts the index after the first probe, so the probe must read on."""
|
||||
monkeypatch.setattr(config, "sample", WINDOW)
|
||||
box = sidx_box()
|
||||
session = RangeSession(one_file(moov_size, box))
|
||||
audio = make_audio("https://x/a.mp4")
|
||||
|
||||
init, byte_range = audio.sample_byte_range(session)
|
||||
|
||||
media = 16 + moov_size + len(box)
|
||||
assert init == session.data[: 16 + moov_size], "the init data stops before the index"
|
||||
assert byte_range == f"{media + 15 * 5000}-{media + 30 * 5000 - 1}"
|
||||
assert audio.sample_start == 60
|
||||
assert all(r.startswith("bytes=") for r in session.ranges), "every request is a byte range"
|
||||
|
||||
|
||||
def test_a_one_file_track_with_no_index_cannot_be_cut(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "sample", WINDOW)
|
||||
mdat = struct.pack(">I4s", 600_000, b"mdat") + b"\x00" * (600_000 - 8)
|
||||
audio = make_audio("https://x/a.mp4")
|
||||
assert audio.sample_byte_range(RangeSession(one_file(700, b"")[:716] + mdat)) is None
|
||||
+140
-7
@@ -46,6 +46,7 @@ from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote
|
||||
from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption
|
||||
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
|
||||
from unshackle.core.constants import (
|
||||
DEFAULT_SAMPLE,
|
||||
DOWNLOAD_ALL_DRM,
|
||||
DOWNLOAD_CANCELLED,
|
||||
DOWNLOAD_LICENCE_ONLY,
|
||||
@@ -107,6 +108,7 @@ from unshackle.core.utils.click_types import (
|
||||
AUDIO_CODEC_LIST,
|
||||
LANGUAGE_RANGE,
|
||||
QUALITY_LIST,
|
||||
SAMPLE_RANGE,
|
||||
SEASON_RANGE,
|
||||
SLOW_DELAY_RANGE,
|
||||
SUBTITLE_CODEC,
|
||||
@@ -118,7 +120,7 @@ from unshackle.core.utils.collections import ci_get, merge_dict
|
||||
from unshackle.core.utils.post_scripts import NO_POST_SCRIPTS, build_context, dispatch, season_context
|
||||
from unshackle.core.utils.redact import mask_proxy
|
||||
from unshackle.core.utils.selector import select_multiple
|
||||
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe
|
||||
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe, run_step
|
||||
from unshackle.core.vault import Vault
|
||||
from unshackle.core.vaults import Vaults
|
||||
|
||||
@@ -259,6 +261,19 @@ def latest_episode_keys(titles: Any, latest_episode: bool, latest_episodes: Opti
|
||||
return list(dict.fromkeys((ep.season, ep.number) for ep in titles))[-count:]
|
||||
|
||||
|
||||
SAMPLE_DIR = "Sample"
|
||||
|
||||
|
||||
def template_type(title: Any) -> str:
|
||||
"""Return the output template key of a title: ``series``, ``songs`` or ``movies``."""
|
||||
return "series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
||||
|
||||
|
||||
def sample_name(name: str, title: Any) -> str:
|
||||
"""Return ``name`` with a SAMPLE tag, joined with the separator of the output template of the title."""
|
||||
return f"{name.rstrip()}{config.get_template_separator(template_type(title))}SAMPLE"
|
||||
|
||||
|
||||
def apply_enrich(title: Title_T, name: Optional[str], year: Optional[int], language: Optional[Language]) -> None:
|
||||
"""Set the --enrich name, year and original language on a title. A song takes only the language.
|
||||
|
||||
@@ -324,6 +339,41 @@ def remux_music_tracks(title: Any, log: logging.Logger) -> None:
|
||||
log.warning(f"Could not remux audio track {track.id} for music: {e}")
|
||||
|
||||
|
||||
def cut_music_sample(title: Any, window: tuple[int, int], log: logging.Logger) -> None:
|
||||
"""Cut each audio track of a Song to the sample window. A Song has no mux step that cuts it."""
|
||||
if not isinstance(title, Song) or not binaries.FFMPEG:
|
||||
return
|
||||
for track in title.tracks.audio:
|
||||
if not track.path or not track.path.exists():
|
||||
continue
|
||||
cut_path = track.path.with_stem(f"{track.path.stem}_cut")
|
||||
try:
|
||||
run_step(
|
||||
[
|
||||
binaries.FFMPEG,
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-ss",
|
||||
f"{max(window[0] - track.sample_start, 0):.3f}",
|
||||
"-i",
|
||||
track.path,
|
||||
"-t",
|
||||
str(window[1] - window[0]),
|
||||
"-map",
|
||||
"0",
|
||||
"-c",
|
||||
"copy",
|
||||
cut_path,
|
||||
],
|
||||
output=cut_path,
|
||||
label="music sample cut",
|
||||
)
|
||||
os.replace(cut_path, track.path)
|
||||
except RuntimeError as e:
|
||||
log.warning(f"Could not cut audio track {track.id} to the sample window, so it holds whole segments: {e}")
|
||||
|
||||
|
||||
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
|
||||
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
|
||||
The config replaces defaults and global ``dl:`` default_map values."""
|
||||
@@ -972,6 +1022,19 @@ class dl:
|
||||
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
|
||||
)
|
||||
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
|
||||
@click.option(
|
||||
"--sample",
|
||||
type=SAMPLE_RANGE,
|
||||
default=None,
|
||||
help="Download a short sample and write it to a Sample folder. "
|
||||
"Use --sample for 60 seconds from 2 minutes in, or --sample START-END (e.g., --sample 0-60) for a custom window.",
|
||||
)
|
||||
@click.option(
|
||||
"--include-sample",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Also cut a 60 second sample from each muxed file into a Sample folder next to it.",
|
||||
)
|
||||
@click.option(
|
||||
"--postscript",
|
||||
"postscript",
|
||||
@@ -1566,6 +1629,8 @@ class dl:
|
||||
no_folder: bool,
|
||||
no_source: bool,
|
||||
no_mux: bool,
|
||||
sample: Optional[tuple[int, int]],
|
||||
include_sample: bool,
|
||||
workers: Optional[int],
|
||||
adaptive_workers: bool,
|
||||
download_processes: int,
|
||||
@@ -1595,6 +1660,10 @@ class dl:
|
||||
postscript = NO_POST_SCRIPTS
|
||||
if continue_downloads:
|
||||
config.continue_downloads = True
|
||||
config.sample = sample
|
||||
if sample and include_sample:
|
||||
raise click.ClickException("--sample and --include-sample cannot be used together.")
|
||||
sample_dir = SAMPLE_DIR if sample else ""
|
||||
self.tmdb_searched = False
|
||||
self.search_source = None
|
||||
self.service_anime = bool(getattr(service, "ANIME", False))
|
||||
@@ -3058,6 +3127,18 @@ class dl:
|
||||
self.prefer_vault_keys(title)
|
||||
self.cache_resolved_keys(title)
|
||||
|
||||
if sample and not skip_dl:
|
||||
whole_files = [
|
||||
track
|
||||
for track in (*title.tracks.videos, *title.tracks.audio)
|
||||
if not track.can_sample(track.session or service.session)
|
||||
]
|
||||
if whole_files:
|
||||
raise click.ClickException(
|
||||
f"--sample cannot cut {len(whole_files)} track(s) of {title}: "
|
||||
f"{self.service} gives each one as one file with no segment index."
|
||||
)
|
||||
|
||||
dl_start_time = time.time()
|
||||
|
||||
try:
|
||||
@@ -3213,6 +3294,10 @@ class dl:
|
||||
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
|
||||
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
|
||||
)
|
||||
if sample:
|
||||
for subtitle in title.tracks.subtitles:
|
||||
if subtitle.path and subtitle.path.exists():
|
||||
subtitle.trim(*sample)
|
||||
for (src, dst), count in sub_conversions.items():
|
||||
log_event(
|
||||
"subtitle_convert",
|
||||
@@ -3340,6 +3425,8 @@ class dl:
|
||||
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
|
||||
|
||||
remux_music_tracks(title, self.log)
|
||||
if sample:
|
||||
cut_music_sample(title, sample, self.log)
|
||||
|
||||
muxed_paths = []
|
||||
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
|
||||
@@ -3522,6 +3609,11 @@ class dl:
|
||||
hybrid_output_path = config.directories.temp / hybrid_filename
|
||||
hybrid_temp_paths.append(hybrid_output_path)
|
||||
|
||||
if len({track.sample_start for track in resolution_tracks}) > 1:
|
||||
raise click.ClickException(
|
||||
"--sample cut the HDR10 and Dolby Vision tracks at different "
|
||||
"points, so a hybrid of them would not line up."
|
||||
)
|
||||
Hybrid(resolution_tracks, self.service, output_name=hybrid_filename)
|
||||
|
||||
if container:
|
||||
@@ -3579,6 +3671,8 @@ class dl:
|
||||
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
|
||||
if media_info:
|
||||
base_filename = title.get_filename(media_info, show_service=not no_source)
|
||||
if sample:
|
||||
base_filename = sample_name(base_filename, title)
|
||||
else:
|
||||
base_filename = str(title)
|
||||
|
||||
@@ -3594,6 +3688,7 @@ class dl:
|
||||
sidecar_dir /= title.get_filename(
|
||||
media_info, show_service=not no_source, folder=True
|
||||
)
|
||||
sidecar_dir /= sample_dir
|
||||
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
|
||||
|
||||
with console.status("Saving subtitle sidecar files..."):
|
||||
@@ -3674,11 +3769,14 @@ class dl:
|
||||
media_info = MediaInfo.parse(sample_track.path)
|
||||
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
||||
|
||||
final_dir /= sample_dir
|
||||
final_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for track_path in muxed_paths:
|
||||
media_info = MediaInfo.parse(track_path)
|
||||
base_filename = title.get_filename(media_info, show_service=not no_source)
|
||||
if sample:
|
||||
base_filename = sample_name(base_filename, title)
|
||||
|
||||
track = next((t for t in title.tracks if t.path == track_path), None)
|
||||
if track:
|
||||
@@ -3710,6 +3808,8 @@ class dl:
|
||||
media_info = MediaInfo.parse(muxed_path)
|
||||
final_dir = self.output_dir or config.directories.downloads
|
||||
final_filename = title.get_filename(media_info, show_service=not no_source)
|
||||
if sample:
|
||||
final_filename = sample_name(final_filename, title)
|
||||
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
|
||||
|
||||
if not no_folder and (
|
||||
@@ -3717,13 +3817,11 @@ class dl:
|
||||
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
||||
):
|
||||
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
||||
final_dir /= sample_dir
|
||||
|
||||
final_dir.mkdir(parents=True, exist_ok=True)
|
||||
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
||||
template_type = (
|
||||
"series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
||||
)
|
||||
sep = config.get_template_separator(template_type)
|
||||
sep = config.get_template_separator(template_type(title))
|
||||
|
||||
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
|
||||
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
|
||||
@@ -3754,6 +3852,8 @@ class dl:
|
||||
self.anime_hint(title),
|
||||
session=service.session,
|
||||
)
|
||||
if include_sample:
|
||||
self.write_sample(final_path, title, media_info)
|
||||
|
||||
post_script_context = build_context(
|
||||
title,
|
||||
@@ -3807,6 +3907,32 @@ class dl:
|
||||
|
||||
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
||||
|
||||
def write_sample(self, source: Path, title: Title_T, media_info: MediaInfo) -> None:
|
||||
"""Cut a 60 second sample from a finished file into a ``Sample`` folder next to it.
|
||||
|
||||
The sample starts 2 minutes in. A file shorter than 3 minutes gives its middle 60 seconds.
|
||||
"""
|
||||
duration = float(media_info.general_tracks[0].duration or 0) / 1000
|
||||
start, end = DEFAULT_SAMPLE
|
||||
length = end - start
|
||||
if duration < end:
|
||||
start = max(0.0, (duration - length) / 2)
|
||||
output = source.parent / SAMPLE_DIR / f"{sample_name(source.stem, title)}{source.suffix}"
|
||||
output.parent.mkdir(exist_ok=True)
|
||||
returncode = subprocess.run(
|
||||
[
|
||||
str(binaries.MKVToolNix),
|
||||
"-q",
|
||||
"-o",
|
||||
str(output),
|
||||
"--split",
|
||||
f"parts:{start:.3f}s-{start + length:.3f}s",
|
||||
source,
|
||||
]
|
||||
).returncode
|
||||
if returncode > 1 or not output.exists():
|
||||
self.log.warning(f"mkvmerge wrote no sample for {source.name}")
|
||||
|
||||
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
|
||||
"""Whether metadata lookups for this title should prefer AniList."""
|
||||
per_title = getattr(title, "anime", None)
|
||||
@@ -4064,7 +4190,12 @@ class dl:
|
||||
doc.add(entry)
|
||||
|
||||
tracks_map = entry.ext("unshackle").setdefault("tracks", {})
|
||||
tracks_map.setdefault(str(track.id), track.to_dict())
|
||||
track_dict = tracks_map.setdefault(str(track.id), track.to_dict())
|
||||
unwanted = (track.data.get("hls") or {}).get("unwanted_segments")
|
||||
if unwanted:
|
||||
track_dict["unwanted_segments"] = unwanted
|
||||
elif unwanted is not None:
|
||||
track_dict.pop("unwanted_segments", None)
|
||||
# the row names the track's KIDs, so an import gives the track only their keys
|
||||
track_kids = [kid.hex for kid in own_kids(drm)] if drm is not None else []
|
||||
for row in entry.tracks:
|
||||
@@ -5449,6 +5580,7 @@ class dl:
|
||||
"""
|
||||
Get CDM for a specified service (either Local or Remote CDM).
|
||||
Now supports quality-based selection when the caller gives a quality.
|
||||
With no ``drm``, an entry gives the DRM system that it lists first.
|
||||
Raises a ValueError if there is a problem getting a CDM.
|
||||
"""
|
||||
# A per-request override (set by the REST API per job) takes precedence over the
|
||||
@@ -5529,7 +5661,8 @@ class dl:
|
||||
"pr": "playready",
|
||||
"playready": "playready",
|
||||
}.get(drm.lower())
|
||||
cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready")
|
||||
first = next(key for key in lower_keys if key in ("widevine", "playready"))
|
||||
cdm_name = lower_keys.get(drm_key or first) or lower_keys.get("playready")
|
||||
else:
|
||||
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(config.cdm, "default")
|
||||
if not cdm_name:
|
||||
|
||||
@@ -96,12 +96,13 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None:
|
||||
|
||||
service_tag = service.__class__.__name__
|
||||
|
||||
with console.status("Authenticating with Service...", spinner="dots"):
|
||||
cookies = dl.get_cookie_jar(service_tag, profile)
|
||||
credential = dl.get_credentials(service_tag, profile)
|
||||
service.authenticate(cookies, credential)
|
||||
if cookies or credential:
|
||||
log.info("Authenticated with Service")
|
||||
if not service.ANONYMOUS_SEARCH:
|
||||
with console.status("Authenticating with Service...", spinner="dots"):
|
||||
cookies = dl.get_cookie_jar(service_tag, profile)
|
||||
credential = dl.get_credentials(service_tag, profile)
|
||||
service.authenticate(cookies, credential)
|
||||
if cookies or credential:
|
||||
log.info("Authenticated with Service")
|
||||
|
||||
search_results = Tree("Search Results", hide_root=True)
|
||||
with console.status("Searching...", spinner="dots"):
|
||||
@@ -116,7 +117,7 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None:
|
||||
result_text += f"\n[bright_black]id: {result.id}[/]"
|
||||
search_results.add(result_text + "\n")
|
||||
|
||||
cookie_file = dl.get_cookie_path(service_tag, profile)
|
||||
cookie_file = None if service.ANONYMOUS_SEARCH else dl.get_cookie_path(service_tag, profile)
|
||||
if cookie_file:
|
||||
dl.save_cookies(cookie_file, service.session.cookies)
|
||||
|
||||
|
||||
+55
-47
@@ -54,6 +54,23 @@ class _MaskAccessKey(logging.Filter):
|
||||
return True
|
||||
|
||||
|
||||
def _install_session_store(app: web.Application) -> None:
|
||||
"""Start the remote session store's idle sweep with the app and stop it, bridges included, on cleanup."""
|
||||
from unshackle.core.api.session_store import get_session_store
|
||||
|
||||
session_store = get_session_store()
|
||||
|
||||
async def start(_app: web.Application) -> None:
|
||||
await session_store.start_cleanup_loop()
|
||||
|
||||
async def stop(_app: web.Application) -> None:
|
||||
await session_store.cancel_all_bridges()
|
||||
await session_store.stop_cleanup_loop()
|
||||
|
||||
app.on_startup.append(start)
|
||||
app.on_cleanup.append(stop)
|
||||
|
||||
|
||||
def _install_service_refresh(app: web.Application) -> None:
|
||||
"""Report service load issues, then periodically pull the service repos and hot-reload changed services."""
|
||||
from unshackle.core import services
|
||||
@@ -298,6 +315,12 @@ def _install_config_reload(app: web.Application, *, no_key: bool, serve_widevine
|
||||
@click.option(
|
||||
"--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine/pyplayready CDM."
|
||||
)
|
||||
@click.option(
|
||||
"--cdm-only",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Serve only the pywidevine/pyplayready CDM endpoints (plus health and the dashboard), not the REST API.",
|
||||
)
|
||||
@click.option("--no-widevine", is_flag=True, default=False, help="Disable Widevine CDM endpoints.")
|
||||
@click.option("--no-playready", is_flag=True, default=False, help="Disable PlayReady CDM endpoints.")
|
||||
@click.option("--no-key", is_flag=True, default=False, help="Disable API key authentication (allows all requests).")
|
||||
@@ -331,6 +354,7 @@ def serve(
|
||||
port: int,
|
||||
caddy: bool,
|
||||
api_only: bool,
|
||||
cdm_only: bool,
|
||||
no_widevine: bool,
|
||||
no_playready: bool,
|
||||
no_key: bool,
|
||||
@@ -408,6 +432,8 @@ def serve(
|
||||
|
||||
if api_only and (no_widevine or no_playready):
|
||||
raise click.ClickException("Cannot use --api-only with --no-widevine or --no-playready.")
|
||||
if cdm_only and (api_only or remote_only):
|
||||
raise click.ClickException("Cannot use --cdm-only with --api-only or --remote-only.")
|
||||
|
||||
if caddy:
|
||||
if not binaries.Caddy:
|
||||
@@ -428,31 +454,34 @@ def serve(
|
||||
if remote_only:
|
||||
api_only = True
|
||||
stats.host, stats.port = host, port
|
||||
stats.mode = "remote_only" if remote_only else "api_only" if api_only else "full"
|
||||
stats.mode = "remote_only" if remote_only else "api_only" if api_only else "cdm_only" if cdm_only else "full"
|
||||
dashboard = bool(dashboard_key())
|
||||
if dashboard:
|
||||
log.info(f"Developer dashboard endpoints available at http://{host}:{port}/api/dashboard/")
|
||||
elif quiet:
|
||||
log.info("Developer dashboard disabled: set serve.dashboard.key in unshackle.yaml to enable it")
|
||||
|
||||
global_speed_limit = parse_speed_limit(config.serve.get("global_speed_limit"))
|
||||
if global_speed_limit:
|
||||
set_speed_limit(global_speed_limit, lock=True)
|
||||
log.info(f"Global speed limit: {format_speed(global_speed_limit)} (shared by all jobs)")
|
||||
if not cdm_only:
|
||||
global_speed_limit = parse_speed_limit(config.serve.get("global_speed_limit"))
|
||||
if global_speed_limit:
|
||||
set_speed_limit(global_speed_limit, lock=True)
|
||||
log.info(f"Global speed limit: {format_speed(global_speed_limit)} (shared by all jobs)")
|
||||
|
||||
global_services = config.serve.get("services")
|
||||
if global_services:
|
||||
log.info(f"Global service allowlist: {', '.join(global_services)}")
|
||||
try:
|
||||
server_accounts = validate_server_accounts()
|
||||
except ValueError as e:
|
||||
raise click.ClickException(str(e))
|
||||
for tag in server_accounts:
|
||||
regions = server_account_regions(tag) or {}
|
||||
covered = list(regions.get("regions") or []) + (["global"] if regions.get("global") else [])
|
||||
log.info(f"Server accounts for {tag}: {', '.join(covered)}")
|
||||
if config.key_vaults and not any(v.get("type") == "SQLite" for v in config.key_vaults):
|
||||
log.warning("No SQLite key vault configured: a content key a remote client proves wrong cannot be flagged")
|
||||
global_services = config.serve.get("services")
|
||||
if global_services:
|
||||
log.info(f"Global service allowlist: {', '.join(global_services)}")
|
||||
try:
|
||||
server_accounts = validate_server_accounts()
|
||||
except ValueError as e:
|
||||
raise click.ClickException(str(e))
|
||||
for tag in server_accounts:
|
||||
regions = server_account_regions(tag) or {}
|
||||
covered = list(regions.get("regions") or []) + (["global"] if regions.get("global") else [])
|
||||
log.info(f"Server accounts for {tag}: {', '.join(covered)}")
|
||||
if config.key_vaults and not any(v.get("type") == "SQLite" for v in config.key_vaults):
|
||||
log.warning(
|
||||
"No SQLite key vault configured: a content key a remote client proves wrong cannot be flagged"
|
||||
)
|
||||
|
||||
serve_widevine = not api_only and not no_widevine
|
||||
serve_playready = not api_only and not no_playready
|
||||
@@ -478,19 +507,7 @@ def serve(
|
||||
app["config"] = serve_config
|
||||
app["debug_api"] = debug_api
|
||||
|
||||
from unshackle.core.api.session_store import get_session_store
|
||||
|
||||
session_store = get_session_store()
|
||||
|
||||
async def start_session_cleanup(_app: web.Application) -> None:
|
||||
await session_store.start_cleanup_loop()
|
||||
|
||||
async def stop_session_cleanup(_app: web.Application) -> None:
|
||||
await session_store.cancel_all_bridges()
|
||||
await session_store.stop_cleanup_loop()
|
||||
|
||||
app.on_startup.append(start_session_cleanup)
|
||||
app.on_cleanup.append(stop_session_cleanup)
|
||||
_install_session_store(app)
|
||||
_install_service_refresh(app)
|
||||
_install_config_reload(app, no_key=no_key, serve_widevine=serve_widevine, serve_playready=serve_playready)
|
||||
|
||||
@@ -571,20 +588,9 @@ def serve(
|
||||
app["config"] = serve_config
|
||||
app["debug_api"] = debug_api
|
||||
|
||||
from unshackle.core.api.session_store import get_session_store
|
||||
|
||||
session_store = get_session_store()
|
||||
|
||||
async def start_session_cleanup(_app: web.Application) -> None:
|
||||
await session_store.start_cleanup_loop()
|
||||
|
||||
async def stop_session_cleanup(_app: web.Application) -> None:
|
||||
await session_store.cancel_all_bridges()
|
||||
await session_store.stop_cleanup_loop()
|
||||
|
||||
app.on_startup.append(start_session_cleanup)
|
||||
app.on_cleanup.append(stop_session_cleanup)
|
||||
_install_service_refresh(app)
|
||||
if not cdm_only:
|
||||
_install_session_store(app)
|
||||
_install_service_refresh(app)
|
||||
_install_config_reload(app, no_key=no_key, serve_widevine=serve_widevine, serve_playready=serve_playready)
|
||||
|
||||
if serve_widevine:
|
||||
@@ -617,11 +623,13 @@ def serve(
|
||||
elif serve_playready:
|
||||
log.info("No PlayReady devices found, skipping PlayReady CDM endpoints")
|
||||
|
||||
setup_routes(app, remote_only=remote_only, dashboard=dashboard)
|
||||
setup_routes(app, remote_only=remote_only, dashboard=dashboard, cdm_only=cdm_only)
|
||||
|
||||
if serve_widevine:
|
||||
log.info(f"Widevine CDM endpoints available at http://{host}:{port}/{{device}}/open")
|
||||
if remote_only:
|
||||
if cdm_only:
|
||||
log.info("REST API disabled (--cdm-only): only /api/health and the dashboard answer under /api/")
|
||||
elif remote_only:
|
||||
log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/")
|
||||
else:
|
||||
setup_swagger(app, dashboard=dashboard)
|
||||
|
||||
@@ -7,7 +7,7 @@ from rich_click.patch import patch as _patch_click_help
|
||||
|
||||
_patch_click_help()
|
||||
|
||||
__version__ = "5.5.0"
|
||||
__version__ = "6.0.0"
|
||||
|
||||
PKG = Path(__file__).parent.parent
|
||||
# Framework code only. Services are user-swappable, so they are not part of the identity.
|
||||
|
||||
@@ -126,6 +126,8 @@ class DownloadJob:
|
||||
|
||||
# Prompt a service raised through prompt_user() and waits on; answered by submit_input().
|
||||
input_prompt: Optional[str] = None
|
||||
# Message a service shows through show_notice() while it waits for an approval; needs no answer.
|
||||
notice: Optional[str] = None
|
||||
|
||||
# Subtitles skipped under skip_subtitle_errors (non-fatal). Each entry is a dl.SkippedSubtitle
|
||||
# dict (id / language / title) so a client can report which weren't available.
|
||||
@@ -160,6 +162,7 @@ class DownloadJob:
|
||||
"speed": self.speed,
|
||||
"skipped_subtitles": self.skipped_subtitles,
|
||||
"input_prompt": self.input_prompt,
|
||||
"notice": self.notice,
|
||||
}
|
||||
|
||||
if include_full_details:
|
||||
@@ -415,6 +418,15 @@ def perform_download(
|
||||
if params.get("export"):
|
||||
params["export"] = bool(params["export"])
|
||||
|
||||
sample_raw = params.get("sample")
|
||||
if sample_raw is not None:
|
||||
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||
|
||||
try:
|
||||
params["sample"] = SAMPLE_RANGE.convert(sample_raw, None, None)
|
||||
except click.BadParameter as exc:
|
||||
raise Exception(f"Invalid sample parameter: {exc}")
|
||||
|
||||
slow_raw = params.get("slow")
|
||||
if slow_raw is not None and not isinstance(slow_raw, tuple):
|
||||
if isinstance(slow_raw, bool):
|
||||
@@ -488,6 +500,7 @@ def perform_download(
|
||||
"v_lang": params.get("v_lang", []),
|
||||
"a_lang": params.get("a_lang", []),
|
||||
"forced_subs": params.get("forced_subs", False),
|
||||
"exact_lang": params.get("exact_lang", False),
|
||||
}
|
||||
# Hand-built context: record parameter sources so service dl overrides
|
||||
# apply to defaults but never clobber client-sent values.
|
||||
@@ -630,6 +643,8 @@ def perform_download(
|
||||
no_folder=params.get("no_folder", False),
|
||||
no_source=params.get("no_source", False),
|
||||
no_mux=params.get("no_mux", False),
|
||||
sample=params.get("sample"),
|
||||
include_sample=params.get("include_sample", False),
|
||||
workers=params.get("workers"),
|
||||
adaptive_workers=params.get("adaptive_workers", False),
|
||||
download_processes=params.get("download_processes", 1),
|
||||
@@ -1109,6 +1124,8 @@ class DownloadQueueManager:
|
||||
job.speed = str(progress_data["speed"])
|
||||
if "input_prompt" in progress_data:
|
||||
job.input_prompt = progress_data["input_prompt"]
|
||||
if "notice" in progress_data:
|
||||
job.notice = progress_data["notice"]
|
||||
if progress_data.get("skipped_subtitles"):
|
||||
job.skipped_subtitles = progress_data["skipped_subtitles"]
|
||||
if "progress" in progress_data:
|
||||
@@ -1195,6 +1212,7 @@ class DownloadQueueManager:
|
||||
|
||||
self._download_processes.pop(job.job_id, None)
|
||||
job.input_prompt = None
|
||||
job.notice = None
|
||||
|
||||
temp_paths = self._job_temp_files.pop(job.job_id, {})
|
||||
for path in temp_paths.values():
|
||||
|
||||
@@ -16,7 +16,7 @@ from typing import Any, Callable, Dict
|
||||
from uuid import uuid4
|
||||
|
||||
from unshackle.core.api.input_bridge import AUTH_INPUT_TIMEOUT
|
||||
from unshackle.core.console import set_prompt_handler
|
||||
from unshackle.core.console import set_notice_handler, set_prompt_handler
|
||||
|
||||
from .download_manager import perform_download
|
||||
|
||||
@@ -64,7 +64,7 @@ def relay_prompts(progress_callback: Callable[[Dict[str, Any]], None]) -> None:
|
||||
The parent writes each answer as one JSON string per line, so an answer can hold a newline. The reader
|
||||
uses unbuffered stdin: a daemon thread blocked in ``sys.stdin.buffer`` aborts the interpreter at shutdown.
|
||||
Prompts go out one at a time, and a prompt discards answers that arrived before it, such as a late
|
||||
answer to a prompt that timed out.
|
||||
answer to a prompt that timed out. A show_notice() message goes out as ``notice`` progress and needs no answer.
|
||||
"""
|
||||
answers: queue.Queue[str] = queue.Queue()
|
||||
one_prompt = threading.Lock()
|
||||
@@ -92,6 +92,7 @@ def relay_prompts(progress_callback: Callable[[Dict[str, Any]], None]) -> None:
|
||||
progress_callback({"input_prompt": None})
|
||||
|
||||
set_prompt_handler(ask)
|
||||
set_notice_handler(lambda message: progress_callback({"notice": message}))
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
|
||||
@@ -20,7 +20,7 @@ from aiohttp import web
|
||||
|
||||
from unshackle.core.api.compression import safe_inflate
|
||||
from unshackle.core.api.errors import APIError, APIErrorCode, categorize_exception, handle_api_exception
|
||||
from unshackle.core.api.input_bridge import AuthStatus, InputBridge
|
||||
from unshackle.core.api.input_bridge import AuthStatus, InputBridge, NoUserBridge
|
||||
from unshackle.core.api.sanitize import MAX_SESSION_CACHE_KEYS, safe_cache_key, sanitize_log
|
||||
from unshackle.core.api.session_log import SessionLogBuffer, SessionLogMirror, capture_service_logs
|
||||
from unshackle.core.api.session_store import SessionStore
|
||||
@@ -31,6 +31,7 @@ from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_C
|
||||
from unshackle.core.providers.anilist import parse_anilist_ref
|
||||
from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.titles import Episode, Movie, Song, Title_T
|
||||
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
|
||||
from unshackle.core.utilities import declared_kwargs, kind_only_conflict
|
||||
@@ -92,6 +93,8 @@ DEFAULT_DOWNLOAD_PARAMS = {
|
||||
"no_folder": False,
|
||||
"no_source": False,
|
||||
"no_mux": False,
|
||||
"sample": None,
|
||||
"include_sample": False,
|
||||
"workers": None,
|
||||
"adaptive_workers": False,
|
||||
"download_processes": 1,
|
||||
@@ -154,7 +157,8 @@ def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str]
|
||||
drm_key = {"wv": "widevine", "widevine": "widevine", "pr": "playready", "playready": "playready"}.get(
|
||||
cdm_type.lower()
|
||||
)
|
||||
cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready")
|
||||
first = next(key for key in lower_keys if key in ("widevine", "playready"))
|
||||
cdm_name = lower_keys.get(drm_key or first) or lower_keys.get("playready")
|
||||
else:
|
||||
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(app_config.cdm, "default")
|
||||
|
||||
@@ -356,7 +360,7 @@ class RequestCache:
|
||||
Such a server keeps nothing that a client sends, so the request runs on a cache directory
|
||||
of its own that the handler removes when the request ends, on success and on error. Before
|
||||
that, a client that logged in with its own cookies, credentials or cache gets the updated
|
||||
files back, the same as when a remote session ends.
|
||||
files back.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
@@ -443,6 +447,7 @@ def setup_list_service(
|
||||
elif request_cache is not None:
|
||||
client_login = cookies is not None or credential is not None
|
||||
request_cache.attach(service_instance, data, normalized_service, request, client_login)
|
||||
service_instance._input_bridge = NoUserBridge()
|
||||
service_instance.authenticate(cookies, credential)
|
||||
return service_instance
|
||||
|
||||
@@ -459,6 +464,8 @@ def run_service_search(
|
||||
The same preamble as :func:`setup_list_service`, without the service option extras,
|
||||
plus the search itself. Every step blocks on disk or network, so the caller runs the
|
||||
whole function in one worker thread and keeps the event loop free for other clients.
|
||||
|
||||
The server account check and its cache directory apply to an ``ANONYMOUS_SEARCH`` service too.
|
||||
"""
|
||||
|
||||
profile = client_profile(data)
|
||||
@@ -472,7 +479,11 @@ def run_service_search(
|
||||
profile = None if account == "default" else account
|
||||
|
||||
cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type"))
|
||||
cookies, credential = server_login_material(data, normalized_service, account, profile)
|
||||
service_module = Services.load(normalized_service)
|
||||
anonymous = getattr(service_module, "ANONYMOUS_SEARCH", False)
|
||||
cookies, credential = (
|
||||
(None, None) if anonymous else server_login_material(data, normalized_service, account, profile)
|
||||
)
|
||||
|
||||
parent_ctx = build_parent_ctx(
|
||||
profile,
|
||||
@@ -483,7 +494,6 @@ def run_service_search(
|
||||
service_config,
|
||||
extra_params={"cookies_supplied": cookies is not None},
|
||||
)
|
||||
service_module = Services.load(normalized_service)
|
||||
|
||||
from click import ClickException
|
||||
|
||||
@@ -503,7 +513,9 @@ def run_service_search(
|
||||
elif request_cache is not None:
|
||||
client_login = cookies is not None or credential is not None
|
||||
request_cache.attach(service_instance, data, normalized_service, request, client_login)
|
||||
service_instance.authenticate(cookies, credential)
|
||||
service_instance._input_bridge = NoUserBridge()
|
||||
if not anonymous:
|
||||
service_instance.authenticate(cookies, credential)
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
try:
|
||||
@@ -1823,6 +1835,16 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]:
|
||||
except APIError as e:
|
||||
return e.message
|
||||
|
||||
if data.get("sample") is not None:
|
||||
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||
|
||||
try:
|
||||
SAMPLE_RANGE.convert(data["sample"], None, None)
|
||||
except click.BadParameter:
|
||||
return "sample must be true or a 'START-END' window in seconds, such as '60-120'."
|
||||
if data.get("sample") and data.get("include_sample"):
|
||||
return "sample and include_sample cannot be used together."
|
||||
|
||||
# The serve user would create and write any directory a client names; keep it under downloads.
|
||||
if data.get("output_dir"):
|
||||
root = config.directories.downloads.resolve()
|
||||
@@ -3256,6 +3278,7 @@ def forwarded_dl_params(data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
except click.BadParameter:
|
||||
log.warning(f"Ignoring unknown dl_params.acodec value: {sanitize_log(name)}")
|
||||
params["forced_subs"] = raw.get("forced_subs") is True
|
||||
params["exact_lang"] = raw.get("exact_lang") is True
|
||||
return params
|
||||
|
||||
|
||||
@@ -3559,7 +3582,7 @@ async def session_create_handler(data: Dict[str, Any], request: Optional[web.Req
|
||||
async def run_auth() -> None:
|
||||
try:
|
||||
await asyncio.to_thread(service_instance.authenticate, cookies, credential)
|
||||
if bridge.answered and not server_account:
|
||||
if not server_account:
|
||||
session.client_auth = True
|
||||
session.auth_status = AuthStatus.AUTHENTICATED
|
||||
bridge.status = AuthStatus.AUTHENTICATED
|
||||
@@ -3647,10 +3670,9 @@ async def session_titles_handler(session_id: str, request: Optional[web.Request]
|
||||
def client_session_auth(session: Any) -> tuple[Dict[str, str], Dict[str, str]]:
|
||||
"""The service session headers and cookies a remote client may receive.
|
||||
|
||||
Only a client that sent its own cookies or credentials gets them back. Otherwise (a server
|
||||
account, or an anonymous login through the server's proxy) the cookie jar stays on the
|
||||
server and the server drops the auth-bearing headers, so the client cannot reuse a login
|
||||
the server paid for. A remote session on the server's lent device keeps its cookies (the client's account)
|
||||
A client whose own login succeeded gets them back. For a server account the cookie jar
|
||||
stays on the server and the server drops the auth-bearing headers, so the client cannot
|
||||
reuse a login the server paid for. A remote session on the server's lent device keeps its cookies (the client's account)
|
||||
but drops those headers too.
|
||||
"""
|
||||
svc_session = session.service_instance.session
|
||||
@@ -3894,7 +3916,8 @@ def resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional[
|
||||
Checks the server's own CDM config (``config.cdm[service]``) to
|
||||
determine the CDM type without loading the full CDM object. This
|
||||
makes sure that when you use ``server_cdm: true``, the server's CDM
|
||||
determines device selection (e.g. PlayReady vs Widevine).
|
||||
determines device selection (e.g. PlayReady vs Widevine). An entry with
|
||||
``widevine`` and ``playready`` gives the DRM system that it lists first.
|
||||
|
||||
Falls back to a lightweight stub from *cdm_type* only if the configuration
|
||||
has no server CDM for the service.
|
||||
@@ -3906,7 +3929,8 @@ def resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional[
|
||||
if isinstance(cdm_name, dict):
|
||||
lower_keys = {k.lower(): v for k, v in cdm_name.items()}
|
||||
if {"widevine", "playready"} & lower_keys.keys():
|
||||
cdm_name = lower_keys.get("playready") or lower_keys.get("widevine")
|
||||
systems = [lower_keys[key] for key in lower_keys if key in ("widevine", "playready")]
|
||||
cdm_name = next(filter(None, systems), None)
|
||||
else:
|
||||
cdm_name = cdm_name.get("default") or next(iter(cdm_name.values()), None)
|
||||
|
||||
@@ -4304,11 +4328,11 @@ def ensure_track_drm(track: Any, session: Any = None, init_data: Optional[bytes]
|
||||
from unshackle.core.manifests import HLS
|
||||
|
||||
http = getattr(track, "session", None) or session
|
||||
playlist = m3u8.loads(http.get(track.url).text, uri=track.url) if http else m3u8.load(track.url)
|
||||
playlist = m3u8.loads(utf8_text(http.get(track.url)), uri=track.url) if http else m3u8.load(track.url)
|
||||
keys = [k for k in (playlist.keys or []) + (playlist.session_keys or []) if k is not None]
|
||||
for key in keys:
|
||||
try:
|
||||
drm = HLS.get_drm(key)
|
||||
drm = HLS.get_drm_or_probe(key, playlist, track, probe_session=http)
|
||||
if isinstance(drm, (Widevine, PlayReady)):
|
||||
track.drm = [drm]
|
||||
return
|
||||
@@ -4784,14 +4808,11 @@ async def session_segment_filter_handler(
|
||||
|
||||
def unwanted_segments() -> list[str]:
|
||||
import m3u8
|
||||
import requests
|
||||
|
||||
response = session.service_instance.session.get(track.url)
|
||||
if not getattr(response, "ok", True):
|
||||
raise ValueError(f"playlist request failed: {response.status_code}")
|
||||
if isinstance(response, requests.Response):
|
||||
response.encoding = response.encoding or "utf-8"
|
||||
playlist = m3u8.loads(response.text, uri=str(track.url))
|
||||
playlist = m3u8.loads(utf8_text(response), uri=str(track.url))
|
||||
return [segment.absolute_uri for segment in playlist.segments if segment_filter(segment)]
|
||||
|
||||
try:
|
||||
|
||||
@@ -49,7 +49,6 @@ class InputBridge:
|
||||
_response: Optional[str] = field(default=None, init=False, repr=False)
|
||||
_status: AuthStatus = field(default=AuthStatus.AUTHENTICATING, init=False)
|
||||
_cancelled: bool = field(default=False, init=False, repr=False)
|
||||
_answered: bool = field(default=False, init=False, repr=False)
|
||||
_cdm_call: Optional[dict[str, Any]] = field(default=None, init=False, repr=False)
|
||||
_response_ready: threading.Event = field(default_factory=threading.Event, init=False, repr=False)
|
||||
_picked_up: threading.Event = field(default_factory=threading.Event, init=False, repr=False)
|
||||
@@ -166,8 +165,6 @@ class InputBridge:
|
||||
if not self._awaiting_answer():
|
||||
return False
|
||||
self._response = response
|
||||
if self._cdm_call is None:
|
||||
self._answered = True
|
||||
self._picked_up.set()
|
||||
self._response_ready.set()
|
||||
return True
|
||||
@@ -188,11 +185,11 @@ class InputBridge:
|
||||
self._picked_up.set()
|
||||
self._response_ready.set()
|
||||
|
||||
@property
|
||||
def answered(self) -> bool:
|
||||
"""``True`` once the bridge has accepted a response from the client."""
|
||||
def ensure_active(self) -> None:
|
||||
"""Raise RuntimeError once the server cancelled the bridge, so a login loop stops."""
|
||||
with self._lock:
|
||||
return self._answered
|
||||
if self._cancelled:
|
||||
raise RuntimeError("Session was cancelled")
|
||||
|
||||
@property
|
||||
def status(self) -> AuthStatus:
|
||||
@@ -203,3 +200,19 @@ class InputBridge:
|
||||
def status(self, value: AuthStatus) -> None:
|
||||
with self._lock:
|
||||
self._status = value
|
||||
|
||||
|
||||
NO_USER_MESSAGE = (
|
||||
"This login needs the user, and this request cannot reach the user. "
|
||||
"Start a download job or a remote session to complete the authentication"
|
||||
)
|
||||
|
||||
|
||||
class NoUserBridge(InputBridge):
|
||||
"""Bridge for a request with no channel to the user: a login step that needs the user fails at once."""
|
||||
|
||||
def request_input(self, prompt: str, timeout: float = AUTH_INPUT_TIMEOUT) -> str:
|
||||
raise RuntimeError(NO_USER_MESSAGE)
|
||||
|
||||
def ensure_active(self) -> None:
|
||||
raise RuntimeError(NO_USER_MESSAGE)
|
||||
|
||||
@@ -388,6 +388,7 @@ async def services(request: web.Request) -> web.Response:
|
||||
if client_config:
|
||||
service_data["client_config"] = list(client_config)
|
||||
service_data["has_search"] = getattr(service_module, "search", None) is not _BaseService.search
|
||||
service_data["anonymous_search"] = bool(getattr(service_module, "ANONYMOUS_SEARCH", False))
|
||||
service_data["has_drm"] = (
|
||||
getattr(service_module, "get_widevine_license", None) is not _BaseService.get_widevine_license
|
||||
or getattr(service_module, "get_playready_license", None) is not _BaseService.get_playready_license
|
||||
@@ -435,7 +436,7 @@ async def search(request: web.Request) -> web.Response:
|
||||
Find titles from a service.
|
||||
---
|
||||
summary: Find titles
|
||||
description: Find titles by query string from a service
|
||||
description: Find titles by query string from a service. A service that reports anonymous_search does the search with no login, and ignores credentials and cookies.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
@@ -725,7 +726,7 @@ async def list_tracks(request: web.Request) -> web.Response:
|
||||
dl_params:
|
||||
type: object
|
||||
description: |
|
||||
The dl track selection (lang, v_lang, a_lang, acodec, forced_subs), in the same
|
||||
The dl track selection (lang, v_lang, a_lang, acodec, forced_subs, exact_lang), in the same
|
||||
form as on `/api/session/create`. The service reads it from `ctx.parent.params`.
|
||||
credentials:
|
||||
type: object
|
||||
@@ -1004,6 +1005,14 @@ async def download(request: web.Request) -> web.Response:
|
||||
no_mux:
|
||||
type: boolean
|
||||
description: Do not mux tracks into a container file (default - false)
|
||||
include_sample:
|
||||
type: boolean
|
||||
description: Also cut a 60 second sample from each muxed file into a Sample folder next to it (default - false)
|
||||
sample:
|
||||
oneOf:
|
||||
- type: boolean
|
||||
- type: string
|
||||
description: Download a short sample and write it to a Sample folder. `true` for 60 seconds from 2 minutes in, or a `"START-END"` string in seconds (e.g., `"0-60"`) (default - null)
|
||||
workers:
|
||||
type: integer
|
||||
description: Max workers/threads per track download (default - None)
|
||||
@@ -1375,7 +1384,8 @@ async def download_job_input(request: web.Request) -> web.Response:
|
||||
description: >
|
||||
Submit the answer to the prompt a running job waits on (an OTP code, a PIN, a device-code
|
||||
confirmation). The job shows the prompt in its `input_prompt` field and in its SSE `progress`
|
||||
events. The job fails if nobody answers within the prompt timeout.
|
||||
events. The job fails if nobody answers within the prompt timeout. A job that waits for an
|
||||
approval on another device shows the message in its `notice` field, which takes no answer.
|
||||
parameters:
|
||||
- name: job_id
|
||||
in: path
|
||||
@@ -1821,6 +1831,9 @@ async def session_create(request: web.Request) -> web.Response:
|
||||
forced_subs:
|
||||
type: boolean
|
||||
default: false
|
||||
exact_lang:
|
||||
type: boolean
|
||||
default: false
|
||||
server_cdm:
|
||||
type: boolean
|
||||
description: |
|
||||
@@ -3081,15 +3094,19 @@ DASHBOARD_ROUTES: list[tuple[str, str, Handler, bool]] = [
|
||||
]
|
||||
|
||||
|
||||
def setup_routes(app: web.Application, remote_only: bool = False, dashboard: bool = False) -> None:
|
||||
def setup_routes(
|
||||
app: web.Application, remote_only: bool = False, dashboard: bool = False, cdm_only: bool = False
|
||||
) -> None:
|
||||
"""Setup API routes. When remote_only=True, only the remote session endpoints operate.
|
||||
When dashboard=True, this also registers the /api/dashboard/ routes."""
|
||||
When cdm_only=True, only /api/health operates. When dashboard=True, this also registers
|
||||
the /api/dashboard/ routes."""
|
||||
add: dict[str, Callable[..., Any]] = {
|
||||
"GET": app.router.add_get,
|
||||
"POST": app.router.add_post,
|
||||
"DELETE": app.router.add_delete,
|
||||
}
|
||||
for method, path, handler, remote in ROUTES + (DASHBOARD_ROUTES if dashboard else []):
|
||||
api_routes = [r for r in ROUTES if r[1] == "/api/health"] if cdm_only else ROUTES
|
||||
for method, path, handler, remote in api_routes + (DASHBOARD_ROUTES if dashboard else []):
|
||||
if remote_only and not remote:
|
||||
continue
|
||||
add[method](path, handler)
|
||||
|
||||
@@ -50,9 +50,7 @@ class SessionEntry:
|
||||
server_cdm_max_height: Optional[int] = None
|
||||
server_vault: bool = False # a session the client's device licenses may still take keys from the server vault
|
||||
server_device: bool = False # its service runs on the server's lent device, so keys come only from a vault
|
||||
client_auth: bool = (
|
||||
False # the client sent its own cookies, credentials, or cache files, or answered a login prompt
|
||||
)
|
||||
client_auth: bool = False # the login is the client's: it succeeded without a server account
|
||||
input_bridge: Optional[InputBridge] = None
|
||||
log_buffer: Optional[Any] = None # SessionLogBuffer mirroring the service's self.log
|
||||
auth_status: AuthStatus = AuthStatus.AUTHENTICATED
|
||||
|
||||
@@ -988,7 +988,7 @@ class CustomRemoteCDM:
|
||||
if "vault_keys" in session:
|
||||
all_keys.extend(session["vault_keys"])
|
||||
|
||||
if "cached_keys" in session:
|
||||
if session.get("cached_keys"):
|
||||
all_keys.extend(session["cached_keys"])
|
||||
|
||||
for license_key in license_keys:
|
||||
|
||||
@@ -193,6 +193,7 @@ class Config:
|
||||
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
||||
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
||||
self.continue_downloads: bool = kwargs.get("continue_downloads", False)
|
||||
self.sample: Optional[tuple[int, int]] = None
|
||||
self.post_scripts: list = kwargs.get("post_scripts") or []
|
||||
|
||||
self.language_tags: dict = kwargs.get("language_tags") or {}
|
||||
|
||||
@@ -490,6 +490,21 @@ def prompt_user(prompt: str) -> str:
|
||||
return console.input(body)
|
||||
|
||||
|
||||
_notice_handler: Optional[Callable[[Optional[str]], None]] = None
|
||||
|
||||
|
||||
def set_notice_handler(handler: Optional[Callable[[Optional[str]], None]]) -> None:
|
||||
"""Route every :func:`show_notice` call to *handler*, for a process whose log output no user reads."""
|
||||
global _notice_handler
|
||||
_notice_handler = handler
|
||||
|
||||
|
||||
def show_notice(message: Optional[str]) -> None:
|
||||
"""Give *message* to the notice handler, or None to clear it. Without a handler this does nothing."""
|
||||
if _notice_handler is not None:
|
||||
_notice_handler(message)
|
||||
|
||||
|
||||
__all__ = (
|
||||
"ComfyLogRenderer",
|
||||
"ComfyRichHandler",
|
||||
@@ -501,5 +516,7 @@ __all__ = (
|
||||
"listing_table",
|
||||
"print_wide",
|
||||
"prompt_user",
|
||||
"set_notice_handler",
|
||||
"set_prompt_handler",
|
||||
"show_notice",
|
||||
)
|
||||
|
||||
@@ -3,6 +3,8 @@ from typing import TypeVar, Union
|
||||
|
||||
DOWNLOAD_CANCELLED = Event()
|
||||
DOWNLOAD_LICENCE_ONLY = Event()
|
||||
|
||||
DEFAULT_SAMPLE = (120, 180)
|
||||
DOWNLOAD_ALL_DRM = Event()
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.utilities import get_boxes, log_event
|
||||
from unshackle.core.utils.subprocess import ffprobe
|
||||
|
||||
@@ -180,7 +181,7 @@ class PlayReady:
|
||||
|
||||
if track.descriptor == track.Descriptor.HLS:
|
||||
m3u_url = track.url
|
||||
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
|
||||
master = m3u8.loads(utf8_text(session.get(m3u_url)), uri=m3u_url)
|
||||
pssh_boxes.extend(
|
||||
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
|
||||
for x in (master.session_keys or master.keys)
|
||||
|
||||
@@ -22,6 +22,7 @@ from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.utilities import get_boxes, log_event
|
||||
from unshackle.core.utils.subprocess import ffprobe
|
||||
|
||||
@@ -94,7 +95,7 @@ class Widevine:
|
||||
|
||||
if track.descriptor == track.Descriptor.HLS:
|
||||
m3u_url = track.url
|
||||
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
|
||||
master = m3u8.loads(utf8_text(session.get(m3u_url)), uri=m3u_url)
|
||||
pssh_boxes.extend(
|
||||
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
|
||||
for x in (master.session_keys or master.keys)
|
||||
|
||||
@@ -18,7 +18,14 @@ from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.credential import Credential
|
||||
from unshackle.core.drm import drm_from_dict, real_kids
|
||||
from unshackle.core.manifests import DASH, HLS, ISM
|
||||
from unshackle.core.remote_service import RemoteService, build_title, match_track, resolve_proxy_arg
|
||||
from unshackle.core.remote_service import (
|
||||
RemoteService,
|
||||
build_title,
|
||||
match_track,
|
||||
resolve_proxy_arg,
|
||||
segment_uri_filter,
|
||||
)
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
|
||||
from unshackle.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video
|
||||
from unshackle.core.tracks.attachment import Attachment
|
||||
@@ -61,9 +68,10 @@ def hls_playlist_kids(track: Any, session: Optional[requests.Session] = None) ->
|
||||
key from its URI, so such a track takes no exported content key. An empty set means that
|
||||
the track is encrypted but names no KID that unshackle can read.
|
||||
"""
|
||||
http = session or requests.Session()
|
||||
try:
|
||||
text = (session or requests.Session()).get(str(track.url)).text
|
||||
keys = [k for k in m3u8.loads(text, uri=str(track.url)).keys if k is not None and k.method != "NONE"]
|
||||
playlist = m3u8.loads(utf8_text(http.get(str(track.url))), uri=str(track.url))
|
||||
keys = [k for k in playlist.keys if k is not None and k.method != "NONE"]
|
||||
except Exception as e:
|
||||
log.warning(f"Cannot read the media playlist of {track.id} for its KIDs: {e!r}")
|
||||
return None
|
||||
@@ -72,7 +80,7 @@ def hls_playlist_kids(track: Any, session: Optional[requests.Session] = None) ->
|
||||
if key.method == "AES-128":
|
||||
continue
|
||||
try:
|
||||
kids.update(real_kids(HLS.get_drm(key)))
|
||||
kids.update(real_kids(HLS.get_drm_or_probe(key, playlist, track, probe_session=http)))
|
||||
except Exception as e:
|
||||
log.debug(f"Skipping HLS key {key.keyformat or key.method} of {track.id}: {e!r}")
|
||||
if kids or any(k.method != "AES-128" for k in keys):
|
||||
@@ -409,6 +417,10 @@ class ImportService:
|
||||
drm = self.rebuild_drm(track_dict, title_id)
|
||||
if drm:
|
||||
track.drm = drm
|
||||
if track_dict.get("unwanted_segments"):
|
||||
track.OnSegmentFilter = segment_uri_filter(track_dict["unwanted_segments"])
|
||||
# HLS.download_track warns when none of these is in the playlist
|
||||
track.data["hls"]["unwanted_segments"] = list(track_dict["unwanted_segments"])
|
||||
tracks.add(track, warn_only=True)
|
||||
|
||||
if manifest_url and parser is None and not [*tracks.videos, *tracks.audio, *tracks.subtitles]:
|
||||
|
||||
@@ -19,7 +19,7 @@ from zlib import crc32
|
||||
|
||||
import requests
|
||||
from langcodes import Language, tag_is_valid
|
||||
from lxml.etree import Element, ElementTree
|
||||
from lxml.etree import Element, ElementTree, XMLSyntaxError
|
||||
from pyplayready.system.pssh import PSSH as PR_PSSH
|
||||
from pywidevine.cdm import Cdm as WidevineCdm
|
||||
from pywidevine.pssh import PSSH
|
||||
@@ -34,7 +34,11 @@ from unshackle.core.drm.verify import decrypt_track
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.session import RnetSession
|
||||
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
||||
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
|
||||
from unshackle.core.tracks.track import (
|
||||
DRM_PREFERENCE_TYPES,
|
||||
assert_fragments_decrypted,
|
||||
parse_sidx,
|
||||
)
|
||||
from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8
|
||||
from unshackle.core.utils.redact import safe_display_url
|
||||
from unshackle.core.utils.xml import load_xml
|
||||
@@ -114,6 +118,7 @@ class DASH:
|
||||
|
||||
@classmethod
|
||||
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH:
|
||||
"""Fetch and parse an MPD. The XML declaration gives the encoding, then the HTTP charset."""
|
||||
if not url:
|
||||
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
|
||||
if not isinstance(url, str):
|
||||
@@ -134,11 +139,15 @@ class DASH:
|
||||
log_event(
|
||||
"manifest_dash_fetch",
|
||||
level="DEBUG",
|
||||
message=f"Fetched DASH manifest ({len(res.text)} bytes)",
|
||||
context={"url": safe_display_url(url), "size": len(res.text)},
|
||||
message=f"Fetched DASH manifest ({len(res.content)} bytes)",
|
||||
context={"url": safe_display_url(url), "size": len(res.content)},
|
||||
)
|
||||
|
||||
return DASH.from_text(res.text, url)
|
||||
try:
|
||||
return cls(load_xml(res.content), url)
|
||||
except XMLSyntaxError:
|
||||
# no declaration and not UTF-8: only the HTTP charset can name the encoding
|
||||
return cls.from_text(res.text, url)
|
||||
|
||||
@classmethod
|
||||
def from_text(cls, text: str, url: str) -> DASH:
|
||||
@@ -458,6 +467,7 @@ class DASH:
|
||||
track_url=track.url,
|
||||
session=session,
|
||||
probe_kid=period_idx == 0,
|
||||
window=config.sample,
|
||||
)
|
||||
|
||||
if period_idx == 0:
|
||||
@@ -471,12 +481,20 @@ class DASH:
|
||||
seen_segments.add(seg)
|
||||
segments.append(seg)
|
||||
segment_durations.extend(p_durations)
|
||||
# ponytail: a sample comes from the first content period only; map the window across
|
||||
# periods if a manifest with a short first period needs it
|
||||
if config.sample and segments:
|
||||
break
|
||||
|
||||
if not segments:
|
||||
log.error("Could not find a way to get segments from this MPD manifest.")
|
||||
log.debug(track.url)
|
||||
sys.exit(1)
|
||||
|
||||
if config.sample and not segment_durations and not isinstance(track, Subtitle):
|
||||
log.error("--sample cannot cut this track: the manifest gives it as one file with no segment index.")
|
||||
sys.exit(1)
|
||||
|
||||
# TODO: Should we floor/ceil/round, or is int() ok?
|
||||
track.data["dash"]["timescale"] = int(segment_timescale)
|
||||
track.data["dash"]["segment_durations"] = segment_durations
|
||||
@@ -533,7 +551,7 @@ class DASH:
|
||||
# When every segment is a byte range of one parent resource and init is its [0, len)
|
||||
# prefix, the whole resource is itself a valid MP4, so one direct download to save_path
|
||||
# replaces the per-segment files, init write and merge pass.
|
||||
collapse_single_url = DASH.collapsible_single_url(
|
||||
collapse_single_url = not config.sample and DASH.collapsible_single_url(
|
||||
isinstance(track, Subtitle),
|
||||
segments,
|
||||
len(init_data) if init_data is not None else None,
|
||||
@@ -753,6 +771,26 @@ class DASH:
|
||||
|
||||
progress(downloaded="Downloaded")
|
||||
|
||||
@staticmethod
|
||||
def find_segment_element(name: str, adaptation_set: Element, representation: Element) -> Optional[Element]:
|
||||
"""Find a segment description element on the representation, or else on its adaptation set."""
|
||||
element = representation.find(name)
|
||||
return element if element is not None else adaptation_set.find(name)
|
||||
|
||||
@staticmethod
|
||||
def has_segment_timing(track: AnyTrack) -> bool:
|
||||
"""Return False when the manifest gives the track as one file with no index, so a sample cannot cut it."""
|
||||
dash = track.data.get("dash") or {}
|
||||
representation, adaptation_set = dash.get("representation"), dash.get("adaptation_set")
|
||||
if representation is None or adaptation_set is None:
|
||||
return True
|
||||
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
||||
return (
|
||||
DASH.merge_segment_templates(adaptation_set, representation) is not None
|
||||
or DASH.find_segment_element("SegmentList", adaptation_set, representation) is not None
|
||||
or (segment_base is not None and bool(segment_base.get("indexRange")))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def collapsible_single_url(
|
||||
is_subtitle: bool,
|
||||
@@ -844,6 +882,7 @@ class DASH:
|
||||
track_url: str,
|
||||
session: Union[Session, RnetSession],
|
||||
probe_kid: bool = True,
|
||||
window: Optional[tuple[int, int]] = None,
|
||||
) -> tuple[
|
||||
Optional[bytes],
|
||||
list[tuple[str, Optional[str]]],
|
||||
@@ -877,13 +916,9 @@ class DASH:
|
||||
|
||||
segment_template = DASH.merge_segment_templates(adaptation_set, representation)
|
||||
|
||||
segment_list = representation.find("SegmentList")
|
||||
if segment_list is None:
|
||||
segment_list = adaptation_set.find("SegmentList")
|
||||
segment_list = DASH.find_segment_element("SegmentList", adaptation_set, representation)
|
||||
|
||||
segment_base = representation.find("SegmentBase")
|
||||
if segment_base is None:
|
||||
segment_base = adaptation_set.find("SegmentBase")
|
||||
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
||||
|
||||
segments: list[tuple[str, Optional[str]]] = []
|
||||
segment_timescale: float = 0
|
||||
@@ -930,6 +965,10 @@ class DASH:
|
||||
segment_durations.append(current_time)
|
||||
current_time += int(s.get("d"))
|
||||
|
||||
keep = track.sample_slice(segment_durations, segment_timescale, window) if window else slice(None)
|
||||
start_number += keep.start or 0
|
||||
segment_durations = segment_durations[keep]
|
||||
|
||||
if not end_number:
|
||||
end_number = start_number + len(segment_durations) - 1
|
||||
|
||||
@@ -956,7 +995,11 @@ class DASH:
|
||||
segment_count = math.ceil(period_duration / (segment_duration / segment_timescale))
|
||||
end_number = start_number + segment_count - 1
|
||||
|
||||
for s in range(start_number, end_number + 1):
|
||||
numbers = range(start_number, end_number + 1)
|
||||
if window:
|
||||
starts = [n * segment_duration for n in range(len(numbers))]
|
||||
numbers = numbers[track.sample_slice(starts, segment_timescale, window)]
|
||||
for s in numbers:
|
||||
segments.append(
|
||||
(
|
||||
DASH.replace_fields(
|
||||
@@ -994,6 +1037,10 @@ class DASH:
|
||||
track_kid = track.get_key_id(init_data)
|
||||
|
||||
segment_urls = segment_list.findall("SegmentURL")
|
||||
list_duration = float(segment_list.get("duration") or 0)
|
||||
if window and list_duration:
|
||||
starts = [n * list_duration for n in range(len(segment_urls))]
|
||||
segment_urls = segment_urls[track.sample_slice(starts, segment_timescale, window)]
|
||||
for segment_url in segment_urls:
|
||||
media_url = segment_url.get("media")
|
||||
if not media_url:
|
||||
@@ -1022,16 +1069,25 @@ class DASH:
|
||||
if total_size:
|
||||
media_range = f"{len(init_data)}-{total_size}"
|
||||
|
||||
segments.append((rep_base_url, media_range))
|
||||
subsegments: list[tuple[int, int, int]] = []
|
||||
index_range = segment_base.get("indexRange")
|
||||
if window and index_range and not isinstance(track, Subtitle):
|
||||
res = session.get(url=rep_base_url, headers={"Range": f"bytes={index_range}"})
|
||||
res.raise_for_status()
|
||||
segment_timescale, subsegments = parse_sidx(res.content, int(index_range.split("-")[0]))
|
||||
|
||||
if subsegments:
|
||||
segment_durations = [time for time, _, _ in subsegments]
|
||||
keep = track.sample_slice(segment_durations, segment_timescale, window)
|
||||
segment_durations = segment_durations[keep]
|
||||
segments.extend((rep_base_url, f"{first}-{last}") for _, first, last in subsegments[keep])
|
||||
else:
|
||||
segments.append((rep_base_url, media_range))
|
||||
elif rep_base_url:
|
||||
segments.append((rep_base_url, None))
|
||||
|
||||
return init_data, segments, segment_timescale, segment_durations, track_kid
|
||||
|
||||
# Deprecated 5.5.0 shims for service repos still on the old underscored names; drop once they migrate.
|
||||
_is_content_period = is_content_period
|
||||
_get_period_segments = get_period_segments
|
||||
|
||||
@staticmethod
|
||||
def get_attr(item: str, adaptation_set: Element, representation: Optional[Element] = None) -> Optional[Any]:
|
||||
"""Helper to get a requested item from the Representation, otherwise from the AdaptationSet."""
|
||||
|
||||
+112
-66
@@ -11,6 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
from contextlib import ExitStack, closing
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
@@ -36,7 +37,7 @@ from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
|
||||
from unshackle.core.drm.verify import decrypt_track
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.manifests.dash import RollingMerge
|
||||
from unshackle.core.session import RnetResponse, RnetSession
|
||||
from unshackle.core.session import RnetResponse, RnetSession, utf8_text
|
||||
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
||||
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
|
||||
from unshackle.core.utilities import get_extension, is_close_match, log_event, try_ensure_utf8
|
||||
@@ -80,17 +81,11 @@ class HLS:
|
||||
|
||||
res = session.get(url, **args)
|
||||
|
||||
if isinstance(res, requests.Response):
|
||||
if not res.ok:
|
||||
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
|
||||
res.encoding = res.encoding or "utf-8"
|
||||
content = res.text
|
||||
elif isinstance(res, RnetResponse):
|
||||
if not res.ok:
|
||||
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
|
||||
content = res.text
|
||||
else:
|
||||
if not isinstance(res, (requests.Response, RnetResponse)):
|
||||
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}")
|
||||
if not res.ok:
|
||||
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
|
||||
content = utf8_text(res)
|
||||
|
||||
master = m3u8.loads(content, uri=url)
|
||||
|
||||
@@ -396,9 +391,7 @@ class HLS:
|
||||
session = Session()
|
||||
|
||||
res = session.get(variant_url)
|
||||
if isinstance(res, requests.Response):
|
||||
res.encoding = res.encoding or "utf-8"
|
||||
variant = m3u8.loads(res.text, uri=variant_url)
|
||||
variant = m3u8.loads(utf8_text(res), uri=variant_url)
|
||||
if not variant.segments:
|
||||
return None
|
||||
|
||||
@@ -658,22 +651,35 @@ class HLS:
|
||||
master = m3u8.load(str(track.from_file))
|
||||
else:
|
||||
response = session.get(track.url)
|
||||
if isinstance(response, requests.Response) or isinstance(response, RnetResponse):
|
||||
if not response.ok:
|
||||
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
|
||||
sys.exit(1)
|
||||
if isinstance(response, requests.Response):
|
||||
response.encoding = response.encoding or "utf-8"
|
||||
playlist_text = response.text
|
||||
else:
|
||||
if not isinstance(response, (requests.Response, RnetResponse)):
|
||||
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}")
|
||||
if not response.ok:
|
||||
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
|
||||
sys.exit(1)
|
||||
|
||||
master = m3u8.loads(playlist_text, uri=track.url)
|
||||
master = m3u8.loads(utf8_text(response), uri=track.url)
|
||||
|
||||
if not master.segments:
|
||||
log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.")
|
||||
sys.exit(1)
|
||||
|
||||
unwanted_segments = [
|
||||
segment for segment in master.segments if callable(track.OnSegmentFilter) and track.OnSegmentFilter(segment)
|
||||
]
|
||||
hls_data = track.data.setdefault("hls", {})
|
||||
if hls_data.get("unwanted_segments") and not unwanted_segments:
|
||||
log.warning(
|
||||
f"None of the {len(hls_data['unwanted_segments'])} segments that the export drops is in the "
|
||||
f"playlist of {track.id}. The download keeps every segment."
|
||||
)
|
||||
hls_data["unwanted_segments"] = [s.absolute_uri for s in unwanted_segments]
|
||||
|
||||
if config.sample:
|
||||
content = [segment for segment in master.segments if segment not in unwanted_segments]
|
||||
starts = list(accumulate((segment.duration or 0 for segment in content[:-1]), initial=0.0))
|
||||
del content[track.sample_slice(starts, window=config.sample)]
|
||||
unwanted_segments += content
|
||||
|
||||
# Get session DRM as fallback but prefer media playlist keys for accurate KID matching
|
||||
if track.drm:
|
||||
session_drm = track.get_drm_for_cdm(cdm)
|
||||
@@ -688,9 +694,14 @@ class HLS:
|
||||
media_playlist_key = HLS.get_supported_key(cdm_media_keys) if cdm_media_keys else None
|
||||
|
||||
if media_playlist_key:
|
||||
media_drm = HLS.get_drm(media_playlist_key, session)
|
||||
init_kid = (
|
||||
None
|
||||
if media_playlist_key.method == "AES-128"
|
||||
else HLS.get_track_kid_from_init(master, track, session)
|
||||
)
|
||||
media_drm = HLS.get_drm(media_playlist_key, session, **({"kid": init_kid} if init_kid else {}))
|
||||
if isinstance(media_drm, (Widevine, PlayReady)):
|
||||
track_kid = HLS.get_track_kid_from_init(master, track, session) or media_drm.kid
|
||||
track_kid = init_kid or media_drm.kid
|
||||
# Preserve pre-existing keys (e.g. from server_cdm)
|
||||
if track.drm:
|
||||
for existing_drm in track.drm:
|
||||
@@ -698,7 +709,7 @@ class HLS:
|
||||
media_drm.content_keys.update(existing_drm.content_keys)
|
||||
track.drm = [media_drm]
|
||||
if DOWNLOAD_ALL_DRM.is_set():
|
||||
track.drm += HLS.alternate_drm(media_keys, media_drm, session)
|
||||
track.drm += HLS.alternate_drm(media_keys, media_drm, session, init_kid)
|
||||
try:
|
||||
if not license_widevine:
|
||||
raise ValueError("license_widevine func must be supplied to use DRM")
|
||||
@@ -745,13 +756,21 @@ class HLS:
|
||||
raise
|
||||
|
||||
if DOWNLOAD_LICENCE_ONLY.is_set():
|
||||
# a playlist can change its key part-way, and only the download loop licenses the later keys
|
||||
licensed_keys = [initial_drm_key]
|
||||
for segment in master.segments:
|
||||
if not segment.keys or segment in unwanted_segments:
|
||||
continue
|
||||
key = HLS.resolve_segment_key(segment.keys, cdm, track.drm_preference)
|
||||
if key is None or key.method == "AES-128" or key in licensed_keys:
|
||||
continue
|
||||
licensed_keys.append(key)
|
||||
later_drm = HLS.get_drm_or_probe(key, master, track, session, segment.init_section)
|
||||
if isinstance(later_drm, (Widevine, PlayReady)) and license_widevine:
|
||||
license_widevine(later_drm, track_kid=later_drm.kid)
|
||||
progress(downloaded="[yellow]SKIPPED")
|
||||
return
|
||||
|
||||
unwanted_segments = [
|
||||
segment for segment in master.segments if callable(track.OnSegmentFilter) and track.OnSegmentFilter(segment)
|
||||
]
|
||||
|
||||
# Downloaded segment files are named by post-filter index; map that back to the wanted
|
||||
# segment so the IV uses the absolute media sequence number, not the download index.
|
||||
wanted_segments = [seg for seg in master.segments if seg not in unwanted_segments]
|
||||
@@ -777,17 +796,17 @@ class HLS:
|
||||
|
||||
range_offset = 0
|
||||
for segment in master.segments:
|
||||
if segment.byterange:
|
||||
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
|
||||
range_offset = int(byte_range.split("-")[1]) + 1
|
||||
else:
|
||||
byte_range = None
|
||||
|
||||
if segment in unwanted_segments:
|
||||
continue
|
||||
|
||||
segment_durations.append(int(segment.duration))
|
||||
|
||||
if segment.byterange:
|
||||
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
|
||||
range_offset = int(byte_range.split("-")[0])
|
||||
else:
|
||||
byte_range = None
|
||||
|
||||
urls.append(
|
||||
{
|
||||
"url": urljoin(segment.base_uri, segment.uri),
|
||||
@@ -1126,22 +1145,18 @@ class HLS:
|
||||
if key is None:
|
||||
encryption_data = None
|
||||
elif not encryption_data or encryption_data[0] != key:
|
||||
drm = HLS.get_drm(key, session)
|
||||
if isinstance(drm, (Widevine, PlayReady)):
|
||||
try:
|
||||
if map_data:
|
||||
track_kid = track.get_key_id(map_data[1])
|
||||
else:
|
||||
track_kid = None
|
||||
if not track_kid:
|
||||
track_kid = drm.kid
|
||||
try:
|
||||
init_kid = track.get_key_id(map_data[1]) if map_data and key.method != "AES-128" else None
|
||||
drm = HLS.get_drm(key, session, **({"kid": init_kid} if init_kid else {}))
|
||||
if isinstance(drm, (Widevine, PlayReady)):
|
||||
track_kid = init_kid or drm.kid
|
||||
progress(downloaded="LICENSING")
|
||||
license_widevine(drm, track_kid=track_kid)
|
||||
progress(downloaded="[yellow]LICENSED")
|
||||
except Exception: # noqa
|
||||
DOWNLOAD_CANCELLED.set() # skip pending track downloads
|
||||
progress(downloaded="[red]FAILED")
|
||||
raise
|
||||
except Exception: # noqa
|
||||
DOWNLOAD_CANCELLED.set() # skip pending track downloads
|
||||
progress(downloaded="[red]FAILED")
|
||||
raise
|
||||
if (
|
||||
encryption_data
|
||||
and isinstance(drm, (Widevine, PlayReady))
|
||||
@@ -1152,9 +1167,6 @@ class HLS:
|
||||
drm.content_keys.setdefault(prev_kid, prev_key)
|
||||
encryption_data = (key, drm)
|
||||
|
||||
if DOWNLOAD_LICENCE_ONLY.is_set():
|
||||
continue
|
||||
|
||||
if is_last_segment:
|
||||
# required as it won't end with EXT-X-DISCONTINUITY nor a new key
|
||||
if encryption_data:
|
||||
@@ -1165,9 +1177,6 @@ class HLS:
|
||||
|
||||
progress(advance=1)
|
||||
|
||||
if DOWNLOAD_LICENCE_ONLY.is_set():
|
||||
return
|
||||
|
||||
def find_segments_recursively(directory: Path) -> list[Path]:
|
||||
"""Find all segment files recursively in any directory structure created by downloaders."""
|
||||
segments = []
|
||||
@@ -1364,8 +1373,7 @@ class HLS:
|
||||
if not value and data.uri:
|
||||
if not session:
|
||||
session = Session()
|
||||
res = session.get(urljoin(manifest.base_uri or "", data.uri))
|
||||
value = res.text
|
||||
value = utf8_text(session.get(urljoin(manifest.base_uri or "", data.uri)))
|
||||
|
||||
if not value:
|
||||
continue
|
||||
@@ -1442,6 +1450,7 @@ class HLS:
|
||||
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]],
|
||||
chosen: DRM_T,
|
||||
session: Optional[Union[Session, RnetSession]] = None,
|
||||
kid: Optional[UUID] = None,
|
||||
) -> list[DRM_T]:
|
||||
"""Build the Widevine or PlayReady DRM that the EXT-X-KEY entries offer next to ``chosen``.
|
||||
|
||||
@@ -1452,7 +1461,7 @@ class HLS:
|
||||
if not other_keys:
|
||||
return []
|
||||
try:
|
||||
return [HLS.get_drm(other_keys[0], session)]
|
||||
return [HLS.get_drm(other_keys[0], session, **({"kid": kid} if kid else {}))]
|
||||
except Exception as e:
|
||||
logging.getLogger("HLS").warning(f"Could not read the {system} EXT-X-KEY entry: {e}")
|
||||
return []
|
||||
@@ -1462,14 +1471,15 @@ class HLS:
|
||||
master: M3U8,
|
||||
track: AnyTrack,
|
||||
session: Union[Session, RnetSession],
|
||||
init_section: Optional[m3u8.model.InitializationSection] = None,
|
||||
) -> Optional[UUID]:
|
||||
"""
|
||||
Extract the track's Key ID from its init segment (EXT-X-MAP).
|
||||
Extract the track's Key ID from an init segment (EXT-X-MAP), by default the first in the playlist.
|
||||
|
||||
Returns None if no init segment exists or KID extraction fails.
|
||||
The caller should fall back to drm.kid from the PSSH if this returns None.
|
||||
"""
|
||||
map_section = next((seg.init_section for seg in master.segments if seg.init_section), None)
|
||||
map_section = init_section or next((seg.init_section for seg in master.segments if seg.init_section), None)
|
||||
if not map_section:
|
||||
return None
|
||||
|
||||
@@ -1483,6 +1493,10 @@ class HLS:
|
||||
map_res = session.get(url=map_uri, headers=headers)
|
||||
if map_res.ok:
|
||||
return track.get_key_id(map_res.content)
|
||||
logging.getLogger("HLS").warning(
|
||||
f"Could not read the Key ID from the init segment {safe_display_url(map_uri)}: "
|
||||
f"HTTP {map_res.status_code}"
|
||||
)
|
||||
# best-effort key-id probe over network + init parse; caller falls back without it
|
||||
except Exception as e:
|
||||
logging.getLogger("HLS").debug(f"Init map key-id probe failed for {map_uri}: {e!r}")
|
||||
@@ -1530,6 +1544,7 @@ class HLS:
|
||||
def get_drm(
|
||||
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
|
||||
session: Optional[Union[Session, RnetSession]] = None,
|
||||
kid: Optional[UUID] = None,
|
||||
) -> DRM_T:
|
||||
"""
|
||||
Convert HLS EXT-X-KEY data to an initialized DRM object.
|
||||
@@ -1538,6 +1553,8 @@ class HLS:
|
||||
key: The m3u8 `EXT-X-KEY` object.
|
||||
session: Optional HTTP session used to request AES-128 URIs.
|
||||
Useful to set headers, proxies, cookies, and so forth.
|
||||
kid: The Key ID from the init segment. Widevine uses it only when the tag
|
||||
and the PSSH both carry no real Key ID.
|
||||
|
||||
Raises a NotImplementedError when unshackle cannot use the DRM system.
|
||||
"""
|
||||
@@ -1553,13 +1570,16 @@ class HLS:
|
||||
drm = Widevine(pssh=WV_PSSH.new(key_ids=[key.uri.split(",")[-1]], system_id=WV_PSSH.SystemId.Widevine))
|
||||
elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn:
|
||||
extra_params = dict(key._extra_params) # noqa
|
||||
# a v0 PSSH carries no KID, leaving the tag's own KEYID as the only source
|
||||
kid = extra_params.pop("keyid", None) or extra_params.pop("kid", None)
|
||||
if isinstance(kid, str):
|
||||
kid = kid.removeprefix("0x").removeprefix("0X")
|
||||
pssh = WV_PSSH(key.uri.split(",")[-1])
|
||||
if kid in Widevine.PLACEHOLDER_KIDS or any(k not in Widevine.PLACEHOLDER_KIDS for k in pssh.key_ids):
|
||||
kid = None
|
||||
# a v0 PSSH can carry no KID: the tag's own KEYID is the first source, the init segment the last
|
||||
tag_kid = extra_params.pop("keyid", None) or extra_params.pop("kid", None)
|
||||
if isinstance(tag_kid, str):
|
||||
tag_kid = tag_kid.removeprefix("0x").removeprefix("0X")
|
||||
drm = Widevine(
|
||||
pssh=WV_PSSH(key.uri.split(",")[-1]),
|
||||
kid=kid,
|
||||
pssh=pssh,
|
||||
kid=tag_kid or kid,
|
||||
**extra_params,
|
||||
)
|
||||
elif key.keyformat and key.keyformat.lower() in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}:
|
||||
@@ -1572,6 +1592,28 @@ class HLS:
|
||||
|
||||
return drm
|
||||
|
||||
@staticmethod
|
||||
def get_drm_or_probe(
|
||||
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
|
||||
master: M3U8,
|
||||
track: AnyTrack,
|
||||
session: Optional[Union[Session, RnetSession]] = None,
|
||||
init_section: Optional[m3u8.model.InitializationSection] = None,
|
||||
probe_session: Any = None,
|
||||
) -> DRM_T:
|
||||
"""
|
||||
Convert HLS EXT-X-KEY data to a DRM object, with the Key ID of the init segment when the key carries none.
|
||||
|
||||
The init segment request uses `probe_session` when the caller has no HTTP session that `get_drm` accepts.
|
||||
"""
|
||||
try:
|
||||
return HLS.get_drm(key, session)
|
||||
except Widevine.Exceptions.KIDNotFound:
|
||||
kid = HLS.get_track_kid_from_init(master, track, probe_session or session or Session(), init_section)
|
||||
if not kid:
|
||||
raise
|
||||
return HLS.get_drm(key, session, kid=kid)
|
||||
|
||||
@staticmethod
|
||||
def get_all_drm(
|
||||
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], proxy: Optional[str] = None
|
||||
@@ -1597,6 +1639,10 @@ class HLS:
|
||||
drm_objects.append(drm)
|
||||
except NotImplementedError:
|
||||
unsupported_keys.append(key)
|
||||
except Widevine.Exceptions.KIDNotFound:
|
||||
logging.getLogger("HLS").warning(
|
||||
"Skipped an EXT-X-SESSION-KEY entry that carries no Key ID. The media playlist must supply the key."
|
||||
)
|
||||
|
||||
if not drm_objects and unsupported_keys:
|
||||
logging.debug(
|
||||
|
||||
@@ -259,6 +259,7 @@ class ISM:
|
||||
)
|
||||
|
||||
track_urls: list[str] = []
|
||||
track_times: list[int] = []
|
||||
fragment_time = 0
|
||||
fragments = stream_index.findall("c")
|
||||
# MS-SSTR UrlPattern; regex over str.format so {Bitrate}/{start_time}
|
||||
@@ -276,6 +277,7 @@ class ISM:
|
||||
first_time = int(fragments[0].get("t") or 0)
|
||||
if first_time != 0:
|
||||
track_urls.append(re.sub(r"\{start[ _]time\}", "0", url_template))
|
||||
track_times.append(first_time)
|
||||
|
||||
for idx, frag in enumerate(fragments):
|
||||
fragment_time = int(frag.get("t", fragment_time))
|
||||
@@ -292,6 +294,7 @@ class ISM:
|
||||
duration_frag = (next_time - fragment_time) // repeat
|
||||
for _ in range(repeat):
|
||||
track_urls.append(re.sub(r"\{start[ _]time\}", str(fragment_time), url_template))
|
||||
track_times.append(fragment_time)
|
||||
fragment_time += duration_frag
|
||||
|
||||
track_id = hashlib.md5(
|
||||
@@ -312,6 +315,8 @@ class ISM:
|
||||
"stream_index": stream_index,
|
||||
"quality_level": ql,
|
||||
"segments": track_urls,
|
||||
"segment_times": track_times,
|
||||
"timescale": int(stream_index.get("TimeScale") or self.manifest.get("TimeScale") or 10000000),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +419,9 @@ class ISM:
|
||||
session.proxies.update({"all": proxy})
|
||||
|
||||
segments: list[str] = track.data["ism"]["segments"]
|
||||
segment_times: list[int] = track.data["ism"].get("segment_times") or []
|
||||
if segment_times:
|
||||
segments = segments[track.sample_slice(segment_times, track.data["ism"]["timescale"], config.sample)]
|
||||
|
||||
session_drm = None
|
||||
if track.drm:
|
||||
|
||||
@@ -22,7 +22,7 @@ from enum import Enum
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any, Callable, Dict, Iterator, Optional, Union
|
||||
from typing import Any, Callable, Dict, Iterable, Iterator, Optional, Union
|
||||
from uuid import UUID
|
||||
|
||||
import click
|
||||
@@ -465,6 +465,22 @@ def resolve_manifest_data(tracks: Tracks, manifests: list) -> None:
|
||||
remote_track.drm = matched.drm
|
||||
|
||||
|
||||
def segment_uri_filter(uris: Iterable[Any]) -> Callable[[Any], bool]:
|
||||
"""Return an HLS ``OnSegmentFilter`` that drops the segments with these URIs.
|
||||
|
||||
A URI also matches without its query string, because a later fetch of the playlist can
|
||||
sign the same segment with a new one.
|
||||
"""
|
||||
exact = {str(u) for u in uris}
|
||||
paths = {u.split("?")[0] for u in exact}
|
||||
|
||||
def segment_filter(segment: Any) -> bool:
|
||||
uri = str(getattr(segment, "absolute_uri", "") or "")
|
||||
return bool(exact) and (uri in exact or uri.split("?")[0] in paths)
|
||||
|
||||
return segment_filter
|
||||
|
||||
|
||||
def same_bitrate(local: Optional[int], remote: Optional[int]) -> bool:
|
||||
"""Equal to the nearest kb/s. A server that predates ``bitrate_bps`` rounds to kb/s in transit."""
|
||||
if not local or not remote:
|
||||
@@ -895,7 +911,7 @@ class RemoteService:
|
||||
self.client_licensed: set[str] = set() # track ids this machine's device licenses by challenge relay
|
||||
self.server_vault_keys: Dict[UUID, str] = {}
|
||||
self.server_vault = ServerVault(self)
|
||||
self._segment_filters: Dict[str, tuple[set[str], set[str]]] = {}
|
||||
self._segment_filters: Dict[str, Callable[[Any], bool]] = {}
|
||||
self._log_seq = 0
|
||||
self._log_drain_lock = Lock()
|
||||
self._keepalive_stop = Event()
|
||||
@@ -1020,6 +1036,9 @@ class RemoteService:
|
||||
if create_data.get("proxy") and is_loopback(create_data["proxy"]) and not is_loopback(self.client.server_url):
|
||||
# the local CLI resolves the proxy, so a local-only URI is useless to a server elsewhere
|
||||
raise click.ClickException("That proxy is on your machine, so --remote cannot use it.")
|
||||
local_proxy = None if no_proxy else (proxy or create_data.get("proxy"))
|
||||
if local_proxy:
|
||||
self._session.proxies["all"] = local_proxy
|
||||
|
||||
if not no_proxy and not proxy and client_region:
|
||||
create_data["client_region"] = client_region
|
||||
@@ -1054,8 +1073,9 @@ class RemoteService:
|
||||
}
|
||||
if params.get("acodec") is not None:
|
||||
dl_params["acodec"] = [c.name for c in params["acodec"]]
|
||||
if "forced_subs" in params:
|
||||
dl_params["forced_subs"] = bool(params["forced_subs"])
|
||||
for key in ("forced_subs", "exact_lang"):
|
||||
if key in params:
|
||||
dl_params[key] = bool(params[key])
|
||||
if dl_params:
|
||||
create_data["dl_params"] = dl_params
|
||||
|
||||
@@ -1404,14 +1424,9 @@ class RemoteService:
|
||||
result = self.client.post_optional(
|
||||
f"/api/session/{self._session_id}/segment_filter", {"track_id": track_id}
|
||||
)
|
||||
uris = [str(u) for u in (result.get("unwanted") or [])]
|
||||
unwanted = (set(uris), {u.split("?")[0] for u in uris})
|
||||
unwanted = segment_uri_filter(result.get("unwanted") or [])
|
||||
self._segment_filters[track_id] = unwanted
|
||||
exact, paths = unwanted
|
||||
if not exact:
|
||||
return False
|
||||
uri = str(getattr(segment, "absolute_uri", "") or "")
|
||||
return uri in exact or uri.split("?")[0] in paths
|
||||
return unwanted(segment)
|
||||
|
||||
return segment_filter
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import logging
|
||||
import time
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from collections.abc import Callable, Generator
|
||||
from dataclasses import dataclass, field
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from unshackle.core.api.input_bridge import InputBridge
|
||||
@@ -18,7 +19,7 @@ from rich.rule import Rule
|
||||
|
||||
from unshackle.core.cacher import Cacher
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.console import console, prompt_user
|
||||
from unshackle.core.console import console, prompt_user, show_notice
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.credential import Credential
|
||||
from unshackle.core.drm import DRM_T
|
||||
@@ -140,6 +141,9 @@ def sanitize_proxy_for_log(uri: Optional[str], mask_host: bool = False) -> Optio
|
||||
return mask_proxy(uri, mask_host=mask_host, allow_debug=False)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Service(metaclass=ABCMeta):
|
||||
"""The Service Base Class.
|
||||
|
||||
@@ -154,6 +158,7 @@ class Service(metaclass=ABCMeta):
|
||||
GEOBLOCK: tuple[str, ...] = () # ip regions where the service refuses to work; everything else is allowed.
|
||||
ANIME: bool = False # service catalogue is anime; metadata lookups prefer AniList. Title.anime overrides per title.
|
||||
DAILY: bool = False # catalog is daily/date-based. episodes are named by air date. Title.daily overrides per title.
|
||||
ANONYMOUS_SEARCH: bool = False # search() needs no login; a search then skips cookies and authenticate().
|
||||
# vault namespace override; when set, key vault read/write uses this tag instead of the service's own.
|
||||
VAULT_TAG: Optional[str] = None
|
||||
# Auth methods the service accepts ("cookies"/"credentials"); when None the REST /services
|
||||
@@ -401,9 +406,6 @@ class Service(metaclass=ABCMeta):
|
||||
|
||||
return all_tracks
|
||||
|
||||
# Deprecated 5.5.0 shim for service repos still on the old underscored name; drop once they migrate.
|
||||
_get_tracks_for_variants = get_tracks_for_variants
|
||||
|
||||
@staticmethod
|
||||
def get_session() -> requests.Session:
|
||||
"""
|
||||
@@ -452,7 +454,8 @@ class Service(metaclass=ABCMeta):
|
||||
|
||||
This is effectively a login() function. Any API calls or object initializations
|
||||
needing to be made, should be made here. unshackle operates this method before
|
||||
any of the following abstract functions.
|
||||
any of the following abstract functions. The one exception is a search on a
|
||||
Service that sets ``ANONYMOUS_SEARCH``, which does not operate this method.
|
||||
|
||||
You should avoid storing or using the Credential outside this function.
|
||||
Make any calls you need for any Cookies, Tokens, or such, then use those.
|
||||
@@ -479,6 +482,32 @@ class Service(metaclass=ABCMeta):
|
||||
return self._input_bridge.request_input(prompt)
|
||||
return prompt_user(prompt)
|
||||
|
||||
def wait_for_approval(
|
||||
self, message: str, check: Callable[[], Optional[T]], timeout: float, interval: float = 3.0
|
||||
) -> T:
|
||||
"""Show ``message``, then call ``check`` until it returns a value.
|
||||
|
||||
Use it for a login that the user approves on another device, such as a pair code.
|
||||
``check`` returns None while the approval is pending, and must return within a few
|
||||
seconds. Raises TimeoutError after ``timeout`` seconds, and RuntimeError when the
|
||||
remote session no longer exists. A REST download job shows ``message`` as its ``notice``.
|
||||
"""
|
||||
self.log.info(message)
|
||||
show_notice(message)
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
while True:
|
||||
if self._input_bridge is not None:
|
||||
self._input_bridge.ensure_active()
|
||||
result = check()
|
||||
if result is not None:
|
||||
return result
|
||||
if time.monotonic() + interval >= deadline:
|
||||
raise TimeoutError(f"No approval within {timeout:.0f}s")
|
||||
time.sleep(interval)
|
||||
finally:
|
||||
show_notice(None)
|
||||
|
||||
def search(self) -> Generator[SearchResult, None, None]:
|
||||
"""
|
||||
Find titles from the Service by query.
|
||||
|
||||
@@ -15,6 +15,7 @@ import click
|
||||
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DEFAULT_SAMPLE
|
||||
from unshackle.core.service import Service
|
||||
from unshackle.core.service_repo import DirtyServiceRepo, head, is_repo_spec, refresh_repo, resolve_service_repo
|
||||
from unshackle.core.utilities import import_module_by_path
|
||||
@@ -360,16 +361,17 @@ class Services(click.Group):
|
||||
super().format_help(ctx, formatter)
|
||||
|
||||
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
||||
"""Preprocess ``--slow`` so it can take an optional range value before Click parses the arguments."""
|
||||
"""Preprocess ``--slow`` and ``--sample`` so each can take an optional range value before Click parses."""
|
||||
defaults = {"--slow": "60-120", "--sample": "{}-{}".format(*DEFAULT_SAMPLE)}
|
||||
processed = []
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--slow":
|
||||
if args[i] in defaults:
|
||||
if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]):
|
||||
processed.append(f"--slow={args[i + 1]}")
|
||||
processed.append(f"{args[i]}={args[i + 1]}")
|
||||
i += 2
|
||||
else:
|
||||
processed.append("--slow=60-120")
|
||||
processed.append(f"{args[i]}={defaults[args[i]]}")
|
||||
i += 1
|
||||
else:
|
||||
processed.append(args[i])
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Any, Optional
|
||||
from urllib.parse import urlencode, urlparse, urlunparse
|
||||
|
||||
import rnet
|
||||
from requests import HTTPError, Request
|
||||
from requests import HTTPError, Request, Response
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from unshackle.core.config import config
|
||||
@@ -227,6 +227,18 @@ class RnetResponse:
|
||||
pass
|
||||
|
||||
|
||||
def utf8_text(res: Response | RnetResponse) -> str:
|
||||
"""Return the response text, decoded as UTF-8 unless the server sent a charset.
|
||||
|
||||
With no charset, requests assumes ISO-8859-1 for a `text/*` type and runs charset detection
|
||||
on the full body for any other type. Use this only for a format that is UTF-8 by
|
||||
specification, such as M3U8.
|
||||
"""
|
||||
if isinstance(res, Response) and "charset" not in res.headers.get("content-type", "").lower():
|
||||
res.encoding = "utf-8"
|
||||
return res.text
|
||||
|
||||
|
||||
class RnetSessionHeaders(CaseInsensitiveDict):
|
||||
"""Dict-like headers that write to the rnet client through update()."""
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Deprecated. Import from unshackle.core.titles.music instead."""
|
||||
|
||||
from unshackle.core.titles.music import Album, Song # noqa: F401
|
||||
|
||||
__all__ = ("Song", "Album")
|
||||
@@ -9,8 +9,9 @@ from enum import Enum
|
||||
from functools import partial
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Optional, Union
|
||||
from typing import Any, Callable, Iterable, Optional, Sequence, Union
|
||||
|
||||
import click
|
||||
import pycaption
|
||||
import requests
|
||||
from construct import Container
|
||||
@@ -1001,6 +1002,43 @@ class Subtitle(Track):
|
||||
|
||||
return captions, language
|
||||
|
||||
def sample_slice(
|
||||
self, starts: Sequence[float], timescale: float = 1, window: Optional[tuple[int, int]] = None
|
||||
) -> slice:
|
||||
"""Return the segments of ``window`` plus one segment on each side.
|
||||
|
||||
Cue times in a segment are absolute, so ``sample_start`` stays zero and ``trim`` makes the cut.
|
||||
A track that ends before the window keeps only its last segment.
|
||||
"""
|
||||
try:
|
||||
keep = super().sample_slice(starts, timescale, window)
|
||||
except click.ClickException:
|
||||
keep = slice(len(starts) - 1, None)
|
||||
self.sample_start = 0.0
|
||||
if keep.start is None:
|
||||
return keep
|
||||
return slice(max(keep.start - 1, 0), None if keep.stop is None else keep.stop + 1)
|
||||
|
||||
def trim(self, start: float, end: float) -> bool:
|
||||
"""Keep the cues between ``start`` and ``end`` seconds and move them to start at zero.
|
||||
|
||||
Returns False and leaves the file whole when pysubs2 cannot read or write its format.
|
||||
"""
|
||||
import pysubs2
|
||||
|
||||
low, high = round(start * 1000), round(end * 1000)
|
||||
try:
|
||||
subs = pysubs2.load(str(self.path), encoding="utf-8")
|
||||
subs.events = [event for event in subs.events if event.end > low and event.start < high]
|
||||
for event in subs.events:
|
||||
event.start = max(event.start, low) - low
|
||||
event.end = min(event.end, high) - low
|
||||
subs.save(str(self.path))
|
||||
except (pysubs2.exceptions.Pysubs2Error, UnicodeDecodeError):
|
||||
return False
|
||||
self.sample_start = start
|
||||
return True
|
||||
|
||||
def strip_hearing_impaired(self) -> None:
|
||||
"""
|
||||
Remove the SDH captions.
|
||||
|
||||
@@ -3,19 +3,24 @@ import html
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
from bisect import bisect_left, bisect_right
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Optional, Union
|
||||
from typing import Any, Callable, Iterable, Optional, Sequence, Union
|
||||
from uuid import UUID
|
||||
from zlib import crc32
|
||||
|
||||
import click
|
||||
from construct import ConstructError
|
||||
from langcodes import Language
|
||||
from requests import Response, Session
|
||||
from pymp4.parser import Box
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
|
||||
from unshackle.core import binaries
|
||||
@@ -26,7 +31,7 @@ from unshackle.core.downloaders import requests
|
||||
from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine
|
||||
from unshackle.core.drm.verify import decrypt_track
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.session import RnetSession
|
||||
from unshackle.core.session import RnetSession, utf8_text
|
||||
from unshackle.core.tracks import resume
|
||||
from unshackle.core.utilities import get_boxes, log_event, try_ensure_utf8
|
||||
from unshackle.core.utils.subprocess import ffprobe
|
||||
@@ -364,6 +369,31 @@ def assert_fragments_decrypted(path: Path) -> None:
|
||||
)
|
||||
|
||||
|
||||
SAMPLE_PROBE_BYTES = 64 * 1024
|
||||
SAMPLE_PROBE_LIMIT = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def parse_sidx(data: bytes, offset: int) -> tuple[int, list[tuple[int, int, int]]]:
|
||||
"""Read a ``sidx`` box that starts at byte ``offset`` of its file.
|
||||
|
||||
Returns the timescale and one (start time, first byte, last byte) for each subsegment. The list
|
||||
is empty when the box is not a ``sidx`` or when it points at more indexes and not at media.
|
||||
"""
|
||||
try:
|
||||
box = Box.parse(data)
|
||||
except ConstructError:
|
||||
return 0, []
|
||||
if box.type != b"sidx" or any(reference.reference_type != "MEDIA" for reference in box.references):
|
||||
return 0, []
|
||||
start, byte = box.earliest_presentation_time, offset + box.end + box.first_offset
|
||||
subsegments: list[tuple[int, int, int]] = []
|
||||
for reference in box.references:
|
||||
subsegments.append((start, byte, byte + reference.referenced_size - 1))
|
||||
start += reference.segment_duration
|
||||
byte += reference.referenced_size
|
||||
return box.timescale, subsegments
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadContext:
|
||||
"""Shared arguments passed to each manifest's ``download_track``."""
|
||||
@@ -396,6 +426,8 @@ class Track:
|
||||
DASH = 3 # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
|
||||
ISM = 4 # https://learn.microsoft.com/en-us/silverlight/smooth-streaming
|
||||
|
||||
sample_tail = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Union[str, list[str]],
|
||||
@@ -497,6 +529,77 @@ class Track:
|
||||
|
||||
# TODO: Currently using OnFoo event naming, change to just segment_filter
|
||||
self.OnSegmentFilter: Optional[Callable] = None
|
||||
self.sample_start: float = 0.0
|
||||
self.sample_bytes: Optional[tuple[bytes, str]] = None
|
||||
|
||||
def sample_slice(
|
||||
self, starts: Sequence[float], timescale: float = 1, window: Optional[tuple[int, int]] = None
|
||||
) -> slice:
|
||||
"""Return the slice of segments that hold ``window``, from the start time of each segment.
|
||||
|
||||
Sets ``sample_start`` to the start time of the first kept segment, in seconds. A video keeps
|
||||
``sample_tail`` segments after the window: mkvmerge ends the cut only at a video keyframe, and
|
||||
with no such keyframe the audio and the subtitles continue to their own end.
|
||||
"""
|
||||
self.sample_start = 0.0
|
||||
if not window or not starts:
|
||||
return slice(None)
|
||||
low, high = (starts[0] + seconds * timescale for seconds in window)
|
||||
if low > starts[-1] and len(starts) > 1:
|
||||
raise click.ClickException(
|
||||
"The title ends before the sample window starts. Use --sample START-END with an earlier window."
|
||||
)
|
||||
first = max(bisect_right(starts, low) - 1, 0)
|
||||
self.sample_start = (starts[first] - starts[0]) / timescale
|
||||
return slice(first, max(bisect_left(starts, high) + self.sample_tail, first + 1))
|
||||
|
||||
def sample_byte_range(self, session: Union[Session, "RnetSession"]) -> Optional[tuple[bytes, str]]:
|
||||
"""Find the bytes of a one-file track that hold the sample window.
|
||||
|
||||
Reads the start of the file for a ``sidx`` box. Returns the bytes before that box, which hold
|
||||
the init data, and the byte range of the window. Returns None when the file has no such index.
|
||||
"""
|
||||
if self.sample_bytes or not isinstance(self.url, str):
|
||||
return self.sample_bytes
|
||||
head = b""
|
||||
|
||||
def read_to(end: int) -> None:
|
||||
nonlocal head
|
||||
if end > len(head):
|
||||
res = session.get(self.url, headers={"Range": f"bytes={len(head)}-{end - 1}"})
|
||||
if res.ok:
|
||||
head += res.content
|
||||
|
||||
pos = 0
|
||||
while pos < SAMPLE_PROBE_LIMIT:
|
||||
if pos + 8 > len(head):
|
||||
read_to(pos + SAMPLE_PROBE_BYTES)
|
||||
if pos + 8 > len(head):
|
||||
return None
|
||||
size, kind = struct.unpack(">I4s", head[pos : pos + 8])
|
||||
if size < 8:
|
||||
return None
|
||||
if kind != b"sidx":
|
||||
pos += size
|
||||
continue
|
||||
read_to(pos + size)
|
||||
timescale, subsegments = parse_sidx(head[pos : pos + size], pos)
|
||||
if not subsegments:
|
||||
return None
|
||||
kept = subsegments[self.sample_slice([start for start, _, _ in subsegments], timescale, config.sample)]
|
||||
self.sample_bytes = head[:pos], f"{kept[0][1]}-{kept[-1][2]}"
|
||||
return self.sample_bytes
|
||||
return None
|
||||
|
||||
def can_sample(self, session: Union[Session, "RnetSession"]) -> bool:
|
||||
"""Return False when a sample cannot cut this track, because it is one file with no segment index."""
|
||||
if self.descriptor == self.Descriptor.URL:
|
||||
return bool(self.sample_byte_range(session))
|
||||
if self.descriptor == self.Descriptor.DASH:
|
||||
from unshackle.core.manifests import DASH
|
||||
|
||||
return DASH.has_segment_timing(self)
|
||||
return True
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "{name}({items})".format(
|
||||
@@ -741,8 +844,17 @@ class Track:
|
||||
if DOWNLOAD_LICENCE_ONLY.is_set():
|
||||
progress(downloaded="[yellow]SKIPPED")
|
||||
else:
|
||||
sample_range = None
|
||||
if config.sample and track_type in ("Video", "Audio"):
|
||||
sample_range = self.sample_byte_range(dl_session)
|
||||
if not sample_range:
|
||||
raise click.ClickException(
|
||||
"--sample cannot cut this track: it is one file with no segment index."
|
||||
)
|
||||
for status_update in self.downloader(
|
||||
urls=self.url,
|
||||
urls=[{"url": self.url, "headers": {"Range": f"bytes={sample_range[1]}"}}]
|
||||
if sample_range
|
||||
else self.url,
|
||||
output_dir=save_path.parent,
|
||||
filename=save_path.name,
|
||||
headers=dl_session.headers,
|
||||
@@ -760,6 +872,14 @@ class Track:
|
||||
status_update["downloaded"] = f"URL {downloaded}"
|
||||
progress(**status_update)
|
||||
|
||||
if sample_range:
|
||||
media_path = save_path.with_name(f"{save_path.name}.media")
|
||||
save_path.replace(media_path)
|
||||
with save_path.open("wb") as joined, media_path.open("rb") as media:
|
||||
joined.write(sample_range[0])
|
||||
shutil.copyfileobj(media, joined)
|
||||
media_path.unlink()
|
||||
|
||||
self.path = save_path
|
||||
events.emit(events.Types.TRACK_DOWNLOADED, track=self)
|
||||
|
||||
@@ -1021,9 +1141,7 @@ class Track:
|
||||
session = getattr(self, "session", None) or Session()
|
||||
|
||||
response = session.get(self.url)
|
||||
if isinstance(response, Response):
|
||||
response.encoding = response.encoding or "utf-8"
|
||||
playlist = m3u8.loads(response.text, self.url)
|
||||
playlist = m3u8.loads(utf8_text(response), self.url)
|
||||
|
||||
drm_list = []
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from unshackle.core.utilities import (
|
||||
valid_language,
|
||||
)
|
||||
from unshackle.core.utils.collections import as_list, flatten
|
||||
from unshackle.core.utils.subprocess import run_step
|
||||
|
||||
MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)")
|
||||
MP4BOX_OPEN_FAILED = re.compile(r"error while opening|Error opening file|Invalid IsoMedia File", re.I)
|
||||
@@ -495,6 +496,11 @@ class Tracks:
|
||||
selected.append(track)
|
||||
return selected
|
||||
|
||||
@property
|
||||
def sample_base(self) -> float:
|
||||
"""The earliest ``sample_start`` of the video and audio tracks, which a sample mux uses as time zero."""
|
||||
return min((t.sample_start for t in (*self.videos, *self.audio)), default=0.0)
|
||||
|
||||
def _mux_mp4(
|
||||
self,
|
||||
title: str,
|
||||
@@ -524,6 +530,7 @@ class Tracks:
|
||||
progress = progress or partial(lambda **kwargs: None)
|
||||
|
||||
cl = [str(mp4box)]
|
||||
sample_base = self.sample_base
|
||||
names: list[str] = []
|
||||
cleaned: list[Path] = []
|
||||
track_id = 0
|
||||
@@ -557,7 +564,8 @@ class Tracks:
|
||||
elif candidate.exists():
|
||||
candidate.unlink()
|
||||
|
||||
cl.extend(["-add", f"{source}:lang={track.language}"])
|
||||
delay = round((track.sample_start - sample_base) * 1000) if config.sample else 0
|
||||
cl.extend(["-add", f"{source}:lang={track.language}" + (f":delay={delay}" if delay else "")])
|
||||
name = "" if isinstance(track, Video) else (track.get_track_name() or "")
|
||||
if name:
|
||||
names.extend(["-name", f"{track_id}={name}"])
|
||||
@@ -568,7 +576,7 @@ class Tracks:
|
||||
cl.extend(names)
|
||||
|
||||
chapters_path = None
|
||||
if self.chapters:
|
||||
if self.chapters and not config.sample:
|
||||
chapters_path = config.directories.temp / config.filenames.chapters.format(
|
||||
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
||||
)
|
||||
@@ -632,6 +640,19 @@ class Tracks:
|
||||
returncode = p.wait()
|
||||
if returncode < 2 and any(MP4BOX_OPEN_FAILED.search(line) for line in errors):
|
||||
returncode = 2
|
||||
if config.sample and returncode == 0:
|
||||
start, end = (seconds - sample_base for seconds in config.sample)
|
||||
cut_path = output_path.with_suffix(".cut.mp4")
|
||||
try:
|
||||
run_step(
|
||||
[mp4box, "-splitx", f"{start:.3f}:{end:.3f}", output_path, "-out", cut_path],
|
||||
output=cut_path,
|
||||
label="MP4Box sample cut",
|
||||
)
|
||||
os.replace(cut_path, output_path)
|
||||
except RuntimeError as e:
|
||||
returncode = 2
|
||||
errors.append(f"MP4Box could not cut the sample window: {e}")
|
||||
mux_duration_ms = round((time.monotonic() - mux_start_time) * 1000, 1)
|
||||
output_size = output_path.stat().st_size if output_path.exists() else 0
|
||||
|
||||
@@ -729,6 +750,16 @@ class Tracks:
|
||||
"--no-date", # remove dates from the output for security
|
||||
]
|
||||
|
||||
sample_base = self.sample_base
|
||||
|
||||
def sample_sync(start: float) -> list[str]:
|
||||
shift = round((start - sample_base) * 1000)
|
||||
return ["--sync", f"0:{shift}"] if config.sample and shift else []
|
||||
|
||||
if config.sample:
|
||||
start, end = (seconds - sample_base for seconds in config.sample)
|
||||
cl.extend(["--split", f"parts:{start:.3f}s-{end:.3f}s"])
|
||||
|
||||
if config.muxing.get("set_title", True):
|
||||
cl.extend(["--title", title])
|
||||
|
||||
@@ -782,6 +813,7 @@ class Tracks:
|
||||
f"0:{vt.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none",
|
||||
*sample_sync(vt.sample_start),
|
||||
]
|
||||
|
||||
# Add FPS fix if needed (typically for hybrid mode to prevent sync issues)
|
||||
@@ -839,6 +871,7 @@ class Tracks:
|
||||
f"0:{at.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none",
|
||||
*sample_sync(at.sample_start),
|
||||
"(",
|
||||
str(at.path),
|
||||
")",
|
||||
@@ -872,6 +905,7 @@ class Tracks:
|
||||
f"0:{st.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none",
|
||||
*sample_sync(st.sample_start),
|
||||
"(",
|
||||
str(st.path),
|
||||
")",
|
||||
@@ -883,6 +917,8 @@ class Tracks:
|
||||
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
||||
)
|
||||
self.chapters.dump(chapters_path, fallback_name=config.chapter_fallback_name)
|
||||
if config.sample and sample_base:
|
||||
cl.extend(["--chapter-sync", str(round(-sample_base * 1000))])
|
||||
cl.extend(["--chapter-charset", "UTF-8", "--chapters", str(chapters_path)])
|
||||
else:
|
||||
chapters_path = None
|
||||
|
||||
@@ -20,6 +20,8 @@ from unshackle.core.utils.subprocess import log_tool_run
|
||||
|
||||
|
||||
class Video(Track):
|
||||
sample_tail = 1
|
||||
|
||||
class Codec(str, Enum):
|
||||
AVC = "H.264"
|
||||
HEVC = "H.265"
|
||||
|
||||
@@ -13,7 +13,7 @@ from urllib.parse import urljoin
|
||||
from requests import Session
|
||||
|
||||
from unshackle.core.binaries import FFProbe
|
||||
from unshackle.core.session import RnetSession
|
||||
from unshackle.core.session import RnetSession, utf8_text
|
||||
from unshackle.core.utils.subprocess import log_tool_run
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -320,7 +320,7 @@ def extract_hls(track: "Track", session: Union[Session, RnetSession]) -> list[Se
|
||||
|
||||
playlist_url = track.url if isinstance(track.url, str) else track.url[0]
|
||||
res = session.get(playlist_url, timeout=PROBE_TIMEOUT)
|
||||
playlist = m3u8.loads(res.text, uri=playlist_url)
|
||||
playlist = m3u8.loads(utf8_text(res), uri=playlist_url)
|
||||
|
||||
out: list[Segment] = []
|
||||
for segment in playlist.segments:
|
||||
|
||||
@@ -6,6 +6,7 @@ import click
|
||||
from click.shell_completion import CompletionItem
|
||||
from pywidevine.cdm import Cdm as WidevineCdm
|
||||
|
||||
from unshackle.core.constants import DEFAULT_SAMPLE
|
||||
from unshackle.core.tracks.audio import Audio
|
||||
from unshackle.core.tracks.subtitle import Subtitle
|
||||
from unshackle.core.tracks.video import Video
|
||||
@@ -505,6 +506,26 @@ class SlowDelayRange(click.ParamType):
|
||||
return (low, high)
|
||||
|
||||
|
||||
class SampleRange(click.ParamType):
|
||||
"""Parses a sample window string like '60-120' into a tuple of (start, end) seconds."""
|
||||
|
||||
name = "sample_range"
|
||||
|
||||
def convert(
|
||||
self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]
|
||||
) -> Optional[tuple[int, int]]:
|
||||
if isinstance(value, bool):
|
||||
return DEFAULT_SAMPLE if value else None
|
||||
if isinstance(value, (list, tuple)) and len(value) == 2:
|
||||
value = f"{value[0]}-{value[1]}"
|
||||
|
||||
match = re.match(r"^(\d+)-(\d+)$", str(value))
|
||||
if not match or int(match.group(1)) >= int(match.group(2)):
|
||||
self.fail(f"'{value}' is not a valid sample window. Use format: START-END (e.g., 60-120)", param, ctx)
|
||||
|
||||
return (int(match.group(1)), int(match.group(2)))
|
||||
|
||||
|
||||
SEASON_RANGE = SeasonRange()
|
||||
LANGUAGE_RANGE = LanguageRange()
|
||||
QUALITY_LIST = QualityList()
|
||||
@@ -512,3 +533,4 @@ AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec)
|
||||
VIDEO_CODEC_LIST = MultipleVideoCodecChoice(Video.Codec)
|
||||
SUBTITLE_CODEC = SubtitleCodecChoice(Subtitle.Codec)
|
||||
SLOW_DELAY_RANGE = SlowDelayRange()
|
||||
SAMPLE_RANGE = SampleRange()
|
||||
|
||||
@@ -79,6 +79,9 @@ class EXAMPLE(Service):
|
||||
# ANIME: this catalogue is anime, so metadata lookups ask AniList first. Set it False (the
|
||||
# default) on a mixed catalogue and flag the anime titles individually in get_titles().
|
||||
ANIME = True
|
||||
# ANONYMOUS_SEARCH: set True only when search() needs no login and no state from
|
||||
# authenticate(). A search then skips the cookie load, authenticate() and the cookie save.
|
||||
ANONYMOUS_SEARCH = False
|
||||
# VAULT_TAG: store/read keys under a different vault namespace than this service's tag.
|
||||
# Lets sibling services share one key vault. Omit to use the service's own tag.
|
||||
VAULT_TAG = "DIFFERENT_NAME"
|
||||
|
||||
Reference in New Issue
Block a user