mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-16 14:05:10 +00:00
Compare commits
9
Commits
0a7c76561c
..
6.0.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
83eef01fad | ||
|
|
e7bb825def | ||
|
|
4fbaf2a61a | ||
|
|
3e64a7d684 | ||
|
|
2ba442f50e | ||
|
|
af4e6d5d47 | ||
|
|
b4136e1cf5 | ||
|
|
4c2e74de80 | ||
|
|
ecdfb667c9 |
+261
@@ -6,6 +6,267 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||
|
||||
This changelog is automatically generated using [git-cliff](https://git-cliff.org).
|
||||
|
||||
## [6.0.0] - 2026-10-13
|
||||
|
||||
### Features
|
||||
|
||||
- *binary*: Add custom binary registration for services
|
||||
- *cdm*: Send license_type and privacy_mode with the request to the server
|
||||
- *click*: Change the minimum time for slow
|
||||
- *downloader*: Adaptive worker scaling, tail boost, session parity, resume fixes
|
||||
- *downloader*: Fan large segment batches across worker processes
|
||||
- *downloader*: Resume failed downloads from completed segments across runs
|
||||
- *dl*: Run user post-download scripts with metadata variables
|
||||
- *dl*: Add --require-audio/--require-video and --warn-only alias
|
||||
- *drm*: Let a track pin the DRM system it licenses with
|
||||
- *dl*: Add --no-attachments to skip cover art and subtitle fonts
|
||||
- *dl*: Add --cdm to pin one CDM device for a run
|
||||
- *post-scripts*: Add wait option to block until the script exits
|
||||
- *post-scripts*: Show a spinner while waiting for a script
|
||||
- *services*: Add services_repo_force and compact refresh output
|
||||
- *serve*: Hot-reload service repos on a timer without a restart
|
||||
- *serve*: Gate server_cdm per service and let clients follow it
|
||||
- *kv*: Show a progress bar while adding keys to a vault
|
||||
- *serve*: [**breaking**] Gate the server's own proxies behind server_proxy
|
||||
- *vaults*: Batch HTTP json vault writes with InsertKeys
|
||||
- *remote*: Forward HLS segment filters to remote clients
|
||||
- *serve*: Add --quiet mode and developer dashboard API
|
||||
- *serve*: [**breaking**] Add server-held accounts for remote sessions
|
||||
- *serve*: Forward per-session service logs to remote-dl clients
|
||||
- *proxy*: Mask proxy credentials in console and API output
|
||||
- *remote*: Spoof a type-only CDM so server_cdm clients need no local .wvd/.prd
|
||||
- *serve*: Add operator dashboard endpoints and per-key rate limits
|
||||
- *audio*: Support DTS:X Profile 2 with MP4 muxing
|
||||
- *dl*: Add --no-postscript to skip post-scripts for one run
|
||||
- *api*: Report the live title and the client's command line to the dashboard
|
||||
- *vaults*: Flag and skip wrong content keys from poisoned vaults
|
||||
- *remote*: Verify server-CDM vault keys on the client and report bad pairs
|
||||
- *dl*: Add --proxy-download for a separate download proxy
|
||||
- *vaults*: Report bad keys to the vault that served them
|
||||
- *service*: Add GEOBLOCK negative geofence
|
||||
- *dash*: Add merge_segments to append segments during the download
|
||||
- *manifests*: Extend merge_segments rolling merge to HLS and ISM
|
||||
- *export*: Write mediaexport files and read legacy exports through the package
|
||||
- *proxies*: Add Control D proxy provider
|
||||
- *export*: Record each track's KIDs
|
||||
- *import*: Build tracks from direct-file export rows
|
||||
- *export*: Name export files after their titles and tracks
|
||||
- *export*: Separate export file name parts with dots
|
||||
- *remote*: Keep slow remote session requests alive with a heartbeat
|
||||
- *remote*: Cap server CDM licensing by height per API key and service
|
||||
- *remote*: List only configured services on the dashboard
|
||||
- *serve*: Relay service prompts from download jobs to the REST client
|
||||
- *remote*: Let a remote client license with its own device identity
|
||||
- *dl*: Add --latest-episodes N to download the N newest episodes
|
||||
- *setup*: Add unshackle setup command and cross-platform install scripts
|
||||
- *remote*: License with the client's device when the user picked one
|
||||
- *dl*: Add --all-drm to license Widevine and PlayReady per track
|
||||
- *dl*: Add --audio-description-only and --forced-subs-only
|
||||
- *serve*: Log CDM calls for the dashboard
|
||||
- *serve*: Reload unshackle.yaml on SIGHUP without a restart
|
||||
- *serve*: Add --cdm-only for a CDM backend with no REST API
|
||||
- *service*: Add wait_for_approval and return the cache for a login with no prompt
|
||||
- *serve*: Relay approval messages to jobs and proxy remote downloads
|
||||
- *search*: Add ANONYMOUS_SEARCH to skip the login for a service
|
||||
- *dl*: Add --sample and --include-sample
|
||||
- *dl*: Load the DRM system a cdm entry lists first
|
||||
- *api*: Forward exact_lang to the service context
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- *downloader*: Release hedge loser file handles before merge sweep
|
||||
- *downloader*: Validate 206 on byte-range segments and force identity encoding
|
||||
- *downloader*: Stop leaked racers holding segment handles after a failed batch
|
||||
- *dash*: License ClearKeyCENC before decrypt
|
||||
- *subtitle*: Convert ASS to SRT in the SDH-strip fallback
|
||||
- *api*: Keep sub_format "original" working through the API
|
||||
- *webvtt*: Don't shorten cues when splicing boundary duplicates
|
||||
- *api*: Reject backslash traversal in safe_cache_key
|
||||
- *downloader*: Honor Retry-After with cancel-safe backoff, add fault-injection bench and tests
|
||||
- *downloader*: Keep speed-limited batches in a single process
|
||||
- *downloader*: Fail closed on rnet content codings outside the old allowlist
|
||||
- *downloader*: Accept a 200 whose body is exactly the requested byte-range slice
|
||||
- *downloader*: Report rolling speed on the multiprocess progress bar
|
||||
- *downloaders*: Submit segments in a window so progress can drain
|
||||
- *api*: Pass adaptive_workers and download_processes to dl.result
|
||||
- *downloader*: Validate Content-Range before placing 206 bytes
|
||||
- *downloader*: Harden ranged reads, teardown and multiprocess sessions
|
||||
- *titles*: Drop folder {year?} when it repeats the dated season label
|
||||
- *cli*: Work around rich-click's patched click classes
|
||||
- *vaults*: Look tables up by the name the vault stores
|
||||
- *hls*: Keep ad-filtered segments out of DRM key state
|
||||
- *music*: Remux songs into the container of their codec
|
||||
- *api*: Survive Windows errno 13 on the job progress file
|
||||
- *api*: Guard two more Windows errno 13 paths in the serve layer
|
||||
- *dl*: Accept a clamped 206 and report audio-description-only titles
|
||||
- *services*: Match service aliases without case
|
||||
- *downloader*: Keep retrying while a cut connection makes progress
|
||||
- *downloader*: Bound requests after a cut and stop retry nesting
|
||||
- *subtitle*: Keep WebVTT-only cue markup out of converted subtitles
|
||||
- *proxies*: Resolve windscribevpn hostname prefixes and uk alias
|
||||
- *mux*: Give each parallel mux its own chapters temp file
|
||||
- *remote*: Carry the full title and track state across the remote API
|
||||
- *HTTP*: Simplify key retrieval condition in get_key method
|
||||
- *tracks*: Match resolutions per range and codec
|
||||
- *remote*: Resolve service names against the remote server
|
||||
- *remote*: Send only the active profile's cache files to the server
|
||||
- *utils*: Preserve _ssl_context when copying SSLCiphers adapter
|
||||
- *util*: Remove :
|
||||
- *cli*: Show banner when dl -q is used
|
||||
- *serve*: License multi-key PlayReady through server CDM instead of vault subset
|
||||
- *remote*: Raise GET timeout to 120s so large catalogs load
|
||||
- *cdm*: Scope remote CDM licence state to its session
|
||||
- *remote*: Pass fallback language to ISM re-parse
|
||||
- *ism*: Signal Dolby Vision and Atmos from Smooth manifests
|
||||
- *dash*: Number SegmentTimeline segments from startNumber
|
||||
- *remote*: Match re-parsed manifest tracks by ID before attributes
|
||||
- *remote*: Send per-track MPD fragments for service-built AdaptationSets
|
||||
- *dl*: Repair damaged cookie files without rewriting them
|
||||
- *kv*: Keep copy log rows single-spaced
|
||||
- *dl*: Fall back to local vaults when the server CDM returns no key
|
||||
- *cli*: Stop service tags from renaming CLI options
|
||||
- *dl*: Log vault writes under their own title
|
||||
- *dl*: Specify 'subtitle tracks' in error log message
|
||||
- *remote*: Keep option choices and optional title in remote -h
|
||||
- *audio*: Accept immersive channel layouts like 5.1.4
|
||||
- *serve*: Resolve DRM keys for manifests with no ContentProtection PSSH
|
||||
- *dl*: Reject local proxies when --remote targets another machine
|
||||
- *dl*: End a cancelled track by raising instead of returning short
|
||||
- *services*: Name the build a compiled service is missing
|
||||
- *serve*: Lock shared session state and throttle progress writes
|
||||
- *downloader*: Correct truncation, ranged sizing, and session carry
|
||||
- *dl*: Cache server-CDM keys to the local vaults
|
||||
- *drm*: License every PlayReady header until all KIDs are covered
|
||||
- *dash*: Treat a missing FFprobe as no PSSH instead of crashing
|
||||
- *api*: License every PlayReady header in the server CDM path
|
||||
- *api*: Reach every PlayReady header on all server CDM routes
|
||||
- *api*: Let the server CDM see service-supplied keys and HLS track DRM
|
||||
- *api*: Tell the service that the client sent a cookie jar
|
||||
- *api*: Stop serve leaking server credentials, paths, and cross-tenant data
|
||||
- *api*: Keep error URLs readable and mask host paths in dashboard logs
|
||||
- *api*: Reject a NUL byte in output_dir before resolving it
|
||||
- *manifests*: Flag DASH audio using the standard description role
|
||||
- *api*: Serialize the job progress replace to stop Windows errno 13
|
||||
- *api*: Decode challenge bytes to UTF-8 for PlayReady license request
|
||||
- *proxy*: Check the proxy exit IP before any service request
|
||||
- *cookies*: Find Firefox profiles in the XDG config directory
|
||||
- *naming*: Correct language tag rule matching and add state conditions
|
||||
- *config*: Keep no and yes as language tags
|
||||
- *import*: Inject only the imported title's keys into each track
|
||||
- *main*: Remove unwanted -
|
||||
- *serve*: Defer service hot reload while a remote session uses it
|
||||
- *remote*: Return session cache for device-code and cache-only logins
|
||||
- *remote*: Honor proxy client cdm_type for DRM system selection
|
||||
- *drm*: Log missing Widevine/PlayReady PSSH at debug level
|
||||
- *cli*: Make the click param types the one source of codec spellings
|
||||
- *vaults*: Verify vault content keys per KID from the fragment map
|
||||
- *vaults*: Harden the KID window check and clear a stale flag on licence
|
||||
- *remote*: Forward nested service cache files to and from the server
|
||||
- *remote*: Stop warning about missing keys for tracks that carry no DRM
|
||||
- *remote*: Keep the server's own CDM when the tier licenses with server_cdm
|
||||
- *post-scripts*: Expose {part} and all title-owned variables on failure
|
||||
- *serve*: Apply staged service reloads when sessions end, with a max staging age
|
||||
- *remote*: Let the server's CDM mapping decide the DRM system under server_cdm
|
||||
- *remote*: Forward dl track selection and profile to remote sessions
|
||||
- *dl*: Skip empty vault pushes and count cached keys once
|
||||
- *serve*: Stop repeating server vault pushes and retry failed ones
|
||||
- *export*: Import side-loaded subtitles and keep conflicting keys apart
|
||||
- *export*: Strip session headers from exports and normalise merged keys
|
||||
- *import*: Read export files through mediaexport
|
||||
- *export*: Skip bad keys and failed writes without stopping the download
|
||||
- *import*: Report refused titles and fail when none is usable
|
||||
- *drm*: Give the zero-KID fallback only the track's own key
|
||||
- *import*: Give each track only the keys it declares
|
||||
- *import*: Stop clearly when no export or vault key exists
|
||||
- *export*: Name the file from the first write
|
||||
- *import*: Say an import uses the export's keys, not a server CDM
|
||||
- *import*: Give no title keys to a track that is clear
|
||||
- *import*: Read an HLS export that names no title language or master key
|
||||
- *remote*: Report a missing title list and a refused track URI clearly
|
||||
- *tracks*: Keep the Dolby Vision record when remuxing video with ffmpeg
|
||||
- *proxies*: Match surfshark's current server api
|
||||
- *proxies*: Skip a proxy provider that fails to load
|
||||
- *proxies*: Return None for unservable queries and repair live providers
|
||||
- *proxies*: Link to the expressvpn device login with the code prefilled
|
||||
- *cdm*: Skip the server header probe for pywidevine and pyplayready remote CDMs
|
||||
- *api*: Stop dashboard event and log reads racing worker threads
|
||||
- *api*: Keep the worker's error on failed jobs, gate tracebacks behind --debug-api
|
||||
- *sanitize*: Keep nonspacing marks on letters in unicode filenames
|
||||
- *sanitize*: Normalise filenames to NFC and keep combining marks
|
||||
- *console*: Write UTF-8 to redirected streams on Windows
|
||||
- *dl*: Keep the --enrich original language after get_tracks
|
||||
- *attachment*: Keep custom fonts after mux cleanup
|
||||
- *dl*: Keep the --enrich title and year after get_tracks
|
||||
- *docs*: Hide the release version in the header repo widget
|
||||
- *dl*: Read the vaults first under --all-drm
|
||||
- *hybrid*: Read the container file for a base that had the DV fixup
|
||||
- *cdm*: Send a PlayReady licence without an XML declaration as XML
|
||||
- *import*: Carry the HLS segment filter in an export
|
||||
- *cdm*: Simplify cached keys retrieval in CustomRemoteCDM
|
||||
- *hls*: Read the key id from the init segment when the PSSH has none
|
||||
- *api*: Follow the cdm config key order for the server CDM
|
||||
|
||||
### Documentation
|
||||
|
||||
- *session*: Correct stale pool-size comment
|
||||
- Strip non-essential comments, fold key facts into docstrings
|
||||
- *services*: Add MUSIC_EXAMPLE reference service
|
||||
- Apply simplified technical english across docs and docstrings.
|
||||
- Phase 2 of the prose style across docs and docstrings
|
||||
- Widen supported shaka-packager range to 2.6.1 or >= 3.10.0
|
||||
- *dev*: Correct get_chapters guidance on markers at timestamp 0
|
||||
- *example*: Show drm_preference pinning for dual-PSSH manifests
|
||||
- *post-scripts*: Use post-script as the one term for hooks
|
||||
- *api*: Describe the dashboard session payload in the OpenAPI yaml
|
||||
- *cli*: Correct --merge-video scope in cli reference
|
||||
- *export*: Describe the export file name and its messages
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
- *downloader*: Slow-start ramp for the adaptive worker controller
|
||||
- *downloader*: Use per-arrival reads only while a segment is hedge-raced
|
||||
- *dash*: Collapse single-URL byte-range segments into one direct download
|
||||
- *downloader*: Stride multiprocess chunk assignment across children
|
||||
- *api*: Skip progress re-parse when the file is unchanged
|
||||
- *api*: Append job history instead of rewriting the file
|
||||
- *downloader*: Unblock hedge-loser reads at teardown
|
||||
- *dash*: Speed up segment collection and URL templating
|
||||
- *dl*: Parallelise mux, licensing, and vault lookups; size the shared pool
|
||||
- *dl*: Overlap hybrid builds with muxing, tag in the mux thread, preload DRM in parallel
|
||||
- *hybrid*: Reuse DV-side files across resolutions, trim the RPU chain, fix L6 fallback
|
||||
- *dl*: Push vault key writes to a background thread
|
||||
- *drm*: Decrypt fMP4 segments during download with mp4decrypt
|
||||
- *kv*: Push whole services to SQL vaults in one call
|
||||
- *kv*: Push keys to vaults concurrently, log one summary per title
|
||||
- *sample*: Download only the subtitle segments around the window
|
||||
- *manifests*: Decode manifests without charset detection
|
||||
|
||||
### Changes
|
||||
|
||||
- *monalisa*: Remove ML-Worker and improve MonaLisa DRM
|
||||
- *downloader*: Extract shared byte-range split helper
|
||||
- *console*: Route interactive prompts through a themed helper
|
||||
- *core*: [**breaking**] Drop leading underscores from internal definitions
|
||||
- *tests*: Drop leading underscores from test helper functions
|
||||
- *music*: [**breaking**] Make Song an ordinary Title on the generic path
|
||||
- *console*: Style env listings like the rich-click help screens
|
||||
- *drm*: Drop the hardcoded service branch in the license flow
|
||||
- *proxies*: Share proxy dispatch between dl, search and resolve_proxy
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Restore memory/ to .gitignore from dev
|
||||
- *scripts*: Expand bench suite for cross-machine runs
|
||||
- *scripts*: Harden bench suite report capture
|
||||
- *scripts*: Add attempt-1-only tail-slow fault to downloader bench
|
||||
- Bump dev version to 5.5.0
|
||||
- Mark install.sh executable
|
||||
|
||||
### Builds
|
||||
|
||||
- *deps*: Pull mediaexport from github in the package metadata
|
||||
- *deps*: Lock mediaexport at fa9357e
|
||||
|
||||
## [5.4.0] - 2026-08-15
|
||||
|
||||
### Features
|
||||
|
||||
@@ -316,7 +316,7 @@ Show the video, audio, and subtitle tracks for a title. For series, you can scop
|
||||
| `part` | int/string | no | Part index of a split episode (combined with `season` and `episode`). |
|
||||
| `profile`, `proxy`, `no_proxy`, `cdm_type` | - | no | As on `list-titles`. |
|
||||
| `credentials`, `cookies`, `cache` | - | no | As on `list-titles`. |
|
||||
| `dl_params` | object | no | The `dl` track selection that the service reads from `ctx.parent.params`: `lang`, `v_lang`, `a_lang`, `acodec` and `forced_subs`, as on [`POST /api/session/create`](#post-apisessioncreate). Some services select the manifests they fetch from these values. |
|
||||
| `dl_params` | object | no | The `dl` track selection that the service reads from `ctx.parent.params`: `lang`, `v_lang`, `a_lang`, `acodec`, `forced_subs` and `exact_lang`, as on [`POST /api/session/create`](#post-apisessioncreate). Some services select the manifests they fetch from these values. |
|
||||
|
||||
When you give both `season` and `episode`, the server combines them into a `"{season}x{episode}"` selector. `part` narrows that to `"{season}x{episode}.{part}"`, one part of a [split episode](../creating-a-service.md#split-episodes). Without `part` you get every part of the episode. The server reads `part` only when you give both `season` and `episode`, and `wanted` takes precedence over all three. To scope a part by range, put it in `wanted` instead (`"S01E01.2"`).
|
||||
|
||||
@@ -462,6 +462,8 @@ Make a download job. It requires `service` and `title_id`. Every other field is
|
||||
| `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. |
|
||||
| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://<resolver>@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. |
|
||||
| `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. |
|
||||
| `sample` | boolean/string | `null` | Download a short sample and write it with a `SAMPLE` tag on the filename in a `Sample` folder inside the output folder. `true` is 60 seconds from 2 minutes in; `"0-60"` is a window in seconds. |
|
||||
| `include_sample` | boolean | `false` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder next to it. |
|
||||
| `workers` | int | `null` | Threads per track. |
|
||||
| `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. |
|
||||
| `download_processes` | int | `1` | Split a track's segments across this many processes. Only engages for large batches. |
|
||||
@@ -956,7 +958,7 @@ the newlines must go out at once.
|
||||
|
||||
Make a remote session for a service and title. Authentication runs asynchronously in the background. This call returns immediately with `status: "authenticating"`, and you then poll the prompt endpoint. The body accepts `service` and `title_id` (both required). It also accepts a broad set of optional keys, because the body allows `additionalProperties`. These are `credentials` (`{username, password, extra?}`), `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, `no_proxy`, `profile`, `cache` (map of forwarded cache files, keyed by the file path relative to the service cache directory with `/` separators and no `.json` suffix, so a nested key such as `session_web/<sha1>` lands in a subdirectory), `client_region`, `proxy_region`, `cdm_type`, `range_`, `vcodec`, `quality`, `best_available`, `dl_params`, `client`, and any service CLI options. `proxy_region` is the country code the client resolved `proxy` from; the server matches it against its own accounts. `client` is a freeform object the dashboard shows as sent (the CLI puts `version`, `code_hash`, `platform` and a redacted `argv` in it); the server ignores it above 4096 bytes of JSON.
|
||||
|
||||
`dl_params` is an object with the `dl` track selection: `lang`, `v_lang` and `a_lang` (arrays of language strings), `acodec` (an array of audio codec names or `dl -a` aliases such as `ddp`) and `forced_subs` (boolean). The service reads these values from `ctx.parent.params`. The server uses the `dl` default for each absent or malformed value, and drops an unknown codec name.
|
||||
`dl_params` is an object with the `dl` track selection: `lang`, `v_lang` and `a_lang` (arrays of language strings), `acodec` (an array of audio codec names or `dl -a` aliases such as `ddp`), and `forced_subs` and `exact_lang` (booleans). The service reads these values from `ctx.parent.params`. The server uses the `dl` default for each absent or malformed value, and drops an unknown codec name.
|
||||
|
||||
Service CLI options also travel in a nested `service_params` object, which wins over a flat key with the same name. `profile` at the top level always means the credentials profile, never a service's own `--profile` option.
|
||||
|
||||
|
||||
@@ -432,11 +432,11 @@ object the dashboard shows as sent), and the track-selection hints `range_`, `vc
|
||||
|
||||
`dl_params` holds the `dl` track selection: `lang`, `v_lang` and `a_lang`
|
||||
(arrays of language strings), `acodec` (an array of audio codec names, such as
|
||||
`EC3`), and `forced_subs` (boolean). The server gives these values to the
|
||||
`EC3`), and `forced_subs` and `exact_lang` (booleans). The server gives these values to the
|
||||
service in `ctx.parent.params`, never as service options, so a service option
|
||||
with the same name, such as an own `--lang`, does not get them. The server uses
|
||||
the `dl` default for each absent or malformed value: `lang` is `["orig"]`,
|
||||
`v_lang`, `a_lang` and `acodec` are `[]`, and `forced_subs` is `false`. The
|
||||
`v_lang`, `a_lang` and `acodec` are `[]`, and `forced_subs` and `exact_lang` are `false`. The
|
||||
server drops an unknown codec name. An empty array is a valid value and stays
|
||||
empty.
|
||||
|
||||
|
||||
@@ -153,6 +153,8 @@ Keep only certain track types, or skip certain track types.
|
||||
| `--no-folder` | Disable folder creation for TV shows. |
|
||||
| `--no-source` | Remove the source tag from the filename/path. |
|
||||
| `--no-mux` | Do not mux; keep individual track files. |
|
||||
| `--sample` | Download a short sample and write it with a `SAMPLE` tag on the filename in a `Sample` folder inside the output folder. Bare `--sample` = 60 seconds from 2 minutes in; `--sample 0-60` = a custom window in seconds. |
|
||||
| `--include-sample` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder next to it. |
|
||||
| `--tag` | Group tag override. |
|
||||
| `--repack` | Add a `REPACK` tag to the filename. |
|
||||
| `--postscript` | Run a command after each output file, with `{variable}` placeholders substituted. Repeatable. Replaces the `post_scripts` config for this run. See [Post-scripts](../reference/configuration/post-scripts.md). |
|
||||
|
||||
@@ -711,9 +711,24 @@ By default, `mkvmerge` muxes the tracks of a movie or an episode into a single M
|
||||
| Flag | Behaviour | Default source |
|
||||
| --- | --- | --- |
|
||||
| `--no-mux` | Do not mux; keep the individual track files. | - |
|
||||
| `--sample` | Download a short sample: 60 seconds from 2 minutes in, or `--sample START-END` for a window in seconds. | - |
|
||||
| `--include-sample` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder. | - |
|
||||
| `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) |
|
||||
| `--merge-video` | Mux video tracks that share a height, range, and codec into one file, so only language varies inside a file. | config `muxing.merge_video` (off) |
|
||||
|
||||
!!! note "`--sample` needs a segmented track"
|
||||
`--sample` downloads only the segments that hold the window, and `mkvmerge` cuts the output
|
||||
to the window. The cut starts at the first video keyframe in the window, so a sample can be
|
||||
some seconds short. unshackle downloads each subtitle track in full and then keeps only the
|
||||
cues of the window.
|
||||
|
||||
The run stops before the download starts in two conditions: a service gives a video or
|
||||
audio track as one file with no segments and no segment index, or the title ends before the
|
||||
window starts.
|
||||
|
||||
A sidecar subtitle file with `sidecar_format: original` is not trimmed. It holds the segments around the
|
||||
window, or the whole file when the track is one file.
|
||||
|
||||
!!! note "Music is never muxed"
|
||||
A song skips the muxer. unshackle keeps the container that the service
|
||||
delivered, such as `.flac`, `.m4a`, or `.mp3`, and writes the metadata into that file.
|
||||
|
||||
@@ -495,6 +495,9 @@ unshackle dl --skip-dl --all-drm --export EXAMPLE 81234567
|
||||
|
||||
- The `cdm` mapping must name a device for each system. A single device name, or `--cdm`,
|
||||
stops the run with an error that names the missing system.
|
||||
- The order of the `widevine` and `playready` config keys in the `cdm` entry sets the order
|
||||
of the two systems, as in a run without the flag. Each track licenses first with the
|
||||
system of the first config key.
|
||||
- A Key Vault is read first, as in a run without the flag. A DRM system sends a challenge
|
||||
only for a KID that no Key Vault and no earlier track of the run gave a content key
|
||||
for. When the first system returns all the KIDs of the second, the second sends no
|
||||
|
||||
@@ -330,6 +330,22 @@ A few muxing behaviors are automatic and not configurable:
|
||||
|
||||
The `--no-mux` flag skips muxing entirely and writes the individual track files, each with a track-type suffix, into the same output/folder structure.
|
||||
|
||||
## Sample files
|
||||
|
||||
[`--sample`](cli-reference.md#output-muxing-files) and `--include-sample` write a sample into a
|
||||
`Sample` folder inside the output folder of the title. The sample keeps the filename of the
|
||||
title, with a `SAMPLE` tag at the end. The tag uses the separator of the output template.
|
||||
|
||||
```text title="A movie with --include-sample"
|
||||
Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG/
|
||||
├── Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG.mkv
|
||||
└── Sample/
|
||||
└── Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG.SAMPLE.mkv
|
||||
```
|
||||
|
||||
With `--sample`, the `Sample` folder holds the only output file. If the run makes no title
|
||||
folder, the `Sample` folder is in the downloads directory.
|
||||
|
||||
## Tags & group naming
|
||||
|
||||
The release-group tag is the `-TAG` portion at the end of scene-style names, which the `{tag}` variable gives.
|
||||
|
||||
@@ -45,6 +45,11 @@ are `widevine` / `playready`. unshackle matches the quality keys first, but only
|
||||
height is known. If the value is still a dict after that, unshackle uses
|
||||
`widevine`/`playready` when present, or else the credential profile name, then `default`.
|
||||
|
||||
When an entry holds `widevine` and `playready`, their order sets which device unshackle loads
|
||||
first. A track that offers the two DRM systems licenses with that system. A track that offers
|
||||
only the other system uses the other device. A `widevine` / `playready` dict in a quality key
|
||||
does not set the order. The server CDM of a remote session follows the same order.
|
||||
|
||||
A quality key can also hold a dict of `widevine` / `playready` devices, and so can the
|
||||
top-level `default` entry:
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "unshackle"
|
||||
version = "5.5.0"
|
||||
version = "6.0.0"
|
||||
description = "Modular Movie, TV, and Music Archival Software."
|
||||
authors = [{ name = "unshackle team" }]
|
||||
requires-python = ">=3.11,<3.15"
|
||||
|
||||
@@ -17,7 +17,7 @@ from pywidevine.pssh import PSSH
|
||||
from rich.table import Table
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
from unshackle.core.api.handlers import validate_download_parameters
|
||||
from unshackle.core.api.handlers import load_full_cdm, resolve_server_cdm, validate_download_parameters
|
||||
from unshackle.core.cdm.detect import cdm_type_stub, is_widevine_cdm
|
||||
from unshackle.core.drm import PlayReady, Widevine
|
||||
from unshackle.core.manifests import HLS
|
||||
@@ -286,3 +286,42 @@ def test_the_init_segment_adds_the_drm_system_the_track_does_not_have(monkeypatc
|
||||
Track.add_init_drm(track, b"init", None)
|
||||
|
||||
assert track.drm == [wv, pr]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("entry", "expected"),
|
||||
[
|
||||
({"playready": "pr_device", "widevine": "wv_device"}, "pr_device"),
|
||||
({"widevine": "wv_device", "playready": "pr_device"}, "wv_device"),
|
||||
],
|
||||
)
|
||||
def test_the_first_drm_system_of_the_config_entry_loads_first(
|
||||
cmd: dl, monkeypatch: pytest.MonkeyPatch, entry: dict, expected: str
|
||||
) -> None:
|
||||
monkeypatch.setattr("unshackle.commands.dl.config.cdm", {"EXAMPLE": entry})
|
||||
monkeypatch.setattr("unshackle.core.cdm.load_cdm", lambda name, **_: name)
|
||||
monkeypatch.setattr("unshackle.core.config.config.cdm", {"EXAMPLE": entry})
|
||||
cmd.vaults = None
|
||||
|
||||
assert cmd.get_cdm("EXAMPLE") == expected
|
||||
assert load_full_cdm("EXAMPLE", None) == expected
|
||||
assert cmd.get_cdm("EXAMPLE", drm="widevine") == "wv_device"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("entry", "expected"),
|
||||
[
|
||||
({"playready": "pr_device", "widevine": "wv_device"}, "playready"),
|
||||
({"widevine": "wv_device", "playready": "pr_device"}, "widevine"),
|
||||
],
|
||||
)
|
||||
def test_the_server_cdm_stub_is_the_first_drm_system_of_the_config_entry(
|
||||
monkeypatch: pytest.MonkeyPatch, entry: dict, expected: str
|
||||
) -> None:
|
||||
monkeypatch.setattr("unshackle.core.config.config.cdm", {"EXAMPLE": entry})
|
||||
monkeypatch.setattr(
|
||||
"unshackle.core.api.handlers.detect_cdm_type",
|
||||
lambda name, _: "playready" if name == "pr_device" else "widevine",
|
||||
)
|
||||
|
||||
assert is_widevine_cdm(resolve_server_cdm("EXAMPLE", None, None)) is (expected == "widevine")
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""An HLS Widevine key whose PSSH holds a content id and no Key ID takes the Key ID of its init segment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import m3u8
|
||||
from pywidevine.license_protocol_pb2 import WidevinePsshData
|
||||
from pywidevine.pssh import PSSH
|
||||
|
||||
from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY
|
||||
from unshackle.core.drm import PlayReady, Widevine
|
||||
from unshackle.core.import_service import hls_playlist_kids
|
||||
from unshackle.core.manifests.hls import HLS
|
||||
from unshackle.core.tracks import Video
|
||||
from unshackle.core.tracks.track import DownloadContext, Track
|
||||
|
||||
INIT_KID = UUID("11111111-1111-1111-1111-111111111111")
|
||||
PSSH_KID = UUID("22222222-2222-2222-2222-222222222222")
|
||||
TAG_KID = UUID("33333333-3333-3333-3333-333333333333")
|
||||
|
||||
ZERO_KID = UUID(int=0)
|
||||
|
||||
|
||||
def no_kid_pssh(content_id: bytes = b"content-id-only") -> str:
|
||||
return PSSH.new(
|
||||
system_id=PSSH.SystemId.Widevine,
|
||||
init_data=WidevinePsshData(content_id=content_id).SerializeToString(),
|
||||
).dumps()
|
||||
|
||||
|
||||
NO_KID_PSSH = no_kid_pssh()
|
||||
KEY_LINE = (
|
||||
'#EXT-X-KEY:METHOD=SAMPLE-AES-CTR,URI="data:text/plain;base64,{pssh}",'
|
||||
'KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed",KEYFORMATVERSIONS="1"'
|
||||
)
|
||||
|
||||
|
||||
def keyed_segment(pssh: str, extra: str = "", init: str = "init.mp4") -> str:
|
||||
return f'#EXT-X-MAP:URI="{init}"\n{KEY_LINE.format(pssh=pssh)}{extra}\n#EXTINF:6.0,\n{init}.m4s\n'
|
||||
|
||||
|
||||
def playlist(*segments: str) -> str:
|
||||
body = "#EXT-X-DISCONTINUITY\n".join(segments)
|
||||
return f"#EXTM3U\n#EXT-X-VERSION:7\n#EXT-X-TARGETDURATION:6\n{body}#EXT-X-ENDLIST\n"
|
||||
|
||||
|
||||
def key_of(pssh: str, extra: str = "") -> m3u8.Key:
|
||||
return m3u8.loads(playlist(keyed_segment(pssh, extra))).keys[0]
|
||||
|
||||
|
||||
def licence_only(text: str, tmp_path: Path) -> list[tuple[list[UUID], Any]]:
|
||||
"""Run a licence-only download of the playlist and return the Key IDs of each licence call."""
|
||||
media = tmp_path / "media.m3u8"
|
||||
media.write_text(text, encoding="utf-8")
|
||||
track = Video(
|
||||
url="https://example.test/media.m3u8",
|
||||
language="en",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
codec=Video.Codec.AVC,
|
||||
)
|
||||
track.from_file = media
|
||||
licensed: list[tuple[list[UUID], Any]] = []
|
||||
ctx = DownloadContext(
|
||||
save_path=tmp_path / "video.mp4",
|
||||
save_dir=tmp_path / "video_segments",
|
||||
progress=lambda **_: None,
|
||||
license_widevine=lambda drm, track_kid=None: licensed.append((drm.kids, track_kid)),
|
||||
cdm=None,
|
||||
)
|
||||
DOWNLOAD_LICENCE_ONLY.set()
|
||||
try:
|
||||
HLS.download_track(track=track, ctx=ctx)
|
||||
finally:
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
return licensed
|
||||
|
||||
|
||||
def test_download_track_licenses_with_the_init_segment_kid(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
# the probe itself needs the network and FFprobe; the order of the calls is under test
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(lambda *args, **kwargs: INIT_KID))
|
||||
assert licence_only(playlist(keyed_segment(NO_KID_PSSH)), tmp_path) == [([INIT_KID], INIT_KID)]
|
||||
|
||||
|
||||
def test_licence_only_reads_each_later_key_from_its_own_init_segment(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
def probe(master: Any, track: Any, session: Any, init_section: Any = None) -> UUID:
|
||||
return PSSH_KID if init_section is not None and init_section.uri == "init2.mp4" else INIT_KID
|
||||
|
||||
monkeypatch.setattr(HLS, "get_track_kid_from_init", staticmethod(probe))
|
||||
text = playlist(keyed_segment(no_kid_pssh(b"first")), keyed_segment(no_kid_pssh(b"later"), init="init2.mp4"))
|
||||
assert licence_only(text, tmp_path) == [([INIT_KID], INIT_KID), ([PSSH_KID], PSSH_KID)]
|
||||
|
||||
|
||||
def test_get_drm_keeps_a_pssh_that_has_its_own_kid() -> None:
|
||||
pssh = PSSH.new(key_ids=[PSSH_KID], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
drm = HLS.get_drm(key_of(pssh), kid=INIT_KID)
|
||||
assert drm.kids == [PSSH_KID]
|
||||
assert drm.pssh.dumps() == pssh
|
||||
|
||||
|
||||
def test_get_drm_prefers_the_tag_keyid_over_the_init_segment_kid() -> None:
|
||||
drm = HLS.get_drm(key_of(NO_KID_PSSH, f",KEYID=0x{TAG_KID.hex}"), kid=INIT_KID)
|
||||
assert drm.kids == [TAG_KID]
|
||||
|
||||
|
||||
def test_get_drm_replaces_a_placeholder_pssh_kid_and_ignores_a_placeholder_init_kid() -> None:
|
||||
placeholder = PSSH.new(key_ids=[ZERO_KID], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
assert HLS.get_drm(key_of(placeholder), kid=INIT_KID).kids == [INIT_KID]
|
||||
assert HLS.get_drm(key_of(placeholder), kid=ZERO_KID).kids == [ZERO_KID]
|
||||
|
||||
|
||||
def test_session_key_without_a_kid_does_not_stop_the_parse() -> None:
|
||||
with_kid = PSSH.new(key_ids=[PSSH_KID], system_id=PSSH.SystemId.Widevine).dumps()
|
||||
drm = HLS.get_all_drm([key_of(NO_KID_PSSH), key_of(with_kid)])
|
||||
assert [d.kids for d in drm] == [[PSSH_KID]]
|
||||
|
||||
|
||||
def test_alternate_drm_builds_the_other_system_with_the_init_segment_kid() -> None:
|
||||
chosen = object.__new__(PlayReady)
|
||||
alternate = HLS.alternate_drm([key_of(NO_KID_PSSH)], chosen, kid=INIT_KID)
|
||||
assert [type(d) for d in alternate] == [Widevine] and alternate[0].kids == [INIT_KID]
|
||||
|
||||
|
||||
def test_import_reads_the_kid_of_a_playlist_whose_key_has_none() -> None:
|
||||
def get(url: str, headers: Any = None) -> Any:
|
||||
return SimpleNamespace(text=playlist(keyed_segment(NO_KID_PSSH)), ok=True, content=b"init")
|
||||
|
||||
track = SimpleNamespace(id="vid", url="https://example.test/media.m3u8", get_key_id=lambda init: INIT_KID)
|
||||
assert hls_playlist_kids(track, SimpleNamespace(get=get)) == {INIT_KID}
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Unit tests for M3U8 response decoding. Reading `.text` on a response with no encoding
|
||||
makes requests run chardet over the whole body, so both playlist fetch paths set utf-8
|
||||
first. A charset the server did send must survive."""
|
||||
"""Unit tests for manifest response decoding. Reading `.text` on a response with no encoding
|
||||
makes requests run chardet over the whole body, so the playlist fetch paths set utf-8
|
||||
first. A charset the server did send must survive. An MPD is parsed from bytes, so the
|
||||
XML declaration gives the encoding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,7 +10,9 @@ from typing import Any, Optional
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from unshackle.core.manifests.dash import DASH
|
||||
from unshackle.core.manifests.hls import HLS
|
||||
from unshackle.core.session import RnetResponse, utf8_text
|
||||
from unshackle.core.tracks import Video
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
@@ -24,6 +27,8 @@ def make_response(body: bytes, url: str, encoding: Optional[str]) -> requests.Re
|
||||
res._content = body
|
||||
res.url = url
|
||||
res.encoding = encoding
|
||||
if encoding:
|
||||
res.headers["Content-Type"] = f"application/vnd.apple.mpegurl; charset={encoding}"
|
||||
return res
|
||||
|
||||
|
||||
@@ -74,3 +79,61 @@ def test_load_drm_from_playlist_keeps_a_charset_the_server_sent() -> None:
|
||||
|
||||
make_track(res).load_drm_from_playlist()
|
||||
assert res.encoding == "iso-8859-1"
|
||||
|
||||
|
||||
MPD_LATIN1 = '<?xml version="1.0" encoding="ISO-8859-1"?><MPD><Title>café</Title></MPD>'.encode("iso-8859-1")
|
||||
|
||||
|
||||
class StubRnetResponse:
|
||||
"""The part of an rnet response that `RnetResponse` reads."""
|
||||
|
||||
ok = True
|
||||
status_code = 200
|
||||
encoding = None
|
||||
|
||||
def __init__(self, body: bytes, url: str) -> None:
|
||||
self.body = body
|
||||
self.url = url
|
||||
|
||||
def bytes(self) -> bytes:
|
||||
return self.body
|
||||
|
||||
|
||||
def test_dash_from_url_follows_the_xml_declaration() -> None:
|
||||
res = make_response(MPD_LATIN1, "https://example.com/manifest.mpd", None)
|
||||
dash = DASH.from_url("https://example.com/manifest.mpd", FakeSession(res))
|
||||
|
||||
assert dash.manifest.findtext("Title") == "café"
|
||||
|
||||
|
||||
def test_dash_from_url_reads_an_rnet_response() -> None:
|
||||
res = RnetResponse(StubRnetResponse(MPD_LATIN1, "https://example.com/manifest.mpd"))
|
||||
dash = DASH.from_url("https://example.com/manifest.mpd", FakeSession(res)) # type: ignore[arg-type]
|
||||
|
||||
assert dash.manifest.findtext("Title") == "café"
|
||||
|
||||
|
||||
def test_utf8_text_reads_an_rnet_response() -> None:
|
||||
assert utf8_text(RnetResponse(StubRnetResponse(MEDIA, "https://example.com/video/720.m3u8"))) == MEDIA.decode()
|
||||
|
||||
|
||||
def test_utf8_text_ignores_the_requests_default_for_a_text_type() -> None:
|
||||
res = make_response('#EXTM3U\n#EXT-X-MEDIA:TYPE=AUDIO,NAME="中文"\n'.encode(), "https://example.com/a.m3u8", None)
|
||||
res.headers["Content-Type"] = "text/plain"
|
||||
res.encoding = "ISO-8859-1"
|
||||
|
||||
assert "中文" in utf8_text(res)
|
||||
|
||||
|
||||
def test_dash_from_url_uses_the_http_charset_without_an_xml_declaration() -> None:
|
||||
res = make_response("<MPD><Title>中文</Title></MPD>".encode("gbk"), "https://example.com/manifest.mpd", "gbk")
|
||||
dash = DASH.from_url("https://example.com/manifest.mpd", FakeSession(res))
|
||||
|
||||
assert dash.manifest.findtext("Title") == "中文"
|
||||
|
||||
|
||||
def test_dash_from_url_rejects_an_empty_body() -> None:
|
||||
res = make_response(b"", "https://example.com/manifest.mpd", None)
|
||||
|
||||
with pytest.raises(ValueError, match="must be provided"):
|
||||
DASH.from_url("https://example.com/manifest.mpd", FakeSession(res))
|
||||
|
||||
@@ -21,6 +21,7 @@ from Cryptodome.Cipher import AES
|
||||
from Cryptodome.Util.Padding import pad
|
||||
from pywidevine.pssh import PSSH
|
||||
|
||||
from tests.core.test_hls_kid_from_init import KEY_LINE, no_kid_pssh
|
||||
from tests.core.test_ism_init import piff_fragment
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
@@ -264,3 +265,34 @@ def test_hls_aes128_keeps_post_merge(server, tmp_path, monkeypatch):
|
||||
|
||||
plain, rolling = download_both_ways(HLS, make, tmp_path, monkeypatch, False)
|
||||
assert rolling == plain == INIT + b"".join(PARTS)
|
||||
|
||||
|
||||
def test_hls_rekey_takes_the_kid_from_the_init_segment_of_the_new_key(server, tmp_path, monkeypatch):
|
||||
"""Each key carries a PSSH with a content id and no Key ID, so only the init segment names the KID."""
|
||||
later_kid = UUID(hex="22222222222222222222222222222222")
|
||||
later_init = bytes(range(100, 200))
|
||||
server.routes["/init2.mp4"] = later_init
|
||||
no_decrypt(monkeypatch, hls_module)
|
||||
monkeypatch.setattr(config, "merge_segments", False)
|
||||
|
||||
def key_line(content_id: bytes) -> str:
|
||||
return KEY_LINE.format(pssh=no_kid_pssh(content_id))
|
||||
|
||||
track = hls_track(server, False)
|
||||
lines = ["#EXTM3U", "#EXT-X-VERSION:7", "#EXT-X-TARGETDURATION:4", '#EXT-X-MAP:URI="init.mp4"', key_line(b"a")]
|
||||
for i in range(len(PARTS)):
|
||||
if i == 2:
|
||||
lines += ["#EXT-X-DISCONTINUITY", '#EXT-X-MAP:URI="init2.mp4"', key_line(b"b")]
|
||||
lines += ["#EXTINF:4.0,", f"part{i}.m4s"]
|
||||
server.routes["/media.m3u8"] = "\n".join(lines + ["#EXT-X-ENDLIST"]).encode()
|
||||
# the real probe needs FFprobe and a tenc box; the KID follows the init bytes the parser hands over
|
||||
monkeypatch.setattr(
|
||||
type(track), "get_key_id", lambda self, init_data=None, **kw: {later_init: later_kid}.get(init_data, KID)
|
||||
)
|
||||
|
||||
licensed: list[UUID] = []
|
||||
ctx = make_ctx(tmp_path, "rekey.mp4")
|
||||
ctx.license_widevine = lambda drm, track_kid=None: licensed.append(track_kid)
|
||||
HLS.download_track(track, ctx)
|
||||
|
||||
assert licensed == [KID, later_kid]
|
||||
|
||||
@@ -192,10 +192,17 @@ class _CapturedCtx(Exception):
|
||||
@pytest.mark.parametrize(
|
||||
("params", "expected"),
|
||||
[
|
||||
({}, (["orig"], [], [], [], False)),
|
||||
({}, (["orig"], [], [], [], False, False)),
|
||||
(
|
||||
{"lang": ["es-419"], "v_lang": ["en"], "a_lang": ["ja"], "acodec": "AAC,EC3", "forced_subs": True},
|
||||
(["es-419"], ["en"], ["ja"], ["AAC", "EC3"], True),
|
||||
{
|
||||
"lang": ["es-419"],
|
||||
"v_lang": ["en"],
|
||||
"a_lang": ["ja"],
|
||||
"acodec": "AAC,EC3",
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
},
|
||||
(["es-419"], ["en"], ["ja"], ["AAC", "EC3"], True, True),
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -224,7 +231,8 @@ def test_perform_download_puts_the_lang_selection_in_the_service_ctx(
|
||||
|
||||
ctx = exc.value.args[0]
|
||||
p = ctx.params
|
||||
assert (p["lang"], p["v_lang"], p["a_lang"], [c.name for c in p["acodec"]], p["forced_subs"]) == expected
|
||||
selection = (p["lang"], p["v_lang"], p["a_lang"], [c.name for c in p["acodec"]], p["forced_subs"], p["exact_lang"])
|
||||
assert selection == expected
|
||||
|
||||
|
||||
def test_perform_download_passes_latest_episodes_to_dl_result(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
@@ -13,6 +13,7 @@ from uuid import UUID
|
||||
import pytest
|
||||
from pywidevine.pssh import PSSH as WvPSSH
|
||||
|
||||
from tests.core.test_hls_kid_from_init import NO_KID_PSSH
|
||||
from unshackle.core.api import handlers
|
||||
from unshackle.core.drm import Widevine
|
||||
from unshackle.core.tracks import Track
|
||||
@@ -22,13 +23,14 @@ pytestmark = pytest.mark.unit
|
||||
KID = UUID("161219ec3df64e0eadbb3827c0ccc46d")
|
||||
|
||||
|
||||
def _media_playlist() -> str:
|
||||
pssh = WvPSSH.new(key_ids=[KID], system_id=WvPSSH.SystemId.Widevine).dumps()
|
||||
def _media_playlist(pssh: str | None = None) -> str:
|
||||
pssh = pssh or WvPSSH.new(key_ids=[KID], system_id=WvPSSH.SystemId.Widevine).dumps()
|
||||
return "\n".join(
|
||||
[
|
||||
"#EXTM3U",
|
||||
"#EXT-X-VERSION:6",
|
||||
"#EXT-X-TARGETDURATION:6",
|
||||
'#EXT-X-MAP:URI="init.mp4"',
|
||||
f'#EXT-X-KEY:METHOD=SAMPLE-AES,URI="data:text/plain;base64,{pssh}",'
|
||||
'KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed",KEYFORMATVERSIONS="1"',
|
||||
"#EXTINF:6.0,",
|
||||
@@ -49,3 +51,21 @@ def test_hls_descriptor_alone_reaches_the_media_playlist_keys():
|
||||
assert track.drm and isinstance(track.drm[0], Widevine), "media playlist DRM not read"
|
||||
assert track.drm[0].kids == [KID]
|
||||
assert base64.b64decode(handlers.extract_pssh_from_track(track, "widevine"))
|
||||
|
||||
|
||||
def test_key_without_a_kid_takes_it_from_the_init_segment():
|
||||
"""A PSSH that holds a content id and no Key ID is DRM, and the server must not call the track clear."""
|
||||
response = SimpleNamespace(text=_media_playlist(NO_KID_PSSH), ok=True, content=b"init")
|
||||
session = SimpleNamespace(get=lambda url, headers=None: response)
|
||||
track = SimpleNamespace(
|
||||
id="vid",
|
||||
url="https://cdn.example.invalid/media.m3u8",
|
||||
descriptor=Track.Descriptor.HLS,
|
||||
data={},
|
||||
drm=None,
|
||||
get_key_id=lambda init: KID,
|
||||
)
|
||||
|
||||
handlers.ensure_track_drm(track, session)
|
||||
|
||||
assert track.drm and track.drm[0].kids == [KID]
|
||||
|
||||
@@ -616,6 +616,7 @@ def test_dl_selection_travels_nested_and_a_service_option_cannot_override_it() -
|
||||
"a_lang": [],
|
||||
"acodec": [Audio.Codec.EC3],
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
}
|
||||
data = _sent_create_data(parent_params, {"lang": 3, "a_lang": "x"})
|
||||
assert data["dl_params"] == {
|
||||
@@ -624,6 +625,7 @@ def test_dl_selection_travels_nested_and_a_service_option_cannot_override_it() -
|
||||
"a_lang": [],
|
||||
"acodec": ["EC3"],
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
}
|
||||
assert data["lang"] == 3
|
||||
assert data["service_params"] == {"lang": 3, "a_lang": "x"}
|
||||
|
||||
@@ -162,6 +162,9 @@ def _create_with_real_ctx(monkeypatch, data):
|
||||
return svc
|
||||
|
||||
|
||||
DL_DEFAULTS = {"lang": ["orig"], "v_lang": [], "a_lang": [], "acodec": [], "forced_subs": False, "exact_lang": False}
|
||||
|
||||
|
||||
def test_nested_dl_params_reach_the_parent_ctx(monkeypatch):
|
||||
from unshackle.core.tracks import Audio
|
||||
|
||||
@@ -172,6 +175,7 @@ def test_nested_dl_params_reach_the_parent_ctx(monkeypatch):
|
||||
"a_lang": ["es-419"],
|
||||
"acodec": ["EC3", "NOPE", "AAC"],
|
||||
"forced_subs": True,
|
||||
"exact_lang": True,
|
||||
}
|
||||
}
|
||||
svc = _create_with_real_ctx(monkeypatch, data)
|
||||
@@ -180,12 +184,13 @@ def test_nested_dl_params_reach_the_parent_ctx(monkeypatch):
|
||||
assert svc.parent_params["a_lang"] == ["es-419"]
|
||||
assert svc.parent_params["acodec"] == [Audio.Codec.EC3, Audio.Codec.AAC]
|
||||
assert svc.parent_params["forced_subs"] is True
|
||||
assert svc.parent_params["exact_lang"] is True
|
||||
|
||||
|
||||
def test_absent_dl_params_fall_back_to_the_dl_defaults(monkeypatch):
|
||||
svc = _create_with_real_ctx(monkeypatch, {})
|
||||
selection = {k: svc.parent_params[k] for k in ("lang", "v_lang", "a_lang", "acodec", "forced_subs")}
|
||||
assert selection == {"lang": ["orig"], "v_lang": [], "a_lang": [], "acodec": [], "forced_subs": False}
|
||||
selection = {k: svc.parent_params[k] for k in (*DL_DEFAULTS,)}
|
||||
assert selection == DL_DEFAULTS
|
||||
|
||||
|
||||
def test_an_empty_lang_list_is_kept(monkeypatch):
|
||||
@@ -204,14 +209,14 @@ def test_a_service_lang_option_never_gets_the_dl_lang(monkeypatch):
|
||||
"dl_params",
|
||||
[
|
||||
"en",
|
||||
{"lang": "en", "v_lang": "ja", "a_lang": {"x": 1}, "acodec": "EC3", "forced_subs": "yes"},
|
||||
{"lang": [1], "v_lang": [None], "a_lang": ["en", None], "acodec": [3], "forced_subs": 1},
|
||||
{"lang": "en", "v_lang": "ja", "a_lang": {"x": 1}, "acodec": "EC3", "forced_subs": "yes", "exact_lang": "yes"},
|
||||
{"lang": [1], "v_lang": [None], "a_lang": ["en", None], "acodec": [3], "forced_subs": 1, "exact_lang": 1},
|
||||
],
|
||||
)
|
||||
def test_malformed_dl_params_are_ignored(monkeypatch, dl_params):
|
||||
svc = _create_with_real_ctx(monkeypatch, {"dl_params": dl_params})
|
||||
selection = {k: svc.parent_params[k] for k in ("lang", "v_lang", "a_lang", "acodec", "forced_subs")}
|
||||
assert selection == {"lang": ["orig"], "v_lang": [], "a_lang": [], "acodec": [], "forced_subs": False}
|
||||
selection = {k: svc.parent_params[k] for k in (*DL_DEFAULTS,)}
|
||||
assert selection == DL_DEFAULTS
|
||||
|
||||
|
||||
def test_profile_reaches_the_parent_ctx_params(monkeypatch):
|
||||
|
||||
@@ -268,7 +268,7 @@ def test_season_context_blanks_every_episode_identity_field():
|
||||
|
||||
def test_every_documented_variable_is_one_build_context_produces():
|
||||
"""The variable tables in docs/reference/configuration/post-scripts.md must not drift from the code."""
|
||||
from unshackle.core.titles.song import Song
|
||||
from unshackle.core.titles.music import Song
|
||||
|
||||
doc = Path(__file__).resolve().parents[1] / "docs" / "reference" / "configuration" / "post-scripts.md"
|
||||
documented = {
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
"""A ``--sample`` run downloads only the segments of its window and cuts the mux to the window."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import http.server
|
||||
import struct
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import click
|
||||
import pytest
|
||||
from requests import Session
|
||||
|
||||
from unshackle.commands.dl import sample_name
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.api.handlers import validate_download_parameters
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.manifests.dash import DASH
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.titles import Episode, Movie
|
||||
from unshackle.core.tracks import Audio, Chapter, Subtitle, Tracks, Video
|
||||
from unshackle.core.tracks.track import parse_sidx
|
||||
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||
from unshackle.core.utils.xml import load_xml
|
||||
|
||||
STARTS = [1000 + i * 4000 for i in range(100)] # 4 s segments at timescale 1000
|
||||
|
||||
|
||||
def make_video() -> Video:
|
||||
return Video(id_="v", url="https://x/v", codec=Video.Codec.AVC, language="en", width=1920, height=1080, fps=24)
|
||||
|
||||
|
||||
def make_audio(url: str = "https://x/a") -> Audio:
|
||||
return Audio(id_="a", url=url, codec=Audio.Codec.AAC, language="en", bitrate=128000)
|
||||
|
||||
|
||||
def test_no_window_keeps_every_segment() -> None:
|
||||
assert make_audio().sample_slice(STARTS, 1000) == slice(None)
|
||||
|
||||
|
||||
def test_the_window_is_measured_from_the_first_segment() -> None:
|
||||
track = make_audio()
|
||||
assert track.sample_slice(STARTS, 1000, (62, 120)) == slice(15, 30)
|
||||
assert track.sample_start == 60
|
||||
|
||||
|
||||
def test_a_video_keeps_one_segment_after_the_window() -> None:
|
||||
"""mkvmerge ends the cut only at a video keyframe, so the video must reach past the window."""
|
||||
assert make_video().sample_slice(STARTS, 1000, (0, 60)) == slice(0, 16)
|
||||
assert make_audio().sample_slice(STARTS, 1000, (0, 60)) == slice(0, 15)
|
||||
|
||||
|
||||
def test_a_subtitle_keeps_one_segment_on_each_side_of_the_window() -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="en")
|
||||
video = make_video()
|
||||
video.sample_tail = 0
|
||||
keep = video.sample_slice(STARTS, 1000, (62, 120))
|
||||
assert subtitle.sample_slice(STARTS, 1000, (62, 120)) == slice(keep.start - 1, keep.stop + 1)
|
||||
assert subtitle.sample_start == 0, "cue times stay absolute, so the mux does not shift the track"
|
||||
assert subtitle.sample_slice(STARTS, 1000, None) == slice(None)
|
||||
|
||||
|
||||
def test_a_subtitle_that_ends_before_the_window_keeps_its_last_segment() -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="en")
|
||||
assert subtitle.sample_slice([0, 4000], 1000, (120, 180)) == slice(0, None)
|
||||
|
||||
|
||||
def test_a_window_after_the_end_stops_the_run() -> None:
|
||||
with pytest.raises(click.ClickException, match="ends before the sample window"):
|
||||
make_audio().sample_slice(STARTS[:10], 1000, (120, 180))
|
||||
|
||||
|
||||
TIMELINE_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<SegmentTemplate timescale="1" media="a/$Number$-$Time$.mp4" startNumber="5">
|
||||
<SegmentTimeline><S t="1000" d="4" r="99"/></SegmentTimeline>
|
||||
</SegmentTemplate>
|
||||
<Representation id="a" bandwidth="1"/>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
DURATION_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S" duration="PT400S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<SegmentTemplate timescale="25" duration="100" media="a/$Number$.mp4" startNumber="5"/>
|
||||
<Representation id="a" bandwidth="1"/>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
LIST_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<Representation id="a" bandwidth="1">
|
||||
<BaseURL>https://example.com/a/</BaseURL>
|
||||
<SegmentList timescale="1" duration="4">{urls}</SegmentList>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
""".format(urls="".join(f'<SegmentURL media="{n}.mp4"/>' for n in range(100)))
|
||||
|
||||
BASE_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<Representation id="a" bandwidth="1"><BaseURL>https://example.com/a.mp4</BaseURL></Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
|
||||
WINDOW = (62, 120)
|
||||
|
||||
|
||||
def dash_names(mpd: str, window: Optional[tuple[int, int]]) -> tuple[list[str], float]:
|
||||
tree = load_xml(mpd.encode())
|
||||
period = tree.find("Period")
|
||||
aset = period.find("AdaptationSet")
|
||||
rep = aset.find("Representation")
|
||||
track = make_audio()
|
||||
_, segments, _, _, _ = DASH.get_period_segments(
|
||||
period, aset, rep, tree, track, "https://example.com/main.mpd", MagicMock(), window=window
|
||||
)
|
||||
return [url.rsplit("/", 1)[1].split(".")[0] for url, _ in segments], track.sample_start
|
||||
|
||||
|
||||
def test_dash_timeline_keeps_the_numbers_and_times_of_the_window() -> None:
|
||||
names, start = dash_names(TIMELINE_MPD, WINDOW)
|
||||
assert names == [f"{5 + n}-{1000 + n * 4}" for n in range(15, 30)]
|
||||
assert start == 60
|
||||
|
||||
|
||||
def test_dash_fixed_duration_keeps_the_numbers_of_the_window() -> None:
|
||||
names, start = dash_names(DURATION_MPD, WINDOW)
|
||||
assert names == [str(5 + n) for n in range(15, 30)]
|
||||
assert start == 60
|
||||
|
||||
|
||||
def test_dash_segment_list_keeps_the_urls_of_the_window() -> None:
|
||||
names, start = dash_names(LIST_MPD, WINDOW)
|
||||
assert names == [str(n) for n in range(15, 30)]
|
||||
assert start == 60
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mpd", [TIMELINE_MPD, DURATION_MPD, LIST_MPD])
|
||||
def test_a_caller_that_passes_no_window_gets_every_segment(mpd: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The bitrate probe divides by the full duration, so a sample run must not cut its segment list."""
|
||||
monkeypatch.setattr(config, "sample", WINDOW)
|
||||
assert len(dash_names(mpd, None)[0]) == 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("mpd", "timed"), [(TIMELINE_MPD, True), (LIST_MPD, True), (BASE_MPD, False)])
|
||||
def test_a_track_given_as_one_file_has_no_segment_timing(mpd: str, timed: bool) -> None:
|
||||
aset = load_xml(mpd.encode()).find("Period").find("AdaptationSet")
|
||||
track = SimpleNamespace(data={"dash": {"adaptation_set": aset, "representation": aset.find("Representation")}})
|
||||
assert DASH.has_segment_timing(track) is timed
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def served(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str:
|
||||
handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(tmp_path))
|
||||
handler.log_message = lambda *args: None # type: ignore[attr-defined]
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
(tmp_path / "temp").mkdir()
|
||||
monkeypatch.setattr(config.directories, "temp", tmp_path / "temp")
|
||||
yield f"http://127.0.0.1:{server.server_port}"
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_an_hls_subtitle_downloads_the_segments_around_the_window(
|
||||
tmp_path: Path, served: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
for n in range(8):
|
||||
(tmp_path / f"s{n}.vtt").write_text(
|
||||
f"WEBVTT\n\n00:00:{n * 10 + 1:02}.000 --> 00:00:{n * 10 + 2:02}.000\ncue {n}\n"
|
||||
)
|
||||
(tmp_path / "s.m3u8").write_text(
|
||||
"#EXTM3U\n#EXT-X-TARGETDURATION:10\n"
|
||||
+ "".join(f"#EXTINF:10.0,\ns{n}.vtt\n" for n in range(8))
|
||||
+ "#EXT-X-ENDLIST\n"
|
||||
)
|
||||
monkeypatch.setattr(config, "sample", (32, 38))
|
||||
subtitle = Subtitle(
|
||||
id_="s", url=f"{served}/s.m3u8", codec=Subtitle.Codec.WebVTT, language="en", descriptor=Subtitle.Descriptor.HLS
|
||||
)
|
||||
|
||||
subtitle.download(session=Session(), prepare_drm=lambda *a, **k: None, progress=lambda **k: None)
|
||||
|
||||
text = subtitle.path.read_text()
|
||||
assert [n for n in range(8) if f"cue {n}" in text] == [2, 3, 4]
|
||||
assert "00:00:31.000 --> 00:00:32.000" in text, "cue times stay absolute"
|
||||
|
||||
|
||||
def test_subtitle_trim_keeps_the_cues_of_the_window_from_zero(tmp_path: Path) -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.SubRip, language="en")
|
||||
subtitle.path = tmp_path / "s.srt"
|
||||
subtitle.path.write_text(
|
||||
"1\n00:00:05,000 --> 00:00:06,000\nbefore\n\n"
|
||||
"2\n00:02:10,000 --> 00:02:12,000\ninside\n\n"
|
||||
"3\n00:10:00,000 --> 00:10:01,000\nafter\n"
|
||||
)
|
||||
|
||||
assert subtitle.trim(120, 180) is True
|
||||
|
||||
text = subtitle.path.read_text()
|
||||
assert "inside" in text and "before" not in text and "after" not in text
|
||||
assert "00:00:10,000 --> 00:00:12,000" in text
|
||||
assert subtitle.sample_start == 120
|
||||
|
||||
|
||||
def test_subtitle_trim_leaves_an_unreadable_file_whole(tmp_path: Path) -> None:
|
||||
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.fTTML, language="en")
|
||||
subtitle.path = tmp_path / "s.stpp"
|
||||
subtitle.path.write_bytes(b"\x00\x01not a subtitle")
|
||||
|
||||
assert subtitle.trim(120, 180) is False
|
||||
assert subtitle.path.read_bytes() == b"\x00\x01not a subtitle"
|
||||
|
||||
|
||||
def mux_command(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
class Stop(Exception):
|
||||
pass
|
||||
|
||||
captured: list[list[str]] = []
|
||||
|
||||
def fake_popen(args: list[str], **kwargs: object) -> None:
|
||||
captured.append(args)
|
||||
raise Stop
|
||||
|
||||
monkeypatch.setattr("unshackle.core.tracks.tracks.subprocess.Popen", fake_popen)
|
||||
monkeypatch.setattr(binaries, "MKVToolNix", Path("mkvmerge"))
|
||||
|
||||
video, audio = make_video(), make_audio()
|
||||
trimmed = Subtitle(id_="s1", url="https://x/s1", codec=Subtitle.Codec.SubRip, language="en")
|
||||
whole = Subtitle(id_="s2", url="https://x/s2", codec=Subtitle.Codec.SubRip, language="fr")
|
||||
for track, name in ((video, "v.mp4"), (audio, "a.mp4"), (trimmed, "s1.srt"), (whole, "s2.srt")):
|
||||
track.path = tmp_path / name
|
||||
track.path.write_bytes(b"\x00" * 16)
|
||||
video.sample_start, audio.sample_start, trimmed.sample_start = 120.0, 119.5, 121.0
|
||||
tracks = Tracks([video, audio, trimmed, whole])
|
||||
tracks.chapters.add(Chapter("00:02:05.000", "Scene"))
|
||||
|
||||
with pytest.raises(Stop):
|
||||
tracks.mux("Title", delete=False, progress=lambda **k: None, output_path=tmp_path / "out.mkv")
|
||||
return captured[0]
|
||||
|
||||
|
||||
def test_the_mux_shifts_each_track_and_cuts_to_the_window(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "sample", (121, 181))
|
||||
command = mux_command(tmp_path, monkeypatch)
|
||||
|
||||
def sync_before(name: str) -> str:
|
||||
file_at = command.index(str(tmp_path / name))
|
||||
options = command[:file_at]
|
||||
sync_at = len(options) - 1 - options[::-1].index("--sync")
|
||||
assert all(not arg.endswith((".mp4", ".srt")) for arg in command[sync_at:file_at]), (
|
||||
"the --sync is another file's"
|
||||
)
|
||||
return command[sync_at + 1]
|
||||
|
||||
assert command[command.index("--split") + 1] == "parts:1.500s-61.500s"
|
||||
assert sync_before("v.mp4") == "0:500"
|
||||
assert "--sync" not in command[: command.index(str(tmp_path / "a.mp4"))][-12:], "the earliest track is not moved"
|
||||
assert sync_before("s1.srt") == "0:1500"
|
||||
assert sync_before("s2.srt") == "0:-119500"
|
||||
assert command[command.index("--chapter-sync") + 1] == "-119500"
|
||||
assert command.index("--chapter-sync") < command.index("--chapters")
|
||||
|
||||
|
||||
def test_a_full_download_adds_no_sample_options(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "sample", None)
|
||||
command = mux_command(tmp_path, monkeypatch)
|
||||
assert not {"--split", "--sync", "--chapter-sync"} & set(command)
|
||||
|
||||
|
||||
def test_sample_range_reads_a_window() -> None:
|
||||
assert SAMPLE_RANGE.convert("60-120", None, None) == (60, 120)
|
||||
assert SAMPLE_RANGE.convert(True, None, None) == (120, 180)
|
||||
assert SAMPLE_RANGE.convert(False, None, None) is None
|
||||
for bad in ("120-60", "60-60", "abc", "-5-10", "60"):
|
||||
with pytest.raises(click.BadParameter):
|
||||
SAMPLE_RANGE.convert(bad, None, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample", [True, "60-120", [60, 120]])
|
||||
def test_the_api_accepts_a_sample_window(sample: object) -> None:
|
||||
assert validate_download_parameters({"sample": sample}) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample", ["abc", "120-60", [1, 2, 3], ["a", "b"], 5, {"start": 1}])
|
||||
def test_the_api_refuses_a_bad_sample_window(sample: object) -> None:
|
||||
assert "sample" in (validate_download_parameters({"sample": sample}) or "")
|
||||
|
||||
|
||||
def test_the_api_refuses_sample_with_include_sample() -> None:
|
||||
assert "include_sample" in (validate_download_parameters({"sample": True, "include_sample": True}) or "")
|
||||
|
||||
|
||||
def test_sample_name_uses_the_separator_of_the_template(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
separators = {"movies": ".", "series": " "}
|
||||
monkeypatch.setattr(config, "get_template_separator", lambda kind="movies": separators[kind])
|
||||
movie = Movie.__new__(Movie)
|
||||
episode = Episode.__new__(Episode)
|
||||
assert sample_name("Some.Movie.2026.1080p-TAG", movie) == "Some.Movie.2026.1080p-TAG.SAMPLE"
|
||||
assert sample_name("Some Show S01E01 1080p-TAG", episode) == "Some Show S01E01 1080p-TAG SAMPLE"
|
||||
|
||||
|
||||
SIDX_MPD = """
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||
<Period id="0" start="PT0S">
|
||||
<AdaptationSet mimeType="audio/mp4">
|
||||
<Representation id="a" bandwidth="1">
|
||||
<BaseURL>https://example.com/a.mp4</BaseURL>
|
||||
<SegmentBase indexRange="800-{last}"><Initialization range="0-799"/></SegmentBase>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>
|
||||
"""
|
||||
|
||||
|
||||
def sidx_box(count: int = 100, size: int = 5000, duration: int = 4000, timescale: int = 1000) -> bytes:
|
||||
body = struct.pack(">IIIII", 0, 1, timescale, 0, 0) + struct.pack(">HH", 0, count)
|
||||
body += b"".join(struct.pack(">III", size, duration, 0x90000000) for _ in range(count))
|
||||
return struct.pack(">I", 8 + len(body)) + b"sidx" + body
|
||||
|
||||
|
||||
def test_parse_sidx_gives_the_time_and_byte_range_of_each_subsegment() -> None:
|
||||
box = sidx_box(count=3)
|
||||
end = 800 + len(box)
|
||||
assert parse_sidx(box, 800) == (
|
||||
1000,
|
||||
[(0, end, end + 4999), (4000, end + 5000, end + 9999), (8000, end + 10000, end + 14999)],
|
||||
)
|
||||
|
||||
|
||||
def test_parse_sidx_refuses_what_it_cannot_cut() -> None:
|
||||
index_of_indexes = bytearray(sidx_box(count=2))
|
||||
index_of_indexes[32] |= 0x80
|
||||
assert parse_sidx(bytes(index_of_indexes), 0) == (0, [])
|
||||
assert parse_sidx(b"\x00\x00\x00\x20moof" + b"\x00" * 24, 0) == (0, [])
|
||||
assert parse_sidx(sidx_box(count=3)[:-4], 0) == (0, [])
|
||||
|
||||
|
||||
def test_a_single_file_dash_track_is_cut_by_its_segment_index(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
box = sidx_box()
|
||||
requested: list[str] = []
|
||||
|
||||
def get(url: str, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
byte_range = (headers or {}).get("Range", "")
|
||||
requested.append(byte_range)
|
||||
content = box if byte_range == f"bytes=800-{800 + len(box) - 1}" else b"\x00" * 800
|
||||
return MagicMock(content=content, headers={"Content-Range": "bytes 0-799/900000"})
|
||||
|
||||
tree = load_xml(SIDX_MPD.format(last=800 + len(box) - 1).encode())
|
||||
period = tree.find("Period")
|
||||
aset = period.find("AdaptationSet")
|
||||
rep = aset.find("Representation")
|
||||
track = make_audio()
|
||||
track.data["dash"] = {"adaptation_set": aset, "representation": rep}
|
||||
session = MagicMock(get=get)
|
||||
first_media_byte = 800 + len(box)
|
||||
|
||||
_, segments, timescale, starts, _ = DASH.get_period_segments(
|
||||
period, aset, rep, tree, track, "https://example.com/main.mpd", session, probe_kid=False, window=WINDOW
|
||||
)
|
||||
|
||||
assert DASH.has_segment_timing(track) is True
|
||||
assert timescale == 1000 and starts == [n * 4000 for n in range(15, 30)]
|
||||
assert segments[0] == (
|
||||
"https://example.com/a.mp4",
|
||||
f"{first_media_byte + 15 * 5000}-{first_media_byte + 16 * 5000 - 1}",
|
||||
)
|
||||
assert len(segments) == 15 and track.sample_start == 60
|
||||
|
||||
requested.clear()
|
||||
_, whole, _, _, _ = DASH.get_period_segments(
|
||||
period, aset, rep, tree, track, "https://example.com/main.mpd", session, probe_kid=False
|
||||
)
|
||||
assert len(whole) == 1 and not any("800-" in r for r in requested), "a full download does not read the index"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "window"),
|
||||
[
|
||||
(["--sample", "TAG", "-t", "x", "id"], (120, 180)),
|
||||
(["--sample", "60-120", "TAG", "id"], (60, 120)),
|
||||
(["TAG", "id"], None),
|
||||
],
|
||||
)
|
||||
def test_a_bare_sample_option_does_not_take_the_service_tag(args: list[str], window: object) -> None:
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
class Group(Services):
|
||||
def list_commands(self, ctx: click.Context) -> list[str]:
|
||||
return ["TAG"]
|
||||
|
||||
def get_command(self, ctx: click.Context, name: str) -> click.Command:
|
||||
return click.Command(
|
||||
name, params=[click.Option(["-t"]), click.Argument(["title"])], callback=lambda **k: None
|
||||
)
|
||||
|
||||
@click.group(cls=Group)
|
||||
@click.option("--sample", type=SAMPLE_RANGE, default=None)
|
||||
def group(sample: object) -> None:
|
||||
seen["sample"] = sample
|
||||
|
||||
group(args, standalone_mode=False)
|
||||
assert seen["sample"] == window
|
||||
|
||||
|
||||
class RangeSession:
|
||||
"""Serves byte ranges of one in-memory file and records each range it was asked for."""
|
||||
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
self.ranges: list[str] = []
|
||||
|
||||
def get(self, url: str, headers: dict[str, str]) -> SimpleNamespace:
|
||||
self.ranges.append(headers["Range"])
|
||||
first, last = (int(n) for n in headers["Range"].removeprefix("bytes=").split("-"))
|
||||
return SimpleNamespace(ok=True, content=self.data[first : last + 1])
|
||||
|
||||
|
||||
def one_file(moov_size: int, index: bytes) -> bytes:
|
||||
ftyp = struct.pack(">I4s", 16, b"ftyp") + b"isom" + b"\x00" * 4
|
||||
moov = struct.pack(">I4s", moov_size, b"moov") + b"\x00" * (moov_size - 8)
|
||||
return ftyp + moov + index + b"\x00" * 600_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moov_size", [700, 200_000])
|
||||
def test_a_one_file_track_is_cut_by_the_index_in_the_file(moov_size: int, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A large moov puts the index after the first probe, so the probe must read on."""
|
||||
monkeypatch.setattr(config, "sample", WINDOW)
|
||||
box = sidx_box()
|
||||
session = RangeSession(one_file(moov_size, box))
|
||||
audio = make_audio("https://x/a.mp4")
|
||||
|
||||
init, byte_range = audio.sample_byte_range(session)
|
||||
|
||||
media = 16 + moov_size + len(box)
|
||||
assert init == session.data[: 16 + moov_size], "the init data stops before the index"
|
||||
assert byte_range == f"{media + 15 * 5000}-{media + 30 * 5000 - 1}"
|
||||
assert audio.sample_start == 60
|
||||
assert all(r.startswith("bytes=") for r in session.ranges), "every request is a byte range"
|
||||
|
||||
|
||||
def test_a_one_file_track_with_no_index_cannot_be_cut(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "sample", WINDOW)
|
||||
mdat = struct.pack(">I4s", 600_000, b"mdat") + b"\x00" * (600_000 - 8)
|
||||
audio = make_audio("https://x/a.mp4")
|
||||
assert audio.sample_byte_range(RangeSession(one_file(700, b"")[:716] + mdat)) is None
|
||||
+134
-6
@@ -46,6 +46,7 @@ from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote
|
||||
from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption
|
||||
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
|
||||
from unshackle.core.constants import (
|
||||
DEFAULT_SAMPLE,
|
||||
DOWNLOAD_ALL_DRM,
|
||||
DOWNLOAD_CANCELLED,
|
||||
DOWNLOAD_LICENCE_ONLY,
|
||||
@@ -107,6 +108,7 @@ from unshackle.core.utils.click_types import (
|
||||
AUDIO_CODEC_LIST,
|
||||
LANGUAGE_RANGE,
|
||||
QUALITY_LIST,
|
||||
SAMPLE_RANGE,
|
||||
SEASON_RANGE,
|
||||
SLOW_DELAY_RANGE,
|
||||
SUBTITLE_CODEC,
|
||||
@@ -118,7 +120,7 @@ from unshackle.core.utils.collections import ci_get, merge_dict
|
||||
from unshackle.core.utils.post_scripts import NO_POST_SCRIPTS, build_context, dispatch, season_context
|
||||
from unshackle.core.utils.redact import mask_proxy
|
||||
from unshackle.core.utils.selector import select_multiple
|
||||
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe
|
||||
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe, run_step
|
||||
from unshackle.core.vault import Vault
|
||||
from unshackle.core.vaults import Vaults
|
||||
|
||||
@@ -259,6 +261,19 @@ def latest_episode_keys(titles: Any, latest_episode: bool, latest_episodes: Opti
|
||||
return list(dict.fromkeys((ep.season, ep.number) for ep in titles))[-count:]
|
||||
|
||||
|
||||
SAMPLE_DIR = "Sample"
|
||||
|
||||
|
||||
def template_type(title: Any) -> str:
|
||||
"""Return the output template key of a title: ``series``, ``songs`` or ``movies``."""
|
||||
return "series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
||||
|
||||
|
||||
def sample_name(name: str, title: Any) -> str:
|
||||
"""Return ``name`` with a SAMPLE tag, joined with the separator of the output template of the title."""
|
||||
return f"{name.rstrip()}{config.get_template_separator(template_type(title))}SAMPLE"
|
||||
|
||||
|
||||
def apply_enrich(title: Title_T, name: Optional[str], year: Optional[int], language: Optional[Language]) -> None:
|
||||
"""Set the --enrich name, year and original language on a title. A song takes only the language.
|
||||
|
||||
@@ -324,6 +339,41 @@ def remux_music_tracks(title: Any, log: logging.Logger) -> None:
|
||||
log.warning(f"Could not remux audio track {track.id} for music: {e}")
|
||||
|
||||
|
||||
def cut_music_sample(title: Any, window: tuple[int, int], log: logging.Logger) -> None:
|
||||
"""Cut each audio track of a Song to the sample window. A Song has no mux step that cuts it."""
|
||||
if not isinstance(title, Song) or not binaries.FFMPEG:
|
||||
return
|
||||
for track in title.tracks.audio:
|
||||
if not track.path or not track.path.exists():
|
||||
continue
|
||||
cut_path = track.path.with_stem(f"{track.path.stem}_cut")
|
||||
try:
|
||||
run_step(
|
||||
[
|
||||
binaries.FFMPEG,
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-ss",
|
||||
f"{max(window[0] - track.sample_start, 0):.3f}",
|
||||
"-i",
|
||||
track.path,
|
||||
"-t",
|
||||
str(window[1] - window[0]),
|
||||
"-map",
|
||||
"0",
|
||||
"-c",
|
||||
"copy",
|
||||
cut_path,
|
||||
],
|
||||
output=cut_path,
|
||||
label="music sample cut",
|
||||
)
|
||||
os.replace(cut_path, track.path)
|
||||
except RuntimeError as e:
|
||||
log.warning(f"Could not cut audio track {track.id} to the sample window, so it holds whole segments: {e}")
|
||||
|
||||
|
||||
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
|
||||
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
|
||||
The config replaces defaults and global ``dl:`` default_map values."""
|
||||
@@ -972,6 +1022,19 @@ class dl:
|
||||
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
|
||||
)
|
||||
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
|
||||
@click.option(
|
||||
"--sample",
|
||||
type=SAMPLE_RANGE,
|
||||
default=None,
|
||||
help="Download a short sample and write it to a Sample folder. "
|
||||
"Use --sample for 60 seconds from 2 minutes in, or --sample START-END (e.g., --sample 0-60) for a custom window.",
|
||||
)
|
||||
@click.option(
|
||||
"--include-sample",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Also cut a 60 second sample from each muxed file into a Sample folder next to it.",
|
||||
)
|
||||
@click.option(
|
||||
"--postscript",
|
||||
"postscript",
|
||||
@@ -1566,6 +1629,8 @@ class dl:
|
||||
no_folder: bool,
|
||||
no_source: bool,
|
||||
no_mux: bool,
|
||||
sample: Optional[tuple[int, int]],
|
||||
include_sample: bool,
|
||||
workers: Optional[int],
|
||||
adaptive_workers: bool,
|
||||
download_processes: int,
|
||||
@@ -1595,6 +1660,10 @@ class dl:
|
||||
postscript = NO_POST_SCRIPTS
|
||||
if continue_downloads:
|
||||
config.continue_downloads = True
|
||||
config.sample = sample
|
||||
if sample and include_sample:
|
||||
raise click.ClickException("--sample and --include-sample cannot be used together.")
|
||||
sample_dir = SAMPLE_DIR if sample else ""
|
||||
self.tmdb_searched = False
|
||||
self.search_source = None
|
||||
self.service_anime = bool(getattr(service, "ANIME", False))
|
||||
@@ -3058,6 +3127,18 @@ class dl:
|
||||
self.prefer_vault_keys(title)
|
||||
self.cache_resolved_keys(title)
|
||||
|
||||
if sample and not skip_dl:
|
||||
whole_files = [
|
||||
track
|
||||
for track in (*title.tracks.videos, *title.tracks.audio)
|
||||
if not track.can_sample(track.session or service.session)
|
||||
]
|
||||
if whole_files:
|
||||
raise click.ClickException(
|
||||
f"--sample cannot cut {len(whole_files)} track(s) of {title}: "
|
||||
f"{self.service} gives each one as one file with no segment index."
|
||||
)
|
||||
|
||||
dl_start_time = time.time()
|
||||
|
||||
try:
|
||||
@@ -3213,6 +3294,10 @@ class dl:
|
||||
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
|
||||
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
|
||||
)
|
||||
if sample:
|
||||
for subtitle in title.tracks.subtitles:
|
||||
if subtitle.path and subtitle.path.exists():
|
||||
subtitle.trim(*sample)
|
||||
for (src, dst), count in sub_conversions.items():
|
||||
log_event(
|
||||
"subtitle_convert",
|
||||
@@ -3340,6 +3425,8 @@ class dl:
|
||||
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
|
||||
|
||||
remux_music_tracks(title, self.log)
|
||||
if sample:
|
||||
cut_music_sample(title, sample, self.log)
|
||||
|
||||
muxed_paths = []
|
||||
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
|
||||
@@ -3522,6 +3609,11 @@ class dl:
|
||||
hybrid_output_path = config.directories.temp / hybrid_filename
|
||||
hybrid_temp_paths.append(hybrid_output_path)
|
||||
|
||||
if len({track.sample_start for track in resolution_tracks}) > 1:
|
||||
raise click.ClickException(
|
||||
"--sample cut the HDR10 and Dolby Vision tracks at different "
|
||||
"points, so a hybrid of them would not line up."
|
||||
)
|
||||
Hybrid(resolution_tracks, self.service, output_name=hybrid_filename)
|
||||
|
||||
if container:
|
||||
@@ -3579,6 +3671,8 @@ class dl:
|
||||
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
|
||||
if media_info:
|
||||
base_filename = title.get_filename(media_info, show_service=not no_source)
|
||||
if sample:
|
||||
base_filename = sample_name(base_filename, title)
|
||||
else:
|
||||
base_filename = str(title)
|
||||
|
||||
@@ -3594,6 +3688,7 @@ class dl:
|
||||
sidecar_dir /= title.get_filename(
|
||||
media_info, show_service=not no_source, folder=True
|
||||
)
|
||||
sidecar_dir /= sample_dir
|
||||
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
|
||||
|
||||
with console.status("Saving subtitle sidecar files..."):
|
||||
@@ -3674,11 +3769,14 @@ class dl:
|
||||
media_info = MediaInfo.parse(sample_track.path)
|
||||
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
||||
|
||||
final_dir /= sample_dir
|
||||
final_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for track_path in muxed_paths:
|
||||
media_info = MediaInfo.parse(track_path)
|
||||
base_filename = title.get_filename(media_info, show_service=not no_source)
|
||||
if sample:
|
||||
base_filename = sample_name(base_filename, title)
|
||||
|
||||
track = next((t for t in title.tracks if t.path == track_path), None)
|
||||
if track:
|
||||
@@ -3710,6 +3808,8 @@ class dl:
|
||||
media_info = MediaInfo.parse(muxed_path)
|
||||
final_dir = self.output_dir or config.directories.downloads
|
||||
final_filename = title.get_filename(media_info, show_service=not no_source)
|
||||
if sample:
|
||||
final_filename = sample_name(final_filename, title)
|
||||
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
|
||||
|
||||
if not no_folder and (
|
||||
@@ -3717,13 +3817,11 @@ class dl:
|
||||
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
||||
):
|
||||
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
||||
final_dir /= sample_dir
|
||||
|
||||
final_dir.mkdir(parents=True, exist_ok=True)
|
||||
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
||||
template_type = (
|
||||
"series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
||||
)
|
||||
sep = config.get_template_separator(template_type)
|
||||
sep = config.get_template_separator(template_type(title))
|
||||
|
||||
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
|
||||
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
|
||||
@@ -3754,6 +3852,8 @@ class dl:
|
||||
self.anime_hint(title),
|
||||
session=service.session,
|
||||
)
|
||||
if include_sample:
|
||||
self.write_sample(final_path, title, media_info)
|
||||
|
||||
post_script_context = build_context(
|
||||
title,
|
||||
@@ -3807,6 +3907,32 @@ class dl:
|
||||
|
||||
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
||||
|
||||
def write_sample(self, source: Path, title: Title_T, media_info: MediaInfo) -> None:
|
||||
"""Cut a 60 second sample from a finished file into a ``Sample`` folder next to it.
|
||||
|
||||
The sample starts 2 minutes in. A file shorter than 3 minutes gives its middle 60 seconds.
|
||||
"""
|
||||
duration = float(media_info.general_tracks[0].duration or 0) / 1000
|
||||
start, end = DEFAULT_SAMPLE
|
||||
length = end - start
|
||||
if duration < end:
|
||||
start = max(0.0, (duration - length) / 2)
|
||||
output = source.parent / SAMPLE_DIR / f"{sample_name(source.stem, title)}{source.suffix}"
|
||||
output.parent.mkdir(exist_ok=True)
|
||||
returncode = subprocess.run(
|
||||
[
|
||||
str(binaries.MKVToolNix),
|
||||
"-q",
|
||||
"-o",
|
||||
str(output),
|
||||
"--split",
|
||||
f"parts:{start:.3f}s-{start + length:.3f}s",
|
||||
source,
|
||||
]
|
||||
).returncode
|
||||
if returncode > 1 or not output.exists():
|
||||
self.log.warning(f"mkvmerge wrote no sample for {source.name}")
|
||||
|
||||
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
|
||||
"""Whether metadata lookups for this title should prefer AniList."""
|
||||
per_title = getattr(title, "anime", None)
|
||||
@@ -5454,6 +5580,7 @@ class dl:
|
||||
"""
|
||||
Get CDM for a specified service (either Local or Remote CDM).
|
||||
Now supports quality-based selection when the caller gives a quality.
|
||||
With no ``drm``, an entry gives the DRM system that it lists first.
|
||||
Raises a ValueError if there is a problem getting a CDM.
|
||||
"""
|
||||
# A per-request override (set by the REST API per job) takes precedence over the
|
||||
@@ -5534,7 +5661,8 @@ class dl:
|
||||
"pr": "playready",
|
||||
"playready": "playready",
|
||||
}.get(drm.lower())
|
||||
cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready")
|
||||
first = next(key for key in lower_keys if key in ("widevine", "playready"))
|
||||
cdm_name = lower_keys.get(drm_key or first) or lower_keys.get("playready")
|
||||
else:
|
||||
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(config.cdm, "default")
|
||||
if not cdm_name:
|
||||
|
||||
@@ -7,7 +7,7 @@ from rich_click.patch import patch as _patch_click_help
|
||||
|
||||
_patch_click_help()
|
||||
|
||||
__version__ = "5.5.0"
|
||||
__version__ = "6.0.0"
|
||||
|
||||
PKG = Path(__file__).parent.parent
|
||||
# Framework code only. Services are user-swappable, so they are not part of the identity.
|
||||
|
||||
@@ -418,6 +418,15 @@ def perform_download(
|
||||
if params.get("export"):
|
||||
params["export"] = bool(params["export"])
|
||||
|
||||
sample_raw = params.get("sample")
|
||||
if sample_raw is not None:
|
||||
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||
|
||||
try:
|
||||
params["sample"] = SAMPLE_RANGE.convert(sample_raw, None, None)
|
||||
except click.BadParameter as exc:
|
||||
raise Exception(f"Invalid sample parameter: {exc}")
|
||||
|
||||
slow_raw = params.get("slow")
|
||||
if slow_raw is not None and not isinstance(slow_raw, tuple):
|
||||
if isinstance(slow_raw, bool):
|
||||
@@ -491,6 +500,7 @@ def perform_download(
|
||||
"v_lang": params.get("v_lang", []),
|
||||
"a_lang": params.get("a_lang", []),
|
||||
"forced_subs": params.get("forced_subs", False),
|
||||
"exact_lang": params.get("exact_lang", False),
|
||||
}
|
||||
# Hand-built context: record parameter sources so service dl overrides
|
||||
# apply to defaults but never clobber client-sent values.
|
||||
@@ -633,6 +643,8 @@ def perform_download(
|
||||
no_folder=params.get("no_folder", False),
|
||||
no_source=params.get("no_source", False),
|
||||
no_mux=params.get("no_mux", False),
|
||||
sample=params.get("sample"),
|
||||
include_sample=params.get("include_sample", False),
|
||||
workers=params.get("workers"),
|
||||
adaptive_workers=params.get("adaptive_workers", False),
|
||||
download_processes=params.get("download_processes", 1),
|
||||
|
||||
@@ -31,6 +31,7 @@ from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_C
|
||||
from unshackle.core.providers.anilist import parse_anilist_ref
|
||||
from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.titles import Episode, Movie, Song, Title_T
|
||||
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
|
||||
from unshackle.core.utilities import declared_kwargs, kind_only_conflict
|
||||
@@ -92,6 +93,8 @@ DEFAULT_DOWNLOAD_PARAMS = {
|
||||
"no_folder": False,
|
||||
"no_source": False,
|
||||
"no_mux": False,
|
||||
"sample": None,
|
||||
"include_sample": False,
|
||||
"workers": None,
|
||||
"adaptive_workers": False,
|
||||
"download_processes": 1,
|
||||
@@ -154,7 +157,8 @@ def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str]
|
||||
drm_key = {"wv": "widevine", "widevine": "widevine", "pr": "playready", "playready": "playready"}.get(
|
||||
cdm_type.lower()
|
||||
)
|
||||
cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready")
|
||||
first = next(key for key in lower_keys if key in ("widevine", "playready"))
|
||||
cdm_name = lower_keys.get(drm_key or first) or lower_keys.get("playready")
|
||||
else:
|
||||
cdm_name = cdm_name.get(profile) or cdm_name.get("default") or ci_get(app_config.cdm, "default")
|
||||
|
||||
@@ -1831,6 +1835,16 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]:
|
||||
except APIError as e:
|
||||
return e.message
|
||||
|
||||
if data.get("sample") is not None:
|
||||
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||
|
||||
try:
|
||||
SAMPLE_RANGE.convert(data["sample"], None, None)
|
||||
except click.BadParameter:
|
||||
return "sample must be true or a 'START-END' window in seconds, such as '60-120'."
|
||||
if data.get("sample") and data.get("include_sample"):
|
||||
return "sample and include_sample cannot be used together."
|
||||
|
||||
# The serve user would create and write any directory a client names; keep it under downloads.
|
||||
if data.get("output_dir"):
|
||||
root = config.directories.downloads.resolve()
|
||||
@@ -3264,6 +3278,7 @@ def forwarded_dl_params(data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
except click.BadParameter:
|
||||
log.warning(f"Ignoring unknown dl_params.acodec value: {sanitize_log(name)}")
|
||||
params["forced_subs"] = raw.get("forced_subs") is True
|
||||
params["exact_lang"] = raw.get("exact_lang") is True
|
||||
return params
|
||||
|
||||
|
||||
@@ -3901,7 +3916,8 @@ def resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional[
|
||||
Checks the server's own CDM config (``config.cdm[service]``) to
|
||||
determine the CDM type without loading the full CDM object. This
|
||||
makes sure that when you use ``server_cdm: true``, the server's CDM
|
||||
determines device selection (e.g. PlayReady vs Widevine).
|
||||
determines device selection (e.g. PlayReady vs Widevine). An entry with
|
||||
``widevine`` and ``playready`` gives the DRM system that it lists first.
|
||||
|
||||
Falls back to a lightweight stub from *cdm_type* only if the configuration
|
||||
has no server CDM for the service.
|
||||
@@ -3913,7 +3929,8 @@ def resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional[
|
||||
if isinstance(cdm_name, dict):
|
||||
lower_keys = {k.lower(): v for k, v in cdm_name.items()}
|
||||
if {"widevine", "playready"} & lower_keys.keys():
|
||||
cdm_name = lower_keys.get("playready") or lower_keys.get("widevine")
|
||||
systems = [lower_keys[key] for key in lower_keys if key in ("widevine", "playready")]
|
||||
cdm_name = next(filter(None, systems), None)
|
||||
else:
|
||||
cdm_name = cdm_name.get("default") or next(iter(cdm_name.values()), None)
|
||||
|
||||
@@ -4311,11 +4328,11 @@ def ensure_track_drm(track: Any, session: Any = None, init_data: Optional[bytes]
|
||||
from unshackle.core.manifests import HLS
|
||||
|
||||
http = getattr(track, "session", None) or session
|
||||
playlist = m3u8.loads(http.get(track.url).text, uri=track.url) if http else m3u8.load(track.url)
|
||||
playlist = m3u8.loads(utf8_text(http.get(track.url)), uri=track.url) if http else m3u8.load(track.url)
|
||||
keys = [k for k in (playlist.keys or []) + (playlist.session_keys or []) if k is not None]
|
||||
for key in keys:
|
||||
try:
|
||||
drm = HLS.get_drm(key)
|
||||
drm = HLS.get_drm_or_probe(key, playlist, track, probe_session=http)
|
||||
if isinstance(drm, (Widevine, PlayReady)):
|
||||
track.drm = [drm]
|
||||
return
|
||||
@@ -4791,14 +4808,11 @@ async def session_segment_filter_handler(
|
||||
|
||||
def unwanted_segments() -> list[str]:
|
||||
import m3u8
|
||||
import requests
|
||||
|
||||
response = session.service_instance.session.get(track.url)
|
||||
if not getattr(response, "ok", True):
|
||||
raise ValueError(f"playlist request failed: {response.status_code}")
|
||||
if isinstance(response, requests.Response):
|
||||
response.encoding = response.encoding or "utf-8"
|
||||
playlist = m3u8.loads(response.text, uri=str(track.url))
|
||||
playlist = m3u8.loads(utf8_text(response), uri=str(track.url))
|
||||
return [segment.absolute_uri for segment in playlist.segments if segment_filter(segment)]
|
||||
|
||||
try:
|
||||
|
||||
@@ -726,7 +726,7 @@ async def list_tracks(request: web.Request) -> web.Response:
|
||||
dl_params:
|
||||
type: object
|
||||
description: |
|
||||
The dl track selection (lang, v_lang, a_lang, acodec, forced_subs), in the same
|
||||
The dl track selection (lang, v_lang, a_lang, acodec, forced_subs, exact_lang), in the same
|
||||
form as on `/api/session/create`. The service reads it from `ctx.parent.params`.
|
||||
credentials:
|
||||
type: object
|
||||
@@ -1005,6 +1005,14 @@ async def download(request: web.Request) -> web.Response:
|
||||
no_mux:
|
||||
type: boolean
|
||||
description: Do not mux tracks into a container file (default - false)
|
||||
include_sample:
|
||||
type: boolean
|
||||
description: Also cut a 60 second sample from each muxed file into a Sample folder next to it (default - false)
|
||||
sample:
|
||||
oneOf:
|
||||
- type: boolean
|
||||
- type: string
|
||||
description: Download a short sample and write it to a Sample folder. `true` for 60 seconds from 2 minutes in, or a `"START-END"` string in seconds (e.g., `"0-60"`) (default - null)
|
||||
workers:
|
||||
type: integer
|
||||
description: Max workers/threads per track download (default - None)
|
||||
@@ -1823,6 +1831,9 @@ async def session_create(request: web.Request) -> web.Response:
|
||||
forced_subs:
|
||||
type: boolean
|
||||
default: false
|
||||
exact_lang:
|
||||
type: boolean
|
||||
default: false
|
||||
server_cdm:
|
||||
type: boolean
|
||||
description: |
|
||||
|
||||
@@ -193,6 +193,7 @@ class Config:
|
||||
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
||||
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
||||
self.continue_downloads: bool = kwargs.get("continue_downloads", False)
|
||||
self.sample: Optional[tuple[int, int]] = None
|
||||
self.post_scripts: list = kwargs.get("post_scripts") or []
|
||||
|
||||
self.language_tags: dict = kwargs.get("language_tags") or {}
|
||||
|
||||
@@ -3,6 +3,8 @@ from typing import TypeVar, Union
|
||||
|
||||
DOWNLOAD_CANCELLED = Event()
|
||||
DOWNLOAD_LICENCE_ONLY = Event()
|
||||
|
||||
DEFAULT_SAMPLE = (120, 180)
|
||||
DOWNLOAD_ALL_DRM = Event()
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.utilities import get_boxes, log_event
|
||||
from unshackle.core.utils.subprocess import ffprobe
|
||||
|
||||
@@ -180,7 +181,7 @@ class PlayReady:
|
||||
|
||||
if track.descriptor == track.Descriptor.HLS:
|
||||
m3u_url = track.url
|
||||
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
|
||||
master = m3u8.loads(utf8_text(session.get(m3u_url)), uri=m3u_url)
|
||||
pssh_boxes.extend(
|
||||
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
|
||||
for x in (master.session_keys or master.keys)
|
||||
|
||||
@@ -22,6 +22,7 @@ from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import AnyTrack
|
||||
from unshackle.core.drm import key_args as drm_key_args
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.utilities import get_boxes, log_event
|
||||
from unshackle.core.utils.subprocess import ffprobe
|
||||
|
||||
@@ -94,7 +95,7 @@ class Widevine:
|
||||
|
||||
if track.descriptor == track.Descriptor.HLS:
|
||||
m3u_url = track.url
|
||||
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
|
||||
master = m3u8.loads(utf8_text(session.get(m3u_url)), uri=m3u_url)
|
||||
pssh_boxes.extend(
|
||||
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
|
||||
for x in (master.session_keys or master.keys)
|
||||
|
||||
@@ -25,6 +25,7 @@ from unshackle.core.remote_service import (
|
||||
resolve_proxy_arg,
|
||||
segment_uri_filter,
|
||||
)
|
||||
from unshackle.core.session import utf8_text
|
||||
from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
|
||||
from unshackle.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video
|
||||
from unshackle.core.tracks.attachment import Attachment
|
||||
@@ -67,9 +68,10 @@ def hls_playlist_kids(track: Any, session: Optional[requests.Session] = None) ->
|
||||
key from its URI, so such a track takes no exported content key. An empty set means that
|
||||
the track is encrypted but names no KID that unshackle can read.
|
||||
"""
|
||||
http = session or requests.Session()
|
||||
try:
|
||||
text = (session or requests.Session()).get(str(track.url)).text
|
||||
keys = [k for k in m3u8.loads(text, uri=str(track.url)).keys if k is not None and k.method != "NONE"]
|
||||
playlist = m3u8.loads(utf8_text(http.get(str(track.url))), uri=str(track.url))
|
||||
keys = [k for k in playlist.keys if k is not None and k.method != "NONE"]
|
||||
except Exception as e:
|
||||
log.warning(f"Cannot read the media playlist of {track.id} for its KIDs: {e!r}")
|
||||
return None
|
||||
@@ -78,7 +80,7 @@ def hls_playlist_kids(track: Any, session: Optional[requests.Session] = None) ->
|
||||
if key.method == "AES-128":
|
||||
continue
|
||||
try:
|
||||
kids.update(real_kids(HLS.get_drm(key)))
|
||||
kids.update(real_kids(HLS.get_drm_or_probe(key, playlist, track, probe_session=http)))
|
||||
except Exception as e:
|
||||
log.debug(f"Skipping HLS key {key.keyformat or key.method} of {track.id}: {e!r}")
|
||||
if kids or any(k.method != "AES-128" for k in keys):
|
||||
|
||||
@@ -19,7 +19,7 @@ from zlib import crc32
|
||||
|
||||
import requests
|
||||
from langcodes import Language, tag_is_valid
|
||||
from lxml.etree import Element, ElementTree
|
||||
from lxml.etree import Element, ElementTree, XMLSyntaxError
|
||||
from pyplayready.system.pssh import PSSH as PR_PSSH
|
||||
from pywidevine.cdm import Cdm as WidevineCdm
|
||||
from pywidevine.pssh import PSSH
|
||||
@@ -34,7 +34,11 @@ from unshackle.core.drm.verify import decrypt_track
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.session import RnetSession
|
||||
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
||||
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
|
||||
from unshackle.core.tracks.track import (
|
||||
DRM_PREFERENCE_TYPES,
|
||||
assert_fragments_decrypted,
|
||||
parse_sidx,
|
||||
)
|
||||
from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8
|
||||
from unshackle.core.utils.redact import safe_display_url
|
||||
from unshackle.core.utils.xml import load_xml
|
||||
@@ -114,6 +118,7 @@ class DASH:
|
||||
|
||||
@classmethod
|
||||
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH:
|
||||
"""Fetch and parse an MPD. The XML declaration gives the encoding, then the HTTP charset."""
|
||||
if not url:
|
||||
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
|
||||
if not isinstance(url, str):
|
||||
@@ -134,11 +139,15 @@ class DASH:
|
||||
log_event(
|
||||
"manifest_dash_fetch",
|
||||
level="DEBUG",
|
||||
message=f"Fetched DASH manifest ({len(res.text)} bytes)",
|
||||
context={"url": safe_display_url(url), "size": len(res.text)},
|
||||
message=f"Fetched DASH manifest ({len(res.content)} bytes)",
|
||||
context={"url": safe_display_url(url), "size": len(res.content)},
|
||||
)
|
||||
|
||||
return DASH.from_text(res.text, url)
|
||||
try:
|
||||
return cls(load_xml(res.content), url)
|
||||
except XMLSyntaxError:
|
||||
# no declaration and not UTF-8: only the HTTP charset can name the encoding
|
||||
return cls.from_text(res.text, url)
|
||||
|
||||
@classmethod
|
||||
def from_text(cls, text: str, url: str) -> DASH:
|
||||
@@ -458,6 +467,7 @@ class DASH:
|
||||
track_url=track.url,
|
||||
session=session,
|
||||
probe_kid=period_idx == 0,
|
||||
window=config.sample,
|
||||
)
|
||||
|
||||
if period_idx == 0:
|
||||
@@ -471,12 +481,20 @@ class DASH:
|
||||
seen_segments.add(seg)
|
||||
segments.append(seg)
|
||||
segment_durations.extend(p_durations)
|
||||
# ponytail: a sample comes from the first content period only; map the window across
|
||||
# periods if a manifest with a short first period needs it
|
||||
if config.sample and segments:
|
||||
break
|
||||
|
||||
if not segments:
|
||||
log.error("Could not find a way to get segments from this MPD manifest.")
|
||||
log.debug(track.url)
|
||||
sys.exit(1)
|
||||
|
||||
if config.sample and not segment_durations and not isinstance(track, Subtitle):
|
||||
log.error("--sample cannot cut this track: the manifest gives it as one file with no segment index.")
|
||||
sys.exit(1)
|
||||
|
||||
# TODO: Should we floor/ceil/round, or is int() ok?
|
||||
track.data["dash"]["timescale"] = int(segment_timescale)
|
||||
track.data["dash"]["segment_durations"] = segment_durations
|
||||
@@ -533,7 +551,7 @@ class DASH:
|
||||
# When every segment is a byte range of one parent resource and init is its [0, len)
|
||||
# prefix, the whole resource is itself a valid MP4, so one direct download to save_path
|
||||
# replaces the per-segment files, init write and merge pass.
|
||||
collapse_single_url = DASH.collapsible_single_url(
|
||||
collapse_single_url = not config.sample and DASH.collapsible_single_url(
|
||||
isinstance(track, Subtitle),
|
||||
segments,
|
||||
len(init_data) if init_data is not None else None,
|
||||
@@ -753,6 +771,26 @@ class DASH:
|
||||
|
||||
progress(downloaded="Downloaded")
|
||||
|
||||
@staticmethod
|
||||
def find_segment_element(name: str, adaptation_set: Element, representation: Element) -> Optional[Element]:
|
||||
"""Find a segment description element on the representation, or else on its adaptation set."""
|
||||
element = representation.find(name)
|
||||
return element if element is not None else adaptation_set.find(name)
|
||||
|
||||
@staticmethod
|
||||
def has_segment_timing(track: AnyTrack) -> bool:
|
||||
"""Return False when the manifest gives the track as one file with no index, so a sample cannot cut it."""
|
||||
dash = track.data.get("dash") or {}
|
||||
representation, adaptation_set = dash.get("representation"), dash.get("adaptation_set")
|
||||
if representation is None or adaptation_set is None:
|
||||
return True
|
||||
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
||||
return (
|
||||
DASH.merge_segment_templates(adaptation_set, representation) is not None
|
||||
or DASH.find_segment_element("SegmentList", adaptation_set, representation) is not None
|
||||
or (segment_base is not None and bool(segment_base.get("indexRange")))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def collapsible_single_url(
|
||||
is_subtitle: bool,
|
||||
@@ -844,6 +882,7 @@ class DASH:
|
||||
track_url: str,
|
||||
session: Union[Session, RnetSession],
|
||||
probe_kid: bool = True,
|
||||
window: Optional[tuple[int, int]] = None,
|
||||
) -> tuple[
|
||||
Optional[bytes],
|
||||
list[tuple[str, Optional[str]]],
|
||||
@@ -877,13 +916,9 @@ class DASH:
|
||||
|
||||
segment_template = DASH.merge_segment_templates(adaptation_set, representation)
|
||||
|
||||
segment_list = representation.find("SegmentList")
|
||||
if segment_list is None:
|
||||
segment_list = adaptation_set.find("SegmentList")
|
||||
segment_list = DASH.find_segment_element("SegmentList", adaptation_set, representation)
|
||||
|
||||
segment_base = representation.find("SegmentBase")
|
||||
if segment_base is None:
|
||||
segment_base = adaptation_set.find("SegmentBase")
|
||||
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
||||
|
||||
segments: list[tuple[str, Optional[str]]] = []
|
||||
segment_timescale: float = 0
|
||||
@@ -930,6 +965,10 @@ class DASH:
|
||||
segment_durations.append(current_time)
|
||||
current_time += int(s.get("d"))
|
||||
|
||||
keep = track.sample_slice(segment_durations, segment_timescale, window) if window else slice(None)
|
||||
start_number += keep.start or 0
|
||||
segment_durations = segment_durations[keep]
|
||||
|
||||
if not end_number:
|
||||
end_number = start_number + len(segment_durations) - 1
|
||||
|
||||
@@ -956,7 +995,11 @@ class DASH:
|
||||
segment_count = math.ceil(period_duration / (segment_duration / segment_timescale))
|
||||
end_number = start_number + segment_count - 1
|
||||
|
||||
for s in range(start_number, end_number + 1):
|
||||
numbers = range(start_number, end_number + 1)
|
||||
if window:
|
||||
starts = [n * segment_duration for n in range(len(numbers))]
|
||||
numbers = numbers[track.sample_slice(starts, segment_timescale, window)]
|
||||
for s in numbers:
|
||||
segments.append(
|
||||
(
|
||||
DASH.replace_fields(
|
||||
@@ -994,6 +1037,10 @@ class DASH:
|
||||
track_kid = track.get_key_id(init_data)
|
||||
|
||||
segment_urls = segment_list.findall("SegmentURL")
|
||||
list_duration = float(segment_list.get("duration") or 0)
|
||||
if window and list_duration:
|
||||
starts = [n * list_duration for n in range(len(segment_urls))]
|
||||
segment_urls = segment_urls[track.sample_slice(starts, segment_timescale, window)]
|
||||
for segment_url in segment_urls:
|
||||
media_url = segment_url.get("media")
|
||||
if not media_url:
|
||||
@@ -1022,16 +1069,25 @@ class DASH:
|
||||
if total_size:
|
||||
media_range = f"{len(init_data)}-{total_size}"
|
||||
|
||||
segments.append((rep_base_url, media_range))
|
||||
subsegments: list[tuple[int, int, int]] = []
|
||||
index_range = segment_base.get("indexRange")
|
||||
if window and index_range and not isinstance(track, Subtitle):
|
||||
res = session.get(url=rep_base_url, headers={"Range": f"bytes={index_range}"})
|
||||
res.raise_for_status()
|
||||
segment_timescale, subsegments = parse_sidx(res.content, int(index_range.split("-")[0]))
|
||||
|
||||
if subsegments:
|
||||
segment_durations = [time for time, _, _ in subsegments]
|
||||
keep = track.sample_slice(segment_durations, segment_timescale, window)
|
||||
segment_durations = segment_durations[keep]
|
||||
segments.extend((rep_base_url, f"{first}-{last}") for _, first, last in subsegments[keep])
|
||||
else:
|
||||
segments.append((rep_base_url, media_range))
|
||||
elif rep_base_url:
|
||||
segments.append((rep_base_url, None))
|
||||
|
||||
return init_data, segments, segment_timescale, segment_durations, track_kid
|
||||
|
||||
# Deprecated 5.5.0 shims for service repos still on the old underscored names; drop once they migrate.
|
||||
_is_content_period = is_content_period
|
||||
_get_period_segments = get_period_segments
|
||||
|
||||
@staticmethod
|
||||
def get_attr(item: str, adaptation_set: Element, representation: Optional[Element] = None) -> Optional[Any]:
|
||||
"""Helper to get a requested item from the Representation, otherwise from the AdaptationSet."""
|
||||
|
||||
@@ -11,6 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
from contextlib import ExitStack, closing
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
@@ -36,7 +37,7 @@ from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
|
||||
from unshackle.core.drm.verify import decrypt_track
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.manifests.dash import RollingMerge
|
||||
from unshackle.core.session import RnetResponse, RnetSession
|
||||
from unshackle.core.session import RnetResponse, RnetSession, utf8_text
|
||||
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
||||
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
|
||||
from unshackle.core.utilities import get_extension, is_close_match, log_event, try_ensure_utf8
|
||||
@@ -80,17 +81,11 @@ class HLS:
|
||||
|
||||
res = session.get(url, **args)
|
||||
|
||||
if isinstance(res, requests.Response):
|
||||
if not res.ok:
|
||||
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
|
||||
res.encoding = res.encoding or "utf-8"
|
||||
content = res.text
|
||||
elif isinstance(res, RnetResponse):
|
||||
if not res.ok:
|
||||
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
|
||||
content = res.text
|
||||
else:
|
||||
if not isinstance(res, (requests.Response, RnetResponse)):
|
||||
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}")
|
||||
if not res.ok:
|
||||
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
|
||||
content = utf8_text(res)
|
||||
|
||||
master = m3u8.loads(content, uri=url)
|
||||
|
||||
@@ -396,9 +391,7 @@ class HLS:
|
||||
session = Session()
|
||||
|
||||
res = session.get(variant_url)
|
||||
if isinstance(res, requests.Response):
|
||||
res.encoding = res.encoding or "utf-8"
|
||||
variant = m3u8.loads(res.text, uri=variant_url)
|
||||
variant = m3u8.loads(utf8_text(res), uri=variant_url)
|
||||
if not variant.segments:
|
||||
return None
|
||||
|
||||
@@ -658,17 +651,13 @@ class HLS:
|
||||
master = m3u8.load(str(track.from_file))
|
||||
else:
|
||||
response = session.get(track.url)
|
||||
if isinstance(response, requests.Response) or isinstance(response, RnetResponse):
|
||||
if not response.ok:
|
||||
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
|
||||
sys.exit(1)
|
||||
if isinstance(response, requests.Response):
|
||||
response.encoding = response.encoding or "utf-8"
|
||||
playlist_text = response.text
|
||||
else:
|
||||
if not isinstance(response, (requests.Response, RnetResponse)):
|
||||
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}")
|
||||
if not response.ok:
|
||||
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
|
||||
sys.exit(1)
|
||||
|
||||
master = m3u8.loads(playlist_text, uri=track.url)
|
||||
master = m3u8.loads(utf8_text(response), uri=track.url)
|
||||
|
||||
if not master.segments:
|
||||
log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.")
|
||||
@@ -685,6 +674,12 @@ class HLS:
|
||||
)
|
||||
hls_data["unwanted_segments"] = [s.absolute_uri for s in unwanted_segments]
|
||||
|
||||
if config.sample:
|
||||
content = [segment for segment in master.segments if segment not in unwanted_segments]
|
||||
starts = list(accumulate((segment.duration or 0 for segment in content[:-1]), initial=0.0))
|
||||
del content[track.sample_slice(starts, window=config.sample)]
|
||||
unwanted_segments += content
|
||||
|
||||
# Get session DRM as fallback but prefer media playlist keys for accurate KID matching
|
||||
if track.drm:
|
||||
session_drm = track.get_drm_for_cdm(cdm)
|
||||
@@ -699,9 +694,14 @@ class HLS:
|
||||
media_playlist_key = HLS.get_supported_key(cdm_media_keys) if cdm_media_keys else None
|
||||
|
||||
if media_playlist_key:
|
||||
media_drm = HLS.get_drm(media_playlist_key, session)
|
||||
init_kid = (
|
||||
None
|
||||
if media_playlist_key.method == "AES-128"
|
||||
else HLS.get_track_kid_from_init(master, track, session)
|
||||
)
|
||||
media_drm = HLS.get_drm(media_playlist_key, session, **({"kid": init_kid} if init_kid else {}))
|
||||
if isinstance(media_drm, (Widevine, PlayReady)):
|
||||
track_kid = HLS.get_track_kid_from_init(master, track, session) or media_drm.kid
|
||||
track_kid = init_kid or media_drm.kid
|
||||
# Preserve pre-existing keys (e.g. from server_cdm)
|
||||
if track.drm:
|
||||
for existing_drm in track.drm:
|
||||
@@ -709,7 +709,7 @@ class HLS:
|
||||
media_drm.content_keys.update(existing_drm.content_keys)
|
||||
track.drm = [media_drm]
|
||||
if DOWNLOAD_ALL_DRM.is_set():
|
||||
track.drm += HLS.alternate_drm(media_keys, media_drm, session)
|
||||
track.drm += HLS.alternate_drm(media_keys, media_drm, session, init_kid)
|
||||
try:
|
||||
if not license_widevine:
|
||||
raise ValueError("license_widevine func must be supplied to use DRM")
|
||||
@@ -765,7 +765,7 @@ class HLS:
|
||||
if key is None or key.method == "AES-128" or key in licensed_keys:
|
||||
continue
|
||||
licensed_keys.append(key)
|
||||
later_drm = HLS.get_drm(key, session)
|
||||
later_drm = HLS.get_drm_or_probe(key, master, track, session, segment.init_section)
|
||||
if isinstance(later_drm, (Widevine, PlayReady)) and license_widevine:
|
||||
license_widevine(later_drm, track_kid=later_drm.kid)
|
||||
progress(downloaded="[yellow]SKIPPED")
|
||||
@@ -796,17 +796,17 @@ class HLS:
|
||||
|
||||
range_offset = 0
|
||||
for segment in master.segments:
|
||||
if segment.byterange:
|
||||
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
|
||||
range_offset = int(byte_range.split("-")[1]) + 1
|
||||
else:
|
||||
byte_range = None
|
||||
|
||||
if segment in unwanted_segments:
|
||||
continue
|
||||
|
||||
segment_durations.append(int(segment.duration))
|
||||
|
||||
if segment.byterange:
|
||||
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
|
||||
range_offset = int(byte_range.split("-")[0])
|
||||
else:
|
||||
byte_range = None
|
||||
|
||||
urls.append(
|
||||
{
|
||||
"url": urljoin(segment.base_uri, segment.uri),
|
||||
@@ -1145,22 +1145,18 @@ class HLS:
|
||||
if key is None:
|
||||
encryption_data = None
|
||||
elif not encryption_data or encryption_data[0] != key:
|
||||
drm = HLS.get_drm(key, session)
|
||||
if isinstance(drm, (Widevine, PlayReady)):
|
||||
try:
|
||||
if map_data:
|
||||
track_kid = track.get_key_id(map_data[1])
|
||||
else:
|
||||
track_kid = None
|
||||
if not track_kid:
|
||||
track_kid = drm.kid
|
||||
try:
|
||||
init_kid = track.get_key_id(map_data[1]) if map_data and key.method != "AES-128" else None
|
||||
drm = HLS.get_drm(key, session, **({"kid": init_kid} if init_kid else {}))
|
||||
if isinstance(drm, (Widevine, PlayReady)):
|
||||
track_kid = init_kid or drm.kid
|
||||
progress(downloaded="LICENSING")
|
||||
license_widevine(drm, track_kid=track_kid)
|
||||
progress(downloaded="[yellow]LICENSED")
|
||||
except Exception: # noqa
|
||||
DOWNLOAD_CANCELLED.set() # skip pending track downloads
|
||||
progress(downloaded="[red]FAILED")
|
||||
raise
|
||||
except Exception: # noqa
|
||||
DOWNLOAD_CANCELLED.set() # skip pending track downloads
|
||||
progress(downloaded="[red]FAILED")
|
||||
raise
|
||||
if (
|
||||
encryption_data
|
||||
and isinstance(drm, (Widevine, PlayReady))
|
||||
@@ -1377,8 +1373,7 @@ class HLS:
|
||||
if not value and data.uri:
|
||||
if not session:
|
||||
session = Session()
|
||||
res = session.get(urljoin(manifest.base_uri or "", data.uri))
|
||||
value = res.text
|
||||
value = utf8_text(session.get(urljoin(manifest.base_uri or "", data.uri)))
|
||||
|
||||
if not value:
|
||||
continue
|
||||
@@ -1455,6 +1450,7 @@ class HLS:
|
||||
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]],
|
||||
chosen: DRM_T,
|
||||
session: Optional[Union[Session, RnetSession]] = None,
|
||||
kid: Optional[UUID] = None,
|
||||
) -> list[DRM_T]:
|
||||
"""Build the Widevine or PlayReady DRM that the EXT-X-KEY entries offer next to ``chosen``.
|
||||
|
||||
@@ -1465,7 +1461,7 @@ class HLS:
|
||||
if not other_keys:
|
||||
return []
|
||||
try:
|
||||
return [HLS.get_drm(other_keys[0], session)]
|
||||
return [HLS.get_drm(other_keys[0], session, **({"kid": kid} if kid else {}))]
|
||||
except Exception as e:
|
||||
logging.getLogger("HLS").warning(f"Could not read the {system} EXT-X-KEY entry: {e}")
|
||||
return []
|
||||
@@ -1475,14 +1471,15 @@ class HLS:
|
||||
master: M3U8,
|
||||
track: AnyTrack,
|
||||
session: Union[Session, RnetSession],
|
||||
init_section: Optional[m3u8.model.InitializationSection] = None,
|
||||
) -> Optional[UUID]:
|
||||
"""
|
||||
Extract the track's Key ID from its init segment (EXT-X-MAP).
|
||||
Extract the track's Key ID from an init segment (EXT-X-MAP), by default the first in the playlist.
|
||||
|
||||
Returns None if no init segment exists or KID extraction fails.
|
||||
The caller should fall back to drm.kid from the PSSH if this returns None.
|
||||
"""
|
||||
map_section = next((seg.init_section for seg in master.segments if seg.init_section), None)
|
||||
map_section = init_section or next((seg.init_section for seg in master.segments if seg.init_section), None)
|
||||
if not map_section:
|
||||
return None
|
||||
|
||||
@@ -1496,6 +1493,10 @@ class HLS:
|
||||
map_res = session.get(url=map_uri, headers=headers)
|
||||
if map_res.ok:
|
||||
return track.get_key_id(map_res.content)
|
||||
logging.getLogger("HLS").warning(
|
||||
f"Could not read the Key ID from the init segment {safe_display_url(map_uri)}: "
|
||||
f"HTTP {map_res.status_code}"
|
||||
)
|
||||
# best-effort key-id probe over network + init parse; caller falls back without it
|
||||
except Exception as e:
|
||||
logging.getLogger("HLS").debug(f"Init map key-id probe failed for {map_uri}: {e!r}")
|
||||
@@ -1543,6 +1544,7 @@ class HLS:
|
||||
def get_drm(
|
||||
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
|
||||
session: Optional[Union[Session, RnetSession]] = None,
|
||||
kid: Optional[UUID] = None,
|
||||
) -> DRM_T:
|
||||
"""
|
||||
Convert HLS EXT-X-KEY data to an initialized DRM object.
|
||||
@@ -1551,6 +1553,8 @@ class HLS:
|
||||
key: The m3u8 `EXT-X-KEY` object.
|
||||
session: Optional HTTP session used to request AES-128 URIs.
|
||||
Useful to set headers, proxies, cookies, and so forth.
|
||||
kid: The Key ID from the init segment. Widevine uses it only when the tag
|
||||
and the PSSH both carry no real Key ID.
|
||||
|
||||
Raises a NotImplementedError when unshackle cannot use the DRM system.
|
||||
"""
|
||||
@@ -1566,13 +1570,16 @@ class HLS:
|
||||
drm = Widevine(pssh=WV_PSSH.new(key_ids=[key.uri.split(",")[-1]], system_id=WV_PSSH.SystemId.Widevine))
|
||||
elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn:
|
||||
extra_params = dict(key._extra_params) # noqa
|
||||
# a v0 PSSH carries no KID, leaving the tag's own KEYID as the only source
|
||||
kid = extra_params.pop("keyid", None) or extra_params.pop("kid", None)
|
||||
if isinstance(kid, str):
|
||||
kid = kid.removeprefix("0x").removeprefix("0X")
|
||||
pssh = WV_PSSH(key.uri.split(",")[-1])
|
||||
if kid in Widevine.PLACEHOLDER_KIDS or any(k not in Widevine.PLACEHOLDER_KIDS for k in pssh.key_ids):
|
||||
kid = None
|
||||
# a v0 PSSH can carry no KID: the tag's own KEYID is the first source, the init segment the last
|
||||
tag_kid = extra_params.pop("keyid", None) or extra_params.pop("kid", None)
|
||||
if isinstance(tag_kid, str):
|
||||
tag_kid = tag_kid.removeprefix("0x").removeprefix("0X")
|
||||
drm = Widevine(
|
||||
pssh=WV_PSSH(key.uri.split(",")[-1]),
|
||||
kid=kid,
|
||||
pssh=pssh,
|
||||
kid=tag_kid or kid,
|
||||
**extra_params,
|
||||
)
|
||||
elif key.keyformat and key.keyformat.lower() in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}:
|
||||
@@ -1585,6 +1592,28 @@ class HLS:
|
||||
|
||||
return drm
|
||||
|
||||
@staticmethod
|
||||
def get_drm_or_probe(
|
||||
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
|
||||
master: M3U8,
|
||||
track: AnyTrack,
|
||||
session: Optional[Union[Session, RnetSession]] = None,
|
||||
init_section: Optional[m3u8.model.InitializationSection] = None,
|
||||
probe_session: Any = None,
|
||||
) -> DRM_T:
|
||||
"""
|
||||
Convert HLS EXT-X-KEY data to a DRM object, with the Key ID of the init segment when the key carries none.
|
||||
|
||||
The init segment request uses `probe_session` when the caller has no HTTP session that `get_drm` accepts.
|
||||
"""
|
||||
try:
|
||||
return HLS.get_drm(key, session)
|
||||
except Widevine.Exceptions.KIDNotFound:
|
||||
kid = HLS.get_track_kid_from_init(master, track, probe_session or session or Session(), init_section)
|
||||
if not kid:
|
||||
raise
|
||||
return HLS.get_drm(key, session, kid=kid)
|
||||
|
||||
@staticmethod
|
||||
def get_all_drm(
|
||||
keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], proxy: Optional[str] = None
|
||||
@@ -1610,6 +1639,10 @@ class HLS:
|
||||
drm_objects.append(drm)
|
||||
except NotImplementedError:
|
||||
unsupported_keys.append(key)
|
||||
except Widevine.Exceptions.KIDNotFound:
|
||||
logging.getLogger("HLS").warning(
|
||||
"Skipped an EXT-X-SESSION-KEY entry that carries no Key ID. The media playlist must supply the key."
|
||||
)
|
||||
|
||||
if not drm_objects and unsupported_keys:
|
||||
logging.debug(
|
||||
|
||||
@@ -259,6 +259,7 @@ class ISM:
|
||||
)
|
||||
|
||||
track_urls: list[str] = []
|
||||
track_times: list[int] = []
|
||||
fragment_time = 0
|
||||
fragments = stream_index.findall("c")
|
||||
# MS-SSTR UrlPattern; regex over str.format so {Bitrate}/{start_time}
|
||||
@@ -276,6 +277,7 @@ class ISM:
|
||||
first_time = int(fragments[0].get("t") or 0)
|
||||
if first_time != 0:
|
||||
track_urls.append(re.sub(r"\{start[ _]time\}", "0", url_template))
|
||||
track_times.append(first_time)
|
||||
|
||||
for idx, frag in enumerate(fragments):
|
||||
fragment_time = int(frag.get("t", fragment_time))
|
||||
@@ -292,6 +294,7 @@ class ISM:
|
||||
duration_frag = (next_time - fragment_time) // repeat
|
||||
for _ in range(repeat):
|
||||
track_urls.append(re.sub(r"\{start[ _]time\}", str(fragment_time), url_template))
|
||||
track_times.append(fragment_time)
|
||||
fragment_time += duration_frag
|
||||
|
||||
track_id = hashlib.md5(
|
||||
@@ -312,6 +315,8 @@ class ISM:
|
||||
"stream_index": stream_index,
|
||||
"quality_level": ql,
|
||||
"segments": track_urls,
|
||||
"segment_times": track_times,
|
||||
"timescale": int(stream_index.get("TimeScale") or self.manifest.get("TimeScale") or 10000000),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +419,9 @@ class ISM:
|
||||
session.proxies.update({"all": proxy})
|
||||
|
||||
segments: list[str] = track.data["ism"]["segments"]
|
||||
segment_times: list[int] = track.data["ism"].get("segment_times") or []
|
||||
if segment_times:
|
||||
segments = segments[track.sample_slice(segment_times, track.data["ism"]["timescale"], config.sample)]
|
||||
|
||||
session_drm = None
|
||||
if track.drm:
|
||||
|
||||
@@ -1073,8 +1073,9 @@ class RemoteService:
|
||||
}
|
||||
if params.get("acodec") is not None:
|
||||
dl_params["acodec"] = [c.name for c in params["acodec"]]
|
||||
if "forced_subs" in params:
|
||||
dl_params["forced_subs"] = bool(params["forced_subs"])
|
||||
for key in ("forced_subs", "exact_lang"):
|
||||
if key in params:
|
||||
dl_params[key] = bool(params[key])
|
||||
if dl_params:
|
||||
create_data["dl_params"] = dl_params
|
||||
|
||||
|
||||
@@ -406,9 +406,6 @@ class Service(metaclass=ABCMeta):
|
||||
|
||||
return all_tracks
|
||||
|
||||
# Deprecated 5.5.0 shim for service repos still on the old underscored name; drop once they migrate.
|
||||
_get_tracks_for_variants = get_tracks_for_variants
|
||||
|
||||
@staticmethod
|
||||
def get_session() -> requests.Session:
|
||||
"""
|
||||
|
||||
@@ -15,6 +15,7 @@ import click
|
||||
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DEFAULT_SAMPLE
|
||||
from unshackle.core.service import Service
|
||||
from unshackle.core.service_repo import DirtyServiceRepo, head, is_repo_spec, refresh_repo, resolve_service_repo
|
||||
from unshackle.core.utilities import import_module_by_path
|
||||
@@ -360,16 +361,17 @@ class Services(click.Group):
|
||||
super().format_help(ctx, formatter)
|
||||
|
||||
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
||||
"""Preprocess ``--slow`` so it can take an optional range value before Click parses the arguments."""
|
||||
"""Preprocess ``--slow`` and ``--sample`` so each can take an optional range value before Click parses."""
|
||||
defaults = {"--slow": "60-120", "--sample": "{}-{}".format(*DEFAULT_SAMPLE)}
|
||||
processed = []
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--slow":
|
||||
if args[i] in defaults:
|
||||
if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]):
|
||||
processed.append(f"--slow={args[i + 1]}")
|
||||
processed.append(f"{args[i]}={args[i + 1]}")
|
||||
i += 2
|
||||
else:
|
||||
processed.append("--slow=60-120")
|
||||
processed.append(f"{args[i]}={defaults[args[i]]}")
|
||||
i += 1
|
||||
else:
|
||||
processed.append(args[i])
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Any, Optional
|
||||
from urllib.parse import urlencode, urlparse, urlunparse
|
||||
|
||||
import rnet
|
||||
from requests import HTTPError, Request
|
||||
from requests import HTTPError, Request, Response
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from unshackle.core.config import config
|
||||
@@ -227,6 +227,18 @@ class RnetResponse:
|
||||
pass
|
||||
|
||||
|
||||
def utf8_text(res: Response | RnetResponse) -> str:
|
||||
"""Return the response text, decoded as UTF-8 unless the server sent a charset.
|
||||
|
||||
With no charset, requests assumes ISO-8859-1 for a `text/*` type and runs charset detection
|
||||
on the full body for any other type. Use this only for a format that is UTF-8 by
|
||||
specification, such as M3U8.
|
||||
"""
|
||||
if isinstance(res, Response) and "charset" not in res.headers.get("content-type", "").lower():
|
||||
res.encoding = "utf-8"
|
||||
return res.text
|
||||
|
||||
|
||||
class RnetSessionHeaders(CaseInsensitiveDict):
|
||||
"""Dict-like headers that write to the rnet client through update()."""
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Deprecated. Import from unshackle.core.titles.music instead."""
|
||||
|
||||
from unshackle.core.titles.music import Album, Song # noqa: F401
|
||||
|
||||
__all__ = ("Song", "Album")
|
||||
@@ -9,8 +9,9 @@ from enum import Enum
|
||||
from functools import partial
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Optional, Union
|
||||
from typing import Any, Callable, Iterable, Optional, Sequence, Union
|
||||
|
||||
import click
|
||||
import pycaption
|
||||
import requests
|
||||
from construct import Container
|
||||
@@ -1001,6 +1002,43 @@ class Subtitle(Track):
|
||||
|
||||
return captions, language
|
||||
|
||||
def sample_slice(
|
||||
self, starts: Sequence[float], timescale: float = 1, window: Optional[tuple[int, int]] = None
|
||||
) -> slice:
|
||||
"""Return the segments of ``window`` plus one segment on each side.
|
||||
|
||||
Cue times in a segment are absolute, so ``sample_start`` stays zero and ``trim`` makes the cut.
|
||||
A track that ends before the window keeps only its last segment.
|
||||
"""
|
||||
try:
|
||||
keep = super().sample_slice(starts, timescale, window)
|
||||
except click.ClickException:
|
||||
keep = slice(len(starts) - 1, None)
|
||||
self.sample_start = 0.0
|
||||
if keep.start is None:
|
||||
return keep
|
||||
return slice(max(keep.start - 1, 0), None if keep.stop is None else keep.stop + 1)
|
||||
|
||||
def trim(self, start: float, end: float) -> bool:
|
||||
"""Keep the cues between ``start`` and ``end`` seconds and move them to start at zero.
|
||||
|
||||
Returns False and leaves the file whole when pysubs2 cannot read or write its format.
|
||||
"""
|
||||
import pysubs2
|
||||
|
||||
low, high = round(start * 1000), round(end * 1000)
|
||||
try:
|
||||
subs = pysubs2.load(str(self.path), encoding="utf-8")
|
||||
subs.events = [event for event in subs.events if event.end > low and event.start < high]
|
||||
for event in subs.events:
|
||||
event.start = max(event.start, low) - low
|
||||
event.end = min(event.end, high) - low
|
||||
subs.save(str(self.path))
|
||||
except (pysubs2.exceptions.Pysubs2Error, UnicodeDecodeError):
|
||||
return False
|
||||
self.sample_start = start
|
||||
return True
|
||||
|
||||
def strip_hearing_impaired(self) -> None:
|
||||
"""
|
||||
Remove the SDH captions.
|
||||
|
||||
@@ -3,19 +3,24 @@ import html
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
from bisect import bisect_left, bisect_right
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Optional, Union
|
||||
from typing import Any, Callable, Iterable, Optional, Sequence, Union
|
||||
from uuid import UUID
|
||||
from zlib import crc32
|
||||
|
||||
import click
|
||||
from construct import ConstructError
|
||||
from langcodes import Language
|
||||
from requests import Response, Session
|
||||
from pymp4.parser import Box
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
|
||||
from unshackle.core import binaries
|
||||
@@ -26,7 +31,7 @@ from unshackle.core.downloaders import requests
|
||||
from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine
|
||||
from unshackle.core.drm.verify import decrypt_track
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.session import RnetSession
|
||||
from unshackle.core.session import RnetSession, utf8_text
|
||||
from unshackle.core.tracks import resume
|
||||
from unshackle.core.utilities import get_boxes, log_event, try_ensure_utf8
|
||||
from unshackle.core.utils.subprocess import ffprobe
|
||||
@@ -364,6 +369,31 @@ def assert_fragments_decrypted(path: Path) -> None:
|
||||
)
|
||||
|
||||
|
||||
SAMPLE_PROBE_BYTES = 64 * 1024
|
||||
SAMPLE_PROBE_LIMIT = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def parse_sidx(data: bytes, offset: int) -> tuple[int, list[tuple[int, int, int]]]:
|
||||
"""Read a ``sidx`` box that starts at byte ``offset`` of its file.
|
||||
|
||||
Returns the timescale and one (start time, first byte, last byte) for each subsegment. The list
|
||||
is empty when the box is not a ``sidx`` or when it points at more indexes and not at media.
|
||||
"""
|
||||
try:
|
||||
box = Box.parse(data)
|
||||
except ConstructError:
|
||||
return 0, []
|
||||
if box.type != b"sidx" or any(reference.reference_type != "MEDIA" for reference in box.references):
|
||||
return 0, []
|
||||
start, byte = box.earliest_presentation_time, offset + box.end + box.first_offset
|
||||
subsegments: list[tuple[int, int, int]] = []
|
||||
for reference in box.references:
|
||||
subsegments.append((start, byte, byte + reference.referenced_size - 1))
|
||||
start += reference.segment_duration
|
||||
byte += reference.referenced_size
|
||||
return box.timescale, subsegments
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadContext:
|
||||
"""Shared arguments passed to each manifest's ``download_track``."""
|
||||
@@ -396,6 +426,8 @@ class Track:
|
||||
DASH = 3 # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
|
||||
ISM = 4 # https://learn.microsoft.com/en-us/silverlight/smooth-streaming
|
||||
|
||||
sample_tail = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Union[str, list[str]],
|
||||
@@ -497,6 +529,77 @@ class Track:
|
||||
|
||||
# TODO: Currently using OnFoo event naming, change to just segment_filter
|
||||
self.OnSegmentFilter: Optional[Callable] = None
|
||||
self.sample_start: float = 0.0
|
||||
self.sample_bytes: Optional[tuple[bytes, str]] = None
|
||||
|
||||
def sample_slice(
|
||||
self, starts: Sequence[float], timescale: float = 1, window: Optional[tuple[int, int]] = None
|
||||
) -> slice:
|
||||
"""Return the slice of segments that hold ``window``, from the start time of each segment.
|
||||
|
||||
Sets ``sample_start`` to the start time of the first kept segment, in seconds. A video keeps
|
||||
``sample_tail`` segments after the window: mkvmerge ends the cut only at a video keyframe, and
|
||||
with no such keyframe the audio and the subtitles continue to their own end.
|
||||
"""
|
||||
self.sample_start = 0.0
|
||||
if not window or not starts:
|
||||
return slice(None)
|
||||
low, high = (starts[0] + seconds * timescale for seconds in window)
|
||||
if low > starts[-1] and len(starts) > 1:
|
||||
raise click.ClickException(
|
||||
"The title ends before the sample window starts. Use --sample START-END with an earlier window."
|
||||
)
|
||||
first = max(bisect_right(starts, low) - 1, 0)
|
||||
self.sample_start = (starts[first] - starts[0]) / timescale
|
||||
return slice(first, max(bisect_left(starts, high) + self.sample_tail, first + 1))
|
||||
|
||||
def sample_byte_range(self, session: Union[Session, "RnetSession"]) -> Optional[tuple[bytes, str]]:
|
||||
"""Find the bytes of a one-file track that hold the sample window.
|
||||
|
||||
Reads the start of the file for a ``sidx`` box. Returns the bytes before that box, which hold
|
||||
the init data, and the byte range of the window. Returns None when the file has no such index.
|
||||
"""
|
||||
if self.sample_bytes or not isinstance(self.url, str):
|
||||
return self.sample_bytes
|
||||
head = b""
|
||||
|
||||
def read_to(end: int) -> None:
|
||||
nonlocal head
|
||||
if end > len(head):
|
||||
res = session.get(self.url, headers={"Range": f"bytes={len(head)}-{end - 1}"})
|
||||
if res.ok:
|
||||
head += res.content
|
||||
|
||||
pos = 0
|
||||
while pos < SAMPLE_PROBE_LIMIT:
|
||||
if pos + 8 > len(head):
|
||||
read_to(pos + SAMPLE_PROBE_BYTES)
|
||||
if pos + 8 > len(head):
|
||||
return None
|
||||
size, kind = struct.unpack(">I4s", head[pos : pos + 8])
|
||||
if size < 8:
|
||||
return None
|
||||
if kind != b"sidx":
|
||||
pos += size
|
||||
continue
|
||||
read_to(pos + size)
|
||||
timescale, subsegments = parse_sidx(head[pos : pos + size], pos)
|
||||
if not subsegments:
|
||||
return None
|
||||
kept = subsegments[self.sample_slice([start for start, _, _ in subsegments], timescale, config.sample)]
|
||||
self.sample_bytes = head[:pos], f"{kept[0][1]}-{kept[-1][2]}"
|
||||
return self.sample_bytes
|
||||
return None
|
||||
|
||||
def can_sample(self, session: Union[Session, "RnetSession"]) -> bool:
|
||||
"""Return False when a sample cannot cut this track, because it is one file with no segment index."""
|
||||
if self.descriptor == self.Descriptor.URL:
|
||||
return bool(self.sample_byte_range(session))
|
||||
if self.descriptor == self.Descriptor.DASH:
|
||||
from unshackle.core.manifests import DASH
|
||||
|
||||
return DASH.has_segment_timing(self)
|
||||
return True
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "{name}({items})".format(
|
||||
@@ -741,8 +844,17 @@ class Track:
|
||||
if DOWNLOAD_LICENCE_ONLY.is_set():
|
||||
progress(downloaded="[yellow]SKIPPED")
|
||||
else:
|
||||
sample_range = None
|
||||
if config.sample and track_type in ("Video", "Audio"):
|
||||
sample_range = self.sample_byte_range(dl_session)
|
||||
if not sample_range:
|
||||
raise click.ClickException(
|
||||
"--sample cannot cut this track: it is one file with no segment index."
|
||||
)
|
||||
for status_update in self.downloader(
|
||||
urls=self.url,
|
||||
urls=[{"url": self.url, "headers": {"Range": f"bytes={sample_range[1]}"}}]
|
||||
if sample_range
|
||||
else self.url,
|
||||
output_dir=save_path.parent,
|
||||
filename=save_path.name,
|
||||
headers=dl_session.headers,
|
||||
@@ -760,6 +872,14 @@ class Track:
|
||||
status_update["downloaded"] = f"URL {downloaded}"
|
||||
progress(**status_update)
|
||||
|
||||
if sample_range:
|
||||
media_path = save_path.with_name(f"{save_path.name}.media")
|
||||
save_path.replace(media_path)
|
||||
with save_path.open("wb") as joined, media_path.open("rb") as media:
|
||||
joined.write(sample_range[0])
|
||||
shutil.copyfileobj(media, joined)
|
||||
media_path.unlink()
|
||||
|
||||
self.path = save_path
|
||||
events.emit(events.Types.TRACK_DOWNLOADED, track=self)
|
||||
|
||||
@@ -1021,9 +1141,7 @@ class Track:
|
||||
session = getattr(self, "session", None) or Session()
|
||||
|
||||
response = session.get(self.url)
|
||||
if isinstance(response, Response):
|
||||
response.encoding = response.encoding or "utf-8"
|
||||
playlist = m3u8.loads(response.text, self.url)
|
||||
playlist = m3u8.loads(utf8_text(response), self.url)
|
||||
|
||||
drm_list = []
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from unshackle.core.utilities import (
|
||||
valid_language,
|
||||
)
|
||||
from unshackle.core.utils.collections import as_list, flatten
|
||||
from unshackle.core.utils.subprocess import run_step
|
||||
|
||||
MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)")
|
||||
MP4BOX_OPEN_FAILED = re.compile(r"error while opening|Error opening file|Invalid IsoMedia File", re.I)
|
||||
@@ -495,6 +496,11 @@ class Tracks:
|
||||
selected.append(track)
|
||||
return selected
|
||||
|
||||
@property
|
||||
def sample_base(self) -> float:
|
||||
"""The earliest ``sample_start`` of the video and audio tracks, which a sample mux uses as time zero."""
|
||||
return min((t.sample_start for t in (*self.videos, *self.audio)), default=0.0)
|
||||
|
||||
def _mux_mp4(
|
||||
self,
|
||||
title: str,
|
||||
@@ -524,6 +530,7 @@ class Tracks:
|
||||
progress = progress or partial(lambda **kwargs: None)
|
||||
|
||||
cl = [str(mp4box)]
|
||||
sample_base = self.sample_base
|
||||
names: list[str] = []
|
||||
cleaned: list[Path] = []
|
||||
track_id = 0
|
||||
@@ -557,7 +564,8 @@ class Tracks:
|
||||
elif candidate.exists():
|
||||
candidate.unlink()
|
||||
|
||||
cl.extend(["-add", f"{source}:lang={track.language}"])
|
||||
delay = round((track.sample_start - sample_base) * 1000) if config.sample else 0
|
||||
cl.extend(["-add", f"{source}:lang={track.language}" + (f":delay={delay}" if delay else "")])
|
||||
name = "" if isinstance(track, Video) else (track.get_track_name() or "")
|
||||
if name:
|
||||
names.extend(["-name", f"{track_id}={name}"])
|
||||
@@ -568,7 +576,7 @@ class Tracks:
|
||||
cl.extend(names)
|
||||
|
||||
chapters_path = None
|
||||
if self.chapters:
|
||||
if self.chapters and not config.sample:
|
||||
chapters_path = config.directories.temp / config.filenames.chapters.format(
|
||||
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
||||
)
|
||||
@@ -632,6 +640,19 @@ class Tracks:
|
||||
returncode = p.wait()
|
||||
if returncode < 2 and any(MP4BOX_OPEN_FAILED.search(line) for line in errors):
|
||||
returncode = 2
|
||||
if config.sample and returncode == 0:
|
||||
start, end = (seconds - sample_base for seconds in config.sample)
|
||||
cut_path = output_path.with_suffix(".cut.mp4")
|
||||
try:
|
||||
run_step(
|
||||
[mp4box, "-splitx", f"{start:.3f}:{end:.3f}", output_path, "-out", cut_path],
|
||||
output=cut_path,
|
||||
label="MP4Box sample cut",
|
||||
)
|
||||
os.replace(cut_path, output_path)
|
||||
except RuntimeError as e:
|
||||
returncode = 2
|
||||
errors.append(f"MP4Box could not cut the sample window: {e}")
|
||||
mux_duration_ms = round((time.monotonic() - mux_start_time) * 1000, 1)
|
||||
output_size = output_path.stat().st_size if output_path.exists() else 0
|
||||
|
||||
@@ -729,6 +750,16 @@ class Tracks:
|
||||
"--no-date", # remove dates from the output for security
|
||||
]
|
||||
|
||||
sample_base = self.sample_base
|
||||
|
||||
def sample_sync(start: float) -> list[str]:
|
||||
shift = round((start - sample_base) * 1000)
|
||||
return ["--sync", f"0:{shift}"] if config.sample and shift else []
|
||||
|
||||
if config.sample:
|
||||
start, end = (seconds - sample_base for seconds in config.sample)
|
||||
cl.extend(["--split", f"parts:{start:.3f}s-{end:.3f}s"])
|
||||
|
||||
if config.muxing.get("set_title", True):
|
||||
cl.extend(["--title", title])
|
||||
|
||||
@@ -782,6 +813,7 @@ class Tracks:
|
||||
f"0:{vt.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none",
|
||||
*sample_sync(vt.sample_start),
|
||||
]
|
||||
|
||||
# Add FPS fix if needed (typically for hybrid mode to prevent sync issues)
|
||||
@@ -839,6 +871,7 @@ class Tracks:
|
||||
f"0:{at.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none",
|
||||
*sample_sync(at.sample_start),
|
||||
"(",
|
||||
str(at.path),
|
||||
")",
|
||||
@@ -872,6 +905,7 @@ class Tracks:
|
||||
f"0:{st.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none",
|
||||
*sample_sync(st.sample_start),
|
||||
"(",
|
||||
str(st.path),
|
||||
")",
|
||||
@@ -883,6 +917,8 @@ class Tracks:
|
||||
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
||||
)
|
||||
self.chapters.dump(chapters_path, fallback_name=config.chapter_fallback_name)
|
||||
if config.sample and sample_base:
|
||||
cl.extend(["--chapter-sync", str(round(-sample_base * 1000))])
|
||||
cl.extend(["--chapter-charset", "UTF-8", "--chapters", str(chapters_path)])
|
||||
else:
|
||||
chapters_path = None
|
||||
|
||||
@@ -20,6 +20,8 @@ from unshackle.core.utils.subprocess import log_tool_run
|
||||
|
||||
|
||||
class Video(Track):
|
||||
sample_tail = 1
|
||||
|
||||
class Codec(str, Enum):
|
||||
AVC = "H.264"
|
||||
HEVC = "H.265"
|
||||
|
||||
@@ -13,7 +13,7 @@ from urllib.parse import urljoin
|
||||
from requests import Session
|
||||
|
||||
from unshackle.core.binaries import FFProbe
|
||||
from unshackle.core.session import RnetSession
|
||||
from unshackle.core.session import RnetSession, utf8_text
|
||||
from unshackle.core.utils.subprocess import log_tool_run
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -320,7 +320,7 @@ def extract_hls(track: "Track", session: Union[Session, RnetSession]) -> list[Se
|
||||
|
||||
playlist_url = track.url if isinstance(track.url, str) else track.url[0]
|
||||
res = session.get(playlist_url, timeout=PROBE_TIMEOUT)
|
||||
playlist = m3u8.loads(res.text, uri=playlist_url)
|
||||
playlist = m3u8.loads(utf8_text(res), uri=playlist_url)
|
||||
|
||||
out: list[Segment] = []
|
||||
for segment in playlist.segments:
|
||||
|
||||
@@ -6,6 +6,7 @@ import click
|
||||
from click.shell_completion import CompletionItem
|
||||
from pywidevine.cdm import Cdm as WidevineCdm
|
||||
|
||||
from unshackle.core.constants import DEFAULT_SAMPLE
|
||||
from unshackle.core.tracks.audio import Audio
|
||||
from unshackle.core.tracks.subtitle import Subtitle
|
||||
from unshackle.core.tracks.video import Video
|
||||
@@ -505,6 +506,26 @@ class SlowDelayRange(click.ParamType):
|
||||
return (low, high)
|
||||
|
||||
|
||||
class SampleRange(click.ParamType):
|
||||
"""Parses a sample window string like '60-120' into a tuple of (start, end) seconds."""
|
||||
|
||||
name = "sample_range"
|
||||
|
||||
def convert(
|
||||
self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]
|
||||
) -> Optional[tuple[int, int]]:
|
||||
if isinstance(value, bool):
|
||||
return DEFAULT_SAMPLE if value else None
|
||||
if isinstance(value, (list, tuple)) and len(value) == 2:
|
||||
value = f"{value[0]}-{value[1]}"
|
||||
|
||||
match = re.match(r"^(\d+)-(\d+)$", str(value))
|
||||
if not match or int(match.group(1)) >= int(match.group(2)):
|
||||
self.fail(f"'{value}' is not a valid sample window. Use format: START-END (e.g., 60-120)", param, ctx)
|
||||
|
||||
return (int(match.group(1)), int(match.group(2)))
|
||||
|
||||
|
||||
SEASON_RANGE = SeasonRange()
|
||||
LANGUAGE_RANGE = LanguageRange()
|
||||
QUALITY_LIST = QualityList()
|
||||
@@ -512,3 +533,4 @@ AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec)
|
||||
VIDEO_CODEC_LIST = MultipleVideoCodecChoice(Video.Codec)
|
||||
SUBTITLE_CODEC = SubtitleCodecChoice(Subtitle.Codec)
|
||||
SLOW_DELAY_RANGE = SlowDelayRange()
|
||||
SAMPLE_RANGE = SampleRange()
|
||||
|
||||
Reference in New Issue
Block a user