mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-18 06:55:07 +00:00
Compare commits
9
Commits
87cdb30229
...
1543e4d22f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1543e4d22f | ||
|
|
65353af5c8 | ||
|
|
e804d09f0a | ||
|
|
8c43250c9b | ||
|
|
1f96a4bd78 | ||
|
|
d4690e5c6b | ||
|
|
e5e1ce14c6 | ||
|
|
d9d6115054 | ||
|
|
0a68cc2452 |
@@ -95,13 +95,13 @@ Declared at class level to configure framework behaviour:
|
||||
|
||||
| Variable | Type | Purpose |
|
||||
|---|---|---|
|
||||
| `ALIASES` | `tuple[str, ...]` | Extra tags that resolve to this service (e.g. `("EX", "DOMAIN")`). Default `()`. |
|
||||
| `GEOFENCE` | `tuple[str, ...]` | IP region codes the service requires. Empty = no geofence. The **first** entry is treated as the main region for auto-proxy. |
|
||||
| `ALIASES` | `tuple[str, ...]` | Extra tags that resolve to this service (e.g. `("EX", "DOMAIN")`). Case does not matter. Default `()`. |
|
||||
| `GEOFENCE` | `tuple[str, ...]` | IP region codes the service requires. Empty = no geofence. unshackle treats the **first** entry as the main region for auto-proxy. |
|
||||
| `VAULT_TAG` | `Optional[str]` | Overrides the key-vault namespace so sibling services can share one vault. Default `None` (use the service's own tag). |
|
||||
| `AUTH_METHODS` | `Optional[tuple[str, ...]]` | Auth methods accepted (`"cookies"` / `"credentials"`). When `None`, the REST `/services` endpoint infers them from `authenticate()`. |
|
||||
| `NO_SUBTITLES` | `bool` | Set `True` on a service with no subtitle tracks to skip subtitle handling entirely. |
|
||||
| `ANIME` | `bool` | Set `True` when the catalogue is anime, so metadata lookups prefer AniList. A title's own `anime` flag overrides it. |
|
||||
| `DAILY` | `bool` | Set `True` when the catalogue is daily/date-based (talk shows, news, sports), so episodes are named by air date. A title's own `daily` flag overrides it. |
|
||||
| `DAILY` | `bool` | Set `True` when the catalogue is daily/date-based (talk shows, news, sports), so unshackle names episodes by air date. A title's own `daily` flag overrides it. |
|
||||
|
||||
!!! note "`NO_SUBTITLES` is a convention, not a base-class attribute"
|
||||
`dl.py` uses `hasattr` to find `NO_SUBTITLES`, and the `Service` base class
|
||||
|
||||
@@ -396,7 +396,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is
|
||||
| `exact_lang` | boolean | `false` | Exact language matching. |
|
||||
| `sub_format` | string | `null` | Output subtitle format. |
|
||||
| `video_only` / `audio_only` / `subs_only` / `chapters_only` | boolean | `false` | Track-type restriction (at most one). |
|
||||
| `no_subs` / `no_audio` / `no_chapters` / `no_video` | boolean | `false` | Skip a track type. |
|
||||
| `no_subs` / `no_audio` / `no_chapters` / `no_video` / `no_attachments` | boolean | `false` | Skip a track type. |
|
||||
| `audio_description` | boolean | `false` | Download audio description. |
|
||||
| `slow` | boolean/string | `null` | Randomized delay between downloads. |
|
||||
| `split_audio` | boolean | `null` | Separate files per audio codec. |
|
||||
|
||||
@@ -61,12 +61,12 @@ Declared at the top of your service class, these are static descriptors:
|
||||
|
||||
| Class variable | Type | Purpose |
|
||||
|---|---|---|
|
||||
| `ALIASES` | `tuple[str, ...]` | Alternate tags the service answers to (resolved by `Services.get_tag`). |
|
||||
| `GEOFENCE` | `tuple[str, ...]` | ISO region codes the service requires; empty means no geofence. The first entry is treated as the "main region" for automatic proxy selection. |
|
||||
| `ALIASES` | `tuple[str, ...]` | Alternate tags the service answers to, in any case (resolved by `Services.get_tag`). |
|
||||
| `GEOFENCE` | `tuple[str, ...]` | ISO region codes the service requires; empty means no geofence. unshackle treats the first entry as the "main region" for automatic proxy selection. |
|
||||
| `VAULT_TAG` | `Optional[str]` | Overrides the key-vault namespace; defaults to the service's own tag. |
|
||||
| `AUTH_METHODS` | `Optional[tuple[str, ...]]` | Accepted auth methods (`"cookies"` / `"credentials"`). When `None`, the REST `/services` endpoint infers them from `authenticate()`. |
|
||||
| `ANIME` | `bool` | The catalogue is anime, so metadata lookups prefer AniList. A title's own `anime` flag overrides it. |
|
||||
| `DAILY` | `bool` | The catalogue is daily/date-based, so episodes are named by air date. A title's own `daily` flag overrides it. |
|
||||
| `DAILY` | `bool` | The catalogue is daily/date-based, so unshackle names episodes by air date. A title's own `daily` flag overrides it. |
|
||||
|
||||
### Required methods
|
||||
|
||||
@@ -141,7 +141,7 @@ families, one per media kind:
|
||||
All items subclass `Title`, which carries the shared identity: a **unique `id`**, the
|
||||
originating `service` class, an optional original `language`, arbitrary `data`, and a
|
||||
freshly-constructed `self.tracks = Tracks()` that `get_tracks` will fill. Collections
|
||||
are sorted lists, so titles stay in a stable order.
|
||||
use sorted lists, so titles stay in a stable order.
|
||||
|
||||
!!! warning "The unique ID is validated"
|
||||
`Title.__init__` rejects a falsy `id_`, and rejects any ID shorter than 4 characters
|
||||
@@ -330,7 +330,7 @@ in `unshackle.core.manifests`. Each parser has the same three-part contract:
|
||||
### The parser contract
|
||||
|
||||
1. **Construct**: `DASH.from_url(url, session=...)` or `DASH.from_text(text, url)`
|
||||
(and likewise for `HLS` / `ISM`). A `url` is always required, even for `from_text`,
|
||||
(and likewise for `HLS` / `ISM`). Always pass a `url`, even to `from_text`,
|
||||
because the parser needs it to make relative segment paths absolute.
|
||||
2. **`to_tracks(...)`**: parses the manifest into a `Tracks` container of `Video`,
|
||||
`Audio`, and `Subtitle` objects, each carrying descriptor-specific state.
|
||||
|
||||
@@ -121,7 +121,7 @@ unshackle honours the special language tokens `orig`, `all`, and `best` everywhe
|
||||
|
||||
### Track-type inclusion / exclusion
|
||||
|
||||
Keep only certain track types, or skip certain track types. Attachments are always kept.
|
||||
Keep only certain track types, or skip certain track types.
|
||||
|
||||
| Include-only | Skip |
|
||||
|---|---|
|
||||
@@ -129,6 +129,7 @@ Keep only certain track types, or skip certain track types. Attachments are alwa
|
||||
| `-A`, `--audio-only` | `-na`, `--no-audio` |
|
||||
| `-S`, `--subs-only` | `-ns`, `--no-subs` |
|
||||
| `-C`, `--chapters-only` | `-nc`, `--no-chapters` |
|
||||
| | `-nt`, `--no-attachments` (cover art, subtitle fonts, and files the service attaches) |
|
||||
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
@@ -185,6 +186,7 @@ order the service used, then renumbers the episodes into the order you asked for
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--cdm-only` / `--vaults-only` | Use only the CDM, or only Key Vaults, for key acquisition. |
|
||||
| `--cdm <name>` | Use the named CDM device from the `cdm` config mapping for this run, ignoring the service/default mapping. |
|
||||
| `--skip-dl` | Skip the download but still retrieve keys. |
|
||||
| `--export` | Export track info and keys to a JSON file in the exports directory. |
|
||||
|
||||
|
||||
@@ -617,6 +617,7 @@ or negatively (everything but these).
|
||||
- `-na` / `--no-audio`
|
||||
- `-ns` / `--no-subs`
|
||||
- `-nc` / `--no-chapters`
|
||||
- `-nt` / `--no-attachments` (cover art, subtitle fonts, and files the service attaches)
|
||||
|
||||
Additional track-type flags:
|
||||
|
||||
@@ -750,6 +751,7 @@ By default, unshackle checks your **key vaults** first and only asks a **CDM** t
|
||||
a content key when the vault misses. You can force one side or the other:
|
||||
|
||||
- `--cdm-only`: only use the CDM (skip vaults).
|
||||
- `--cdm <name>`: use the named CDM device for this run, ignoring the `cdm` config mapping (including its quality and Widevine/PlayReady sub-entries).
|
||||
- `--vaults-only`: only use key vaults and never license through the CDM. A missing
|
||||
content key fails.
|
||||
|
||||
@@ -932,7 +934,7 @@ authoritative list.
|
||||
| `--select-titles` | | Interactively pick episodes or films. |
|
||||
| `--latest-episode` | | Only the newest episode. |
|
||||
| `--video-only` / `--audio-only` / `--subs-only` | `-V` / `-A` / `-S` | Restrict track types. |
|
||||
| `--no-video` / `--no-audio` / `--no-subs` / `--no-chapters` | `-nv` / `-na` / `-ns` / `-nc` | Skip track types. |
|
||||
| `--no-video` / `--no-audio` / `--no-subs` / `--no-chapters` / `--no-attachments` | `-nv` / `-na` / `-ns` / `-nc` / `-nt` | Skip track types. |
|
||||
| `--worst` | | Lowest bitrate within `-q`. |
|
||||
| `--require-audio` | | Audio languages that must exist. The title fails if one is missing. |
|
||||
| `--require-video` | | Video languages that must exist. The title fails if one is missing. |
|
||||
@@ -944,6 +946,7 @@ authoritative list.
|
||||
| `--workers` / `--downloads` / `--slow` | | Concurrency and pacing. |
|
||||
| `--list` / `--list-titles` / `--skip-dl` | | Dry runs. |
|
||||
| `--cdm-only` / `--vaults-only` | | Content key source control. |
|
||||
| `--cdm <name>` | | Pin one CDM device, bypassing the `cdm` config mapping. |
|
||||
| `--export` | | Export track info and keys to JSON. |
|
||||
| `--tmdb` / `--imdb` / `--tvdb` / `--anilist` / `--enrich` | | Metadata overrides. |
|
||||
| `--tvdb-order` | | Renumber episodes to a TVDB season order. |
|
||||
|
||||
@@ -498,6 +498,7 @@ control this trade-off on the `dl` command:
|
||||
|------|--------|
|
||||
| `--vaults-only` | Never send a license request; use only cached/vault keys. Fails if a content key is missing. |
|
||||
| `--cdm-only` | Ignore vaults for sourcing keys and always perform a fresh license request. |
|
||||
| `--cdm <name>` | Use the named CDM device for this run instead of the one the `cdm` config maps for the service. |
|
||||
|
||||
See [key vaults](vaults.md) for how to configure vault backends.
|
||||
|
||||
|
||||
@@ -55,6 +55,9 @@ format-appropriate clean-up to make a single, valid file:
|
||||
that confuse downstream parsers, and merges multi-line cues that a service split into
|
||||
separate overlapping cues. By default unshackle keeps the original styling and layout
|
||||
(see [`preserve_formatting`](#subtitle-configuration)).
|
||||
- **Cue markup normalisation**: unshackle removes the WebVTT-only parts of a cue
|
||||
payload, so no player draws them as dialogue. See
|
||||
[Cue markup](#cue-markup-and-html-entities).
|
||||
- **TTML for muxing**: Matroska cannot carry TTML. If you do not request a specific
|
||||
output format, unshackle automatically converts any plain TTML track to WebVTT before
|
||||
muxing (the closest lossless equivalent). Boxed `fTTML` is likewise unwrapped to TTML
|
||||
@@ -188,6 +191,20 @@ itself. The default (`auto`) prefers the `subby` cleaner for SRT, falls back to
|
||||
SubtitleEdit when you install it, and otherwise uses the built-in `filter-subs`
|
||||
approach.
|
||||
|
||||
Every engine removes the same three things: bracketed sound effects (`[MUSIC PLAYING]`,
|
||||
`(door creaks)`), speaker labels (`MAN:`), and font tags. Every engine keeps plain
|
||||
dialogue and inline `<i>` styling. They differ on music and lyric lines:
|
||||
|
||||
| `sdh_method` | Engine that runs | Lyric lines (`♪ ... ♪`) |
|
||||
| --- | --- | --- |
|
||||
| `auto` | `subby` for an SRT track. SubtitleEdit for any other format when you install it, otherwise `filter-subs`. | Kept, unless the fallback reaches `filter-subs`. |
|
||||
| `subby` | `subby`, but only for a track that is already SRT. Any other format falls through to `filter-subs`. | Kept for SRT, removed otherwise. |
|
||||
| `subtitleedit` | SubtitleEdit when you install it, otherwise `filter-subs`. | Kept with SubtitleEdit, removed without it. |
|
||||
| `filter-subs` | `filter-subs` always. | Removed. |
|
||||
|
||||
Pick `filter-subs` when you want lyrics gone as well. Pick `subby` or install
|
||||
SubtitleEdit when you want to read along with the songs.
|
||||
|
||||
!!! note "Why `convert_before_strip` exists"
|
||||
`subby`'s SDH stripper only operates on SRT. Handed any other codec it returns
|
||||
silently without stripping anything, not an error but a no-op that leaves the SDH
|
||||
@@ -288,6 +305,28 @@ Reading the table:
|
||||
Install [SubtitleEdit / `seconv`](#subtitle-configuration) if you need colour or the
|
||||
highest → SRT fidelity.
|
||||
|
||||
### Cue markup and HTML entities
|
||||
|
||||
Styling is not the only thing a conversion has to handle. WebVTT carries markup that no
|
||||
other format understands, and a player that does not know a tag draws it as dialogue.
|
||||
unshackle removes that markup before any backend runs, so the result is the same for
|
||||
every `conversion_method`:
|
||||
|
||||
| WebVTT markup | Result |
|
||||
| --- | --- |
|
||||
| `<c.bg_transparent>text</c>` | Tags removed, text kept. A class only selects a CSS `::cue` rule that no player ever receives, so it carries no styling. |
|
||||
| `<i.loud>`, `<b.x>`, `<u.y>` | Class list removed, the styling tag kept. |
|
||||
| `<v Bob>`, `<lang ja>`, `<ruby>`, `<rt>` | Tags removed, text kept. |
|
||||
| `<00:00:02.000>` | Removed. These are karaoke timings that no subtitle player renders. |
|
||||
| `&`, ` `, `'` | Decoded to real characters for SRT, SSA, and ASS output. Kept escaped for WebVTT and TTML, which need them. |
|
||||
|
||||
!!! warning "A class list breaks more than you expect"
|
||||
FFmpeg reads SubRip markup as a tag only while the tag name stays inside
|
||||
`[0-9a-zA-Z_/]`. The dot in `<i.loud>` fails that test, so FFmpeg prints the opening
|
||||
tag on screen and then closes an italic run that never opened. The FFmpeg WebVTT
|
||||
reader matches `i`, `b`, and `u` exactly, so there the same class silently loses the
|
||||
styling. mpv, VLC, Plex, and Jellyfin all use one of those two readers.
|
||||
|
||||
!!! note "Config wins over a service's preference"
|
||||
A service may set a `preferred_conversion_method` on its own tracks (for example when
|
||||
it ships subtitles that a particular backend handles best). An explicit
|
||||
@@ -381,6 +420,32 @@ subtitle:
|
||||
| `group_by` | `type` | `type`, `language` | Whether all tracks of one type stay together, or each language stays next to its own variants. |
|
||||
| `language_priority` | *(unset)* | a list of languages | Languages to sort to the top. The rest keep the default order. Nothing is removed. |
|
||||
|
||||
### Which settings to change
|
||||
|
||||
The defaults give a clean, player-safe file, so most people change nothing. Start here
|
||||
only when you want something the defaults do not do:
|
||||
|
||||
| If you want | Set |
|
||||
| --- | --- |
|
||||
| Every original style kept, with no re-encode at all | `--sub-format original`, and `sidecar_format: original` for sidecars |
|
||||
| The highest conversion fidelity, including colour | Install SubtitleEdit. `conversion_method` stays `auto` |
|
||||
| Music and lyric lines stripped along with the sound effects | `sdh_method: filter-subs` |
|
||||
| Lyric lines kept | `sdh_method: subby`, or install SubtitleEdit |
|
||||
| No stripped copies made at all | `strip_sdh: false` |
|
||||
| One backend for every conversion, whatever you install | `conversion_method: <backend>` |
|
||||
|
||||
Two keys read as if they control markup, and do not:
|
||||
|
||||
- **`preserve_formatting`** decides whether unshackle re-encodes a WebVTT file after
|
||||
the repair pipeline. Left `true`, the cue text passes through as written. Set to
|
||||
`false`, unshackle re-encodes the file through pycaption, which merges identical cues
|
||||
and drops empty ones but also flattens styling. Neither setting leaves WebVTT-only
|
||||
markup in the file: [cue markup normalisation](#cue-markup-and-html-entities) runs
|
||||
either way.
|
||||
- **`convert_before_strip`** applies to SDH stripping, not to your output format. It
|
||||
converts the working copy to SRT so a non-SubtitleEdit engine can read it. Turning it
|
||||
off makes `subby` and `filter-subs` no-ops on anything that is not already SRT.
|
||||
|
||||
!!! tip "SubtitleEdit is optional but recommended"
|
||||
Several high-fidelity paths (the best-quality conversions and the SubtitleEdit SDH
|
||||
stripper) only activate when you install the SubtitleEdit CLI and put it on your `PATH`.
|
||||
|
||||
@@ -43,7 +43,7 @@ Common config keys. This is a useful subset, and every `dl` flag works:
|
||||
| `sub_format` | str | *(unset)* | Convert subtitles to this format (`srt`, `vtt`, `original`, ...). |
|
||||
| `forced_subs` | bool | `false` | Include forced subtitle tracks. |
|
||||
| `forced_s_lang` | list | `[]` | Forced subtitle language(s); implies `forced_subs`. A `-` prefix excludes. |
|
||||
| `no_subs` / `no_audio` / `no_chapters` | bool | `false` | Skip that track type. |
|
||||
| `no_subs` / `no_audio` / `no_chapters` / `no_attachments` | bool | `false` | Skip that track type. `no_attachments` also skips attaching subtitle fonts. |
|
||||
| `downloads` | int | `1` | Tracks downloaded concurrently. |
|
||||
| `workers` | int | *(downloader default)* | Threads per track. |
|
||||
| `speed_limit` | str | *(unlimited)* | Total download speed cap across all downloads combined, e.g. `500k`, `5M`, `1.5G` or plain bytes/sec. Values are bytes, not bits (`5M` = 5.0 MB/s). |
|
||||
@@ -132,7 +132,7 @@ in or writes them as sidecar files. See [Subtitles](../../guide/subtitles.md) fo
|
||||
|---------|------|---------|-------------|
|
||||
| `strip_sdh` | bool | `true` | Strip SDH/CC cues (hearing-impaired annotations) into a clean track. |
|
||||
| `sdh_method` | str | `"auto"` | SDH-stripping backend to use. |
|
||||
| `preserve_formatting` | bool | `true` | Keep styling/positioning when converting or stripping. |
|
||||
| `preserve_formatting` | bool | `true` | Pass WebVTT cue text through as written. `false` re-encodes it through pycaption, which merges identical cues but flattens styling. unshackle removes WebVTT-only cue markup either way. |
|
||||
| `convert_before_strip` | bool | `true` | Convert to the working format before stripping SDH. |
|
||||
| `conversion_method` | str | `"auto"` | Subtitle conversion backend. |
|
||||
| `output_mode` | str | `"mux"` | `mux` embeds subtitles in the MKV; `sidecar` writes separate files; `both` writes sidecars and still muxes. |
|
||||
|
||||
@@ -9,8 +9,9 @@ the [DRM & CDM guide](../../guide/drm-and-cdm.md). Device files themselves live
|
||||
- **Type:** `dict` · **Default:** `{}`
|
||||
|
||||
Maps a service tag to the CDM (Widevine/PlayReady device) to use, with a `default` fallback.
|
||||
Resolution is case-insensitive: a per-request API override wins, then the per-service entry,
|
||||
then `default`.
|
||||
Resolution is case-insensitive: an override (`dl --cdm <name>`, or the `cdm` field of an API
|
||||
job) wins, then the per-service entry, then `default`. An override pins one device for the whole
|
||||
run and skips any quality or Widevine/PlayReady sub-entries.
|
||||
|
||||
```yaml
|
||||
cdm:
|
||||
|
||||
+76
-8
@@ -15,7 +15,9 @@ Fault profiles:
|
||||
rate-limit reply 429 + Retry-After during recurring throttle windows
|
||||
(window_open seconds serving / window_closed seconds throttled)
|
||||
tail-slow heavily throttle the last K segments' bodies
|
||||
throttle-cut throttle every body and close it (FIN) after N seconds or N bytes
|
||||
flaky-first fail attempt 1 per path (503), succeed on retry
|
||||
deny reply 403 to every request (a permanent error no retry can fix)
|
||||
loopback no faults; payloads served from memory (overhead ceiling)
|
||||
|
||||
Range requests are honored (206 + Content-Range) so byte-range / hedge / tail-boost
|
||||
@@ -24,6 +26,8 @@ download paths exercise correctly, and an unsatisfiable resume gets a 416.
|
||||
Standalone:
|
||||
uv run python scripts/fault_server.py --profile rate-limit --segments 32
|
||||
uv run python scripts/fault_server.py --profile stall --stall-secs 4 --port 8080
|
||||
uv run python scripts/fault_server.py --profile throttle-cut --segments 1 --seg-size 71303168
|
||||
uv run python scripts/fault_server.py --profile deny --deny-after 1
|
||||
|
||||
Importable: build a ``FaultServer``, serve it on a thread, and hash output against
|
||||
``segment_payload(path, size)``.
|
||||
@@ -74,6 +78,11 @@ class FaultProfile:
|
||||
tail_rate_kib: int = 256 # throttle rate for tail-slow segments (KiB/s)
|
||||
flaky_first: bool = False # fail attempt 1 per path, succeed after
|
||||
flaky_status: int = 503 # status returned by a flaky-first failure
|
||||
cut_secs: float = 0.0 # close the body cleanly (FIN) after this long (0 = never)
|
||||
cut_bytes: int = 0 # close the body cleanly (FIN) after this many bytes (0 = never)
|
||||
deny: bool = False # reply deny_status to every request past deny_after
|
||||
deny_status: int = 403 # status returned by a deny
|
||||
deny_after: int = 0 # serve this many attempts per path before denying (0 = deny all)
|
||||
|
||||
|
||||
PROFILES: dict[str, FaultProfile] = {
|
||||
@@ -88,6 +97,14 @@ PROFILES: dict[str, FaultProfile] = {
|
||||
),
|
||||
"tail-slow": FaultProfile("tail-slow", tail_slow=4, tail_rate_kib=256),
|
||||
"flaky-first": FaultProfile("flaky-first", flaky_first=True, flaky_status=503),
|
||||
# a CDN that throttles one flow and drops it on a timer: every connection delivers bytes
|
||||
# and then ends short, so each retry makes progress but no single attempt finishes. Not
|
||||
# seed-gated like stall and reset, because the real host cuts every attempt, not a share.
|
||||
"throttle-cut": FaultProfile("throttle-cut", body_rate_kib=480, cut_secs=6.5, cut_bytes=5 * MIB // 2),
|
||||
# a permanently refused resource (expired token, geo block): no retry and no fallback
|
||||
# recovers, so a client that keeps trying is burning attempts for nothing. --deny-after
|
||||
# serves the first N attempts per path, which lets a range probe through first.
|
||||
"deny": FaultProfile("deny", deny=True),
|
||||
}
|
||||
|
||||
|
||||
@@ -223,6 +240,9 @@ class FaultHandler(BaseHTTPRequestHandler):
|
||||
attempt = server.note_attempt(self.path)
|
||||
prof = server.profile
|
||||
|
||||
if prof.deny and attempt > prof.deny_after:
|
||||
self._send_status(prof.deny_status)
|
||||
return
|
||||
if prof.flaky_first and attempt == 1: # first attempt per path always fails, retry recovers
|
||||
self._send_status(prof.flaky_status)
|
||||
return
|
||||
@@ -235,7 +255,14 @@ class FaultHandler(BaseHTTPRequestHandler):
|
||||
rate = prof.tail_rate_kib
|
||||
stall = server.decides("stall", self.path, attempt, prof.stall_pct)
|
||||
reset = server.decides("reset", self.path, attempt, prof.reset_pct)
|
||||
self._serve(server.payload(self.path), stall=stall, reset=reset, rate_kib=rate)
|
||||
self._serve(
|
||||
server.payload(self.path),
|
||||
stall=stall,
|
||||
reset=reset,
|
||||
rate_kib=rate,
|
||||
cut_secs=prof.cut_secs,
|
||||
cut_bytes=prof.cut_bytes,
|
||||
)
|
||||
|
||||
def _send_status(self, code: int) -> None:
|
||||
server: FaultServer = self.server # type: ignore[assignment]
|
||||
@@ -252,7 +279,15 @@ class FaultHandler(BaseHTTPRequestHandler):
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def _serve(self, full: bytes, stall: bool = False, reset: bool = False, rate_kib: int = 0) -> None:
|
||||
def _serve(
|
||||
self,
|
||||
full: bytes,
|
||||
stall: bool = False,
|
||||
reset: bool = False,
|
||||
rate_kib: int = 0,
|
||||
cut_secs: float = 0.0,
|
||||
cut_bytes: int = 0,
|
||||
) -> None:
|
||||
server: FaultServer = self.server # type: ignore[assignment]
|
||||
rng = self._parse_range(len(full))
|
||||
if rng is not None and rng[0] >= len(full): # unsatisfiable resume: a real CDN answers 416
|
||||
@@ -294,20 +329,47 @@ class FaultHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(body[half:])
|
||||
self.wfile.flush()
|
||||
else:
|
||||
self._write_body(body, rate_kib)
|
||||
self._write_body(body, rate_kib, cut_secs, cut_bytes)
|
||||
except (BrokenPipeError, ConnectionError, OSError):
|
||||
self.close_connection = True
|
||||
|
||||
def _write_body(self, body: bytes, rate_kib: int) -> None:
|
||||
if rate_kib <= 0:
|
||||
def _cut(self) -> None:
|
||||
"""End the body early with a clean half-close.
|
||||
|
||||
FIN rather than RST: the client reads fewer bytes than Content-Length promised and
|
||||
raises IncompleteRead, which is the resumable short read a throttling CDN produces.
|
||||
An RST (see the reset fault) is a different error class on the client side.
|
||||
"""
|
||||
try:
|
||||
self.wfile.flush()
|
||||
self.connection.shutdown(socket.SHUT_WR)
|
||||
except OSError:
|
||||
pass
|
||||
self.close_connection = True
|
||||
|
||||
def _write_body(self, body: bytes, rate_kib: int, cut_secs: float = 0.0, cut_bytes: int = 0) -> None:
|
||||
if rate_kib <= 0 and cut_secs <= 0 and cut_bytes <= 0:
|
||||
self.wfile.write(body)
|
||||
return
|
||||
chunk = 64 * 1024 # throttle: fixed-size chunks paced to rate_kib KiB/s
|
||||
per_chunk = chunk / (rate_kib * 1024)
|
||||
per_chunk = chunk / (rate_kib * 1024) if rate_kib > 0 else 0.0
|
||||
deadline = time.monotonic() + cut_secs if cut_secs > 0 else None
|
||||
sent = 0
|
||||
for off in range(0, len(body), chunk):
|
||||
self.wfile.write(body[off : off + chunk])
|
||||
size = min(chunk, len(body) - off)
|
||||
if cut_bytes > 0:
|
||||
size = min(size, cut_bytes - sent)
|
||||
self.wfile.write(body[off : off + size])
|
||||
self.wfile.flush()
|
||||
time.sleep(per_chunk)
|
||||
sent += size
|
||||
if cut_bytes > 0 and sent >= cut_bytes:
|
||||
self._cut()
|
||||
return
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
self._cut()
|
||||
return
|
||||
if per_chunk:
|
||||
time.sleep(per_chunk)
|
||||
|
||||
|
||||
def resolve_profile(name: str, overrides: Optional[dict[str, Any]] = None) -> FaultProfile:
|
||||
@@ -338,6 +400,9 @@ def _build_arg_parser() -> argparse.ArgumentParser:
|
||||
p.add_argument("--body-rate-kib", type=int, help="Override: throttle every body to this KiB/s (0 = off).")
|
||||
p.add_argument("--tail-slow", type=int, help="Override: throttle the last K segments.")
|
||||
p.add_argument("--tail-rate-kib", type=int, help="Override: tail-slow throttle rate (KiB/s).")
|
||||
p.add_argument("--deny-after", type=int, help="Override: serve this many attempts per path before denying.")
|
||||
p.add_argument("--cut-secs", type=float, help="Override: close each body (FIN) after this long (0 = never).")
|
||||
p.add_argument("--cut-bytes", type=int, help="Override: close each body (FIN) after this many bytes (0 = never).")
|
||||
return p
|
||||
|
||||
|
||||
@@ -353,6 +418,9 @@ def main() -> None:
|
||||
"body_rate_kib": args.body_rate_kib,
|
||||
"tail_slow": args.tail_slow,
|
||||
"tail_rate_kib": args.tail_rate_kib,
|
||||
"cut_secs": args.cut_secs,
|
||||
"cut_bytes": args.cut_bytes,
|
||||
"deny_after": args.deny_after,
|
||||
}
|
||||
profile = resolve_profile(args.profile, overrides)
|
||||
server = FaultServer(profile, args.seg_size, args.segments, args.seed, port=args.port)
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Service tags resolve from an alias in any case, and a real tag beats another service's alias."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_services(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""SVCA claims "SVCB" as an alias, but SVCB is a service in its own right."""
|
||||
from unshackle.core import services
|
||||
|
||||
monkeypatch.setattr(services, "SERVICES", [Path("SVCA") / "__init__.py", Path("SVCB") / "__init__.py"])
|
||||
monkeypatch.setattr(services, "ALIASES", {"SVCA": ("SVCX", "SVCB"), "SVCB": ()})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["SVCX", "svcx", "SvCx"])
|
||||
def test_alias_resolves_in_any_case(two_services: None, value: str) -> None:
|
||||
from unshackle.core.services import Services
|
||||
|
||||
assert Services.get_tag(value) == "SVCA"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["SVCB", "svcb", "SvCb"])
|
||||
def test_real_tag_beats_another_services_alias(two_services: None, value: str) -> None:
|
||||
from unshackle.core.services import Services
|
||||
|
||||
assert Services.get_tag(value) == "SVCB"
|
||||
|
||||
|
||||
def test_unknown_value_is_returned_unchanged(two_services: None) -> None:
|
||||
from unshackle.core.services import Services
|
||||
|
||||
assert Services.get_tag("NOPE") == "NOPE"
|
||||
@@ -0,0 +1,93 @@
|
||||
import sys
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
import requests as rq
|
||||
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
from unshackle.core.downloaders.requests import download
|
||||
|
||||
# the package re-exports the `requests` function, shadowing the module of the same name
|
||||
downloader = sys.modules[download.__module__]
|
||||
|
||||
BODY = bytes(range(256)) * 256 # 64 KiB
|
||||
CUT_AFTER = 10 * 1024
|
||||
CHUNK = 16 * 1024
|
||||
|
||||
|
||||
class _CutServer(ThreadingHTTPServer):
|
||||
"""Serves BODY but ends any body longer than CUT_AFTER early, like a per-request-limited CDN."""
|
||||
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self):
|
||||
self.ranges: list[str] = []
|
||||
self.cuts = 0
|
||||
self.connections = 0
|
||||
super().__init__(("127.0.0.1", 0), _CutHandler)
|
||||
self.url = f"http://127.0.0.1:{self.server_address[1]}/seg.mp4"
|
||||
|
||||
def handle_error(self, request, client_address): # the client hanging up is expected here
|
||||
pass
|
||||
|
||||
|
||||
class _CutHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.server.connections += 1
|
||||
|
||||
def do_GET(self):
|
||||
header = self.headers.get("Range")
|
||||
self.server.ranges.append(header)
|
||||
start, end = 0, len(BODY) - 1
|
||||
if header:
|
||||
start_s, _, end_s = header.removeprefix("bytes=").partition("-")
|
||||
start, end = int(start_s), (int(end_s) if end_s else len(BODY) - 1)
|
||||
body = BODY[start : end + 1]
|
||||
self.send_response(206 if header else 200)
|
||||
if header:
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{len(BODY)}")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if len(body) > CUT_AFTER:
|
||||
self.server.cuts += 1
|
||||
self.wfile.write(body[:CUT_AFTER])
|
||||
self.close_connection = True
|
||||
return
|
||||
self.wfile.write(body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server(monkeypatch):
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
monkeypatch.setattr(downloader, "RETRY_WAIT", 0)
|
||||
monkeypatch.setattr(downloader, "REQUEST_CHUNK_SIZE", CHUNK)
|
||||
monkeypatch.setattr(downloader, "MIN_REQUEST_CHUNK", 4 * 1024)
|
||||
monkeypatch.setattr(downloader, "MIN_RESUME_PROGRESS", 1024)
|
||||
server = _CutServer()
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
yield server
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def test_a_cut_body_bounds_the_following_requests(tmp_path, server):
|
||||
save_path = tmp_path / "0.mp4"
|
||||
list(download(url=server.url, save_path=save_path, session=rq.Session()))
|
||||
|
||||
assert save_path.read_bytes() == BODY
|
||||
# the open-ended first request is cut, the chunk-sized one after it is cut too, and the
|
||||
# halved chunk fits under the host's limit, so every later request completes whole
|
||||
assert server.cuts == 2
|
||||
spans = [r.removeprefix("bytes=").split("-") for r in server.ranges if r and r.split("-")[-1]]
|
||||
assert spans and all(int(end) - int(start) + 1 <= CHUNK for start, end in spans)
|
||||
# the bounded requests are cheap because they share one pooled connection
|
||||
assert server.connections < len(server.ranges)
|
||||
@@ -201,3 +201,27 @@ def test_transient_429_with_retry_after_does_not_abort_track(server, tmp_path, m
|
||||
assert all(f.read_bytes() == PARENT for f in files)
|
||||
assert not dl.DOWNLOAD_CANCELLED.is_set()
|
||||
assert len(server.requests) == 5 # 3 successes + 2 rate-limited attempts
|
||||
|
||||
|
||||
def test_session_retry_does_not_nest_inside_an_attempt(server, tmp_path, monkeypatch):
|
||||
# a service session mounts urllib3 Retry with a 429 forcelist; download() owns segment
|
||||
# retries, so the two must not multiply (MAX_ATTEMPTS requests, not MAX_ATTEMPTS x 6)
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
server.fail_429_first = 10**6
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
session = Session()
|
||||
adapter = HTTPAdapter(
|
||||
max_retries=Retry(total=5, backoff_factor=0, status_forcelist=[429, 500, 502, 503, 504]),
|
||||
)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
run(server, tmp_path, [{"url": url(server)}], max_workers=1, session=session)
|
||||
|
||||
assert len(server.requests) == dl.MAX_ATTEMPTS
|
||||
# the caller's session keeps its own retry policy
|
||||
assert session.get_adapter("http://x/").max_retries.total == 5
|
||||
|
||||
@@ -10,12 +10,14 @@ Pins the byte-range fan-out that splits one large file across parts:
|
||||
- that failure sets only the ranged download's LOCAL abort event, never the
|
||||
process-global DOWNLOAD_CANCELLED (which would poison sibling tracks);
|
||||
- ``probe_ranged`` parses the Content-Range total on a 206 and declines a 200
|
||||
or a content-encoded 206.
|
||||
or a content-encoded 206;
|
||||
- the public single-URL path falls back to a sequential download only when the
|
||||
server does not really support ranges, and re-raises a 4xx instead of paying
|
||||
MAX_ATTEMPTS a second time from byte 0.
|
||||
|
||||
Tests 2/3 exercise ``dispatch_parts`` directly: the public single-URL path in
|
||||
``requests()`` deliberately swallows a ranged failure and falls back to a plain
|
||||
sequential download, so the raise the invariant guarantees is only observable at
|
||||
the dispatch layer.
|
||||
Tests 2/3 exercise ``dispatch_parts`` directly: the public path swallows a
|
||||
range-support failure and falls back, so the raise that the invariant guarantees
|
||||
is only observable at the dispatch layer.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
@@ -23,13 +25,18 @@ import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
from requests import Session
|
||||
from requests import HTTPError, Session
|
||||
|
||||
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
PAYLOAD = bytes(range(256)) * 128 # 32 KiB, distinctive repeating byte pattern
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code):
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class _RangeHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
srv = self.server
|
||||
@@ -48,6 +55,13 @@ class _RangeHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(b"\x00")
|
||||
return
|
||||
|
||||
if mode == "deny_parts" and rng != "bytes=0-0":
|
||||
# probe passes, every part window is refused for good (expired token, geo block)
|
||||
self.send_response(403)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# always_200 ignores Range entirely; probe_only_206 answers the 0-0 probe
|
||||
# with a real 206 but hands every wider part window a 200 full body
|
||||
serve_full = mode == "always_200" or not rng or (mode == "probe_only_206" and rng != "bytes=0-0")
|
||||
@@ -162,3 +176,41 @@ def test_probe_ranged_parses_content_range(server):
|
||||
assert dl.probe_ranged(url(server), session) == (0, False)
|
||||
server.mode = "gzip_probe"
|
||||
assert dl.probe_ranged(url(server), session) == (0, False)
|
||||
|
||||
|
||||
def test_permanent_part_failure_is_not_retried_sequentially(server, tmp_path, monkeypatch):
|
||||
server.mode = "deny_parts"
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 8 * 1024)
|
||||
with pytest.raises(HTTPError):
|
||||
run(server, tmp_path, [{"url": url(server)}], max_workers=4)
|
||||
# a sequential fallback would issue a plain unranged GET; only the probe and the
|
||||
# part windows may appear, each part capped at MAX_ATTEMPTS
|
||||
assert [r for r in server.requests if r is None] == []
|
||||
parts = len({r for r in server.requests if r and r != "bytes=0-0"})
|
||||
assert len(server.requests) <= 1 + parts * dl.MAX_ATTEMPTS
|
||||
# nothing half-written is left behind for a later run to mistake for a finished file
|
||||
assert sorted(tmp_path.glob("seg_*")) == []
|
||||
|
||||
|
||||
def test_range_unsupported_falls_back_to_sequential(server, tmp_path, monkeypatch):
|
||||
server.mode = "probe_only_206"
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 8 * 1024)
|
||||
files = run(server, tmp_path, [{"url": url(server)}], max_workers=4)
|
||||
assert len(files) == 1
|
||||
assert files[0].read_bytes() == PAYLOAD
|
||||
assert not files[0].with_name(files[0].name + ".!dev").exists()
|
||||
|
||||
|
||||
def test_permanent_status_unwraps_causes():
|
||||
assert dl.permanent_status(HTTPError(response=_Resp(403))) == 403
|
||||
assert dl.permanent_status(HTTPError(response=_Resp(429))) is None
|
||||
assert dl.permanent_status(HTTPError(response=_Resp(503))) is None
|
||||
assert dl.permanent_status(IOError("short read")) is None
|
||||
# RnetSession wraps the HTTPError one level down as __cause__
|
||||
wrapped = RuntimeError("max retries")
|
||||
wrapped.__cause__ = HTTPError(response=_Resp(404))
|
||||
assert dl.permanent_status(wrapped) == 404
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
from unshackle.core.tracks.subtitle import Subtitle
|
||||
|
||||
|
||||
def test_strip_webvtt_cue_classes():
|
||||
text = (
|
||||
"WEBVTT\n\n"
|
||||
"00:00:01.000 --> 00:00:04.000\n"
|
||||
"<c.bg_transparent>Mor, da vi blev gift,</c>\n"
|
||||
"<c>kom Ben</c> <c.a.b>og jeg</c.a.b> til dig.\n"
|
||||
"<i>Keep</i> <b>these</b> and <code>this</code>\n"
|
||||
"<i.loud>class</i> <b.x>on</b> <u.y>styling</u>\n"
|
||||
"<v.z Bob>voice</v> <lang ja>jp</lang> <ruby>\u6f22<rt>\u304b\u3093</rt></ruby>\n"
|
||||
"kara <00:00:01.500>timing <00:01.500>short\n"
|
||||
)
|
||||
out = Subtitle.strip_webvtt_cue_classes(text)
|
||||
assert "bg_transparent" not in out and "</c>" not in out and "<c>" not in out
|
||||
assert "Mor, da vi blev gift," in out
|
||||
assert "<i>Keep</i> <b>these</b> and <code>this</code>" in out
|
||||
# class list dropped, the styling tag itself kept
|
||||
assert "<i>class</i> <b>on</b> <u>styling</u>" in out
|
||||
# WebVTT-only tags go, their text stays
|
||||
assert "voice jp \u6f22\u304b\u3093" in out
|
||||
assert "kara timing short" in out
|
||||
|
||||
|
||||
def test_decode_entities(tmp_path):
|
||||
from unshackle.core.tracks.subtitle_convert import decode_entities
|
||||
|
||||
out = tmp_path / "a.srt"
|
||||
out.write_text(
|
||||
"1\n00:00:01,000 --> 00:00:04,000\nROBERT & RA'ED x 'q'\nTom ¬ sure, AT&T\n",
|
||||
encoding="utf8",
|
||||
)
|
||||
decode_entities(out)
|
||||
text = out.read_text(encoding="utf8")
|
||||
assert "ROBERT & RA'ED \xa0x 'q'" in text
|
||||
# no-semicolon legacy forms stay as written
|
||||
assert "Tom ¬ sure, AT&T" in text
|
||||
|
||||
|
||||
def test_filter_subs_converts_before_stripping(tmp_path, monkeypatch):
|
||||
"""``sdh_method: filter-subs`` used to hand subtitle-filter a .vtt and raise OSError."""
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
|
||||
monkeypatch.setattr(binaries, "SubtitleEdit", None)
|
||||
monkeypatch.setattr(config, "subtitle", {"sdh_method": "filter-subs"}, raising=False)
|
||||
|
||||
path = tmp_path / "sub.vtt"
|
||||
path.write_text(
|
||||
"WEBVTT\n\n00:00:01.000 --> 00:00:03.000\n[MUSIC PLAYING]\n\n"
|
||||
"00:00:04.000 --> 00:00:06.000\nPlain dialogue stays.\n",
|
||||
encoding="utf8",
|
||||
)
|
||||
sub = Subtitle(url="https://example.test/x", language="en", codec=Subtitle.Codec.WebVTT)
|
||||
sub.path = path
|
||||
sub.strip_hearing_impaired()
|
||||
|
||||
assert sub.codec == Subtitle.Codec.SubRip
|
||||
text = sub.path.read_text(encoding="utf8")
|
||||
assert "MUSIC PLAYING" not in text
|
||||
assert "Plain dialogue stays." in text
|
||||
+48
-20
@@ -781,6 +781,13 @@ class dl:
|
||||
@click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.")
|
||||
@click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapter markers.")
|
||||
@click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.")
|
||||
@click.option(
|
||||
"-nt",
|
||||
"--no-attachments",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Do not download or mux attachments (cover art, subtitle fonts, and files the service attaches).",
|
||||
)
|
||||
@click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.")
|
||||
@click.option(
|
||||
"--slow",
|
||||
@@ -826,6 +833,13 @@ class dl:
|
||||
default=None,
|
||||
help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.",
|
||||
)
|
||||
@click.option(
|
||||
"--cdm",
|
||||
"cdm_name",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Use this CDM device for the run, overriding the cdm config mapping.",
|
||||
)
|
||||
@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.")
|
||||
@click.option(
|
||||
"--no-proxy-download",
|
||||
@@ -939,7 +953,7 @@ class dl:
|
||||
|
||||
DRM_LOCKS: dict[str, Lock] = {}
|
||||
DRM_LOCKS_GUARD = Lock()
|
||||
VAULT_WRITE_LOCK = Lock()
|
||||
VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer")
|
||||
EXPORT_LOCK = Lock()
|
||||
LICENSE_KEY_CACHE: dict[UUID, str] = {}
|
||||
|
||||
@@ -1194,6 +1208,7 @@ class dl:
|
||||
f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}"
|
||||
)
|
||||
|
||||
self.cdm_override = ctx.params.get("cdm_name")
|
||||
cdm_only = ctx.params.get("cdm_only")
|
||||
|
||||
if cdm_only:
|
||||
@@ -1460,6 +1475,7 @@ class dl:
|
||||
no_audio: bool,
|
||||
no_chapters: bool,
|
||||
no_video: bool,
|
||||
no_attachments: bool,
|
||||
audio_description: bool,
|
||||
slow: Optional[tuple[int, int]],
|
||||
list_: bool,
|
||||
@@ -1496,6 +1512,8 @@ class dl:
|
||||
self.service_anime = bool(getattr(service, "ANIME", False))
|
||||
self.service_daily = bool(getattr(service, "DAILY", False))
|
||||
self.server_cdm = getattr(service, "_server_cdm", False)
|
||||
if self.server_cdm and self.cdm_override:
|
||||
self.log.warning("--cdm is ignored: this remote service licenses with the server CDM")
|
||||
self._remote_service = service if hasattr(service, "_server_cdm") else None
|
||||
start_time = time.time()
|
||||
|
||||
@@ -2215,6 +2233,9 @@ class dl:
|
||||
console.log("Skipped chapters as --no-chapters was used...")
|
||||
keep_chapters = False
|
||||
|
||||
if no_attachments:
|
||||
console.log("Skipped attachments as --no-attachments was used...")
|
||||
|
||||
if no_proxy_download and any(service.session.proxies.values()):
|
||||
console.log("Bypassing proxy for downloads as --no-proxy-download was used...")
|
||||
|
||||
@@ -2770,7 +2791,8 @@ class dl:
|
||||
kept_tracks.extend(title.tracks.subtitles)
|
||||
if keep_chapters:
|
||||
kept_tracks.extend(title.tracks.chapters)
|
||||
kept_tracks.extend(title.tracks.attachments)
|
||||
if not no_attachments:
|
||||
kept_tracks.extend(title.tracks.attachments)
|
||||
|
||||
title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url)
|
||||
|
||||
@@ -2927,6 +2949,7 @@ class dl:
|
||||
message="Download cancelled by user",
|
||||
context={"title": str(title)},
|
||||
)
|
||||
self.wait_vault_writes()
|
||||
return
|
||||
except Exception as e: # noqa
|
||||
# Reported and swallowed (no re-raise) so the CLI exits cleanly; flag it so the
|
||||
@@ -2968,6 +2991,7 @@ class dl:
|
||||
),
|
||||
postscript,
|
||||
)
|
||||
self.wait_vault_writes()
|
||||
return
|
||||
|
||||
if skip_dl:
|
||||
@@ -3021,7 +3045,7 @@ class dl:
|
||||
|
||||
with console.status("Checking Subtitles for Fonts..."):
|
||||
font_names: list[str] = []
|
||||
for subtitle in title.tracks.subtitles:
|
||||
for subtitle in [] if no_attachments else title.tracks.subtitles:
|
||||
if subtitle.codec in (Subtitle.Codec.SubStationAlpha, Subtitle.Codec.SubStationAlphav4):
|
||||
font_names.extend(Subtitle.extract_fonts(subtitle.path.read_text("utf8")))
|
||||
|
||||
@@ -3604,6 +3628,7 @@ class dl:
|
||||
run_context["folder"] = str(folder)
|
||||
dispatch("success", "run", run_context, postscript)
|
||||
|
||||
self.wait_vault_writes()
|
||||
dl_time = time_elapsed_since(start_time)
|
||||
|
||||
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
||||
@@ -3894,13 +3919,19 @@ class dl:
|
||||
export.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf8")
|
||||
|
||||
def flush_vault_writes(self, pending: list[Callable[[], Any]]) -> None:
|
||||
if not pending:
|
||||
return
|
||||
with self.VAULT_WRITE_LOCK:
|
||||
for write in pending:
|
||||
write()
|
||||
"""Hand the queued vault writes to the background writer so the track does not wait on them."""
|
||||
for write in pending:
|
||||
self.VAULT_WRITER.submit(write).add_done_callback(self.report_vault_write)
|
||||
pending.clear()
|
||||
|
||||
def report_vault_write(self, future: futures.Future[Any]) -> None:
|
||||
if exc := future.exception():
|
||||
self.log.error(f"Vault write failed: {exc!r}")
|
||||
|
||||
def wait_vault_writes(self) -> None:
|
||||
"""Block until every queued vault write has run."""
|
||||
self.VAULT_WRITER.submit(lambda: None).result()
|
||||
|
||||
def cache_keys_to_vaults(self, content_keys: dict[UUID, str]) -> None:
|
||||
successful_caches = self.vaults.add_keys(content_keys)
|
||||
self.log.info(
|
||||
@@ -4223,7 +4254,7 @@ class dl:
|
||||
|
||||
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
||||
|
||||
pending_vault_writes.append(partial(self.cache_keys_to_vaults, drm.content_keys))
|
||||
pending_vault_writes.append(partial(self.cache_keys_to_vaults, dict(drm.content_keys)))
|
||||
|
||||
if track_kid and track_kid not in drm.content_keys:
|
||||
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
||||
@@ -4237,10 +4268,9 @@ class dl:
|
||||
table.add_row()
|
||||
table.add_row(cek_tree)
|
||||
|
||||
if export:
|
||||
self.write_export(export, title, track, drm)
|
||||
|
||||
self.flush_vault_writes(pending_vault_writes)
|
||||
if export:
|
||||
self.write_export(export, title, track, drm)
|
||||
|
||||
elif isinstance(drm, PlayReady):
|
||||
if self.debug_logger:
|
||||
@@ -4396,7 +4426,7 @@ class dl:
|
||||
|
||||
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
||||
|
||||
pending_vault_writes.append(partial(self.cache_keys_to_vaults, drm.content_keys))
|
||||
pending_vault_writes.append(partial(self.cache_keys_to_vaults, dict(drm.content_keys)))
|
||||
|
||||
if track_kid and track_kid not in drm.content_keys:
|
||||
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
||||
@@ -4410,10 +4440,9 @@ class dl:
|
||||
table.add_row()
|
||||
table.add_row(cek_tree)
|
||||
|
||||
if export:
|
||||
self.write_export(export, title, track, drm)
|
||||
|
||||
self.flush_vault_writes(pending_vault_writes)
|
||||
if export:
|
||||
self.write_export(export, title, track, drm)
|
||||
|
||||
elif isinstance(drm, ClearKeyCENC):
|
||||
pending_vault_writes = []
|
||||
@@ -4536,7 +4565,7 @@ class dl:
|
||||
|
||||
self.LICENSE_KEY_CACHE.update(drm.content_keys)
|
||||
|
||||
pending_vault_writes.append(partial(self.cache_keys_to_vaults, drm.content_keys))
|
||||
pending_vault_writes.append(partial(self.cache_keys_to_vaults, dict(drm.content_keys)))
|
||||
|
||||
if track_kid and track_kid not in drm.content_keys:
|
||||
msg = f"No Content Key for KID {track_kid.hex} was returned in the License"
|
||||
@@ -4550,10 +4579,9 @@ class dl:
|
||||
table.add_row()
|
||||
table.add_row(cek_tree)
|
||||
|
||||
if export:
|
||||
self.write_export(export, title, track, drm)
|
||||
|
||||
self.flush_vault_writes(pending_vault_writes)
|
||||
if export:
|
||||
self.write_export(export, title, track, drm)
|
||||
|
||||
elif isinstance(drm, MonaLisa):
|
||||
with self.drm_lock(drm):
|
||||
|
||||
@@ -440,6 +440,7 @@ def perform_download(
|
||||
"no_proxy": params.get("no_proxy", False),
|
||||
"no_proxy_download": params.get("no_proxy_download", False),
|
||||
"profile": params.get("profile"),
|
||||
"cdm_name": params.get("cdm"),
|
||||
"repack": params.get("repack", False),
|
||||
"tag": params.get("tag"),
|
||||
"tmdb_id": params.get("tmdb_id"),
|
||||
@@ -484,10 +485,6 @@ def perform_download(
|
||||
daily=params.get("daily", False),
|
||||
output_dir=Path(params["output_dir"]) if params.get("output_dir") else None,
|
||||
)
|
||||
# Per-request CDM override (a device name in the WVDs dir); get_cdm() takes it first.
|
||||
if params.get("cdm"):
|
||||
dl_instance.cdm_override = params["cdm"]
|
||||
|
||||
# Per-request credential ("user:pass"); feed it into the map get_credentials() reads so a
|
||||
# client can authenticate without anything being persisted to disk. Without a profile,
|
||||
# get_credentials() falls back to "default", so store it there too rather than dropping it
|
||||
@@ -579,6 +576,7 @@ def perform_download(
|
||||
no_audio=params.get("no_audio", False),
|
||||
no_chapters=params.get("no_chapters", False),
|
||||
no_video=params.get("no_video", False),
|
||||
no_attachments=params.get("no_attachments", False),
|
||||
audio_description=params.get("audio_description", False),
|
||||
slow=params.get("slow", None),
|
||||
list_=False,
|
||||
|
||||
@@ -59,6 +59,7 @@ DEFAULT_DOWNLOAD_PARAMS = {
|
||||
"no_audio": False,
|
||||
"no_chapters": False,
|
||||
"no_video": False,
|
||||
"no_attachments": False,
|
||||
"audio_description": False,
|
||||
"slow": None,
|
||||
"split_audio": None,
|
||||
|
||||
@@ -734,6 +734,9 @@ async def download(request: web.Request) -> web.Response:
|
||||
no_video:
|
||||
type: boolean
|
||||
description: Do not download video tracks (default - false)
|
||||
no_attachments:
|
||||
type: boolean
|
||||
description: Do not download or mux attachments such as cover art or subtitle fonts (default - false)
|
||||
audio_description:
|
||||
type: boolean
|
||||
description: Download audio description tracks (default - false)
|
||||
@@ -754,6 +757,9 @@ async def download(request: web.Request) -> web.Response:
|
||||
cdm_only:
|
||||
type: boolean
|
||||
description: Only use CDM for content key retrieval (true) or only vaults (false) (default - None)
|
||||
cdm:
|
||||
type: string
|
||||
description: CDM device name on the server to license with, overriding the cdm config mapping. Requires serve.cdm_overrides to allow it (default - None)
|
||||
proxy:
|
||||
type: string
|
||||
description: Proxy URI or country code (default - None)
|
||||
|
||||
@@ -9,9 +9,10 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from collections import deque
|
||||
from collections import OrderedDict, deque
|
||||
from concurrent.futures import FIRST_COMPLETED, wait
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from copy import copy
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from http.cookiejar import CookieJar
|
||||
@@ -21,7 +22,7 @@ from string import Formatter
|
||||
from typing import Any, Callable, Generator, MutableMapping, Optional, Union, cast
|
||||
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
from rich import filesize
|
||||
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
@@ -29,6 +30,18 @@ from unshackle.core.utilities import get_debug_logger, get_extension
|
||||
|
||||
MAX_ATTEMPTS = 5
|
||||
RETRY_WAIT = 2
|
||||
# a cut that still delivered bytes is resumable and refreshes the attempt budget; bound it
|
||||
# so a server that sends a little and closes, over and over, still terminates
|
||||
MAX_RESUMES = 50
|
||||
MIN_RESUME_PROGRESS = 1024 * 1024
|
||||
|
||||
# a host that cuts a body it already started (per-request time limit, per-flow throttle) will
|
||||
# cut the next one at the same place. After such a cut each following request asks for a
|
||||
# bounded sub-range instead of the whole remainder, halving it on every further cut until the
|
||||
# requests fit under the limit: a sub-range that completes is a success, so it costs neither
|
||||
# an attempt nor a backoff nap.
|
||||
REQUEST_CHUNK_SIZE = 4 * 1024 * 1024
|
||||
MIN_REQUEST_CHUNK = 256 * 1024
|
||||
PROGRESS_WINDOW = 2
|
||||
|
||||
# read timeout bounds the gap between chunks so a quiet connection errors instead of hanging
|
||||
@@ -298,6 +311,23 @@ class AdaptiveWorkerController:
|
||||
return self.target
|
||||
|
||||
|
||||
def permanent_status(exc: BaseException) -> Optional[int]:
|
||||
"""HTTP status of ``exc`` when it is a client error that no retry can fix.
|
||||
|
||||
Both session types raise requests' HTTPError; RnetSession wraps it in MaxRetriesError
|
||||
as ``__cause__``, so unwrap one level like retry_sleep does. 408 and 429 do not count:
|
||||
they are transient, and a fan-out that trips a per-client rate limit can still succeed
|
||||
over one sequential connection.
|
||||
"""
|
||||
response = getattr(exc, "response", None)
|
||||
if response is None:
|
||||
response = getattr(getattr(exc, "__cause__", None), "response", None)
|
||||
status = getattr(response, "status_code", None)
|
||||
if isinstance(status, int) and 400 <= status < 500 and status not in (408, 429):
|
||||
return status
|
||||
return None
|
||||
|
||||
|
||||
def retry_sleep(exc: Exception, attempts: int) -> float:
|
||||
"""Seconds to wait before retry number ``attempts``.
|
||||
|
||||
@@ -340,6 +370,24 @@ def is_requests_session(session: Any) -> bool:
|
||||
return isinstance(session, Session)
|
||||
|
||||
|
||||
def no_retry_session(session: Session) -> Session:
|
||||
"""A shallow copy of ``session`` whose adapters never retry, sharing its state and pools.
|
||||
|
||||
download() owns segment retries, so a mounted urllib3 ``Retry`` nests inside a single
|
||||
attempt: one 429 carrying ``Retry-After: 60`` costs five one-minute sleeps before
|
||||
download() even sees the failure. requests takes no per-call retry override, and track
|
||||
threads share the caller's session, so remounting it is not an option. A shallow adapter
|
||||
copy keeps the poolmanager object, so TLS and pooling behaviour does not change.
|
||||
"""
|
||||
view = copy(session)
|
||||
view.adapters = OrderedDict()
|
||||
for prefix, adapter in session.adapters.items():
|
||||
twin = copy(adapter)
|
||||
twin.max_retries = Retry(0, read=False)
|
||||
view.adapters[prefix] = twin
|
||||
return view
|
||||
|
||||
|
||||
def is_rnet_session(session: Any) -> bool:
|
||||
"""Whether the HTTP session is an RnetSession (it uses ``resp.stream()``)."""
|
||||
from unshackle.core.session import RnetSession
|
||||
@@ -368,6 +416,7 @@ def probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]:
|
||||
rest.setdefault("max_retries", 0)
|
||||
else:
|
||||
rest.setdefault("timeout", (CONNECT_TIMEOUT, READ_TIMEOUT))
|
||||
session = no_retry_session(session)
|
||||
try:
|
||||
resp = session.get(url, stream=True, headers=headers, **rest)
|
||||
except Exception:
|
||||
@@ -680,6 +729,9 @@ def download(
|
||||
to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
|
||||
"""
|
||||
session = session or Session()
|
||||
if is_requests_session(session):
|
||||
# same reason as the rnet max_retries=0 below: download() owns segment retries
|
||||
session = no_retry_session(session)
|
||||
part_mode = part_offset is not None and part_end is not None
|
||||
|
||||
# partial data lives at the .!dev name and is renamed into place on completion:
|
||||
@@ -710,6 +762,12 @@ def download(
|
||||
|
||||
attempts = 1
|
||||
written = 0
|
||||
resumes = 0
|
||||
# None until a cut proves the host bounds one request; the sequential path also needs a
|
||||
# known complete length, or a short body could not be told apart from a finished download
|
||||
request_limit: Optional[int] = None
|
||||
known_total: Optional[int] = None
|
||||
progress_seeded = False
|
||||
while True:
|
||||
if claimed is not None and claimed():
|
||||
return
|
||||
@@ -719,6 +777,7 @@ def download(
|
||||
return
|
||||
if not part_mode:
|
||||
written = 0
|
||||
secured = written if part_mode else resume_offset
|
||||
last_speed_refresh = _time()
|
||||
|
||||
try:
|
||||
@@ -736,10 +795,24 @@ def download(
|
||||
# Content-Length accounting and is meaningless on ranged requests
|
||||
if not any(str(k).lower() == "accept-encoding" for k in req_headers):
|
||||
req_headers["Accept-Encoding"] = "identity"
|
||||
if part_mode:
|
||||
req_headers["Range"] = f"bytes={part_offset + written}-{part_end}"
|
||||
# inclusive end of this request when it covers only part of what is left, else None
|
||||
chunk_end: Optional[int] = None
|
||||
if part_mode and part_offset is not None and part_end is not None: # spelled so mypy narrows
|
||||
req_start = part_offset + written
|
||||
chunk_end = part_end
|
||||
if request_limit is not None and part_end - req_start + 1 > request_limit:
|
||||
chunk_end = req_start + request_limit - 1
|
||||
req_headers["Range"] = f"bytes={req_start}-{chunk_end}"
|
||||
elif resume_offset > 0 and not item_range:
|
||||
req_headers["Range"] = f"bytes={resume_offset}-"
|
||||
if (
|
||||
request_limit is not None
|
||||
and known_total is not None
|
||||
and known_total - resume_offset > request_limit
|
||||
):
|
||||
chunk_end = resume_offset + request_limit - 1
|
||||
req_headers["Range"] = f"bytes={resume_offset}-{chunk_end}"
|
||||
else:
|
||||
req_headers["Range"] = f"bytes={resume_offset}-"
|
||||
request_kwargs["headers"] = req_headers
|
||||
|
||||
stream = session.get(url, stream=True, **request_kwargs)
|
||||
@@ -783,6 +856,8 @@ def download(
|
||||
resume_offset = 0
|
||||
written = 0
|
||||
continue
|
||||
elif content_range[2] is not None:
|
||||
known_total = content_range[2]
|
||||
if (not part_mode) and resume_offset > 0 and not resumed:
|
||||
resume_offset = 0
|
||||
if part_mode and part_offset is not None and part_end is not None: # spelled so mypy narrows
|
||||
@@ -791,10 +866,16 @@ def download(
|
||||
content_range = parse_content_range(
|
||||
stream.headers.get("Content-Range") or stream.headers.get("content-range")
|
||||
)
|
||||
if content_range is None or content_range[0] != part_offset + written or content_range[1] > part_end:
|
||||
if (
|
||||
content_range is None
|
||||
or content_range[0] != part_offset + written
|
||||
or (chunk_end is not None and content_range[1] > chunk_end)
|
||||
or (content_range[2] is not None and content_range[2] <= part_end)
|
||||
):
|
||||
# a wrong start would land bytes at the wrong offset in the shared file,
|
||||
# and an end past part_end would spill into a sibling part's region; fail
|
||||
# the attempt before any byte is written
|
||||
# an end past what this request asked for would spill into a sibling
|
||||
# part's region, and a total at or below part_end means the resource
|
||||
# changed under the part plan; fail the attempt before any byte is written
|
||||
raise IOError(f"part {part_offset}-{part_end} got Content-Range {content_range!r}")
|
||||
if use_rnet:
|
||||
content_encoded = is_content_encoded(
|
||||
@@ -810,6 +891,10 @@ def download(
|
||||
except ValueError:
|
||||
content_length = 0
|
||||
|
||||
if known_total is None and not part_mode and not item_range and stream.status_code == 200:
|
||||
# a 200 carries the whole resource, so its length is the complete length
|
||||
known_total = content_length or None
|
||||
|
||||
if item_range and not part_mode and stream.status_code != 206:
|
||||
# server ignored the slice's Range and sent the whole parent (RFC 9110 allows
|
||||
# a 200 here); writing that as the segment would silently corrupt the merge, so
|
||||
@@ -855,9 +940,13 @@ def download(
|
||||
chunk_size = adaptive_chunk_size(content_length)
|
||||
if limiter:
|
||||
chunk_size = min(chunk_size, max(8192, int(limiter.rate / 4)))
|
||||
secured = written if part_mode else resume_offset
|
||||
total_size = (resume_offset + content_length) if resumed and content_length > 0 else content_length
|
||||
|
||||
if not segmented and not part_mode:
|
||||
if not segmented and not part_mode and not progress_seeded:
|
||||
# only the first response of this call sees the true total, and the bytes a
|
||||
# previous run left on disk are progress that the loop below never reports
|
||||
progress_seeded = True
|
||||
if total_size > 0:
|
||||
yield dict(total=total_size)
|
||||
else:
|
||||
@@ -901,47 +990,46 @@ def download(
|
||||
emit_progress = (not segmented) or part_mode
|
||||
if register is not None:
|
||||
register(stream, True)
|
||||
for chunk in chunks:
|
||||
if DOWNLOAD_CANCELLED.is_set() or (abort is not None and abort.is_set()):
|
||||
break
|
||||
if claimed is not None and claimed():
|
||||
# close the handle or Windows can't delete the stray .!dev at merge (WinError 32)
|
||||
if register is not None:
|
||||
register(stream, False)
|
||||
try:
|
||||
stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
if limiter is not None:
|
||||
limiter.consume(len(chunk))
|
||||
_write(chunk)
|
||||
download_size = len(chunk)
|
||||
written += download_size
|
||||
|
||||
if emit_progress:
|
||||
_bytes_since_yield += download_size
|
||||
_data_accumulated += download_size
|
||||
now = _time()
|
||||
time_since = now - last_speed_refresh
|
||||
if time_since > PROGRESS_WINDOW:
|
||||
yield dict(advance=_bytes_since_yield)
|
||||
_bytes_since_yield = 0
|
||||
if not part_mode:
|
||||
download_speed = math.ceil(_data_accumulated / (time_since or 1))
|
||||
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
|
||||
last_speed_refresh = now
|
||||
_data_accumulated = 0
|
||||
|
||||
if emit_progress and _bytes_since_yield > 0:
|
||||
yield dict(advance=_bytes_since_yield)
|
||||
|
||||
if register is not None:
|
||||
register(stream, False)
|
||||
# finally, not a trailing close: when the consumer abandons this generator it
|
||||
# raises GeneratorExit at a yield, and the leaked stream then holds a
|
||||
# pool_block=True connection forever, plus the .!dev handle on Windows. The
|
||||
# same finally closes a superseded hedge loser, or Windows can't delete its
|
||||
# stray .!dev at merge (WinError 32)
|
||||
try:
|
||||
stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
for chunk in chunks:
|
||||
if DOWNLOAD_CANCELLED.is_set() or (abort is not None and abort.is_set()):
|
||||
break
|
||||
if claimed is not None and claimed():
|
||||
return
|
||||
if limiter is not None:
|
||||
limiter.consume(len(chunk))
|
||||
_write(chunk)
|
||||
download_size = len(chunk)
|
||||
written += download_size
|
||||
|
||||
if emit_progress:
|
||||
_bytes_since_yield += download_size
|
||||
_data_accumulated += download_size
|
||||
now = _time()
|
||||
time_since = now - last_speed_refresh
|
||||
if time_since > PROGRESS_WINDOW:
|
||||
yield dict(advance=_bytes_since_yield)
|
||||
_bytes_since_yield = 0
|
||||
if not part_mode:
|
||||
download_speed = math.ceil(_data_accumulated / (time_since or 1))
|
||||
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
|
||||
last_speed_refresh = now
|
||||
_data_accumulated = 0
|
||||
|
||||
if emit_progress and _bytes_since_yield > 0:
|
||||
yield dict(advance=_bytes_since_yield)
|
||||
finally:
|
||||
if register is not None:
|
||||
register(stream, False)
|
||||
try:
|
||||
stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
aborted = abort is not None and abort.is_set()
|
||||
if (DOWNLOAD_CANCELLED.is_set() or aborted) and (not content_length or written < content_length):
|
||||
@@ -949,10 +1037,16 @@ def download(
|
||||
# and skip the size checks below (a short read here is expected, not an error)
|
||||
return
|
||||
|
||||
if part_mode:
|
||||
if part_mode and part_offset is not None and part_end is not None: # spelled so mypy narrows
|
||||
expected = part_end - part_offset + 1
|
||||
if written < expected:
|
||||
target = expected if chunk_end is None else chunk_end - part_offset + 1
|
||||
if written < target:
|
||||
raise IOError(f"Failed to read part {part_offset}-{part_end}: got {written}/{expected}")
|
||||
if written < expected:
|
||||
# this request's sub-range arrived whole; the rest of the part is the next
|
||||
# request, not a retry, so the attempt budget starts over for it
|
||||
attempts = 1
|
||||
continue
|
||||
elif content_length and written < content_length:
|
||||
# applies to segments too: a connection FIN'd mid-body is a clean EOF, so the
|
||||
# chunk loop ends without raising, so without this check a truncated segment would
|
||||
@@ -961,6 +1055,10 @@ def download(
|
||||
raise IOError(f"Failed to read {content_length} bytes from the track URI.")
|
||||
|
||||
if not part_mode:
|
||||
if chunk_end is not None and known_total is not None and resume_offset + written < known_total:
|
||||
resume_offset += written
|
||||
attempts = 1
|
||||
continue
|
||||
os.replace(tmp_file, save_path)
|
||||
yield dict(file_downloaded=save_path, written=resume_offset + written)
|
||||
if segmented:
|
||||
@@ -981,14 +1079,27 @@ def download(
|
||||
if not cancelled:
|
||||
raise
|
||||
return
|
||||
if on_retry is not None:
|
||||
if not part_mode:
|
||||
resume_offset = tmp_file.stat().st_size if tmp_file.exists() else 0
|
||||
# a CDN that throttles a flow and closes it every few seconds would otherwise
|
||||
# exhaust MAX_ATTEMPTS partway into a large part; a cut with no progress (or a
|
||||
# slice item, which rewrites from scratch) still spends an attempt, and only
|
||||
# those reach on_retry so the adaptive controller does not slow a healthy flow
|
||||
progress = written if part_mode else (0 if item_range else resume_offset)
|
||||
if progress - secured >= MIN_RESUME_PROGRESS and resumes < MAX_RESUMES:
|
||||
resumes += 1
|
||||
attempts = 0
|
||||
if not item_range:
|
||||
# the host cut a body it was already delivering, so ask for less next time
|
||||
request_limit = (
|
||||
REQUEST_CHUNK_SIZE if request_limit is None else max(MIN_REQUEST_CHUNK, request_limit // 2)
|
||||
)
|
||||
elif on_retry is not None:
|
||||
try:
|
||||
on_retry(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not part_mode:
|
||||
resume_offset = tmp_file.stat().st_size if tmp_file.exists() else 0
|
||||
delay = retry_sleep(exc, attempts)
|
||||
delay = retry_sleep(exc, max(1, attempts))
|
||||
if abort is not None:
|
||||
# interruptible nap: backoff can reach MAX_BACKOFF, and teardown always
|
||||
# sets the batch abort, so a parked worker must wake and exit at once
|
||||
@@ -1462,7 +1573,12 @@ def requests(
|
||||
url_item = urls[0]
|
||||
try:
|
||||
ranged_used = False
|
||||
if max_workers > 1 and speed_limiter is None and not has_range_header(url_item):
|
||||
if (
|
||||
max_workers > 1
|
||||
and speed_limiter is None
|
||||
and not has_range_header(url_item)
|
||||
and not url_item["save_path"].exists()
|
||||
):
|
||||
total_size, supports_ranges = probe_ranged(url_item["url"], session)
|
||||
if supports_ranges and total_size >= RANGE_PARALLEL_MIN_SIZE:
|
||||
try:
|
||||
@@ -1475,7 +1591,9 @@ def requests(
|
||||
ranged_used = True
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# the pre-truncated save_path is full-size, so the sequential path would
|
||||
# read it as already complete: drop it either way, no resume from it
|
||||
save_path = url_item.get("save_path")
|
||||
if save_path:
|
||||
sp = Path(save_path)
|
||||
@@ -1484,6 +1602,25 @@ def requests(
|
||||
stray.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
if DOWNLOAD_CANCELLED.is_set():
|
||||
return
|
||||
status = permanent_status(exc)
|
||||
fatal = status is not None
|
||||
if debug_logger:
|
||||
debug_logger.log(
|
||||
level="DEBUG",
|
||||
operation="ranged_parallel_failed",
|
||||
message=f"ranged download failed: {exc.__class__.__name__}",
|
||||
context={
|
||||
"error": exc.__class__.__name__,
|
||||
"status": status,
|
||||
"action": "raise" if fatal else "sequential_fallback",
|
||||
},
|
||||
)
|
||||
if fatal:
|
||||
# retrying a 4xx from byte 0 only burns MAX_ATTEMPTS again at
|
||||
# twice the wall clock
|
||||
raise
|
||||
if not ranged_used:
|
||||
yield from download(
|
||||
session=session,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
from typing import Iterator, Optional
|
||||
|
||||
import requests
|
||||
|
||||
@@ -35,12 +35,7 @@ class WindscribeVPN(Proxy):
|
||||
|
||||
def __repr__(self) -> str:
|
||||
countries = len(set(x.get("country_code") for x in self.countries if x.get("country_code")))
|
||||
servers = sum(
|
||||
len(host)
|
||||
for location in self.countries
|
||||
for group in location.get("groups", [])
|
||||
for host in group.get("hosts", [])
|
||||
)
|
||||
servers = sum(1 for _ in self.iter_hostnames())
|
||||
|
||||
return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
|
||||
|
||||
@@ -50,10 +45,11 @@ class WindscribeVPN(Proxy):
|
||||
|
||||
Supports:
|
||||
- Country code: "us", "ca", "gb"
|
||||
- Specific server: "sg007", "us150"
|
||||
- Specific server: "sg007", "us150", "us-central-150"
|
||||
- City selection: "us:seattle", "ca:toronto"
|
||||
"""
|
||||
query = query.lower()
|
||||
# Windscribe names GB hosts "uk-NNN"; Ukraine is "ua", so "uk" is unambiguous.
|
||||
query = re.sub(r"^uk(?=$|[:\d])", "gb", query.lower())
|
||||
city = None
|
||||
|
||||
if ":" in query:
|
||||
@@ -62,7 +58,7 @@ class WindscribeVPN(Proxy):
|
||||
|
||||
server_map_key = f"{query}:{city}" if city else query
|
||||
if server_map_key in self.server_map:
|
||||
hostname = self.server_map[server_map_key]
|
||||
hostname: Optional[str] = self.server_map[server_map_key]
|
||||
elif query in self.server_map:
|
||||
hostname = self.server_map[query]
|
||||
else:
|
||||
@@ -77,45 +73,44 @@ class WindscribeVPN(Proxy):
|
||||
)
|
||||
elif re.match(r"^[a-z]+$", query):
|
||||
hostname = self.get_random_server(query, city)
|
||||
elif "-" in query:
|
||||
hostname = next((h for h in self.iter_hostnames() if h.startswith(f"{query}.")), None)
|
||||
if not hostname:
|
||||
raise ValueError(f"No WindscribeVPN server found matching '{query}'.")
|
||||
else:
|
||||
raise ValueError(f"The query provided is unsupported and unrecognized: {query}")
|
||||
|
||||
if not hostname:
|
||||
return None
|
||||
if not hostname:
|
||||
return None
|
||||
|
||||
hostname = hostname.split(":")[0]
|
||||
return f"https://{self.username}:{self.password}@{hostname}:443"
|
||||
|
||||
def iter_hostnames(self) -> Iterator[str]:
|
||||
"""Yield every known server hostname."""
|
||||
for location in self.countries:
|
||||
for group in location.get("groups", []):
|
||||
for host in group.get("hosts", []):
|
||||
if host.get("hostname"):
|
||||
yield host["hostname"]
|
||||
|
||||
def get_specific_server(self, country_code: str, server_num: str) -> Optional[str]:
|
||||
"""
|
||||
Find a specific server by country code and server number.
|
||||
Find a server by country code and number, e.g. "sg007" -> "sg-007.totallyacdn.com".
|
||||
|
||||
Matches against hostnames like "sg-007.totallyacdn.com" for query "sg007".
|
||||
Tries both the raw number and zero-padded variants.
|
||||
|
||||
Args:
|
||||
country_code: Two-letter country code (e.g., "sg", "us")
|
||||
server_num: Server number as string (e.g., "007", "7", "150")
|
||||
|
||||
Returns:
|
||||
The matching hostname, or None if not found.
|
||||
Matches on the trailing number of the hostname, so "us150" also finds "us-central-150"
|
||||
and "gb044" finds "uk-044" (hostnames do not always share the country code).
|
||||
"""
|
||||
num_stripped = server_num.lstrip("0") or "0"
|
||||
candidates = {
|
||||
f"{country_code}-{server_num}.",
|
||||
f"{country_code}-{num_stripped}.",
|
||||
f"{country_code}-{server_num.zfill(3)}.",
|
||||
}
|
||||
|
||||
num = server_num.lstrip("0") or "0"
|
||||
for location in self.countries:
|
||||
if location.get("country_code", "").lower() != country_code:
|
||||
continue
|
||||
for group in location.get("groups", []):
|
||||
for host in group.get("hosts", []):
|
||||
hostname = host.get("hostname", "")
|
||||
if any(hostname.startswith(prefix) for prefix in candidates):
|
||||
match = re.match(r"^[a-z]+(?:-[a-z]+)?-0*(\d+)\.", hostname)
|
||||
if match and match.group(1) == num:
|
||||
return hostname
|
||||
|
||||
return None
|
||||
|
||||
def get_random_server(self, country_code: str, city: Optional[str] = None) -> Optional[str]:
|
||||
|
||||
@@ -171,7 +171,7 @@ class Service(metaclass=ABCMeta):
|
||||
still, during track download.
|
||||
"""
|
||||
|
||||
ALIASES: tuple[str, ...] = () # list of aliases for the service; alternatives to the service tag.
|
||||
ALIASES: tuple[str, ...] = () # alternative tags for the service, matched without case.
|
||||
GEOFENCE: tuple[str, ...] = () # list of ip regions required to use the service. empty list == no specific region.
|
||||
ANIME: bool = False # service catalogue is anime; metadata lookups prefer AniList. Title.anime overrides per title.
|
||||
DAILY: bool = False # catalog is daily/date-based. episodes are named by air date. Title.daily overrides per title.
|
||||
@@ -489,8 +489,8 @@ class Service(metaclass=ABCMeta):
|
||||
read by the Cdm and decrypted.
|
||||
|
||||
This is a very important request to get correct. A bad, unexpected, or missing
|
||||
value in the request can cause the service to detect your CDM device. The device
|
||||
can then be banned, revoked, disabled, or downgraded.
|
||||
value in the request can cause the service to detect your CDM device.
|
||||
The service can then ban, revoke, disable, or downgrade that device.
|
||||
|
||||
:param challenge: The license challenge from the Widevine CDM.
|
||||
:param title: The current `Title` from get_titles that unshackle processes now. unshackle
|
||||
@@ -510,8 +510,8 @@ class Service(metaclass=ABCMeta):
|
||||
read by the CDM and decrypted.
|
||||
|
||||
This is a very important request to get correct. A bad, unexpected, or missing
|
||||
value in the request can cause the service to detect your CDM device. The device
|
||||
can then be banned, revoked, disabled, or downgraded.
|
||||
value in the request can cause the service to detect your CDM device.
|
||||
The service can then ban, revoke, disable, or downgrade that device.
|
||||
|
||||
:param challenge: The license challenge from the PlayReady CDM.
|
||||
:param title: The current `Title` from get_titles that unshackle processes now. unshackle
|
||||
|
||||
@@ -127,8 +127,15 @@ class Services(click.Group):
|
||||
|
||||
remote_services_cache: list[dict] | None = None
|
||||
|
||||
def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
||||
"""Name the commands panel ``Services``, because this group's commands are services."""
|
||||
config = getattr(formatter, "config", None)
|
||||
if config is not None:
|
||||
config.commands_panel_title = "Services"
|
||||
super().format_help(ctx, formatter)
|
||||
|
||||
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
||||
"""Preprocess --slow so it can take an optional range value before Click parses args."""
|
||||
"""Preprocess ``--slow`` so it can take an optional range value before Click parses the arguments."""
|
||||
processed = []
|
||||
i = 0
|
||||
while i < len(args):
|
||||
@@ -317,18 +324,22 @@ class Services(click.Group):
|
||||
def get_tag(value: str) -> str:
|
||||
"""
|
||||
Get the Service Tag (the name of the service's directory, not one of its aliases) by an Alias.
|
||||
Input value can be of any case-sensitivity.
|
||||
Input value can be of any case-sensitivity, and so can the alias it matches.
|
||||
A real tag always wins over an alias: services do declare aliases that collide with another service's tag.
|
||||
This method returns the original input value if it does not match a service tag.
|
||||
"""
|
||||
original_value = value
|
||||
value = value.lower()
|
||||
value_lower = value.lower()
|
||||
tags = [path.parent.stem for path in SERVICES]
|
||||
|
||||
for path in SERVICES:
|
||||
tag = path.parent.stem
|
||||
if value in (tag.lower(), *ALIASES.get(tag, [])):
|
||||
for tag in tags:
|
||||
if value_lower == tag.lower():
|
||||
return tag
|
||||
|
||||
return original_value
|
||||
for tag in tags:
|
||||
if any(value_lower == alias.lower() for alias in ALIASES.get(tag, ())):
|
||||
return tag
|
||||
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def load(tag: str) -> Service:
|
||||
|
||||
@@ -193,7 +193,7 @@ def resolve_palette(name: object) -> dict[str, str] | None:
|
||||
|
||||
|
||||
def build_help_config(palette: dict[str, str]) -> RichHelpConfiguration:
|
||||
"""Assemble the rich-click help styling for a palette."""
|
||||
"""Assemble the rich-click help configuration for a palette."""
|
||||
return RichHelpConfiguration(
|
||||
style_option=palette["text"],
|
||||
style_switch=palette["green"],
|
||||
@@ -224,6 +224,13 @@ def build_help_config(palette: dict[str, str]) -> RichHelpConfiguration:
|
||||
style_options_table_box="HORIZONTALS",
|
||||
style_options_table_show_lines=True,
|
||||
style_options_table_border_style=palette["dark_gray"],
|
||||
padding_helptext=(0, 1, 1, 5),
|
||||
padding_usage=(1, 1, 1, 5),
|
||||
style_options_panel_padding=(0, 1, 0, 3),
|
||||
style_commands_panel_padding=(0, 1, 0, 4),
|
||||
panel_title_padding=3,
|
||||
padding_errors_panel=(0, 1, 1, 5),
|
||||
padding_errors_suggestion=(0, 1, 0, 5),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -288,6 +288,7 @@ class Subtitle(Track):
|
||||
text = Subtitle.sanitize_webvtt_timestamps(text)
|
||||
text = Subtitle.sanitize_webvtt_cue_identifiers(text)
|
||||
text = Subtitle.merge_overlapping_webvtt_cues(text)
|
||||
text = Subtitle.strip_webvtt_cue_classes(text)
|
||||
|
||||
preserve_formatting = config.subtitle.get("preserve_formatting", True)
|
||||
|
||||
@@ -315,6 +316,29 @@ class Subtitle(Track):
|
||||
# Keep the sanitized version even if parsing failed
|
||||
self.path.write_text(text, encoding="utf8")
|
||||
|
||||
@staticmethod
|
||||
def strip_webvtt_cue_classes(text: str) -> str:
|
||||
"""
|
||||
Reduce a cue payload to the markup every target format shares: ``<i>``, ``<b>``
|
||||
and ``<u>`` without their class list. It removes class spans (``<c.foo>``),
|
||||
``<v>``, ``<lang>``, ``<ruby>``/``<rt>`` and karaoke timestamps
|
||||
(``<00:00:01.000>``).
|
||||
|
||||
A class only selects a CSS ``::cue`` rule that no player receives, so it never
|
||||
carries styling, but it does break rendering. FFmpeg's SubRip reader treats a tag
|
||||
name as a tag only while it matches ``[0-9a-zA-Z_/]``. The dot fails that test, so
|
||||
FFmpeg prints ``<i.loud>`` as cue text and then emits the closing italic-off with
|
||||
nothing opened. FFmpeg's WebVTT reader matches ``i``/``b``/``u`` exactly and drops
|
||||
the rest, so there a class silently loses the styling instead. The tags this
|
||||
method removes outright are the ones both readers already discard, and that
|
||||
libass would draw as text if a conversion carried them into ASS.
|
||||
"""
|
||||
text = re.sub(r"</?c(?:\.[^\s>]*)?(?:\s[^>]*)?>", "", text)
|
||||
text = re.sub(r"(</?)([ibu])\.[^\s>]*", r"\1\2", text)
|
||||
text = re.sub(r"</?(?:v|lang)(?:\.[^\s>]*)?(?:\s[^>]*)?>", "", text)
|
||||
text = re.sub(r"</?(?:ruby|rt)>", "", text)
|
||||
return re.sub(r"<\d{2,}:\d{2}(?::\d{2})?\.\d{3}>", "", text)
|
||||
|
||||
@staticmethod
|
||||
def strip_webvtt_timestamp_map(text: str) -> str:
|
||||
"""
|
||||
@@ -1000,22 +1024,6 @@ class Subtitle(Track):
|
||||
return
|
||||
elif sdh_method == "subtitleedit" and binaries.SubtitleEdit:
|
||||
pass # Continue to SubtitleEdit section below
|
||||
elif sdh_method == "filter-subs":
|
||||
sub = Subtitles(self.path)
|
||||
try:
|
||||
sub.filter(rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True)
|
||||
except ValueError as e:
|
||||
if "too many values to unpack" in str(e):
|
||||
# Retry without name removal if the error is due to multiple colons in time references
|
||||
# This can happen with lines like "at 10:00 and 2:00"
|
||||
sub = Subtitles(self.path)
|
||||
sub.filter(
|
||||
rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True
|
||||
)
|
||||
else:
|
||||
raise
|
||||
sub.save()
|
||||
return
|
||||
elif sdh_method == "auto":
|
||||
if self.codec == Subtitle.Codec.SubRip:
|
||||
try:
|
||||
|
||||
@@ -14,7 +14,9 @@ the codec enum, ``parse``, sanitizers and cue helpers (the collaborators backend
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -204,7 +206,7 @@ class Pysubs2Backend:
|
||||
return 1 if source in (Codec.SubStationAlpha, Codec.SubStationAlphav4) else 2
|
||||
|
||||
def convert(self, source: Codec, src: Path, target: Codec, out: Path) -> None:
|
||||
subs = pysubs2.load(str(src), encoding="utf-8")
|
||||
subs = pysubs2.load(str(src), encoding="utf-8-sig")
|
||||
subs.save(str(out), format_=PYSUBS2_FORMATS[target], encoding="utf-8")
|
||||
|
||||
|
||||
@@ -289,6 +291,30 @@ def resolve_backends(source: Codec, target: Codec, *, pin: Optional[str] = None)
|
||||
return sorted(available, key=lambda b: b.rank(source, target))
|
||||
|
||||
|
||||
# Formats with no entity syntax of their own. WebVTT and TTML keep theirs escaped.
|
||||
PLAIN_TEXT_TARGETS: frozenset[Codec] = frozenset({Codec.SubRip, Codec.SubStationAlpha, Codec.SubStationAlphav4})
|
||||
|
||||
|
||||
def decode_entities(out: Path) -> None:
|
||||
"""
|
||||
Turn HTML entities into real characters in a format that has no entity syntax.
|
||||
|
||||
FFmpeg's SubRip reader (mpv, VLC, Plex, Jellyfin) and libass draw ``&`` and the
|
||||
other entities as literal text. Backends differ here: subby and SubtitleEdit
|
||||
decode, pysubs2 does not.
|
||||
"""
|
||||
text = out.read_text(encoding="utf8")
|
||||
if "&" not in text:
|
||||
return
|
||||
decoded = re.sub(
|
||||
r"&(?:#\d{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});",
|
||||
lambda m: html.unescape(m.group(0)),
|
||||
text,
|
||||
)
|
||||
if decoded != text:
|
||||
out.write_text(decoded, encoding="utf8")
|
||||
|
||||
|
||||
def finalize(sub: Subtitle, target: Codec, out: Path) -> Path:
|
||||
"""Swap the track onto the converted file and fire the OnConverted callback."""
|
||||
original = sub.path
|
||||
@@ -338,6 +364,8 @@ def run_conversion(sub: Subtitle, target: Codec, *, pin: Optional[str] = None, f
|
||||
last_exc = e
|
||||
log.debug(f"Subtitle backend {backend.name} failed ({source.name}->{target.name}): {e}")
|
||||
continue
|
||||
if target in PLAIN_TEXT_TARGETS:
|
||||
decode_entities(out)
|
||||
return finalize(sub, target, out)
|
||||
|
||||
raise RuntimeError(f"All subtitle backends failed for {source.name}->{target.name}") from last_exc
|
||||
|
||||
Reference in New Issue
Block a user