From a0515f8d5778951cea41bf0ea81b6d685766fcf4 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sun, 13 Sep 2026 14:41:01 -0600 Subject: [PATCH] feat(dl): add --proxy-download for a separate download proxy Manifest, licence and auth stay on --proxy; track, subtitle and attachment downloads use the second proxy. Also exposed as proxy_download on the REST download job with the same server_proxy gate. The download session now ignores HTTP(S)_PROXY and copies rnet cookies set by name. --- docs/dev/rest-api/endpoints.md | 2 +- docs/guide/cli-reference.md | 1 + docs/guide/downloading.md | 7 ++- docs/guide/proxies-and-vpn.md | 18 +++++++- tests/tracks/test_attachment_download.py | 57 ++++++++++++++++++++++++ unshackle/commands/dl.py | 23 +++++++++- unshackle/core/api/download_manager.py | 9 ++-- unshackle/core/api/handlers.py | 11 +++++ unshackle/core/api/routes.py | 3 ++ unshackle/core/tracks/attachment.py | 8 +++- unshackle/core/tracks/subtitle.py | 2 + unshackle/core/tracks/track.py | 30 ++++++++++--- 12 files changed, 156 insertions(+), 15 deletions(-) diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 09f140a4..78b6ed08 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -415,7 +415,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `skip_dl` | boolean | `false` | Only fetch keys, do not download. | | `export` | boolean | `false` | Export manifest/keys/subs to JSON. | | `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. | -| `proxy` / `no_proxy` / `no_proxy_download` | string / bool / bool | `null` / `false` / `false` | Proxy controls. `proxy` must be a full proxy URI unless the API key has `server_proxy`. | +| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. | | `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. | | `workers` | int | `null` | Threads per track. | | `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 13cd897a..1e7a6367 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -198,6 +198,7 @@ order the service used, then renumbers the episodes into the order you asked for | `--proxy` | Proxy URI, a 2-letter country code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`). | | `--no-proxy` | Force-disable all proxy use. | | `--no-proxy-download` | Bypass the proxy for **all downloads** (manifest, licence, and auth stay proxied). | +| `--proxy-download` | Use a different proxy for **all downloads**, in the same forms as `--proxy` (manifest, licence, and auth stay on `--proxy`). | | `--remote` | Use a remote unshackle server. | | `--server` | Name a remote server from the `remote_services` config. | diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 744eb9f0..5e5b37ef 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -729,12 +729,15 @@ unshackle dl --proxy nordvpn:ca EXAMPLE 81234567 unshackle dl --proxy 'http://user:pass@host:8080' EXAMPLE 81234567 ``` -Two related flags: +Three related flags: - `--no-proxy`: force-disable all proxy use for this run. - `--no-proxy-download`: bypass the proxy for **all downloads**. The manifest, license, and authentication requests still go through the proxy. This is useful when you need the proxy only to satisfy geo-checks, not to move the bulk of the data. +- `--proxy-download`: use a different proxy for **all downloads**. The manifest, + license, and authentication requests still go through `--proxy`. Takes the same forms + as `--proxy`. ## Performance and caching @@ -954,7 +957,7 @@ authoritative list. | `--output` | `-o` | Output directory for this run. | | `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. | | `--postscript` / `--no-postscript` | | Run a command after each output file, or run none. | -| `--proxy` / `--no-proxy` / `--no-proxy-download` | | Proxy control. | +| `--proxy` / `--no-proxy` / `--no-proxy-download` / `--proxy-download` | | Proxy control. | | `--workers` / `--downloads` / `--slow` | | Concurrency and pacing. | | `--list` / `--list-titles` / `--skip-dl` | | Dry runs. | | `--cdm-only` / `--vaults-only` | | Content key source control. | diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md index 8074f69e..cf0f0445 100644 --- a/docs/guide/proxies-and-vpn.md +++ b/docs/guide/proxies-and-vpn.md @@ -22,13 +22,14 @@ a query to a real proxy URL: [basic static proxies](#basic-static-proxies), ## The `--proxy` flag -Three options on the `dl` command control proxying at download time: +Four options on the `dl` command control proxying at download time: | Option | Effect | |---|---| | `--proxy` | The proxy to use. Either an explicit URI, or a query that unshackle resolves against your configured providers. | | `--no-proxy` | Force **all** proxy use off. No providers are initialised and no proxy query is resolved. | | `--no-proxy-download` | Use the proxy for the manifest, licence, and authentication, but bypass it for the **downloads** themselves. | +| `--proxy-download` | Use `--proxy` for the manifest, licence, and authentication, and a different proxy for the **downloads**. Takes the same forms as `--proxy`. | ```shell title="Explicit proxy URI" unshackle dl --proxy http://user:pass@1.2.3.4:8080 EXAMPLE 81234567 @@ -64,6 +65,21 @@ unshackle dl --proxy gb --no-proxy-download EXAMPLE 10a1234 cause segment fetches to fail or return the wrong region. If downloads break with `--no-proxy-download`, remove it. +### `--proxy-download` + +`--proxy-download` is the middle ground: the manifest, licence, and authentication go through +`--proxy`, and the downloads go through a second proxy. Use it when one proxy passes the +geo-check and another one is faster for bulk traffic. The value takes the same forms as +`--proxy`, so a country code or `provider:region` resolves against the same providers. + +```shell title="Manifest and licence over NordVPN, downloads over Windscribe" +unshackle dl --proxy nordvpn:us --proxy-download windscribevpn:us EXAMPLE 10a1234 +``` + +`--no-proxy` and `--no-proxy-download` both override `--proxy-download`. The same +region warning applies: if the service ties segment delivery to the manifest region, keep +both proxies in that region. + ## How proxy resolution works unshackle interprets the value you pass to `--proxy` in one of three ways. It tries diff --git a/tests/tracks/test_attachment_download.py b/tests/tracks/test_attachment_download.py index caab4ea7..b1777315 100644 --- a/tests/tracks/test_attachment_download.py +++ b/tests/tracks/test_attachment_download.py @@ -57,3 +57,60 @@ def test_download_with_flag_bypasses_proxy(tmp_path, monkeypatch) -> None: assert not proxied.got assert direct.got == [URL] assert attachment.path is not None and attachment.path.read_bytes() == b"jpegdata" + + +def test_download_with_proxy_download_uses_only_that_proxy(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(config.directories, "temp", tmp_path) + seen: dict = {} + + def fake_direct(session, proxy=None): + seen["proxy"] = proxy + return FakeSession(proxies={"all": proxy}) + + monkeypatch.setattr(track_module, "direct_session", fake_direct) + proxied = FakeSession(proxies={"https": "http://manifest-proxy:1"}) + attachment = Attachment.from_url(URL, name="thumbnail", session=proxied) + attachment.download(proxied, proxy_download="http://dl-proxy:2") + assert not proxied.got + assert seen["proxy"] == "http://dl-proxy:2" + assert proxied.proxies == {"https": "http://manifest-proxy:1"} + + +def test_no_proxy_download_wins_over_proxy_download(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(config.directories, "temp", tmp_path) + seen: dict = {} + monkeypatch.setattr( + track_module, "direct_session", lambda s, proxy=None: seen.setdefault("p", proxy) or FakeSession() + ) + plain = FakeSession() + Attachment.from_url(URL, name="t", session=plain).download( + plain, no_proxy_download=True, proxy_download="http://dl-proxy:2" + ) + assert "p" not in seen and plain.got == [URL] + + +def test_direct_session_routes_through_the_given_proxy() -> None: + base = requests.Session() + base.headers["X-Test"] = "1" + new = track_module.direct_session(base, "http://dl-proxy:2") + assert new.headers["X-Test"] == "1" + assert new.proxies == {"http": "http://dl-proxy:2", "https": "http://dl-proxy:2"} + assert requests.utils.select_proxy("https://cdn.example.com/a", new.proxies) == "http://dl-proxy:2" + + +def test_direct_session_proxy_beats_environment(monkeypatch) -> None: + monkeypatch.setenv("HTTPS_PROXY", "http://env-proxy:9") + new = track_module.direct_session(requests.Session(), "http://dl-proxy:2") + settings = new.merge_environment_settings("https://cdn.example.com/a", {}, None, None, None) + assert settings["proxies"]["https"] == "http://dl-proxy:2" + + +def test_direct_session_copies_rnet_cookies_set_by_name() -> None: + from unshackle.core.session import RnetSession + + rs = RnetSession() + rs.cookies.set("via_set", "1", domain="cdn.example.com") + rs.cookies["via_item"] = "3" + new = track_module.direct_session(rs, "http://dl-proxy:2") + assert new.cookies.get("via_set") == "1" + assert new.cookies.get("via_item") == "3" diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index ef5fd6c9..58f4866d 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -53,7 +53,7 @@ from unshackle.core.events import events from unshackle.core.providers.anilist import parse_anilist_ref from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN -from unshackle.core.proxies.resolve import is_loopback +from unshackle.core.proxies.resolve import is_loopback, resolve_proxy from unshackle.core.service import Service, grow_session_pool from unshackle.core.services import Services from unshackle.core.temp import with_task_temp @@ -860,6 +860,12 @@ class dl: default=False, help="Bypass proxy for all downloads. Manifest, license, and auth still use proxy.", ) + @click.option( + "--proxy-download", + type=str, + default=None, + help="Proxy for the downloads only, in the same form as --proxy. Manifest, license, and auth use --proxy.", + ) @click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.") @click.option( "--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path." @@ -1525,6 +1531,7 @@ class dl: cdm_only: Optional[bool], no_proxy: bool, no_proxy_download: bool, + proxy_download: Optional[str], no_folder: bool, no_source: bool, no_mux: bool, @@ -1628,6 +1635,16 @@ class dl: if speed_limit_bps: self.log.info(f"Speed limit: {format_speed(speed_limit_bps)}") + if no_proxy or no_proxy_download: + proxy_download = None + elif proxy_download and re.match(r"^(?:[a-z]+:){0,2}[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy_download, re.I): + # same shapes --proxy resolves against providers (two prefixes for gluetun:nordvpn:ca); else an explicit URI + try: + proxy_download = resolve_proxy(proxy_download.lower(), self.proxy_providers) + except ValueError as e: + self.log.error(f"--proxy-download: {e}") + sys.exit(1) + if export: config.directories.exports.mkdir(parents=True, exist_ok=True) export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json" @@ -2281,6 +2298,8 @@ class dl: if no_proxy_download and any(service.session.proxies.values()): console.log("Bypassing proxy for downloads as --no-proxy-download was used...") + elif proxy_download: + console.log(f"Using a separate proxy for downloads: {mask_proxy(proxy_download)}") for kind, required, available in ( ( @@ -2934,6 +2953,7 @@ class dl: track.download( session=track.session or service.session, no_proxy_download=no_proxy_download, + proxy_download=proxy_download, prepare_drm=prepare_drm_for(track), cdm=self.cdm, max_workers=workers, @@ -2970,6 +2990,7 @@ class dl: attachment.download( attachment.session or service.session, no_proxy_download=no_proxy_download, + proxy_download=proxy_download, ) if ( diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 5c4ad7ba..d51082cc 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -37,9 +37,10 @@ def redact_parameters(parameters: Dict[str, Any]) -> Dict[str, Any]: for key in SENSITIVE_PARAM_KEYS: if redacted.get(key): redacted[key] = REDACTED - proxy = redacted.get("proxy") - if isinstance(proxy, str) and "@" in proxy: - redacted["proxy"] = URL_USERINFO_RE.sub(f"{REDACTED}@", proxy) + for key in ("proxy", "proxy_download"): + proxy = redacted.get(key) + if isinstance(proxy, str) and "@" in proxy: + redacted[key] = URL_USERINFO_RE.sub(f"{REDACTED}@", proxy) return redacted @@ -455,6 +456,7 @@ def perform_download( "proxy": params.get("proxy"), "no_proxy": params.get("no_proxy", False), "no_proxy_download": params.get("no_proxy_download", False), + "proxy_download": params.get("proxy_download"), "profile": params.get("profile"), "cdm_name": params.get("cdm"), "repack": params.get("repack", False), @@ -608,6 +610,7 @@ def perform_download( cdm_only=params.get("cdm_only"), no_proxy=params.get("no_proxy", False), no_proxy_download=params.get("no_proxy_download", False), + proxy_download=params.get("proxy_download"), no_folder=params.get("no_folder", False), no_source=params.get("no_source", False), no_mux=params.get("no_mux", False), diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 82ff4993..3fd95ed9 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -82,6 +82,7 @@ DEFAULT_DOWNLOAD_PARAMS = { "proxy": None, "no_proxy": False, "no_proxy_download": False, + "proxy_download": None, "no_folder": False, "no_source": False, "no_mux": False, @@ -1696,8 +1697,18 @@ def enforce_download_gates(params: Dict[str, Any], request: Optional[web.Request "Download jobs license with the server CDM, which is not enabled for this key on this service.", ) + if params.get("proxy_download") is not None and not isinstance(params["proxy_download"], str): + raise APIError(APIErrorCode.INVALID_INPUT, "proxy_download must be a string.") + if not server_proxy_allowed(request): resolve_handler_proxy(params, params.get("service") or "", request) + if params.get("proxy_download"): + # the download proxy is gated like the main one: a full URI or nothing, never a server provider + resolve_handler_proxy( + {**params, "proxy": params["proxy_download"], "client_region": None}, + params.get("service") or "", + request, + ) requested_cdm = params.get("cdm") if requested_cdm: diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index e77d8ce4..caf3d9c9 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -810,6 +810,9 @@ async def download(request: web.Request) -> web.Response: no_proxy_download: type: boolean description: Bypass proxy for all downloads. Manifest, license, and auth still use proxy (default - false) + proxy_download: + type: string + description: Proxy for the downloads only, in the same form as proxy. Manifest, license, and auth use proxy (default - None) tag: type: string description: Set the group tag (default - None) diff --git a/unshackle/core/tracks/attachment.py b/unshackle/core/tracks/attachment.py index 4285d761..d73aedb0 100644 --- a/unshackle/core/tracks/attachment.py +++ b/unshackle/core/tracks/attachment.py @@ -102,6 +102,7 @@ class Attachment: session: Optional[AnySession] = None, *, no_proxy_download: bool = False, + proxy_download: Optional[str] = None, ) -> None: """Download a URL-backed attachment to the temp directory.""" if self.path is not None or not self.url or DOWNLOAD_LICENCE_ONLY.is_set(): @@ -110,8 +111,11 @@ class Attachment: from unshackle.core.tracks.track import direct_session session = session or self.session or requests.Session() - if no_proxy_download and any(session.proxies.values()): - session = direct_session(session) + if no_proxy_download: + if any(session.proxies.values()): + session = direct_session(session) + elif proxy_download: + session = direct_session(session, proxy_download) download_path = config.directories.temp / (self.file_name or "attachment") try: diff --git a/unshackle/core/tracks/subtitle.py b/unshackle/core/tracks/subtitle.py index d4aa19cb..f0e5d0ab 100644 --- a/unshackle/core/tracks/subtitle.py +++ b/unshackle/core/tracks/subtitle.py @@ -247,6 +247,7 @@ class Subtitle(Track): *, cdm: Optional[object] = None, no_proxy_download: bool = False, + proxy_download: Optional[str] = None, adaptive_workers: bool = False, download_processes: int = 1, ): @@ -257,6 +258,7 @@ class Subtitle(Track): progress, cdm=cdm, no_proxy_download=no_proxy_download, + proxy_download=proxy_download, adaptive_workers=adaptive_workers, download_processes=download_processes, ) diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 8c7d1bb1..28ceeb4c 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -39,9 +39,13 @@ DRM_PREFERENCE_TYPES: dict[str, Union[type[Widevine], type[PlayReady]]] = { } -def direct_session(session: Union[Session, "RnetSession"]) -> Session: - """requests.Session with copied headers/cookies and no proxy.""" +def direct_session(session: Union[Session, "RnetSession"], proxy: Optional[str] = None) -> Session: + """requests.Session with copied headers/cookies and no proxy, or only ``proxy`` when given.""" new = Session() + if proxy: + # requests lets HTTP(S)_PROXY override session proxies, so the env must be ignored for a chosen proxy + new.trust_env = False + new.proxies = {"http": proxy, "https": proxy} headers = getattr(session, "headers", None) if headers is not None: try: @@ -55,6 +59,17 @@ def direct_session(session: Union[Session, "RnetSession"]) -> Session: new.cookies.update(jar if jar is not None else cookies) except Exception: pass + # RnetCookieAdapter.jar holds only cookies that arrived as a CookieJar; the ones set by name or + # received from a server live in its domain map, so copy those across too + by_domain = getattr(cookies, "get_dict_by_domain", None) + if by_domain is not None: + try: + for domain, named in by_domain().items(): + for name, value in named.items(): + if name not in new.cookies: + new.cookies.set(name, value, **({"domain": domain} if domain else {})) + except Exception: + pass new.mount( "https://", HTTPAdapter( @@ -563,6 +578,7 @@ class Track: *, cdm: Optional[object] = None, no_proxy_download: bool = False, + proxy_download: Optional[str] = None, adaptive_workers: bool = False, download_processes: int = 1, ): @@ -585,9 +601,13 @@ class Track: proxy = next(iter(session.proxies.values()), None) dl_session = session - if no_proxy_download and proxy: - dl_session = direct_session(session) - proxy = None + if no_proxy_download: + if proxy: + dl_session = direct_session(session) + proxy = None + elif proxy_download: + dl_session = direct_session(session, proxy_download) + proxy = proxy_download track_type = self.__class__.__name__ save_path = config.directories.temp / f"{track_type}_{self.id}.mp4"