From 783c8ae1c2c14e7c7f6bdf4d444d10f5db48b372 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Wed, 23 Sep 2026 13:23:50 -0600 Subject: [PATCH] feat(proxies): add Control D proxy provider Runs a loopback forwarder per region-named profile (unshackle-), so --proxy and --proxy-download can exit in different countries Thanks to Orni_ for the original code and the test account. Co-authored-by: AntCardFr <24798811+AntCardFR@users.noreply.github.com> --- docs/dev/rest-api/endpoints.md | 8 +- docs/dev/rest-api/remote-sessions.md | 10 +- docs/guide/cli-reference.md | 4 +- docs/guide/downloading.md | 6 +- docs/guide/proxies-and-vpn.md | 198 +++++++++- docs/index.md | 2 +- docs/reference/configuration/network.md | 14 + tests/core/test_controld_proxy.py | 402 ++++++++++++++++++++ unshackle/commands/dl.py | 30 +- unshackle/commands/search.py | 22 +- unshackle/core/api/handlers.py | 2 +- unshackle/core/api/routes.py | 6 +- unshackle/core/proxies/__init__.py | 13 +- unshackle/core/proxies/controld.py | 474 ++++++++++++++++++++++++ unshackle/core/proxies/resolve.py | 12 + unshackle/core/remote_service.py | 31 +- 16 files changed, 1194 insertions(+), 40 deletions(-) create mode 100644 tests/core/test_controld_proxy.py create mode 100644 unshackle/core/proxies/controld.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 2c763fe3..86fc3f8f 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -141,7 +141,7 @@ Find titles in a service by query string. The service must have a `search()` met | `service` | string | yes | - | Service tag. | | `query` | string | yes | - | Search query. | | `profile` | string | no | `null` | Credential/cookie profile to use. | -| `proxy` | string | no | `null` | Full proxy URI, or a country code when the API key has `server_proxy`. | +| `proxy` | string | no | `null` | Full proxy URI, a Control D resolver as `controld://@dns.controld.com`, or a country code when the API key has `server_proxy`. | | `no_proxy` | boolean | no | `false` | Force-disable all proxy use. | | `credentials`, `cookies`, `cache` | - | no | `null` | Your own login material, on a `--remote-only` server only. See [Client cache on a remote-only server](#client-cache-on-a-remote-only-server). | @@ -190,7 +190,7 @@ Get the list of titles behind a title ID, for example episodes or a movie, witho | `service` | string | yes | Service tag. | | `title_id` | string | yes | Title identifier. | | `profile` | string | no | Credential/cookie profile. | -| `proxy` | string | no | Full proxy URI, or a country code when the API key has `server_proxy`. | +| `proxy` | string | no | Full proxy URI, a Control D resolver as `controld://@dns.controld.com`, or a country code when the API key has `server_proxy`. | | `no_proxy` | boolean | no | Force-disable proxy. | | `cdm_type` | string | no | Preferred CDM type. | | `credentials`, `cookies`, `cache` | - | no | Your own login material, on a `--remote-only` server only. See [Client cache on a remote-only server](#client-cache-on-a-remote-only-server). | @@ -445,7 +445,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is | `skip_dl` | boolean | `false` | Only fetch keys, do not download. | | `export` | boolean | `false` | Export manifest/keys/subs to JSON. | | `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. | -| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. | +| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. | | `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. | | `workers` | int | `null` | Threads per track. | | `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. | @@ -891,7 +891,7 @@ Make a remote session for a service and title. Authentication runs asynchronousl Service CLI options also travel in a nested `service_params` object, which wins over a flat key with the same name. `profile` at the top level always means the credentials profile, never a service's own `--profile` option. -The `proxy` value must be a full proxy URI, unless the operator gives your API key `server_proxy` in the `serve.users` config. Without it, the server does not resolve country codes with its own proxy providers. It rejects the request with `INVALID_PROXY` when no `proxy` is set and the reported `client_region` differs from the server's region. A request that reports no `client_region` is not blocked. Pass `proxy` with your own proxy, or `no_proxy` to accept the server's own connection. With `server_proxy`, the server resolves a country code and picks a proxy for your `client_region` itself. +The `proxy` value must be a full proxy URI, unless the operator gives your API key `server_proxy` in the `serve.users` config. Without `server_proxy`, the server does not resolve country codes with its own proxy providers. It rejects the request with `INVALID_PROXY` when no `proxy` is set and the reported `client_region` differs from the server's region. A request that reports no `client_region` is not blocked. Pass `proxy` with your own proxy, or `no_proxy` to accept the server's own connection. With `server_proxy`, the server resolves a country code and picks a proxy for your `client_region` itself. A Control D resolver, `controld://@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it and needs no Control D configuration of its own. === "Response `200`" diff --git a/docs/dev/rest-api/remote-sessions.md b/docs/dev/rest-api/remote-sessions.md index 605307d9..3302791a 100644 --- a/docs/dev/rest-api/remote-sessions.md +++ b/docs/dev/rest-api/remote-sessions.md @@ -121,7 +121,12 @@ hood the client walks a remote session through its lifecycle. it sits in a different region from the reported one, asking the client to pass `--proxy` with its own proxy, or `--no-proxy` to accept the server's own connection. With `server_proxy` the server picks a proxy that matches - the client region itself + the client region itself. For `--proxy controld:ca` the client points its + Control D profile at the region and sends only the resolver, as + `controld://@dns.controld.com`. The server counts it as a full proxy + URI and runs its own forwarder for it. A bare region skips Control D, because its + proxy is a forwarder on the client. See + [Control D](../../guide/proxies-and-vpn.md#control-d) - Track-selection hints (`range_`, `vcodec`, `quality`, `best_available`) so the server fetches the right manifests - Your language and audio codec selection (`-l`, `-vl`, `-al`, `-a`, `-fs`) @@ -145,7 +150,8 @@ hood the client walks a remote session through its lifecycle. `GET /api/services` advertises the regions those accounts cover. When your own region is not one of them and you set no `--proxy`, the client resolves a proxy for the first advertised region itself, so both sides sit in a region the account - works in. + works in. This is a bare region query, so it skips Control D. A server-account + service does not accept a `controld://` resolver from the client. The server responds immediately with a session ID and a status. Authentication runs in the background on the server. diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 9bf3069a..3aa36d0f 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -195,7 +195,7 @@ order the service used, then renumbers the episodes into the order you asked for | Flag | Description | |---|---| -| `--proxy` | Proxy URI, a 2-letter country code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`). | +| `--proxy` | Proxy URI, a country or location code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`, `controld:yul`). | | `--no-proxy` | Force-disable all proxy use. | | `--no-proxy-download` | Bypass the proxy for **all downloads** (manifest, licence, and auth stay proxied). | | `--proxy-download` | Use a different proxy for **all downloads**, in the same forms as `--proxy` (manifest, licence, and auth stay on `--proxy`). | @@ -235,7 +235,7 @@ Each service defines its own query syntax. unshackle prints the results as a tre | Option | Description | |---|---| | `-p`, `--profile` | Profile for credentials and cookies. | -| `--proxy` | Proxy URI, 2-letter country code, or `provider:region`. | +| `--proxy` | Proxy URI, country or location code, or `provider:region` (e.g. `controld:ca`). | | `--no-proxy` | Force-disable all proxy use. | !!! example diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 1a4d0da2..e10943f2 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -722,12 +722,14 @@ unshackle dl --tag MYGRP --repack EXAMPLE 81234567 ## Proxies -`--proxy` accepts a full proxy URI, a 2-letter country code (resolved through your -configured proxy providers), or a `provider:region` form. +`--proxy` accepts a full proxy URI, a country or location code (resolved through your +configured proxy providers), or a `provider:region` form. See +[Proxies & VPN](proxies-and-vpn.md#how-proxy-resolution-works) for the exact grammar. ```shell title="Proxy forms" unshackle dl --proxy us EXAMPLE 81234567 unshackle dl --proxy nordvpn:ca EXAMPLE 81234567 +unshackle dl --proxy controld:yul EXAMPLE 81234567 unshackle dl --proxy 'http://user:pass@host:8080' EXAMPLE 81234567 ``` diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md index cf0f0445..8c4b87ab 100644 --- a/docs/guide/proxies-and-vpn.md +++ b/docs/guide/proxies-and-vpn.md @@ -9,7 +9,7 @@ unshackle pick a working server for you. The `--proxy` flag controls proxying, and the proxy providers unshackle supports change a query to a real proxy URL: [basic static proxies](#basic-static-proxies), -[Gluetun](#gluetun) (documented in the most detail, since it is the most flexible), +[Control D](#control-d), [Gluetun](#gluetun) (documented in the most detail, since it is the most flexible), [ExpressVPN](#expressvpn), [NordVPN](#nordvpn), [Proton VPN](#proton-vpn), [Hola](#hola), [Surfshark](#surfshark), and [Windscribe](#windscribe). @@ -106,23 +106,36 @@ flowchart TD asks each loaded proxy provider **in order** and uses the first proxy any of them returns. -unshackle compares the query against a region grammar: a two-letter country code, -optionally followed by a server number and/or a `:city` or `-city` part. It then +unshackle compares the query against a region grammar: a country or location code of two +to four letters, with an optional `_code` part (`res_yyz`), then an optional server +number and/or a `:city` or `-city` part. A `host:port` value never matches. It then lowercases the query and hands it to the proxy provider. Its section below gives the exact forms each proxy provider accepts. +A Control D resolver, `controld://@dns.controld.com`, is a fourth form. A +`--remote` client sends it to a server in place of a proxy URI, and the server starts a +forwarder for it. See [Control D](#control-d). + ### Provider load order When you do a download without `--no-proxy`, unshackle instantiates every configured proxy provider once, in this fixed order. It uses that same order for bare-query resolution: -**Basic → ExpressVPN → NordVPN → Proton VPN → Surfshark → Windscribe → Gluetun → Hola** +**Basic → ExpressVPN → NordVPN → Proton VPN → Surfshark → Windscribe → Gluetun → Hola → Control D** So if you have both Basic and NordVPN configured and use `--proxy us`, a `us` entry in your Basic config wins, because unshackle tries Basic first. To skip ahead to a specific proxy provider, prefix the query (`--proxy nordvpn:us`). +!!! note "A bare region reaches Control D last" + unshackle asks Control D for a bare region such as `--proxy us` only when no other + loaded proxy provider has a proxy for it. Hola loads automatically when the + `hola-proxy` binary is on your `PATH`, and it comes before Control D. When Control D answers, it changes your Control D + account: it points a profile at that country, and it can create an `unshackle-us` + profile and endpoint. The geofence auto-proxy uses the same order. To use Control D + directly, prefix the query (`--proxy controld:us`). + !!! note "Auto-loading providers" Most providers only load when you configure them under `proxy_providers:`. Three are special: **ExpressVPN** and **Proton VPN** also load automatically once their cache @@ -190,6 +203,167 @@ proxy_providers: unshackle dl --proxy de2 EXAMPLE 81234567 ``` +## Control D + +[Control D](https://controld.com) is a DNS service, not a proxy. It answers the hostnames +it redirects with the IP of a transparent proxy in the region you chose, and that proxy +relays onwards by SNI. There is no proxy URI to hand to Requests, so this proxy provider runs a +small **forwarder on loopback**: it resolves each connection through your Control D +resolver and relays the bytes. Everything downstream (the service session, the downloads, +`--proxy-download`) sees an ordinary HTTP proxy. + +unshackle picks the region itself, by pointing a profile's **default rule** at it. A profile +holds one region at a time, so unshackle gives each region in use a profile of its own, +each with its own endpoint and its own forwarder port. `--proxy controld:yyz +--proxy-download controld:yul` therefore uses two profiles, when two are free: Toronto for +the service and Montreal for the downloads. When no second profile is free, the second +region moves the first profile (see [step 5](#how-unshackle-chooses-a-profile)). + +Keep both locations in one country. Some CDNs also check the country of the download IP, +and refuse a `--proxy-download` exit in another country with a 403. This applies to every +proxy provider. + +`unshackle search --proxy controld:ca` works the same way as `dl`. + +### What you need + +- **A plan with traffic redirection.** On the personal plans that is *Full Control*; + *Some Control* has no redirection, and every redirect fails. The 14-day trial includes + it. See [Control D's plans](https://controld.com/plans) for current prices. +- **An API token of type *Write*.** A *Read* token cannot change a profile's default rule. + Create it in the dashboard under *API*. If you set *Allowed IPs* on the token, include + the address of each machine that runs unshackle with it. + +```yaml title="unshackle.yaml" +proxy_providers: + controld: + token: api.xxxxxxxxxxxx # a Write API token, from the Control D dashboard +``` + +```shell +unshackle dl --proxy controld:ca EXAMPLE 81234567 +``` + +The token is the only required key. You do not need to create a profile or an endpoint, +or to give a `profile` and `resolver`: unshackle creates an `unshackle-` profile +and endpoint on your account the first time it needs one, and uses it again on the next +run. Give `profile` and `resolver` only to use a profile that you made yourself (see +[How unshackle chooses a profile](#how-unshackle-chooses-a-profile)). + +The returned proxy is HTTP on loopback, +`http://127.0.0.1:{port}`. The locations themselves are read from Control D at startup +(107 of them across 68 countries, at the time of writing), so there is nothing to list: + +| Query | Meaning | +|---|---| +| `ca` | One of Control D's Canadian locations, at random. | +| `yul` | That exact location, by its code. | +| `res_yyz` | A residential location, by its code. A country pick never uses a residential location, so name it this way. | + +The [configuration reference](../reference/configuration/network.md#controld) lists +every `controld` config key. + +### How unshackle chooses a profile + +Each region query gets a profile and an endpoint named after it: `--proxy controld:ca` uses +`unshackle-ca`. For each region, unshackle uses the first match in this list: + +1. A profile that this process already pointed at that region. +2. The endpoint named `unshackle-` on your account, made by an earlier run. +3. A profile from your config that this process does not use yet. These are optional: + + ```yaml title="unshackle.yaml" + proxy_providers: + controld: + token: api.xxxxxxxxxxxx + profile: abcd123 # a profile kept for unshackle, and nothing else + resolver: efgh456 # the resolver ID of an endpoint using that profile + profiles: # more pairs, in the same shape + - profile: ijkl789 + resolver: mnop012 + max_profiles: 4 # the most profiles unshackle uses, yours included (default 4) + ``` + + The resolver ID is the last part of the endpoint's DoH URL, + `https://dns.controld.com/`. You can also give the full URL. + +4. A new profile and endpoint named `unshackle-`, while your config pairs plus the + `unshackle-*` endpoints on the account are fewer than `max_profiles`. unshackle keeps + them for the next run. `max_profiles` limits only this step: unshackle uses every pair + in your config, even more than `max_profiles`. +5. When no profile is free, a profile moves to the new region, with everything that still + uses it. unshackle first moves a profile that this process has not pointed at a region, + which another process can still use. Next, it moves the profile that this process + pointed at a region first. An `unshackle-*` profile that moves, and its endpoint, get the + new region's name. unshackle logs `Every Control D profile is in use; moving profile … to …`, + and connections already open keep the old exit until they close. + +Because the names live on the account, every unshackle process agrees on them: two runs, +or two jobs on a `serve` instance, that ask for `ca` both use `unshackle-ca`. Two processes +can still move a profile under each other when they run out of profiles (step 5). + +unshackle changes only the profiles in your config and the profiles of `unshackle-*` +endpoints. It skips an `unshackle-*` endpoint whose profile an endpoint without that name +also uses, so unshackle never redirects a profile that serves your own devices. Leave the `unshackle-*` +profiles empty: no services, no custom rules. unshackle sets their default rule to +`REDIRECT`, and the *Services* and *Custom Rules* tabs stay at zero by design. To remove +them, delete the endpoints and profiles in the dashboard. unshackle creates new ones when +it next needs them. + +!!! warning "Business accounts pay per endpoint" + Personal plans allow an unlimited number of profiles and endpoints. Business plans bill + each endpoint every month, so each `unshackle-` endpoint adds to the bill. On a business + account, configure your own pairs, as many as the countries you use at the same time, + and set `max_profiles` to that number. unshackle then never creates a profile. + +!!! note "Everything goes through the proxy, including the segments" + With the default rule redirecting, the whole download leaves through Control D's + transparent proxy. Control D serves it on a best-effort basis. Pair it with `--no-proxy-download` to + keep the geo-check on the manifest and the licence while pulling the files over your + own connection. + +!!! note "The switch is not instant" + unshackle drops its own DNS cache when it changes region, but Control D's edge may + still serve the previous answer for the remainder of the record's TTL. + +!!! note "Same machine, same IP, over IPv4" + Control D authorises its proxies by source IP: the address that queried DNS is the one + allowed to use the proxy. The forwarder resolves and connects from the same host, so + this holds. Control D also answers a redirected name only to a query over IPv4, so the + forwarder asks over IPv4 even on a host with IPv6, and ignores `HTTPS_PROXY` for these + queries. + + The forwarder refuses a connection to an IP address with a `502`, because an IP address + skips Control D's DNS and connects from your own address. A request that names a host + by IP address fails with this proxy provider. + +!!! note "With `--remote`, the server resolves" + unshackle does not send your API token to a server. For `--proxy controld:ca` with + `--remote`, unshackle points `unshackle-ca` at Canada with your token, then sends the + server only that endpoint's resolver ID, as `controld://@dns.controld.com`. + The server runs its own forwarder with it, from its own address, and needs no Control D + configuration. The resolver ID lets anyone who has it use that endpoint's proxy, so treat + it as a secret: unshackle masks it in logs and in stored job parameters. + + The server must run a version of unshackle that knows `controld://`. An older server + rejects it with `INVALID_PROXY`: `Proxy provider 'controld' not found` when your API key + has `server_proxy`, else a message that asks for a full proxy URI. A bare region, such + as `--proxy ca`, skips Control D with `--remote`, because the proxy it gives is on your + machine. A service whose accounts the server manages does not accept a proxy from the + client, and that includes a Control D resolver. + +### Troubleshooting Control D + +| Symptom | Cause and fix | +|---|---| +| `Proxy check failed`, because the exit IP is your own IP. | The profile does not redirect. Make sure that the plan includes traffic redirection (*Full Control*, not *Some Control*). After a region change, wait for the TTL of the previous answer to end. | +| A `502 Bad Gateway` for one host. With `-d`, the log shows `Control D returned no A record for …`. | Control D gave no IPv4 address for that host. | +| A `502 Bad Gateway` for one host only. With `-d`, the log shows `Refused …, an address that Control D cannot redirect`. | The request names the host by IP address. The forwarder refuses it. See the note on IPv4 above. | +| An HTTP `401` or `403` from `api.controld.com`. | The token is a *Read* token, or its *Allowed IPs* do not include this machine. Use a *Write* token. | +| `Control D needs both a profile and a resolver` or `Control D needs max_profiles of 1 or more` at startup. | The `controld` config is not valid. Give `profile` and `resolver` together, and a `max_profiles` of 1 or more. Every run fails until you correct it, also a run that does not use Control D. | +| `Could not read Control D's locations`. | unshackle could not get the location list. It tries again at the next Control D query. | +| A `403` for the segments only, with `--proxy-download` in another country. | The CDN checks the country of the download IP. Keep `--proxy-download` in the same country as `--proxy`. | + ## Gluetun Gluetun is unshackle's most flexible proxy backend, and the one to reach for if you want @@ -651,16 +825,22 @@ The returned proxy is HTTPS on **port 443**. unshackle has **two** proxy-resolution code paths that are intentionally not identical: - - `unshackle/commands/dl.py` inlines its own logic and loads **all eight** providers, + - `unshackle/commands/dl.py` inlines its own logic and loads **all nine** providers, including Windscribe and Gluetun. - `unshackle/core/proxies/resolve.py::initialize_proxy_providers()` is the shared resolver used by the REST API handlers and the remote-service client. It **omits** Windscribe and Gluetun. - Both paths accept the same query grammar (a direct URI, a `provider:query`, or a bare - country query tried against providers in order), but the set of available providers - differs. Keep this in mind when you add a proxy provider: an entry in `dl.py` does not - make it available over the API, and the opposite is also true. + Both paths accept the same query grammar (a direct URI, a `provider:query`, a bare + country query tried against providers in order, or a `controld://` resolver), but the + set of available providers differs. Keep this in mind when you add a proxy provider: + an entry in `dl.py` does not make it available over the API, and the opposite is also + true. + + The remote-service client resolves `--proxy` with its own + `resolve_remote_proxy_arg()`. It changes a `controld:` query to a `controld://` + resolver for the server with `get_remote_proxy()`, and it removes Control D from a bare + query, because a loopback forwarder is of no use to a server on another machine. !!! note "Implementing a new provider (developers)" Every proxy provider subclasses `unshackle.core.proxies.proxy.Proxy` and has three diff --git a/docs/index.md b/docs/index.md index b37b6506..43633113 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,7 +41,7 @@ at a title and it will: | Audio | AAC, AC-3, E-AC-3, AC-4, Opus, Vorbis, DTS, DTS-X, ALAC, FLAC; Dolby Atmos handling; per-channel-layout selection | | Subtitles | SRT, WebVTT, ASS/SSA, TTML and more, with optional SDH stripping and format conversion | | Key vaults | SQLite, MySQL, and HTTP/API vaults to store and share content keys | -| Proxies | Basic proxies plus NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers | +| Proxies | Basic proxies plus Control D, NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers | | Services | A plugin API for the service modules you write. unshackle includes none | | Automation | A REST API with a job queue, live progress, history, and remote-download sessions | diff --git a/docs/reference/configuration/network.md b/docs/reference/configuration/network.md index b2bdf856..e7c2f2dc 100644 --- a/docs/reference/configuration/network.md +++ b/docs/reference/configuration/network.md @@ -67,6 +67,7 @@ for the full proxy provider guide. Recognised proxy providers and their exit por | Provider | Config key | Credentials | Proxy scheme/port | |----------|-----------|-------------|-------------------| | Basic (static) | `basic` | country → URI(s) | as specified | +| Control D | `controld` | Write API token (profile/resolver pairs optional) | `http://127.0.0.1:{port}` (local forwarder) | | NordVPN | `nordvpn` | **service** credentials (48 chars combined) | `https://...:89` | | Surfshark | `surfsharkvpn` | **service** credentials (48 chars combined) | `https://...:443` | | Windscribe | `windscribevpn` | service credentials | `https://...:443` | @@ -87,6 +88,19 @@ proxy_providers: password: ``` +### `controld` + +Config keys of the Control D proxy provider. See +[Control D](../../guide/proxies-and-vpn.md#control-d) for how unshackle uses them. + +| Key | Type | Default | Meaning | +|---|---|---|---| +| `token` | `str` | required | A Control D API token of type *Write*. | +| `profile` | `str` | none | Optional. The ID of a profile you made for unshackle. Give it together with `resolver`. Without it, unshackle creates `unshackle-` profiles itself. | +| `resolver` | `str` | none | Optional. The resolver ID of an endpoint that uses `profile`, or its DoH URL `https://dns.controld.com/`. | +| `profiles` | `list` | `[]` | Optional. More pairs, each a mapping with `profile` and `resolver`. | +| `max_profiles` | `int` | `4` | The most profiles unshackle uses: your pairs plus the `unshackle-*` endpoints on the account. It must be 1 or more. unshackle creates an `unshackle-` profile and endpoint only while the total is lower. | + !!! note "Provider loading differs between CLI and REST server" The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API / remote-client path uses a separate resolver that does **not** load `windscribevpn` or diff --git a/tests/core/test_controld_proxy.py b/tests/core/test_controld_proxy.py new file mode 100644 index 00000000..ff4a834d --- /dev/null +++ b/tests/core/test_controld_proxy.py @@ -0,0 +1,402 @@ +"""Unit tests for the Control D proxy provider: the DoH answer parser, the loopback forwarder +that turns a resolver into something Python-Requests can use as a proxy, the pool of +region-named profiles, and the resolver form a client sends to a remote server.""" + +from __future__ import annotations + +import base64 +import http.server +import re +import socket +import socketserver +import threading +import time +from types import SimpleNamespace +from typing import Iterator + +import click +import pytest +import requests + +from unshackle.core.proxies import controld +from unshackle.core.proxies.controld import ( + MIN_TTL, + ControlD, + IPv4Adapter, + LocalProxy, + Resolver, + build_query, + local_proxy, + parse_a_records, +) +from unshackle.core.proxies.resolve import REGION, resolve_proxy + +pytestmark = pytest.mark.unit + + +def dns_response(ip: bytes, ttl: int = 300) -> bytes: + """A one-question, one-answer A-record response whose answer name is a compression pointer.""" + header = b"\x00\x00\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" + question = b"\x01a\x04test\x00" + b"\x00\x01\x00\x01" + answer = b"\xc0\x0c" + b"\x00\x01\x00\x01" + ttl.to_bytes(4, "big") + b"\x00\x04" + ip + return header + question + answer + + +def test_query_is_a_well_formed_a_record_question() -> None: + query = build_query("a.test.") + assert query[4:6] == b"\x00\x01" + assert query[12:] == b"\x01a\x04test\x00\x00\x01\x00\x01" + + +def test_parses_the_answer_past_a_compression_pointer() -> None: + assert parse_a_records(dns_response(bytes([1, 2, 3, 4]))) == [("1.2.3.4", 300)] + assert parse_a_records(b"") == [] + + +def origin(body: bytes) -> http.server.HTTPServer: + """A local HTTP origin that answers every GET with `body`, and keeps connections alive.""" + + class Origin(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args: object) -> None: + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Origin) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server + + +@pytest.fixture +def forwarder() -> Iterator[LocalProxy]: + """A forwarder whose resolver sends every hostname to 127.0.0.1, as Control D sends it to its proxy.""" + proxy = LocalProxy(SimpleNamespace(resolve=lambda host: "127.0.0.1")) + threading.Thread(target=proxy.serve_forever, daemon=True).start() + yield proxy + proxy.shutdown() + proxy.server_close() + + +def test_forwards_a_request_to_the_resolved_host(forwarder: LocalProxy) -> None: + pong = origin(b"pong") + try: + response = requests.get( + f"http://unresolvable.test:{pong.server_address[1]}/", proxies={"http": forwarder.uri}, timeout=10 + ) + assert response.status_code == 200 + assert response.content == b"pong" + finally: + pong.shutdown() + + +def test_a_reused_connection_never_carries_a_request_to_the_wrong_host(forwarder: LocalProxy) -> None: + """A client pools one proxy connection for every plain-HTTP host, and the relay is bound to the first.""" + first, second = origin(b"A"), origin(b"B") + try: + with requests.Session() as session: + session.proxies = {"http": forwarder.uri} + assert session.get(f"http://a.test:{first.server_address[1]}/", timeout=10).content == b"A" + assert session.get(f"http://b.test:{second.server_address[1]}/", timeout=10).content == b"B" + finally: + first.shutdown() + second.shutdown() + + +def test_connect_tunnels_bytes_both_ways(forwarder: LocalProxy) -> None: + """Every real download is HTTPS, so it all goes through CONNECT rather than absolute-form.""" + echo = socketserver.TCPServer( + ("127.0.0.1", 0), + type( + "Echo", + (socketserver.BaseRequestHandler,), + {"handle": lambda self: self.request.sendall(self.request.recv(64)[::-1])}, + ), + ) + threading.Thread(target=echo.serve_forever, daemon=True).start() + + try: + with socket.create_connection(forwarder.server_address[:2], timeout=10) as client: + client.sendall(f"CONNECT unresolvable.test:{echo.server_address[1]} HTTP/1.1\r\n\r\n".encode()) + assert b"200 Connection established" in client.recv(128) + client.sendall(b"stressed") + assert client.recv(64) == b"desserts" + finally: + echo.shutdown() + echo.server_close() + + +@pytest.mark.parametrize( + ("target", "status"), + [ + ("1.2.3.4:443", b"502"), # an IP literal would bypass Control D + ("[::1]:443", b"502"), + ("a.test:x", b"400"), + ], +) +def test_connect_refuses_what_it_cannot_redirect(forwarder: LocalProxy, target: str, status: bytes) -> None: + with socket.create_connection(forwarder.server_address[:2], timeout=10) as client: + client.sendall(f"CONNECT {target} HTTP/1.1\r\n\r\n".encode()) + assert client.recv(128).startswith(b"HTTP/1.1 " + status) + + +def test_an_unreachable_host_answers_502() -> None: + def refuse(host: str) -> str: + raise ConnectionError("no A record") + + proxy = LocalProxy(SimpleNamespace(resolve=refuse)) + threading.Thread(target=proxy.serve_forever, daemon=True).start() + try: + response = requests.get("http://unresolvable.test/", proxies={"http": proxy.uri}, timeout=10) + assert response.status_code == 502 + finally: + proxy.shutdown() + proxy.server_close() + + +def stub_session(response: bytes, seen: list) -> object: + class Session: + @staticmethod + def get(url: str, params: dict, headers: dict, timeout: int) -> object: + seen.append((url, params, headers)) + + class Answer: + content = response + + @staticmethod + def raise_for_status() -> None: + pass + + return Answer() + + return Session() + + +def test_resolve_asks_control_d_in_wireformat_and_caches_it() -> None: + seen: list = [] + resolver = Resolver("https://dns.controld.com/efgh456") + resolver.session = stub_session(dns_response(bytes([9, 8, 7, 6]), ttl=1), seen) + + assert resolver.resolve("a.test") == "9.8.7.6" + url, params, headers = seen[0] + assert url == "https://dns.controld.com/efgh456" + assert headers == {"accept": "application/dns-message"} + assert params["dns"] == base64.urlsafe_b64encode(build_query("a.test")).rstrip(b"=").decode() + assert "=" not in params["dns"] # RFC 8484 wants the padding stripped + + assert resolver.resolve("a.test") == "9.8.7.6" + assert len(seen) == 1 + assert resolver.cache["a.test"][1] > time.monotonic() + MIN_TTL - 1 # a 1s TTL is floored + + +def test_resolve_raises_when_control_d_has_no_address() -> None: + resolver = Resolver("https://dns.controld.com/efgh456") + resolver.session = stub_session(b"\x00\x00\x81\x83" + b"\x00\x00" * 4, []) + + with pytest.raises(ConnectionError): + resolver.resolve("nowhere.test") + + +def test_resolve_asks_over_ipv4_and_ignores_environment_proxies() -> None: + """Control D answers a redirected name only over IPv4, and authorises the address that asked.""" + resolver = Resolver("https://dns.controld.com/efgh456") + assert resolver.session.trust_env is False + assert isinstance(resolver.session.get_adapter("https://dns.controld.com/"), IPv4Adapter) + + +CATALOGUE = [ + {"PK": "YUL", "country": "CA", "city": "Montreal"}, + {"PK": "YYZ", "country": "CA", "city": "Toronto"}, + {"PK": "RES_YYZ", "country": "CA", "city": "Res Toronto", "hidden": True}, + {"PK": "JFK", "country": "US", "city": "New York"}, +] + + +@pytest.fixture +def catalogue(monkeypatch: pytest.MonkeyPatch) -> None: + """Serve Control D's location list without going near the network.""" + response = SimpleNamespace(raise_for_status=lambda: None, json=lambda: {"body": {"proxies": CATALOGUE}}) + monkeypatch.setattr("unshackle.core.proxies.controld.requests.get", lambda *a, **kw: response) + + +class Account: + """A fake Control D account API: holds endpoints, records every call, and answers them.""" + + def __init__(self) -> None: + self.devices: list[dict] = [] + self.calls: list[tuple[str, str, dict]] = [] + self.fail_device = False + + def device(self, name: str, profile: str) -> None: + self.devices.append( + {"PK": f"d-{name}", "name": name, "profile": {"PK": profile}, "resolvers": {"uid": f"r{profile}"}} + ) + + def request(self, method: str, url: str, data: dict, headers: dict, timeout: int) -> SimpleNamespace: + assert headers == {"authorization": "Bearer secret"} + path = url.removeprefix("https://api.controld.com") + self.calls.append((method, path, data)) + body: dict = {} + if (method, path) == ("GET", "/devices"): + body = {"devices": [dict(x) for x in self.devices]} + elif (method, path) == ("POST", "/profiles"): + body = {"profiles": [{"PK": f"p{data['name'].replace('-', '')}"}]} + elif (method, path) == ("POST", "/devices"): + if self.fail_device: + raise requests.HTTPError("endpoint limit") + self.device(data["name"], data["profile_id"]) + body = self.devices[-1] + elif method == "PUT" and path.startswith("/devices/"): + next(x for x in self.devices if x["PK"] == path.split("/")[2])["name"] = data["name"] + return SimpleNamespace(raise_for_status=lambda: None, json=lambda: {"body": body}) + + def moves(self) -> list[tuple[str, str]]: + """The (profile, location) of every default-rule change, in order.""" + return [(path.split("/")[2], data["via"]) for method, path, data in self.calls if path.endswith("/default")] + + +@pytest.fixture +def account(catalogue: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[Account]: + """A fake account, on a fresh process state whose forwarders are shut down when the test ends.""" + made = Account() + monkeypatch.setattr("unshackle.core.proxies.controld.requests.request", made.request) + controld.FORWARDERS.clear() + controld.REGIONS.clear() + yield made + for server in controld.FORWARDERS.values(): + server.shutdown() + server.server_close() + controld.FORWARDERS.clear() + controld.REGIONS.clear() + + +def test_locations_come_from_control_d(account: Account) -> None: + provider = ControlD(token="secret") + assert provider.location("ca") in ("YUL", "YYZ") # never the hidden residential one + assert provider.location("res_yyz") == "RES_YYZ" # unless asked for by name + assert provider.location("jp") is None + assert repr(provider) == "2 Countries (4 Servers)" + + +def test_a_query_steers_the_profile_default_rule(account: Account) -> None: + provider = ControlD(token="secret", profile="abcd123", resolver="efgh456", max_profiles=1) + + assert provider.get_proxy("jp") is None + assert not account.moves() + + uri = provider.get_proxy("jfk") + assert uri and uri.startswith("http://127.0.0.1:") + assert ("PUT", "/profiles/abcd123/default", {"do": "3", "via": "JFK", "status": "1"}) in account.calls + + assert provider.get_proxy("JFK") == uri + assert len(account.moves()) == 1 + + # one profile only: a second country moves it and must clear its cache + cache = controld.FORWARDERS["https://dns.controld.com/efgh456"].resolver.cache + cache["cached.test"] = ("1.2.3.4", float("inf")) + assert provider.get_proxy("yul") == uri + assert account.moves()[-1] == ("abcd123", "YUL") + assert cache == {} + + +def test_each_region_gets_its_own_profile_and_forwarder(account: Account) -> None: + """--proxy and --proxy-download in two countries must not move each other.""" + provider = ControlD( + token="secret", profile="one", resolver="r1", profiles=[{"profile": "two", "resolver": "r2"}], max_profiles=2 + ) + us = provider.get_proxy("jfk") + ca = provider.get_proxy("yul") + assert us != ca + assert account.moves() == [("one", "JFK"), ("two", "YUL")] + + # out of profiles: the oldest claim moves + assert provider.get_proxy("yyz") == us + assert account.moves()[-1] == ("one", "YYZ") + assert provider.get_proxy("yul") == ca + assert len(account.moves()) == 3 + + +def test_a_region_uses_the_endpoint_named_after_it_or_creates_one(account: Account) -> None: + account.device("unshackle-us", "kept") + account.device("Phone", "theirs") + account.device("unshackle-gb", "theirs") # shares the user's own profile, so never touched + provider = ControlD(token="secret") + + us = provider.get_proxy("us") + ca = provider.get_proxy("ca") + assert us != ca + assert ("POST", "/profiles", {"name": "unshackle-ca"}) in account.calls + assert [x[0] for x in account.moves()] == ["kept", "punshackleca"] + assert "theirs" not in provider.resolvers + + # the REST API makes a provider per request; it must find the endpoint by name + assert ControlD(token="secret").get_proxy("ca") == ca + assert len(account.moves()) == 2 + + +def test_a_moved_profile_is_renamed_to_its_new_region(account: Account) -> None: + account.device("unshackle-us", "kept") + provider = ControlD(token="secret", max_profiles=1) + + provider.get_proxy("us") + provider.get_proxy("ca") + assert ("PUT", "/profiles/kept", {"name": "unshackle-ca"}) in account.calls + assert account.devices[0]["name"] == "unshackle-ca" + assert account.moves()[-1][0] == "kept" + + +def test_a_failed_endpoint_leaves_no_profile_behind(account: Account) -> None: + account.fail_device = True + with pytest.raises(requests.HTTPError): + ControlD(token="secret").get_proxy("us") + assert ("DELETE", "/profiles/punshackleus", None) in [(m, p, d or None) for m, p, d in account.calls] + + +def test_a_bad_config_is_refused(catalogue: None) -> None: + with pytest.raises(ValueError): + ControlD(token="secret", profile="one") + with pytest.raises(ValueError): + ControlD(token="secret", max_profiles=0) + with pytest.raises(ValueError): + ControlD(token="secret", profile="one", resolver="bad id?") + + +def test_a_server_runs_a_forwarder_for_a_resolver_a_client_sends(account: Account) -> None: + provider = ControlD(token="secret", profile="abcd123", resolver="efgh456") + sent = provider.get_remote_proxy("jfk") + assert sent == "controld://efgh456@dns.controld.com" + + uri = resolve_proxy(sent, []) + assert uri and uri.startswith("http://127.0.0.1:") + assert local_proxy(sent) == uri # one forwarder per resolver + with pytest.raises(ValueError): + local_proxy("controld://evil.example/x@dns.controld.com") + + +def test_remote_sends_the_resolver_and_a_bare_region_skips_control_d( + account: Account, monkeypatch: pytest.MonkeyPatch +) -> None: + from unshackle.core import remote_service + + provider = ControlD(token="secret", profile="abcd123", resolver="efgh456") + monkeypatch.setattr("unshackle.core.proxies.resolve.initialize_proxy_providers", lambda: [provider]) + + assert remote_service.resolve_remote_proxy_arg("controld:us") == "controld://efgh456@dns.controld.com" + with pytest.raises(click.ClickException): + remote_service.resolve_remote_proxy_arg("us") # a loopback forwarder is useless to a server + + +@pytest.mark.parametrize("query", ["ca", "us:seattle", "us-ny", "uk1", "yul", "jfk", "res_yyz", "nordvpn:ca"]) +def test_a_region_query_matches(query: str) -> None: + assert re.fullmatch(rf"(?:[a-z]+:)?{REGION}", query, re.IGNORECASE) + assert re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", f"gluetun:{query}", re.IGNORECASE) + + +@pytest.mark.parametrize("uri", ["nas:3128", "localhost:8080", "proxy.example.com:8080", "1.2.3.4:8080", "us:1080"]) +def test_a_host_and_port_is_never_a_region_query(uri: str) -> None: + assert not re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", uri, re.IGNORECASE) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index a186f912..0abd1d9e 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -52,8 +52,18 @@ from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevin from unshackle.core.events import events from unshackle.core.providers.anilist import parse_anilist_ref from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int -from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN -from unshackle.core.proxies.resolve import is_loopback, resolve_proxy +from unshackle.core.proxies import ( + Basic, + ControlD, + ExpressVPN, + Gluetun, + Hola, + NordVPN, + ProtonVPN, + SurfsharkVPN, + WindscribeVPN, +) +from unshackle.core.proxies.resolve import REGION, is_loopback, resolve_proxy from unshackle.core.service import Service, grow_session_pool from unshackle.core.services import Services from unshackle.core.temp import with_task_temp @@ -1371,9 +1381,13 @@ class dl: self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) if binaries.HolaProxy: self.proxy_providers.append(Hola()) + if config.proxy_providers.get("controld"): + self.proxy_providers.append(ControlD(**config.proxy_providers["controld"])) for proxy_provider in self.proxy_providers: self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") + if proxy and proxy.startswith("controld://"): + proxy = ctx.params["proxy"] = resolve_proxy(proxy, []) if proxy: def log_proxy_used(provider: object, uri: str) -> None: @@ -1385,11 +1399,8 @@ class dl: if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): # requesting proxy from a specific proxy provider requested_provider, proxy = proxy.split(":", maxsplit=1) - # Match simple region codes (us, ca, uk1, us:ny) or provider:region (nordvpn:ca, protonvpn:us:ny). - # ':' is allowed as a city separator (e.g. nordvpn:us:seattle, protonvpn:de:berlin). - if re.match(r"^[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE) or re.match( - r"^[a-z]+:[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE - ): + # Match region codes (us, ca, uk1, us:ny, yul) or provider:region (nordvpn:ca, protonvpn:us:ny). + if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", proxy, re.IGNORECASE): proxy = proxy.lower() # Preserve the original user query (region code) for service-specific proxy_map overrides. # NOTE: `proxy` may be overwritten with the resolved proxy URI later. @@ -1637,7 +1648,10 @@ class dl: if no_proxy or no_proxy_download: proxy_download = None - elif proxy_download and re.match(r"^(?:[a-z]+:){0,2}[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy_download, re.I): + elif proxy_download and ( + proxy_download.startswith("controld://") + or re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", proxy_download, re.I) + ): # same shapes --proxy resolves against providers (two prefixes for gluetun:nordvpn:ca); else an explicit URI try: proxy_download = resolve_proxy(proxy_download.lower(), self.proxy_providers) diff --git a/unshackle/commands/search.py b/unshackle/commands/search.py index f88da5dd..eaf6c9cc 100644 --- a/unshackle/commands/search.py +++ b/unshackle/commands/search.py @@ -16,7 +16,19 @@ from unshackle.core import binaries from unshackle.core.config import config from unshackle.core.console import console from unshackle.core.constants import context_settings -from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN +from unshackle.core.proxies import ( + Basic, + ControlD, + ExpressVPN, + Gluetun, + Hola, + NordVPN, + ProtonVPN, + SurfsharkVPN, + WindscribeVPN, +) +from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.resolve import REGION from unshackle.core.service import Service from unshackle.core.services import Services from unshackle.core.utils.click_types import ContextData @@ -61,7 +73,7 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr service_config = {} merge_dict(config.services.get(service), service_config) - proxy_providers = [] + proxy_providers: list[Proxy] = [] if no_proxy: ctx.params["proxy"] = None else: @@ -84,6 +96,8 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) if binaries.HolaProxy: proxy_providers.append(Hola()) + if config.proxy_providers.get("controld"): + proxy_providers.append(ControlD(**config.proxy_providers["controld"])) for proxy_provider in proxy_providers: log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") @@ -91,9 +105,7 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr requested_provider = None if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): requested_provider, proxy = proxy.split(":", maxsplit=1) - if re.match(r"^[a-z]{2}(?:[-][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE) or re.match( - r"^[a-z]+:[a-z]{2}(?:[-][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE - ): + if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", proxy, re.IGNORECASE): proxy = proxy.lower() with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"): if requested_provider: diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 92cadeee..346aba35 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -3734,7 +3734,7 @@ def resolve_handler_proxy( if ( proxy_param - and re.match(r"^(https?://|socks)", str(proxy_param)) + and re.match(r"^(https?://|socks|controld://)", str(proxy_param)) and server_account_for(request, normalized_service) ): raise APIError( diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 028e4fa1..88326ca3 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -404,7 +404,7 @@ async def search(request: web.Request) -> web.Response: description: Profile to use for credentials and cookies (default - None) proxy: type: string - description: Full proxy URI, or a country code when the API key has server_proxy (default - None) + description: Full proxy URI, a Control D resolver as controld://@dns.controld.com, or a country code when the API key has server_proxy (default - None) no_proxy: type: boolean description: Force disable all proxy use (default - false) @@ -668,7 +668,7 @@ async def list_tracks(request: web.Request) -> web.Response: description: Specific episode/season, or song ("1-5", "2x3" for disc 2 track 3) (optional) proxy: type: string - description: Proxy configuration (optional) + description: Full proxy URI, a Control D resolver as controld://@dns.controld.com, or a country code when the API key has server_proxy (default - None) dl_params: type: object description: | @@ -898,7 +898,7 @@ async def download(request: web.Request) -> web.Response: description: CDM device name on the server to license with, overriding the cdm config mapping. Requires serve.cdm_overrides to allow it (default - None) proxy: type: string - description: Full proxy URI, or a country code when the API key has server_proxy (default - None) + description: Full proxy URI, a Control D resolver as controld://@dns.controld.com, or a country code when the API key has server_proxy (default - None) no_proxy: type: boolean description: Force disable all proxy use (default - false) diff --git a/unshackle/core/proxies/__init__.py b/unshackle/core/proxies/__init__.py index 2b313043..94a5eb94 100644 --- a/unshackle/core/proxies/__init__.py +++ b/unshackle/core/proxies/__init__.py @@ -1,4 +1,5 @@ from .basic import Basic +from .controld import ControlD from .expressvpn import ExpressVPN from .gluetun import Gluetun from .hola import Hola @@ -7,4 +8,14 @@ from .proton import ProtonVPN from .surfsharkvpn import SurfsharkVPN from .windscribevpn import WindscribeVPN -__all__ = ("Basic", "ExpressVPN", "Gluetun", "Hola", "NordVPN", "ProtonVPN", "SurfsharkVPN", "WindscribeVPN") +__all__ = ( + "Basic", + "ControlD", + "ExpressVPN", + "Gluetun", + "Hola", + "NordVPN", + "ProtonVPN", + "SurfsharkVPN", + "WindscribeVPN", +) diff --git a/unshackle/core/proxies/controld.py b/unshackle/core/proxies/controld.py new file mode 100644 index 00000000..b88d95d1 --- /dev/null +++ b/unshackle/core/proxies/controld.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import base64 +import ipaddress +import logging +import random +import re +import socket +import socketserver +import threading +import time +from typing import Optional +from urllib.parse import urlsplit + +import requests +from requests.adapters import HTTPAdapter + +from unshackle.core.proxies.proxy import Proxy + +log = logging.getLogger("ControlD") + +API = "https://api.controld.com" +DOH_TIMEOUT = 10 +MIN_TTL = 30 +MAX_TTL = 3600 +HEAD_TIMEOUT = 30 +IDLE_TIMEOUT = 600 +CHUNK = 65536 +PREFIX = "unshackle-" +SCHEME = "controld://" +RESOLVER_ID = re.compile(r"[a-z0-9]+", re.IGNORECASE) +MAX_FORWARDERS = 64 + + +def doh_url(resolver: str) -> str: + """Get the DoH URL of a Control D endpoint's resolver ID.""" + return f"https://dns.controld.com/{resolver}" + + +def build_query(host: str) -> bytes: + """Build a minimal DNS A-record query (RFC 1035) for wireformat DoH (RFC 8484).""" + labels = host.rstrip(".").encode("idna").split(b".") + qname = b"".join(bytes([len(label)]) + label for label in labels) + b"\x00" + # header: id 0 (unused by DoH), RD set, 1 question + return b"\x00\x00\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" + qname + b"\x00\x01\x00\x01" + + +def skip_name(buf: bytes, i: int) -> int: + """Return the offset past the domain name at `i`. A compression pointer ends it.""" + while i < len(buf): + length = buf[i] + if length == 0: + return i + 1 + if length & 0xC0 == 0xC0: + return i + 2 + i += length + 1 + return i + + +def parse_a_records(answer: bytes) -> list[tuple[str, int]]: + """Return the (IPv4, TTL) pairs of a DNS response, in order.""" + if len(answer) < 12: + return [] + questions = int.from_bytes(answer[4:6], "big") + answers = int.from_bytes(answer[6:8], "big") + i = 12 + for _ in range(questions): + i = skip_name(answer, i) + 4 # QTYPE + QCLASS + records = [] + for _ in range(answers): + i = skip_name(answer, i) + if i + 10 > len(answer): + break + rtype = int.from_bytes(answer[i : i + 2], "big") + ttl = int.from_bytes(answer[i + 4 : i + 8], "big") + rdlength = int.from_bytes(answer[i + 8 : i + 10], "big") + rdata = answer[i + 10 : i + 10 + rdlength] + i += 10 + rdlength + if rtype == 1 and len(rdata) == 4: + records.append((".".join(str(octet) for octet in rdata), ttl)) + return records + + +class IPv4Adapter(HTTPAdapter): + """Connect over IPv4 only: bound to 0.0.0.0, an IPv6 address fails and the next one is tried.""" + + def init_poolmanager(self, *args: object, **kwargs: object) -> None: + kwargs["source_address"] = ("0.0.0.0", 0) # nosec B104 - a source address to bind, not a listener + super().init_poolmanager(*args, **kwargs) + + +class Resolver: + def __init__(self, url: str): + """Resolve hostnames through one Control D DoH endpoint, caching answers for their TTL.""" + self.url = url + self.session = requests.Session() + # Control D redirects only IPv4 queries and authorises the proxy by the asking IP: no IPv6, no env proxy. + self.session.trust_env = False + self.session.mount("https://", IPv4Adapter()) + self.cache: dict[str, tuple[str, float]] = {} + self.lock = threading.Lock() + + def resolve(self, host: str) -> str: + host = host.lower() + with self.lock: + cached = self.cache.get(host) + if cached and cached[1] > time.monotonic(): + return cached[0] + + query = base64.urlsafe_b64encode(build_query(host)).rstrip(b"=").decode() + response = self.session.get( + self.url, + params={"dns": query}, + headers={"accept": "application/dns-message"}, + timeout=DOH_TIMEOUT, + ) + response.raise_for_status() + + records = parse_a_records(response.content) + if not records: + raise ConnectionError(f"Control D returned no A record for {host}") + + ip, ttl = records[0] + with self.lock: + self.cache[host] = (ip, time.monotonic() + min(max(ttl, MIN_TTL), MAX_TTL)) + return ip + + +def relay(source: socket.socket, sink: socket.socket) -> None: + """Copy bytes one way until the source closes, then half-close the sink.""" + try: + while True: + data = source.recv(CHUNK) + if not data: + break + sink.sendall(data) + except OSError: + pass + finally: + try: + sink.shutdown(socket.SHUT_WR) + except OSError: + pass + + +HOP_BY_HOP = (b"connection:", b"proxy-connection:", b"keep-alive:") + + +class Handler(socketserver.StreamRequestHandler): + server: LocalProxy + rbufsize = 0 # unbuffered, so no body bytes are held back from the relay + timeout = HEAD_TIMEOUT + + def reply(self, status: bytes) -> None: + try: + self.connection.sendall(b"HTTP/1.1 " + status + b"\r\n\r\n") + except OSError: + pass + + def forward_response_head(self, upstream: socket.socket) -> None: + """Send the response head on with `Connection: close`, so the client never reuses this connection.""" + data = b"" + try: + while b"\r\n\r\n" not in data and len(data) < CHUNK: + chunk = upstream.recv(CHUNK) + if not chunk: + break + data += chunk + except OSError: + pass + head, found, body = data.partition(b"\r\n\r\n") + if found: + lines = [x for x in head.split(b"\r\n") if not x.lower().startswith(HOP_BY_HOP)] + data = b"\r\n".join(lines) + b"\r\nConnection: close\r\n\r\n" + body + try: + self.connection.sendall(data) + except OSError: + pass + + def handle(self) -> None: + try: + head = self.rfile.readline(CHUNK) + headers = [] + while True: + line = self.rfile.readline(CHUNK) + if not line or line in (b"\r\n", b"\n"): + break + headers.append(line) + except OSError: + return + parts = head.split() + if len(parts) < 3: + return + + method = parts[0].decode("latin-1") + target = parts[1].decode("latin-1") + try: + if method == "CONNECT": + host, _, port_text = target.rpartition(":") if ":" in target else (target, "", "") + host, port = host.strip("[]"), int(port_text or 443) + else: + parsed = urlsplit(target) + host, port = parsed.hostname or "", parsed.port or 80 + except ValueError: + return self.reply(b"400 Bad Request") + if not host: + return self.reply(b"400 Bad Request") + + try: + ipaddress.ip_address(host) + except ValueError: + pass + else: + # an IP literal bypasses Control D's DNS and would leak this machine's own IP + log.debug(f"Refused {host}:{port}, an address that Control D cannot redirect") + return self.reply(b"502 Bad Gateway") + + try: + ip = self.server.resolver.resolve(host) + upstream = socket.create_connection((ip, port), timeout=HEAD_TIMEOUT) + except Exception as e: + log.debug(f"Could not reach {host}:{port}: {e}") + return self.reply(b"502 Bad Gateway") + + self.connection.settimeout(IDLE_TIMEOUT) + upstream.settimeout(IDLE_TIMEOUT) + try: + if method == "CONNECT": + self.connection.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + else: + headers = [x for x in headers if not x.lower().startswith(HOP_BY_HOP)] + upstream.sendall(head + b"".join(headers) + b"Connection: close\r\n\r\n") + upward = threading.Thread(target=relay, args=(self.connection, upstream), daemon=True) + upward.start() + if method != "CONNECT": + self.forward_response_head(upstream) + relay(upstream, self.connection) + # origin done: unblock the upward relay instead of waiting on a client that holds its side open + try: + self.connection.shutdown(socket.SHUT_RD) + except OSError: + pass + upward.join() + finally: + upstream.close() + + +class LocalProxy(socketserver.ThreadingTCPServer): + daemon_threads = True + + def __init__(self, resolver: Resolver): + """An HTTP forward proxy on loopback that resolves every host through `resolver`.""" + super().__init__(("127.0.0.1", 0), Handler) + self.resolver = resolver + + @property + def uri(self) -> str: + return f"http://127.0.0.1:{self.socket.getsockname()[1]}" + + +# process-wide across provider instances; separate processes share only the account's unshackle- names +FORWARDERS: dict[str, LocalProxy] = {} +REGIONS: dict[str, str] = {} # profile -> region, oldest claim first +STATE = threading.Lock() + + +def forwarder(resolver: str) -> LocalProxy: + """Get the loopback forwarder for a resolver ID, starting it on first use. Hold STATE.""" + url = doh_url(resolver) + if url not in FORWARDERS: + if len(FORWARDERS) >= MAX_FORWARDERS: + raise ValueError("Too many Control D resolvers in use in this process") + server = LocalProxy(Resolver(url)) + threading.Thread(target=server.serve_forever, daemon=True, name="controld").start() + FORWARDERS[url] = server + log.debug(f"Listening on {server.uri}") + return FORWARDERS[url] + + +def local_proxy(uri: str) -> str: + """ + Get a loopback proxy URI for a `controld://@dns.controld.com` proxy. + + That is the form a client sends to a remote server: the resolver ID alone, never the + API token. The client points the profile at the region; the server only resolves + through it, from its own address. + """ + resolver = urlsplit(uri).username or "" + if not RESOLVER_ID.fullmatch(resolver): + raise ValueError("A controld:// proxy needs a Control D resolver ID") + with STATE: + return forwarder(resolver.lower()).uri + + +def resolver_id(value: str) -> str: + """Get the resolver ID from a resolver ID or its DoH URL.""" + value = value.rstrip("/").rsplit("/", 1)[-1] + if not RESOLVER_ID.fullmatch(value): + raise ValueError(f"Not a Control D resolver ID: {value}") + return value.lower() + + +class ControlD(Proxy): + def __init__( + self, + *, + token: str, + profile: Optional[str] = None, + resolver: Optional[str] = None, + profiles: Optional[list[dict]] = None, + max_profiles: int = 4, + ): + """ + Proxy provider that uses Control D's transparent proxies. + + Control D is a DNS service rather than a proxy: it answers redirected hostnames + with the IP of a proxy in the region you chose, and that proxy relays onwards by + SNI. There is no proxy URI to hand to Python-Requests, so this proxy provider runs a + forwarder on loopback per resolver, and the rest of unshackle sees an ordinary + HTTP proxy. + + A query points a profile's default rule at the region asked for, so everything + that profile resolves leaves through it, and each profile must be one kept for + unshackle alone. A profile holds one region at a time, so each region in use gets + a profile of its own: the account's endpoint named `unshackle-`, else a + configured pair (`profile` and `resolver`, `profiles`, or both), else a new + `unshackle-` endpoint and profile, up to `max_profiles` in all. The + locations come from Control D itself, so nothing needs listing here. + """ + if bool(profile) != bool(resolver): + raise ValueError("Control D needs both a profile and a resolver") + if max_profiles < 1: + raise ValueError("Control D needs max_profiles of 1 or more") + pairs = ([{"profile": profile, "resolver": resolver}] if profile else []) + (profiles or []) + + self.token = token + self.configured = {x["profile"]: resolver_id(x["resolver"]) for x in pairs} + self.max_profiles = max_profiles + self.resolvers = dict(self.configured) + + self.catalogue = self.get_locations() + + def __repr__(self) -> str: + countries = len({x["country"] for x in self.catalogue}) + servers = len(self.catalogue) + + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + def get_locations(self) -> list[dict]: + """Get Control D's proxy locations. Public data, so no token is needed.""" + try: + response = requests.get(f"{API}/proxies", timeout=DOH_TIMEOUT) + response.raise_for_status() + return response.json()["body"]["proxies"] + except Exception as e: + # never fail the command: all providers load up front, even for a run that wants no proxy + log.warning(f"Could not read Control D's locations: {e}") + return [] + + def location(self, query: str) -> Optional[str]: + """Get the identifier of a Control D proxy location for a country or location code.""" + if not self.catalogue: + self.catalogue = self.get_locations() + + # a hidden (residential) location is reachable by exact code only, never by country + codes = [x["PK"] for x in self.catalogue if x["PK"].lower() == query] + if not codes: + codes = [x["PK"] for x in self.catalogue if x["country"].lower() == query and not x.get("hidden")] + + return random.choice(codes) if codes else None + + def api(self, method: str, path: str, **data: str) -> dict: + """Call the Control D API with the account token and return the response body.""" + response = requests.request( + method, + f"{API}{path}", + data=data or None, + headers={"authorization": f"Bearer {self.token}"}, + timeout=DOH_TIMEOUT, + ) + response.raise_for_status() + return response.json()["body"] + + def owned(self) -> dict[str, dict]: + """ + Get the account's `unshackle-` endpoints, by profile. + + An endpoint counts only when no endpoint without that name shares its profile, so a + profile that also serves the user's own devices is never redirected. + """ + devices = self.api("GET", "/devices")["devices"] + shared = {x["profile"]["PK"] for x in devices if x.get("profile") and not x["name"].startswith(PREFIX)} + return { + x["profile"]["PK"]: {"device": x["PK"], "name": x["name"], "resolver": resolver_id(x["resolvers"]["uid"])} + for x in devices + if x.get("profile") and x["name"].startswith(PREFIX) and x["profile"]["PK"] not in shared + } + + def pick(self, query: str) -> str: + """ + Get the profile for a region: one this process already has on it, else the account's + `unshackle-`, else a configured pair this process has not claimed, else a new + `unshackle-`, else one to move, which is renamed if unshackle made it. + """ + for profile, region in REGIONS.items(): + if region == query and profile in self.resolvers: + return profile + + owned = self.owned() + for profile, x in owned.items(): + self.resolvers[profile] = x["resolver"] + if x["name"] == PREFIX + query: + return profile + + for profile in self.configured: + if profile not in REGIONS: + return profile + + if len(self.configured) + len(owned) < self.max_profiles: + return self.create(query) + + # out of profiles: move one this process has not claimed (maybe idle) first, then the oldest claim + candidates = [x for x in [*owned, *self.configured] if x not in REGIONS] + candidates += [x for x in REGIONS if x in self.resolvers] + profile = candidates[0] + log.warning(f"Every Control D profile is in use; moving profile {profile} to {query}") + if profile in owned: + self.api("PUT", f"/profiles/{profile}", name=PREFIX + query) + self.api("PUT", f"/devices/{owned[profile]['device']}", name=PREFIX + query) + return profile + + def create(self, query: str) -> str: + """Create a profile and an endpoint that uses it, for unshackle alone.""" + name = PREFIX + query + profile = self.api("POST", "/profiles", name=name)["profiles"][0]["PK"] + try: + device = self.api("POST", "/devices", name=name, profile_id=profile, icon="desktop-linux") + except Exception: + self.api("DELETE", f"/profiles/{profile}") # an orphan profile is never found again + raise + self.resolvers[profile] = resolver_id(device["resolvers"]["uid"]) + log.info(f"Created profile {name} ({profile}) on the Control D account") + return profile + + def claim(self, query: str) -> Optional[str]: + """Point a profile at this region, unless one already is, and get its resolver ID. Hold STATE.""" + location = self.location(query) + if not location: + return None + + profile = self.pick(query) + resolver = self.resolvers[profile] + if REGIONS.get(profile) != query: + self.api("PUT", f"/profiles/{profile}/default", do="3", via=location, status="1") + log.info(f"Profile {profile} now redirects through {location}") + server = FORWARDERS.get(doh_url(resolver)) + if server: + with server.resolver.lock: + server.resolver.cache.clear() + REGIONS.pop(profile, None) + REGIONS[profile] = query + return resolver + + def get_proxy(self, query: str) -> Optional[str]: + """Get the local HTTP proxy URI that resolves through Control D for this region.""" + with STATE: + resolver = self.claim(query.lower()) + return forwarder(resolver).uri if resolver else None + + def get_remote_proxy(self, query: str) -> Optional[str]: + """Get the `controld://` proxy URI for this region, for a server on another machine.""" + with STATE: + resolver = self.claim(query.lower()) + return f"{SCHEME}{resolver}@dns.controld.com" if resolver else None diff --git a/unshackle/core/proxies/resolve.py b/unshackle/core/proxies/resolve.py index 0d9a5eb0..0e7b3f01 100644 --- a/unshackle/core/proxies/resolve.py +++ b/unshackle/core/proxies/resolve.py @@ -15,6 +15,8 @@ from unshackle.core.utils.redact import mask_proxy log = logging.getLogger("proxies") +REGION = r"[a-z]{2,4}(?:_[a-z0-9]{2,4})?(?:-[a-z0-9]+|:[a-z][a-z0-9]*)*\d*" + def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False) -> List[Any]: """Initialise and return available proxy providers from config. @@ -28,6 +30,7 @@ def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False) from unshackle.core import binaries from unshackle.core.config import config as main_config from unshackle.core.proxies.basic import Basic + from unshackle.core.proxies.controld import ControlD from unshackle.core.proxies.expressvpn import ExpressVPN from unshackle.core.proxies.hola import Hola from unshackle.core.proxies.nordvpn import NordVPN @@ -51,6 +54,9 @@ def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False) proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"])) if hasattr(binaries, "HolaProxy") and binaries.HolaProxy: proxy_providers.append(Hola()) + # last: a bare region reaches it only when nothing else answers, since it changes the account + if proxy_config.get("controld"): + proxy_providers.append(ControlD(**proxy_config["controld"])) if not quiet: for provider in proxy_providers: @@ -74,10 +80,16 @@ def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]: - Direct URI: "https://...", "socks5://..." - Country code: "us", "uk" - Provider:country: "nordvpn:us" + - A Control D resolver sent by a client: "controld://@dns.controld.com" """ if not proxy: return None + if proxy.startswith("controld://"): + from unshackle.core.proxies.controld import local_proxy + + return local_proxy(proxy) + if re.match(r"^(https?://|socks)", proxy): return proxy diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index 979ff662..5b750d2e 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -684,6 +684,33 @@ def resolve_proxy_arg(proxy_arg: Optional[str]) -> Optional[str]: raise click.ClickException(str(e)) +def resolve_remote_proxy_arg(proxy_arg: str) -> Optional[str]: + """ + Get the proxy URI to send to a server for a --proxy value. + + Control D's proxy URI is a forwarder on this machine, so for a server it becomes the + profile's resolver instead (`controld://`), and a bare region query skips Control D. + """ + from unshackle.core.proxies.controld import ControlD + from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy + + try: + providers = initialize_proxy_providers() + provider, _, query = proxy_arg.partition(":") + if provider.lower() == "controld" and query: + controld = next((x for x in providers if isinstance(x, ControlD)), None) + if not controld: + raise ValueError("Proxy provider 'controld' is not configured") + uri = controld.get_remote_proxy(query) + if not uri: + raise ValueError(f"Control D has no location for {query}") + log.info(f"Using a Control D proxy in {query}, resolved by the server") + return uri + return resolve_proxy(proxy_arg, [x for x in providers if not isinstance(x, ControlD)]) + except ValueError as e: + raise click.ClickException(str(e)) + + _CACHE_HEX_ID_RE = re.compile(r"[0-9a-f]{32,}", re.IGNORECASE) @@ -924,7 +951,7 @@ class RemoteService: regions = [str(r).lower() for r in self._server_accounts.get("regions") or []] if regions and client_region and client_region not in regions: try: - create_data["proxy"] = resolve_proxy_arg(regions[0]) + create_data["proxy"] = resolve_remote_proxy_arg(regions[0]) create_data["proxy_region"] = regions[0] self.log.info(f"Using a '{regions[0]}' proxy to match a server account ({', '.join(regions)})") except click.ClickException: @@ -934,7 +961,7 @@ class RemoteService: "rejects the remote session" ) elif not no_proxy and proxy: - resolved_proxy = resolve_proxy_arg(proxy) + resolved_proxy = resolve_remote_proxy_arg(proxy) if resolved_proxy: create_data["proxy"] = resolved_proxy query = (self.ctx.parent.params.get("proxy_query") if self.ctx.parent else None) or proxy