mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-11 19:45:06 +00:00
feat(proxies): add Control D proxy provider
Runs a loopback forwarder per region-named profile (unshackle-<region>), so --proxy and --proxy-download can exit in different countries Thanks to Orni_ for the original code and the test account. Co-authored-by: AntCardFr <24798811+AntCardFR@users.noreply.github.com>
This commit is contained in:
imSp4rky
@@ -141,7 +141,7 @@ Find titles in a service by query string. The service must have a `search()` met
|
||||
| `service` | string | yes | - | Service tag. |
|
||||
| `query` | string | yes | - | Search query. |
|
||||
| `profile` | string | no | `null` | Credential/cookie profile to use. |
|
||||
| `proxy` | string | no | `null` | Full proxy URI, or a country code when the API key has `server_proxy`. |
|
||||
| `proxy` | string | no | `null` | Full proxy URI, a Control D resolver as `controld://<resolver>@dns.controld.com`, or a country code when the API key has `server_proxy`. |
|
||||
| `no_proxy` | boolean | no | `false` | Force-disable all proxy use. |
|
||||
| `credentials`, `cookies`, `cache` | - | no | `null` | Your own login material, on a `--remote-only` server only. See [Client cache on a remote-only server](#client-cache-on-a-remote-only-server). |
|
||||
|
||||
@@ -190,7 +190,7 @@ Get the list of titles behind a title ID, for example episodes or a movie, witho
|
||||
| `service` | string | yes | Service tag. |
|
||||
| `title_id` | string | yes | Title identifier. |
|
||||
| `profile` | string | no | Credential/cookie profile. |
|
||||
| `proxy` | string | no | Full proxy URI, or a country code when the API key has `server_proxy`. |
|
||||
| `proxy` | string | no | Full proxy URI, a Control D resolver as `controld://<resolver>@dns.controld.com`, or a country code when the API key has `server_proxy`. |
|
||||
| `no_proxy` | boolean | no | Force-disable proxy. |
|
||||
| `cdm_type` | string | no | Preferred CDM type. |
|
||||
| `credentials`, `cookies`, `cache` | - | no | Your own login material, on a `--remote-only` server only. See [Client cache on a remote-only server](#client-cache-on-a-remote-only-server). |
|
||||
@@ -445,7 +445,7 @@ Make a download job. It requires `service` and `title_id`. Every other field is
|
||||
| `skip_dl` | boolean | `false` | Only fetch keys, do not download. |
|
||||
| `export` | boolean | `false` | Export manifest/keys/subs to JSON. |
|
||||
| `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. |
|
||||
| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. |
|
||||
| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://<resolver>@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. |
|
||||
| `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. |
|
||||
| `workers` | int | `null` | Threads per track. |
|
||||
| `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. |
|
||||
@@ -891,7 +891,7 @@ Make a remote session for a service and title. Authentication runs asynchronousl
|
||||
|
||||
Service CLI options also travel in a nested `service_params` object, which wins over a flat key with the same name. `profile` at the top level always means the credentials profile, never a service's own `--profile` option.
|
||||
|
||||
The `proxy` value must be a full proxy URI, unless the operator gives your API key `server_proxy` in the `serve.users` config. Without it, the server does not resolve country codes with its own proxy providers. It rejects the request with `INVALID_PROXY` when no `proxy` is set and the reported `client_region` differs from the server's region. A request that reports no `client_region` is not blocked. Pass `proxy` with your own proxy, or `no_proxy` to accept the server's own connection. With `server_proxy`, the server resolves a country code and picks a proxy for your `client_region` itself.
|
||||
The `proxy` value must be a full proxy URI, unless the operator gives your API key `server_proxy` in the `serve.users` config. Without `server_proxy`, the server does not resolve country codes with its own proxy providers. It rejects the request with `INVALID_PROXY` when no `proxy` is set and the reported `client_region` differs from the server's region. A request that reports no `client_region` is not blocked. Pass `proxy` with your own proxy, or `no_proxy` to accept the server's own connection. With `server_proxy`, the server resolves a country code and picks a proxy for your `client_region` itself. A Control D resolver, `controld://<resolver>@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it and needs no Control D configuration of its own.
|
||||
|
||||
=== "Response `200`"
|
||||
|
||||
|
||||
@@ -121,7 +121,12 @@ hood the client walks a remote session through its lifecycle.
|
||||
it sits in a different region from the reported one, asking the client to
|
||||
pass `--proxy` with its own proxy, or `--no-proxy` to accept the server's
|
||||
own connection. With `server_proxy` the server picks a proxy that matches
|
||||
the client region itself
|
||||
the client region itself. For `--proxy controld:ca` the client points its
|
||||
Control D profile at the region and sends only the resolver, as
|
||||
`controld://<resolver>@dns.controld.com`. The server counts it as a full proxy
|
||||
URI and runs its own forwarder for it. A bare region skips Control D, because its
|
||||
proxy is a forwarder on the client. See
|
||||
[Control D](../../guide/proxies-and-vpn.md#control-d)
|
||||
- Track-selection hints (`range_`, `vcodec`, `quality`, `best_available`) so
|
||||
the server fetches the right manifests
|
||||
- Your language and audio codec selection (`-l`, `-vl`, `-al`, `-a`, `-fs`)
|
||||
@@ -145,7 +150,8 @@ hood the client walks a remote session through its lifecycle.
|
||||
`GET /api/services` advertises the regions those accounts cover. When your own
|
||||
region is not one of them and you set no `--proxy`, the client resolves a proxy for
|
||||
the first advertised region itself, so both sides sit in a region the account
|
||||
works in.
|
||||
works in. This is a bare region query, so it skips Control D. A server-account
|
||||
service does not accept a `controld://` resolver from the client.
|
||||
|
||||
The server responds immediately with a session ID and a status. Authentication
|
||||
runs in the background on the server.
|
||||
|
||||
@@ -195,7 +195,7 @@ order the service used, then renumbers the episodes into the order you asked for
|
||||
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--proxy` | Proxy URI, a 2-letter country code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`). |
|
||||
| `--proxy` | Proxy URI, a country or location code resolved from configured providers, or `provider:region` (e.g. `nordvpn:ca`, `gluetun:us`, `protonvpn:de:berlin`, `controld:yul`). |
|
||||
| `--no-proxy` | Force-disable all proxy use. |
|
||||
| `--no-proxy-download` | Bypass the proxy for **all downloads** (manifest, licence, and auth stay proxied). |
|
||||
| `--proxy-download` | Use a different proxy for **all downloads**, in the same forms as `--proxy` (manifest, licence, and auth stay on `--proxy`). |
|
||||
@@ -235,7 +235,7 @@ Each service defines its own query syntax. unshackle prints the results as a tre
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `-p`, `--profile` | Profile for credentials and cookies. |
|
||||
| `--proxy` | Proxy URI, 2-letter country code, or `provider:region`. |
|
||||
| `--proxy` | Proxy URI, country or location code, or `provider:region` (e.g. `controld:ca`). |
|
||||
| `--no-proxy` | Force-disable all proxy use. |
|
||||
|
||||
!!! example
|
||||
|
||||
@@ -722,12 +722,14 @@ unshackle dl --tag MYGRP --repack EXAMPLE 81234567
|
||||
|
||||
## Proxies
|
||||
|
||||
`--proxy` accepts a full proxy URI, a 2-letter country code (resolved through your
|
||||
configured proxy providers), or a `provider:region` form.
|
||||
`--proxy` accepts a full proxy URI, a country or location code (resolved through your
|
||||
configured proxy providers), or a `provider:region` form. See
|
||||
[Proxies & VPN](proxies-and-vpn.md#how-proxy-resolution-works) for the exact grammar.
|
||||
|
||||
```shell title="Proxy forms"
|
||||
unshackle dl --proxy us EXAMPLE 81234567
|
||||
unshackle dl --proxy nordvpn:ca EXAMPLE 81234567
|
||||
unshackle dl --proxy controld:yul EXAMPLE 81234567
|
||||
unshackle dl --proxy 'http://user:pass@host:8080' EXAMPLE 81234567
|
||||
```
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ unshackle pick a working server for you.
|
||||
|
||||
The `--proxy` flag controls proxying, and the proxy providers unshackle supports change
|
||||
a query to a real proxy URL: [basic static proxies](#basic-static-proxies),
|
||||
[Gluetun](#gluetun) (documented in the most detail, since it is the most flexible),
|
||||
[Control D](#control-d), [Gluetun](#gluetun) (documented in the most detail, since it is the most flexible),
|
||||
[ExpressVPN](#expressvpn), [NordVPN](#nordvpn), [Proton VPN](#proton-vpn),
|
||||
[Hola](#hola), [Surfshark](#surfshark), and [Windscribe](#windscribe).
|
||||
|
||||
@@ -106,23 +106,36 @@ flowchart TD
|
||||
asks each loaded proxy provider **in order** and uses the first proxy any of them
|
||||
returns.
|
||||
|
||||
unshackle compares the query against a region grammar: a two-letter country code,
|
||||
optionally followed by a server number and/or a `:city` or `-city` part. It then
|
||||
unshackle compares the query against a region grammar: a country or location code of two
|
||||
to four letters, with an optional `_code` part (`res_yyz`), then an optional server
|
||||
number and/or a `:city` or `-city` part. A `host:port` value never matches. It then
|
||||
lowercases the query and hands it to the proxy provider. Its section below gives the
|
||||
exact forms each proxy provider accepts.
|
||||
|
||||
A Control D resolver, `controld://<resolver>@dns.controld.com`, is a fourth form. A
|
||||
`--remote` client sends it to a server in place of a proxy URI, and the server starts a
|
||||
forwarder for it. See [Control D](#control-d).
|
||||
|
||||
### Provider load order
|
||||
|
||||
When you do a download without `--no-proxy`, unshackle instantiates every configured
|
||||
proxy provider once, in this fixed order. It uses that same order for bare-query
|
||||
resolution:
|
||||
|
||||
**Basic → ExpressVPN → NordVPN → Proton VPN → Surfshark → Windscribe → Gluetun → Hola**
|
||||
**Basic → ExpressVPN → NordVPN → Proton VPN → Surfshark → Windscribe → Gluetun → Hola → Control D**
|
||||
|
||||
So if you have both Basic and NordVPN configured and use `--proxy us`, a `us` entry in
|
||||
your Basic config wins, because unshackle tries Basic first. To skip ahead to a specific
|
||||
proxy provider, prefix the query (`--proxy nordvpn:us`).
|
||||
|
||||
!!! note "A bare region reaches Control D last"
|
||||
unshackle asks Control D for a bare region such as `--proxy us` only when no other
|
||||
loaded proxy provider has a proxy for it. Hola loads automatically when the
|
||||
`hola-proxy` binary is on your `PATH`, and it comes before Control D. When Control D answers, it changes your Control D
|
||||
account: it points a profile at that country, and it can create an `unshackle-us`
|
||||
profile and endpoint. The geofence auto-proxy uses the same order. To use Control D
|
||||
directly, prefix the query (`--proxy controld:us`).
|
||||
|
||||
!!! note "Auto-loading providers"
|
||||
Most providers only load when you configure them under `proxy_providers:`. Three are
|
||||
special: **ExpressVPN** and **Proton VPN** also load automatically once their cache
|
||||
@@ -190,6 +203,167 @@ proxy_providers:
|
||||
unshackle dl --proxy de2 EXAMPLE 81234567
|
||||
```
|
||||
|
||||
## Control D
|
||||
|
||||
[Control D](https://controld.com) is a DNS service, not a proxy. It answers the hostnames
|
||||
it redirects with the IP of a transparent proxy in the region you chose, and that proxy
|
||||
relays onwards by SNI. There is no proxy URI to hand to Requests, so this proxy provider runs a
|
||||
small **forwarder on loopback**: it resolves each connection through your Control D
|
||||
resolver and relays the bytes. Everything downstream (the service session, the downloads,
|
||||
`--proxy-download`) sees an ordinary HTTP proxy.
|
||||
|
||||
unshackle picks the region itself, by pointing a profile's **default rule** at it. A profile
|
||||
holds one region at a time, so unshackle gives each region in use a profile of its own,
|
||||
each with its own endpoint and its own forwarder port. `--proxy controld:yyz
|
||||
--proxy-download controld:yul` therefore uses two profiles, when two are free: Toronto for
|
||||
the service and Montreal for the downloads. When no second profile is free, the second
|
||||
region moves the first profile (see [step 5](#how-unshackle-chooses-a-profile)).
|
||||
|
||||
Keep both locations in one country. Some CDNs also check the country of the download IP,
|
||||
and refuse a `--proxy-download` exit in another country with a 403. This applies to every
|
||||
proxy provider.
|
||||
|
||||
`unshackle search --proxy controld:ca` works the same way as `dl`.
|
||||
|
||||
### What you need
|
||||
|
||||
- **A plan with traffic redirection.** On the personal plans that is *Full Control*;
|
||||
*Some Control* has no redirection, and every redirect fails. The 14-day trial includes
|
||||
it. See [Control D's plans](https://controld.com/plans) for current prices.
|
||||
- **An API token of type *Write*.** A *Read* token cannot change a profile's default rule.
|
||||
Create it in the dashboard under *API*. If you set *Allowed IPs* on the token, include
|
||||
the address of each machine that runs unshackle with it.
|
||||
|
||||
```yaml title="unshackle.yaml"
|
||||
proxy_providers:
|
||||
controld:
|
||||
token: api.xxxxxxxxxxxx # a Write API token, from the Control D dashboard
|
||||
```
|
||||
|
||||
```shell
|
||||
unshackle dl --proxy controld:ca EXAMPLE 81234567
|
||||
```
|
||||
|
||||
The token is the only required key. You do not need to create a profile or an endpoint,
|
||||
or to give a `profile` and `resolver`: unshackle creates an `unshackle-<region>` profile
|
||||
and endpoint on your account the first time it needs one, and uses it again on the next
|
||||
run. Give `profile` and `resolver` only to use a profile that you made yourself (see
|
||||
[How unshackle chooses a profile](#how-unshackle-chooses-a-profile)).
|
||||
|
||||
The returned proxy is HTTP on loopback,
|
||||
`http://127.0.0.1:{port}`. The locations themselves are read from Control D at startup
|
||||
(107 of them across 68 countries, at the time of writing), so there is nothing to list:
|
||||
|
||||
| Query | Meaning |
|
||||
|---|---|
|
||||
| `ca` | One of Control D's Canadian locations, at random. |
|
||||
| `yul` | That exact location, by its code. |
|
||||
| `res_yyz` | A residential location, by its code. A country pick never uses a residential location, so name it this way. |
|
||||
|
||||
The [configuration reference](../reference/configuration/network.md#controld) lists
|
||||
every `controld` config key.
|
||||
|
||||
### How unshackle chooses a profile
|
||||
|
||||
Each region query gets a profile and an endpoint named after it: `--proxy controld:ca` uses
|
||||
`unshackle-ca`. For each region, unshackle uses the first match in this list:
|
||||
|
||||
1. A profile that this process already pointed at that region.
|
||||
2. The endpoint named `unshackle-<region>` on your account, made by an earlier run.
|
||||
3. A profile from your config that this process does not use yet. These are optional:
|
||||
|
||||
```yaml title="unshackle.yaml"
|
||||
proxy_providers:
|
||||
controld:
|
||||
token: api.xxxxxxxxxxxx
|
||||
profile: abcd123 # a profile kept for unshackle, and nothing else
|
||||
resolver: efgh456 # the resolver ID of an endpoint using that profile
|
||||
profiles: # more pairs, in the same shape
|
||||
- profile: ijkl789
|
||||
resolver: mnop012
|
||||
max_profiles: 4 # the most profiles unshackle uses, yours included (default 4)
|
||||
```
|
||||
|
||||
The resolver ID is the last part of the endpoint's DoH URL,
|
||||
`https://dns.controld.com/<resolver>`. You can also give the full URL.
|
||||
|
||||
4. A new profile and endpoint named `unshackle-<region>`, while your config pairs plus the
|
||||
`unshackle-*` endpoints on the account are fewer than `max_profiles`. unshackle keeps
|
||||
them for the next run. `max_profiles` limits only this step: unshackle uses every pair
|
||||
in your config, even more than `max_profiles`.
|
||||
5. When no profile is free, a profile moves to the new region, with everything that still
|
||||
uses it. unshackle first moves a profile that this process has not pointed at a region,
|
||||
which another process can still use. Next, it moves the profile that this process
|
||||
pointed at a region first. An `unshackle-*` profile that moves, and its endpoint, get the
|
||||
new region's name. unshackle logs `Every Control D profile is in use; moving profile … to …`,
|
||||
and connections already open keep the old exit until they close.
|
||||
|
||||
Because the names live on the account, every unshackle process agrees on them: two runs,
|
||||
or two jobs on a `serve` instance, that ask for `ca` both use `unshackle-ca`. Two processes
|
||||
can still move a profile under each other when they run out of profiles (step 5).
|
||||
|
||||
unshackle changes only the profiles in your config and the profiles of `unshackle-*`
|
||||
endpoints. It skips an `unshackle-*` endpoint whose profile an endpoint without that name
|
||||
also uses, so unshackle never redirects a profile that serves your own devices. Leave the `unshackle-*`
|
||||
profiles empty: no services, no custom rules. unshackle sets their default rule to
|
||||
`REDIRECT`, and the *Services* and *Custom Rules* tabs stay at zero by design. To remove
|
||||
them, delete the endpoints and profiles in the dashboard. unshackle creates new ones when
|
||||
it next needs them.
|
||||
|
||||
!!! warning "Business accounts pay per endpoint"
|
||||
Personal plans allow an unlimited number of profiles and endpoints. Business plans bill
|
||||
each endpoint every month, so each `unshackle-<country>` endpoint adds to the bill. On a business
|
||||
account, configure your own pairs, as many as the countries you use at the same time,
|
||||
and set `max_profiles` to that number. unshackle then never creates a profile.
|
||||
|
||||
!!! note "Everything goes through the proxy, including the segments"
|
||||
With the default rule redirecting, the whole download leaves through Control D's
|
||||
transparent proxy. Control D serves it on a best-effort basis. Pair it with `--no-proxy-download` to
|
||||
keep the geo-check on the manifest and the licence while pulling the files over your
|
||||
own connection.
|
||||
|
||||
!!! note "The switch is not instant"
|
||||
unshackle drops its own DNS cache when it changes region, but Control D's edge may
|
||||
still serve the previous answer for the remainder of the record's TTL.
|
||||
|
||||
!!! note "Same machine, same IP, over IPv4"
|
||||
Control D authorises its proxies by source IP: the address that queried DNS is the one
|
||||
allowed to use the proxy. The forwarder resolves and connects from the same host, so
|
||||
this holds. Control D also answers a redirected name only to a query over IPv4, so the
|
||||
forwarder asks over IPv4 even on a host with IPv6, and ignores `HTTPS_PROXY` for these
|
||||
queries.
|
||||
|
||||
The forwarder refuses a connection to an IP address with a `502`, because an IP address
|
||||
skips Control D's DNS and connects from your own address. A request that names a host
|
||||
by IP address fails with this proxy provider.
|
||||
|
||||
!!! note "With `--remote`, the server resolves"
|
||||
unshackle does not send your API token to a server. For `--proxy controld:ca` with
|
||||
`--remote`, unshackle points `unshackle-ca` at Canada with your token, then sends the
|
||||
server only that endpoint's resolver ID, as `controld://<resolver>@dns.controld.com`.
|
||||
The server runs its own forwarder with it, from its own address, and needs no Control D
|
||||
configuration. The resolver ID lets anyone who has it use that endpoint's proxy, so treat
|
||||
it as a secret: unshackle masks it in logs and in stored job parameters.
|
||||
|
||||
The server must run a version of unshackle that knows `controld://`. An older server
|
||||
rejects it with `INVALID_PROXY`: `Proxy provider 'controld' not found` when your API key
|
||||
has `server_proxy`, else a message that asks for a full proxy URI. A bare region, such
|
||||
as `--proxy ca`, skips Control D with `--remote`, because the proxy it gives is on your
|
||||
machine. A service whose accounts the server manages does not accept a proxy from the
|
||||
client, and that includes a Control D resolver.
|
||||
|
||||
### Troubleshooting Control D
|
||||
|
||||
| Symptom | Cause and fix |
|
||||
|---|---|
|
||||
| `Proxy check failed`, because the exit IP is your own IP. | The profile does not redirect. Make sure that the plan includes traffic redirection (*Full Control*, not *Some Control*). After a region change, wait for the TTL of the previous answer to end. |
|
||||
| A `502 Bad Gateway` for one host. With `-d`, the log shows `Control D returned no A record for …`. | Control D gave no IPv4 address for that host. |
|
||||
| A `502 Bad Gateway` for one host only. With `-d`, the log shows `Refused …, an address that Control D cannot redirect`. | The request names the host by IP address. The forwarder refuses it. See the note on IPv4 above. |
|
||||
| An HTTP `401` or `403` from `api.controld.com`. | The token is a *Read* token, or its *Allowed IPs* do not include this machine. Use a *Write* token. |
|
||||
| `Control D needs both a profile and a resolver` or `Control D needs max_profiles of 1 or more` at startup. | The `controld` config is not valid. Give `profile` and `resolver` together, and a `max_profiles` of 1 or more. Every run fails until you correct it, also a run that does not use Control D. |
|
||||
| `Could not read Control D's locations`. | unshackle could not get the location list. It tries again at the next Control D query. |
|
||||
| A `403` for the segments only, with `--proxy-download` in another country. | The CDN checks the country of the download IP. Keep `--proxy-download` in the same country as `--proxy`. |
|
||||
|
||||
## Gluetun
|
||||
|
||||
Gluetun is unshackle's most flexible proxy backend, and the one to reach for if you want
|
||||
@@ -651,16 +825,22 @@ The returned proxy is HTTPS on **port 443**.
|
||||
unshackle has **two** proxy-resolution code paths that are intentionally not
|
||||
identical:
|
||||
|
||||
- `unshackle/commands/dl.py` inlines its own logic and loads **all eight** providers,
|
||||
- `unshackle/commands/dl.py` inlines its own logic and loads **all nine** providers,
|
||||
including Windscribe and Gluetun.
|
||||
- `unshackle/core/proxies/resolve.py::initialize_proxy_providers()` is the shared
|
||||
resolver used by the REST API handlers and the remote-service client. It **omits**
|
||||
Windscribe and Gluetun.
|
||||
|
||||
Both paths accept the same query grammar (a direct URI, a `provider:query`, or a bare
|
||||
country query tried against providers in order), but the set of available providers
|
||||
differs. Keep this in mind when you add a proxy provider: an entry in `dl.py` does not
|
||||
make it available over the API, and the opposite is also true.
|
||||
Both paths accept the same query grammar (a direct URI, a `provider:query`, a bare
|
||||
country query tried against providers in order, or a `controld://` resolver), but the
|
||||
set of available providers differs. Keep this in mind when you add a proxy provider:
|
||||
an entry in `dl.py` does not make it available over the API, and the opposite is also
|
||||
true.
|
||||
|
||||
The remote-service client resolves `--proxy` with its own
|
||||
`resolve_remote_proxy_arg()`. It changes a `controld:` query to a `controld://`
|
||||
resolver for the server with `get_remote_proxy()`, and it removes Control D from a bare
|
||||
query, because a loopback forwarder is of no use to a server on another machine.
|
||||
|
||||
!!! note "Implementing a new provider (developers)"
|
||||
Every proxy provider subclasses `unshackle.core.proxies.proxy.Proxy` and has three
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ at a title and it will:
|
||||
| Audio | AAC, AC-3, E-AC-3, AC-4, Opus, Vorbis, DTS, DTS-X, ALAC, FLAC; Dolby Atmos handling; per-channel-layout selection |
|
||||
| Subtitles | SRT, WebVTT, ASS/SSA, TTML and more, with optional SDH stripping and format conversion |
|
||||
| Key vaults | SQLite, MySQL, and HTTP/API vaults to store and share content keys |
|
||||
| Proxies | Basic proxies plus NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers |
|
||||
| Proxies | Basic proxies plus Control D, NordVPN, ProtonVPN, Surfshark, Windscribe, ExpressVPN, Gluetun, and Hola providers |
|
||||
| Services | A plugin API for the service modules you write. unshackle includes none |
|
||||
| Automation | A REST API with a job queue, live progress, history, and remote-download sessions |
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ for the full proxy provider guide. Recognised proxy providers and their exit por
|
||||
| Provider | Config key | Credentials | Proxy scheme/port |
|
||||
|----------|-----------|-------------|-------------------|
|
||||
| Basic (static) | `basic` | country → URI(s) | as specified |
|
||||
| Control D | `controld` | Write API token (profile/resolver pairs optional) | `http://127.0.0.1:{port}` (local forwarder) |
|
||||
| NordVPN | `nordvpn` | **service** credentials (48 chars combined) | `https://...:89` |
|
||||
| Surfshark | `surfsharkvpn` | **service** credentials (48 chars combined) | `https://...:443` |
|
||||
| Windscribe | `windscribevpn` | service credentials | `https://...:443` |
|
||||
@@ -87,6 +88,19 @@ proxy_providers:
|
||||
password: <service password>
|
||||
```
|
||||
|
||||
### `controld`
|
||||
|
||||
Config keys of the Control D proxy provider. See
|
||||
[Control D](../../guide/proxies-and-vpn.md#control-d) for how unshackle uses them.
|
||||
|
||||
| Key | Type | Default | Meaning |
|
||||
|---|---|---|---|
|
||||
| `token` | `str` | required | A Control D API token of type *Write*. |
|
||||
| `profile` | `str` | none | Optional. The ID of a profile you made for unshackle. Give it together with `resolver`. Without it, unshackle creates `unshackle-<region>` profiles itself. |
|
||||
| `resolver` | `str` | none | Optional. The resolver ID of an endpoint that uses `profile`, or its DoH URL `https://dns.controld.com/<resolver>`. |
|
||||
| `profiles` | `list` | `[]` | Optional. More pairs, each a mapping with `profile` and `resolver`. |
|
||||
| `max_profiles` | `int` | `4` | The most profiles unshackle uses: your pairs plus the `unshackle-*` endpoints on the account. It must be 1 or more. unshackle creates an `unshackle-<region>` profile and endpoint only while the total is lower. |
|
||||
|
||||
!!! note "Provider loading differs between CLI and REST server"
|
||||
The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API /
|
||||
remote-client path uses a separate resolver that does **not** load `windscribevpn` or
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
"""Unit tests for the Control D proxy provider: the DoH answer parser, the loopback forwarder
|
||||
that turns a resolver into something Python-Requests can use as a proxy, the pool of
|
||||
region-named profiles, and the resolver form a client sends to a remote server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import http.server
|
||||
import re
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import Iterator
|
||||
|
||||
import click
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from unshackle.core.proxies import controld
|
||||
from unshackle.core.proxies.controld import (
|
||||
MIN_TTL,
|
||||
ControlD,
|
||||
IPv4Adapter,
|
||||
LocalProxy,
|
||||
Resolver,
|
||||
build_query,
|
||||
local_proxy,
|
||||
parse_a_records,
|
||||
)
|
||||
from unshackle.core.proxies.resolve import REGION, resolve_proxy
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def dns_response(ip: bytes, ttl: int = 300) -> bytes:
|
||||
"""A one-question, one-answer A-record response whose answer name is a compression pointer."""
|
||||
header = b"\x00\x00\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00"
|
||||
question = b"\x01a\x04test\x00" + b"\x00\x01\x00\x01"
|
||||
answer = b"\xc0\x0c" + b"\x00\x01\x00\x01" + ttl.to_bytes(4, "big") + b"\x00\x04" + ip
|
||||
return header + question + answer
|
||||
|
||||
|
||||
def test_query_is_a_well_formed_a_record_question() -> None:
|
||||
query = build_query("a.test.")
|
||||
assert query[4:6] == b"\x00\x01"
|
||||
assert query[12:] == b"\x01a\x04test\x00\x00\x01\x00\x01"
|
||||
|
||||
|
||||
def test_parses_the_answer_past_a_compression_pointer() -> None:
|
||||
assert parse_a_records(dns_response(bytes([1, 2, 3, 4]))) == [("1.2.3.4", 300)]
|
||||
assert parse_a_records(b"") == []
|
||||
|
||||
|
||||
def origin(body: bytes) -> http.server.HTTPServer:
|
||||
"""A local HTTP origin that answers every GET with `body`, and keeps connections alive."""
|
||||
|
||||
class Origin(http.server.BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *_args: object) -> None:
|
||||
pass
|
||||
|
||||
server = http.server.HTTPServer(("127.0.0.1", 0), Origin)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
return server
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def forwarder() -> Iterator[LocalProxy]:
|
||||
"""A forwarder whose resolver sends every hostname to 127.0.0.1, as Control D sends it to its proxy."""
|
||||
proxy = LocalProxy(SimpleNamespace(resolve=lambda host: "127.0.0.1"))
|
||||
threading.Thread(target=proxy.serve_forever, daemon=True).start()
|
||||
yield proxy
|
||||
proxy.shutdown()
|
||||
proxy.server_close()
|
||||
|
||||
|
||||
def test_forwards_a_request_to_the_resolved_host(forwarder: LocalProxy) -> None:
|
||||
pong = origin(b"pong")
|
||||
try:
|
||||
response = requests.get(
|
||||
f"http://unresolvable.test:{pong.server_address[1]}/", proxies={"http": forwarder.uri}, timeout=10
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.content == b"pong"
|
||||
finally:
|
||||
pong.shutdown()
|
||||
|
||||
|
||||
def test_a_reused_connection_never_carries_a_request_to_the_wrong_host(forwarder: LocalProxy) -> None:
|
||||
"""A client pools one proxy connection for every plain-HTTP host, and the relay is bound to the first."""
|
||||
first, second = origin(b"A"), origin(b"B")
|
||||
try:
|
||||
with requests.Session() as session:
|
||||
session.proxies = {"http": forwarder.uri}
|
||||
assert session.get(f"http://a.test:{first.server_address[1]}/", timeout=10).content == b"A"
|
||||
assert session.get(f"http://b.test:{second.server_address[1]}/", timeout=10).content == b"B"
|
||||
finally:
|
||||
first.shutdown()
|
||||
second.shutdown()
|
||||
|
||||
|
||||
def test_connect_tunnels_bytes_both_ways(forwarder: LocalProxy) -> None:
|
||||
"""Every real download is HTTPS, so it all goes through CONNECT rather than absolute-form."""
|
||||
echo = socketserver.TCPServer(
|
||||
("127.0.0.1", 0),
|
||||
type(
|
||||
"Echo",
|
||||
(socketserver.BaseRequestHandler,),
|
||||
{"handle": lambda self: self.request.sendall(self.request.recv(64)[::-1])},
|
||||
),
|
||||
)
|
||||
threading.Thread(target=echo.serve_forever, daemon=True).start()
|
||||
|
||||
try:
|
||||
with socket.create_connection(forwarder.server_address[:2], timeout=10) as client:
|
||||
client.sendall(f"CONNECT unresolvable.test:{echo.server_address[1]} HTTP/1.1\r\n\r\n".encode())
|
||||
assert b"200 Connection established" in client.recv(128)
|
||||
client.sendall(b"stressed")
|
||||
assert client.recv(64) == b"desserts"
|
||||
finally:
|
||||
echo.shutdown()
|
||||
echo.server_close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("target", "status"),
|
||||
[
|
||||
("1.2.3.4:443", b"502"), # an IP literal would bypass Control D
|
||||
("[::1]:443", b"502"),
|
||||
("a.test:x", b"400"),
|
||||
],
|
||||
)
|
||||
def test_connect_refuses_what_it_cannot_redirect(forwarder: LocalProxy, target: str, status: bytes) -> None:
|
||||
with socket.create_connection(forwarder.server_address[:2], timeout=10) as client:
|
||||
client.sendall(f"CONNECT {target} HTTP/1.1\r\n\r\n".encode())
|
||||
assert client.recv(128).startswith(b"HTTP/1.1 " + status)
|
||||
|
||||
|
||||
def test_an_unreachable_host_answers_502() -> None:
|
||||
def refuse(host: str) -> str:
|
||||
raise ConnectionError("no A record")
|
||||
|
||||
proxy = LocalProxy(SimpleNamespace(resolve=refuse))
|
||||
threading.Thread(target=proxy.serve_forever, daemon=True).start()
|
||||
try:
|
||||
response = requests.get("http://unresolvable.test/", proxies={"http": proxy.uri}, timeout=10)
|
||||
assert response.status_code == 502
|
||||
finally:
|
||||
proxy.shutdown()
|
||||
proxy.server_close()
|
||||
|
||||
|
||||
def stub_session(response: bytes, seen: list) -> object:
|
||||
class Session:
|
||||
@staticmethod
|
||||
def get(url: str, params: dict, headers: dict, timeout: int) -> object:
|
||||
seen.append((url, params, headers))
|
||||
|
||||
class Answer:
|
||||
content = response
|
||||
|
||||
@staticmethod
|
||||
def raise_for_status() -> None:
|
||||
pass
|
||||
|
||||
return Answer()
|
||||
|
||||
return Session()
|
||||
|
||||
|
||||
def test_resolve_asks_control_d_in_wireformat_and_caches_it() -> None:
|
||||
seen: list = []
|
||||
resolver = Resolver("https://dns.controld.com/efgh456")
|
||||
resolver.session = stub_session(dns_response(bytes([9, 8, 7, 6]), ttl=1), seen)
|
||||
|
||||
assert resolver.resolve("a.test") == "9.8.7.6"
|
||||
url, params, headers = seen[0]
|
||||
assert url == "https://dns.controld.com/efgh456"
|
||||
assert headers == {"accept": "application/dns-message"}
|
||||
assert params["dns"] == base64.urlsafe_b64encode(build_query("a.test")).rstrip(b"=").decode()
|
||||
assert "=" not in params["dns"] # RFC 8484 wants the padding stripped
|
||||
|
||||
assert resolver.resolve("a.test") == "9.8.7.6"
|
||||
assert len(seen) == 1
|
||||
assert resolver.cache["a.test"][1] > time.monotonic() + MIN_TTL - 1 # a 1s TTL is floored
|
||||
|
||||
|
||||
def test_resolve_raises_when_control_d_has_no_address() -> None:
|
||||
resolver = Resolver("https://dns.controld.com/efgh456")
|
||||
resolver.session = stub_session(b"\x00\x00\x81\x83" + b"\x00\x00" * 4, [])
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
resolver.resolve("nowhere.test")
|
||||
|
||||
|
||||
def test_resolve_asks_over_ipv4_and_ignores_environment_proxies() -> None:
|
||||
"""Control D answers a redirected name only over IPv4, and authorises the address that asked."""
|
||||
resolver = Resolver("https://dns.controld.com/efgh456")
|
||||
assert resolver.session.trust_env is False
|
||||
assert isinstance(resolver.session.get_adapter("https://dns.controld.com/"), IPv4Adapter)
|
||||
|
||||
|
||||
CATALOGUE = [
|
||||
{"PK": "YUL", "country": "CA", "city": "Montreal"},
|
||||
{"PK": "YYZ", "country": "CA", "city": "Toronto"},
|
||||
{"PK": "RES_YYZ", "country": "CA", "city": "Res Toronto", "hidden": True},
|
||||
{"PK": "JFK", "country": "US", "city": "New York"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def catalogue(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Serve Control D's location list without going near the network."""
|
||||
response = SimpleNamespace(raise_for_status=lambda: None, json=lambda: {"body": {"proxies": CATALOGUE}})
|
||||
monkeypatch.setattr("unshackle.core.proxies.controld.requests.get", lambda *a, **kw: response)
|
||||
|
||||
|
||||
class Account:
|
||||
"""A fake Control D account API: holds endpoints, records every call, and answers them."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.devices: list[dict] = []
|
||||
self.calls: list[tuple[str, str, dict]] = []
|
||||
self.fail_device = False
|
||||
|
||||
def device(self, name: str, profile: str) -> None:
|
||||
self.devices.append(
|
||||
{"PK": f"d-{name}", "name": name, "profile": {"PK": profile}, "resolvers": {"uid": f"r{profile}"}}
|
||||
)
|
||||
|
||||
def request(self, method: str, url: str, data: dict, headers: dict, timeout: int) -> SimpleNamespace:
|
||||
assert headers == {"authorization": "Bearer secret"}
|
||||
path = url.removeprefix("https://api.controld.com")
|
||||
self.calls.append((method, path, data))
|
||||
body: dict = {}
|
||||
if (method, path) == ("GET", "/devices"):
|
||||
body = {"devices": [dict(x) for x in self.devices]}
|
||||
elif (method, path) == ("POST", "/profiles"):
|
||||
body = {"profiles": [{"PK": f"p{data['name'].replace('-', '')}"}]}
|
||||
elif (method, path) == ("POST", "/devices"):
|
||||
if self.fail_device:
|
||||
raise requests.HTTPError("endpoint limit")
|
||||
self.device(data["name"], data["profile_id"])
|
||||
body = self.devices[-1]
|
||||
elif method == "PUT" and path.startswith("/devices/"):
|
||||
next(x for x in self.devices if x["PK"] == path.split("/")[2])["name"] = data["name"]
|
||||
return SimpleNamespace(raise_for_status=lambda: None, json=lambda: {"body": body})
|
||||
|
||||
def moves(self) -> list[tuple[str, str]]:
|
||||
"""The (profile, location) of every default-rule change, in order."""
|
||||
return [(path.split("/")[2], data["via"]) for method, path, data in self.calls if path.endswith("/default")]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def account(catalogue: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[Account]:
|
||||
"""A fake account, on a fresh process state whose forwarders are shut down when the test ends."""
|
||||
made = Account()
|
||||
monkeypatch.setattr("unshackle.core.proxies.controld.requests.request", made.request)
|
||||
controld.FORWARDERS.clear()
|
||||
controld.REGIONS.clear()
|
||||
yield made
|
||||
for server in controld.FORWARDERS.values():
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
controld.FORWARDERS.clear()
|
||||
controld.REGIONS.clear()
|
||||
|
||||
|
||||
def test_locations_come_from_control_d(account: Account) -> None:
|
||||
provider = ControlD(token="secret")
|
||||
assert provider.location("ca") in ("YUL", "YYZ") # never the hidden residential one
|
||||
assert provider.location("res_yyz") == "RES_YYZ" # unless asked for by name
|
||||
assert provider.location("jp") is None
|
||||
assert repr(provider) == "2 Countries (4 Servers)"
|
||||
|
||||
|
||||
def test_a_query_steers_the_profile_default_rule(account: Account) -> None:
|
||||
provider = ControlD(token="secret", profile="abcd123", resolver="efgh456", max_profiles=1)
|
||||
|
||||
assert provider.get_proxy("jp") is None
|
||||
assert not account.moves()
|
||||
|
||||
uri = provider.get_proxy("jfk")
|
||||
assert uri and uri.startswith("http://127.0.0.1:")
|
||||
assert ("PUT", "/profiles/abcd123/default", {"do": "3", "via": "JFK", "status": "1"}) in account.calls
|
||||
|
||||
assert provider.get_proxy("JFK") == uri
|
||||
assert len(account.moves()) == 1
|
||||
|
||||
# one profile only: a second country moves it and must clear its cache
|
||||
cache = controld.FORWARDERS["https://dns.controld.com/efgh456"].resolver.cache
|
||||
cache["cached.test"] = ("1.2.3.4", float("inf"))
|
||||
assert provider.get_proxy("yul") == uri
|
||||
assert account.moves()[-1] == ("abcd123", "YUL")
|
||||
assert cache == {}
|
||||
|
||||
|
||||
def test_each_region_gets_its_own_profile_and_forwarder(account: Account) -> None:
|
||||
"""--proxy and --proxy-download in two countries must not move each other."""
|
||||
provider = ControlD(
|
||||
token="secret", profile="one", resolver="r1", profiles=[{"profile": "two", "resolver": "r2"}], max_profiles=2
|
||||
)
|
||||
us = provider.get_proxy("jfk")
|
||||
ca = provider.get_proxy("yul")
|
||||
assert us != ca
|
||||
assert account.moves() == [("one", "JFK"), ("two", "YUL")]
|
||||
|
||||
# out of profiles: the oldest claim moves
|
||||
assert provider.get_proxy("yyz") == us
|
||||
assert account.moves()[-1] == ("one", "YYZ")
|
||||
assert provider.get_proxy("yul") == ca
|
||||
assert len(account.moves()) == 3
|
||||
|
||||
|
||||
def test_a_region_uses_the_endpoint_named_after_it_or_creates_one(account: Account) -> None:
|
||||
account.device("unshackle-us", "kept")
|
||||
account.device("Phone", "theirs")
|
||||
account.device("unshackle-gb", "theirs") # shares the user's own profile, so never touched
|
||||
provider = ControlD(token="secret")
|
||||
|
||||
us = provider.get_proxy("us")
|
||||
ca = provider.get_proxy("ca")
|
||||
assert us != ca
|
||||
assert ("POST", "/profiles", {"name": "unshackle-ca"}) in account.calls
|
||||
assert [x[0] for x in account.moves()] == ["kept", "punshackleca"]
|
||||
assert "theirs" not in provider.resolvers
|
||||
|
||||
# the REST API makes a provider per request; it must find the endpoint by name
|
||||
assert ControlD(token="secret").get_proxy("ca") == ca
|
||||
assert len(account.moves()) == 2
|
||||
|
||||
|
||||
def test_a_moved_profile_is_renamed_to_its_new_region(account: Account) -> None:
|
||||
account.device("unshackle-us", "kept")
|
||||
provider = ControlD(token="secret", max_profiles=1)
|
||||
|
||||
provider.get_proxy("us")
|
||||
provider.get_proxy("ca")
|
||||
assert ("PUT", "/profiles/kept", {"name": "unshackle-ca"}) in account.calls
|
||||
assert account.devices[0]["name"] == "unshackle-ca"
|
||||
assert account.moves()[-1][0] == "kept"
|
||||
|
||||
|
||||
def test_a_failed_endpoint_leaves_no_profile_behind(account: Account) -> None:
|
||||
account.fail_device = True
|
||||
with pytest.raises(requests.HTTPError):
|
||||
ControlD(token="secret").get_proxy("us")
|
||||
assert ("DELETE", "/profiles/punshackleus", None) in [(m, p, d or None) for m, p, d in account.calls]
|
||||
|
||||
|
||||
def test_a_bad_config_is_refused(catalogue: None) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
ControlD(token="secret", profile="one")
|
||||
with pytest.raises(ValueError):
|
||||
ControlD(token="secret", max_profiles=0)
|
||||
with pytest.raises(ValueError):
|
||||
ControlD(token="secret", profile="one", resolver="bad id?")
|
||||
|
||||
|
||||
def test_a_server_runs_a_forwarder_for_a_resolver_a_client_sends(account: Account) -> None:
|
||||
provider = ControlD(token="secret", profile="abcd123", resolver="efgh456")
|
||||
sent = provider.get_remote_proxy("jfk")
|
||||
assert sent == "controld://efgh456@dns.controld.com"
|
||||
|
||||
uri = resolve_proxy(sent, [])
|
||||
assert uri and uri.startswith("http://127.0.0.1:")
|
||||
assert local_proxy(sent) == uri # one forwarder per resolver
|
||||
with pytest.raises(ValueError):
|
||||
local_proxy("controld://evil.example/x@dns.controld.com")
|
||||
|
||||
|
||||
def test_remote_sends_the_resolver_and_a_bare_region_skips_control_d(
|
||||
account: Account, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from unshackle.core import remote_service
|
||||
|
||||
provider = ControlD(token="secret", profile="abcd123", resolver="efgh456")
|
||||
monkeypatch.setattr("unshackle.core.proxies.resolve.initialize_proxy_providers", lambda: [provider])
|
||||
|
||||
assert remote_service.resolve_remote_proxy_arg("controld:us") == "controld://efgh456@dns.controld.com"
|
||||
with pytest.raises(click.ClickException):
|
||||
remote_service.resolve_remote_proxy_arg("us") # a loopback forwarder is useless to a server
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query", ["ca", "us:seattle", "us-ny", "uk1", "yul", "jfk", "res_yyz", "nordvpn:ca"])
|
||||
def test_a_region_query_matches(query: str) -> None:
|
||||
assert re.fullmatch(rf"(?:[a-z]+:)?{REGION}", query, re.IGNORECASE)
|
||||
assert re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", f"gluetun:{query}", re.IGNORECASE)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uri", ["nas:3128", "localhost:8080", "proxy.example.com:8080", "1.2.3.4:8080", "us:1080"])
|
||||
def test_a_host_and_port_is_never_a_region_query(uri: str) -> None:
|
||||
assert not re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", uri, re.IGNORECASE)
|
||||
@@ -52,8 +52,18 @@ from unshackle.core.drm import DRM_T, ClearKeyCENC, MonaLisa, PlayReady, Widevin
|
||||
from unshackle.core.events import events
|
||||
from unshackle.core.providers.anilist import parse_anilist_ref
|
||||
from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int
|
||||
from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN
|
||||
from unshackle.core.proxies.resolve import is_loopback, resolve_proxy
|
||||
from unshackle.core.proxies import (
|
||||
Basic,
|
||||
ControlD,
|
||||
ExpressVPN,
|
||||
Gluetun,
|
||||
Hola,
|
||||
NordVPN,
|
||||
ProtonVPN,
|
||||
SurfsharkVPN,
|
||||
WindscribeVPN,
|
||||
)
|
||||
from unshackle.core.proxies.resolve import REGION, is_loopback, resolve_proxy
|
||||
from unshackle.core.service import Service, grow_session_pool
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.temp import with_task_temp
|
||||
@@ -1371,9 +1381,13 @@ class dl:
|
||||
self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"]))
|
||||
if binaries.HolaProxy:
|
||||
self.proxy_providers.append(Hola())
|
||||
if config.proxy_providers.get("controld"):
|
||||
self.proxy_providers.append(ControlD(**config.proxy_providers["controld"]))
|
||||
for proxy_provider in self.proxy_providers:
|
||||
self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}")
|
||||
|
||||
if proxy and proxy.startswith("controld://"):
|
||||
proxy = ctx.params["proxy"] = resolve_proxy(proxy, [])
|
||||
if proxy:
|
||||
|
||||
def log_proxy_used(provider: object, uri: str) -> None:
|
||||
@@ -1385,11 +1399,8 @@ class dl:
|
||||
if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE):
|
||||
# requesting proxy from a specific proxy provider
|
||||
requested_provider, proxy = proxy.split(":", maxsplit=1)
|
||||
# Match simple region codes (us, ca, uk1, us:ny) or provider:region (nordvpn:ca, protonvpn:us:ny).
|
||||
# ':' is allowed as a city separator (e.g. nordvpn:us:seattle, protonvpn:de:berlin).
|
||||
if re.match(r"^[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE) or re.match(
|
||||
r"^[a-z]+:[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE
|
||||
):
|
||||
# Match region codes (us, ca, uk1, us:ny, yul) or provider:region (nordvpn:ca, protonvpn:us:ny).
|
||||
if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", proxy, re.IGNORECASE):
|
||||
proxy = proxy.lower()
|
||||
# Preserve the original user query (region code) for service-specific proxy_map overrides.
|
||||
# NOTE: `proxy` may be overwritten with the resolved proxy URI later.
|
||||
@@ -1637,7 +1648,10 @@ class dl:
|
||||
|
||||
if no_proxy or no_proxy_download:
|
||||
proxy_download = None
|
||||
elif proxy_download and re.match(r"^(?:[a-z]+:){0,2}[a-z]{2}(?:[-:][a-z0-9]+)*(?:\d+)?$", proxy_download, re.I):
|
||||
elif proxy_download and (
|
||||
proxy_download.startswith("controld://")
|
||||
or re.fullmatch(rf"(?:[a-z]+:){{0,2}}{REGION}", proxy_download, re.I)
|
||||
):
|
||||
# same shapes --proxy resolves against providers (two prefixes for gluetun:nordvpn:ca); else an explicit URI
|
||||
try:
|
||||
proxy_download = resolve_proxy(proxy_download.lower(), self.proxy_providers)
|
||||
|
||||
@@ -16,7 +16,19 @@ from unshackle.core import binaries
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import context_settings
|
||||
from unshackle.core.proxies import Basic, ExpressVPN, Gluetun, Hola, NordVPN, ProtonVPN, SurfsharkVPN, WindscribeVPN
|
||||
from unshackle.core.proxies import (
|
||||
Basic,
|
||||
ControlD,
|
||||
ExpressVPN,
|
||||
Gluetun,
|
||||
Hola,
|
||||
NordVPN,
|
||||
ProtonVPN,
|
||||
SurfsharkVPN,
|
||||
WindscribeVPN,
|
||||
)
|
||||
from unshackle.core.proxies.proxy import Proxy
|
||||
from unshackle.core.proxies.resolve import REGION
|
||||
from unshackle.core.service import Service
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.utils.click_types import ContextData
|
||||
@@ -61,7 +73,7 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr
|
||||
service_config = {}
|
||||
merge_dict(config.services.get(service), service_config)
|
||||
|
||||
proxy_providers = []
|
||||
proxy_providers: list[Proxy] = []
|
||||
if no_proxy:
|
||||
ctx.params["proxy"] = None
|
||||
else:
|
||||
@@ -84,6 +96,8 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr
|
||||
proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"]))
|
||||
if binaries.HolaProxy:
|
||||
proxy_providers.append(Hola())
|
||||
if config.proxy_providers.get("controld"):
|
||||
proxy_providers.append(ControlD(**config.proxy_providers["controld"]))
|
||||
for proxy_provider in proxy_providers:
|
||||
log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}")
|
||||
|
||||
@@ -91,9 +105,7 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr
|
||||
requested_provider = None
|
||||
if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE):
|
||||
requested_provider, proxy = proxy.split(":", maxsplit=1)
|
||||
if re.match(r"^[a-z]{2}(?:[-][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE) or re.match(
|
||||
r"^[a-z]+:[a-z]{2}(?:[-][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE
|
||||
):
|
||||
if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", proxy, re.IGNORECASE):
|
||||
proxy = proxy.lower()
|
||||
with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"):
|
||||
if requested_provider:
|
||||
|
||||
@@ -3734,7 +3734,7 @@ def resolve_handler_proxy(
|
||||
|
||||
if (
|
||||
proxy_param
|
||||
and re.match(r"^(https?://|socks)", str(proxy_param))
|
||||
and re.match(r"^(https?://|socks|controld://)", str(proxy_param))
|
||||
and server_account_for(request, normalized_service)
|
||||
):
|
||||
raise APIError(
|
||||
|
||||
@@ -404,7 +404,7 @@ async def search(request: web.Request) -> web.Response:
|
||||
description: Profile to use for credentials and cookies (default - None)
|
||||
proxy:
|
||||
type: string
|
||||
description: Full proxy URI, or a country code when the API key has server_proxy (default - None)
|
||||
description: Full proxy URI, a Control D resolver as controld://<resolver>@dns.controld.com, or a country code when the API key has server_proxy (default - None)
|
||||
no_proxy:
|
||||
type: boolean
|
||||
description: Force disable all proxy use (default - false)
|
||||
@@ -668,7 +668,7 @@ async def list_tracks(request: web.Request) -> web.Response:
|
||||
description: Specific episode/season, or song ("1-5", "2x3" for disc 2 track 3) (optional)
|
||||
proxy:
|
||||
type: string
|
||||
description: Proxy configuration (optional)
|
||||
description: Full proxy URI, a Control D resolver as controld://<resolver>@dns.controld.com, or a country code when the API key has server_proxy (default - None)
|
||||
dl_params:
|
||||
type: object
|
||||
description: |
|
||||
@@ -898,7 +898,7 @@ async def download(request: web.Request) -> web.Response:
|
||||
description: CDM device name on the server to license with, overriding the cdm config mapping. Requires serve.cdm_overrides to allow it (default - None)
|
||||
proxy:
|
||||
type: string
|
||||
description: Full proxy URI, or a country code when the API key has server_proxy (default - None)
|
||||
description: Full proxy URI, a Control D resolver as controld://<resolver>@dns.controld.com, or a country code when the API key has server_proxy (default - None)
|
||||
no_proxy:
|
||||
type: boolean
|
||||
description: Force disable all proxy use (default - false)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from .basic import Basic
|
||||
from .controld import ControlD
|
||||
from .expressvpn import ExpressVPN
|
||||
from .gluetun import Gluetun
|
||||
from .hola import Hola
|
||||
@@ -7,4 +8,14 @@ from .proton import ProtonVPN
|
||||
from .surfsharkvpn import SurfsharkVPN
|
||||
from .windscribevpn import WindscribeVPN
|
||||
|
||||
__all__ = ("Basic", "ExpressVPN", "Gluetun", "Hola", "NordVPN", "ProtonVPN", "SurfsharkVPN", "WindscribeVPN")
|
||||
__all__ = (
|
||||
"Basic",
|
||||
"ControlD",
|
||||
"ExpressVPN",
|
||||
"Gluetun",
|
||||
"Hola",
|
||||
"NordVPN",
|
||||
"ProtonVPN",
|
||||
"SurfsharkVPN",
|
||||
"WindscribeVPN",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import ipaddress
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
from unshackle.core.proxies.proxy import Proxy
|
||||
|
||||
log = logging.getLogger("ControlD")
|
||||
|
||||
API = "https://api.controld.com"
|
||||
DOH_TIMEOUT = 10
|
||||
MIN_TTL = 30
|
||||
MAX_TTL = 3600
|
||||
HEAD_TIMEOUT = 30
|
||||
IDLE_TIMEOUT = 600
|
||||
CHUNK = 65536
|
||||
PREFIX = "unshackle-"
|
||||
SCHEME = "controld://"
|
||||
RESOLVER_ID = re.compile(r"[a-z0-9]+", re.IGNORECASE)
|
||||
MAX_FORWARDERS = 64
|
||||
|
||||
|
||||
def doh_url(resolver: str) -> str:
|
||||
"""Get the DoH URL of a Control D endpoint's resolver ID."""
|
||||
return f"https://dns.controld.com/{resolver}"
|
||||
|
||||
|
||||
def build_query(host: str) -> bytes:
|
||||
"""Build a minimal DNS A-record query (RFC 1035) for wireformat DoH (RFC 8484)."""
|
||||
labels = host.rstrip(".").encode("idna").split(b".")
|
||||
qname = b"".join(bytes([len(label)]) + label for label in labels) + b"\x00"
|
||||
# header: id 0 (unused by DoH), RD set, 1 question
|
||||
return b"\x00\x00\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" + qname + b"\x00\x01\x00\x01"
|
||||
|
||||
|
||||
def skip_name(buf: bytes, i: int) -> int:
|
||||
"""Return the offset past the domain name at `i`. A compression pointer ends it."""
|
||||
while i < len(buf):
|
||||
length = buf[i]
|
||||
if length == 0:
|
||||
return i + 1
|
||||
if length & 0xC0 == 0xC0:
|
||||
return i + 2
|
||||
i += length + 1
|
||||
return i
|
||||
|
||||
|
||||
def parse_a_records(answer: bytes) -> list[tuple[str, int]]:
|
||||
"""Return the (IPv4, TTL) pairs of a DNS response, in order."""
|
||||
if len(answer) < 12:
|
||||
return []
|
||||
questions = int.from_bytes(answer[4:6], "big")
|
||||
answers = int.from_bytes(answer[6:8], "big")
|
||||
i = 12
|
||||
for _ in range(questions):
|
||||
i = skip_name(answer, i) + 4 # QTYPE + QCLASS
|
||||
records = []
|
||||
for _ in range(answers):
|
||||
i = skip_name(answer, i)
|
||||
if i + 10 > len(answer):
|
||||
break
|
||||
rtype = int.from_bytes(answer[i : i + 2], "big")
|
||||
ttl = int.from_bytes(answer[i + 4 : i + 8], "big")
|
||||
rdlength = int.from_bytes(answer[i + 8 : i + 10], "big")
|
||||
rdata = answer[i + 10 : i + 10 + rdlength]
|
||||
i += 10 + rdlength
|
||||
if rtype == 1 and len(rdata) == 4:
|
||||
records.append((".".join(str(octet) for octet in rdata), ttl))
|
||||
return records
|
||||
|
||||
|
||||
class IPv4Adapter(HTTPAdapter):
|
||||
"""Connect over IPv4 only: bound to 0.0.0.0, an IPv6 address fails and the next one is tried."""
|
||||
|
||||
def init_poolmanager(self, *args: object, **kwargs: object) -> None:
|
||||
kwargs["source_address"] = ("0.0.0.0", 0) # nosec B104 - a source address to bind, not a listener
|
||||
super().init_poolmanager(*args, **kwargs)
|
||||
|
||||
|
||||
class Resolver:
|
||||
def __init__(self, url: str):
|
||||
"""Resolve hostnames through one Control D DoH endpoint, caching answers for their TTL."""
|
||||
self.url = url
|
||||
self.session = requests.Session()
|
||||
# Control D redirects only IPv4 queries and authorises the proxy by the asking IP: no IPv6, no env proxy.
|
||||
self.session.trust_env = False
|
||||
self.session.mount("https://", IPv4Adapter())
|
||||
self.cache: dict[str, tuple[str, float]] = {}
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def resolve(self, host: str) -> str:
|
||||
host = host.lower()
|
||||
with self.lock:
|
||||
cached = self.cache.get(host)
|
||||
if cached and cached[1] > time.monotonic():
|
||||
return cached[0]
|
||||
|
||||
query = base64.urlsafe_b64encode(build_query(host)).rstrip(b"=").decode()
|
||||
response = self.session.get(
|
||||
self.url,
|
||||
params={"dns": query},
|
||||
headers={"accept": "application/dns-message"},
|
||||
timeout=DOH_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
records = parse_a_records(response.content)
|
||||
if not records:
|
||||
raise ConnectionError(f"Control D returned no A record for {host}")
|
||||
|
||||
ip, ttl = records[0]
|
||||
with self.lock:
|
||||
self.cache[host] = (ip, time.monotonic() + min(max(ttl, MIN_TTL), MAX_TTL))
|
||||
return ip
|
||||
|
||||
|
||||
def relay(source: socket.socket, sink: socket.socket) -> None:
|
||||
"""Copy bytes one way until the source closes, then half-close the sink."""
|
||||
try:
|
||||
while True:
|
||||
data = source.recv(CHUNK)
|
||||
if not data:
|
||||
break
|
||||
sink.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
sink.shutdown(socket.SHUT_WR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
HOP_BY_HOP = (b"connection:", b"proxy-connection:", b"keep-alive:")
|
||||
|
||||
|
||||
class Handler(socketserver.StreamRequestHandler):
|
||||
server: LocalProxy
|
||||
rbufsize = 0 # unbuffered, so no body bytes are held back from the relay
|
||||
timeout = HEAD_TIMEOUT
|
||||
|
||||
def reply(self, status: bytes) -> None:
|
||||
try:
|
||||
self.connection.sendall(b"HTTP/1.1 " + status + b"\r\n\r\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def forward_response_head(self, upstream: socket.socket) -> None:
|
||||
"""Send the response head on with `Connection: close`, so the client never reuses this connection."""
|
||||
data = b""
|
||||
try:
|
||||
while b"\r\n\r\n" not in data and len(data) < CHUNK:
|
||||
chunk = upstream.recv(CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
except OSError:
|
||||
pass
|
||||
head, found, body = data.partition(b"\r\n\r\n")
|
||||
if found:
|
||||
lines = [x for x in head.split(b"\r\n") if not x.lower().startswith(HOP_BY_HOP)]
|
||||
data = b"\r\n".join(lines) + b"\r\nConnection: close\r\n\r\n" + body
|
||||
try:
|
||||
self.connection.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def handle(self) -> None:
|
||||
try:
|
||||
head = self.rfile.readline(CHUNK)
|
||||
headers = []
|
||||
while True:
|
||||
line = self.rfile.readline(CHUNK)
|
||||
if not line or line in (b"\r\n", b"\n"):
|
||||
break
|
||||
headers.append(line)
|
||||
except OSError:
|
||||
return
|
||||
parts = head.split()
|
||||
if len(parts) < 3:
|
||||
return
|
||||
|
||||
method = parts[0].decode("latin-1")
|
||||
target = parts[1].decode("latin-1")
|
||||
try:
|
||||
if method == "CONNECT":
|
||||
host, _, port_text = target.rpartition(":") if ":" in target else (target, "", "")
|
||||
host, port = host.strip("[]"), int(port_text or 443)
|
||||
else:
|
||||
parsed = urlsplit(target)
|
||||
host, port = parsed.hostname or "", parsed.port or 80
|
||||
except ValueError:
|
||||
return self.reply(b"400 Bad Request")
|
||||
if not host:
|
||||
return self.reply(b"400 Bad Request")
|
||||
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
# an IP literal bypasses Control D's DNS and would leak this machine's own IP
|
||||
log.debug(f"Refused {host}:{port}, an address that Control D cannot redirect")
|
||||
return self.reply(b"502 Bad Gateway")
|
||||
|
||||
try:
|
||||
ip = self.server.resolver.resolve(host)
|
||||
upstream = socket.create_connection((ip, port), timeout=HEAD_TIMEOUT)
|
||||
except Exception as e:
|
||||
log.debug(f"Could not reach {host}:{port}: {e}")
|
||||
return self.reply(b"502 Bad Gateway")
|
||||
|
||||
self.connection.settimeout(IDLE_TIMEOUT)
|
||||
upstream.settimeout(IDLE_TIMEOUT)
|
||||
try:
|
||||
if method == "CONNECT":
|
||||
self.connection.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||
else:
|
||||
headers = [x for x in headers if not x.lower().startswith(HOP_BY_HOP)]
|
||||
upstream.sendall(head + b"".join(headers) + b"Connection: close\r\n\r\n")
|
||||
upward = threading.Thread(target=relay, args=(self.connection, upstream), daemon=True)
|
||||
upward.start()
|
||||
if method != "CONNECT":
|
||||
self.forward_response_head(upstream)
|
||||
relay(upstream, self.connection)
|
||||
# origin done: unblock the upward relay instead of waiting on a client that holds its side open
|
||||
try:
|
||||
self.connection.shutdown(socket.SHUT_RD)
|
||||
except OSError:
|
||||
pass
|
||||
upward.join()
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
|
||||
class LocalProxy(socketserver.ThreadingTCPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, resolver: Resolver):
|
||||
"""An HTTP forward proxy on loopback that resolves every host through `resolver`."""
|
||||
super().__init__(("127.0.0.1", 0), Handler)
|
||||
self.resolver = resolver
|
||||
|
||||
@property
|
||||
def uri(self) -> str:
|
||||
return f"http://127.0.0.1:{self.socket.getsockname()[1]}"
|
||||
|
||||
|
||||
# process-wide across provider instances; separate processes share only the account's unshackle-<region> names
|
||||
FORWARDERS: dict[str, LocalProxy] = {}
|
||||
REGIONS: dict[str, str] = {} # profile -> region, oldest claim first
|
||||
STATE = threading.Lock()
|
||||
|
||||
|
||||
def forwarder(resolver: str) -> LocalProxy:
|
||||
"""Get the loopback forwarder for a resolver ID, starting it on first use. Hold STATE."""
|
||||
url = doh_url(resolver)
|
||||
if url not in FORWARDERS:
|
||||
if len(FORWARDERS) >= MAX_FORWARDERS:
|
||||
raise ValueError("Too many Control D resolvers in use in this process")
|
||||
server = LocalProxy(Resolver(url))
|
||||
threading.Thread(target=server.serve_forever, daemon=True, name="controld").start()
|
||||
FORWARDERS[url] = server
|
||||
log.debug(f"Listening on {server.uri}")
|
||||
return FORWARDERS[url]
|
||||
|
||||
|
||||
def local_proxy(uri: str) -> str:
|
||||
"""
|
||||
Get a loopback proxy URI for a `controld://<resolver>@dns.controld.com` proxy.
|
||||
|
||||
That is the form a client sends to a remote server: the resolver ID alone, never the
|
||||
API token. The client points the profile at the region; the server only resolves
|
||||
through it, from its own address.
|
||||
"""
|
||||
resolver = urlsplit(uri).username or ""
|
||||
if not RESOLVER_ID.fullmatch(resolver):
|
||||
raise ValueError("A controld:// proxy needs a Control D resolver ID")
|
||||
with STATE:
|
||||
return forwarder(resolver.lower()).uri
|
||||
|
||||
|
||||
def resolver_id(value: str) -> str:
|
||||
"""Get the resolver ID from a resolver ID or its DoH URL."""
|
||||
value = value.rstrip("/").rsplit("/", 1)[-1]
|
||||
if not RESOLVER_ID.fullmatch(value):
|
||||
raise ValueError(f"Not a Control D resolver ID: {value}")
|
||||
return value.lower()
|
||||
|
||||
|
||||
class ControlD(Proxy):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
token: str,
|
||||
profile: Optional[str] = None,
|
||||
resolver: Optional[str] = None,
|
||||
profiles: Optional[list[dict]] = None,
|
||||
max_profiles: int = 4,
|
||||
):
|
||||
"""
|
||||
Proxy provider that uses Control D's transparent proxies.
|
||||
|
||||
Control D is a DNS service rather than a proxy: it answers redirected hostnames
|
||||
with the IP of a proxy in the region you chose, and that proxy relays onwards by
|
||||
SNI. There is no proxy URI to hand to Python-Requests, so this proxy provider runs a
|
||||
forwarder on loopback per resolver, and the rest of unshackle sees an ordinary
|
||||
HTTP proxy.
|
||||
|
||||
A query points a profile's default rule at the region asked for, so everything
|
||||
that profile resolves leaves through it, and each profile must be one kept for
|
||||
unshackle alone. A profile holds one region at a time, so each region in use gets
|
||||
a profile of its own: the account's endpoint named `unshackle-<region>`, else a
|
||||
configured pair (`profile` and `resolver`, `profiles`, or both), else a new
|
||||
`unshackle-<region>` endpoint and profile, up to `max_profiles` in all. The
|
||||
locations come from Control D itself, so nothing needs listing here.
|
||||
"""
|
||||
if bool(profile) != bool(resolver):
|
||||
raise ValueError("Control D needs both a profile and a resolver")
|
||||
if max_profiles < 1:
|
||||
raise ValueError("Control D needs max_profiles of 1 or more")
|
||||
pairs = ([{"profile": profile, "resolver": resolver}] if profile else []) + (profiles or [])
|
||||
|
||||
self.token = token
|
||||
self.configured = {x["profile"]: resolver_id(x["resolver"]) for x in pairs}
|
||||
self.max_profiles = max_profiles
|
||||
self.resolvers = dict(self.configured)
|
||||
|
||||
self.catalogue = self.get_locations()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
countries = len({x["country"] for x in self.catalogue})
|
||||
servers = len(self.catalogue)
|
||||
|
||||
return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
|
||||
|
||||
def get_locations(self) -> list[dict]:
|
||||
"""Get Control D's proxy locations. Public data, so no token is needed."""
|
||||
try:
|
||||
response = requests.get(f"{API}/proxies", timeout=DOH_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
return response.json()["body"]["proxies"]
|
||||
except Exception as e:
|
||||
# never fail the command: all providers load up front, even for a run that wants no proxy
|
||||
log.warning(f"Could not read Control D's locations: {e}")
|
||||
return []
|
||||
|
||||
def location(self, query: str) -> Optional[str]:
|
||||
"""Get the identifier of a Control D proxy location for a country or location code."""
|
||||
if not self.catalogue:
|
||||
self.catalogue = self.get_locations()
|
||||
|
||||
# a hidden (residential) location is reachable by exact code only, never by country
|
||||
codes = [x["PK"] for x in self.catalogue if x["PK"].lower() == query]
|
||||
if not codes:
|
||||
codes = [x["PK"] for x in self.catalogue if x["country"].lower() == query and not x.get("hidden")]
|
||||
|
||||
return random.choice(codes) if codes else None
|
||||
|
||||
def api(self, method: str, path: str, **data: str) -> dict:
|
||||
"""Call the Control D API with the account token and return the response body."""
|
||||
response = requests.request(
|
||||
method,
|
||||
f"{API}{path}",
|
||||
data=data or None,
|
||||
headers={"authorization": f"Bearer {self.token}"},
|
||||
timeout=DOH_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["body"]
|
||||
|
||||
def owned(self) -> dict[str, dict]:
|
||||
"""
|
||||
Get the account's `unshackle-<region>` endpoints, by profile.
|
||||
|
||||
An endpoint counts only when no endpoint without that name shares its profile, so a
|
||||
profile that also serves the user's own devices is never redirected.
|
||||
"""
|
||||
devices = self.api("GET", "/devices")["devices"]
|
||||
shared = {x["profile"]["PK"] for x in devices if x.get("profile") and not x["name"].startswith(PREFIX)}
|
||||
return {
|
||||
x["profile"]["PK"]: {"device": x["PK"], "name": x["name"], "resolver": resolver_id(x["resolvers"]["uid"])}
|
||||
for x in devices
|
||||
if x.get("profile") and x["name"].startswith(PREFIX) and x["profile"]["PK"] not in shared
|
||||
}
|
||||
|
||||
def pick(self, query: str) -> str:
|
||||
"""
|
||||
Get the profile for a region: one this process already has on it, else the account's
|
||||
`unshackle-<region>`, else a configured pair this process has not claimed, else a new
|
||||
`unshackle-<region>`, else one to move, which is renamed if unshackle made it.
|
||||
"""
|
||||
for profile, region in REGIONS.items():
|
||||
if region == query and profile in self.resolvers:
|
||||
return profile
|
||||
|
||||
owned = self.owned()
|
||||
for profile, x in owned.items():
|
||||
self.resolvers[profile] = x["resolver"]
|
||||
if x["name"] == PREFIX + query:
|
||||
return profile
|
||||
|
||||
for profile in self.configured:
|
||||
if profile not in REGIONS:
|
||||
return profile
|
||||
|
||||
if len(self.configured) + len(owned) < self.max_profiles:
|
||||
return self.create(query)
|
||||
|
||||
# out of profiles: move one this process has not claimed (maybe idle) first, then the oldest claim
|
||||
candidates = [x for x in [*owned, *self.configured] if x not in REGIONS]
|
||||
candidates += [x for x in REGIONS if x in self.resolvers]
|
||||
profile = candidates[0]
|
||||
log.warning(f"Every Control D profile is in use; moving profile {profile} to {query}")
|
||||
if profile in owned:
|
||||
self.api("PUT", f"/profiles/{profile}", name=PREFIX + query)
|
||||
self.api("PUT", f"/devices/{owned[profile]['device']}", name=PREFIX + query)
|
||||
return profile
|
||||
|
||||
def create(self, query: str) -> str:
|
||||
"""Create a profile and an endpoint that uses it, for unshackle alone."""
|
||||
name = PREFIX + query
|
||||
profile = self.api("POST", "/profiles", name=name)["profiles"][0]["PK"]
|
||||
try:
|
||||
device = self.api("POST", "/devices", name=name, profile_id=profile, icon="desktop-linux")
|
||||
except Exception:
|
||||
self.api("DELETE", f"/profiles/{profile}") # an orphan profile is never found again
|
||||
raise
|
||||
self.resolvers[profile] = resolver_id(device["resolvers"]["uid"])
|
||||
log.info(f"Created profile {name} ({profile}) on the Control D account")
|
||||
return profile
|
||||
|
||||
def claim(self, query: str) -> Optional[str]:
|
||||
"""Point a profile at this region, unless one already is, and get its resolver ID. Hold STATE."""
|
||||
location = self.location(query)
|
||||
if not location:
|
||||
return None
|
||||
|
||||
profile = self.pick(query)
|
||||
resolver = self.resolvers[profile]
|
||||
if REGIONS.get(profile) != query:
|
||||
self.api("PUT", f"/profiles/{profile}/default", do="3", via=location, status="1")
|
||||
log.info(f"Profile {profile} now redirects through {location}")
|
||||
server = FORWARDERS.get(doh_url(resolver))
|
||||
if server:
|
||||
with server.resolver.lock:
|
||||
server.resolver.cache.clear()
|
||||
REGIONS.pop(profile, None)
|
||||
REGIONS[profile] = query
|
||||
return resolver
|
||||
|
||||
def get_proxy(self, query: str) -> Optional[str]:
|
||||
"""Get the local HTTP proxy URI that resolves through Control D for this region."""
|
||||
with STATE:
|
||||
resolver = self.claim(query.lower())
|
||||
return forwarder(resolver).uri if resolver else None
|
||||
|
||||
def get_remote_proxy(self, query: str) -> Optional[str]:
|
||||
"""Get the `controld://` proxy URI for this region, for a server on another machine."""
|
||||
with STATE:
|
||||
resolver = self.claim(query.lower())
|
||||
return f"{SCHEME}{resolver}@dns.controld.com" if resolver else None
|
||||
@@ -15,6 +15,8 @@ from unshackle.core.utils.redact import mask_proxy
|
||||
|
||||
log = logging.getLogger("proxies")
|
||||
|
||||
REGION = r"[a-z]{2,4}(?:_[a-z0-9]{2,4})?(?:-[a-z0-9]+|:[a-z][a-z0-9]*)*\d*"
|
||||
|
||||
|
||||
def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False) -> List[Any]:
|
||||
"""Initialise and return available proxy providers from config.
|
||||
@@ -28,6 +30,7 @@ def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False)
|
||||
from unshackle.core import binaries
|
||||
from unshackle.core.config import config as main_config
|
||||
from unshackle.core.proxies.basic import Basic
|
||||
from unshackle.core.proxies.controld import ControlD
|
||||
from unshackle.core.proxies.expressvpn import ExpressVPN
|
||||
from unshackle.core.proxies.hola import Hola
|
||||
from unshackle.core.proxies.nordvpn import NordVPN
|
||||
@@ -51,6 +54,9 @@ def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False)
|
||||
proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"]))
|
||||
if hasattr(binaries, "HolaProxy") and binaries.HolaProxy:
|
||||
proxy_providers.append(Hola())
|
||||
# last: a bare region reaches it only when nothing else answers, since it changes the account
|
||||
if proxy_config.get("controld"):
|
||||
proxy_providers.append(ControlD(**proxy_config["controld"]))
|
||||
|
||||
if not quiet:
|
||||
for provider in proxy_providers:
|
||||
@@ -74,10 +80,16 @@ def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]:
|
||||
- Direct URI: "https://...", "socks5://..."
|
||||
- Country code: "us", "uk"
|
||||
- Provider:country: "nordvpn:us"
|
||||
- A Control D resolver sent by a client: "controld://<resolver>@dns.controld.com"
|
||||
"""
|
||||
if not proxy:
|
||||
return None
|
||||
|
||||
if proxy.startswith("controld://"):
|
||||
from unshackle.core.proxies.controld import local_proxy
|
||||
|
||||
return local_proxy(proxy)
|
||||
|
||||
if re.match(r"^(https?://|socks)", proxy):
|
||||
return proxy
|
||||
|
||||
|
||||
@@ -684,6 +684,33 @@ def resolve_proxy_arg(proxy_arg: Optional[str]) -> Optional[str]:
|
||||
raise click.ClickException(str(e))
|
||||
|
||||
|
||||
def resolve_remote_proxy_arg(proxy_arg: str) -> Optional[str]:
|
||||
"""
|
||||
Get the proxy URI to send to a server for a --proxy value.
|
||||
|
||||
Control D's proxy URI is a forwarder on this machine, so for a server it becomes the
|
||||
profile's resolver instead (`controld://`), and a bare region query skips Control D.
|
||||
"""
|
||||
from unshackle.core.proxies.controld import ControlD
|
||||
from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy
|
||||
|
||||
try:
|
||||
providers = initialize_proxy_providers()
|
||||
provider, _, query = proxy_arg.partition(":")
|
||||
if provider.lower() == "controld" and query:
|
||||
controld = next((x for x in providers if isinstance(x, ControlD)), None)
|
||||
if not controld:
|
||||
raise ValueError("Proxy provider 'controld' is not configured")
|
||||
uri = controld.get_remote_proxy(query)
|
||||
if not uri:
|
||||
raise ValueError(f"Control D has no location for {query}")
|
||||
log.info(f"Using a Control D proxy in {query}, resolved by the server")
|
||||
return uri
|
||||
return resolve_proxy(proxy_arg, [x for x in providers if not isinstance(x, ControlD)])
|
||||
except ValueError as e:
|
||||
raise click.ClickException(str(e))
|
||||
|
||||
|
||||
_CACHE_HEX_ID_RE = re.compile(r"[0-9a-f]{32,}", re.IGNORECASE)
|
||||
|
||||
|
||||
@@ -924,7 +951,7 @@ class RemoteService:
|
||||
regions = [str(r).lower() for r in self._server_accounts.get("regions") or []]
|
||||
if regions and client_region and client_region not in regions:
|
||||
try:
|
||||
create_data["proxy"] = resolve_proxy_arg(regions[0])
|
||||
create_data["proxy"] = resolve_remote_proxy_arg(regions[0])
|
||||
create_data["proxy_region"] = regions[0]
|
||||
self.log.info(f"Using a '{regions[0]}' proxy to match a server account ({', '.join(regions)})")
|
||||
except click.ClickException:
|
||||
@@ -934,7 +961,7 @@ class RemoteService:
|
||||
"rejects the remote session"
|
||||
)
|
||||
elif not no_proxy and proxy:
|
||||
resolved_proxy = resolve_proxy_arg(proxy)
|
||||
resolved_proxy = resolve_remote_proxy_arg(proxy)
|
||||
if resolved_proxy:
|
||||
create_data["proxy"] = resolved_proxy
|
||||
query = (self.ctx.parent.params.get("proxy_query") if self.ctx.parent else None) or proxy
|
||||
|
||||
Reference in New Issue
Block a user