From 4753fcd7bc878e57d68f77d784af6f1097b39c8c Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Thu, 22 Oct 2026 13:53:30 -0600 Subject: [PATCH] feat(proxy): select a vpn server by network or ip with @ On NordVPN, Windscribe and Surfshark, `@` keeps and `@-` skips a hosting network; `@` and `@-` pick or skip one server. The connect line names the server, IP and network. Surfshark pins an IP through a loopback TLS port. --- docs/guide/proxies-and-vpn.md | 54 +++- tests/core/test_proxy_network_selector.py | 307 ++++++++++++++++++++ tests/core/test_proxy_unservable_queries.py | 6 +- unshackle/core/proxies/asn.py | 110 +++++++ unshackle/core/proxies/nordvpn.py | 58 +++- unshackle/core/proxies/resolve.py | 2 +- unshackle/core/proxies/surfsharkvpn.py | 151 ++++++++-- unshackle/core/proxies/windscribevpn.py | 32 +- unshackle/core/remote_service.py | 4 +- 9 files changed, 689 insertions(+), 35 deletions(-) create mode 100644 tests/core/test_proxy_network_selector.py create mode 100644 unshackle/core/proxies/asn.py diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md index 3f8548c3..7d2ccb7d 100644 --- a/docs/guide/proxies-and-vpn.md +++ b/docs/guide/proxies-and-vpn.md @@ -113,7 +113,8 @@ flowchart TD unshackle compares the query against a region grammar: a country or location code of two to four letters, with an optional `_code` part (`res_yyz`), then an optional server -number and/or a `:city` or `-city` part. A `host:port` value never matches. It then +number and/or a `:city` or `-city` part, and an optional `@` network part (see +[Choose a server network](#choose-a-server-network)). A `host:port` value never matches. It then lowercases the query and hands it to the proxy provider. Its section below gives the exact forms each proxy provider accepts. @@ -160,8 +161,43 @@ where you came out: - **Gluetun** reports the verified exit IP, country, and city of the container. - **ExpressVPN** and **Proton VPN** print a location summary such as `(USA - New York, #3 of 5): .214`. +- **NordVPN** and **Windscribe** name the server, its IP, and its network, for example + `Using NordVPN Proxy fr1154.proxy.nordvpn.com (187.14.108.15, Datacamp Limited)`. + **Surfshark** prints the same line when the query has an `@` part. - Other proxy providers log the resolved proxy URL. +### Choose a server network + +A VPN rents its servers from different hosting companies, and each hosting company has its own +network. A service often blocks all the IPs of one network, so a random server in a country +works on one run and fails on the next. NordVPN, Surfshark and Windscribe accept an `@` part after the query that keeps or skips a network, or one server IP: + +| Query | Meaning | +|---|---| +| `nordvpn:ca@datacamp` | A random Canadian server on a network whose name contains `datacamp`. | +| `nordvpn:ca@-datacamp` | A random Canadian server on any other network. | +| `windscribevpn:fr:paris@m247` | A random Paris server on an M247 network. | +| `nordvpn:fr@187.14.108.15` | The French server with that IP. Any of the three proxy providers takes an IP. | +| `surfsharkvpn:fr-bod@-195.86.24.103` | A random Bordeaux server IP other than that one. | + +A network name matches without case, spaces or punctuation, and part of the name is enough. +Read the name or IP to use from the connection line of an earlier run. To keep a server that works, +run again with `@`. When a server is blocked, run again with `@-` to skip its network. + +unshackle gets the network of each server IP from the Team Cymru whois service +(`whois.cymru.com`, TCP port 43). The NordVPN and Windscribe connection line uses it too, on every run. +If that port is blocked, a query with an `@` part fails, and the connection line shows no network +name. unshackle tries the port once per run. +An `@` part skips `server_map`, except on Surfshark, where it selects from the servers of +the mapped cluster. A query that already names one server, such as `nordvpn:us1234`, +does not accept an `@` part. An IP selector needs no whois lookup. On NordVPN it searches every server in +the country, not only the recommended ones, so a pinned server still works when it is busy. + +!!! note "Not every proxy provider" + Only NordVPN, Surfshark and Windscribe read the `@` part. With a bare query such as + `--proxy fr@datacamp`, the other proxy providers have no proxy for it, so prefix + the query to be sure which proxy provider answers. + ### Exit check Before the service sends a request, unshackle looks up the proxy exit IP through the @@ -771,6 +807,7 @@ that server for a day or two as a workaround, until it degrades and you swap in | `us1234` | The specific server `us1234`. | | `nordvpn:us:seattle` | A recommended server in Seattle. | | `228` | A NordVPN numeric country ID. | +| `nordvpn:us@-datacamp` | A recommended US server that is not on a Datacamp network. See [Choose a server network](#choose-a-server-network). | The returned proxy is HTTPS on **port 89** (NordVPN disabled its plain-HTTP proxies on port 80 in 2021). @@ -906,6 +943,8 @@ fall back to the `us` entry. | `surfsharkvpn:us` | A random US server. `uk` and `gb` both give a UK server. | | `surfsharkvpn:us-bos` | A specific server from Surfshark's server list. | | `surfsharkvpn:us:seattle` | A random server in Seattle. | +| `surfsharkvpn:fr-bod@datacamp` | A random Bordeaux server IP on a Datacamp network. See [Choose a server network](#choose-a-server-network). | +| `surfsharkvpn:fr-bod@195.86.24.103` | The server at that IP, in the `fr-bod` cluster. | unshackle ignores case, spaces, hyphens and accents in a city name, so `us:new-york` matches New York. A city also matches the start of a longer name: `de:frankfurt` matches @@ -915,6 +954,18 @@ proxy provider earlier in the order, and a bare `us:seattle` does not work, so u The returned proxy is HTTPS on **port 443**. +A Surfshark server name such as `fr-bod` is a cluster: its DNS name points at many server +IPs, and each connection can go to a different one. Surfshark publishes no name for each server, so +unshackle lists the IPs of a cluster from its nameserver. With an `@` part, unshackle connects to one +of those IPs through a local port on `127.0.0.1`. That port checks the server certificate against the +cluster name, so your credentials go only to a real Surfshark server. The connection line shows the +IP, so you can pin a server that works with `@`. Surfshark changes its server IPs from time to +time, and a pinned IP can stop working. + +With `--remote`, the client sends a Surfshark query with an `@` part to the server as you typed it, +because the local port is on your machine. The server then selects and pins the server with its own +Surfshark configuration. This needs the `server_proxy` grant on your API key. + ## Windscribe Windscribe uses **Service Credentials** from its @@ -941,6 +992,7 @@ proxy_providers: | `us` | A random US server. | | `us150` / `sg007` | A specific numbered server. | | `windscribevpn:us:seattle` | A random server in Seattle. | +| `windscribevpn:fr@-datacamp` | A random French server that is not on a Datacamp network. See [Choose a server network](#choose-a-server-network). | The returned proxy is HTTPS on **port 443**. diff --git a/tests/core/test_proxy_network_selector.py b/tests/core/test_proxy_network_selector.py new file mode 100644 index 00000000..54fa5064 --- /dev/null +++ b/tests/core/test_proxy_network_selector.py @@ -0,0 +1,307 @@ +"""Unit tests for the "@" proxy network selector and the SurfsharkVPN pinned server port.""" + +from __future__ import annotations + +import datetime +import re +import socket +import ssl +import threading +from pathlib import Path +from typing import Iterator +from unittest.mock import MagicMock + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID + +from unshackle.core.proxies import asn, surfsharkvpn +from unshackle.core.proxies.nordvpn import NordVPN +from unshackle.core.proxies.resolve import REGION +from unshackle.core.proxies.surfsharkvpn import SurfsharkVPN +from unshackle.core.proxies.windscribevpn import WindscribeVPN +from unshackle.core.remote_service import resolve_remote_proxy_arg + +pytestmark = pytest.mark.unit + +DATACAMP = "CDNEXT - Datacamp Limited, GB" +HOSTROYALE = "HOSTROYALE - HostRoyale Technologies Pvt Ltd, IN" +NETWORKS = {"10.0.0.1": DATACAMP, "10.0.0.2": HOSTROYALE, "10.0.0.3": HOSTROYALE} + + +@pytest.fixture(autouse=True) +def known_networks(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(asn, "NETWORKS", dict(NETWORKS)) + monkeypatch.setattr(asn, "FAILED", None) + + +@pytest.mark.parametrize( + ("value", "is_query"), + [ + ("nordvpn:ca@datacamp", True), + ("nordvpn:ca:toronto@-datacamp", True), + ("windscribevpn:fr@host-royale", True), + ("fr@data_camp", True), + ("surfsharkvpn:fr-bod@195.86.24.103", True), + ("nordvpn:us", True), + ("fr@", False), + ("fr@-", False), + ("user:pass@host:8080", False), + ("http://user:pass@host:8080", False), + ], +) +def test_region_grammar_takes_the_selector_but_not_a_uri(value: str, is_query: bool) -> None: + """dl and search send a value to the proxy providers only when it matches; anything else is an explicit URI.""" + assert bool(re.fullmatch(rf"(?:[a-z]+:)?{REGION}", value, re.IGNORECASE)) is is_query + + +def test_remote_sends_a_surfshark_selector_unresolved() -> None: + assert resolve_remote_proxy_arg("surfsharkvpn:fr-bod@datacamp", []) == "surfsharkvpn:fr-bod@datacamp" + + +@pytest.mark.parametrize( + ("query", "expected"), + [("fr", ("fr", None)), ("fr:paris@-datacamp", ("fr:paris", "-datacamp")), ("fr@", ("fr", None))], +) +def test_split_selector(query: str, expected: tuple) -> None: + assert asn.split_selector(query) == expected + + +@pytest.mark.parametrize( + ("selector", "expected"), + [("datacamp", ["a"]), ("Data-Camp", ["a"]), ("host royale", ["b", "c"]), ("-datacamp", ["b", "c"])], +) +def test_select_includes_or_excludes_a_network(selector: str, expected: list[str]) -> None: + servers = {"a": "10.0.0.1", "b": "10.0.0.2", "c": "10.0.0.3"} + assert sorted(asn.select(servers, selector)) == expected + + +@pytest.mark.parametrize(("selector", "expected"), [("10.9.9.9", ["x"]), ("-10.9.9.9", ["a", "b"])]) +def test_select_by_ip_needs_no_network_lookup( + selector: str, expected: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(asn, "lookup", lambda ips: pytest.fail("an IP selector looked up networks")) + assert sorted(asn.select({"a": "10.0.0.1", "b": "10.0.0.2", "x": "10.9.9.9"}, selector)) == expected + + +def test_select_drops_a_server_whose_network_is_unknown_even_when_excluding(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(asn, "lookup", lambda ips: {k: v for k, v in NETWORKS.items() if k in set(ips)}) + assert asn.select({"a": "10.0.0.1", "x": "10.9.9.9"}, "-hostroyale") == ["a"] + + +def test_select_needs_a_name() -> None: + with pytest.raises(ValueError, match="needs a network name"): + asn.select({"a": "10.0.0.1"}, "-") + + +def test_lookup_parses_cymru_bulk_whois(monkeypatch: pytest.MonkeyPatch) -> None: + answer = ( + b"Bulk mode; whois.cymru.com [2026-10-22 18:57:29 +0000]\n" + b"212238 | 195.86.24.103 | CDNEXT - Datacamp Limited, GB\n" + b"NA | 10.1.1.1 | NA\n" + ) + + conn = MagicMock() + conn.__enter__.return_value = conn + conn.recv.side_effect = [answer, b""] + monkeypatch.setattr(asn.socket, "create_connection", lambda *args, **kwargs: conn) + assert asn.lookup(["195.86.24.103", "10.1.1.1", "10.0.0.1"]) == { + "195.86.24.103": DATACAMP, + "10.0.0.1": DATACAMP, + } + conn.sendall.assert_called_once_with(b"begin\n195.86.24.103\n10.1.1.1\nend\n") + + +def test_lookup_failure_raises_connection_error(monkeypatch: pytest.MonkeyPatch) -> None: + def refuse(*args: object, **kwargs: object) -> None: + raise OSError("refused") + + monkeypatch.setattr(asn.socket, "create_connection", refuse) + with pytest.raises(ConnectionError, match="whois"): + asn.lookup(["10.9.9.9"]) + + monkeypatch.setattr(asn.socket, "create_connection", lambda *args, **kwargs: pytest.fail("whois asked twice")) + with pytest.raises(ConnectionError): + asn.lookup(["10.9.9.8"]) + assert asn.lookup(["10.0.0.1"]) == {"10.0.0.1": DATACAMP} + + +def test_describe_names_the_network() -> None: + assert asn.describe("fr-028.example", "10.0.0.1") == "fr-028.example (10.0.0.1, Datacamp Limited)" + + +def test_describe_never_raises(monkeypatch: pytest.MonkeyPatch) -> None: + def unresolvable(host: str) -> str: + raise socket.gaierror("no such host") + + monkeypatch.setattr(asn.socket, "gethostbyname", unresolvable) + assert asn.describe("gone.example") == "gone.example" + + +@pytest.fixture +def nordvpn(monkeypatch: pytest.MonkeyPatch) -> NordVPN: + servers = [ + {"hostname": "ca1.nordvpn.com", "station": "10.0.0.1"}, + {"hostname": "ca2.nordvpn.com", "station": "10.0.0.2"}, + ] + countries = [{"id": 38, "code": "CA", "name": "Canada", "serverCount": 2, "cities": []}] + monkeypatch.setattr(NordVPN, "get_countries", staticmethod(lambda: countries)) + monkeypatch.setattr(NordVPN, "get_recommended_servers", staticmethod(lambda c, city=None, limit=None: servers)) + unrecommended = [{"hostname": "ca3.nordvpn.com", "station": "10.0.0.9"}] + monkeypatch.setattr(NordVPN, "get_all_servers", staticmethod(lambda c: servers + unrecommended)) + return NordVPN("a" * 24, "b" * 24, server_map={"ca": 9}) + + +@pytest.mark.parametrize( + ("query", "host"), + [("ca@datacamp", "ca1"), ("ca@-datacamp", "ca2"), ("ca", "ca9"), ("ca@10.0.0.9", "ca3"), ("ca@-10.0.0.1", "ca2")], +) +def test_nordvpn_network_selector_skips_the_server_map(nordvpn: NordVPN, query: str, host: str) -> None: + uri = nordvpn.get_proxy(query) + assert uri is not None and uri.endswith(f"@{host}.proxy.nordvpn.com:89") + assert nordvpn.last_hostname == f"{host}.proxy.nordvpn.com" + + +def test_nordvpn_selector_with_no_match_raises(nordvpn: NordVPN) -> None: + with pytest.raises(ValueError, match="No NordVPN server in ca"): + nordvpn.get_proxy("ca@m247") + + +def test_nordvpn_specific_server_takes_no_selector(nordvpn: NordVPN) -> None: + with pytest.raises(ValueError, match="already one NordVPN server"): + nordvpn.get_proxy("ca1@datacamp") + + +@pytest.fixture +def windscribe(monkeypatch: pytest.MonkeyPatch) -> WindscribeVPN: + countries = [ + { + "country_code": "FR", + "groups": [ + {"city": "Paris", "hosts": [{"hostname": "fr-001.example"}, {"hostname": "fr-002.example"}]}, + {"city": "Bordeaux", "hosts": [{"hostname": "fr-003.example"}]}, + ], + } + ] + ips = {"fr-001.example": "10.0.0.2", "fr-002.example": "10.0.0.3", "fr-003.example": "10.0.0.1"} + monkeypatch.setattr(WindscribeVPN, "get_countries", staticmethod(lambda: countries)) + monkeypatch.setattr("unshackle.core.proxies.windscribevpn.resolve_all", lambda hosts: {x: ips[x] for x in hosts}) + return WindscribeVPN("user", "pass") + + +@pytest.mark.parametrize( + ("query", "hosts"), + [ + ("fr@datacamp", {"fr-003.example"}), + ("fr@-datacamp", {"fr-001.example", "fr-002.example"}), + ("fr:paris@hostroyale", {"fr-001.example", "fr-002.example"}), + ("fr@10.0.0.3", {"fr-002.example"}), + ("fr@-10.0.0.3", {"fr-001.example", "fr-003.example"}), + ], +) +def test_windscribe_network_selector(windscribe: WindscribeVPN, query: str, hosts: set[str]) -> None: + uri = windscribe.get_proxy(query) + assert uri is not None and uri.rsplit("@", 1)[1].removesuffix(":443") in hosts + + +def test_windscribe_selector_with_no_match_raises(windscribe: WindscribeVPN) -> None: + with pytest.raises(ValueError, match="No WindscribeVPN server in fr"): + windscribe.get_proxy("fr:paris@datacamp") + + +@pytest.fixture +def surfshark(monkeypatch: pytest.MonkeyPatch) -> SurfsharkVPN: + clusters = [{"countryCode": "FR", "connectionName": "fr-bod.prod.surfshark.com", "location": "Bordeaux"}] + monkeypatch.setattr(SurfsharkVPN, "get_countries", staticmethod(lambda: clusters)) + monkeypatch.setattr(SurfsharkVPN, "get_pool", staticmethod(lambda hostname: list(NETWORKS))) + monkeypatch.setattr(surfsharkvpn, "pinned", lambda ip, hostname: 4000 + int(ip.rsplit(".", 1)[1])) + return SurfsharkVPN("a" * 24, "b" * 24) + + +@pytest.mark.parametrize( + ("query", "ports"), + [ + ("fr-bod@datacamp", {4001}), + ("fr@-datacamp", {4002, 4003}), + ("fr-bod@10.0.0.3", {4003}), + ("fr-bod@-10.0.0.3", {4001, 4002}), + ], +) +def test_surfshark_selector_pins_a_server_ip(surfshark: SurfsharkVPN, query: str, ports: set[int]) -> None: + uri = surfshark.get_proxy(query) + assert uri is not None and uri.startswith(f"http://{'a' * 24}:{'b' * 24}@127.0.0.1:") + assert int(uri.rsplit(":", 1)[1]) in ports + assert surfshark.last_server is not None and surfshark.last_server[0] == "fr-bod.prod.surfshark.com" + + +def test_surfshark_without_selector_is_unchanged(surfshark: SurfsharkVPN) -> None: + assert surfshark.get_proxy("fr") == f"https://{'a' * 24}:{'b' * 24}@fr-bod.prod.surfshark.com:443" + assert surfshark.last_connection_display() is None + + +@pytest.fixture +def tls_echo(tmp_path: Path) -> Iterator[tuple[int, ssl.SSLContext]]: + """A TLS echo server for *.prod.surfshark.com on loopback, and a client context that trusts it.""" + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "*.prod.surfshark.com")]) + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("*.prod.surfshark.com")]), critical=False) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + pem = cert.public_bytes(serialization.Encoding.PEM) + (tmp_path / "cert.pem").write_bytes(pem) + (tmp_path / "key.pem").write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + server_tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_tls.load_cert_chain(tmp_path / "cert.pem", tmp_path / "key.pem") + + listener = socket.create_server(("127.0.0.1", 0)) + + def serve() -> None: + while True: + try: + conn, _ = listener.accept() + except OSError: + return + try: + with server_tls.wrap_socket(conn, server_side=True) as tls: + while data := tls.recv(65536): + tls.sendall(data) + except (OSError, ssl.SSLError): + pass + + threading.Thread(target=serve, daemon=True).start() + yield listener.getsockname()[1], ssl.create_default_context(cadata=pem.decode()) + listener.close() + + +def test_pinned_port_relays_over_verified_tls( + tls_echo: tuple[int, ssl.SSLContext], monkeypatch: pytest.MonkeyPatch +) -> None: + port, client_tls = tls_echo + monkeypatch.setattr(surfsharkvpn, "PORT", port) + monkeypatch.setattr(surfsharkvpn, "TLS", client_tls) + monkeypatch.setattr(surfsharkvpn, "PINS", {}) + + with socket.create_connection(("127.0.0.1", surfsharkvpn.pinned("127.0.0.1", "fr-bod.prod.surfshark.com"))) as c: + c.settimeout(5) + c.sendall(b"CONNECT example.com:443 HTTP/1.1\r\n\r\n") + assert c.recv(65536) == b"CONNECT example.com:443 HTTP/1.1\r\n\r\n" + + # a hostname the certificate does not name fails the TLS check, and the client sees the connection close + with socket.create_connection(("127.0.0.1", surfsharkvpn.pinned("127.0.0.1", "evil.example.com"))) as c: + c.settimeout(5) + c.sendall(b"CONNECT example.com:443 HTTP/1.1\r\n\r\n") + assert c.recv(65536) == b"" diff --git a/tests/core/test_proxy_unservable_queries.py b/tests/core/test_proxy_unservable_queries.py index 5a5b6338..5457ac17 100644 --- a/tests/core/test_proxy_unservable_queries.py +++ b/tests/core/test_proxy_unservable_queries.py @@ -19,7 +19,9 @@ CREDENTIALS = {"username": "a" * 24, "password": "b" * 24} def nordvpn(monkeypatch: pytest.MonkeyPatch) -> NordVPN: countries = [{"id": 228, "code": "US", "name": "United States", "serverCount": 1}] monkeypatch.setattr(NordVPN, "get_countries", staticmethod(lambda: countries)) - monkeypatch.setattr(NordVPN, "get_recommended_servers", staticmethod(lambda country_id, city_id=None: [])) + monkeypatch.setattr( + NordVPN, "get_recommended_servers", staticmethod(lambda country_id, city_id=None, limit=None: []) + ) return NordVPN(**CREDENTIALS) @@ -108,7 +110,7 @@ def nordvpn_with_cities(monkeypatch: pytest.MonkeyPatch) -> NordVPN: ] by_city = {None: "us100.nordvpn.com", 1: "us1.nordvpn.com", 2: "us2.nordvpn.com"} - def recommended(country_id: int, city_id: int | None = None) -> list[dict]: + def recommended(country_id: int, city_id: int | None = None, limit: int | None = None) -> list[dict]: return [{"hostname": "uk6018.nordvpn.com" if country_id == 227 else by_city[city_id]}] monkeypatch.setattr(NordVPN, "get_countries", staticmethod(lambda: countries)) diff --git a/unshackle/core/proxies/asn.py b/unshackle/core/proxies/asn.py new file mode 100644 index 00000000..af09601f --- /dev/null +++ b/unshackle/core/proxies/asn.py @@ -0,0 +1,110 @@ +"""Tell VPN servers apart by the network (AS) that hosts them, for the `@` proxy selector.""" + +from __future__ import annotations + +import ipaddress +import re +import socket +from concurrent.futures import ThreadPoolExecutor +from typing import Iterable, Optional + +from unidecode import unidecode + +WHOIS = ("whois.cymru.com", 43) +TIMEOUT = 3 +NETWORKS: dict[str, str] = {} +FAILED: Optional[str] = None + + +def split_selector(query: str) -> tuple[str, Optional[str]]: + """Split "fr:paris@-datacamp" into the region query and the network selector after "@".""" + query, _, selector = query.partition("@") + return query, selector or None + + +def lookup(ips: Iterable[str]) -> dict[str, str]: + """ + Get the network (AS) name of each IP from Team Cymru's bulk whois. An IP it does not know is left out. + + A failure is kept for the process, so a blocked port 43 costs one timeout, not one per proxy. + """ + global FAILED + ips = list(dict.fromkeys(ips)) + wanted = [x for x in ips if x not in NETWORKS] + if wanted: + if FAILED: + raise ConnectionError(FAILED) + data = b"" + try: + with socket.create_connection(WHOIS, timeout=TIMEOUT) as conn: + conn.sendall(("begin\n" + "\n".join(wanted) + "\nend\n").encode()) + while chunk := conn.recv(65536): + data += chunk + except OSError as e: + FAILED = f"Could not get the networks of the VPN servers from {WHOIS[0]}: {e}" + raise ConnectionError(FAILED) from None + for line in data.decode("utf-8", "replace").splitlines(): + fields = [x.strip() for x in line.split("|")] + if len(fields) == 3 and fields[0].isdigit(): + NETWORKS[fields[1]] = fields[2] + return {x: NETWORKS[x] for x in ips if x in NETWORKS} + + +def fold(text: str) -> str: + """Reduce a name to lowercase ASCII letters and digits, so "Frankfurt am Main" and "frankfurt-am-main" match.""" + return re.sub(r"[^a-z0-9]", "", unidecode(text).lower()) + + +def short_name(name: str) -> str: + """Shorten an AS name such as "CDNEXT - Datacamp Limited, GB" to "Datacamp Limited".""" + return re.sub(r",\s*[A-Z]{2}$", "", name).split(" - ", 1)[-1] + + +def is_ip(selector: str) -> bool: + """Whether a selector names one server IPv4 address, not a network.""" + try: + ipaddress.IPv4Address(selector) + except ValueError: + return False + return True + + +def select(servers: dict[str, str], selector: str) -> list[str]: + """ + Get the servers (a server -> IP map) that match the selector, or with a leading "-", the servers that do not. + + An IPv4 selector matches that server IP. Any other selector matches a network whose name contains it, + ignoring case, spaces and punctuation; a server whose network is not known is left out either way. + """ + exclude = selector.startswith("-") + if is_ip(address := selector.removeprefix("-")): + return [x for x, ip in servers.items() if (ip == address) != exclude] + selector = fold(address) + if not selector: + raise ValueError("A proxy network selector needs a network name after '@', for example @datacamp") + networks = lookup(servers.values()) + return [x for x, ip in servers.items() if ip in networks and (selector in fold(networks[ip])) != exclude] + + +def resolve_all(hostnames: Iterable[str]) -> dict[str, str]: + """Resolve hostnames to an IPv4 address each, in parallel. A hostname that does not resolve is left out.""" + + def resolve(hostname: str) -> Optional[str]: + try: + return socket.gethostbyname(hostname) + except OSError: + return None + + hostnames = list(dict.fromkeys(hostnames)) + with ThreadPoolExecutor(16) as pool: + return {x: ip for x, ip in zip(hostnames, pool.map(resolve, hostnames)) if ip} + + +def describe(hostname: str, ip: Optional[str] = None) -> str: + """Name a server with its IP and network, for the line that says which proxy is in use. This never raises.""" + try: + ip = ip or socket.gethostbyname(hostname) + network = lookup([ip]).get(ip) + except OSError: + return f"{hostname} ({ip})" if ip else hostname + return f"{hostname} ({ip}, {short_name(network)})" if network else f"{hostname} ({ip})" diff --git a/unshackle/core/proxies/nordvpn.py b/unshackle/core/proxies/nordvpn.py index bb6e1a34..d74c4186 100644 --- a/unshackle/core/proxies/nordvpn.py +++ b/unshackle/core/proxies/nordvpn.py @@ -5,6 +5,7 @@ from typing import Optional import requests +from unshackle.core.proxies.asn import describe, is_ip, select, split_selector from unshackle.core.proxies.proxy import Proxy from unshackle.core.utilities import COUNTRY_CODE_ALIASES @@ -33,6 +34,7 @@ class NordVPN(Proxy): self.username = username self.password = password self.server_map = server_map or {} + self.last_hostname: Optional[str] = None self.countries = self.get_countries() @@ -54,11 +56,13 @@ class NordVPN(Proxy): - Country ID: "228" - Specific server: "us1234" - City selection: "us:seattle", "us:new-york", "ca:calgary" + - Network selection: "ca@datacamp", "ca:toronto@-datacamp" (a leading "-" excludes) + - One server IP: "fr@187.14.108.15" Returns None for a query NordVPN cannot serve: an unrecognised form, a country it does not list, or a country with no recommended servers. A city with no matching server raises ValueError. """ - query = query.lower() + query, selector = split_selector(query.lower()) city = None if ":" in query: @@ -66,6 +70,8 @@ class NordVPN(Proxy): city = city.strip() if re.fullmatch(r"[a-z]{2}\d+", query): + if selector: + raise ValueError(f"'{query}' is already one NordVPN server, so it takes no '@' network selector.") hostname = f"{query}.proxy.nordvpn.com" else: if query.isdigit(): @@ -82,7 +88,7 @@ class NordVPN(Proxy): self.server_map.get(country["code"].lower()) if not city else None ) - if server_mapping: + if server_mapping and not selector: hostname = f"{country['code'].lower()}{server_mapping}.proxy.nordvpn.com" else: city_id = None @@ -97,11 +103,21 @@ class NordVPN(Proxy): f"No servers found in city '{city}' for country '{country['name']}'. " "Try a different city or check the city name spelling." ) - recommended_servers = self.get_recommended_servers(country["id"], city_id) + if selector and is_ip(selector): + recommended_servers = self.get_all_servers(country["id"]) + else: + recommended_servers = self.get_recommended_servers( + country["id"], city_id, limit=0 if selector else None + ) if not recommended_servers: return None + hostnames = [x["hostname"] for x in recommended_servers] + if selector: + hostnames = select({x["hostname"]: x["station"] for x in recommended_servers}, selector) + if not hostnames: + raise ValueError(f"No NordVPN server in {query} matches '@{selector}'.") - hostname = random.choice(recommended_servers)["hostname"] + hostname = random.choice(hostnames) if hostname.startswith("gb"): # NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname @@ -110,8 +126,13 @@ class NordVPN(Proxy): if hostname.endswith(".nordvpn.com") and not hostname.endswith(".proxy.nordvpn.com"): hostname = hostname[: -len(".nordvpn.com")] + ".proxy.nordvpn.com" + self.last_hostname = hostname return f"https://{self.username}:{self.password}@{hostname}:89" + def last_connection_display(self) -> Optional[str]: + """Name the last server with its IP and network, so a blocked network can be excluded with '@-'.""" + return describe(self.last_hostname) if self.last_hostname else None + def get_country(self, by_id: Optional[int] = None, by_code: Optional[str] = None) -> Optional[dict]: """Find a Country and its metadata.""" if all(x is None for x in (by_id, by_code)): @@ -124,15 +145,40 @@ class NordVPN(Proxy): return country @staticmethod - def get_recommended_servers(country_id: int, city_id: Optional[int] = None) -> list[dict]: + def get_all_servers(country_id: int) -> list[dict]: + """Get the hostname and IP of every server in a Country, recommended or not.""" + res = requests.get( + url="https://api.nordvpn.com/v1/servers", + params={ + "filters[country_id]": country_id, + "limit": 0, + "fields[servers.hostname]": 1, + "fields[servers.station]": 1, + }, + timeout=10, + ) + if not res.ok: + raise ValueError(f"Failed to get a list of NordVPN servers [{res.status_code}]") + + try: + return res.json() + except json.JSONDecodeError: + raise ValueError("Could not decode list of NordVPN servers, not JSON data.") + + @staticmethod + def get_recommended_servers( + country_id: int, city_id: Optional[int] = None, limit: Optional[int] = None + ) -> list[dict]: """ Get the list of recommended Servers for a Country, or for one of its cities. - Note: There may not always be more than one recommended server. + Note: There may not always be more than one recommended server. A limit of 0 gets all of them. """ params = {"filters[country_id]": country_id} if city_id is not None: params["filters[country_city_id]"] = city_id + if limit is not None: + params.update({"limit": limit, "fields[servers.hostname]": 1, "fields[servers.station]": 1}) res = requests.get(url="https://api.nordvpn.com/v1/servers/recommendations", params=params, timeout=10) if not res.ok: raise ValueError(f"Failed to get a list of NordVPN countries [{res.status_code}]") diff --git a/unshackle/core/proxies/resolve.py b/unshackle/core/proxies/resolve.py index 43c71538..b30b84ff 100644 --- a/unshackle/core/proxies/resolve.py +++ b/unshackle/core/proxies/resolve.py @@ -16,7 +16,7 @@ from unshackle.core.utils.redact import mask_proxy log = logging.getLogger("proxies") -REGION = r"[a-z]{2,4}(?:_[a-z0-9]{2,4})?(?:-[a-z0-9]+|:[a-z][a-z0-9]*)*\d*" +REGION = r"[a-z]{2,4}(?:_[a-z0-9]{2,4})?(?:-[a-z0-9]+|:[a-z][a-z0-9]*)*\d*(?:@-?[a-z0-9][a-z0-9._-]*)?" class Unavailable(Proxy): diff --git a/unshackle/core/proxies/surfsharkvpn.py b/unshackle/core/proxies/surfsharkvpn.py index 0dfa6d26..6ba284be 100644 --- a/unshackle/core/proxies/surfsharkvpn.py +++ b/unshackle/core/proxies/surfsharkvpn.py @@ -1,17 +1,86 @@ +import asyncio import json import logging import random import re +import socket +import ssl +import threading +from functools import cache, partial from typing import Optional +import certifi import requests -from unidecode import unidecode +from unshackle.core.proxies.asn import describe, fold, is_ip, select, split_selector +from unshackle.core.proxies.controld import build_query, parse_a_records from unshackle.core.proxies.proxy import Proxy from unshackle.core.utilities import COUNTRY_CODE_ALIASES log = logging.getLogger("proxies.surfsharkvpn") +POOL_QUERIES = 48 +PORT = 443 +CONNECT_TIMEOUT = 30 +CHUNK = 65536 +TLS = ssl.create_default_context(cafile=certifi.where()) # the same CA bundle as requests +PINS: dict[tuple[str, str], int] = {} +LOOP: Optional[asyncio.AbstractEventLoop] = None +STATE = threading.Lock() + + +async def pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Copy bytes one way until the source closes, then close the sink.""" + try: + while data := await reader.read(CHUNK): + writer.write(data) + await writer.drain() + except OSError: + pass + finally: + writer.close() + + +async def tunnel(ip: str, hostname: str, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Carry one client connection over TLS to the server at `ip`, with its certificate checked against `hostname`.""" + try: + upstream = await asyncio.wait_for( + asyncio.open_connection(ip, PORT, ssl=TLS, server_hostname=hostname), CONNECT_TIMEOUT + ) + except OSError as e: + log.warning(f"Could not reach {hostname} at {ip}: {e}") + writer.close() + return + await asyncio.gather(pipe(reader, upstream[1]), pipe(upstream[0], writer)) + + +@cache +def nameserver(zone: str) -> str: + """Get the IPv4 address of a zone's first authoritative nameserver, through Google's DNS-over-HTTPS JSON API.""" + res = requests.get("https://dns.google/resolve", params={"name": zone, "type": "NS"}, timeout=10) + res.raise_for_status() + return socket.gethostbyname([x["data"] for x in res.json().get("Answer", []) if x.get("type") == 2][0]) + + +def pinned(ip: str, hostname: str) -> int: + """ + Get a loopback port that carries each connection over TLS to one server IP, starting it on first use. + + The client still speaks the HTTP proxy protocol with the credentials itself: the port only fixes the IP, + which a proxy URI cannot do because the certificate names the hostname, not the IP. + """ + global LOOP + with STATE: + if (ip, hostname) not in PINS: + if LOOP is None: + LOOP = asyncio.new_event_loop() + threading.Thread(target=LOOP.run_forever, daemon=True, name="surfshark").start() + server = asyncio.run_coroutine_threadsafe( + asyncio.start_server(partial(tunnel, ip, hostname), "127.0.0.1", 0), LOOP + ).result() + PINS[ip, hostname] = server.sockets[0].getsockname()[1] + return PINS[ip, hostname] + class SurfsharkVPN(Proxy): def __init__(self, username: str, password: str, server_map: Optional[dict[str, str]] = None): @@ -53,6 +122,7 @@ class SurfsharkVPN(Proxy): country, sep, city = str(region).lower().partition(":") self.server_map[COUNTRY_CODE_ALIASES.get(country, country) + sep + city] = server + self.last_server: Optional[tuple[str, str]] = None self.countries = self.get_countries() def __repr__(self) -> str: @@ -69,11 +139,14 @@ class SurfsharkVPN(Proxy): - Country code: "us", "ca", "gb" (or "uk") - Specific server: "us-bos" (Boston) - City selection: "us:seattle", "us:new-york" + - Network selection: "fr-bod@datacamp", "fr@-hostroyale" (a leading "-" excludes) + - One server IP: "fr-bod@195.86.24.103" + A network or IP selection goes through a loopback port pinned to the server IP (see `pinned`). Returns None for a query that names no SurfsharkVPN country or server, so that the next proxy provider can answer it. A city with no matching server raises ValueError. """ - query = query.strip().lower() + query, selector = split_selector(query.strip().lower()) city = None if ":" in query: @@ -83,18 +156,65 @@ class SurfsharkVPN(Proxy): server_map_key = f"{query}:{city}" if city else query if server_map_key in self.server_map: - server = self.server_map[server_map_key] + servers = [self.server_map[server_map_key]] elif not city and any(x["connectionName"].split(".", 1)[0] == query for x in self.countries): - server = query + servers = [query] elif re.fullmatch(r"[a-z]{2}", query): - servers = [x for x in self.countries if x["countryCode"] == query.upper()] - if not servers: + in_country = [x for x in self.countries if x["countryCode"] == query.upper()] + if not in_country: return None - server = self.get_random_server(servers, city) + servers = [x["connectionName"] for x in self.get_city_servers(in_country, city)] else: return None - return f"https://{self.username}:{self.password}@{self.get_hostname(server)}:443" + hostnames = [self.get_hostname(x) for x in servers] + self.last_server = None + if not selector: + return f"https://{self.username}:{self.password}@{random.choice(hostnames)}:443" + if is_ip(selector): + hostname, ip = random.choice(hostnames), selector + else: + hostname, ip = self.get_server_on_network(hostnames, selector) + self.last_server = (hostname, ip) + return f"http://{self.username}:{self.password}@127.0.0.1:{pinned(ip, hostname)}" + + def last_connection_display(self) -> Optional[str]: + """Name the pinned server with its IP and network, so it can be pinned again or excluded with '@-'.""" + return describe(*self.last_server) if self.last_server else None + + def get_server_on_network(self, hostnames: list[str], selector: str) -> tuple[str, str]: + """Get a random (hostname, IP) that matches the selector (see `asn.select`), trying the clusters in random order.""" + for hostname in random.sample(hostnames, len(hostnames)): + ips = select({ip: ip for ip in self.get_pool(hostname)}, selector) + if ips: + return hostname, random.choice(ips) + raise ValueError(f"No SurfsharkVPN server matches '@{selector}'.") + + @staticmethod + def get_pool(hostname: str) -> list[str]: + """ + Get the server IPs behind a cluster hostname. + + Its nameserver answers with two of them at random, so this sends it many queries at once and keeps every answer. + """ + ips: set[str] = set() + try: + address = nameserver(hostname.split(".", 1)[1]) + query = build_query(hostname) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.settimeout(1) + for _ in range(POOL_QUERIES): + sock.sendto(query, (address, 53)) + try: + for _ in range(POOL_QUERIES): + ips.update(ip for ip, _ in parse_a_records(sock.recv(4096))) + except TimeoutError: + pass + except (requests.RequestException, ValueError, IndexError, OSError) as e: + raise ValueError(f"Could not list the SurfsharkVPN servers of {hostname}: {e}") from None + if not ips: + raise ValueError(f"The nameserver of {hostname} did not answer, so its servers could not be listed.") + return sorted(ips) @staticmethod def get_hostname(server: str) -> str: @@ -102,16 +222,16 @@ class SurfsharkVPN(Proxy): server = server.lower() return server if server.endswith(".surfshark.com") else f"{server}.prod.surfshark.com" - def get_random_server(self, servers: list[dict], city: Optional[str] = None) -> str: + def get_city_servers(self, servers: list[dict], city: Optional[str] = None) -> list[dict]: """ - Get a random server name from a country's servers, optionally filtered by city. + Get a country's servers, optionally filtered by city. A city matches with case, spaces, hyphens and accents ignored ("new-york" matches "New York"), exactly first, then as a prefix ("frankfurt" matches "Frankfurt am Main"). """ if city: - wanted = self.fold(city) - located = [(self.fold(x.get("location", "")), x) for x in servers] + wanted = fold(city) + located = [(fold(x.get("location", "")), x) for x in servers] servers = [x for location, x in located if location == wanted] or [ x for location, x in located if location.startswith(wanted) ] @@ -120,12 +240,7 @@ class SurfsharkVPN(Proxy): f"No servers found in city '{city}'. Try a different city or check the city name spelling." ) - return random.choice(servers)["connectionName"] - - @staticmethod - def fold(text: str) -> str: - """Reduce a place name to lowercase ASCII letters and digits.""" - return re.sub(r"[^a-z0-9]", "", unidecode(text).lower()) + return servers @staticmethod def get_countries() -> list[dict]: diff --git a/unshackle/core/proxies/windscribevpn.py b/unshackle/core/proxies/windscribevpn.py index a5bbf34d..ca023378 100644 --- a/unshackle/core/proxies/windscribevpn.py +++ b/unshackle/core/proxies/windscribevpn.py @@ -5,6 +5,7 @@ from typing import Iterator, Optional import requests +from unshackle.core.proxies.asn import describe, resolve_all, select, split_selector from unshackle.core.proxies.proxy import Proxy @@ -30,6 +31,7 @@ class WindscribeVPN(Proxy): self.username = username self.password = password self.server_map = server_map or {} + self.last_hostname: Optional[str] = None self.countries = self.get_countries() @@ -47,11 +49,13 @@ class WindscribeVPN(Proxy): - Country code: "us", "ca", "gb" - Specific server: "sg007", "us150", "us-central-150" - City selection: "us:seattle", "ca:toronto" + - Network selection: "fr@datacamp", "ca:toronto@-datacamp" (a leading "-" excludes) + - One server IP: "fr@195.86.24.129" Returns None for a query WindscribeVPN cannot serve. A city with no matching server raises ValueError. """ # Windscribe names GB hosts "uk-NNN"; Ukraine is "ua", so "uk" is unambiguous. - query = re.sub(r"^uk(?=$|[:\d])", "gb", query.lower()) + query, selector = split_selector(re.sub(r"^uk(?=$|[:\d@])", "gb", query.lower())) city = None if ":" in query: @@ -59,8 +63,12 @@ class WindscribeVPN(Proxy): city = city.strip() server_map_key = f"{query}:{city}" if city else query - if server_map_key in self.server_map: - hostname: Optional[str] = self.server_map[server_map_key] + if selector: + if not re.fullmatch(r"[a-z]+", query): + raise ValueError(f"'{query}' is already one WindscribeVPN server, so it takes no '@' network selector.") + hostname: Optional[str] = self.get_random_server(query, city, selector) + elif server_map_key in self.server_map: + hostname = self.server_map[server_map_key] elif query in self.server_map: hostname = self.server_map[query] else: @@ -77,9 +85,13 @@ class WindscribeVPN(Proxy): if not hostname: return None - hostname = hostname.split(":")[0] + hostname = self.last_hostname = hostname.split(":")[0] return f"https://{self.username}:{self.password}@{hostname}:443" + def last_connection_display(self) -> Optional[str]: + """Name the last server with its IP and network, so a blocked network can be excluded with '@-'.""" + return describe(self.last_hostname) if self.last_hostname else None + def iter_hostnames(self) -> Iterator[str]: """Yield every known server hostname.""" for location in self.countries: @@ -107,19 +119,23 @@ class WindscribeVPN(Proxy): return hostname return None - def get_random_server(self, country_code: str, city: Optional[str] = None) -> Optional[str]: + def get_random_server( + self, country_code: str, city: Optional[str] = None, selector: Optional[str] = None + ) -> Optional[str]: """ Get a random server hostname for a country, optionally filtered by city. Args: country_code: The country code (e.g., "us", "ca") city: Optional city name to filter by (case-insensitive) + selector: Optional network selector, such as "datacamp" or "-datacamp" (see asn.select) Returns: A random hostname from matching servers, or None if the country has no servers. Raises: - ValueError: If you give a city and no server in that country matches it. + ValueError: If you give a city and no server in that country matches it, or a selector + and no server is on a matching network. """ hostnames = [] @@ -135,6 +151,10 @@ class WindscribeVPN(Proxy): if hostname := host.get("hostname"): hostnames.append(hostname) + if hostnames and selector: + hostnames = select(resolve_all(hostnames), selector) + if not hostnames: + raise ValueError(f"No WindscribeVPN server in {country_code} matches '@{selector}'.") if hostnames: return random.choice(hostnames) elif city: diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index ef977f86..889a4f3f 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -916,6 +916,8 @@ def resolve_remote_proxy_arg(proxy_arg: str, providers: Optional[list] = None) - # the providers dl or a serve already loaded, which a serve leaves empty for a key without server_proxy providers = initialize_proxy_providers() if providers is None else providers provider, _, query = proxy_arg.partition(":") + if provider.lower() == "surfsharkvpn" and "@" in query: + return proxy_arg if provider.lower() == "controld" and query: controld = find_provider(providers, "controld") uri = controld.get_remote_proxy(query) @@ -1281,7 +1283,7 @@ class RemoteService: if resolved_proxy: create_data["proxy"] = resolved_proxy query = (self.ctx.parent.params.get("proxy_query") if self.ctx.parent else None) or proxy - country = re.match(r"^(?:[a-z]+:)?([a-z]{2})\d*$", query, re.IGNORECASE) + country = re.match(r"^(?:[a-z]+:)?([a-z]{2})\d*(?:@.*)?$", query, re.IGNORECASE) if country: create_data["proxy_region"] = country.group(1).lower() self.require_reachable_proxy(create_data.get("proxy"))