fix(proxies): return None for unservable queries and repair live providers

Basic, NordVPN, Windscribe, Gluetun and Hola no longer raise on a query they cannot serve. Hola parses stdout only and reports a ban; NordVPN cities use the API filter; uk maps to gb.
This commit is contained in:
imSp4rky
2026-09-25 18:13:31 -06:00
parent 40281660f1
commit 80c0dc4ebf
11 changed files with 278 additions and 108 deletions
+12 -7
View File
@@ -744,12 +744,16 @@ for a plain proxy. On success, unshackle prints a summary such as `(Name - City)
## Hola
Hola requires no configuration at all. It uses the
[`hola-proxy`](https://github.com/Snawoot/hola-proxy) binary, and **auto-loads** whenever
that binary is found on your `PATH`. If you query Hola but the binary is missing, the
proxy provider raises an error that tells you to install it.
Hola requires no configuration at all. It uses the `hola-proxy` binary, and
**auto-loads** whenever that binary is found on your `PATH`. If you query Hola but the
binary is missing, the proxy provider raises an error that tells you to install it.
```shell title="Install hola-proxy, then just query a country"
The `hola-proxy` GitHub repository is no longer online, so build the binary with Go. The
Go module proxy still has the source for version 1.18.2. `go install` puts the binary in
`$(go env GOPATH)/bin`, which must be on your `PATH`.
```shell title="Build hola-proxy, then query a country"
go install github.com/Snawoot/hola-proxy@v1.18.2
unshackle dl --proxy hola:us EXAMPLE 81234567
```
@@ -758,8 +762,9 @@ for available proxies in that country and picks one at random, returning an HTTP
!!! warning "Temporary bans"
Hola's free tier can rate-limit or temporarily ban you if queried too aggressively. If
you see a *temporary ban detected* error, wait before retrying. Hola currently uses
only datacenter proxies.
you see a *temporary ban detected* error, wait before retrying. While a ban is active,
`hola-proxy` does not stop, so unshackle waits 30 seconds and then reports the ban.
Hola currently uses only datacenter proxies.
## Surfshark
+30
View File
@@ -0,0 +1,30 @@
"""Unit tests for the Basic proxy provider: a query it cannot serve returns None so a bare query moves on."""
from __future__ import annotations
import pytest
from unshackle.core.proxies import Basic
pytestmark = pytest.mark.unit
@pytest.fixture
def basic() -> Basic:
return Basic(us=["http://one.example:8080", "http://two.example:8080"], gb="http://gb.example:8080")
@pytest.mark.parametrize(
("query", "expected"), [("us1", "http://one.example:8080"), ("US2", "http://two.example:8080")]
)
def test_numbered_entry(basic: Basic, query: str, expected: str) -> None:
assert basic.get_proxy(query) == expected
def test_single_entry_region(basic: Basic) -> None:
assert basic.get_proxy("gb") == "http://gb.example:8080"
@pytest.mark.parametrize("query", ["us-bos", "yul", "us:seattle", "us150", "us0", "ca"])
def test_query_it_cannot_serve_returns_none(basic: Basic, query: str) -> None:
assert basic.get_proxy(query) is None
+138
View File
@@ -0,0 +1,138 @@
"""Every proxy provider returns None for a query it cannot serve, so a bare --proxy query moves on
to the next proxy provider instead of stopping at the first one that does not know the form."""
from __future__ import annotations
import subprocess
import pytest
from unshackle.core.proxies import Gluetun, Hola, NordVPN, WindscribeVPN
from unshackle.core.proxies import hola as hola_module
pytestmark = pytest.mark.unit
CREDENTIALS = {"username": "a" * 24, "password": "b" * 24}
@pytest.fixture
def nordvpn(monkeypatch: pytest.MonkeyPatch) -> NordVPN:
countries = [{"id": 228, "code": "US", "name": "United States", "serverCount": 1}]
monkeypatch.setattr(NordVPN, "get_countries", staticmethod(lambda: countries))
monkeypatch.setattr(NordVPN, "get_recommended_servers", staticmethod(lambda country_id, city_id=None: []))
return NordVPN(**CREDENTIALS)
@pytest.fixture
def windscribe(monkeypatch: pytest.MonkeyPatch) -> WindscribeVPN:
countries = [
{"country_code": "US", "groups": [{"city": "Dallas", "hosts": [{"hostname": "us-central-150.x.com"}]}]}
]
monkeypatch.setattr(WindscribeVPN, "get_countries", staticmethod(lambda: countries))
return WindscribeVPN(**CREDENTIALS)
@pytest.mark.parametrize("query", ["us-bos", "us_yyz", "zz", "us"])
def test_nordvpn(nordvpn: NordVPN, query: str) -> None:
# "us" has no recommended servers in the fixture
assert nordvpn.get_proxy(query) is None
@pytest.mark.parametrize("query", ["us999", "us-bos", "us_yyz", "zz"])
def test_windscribe(windscribe: WindscribeVPN, query: str) -> None:
assert windscribe.get_proxy(query) is None
def test_windscribe_still_finds_a_listed_server(windscribe: WindscribeVPN) -> None:
assert windscribe.get_proxy("us150") == f"https://{'a' * 24}:{'b' * 24}@us-central-150.x.com:443"
def test_gluetun_skips_a_bare_region() -> None:
assert Gluetun.__new__(Gluetun).get_proxy("us") is None
def hola_with_output(monkeypatch: pytest.MonkeyPatch, stdout: str, stderr: str = "") -> Hola:
def run(*args: object, **kwargs: object) -> subprocess.CompletedProcess:
return subprocess.CompletedProcess(args, 0, stdout=stdout, stderr=stderr)
monkeypatch.setattr(hola_module.subprocess, "run", run)
hola = Hola.__new__(Hola)
hola.binary = "hola-proxy"
hola.countries = [{"us": "United States"}, {"gb": "United Kingdom"}]
return hola
HOLA_STDOUT = (
"Login: user-uuid-1\nPassword: secret\nProxy-Authorization: basic abc\n\n"
"host,ip_address,direct,peer,hola,trial,trial_peer,vendor\n"
"zagent1.hola.org,203.0.113.7,22222,22223,22224,22225,22226,digitalocean\n"
)
def test_hola_parses_stdout_while_the_log_goes_to_stderr(monkeypatch: pytest.MonkeyPatch) -> None:
hola = hola_with_output(monkeypatch, HOLA_STDOUT, 'MAIN: INFO Action "list proxies" succeeded\n')
assert hola.get_proxy("us") == "http://user-uuid-1:secret@203.0.113.7:22223"
@pytest.mark.parametrize(("query", "stdout"), [("us-bos", ""), ("zz", HOLA_STDOUT), ("us", "no login"), ("us", "")])
def test_hola(monkeypatch: pytest.MonkeyPatch, query: str, stdout: str) -> None:
assert hola_with_output(monkeypatch, stdout).get_proxy(query) is None
def test_hola_ban_while_hola_proxy_keeps_retrying(monkeypatch: pytest.MonkeyPatch) -> None:
def run(*args: object, **kwargs: object) -> None:
raise subprocess.TimeoutExpired("hola-proxy", 30, stderr=b"Transaction error: temporary ban detected. Retrying")
monkeypatch.setattr(hola_module.subprocess, "run", run)
hola = Hola.__new__(Hola)
hola.binary = "hola-proxy"
hola.countries = [{"us": "United States"}]
with pytest.raises(ConnectionError, match="banned"):
hola.get_proxy("us")
@pytest.fixture
def nordvpn_with_cities(monkeypatch: pytest.MonkeyPatch) -> NordVPN:
countries = [
{
"id": 228,
"code": "US",
"name": "United States",
"serverCount": 2,
"cities": [
{"id": 1, "name": "Seattle", "dns_name": "seattle"},
{"id": 2, "name": "New York", "dns_name": "new-york"},
],
},
{"id": 227, "code": "GB", "name": "United Kingdom", "serverCount": 1, "cities": []},
]
by_city = {None: "us100.nordvpn.com", 1: "us1.nordvpn.com", 2: "us2.nordvpn.com"}
def recommended(country_id: int, city_id: int | None = None) -> list[dict]:
return [{"hostname": "uk6018.nordvpn.com" if country_id == 227 else by_city[city_id]}]
monkeypatch.setattr(NordVPN, "get_countries", staticmethod(lambda: countries))
monkeypatch.setattr(NordVPN, "get_recommended_servers", staticmethod(recommended))
return NordVPN(**CREDENTIALS)
@pytest.mark.parametrize(
("query", "host"),
[
("us", "us100"),
("us:new-york", "us2"),
("us:new york", "us2"),
("us:seattle", "us1"),
("uk", "uk6018"),
("gb", "uk6018"),
("gb6018", "uk6018"),
],
)
def test_nordvpn_city_and_uk(nordvpn_with_cities: NordVPN, query: str, host: str) -> None:
uri = nordvpn_with_cities.get_proxy(query)
assert uri is not None and uri.endswith(f"@{host}.proxy.nordvpn.com:89")
def test_nordvpn_unknown_city_raises(nordvpn_with_cities: NordVPN) -> None:
with pytest.raises(ValueError, match="No servers found in city 'boston'"):
nordvpn_with_cities.get_proxy("us:boston")
+7
View File
@@ -109,3 +109,10 @@ def test_unusable_server_map_entry_is_ignored_with_a_warning(server: object, cap
proxy = SurfsharkVPN(USERNAME, PASSWORD, server_map={"us": server}) # type: ignore[dict-item]
assert "ignoring server_map entry" in caplog.text
assert host(proxy.get_proxy("us")) in US_HOSTS
@pytest.mark.parametrize("key", ["gb", "uk", "GB"])
@pytest.mark.parametrize("query", ["gb", "uk"])
def test_server_map_uk_and_gb_are_the_same_region(key: str, query: str) -> None:
proxy = SurfsharkVPN(USERNAME, PASSWORD, server_map={key: "uk-lon"})
assert host(proxy.get_proxy(query)) == "uk-lon.prod.surfshark.com:443"
+7 -10
View File
@@ -10,7 +10,7 @@ from unshackle.core.utils.redact import mask_proxy
class Basic(Proxy):
def __init__(self, **countries: dict[str, Union[str, list[str]]]):
def __init__(self, **countries: Union[str, list[str]]):
"""Basic proxy provider that uses the proxies given in the configuration file."""
self.countries = {k.lower(): v for k, v in countries.items()}
@@ -21,12 +21,12 @@ class Basic(Proxy):
return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
def get_proxy(self, query: str) -> Optional[str]:
"""Get a proxy URI from the config."""
"""Get a proxy URI from the config. A query for a region or an entry number not in the config returns None."""
query = query.lower()
match = re.match(r"^([a-z]{2})(\d+)?$", query, re.IGNORECASE)
match = re.fullmatch(r"([a-z]{2})(\d+)?", query)
if not match:
raise ValueError(f'The query "{query}" was not recognized...')
return None
country_code = match.group(1)
entry = match.group(2)
@@ -38,12 +38,9 @@ class Basic(Proxy):
if isinstance(servers, str):
proxy = servers
elif entry:
try:
proxy = servers[int(entry) - 1]
except IndexError:
raise ValueError(
f'There\'s only {len(servers)} prox{"y" if len(servers) == 1 else "ies"} for "{country_code}"...'
)
if not 1 <= int(entry) <= len(servers):
return None
proxy = servers[int(entry) - 1]
else:
proxy = random.choice(servers)
+3 -1
View File
@@ -240,9 +240,11 @@ class Gluetun(Proxy):
query: Query format: "provider:region" (e.g., "windscribe:us", "nordvpn:uk")
Returns:
HTTP proxy URI or None if unavailable
HTTP proxy URI, or None for a bare region, so that a bare query moves on to the next proxy provider
"""
parts = query.split(":")
if len(parts) == 1:
return None
if len(parts) != 2:
raise ValueError(f"Invalid query format: '{query}'. Expected 'provider:region' (e.g., 'windscribe:us')")
+41 -15
View File
@@ -10,8 +10,7 @@ from unshackle.core.proxies.proxy import Proxy
class Hola(Proxy):
def __init__(self):
"""
Proxy provider that uses Hola's direct connections through the hola-proxy project.
https://github.com/Snawoot/hola-proxy
Proxy provider that uses Hola's direct connections through the hola-proxy binary.
"""
self.binary = binaries.HolaProxy
if not self.binary:
@@ -28,33 +27,60 @@ class Hola(Proxy):
"""
Get an HTTP proxy URI for a Datacenter ('direct') or Residential ('lum') Hola server.
Returns None for a query that is not a country Hola lists, or when Hola gives no proxy for it.
TODO: - Add ability to select 'lum' proxies (residential proxies).
- Return and use Proxy Authorization
"""
query = query.lower()
if not any(query in country for country in self.countries):
return None
p = subprocess.check_output([self.binary, "-country", query, "-list-proxies"], stderr=subprocess.STDOUT).decode(
"utf-8", errors="replace"
)
ban = "Transaction error: temporary ban detected."
banned = ConnectionError("Hola banned your IP temporarily from it's services. Try change your IP.")
try:
# hola-proxy logs to stderr from another thread, so a merged stream splits the Login/Password lines.
result = subprocess.run(
[self.binary, "-country", query, "-list-proxies"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=30,
)
except subprocess.TimeoutExpired as e:
# While banned, hola-proxy retries forever and never exits, so the ban only shows in its log so far.
log = e.stderr.decode("utf-8", errors="replace") if isinstance(e.stderr, bytes) else (e.stderr or "")
if ban in log:
raise banned from None
raise
if ban in result.stdout + result.stderr:
raise banned
result.check_returncode()
if "Transaction error: temporary ban detected." in p:
raise ConnectionError("Hola banned your IP temporarily from it's services. Try change your IP.")
login = re.search(r"Login: (.*)\nPassword: (.*)\nProxy-Authorization: (.*)", result.stdout)
if not login:
return None
username, password, proxy_authorization = login.groups()
username, password, proxy_authorization = re.search(
r"Login: (.*)\nPassword: (.*)\nProxy-Authorization: (.*)", p
).groups()
servers = re.findall(r"(zagent.*)", p)
servers = re.findall(r"(zagent.*)", result.stdout)
proxies = []
for server in servers:
host, ip_address, direct, peer, hola, trial, trial_peer, vendor = server.split(",")
proxies.append(f"http://{username}:{password}@{ip_address}:{peer}")
proxy = random.choice(proxies)
return proxy
return random.choice(proxies) if proxies else None
def get_countries(self) -> list[dict[str, str]]:
"""Get a list of available Countries."""
p = subprocess.check_output([self.binary, "-list-countries"]).decode("utf-8", errors="replace")
p = subprocess.run(
[self.binary, "-list-countries"],
capture_output=True,
check=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=60,
).stdout
return [{code: name} for country in p.splitlines() for (code, name) in [country.split(" - ", maxsplit=1)]]
+26 -59
View File
@@ -6,6 +6,7 @@ from typing import Optional
import requests
from unshackle.core.proxies.proxy import Proxy
from unshackle.core.utilities import COUNTRY_CODE_ALIASES
class NordVPN(Proxy):
@@ -52,10 +53,10 @@ class NordVPN(Proxy):
- Country code: "us", "ca", "gb"
- Country ID: "228"
- Specific server: "us1234"
- City selection: "us:seattle", "ca:calgary"
- City selection: "us:seattle", "us:new-york", "ca:calgary"
Returns None if the country code or ID is not one NordVPN lists. A country with no recommended
servers, or a city with no matching server, raises ValueError.
Returns None for a query NordVPN cannot serve: an unrecognised form, a country it does not list,
or a country with no recommended servers. A city with no matching server raises ValueError.
"""
query = query.lower()
city = None
@@ -64,17 +65,17 @@ class NordVPN(Proxy):
query, city = query.split(":", maxsplit=1)
city = city.strip()
if re.match(r"^[a-z]{2}\d+$", query):
if re.fullmatch(r"[a-z]{2}\d+", query):
hostname = f"{query}.proxy.nordvpn.com"
else:
if query.isdigit():
country = self.get_country(by_id=int(query))
elif re.match(r"^[a-z]+$", query):
country = self.get_country(by_code=query)
elif re.fullmatch(r"[a-z]+", query):
country = self.get_country(by_code=COUNTRY_CODE_ALIASES.get(query, query))
else:
raise ValueError(f"The query provided is unsupported and unrecognized: {query}")
return None
if not country:
return
return None
server_map_key = f"{country['code'].lower()}:{city}" if city else country["code"].lower()
server_mapping = self.server_map.get(server_map_key) or (
@@ -84,27 +85,27 @@ class NordVPN(Proxy):
if server_mapping:
hostname = f"{country['code'].lower()}{server_mapping}.proxy.nordvpn.com"
else:
recommended_servers = self.get_recommended_servers(country["id"])
if not recommended_servers:
raise ValueError(
f"The NordVPN Country {query} currently has no recommended servers. "
"Try again later. If the issue persists, double-check the query."
)
city_id = None
if city:
city_servers = self.filter_servers_by_city(recommended_servers, city)
if not city_servers:
# Recommendations are ordered by distance from the caller, so the API has to filter by city.
city_id = next(
(x["id"] for x in country.get("cities", []) if city in (x["name"].lower(), x.get("dns_name"))),
None,
)
if city_id is None:
raise ValueError(
f"No servers found in city '{city}' for country '{country['name']}'. "
"Try a different city or check the city name spelling."
)
recommended_servers = city_servers
recommended_servers = self.get_recommended_servers(country["id"], city_id)
if not recommended_servers:
return None
hostname = random.choice(recommended_servers)["hostname"]
if hostname.startswith("gb"):
# NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname
hostname = f"gb{hostname[2:]}"
hostname = f"uk{hostname[2:]}"
if hostname.endswith(".nordvpn.com") and not hostname.endswith(".proxy.nordvpn.com"):
hostname = hostname[: -len(".nordvpn.com")] + ".proxy.nordvpn.com"
@@ -123,50 +124,16 @@ class NordVPN(Proxy):
return country
@staticmethod
def filter_servers_by_city(servers: list[dict], city: str) -> list[dict]:
def get_recommended_servers(country_id: int, city_id: Optional[int] = None) -> list[dict]:
"""
Filter servers by city name.
The API returns servers with location data that includes city information.
This method filters servers to only those in the specified city.
Args:
servers: List of server dictionaries from the NordVPN API
city: City name to filter by (case-insensitive)
Returns:
List of servers in the specified city
"""
city_lower = city.lower()
filtered = []
for server in servers:
locations = server.get("locations", [])
for location in locations:
# City data can be in different formats:
# - {"city": {"name": "Seattle", ...}}
# - {"city": "Seattle"}
city_data = location.get("city")
if city_data:
city_name = city_data.get("name") if isinstance(city_data, dict) else city_data
if city_name and city_name.lower() == city_lower:
filtered.append(server)
break # Found a match, no need to check other locations for this server
return filtered
@staticmethod
def get_recommended_servers(country_id: int) -> list[dict]:
"""
Get the list of recommended Servers for a Country.
Get the list of recommended Servers for a Country, or for one of its cities.
Note: There may not always be more than one recommended server.
"""
res = requests.get(
url="https://api.nordvpn.com/v1/servers/recommendations",
params={"filters[country_id]": country_id},
timeout=10,
)
params = {"filters[country_id]": country_id}
if city_id is not None:
params["filters[country_city_id]"] = city_id
res = requests.get(url="https://api.nordvpn.com/v1/servers/recommendations", params=params, timeout=10)
if not res.ok:
raise ValueError(f"Failed to get a list of NordVPN countries [{res.status_code}]")
+4 -1
View File
@@ -187,9 +187,12 @@ class ProtonVPN(Proxy):
return chosen
def get_logicals(self) -> list[dict]:
"""Get Proton's server list. Only a list that loaded is cached, so a later session can still fetch it."""
if self.logicals is None:
response = self.api("GET", "/vpn/v1/logicals")
self.logicals = (response.json().get("LogicalServers") or []) if response is not None else []
if response is None:
return []
self.logicals = response.json().get("LogicalServers") or []
return self.logicals
def get_max_tier(self) -> int:
+4 -3
View File
@@ -50,7 +50,8 @@ class SurfsharkVPN(Proxy):
server,
)
continue
self.server_map[region] = server
country, sep, city = str(region).lower().partition(":")
self.server_map[COUNTRY_CODE_ALIASES.get(country, country) + sep + city] = server
self.countries = self.get_countries()
@@ -78,6 +79,7 @@ class SurfsharkVPN(Proxy):
if ":" in query:
query, city = query.split(":", maxsplit=1)
city = city.strip()
query = COUNTRY_CODE_ALIASES.get(query, query)
server_map_key = f"{query}:{city}" if city else query
if server_map_key in self.server_map:
@@ -85,8 +87,7 @@ class SurfsharkVPN(Proxy):
elif not city and any(x["connectionName"].split(".", 1)[0] == query for x in self.countries):
server = query
elif re.fullmatch(r"[a-z]{2}", query):
country_code = COUNTRY_CODE_ALIASES.get(query, query).upper()
servers = [x for x in self.countries if x["countryCode"] == country_code]
servers = [x for x in self.countries if x["countryCode"] == query.upper()]
if not servers:
return None
server = self.get_random_server(servers, city)
+6 -12
View File
@@ -47,6 +47,8 @@ class WindscribeVPN(Proxy):
- Country code: "us", "ca", "gb"
- Specific server: "sg007", "us150", "us-central-150"
- City selection: "us:seattle", "ca:toronto"
Returns None for a query WindscribeVPN cannot serve. A city with no matching server raises ValueError.
"""
# Windscribe names GB hosts "uk-NNN"; Ukraine is "ua", so "uk" is unambiguous.
query = re.sub(r"^uk(?=$|[:\d])", "gb", query.lower())
@@ -62,23 +64,15 @@ class WindscribeVPN(Proxy):
elif query in self.server_map:
hostname = self.server_map[query]
else:
server_match = re.match(r"^([a-z]{2})(\d+)$", query)
server_match = re.fullmatch(r"([a-z]{2})(\d+)", query)
if server_match:
country_code, server_num = server_match.groups()
hostname = self.get_specific_server(country_code, server_num)
if not hostname:
raise ValueError(
f"No WindscribeVPN server found matching '{query}'. "
f"Check the server number or use just '{country_code}' for a random server."
)
elif re.match(r"^[a-z]+$", query):
hostname = self.get_specific_server(*server_match.groups())
elif re.fullmatch(r"[a-z]+", query):
hostname = self.get_random_server(query, city)
elif "-" in query:
hostname = next((h for h in self.iter_hostnames() if h.startswith(f"{query}.")), None)
if not hostname:
raise ValueError(f"No WindscribeVPN server found matching '{query}'.")
else:
raise ValueError(f"The query provided is unsupported and unrecognized: {query}")
hostname = None
if not hostname:
return None