diff --git a/docs/guide/proxies-and-vpn.md b/docs/guide/proxies-and-vpn.md index 92a578b5..42791707 100644 --- a/docs/guide/proxies-and-vpn.md +++ b/docs/guide/proxies-and-vpn.md @@ -96,15 +96,19 @@ flowchart TD 1. **An explicit URI.** unshackle uses anything shaped like `http://...`, `https://...`, or a `socks...` URI exactly as you give it. unshackle logs - `Using explicit Proxy: ...` and does no lookup. + `Using explicit Proxy: ...` and does no lookup. A `host:port` value such as + `localhost:8080` is also an explicit proxy. 2. **A provider-prefixed query**: `provider:query`, for example `nordvpn:us` or `gluetun:windscribe:us`. unshackle finds the proxy provider whose name matches the prefix (case-insensitive). It then asks *only* that proxy provider to find a proxy for the remainder. If unshackle loads no such proxy provider, or if that proxy provider returns no proxy, the download errors. -3. **A bare query**: a region like `us`, `gb`, `us:seattle`, or `us1234`. unshackle - asks each loaded proxy provider **in order** and uses the first proxy any of them - returns. +3. **A bare query**: a region like `us`, `gb`, or `us1234`. unshackle asks each + loaded proxy provider **in order** and uses the first proxy any of them returns. A + bare city query such as `us:seattle` does not work: unshackle reads `us` as a proxy + provider name. Use the proxy provider prefix, for example `nordvpn:us:seattle`. + If no proxy provider has a proxy for the query, unshackle stops with + `No proxy provider had a proxy for ` and does not continue without a proxy. unshackle compares the query against a region grammar: a country or location code of two to four letters, with an optional `_code` part (`res_yyz`), then an optional server @@ -659,7 +663,7 @@ that server for a day or two as a workaround, until it degrades and you swap in |---|---| | `us` | A recommended US server. | | `us1234` | The specific server `us1234`. | -| `us:seattle` | A recommended server in Seattle. | +| `nordvpn:us:seattle` | A recommended server in Seattle. | | `228` | A NordVPN numeric country ID. | The returned proxy is HTTPS on **port 89** (NordVPN disabled its plain-HTTP proxies on @@ -799,8 +803,8 @@ fall back to the `us` entry. unshackle ignores case, spaces, hyphens and accents in a city name, so `us:new-york` matches New York. A city also matches the start of a longer name: `de:frankfurt` matches -Frankfurt am Main. A bare `us` also reaches Surfshark, but a bare `us-bos` or -`us:seattle` can stop at a proxy provider earlier in the order, so use the +Frankfurt am Main. A bare `us` also reaches Surfshark. A bare `us-bos` can stop at a +proxy provider earlier in the order, and a bare `us:seattle` does not work, so use the `surfsharkvpn:` prefix for those forms. The returned proxy is HTTPS on **port 443**. @@ -830,7 +834,7 @@ proxy_providers: |---|---| | `us` | A random US server. | | `us150` / `sg007` | A specific numbered server. | -| `us:seattle` | A random server in Seattle. | +| `windscribevpn:us:seattle` | A random server in Seattle. | The returned proxy is HTTPS on **port 443**. diff --git a/docs/reference/configuration/network.md b/docs/reference/configuration/network.md index e7c2f2dc..c40a8cac 100644 --- a/docs/reference/configuration/network.md +++ b/docs/reference/configuration/network.md @@ -102,7 +102,6 @@ Config keys of the Control D proxy provider. See | `max_profiles` | `int` | `4` | The most profiles unshackle uses: your pairs plus the `unshackle-*` endpoints on the account. It must be 1 or more. unshackle creates an `unshackle-` profile and endpoint only while the total is lower. | !!! note "Provider loading differs between CLI and REST server" - The `dl` CLI loads all providers, including `windscribevpn` and `gluetun`. The REST API / - remote-client path uses a separate resolver that does **not** load `windscribevpn` or - `gluetun`. ExpressVPN and ProtonVPN also auto-load when their cached session exists, and Hola + The `dl` CLI loads all providers, including `gluetun`. The REST API / remote-client path + does **not** load `gluetun`, because it starts a local Docker container. ExpressVPN and ProtonVPN also auto-load when their cached session exists, and Hola auto-loads whenever the `hola-proxy` binary is present. diff --git a/tests/core/test_proxy_dispatch.py b/tests/core/test_proxy_dispatch.py new file mode 100644 index 00000000..ee985217 --- /dev/null +++ b/tests/core/test_proxy_dispatch.py @@ -0,0 +1,197 @@ +"""Unit tests for the shared proxy dispatch that dl, search and resolve_proxy use: the provider:query +split, the named or bare provider lookup, and the line that reports which proxy provider answered.""" + +from __future__ import annotations + +import logging +from typing import Iterator, Optional + +import pytest + +from unshackle.core.proxies import Basic +from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.resolve import ( + Unavailable, + describe_proxy, + pick_proxy, + resolve_proxy, + split_proxy_query, +) + +pytestmark = pytest.mark.unit + +HINT = r"A city query needs a proxy provider prefix" + + +class Alpha(Proxy): + def __init__(self, **answers: str) -> None: + self.answers = answers + + def __repr__(self) -> str: + return "Alpha" + + def get_proxy(self, query: str) -> Optional[str]: + return self.answers.get(query) + + +class Beta(Alpha): + def last_connection_display(self) -> Optional[str]: + return "via Dallas (server 12)" + + +class Tunnel(Alpha): + def __init__(self, info: Optional[dict] = None, **answers: str) -> None: + super().__init__(**answers) + self.info = info + self.asked: list[str] = [] + + def get_connection_info(self, query: str) -> Optional[dict]: + self.asked.append(query) + return self.info + + +@pytest.fixture +def providers() -> list: + return [ + Unavailable("Gamma", RuntimeError("no route")), + Alpha(us="http://a:b@alpha.example:80"), + Beta(us="http://c:d@beta.example:80", ca="http://c:d@beta-ca.example:80"), + ] + + +@pytest.fixture(autouse=True) +def info_level() -> Iterator[None]: + """mask_proxy shows the full URI while DEBUG is active, so pin the level the masking tests expect.""" + root = logging.getLogger() + level = root.level + root.setLevel(logging.INFO) + yield + root.setLevel(level) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("us", (None, "us")), + ("alpha:us", ("alpha", "us")), + ("Beta:us:seattle", ("Beta", "us:seattle")), + ("http://user:pass@proxy.example:8080", (None, "http://user:pass@proxy.example:8080")), + ("socks5://proxy.example:1080", (None, "socks5://proxy.example:1080")), + ("localhost:8080", (None, "localhost:8080")), + ], +) +def test_split_proxy_query(providers: list, value: str, expected: tuple) -> None: + assert split_proxy_query(value, providers) == expected + + +@pytest.mark.parametrize("value", ["us:seattle", "us:new-york"]) +def test_a_city_query_without_a_provider_prefix_fails_with_a_hint(providers: list, value: str) -> None: + with pytest.raises(ValueError, match=rf"'us' was not found. Available: \['Alpha', 'Beta'\]. {HINT}"): + split_proxy_query(value, providers) + with pytest.raises(ValueError, match=HINT): + resolve_proxy(value, providers) + + +def test_a_prefix_naming_a_failed_provider_reports_why(providers: list) -> None: + with pytest.raises(ValueError, match="Gamma could not load: no route"): + split_proxy_query("gamma:us", providers) + + +def test_named_query_asks_only_that_provider(providers: list) -> None: + provider, uri = pick_proxy(providers, "ca", "beta") + assert isinstance(provider, Beta) and uri == "http://c:d@beta-ca.example:80" + + +def test_named_query_without_an_answer_raises(providers: list) -> None: + with pytest.raises(ValueError, match="The proxy provider alpha had no proxy for ca"): + pick_proxy(providers, "ca", "alpha") + + +def test_bare_query_takes_the_first_provider_that_answers(providers: list) -> None: + provider, uri = pick_proxy(providers, "ca") + assert isinstance(provider, Beta) and uri == "http://c:d@beta-ca.example:80" + provider, uri = pick_proxy(providers, "us") + assert type(provider) is Alpha and uri == "http://a:b@alpha.example:80" + + +def test_bare_query_without_an_answer_raises(providers: list) -> None: + with pytest.raises(ValueError, match="No proxy provider had a proxy for de"): + pick_proxy(providers, "de") + + +def test_describe_masks_the_credentials() -> None: + line = describe_proxy(Alpha(), "http://user:pass@alpha.example:80", "us") + assert line == "Using Alpha Proxy: http://xxxxx:xxxxx@alpha.example:80" + + +def test_describe_masks_the_host_of_a_basic_proxy() -> None: + line = describe_proxy(Basic(us="http://user:pass@mine.example:3128"), "http://user:pass@mine.example:3128", "us") + assert line == "Using Basic Proxy: http://xxxxx:xxxxx@xxxxx:3128" + + +def test_describe_prefers_the_provider_display() -> None: + assert describe_proxy(Beta(), "http://c:d@beta.example:80", "us") == "Using Beta Proxy via Dallas (server 12)" + + +def test_describe_reports_a_vpn_connection() -> None: + tunnel = Tunnel({"public_ip": "203.0.113.9", "city": "Toronto", "country": "Canada"}) + assert describe_proxy(tunnel, "http://127.0.0.1:8888", "alpha:ca") == "VPN Connected: 203.0.113.9 (Toronto, Canada)" + assert tunnel.asked == ["alpha:ca"] + + +def test_describe_falls_back_when_the_vpn_has_no_connection_info() -> None: + assert describe_proxy(Tunnel(None), "http://127.0.0.1:8888", "ca") == "Using Tunnel Proxy: http://127.0.0.1:8888" + + +def test_describe_shows_the_full_uri_in_debug_only_when_allowed() -> None: + logging.getLogger().setLevel(logging.DEBUG) + uri = "http://user:pass@alpha.example:80" + assert describe_proxy(Alpha(), uri, "us") == f"Using Alpha Proxy: {uri}" + assert "user:pass" not in describe_proxy(Alpha(), uri, "us", allow_debug=False) + + +@pytest.mark.parametrize( + "uri", ["http://user:pass@proxy.example:8080", "https://proxy.example:443", "socks5h://proxy.example:1080"] +) +def test_resolve_proxy_passes_a_uri_through(providers: list, uri: str) -> None: + assert resolve_proxy(uri, providers) == uri + + +def test_resolve_proxy_logs_the_provider_without_credentials(providers: list, caplog: pytest.LogCaptureFixture) -> None: + logging.getLogger().setLevel(logging.DEBUG) + with caplog.at_level(logging.INFO, logger="proxies"): + assert resolve_proxy("beta:ca", providers) == "http://c:d@beta-ca.example:80" + assert resolve_proxy("us", providers) == "http://a:b@alpha.example:80" + assert "Using Beta Proxy via Dallas (server 12)" in caplog.text + assert "Using Alpha Proxy: http://xxxxx:xxxxx@alpha.example:80" in caplog.text + + +def test_resolve_proxy_raises_when_no_provider_answers(providers: list) -> None: + with pytest.raises(ValueError, match="No proxy provider had a proxy for de"): + resolve_proxy("de", providers) + + +def test_bare_query_skips_a_provider_that_raises(caplog: pytest.LogCaptureFixture) -> None: + class Broken: + def get_proxy(self, query: str) -> str: + raise ConnectionError("api down") + + class Working: + def get_proxy(self, query: str) -> str: + return f"http://working/{query}" + + with caplog.at_level(logging.WARNING): + provider, uri = pick_proxy([Broken(), Working()], "us") + assert uri == "http://working/us" and isinstance(provider, Working) + assert "Broken proxy provider failed for us" in caplog.text + with pytest.raises(ValueError, match=r"No proxy provider had a proxy for us \(Broken: api down\)"): + pick_proxy([Broken()], "us") + + +def test_named_query_passes_the_provider_error_on() -> None: + class Broken: + def get_proxy(self, query: str) -> str: + raise ConnectionError("api down") + + with pytest.raises(ConnectionError, match="api down"): + pick_proxy([Broken()], "us", "broken") diff --git a/tests/core/test_proxy_loading.py b/tests/core/test_proxy_loading.py index d6505244..576046a6 100644 --- a/tests/core/test_proxy_loading.py +++ b/tests/core/test_proxy_loading.py @@ -61,3 +61,19 @@ def test_query_naming_an_unknown_provider_lists_the_loaded_ones() -> None: def test_raise_errors_raises_the_load_failure() -> None: with pytest.raises(requests.ConnectionError): load_proxy_providers(OTHERS, raise_errors=True) + + +def test_rest_path_loads_windscribe_but_not_gluetun(monkeypatch: pytest.MonkeyPatch) -> None: + from unshackle.core.proxies import Gluetun, WindscribeVPN + from unshackle.core.proxies.resolve import initialize_proxy_providers + + countries = [ + {"country_code": "US", "groups": [{"city": "Dallas", "hosts": [{"hostname": "us-central-1.example"}]}]} + ] + monkeypatch.setattr(config, "proxy_providers", {"windscribevpn": CREDENTIALS, "gluetun": {"windscribe": {}}}) + monkeypatch.setattr(WindscribeVPN, "get_countries", staticmethod(lambda: countries)) + + providers = initialize_proxy_providers(quiet=True) + assert any(isinstance(p, WindscribeVPN) for p in providers) + assert not any(isinstance(p, Gluetun) or getattr(p, "name", None) == "Gluetun" for p in providers) + assert resolve_proxy("windscribevpn:us", providers).endswith("@us-central-1.example:443") diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 59c1a6e4..b9558e14 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -53,8 +53,15 @@ from unshackle.core.events import events from unshackle.core.export_name import ExportNamer, move_export, plural, season_episodes, title_label from unshackle.core.providers.anilist import parse_anilist_ref from unshackle.core.providers.tvdb import SEASON_TYPES, parse_int -from unshackle.core.proxies import Basic -from unshackle.core.proxies.resolve import REGION, find_provider, is_loopback, load_proxy_providers, resolve_proxy +from unshackle.core.proxies.resolve import ( + REGION, + describe_proxy, + is_loopback, + load_proxy_providers, + pick_proxy, + resolve_proxy, + split_proxy_query, +) from unshackle.core.service import Service, grow_session_pool from unshackle.core.services import Services from unshackle.core.temp import with_task_temp @@ -1358,85 +1365,34 @@ class dl: if proxy and proxy.startswith("controld://"): proxy = ctx.params["proxy"] = resolve_proxy(proxy, []) if proxy: - - def log_proxy_used(provider: object, uri: str) -> None: - self.log.info( - f"Using {provider.__class__.__name__} Proxy: {mask_proxy(uri, isinstance(provider, Basic))}" - ) - - requested_provider = None - if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): - # requesting proxy from a specific proxy provider - requested_provider, proxy = proxy.split(":", maxsplit=1) + try: + requested_provider, query = split_proxy_query(proxy, self.proxy_providers) + except ValueError as e: + self.log.error(str(e)) + sys.exit(1) # Match region codes (us, ca, uk1, us:ny, yul) or provider:region (nordvpn:ca, protonvpn:us:ny). - if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", proxy, re.IGNORECASE): - proxy = proxy.lower() - # Preserve the original user query (region code) for service-specific proxy_map overrides. - # NOTE: `proxy` may be overwritten with the resolved proxy URI later. - proxy_query = proxy + if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", query, re.IGNORECASE): + # Kept apart from ctx.params["proxy"], which becomes the URI: a service proxy_map keys on it. + proxy_query = query.lower() status_msg = ( - f"Connecting to VPN ({proxy})..." + f"Connecting to VPN ({proxy_query})..." if requested_provider == "gluetun" - else f"Getting a Proxy to {proxy}..." + else f"Getting a Proxy to {proxy_query}..." ) with console.status(status_msg, spinner="dots"): - if requested_provider: - try: - proxy_provider = find_provider(self.proxy_providers, requested_provider) - except ValueError as e: - self.log.error(str(e)) - sys.exit(1) - proxy_uri = proxy_provider.get_proxy(proxy) - if not proxy_uri: - self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") - sys.exit(1) - proxy = ctx.params["proxy"] = proxy_uri - # Show connection info for Gluetun (IP, location) instead of proxy URL - if hasattr(proxy_provider, "get_connection_info"): - conn_info = proxy_provider.get_connection_info(proxy_query) - if conn_info and conn_info.get("public_ip"): - location_parts = [conn_info.get("city"), conn_info.get("country")] - location = ", ".join(p for p in location_parts if p) - self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") - else: - log_proxy_used(proxy_provider, proxy) - else: - display = None - if hasattr(proxy_provider, "last_connection_display"): - display = proxy_provider.last_connection_display() - if display: - self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy {display}") - else: - log_proxy_used(proxy_provider, proxy) - else: - for proxy_provider in self.proxy_providers: - proxy_uri = proxy_provider.get_proxy(proxy) - if proxy_uri: - proxy = ctx.params["proxy"] = proxy_uri - # Show connection info for Gluetun (IP, location) instead of proxy URL - if hasattr(proxy_provider, "get_connection_info"): - conn_info = proxy_provider.get_connection_info(proxy_query) - if conn_info and conn_info.get("public_ip"): - location_parts = [conn_info.get("city"), conn_info.get("country")] - location = ", ".join(p for p in location_parts if p) - self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") - else: - log_proxy_used(proxy_provider, proxy) - else: - display = None - if hasattr(proxy_provider, "last_connection_display"): - display = proxy_provider.last_connection_display() - if display: - self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy {display}") - else: - log_proxy_used(proxy_provider, proxy) - break - # Store proxy query info for service-specific overrides + try: + proxy_provider, proxy_uri = pick_proxy( + self.proxy_providers, proxy_query, requested_provider + ) + except ValueError as e: + self.log.error(str(e)) + sys.exit(1) + ctx.params["proxy"] = proxy_uri + self.log.info(describe_proxy(proxy_provider, proxy_uri, proxy_query)) ctx.params["proxy_query"] = proxy_query ctx.params["proxy_provider"] = requested_provider else: self.log.info(f"Using explicit Proxy: {mask_proxy(ctx.params['proxy'])}") - # For explicit proxies, store None for query/provider ctx.params["proxy_query"] = None ctx.params["proxy_provider"] = None diff --git a/unshackle/commands/search.py b/unshackle/commands/search.py index 943fa903..d46dd3e7 100644 --- a/unshackle/commands/search.py +++ b/unshackle/commands/search.py @@ -15,9 +15,8 @@ from unshackle.commands.dl import dl from unshackle.core.config import config from unshackle.core.console import console from unshackle.core.constants import context_settings -from unshackle.core.proxies import Basic from unshackle.core.proxies.proxy import Proxy -from unshackle.core.proxies.resolve import REGION, find_provider, load_proxy_providers +from unshackle.core.proxies.resolve import REGION, describe_proxy, load_proxy_providers, pick_proxy, split_proxy_query from unshackle.core.service import Service from unshackle.core.services import Services from unshackle.core.utils.click_types import ContextData @@ -70,49 +69,21 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr proxy_providers = load_proxy_providers() if proxy: - requested_provider = None - if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): - requested_provider, proxy = proxy.split(":", maxsplit=1) - if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", proxy, re.IGNORECASE): - proxy = proxy.lower() - with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"): - if requested_provider: - try: - proxy_provider = find_provider(proxy_providers, requested_provider) - except ValueError as e: - log.error(str(e)) - sys.exit(1) - proxy_uri = proxy_provider.get_proxy(proxy) - if not proxy_uri: - log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") - sys.exit(1) - proxy = ctx.params["proxy"] = proxy_uri - display = None - if hasattr(proxy_provider, "last_connection_display"): - display = proxy_provider.last_connection_display() - if display: - log.info(f"Using {proxy_provider.__class__.__name__} Proxy {display}") - else: - log.info( - f"Using {proxy_provider.__class__.__name__} Proxy: " - f"{mask_proxy(proxy, isinstance(proxy_provider, Basic))}" - ) - else: - for proxy_provider in proxy_providers: - proxy_uri = proxy_provider.get_proxy(proxy) - if proxy_uri: - proxy = ctx.params["proxy"] = proxy_uri - display = None - if hasattr(proxy_provider, "last_connection_display"): - display = proxy_provider.last_connection_display() - if display: - log.info(f"Using {proxy_provider.__class__.__name__} Proxy {display}") - else: - log.info( - f"Using {proxy_provider.__class__.__name__} Proxy: " - f"{mask_proxy(proxy, isinstance(proxy_provider, Basic))}" - ) - break + try: + requested_provider, query = split_proxy_query(proxy, proxy_providers) + except ValueError as e: + log.error(str(e)) + sys.exit(1) + if re.fullmatch(rf"(?:[a-z]+:)?{REGION}", query, re.IGNORECASE): + query = query.lower() + with console.status(f"Getting a Proxy to {query}...", spinner="dots"): + try: + proxy_provider, proxy_uri = pick_proxy(proxy_providers, query, requested_provider) + except ValueError as e: + log.error(str(e)) + sys.exit(1) + ctx.params["proxy"] = proxy_uri + log.info(describe_proxy(proxy_provider, proxy_uri, query)) else: log.info(f"Using explicit Proxy: {mask_proxy(ctx.params['proxy'])}") diff --git a/unshackle/core/proxies/resolve.py b/unshackle/core/proxies/resolve.py index 100afb38..cf43e48f 100644 --- a/unshackle/core/proxies/resolve.py +++ b/unshackle/core/proxies/resolve.py @@ -31,11 +31,12 @@ class Unavailable(Proxy): return None -def find_provider(proxy_providers: List[Any], name: str) -> Any: +def find_provider(proxy_providers: List[Any], name: str, hint: str = "") -> Any: """ Find a proxy provider by its class name, case-insensitive. Raises ValueError when no proxy provider has that name, or when it failed to load. + *hint* follows the message when no proxy provider has that name. """ for provider in proxy_providers: if isinstance(provider, Unavailable): @@ -44,7 +45,8 @@ def find_provider(proxy_providers: List[Any], name: str) -> Any: elif provider.__class__.__name__.lower() == name.lower(): return provider available = [x.__class__.__name__ for x in proxy_providers if not isinstance(x, Unavailable)] - raise ValueError(f"The proxy provider '{name}' was not found. Available: {available}") + message = f"The proxy provider '{name}' was not found. Available: {available}" + raise ValueError(f"{message}. {hint}" if hint else message) def load_proxy_providers(exclude: tuple[str, ...] = (), raise_errors: bool = False, quiet: bool = False) -> List[Any]: @@ -109,16 +111,84 @@ def initialize_proxy_providers(raise_errors: bool = False, quiet: bool = False) """ Load the proxy providers for the REST API and the remote service client. - WindscribeVPN and Gluetun are not offered on this path. *quiet* drops the per-provider summary - lines: rendering a proxy provider asks some of them for their server catalogue over the network, - which a repeated caller must not pay for. + Gluetun is not offered on this path, because it starts a local Docker container. *quiet* drops the + per-provider summary lines: rendering a proxy provider asks some of them for their server catalogue + over the network, which a repeated caller must not pay for. """ - proxy_providers = load_proxy_providers(("windscribevpn", "gluetun"), raise_errors, quiet) + proxy_providers = load_proxy_providers(("gluetun",), raise_errors, quiet) if not quiet and not proxy_providers: log.warning("No proxy providers were loaded. Check your proxy provider configuration in unshackle.yaml") return proxy_providers +CITY_HINT = "A city query needs a proxy provider prefix, for example nordvpn:us:seattle." + + +def split_proxy_query(proxy: str, proxy_providers: List[Any]) -> tuple[Optional[str], str]: + """ + Split a proxy value into the proxy provider that its prefix names and the query for it. + + A URI, a bare query and a ``host:port`` name no proxy provider. Raises ValueError when the prefix + is not a loaded proxy provider, so a city query without a proxy provider prefix fails here. + """ + if re.match(r"^[a-z][a-z0-9+.-]*://", proxy, re.IGNORECASE) or not re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + return None, proxy + name, query = proxy.split(":", maxsplit=1) + if query.isdigit(): + return None, proxy + find_provider(proxy_providers, name, hint=CITY_HINT) + return name, query + + +def pick_proxy(proxy_providers: List[Any], query: str, provider_name: Optional[str] = None) -> tuple[Any, str]: + """ + Get a proxy provider and its proxy URI: from the named one, or else from the first one that answers. + + A named proxy provider's error goes to the caller. In a bare query, a proxy provider that fails is + logged and skipped, so a later one can still answer. Raises ValueError when no proxy provider answers. + """ + if provider_name: + provider = find_provider(proxy_providers, provider_name) + uri = provider.get_proxy(query) + if not uri: + raise ValueError(f"The proxy provider {provider_name} had no proxy for {query}") + return provider, uri + errors = [] + for provider in proxy_providers: + name = provider.__class__.__name__ + try: + uri = provider.get_proxy(query) + except Exception as e: + log.warning(f"The {name} proxy provider failed for {query}, so the next one is tried: {e}") + errors.append(f"{name}: {e}") + continue + if uri: + return provider, uri + detail = f" ({'; '.join(errors)})" if errors else "" + raise ValueError(f"No proxy provider had a proxy for {query}{detail}") + + +def describe_proxy(provider: Any, uri: str, query: str, allow_debug: bool = True) -> str: + """ + Get the log line that names the proxy provider that answered, with the proxy credentials masked. + + Set *allow_debug* to False where the line can reach more than the operator's own terminal. + """ + from unshackle.core.proxies.basic import Basic + + name = provider.__class__.__name__ + if hasattr(provider, "get_connection_info"): + info = provider.get_connection_info(query) + if info and info.get("public_ip"): + location = ", ".join(x for x in (info.get("city"), info.get("country")) if x) + return f"VPN Connected: {info['public_ip']} ({location})" + elif hasattr(provider, "last_connection_display"): + display = provider.last_connection_display() + if display: + return f"Using {name} Proxy {display}" + return f"Using {name} Proxy: {mask_proxy(uri, isinstance(provider, Basic), allow_debug)}" + + def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]: """Change a proxy parameter to a real proxy URI. @@ -139,32 +209,10 @@ def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]: if re.match(r"^(https?://|socks)", proxy): return proxy - requested_provider = None - query = proxy - if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): - requested_provider, query = proxy.split(":", maxsplit=1) - - if requested_provider: - provider = find_provider(proxy_providers, requested_provider) - proxy_uri = provider.get_proxy(query) - if not proxy_uri: - raise ValueError(f"Proxy provider {requested_provider} had no proxy for {query}") - log.info( - f"Using {provider.__class__.__name__} Proxy: " - f"{mask_proxy(proxy_uri, provider.__class__.__name__ == 'Basic', allow_debug=False)}" - ) - return proxy_uri - - for provider in proxy_providers: - proxy_uri = provider.get_proxy(query) - if proxy_uri: - log.info( - f"Using {provider.__class__.__name__} Proxy: " - f"{mask_proxy(proxy_uri, provider.__class__.__name__ == 'Basic', allow_debug=False)}" - ) - return proxy_uri - - raise ValueError(f"No proxy provider had a proxy for {proxy}") + provider_name, query = split_proxy_query(proxy, proxy_providers) + provider, uri = pick_proxy(proxy_providers, query, provider_name) + log.info(describe_proxy(provider, uri, query, allow_debug=False)) + return uri def is_loopback(uri: str) -> bool: