fix(proxy): check the proxy exit IP before any service request

A dead proxy, or one that exits from this machine's IP, now fails with INVALID_PROXY instead of retrying every request; remote session create also builds the service off the event loop so a stuck check cannot block the server.

docs/dev/rest-api/errors.md

@@ -96,7 +96,7 @@ but the defaults hold for the overwhelming majority of responses.
 |---|---|---|
 | `INVALID_INPUT` | 400 | A required field is missing, or the request body could not be parsed. Also used as the generic "your input was malformed" code. |
 | `INVALID_SERVICE` | 400 | The named service is unknown, or is not permitted for your API key. |
-| `INVALID_PROXY` | 400 | The supplied proxy specification could not be resolved or is malformed. |
+| `INVALID_PROXY` | 400 | The supplied proxy specification could not be resolved or is malformed, or the proxy failed the [exit check](../../guide/proxies-and-vpn.md#exit-check). |
 | `INVALID_PARAMETERS` | 400 | One or more download or query parameters failed validation (bad codec, bitrate, sort field, and so on). |
 | `AUTH_FAILED` | 401 | Authentication with the streaming service failed (bad credentials or cookies). |
 | `FORBIDDEN` | 403 | The action is not allowed. Raised by server-side gates (per-key restrictions on CDM or credential overrides) and by remote session IP binding. |

docs/guide/proxies-and-vpn.md

@@ -123,6 +123,21 @@ where you came out:
   `(USA - New York, #3 of 5): .214`.
 - Other proxy providers log the resolved proxy URL.
 
+### Exit check
+
+Before the service sends a request, unshackle looks up the proxy exit IP through the
+proxy. The service stops with `Proxy check failed` in two cases:
+
+- No IP lookup gets through the proxy. The proxy is down, or it refused the connection.
+- The proxy exit IP is the same as your own IP, so traffic does not go through the proxy.
+
+unshackle tries each geolocation API in turn, and a 429 moves on to the next one. If at
+least one API answers through the proxy but none of them gives an IP, the proxy passes
+the check without the IP comparison. unshackle does not compare the exit country with the region you asked for,
+because geolocation data can be out of date.
+
+Over the REST API, the server returns this failure to the client as `INVALID_PROXY`.
+
 ## Basic (static proxies)
 
 The `Basic` proxy provider is pure static configuration, with no accounts and no network

tests/core/test_verify_proxy_exit.py

@@ -0,0 +1,82 @@
+from types import SimpleNamespace
+
+import pytest
+
+from unshackle.core.utils import ip_info
+
+SERVER = {"ip": "1.1.1.1", "country": "gb"}
+
+
+def patch_lookups(monkeypatch, proxied, server=SERVER, live_server=None, proxy_errors=0):
+    """Answer the proxied lookup, the cached server lookup and the live server lookup."""
+    live_server = live_server if live_server is not None else server
+
+    def fake(session=None, cached=False, errors=None):
+        if cached:
+            return server
+        if session is None:
+            return live_server
+        if errors is not None:
+            errors.extend(OSError("refused") for _ in range(proxy_errors))
+        return proxied
+
+    monkeypatch.setattr(ip_info, "get_ip_info", fake)
+
+
+def test_dead_proxy_raises(monkeypatch):
+    patch_lookups(monkeypatch, None, proxy_errors=len(ip_info.build_providers()))
+    with pytest.raises(ConnectionError, match=r"no IP lookup got through the proxy \(OSError\)"):
+        ip_info.verify_proxy_exit(object())
+
+
+def test_rate_limited_providers_do_not_reject(monkeypatch):
+    patch_lookups(monkeypatch, None, proxy_errors=len(ip_info.build_providers()) - 1)
+    assert ip_info.verify_proxy_exit(object()) == {}
+
+
+def test_exit_ip_equal_to_server_raises(monkeypatch):
+    patch_lookups(monkeypatch, {"ip": "1.1.1.1", "country": "gb"})
+    with pytest.raises(ConnectionError, match="proxy exit IP is the IP of this machine"):
+        ip_info.verify_proxy_exit(object())
+
+
+def test_stale_cached_server_ip_does_not_reject(monkeypatch):
+    patch_lookups(monkeypatch, {"ip": "1.1.1.1", "country": "gb"}, live_server={"ip": "3.3.3.3", "country": "gb"})
+    assert ip_info.verify_proxy_exit(object())["country"] == "gb"
+
+
+def test_distinct_exit_ip_passes(monkeypatch):
+    patch_lookups(monkeypatch, {"ip": "2.2.2.2", "country": "us"})
+    assert ip_info.verify_proxy_exit(object())["country"] == "us"
+
+
+def test_unknown_server_ip_skips_comparison(monkeypatch):
+    patch_lookups(monkeypatch, {"ip": "1.1.1.1", "country": "us"}, server=None)
+    assert ip_info.verify_proxy_exit(object())["country"] == "us"
+
+
+def test_proxied_lookup_ignores_env_proxy(monkeypatch):
+    monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:1")
+    sess = ip_info.lookup_session(SimpleNamespace(proxies={"all": "http://127.0.0.1:2"}))
+    settings = sess.merge_environment_settings("https://ipinfo.io/json", {}, None, None, None)
+    assert settings["proxies"]["https"] == "http://127.0.0.1:2"
+
+
+def test_local_lookup_keeps_env_proxy(monkeypatch):
+    monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:1")
+    sess = ip_info.lookup_session(None)
+    settings = sess.merge_environment_settings("https://ipinfo.io/json", {}, None, None, None)
+    assert settings["proxies"]["https"] == "http://127.0.0.1:1"
+
+
+def test_get_ip_info_collects_connection_errors_only(monkeypatch):
+    def refused(session):
+        raise OSError("refused")
+
+    def no_usable_data(session):
+        return None
+
+    monkeypatch.setattr(ip_info, "build_providers", lambda: [("a", refused), ("b", no_usable_data)])
+    errors: list[Exception] = []
+    assert ip_info.get_ip_info(SimpleNamespace(proxies={"all": "http://127.0.0.1:9"}), errors=errors) is None
+    assert [str(e) for e in errors] == ["refused"]

tests/remote/unit/test_handler_proxy_policy.py

@@ -179,6 +179,25 @@ def test_setup_list_service_threads_request(monkeypatch):
     assert seen["authenticated"]
 
 
+def test_search_reports_failed_proxy_check_as_invalid_proxy(monkeypatch):
+    def failed_check(*args):
+        raise ConnectionError("Proxy check failed: no IP lookup got through the proxy.")
+
+    monkeypatch.setattr(handlers, "load_service_yaml", lambda service: {})
+    monkeypatch.setattr(handlers, "resolve_handler_proxy", lambda *args: ("http://127.0.0.1:9", []))
+    monkeypatch.setattr(handlers, "load_full_cdm", lambda *args: None)
+    monkeypatch.setattr(handlers, "build_parent_ctx", lambda *args, **kwargs: None)
+    monkeypatch.setattr(handlers.Services, "load", lambda service: None)
+    monkeypatch.setattr(handlers, "instantiate_service", failed_check)
+    monkeypatch.setattr("unshackle.commands.dl.dl.get_cookie_jar", staticmethod(lambda service, profile: None))
+    monkeypatch.setattr("unshackle.commands.dl.dl.get_credentials", staticmethod(lambda service, profile: None))
+
+    with pytest.raises(APIError) as exc_info:
+        handlers.run_service_search({}, "EXAMPLE", "query", request("plainkey"))
+    assert exc_info.value.error_code == APIErrorCode.INVALID_PROXY
+    assert exc_info.value.retryable is False
+
+
 class TestDownloadGates:
     """/api/download submissions and retries obey the same proxy policy as sessions."""
 
@@ -288,3 +307,35 @@ class TestDownloadEnforcement:
         with pytest.raises(Stop):
             dm.perform_download("job1", "EXAMPLE", "t1", dict(params))
         assert seen["proxy_providers"] == expected
+
+
+def test_session_create_builds_service_off_the_event_loop(monkeypatch):
+    import asyncio
+    import json
+    import logging
+    import threading
+
+    seen: dict = {}
+
+    class RecordingBuffer(handlers.SessionLogBuffer):
+        def __init__(self):
+            super().__init__()
+            seen["buffer"] = self
+
+    def failed_check(*args, **kwargs):
+        seen["thread"] = threading.get_ident()
+        logging.getLogger("SlowProxySvc").warning("checking proxy")
+        raise ConnectionError("Proxy check failed: no IP lookup got through the proxy.")
+
+    monkeypatch.setattr(handlers, "validate_service", lambda service, request=None: service)
+    monkeypatch.setattr(handlers, "resolve_handler_proxy", lambda *args: ("http://127.0.0.1:9", []))
+    monkeypatch.setattr(handlers, "server_account_for", lambda *args: False)
+    monkeypatch.setattr(handlers.Services, "load", lambda service: type("SlowProxySvc", (), {}))
+    monkeypatch.setattr(handlers, "SessionLogBuffer", RecordingBuffer)
+    monkeypatch.setattr(handlers, "create_service_instance", failed_check)
+
+    response = asyncio.run(handlers.session_create_handler({"service": "EXAMPLE", "title_id": "t"}))
+
+    assert seen["thread"] != threading.get_ident()
+    assert [r["message"] for r in seen["buffer"].since(0)] == ["checking proxy"]
+    assert json.loads(response.body)["error_code"] == "INVALID_PROXY"

unshackle/core/api/handlers.py

@@ -18,7 +18,7 @@ from typing import Any, Dict, Iterable, List, Optional
 from aiohttp import web
 
 from unshackle.core.api.compression import safe_inflate
-from unshackle.core.api.errors import APIError, APIErrorCode, handle_api_exception
+from unshackle.core.api.errors import APIError, APIErrorCode, categorize_exception, handle_api_exception
 from unshackle.core.api.input_bridge import AuthStatus, InputBridge
 from unshackle.core.api.sanitize import safe_cache_key, sanitize_log
 from unshackle.core.api.session_log import SessionLogBuffer, SessionLogMirror, capture_service_logs
@@ -358,6 +358,8 @@ def run_service_search(
 
     try:
         service_instance = instantiate_service(parent_ctx, service_module, query)
+    except ConnectionError as exc:
+        raise categorize_exception(exc, {"service": normalized_service}) from exc
     except Exception as exc:
         raise APIError(
             APIErrorCode.SERVICE_ERROR,
@@ -2947,22 +2949,26 @@ async def session_create_handler(data: Dict[str, Any], request: Optional[web.Req
             if region is not None and not (isinstance(region, str) and re.fullmatch(r"[A-Za-z]{2}", region)):
                 raise APIError(APIErrorCode.INVALID_INPUT, "proxy_region must be a two-letter country code.")
             if region is None and data.get("no_proxy"):
-                region = server_region()
+                region = await asyncio.to_thread(server_region)
             profile = next_server_profile(normalized_service, region)
             log.info(f"Using server account '{sanitize_log(profile or 'default')}' for {normalized_service}")
 
         log_buffer = None if server_account else SessionLogBuffer()
         service_class_name = getattr(Services.load(normalized_service), "__name__", normalized_service)
-        with capture_service_logs(service_class_name, log_buffer):
-            service_instance, cookies, credential = create_service_instance(
-                normalized_service,
-                title_id,
-                data,
-                proxy_param,
-                proxy_providers,
-                profile,
-                server_account=server_account,
-            )
+
+        def build_service() -> Any:
+            with capture_service_logs(service_class_name, log_buffer):
+                return create_service_instance(
+                    normalized_service,
+                    title_id,
+                    data,
+                    proxy_param,
+                    proxy_providers,
+                    profile,
+                    server_account=server_account,
+                )
+
+        service_instance, cookies, credential = await asyncio.to_thread(build_service)
         if log_buffer:
             service_instance.log = SessionLogMirror(service_instance.log, log_buffer)
 

unshackle/core/service.py

@@ -34,12 +34,12 @@ from unshackle.core.session import (
     RETRY_METHODS,
     STATUS_FORCELIST,
 )
-from unshackle.core.title_cacher import TitleCacher, get_account_hash, get_region_from_proxy
+from unshackle.core.title_cacher import TitleCacher, get_account_hash
 from unshackle.core.titles import Title_T, Titles_T, remap_titles
 from unshackle.core.tracks import Chapters, Tracks
 from unshackle.core.tracks.video import Video
 from unshackle.core.utilities import declared_kwargs
-from unshackle.core.utils.ip_info import get_ip_info
+from unshackle.core.utils.ip_info import get_ip_info, verify_proxy_exit
 from unshackle.core.utils.redact import mask_proxy
 
 # Default (connect, read) timeout for the requests path, mirroring RnetSession's
@@ -280,13 +280,9 @@ class Service(metaclass=ABCMeta):
                 # requests authenticate from the credentials embedded in the proxy URL.
                 # A manual header here was malformed (no "Basic " scheme) and broke
                 # plaintext-http forward-proxy requests with HTTP 407.
-                # Always verify proxy IP - proxies can change exit nodes
-                try:
-                    proxy_ip_info = get_ip_info(self.session)
-                    self.current_region = proxy_ip_info.get("country", "").lower() if proxy_ip_info else None
-                except Exception as e:
-                    self.log.warning(f"Failed to verify proxy IP: {e}")
-                    self.current_region = get_region_from_proxy(proxy)
+                # Verify the proxy IP every time, because a proxy can change its exit node. A dead
+                # proxy fails here, not after every service request has used up its retries.
+                self.current_region = verify_proxy_exit(self.session).get("country")
             else:
                 # No proxy, use cached IP info for title caching (non-critical)
                 try:

unshackle/core/title_cacher.py

@@ -265,36 +265,6 @@ class TitleCacher:
         self.log.debug(f"Cached {provider_name} data for {title_id}")
 
 
-def get_region_from_proxy(proxy_url: Optional[str]) -> Optional[str]:
-    """
-    Extract region identifier from proxy URL.
-
-    Args:
-        proxy_url: The proxy URL string
-
-    Returns:
-        Region identifier or None
-    """
-    if not proxy_url:
-        return None
-
-    import re
-
-    nord_match = re.search(r"([a-z]{2})\d+\.nordvpn", proxy_url.lower())
-    if nord_match:
-        return nord_match.group(1)
-
-    cc_match = re.search(r"([a-z]{2})[-_]", proxy_url.lower())
-    if cc_match:
-        return cc_match.group(1)
-
-    subdomain_match = re.search(r"://([a-z]{2})\.", proxy_url.lower())
-    if subdomain_match:
-        return subdomain_match.group(1)
-
-    return None
-
-
 def get_account_hash(credential) -> Optional[str]:
     """
     Make a hash for account identification.

unshackle/core/utils/ip_info.py

@@ -8,14 +8,15 @@ import requests
 
 from unshackle.core.cacher import Cacher
 
-CACHE_KEY = "ip_info_v3"
+CACHE_KEY = "ip_info_v4"
 CACHE_TTL = 86400  # 24 hours
 PROVIDER_STATE_KEY = "ip_provider_state"
 RATE_LIMIT_COOLDOWN = 300  # 5 minutes
 REQUEST_TIMEOUT = 10
 
-# Only these keys are persisted to the global cache.
-GEO_CACHE_KEYS = ("country", "country_code")
+# Only these keys are persisted to the global cache. "ip" is the local IP, which
+# verify_proxy_exit compares against the proxy exit IP.
+GEO_CACHE_KEYS = ("ip", "country", "country_code")
 
 Fetcher = Callable[[requests.Session], Optional[dict]]
 
@@ -108,6 +109,7 @@ def lookup_session(source: Optional[requests.Session]) -> requests.Session:
     if proxies:
         proxy = proxies.get("all") or proxies.get("https") or proxies.get("http")
         if proxy:
+            sess.trust_env = False
             sess.proxies.update({"http": proxy, "https": proxy})
     return sess
 
@@ -184,6 +186,7 @@ def get_ip_info(
     session: Optional[requests.Session] = None,
     *,
     cached: bool = False,
+    errors: Optional[list[Exception]] = None,
 ) -> Optional[dict]:
     """
     Look up IP/geolocation info through ipinfo.io (Lite when `ipinfo_api_key` configured)
@@ -191,7 +194,7 @@ def get_ip_info(
 
     Live lookups return a dict with `ip`, `country` (lowercase ISO2), `country_code`
     (uppercase ISO2), `region`, `city`, `org`, `asn`, `as_name`, `continent_code` and
-    `_provider`. Cached lookups return only `country`/`country_code` (see GEO_CACHE_KEYS).
+    `_provider`. Cached lookups return only `ip`/`country`/`country_code` (see GEO_CACHE_KEYS).
     Returns None if every geolocation API fails.
 
     Args:
@@ -201,6 +204,8 @@ def get_ip_info(
             session.headers.
         cached: When True, read/write a 24h Cacher-backed entry. Use only for
             local IP lookups, never with a proxied HTTP session.
+        errors: When given, each geolocation API that raised an exception adds it to
+            this list. A 429 or an unusable response adds nothing.
     """
     cache = None
     if cached:
@@ -233,6 +238,8 @@ def get_ip_info(
             continue
         except Exception as e:
             log.debug(f"Provider {name} failed with exception: {e}")
+            if errors is not None:
+                errors.append(e)
             continue
 
         if not normalized:
@@ -252,3 +259,36 @@ def get_ip_info(
 
     log.warning("All IP geolocation providers failed")
     return None
+
+
+def verify_proxy_exit(session: requests.Session) -> dict:
+    """
+    Look up the exit IP of the proxy on `session`, and check that it is not the local IP.
+
+    Returns the live IP info of the proxy exit, or an empty dict when geolocation APIs
+    answered through the proxy (a 429, or no usable data) but none returned an IP. Raises
+    ConnectionError when no geolocation API got a response through the proxy, or
+    when the proxy exit IP is the local IP. This function does not check the exit country.
+    The local IP comes from the 24h cached local lookup, and a live local lookup must
+    confirm a match before this function rejects the proxy. If the local IP is not known,
+    the IP comparison does not occur.
+    """
+    errors: list[Exception] = []
+    info = get_ip_info(session, errors=errors)
+    if not info:
+        if len(errors) < len(build_providers()):
+            log.warning("No geolocation API returned the proxy exit IP, so the exit IP check does not occur")
+            return {}
+        raise ConnectionError(
+            f"Proxy check failed: no IP lookup got through the proxy ({type(errors[-1]).__name__}). "
+            "The proxy is down or refused the connection."
+        )
+    exit_ip = info.get("ip")
+    server_ip = (get_ip_info(cached=True) or {}).get("ip")
+    # The cached IP can be up to 24h old: a machine that was on a VPN when it cached its IP,
+    # and now uses that VPN as a proxy, matches the old IP. A live lookup runs only on a match.
+    if exit_ip and server_ip == exit_ip and (get_ip_info() or {}).get("ip") == exit_ip:
+        raise ConnectionError(
+            "Proxy check failed: the proxy exit IP is the IP of this machine, so traffic does not go through the proxy."
+        )
+    return info