"""The reverse tunnel: a server-side SOCKS5 proxy whose connections the client dials and relays.""" from __future__ import annotations import asyncio import http.server import socket import socketserver import threading import time from typing import Callable, Iterator import aiohttp import click import pytest import requests import responses from aiohttp import web from unshackle.core import tunnel from unshackle.core.api.routes import tunnel as tunnel_route from unshackle.core.api.routes import tunnel_check from unshackle.core.tunnel import ( CHECK_ROUTE, OPEN, ROUTE, TUNNELS, TunnelClient, connect_public, frame, is_ip_literal, is_local_ip, is_public_ip, tunnel_proxy, ) class Hello(http.server.BaseHTTPRequestHandler): hits = 0 def do_GET(self) -> None: Hello.hits += 1 body = b"hello through the tunnel" self.send_response(200) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *_: object) -> None: pass @pytest.fixture def target() -> Iterator[int]: server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Hello) threading.Thread(target=server.serve_forever, daemon=True).start() yield server.server_address[1] server.shutdown() @pytest.fixture def server() -> Iterator[str]: loop = asyncio.new_event_loop() app = web.Application() app.router.add_get(ROUTE, tunnel_route) app.router.add_get(CHECK_ROUTE, tunnel_check) runner = web.AppRunner(app) loop.run_until_complete(runner.setup()) site = web.TCPSite(runner, "127.0.0.1", 0) loop.run_until_complete(site.start()) port = site._server.sockets[0].getsockname()[1] # type: ignore[union-attr] threading.Thread(target=loop.run_forever, daemon=True).start() yield f"http://127.0.0.1:{port}" asyncio.run_coroutine_threadsafe(tunnel.close_tunnels(app), loop).result(5) loop.call_soon_threadsafe(loop.stop) def test_requests_through_the_tunnel_reach_the_client_side_target( server: str, target: int, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(tunnel, "is_public_ip", lambda address: True) # the test target is on loopback monkeypatch.setattr(tunnel, "dialled_itself", lambda writer: False) client = TunnelClient(server, {"X-Secret-Key": "alice"}) tunnel_id = client.start() proxy = tunnel_proxy(tunnel_id, "alice") assert proxy and proxy.startswith(f"socks5h://{tunnel_id}:") and "@127.0.0.1:" in proxy for _ in range(3): resp = requests.get(f"http://127.0.0.1:{target}/", proxies={"all": proxy}, timeout=10) assert resp.text == "hello through the tunnel" client.stop() def test_a_tunnel_is_only_usable_by_its_owner(server: str) -> None: client = TunnelClient(server, {"X-Secret-Key": "alice"}) tunnel_id = client.start() assert tunnel_proxy(tunnel_id, "mallory") is None assert tunnel_proxy("unknown", "alice") is None client.stop() def test_the_client_refuses_a_target_on_its_own_network(server: str, target: int) -> None: client = TunnelClient(server, {"X-Secret-Key": "alice"}) proxy = tunnel_proxy(client.start(), "alice") with pytest.raises(requests.exceptions.ConnectionError): requests.get(f"http://127.0.0.1:{target}/", proxies={"all": proxy}, timeout=10) client.stop() @pytest.mark.parametrize( ("address", "public"), [ ("8.8.8.8", True), ("127.0.0.1", False), ("192.168.1.10", False), ("10.0.0.1", False), ("::1", False), ("198.18.2.36", False), ("64:ff9b::c0a8:101", False), ], ) def test_is_public_ip(address: str, public: bool) -> None: assert is_public_ip(address) is public @pytest.mark.parametrize( ("address", "local"), [ ("8.8.8.8", False), ("198.18.2.36", False), ("2001:2::1", False), ("192.0.0.69", False), ("192.0.0.88", False), ("127.0.0.1", True), ("10.0.0.1", True), ("172.16.0.1", True), ("192.168.1.1", True), ("169.254.169.254", True), ("100.64.0.1", True), ("192.0.0.1", True), ("0.0.0.0", True), ("224.0.0.1", True), ("::1", True), ("fdfe:dcba:9876::1", True), ("fe80::1%eth0", True), ("::ffff:192.168.1.1", True), ("64:ff9b::c0a8:101", True), ("64:ff9b::808:808", False), ("::", True), ("::7f00:1", True), ("::ffff:0:7f00:1", True), ("64:ff9b:1::c0a8:101", True), ("192.0.0.9", True), ("192.0.0.10", True), ("not-an-address", True), ], ) def test_is_local_ip(address: str, local: bool) -> None: assert is_local_ip(address) is local @pytest.mark.parametrize("address", ["198.18.2.36", "192.0.0.69", "2001:2::1", "8.8.8.8"]) def test_a_name_that_resolves_to_a_non_local_address_is_dialled(monkeypatch: pytest.MonkeyPatch, address: str) -> None: async def resolve(self: object, host: str, port: int, **_: object) -> list: return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, port))] dialled: list[str] = [] async def open_connection(address: str, port: int) -> tuple: dialled.append(address) return None, FakeWriter(("10.0.0.2", 50000), (address, port)) monkeypatch.setattr(asyncio.BaseEventLoop, "getaddrinfo", resolve) monkeypatch.setattr(asyncio, "open_connection", open_connection) asyncio.run(connect_public("api.example", 443)) assert dialled == [address] class FakeWriter: def __init__(self, sockname: tuple, peername: tuple) -> None: self.extra = {"sockname": sockname, "peername": peername} self.closed = False def get_extra_info(self, name: str) -> tuple: return self.extra[name] def close(self) -> None: self.closed = True @pytest.mark.parametrize( ("host", "literal"), [ ("8.8.8.8", True), ("::1", True), ("198.18.1", True), ("3323068417", True), ("0x7f.1", True), ("api.example", False), ], ) def test_is_ip_literal(host: str, literal: bool) -> None: assert is_ip_literal(host) is literal def test_a_dial_that_reaches_the_client_itself_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: async def resolve(self: object, host: str, port: int, **_: object) -> list: return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("198.18.0.1", port))] writer = FakeWriter(("198.18.0.1", 50000), ("198.18.0.1", 443)) async def open_connection(address: str, port: int) -> tuple: return None, writer monkeypatch.setattr(asyncio.BaseEventLoop, "getaddrinfo", resolve) monkeypatch.setattr(asyncio, "open_connection", open_connection) with pytest.raises(PermissionError, match="this machine"): asyncio.run(connect_public("tun.example", 443)) assert writer.closed def test_localhost_is_refused() -> None: with pytest.raises(PermissionError): asyncio.run(connect_public("localhost", 443)) @pytest.mark.parametrize( ("host", "address"), [ ("rebind.example", "192.168.1.1"), ("metadata.example", "169.254.169.254"), ("nat64.example", "64:ff9b::c0a8:101"), ("198.18.0.1", "198.18.0.1"), ("192.0.0.69", "192.0.0.69"), ("198.18.1", "198.18.0.1"), ("3221225541", "192.0.0.69"), ], ) def test_a_target_that_resolves_to_a_local_address_is_refused( monkeypatch: pytest.MonkeyPatch, host: str, address: str ) -> None: async def resolve(self: object, host: str, port: int, **_: object) -> list: return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, port))] monkeypatch.setattr(asyncio.BaseEventLoop, "getaddrinfo", resolve) with pytest.raises(PermissionError): asyncio.run(connect_public(host, 443)) def local_proxy_ctx(params: dict, typed: set) -> click.Context: from click.core import ParameterSource ctx = click.Context(click.Command("dl")) ctx.params = {"local_proxy": True, **params} for key in params: ctx.set_parameter_source(key, ParameterSource.COMMANDLINE if key in typed else ParameterSource.DEFAULT_MAP) return ctx def test_local_proxy_works_with_no_other_proxy_option() -> None: from unshackle.core.proxies.resolve import require_alone_local_proxy with pytest.raises(click.UsageError, match="--proxy, --proxy-download"): require_alone_local_proxy( local_proxy_ctx({"remote": True, "proxy": "us", "proxy_download": "us"}, {"proxy", "proxy_download"}) ) with pytest.raises(click.UsageError, match="needs --remote"): require_alone_local_proxy(local_proxy_ctx({"remote": False}, set())) def test_local_proxy_turns_off_a_proxy_from_the_config_defaults() -> None: from unshackle.core.proxies.resolve import require_alone_local_proxy ctx = local_proxy_ctx({"remote": True, "proxy": "us", "no_proxy_download": True}, set()) require_alone_local_proxy(ctx) assert ctx.params["proxy"] is None and ctx.params["no_proxy_download"] is False def test_the_tunnel_port_refuses_a_proxy_uri_without_the_tunnel_credentials( server: str, target: int, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(tunnel, "is_public_ip", lambda address: True) monkeypatch.setattr(tunnel, "dialled_itself", lambda writer: False) client = TunnelClient(server, {"X-Secret-Key": "alice"}) tunnel_id = client.start() port = TUNNELS[tunnel_id].port for uri in (f"socks5h://127.0.0.1:{port}", f"socks5h://{tunnel_id}:wrong@127.0.0.1:{port}"): with pytest.raises(requests.exceptions.ConnectionError): requests.get(f"http://127.0.0.1:{target}/", proxies={"all": uri}, timeout=10) client.stop() def test_the_server_never_dials_for_a_client(server: str, target: int) -> None: """An OPEN from the client must not make the server connect anywhere: only the client dials.""" async def send_open() -> None: async with aiohttp.ClientSession(headers={"X-Secret-Key": "mallory"}) as http: async with http.ws_connect(server + ROUTE) as ws: await ws.receive_json() await ws.send_bytes(frame(1, OPEN, target.to_bytes(2, "big") + b"127.0.0.1")) Hello.hits = 0 asyncio.run(send_open()) assert Hello.hits == 0 def test_the_check_route_is_only_for_the_tunnel_owner(server: str) -> None: client = TunnelClient(server, {"X-Secret-Key": "alice"}) tunnel_id = client.start() resp = requests.get(f"{server}{CHECK_ROUTE}?id={tunnel_id}", headers={"X-Secret-Key": "mallory"}, timeout=10) assert resp.status_code == 404 client.stop() @pytest.mark.parametrize( ("uri", "line"), [ ("socks5://127.0.0.1:1080", "Using a local proxy on port 1080"), ("http://localhost:8888", "Using a local proxy on port 8888"), ("http://1.2.3.4:8080", "Using explicit Proxy: http://1.2.3.4:8080"), ("http://localhost", "Using a local proxy"), ("http://127.0.0.1:99999", "Using a local proxy"), ], ) def test_a_local_proxy_logs_only_its_port(uri: str, line: str) -> None: from unshackle.core.proxies.resolve import describe_explicit_proxy assert describe_explicit_proxy(uri) == line def test_another_api_key_cannot_reattach_to_a_tunnel(server: str) -> None: client = TunnelClient(server, {"X-Secret-Key": "alice"}) tunnel_id = client.start() async def reattach() -> None: async with aiohttp.ClientSession(headers={"X-Secret-Key": "mallory"}) as http: async with http.ws_connect(f"{server}{ROUTE}?id={tunnel_id}"): pass with pytest.raises(aiohttp.WSServerHandshakeError) as e: asyncio.run(reattach()) assert e.value.status == 404 client.stop() def test_a_request_that_is_not_a_websocket_opens_no_tunnel(server: str) -> None: before = len(TUNNELS) resp = requests.get(server + ROUTE, headers={"X-Secret-Key": "alice"}, timeout=10) assert resp.status_code == 400 assert len(TUNNELS) == before def test_a_malformed_frame_still_leaves_the_tunnel_to_expire(server: str) -> None: async def send_short_frame() -> str: async with aiohttp.ClientSession(headers={"X-Secret-Key": "mallory"}) as http: async with http.ws_connect(server + ROUTE) as ws: tunnel_id = (await ws.receive_json())["id"] await ws.send_bytes(b"\x01") return str(tunnel_id) tunnel_id = asyncio.run(send_short_frame()) deadline = time.monotonic() + 5 while TUNNELS[tunnel_id].expiry is None and time.monotonic() < deadline: time.sleep(0.05) assert TUNNELS[tunnel_id].expiry is not None def test_an_api_key_has_a_tunnel_limit(server: str) -> None: clients = [TunnelClient(server, {"X-Secret-Key": "bob"}) for _ in range(tunnel.MAX_TUNNELS)] for client in clients: client.start() with pytest.raises(ConnectionError, match="429"): TunnelClient(server, {"X-Secret-Key": "bob"}).start() for client in clients: client.stop() def test_a_connection_that_stops_reading_is_dropped() -> None: sent: list = [] class FakeWS: closed = False async def send_bytes(self, data: bytes) -> None: sent.append(data) async def flood() -> tunnel.Link: link = tunnel.Link(FakeWS()) link.streams[7] = tunnel.new_queue() for _ in range(tunnel.BACKLOG + 5): await link.feed(7, b"x") return link link = asyncio.run(flood()) assert 7 not in link.streams assert sent == [frame(7, tunnel.CLOSE)] def test_a_download_job_refuses_a_tunnel_proxy(monkeypatch: pytest.MonkeyPatch) -> None: from unshackle.core.api import handlers from unshackle.core.api.errors import APIError monkeypatch.setattr(handlers, "server_cdm_allowed", lambda request, service: True) monkeypatch.setattr(handlers, "server_cdm_max_height", lambda request, service: None) with pytest.raises(APIError, match="tunnel"): handlers.enforce_download_gates({"service": "EXAMPLE", "proxy": "tunnel:abc"}, None) @responses.activate def test_a_server_error_after_the_tunnel_closed_names_the_tunnel() -> None: from types import SimpleNamespace from unshackle.core.remote_service import RemoteClient, RemoteError responses.add( responses.GET, "http://srv:8786/api/session/x/titles", json={"status": "error", "error_code": "SERVICE_ERROR", "message": "proxy refused"}, status=422, ) client = RemoteClient("http://srv:8786", "") client.tunnel = SimpleNamespace(lost=True) # type: ignore[assignment] with pytest.raises(RemoteError) as e: client.get("/api/session/x/titles") assert e.value.kind == "TUNNEL_CLOSED" def spy_links(monkeypatch: pytest.MonkeyPatch) -> list: """Record every Link the client makes, so a test can look at its open streams.""" links: list = [] class SpyLink(tunnel.Link): def __init__(self, ws: object) -> None: super().__init__(ws) links.append(self) monkeypatch.setattr(tunnel, "Link", SpyLink) return links def wait_until(check: "Callable[[], bool]", seconds: float = 3) -> bool: deadline = time.monotonic() + seconds while not check() and time.monotonic() < deadline: time.sleep(0.05) return check() def test_a_dial_the_server_gave_up_on_is_dropped_by_the_client( server: str, target: int, monkeypatch: pytest.MonkeyPatch ) -> None: """When the server times out an OPEN, the client must not keep the connection it makes later.""" monkeypatch.setattr(tunnel, "is_public_ip", lambda address: True) monkeypatch.setattr(tunnel, "dialled_itself", lambda writer: False) monkeypatch.setattr(tunnel, "OPEN_TIMEOUT", 0.3) real_connect = tunnel.connect_public async def slow_connect(host: str, port: int) -> tuple: await asyncio.sleep(1) return await real_connect(host, port) monkeypatch.setattr(tunnel, "connect_public", slow_connect) links = spy_links(monkeypatch) client = TunnelClient(server, {"X-Secret-Key": "alice"}) proxy = tunnel_proxy(client.start(), "alice") with pytest.raises(requests.exceptions.ConnectionError): requests.get(f"http://127.0.0.1:{target}/", proxies={"all": proxy}, timeout=10) assert wait_until(lambda: not links[-1].streams and not client.relays) client.stop() def socks5_connect(proxy: str, host: bytes) -> int: """Send one SOCKS5 CONNECT for ``host`` with the credentials in ``proxy`` and return the reply code.""" from urllib.parse import urlsplit parts = urlsplit(proxy) user, password = (parts.username or "").encode(), (parts.password or "").encode() with socket.create_connection(("127.0.0.1", parts.port or 0), timeout=10) as sock: sock.sendall(b"\x05\x01\x02") assert sock.recv(2) == b"\x05\x02" sock.sendall(b"\x01" + bytes([len(user)]) + user + bytes([len(password)]) + password) assert sock.recv(2) == b"\x01\x00" sock.sendall(b"\x05\x01\x00\x03" + bytes([len(host)]) + host + (443).to_bytes(2, "big")) return sock.recv(10)[1] def test_a_hostname_the_resolver_rejects_closes_its_stream(server: str, monkeypatch: pytest.MonkeyPatch) -> None: async def resolve(self: object, host: str, port: int, **_: object) -> list: host.encode("idna") # as socket.getaddrinfo does, so a long label raises UnicodeError return [] monkeypatch.setattr(asyncio.BaseEventLoop, "getaddrinfo", resolve) monkeypatch.setattr(tunnel, "OPEN_TIMEOUT", 1) links = spy_links(monkeypatch) client = TunnelClient(server, {"X-Secret-Key": "alice"}) proxy = tunnel_proxy(client.start(), "alice") assert proxy assert socks5_connect(proxy, b"a" * 64 + b".example") == 5 assert wait_until(lambda: not links[-1].streams) client.stop() def test_a_failed_start_leaves_no_thread_behind() -> None: silent = socket.socket() silent.bind(("127.0.0.1", 0)) silent.listen() try: client = TunnelClient(f"http://127.0.0.1:{silent.getsockname()[1]}", {"X-Secret-Key": "alice"}) with pytest.raises(ConnectionError, match="timed out"): client.start(timeout=0.5) assert not client.thread.is_alive() finally: silent.close()