Files
unshackle/unshackle/core/session.py
T

832 lines
31 KiB
Python

"""Session utilities for creating HTTP sessions with TLS fingerprinting through rnet (Rust/BoringSSL)."""
from __future__ import annotations
import http
import logging
import math
import random
import time
from collections.abc import Iterator, MutableMapping
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from http.cookiejar import CookieJar
from typing import Any, Optional
from urllib.parse import urlencode, urlparse, urlunparse
import rnet
from requests import HTTPError, Request
from requests.structures import CaseInsensitiveDict
from unshackle.core.config import config
# rnet's only distinguishing role is TLS fingerprinting; retry, backoff,
# pooling, and timeouts must match the plain-requests path. Both RnetSession
# (below) and Service.get_session import these so the two can't drift.
# The unified downloader keeps its own copies (downloaders/requests.py) by design.
MAX_RETRIES = 5
BACKOFF_FACTOR = 0.2
MAX_BACKOFF = 60.0 # shared backoff cap (rnet max_backoff == requests Retry backoff_max)
STATUS_FORCELIST = [429, 500, 502, 503, 504]
RETRY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"})
# pool_block=True means a pool smaller than the in-flight request count stalls threads
POOL_MAX_SIZE = 64 # rnet pool_max_idle_per_host == requests pool_maxsize/pool_connections
CONNECT_TIMEOUT = 10
READ_TIMEOUT = 30
DEFAULT_IMPERSONATE = rnet.Impersonate.Chrome131
def resolve_impersonate(browser: str) -> rnet.Impersonate:
"""Change a browser string to an rnet.Impersonate preset.
Accepts exact rnet preset names (e.g. "Chrome131", "OkHttp4_12", "Edge101").
See https://github.com/0x676e67/rnet for the full list of available presets.
"""
preset = getattr(rnet.Impersonate, browser, None)
if preset is not None:
return preset
raise ValueError(
f"Unknown impersonate preset: {browser!r}. "
f"Use exact rnet preset names like 'Chrome131', 'OkHttp4_12', 'Edge101'. "
f"See rnet.Impersonate for all available presets."
)
METHOD_MAP: dict[str, rnet.Method] = {
"GET": rnet.Method.GET,
"POST": rnet.Method.POST,
"PUT": rnet.Method.PUT,
"DELETE": rnet.Method.DELETE,
"HEAD": rnet.Method.HEAD,
"OPTIONS": rnet.Method.OPTIONS,
"PATCH": rnet.Method.PATCH,
"TRACE": rnet.Method.TRACE,
}
class RnetResponseHeaders(MutableMapping):
"""Read-only str-based view over rnet's bytes-based HeaderMap."""
def __init__(self, header_map: Any) -> None:
self._map = header_map
def decode(self, val: Any) -> str:
return val.decode("utf-8", errors="replace") if isinstance(val, (bytes, bytearray)) else str(val)
def __getitem__(self, key: str) -> str:
val = self._map[key]
return self.decode(val)
def __setitem__(self, key: str, value: str) -> None:
raise TypeError("Response headers are read-only")
def __delitem__(self, key: str) -> None:
raise TypeError("Response headers are read-only")
def __contains__(self, key: object) -> bool:
if not isinstance(key, str):
return False
return self._map.contains_key(key)
def __iter__(self) -> Iterator[str]:
seen: set[str] = set()
for k, _ in self._map.items():
dk = self.decode(k)
if dk not in seen:
seen.add(dk)
yield dk
def __len__(self) -> int:
return self._map.keys_len()
def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
val = self._map.get(key)
if val is None:
return default
return self.decode(val)
def items(self) -> list[tuple[str, str]]:
return [(self.decode(k), self.decode(v)) for k, v in self._map.items()]
class RnetResponse:
"""Wraps rnet.BlockingResponse with a requests-compatible API."""
def __init__(self, resp: Any) -> None:
self._resp = resp
self._headers: Optional[RnetResponseHeaders] = None
self._content: Optional[bytes] = None
self._text: Optional[str] = None
self._streamed = False
@property
def status_code(self) -> int:
return int(str(self._resp.status_code))
@property
def ok(self) -> bool:
return self._resp.ok
@property
def headers(self) -> RnetResponseHeaders:
if self._headers is None:
self._headers = RnetResponseHeaders(self._resp.headers)
return self._headers
@property
def url(self) -> str:
return str(self._resp.url)
@property
def content_length(self) -> Optional[int]:
return self._resp.content_length
@property
def content(self) -> bytes:
if self._content is None:
self._content = self._resp.bytes()
return self._content
@property
def text(self) -> str:
if self._text is None:
encoding = self._resp.encoding or "utf-8"
self._text = self.content.decode(encoding, errors="replace")
return self._text
@property
def reason(self) -> str:
try:
return http.HTTPStatus(self.status_code).phrase
except ValueError:
return "Unknown"
@property
def cookies(self) -> Any:
return self._resp.cookies
@property
def version(self) -> str:
"""Get the HTTP version used for the response as a string (e.g. 'HTTP/2', 'HTTP/1.1')."""
try:
v_str = str(self._resp.version)
if "HTTP_2" in v_str:
return "HTTP/2"
elif "HTTP_11" in v_str:
return "HTTP/1.1"
elif "HTTP_3" in v_str:
return "HTTP/3"
elif "HTTP_10" in v_str:
return "HTTP/1.0"
elif "HTTP_09" in v_str:
return "HTTP/0.9"
return v_str.replace("Version.", "").replace("_", "/")
except Exception:
return "HTTP/1.1"
def json(self, **kwargs: Any) -> Any:
import json as _json
return _json.loads(self.content)
def raise_for_status(self) -> None:
if not self.ok:
raise HTTPError(
f"{self.status_code} {self.reason}: {self.url}",
response=self,
)
def iter_content(self, chunk_size: Optional[int] = None) -> Iterator[bytes]:
"""Re-chunk rnet's variable-size data into fixed-size pieces."""
self._streamed = True
if chunk_size is None or chunk_size <= 0:
yield from self._resp.stream()
return
buf = bytearray()
for chunk in self._resp.stream():
buf.extend(chunk)
while len(buf) >= chunk_size:
yield bytes(buf[:chunk_size])
buf = buf[chunk_size:]
if buf:
yield bytes(buf)
def stream(self) -> Iterator[bytes]:
"""Direct pass-through of rnet's native ``stream()`` iterator."""
self._streamed = True
yield from self._resp.stream()
def close(self) -> None:
try:
self._resp.close()
except Exception:
pass
class RnetSessionHeaders(CaseInsensitiveDict):
"""Dict-like headers that write to the rnet client through update()."""
def __init__(self, client: Any) -> None:
self._client = client
super().__init__()
def sync(self) -> None:
"""Push current headers to the rnet client."""
if self._client is not None and hasattr(self, "_store") and self._store:
self._client.update(headers={k: v for k, v in self.items()})
def __setitem__(self, key: str, value: str) -> None:
super().__setitem__(key, value)
self.sync()
def update(self, __m: Any = None, **kwargs: Any) -> None:
if __m:
if hasattr(__m, "items"):
for k, v in __m.items():
super().__setitem__(k, v)
else:
for k, v in __m:
super().__setitem__(k, v)
for k, v in kwargs.items():
super().__setitem__(k, v)
self.sync()
def pop(self, key: str, *args: Any) -> Any:
result = super().pop(key, *args)
# rnet doesn't support removing individual headers, but we track locally
# and always send the full set on next update
return result
def __delitem__(self, key: str) -> None:
super().__delitem__(key)
class RnetCookieAdapter(MutableMapping):
"""Cookie adapter that bridges requests-style cookie access to rnet."""
def __init__(self, client: Any) -> None:
self._client = client
self._cookies: dict[str, dict[str, str]] = {}
self._flat: dict[str, str] = {}
self._original_cookies: list[Any] = []
def set_cookie_on_client(self, url: str, name: str, value: str) -> None:
"""Set a cookie on the rnet client, or buffer locally if the client is not yet created."""
if self._client is not None:
try:
self._client.set_cookie(url, rnet.Cookie(name, value))
except Exception:
pass
def flush_to_client(self) -> None:
"""Push all buffered cookies to the rnet client after unshackle makes it."""
if self._client is None:
return
for domain, cookies in self._cookies.items():
url = f"https://{domain.lstrip('.')}" if domain else "https://localhost"
for name, value in cookies.items():
try:
self._client.set_cookie(url, rnet.Cookie(name, value))
except Exception:
pass
@property
def jar(self) -> CookieJar:
"""Return a CookieJar with original Cookie objects (requests compat).
``save_cookies`` in dl.py uses this to write cookies back to disk.
"""
jar = CookieJar()
for cookie in self._original_cookies:
jar.set_cookie(cookie)
return jar
def update(self, other: Any = None, **kwargs: Any) -> None:
if other is None:
other = {}
if isinstance(other, CookieJar):
for cookie in other:
domain = cookie.domain or ""
name = cookie.name
value = cookie.value or ""
self._flat[name] = value
self._cookies.setdefault(domain, {})[name] = value
self._original_cookies.append(cookie)
url = f"https://{domain.lstrip('.')}" if domain else "https://localhost"
self.set_cookie_on_client(url, name, value)
elif isinstance(other, dict):
for name, value in other.items():
self._flat[name] = value
self.set_cookie_on_client("https://localhost", name, str(value))
self._flat.update(other)
elif hasattr(other, "items"):
for name, value in other.items():
self._flat[name] = str(value)
self.set_cookie_on_client("https://localhost", name, str(value))
for name, value in kwargs.items():
self._flat[name] = value
self.set_cookie_on_client("https://localhost", name, value)
def get(
self, name: str, default: Optional[str] = None, domain: Optional[str] = None, path: Optional[str] = None
) -> Optional[str]:
if domain and domain in self._cookies:
return self._cookies[domain].get(name, default)
return self._flat.get(name, default)
def set(self, name: str, value: str, domain: str = "localhost") -> None:
self._flat[name] = value
self._cookies.setdefault(domain, {})[name] = value
url = f"https://{domain.lstrip('.')}"
self.set_cookie_on_client(url, name, value)
def __getitem__(self, name: str) -> str:
return self._flat[name]
def __setitem__(self, name: str, value: str) -> None:
self.set(name, value)
def __delitem__(self, name: str) -> None:
self._flat.pop(name, None)
for domain_cookies in self._cookies.values():
domain_cookies.pop(name, None)
def __contains__(self, name: object) -> bool:
return name in self._flat
def __iter__(self) -> Iterator:
return iter(self._flat)
def __len__(self) -> int:
return len(self._flat)
def __bool__(self) -> bool:
return bool(self._flat)
def get_dict(self, domain: Optional[str] = None, path: Optional[str] = None) -> dict[str, str]:
"""Return cookies as a plain dict (requests RequestsCookieJar compat).
With a *domain*, this method returns only the cookies for that domain.
It accepts *path* for API compatibility but ignores it (flat storage).
"""
if domain is not None:
return dict(self._cookies.get(domain, {}))
return dict(self._flat)
def get_dict_by_domain(self) -> dict[str, dict[str, str]]:
"""Return cookies grouped by domain, domain-less ones under ``""``.
rnet scopes each cookie to the host of the URL it was set on, so a copy made with
``get_dict`` can only ever replay against one host. This class records cookies that
arrived as a plain dict flat only, and reports them under the empty domain, to match
the localhost fallback in :meth:`flush_to_client`.
"""
grouped = {domain: dict(cookies) for domain, cookies in self._cookies.items() if cookies}
scoped = {name for cookies in grouped.values() for name in cookies}
leftovers = {name: value for name, value in self._flat.items() if name not in scoped}
if leftovers:
grouped[""] = {**grouped.get("", {}), **leftovers}
return grouped
def clear(self, domain: Optional[str] = None, path: Optional[str] = None, name: Optional[str] = None) -> None:
"""Remove cookies (requests RequestsCookieJar compat).
- ``clear()`` removes all cookies.
- ``clear(domain=..., path=..., name=...)`` removes a specific cookie.
"""
if name is not None:
self._flat.pop(name, None)
if domain is not None and domain in self._cookies:
self._cookies[domain].pop(name, None)
else:
for domain_cookies in self._cookies.values():
domain_cookies.pop(name, None)
elif domain is not None:
removed = self._cookies.pop(domain, {})
for k in removed:
# Only remove from flat if no other domain has same key
still_exists = any(k in dc for dc in self._cookies.values())
if not still_exists:
self._flat.pop(k, None)
else:
self._flat.clear()
self._cookies.clear()
def items(self) -> list[tuple[str, str]]:
return list(self._flat.items())
def keys(self) -> list[str]:
return list(self._flat.keys())
def values(self) -> list[str]:
return list(self._flat.values())
class RnetProxyDict(dict):
"""Dict-like proxy config that syncs to the rnet client.
Accepts ``{"all": url}``, ``{"https": url}``, or ``{"http": url}``
and applies through rnet's native ``proxies`` parameter (``List[rnet.Proxy]``).
Supports both lazy (pre-client) and live (post-client) proxy updates.
"""
def __init__(self, session: "RnetSession") -> None:
super().__init__()
self._session = session
def sync(self) -> None:
proxy = self.get("all") or self.get("https") or self.get("http")
proxies = [rnet.Proxy.all(proxy)] if proxy else []
self._session._client_kwargs["proxies"] = proxies or None
if self._session._client is not None:
self._session._client.update(proxies=proxies or None)
def update(self, __m: Any = None, **kwargs: Any) -> None:
super().update(__m or {}, **kwargs)
self.sync()
def __setitem__(self, key: str, value: str) -> None:
super().__setitem__(key, value)
self.sync()
class MaxRetriesError(Exception):
def __init__(self, message: str, cause: Optional[Exception] = None) -> None:
super().__init__(message)
self.__cause__ = cause
class RnetSession:
"""TLS-fingerprinted HTTP session powered by rnet (Rust/BoringSSL).
Drop-in replacement for CurlSession with requests-compatible API.
Supports browser impersonation (Chrome, Firefox, Edge, Safari, OkHttp),
retry with exponential backoff, cookie persistence, and proxies.
unshackle makes the client lazily on the first request, so you can
configure headers, cookies, and proxies freely before it opens any
connection.
"""
def __init__(
self,
max_retries: int = MAX_RETRIES,
backoff_factor: float = BACKOFF_FACTOR,
max_backoff: float = MAX_BACKOFF,
status_forcelist: Optional[list[int]] = None,
allowed_methods: Optional[set[str]] = None,
catch_exceptions: Optional[tuple[type[Exception], ...]] = None,
**session_kwargs: Any,
) -> None:
self.max_retries = max_retries
self.backoff_factor = backoff_factor
self.max_backoff = max_backoff
self.status_forcelist = status_forcelist or list(STATUS_FORCELIST)
self.allowed_methods = allowed_methods or set(RETRY_METHODS)
self.catch_exceptions = catch_exceptions or (
rnet.ConnectionError,
rnet.ConnectionResetError, # sibling of ConnectionError; mid-request resets are retryable
rnet.TimeoutError,
rnet.RequestError,
)
self.log = logging.getLogger(self.__class__.__name__)
client_kwargs: dict[str, Any] = {}
for key in (
"impersonate",
"timeout",
"connect_timeout",
"read_timeout",
"proxies",
"verify",
"allow_redirects",
"pool_idle_timeout",
"pool_max_idle_per_host",
"pool_max_size",
"tcp_keepalive",
"tcp_keepalive_interval",
"tcp_keepalive_retries",
"http1_only",
"http2_only",
"tcp_nodelay",
):
if key in session_kwargs:
client_kwargs[key] = session_kwargs.pop(key)
if "proxy" in session_kwargs:
proxy_url = session_kwargs.pop("proxy")
if proxy_url:
client_kwargs["proxies"] = [rnet.Proxy.all(proxy_url)]
client_kwargs["cookie_store"] = True
self.verify: bool = client_kwargs.pop("verify", True)
if not self.verify:
client_kwargs["danger_accept_invalid_certs"] = True
self._client_kwargs = dict(client_kwargs)
self._client: Optional[rnet.BlockingClient] = None
self.headers = RnetSessionHeaders(None)
self.cookies = RnetCookieAdapter(None)
self.proxies = RnetProxyDict(self)
if "headers" in session_kwargs:
self.headers.update(session_kwargs.pop("headers"))
if "cookies" in session_kwargs:
self.cookies.update(session_kwargs.pop("cookies"))
if "proxies" in session_kwargs:
self.proxies.update(session_kwargs.pop("proxies"))
@property
def impersonate_name(self) -> Optional[str]:
"""Preset name (e.g. 'Chrome131') for rebuilding this HTTP session in another process.
rnet enums stringify as 'Impersonate.Chrome131'. Return the trailing name, or None
when no preset was set (then the HTTP session is not cheaply rebuildable across
processes).
"""
preset = self._client_kwargs.get("impersonate")
if preset is None:
return None
name = str(preset).rsplit(".", 1)[-1]
return name or None
def ensure_client(self) -> rnet.BlockingClient:
"""Lazily make the rnet client on first use, flushing any buffered state."""
if self._client is None:
self._client = rnet.BlockingClient(**self._client_kwargs)
self.headers._client = self._client
self.headers.sync()
self.cookies._client = self._client
self.cookies.flush_to_client()
return self._client
def build_url(self, url: str, params: Optional[Any] = None) -> str:
"""Encode params into the URL (rnet ignores the params kwarg).
Accepts the same shapes as requests: a mapping, a sequence of pairs, or a
pre-built query string/bytes. This method appends a string verbatim (already
encoded). urlencode() would raise TypeError on it.
"""
if not params:
return url
if isinstance(params, bytes):
extra = params.decode("utf-8")
elif isinstance(params, str):
extra = params
else:
extra = urlencode(params, doseq=True)
parsed = urlparse(url)
separator = "&" if parsed.query else ""
query = parsed.query + separator + extra if parsed.query else extra
return urlunparse(parsed._replace(query=query))
def get_sleep_time(self, response: Optional[RnetResponse], attempt: int) -> Optional[float]:
if response:
retry_after = response.headers.get("Retry-After")
if retry_after:
wait: Optional[float] = None
try:
wait = float(retry_after)
except ValueError:
try:
retry_date = parsedate_to_datetime(retry_after)
if retry_date.tzinfo is None:
retry_date = retry_date.replace(tzinfo=timezone.utc)
wait = (retry_date - datetime.now(timezone.utc)).total_seconds()
except Exception:
# parsedate_to_datetime itself raises ValueError on malformed dates
# (Python >= 3.10); an unusable header must not escape the retry loop
wait = None
if wait is not None and math.isfinite(wait):
# a hostile Retry-After (e.g. 86400) would otherwise park the caller for a day
return min(wait, self.max_backoff)
if attempt == 0:
return 0.0
backoff_value = self.backoff_factor * (2 ** (attempt - 1))
jitter = backoff_value * 0.1
sleep_time = backoff_value + random.uniform(-jitter, jitter)
return min(sleep_time, self.max_backoff)
def request(self, method: str, url: str, **kwargs: Any) -> RnetResponse:
"""Send a request, retrying on the status forcelist and on the caught exception types.
This method retries only the methods in allowed_methods. Any other method gets one try.
A max_retries kwarg overrides the HTTP session retry budget for this call alone, where 0
means one try with no retries. Once the budget is spent, this method raises
MaxRetriesError with the last failure as its cause.
"""
client = self.ensure_client()
method_upper = method.upper() if isinstance(method, str) else str(method).upper()
max_retries = kwargs.pop("max_retries", None)
if max_retries is None:
max_retries = self.max_retries
url = self.build_url(url, kwargs.pop("params", None))
kwargs.setdefault("allow_redirects", True)
if not self.verify:
kwargs.setdefault("verify", False)
# Remove kwargs rnet doesn't understand
kwargs.pop("stream", None) # rnet responses are always lazy
data = kwargs.pop("data", None)
if data is not None:
if isinstance(data, dict):
kwargs["form"] = list(data.items())
elif isinstance(data, (str, bytes)):
kwargs["body"] = data
else:
kwargs["body"] = data
rnet_method = METHOD_MAP.get(method_upper)
if rnet_method is None:
raise ValueError(f"Unsupported HTTP method: {method}")
# Convert headers to standard dict once to resolve PyO3 CaseInsensitiveDict rejection.
if kwargs.get("headers") is not None:
kwargs["headers"] = dict(kwargs["headers"])
if method_upper not in self.allowed_methods:
raw_resp = client.request(rnet_method, url, **kwargs)
return RnetResponse(raw_resp)
last_exception: Optional[Exception] = None
response: Optional[RnetResponse] = None
for attempt in range(max_retries + 1):
try:
raw_resp = client.request(rnet_method, url, **kwargs)
response = RnetResponse(raw_resp)
if config.debug_requests:
parsed_url = urlparse(url)
port_str = f":{parsed_url.port}" if parsed_url.port else ""
if not port_str:
port_str = ":443" if parsed_url.scheme == "https" else ":80"
host_url = f"{parsed_url.scheme}://{parsed_url.hostname}{port_str}"
path_and_query = parsed_url.path + (f"?{parsed_url.query}" if parsed_url.query else "")
if not path_and_query:
path_and_query = "/"
content_length = response.headers.get("content-length") or response.content_length
log_msg = f'{host_url} "{method_upper} {path_and_query} {response.version}" {response.status_code} {content_length}'
self.log.debug(log_msg)
if response.status_code not in self.status_forcelist:
return response
last_exception = HTTPError(f"Received status code: {response.status_code}", response=response)
self.log.warning(
f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... "
f"({attempt + 1}/{max_retries})"
)
except self.catch_exceptions as e:
last_exception = e
response = None
self.log.warning(
f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{max_retries})"
)
if attempt < max_retries:
if sleep_duration := self.get_sleep_time(response, attempt + 1):
if sleep_duration > 0:
time.sleep(sleep_duration)
else:
break
raise MaxRetriesError(f"Max retries exceeded for {method} {url}", cause=last_exception)
def get(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("GET", url, **kwargs)
def post(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("POST", url, **kwargs)
def put(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("PUT", url, **kwargs)
def delete(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("DELETE", url, **kwargs)
def head(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("HEAD", url, **kwargs)
def options(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("OPTIONS", url, **kwargs)
def patch(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("PATCH", url, **kwargs)
def prepare_request(self, req: Request) -> Request:
"""Compatibility shim for services using prepared requests."""
# Merge session headers into request headers
if req.headers:
merged = dict(self.headers)
merged.update(req.headers)
req.headers = merged
else:
req.headers = dict(self.headers)
return req
def send(self, req: Request, **kwargs: Any) -> RnetResponse:
"""Compatibility shim for services using prepared requests."""
method = req.method or "GET"
url = req.url or ""
send_kwargs: dict[str, Any] = {}
if req.headers:
send_kwargs["headers"] = dict(req.headers)
if req.body:
send_kwargs["data"] = req.body
if req.json:
send_kwargs["json"] = req.json
send_kwargs.update(kwargs)
return self.request(method, url, **send_kwargs)
def mount(self, prefix: str, adapter: Any) -> None:
"""No-op. rnet does TLS and connection pooling natively."""
pass
def close(self) -> None:
"""No-op. rnet manages its own resources."""
pass
def session(
browser: Optional[str] = None,
**kwargs: Any,
) -> RnetSession:
"""
Make an rnet HTTP session with TLS fingerprinting (browser/app impersonation).
Args:
browser: Exact rnet.Impersonate preset name. Examples:
"Chrome131", "OkHttp4_12", "Edge101", "Firefox135",
"Safari18", "OkHttp5", "Opera118"
Uses the configured default from config if not specified.
See rnet.Impersonate for all available presets.
**kwargs: Additional arguments passed to RnetSession constructor.
Returns:
RnetSession configured with browser impersonation and retry behaviour.
Examples:
session() # Default browser from config
session("OkHttp4_12") # OkHttp 4.12 fingerprint
session("Chrome131") # Chrome 131
session("Edge101", max_retries=3) # Edge 101 with custom retry
"""
if browser is None:
browser = config.network.get("browser", "Chrome131")
impersonate = resolve_impersonate(browser)
session_kwargs: dict[str, Any] = {"impersonate": impersonate}
# optional rnet client knobs, see docs/NETWORK_CONFIG.md
for key in (
"http1_only",
"http2_only",
"pool_max_idle_per_host",
"pool_max_size",
"tcp_nodelay",
"connect_timeout",
"read_timeout",
"timeout",
"pool_idle_timeout",
"tcp_keepalive",
"tcp_keepalive_interval",
"tcp_keepalive_retries",
"allow_redirects",
):
if key in config.network:
session_kwargs[key] = config.network[key]
session_kwargs.update(kwargs)
# Connection-pool / timeout defaults applied only when neither config.network nor the caller set them.
# connect_timeout + read_timeout mirror the requests-path default timeout (CONNECT_TIMEOUT, READ_TIMEOUT)
# so an unset config still gets a bounded connect and read like requests; pool_idle_timeout stays under
# the typical ~60s CDN idle kill; pool_max_idle_per_host follows POOL_MAX_SIZE, sized above the worker
# cap because hedge racers and tail-boost parts push in-flight requests past it; tcp_keepalive keeps
# long idle segments warm.
session_kwargs.setdefault("connect_timeout", CONNECT_TIMEOUT)
session_kwargs.setdefault("read_timeout", READ_TIMEOUT)
session_kwargs.setdefault("pool_idle_timeout", 55)
session_kwargs.setdefault("pool_max_idle_per_host", POOL_MAX_SIZE)
session_kwargs.setdefault("tcp_keepalive", 30)
session_obj = RnetSession(**session_kwargs)
session_obj.headers.update(config.headers)
return session_obj