feat(downloader): adaptive worker scaling, tail boost, session parity, resume fixes

Opt-in --adaptive-workers (AIMD scaling + tail range-split), request/rnet session parity, and fixes for resume poisoning, truncated-segment acceptance, 416 handling, and byte-range-slice corruption. Tests + docs.
This commit is contained in:
imSp4rky
2026-07-06 11:25:29 -06:00
parent 774c1a4555
commit 2f8492ebaf
16 changed files with 1085 additions and 146 deletions
+65 -11
View File
@@ -19,6 +19,24 @@ from requests.structures import CaseInsensitiveDict
from unshackle.core.config import config
# ---------------------------------------------------------------------------
# Shared retry / pool / timeout policy: single source of truth
# ---------------------------------------------------------------------------
# rnet's only distinguishing role is TLS fingerprinting; retry, backoff,
# pooling, and timeouts must match the plain-requests path. Both RnetSession
# (below) and Service.get_session import these so the two can't drift.
# The unified downloader keeps its own copies (downloaders/requests.py) by design.
MAX_RETRIES = 5
BACKOFF_FACTOR = 0.2
MAX_BACKOFF = 60.0 # shared backoff cap (rnet max_backoff == requests Retry backoff_max)
STATUS_FORCELIST = [429, 500, 502, 503, 504]
RETRY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"})
POOL_MAX_SIZE = 16 # rnet pool_max_idle_per_host == requests pool_maxsize/pool_connections
CONNECT_TIMEOUT = 10
READ_TIMEOUT = 30
# ---------------------------------------------------------------------------
# Impersonate preset mapping — rnet uses named presets (no custom JA3/Akamai)
# ---------------------------------------------------------------------------
@@ -483,9 +501,9 @@ class RnetSession:
def __init__(
self,
max_retries: int = 5,
backoff_factor: float = 0.2,
max_backoff: float = 60.0,
max_retries: int = MAX_RETRIES,
backoff_factor: float = BACKOFF_FACTOR,
max_backoff: float = MAX_BACKOFF,
status_forcelist: Optional[list[int]] = None,
allowed_methods: Optional[set[str]] = None,
catch_exceptions: Optional[tuple[type[Exception], ...]] = None,
@@ -494,10 +512,11 @@ class RnetSession:
self.max_retries = max_retries
self.backoff_factor = backoff_factor
self.max_backoff = max_backoff
self.status_forcelist = status_forcelist or [429, 500, 502, 503, 504]
self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"}
self.status_forcelist = status_forcelist or list(STATUS_FORCELIST)
self.allowed_methods = allowed_methods or set(RETRY_METHODS)
self.catch_exceptions = catch_exceptions or (
rnet.ConnectionError,
rnet.ConnectionResetError, # sibling of ConnectionError; mid-request resets are retryable
rnet.TimeoutError,
rnet.RequestError,
)
@@ -511,9 +530,13 @@ class RnetSession:
"read_timeout",
"proxies",
"verify",
"redirect",
"allow_redirects",
"pool_idle_timeout",
"pool_max_idle_per_host",
"pool_max_size",
"tcp_keepalive",
"tcp_keepalive_interval",
"tcp_keepalive_retries",
"http1_only",
"http2_only",
"tcp_nodelay",
@@ -597,6 +620,12 @@ class RnetSession:
client = self._ensure_client()
method_upper = method.upper() if isinstance(method, str) else str(method).upper()
# Per-request override of the session retry budget (rnet.request() would reject the kwarg).
# max_retries=0 means a single attempt with no session-level retries.
max_retries = kwargs.pop("max_retries", None)
if max_retries is None:
max_retries = self.max_retries
# Build URL with params
url = self._build_url(url, kwargs.pop("params", None))
@@ -637,7 +666,7 @@ class RnetSession:
last_exception: Optional[Exception] = None
response: Optional[RnetResponse] = None
for attempt in range(self.max_retries + 1):
for attempt in range(max_retries + 1):
try:
raw_resp = client.request(rnet_method, url, **kwargs)
response = RnetResponse(raw_resp)
@@ -662,17 +691,17 @@ class RnetSession:
last_exception = HTTPError(f"Received status code: {response.status_code}")
self.log.warning(
f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... "
f"({attempt + 1}/{self.max_retries})"
f"({attempt + 1}/{max_retries})"
)
except self.catch_exceptions as e:
last_exception = e
response = None
self.log.warning(
f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{self.max_retries})"
f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{max_retries})"
)
if attempt < self.max_retries:
if attempt < max_retries:
if sleep_duration := self.get_sleep_time(response, attempt + 1):
if sleep_duration > 0:
time.sleep(sleep_duration)
@@ -774,11 +803,36 @@ def session(
session_kwargs: dict[str, Any] = {"impersonate": impersonate}
# optional rnet client knobs, see docs/NETWORK_CONFIG.md
for key in ("http1_only", "http2_only", "pool_max_idle_per_host", "pool_max_size", "tcp_nodelay"):
for key in (
"http1_only",
"http2_only",
"pool_max_idle_per_host",
"pool_max_size",
"tcp_nodelay",
"connect_timeout",
"read_timeout",
"timeout",
"pool_idle_timeout",
"tcp_keepalive",
"tcp_keepalive_interval",
"tcp_keepalive_retries",
"allow_redirects",
):
if key in config.network:
session_kwargs[key] = config.network[key]
session_kwargs.update(kwargs)
# Connection-pool / timeout defaults applied only when neither config.network nor the caller set them.
# connect_timeout + read_timeout mirror the requests-path default timeout (CONNECT_TIMEOUT, READ_TIMEOUT)
# so an unset config still gets a bounded connect and read like requests; pool_idle_timeout stays under
# the typical ~60s CDN idle kill; pool_max_idle_per_host matches the downloader worker cap to avoid
# HTTP/1.1 connection churn; tcp_keepalive keeps long idle segments warm.
session_kwargs.setdefault("connect_timeout", CONNECT_TIMEOUT)
session_kwargs.setdefault("read_timeout", READ_TIMEOUT)
session_kwargs.setdefault("pool_idle_timeout", 55)
session_kwargs.setdefault("pool_max_idle_per_host", POOL_MAX_SIZE)
session_kwargs.setdefault("tcp_keepalive", 30)
session_obj = RnetSession(**session_kwargs)
session_obj.headers.update(config.headers)
return session_obj