mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-21 16:35:07 +00:00
feat(downloader): adaptive worker scaling, tail boost, session parity, resume fixes
Opt-in --adaptive-workers (AIMD scaling + tail range-split), request/rnet session parity, and fixes for resume poisoning, truncated-segment acceptance, 416 handling, and byte-range-slice corruption. Tests + docs.
This commit is contained in:
+65
-11
@@ -19,6 +19,24 @@ from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from unshackle.core.config import config
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared retry / pool / timeout policy: single source of truth
|
||||
# ---------------------------------------------------------------------------
|
||||
# rnet's only distinguishing role is TLS fingerprinting; retry, backoff,
|
||||
# pooling, and timeouts must match the plain-requests path. Both RnetSession
|
||||
# (below) and Service.get_session import these so the two can't drift.
|
||||
# The unified downloader keeps its own copies (downloaders/requests.py) by design.
|
||||
|
||||
MAX_RETRIES = 5
|
||||
BACKOFF_FACTOR = 0.2
|
||||
MAX_BACKOFF = 60.0 # shared backoff cap (rnet max_backoff == requests Retry backoff_max)
|
||||
STATUS_FORCELIST = [429, 500, 502, 503, 504]
|
||||
RETRY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"})
|
||||
POOL_MAX_SIZE = 16 # rnet pool_max_idle_per_host == requests pool_maxsize/pool_connections
|
||||
CONNECT_TIMEOUT = 10
|
||||
READ_TIMEOUT = 30
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Impersonate preset mapping — rnet uses named presets (no custom JA3/Akamai)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -483,9 +501,9 @@ class RnetSession:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_retries: int = 5,
|
||||
backoff_factor: float = 0.2,
|
||||
max_backoff: float = 60.0,
|
||||
max_retries: int = MAX_RETRIES,
|
||||
backoff_factor: float = BACKOFF_FACTOR,
|
||||
max_backoff: float = MAX_BACKOFF,
|
||||
status_forcelist: Optional[list[int]] = None,
|
||||
allowed_methods: Optional[set[str]] = None,
|
||||
catch_exceptions: Optional[tuple[type[Exception], ...]] = None,
|
||||
@@ -494,10 +512,11 @@ class RnetSession:
|
||||
self.max_retries = max_retries
|
||||
self.backoff_factor = backoff_factor
|
||||
self.max_backoff = max_backoff
|
||||
self.status_forcelist = status_forcelist or [429, 500, 502, 503, 504]
|
||||
self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"}
|
||||
self.status_forcelist = status_forcelist or list(STATUS_FORCELIST)
|
||||
self.allowed_methods = allowed_methods or set(RETRY_METHODS)
|
||||
self.catch_exceptions = catch_exceptions or (
|
||||
rnet.ConnectionError,
|
||||
rnet.ConnectionResetError, # sibling of ConnectionError; mid-request resets are retryable
|
||||
rnet.TimeoutError,
|
||||
rnet.RequestError,
|
||||
)
|
||||
@@ -511,9 +530,13 @@ class RnetSession:
|
||||
"read_timeout",
|
||||
"proxies",
|
||||
"verify",
|
||||
"redirect",
|
||||
"allow_redirects",
|
||||
"pool_idle_timeout",
|
||||
"pool_max_idle_per_host",
|
||||
"pool_max_size",
|
||||
"tcp_keepalive",
|
||||
"tcp_keepalive_interval",
|
||||
"tcp_keepalive_retries",
|
||||
"http1_only",
|
||||
"http2_only",
|
||||
"tcp_nodelay",
|
||||
@@ -597,6 +620,12 @@ class RnetSession:
|
||||
client = self._ensure_client()
|
||||
method_upper = method.upper() if isinstance(method, str) else str(method).upper()
|
||||
|
||||
# Per-request override of the session retry budget (rnet.request() would reject the kwarg).
|
||||
# max_retries=0 means a single attempt with no session-level retries.
|
||||
max_retries = kwargs.pop("max_retries", None)
|
||||
if max_retries is None:
|
||||
max_retries = self.max_retries
|
||||
|
||||
# Build URL with params
|
||||
url = self._build_url(url, kwargs.pop("params", None))
|
||||
|
||||
@@ -637,7 +666,7 @@ class RnetSession:
|
||||
last_exception: Optional[Exception] = None
|
||||
response: Optional[RnetResponse] = None
|
||||
|
||||
for attempt in range(self.max_retries + 1):
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
raw_resp = client.request(rnet_method, url, **kwargs)
|
||||
response = RnetResponse(raw_resp)
|
||||
@@ -662,17 +691,17 @@ class RnetSession:
|
||||
last_exception = HTTPError(f"Received status code: {response.status_code}")
|
||||
self.log.warning(
|
||||
f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... "
|
||||
f"({attempt + 1}/{self.max_retries})"
|
||||
f"({attempt + 1}/{max_retries})"
|
||||
)
|
||||
|
||||
except self.catch_exceptions as e:
|
||||
last_exception = e
|
||||
response = None
|
||||
self.log.warning(
|
||||
f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{self.max_retries})"
|
||||
f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{max_retries})"
|
||||
)
|
||||
|
||||
if attempt < self.max_retries:
|
||||
if attempt < max_retries:
|
||||
if sleep_duration := self.get_sleep_time(response, attempt + 1):
|
||||
if sleep_duration > 0:
|
||||
time.sleep(sleep_duration)
|
||||
@@ -774,11 +803,36 @@ def session(
|
||||
|
||||
session_kwargs: dict[str, Any] = {"impersonate": impersonate}
|
||||
# optional rnet client knobs, see docs/NETWORK_CONFIG.md
|
||||
for key in ("http1_only", "http2_only", "pool_max_idle_per_host", "pool_max_size", "tcp_nodelay"):
|
||||
for key in (
|
||||
"http1_only",
|
||||
"http2_only",
|
||||
"pool_max_idle_per_host",
|
||||
"pool_max_size",
|
||||
"tcp_nodelay",
|
||||
"connect_timeout",
|
||||
"read_timeout",
|
||||
"timeout",
|
||||
"pool_idle_timeout",
|
||||
"tcp_keepalive",
|
||||
"tcp_keepalive_interval",
|
||||
"tcp_keepalive_retries",
|
||||
"allow_redirects",
|
||||
):
|
||||
if key in config.network:
|
||||
session_kwargs[key] = config.network[key]
|
||||
session_kwargs.update(kwargs)
|
||||
|
||||
# Connection-pool / timeout defaults applied only when neither config.network nor the caller set them.
|
||||
# connect_timeout + read_timeout mirror the requests-path default timeout (CONNECT_TIMEOUT, READ_TIMEOUT)
|
||||
# so an unset config still gets a bounded connect and read like requests; pool_idle_timeout stays under
|
||||
# the typical ~60s CDN idle kill; pool_max_idle_per_host matches the downloader worker cap to avoid
|
||||
# HTTP/1.1 connection churn; tcp_keepalive keeps long idle segments warm.
|
||||
session_kwargs.setdefault("connect_timeout", CONNECT_TIMEOUT)
|
||||
session_kwargs.setdefault("read_timeout", READ_TIMEOUT)
|
||||
session_kwargs.setdefault("pool_idle_timeout", 55)
|
||||
session_kwargs.setdefault("pool_max_idle_per_host", POOL_MAX_SIZE)
|
||||
session_kwargs.setdefault("tcp_keepalive", 30)
|
||||
|
||||
session_obj = RnetSession(**session_kwargs)
|
||||
session_obj.headers.update(config.headers)
|
||||
return session_obj
|
||||
|
||||
Reference in New Issue
Block a user