fix(downloader): bound requests after a cut and stop retry nesting

Request 4 MiB sub-ranges once a host cuts a body, skip the session's own urllib3 Retry inside download(), fail the ranged path at once on a 4xx, and cap resumes at 50 of 1 MiB each. Adds a deny fault profile.
This commit is contained in:
imSp4rky
2026-08-24 15:10:36 -06:00
parent e5e1ce14c6
commit d4690e5c6b
5 changed files with 316 additions and 21 deletions
+58 -6
View File
@@ -10,12 +10,14 @@ Pins the byte-range fan-out that splits one large file across parts:
- that failure sets only the ranged download's LOCAL abort event, never the
process-global DOWNLOAD_CANCELLED (which would poison sibling tracks);
- ``probe_ranged`` parses the Content-Range total on a 206 and declines a 200
or a content-encoded 206.
or a content-encoded 206;
- the public single-URL path falls back to a sequential download only when the
server does not really support ranges, and re-raises a 4xx instead of paying
MAX_ATTEMPTS a second time from byte 0.
Tests 2/3 exercise ``dispatch_parts`` directly: the public single-URL path in
``requests()`` deliberately swallows a ranged failure and falls back to a plain
sequential download, so the raise the invariant guarantees is only observable at
the dispatch layer.
Tests 2/3 exercise ``dispatch_parts`` directly: the public path swallows a
range-support failure and falls back, so the raise that the invariant guarantees
is only observable at the dispatch layer.
"""
import importlib
@@ -23,13 +25,18 @@ import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from requests import Session
from requests import HTTPError, Session
dl = importlib.import_module("unshackle.core.downloaders.requests")
PAYLOAD = bytes(range(256)) * 128 # 32 KiB, distinctive repeating byte pattern
class _Resp:
def __init__(self, status_code):
self.status_code = status_code
class _RangeHandler(BaseHTTPRequestHandler):
def do_GET(self):
srv = self.server
@@ -48,6 +55,13 @@ class _RangeHandler(BaseHTTPRequestHandler):
self.wfile.write(b"\x00")
return
if mode == "deny_parts" and rng != "bytes=0-0":
# probe passes, every part window is refused for good (expired token, geo block)
self.send_response(403)
self.send_header("Content-Length", "0")
self.end_headers()
return
# always_200 ignores Range entirely; probe_only_206 answers the 0-0 probe
# with a real 206 but hands every wider part window a 200 full body
serve_full = mode == "always_200" or not rng or (mode == "probe_only_206" and rng != "bytes=0-0")
@@ -162,3 +176,41 @@ def test_probe_ranged_parses_content_range(server):
assert dl.probe_ranged(url(server), session) == (0, False)
server.mode = "gzip_probe"
assert dl.probe_ranged(url(server), session) == (0, False)
def test_permanent_part_failure_is_not_retried_sequentially(server, tmp_path, monkeypatch):
server.mode = "deny_parts"
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 8 * 1024)
with pytest.raises(HTTPError):
run(server, tmp_path, [{"url": url(server)}], max_workers=4)
# a sequential fallback would issue a plain unranged GET; only the probe and the
# part windows may appear, each part capped at MAX_ATTEMPTS
assert [r for r in server.requests if r is None] == []
parts = len({r for r in server.requests if r and r != "bytes=0-0"})
assert len(server.requests) <= 1 + parts * dl.MAX_ATTEMPTS
# nothing half-written is left behind for a later run to mistake for a finished file
assert sorted(tmp_path.glob("seg_*")) == []
def test_range_unsupported_falls_back_to_sequential(server, tmp_path, monkeypatch):
server.mode = "probe_only_206"
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 8 * 1024)
files = run(server, tmp_path, [{"url": url(server)}], max_workers=4)
assert len(files) == 1
assert files[0].read_bytes() == PAYLOAD
assert not files[0].with_name(files[0].name + ".!dev").exists()
def test_permanent_status_unwraps_causes():
assert dl.permanent_status(HTTPError(response=_Resp(403))) == 403
assert dl.permanent_status(HTTPError(response=_Resp(429))) is None
assert dl.permanent_status(HTTPError(response=_Resp(503))) is None
assert dl.permanent_status(IOError("short read")) is None
# RnetSession wraps the HTTPError one level down as __cause__
wrapped = RuntimeError("max retries")
wrapped.__cause__ = HTTPError(response=_Resp(404))
assert dl.permanent_status(wrapped) == 404