mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-15 21:45:10 +00:00
fix(downloader): bound requests after a cut and stop retry nesting
Request 4 MiB sub-ranges once a host cuts a body, skip the session's own urllib3 Retry inside download(), fail the ranged path at once on a 4xx, and cap resumes at 50 of 1 MiB each. Adds a deny fault profile.
This commit is contained in:
@@ -10,12 +10,14 @@ Pins the byte-range fan-out that splits one large file across parts:
|
||||
- that failure sets only the ranged download's LOCAL abort event, never the
|
||||
process-global DOWNLOAD_CANCELLED (which would poison sibling tracks);
|
||||
- ``probe_ranged`` parses the Content-Range total on a 206 and declines a 200
|
||||
or a content-encoded 206.
|
||||
or a content-encoded 206;
|
||||
- the public single-URL path falls back to a sequential download only when the
|
||||
server does not really support ranges, and re-raises a 4xx instead of paying
|
||||
MAX_ATTEMPTS a second time from byte 0.
|
||||
|
||||
Tests 2/3 exercise ``dispatch_parts`` directly: the public single-URL path in
|
||||
``requests()`` deliberately swallows a ranged failure and falls back to a plain
|
||||
sequential download, so the raise the invariant guarantees is only observable at
|
||||
the dispatch layer.
|
||||
Tests 2/3 exercise ``dispatch_parts`` directly: the public path swallows a
|
||||
range-support failure and falls back, so the raise that the invariant guarantees
|
||||
is only observable at the dispatch layer.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
@@ -23,13 +25,18 @@ import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
from requests import Session
|
||||
from requests import HTTPError, Session
|
||||
|
||||
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
PAYLOAD = bytes(range(256)) * 128 # 32 KiB, distinctive repeating byte pattern
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code):
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class _RangeHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
srv = self.server
|
||||
@@ -48,6 +55,13 @@ class _RangeHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(b"\x00")
|
||||
return
|
||||
|
||||
if mode == "deny_parts" and rng != "bytes=0-0":
|
||||
# probe passes, every part window is refused for good (expired token, geo block)
|
||||
self.send_response(403)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# always_200 ignores Range entirely; probe_only_206 answers the 0-0 probe
|
||||
# with a real 206 but hands every wider part window a 200 full body
|
||||
serve_full = mode == "always_200" or not rng or (mode == "probe_only_206" and rng != "bytes=0-0")
|
||||
@@ -162,3 +176,41 @@ def test_probe_ranged_parses_content_range(server):
|
||||
assert dl.probe_ranged(url(server), session) == (0, False)
|
||||
server.mode = "gzip_probe"
|
||||
assert dl.probe_ranged(url(server), session) == (0, False)
|
||||
|
||||
|
||||
def test_permanent_part_failure_is_not_retried_sequentially(server, tmp_path, monkeypatch):
|
||||
server.mode = "deny_parts"
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 8 * 1024)
|
||||
with pytest.raises(HTTPError):
|
||||
run(server, tmp_path, [{"url": url(server)}], max_workers=4)
|
||||
# a sequential fallback would issue a plain unranged GET; only the probe and the
|
||||
# part windows may appear, each part capped at MAX_ATTEMPTS
|
||||
assert [r for r in server.requests if r is None] == []
|
||||
parts = len({r for r in server.requests if r and r != "bytes=0-0"})
|
||||
assert len(server.requests) <= 1 + parts * dl.MAX_ATTEMPTS
|
||||
# nothing half-written is left behind for a later run to mistake for a finished file
|
||||
assert sorted(tmp_path.glob("seg_*")) == []
|
||||
|
||||
|
||||
def test_range_unsupported_falls_back_to_sequential(server, tmp_path, monkeypatch):
|
||||
server.mode = "probe_only_206"
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 8 * 1024)
|
||||
files = run(server, tmp_path, [{"url": url(server)}], max_workers=4)
|
||||
assert len(files) == 1
|
||||
assert files[0].read_bytes() == PAYLOAD
|
||||
assert not files[0].with_name(files[0].name + ".!dev").exists()
|
||||
|
||||
|
||||
def test_permanent_status_unwraps_causes():
|
||||
assert dl.permanent_status(HTTPError(response=_Resp(403))) == 403
|
||||
assert dl.permanent_status(HTTPError(response=_Resp(429))) is None
|
||||
assert dl.permanent_status(HTTPError(response=_Resp(503))) is None
|
||||
assert dl.permanent_status(IOError("short read")) is None
|
||||
# RnetSession wraps the HTTPError one level down as __cause__
|
||||
wrapped = RuntimeError("max retries")
|
||||
wrapped.__cause__ = HTTPError(response=_Resp(404))
|
||||
assert dl.permanent_status(wrapped) == 404
|
||||
|
||||
Reference in New Issue
Block a user