fix(downloader): honor Retry-After with cancel-safe backoff, add fault-injection bench and tests

This commit is contained in:
imSp4rky
2026-07-18 20:21:30 -06:00
parent 2c8b6774af
commit b54aadea7a
12 changed files with 1589 additions and 19 deletions
+26 -3
View File
@@ -11,9 +11,16 @@ from pathlib import Path
import pytest
from unshackle.core.downloaders.requests import (TAIL_BOOST_MIN_SEGMENT_SIZE, TAIL_BOOST_PART_SIZE,
AdaptiveWorkerController, _has_range_header, _plan_tail_parts,
_tail_boost_engages, requests)
from unshackle.core.downloaders.requests import (
TAIL_BOOST_MIN_SEGMENT_SIZE,
TAIL_BOOST_PART_SIZE,
AdaptiveWorkerController,
_has_range_header,
_plan_tail_parts,
_split_ranges,
_tail_boost_engages,
requests,
)
from unshackle.core.tracks.track import DownloadContext
pytestmark = pytest.mark.unit
@@ -202,6 +209,22 @@ def test_plan_tail_parts_skips_small_or_no_spare() -> None:
assert _plan_tail_parts(20 * 1024 * 1024, spare=0) == []
def test_split_ranges_covers_exactly_no_gaps_no_overlaps() -> None:
for size in (1, 100, 4 * 1024 * 1024, 16 * 1024 * 1024 + 7, 64 * 1024 * 1024):
parts = _split_ranges(size, max_parts=8, part_target=4 * 1024 * 1024)
assert parts[0][0] == 0 and parts[-1][1] == size - 1
for (_, prev_end), (start, _) in zip(parts, parts[1:]):
assert start == prev_end + 1
assert all(start <= end for start, end in parts)
assert sum(end - start + 1 for start, end in parts) == size
def test_split_ranges_respects_max_parts_and_target() -> None:
assert len(_split_ranges(100 * 1024 * 1024, max_parts=4, part_target=16 * 1024 * 1024)) == 4 # capped
assert len(_split_ranges(10 * 1024 * 1024, max_parts=8, part_target=16 * 1024 * 1024)) == 1 # below one target
assert _split_ranges(1, max_parts=8, part_target=16 * 1024 * 1024) == [(0, 0)]
def test_tail_boost_engages_on_idle_capacity() -> None:
# engages at the tail: idle workers outnumber the remaining whole segments
assert _tail_boost_engages(remaining=4, pending=2, target=8) # spare 6 >= 4
@@ -0,0 +1,180 @@
"""Integration coverage for the CONSUMING side of the DASH single-URL byte-range collapse.
The pure predicate ``DASH._collapsible_single_url`` is unit-tested in
``test_dash_collapse.py``; this file drives the real ``DASH.download_track`` code path that
acts on it. A minimal MPD is parsed with ``DASH.from_text().to_tracks()`` (so the track's
``data["dash"]`` is built by real parser code, not hand-faked) and downloaded against a
localhost server, asserting that:
- a SegmentList whose ranges collapse fetches the parent resource whole in one direct
(Range-less) request and writes a byte-correct file at ``save_path``, leaving no
segment dir or ``.!dev`` artifacts, and
- a track with mixed segment URLs still takes the per-segment download-and-merge path.
"""
from __future__ import annotations
import threading
from functools import partial
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from unshackle.core.constants import DOWNLOAD_CANCELLED
from unshackle.core.manifests import DASH
from unshackle.core.tracks.track import DownloadContext
# distinctive parent resource for the collapse case; init is its [0, INIT_LEN) prefix
PARENT = bytes(i % 256 for i in range(10240))
INIT_LEN = 512
# separate resources for the non-collapse (mixed-URL) case
INIT2 = bytes(range(100))
PART0 = bytes([0xA0]) * 300
PART1 = bytes([0xB1]) * 200
class _Handler(BaseHTTPRequestHandler):
def do_GET(self):
server = self.server
rng = self.headers.get("Range")
with server.lock:
server.requests.append((self.path, rng))
body = server.routes.get(self.path)
if body is None:
self.send_response(404)
self.send_header("Content-Length", "0")
self.end_headers()
return
if rng:
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
chunk = body[start : end + 1]
self.send_response(206)
self.send_header("Content-Range", f"bytes {start}-{end}/{len(body)}")
body = chunk
else:
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
@pytest.fixture
def server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
srv.lock = threading.Lock()
srv.requests = []
srv.routes = {}
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
srv.shutdown()
thread.join(timeout=5)
@pytest.fixture(autouse=True)
def _clear_cancel():
was_set = DOWNLOAD_CANCELLED.is_set()
DOWNLOAD_CANCELLED.clear()
yield
if was_set:
DOWNLOAD_CANCELLED.set()
else:
DOWNLOAD_CANCELLED.clear()
def _base(srv) -> str:
return f"http://127.0.0.1:{srv.server_address[1]}/"
def _collapse_mpd(srv) -> str:
# SegmentList on one media.mp4: init is bytes [0, INIT_LEN), media ranges cover the rest
# contiguously -> _collapsible_single_url returns the URL and download_track collapses.
return f"""<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
<BaseURL>{_base(srv)}</BaseURL>
<Period id="0">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
<BaseURL>media.mp4</BaseURL>
<SegmentList timescale="1">
<Initialization range="0-{INIT_LEN - 1}"/>
<SegmentURL mediaRange="{INIT_LEN}-5119"/>
<SegmentURL mediaRange="5120-{len(PARENT) - 1}"/>
</SegmentList>
</Representation>
</AdaptationSet>
</Period>
</MPD>"""
def _mixed_mpd(srv) -> str:
# Two SegmentURLs on distinct paths -> mixed URLs -> predicate False -> segmented path.
return f"""<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
<BaseURL>{_base(srv)}</BaseURL>
<Period id="0">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
<SegmentList timescale="1">
<Initialization sourceURL="init.bin"/>
<SegmentURL media="part0.bin"/>
<SegmentURL media="part1.bin"/>
</SegmentList>
</Representation>
</AdaptationSet>
</Period>
</MPD>"""
def _track(mpd: str, srv):
return DASH.from_text(mpd, url=f"{_base(srv)}manifest.mpd").to_tracks(language="en").videos[0]
def _ctx(tmp_path, name="out.mp4"):
return DownloadContext(
save_path=tmp_path / name,
save_dir=tmp_path / "v0_segments",
progress=partial(lambda **kw: None),
)
def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path):
server.routes["/media.mp4"] = PARENT
track = _track(_collapse_mpd(server), server)
ctx = _ctx(tmp_path, "collapsed.mp4")
DASH.download_track(track, ctx)
# the whole parent lands at save_path, byte-for-byte
assert ctx.save_path.read_bytes() == PARENT
assert track.path == ctx.save_path
# exactly one Range-less GET (the direct whole-resource download); the only other hit is
# the ranged init probe done while parsing SegmentList, never a per-segment slice fetch.
no_range = [path for path, rng in server.requests if rng is None]
assert no_range == ["/media.mp4"]
# collapse skips the segment dir + merge entirely: no dir, no leftover control files
assert not ctx.save_dir.exists()
assert not list(tmp_path.glob("*.!dev"))
assert not list(tmp_path.glob("**/*.!dev"))
def test_non_collapse_takes_segmented_merge_path(server, tmp_path):
server.routes["/init.bin"] = INIT2
server.routes["/part0.bin"] = PART0
server.routes["/part1.bin"] = PART1
track = _track(_mixed_mpd(server), server)
ctx = _ctx(tmp_path, "merged.mp4")
DASH.download_track(track, ctx)
# merged output is init followed by both segments, in order
assert ctx.save_path.read_bytes() == INIT2 + PART0 + PART1
# both distinct segments were fetched -> genuinely the per-segment path, not a collapse
fetched = {path for path, _ in server.requests}
assert {"/part0.bin", "/part1.bin"} <= fetched
# segment files are consumed and the segment dir is cleaned up after merge
assert not list(ctx.save_dir.glob("*")) if ctx.save_dir.exists() else True
+145
View File
@@ -0,0 +1,145 @@
"""Cancel/abort hardening for the downloader.
Pins two teardown-safety invariants:
- ``_dispatch_parts`` must NOT finalize when the process-global ``DOWNLOAD_CANCELLED``
fires mid-download: part workers return silently keeping partials, so every future
completes without error, and finalizing would strip the ``.!dev`` control file and
pass off a hole-filled pre-truncated file as complete on the next run.
- A worker parked in the retry backoff wait (which can reach MAX_BACKOFF) must wake and
exit as soon as the batch abort is set, instead of sleeping out the full delay and
stalling shutdown or the merge.
"""
import importlib
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from requests import Session
dl = importlib.import_module("unshackle.core.downloaders.requests")
pytestmark = pytest.mark.unit
BODY = b"x" * 4096
class _Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
server = self.server
if server.always_status:
self.send_response(server.always_status)
self.send_header("Content-Length", "0")
self.end_headers()
return
rng = self.headers.get("Range")
if rng:
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
body = BODY[start : end + 1]
self.send_response(206)
self.send_header("Content-Range", f"bytes {start}-{end}/{len(BODY)}")
else:
body = BODY
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if server.gate is not None:
self.wfile.write(body[:64])
self.wfile.flush()
server.started.set()
server.gate.wait(timeout=10)
self.wfile.write(body[64:])
else:
self.wfile.write(body)
def log_message(self, *args): # keep the test output quiet
pass
class _Server(ThreadingHTTPServer):
daemon_threads = True
def handle_error(self, request, client_address): # cancelled readers close mid-write
pass
@pytest.fixture
def server():
srv = _Server(("127.0.0.1", 0), _Handler)
srv.always_status = 0
srv.gate = None
srv.started = threading.Event()
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
if srv.gate is not None:
srv.gate.set()
srv.shutdown()
thread.join(timeout=5)
@pytest.fixture(autouse=True)
def _clear_cancel():
dl.DOWNLOAD_CANCELLED.clear()
yield
dl.DOWNLOAD_CANCELLED.clear()
def _url(server: _Server) -> str:
host, port = server.server_address
return f"http://{host}:{port}/seg.bin"
def test_dispatch_parts_does_not_finalize_on_global_cancel(server, tmp_path):
server.gate = threading.Event()
save_path = tmp_path / "big.bin"
def _cancel_when_started() -> None:
assert server.started.wait(timeout=10)
dl.DOWNLOAD_CANCELLED.set()
server.gate.set()
canceller = threading.Thread(target=_cancel_when_started)
canceller.start()
events = list(
dl._dispatch_parts(
url=_url(server),
save_path=save_path,
session=Session(),
total_size=len(BODY),
max_workers=2,
)
)
canceller.join(timeout=10)
assert not any("file_downloaded" in ev for ev in events)
# the control file must survive so the next run restarts instead of trusting the file
assert save_path.with_name(f"{save_path.name}.!dev").exists()
def test_backoff_wait_exits_promptly_on_batch_abort(server, tmp_path, monkeypatch):
server.always_status = 503
monkeypatch.setattr(dl, "_retry_sleep", lambda exc, attempts: 30.0)
abort = threading.Event()
def _consume() -> None:
for _ in dl.download(
url=_url(server),
save_path=tmp_path / "seg.bin",
session=Session(),
segmented=True,
abort=abort,
):
pass
worker = threading.Thread(target=_consume)
start = time.monotonic()
worker.start()
time.sleep(0.5) # let the first attempt fail and enter the backoff wait
abort.set()
worker.join(timeout=10)
assert not worker.is_alive()
# generous bound: far below the 30s nap, so the wait was interrupted, not slept out
assert time.monotonic() - start < 10
@@ -9,7 +9,10 @@ media requests must ask for identity encoding.
import importlib
import threading
from datetime import datetime, timedelta, timezone
from email.utils import format_datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from types import SimpleNamespace
import pytest
@@ -26,6 +29,12 @@ class _Handler(BaseHTTPRequestHandler):
with server.lock:
server.requests.append(dict(self.headers))
nth = len(server.requests)
if nth <= server.fail_429_first:
self.send_response(429)
self.send_header("Retry-After", "0")
self.send_header("Content-Length", "0")
self.end_headers()
return
rng = self.headers.get("Range")
if server.honor_range and rng:
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
@@ -53,6 +62,7 @@ def server():
srv.requests = []
srv.honor_range = True
srv.truncate_first = False
srv.fail_429_first = 0
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
@@ -120,3 +130,64 @@ def test_range_item_retry_rewrites_slice(server, tmp_path, monkeypatch):
def test_segment_requests_ask_identity_encoding(server, tmp_path):
_run(server, tmp_path, [{"url": _url(server)}], max_workers=1)
assert all(r.get("Accept-Encoding") == "identity" for r in server.requests)
def _exc_with_headers(headers):
exc = Exception("boom")
exc.response = SimpleNamespace(headers=headers)
return exc
def test_retry_sleep_honors_numeric_retry_after():
assert dl._retry_sleep(_exc_with_headers({"Retry-After": "3"}), 1) == 3.0
assert dl._retry_sleep(_exc_with_headers({"Retry-After": "9999"}), 1) == 60.0 # session.MAX_BACKOFF cap
def test_retry_sleep_honors_http_date_retry_after():
when = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=10), usegmt=True)
assert 5 <= dl._retry_sleep(_exc_with_headers({"Retry-After": when}), 1) <= 10
def test_retry_sleep_exponential_backoff_with_jitter():
# no response on the failure: exponential from RETRY_WAIT with up to 10% jitter
for attempts, base in ((1, dl.RETRY_WAIT), (2, dl.RETRY_WAIT * 2), (3, dl.RETRY_WAIT * 4)):
assert base <= dl._retry_sleep(Exception("reset"), attempts) <= base * 1.1
assert dl._retry_sleep(Exception("reset"), 30) == 60.0 # capped
def test_retry_sleep_invalid_retry_after_falls_back():
v = dl._retry_sleep(_exc_with_headers({"Retry-After": "soon"}), 1)
assert dl.RETRY_WAIT <= v <= dl.RETRY_WAIT * 1.1
def test_retry_sleep_naive_http_date_treated_as_utc():
# "-0000" makes parsedate_to_datetime return a naive datetime; must clamp, not raise
when = format_datetime(datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=10))
assert when.endswith("-0000")
v = dl._retry_sleep(_exc_with_headers({"Retry-After": when}), 1)
assert 5 <= v <= 10
def test_retry_sleep_nan_retry_after_falls_back():
# float("nan") parses; a non-finite wait must fall back to the exponential path
v = dl._retry_sleep(_exc_with_headers({"Retry-After": "nan"}), 1)
assert dl.RETRY_WAIT <= v <= dl.RETRY_WAIT * 1.1
def test_retry_sleep_honors_retry_after_on_wrapped_cause():
# RnetSession raises MaxRetriesError whose __cause__ HTTPError carries the response
exc = Exception("max retries exceeded")
exc.__cause__ = _exc_with_headers({"Retry-After": "3"})
assert dl._retry_sleep(exc, 1) == 3.0
def test_transient_429_with_retry_after_does_not_abort_track(server, tmp_path, monkeypatch):
# a short rate-limit window must retry the segment, not kill the whole batch
dl.DOWNLOAD_CANCELLED.clear()
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
server.fail_429_first = 2
files = _run(server, tmp_path, [{"url": _url(server, f"/seg{i}.bin")} for i in range(3)], max_workers=1)
assert len(files) == 3
assert all(f.read_bytes() == PARENT for f in files)
assert not dl.DOWNLOAD_CANCELLED.is_set()
assert len(server.requests) == 5 # 3 successes + 2 rate-limited attempts
+156
View File
@@ -0,0 +1,156 @@
"""Multiprocess segment-download invariants.
Pins the spawned-child fan-out path in ``requests()``/``_download_multiprocess``:
- Strided assignment plus the ``index_offset``/``index_stride`` filename mapping puts every
segment's bytes in the correctly named file (a broken stride mapping misfiles content).
- The parent normalizes child progress to segment granularity: it drops child ``advance``
and ``total``/speed events and emits exactly one ``advance=1`` per ``file_downloaded``, so
the advances summed by a caller equal the segment count.
- A child that dies (its worker sends ``__mp_error__``) surfaces as a ``RuntimeError`` from
the generator without hanging.
- A process-global ``DOWNLOAD_CANCELLED`` set mid-batch terminates the spawned children (which
carry their own fresh flag and can't see it) and the teardown join reaps them promptly.
Children are spawned and re-import the module, so monkeypatched module attributes do NOT reach
them; these tests drive real child processes against a localhost server on 127.0.0.1 (reachable
from the children) and avoid any child-side patching. Segments are a few hundred bytes so the
real spawns stay fast.
"""
import importlib
import multiprocessing
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
dl = importlib.import_module("unshackle.core.downloaders.requests")
SEG_REPEAT = 32 # 8-digit index * 32 = 256-byte body, distinct per index
def _seg_body(index: int) -> bytes:
"""Segment body that encodes its own index, so misfiled striding is detectable."""
return (f"{index:08d}" * SEG_REPEAT).encode()
class _Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
server = self.server
delay = getattr(server, "delay", 0.0)
if delay:
time.sleep(delay)
try:
index = int(self.path.rsplit("/", 1)[1].split(".")[0])
except ValueError:
self.send_response(404)
self.send_header("Content-Length", "0")
self.end_headers()
return
body = _seg_body(index)
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args): # keep the test output quiet
pass
class _Server(ThreadingHTTPServer):
daemon_threads = True
def handle_error(self, request, client_address): # a cancelled child closes its socket mid-write
pass
@pytest.fixture
def server():
srv = _Server(("127.0.0.1", 0), _Handler)
srv.delay = 0.0
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
srv.shutdown()
thread.join(timeout=5)
@pytest.fixture(autouse=True)
def _clear_cancel():
# spawned children carry their own flag, but the parent loop reads this process's flag;
# keep it clean so cancel state never leaks between tests
dl.DOWNLOAD_CANCELLED.clear()
yield
dl.DOWNLOAD_CANCELLED.clear()
def _url(srv, path):
return f"http://127.0.0.1:{srv.server_address[1]}{path}"
def test_multiprocess_stride_downloads_all_segments_correctly(server, tmp_path):
# >= MP_MIN_SEGMENTS with processes=2 engages the fan-out via the public entry point
n = dl.MP_MIN_SEGMENTS
urls = [{"url": _url(server, f"/seg/{i}.bin")} for i in range(n)]
total_advance = 0
for event in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
total_advance += event.get("advance", 0)
files = sorted(tmp_path.glob("seg_*.bin"))
assert len(files) == n
for f in files:
index = int(f.stem.split("_")[1])
assert f.read_bytes() == _seg_body(index), f"segment {index} holds another index's bytes (stride mapping bug)"
# parent drops child advances and emits one advance=1 per file_downloaded
assert total_advance == n
def test_multiprocess_child_crash_raises(server, tmp_path):
# _download_multiprocess has no MP_MIN_SEGMENTS gate, so drive it directly with 2 urls.
# a dict url without "url" raises KeyError while requests() builds the save path (before
# any retry loop), so the child sends __mp_error__ fast; the sibling downloads normally.
good = {"url": _url(server, "/seg/0.bin")}
bad = {"nourl": True}
with pytest.raises(RuntimeError):
for _ in dl._download_multiprocess(
urls=[good, bad],
output_dir=tmp_path,
filename="seg_{i:04}.bin",
headers=None,
cookies=None,
proxy=None,
max_workers=2,
adaptive=False,
spec={"kind": "none"},
processes=2,
debug_logger=None,
):
pass
def test_multiprocess_cancel_terminates_children(server, tmp_path):
server.delay = 0.5 # slow segments so cancel lands well before the batch could finish
n = dl.MP_MIN_SEGMENTS
urls = [{"url": _url(server, f"/seg/{i}.bin")} for i in range(n)]
baseline = len(multiprocessing.active_children())
start = time.time()
progressed = 0
for event in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
if event.get("advance") or event.get("written"):
progressed += 1
if progressed >= 2:
dl.DOWNLOAD_CANCELLED.set() # sibling-track cancel: parent must tear the children down
elapsed = time.time() - start
# cancel stopped the work: nowhere near all segments finished, and no hang
assert len(list(tmp_path.glob("seg_*.bin"))) < n
assert elapsed < 15
# the finally-block join reaps the spawned children promptly
deadline = time.time() + 5
while time.time() < deadline and len(multiprocessing.active_children()) > baseline:
time.sleep(0.05)
assert len(multiprocessing.active_children()) <= baseline
+164
View File
@@ -0,0 +1,164 @@
"""Ranged-parallel single-URL download invariants.
Pins the byte-range fan-out that splits one large file across parts:
- a range-capable server above the size threshold is split into several byte
windows, the parts merge byte-identical, and the ``.!dev`` control marker is
removed on success;
- a ranged part that receives a 200 (server ignored the Range) fails after
retries rather than writing a wrong-length body into the pre-truncated file;
- that failure sets only the ranged download's LOCAL abort event, never the
process-global DOWNLOAD_CANCELLED (which would poison sibling tracks);
- ``_probe_ranged`` parses the Content-Range total on a 206 and declines a 200
or a content-encoded 206.
Tests 2/3 exercise ``_dispatch_parts`` directly: the public single-URL path in
``requests()`` deliberately swallows a ranged failure and falls back to a plain
sequential download, so the raise the invariant guarantees is only observable at
the dispatch layer.
"""
import importlib
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from requests import Session
dl = importlib.import_module("unshackle.core.downloaders.requests")
PAYLOAD = bytes(range(256)) * 128 # 32 KiB, distinctive repeating byte pattern
class _RangeHandler(BaseHTTPRequestHandler):
def do_GET(self):
srv = self.server
rng = self.headers.get("Range")
with srv.lock:
srv.requests.append(rng)
mode = srv.mode
if mode == "gzip_probe":
# a 206 that (wrongly) carries a content-encoding: the probe must decline it
self.send_response(206)
self.send_header("Content-Range", f"bytes 0-0/{len(PAYLOAD)}")
self.send_header("Content-Encoding", "gzip")
self.send_header("Content-Length", "1")
self.end_headers()
self.wfile.write(b"\x00")
return
# always_200 ignores Range entirely; probe_only_206 answers the 0-0 probe
# with a real 206 but hands every wider part window a 200 full body
serve_full = mode == "always_200" or not rng or (mode == "probe_only_206" and rng != "bytes=0-0")
if serve_full:
self.send_response(200)
self.send_header("Content-Length", str(len(PAYLOAD)))
self.end_headers()
self.wfile.write(PAYLOAD)
return
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
body = PAYLOAD[start : end + 1]
self.send_response(206)
self.send_header("Content-Range", f"bytes {start}-{end}/{len(PAYLOAD)}")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
@pytest.fixture(autouse=True)
def _clear_cancelled():
# ranged failures must never touch the process-global flag; isolate every test from it
dl.DOWNLOAD_CANCELLED.clear()
yield
dl.DOWNLOAD_CANCELLED.clear()
@pytest.fixture
def server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _RangeHandler)
srv.lock = threading.Lock()
srv.requests = []
srv.mode = "range" # honor every Range with a proper 206
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
srv.shutdown()
thread.join(timeout=5)
def _url(srv, path="/payload.bin"):
return f"http://127.0.0.1:{srv.server_address[1]}{path}"
def _run(srv, tmp_path, urls, **kwargs):
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", **kwargs):
pass
return sorted(tmp_path.glob("seg_*.bin"))
def test_ranged_parallel_merges_byte_identical(server, tmp_path, monkeypatch):
# shrink the thresholds so a 32 KiB payload splits into several parts
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 2 * 1024)
files = _run(server, tmp_path, [{"url": _url(server)}], max_workers=4)
assert len(files) == 1
assert files[0].read_bytes() == PAYLOAD
# the split must actually have happened: >1 distinct window beyond the 0-0 probe
windows = {r for r in server.requests if r and r != "bytes=0-0"}
assert len(windows) >= 2
# control marker removed once the parts finalize
assert not files[0].with_name(files[0].name + ".!dev").exists()
def test_ranged_part_requires_206(server, tmp_path, monkeypatch):
server.mode = "probe_only_206"
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
save_path = tmp_path / "seg_0000.bin"
gen = dl._dispatch_parts(
url=_url(server),
save_path=save_path,
session=Session(),
total_size=len(PAYLOAD),
max_workers=4,
)
with pytest.raises(IOError, match="expected 206 for ranged part"):
for _ in gen:
pass
# not finalized: the control marker survives (completed=False) and the target
# is still the pre-truncated zero-fill, never the served bytes
assert save_path.with_name(save_path.name + ".!dev").exists()
assert save_path.read_bytes() != PAYLOAD
def test_ranged_part_failure_stays_local(server, tmp_path, monkeypatch):
server.mode = "probe_only_206"
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
dl.DOWNLOAD_CANCELLED.clear()
save_path = tmp_path / "seg_0000.bin"
gen = dl._dispatch_parts(
url=_url(server),
save_path=save_path,
session=Session(),
total_size=len(PAYLOAD),
max_workers=4,
)
with pytest.raises(IOError, match="expected 206 for ranged part"):
for _ in gen:
pass
# a failed part rides the local abort event; the global cancel must stay clear
assert not dl.DOWNLOAD_CANCELLED.is_set()
def test_probe_ranged_parses_content_range(server):
session = Session()
server.mode = "range"
assert dl._probe_ranged(_url(server), session) == (len(PAYLOAD), True)
server.mode = "always_200"
assert dl._probe_ranged(_url(server), session) == (0, False)
server.mode = "gzip_probe"
assert dl._probe_ranged(_url(server), session) == (0, False)
+175
View File
@@ -0,0 +1,175 @@
"""Tail-boost success-path invariants.
Pins the tail-boost engage path in ``requests()``: when only a few segments remain
and workers would otherwise idle, each remaining segment is probed for range support
and split into intra-segment byte-range parts written into a pre-truncated ``.tp.!dev``
target, finalized by the last part worker via os.replace. The invariants:
- a boosted segment's final file is byte-identical to the served content, every
segment reports exactly one advance=1 (part-mode byte advances are swallowed), and
a clean run leaves no ``.!dev`` / ``.tp.!dev`` strays;
- finalize is last-part-wins: on success save_path exists and the ``.tp.!dev`` target
is gone;
- a permanently failing part fails the batch and the boosted segment is never
os.replace'd into place (no corrupt finalized segment).
Determinism: with ``max_workers`` < segment count <= 2*max_workers and adaptive=True,
the leading ``max_workers`` segments submit upfront and the trailing few stay in
``remaining`` (the tail top-up only releases boost-declined indices). The controller
starts at the cap and never grows, so as soon as two leaders finish the spare workers
outnumber the tail and ``_tail_boost_engages`` fires. Module timing/size constants are
lowered via monkeypatch; the whole batch runs in-process (threads), so the patched
values are seen by the workers.
"""
import importlib
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
dl = importlib.import_module("unshackle.core.downloaders.requests")
BODY = bytes((i * 7 + 11) % 256 for i in range(16384)) # 16 KiB, distinctive, range-sliceable
# lowered so a 16 KiB segment clears the min and splits into several 4 KiB parts
BOOST_MIN = 4096
BOOST_PART = 4096
MAX_WORKERS = 6
SEG_COUNT = 8 # > MAX_WORKERS and <= 2*MAX_WORKERS: leaders 0-5 submit, tail 6,7 boost
class _Handler(BaseHTTPRequestHandler):
def do_GET(self):
server = self.server
rng = self.headers.get("Range")
with server.lock:
server.requests.append((self.path, rng))
if rng and rng.startswith("bytes="):
start_s, _, end_s = rng[len("bytes=") :].partition("-")
start = int(start_s)
end = int(end_s) if end_s else len(BODY) - 1
if server.break_offset_ranges and start > 0:
# misbehave for every non-leading window: send the whole body as 200 so
# part-mode's 206 check fails permanently for that part
self.send_response(200)
self.send_header("Content-Length", str(len(BODY)))
self.end_headers()
self.wfile.write(BODY)
return
body = BODY[start : end + 1]
self.send_response(206)
self.send_header("Content-Range", f"bytes {start}-{end}/{len(BODY)}")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_response(200)
self.send_header("Content-Length", str(len(BODY)))
self.end_headers()
self.wfile.write(BODY)
def log_message(self, *args):
pass
@pytest.fixture
def server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
srv.lock = threading.Lock()
srv.requests = []
srv.break_offset_ranges = False
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
srv.shutdown()
thread.join(timeout=5)
@pytest.fixture(autouse=True)
def _clean_cancel():
# a failing batch sets the process-global cancel; keep tests independent
dl.DOWNLOAD_CANCELLED.clear()
yield
dl.DOWNLOAD_CANCELLED.clear()
@pytest.fixture(autouse=True)
def _boost_constants(monkeypatch):
monkeypatch.setattr(dl, "TAIL_BOOST_MIN_SEGMENT_SIZE", BOOST_MIN)
monkeypatch.setattr(dl, "TAIL_BOOST_PART_SIZE", BOOST_PART)
def _url(srv, path):
return f"http://127.0.0.1:{srv.server_address[1]}{path}"
def _urls(srv):
return [{"url": _url(srv, f"/seg{i}.bin")} for i in range(SEG_COUNT)]
def _run(srv, tmp_path, urls):
advances = 0
for ev in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=MAX_WORKERS, adaptive=True):
a = ev.get("advance")
if a:
advances += a
return sorted(tmp_path.glob("seg_*.bin")), advances
def _ranged_counts(srv):
with srv.lock:
reqs = list(srv.requests)
counts: dict[str, int] = {}
for path, rng in reqs:
if rng:
counts[path] = counts.get(path, 0) + 1
return counts
def _boost_engaged(srv):
# a boosted path sees a 0-0 probe plus >=2 part windows; a normal segment sees no Range
return any(n > 1 for n in _ranged_counts(srv).values())
def test_tail_boost_split_produces_byte_identical_segment(server, tmp_path):
files, advances = _run(server, tmp_path, _urls(server))
assert _boost_engaged(server), "tail boost never engaged; test would silently cover the normal path"
assert len(files) == SEG_COUNT
assert all(f.read_bytes() == BODY for f in files)
# part-mode byte advances are swallowed; each segment reports exactly one advance=1
assert advances == SEG_COUNT
# a clean run renames every target into place, so no partial/part strays survive
assert not list(tmp_path.glob("*.!dev"))
assert not list(tmp_path.glob("*.tp.!dev"))
def test_tail_boost_finalize_last_part_wins(server, tmp_path):
files, _ = _run(server, tmp_path, _urls(server))
assert _boost_engaged(server), "tail boost never engaged; finalize path untested"
assert len(files) == SEG_COUNT
# last part wins: the pre-truncated .tp.!dev is os.replace'd into save_path, leaving no target
assert not list(tmp_path.glob("*.tp.!dev"))
for path in _ranged_counts(server):
i = int(path.removeprefix("/seg").removesuffix(".bin"))
save_path = tmp_path / f"seg_{i:04}.bin"
assert save_path.exists()
assert not save_path.with_name(f"{save_path.name}.tp.!dev").exists()
def test_tail_boost_part_failure_fails_batch_but_keeps_siblings_files_clean(server, tmp_path, monkeypatch):
server.break_offset_ranges = True # every non-leading part window 200s -> permanent 206 failure
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
with pytest.raises(Exception):
_run(server, tmp_path, _urls(server))
# the boosted tail segments (6, 7) hit the failing windows: no part run reaches parts_left==0
# cleanly, so neither is finalized. sibling leaders that did finalize must be uncorrupted.
for i in (6, 7):
assert not (tmp_path / f"seg_{i:04}.bin").exists()
for f in tmp_path.glob("seg_*.bin"):
assert f.read_bytes() == BODY