mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-19 07:25:11 +00:00
fix(downloader): honor Retry-After with cancel-safe backoff, add fault-injection bench and tests
This commit is contained in:
@@ -11,9 +11,16 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.downloaders.requests import (TAIL_BOOST_MIN_SEGMENT_SIZE, TAIL_BOOST_PART_SIZE,
|
||||
AdaptiveWorkerController, _has_range_header, _plan_tail_parts,
|
||||
_tail_boost_engages, requests)
|
||||
from unshackle.core.downloaders.requests import (
|
||||
TAIL_BOOST_MIN_SEGMENT_SIZE,
|
||||
TAIL_BOOST_PART_SIZE,
|
||||
AdaptiveWorkerController,
|
||||
_has_range_header,
|
||||
_plan_tail_parts,
|
||||
_split_ranges,
|
||||
_tail_boost_engages,
|
||||
requests,
|
||||
)
|
||||
from unshackle.core.tracks.track import DownloadContext
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
@@ -202,6 +209,22 @@ def test_plan_tail_parts_skips_small_or_no_spare() -> None:
|
||||
assert _plan_tail_parts(20 * 1024 * 1024, spare=0) == []
|
||||
|
||||
|
||||
def test_split_ranges_covers_exactly_no_gaps_no_overlaps() -> None:
|
||||
for size in (1, 100, 4 * 1024 * 1024, 16 * 1024 * 1024 + 7, 64 * 1024 * 1024):
|
||||
parts = _split_ranges(size, max_parts=8, part_target=4 * 1024 * 1024)
|
||||
assert parts[0][0] == 0 and parts[-1][1] == size - 1
|
||||
for (_, prev_end), (start, _) in zip(parts, parts[1:]):
|
||||
assert start == prev_end + 1
|
||||
assert all(start <= end for start, end in parts)
|
||||
assert sum(end - start + 1 for start, end in parts) == size
|
||||
|
||||
|
||||
def test_split_ranges_respects_max_parts_and_target() -> None:
|
||||
assert len(_split_ranges(100 * 1024 * 1024, max_parts=4, part_target=16 * 1024 * 1024)) == 4 # capped
|
||||
assert len(_split_ranges(10 * 1024 * 1024, max_parts=8, part_target=16 * 1024 * 1024)) == 1 # below one target
|
||||
assert _split_ranges(1, max_parts=8, part_target=16 * 1024 * 1024) == [(0, 0)]
|
||||
|
||||
|
||||
def test_tail_boost_engages_on_idle_capacity() -> None:
|
||||
# engages at the tail: idle workers outnumber the remaining whole segments
|
||||
assert _tail_boost_engages(remaining=4, pending=2, target=8) # spare 6 >= 4
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Integration coverage for the CONSUMING side of the DASH single-URL byte-range collapse.
|
||||
|
||||
The pure predicate ``DASH._collapsible_single_url`` is unit-tested in
|
||||
``test_dash_collapse.py``; this file drives the real ``DASH.download_track`` code path that
|
||||
acts on it. A minimal MPD is parsed with ``DASH.from_text().to_tracks()`` (so the track's
|
||||
``data["dash"]`` is built by real parser code, not hand-faked) and downloaded against a
|
||||
localhost server, asserting that:
|
||||
|
||||
- a SegmentList whose ranges collapse fetches the parent resource whole in one direct
|
||||
(Range-less) request and writes a byte-correct file at ``save_path``, leaving no
|
||||
segment dir or ``.!dev`` artifacts, and
|
||||
- a track with mixed segment URLs still takes the per-segment download-and-merge path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
from unshackle.core.manifests import DASH
|
||||
from unshackle.core.tracks.track import DownloadContext
|
||||
|
||||
# distinctive parent resource for the collapse case; init is its [0, INIT_LEN) prefix
|
||||
PARENT = bytes(i % 256 for i in range(10240))
|
||||
INIT_LEN = 512
|
||||
|
||||
# separate resources for the non-collapse (mixed-URL) case
|
||||
INIT2 = bytes(range(100))
|
||||
PART0 = bytes([0xA0]) * 300
|
||||
PART1 = bytes([0xB1]) * 200
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
server = self.server
|
||||
rng = self.headers.get("Range")
|
||||
with server.lock:
|
||||
server.requests.append((self.path, rng))
|
||||
body = server.routes.get(self.path)
|
||||
if body is None:
|
||||
self.send_response(404)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
if rng:
|
||||
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
|
||||
chunk = body[start : end + 1]
|
||||
self.send_response(206)
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{len(body)}")
|
||||
body = chunk
|
||||
else:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
||||
srv.lock = threading.Lock()
|
||||
srv.requests = []
|
||||
srv.routes = {}
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cancel():
|
||||
was_set = DOWNLOAD_CANCELLED.is_set()
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
if was_set:
|
||||
DOWNLOAD_CANCELLED.set()
|
||||
else:
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def _base(srv) -> str:
|
||||
return f"http://127.0.0.1:{srv.server_address[1]}/"
|
||||
|
||||
|
||||
def _collapse_mpd(srv) -> str:
|
||||
# SegmentList on one media.mp4: init is bytes [0, INIT_LEN), media ranges cover the rest
|
||||
# contiguously -> _collapsible_single_url returns the URL and download_track collapses.
|
||||
return f"""<?xml version="1.0"?>
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
|
||||
<BaseURL>{_base(srv)}</BaseURL>
|
||||
<Period id="0">
|
||||
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
|
||||
<Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
|
||||
<BaseURL>media.mp4</BaseURL>
|
||||
<SegmentList timescale="1">
|
||||
<Initialization range="0-{INIT_LEN - 1}"/>
|
||||
<SegmentURL mediaRange="{INIT_LEN}-5119"/>
|
||||
<SegmentURL mediaRange="5120-{len(PARENT) - 1}"/>
|
||||
</SegmentList>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>"""
|
||||
|
||||
|
||||
def _mixed_mpd(srv) -> str:
|
||||
# Two SegmentURLs on distinct paths -> mixed URLs -> predicate False -> segmented path.
|
||||
return f"""<?xml version="1.0"?>
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
|
||||
<BaseURL>{_base(srv)}</BaseURL>
|
||||
<Period id="0">
|
||||
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
|
||||
<Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
|
||||
<SegmentList timescale="1">
|
||||
<Initialization sourceURL="init.bin"/>
|
||||
<SegmentURL media="part0.bin"/>
|
||||
<SegmentURL media="part1.bin"/>
|
||||
</SegmentList>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>"""
|
||||
|
||||
|
||||
def _track(mpd: str, srv):
|
||||
return DASH.from_text(mpd, url=f"{_base(srv)}manifest.mpd").to_tracks(language="en").videos[0]
|
||||
|
||||
|
||||
def _ctx(tmp_path, name="out.mp4"):
|
||||
return DownloadContext(
|
||||
save_path=tmp_path / name,
|
||||
save_dir=tmp_path / "v0_segments",
|
||||
progress=partial(lambda **kw: None),
|
||||
)
|
||||
|
||||
|
||||
def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path):
|
||||
server.routes["/media.mp4"] = PARENT
|
||||
track = _track(_collapse_mpd(server), server)
|
||||
ctx = _ctx(tmp_path, "collapsed.mp4")
|
||||
|
||||
DASH.download_track(track, ctx)
|
||||
|
||||
# the whole parent lands at save_path, byte-for-byte
|
||||
assert ctx.save_path.read_bytes() == PARENT
|
||||
assert track.path == ctx.save_path
|
||||
# exactly one Range-less GET (the direct whole-resource download); the only other hit is
|
||||
# the ranged init probe done while parsing SegmentList, never a per-segment slice fetch.
|
||||
no_range = [path for path, rng in server.requests if rng is None]
|
||||
assert no_range == ["/media.mp4"]
|
||||
# collapse skips the segment dir + merge entirely: no dir, no leftover control files
|
||||
assert not ctx.save_dir.exists()
|
||||
assert not list(tmp_path.glob("*.!dev"))
|
||||
assert not list(tmp_path.glob("**/*.!dev"))
|
||||
|
||||
|
||||
def test_non_collapse_takes_segmented_merge_path(server, tmp_path):
|
||||
server.routes["/init.bin"] = INIT2
|
||||
server.routes["/part0.bin"] = PART0
|
||||
server.routes["/part1.bin"] = PART1
|
||||
track = _track(_mixed_mpd(server), server)
|
||||
ctx = _ctx(tmp_path, "merged.mp4")
|
||||
|
||||
DASH.download_track(track, ctx)
|
||||
|
||||
# merged output is init followed by both segments, in order
|
||||
assert ctx.save_path.read_bytes() == INIT2 + PART0 + PART1
|
||||
# both distinct segments were fetched -> genuinely the per-segment path, not a collapse
|
||||
fetched = {path for path, _ in server.requests}
|
||||
assert {"/part0.bin", "/part1.bin"} <= fetched
|
||||
# segment files are consumed and the segment dir is cleaned up after merge
|
||||
assert not list(ctx.save_dir.glob("*")) if ctx.save_dir.exists() else True
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Cancel/abort hardening for the downloader.
|
||||
|
||||
Pins two teardown-safety invariants:
|
||||
|
||||
- ``_dispatch_parts`` must NOT finalize when the process-global ``DOWNLOAD_CANCELLED``
|
||||
fires mid-download: part workers return silently keeping partials, so every future
|
||||
completes without error, and finalizing would strip the ``.!dev`` control file and
|
||||
pass off a hole-filled pre-truncated file as complete on the next run.
|
||||
- A worker parked in the retry backoff wait (which can reach MAX_BACKOFF) must wake and
|
||||
exit as soon as the batch abort is set, instead of sleeping out the full delay and
|
||||
stalling shutdown or the merge.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
from requests import Session
|
||||
|
||||
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
BODY = b"x" * 4096
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
server = self.server
|
||||
if server.always_status:
|
||||
self.send_response(server.always_status)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
rng = self.headers.get("Range")
|
||||
if rng:
|
||||
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
|
||||
body = BODY[start : end + 1]
|
||||
self.send_response(206)
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{len(BODY)}")
|
||||
else:
|
||||
body = BODY
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if server.gate is not None:
|
||||
self.wfile.write(body[:64])
|
||||
self.wfile.flush()
|
||||
server.started.set()
|
||||
server.gate.wait(timeout=10)
|
||||
self.wfile.write(body[64:])
|
||||
else:
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args): # keep the test output quiet
|
||||
pass
|
||||
|
||||
|
||||
class _Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def handle_error(self, request, client_address): # cancelled readers close mid-write
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = _Server(("127.0.0.1", 0), _Handler)
|
||||
srv.always_status = 0
|
||||
srv.gate = None
|
||||
srv.started = threading.Event()
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
if srv.gate is not None:
|
||||
srv.gate.set()
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cancel():
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def _url(server: _Server) -> str:
|
||||
host, port = server.server_address
|
||||
return f"http://{host}:{port}/seg.bin"
|
||||
|
||||
|
||||
def test_dispatch_parts_does_not_finalize_on_global_cancel(server, tmp_path):
|
||||
server.gate = threading.Event()
|
||||
save_path = tmp_path / "big.bin"
|
||||
|
||||
def _cancel_when_started() -> None:
|
||||
assert server.started.wait(timeout=10)
|
||||
dl.DOWNLOAD_CANCELLED.set()
|
||||
server.gate.set()
|
||||
|
||||
canceller = threading.Thread(target=_cancel_when_started)
|
||||
canceller.start()
|
||||
events = list(
|
||||
dl._dispatch_parts(
|
||||
url=_url(server),
|
||||
save_path=save_path,
|
||||
session=Session(),
|
||||
total_size=len(BODY),
|
||||
max_workers=2,
|
||||
)
|
||||
)
|
||||
canceller.join(timeout=10)
|
||||
|
||||
assert not any("file_downloaded" in ev for ev in events)
|
||||
# the control file must survive so the next run restarts instead of trusting the file
|
||||
assert save_path.with_name(f"{save_path.name}.!dev").exists()
|
||||
|
||||
|
||||
def test_backoff_wait_exits_promptly_on_batch_abort(server, tmp_path, monkeypatch):
|
||||
server.always_status = 503
|
||||
monkeypatch.setattr(dl, "_retry_sleep", lambda exc, attempts: 30.0)
|
||||
abort = threading.Event()
|
||||
|
||||
def _consume() -> None:
|
||||
for _ in dl.download(
|
||||
url=_url(server),
|
||||
save_path=tmp_path / "seg.bin",
|
||||
session=Session(),
|
||||
segmented=True,
|
||||
abort=abort,
|
||||
):
|
||||
pass
|
||||
|
||||
worker = threading.Thread(target=_consume)
|
||||
start = time.monotonic()
|
||||
worker.start()
|
||||
time.sleep(0.5) # let the first attempt fail and enter the backoff wait
|
||||
abort.set()
|
||||
worker.join(timeout=10)
|
||||
assert not worker.is_alive()
|
||||
# generous bound: far below the 30s nap, so the wait was interrupted, not slept out
|
||||
assert time.monotonic() - start < 10
|
||||
@@ -9,7 +9,10 @@ media requests must ask for identity encoding.
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import format_datetime
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -26,6 +29,12 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
with server.lock:
|
||||
server.requests.append(dict(self.headers))
|
||||
nth = len(server.requests)
|
||||
if nth <= server.fail_429_first:
|
||||
self.send_response(429)
|
||||
self.send_header("Retry-After", "0")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
rng = self.headers.get("Range")
|
||||
if server.honor_range and rng:
|
||||
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
|
||||
@@ -53,6 +62,7 @@ def server():
|
||||
srv.requests = []
|
||||
srv.honor_range = True
|
||||
srv.truncate_first = False
|
||||
srv.fail_429_first = 0
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
@@ -120,3 +130,64 @@ def test_range_item_retry_rewrites_slice(server, tmp_path, monkeypatch):
|
||||
def test_segment_requests_ask_identity_encoding(server, tmp_path):
|
||||
_run(server, tmp_path, [{"url": _url(server)}], max_workers=1)
|
||||
assert all(r.get("Accept-Encoding") == "identity" for r in server.requests)
|
||||
|
||||
|
||||
def _exc_with_headers(headers):
|
||||
exc = Exception("boom")
|
||||
exc.response = SimpleNamespace(headers=headers)
|
||||
return exc
|
||||
|
||||
|
||||
def test_retry_sleep_honors_numeric_retry_after():
|
||||
assert dl._retry_sleep(_exc_with_headers({"Retry-After": "3"}), 1) == 3.0
|
||||
assert dl._retry_sleep(_exc_with_headers({"Retry-After": "9999"}), 1) == 60.0 # session.MAX_BACKOFF cap
|
||||
|
||||
|
||||
def test_retry_sleep_honors_http_date_retry_after():
|
||||
when = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=10), usegmt=True)
|
||||
assert 5 <= dl._retry_sleep(_exc_with_headers({"Retry-After": when}), 1) <= 10
|
||||
|
||||
|
||||
def test_retry_sleep_exponential_backoff_with_jitter():
|
||||
# no response on the failure: exponential from RETRY_WAIT with up to 10% jitter
|
||||
for attempts, base in ((1, dl.RETRY_WAIT), (2, dl.RETRY_WAIT * 2), (3, dl.RETRY_WAIT * 4)):
|
||||
assert base <= dl._retry_sleep(Exception("reset"), attempts) <= base * 1.1
|
||||
assert dl._retry_sleep(Exception("reset"), 30) == 60.0 # capped
|
||||
|
||||
|
||||
def test_retry_sleep_invalid_retry_after_falls_back():
|
||||
v = dl._retry_sleep(_exc_with_headers({"Retry-After": "soon"}), 1)
|
||||
assert dl.RETRY_WAIT <= v <= dl.RETRY_WAIT * 1.1
|
||||
|
||||
|
||||
def test_retry_sleep_naive_http_date_treated_as_utc():
|
||||
# "-0000" makes parsedate_to_datetime return a naive datetime; must clamp, not raise
|
||||
when = format_datetime(datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=10))
|
||||
assert when.endswith("-0000")
|
||||
v = dl._retry_sleep(_exc_with_headers({"Retry-After": when}), 1)
|
||||
assert 5 <= v <= 10
|
||||
|
||||
|
||||
def test_retry_sleep_nan_retry_after_falls_back():
|
||||
# float("nan") parses; a non-finite wait must fall back to the exponential path
|
||||
v = dl._retry_sleep(_exc_with_headers({"Retry-After": "nan"}), 1)
|
||||
assert dl.RETRY_WAIT <= v <= dl.RETRY_WAIT * 1.1
|
||||
|
||||
|
||||
def test_retry_sleep_honors_retry_after_on_wrapped_cause():
|
||||
# RnetSession raises MaxRetriesError whose __cause__ HTTPError carries the response
|
||||
exc = Exception("max retries exceeded")
|
||||
exc.__cause__ = _exc_with_headers({"Retry-After": "3"})
|
||||
assert dl._retry_sleep(exc, 1) == 3.0
|
||||
|
||||
|
||||
def test_transient_429_with_retry_after_does_not_abort_track(server, tmp_path, monkeypatch):
|
||||
# a short rate-limit window must retry the segment, not kill the whole batch
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
server.fail_429_first = 2
|
||||
files = _run(server, tmp_path, [{"url": _url(server, f"/seg{i}.bin")} for i in range(3)], max_workers=1)
|
||||
assert len(files) == 3
|
||||
assert all(f.read_bytes() == PARENT for f in files)
|
||||
assert not dl.DOWNLOAD_CANCELLED.is_set()
|
||||
assert len(server.requests) == 5 # 3 successes + 2 rate-limited attempts
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Multiprocess segment-download invariants.
|
||||
|
||||
Pins the spawned-child fan-out path in ``requests()``/``_download_multiprocess``:
|
||||
|
||||
- Strided assignment plus the ``index_offset``/``index_stride`` filename mapping puts every
|
||||
segment's bytes in the correctly named file (a broken stride mapping misfiles content).
|
||||
- The parent normalizes child progress to segment granularity: it drops child ``advance``
|
||||
and ``total``/speed events and emits exactly one ``advance=1`` per ``file_downloaded``, so
|
||||
the advances summed by a caller equal the segment count.
|
||||
- A child that dies (its worker sends ``__mp_error__``) surfaces as a ``RuntimeError`` from
|
||||
the generator without hanging.
|
||||
- A process-global ``DOWNLOAD_CANCELLED`` set mid-batch terminates the spawned children (which
|
||||
carry their own fresh flag and can't see it) and the teardown join reaps them promptly.
|
||||
|
||||
Children are spawned and re-import the module, so monkeypatched module attributes do NOT reach
|
||||
them; these tests drive real child processes against a localhost server on 127.0.0.1 (reachable
|
||||
from the children) and avoid any child-side patching. Segments are a few hundred bytes so the
|
||||
real spawns stay fast.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import multiprocessing
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
SEG_REPEAT = 32 # 8-digit index * 32 = 256-byte body, distinct per index
|
||||
|
||||
|
||||
def _seg_body(index: int) -> bytes:
|
||||
"""Segment body that encodes its own index, so misfiled striding is detectable."""
|
||||
return (f"{index:08d}" * SEG_REPEAT).encode()
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
server = self.server
|
||||
delay = getattr(server, "delay", 0.0)
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
try:
|
||||
index = int(self.path.rsplit("/", 1)[1].split(".")[0])
|
||||
except ValueError:
|
||||
self.send_response(404)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
body = _seg_body(index)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args): # keep the test output quiet
|
||||
pass
|
||||
|
||||
|
||||
class _Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def handle_error(self, request, client_address): # a cancelled child closes its socket mid-write
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = _Server(("127.0.0.1", 0), _Handler)
|
||||
srv.delay = 0.0
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cancel():
|
||||
# spawned children carry their own flag, but the parent loop reads this process's flag;
|
||||
# keep it clean so cancel state never leaks between tests
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def _url(srv, path):
|
||||
return f"http://127.0.0.1:{srv.server_address[1]}{path}"
|
||||
|
||||
|
||||
def test_multiprocess_stride_downloads_all_segments_correctly(server, tmp_path):
|
||||
# >= MP_MIN_SEGMENTS with processes=2 engages the fan-out via the public entry point
|
||||
n = dl.MP_MIN_SEGMENTS
|
||||
urls = [{"url": _url(server, f"/seg/{i}.bin")} for i in range(n)]
|
||||
|
||||
total_advance = 0
|
||||
for event in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
|
||||
total_advance += event.get("advance", 0)
|
||||
|
||||
files = sorted(tmp_path.glob("seg_*.bin"))
|
||||
assert len(files) == n
|
||||
for f in files:
|
||||
index = int(f.stem.split("_")[1])
|
||||
assert f.read_bytes() == _seg_body(index), f"segment {index} holds another index's bytes (stride mapping bug)"
|
||||
# parent drops child advances and emits one advance=1 per file_downloaded
|
||||
assert total_advance == n
|
||||
|
||||
|
||||
def test_multiprocess_child_crash_raises(server, tmp_path):
|
||||
# _download_multiprocess has no MP_MIN_SEGMENTS gate, so drive it directly with 2 urls.
|
||||
# a dict url without "url" raises KeyError while requests() builds the save path (before
|
||||
# any retry loop), so the child sends __mp_error__ fast; the sibling downloads normally.
|
||||
good = {"url": _url(server, "/seg/0.bin")}
|
||||
bad = {"nourl": True}
|
||||
with pytest.raises(RuntimeError):
|
||||
for _ in dl._download_multiprocess(
|
||||
urls=[good, bad],
|
||||
output_dir=tmp_path,
|
||||
filename="seg_{i:04}.bin",
|
||||
headers=None,
|
||||
cookies=None,
|
||||
proxy=None,
|
||||
max_workers=2,
|
||||
adaptive=False,
|
||||
spec={"kind": "none"},
|
||||
processes=2,
|
||||
debug_logger=None,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def test_multiprocess_cancel_terminates_children(server, tmp_path):
|
||||
server.delay = 0.5 # slow segments so cancel lands well before the batch could finish
|
||||
n = dl.MP_MIN_SEGMENTS
|
||||
urls = [{"url": _url(server, f"/seg/{i}.bin")} for i in range(n)]
|
||||
|
||||
baseline = len(multiprocessing.active_children())
|
||||
start = time.time()
|
||||
progressed = 0
|
||||
for event in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
|
||||
if event.get("advance") or event.get("written"):
|
||||
progressed += 1
|
||||
if progressed >= 2:
|
||||
dl.DOWNLOAD_CANCELLED.set() # sibling-track cancel: parent must tear the children down
|
||||
elapsed = time.time() - start
|
||||
|
||||
# cancel stopped the work: nowhere near all segments finished, and no hang
|
||||
assert len(list(tmp_path.glob("seg_*.bin"))) < n
|
||||
assert elapsed < 15
|
||||
# the finally-block join reaps the spawned children promptly
|
||||
deadline = time.time() + 5
|
||||
while time.time() < deadline and len(multiprocessing.active_children()) > baseline:
|
||||
time.sleep(0.05)
|
||||
assert len(multiprocessing.active_children()) <= baseline
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Ranged-parallel single-URL download invariants.
|
||||
|
||||
Pins the byte-range fan-out that splits one large file across parts:
|
||||
|
||||
- a range-capable server above the size threshold is split into several byte
|
||||
windows, the parts merge byte-identical, and the ``.!dev`` control marker is
|
||||
removed on success;
|
||||
- a ranged part that receives a 200 (server ignored the Range) fails after
|
||||
retries rather than writing a wrong-length body into the pre-truncated file;
|
||||
- that failure sets only the ranged download's LOCAL abort event, never the
|
||||
process-global DOWNLOAD_CANCELLED (which would poison sibling tracks);
|
||||
- ``_probe_ranged`` parses the Content-Range total on a 206 and declines a 200
|
||||
or a content-encoded 206.
|
||||
|
||||
Tests 2/3 exercise ``_dispatch_parts`` directly: the public single-URL path in
|
||||
``requests()`` deliberately swallows a ranged failure and falls back to a plain
|
||||
sequential download, so the raise the invariant guarantees is only observable at
|
||||
the dispatch layer.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
from requests import Session
|
||||
|
||||
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
PAYLOAD = bytes(range(256)) * 128 # 32 KiB, distinctive repeating byte pattern
|
||||
|
||||
|
||||
class _RangeHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
srv = self.server
|
||||
rng = self.headers.get("Range")
|
||||
with srv.lock:
|
||||
srv.requests.append(rng)
|
||||
mode = srv.mode
|
||||
|
||||
if mode == "gzip_probe":
|
||||
# a 206 that (wrongly) carries a content-encoding: the probe must decline it
|
||||
self.send_response(206)
|
||||
self.send_header("Content-Range", f"bytes 0-0/{len(PAYLOAD)}")
|
||||
self.send_header("Content-Encoding", "gzip")
|
||||
self.send_header("Content-Length", "1")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"\x00")
|
||||
return
|
||||
|
||||
# always_200 ignores Range entirely; probe_only_206 answers the 0-0 probe
|
||||
# with a real 206 but hands every wider part window a 200 full body
|
||||
serve_full = mode == "always_200" or not rng or (mode == "probe_only_206" and rng != "bytes=0-0")
|
||||
if serve_full:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(PAYLOAD)))
|
||||
self.end_headers()
|
||||
self.wfile.write(PAYLOAD)
|
||||
return
|
||||
|
||||
start, end = (int(x) for x in rng.removeprefix("bytes=").split("-"))
|
||||
body = PAYLOAD[start : end + 1]
|
||||
self.send_response(206)
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{len(PAYLOAD)}")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cancelled():
|
||||
# ranged failures must never touch the process-global flag; isolate every test from it
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", 0), _RangeHandler)
|
||||
srv.lock = threading.Lock()
|
||||
srv.requests = []
|
||||
srv.mode = "range" # honor every Range with a proper 206
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def _url(srv, path="/payload.bin"):
|
||||
return f"http://127.0.0.1:{srv.server_address[1]}{path}"
|
||||
|
||||
|
||||
def _run(srv, tmp_path, urls, **kwargs):
|
||||
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", **kwargs):
|
||||
pass
|
||||
return sorted(tmp_path.glob("seg_*.bin"))
|
||||
|
||||
|
||||
def test_ranged_parallel_merges_byte_identical(server, tmp_path, monkeypatch):
|
||||
# shrink the thresholds so a 32 KiB payload splits into several parts
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_MIN_SIZE", 8 * 1024)
|
||||
monkeypatch.setattr(dl, "RANGE_PARALLEL_PART_SIZE", 2 * 1024)
|
||||
files = _run(server, tmp_path, [{"url": _url(server)}], max_workers=4)
|
||||
assert len(files) == 1
|
||||
assert files[0].read_bytes() == PAYLOAD
|
||||
# the split must actually have happened: >1 distinct window beyond the 0-0 probe
|
||||
windows = {r for r in server.requests if r and r != "bytes=0-0"}
|
||||
assert len(windows) >= 2
|
||||
# control marker removed once the parts finalize
|
||||
assert not files[0].with_name(files[0].name + ".!dev").exists()
|
||||
|
||||
|
||||
def test_ranged_part_requires_206(server, tmp_path, monkeypatch):
|
||||
server.mode = "probe_only_206"
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
save_path = tmp_path / "seg_0000.bin"
|
||||
gen = dl._dispatch_parts(
|
||||
url=_url(server),
|
||||
save_path=save_path,
|
||||
session=Session(),
|
||||
total_size=len(PAYLOAD),
|
||||
max_workers=4,
|
||||
)
|
||||
with pytest.raises(IOError, match="expected 206 for ranged part"):
|
||||
for _ in gen:
|
||||
pass
|
||||
# not finalized: the control marker survives (completed=False) and the target
|
||||
# is still the pre-truncated zero-fill, never the served bytes
|
||||
assert save_path.with_name(save_path.name + ".!dev").exists()
|
||||
assert save_path.read_bytes() != PAYLOAD
|
||||
|
||||
|
||||
def test_ranged_part_failure_stays_local(server, tmp_path, monkeypatch):
|
||||
server.mode = "probe_only_206"
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
save_path = tmp_path / "seg_0000.bin"
|
||||
gen = dl._dispatch_parts(
|
||||
url=_url(server),
|
||||
save_path=save_path,
|
||||
session=Session(),
|
||||
total_size=len(PAYLOAD),
|
||||
max_workers=4,
|
||||
)
|
||||
with pytest.raises(IOError, match="expected 206 for ranged part"):
|
||||
for _ in gen:
|
||||
pass
|
||||
# a failed part rides the local abort event; the global cancel must stay clear
|
||||
assert not dl.DOWNLOAD_CANCELLED.is_set()
|
||||
|
||||
|
||||
def test_probe_ranged_parses_content_range(server):
|
||||
session = Session()
|
||||
server.mode = "range"
|
||||
assert dl._probe_ranged(_url(server), session) == (len(PAYLOAD), True)
|
||||
server.mode = "always_200"
|
||||
assert dl._probe_ranged(_url(server), session) == (0, False)
|
||||
server.mode = "gzip_probe"
|
||||
assert dl._probe_ranged(_url(server), session) == (0, False)
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Tail-boost success-path invariants.
|
||||
|
||||
Pins the tail-boost engage path in ``requests()``: when only a few segments remain
|
||||
and workers would otherwise idle, each remaining segment is probed for range support
|
||||
and split into intra-segment byte-range parts written into a pre-truncated ``.tp.!dev``
|
||||
target, finalized by the last part worker via os.replace. The invariants:
|
||||
|
||||
- a boosted segment's final file is byte-identical to the served content, every
|
||||
segment reports exactly one advance=1 (part-mode byte advances are swallowed), and
|
||||
a clean run leaves no ``.!dev`` / ``.tp.!dev`` strays;
|
||||
- finalize is last-part-wins: on success save_path exists and the ``.tp.!dev`` target
|
||||
is gone;
|
||||
- a permanently failing part fails the batch and the boosted segment is never
|
||||
os.replace'd into place (no corrupt finalized segment).
|
||||
|
||||
Determinism: with ``max_workers`` < segment count <= 2*max_workers and adaptive=True,
|
||||
the leading ``max_workers`` segments submit upfront and the trailing few stay in
|
||||
``remaining`` (the tail top-up only releases boost-declined indices). The controller
|
||||
starts at the cap and never grows, so as soon as two leaders finish the spare workers
|
||||
outnumber the tail and ``_tail_boost_engages`` fires. Module timing/size constants are
|
||||
lowered via monkeypatch; the whole batch runs in-process (threads), so the patched
|
||||
values are seen by the workers.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
BODY = bytes((i * 7 + 11) % 256 for i in range(16384)) # 16 KiB, distinctive, range-sliceable
|
||||
|
||||
# lowered so a 16 KiB segment clears the min and splits into several 4 KiB parts
|
||||
BOOST_MIN = 4096
|
||||
BOOST_PART = 4096
|
||||
|
||||
MAX_WORKERS = 6
|
||||
SEG_COUNT = 8 # > MAX_WORKERS and <= 2*MAX_WORKERS: leaders 0-5 submit, tail 6,7 boost
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
server = self.server
|
||||
rng = self.headers.get("Range")
|
||||
with server.lock:
|
||||
server.requests.append((self.path, rng))
|
||||
if rng and rng.startswith("bytes="):
|
||||
start_s, _, end_s = rng[len("bytes=") :].partition("-")
|
||||
start = int(start_s)
|
||||
end = int(end_s) if end_s else len(BODY) - 1
|
||||
if server.break_offset_ranges and start > 0:
|
||||
# misbehave for every non-leading window: send the whole body as 200 so
|
||||
# part-mode's 206 check fails permanently for that part
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(BODY)))
|
||||
self.end_headers()
|
||||
self.wfile.write(BODY)
|
||||
return
|
||||
body = BODY[start : end + 1]
|
||||
self.send_response(206)
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{len(BODY)}")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(BODY)))
|
||||
self.end_headers()
|
||||
self.wfile.write(BODY)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
||||
srv.lock = threading.Lock()
|
||||
srv.requests = []
|
||||
srv.break_offset_ranges = False
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_cancel():
|
||||
# a failing batch sets the process-global cancel; keep tests independent
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
dl.DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _boost_constants(monkeypatch):
|
||||
monkeypatch.setattr(dl, "TAIL_BOOST_MIN_SEGMENT_SIZE", BOOST_MIN)
|
||||
monkeypatch.setattr(dl, "TAIL_BOOST_PART_SIZE", BOOST_PART)
|
||||
|
||||
|
||||
def _url(srv, path):
|
||||
return f"http://127.0.0.1:{srv.server_address[1]}{path}"
|
||||
|
||||
|
||||
def _urls(srv):
|
||||
return [{"url": _url(srv, f"/seg{i}.bin")} for i in range(SEG_COUNT)]
|
||||
|
||||
|
||||
def _run(srv, tmp_path, urls):
|
||||
advances = 0
|
||||
for ev in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=MAX_WORKERS, adaptive=True):
|
||||
a = ev.get("advance")
|
||||
if a:
|
||||
advances += a
|
||||
return sorted(tmp_path.glob("seg_*.bin")), advances
|
||||
|
||||
|
||||
def _ranged_counts(srv):
|
||||
with srv.lock:
|
||||
reqs = list(srv.requests)
|
||||
counts: dict[str, int] = {}
|
||||
for path, rng in reqs:
|
||||
if rng:
|
||||
counts[path] = counts.get(path, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def _boost_engaged(srv):
|
||||
# a boosted path sees a 0-0 probe plus >=2 part windows; a normal segment sees no Range
|
||||
return any(n > 1 for n in _ranged_counts(srv).values())
|
||||
|
||||
|
||||
def test_tail_boost_split_produces_byte_identical_segment(server, tmp_path):
|
||||
files, advances = _run(server, tmp_path, _urls(server))
|
||||
|
||||
assert _boost_engaged(server), "tail boost never engaged; test would silently cover the normal path"
|
||||
assert len(files) == SEG_COUNT
|
||||
assert all(f.read_bytes() == BODY for f in files)
|
||||
# part-mode byte advances are swallowed; each segment reports exactly one advance=1
|
||||
assert advances == SEG_COUNT
|
||||
# a clean run renames every target into place, so no partial/part strays survive
|
||||
assert not list(tmp_path.glob("*.!dev"))
|
||||
assert not list(tmp_path.glob("*.tp.!dev"))
|
||||
|
||||
|
||||
def test_tail_boost_finalize_last_part_wins(server, tmp_path):
|
||||
files, _ = _run(server, tmp_path, _urls(server))
|
||||
|
||||
assert _boost_engaged(server), "tail boost never engaged; finalize path untested"
|
||||
assert len(files) == SEG_COUNT
|
||||
# last part wins: the pre-truncated .tp.!dev is os.replace'd into save_path, leaving no target
|
||||
assert not list(tmp_path.glob("*.tp.!dev"))
|
||||
for path in _ranged_counts(server):
|
||||
i = int(path.removeprefix("/seg").removesuffix(".bin"))
|
||||
save_path = tmp_path / f"seg_{i:04}.bin"
|
||||
assert save_path.exists()
|
||||
assert not save_path.with_name(f"{save_path.name}.tp.!dev").exists()
|
||||
|
||||
|
||||
def test_tail_boost_part_failure_fails_batch_but_keeps_siblings_files_clean(server, tmp_path, monkeypatch):
|
||||
server.break_offset_ranges = True # every non-leading part window 200s -> permanent 206 failure
|
||||
monkeypatch.setattr(dl, "RETRY_WAIT", 0.01)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
_run(server, tmp_path, _urls(server))
|
||||
|
||||
# the boosted tail segments (6, 7) hit the failing windows: no part run reaches parts_left==0
|
||||
# cleanly, so neither is finalized. sibling leaders that did finalize must be uncorrupted.
|
||||
for i in (6, 7):
|
||||
assert not (tmp_path / f"seg_{i:04}.bin").exists()
|
||||
for f in tmp_path.glob("seg_*.bin"):
|
||||
assert f.read_bytes() == BODY
|
||||
Reference in New Issue
Block a user