mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-16 22:15:07 +00:00
fix(downloader): keep speed-limited batches in a single process
Spawned children re-import the module and start with no TokenBucket, so --download-processes silently dropped a configured --speed-limit / global_speed_limit for any 24+ segment batch. Fall back to the in-process path while a limit is set (same gate as the ranged-parallel paths), log the fallback, and document the interaction.
This commit is contained in:
@@ -154,3 +154,26 @@ def test_multiprocess_cancel_terminates_children(server, tmp_path):
|
||||
while time.time() < deadline and len(multiprocessing.active_children()) > baseline:
|
||||
time.sleep(0.05)
|
||||
assert len(multiprocessing.active_children()) <= baseline
|
||||
|
||||
|
||||
def test_speed_limit_forces_single_process(server, tmp_path, monkeypatch):
|
||||
"""A set speed limit must keep the batch in-process: spawned children re-import the
|
||||
module and would run with no TokenBucket, silently ignoring the configured cap."""
|
||||
|
||||
def _no_mp(**kwargs):
|
||||
raise AssertionError("multiprocess fan-out engaged despite a speed limit")
|
||||
|
||||
monkeypatch.setattr(dl, "_download_multiprocess", _no_mp)
|
||||
n = dl.MP_MIN_SEGMENTS
|
||||
urls = [{"url": _url(server, f"/seg/{i}.bin")} for i in range(n)]
|
||||
dl.set_speed_limit(1_000_000_000) # far above the tiny batch, so the test is not slowed
|
||||
try:
|
||||
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
|
||||
pass
|
||||
finally:
|
||||
dl.set_speed_limit(None)
|
||||
|
||||
files = sorted(tmp_path.glob("seg_*.bin"))
|
||||
assert len(files) == n
|
||||
for f in files:
|
||||
assert f.read_bytes() == _seg_body(int(f.stem.split("_")[1]))
|
||||
|
||||
Reference in New Issue
Block a user