mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 20:15:10 +00:00
imSp4rky
85503fd87c
Assert that a probe is judged on the trailing tick, that a backoff is temporary and climbs back to the cap, and that a worker always finds its next segment staged. Documents adaptive-workers behaviour in the downloading guide.
106 lines
3.4 KiB
Python
106 lines
3.4 KiB
Python
"""Submission-window invariants for the segmented drain loop.
|
|
|
|
Segments are not all submitted up front: that convoys on the pool's global lock and starves
|
|
the event drain. The loop meters submission through a window it tops up. The window must hold
|
|
two segments for every worker, so a worker that finishes one finds the next already staged
|
|
instead of idling until the next drain cycle. Fixed and adaptive mode must agree on that
|
|
ratio; adaptive sizes it from its live target instead of from ``max_workers``.
|
|
|
|
The test stalls every response, so no future can complete and the queue settles at exactly
|
|
the window. Counting ``pool.submit`` calls makes the queue depth observable: a blocked
|
|
server only ever sees ``max_workers`` requests, whatever is staged behind them.
|
|
"""
|
|
|
|
import importlib
|
|
import threading
|
|
import time
|
|
from concurrent.futures.thread import ThreadPoolExecutor
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
import pytest
|
|
|
|
dl = importlib.import_module("unshackle.core.downloaders.requests")
|
|
|
|
BODY = b"segment-body" * 64
|
|
MAX_WORKERS = 4
|
|
SEG_COUNT = 20 # more than the window, so queue depth is the only bound
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
class _Handler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
self.server.release.wait(timeout=10)
|
|
self.send_response(200)
|
|
self.send_header("Content-Length", str(len(BODY)))
|
|
self.end_headers()
|
|
self.wfile.write(BODY)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
|
|
@pytest.fixture
|
|
def server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
|
srv.release = threading.Event()
|
|
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
thread.start()
|
|
yield srv
|
|
srv.release.set()
|
|
srv.shutdown()
|
|
thread.join(timeout=5)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clean_cancel():
|
|
dl.DOWNLOAD_CANCELLED.clear()
|
|
yield
|
|
dl.DOWNLOAD_CANCELLED.clear()
|
|
|
|
|
|
@pytest.fixture
|
|
def submissions(monkeypatch):
|
|
counter: list[int] = []
|
|
|
|
class _CountingPool(ThreadPoolExecutor):
|
|
def submit(self, fn, /, *args, **kwargs):
|
|
counter.append(1) # list.append is atomic, so no lock is needed here
|
|
return super().submit(fn, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(dl, "ThreadPoolExecutor", _CountingPool)
|
|
return counter
|
|
|
|
|
|
@pytest.mark.parametrize("adaptive", [False, True])
|
|
def test_queue_holds_two_segments_per_worker(server, submissions, tmp_path, adaptive):
|
|
urls = [{"url": f"http://127.0.0.1:{server.server_address[1]}/seg{i}.bin"} for i in range(SEG_COUNT)]
|
|
events: list[dict] = []
|
|
|
|
def run():
|
|
events.extend(
|
|
dl.requests(
|
|
urls,
|
|
output_dir=tmp_path,
|
|
filename="seg_{i:04}.bin",
|
|
max_workers=MAX_WORKERS,
|
|
adaptive=adaptive,
|
|
)
|
|
)
|
|
|
|
worker = threading.Thread(target=run, daemon=True)
|
|
worker.start()
|
|
try:
|
|
deadline = time.time() + 3.0
|
|
while time.time() < deadline and len(submissions) < 2 * MAX_WORKERS:
|
|
time.sleep(0.02)
|
|
time.sleep(0.2) # settle, so an over-filled window is caught as well as an under-filled one
|
|
assert len(submissions) == 2 * MAX_WORKERS
|
|
finally:
|
|
server.release.set()
|
|
worker.join(timeout=30)
|
|
|
|
assert not worker.is_alive()
|
|
assert sum(event.get("advance", 0) for event in events) == SEG_COUNT
|
|
assert len(sorted(tmp_path.glob("seg_*.bin"))) == SEG_COUNT
|