fix(downloaders): submit segments in a window so progress can drain

Submitting every segment up front convoyed on ThreadPoolExecutor's process-global lock, starving the event drain so per-track progress bars only advanced once the download had finished.
This commit is contained in:
imSp4rky
2026-08-16 11:06:17 -06:00
parent 21d5f1c5e1
commit 1af77d395d
2 changed files with 90 additions and 34 deletions
@@ -149,6 +149,29 @@ def test_failed_segment_fails_the_batch(tmp_path, server):
assert sorted(p.name for p in tmp_path.iterdir() if p.suffix == ".mp4") == [] assert sorted(p.name for p in tmp_path.iterdir() if p.suffix == ".mp4") == []
@pytest.mark.parametrize("server", ["ok"], indirect=True)
def test_batch_larger_than_the_queue_window_downloads_every_segment(tmp_path, server):
# a broken top-up livelocks the drain loop instead of erroring, so consume on a thread and
# let the timeout fail the test
urls = [server.url] * 6
advanced = 0
def consume():
nonlocal advanced
advanced = sum(
event["advance"]
for event in requests_downloader(urls=urls, output_dir=tmp_path, filename="{i:02}.mp4", max_workers=1)
if "advance" in event
)
consumer = threading.Thread(target=consume, daemon=True)
consumer.start()
consumer.join(timeout=30)
assert not consumer.is_alive(), "downloader never finished the batch"
assert advanced == len(urls)
assert sorted(p.name for p in tmp_path.iterdir() if p.suffix == ".mp4") == [f"{i:02}.mp4" for i in range(6)]
@pytest.mark.parametrize( @pytest.mark.parametrize(
("item", "expected"), ("item", "expected"),
[ [
+67 -34
View File
@@ -5,7 +5,7 @@ import statistics
import threading import threading
import time import time
from collections import deque from collections import deque
from concurrent.futures import FIRST_COMPLETED, wait from concurrent.futures import FIRST_COMPLETED, Future, wait
from concurrent.futures.thread import ThreadPoolExecutor from concurrent.futures.thread import ThreadPoolExecutor
from http.cookiejar import CookieJar from http.cookiejar import CookieJar
from pathlib import Path from pathlib import Path
@@ -824,19 +824,48 @@ def requests(
with seg_lock: with seg_lock:
seg_active[index] -= 1 seg_active[index] -= 1
futures = [pool.submit(download_worker, i, url) for i, url in enumerate(urls)] outstanding = 0
pending = set(futures) failures: list[BaseException] = []
def track_future(future: Future) -> None:
if not future.cancelled():
exc = future.exception()
if exc is not None:
failures.append(exc)
event_queue.put(dict(worker_done=True))
def spawn(index: int, url_item: dict[str, Any], hedge: bool = False) -> None:
nonlocal outstanding
outstanding += 1
pool.submit(download_worker, index, url_item, hedge).add_done_callback(track_future)
queue_depth = max_workers * 2
next_index = 0
def fill_queue() -> None:
nonlocal next_index
while next_index < len(urls) and outstanding < queue_depth:
spawn(next_index, urls[next_index])
next_index += 1
fill_queue()
pending_advance = 0 pending_advance = 0
try: try:
while pending: while outstanding or next_index < len(urls):
# Drain queued events — batch advances, track bytes for speed # Drain queued events - batch advances, track bytes for speed. The first get
# blocks so the loop idles between events instead of spinning.
blocking = True
while True: while True:
try: try:
event = event_queue.get_nowait() event = event_queue.get(timeout=0.1) if blocking else event_queue.get_nowait()
except Empty: except Empty:
break break
blocking = False
if event.get("worker_done"):
outstanding -= 1
continue
# Accumulate advance events for batched yield # Accumulate advance events for batched yield
advance = event.get("advance") advance = event.get("advance")
if advance: if advance:
@@ -849,6 +878,8 @@ def requests(
# Pass through other events (file_downloaded, total, etc.) # Pass through other events (file_downloaded, total, etc.)
yield event yield event
fill_queue()
# Yield batched advances every drain cycle for responsive progress bar # Yield batched advances every drain cycle for responsive progress bar
if pending_advance > 0: if pending_advance > 0:
yield dict(advance=pending_advance) yield dict(advance=pending_advance)
@@ -862,10 +893,10 @@ def requests(
yield dict(downloaded=f"{filesize.decimal(math.ceil(rate))}/s") yield dict(downloaded=f"{filesize.decimal(math.ceil(rate))}/s")
last_speed_report = now last_speed_report = now
# hedge stuck segments once spare workers exist (pending < max_workers); # hedge stuck segments once spare workers exist; throttled to ~0.5s and
# throttled to ~0.5s and median recomputed only when seg_durations grows # median recomputed only when seg_durations grows
if ( if (
len(pending) < max_workers outstanding < max_workers
and seg_durations and seg_durations
and speed_limiter is None and speed_limiter is None
and now - last_hedge_check > 0.5 and now - last_hedge_check > 0.5
@@ -883,37 +914,37 @@ def requests(
for i, started in seg_start.items() for i, started in seg_start.items()
if i not in seg_done and i not in hedged and now - started > threshold if i not in seg_done and i not in hedged and now - started > threshold
] ]
for i in stuck[: max_workers - len(pending)]: for i in stuck[: max_workers - outstanding]:
hedged.add(i) hedged.add(i)
pending.add(pool.submit(download_worker, i, urls[i], True)) spawn(i, urls[i], True)
# all segments claimed; superseded losers exit via their claimed() check # all segments claimed; superseded losers exit via their claimed() check
if len(seg_done) == len(urls): if len(seg_done) == len(urls):
break break
# Wait efficiently for next future completion (OS condition variable) if failures:
completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) break
for future in completed:
exc = future.exception() if any(isinstance(exc, KeyboardInterrupt) for exc in failures):
if isinstance(exc, KeyboardInterrupt): raise KeyboardInterrupt()
raise KeyboardInterrupt() if failures and len(seg_done) != len(urls):
elif exc: exc = failures[0]
DOWNLOAD_CANCELLED.set() DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[red]FAILING") yield dict(downloaded="[red]FAILING")
pool.shutdown(wait=False, cancel_futures=True) pool.shutdown(wait=False, cancel_futures=True)
yield dict(downloaded="[red]FAILED") yield dict(downloaded="[red]FAILED")
if debug_logger: if debug_logger:
debug_logger.log( debug_logger.log(
level="ERROR", level="ERROR",
operation="downloader_failed", operation="downloader_failed",
message=f"Download failed: {exc}", message=f"Download failed: {exc}",
error=exc, error=exc,
context={ context={
"url_count": len(urls), "url_count": len(urls),
"output_dir": str(output_dir), "output_dir": str(output_dir),
}, },
) )
raise exc raise exc
except KeyboardInterrupt: except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set() DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[yellow]CANCELLING") yield dict(downloaded="[yellow]CANCELLING")
@@ -931,6 +962,8 @@ def requests(
event = event_queue.get_nowait() event = event_queue.get_nowait()
except Empty: except Empty:
break break
if event.get("worker_done"):
continue
advance = event.get("advance") advance = event.get("advance")
if advance: if advance:
pending_advance += advance pending_advance += advance