diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index 396a1224..5a79f664 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -529,7 +529,7 @@ Two related flags: | --- | --- | | `--workers N` | Threads used per track for segment downloads. Default depends on the downloader. | | `--adaptive-workers` | Opt-in: start with a moderate per-track worker count and ramp it up or back off based on measured CDN throughput and errors, capped at `--workers`. Off by default (fixed worker count). | -| `--download-processes N` | Opt-in: split a large segment batch across `N` download processes, each with its own worker pool. A single process tops out around 1.3 Gb/s (Python interpreter limit); `2` reaches 2.5GbE line rate on fast CDNs. Only engages for batches of 24+ segments. Ignored while `--speed-limit` (or serve's `global_speed_limit`) is set — the cap is one shared budget, which extra processes cannot share, so the download stays in a single process. Default `1`. | +| `--download-processes N` | Opt-in: split a large segment batch across `N` download processes, each with its own worker pool. A single process tops out around 1.3 Gb/s (Python interpreter limit); `2` reaches 2.5GbE line rate on fast CDNs. Only engages for batches of 24+ segments. Ignored while `--speed-limit` (or serve's `global_speed_limit`) is set: the cap is one shared budget, which extra processes cannot share, so the download stays in a single process. Default `1`. | | `--downloads N` | Number of tracks downloaded concurrently. Default `1`. | | `--slow [MIN-MAX]` | Add a delay between titles to look more like a real device. `--slow` alone means 60-120s; `--slow 20-40` sets a custom range. Minimum 20s. | | `--no-cache` | Bypass the title cache for this download. | diff --git a/tests/downloaders/test_content_encoding.py b/tests/downloaders/test_content_encoding.py index 7aa58d7c..6e3f7b49 100644 --- a/tests/downloaders/test_content_encoding.py +++ b/tests/downloaders/test_content_encoding.py @@ -151,6 +151,6 @@ def test_rnet_encoded_body_discards_a_partial_instead_of_range_resuming(tmp_path save_path = tmp_path / "sub.vtt" save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000]) list(download(url="http://127.0.0.1:1/sub.vtt", save_path=save_path, session=session)) - # the encoded 206 must be dropped (never appended); the restart fetches the whole body + # the encoded 206 must be dropped, not appended; the clean restart fetches the whole body assert session.range_requests == 1 assert save_path.read_bytes() == PLAIN diff --git a/tests/downloaders/test_download_hardening.py b/tests/downloaders/test_download_hardening.py index b48a6824..080fd273 100644 --- a/tests/downloaders/test_download_hardening.py +++ b/tests/downloaders/test_download_hardening.py @@ -116,9 +116,8 @@ def test_range_item_ignored_by_server_fails_not_corrupts(server, tmp_path, monke def test_range_item_spanning_whole_resource_accepts_200(server, tmp_path): - # RFC 9110 lets a server ignore Range and answer 200 with the whole resource; when - # the slice spans that whole resource the body is byte-identical to the 206, so it - # must be kept instead of burning five attempts and failing the track + # RFC 9110 lets a server ignore Range and answer 200 with the whole resource; when the + # slice spans that resource the body is byte-identical to the 206 and must be kept server.honor_range = False urls = [{"url": _url(server), "headers": {"Range": f"bytes=0-{len(PARENT) - 1}"}}] files = _run(server, tmp_path, urls, max_workers=1) diff --git a/tests/downloaders/test_speed_limit.py b/tests/downloaders/test_speed_limit.py index 68303021..3d32bb5d 100644 --- a/tests/downloaders/test_speed_limit.py +++ b/tests/downloaders/test_speed_limit.py @@ -14,8 +14,8 @@ requests_downloader = importlib.import_module("unshackle.core.downloaders.reques @pytest.fixture(autouse=True) def _clean_cancel(): - # a failing batch anywhere sets the process-global cancel, and TokenBucket.consume - # waits on it, so a leaked flag voids every timing assertion; keep tests independent + # a failing batch sets the process-global cancel, which TokenBucket.consume waits on; + # keep tests independent DOWNLOAD_CANCELLED.clear() yield DOWNLOAD_CANCELLED.clear() @@ -97,8 +97,8 @@ def test_bucket_holds_aggregate_rate_across_threads(): def test_bucket_chunk_larger_than_capacity_does_not_deadlock(monkeypatch): - # capture the debt sleep instead of racing the wall clock: Event.wait may wake - # early spuriously, which intermittently tripped a lower bound on real sleeps + # capture the debt sleep instead of timing it: Event.wait can wake early spuriously, + # so a wall-clock lower bound flakes waits: list[float] = [] monkeypatch.setattr(requests_downloader, "DOWNLOAD_CANCELLED", SimpleNamespace(wait=waits.append)) bucket = TokenBucket(1_000_000) diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index 2b87c847..79defb71 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -742,9 +742,9 @@ def download( if item_range and not part_mode and stream.status_code != 206: # server ignored the slice's Range and sent the whole parent (RFC 9110 allows - # a 200 here); writing that as the segment would silently corrupt the merge, - # so fail the attempt — unless the slice spans the whole resource, in which - # case the 200 body is byte-identical to the 206 and safe to keep + # a 200 here); writing that as the segment would silently corrupt the merge, so + # fail the attempt unless the slice spans the whole resource, in which case the + # 200 body is byte-identical to the 206 and safe to keep if not (stream.status_code == 200 and _range_covers_full_body(req_headers, content_length)): raise IOError(f"expected 206 for byte-range segment, got {stream.status_code}") diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 077dab77..edb584fa 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -840,8 +840,8 @@ def session( # connect_timeout + read_timeout mirror the requests-path default timeout (CONNECT_TIMEOUT, READ_TIMEOUT) # so an unset config still gets a bounded connect and read like requests; pool_idle_timeout stays under # the typical ~60s CDN idle kill; pool_max_idle_per_host follows POOL_MAX_SIZE, sized above the worker - # cap because hedge racers and tail-boost parts push in-flight requests past it (see POOL_MAX_SIZE); - # tcp_keepalive keeps long idle segments warm. + # cap because hedge racers and tail-boost parts push in-flight requests past it; tcp_keepalive keeps + # long idle segments warm. session_kwargs.setdefault("connect_timeout", CONNECT_TIMEOUT) session_kwargs.setdefault("read_timeout", READ_TIMEOUT) session_kwargs.setdefault("pool_idle_timeout", 55)