mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-21 16:35:07 +00:00
style: tighten comments added on this branch
This commit is contained in:
@@ -529,7 +529,7 @@ Two related flags:
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `--workers N` | Threads used per track for segment downloads. Default depends on the downloader. |
|
| `--workers N` | Threads used per track for segment downloads. Default depends on the downloader. |
|
||||||
| `--adaptive-workers` | Opt-in: start with a moderate per-track worker count and ramp it up or back off based on measured CDN throughput and errors, capped at `--workers`. Off by default (fixed worker count). |
|
| `--adaptive-workers` | Opt-in: start with a moderate per-track worker count and ramp it up or back off based on measured CDN throughput and errors, capped at `--workers`. Off by default (fixed worker count). |
|
||||||
| `--download-processes N` | Opt-in: split a large segment batch across `N` download processes, each with its own worker pool. A single process tops out around 1.3 Gb/s (Python interpreter limit); `2` reaches 2.5GbE line rate on fast CDNs. Only engages for batches of 24+ segments. Ignored while `--speed-limit` (or serve's `global_speed_limit`) is set — the cap is one shared budget, which extra processes cannot share, so the download stays in a single process. Default `1`. |
|
| `--download-processes N` | Opt-in: split a large segment batch across `N` download processes, each with its own worker pool. A single process tops out around 1.3 Gb/s (Python interpreter limit); `2` reaches 2.5GbE line rate on fast CDNs. Only engages for batches of 24+ segments. Ignored while `--speed-limit` (or serve's `global_speed_limit`) is set: the cap is one shared budget, which extra processes cannot share, so the download stays in a single process. Default `1`. |
|
||||||
| `--downloads N` | Number of tracks downloaded concurrently. Default `1`. |
|
| `--downloads N` | Number of tracks downloaded concurrently. Default `1`. |
|
||||||
| `--slow [MIN-MAX]` | Add a delay between titles to look more like a real device. `--slow` alone means 60-120s; `--slow 20-40` sets a custom range. Minimum 20s. |
|
| `--slow [MIN-MAX]` | Add a delay between titles to look more like a real device. `--slow` alone means 60-120s; `--slow 20-40` sets a custom range. Minimum 20s. |
|
||||||
| `--no-cache` | Bypass the title cache for this download. |
|
| `--no-cache` | Bypass the title cache for this download. |
|
||||||
|
|||||||
@@ -151,6 +151,6 @@ def test_rnet_encoded_body_discards_a_partial_instead_of_range_resuming(tmp_path
|
|||||||
save_path = tmp_path / "sub.vtt"
|
save_path = tmp_path / "sub.vtt"
|
||||||
save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000])
|
save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000])
|
||||||
list(download(url="http://127.0.0.1:1/sub.vtt", save_path=save_path, session=session))
|
list(download(url="http://127.0.0.1:1/sub.vtt", save_path=save_path, session=session))
|
||||||
# the encoded 206 must be dropped (never appended); the restart fetches the whole body
|
# the encoded 206 must be dropped, not appended; the clean restart fetches the whole body
|
||||||
assert session.range_requests == 1
|
assert session.range_requests == 1
|
||||||
assert save_path.read_bytes() == PLAIN
|
assert save_path.read_bytes() == PLAIN
|
||||||
|
|||||||
@@ -116,9 +116,8 @@ def test_range_item_ignored_by_server_fails_not_corrupts(server, tmp_path, monke
|
|||||||
|
|
||||||
|
|
||||||
def test_range_item_spanning_whole_resource_accepts_200(server, tmp_path):
|
def test_range_item_spanning_whole_resource_accepts_200(server, tmp_path):
|
||||||
# RFC 9110 lets a server ignore Range and answer 200 with the whole resource; when
|
# RFC 9110 lets a server ignore Range and answer 200 with the whole resource; when the
|
||||||
# the slice spans that whole resource the body is byte-identical to the 206, so it
|
# slice spans that resource the body is byte-identical to the 206 and must be kept
|
||||||
# must be kept instead of burning five attempts and failing the track
|
|
||||||
server.honor_range = False
|
server.honor_range = False
|
||||||
urls = [{"url": _url(server), "headers": {"Range": f"bytes=0-{len(PARENT) - 1}"}}]
|
urls = [{"url": _url(server), "headers": {"Range": f"bytes=0-{len(PARENT) - 1}"}}]
|
||||||
files = _run(server, tmp_path, urls, max_workers=1)
|
files = _run(server, tmp_path, urls, max_workers=1)
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ requests_downloader = importlib.import_module("unshackle.core.downloaders.reques
|
|||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def _clean_cancel():
|
def _clean_cancel():
|
||||||
# a failing batch anywhere sets the process-global cancel, and TokenBucket.consume
|
# a failing batch sets the process-global cancel, which TokenBucket.consume waits on;
|
||||||
# waits on it, so a leaked flag voids every timing assertion; keep tests independent
|
# keep tests independent
|
||||||
DOWNLOAD_CANCELLED.clear()
|
DOWNLOAD_CANCELLED.clear()
|
||||||
yield
|
yield
|
||||||
DOWNLOAD_CANCELLED.clear()
|
DOWNLOAD_CANCELLED.clear()
|
||||||
@@ -97,8 +97,8 @@ def test_bucket_holds_aggregate_rate_across_threads():
|
|||||||
|
|
||||||
|
|
||||||
def test_bucket_chunk_larger_than_capacity_does_not_deadlock(monkeypatch):
|
def test_bucket_chunk_larger_than_capacity_does_not_deadlock(monkeypatch):
|
||||||
# capture the debt sleep instead of racing the wall clock: Event.wait may wake
|
# capture the debt sleep instead of timing it: Event.wait can wake early spuriously,
|
||||||
# early spuriously, which intermittently tripped a lower bound on real sleeps
|
# so a wall-clock lower bound flakes
|
||||||
waits: list[float] = []
|
waits: list[float] = []
|
||||||
monkeypatch.setattr(requests_downloader, "DOWNLOAD_CANCELLED", SimpleNamespace(wait=waits.append))
|
monkeypatch.setattr(requests_downloader, "DOWNLOAD_CANCELLED", SimpleNamespace(wait=waits.append))
|
||||||
bucket = TokenBucket(1_000_000)
|
bucket = TokenBucket(1_000_000)
|
||||||
|
|||||||
@@ -742,9 +742,9 @@ def download(
|
|||||||
|
|
||||||
if item_range and not part_mode and stream.status_code != 206:
|
if item_range and not part_mode and stream.status_code != 206:
|
||||||
# server ignored the slice's Range and sent the whole parent (RFC 9110 allows
|
# server ignored the slice's Range and sent the whole parent (RFC 9110 allows
|
||||||
# a 200 here); writing that as the segment would silently corrupt the merge,
|
# a 200 here); writing that as the segment would silently corrupt the merge, so
|
||||||
# so fail the attempt — unless the slice spans the whole resource, in which
|
# fail the attempt unless the slice spans the whole resource, in which case the
|
||||||
# case the 200 body is byte-identical to the 206 and safe to keep
|
# 200 body is byte-identical to the 206 and safe to keep
|
||||||
if not (stream.status_code == 200 and _range_covers_full_body(req_headers, content_length)):
|
if not (stream.status_code == 200 and _range_covers_full_body(req_headers, content_length)):
|
||||||
raise IOError(f"expected 206 for byte-range segment, got {stream.status_code}")
|
raise IOError(f"expected 206 for byte-range segment, got {stream.status_code}")
|
||||||
|
|
||||||
|
|||||||
@@ -840,8 +840,8 @@ def session(
|
|||||||
# connect_timeout + read_timeout mirror the requests-path default timeout (CONNECT_TIMEOUT, READ_TIMEOUT)
|
# connect_timeout + read_timeout mirror the requests-path default timeout (CONNECT_TIMEOUT, READ_TIMEOUT)
|
||||||
# so an unset config still gets a bounded connect and read like requests; pool_idle_timeout stays under
|
# so an unset config still gets a bounded connect and read like requests; pool_idle_timeout stays under
|
||||||
# the typical ~60s CDN idle kill; pool_max_idle_per_host follows POOL_MAX_SIZE, sized above the worker
|
# the typical ~60s CDN idle kill; pool_max_idle_per_host follows POOL_MAX_SIZE, sized above the worker
|
||||||
# cap because hedge racers and tail-boost parts push in-flight requests past it (see POOL_MAX_SIZE);
|
# cap because hedge racers and tail-boost parts push in-flight requests past it; tcp_keepalive keeps
|
||||||
# tcp_keepalive keeps long idle segments warm.
|
# long idle segments warm.
|
||||||
session_kwargs.setdefault("connect_timeout", CONNECT_TIMEOUT)
|
session_kwargs.setdefault("connect_timeout", CONNECT_TIMEOUT)
|
||||||
session_kwargs.setdefault("read_timeout", READ_TIMEOUT)
|
session_kwargs.setdefault("read_timeout", READ_TIMEOUT)
|
||||||
session_kwargs.setdefault("pool_idle_timeout", 55)
|
session_kwargs.setdefault("pool_idle_timeout", 55)
|
||||||
|
|||||||
Reference in New Issue
Block a user