fix(downloader): harden ranged reads, teardown and multiprocess sessions

Validates the 206 range end, bounds the DASH collapse, and preserves cookie domains and pool size in spawned children.

tests/core/test_adaptive_workers.py

@@ -1,7 +1,6 @@
-"""Unit tests for the opt-in adaptive worker controller (slow-start then AIMD hill-climb
-over segment concurrency) and the plumbing that carries the flag into the downloader. Pure
-controller tests only. No threads, sockets, or real downloads; the clock is supplied
-explicitly."""
+"""Unit tests for the opt-in adaptive worker controller (AIMD hill-climb over segment
+concurrency) and the plumbing that carries the flag into the downloader. Pure controller
+tests only. No threads, sockets, or real downloads; the clock is supplied explicitly."""
 
 from __future__ import annotations
 
@@ -55,31 +54,31 @@ def run_tick(c: AdaptiveWorkerController, rate: float, now: float, errors: int =
     return c.update(now), now
 
 
-def test_starts_at_min_six_cap() -> None:
-    assert AdaptiveWorkerController(cap=16).target == 6
-    assert AdaptiveWorkerController(cap=8).target == 6
-    assert AdaptiveWorkerController(cap=4).target == 4  # min(6, cap)
+def test_starts_at_cap() -> None:
+    assert AdaptiveWorkerController(cap=16).target == 16
+    assert AdaptiveWorkerController(cap=4).target == 4
     assert AdaptiveWorkerController(cap=1).target == 2  # clamped up to ADAPTIVE_MIN
+    assert AdaptiveWorkerController(cap=16, start=6).target == 6  # explicit start wins
 
 
 def test_no_op_before_warmup() -> None:
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     assert c.update(0.0) == 6  # arms last_tick before any samples are recorded
     c.record_bytes(500_000, 3.0)
     # a full tick has elapsed since arming, but under half a tick of samples exists -> hold
     assert c.update(4.5) == 6
     c.record_bytes(500_000, 6.5)
-    # warmed up (first sample 3.0, now 8.5) -> slow-start probe fires
-    assert c.update(8.5) == 12
+    # warmed up (first sample 3.0, now 8.5) -> the probe fires
+    assert c.update(8.5) == 8
 
 
-def test_ramps_up_while_speed_improves() -> None:
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+def test_climbs_while_speed_improves() -> None:
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     now = 0.0
     c.update(now)  # arm
     rate = 1_000_000.0
     now = fill_window(c, rate, now)
-    targets = [c.update(now)]  # first evaluation: slow-start doubles 6 -> 12
+    targets = [c.update(now)]  # first evaluation: 6 -> 8
     for _ in range(6):
         rate *= 2  # sustained >10% windowed gain each tick
         target, now = run_tick(c, rate, now)
@@ -89,23 +88,23 @@ def test_ramps_up_while_speed_improves() -> None:
     assert max(targets) <= 16
 
 
-def test_reverts_on_plateau_and_ends_slow_start() -> None:
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+def test_reverts_on_plateau() -> None:
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     now = 0.0
     c.update(now)
     rate = 1_000_000.0
     now = fill_window(c, rate, now)
-    assert c.update(now) == 12  # first increase slow-starts 6 -> 12
+    assert c.update(now) == 8  # first increase 6 -> 8
     # hold the rate flat: <10% gain -> the tick reverts the increase to its restore point
     target, now = run_tick(c, rate, now)
     assert target == 6
-    # slow-start is over: the next successful probe is additive (+2), not a double
+    # a paying probe climbs again from the restore point
     target, now = run_tick(c, rate * 4, now)
     assert target == 8
 
 
 def test_halves_and_cooldown_on_error_burst() -> None:
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     now = 0.0
     c.update(now)
     now = fill_window(c, 1_000_000.0, now)
@@ -150,7 +149,7 @@ def test_error_burst_floors_at_min() -> None:
 
 def test_tail_guard_suppresses_increase() -> None:
     # with fewer units of work than the target, the tick must hold instead of probing up
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     now = 0.0
     c.update(now)  # arm at target 6
     now = fill_window(c, 1_000_000.0, now)
@@ -159,16 +158,16 @@ def test_tail_guard_suppresses_increase() -> None:
 
 def test_tail_guard_none_path_unchanged() -> None:
     # without a tail count, the update still probes upward on the first evaluation
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     now = 0.0
     c.update(now)
     now = fill_window(c, 1_000_000.0, now)
-    assert c.update(now) == 12  # first evaluation slow-starts 6 -> 12
+    assert c.update(now) == 8  # first evaluation 6 -> 8
 
 
 def test_tail_guard_still_halves_on_error_burst() -> None:
     # the guard suppresses growth but never blocks an error-burst decrease
-    c = AdaptiveWorkerController(cap=16, tick=TICK, window=WINDOW)
+    c = AdaptiveWorkerController(cap=16, start=6, tick=TICK, window=WINDOW)
     now = 0.0
     c.update(now)
     now = fill_window(c, 1_000_000.0, now)

tests/core/test_dash_collapse.py

@@ -48,6 +48,12 @@ def test_first_segment_gap_after_init_ineligible():
     assert collapsible([(URL, "900-1999")], init_len=863) is False
 
 
+def test_gap_between_segments_ineligible():
+    # the merge path would skip 2000-2999; collapsing the parent would include it
+    segments = [(URL, "863-1999"), (URL, "3000-3999")]
+    assert collapsible(segments, init_len=863) is False
+
+
 def test_overlapping_ranges_ineligible():
     segments = [(URL, "863-2500"), (URL, "2000-3999")]
     assert collapsible(segments, init_len=863) is False

tests/core/test_dash_collapse_integration.py

@@ -27,6 +27,8 @@ from unshackle.core.tracks.track import DownloadContext
 # distinctive parent resource for the collapse case; init is its [0, INIT_LEN) prefix
 PARENT = bytes(i % 256 for i in range(10240))
 INIT_LEN = 512
+# last indexed byte for the clipped case; the parent carries 1024 bytes past it
+CLIP_END = len(PARENT) - 1025
 
 # separate resources for the non-collapse (mixed-URL) case
 INIT2 = bytes(range(100))
@@ -111,6 +113,27 @@ def collapse_mpd(srv) -> str:
 </MPD>"""
 
 
+def clipped_mpd(srv) -> str:
+    # same parent, but the ranges stop short of its end -> the collapsed file must be trimmed
+    # to the last indexed byte, not left holding the parent's trailing bytes.
+    return f"""<?xml version="1.0"?>
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
+  <BaseURL>{base(srv)}</BaseURL>
+  <Period id="0">
+    <AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
+      <Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
+        <BaseURL>media.mp4</BaseURL>
+        <SegmentList timescale="1">
+          <Initialization range="0-{INIT_LEN - 1}"/>
+          <SegmentURL mediaRange="{INIT_LEN}-5119"/>
+          <SegmentURL mediaRange="5120-{CLIP_END}"/>
+        </SegmentList>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>"""
+
+
 def mixed_mpd(srv) -> str:
     # Two SegmentURLs on distinct paths -> mixed URLs -> predicate False -> segmented path.
     return f"""<?xml version="1.0"?>
@@ -162,6 +185,17 @@ def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path)
     assert not list(tmp_path.glob("**/*.!dev"))
 
 
+def test_collapse_trims_parent_bytes_past_the_last_segment(server, tmp_path):
+    server.routes["/media.mp4"] = PARENT
+    track = make_track(clipped_mpd(server), server)
+    ctx = make_ctx(tmp_path, "clipped.mp4")
+
+    DASH.download_track(track, ctx)
+
+    # the parent's trailing 1024 bytes are not part of any segment, so they must not survive
+    assert ctx.save_path.read_bytes() == PARENT[: CLIP_END + 1]
+
+
 def test_non_collapse_takes_segmented_merge_path(server, tmp_path):
     server.routes["/init.bin"] = INIT2
     server.routes["/part0.bin"] = PART0

tests/core/test_retry_after.py

@@ -0,0 +1,69 @@
+"""RnetSession.get_sleep_time must survive hostile or malformed Retry-After headers."""
+
+from datetime import datetime, timedelta, timezone
+from email.utils import format_datetime
+from types import SimpleNamespace
+
+import pytest
+
+from unshackle.core.session import RnetSession
+
+
+@pytest.fixture
+def session():
+    return RnetSession()
+
+
+def response(retry_after=None):
+    headers = {} if retry_after is None else {"Retry-After": retry_after}
+    return SimpleNamespace(headers=headers)
+
+
+def test_numeric_retry_after_passes_through(session):
+    assert session.get_sleep_time(response("3"), 1) == 3.0
+
+
+def test_huge_numeric_retry_after_is_clamped(session):
+    assert session.get_sleep_time(response("86400"), 1) == session.max_backoff
+
+
+def test_huge_numeric_retry_after_respects_session_cap():
+    session = RnetSession(max_backoff=5.0)
+    assert session.get_sleep_time(response("86400"), 1) == 5.0
+
+
+def test_http_date_retry_after_honoured(session):
+    when = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=10), usegmt=True)
+    assert 5 <= session.get_sleep_time(response(when), 1) <= 10
+
+
+def test_naive_http_date_does_not_raise(session):
+    # "-0000" makes parsedate_to_datetime return a naive datetime
+    when = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=10)).replace("+0000", "-0000")
+    assert 5 <= session.get_sleep_time(response(when), 1) <= 10
+
+
+def test_past_http_date_yields_no_sleep(session):
+    when = format_datetime(datetime.now(timezone.utc) - timedelta(seconds=30), usegmt=True)
+    assert session.get_sleep_time(response(when), 1) <= 0
+
+
+def test_malformed_retry_after_falls_back_to_backoff(session):
+    value = session.get_sleep_time(response("not-a-date"), 1)
+    assert 0 < value <= session.max_backoff
+
+
+def test_empty_retry_after_falls_back_to_backoff(session):
+    value = session.get_sleep_time(response(""), 1)
+    assert 0 < value <= session.max_backoff
+
+
+def test_exponential_backoff_without_retry_after(session):
+    assert session.get_sleep_time(response(), 0) == 0.0
+    base = session.backoff_factor * 4
+    assert base * 0.9 <= session.get_sleep_time(response(), 3) <= base * 1.1
+    assert session.get_sleep_time(response(), 30) == session.max_backoff
+
+
+def test_no_response_still_backs_off(session):
+    assert session.get_sleep_time(None, 30) == session.max_backoff