feat(downloader): resume failed downloads from completed segments across runs

Fingerprint-gated reuse of atomically-published segment files (DASH/ISM/HLS) behind a continue_downloads config option and one-off dl --continue-downloads flag.
This commit is contained in:
imSp4rky
2026-08-16 19:38:06 -06:00
parent dd9dafef91
commit 619c86cd10
18 changed files with 675 additions and 11 deletions
+86
View File
@@ -0,0 +1,86 @@
"""Offline unit tests for the cross-run segment-resume sidecar: fingerprint stability
across rotating query tokens, sensitivity to real segmentation changes, sidecar
read/write robustness, and the sidecar-outside-save_dir invariant that protects the
merge glob from concatenating it into the output."""
from __future__ import annotations
import json
from pathlib import Path
from unshackle.core.tracks import resume
MPD = "https://cdn/manifest.mpd"
SEGMENTS = [
("https://cdn/media.mp4", "863-1999"),
("https://cdn/media.mp4", "2000-3999"),
("https://cdn/media.mp4", "4000-5999"),
]
def test_identical_inputs_identical_digest():
assert resume.fingerprint(MPD, SEGMENTS) == resume.fingerprint(MPD, SEGMENTS)
def test_rotating_query_token_ignored():
tokenised = [(f"{url}?token=abc123&expires=999", rng) for url, rng in SEGMENTS]
assert resume.fingerprint(f"{MPD}?auth=xyz", tokenised) == resume.fingerprint(MPD, SEGMENTS)
def test_changed_byte_range_changes_digest():
changed = [SEGMENTS[0], ("https://cdn/media.mp4", "2000-4000"), SEGMENTS[2]]
assert resume.fingerprint(MPD, changed) != resume.fingerprint(MPD, SEGMENTS)
def test_changed_segment_count_changes_digest():
assert resume.fingerprint(MPD, SEGMENTS[:2]) != resume.fingerprint(MPD, SEGMENTS)
def test_changed_url_path_changes_digest():
moved = [("https://cdn/other.mp4", rng) for _, rng in SEGMENTS]
assert resume.fingerprint(MPD, moved) != resume.fingerprint(MPD, SEGMENTS)
def test_query_carried_identity_is_not_collapsed():
# all segments share a path and differ only in the query (e.g. $Time$ as a query
# param): stripping must not collapse them into false matches; the digest goes
# verbatim and distinct segmentations stay distinct
a = [("https://cdn/seg?t=0", None), ("https://cdn/seg?t=100", None)]
b = [("https://cdn/seg?t=0", None), ("https://cdn/seg?t=200", None)]
assert resume.fingerprint(MPD, a) != resume.fingerprint(MPD, b)
def test_extra_inputs_change_digest():
assert resume.fingerprint(MPD, SEGMENTS, extra=["100", "3"]) != resume.fingerprint(MPD, SEGMENTS)
def test_reusable_roundtrip(tmp_path: Path):
save_dir = tmp_path / "Video_x_segments"
digest = resume.fingerprint(MPD, SEGMENTS)
resume.write_sidecar(save_dir, digest)
assert resume.reusable(save_dir, digest) is True
assert resume.reusable(save_dir, "0" * 64) is False
def test_reusable_never_raises(tmp_path: Path):
save_dir = tmp_path / "Video_x_segments"
digest = resume.fingerprint(MPD, SEGMENTS)
assert resume.reusable(save_dir, digest) is False # no sidecar
resume.sidecar_path(save_dir).write_text("{not json", encoding="utf-8")
assert resume.reusable(save_dir, digest) is False # malformed
resume.sidecar_path(save_dir).write_text(json.dumps({"version": 1}), encoding="utf-8")
assert resume.reusable(save_dir, digest) is False # no digest key
def test_clear_sidecar(tmp_path: Path):
save_dir = tmp_path / "Video_x_segments"
digest = resume.fingerprint(MPD, SEGMENTS)
resume.write_sidecar(save_dir, digest)
resume.clear_sidecar(save_dir)
assert resume.reusable(save_dir, digest) is False
resume.clear_sidecar(save_dir) # idempotent on a missing file
def test_sidecar_is_never_inside_save_dir(tmp_path: Path):
save_dir = tmp_path / "Video_x_segments"
assert resume.sidecar_path(save_dir).parent == save_dir.parent
+139
View File
@@ -0,0 +1,139 @@
"""Integration coverage for cross-run segment resume on the HLS path.
Same two-run shape as the DASH test, plus the HLS-specific rules: the sidecar (the
"download phase completed cleanly" proof) survives a mid-download crash and is gone once
the merge pass runs, and AES-128 content is excluded from resume entirely because its
segments are decrypted in place during the merge pass.
"""
from __future__ import annotations
import importlib
import threading
from functools import partial
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from unshackle.core.config import config
from unshackle.core.constants import DOWNLOAD_CANCELLED
from unshackle.core.manifests import HLS
from unshackle.core.tracks import resume
from unshackle.core.tracks.track import DownloadContext
PART0 = bytes([0xA0]) * 300
PART1 = bytes([0xB1]) * 200
AES_KEY = bytes(range(16))
class _Handler(BaseHTTPRequestHandler):
def do_GET(self):
server = self.server
with server.lock:
server.requests.append(self.path)
body = server.routes.get(self.path)
if body is None:
self.send_response(404)
self.send_header("Content-Length", "0")
self.end_headers()
return
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
@pytest.fixture
def server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
srv.lock = threading.Lock()
srv.requests = []
srv.routes = {}
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
srv.shutdown()
thread.join(timeout=5)
@pytest.fixture(autouse=True)
def resume_enabled(monkeypatch):
monkeypatch.setattr(config, "continue_downloads", True)
monkeypatch.setattr(importlib.import_module("unshackle.core.downloaders.requests"), "MAX_ATTEMPTS", 1)
DOWNLOAD_CANCELLED.clear()
yield
DOWNLOAD_CANCELLED.clear()
def base(srv) -> str:
return f"http://127.0.0.1:{srv.server_address[1]}/"
MASTER = """#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=1920x1080,CODECS="avc1.640028"
media.m3u8
"""
def media_playlist(key_line: str = "") -> bytes:
return f"""#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:4
{key_line}#EXTINF:4.0,
part0.bin
#EXTINF:4.0,
part1.bin
#EXT-X-ENDLIST
""".encode()
def make_track(srv):
return HLS.from_text(MASTER, url=f"{base(srv)}master.m3u8").to_tracks(language="en").videos[0]
def make_ctx(tmp_path):
return DownloadContext(
save_path=tmp_path / "out.mp4",
save_dir=tmp_path / "v0_segments",
progress=partial(lambda **kw: None),
max_workers=1,
)
def test_hls_second_run_reuses_completed_segments(server, tmp_path):
server.routes["/media.m3u8"] = media_playlist()
server.routes["/part0.bin"] = PART0
# part1.bin missing -> run 1 completes part0 then dies during the download phase
ctx = make_ctx(tmp_path)
with pytest.raises(Exception):
HLS.download_track(make_track(server), ctx)
assert (ctx.save_dir / "segments" / "0.bin").read_bytes() == PART0
assert resume.sidecar_path(ctx.save_dir).exists()
part0_fetches_run1 = server.requests.count("/part0.bin")
DOWNLOAD_CANCELLED.clear()
server.routes["/part1.bin"] = PART1
HLS.download_track(make_track(server), ctx)
assert ctx.save_path.read_bytes() == PART0 + PART1
assert server.requests.count("/part0.bin") == part0_fetches_run1
assert not resume.sidecar_path(ctx.save_dir).exists()
def test_hls_aes128_content_never_writes_sidecar(server, tmp_path):
key_line = f'#EXT-X-KEY:METHOD=AES-128,URI="{base(server)}key.bin"\n'
server.routes["/media.m3u8"] = media_playlist(key_line)
server.routes["/key.bin"] = AES_KEY
server.routes["/part0.bin"] = PART0
# part1.bin missing -> crash mid-download, the state resume would normally preserve
ctx = make_ctx(tmp_path)
with pytest.raises(Exception):
HLS.download_track(make_track(server), ctx)
assert not resume.sidecar_path(ctx.save_dir).exists()
+149
View File
@@ -0,0 +1,149 @@
"""Integration coverage for cross-run segment resume on the segmented DASH path.
Drives the real ``DASH.download_track`` twice against a localhost server with
``continue_downloads`` on: the first run dies on a missing segment (after completing the
other one), the second run finds the sidecar digest unchanged and must reuse the completed
segment file instead of re-fetching it. A third scenario changes the segmentation and
asserts the stale directory is wiped rather than mixed into the new download.
"""
from __future__ import annotations
import importlib
import threading
from functools import partial
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from unshackle.core.config import config
from unshackle.core.constants import DOWNLOAD_CANCELLED
from unshackle.core.manifests import DASH
from unshackle.core.tracks import resume
from unshackle.core.tracks.track import DownloadContext
INIT = bytes(range(100))
PART0 = bytes([0xA0]) * 300
PART1 = bytes([0xB1]) * 200
class _Handler(BaseHTTPRequestHandler):
def do_GET(self):
server = self.server
with server.lock:
server.requests.append(self.path)
body = server.routes.get(self.path)
if body is None:
self.send_response(404)
self.send_header("Content-Length", "0")
self.end_headers()
return
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
@pytest.fixture
def server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
srv.lock = threading.Lock()
srv.requests = []
srv.routes = {}
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
yield srv
srv.shutdown()
thread.join(timeout=5)
@pytest.fixture(autouse=True)
def resume_enabled(monkeypatch):
monkeypatch.setattr(config, "continue_downloads", True)
# fail fast on the deliberately-missing segment instead of the full retry ladder
# (import via importlib: the downloaders package exports a same-named function)
monkeypatch.setattr(importlib.import_module("unshackle.core.downloaders.requests"), "MAX_ATTEMPTS", 1)
DOWNLOAD_CANCELLED.clear()
yield
DOWNLOAD_CANCELLED.clear()
def base(srv) -> str:
return f"http://127.0.0.1:{srv.server_address[1]}/"
def mpd(srv, part1: str = "part1.bin") -> str:
return f"""<?xml version="1.0"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
<BaseURL>{base(srv)}</BaseURL>
<Period id="0">
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
<Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
<SegmentList timescale="1">
<Initialization sourceURL="init.bin"/>
<SegmentURL media="part0.bin"/>
<SegmentURL media="{part1}"/>
</SegmentList>
</Representation>
</AdaptationSet>
</Period>
</MPD>"""
def make_track(text: str, srv):
return DASH.from_text(text, url=f"{base(srv)}manifest.mpd").to_tracks(language="en").videos[0]
def make_ctx(tmp_path):
return DownloadContext(
save_path=tmp_path / "out.mp4",
save_dir=tmp_path / "v0_segments",
progress=partial(lambda **kw: None),
# sequential: part0 must be complete before the deliberate part1 failure aborts run 1
max_workers=1,
)
def test_second_run_reuses_completed_segments(server, tmp_path):
server.routes["/init.bin"] = INIT
server.routes["/part0.bin"] = PART0
# part1.bin is missing -> the first run completes part0 then dies on part1
ctx = make_ctx(tmp_path)
with pytest.raises(Exception):
DASH.download_track(make_track(mpd(server), server), ctx)
assert (ctx.save_dir / "0.mp4").read_bytes() == PART0
assert resume.sidecar_path(ctx.save_dir).exists()
part0_fetches_run1 = server.requests.count("/part0.bin")
DOWNLOAD_CANCELLED.clear()
server.routes["/part1.bin"] = PART1
DASH.download_track(make_track(mpd(server), server), ctx)
assert ctx.save_path.read_bytes() == INIT + PART0 + PART1
assert server.requests.count("/part0.bin") == part0_fetches_run1
assert not resume.sidecar_path(ctx.save_dir).exists()
assert not ctx.save_dir.exists()
def test_changed_segmentation_wipes_stale_segments(server, tmp_path):
server.routes["/init.bin"] = INIT
server.routes["/part0.bin"] = PART0
ctx = make_ctx(tmp_path)
with pytest.raises(Exception):
DASH.download_track(make_track(mpd(server), server), ctx)
assert (ctx.save_dir / "0.mp4").exists()
DOWNLOAD_CANCELLED.clear()
# different final segment path -> different digest -> stale part0 must be re-fetched
server.routes["/part2.bin"] = PART1
part0_fetches_run1 = server.requests.count("/part0.bin")
DASH.download_track(make_track(mpd(server, part1="part2.bin"), server), ctx)
assert ctx.save_path.read_bytes() == INIT + PART0 + PART1
assert server.requests.count("/part0.bin") == part0_fetches_run1 + 1
+109
View File
@@ -0,0 +1,109 @@
"""Lifecycle tests for the conditional wipe in Track.download itself.
The manifest-parser gates are covered by the DASH/HLS integration tests; these cover the
layer above them: with continue_downloads on, Track.download's entry sweep and its
exception handler must both preserve completed segments and the sidecar while purging
every partial, and with the flag off both must wipe the directory entirely. The manifest
parser is stubbed so only Track.download's own behaviour is under test.
"""
from __future__ import annotations
from functools import partial
from pathlib import Path
import pytest
from unshackle.core.config import config
from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY
from unshackle.core.tracks import Video, resume
@pytest.fixture(autouse=True)
def clean_flags(tmp_path, monkeypatch):
monkeypatch.setattr(config.directories, "temp", tmp_path)
DOWNLOAD_CANCELLED.clear()
DOWNLOAD_LICENCE_ONLY.clear()
yield
DOWNLOAD_CANCELLED.clear()
def make_track() -> Video:
return Video(
id_="t1",
url="https://example.test/manifest.mpd",
language="en",
codec=Video.Codec.AVC,
range_=Video.Range.SDR,
width=1920,
height=1080,
bitrate=5_000_000,
descriptor=Video.Descriptor.DASH,
)
def seed_prior_run(tmp_path: Path) -> Path:
save_dir = tmp_path / "Video_t1.mp4_segments"
(save_dir / "segments").mkdir(parents=True)
(save_dir / "0.mp4").write_bytes(b"complete segment")
(save_dir / "1.mp4.p.!dev").write_bytes(b"partial")
(save_dir / "segments" / "2.bin.h.!dev").write_bytes(b"hedge partial")
resume.write_sidecar(save_dir, "f" * 64)
return save_dir
def run_download(track: Video, monkeypatch, fail: bool):
import requests as requests_lib
from unshackle.core.manifests import DASH
def stub_download_track(track, ctx):
if fail:
raise RuntimeError("simulated mid-download crash")
ctx.save_path.write_bytes(b"output")
track.path = ctx.save_path
monkeypatch.setattr(DASH, "download_track", staticmethod(stub_download_track))
track.download(
session=requests_lib.Session(),
prepare_drm=partial(lambda *a, **kw: None),
progress=partial(lambda **kw: None),
)
def test_flag_on_keeps_segments_and_sidecar_sweeps_partials(tmp_path, monkeypatch):
monkeypatch.setattr(config, "continue_downloads", True)
save_dir = seed_prior_run(tmp_path)
track = make_track()
with pytest.raises(RuntimeError):
run_download(track, monkeypatch, fail=True)
assert (save_dir / "0.mp4").read_bytes() == b"complete segment"
assert resume.sidecar_path(save_dir).exists()
assert not list(save_dir.rglob("*.!dev"))
def test_flag_off_wipes_segments_and_sidecar(tmp_path, monkeypatch):
monkeypatch.setattr(config, "continue_downloads", False)
save_dir = seed_prior_run(tmp_path)
track = make_track()
with pytest.raises(RuntimeError):
run_download(track, monkeypatch, fail=True)
assert not save_dir.exists()
assert not resume.sidecar_path(save_dir).exists()
def test_flag_off_clears_orphaned_sidecar_without_dir(tmp_path, monkeypatch):
monkeypatch.setattr(config, "continue_downloads", False)
save_dir = tmp_path / "Video_t1.mp4_segments"
# a kill between a parser's dir removal and its sidecar clear leaves exactly this
resume.write_sidecar(save_dir, "f" * 64)
track = make_track()
with pytest.raises(RuntimeError):
run_download(track, monkeypatch, fail=True)
assert not resume.sidecar_path(save_dir).exists()