mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-19 07:25:11 +00:00
feat(downloader): resume failed downloads from completed segments across runs
Fingerprint-gated reuse of atomically-published segment files (DASH/ISM/HLS) behind a continue_downloads config option and one-off dl --continue-downloads flag.
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
"""Offline unit tests for the cross-run segment-resume sidecar: fingerprint stability
|
||||
across rotating query tokens, sensitivity to real segmentation changes, sidecar
|
||||
read/write robustness, and the sidecar-outside-save_dir invariant that protects the
|
||||
merge glob from concatenating it into the output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from unshackle.core.tracks import resume
|
||||
|
||||
MPD = "https://cdn/manifest.mpd"
|
||||
SEGMENTS = [
|
||||
("https://cdn/media.mp4", "863-1999"),
|
||||
("https://cdn/media.mp4", "2000-3999"),
|
||||
("https://cdn/media.mp4", "4000-5999"),
|
||||
]
|
||||
|
||||
|
||||
def test_identical_inputs_identical_digest():
|
||||
assert resume.fingerprint(MPD, SEGMENTS) == resume.fingerprint(MPD, SEGMENTS)
|
||||
|
||||
|
||||
def test_rotating_query_token_ignored():
|
||||
tokenised = [(f"{url}?token=abc123&expires=999", rng) for url, rng in SEGMENTS]
|
||||
assert resume.fingerprint(f"{MPD}?auth=xyz", tokenised) == resume.fingerprint(MPD, SEGMENTS)
|
||||
|
||||
|
||||
def test_changed_byte_range_changes_digest():
|
||||
changed = [SEGMENTS[0], ("https://cdn/media.mp4", "2000-4000"), SEGMENTS[2]]
|
||||
assert resume.fingerprint(MPD, changed) != resume.fingerprint(MPD, SEGMENTS)
|
||||
|
||||
|
||||
def test_changed_segment_count_changes_digest():
|
||||
assert resume.fingerprint(MPD, SEGMENTS[:2]) != resume.fingerprint(MPD, SEGMENTS)
|
||||
|
||||
|
||||
def test_changed_url_path_changes_digest():
|
||||
moved = [("https://cdn/other.mp4", rng) for _, rng in SEGMENTS]
|
||||
assert resume.fingerprint(MPD, moved) != resume.fingerprint(MPD, SEGMENTS)
|
||||
|
||||
|
||||
def test_query_carried_identity_is_not_collapsed():
|
||||
# all segments share a path and differ only in the query (e.g. $Time$ as a query
|
||||
# param): stripping must not collapse them into false matches; the digest goes
|
||||
# verbatim and distinct segmentations stay distinct
|
||||
a = [("https://cdn/seg?t=0", None), ("https://cdn/seg?t=100", None)]
|
||||
b = [("https://cdn/seg?t=0", None), ("https://cdn/seg?t=200", None)]
|
||||
assert resume.fingerprint(MPD, a) != resume.fingerprint(MPD, b)
|
||||
|
||||
|
||||
def test_extra_inputs_change_digest():
|
||||
assert resume.fingerprint(MPD, SEGMENTS, extra=["100", "3"]) != resume.fingerprint(MPD, SEGMENTS)
|
||||
|
||||
|
||||
def test_reusable_roundtrip(tmp_path: Path):
|
||||
save_dir = tmp_path / "Video_x_segments"
|
||||
digest = resume.fingerprint(MPD, SEGMENTS)
|
||||
resume.write_sidecar(save_dir, digest)
|
||||
assert resume.reusable(save_dir, digest) is True
|
||||
assert resume.reusable(save_dir, "0" * 64) is False
|
||||
|
||||
|
||||
def test_reusable_never_raises(tmp_path: Path):
|
||||
save_dir = tmp_path / "Video_x_segments"
|
||||
digest = resume.fingerprint(MPD, SEGMENTS)
|
||||
assert resume.reusable(save_dir, digest) is False # no sidecar
|
||||
resume.sidecar_path(save_dir).write_text("{not json", encoding="utf-8")
|
||||
assert resume.reusable(save_dir, digest) is False # malformed
|
||||
resume.sidecar_path(save_dir).write_text(json.dumps({"version": 1}), encoding="utf-8")
|
||||
assert resume.reusable(save_dir, digest) is False # no digest key
|
||||
|
||||
|
||||
def test_clear_sidecar(tmp_path: Path):
|
||||
save_dir = tmp_path / "Video_x_segments"
|
||||
digest = resume.fingerprint(MPD, SEGMENTS)
|
||||
resume.write_sidecar(save_dir, digest)
|
||||
resume.clear_sidecar(save_dir)
|
||||
assert resume.reusable(save_dir, digest) is False
|
||||
resume.clear_sidecar(save_dir) # idempotent on a missing file
|
||||
|
||||
|
||||
def test_sidecar_is_never_inside_save_dir(tmp_path: Path):
|
||||
save_dir = tmp_path / "Video_x_segments"
|
||||
assert resume.sidecar_path(save_dir).parent == save_dir.parent
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Integration coverage for cross-run segment resume on the HLS path.
|
||||
|
||||
Same two-run shape as the DASH test, plus the HLS-specific rules: the sidecar (the
|
||||
"download phase completed cleanly" proof) survives a mid-download crash and is gone once
|
||||
the merge pass runs, and AES-128 content is excluded from resume entirely because its
|
||||
segments are decrypted in place during the merge pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
from unshackle.core.manifests import HLS
|
||||
from unshackle.core.tracks import resume
|
||||
from unshackle.core.tracks.track import DownloadContext
|
||||
|
||||
PART0 = bytes([0xA0]) * 300
|
||||
PART1 = bytes([0xB1]) * 200
|
||||
AES_KEY = bytes(range(16))
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
server = self.server
|
||||
with server.lock:
|
||||
server.requests.append(self.path)
|
||||
body = server.routes.get(self.path)
|
||||
if body is None:
|
||||
self.send_response(404)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
||||
srv.lock = threading.Lock()
|
||||
srv.requests = []
|
||||
srv.routes = {}
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def resume_enabled(monkeypatch):
|
||||
monkeypatch.setattr(config, "continue_downloads", True)
|
||||
monkeypatch.setattr(importlib.import_module("unshackle.core.downloaders.requests"), "MAX_ATTEMPTS", 1)
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def base(srv) -> str:
|
||||
return f"http://127.0.0.1:{srv.server_address[1]}/"
|
||||
|
||||
|
||||
MASTER = """#EXTM3U
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=1920x1080,CODECS="avc1.640028"
|
||||
media.m3u8
|
||||
"""
|
||||
|
||||
|
||||
def media_playlist(key_line: str = "") -> bytes:
|
||||
return f"""#EXTM3U
|
||||
#EXT-X-VERSION:3
|
||||
#EXT-X-TARGETDURATION:4
|
||||
{key_line}#EXTINF:4.0,
|
||||
part0.bin
|
||||
#EXTINF:4.0,
|
||||
part1.bin
|
||||
#EXT-X-ENDLIST
|
||||
""".encode()
|
||||
|
||||
|
||||
def make_track(srv):
|
||||
return HLS.from_text(MASTER, url=f"{base(srv)}master.m3u8").to_tracks(language="en").videos[0]
|
||||
|
||||
|
||||
def make_ctx(tmp_path):
|
||||
return DownloadContext(
|
||||
save_path=tmp_path / "out.mp4",
|
||||
save_dir=tmp_path / "v0_segments",
|
||||
progress=partial(lambda **kw: None),
|
||||
max_workers=1,
|
||||
)
|
||||
|
||||
|
||||
def test_hls_second_run_reuses_completed_segments(server, tmp_path):
|
||||
server.routes["/media.m3u8"] = media_playlist()
|
||||
server.routes["/part0.bin"] = PART0
|
||||
# part1.bin missing -> run 1 completes part0 then dies during the download phase
|
||||
|
||||
ctx = make_ctx(tmp_path)
|
||||
with pytest.raises(Exception):
|
||||
HLS.download_track(make_track(server), ctx)
|
||||
|
||||
assert (ctx.save_dir / "segments" / "0.bin").read_bytes() == PART0
|
||||
assert resume.sidecar_path(ctx.save_dir).exists()
|
||||
part0_fetches_run1 = server.requests.count("/part0.bin")
|
||||
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
server.routes["/part1.bin"] = PART1
|
||||
HLS.download_track(make_track(server), ctx)
|
||||
|
||||
assert ctx.save_path.read_bytes() == PART0 + PART1
|
||||
assert server.requests.count("/part0.bin") == part0_fetches_run1
|
||||
assert not resume.sidecar_path(ctx.save_dir).exists()
|
||||
|
||||
|
||||
def test_hls_aes128_content_never_writes_sidecar(server, tmp_path):
|
||||
key_line = f'#EXT-X-KEY:METHOD=AES-128,URI="{base(server)}key.bin"\n'
|
||||
server.routes["/media.m3u8"] = media_playlist(key_line)
|
||||
server.routes["/key.bin"] = AES_KEY
|
||||
server.routes["/part0.bin"] = PART0
|
||||
# part1.bin missing -> crash mid-download, the state resume would normally preserve
|
||||
|
||||
ctx = make_ctx(tmp_path)
|
||||
with pytest.raises(Exception):
|
||||
HLS.download_track(make_track(server), ctx)
|
||||
|
||||
assert not resume.sidecar_path(ctx.save_dir).exists()
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Integration coverage for cross-run segment resume on the segmented DASH path.
|
||||
|
||||
Drives the real ``DASH.download_track`` twice against a localhost server with
|
||||
``continue_downloads`` on: the first run dies on a missing segment (after completing the
|
||||
other one), the second run finds the sidecar digest unchanged and must reuse the completed
|
||||
segment file instead of re-fetching it. A third scenario changes the segmentation and
|
||||
asserts the stale directory is wiped rather than mixed into the new download.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED
|
||||
from unshackle.core.manifests import DASH
|
||||
from unshackle.core.tracks import resume
|
||||
from unshackle.core.tracks.track import DownloadContext
|
||||
|
||||
INIT = bytes(range(100))
|
||||
PART0 = bytes([0xA0]) * 300
|
||||
PART1 = bytes([0xB1]) * 200
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
server = self.server
|
||||
with server.lock:
|
||||
server.requests.append(self.path)
|
||||
body = server.routes.get(self.path)
|
||||
if body is None:
|
||||
self.send_response(404)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
||||
srv.lock = threading.Lock()
|
||||
srv.requests = []
|
||||
srv.routes = {}
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield srv
|
||||
srv.shutdown()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def resume_enabled(monkeypatch):
|
||||
monkeypatch.setattr(config, "continue_downloads", True)
|
||||
# fail fast on the deliberately-missing segment instead of the full retry ladder
|
||||
# (import via importlib: the downloaders package exports a same-named function)
|
||||
monkeypatch.setattr(importlib.import_module("unshackle.core.downloaders.requests"), "MAX_ATTEMPTS", 1)
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
yield
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def base(srv) -> str:
|
||||
return f"http://127.0.0.1:{srv.server_address[1]}/"
|
||||
|
||||
|
||||
def mpd(srv, part1: str = "part1.bin") -> str:
|
||||
return f"""<?xml version="1.0"?>
|
||||
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
|
||||
<BaseURL>{base(srv)}</BaseURL>
|
||||
<Period id="0">
|
||||
<AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
|
||||
<Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
|
||||
<SegmentList timescale="1">
|
||||
<Initialization sourceURL="init.bin"/>
|
||||
<SegmentURL media="part0.bin"/>
|
||||
<SegmentURL media="{part1}"/>
|
||||
</SegmentList>
|
||||
</Representation>
|
||||
</AdaptationSet>
|
||||
</Period>
|
||||
</MPD>"""
|
||||
|
||||
|
||||
def make_track(text: str, srv):
|
||||
return DASH.from_text(text, url=f"{base(srv)}manifest.mpd").to_tracks(language="en").videos[0]
|
||||
|
||||
|
||||
def make_ctx(tmp_path):
|
||||
return DownloadContext(
|
||||
save_path=tmp_path / "out.mp4",
|
||||
save_dir=tmp_path / "v0_segments",
|
||||
progress=partial(lambda **kw: None),
|
||||
# sequential: part0 must be complete before the deliberate part1 failure aborts run 1
|
||||
max_workers=1,
|
||||
)
|
||||
|
||||
|
||||
def test_second_run_reuses_completed_segments(server, tmp_path):
|
||||
server.routes["/init.bin"] = INIT
|
||||
server.routes["/part0.bin"] = PART0
|
||||
# part1.bin is missing -> the first run completes part0 then dies on part1
|
||||
|
||||
ctx = make_ctx(tmp_path)
|
||||
with pytest.raises(Exception):
|
||||
DASH.download_track(make_track(mpd(server), server), ctx)
|
||||
|
||||
assert (ctx.save_dir / "0.mp4").read_bytes() == PART0
|
||||
assert resume.sidecar_path(ctx.save_dir).exists()
|
||||
part0_fetches_run1 = server.requests.count("/part0.bin")
|
||||
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
server.routes["/part1.bin"] = PART1
|
||||
DASH.download_track(make_track(mpd(server), server), ctx)
|
||||
|
||||
assert ctx.save_path.read_bytes() == INIT + PART0 + PART1
|
||||
assert server.requests.count("/part0.bin") == part0_fetches_run1
|
||||
assert not resume.sidecar_path(ctx.save_dir).exists()
|
||||
assert not ctx.save_dir.exists()
|
||||
|
||||
|
||||
def test_changed_segmentation_wipes_stale_segments(server, tmp_path):
|
||||
server.routes["/init.bin"] = INIT
|
||||
server.routes["/part0.bin"] = PART0
|
||||
|
||||
ctx = make_ctx(tmp_path)
|
||||
with pytest.raises(Exception):
|
||||
DASH.download_track(make_track(mpd(server), server), ctx)
|
||||
assert (ctx.save_dir / "0.mp4").exists()
|
||||
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
# different final segment path -> different digest -> stale part0 must be re-fetched
|
||||
server.routes["/part2.bin"] = PART1
|
||||
part0_fetches_run1 = server.requests.count("/part0.bin")
|
||||
DASH.download_track(make_track(mpd(server, part1="part2.bin"), server), ctx)
|
||||
|
||||
assert ctx.save_path.read_bytes() == INIT + PART0 + PART1
|
||||
assert server.requests.count("/part0.bin") == part0_fetches_run1 + 1
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Lifecycle tests for the conditional wipe in Track.download itself.
|
||||
|
||||
The manifest-parser gates are covered by the DASH/HLS integration tests; these cover the
|
||||
layer above them: with continue_downloads on, Track.download's entry sweep and its
|
||||
exception handler must both preserve completed segments and the sidecar while purging
|
||||
every partial, and with the flag off both must wipe the directory entirely. The manifest
|
||||
parser is stubbed so only Track.download's own behaviour is under test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY
|
||||
from unshackle.core.tracks import Video, resume
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_flags(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config.directories, "temp", tmp_path)
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
DOWNLOAD_LICENCE_ONLY.clear()
|
||||
yield
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
def make_track() -> Video:
|
||||
return Video(
|
||||
id_="t1",
|
||||
url="https://example.test/manifest.mpd",
|
||||
language="en",
|
||||
codec=Video.Codec.AVC,
|
||||
range_=Video.Range.SDR,
|
||||
width=1920,
|
||||
height=1080,
|
||||
bitrate=5_000_000,
|
||||
descriptor=Video.Descriptor.DASH,
|
||||
)
|
||||
|
||||
|
||||
def seed_prior_run(tmp_path: Path) -> Path:
|
||||
save_dir = tmp_path / "Video_t1.mp4_segments"
|
||||
(save_dir / "segments").mkdir(parents=True)
|
||||
(save_dir / "0.mp4").write_bytes(b"complete segment")
|
||||
(save_dir / "1.mp4.p.!dev").write_bytes(b"partial")
|
||||
(save_dir / "segments" / "2.bin.h.!dev").write_bytes(b"hedge partial")
|
||||
resume.write_sidecar(save_dir, "f" * 64)
|
||||
return save_dir
|
||||
|
||||
|
||||
def run_download(track: Video, monkeypatch, fail: bool):
|
||||
import requests as requests_lib
|
||||
|
||||
from unshackle.core.manifests import DASH
|
||||
|
||||
def stub_download_track(track, ctx):
|
||||
if fail:
|
||||
raise RuntimeError("simulated mid-download crash")
|
||||
ctx.save_path.write_bytes(b"output")
|
||||
track.path = ctx.save_path
|
||||
|
||||
monkeypatch.setattr(DASH, "download_track", staticmethod(stub_download_track))
|
||||
track.download(
|
||||
session=requests_lib.Session(),
|
||||
prepare_drm=partial(lambda *a, **kw: None),
|
||||
progress=partial(lambda **kw: None),
|
||||
)
|
||||
|
||||
|
||||
def test_flag_on_keeps_segments_and_sidecar_sweeps_partials(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "continue_downloads", True)
|
||||
save_dir = seed_prior_run(tmp_path)
|
||||
track = make_track()
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
run_download(track, monkeypatch, fail=True)
|
||||
|
||||
assert (save_dir / "0.mp4").read_bytes() == b"complete segment"
|
||||
assert resume.sidecar_path(save_dir).exists()
|
||||
assert not list(save_dir.rglob("*.!dev"))
|
||||
|
||||
|
||||
def test_flag_off_wipes_segments_and_sidecar(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "continue_downloads", False)
|
||||
save_dir = seed_prior_run(tmp_path)
|
||||
track = make_track()
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
run_download(track, monkeypatch, fail=True)
|
||||
|
||||
assert not save_dir.exists()
|
||||
assert not resume.sidecar_path(save_dir).exists()
|
||||
|
||||
|
||||
def test_flag_off_clears_orphaned_sidecar_without_dir(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "continue_downloads", False)
|
||||
save_dir = tmp_path / "Video_t1.mp4_segments"
|
||||
# a kill between a parser's dir removal and its sidecar clear leaves exactly this
|
||||
resume.write_sidecar(save_dir, "f" * 64)
|
||||
track = make_track()
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
run_download(track, monkeypatch, fail=True)
|
||||
|
||||
assert not resume.sidecar_path(save_dir).exists()
|
||||
Reference in New Issue
Block a user