mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 20:15:10 +00:00
perf(dash): collapse single-URL byte-range segments into one direct download
Also throttle hedge-median recompute to 0.5s, cache replace_fields format patterns and short-circuit when no $ token, and stream HLS merge with copyfileobj. Adds test_dash_collapse and live A/B tooling.
This commit is contained in:
imSp4rky
@@ -7,6 +7,9 @@ The controller logs its worker-target changes to a JSON-lines debug log; this sc
|
||||
reads them back and prints the scaling sequence per stream, so you can see whether the
|
||||
downloader ramps workers up against each CDN or finishes before the controller acts.
|
||||
|
||||
STREAMS carries a public 4K/HDR/Dolby-Vision tier (labels uhd-hevc, dv-atmos, hevc-hls,
|
||||
h265-cmaf) for exercising large high-bitrate tracks; filter it with --only uhd / dv / hevc / h265.
|
||||
|
||||
Throwaway-ish operational tool: it makes real network requests to public test assets.
|
||||
Run: uv run python scripts/live_manifest_test.py [--workers N] [--track smallest|largest]
|
||||
"""
|
||||
@@ -17,7 +20,6 @@ import json
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from tempfile import mkdtemp
|
||||
from typing import Any, Optional
|
||||
@@ -33,16 +35,51 @@ from unshackle.core.utilities import close_debug_logger, init_debug_logger
|
||||
# all three formats on Unified Streaming, so the formats are comparable on one CDN.
|
||||
STREAMS: list[tuple[str, Any, str, str]] = [
|
||||
("HLS", HLS, "mux", "https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8"),
|
||||
("HLS", HLS, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8"),
|
||||
(
|
||||
"HLS",
|
||||
HLS,
|
||||
"unified",
|
||||
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8",
|
||||
),
|
||||
("DASH", DASH, "akamai", "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps.mpd"),
|
||||
("DASH", DASH, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.mpd"),
|
||||
(
|
||||
"DASH",
|
||||
DASH,
|
||||
"unified",
|
||||
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.mpd",
|
||||
),
|
||||
("DASH", DASH, "dashif", "https://livesim2.dashif.org/vod/testpic_2s/Manifest.mpd"),
|
||||
("ISM", ISM, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/Manifest"),
|
||||
("HLS", HLS, "apple", "https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8"),
|
||||
(
|
||||
"ISM",
|
||||
ISM,
|
||||
"unified",
|
||||
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/Manifest",
|
||||
),
|
||||
(
|
||||
"HLS",
|
||||
HLS,
|
||||
"apple",
|
||||
"https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8",
|
||||
),
|
||||
("HLS", HLS, "gcs", "https://storage.googleapis.com/shaka-demo-assets/angel-one-hls/hls.m3u8"),
|
||||
("DASH", DASH, "gcs", "https://storage.googleapis.com/shaka-demo-assets/angel-one/dash.mpd"),
|
||||
("DASH", DASH, "axinom", "https://media.axprod.net/TestVectors/v7-Clear/Manifest_1080p.mpd"),
|
||||
("ISM", ISM, "dtaps", "https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest"),
|
||||
(
|
||||
"ISM",
|
||||
ISM,
|
||||
"dtaps",
|
||||
"https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest",
|
||||
),
|
||||
# public 4K / HDR / Dolby-Vision tier: large high-bitrate tracks, no auth
|
||||
("DASH", DASH, "uhd-hevc", "https://dash.akamaized.net/dash264/TestCasesUHD/2b/11/MultiRate.mpd"),
|
||||
("HLS", HLS, "dv-atmos", "https://devstreaming-cdn.apple.com/videos/streaming/examples/adv_dv_atmos/main.m3u8"),
|
||||
(
|
||||
"HLS",
|
||||
HLS,
|
||||
"hevc-hls",
|
||||
"https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_adv_example_hevc/master.m3u8",
|
||||
),
|
||||
("DASH", DASH, "h265-cmaf", "https://media.axprod.net/TestVectors/H265/clear_cmaf_1080p_h265/manifest.mpd"),
|
||||
]
|
||||
|
||||
|
||||
@@ -97,18 +134,20 @@ def run_one(label: str, parser: Any, cdn: str, url: str, workers: int, which: st
|
||||
save_path = work / "out.mp4"
|
||||
counter = Counter()
|
||||
# adaptive_workers only exists on the downloader-speed branch; skip on older checkouts
|
||||
extra = {"adaptive_workers": True} if "adaptive_workers" in getattr(DownloadContext, "__dataclass_fields__", {}) else {}
|
||||
extra = (
|
||||
{"adaptive_workers": True} if "adaptive_workers" in getattr(DownloadContext, "__dataclass_fields__", {}) else {}
|
||||
)
|
||||
ctx = DownloadContext(
|
||||
save_path=save_path,
|
||||
save_dir=work / "segs",
|
||||
progress=partial(counter.__call__),
|
||||
progress=counter,
|
||||
session=session,
|
||||
proxy=None,
|
||||
max_workers=workers,
|
||||
**extra,
|
||||
)
|
||||
ctx.progress = counter # type: ignore[assignment]
|
||||
|
||||
track.drm = None # download-speed only: time raw segment download, no license/decrypt
|
||||
t0 = time.time()
|
||||
err = ""
|
||||
try:
|
||||
@@ -123,14 +162,17 @@ def run_one(label: str, parser: Any, cdn: str, url: str, workers: int, which: st
|
||||
moves, _ = read_new_transitions(log_path, line0)
|
||||
scaling = " ".join(moves) if moves else "none (finished under controller tick)"
|
||||
res = f"{getattr(track, 'width', '?')}x{getattr(track, 'height', '?')}"
|
||||
return f"{label:5} {cdn:8} {dt:6.1f}s {size/1e6:7.1f}MB {mbps:6.1f}MB/s [{res}] workers: {scaling}{err}"
|
||||
return f"{label:5} {cdn:8} {dt:6.1f}s {size / 1e6:7.1f}MB {mbps:6.1f}MB/s [{res}] workers: {scaling}{err}"
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--workers", default=16, help="max workers (the adaptive ceiling)")
|
||||
@click.option("--track", "which", type=click.Choice(["smallest", "largest"]), default="largest")
|
||||
@click.option("--only", default="", help="substring filter on CDN/format labels (e.g. 'gcs' or 'ISM')")
|
||||
@click.option("--only", default="", help="substring filter on CDN/format labels (e.g. 'apple' or 'ISM')")
|
||||
def main(workers: int, which: str, only: str) -> None:
|
||||
# download-speed only: skip HLS key handling so segments just download, no license server
|
||||
HLS.get_drm = staticmethod(lambda *a, **k: None) # type: ignore[assignment]
|
||||
|
||||
log_path = Path(mkdtemp(prefix="live_dbg_")) / "debug.jsonl"
|
||||
init_debug_logger(log_path=log_path, enabled=True)
|
||||
print(f"debug log: {log_path}\ntrack: {which}, max_workers: {workers}\n")
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Live A/B test for SegmentBase (byte-range) DASH downloads.
|
||||
|
||||
Drives the real DASH pipeline against public SegmentBase test manifests and reports
|
||||
wall time, size, throughput and output validity (ffprobe). On branches that have the
|
||||
single-URL collapse (DASH._collapsible_single_url) it also reports whether the
|
||||
collapse fired; on checkouts without it (e.g. dev) the same script runs the classic
|
||||
per-segment + merge path, so running it from both checkouts gives an A/B comparison:
|
||||
|
||||
uv run python scripts/live_segmentbase_ab.py --track largest
|
||||
/path/to/dev/.venv/bin/python scripts/live_segmentbase_ab.py --track largest
|
||||
|
||||
Throwaway-ish operational tool: it makes real network requests to public test assets.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from tempfile import mkdtemp
|
||||
from typing import Any, Optional
|
||||
|
||||
import click
|
||||
from requests import Session
|
||||
|
||||
from unshackle.core.manifests import DASH
|
||||
from unshackle.core.tracks.track import DownloadContext
|
||||
|
||||
# (label, SegmentBase MPD). netflix-1a is collapse-eligible (Initialization range from
|
||||
# byte 0); sony-1a has no Initialization range so every checkout takes the classic path.
|
||||
STREAMS: list[tuple[str, str]] = [
|
||||
("netflix-1a", "https://dash.akamaized.net/dash264/TestCases/1a/netflix/exMPD_BIP_TC1.mpd"),
|
||||
("sony-1a", "https://dash.akamaized.net/dash264/TestCases/1a/sony/SNE_DASH_SD_CASE1A_REVISED.mpd"),
|
||||
]
|
||||
|
||||
guard_results: list[bool] = []
|
||||
|
||||
|
||||
def install_guard_spy() -> bool:
|
||||
"""Record collapse decisions when this checkout has the guard; no-op otherwise."""
|
||||
orig = getattr(DASH, "_collapsible_single_url", None)
|
||||
if orig is None:
|
||||
return False
|
||||
|
||||
def spy(is_subtitle: bool, segments: Any, init_len: Optional[int]) -> bool:
|
||||
result = orig(is_subtitle, segments, init_len)
|
||||
guard_results.append(result)
|
||||
return result
|
||||
|
||||
DASH._collapsible_single_url = staticmethod(spy) # type: ignore[method-assign]
|
||||
return True
|
||||
|
||||
|
||||
def ffprobe(path: Path) -> str:
|
||||
if not path.exists() or not shutil.which("ffprobe"):
|
||||
return "n/a"
|
||||
r = subprocess.run(
|
||||
["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return f"{float(r.stdout.strip()):.0f}s" if r.stdout.strip() else "INVALID"
|
||||
|
||||
|
||||
def run_one(label: str, url: str, which: str, workers: int, has_guard: bool) -> str:
|
||||
guard_results.clear()
|
||||
session = Session()
|
||||
try:
|
||||
tracks = DASH.from_url(url=url, session=session).to_tracks(language="en")
|
||||
except Exception as e: # noqa: BLE001
|
||||
return f"{label:12} PARSE ERROR {type(e).__name__}: {e}"
|
||||
vids = list(tracks.videos)
|
||||
if not vids:
|
||||
return f"{label:12} NO VIDEO TRACKS"
|
||||
key = lambda t: t.bitrate or 0 # noqa: E731
|
||||
track = min(vids, key=key) if which == "smallest" else max(vids, key=key)
|
||||
|
||||
work = Path(mkdtemp(prefix="segbase_ab_"))
|
||||
save_path = work / "out.mp4"
|
||||
# adaptive_workers only exists on the downloader-speed branch; skip elsewhere
|
||||
extra = {"adaptive_workers": True} if "adaptive_workers" in DownloadContext.__dataclass_fields__ else {}
|
||||
ctx = DownloadContext(
|
||||
save_path=save_path,
|
||||
save_dir=work / "segs",
|
||||
progress=lambda **ev: None,
|
||||
session=session,
|
||||
proxy=None,
|
||||
max_workers=workers,
|
||||
**extra,
|
||||
)
|
||||
|
||||
t0 = time.time()
|
||||
err = ""
|
||||
try:
|
||||
DASH.download_track(track=track, ctx=ctx)
|
||||
except Exception as e: # noqa: BLE001
|
||||
err = f" ERROR {type(e).__name__}: {e}"
|
||||
dt = time.time() - t0
|
||||
size = save_path.stat().st_size if save_path.exists() else 0
|
||||
mbps = (size / 1e6 / dt) if dt > 0 and size else 0.0
|
||||
probe = ffprobe(save_path)
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
||||
if has_guard:
|
||||
collapse = "yes" if guard_results and guard_results[0] else "no"
|
||||
else:
|
||||
collapse = "unsupported"
|
||||
res = f"{getattr(track, 'width', '?')}x{getattr(track, 'height', '?')}"
|
||||
return (
|
||||
f"{label:12} {dt:6.1f}s {size / 1e6:8.1f}MB {mbps:6.1f}MB/s [{res}] "
|
||||
f"collapse: {collapse:11} duration: {probe}{err}"
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--workers", default=16, show_default=True, help="max workers (adaptive cap where supported)")
|
||||
@click.option("--track", "which", type=click.Choice(["smallest", "largest"]), default="smallest", show_default=True)
|
||||
@click.option("--only", default="", help="substring filter on stream labels")
|
||||
def main(workers: int, which: str, only: str) -> None:
|
||||
has_guard = install_guard_spy()
|
||||
print(f"track: {which}, max_workers: {workers}, collapse guard present: {has_guard}\n")
|
||||
for label, url in STREAMS:
|
||||
if only and only.lower() not in label.lower():
|
||||
continue
|
||||
print(run_one(label, url, which, workers, has_guard))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user