perf(dash): collapse single-URL byte-range segments into one direct download

Also throttle hedge-median recompute to 0.5s, cache replace_fields format
patterns and short-circuit when no $ token, and stream HLS merge with
copyfileobj. Adds test_dash_collapse and live A/B tooling.
This commit is contained in:
imSp4rky
2026-07-16 12:42:16 -06:00
parent 78d80eb68a
commit 7eee34014e
6 changed files with 428 additions and 97 deletions
+53 -11
View File
@@ -7,6 +7,9 @@ The controller logs its worker-target changes to a JSON-lines debug log; this sc
reads them back and prints the scaling sequence per stream, so you can see whether the
downloader ramps workers up against each CDN or finishes before the controller acts.
STREAMS carries a public 4K/HDR/Dolby-Vision tier (labels uhd-hevc, dv-atmos, hevc-hls,
h265-cmaf) for exercising large high-bitrate tracks; filter it with --only uhd / dv / hevc / h265.
Throwaway-ish operational tool: it makes real network requests to public test assets.
Run: uv run python scripts/live_manifest_test.py [--workers N] [--track smallest|largest]
"""
@@ -17,7 +20,6 @@ import json
import shutil
import sys
import time
from functools import partial
from pathlib import Path
from tempfile import mkdtemp
from typing import Any, Optional
@@ -33,16 +35,51 @@ from unshackle.core.utilities import close_debug_logger, init_debug_logger
# all three formats on Unified Streaming, so the formats are comparable on one CDN.
STREAMS: list[tuple[str, Any, str, str]] = [
("HLS", HLS, "mux", "https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8"),
("HLS", HLS, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8"),
(
"HLS",
HLS,
"unified",
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8",
),
("DASH", DASH, "akamai", "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps.mpd"),
("DASH", DASH, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.mpd"),
(
"DASH",
DASH,
"unified",
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.mpd",
),
("DASH", DASH, "dashif", "https://livesim2.dashif.org/vod/testpic_2s/Manifest.mpd"),
("ISM", ISM, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/Manifest"),
("HLS", HLS, "apple", "https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8"),
(
"ISM",
ISM,
"unified",
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/Manifest",
),
(
"HLS",
HLS,
"apple",
"https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8",
),
("HLS", HLS, "gcs", "https://storage.googleapis.com/shaka-demo-assets/angel-one-hls/hls.m3u8"),
("DASH", DASH, "gcs", "https://storage.googleapis.com/shaka-demo-assets/angel-one/dash.mpd"),
("DASH", DASH, "axinom", "https://media.axprod.net/TestVectors/v7-Clear/Manifest_1080p.mpd"),
("ISM", ISM, "dtaps", "https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest"),
(
"ISM",
ISM,
"dtaps",
"https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest",
),
# public 4K / HDR / Dolby-Vision tier: large high-bitrate tracks, no auth
("DASH", DASH, "uhd-hevc", "https://dash.akamaized.net/dash264/TestCasesUHD/2b/11/MultiRate.mpd"),
("HLS", HLS, "dv-atmos", "https://devstreaming-cdn.apple.com/videos/streaming/examples/adv_dv_atmos/main.m3u8"),
(
"HLS",
HLS,
"hevc-hls",
"https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_adv_example_hevc/master.m3u8",
),
("DASH", DASH, "h265-cmaf", "https://media.axprod.net/TestVectors/H265/clear_cmaf_1080p_h265/manifest.mpd"),
]
@@ -97,18 +134,20 @@ def run_one(label: str, parser: Any, cdn: str, url: str, workers: int, which: st
save_path = work / "out.mp4"
counter = Counter()
# adaptive_workers only exists on the downloader-speed branch; skip on older checkouts
extra = {"adaptive_workers": True} if "adaptive_workers" in getattr(DownloadContext, "__dataclass_fields__", {}) else {}
extra = (
{"adaptive_workers": True} if "adaptive_workers" in getattr(DownloadContext, "__dataclass_fields__", {}) else {}
)
ctx = DownloadContext(
save_path=save_path,
save_dir=work / "segs",
progress=partial(counter.__call__),
progress=counter,
session=session,
proxy=None,
max_workers=workers,
**extra,
)
ctx.progress = counter # type: ignore[assignment]
track.drm = None # download-speed only: time raw segment download, no license/decrypt
t0 = time.time()
err = ""
try:
@@ -123,14 +162,17 @@ def run_one(label: str, parser: Any, cdn: str, url: str, workers: int, which: st
moves, _ = read_new_transitions(log_path, line0)
scaling = " ".join(moves) if moves else "none (finished under controller tick)"
res = f"{getattr(track, 'width', '?')}x{getattr(track, 'height', '?')}"
return f"{label:5} {cdn:8} {dt:6.1f}s {size/1e6:7.1f}MB {mbps:6.1f}MB/s [{res}] workers: {scaling}{err}"
return f"{label:5} {cdn:8} {dt:6.1f}s {size / 1e6:7.1f}MB {mbps:6.1f}MB/s [{res}] workers: {scaling}{err}"
@click.command()
@click.option("--workers", default=16, help="max workers (the adaptive ceiling)")
@click.option("--track", "which", type=click.Choice(["smallest", "largest"]), default="largest")
@click.option("--only", default="", help="substring filter on CDN/format labels (e.g. 'gcs' or 'ISM')")
@click.option("--only", default="", help="substring filter on CDN/format labels (e.g. 'apple' or 'ISM')")
def main(workers: int, which: str, only: str) -> None:
# download-speed only: skip HLS key handling so segments just download, no license server
HLS.get_drm = staticmethod(lambda *a, **k: None) # type: ignore[assignment]
log_path = Path(mkdtemp(prefix="live_dbg_")) / "debug.jsonl"
init_debug_logger(log_path=log_path, enabled=True)
print(f"debug log: {log_path}\ntrack: {which}, max_workers: {workers}\n")