mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-15 05:25:12 +00:00
perf(dash): collapse single-URL byte-range segments into one direct download
Also throttle hedge-median recompute to 0.5s, cache replace_fields format patterns and short-circuit when no $ token, and stream HLS merge with copyfileobj. Adds test_dash_collapse and live A/B tooling.
This commit is contained in:
@@ -7,6 +7,9 @@ The controller logs its worker-target changes to a JSON-lines debug log; this sc
|
||||
reads them back and prints the scaling sequence per stream, so you can see whether the
|
||||
downloader ramps workers up against each CDN or finishes before the controller acts.
|
||||
|
||||
STREAMS carries a public 4K/HDR/Dolby-Vision tier (labels uhd-hevc, dv-atmos, hevc-hls,
|
||||
h265-cmaf) for exercising large high-bitrate tracks; filter it with --only uhd / dv / hevc / h265.
|
||||
|
||||
Throwaway-ish operational tool: it makes real network requests to public test assets.
|
||||
Run: uv run python scripts/live_manifest_test.py [--workers N] [--track smallest|largest]
|
||||
"""
|
||||
@@ -17,7 +20,6 @@ import json
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from tempfile import mkdtemp
|
||||
from typing import Any, Optional
|
||||
@@ -33,16 +35,51 @@ from unshackle.core.utilities import close_debug_logger, init_debug_logger
|
||||
# all three formats on Unified Streaming, so the formats are comparable on one CDN.
|
||||
STREAMS: list[tuple[str, Any, str, str]] = [
|
||||
("HLS", HLS, "mux", "https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8"),
|
||||
("HLS", HLS, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8"),
|
||||
(
|
||||
"HLS",
|
||||
HLS,
|
||||
"unified",
|
||||
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8",
|
||||
),
|
||||
("DASH", DASH, "akamai", "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps.mpd"),
|
||||
("DASH", DASH, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.mpd"),
|
||||
(
|
||||
"DASH",
|
||||
DASH,
|
||||
"unified",
|
||||
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.mpd",
|
||||
),
|
||||
("DASH", DASH, "dashif", "https://livesim2.dashif.org/vod/testpic_2s/Manifest.mpd"),
|
||||
("ISM", ISM, "unified", "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/Manifest"),
|
||||
("HLS", HLS, "apple", "https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8"),
|
||||
(
|
||||
"ISM",
|
||||
ISM,
|
||||
"unified",
|
||||
"https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/Manifest",
|
||||
),
|
||||
(
|
||||
"HLS",
|
||||
HLS,
|
||||
"apple",
|
||||
"https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8",
|
||||
),
|
||||
("HLS", HLS, "gcs", "https://storage.googleapis.com/shaka-demo-assets/angel-one-hls/hls.m3u8"),
|
||||
("DASH", DASH, "gcs", "https://storage.googleapis.com/shaka-demo-assets/angel-one/dash.mpd"),
|
||||
("DASH", DASH, "axinom", "https://media.axprod.net/TestVectors/v7-Clear/Manifest_1080p.mpd"),
|
||||
("ISM", ISM, "dtaps", "https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest"),
|
||||
(
|
||||
"ISM",
|
||||
ISM,
|
||||
"dtaps",
|
||||
"https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest",
|
||||
),
|
||||
# public 4K / HDR / Dolby-Vision tier: large high-bitrate tracks, no auth
|
||||
("DASH", DASH, "uhd-hevc", "https://dash.akamaized.net/dash264/TestCasesUHD/2b/11/MultiRate.mpd"),
|
||||
("HLS", HLS, "dv-atmos", "https://devstreaming-cdn.apple.com/videos/streaming/examples/adv_dv_atmos/main.m3u8"),
|
||||
(
|
||||
"HLS",
|
||||
HLS,
|
||||
"hevc-hls",
|
||||
"https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_adv_example_hevc/master.m3u8",
|
||||
),
|
||||
("DASH", DASH, "h265-cmaf", "https://media.axprod.net/TestVectors/H265/clear_cmaf_1080p_h265/manifest.mpd"),
|
||||
]
|
||||
|
||||
|
||||
@@ -97,18 +134,20 @@ def run_one(label: str, parser: Any, cdn: str, url: str, workers: int, which: st
|
||||
save_path = work / "out.mp4"
|
||||
counter = Counter()
|
||||
# adaptive_workers only exists on the downloader-speed branch; skip on older checkouts
|
||||
extra = {"adaptive_workers": True} if "adaptive_workers" in getattr(DownloadContext, "__dataclass_fields__", {}) else {}
|
||||
extra = (
|
||||
{"adaptive_workers": True} if "adaptive_workers" in getattr(DownloadContext, "__dataclass_fields__", {}) else {}
|
||||
)
|
||||
ctx = DownloadContext(
|
||||
save_path=save_path,
|
||||
save_dir=work / "segs",
|
||||
progress=partial(counter.__call__),
|
||||
progress=counter,
|
||||
session=session,
|
||||
proxy=None,
|
||||
max_workers=workers,
|
||||
**extra,
|
||||
)
|
||||
ctx.progress = counter # type: ignore[assignment]
|
||||
|
||||
track.drm = None # download-speed only: time raw segment download, no license/decrypt
|
||||
t0 = time.time()
|
||||
err = ""
|
||||
try:
|
||||
@@ -123,14 +162,17 @@ def run_one(label: str, parser: Any, cdn: str, url: str, workers: int, which: st
|
||||
moves, _ = read_new_transitions(log_path, line0)
|
||||
scaling = " ".join(moves) if moves else "none (finished under controller tick)"
|
||||
res = f"{getattr(track, 'width', '?')}x{getattr(track, 'height', '?')}"
|
||||
return f"{label:5} {cdn:8} {dt:6.1f}s {size/1e6:7.1f}MB {mbps:6.1f}MB/s [{res}] workers: {scaling}{err}"
|
||||
return f"{label:5} {cdn:8} {dt:6.1f}s {size / 1e6:7.1f}MB {mbps:6.1f}MB/s [{res}] workers: {scaling}{err}"
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--workers", default=16, help="max workers (the adaptive ceiling)")
|
||||
@click.option("--track", "which", type=click.Choice(["smallest", "largest"]), default="largest")
|
||||
@click.option("--only", default="", help="substring filter on CDN/format labels (e.g. 'gcs' or 'ISM')")
|
||||
@click.option("--only", default="", help="substring filter on CDN/format labels (e.g. 'apple' or 'ISM')")
|
||||
def main(workers: int, which: str, only: str) -> None:
|
||||
# download-speed only: skip HLS key handling so segments just download, no license server
|
||||
HLS.get_drm = staticmethod(lambda *a, **k: None) # type: ignore[assignment]
|
||||
|
||||
log_path = Path(mkdtemp(prefix="live_dbg_")) / "debug.jsonl"
|
||||
init_debug_logger(log_path=log_path, enabled=True)
|
||||
print(f"debug log: {log_path}\ntrack: {which}, max_workers: {workers}\n")
|
||||
|
||||
Reference in New Issue
Block a user