perf(dash): speed up segment collection and URL templating

This commit is contained in:
imSp4rky
2026-07-18 23:48:10 -06:00
parent b54aadea7a
commit 100857d9bb
3 changed files with 51 additions and 5 deletions
+35
View File
@@ -0,0 +1,35 @@
"""Regression tests for DASH.replace_fields template substitution.
Covers the fast-path short-circuits (no ``$``, no ``%``) and the printf-style
``$Field%fmt$`` padding, so plain and padded templates both resolve correctly.
"""
from unshackle.core.manifests.dash import DASH
def test_plain_fields_substituted() -> None:
url = "seg-$RepresentationID$-$Number$-$Time$.mp4"
out = DASH.replace_fields(url, RepresentationID="video=6M", Number=5, Time=20000)
assert out == "seg-video=6M-5-20000.mp4"
def test_no_placeholder_returns_unchanged() -> None:
url = "https://cdn.example.com/init.mp4"
assert DASH.replace_fields(url, Number=1, Time=0) is url
def test_printf_padding_applied() -> None:
# the '%05d' padding path must still fire when a '%' spec is present
url = "seg-$Number%05d$.mp4"
assert DASH.replace_fields(url, Number=7) == "seg-00007.mp4"
def test_printf_hex_padding_applied() -> None:
url = "seg-$Number%08x$.m4s"
assert DASH.replace_fields(url, Number=255) == "seg-000000ff.m4s"
def test_mixed_plain_and_padded_fields() -> None:
url = "$RepresentationID$/seg-$Number%04d$.mp4"
out = DASH.replace_fields(url, RepresentationID="v1", Number=42)
assert out == "v1/seg-0042.mp4"
+6 -1
View File
@@ -16,6 +16,7 @@ from email.utils import parsedate_to_datetime
from http.cookiejar import CookieJar from http.cookiejar import CookieJar
from pathlib import Path from pathlib import Path
from queue import Empty, Queue from queue import Empty, Queue
from string import Formatter
from typing import Any, Callable, Generator, MutableMapping, Optional, Union, cast from typing import Any, Callable, Generator, MutableMapping, Optional, Union, cast
from requests import Session from requests import Session
@@ -1118,13 +1119,17 @@ def requests(
context={"session_type": type(session).__name__, "url_count": len(urls)}, context={"session_type": type(session).__name__, "url_count": len(urls)},
) )
# only resolve each URL's extension when the template actually references {ext};
# DASH/ISM use "{i}.mp4" (no ext), so this skips a urlparse per segment for them
needs_ext = any(name == "ext" for _, name, _, _ in Formatter().parse(filename))
urls = [ urls = [
dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path) dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path)
for i, url in enumerate(urls) for i, url in enumerate(urls)
for save_path in [ for save_path in [
output_dir output_dir
/ filename.format( / filename.format(
i=i * index_stride + index_offset, ext=get_extension(url["url"] if isinstance(url, dict) else url) i=i * index_stride + index_offset,
ext=get_extension(url["url"] if isinstance(url, dict) else url) if needs_ext else "",
) )
] ]
] ]
+10 -4
View File
@@ -339,6 +339,7 @@ class DASH:
# Collect segments from all content periods in the manifest # Collect segments from all content periods in the manifest
all_periods = manifest.findall("Period") all_periods = manifest.findall("Period")
segments: list[tuple[str, Optional[str]]] = [] segments: list[tuple[str, Optional[str]]] = []
seen_segments: set[tuple[str, Optional[str]]] = set() # dedupe segments across periods in O(1)
segment_durations: list[int] = [] segment_durations: list[int] = []
segment_timescale: float = 0 segment_timescale: float = 0
init_data: Optional[bytes] = None init_data: Optional[bytes] = None
@@ -390,7 +391,8 @@ class DASH:
log.debug(f"Period {content_period.get('id', period_idx)} has different KID: {p_kid}") log.debug(f"Period {content_period.get('id', period_idx)} has different KID: {p_kid}")
for seg in p_segments: for seg in p_segments:
if seg not in segments: if seg not in seen_segments:
seen_segments.add(seg)
segments.append(seg) segments.append(seg)
segment_durations.extend(p_durations) segment_durations.extend(p_durations)
@@ -1187,11 +1189,15 @@ class DASH:
def replace_fields(url: str, **kwargs: Any) -> str: def replace_fields(url: str, **kwargs: Any) -> str:
if "$" not in url: if "$" not in url:
return url return url
# printf-style `$Field%fmt$` padding always contains a literal '%'; skip the
# per-field regex entirely for the common case (plain `$Field$` templates)
has_fmt = "%" in url
for field, value in kwargs.items(): for field, value in kwargs.items():
url = url.replace(f"${field}$", str(value)) url = url.replace(f"${field}$", str(value))
m = DASH._field_format_pattern(field).search(url) if has_fmt:
if m: m = DASH._field_format_pattern(field).search(url)
url = url.replace(m.group(), f"{value:{m.group(1)}}") if m:
url = url.replace(m.group(), f"{value:{m.group(1)}}")
return url return url