perf(dash): speed up segment collection and URL templating

tests/core/test_dash_replace_fields.py

@@ -0,0 +1,35 @@
+"""Regression tests for DASH.replace_fields template substitution.
+
+Covers the fast-path short-circuits (no ``$``, no ``%``) and the printf-style
+``$Field%fmt$`` padding, so plain and padded templates both resolve correctly.
+"""
+
+from unshackle.core.manifests.dash import DASH
+
+
+def test_plain_fields_substituted() -> None:
+    url = "seg-$RepresentationID$-$Number$-$Time$.mp4"
+    out = DASH.replace_fields(url, RepresentationID="video=6M", Number=5, Time=20000)
+    assert out == "seg-video=6M-5-20000.mp4"
+
+
+def test_no_placeholder_returns_unchanged() -> None:
+    url = "https://cdn.example.com/init.mp4"
+    assert DASH.replace_fields(url, Number=1, Time=0) is url
+
+
+def test_printf_padding_applied() -> None:
+    # the '%05d' padding path must still fire when a '%' spec is present
+    url = "seg-$Number%05d$.mp4"
+    assert DASH.replace_fields(url, Number=7) == "seg-00007.mp4"
+
+
+def test_printf_hex_padding_applied() -> None:
+    url = "seg-$Number%08x$.m4s"
+    assert DASH.replace_fields(url, Number=255) == "seg-000000ff.m4s"
+
+
+def test_mixed_plain_and_padded_fields() -> None:
+    url = "$RepresentationID$/seg-$Number%04d$.mp4"
+    out = DASH.replace_fields(url, RepresentationID="v1", Number=42)
+    assert out == "v1/seg-0042.mp4"

unshackle/core/downloaders/requests.py

@@ -16,6 +16,7 @@ from email.utils import parsedate_to_datetime
 from http.cookiejar import CookieJar
 from pathlib import Path
 from queue import Empty, Queue
+from string import Formatter
 from typing import Any, Callable, Generator, MutableMapping, Optional, Union, cast
 
 from requests import Session
@@ -1118,13 +1119,17 @@ def requests(
                 context={"session_type": type(session).__name__, "url_count": len(urls)},
             )
 
+    # only resolve each URL's extension when the template actually references {ext};
+    # DASH/ISM use "{i}.mp4" (no ext), so this skips a urlparse per segment for them
+    needs_ext = any(name == "ext" for _, name, _, _ in Formatter().parse(filename))
     urls = [
         dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path)
         for i, url in enumerate(urls)
         for save_path in [
             output_dir
             / filename.format(
-                i=i * index_stride + index_offset, ext=get_extension(url["url"] if isinstance(url, dict) else url)
+                i=i * index_stride + index_offset,
+                ext=get_extension(url["url"] if isinstance(url, dict) else url) if needs_ext else "",
             )
         ]
     ]

unshackle/core/manifests/dash.py

@@ -339,6 +339,7 @@ class DASH:
         # Collect segments from all content periods in the manifest
         all_periods = manifest.findall("Period")
         segments: list[tuple[str, Optional[str]]] = []
+        seen_segments: set[tuple[str, Optional[str]]] = set()  # dedupe segments across periods in O(1)
         segment_durations: list[int] = []
         segment_timescale: float = 0
         init_data: Optional[bytes] = None
@@ -390,7 +391,8 @@ class DASH:
                     log.debug(f"Period {content_period.get('id', period_idx)} has different KID: {p_kid}")
 
             for seg in p_segments:
-                if seg not in segments:
+                if seg not in seen_segments:
+                    seen_segments.add(seg)
                     segments.append(seg)
             segment_durations.extend(p_durations)
 
@@ -1187,11 +1189,15 @@ class DASH:
     def replace_fields(url: str, **kwargs: Any) -> str:
         if "$" not in url:
             return url
+        # printf-style `$Field%fmt$` padding always contains a literal '%'; skip the
+        # per-field regex entirely for the common case (plain `$Field$` templates)
+        has_fmt = "%" in url
         for field, value in kwargs.items():
             url = url.replace(f"${field}$", str(value))
-            m = DASH._field_format_pattern(field).search(url)
-            if m:
-                url = url.replace(m.group(), f"{value:{m.group(1)}}")
+            if has_fmt:
+                m = DASH._field_format_pattern(field).search(url)
+                if m:
+                    url = url.replace(m.group(), f"{value:{m.group(1)}}")
         return url