perf(dash): collapse single-file tracks that have a sidx gap

scripts/live_segmentbase_ab.py

@@ -34,7 +34,7 @@ STREAMS: list[tuple[str, str]] = [
     ("sony-1a", "https://dash.akamaized.net/dash264/TestCases/1a/sony/SNE_DASH_SD_CASE1A_REVISED.mpd"),
 ]
 
-guard_results: list[bool] = []
+guard_results: list[Optional[tuple[int, int]]] = []
 
 
 def install_guard_spy() -> bool:
@@ -43,7 +43,7 @@ def install_guard_spy() -> bool:
     if orig is None:
         return False
 
-    def spy(is_subtitle: bool, segments: Any, init_len: Optional[int]) -> bool:
+    def spy(is_subtitle: bool, segments: Any, init_len: int) -> Optional[tuple[int, int]]:
         result = orig(is_subtitle, segments, init_len)
         guard_results.append(result)
         return result

tests/core/test_dash_collapse.py

@@ -4,15 +4,13 @@ resource instead of per-segment files + merge."""
 
 from __future__ import annotations
 
-from typing import Optional
-
 from unshackle.core.manifests import DASH
 
 URL = "https://cdn/media.mp4"
 
 
-def collapsible(segments, init_len: Optional[int], is_subtitle: bool = False) -> bool:
-    return DASH.collapsible_single_url(is_subtitle, segments, init_len)
+def collapsible(segments, init_len: int, is_subtitle: bool = False) -> bool:
+    return DASH.collapsible_single_url(is_subtitle, segments, init_len) is not None
 
 
 def test_segmentbase_single_range_eligible():
@@ -29,10 +27,6 @@ def test_subtitle_never_eligible():
     assert collapsible([(URL, "863-999999")], init_len=863, is_subtitle=True) is False
 
 
-def test_missing_init_len_ineligible():
-    assert collapsible([(URL, "863-999999")], init_len=None) is False
-
-
 def test_missing_range_ineligible():
     assert collapsible([(URL, None)], init_len=863) is False
     assert collapsible([(URL, "863-1999"), (URL, None)], init_len=863) is False
@@ -43,9 +37,14 @@ def test_mixed_urls_ineligible():
     assert collapsible(segments, init_len=863) is False
 
 
-def test_first_segment_gap_after_init_ineligible():
-    # first media byte does not immediately follow the init prefix
-    assert collapsible([(URL, "900-1999")], init_len=863) is False
+def test_first_segment_gap_after_init_left_to_prefix_check():
+    # the gap may be the sidx; single_url_prefix_matches decides from the bytes
+    segments = [(URL, "900-1999"), (URL, "2000-2999")]
+    assert DASH.collapsible_single_url(False, segments, 863) == (900, 2999)
+
+
+def test_first_segment_inside_init_ineligible():
+    assert collapsible([(URL, "800-1999")], init_len=863) is False
 
 
 def test_gap_between_segments_ineligible():

tests/core/test_dash_collapse_integration.py

@@ -18,6 +18,7 @@ localhost server, asserting that:
 from __future__ import annotations
 
 import shutil
+import struct
 import threading
 from functools import partial
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -99,48 +100,6 @@ def base(srv) -> str:
     return f"http://127.0.0.1:{srv.server_address[1]}/"
 
 
-def collapse_mpd(srv) -> str:
-    # SegmentList on one media.mp4: init is bytes [0, INIT_LEN), media ranges cover the rest
-    # contiguously -> collapsible_single_url returns the URL and download_track collapses.
-    return f"""<?xml version="1.0"?>
-<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
-  <BaseURL>{base(srv)}</BaseURL>
-  <Period id="0">
-    <AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
-      <Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
-        <BaseURL>media.mp4</BaseURL>
-        <SegmentList timescale="1">
-          <Initialization range="0-{INIT_LEN - 1}"/>
-          <SegmentURL mediaRange="{INIT_LEN}-5119"/>
-          <SegmentURL mediaRange="5120-{len(PARENT) - 1}"/>
-        </SegmentList>
-      </Representation>
-    </AdaptationSet>
-  </Period>
-</MPD>"""
-
-
-def clipped_mpd(srv) -> str:
-    # same parent, but the ranges stop short of its end -> the collapsed file must be trimmed
-    # to the last indexed byte, not left holding the parent's trailing bytes.
-    return f"""<?xml version="1.0"?>
-<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
-  <BaseURL>{base(srv)}</BaseURL>
-  <Period id="0">
-    <AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
-      <Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
-        <BaseURL>media.mp4</BaseURL>
-        <SegmentList timescale="1">
-          <Initialization range="0-{INIT_LEN - 1}"/>
-          <SegmentURL mediaRange="{INIT_LEN}-5119"/>
-          <SegmentURL mediaRange="5120-{CLIP_END}"/>
-        </SegmentList>
-      </Representation>
-    </AdaptationSet>
-  </Period>
-</MPD>"""
-
-
 def mixed_mpd(srv) -> str:
     # Two SegmentURLs on distinct paths -> mixed URLs -> predicate False -> segmented path.
     return f"""<?xml version="1.0"?>
@@ -160,6 +119,33 @@ def mixed_mpd(srv) -> str:
 </MPD>"""
 
 
+def sidx_box(sizes: list[int]) -> bytes:
+    # version 0 sidx: one MEDIA reference per size, each starting with a SAP
+    refs = b"".join(struct.pack(">III", size, 1, 0x90000000) for size in sizes)
+    body = struct.pack(">IIIIIHH", 0, 1, 1, 0, 0, 0, len(sizes)) + refs
+    return struct.pack(">I", 8 + len(body)) + b"sidx" + body
+
+
+def ranged_mpd(srv, ranges: list[tuple[int, int]], init: str = f'range="0-{INIT_LEN - 1}"') -> str:
+    # SegmentList of byte ranges on one media.mp4, with the given Initialization attributes
+    segment_urls = "\n".join(f'          <SegmentURL mediaRange="{s}-{e}"/>' for s, e in ranges)
+    return f"""<?xml version="1.0"?>
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">
+  <BaseURL>{base(srv)}</BaseURL>
+  <Period id="0">
+    <AdaptationSet contentType="video" mimeType="video/mp4" lang="en">
+      <Representation id="v0" codecs="avc1.640028" bandwidth="1000000" width="1920" height="1080">
+        <BaseURL>media.mp4</BaseURL>
+        <SegmentList timescale="1">
+          <Initialization {init}/>
+{segment_urls}
+        </SegmentList>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>"""
+
+
 def make_track(mpd: str, srv):
     return DASH.from_text(mpd, url=f"{base(srv)}manifest.mpd").to_tracks(language="en").videos[0]
 
@@ -174,7 +160,7 @@ def make_ctx(tmp_path, name="out.mp4"):
 
 def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path):
     server.routes["/media.mp4"] = PARENT
-    track = make_track(collapse_mpd(server), server)
+    track = make_track(ranged_mpd(server, [(INIT_LEN, 5119), (5120, len(PARENT) - 1)]), server)
     ctx = make_ctx(tmp_path, "collapsed.mp4")
 
     DASH.download_track(track, ctx)
@@ -194,7 +180,8 @@ def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path)
 
 def test_collapse_trims_parent_bytes_past_the_last_segment(server, tmp_path):
     server.routes["/media.mp4"] = PARENT
-    track = make_track(clipped_mpd(server), server)
+    # the ranges stop short of the parent's end, so the collapsed file must be trimmed
+    track = make_track(ranged_mpd(server, [(INIT_LEN, 5119), (5120, CLIP_END)]), server)
     ctx = make_ctx(tmp_path, "clipped.mp4")
 
     DASH.download_track(track, ctx)
@@ -203,6 +190,60 @@ def test_collapse_trims_parent_bytes_past_the_last_segment(server, tmp_path):
     assert ctx.save_path.read_bytes() == PARENT[: CLIP_END + 1]
 
 
+def test_collapse_keeps_the_sidx_between_init_and_first_segment(server, tmp_path):
+    media = [bytes([0xC0]) * 3000, bytes([0xD1]) * 2000]
+    head = PARENT[:INIT_LEN] + sidx_box([len(m) for m in media])
+    parent = head + b"".join(media)
+    server.routes["/media.mp4"] = parent
+    ranges = [(len(head), len(head) + 2999), (len(head) + 3000, len(parent) - 1)]
+    ctx = make_ctx(tmp_path, "sidx.mp4")
+
+    DASH.download_track(make_track(ranged_mpd(server, ranges), server), ctx)
+
+    assert ctx.save_path.read_bytes() == parent
+    assert [path for path, rng in server.requests if rng is None] == ["/media.mp4"]
+
+
+def test_leading_gap_that_is_not_a_sidx_takes_segmented_path(server, tmp_path):
+    # a filtered period cut from the same file leaves a gap of media, not an index
+    gap = struct.pack(">I", 1000) + b"moof" + bytes(992)
+    media = bytes([0xC0]) * 3000
+    server.routes["/media.mp4"] = PARENT[:INIT_LEN] + gap + media
+    start = INIT_LEN + len(gap)
+    ctx = make_ctx(tmp_path, "gap.mp4")
+
+    DASH.download_track(make_track(ranged_mpd(server, [(start, start + 2999)]), server), ctx)
+
+    assert ctx.save_path.read_bytes() == PARENT[:INIT_LEN] + media
+    assert not [path for path, rng in server.requests if rng is None]
+
+
+def test_sidx_indexing_more_than_the_listed_segments_takes_segmented_path(server, tmp_path):
+    # the file carries a post-roll after the listed segments, and the sidx indexes it too
+    media, post_roll = bytes([0xC0]) * 3000, bytes([0xEE]) * 500
+    head = PARENT[:INIT_LEN] + sidx_box([len(media), len(post_roll)])
+    server.routes["/media.mp4"] = head + media + post_roll
+    ctx = make_ctx(tmp_path, "post-roll.mp4")
+
+    DASH.download_track(make_track(ranged_mpd(server, [(len(head), len(head) + 2999)]), server), ctx)
+
+    assert ctx.save_path.read_bytes() == PARENT[:INIT_LEN] + media
+
+
+def test_init_from_another_file_takes_segmented_path(server, tmp_path):
+    # the media ranges start where an init of the same length would end, but the init is elsewhere
+    init = bytes([0x11]) * INIT_LEN
+    server.routes["/init.bin"] = init
+    server.routes["/media.mp4"] = PARENT
+    ranges = [(INIT_LEN, 5119), (5120, len(PARENT) - 1)]
+    ctx = make_ctx(tmp_path, "init-elsewhere.mp4")
+
+    DASH.download_track(make_track(ranged_mpd(server, ranges, init='sourceURL="init.bin"'), server), ctx)
+
+    assert ctx.save_path.read_bytes() == init + PARENT[INIT_LEN:]
+    assert not [path for path, rng in server.requests if rng is None and path == "/media.mp4"]
+
+
 def test_non_collapse_takes_segmented_merge_path(server, tmp_path):
     server.routes["/init.bin"] = INIT2
     server.routes["/part0.bin"] = PART0

unshackle/core/manifests/dash.py

@@ -40,6 +40,7 @@ from unshackle.core.tracks.track import (
 )
 from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8
 from unshackle.core.utils import run_proxy
+from unshackle.core.utils.bitrate import ranged_get
 from unshackle.core.utils.redact import safe_display_url
 from unshackle.core.utils.xml import load_xml
 
@@ -558,14 +559,17 @@ class DASH:
 
         downloader = track.downloader
 
-        # When every segment is a byte range of one parent resource and init is its [0, len)
-        # prefix, the whole resource is itself a valid MP4, so one direct download to save_path
-        # replaces the per-segment files, init write and merge pass.
-        collapse_single_url = not config.sample and DASH.collapsible_single_url(
-            isinstance(track, Subtitle),
-            segments,
-            len(init_data) if init_data is not None else None,
+        # A parent resource that is itself a valid MP4 downloads in one pass to save_path,
+        # replacing the per-segment files, init write and merge pass.
+        collapse_span = (
+            DASH.collapsible_single_url(isinstance(track, Subtitle), segments, len(init_data))
+            if init_data is not None and not config.sample
+            else None
         )
+        if collapse_span and init_data:
+            if not DASH.single_url_prefix_matches(session, segments[0][0], collapse_span, init_data):
+                collapse_span = None
+        collapse_single_url = collapse_span is not None
 
         use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption)
         # subtitle segments are tiny, the post-download merge costs nothing there
@@ -674,9 +678,9 @@ class DASH:
                 save_path.unlink(missing_ok=True)
             raise
 
-        if collapse_single_url:
+        if collapse_span:
             with open(save_path, "r+b") as collapsed:
-                collapsed.truncate(int(segments[-1][1].split("-")[1]) + 1)
+                collapsed.truncate(collapse_span[1] + 1)
         elif merger:
             for control_file in save_dir.glob("*.!dev"):
                 control_file.unlink(missing_ok=True)
@@ -805,42 +809,67 @@ class DASH:
     def collapsible_single_url(
         is_subtitle: bool,
         segments: list[tuple[str, Optional[str]]],
-        init_len: Optional[int],
-    ) -> bool:
+        init_len: int,
+    ) -> Optional[tuple[int, int]]:
         """
-        Whether every segment is a byte range of one parent resource whose [0, init_len)
-        prefix is the init segment, so unshackle can download the whole resource in one pass.
+        Get the first and last byte of the segments when every segment is a byte range of one
+        parent resource that follows an init of init_len bytes, so unshackle can download the
+        whole resource in one pass.
 
-        Returns False for subtitle tracks, an unknown init length, mixed URLs, malformed
-        ranges, and anything but one gapless run in document order starting at the init
-        prefix. A gap would pull bytes the segmented merge path excludes (a period the
-        manifest filters out) into the collapsed file, so it fails closed.
+        Returns None for subtitle tracks, mixed URLs, malformed ranges, and anything but one
+        gapless run in document order. A gap between segments would pull bytes the segmented
+        merge path excludes (a period the manifest filters out) into the collapsed file, so it
+        fails closed. A gap before the first segment passes here, and single_url_prefix_matches
+        must then prove that it is the index.
         """
-        if is_subtitle or init_len is None or not segments:
-            return False
+        if is_subtitle or not segments:
+            return None
 
         first_url = segments[0][0]
         ranges: list[tuple[int, int]] = []
         for url, bytes_range in segments:
             if url != first_url or not bytes_range:
-                return False
+                return None
             parts = bytes_range.split("-")
             if len(parts) != 2 or not parts[0] or not parts[1]:
-                return False
+                return None
             try:
                 start, end = int(parts[0]), int(parts[1])
             except ValueError:
-                return False
+                return None
             if end < start:
-                return False
+                return None
             ranges.append((start, end))
 
-        prev_end = init_len - 1
+        if ranges[0][0] < init_len:
+            return None
+        prev_end = ranges[0][0] - 1
         for start, end in ranges:
             if start != prev_end + 1:
-                return False
+                return None
             prev_end = end
-        return True
+        return ranges[0][0], prev_end
+
+    @staticmethod
+    def single_url_prefix_matches(
+        session: Union[Session, RnetSession],
+        url: str,
+        span: tuple[int, int],
+        init_data: bytes,
+    ) -> bool:
+        """
+        Whether the resource at url starts with init_data and, when bytes sit between the init
+        and the first byte of span, whether they are one sidx that indexes exactly span.
+        Fetches the resource up to the start of span.
+        """
+        first_start, last_end = span
+        head = ranged_get(url, session, f"bytes=0-{first_start - 1}")
+        if not head or len(head) != first_start or not head.startswith(init_data):
+            return False
+        if first_start == len(init_data):
+            return True
+        _, subsegments = parse_sidx(head[len(init_data) :], len(init_data))
+        return bool(subsegments) and subsegments[0][1] == first_start and subsegments[-1][2] == last_end
 
     @staticmethod
     def is_content_period(period: Element, filtered_period_ids: list[str]) -> bool: