diff --git a/scripts/live_segmentbase_ab.py b/scripts/live_segmentbase_ab.py index e38d284b..04082fc6 100644 --- a/scripts/live_segmentbase_ab.py +++ b/scripts/live_segmentbase_ab.py @@ -34,7 +34,7 @@ STREAMS: list[tuple[str, str]] = [ ("sony-1a", "https://dash.akamaized.net/dash264/TestCases/1a/sony/SNE_DASH_SD_CASE1A_REVISED.mpd"), ] -guard_results: list[bool] = [] +guard_results: list[Optional[tuple[int, int]]] = [] def install_guard_spy() -> bool: @@ -43,7 +43,7 @@ def install_guard_spy() -> bool: if orig is None: return False - def spy(is_subtitle: bool, segments: Any, init_len: Optional[int]) -> bool: + def spy(is_subtitle: bool, segments: Any, init_len: int) -> Optional[tuple[int, int]]: result = orig(is_subtitle, segments, init_len) guard_results.append(result) return result diff --git a/tests/core/test_dash_collapse.py b/tests/core/test_dash_collapse.py index 0305778b..40870387 100644 --- a/tests/core/test_dash_collapse.py +++ b/tests/core/test_dash_collapse.py @@ -4,15 +4,13 @@ resource instead of per-segment files + merge.""" from __future__ import annotations -from typing import Optional - from unshackle.core.manifests import DASH URL = "https://cdn/media.mp4" -def collapsible(segments, init_len: Optional[int], is_subtitle: bool = False) -> bool: - return DASH.collapsible_single_url(is_subtitle, segments, init_len) +def collapsible(segments, init_len: int, is_subtitle: bool = False) -> bool: + return DASH.collapsible_single_url(is_subtitle, segments, init_len) is not None def test_segmentbase_single_range_eligible(): @@ -29,10 +27,6 @@ def test_subtitle_never_eligible(): assert collapsible([(URL, "863-999999")], init_len=863, is_subtitle=True) is False -def test_missing_init_len_ineligible(): - assert collapsible([(URL, "863-999999")], init_len=None) is False - - def test_missing_range_ineligible(): assert collapsible([(URL, None)], init_len=863) is False assert collapsible([(URL, "863-1999"), (URL, None)], init_len=863) is False @@ -43,9 +37,14 @@ def test_mixed_urls_ineligible(): assert collapsible(segments, init_len=863) is False -def test_first_segment_gap_after_init_ineligible(): - # first media byte does not immediately follow the init prefix - assert collapsible([(URL, "900-1999")], init_len=863) is False +def test_first_segment_gap_after_init_left_to_prefix_check(): + # the gap may be the sidx; single_url_prefix_matches decides from the bytes + segments = [(URL, "900-1999"), (URL, "2000-2999")] + assert DASH.collapsible_single_url(False, segments, 863) == (900, 2999) + + +def test_first_segment_inside_init_ineligible(): + assert collapsible([(URL, "800-1999")], init_len=863) is False def test_gap_between_segments_ineligible(): diff --git a/tests/core/test_dash_collapse_integration.py b/tests/core/test_dash_collapse_integration.py index 133f7cbc..3976164c 100644 --- a/tests/core/test_dash_collapse_integration.py +++ b/tests/core/test_dash_collapse_integration.py @@ -18,6 +18,7 @@ localhost server, asserting that: from __future__ import annotations import shutil +import struct import threading from functools import partial from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -99,48 +100,6 @@ def base(srv) -> str: return f"http://127.0.0.1:{srv.server_address[1]}/" -def collapse_mpd(srv) -> str: - # SegmentList on one media.mp4: init is bytes [0, INIT_LEN), media ranges cover the rest - # contiguously -> collapsible_single_url returns the URL and download_track collapses. - return f""" - - {base(srv)} - - - - media.mp4 - - - - - - - - -""" - - -def clipped_mpd(srv) -> str: - # same parent, but the ranges stop short of its end -> the collapsed file must be trimmed - # to the last indexed byte, not left holding the parent's trailing bytes. - return f""" - - {base(srv)} - - - - media.mp4 - - - - - - - - -""" - - def mixed_mpd(srv) -> str: # Two SegmentURLs on distinct paths -> mixed URLs -> predicate False -> segmented path. return f""" @@ -160,6 +119,33 @@ def mixed_mpd(srv) -> str: """ +def sidx_box(sizes: list[int]) -> bytes: + # version 0 sidx: one MEDIA reference per size, each starting with a SAP + refs = b"".join(struct.pack(">III", size, 1, 0x90000000) for size in sizes) + body = struct.pack(">IIIIIHH", 0, 1, 1, 0, 0, 0, len(sizes)) + refs + return struct.pack(">I", 8 + len(body)) + b"sidx" + body + + +def ranged_mpd(srv, ranges: list[tuple[int, int]], init: str = f'range="0-{INIT_LEN - 1}"') -> str: + # SegmentList of byte ranges on one media.mp4, with the given Initialization attributes + segment_urls = "\n".join(f' ' for s, e in ranges) + return f""" + + {base(srv)} + + + + media.mp4 + + +{segment_urls} + + + + +""" + + def make_track(mpd: str, srv): return DASH.from_text(mpd, url=f"{base(srv)}manifest.mpd").to_tracks(language="en").videos[0] @@ -174,7 +160,7 @@ def make_ctx(tmp_path, name="out.mp4"): def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path): server.routes["/media.mp4"] = PARENT - track = make_track(collapse_mpd(server), server) + track = make_track(ranged_mpd(server, [(INIT_LEN, 5119), (5120, len(PARENT) - 1)]), server) ctx = make_ctx(tmp_path, "collapsed.mp4") DASH.download_track(track, ctx) @@ -194,7 +180,8 @@ def test_collapse_downloads_parent_whole_in_one_direct_request(server, tmp_path) def test_collapse_trims_parent_bytes_past_the_last_segment(server, tmp_path): server.routes["/media.mp4"] = PARENT - track = make_track(clipped_mpd(server), server) + # the ranges stop short of the parent's end, so the collapsed file must be trimmed + track = make_track(ranged_mpd(server, [(INIT_LEN, 5119), (5120, CLIP_END)]), server) ctx = make_ctx(tmp_path, "clipped.mp4") DASH.download_track(track, ctx) @@ -203,6 +190,60 @@ def test_collapse_trims_parent_bytes_past_the_last_segment(server, tmp_path): assert ctx.save_path.read_bytes() == PARENT[: CLIP_END + 1] +def test_collapse_keeps_the_sidx_between_init_and_first_segment(server, tmp_path): + media = [bytes([0xC0]) * 3000, bytes([0xD1]) * 2000] + head = PARENT[:INIT_LEN] + sidx_box([len(m) for m in media]) + parent = head + b"".join(media) + server.routes["/media.mp4"] = parent + ranges = [(len(head), len(head) + 2999), (len(head) + 3000, len(parent) - 1)] + ctx = make_ctx(tmp_path, "sidx.mp4") + + DASH.download_track(make_track(ranged_mpd(server, ranges), server), ctx) + + assert ctx.save_path.read_bytes() == parent + assert [path for path, rng in server.requests if rng is None] == ["/media.mp4"] + + +def test_leading_gap_that_is_not_a_sidx_takes_segmented_path(server, tmp_path): + # a filtered period cut from the same file leaves a gap of media, not an index + gap = struct.pack(">I", 1000) + b"moof" + bytes(992) + media = bytes([0xC0]) * 3000 + server.routes["/media.mp4"] = PARENT[:INIT_LEN] + gap + media + start = INIT_LEN + len(gap) + ctx = make_ctx(tmp_path, "gap.mp4") + + DASH.download_track(make_track(ranged_mpd(server, [(start, start + 2999)]), server), ctx) + + assert ctx.save_path.read_bytes() == PARENT[:INIT_LEN] + media + assert not [path for path, rng in server.requests if rng is None] + + +def test_sidx_indexing_more_than_the_listed_segments_takes_segmented_path(server, tmp_path): + # the file carries a post-roll after the listed segments, and the sidx indexes it too + media, post_roll = bytes([0xC0]) * 3000, bytes([0xEE]) * 500 + head = PARENT[:INIT_LEN] + sidx_box([len(media), len(post_roll)]) + server.routes["/media.mp4"] = head + media + post_roll + ctx = make_ctx(tmp_path, "post-roll.mp4") + + DASH.download_track(make_track(ranged_mpd(server, [(len(head), len(head) + 2999)]), server), ctx) + + assert ctx.save_path.read_bytes() == PARENT[:INIT_LEN] + media + + +def test_init_from_another_file_takes_segmented_path(server, tmp_path): + # the media ranges start where an init of the same length would end, but the init is elsewhere + init = bytes([0x11]) * INIT_LEN + server.routes["/init.bin"] = init + server.routes["/media.mp4"] = PARENT + ranges = [(INIT_LEN, 5119), (5120, len(PARENT) - 1)] + ctx = make_ctx(tmp_path, "init-elsewhere.mp4") + + DASH.download_track(make_track(ranged_mpd(server, ranges, init='sourceURL="init.bin"'), server), ctx) + + assert ctx.save_path.read_bytes() == init + PARENT[INIT_LEN:] + assert not [path for path, rng in server.requests if rng is None and path == "/media.mp4"] + + def test_non_collapse_takes_segmented_merge_path(server, tmp_path): server.routes["/init.bin"] = INIT2 server.routes["/part0.bin"] = PART0 diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 8c05fa32..f530acc0 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -40,6 +40,7 @@ from unshackle.core.tracks.track import ( ) from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8 from unshackle.core.utils import run_proxy +from unshackle.core.utils.bitrate import ranged_get from unshackle.core.utils.redact import safe_display_url from unshackle.core.utils.xml import load_xml @@ -558,14 +559,17 @@ class DASH: downloader = track.downloader - # When every segment is a byte range of one parent resource and init is its [0, len) - # prefix, the whole resource is itself a valid MP4, so one direct download to save_path - # replaces the per-segment files, init write and merge pass. - collapse_single_url = not config.sample and DASH.collapsible_single_url( - isinstance(track, Subtitle), - segments, - len(init_data) if init_data is not None else None, + # A parent resource that is itself a valid MP4 downloads in one pass to save_path, + # replacing the per-segment files, init write and merge pass. + collapse_span = ( + DASH.collapsible_single_url(isinstance(track, Subtitle), segments, len(init_data)) + if init_data is not None and not config.sample + else None ) + if collapse_span and init_data: + if not DASH.single_url_prefix_matches(session, segments[0][0], collapse_span, init_data): + collapse_span = None + collapse_single_url = collapse_span is not None use_segment_decrypt = not collapse_single_url and bool(init_data) and can_use(drm, config.decryption) # subtitle segments are tiny, the post-download merge costs nothing there @@ -674,9 +678,9 @@ class DASH: save_path.unlink(missing_ok=True) raise - if collapse_single_url: + if collapse_span: with open(save_path, "r+b") as collapsed: - collapsed.truncate(int(segments[-1][1].split("-")[1]) + 1) + collapsed.truncate(collapse_span[1] + 1) elif merger: for control_file in save_dir.glob("*.!dev"): control_file.unlink(missing_ok=True) @@ -805,42 +809,67 @@ class DASH: def collapsible_single_url( is_subtitle: bool, segments: list[tuple[str, Optional[str]]], - init_len: Optional[int], - ) -> bool: + init_len: int, + ) -> Optional[tuple[int, int]]: """ - Whether every segment is a byte range of one parent resource whose [0, init_len) - prefix is the init segment, so unshackle can download the whole resource in one pass. + Get the first and last byte of the segments when every segment is a byte range of one + parent resource that follows an init of init_len bytes, so unshackle can download the + whole resource in one pass. - Returns False for subtitle tracks, an unknown init length, mixed URLs, malformed - ranges, and anything but one gapless run in document order starting at the init - prefix. A gap would pull bytes the segmented merge path excludes (a period the - manifest filters out) into the collapsed file, so it fails closed. + Returns None for subtitle tracks, mixed URLs, malformed ranges, and anything but one + gapless run in document order. A gap between segments would pull bytes the segmented + merge path excludes (a period the manifest filters out) into the collapsed file, so it + fails closed. A gap before the first segment passes here, and single_url_prefix_matches + must then prove that it is the index. """ - if is_subtitle or init_len is None or not segments: - return False + if is_subtitle or not segments: + return None first_url = segments[0][0] ranges: list[tuple[int, int]] = [] for url, bytes_range in segments: if url != first_url or not bytes_range: - return False + return None parts = bytes_range.split("-") if len(parts) != 2 or not parts[0] or not parts[1]: - return False + return None try: start, end = int(parts[0]), int(parts[1]) except ValueError: - return False + return None if end < start: - return False + return None ranges.append((start, end)) - prev_end = init_len - 1 + if ranges[0][0] < init_len: + return None + prev_end = ranges[0][0] - 1 for start, end in ranges: if start != prev_end + 1: - return False + return None prev_end = end - return True + return ranges[0][0], prev_end + + @staticmethod + def single_url_prefix_matches( + session: Union[Session, RnetSession], + url: str, + span: tuple[int, int], + init_data: bytes, + ) -> bool: + """ + Whether the resource at url starts with init_data and, when bytes sit between the init + and the first byte of span, whether they are one sidx that indexes exactly span. + Fetches the resource up to the start of span. + """ + first_start, last_end = span + head = ranged_get(url, session, f"bytes=0-{first_start - 1}") + if not head or len(head) != first_start or not head.startswith(init_data): + return False + if first_start == len(init_data): + return True + _, subsegments = parse_sidx(head[len(init_data) :], len(init_data)) + return bool(subsegments) and subsegments[0][1] == first_start and subsegments[-1][2] == last_end @staticmethod def is_content_period(period: Element, filtered_period_ids: list[str]) -> bool: