fix(ism): derive video colour range from CodecPrivateData SPS VUI

Smooth Streaming manifests carry no range attributes, so every ISM video track was labelled SDR even when the stream is HDR10/HLG/DV, breaking range-based selection (-r HDR10 / -r DV) for ISM services. - ism_init: walk the full HEVC SPS (incl. scaling-list and st_ref_pic_set skippers) to read the VUI colour triple (colour_primaries, transfer_characteristics, matrix_coeffs); expose parse_codec_private_data_colour() keyed by FourCC. No unshackle imports added. - ism: new ISM.get_video_range() maps the CICP triple via Video.Range.from_cicp (PQ -> HDR10, HLG -> HLG, BT.709/absent -> SDR); DVHE/DVH1 FourCCs map straight to DV since DV bitstreams signal Unspecified (2,2,2) in the VUI. to_tracks() now sets range_ on every video track. Soft-fails to SDR on malformed data. - ism: accept RnetSession in download_track() so TLS-impersonated sessions pass the type check. - tests: real PQ/HLG/BT.709 (x265-minted) and Dolby Vision (live-manifest, DoViProfile=stn, out-of-order SPS,PPS,VPS NALs) CodecPrivateData samples; byte-level VUI assertions in test_ism_init and manifest->Range characterization in new test_ism_range.
2026-06-22 17:07:23 +00:00 · 2026-06-11 18:28:35 -06:00
parent 39034f2bb5
commit e207116d30
4 changed files with 278 additions and 11 deletions
--- a/tests/core/test_ism_init.py
+++ b/tests/core/test_ism_init.py
@@ -13,7 +13,8 @@ import struct
 import pytest

 from unshackle.core.manifests.ism_init import (NAL_START_CODE, PIFF_SENC_UUID, box, build_avcc, build_dec3,
-                                               build_hvcc, build_init_segment, full_box, parse_hevc_sps_format,
+                                               build_hvcc, build_init_segment, full_box,
+                                               parse_codec_private_data_colour, parse_hevc_sps_format,
                                               read_per_sample_iv_size, read_track_id, remove_emulation_prevention,
                                               split_nal_units, synthesize_aac_codec_private_data)

@@ -30,6 +31,28 @@ VIDEO_HEVC10_CPD = (
    "0000000140010c01ffff02200000030090000003000003003c959809000000000142010102200000030090"
    "000003000003003ca00a080b9f6d96566924caf0168080000003008000000c8400000000014401c172b4624000"
 )
+# HEVC VPS+SPS+PPS minted with x265, explicit SPS VUI colour signalling:
+# PQ (bt2020/smpte2084/bt2020nc), HLG (arib-std-b67) and BT.709 SDR.
+VIDEO_HEVC_PQ_CPD = (
+    "0000000140010c01ffff02200000030090000003000003001e9598090000000142010102200000030090000003000003001ea020"
+    "8104d96566924caf016a12201208000003000800000300c840000000014401c172b42240"
+)
+VIDEO_HEVC_HLG_CPD = (
+    "0000000140010c01ffff02200000030090000003000003001e9598090000000142010102200000030090000003000003001ea020"
+    "8104d96566924caf016a12241208000003000800000300c840000000014401c172b42240"
+)
+VIDEO_HEVC_SDR_CPD = (
+    "0000000140010c01ffff02200000030090000003000003001e9598090000000142010102200000030090000003000003001ea020"
+    "8104d96566924caf016a02020208000003000800000300c840000000014401c172b42240"
+)
+# Real Dolby Vision (dvhe, DoViProfile "stn") CodecPrivateData from a Smooth
+# manifest: NALs arrive SPS,PPS,VPS (VPS last) and the VUI colour triple is
+# Unspecified (2,2,2) — DV is signalled by FourCC only, never by CICP.
+VIDEO_HEVC_DV_CPD = (
+    "00000001420101022000000300B00000030000030096A001E020021C4D9457B91CAF016E0404042800001F480002EE0401F4E1"
+    "15EE7E0001312D00002FAF0C80000000014401C1ACBE0EC90000000140010C01FFFF022000000300B00000030000030096"
+    "15C0C00000FA40001770200FA680"
+)
 AAC_LC_CPD = "1190"
 # Real Smooth EC-3 CodecPrivateData: WAVEFORMATEXTENSIBLE extension (samples
 # per block + channel mask + DD+ GUID) followed by the 5-byte dec3 payload.
@@ -400,6 +423,55 @@ def test_read_track_id_truncated_tfhd_returns_none():
    assert read_track_id(fragment) is None


+def test_parse_colour_hevc_pq():
+    # PQ master: bt2020 primaries (9), smpte2084 transfer (16), bt2020nc matrix (9).
+    assert parse_codec_private_data_colour("HVC1", bytes.fromhex(VIDEO_HEVC_PQ_CPD)) == (9, 16, 9)
+
+
+def test_parse_colour_hevc_hlg():
+    assert parse_codec_private_data_colour("HVC1", bytes.fromhex(VIDEO_HEVC_HLG_CPD)) == (9, 18, 9)
+
+
+def test_parse_colour_hevc_bt709():
+    assert parse_codec_private_data_colour("HVC1", bytes.fromhex(VIDEO_HEVC_SDR_CPD)) == (1, 1, 1)
+    # The real-manifest 8-bit sample also signals BT.709 explicitly.
+    assert parse_codec_private_data_colour("HVC1", bytes.fromhex(VIDEO_HEVC_CPD)) == (1, 1, 1)
+
+
+def test_parse_colour_dv_is_unspecified():
+    # DV carries no usable CICP; the DV decision must come from the FourCC.
+    assert parse_codec_private_data_colour("DVHE", bytes.fromhex(VIDEO_HEVC_DV_CPD)) == (2, 2, 2)
+
+
+def test_dv_cpd_with_vps_last_builds_init():
+    cpd = bytes.fromhex(VIDEO_HEVC_DV_CPD)
+    nals = split_nal_units(cpd)
+    assert [(n[0] >> 1) & 0x3F for n in nals] == [33, 34, 32]  # SPS, PPS, VPS
+    sps = remove_emulation_prevention(nals[0])
+    assert parse_hevc_sps_format(sps) == (1, 2, 2)  # 4:2:0, 10-bit
+    hvcc = build_hvcc(cpd)
+    for nal in nals:
+        assert nal in hvcc
+    init = build_init_segment(
+        stream_type="video",
+        fourcc="DVHE",
+        codec_private_data=VIDEO_HEVC_DV_CPD,
+        timescale=10000000,
+        width=3840,
+        height=2160,
+    )
+    assert b"dvh1" in init and b"hvcC" in init
+
+
+def test_parse_colour_absent_or_unknown_returns_none():
+    # Real sample without a VUI colour description.
+    assert parse_codec_private_data_colour("HVC1", bytes.fromhex(VIDEO_HEVC10_CPD)) is None
+    # Non-HEVC codecs (AVC has no HDR deployment) and truncated data must not raise.
+    assert parse_codec_private_data_colour("H264", bytes.fromhex(VIDEO_AVC_CPD)) is None
+    assert parse_codec_private_data_colour("WVC1", bytes.fromhex(VIDEO_AVC_CPD)) is None
+    assert parse_codec_private_data_colour("HVC1", b"\x00\x00\x00\x01\x42") is None
+
+
 def test_hvcc_profile_tier_level_is_nonzero():
    # De-emulated PTL must yield real profile/level, not the off-by-one garbage.
    hvcc = build_hvcc(bytes.fromhex(VIDEO_HEVC_CPD))
--- a/tests/core/test_ism_range.py
+++ b/tests/core/test_ism_range.py
@@ -0,0 +1,66 @@
+"""Offline characterization: ISM CodecPrivateData SPS VUI -> Video.Range
+(PQ -> HDR10, HLG -> HLG, BT.709/absent -> SDR). HDR10+ is per-frame SEI,
+undecidable from the manifest; the post-mux bitstream probe names it."""
+
+from __future__ import annotations
+
+from unshackle.core.manifests import ISM
+from unshackle.core.tracks import Video
+
+from tests.core.test_ism_init import (VIDEO_HEVC10_CPD, VIDEO_HEVC_DV_CPD, VIDEO_HEVC_HLG_CPD,  # isort: skip
+                                      VIDEO_HEVC_PQ_CPD, VIDEO_HEVC_SDR_CPD)
+
+
+def manifest_xml(cpd: str, fourcc: str = "HVC1") -> str:
+    return (
+        '<SmoothStreamingMedia MajorVersion="2" MinorVersion="0" TimeScale="10000000" Duration="100000000">'
+        '<StreamIndex Type="video" Name="video" Chunks="1" QualityLevels="1" MaxWidth="3840" MaxHeight="2160" '
+        'Url="QualityLevels({bitrate})/Fragments(video={start time})">'
+        f'<QualityLevel Index="0" Bitrate="15000000" FourCC="{fourcc}" MaxWidth="3840" MaxHeight="2160" '
+        f'CodecPrivateData="{cpd}"/>'
+        '<c t="0" d="100000000"/>'
+        "</StreamIndex>"
+        "</SmoothStreamingMedia>"
+    )
+
+
+def parse_video(cpd: str, fourcc: str = "HVC1") -> Video:
+    tracks = ISM.from_text(manifest_xml(cpd, fourcc), url="https://x/ism/manifest").to_tracks(language="en")
+    assert len(tracks.videos) == 1
+    return tracks.videos[0]
+
+
+def test_pq_codec_private_data_yields_hdr10() -> None:
+    assert parse_video(VIDEO_HEVC_PQ_CPD).range == Video.Range.HDR10
+
+
+def test_hlg_codec_private_data_yields_hlg() -> None:
+    assert parse_video(VIDEO_HEVC_HLG_CPD).range == Video.Range.HLG
+
+
+def test_bt709_codec_private_data_stays_sdr() -> None:
+    assert parse_video(VIDEO_HEVC_SDR_CPD).range == Video.Range.SDR
+
+
+def test_colourless_codec_private_data_defaults_sdr() -> None:
+    # Real 10-bit sample without a VUI colour description: unspecified -> SDR.
+    assert parse_video(VIDEO_HEVC10_CPD).range == Video.Range.SDR
+
+
+def test_get_video_range_dolby_vision_fourcc() -> None:
+    assert ISM.get_video_range("DVH1", VIDEO_HEVC_PQ_CPD) == Video.Range.DV
+    assert ISM.get_video_range("DVHE", "") == Video.Range.DV
+
+
+def test_dv_track_from_real_smooth_cpd() -> None:
+    # Live manifests ship lowercase "dvhe"; its VUI is Unspecified so the
+    # FourCC short-circuit is the only thing standing between DV and SDR.
+    video = parse_video(VIDEO_HEVC_DV_CPD, fourcc="dvhe")
+    assert video.range == Video.Range.DV
+    assert ISM.get_video_range("hvc1", VIDEO_HEVC_DV_CPD) == Video.Range.SDR
+
+
+def test_get_video_range_malformed_data_soft_fails_sdr() -> None:
+    assert ISM.get_video_range("HVC1", "not-hex") == Video.Range.SDR
+    assert ISM.get_video_range("HVC1", "") == Video.Range.SDR
+    assert ISM.get_video_range("", VIDEO_HEVC_PQ_CPD) == Video.Range.SDR