fix(ism): signal Dolby Vision and Atmos from Smooth manifests

Init segments now carry dvcC/dvvC with a profile-keyed sample entry and the dby1 brand; track ids cover the manifest URL so per-range ladders stop colliding, and the remote path carries the Atmos flag without inventing a JOC count.

docs/dev/rest-api/endpoints.md

@@ -434,6 +434,26 @@ Make a download job. It requires `service` and `title_id`. Every other field is
 
 **Validation.** Invalid values return `400 INVALID_PARAMETERS`. `vcodec` must be one of H264/H265/H.264/H.265/AVC/HEVC/VC1/VC-1/VP8/VP9/AV1. `acodec` must be one of AAC/AC3/EC3/EAC3/DD/DD+/AC4/OPUS/FLAC/ALAC/VORBIS/OGG/DTS. `range` must be one of SDR/HDR10/HDR10P/DV/HLG/HYBRID, and `HDR10+` is also valid. The bitrate, download worker, and download counts must be positive integers. You may set at most one of the `*_only` flags. You cannot combine `no_subs` with `subs_only`, or `no_audio` with `audio_only`.
 
+**Service options.** A service's own CLI options travel in the same body. Send them in a nested
+`service_params` object, or as flat top-level keys. `service_params` wins over a flat key of the
+same name. An option you leave out takes the default declared on the service's `cli` command, so
+the job matches the equivalent `dl` run. `profile` at the top level always means the credential
+profile, never a service's own `--profile` option.
+
+=== "Request"
+
+    ```bash
+    curl -X POST http://127.0.0.1:8786/api/download \
+      -H "X-Secret-Key: $KEY" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "service": "EXAMPLE",
+        "title_id": "12345",
+        "quality": [2160],
+        "service_params": {"device": "chrome", "movie": true}
+      }'
+    ```
+
 !!! danger "Rejected parameters"
     `postscript`, `post_script` and `post_scripts` are never accepted. A body containing any
     of them gets `400 INVALID_PARAMETERS`, because a command string that arrives from

docs/dev/service-architecture.md

@@ -365,7 +365,9 @@ A typical `get_tracks` therefore looks like fetching the manifest text with
   the media-sequence-number IV for AES-128 keys that have no explicit IV.
 - **ISM** has no list of segments and no init segments: the parser synthesizes segment URLs
   from the `<c>` fragment timeline, and rebuilds a `moov`/`ftyp` init box from `CodecPrivateData`
-  (supported for AVC, HEVC, DV-over-HEVC, AAC, EC-3, and TTML).
+  (supported for AVC, HEVC, DV-over-HEVC, AAC, EC-3, and TTML). A track id covers the manifest
+  URL, so a title you build from one ISM manifest per codec or range keeps every variant:
+  `Tracks.add` no longer reads the second manifest's tracks as duplicates of the first.
 
 ## DRM hooks
 

tests/core/test_ism_init.py

@@ -18,6 +18,7 @@ from unshackle.core.manifests.ism_init import (
     box,
     build_avcc,
     build_dec3,
+    build_dvcc,
     build_hvcc,
     build_init_segment,
     full_box,
@@ -149,6 +150,13 @@ def find_stsd(init: bytes) -> int:
     return offset - 8
 
 
+def sample_entry_bounds(init: bytes) -> tuple[int, int]:
+    """(start, end) byte range of the first stsd sample entry."""
+    stsd = find_stsd(init)
+    entry_size = struct.unpack_from(">I", init, stsd + 16)[0]
+    return stsd + 16, stsd + 16 + entry_size
+
+
 # The two-entry stsd is unconditional, so every stream type and codec the builder
 # emits needs covering, encrypted and clear alike.
 STSD_CASES = [
@@ -816,3 +824,154 @@ def test_encrypted_hev1_frma_carries_the_same_fourcc():
     )
     assert b"encv" in init
     assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"hev1"
+
+
+def test_dolby_vision_sample_entry_carries_dvcc():
+    # a dvh1 sample entry without dvcC describes itself as plain HEVC, so every reader
+    # downstream loses the Dolby Vision signalling even though the RPU NAL units are in
+    # the elementary stream.
+    init = build_init_segment(
+        stream_type="video", fourcc="dvhe", codec_private_data=VIDEO_HEVC_CPD, width=3840, height=2160
+    )
+    offset = find_stsd(init)
+    assert init[offset + 20 : offset + 24] == b"dvh1"
+
+    # readers cannot reach a dvcC outside the sample entry, so byte presence alone
+    # proves nothing.
+    entry_start, entry_end = sample_entry_bounds(init)
+    assert entry_start <= init.index(b"dvcC") < entry_end
+
+    payload = init[init.index(b"dvcC") + 4 :][:24]
+    assert payload[0] == 1 and payload[1] == 0
+    flags = struct.unpack(">H", payload[2:4])[0]
+    assert flags >> 9 == 5  # profile 5: base layer carries no usable colour description
+    assert (flags >> 3) & 0x3F == 6  # level 6: 3840x2160 at 24 fps
+    assert flags & 0b111 == 0b101  # RPU present, no enhancement layer, base layer present
+
+
+@pytest.mark.parametrize(
+    "cpd,expected_box,expected_profile,expected_compat,expected_entry",
+    [
+        (VIDEO_HEVC_DV_CPD, b"dvcC", 5, 0, b"dvh1"),
+        (VIDEO_HEVC_PQ_CPD, b"dvvC", 8, 1, b"hvc1"),
+        (VIDEO_HEVC_HLG_CPD, b"dvvC", 8, 4, b"hvc1"),
+    ],
+)
+def test_dolby_vision_profile_follows_the_base_layer_colour(
+    cpd, expected_box, expected_profile, expected_compat, expected_entry
+):
+    # a base layer with no usable colour description is profile 5; one that stays readable as
+    # HDR10 or HLG is a cross-compatible profile, which readers expect under dvvC, not dvcC.
+    # The sample entry follows base-layer decodability: hvc1 for a compliant base, dvh1 otherwise.
+    init = build_init_segment(stream_type="video", fourcc="DVH1", codec_private_data=cpd, width=3840, height=2160)
+    assert expected_box in init
+    entry_start, entry_end = sample_entry_bounds(init)
+    assert entry_start <= init.index(expected_box) < entry_end
+    payload = init[init.index(expected_box) + 4 :][:24]
+    assert struct.unpack(">H", payload[2:4])[0] >> 9 == expected_profile
+    assert struct.unpack(">I", payload[4:8])[0] >> 28 == expected_compat
+    offset = find_stsd(init)
+    assert init[offset + 20 : offset + 24] == expected_entry
+
+
+def test_dolby_vision_profile_8_dvhe_takes_hev1_and_frma_follows():
+    # DVHE maps to the in-band-parameter-set entry (hev1) when the base layer is
+    # HEVC-decodable, and the frma inside sinf must carry the same fourcc.
+    init = build_init_segment(
+        stream_type="video", fourcc="dvhe", codec_private_data=VIDEO_HEVC_PQ_CPD, width=3840, height=2160
+    )
+    offset = find_stsd(init)
+    assert init[offset + 20 : offset + 24] == b"hev1"
+    assert b"dvvC" in init
+
+    init = build_init_segment(
+        stream_type="video", fourcc="dvhe", codec_private_data=VIDEO_HEVC_PQ_CPD, width=3840, height=2160, kid=KID
+    )
+    assert b"encv" in init and b"dvvC" in init
+    assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"hev1"
+    entry_start, entry_end = sample_entry_bounds(init)
+    assert entry_start <= init.index(b"dvvC") < entry_end
+
+
+def test_encrypted_dolby_vision_wraps_encv_with_dvcc_before_sinf():
+    # the encrypted entry must keep the dvcC inside the encv wrapper and ahead of the
+    # sinf, or tooling that decrypts and remuxes drops the Dolby Vision config with the
+    # wrapper.
+    init = build_init_segment(
+        stream_type="video", fourcc="DVH1", codec_private_data=VIDEO_HEVC_DV_CPD, width=3840, height=2160, kid=KID
+    )
+    assert b"encv" in init and b"tenc" in init
+    assert init[init.index(b"frma") + 4 : init.index(b"frma") + 8] == b"dvh1"
+    entry_start, entry_end = sample_entry_bounds(init)
+    dvcc_at = init.index(b"dvcC")
+    assert entry_start <= dvcc_at < entry_end
+    assert dvcc_at < init.index(b"sinf") < entry_end
+
+
+@pytest.mark.parametrize(
+    "fps,expected_level",
+    [
+        (24.0, 6),
+        (30.0, 7),
+        (48.0, 8),
+        (60.0, 9),  # same 4K frame, higher rate: the pixel-rate limit must move the level
+        (120.0, 10),
+        (None, 7),  # no VUI timing: the 25 fps assumption sits one level above 2160p24
+    ],
+)
+def test_dolby_vision_level_scales_with_frame_rate(fps, expected_level):
+    payload = build_dvcc(3840, 2160, fps, (2, 2, 2))[8:]
+    flags = struct.unpack(">H", payload[2:4])[0]
+    assert (flags >> 3) & 0x3F == expected_level
+
+
+def test_dolby_vision_ftyp_carries_dby1_brand():
+    # Dolby ISOBMFF requires the dby1 compatible brand on files with a Dolby Vision
+    # config box; plain HEVC must not claim it.
+    dv_init = build_init_segment(
+        stream_type="video", fourcc="DVH1", codec_private_data=VIDEO_HEVC_DV_CPD, width=3840, height=2160
+    )
+    ftyp_size = struct.unpack(">I", dv_init[:4])[0]
+    assert b"dby1" in dv_init[:ftyp_size]
+
+    hevc_init = build_init_segment(
+        stream_type="video", fourcc="HVC1", codec_private_data=VIDEO_HEVC_CPD, width=3840, height=2160
+    )
+    assert b"dby1" not in hevc_init
+
+
+def test_hevc_cpd_without_vps_builds_init():
+    # MS-SSTR HEVC CodecPrivateData is SPS+PPS only (no VPS); the hvcC must build
+    # from what is there, and the encrypted path must still produce sinf/tenc.
+    nals = split_nal_units(bytes.fromhex(VIDEO_HEVC_PQ_CPD))
+    cpd = b"".join(b"\x00\x00\x00\x01" + n for n in nals if (n[0] >> 1) & 0x3F != 32).hex()
+
+    init = build_init_segment(stream_type="video", fourcc="HVC1", codec_private_data=cpd, width=1920, height=1080)
+    assert b"hvcC" in init
+
+    init = build_init_segment(
+        stream_type="video", fourcc="HVC1", codec_private_data=cpd, width=1920, height=1080, kid=KID
+    )
+    assert b"sinf" in init and b"tenc" in init
+
+    # SPS and PPS stay mandatory: dropping the SPS must still raise.
+    no_sps = b"".join(b"\x00\x00\x00\x01" + n for n in nals if (n[0] >> 1) & 0x3F not in (32, 33)).hex()
+    with pytest.raises(ValueError):
+        build_init_segment(stream_type="video", fourcc="HVC1", codec_private_data=no_sps, width=1920, height=1080)
+
+
+@pytest.mark.parametrize(
+    "width,height,expected_level",
+    [
+        (1920, 1080, 3),  # 1080p24
+        (3840, 2160, 6),  # 2160p24
+        (7680, 4320, 11),  # 8K24 falls in the level that shares 4K120's pixel rate
+        (3840, 600, 5),  # a pixel rate that fits level 4, a width that does not
+    ],
+)
+def test_dolby_vision_level_needs_both_pixel_rate_and_width(width, height, expected_level):
+    init = build_init_segment(
+        stream_type="video", fourcc="DVH1", codec_private_data=VIDEO_HEVC_DV_CPD, width=width, height=height
+    )
+    payload = init[init.index(b"dvcC") + 4 :][:24]
+    assert (struct.unpack(">H", payload[2:4])[0] >> 3) & 0x3F == expected_level

tests/remote/unit/test_remote_service_helpers.py

@@ -56,17 +56,25 @@ def test_deserialize_video_unknown_codec_falls_back_to_none() -> None:
     assert v.codec is None
 
 
-def test_deserialize_audio_atmos_flag_sets_joc() -> None:
+def test_deserialize_audio_atmos_flag_without_a_count() -> None:
     a = deserialize_audio({"id": "a1", "codec": "AAC", "atmos": True, "channels": 6, "bitrate": 256})
     assert isinstance(a, Audio)
-    assert a.joc == 1
+    assert a.atmos is True
+    assert a.joc is None
     assert a.channels == 6
     assert a.bitrate == 256_000
 
 
+def test_deserialize_audio_atmos_keeps_a_real_joc_count() -> None:
+    a = deserialize_audio({"id": "a3", "codec": "EC3", "atmos": True, "joc": 16, "channels": 6})
+    assert a.atmos is True
+    assert a.joc == 16
+
+
 def test_deserialize_audio_no_atmos() -> None:
     a = deserialize_audio({"id": "a2", "codec": "AAC", "channels": 2})
-    assert a.joc == 0
+    assert a.atmos is False
+    assert not a.joc
 
 
 def test_deserialize_subtitle_forced_flag() -> None:

unshackle/core/api/handlers.py

@@ -717,7 +717,7 @@ def extract_manifests(tracks) -> List[Dict[str, Any]]:
         if not manifest_url or manifest_url in seen:
             continue
 
-        if track.data.get("dash") and track.data["dash"].get("manifest"):
+        if track.data.get("dash") and track.data["dash"].get("manifest") is not None:
             seen.add(manifest_url)
             xml_bytes = etree.tostring(track.data["dash"]["manifest"], xml_declaration=True, encoding="UTF-8")
             compressed = zlib.compress(xml_bytes, compression_level) if compression_level else xml_bytes
@@ -728,7 +728,7 @@ def extract_manifests(tracks) -> List[Dict[str, Any]]:
                     "data": base64.b64encode(compressed).decode("ascii"),
                 }
             )
-        elif track.data.get("ism") and track.data["ism"].get("manifest"):
+        elif track.data.get("ism") and track.data["ism"].get("manifest") is not None:
             seen.add(manifest_url)
             xml_bytes = etree.tostring(track.data["ism"]["manifest"], xml_declaration=True, encoding="UTF-8")
             compressed = zlib.compress(xml_bytes, compression_level) if compression_level else xml_bytes

unshackle/core/manifests/ism.py

@@ -193,8 +193,10 @@ class ISM:
                     timescale=timescale,
                     duration=duration,
                     language=language,
-                    width=int(quality_level.get("MaxWidth") or stream_index.get("MaxWidth") or 0),
-                    height=int(quality_level.get("MaxHeight") or stream_index.get("MaxHeight") or 0),
+                    width=int(quality_level.get("MaxWidth") or quality_level.get("Width") or 0)
+                    or int(stream_index.get("MaxWidth") or stream_index.get("Width") or 0),
+                    height=int(quality_level.get("MaxHeight") or quality_level.get("Height") or 0)
+                    or int(stream_index.get("MaxHeight") or stream_index.get("Height") or 0),
                     track_id=track_id,
                     nal_length_size=nal_length_size,
                     kid=kid,
@@ -289,13 +291,14 @@ class ISM:
                         fragment_time += duration_frag
 
                 track_id = hashlib.md5(
-                    "{codec}-{lang}-{bitrate}-{index}-{name}-{url}".format(
+                    "{codec}-{lang}-{bitrate}-{index}-{name}-{url}-{manifest}".format(
                         codec=codec,
                         lang=track_lang,
                         bitrate=ql.get("Bitrate") or 0,
                         index=ql.get("Index") or 0,
                         name=stream_index.get("Name") or "",
                         url=stream_index.get("Url") or "",
+                        manifest=(self.url or "").split("?")[0],
                     ).encode()
                 ).hexdigest()
 
@@ -348,6 +351,7 @@ class ISM:
                             is_original_lang=bool(language and track_lang and str(track_lang) == str(language)),
                             bitrate=ql.get("Bitrate"),
                             channels=ql.get("Channels"),
+                            extra={"atmos": True} if (ql.get("HasAtmos") or "").lower() == "true" else None,
                             descriptor=Track.Descriptor.ISM,
                             drm=drm,
                             data=data,

unshackle/core/manifests/ism_init.py

@@ -522,9 +522,65 @@ def build_avcc(codec_private_data: bytes, nal_length_size: int = 4) -> bytes:
     return box(b"avcC", payload)
 
 
+DV_LEVELS = (
+    (1280 * 720 * 24, 1280, 1),
+    (1280 * 720 * 30, 1280, 2),
+    (1920 * 1080 * 24, 1920, 3),
+    (1920 * 1080 * 30, 2560, 4),
+    (1920 * 1080 * 60, 3840, 5),
+    (3840 * 2160 * 24, 3840, 6),
+    (3840 * 2160 * 30, 3840, 7),
+    (3840 * 2160 * 48, 3840, 8),
+    (3840 * 2160 * 60, 3840, 9),
+    (3840 * 2160 * 120, 3840, 10),
+    (3840 * 2160 * 120, 7680, 11),
+    (7680 * 4320 * 60, 7680, 12),
+    (7680 * 4320 * 120, 7680, 13),
+)
+
+
+def infer_dv_profile(cicp: Optional[tuple[int, int, int]]) -> tuple[int, int]:
+    """Derive (dv_profile, dv_bl_signal_compatibility_id) from the base layer's colour description.
+
+    Profile 5 carries no usable description, while profile 8 stays backward compatible
+    with HDR10 (PQ) or HLG and says so.
+    """
+    transfer = cicp[1] if cicp else 0
+    if transfer == 16:
+        return 8, 1
+    if transfer == 18:
+        return 8, 4
+    return 5, 0
+
+
+def build_dvcc(width: int, height: int, fps: Optional[float], cicp: Optional[tuple[int, int, int]]) -> bytes:
+    """
+    Assemble the Dolby Vision decoder config box for a single-layer Dolby Vision track:
+    dvcC, or dvvC when the profile is cross-compatible.
+
+    A dvh1/dvhe sample entry without this box describes itself as plain HEVC, so every
+    reader downstream (muxers, players, MediaInfo) loses the Dolby Vision signalling even
+    though the RPU NAL units are present in the elementary stream.
+
+    The base layer's colour description separates the two single-layer profiles: profile 5
+    carries no usable description, while profile 8 stays backward compatible with HDR10 (PQ)
+    or HLG and says so (see infer_dv_profile). Level is the first entry in the Dolby table
+    that allows both the pixel rate and the width.
+    """
+    profile, compatibility_id = infer_dv_profile(cicp)
+
+    pixel_rate = width * height * (fps or 25.0)
+    level = next((lvl for pps, max_width, lvl in DV_LEVELS if pixel_rate <= pps and width <= max_width), 13)
+
+    flags = (profile << 9) | (level << 3) | (1 << 2) | (0 << 1) | 1
+    payload = u8.pack(1) + u8.pack(0) + u16.pack(flags) + u32.pack(compatibility_id << 28) + b"\x00" * 16
+    return box(b"dvvC" if profile > 7 else b"dvcC", payload)
+
+
 def build_hvcc(codec_private_data: bytes, nal_length_size: int = 4) -> bytes:
     """
-    Assemble an hvcC (HEVC decoder config) box from VPS+SPS+PPS CodecPrivateData.
+    Assemble an hvcC (HEVC decoder config) box from SPS+PPS CodecPrivateData, and
+    include the VPS when the CodecPrivateData carries one.
 
     The profile/tier/level bytes come from the SPS profile_tier_level. This
     function reads the chroma format and the bit depths from the SPS, so 10-bit
@@ -532,8 +588,6 @@ def build_hvcc(codec_private_data: bytes, nal_length_size: int = 4) -> bytes:
     8-bit 4:2:0.
     """
     nals = split_nal_units(codec_private_data)
-    if len(nals) < 3:
-        raise ValueError("HEVC CodecPrivateData must contain VPS, SPS and PPS NAL units")
 
     # Group NAL units by type (HEVC NAL type = (first byte >> 1) & 0x3F).
     by_type: dict[int, list[bytes]] = {}
@@ -541,7 +595,10 @@ def build_hvcc(codec_private_data: bytes, nal_length_size: int = 4) -> bytes:
         nal_type = (nal[0] >> 1) & 0x3F
         by_type.setdefault(nal_type, []).append(nal)
 
-    sps = by_type.get(33, [b""])[0]
+    if not by_type.get(33) or not by_type.get(34):
+        raise ValueError("HEVC CodecPrivateData must contain SPS and PPS NAL units")
+
+    sps = by_type[33][0]
     # profile_tier_level must be read from the de-emulated SPS RBSP, after the
     # 2-byte NAL header + 1 byte (sps_video_parameter_set_id(4) +
     # sps_max_sub_layers_minus1(3) + sps_temporal_id_nesting_flag(1)). PTL is 12
@@ -715,7 +772,10 @@ def build_init_segment(
     if len(lang) != 3 or not all("a" <= c <= "z" for c in lang):
         lang = "und"
 
-    ftyp = box(b"ftyp", b"isml" + u32.pack(1) + b"iso5" + b"iso6" + b"piff" + b"msdh")
+    brands = b"iso5" + b"iso6" + b"piff" + b"msdh"
+    if stream_type == "video" and fourcc in ("DVHE", "DVH1"):
+        brands += b"dby1"
+    ftyp = box(b"ftyp", b"isml" + u32.pack(1) + brands)
 
     # version 1: at the 10 MHz ISM timescale a 32-bit duration overflows after ~429s.
     mvhd = full_box(
@@ -805,9 +865,14 @@ def build_init_segment(
             # (and the frma inside sinf).
             codec_fourcc = b"hev1" if fourcc == "HEV1" else b"hvc1"
         elif fourcc in ("DVHE", "DVH1"):
-            # Dolby Vision over HEVC: same hvcC config, dvh1 sample entry.
-            config_box = build_hvcc(cpd, nal_length_size)
-            codec_fourcc = b"dvh1"
+            # Dolby Vision over HEVC: same hvcC config plus the dvcC/dvvC that tells
+            # readers the RPU NAL units in the elementary stream are Dolby Vision.
+            cicp, fps = parse_codec_private_data_vui(fourcc, cpd)
+            config_box = build_hvcc(cpd, nal_length_size) + build_dvcc(width, height, fps, cicp)
+            if infer_dv_profile(cicp)[0] > 7:
+                codec_fourcc = b"hev1" if fourcc == "DVHE" else b"hvc1"
+            else:
+                codec_fourcc = b"dvh1"
         else:
             raise NotImplementedError(f"Unsupported video FourCC: {fourcc}")
         sample_entry_payload += config_box

unshackle/core/remote_service.py

@@ -191,7 +191,8 @@ def deserialize_audio(data: Dict[str, Any]) -> Audio:
         codec=enum_get(Audio.Codec, data.get("codec")),
         bitrate=deserialize_bitrate(data),
         channels=data.get("channels"),
-        joc=joc if joc is not None else (1 if data.get("atmos") else 0),
+        joc=joc,
+        extra={"atmos": True} if joc is None and data.get("atmos") else None,
         descriptive=data.get("descriptive", False),
     )