Compare commits

...
4 Commits
Author SHA1 Message Date
imSp4rky 65e807bf5f fix(subtitle): join cues that a segment boundary splits in stpp and wvtt
Share one append_cue join across DASH fTTML, ISM and fVTT parsing. Also map "geo-block" and "geolocation" errors to GEOFENCE in the API.
2026-10-31 15:59:12 -06:00
imSp4rky c6c0331007 refactor(dl): share the cache, vault and key-label steps in prepare_drm
take_cached_key, take_vault_key and add_key_label replace the code copied across the server-CDM, Widevine, PlayReady, ClearKey and MonaLisa branches.
2026-10-31 13:28:29 -06:00
imSp4rky b3c86413d5 fix(dl): take turns on licence requests so one licence serves every track
Tracks with different KID sets still query the vaults at the same time, but wait for one licence lock and check the key cache again before they licence.
2026-10-31 12:42:43 -06:00
imSp4rky f2747076a6 fix(tracks): retry ccextractor on the repackaged file 2026-10-31 12:21:19 -06:00
10 changed files with 376 additions and 197 deletions
+1 -1
View File
@@ -52,7 +52,7 @@ format-appropriate clean-up to make a single, valid file:
`‎` / `‏` bidirectional entities.
- **Segment merging**: when a WebVTT or boxed subtitle arrives as many DASH/HLS
segments, unshackle merges the segments back into one continuous document with
correct timing.
correct timing. A cue that a segment boundary cuts in two comes out as one cue.
- **WebVTT sanitization**: merged WebVTT runs through a repair pipeline that clamps
negative timestamps to `00:00:00.000`, removes stray cue identifiers (like `Q0`, `Q1`)
that confuse downstream parsers, and merges multi-line cues that a service split into
+153
View File
@@ -0,0 +1,153 @@
"""Licences take turns across DRMs with different KIDs, so one licence that carries every key is sent once."""
from __future__ import annotations
import logging
import threading
import time
from types import SimpleNamespace
from typing import Any
from uuid import UUID
import pytest
from pywidevine.pssh import PSSH
from rich.table import Table
from rich.tree import Tree
from unshackle.commands.dl import dl
from unshackle.core.cdm.detect import cdm_type_stub
from unshackle.core.constants import DOWNLOAD_CANCELLED, DownloadCancelled
from unshackle.core.drm import Widevine
pytestmark = pytest.mark.unit
KID_A, KID_B, KID_C = (UUID(c * 32) for c in "abc")
KEYS = {KID_A: "aa" * 16, KID_B: "bb" * 16, KID_C: "cc" * 16}
class NoVaults:
sources: dict = {}
def __bool__(self) -> bool:
return False
def get_key(self, kid: UUID) -> tuple[None, None]:
return None, None
@pytest.fixture
def licences(monkeypatch: pytest.MonkeyPatch) -> list[str]:
"""Each licence returns every content key of the title, after a delay that lets the tracks overlap."""
calls: list[str] = []
def get_content_keys(self: Widevine, **_: Any) -> None:
calls.append(threading.current_thread().name)
time.sleep(0.2)
self.content_keys = dict(KEYS)
monkeypatch.setattr(Widevine, "get_content_keys", get_content_keys)
monkeypatch.setattr(dl, "LICENSE_KEY_CACHE", {})
monkeypatch.setattr(dl, "DRM_LOCKS", {})
DOWNLOAD_CANCELLED.clear()
yield calls
DOWNLOAD_CANCELLED.clear()
@pytest.fixture
def cmd() -> dl:
cmd = dl.__new__(dl)
cmd.log = logging.getLogger(__name__)
cmd.service = "TEST"
cmd.vaults = NoVaults()
cmd.vault_cache_tally = None
cmd._remote_service = None
cmd.debug_logger = None
cmd.cdm = cdm_type_stub("widevine")
return cmd
def widevine(*kids: UUID) -> Widevine:
return Widevine(pssh=PSSH.new(system_id=PSSH.SystemId.Widevine, key_ids=list(kids)))
def prepare(cmd: dl, drm: Widevine, track_kid: UUID, table: Table, **kwargs: Any) -> None:
cmd.prepare_drm(
drm,
track=SimpleNamespace(drm=[drm]),
title=None,
certificate=lambda **_: None,
licence=lambda **_: None,
table=table,
track_kid=track_kid,
**kwargs,
)
def new_table() -> Table:
table = Table.grid()
table.add_row("")
return table
def run_together(cmd: dl, *jobs: tuple[Widevine, UUID]) -> tuple[Table, list[BaseException]]:
table = new_table()
errors: list[BaseException] = []
def run(drm: Widevine, kid: UUID) -> None:
try:
prepare(cmd, drm, kid, table)
except BaseException as e:
errors.append(e)
threads = [threading.Thread(target=run, args=job, name=f"track-{i}") for i, job in enumerate(jobs)]
for t in threads:
t.start()
for t in threads:
t.join()
return table, errors
def tree_labels(table: Table, kid: UUID) -> list[str]:
trees = [cell for cell in table.columns[0].cells if isinstance(cell, Tree)]
return [str(child.label) for tree in trees for child in tree.children if kid.hex in str(child.label)]
def test_one_licence_for_two_pssh_with_different_kids(cmd: dl, licences: list[str]) -> None:
first, second = widevine(KID_A, KID_B), widevine(KID_C)
table, errors = run_together(cmd, (first, KID_A), (second, KID_C))
assert not errors
assert len(licences) == 1
assert first.content_keys[KID_A] == KEYS[KID_A] and second.content_keys[KID_C] == KEYS[KID_C]
assert any(label.endswith(" from cache") for kid in (KID_A, KID_C) for label in tree_labels(table, kid))
def test_a_track_waiting_for_its_turn_sends_no_licence_after_a_failure(cmd: dl, licences: list[str]) -> None:
DOWNLOAD_CANCELLED.set()
with pytest.raises(DownloadCancelled):
prepare(cmd, widevine(KID_C), KID_C, new_table())
assert licences == []
def test_force_licenses_again_although_the_keys_are_cached(cmd: dl, licences: list[str]) -> None:
dl.LICENSE_KEY_CACHE.update(KEYS)
prepare(cmd, widevine(KID_C), KID_C, new_table(), force=True)
assert len(licences) == 1
def test_a_key_label_is_added_once_with_its_marker_and_source() -> None:
tree = Tree("Widevine")
dl.add_key_label(tree, KID_A, KEYS[KID_A], KID_A, "from cache")
dl.add_key_label(tree, KID_A, KEYS[KID_A])
dl.add_key_label(tree, KID_B, KEYS[KID_B], KID_A)
assert [str(child.label) for child in tree.children] == [
f"[text2]{KID_A.hex}:{KEYS[KID_A]}* from cache",
f"[text2]{KID_B.hex}:{KEYS[KID_B]}",
]
+2
View File
@@ -114,6 +114,8 @@ def test_build_error_response_debug_mode_includes_traceback() -> None:
(TimeoutError("read timeout"), APIErrorCode.NETWORK_ERROR),
(Exception("Not available in your region"), APIErrorCode.GEOFENCE),
(Exception("Service is geoblocked in GB"), APIErrorCode.GEOFENCE),
(Exception("Content is geo-blocked."), APIErrorCode.GEOFENCE),
(Exception("Manifest request failed: GeoLocationBlocked"), APIErrorCode.GEOFENCE),
(
Exception("Service is not available in your region (GB)"),
APIErrorCode.GEOFENCE,
@@ -67,6 +67,7 @@ def make_cmd(vaults):
cmd.service = "TEST"
cmd.vaults = vaults
cmd.vault_cache_tally = None
cmd.debug_logger = None
cmd._remote_service = SimpleNamespace(_server_cdm=True, _server_cdm_type="widevine")
dl.LICENSE_KEY_CACHE.clear()
dl.DRM_LOCKS.clear()
@@ -120,6 +121,7 @@ def test_batch_resolved_keys_are_cached_before_download():
cmd.service = "TEST"
cmd.vaults = vaults
cmd.vault_cache_tally = None
cmd.debug_logger = None
dl.LICENSE_KEY_CACHE.clear()
drm = FakeWidevine([NEW])
+41
View File
@@ -0,0 +1,41 @@
"""Tests for the repackaged-copy retry in ``Video.ccextractor``."""
from __future__ import annotations
import subprocess
from pathlib import Path
from typing import Any
import pytest
from langcodes import Language
from unshackle.core.tracks import Video
from unshackle.core.tracks import video as video_module
def test_retry_runs_on_the_repackaged_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
original = tmp_path / "v.mp4"
original.write_bytes(b"x")
repacked = tmp_path / "v_repack.mp4"
track = Video(id_="v", url="https://example.test/v.m3u8", language="en", codec=Video.Codec.AVC)
track.path = original
def fake_repackage() -> None:
original.unlink()
repacked.write_bytes(b"x")
track.path = repacked
inputs: list[Path] = []
def fake_run(args: list[Any], **_: Any) -> None:
inputs.append(Path(args[-1]))
if not Path(args[-1]).exists():
raise subprocess.CalledProcessError(2, args) # ccextractor: no input file
raise subprocess.CalledProcessError(10, args) # ccextractor: no captions found
monkeypatch.setattr(video_module.binaries, "CCExtractor", Path("/usr/bin/ccextractor"))
monkeypatch.setattr(video_module.subprocess, "run", fake_run)
monkeypatch.setattr(track, "repackage", fake_repackage)
assert track.ccextractor("cc", tmp_path / "cc.srt", Language.get("en")) is None
assert inputs == [original, repacked]
+16 -2
View File
@@ -351,6 +351,20 @@ def test_segmented_fvtt_parses_and_converts(tmp_path, monkeypatch):
assert cue_count(out) == 2
def test_segmented_fttml_joins_a_cue_split_at_a_segment_boundary():
data = build_stpp_mp4(
ttml_fragment(("00:00:01.000", "00:00:02.000", "Carried")),
ttml_fragment(("00:00:02.000", "00:00:03.500", "Carried"), ("00:00:03.500", "00:00:04.000", "Next")),
)
captions = Subtitle.parse(data, Codec.fTTML).get_captions("en")
assert [(c.start / 1e6, c.end / 1e6, c.get_text()) for c in captions] == [
(1.0, 3.5, "Carried"),
(3.5, 4.0, "Next"),
]
def test_segmented_fttml_parses_and_converts(tmp_path, monkeypatch):
monkeypatch.setattr("unshackle.core.config.config.subtitle", {}, raising=False)
frag = (
@@ -448,8 +462,8 @@ def test_dash_fvtt_cues_lose_the_presentation_time_offset(tmp_path, monkeypatch)
assert "00:00:02.000 --> 00:00:03.000" in text
def ttml_fragment(*cues: tuple[str, str]) -> str:
body = "".join(f'<p begin="{begin}" end="{end}">Line</p>' for begin, end in cues)
def ttml_fragment(*cues: tuple[str, ...]) -> str:
body = "".join(f'<p begin="{cue[0]}" end="{cue[1]}">{cue[2] if len(cue) > 2 else "Line"}</p>' for cue in cues)
return f'<tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en"><body><div>{body}</div></body></tt>'
+139 -178
View File
@@ -14,6 +14,7 @@ from collections import Counter, defaultdict
from collections.abc import Iterable
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
from copy import deepcopy
from datetime import date, timedelta
from functools import partial
@@ -50,6 +51,7 @@ from unshackle.core.constants import (
DOWNLOAD_CANCELLED,
DOWNLOAD_LICENCE_ONLY,
AnyTrack,
DownloadCancelled,
context_settings,
)
from unshackle.core.credential import Credential
@@ -1241,6 +1243,7 @@ class dl:
DRM_LOCKS: dict[str, Lock] = {}
DRM_LOCKS_GUARD = Lock()
LICENCE_LOCK = Lock()
VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer")
EXPORT_LOCK = Lock()
LICENSE_KEY_CACHE: dict[UUID, str] = {}
@@ -4749,8 +4752,8 @@ class dl:
"""Return the lock for one DRM's set of content keys.
Tracks that share KIDs still take turns, so the second one finds the content keys in
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs send a challenge
and query the key vaults at the same time.
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs query the key vaults
at the same time; their licences take turns under ``take_licence_turn``.
Callers take the lock after get_drm_for_cdm(), which folds a track's sibling PlayReady
objects into the returned one, so the KID set is complete for that track's DRM list
@@ -4764,6 +4767,93 @@ class dl:
with cls.DRM_LOCKS_GUARD:
return cls.DRM_LOCKS.setdefault(key or type(drm).__name__, Lock())
def take_licence_turn(
self,
turn: ExitStack,
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
kids: list[UUID],
track: AnyTrack,
track_kid: Optional[UUID],
cek_tree: Optional[Tree],
) -> None:
"""Hold LICENCE_LOCK until ``turn`` closes, then take the content keys of ``kids`` an earlier licence cached.
One licence can carry the keys of a DRM with other KIDs, so licences take turns. The caller
holds its ``drm_lock`` first, so the vault lookups before this still run at the same time.
"""
turn.enter_context(self.LICENCE_LOCK)
if DOWNLOAD_CANCELLED.is_set():
raise DownloadCancelled()
for kid in kids:
if kid not in drm.content_keys:
self.take_cached_key(drm, kid, track, track_kid, cek_tree)
def take_cached_key(
self,
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
kid: UUID,
track: AnyTrack,
track_kid: Optional[UUID],
cek_tree: Optional[Tree],
) -> bool:
"""Give the DRM the content key LICENSE_KEY_CACHE holds for ``kid``, and return whether it held one."""
key = self.LICENSE_KEY_CACHE.get(kid)
if not key:
return False
drm.content_keys[kid] = key
if cek_tree is not None:
self.add_key_label(cek_tree, kid, key, track_kid, "from cache")
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={"kid": kid.hex, "content_key": key, "track": str(track), "drm_type": type(drm).__name__},
)
return True
def take_vault_key(
self,
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
kid: UUID,
track: AnyTrack,
track_kid: Optional[UUID],
cek_tree: Optional[Tree],
) -> bool:
"""Give the DRM the content key a key vault holds for ``kid``, cache it for the run, and return whether one did."""
key, vault = self.vaults.get_key(kid)
if not key:
return False
drm.content_keys[kid] = key
self.LICENSE_KEY_CACHE[kid] = key
if cek_tree is not None:
self.add_key_label(cek_tree, kid, key, track_kid, f"from {vault}")
if self.debug_logger:
self.debug_logger.log_vault_query(
vault_name=str(vault),
operation="get_key_success",
service=self.service,
context={
"kid": kid.hex,
"content_key": key,
"track": str(track),
"from_cache": True,
"drm_type": type(drm).__name__,
},
)
return True
@staticmethod
def add_key_label(tree: Tree, kid: UUID, key: str, track_kid: Optional[UUID] = None, source: str = "") -> None:
"""Add ``kid:key`` to a content key tree once, with ``*`` on the track's own KID and ``source`` after it."""
if any(f"{kid.hex}:{key}" in str(x.label) for x in tree.children):
return
label = f"[text2]{kid.hex}:{key}"
if kid == track_kid:
label += "*"
if source:
label += f" {source}"
tree.add(label)
def service_licence(self, service: Service, drm_system: Optional[str] = None, **kwargs: Any) -> Any:
"""Call the service's licence function for the track's DRM system.
@@ -4908,26 +4998,24 @@ class dl:
licensed_locally = bool(client_licensed) and str(track.id) in client_licensed
if server_cdm and not licensed_locally:
with self.drm_lock(drm):
with self.drm_lock(drm), ExitStack() as licence_turn:
pending_vault_writes: list[Callable[[], Any]] = []
vault_kids = list(getattr(drm, "kids", None) or [])
if track_kid and track_kid not in vault_kids:
vault_kids.append(track_kid)
for kid in vault_kids:
if kid in drm.content_keys:
if kid in drm.content_keys or self.take_cached_key(drm, kid, track, track_kid, None):
continue
content_key = self.LICENSE_KEY_CACHE.get(kid)
if not content_key and not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
self.LICENSE_KEY_CACHE[kid] = content_key
known_keys = set(drm.content_keys)
if not cdm_only:
self.take_vault_key(drm, kid, track, track_kid, None)
def missing_track_key() -> bool:
return not drm.content_keys or bool(track_kid and track_kid not in drm.content_keys)
if missing_track_key():
self.take_licence_turn(licence_turn, drm, vault_kids, track, track_kid, None)
known_keys = set(drm.content_keys)
if missing_track_key():
from unshackle.core.remote_service import RemoteError
@@ -5003,12 +5091,10 @@ class dl:
all_kids.append(track_kid)
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
cek_tree.add(f"[text2]{kid.hex}:{key}{is_track_kid}")
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
for kid, key in drm.content_keys.items():
if kid not in all_kids:
cek_tree.add(f"[text2]{kid.hex}:{key}")
self.add_key_label(cek_tree, kid, key)
if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells):
table.add_row(cek_tree)
if export:
@@ -5079,7 +5165,7 @@ class dl:
)
pending_vault_writes = []
with self.drm_lock(drm):
with self.drm_lock(drm), ExitStack() as licence_turn:
pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine")
cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
pre_existing_tree = next(
@@ -5095,56 +5181,16 @@ class dl:
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
if not any(f"{kid.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
continue
is_track_kid = ["", "*"][kid == track_kid]
cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid)
if cached_key:
drm.content_keys[kid] = cached_key
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
if not any(f"{kid.hex}:{cached_key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={
"kid": kid.hex,
"content_key": cached_key,
"track": str(track),
"drm_type": "Widevine",
},
)
if not force and self.take_cached_key(drm, kid, track, track_kid, cek_tree):
continue
if not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
if not any(f"{kid.hex}:{content_key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.LICENSE_KEY_CACHE[kid] = content_key
if self.debug_logger:
self.debug_logger.log_vault_query(
vault_name=str(vault_used),
operation="get_key_success",
service=self.service,
context={
"kid": kid.hex,
"content_key": content_key,
"track": str(track),
"from_cache": True,
},
)
elif vaults_only:
if cdm_only:
need_license = True
elif not self.take_vault_key(drm, kid, track, track_kid, cek_tree):
if vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
@@ -5158,14 +5204,12 @@ class dl:
)
self.flush_vault_writes(pending_vault_writes)
raise Widevine.Exceptions.CEKNotFound(msg)
else:
need_license = True
if kid not in drm.content_keys and cdm_only:
need_license = True
if need_license and all(kid in drm.content_keys for kid in all_kids):
need_license = False
if need_license:
# force licenses again, so the turn takes no cached keys
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
need_license = not all(kid in drm.content_keys for kid in all_kids)
if need_license and not vaults_only:
from_vaults = drm.content_keys.copy()
@@ -5218,10 +5262,7 @@ class dl:
for kid_, key in drm.content_keys.items():
if key == "0" * 32:
key = f"[red]{key}[/]"
is_track_kid_marker = ["", "*"][kid_ == track_kid]
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid_, key, track_kid)
drm.content_keys = {
kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key)
@@ -5272,7 +5313,7 @@ class dl:
)
pending_vault_writes = []
with self.drm_lock(drm):
with self.drm_lock(drm), ExitStack() as licence_turn:
pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady")
cek_tree = Tree(
Text.assemble(
@@ -5294,57 +5335,16 @@ class dl:
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
if not any(f"{kid.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
continue
is_track_kid = ["", "*"][kid == track_kid]
cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid)
if cached_key:
drm.content_keys[kid] = cached_key
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
if not any(f"{kid.hex}:{cached_key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={
"kid": kid.hex,
"content_key": cached_key,
"track": str(track),
"drm_type": "PlayReady",
},
)
if not force and self.take_cached_key(drm, kid, track, track_kid, cek_tree):
continue
if not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
if not any(f"{kid.hex}:{content_key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.LICENSE_KEY_CACHE[kid] = content_key
if self.debug_logger:
self.debug_logger.log_vault_query(
vault_name=str(vault_used),
operation="get_key_success",
service=self.service,
context={
"kid": kid.hex,
"content_key": content_key,
"track": str(track),
"from_cache": True,
"drm_type": "PlayReady",
},
)
elif vaults_only:
if cdm_only:
need_license = True
elif not self.take_vault_key(drm, kid, track, track_kid, cek_tree):
if vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
@@ -5358,14 +5358,12 @@ class dl:
)
self.flush_vault_writes(pending_vault_writes)
raise PlayReady.Exceptions.CEKNotFound(msg)
else:
need_license = True
if kid not in drm.content_keys and cdm_only:
need_license = True
if need_license and all(kid in drm.content_keys for kid in all_kids):
need_license = False
if need_license:
# force licenses again, so the turn takes no cached keys
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
need_license = not all(kid in drm.content_keys for kid in all_kids)
if need_license and not vaults_only:
from_vaults = drm.content_keys.copy()
@@ -5398,10 +5396,7 @@ class dl:
raise e
for kid_, key in drm.content_keys.items():
is_track_kid_marker = ["", "*"][kid_ == track_kid]
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid_, key, track_kid)
drm.content_keys.update(from_vaults)
@@ -5432,7 +5427,7 @@ class dl:
elif isinstance(drm, ClearKeyCENC):
pending_vault_writes = []
with self.drm_lock(drm):
with self.drm_lock(drm), ExitStack() as licence_turn:
cek_tree = Tree(Text.assemble(("ClearKey", "cyan"), overflow="fold"))
pre_existing_tree = next(
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
@@ -5447,43 +5442,16 @@ class dl:
for kid in all_kids:
if kid in drm.content_keys:
is_track_kid = ["", "*"][kid == track_kid]
key = drm.content_keys[kid]
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
if not any(f"{kid.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
continue
is_track_kid = ["", "*"][kid == track_kid]
cached_key = self.LICENSE_KEY_CACHE.get(kid)
if cached_key:
drm.content_keys[kid] = cached_key
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
if not any(f"{kid.hex}:{cached_key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
log_event(
"license_cache_hit",
level="INFO",
service=self.service,
context={
"kid": kid.hex,
"content_key": cached_key,
"track": str(track),
"drm_type": "ClearKeyCENC",
},
)
if self.take_cached_key(drm, kid, track, track_kid, cek_tree):
continue
if not cdm_only:
content_key, vault_used = self.vaults.get_key(kid)
if content_key:
drm.content_keys[kid] = content_key
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
if not any(f"{kid.hex}:{content_key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.LICENSE_KEY_CACHE[kid] = content_key
elif vaults_only:
if cdm_only:
need_license = True
elif not self.take_vault_key(drm, kid, track, track_kid, cek_tree):
if vaults_only:
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
cek_tree.add(f"[logging.level.error]{msg}")
if not pre_existing_tree:
@@ -5497,14 +5465,12 @@ class dl:
)
self.flush_vault_writes(pending_vault_writes)
raise ClearKeyCENC.Exceptions.CEKNotFound(msg)
else:
need_license = True
if kid not in drm.content_keys and cdm_only:
need_license = True
if need_license and all(kid in drm.content_keys for kid in all_kids):
need_license = False
if need_license:
# force licenses again, so the turn takes no cached keys
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
need_license = not all(kid in drm.content_keys for kid in all_kids)
if need_license and not vaults_only:
from_vaults = drm.content_keys.copy()
@@ -5539,10 +5505,7 @@ class dl:
raise e
for kid_, key in drm.content_keys.items():
is_track_kid_marker = ["", "*"][kid_ == track_kid]
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid_, key, track_kid)
drm.content_keys.update(from_vaults)
@@ -5583,9 +5546,7 @@ class dl:
cek_tree = pre_existing_tree
for kid_, key in drm.content_keys.items():
label = f"[text2]{kid_.hex}:{key}*"
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
cek_tree.add(label)
self.add_key_label(cek_tree, kid_, key, track_kid=kid_) # every MonaLisa key is the track's own
if cek_tree.children and not pre_existing_tree:
table.add_row()
+4 -1
View File
@@ -208,7 +208,10 @@ def categorize_exception(
exc_str = f"{exc} {root}".lower()
exc_type = type(root).__name__
if any(keyword in exc_str for keyword in ["geofence", "geoblock", "region", "not available in", "territory"]):
if any(
keyword in exc_str
for keyword in ["geofence", "geoblock", "geo-block", "geolocation", "region", "not available in", "territory"]
):
return APIError(
error_code=APIErrorCode.GEOFENCE,
message=f"Content not available in your region: {redact_secrets(str(exc))}",
+13 -11
View File
@@ -376,6 +376,14 @@ class Subtitle(Track):
boxes = MP4.parse_stream(BytesIO(data))
return [box.data for box in boxes if box.type == b"mdat"]
@staticmethod
def append_cue(kept: CaptionList, caption: Caption) -> None:
"""Append caption to kept, or extend the last cue that a segment boundary cut and caption carries on."""
if kept and kept[-1].end == caption.start and kept[-1].get_text() == caption.get_text():
kept[-1].end = caption.end
else:
kept.append(caption)
@staticmethod
def first_cue_start(caption_set: pycaption.CaptionSet) -> Optional[int]:
"""The start of the first cue, in microseconds, looking past empty languages; None with no cues."""
@@ -413,10 +421,8 @@ class Subtitle(Track):
for caption in fragment.get_captions(language):
caption.start = max(caption.start + shift, 0)
caption.end += shift
if kept and kept[-1].end == caption.start and kept[-1].get_text() == caption.get_text():
kept[-1].end = caption.end # a cue carried into the next fragment
elif caption.end > 0:
kept.append(caption)
if caption.end > 0:
Subtitle.append_cue(kept, caption)
if not any(captions.values()):
logging.getLogger("Subtitle").debug(f"ISM text: {len(payloads)} fragments, {len(times)} fragment times")
self.convert(Subtitle.Codec.TimedTextMarkupLang)
@@ -837,7 +843,8 @@ class Subtitle(Track):
for payload in Subtitle.mdat_payloads(data)
):
for lang in segment.get_languages():
caption_lists[lang].extend(segment.get_captions(lang))
for caption in segment.get_captions(lang):
Subtitle.append_cue(caption_lists[lang], caption)
caption_set = pycaption.CaptionSet(caption_lists)
elif codec == Subtitle.Codec.TimedTextMarkupLang:
text = try_ensure_utf8(data).decode("utf8")
@@ -1109,12 +1116,7 @@ class Subtitle(Track):
nodes=nodes,
layout_info=layout,
)
p_caption = captions[-1] if captions else None
if p_caption and caption.start == p_caption.end and str(caption.nodes) == str(p_caption.nodes):
# it's a duplicate, but lets take its end time
p_caption.end = caption.end
continue
captions.append(caption)
Subtitle.append_cue(captions, caption)
return captions, language
+5 -4
View File
@@ -655,9 +655,9 @@ class Video(Track):
raise EnvironmentError("ccextractor executable was not found.")
out_path = Path(out_path)
video_path, ccextractor = self.path, binaries.CCExtractor
ccextractor = binaries.CCExtractor
def run_ccextractor(fallback: str) -> bool:
def run_ccextractor(video_path: Path, fallback: str) -> bool:
cc_start = time.monotonic()
try:
subprocess.run(
@@ -692,9 +692,10 @@ class Video(Track):
# Try on the original file first (preserves container-level CC data like c608 boxes),
# then fall back to repacked file (ccextractor can fail on some container formats).
if not run_ccextractor("retrying on a repackaged copy"):
if not run_ccextractor(self.path, "retrying on a repackaged copy"):
# repackage() deletes the original file and moves self.path to the repacked copy
self.repackage()
run_ccextractor("skipping CC extraction")
run_ccextractor(self.path, "skipping CC extraction")
if out_path.exists():
cc_track = Subtitle(