mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-19 07:25:11 +00:00
Compare commits
4
Commits
cbeff379fb
...
65e807bf5f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65e807bf5f | ||
|
|
c6c0331007 | ||
|
|
b3c86413d5 | ||
|
|
f2747076a6 |
@@ -52,7 +52,7 @@ format-appropriate clean-up to make a single, valid file:
|
||||
`‎` / `‏` bidirectional entities.
|
||||
- **Segment merging**: when a WebVTT or boxed subtitle arrives as many DASH/HLS
|
||||
segments, unshackle merges the segments back into one continuous document with
|
||||
correct timing.
|
||||
correct timing. A cue that a segment boundary cuts in two comes out as one cue.
|
||||
- **WebVTT sanitization**: merged WebVTT runs through a repair pipeline that clamps
|
||||
negative timestamps to `00:00:00.000`, removes stray cue identifiers (like `Q0`, `Q1`)
|
||||
that confuse downstream parsers, and merges multi-line cues that a service split into
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Licences take turns across DRMs with different KIDs, so one licence that carries every key is sent once."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from pywidevine.pssh import PSSH
|
||||
from rich.table import Table
|
||||
from rich.tree import Tree
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
from unshackle.core.cdm.detect import cdm_type_stub
|
||||
from unshackle.core.constants import DOWNLOAD_CANCELLED, DownloadCancelled
|
||||
from unshackle.core.drm import Widevine
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
KID_A, KID_B, KID_C = (UUID(c * 32) for c in "abc")
|
||||
KEYS = {KID_A: "aa" * 16, KID_B: "bb" * 16, KID_C: "cc" * 16}
|
||||
|
||||
|
||||
class NoVaults:
|
||||
sources: dict = {}
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_key(self, kid: UUID) -> tuple[None, None]:
|
||||
return None, None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def licences(monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
"""Each licence returns every content key of the title, after a delay that lets the tracks overlap."""
|
||||
calls: list[str] = []
|
||||
|
||||
def get_content_keys(self: Widevine, **_: Any) -> None:
|
||||
calls.append(threading.current_thread().name)
|
||||
time.sleep(0.2)
|
||||
self.content_keys = dict(KEYS)
|
||||
|
||||
monkeypatch.setattr(Widevine, "get_content_keys", get_content_keys)
|
||||
monkeypatch.setattr(dl, "LICENSE_KEY_CACHE", {})
|
||||
monkeypatch.setattr(dl, "DRM_LOCKS", {})
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
yield calls
|
||||
DOWNLOAD_CANCELLED.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cmd() -> dl:
|
||||
cmd = dl.__new__(dl)
|
||||
cmd.log = logging.getLogger(__name__)
|
||||
cmd.service = "TEST"
|
||||
cmd.vaults = NoVaults()
|
||||
cmd.vault_cache_tally = None
|
||||
cmd._remote_service = None
|
||||
cmd.debug_logger = None
|
||||
cmd.cdm = cdm_type_stub("widevine")
|
||||
return cmd
|
||||
|
||||
|
||||
def widevine(*kids: UUID) -> Widevine:
|
||||
return Widevine(pssh=PSSH.new(system_id=PSSH.SystemId.Widevine, key_ids=list(kids)))
|
||||
|
||||
|
||||
def prepare(cmd: dl, drm: Widevine, track_kid: UUID, table: Table, **kwargs: Any) -> None:
|
||||
cmd.prepare_drm(
|
||||
drm,
|
||||
track=SimpleNamespace(drm=[drm]),
|
||||
title=None,
|
||||
certificate=lambda **_: None,
|
||||
licence=lambda **_: None,
|
||||
table=table,
|
||||
track_kid=track_kid,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def new_table() -> Table:
|
||||
table = Table.grid()
|
||||
table.add_row("")
|
||||
return table
|
||||
|
||||
|
||||
def run_together(cmd: dl, *jobs: tuple[Widevine, UUID]) -> tuple[Table, list[BaseException]]:
|
||||
table = new_table()
|
||||
errors: list[BaseException] = []
|
||||
|
||||
def run(drm: Widevine, kid: UUID) -> None:
|
||||
try:
|
||||
prepare(cmd, drm, kid, table)
|
||||
except BaseException as e:
|
||||
errors.append(e)
|
||||
|
||||
threads = [threading.Thread(target=run, args=job, name=f"track-{i}") for i, job in enumerate(jobs)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return table, errors
|
||||
|
||||
|
||||
def tree_labels(table: Table, kid: UUID) -> list[str]:
|
||||
trees = [cell for cell in table.columns[0].cells if isinstance(cell, Tree)]
|
||||
return [str(child.label) for tree in trees for child in tree.children if kid.hex in str(child.label)]
|
||||
|
||||
|
||||
def test_one_licence_for_two_pssh_with_different_kids(cmd: dl, licences: list[str]) -> None:
|
||||
first, second = widevine(KID_A, KID_B), widevine(KID_C)
|
||||
|
||||
table, errors = run_together(cmd, (first, KID_A), (second, KID_C))
|
||||
|
||||
assert not errors
|
||||
assert len(licences) == 1
|
||||
assert first.content_keys[KID_A] == KEYS[KID_A] and second.content_keys[KID_C] == KEYS[KID_C]
|
||||
assert any(label.endswith(" from cache") for kid in (KID_A, KID_C) for label in tree_labels(table, kid))
|
||||
|
||||
|
||||
def test_a_track_waiting_for_its_turn_sends_no_licence_after_a_failure(cmd: dl, licences: list[str]) -> None:
|
||||
DOWNLOAD_CANCELLED.set()
|
||||
|
||||
with pytest.raises(DownloadCancelled):
|
||||
prepare(cmd, widevine(KID_C), KID_C, new_table())
|
||||
|
||||
assert licences == []
|
||||
|
||||
|
||||
def test_force_licenses_again_although_the_keys_are_cached(cmd: dl, licences: list[str]) -> None:
|
||||
dl.LICENSE_KEY_CACHE.update(KEYS)
|
||||
|
||||
prepare(cmd, widevine(KID_C), KID_C, new_table(), force=True)
|
||||
|
||||
assert len(licences) == 1
|
||||
|
||||
|
||||
def test_a_key_label_is_added_once_with_its_marker_and_source() -> None:
|
||||
tree = Tree("Widevine")
|
||||
|
||||
dl.add_key_label(tree, KID_A, KEYS[KID_A], KID_A, "from cache")
|
||||
dl.add_key_label(tree, KID_A, KEYS[KID_A])
|
||||
dl.add_key_label(tree, KID_B, KEYS[KID_B], KID_A)
|
||||
|
||||
assert [str(child.label) for child in tree.children] == [
|
||||
f"[text2]{KID_A.hex}:{KEYS[KID_A]}* from cache",
|
||||
f"[text2]{KID_B.hex}:{KEYS[KID_B]}",
|
||||
]
|
||||
@@ -114,6 +114,8 @@ def test_build_error_response_debug_mode_includes_traceback() -> None:
|
||||
(TimeoutError("read timeout"), APIErrorCode.NETWORK_ERROR),
|
||||
(Exception("Not available in your region"), APIErrorCode.GEOFENCE),
|
||||
(Exception("Service is geoblocked in GB"), APIErrorCode.GEOFENCE),
|
||||
(Exception("Content is geo-blocked."), APIErrorCode.GEOFENCE),
|
||||
(Exception("Manifest request failed: GeoLocationBlocked"), APIErrorCode.GEOFENCE),
|
||||
(
|
||||
Exception("Service is not available in your region (GB)"),
|
||||
APIErrorCode.GEOFENCE,
|
||||
|
||||
@@ -67,6 +67,7 @@ def make_cmd(vaults):
|
||||
cmd.service = "TEST"
|
||||
cmd.vaults = vaults
|
||||
cmd.vault_cache_tally = None
|
||||
cmd.debug_logger = None
|
||||
cmd._remote_service = SimpleNamespace(_server_cdm=True, _server_cdm_type="widevine")
|
||||
dl.LICENSE_KEY_CACHE.clear()
|
||||
dl.DRM_LOCKS.clear()
|
||||
@@ -120,6 +121,7 @@ def test_batch_resolved_keys_are_cached_before_download():
|
||||
cmd.service = "TEST"
|
||||
cmd.vaults = vaults
|
||||
cmd.vault_cache_tally = None
|
||||
cmd.debug_logger = None
|
||||
dl.LICENSE_KEY_CACHE.clear()
|
||||
|
||||
drm = FakeWidevine([NEW])
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Tests for the repackaged-copy retry in ``Video.ccextractor``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from langcodes import Language
|
||||
|
||||
from unshackle.core.tracks import Video
|
||||
from unshackle.core.tracks import video as video_module
|
||||
|
||||
|
||||
def test_retry_runs_on_the_repackaged_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
original = tmp_path / "v.mp4"
|
||||
original.write_bytes(b"x")
|
||||
repacked = tmp_path / "v_repack.mp4"
|
||||
track = Video(id_="v", url="https://example.test/v.m3u8", language="en", codec=Video.Codec.AVC)
|
||||
track.path = original
|
||||
|
||||
def fake_repackage() -> None:
|
||||
original.unlink()
|
||||
repacked.write_bytes(b"x")
|
||||
track.path = repacked
|
||||
|
||||
inputs: list[Path] = []
|
||||
|
||||
def fake_run(args: list[Any], **_: Any) -> None:
|
||||
inputs.append(Path(args[-1]))
|
||||
if not Path(args[-1]).exists():
|
||||
raise subprocess.CalledProcessError(2, args) # ccextractor: no input file
|
||||
raise subprocess.CalledProcessError(10, args) # ccextractor: no captions found
|
||||
|
||||
monkeypatch.setattr(video_module.binaries, "CCExtractor", Path("/usr/bin/ccextractor"))
|
||||
monkeypatch.setattr(video_module.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(track, "repackage", fake_repackage)
|
||||
|
||||
assert track.ccextractor("cc", tmp_path / "cc.srt", Language.get("en")) is None
|
||||
assert inputs == [original, repacked]
|
||||
@@ -351,6 +351,20 @@ def test_segmented_fvtt_parses_and_converts(tmp_path, monkeypatch):
|
||||
assert cue_count(out) == 2
|
||||
|
||||
|
||||
def test_segmented_fttml_joins_a_cue_split_at_a_segment_boundary():
|
||||
data = build_stpp_mp4(
|
||||
ttml_fragment(("00:00:01.000", "00:00:02.000", "Carried")),
|
||||
ttml_fragment(("00:00:02.000", "00:00:03.500", "Carried"), ("00:00:03.500", "00:00:04.000", "Next")),
|
||||
)
|
||||
|
||||
captions = Subtitle.parse(data, Codec.fTTML).get_captions("en")
|
||||
|
||||
assert [(c.start / 1e6, c.end / 1e6, c.get_text()) for c in captions] == [
|
||||
(1.0, 3.5, "Carried"),
|
||||
(3.5, 4.0, "Next"),
|
||||
]
|
||||
|
||||
|
||||
def test_segmented_fttml_parses_and_converts(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("unshackle.core.config.config.subtitle", {}, raising=False)
|
||||
frag = (
|
||||
@@ -448,8 +462,8 @@ def test_dash_fvtt_cues_lose_the_presentation_time_offset(tmp_path, monkeypatch)
|
||||
assert "00:00:02.000 --> 00:00:03.000" in text
|
||||
|
||||
|
||||
def ttml_fragment(*cues: tuple[str, str]) -> str:
|
||||
body = "".join(f'<p begin="{begin}" end="{end}">Line</p>' for begin, end in cues)
|
||||
def ttml_fragment(*cues: tuple[str, ...]) -> str:
|
||||
body = "".join(f'<p begin="{cue[0]}" end="{cue[1]}">{cue[2] if len(cue) > 2 else "Line"}</p>' for cue in cues)
|
||||
return f'<tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en"><body><div>{body}</div></body></tt>'
|
||||
|
||||
|
||||
|
||||
+139
-178
@@ -14,6 +14,7 @@ from collections import Counter, defaultdict
|
||||
from collections.abc import Iterable
|
||||
from concurrent import futures
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from copy import deepcopy
|
||||
from datetime import date, timedelta
|
||||
from functools import partial
|
||||
@@ -50,6 +51,7 @@ from unshackle.core.constants import (
|
||||
DOWNLOAD_CANCELLED,
|
||||
DOWNLOAD_LICENCE_ONLY,
|
||||
AnyTrack,
|
||||
DownloadCancelled,
|
||||
context_settings,
|
||||
)
|
||||
from unshackle.core.credential import Credential
|
||||
@@ -1241,6 +1243,7 @@ class dl:
|
||||
|
||||
DRM_LOCKS: dict[str, Lock] = {}
|
||||
DRM_LOCKS_GUARD = Lock()
|
||||
LICENCE_LOCK = Lock()
|
||||
VAULT_WRITER = ThreadPoolExecutor(1, thread_name_prefix="vault-writer")
|
||||
EXPORT_LOCK = Lock()
|
||||
LICENSE_KEY_CACHE: dict[UUID, str] = {}
|
||||
@@ -4749,8 +4752,8 @@ class dl:
|
||||
"""Return the lock for one DRM's set of content keys.
|
||||
|
||||
Tracks that share KIDs still take turns, so the second one finds the content keys in
|
||||
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs send a challenge
|
||||
and query the key vaults at the same time.
|
||||
LICENSE_KEY_CACHE and skips the challenge. Tracks with different KIDs query the key vaults
|
||||
at the same time; their licences take turns under ``take_licence_turn``.
|
||||
|
||||
Callers take the lock after get_drm_for_cdm(), which folds a track's sibling PlayReady
|
||||
objects into the returned one, so the KID set is complete for that track's DRM list
|
||||
@@ -4764,6 +4767,93 @@ class dl:
|
||||
with cls.DRM_LOCKS_GUARD:
|
||||
return cls.DRM_LOCKS.setdefault(key or type(drm).__name__, Lock())
|
||||
|
||||
def take_licence_turn(
|
||||
self,
|
||||
turn: ExitStack,
|
||||
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
||||
kids: list[UUID],
|
||||
track: AnyTrack,
|
||||
track_kid: Optional[UUID],
|
||||
cek_tree: Optional[Tree],
|
||||
) -> None:
|
||||
"""Hold LICENCE_LOCK until ``turn`` closes, then take the content keys of ``kids`` an earlier licence cached.
|
||||
|
||||
One licence can carry the keys of a DRM with other KIDs, so licences take turns. The caller
|
||||
holds its ``drm_lock`` first, so the vault lookups before this still run at the same time.
|
||||
"""
|
||||
turn.enter_context(self.LICENCE_LOCK)
|
||||
if DOWNLOAD_CANCELLED.is_set():
|
||||
raise DownloadCancelled()
|
||||
for kid in kids:
|
||||
if kid not in drm.content_keys:
|
||||
self.take_cached_key(drm, kid, track, track_kid, cek_tree)
|
||||
|
||||
def take_cached_key(
|
||||
self,
|
||||
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
||||
kid: UUID,
|
||||
track: AnyTrack,
|
||||
track_kid: Optional[UUID],
|
||||
cek_tree: Optional[Tree],
|
||||
) -> bool:
|
||||
"""Give the DRM the content key LICENSE_KEY_CACHE holds for ``kid``, and return whether it held one."""
|
||||
key = self.LICENSE_KEY_CACHE.get(kid)
|
||||
if not key:
|
||||
return False
|
||||
drm.content_keys[kid] = key
|
||||
if cek_tree is not None:
|
||||
self.add_key_label(cek_tree, kid, key, track_kid, "from cache")
|
||||
log_event(
|
||||
"license_cache_hit",
|
||||
level="INFO",
|
||||
service=self.service,
|
||||
context={"kid": kid.hex, "content_key": key, "track": str(track), "drm_type": type(drm).__name__},
|
||||
)
|
||||
return True
|
||||
|
||||
def take_vault_key(
|
||||
self,
|
||||
drm: Union[ClearKeyCENC, Widevine, PlayReady, MonaLisa],
|
||||
kid: UUID,
|
||||
track: AnyTrack,
|
||||
track_kid: Optional[UUID],
|
||||
cek_tree: Optional[Tree],
|
||||
) -> bool:
|
||||
"""Give the DRM the content key a key vault holds for ``kid``, cache it for the run, and return whether one did."""
|
||||
key, vault = self.vaults.get_key(kid)
|
||||
if not key:
|
||||
return False
|
||||
drm.content_keys[kid] = key
|
||||
self.LICENSE_KEY_CACHE[kid] = key
|
||||
if cek_tree is not None:
|
||||
self.add_key_label(cek_tree, kid, key, track_kid, f"from {vault}")
|
||||
if self.debug_logger:
|
||||
self.debug_logger.log_vault_query(
|
||||
vault_name=str(vault),
|
||||
operation="get_key_success",
|
||||
service=self.service,
|
||||
context={
|
||||
"kid": kid.hex,
|
||||
"content_key": key,
|
||||
"track": str(track),
|
||||
"from_cache": True,
|
||||
"drm_type": type(drm).__name__,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def add_key_label(tree: Tree, kid: UUID, key: str, track_kid: Optional[UUID] = None, source: str = "") -> None:
|
||||
"""Add ``kid:key`` to a content key tree once, with ``*`` on the track's own KID and ``source`` after it."""
|
||||
if any(f"{kid.hex}:{key}" in str(x.label) for x in tree.children):
|
||||
return
|
||||
label = f"[text2]{kid.hex}:{key}"
|
||||
if kid == track_kid:
|
||||
label += "*"
|
||||
if source:
|
||||
label += f" {source}"
|
||||
tree.add(label)
|
||||
|
||||
def service_licence(self, service: Service, drm_system: Optional[str] = None, **kwargs: Any) -> Any:
|
||||
"""Call the service's licence function for the track's DRM system.
|
||||
|
||||
@@ -4908,26 +4998,24 @@ class dl:
|
||||
licensed_locally = bool(client_licensed) and str(track.id) in client_licensed
|
||||
|
||||
if server_cdm and not licensed_locally:
|
||||
with self.drm_lock(drm):
|
||||
with self.drm_lock(drm), ExitStack() as licence_turn:
|
||||
pending_vault_writes: list[Callable[[], Any]] = []
|
||||
vault_kids = list(getattr(drm, "kids", None) or [])
|
||||
if track_kid and track_kid not in vault_kids:
|
||||
vault_kids.append(track_kid)
|
||||
for kid in vault_kids:
|
||||
if kid in drm.content_keys:
|
||||
if kid in drm.content_keys or self.take_cached_key(drm, kid, track, track_kid, None):
|
||||
continue
|
||||
content_key = self.LICENSE_KEY_CACHE.get(kid)
|
||||
if not content_key and not cdm_only:
|
||||
content_key, vault_used = self.vaults.get_key(kid)
|
||||
if content_key:
|
||||
drm.content_keys[kid] = content_key
|
||||
self.LICENSE_KEY_CACHE[kid] = content_key
|
||||
|
||||
known_keys = set(drm.content_keys)
|
||||
if not cdm_only:
|
||||
self.take_vault_key(drm, kid, track, track_kid, None)
|
||||
|
||||
def missing_track_key() -> bool:
|
||||
return not drm.content_keys or bool(track_kid and track_kid not in drm.content_keys)
|
||||
|
||||
if missing_track_key():
|
||||
self.take_licence_turn(licence_turn, drm, vault_kids, track, track_kid, None)
|
||||
known_keys = set(drm.content_keys)
|
||||
|
||||
if missing_track_key():
|
||||
from unshackle.core.remote_service import RemoteError
|
||||
|
||||
@@ -5003,12 +5091,10 @@ class dl:
|
||||
all_kids.append(track_kid)
|
||||
for kid in all_kids:
|
||||
if kid in drm.content_keys:
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
key = drm.content_keys[kid]
|
||||
cek_tree.add(f"[text2]{kid.hex}:{key}{is_track_kid}")
|
||||
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
|
||||
for kid, key in drm.content_keys.items():
|
||||
if kid not in all_kids:
|
||||
cek_tree.add(f"[text2]{kid.hex}:{key}")
|
||||
self.add_key_label(cek_tree, kid, key)
|
||||
if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells):
|
||||
table.add_row(cek_tree)
|
||||
if export:
|
||||
@@ -5079,7 +5165,7 @@ class dl:
|
||||
)
|
||||
|
||||
pending_vault_writes = []
|
||||
with self.drm_lock(drm):
|
||||
with self.drm_lock(drm), ExitStack() as licence_turn:
|
||||
pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine")
|
||||
cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
|
||||
pre_existing_tree = next(
|
||||
@@ -5095,56 +5181,16 @@ class dl:
|
||||
|
||||
for kid in all_kids:
|
||||
if kid in drm.content_keys:
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
key = drm.content_keys[kid]
|
||||
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
|
||||
if not any(f"{kid.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
|
||||
continue
|
||||
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
|
||||
cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid)
|
||||
if cached_key:
|
||||
drm.content_keys[kid] = cached_key
|
||||
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
|
||||
if not any(f"{kid.hex}:{cached_key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
log_event(
|
||||
"license_cache_hit",
|
||||
level="INFO",
|
||||
service=self.service,
|
||||
context={
|
||||
"kid": kid.hex,
|
||||
"content_key": cached_key,
|
||||
"track": str(track),
|
||||
"drm_type": "Widevine",
|
||||
},
|
||||
)
|
||||
if not force and self.take_cached_key(drm, kid, track, track_kid, cek_tree):
|
||||
continue
|
||||
|
||||
if not cdm_only:
|
||||
content_key, vault_used = self.vaults.get_key(kid)
|
||||
if content_key:
|
||||
drm.content_keys[kid] = content_key
|
||||
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
|
||||
if not any(f"{kid.hex}:{content_key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.LICENSE_KEY_CACHE[kid] = content_key
|
||||
|
||||
if self.debug_logger:
|
||||
self.debug_logger.log_vault_query(
|
||||
vault_name=str(vault_used),
|
||||
operation="get_key_success",
|
||||
service=self.service,
|
||||
context={
|
||||
"kid": kid.hex,
|
||||
"content_key": content_key,
|
||||
"track": str(track),
|
||||
"from_cache": True,
|
||||
},
|
||||
)
|
||||
elif vaults_only:
|
||||
if cdm_only:
|
||||
need_license = True
|
||||
elif not self.take_vault_key(drm, kid, track, track_kid, cek_tree):
|
||||
if vaults_only:
|
||||
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
|
||||
cek_tree.add(f"[logging.level.error]{msg}")
|
||||
if not pre_existing_tree:
|
||||
@@ -5158,14 +5204,12 @@ class dl:
|
||||
)
|
||||
self.flush_vault_writes(pending_vault_writes)
|
||||
raise Widevine.Exceptions.CEKNotFound(msg)
|
||||
else:
|
||||
need_license = True
|
||||
|
||||
if kid not in drm.content_keys and cdm_only:
|
||||
need_license = True
|
||||
|
||||
if need_license and all(kid in drm.content_keys for kid in all_kids):
|
||||
need_license = False
|
||||
if need_license:
|
||||
# force licenses again, so the turn takes no cached keys
|
||||
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
|
||||
need_license = not all(kid in drm.content_keys for kid in all_kids)
|
||||
|
||||
if need_license and not vaults_only:
|
||||
from_vaults = drm.content_keys.copy()
|
||||
@@ -5218,10 +5262,7 @@ class dl:
|
||||
for kid_, key in drm.content_keys.items():
|
||||
if key == "0" * 32:
|
||||
key = f"[red]{key}[/]"
|
||||
is_track_kid_marker = ["", "*"][kid_ == track_kid]
|
||||
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
|
||||
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid_, key, track_kid)
|
||||
|
||||
drm.content_keys = {
|
||||
kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key)
|
||||
@@ -5272,7 +5313,7 @@ class dl:
|
||||
)
|
||||
|
||||
pending_vault_writes = []
|
||||
with self.drm_lock(drm):
|
||||
with self.drm_lock(drm), ExitStack() as licence_turn:
|
||||
pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady")
|
||||
cek_tree = Tree(
|
||||
Text.assemble(
|
||||
@@ -5294,57 +5335,16 @@ class dl:
|
||||
|
||||
for kid in all_kids:
|
||||
if kid in drm.content_keys:
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
key = drm.content_keys[kid]
|
||||
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
|
||||
if not any(f"{kid.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
|
||||
continue
|
||||
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
|
||||
cached_key = None if force else self.LICENSE_KEY_CACHE.get(kid)
|
||||
if cached_key:
|
||||
drm.content_keys[kid] = cached_key
|
||||
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
|
||||
if not any(f"{kid.hex}:{cached_key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
log_event(
|
||||
"license_cache_hit",
|
||||
level="INFO",
|
||||
service=self.service,
|
||||
context={
|
||||
"kid": kid.hex,
|
||||
"content_key": cached_key,
|
||||
"track": str(track),
|
||||
"drm_type": "PlayReady",
|
||||
},
|
||||
)
|
||||
if not force and self.take_cached_key(drm, kid, track, track_kid, cek_tree):
|
||||
continue
|
||||
|
||||
if not cdm_only:
|
||||
content_key, vault_used = self.vaults.get_key(kid)
|
||||
if content_key:
|
||||
drm.content_keys[kid] = content_key
|
||||
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
|
||||
if not any(f"{kid.hex}:{content_key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.LICENSE_KEY_CACHE[kid] = content_key
|
||||
|
||||
if self.debug_logger:
|
||||
self.debug_logger.log_vault_query(
|
||||
vault_name=str(vault_used),
|
||||
operation="get_key_success",
|
||||
service=self.service,
|
||||
context={
|
||||
"kid": kid.hex,
|
||||
"content_key": content_key,
|
||||
"track": str(track),
|
||||
"from_cache": True,
|
||||
"drm_type": "PlayReady",
|
||||
},
|
||||
)
|
||||
elif vaults_only:
|
||||
if cdm_only:
|
||||
need_license = True
|
||||
elif not self.take_vault_key(drm, kid, track, track_kid, cek_tree):
|
||||
if vaults_only:
|
||||
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
|
||||
cek_tree.add(f"[logging.level.error]{msg}")
|
||||
if not pre_existing_tree:
|
||||
@@ -5358,14 +5358,12 @@ class dl:
|
||||
)
|
||||
self.flush_vault_writes(pending_vault_writes)
|
||||
raise PlayReady.Exceptions.CEKNotFound(msg)
|
||||
else:
|
||||
need_license = True
|
||||
|
||||
if kid not in drm.content_keys and cdm_only:
|
||||
need_license = True
|
||||
|
||||
if need_license and all(kid in drm.content_keys for kid in all_kids):
|
||||
need_license = False
|
||||
if need_license:
|
||||
# force licenses again, so the turn takes no cached keys
|
||||
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
|
||||
need_license = not all(kid in drm.content_keys for kid in all_kids)
|
||||
|
||||
if need_license and not vaults_only:
|
||||
from_vaults = drm.content_keys.copy()
|
||||
@@ -5398,10 +5396,7 @@ class dl:
|
||||
raise e
|
||||
|
||||
for kid_, key in drm.content_keys.items():
|
||||
is_track_kid_marker = ["", "*"][kid_ == track_kid]
|
||||
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
|
||||
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid_, key, track_kid)
|
||||
|
||||
drm.content_keys.update(from_vaults)
|
||||
|
||||
@@ -5432,7 +5427,7 @@ class dl:
|
||||
|
||||
elif isinstance(drm, ClearKeyCENC):
|
||||
pending_vault_writes = []
|
||||
with self.drm_lock(drm):
|
||||
with self.drm_lock(drm), ExitStack() as licence_turn:
|
||||
cek_tree = Tree(Text.assemble(("ClearKey", "cyan"), overflow="fold"))
|
||||
pre_existing_tree = next(
|
||||
(x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
|
||||
@@ -5447,43 +5442,16 @@ class dl:
|
||||
|
||||
for kid in all_kids:
|
||||
if kid in drm.content_keys:
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
key = drm.content_keys[kid]
|
||||
label = f"[text2]{kid.hex}:{key}{is_track_kid}"
|
||||
if not any(f"{kid.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid, drm.content_keys[kid], track_kid)
|
||||
continue
|
||||
|
||||
is_track_kid = ["", "*"][kid == track_kid]
|
||||
|
||||
cached_key = self.LICENSE_KEY_CACHE.get(kid)
|
||||
if cached_key:
|
||||
drm.content_keys[kid] = cached_key
|
||||
label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache"
|
||||
if not any(f"{kid.hex}:{cached_key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
log_event(
|
||||
"license_cache_hit",
|
||||
level="INFO",
|
||||
service=self.service,
|
||||
context={
|
||||
"kid": kid.hex,
|
||||
"content_key": cached_key,
|
||||
"track": str(track),
|
||||
"drm_type": "ClearKeyCENC",
|
||||
},
|
||||
)
|
||||
if self.take_cached_key(drm, kid, track, track_kid, cek_tree):
|
||||
continue
|
||||
|
||||
if not cdm_only:
|
||||
content_key, vault_used = self.vaults.get_key(kid)
|
||||
if content_key:
|
||||
drm.content_keys[kid] = content_key
|
||||
label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}"
|
||||
if not any(f"{kid.hex}:{content_key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.LICENSE_KEY_CACHE[kid] = content_key
|
||||
elif vaults_only:
|
||||
if cdm_only:
|
||||
need_license = True
|
||||
elif not self.take_vault_key(drm, kid, track, track_kid, cek_tree):
|
||||
if vaults_only:
|
||||
msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used"
|
||||
cek_tree.add(f"[logging.level.error]{msg}")
|
||||
if not pre_existing_tree:
|
||||
@@ -5497,14 +5465,12 @@ class dl:
|
||||
)
|
||||
self.flush_vault_writes(pending_vault_writes)
|
||||
raise ClearKeyCENC.Exceptions.CEKNotFound(msg)
|
||||
else:
|
||||
need_license = True
|
||||
|
||||
if kid not in drm.content_keys and cdm_only:
|
||||
need_license = True
|
||||
|
||||
if need_license and all(kid in drm.content_keys for kid in all_kids):
|
||||
need_license = False
|
||||
if need_license:
|
||||
# force licenses again, so the turn takes no cached keys
|
||||
self.take_licence_turn(licence_turn, drm, [] if force else all_kids, track, track_kid, cek_tree)
|
||||
need_license = not all(kid in drm.content_keys for kid in all_kids)
|
||||
|
||||
if need_license and not vaults_only:
|
||||
from_vaults = drm.content_keys.copy()
|
||||
@@ -5539,10 +5505,7 @@ class dl:
|
||||
raise e
|
||||
|
||||
for kid_, key in drm.content_keys.items():
|
||||
is_track_kid_marker = ["", "*"][kid_ == track_kid]
|
||||
label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}"
|
||||
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid_, key, track_kid)
|
||||
|
||||
drm.content_keys.update(from_vaults)
|
||||
|
||||
@@ -5583,9 +5546,7 @@ class dl:
|
||||
cek_tree = pre_existing_tree
|
||||
|
||||
for kid_, key in drm.content_keys.items():
|
||||
label = f"[text2]{kid_.hex}:{key}*"
|
||||
if not any(f"{kid_.hex}:{key}" in str(x.label) for x in cek_tree.children):
|
||||
cek_tree.add(label)
|
||||
self.add_key_label(cek_tree, kid_, key, track_kid=kid_) # every MonaLisa key is the track's own
|
||||
|
||||
if cek_tree.children and not pre_existing_tree:
|
||||
table.add_row()
|
||||
|
||||
@@ -208,7 +208,10 @@ def categorize_exception(
|
||||
exc_str = f"{exc} {root}".lower()
|
||||
exc_type = type(root).__name__
|
||||
|
||||
if any(keyword in exc_str for keyword in ["geofence", "geoblock", "region", "not available in", "territory"]):
|
||||
if any(
|
||||
keyword in exc_str
|
||||
for keyword in ["geofence", "geoblock", "geo-block", "geolocation", "region", "not available in", "territory"]
|
||||
):
|
||||
return APIError(
|
||||
error_code=APIErrorCode.GEOFENCE,
|
||||
message=f"Content not available in your region: {redact_secrets(str(exc))}",
|
||||
|
||||
@@ -376,6 +376,14 @@ class Subtitle(Track):
|
||||
boxes = MP4.parse_stream(BytesIO(data))
|
||||
return [box.data for box in boxes if box.type == b"mdat"]
|
||||
|
||||
@staticmethod
|
||||
def append_cue(kept: CaptionList, caption: Caption) -> None:
|
||||
"""Append caption to kept, or extend the last cue that a segment boundary cut and caption carries on."""
|
||||
if kept and kept[-1].end == caption.start and kept[-1].get_text() == caption.get_text():
|
||||
kept[-1].end = caption.end
|
||||
else:
|
||||
kept.append(caption)
|
||||
|
||||
@staticmethod
|
||||
def first_cue_start(caption_set: pycaption.CaptionSet) -> Optional[int]:
|
||||
"""The start of the first cue, in microseconds, looking past empty languages; None with no cues."""
|
||||
@@ -413,10 +421,8 @@ class Subtitle(Track):
|
||||
for caption in fragment.get_captions(language):
|
||||
caption.start = max(caption.start + shift, 0)
|
||||
caption.end += shift
|
||||
if kept and kept[-1].end == caption.start and kept[-1].get_text() == caption.get_text():
|
||||
kept[-1].end = caption.end # a cue carried into the next fragment
|
||||
elif caption.end > 0:
|
||||
kept.append(caption)
|
||||
if caption.end > 0:
|
||||
Subtitle.append_cue(kept, caption)
|
||||
if not any(captions.values()):
|
||||
logging.getLogger("Subtitle").debug(f"ISM text: {len(payloads)} fragments, {len(times)} fragment times")
|
||||
self.convert(Subtitle.Codec.TimedTextMarkupLang)
|
||||
@@ -837,7 +843,8 @@ class Subtitle(Track):
|
||||
for payload in Subtitle.mdat_payloads(data)
|
||||
):
|
||||
for lang in segment.get_languages():
|
||||
caption_lists[lang].extend(segment.get_captions(lang))
|
||||
for caption in segment.get_captions(lang):
|
||||
Subtitle.append_cue(caption_lists[lang], caption)
|
||||
caption_set = pycaption.CaptionSet(caption_lists)
|
||||
elif codec == Subtitle.Codec.TimedTextMarkupLang:
|
||||
text = try_ensure_utf8(data).decode("utf8")
|
||||
@@ -1109,12 +1116,7 @@ class Subtitle(Track):
|
||||
nodes=nodes,
|
||||
layout_info=layout,
|
||||
)
|
||||
p_caption = captions[-1] if captions else None
|
||||
if p_caption and caption.start == p_caption.end and str(caption.nodes) == str(p_caption.nodes):
|
||||
# it's a duplicate, but lets take its end time
|
||||
p_caption.end = caption.end
|
||||
continue
|
||||
captions.append(caption)
|
||||
Subtitle.append_cue(captions, caption)
|
||||
|
||||
return captions, language
|
||||
|
||||
|
||||
@@ -655,9 +655,9 @@ class Video(Track):
|
||||
raise EnvironmentError("ccextractor executable was not found.")
|
||||
|
||||
out_path = Path(out_path)
|
||||
video_path, ccextractor = self.path, binaries.CCExtractor
|
||||
ccextractor = binaries.CCExtractor
|
||||
|
||||
def run_ccextractor(fallback: str) -> bool:
|
||||
def run_ccextractor(video_path: Path, fallback: str) -> bool:
|
||||
cc_start = time.monotonic()
|
||||
try:
|
||||
subprocess.run(
|
||||
@@ -692,9 +692,10 @@ class Video(Track):
|
||||
|
||||
# Try on the original file first (preserves container-level CC data like c608 boxes),
|
||||
# then fall back to repacked file (ccextractor can fail on some container formats).
|
||||
if not run_ccextractor("retrying on a repackaged copy"):
|
||||
if not run_ccextractor(self.path, "retrying on a repackaged copy"):
|
||||
# repackage() deletes the original file and moves self.path to the repacked copy
|
||||
self.repackage()
|
||||
run_ccextractor("skipping CC extraction")
|
||||
run_ccextractor(self.path, "skipping CC extraction")
|
||||
|
||||
if out_path.exists():
|
||||
cc_track = Subtitle(
|
||||
|
||||
Reference in New Issue
Block a user