merge upstream

- Always send get_cached_keys_if_exists=True for L1/L2 devices to leverage
- the API's automatic caching optimization. This reduces unnecessary license
- requests by prioritizing cached keys for these security levels.

unshackle/commands/dl.py

@@ -66,6 +66,18 @@ from unshackle.core.vaults import Vaults
 
 
 class dl:
+    @staticmethod
+    def _truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str:
+        """Truncate PSSH string for display when not in debug mode."""
+        if logging.root.level == logging.DEBUG or not pssh_string:
+            return pssh_string
+
+        max_width = console.width - len(drm_type) - 12
+        if len(pssh_string) <= max_width:
+            return pssh_string
+
+        return pssh_string[: max_width - 3] + "..."
+
     @click.command(
         short_help="Download, Decrypt, and Mux tracks for titles from a Service.",
         cls=Services,
@@ -1237,7 +1249,8 @@ class dl:
 
         if isinstance(drm, Widevine):
             with self.DRM_TABLE_LOCK:
-                cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({drm.pssh.dumps()})", "text"), overflow="fold"))
+                pssh_display = self._truncate_pssh_for_display(drm.pssh.dumps(), "Widevine")
+                cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold"))
                 pre_existing_tree = next(
                     (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None
                 )
@@ -1329,10 +1342,11 @@ class dl:
 
         elif isinstance(drm, PlayReady):
             with self.DRM_TABLE_LOCK:
+                pssh_display = self._truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady")
                 cek_tree = Tree(
                     Text.assemble(
                         ("PlayReady", "cyan"),
-                        (f"({drm.pssh_b64 or ''})", "text"),
+                        (f"({pssh_display})", "text"),
                         overflow="fold",
                     )
                 )

unshackle/commands/kv.py

@@ -12,84 +12,113 @@ from unshackle.core.vault import Vault
 from unshackle.core.vaults import Vaults
 
 
+def _load_vaults(vault_names: list[str]) -> Vaults:
+    """Load and validate vaults by name."""
+    vaults = Vaults()
+    for vault_name in vault_names:
+        vault_config = next((x for x in config.key_vaults if x["name"] == vault_name), None)
+        if not vault_config:
+            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
+
+        vault_type = vault_config["type"]
+        vault_args = vault_config.copy()
+        del vault_args["type"]
+
+        if not vaults.load(vault_type, **vault_args):
+            raise click.ClickException(f"Failed to load vault ({vault_name}).")
+
+    return vaults
+
+
+def _process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -> dict[str, str]:
+    """Get and validate keys from a vault for a specific service."""
+    content_keys = list(from_vault.get_keys(service))
+
+    bad_keys = {kid: key for kid, key in content_keys if not key or key.count("0") == len(key)}
+    for kid, key in bad_keys.items():
+        log.warning(f"Skipping NULL key: {kid}:{key}")
+
+    return {kid: key for kid, key in content_keys if kid not in bad_keys}
+
+
+def _copy_service_data(to_vault: Vault, from_vault: Vault, service: str, log: logging.Logger) -> int:
+    """Copy data for a single service between vaults."""
+    content_keys = _process_service_keys(from_vault, service, log)
+    total_count = len(content_keys)
+
+    if total_count == 0:
+        log.info(f"{service}: No keys found in {from_vault}")
+        return 0
+
+    try:
+        added = to_vault.add_keys(service, content_keys)
+    except PermissionError:
+        log.warning(f"{service}: No permission to create table in {to_vault}, skipped")
+        return 0
+
+    existed = total_count - added
+
+    if added > 0 and existed > 0:
+        log.info(f"{service}: {added} added, {existed} skipped ({total_count} total)")
+    elif added > 0:
+        log.info(f"{service}: {added} added ({total_count} total)")
+    else:
+        log.info(f"{service}: {existed} skipped (all existed)")
+
+    return added
+
+
 @click.group(short_help="Manage and configure Key Vaults.", context_settings=context_settings)
 def kv() -> None:
     """Manage and configure Key Vaults."""
 
 
 @kv.command()
-@click.argument("to_vault", type=str)
+@click.argument("to_vault_name", type=str)
-@click.argument("from_vaults", nargs=-1, type=click.UNPROCESSED)
+@click.argument("from_vault_names", nargs=-1, type=click.UNPROCESSED)
 @click.option("-s", "--service", type=str, default=None, help="Only copy data to and from a specific service.")
-def copy(to_vault: str, from_vaults: list[str], service: Optional[str] = None) -> None:
+def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] = None) -> None:
     """
     Copy data from multiple Key Vaults into a single Key Vault.
     Rows with matching KIDs are skipped unless there's no KEY set.
     Existing data is not deleted or altered.
 
-    The `to_vault` argument is the key vault you wish to copy data to.
+    The `to_vault_name` argument is the key vault you wish to copy data to.
     It should be the name of a Key Vault defined in the config.
 
-    The `from_vaults` argument is the key vault(s) you wish to take
+    The `from_vault_names` argument is the key vault(s) you wish to take
     data from. You may supply multiple key vaults.
     """
-    if not from_vaults:
+    if not from_vault_names:
         raise click.ClickException("No Vaults were specified to copy data from.")
 
     log = logging.getLogger("kv")
 
-    vaults = Vaults()
+    all_vault_names = [to_vault_name] + list(from_vault_names)
-    for vault_name in [to_vault] + list(from_vaults):
+    vaults = _load_vaults(all_vault_names)
-        vault = next((x for x in config.key_vaults if x["name"] == vault_name), None)
-        if not vault:
-            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
-        vault_type = vault["type"]
-        vault_args = vault.copy()
-        del vault_args["type"]
-        if not vaults.load(vault_type, **vault_args):
-            raise click.ClickException(f"Failed to load vault ({vault_name}).")
 
-    to_vault: Vault = vaults.vaults[0]
+    to_vault = vaults.vaults[0]
-    from_vaults: list[Vault] = vaults.vaults[1:]
+    from_vaults = vaults.vaults[1:]
+
+    vault_names = ", ".join([v.name for v in from_vaults])
+    log.info(f"Copying data from {vault_names} → {to_vault.name}")
 
-    log.info(f"Copying data from {', '.join([x.name for x in from_vaults])}, into {to_vault.name}")
     if service:
         service = Services.get_tag(service)
-        log.info(f"Only copying data for service {service}")
+        log.info(f"Filtering by service: {service}")
 
     total_added = 0
     for from_vault in from_vaults:
-        if service:
+        services_to_copy = [service] if service else from_vault.get_services()
-            services = [service]
-        else:
-            services = from_vault.get_services()
-
-        for service_ in services:
-            log.info(f"Getting data from {from_vault} for {service_}")
-            content_keys = list(from_vault.get_keys(service_))  # important as it's a generator we iterate twice
-
-            bad_keys = {kid: key for kid, key in content_keys if not key or key.count("0") == len(key)}
-
-            for kid, key in bad_keys.items():
-                log.warning(f"Cannot add a NULL Content Key to a Vault, skipping: {kid}:{key}")
-
-            content_keys = {kid: key for kid, key in content_keys if kid not in bad_keys}
-
-            total_count = len(content_keys)
-            log.info(f"Adding {total_count} Content Keys to {to_vault} for {service_}")
-
-            try:
-                added = to_vault.add_keys(service_, content_keys)
-            except PermissionError:
-                log.warning(f" - No permission to create table ({service_}) in {to_vault}, skipping...")
-                continue
 
+        for service_tag in services_to_copy:
+            added = _copy_service_data(to_vault, from_vault, service_tag, log)
             total_added += added
-            existed = total_count - added
 
-            log.info(f"{to_vault} ({service_}): {added} newly added, {existed} already existed (skipped)")
+    if total_added > 0:
-
+        log.info(f"Successfully added {total_added} new keys to {to_vault}")
-    log.info(f"{to_vault}: {total_added} total newly added")
+    else:
+        log.info("Copy completed - no new keys to add")
 
 
 @kv.command()
@@ -106,9 +135,9 @@ def sync(ctx: click.Context, vaults: list[str], service: Optional[str] = None) -
     if not len(vaults) > 1:
         raise click.ClickException("You must provide more than one Vault to sync.")
 
-    ctx.invoke(copy, to_vault=vaults[0], from_vaults=vaults[1:], service=service)
+    ctx.invoke(copy, to_vault_name=vaults[0], from_vault_names=vaults[1:], service=service)
     for i in range(1, len(vaults)):
-        ctx.invoke(copy, to_vault=vaults[i], from_vaults=[vaults[i - 1]], service=service)
+        ctx.invoke(copy, to_vault_name=vaults[i], from_vault_names=[vaults[i - 1]], service=service)
 
 
 @kv.command()
@@ -135,15 +164,7 @@ def add(file: Path, service: str, vaults: list[str]) -> None:
     log = logging.getLogger("kv")
     service = Services.get_tag(service)
 
-    vaults_ = Vaults()
+    vaults_ = _load_vaults(list(vaults))
-    for vault_name in vaults:
-        vault = next((x for x in config.key_vaults if x["name"] == vault_name), None)
-        if not vault:
-            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
-        vault_type = vault["type"]
-        vault_args = vault.copy()
-        del vault_args["type"]
-        vaults_.load(vault_type, **vault_args)
 
     data = file.read_text(encoding="utf8")
     kid_keys: dict[str, str] = {}
@@ -173,15 +194,7 @@ def prepare(vaults: list[str]) -> None:
     """Create Service Tables on Vaults if not yet created."""
     log = logging.getLogger("kv")
 
-    vaults_ = Vaults()
+    vaults_ = _load_vaults(vaults)
-    for vault_name in vaults:
-        vault = next((x for x in config.key_vaults if x["name"] == vault_name), None)
-        if not vault:
-            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
-        vault_type = vault["type"]
-        vault_args = vault.copy()
-        del vault_args["type"]
-        vaults_.load(vault_type, **vault_args)
 
     for vault in vaults_:
         if hasattr(vault, "has_table") and hasattr(vault, "create_table"):

unshackle/core/cdm/decrypt_labs_remote_cdm.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Union
 from uuid import UUID
 
 import requests
+from pywidevine.cdm import Cdm as WidevineCdm
 from pywidevine.device import DeviceTypes
 from requests import Session
 
@@ -79,15 +80,17 @@ class DecryptLabsRemoteCDM:
     Key Features:
     - Compatible with both Widevine and PlayReady DRM schemes
     - Intelligent caching that compares required vs. available keys
+    - Optimized caching for L1/L2 devices (leverages API auto-optimization)
     - Automatic key combination for mixed cache/license scenarios
     - Seamless fallback to license requests when keys are missing
 
     Intelligent Caching System:
     1. DRM classes (PlayReady/Widevine) provide required KIDs via set_required_kids()
     2. get_license_challenge() first checks for cached keys
-    3. If cached keys satisfy requirements, returns empty challenge (no license needed)
+    3. For L1/L2 devices, always attempts cached keys first (API optimized)
-    4. If keys are missing, makes targeted license request for remaining keys
+    4. If cached keys satisfy requirements, returns empty challenge (no license needed)
-    5. parse_license() combines cached and license keys intelligently
+    5. If keys are missing, makes targeted license request for remaining keys
+    6. parse_license() combines cached and license keys intelligently
     """
 
     service_certificate_challenge = b"\x08\x04"
@@ -250,12 +253,14 @@ class DecryptLabsRemoteCDM:
             "pssh": None,
             "challenge": None,
             "decrypt_labs_session_id": None,
+            "tried_cache": False,
+            "cached_keys": None,
         }
         return session_id
 
     def close(self, session_id: bytes) -> None:
         """
-        Close a CDM session.
+        Close a CDM session and perform comprehensive cleanup.
 
         Args:
             session_id: Session identifier
@@ -266,6 +271,8 @@ class DecryptLabsRemoteCDM:
         if session_id not in self._sessions:
             raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
 
+        session = self._sessions[session_id]
+        session.clear()
         del self._sessions[session_id]
 
     def get_service_certificate(self, session_id: bytes) -> Optional[bytes]:
@@ -304,8 +311,13 @@ class DecryptLabsRemoteCDM:
             raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
 
         if certificate is None:
+            if not self._is_playready and self.device_name == "L1":
+                certificate = WidevineCdm.common_privacy_cert
+                self._sessions[session_id]["service_certificate"] = base64.b64decode(certificate)
+                return "Using default Widevine common privacy certificate for L1"
+            else:
                 self._sessions[session_id]["service_certificate"] = None
-            return "Removed"
+                return "No certificate set (not required for this device type)"
 
         if isinstance(certificate, str):
             certificate = base64.b64decode(certificate)
@@ -346,6 +358,8 @@ class DecryptLabsRemoteCDM:
         4. Returns empty challenge if all required keys are cached
 
         The intelligent caching works as follows:
+        - For L1/L2 devices: Always prioritizes cached keys (API automatically optimizes)
+        - For other devices: Uses cache retry logic based on session state
         - With required KIDs set: Only requests license for missing keys
         - Without required KIDs: Returns any available cached keys
         - For PlayReady: Combines cached keys with license keys seamlessly
@@ -365,6 +379,7 @@ class DecryptLabsRemoteCDM:
 
         Note:
             Call set_required_kids() before this method for optimal caching behavior.
+            L1/L2 devices automatically use cached keys when available per API design.
         """
         _ = license_type, privacy_mode
 
@@ -377,10 +392,15 @@ class DecryptLabsRemoteCDM:
         init_data = self._get_init_data_from_pssh(pssh_or_wrm)
         already_tried_cache = session.get("tried_cache", False)
 
+        if self.device_name in ["L1", "L2"]:
+            get_cached_keys = True
+        else:
+            get_cached_keys = not already_tried_cache
+
         request_data = {
             "scheme": self.device_name,
             "init_data": init_data,
-            "get_cached_keys_if_exists": not already_tried_cache,
+            "get_cached_keys_if_exists": get_cached_keys,
         }
 
         if self.device_name in ["L1", "L2", "SL2", "SL3"] and self.service_name:
@@ -434,8 +454,30 @@ class DecryptLabsRemoteCDM:
 
                 if missing_kids:
                     session["cached_keys"] = parsed_keys
-                    request_data["get_cached_keys_if_exists"] = False
+
-                    response = self._http_session.post(f"{self.host}/get-request", json=request_data, timeout=30)
+                    if self.device_name in ["L1", "L2"]:
+                        license_request_data = {
+                            "scheme": self.device_name,
+                            "init_data": init_data,
+                            "get_cached_keys_if_exists": False,
+                        }
+                        if self.service_name:
+                            license_request_data["service"] = self.service_name
+                        if session["service_certificate"]:
+                            license_request_data["service_certificate"] = base64.b64encode(
+                                session["service_certificate"]
+                            ).decode("utf-8")
+                    else:
+                        license_request_data = request_data.copy()
+                        license_request_data["get_cached_keys_if_exists"] = False
+
+                    session["decrypt_labs_session_id"] = None
+                    session["challenge"] = None
+                    session["tried_cache"] = False
+
+                    response = self._http_session.post(
+                        f"{self.host}/get-request", json=license_request_data, timeout=30
+                    )
                     if response.status_code == 200:
                         data = response.json()
                         if data.get("message") == "success" and "challenge" in data:
@@ -580,6 +622,7 @@ class DecryptLabsRemoteCDM:
                     all_keys.append(license_key)
 
             session["keys"] = all_keys
+            session["cached_keys"] = None
         else:
             session["keys"] = license_keys
 

unshackle/services/EXAMPLE/__init__.py

@@ -282,6 +282,10 @@ class EXAMPLE(Service):
 
         return chapters
 
+    def get_widevine_service_certificate(self, **_: any) -> str:
+        """Return the Widevine service certificate from config, if available."""
+        return self.config.get("certificate")
+
     def get_playready_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[bytes]:
         """Retrieve a PlayReady license for a given track."""
 

unshackle/vaults/MySQL.py

@@ -131,16 +131,27 @@ class MySQL(Vault):
         if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()):
             kid_keys = {kid.hex if isinstance(kid, UUID) else kid: key_ for kid, key_ in kid_keys.items()}
 
+        if not kid_keys:
+            return 0
+
         conn = self.conn_factory.get()
         cursor = conn.cursor()
 
         try:
+            placeholders = ",".join(["%s"] * len(kid_keys))
+            cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys()))
+            existing_kids = {row["kid"] for row in cursor.fetchall()}
+
+            new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids}
+
+            if not new_keys:
+                return 0
+
             cursor.executemany(
-                # TODO: SQL injection risk
+                f"INSERT INTO `{service}` (kid, key_) VALUES (%s, %s)",
-                f"INSERT IGNORE INTO `{service}` (kid, key_) VALUES (%s, %s)",
+                new_keys.items(),
-                kid_keys.items(),
             )
-            return cursor.rowcount
+            return len(new_keys)
         finally:
             conn.commit()
             cursor.close()

unshackle/vaults/SQLite.py

@@ -102,16 +102,27 @@ class SQLite(Vault):
         if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()):
             kid_keys = {kid.hex if isinstance(kid, UUID) else kid: key_ for kid, key_ in kid_keys.items()}
 
+        if not kid_keys:
+            return 0
+
         conn = self.conn_factory.get()
         cursor = conn.cursor()
 
         try:
+            placeholders = ",".join(["?"] * len(kid_keys))
+            cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys()))
+            existing_kids = {row[0] for row in cursor.fetchall()}
+
+            new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids}
+
+            if not new_keys:
+                return 0
+
             cursor.executemany(
-                # TODO: SQL injection risk
+                f"INSERT INTO `{service}` (kid, key_) VALUES (?, ?)",
-                f"INSERT OR IGNORE INTO `{service}` (kid, key_) VALUES (?, ?)",
+                new_keys.items(),
-                kid_keys.items(),
             )
-            return cursor.rowcount
+            return len(new_keys)
         finally:
             conn.commit()
             cursor.close()