perf(kv): push whole services to SQL vaults in one call

This commit is contained in:
imSp4rky
2026-09-03 13:30:39 -06:00
parent 993025fba6
commit e15c2f067e
3 changed files with 82 additions and 10 deletions
+9 -7
View File
@@ -2,7 +2,7 @@ import logging
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Optional, Union
from typing import Optional, Union, cast
from uuid import UUID
import click
@@ -64,19 +64,20 @@ class _PaddedProgress(Progress):
def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str], log: logging.Logger) -> int:
"""Add content keys to a vault in batches, with a progress bar of counts and time left."""
"""Add content keys to a vault. Network vaults write in batches behind a progress bar."""
if type(vault).__name__ in ("MySQL", "SQLite"):
return vault.add_keys(service, cast(dict[Union[UUID, str], str], kid_keys))
def chunk(i: int) -> int:
if type(vault).__name__ != "HTTP":
return 500
return 500 if i and getattr(vault, "batch_insert", False) else 1
# Probe with one key so a server that rejects batches costs one row, not 500.
return 500 if i and getattr(vault, "batch_insert", True) else 1
kids = list(kid_keys)
added = 0
with _PaddedProgress(
SpinnerColumn(finished_text=""),
TextColumn("[bold]{task.description}"),
GradientPulseBarColumn(),
GradientPulseBarColumn(bar_width=None),
MofNCompleteColumn(),
"•",
TextColumn("[green]{task.fields[added]} new"),
@@ -86,6 +87,7 @@ def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str],
TimeRemainingColumn(compact=True),
console=console,
transient=True,
expand=True,
) as progress:
task = progress.add_task(f"{service} → {vault}", total=len(kids), added=0)
i = 0
@@ -196,7 +198,7 @@ def copy(
try:
services_to_copy = list(from_vault.get_services())
except Exception as e:
log.warning(f"{from_vault.name}: cannot list services ({e}), skipped")
log.debug(f"{from_vault.name}: cannot list services ({e}), skipped")
continue
if installed is not None:
+7 -3
View File
@@ -138,9 +138,13 @@ class MySQL(Vault):
cursor = conn.cursor()
try:
placeholders = ",".join(["%s"] * len(kid_keys))
cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys()))
existing_kids = {row["kid"] for row in cursor.fetchall()}
existing_kids: set[str] = set()
kid_list = list(kid_keys)
for i in range(0, len(kid_list), 10000):
batch = kid_list[i : i + 10000]
placeholders = ",".join(["%s"] * len(batch))
cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", batch)
existing_kids.update(row["kid"] for row in cursor.fetchall())
new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids}