forked from kenzuya/unshackle
Probe locally installed service tags in parallel when a vault has no table listing, and report every vault holding the KID instead of the first hit.
367 lines
13 KiB
Python
367 lines
13 KiB
Python
import logging
|
|
import re
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import click
|
|
from rich.padding import Padding
|
|
from rich.text import Text
|
|
from rich.tree import Tree
|
|
|
|
from unshackle.core.config import config
|
|
from unshackle.core.console import console
|
|
from unshackle.core.constants import context_settings
|
|
from unshackle.core.services import Services
|
|
from unshackle.core.vault import Vault
|
|
from unshackle.core.vaults import Vaults
|
|
|
|
|
|
def load_vaults(vault_names: list[str]) -> Vaults:
|
|
"""Load and validate vaults by name."""
|
|
vaults = Vaults()
|
|
for vault_name in vault_names:
|
|
vault_config = next((x for x in config.key_vaults if x["name"] == vault_name), None)
|
|
if not vault_config:
|
|
raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
|
|
|
|
vault_type = vault_config["type"]
|
|
vault_args = vault_config.copy()
|
|
del vault_args["type"]
|
|
|
|
if not vaults.load(vault_type, **vault_args):
|
|
raise click.ClickException(f"Failed to load vault ({vault_name}).")
|
|
|
|
return vaults
|
|
|
|
|
|
def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -> dict[str, str]:
|
|
"""Get and validate keys from a vault for a specific service."""
|
|
content_keys = list(from_vault.get_keys(service))
|
|
|
|
bad_keys = {kid: key for kid, key in content_keys if not key or key.count("0") == len(key)}
|
|
for kid, key in bad_keys.items():
|
|
log.warning(f"Skipping NULL key: {kid}:{key}")
|
|
|
|
return {kid: key for kid, key in content_keys if kid not in bad_keys}
|
|
|
|
|
|
def copy_service_data(to_vault: Vault, from_vault: Vault, service: str, log: logging.Logger) -> int:
|
|
"""Copy data for a single service between vaults."""
|
|
content_keys = process_service_keys(from_vault, service, log)
|
|
total_count = len(content_keys)
|
|
|
|
if total_count == 0:
|
|
log.info(f"{service}: No keys found in {from_vault}")
|
|
return 0
|
|
|
|
try:
|
|
added = to_vault.add_keys(service, content_keys)
|
|
except PermissionError:
|
|
log.warning(f"{service}: No permission to create table in {to_vault}, skipped")
|
|
return 0
|
|
|
|
existed = total_count - added
|
|
|
|
if added > 0 and existed > 0:
|
|
log.info(f"{service}: {added} added, {existed} skipped ({total_count} total)")
|
|
elif added > 0:
|
|
log.info(f"{service}: {added} added ({total_count} total)")
|
|
else:
|
|
log.info(f"{service}: {existed} skipped (all existed)")
|
|
|
|
return added
|
|
|
|
|
|
@click.group(short_help="Manage and configure Key Vaults.", context_settings=context_settings)
|
|
def kv() -> None:
|
|
"""Manage and configure Key Vaults."""
|
|
|
|
|
|
@kv.command()
|
|
@click.argument("to_vault_name", type=str)
|
|
@click.argument("from_vault_names", nargs=-1, type=click.UNPROCESSED)
|
|
@click.option("-s", "--service", type=str, default=None, help="Only copy data to and from a specific service.")
|
|
@click.option(
|
|
"-l",
|
|
"--local-only",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Only copy data for services installed locally (skip vault tables for services not present in the configured services path).",
|
|
)
|
|
def copy(
|
|
to_vault_name: str,
|
|
from_vault_names: list[str],
|
|
service: Optional[str] = None,
|
|
local_only: bool = False,
|
|
) -> None:
|
|
"""
|
|
Copy data from multiple Key Vaults into a single Key Vault.
|
|
Rows with matching KIDs are skipped unless there's no KEY set.
|
|
Existing data is not deleted or altered.
|
|
|
|
The `to_vault_name` argument is the key vault you wish to copy data to.
|
|
It should be the name of a Key Vault defined in the config.
|
|
|
|
The `from_vault_names` argument is the key vault(s) you wish to take
|
|
data from. You may supply multiple key vaults.
|
|
"""
|
|
if not from_vault_names:
|
|
raise click.ClickException("No Vaults were specified to copy data from.")
|
|
|
|
log = logging.getLogger("kv")
|
|
|
|
all_vault_names = [to_vault_name] + list(from_vault_names)
|
|
vaults = load_vaults(all_vault_names)
|
|
|
|
to_vault = vaults.vaults[0]
|
|
from_vaults = vaults.vaults[1:]
|
|
|
|
vault_names = ", ".join([v.name for v in from_vaults])
|
|
log.info(f"Copying data from {vault_names} → {to_vault.name}")
|
|
|
|
if service and local_only:
|
|
raise click.UsageError("--service and --local-only are mutually exclusive.")
|
|
|
|
if service:
|
|
service = Services.get_tag(service)
|
|
log.info(f"Filtering by service: {service}")
|
|
|
|
installed: Optional[set[str]] = None
|
|
if local_only:
|
|
installed = {t.upper() for t in Services.get_tags()}
|
|
log.info(f"Filtering by locally installed services ({len(installed)} found)")
|
|
|
|
total_added = 0
|
|
for from_vault in from_vaults:
|
|
services_to_copy = [service] if service else list(from_vault.get_services())
|
|
|
|
if installed is not None:
|
|
before = len(services_to_copy)
|
|
services_to_copy = [s for s in services_to_copy if s and s.upper() in installed]
|
|
skipped = before - len(services_to_copy)
|
|
if skipped:
|
|
log.info(f"{from_vault.name}: skipping {skipped} service(s) not installed locally")
|
|
|
|
for service_tag in services_to_copy:
|
|
added = copy_service_data(to_vault, from_vault, service_tag, log)
|
|
total_added += added
|
|
|
|
if total_added > 0:
|
|
log.info(f"Successfully added {total_added} new keys to {to_vault}")
|
|
else:
|
|
log.info("Copy completed - no new keys to add")
|
|
|
|
|
|
@kv.command()
|
|
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
|
|
@click.option("-s", "--service", type=str, default=None, help="Only sync data to and from a specific service.")
|
|
@click.option(
|
|
"-l",
|
|
"--local-only",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Only sync data for services installed locally (skip vault tables for services not present in the configured services path).",
|
|
)
|
|
@click.pass_context
|
|
def sync(
|
|
ctx: click.Context,
|
|
vaults: list[str],
|
|
service: Optional[str] = None,
|
|
local_only: bool = False,
|
|
) -> None:
|
|
"""
|
|
Ensure multiple Key Vaults copies of all keys as each other.
|
|
It's essentially just a bi-way copy between each vault.
|
|
To see the precise details of what it's doing between each
|
|
provided vault, see the documentation for the `copy` command.
|
|
"""
|
|
if not len(vaults) > 1:
|
|
raise click.ClickException("You must provide more than one Vault to sync.")
|
|
|
|
ctx.invoke(
|
|
copy,
|
|
to_vault_name=vaults[0],
|
|
from_vault_names=vaults[1:],
|
|
service=service,
|
|
local_only=local_only,
|
|
)
|
|
for i in range(1, len(vaults)):
|
|
ctx.invoke(
|
|
copy,
|
|
to_vault_name=vaults[i],
|
|
from_vault_names=[vaults[i - 1]],
|
|
service=service,
|
|
local_only=local_only,
|
|
)
|
|
|
|
|
|
@kv.command()
|
|
@click.argument("file", type=Path)
|
|
@click.argument("service", type=str)
|
|
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
|
|
def add(file: Path, service: str, vaults: list[str]) -> None:
|
|
"""
|
|
Add new Content Keys to Key Vault(s) by service.
|
|
|
|
File should contain one key per line in the format KID:KEY (HEX:HEX).
|
|
Each line should have nothing else within it except for the KID:KEY.
|
|
Encoding is presumed to be UTF8.
|
|
"""
|
|
if not file.exists():
|
|
raise click.ClickException(f"File provided ({file}) does not exist.")
|
|
if not file.is_file():
|
|
raise click.ClickException(f"File provided ({file}) is not a file.")
|
|
if not service or not isinstance(service, str):
|
|
raise click.ClickException(f"Service provided ({service}) is invalid.")
|
|
if len(vaults) < 1:
|
|
raise click.ClickException("You must provide at least one Vault.")
|
|
|
|
log = logging.getLogger("kv")
|
|
service = Services.get_tag(service)
|
|
|
|
vaults_ = load_vaults(list(vaults))
|
|
|
|
data = file.read_text(encoding="utf8")
|
|
kid_keys: dict[str, str] = {}
|
|
for line in data.splitlines(keepends=False):
|
|
line = line.strip()
|
|
match = re.search(r"^(?P<kid>[0-9a-fA-F]{32}):(?P<key>[0-9a-fA-F]{32})$", line)
|
|
if not match:
|
|
continue
|
|
kid = match.group("kid").lower()
|
|
key = match.group("key").lower()
|
|
kid_keys[kid] = key
|
|
|
|
total_count = len(kid_keys)
|
|
|
|
for vault in vaults_:
|
|
log.info(f"Adding {total_count} Content Keys to {vault}")
|
|
added_count = vault.add_keys(service, kid_keys)
|
|
existed_count = total_count - added_count
|
|
log.info(f"{vault}: {added_count} newly added, {existed_count} already existed (skipped)")
|
|
|
|
log.info("Done!")
|
|
|
|
|
|
def search_vault(vault: Vault, kid: str, services: list[str], log: logging.Logger) -> Optional[tuple[str, str]]:
|
|
"""Return the (service, key) of the first service table in a vault holding the KID."""
|
|
|
|
def probe(svc: str) -> Optional[tuple[str, str]]:
|
|
try:
|
|
key = vault.get_key(kid, svc)
|
|
except Exception as e:
|
|
log.debug(f"{vault} [{svc}]: lookup error ({e})")
|
|
return None
|
|
if key and key.count("0") != len(key):
|
|
return svc, key
|
|
return None
|
|
|
|
if len(services) == 1:
|
|
return probe(services[0])
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
futures = [pool.submit(probe, svc) for svc in services]
|
|
try:
|
|
for future in as_completed(futures):
|
|
found = future.result()
|
|
if found:
|
|
return found
|
|
finally:
|
|
for future in futures:
|
|
future.cancel()
|
|
|
|
return None
|
|
|
|
|
|
@kv.command()
|
|
@click.argument("kid", type=str)
|
|
@click.option("-s", "--service", type=str, default=None, help="Limit search to a specific service tag.")
|
|
@click.option(
|
|
"-v", "--vault", "vault_name", type=str, default=None, help="Limit search to a specific configured vault by name."
|
|
)
|
|
def search(kid: str, service: Optional[str], vault_name: Optional[str]) -> None:
|
|
"""
|
|
Search configured Key Vault(s) for a KID and report any matching KEY.
|
|
|
|
KID must be 32 hex characters (no dashes). If --service is omitted, every
|
|
service table in each vault is scanned; vaults that cannot list their
|
|
tables are probed with every locally installed service tag instead, so
|
|
passing --service is much faster. If --vault is omitted, every vault in
|
|
the config is searched.
|
|
"""
|
|
log = logging.getLogger("kv")
|
|
|
|
kid_norm = kid.replace("-", "").lower()
|
|
if not re.fullmatch(r"[0-9a-f]{32}", kid_norm):
|
|
raise click.ClickException(f"KID '{kid}' is not 32 hex characters.")
|
|
|
|
if vault_name:
|
|
vault_names = [vault_name]
|
|
else:
|
|
vault_names = [v["name"] for v in config.key_vaults]
|
|
if not vault_names:
|
|
raise click.ClickException("No Key Vaults are configured.")
|
|
|
|
vaults_ = load_vaults(vault_names)
|
|
|
|
service_tag = Services.get_tag(service) if service else None
|
|
|
|
hits: list[tuple[str, str, str]] = []
|
|
for vault in vaults_:
|
|
probed = False
|
|
if service_tag:
|
|
services_to_check: list[str] = [service_tag]
|
|
else:
|
|
try:
|
|
services_to_check = list(vault.get_services())
|
|
except Exception as e:
|
|
log.debug(f"{vault}: get_services() failed ({e})")
|
|
services_to_check = []
|
|
if not services_to_check:
|
|
services_to_check = list(Services.get_tags())
|
|
probed = True
|
|
log.debug(f"{vault}: cannot list tables, probing {len(services_to_check)} local service tag(s)")
|
|
|
|
found = search_vault(vault, kid_norm, services_to_check, log)
|
|
if found:
|
|
svc_hit, key_hit = found
|
|
hits.append((vault.name, f"{svc_hit}?" if probed else svc_hit, key_hit))
|
|
|
|
if not hits:
|
|
log.info(f"KID {kid_norm} not found in {len(vaults_)} vault(s).")
|
|
return
|
|
|
|
for svc in dict.fromkeys(svc for _, svc, _ in hits):
|
|
tree = Tree(Text.assemble((svc, "cyan"), overflow="fold"))
|
|
for vname, hit_svc, key in hits:
|
|
if hit_svc == svc:
|
|
tree.add(f"[text2]{kid_norm}:{key} from {vname}")
|
|
console.print(Padding(tree, (1, 5)))
|
|
|
|
|
|
@kv.command()
|
|
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
|
|
def prepare(vaults: list[str]) -> None:
|
|
"""Create Service Tables on Vaults if not yet created."""
|
|
log = logging.getLogger("kv")
|
|
|
|
vaults_ = load_vaults(vaults)
|
|
|
|
for vault in vaults_:
|
|
if hasattr(vault, "has_table") and hasattr(vault, "create_table"):
|
|
for service_tag in Services.get_tags():
|
|
if vault.has_table(service_tag):
|
|
log.info(f"{vault} already has a {service_tag} Table")
|
|
else:
|
|
try:
|
|
vault.create_table(service_tag, commit=True)
|
|
log.info(f"{vault}: Created {service_tag} Table")
|
|
except PermissionError:
|
|
log.error(f"{vault} user has no create table permission, skipping...")
|
|
continue
|
|
else:
|
|
log.info(f"{vault} does not use tables, skipping...")
|
|
|
|
log.info("Done!")
|