mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-11 19:45:06 +00:00
feat(vaults): batch HTTP json vault writes with InsertKeys
Try one InsertKeys call per 500 content keys, falling back to per-key InsertKey only when the server answers 404; kv copy/sync chunk 500 for json mode.
This commit is contained in:
@@ -167,7 +167,7 @@ A flexible HTTP vault that speaks one of three wire protocols, selected with `ap
|
||||
password: "your-api-token"
|
||||
```
|
||||
|
||||
Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling.
|
||||
Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. Bulk writes (`kv copy`, `kv add`) first try `InsertKeys` with `params: {"keys": {kid: key, ...}, "service": ..., "title": ...}` (up to 500 content keys per call) and expect `{"inserted": <count>}` back. A server that answers HTTP 404 to `InsertKeys` gets one `InsertKey` call per content key for the rest of the run instead. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling.
|
||||
|
||||
=== "decrypt_labs mode"
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -
|
||||
|
||||
def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str], log: logging.Logger) -> int:
|
||||
"""Add content keys to a vault in batches, with a progress bar of counts and time left."""
|
||||
chunk = 1 if type(vault).__name__ == "HTTP" else 500
|
||||
chunk = 1 if type(vault).__name__ == "HTTP" and getattr(vault, "api_mode", None) != "json" else 500
|
||||
kids = list(kid_keys)
|
||||
added = 0
|
||||
console.print()
|
||||
|
||||
@@ -57,6 +57,7 @@ class HTTP(Vault):
|
||||
|
||||
self.username = username
|
||||
self.api_mode = api_mode.lower()
|
||||
self.batch_insert = True
|
||||
self.current_title = None
|
||||
self.session = Session()
|
||||
self.session.headers.update({"User-Agent": f"unshackle v{__version__}"})
|
||||
@@ -278,6 +279,18 @@ class HTTP(Vault):
|
||||
title = getattr(self, "current_title", None)
|
||||
|
||||
if self.api_mode == "json":
|
||||
if self.batch_insert:
|
||||
try:
|
||||
response = self.request(
|
||||
"InsertKeys", {"keys": processed_kid_keys, "service": service.lower(), "title": title}
|
||||
)
|
||||
if response.get("status") != "not_found":
|
||||
return int(response.get("inserted", 0))
|
||||
# only a 404 proves the server has no InsertKeys method; other
|
||||
# failures may be transient, so keep batching for the next call
|
||||
self.batch_insert = False
|
||||
except Exception:
|
||||
pass
|
||||
for kid, key in processed_kid_keys.items():
|
||||
try:
|
||||
response = self.request(
|
||||
|
||||
Reference in New Issue
Block a user