mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 20:15:10 +00:00
feat(vaults): batch HTTP json vault writes with InsertKeys
Try one InsertKeys call per 500 content keys, falling back to per-key InsertKey only when the server answers 404; kv copy/sync chunk 500 for json mode.
This commit is contained in:
@@ -167,7 +167,7 @@ A flexible HTTP vault that speaks one of three wire protocols, selected with `ap
|
|||||||
password: "your-api-token"
|
password: "your-api-token"
|
||||||
```
|
```
|
||||||
|
|
||||||
Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling.
|
Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. Bulk writes (`kv copy`, `kv add`) first try `InsertKeys` with `params: {"keys": {kid: key, ...}, "service": ..., "title": ...}` (up to 500 content keys per call) and expect `{"inserted": <count>}` back. A server that answers HTTP 404 to `InsertKeys` gets one `InsertKey` call per content key for the rest of the run instead. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling.
|
||||||
|
|
||||||
=== "decrypt_labs mode"
|
=== "decrypt_labs mode"
|
||||||
|
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -
|
|||||||
|
|
||||||
def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str], log: logging.Logger) -> int:
|
def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str], log: logging.Logger) -> int:
|
||||||
"""Add content keys to a vault in batches, with a progress bar of counts and time left."""
|
"""Add content keys to a vault in batches, with a progress bar of counts and time left."""
|
||||||
chunk = 1 if type(vault).__name__ == "HTTP" else 500
|
chunk = 1 if type(vault).__name__ == "HTTP" and getattr(vault, "api_mode", None) != "json" else 500
|
||||||
kids = list(kid_keys)
|
kids = list(kid_keys)
|
||||||
added = 0
|
added = 0
|
||||||
console.print()
|
console.print()
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ class HTTP(Vault):
|
|||||||
|
|
||||||
self.username = username
|
self.username = username
|
||||||
self.api_mode = api_mode.lower()
|
self.api_mode = api_mode.lower()
|
||||||
|
self.batch_insert = True
|
||||||
self.current_title = None
|
self.current_title = None
|
||||||
self.session = Session()
|
self.session = Session()
|
||||||
self.session.headers.update({"User-Agent": f"unshackle v{__version__}"})
|
self.session.headers.update({"User-Agent": f"unshackle v{__version__}"})
|
||||||
@@ -278,6 +279,18 @@ class HTTP(Vault):
|
|||||||
title = getattr(self, "current_title", None)
|
title = getattr(self, "current_title", None)
|
||||||
|
|
||||||
if self.api_mode == "json":
|
if self.api_mode == "json":
|
||||||
|
if self.batch_insert:
|
||||||
|
try:
|
||||||
|
response = self.request(
|
||||||
|
"InsertKeys", {"keys": processed_kid_keys, "service": service.lower(), "title": title}
|
||||||
|
)
|
||||||
|
if response.get("status") != "not_found":
|
||||||
|
return int(response.get("inserted", 0))
|
||||||
|
# only a 404 proves the server has no InsertKeys method; other
|
||||||
|
# failures may be transient, so keep batching for the next call
|
||||||
|
self.batch_insert = False
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
for kid, key in processed_kid_keys.items():
|
for kid, key in processed_kid_keys.items():
|
||||||
try:
|
try:
|
||||||
response = self.request(
|
response = self.request(
|
||||||
|
|||||||
Reference in New Issue
Block a user