From e9ed05d4b20fcceb25519a9544d1c712fa8a6dfe Mon Sep 17 00:00:00 2001
From: imSp4rky <george+slight@slight.me>
Date: Thu, 27 Aug 2026 18:21:45 -0600
Subject: [PATCH] feat(vaults): batch HTTP json vault writes with InsertKeys

---
 docs/guide/vaults.md     |  2 +-
 unshackle/commands/kv.py | 13 ++++++++++---
 unshackle/vaults/HTTP.py | 11 ++++++++---
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/docs/guide/vaults.md b/docs/guide/vaults.md
index 5bf189e6..cead2ee1 100644
--- a/docs/guide/vaults.md
+++ b/docs/guide/vaults.md
@@ -167,7 +167,7 @@ A flexible HTTP vault that speaks one of three wire protocols, selected with `ap
         password: "your-api-token"
     ```
 
-    Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. Bulk writes (`kv copy`, `kv add`) first try `InsertKeys` with `params: {"keys": {kid: key, ...}, "service": ..., "title": ...}` (up to 500 content keys per call) and expect `{"inserted": <count>}` back. A server that answers HTTP 404 to `InsertKeys` gets one `InsertKey` call per content key for the rest of the run instead. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling.
+    Requests are `POST`ed as `{"method": ..., "params": ..., "token": ...}` using methods like `GetKey`, `InsertKey`, and `GetServices`. Bulk writes (`kv copy`, `kv add`) first try `InsertKeys` with `params: {"keys": {kid: key, ...}, "service": ..., "title": ...}` (up to 500 content keys per call) and expect `{"inserted": <count>}` back. A server that answers HTTP 404 or `Unknown JSON-RPC method` to `InsertKeys` gets one `InsertKey` call per content key for the rest of the run instead. This mode cannot list all keys in bulk, so a `kv copy` from a json vault relies on the remote server's own duplicate handling.
 
 === "decrypt_labs mode"
 
diff --git a/unshackle/commands/kv.py b/unshackle/commands/kv.py
index fc8cc828..5f73d4ef 100644
--- a/unshackle/commands/kv.py
+++ b/unshackle/commands/kv.py
@@ -57,7 +57,12 @@ def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -
 
 def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str], log: logging.Logger) -> int:
     """Add content keys to a vault in batches, with a progress bar of counts and time left."""
-    chunk = 1 if type(vault).__name__ == "HTTP" and getattr(vault, "api_mode", None) != "json" else 500
+
+    def chunk(i: int) -> int:
+        if type(vault).__name__ != "HTTP":
+            return 500
+        return 500 if i and getattr(vault, "batch_insert", False) else 1
+
     kids = list(kid_keys)
     added = 0
     console.print()
@@ -76,9 +81,11 @@ def add_keys_with_progress(vault: Vault, service: str, kid_keys: dict[str, str],
         transient=True,
     ) as progress:
         task = progress.add_task(f"{service} → {vault}", total=len(kids), added=0)
-        for i in range(0, len(kids), chunk):
-            batch: dict[Union[UUID, str], str] = {kid: kid_keys[kid] for kid in kids[i : i + chunk]}
+        i = 0
+        while i < len(kids):
+            batch: dict[Union[UUID, str], str] = {kid: kid_keys[kid] for kid in kids[i : i + chunk(i)]}
             added += vault.add_keys(service, batch)
+            i += len(batch)
             progress.update(task, advance=len(batch), added=added)
     return added
 
diff --git a/unshackle/vaults/HTTP.py b/unshackle/vaults/HTTP.py
index 3a79cdf6..2a0561ef 100644
--- a/unshackle/vaults/HTTP.py
+++ b/unshackle/vaults/HTTP.py
@@ -93,7 +93,11 @@ class HTTP(Vault):
             return {"status": "not_found"}
 
         if not r.ok:
-            raise ValueError(f"API returned HTTP Error {r.status_code}: {r.reason.title()}")
+            try:
+                detail = r.json().get("message") or r.reason.title()
+            except (json.JSONDecodeError, AttributeError):
+                detail = r.reason.title()
+            raise ValueError(f"API returned HTTP Error {r.status_code}: {detail}")
 
         try:
             res = r.json()
@@ -286,9 +290,10 @@ class HTTP(Vault):
                     )
                     if response.get("status") != "not_found":
                         return int(response.get("inserted", 0))
-                    # only a 404 proves the server has no InsertKeys method; other
-                    # failures may be transient, so keep batching for the next call
                     self.batch_insert = False
+                except ValueError as e:
+                    if "unknown" in str(e).lower() and "method" in str(e).lower():
+                        self.batch_insert = False
                 except Exception:
                     pass
             for kid, key in processed_kid_keys.items():