feat(api)!: add a server drain and never answer with a 5xx

POST/DELETE /api/dashboard/drain refuses new sessions and jobs (SERVER_DRAINING, 409) and warns open sessions. A CDN can replace a 5xx body, so server errors are now 422, retryable ones and the tunnel check 424, and a new middleware turns any 5xx into 422.

BREAKING CHANGE: INTERNAL_ERROR, DOWNLOAD_ERROR, WORKER_ERROR and the fallback return 422 instead of 500; NETWORK_ERROR and SERVICE_UNAVAILABLE return 424 instead of 503; /api/tunnel/check failures return 424 instead of 502.

tests/remote/unit/test_dashboard.py

@@ -281,6 +281,37 @@ async def test_dashboard_session_logs(aiohttp_client, dashboard_cfg) -> None:
     await store.delete(entry.session_id)
 
 
+async def test_drain_refuses_new_work_and_warns_open_sessions(aiohttp_client, dashboard_cfg) -> None:
+    from unshackle.core.api.download_manager import get_download_manager
+    from unshackle.core.api.errors import APIError
+    from unshackle.core.api.session_log import SessionLogBuffer
+    from unshackle.core.api.session_store import get_session_store
+
+    store = get_session_store()
+    entry = await store.create("EXAMPLE", object())
+    entry.log_buffer = SessionLogBuffer()
+    client = await aiohttp_client(make_app())
+    headers = {"X-Secret-Key": "dash-secret"}
+    try:
+        body = await (await client.post("/api/dashboard/drain", headers=headers)).json()
+        assert body["draining"] is True and body["sessions"] == 1
+        await client.post("/api/dashboard/drain", headers=headers)
+        assert [r["level"] for r in entry.log_buffer.since(0)] == ["WARNING"]
+
+        refused = await client.post("/api/session/create", json={})
+        assert refused.status == 409
+        assert (await refused.json())["error_code"] == "SERVER_DRAINING"
+        with pytest.raises(APIError, match="restarts for an update"):
+            get_download_manager().create_job("EXAMPLE", "t")
+
+        body = await (await client.delete("/api/dashboard/drain", headers=headers)).json()
+        assert body["draining"] is False
+        assert (await client.post("/api/session/create", json={})).status == 400
+    finally:
+        store.stop_drain()
+        await store.delete(entry.session_id)
+
+
 async def test_dashboard_log_read_does_not_touch_the_session() -> None:
     from unshackle.core.api.session_store import SessionStore
 

tests/remote/unit/test_errors.py

@@ -40,20 +40,21 @@ def test_api_error_default_http_status_per_code() -> None:
         APIErrorCode.TRACK_NOT_FOUND: 404,
         APIErrorCode.CONFLICT: 409,
         APIErrorCode.RATE_LIMITED: 429,
-        APIErrorCode.INTERNAL_ERROR: 500,
-        APIErrorCode.DOWNLOAD_ERROR: 500,
-        APIErrorCode.WORKER_ERROR: 500,
+        APIErrorCode.INTERNAL_ERROR: 422,
+        APIErrorCode.DOWNLOAD_ERROR: 422,
+        APIErrorCode.WORKER_ERROR: 422,
         APIErrorCode.SERVICE_ERROR: 422,
         APIErrorCode.DRM_ERROR: 422,
-        APIErrorCode.NETWORK_ERROR: 503,
-        APIErrorCode.SERVICE_UNAVAILABLE: 503,
+        APIErrorCode.NETWORK_ERROR: 424,
+        APIErrorCode.SERVICE_UNAVAILABLE: 424,
+        APIErrorCode.SERVER_DRAINING: 409,
     }
     # Lock that the cases above are the *complete* live set; a new/removed code trips this.
     assert {c.name for c in cases} == {c.name for c in APIErrorCode}
     for code, expected in cases.items():
         assert APIError(code, "x").http_status == expected, code
-    # the edge proxy replaces a 502 or 504 body, so no code may default to either
-    assert not {502, 504} & set(cases.values())
+    # the edge proxy can replace a 5xx body, so no code may default to one
+    assert all(status < 500 for status in cases.values())
 
 
 def test_api_error_explicit_http_status_overrides_default() -> None:
@@ -88,7 +89,7 @@ def test_build_error_response_retryable_flag() -> None:
 
 def test_build_error_response_from_generic_exception() -> None:
     resp = build_error_response(RuntimeError("oops"))
-    assert resp.status == 500
+    assert resp.status == 422
     body = _body(resp)
     assert body["error_code"] == "INTERNAL_ERROR"
     assert body["message"] == f"oops (ref {body['ref']})"
@@ -161,15 +162,15 @@ def test_handle_api_exception_categorizes_generic() -> None:
     "exc, status, code, retryable",
     [
         (Exception("Invalid credentials provided"), 401, "AUTH_FAILED", False),
-        (Exception("Connection refused"), 503, "NETWORK_ERROR", True),
-        (TimeoutError("read timeout"), 503, "NETWORK_ERROR", True),
+        (Exception("Connection refused"), 424, "NETWORK_ERROR", True),
+        (TimeoutError("read timeout"), 424, "NETWORK_ERROR", True),
         (Exception("Not available in your region"), 403, "GEOFENCE", False),
         (Exception("Title not found"), 404, "NOT_FOUND", False),
         (Exception("HTTP 429 too many requests"), 429, "RATE_LIMITED", True),
         (Exception("DRM license fetch failed"), 422, "DRM_ERROR", False),
-        (Exception("503 service unavailable"), 503, "SERVICE_UNAVAILABLE", True),
+        (Exception("503 service unavailable"), 424, "SERVICE_UNAVAILABLE", True),
         (ValueError("malformed body"), 400, "INVALID_INPUT", False),
-        (RuntimeError("totally novel failure xyz"), 500, "INTERNAL_ERROR", False),
+        (RuntimeError("totally novel failure xyz"), 422, "INTERNAL_ERROR", False),
     ],
 )
 def test_error_contract_generic_exception(exc: Exception, status: int, code: str, retryable: bool) -> None:
@@ -191,3 +192,29 @@ def test_error_contract_apierror_passthrough_full_shape() -> None:
     assert body["message"] == f"slow down (ref {body['ref']})"
     assert body["details"] == {"retry_after": 5}
     assert body["retryable"] is True
+
+
+async def test_outer_middleware_never_answers_5xx(aiohttp_client) -> None:
+    from aiohttp import web
+
+    from unshackle.core.api.routes import no_5xx_middleware
+
+    async def broken(request: web.Request) -> web.Response:
+        raise KeyError(request.headers["X-Secret-Key"])
+
+    async def subapp_500(_request: web.Request) -> web.Response:
+        return web.json_response({"status": 500, "message": "device error"}, status=500)
+
+    app = web.Application(middlewares=[no_5xx_middleware])
+    app.router.add_get("/broken", broken)
+    app.router.add_get("/subapp", subapp_500)
+    client = await aiohttp_client(app)
+
+    resp = await client.get("/broken", headers={"X-Secret-Key": "leaky-key"})
+    assert resp.status == 422
+    body = await resp.json()
+    assert body["error_code"] == "INTERNAL_ERROR" and "leaky-key" not in json.dumps(body)
+
+    resp = await client.get("/subapp")
+    assert resp.status == 422 and (await resp.json())["message"] == "device error"
+    assert (await client.get("/missing")).status == 404

tests/remote/unit/test_remote_service_helpers.py

@@ -558,11 +558,12 @@ def test_session_create_writes_nested_cache_and_skips_colliding_keys(tmp_path, m
         "create_service_instance",
         lambda *args, **kwargs: (SimpleNamespace(log=logging.getLogger("Svc")), None, None),
     )
+    monkeypatch.setattr("unshackle.core.api.session_store.refuse_while_draining", lambda: None)
     monkeypatch.setattr("unshackle.core.api.session_store.get_session_store", lambda: (_ for _ in ()).throw(Stop()))
 
     cache = {"x": blob('"flat"'), "x.json/y": blob('"collides"'), "MSL/keys.v2": blob('"dotted"')}
     response = asyncio.run(handlers.session_create_handler({"service": "TESTSVC", "title_id": "t", "cache": cache}))
-    assert response.status == 500  # the Stop stand-in fired after the cache write
+    assert response.status == 422  # the Stop stand-in fired after the cache write
 
     written = {p.name: p.read_text() for p in tmp_path.rglob("*.json") if p.is_file()}
     assert written == {"x.json": '"flat"', "keys.v2.json": '"dotted"'}