fix(api): stop a service's sys.exit from killing the serve process

Catch SystemExit at service import, in the heartbeat task, the 5xx middleware, search, samples and chapters, and answer it as SERVICE_ERROR.

tests/remote/unit/test_errors.py

@@ -216,3 +216,9 @@ async def test_outer_middleware_never_answers_5xx(aiohttp_client) -> None:
     resp = await client.get("/subapp")
     assert resp.status == 422 and (await resp.json())["message"] == "device error"
     assert (await client.get("/missing")).status == 404
+
+
+def test_categorize_system_exit_is_a_service_error() -> None:
+    err = categorize_exception(SystemExit(1), {"service": "EXAMPLE"})
+    assert err.error_code == APIErrorCode.SERVICE_ERROR
+    assert err.message == "Service EXAMPLE exited"

tests/remote/unit/test_request_cache.py

@@ -4,8 +4,10 @@ The directory holds only what the client sent, goes back to the client when it l
 itself, and is always removed when the request ends.
 """
 
+import asyncio
 import base64
 import json
+import types
 import zlib
 
 import click
@@ -123,6 +125,17 @@ async def test_cache_dir_removed_on_error(seen, tmp_path):
     assert not request_dirs(tmp_path)
 
 
+@pytest.mark.asyncio
+async def test_search_answers_a_service_sys_exit_with_an_error(seen, tmp_path, monkeypatch):
+    def search(self):
+        raise SystemExit(1)
+
+    monkeypatch.setattr(_CacheService, "search", search)
+    response = await handlers.search_handler({"service": "EXAMPLE", "query": "q"})
+    assert json.loads(response.body)["error_code"] == "SERVICE_ERROR"
+    assert not request_dirs(tmp_path)
+
+
 @pytest.mark.asyncio
 async def test_invalid_client_cache_still_removes_the_dir(seen, tmp_path):
     data = {"service": "EXAMPLE", "query": "q", "cache": {"tokens": 1}}
@@ -192,3 +205,24 @@ async def test_list_tracks_service_sees_dl_params(seen):
 async def test_search_service_gets_no_dl_params(seen):
     await handlers.search_handler({"service": "EXAMPLE", "query": "q", "dl_params": {"a_lang": ["es-419"]}})
     assert "a_lang" not in seen["parent_params"]
+
+
+@pytest.mark.asyncio
+async def test_samples_answers_a_service_exit_in_decrypts_samples(monkeypatch):
+    class Service:
+        def decrypts_samples(self, track):
+            raise SystemExit(1)
+
+    session = types.SimpleNamespace(
+        tracks={"t": object()}, service_instance=Service(), service_tag="EXAMPLE", lock=asyncio.Lock()
+    )
+
+    async def get_session(session_id, request=None):
+        return session
+
+    monkeypatch.setattr(handlers, "get_validated_session", get_session)
+    monkeypatch.setattr(handlers, "require_authenticated", lambda s: None)
+    monkeypatch.setattr(handlers, "server_cdm_allowed", lambda request, tag: True)
+    with pytest.raises(handlers.APIError) as e:
+        await handlers.session_samples_handler("sid", "t", "1", b"")
+    assert e.value.error_code.value == "SERVICE_ERROR"

tests/remote/unit/test_routes.py

@@ -175,6 +175,41 @@ async def test_heartbeat_keeps_slow_request_alive(aiohttp_client, monkeypatch: p
     assert resp.status == 200 and body["status"] == "error" and body["error_code"] == "SERVICE_ERROR"
 
 
+@pytest.mark.parametrize("delay", [0, 0.2])
+async def test_heartbeat_answers_a_service_exit_and_keeps_serving(
+    aiohttp_client, monkeypatch: pytest.MonkeyPatch, delay: float
+) -> None:
+    """A SystemExit inside the heartbeat task becomes a SERVICE_ERROR instead of stopping the event loop."""
+    monkeypatch.setattr(routes, "HEARTBEAT_INTERVAL", 0.05)
+
+    @heartbeat
+    async def exits(request: web.Request) -> web.Response:
+        await asyncio.sleep(delay)
+        raise SystemExit(1)
+
+    app = web.Application()
+    app["debug_api"] = False
+    app.router.add_get("/exits", api_handler(exits))
+    client = await aiohttp_client(app)
+
+    for _ in range(2):
+        resp = await client.get("/exits")
+        assert json.loads(await resp.text())["error_code"] == "SERVICE_ERROR"
+
+
+async def test_unguarded_route_answers_a_service_exit(aiohttp_client) -> None:
+    async def exits(request: web.Request) -> web.Response:
+        raise SystemExit(1)
+
+    app = web.Application(middlewares=[routes.no_5xx_middleware])
+    app.router.add_get("/exits", exits)
+    client = await aiohttp_client(app)
+
+    for _ in range(2):
+        resp = await client.get("/exits")
+        assert resp.status == 422 and (await resp.json())["error_code"] == "SERVICE_ERROR"
+
+
 async def test_heartbeat_late_error_counts_as_rejected(aiohttp_client, monkeypatch: pytest.MonkeyPatch) -> None:
     """Stats count a late failure by its real status and the bytes the stream sent (with framing), not as an empty 200."""
     from unshackle.core.api.stats import stats, stats_middleware