unshackle/tests/remote/unit/test_dashboard.py

"""Dashboard auth split, stats middleware, log ring buffer and session events."""

from __future__ import annotations

import asyncio
import logging
import sys
import threading
import time
from collections.abc import Callable
from datetime import datetime, timedelta, timezone

import pytest
from aiohttp import web

from unshackle.core.api.events import EventBus, bus
from unshackle.core.api.handlers import api_key_authentication, dashboard_authentication
from unshackle.core.api.routes import cors_middleware, setup_routes
from unshackle.core.api.session_store import SessionStore
from unshackle.core.api.stats import RATE_LIMIT_WINDOW, RingLogHandler, key_id, stats, stats_middleware
from unshackle.core.config import config

pytestmark = pytest.mark.unit


@pytest.fixture
def dashboard_cfg(monkeypatch):
    monkeypatch.setitem(config.serve, "dashboard", {"key": "dash-secret"})
    monkeypatch.setitem(config.serve, "users", {"tier-key": {"username": "tier1"}})
    monkeypatch.setitem(config.serve, "services", None)
    stats.requests_total = 0
    stats.requests_rejected = 0
    stats.keys.clear()


def make_app(dashboard: bool = True) -> web.Application:
    app = web.Application(middlewares=[cors_middleware, stats_middleware, dashboard_authentication])
    app["config"] = {"users": {}}
    app["debug_api"] = False
    setup_routes(app, remote_only=True, dashboard=dashboard)
    return app


async def test_dashboard_key_split(aiohttp_client, dashboard_cfg) -> None:
    client = await aiohttp_client(make_app())
    assert (await client.get("/api/dashboard/status", headers={"X-Secret-Key": "tier-key"})).status == 401
    assert (await client.get("/api/dashboard/status")).status == 401
    resp = await client.get("/api/dashboard/status", headers={"X-Secret-Key": "dash-secret"})
    assert resp.status == 200
    body = await resp.json()
    assert body["mode"] and "uptime_seconds" in body and body["sessions"] == 0
    # rejected requests count only as rejected; accepted ones are keyed by masked label
    assert stats.requests_total == 3 and stats.requests_rejected == 2 and stats.requests_by_key == {"dash…": 1}
    await client.get("/api/health", headers={"X-Secret-Key": "tier-key"})
    assert stats.requests_by_key["tier1"] == 1


async def test_dashboard_routes_absent_without_key(aiohttp_client, dashboard_cfg) -> None:
    client = await aiohttp_client(make_app(dashboard=False))
    assert (await client.get("/api/dashboard/status", headers={"X-Secret-Key": "dash-secret"})).status == 404


def test_ring_since_and_level() -> None:
    ring = RingLogHandler(maxlen=3)
    ring.setFormatter(logging.Formatter("%(message)s"))
    logger = logging.getLogger("test.ring")
    logger.propagate = False
    logger.addHandler(ring)
    logger.setLevel(logging.DEBUG)
    for i in range(5):
        (logger.warning if i % 2 else logger.info)(f"m{i}")
    assert [r["msg"] for r in ring.records] == ["m2", "m3", "m4"]
    assert ring.seq == 5
    assert [r["msg"] for r in ring.since(3)] == ["m3", "m4"]
    assert [r["msg"] for r in ring.since(0, "warning")] == ["m3"]


def _hammer(write: Callable[[], object], read: Callable[[], object]) -> None:
    """Call ``read`` on this thread while four threads call ``write`` in a loop."""
    stop = threading.Event()

    def spin() -> None:
        while not stop.is_set():
            write()

    old_interval = sys.getswitchinterval()
    sys.setswitchinterval(1e-6)
    threads = [threading.Thread(target=spin, daemon=True) for _ in range(4)]
    try:
        for t in threads:
            t.start()
        for _ in range(300):
            read()
    finally:
        stop.set()
        sys.setswitchinterval(old_interval)
        for t in threads:
            t.join()


def test_bus_since_survives_concurrent_publish() -> None:
    local = EventBus(history=5000)
    _hammer(lambda: local.publish("log", {}), lambda: local.since(0))
    seqs = [item["seq"] for item in local.history]
    assert seqs == list(range(seqs[0], seqs[0] + len(seqs)))


def test_bus_publish_reenters_its_lock() -> None:
    local = EventBus()

    def nested() -> None:
        with local._lock:
            local.publish("log", {})

    worker = threading.Thread(target=nested, daemon=True)
    worker.start()
    worker.join(5)
    assert not worker.is_alive(), "publish() deadlocked re-entering its own lock"
    assert [item["seq"] for item in local.since(0)] == [1]


def test_ring_since_survives_concurrent_emit() -> None:
    ring = RingLogHandler(maxlen=5000)
    record = logging.LogRecord("test.ring", logging.INFO, __file__, 0, "m", None, None)
    _hammer(lambda: ring.handle(record), lambda: ring.since(0))


def test_ring_masks_host_paths_and_secrets_in_records() -> None:
    """The ring feeds both /api/logs and the SSE stream, so it must mask on the way in."""
    from pathlib import Path

    import unshackle.core.utils.redact as redact_mod

    root = str(Path(redact_mod.__file__).resolve().parents[3])
    ring = RingLogHandler(maxlen=3)
    ring.setFormatter(logging.Formatter("%(message)s"))
    logger = logging.getLogger("test.ring.redact")
    logger.setLevel(logging.INFO)
    logger.addHandler(ring)
    logger.error(f'File "{root}/unshackle/services/HMAX/__init__.py" 404: https://api.x/y?token=abc')
    logger.removeHandler(ring)

    msg = ring.records[-1]["msg"]
    assert root not in msg
    assert "<unshackle>/unshackle/services/HMAX" in msg
    assert "token=abc" not in msg
    assert "https://api.x/y" in msg
    assert ring.since(0, logger="test") and not ring.since(0, logger="other")


async def test_session_events() -> None:
    queue = bus.subscribe()
    try:
        store = SessionStore()
        entry = await store.create("EXAMPLE", object())
        assert [e.session_id for e in store.list()] == [entry.session_id]
        await store.delete(entry.session_id)
        create, delete = queue.get_nowait(), queue.get_nowait()
        assert create["event"] == "session" and create["data"]["action"] == "create"
        assert delete["data"]["action"] == "delete" and delete["data"]["id"] == entry.session_id
        assert "owner_key" not in create["data"]
    finally:
        bus.unsubscribe(queue)
    await asyncio.sleep(0)


async def test_events_burst_mode(aiohttp_client, dashboard_cfg) -> None:
    client = await aiohttp_client(make_app())
    mark = bus.seq
    bus.publish("session", {"action": "create", "id": "s1"})
    bus.publish("session", {"action": "delete", "id": "s1"})
    resp = await client.get(f"/api/dashboard/events?since={mark}", headers={"X-Secret-Key": "dash-secret"})
    assert resp.status == 200
    body = await resp.json()
    assert body["seq"] == bus.seq and "stats" in body
    assert [e["data"]["action"] for e in body["events"] if e["event"] == "session"] == ["create", "delete"]


async def test_session_action_log() -> None:
    store = SessionStore()
    entry = await store.create("EXAMPLE", object(), owner_key="k")
    queue = bus.subscribe()
    try:
        store.record_action(entry.session_id, {"action": "titles", "status": 200})
        store.record_action("missing", {"action": "titles", "status": 200})
        assert entry.summary()["actions"] == [{"action": "titles", "status": 200}]
        assert queue.get_nowait()["data"]["action"] == "update" and queue.empty()
    finally:
        bus.unsubscribe(queue)


async def test_key_stats_separate_unnamed_keys_sharing_a_prefix(aiohttp_client, monkeypatch) -> None:
    """mask_key truncates to four characters, so two unnamed keys can share a label."""
    monkeypatch.setitem(config.serve, "dashboard", {"key": "dash-secret"})
    monkeypatch.setitem(config.serve, "users", {"abcd-one": {}, "abcd-two": {}})
    stats.keys.clear()
    client = await aiohttp_client(make_app())
    await client.get("/api/health", headers={"X-Secret-Key": "abcd-one"})
    for _ in range(2):
        await client.get("/api/health", headers={"X-Secret-Key": "abcd-two"})

    # The legacy label-keyed view still merges them; that is why /keys exists.
    assert stats.requests_by_key["abcd…"] == 3
    assert stats.keys[key_id("abcd-one")].requests == 1
    assert stats.keys[key_id("abcd-two")].requests == 2

    resp = await client.get("/api/dashboard/keys", headers={"X-Secret-Key": "dash-secret"})
    rows = {r["id"]: r for r in await resp.json()}
    assert rows[key_id("abcd-one")]["requests"] == 1
    assert rows[key_id("abcd-two")]["requests"] == 2
    assert all(r["rate_limit"] is None for r in rows.values())


def test_rate_limit_window(monkeypatch) -> None:
    monkeypatch.setitem(config.serve, "tiers", {"bot": {"rate_limit": 2}})
    monkeypatch.setitem(config.serve, "users", {"k": {"tier": "bot"}, "free": {}})
    stats.keys.clear()

    assert stats.check_rate_limit("k") is None
    assert stats.check_rate_limit("k") is None
    retry_after = stats.check_rate_limit("k")
    assert retry_after is not None and 0 < retry_after <= RATE_LIMIT_WINDOW

    # A key with no tier and no override is unlimited and never opens a window.
    assert all(stats.check_rate_limit("free") is None for _ in range(5))

    # The window is fixed, not sliding: it reopens once the hour has passed.
    stats.keys[key_id("k")].window_start -= RATE_LIMIT_WINDOW + 1
    assert stats.check_rate_limit("k") is None


def test_rate_limit_per_key_override_beats_tier(monkeypatch) -> None:
    monkeypatch.setitem(config.serve, "tiers", {"bot": {"rate_limit": 2}})
    monkeypatch.setitem(config.serve, "users", {"k": {"tier": "bot", "rate_limit": 5}})
    stats.keys.clear()
    assert [stats.check_rate_limit("k") for _ in range(5)] == [None] * 5
    assert stats.check_rate_limit("k") is not None


async def test_rate_limited_request_returns_429(aiohttp_client, monkeypatch) -> None:
    monkeypatch.setitem(config.serve, "dashboard", {"key": "dash-secret"})
    monkeypatch.setitem(config.serve, "tiers", {"bot": {"rate_limit": 1}})
    monkeypatch.setitem(config.serve, "users", {"tier-key": {"username": "tier1", "tier": "bot"}})
    stats.keys.clear()

    app = web.Application(
        middlewares=[cors_middleware, stats_middleware, dashboard_authentication, api_key_authentication]
    )
    app["config"] = {"users": {"tier-key": {}}}
    app["debug_api"] = False
    setup_routes(app, remote_only=True, dashboard=True)
    client = await aiohttp_client(app)

    assert (await client.get("/api/services", headers={"X-Secret-Key": "tier-key"})).status == 200
    resp = await client.get("/api/services", headers={"X-Secret-Key": "tier-key"})
    assert resp.status == 429 and int(resp.headers["Retry-After"]) > 0
    # The dashboard key leaves the middleware before the limit check, so it always answers.
    assert (await client.get("/api/dashboard/status", headers={"X-Secret-Key": "dash-secret"})).status == 200


async def test_dashboard_session_logs(aiohttp_client, dashboard_cfg) -> None:
    from unshackle.core.api.session_log import SessionLogBuffer
    from unshackle.core.api.session_store import get_session_store

    store = get_session_store()
    entry = await store.create("EXAMPLE", object())
    entry.log_buffer = SessionLogBuffer()
    entry.log_buffer.append(logging.ERROR, "login rejected: MFA required")
    assert entry.summary()["log_seq"] == 1

    client = await aiohttp_client(make_app())
    headers = {"X-Secret-Key": "dash-secret"}
    body = await (await client.get(f"/api/dashboard/sessions/{entry.session_id}/logs", headers=headers)).json()
    assert body["last_seq"] == 1
    assert body["records"][0]["message"] == "login rejected: MFA required"

    # A cursor read, so the client draining the same buffer still gets every record.
    empty = await (await client.get(f"/api/dashboard/sessions/{entry.session_id}/logs?since=1", headers=headers)).json()
    assert empty["records"] == [] and empty["last_seq"] == 1

    assert (await client.get("/api/dashboard/sessions/nope/logs", headers=headers)).status == 404
    await store.delete(entry.session_id)


async def test_dashboard_log_read_does_not_touch_the_session() -> None:
    from unshackle.core.api.session_store import SessionStore

    store = SessionStore()
    entry = await store.create("EXAMPLE", object())

    stale = datetime.now(timezone.utc) - timedelta(seconds=1)
    entry.last_accessed = stale
    assert store.peek(entry.session_id) is entry
    assert entry.last_accessed == stale

    # get() is the touching read, and is why the dashboard must not use it.
    await store.get(entry.session_id)
    assert entry.last_accessed > stale


async def test_dashboard_services_states(aiohttp_client, dashboard_cfg, monkeypatch) -> None:
    from unshackle.core import services as services_module

    tags = services_module.Services.get_tags()
    if not tags:
        pytest.skip("no services installed")
    staged, failed = tags[0], tags[-1]
    monkeypatch.setattr(services_module, "PENDING", {staged})
    monkeypatch.setattr(services_module, "PENDING_SINCE", {staged: 1756908900.0})
    monkeypatch.setattr(services_module, "LOAD_ERRORS", [f"{failed}: failed to import - ImportError: boom"])

    client = await aiohttp_client(make_app())
    rows = {
        r["tag"]: r
        for r in await (await client.get("/api/dashboard/services", headers={"X-Secret-Key": "dash-secret"})).json()
    }
    assert rows[failed]["state"] == "failed" and "boom" in rows[failed]["error"]
    if staged != failed:
        assert rows[staged]["state"] == "staged" and rows[staged]["staged_since"] == 1756908900.0
        assert all(r["state"] == "loaded" for t, r in rows.items() if t not in (staged, failed))


async def test_dashboard_services_lists_only_configured(aiohttp_client, dashboard_cfg, monkeypatch) -> None:
    from pathlib import Path

    from unshackle.core import services as services_module

    monkeypatch.setattr(services_module, "SERVICES", [Path(f"/svc/{t}/__init__.py") for t in ("AAA", "BBB", "CCC")])
    client = await aiohttp_client(make_app())
    headers = {"X-Secret-Key": "dash-secret"}

    async def listed() -> set[str]:
        rows = await (await client.get("/api/dashboard/services", headers=headers)).json()
        count = (await (await client.get("/api/dashboard/status", headers=headers)).json())["services"]
        assert count == len(rows)
        return {r["tag"] for r in rows}

    assert await listed() == {"AAA", "BBB", "CCC"}
    users = {"k1": {"services": ["aaa"]}, "k2": {"services": ["BBB", 1]}, "k3": {"services": "CCC"}, "k4": {}}
    monkeypatch.setitem(config.serve, "users", users)
    assert await listed() == {"AAA", "BBB"}
    monkeypatch.setitem(config.serve, "users", ["not", "a", "mapping"])
    assert await listed() == {"AAA", "BBB", "CCC"}
    monkeypatch.setitem(config.serve, "services", ["CCC"])
    assert await listed() == {"CCC"}
    everything = await (await client.get("/api/dashboard/services?all=1", headers=headers)).json()
    assert {r["tag"] for r in everything} == {"AAA", "BBB", "CCC"}


async def test_status_reports_no_session_limit(aiohttp_client, dashboard_cfg, monkeypatch) -> None:
    monkeypatch.setitem(config.serve, "max_sessions", None)
    client = await aiohttp_client(make_app())
    body = await (await client.get("/api/dashboard/status", headers={"X-Secret-Key": "dash-secret"})).json()
    assert body["max_sessions"] is None


async def test_dashboard_health(aiohttp_client, dashboard_cfg) -> None:
    from unshackle.core.api import handlers

    handlers._health_cache.clear()
    client = await aiohttp_client(make_app())
    body = await (await client.get("/api/dashboard/health", headers={"X-Secret-Key": "dash-secret"})).json()
    assert body["status"] in {"ok", "degraded", "failing"}
    assert {"id", "label", "status", "detail", "ms"} <= set(body["checks"][0])
    assert all(c["status"] in {"ok", "warn", "fail"} for c in body["checks"])


async def test_unknown_key_on_open_route_does_not_grow_key_stats(aiohttp_client, monkeypatch) -> None:
    """/api/health answers 200 to any header, so counters keyed by the raw header are a memory leak."""
    monkeypatch.setitem(config.serve, "users", {"real": {}})
    stats.keys.clear()
    client = await aiohttp_client(make_app())
    for i in range(20):
        assert (await client.get("/api/health", headers={"X-Secret-Key": f"garbage-{i}"})).status == 200
    await client.get("/api/health", headers={"X-Secret-Key": "real"})
    assert set(stats.keys) == {"anonymous", key_id("real")}
    assert stats.keys["anonymous"].requests == 20


async def test_refresh_events_never_report_a_failed_reload_as_applied() -> None:
    from unshackle.core.api.events import publish_refresh_events

    queue: asyncio.Queue = bus.subscribe()
    try:
        publish_refresh_events(
            [
                {
                    "spec": "x",
                    "updated": True,
                    "changes": ["~GOOD", "~BAD", "~BUSY"],
                    "deferred": ["BUSY"],
                    "load_errors": ["BAD: failed to import - ImportError: boom"],
                }
            ]
        )
        events = []
        while not queue.empty():
            events.append(queue.get_nowait()["data"])
    finally:
        bus.unsubscribe(queue)
    by_action = {e["action"]: e for e in events}
    assert by_action["applied"]["tags"] == ["GOOD"]
    assert by_action["staged"]["tags"] == ["BUSY"]
    assert by_action["failed"]["tags"] == [] and "boom" in by_action["failed"]["errors"][0]


def test_health_check_masks_vault_secrets() -> None:
    from unshackle.core.api.handlers import health_check

    def probe() -> tuple[str, str]:
        raise ConnectionError("could not connect to https://user:hunter2@vault.example/api?token=hunter2")

    detail = health_check("vault:API", "vault API", probe, ["hunter2"])["detail"]
    assert "hunter2" not in detail and detail.startswith("ConnectionError")


async def test_health_vault_probe_tolerates_api_vault_rejecting_the_probe(aiohttp_server, monkeypatch) -> None:
    """An API vault answers code 3/4 to a dummy service or KID; that proves it is up, not down."""
    from unshackle.core.api.handlers import run_health_checks

    async def reject(request: web.Request) -> web.Response:
        return web.json_response({"code": 4, "message": "bad kid"})

    vault_app = web.Application()
    vault_app.router.add_get("/{tail:.*}", reject)
    server = await aiohttp_server(vault_app)
    monkeypatch.setattr(
        config, "key_vaults", [{"type": "API", "name": "t", "uri": str(server.make_url("/")), "token": "tok"}]
    )
    monkeypatch.setattr(config, "proxy_providers", {})
    checks = {c["id"]: c for c in await asyncio.to_thread(run_health_checks)}
    assert checks["vault:t"]["status"] == "ok", checks["vault:t"]


def test_health_warns_without_a_sqlite_vault_to_store_flags(monkeypatch) -> None:
    from unshackle.core.api.handlers import run_health_checks

    monkeypatch.setattr(config, "proxy_providers", {})
    monkeypatch.setattr(
        config, "key_vaults", [{"type": "API", "name": "shared", "uri": "http://127.0.0.1:9", "token": "t"}]
    )
    assert {c["id"]: c for c in run_health_checks()}["bad_keys"]["status"] == "warn"

    monkeypatch.setattr(config, "key_vaults", [])
    assert {c["id"]: c for c in run_health_checks()}["bad_keys"]["status"] == "ok"


def test_health_proxy_probe_reports_a_provider_that_fails_to_build(monkeypatch) -> None:
    from unshackle.core.api.handlers import run_health_checks

    monkeypatch.setattr(config, "key_vaults", [])
    monkeypatch.setattr(config, "proxy_providers", {"nordvpn": {"username": "u"}})  # missing password
    checks = {c["id"]: c for c in run_health_checks()}
    assert checks["proxies"]["status"] == "fail" and "TypeError" in checks["proxies"]["detail"]


def test_health_proxy_probe_stays_out_of_the_log_ring(monkeypatch, caplog) -> None:
    """Rendering a provider fetches its server catalog, and the panel refreshes every 30 s."""
    from unshackle.core.api.handlers import run_health_checks

    monkeypatch.setattr(config, "key_vaults", [])
    monkeypatch.setattr(config, "proxy_providers", {"basic": {"us": "http://user:pass@host:8080"}})
    with caplog.at_level(logging.INFO, logger="proxies"):
        checks = {c["id"]: c for c in run_health_checks()}
    assert checks["proxies"]["status"] == "ok" and "Basic" in checks["proxies"]["detail"]
    assert [r.message for r in caplog.records if r.name == "proxies"] == []


async def test_health_vault_checks_are_told_apart_by_name(aiohttp_server, monkeypatch) -> None:
    """Two vaults of one type used to answer under the same id and the same label."""
    from unshackle.core.api.handlers import run_health_checks

    async def reject(request: web.Request) -> web.Response:
        return web.json_response({"code": 4, "message": "bad kid"})

    vault_app = web.Application()
    vault_app.router.add_get("/{tail:.*}", reject)
    server = await aiohttp_server(vault_app)
    uri = str(server.make_url("/"))
    monkeypatch.setattr(
        config,
        "key_vaults",
        [
            {"type": "API", "name": "primary", "uri": uri, "token": "tok"},
            {"type": "API", "name": "backup", "uri": uri, "token": "tok"},
        ],
    )
    monkeypatch.setattr(config, "proxy_providers", {})
    ids = [c["id"] for c in await asyncio.to_thread(run_health_checks) if c["id"].startswith("vault:")]
    assert ids == ["vault:primary", "vault:backup"]


def test_rate_limit_value_must_be_a_positive_whole_number() -> None:
    """key_rate_limit reads anything else as unlimited, so it has to be caught at startup."""
    from unshackle.core.api.stats import key_rate_limit, rate_limit_error

    assert rate_limit_error("serve.tiers.bot", 600) is None
    assert rate_limit_error("serve.tiers.bot", None) is None
    for bad in ("600", 600.0, 0, -1, True):
        assert rate_limit_error("serve.tiers.bot", bad), bad

    with pytest.MonkeyPatch().context() as m:
        m.setitem(config.serve, "tiers", {"bot": {"rate_limit": "600"}})
        m.setitem(config.serve, "users", {"k": {"tier": "bot"}})
        assert key_rate_limit("k") is None


async def test_dashboard_key_gets_its_own_row(aiohttp_client, dashboard_cfg) -> None:
    """configured_key counts the dashboard key, so a row must exist to attribute its polling to."""
    client = await aiohttp_client(make_app())
    headers = {"X-Secret-Key": "dash-secret"}
    await client.get("/api/dashboard/status", headers=headers)
    rows = await (await client.get("/api/dashboard/keys", headers=headers)).json()

    by_id = {r["id"]: r for r in rows}
    assert len(by_id) == len(rows)
    dash = by_id[key_id("dash-secret")]
    assert dash["role"] == "dashboard" and dash["requests"] > 0
    # null would render as "all services" on a key that reaches no service route.
    assert dash["services"] == []
    assert not any((dash["server_cdm"], dash["server_accounts"], dash["server_proxy"]))
    assert by_id[key_id("tier-key")]["role"] == "user"

    # Every counted bucket now has a row to attribute it to; only anonymous is left over.
    labels = {r["label"] for r in rows} | {"anonymous"}
    assert set(stats.requests_by_key) <= labels


async def test_dashboard_key_listed_as_a_user_appears_once(aiohttp_client, monkeypatch) -> None:
    monkeypatch.setitem(config.serve, "dashboard", {"key": "dash-secret"})
    monkeypatch.setitem(config.serve, "users", {"dash-secret": {"username": "ops", "services": ["EXAMPLE"]}})
    monkeypatch.setitem(config.serve, "api_secret", "dash-secret")
    stats.keys.clear()

    client = await aiohttp_client(make_app())
    rows = await (await client.get("/api/dashboard/keys", headers={"X-Secret-Key": "dash-secret"})).json()
    assert len(rows) == 1
    # The dashboard role wins as the most specific, and a real users entry keeps its allowlist.
    assert rows[0]["role"] == "dashboard" and rows[0]["label"] == "ops"
    assert rows[0]["services"] == ["EXAMPLE"]


async def test_admin_row_is_distinguishable(aiohttp_client, monkeypatch) -> None:
    monkeypatch.setitem(config.serve, "dashboard", {"key": "dash-secret"})
    monkeypatch.setitem(config.serve, "users", {"tier-key": {"username": "tier1"}})
    monkeypatch.setitem(config.serve, "api_secret", "master-secret")
    stats.keys.clear()

    client = await aiohttp_client(make_app())
    rows = await (await client.get("/api/dashboard/keys", headers={"X-Secret-Key": "dash-secret"})).json()
    roles = {r["role"]: r for r in rows}
    assert set(roles) == {"user", "admin", "dashboard"}
    # No serve.users entry, so the master secret keeps the implicit access the resolvers give it.
    assert roles["admin"]["services"] is None and roles["admin"]["server_cdm"] is True


def test_health_check_masks_a_pymysql_password_alias() -> None:
    """pymysql accepts passwd and ssl_key_password as well as password; all three reach the driver."""
    from unshackle.core.api.handlers import config_secrets, health_check

    def probe() -> tuple[str, str]:
        raise ConnectionError("connect failed for passwd=hunter2 key=keypass1")

    secrets = config_secrets({"type": "MySQL", "host": "db", "passwd": "hunter2", "ssl_key_password": "keypass1"})
    detail = health_check("vault:x", "vault x", probe, secrets)["detail"]
    assert "hunter2" not in detail and "keypass1" not in detail


def test_health_check_masks_a_nested_config_secret() -> None:
    """An API vault's headers and a MySQL vault's ssl are driver arguments with no fixed names."""
    from unshackle.core.api.handlers import config_secrets, health_check

    def probe() -> tuple[str, str]:
        raise ConnectionError("rejected header nested-token-1 and cert nested-pass-2")

    secrets = config_secrets(
        {"type": "API", "uri": "https://vault.example", "headers": {"X-Api-Key": "nested-token-1"}}
    ) + config_secrets({"type": "MySQL", "ssl": {"key_password": "nested-pass-2"}})
    detail = health_check("vault:x", "vault x", probe, secrets)["detail"]
    assert "nested-token-1" not in detail and "nested-pass-2" not in detail
    # Too short to mask usefully: replacing it would garble the detail without hiding anything.
    assert config_secrets({"token": "ab"}) == []


def test_health_check_detail_is_capped_after_redaction() -> None:
    """A driver can raise with a whole response body; redaction only hides what it recognises."""
    from unshackle.core.api.handlers import HEALTH_DETAIL_MAX, health_check

    def probe() -> tuple[str, str]:
        raise ValueError("API returned an invalid response: " + "hunter2 filler " * 200)

    detail = health_check("vault:x", "vault x", probe, ["hunter2"])["detail"]
    assert len(detail) == HEALTH_DETAIL_MAX + 1 and detail.endswith("…")
    # Capped after masking, so no secret survives in the part that is kept.
    assert "hunter2" not in detail


def test_health_proxy_probe_masks_provider_credentials(monkeypatch) -> None:
    """The proxy check held only by luck: no provider happens to raise with its own password."""
    from unshackle.core.api.handlers import config_secrets, run_health_checks

    providers = {
        "basic": {"us": "http://someone:proxypass1@host:8080"},
        "nordvpn": {"username": "u" * 24, "password": "nordpass1234"},
    }
    secrets = config_secrets(providers)
    # basic maps a country to a whole proxy URI, so the URI itself is the collected secret.
    assert "nordpass1234" in secrets and any("proxypass1" in s for s in secrets)

    monkeypatch.setattr(config, "key_vaults", [])
    monkeypatch.setattr(config, "proxy_providers", providers)
    detail = {c["id"]: c for c in run_health_checks()}["proxies"]["detail"]
    assert "nordpass1234" not in detail and "proxypass1" not in detail


async def test_health_probe_timeout_reports_what_finished(aiohttp_client, dashboard_cfg, monkeypatch) -> None:
    """A vault that accepts the connection then stalls must not hang the panel forever."""
    from unshackle.core.api import handlers

    def stalling_probe(checks=None):
        checks = [] if checks is None else checks
        checks.append({"id": "ffmpeg", "label": "ffmpeg", "status": "ok", "detail": "7.1", "ms": 1.0})
        time.sleep(6)  # the stalled vault; the deadline fires long before this returns
        checks.append({"id": "never", "label": "never", "status": "ok", "detail": "", "ms": 0.0})
        return checks

    handlers._health_cache.clear()
    monkeypatch.setattr(handlers, "run_health_checks", stalling_probe)
    monkeypatch.setattr(handlers, "HEALTH_PROBE_TIMEOUT", 1.0)

    client = await aiohttp_client(make_app())
    body = await (await client.get("/api/dashboard/health", headers={"X-Secret-Key": "dash-secret"})).json()

    ids = [c["id"] for c in body["checks"]]
    assert "ffmpeg" in ids, "the probe that finished before the deadline must survive"
    assert "never" not in ids
    assert body["status"] == "failing"
    timed_out = next(c for c in body["checks"] if c["id"] == "probe")
    assert timed_out["status"] == "fail" and "timed out" in timed_out["detail"]

    # The timeout is cached for the full TTL, so callers cannot stack up more stalled threads.
    assert (await client.get("/api/dashboard/health", headers={"X-Secret-Key": "dash-secret"})).status == 200
    assert handlers._health_cache["payload"]["status"] == "failing"