Files
unshackle/unshackle/core/api/stats.py
T
imSp4rky afaff8e070 feat(api): tag each error with a ref and count errors by service
Every structured error carries a short ref, appended to its message and logged with the request; /api/dashboard/status gains per-service error counts. Gzip now compresses in place.
2026-10-21 14:39:37 -06:00

486 lines
18 KiB
Python

"""Request counters and a log ring buffer for the developer dashboard."""
from __future__ import annotations
import gzip
import hashlib
import json
import logging
import time
from collections import Counter, defaultdict, deque
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any, Deque, Dict, List, Optional, Tuple
from aiohttp import web
from unshackle.core import __code_hash__, __version__
from unshackle.core.api.events import bus
from unshackle.core.config import config
from unshackle.core.utils.redact import redact_secrets
RATE_LIMIT_WINDOW = 3600.0
CDM_OPS = frozenset(
{
"open",
"close",
"set_service_certificate",
"get_service_certificate",
"get_license_challenge",
"parse_license",
"get_keys",
}
)
CDM_ERROR_LIMIT = 300
CDM_SESSION_ID_LIMIT = 64
CDM_DEVICE_LIMIT = 255
def _users() -> Dict[str, Any]:
users = (config.serve or {}).get("users")
return users if isinstance(users, dict) else {}
def _user_config(key: Optional[str]) -> Optional[Dict[str, Any]]:
user = _users().get(key) if key else None
return user if isinstance(user, dict) else None
def configured_key(key: Optional[str]) -> Optional[str]:
"""``key`` when the config knows it (a user API key, the master secret or the dashboard API key), else None.
Open routes answer 200 to any header value, so attributing by the raw header would let an
unauthenticated caller grow ``stats.keys`` without bound.
"""
if not key:
return None
serve = config.serve or {}
dashboard = serve.get("dashboard")
known = {str(k) for k in _users()} | {str(serve.get("api_secret") or "")}
if isinstance(dashboard, dict) and dashboard.get("key"):
known.add(str(dashboard["key"]))
return key if key in known else None
def mask_key(key: Optional[str]) -> str:
"""A display label for an API key: its configured username, else a truncated prefix."""
if not key:
return "anonymous"
user = _user_config(key)
if user and user.get("username"):
return str(user["username"])
return key[:4] + "…"
def key_id(key: Optional[str]) -> str:
"""A stable id for an API key. A hash prefix, so two unnamed keys never collide the way
``mask_key`` does, and the id carries no key material."""
if not key:
return "anonymous"
return hashlib.sha256(key.encode()).hexdigest()[:12]
def key_tier(key: Optional[str]) -> Optional[str]:
"""The tier name an API key references, when it names one that exists."""
user = _user_config(key)
tier = user.get("tier") if user else None
return str(tier) if tier else None
def key_rate_limit(key: Optional[str], serve: Optional[Dict[str, Any]] = None) -> Optional[int]:
"""Requests per hour allowed for a key: its own override, else its tier's, else unlimited.
``serve`` reads a config block that is not yet the live one, as the reload validator does.
"""
if serve is None:
serve = config.serve or {}
users = serve.get("users")
user = users.get(key) if isinstance(users, dict) and key else None
if not isinstance(user, dict):
return None
own = user.get("rate_limit")
if isinstance(own, int) and own > 0:
return own
tiers = serve.get("tiers")
tier = tiers.get(user.get("tier")) if isinstance(tiers, dict) and user.get("tier") else None
limit = tier.get("rate_limit") if isinstance(tier, dict) else None
return limit if isinstance(limit, int) and limit > 0 else None
def rate_limit_error(where: str, value: Any) -> Optional[str]:
"""Why *value* cannot serve as a rate limit, or None when it can. Unset is fine.
``key_rate_limit`` reads anything else as "no limit", so a mistyped value would silently
remove the cap the operator wrote down. It lives beside the reader so the two rules cannot
drift apart.
"""
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
return f"{where}.rate_limit must be a positive whole number of requests per hour, not {value!r}"
return None
@dataclass
class KeyStats:
"""Per-key counters and the current rate-limit window."""
label: str
requests: int = 0
rejected: int = 0
bytes_out: int = 0
last_seen: float = 0.0
window_start: float = 0.0
window_used: int = 0
@dataclass
class ServerStats:
started_at: float = field(default_factory=time.time)
host: str = ""
port: int = 0
mode: str = "full"
requests_total: int = 0
requests_rejected: int = 0
keys: Dict[str, KeyStats] = field(default_factory=dict)
errors: Dict[str, Counter] = field(default_factory=lambda: defaultdict(Counter)) # service -> error_code -> count
def key_stats(self, key: Optional[str]) -> KeyStats:
"""The counters for an API key, created on first sight."""
kid = key_id(key)
entry = self.keys.get(kid)
if entry is None:
entry = KeyStats(label=mask_key(key))
self.keys[kid] = entry
else:
entry.label = mask_key(key)
return entry
@property
def requests_by_key(self) -> Counter:
"""Legacy label-keyed request counts. Two keys sharing a masked label still merge here;
the dashboard reads /api/dashboard/keys for unambiguous per-key figures."""
counter: Counter = Counter()
for entry in self.keys.values():
counter[entry.label] += entry.requests
return counter
def check_rate_limit(self, key: str) -> Optional[int]:
"""Seconds to wait when ``key`` is over its hourly limit, else None after counting the request.
A fixed window, not a sliding one: cheap, and an operator cap does not need the precision.
"""
limit = key_rate_limit(key)
if limit is None:
return None
entry = self.key_stats(key)
now = time.time()
if now - entry.window_start >= RATE_LIMIT_WINDOW:
entry.window_start = now
entry.window_used = 0
if entry.window_used >= limit:
return max(1, int(entry.window_start + RATE_LIMIT_WINDOW - now))
entry.window_used += 1
return None
def to_dict(self) -> Dict[str, Any]:
return {
"version": __version__,
"code_hash": __code_hash__,
"host": self.host,
"port": self.port,
"mode": self.mode,
"started_at": self.started_at,
"uptime_seconds": int(time.time() - self.started_at),
"requests_total": self.requests_total,
"requests_rejected": self.requests_rejected,
"requests_by_key": dict(self.requests_by_key),
"errors": {service: dict(codes) for service, codes in self.errors.items()},
}
stats = ServerStats()
@web.middleware
async def stats_middleware(request: web.Request, handler: Any) -> web.StreamResponse:
from unshackle.core.api.handlers import request_secret_key
from unshackle.core.api.sanitize import sanitize_log
stats.requests_total += 1
started = time.perf_counter()
response = await handler(request)
status = response.get("late_status", response.status)
bytes_out = getattr(response, "content_length", None) or response.body_length
key = None if status == 401 else configured_key(request_secret_key(request))
if status == 401:
stats.requests_rejected += 1
else:
entry = stats.key_stats(key)
entry.requests += 1
entry.last_seen = time.time()
entry.bytes_out += bytes_out
if status >= 400:
entry.rejected += 1
session_id = request.match_info.get("session_id")
if session_id and request.method != "OPTIONS" and not request.path.endswith("/logs"):
from unshackle.core.api.session_store import get_session_store
get_session_store().record_action(
session_id,
{
"ts": time.time(),
"method": request.method,
"action": request.path.split(f"/{session_id}", 1)[-1].strip("/") or "info",
"query": dict(request.query),
"status": status,
"ms": round((time.perf_counter() - started) * 1000, 1),
"bytes_in": request.content_length or 0,
"bytes_out": bytes_out,
},
)
error = response.get("error")
if error:
service = await _request_service(request, session_id)
stats.errors[service][error["code"]] += 1
logging.getLogger("api").warning(
f"Error ref {error['ref']}: {error['code']} {status} {request.method} {sanitize_log(request.path)} "
f"service={service} user={sanitize_log(mask_key(key))}: {sanitize_log(error['message'])}"
)
return response
async def _request_service(request: web.Request, session_id: Optional[str]) -> str:
"""The installed service tag a request is for, from its remote session or its JSON body; ``-`` if none.
Only an installed tag counts, so a caller cannot grow ``stats.errors`` with made-up names.
"""
from unshackle.core.api.session_store import get_session_store
from unshackle.core.services import Services
entry = get_session_store().peek(session_id) if session_id else None
service = entry.service_tag if entry is not None else (await _consumed_json(request)).get("service")
if not isinstance(service, str):
return "-"
tag = Services.get_tag(service)
return tag if tag in Services.get_tags() else "-"
class SeqRing:
"""Keep the last N records in memory, each with a ``seq`` one above the record before it."""
def __init__(self, maxlen: int) -> None:
self.records: Deque[Dict[str, Any]] = deque(maxlen=maxlen)
self.seq = 0
def append(self, record: Dict[str, Any]) -> Dict[str, Any]:
"""Buffer one record; ``seq`` moves only after the append, as in ``EventBus``."""
seq = self.seq + 1
item = {"seq": seq, **record}
self.records.append(item)
self.seq = seq
return item
def since(self, seq: int = 0) -> List[Dict[str, Any]]:
"""The records after ``seq``, oldest first, from a copy: ``append()`` can run on a worker thread."""
return [r for r in self.records.copy() if r["seq"] > seq]
class RingLogHandler(SeqRing, logging.Handler):
"""Keep the last N log records in memory and publish each one on the event bus."""
def __init__(self, maxlen: int = 1000) -> None:
logging.Handler.__init__(self)
SeqRing.__init__(self, maxlen)
def emit(self, record: logging.LogRecord) -> None:
"""Buffer and publish one record."""
item = self.append(
{
"ts": record.created,
"level": record.levelname,
"logger": record.name,
"msg": redact_secrets(self.format(record)),
}
)
bus.publish("log", item)
def since(self, seq: int = 0, level: Optional[str] = None, logger: Optional[str] = None) -> List[Dict[str, Any]]:
"""The records after ``seq`` at or above ``level``, from ``logger`` or one of its children."""
min_level = logging.getLevelName(level.upper()) if level else 0
if not isinstance(min_level, int):
min_level = 0
return [
r
for r in super().since(seq)
if logging.getLevelName(r["level"]) >= min_level
and (not logger or r["logger"] == logger or r["logger"].startswith(logger + "."))
]
ring = RingLogHandler()
cdm_calls = SeqRing(maxlen=5000)
@lru_cache(maxsize=None)
def _cdm_op(pattern: str) -> Optional[Tuple[str, str]]:
"""The ``(drm, op)`` of a route pattern, else None. Cached: the route table does not change."""
parts = pattern.strip("/").split("/")
drm = "widevine"
if parts[:1] == ["playready"]:
drm, parts = "playready", parts[1:]
if len(parts) < 2 or parts[0] != "{device}" or parts[1] not in CDM_OPS:
return None
return drm, parts[1]
def _cdm_route(request: web.Request) -> Optional[Tuple[str, str]]:
"""The ``(drm, op)`` of a CDM device route, else None.
Reads the matched route pattern, not the path, so a device name cannot pass for an operation
or for the ``/playready`` prefix.
"""
resource = request.match_info.route.resource
return _cdm_op(resource.canonical) if resource is not None else None
def _hex_id(value: Any) -> Optional[str]:
"""``value`` in lowercase when it is a hexadecimal string of a plausible length, else None."""
if not isinstance(value, str) or not value or len(value) > CDM_SESSION_ID_LIMIT:
return None
try:
bytes.fromhex(value)
except ValueError:
return None
return value.lower()
def _key_fields(secret_key: Optional[str]) -> Dict[str, str]:
"""The ``key_id`` and ``user`` of an API key: the ``id`` and ``label`` of its ``stats.keys`` entry."""
key = configured_key(secret_key)
return {"key_id": key_id(key), "user": mask_key(key)}
def _reply_json(response: Any) -> Dict[str, Any]:
"""The JSON object a handler answered, or ``{}``. ``compression_middleware`` runs inside the
CDM log middleware, so the body can arrive as gzip."""
try:
body = response.body
if response.headers.get("Content-Encoding") == "gzip":
body = gzip.decompress(body)
data = json.loads(body)
except Exception:
return {}
return data if isinstance(data, dict) else {}
async def _consumed_json(request: web.Request) -> Dict[str, Any]:
"""The JSON object of the request body when the handler consumed it (aiohttp caches that read), else ``{}``.
A body nobody read belongs to a call refused before the handler, a 401 or a 429. It stays
unread: reading it would spend time and memory on an unauthenticated caller, and wait on a
body that never finishes arriving.
"""
if not request.content.at_eof():
return {}
try:
body = await request.json()
except Exception:
return {}
return body if isinstance(body, dict) else {}
async def _cdm_record(request: web.Request, drm: str, op: str, outcome: Any) -> Dict[str, Any]:
"""The record of one call. ``outcome`` is the response, or the exception the handler raised.
An exception that is not an HTTP one gives its class name only: its text can hold the
presented API key. The reply body is read only for a refusal and for the session id of an
``open``, so a challenge, a licence or a content key never passes through here.
"""
from unshackle.core.api.handlers import request_secret_key
reply: Dict[str, Any] = {}
if isinstance(outcome, web.StreamResponse):
status = outcome.status
if status >= 400 or op == "open":
reply = _reply_json(outcome)
message = reply.get("message")
error = message if isinstance(message, str) else outcome.reason
else:
status, error = 500, type(outcome).__name__
ok = status < 400
if op == "open":
data = reply.get("data") if ok else None
session_id = data.get("session_id") if isinstance(data, dict) else None
elif op == "close":
session_id = request.match_info.get("session_id")
else:
session_id = (await _consumed_json(request)).get("session_id")
return {
"drm": drm,
"op": op,
"device": request.match_info["device"][:CDM_DEVICE_LIMIT],
**_key_fields(request_secret_key(request)),
"session_id": _hex_id(session_id),
"status": status,
"ok": ok,
"error": None if ok else str(error)[:CDM_ERROR_LIMIT],
}
async def _record_cdm_call(
request: web.Request, route: Tuple[str, str], ts: float, started: float, outcome: Any
) -> None:
"""Append one call to ``cdm_calls``. Logs a failure and does not raise it: it must never fail the CDM call."""
ms = round((time.perf_counter() - started) * 1000, 1)
try:
cdm_calls.append({"ts": ts, **await _cdm_record(request, *route, outcome), "ms": ms})
except Exception:
logging.getLogger("serve").debug("CDM call not recorded", exc_info=True)
@web.middleware
async def cdm_log_middleware(request: web.Request, handler: Any) -> web.StreamResponse:
"""Record each call to a CDM device route in ``cdm_calls``.
Must run outside the serve authentication middleware, so the log holds a 401 too.
"""
route = _cdm_route(request)
if route is None:
return await handler(request)
ts, started = time.time(), time.perf_counter()
try:
response = await handler(request)
except Exception as e:
await _record_cdm_call(request, route, ts, started, e)
raise
await _record_cdm_call(request, route, ts, started, response)
return response
def cdm_sessions(app: web.Application) -> List[Dict[str, Any]]:
"""One row per live Cdm object: its open session count and the most it allows.
Both CDM libraries refuse an ``open`` only above ``MAX_NUM_OF_SESSIONS``, so the true limit
is one more. A Cdm with a ``SESSION_TIMEOUT`` drops older sessions on its next ``open``,
so those do not count as open.
"""
now = time.time()
rows: List[Dict[str, Any]] = []
for drm, owner in (("widevine", app), ("playready", app.get("playready_app"))):
cdms = owner.get("cdms") if owner is not None else None
for (secret_key, device), cdm in (cdms or {}).items():
sessions = getattr(cdm, "_Cdm__sessions", {}).values()
timeout = getattr(cdm, "SESSION_TIMEOUT", None)
rows.append(
{
"drm": drm,
"device": device,
**_key_fields(secret_key),
"open": len(sessions) if timeout is None else sum(now - s.opened_at <= timeout for s in sessions),
"max": cdm.MAX_NUM_OF_SESSIONS + 1,
}
)
return rows