diff --git a/docs/dev/rest-api/dashboard.md b/docs/dev/rest-api/dashboard.md index e1d8cf68..03a151ae 100644 --- a/docs/dev/rest-api/dashboard.md +++ b/docs/dev/rest-api/dashboard.md @@ -44,7 +44,7 @@ All are `GET`. | Path | Returns | |------|---------| | `/api/dashboard/status` | Version, `code_hash`, bind host and port, `mode` (`full`, `api_only`, `remote_only`), uptime, `requests_total`, `requests_rejected` (401s), `requests_by_key` (label → count), loaded service count, session count, `max_sessions` (`null` when the operator configured no limit), `session_ttl`, job counts by status. The service count includes services that failed to import - use `/api/dashboard/services` for the number that loaded. | -| `/api/dashboard/sessions` | Every live remote session: `id`, `owner` (username or masked key), `creator_ip`, `service`, `title_id`, `titles`, `tracks`, `title` (the first resolved title's display name), `auth_status`, `auth_error`, `server_account` (the server profile lent to the remote session, else `null`), `client`, `actions`, `created_at` and `last_accessed` (ISO 8601) with `created_ts` and `last_accessed_ts` (Unix epoch, the same clock as log `ts`), `age_seconds`, `idle_seconds`, `log_seq`. `client` is whatever the remote client sent as `client` in its session create request: the CLI sends `version` and `platform`, an older client sends nothing and the field is `{}`. `log_seq` is the last sequence number in the remote session's service log - poll `/api/dashboard/sessions/{id}/logs` when it changes. `actions` is the session's request log, newest last, capped at 500: `ts`, `method`, `action` (`titles`, `tracks`, `segments`, `license`, `prompt`, …), `query`, `status`, `ms`, `bytes_in`, `bytes_out`. | +| `/api/dashboard/sessions` | Every live remote session: `id`, `owner` (username or masked key), `creator_ip`, `service`, `title_id`, `titles`, `tracks`, `title` (the display name of the title the client is on: the last one it asked tracks for, else the first resolved title), `auth_status`, `auth_error`, `server_account` (the server profile lent to the remote session, else `null`), `client`, `actions`, `created_at` and `last_accessed` (ISO 8601) with `created_ts` and `last_accessed_ts` (Unix epoch, the same clock as log `ts`), `age_seconds`, `idle_seconds`, `log_seq`. `client` is whatever the remote client sent as `client` in its session create request. The CLI sends `version`, `code_hash` (the commit the client runs, `null` when its source cannot be read), `platform` and `argv`. `argv` is the command line the user ran: proxy and URL userinfo, secret query parameters and credential values become `***`, home and install paths shorten as in the logs, and the line is cut at 3000 characters. An older client sends less or nothing and the field is `{}`. `log_seq` is the last sequence number in the remote session's service log - poll `/api/dashboard/sessions/{id}/logs` when it changes. `actions` is the session's request log, newest last, capped at 500: `ts`, `method`, `action` (`titles`, `tracks`, `segments`, `license`, `prompt`, …), `query`, `status`, `ms`, `bytes_in`, `bytes_out`. | | `/api/dashboard/jobs` | Every download job with full detail, regardless of owner. Empty in `--remote-only` mode. | | `/api/dashboard/logs` | `{"seq": N, "records": [...]}`: the last 1000 log records. `?since=` returns only newer records; `?level=WARNING` sets the minimum level; `?logger=serve` keeps one logger and its children (`aiohttp.access` is the noisiest). `ts` is a Unix epoch in seconds. | | `/api/dashboard/sessions/{id}/logs` | One remote session's service log: `{"session_id", "last_seq", "records": [...]}`. Each record has `seq`, `ts`, `level`, `message`. `?since=` returns only newer records. 404 when the remote session is unknown. | diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 6168e07d..a192b893 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -854,7 +854,7 @@ DELETE /api/session/{id} → tear down, harvest updated cache ### `POST /api/session/create` -Make a remote session for a service and title. Authentication runs asynchronously in the background. This call returns immediately with `status: "authenticating"`, and you then poll the prompt endpoint. The body accepts `service` and `title_id` (both required). It also accepts a broad set of optional keys, because the body allows `additionalProperties`. These are `credentials` (`{username, password, extra?}`), `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, `no_proxy`, `profile`, `cache` (map of forwarded cache files), `client_region`, `proxy_region`, `cdm_type`, `range_`, `vcodec`, `quality`, `best_available`, and any service CLI options. `proxy_region` is the country code the client resolved `proxy` from; the server matches it against its own accounts. +Make a remote session for a service and title. Authentication runs asynchronously in the background. This call returns immediately with `status: "authenticating"`, and you then poll the prompt endpoint. The body accepts `service` and `title_id` (both required). It also accepts a broad set of optional keys, because the body allows `additionalProperties`. These are `credentials` (`{username, password, extra?}`), `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, `no_proxy`, `profile`, `cache` (map of forwarded cache files), `client_region`, `proxy_region`, `cdm_type`, `range_`, `vcodec`, `quality`, `best_available`, `client`, and any service CLI options. `proxy_region` is the country code the client resolved `proxy` from; the server matches it against its own accounts. `client` is a freeform object the dashboard shows as sent (the CLI puts `version`, `code_hash`, `platform` and a redacted `argv` in it); the server ignores it above 4096 bytes of JSON. Service CLI options also travel in a nested `service_params` object, which wins over a flat key with the same name. `profile` at the top level always means the credentials profile, never a service's own `--profile` option. diff --git a/docs/dev/rest-api/remote-sessions.md b/docs/dev/rest-api/remote-sessions.md index 9a8b235e..b1489896 100644 --- a/docs/dev/rest-api/remote-sessions.md +++ b/docs/dev/rest-api/remote-sessions.md @@ -324,7 +324,8 @@ The server mounts all these routes, even in `--remote-only` mode. Paths use the `POST /api/session/create` requires `service` and `title_id`. It also accepts `credentials`, `cookies` (base64 of zlib-compressed Netscape cookie file), `proxy`, `no_proxy`, `profile`, `cache` (a map of `filename → base64(zlib(bytes))`), -`client_region`, `cdm_type`, and the track-selection hints `range_`, `vcodec`, +`client_region`, `cdm_type`, `client` (a freeform identity object the dashboard +shows as sent), and the track-selection hints `range_`, `vcodec`, `quality`, `best_available`, plus arbitrary service CLI options (`additionalProperties: true`). diff --git a/tests/remote/unit/test_session_reporting.py b/tests/remote/unit/test_session_reporting.py new file mode 100644 index 00000000..e99af4e5 --- /dev/null +++ b/tests/remote/unit/test_session_reporting.py @@ -0,0 +1,145 @@ +"""The dashboard's view of a remote session: the live title, and the client's redacted command line.""" + +import asyncio +import shlex +from types import SimpleNamespace +from typing import Any + +import pytest + +from unshackle.core.api import session_store as store_mod +from unshackle.core.api.handlers import find_title_for_track, session_tracks_handler +from unshackle.core.api.session_store import SessionEntry +from unshackle.core.tracks import Tracks +from unshackle.core.utils.redact import redact_text + + +class FakeTitle: + def __init__(self, name: str) -> None: + self.name = name + + def __str__(self) -> str: + return self.name + + +def make_entry() -> SessionEntry: + entry = SessionEntry(session_id="s1", service_tag="EXAMPLE", service_instance=None) + entry.title_map = {"t1": FakeTitle("Example Show 2024 S01E01"), "t3": FakeTitle("Example Show 2024 S01E03")} + return entry + + +def test_summary_falls_back_to_the_first_title() -> None: + summary = make_entry().summary() + assert summary["title_id"] == "t1" + assert summary["title"] == "Example Show 2024 S01E01" + + +def test_summary_reports_the_title_the_client_asked_for() -> None: + entry = make_entry() + entry.current_title_id = "t3" + summary = entry.summary() + assert summary["title_id"] == "t3" + assert summary["title"] == "Example Show 2024 S01E03" + + +def test_summary_ignores_a_title_id_that_left_the_map() -> None: + entry = make_entry() + entry.current_title_id = "gone" + assert entry.summary()["title_id"] == "t1" + + +def test_unknown_track_resolves_to_the_live_title() -> None: + entry = make_entry() + entry.current_title_id = "t3" + assert find_title_for_track("nope", entry) is entry.title_map["t3"] + + +def argv_line(argv: list[str], credentials: dict[str, str] | None = None) -> str: + """Mirror the argv capture in RemoteService.authenticate.""" + secrets = [v for v in (credentials or {}).values() if isinstance(v, str) and len(v) >= 4] + return redact_text(shlex.join(argv), secrets) or "" + + +def test_argv_keeps_the_flags_and_drops_the_proxy_password() -> None: + line = argv_line( + [ + "dl", + "-w", + "s1e1-s1e4", + "-r", + "HDR10", + "-v", + "H.264,H.265", + "--list", + "--proxy", + "http://bob:hunter2@proxy.example.com:8080", + "--remote", + "EXAMPLE", + "https://example.com/title/1", + ] + ) + assert "hunter2" not in line + assert "bob" not in line + assert "***@proxy.example.com:8080" in line + assert "-w s1e1-s1e4" in line + assert "-r HDR10" in line + assert "--list" in line + + +def test_argv_drops_credential_values_and_url_tokens() -> None: + line = argv_line( + ["dl", "--remote", "EXAMPLE", "https://example.com/t?token=abc123"], + credentials={"username": "bob", "password": "hunter2", "extra": "totp-seed"}, + ) + assert "hunter2" not in line + assert "totp-seed" not in line + assert "abc123" not in line + assert "token=***" in line + + +def test_argv_does_not_mask_a_short_credential_value() -> None: + line = argv_line( + ["dl", "-w", "s1e1", "--remote", "EXAMPLE", "t1"], credentials={"username": "s1", "password": "e1"} + ) + assert "-w s1e1" in line + + +@pytest.mark.parametrize("param", ["access_token", "auth_token", "client_secret", "api-key", "X-Api-Key"]) +def test_argv_masks_affixed_secret_query_params(param: str) -> None: + line = argv_line(["dl", "--remote", "EXAMPLE", f"https://example.com/t?{param}=abc123&kind=movie"]) + assert "abc123" not in line + assert f"{param}=***" in line + assert "kind=movie" in line + + +def test_tracks_handler_marks_the_title_live_before_it_fetches(monkeypatch: pytest.MonkeyPatch) -> None: + """The dashboard should see the new title while get_tracks runs, not a title later.""" + published: list[tuple[str, dict[str, Any]]] = [] + seen_during_fetch: list[str | None] = [] + monkeypatch.setattr(store_mod.bus, "publish", lambda topic, data: published.append((topic, data))) + + class StubService: + session = SimpleNamespace(headers={}, cookies=[]) + + def get_tracks(self, title: Any) -> Tracks: + seen_during_fetch.append(published[-1][1]["title"] if published else None) + return Tracks() + + def get_chapters(self, title: Any) -> None: + raise NotImplementedError + + async def run() -> None: + store = store_mod.get_session_store() + entry = await store.create( + service_tag="EXAMPLE", service_instance=StubService(), session_id="live", owner_key=None + ) + entry.title_map = {"t1": FakeTitle("Example Show 2024 S01E01"), "t3": FakeTitle("Example Show 2024 S01E03")} + published.clear() + resp = await session_tracks_handler({"title_id": "t3"}, "live") + assert resp.status == 200, resp.text + assert seen_during_fetch == ["Example Show 2024 S01E03"] + assert entry.current_title_id == "t3" + assert entry.summary()["title"] == "Example Show 2024 S01E03" + await store.delete("live") + + asyncio.run(run()) diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 2647eef1..809241bd 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -3023,6 +3023,9 @@ async def session_tracks_handler( details={"available_titles": list(session.title_map.keys())}, ) + session.current_title_id = str(title_id) + SessionStore.publish_update(session) + try: service_instance = session.service_instance async with session.lock: @@ -3473,7 +3476,7 @@ def find_title_for_track(track_id: str, session: Any) -> Any: if track_id in tracks_dict: return session.title_map.get(t_id) if session.title_map: - return next(iter(session.title_map.values())) + return session.title_map.get(session.current_title_id) or next(iter(session.title_map.values())) return None diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 8206ee6f..fac03588 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -1557,6 +1557,13 @@ async def session_create(request: web.Request) -> web.Response: proxy_region: type: string description: Two-letter country the client resolved its proxy for; picks a server account + client: + type: object + additionalProperties: true + description: | + Freeform client identity, shown to dashboard viewers as sent. The CLI sends + `version`, `code_hash`, `platform` and a redacted `argv`. Ignored above 4096 + bytes of JSON. responses: '200': description: Remote session created; authentication continues in the background diff --git a/unshackle/core/api/session_store.py b/unshackle/core/api/session_store.py index 2989d0a0..c43f16e9 100644 --- a/unshackle/core/api/session_store.py +++ b/unshackle/core/api/session_store.py @@ -34,6 +34,7 @@ class SessionEntry: service_instance: Any titles: Any = None # Titles_T from get_titles() title_map: Dict[str, Any] = field(default_factory=dict) + current_title_id: Optional[str] = None # title the client last asked tracks for tracks: Dict[str, Track] = field(default_factory=dict) tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) @@ -60,13 +61,17 @@ class SessionEntry: from unshackle.core.api.stats import mask_key now = datetime.now(timezone.utc) + title_id = ( + self.current_title_id if self.current_title_id in self.title_map else next(iter(self.title_map), None) + ) + title = self.title_map.get(title_id) if title_id else None return { "id": self.session_id, "owner": mask_key(self.owner_key), "creator_ip": self.creator_ip, "service": self.service_tag, - "title_id": next(iter(self.title_map), None), - "title": str(next(iter(self.title_map.values()), "")) or None, + "title_id": title_id, + "title": str(title) if title else None, "titles": len(self.title_map), "tracks": len(self.tracks), "auth_status": self.auth_status.value, diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index bb06d0b7..f133e3ae 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -11,6 +11,7 @@ import base64 import hashlib import logging import re +import shlex import sys import time from datetime import date as date_ @@ -39,7 +40,7 @@ from unshackle.core.titles.music import Album, Song from unshackle.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.track import Track -from unshackle.core.utils.redact import redact_text, safe_display_url +from unshackle.core.utils.redact import redact_path, redact_text, safe_display_url log = logging.getLogger("remote_service") @@ -908,9 +909,16 @@ class RemoteService: if cache_data: create_data["cache"] = cache_data - from unshackle.core import __version__ + from unshackle.core import __code_hash__, __version__ - create_data["client"] = {"version": __version__, "platform": sys.platform} + secrets = [v for v in (create_data.get("credentials") or {}).values() if isinstance(v, str) and len(v) >= 4] + argv = (redact_path(redact_text(shlex.join(sys.argv[1:]), secrets)) or "")[:3000] + create_data["client"] = { + "version": __version__, + "code_hash": __code_hash__ or None, + "platform": sys.platform, + "argv": argv, + } result = self.client.post("/api/session/create", create_data) self._session_id = result["session_id"] diff --git a/unshackle/core/utils/redact.py b/unshackle/core/utils/redact.py index e06f52a1..ce82da99 100644 --- a/unshackle/core/utils/redact.py +++ b/unshackle/core/utils/redact.py @@ -14,8 +14,11 @@ REDACTED = "***" # user:pass@ userinfo embedded in any URL (proxy URLs, remote server URLs) URL_USERINFO_RE = re.compile(r"(?<=://)[^/@]+@") -# secret-bearing query parameters in URLs that end up in free text -SENSITIVE_QUERY_PARAM_RE = re.compile(r"(?i)\b(password|passwd|pwd|token|api_key|apikey|secret|auth)=([^&#\s\"']+)") +# secret-bearing query parameters in URLs that end up in free text; the affixes catch +# access_token=, client_secret=, api-key= - "_" is a word character, so \b alone misses them +SENSITIVE_QUERY_PARAM_RE = re.compile( + r"(?i)(? Optional[str]: