From eeaf16cd83808c79fcb2db5e12fc0ee7282ef0fd Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Fri, 23 Oct 2026 09:42:01 -0600 Subject: [PATCH] fix(remote): keep each session cookie's domain, path and secure flag The client set server cookies with no domain and sent all of them to every host, so a CDN refused segment requests with 400 header too large. rnet sessions now send cookies too. --- docs/dev/rest-api/endpoints.md | 5 +- docs/dev/rest-api/remote-sessions.md | 8 +- tests/remote/unit/test_client_device_cdm.py | 2 +- tests/test_client_session_auth.py | 123 +++++++++++++++++++- unshackle/core/api/handlers.py | 40 +++++-- unshackle/core/remote_service.py | 24 +++- unshackle/core/session.py | 36 +++++- 7 files changed, 211 insertions(+), 27 deletions(-) diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index bb8e02d6..d7680294 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -1096,6 +1096,7 @@ Get tracks and chapters for a specific title within the remote session. Body: `{ "track_manifests": [ { "track_id": "s-1", "type": "dash", "url": "https://...", "data": "" } ], "session_headers": { "User-Agent": "..." }, "session_cookies": { "sid": "..." }, + "session_cookie_jar": [ { "name": "sid", "value": "...", "domain": ".example.com", "path": "/", "secure": true } ], "server_cdm_type": "widevine" } ``` @@ -1115,11 +1116,13 @@ Get the download URL, DRM info, headers, cookies, and raw track data for the sel ```json { "tracks": { - "v-1": { "descriptor": "DASH", "url": "https://...", "drm": [ ... ], "headers": { ... }, "cookies": { ... }, "data": { ... } } + "v-1": { "descriptor": "DASH", "url": "https://...", "drm": [ ... ], "headers": { ... }, "cookies": { ... }, "cookie_jar": [ ... ], "data": { ... } } } } ``` +`cookies` gives names and values only. `cookie_jar` gives the same cookies with their domain, path and `secure` flag, in the `session_cookie_jar` form of the tracks response. + | Status | Error code | Meaning | | --- | --- | --- | | `200` | - | Segment info returned. | diff --git a/docs/dev/rest-api/remote-sessions.md b/docs/dev/rest-api/remote-sessions.md index b03a4d0c..e64f9dc2 100644 --- a/docs/dev/rest-api/remote-sessions.md +++ b/docs/dev/rest-api/remote-sessions.md @@ -214,8 +214,10 @@ hood the client walks a remote session through its lifecycle. downloading. A service can build an AdaptationSet of its own that the served manifest does not contain. Each such track gets a one-AdaptationSet MPD in `track_manifests`, and the client re-parses that instead. Any - `session_headers` / `session_cookies` the server used are merged into the - client's local HTTP session. + `session_headers` and cookies the server used are merged into the + client's local HTTP session. Each cookie in `session_cookie_jar` keeps its + domain, path and `secure` flag, so it goes only to its own hosts. `session_cookies` carries + the same cookies as names and values only, for older clients. The tracks response repeats the title. A service can rewrite title fields inside `get_tracks`, so the client takes this copy in place of the one from the title @@ -653,7 +655,7 @@ remote session API and a good template for any client. - **Flow.** `authenticate()` → `create` (+ poll `prompt` every 2s up to a 600s deadline, answering `pending_input` prompts. With stdin closed the client logs an error and raises `SystemExit(1)` instead of posting an empty answer) → `get_titles()` → `get_tracks()` - (merging returned `session_headers`/`session_cookies`, re-parsing `manifests`) → + (merging returned `session_headers`/`session_cookie_jar`, re-parsing `manifests`) → get the licence in proxy mode or `server_cdm` mode → `close()` (`DELETE`, saving any returned `cache`). - **Server resolution.** `resolve_server()` reads `config.remote_services.` diff --git a/tests/remote/unit/test_client_device_cdm.py b/tests/remote/unit/test_client_device_cdm.py index b671c90e..6488760b 100644 --- a/tests/remote/unit/test_client_device_cdm.py +++ b/tests/remote/unit/test_client_device_cdm.py @@ -389,7 +389,7 @@ def test_a_lent_device_session_keeps_its_auth_headers_and_track_secrets() -> Non headers, cookies = handlers.client_session_auth(session) assert headers == {"User-Agent": "ua"} - assert cookies == {"account": "client"} + assert handlers.cookie_values(cookies) == {"account": "client"} assert handlers.scrub_track_data({"token": "server", "id": 1}, session) == {"id": 1} diff --git a/tests/test_client_session_auth.py b/tests/test_client_session_auth.py index 1b160694..27b64bdc 100644 --- a/tests/test_client_session_auth.py +++ b/tests/test_client_session_auth.py @@ -1,13 +1,15 @@ from types import SimpleNamespace import pytest +import requests from unshackle.core.api.errors import APIError from unshackle.core.api.handlers import client_session_auth, resolve_handler_proxy, scrub_track_data +from unshackle.core.remote_service import apply_session_cookies def _session(client_auth): - cookie = SimpleNamespace(name="sid", value="secret") + cookie = SimpleNamespace(name="sid", value="secret", domain=".example.com", path="/") svc = SimpleNamespace(headers={"Authorization": "Bearer x", "User-Agent": "ua"}, cookies=[cookie]) return SimpleNamespace(service_instance=SimpleNamespace(session=svc), client_auth=client_auth) @@ -15,14 +17,129 @@ def _session(client_auth): def test_client_login_keeps_auth(): headers, cookies = client_session_auth(_session(True)) assert headers == {"Authorization": "Bearer x", "User-Agent": "ua"} - assert cookies == {"sid": "secret"} + assert cookies == [{"name": "sid", "value": "secret", "domain": ".example.com", "path": "/", "secure": False}] assert scrub_track_data({"token": "t", "hls": {}}, _session(True)) == {"token": "t", "hls": {}} +def test_client_cookies_keep_their_domain(): + session = requests.Session() + _, cookies = client_session_auth(_session(True)) + apply_session_cookies(session, {"session_cookie_jar": cookies}) + assert session.cookies.get("sid", domain=".example.com") == "secret" + cdn = session.prepare_request(requests.Request("GET", "https://cdn.other.net/seg.mp4")) + assert "Cookie" not in cdn.headers + own = session.prepare_request(requests.Request("GET", "https://www.example.com/api")) + assert own.headers["Cookie"] == "sid=secret" + + +def test_client_takes_old_server_cookies_without_domain(): + session = requests.Session() + apply_session_cookies(session, {"session_cookies": {"sid": "secret"}}) + assert session.cookies.get("sid") == "secret" + + +def test_client_keeps_secure_cookies_off_http(): + session = requests.Session() + jar = [{"name": "sid", "value": "s", "domain": ".example.com", "path": "/", "secure": True}] + apply_session_cookies(session, {"session_cookie_jar": jar}) + plain = session.prepare_request(requests.Request("GET", "http://cdn.example.com/seg.mp4")) + assert "Cookie" not in plain.headers + tls = session.prepare_request(requests.Request("GET", "https://cdn.example.com/seg.mp4")) + assert tls.headers["Cookie"] == "sid=s" + + +def test_client_skips_malformed_server_cookies(): + session = requests.Session() + jar = [{"value": "v"}, "str", {"name": "n", "value": 1}, {"name": "ok", "value": "v", "domain": ".example.com"}] + apply_session_cookies(session, {"session_cookie_jar": jar}) + assert [c.name for c in session.cookies] == ["ok"] + session = requests.Session() + apply_session_cookies(session, {"session_cookie_jar": 5, "session_cookies": ["x"]}) + assert not session.cookies + + +def test_rnet_session_sends_client_and_set_cookies(): + from unshackle.core.session import RnetSession + + svc = RnetSession() + svc.cookies.set("sid", "stale", domain=".example.com") + svc.cookies.set("cdn", "c", domain="cdn.other.net") + svc.export_origin_cookies = lambda: {"https://www.example.com": "sid=live; cf=1"} + _, cookies = client_session_auth(SimpleNamespace(service_instance=SimpleNamespace(session=svc), client_auth=True)) + assert {(c["name"], c["value"], c["domain"]) for c in cookies} == { + ("sid", "live", "www.example.com"), + ("cf", "1", "www.example.com"), + ("cdn", "c", "cdn.other.net"), + } + + +def test_tracks_response_carries_the_cookie_jar(): + import asyncio + import json + + from unshackle.core.api import session_store as store_mod + from unshackle.core.api.handlers import session_tracks_handler + from unshackle.core.tracks import Tracks + + cookie = SimpleNamespace(name="sid", value="secret", domain=".example.com", path="/", secure=True) + + class StubService: + session = SimpleNamespace(headers={}, cookies=[cookie]) + + def get_tracks(self, title): + return Tracks() + + def get_chapters(self, title): + raise NotImplementedError + + async def run(): + store = store_mod.get_session_store() + entry = await store.create( + service_tag="EXAMPLE", service_instance=StubService(), session_id="jar", owner_key=None + ) + entry.title_map = {"t1": "Example Title"} + entry.client_auth = True + try: + return json.loads((await session_tracks_handler({"title_id": "t1"}, "jar")).body) + finally: + await store.delete("jar") + + body = asyncio.run(run()) + assert body["session_cookies"] == {"sid": "secret"} + assert body["session_cookie_jar"] == [ + {"name": "sid", "value": "secret", "domain": ".example.com", "path": "/", "secure": True} + ] + + +def test_remote_get_tracks_scopes_server_cookies(): + import logging + + from unshackle.core.remote_service import RemoteService + + jar = [{"name": "sid", "value": "secret", "domain": ".example.com", "path": "/"}] + service = object.__new__(RemoteService) + service.__dict__.update( + _tracks_by_title={}, + _chapters_by_title={}, + _sample_decrypt_tracks=set(), + _server_cdm=None, + _session=requests.Session(), + _session_id="s", + service_tag="EXAMPLE", + log=logging.getLogger("test"), + client=SimpleNamespace(post=lambda path, body: {"session_cookie_jar": jar}), + drain_server_logs=lambda: None, + ) + service.get_tracks(SimpleNamespace(id="t1")) + cdn = service._session.prepare_request(requests.Request("GET", "https://cdn.other.net/seg.mp4")) + assert "Cookie" not in cdn.headers + assert service._session.cookies.get("sid", domain=".example.com") == "secret" + + def test_server_login_strips_auth(): headers, cookies = client_session_auth(_session(False)) assert headers == {"User-Agent": "ua"} - assert cookies == {} + assert cookies == [] assert scrub_track_data({"token": "t", "hls": {}}, _session(False)) == {"hls": {}} diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index da76d12e..85f2e2a3 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -34,7 +34,7 @@ from unshackle.core.drm import cbcs from unshackle.core.providers.anilist import parse_anilist_ref from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy from unshackle.core.services import Services -from unshackle.core.session import utf8_text +from unshackle.core.session import RnetSession, utf8_text from unshackle.core.titles import Episode, Movie, Song, Title_T from unshackle.core.titles.music import SONG_FIELDS from unshackle.core.tracks import Audio, Subtitle, Tracks, Video @@ -4068,8 +4068,8 @@ async def session_titles_handler(session_id: str, request: Optional[web.Request] ) -def client_session_auth(session: Any) -> tuple[Dict[str, str], Dict[str, str]]: - """The service session headers and cookies a remote client may receive. +def client_session_auth(session: Any) -> tuple[Dict[str, str], list[Dict[str, Any]]]: + """The service session headers and cookies (name, value, domain, path, secure) a remote client may receive. A client whose own login succeeded gets them back. For a server account the cookie jar stays on the server and the server drops the auth-bearing headers, so the client cannot @@ -4078,18 +4078,31 @@ def client_session_auth(session: Any) -> tuple[Dict[str, str], Dict[str, str]]: """ svc_session = session.service_instance.session headers = dict(svc_session.headers) if hasattr(svc_session, "headers") and svc_session.headers else {} - cookies: Dict[str, str] = {} - if hasattr(svc_session, "cookies"): - for cookie in svc_session.cookies: - if hasattr(cookie, "name") and hasattr(cookie, "value"): - cookies[cookie.name] = cookie.value + cookies: list[Dict[str, Any]] = [] + jar = svc_session.scoped_cookies() if isinstance(svc_session, RnetSession) else getattr(svc_session, "cookies", []) + for cookie in jar: + if hasattr(cookie, "name") and hasattr(cookie, "value"): + cookies.append( + { + "name": cookie.name, + "value": cookie.value, + "domain": getattr(cookie, "domain", None) or "", + "path": getattr(cookie, "path", None) or "/", + "secure": bool(getattr(cookie, "secure", False)), + } + ) if not client_owns_login(session): headers = {k: v for k, v in headers.items() if not CONFIG_SECRET_KEY_RE.search(str(k))} if not session.client_auth: - cookies = {} + cookies = [] return headers, cookies +def cookie_values(cookies: list[Dict[str, Any]]) -> Dict[str, str]: + """The ``{name: value}`` form of :func:`client_session_auth` cookies, for clients that predate the jar.""" + return {c["name"]: c["value"] for c in cookies} + + def client_owns_login(session: Any) -> bool: """Whether the login state of ``session`` is the client's to take back. @@ -4232,7 +4245,8 @@ async def session_tracks_handler( "manifests": manifests, "track_manifests": track_manifests, "session_headers": session_headers, - "session_cookies": session_cookies, + "session_cookies": cookie_values(session_cookies), + "session_cookie_jar": session_cookies, "server_cdm": session.server_cdm, "server_cdm_type": server_cdm_type, "sample_decrypt_tracks": [ @@ -4269,6 +4283,7 @@ async def session_segments_handler( try: result: Dict[str, Any] = {} + headers, cookies = client_session_auth(session) for track_id in track_ids: track = session.tracks.get(track_id) @@ -4285,10 +4300,11 @@ async def session_segments_handler( "descriptor": descriptor_name, "url": str(track.url) if track.url else None, "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + "headers": headers, + "cookies": cookie_values(cookies), + "cookie_jar": cookies, } - track_info["headers"], track_info["cookies"] = client_session_auth(session) - # Include manifest-specific data for segment resolution. Round-trip through # JSON so any non-serializable value becomes its str() (default=str). if hasattr(track, "data") and track.data: diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py index 889a4f3f..b8a365fd 100644 --- a/unshackle/core/remote_service.py +++ b/unshackle/core/remote_service.py @@ -426,6 +426,27 @@ def apply_service_track_data(tracks: Tracks, data: Dict[str, Any]) -> None: track.data.update(extra) +def apply_session_cookies(session: requests.Session, result: dict) -> None: + """Copy the server's session cookies into ``session``, each scoped to its own domain. + + An older server sends only ``session_cookies`` names and values, so those cookies go to every host. + It skips malformed entries. + """ + jar = result.get("session_cookie_jar") + if not isinstance(jar, list): + old = result.get("session_cookies") + jar = [{"name": k, "value": v} for k, v in old.items()] if isinstance(old, dict) else [] + for c in jar: + if isinstance(c, dict) and isinstance(c.get("name"), str) and isinstance(c.get("value"), str): + session.cookies.set( + c["name"], + c["value"], + domain=str(c.get("domain") or ""), + path=str(c.get("path") or "/"), + secure=bool(c.get("secure")), + ) + + def resolve_track_manifests(tracks: Tracks, track_manifests: list) -> None: """Fill track.data from the per-track MPD fragments the server sent. @@ -1651,8 +1672,7 @@ class RemoteService: for k, v in result.get("session_headers", {}).items(): if k.lower() not in ("host", "content-length", "content-type"): self._session.headers[k] = v - for k, v in result.get("session_cookies", {}).items(): - self._session.cookies.set(k, v) + apply_session_cookies(self._session, result) resolve_track_manifests(tracks, result.get("track_manifests", [])) resolve_manifest_data(tracks, result.get("manifests", [])) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 61861b96..8e5e1c3a 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -17,6 +17,7 @@ from urllib.parse import urlencode, urlparse, urlunparse import rnet from requests import HTTPError, Request, Response +from requests.cookies import create_cookie from requests.structures import CaseInsensitiveDict from unshackle.core.config import config @@ -664,14 +665,19 @@ class RnetSession: self._client = client @staticmethod - def set_cookie_header(client: rnet.BlockingClient, origin: str, header: str) -> None: - """Set every cookie in a ``name=value; ...`` header string on one origin of *client*.""" + def parse_cookie_header(header: str) -> Iterator[tuple[str, str]]: + """The ``(name, value)`` pairs of a ``name=value; ...`` header string.""" for pair in header.split(";"): name, separator, value = pair.partition("=") - if not separator: - continue + if separator: + yield name.strip(), value.strip() + + @staticmethod + def set_cookie_header(client: rnet.BlockingClient, origin: str, header: str) -> None: + """Set every cookie in a ``name=value; ...`` header string on one origin of *client*.""" + for name, value in RnetSession.parse_cookie_header(header): try: - client.set_cookie(origin, rnet.Cookie(name.strip(), value.strip())) + client.set_cookie(origin, rnet.Cookie(name, value)) except Exception: pass @@ -705,6 +711,26 @@ class RnetSession: exported[origin] = raw.decode("utf-8", errors="replace") return exported + def scoped_cookies(self) -> CookieJar: + """Every cookie this HTTP session sends, each scoped to its domain. + + The live client's cookies of each requested origin come first, then the cookies set + through :attr:`cookies` for any other host. + """ + jar = CookieJar() + hosts: dict[str, set[str]] = {} + for origin, header in self.export_origin_cookies().items(): + host = urlparse(origin).hostname or "" + for name, value in self.parse_cookie_header(header): + jar.set_cookie(create_cookie(name, value, domain=host)) + hosts.setdefault(name, set()).add(host) + for domain, values in self.cookies.get_dict_by_domain().items(): + bare = domain.lstrip(".") + for name, value in values.items(): + if not any(h == bare or h.endswith("." + bare) for h in hosts.get(name, ())): + jar.set_cookie(create_cookie(name, value, domain=domain)) + return jar + def import_origin_cookies(self, exported: dict[str, str]) -> None: """Load cookie headers from :meth:`export_origin_cookies` into this HTTP session's client.""" if not exported: