From 96c3840b82380cfee8e970666d6388a577badc35 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Sat, 31 Oct 2026 22:57:22 -0600 Subject: [PATCH] fix(session): keep cookie paths so same-name cookies stay on their own path RnetCookieAdapter stores cookies by domain, path and name and sends path, secure and a leading-dot domain to rnet. It also takes server-rotated values, and worker children and track sessions get the cookies with their paths. --- docs/getting-started/quickstart.md | 3 + tests/core/test_session_headers.py | 96 +++++++- tests/downloaders/test_session_spec.py | 31 ++- unshackle/core/downloaders/requests.py | 10 +- unshackle/core/session.py | 305 ++++++++++++++----------- unshackle/core/tracks/track.py | 19 +- 6 files changed, 297 insertions(+), 167 deletions(-) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 62819448..6680df85 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -135,6 +135,9 @@ Services that require a login read either **cookies** or **credentials**. rows a text editor broke. unshackle refuses a JSON export. Export the cookies again in Netscape format. + unshackle sends each cookie only to the host and path that its row names, as a + browser does. A domain that starts with a dot also covers its subdomains. + - **Credentials**: store a username and password per service in your config: ```yaml title="unshackle.yaml" diff --git a/tests/core/test_session_headers.py b/tests/core/test_session_headers.py index 7d4ce7aa..cb7aa6d0 100644 --- a/tests/core/test_session_headers.py +++ b/tests/core/test_session_headers.py @@ -37,6 +37,10 @@ class Recorder(BaseHTTPRequestHandler): self.send_header("Set-Cookie", "sid=abc123; Path=/") elif self.path.startswith("/set/"): self.send_header("Set-Cookie", f"{self.path[len('/set/') :]}; Path=/") + elif self.path.startswith("/scoped/set/"): + self.send_header("Set-Cookie", f"{self.path[len('/scoped/set/') :]}; Path=/scoped") + elif self.path.startswith("/scoped/expire/"): + self.send_header("Set-Cookie", f"{self.path[len('/scoped/expire/') :]}=; Max-Age=0; Path=/scoped") self.end_headers() self.wfile.write(body) @@ -57,7 +61,7 @@ def server() -> Iterator[str]: httpd.server_close() -def make_cookie(name: str, value: str, domain: str) -> Cookie: +def make_cookie(name: str, value: str, domain: str, path: str = "/") -> Cookie: return Cookie( version=0, name=name, @@ -67,7 +71,7 @@ def make_cookie(name: str, value: str, domain: str) -> Cookie: domain=domain, domain_specified=True, domain_initial_dot=domain.startswith("."), - path="/", + path=path, path_specified=True, secure=False, expires=None, @@ -333,3 +337,91 @@ def test_the_rebuilt_client_is_published_only_when_it_is_ready(server: str, monk assert seen and all(client is old for client in seen) assert session._client is not old + + +def path_session(*cookies: Cookie) -> RnetSession: + session = RnetSession() + session.cookies.update(list(cookies)) + return session + + +def test_same_name_cookies_reach_only_their_own_path(server: str) -> None: + """Two cookies with one name on two paths must each reach only their own path.""" + session = path_session(make_cookie("h", "A", "127.0.0.1", "/a"), make_cookie("h", "B", "127.0.0.1", "/b")) + + for path, expected in (("/a/seg", "h=A"), ("/b/seg", "h=B"), ("/", "")): + session.get(f"{server}{path}") + assert Recorder.cookies_sent[-1] == expected + + session.headers["X-Rebuild"] = "1" + del session.headers["X-Rebuild"] + session.get(f"{server}/a/seg") + assert Recorder.cookies_sent[-1] == "h=A" + + +def test_delete_drops_every_path_copy(server: str) -> None: + """``del cookies[name]`` must reach a cookie stored on a path, not only the root-path one.""" + session = path_session(make_cookie("h", "A", "127.0.0.1", "/a"), make_cookie("h", "B", "127.0.0.1", "/b")) + session.get(f"{server}/a/seg") + + del session.cookies["h"] + + session.get(f"{server}/a/seg") + session.get(f"{server}/b/seg") + assert Recorder.cookies_sent[-2:] == ["", ""] + assert list(session.cookies.jar) == [] + + +def test_clear_by_path_spares_the_other_path(server: str) -> None: + session = path_session(make_cookie("h", "A", "127.0.0.1", "/a"), make_cookie("h", "B", "127.0.0.1", "/b")) + session.get(f"{server}/a/seg") + + session.cookies.clear(domain="127.0.0.1", path="/a", name="h") + + session.get(f"{server}/a/seg") + session.get(f"{server}/b/seg") + assert Recorder.cookies_sent[-2:] == ["", "h=B"] + assert session.cookies.get("h", path="/b") == "B" + assert session.cookies.get("h", path="/a") is None + + +def test_a_rotated_path_cookie_survives_a_rebuild(server: str) -> None: + """A value the server set for a stored path cookie must win after ``rebuild_client`` and on save.""" + session = path_session(make_cookie("h", "STALE", "127.0.0.1", "/scoped")) + session.headers["X-Auth"] = "secret" + + session.get(f"{server}/scoped/set/h=FRESH") + session.headers.pop("X-Auth", None) + session.get(f"{server}/scoped/seg") + + assert Recorder.cookies_sent[-1] == "h=FRESH" + assert [(c.path, c.value) for c in session.cookies.jar] == [("/scoped", "FRESH")] + + +def test_a_server_expiry_drops_the_stored_cookie(server: str) -> None: + session = path_session(make_cookie("h", "STALE", "127.0.0.1", "/scoped")) + session.headers["X-Auth"] = "secret" + + session.get(f"{server}/scoped/expire/h") + session.headers.pop("X-Auth", None) + session.get(f"{server}/scoped/seg") + + assert Recorder.cookies_sent[-1] == "" + assert "h" not in session.cookies + + +def test_only_a_leading_dot_widens_a_cookie_to_subdomains() -> None: + """A Netscape row with a leading dot covers subdomains; any other domain stays host-only.""" + assert RnetCookieAdapter.to_rnet(make_cookie("s", "1", ".example.com")).domain == "example.com" + assert RnetCookieAdapter.to_rnet(make_cookie("s", "1", "www.example.com")).domain is None + + +def test_the_latest_write_answers_a_lookup_by_name() -> None: + session = path_session(make_cookie("tok", "file", ".example.com")) + session.cookies.update({"tok": "dict"}) + assert session.cookies.get("tok") == "dict" + assert session.cookies["tok"] == "dict" + session.cookies.set("tok", "set", domain=".example.com") + assert session.cookies.get("tok") == "set" + assert session.cookies.get("tok", domain="") == "dict" + assert [c.value for c in session.cookies.jar] == ["set"] diff --git a/tests/downloaders/test_session_spec.py b/tests/downloaders/test_session_spec.py index 95b2e581..b681b883 100644 --- a/tests/downloaders/test_session_spec.py +++ b/tests/downloaders/test_session_spec.py @@ -47,7 +47,7 @@ def test_rnet_spec_round_trips_cookie_domains() -> None: rebuilt = rebuild_session(pickle.loads(pickle.dumps(spec)), 1) assert rebuilt is not None - assert rebuilt.cookies.get_dict_by_domain() == ns.cookies.get_dict_by_domain() + assert [vars(c) for c in rebuilt.cookies.snapshot()] == [vars(c) for c in ns.cookies.snapshot()] assert rebuilt.cookies.get_dict() == ns.cookies.get_dict() assert rebuilt.cookies.get("cdn_token", domain="cdn.example.com") == "abc" assert rebuilt.cookies.get("api_token", domain="api.example.com") == "def" @@ -71,17 +71,36 @@ def test_rnet_spec_cookies_carry_domain_information() -> None: spec = build_session_spec(ns) assert spec is not None - cookies = spec["cookies"] - assert set(cookies) == {"cdn.example.com", "api.example.com", ""} - assert all(isinstance(jar, dict) for jar in cookies.values()) - assert cookies["cdn.example.com"] == {"cdn_token": "abc"} + assert {(c.domain, c.path, c.name, c.value) for c in spec["cookies"]} == { + ("cdn.example.com", "/", "cdn_token", "abc"), + ("api.example.com", "/", "api_token", "def"), + ("", "/", "loose", "ghi"), + } + + +def test_rnet_spec_keeps_same_name_cookies_on_their_paths() -> None: + """Same-name cookies on two paths must reach a child apart, or a stale one goes to every path.""" + ns = make_session("Chrome131") + ns.cookies.set("hdntl", "one", domain=".cdn.example.com", path="/t/1") + ns.cookies.set("hdntl", "two", domain=".cdn.example.com", path="/t/2") + + spec = build_session_spec(ns) + assert spec is not None + rebuilt = rebuild_session(pickle.loads(pickle.dumps(spec)), 1) + assert rebuilt is not None + + assert rebuilt.cookies.get("hdntl", path="/t/1") == "one" + assert rebuilt.cookies.get("hdntl", path="/t/2") == "two" def test_rnet_spec_is_picklable() -> None: spec = build_session_spec(rnet_session_with_cookies()) assert spec is not None - assert pickle.loads(pickle.dumps(spec)) == spec + loaded = pickle.loads(pickle.dumps(spec)) + # stdlib Cookie has no __eq__, so compare the cookies by their fields + assert [vars(c) for c in loaded.pop("cookies")] == [vars(c) for c in spec.pop("cookies")] + assert loaded == spec def test_rnet_spec_without_impersonate_is_not_rebuildable() -> None: diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index 8f09420b..b241625b 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -1247,7 +1247,7 @@ def build_session_spec(session: Optional[Any]) -> Optional[dict[str, Any]]: "kind": "rnet", "impersonate": name, "headers": dict(session.headers), - "cookies": session.cookies.get_dict_by_domain(), + "cookies": session.cookies.snapshot(), # stdlib Cookies pickle with their paths "origin_cookies": session.export_origin_cookies(), "proxy": session.proxies.get("all") or session.proxies.get("https") or session.proxies.get("http"), } @@ -1290,13 +1290,7 @@ def rebuild_session(spec: dict[str, Any], max_workers: int) -> Optional[Any]: ns = make_session(spec["impersonate"]) if spec.get("headers"): ns.headers.update(spec["headers"]) - for domain, jar in spec.get("cookies", {}).items(): - if domain: - for name, value in jar.items(): - ns.cookies.set(name, value, domain=domain) - else: - # no domain to scope to; update() reproduces the parent's localhost-scoped state - ns.cookies.update(jar) + ns.cookies.update(spec.get("cookies") or []) if spec.get("proxy"): ns.proxies.update({"all": spec["proxy"]}) ns.import_origin_cookies(spec.get("origin_cookies") or {}) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index dedebb92..b4b98d76 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -10,9 +10,10 @@ import re import sys import time from collections.abc import Iterator, MutableMapping +from copy import copy from datetime import datetime, timezone from email.utils import parsedate_to_datetime -from http.cookiejar import CookieJar +from http.cookiejar import Cookie, CookieJar from typing import Any, Optional, Union from urllib.parse import urlencode, urlparse, urlunparse @@ -288,210 +289,231 @@ class RnetSessionHeaders(CaseInsensitiveDict): class RnetCookieAdapter(MutableMapping): - """Cookie adapter that bridges requests-style cookie access to rnet.""" + """Cookie adapter that bridges requests-style cookie access to rnet. + + One store keyed by (domain, path, name) holds every cookie set through this class, in write + order. Same-name cookies on different paths stay apart on the wire, and the latest write + answers a lookup by name. + """ def __init__(self, client: Any, session: Optional[RnetSession] = None) -> None: self._client = client self._session = session - self._cookies: dict[str, dict[str, str]] = {} - self._flat: dict[str, str] = {} - self._original_cookies: list[Any] = [] + self._store: dict[tuple[str, str, str], Cookie] = {} + self._from_jar: set[tuple[str, str, str]] = set() - def set_cookie_on_client(self, url: str, name: str, value: str) -> None: - """Set a cookie on the rnet client, or buffer locally if the client is not yet created.""" + @staticmethod + def client_url(cookie: Cookie) -> str: + """The URL rnet stores *cookie* under; a domain-less cookie goes to localhost.""" + host = (cookie.domain or "").lstrip(".") + return f"https://{host}" if host else "https://localhost" + + @staticmethod + def to_rnet(cookie: Cookie) -> rnet.Cookie: + """The rnet form of *cookie*. + + A leading dot marks a cookie for subdomains too, as in a Netscape cookie file; without + one the cookie stays on its own host. Expiry stays out, so no layer drops a cookie the + cookie loader deliberately kept. + """ + domain = cookie.domain or "" + return rnet.Cookie( + cookie.name, + cookie.value or "", + domain=domain if domain.startswith(".") else None, + path=cookie.path or "/", + secure=bool(cookie.secure), + ) + + def push(self, cookie: Cookie) -> None: + """Set *cookie* on the rnet client; before the client exists the store buffers it.""" if self._client is not None: try: - self._client.set_cookie(url, rnet.Cookie(name, value)) + self._client.set_cookie(self.client_url(cookie), self.to_rnet(cookie)) except Exception: pass + def add(self, cookie: Cookie, from_jar: bool = False) -> None: + """Store *cookie* as the latest write for its (domain, path, name) and send it to the client.""" + key = (cookie.domain or "", cookie.path or "/", cookie.name) + self._store.pop(key, None) + self._store[key] = cookie + if from_jar: + self._from_jar.add(key) + self.push(cookie) + + def snapshot(self) -> list[Cookie]: + """Every stored cookie, oldest write first. Download threads write the store, so read a copy.""" + return list(self._store.values()) + def flush_to_client(self) -> None: """Push all buffered cookies to the rnet client after unshackle makes it.""" - if self._client is None: - return - for domain, cookies in self._cookies.items(): - url = f"https://{domain.lstrip('.')}" if domain else "https://localhost" - for name, value in cookies.items(): - try: - self._client.set_cookie(url, rnet.Cookie(name, value)) - except Exception: - pass + for cookie in self.snapshot(): + self.push(cookie) - def client_urls(self, domain: Optional[str] = None) -> list[str]: + def client_urls(self, domain: Optional[str] = None, name: Optional[str] = None) -> list[str]: """URLs the rnet cookie jar could hold a cookie under. - rnet keys its jar by URL and cannot list the jar whole, so a removal has to name every - URL this class has set a cookie on and every origin the HTTP session has requested. + rnet keys its jar by URL, removes a cookie only at its exact path, and cannot list the + jar. So a removal names every host this class has set a cookie on, every origin the + HTTP session has requested, and every stored path of the cookie *name*. """ - urls = {f"https://{d.lstrip('.')}" for d in self._cookies if d} - urls.add("https://localhost") + cookies = self.snapshot() + hosts = {self.client_url(c) for c in cookies} | {"https://localhost"} if self._session is not None: - urls |= set(self._session._origins) + hosts |= set(self._session._origins) if domain is not None: host = domain.lstrip(".") - urls = {url for url in urls if (urlparse(url).hostname or "") == host} - urls.add(f"https://{host}") - return sorted(urls) + hosts = {url for url in hosts if (urlparse(url).hostname or "") == host} | {f"https://{host}"} + paths = {c.path for c in cookies if c.path and c.path != "/" and (name is None or c.name == name)} + return sorted(hosts | {url + path for url in hosts for path in paths}) def remove_cookie_on_client(self, name: str, domain: Optional[str] = None) -> None: - """Drop a cookie from the rnet cookie jar. + """Drop a cookie from the rnet cookie jar; call it before the store forgets the cookie's paths. The client is built with cookie_store=True, so a cookie the server set lives in that - jar alone. Dropping it from the local dicts leaves it going out on the wire. + jar alone. Dropping it from the store leaves it going out on the wire. """ if self._client is None: return - for url in self.client_urls(domain): + for url in self.client_urls(domain, name): try: self._client.remove_cookie(url, name) except Exception: pass + def absorb(self, response: Any) -> None: + """Take a value the server set for a stored cookie, so a client rebuild or a cookie save keeps it. + + A server cookie replaces each stored cookie of the same name and path that covers the + response host, and an expired one drops it. + """ + server_cookies = response.cookies if self._store else None + if not server_cookies: + return + parsed = urlparse(response.url) + host = parsed.hostname or "" + default_path = (parsed.path[: parsed.path.rfind("/")] or "/") if parsed.path.startswith("/") else "/" + now = datetime.now(timezone.utc) + for server_cookie in server_cookies: + path = server_cookie.path if (server_cookie.path or "").startswith("/") else default_path + expires, max_age = server_cookie.expires, server_cookie.max_age + if expires is not None and expires.tzinfo is None: + expires = expires.replace(tzinfo=timezone.utc) + expired = (max_age is not None and max_age.total_seconds() <= 0) or (expires is not None and expires <= now) + for cookie in self.snapshot(): + domain = cookie.domain or "" + if cookie.name != server_cookie.name or cookie.path != path or not domain: + continue + if host != domain.lstrip(".") and not (domain.startswith(".") and host.endswith(domain)): + continue + if expired: + self.clear(domain=domain, path=path, name=cookie.name) + elif cookie.value != server_cookie.value: + fresh = copy(cookie) + fresh.value = server_cookie.value + self.add(fresh) + @property def jar(self) -> CookieJar: - """Return a CookieJar with original Cookie objects (requests compat). + """Return a CookieJar of the cookies that arrived as a CookieJar (requests compat). - ``save_cookies`` in dl.py uses this to write cookies back to disk. + ``save_cookies`` in dl.py uses this to write cookies back to disk, so it carries a value + the server rotated for one of those cookies but not the cookies set by name. """ jar = CookieJar() - for cookie in self._original_cookies: - jar.set_cookie(cookie) + for key, cookie in list(self._store.items()): + if key in self._from_jar: + jar.set_cookie(cookie) return jar def update(self, other: Any = None, /, **kwargs: Any) -> None: if other is None: other = {} - if isinstance(other, CookieJar): + if isinstance(other, (CookieJar, list)): for cookie in other: - domain = cookie.domain or "" - name = cookie.name - value = cookie.value or "" - self._flat[name] = value - self._cookies.setdefault(domain, {})[name] = value - self._original_cookies.append(cookie) - url = f"https://{domain.lstrip('.')}" if domain else "https://localhost" - self.set_cookie_on_client(url, name, value) - elif isinstance(other, dict): - for name, value in other.items(): - self._flat[name] = value - self.set_cookie_on_client("https://localhost", name, str(value)) - self._flat.update(other) + self.add(cookie, from_jar=True) elif hasattr(other, "items"): for name, value in other.items(): - self._flat[name] = str(value) - self.set_cookie_on_client("https://localhost", name, str(value)) - + self.add(create_cookie(name, str(value))) for name, value in kwargs.items(): - self._flat[name] = value - self.set_cookie_on_client("https://localhost", name, value) + self.add(create_cookie(name, str(value))) def get( # type: ignore[override] self, name: str, default: Optional[str] = None, domain: Optional[str] = None, path: Optional[str] = None ) -> Optional[str]: - if domain and domain in self._cookies: - return self._cookies[domain].get(name, default) - return self._flat.get(name, default) + """The latest value of *name*.""" + for cookie in reversed(self.snapshot()): + if cookie.name == name and domain in (None, cookie.domain) and path in (None, cookie.path): + return cookie.value + return default - def set(self, name: str, value: str, domain: str = "localhost") -> None: - self._flat[name] = value - self._cookies.setdefault(domain, {})[name] = value - url = f"https://{domain.lstrip('.')}" - self.set_cookie_on_client(url, name, value) + def set(self, name: str, value: str, domain: str = "localhost", path: str = "/", **kwargs: Any) -> None: + self.add(create_cookie(name, value, domain=domain, path=path, **kwargs)) + + def flat(self) -> dict[str, str]: + """Each cookie name with its latest value.""" + return {cookie.name: cookie.value or "" for cookie in self.snapshot()} def __getitem__(self, name: str) -> str: - return self._flat[name] + return self.flat()[name] def __setitem__(self, name: str, value: str) -> None: self.set(name, value) def __delitem__(self, name: str) -> None: - self._flat.pop(name, None) - for domain_cookies in self._cookies.values(): - domain_cookies.pop(name, None) - self._original_cookies = [cookie for cookie in self._original_cookies if cookie.name != name] - self.remove_cookie_on_client(name) + self.clear(name=name) def __contains__(self, name: object) -> bool: - return name in self._flat + return any(cookie.name == name for cookie in self.snapshot()) def __iter__(self) -> Iterator: - return iter(self._flat) + return iter(self.flat()) def __len__(self) -> int: - return len(self._flat) + return len(self.flat()) def __bool__(self) -> bool: - return bool(self._flat) + return bool(self._store) def get_dict(self, domain: Optional[str] = None, path: Optional[str] = None) -> dict[str, str]: - """Return cookies as a plain dict (requests RequestsCookieJar compat). - - With a *domain*, this method returns only the cookies for that domain. - It accepts *path* for API compatibility but ignores it (flat storage). - """ - if domain is not None: - return dict(self._cookies.get(domain, {})) - return dict(self._flat) - - def get_dict_by_domain(self) -> dict[str, dict[str, str]]: - """Return cookies grouped by domain, domain-less ones under ``""``. - - rnet scopes each cookie to the host of the URL it was set on, so a copy made with - ``get_dict`` can only ever replay against one host. This class records cookies that - arrived as a plain dict flat only, and reports them under the empty domain, to match - the localhost fallback in :meth:`flush_to_client`. - """ - grouped = {domain: dict(cookies) for domain, cookies in self._cookies.items() if cookies} - scoped = {name for cookies in grouped.values() for name in cookies} - leftovers = {name: value for name, value in self._flat.items() if name not in scoped} - if leftovers: - grouped[""] = {**grouped.get("", {}), **leftovers} - return grouped + """Return cookies as a plain dict (requests RequestsCookieJar compat), filtered by exact domain and path.""" + return {c.name: c.value or "" for c in self.snapshot() if domain in (None, c.domain) and path in (None, c.path)} def clear(self, domain: Optional[str] = None, path: Optional[str] = None, name: Optional[str] = None) -> None: """Remove cookies (requests RequestsCookieJar compat). - ``clear()`` removes all cookies. - - ``clear(domain=..., path=..., name=...)`` removes a specific cookie. + - ``clear(domain=..., path=..., name=...)`` removes the cookies that match every given filter. """ - if name is not None: - self._flat.pop(name, None) - if domain is not None and domain in self._cookies: - self._cookies[domain].pop(name, None) - else: - for domain_cookies in self._cookies.values(): - domain_cookies.pop(name, None) - self._original_cookies = [ - cookie - for cookie in self._original_cookies - if cookie.name != name or (domain is not None and cookie.domain != domain) - ] - self.remove_cookie_on_client(name, domain) - elif domain is not None: - removed = self._cookies.pop(domain, {}) - for k in removed: - # Only remove from flat if no other domain has same key - still_exists = any(k in dc for dc in self._cookies.values()) - if not still_exists: - self._flat.pop(k, None) - self._original_cookies = [cookie for cookie in self._original_cookies if cookie.domain != domain] - for k in removed: - self.remove_cookie_on_client(k, domain) - else: - self._flat.clear() - self._cookies.clear() - self._original_cookies.clear() + if domain is None and path is None and name is None: + self._store.clear() + self._from_jar.clear() if self._client is not None: self._client.clear_cookies() + return + + def hit(cookie: Cookie) -> bool: + return domain in (None, cookie.domain) and path in (None, cookie.path) and name in (None, cookie.name) + + names = {name} if name is not None else {c.name for c in self.snapshot() if hit(c)} + for each in names: + self.remove_cookie_on_client(each, domain) + self._store = {key: c for key, c in list(self._store.items()) if not hit(c)} + self._from_jar &= self._store.keys() + # removal on the client is by host and name, so put back what a path filter spared + for cookie in self.snapshot(): + if cookie.name in names: + self.push(cookie) def items(self) -> list[tuple[str, str]]: # type: ignore[override] - return list(self._flat.items()) + return list(self.flat().items()) def keys(self) -> list[str]: # type: ignore[override] - return list(self._flat.keys()) + return list(self.flat().keys()) def values(self) -> list[str]: # type: ignore[override] - return list(self._flat.values()) + return list(self.flat().values()) class RnetProxyDict(dict): @@ -671,9 +693,10 @@ class RnetSession: that replaces the impersonate header set instead of overlaying it, which breaks the fingerprint. - A new client also starts with an empty cookie jar. This method replays the cookies - unshackle set itself, then the cookies of every origin this HTTP session has used. - That order lets a value the server rotated win over the buffered one. The new client + A new client also starts with an empty cookie jar. This method replays the stored + cookies, which hold any value the server rotated for them, then the root-path cookies + of every origin this HTTP session has used. A cookie the server set on a deeper path + that the store never held is lost, because rnet cannot list its jar. The new client goes on the HTTP session last, so a request from another thread keeps the old client until the new one holds the headers and the cookies. """ @@ -739,8 +762,8 @@ class RnetSession: def scoped_cookies(self) -> CookieJar: """Every cookie this HTTP session sends, each scoped to its domain. - The live client's cookies of each requested origin come first, then the cookies set - through :attr:`cookies` for any other host. + The live client's root-path cookies of each requested origin come first, then the + cookies stored in :attr:`cookies` with their own paths. """ jar = CookieJar() hosts: dict[str, set[str]] = {} @@ -749,11 +772,11 @@ class RnetSession: for name, value in self.parse_cookie_header(header): jar.set_cookie(create_cookie(name, value, domain=host)) hosts.setdefault(name, set()).add(host) - for domain, values in self.cookies.get_dict_by_domain().items(): - bare = domain.lstrip(".") - for name, value in values.items(): - if not any(h == bare or h.endswith("." + bare) for h in hosts.get(name, ())): - jar.set_cookie(create_cookie(name, value, domain=domain)) + for cookie in self.cookies.snapshot(): + bare = (cookie.domain or "").lstrip(".") + live = any(h == bare or h.endswith("." + bare) for h in hosts.get(cookie.name, ())) + if not (live and cookie.path in ("", "/")): + jar.set_cookie(cookie) return jar def import_origin_cookies(self, exported: dict[str, str]) -> None: @@ -813,6 +836,14 @@ class RnetSession: sleep_time = backoff_value + random.uniform(-jitter, jitter) return min(sleep_time, self.max_backoff) + def absorb_cookies(self, response: RnetResponse) -> RnetResponse: + """Record a value the server set for a stored cookie; a cookie fault must never fail the request.""" + try: + self.cookies.absorb(response) + except Exception: # nosec B110 - the store keeps its old value, the rnet jar still has the new one + pass + return response + def request(self, method: str, url: str, **kwargs: Any) -> RnetResponse: """Send a request, retrying on the status forcelist and on the caught exception types. @@ -865,16 +896,14 @@ class RnetSession: kwargs["headers"] = dict(kwargs["headers"]) if method_upper not in self.allowed_methods: - raw_resp = client.request(rnet_method, url, **kwargs) - return RnetResponse(raw_resp) + return self.absorb_cookies(RnetResponse(client.request(rnet_method, url, **kwargs))) last_exception: Optional[Exception] = None response: Optional[RnetResponse] = None for attempt in range(max_retries + 1): try: - raw_resp = client.request(rnet_method, url, **kwargs) - response = RnetResponse(raw_resp) + response = self.absorb_cookies(RnetResponse(client.request(rnet_method, url, **kwargs))) if config.debug_requests: parsed_url = urlparse(url) diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index da4290d6..464aec49 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -59,22 +59,15 @@ def direct_session(session: AnySession, proxy: Optional[str] = None) -> Session: pass cookies = getattr(session, "cookies", None) if cookies is not None: - jar = getattr(cookies, "jar", None) + snapshot = getattr(cookies, "snapshot", None) try: - new.cookies.update(jar if jar is not None else cookies) + if snapshot is None: + new.cookies.update(cookies) + else: + for cookie in snapshot(): + new.cookies.set_cookie(copy(cookie)) # set_cookie edits a quoted value in place except Exception: pass - # RnetCookieAdapter.jar holds only cookies that arrived as a CookieJar; the ones set by name or - # received from a server live in its domain map, so copy those across too - by_domain = getattr(cookies, "get_dict_by_domain", None) - if by_domain is not None: - try: - for domain, named in by_domain().items(): - for name, value in named.items(): - if name not in new.cookies: - new.cookies.set(name, value, **({"domain": domain} if domain else {})) - except Exception: - pass new.mount( "https://", HTTPAdapter(