diff --git a/tests/conftest.py b/tests/conftest.py index ae36af88..5709f220 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ import pytest from unshackle.core import binaries from unshackle.core.config import config +from unshackle.core.proxies.proxy import CATALOGUES from unshackle.core.update_checker import UpdateChecker # shutdown() blocks until the serve loop polls again, and the 0.5s default is paid by every @@ -40,6 +41,12 @@ def no_update_check(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(UpdateChecker, "check_for_updates_sync", lambda *args, **kwargs: None) +@pytest.fixture(autouse=True) +def fresh_proxy_catalogues() -> None: + """Fetch each proxy provider catalogue again, so one test's stub never answers another test.""" + CATALOGUES.clear() + + @pytest.fixture(autouse=True) def default_muxing(monkeypatch: pytest.MonkeyPatch) -> None: """Mux with the default options, whatever the muxing block of the local unshackle.yaml sets.""" diff --git a/tests/core/test_proxy_catalogue_cache.py b/tests/core/test_proxy_catalogue_cache.py new file mode 100644 index 00000000..43bed6d1 --- /dev/null +++ b/tests/core/test_proxy_catalogue_cache.py @@ -0,0 +1,37 @@ +"""The proxy provider catalogue cache.""" + +from __future__ import annotations + +import pytest + +from unshackle.core.proxies import proxy +from unshackle.core.proxies.proxy import cached_catalogue + + +def test_catalogue_is_fetched_once_until_it_expires(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[int] = [] + + @cached_catalogue + def fetch() -> list[dict]: + calls.append(1) + return [{"country": "CA"}] + + assert fetch() == fetch() == [{"country": "CA"}] + assert len(calls) == 1 + + monkeypatch.setattr(proxy, "CATALOGUE_TTL", 0.0) + fetch() + assert len(calls) == 2 + + +def test_empty_catalogue_is_not_kept() -> None: + calls: list[int] = [] + + @cached_catalogue + def fetch() -> list[dict]: + calls.append(1) + return [] + + fetch() + fetch() + assert len(calls) == 2 diff --git a/tests/core/test_vault_push_retry.py b/tests/core/test_vault_push_retry.py index 5f8870a3..389f981a 100644 --- a/tests/core/test_vault_push_retry.py +++ b/tests/core/test_vault_push_retry.py @@ -66,3 +66,20 @@ def test_server_pushes_again_after_one_vault_failed(monkeypatch: pytest.MonkeyPa handlers.cache_to_vaults({KID.hex: KEY}, "SVC") handlers.cache_to_vaults({KID.hex: KEY}, "SVC") assert len(pushes) == 2 + + +def test_one_licence_request_builds_the_server_vaults_once(monkeypatch: pytest.MonkeyPatch) -> None: + builds: list = [] + pushes: list = [] + + def build(name: str) -> Any: + builds.append(name) + return fake_server_vaults(pushes, [FakeVault(False)]) + + monkeypatch.setattr(handlers, "load_server_vaults", build) + monkeypatch.setattr(handlers, "CACHED_PAIRS", set()) + shared: dict = {} + handlers.cache_to_vaults({KID.hex: KEY}, "SVC", shared=shared) + handlers.cache_to_vaults({KID.hex: KEY}, "SVC", shared=shared) + assert len(builds) == 1 + assert len(pushes) == 2 diff --git a/tests/remote/unit/test_client_device_cdm.py b/tests/remote/unit/test_client_device_cdm.py index 6488760b..7e732b50 100644 --- a/tests/remote/unit/test_client_device_cdm.py +++ b/tests/remote/unit/test_client_device_cdm.py @@ -238,7 +238,7 @@ def test_a_lent_device_session_never_licenses_live() -> None: def test_a_vault_only_miss_raises_the_refusal(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(handlers, "check_vaults", lambda kids, service: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, service, **_: None) monkeypatch.setattr(handlers, "ensure_track_drm", lambda *a, **k: None) refusal = handlers.live_licence_refusal(SimpleNamespace(server_device=True), object()) with pytest.raises(APIError) as caught: @@ -257,7 +257,7 @@ def test_a_vault_only_miss_raises_the_refusal(monkeypatch: pytest.MonkeyPatch) - def test_a_vault_only_hit_needs_no_server_device(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( - handlers, "check_vaults", lambda kids, service: ({SESSION_KID.hex: "k"}, {SESSION_KID.hex: "v"}) + handlers, "check_vaults", lambda kids, service, **_: ({SESSION_KID.hex: "k"}, {SESSION_KID.hex: "v"}) ) monkeypatch.setattr(handlers, "ensure_track_drm", lambda *a, **k: None) monkeypatch.setattr( diff --git a/tests/remote/unit/test_extract_pssh_strict.py b/tests/remote/unit/test_extract_pssh_strict.py index ed53566d..06db2a09 100644 --- a/tests/remote/unit/test_extract_pssh_strict.py +++ b/tests/remote/unit/test_extract_pssh_strict.py @@ -55,7 +55,9 @@ async def _run_batch(monkeypatch, tracks, cdm_type, calls): async def fake_get_session(session_id, request): return session - def fake_single(service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False): + def fake_single( + service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False, shared=None + ): calls.append((pssh_str, drm_type)) return {"00" * 16: "key"} diff --git a/tests/remote/unit/test_handler_proxy_policy.py b/tests/remote/unit/test_handler_proxy_policy.py index 9b1f6c9d..3ca2f5ec 100644 --- a/tests/remote/unit/test_handler_proxy_policy.py +++ b/tests/remote/unit/test_handler_proxy_policy.py @@ -34,7 +34,7 @@ def patch_server_region(monkeypatch, region): def patch_providers(monkeypatch, proxy): """Give the server one provider that answers every country code with ``proxy``.""" provider = SimpleNamespace(get_proxy=lambda query: proxy) - monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda: [provider]) + monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda **_: [provider]) return provider @@ -101,7 +101,7 @@ def test_provider_country_rejected(monkeypatch): def test_no_proxy_short_circuits(monkeypatch): """no_proxy skips resolution and never loads providers, even with a proxy param set.""" - monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda: pytest.fail("providers loaded")) + monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda **_: pytest.fail("providers loaded")) _, providers = handlers.resolve_handler_proxy( {"proxy": "us", "no_proxy": True, "client_region": "ca"}, "EXAMPLE", request("proxykey") ) @@ -154,7 +154,7 @@ def test_user_without_server_proxy_denied(monkeypatch): def test_no_region_sent_proceeds(monkeypatch): """An out-of-date client that reports no region is unknown and never blocked.""" patch_server_region(monkeypatch, "IN") - monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda: []) + monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda **_: []) assert handlers.resolve_handler_proxy({}, "EXAMPLE", request("plainkey")) == (None, []) assert handlers.resolve_handler_proxy({}, "EXAMPLE", request("proxykey")) == (None, []) @@ -241,7 +241,7 @@ class TestDownloadGates: def test_country_code_passes_when_opted_in(self, monkeypatch): # The gate must not even try to resolve for an opted-in key: dl resolves at run time, # exactly as before the gate existed. - monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda: pytest.fail("providers loaded")) + monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda **_: pytest.fail("providers loaded")) handlers.enforce_download_gates({"service": "EXAMPLE", "proxy": "us"}, request("proxykey")) def test_region_mismatch_rejected_at_submit(self, monkeypatch): @@ -388,7 +388,7 @@ def test_local_proxy_uri_refused_without_server_proxy(proxy): @pytest.mark.parametrize("key", ["proxykey", "adminkey"]) # adminkey has no serve.users entry def test_local_proxy_uri_allowed_with_server_proxy_or_admin(monkeypatch, key): - monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda: []) + monkeypatch.setattr(handlers, "initialize_proxy_providers", lambda **_: []) proxy, _ = handlers.resolve_handler_proxy({"proxy": "http://127.0.0.1:8080"}, "EXAMPLE", request(key)) assert proxy == "http://127.0.0.1:8080" diff --git a/tests/remote/unit/test_licence_declared_kwargs.py b/tests/remote/unit/test_licence_declared_kwargs.py index 4869e54b..1093b81f 100644 --- a/tests/remote/unit/test_licence_declared_kwargs.py +++ b/tests/remote/unit/test_licence_declared_kwargs.py @@ -157,8 +157,8 @@ def widevine_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(detect_mod, "is_widevine_cdm", lambda cdm: True) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) @@ -170,7 +170,7 @@ def playready_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(detect_mod, "is_playready_cdm", lambda cdm: True) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) diff --git a/tests/remote/unit/test_server_cdm_absorb_routes.py b/tests/remote/unit/test_server_cdm_absorb_routes.py index ce2ff9ff..b6a5f13e 100644 --- a/tests/remote/unit/test_server_cdm_absorb_routes.py +++ b/tests/remote/unit/test_server_cdm_absorb_routes.py @@ -59,8 +59,8 @@ def env(monkeypatch): monkeypatch.setattr(detect_mod, "is_playready_cdm", lambda cdm: True) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers, "server_cdm_allowed", lambda request, tag: True) monkeypatch.setattr(handlers, "find_title_for_track", lambda tid, session: SimpleNamespace()) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) diff --git a/tests/remote/unit/test_server_cdm_absorbs_headers.py b/tests/remote/unit/test_server_cdm_absorbs_headers.py index 693e6927..1e71f15b 100644 --- a/tests/remote/unit/test_server_cdm_absorbs_headers.py +++ b/tests/remote/unit/test_server_cdm_absorbs_headers.py @@ -58,8 +58,8 @@ def playready_env(monkeypatch): monkeypatch.setattr(detect_mod, "is_playready_cdm", lambda cdm: True) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) diff --git a/tests/remote/unit/test_server_cdm_key_verification.py b/tests/remote/unit/test_server_cdm_key_verification.py index 4fb764fd..9718f78b 100644 --- a/tests/remote/unit/test_server_cdm_key_verification.py +++ b/tests/remote/unit/test_server_cdm_key_verification.py @@ -120,7 +120,9 @@ async def test_batch_reports_vault_keys_from_the_init_segment_pssh(monkeypatch): async def validated(sid, req): return session - def fake_single(service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False): + def fake_single( + service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False, shared=None + ): if pssh_str == "init": sources[CDM_KID.hex] = "sqlite" return {CDM_KID.hex: "v"} diff --git a/tests/remote/unit/test_server_cdm_manifest_kid.py b/tests/remote/unit/test_server_cdm_manifest_kid.py index 92c7a4d1..957448e7 100644 --- a/tests/remote/unit/test_server_cdm_manifest_kid.py +++ b/tests/remote/unit/test_server_cdm_manifest_kid.py @@ -83,8 +83,8 @@ def server_env(monkeypatch): monkeypatch.setattr(detect_mod, "is_playready_cdm", lambda cdm: True) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers, "server_cdm_allowed", lambda request, tag: True) monkeypatch.setattr(handlers, "find_title_for_track", lambda tid, session: SimpleNamespace()) monkeypatch.setattr(handlers, "detect_cdm_type_for_service", lambda tag, cfg: "playready") diff --git a/tests/remote/unit/test_server_cdm_max_height.py b/tests/remote/unit/test_server_cdm_max_height.py index ea0aef21..4c6970eb 100644 --- a/tests/remote/unit/test_server_cdm_max_height.py +++ b/tests/remote/unit/test_server_cdm_max_height.py @@ -156,13 +156,13 @@ def single(refusal): def test_a_vault_hit_is_served_above_the_cap(widevine_env, monkeypatch): - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: ({"11" * 16: "key"}, {"11" * 16: "sqlite"})) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: ({"11" * 16: "key"}, {"11" * 16: "sqlite"})) refusal = handlers.live_licence_refusal(SimpleNamespace(server_cdm_max_height=1080), video(2160)) assert single(refusal) == {"11" * 16: "key"} def test_a_vault_miss_raises_the_refusal_before_the_device_loads(widevine_env, monkeypatch): - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) refusal = handlers.live_licence_refusal(SimpleNamespace(server_cdm_max_height=1080), video(2160)) with pytest.raises(APIError) as ei: single(refusal) @@ -406,7 +406,9 @@ def licence_env(monkeypatch): calls: list = [] vault = {} - def fake_single(service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False): + def fake_single( + service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False, shared=None + ): calls.append((track.id, pssh_str)) if pssh_str in vault: return dict(vault[pssh_str]) diff --git a/tests/remote/unit/test_server_cdm_multikey.py b/tests/remote/unit/test_server_cdm_multikey.py index 89648cb9..aae54647 100644 --- a/tests/remote/unit/test_server_cdm_multikey.py +++ b/tests/remote/unit/test_server_cdm_multikey.py @@ -61,9 +61,9 @@ def playready_env(monkeypatch): def test_playready_multikey_not_shortcircuited_by_vault(playready_env, monkeypatch): # The vault holds only the PSSH KID (a stale subset). The old code returned it and skipped # the license, hiding REAL and OTHER; the fix licenses and returns the whole bundle. - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: {PSSH_KID.hex: "STALE"}) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: {PSSH_KID.hex: "STALE"}) cached: dict = {} - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: cached.update(keys)) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: cached.update(keys)) keys = handlers.handle_single_server_cdm( service=SimpleNamespace(), @@ -99,7 +99,7 @@ def test_widevine_still_uses_vault_shortcut(monkeypatch): monkeypatch.setattr(drm_mod, "Widevine", _FakeWidevine) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: ({kid.hex: "vaultkey"}, {kid.hex: "sqlite"})) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: ({kid.hex: "vaultkey"}, {kid.hex: "sqlite"})) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) sources: dict = {} @@ -133,7 +133,9 @@ async def test_batch_shares_full_bundle_per_pssh(monkeypatch): bundle_a = {REAL.hex: "k_real", PSSH_KID.hex: "k_pssh", OTHER.hex: "k_other"} bundle_b = {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "k_b"} - def fake_single(service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False): + def fake_single( + service, title, track, pssh_str, drm_type, request, sources=None, refusal=None, vault_only=False, shared=None + ): calls.append(pssh_str) return dict(bundle_a) if pssh_str == pssh_a else dict(bundle_b) diff --git a/tests/remote/unit/test_server_cdm_service_injected_keys.py b/tests/remote/unit/test_server_cdm_service_injected_keys.py index f716e0c0..12d71acb 100644 --- a/tests/remote/unit/test_server_cdm_service_injected_keys.py +++ b/tests/remote/unit/test_server_cdm_service_injected_keys.py @@ -56,8 +56,8 @@ def server_env(monkeypatch): monkeypatch.setattr(detect_mod, "is_playready_cdm", lambda cdm: True) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) monkeypatch.setattr(handlers, "resolve_device_name", lambda *a, **k: "dev") - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) diff --git a/tests/remote/unit/test_server_cdm_type_override.py b/tests/remote/unit/test_server_cdm_type_override.py index 4cc6daf7..67eebf8b 100644 --- a/tests/remote/unit/test_server_cdm_type_override.py +++ b/tests/remote/unit/test_server_cdm_type_override.py @@ -95,8 +95,8 @@ def env(monkeypatch): monkeypatch.setattr(detect_mod, "is_widevine_cdm", lambda cdm: cdm.device.startswith("wv")) monkeypatch.setattr(detect_mod, "is_playready_cdm", lambda cdm: cdm.device.startswith("pr")) monkeypatch.setattr(handlers, "ensure_track_drm", lambda track, session=None, init_data=None: None) - monkeypatch.setattr(handlers, "check_vaults", lambda kids, name: None) - monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name: None) + monkeypatch.setattr(handlers, "check_vaults", lambda kids, name, **_: None) + monkeypatch.setattr(handlers, "cache_to_vaults", lambda keys, name, **_: None) monkeypatch.setattr(handlers, "server_cdm_allowed", lambda request, tag: True) monkeypatch.setattr(handlers, "find_title_for_track", lambda tid, session: SimpleNamespace()) monkeypatch.setattr(handlers.config, "serve", {"users": {}}, raising=False) diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index a36b56f8..71345535 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -1,5 +1,6 @@ import asyncio import base64 +import copy import enum import json import logging @@ -202,15 +203,26 @@ def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str] return resolve_server_cdm(service, profile, cdm_type) +SERVICE_YAML: Dict[Path, tuple[int, dict]] = {} + + def load_service_yaml(normalized_service: str) -> dict: - """Load a service's config.yaml and merge it with the global override block.""" + """Load a service's config.yaml and merge it with the global override block. + + The parsed file is kept until its mtime changes; each caller gets its own copy to change. + """ from unshackle.core.utils.collections import merge_dict service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = load_yaml(service_config_path) or {} + try: + mtime = service_config_path.stat().st_mtime_ns + except OSError: + service_config: dict = {} else: - service_config = {} + hit = SERVICE_YAML.get(service_config_path) + if not hit or hit[0] != mtime: + hit = SERVICE_YAML[service_config_path] = (mtime, load_yaml(service_config_path) or {}) + service_config = copy.deepcopy(hit[1]) merge_dict(config.services.get(normalized_service) or {}, service_config) return service_config @@ -4638,7 +4650,7 @@ def resolve_handler_proxy( ) if allowed and not no_proxy: - proxy_providers = initialize_proxy_providers() + proxy_providers = initialize_proxy_providers(quiet=True) if proxy_param and not no_proxy: try: @@ -4943,7 +4955,16 @@ def load_server_vaults(service_name: str) -> Any: return vaults -def check_vaults(kids: list, service_name: str) -> Optional[tuple[Dict[str, str], Dict[str, str]]]: +def once(shared: Dict[str, Any], key: Any, make: Callable[[], Any]) -> Any: + """``make()`` the first time ``key`` is asked for in ``shared``, the stored value after that.""" + if key not in shared: + shared[key] = make() + return shared[key] + + +def check_vaults( + kids: list, service_name: str, shared: Optional[Dict[str, Any]] = None +) -> Optional[tuple[Dict[str, str], Dict[str, str]]]: """Examine the server vaults for existing content keys that match all KIDs. Returns `(KID:KEY, KID:vault name)` if ALL KIDs are found, None otherwise. The vault @@ -4952,7 +4973,7 @@ def check_vaults(kids: list, service_name: str) -> Optional[tuple[Dict[str, str] from uuid import UUID try: - vaults = load_server_vaults(service_name) + vaults = once({} if shared is None else shared, "vaults", lambda: load_server_vaults(service_name)) if not vaults.vaults: return None keys: Dict[str, str] = {} @@ -4977,7 +4998,7 @@ def check_vaults(kids: list, service_name: str) -> Optional[tuple[Dict[str, str] CACHED_PAIRS: set[tuple[str, str, str]] = set() -def cache_to_vaults(keys: Dict[str, str], service_name: str) -> None: +def cache_to_vaults(keys: Dict[str, str], service_name: str, shared: Optional[Dict[str, Any]] = None) -> None: """Cache the content keys of a CDM licence to the server vaults. A PlayReady licence always runs, and a Widevine licence runs when one PSSH KID has no content @@ -4991,7 +5012,7 @@ def cache_to_vaults(keys: Dict[str, str], service_name: str) -> None: new = {kid: key for kid, key in keys.items() if (service_name, kid, key) not in CACHED_PAIRS} if not new: return - vaults = load_server_vaults(service_name) + vaults = once({} if shared is None else shared, "vaults", lambda: load_server_vaults(service_name)) if not vaults.vaults: return @@ -5084,6 +5105,7 @@ def handle_single_server_cdm( sources: Optional[Dict[str, str]] = None, refusal: Optional[APIError] = None, vault_only: bool = False, + shared: Optional[Dict[str, Any]] = None, ) -> Dict[str, str]: """Do the single-track server_cdm licensing with the DRM class get_content_keys() flow. @@ -5091,6 +5113,7 @@ def handle_single_server_cdm( supplied. A content key the CDM licensed gets no entry. It raises ``refusal`` before a live licence: after a Widevine vault miss, or at once for PlayReady, whose licence can carry keys its header does not name. With ``vault_only`` no server device is involved: the header's KIDs come from the vault, else ``refusal``. + ``shared`` keeps the vaults and CDMs one ``license_response`` call built, for its next track. """ import base64 @@ -5098,6 +5121,8 @@ def handle_single_server_cdm( from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm ensure_track_drm(track, getattr(service, "session", None)) + shared = {} if shared is None else shared + service_name = service.__class__.__name__ if not pssh_b64: pssh_b64 = extract_pssh_from_track(track, drm_type) @@ -5115,7 +5140,7 @@ def handle_single_server_cdm( from pywidevine.pssh import PSSH as WvPSSH kids = Widevine(pssh=WvPSSH(pssh_b64)).kids - vault_hit = check_vaults(kids, service.__class__.__name__) + vault_hit = check_vaults(kids, service_name, shared=shared) if vault_hit: if sources is not None: sources.update(vault_hit[1]) @@ -5143,7 +5168,7 @@ def handle_single_server_cdm( if refusal: raise refusal - cdm = load_cdm(device_name, service_name=service.__class__.__name__) + cdm = once(shared, ("cdm", device_name), lambda: load_cdm(device_name, service_name=service_name)) if not is_playready_cdm(cdm): raise APIError(APIErrorCode.INVALID_INPUT, f"CDM device '{device_name}' is not a PlayReady device") keys = license_with_track_fallback( @@ -5167,7 +5192,7 @@ def handle_single_server_cdm( # harvest server-side keys from the vault fallback below. device_name = resolve_device_name(user_config, drm_type, service.__class__.__name__) - vault_hit = check_vaults(wv_drm.kids, service.__class__.__name__) + vault_hit = check_vaults(wv_drm.kids, service_name, shared=shared) if vault_hit: vault_keys, vault_sources = vault_hit if sources is not None: @@ -5176,7 +5201,7 @@ def handle_single_server_cdm( if refusal: raise refusal - cdm = load_cdm(device_name, service_name=service.__class__.__name__) + cdm = once(shared, ("cdm", device_name), lambda: load_cdm(device_name, service_name=service_name)) if not is_widevine_cdm(cdm): raise APIError(APIErrorCode.INVALID_INPUT, f"CDM device '{device_name}' is not a Widevine device") keys = license_with_track_fallback( @@ -5199,7 +5224,7 @@ def handle_single_server_cdm( if not keys: raise APIError(APIErrorCode.NO_CONTENT, "Server CDM returned no content keys") - cache_to_vaults(keys, service.__class__.__name__) + cache_to_vaults(keys, service_name, shared=shared) return keys @@ -5414,6 +5439,7 @@ def license_response( clear_tracks: list[str] = [] drm_types: Dict[str, str] = {} keys_by_pssh: Dict[tuple, Dict[str, str]] = {} + shared: Dict[str, Any] = {} sources_by_pssh: Dict[tuple, Dict[str, str]] = {} drm_type_by_pssh: Dict[tuple, str] = {} actual_drm_type: Optional[str] = None @@ -5471,6 +5497,7 @@ def license_response( sources_by_pssh[cache_key], live_licence_refusal(session, track), vault_only=vault_only, + shared=shared, ) if keys: keys_by_pssh[cache_key] = keys diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index a8915fab..2cf6d3b1 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -2,8 +2,9 @@ import asyncio import functools import logging import re +import time from collections.abc import Awaitable, Callable -from typing import Any +from typing import Any, Optional import click from aiohttp import web @@ -197,6 +198,20 @@ def heartbeat(handler: Handler) -> Handler: return wrapper +UPDATE_CHECK_TTL = 300.0 +UPDATE_CHECK: dict[str, tuple[float, Optional[str]]] = {} + + +def latest_release() -> Optional[str]: + """The newer release the update check reports, kept for ``UPDATE_CHECK_TTL`` seconds: clients poll health.""" + hit = UPDATE_CHECK.get(__version__) + if hit and time.monotonic() - hit[0] < UPDATE_CHECK_TTL: + return hit[1] + latest = UpdateChecker.check_for_updates_sync(__version__) + UPDATE_CHECK[__version__] = (time.monotonic(), latest) + return latest + + @api_handler async def health(request: web.Request) -> web.Response: """ @@ -238,7 +253,7 @@ async def health(request: web.Request) -> web.Response: nullable: true """ try: - latest_version = await asyncio.to_thread(UpdateChecker.check_for_updates_sync, __version__) + latest_version = await asyncio.to_thread(latest_release) update_info = { "update_available": latest_version is not None, "current_version": __version__, @@ -261,6 +276,124 @@ async def health(request: web.Request) -> web.Response: ) +SERVICE_DEFAULTS: dict[str, Any] = { + "aliases": [], + "geofence": [], + "geoblock": [], + "title_regex": None, + "url": None, + "help": None, +} +SERVICE_DETAILS: dict[str, tuple[Any, dict[str, Any]]] = {} + + +def service_details(tag: str) -> dict[str, Any]: + """The catalogue entry of one service, built again only when its service module reloads.""" + service_module = Services.load(tag) + hit = SERVICE_DETAILS.get(tag) + if hit and hit[0] is service_module: + return hit[1] + service_data: dict[str, Any] = {**SERVICE_DEFAULTS, "tag": tag} + + if hasattr(service_module, "ALIASES"): + service_data["aliases"] = list(service_module.ALIASES) + + if hasattr(service_module, "GEOFENCE"): + service_data["geofence"] = list(service_module.GEOFENCE) + service_data["geoblock"] = list(getattr(service_module, "GEOBLOCK", ()) or ()) + + if hasattr(service_module, "TITLE_RE"): + title_re = service_module.TITLE_RE + if isinstance(title_re, re.Pattern): + service_data["title_regex"] = title_re.pattern + elif isinstance(title_re, str): + service_data["title_regex"] = title_re + elif isinstance(title_re, (list, tuple)): + # Convert list/tuple of patterns to list of strings + patterns = [] + for item in title_re: + if isinstance(item, re.Pattern): + patterns.append(item.pattern) + elif isinstance(item, str): + patterns.append(item) + service_data["title_regex"] = patterns if patterns else None + + if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"): + service_data["url"] = service_module.cli.short_help + + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + cli_params = [] + for param in service_module.cli.params: + param_info: dict = {"name": getattr(param, "name", None)} + if param.param_type_name == "argument": + param_info["kind"] = "argument" + param_info["required"] = param.required + else: + param_info["kind"] = "option" + param_info["opts"] = list(param.opts) if hasattr(param, "opts") else [] + param_info["is_flag"] = getattr(param, "is_flag", False) + default = param.default + if default is None: + pass + elif callable(default) or type(default).__name__ == "Sentinel": + default = None + elif hasattr(default, "name"): + default = default.name + elif not isinstance(default, (str, int, float, bool, list)): + default = str(default) + param_info["default"] = default + param_info["help"] = getattr(param, "help", None) + param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type) + if isinstance(param.type, click.Choice): + param_info["choices"] = list(param.type.choices) + param_info["multiple"] = getattr(param, "multiple", False) + cli_params.append(param_info) + service_data["cli_params"] = cli_params + + if service_module.__doc__: + service_data["help"] = service_module.__doc__.strip() + + # Capability flags, derived from which Service hooks the service overrides. + from unshackle.core.service import Service as _BaseService + + service_data["needs_auth"] = getattr(service_module, "authenticate", None) is not _BaseService.authenticate + client_config = getattr(service_module, "CLIENT_CONFIG", ()) + if client_config: + service_data["client_config"] = list(client_config) + service_data["has_search"] = getattr(service_module, "search", None) is not _BaseService.search + service_data["anonymous_search"] = bool(getattr(service_module, "ANONYMOUS_SEARCH", False)) + service_data["has_drm"] = ( + getattr(service_module, "get_widevine_license", None) is not _BaseService.get_widevine_license + or getattr(service_module, "get_playready_license", None) is not _BaseService.get_playready_license + ) + + # Prefer the service's explicit AUTH_METHODS; otherwise infer from authenticate(). + methods = [] + if service_data["needs_auth"]: + declared = getattr(service_module, "AUTH_METHODS", None) + if declared: + methods = list(declared) + else: + try: + import inspect as _inspect + + src_lines = _inspect.getsource(service_module.authenticate).splitlines() + start = next((i + 1 for i, ln in enumerate(src_lines) if ln.rstrip().endswith(":")), 1) + body = "\n".join(src_lines[start:]) + if "cookies" in body: + methods.append("cookies") + if "credential" in body: + methods.append("credentials") + except (OSError, TypeError): + pass + if not methods: + methods = ["cookies"] + service_data["auth_methods"] = methods + + SERVICE_DETAILS[tag] = (service_module, service_data) + return service_data + + @api_handler async def services(request: web.Request) -> web.Response: """ @@ -374,123 +507,16 @@ async def services(request: web.Request) -> web.Response: services_info = [] for tag in service_tags: - service_data: dict[str, Any] = { - "tag": tag, - "aliases": [], - "geofence": [], - "geoblock": [], - "title_regex": None, - "url": None, - "help": None, - } + service_data: dict[str, Any] = {**SERVICE_DEFAULTS, "tag": tag} + try: + service_data = {**service_details(tag)} + except Exception as e: + log.warning(f"Could not load details for service {tag}: {e}") if tag in services_module.PENDING: service_data["pending_update"] = True if server_accounts_allowed(request, tag) and (server_accounts := server_account_regions(tag)) is not None: service_data["server_accounts"] = server_accounts - try: - service_module = Services.load(tag) - - if hasattr(service_module, "ALIASES"): - service_data["aliases"] = list(service_module.ALIASES) - - if hasattr(service_module, "GEOFENCE"): - service_data["geofence"] = list(service_module.GEOFENCE) - service_data["geoblock"] = list(getattr(service_module, "GEOBLOCK", ()) or ()) - - if hasattr(service_module, "TITLE_RE"): - title_re = service_module.TITLE_RE - if isinstance(title_re, re.Pattern): - service_data["title_regex"] = title_re.pattern - elif isinstance(title_re, str): - service_data["title_regex"] = title_re - elif isinstance(title_re, (list, tuple)): - # Convert list/tuple of patterns to list of strings - patterns = [] - for item in title_re: - if isinstance(item, re.Pattern): - patterns.append(item.pattern) - elif isinstance(item, str): - patterns.append(item) - service_data["title_regex"] = patterns if patterns else None - - if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"): - service_data["url"] = service_module.cli.short_help - - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - cli_params = [] - for param in service_module.cli.params: - param_info: dict = {"name": getattr(param, "name", None)} - if param.param_type_name == "argument": - param_info["kind"] = "argument" - param_info["required"] = param.required - else: - param_info["kind"] = "option" - param_info["opts"] = list(param.opts) if hasattr(param, "opts") else [] - param_info["is_flag"] = getattr(param, "is_flag", False) - default = param.default - if default is None: - pass - elif callable(default) or type(default).__name__ == "Sentinel": - default = None - elif hasattr(default, "name"): - default = default.name - elif not isinstance(default, (str, int, float, bool, list)): - default = str(default) - param_info["default"] = default - param_info["help"] = getattr(param, "help", None) - param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type) - if isinstance(param.type, click.Choice): - param_info["choices"] = list(param.type.choices) - param_info["multiple"] = getattr(param, "multiple", False) - cli_params.append(param_info) - service_data["cli_params"] = cli_params - - if service_module.__doc__: - service_data["help"] = service_module.__doc__.strip() - - # Capability flags, derived from which Service hooks the service overrides. - from unshackle.core.service import Service as _BaseService - - service_data["needs_auth"] = ( - getattr(service_module, "authenticate", None) is not _BaseService.authenticate - ) - client_config = getattr(service_module, "CLIENT_CONFIG", ()) - if client_config: - service_data["client_config"] = list(client_config) - service_data["has_search"] = getattr(service_module, "search", None) is not _BaseService.search - service_data["anonymous_search"] = bool(getattr(service_module, "ANONYMOUS_SEARCH", False)) - service_data["has_drm"] = ( - getattr(service_module, "get_widevine_license", None) is not _BaseService.get_widevine_license - or getattr(service_module, "get_playready_license", None) is not _BaseService.get_playready_license - ) - - # Prefer the service's explicit AUTH_METHODS; otherwise infer from authenticate(). - methods = [] - if service_data["needs_auth"]: - declared = getattr(service_module, "AUTH_METHODS", None) - if declared: - methods = list(declared) - else: - try: - import inspect as _inspect - - src_lines = _inspect.getsource(service_module.authenticate).splitlines() - start = next((i + 1 for i, ln in enumerate(src_lines) if ln.rstrip().endswith(":")), 1) - body = "\n".join(src_lines[start:]) - if "cookies" in body: - methods.append("cookies") - if "credential" in body: - methods.append("credentials") - except (OSError, TypeError): - pass - if not methods: - methods = ["cookies"] - service_data["auth_methods"] = methods - - except Exception as e: - log.warning(f"Could not load details for service {tag}: {e}") - services_info.append(service_data) load_errors = [redact_path(re.sub(r" \([^()]*\)$", "", err)) for err in services_module.LOAD_ERRORS] diff --git a/unshackle/core/proxies/controld.py b/unshackle/core/proxies/controld.py index a0cfc1a7..9611aa99 100644 --- a/unshackle/core/proxies/controld.py +++ b/unshackle/core/proxies/controld.py @@ -15,7 +15,7 @@ from urllib.parse import urlsplit import requests from requests.adapters import HTTPAdapter -from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.proxy import Proxy, cached_catalogue from unshackle.core.utils.redact import REDACTED log = logging.getLogger("ControlD") @@ -369,6 +369,7 @@ class ControlD(Proxy): return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + @cached_catalogue def get_locations(self) -> list[dict]: """Get Control D's proxy locations. Public data, so no token is needed.""" try: diff --git a/unshackle/core/proxies/hola.py b/unshackle/core/proxies/hola.py index c21ee5d9..6d81684b 100644 --- a/unshackle/core/proxies/hola.py +++ b/unshackle/core/proxies/hola.py @@ -4,7 +4,7 @@ import subprocess from typing import Optional from unshackle.core import binaries -from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.proxy import Proxy, cached_catalogue class Hola(Proxy): @@ -71,6 +71,7 @@ class Hola(Proxy): return random.choice(proxies) if proxies else None + @cached_catalogue def get_countries(self) -> list[dict[str, str]]: """Get a list of available Countries.""" p = subprocess.run( diff --git a/unshackle/core/proxies/nordvpn.py b/unshackle/core/proxies/nordvpn.py index 095f49dd..c6cfb4df 100644 --- a/unshackle/core/proxies/nordvpn.py +++ b/unshackle/core/proxies/nordvpn.py @@ -6,7 +6,7 @@ from typing import Optional, Union import requests from unshackle.core.proxies.asn import describe, is_ip, mapped_query, select, split_selector -from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.proxy import Proxy, cached_catalogue from unshackle.core.utilities import COUNTRY_CODE_ALIASES @@ -186,6 +186,7 @@ class NordVPN(Proxy): raise ValueError("Could not decode list of NordVPN countries, not JSON data.") @staticmethod + @cached_catalogue def get_countries() -> list[dict]: """Get a list of available Countries and their metadata.""" res = requests.get(url="https://api.nordvpn.com/v1/servers/countries", timeout=10) diff --git a/unshackle/core/proxies/proxy.py b/unshackle/core/proxies/proxy.py index fd00fa95..6be90b54 100644 --- a/unshackle/core/proxies/proxy.py +++ b/unshackle/core/proxies/proxy.py @@ -1,5 +1,31 @@ +import functools +import time from abc import abstractmethod -from typing import Any, Optional +from typing import Any, Callable, Optional, TypeVar + +CATALOGUE_TTL = 600.0 +CATALOGUES: dict[str, tuple[float, Any]] = {} + +F = TypeVar("F", bound=Callable[..., Any]) + + +def cached_catalogue(fetch: F) -> F: + """Keep a proxy provider's server catalogue for ``CATALOGUE_TTL`` seconds; the next call fetches an empty answer again. + + A server builds its proxy providers on every request, and each build fetches the catalogue. + """ + + @functools.wraps(fetch) + def wrapper(*args: Any, **kwargs: Any) -> Any: + hit = CATALOGUES.get(fetch.__qualname__) + if hit and time.monotonic() - hit[0] < CATALOGUE_TTL: + return hit[1] + value = fetch(*args, **kwargs) + if value: + CATALOGUES[fetch.__qualname__] = (time.monotonic(), value) + return value + + return wrapper # type: ignore[return-value] class Proxy: diff --git a/unshackle/core/proxies/surfsharkvpn.py b/unshackle/core/proxies/surfsharkvpn.py index cd00401b..1b6c715b 100644 --- a/unshackle/core/proxies/surfsharkvpn.py +++ b/unshackle/core/proxies/surfsharkvpn.py @@ -14,7 +14,7 @@ import requests from unshackle.core.proxies.asn import describe, fold, is_ip, mapped_query, select, split_selector from unshackle.core.proxies.controld import build_query, parse_a_records -from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.proxy import Proxy, cached_catalogue from unshackle.core.utilities import COUNTRY_CODE_ALIASES log = logging.getLogger("proxies.surfsharkvpn") @@ -241,6 +241,7 @@ class SurfsharkVPN(Proxy): return servers @staticmethod + @cached_catalogue def get_countries() -> list[dict]: """Get a list of available Countries and their metadata.""" res = requests.get( diff --git a/unshackle/core/proxies/windscribevpn.py b/unshackle/core/proxies/windscribevpn.py index 16dd7b65..1513f79f 100644 --- a/unshackle/core/proxies/windscribevpn.py +++ b/unshackle/core/proxies/windscribevpn.py @@ -6,7 +6,7 @@ from typing import Iterator, Optional import requests from unshackle.core.proxies.asn import describe, mapped_query, resolve_all, select, split_selector -from unshackle.core.proxies.proxy import Proxy +from unshackle.core.proxies.proxy import Proxy, cached_catalogue class WindscribeVPN(Proxy): @@ -163,6 +163,7 @@ class WindscribeVPN(Proxy): return None @staticmethod + @cached_catalogue def get_countries() -> list[dict]: """Get a list of available Countries and their metadata.""" res = requests.get(