diff --git a/docs/reference/configuration/services.md b/docs/reference/configuration/services.md index 6263f818..0131eff3 100644 --- a/docs/reference/configuration/services.md +++ b/docs/reference/configuration/services.md @@ -16,6 +16,8 @@ Per-service configuration, keyed by the canonical service tag (for example `EXAM [`dl`](download.md#dl) block. Set `remote: true` to send the `dl` and `search` runs of that service to a [remote server](#remote_services) without `--remote`. Add `server` to select one of several servers. On the command line, `--local` and `--server` override these keys. A REST API job ignores both keys. + A REST API job takes the other keys that it accepts as download parameters as its defaults, + above the `serve` defaults and below the values that the request sends. See [making a service remote by default](../../dev/rest-api/remote-sessions.md#making-a-service-remote-by-default). Individual services may read any additional keys they define. unshackle hands the merged result diff --git a/tests/remote/unit/test_api_service_dl_overrides.py b/tests/remote/unit/test_api_service_dl_overrides.py new file mode 100644 index 00000000..69060be7 --- /dev/null +++ b/tests/remote/unit/test_api_service_dl_overrides.py @@ -0,0 +1,105 @@ +"""services..dl defaults reach REST API download jobs, below the values the request sends.""" + +import asyncio +from datetime import datetime +from types import SimpleNamespace + +import pytest + +from unshackle.core.api import download_manager as dm +from unshackle.core.api import handlers + +pytestmark = pytest.mark.unit + + +class FakeManager: + def __init__(self): + self.params = None + + async def start_workers(self): + pass + + def create_job(self, service, title_id, owner_key=None, **params): + self.params = params + return SimpleNamespace(job_id="j", status=SimpleNamespace(value="queued"), created_time=datetime.now()) + + +@pytest.fixture +def env(monkeypatch): + manager = FakeManager() + gate_input = {} + monkeypatch.setattr(handlers, "validate_service", lambda service, request=None: "EXAMPLE1") + monkeypatch.setattr(handlers.Services, "load", lambda service: SimpleNamespace()) + monkeypatch.setattr(dm, "get_download_manager", lambda: manager) + monkeypatch.setattr(handlers, "enforce_download_gates", lambda params, request: gate_input.update(params)) + monkeypatch.setattr(handlers, "caller_key", lambda request: None) + monkeypatch.setattr(handlers.config, "serve", {}, raising=False) + monkeypatch.setattr(handlers.config, "services", {}, raising=False) + + def run(body=None, dl=None, serve=None): + if dl is not None: + monkeypatch.setattr(handlers.config, "services", {"EXAMPLE1": {"dl": dl}}) + if serve is not None: + monkeypatch.setattr(handlers.config, "serve", serve) + asyncio.run(handlers.download_handler({"service": "EXAMPLE1", "title_id": "t1", **(body or {})}, None)) + return manager.params + + run.gate = gate_input + return run + + +def test_service_dl_sets_defaults(env): + params = env(dl={"workers": 7, "range": ["HDR10"]}) + assert params["workers"] == 7 + assert params["range"] == ["HDR10"] + + +def test_request_value_beats_service_dl(env): + assert env(body={"workers": 3}, dl={"workers": 7})["workers"] == 3 + + +def test_service_dl_beats_serve(env): + assert env(dl={"workers": 7}, serve={"workers": 5})["workers"] == 7 + + +def test_routing_and_unknown_keys_are_ignored(env): + baseline = env() + params = env(dl={"remote": True, "server": "x", "speed_limit": "1M"}) + assert params.get("remote") == baseline.get("remote") + assert params.get("server") == baseline.get("server") + assert "speed_limit" not in params + + +def test_click_spelling_of_range_lands_as_range(env): + assert env(dl={"range_": ["HLG"]})["range"] == ["HLG"] + + +def test_gate_sees_service_dl(env): + env(dl={"quality": [2160]}) + assert env.gate["quality"] == [2160] + + +def test_client_keys_are_the_body_keys(env): + params = env(body={"workers": 3, "client_keys": ["forged"], "quality": [1080]}, dl={"workers": 7}) + assert params["client_keys"] == ["client_keys", "quality", "workers"] + + +def test_relayed_job_skips_service_dl(env, monkeypatch): + async def relay(data, request): + return "remote-server", {"tag": "EXAMPLE1"}, data + + monkeypatch.setattr(handlers, "relay_target", relay) + baseline_workers = handlers.DEFAULT_DOWNLOAD_PARAMS["workers"] + params = env(dl={"workers": 7}) + assert params["workers"] == baseline_workers + assert set(params["client_keys"]) >= set(handlers.DEFAULT_DOWNLOAD_PARAMS) + + +def test_client_sent_keys_uses_the_stamp(): + params = {"workers": 1, "quality": [1080], "client_keys": ["workers"]} + assert dm.client_sent_keys(params) == {"workers"} + assert "client_keys" not in params + + +def test_client_sent_keys_without_stamp_counts_all_keys(): + assert dm.client_sent_keys({"workers": 1, "quality": None}) == {"workers", "quality"} diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 0bc6a89c..bf1a159b 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -344,6 +344,11 @@ def normalize_sub_format(sub_format_raw: str) -> Any: return mapped[0] if mapped else None +def client_sent_keys(params: Dict[str, Any]) -> set[str]: + """Pop the server's ``client_keys`` stamp; a job without one counts every parameter as client-sent.""" + return set(params.pop("client_keys", params)) + + def perform_download( job_id: str, service: str, @@ -393,6 +398,8 @@ def perform_download( if params.pop(banned, None) is not None: log.warning(f"Ignoring '{banned}' in API download parameters; post-scripts are config-only") + client_keys = client_sent_keys(params) + vcodec_raw = params.get("vcodec") if vcodec_raw: params["vcodec"] = VIDEO_CODEC_LIST.convert(vcodec_raw) @@ -515,7 +522,7 @@ def perform_download( for param_name in ctx.params: source_key = "range" if param_name == "range_" else param_name ctx.set_parameter_source( - param_name, ParameterSource.COMMANDLINE if source_key in params else ParameterSource.DEFAULT + param_name, ParameterSource.COMMANDLINE if source_key in client_keys else ParameterSource.DEFAULT ) dl_instance = dl( diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 835ced0d..84fea94f 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -2398,6 +2398,24 @@ def enforce_download_gates(params: Dict[str, Any], request: Optional[web.Request ) +def service_dl_defaults(service: str) -> Dict[str, Any]: + """The ``services..dl`` values that name an API download parameter, keyed as the API names them.""" + from unshackle.commands.dl import DL_OPTION_ALIASES + from unshackle.core.services import ROUTING_PARAMS + + api_names = {click_name: api_name for api_name, click_name in DL_OPTION_ALIASES.items()} + result: Dict[str, Any] = {} + for key, value in ((ci_get(config.services, service) or {}).get("dl") or {}).items(): + key = api_names.get(key, key) + if value is None or key in ROUTING_PARAMS: + continue + if key not in DEFAULT_DOWNLOAD_PARAMS: + log.debug("Ignoring services.%s.dl key %r: not an API download parameter", service, key) + continue + result[key] = value + return result + + async def download_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: """Answer the download request: make a download job and add it to the queue.""" from unshackle.core.api.download_manager import get_download_manager @@ -2442,10 +2460,13 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] serve_overrides = { k: v for k, v in (config.serve or {}).items() if k in DEFAULT_DOWNLOAD_PARAMS and v is not None } + # A relayed job gets no local service overrides, so none of its keys may count as defaults. + service_dl = {} if server else service_dl_defaults(normalized_service) params_with_defaults = { **DEFAULT_DOWNLOAD_PARAMS, **serve_overrides, **service_specific_defaults, + **service_dl, **filtered_params, } await asyncio.to_thread(enforce_download_gates, {**params_with_defaults, "service": service_tag}, request) @@ -2455,6 +2476,7 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] # The download worker reads this to decide whether dl may load the server's proxy # providers. Always stamped server-side so a client-sent value cannot grant it. params_with_defaults["server_proxy"] = server_proxy_allowed(request) + params_with_defaults["client_keys"] = sorted(params_with_defaults if server else filtered_params) job = manager.create_job(normalized_service, title_id, owner_key=caller_key(request), **params_with_defaults) return web.json_response(