Files
unshackle/unshackle/core/api/download_worker.py
T
imSp4rky f6ce600d71 perf(api)!: cut per-request and per-tick event bus traffic
Session update events carry the new action as action_entry instead of the full log; access lines stay out of the log ring unless --debug. Job events build and redact their dict once, the progress interval matches the 0.5 s poll, and dashboard stats are serialized once.
2026-11-02 14:52:18 -07:00

187 lines
6.6 KiB
Python

"""Standalone job worker process entry point for executing download jobs."""
from __future__ import annotations
import io
import json
import logging
import os
import queue
import sys
import threading
import time
import traceback
from pathlib import Path
from typing import Any, Callable, Dict
from uuid import uuid4
from unshackle.core.api.input_bridge import AUTH_INPUT_TIMEOUT
from unshackle.core.console import set_notice_handler, set_prompt_handler
from unshackle.core.downloaders import format_speed, set_speed_limit
from .download_manager import perform_download
log = logging.getLogger("download_worker")
def read_payload(path: Path) -> Dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
_replace_lock = threading.Lock()
def write_result(path: Path, payload: Dict[str, Any]) -> None:
"""Write the payload with an atomic replace, because the parent polls this file during the write.
Each call gets its own temp name, so two threads writing this destination cannot replace away
the temp file the other is about to move.
Windows denies the replace while another thread or the parent holds the destination open. The
lock removes the in-process contention; the retry covers the parent process.
"""
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
try:
tmp.write_text(json.dumps(payload), encoding="utf-8")
with _replace_lock:
for attempt in range(10):
try:
os.replace(tmp, path)
return
except PermissionError:
if attempt == 9:
raise
time.sleep(0.02 * (attempt + 1))
except OSError:
tmp.unlink(missing_ok=True)
raise
def relay_prompts(progress_callback: Callable[[Dict[str, Any]], None]) -> None:
"""Send each prompt_user() prompt to the parent as ``input_prompt`` progress, and read the answer from stdin.
The parent writes each answer as one JSON string per line, so an answer can hold a newline. The reader
uses unbuffered stdin: a daemon thread blocked in ``sys.stdin.buffer`` aborts the interpreter at shutdown.
Prompts go out one at a time, and a prompt discards answers that arrived before it, such as a late
answer to a prompt that timed out. A show_notice() message goes out as ``notice`` progress and needs no answer.
"""
answers: queue.Queue[str] = queue.Queue()
one_prompt = threading.Lock()
def read_answers() -> None:
for line in io.FileIO(sys.stdin.fileno(), "rb", closefd=False):
answers.put(json.loads(line.decode("utf-8", errors="replace")))
threading.Thread(target=read_answers, name="prompt-answers", daemon=True).start()
def ask(prompt: str) -> str:
with one_prompt:
while not answers.empty():
answers.get_nowait()
progress_callback({"input_prompt": prompt})
try:
return answers.get(timeout=AUTH_INPUT_TIMEOUT)
except queue.Empty:
# "authentication" makes categorize_exception report AUTH_FAILED, not a retryable network
# error: a retry only waits on another prompt.
raise TimeoutError(
f"No client answer to the authentication prompt within {AUTH_INPUT_TIMEOUT:.0f}s"
) from None
finally:
progress_callback({"input_prompt": None})
set_prompt_handler(ask)
set_notice_handler(lambda message: progress_callback({"notice": message}))
def apply_payload_speed_limit(payload: Dict[str, Any]) -> None:
"""Lock this worker's share of the server speed limit so the job cannot clear it."""
limit = payload.get("speed_limit")
if not limit:
return
set_speed_limit(limit, lock=True)
log.info(f"Job speed limit: {format_speed(limit)} (share of the global speed limit)")
def main(argv: list[str]) -> int:
if len(argv) not in [3, 4]:
print(
"Usage: python -m unshackle.core.api.download_worker <payload_path> <result_path> [progress_path]",
file=sys.stderr,
)
return 2
payload_path = Path(argv[1])
result_path = Path(argv[2])
progress_path = Path(argv[3]) if len(argv) > 3 else None
result: Dict[str, Any] = {}
exit_code = 0
try:
payload = read_payload(payload_path)
job_id = payload["job_id"]
service = payload["service"]
title_id = payload["title_id"]
params = payload.get("parameters", {})
apply_payload_speed_limit(payload)
log.info(f"Worker starting job {job_id} ({service}:{title_id})")
# Merged so sparse keys (current_title, output_files) survive later writes.
progress_state: Dict[str, Any] = {}
progress_lock = threading.Lock()
def progress_callback(progress_data: Dict[str, Any]) -> None:
"""Write progress updates to file for main process to read."""
if progress_path:
try:
with progress_lock:
progress_state.update(progress_data)
write_result(progress_path, progress_state)
except Exception as e:
log.error(f"Failed to write progress update: {e}")
relay_prompts(progress_callback)
output_files = perform_download(job_id, service, title_id, params, progress_callback=progress_callback)
result = {"status": "success", "output_files": output_files}
except Exception as exc: # noqa: BLE001 - capture for parent process
from unshackle.core.api.errors import categorize_exception
exit_code = 1
tb = traceback.format_exc()
log.error(f"Worker failed with error: {exc}")
api_error = categorize_exception(
exc,
context={
"service": payload.get("service") if "payload" in locals() else None,
"title_id": payload.get("title_id") if "payload" in locals() else None,
"job_id": payload.get("job_id") if "payload" in locals() else None,
},
)
result = {
"status": "error",
"message": str(exc),
"error_details": api_error.message,
"error_code": api_error.error_code.value,
"traceback": tb,
}
finally:
try:
write_result(result_path, result)
except Exception as exc: # noqa: BLE001 - last resort logging
log.error(f"Failed to write worker result file: {exc}")
return exit_code
if __name__ == "__main__":
sys.exit(main(sys.argv))