forked from kenzuya/unshackle
Session update events carry the new action as action_entry instead of the full log; access lines stay out of the log ring unless --debug. Job events build and redact their dict once, the progress interval matches the 0.5 s poll, and dashboard stats are serialized once.
187 lines
6.6 KiB
Python
187 lines
6.6 KiB
Python
"""Standalone job worker process entry point for executing download jobs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import queue
|
|
import sys
|
|
import threading
|
|
import time
|
|
import traceback
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Dict
|
|
from uuid import uuid4
|
|
|
|
from unshackle.core.api.input_bridge import AUTH_INPUT_TIMEOUT
|
|
from unshackle.core.console import set_notice_handler, set_prompt_handler
|
|
from unshackle.core.downloaders import format_speed, set_speed_limit
|
|
|
|
from .download_manager import perform_download
|
|
|
|
log = logging.getLogger("download_worker")
|
|
|
|
|
|
def read_payload(path: Path) -> Dict[str, Any]:
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
|
|
|
|
_replace_lock = threading.Lock()
|
|
|
|
|
|
def write_result(path: Path, payload: Dict[str, Any]) -> None:
|
|
"""Write the payload with an atomic replace, because the parent polls this file during the write.
|
|
|
|
Each call gets its own temp name, so two threads writing this destination cannot replace away
|
|
the temp file the other is about to move.
|
|
|
|
Windows denies the replace while another thread or the parent holds the destination open. The
|
|
lock removes the in-process contention; the retry covers the parent process.
|
|
"""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
|
|
try:
|
|
tmp.write_text(json.dumps(payload), encoding="utf-8")
|
|
with _replace_lock:
|
|
for attempt in range(10):
|
|
try:
|
|
os.replace(tmp, path)
|
|
return
|
|
except PermissionError:
|
|
if attempt == 9:
|
|
raise
|
|
time.sleep(0.02 * (attempt + 1))
|
|
except OSError:
|
|
tmp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
def relay_prompts(progress_callback: Callable[[Dict[str, Any]], None]) -> None:
|
|
"""Send each prompt_user() prompt to the parent as ``input_prompt`` progress, and read the answer from stdin.
|
|
|
|
The parent writes each answer as one JSON string per line, so an answer can hold a newline. The reader
|
|
uses unbuffered stdin: a daemon thread blocked in ``sys.stdin.buffer`` aborts the interpreter at shutdown.
|
|
Prompts go out one at a time, and a prompt discards answers that arrived before it, such as a late
|
|
answer to a prompt that timed out. A show_notice() message goes out as ``notice`` progress and needs no answer.
|
|
"""
|
|
answers: queue.Queue[str] = queue.Queue()
|
|
one_prompt = threading.Lock()
|
|
|
|
def read_answers() -> None:
|
|
for line in io.FileIO(sys.stdin.fileno(), "rb", closefd=False):
|
|
answers.put(json.loads(line.decode("utf-8", errors="replace")))
|
|
|
|
threading.Thread(target=read_answers, name="prompt-answers", daemon=True).start()
|
|
|
|
def ask(prompt: str) -> str:
|
|
with one_prompt:
|
|
while not answers.empty():
|
|
answers.get_nowait()
|
|
progress_callback({"input_prompt": prompt})
|
|
try:
|
|
return answers.get(timeout=AUTH_INPUT_TIMEOUT)
|
|
except queue.Empty:
|
|
# "authentication" makes categorize_exception report AUTH_FAILED, not a retryable network
|
|
# error: a retry only waits on another prompt.
|
|
raise TimeoutError(
|
|
f"No client answer to the authentication prompt within {AUTH_INPUT_TIMEOUT:.0f}s"
|
|
) from None
|
|
finally:
|
|
progress_callback({"input_prompt": None})
|
|
|
|
set_prompt_handler(ask)
|
|
set_notice_handler(lambda message: progress_callback({"notice": message}))
|
|
|
|
|
|
def apply_payload_speed_limit(payload: Dict[str, Any]) -> None:
|
|
"""Lock this worker's share of the server speed limit so the job cannot clear it."""
|
|
limit = payload.get("speed_limit")
|
|
if not limit:
|
|
return
|
|
set_speed_limit(limit, lock=True)
|
|
log.info(f"Job speed limit: {format_speed(limit)} (share of the global speed limit)")
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
if len(argv) not in [3, 4]:
|
|
print(
|
|
"Usage: python -m unshackle.core.api.download_worker <payload_path> <result_path> [progress_path]",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
payload_path = Path(argv[1])
|
|
result_path = Path(argv[2])
|
|
progress_path = Path(argv[3]) if len(argv) > 3 else None
|
|
|
|
result: Dict[str, Any] = {}
|
|
exit_code = 0
|
|
|
|
try:
|
|
payload = read_payload(payload_path)
|
|
job_id = payload["job_id"]
|
|
service = payload["service"]
|
|
title_id = payload["title_id"]
|
|
params = payload.get("parameters", {})
|
|
apply_payload_speed_limit(payload)
|
|
|
|
log.info(f"Worker starting job {job_id} ({service}:{title_id})")
|
|
|
|
# Merged so sparse keys (current_title, output_files) survive later writes.
|
|
progress_state: Dict[str, Any] = {}
|
|
progress_lock = threading.Lock()
|
|
|
|
def progress_callback(progress_data: Dict[str, Any]) -> None:
|
|
"""Write progress updates to file for main process to read."""
|
|
if progress_path:
|
|
try:
|
|
with progress_lock:
|
|
progress_state.update(progress_data)
|
|
write_result(progress_path, progress_state)
|
|
except Exception as e:
|
|
log.error(f"Failed to write progress update: {e}")
|
|
|
|
relay_prompts(progress_callback)
|
|
output_files = perform_download(job_id, service, title_id, params, progress_callback=progress_callback)
|
|
|
|
result = {"status": "success", "output_files": output_files}
|
|
|
|
except Exception as exc: # noqa: BLE001 - capture for parent process
|
|
from unshackle.core.api.errors import categorize_exception
|
|
|
|
exit_code = 1
|
|
tb = traceback.format_exc()
|
|
log.error(f"Worker failed with error: {exc}")
|
|
|
|
api_error = categorize_exception(
|
|
exc,
|
|
context={
|
|
"service": payload.get("service") if "payload" in locals() else None,
|
|
"title_id": payload.get("title_id") if "payload" in locals() else None,
|
|
"job_id": payload.get("job_id") if "payload" in locals() else None,
|
|
},
|
|
)
|
|
|
|
result = {
|
|
"status": "error",
|
|
"message": str(exc),
|
|
"error_details": api_error.message,
|
|
"error_code": api_error.error_code.value,
|
|
"traceback": tb,
|
|
}
|
|
|
|
finally:
|
|
try:
|
|
write_result(result_path, result)
|
|
except Exception as exc: # noqa: BLE001 - last resort logging
|
|
log.error(f"Failed to write worker result file: {exc}")
|
|
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|