Files
unshackle/unshackle/core/api/download_manager.py
T
imSp4rky 8a9a1e1bdc refactor(api): remove dead code paths in serve/remote layer
Delete code paths that never executed in the serve/API/remote stack:

- BridgeCancelledError: raised but never caught; replace raises with RuntimeError and drop the class.
- InputBridge.error setter and _error field: written at the handler layer, never read (session.auth_error is the surfaced channel).
- cancel_event param and _check_cancel inner fn on _perform_download: the only caller always passed None (the live job.cancel_event is untouched).
- result_with_progress monkeypatch: its status pings were redundant with the file-polled progress_sink, which is now the sole path.
- session param on _resolve_manifest_data: unused in the body.

Reconcile SessionStore's hardcoded 600s auth-state TTL into a shared AUTH_INPUT_TIMEOUT constant referenced by both the InputBridge input timeout and the store, removing the magic number without changing the longer grace window.

Update input_bridge/session_store tests for the new channels and add an assertion that auth-state entries expire at the single rule.
2026-06-21 13:07:45 -06:00

852 lines
36 KiB
Python

import asyncio
import json
import logging
import os
import re
import sys
import tempfile
import threading
import uuid
from contextlib import suppress
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from unshackle.core.api.sanitize import sanitize_log
from unshackle.core.utils.redact import REDACTED, URL_USERINFO_RE, redact_text
log = logging.getLogger("download_manager")
# Job parameters may carry secrets (a raw "user:pass" credential, a proxy URL with embedded
# userinfo). These must never leave the process via the API or logs, so they are masked
# wherever parameters are serialized for a response.
_SENSITIVE_PARAM_KEYS = ("credential", "credentials", "password", "token", "api_key")
def _redact_parameters(parameters: Dict[str, Any]) -> Dict[str, Any]:
"""Return a copy of job parameters with secrets masked, safe to serialize."""
if not isinstance(parameters, dict):
return parameters
redacted = dict(parameters)
for key in _SENSITIVE_PARAM_KEYS:
if redacted.get(key):
redacted[key] = REDACTED
proxy = redacted.get("proxy")
if isinstance(proxy, str) and "@" in proxy:
redacted["proxy"] = URL_USERINFO_RE.sub(f"{REDACTED}@", proxy)
return redacted
def _secret_values(parameters: Dict[str, Any]) -> List[str]:
"""Raw secret strings carried in job parameters, longest first, for scrubbing free text."""
if not isinstance(parameters, dict):
return []
secrets: List[str] = []
for key in ("credential", "password", "token", "api_key"):
value = parameters.get(key)
if isinstance(value, str) and value:
secrets.append(value)
if key == "credential" and ":" in value:
password = value.split(":", 1)[1]
if len(password) >= 4: # short passwords would blanket-replace and garble the text
secrets.append(password)
creds = parameters.get("credentials")
if isinstance(creds, dict):
secrets.extend(v for v in creds.values() if isinstance(v, str) and v)
elif isinstance(creds, str) and creds:
secrets.append(creds)
return sorted(set(secrets), key=len, reverse=True) # longest first so substrings don't survive
def _redact_text(text: Optional[str], parameters: Dict[str, Any]) -> Optional[str]:
"""Mask proxy userinfo and any known parameter secrets that leaked into a free-text field
(error message / details / traceback / worker stderr) before it is returned via the API."""
return redact_text(text, _secret_values(parameters))
class JobStatus(Enum):
QUEUED = "queued"
DOWNLOADING = "downloading"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
@dataclass
class DownloadJob:
"""Represents a download job with all its parameters and status."""
job_id: str
status: JobStatus
created_time: datetime
service: str
title_id: str
parameters: Dict[str, Any]
# Progress tracking
started_time: Optional[datetime] = None
completed_time: Optional[datetime] = None
progress: float = 0.0
# Results and error info
output_files: List[str] = field(default_factory=list)
error_message: Optional[str] = None
error_details: Optional[str] = None
error_code: Optional[str] = None
error_traceback: Optional[str] = None
worker_stderr: Optional[str] = None
# Current phase, track counts, and labels of the tracks downloading now.
phase: Optional[str] = None
completed_tracks: int = 0
total_tracks: int = 0
active_tracks: List[str] = field(default_factory=list)
# Segment counts and transfer speed of the track downloading now (granular display)
segments_done: float = 0.0
segments_total: float = 0.0
speed: Optional[str] = None
# Subtitles skipped under skip_subtitle_errors (non-fatal). Each entry is a dl.SkippedSubtitle
# dict (id / language / title) so a client can report which weren't available.
skipped_subtitles: List[Dict[str, Any]] = field(default_factory=list)
# Cancellation support
cancel_event: threading.Event = field(default_factory=threading.Event)
def to_dict(self, include_full_details: bool = False) -> Dict[str, Any]:
"""Convert job to dictionary for JSON response."""
result = {
"job_id": self.job_id,
"status": self.status.value,
"created_time": self.created_time.isoformat(),
"service": self.service,
"title_id": self.title_id,
"progress": self.progress,
"phase": self.phase,
"completed_tracks": self.completed_tracks,
"total_tracks": self.total_tracks,
"active_tracks": self.active_tracks,
"segments_done": self.segments_done,
"segments_total": self.segments_total,
"speed": self.speed,
"skipped_subtitles": self.skipped_subtitles,
}
if include_full_details:
# Error/stderr/traceback are free text a service may have echoed a credential or proxy
# URL into, so scrub them with the same secrets that _redact_parameters masks.
result.update(
{
"parameters": _redact_parameters(self.parameters),
"started_time": self.started_time.isoformat() if self.started_time else None,
"completed_time": self.completed_time.isoformat() if self.completed_time else None,
"output_files": self.output_files,
"error_message": _redact_text(self.error_message, self.parameters),
"error_details": _redact_text(self.error_details, self.parameters),
"error_code": self.error_code,
"error_traceback": _redact_text(self.error_traceback, self.parameters),
"worker_stderr": _redact_text(self.worker_stderr, self.parameters),
}
)
return result
def _perform_download(
job_id: str,
service: str,
title_id: str,
params: Dict[str, Any],
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
) -> List[str]:
"""Execute the synchronous download logic for a job."""
from contextlib import redirect_stderr, redirect_stdout
from io import StringIO
# Import dl.py components lazily to avoid circular deps during module import
import click
import yaml
from unshackle.commands.dl import dl
from unshackle.core.api.errors import APIError, APIErrorCode
from unshackle.core.config import config
from unshackle.core.services import Services
from unshackle.core.tracks import Subtitle, Video
from unshackle.core.utils.click_types import ContextData
from unshackle.core.utils.collections import merge_dict
log.info(f"Starting sync download for job {job_id}")
# A service caches tokens under cache/<Service>/, keyed by service name only, so two jobs on
# one service with different credentials would share a cache. When a per-job credential is set,
# namespace the cache dir by a hash of it so the sessions can't cross.
job_credential = params.get("credential")
if job_credential:
import hashlib
cred_hash = hashlib.sha256(job_credential.encode("utf-8")).hexdigest()[:12]
config.directories.cache = config.directories.cache / "_jobs" / cred_hash
# Convert string parameters to enums (API receives strings, dl.result() expects enums)
vcodec_raw = params.get("vcodec")
if vcodec_raw:
if isinstance(vcodec_raw, str):
vcodec_raw = [vcodec_raw]
if isinstance(vcodec_raw, list) and vcodec_raw and not isinstance(vcodec_raw[0], Video.Codec):
codec_map = {c.name.upper(): c for c in Video.Codec}
codec_map.update({c.value.upper(): c for c in Video.Codec})
params["vcodec"] = [codec_map[v.upper()] for v in vcodec_raw if v.upper() in codec_map]
else:
params["vcodec"] = []
range_raw = params.get("range")
if range_raw:
if isinstance(range_raw, str):
range_raw = [range_raw]
if isinstance(range_raw, list) and range_raw and not isinstance(range_raw[0], Video.Range):
range_map = {r.name.upper(): r for r in Video.Range}
range_map.update({r.value.upper(): r for r in Video.Range})
params["range"] = [range_map[r.upper()] for r in range_raw if r.upper() in range_map]
else:
params["range"] = [Video.Range.SDR]
sub_format_raw = params.get("sub_format")
if sub_format_raw and isinstance(sub_format_raw, str):
sub_map = {c.name.upper(): c for c in Subtitle.Codec}
sub_map.update({c.value.upper(): c for c in Subtitle.Codec})
params["sub_format"] = sub_map.get(sub_format_raw.upper())
if params.get("export"):
params["export"] = bool(params["export"])
# Normalize slow: accept string "MIN-MAX", list/tuple, or True (default 60-120)
slow_raw = params.get("slow")
if slow_raw is not None and not isinstance(slow_raw, tuple):
if isinstance(slow_raw, bool):
params["slow"] = (60, 120) if slow_raw else None
elif isinstance(slow_raw, list) and len(slow_raw) == 2:
params["slow"] = (int(slow_raw[0]), int(slow_raw[1]))
elif isinstance(slow_raw, str):
from unshackle.core.utils.click_types import SLOW_DELAY_RANGE
try:
params["slow"] = SLOW_DELAY_RANGE.convert(slow_raw, None, None)
except click.BadParameter as exc:
raise Exception(f"Invalid slow parameter: {exc}")
# Convert wanted episode strings to internal "SxE" format
# Accepts: "S01E01", "S01-S03", "s1e1", "1x1", or already-parsed format
wanted_raw = params.get("wanted")
if wanted_raw:
from unshackle.core.utils.click_types import SeasonRange
if isinstance(wanted_raw, str):
wanted_raw = [wanted_raw]
# Only convert if not already in internal "SxE" format
needs_conversion = any(not re.match(r"^\d+x\d+$", w) for w in wanted_raw)
if needs_conversion:
season_range = SeasonRange()
params["wanted"] = season_range.parse_tokens(*wanted_raw)
# Load service configuration
service_config_path = Services.get_path(service) / config.filenames.config
if service_config_path.exists():
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
else:
service_config = {}
merge_dict(config.services.get(service), service_config)
from unshackle.commands.dl import dl as dl_command
ctx = click.Context(dl_command.cli)
ctx.invoked_subcommand = service
from unshackle.core.api.handlers import load_full_cdm
cdm = load_full_cdm(service, params.get("profile"), params.get("cdm_type"))
ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=params.get("profile"))
ctx.params = {
"proxy": params.get("proxy"),
"no_proxy": params.get("no_proxy", False),
"no_proxy_download": params.get("no_proxy_download", False),
"profile": params.get("profile"),
"repack": params.get("repack", False),
"tag": params.get("tag"),
"tmdb_id": params.get("tmdb_id"),
"imdb_id": params.get("imdb_id"),
"animeapi_id": params.get("animeapi_id"),
"enrich": params.get("enrich", False),
"output_dir": Path(params["output_dir"]) if params.get("output_dir") else None,
"no_cache": params.get("no_cache", False),
"reset_cache": params.get("reset_cache", False),
}
# Hand-built context: record parameter sources so service dl overrides
# apply to defaults but never clobber client-sent values.
from click.core import ParameterSource
for param_name in ctx.params:
ctx.set_parameter_source(
param_name, ParameterSource.COMMANDLINE if param_name in params else ParameterSource.DEFAULT
)
dl_instance = dl(
ctx=ctx,
no_proxy=params.get("no_proxy", False),
profile=params.get("profile"),
proxy=params.get("proxy"),
repack=params.get("repack", False),
tag=params.get("tag"),
tmdb_id=params.get("tmdb_id"),
imdb_id=params.get("imdb_id"),
animeapi_id=params.get("animeapi_id"),
enrich=params.get("enrich", False),
output_dir=Path(params["output_dir"]) if params.get("output_dir") else None,
)
# Per-request CDM override (a device name in the WVDs dir); get_cdm() takes it first.
if params.get("cdm"):
dl_instance.cdm_override = params["cdm"]
# Per-request credential ("user:pass"); feed it into the map get_credentials() reads so a
# client can authenticate without anything being persisted to disk. Without a profile,
# get_credentials() falls back to "default", so store it there too rather than dropping it
# (which would silently authenticate as the server's own default account).
if params.get("credential"):
svc_creds = config.credentials.get(service)
if not isinstance(svc_creds, dict):
config.credentials[service] = svc_creds = {}
svc_creds[params.get("profile") or "default"] = params["credential"]
service_module = Services.load(service)
try:
import inspect
service_init_params = inspect.signature(service_module.__init__).parameters
service_ctx = click.Context(click.Command(service))
service_ctx.parent = ctx
service_ctx.obj = ctx.obj
service_kwargs = {}
if "title" in service_init_params:
service_kwargs["title"] = title_id
for key, value in params.items():
if key in service_init_params and key not in ["service", "title_id"]:
service_kwargs[key] = value
for param_name, param_info in service_init_params.items():
if param_name not in service_kwargs and param_name not in ["self", "ctx"]:
if param_info.default is inspect.Parameter.empty:
if param_name == "movie":
service_kwargs[param_name] = "/movies/" in title_id
elif param_name == "meta_lang":
service_kwargs[param_name] = None
else:
log.warning(f"Unknown required parameter '{param_name}' for service {service}, using None")
service_kwargs[param_name] = None
service_instance = service_module(service_ctx, **service_kwargs)
except Exception as exc: # noqa: BLE001 - propagate meaningful failure
log.error(f"Failed to create service instance: {exc}")
raise
original_download_dir = config.directories.downloads
stdout_capture = StringIO()
stderr_capture = StringIO()
# progress_sink (dl.build_job_progress_callables) owns percentage; terminal status
# (success/failure) is reported by the worker's result file.
try:
with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture):
dl_instance.result(
service=service_instance,
quality=params.get("quality", []),
vcodec=params.get("vcodec", []),
acodec=params.get("acodec"),
vbitrate=params.get("vbitrate"),
abitrate=params.get("abitrate"),
vbitrate_range=params.get("vbitrate_range"),
abitrate_range=params.get("abitrate_range"),
range_=params.get("range", ["SDR"]),
channels=params.get("channels"),
no_atmos=params.get("no_atmos", False),
select_titles=False,
wanted=params.get("wanted", []),
latest_episode=params.get("latest_episode", False),
lang=params.get("lang", ["orig"]),
v_lang=params.get("v_lang", []),
a_lang=params.get("a_lang", []),
s_lang=params.get("s_lang", ["all"]),
require_subs=params.get("require_subs", []),
forced_subs=params.get("forced_subs", False),
exact_lang=params.get("exact_lang", False),
sub_format=params.get("sub_format"),
video_only=params.get("video_only", False),
audio_only=params.get("audio_only", False),
subs_only=params.get("subs_only", False),
chapters_only=params.get("chapters_only", False),
no_subs=params.get("no_subs", False),
skip_subtitle_errors=params.get("skip_subtitle_errors", False),
no_audio=params.get("no_audio", False),
no_chapters=params.get("no_chapters", False),
no_video=params.get("no_video", False),
audio_description=params.get("audio_description", False),
slow=params.get("slow", None),
list_=False,
list_titles=False,
skip_dl=params.get("skip_dl", False),
export=params.get("export"),
cdm_only=params.get("cdm_only"),
no_proxy=params.get("no_proxy", False),
no_proxy_download=params.get("no_proxy_download", False),
no_folder=params.get("no_folder", False),
no_source=params.get("no_source", False),
no_mux=params.get("no_mux", False),
workers=params.get("workers"),
downloads=params.get("downloads", 1),
worst=params.get("worst", False),
best_available=params.get("best_available", False),
split_audio=params.get("split_audio"),
progress_sink=progress_callback,
)
except SystemExit as exc:
if exc.code != 0:
stdout_str = stdout_capture.getvalue()
stderr_str = stderr_capture.getvalue()
log.error(f"Download exited with code {exc.code}")
log.error(f"Stdout: {stdout_str}")
log.error(f"Stderr: {stderr_str}")
raise APIError(APIErrorCode.DOWNLOAD_ERROR, f"Download failed with exit code {exc.code}")
except Exception as exc: # noqa: BLE001 - propagate to caller
stdout_str = stdout_capture.getvalue()
stderr_str = stderr_capture.getvalue()
log.error(f"Download execution failed: {exc}")
log.error(f"Stdout: {stdout_str}")
log.error(f"Stderr: {stderr_str}")
raise
# dl.result() catches a download-worker exception, reports it, but returns normally (exit 0).
# It sets download_failed in that case, so the job isn't reported as completed with no output.
if getattr(dl_instance, "download_failed", False):
detail = (stdout_capture.getvalue() + stderr_capture.getvalue())[-200:].strip()
raise APIError(APIErrorCode.WORKER_ERROR, "download worker failed: " + (detail or "see logs"))
# Surface any subtitles that were skipped (non-fatal failures) so the client can report them.
if progress_callback:
skipped_subs = getattr(dl_instance, "skipped_subtitles", None)
if skipped_subs:
progress_callback({"skipped_subtitles": list(skipped_subs)})
output_files = [str(p) for p in dl_instance.completed_files]
log.info(f"Download completed for job {job_id}, {len(output_files)} file(s) in {original_download_dir}")
return output_files
class DownloadQueueManager:
"""Manages download job queue with configurable concurrency limits."""
def __init__(self, max_concurrent_downloads: int = 2, job_retention_hours: int = 24):
self.max_concurrent_downloads = max_concurrent_downloads
self.job_retention_hours = job_retention_hours
self._jobs: Dict[str, DownloadJob] = {}
self._job_queue: asyncio.Queue = asyncio.Queue()
self._active_downloads: Dict[str, asyncio.Task] = {}
self._download_processes: Dict[str, asyncio.subprocess.Process] = {}
self._job_temp_files: Dict[str, Dict[str, str]] = {}
self._workers_started = False
self._shutdown_event = asyncio.Event()
log.info(
f"Initialized download queue manager: max_concurrent={max_concurrent_downloads}, retention_hours={job_retention_hours}"
)
def create_job(self, service: str, title_id: str, **parameters) -> DownloadJob:
"""Create a new download job and add it to the queue."""
job_id = str(uuid.uuid4())
job = DownloadJob(
job_id=job_id,
status=JobStatus.QUEUED,
created_time=datetime.now(),
service=service,
title_id=title_id,
parameters=parameters,
)
self._jobs[job_id] = job
self._job_queue.put_nowait(job)
log.info(f"Created download job {job_id} for {sanitize_log(service)}:{sanitize_log(title_id)}")
return job
def get_job(self, job_id: str) -> Optional[DownloadJob]:
"""Get job by ID."""
return self._jobs.get(job_id)
def list_jobs(self) -> List[DownloadJob]:
"""List all jobs."""
return list(self._jobs.values())
def cancel_job(self, job_id: str) -> bool:
"""Cancel a job if it's queued or downloading."""
job = self._jobs.get(job_id)
if not job:
return False
if job.status == JobStatus.QUEUED:
job.status = JobStatus.CANCELLED
job.cancel_event.set() # Signal cancellation
log.info(f"Cancelled queued job {sanitize_log(job_id)}")
return True
elif job.status == JobStatus.DOWNLOADING:
# Set the cancellation event first - this will be checked by the download thread
job.cancel_event.set()
job.status = JobStatus.CANCELLED
log.info(f"Signaled cancellation for downloading job {sanitize_log(job_id)}")
# Cancel the active download task
task = self._active_downloads.get(job_id)
if task:
task.cancel()
log.info(f"Cancelled download task for job {sanitize_log(job_id)}")
process = self._download_processes.get(job_id)
if process:
try:
process.terminate()
log.info(f"Terminated worker process for job {sanitize_log(job_id)}")
except ProcessLookupError:
log.debug(f"Worker process for job {sanitize_log(job_id)} already exited")
return True
return False
def cleanup_old_jobs(self) -> int:
"""Remove jobs older than retention period."""
cutoff_time = datetime.now() - timedelta(hours=self.job_retention_hours)
jobs_to_remove = []
for job_id, job in self._jobs.items():
if job.status in [JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED]:
if job.completed_time and job.completed_time < cutoff_time:
jobs_to_remove.append(job_id)
elif not job.completed_time and job.created_time < cutoff_time:
jobs_to_remove.append(job_id)
for job_id in jobs_to_remove:
del self._jobs[job_id]
if jobs_to_remove:
log.info(f"Cleaned up {len(jobs_to_remove)} old jobs")
return len(jobs_to_remove)
async def start_workers(self):
"""Start worker tasks to process the download queue."""
if self._workers_started:
return
self._workers_started = True
# Start worker tasks
for i in range(self.max_concurrent_downloads):
asyncio.create_task(self._download_worker(f"worker-{i}"))
# Start cleanup task
asyncio.create_task(self._cleanup_worker())
log.info(f"Started {self.max_concurrent_downloads} download workers")
async def shutdown(self):
"""Shutdown the queue manager and cancel all active downloads."""
log.info("Shutting down download queue manager")
self._shutdown_event.set()
# Cancel all active downloads
for task in self._active_downloads.values():
task.cancel()
# Terminate worker processes
for job_id, process in list(self._download_processes.items()):
try:
process.terminate()
except ProcessLookupError:
log.debug(f"Worker process for job {job_id} already exited during shutdown")
for job_id, process in list(self._download_processes.items()):
try:
await asyncio.wait_for(process.wait(), timeout=5)
except asyncio.TimeoutError:
log.warning(f"Worker process for job {job_id} did not exit, killing")
process.kill()
await process.wait()
finally:
self._download_processes.pop(job_id, None)
# Clean up any remaining temp files
for paths in self._job_temp_files.values():
for path in paths.values():
try:
os.remove(path)
except OSError:
pass
self._job_temp_files.clear()
# Wait for workers to finish
if self._active_downloads:
await asyncio.gather(*self._active_downloads.values(), return_exceptions=True)
async def _download_worker(self, worker_name: str):
"""Worker task that processes jobs from the queue."""
log.debug(f"Download worker {worker_name} started")
while not self._shutdown_event.is_set():
try:
# Wait for a job or shutdown signal
job = await asyncio.wait_for(self._job_queue.get(), timeout=1.0)
if job.status == JobStatus.CANCELLED:
continue
# Start processing the job
job.status = JobStatus.DOWNLOADING
job.started_time = datetime.now()
log.info(f"Worker {worker_name} starting job {job.job_id}")
# Create download task
download_task = asyncio.create_task(self._execute_download(job))
self._active_downloads[job.job_id] = download_task
try:
await download_task
except asyncio.CancelledError:
job.status = JobStatus.CANCELLED
log.info(f"Job {job.job_id} was cancelled")
except Exception as e:
job.status = JobStatus.FAILED
job.error_message = str(e)
log.error(f"Job {job.job_id} failed: {e}")
finally:
job.completed_time = datetime.now()
if job.job_id in self._active_downloads:
del self._active_downloads[job.job_id]
except asyncio.TimeoutError:
continue
except Exception as e:
log.error(f"Worker {worker_name} error: {e}")
async def _execute_download(self, job: DownloadJob):
"""Execute the actual download for a job."""
log.info(f"Executing download for job {job.job_id}")
try:
output_files = await self._run_download_async(job)
job.status = JobStatus.COMPLETED
job.output_files = output_files
job.progress = 100.0
log.info(f"Download completed for job {job.job_id}: {len(output_files)} files")
except Exception as e:
import traceback
from unshackle.core.api.errors import categorize_exception
job.status = JobStatus.FAILED
job.error_message = str(e)
job.error_details = str(e)
api_error = categorize_exception(
e, context={"service": job.service, "title_id": job.title_id, "job_id": job.job_id}
)
job.error_code = api_error.error_code.value
job.error_traceback = traceback.format_exc()
log.error(f"Download failed for job {job.job_id}: {e}")
raise
async def _run_download_async(self, job: DownloadJob) -> List[str]:
"""Invoke a worker subprocess to execute the download."""
payload = {
"job_id": job.job_id,
"service": job.service,
"title_id": job.title_id,
"parameters": job.parameters,
}
payload_fd, payload_path = tempfile.mkstemp(prefix=f"unshackle_job_{job.job_id}_", suffix="_payload.json")
os.close(payload_fd)
result_fd, result_path = tempfile.mkstemp(prefix=f"unshackle_job_{job.job_id}_", suffix="_result.json")
os.close(result_fd)
progress_fd, progress_path = tempfile.mkstemp(prefix=f"unshackle_job_{job.job_id}_", suffix="_progress.json")
os.close(progress_fd)
with open(payload_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle)
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"unshackle.core.api.download_worker",
payload_path,
result_path,
progress_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
self._download_processes[job.job_id] = process
self._job_temp_files[job.job_id] = {"payload": payload_path, "result": result_path, "progress": progress_path}
communicate_task = asyncio.create_task(process.communicate())
stdout_bytes = b""
stderr_bytes = b""
try:
while True:
done, _ = await asyncio.wait({communicate_task}, timeout=0.5)
if communicate_task in done:
stdout_bytes, stderr_bytes = communicate_task.result()
break
# Check for progress updates
try:
if os.path.exists(progress_path):
with open(progress_path, "r", encoding="utf-8") as handle:
progress_data = json.load(handle)
if progress_data.get("phase") and progress_data["phase"] != job.phase:
job.phase = progress_data["phase"]
if progress_data.get("total_tracks"):
job.total_tracks = int(progress_data["total_tracks"])
if progress_data.get("completed_tracks") is not None:
job.completed_tracks = int(progress_data["completed_tracks"])
if "active_tracks" in progress_data:
job.active_tracks = list(progress_data["active_tracks"])
for _sk in ("segments_done", "segments_total"):
if _sk in progress_data:
try:
setattr(job, _sk, float(progress_data[_sk]))
except (TypeError, ValueError):
pass
if progress_data.get("speed"):
job.speed = str(progress_data["speed"])
if progress_data.get("skipped_subtitles"):
job.skipped_subtitles = progress_data["skipped_subtitles"]
if "progress" in progress_data:
new_progress = float(progress_data["progress"])
if new_progress != job.progress:
job.progress = new_progress
log.info(f"Job {job.job_id} progress updated: {job.progress}%")
except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
log.debug(f"Could not read progress for job {job.job_id}: {e}")
if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED:
log.info(f"Cancellation detected for job {job.job_id}, terminating worker process")
process.terminate()
try:
await asyncio.wait_for(communicate_task, timeout=5)
except asyncio.TimeoutError:
log.warning(f"Worker process for job {job.job_id} did not terminate, killing")
process.kill()
await asyncio.wait_for(communicate_task, timeout=5)
raise asyncio.CancelledError("Job was cancelled")
returncode = process.returncode
stdout = stdout_bytes.decode("utf-8", errors="ignore")
stderr = stderr_bytes.decode("utf-8", errors="ignore")
# A service can echo a credential or a proxy URL into its output, so scrub it before
# it reaches the log as well, not only the API response.
safe_stdout = _redact_text(stdout.strip(), job.parameters)
safe_stderr = _redact_text(stderr.strip(), job.parameters)
if stdout.strip():
log.debug(f"Worker stdout for job {job.job_id}: {safe_stdout}")
if stderr.strip():
job.worker_stderr = stderr.strip()
if returncode != 0:
log.warning(f"Worker stderr for job {job.job_id}: {safe_stderr}")
else:
log.debug(f"Worker stderr for job {job.job_id}: {safe_stderr}")
result_data: Optional[Dict[str, Any]] = None
try:
with open(result_path, "r", encoding="utf-8") as handle:
result_data = json.load(handle)
except FileNotFoundError:
log.error(f"Result file missing for job {job.job_id}")
except json.JSONDecodeError as exc:
log.error(f"Failed to parse worker result for job {job.job_id}: {exc}")
if returncode != 0:
message = result_data.get("message") if result_data else "unknown error"
if result_data:
job.error_details = result_data.get("error_details", message)
job.error_code = result_data.get("error_code")
raise Exception(f"Worker exited with code {returncode}: {message}")
if not result_data or result_data.get("status") != "success":
message = result_data.get("message") if result_data else "worker did not report success"
if result_data:
job.error_details = result_data.get("error_details", message)
job.error_code = result_data.get("error_code")
raise Exception(f"Worker failure: {message}")
return result_data.get("output_files", [])
finally:
if not communicate_task.done():
communicate_task.cancel()
with suppress(asyncio.CancelledError):
await communicate_task
self._download_processes.pop(job.job_id, None)
temp_paths = self._job_temp_files.pop(job.job_id, {})
for path in temp_paths.values():
try:
os.remove(path)
except OSError:
pass
async def _cleanup_worker(self):
"""Worker that periodically cleans up old jobs."""
while not self._shutdown_event.is_set():
try:
await asyncio.sleep(3600) # Run every hour
self.cleanup_old_jobs()
except Exception as e:
log.error(f"Cleanup worker error: {e}")
# Global instance
download_manager: Optional[DownloadQueueManager] = None
def get_download_manager() -> DownloadQueueManager:
"""Get the global download manager instance."""
global download_manager
if download_manager is None:
# Load configuration from unshackle config
from unshackle.core.config import config
max_concurrent = getattr(config, "max_concurrent_downloads", 2)
retention_hours = getattr(config, "download_job_retention_hours", 24)
download_manager = DownloadQueueManager(max_concurrent, retention_hours)
return download_manager