fix(serve): defer service hot reload while a remote session uses it

Remote-only serve no longer builds the download queue manager just to read busy services.
This commit is contained in:
imSp4rky
2026-09-13 12:21:31 -06:00
parent 2ae26cf731
commit e40421a855
7 changed files with 34 additions and 12 deletions
+1 -1
View File
@@ -190,7 +190,7 @@ here instead of disappearing, and `/status` still counts it in the service total
`state` values here for the number that loaded.
A `staged` service has an update on disk that a busy service blocks from the import: the repo
pull already happened, so `commit` (what runs now) and `staged_commit` (what waits) differ. It swaps in when the last job for that tag finishes. Both commits are `null`
pull already happened, so `commit` (what runs now) and `staged_commit` (what waits) differ. It swaps in when the last job and the last remote session for that tag finish. Both commits are `null`
for a service from a plain local directory rather than a git repo.
Watch the `service` event rather than polling for the swap.
+1 -1
View File
@@ -62,7 +62,7 @@ If the update check fails (for example, no network), `update_available` and `lat
### `GET /api/services`
Show the streaming services available on this server, filtered by your allowlist. Each entry gives the service's tag, matching rules, capability flags, and its CLI parameters (useful when you make a UI that accepts service-specific options). `load_errors` lists the services the server skipped because they failed to import, at startup or after a service repository refresh; the server also writes each one to its log. An entry marked `pending_update` has newer code staged: it keeps its current code until its running or queued jobs finish.
Show the streaming services available on this server, filtered by your allowlist. Each entry gives the service's tag, matching rules, capability flags, and its CLI parameters (useful when you make a UI that accepts service-specific options). `load_errors` lists the services the server skipped because they failed to import, at startup or after a service repository refresh; the server also writes each one to its log. An entry marked `pending_update` has newer code staged: it keeps its current code until its running or queued jobs and its live remote sessions finish.
=== "Request"
+1 -1
View File
@@ -125,7 +125,7 @@ is the [REST API](../../dev/rest-api/index.md) section. These are the config key
| `max_sessions` | int or null | `100` | Maximum concurrent remote sessions; at the cap the server evicts the oldest. Set it to `null` or `0` for no limit, which is also what `/api/dashboard/status` then reports, so a dashboard can tell "no cap" from a cap that happens to sit at the default. |
| `history_limit` | int | `100` | How many finished jobs to retain in history. |
| `compression_level` | int | `1` | gzip level for responses. |
| `services_refresh_interval` | int (s) | `0` | How often the server pulls the git-backed service repositories in `directories.services` and hot-reloads the services that changed. `0` turns it off. A service with a running or queued job swaps to the new code as soon as its last job finishes. |
| `services_refresh_interval` | int (s) | `0` | How often the server pulls the git-backed service repositories in `directories.services` and hot-reloads the services that changed. `0` turns it off. A service with a running or queued job, or a live remote session, swaps to the new code as soon as its last job and session finish. |
| `global_speed_limit` | str | *(unlimited)* | Server-wide download speed cap, e.g. `10M`, `1.5G` or plain bytes/sec (same format as `speed_limit`). One shared budget across all concurrent jobs; the server ignores per-job speed limits while it is set. |
| `cdm_overrides` | list or bool | *(unset)* | Allowed per-request CDM overrides: a list of permitted device names, or `true` for any. Unset rejects every override. |
| `allow_job_credentials` | bool | `false` | Permit clients to supply credentials per job. |
@@ -111,6 +111,19 @@ def test_get_download_manager_returns_singleton() -> None:
assert a is b
async def test_busy_services_counts_a_live_remote_session() -> None:
from unshackle.core.api.download_manager import busy_services
from unshackle.core.api.session_store import get_session_store
store = get_session_store()
session = await store.create("EXAMPLE", object(), session_id="busy-session")
try:
assert "EXAMPLE" in busy_services()
finally:
await store.delete(session.session_id)
assert "EXAMPLE" not in busy_services()
def test_job_status_values() -> None:
assert {s.value for s in JobStatus} == {
"queued",
+4 -5
View File
@@ -52,22 +52,21 @@ def _install_service_refresh(app: web.Application) -> None:
return
async def loop() -> None:
from unshackle.core.api.download_manager import get_download_manager
from unshackle.core.api.download_manager import busy_services
manager = get_download_manager()
while True:
await asyncio.sleep(interval)
try:
applied = await asyncio.to_thread(services.apply_pending, manager.busy_services())
applied = await asyncio.to_thread(services.apply_pending, busy_services())
if applied:
log.info(f"Services reloaded: {', '.join(applied)}")
publish_service_event("applied", applied)
repos = await asyncio.to_thread(services.refresh_and_reload, manager.busy_services())
repos = await asyncio.to_thread(services.refresh_and_reload, busy_services())
for r in repos:
if r["changes"]:
log.info(f"Services refreshed {r['spec']}: {', '.join(r['changes'])}")
if r["deferred"]:
log.info(f"Services staged until their jobs finish: {', '.join(r['deferred'])}")
log.info(f"Services staged until their jobs and sessions finish: {', '.join(r['deferred'])}")
for err in r["load_errors"]:
log.error(f"Service reload failed: {err}")
publish_refresh_events(repos)
+12 -1
View File
@@ -753,7 +753,7 @@ class DownloadQueueManager:
async def run() -> None:
try:
applied = await asyncio.to_thread(services.apply_pending, self.busy_services())
applied = await asyncio.to_thread(services.apply_pending, busy_services())
if applied:
log.info(f"Services reloaded after job completion: {', '.join(applied)}")
publish_service_event("applied", applied)
@@ -1198,3 +1198,14 @@ def get_download_manager() -> DownloadQueueManager:
download_manager = DownloadQueueManager(max_concurrent, retention_hours)
return download_manager
def busy_services() -> set[str]:
"""Tags a hot reload must not swap: those with an unfinished job and those a live remote session uses."""
from unshackle.core.api.session_store import get_session_store
from unshackle.core.api.stats import stats
busy = {entry.service_tag for entry in get_session_store().list()}
if stats.mode != "remote_only":
busy |= get_download_manager().busy_services()
return busy
+2 -3
View File
@@ -2734,14 +2734,13 @@ async def clear_temp_handler(request: Optional[web.Request] = None) -> web.Respo
async def refresh_services_handler(request: Optional[web.Request] = None) -> web.Response:
"""Refresh the service repos configured in directories.services and reload the changed services."""
from unshackle.core.api.download_manager import get_download_manager
from unshackle.core.api.download_manager import busy_services
from unshackle.core.api.events import publish_refresh_events
from unshackle.core.services import refresh_and_reload
require_admin(request)
try:
busy = get_download_manager().busy_services()
repos = await asyncio.to_thread(refresh_and_reload, busy)
repos = await asyncio.to_thread(refresh_and_reload, busy_services())
publish_refresh_events(repos)
return web.json_response({"refreshed": all(r["updated"] for r in repos), "repos": repos})