mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-22 00:45:10 +00:00
feat(services): load service plugins from git repos
Allow directories.services entries to be git repo specs (URL/owner-repo/ @branch) alongside local paths: clone + TTL-pull as cache, list-order priority with a duplicate-ignored summary, a refresh-services command that force-syncs and reports the per-service diff, and a dirty-clone guard that protects local edits on the automated path. Adds binaries.Git and a redact_paths config toggle.
This commit is contained in:
@@ -96,6 +96,7 @@ def check() -> None:
|
||||
},
|
||||
{"name": "Caddy", "binary": binaries.Caddy, "required": False, "desc": "Web server", "cat": "Network"},
|
||||
{"name": "Docker", "binary": binaries.Docker, "required": False, "desc": "Gluetun VPN", "cat": "Network"},
|
||||
{"name": "git", "binary": binaries.Git, "required": False, "desc": "Service repos", "cat": "Network"},
|
||||
]
|
||||
|
||||
# Track overall status
|
||||
|
||||
@@ -19,6 +19,34 @@ def util() -> None:
|
||||
"""Various helper scripts and programs."""
|
||||
|
||||
|
||||
@util.command(name="refresh-services")
|
||||
def refresh_services() -> None:
|
||||
"""Force a refresh (git pull) of all service repos configured in directories.services."""
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.service_repo import is_repo_spec, refresh_repo
|
||||
from unshackle.core.utils.redact import redact_path
|
||||
|
||||
entries = config.directories.services
|
||||
if not isinstance(entries, list):
|
||||
entries = [entries]
|
||||
specs = [e for e in entries if isinstance(e, str) and is_repo_spec(e)]
|
||||
if not specs:
|
||||
click.echo("No service repos configured in directories.services.")
|
||||
return
|
||||
# manual refresh force-overwrites local changes (hard reset to upstream), then reports the diff
|
||||
for spec in specs:
|
||||
dest, changes = refresh_repo(spec)
|
||||
if not dest:
|
||||
click.echo(f"Failed to update {spec} (see log).")
|
||||
continue
|
||||
if changes:
|
||||
click.echo(f"Updated {spec} → {redact_path(str(dest))}")
|
||||
for line in changes:
|
||||
click.echo(f" {line}")
|
||||
else:
|
||||
click.echo(f"No changes {spec}")
|
||||
|
||||
|
||||
@util.command()
|
||||
@click.argument("path", type=Path)
|
||||
@click.argument("aspect", type=str)
|
||||
|
||||
@@ -57,6 +57,7 @@ HDR10PlusTool = find("hdr10plus_tool", "HDR10Plus_tool")
|
||||
Mp4decrypt = find("mp4decrypt")
|
||||
Docker = find("docker")
|
||||
ML_Worker = find("ML-Worker")
|
||||
Git = find("git")
|
||||
|
||||
|
||||
__all__ = (
|
||||
@@ -76,5 +77,6 @@ __all__ = (
|
||||
"Mp4decrypt",
|
||||
"Docker",
|
||||
"ML_Worker",
|
||||
"Git",
|
||||
"find",
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any, Optional
|
||||
import yaml
|
||||
from appdirs import AppDirs
|
||||
|
||||
from unshackle.core.service_repo import is_repo_spec
|
||||
from unshackle.core.utils.collections import ci_get
|
||||
|
||||
|
||||
@@ -68,7 +69,8 @@ class Config:
|
||||
# these must not be modified by the user
|
||||
continue
|
||||
if name == "services" and isinstance(path, list):
|
||||
setattr(self.directories, name, [Path(p).expanduser() for p in path])
|
||||
# repo specs (git URLs / owner-repo) stay raw strings; resolved lazily in services.py
|
||||
setattr(self.directories, name, [p if is_repo_spec(p) else Path(p).expanduser() for p in path])
|
||||
else:
|
||||
setattr(self.directories, name, Path(path).expanduser())
|
||||
|
||||
@@ -111,6 +113,8 @@ class Config:
|
||||
self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or ""
|
||||
self.update_checks: bool = kwargs.get("update_checks", True)
|
||||
self.update_check_interval: int = kwargs.get("update_check_interval", 24)
|
||||
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
||||
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
||||
|
||||
self.language_tags: dict = kwargs.get("language_tags") or {}
|
||||
self.output_template: dict = kwargs.get("output_template") or {}
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import subprocess # nosec B404 - git is resolved via binaries.find, args are composed locally
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from unshackle.core import binaries
|
||||
|
||||
log = logging.getLogger("services")
|
||||
|
||||
|
||||
class DirtyServiceRepo(Exception):
|
||||
"""A clone has uncommitted or unpushed local changes; refresh is refused to protect them."""
|
||||
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
super().__init__(str(path))
|
||||
|
||||
DEFAULT_TTL = 24 * 60 * 60 # refresh at most once a day (mirrors UpdateChecker)
|
||||
STAMP_SUFFIX = ".fetch" # last-fetch unix time, kept beside (not inside) the clone
|
||||
|
||||
|
||||
def stamp_for(dest: Path) -> Path:
|
||||
"""Fetch-timestamp file path, kept OUTSIDE the clone so it never shows up in `git status`."""
|
||||
return dest.parent / f"{dest.name}{STAMP_SUFFIX}"
|
||||
|
||||
# owner/repo shorthand: exactly two path-like segments, no scheme
|
||||
SHORTHAND_RE = re.compile(r"^[\w.-]+/[\w.-]+$")
|
||||
|
||||
|
||||
def is_repo_spec(value: object) -> bool:
|
||||
"""True if value is a git repo spec (URL or owner/repo shorthand), not a local path."""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
if value.startswith(("http://", "https://", "ssh://", "git@")) or value.endswith(".git"):
|
||||
return True
|
||||
# owner/repo shorthand, but only if there's no local dir by that name (paths win)
|
||||
base = value.split("@", 1)[0]
|
||||
return bool(SHORTHAND_RE.match(base)) and not Path(value).expanduser().exists()
|
||||
|
||||
|
||||
def parse_spec(spec: str) -> tuple[str, Optional[str], str]:
|
||||
"""Return (clone_url, branch, dest_name). `@branch` is honored for http/shorthand, not ssh."""
|
||||
is_ssh = spec.startswith(("git@", "ssh://"))
|
||||
branch: Optional[str] = None
|
||||
url = spec
|
||||
if not is_ssh and "@" in spec:
|
||||
url, branch = spec.rsplit("@", 1)
|
||||
if SHORTHAND_RE.match(url) and not url.startswith(("http", "ssh", "git@")):
|
||||
url = f"https://github.com/{url}"
|
||||
return url, branch, _slug(url)
|
||||
|
||||
|
||||
def _slug(url: str) -> str:
|
||||
"""Filesystem-safe clone dir name from a repo URL, host included so same owner/repo on
|
||||
different hosts don't collide, e.g. github.com__owner__repo."""
|
||||
cleaned = re.sub(r"^[a-z]+://", "", url).split("@")[-1].replace(":", "/")
|
||||
cleaned = re.sub(r"\.git$", "", cleaned)
|
||||
return "__".join(p for p in cleaned.split("/") if p)
|
||||
|
||||
|
||||
def repos_base() -> Path:
|
||||
"""Where clones live: <first configured local services dir>/_repos (else the default)."""
|
||||
from unshackle.core.config import Config, config # lazy: avoid config <-> service_repo import cycle
|
||||
|
||||
entries = config.directories.services
|
||||
if not isinstance(entries, list):
|
||||
entries = [entries]
|
||||
for entry in entries:
|
||||
if not (isinstance(entry, str) and is_repo_spec(entry)):
|
||||
return Path(entry).expanduser() / "_repos"
|
||||
return Config._Directories.namespace_dir / "services" / "_repos"
|
||||
|
||||
|
||||
def resolve_service_repo(spec: str, *, ttl: int = DEFAULT_TTL, force: bool = False) -> Optional[Path]:
|
||||
"""Resolve a repo spec to its local clone dir.
|
||||
|
||||
First use clones it. ``force=True`` (manual ``refresh-services``) hard-resets the clone to
|
||||
upstream, discarding local changes. Otherwise a TTL-stale clone is fast-forwarded, but a dirty
|
||||
clone raises ``DirtyServiceRepo`` so the automated dl/search path can warn and exit instead of
|
||||
clobbering in-progress edits.
|
||||
"""
|
||||
url, branch, dest_name = parse_spec(spec)
|
||||
dest = repos_base() / dest_name
|
||||
stamp = stamp_for(dest)
|
||||
|
||||
if not dest.exists():
|
||||
return _clone(url, branch, dest, stamp)
|
||||
if force:
|
||||
_force_sync(dest, stamp)
|
||||
elif _is_stale(stamp, ttl):
|
||||
_pull(dest, stamp)
|
||||
return dest # ponytail: git IS the cache layer — no bespoke fetch/integrity code
|
||||
|
||||
|
||||
def _clone(url: str, branch: Optional[str], dest: Path, stamp: Path) -> Optional[Path]:
|
||||
if not binaries.Git:
|
||||
log.error("git not found on PATH — cannot fetch service repo %s", url)
|
||||
return None
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
args = [str(binaries.Git), "clone", "--depth", "1"]
|
||||
if branch:
|
||||
args += ["-b", branch]
|
||||
args += [url, str(dest)]
|
||||
try:
|
||||
subprocess.run(args, check=True, capture_output=True) # nosec B603
|
||||
except subprocess.CalledProcessError as e:
|
||||
log.error("failed to clone service repo %s: %s", url, _stderr(e))
|
||||
shutil.rmtree(dest, ignore_errors=True) # don't leave a half-cloned dir that sticks on retries
|
||||
return None
|
||||
_write_stamp(stamp)
|
||||
log.info("cloned service repo %s", url)
|
||||
return dest
|
||||
|
||||
|
||||
def _pull(dest: Path, stamp: Path) -> None:
|
||||
if not binaries.Git:
|
||||
_write_stamp(stamp)
|
||||
return
|
||||
# never clobber local edits — refuse to refresh a dirty clone and let the caller exit cleanly
|
||||
if _is_dirty(dest):
|
||||
raise DirtyServiceRepo(dest)
|
||||
try:
|
||||
subprocess.run(
|
||||
[str(binaries.Git), "-C", str(dest), "pull", "--ff-only"], check=True, capture_output=True
|
||||
) # nosec B603
|
||||
except subprocess.CalledProcessError as e:
|
||||
log.warning("could not update service repo at %s, using existing copy: %s", dest, _stderr(e))
|
||||
_write_stamp(stamp) # stamp regardless so a flaky remote can't trigger a fetch every run (force overrides)
|
||||
|
||||
|
||||
def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]:
|
||||
"""Force-sync a repo to upstream and return (clone dir, git-style per-service change lines).
|
||||
|
||||
Used by the manual ``refresh-services`` command. Change lines look like ``+ TAG (added)`` /
|
||||
``~ TAG (modified)`` / ``- TAG (removed)`` plus a shortstat; empty list means already up to date.
|
||||
"""
|
||||
url, branch, dest_name = parse_spec(spec)
|
||||
dest = repos_base() / dest_name
|
||||
stamp = stamp_for(dest)
|
||||
if not dest.exists():
|
||||
return _clone(url, branch, dest, stamp), ["cloned (new)"]
|
||||
if not binaries.Git:
|
||||
return dest, []
|
||||
old = _head(dest)
|
||||
discarded = _local_dirty_services(dest) # uncommitted edits the force reset is about to wipe
|
||||
_force_sync(dest, stamp)
|
||||
lines = [f"! {tag} (local changes discarded)" for tag in discarded]
|
||||
lines += _changed_services(dest, old, _head(dest)) # upstream changes pulled in
|
||||
return dest, lines
|
||||
|
||||
|
||||
def _head(dest: Path) -> Optional[str]:
|
||||
r = subprocess.run([str(binaries.Git), "-C", str(dest), "rev-parse", "HEAD"], capture_output=True) # nosec B603
|
||||
return r.stdout.decode(errors="ignore").strip() if r.returncode == 0 else None
|
||||
|
||||
|
||||
def _local_dirty_services(dest: Path) -> list[str]:
|
||||
"""Top-level service dirs with uncommitted edits to tracked files (lost on a force reset)."""
|
||||
r = subprocess.run([str(binaries.Git), "-C", str(dest), "diff", "--name-only", "HEAD"], capture_output=True) # nosec B603
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
return sorted({line.split("/")[0] for line in r.stdout.decode(errors="ignore").splitlines() if line})
|
||||
|
||||
|
||||
def _changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> list[str]:
|
||||
"""Summarise a HEAD old->new diff grouped by top-level service dir, plus a line shortstat."""
|
||||
if not old or not new or old == new:
|
||||
return []
|
||||
git = str(binaries.Git)
|
||||
name_status = subprocess.run(
|
||||
[git, "-C", str(dest), "diff", "--name-status", f"{old}..{new}"], capture_output=True
|
||||
) # nosec B603
|
||||
by_dir: dict[str, set[str]] = {}
|
||||
if name_status.returncode == 0:
|
||||
for raw in name_status.stdout.decode(errors="ignore").splitlines():
|
||||
parts = raw.split("\t")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
by_dir.setdefault(parts[-1].split("/")[0], set()).add(parts[0][0]) # parts[-1]=new path (handles renames)
|
||||
sym = {"added": "+", "modified": "~", "removed": "-"}
|
||||
lines = []
|
||||
for top in sorted(by_dir):
|
||||
codes = by_dir[top]
|
||||
kind = "added" if codes == {"A"} else "removed" if codes == {"D"} else "modified"
|
||||
lines.append(f"{sym[kind]} {top} ({kind})")
|
||||
short = subprocess.run([git, "-C", str(dest), "diff", "--shortstat", f"{old}..{new}"], capture_output=True) # nosec B603
|
||||
if short.returncode == 0 and short.stdout.strip():
|
||||
lines.append(short.stdout.decode(errors="ignore").strip())
|
||||
return lines
|
||||
|
||||
|
||||
def _force_sync(dest: Path, stamp: Path) -> None:
|
||||
"""Hard-reset the clone to its upstream, discarding local changes (manual refresh only)."""
|
||||
if not binaries.Git:
|
||||
_write_stamp(stamp)
|
||||
return
|
||||
git = str(binaries.Git)
|
||||
try:
|
||||
subprocess.run([git, "-C", str(dest), "fetch", "--depth", "1", "origin"], check=True, capture_output=True) # nosec B603
|
||||
subprocess.run([git, "-C", str(dest), "reset", "--hard", "@{u}"], check=True, capture_output=True) # nosec B603
|
||||
except subprocess.CalledProcessError as e:
|
||||
log.warning("could not force-refresh service repo at %s, using existing copy: %s", dest, _stderr(e))
|
||||
_write_stamp(stamp)
|
||||
|
||||
|
||||
def _is_dirty(dest: Path) -> bool:
|
||||
"""True if the clone has uncommitted edits to tracked files or local commits not on its upstream.
|
||||
|
||||
Untracked files are intentionally ignored: a `git pull --ff-only` only touches tracked files, so
|
||||
untracked content (including `__pycache__` that importing the services creates) is never at risk
|
||||
and must not falsely flag the clone as dirty.
|
||||
"""
|
||||
git = str(binaries.Git)
|
||||
# modifications to tracked files only (--untracked-files=no skips __pycache__ etc.)
|
||||
status = subprocess.run(
|
||||
[git, "-C", str(dest), "status", "--porcelain", "--untracked-files=no"], capture_output=True
|
||||
) # nosec B603
|
||||
if status.returncode == 0 and status.stdout.strip():
|
||||
return True
|
||||
# local commits ahead of upstream (committed but not pushed); no upstream → treat as clean
|
||||
ahead = subprocess.run(
|
||||
[git, "-C", str(dest), "rev-list", "--count", "@{u}..HEAD"], capture_output=True
|
||||
) # nosec B603
|
||||
if ahead.returncode == 0:
|
||||
try:
|
||||
return int(ahead.stdout.strip() or b"0") > 0
|
||||
except ValueError:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _is_stale(stamp: Path, ttl: int) -> bool:
|
||||
try:
|
||||
return (time.time() - float(stamp.read_text().strip())) >= ttl
|
||||
except (OSError, ValueError):
|
||||
return True
|
||||
|
||||
|
||||
def _write_stamp(stamp: Path) -> None:
|
||||
try:
|
||||
stamp.parent.mkdir(parents=True, exist_ok=True)
|
||||
stamp.write_text(str(time.time()))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _stderr(e: subprocess.CalledProcessError) -> str:
|
||||
return (e.stderr or b"").decode(errors="ignore").strip()
|
||||
@@ -8,18 +8,43 @@ import click
|
||||
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.service import Service
|
||||
from unshackle.core.service_repo import DirtyServiceRepo, is_repo_spec, resolve_service_repo
|
||||
from unshackle.core.utilities import import_module_by_path
|
||||
from unshackle.core.utils.redact import redact_path
|
||||
|
||||
log = logging.getLogger("services")
|
||||
|
||||
_service_dirs = config.directories.services
|
||||
if not isinstance(_service_dirs, list):
|
||||
_service_dirs = [_service_dirs]
|
||||
_raw = config.directories.services
|
||||
if not isinstance(_raw, list):
|
||||
_raw = [_raw]
|
||||
# resolve in config order — priority IS list order: the first source to define a tag is the source,
|
||||
# later sources (local or repo) with the same tag are shadowed. List local last to make it a fallback.
|
||||
_service_dirs: list[Path] = []
|
||||
DIRTY_REPOS: list[str] = []
|
||||
for _entry in _raw:
|
||||
if isinstance(_entry, str) and is_repo_spec(_entry):
|
||||
try:
|
||||
_resolved = resolve_service_repo(_entry)
|
||||
except DirtyServiceRepo as e:
|
||||
# local edits in the clone — record it and let check_load_errors() exit cleanly
|
||||
DIRTY_REPOS.append(redact_path(str(e.path)))
|
||||
_resolved = None
|
||||
if _resolved:
|
||||
_service_dirs.append(_resolved)
|
||||
else:
|
||||
_service_dirs.append(Path(_entry))
|
||||
|
||||
_SERVICES = sorted(
|
||||
(path for service_dir in _service_dirs for path in service_dir.glob("*/__init__.py")),
|
||||
key=lambda x: x.parent.stem,
|
||||
)
|
||||
# dedupe by tag honoring list order (first wins); track shadows for the load summary
|
||||
_seen: dict[str, Path] = {}
|
||||
SHADOWED: list[str] = []
|
||||
for _dir in _service_dirs:
|
||||
for _path in _dir.glob("*/__init__.py"):
|
||||
tag = _path.parent.stem
|
||||
if tag in _seen:
|
||||
SHADOWED.append(f"{tag}: using {redact_path(str(_seen[tag]))}, ignoring duplicate {redact_path(str(_path))}")
|
||||
else:
|
||||
_seen[tag] = _path
|
||||
_SERVICES = sorted(_seen.values(), key=lambda x: x.parent.stem)
|
||||
|
||||
|
||||
def load_service(path: Path) -> object:
|
||||
@@ -64,13 +89,33 @@ _MODULES, LOAD_ERRORS = load_services(_SERVICES)
|
||||
_ALIASES = {tag: getattr(module, "ALIASES", ()) for tag, module in _MODULES.items()}
|
||||
|
||||
|
||||
_SUMMARY_LOGGED = False
|
||||
|
||||
|
||||
def check_load_errors() -> None:
|
||||
"""Raise a single clean error if any Service failed to load.
|
||||
"""Log a one-line load summary (once) and raise a single clean error on failures.
|
||||
|
||||
Called when services are actually needed (listing/resolving) so the message
|
||||
is rendered once by Click, without a traceback and without cascading through
|
||||
every command that imports this module.
|
||||
every command that imports this module. Duplicate services (a tag also served by an
|
||||
earlier, higher-priority source, which is ignored) are summarised here; the full
|
||||
list — naming the path used and the duplicate ignored — is debug-only.
|
||||
"""
|
||||
if DIRTY_REPOS:
|
||||
joined = "\n".join(f" - {p}" for p in DIRTY_REPOS)
|
||||
raise click.ClickException(
|
||||
"Service repo has local changes — refusing to refresh so your edits are not lost.\n"
|
||||
"Commit and push them to the upstream repo (or revert them), then retry:\n" + joined
|
||||
)
|
||||
global _SUMMARY_LOGGED
|
||||
if not _SUMMARY_LOGGED:
|
||||
_SUMMARY_LOGGED = True
|
||||
summary = f"Loaded {len(_MODULES)} services"
|
||||
if SHADOWED:
|
||||
summary += f" ({len(SHADOWED)} duplicate(s) ignored)"
|
||||
log.info(summary)
|
||||
for shadow in SHADOWED:
|
||||
log.debug("%s", shadow)
|
||||
if LOAD_ERRORS:
|
||||
joined = "\n".join(f" - {err}" for err in LOAD_ERRORS)
|
||||
raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} service(s):\n{joined}")
|
||||
|
||||
@@ -77,13 +77,23 @@ def _path_bases() -> list[tuple[str, str]]:
|
||||
_PATH_BASES = _path_bases()
|
||||
|
||||
|
||||
def _path_redaction_enabled() -> bool:
|
||||
"""Path-prefix redaction is on unless ``config.redact_paths`` is explicitly False."""
|
||||
try:
|
||||
from unshackle.core.config import config # lazy: config doesn't import redact, so no cycle
|
||||
|
||||
return bool(getattr(config, "redact_paths", True))
|
||||
except (ImportError, AttributeError):
|
||||
return True
|
||||
|
||||
|
||||
def redact_path(text: Optional[str]) -> Optional[str]:
|
||||
"""Replace local base-directory prefixes (install root, venv, home) in ``text`` with tokens.
|
||||
|
||||
Idempotent and cheap; only touches strings that actually contain a known base dir, so URLs
|
||||
and relative paths pass through unchanged.
|
||||
and relative paths pass through unchanged. Disabled by ``config.redact_paths: false``.
|
||||
"""
|
||||
if not isinstance(text, str) or not text:
|
||||
if not isinstance(text, str) or not text or not _path_redaction_enabled():
|
||||
return text
|
||||
for base, token in _PATH_BASES:
|
||||
if base in text:
|
||||
|
||||
Reference in New Issue
Block a user