diff --git a/docs/DEBUG_LOGGING.md b/docs/DEBUG_LOGGING.md index b8b62796..34a52283 100644 --- a/docs/DEBUG_LOGGING.md +++ b/docs/DEBUG_LOGGING.md @@ -52,6 +52,9 @@ Route binary calls through the helpers in `unshackle.core.utils.subprocess`: dirs: install root → ``, venv → ``, home → `~`). `key` fields are also redacted unless `config.debug_keys`. Pass user data via `context=`/`request=`/kwargs so it is sanitized. Net effect: a shared JSONL leaks no account URLs, machine paths, or usernames. + Set `redact_paths: false` in `unshackle.yaml` to show full absolute paths instead of ``/ + ``/`~` tokens (config-only, no CLI flag). Affects `redact_path` everywhere it is used — + the debug log and the service load-summary detail. Secret/URL redaction is unaffected. - **Service calls are intentionally not logged** (no per-request POST/GET to services). Manifest parsing (`manifest_*_parse`) is the seam for request-level visibility. diff --git a/docs/OUTPUT_CONFIG.md b/docs/OUTPUT_CONFIG.md index 5ac343ff..b05ab22c 100644 --- a/docs/OUTPUT_CONFIG.md +++ b/docs/OUTPUT_CONFIG.md @@ -343,6 +343,7 @@ The following directories are available and may be overridden, Notes: - `services` accepts either a single directory or a list of directories to search for service modules. + Entries may also be **git repo specs** to load service packs hosted in a repo (see below). For example, @@ -354,4 +355,64 @@ directories: There are directories not listed that cannot be modified as they are crucial to the operation of unshackle. +### Loading services from a git repo + +A `services` entry may be a git repo instead of a local path, letting you host service packs on +GitHub or any git host (GitLab, Gitea, self-hosted). Local paths and repo specs can be mixed: + +```yaml +directories: + services: + - https://github.com/you/your-services # https repo (highest priority — listed first) + - git@gitlab.com:you/your-services.git # ssh repo (private, via your git auth) + - you/your-services@main # owner/repo shorthand + optional @branch + - ~/my-local-services # local path (fallback — listed last) +``` + +How it works: + +- On first use the repo is cloned (shallow) to `/_repos//` — the first + local `services` entry, or the bundled `unshackle/services` if you configured none. **Nothing is + written to the `cache` directory.** +- After that, unshackle does **not** hit the network on every run. It re-pulls at most once every 24h, + or immediately when you run `unshackle util refresh-services`. +- Requires `git` on your PATH. Private repos use your existing git credential helper — unshackle + stores no tokens. Git use is **read-only on the remote** — only `clone`, `fetch`, `pull`, and a + local `reset` are run; nothing is ever pushed. +- **Local edits and refresh.** You can edit a clone under `_repos/` directly. The two refresh paths + treat your edits differently: + - *Automatic* (the 24h TTL during a normal `dl`/`search` run): if the clone has uncommitted + changes to tracked files or unpushed local commits, unshackle **refuses to refresh and exits**, + naming the clone, so a background pull never clobbers work in progress. Commit and push it + upstream (or revert), then it refreshes normally. + - *Manual* (`unshackle util refresh-services`): an explicit "get upstream's latest" — it + **hard-resets the clone to upstream, discarding local changes**. Run it only when you want to + throw away local edits. + - Untracked files (new service folders, `__pycache__`) never block the automatic path — a + fast-forward pull doesn't touch them. +- The repo's **top level** must contain `/__init__.py` service dirs (same layout as + `unshackle/services/`). +- **Priority is list order.** The first source to define a tag is the one that loads; if a later + source (repo or local) has the same tag, that copy is treated as a duplicate and ignored. So list + the sources you trust most first — e.g. repos first and local last to make local a fallback, or + local first to let your local tweaks override a repo. +- **What you see on load.** A one-line summary is logged each run, e.g.: + + ``` + Loaded 103 services (36 duplicate(s) ignored) + ``` + + The full per-duplicate detail is logged only at debug verbosity (`unshackle -d ...`), one line per + duplicate, naming the path that loaded and the path that was ignored: + + ``` + EXAMPLE: using /services/EXAMPLE/__init__.py, ignoring duplicate /services/_repos/your-repo/EXAMPLE/__init__.py + ``` + + Paths are shortened to ``/``/`~` tokens; set `redact_paths: false` to show full + absolute paths (see [Debug Logging](DEBUG_LOGGING.md)). + +If `` is inside the installed package, a reinstall may remove the clones; they are +simply re-cloned on next use. On read-only installs, point `services` at a writable path. + --- diff --git a/tests/core/test_debug_logging.py b/tests/core/test_debug_logging.py index 2e68a381..b52bfa5f 100644 --- a/tests/core/test_debug_logging.py +++ b/tests/core/test_debug_logging.py @@ -261,6 +261,19 @@ def test_redact_path_leaves_relative_paths_untouched(): assert redact_path("temp/Audio.mp4") == "temp/Audio.mp4" +def test_redact_path_disabled_by_config(monkeypatch): + # config.redact_paths: false shows full paths verbatim + from unshackle.core.config import config + from unshackle.core.utils import redact as redact_mod + + root = str(Path(redact_mod.__file__).resolve().parents[3]) + full = f"{root}/temp/Audio.mp4" + monkeypatch.setattr(config, "redact_paths", False, raising=False) + assert redact_path(full) == full + monkeypatch.setattr(config, "redact_paths", True, raising=False) + assert redact_path(full) == "/temp/Audio.mp4" + + # ---------- redact_all + output_dir integration ---------- diff --git a/tests/core/test_service_repo.py b/tests/core/test_service_repo.py new file mode 100644 index 00000000..17b12782 --- /dev/null +++ b/tests/core/test_service_repo.py @@ -0,0 +1,272 @@ +"""Unit tests for remote git service-repo resolution. No network, no real git.""" + +from __future__ import annotations + +import subprocess +import time +from pathlib import Path + +import pytest + +from unshackle.core import service_repo + + +@pytest.mark.parametrize( + "value, expected", + [ + ("https://github.com/owner/repo", True), + ("http://example.com/me/svc", True), + ("git@gitlab.com:me/my-services.git", True), + ("ssh://git@host/me/svc", True), + ("me/svc.git", True), + ("owner/repo", True), + ("owner/repo@main", True), + ("/abs/local/path", False), + ("~/my-local-services", False), + (123, False), + ], +) +def test_is_repo_spec(value, expected): + assert service_repo.is_repo_spec(value) is expected + + +def test_is_repo_spec_existing_path_wins(tmp_path): + # an owner/repo-looking value that exists locally is a path, not a repo + local = tmp_path / "owner" / "repo" + local.mkdir(parents=True) + assert service_repo.is_repo_spec(str(local)) is False + + +@pytest.mark.parametrize( + "spec, url, branch, dest", + [ + ("owner/repo@main", "https://github.com/owner/repo", "main", "github.com__owner__repo"), + ("owner/repo", "https://github.com/owner/repo", None, "github.com__owner__repo"), + ("git@gitlab.com:me/my-services.git", "git@gitlab.com:me/my-services.git", None, "gitlab.com__me__my-services"), + ("https://github.com/a/b", "https://github.com/a/b", None, "github.com__a__b"), + ], +) +def test_parse_spec(spec, url, branch, dest): + assert service_repo.parse_spec(spec) == (url, branch, dest) + + +def test_slug_distinguishes_hosts(): + # same owner/repo on different hosts must not share a clone dir + a = service_repo.parse_spec("https://github.com/owner/repo")[2] + b = service_repo.parse_spec("https://gitlab.com/owner/repo")[2] + assert a != b + + +def test_repos_base_uses_configured_local_dir(monkeypatch, tmp_path): + from unshackle.core.config import config + + monkeypatch.setattr(config.directories, "services", ["owner/repo", str(tmp_path / "svc")], raising=False) + assert service_repo.repos_base() == (tmp_path / "svc" / "_repos") + + +def test_is_stale(tmp_path): + stamp = tmp_path / "stamp" + assert service_repo._is_stale(stamp, ttl=10) is True # missing → stale + stamp.write_text(str(time.time())) + assert service_repo._is_stale(stamp, ttl=10) is False # fresh + stamp.write_text(str(time.time() - 100)) + assert service_repo._is_stale(stamp, ttl=10) is True # old + + +def test_clone_invokes_git(monkeypatch, tmp_path): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path) + + calls = [] + + def fake_run(args, **kw): + calls.append(args) + (tmp_path / "github.com__owner__repo").mkdir(parents=True, exist_ok=True) # simulate clone output + return subprocess.CompletedProcess(args, 0, b"", b"") + + monkeypatch.setattr(service_repo.subprocess, "run", fake_run) + + dest = service_repo.resolve_service_repo("owner/repo@main") + assert dest == tmp_path / "github.com__owner__repo" + assert calls[0][:4] == ["git", "clone", "--depth", "1"] + assert "-b" in calls[0] and "main" in calls[0] + assert service_repo.stamp_for(dest).exists() # stamp written (beside the clone, not inside) + + +def test_clone_missing_git_returns_none(monkeypatch, tmp_path): + monkeypatch.setattr(service_repo.binaries, "Git", None) + monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path) + assert service_repo.resolve_service_repo("a/b") is None + + +def test_fresh_clone_skips_pull(monkeypatch, tmp_path): + dest = tmp_path / "github.com__a__b" + dest.mkdir(parents=True) + service_repo.stamp_for(dest).write_text(str(time.time())) + monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path) + monkeypatch.setattr( + service_repo.subprocess, "run", lambda *a, **k: pytest.fail("git should not run for a fresh clone") + ) + assert service_repo.resolve_service_repo("a/b") == dest + + +def _git_stub(porcelain=b"", ahead=b"0"): + """Fake subprocess.run for git: dirty checks return given output, pull succeeds.""" + + def run(args, **kw): + sub = args[3] if len(args) > 3 else "" + if sub == "status": + return subprocess.CompletedProcess(args, 0, porcelain, b"") + if sub == "rev-list": + return subprocess.CompletedProcess(args, 0, ahead, b"") + return subprocess.CompletedProcess(args, 0, b"", b"") # pull + + return run + + +def test_is_dirty_uncommitted(monkeypatch): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + monkeypatch.setattr(service_repo.subprocess, "run", _git_stub(porcelain=b" M EXAMPLE/__init__.py\n")) + assert service_repo._is_dirty(Path("/x")) is True + + +def test_is_dirty_commits_ahead(monkeypatch): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + monkeypatch.setattr(service_repo.subprocess, "run", _git_stub(porcelain=b"", ahead=b"2")) + assert service_repo._is_dirty(Path("/x")) is True + + +def test_is_dirty_clean(monkeypatch): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + monkeypatch.setattr(service_repo.subprocess, "run", _git_stub(porcelain=b"", ahead=b"0")) + assert service_repo._is_dirty(Path("/x")) is False + + +def test_is_dirty_ignores_untracked_files(monkeypatch): + # status must ask git to skip untracked files so importing services (__pycache__) isn't "dirty" + seen = [] + monkeypatch.setattr(service_repo.binaries, "Git", "git") + + def run(args, **kw): + seen.append(args) + out = b"0" if args[3] == "rev-list" else b"" + return subprocess.CompletedProcess(args, 0, out, b"") + + monkeypatch.setattr(service_repo.subprocess, "run", run) + service_repo._is_dirty(Path("/x")) + status_args = next(a for a in seen if a[3] == "status") + assert "--untracked-files=no" in status_args + + +def test_auto_refresh_refuses_dirty_clone(monkeypatch, tmp_path): + """Auto path (force=False): a stale clone with local edits raises instead of clobbering.""" + dest = tmp_path / "github.com__a__b" + dest.mkdir(parents=True) + service_repo.stamp_for(dest).write_text("0") # stale → refresh due + monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path) + monkeypatch.setattr(service_repo.binaries, "Git", "git") + monkeypatch.setattr(service_repo.subprocess, "run", _git_stub(porcelain=b" M f\n")) + with pytest.raises(service_repo.DirtyServiceRepo) as ei: + service_repo.resolve_service_repo("a/b") + assert ei.value.path == dest + + +def test_manual_refresh_force_overwrites(monkeypatch, tmp_path): + """Manual path (force=True): hard-reset to upstream, never raises on a dirty clone.""" + dest = tmp_path / "github.com__a__b" + dest.mkdir(parents=True) + monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path) + monkeypatch.setattr(service_repo.binaries, "Git", "git") + calls = [] + monkeypatch.setattr( + service_repo.subprocess, + "run", + lambda args, **kw: (calls.append(args[3]), subprocess.CompletedProcess(args, 0, b"", b""))[1], + ) + assert service_repo.resolve_service_repo("a/b", force=True) == dest + assert "fetch" in calls and "reset" in calls # force-sync ran + assert "status" not in calls # no dirty gate on the force path + + +def test_changed_services_groups_by_top_dir(monkeypatch): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + + def run(args, **kw): + if "--name-status" in args: + out = b"A\tNEW/__init__.py\nM\tOLD/__init__.py\nM\tOLD/config.yaml\nD\tGONE/__init__.py\n" + elif "--shortstat" in args: + out = b" 4 files changed, 10 insertions(+), 2 deletions(-)\n" + else: + out = b"" + return subprocess.CompletedProcess(args, 0, out, b"") + + monkeypatch.setattr(service_repo.subprocess, "run", run) + lines = service_repo._changed_services(Path("/x"), "aaa", "bbb") + assert "+ NEW (added)" in lines + assert "~ OLD (modified)" in lines + assert "- GONE (removed)" in lines + assert any("4 files changed" in line for line in lines) + + +def test_local_dirty_services_lists_edited_dirs(monkeypatch): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + monkeypatch.setattr( + service_repo.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a, 0, b"CR/__init__.py\nCR/config.yaml\nNF/__init__.py\n", b""), + ) + assert service_repo._local_dirty_services(Path("/x")) == ["CR", "NF"] + + +def test_refresh_reports_discarded_local_edits(monkeypatch, tmp_path): + """Force refresh with no upstream advance still reports the local edits it wiped.""" + dest = tmp_path / "github.com__a__b" + dest.mkdir(parents=True) + monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path) + monkeypatch.setattr(service_repo.binaries, "Git", "git") + + def run(args, **kw): + if "diff" in args and "--name-only" in args: + return subprocess.CompletedProcess(args, 0, b"CR/__init__.py\n", b"") # local edit present + if "rev-parse" in args: + return subprocess.CompletedProcess(args, 0, b"samehead\n", b"") # upstream did not advance + return subprocess.CompletedProcess(args, 0, b"", b"") # fetch/reset/diff + + monkeypatch.setattr(service_repo.subprocess, "run", run) + _, changes = service_repo.refresh_repo("a/b") + assert changes == ["! CR (local changes discarded)"] + + +def test_changed_services_up_to_date(monkeypatch): + monkeypatch.setattr(service_repo.binaries, "Git", "git") + # same HEAD before/after → no diff computed at all + monkeypatch.setattr( + service_repo.subprocess, "run", lambda *a, **k: pytest.fail("no git diff when HEAD unchanged") + ) + assert service_repo._changed_services(Path("/x"), "same", "same") == [] + + +@pytest.mark.parametrize("order", [["a", "b"], ["b", "a"]]) +def test_collision_first_source_wins(tmp_path, order): + """Discovery dedupes by tag honoring list order — the FIRST source to define a tag wins, + whether it's a local dir or a repo clone; later duplicates are shadowed.""" + dirs = {} + for name in ("a", "b"): + svc = tmp_path / name / "EXAMPLE" + svc.mkdir(parents=True) + (svc / "__init__.py").touch() + dirs[name] = tmp_path / name + + seen: dict = {} + shadowed: list = [] + for name in order: # services.py preserves config order + for path in dirs[name].glob("*/__init__.py"): + tag = path.parent.stem + if tag in seen: + shadowed.append((tag, path, seen[tag])) + else: + seen[tag] = path + + winner, loser = order[0], order[1] + assert seen["EXAMPLE"] == dirs[winner] / "EXAMPLE" / "__init__.py" + assert shadowed and shadowed[0][1] == dirs[loser] / "EXAMPLE" / "__init__.py" diff --git a/unshackle/commands/env.py b/unshackle/commands/env.py index dced7c0e..5cc9c8e7 100644 --- a/unshackle/commands/env.py +++ b/unshackle/commands/env.py @@ -96,6 +96,7 @@ def check() -> None: }, {"name": "Caddy", "binary": binaries.Caddy, "required": False, "desc": "Web server", "cat": "Network"}, {"name": "Docker", "binary": binaries.Docker, "required": False, "desc": "Gluetun VPN", "cat": "Network"}, + {"name": "git", "binary": binaries.Git, "required": False, "desc": "Service repos", "cat": "Network"}, ] # Track overall status diff --git a/unshackle/commands/util.py b/unshackle/commands/util.py index 612ac2bc..7a3c42c0 100644 --- a/unshackle/commands/util.py +++ b/unshackle/commands/util.py @@ -19,6 +19,34 @@ def util() -> None: """Various helper scripts and programs.""" +@util.command(name="refresh-services") +def refresh_services() -> None: + """Force a refresh (git pull) of all service repos configured in directories.services.""" + from unshackle.core.config import config + from unshackle.core.service_repo import is_repo_spec, refresh_repo + from unshackle.core.utils.redact import redact_path + + entries = config.directories.services + if not isinstance(entries, list): + entries = [entries] + specs = [e for e in entries if isinstance(e, str) and is_repo_spec(e)] + if not specs: + click.echo("No service repos configured in directories.services.") + return + # manual refresh force-overwrites local changes (hard reset to upstream), then reports the diff + for spec in specs: + dest, changes = refresh_repo(spec) + if not dest: + click.echo(f"Failed to update {spec} (see log).") + continue + if changes: + click.echo(f"Updated {spec} → {redact_path(str(dest))}") + for line in changes: + click.echo(f" {line}") + else: + click.echo(f"No changes {spec}") + + @util.command() @click.argument("path", type=Path) @click.argument("aspect", type=str) diff --git a/unshackle/core/binaries.py b/unshackle/core/binaries.py index d2245882..5ddf3a02 100644 --- a/unshackle/core/binaries.py +++ b/unshackle/core/binaries.py @@ -57,6 +57,7 @@ HDR10PlusTool = find("hdr10plus_tool", "HDR10Plus_tool") Mp4decrypt = find("mp4decrypt") Docker = find("docker") ML_Worker = find("ML-Worker") +Git = find("git") __all__ = ( @@ -76,5 +77,6 @@ __all__ = ( "Mp4decrypt", "Docker", "ML_Worker", + "Git", "find", ) diff --git a/unshackle/core/config.py b/unshackle/core/config.py index f847cbd6..47407014 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -8,6 +8,7 @@ from typing import Any, Optional import yaml from appdirs import AppDirs +from unshackle.core.service_repo import is_repo_spec from unshackle.core.utils.collections import ci_get @@ -68,7 +69,8 @@ class Config: # these must not be modified by the user continue if name == "services" and isinstance(path, list): - setattr(self.directories, name, [Path(p).expanduser() for p in path]) + # repo specs (git URLs / owner-repo) stay raw strings; resolved lazily in services.py + setattr(self.directories, name, [p if is_repo_spec(p) else Path(p).expanduser() for p in path]) else: setattr(self.directories, name, Path(path).expanduser()) @@ -111,6 +113,8 @@ class Config: self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or "" self.update_checks: bool = kwargs.get("update_checks", True) self.update_check_interval: int = kwargs.get("update_check_interval", 24) + # mask local base dirs (install root/venv/home) in logged paths; False shows full paths + self.redact_paths: bool = kwargs.get("redact_paths", True) self.language_tags: dict = kwargs.get("language_tags") or {} self.output_template: dict = kwargs.get("output_template") or {} diff --git a/unshackle/core/service_repo.py b/unshackle/core/service_repo.py new file mode 100644 index 00000000..dcd96d0f --- /dev/null +++ b/unshackle/core/service_repo.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import logging +import re +import shutil +import subprocess # nosec B404 - git is resolved via binaries.find, args are composed locally +import time +from pathlib import Path +from typing import Optional + +from unshackle.core import binaries + +log = logging.getLogger("services") + + +class DirtyServiceRepo(Exception): + """A clone has uncommitted or unpushed local changes; refresh is refused to protect them.""" + + def __init__(self, path: Path): + self.path = path + super().__init__(str(path)) + +DEFAULT_TTL = 24 * 60 * 60 # refresh at most once a day (mirrors UpdateChecker) +STAMP_SUFFIX = ".fetch" # last-fetch unix time, kept beside (not inside) the clone + + +def stamp_for(dest: Path) -> Path: + """Fetch-timestamp file path, kept OUTSIDE the clone so it never shows up in `git status`.""" + return dest.parent / f"{dest.name}{STAMP_SUFFIX}" + +# owner/repo shorthand: exactly two path-like segments, no scheme +SHORTHAND_RE = re.compile(r"^[\w.-]+/[\w.-]+$") + + +def is_repo_spec(value: object) -> bool: + """True if value is a git repo spec (URL or owner/repo shorthand), not a local path.""" + if not isinstance(value, str): + return False + if value.startswith(("http://", "https://", "ssh://", "git@")) or value.endswith(".git"): + return True + # owner/repo shorthand, but only if there's no local dir by that name (paths win) + base = value.split("@", 1)[0] + return bool(SHORTHAND_RE.match(base)) and not Path(value).expanduser().exists() + + +def parse_spec(spec: str) -> tuple[str, Optional[str], str]: + """Return (clone_url, branch, dest_name). `@branch` is honored for http/shorthand, not ssh.""" + is_ssh = spec.startswith(("git@", "ssh://")) + branch: Optional[str] = None + url = spec + if not is_ssh and "@" in spec: + url, branch = spec.rsplit("@", 1) + if SHORTHAND_RE.match(url) and not url.startswith(("http", "ssh", "git@")): + url = f"https://github.com/{url}" + return url, branch, _slug(url) + + +def _slug(url: str) -> str: + """Filesystem-safe clone dir name from a repo URL, host included so same owner/repo on + different hosts don't collide, e.g. github.com__owner__repo.""" + cleaned = re.sub(r"^[a-z]+://", "", url).split("@")[-1].replace(":", "/") + cleaned = re.sub(r"\.git$", "", cleaned) + return "__".join(p for p in cleaned.split("/") if p) + + +def repos_base() -> Path: + """Where clones live: /_repos (else the default).""" + from unshackle.core.config import Config, config # lazy: avoid config <-> service_repo import cycle + + entries = config.directories.services + if not isinstance(entries, list): + entries = [entries] + for entry in entries: + if not (isinstance(entry, str) and is_repo_spec(entry)): + return Path(entry).expanduser() / "_repos" + return Config._Directories.namespace_dir / "services" / "_repos" + + +def resolve_service_repo(spec: str, *, ttl: int = DEFAULT_TTL, force: bool = False) -> Optional[Path]: + """Resolve a repo spec to its local clone dir. + + First use clones it. ``force=True`` (manual ``refresh-services``) hard-resets the clone to + upstream, discarding local changes. Otherwise a TTL-stale clone is fast-forwarded, but a dirty + clone raises ``DirtyServiceRepo`` so the automated dl/search path can warn and exit instead of + clobbering in-progress edits. + """ + url, branch, dest_name = parse_spec(spec) + dest = repos_base() / dest_name + stamp = stamp_for(dest) + + if not dest.exists(): + return _clone(url, branch, dest, stamp) + if force: + _force_sync(dest, stamp) + elif _is_stale(stamp, ttl): + _pull(dest, stamp) + return dest # ponytail: git IS the cache layer — no bespoke fetch/integrity code + + +def _clone(url: str, branch: Optional[str], dest: Path, stamp: Path) -> Optional[Path]: + if not binaries.Git: + log.error("git not found on PATH — cannot fetch service repo %s", url) + return None + dest.parent.mkdir(parents=True, exist_ok=True) + args = [str(binaries.Git), "clone", "--depth", "1"] + if branch: + args += ["-b", branch] + args += [url, str(dest)] + try: + subprocess.run(args, check=True, capture_output=True) # nosec B603 + except subprocess.CalledProcessError as e: + log.error("failed to clone service repo %s: %s", url, _stderr(e)) + shutil.rmtree(dest, ignore_errors=True) # don't leave a half-cloned dir that sticks on retries + return None + _write_stamp(stamp) + log.info("cloned service repo %s", url) + return dest + + +def _pull(dest: Path, stamp: Path) -> None: + if not binaries.Git: + _write_stamp(stamp) + return + # never clobber local edits — refuse to refresh a dirty clone and let the caller exit cleanly + if _is_dirty(dest): + raise DirtyServiceRepo(dest) + try: + subprocess.run( + [str(binaries.Git), "-C", str(dest), "pull", "--ff-only"], check=True, capture_output=True + ) # nosec B603 + except subprocess.CalledProcessError as e: + log.warning("could not update service repo at %s, using existing copy: %s", dest, _stderr(e)) + _write_stamp(stamp) # stamp regardless so a flaky remote can't trigger a fetch every run (force overrides) + + +def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]: + """Force-sync a repo to upstream and return (clone dir, git-style per-service change lines). + + Used by the manual ``refresh-services`` command. Change lines look like ``+ TAG (added)`` / + ``~ TAG (modified)`` / ``- TAG (removed)`` plus a shortstat; empty list means already up to date. + """ + url, branch, dest_name = parse_spec(spec) + dest = repos_base() / dest_name + stamp = stamp_for(dest) + if not dest.exists(): + return _clone(url, branch, dest, stamp), ["cloned (new)"] + if not binaries.Git: + return dest, [] + old = _head(dest) + discarded = _local_dirty_services(dest) # uncommitted edits the force reset is about to wipe + _force_sync(dest, stamp) + lines = [f"! {tag} (local changes discarded)" for tag in discarded] + lines += _changed_services(dest, old, _head(dest)) # upstream changes pulled in + return dest, lines + + +def _head(dest: Path) -> Optional[str]: + r = subprocess.run([str(binaries.Git), "-C", str(dest), "rev-parse", "HEAD"], capture_output=True) # nosec B603 + return r.stdout.decode(errors="ignore").strip() if r.returncode == 0 else None + + +def _local_dirty_services(dest: Path) -> list[str]: + """Top-level service dirs with uncommitted edits to tracked files (lost on a force reset).""" + r = subprocess.run([str(binaries.Git), "-C", str(dest), "diff", "--name-only", "HEAD"], capture_output=True) # nosec B603 + if r.returncode != 0: + return [] + return sorted({line.split("/")[0] for line in r.stdout.decode(errors="ignore").splitlines() if line}) + + +def _changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> list[str]: + """Summarise a HEAD old->new diff grouped by top-level service dir, plus a line shortstat.""" + if not old or not new or old == new: + return [] + git = str(binaries.Git) + name_status = subprocess.run( + [git, "-C", str(dest), "diff", "--name-status", f"{old}..{new}"], capture_output=True + ) # nosec B603 + by_dir: dict[str, set[str]] = {} + if name_status.returncode == 0: + for raw in name_status.stdout.decode(errors="ignore").splitlines(): + parts = raw.split("\t") + if len(parts) < 2: + continue + by_dir.setdefault(parts[-1].split("/")[0], set()).add(parts[0][0]) # parts[-1]=new path (handles renames) + sym = {"added": "+", "modified": "~", "removed": "-"} + lines = [] + for top in sorted(by_dir): + codes = by_dir[top] + kind = "added" if codes == {"A"} else "removed" if codes == {"D"} else "modified" + lines.append(f"{sym[kind]} {top} ({kind})") + short = subprocess.run([git, "-C", str(dest), "diff", "--shortstat", f"{old}..{new}"], capture_output=True) # nosec B603 + if short.returncode == 0 and short.stdout.strip(): + lines.append(short.stdout.decode(errors="ignore").strip()) + return lines + + +def _force_sync(dest: Path, stamp: Path) -> None: + """Hard-reset the clone to its upstream, discarding local changes (manual refresh only).""" + if not binaries.Git: + _write_stamp(stamp) + return + git = str(binaries.Git) + try: + subprocess.run([git, "-C", str(dest), "fetch", "--depth", "1", "origin"], check=True, capture_output=True) # nosec B603 + subprocess.run([git, "-C", str(dest), "reset", "--hard", "@{u}"], check=True, capture_output=True) # nosec B603 + except subprocess.CalledProcessError as e: + log.warning("could not force-refresh service repo at %s, using existing copy: %s", dest, _stderr(e)) + _write_stamp(stamp) + + +def _is_dirty(dest: Path) -> bool: + """True if the clone has uncommitted edits to tracked files or local commits not on its upstream. + + Untracked files are intentionally ignored: a `git pull --ff-only` only touches tracked files, so + untracked content (including `__pycache__` that importing the services creates) is never at risk + and must not falsely flag the clone as dirty. + """ + git = str(binaries.Git) + # modifications to tracked files only (--untracked-files=no skips __pycache__ etc.) + status = subprocess.run( + [git, "-C", str(dest), "status", "--porcelain", "--untracked-files=no"], capture_output=True + ) # nosec B603 + if status.returncode == 0 and status.stdout.strip(): + return True + # local commits ahead of upstream (committed but not pushed); no upstream → treat as clean + ahead = subprocess.run( + [git, "-C", str(dest), "rev-list", "--count", "@{u}..HEAD"], capture_output=True + ) # nosec B603 + if ahead.returncode == 0: + try: + return int(ahead.stdout.strip() or b"0") > 0 + except ValueError: + return False + return False + + +def _is_stale(stamp: Path, ttl: int) -> bool: + try: + return (time.time() - float(stamp.read_text().strip())) >= ttl + except (OSError, ValueError): + return True + + +def _write_stamp(stamp: Path) -> None: + try: + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text(str(time.time())) + except OSError: + pass + + +def _stderr(e: subprocess.CalledProcessError) -> str: + return (e.stderr or b"").decode(errors="ignore").strip() diff --git a/unshackle/core/services.py b/unshackle/core/services.py index 0f600232..b4dda122 100644 --- a/unshackle/core/services.py +++ b/unshackle/core/services.py @@ -8,18 +8,43 @@ import click from unshackle.core.config import config from unshackle.core.service import Service +from unshackle.core.service_repo import DirtyServiceRepo, is_repo_spec, resolve_service_repo from unshackle.core.utilities import import_module_by_path +from unshackle.core.utils.redact import redact_path log = logging.getLogger("services") -_service_dirs = config.directories.services -if not isinstance(_service_dirs, list): - _service_dirs = [_service_dirs] +_raw = config.directories.services +if not isinstance(_raw, list): + _raw = [_raw] +# resolve in config order — priority IS list order: the first source to define a tag is the source, +# later sources (local or repo) with the same tag are shadowed. List local last to make it a fallback. +_service_dirs: list[Path] = [] +DIRTY_REPOS: list[str] = [] +for _entry in _raw: + if isinstance(_entry, str) and is_repo_spec(_entry): + try: + _resolved = resolve_service_repo(_entry) + except DirtyServiceRepo as e: + # local edits in the clone — record it and let check_load_errors() exit cleanly + DIRTY_REPOS.append(redact_path(str(e.path))) + _resolved = None + if _resolved: + _service_dirs.append(_resolved) + else: + _service_dirs.append(Path(_entry)) -_SERVICES = sorted( - (path for service_dir in _service_dirs for path in service_dir.glob("*/__init__.py")), - key=lambda x: x.parent.stem, -) +# dedupe by tag honoring list order (first wins); track shadows for the load summary +_seen: dict[str, Path] = {} +SHADOWED: list[str] = [] +for _dir in _service_dirs: + for _path in _dir.glob("*/__init__.py"): + tag = _path.parent.stem + if tag in _seen: + SHADOWED.append(f"{tag}: using {redact_path(str(_seen[tag]))}, ignoring duplicate {redact_path(str(_path))}") + else: + _seen[tag] = _path +_SERVICES = sorted(_seen.values(), key=lambda x: x.parent.stem) def load_service(path: Path) -> object: @@ -64,13 +89,33 @@ _MODULES, LOAD_ERRORS = load_services(_SERVICES) _ALIASES = {tag: getattr(module, "ALIASES", ()) for tag, module in _MODULES.items()} +_SUMMARY_LOGGED = False + + def check_load_errors() -> None: - """Raise a single clean error if any Service failed to load. + """Log a one-line load summary (once) and raise a single clean error on failures. Called when services are actually needed (listing/resolving) so the message is rendered once by Click, without a traceback and without cascading through - every command that imports this module. + every command that imports this module. Duplicate services (a tag also served by an + earlier, higher-priority source, which is ignored) are summarised here; the full + list — naming the path used and the duplicate ignored — is debug-only. """ + if DIRTY_REPOS: + joined = "\n".join(f" - {p}" for p in DIRTY_REPOS) + raise click.ClickException( + "Service repo has local changes — refusing to refresh so your edits are not lost.\n" + "Commit and push them to the upstream repo (or revert them), then retry:\n" + joined + ) + global _SUMMARY_LOGGED + if not _SUMMARY_LOGGED: + _SUMMARY_LOGGED = True + summary = f"Loaded {len(_MODULES)} services" + if SHADOWED: + summary += f" ({len(SHADOWED)} duplicate(s) ignored)" + log.info(summary) + for shadow in SHADOWED: + log.debug("%s", shadow) if LOAD_ERRORS: joined = "\n".join(f" - {err}" for err in LOAD_ERRORS) raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} service(s):\n{joined}") diff --git a/unshackle/core/utils/redact.py b/unshackle/core/utils/redact.py index 28ac7fa4..e4c1f5b8 100644 --- a/unshackle/core/utils/redact.py +++ b/unshackle/core/utils/redact.py @@ -77,13 +77,23 @@ def _path_bases() -> list[tuple[str, str]]: _PATH_BASES = _path_bases() +def _path_redaction_enabled() -> bool: + """Path-prefix redaction is on unless ``config.redact_paths`` is explicitly False.""" + try: + from unshackle.core.config import config # lazy: config doesn't import redact, so no cycle + + return bool(getattr(config, "redact_paths", True)) + except (ImportError, AttributeError): + return True + + def redact_path(text: Optional[str]) -> Optional[str]: """Replace local base-directory prefixes (install root, venv, home) in ``text`` with tokens. Idempotent and cheap; only touches strings that actually contain a known base dir, so URLs - and relative paths pass through unchanged. + and relative paths pass through unchanged. Disabled by ``config.redact_paths: false``. """ - if not isinstance(text, str) or not text: + if not isinstance(text, str) or not text or not _path_redaction_enabled(): return text for base, token in _PATH_BASES: if base in text: