Files
unshackle/unshackle/core/service_repo.py
T
imSp4rky 4964869258 feat(services): add services_repo_force and compact refresh output
Force-reset repo clones on the automatic TTL refresh when enabled. Report refresh changes as one +/~/-/!TAG line per repo.
2026-08-26 22:15:39 -06:00

246 lines
10 KiB
Python

from __future__ import annotations
import logging
import re
import shutil
import subprocess # nosec B404 - git is resolved via binaries.find, args are composed locally
import time
from pathlib import Path
from typing import Optional
from unshackle.core import binaries
log = logging.getLogger("services")
class DirtyServiceRepo(Exception):
"""A clone has uncommitted or unpushed local changes. unshackle refuses to refresh it, to protect them."""
def __init__(self, path: Path):
self.path = path
super().__init__(str(path))
DEFAULT_TTL = 24 * 60 * 60 # refresh at most once a day (mirrors UpdateChecker)
STAMP_SUFFIX = ".fetch" # last-fetch unix time, kept beside (not inside) the clone
def stamp_for(dest: Path) -> Path:
"""Fetch-timestamp file path, kept OUTSIDE the clone so it never shows up in `git status`."""
return dest.parent / f"{dest.name}{STAMP_SUFFIX}"
SHORTHAND_RE = re.compile(r"^[\w.-]+/[\w.-]+$")
def is_repo_spec(value: object) -> bool:
"""True if value is a git repo spec (URL or owner/repo shorthand), not a local path."""
if not isinstance(value, str):
return False
if value.startswith(("http://", "https://", "ssh://", "git@")) or value.endswith(".git"):
return True
# owner/repo shorthand, but only if there's no local dir by that name (paths win)
base = value.split("@", 1)[0]
return bool(SHORTHAND_RE.match(base)) and not Path(value).expanduser().exists()
def parse_spec(spec: str) -> tuple[str, Optional[str], str]:
"""Return (clone_url, branch, dest_name). This function honours `@branch` for http/shorthand, not for ssh."""
is_ssh = spec.startswith(("git@", "ssh://"))
branch: Optional[str] = None
url = spec
if not is_ssh and "@" in spec:
url, branch = spec.rsplit("@", 1)
if SHORTHAND_RE.match(url) and not url.startswith(("http", "ssh", "git@")):
url = f"https://github.com/{url}"
return url, branch, slug(url)
def slug(url: str) -> str:
"""Filesystem-safe clone dir name from a repo URL, host included so same owner/repo on
different hosts do not collide, e.g. github.com__owner__repo."""
cleaned = re.sub(r"^[a-z]+://", "", url).split("@")[-1].replace(":", "/")
cleaned = re.sub(r"\.git$", "", cleaned)
return "__".join(p for p in cleaned.split("/") if p)
def repos_base() -> Path:
"""Where clones live: <first configured local services dir>/_repos (else the default)."""
from unshackle.core.config import Config, config # lazy: avoid config <-> service_repo import cycle
entries = config.directories.services
if not isinstance(entries, list):
entries = [entries]
for entry in entries:
if not (isinstance(entry, str) and is_repo_spec(entry)):
return Path(entry).expanduser() / "_repos"
return Config._Directories.namespace_dir / "services" / "_repos"
def resolve_service_repo(spec: str, *, ttl: int = DEFAULT_TTL, force: bool = False) -> Optional[Path]:
"""Find the local clone dir of a repo spec.
First use clones it. Only a TTL-stale clone touches the network. With ``force=True``
(``services_repo_force``) the refresh hard-resets the clone to upstream and discards local
changes. Without it the refresh fast-forwards, and a dirty clone raises ``DirtyServiceRepo``
so the automated dl/search path can warn and exit instead of clobbering in-progress edits.
Manual ``refresh-services`` uses ``refresh_repo`` and ignores the TTL.
"""
url, branch, dest_name = parse_spec(spec)
dest = repos_base() / dest_name
stamp = stamp_for(dest)
if not dest.exists():
return clone(url, branch, dest, stamp)
if is_stale(stamp, ttl):
force_sync(dest, stamp) if force else pull(dest, stamp)
return dest
def clone(url: str, branch: Optional[str], dest: Path, stamp: Path) -> Optional[Path]:
if not binaries.Git:
log.error("git not found on PATH; cannot fetch service repo %s", url)
return None
dest.parent.mkdir(parents=True, exist_ok=True)
args = [str(binaries.Git), "clone", "--depth", "1"]
if branch:
args += ["-b", branch]
args += [url, str(dest)]
try:
subprocess.run(args, check=True, capture_output=True) # nosec B603
except subprocess.CalledProcessError as e:
log.error("failed to clone service repo %s: %s", url, stderr_text(e))
shutil.rmtree(dest, ignore_errors=True) # don't leave a half-cloned dir that sticks on retries
return None
write_stamp(stamp)
log.info("cloned service repo %s", url)
return dest
def pull(dest: Path, stamp: Path) -> None:
if not binaries.Git:
write_stamp(stamp)
return
# never clobber local edits: refuse to refresh a dirty clone and let the caller exit cleanly
if is_dirty(dest):
raise DirtyServiceRepo(dest)
try:
subprocess.run([str(binaries.Git), "-C", str(dest), "pull", "--ff-only"], check=True, capture_output=True) # nosec B603
except subprocess.CalledProcessError as e:
log.warning("could not update service repo at %s, using existing copy: %s", dest, stderr_text(e))
write_stamp(stamp) # stamp regardless so a flaky remote can't trigger a fetch every run (force overrides)
def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]:
"""Force-sync a repo to upstream and return (clone dir, git-style per-service change lines).
Used by the manual ``refresh-services`` command. Each line is one service tag with a status
prefix: ``+TAG`` added, ``~TAG`` modified, ``-TAG`` removed, ``!TAG`` local changes discarded.
An empty list means the clone is already up to date.
"""
url, branch, dest_name = parse_spec(spec)
dest = repos_base() / dest_name
stamp = stamp_for(dest)
if not dest.exists():
return clone(url, branch, dest, stamp), ["cloned (new)"]
if not binaries.Git:
return dest, []
old = head(dest)
discarded = local_dirty_services(dest)
force_sync(dest, stamp)
lines = [f"!{tag}" for tag in discarded] # ! = local changes discarded
lines += changed_services(dest, old, head(dest))
return dest, lines
def head(dest: Path) -> Optional[str]:
r = subprocess.run([str(binaries.Git), "-C", str(dest), "rev-parse", "HEAD"], capture_output=True) # nosec B603
return r.stdout.decode(errors="ignore").strip() if r.returncode == 0 else None
def local_dirty_services(dest: Path) -> list[str]:
"""Top-level service dirs with uncommitted edits to tracked files (lost on a force reset)."""
r = subprocess.run([str(binaries.Git), "-C", str(dest), "diff", "--name-only", "HEAD"], capture_output=True) # nosec B603
if r.returncode != 0:
return []
return sorted({line.split("/")[0] for line in r.stdout.decode(errors="ignore").splitlines() if line})
def changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> list[str]:
"""Summarise a HEAD old->new diff as one ``+TAG``/``~TAG``/``-TAG`` token per top-level service directory."""
if not old or not new or old == new:
return []
git = str(binaries.Git)
name_status = subprocess.run([git, "-C", str(dest), "diff", "--name-status", f"{old}..{new}"], capture_output=True) # nosec B603
by_dir: dict[str, set[str]] = {}
if name_status.returncode == 0:
for raw in name_status.stdout.decode(errors="ignore").splitlines():
parts = raw.split("\t")
if len(parts) < 2:
continue
by_dir.setdefault(parts[-1].split("/")[0], set()).add(parts[0][0]) # parts[-1]=new path (handles renames)
sym = {"added": "+", "modified": "~", "removed": "-"}
lines = []
for top in sorted(by_dir):
codes = by_dir[top]
kind = "added" if codes == {"A"} else "removed" if codes == {"D"} else "modified"
lines.append(f"{sym[kind]}{top}")
return lines
def force_sync(dest: Path, stamp: Path) -> None:
"""Hard-reset the clone to its upstream, discarding local changes."""
if not binaries.Git:
write_stamp(stamp)
return
git = str(binaries.Git)
try:
subprocess.run([git, "-C", str(dest), "fetch", "--depth", "1", "origin"], check=True, capture_output=True) # nosec B603
subprocess.run([git, "-C", str(dest), "reset", "--hard", "@{u}"], check=True, capture_output=True) # nosec B603
except subprocess.CalledProcessError as e:
log.warning("could not force-refresh service repo at %s, using existing copy: %s", dest, stderr_text(e))
write_stamp(stamp)
def is_dirty(dest: Path) -> bool:
"""True if the clone has uncommitted edits to tracked files or local commits not on its upstream.
This function intentionally ignores untracked files: a `git pull --ff-only` only touches tracked
files, so untracked files (including `__pycache__` that importing the services creates) are never
at risk and must not falsely flag the clone as dirty.
"""
git = str(binaries.Git)
# modifications to tracked files only (--untracked-files=no skips __pycache__ etc.)
status = subprocess.run(
[git, "-C", str(dest), "status", "--porcelain", "--untracked-files=no"], capture_output=True
) # nosec B603
if status.returncode == 0 and status.stdout.strip():
return True
# local commits ahead of upstream (committed but not pushed); no upstream → treat as clean
ahead = subprocess.run([git, "-C", str(dest), "rev-list", "--count", "@{u}..HEAD"], capture_output=True) # nosec B603
if ahead.returncode == 0:
try:
return int(ahead.stdout.strip() or b"0") > 0
except ValueError:
return False
return False
def is_stale(stamp: Path, ttl: int) -> bool:
try:
return (time.time() - float(stamp.read_text().strip())) >= ttl
except (OSError, ValueError):
return True
def write_stamp(stamp: Path) -> None:
try:
stamp.parent.mkdir(parents=True, exist_ok=True)
stamp.write_text(str(time.time()))
except OSError:
pass
def stderr_text(e: subprocess.CalledProcessError) -> str:
return (e.stderr or b"").decode(errors="ignore").strip()