mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-13 12:35:05 +00:00
feat(services): add services_repo_force and compact refresh output
Force-reset repo clones on the automatic TTL refresh when enabled. Report refresh changes as one +/~/-/!TAG line per repo.
This commit is contained in:
@@ -781,7 +781,7 @@ git-pull every service repository configured under `directories.services`. `refr
|
|||||||
{
|
{
|
||||||
"refreshed": true,
|
"refreshed": true,
|
||||||
"repos": [
|
"repos": [
|
||||||
{ "spec": "github.com/example/services", "updated": true, "changes": ["a1b2c3d Update EXAMPLE"] }
|
{ "spec": "github.com/example/services", "updated": true, "changes": ["~EXAMPLE"] }
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -67,6 +67,15 @@ directories:
|
|||||||
`unshackle util refresh-services` command instead **hard-resets** the clone to upstream,
|
`unshackle util refresh-services` command instead **hard-resets** the clone to upstream,
|
||||||
discarding any local edits.
|
discarding any local edits.
|
||||||
|
|
||||||
|
Set `services_repo_force: true` to make the automatic 24 h refresh behave like
|
||||||
|
`refresh-services`. When a refresh is due, the run fetches and hard-resets every clone,
|
||||||
|
and discards any local edits in `_repos/`. Runs between refreshes do not touch the
|
||||||
|
network. Use it on unattended hosts where nobody edits the clones by hand.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services_repo_force: true
|
||||||
|
```
|
||||||
|
|
||||||
!!! warning "Read-only installs and reinstalls"
|
!!! warning "Read-only installs and reinstalls"
|
||||||
If the services directory lives inside the installed package, a reinstall can delete the
|
If the services directory lives inside the installed package, a reinstall can delete the
|
||||||
`_repos` clones. They are re-cloned on next use. On **read-only installs** you must
|
`_repos` clones. They are re-cloned on next use. On **read-only installs** you must
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ Where the file lives, how unshackle discovers it, and how to edit it:
|
|||||||
|
|
||||||
| Area | Keys |
|
| Area | Keys |
|
||||||
|------|------|
|
|------|------|
|
||||||
| [Directories](directories.md) | `directories`, `filenames` |
|
| [Directories](directories.md) | `directories`, `filenames`, `services_repo_force` |
|
||||||
| [Services & authentication](services.md) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` |
|
| [Services & authentication](services.md) | `services`, `credentials`, `firefox_cookies`, `remote_services`, `serve` |
|
||||||
| [Download & processing](download.md) | `dl`, `continue_downloads`, `subtitle`, `audio`, `muxing`, `language_tags`, `dual_multi_mode` |
|
| [Download & processing](download.md) | `dl`, `continue_downloads`, `subtitle`, `audio`, `muxing`, `language_tags`, `dual_multi_mode` |
|
||||||
| [Output & naming](output.md) | `output_template`, `tag`, `tag_rules`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` |
|
| [Output & naming](output.md) | `output_template`, `tag`, `tag_rules`, `tag_group_name`, `tag_imdb_tmdb`, `chapter_fallback_name`, `unicode_filenames` |
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ def test_auto_refresh_refuses_dirty_clone(monkeypatch, tmp_path):
|
|||||||
|
|
||||||
|
|
||||||
def test_manual_refresh_force_overwrites(monkeypatch, tmp_path):
|
def test_manual_refresh_force_overwrites(monkeypatch, tmp_path):
|
||||||
"""Manual path (force=True): hard-reset to upstream, never raises on a dirty clone."""
|
"""Force path (services_repo_force): hard-resets a stale clone, never raises; leaves a fresh clone alone."""
|
||||||
dest = tmp_path / "github.com__a__b"
|
dest = tmp_path / "github.com__a__b"
|
||||||
dest.mkdir(parents=True)
|
dest.mkdir(parents=True)
|
||||||
monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path)
|
monkeypatch.setattr(service_repo, "repos_base", lambda: tmp_path)
|
||||||
@@ -186,6 +186,10 @@ def test_manual_refresh_force_overwrites(monkeypatch, tmp_path):
|
|||||||
assert service_repo.resolve_service_repo("a/b", force=True) == dest
|
assert service_repo.resolve_service_repo("a/b", force=True) == dest
|
||||||
assert "fetch" in calls and "reset" in calls # force-sync ran
|
assert "fetch" in calls and "reset" in calls # force-sync ran
|
||||||
assert "status" not in calls # no dirty gate on the force path
|
assert "status" not in calls # no dirty gate on the force path
|
||||||
|
calls.clear()
|
||||||
|
service_repo.write_stamp(service_repo.stamp_for(dest)) # fresh stamp, inside the TTL
|
||||||
|
assert service_repo.resolve_service_repo("a/b", force=True) == dest
|
||||||
|
assert calls == [] # no network before the TTL expires
|
||||||
|
|
||||||
|
|
||||||
def test_changed_services_groups_by_top_dir(monkeypatch):
|
def test_changed_services_groups_by_top_dir(monkeypatch):
|
||||||
@@ -194,18 +198,13 @@ def test_changed_services_groups_by_top_dir(monkeypatch):
|
|||||||
def run(args, **kw):
|
def run(args, **kw):
|
||||||
if "--name-status" in args:
|
if "--name-status" in args:
|
||||||
out = b"A\tNEW/__init__.py\nM\tOLD/__init__.py\nM\tOLD/config.yaml\nD\tGONE/__init__.py\n"
|
out = b"A\tNEW/__init__.py\nM\tOLD/__init__.py\nM\tOLD/config.yaml\nD\tGONE/__init__.py\n"
|
||||||
elif "--shortstat" in args:
|
|
||||||
out = b" 4 files changed, 10 insertions(+), 2 deletions(-)\n"
|
|
||||||
else:
|
else:
|
||||||
out = b""
|
out = b""
|
||||||
return subprocess.CompletedProcess(args, 0, out, b"")
|
return subprocess.CompletedProcess(args, 0, out, b"")
|
||||||
|
|
||||||
monkeypatch.setattr(service_repo.subprocess, "run", run)
|
monkeypatch.setattr(service_repo.subprocess, "run", run)
|
||||||
lines = service_repo.changed_services(Path("/x"), "aaa", "bbb")
|
lines = service_repo.changed_services(Path("/x"), "aaa", "bbb")
|
||||||
assert "+ NEW (added)" in lines
|
assert lines == ["-GONE", "+NEW", "~OLD"]
|
||||||
assert "~ OLD (modified)" in lines
|
|
||||||
assert "- GONE (removed)" in lines
|
|
||||||
assert any("4 files changed" in line for line in lines)
|
|
||||||
|
|
||||||
|
|
||||||
def test_local_dirty_services_lists_edited_dirs(monkeypatch):
|
def test_local_dirty_services_lists_edited_dirs(monkeypatch):
|
||||||
@@ -234,7 +233,7 @@ def test_refresh_reports_discarded_local_edits(monkeypatch, tmp_path):
|
|||||||
|
|
||||||
monkeypatch.setattr(service_repo.subprocess, "run", run)
|
monkeypatch.setattr(service_repo.subprocess, "run", run)
|
||||||
_, changes = service_repo.refresh_repo("a/b")
|
_, changes = service_repo.refresh_repo("a/b")
|
||||||
assert changes == ["! CR (local changes discarded)"]
|
assert changes == ["!CR"]
|
||||||
|
|
||||||
|
|
||||||
def test_changed_services_up_to_date(monkeypatch):
|
def test_changed_services_up_to_date(monkeypatch):
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ def refresh_services() -> None:
|
|||||||
"""Force a refresh (git pull) of all service repos configured in directories.services."""
|
"""Force a refresh (git pull) of all service repos configured in directories.services."""
|
||||||
from unshackle.core.config import config
|
from unshackle.core.config import config
|
||||||
from unshackle.core.service_repo import is_repo_spec, refresh_repo
|
from unshackle.core.service_repo import is_repo_spec, refresh_repo
|
||||||
from unshackle.core.utils.redact import redact_path
|
|
||||||
|
|
||||||
log = logging.getLogger("util")
|
log = logging.getLogger("util")
|
||||||
|
|
||||||
@@ -43,11 +42,9 @@ def refresh_services() -> None:
|
|||||||
log.error(f"Failed to update {spec} (see log).")
|
log.error(f"Failed to update {spec} (see log).")
|
||||||
continue
|
continue
|
||||||
if changes:
|
if changes:
|
||||||
log.info(f"Updated {spec} → {redact_path(str(dest))}")
|
log.info(f"Updated {spec}: {' '.join(changes)}")
|
||||||
for line in changes:
|
|
||||||
log.info(f" {line}")
|
|
||||||
else:
|
else:
|
||||||
log.info(f"No changes {spec}")
|
log.info(f"Up to date: {spec}")
|
||||||
|
|
||||||
|
|
||||||
@util.command()
|
@util.command()
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ class Config:
|
|||||||
self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or ""
|
self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or ""
|
||||||
self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or ""
|
self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or ""
|
||||||
self.update_checks: bool = kwargs.get("update_checks", True)
|
self.update_checks: bool = kwargs.get("update_checks", True)
|
||||||
|
self.services_repo_force: bool = bool(kwargs.get("services_repo_force", False))
|
||||||
self.update_check_interval: int = kwargs.get("update_check_interval", 24)
|
self.update_check_interval: int = kwargs.get("update_check_interval", 24)
|
||||||
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
||||||
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
||||||
|
|||||||
@@ -80,10 +80,11 @@ def repos_base() -> Path:
|
|||||||
def resolve_service_repo(spec: str, *, ttl: int = DEFAULT_TTL, force: bool = False) -> Optional[Path]:
|
def resolve_service_repo(spec: str, *, ttl: int = DEFAULT_TTL, force: bool = False) -> Optional[Path]:
|
||||||
"""Find the local clone dir of a repo spec.
|
"""Find the local clone dir of a repo spec.
|
||||||
|
|
||||||
First use clones it. ``force=True`` (manual ``refresh-services``) hard-resets the clone to
|
First use clones it. Only a TTL-stale clone touches the network. With ``force=True``
|
||||||
upstream, discarding local changes. Otherwise a TTL-stale clone is fast-forwarded, but a dirty
|
(``services_repo_force``) the refresh hard-resets the clone to upstream and discards local
|
||||||
clone raises ``DirtyServiceRepo`` so the automated dl/search path can warn and exit instead of
|
changes. Without it the refresh fast-forwards, and a dirty clone raises ``DirtyServiceRepo``
|
||||||
clobbering in-progress edits.
|
so the automated dl/search path can warn and exit instead of clobbering in-progress edits.
|
||||||
|
Manual ``refresh-services`` uses ``refresh_repo`` and ignores the TTL.
|
||||||
"""
|
"""
|
||||||
url, branch, dest_name = parse_spec(spec)
|
url, branch, dest_name = parse_spec(spec)
|
||||||
dest = repos_base() / dest_name
|
dest = repos_base() / dest_name
|
||||||
@@ -91,10 +92,8 @@ def resolve_service_repo(spec: str, *, ttl: int = DEFAULT_TTL, force: bool = Fal
|
|||||||
|
|
||||||
if not dest.exists():
|
if not dest.exists():
|
||||||
return clone(url, branch, dest, stamp)
|
return clone(url, branch, dest, stamp)
|
||||||
if force:
|
if is_stale(stamp, ttl):
|
||||||
force_sync(dest, stamp)
|
force_sync(dest, stamp) if force else pull(dest, stamp)
|
||||||
elif is_stale(stamp, ttl):
|
|
||||||
pull(dest, stamp)
|
|
||||||
return dest
|
return dest
|
||||||
|
|
||||||
|
|
||||||
@@ -135,9 +134,9 @@ def pull(dest: Path, stamp: Path) -> None:
|
|||||||
def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]:
|
def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]:
|
||||||
"""Force-sync a repo to upstream and return (clone dir, git-style per-service change lines).
|
"""Force-sync a repo to upstream and return (clone dir, git-style per-service change lines).
|
||||||
|
|
||||||
Used by the manual ``refresh-services`` command. Change lines look like ``+ TAG (added)`` /
|
Used by the manual ``refresh-services`` command. Each line is one service tag with a status
|
||||||
``~ TAG (modified)`` / ``- TAG (removed)`` plus a shortstat. An empty list means the clone is
|
prefix: ``+TAG`` added, ``~TAG`` modified, ``-TAG`` removed, ``!TAG`` local changes discarded.
|
||||||
already up to date.
|
An empty list means the clone is already up to date.
|
||||||
"""
|
"""
|
||||||
url, branch, dest_name = parse_spec(spec)
|
url, branch, dest_name = parse_spec(spec)
|
||||||
dest = repos_base() / dest_name
|
dest = repos_base() / dest_name
|
||||||
@@ -149,7 +148,7 @@ def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]:
|
|||||||
old = head(dest)
|
old = head(dest)
|
||||||
discarded = local_dirty_services(dest)
|
discarded = local_dirty_services(dest)
|
||||||
force_sync(dest, stamp)
|
force_sync(dest, stamp)
|
||||||
lines = [f"! {tag} (local changes discarded)" for tag in discarded]
|
lines = [f"!{tag}" for tag in discarded] # ! = local changes discarded
|
||||||
lines += changed_services(dest, old, head(dest))
|
lines += changed_services(dest, old, head(dest))
|
||||||
return dest, lines
|
return dest, lines
|
||||||
|
|
||||||
@@ -168,7 +167,7 @@ def local_dirty_services(dest: Path) -> list[str]:
|
|||||||
|
|
||||||
|
|
||||||
def changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> list[str]:
|
def changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> list[str]:
|
||||||
"""Summarise a HEAD old->new diff grouped by top-level service dir, plus a line shortstat."""
|
"""Summarise a HEAD old->new diff as one ``+TAG``/``~TAG``/``-TAG`` token per top-level service directory."""
|
||||||
if not old or not new or old == new:
|
if not old or not new or old == new:
|
||||||
return []
|
return []
|
||||||
git = str(binaries.Git)
|
git = str(binaries.Git)
|
||||||
@@ -185,15 +184,12 @@ def changed_services(dest: Path, old: Optional[str], new: Optional[str]) -> list
|
|||||||
for top in sorted(by_dir):
|
for top in sorted(by_dir):
|
||||||
codes = by_dir[top]
|
codes = by_dir[top]
|
||||||
kind = "added" if codes == {"A"} else "removed" if codes == {"D"} else "modified"
|
kind = "added" if codes == {"A"} else "removed" if codes == {"D"} else "modified"
|
||||||
lines.append(f"{sym[kind]} {top} ({kind})")
|
lines.append(f"{sym[kind]}{top}")
|
||||||
short = subprocess.run([git, "-C", str(dest), "diff", "--shortstat", f"{old}..{new}"], capture_output=True) # nosec B603
|
|
||||||
if short.returncode == 0 and short.stdout.strip():
|
|
||||||
lines.append(short.stdout.decode(errors="ignore").strip())
|
|
||||||
return lines
|
return lines
|
||||||
|
|
||||||
|
|
||||||
def force_sync(dest: Path, stamp: Path) -> None:
|
def force_sync(dest: Path, stamp: Path) -> None:
|
||||||
"""Hard-reset the clone to its upstream, discarding local changes (manual refresh only)."""
|
"""Hard-reset the clone to its upstream, discarding local changes."""
|
||||||
if not binaries.Git:
|
if not binaries.Git:
|
||||||
write_stamp(stamp)
|
write_stamp(stamp)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ DIRTY_REPOS: list[str] = []
|
|||||||
for _entry in raw:
|
for _entry in raw:
|
||||||
if isinstance(_entry, str) and is_repo_spec(_entry):
|
if isinstance(_entry, str) and is_repo_spec(_entry):
|
||||||
try:
|
try:
|
||||||
_resolved = resolve_service_repo(_entry)
|
_resolved = resolve_service_repo(_entry, force=config.services_repo_force)
|
||||||
except DirtyServiceRepo as e:
|
except DirtyServiceRepo as e:
|
||||||
# local edits in the clone - record it and let check_load_errors() exit cleanly
|
# local edits in the clone - record it and let check_load_errors() exit cleanly
|
||||||
DIRTY_REPOS.append(redact_path(str(e.path)))
|
DIRTY_REPOS.append(redact_path(str(e.path)))
|
||||||
|
|||||||
Reference in New Issue
Block a user