From 09fe2e88b2fa05bc17b8ff4222044e251cee4416 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Thu, 16 Jul 2026 22:47:25 -0600 Subject: [PATCH] perf(api): append job history instead of rewriting the file --- unshackle/core/api/download_manager.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index e9393bc8..e54db8e9 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -189,8 +189,12 @@ def _history_limit() -> int: return 100 +_history_appends_since_trim = 0 # module-level; record_job_history runs serialized on the event loop + + def record_job_history(job: DownloadJob) -> None: """Append a terminal job's summary as one JSON line to the history file (best effort).""" + global _history_appends_since_trim if job.history_recorded or job.status not in TERMINAL_STATUSES: return job.history_recorded = True @@ -211,14 +215,18 @@ def record_job_history(job: DownloadJob) -> None: path.parent.mkdir(parents=True, exist_ok=True) line = json.dumps(entry) limit = _history_limit() - # Serialized on the manager's event loop, so read-append-trim doesn't race. - if limit > 0 and path.exists(): - existing = [ln for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()] - existing.append(line) - path.write_text("\n".join(existing[-limit:]) + "\n", encoding="utf-8") # keep newest N - else: - with open(path, "a", encoding="utf-8") as handle: - handle.write(line + "\n") + # Serialized on the manager's event loop, so append + periodic trim doesn't race. + with open(path, "a", encoding="utf-8") as handle: + handle.write(line + "\n") + # Appending is O(1); only pay the full read-rewrite once appends drift past a slack + # margin, keeping the newest N. + if limit > 0: + _history_appends_since_trim += 1 + if _history_appends_since_trim >= max(16, limit // 4): + _history_appends_since_trim = 0 + existing = [ln for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()] + if len(existing) > limit: + path.write_text("\n".join(existing[-limit:]) + "\n", encoding="utf-8") # keep newest N except OSError as e: log.warning(f"Could not write job history: {e}")