feat(dl): run user post-download scripts with metadata variables

New post_scripts config and repeatable --postscript flag run a command per output, season/album folder, or run, substituting filename-template variables into pre-tokenized argv. The REST API rejects hook commands.

Closes #150
This commit is contained in:
imSp4rky
2026-08-18 08:26:14 -06:00
parent 511f96d8db
commit 89acfab279
17 changed files with 1154 additions and 23 deletions
+198
View File
@@ -0,0 +1,198 @@
"""Post-script command building: the injection guard, and how variables become argv."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from unshackle.core.config import config
from unshackle.core.utils.post_scripts import (
SIDECAR_SEPARATOR,
_entries,
build_context,
dispatch,
substitute,
tokenize,
)
def build(command: str, context: dict[str, str]) -> list[str]:
return substitute(tokenize(command), context)
def test_metacharacters_in_a_value_stay_one_argument():
"""The whole point of tokenizing before substituting: a title can't become a command."""
argv = build(
"upload.py {filepath} --title={title}",
{
"filepath": "/downloads/Bob's Burgers S01E01.mkv",
"title": 'Bob"; rm -rf ~',
},
)
assert argv == ["upload.py", "/downloads/Bob's Burgers S01E01.mkv", '--title=Bob"; rm -rf ~']
def test_empty_variable_substitutes_empty():
argv = build("upload.py {filepath} --tmdb={tmdb}", {"filepath": "/x.mkv", "tmdb": ""})
assert argv == ["upload.py", "/x.mkv", "--tmdb="]
def test_sidecars_join_as_a_single_token():
subs = [Path("/dl/Show S01E01.en.srt"), Path("/dl/Show S01E01.fr.srt")]
argv = build("upload.py --subs={sidecars}", {"sidecars": SIDECAR_SEPARATOR.join(map(str, subs))})
assert len(argv) == 2
assert argv[1].removeprefix("--subs=").split(SIDECAR_SEPARATOR) == [str(p) for p in subs]
@pytest.mark.parametrize(
"hostile_value",
["--upload-file=/etc/passwd", "-rf", "--output=/etc/cron.d/x"],
)
def test_bare_variable_expanding_to_a_flag_is_refused(caplog, hostile_value):
"""A service-controlled value that becomes a whole -flag must not run the command."""
dispatch("success", "file", {"title_raw": hostile_value}, ["curl example.com {title_raw}"])
assert "option-like token" in caplog.text
def test_flag_prefixed_variable_is_not_refused(caplog):
"""--opt={var} is the user's own flag; the value in the middle can never forge one."""
calls = []
import unshackle.core.utils.post_scripts as ps
original = ps.subprocess.Popen
ps.subprocess.Popen = lambda argv, **kw: calls.append(argv) or original(["true"])
try:
dispatch("success", "file", {"title": "--evil=1"}, ["echo --title={title}"])
finally:
ps.subprocess.Popen = original
assert calls and calls[0] == ["echo", "--title=--evil=1"]
assert "option-like token" not in caplog.text
def test_unknown_variable_becomes_empty(caplog):
argv = build("upload.py --x={nope}", {})
assert argv == ["upload.py", "--x="]
assert "nope" in caplog.text
@pytest.mark.parametrize(
"command,expected",
[
('python "/opt/my scripts/upload.py" {filepath}', ["python", "/opt/my scripts/upload.py", "/x.mkv"]),
("python upload.py {filepath}", ["python", "upload.py", "/x.mkv"]),
],
)
def test_quoted_paths_in_the_template_survive_tokenizing(command, expected):
assert build(command, {"filepath": "/x.mkv"}) == expected
@pytest.mark.parametrize(
"command,expected",
[
(r"python C:\Scripts\up.py {filepath}", ["python", "C:\\Scripts\\up.py", "/x.mkv"]),
(r'python "C:\My Scripts\up.py" {filepath}', ["python", "C:\\My Scripts\\up.py", "/x.mkv"]),
('python up.py --title="{title}"', ["python", "up.py", "--title=T"]),
],
)
def test_windows_tokenizing_matches_posix_semantics(monkeypatch, command, expected):
import os
monkeypatch.setattr(os, "name", "nt")
assert build(command, {"filepath": "/x.mkv", "title": "T"}) == expected
def test_control_chars_stripped_so_sidecar_newline_separator_holds():
"""A service title with a newline must not survive into a filename and forge a {sidecars} entry."""
from unshackle.core.utilities import sanitize_filename
assert "\n" not in sanitize_filename("Ep\n/etc/cron.d/x", " ")
assert "\t" not in sanitize_filename("Ep\tfoo", " ")
def test_failure_context_still_has_title_and_year():
"""A failure dispatch has no MediaInfo, but a notifier's --title={title} must not go empty."""
title = SimpleNamespace(title="Bob's Burgers", name="Ep", year=2011, season=1, number=2, id="x")
context = build_context(title, None, service="SVC", error="Boom")
assert context["title"] == "Bob's Burgers"
assert context["year"] == "2011"
assert context["error"] == "Boom"
assert context["filepath"] == ""
def test_success_context_prefers_the_template_title():
title = SimpleNamespace(
title="Raw $how",
name="Ep",
year=2011,
season=1,
number=2,
id="x",
build_template_context=lambda media_info, show_service: {"title": "From Template", "year": 2020},
)
context = build_context(title, media_info=object(), service="SVC")
assert context["title"] == "From Template"
assert context["year"] == "2020"
assert context["title_raw"] == "Raw $how"
def test_dispatch_survives_an_unbalanced_quote(caplog):
"""dispatch never raises: a malformed template is skipped with a warning, not a crash."""
dispatch("success", "file", {}, ['python "unclosed'])
assert "malformed" in caplog.text
@pytest.mark.parametrize(
"raw,expected",
[
("python x.py {filepath}", ["python x.py {filepath}"]), # lone string, not a list
({"command": "python x.py"}, ["python x.py"]), # lone mapping, not a list
(42, []), # nonsense is ignored, never iterated
([42, None, "ok.py"], ["ok.py"]), # junk entries are dropped, valid ones survive
],
)
def test_malformed_post_scripts_config_shapes(monkeypatch, raw, expected):
monkeypatch.setattr(config, "post_scripts", raw)
assert list(_entries("success", "file")) == expected
def test_typoed_event_or_mode_warns_instead_of_silently_dropping(monkeypatch, caplog):
"""Fire-and-forget gives no other feedback, so a dead entry must at least warn."""
monkeypatch.setattr(
config,
"post_scripts",
[{"command": "typo.py", "mode": "seson"}, {"command": "ok.py", "mode": "season"}],
)
for dispatch_mode in ("file", "season", "run"):
assert list(_entries("success", dispatch_mode)) == (["ok.py"] if dispatch_mode == "season" else [])
assert caplog.text.count("unknown event/mode") == 1 # warned once, not per dispatch
def test_tagging_ids_are_read_at_dispatch_not_snapshotted():
"""A series' TMDB ID is resolved inside the title loop, after any pre-loop snapshot.
Guards a regression where ``post_script_ids`` was a dict built before the loop, so
``{tmdb}`` was empty for every download that did not pass ``--tmdb`` by hand.
"""
import ast
import inspect
import textwrap
from unshackle.commands.dl import dl
holder = SimpleNamespace(tmdb_id=None, imdb_id="tt0000001", tvdb_id=None)
assert build_context(SimpleNamespace(), ids=dl.post_script_ids(holder))["tmdb"] == ""
holder.tmdb_id = 1396
assert build_context(SimpleNamespace(), ids=dl.post_script_ids(holder))["tmdb"] == "1396"
body = ast.parse(textwrap.dedent(inspect.getsource(dl.result)))
for node in ast.walk(body):
targets = getattr(node, "targets", [])
assert not any(isinstance(t, ast.Name) and t.id == "post_script_ids" for t in targets), (
"post_script_ids is snapshotted again; it must be read per dispatch"
)
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-q"]))
+232
View File
@@ -0,0 +1,232 @@
"""Music post-script hooks, which no live service was available to test against.
Music returns from ``dl.result`` before the title loop, so its hooks are dispatched inside
the music pipeline. These check the pieces that pipeline feeds them: that a Song builds a
usable context, that an Album groups the way a season does, and that the album and run
contexts blank the file-level variables.
"""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from unshackle.core.utils.post_scripts import build_context, dispatch, substitute, tokenize
class FakeSong(SimpleNamespace):
"""A Song as the hook sees it: name/artist/album, no season or episode."""
def build_template_context(self, media_info, show_service=True):
return {
"title": self.name,
"artist": self.artist,
"album": self.album,
"track_number": f"{self.track:02}",
"year": self.year,
"audio": "FLAC",
"video": "",
}
def song(**kwargs):
defaults = dict(
name="Blue Monday", artist="New Order", album="Power, Corruption & Lies", track=4, year=1983, id="s1"
)
return FakeSong(**{**defaults, "id": "s1", **kwargs})
def test_song_context_has_music_fields_and_blank_episode_fields():
context = build_context(
song(),
media_info=object(),
filepath=Path("/dl/New Order/04. Blue Monday.flac"),
service="EXAMPLE",
)
assert context["title"] == "Blue Monday"
assert context["artist"] == "New Order"
assert context["album"] == "Power, Corruption & Lies"
assert context["track_number"] == "04"
assert context["service"] == "EXAMPLE"
assert context["filename"] == "04. Blue Monday.flac"
assert context["ext"] == ".flac"
assert context["folder"] == "/dl/New Order"
assert context["season"] == ""
assert context["episode"] == ""
def test_song_context_survives_without_media_info():
"""The failure path and any pre-mux dispatch pass no MediaInfo."""
context = build_context(song(), None, service="EXAMPLE", error="HTTPError: 403")
assert context["title"] == "Blue Monday"
assert context["error"] == "HTTPError: 403"
assert context["filepath"] == ""
def test_album_grouping_matches_the_season_grouping_rule():
"""post_script_group's music branch: albums group by (album artist, album)."""
from unshackle.commands.dl import dl # noqa: F401 - import guard, the grouping is inlined in result()
def group(candidate):
artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "")
return ("album", artist, getattr(candidate, "album", ""))
a = song(track=1)
b = song(track=2)
c = song(album="Technique")
assert group(a) == group(b)
assert group(a) != group(c)
@pytest.mark.parametrize("mode", ["season", "run"])
def test_album_and_run_contexts_blank_the_file_variables(mode):
"""Mirrors the blanking dl.py applies before dispatching an album or run hook."""
base = build_context(
song(),
media_info=object(),
filepath=Path("/dl/New Order/04. Blue Monday.flac"),
service="EXAMPLE",
)
context = dict(base)
if mode == "season":
for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"):
context[key] = ""
else:
context = dict.fromkeys(base, "")
context["folder"] = "/dl/New Order"
assert context["filepath"] == ""
assert context["sidecars"] == ""
assert context["folder"] == "/dl/New Order"
if mode == "run":
assert context["title"] == ""
assert set(context) == set(base)
else:
assert context["album"] == "Power, Corruption & Lies"
def test_a_music_hook_command_builds_the_argv_a_script_would_get(monkeypatch, tmp_path):
"""End to end through the real dispatcher: config entry in, argv out."""
from unshackle.core.config import config
marker = tmp_path / "argv.txt"
script = tmp_path / "hook.py"
script.write_text("import sys, pathlib\npathlib.Path(sys.argv[1]).write_text(chr(10).join(sys.argv[2:]))\n")
import sys
command = (
f'"{sys.executable}" "{script}" "{marker}" '
"--album={album} --artist={artist} --track={track_number} --file={filepath}"
)
monkeypatch.setattr(config, "post_scripts", [{"command": command, "mode": "file", "event": "success"}])
context = build_context(
song(),
media_info=object(),
filepath=Path("/dl/New Order/04. Blue Monday.flac"),
service="EXAMPLE",
)
dispatch("success", "file", context)
for _ in range(100):
if marker.exists():
break
import time
time.sleep(0.05)
assert marker.exists(), "the hook process never ran"
argv = marker.read_text().splitlines()
assert "--album=Power, Corruption & Lies" in argv
assert "--artist=New Order" in argv
assert "--track=04" in argv
assert "--file=/dl/New Order/04. Blue Monday.flac" in argv
def test_album_hook_mapping_is_bound_before_any_early_branch():
"""--skip-dl skips the final-move block; the album hook still runs after it.
Guards a NameError: the mapping used to be created inside that block, so a music
download with --skip-dl reached the album hook with an unbound name.
"""
import inspect
from unshackle.commands.dl import dl
source = inspect.getsource(dl.result)
body = source[source.index("def download_music_title") :]
bind = body.index("album_post_script: dict")
for use in ("album_post_script[final_path.parent]", "album_post_script.items()"):
assert bind < body.index(use), f"{use} can run before the mapping is bound"
line = body[:bind].rsplit("\n", 1)[-1]
assert len(line) == 16, f"bound at indent {len(line)}, expected the function body level"
def test_album_hook_identifies_the_album_not_the_last_track():
"""A Song's ``{title}`` is the song, so blanking only the video fields is not enough.
Guards a regression where an album hook was handed whichever track landed last:
``{title}`` was a song name and ``{track_number}``/``{isrc}`` were that track's.
"""
from unshackle.commands.dl import dl
context = build_context(
song(track=11, isrc="GBAAA8300001"),
media_info=object(),
filepath=Path("/dl/New Order - PCL (1983)/11. Blue Monday.flac"),
service="EXAMPLE",
)
context.setdefault("isrc", "GBAAA8300001")
album = dict(context)
for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"):
album[key] = ""
for key in ("track_number", "disc", "isrc"):
if key in album:
album[key] = ""
album["title"] = album["title_raw"] = album.get("album", "")
assert album["title"] == "Power, Corruption & Lies"
assert album["title_raw"] == "Power, Corruption & Lies"
assert album["track_number"] == ""
assert album["isrc"] == ""
assert album["artist"] == "New Order" # album-level, kept
import inspect
body = inspect.getsource(dl.result)
body = body[body.index("for folder, context in album_post_script.items()") :][:900]
for key in ("track_number", "disc", "isrc"):
assert f'"{key}"' in body, f"the album hook no longer blanks {key}"
assert 'album_context["title"]' in body
def test_a_music_download_failure_dispatches_a_failure_hook():
"""Parity with the video worker, which dispatches failure/file when it raises."""
import inspect
from unshackle.commands.dl import dl
body = inspect.getsource(dl.result)
body = body[body.index("def download_music_title") :]
assert body.count("music_failure(") == 3, "expected one definition and both failure exits"
assert body.index("def music_failure") < body.index("music_failure(e)")
assert "music_failure(error)" in body
release = SimpleNamespace(title="Power, Corruption & Lies", artist="New Order", year=1983)
context = build_context(release, service="EXAMPLE", error="HTTPError: 403")
assert context["title"] == "Power, Corruption & Lies"
assert context["error"] == "HTTPError: 403"
assert context["filepath"] == ""
def test_an_album_name_with_a_comma_stays_one_argument():
"""'Power, Corruption & Lies' must not split, which is the whole tokenize-first point."""
argv = substitute(tokenize("upload.py --album={album}"), {"album": "Power, Corruption & Lies"})
assert argv == ["upload.py", "--album=Power, Corruption & Lies"]
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-q"]))