mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-16 05:55:09 +00:00
feat(dl): run user post-download scripts with metadata variables
New post_scripts config and repeatable --postscript flag run a command per output, season/album folder, or run, substituting filename-template variables into pre-tokenized argv. The REST API rejects hook commands. Closes #150
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
"""Post-script command building: the injection guard, and how variables become argv."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.utils.post_scripts import (
|
||||
SIDECAR_SEPARATOR,
|
||||
_entries,
|
||||
build_context,
|
||||
dispatch,
|
||||
substitute,
|
||||
tokenize,
|
||||
)
|
||||
|
||||
|
||||
def build(command: str, context: dict[str, str]) -> list[str]:
|
||||
return substitute(tokenize(command), context)
|
||||
|
||||
|
||||
def test_metacharacters_in_a_value_stay_one_argument():
|
||||
"""The whole point of tokenizing before substituting: a title can't become a command."""
|
||||
argv = build(
|
||||
"upload.py {filepath} --title={title}",
|
||||
{
|
||||
"filepath": "/downloads/Bob's Burgers S01E01.mkv",
|
||||
"title": 'Bob"; rm -rf ~',
|
||||
},
|
||||
)
|
||||
assert argv == ["upload.py", "/downloads/Bob's Burgers S01E01.mkv", '--title=Bob"; rm -rf ~']
|
||||
|
||||
|
||||
def test_empty_variable_substitutes_empty():
|
||||
argv = build("upload.py {filepath} --tmdb={tmdb}", {"filepath": "/x.mkv", "tmdb": ""})
|
||||
assert argv == ["upload.py", "/x.mkv", "--tmdb="]
|
||||
|
||||
|
||||
def test_sidecars_join_as_a_single_token():
|
||||
subs = [Path("/dl/Show S01E01.en.srt"), Path("/dl/Show S01E01.fr.srt")]
|
||||
argv = build("upload.py --subs={sidecars}", {"sidecars": SIDECAR_SEPARATOR.join(map(str, subs))})
|
||||
assert len(argv) == 2
|
||||
assert argv[1].removeprefix("--subs=").split(SIDECAR_SEPARATOR) == [str(p) for p in subs]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hostile_value",
|
||||
["--upload-file=/etc/passwd", "-rf", "--output=/etc/cron.d/x"],
|
||||
)
|
||||
def test_bare_variable_expanding_to_a_flag_is_refused(caplog, hostile_value):
|
||||
"""A service-controlled value that becomes a whole -flag must not run the command."""
|
||||
dispatch("success", "file", {"title_raw": hostile_value}, ["curl example.com {title_raw}"])
|
||||
assert "option-like token" in caplog.text
|
||||
|
||||
|
||||
def test_flag_prefixed_variable_is_not_refused(caplog):
|
||||
"""--opt={var} is the user's own flag; the value in the middle can never forge one."""
|
||||
calls = []
|
||||
import unshackle.core.utils.post_scripts as ps
|
||||
|
||||
original = ps.subprocess.Popen
|
||||
ps.subprocess.Popen = lambda argv, **kw: calls.append(argv) or original(["true"])
|
||||
try:
|
||||
dispatch("success", "file", {"title": "--evil=1"}, ["echo --title={title}"])
|
||||
finally:
|
||||
ps.subprocess.Popen = original
|
||||
assert calls and calls[0] == ["echo", "--title=--evil=1"]
|
||||
assert "option-like token" not in caplog.text
|
||||
|
||||
|
||||
def test_unknown_variable_becomes_empty(caplog):
|
||||
argv = build("upload.py --x={nope}", {})
|
||||
assert argv == ["upload.py", "--x="]
|
||||
assert "nope" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command,expected",
|
||||
[
|
||||
('python "/opt/my scripts/upload.py" {filepath}', ["python", "/opt/my scripts/upload.py", "/x.mkv"]),
|
||||
("python upload.py {filepath}", ["python", "upload.py", "/x.mkv"]),
|
||||
],
|
||||
)
|
||||
def test_quoted_paths_in_the_template_survive_tokenizing(command, expected):
|
||||
assert build(command, {"filepath": "/x.mkv"}) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command,expected",
|
||||
[
|
||||
(r"python C:\Scripts\up.py {filepath}", ["python", "C:\\Scripts\\up.py", "/x.mkv"]),
|
||||
(r'python "C:\My Scripts\up.py" {filepath}', ["python", "C:\\My Scripts\\up.py", "/x.mkv"]),
|
||||
('python up.py --title="{title}"', ["python", "up.py", "--title=T"]),
|
||||
],
|
||||
)
|
||||
def test_windows_tokenizing_matches_posix_semantics(monkeypatch, command, expected):
|
||||
import os
|
||||
|
||||
monkeypatch.setattr(os, "name", "nt")
|
||||
assert build(command, {"filepath": "/x.mkv", "title": "T"}) == expected
|
||||
|
||||
|
||||
def test_control_chars_stripped_so_sidecar_newline_separator_holds():
|
||||
"""A service title with a newline must not survive into a filename and forge a {sidecars} entry."""
|
||||
from unshackle.core.utilities import sanitize_filename
|
||||
|
||||
assert "\n" not in sanitize_filename("Ep\n/etc/cron.d/x", " ")
|
||||
assert "\t" not in sanitize_filename("Ep\tfoo", " ")
|
||||
|
||||
|
||||
def test_failure_context_still_has_title_and_year():
|
||||
"""A failure dispatch has no MediaInfo, but a notifier's --title={title} must not go empty."""
|
||||
title = SimpleNamespace(title="Bob's Burgers", name="Ep", year=2011, season=1, number=2, id="x")
|
||||
context = build_context(title, None, service="SVC", error="Boom")
|
||||
assert context["title"] == "Bob's Burgers"
|
||||
assert context["year"] == "2011"
|
||||
assert context["error"] == "Boom"
|
||||
assert context["filepath"] == ""
|
||||
|
||||
|
||||
def test_success_context_prefers_the_template_title():
|
||||
title = SimpleNamespace(
|
||||
title="Raw $how",
|
||||
name="Ep",
|
||||
year=2011,
|
||||
season=1,
|
||||
number=2,
|
||||
id="x",
|
||||
build_template_context=lambda media_info, show_service: {"title": "From Template", "year": 2020},
|
||||
)
|
||||
context = build_context(title, media_info=object(), service="SVC")
|
||||
assert context["title"] == "From Template"
|
||||
assert context["year"] == "2020"
|
||||
assert context["title_raw"] == "Raw $how"
|
||||
|
||||
|
||||
def test_dispatch_survives_an_unbalanced_quote(caplog):
|
||||
"""dispatch never raises: a malformed template is skipped with a warning, not a crash."""
|
||||
dispatch("success", "file", {}, ['python "unclosed'])
|
||||
assert "malformed" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw,expected",
|
||||
[
|
||||
("python x.py {filepath}", ["python x.py {filepath}"]), # lone string, not a list
|
||||
({"command": "python x.py"}, ["python x.py"]), # lone mapping, not a list
|
||||
(42, []), # nonsense is ignored, never iterated
|
||||
([42, None, "ok.py"], ["ok.py"]), # junk entries are dropped, valid ones survive
|
||||
],
|
||||
)
|
||||
def test_malformed_post_scripts_config_shapes(monkeypatch, raw, expected):
|
||||
monkeypatch.setattr(config, "post_scripts", raw)
|
||||
assert list(_entries("success", "file")) == expected
|
||||
|
||||
|
||||
def test_typoed_event_or_mode_warns_instead_of_silently_dropping(monkeypatch, caplog):
|
||||
"""Fire-and-forget gives no other feedback, so a dead entry must at least warn."""
|
||||
monkeypatch.setattr(
|
||||
config,
|
||||
"post_scripts",
|
||||
[{"command": "typo.py", "mode": "seson"}, {"command": "ok.py", "mode": "season"}],
|
||||
)
|
||||
for dispatch_mode in ("file", "season", "run"):
|
||||
assert list(_entries("success", dispatch_mode)) == (["ok.py"] if dispatch_mode == "season" else [])
|
||||
assert caplog.text.count("unknown event/mode") == 1 # warned once, not per dispatch
|
||||
|
||||
|
||||
def test_tagging_ids_are_read_at_dispatch_not_snapshotted():
|
||||
"""A series' TMDB ID is resolved inside the title loop, after any pre-loop snapshot.
|
||||
|
||||
Guards a regression where ``post_script_ids`` was a dict built before the loop, so
|
||||
``{tmdb}`` was empty for every download that did not pass ``--tmdb`` by hand.
|
||||
"""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
|
||||
holder = SimpleNamespace(tmdb_id=None, imdb_id="tt0000001", tvdb_id=None)
|
||||
assert build_context(SimpleNamespace(), ids=dl.post_script_ids(holder))["tmdb"] == ""
|
||||
holder.tmdb_id = 1396
|
||||
assert build_context(SimpleNamespace(), ids=dl.post_script_ids(holder))["tmdb"] == "1396"
|
||||
|
||||
body = ast.parse(textwrap.dedent(inspect.getsource(dl.result)))
|
||||
for node in ast.walk(body):
|
||||
targets = getattr(node, "targets", [])
|
||||
assert not any(isinstance(t, ast.Name) and t.id == "post_script_ids" for t in targets), (
|
||||
"post_script_ids is snapshotted again; it must be read per dispatch"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__, "-q"]))
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Music post-script hooks, which no live service was available to test against.
|
||||
|
||||
Music returns from ``dl.result`` before the title loop, so its hooks are dispatched inside
|
||||
the music pipeline. These check the pieces that pipeline feeds them: that a Song builds a
|
||||
usable context, that an Album groups the way a season does, and that the album and run
|
||||
contexts blank the file-level variables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.utils.post_scripts import build_context, dispatch, substitute, tokenize
|
||||
|
||||
|
||||
class FakeSong(SimpleNamespace):
|
||||
"""A Song as the hook sees it: name/artist/album, no season or episode."""
|
||||
|
||||
def build_template_context(self, media_info, show_service=True):
|
||||
return {
|
||||
"title": self.name,
|
||||
"artist": self.artist,
|
||||
"album": self.album,
|
||||
"track_number": f"{self.track:02}",
|
||||
"year": self.year,
|
||||
"audio": "FLAC",
|
||||
"video": "",
|
||||
}
|
||||
|
||||
|
||||
def song(**kwargs):
|
||||
defaults = dict(
|
||||
name="Blue Monday", artist="New Order", album="Power, Corruption & Lies", track=4, year=1983, id="s1"
|
||||
)
|
||||
return FakeSong(**{**defaults, "id": "s1", **kwargs})
|
||||
|
||||
|
||||
def test_song_context_has_music_fields_and_blank_episode_fields():
|
||||
context = build_context(
|
||||
song(),
|
||||
media_info=object(),
|
||||
filepath=Path("/dl/New Order/04. Blue Monday.flac"),
|
||||
service="EXAMPLE",
|
||||
)
|
||||
assert context["title"] == "Blue Monday"
|
||||
assert context["artist"] == "New Order"
|
||||
assert context["album"] == "Power, Corruption & Lies"
|
||||
assert context["track_number"] == "04"
|
||||
assert context["service"] == "EXAMPLE"
|
||||
assert context["filename"] == "04. Blue Monday.flac"
|
||||
assert context["ext"] == ".flac"
|
||||
assert context["folder"] == "/dl/New Order"
|
||||
assert context["season"] == ""
|
||||
assert context["episode"] == ""
|
||||
|
||||
|
||||
def test_song_context_survives_without_media_info():
|
||||
"""The failure path and any pre-mux dispatch pass no MediaInfo."""
|
||||
context = build_context(song(), None, service="EXAMPLE", error="HTTPError: 403")
|
||||
assert context["title"] == "Blue Monday"
|
||||
assert context["error"] == "HTTPError: 403"
|
||||
assert context["filepath"] == ""
|
||||
|
||||
|
||||
def test_album_grouping_matches_the_season_grouping_rule():
|
||||
"""post_script_group's music branch: albums group by (album artist, album)."""
|
||||
from unshackle.commands.dl import dl # noqa: F401 - import guard, the grouping is inlined in result()
|
||||
|
||||
def group(candidate):
|
||||
artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "")
|
||||
return ("album", artist, getattr(candidate, "album", ""))
|
||||
|
||||
a = song(track=1)
|
||||
b = song(track=2)
|
||||
c = song(album="Technique")
|
||||
assert group(a) == group(b)
|
||||
assert group(a) != group(c)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["season", "run"])
|
||||
def test_album_and_run_contexts_blank_the_file_variables(mode):
|
||||
"""Mirrors the blanking dl.py applies before dispatching an album or run hook."""
|
||||
base = build_context(
|
||||
song(),
|
||||
media_info=object(),
|
||||
filepath=Path("/dl/New Order/04. Blue Monday.flac"),
|
||||
service="EXAMPLE",
|
||||
)
|
||||
context = dict(base)
|
||||
if mode == "season":
|
||||
for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"):
|
||||
context[key] = ""
|
||||
else:
|
||||
context = dict.fromkeys(base, "")
|
||||
context["folder"] = "/dl/New Order"
|
||||
|
||||
assert context["filepath"] == ""
|
||||
assert context["sidecars"] == ""
|
||||
assert context["folder"] == "/dl/New Order"
|
||||
if mode == "run":
|
||||
assert context["title"] == ""
|
||||
assert set(context) == set(base)
|
||||
else:
|
||||
assert context["album"] == "Power, Corruption & Lies"
|
||||
|
||||
|
||||
def test_a_music_hook_command_builds_the_argv_a_script_would_get(monkeypatch, tmp_path):
|
||||
"""End to end through the real dispatcher: config entry in, argv out."""
|
||||
from unshackle.core.config import config
|
||||
|
||||
marker = tmp_path / "argv.txt"
|
||||
script = tmp_path / "hook.py"
|
||||
script.write_text("import sys, pathlib\npathlib.Path(sys.argv[1]).write_text(chr(10).join(sys.argv[2:]))\n")
|
||||
|
||||
import sys
|
||||
|
||||
command = (
|
||||
f'"{sys.executable}" "{script}" "{marker}" '
|
||||
"--album={album} --artist={artist} --track={track_number} --file={filepath}"
|
||||
)
|
||||
monkeypatch.setattr(config, "post_scripts", [{"command": command, "mode": "file", "event": "success"}])
|
||||
|
||||
context = build_context(
|
||||
song(),
|
||||
media_info=object(),
|
||||
filepath=Path("/dl/New Order/04. Blue Monday.flac"),
|
||||
service="EXAMPLE",
|
||||
)
|
||||
dispatch("success", "file", context)
|
||||
|
||||
for _ in range(100):
|
||||
if marker.exists():
|
||||
break
|
||||
import time
|
||||
|
||||
time.sleep(0.05)
|
||||
assert marker.exists(), "the hook process never ran"
|
||||
argv = marker.read_text().splitlines()
|
||||
assert "--album=Power, Corruption & Lies" in argv
|
||||
assert "--artist=New Order" in argv
|
||||
assert "--track=04" in argv
|
||||
assert "--file=/dl/New Order/04. Blue Monday.flac" in argv
|
||||
|
||||
|
||||
def test_album_hook_mapping_is_bound_before_any_early_branch():
|
||||
"""--skip-dl skips the final-move block; the album hook still runs after it.
|
||||
|
||||
Guards a NameError: the mapping used to be created inside that block, so a music
|
||||
download with --skip-dl reached the album hook with an unbound name.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
|
||||
source = inspect.getsource(dl.result)
|
||||
body = source[source.index("def download_music_title") :]
|
||||
bind = body.index("album_post_script: dict")
|
||||
for use in ("album_post_script[final_path.parent]", "album_post_script.items()"):
|
||||
assert bind < body.index(use), f"{use} can run before the mapping is bound"
|
||||
line = body[:bind].rsplit("\n", 1)[-1]
|
||||
assert len(line) == 16, f"bound at indent {len(line)}, expected the function body level"
|
||||
|
||||
|
||||
def test_album_hook_identifies_the_album_not_the_last_track():
|
||||
"""A Song's ``{title}`` is the song, so blanking only the video fields is not enough.
|
||||
|
||||
Guards a regression where an album hook was handed whichever track landed last:
|
||||
``{title}`` was a song name and ``{track_number}``/``{isrc}`` were that track's.
|
||||
"""
|
||||
from unshackle.commands.dl import dl
|
||||
|
||||
context = build_context(
|
||||
song(track=11, isrc="GBAAA8300001"),
|
||||
media_info=object(),
|
||||
filepath=Path("/dl/New Order - PCL (1983)/11. Blue Monday.flac"),
|
||||
service="EXAMPLE",
|
||||
)
|
||||
context.setdefault("isrc", "GBAAA8300001")
|
||||
|
||||
album = dict(context)
|
||||
for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"):
|
||||
album[key] = ""
|
||||
for key in ("track_number", "disc", "isrc"):
|
||||
if key in album:
|
||||
album[key] = ""
|
||||
album["title"] = album["title_raw"] = album.get("album", "")
|
||||
|
||||
assert album["title"] == "Power, Corruption & Lies"
|
||||
assert album["title_raw"] == "Power, Corruption & Lies"
|
||||
assert album["track_number"] == ""
|
||||
assert album["isrc"] == ""
|
||||
assert album["artist"] == "New Order" # album-level, kept
|
||||
|
||||
import inspect
|
||||
|
||||
body = inspect.getsource(dl.result)
|
||||
body = body[body.index("for folder, context in album_post_script.items()") :][:900]
|
||||
for key in ("track_number", "disc", "isrc"):
|
||||
assert f'"{key}"' in body, f"the album hook no longer blanks {key}"
|
||||
assert 'album_context["title"]' in body
|
||||
|
||||
|
||||
def test_a_music_download_failure_dispatches_a_failure_hook():
|
||||
"""Parity with the video worker, which dispatches failure/file when it raises."""
|
||||
import inspect
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
|
||||
body = inspect.getsource(dl.result)
|
||||
body = body[body.index("def download_music_title") :]
|
||||
assert body.count("music_failure(") == 3, "expected one definition and both failure exits"
|
||||
assert body.index("def music_failure") < body.index("music_failure(e)")
|
||||
assert "music_failure(error)" in body
|
||||
|
||||
release = SimpleNamespace(title="Power, Corruption & Lies", artist="New Order", year=1983)
|
||||
context = build_context(release, service="EXAMPLE", error="HTTPError: 403")
|
||||
assert context["title"] == "Power, Corruption & Lies"
|
||||
assert context["error"] == "HTTPError: 403"
|
||||
assert context["filepath"] == ""
|
||||
|
||||
|
||||
def test_an_album_name_with_a_comma_stays_one_argument():
|
||||
"""'Power, Corruption & Lies' must not split, which is the whole tokenize-first point."""
|
||||
argv = substitute(tokenize("upload.py --album={album}"), {"album": "Power, Corruption & Lies"})
|
||||
assert argv == ["upload.py", "--album=Power, Corruption & Lies"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__, "-q"]))
|
||||
Reference in New Issue
Block a user