From 89acfab279629bd29d0c0a619a1158bfd7dfe4c1 Mon Sep 17 00:00:00 2001 From: imSp4rky Date: Mon, 17 Aug 2026 16:10:31 -0600 Subject: [PATCH] feat(dl): run user post-download scripts with metadata variables New post_scripts config and repeatable --postscript flag run a command per output, season/album folder, or run, substituting filename-template variables into pre-tokenized argv. The REST API rejects hook commands. Closes #150 --- docs/dev/rest-api/endpoints.md | 7 + docs/getting-started/configuration-file.md | 1 + docs/guide/cli-reference.md | 1 + docs/guide/downloading.md | 16 ++ docs/reference/configuration.md | 1 + docs/reference/configuration/index.md | 1 + docs/reference/configuration/post-scripts.md | 288 +++++++++++++++++++ mkdocs.yml | 1 + tests/test_post_scripts.py | 198 +++++++++++++ tests/test_post_scripts_music.py | 232 +++++++++++++++ unshackle/commands/dl.py | 199 +++++++++++-- unshackle/core/api/download_manager.py | 4 + unshackle/core/api/handlers.py | 4 + unshackle/core/api/routes.py | 5 +- unshackle/core/config.py | 1 + unshackle/core/utilities.py | 2 +- unshackle/core/utils/post_scripts.py | 216 ++++++++++++++ 17 files changed, 1154 insertions(+), 23 deletions(-) create mode 100644 docs/reference/configuration/post-scripts.md create mode 100644 tests/test_post_scripts.py create mode 100644 tests/test_post_scripts_music.py create mode 100644 unshackle/core/utils/post_scripts.py diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 10963597..f40ee4e9 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -413,6 +413,13 @@ Create a download job. Requires `service` and `title_id`; every other field is a **Validation.** Invalid values return `400 INVALID_PARAMETERS`. Notably: `vcodec` must be one of H264/H265/H.264/H.265/AVC/HEVC/VC1/VC-1/VP8/VP9/AV1; `acodec` one of AAC/AC3/EC3/EAC3/DD/DD+/AC4/OPUS/FLAC/ALAC/VORBIS/OGG/DTS; `range` one of SDR/HDR10/HDR10P/DV/HLG/HYBRID (`HDR10+` is accepted); bitrate/worker/download counts must be positive integers; at most one of the `*_only` flags may be set; `no_subs` cannot be combined with `subs_only`, `no_audio` with `audio_only`, or `s_lang` with `require_subs`. +!!! danger "Rejected parameters" + `postscript`, `post_script` and `post_scripts` are never accepted. A body containing any + of them is rejected with `400 INVALID_PARAMETERS`, because a command string arriving from + an HTTP caller would be remote code execution. Post-download hooks are configured only in + `unshackle.yaml`, where they still run for API jobs. See + [Post-download scripts](../../reference/configuration/post-scripts.md). + !!! warning "Gated parameters (developer)" A few keys are accepted but only permitted when the server config opts in, otherwise the request is rejected with `403 FORBIDDEN`: diff --git a/docs/getting-started/configuration-file.md b/docs/getting-started/configuration-file.md index 04c6a4ac..572e93c3 100644 --- a/docs/getting-started/configuration-file.md +++ b/docs/getting-started/configuration-file.md @@ -87,6 +87,7 @@ The top-level keys group loosely into these areas: | Naming & tagging | `tag`, `output_template`, `chapter_fallback_name` | | External API keys | `tmdb_api_key`, `tvdb_api_key`, `simkl_client_id`, `ipinfo_api_key`, `metadata_providers` | | Behavior & logging | `update_checks`, `redact_paths`, `debug`, `unicode_filenames` | +| Post-download scripts | `post_scripts` | | Paths | `directories`, `filenames` | For the full list with types and defaults, see the [Configuration Reference](../reference/configuration/index.md). diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 58eed11b..3c25caff 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -145,6 +145,7 @@ Keep only certain track types, or skip certain track types. Attachments are alwa | `--no-mux` | Do not mux; keep individual track files. | | `--tag` | Group tag override. | | `--repack` | Add a `REPACK` tag to the filename. | +| `--postscript` | Run a command after each output file, with `{variable}` placeholders substituted. Repeatable. Replaces the `post_scripts` config for this run. See [Post-download scripts](../reference/configuration/post-scripts.md). | ### Metadata & tagging diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index e062e412..0a504c73 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -579,6 +579,21 @@ flags change how the output is assembled: | `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) | | `--merge-video` | Mux video tracks that share a height, range, and codec into one file, so only language varies inside a file. | config `muxing.merge_video` (off) | +### After the download + +`--postscript ""` runs your own command once per output file, with unshackle's +metadata substituted into `{variable}` placeholders. It is repeatable, and it replaces the +`post_scripts` config for that run. Nothing runs under `--no-mux`, since no output is +written. + +```shell title="Hand each finished file to an uploader" +unshackle dl --postscript "python /opt/upload.py {filepath} --service={service}" EXAMPLE 81234567 +``` + +For hooks that persist across runs, for the `season` and `run` modes, for `failure` hooks +and for the full variable list, see +[Post-download scripts](../reference/configuration/post-scripts.md). + ### Naming tags - `--tag`: set the release group tag (overrides the configured tag). @@ -815,6 +830,7 @@ authoritative list. | `--best-available` | | Degrade gracefully instead of failing. | | `--output` | `-o` | Output directory for this run. | | `--split-audio` / `--merge-video` / `--no-mux` | | Muxing behaviour. | +| `--postscript` | | Run a command after each output file. Repeatable. | | `--proxy` / `--no-proxy` / `--no-proxy-download` | | Proxy control. | | `--workers` / `--downloads` / `--slow` | | Concurrency and pacing. | | `--list` / `--list-titles` / `--skip-dl` | | Dry runs. | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index e6bfa177..0ac1bf50 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -14,5 +14,6 @@ This page has moved. The configuration reference is now split by area under | DRM & CDM | [configuration/drm.md](configuration/drm.md) | | Network & proxy | [configuration/network.md](configuration/network.md) | | Key vaults | [configuration/vaults.md](configuration/vaults.md) | +| Post-download scripts | [configuration/post-scripts.md](configuration/post-scripts.md) | | Other settings | [configuration/misc.md](configuration/misc.md) | | Complete annotated example | [configuration/example.md](configuration/example.md) | diff --git a/docs/reference/configuration/index.md b/docs/reference/configuration/index.md index 35dc3bbc..57f3ced7 100644 --- a/docs/reference/configuration/index.md +++ b/docs/reference/configuration/index.md @@ -36,6 +36,7 @@ Where the file lives, how unshackle discovers it, and how to edit it: | [DRM & CDM](drm.md) | `cdm`, `remote_cdm`, `decryption` | | [Network & proxy](network.md) | `network`, `headers`, `proxy_providers` | | [Key vaults](vaults.md) | `key_vaults`, `vault_timeout` | +| [Post-download scripts](post-scripts.md) | `post_scripts` | | [External API keys](misc.md#external-api-keys) | `omdb_api_key`, `tmdb_api_key`, `tvdb_api_key`, `tvdb_pin`, `tvdb_order`, `simkl_client_id`, `metadata_providers`, `anilist_title_language`, `disable_metadata`, `decrypt_labs_api_key`, `ipinfo_api_key` | | [Caching & updates](misc.md#caching-updates) | `title_cache_enabled`, `title_cache_time`, `title_cache_max_retention`, `update_checks`, `update_check_interval` | | [Appearance](misc.md#appearance) | `theme` | diff --git a/docs/reference/configuration/post-scripts.md b/docs/reference/configuration/post-scripts.md new file mode 100644 index 00000000..26e27ea3 --- /dev/null +++ b/docs/reference/configuration/post-scripts.md @@ -0,0 +1,288 @@ +# Post-download scripts { #post-scripts } + +Run your own script when a download finishes, with unshackle's metadata passed in as +`{variable}` placeholders. Use it to hand a finished file to an uploader, send a +notification, or start a library scan. + +## `post_scripts` + +- **Type:** `list[dict]`  ยท  **Default:** `[]` (no scripts run) + +```yaml title="unshackle.yaml" +post_scripts: + - command: "python /opt/upload.py {filepath} --service={service} --season={season} --episode={episode} --tmdb={tmdb} --subs={sidecars}" + event: success # success | failure + mode: file # file | season | run + - command: "python /opt/notify.py --title={title} --episode={episode}" +``` + +Every entry that matches the event and mode runs, so one download can feed an uploader and +a notifier without a wrapper script. An entry may also be written as a plain string, which +is the same as `{command: "...", event: success, mode: file}`. + +| Key | Default | Meaning | +| --- | --- | --- | +| `command` | required | The command to run, with `{variable}` placeholders | +| `event` | `success` | `success` when a file was written, `failure` when the download failed | +| `mode` | `file` | `file`, `season`, or `run`. See [Modes](#modes) | + +An entry whose `event` or `mode` is not one of the listed values is skipped and logs a +warning naming the bad value, once per command. An entry whose `command` cannot be split +into arguments, such as one with an unbalanced quote, is skipped with a warning too. Neither +stops the download or the other entries. + +## How the command runs + +The command is split into arguments **first**, and variables are substituted into those +arguments afterwards. The process runs without a shell. + +This is what keeps a title like `Bob"; rm -rf ~` harmless: a value can never turn into an +extra argument or a shell operator, and you never have to think about quoting. It also +means a path containing spaces always arrives as a single argument on every OS. + +!!! warning "Prefix your variables" + Titles, episode names and IDs come from the streaming service, not from you. Tokenizing + first stops a value becoming a *new* argument, but a value on its own can still be a + whole argument. A bare `{title}` whose value is `--upload-file=/etc/passwd` would arrive + as an option to your script, not as text. Write `--title={title}`, not a bare `{title}`, + so a service can never forge a flag. unshackle refuses to run a command when a + substituted value would become an option-like token (one that starts with `-`), and logs + a warning instead. + +!!! warning "No shell features, and name your interpreter" + Pipes, `&&`, `>` redirection, `~` and globs do not work, because no shell is involved. + If you want them, make the shell explicit: `bash -c "..."`. + + For the same reason there is no interpreter lookup. Write `python /opt/upload.py` + rather than `/opt/upload.py`. unshackle will not guess whether you meant the system + Python or the one in its own virtual environment. + + On Windows, when a **fixed** part of an argument contains a space, quote the whole + argument (`"--out=C:\My Dir\{filename}"`), not just the value. Substituted values + never need quoting on any OS. + + On Windows, do not target a `.bat` or `.cmd` file directly. Windows runs batch files + through `cmd.exe`, whose argument quoting differs from every other program and has a + history of letting a crafted value break out (CVE-2024-3566). Call the interpreter and + pass the script as an argument (`python upload.py`, `pwsh -File upload.ps1`) instead. + +Scripts are fire and forget. unshackle starts the process, writes the command to the debug +log, and carries on. It does not wait for the script, capture its output, time it out, or +change its own exit code when a script fails. A script keeps running if unshackle exits +first, and Ctrl+C during a season pack does not kill the scripts +already started. + +!!! note "One process per file" + In `file` mode a large season pack starts one process per episode with nothing + throttling them. If your script is heavy, queue the work inside it or use `mode: season`. + +## Variables + +Metadata comes from the same naming context that produced the file's name, built per output +file. So `{quality}` and `{hdr}` always describe the file being passed. Downloading +`-q 1080,2160 -r HDR10,SDR` produces four files, each with its own invocation and its own +values. + +| Variable | Value | +| --- | --- | +| `{filepath}` | Absolute path of the output file | +| `{filename}` | Its filename, with extension | +| `{folder}` | Absolute path of its parent folder | +| `{ext}` | Its extension, including the dot | +| `{sidecars}` | Sidecar subtitle files written for this output, newline separated | +| `{title}` | Show, movie or track name, sanitized | +| `{title_raw}` | Exactly what the service returned, untouched | +| `{title_id}` | The service's ID for the title | +| `{service}` | Service tag | +| `{year}` `{season}` `{episode}` `{episode_name}` | Season and episode are plain numbers (`1`, `5`) | +| `{quality}` `{resolution}` `{hdr}` `{hfr}` `{vcodec}` `{acodec}` `{edition}` `{tag}` `{lang_tag}` | As in the filename template, for this file | +| `{tmdb}` `{imdb}` `{tvdb}` | The tagging IDs in use, otherwise empty. See [Tagging IDs](#tagging-ids) | +| `{error}` | Failure message. Empty on success | + +Every variable from your `output_template` is available here as well, including +`{video}` and `{audio}`, of which `{vcodec}` and `{acodec}` are aliases. + +!!! warning "Empty variables become an empty string" + `--tmdb={tmdb}` with no ID resolved hands your script `--tmdb=`, so treat an empty + string as absent. Write your script to handle every variable being empty. + + Empty is common, and not only when data is missing: + + - Nothing was resolved. `{tmdb}` is empty when no metadata provider found the title, + the matching API key is missing, or the title is a movie you gave no ID for. + - The title has no such field. Movies have no `{season}` or `{episode}`, and music has + neither. + - **The naming context renders it empty on purpose.** `{hdr}` is empty for an SDR + file, because SDR is the absence of an HDR tag in a filename rather than a tag that + reads `SDR`. `{edition}`, `{atmos}`, `{multi}`, `{dual}` and `{hfr}` behave the same + way. If your script needs the word `SDR`, derive it from an empty `{hdr}` yourself. + - The hook is a `failure` hook. A failed download has no output file to read metadata + from, so everything that comes from the naming context is empty. See + [Events](#events). + + There is no `{variable?}` conditional here, unlike `output_template`. The token is + always passed, so the argument list your script receives keeps the same shape on every + run and your argument parser does not have to cope with a flag appearing and + disappearing. + + Season and episode are the plain numbers, not the `S01E05` form used in filenames. The + padding and the separator belong to the filename template rather than to the data. + +### Music variables + +A music download builds its variables from the music naming context, so a hook gets the +release fields instead of the season and episode ones. `{season}`, `{episode}` and +`{episode_name}` are always empty for music. + +| Variable | Value | +| --- | --- | +| `{artist}` | Track artist | +| `{album_artist}` | Album artist, falling back to the track artist | +| `{album}` | Album name | +| `{track_number}` `{disc}` | Zero padded. `{disc}` is empty on a single-disc release | +| `{track_total}` `{disc_total}` | Zero padded totals, empty when the service did not give them | +| `{isrc}` `{upc}` `{label}` `{genre}` `{release_type}` `{explicit}` | As the service reported them, empty when absent | + +In a `file` hook `{title}` is the track name. See [Modes](#modes) for what an album hook +puts in `{title}`. + +### Tagging IDs { #tagging-ids } + +`{tmdb}`, `{imdb}` and `{tvdb}` carry the IDs unshackle is tagging the file with, read at +the moment the hook is dispatched rather than snapshotted at the start of the run. An +episode's TMDB ID is resolved by the title search inside the download loop, so a hook fires +with it already filled in. + +They hold an ID you passed with `--tmdb`, `--imdb` or `--tvdb`, plus a TMDB ID the episode +search resolved. No such search runs for a movie, so a movie hook gets them empty unless you +passed one. + +### Sidecar files + +`{sidecars}` holds the sidecar subtitle files written beside the output, which are the only +extra files a download leaves next to it. Attachments such as fonts are muxed into the +container and temporary artwork is deleted, so neither appears. Expect `{sidecars}` to be +empty unless [`subtitle.output_mode`](download.md#subtitle) is `sidecar` or `both`, which is +not the default. A music download writes no sidecars at all. + +Paths are absolute and separated by a newline, because a newline is the only character that +cannot appear in a filename on any supported OS. The whole list arrives as a single +argument, so no quoting is needed in your template. + +When one download writes several outputs (`-q 1080,720`), each output's folder gets its own +copy of the sidecar files, and each invocation lists the copies sitting beside its own +`{filepath}`. + +```python title="Reading {sidecars} in Python" +subs = [p for p in sys.argv[i].removeprefix("--subs=").split("\n") if p] +``` + +## Modes { #modes } + +| `mode` | Runs | `{filepath}` | `{sidecars}` | +| --- | --- | --- | --- | +| `file` | Once per output file | That file | That output's sidecars | +| `season` | Once per season folder, after that season's last queued title | Empty | Empty | +| `run` | Once per folder written, at the end of the run | Empty | Empty | + +!!! warning "`season` means everything this run asked for" + A `season` hook fires when the last title **this run queued** for that season finishes. + With `-w S01-S02` the S01 hook runs while S02 is still downloading. It says nothing + about whether the season is complete on disk. Downloading a single episode fires the + hook for that season as soon as the episode lands. + + A title that fails takes its group with it. If one episode of S01 fails, the S01 hook + does not fire at all, because the run never finished everything it queued for that + season. + +The hook is keyed to the folder, not to the season, so one season landing in several +folders (`-q 1080,720` with a quality-dependent folder template) fires it once per folder, +each with its own `{folder}`. A movie is a group of its own: a `season` hook fires for a +movie's folder as soon as the movie finishes. + +In `run` mode every variable except `{folder}` is empty, because one run can cover several +shows. Only `{folder}` is meaningful, which makes it useful for starting a library scan +once. + +### Music + +Music uses the same three modes, with the album standing in for the season. + +| `mode` | Runs | +| --- | --- | +| `file` | Once per track, after its metadata has been written | +| `season` | Once per album folder, after every queued track of that album has landed | +| `run` | Once per folder written, at the end of the run | + +In a `file` hook `{title}` and `{title_raw}` are the track name. In an album hook they are +the album name, and the per-track variables `{track_number}`, `{disc}` and `{isrc}` are +empty, in the same way that `{episode}` is empty in a season hook. + +## Events { #events } + +`success` runs after the file has moved to its final path. `failure` runs when the download +fails, with `{filepath}` empty and `{error}` set. A music `failure` hook describes the +release rather than a track, because the whole album stops when one track fails. + +!!! warning "`failure` only fires in `file` mode" + A `failure` entry with `mode: season` or `mode: run` never runs. Leave `mode` at its + default on a failure entry. + +A `failure` hook has no output file to read metadata from, so it carries only what the +title object already knew: `{title}`, `{title_raw}`, `{title_id}`, `{year}`, `{season}`, +`{episode}`, `{episode_name}`, `{service}`, the tagging IDs and `{error}`. Everything that +comes from the naming context, such as `{quality}`, `{hdr}`, `{artist}` and `{album}`, is +empty. + +A resumed `--continue` download that finishes runs `success` as usual. Because unshackle +does not wait for your script, it never sees its exit code, so a failing script is not +reported anywhere. Your script owns its own error handling and logging. + +A `success` hook always names a file this run wrote. There is no skip-if-exists path in +`dl` today, so nothing fires for a title that was not downloaded. + +## `--postscript` + +`dl` accepts `--postscript ""`, repeatable. Every command given this way runs with +`event: success` and `mode: file`. + +```console +$ unshackle dl --postscript "python /opt/upload.py {filepath}" SERVICE TITLE_ID +``` + +It replaces the `post_scripts` config for that run rather than adding to it, so no +configured entry runs while it is in use. That includes your `season`, `run` and `failure` +entries, which have no `--postscript` equivalent. + +## The REST API + +The API never accepts a post-script command. `postscript`, `post_script` and `post_scripts` +in a `POST /api/download` body are rejected with `400 INVALID_PARAMETERS`, because a command +arriving from an HTTP caller would be remote code execution. + +Scripts defined in `unshackle.yaml` do run for API jobs, music jobs included. Only the +command itself may not cross the network. + +## Logging + +The command that was run is logged twice: at `DEBUG` level on the console, and as a +`post_script_dispatch` entry in the +[structured JSON debug log](../../guide/troubleshooting.md#the-structured-json-debug-log) +when that log is enabled. + +Both are masked the same way, honouring [`redact_paths`](misc.md), which is on by default. +Debug logs get shared in issue reports, and a hook command line is mostly paths. URL +credentials and `token=` / `api_key=` query parameters are also masked. + +!!! warning "Do not hardcode secrets in the command" + An arbitrary secret in the command line, such as `-H "Authorization: Bearer "`, + cannot be recognised and is logged verbatim. Read the secret from a file or an + environment variable inside your script rather than putting it in the `command`. + +## Limitations + +- No `success` hooks run under `--no-mux` or `--skip-dl`, because neither writes a muxed + output. A `failure` hook still runs under `--no-mux` if the download itself fails. +- `failure` hooks only run in `file` mode. +- There is no webhook sender. Call `curl` from a script instead. +- Script output is not captured. Redirect it inside your own script if you want a log. diff --git a/mkdocs.yml b/mkdocs.yml index 8fa8d712..d796e069 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -128,6 +128,7 @@ nav: - DRM & CDM: reference/configuration/drm.md - Network & Proxy: reference/configuration/network.md - Key Vaults: reference/configuration/vaults.md + - Post-Download Scripts: reference/configuration/post-scripts.md - Other Settings: reference/configuration/misc.md - Annotated Example: reference/configuration/example.md - Developer: diff --git a/tests/test_post_scripts.py b/tests/test_post_scripts.py new file mode 100644 index 00000000..94a15cbd --- /dev/null +++ b/tests/test_post_scripts.py @@ -0,0 +1,198 @@ +"""Post-script command building: the injection guard, and how variables become argv.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from unshackle.core.config import config +from unshackle.core.utils.post_scripts import ( + SIDECAR_SEPARATOR, + _entries, + build_context, + dispatch, + substitute, + tokenize, +) + + +def build(command: str, context: dict[str, str]) -> list[str]: + return substitute(tokenize(command), context) + + +def test_metacharacters_in_a_value_stay_one_argument(): + """The whole point of tokenizing before substituting: a title can't become a command.""" + argv = build( + "upload.py {filepath} --title={title}", + { + "filepath": "/downloads/Bob's Burgers S01E01.mkv", + "title": 'Bob"; rm -rf ~', + }, + ) + assert argv == ["upload.py", "/downloads/Bob's Burgers S01E01.mkv", '--title=Bob"; rm -rf ~'] + + +def test_empty_variable_substitutes_empty(): + argv = build("upload.py {filepath} --tmdb={tmdb}", {"filepath": "/x.mkv", "tmdb": ""}) + assert argv == ["upload.py", "/x.mkv", "--tmdb="] + + +def test_sidecars_join_as_a_single_token(): + subs = [Path("/dl/Show S01E01.en.srt"), Path("/dl/Show S01E01.fr.srt")] + argv = build("upload.py --subs={sidecars}", {"sidecars": SIDECAR_SEPARATOR.join(map(str, subs))}) + assert len(argv) == 2 + assert argv[1].removeprefix("--subs=").split(SIDECAR_SEPARATOR) == [str(p) for p in subs] + + +@pytest.mark.parametrize( + "hostile_value", + ["--upload-file=/etc/passwd", "-rf", "--output=/etc/cron.d/x"], +) +def test_bare_variable_expanding_to_a_flag_is_refused(caplog, hostile_value): + """A service-controlled value that becomes a whole -flag must not run the command.""" + dispatch("success", "file", {"title_raw": hostile_value}, ["curl example.com {title_raw}"]) + assert "option-like token" in caplog.text + + +def test_flag_prefixed_variable_is_not_refused(caplog): + """--opt={var} is the user's own flag; the value in the middle can never forge one.""" + calls = [] + import unshackle.core.utils.post_scripts as ps + + original = ps.subprocess.Popen + ps.subprocess.Popen = lambda argv, **kw: calls.append(argv) or original(["true"]) + try: + dispatch("success", "file", {"title": "--evil=1"}, ["echo --title={title}"]) + finally: + ps.subprocess.Popen = original + assert calls and calls[0] == ["echo", "--title=--evil=1"] + assert "option-like token" not in caplog.text + + +def test_unknown_variable_becomes_empty(caplog): + argv = build("upload.py --x={nope}", {}) + assert argv == ["upload.py", "--x="] + assert "nope" in caplog.text + + +@pytest.mark.parametrize( + "command,expected", + [ + ('python "/opt/my scripts/upload.py" {filepath}', ["python", "/opt/my scripts/upload.py", "/x.mkv"]), + ("python upload.py {filepath}", ["python", "upload.py", "/x.mkv"]), + ], +) +def test_quoted_paths_in_the_template_survive_tokenizing(command, expected): + assert build(command, {"filepath": "/x.mkv"}) == expected + + +@pytest.mark.parametrize( + "command,expected", + [ + (r"python C:\Scripts\up.py {filepath}", ["python", "C:\\Scripts\\up.py", "/x.mkv"]), + (r'python "C:\My Scripts\up.py" {filepath}', ["python", "C:\\My Scripts\\up.py", "/x.mkv"]), + ('python up.py --title="{title}"', ["python", "up.py", "--title=T"]), + ], +) +def test_windows_tokenizing_matches_posix_semantics(monkeypatch, command, expected): + import os + + monkeypatch.setattr(os, "name", "nt") + assert build(command, {"filepath": "/x.mkv", "title": "T"}) == expected + + +def test_control_chars_stripped_so_sidecar_newline_separator_holds(): + """A service title with a newline must not survive into a filename and forge a {sidecars} entry.""" + from unshackle.core.utilities import sanitize_filename + + assert "\n" not in sanitize_filename("Ep\n/etc/cron.d/x", " ") + assert "\t" not in sanitize_filename("Ep\tfoo", " ") + + +def test_failure_context_still_has_title_and_year(): + """A failure dispatch has no MediaInfo, but a notifier's --title={title} must not go empty.""" + title = SimpleNamespace(title="Bob's Burgers", name="Ep", year=2011, season=1, number=2, id="x") + context = build_context(title, None, service="SVC", error="Boom") + assert context["title"] == "Bob's Burgers" + assert context["year"] == "2011" + assert context["error"] == "Boom" + assert context["filepath"] == "" + + +def test_success_context_prefers_the_template_title(): + title = SimpleNamespace( + title="Raw $how", + name="Ep", + year=2011, + season=1, + number=2, + id="x", + build_template_context=lambda media_info, show_service: {"title": "From Template", "year": 2020}, + ) + context = build_context(title, media_info=object(), service="SVC") + assert context["title"] == "From Template" + assert context["year"] == "2020" + assert context["title_raw"] == "Raw $how" + + +def test_dispatch_survives_an_unbalanced_quote(caplog): + """dispatch never raises: a malformed template is skipped with a warning, not a crash.""" + dispatch("success", "file", {}, ['python "unclosed']) + assert "malformed" in caplog.text + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("python x.py {filepath}", ["python x.py {filepath}"]), # lone string, not a list + ({"command": "python x.py"}, ["python x.py"]), # lone mapping, not a list + (42, []), # nonsense is ignored, never iterated + ([42, None, "ok.py"], ["ok.py"]), # junk entries are dropped, valid ones survive + ], +) +def test_malformed_post_scripts_config_shapes(monkeypatch, raw, expected): + monkeypatch.setattr(config, "post_scripts", raw) + assert list(_entries("success", "file")) == expected + + +def test_typoed_event_or_mode_warns_instead_of_silently_dropping(monkeypatch, caplog): + """Fire-and-forget gives no other feedback, so a dead entry must at least warn.""" + monkeypatch.setattr( + config, + "post_scripts", + [{"command": "typo.py", "mode": "seson"}, {"command": "ok.py", "mode": "season"}], + ) + for dispatch_mode in ("file", "season", "run"): + assert list(_entries("success", dispatch_mode)) == (["ok.py"] if dispatch_mode == "season" else []) + assert caplog.text.count("unknown event/mode") == 1 # warned once, not per dispatch + + +def test_tagging_ids_are_read_at_dispatch_not_snapshotted(): + """A series' TMDB ID is resolved inside the title loop, after any pre-loop snapshot. + + Guards a regression where ``post_script_ids`` was a dict built before the loop, so + ``{tmdb}`` was empty for every download that did not pass ``--tmdb`` by hand. + """ + import ast + import inspect + import textwrap + + from unshackle.commands.dl import dl + + holder = SimpleNamespace(tmdb_id=None, imdb_id="tt0000001", tvdb_id=None) + assert build_context(SimpleNamespace(), ids=dl.post_script_ids(holder))["tmdb"] == "" + holder.tmdb_id = 1396 + assert build_context(SimpleNamespace(), ids=dl.post_script_ids(holder))["tmdb"] == "1396" + + body = ast.parse(textwrap.dedent(inspect.getsource(dl.result))) + for node in ast.walk(body): + targets = getattr(node, "targets", []) + assert not any(isinstance(t, ast.Name) and t.id == "post_script_ids" for t in targets), ( + "post_script_ids is snapshotted again; it must be read per dispatch" + ) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/test_post_scripts_music.py b/tests/test_post_scripts_music.py new file mode 100644 index 00000000..426fa810 --- /dev/null +++ b/tests/test_post_scripts_music.py @@ -0,0 +1,232 @@ +"""Music post-script hooks, which no live service was available to test against. + +Music returns from ``dl.result`` before the title loop, so its hooks are dispatched inside +the music pipeline. These check the pieces that pipeline feeds them: that a Song builds a +usable context, that an Album groups the way a season does, and that the album and run +contexts blank the file-level variables. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from unshackle.core.utils.post_scripts import build_context, dispatch, substitute, tokenize + + +class FakeSong(SimpleNamespace): + """A Song as the hook sees it: name/artist/album, no season or episode.""" + + def build_template_context(self, media_info, show_service=True): + return { + "title": self.name, + "artist": self.artist, + "album": self.album, + "track_number": f"{self.track:02}", + "year": self.year, + "audio": "FLAC", + "video": "", + } + + +def song(**kwargs): + defaults = dict( + name="Blue Monday", artist="New Order", album="Power, Corruption & Lies", track=4, year=1983, id="s1" + ) + return FakeSong(**{**defaults, "id": "s1", **kwargs}) + + +def test_song_context_has_music_fields_and_blank_episode_fields(): + context = build_context( + song(), + media_info=object(), + filepath=Path("/dl/New Order/04. Blue Monday.flac"), + service="EXAMPLE", + ) + assert context["title"] == "Blue Monday" + assert context["artist"] == "New Order" + assert context["album"] == "Power, Corruption & Lies" + assert context["track_number"] == "04" + assert context["service"] == "EXAMPLE" + assert context["filename"] == "04. Blue Monday.flac" + assert context["ext"] == ".flac" + assert context["folder"] == "/dl/New Order" + assert context["season"] == "" + assert context["episode"] == "" + + +def test_song_context_survives_without_media_info(): + """The failure path and any pre-mux dispatch pass no MediaInfo.""" + context = build_context(song(), None, service="EXAMPLE", error="HTTPError: 403") + assert context["title"] == "Blue Monday" + assert context["error"] == "HTTPError: 403" + assert context["filepath"] == "" + + +def test_album_grouping_matches_the_season_grouping_rule(): + """post_script_group's music branch: albums group by (album artist, album).""" + from unshackle.commands.dl import dl # noqa: F401 - import guard, the grouping is inlined in result() + + def group(candidate): + artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "") + return ("album", artist, getattr(candidate, "album", "")) + + a = song(track=1) + b = song(track=2) + c = song(album="Technique") + assert group(a) == group(b) + assert group(a) != group(c) + + +@pytest.mark.parametrize("mode", ["season", "run"]) +def test_album_and_run_contexts_blank_the_file_variables(mode): + """Mirrors the blanking dl.py applies before dispatching an album or run hook.""" + base = build_context( + song(), + media_info=object(), + filepath=Path("/dl/New Order/04. Blue Monday.flac"), + service="EXAMPLE", + ) + context = dict(base) + if mode == "season": + for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"): + context[key] = "" + else: + context = dict.fromkeys(base, "") + context["folder"] = "/dl/New Order" + + assert context["filepath"] == "" + assert context["sidecars"] == "" + assert context["folder"] == "/dl/New Order" + if mode == "run": + assert context["title"] == "" + assert set(context) == set(base) + else: + assert context["album"] == "Power, Corruption & Lies" + + +def test_a_music_hook_command_builds_the_argv_a_script_would_get(monkeypatch, tmp_path): + """End to end through the real dispatcher: config entry in, argv out.""" + from unshackle.core.config import config + + marker = tmp_path / "argv.txt" + script = tmp_path / "hook.py" + script.write_text("import sys, pathlib\npathlib.Path(sys.argv[1]).write_text(chr(10).join(sys.argv[2:]))\n") + + import sys + + command = ( + f'"{sys.executable}" "{script}" "{marker}" ' + "--album={album} --artist={artist} --track={track_number} --file={filepath}" + ) + monkeypatch.setattr(config, "post_scripts", [{"command": command, "mode": "file", "event": "success"}]) + + context = build_context( + song(), + media_info=object(), + filepath=Path("/dl/New Order/04. Blue Monday.flac"), + service="EXAMPLE", + ) + dispatch("success", "file", context) + + for _ in range(100): + if marker.exists(): + break + import time + + time.sleep(0.05) + assert marker.exists(), "the hook process never ran" + argv = marker.read_text().splitlines() + assert "--album=Power, Corruption & Lies" in argv + assert "--artist=New Order" in argv + assert "--track=04" in argv + assert "--file=/dl/New Order/04. Blue Monday.flac" in argv + + +def test_album_hook_mapping_is_bound_before_any_early_branch(): + """--skip-dl skips the final-move block; the album hook still runs after it. + + Guards a NameError: the mapping used to be created inside that block, so a music + download with --skip-dl reached the album hook with an unbound name. + """ + import inspect + + from unshackle.commands.dl import dl + + source = inspect.getsource(dl.result) + body = source[source.index("def download_music_title") :] + bind = body.index("album_post_script: dict") + for use in ("album_post_script[final_path.parent]", "album_post_script.items()"): + assert bind < body.index(use), f"{use} can run before the mapping is bound" + line = body[:bind].rsplit("\n", 1)[-1] + assert len(line) == 16, f"bound at indent {len(line)}, expected the function body level" + + +def test_album_hook_identifies_the_album_not_the_last_track(): + """A Song's ``{title}`` is the song, so blanking only the video fields is not enough. + + Guards a regression where an album hook was handed whichever track landed last: + ``{title}`` was a song name and ``{track_number}``/``{isrc}`` were that track's. + """ + from unshackle.commands.dl import dl + + context = build_context( + song(track=11, isrc="GBAAA8300001"), + media_info=object(), + filepath=Path("/dl/New Order - PCL (1983)/11. Blue Monday.flac"), + service="EXAMPLE", + ) + context.setdefault("isrc", "GBAAA8300001") + + album = dict(context) + for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"): + album[key] = "" + for key in ("track_number", "disc", "isrc"): + if key in album: + album[key] = "" + album["title"] = album["title_raw"] = album.get("album", "") + + assert album["title"] == "Power, Corruption & Lies" + assert album["title_raw"] == "Power, Corruption & Lies" + assert album["track_number"] == "" + assert album["isrc"] == "" + assert album["artist"] == "New Order" # album-level, kept + + import inspect + + body = inspect.getsource(dl.result) + body = body[body.index("for folder, context in album_post_script.items()") :][:900] + for key in ("track_number", "disc", "isrc"): + assert f'"{key}"' in body, f"the album hook no longer blanks {key}" + assert 'album_context["title"]' in body + + +def test_a_music_download_failure_dispatches_a_failure_hook(): + """Parity with the video worker, which dispatches failure/file when it raises.""" + import inspect + + from unshackle.commands.dl import dl + + body = inspect.getsource(dl.result) + body = body[body.index("def download_music_title") :] + assert body.count("music_failure(") == 3, "expected one definition and both failure exits" + assert body.index("def music_failure") < body.index("music_failure(e)") + assert "music_failure(error)" in body + + release = SimpleNamespace(title="Power, Corruption & Lies", artist="New Order", year=1983) + context = build_context(release, service="EXAMPLE", error="HTTPError: 403") + assert context["title"] == "Power, Corruption & Lies" + assert context["error"] == "HTTPError: 403" + assert context["filepath"] == "" + + +def test_an_album_name_with_a_comma_stays_one_argument(): + """'Power, Corruption & Lies' must not split, which is the whole tokenize-first point.""" + argv = substitute(tokenize("upload.py --album={album}"), {"album": "Power, Corruption & Lies"}) + assert argv == ["upload.py", "--album=Power, Corruption & Lies"] + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 3a672372..d7de4b31 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -22,7 +22,7 @@ from http.cookiejar import CookieJar, MozillaCookieJar from itertools import product from pathlib import Path from threading import Lock -from typing import Any, Callable, Optional, TypedDict, Union +from typing import Any, Callable, Optional, Sequence, TypedDict, Union from uuid import UUID import click @@ -107,6 +107,7 @@ from unshackle.core.utils.click_types import ( SubtitleCodecChoice, ) from unshackle.core.utils.collections import ci_get, merge_dict +from unshackle.core.utils.post_scripts import build_context, dispatch from unshackle.core.utils.selector import select_multiple from unshackle.core.utils.subprocess import ffprobe from unshackle.core.vaults import Vaults @@ -466,6 +467,14 @@ class dl: return created_paths + def post_script_ids(self) -> dict[str, Any]: + """Tagging IDs as they stand right now. + + Read per dispatch, never snapshotted: an Episode's TMDB ID is resolved by the + search inside the title loop, so a snapshot taken before it is always empty. + """ + return {"tmdb": self.tmdb_id, "imdb": self.imdb_id, "tvdb": self.tvdb_id} + @click.command( short_help="Download, Decrypt, and Mux tracks for titles from a Service.", cls=Services, @@ -791,6 +800,17 @@ class dl: "--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path." ) @click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.") + @click.option( + "--postscript", + "postscript", + type=str, + multiple=True, + metavar="COMMAND", + help=( + "Run COMMAND after each downloaded file, with {filepath}, {title}, {season} and other " + "variables substituted. Repeatable. Replaces the post_scripts config for this run." + ), + ) @click.option( "--workers", type=int, @@ -1423,6 +1443,7 @@ class dl: real_video_bitrate: bool = False, real_audio_bitrate: bool = False, progress_sink: Optional[Callable[[dict[str, Any]], None]] = None, + postscript: Sequence[str] = (), *_: Any, **__: Any, ) -> None: @@ -1714,6 +1735,7 @@ class dl: ) music_titles = list(titles) if music_collection_mode else ([titles] if music_mode else []) music_plans = {} + music_post_script: dict[Path, dict[str, str]] = {} if music_titles: music_renderer = MusicRenderer() @@ -1907,6 +1929,22 @@ class dl: if music_group_download: def download_music_title(titles: Music, plan: Any) -> bool: + album_post_script: dict[Path, dict[str, str]] = {} + + def music_failure(error: BaseException) -> None: + """The music twin of the video worker's failure hook, keyed to the release.""" + dispatch( + "failure", + "file", + build_context( + titles, + service=self.service, + ids=self.post_script_ids(), + error=f"{type(error).__name__}: {error}", + ), + postscript, + ) + music_items: list[tuple[Song, Audio, Callable[..., None]]] = [] music_song_plans = {id(song_plan.song): song_plan for disc in plan.discs for song_plan in disc.songs} music_renderer = MusicRenderer() @@ -2146,6 +2184,7 @@ class dl: ) ) console.print_exception() + music_failure(e) return if skip_dl: @@ -2182,6 +2221,7 @@ class dl: ) except MusicAudioIntegrityError as error: console.print(Padding(f"Audio integrity failed: {error}", (0, 5, 1, 5))) + music_failure(error) return integrity_time = format_elapsed_seconds(time.time() - integrity_start) @@ -2276,6 +2316,16 @@ class dl: } ) + song_context = build_context( + song, + media_infos.get(id(track)), + filepath=final_path, + service=self.service, + ids=self.post_script_ids(), + ) + dispatch("success", "file", song_context, postscript) + album_post_script[final_path.parent] = song_context + if final_paths: manifest_slug = ".".join( part @@ -2347,6 +2397,18 @@ class dl: Padding(f"{release_label} downloaded in [progress.elapsed]{album_time}[/]!", (0, 5, 1, 5)) ) + for folder, context in album_post_script.items(): + album_context = dict(context) + for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"): + album_context[key] = "" + for key in ("track_number", "disc", "isrc"): + if key in album_context: + album_context[key] = "" + album_context["title"] = album_context["title_raw"] = album_context.get("album", "") + album_context["folder"] = str(folder) + dispatch("success", "season", album_context, postscript) + music_post_script.update(album_post_script) + return True for music_title in music_titles: @@ -2354,6 +2416,12 @@ class dl: if not download_music_title(music_title, current_plan): return + if music_post_script: + run_context = dict.fromkeys(next(iter(music_post_script.values())), "") + for folder in music_post_script: + run_context["folder"] = str(folder) + dispatch("success", "run", run_context, postscript) + if not hasattr(service, "close"): cookie_file = self.get_cookie_path(self.service, self.profile) if cookie_file: @@ -2371,6 +2439,32 @@ class dl: base_selection = (v_lang, a_lang, s_lang, range_) + def post_script_group(candidate: Any) -> Any: + """Key a title by the folder its outputs share, for post-scripts in season mode.""" + if isinstance(candidate, Episode): + return ("episode", candidate.title, candidate.season) + if isinstance(candidate, (Song, Music)): + artist = getattr(candidate, "album_artist", None) or getattr(candidate, "artist", "") + return ("album", artist, getattr(candidate, "album", "")) + return ("title", id(candidate)) + + def post_script_queued(candidate: Any) -> bool: + """Mirror of the filters the loop below applies, so the season counter matches it.""" + if isinstance(candidate, Episode) and latest_episode and latest_episode_id: + return f"{candidate.season}x{candidate.number}" == latest_episode_id + if isinstance(candidate, Episode) and wanted: + return bool(candidate.matches_wanted(wanted)) + return True + + post_script_pending: dict[Any, int] = {} + for candidate in titles: + if post_script_queued(candidate): + key = post_script_group(candidate) + post_script_pending[key] = post_script_pending.get(key, 0) + 1 + post_script_last: dict[Any, dict[Path, dict[str, str]]] = {} + post_script_folders: list[Path] = [] + post_script_sample: dict[str, str] = {} + for i, title in enumerate(titles): v_lang, a_lang, s_lang, range_ = base_selection if isinstance(title, Episode) and latest_episode and latest_episode_id: @@ -3376,6 +3470,18 @@ class dl: "returncode": getattr(e, "returncode", None), }, ) + + dispatch( + "failure", + "file", + build_context( + title, + service=self.service, + ids=self.post_script_ids(), + error=f"{type(e).__name__}: {e}", + ), + postscript, + ) return if skip_dl: @@ -3389,6 +3495,7 @@ class dl: skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) sidecar_subtitles: list[Subtitle] = [] sidecar_original_paths: dict[str, Path] = {} + sidecar_files: dict[Optional[Path], list[Path]] = {} if subtitle_output_mode in ("sidecar", "both") and not no_mux: sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()] if sidecar_format == "original": @@ -3750,37 +3857,54 @@ class dl: sys.exit(1) if sidecar_subtitles and not no_mux: - media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None - if media_info: - base_filename = title.get_filename(media_info, show_service=not no_source) - else: - base_filename = str(title) + sidecar_targets: list[tuple[Optional[Path], str, Path]] = [] + for muxed_path in muxed_paths or [None]: + media_info = MediaInfo.parse(muxed_path) if muxed_path else None + if media_info: + base_filename = title.get_filename(media_info, show_service=not no_source) + else: + base_filename = str(title) - sidecar_dir = self.output_dir or config.directories.downloads - if ( - not no_folder - and media_info - and ( - isinstance(title, (Episode, Song)) - or (isinstance(title, Movie) and config.get_folder_template("movies")) - ) - ): - sidecar_dir /= title.get_filename( - media_info, show_service=not no_source, folder=True - ) - sidecar_dir.mkdir(parents=True, exist_ok=True) + sidecar_dir = self.output_dir or config.directories.downloads + if ( + not no_folder + and media_info + and ( + isinstance(title, (Episode, Song)) + or (isinstance(title, Movie) and config.get_folder_template("movies")) + ) + ): + sidecar_dir /= title.get_filename( + media_info, show_service=not no_source, folder=True + ) + sidecar_targets.append((muxed_path, base_filename, sidecar_dir)) with console.status("Saving subtitle sidecar files..."): + first_path, first_base, first_dir = sidecar_targets[0] + first_dir.mkdir(parents=True, exist_ok=True) created = self.output_subtitle_sidecars( sidecar_subtitles, - base_filename, - sidecar_dir, + first_base, + first_dir, sidecar_format, original_paths=sidecar_original_paths or None, ) if created: + sidecar_files[first_path] = list(created) self.log.info(f"Saved {len(created)} sidecar subtitle files") + for muxed_path, base_filename, sidecar_dir in sidecar_targets[1:]: + sidecar_dir.mkdir(parents=True, exist_ok=True) + copies = [] + for source in created: + target = sidecar_dir / f"{base_filename}{source.name[len(first_base) :]}" + if target != source: + shutil.copy2(source, target) + copies.append(target) + if copies: + sidecar_files[muxed_path] = copies + self.log.info(f"Saved {len(copies)} sidecar subtitle files") + for track in title.tracks: track.delete() @@ -3910,6 +4034,33 @@ class dl: self.anime_hint(title), ) + post_script_context = build_context( + title, + media_info, + filepath=final_path, + sidecars=sidecar_files.get(muxed_path, []), + service=self.service, + ids=self.post_script_ids(), + ) + dispatch("success", "file", post_script_context, postscript) + post_script_last.setdefault(post_script_group(title), {})[final_path.parent] = ( + post_script_context + ) + post_script_sample = post_script_context + if final_path.parent not in post_script_folders: + post_script_folders.append(final_path.parent) + + if not no_mux: + group_key = post_script_group(title) + post_script_pending[group_key] = post_script_pending.get(group_key, 1) - 1 + if post_script_pending[group_key] <= 0: + for folder, context in post_script_last.get(group_key, {}).items(): + season_context = dict(context) + for key in ("filepath", "filename", "ext", "sidecars", "episode", "episode_name"): + season_context[key] = "" + season_context["folder"] = str(folder) + dispatch("success", "season", season_context, postscript) + title_dl_time = time_elapsed_since(dl_start_time) downloaded_label = "Track" if music_mode and isinstance(title, Song) else "Title" console.print( @@ -3930,6 +4081,12 @@ class dl: if hasattr(service, "close"): service.close() + if post_script_folders and post_script_sample: + run_context = dict.fromkeys(post_script_sample, "") + for folder in post_script_folders: + run_context["folder"] = str(folder) + dispatch("success", "run", run_context, postscript) + dl_time = time_elapsed_since(start_time) console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index affc1716..ac9c9cad 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -360,6 +360,10 @@ def perform_download( ] config.directories.cache = config.directories.cache / "_jobs" / cred_hash + for banned in ("postscript", "post_script", "post_scripts"): + if params.pop(banned, None) is not None: + log.warning(f"Ignoring '{banned}' in API download parameters; post-scripts are config-only") + vcodec_raw = params.get("vcodec") if vcodec_raw: if isinstance(vcodec_raw, str): diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 834e2744..942e78f2 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -942,6 +942,10 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: Returns: None if valid, error message string if invalid """ + for banned in ("postscript", "post_script", "post_scripts"): + if data.get(banned): + return f"'{banned}' is not accepted over the API. Define post_scripts in unshackle.yaml." + if "vcodec" in data and data["vcodec"]: err = check_codec(data["vcodec"], VALID_VCODECS, "vcodec") if err: diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 12d8ee0b..a7165c7b 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -594,7 +594,10 @@ async def download(request: web.Request) -> web.Response: Download content based on provided parameters. --- summary: Download content - description: Download video content based on specified parameters + description: >- + Download video content based on specified parameters. + 'postscript', 'post_script' and 'post_scripts' are never accepted and return 400; + post-download hooks are configured only in unshackle.yaml. requestBody: required: true content: diff --git a/unshackle/core/config.py b/unshackle/core/config.py index ac8112af..6baf329e 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -131,6 +131,7 @@ class Config: # mask local base dirs (install root/venv/home) in logged paths; False shows full paths self.redact_paths: bool = kwargs.get("redact_paths", True) self.continue_downloads: bool = kwargs.get("continue_downloads", False) + self.post_scripts: list = kwargs.get("post_scripts") or [] self.language_tags: dict = kwargs.get("language_tags") or {} self.tag_rules: list = kwargs.get("tag_rules") or [] diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 38dffeab..b953e4ea 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -132,7 +132,7 @@ def sanitize_filename(filename: str, spacer: str = ".") -> str: filename = unidecode(filename) filename = re.sub(r"\[\(+", "[", filename) filename = re.sub(r"\)+\]", "]", filename) - filename = "".join(c for c in filename if unicodedata.category(c) != "Mn") # hidden characters + filename = "".join(c for c in filename if unicodedata.category(c) not in ("Mn", "Cc")) filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames if spacer == ".": filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern) diff --git a/unshackle/core/utils/post_scripts.py b/unshackle/core/utils/post_scripts.py new file mode 100644 index 00000000..47ce5cc9 --- /dev/null +++ b/unshackle/core/utils/post_scripts.py @@ -0,0 +1,216 @@ +"""Run user-defined scripts after a download completes. + +A command is tokenized once, then variables are substituted into the resulting argv +tokens and the process is spawned with ``shell=False``. Substituting after tokenizing +is what makes a title like ``Bob"; rm -rf ~`` harmless: a value can never become a new +token or a shell operator. The cost is that the command has no shell features and no +interpreter lookup, so users name the interpreter themselves (``python upload.py``). + +Scripts are fire-and-forget: unshackle spawns them, logs the command, and moves on. It +does not wait, capture output, time them out, or let a script's exit code change its own. +""" + +from __future__ import annotations + +import logging +import os +import re +import shlex +import subprocess +from pathlib import Path +from typing import Any, Iterator, Optional, Sequence + +from unshackle.core.config import config +from unshackle.core.utilities import log_event +from unshackle.core.utils.redact import redact_path, redact_text + +log = logging.getLogger("post-script") + +VARIABLE_RE = re.compile(r"\{([a-z_]+)\}") + +EVENTS = ("success", "failure") +MODES = ("file", "season", "run") + +SIDECAR_SEPARATOR = "\n" + +_warned_entries: set[str] = set() + + +def tokenize(command: str) -> list[str]: + """Split a command string into argv tokens, before any variable is substituted.""" + if os.name == "nt": + + def unquote(token: str) -> str: + if len(token) > 1 and token[0] == token[-1] and token[0] in "\"'": + return token[1:-1] + match = re.match(r"^([^\"']*=)([\"'])(.*)\2$", token) + return f"{match.group(1)}{match.group(3)}" if match else token + + return [unquote(token) for token in shlex.split(command, posix=False)] + return shlex.split(command) + + +def substitute(tokens: Sequence[str], context: dict[str, str]) -> list[str]: + """Replace every ``{variable}`` inside each token; unknown or empty ones become ''.""" + unknown: set[str] = set() + + def replace(match: re.Match[str]) -> str: + name = match.group(1) + if name not in context: + unknown.add(name) + return context.get(name, "") + + argv = [VARIABLE_RE.sub(replace, token) for token in tokens] + if unknown: + log.warning("Unknown post-script variables treated as empty: %s", ", ".join(sorted(unknown))) + return argv + + +def _data_injected_flag(tokens: Sequence[str], argv: Sequence[str]) -> Optional[str]: + """A token that only became a ``-flag`` after substitution. + + Tokenizing before substituting stops a value becoming a *new* token, but a bare + ``{title}`` token whose value is service-controlled (``--upload-file=x``) still becomes a + whole option token to the user's script. Flag it when the leading ``-`` came from the + value, not the template. A literal flag like ``--opt={var}`` starts with ``-`` in the + template itself and is the user's own, so it is never flagged. + """ + for template, token in zip(tokens, argv): + if token.startswith("-") and not template.startswith("-"): + return token + return None + + +def build_context( + title: Any, + media_info: Optional[Any] = None, + *, + filepath: Optional[Path] = None, + folder: Optional[Path] = None, + sidecars: Sequence[Path] = (), + service: str = "", + ids: Optional[dict[str, Any]] = None, + error: str = "", +) -> dict[str, str]: + """Build the variable set for one output file. + + Metadata comes from the same naming context that produced the file's name, so + ``{quality}`` and ``{hdr}`` always describe *this* file. That is what lets + ``-q 1080,2160 -r HDR10,SDR`` label its four outputs correctly. Season and episode + are handed over as plain numbers rather than the filename's ``S01E05`` form: the + spacer and padding belong to the filename template, not to the data. + """ + context: dict[str, str] = {} + + if media_info is not None: + try: + for key, value in title.build_template_context(media_info, show_service=True).items(): + context[key] = "" if value is None else str(value) + except Exception as e: # noqa: BLE001 - a naming quirk must never stop the download + log.debug("Could not build post-script metadata context: %s", e) + + context["vcodec"] = context.get("video", "") + context["acodec"] = context.get("audio", "") + + season = getattr(title, "season", None) + number = getattr(title, "number", None) + context["season"] = "" if season is None else str(season) + context["episode"] = "" if number is None else str(number) + context["episode_name"] = str(getattr(title, "name", "") or "") if season is not None else "" + context["title_raw"] = str(getattr(title, "title", None) or getattr(title, "name", "") or "") + context.setdefault("title", context["title_raw"].replace("$", "S")) + context.setdefault("year", str(getattr(title, "year", "") or "")) + context["title_id"] = str(getattr(title, "id", "") or "") + context["service"] = service + + context["filepath"] = str(filepath) if filepath else "" + context["filename"] = filepath.name if filepath else "" + context["ext"] = filepath.suffix if filepath else "" + context["folder"] = str(folder or (filepath.parent if filepath else "")) + context["sidecars"] = SIDECAR_SEPARATOR.join(str(path) for path in sidecars) + + ids = ids or {} + for name in ("tmdb", "imdb", "tvdb"): + value = ids.get(name) + context[name] = "" if value is None else str(value) + + context["error"] = error + return context + + +def _entries(event: str, mode: str, overrides: Sequence[str] = ()) -> Iterator[str]: + """Commands configured for this event and mode; --postscript replaces the config list.""" + if overrides: + if event == "success" and mode == "file": + yield from overrides + return + + raw = config.post_scripts + if isinstance(raw, (str, dict)): + raw = [raw] + elif not isinstance(raw, list): + if raw: + log.warning("Ignoring malformed post_scripts config: %r", raw) + raw = [] + for entry in raw: + if isinstance(entry, str): + entry = {"command": entry} + if not isinstance(entry, dict): + log.warning("Ignoring malformed post_scripts entry: %r", entry) + continue + command = str(entry.get("command") or "").strip() + if not command: + continue + entry_event = str(entry.get("event") or "success") + entry_mode = str(entry.get("mode") or "file") + if entry_event not in EVENTS or entry_mode not in MODES: + if command not in _warned_entries: + _warned_entries.add(command) + log.warning( + "Ignoring post_scripts entry with unknown event/mode %r/%r (valid: %s / %s)", + entry_event, + entry_mode, + "|".join(EVENTS), + "|".join(MODES), + ) + continue + if entry_event != event or entry_mode != mode: + continue + yield command + + +def dispatch(event: str, mode: str, context: dict[str, str], overrides: Sequence[str] = ()) -> None: + """Spawn every post-script configured for `event` and `mode`. Never raises.""" + for command in _entries(event, mode, overrides): + try: + tokens = tokenize(command) + except ValueError as e: # shlex: unbalanced quote in the command template + log.warning("Skipping malformed post-script command (%s): %s", e, redact_path(command)) + continue + argv = substitute(tokens, context) + if not argv: + continue + injected = _data_injected_flag(tokens, argv) + if injected is not None: + log.warning( + "Skipping post-script: a substituted value expands to the option-like token %r; " + "prefix the variable (--opt={var}) so service metadata cannot forge a flag", + redact_text(redact_path(injected)), + ) + continue + safe_command = redact_text(redact_path(shlex.join(argv))) + log.debug("Running post-script: %s", safe_command) + log_event( + "post_script_dispatch", + message=f"Running post-script for {event}/{mode}", + context={"event": event, "mode": mode, "command": safe_command, "argc": len(argv)}, + ) + detach: dict[str, Any] = ( + {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} # type: ignore[attr-defined] + if os.name == "nt" + else {"start_new_session": True} + ) + try: + subprocess.Popen(argv, stdin=subprocess.DEVNULL, **detach) + except (OSError, ValueError) as e: + log.warning("Post-script failed to start (%s): %s", e, redact_text(redact_path(argv[0])))