fix: drop null vault keys, cap snapshot restore size, isolate test globals

Vaults.add_keys and kv add skip null content keys instead of failing the batch; is_null_key moves to core.vault. A manifest snapshot restore has one 32 MiB inflate budget. Tests reset the speed limiter and config between runs.
This commit is contained in:
imSp4rky
2026-11-02 22:01:02 -07:00
parent c4677341c8
commit 3027592e31
19 changed files with 170 additions and 81 deletions
+16
View File
@@ -36,3 +36,19 @@ def test_kv_copy_rejects_a_name_that_is_neither_configured_nor_a_file(monkeypatc
assert result.exit_code != 0
assert "not defined in the config" in result.output
def test_kv_add_skips_a_null_key_line_and_adds_the_rest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
keys = tmp_path / "keys.txt"
null_kid = "0" * 31 + "2"
keys.write_text(f"{KID}:{KEY}\n{null_kid}:{'0' * 32}\n", encoding="utf8")
dest = tmp_path / "local.db"
monkeypatch.setattr(kv_module.config, "key_vaults", [{"type": "SQLite", "name": "local", "path": str(dest)}])
monkeypatch.setattr(kv_module.Services, "get_vault_tag", lambda name: name)
result = CliRunner().invoke(kv_module.kv, ["add", str(keys), "TESTSVC", "local"])
assert result.exit_code == 0, result.output
check = SQLite("check", dest)
assert check.get_key(KID, "TESTSVC") == KEY
assert check.get_key(null_kid, "TESTSVC") is None
+10
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import importlib
import socket
import socketserver
from pathlib import Path
@@ -14,6 +15,8 @@ from unshackle.core.config import config
from unshackle.core.proxies.proxy import CATALOGUES
from unshackle.core.update_checker import UpdateChecker
requests_downloader = importlib.import_module("unshackle.core.downloaders.requests")
# shutdown() blocks until the serve loop polls again, and the 0.5s default is paid by every
# test server at teardown. The default changes for the whole process, servers in core included.
socketserver.BaseServer.serve_forever.__defaults__ = (0.01,)
@@ -59,6 +62,13 @@ def default_codec_names(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "codec_names", {})
@pytest.fixture(autouse=True)
def no_speed_limit(monkeypatch: pytest.MonkeyPatch) -> None:
"""Start with no speed limit, so a limit that an earlier test locked cannot block a later one."""
monkeypatch.setattr(requests_downloader, "speed_limiter", None)
monkeypatch.setattr(requests_downloader, "speed_limit_locked", False)
@pytest.fixture
def mux_args(monkeypatch: pytest.MonkeyPatch) -> Callable[..., list[str]]:
"""Run Tracks.mux up to mkvmerge and return its arguments; ``ids`` is what the track ID probe reports."""
+36
View File
@@ -2,13 +2,17 @@
from __future__ import annotations
import base64
import json
import logging
import zlib
from copy import deepcopy
from typing import Any, Optional
import pytest
from unshackle.core.manifests import DASH
from unshackle.core.manifests import snapshot as snapshot_module
from unshackle.core.manifests.snapshot import capture, parse, restore
from unshackle.core.tracks import Tracks
from unshackle.core.tracks.track import Track
@@ -117,6 +121,10 @@ def round_trip(tracks: Tracks, level: int = -1) -> Tracks:
return client
def inflated_size(entry: dict[str, Any]) -> int:
return len(zlib.decompress(base64.b64decode(entry["data"])))
def test_service_filter_drops_every_ad_period_after_the_first_content_period():
assert stitched(service_tracks().videos[0]) == ["main-1", "main-2", "main-3"]
@@ -176,6 +184,34 @@ def test_a_saved_manifest_that_does_not_parse_gives_none():
assert parse({"manifests": [{"type": "dash", "url": URL, "data": "bm90IHpsaWI="}]}, URL, "en") is None
def test_the_inflate_budget_covers_every_entry_of_one_restore(monkeypatch, caplog):
saved = capture(service_tracks())
entry = saved["manifests"][0]
size = inflated_size(entry)
saved["manifests"].append({**entry, "url": URL + "?copy"})
monkeypatch.setattr(snapshot_module, "MAX_SNAPSHOT_BYTES", size)
assert parse(saved, URL, "en") is not None
client = Tracks()
client.add(Track.from_dict(service_tracks().videos[0].to_dict()))
with caplog.at_level(logging.WARNING, logger="snapshot"):
restore(client, saved)
assert client.videos[0].data.get("dash")
assert ["expands beyond allowed size" in r.getMessage() for r in caplog.records] == [True]
monkeypatch.setattr(snapshot_module, "MAX_SNAPSHOT_BYTES", size - 1)
assert parse(saved, URL, "en") is None
def test_the_fragments_and_the_whole_manifests_share_one_budget(monkeypatch):
tracks = service_tracks()
detach_video(tracks)
monkeypatch.setattr(snapshot_module, "MAX_SNAPSHOT_BYTES", inflated_size(capture(tracks)["manifests"][0]))
client = round_trip(tracks)
assert client.videos[0].data.get("dash")
assert not client.audio[0].data.get("dash")
def test_compression_level_zero_restores():
video = round_trip(service_tracks(), level=0).videos[0]
assert stitched(video) == ["main-1", "main-2", "main-3"]
+32 -3
View File
@@ -17,7 +17,8 @@ from uuid import UUID
import pytest
from unshackle.core.vault import Vault
from unshackle.core.vault import Vault, is_null_key
from unshackle.core.vaults import Vaults
from unshackle.vaults.SQLite import SQLite
LOG = logging.getLogger("kv-test")
@@ -119,10 +120,29 @@ def test_synced_lowercase_table_survives_the_service_being_added_later(tmp_path:
assert len(list(v.get_keys("SVCC"))) == 2
@pytest.mark.parametrize(
("key", "expected"),
[
(None, True),
("", True),
("0" * 32, True),
("00", True),
("00112233445566778899aabbccddeeff", False),
("00112233445566778899AABBCCDDEEFF", False),
],
)
def test_is_null_key(key: Optional[str], expected: bool) -> None:
assert is_null_key(key) is expected
def test_null_keys_are_rejected_and_filtered(tmp_path: Path) -> None:
v = SQLite("test", tmp_path / "kv.db")
with pytest.raises(ValueError):
v.add_keys("SVCA", {KID: "0" * 32})
null_key_error = r"^You cannot add a NULL Content Key to a Vault\.$"
for key in ("0" * 32, ""):
with pytest.raises(ValueError, match=null_key_error):
v.add_keys("SVCA", {KID: key})
with pytest.raises(ValueError, match=null_key_error):
v.add_key("SVCA", KID, key)
v.create_table("svca")
conn = sqlite3.connect(v.path)
@@ -134,6 +154,15 @@ def test_null_keys_are_rejected_and_filtered(tmp_path: Path) -> None:
assert list(v.get_keys("SVCA")) == []
def test_vaults_push_drops_null_keys_and_keeps_the_rest(tmp_path: Path) -> None:
vaults = Vaults("SVCA")
vaults.vaults = [SQLite("test", tmp_path / "kv.db")]
other_kid = "0" * 31 + "2"
assert vaults.add_keys({KID: KEY, other_kid: "0" * 32}) == 1
assert vaults.vaults[0].get_key(KID, "SVCA") == KEY
assert vaults.vaults[0].get_key(other_kid, "SVCA") is None
def test_get_services_yields_stored_table_names(vault: SQLite, aliased_service: str) -> None:
"""Names are yielded verbatim. Normalising the `svcdalias` table to the installed SVCD
tag would hand back a name that addresses no table at all."""
+2 -5
View File
@@ -180,11 +180,8 @@ def test_speed_limit_forces_single_process(server, tmp_path, monkeypatch):
n = dl.MP_MIN_SEGMENTS
urls = [{"url": url(server, f"/seg/{i}.bin")} for i in range(n)]
dl.set_speed_limit(1_000_000_000) # far above the tiny batch, so the test is not slowed
try:
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
pass
finally:
dl.set_speed_limit(None)
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
pass
files = sorted(tmp_path.glob("seg_*.bin"))
assert len(files) == n
+5 -9
View File
@@ -121,15 +121,11 @@ def test_bucket_chunk_larger_than_capacity_does_not_deadlock(monkeypatch):
def test_locked_limit_ignores_unlocked_calls():
"""serve's global_speed_limit must survive the per-job set in dl.result()."""
try:
set_speed_limit(5_000_000, lock=True)
set_speed_limit(None)
set_speed_limit(1_000_000)
limiter = requests_downloader.speed_limiter
assert limiter is not None and limiter.rate == 5_000_000
finally:
requests_downloader.speed_limit_locked = False
requests_downloader.speed_limiter = None
set_speed_limit(5_000_000, lock=True)
set_speed_limit(None)
set_speed_limit(1_000_000)
limiter = requests_downloader.speed_limiter
assert limiter is not None and limiter.rate == 5_000_000
def test_bucket_wait_aborts_on_cancellation():
+3
View File
@@ -26,6 +26,9 @@ def running(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> tuple[web.Applic
wvds.mkdir()
prds.mkdir()
(wvds / "dev.wvd").write_bytes(b"x")
# a valid reload replaces every config section, so put each one back for the next test
for name, value in list(vars(config).items()):
monkeypatch.setattr(config, name, value)
monkeypatch.setattr(config.directories, "wvds", wvds)
monkeypatch.setattr(config.directories, "prds", prds)
monkeypatch.setattr(config, "serve", {"api_secret": "api", "users": {"old": {"devices": ["dev"]}}})
@@ -15,16 +15,6 @@ requests_downloader = importlib.import_module("unshackle.core.downloaders.reques
pytestmark = pytest.mark.unit
@pytest.fixture(autouse=True)
def clean_limiter():
"""Start and end each test with no limiter, whatever an earlier test leaked."""
requests_downloader.speed_limit_locked = False
requests_downloader.speed_limiter = None
yield
requests_downloader.speed_limit_locked = False
requests_downloader.speed_limiter = None
@pytest.mark.parametrize(("lock", "expected"), [(True, 2_000_000.0), (False, None)])
def test_worker_share_of_limit(lock, expected):
requests_downloader.set_speed_limit(8_000_000, lock=lock)