mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-22 00:45:10 +00:00
fix: drop null vault keys, cap snapshot restore size, isolate test globals
Vaults.add_keys and kv add skip null content keys instead of failing the batch; is_null_key moves to core.vault. A manifest snapshot restore has one 32 MiB inflate budget. Tests reset the speed limiter and config between runs.
This commit is contained in:
@@ -36,3 +36,19 @@ def test_kv_copy_rejects_a_name_that_is_neither_configured_nor_a_file(monkeypatc
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "not defined in the config" in result.output
|
||||
|
||||
|
||||
def test_kv_add_skips_a_null_key_line_and_adds_the_rest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
keys = tmp_path / "keys.txt"
|
||||
null_kid = "0" * 31 + "2"
|
||||
keys.write_text(f"{KID}:{KEY}\n{null_kid}:{'0' * 32}\n", encoding="utf8")
|
||||
dest = tmp_path / "local.db"
|
||||
monkeypatch.setattr(kv_module.config, "key_vaults", [{"type": "SQLite", "name": "local", "path": str(dest)}])
|
||||
monkeypatch.setattr(kv_module.Services, "get_vault_tag", lambda name: name)
|
||||
|
||||
result = CliRunner().invoke(kv_module.kv, ["add", str(keys), "TESTSVC", "local"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
check = SQLite("check", dest)
|
||||
assert check.get_key(KID, "TESTSVC") == KEY
|
||||
assert check.get_key(null_kid, "TESTSVC") is None
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import socket
|
||||
import socketserver
|
||||
from pathlib import Path
|
||||
@@ -14,6 +15,8 @@ from unshackle.core.config import config
|
||||
from unshackle.core.proxies.proxy import CATALOGUES
|
||||
from unshackle.core.update_checker import UpdateChecker
|
||||
|
||||
requests_downloader = importlib.import_module("unshackle.core.downloaders.requests")
|
||||
|
||||
# shutdown() blocks until the serve loop polls again, and the 0.5s default is paid by every
|
||||
# test server at teardown. The default changes for the whole process, servers in core included.
|
||||
socketserver.BaseServer.serve_forever.__defaults__ = (0.01,)
|
||||
@@ -59,6 +62,13 @@ def default_codec_names(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "codec_names", {})
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_speed_limit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Start with no speed limit, so a limit that an earlier test locked cannot block a later one."""
|
||||
monkeypatch.setattr(requests_downloader, "speed_limiter", None)
|
||||
monkeypatch.setattr(requests_downloader, "speed_limit_locked", False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mux_args(monkeypatch: pytest.MonkeyPatch) -> Callable[..., list[str]]:
|
||||
"""Run Tracks.mux up to mkvmerge and return its arguments; ``ids`` is what the track ID probe reports."""
|
||||
|
||||
@@ -2,13 +2,17 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import zlib
|
||||
from copy import deepcopy
|
||||
from typing import Any, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.manifests import DASH
|
||||
from unshackle.core.manifests import snapshot as snapshot_module
|
||||
from unshackle.core.manifests.snapshot import capture, parse, restore
|
||||
from unshackle.core.tracks import Tracks
|
||||
from unshackle.core.tracks.track import Track
|
||||
@@ -117,6 +121,10 @@ def round_trip(tracks: Tracks, level: int = -1) -> Tracks:
|
||||
return client
|
||||
|
||||
|
||||
def inflated_size(entry: dict[str, Any]) -> int:
|
||||
return len(zlib.decompress(base64.b64decode(entry["data"])))
|
||||
|
||||
|
||||
def test_service_filter_drops_every_ad_period_after_the_first_content_period():
|
||||
assert stitched(service_tracks().videos[0]) == ["main-1", "main-2", "main-3"]
|
||||
|
||||
@@ -176,6 +184,34 @@ def test_a_saved_manifest_that_does_not_parse_gives_none():
|
||||
assert parse({"manifests": [{"type": "dash", "url": URL, "data": "bm90IHpsaWI="}]}, URL, "en") is None
|
||||
|
||||
|
||||
def test_the_inflate_budget_covers_every_entry_of_one_restore(monkeypatch, caplog):
|
||||
saved = capture(service_tracks())
|
||||
entry = saved["manifests"][0]
|
||||
size = inflated_size(entry)
|
||||
saved["manifests"].append({**entry, "url": URL + "?copy"})
|
||||
monkeypatch.setattr(snapshot_module, "MAX_SNAPSHOT_BYTES", size)
|
||||
assert parse(saved, URL, "en") is not None
|
||||
|
||||
client = Tracks()
|
||||
client.add(Track.from_dict(service_tracks().videos[0].to_dict()))
|
||||
with caplog.at_level(logging.WARNING, logger="snapshot"):
|
||||
restore(client, saved)
|
||||
assert client.videos[0].data.get("dash")
|
||||
assert ["expands beyond allowed size" in r.getMessage() for r in caplog.records] == [True]
|
||||
|
||||
monkeypatch.setattr(snapshot_module, "MAX_SNAPSHOT_BYTES", size - 1)
|
||||
assert parse(saved, URL, "en") is None
|
||||
|
||||
|
||||
def test_the_fragments_and_the_whole_manifests_share_one_budget(monkeypatch):
|
||||
tracks = service_tracks()
|
||||
detach_video(tracks)
|
||||
monkeypatch.setattr(snapshot_module, "MAX_SNAPSHOT_BYTES", inflated_size(capture(tracks)["manifests"][0]))
|
||||
client = round_trip(tracks)
|
||||
assert client.videos[0].data.get("dash")
|
||||
assert not client.audio[0].data.get("dash")
|
||||
|
||||
|
||||
def test_compression_level_zero_restores():
|
||||
video = round_trip(service_tracks(), level=0).videos[0]
|
||||
assert stitched(video) == ["main-1", "main-2", "main-3"]
|
||||
|
||||
@@ -17,7 +17,8 @@ from uuid import UUID
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.vault import Vault
|
||||
from unshackle.core.vault import Vault, is_null_key
|
||||
from unshackle.core.vaults import Vaults
|
||||
from unshackle.vaults.SQLite import SQLite
|
||||
|
||||
LOG = logging.getLogger("kv-test")
|
||||
@@ -119,10 +120,29 @@ def test_synced_lowercase_table_survives_the_service_being_added_later(tmp_path:
|
||||
assert len(list(v.get_keys("SVCC"))) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "expected"),
|
||||
[
|
||||
(None, True),
|
||||
("", True),
|
||||
("0" * 32, True),
|
||||
("00", True),
|
||||
("00112233445566778899aabbccddeeff", False),
|
||||
("00112233445566778899AABBCCDDEEFF", False),
|
||||
],
|
||||
)
|
||||
def test_is_null_key(key: Optional[str], expected: bool) -> None:
|
||||
assert is_null_key(key) is expected
|
||||
|
||||
|
||||
def test_null_keys_are_rejected_and_filtered(tmp_path: Path) -> None:
|
||||
v = SQLite("test", tmp_path / "kv.db")
|
||||
with pytest.raises(ValueError):
|
||||
v.add_keys("SVCA", {KID: "0" * 32})
|
||||
null_key_error = r"^You cannot add a NULL Content Key to a Vault\.$"
|
||||
for key in ("0" * 32, ""):
|
||||
with pytest.raises(ValueError, match=null_key_error):
|
||||
v.add_keys("SVCA", {KID: key})
|
||||
with pytest.raises(ValueError, match=null_key_error):
|
||||
v.add_key("SVCA", KID, key)
|
||||
|
||||
v.create_table("svca")
|
||||
conn = sqlite3.connect(v.path)
|
||||
@@ -134,6 +154,15 @@ def test_null_keys_are_rejected_and_filtered(tmp_path: Path) -> None:
|
||||
assert list(v.get_keys("SVCA")) == []
|
||||
|
||||
|
||||
def test_vaults_push_drops_null_keys_and_keeps_the_rest(tmp_path: Path) -> None:
|
||||
vaults = Vaults("SVCA")
|
||||
vaults.vaults = [SQLite("test", tmp_path / "kv.db")]
|
||||
other_kid = "0" * 31 + "2"
|
||||
assert vaults.add_keys({KID: KEY, other_kid: "0" * 32}) == 1
|
||||
assert vaults.vaults[0].get_key(KID, "SVCA") == KEY
|
||||
assert vaults.vaults[0].get_key(other_kid, "SVCA") is None
|
||||
|
||||
|
||||
def test_get_services_yields_stored_table_names(vault: SQLite, aliased_service: str) -> None:
|
||||
"""Names are yielded verbatim. Normalising the `svcdalias` table to the installed SVCD
|
||||
tag would hand back a name that addresses no table at all."""
|
||||
|
||||
@@ -180,11 +180,8 @@ def test_speed_limit_forces_single_process(server, tmp_path, monkeypatch):
|
||||
n = dl.MP_MIN_SEGMENTS
|
||||
urls = [{"url": url(server, f"/seg/{i}.bin")} for i in range(n)]
|
||||
dl.set_speed_limit(1_000_000_000) # far above the tiny batch, so the test is not slowed
|
||||
try:
|
||||
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
|
||||
pass
|
||||
finally:
|
||||
dl.set_speed_limit(None)
|
||||
for _ in dl.requests(urls, output_dir=tmp_path, filename="seg_{i:04}.bin", max_workers=4, processes=2):
|
||||
pass
|
||||
|
||||
files = sorted(tmp_path.glob("seg_*.bin"))
|
||||
assert len(files) == n
|
||||
|
||||
@@ -121,15 +121,11 @@ def test_bucket_chunk_larger_than_capacity_does_not_deadlock(monkeypatch):
|
||||
|
||||
def test_locked_limit_ignores_unlocked_calls():
|
||||
"""serve's global_speed_limit must survive the per-job set in dl.result()."""
|
||||
try:
|
||||
set_speed_limit(5_000_000, lock=True)
|
||||
set_speed_limit(None)
|
||||
set_speed_limit(1_000_000)
|
||||
limiter = requests_downloader.speed_limiter
|
||||
assert limiter is not None and limiter.rate == 5_000_000
|
||||
finally:
|
||||
requests_downloader.speed_limit_locked = False
|
||||
requests_downloader.speed_limiter = None
|
||||
set_speed_limit(5_000_000, lock=True)
|
||||
set_speed_limit(None)
|
||||
set_speed_limit(1_000_000)
|
||||
limiter = requests_downloader.speed_limiter
|
||||
assert limiter is not None and limiter.rate == 5_000_000
|
||||
|
||||
|
||||
def test_bucket_wait_aborts_on_cancellation():
|
||||
|
||||
@@ -26,6 +26,9 @@ def running(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> tuple[web.Applic
|
||||
wvds.mkdir()
|
||||
prds.mkdir()
|
||||
(wvds / "dev.wvd").write_bytes(b"x")
|
||||
# a valid reload replaces every config section, so put each one back for the next test
|
||||
for name, value in list(vars(config).items()):
|
||||
monkeypatch.setattr(config, name, value)
|
||||
monkeypatch.setattr(config.directories, "wvds", wvds)
|
||||
monkeypatch.setattr(config.directories, "prds", prds)
|
||||
monkeypatch.setattr(config, "serve", {"api_secret": "api", "users": {"old": {"devices": ["dev"]}}})
|
||||
|
||||
@@ -15,16 +15,6 @@ requests_downloader = importlib.import_module("unshackle.core.downloaders.reques
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_limiter():
|
||||
"""Start and end each test with no limiter, whatever an earlier test leaked."""
|
||||
requests_downloader.speed_limit_locked = False
|
||||
requests_downloader.speed_limiter = None
|
||||
yield
|
||||
requests_downloader.speed_limit_locked = False
|
||||
requests_downloader.speed_limiter = None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("lock", "expected"), [(True, 2_000_000.0), (False, None)])
|
||||
def test_worker_share_of_limit(lock, expected):
|
||||
requests_downloader.set_speed_limit(8_000_000, lock=lock)
|
||||
|
||||
Reference in New Issue
Block a user