Compare commits

...
9 changed files with 185 additions and 17 deletions
+1 -1
View File
@@ -339,7 +339,7 @@ unshackle cfg [KEY] [VALUE] [OPTIONS]
| `--list` | List all set configuration values. |
!!! warning "Comments are stripped"
Writing a value through `cfg` rewrites the configuration file and **removes all comments** from it. If your config relies on comments, edit it by hand instead. Setting a value and using `--unset` together is an error.
Writing a value through `cfg` rewrites the configuration file and **removes all comments** from it. If your config relies on comments, edit it by hand instead. Setting a value and using `--unset` together is an error. If a config key is set twice in the same block, every command warns with the line numbers of the two copies and uses the later copy. `cfg` stops without a change.
!!! example
```shell
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import logging
from pathlib import Path
import pytest
from click.testing import CliRunner
from unshackle.commands import cfg as cfg_module
@pytest.fixture
def config_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
path = tmp_path / "unshackle.yaml"
monkeypatch.setattr(cfg_module, "get_config_path", lambda: path)
return path
def test_a_duplicate_key_logs_its_lines_once_and_names_no_key_or_value(
config_file: Path, caplog: pytest.LogCaptureFixture
) -> None:
config_file.write_text(
"theme: rainbow\ncredentials:\n EXAMPLE: first-secret\n OTHER: other\n EXAMPLE: second-secret\n"
"serve:\n users:\n key-secret: {tier: a}\n key-secret: {tier: b}\n",
encoding="utf-8",
)
with caplog.at_level(logging.WARNING, logger="config"):
result = CliRunner().invoke(cfg_module.cfg, ["theme"])
CliRunner().invoke(cfg_module.cfg, ["theme"])
assert isinstance(result.exception, SystemExit), result.exception
assert result.exit_code == 1
assert caplog.messages == [
f"{config_file}: duplicate key on lines 3 and 5, lines 8 and 9. Unshackle uses the later one."
]
assert "Error" not in result.output and "lines" not in result.output
for output in (result.output, caplog.text):
assert "secret" not in output and "EXAMPLE" not in output
def test_a_duplicate_key_blocks_a_write(config_file: Path) -> None:
original = "theme: rainbow\ncredentials:\n EXAMPLE: a\n EXAMPLE: b\n"
config_file.write_text(original, encoding="utf-8")
result = CliRunner().invoke(cfg_module.cfg, ["theme", "plain"])
assert result.exit_code == 1
assert config_file.read_text(encoding="utf-8") == original
def test_a_valid_config_still_reads_a_key(config_file: Path) -> None:
config_file.write_text("theme: rainbow\n", encoding="utf-8")
result = CliRunner().invoke(cfg_module.cfg, ["theme"])
assert result.exit_code == 0, result.output
assert "rainbow" in result.output
def test_a_duplicate_only_ruamel_sees_still_gives_its_line(config_file: Path, caplog: pytest.LogCaptureFixture) -> None:
config_file.write_text("a: 1\n1: x\n0x1: y\n", encoding="utf-8")
with caplog.at_level(logging.WARNING):
result = CliRunner().invoke(cfg_module.cfg, ["a"])
assert result.exit_code == 1
assert caplog.messages == [f"{config_file}: duplicate key on line 3. Unshackle uses the later one."]
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import logging
from pathlib import Path
import pytest
import yaml
from ruamel.yaml import YAML
from unshackle.core.config import Config, duplicate_key_lines, load_yaml
ALIASES = "base: &b\n x: 1\n x: 2\n self: *b\nother: *b\nlist:\n - y: 1\n y: 2\n"
@pytest.mark.parametrize("compose", [yaml.compose, YAML().compose], ids=["pyyaml", "ruamel"])
def test_duplicate_key_lines_follows_aliases_without_looping(compose) -> None:
assert duplicate_key_lines(compose(ALIASES)) == [(2, 3), (7, 8)]
def test_load_yaml_warns_once_with_lines_and_keeps_the_later_copy(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
path = tmp_path / "unshackle.yaml"
path.write_text("credentials:\n EXAMPLE: first-secret\n EXAMPLE: second-secret\n", encoding="utf-8")
with caplog.at_level(logging.WARNING, logger="config"):
data = load_yaml(path)
load_yaml(path)
assert data == {"credentials": {"EXAMPLE": "second-secret"}}
assert caplog.messages == [f"{path}: duplicate key on lines 2 and 3. Unshackle uses the later one."]
assert "EXAMPLE" not in caplog.text and "secret" not in caplog.text
def test_config_from_yaml_warns_on_a_duplicate_key(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
path = tmp_path / "unshackle.yaml"
path.write_text("tag: first\ntag: second\n", encoding="utf-8")
with caplog.at_level(logging.WARNING, logger="config"):
assert Config.from_yaml(path).tag == "second"
assert "duplicate key on lines 1 and 2" in caplog.text
@pytest.mark.parametrize(
"text",
["a: 1\nb: [1, 2]\n", "", "a: &a\n x: 1\nb: &b\n y: 2\nc:\n <<: *a\n <<: *b\n"],
ids=["valid", "empty", "two-merges"],
)
def test_load_yaml_matches_safe_load_without_a_warning(
tmp_path: Path, text: str, caplog: pytest.LogCaptureFixture
) -> None:
path = tmp_path / "config.yaml"
path.write_text(text, encoding="utf-8")
with caplog.at_level(logging.WARNING, logger="config"):
assert load_yaml(path) == yaml.safe_load(text)
assert not caplog.records
+12 -2
View File
@@ -4,8 +4,9 @@ import sys
import click
from ruamel.yaml import YAML
from ruamel.yaml.constructor import DuplicateKeyError
from unshackle.core.config import config, get_config_path
from unshackle.core.config import config, duplicate_key_lines, get_config_path, load_yaml
from unshackle.core.constants import context_settings
@@ -41,7 +42,16 @@ def cfg(ctx: click.Context, key: str, value: str, unset: bool, list_: bool) -> N
config_path = get_config_path() or config.directories.user_configs / config.filenames.root_config
if config_path.exists():
data = yaml.load(config_path)
try:
data = yaml.load(config_path)
except DuplicateKeyError as e:
if duplicate_key_lines(yaml.compose(config_path)):
load_yaml(config_path)
else:
log.warning(
f"{config_path}: duplicate key on line {e.problem_mark.line + 1}. Unshackle uses the later one."
)
sys.exit(1)
if not data:
log.warning("No config file was found or it has no data, yet")
+2 -3
View File
@@ -27,7 +27,6 @@ from uuid import UUID
import click
import mediaexport
import yaml
from click.core import ParameterSource
from langcodes import Language, tag_is_valid
from pymediainfo import MediaInfo
@@ -43,7 +42,7 @@ from rich.tree import Tree
from unshackle.core import __version__, binaries, providers
from unshackle.core.cdm import DecryptLabsRemoteCDM
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote_cdm, is_widevine_cdm
from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption
from unshackle.core.config import cdm_entry_names_device, config, load_yaml, resolve_cdm_name, resolve_decryption
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
from unshackle.core.constants import (
DEFAULT_SAMPLE,
@@ -1454,7 +1453,7 @@ class dl:
try:
service_config_path = Services.get_path(self.service) / config.filenames.config
if service_config_path.exists():
self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
self.service_config = load_yaml(service_config_path)
self.log.info("Service Config loaded")
# log key names only: the full config carries service certificates,
# device fingerprints and endpoints that bloat the log and may be sensitive
+2 -3
View File
@@ -6,13 +6,12 @@ import sys
from typing import Any, Optional
import click
import yaml
from rich.padding import Padding
from rich.rule import Rule
from rich.tree import Tree
from unshackle.commands.dl import dl
from unshackle.core.config import config
from unshackle.core.config import config, load_yaml
from unshackle.core.console import console
from unshackle.core.constants import context_settings
from unshackle.core.proxies.proxy import Proxy
@@ -91,7 +90,7 @@ def search(
with console.status("Loading Service Config...", spinner="dots"):
service_config_path = Services.get_path(service) / config.filenames.config
if service_config_path.exists():
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
service_config = load_yaml(service_config_path)
log.info("Service Config loaded")
merge_dict(ci_get(config.services, service), service_config)
+2 -3
View File
@@ -371,11 +371,10 @@ def perform_download(
# Import dl.py components lazily to avoid circular deps during module import
import click
import yaml
from unshackle.commands.dl import dl
from unshackle.core.api.errors import APIError, APIErrorCode
from unshackle.core.config import config
from unshackle.core.config import config, load_yaml
from unshackle.core.services import Services
from unshackle.core.tracks import Video
from unshackle.core.utils.click_types import ContextData
@@ -467,7 +466,7 @@ def perform_download(
remote = params.get("remote")
service_config_path = None if remote else Services.get_path(service) / config.filenames.config
if service_config_path and service_config_path.exists():
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
service_config = load_yaml(service_config_path)
else:
service_config = {}
merge_dict(config.services.get(service), service_config)
+2 -4
View File
@@ -28,7 +28,7 @@ from unshackle.core.api.session_log import SessionLogBuffer, SessionLogMirror, c
from unshackle.core.api.session_store import SessionStore
from unshackle.core.cacher import Cacher
from unshackle.core.cdm.detect import cdm_type_stub
from unshackle.core.config import config
from unshackle.core.config import config, load_yaml
from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP
from unshackle.core.drm import cbcs
from unshackle.core.providers.anilist import parse_anilist_ref
@@ -190,13 +190,11 @@ def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str]
def load_service_yaml(normalized_service: str) -> dict:
"""Load a service's config.yaml and merge it with the global override block."""
import yaml
from unshackle.core.utils.collections import merge_dict
service_config_path = Services.get_path(normalized_service) / config.filenames.config
if service_config_path.exists():
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) or {}
service_config = load_yaml(service_config_path) or {}
else:
service_config = {}
merge_dict(config.services.get(normalized_service), service_config)
+51 -1
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import logging
import re
import warnings
from pathlib import Path
@@ -374,7 +375,56 @@ class Config:
raise FileNotFoundError(f"Config file path ({path}) was not found")
if not path.is_file():
raise FileNotFoundError(f"Config file path ({path}) is not to a file.")
return cls(**restore_bool_languages(yaml.safe_load(path.read_text(encoding="utf8")) or {}))
return cls(**restore_bool_languages(load_yaml(path) or {}))
def duplicate_key_lines(root: Any) -> list[tuple[int, int]]:
"""
Return (first line, repeat line), counted from 1, for each repeated scalar key in a PyYAML or ruamel node tree.
Keys compare as written, so two spellings that load as the same key (yes and true, 1 and 0x1) are not found.
"""
pairs, stack, visited = [], [root], set()
while stack:
node = stack.pop()
if node in visited:
continue
visited.add(node)
if node.id == "mapping":
first: dict[tuple[str, str], int] = {}
for key_node, value_node in node.value:
if key_node.id == "scalar" and key_node.tag != "tag:yaml.org,2002:merge":
key = (key_node.tag, key_node.value)
line = key_node.start_mark.line + 1
if key in first:
pairs.append((first[key], line))
else:
first[key] = line
stack += [key_node, value_node]
elif node.id == "sequence":
stack += node.value
return sorted(pairs)
_warned: set[str] = set()
def load_yaml(path: Path) -> Any:
"""yaml.safe_load a file, and warn with the lines of each duplicate key, which PyYAML takes silently."""
loader = yaml.SafeLoader(path.read_text(encoding="utf8"))
try:
node = loader.get_single_node()
if node is None:
return None
if pairs := duplicate_key_lines(node):
where = ", ".join(f"lines {first} and {repeat}" for first, repeat in pairs)
message = f"{path}: duplicate key on {where}. Unshackle uses the later one."
if message not in _warned:
_warned.add(message)
logging.getLogger("config").warning(message)
return loader.construct_document(node)
finally:
loader.dispose()
# noinspection PyProtectedMember