mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-17 06:25:07 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
71f293161e | ||
|
|
3e9c26bd46 | ||
|
|
9dd97bcd25 | ||
|
|
f83a594c91 |
@@ -339,7 +339,7 @@ unshackle cfg [KEY] [VALUE] [OPTIONS]
|
||||
| `--list` | List all set configuration values. |
|
||||
|
||||
!!! warning "Comments are stripped"
|
||||
Writing a value through `cfg` rewrites the configuration file and **removes all comments** from it. If your config relies on comments, edit it by hand instead. Setting a value and using `--unset` together is an error.
|
||||
Writing a value through `cfg` rewrites the configuration file and **removes all comments** from it. If your config relies on comments, edit it by hand instead. Setting a value and using `--unset` together is an error. If a config key is set twice in the same block, every command warns with the line numbers of the two copies and uses the later copy. `cfg` stops without a change.
|
||||
|
||||
!!! example
|
||||
```shell
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from unshackle.commands import cfg as cfg_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
path = tmp_path / "unshackle.yaml"
|
||||
monkeypatch.setattr(cfg_module, "get_config_path", lambda: path)
|
||||
return path
|
||||
|
||||
|
||||
def test_a_duplicate_key_logs_its_lines_once_and_names_no_key_or_value(
|
||||
config_file: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
config_file.write_text(
|
||||
"theme: rainbow\ncredentials:\n EXAMPLE: first-secret\n OTHER: other\n EXAMPLE: second-secret\n"
|
||||
"serve:\n users:\n key-secret: {tier: a}\n key-secret: {tier: b}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with caplog.at_level(logging.WARNING, logger="config"):
|
||||
result = CliRunner().invoke(cfg_module.cfg, ["theme"])
|
||||
CliRunner().invoke(cfg_module.cfg, ["theme"])
|
||||
assert isinstance(result.exception, SystemExit), result.exception
|
||||
assert result.exit_code == 1
|
||||
assert caplog.messages == [
|
||||
f"{config_file}: duplicate key on lines 3 and 5, lines 8 and 9. Unshackle uses the later one."
|
||||
]
|
||||
assert "Error" not in result.output and "lines" not in result.output
|
||||
for output in (result.output, caplog.text):
|
||||
assert "secret" not in output and "EXAMPLE" not in output
|
||||
|
||||
|
||||
def test_a_duplicate_key_blocks_a_write(config_file: Path) -> None:
|
||||
original = "theme: rainbow\ncredentials:\n EXAMPLE: a\n EXAMPLE: b\n"
|
||||
config_file.write_text(original, encoding="utf-8")
|
||||
result = CliRunner().invoke(cfg_module.cfg, ["theme", "plain"])
|
||||
assert result.exit_code == 1
|
||||
assert config_file.read_text(encoding="utf-8") == original
|
||||
|
||||
|
||||
def test_a_valid_config_still_reads_a_key(config_file: Path) -> None:
|
||||
config_file.write_text("theme: rainbow\n", encoding="utf-8")
|
||||
result = CliRunner().invoke(cfg_module.cfg, ["theme"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "rainbow" in result.output
|
||||
|
||||
|
||||
def test_a_duplicate_only_ruamel_sees_still_gives_its_line(config_file: Path, caplog: pytest.LogCaptureFixture) -> None:
|
||||
config_file.write_text("a: 1\n1: x\n0x1: y\n", encoding="utf-8")
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = CliRunner().invoke(cfg_module.cfg, ["a"])
|
||||
assert result.exit_code == 1
|
||||
assert caplog.messages == [f"{config_file}: duplicate key on line 3. Unshackle uses the later one."]
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
from unshackle.core.config import Config, duplicate_key_lines, load_yaml
|
||||
|
||||
ALIASES = "base: &b\n x: 1\n x: 2\n self: *b\nother: *b\nlist:\n - y: 1\n y: 2\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compose", [yaml.compose, YAML().compose], ids=["pyyaml", "ruamel"])
|
||||
def test_duplicate_key_lines_follows_aliases_without_looping(compose) -> None:
|
||||
assert duplicate_key_lines(compose(ALIASES)) == [(2, 3), (7, 8)]
|
||||
|
||||
|
||||
def test_load_yaml_warns_once_with_lines_and_keeps_the_later_copy(
|
||||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
path = tmp_path / "unshackle.yaml"
|
||||
path.write_text("credentials:\n EXAMPLE: first-secret\n EXAMPLE: second-secret\n", encoding="utf-8")
|
||||
with caplog.at_level(logging.WARNING, logger="config"):
|
||||
data = load_yaml(path)
|
||||
load_yaml(path)
|
||||
assert data == {"credentials": {"EXAMPLE": "second-secret"}}
|
||||
assert caplog.messages == [f"{path}: duplicate key on lines 2 and 3. Unshackle uses the later one."]
|
||||
assert "EXAMPLE" not in caplog.text and "secret" not in caplog.text
|
||||
|
||||
|
||||
def test_config_from_yaml_warns_on_a_duplicate_key(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
|
||||
path = tmp_path / "unshackle.yaml"
|
||||
path.write_text("tag: first\ntag: second\n", encoding="utf-8")
|
||||
with caplog.at_level(logging.WARNING, logger="config"):
|
||||
assert Config.from_yaml(path).tag == "second"
|
||||
assert "duplicate key on lines 1 and 2" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
["a: 1\nb: [1, 2]\n", "", "a: &a\n x: 1\nb: &b\n y: 2\nc:\n <<: *a\n <<: *b\n"],
|
||||
ids=["valid", "empty", "two-merges"],
|
||||
)
|
||||
def test_load_yaml_matches_safe_load_without_a_warning(
|
||||
tmp_path: Path, text: str, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
path = tmp_path / "config.yaml"
|
||||
path.write_text(text, encoding="utf-8")
|
||||
with caplog.at_level(logging.WARNING, logger="config"):
|
||||
assert load_yaml(path) == yaml.safe_load(text)
|
||||
assert not caplog.records
|
||||
@@ -4,8 +4,9 @@ import sys
|
||||
|
||||
import click
|
||||
from ruamel.yaml import YAML
|
||||
from ruamel.yaml.constructor import DuplicateKeyError
|
||||
|
||||
from unshackle.core.config import config, get_config_path
|
||||
from unshackle.core.config import config, duplicate_key_lines, get_config_path, load_yaml
|
||||
from unshackle.core.constants import context_settings
|
||||
|
||||
|
||||
@@ -41,7 +42,16 @@ def cfg(ctx: click.Context, key: str, value: str, unset: bool, list_: bool) -> N
|
||||
|
||||
config_path = get_config_path() or config.directories.user_configs / config.filenames.root_config
|
||||
if config_path.exists():
|
||||
data = yaml.load(config_path)
|
||||
try:
|
||||
data = yaml.load(config_path)
|
||||
except DuplicateKeyError as e:
|
||||
if duplicate_key_lines(yaml.compose(config_path)):
|
||||
load_yaml(config_path)
|
||||
else:
|
||||
log.warning(
|
||||
f"{config_path}: duplicate key on line {e.problem_mark.line + 1}. Unshackle uses the later one."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if not data:
|
||||
log.warning("No config file was found or it has no data, yet")
|
||||
|
||||
@@ -27,7 +27,6 @@ from uuid import UUID
|
||||
|
||||
import click
|
||||
import mediaexport
|
||||
import yaml
|
||||
from click.core import ParameterSource
|
||||
from langcodes import Language, tag_is_valid
|
||||
from pymediainfo import MediaInfo
|
||||
@@ -43,7 +42,7 @@ from rich.tree import Tree
|
||||
from unshackle.core import __version__, binaries, providers
|
||||
from unshackle.core.cdm import DecryptLabsRemoteCDM
|
||||
from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote_cdm, is_widevine_cdm
|
||||
from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption
|
||||
from unshackle.core.config import cdm_entry_names_device, config, load_yaml, resolve_cdm_name, resolve_decryption
|
||||
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
|
||||
from unshackle.core.constants import (
|
||||
DEFAULT_SAMPLE,
|
||||
@@ -1454,7 +1453,7 @@ class dl:
|
||||
try:
|
||||
service_config_path = Services.get_path(self.service) / config.filenames.config
|
||||
if service_config_path.exists():
|
||||
self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
|
||||
self.service_config = load_yaml(service_config_path)
|
||||
self.log.info("Service Config loaded")
|
||||
# log key names only: the full config carries service certificates,
|
||||
# device fingerprints and endpoints that bloat the log and may be sensitive
|
||||
|
||||
@@ -6,13 +6,12 @@ import sys
|
||||
from typing import Any, Optional
|
||||
|
||||
import click
|
||||
import yaml
|
||||
from rich.padding import Padding
|
||||
from rich.rule import Rule
|
||||
from rich.tree import Tree
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.config import config, load_yaml
|
||||
from unshackle.core.console import console
|
||||
from unshackle.core.constants import context_settings
|
||||
from unshackle.core.proxies.proxy import Proxy
|
||||
@@ -91,7 +90,7 @@ def search(
|
||||
with console.status("Loading Service Config...", spinner="dots"):
|
||||
service_config_path = Services.get_path(service) / config.filenames.config
|
||||
if service_config_path.exists():
|
||||
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
|
||||
service_config = load_yaml(service_config_path)
|
||||
log.info("Service Config loaded")
|
||||
merge_dict(ci_get(config.services, service), service_config)
|
||||
|
||||
|
||||
@@ -371,11 +371,10 @@ def perform_download(
|
||||
|
||||
# Import dl.py components lazily to avoid circular deps during module import
|
||||
import click
|
||||
import yaml
|
||||
|
||||
from unshackle.commands.dl import dl
|
||||
from unshackle.core.api.errors import APIError, APIErrorCode
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.config import config, load_yaml
|
||||
from unshackle.core.services import Services
|
||||
from unshackle.core.tracks import Video
|
||||
from unshackle.core.utils.click_types import ContextData
|
||||
@@ -467,7 +466,7 @@ def perform_download(
|
||||
remote = params.get("remote")
|
||||
service_config_path = None if remote else Services.get_path(service) / config.filenames.config
|
||||
if service_config_path and service_config_path.exists():
|
||||
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
|
||||
service_config = load_yaml(service_config_path)
|
||||
else:
|
||||
service_config = {}
|
||||
merge_dict(config.services.get(service), service_config)
|
||||
|
||||
@@ -28,7 +28,7 @@ from unshackle.core.api.session_log import SessionLogBuffer, SessionLogMirror, c
|
||||
from unshackle.core.api.session_store import SessionStore
|
||||
from unshackle.core.cacher import Cacher
|
||||
from unshackle.core.cdm.detect import cdm_type_stub
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.config import config, load_yaml
|
||||
from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP
|
||||
from unshackle.core.drm import cbcs
|
||||
from unshackle.core.providers.anilist import parse_anilist_ref
|
||||
@@ -190,13 +190,11 @@ def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str]
|
||||
|
||||
def load_service_yaml(normalized_service: str) -> dict:
|
||||
"""Load a service's config.yaml and merge it with the global override block."""
|
||||
import yaml
|
||||
|
||||
from unshackle.core.utils.collections import merge_dict
|
||||
|
||||
service_config_path = Services.get_path(normalized_service) / config.filenames.config
|
||||
if service_config_path.exists():
|
||||
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) or {}
|
||||
service_config = load_yaml(service_config_path) or {}
|
||||
else:
|
||||
service_config = {}
|
||||
merge_dict(config.services.get(normalized_service), service_config)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
@@ -374,7 +375,56 @@ class Config:
|
||||
raise FileNotFoundError(f"Config file path ({path}) was not found")
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"Config file path ({path}) is not to a file.")
|
||||
return cls(**restore_bool_languages(yaml.safe_load(path.read_text(encoding="utf8")) or {}))
|
||||
return cls(**restore_bool_languages(load_yaml(path) or {}))
|
||||
|
||||
|
||||
def duplicate_key_lines(root: Any) -> list[tuple[int, int]]:
|
||||
"""
|
||||
Return (first line, repeat line), counted from 1, for each repeated scalar key in a PyYAML or ruamel node tree.
|
||||
|
||||
Keys compare as written, so two spellings that load as the same key (yes and true, 1 and 0x1) are not found.
|
||||
"""
|
||||
pairs, stack, visited = [], [root], set()
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if node in visited:
|
||||
continue
|
||||
visited.add(node)
|
||||
if node.id == "mapping":
|
||||
first: dict[tuple[str, str], int] = {}
|
||||
for key_node, value_node in node.value:
|
||||
if key_node.id == "scalar" and key_node.tag != "tag:yaml.org,2002:merge":
|
||||
key = (key_node.tag, key_node.value)
|
||||
line = key_node.start_mark.line + 1
|
||||
if key in first:
|
||||
pairs.append((first[key], line))
|
||||
else:
|
||||
first[key] = line
|
||||
stack += [key_node, value_node]
|
||||
elif node.id == "sequence":
|
||||
stack += node.value
|
||||
return sorted(pairs)
|
||||
|
||||
|
||||
_warned: set[str] = set()
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> Any:
|
||||
"""yaml.safe_load a file, and warn with the lines of each duplicate key, which PyYAML takes silently."""
|
||||
loader = yaml.SafeLoader(path.read_text(encoding="utf8"))
|
||||
try:
|
||||
node = loader.get_single_node()
|
||||
if node is None:
|
||||
return None
|
||||
if pairs := duplicate_key_lines(node):
|
||||
where = ", ".join(f"lines {first} and {repeat}" for first, repeat in pairs)
|
||||
message = f"{path}: duplicate key on {where}. Unshackle uses the later one."
|
||||
if message not in _warned:
|
||||
_warned.add(message)
|
||||
logging.getLogger("config").warning(message)
|
||||
return loader.construct_document(node)
|
||||
finally:
|
||||
loader.dispose()
|
||||
|
||||
|
||||
# noinspection PyProtectedMember
|
||||
|
||||
Reference in New Issue
Block a user