mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-22 00:45:10 +00:00
Compare commits
3
Commits
9a1abd23b6
...
73bdb98d3e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73bdb98d3e | ||
|
|
34eb94b55d | ||
|
|
7b3794333f |
@@ -380,7 +380,9 @@ unicode_filenames: true
|
||||
| Value | Result |
|
||||
|---|---|
|
||||
| `false` (default) | `기생충` → transliterated ASCII; accents stripped. |
|
||||
| `true` | Native characters preserved in filenames and folders. |
|
||||
| `true` | Native characters preserved in filenames and folders, with the vowel and tone marks of languages such as Thai, Hindi and Bengali. |
|
||||
|
||||
With each value, unshackle first normalises the name to Unicode NFC. A title that a service sends in composed form and in decomposed form gives the same name.
|
||||
|
||||
Regardless of this setting, unshackle always removes or replaces a set of filesystem-unsafe and structural characters during sanitisation. For example, `/` and `;` become ` & `, and unshackle removes characters like `\ * ! ? , ' " < > | $ # ~`. This is why you must never put path-unsafe characters directly in a template.
|
||||
|
||||
|
||||
@@ -2,12 +2,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from unshackle.core import __version__
|
||||
from unshackle.core.__main__ import main
|
||||
from unshackle.core.__main__ import main, print_banner
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.console import force_utf8_streams
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", [["--help"], ["env", "--help"]])
|
||||
@@ -19,3 +23,13 @@ def test_help_prints_the_banner_once(monkeypatch: pytest.MonkeyPatch, args: list
|
||||
assert result.exit_code == 0, result.output
|
||||
assert result.output.count(f"v {__version__}") == 1
|
||||
assert "Usage:" in result.output
|
||||
|
||||
|
||||
def test_banner_prints_to_a_legacy_code_page_stream(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
buffer = io.BytesIO()
|
||||
monkeypatch.setattr(sys, "stdout", io.TextIOWrapper(buffer, encoding="cp1252"))
|
||||
|
||||
force_utf8_streams()
|
||||
print_banner()
|
||||
|
||||
assert "▄".encode() in buffer.getvalue()
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Combining marks and Unicode normalisation in sanitize_filename.
|
||||
|
||||
Thai, Devanagari, Bengali and other scripts write vowels and tone marks as combining marks on
|
||||
a base letter. With ``unicode_filenames`` those marks are part of the word and must survive.
|
||||
In both modes, composed and decomposed spellings of a title must give the same name.
|
||||
Combining characters are written as escapes so an editor that normalises the file cannot
|
||||
change what a case tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from unshackle.core.config import config
|
||||
from unshackle.core.utilities import sanitize_filename
|
||||
|
||||
THAI = "\u0e40\u0e1b\u0e47\u0e19\u0e15\u0e48\u0e2d" # a Thai title with vowel and tone marks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
THAI,
|
||||
"\u0e01\u0e31\u0e49\u0e19", # Thai: a tone mark stacked on an upper vowel
|
||||
"\u0939\u093f\u0902\u0926\u0940", # Devanagari
|
||||
"\u09b0\u200d\u09cd\u09af\u09be\u09ac", # Bengali: the virama follows a zero width joiner
|
||||
"\u1040\u102d\u102f\u1004\u103a\u1038", # Burmese: a mark on digit zero, typed for the letter wa
|
||||
"\u0e19\u0e49\u0e33", # Thai sara am, which NFKC would split into two characters
|
||||
],
|
||||
)
|
||||
def test_unicode_filenames_keep_marks(name: str) -> None:
|
||||
assert sanitize_filename(name, unicode=True) == name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "expected"),
|
||||
[
|
||||
# Decomposed and composed spellings give the same name.
|
||||
("Cafe\u0301", "Caf\u00e9"),
|
||||
("Caf\u00e9", "Caf\u00e9"),
|
||||
("Show \u2764\ufe0f", "Show.\u2764\ufe0f"),
|
||||
("Some\x07Title", "SomeTitle"),
|
||||
],
|
||||
)
|
||||
def test_unicode_filenames_use_nfc_and_drop_control_characters(name: str, expected: str) -> None:
|
||||
assert sanitize_filename(name, unicode=True) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "expected"),
|
||||
[
|
||||
("Cafe\u0301 Name", "Cafe.Name"),
|
||||
("Caf\u00e9 Name", "Cafe.Name"),
|
||||
# Katakana with separate voicing marks transliterates like the composed spelling.
|
||||
("\u30ab\u3099\u30f3\u30bf\u3099\u30e0", "gandamu"),
|
||||
("\u30ac\u30f3\u30c0\u30e0", "gandamu"),
|
||||
("Some\x07Title", "SomeTitle"),
|
||||
],
|
||||
)
|
||||
def test_ascii_filenames_transliterate_both_spellings(name: str, expected: str) -> None:
|
||||
assert sanitize_filename(name, unicode=False) == expected
|
||||
|
||||
|
||||
def test_config_option_decides_when_unicode_is_not_passed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(config, "unicode_filenames", True)
|
||||
assert sanitize_filename(THAI) == THAI
|
||||
monkeypatch.setattr(config, "unicode_filenames", False)
|
||||
assert sanitize_filename(THAI).isascii()
|
||||
|
||||
|
||||
def test_sanitising_a_name_again_does_not_change_it() -> None:
|
||||
# The removed "?" sits between the letter and its mark.
|
||||
once = sanitize_filename("e?\u0301", unicode=True)
|
||||
assert once == "\u00e9"
|
||||
assert sanitize_filename(once, unicode=True) == once
|
||||
@@ -1,6 +1,9 @@
|
||||
import codecs
|
||||
import io
|
||||
import logging
|
||||
import math
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from types import ModuleType
|
||||
from typing import IO, Any, Callable, Iterable, List, Literal, Mapping, Optional, TextIO, Union
|
||||
@@ -402,6 +405,15 @@ class _GradientPulse:
|
||||
return Measurement(4, self.width or options.max_width)
|
||||
|
||||
|
||||
def force_utf8_streams() -> None:
|
||||
"""Make stdout and stderr write UTF-8. A redirected stream on Windows uses a legacy code page that cannot encode the console output."""
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
if isinstance(stream, io.TextIOWrapper) and codecs.lookup(stream.encoding).name != "utf-8":
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
force_utf8_streams()
|
||||
|
||||
console = ComfyConsole(
|
||||
log_time=False,
|
||||
log_path=False,
|
||||
|
||||
@@ -128,13 +128,15 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
|
||||
Set `unicode_filenames: true` in config to preserve the characters of the
|
||||
original language (for example Korean, Japanese, or Chinese) instead of
|
||||
transliterating them to ASCII equivalents. Pass ``unicode`` to decide that
|
||||
for one call instead of the config value.
|
||||
for one call instead of the config value. The name goes to NFC first, so
|
||||
composed and decomposed spellings of a title give the same name.
|
||||
"""
|
||||
filename = unicodedata.normalize("NFC", filename)
|
||||
if not (config.unicode_filenames if unicode is None else unicode):
|
||||
filename = unidecode(filename)
|
||||
filename = re.sub(r"\[\(+", "[", filename)
|
||||
filename = re.sub(r"\)+\]", "]", filename)
|
||||
filename = "".join(c for c in filename if unicodedata.category(c) not in ("Mn", "Cc"))
|
||||
filename = "".join(c for c in filename if unicodedata.category(c) != "Cc")
|
||||
filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames
|
||||
if spacer == ".":
|
||||
filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern)
|
||||
@@ -143,7 +145,8 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
|
||||
filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s
|
||||
filename = filename.strip(" .") # strip leading and trailing spaces and dots for OS path safety
|
||||
|
||||
return filename
|
||||
# A removed character can leave a letter next to a mark it did not have before.
|
||||
return unicodedata.normalize("NFC", filename)
|
||||
|
||||
|
||||
def is_close_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user