Compare commits

...
3 Commits
Author SHA1 Message Date
imSp4rky 73bdb98d3e fix(console): write UTF-8 to redirected streams on Windows 2026-10-02 14:07:05 -06:00
imSp4rky 34eb94b55d fix(sanitize): normalise filenames to NFC and keep combining marks
Remove only control characters in unicode mode. A mark after a zero width joiner or a digit was still dropped (#161). In ASCII mode a decomposed title now transliterates like the composed one.

Names change for titles with combining marks, compatibility ideographs or decomposed spellings, so an existing library can get a second folder.
2026-10-02 13:22:04 -06:00
Chaloemchai 7b3794333f fix(sanitize): keep nonspacing marks on letters in unicode filenames (#162) 2026-10-02 11:56:56 -06:00
5 changed files with 111 additions and 5 deletions
+3 -1
View File
@@ -380,7 +380,9 @@ unicode_filenames: true
| Value | Result |
|---|---|
| `false` (default) | `기생충` → transliterated ASCII; accents stripped. |
| `true` | Native characters preserved in filenames and folders. |
| `true` | Native characters preserved in filenames and folders, with the vowel and tone marks of languages such as Thai, Hindi and Bengali. |
With each value, unshackle first normalises the name to Unicode NFC. A title that a service sends in composed form and in decomposed form gives the same name.
Regardless of this setting, unshackle always removes or replaces a set of filesystem-unsafe and structural characters during sanitisation. For example, `/` and `;` become ` & `, and unshackle removes characters like `\ * ! ? , ' " < > | $ # ~`. This is why you must never put path-unsafe characters directly in a template.
+15 -1
View File
@@ -2,12 +2,16 @@
from __future__ import annotations
import io
import sys
import pytest
from click.testing import CliRunner
from unshackle.core import __version__
from unshackle.core.__main__ import main
from unshackle.core.__main__ import main, print_banner
from unshackle.core.config import config
from unshackle.core.console import force_utf8_streams
@pytest.mark.parametrize("args", [["--help"], ["env", "--help"]])
@@ -19,3 +23,13 @@ def test_help_prints_the_banner_once(monkeypatch: pytest.MonkeyPatch, args: list
assert result.exit_code == 0, result.output
assert result.output.count(f"v {__version__}") == 1
assert "Usage:" in result.output
def test_banner_prints_to_a_legacy_code_page_stream(monkeypatch: pytest.MonkeyPatch) -> None:
buffer = io.BytesIO()
monkeypatch.setattr(sys, "stdout", io.TextIOWrapper(buffer, encoding="cp1252"))
force_utf8_streams()
print_banner()
assert "▄".encode() in buffer.getvalue()
+75
View File
@@ -0,0 +1,75 @@
"""Combining marks and Unicode normalisation in sanitize_filename.
Thai, Devanagari, Bengali and other scripts write vowels and tone marks as combining marks on
a base letter. With ``unicode_filenames`` those marks are part of the word and must survive.
In both modes, composed and decomposed spellings of a title must give the same name.
Combining characters are written as escapes so an editor that normalises the file cannot
change what a case tests.
"""
from __future__ import annotations
import pytest
from unshackle.core.config import config
from unshackle.core.utilities import sanitize_filename
THAI = "\u0e40\u0e1b\u0e47\u0e19\u0e15\u0e48\u0e2d" # a Thai title with vowel and tone marks
@pytest.mark.parametrize(
"name",
[
THAI,
"\u0e01\u0e31\u0e49\u0e19", # Thai: a tone mark stacked on an upper vowel
"\u0939\u093f\u0902\u0926\u0940", # Devanagari
"\u09b0\u200d\u09cd\u09af\u09be\u09ac", # Bengali: the virama follows a zero width joiner
"\u1040\u102d\u102f\u1004\u103a\u1038", # Burmese: a mark on digit zero, typed for the letter wa
"\u0e19\u0e49\u0e33", # Thai sara am, which NFKC would split into two characters
],
)
def test_unicode_filenames_keep_marks(name: str) -> None:
assert sanitize_filename(name, unicode=True) == name
@pytest.mark.parametrize(
("name", "expected"),
[
# Decomposed and composed spellings give the same name.
("Cafe\u0301", "Caf\u00e9"),
("Caf\u00e9", "Caf\u00e9"),
("Show \u2764\ufe0f", "Show.\u2764\ufe0f"),
("Some\x07Title", "SomeTitle"),
],
)
def test_unicode_filenames_use_nfc_and_drop_control_characters(name: str, expected: str) -> None:
assert sanitize_filename(name, unicode=True) == expected
@pytest.mark.parametrize(
("name", "expected"),
[
("Cafe\u0301 Name", "Cafe.Name"),
("Caf\u00e9 Name", "Cafe.Name"),
# Katakana with separate voicing marks transliterates like the composed spelling.
("\u30ab\u3099\u30f3\u30bf\u3099\u30e0", "gandamu"),
("\u30ac\u30f3\u30c0\u30e0", "gandamu"),
("Some\x07Title", "SomeTitle"),
],
)
def test_ascii_filenames_transliterate_both_spellings(name: str, expected: str) -> None:
assert sanitize_filename(name, unicode=False) == expected
def test_config_option_decides_when_unicode_is_not_passed(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config, "unicode_filenames", True)
assert sanitize_filename(THAI) == THAI
monkeypatch.setattr(config, "unicode_filenames", False)
assert sanitize_filename(THAI).isascii()
def test_sanitising_a_name_again_does_not_change_it() -> None:
# The removed "?" sits between the letter and its mark.
once = sanitize_filename("e?\u0301", unicode=True)
assert once == "\u00e9"
assert sanitize_filename(once, unicode=True) == once
+12
View File
@@ -1,6 +1,9 @@
import codecs
import io
import logging
import math
import shutil
import sys
from datetime import datetime
from types import ModuleType
from typing import IO, Any, Callable, Iterable, List, Literal, Mapping, Optional, TextIO, Union
@@ -402,6 +405,15 @@ class _GradientPulse:
return Measurement(4, self.width or options.max_width)
def force_utf8_streams() -> None:
"""Make stdout and stderr write UTF-8. A redirected stream on Windows uses a legacy code page that cannot encode the console output."""
for stream in (sys.stdout, sys.stderr):
if isinstance(stream, io.TextIOWrapper) and codecs.lookup(stream.encoding).name != "utf-8":
stream.reconfigure(encoding="utf-8", errors="replace")
force_utf8_streams()
console = ComfyConsole(
log_time=False,
log_path=False,
+6 -3
View File
@@ -128,13 +128,15 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
Set `unicode_filenames: true` in config to preserve the characters of the
original language (for example Korean, Japanese, or Chinese) instead of
transliterating them to ASCII equivalents. Pass ``unicode`` to decide that
for one call instead of the config value.
for one call instead of the config value. The name goes to NFC first, so
composed and decomposed spellings of a title give the same name.
"""
filename = unicodedata.normalize("NFC", filename)
if not (config.unicode_filenames if unicode is None else unicode):
filename = unidecode(filename)
filename = re.sub(r"\[\(+", "[", filename)
filename = re.sub(r"\)+\]", "]", filename)
filename = "".join(c for c in filename if unicodedata.category(c) not in ("Mn", "Cc"))
filename = "".join(c for c in filename if unicodedata.category(c) != "Cc")
filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames
if spacer == ".":
filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern)
@@ -143,7 +145,8 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s
filename = filename.strip(" .") # strip leading and trailing spaces and dots for OS path safety
return filename
# A removed character can leave a letter next to a mark it did not have before.
return unicodedata.normalize("NFC", filename)
def is_close_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: