diff --git a/docs/guide/output-and-naming.md b/docs/guide/output-and-naming.md index d30be8ea..fbdb2cc9 100644 --- a/docs/guide/output-and-naming.md +++ b/docs/guide/output-and-naming.md @@ -380,7 +380,9 @@ unicode_filenames: true | Value | Result | |---|---| | `false` (default) | `기생충` → transliterated ASCII; accents stripped. | -| `true` | Native characters preserved in filenames and folders. | +| `true` | Native characters preserved in filenames and folders, with the vowel and tone marks of languages such as Thai, Hindi and Bengali. | + +With each value, unshackle first normalises the name to Unicode NFC. A title that a service sends in composed form and in decomposed form gives the same name. Regardless of this setting, unshackle always removes or replaces a set of filesystem-unsafe and structural characters during sanitisation. For example, `/` and `;` become ` & `, and unshackle removes characters like `\ * ! ? , ' " < > | $ # ~`. This is why you must never put path-unsafe characters directly in a template. diff --git a/tests/core/test_sanitize_filename.py b/tests/core/test_sanitize_filename.py index b4b9ee13..ce057706 100644 --- a/tests/core/test_sanitize_filename.py +++ b/tests/core/test_sanitize_filename.py @@ -1,47 +1,75 @@ -"""Nonspacing marks in sanitize_filename. +"""Combining marks and Unicode normalisation in sanitize_filename. -Thai (and Devanagari, Khmer, ...) write vowels and tone marks as nonspacing marks (Mn) on a -base letter. With ``unicode_filenames`` those marks are part of the word and must survive, -while hidden marks with no letter to sit on are still removed. +Thai, Devanagari, Bengali and other scripts write vowels and tone marks as combining marks on +a base letter. With ``unicode_filenames`` those marks are part of the word and must survive. +In both modes, composed and decomposed spellings of a title must give the same name. +Combining characters are written as escapes so an editor that normalises the file cannot +change what a case tests. """ from __future__ import annotations import pytest +from unshackle.core.config import config from unshackle.core.utilities import sanitize_filename +THAI = "\u0e40\u0e1b\u0e47\u0e19\u0e15\u0e48\u0e2d" # a Thai title with vowel and tone marks + + +@pytest.mark.parametrize( + "name", + [ + THAI, + "\u0e01\u0e31\u0e49\u0e19", # Thai: a tone mark stacked on an upper vowel + "\u0939\u093f\u0902\u0926\u0940", # Devanagari + "\u09b0\u200d\u09cd\u09af\u09be\u09ac", # Bengali: the virama follows a zero width joiner + "\u1040\u102d\u102f\u1004\u103a\u1038", # Burmese: a mark on digit zero, typed for the letter wa + "\u0e19\u0e49\u0e33", # Thai sara am, which NFKC would split into two characters + ], +) +def test_unicode_filenames_keep_marks(name: str) -> None: + assert sanitize_filename(name, unicode=True) == name + @pytest.mark.parametrize( ("name", "expected"), [ - # Thai vowel and tone marks stay on their letters ("เป็นต่อ" used to become "เปนตอ"). - ("เป็นต่อ 2026", "เป็นต่อ.2026"), - ("บัญชีดำอาชญากรรมซ่อนเงื่อน", "บัญชีดำอาชญากรรมซ่อนเงื่อน"), - # Stacked marks: a tone mark on top of an upper vowel. - ("กั้น", "กั้น"), - ("हिंदी", "हिंदी"), - # A decomposed accent keeps its letter's accent. - ("Café", "Café"), - # Hidden marks with nothing to attach to are still removed. - ("́Title", "Title"), - ("Title ́x", "Title.x"), - # A variation selector on an emoji is not a letter mark. - ("Show ❤️", "Show.❤"), - # Control characters are always removed. + # Decomposed and composed spellings give the same name. + ("Cafe\u0301", "Caf\u00e9"), + ("Caf\u00e9", "Caf\u00e9"), + ("Show \u2764\ufe0f", "Show.\u2764\ufe0f"), ("Some\x07Title", "SomeTitle"), ], ) -def test_unicode_filenames_keep_marks_on_letters(name: str, expected: str) -> None: +def test_unicode_filenames_use_nfc_and_drop_control_characters(name: str, expected: str) -> None: assert sanitize_filename(name, unicode=True) == expected @pytest.mark.parametrize( ("name", "expected"), [ - ("Café Name", "Cafe.Name"), + ("Cafe\u0301 Name", "Cafe.Name"), + ("Caf\u00e9 Name", "Cafe.Name"), + # Katakana with separate voicing marks transliterates like the composed spelling. + ("\u30ab\u3099\u30f3\u30bf\u3099\u30e0", "gandamu"), + ("\u30ac\u30f3\u30c0\u30e0", "gandamu"), ("Some\x07Title", "SomeTitle"), ], ) -def test_ascii_filenames_drop_all_marks(name: str, expected: str) -> None: +def test_ascii_filenames_transliterate_both_spellings(name: str, expected: str) -> None: assert sanitize_filename(name, unicode=False) == expected + + +def test_config_option_decides_when_unicode_is_not_passed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(config, "unicode_filenames", True) + assert sanitize_filename(THAI) == THAI + monkeypatch.setattr(config, "unicode_filenames", False) + assert sanitize_filename(THAI).isascii() + + +def test_sanitising_a_name_again_does_not_change_it() -> None: + # The removed "?" sits between the letter and its mark. + once = sanitize_filename("e?\u0301", unicode=True) + assert once == "\u00e9" + assert sanitize_filename(once, unicode=True) == once diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index f1259a1a..39455238 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -118,25 +118,6 @@ def import_module_by_path(path: Path) -> ModuleType: return module -def _strip_hidden_characters(text: str, keep_letter_marks: bool) -> str: - """Drop control characters (Cc) and hidden nonspacing marks (Mn). - - Several scripts write vowels and tone marks as nonspacing marks on a base letter, e.g. - Thai "เป็นต่อ" or Devanagari "हिंदी"; dropping those changes the word ("เปนตอ"). With - ``keep_letter_marks`` a mark that sits on a letter (or on another such mark) is kept, while - one with nothing to attach to, or on a symbol (a variation selector after an emoji), is not. - """ - kept: list[str] = [] - for c in text: - category = unicodedata.category(c) - if category == "Cc": - continue - if category == "Mn" and not (keep_letter_marks and kept and unicodedata.category(kept[-1])[0] in "LM"): - continue - kept.append(c) - return "".join(kept) - - def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool] = None) -> str: """ Sanitise a string to be filename safe. @@ -147,14 +128,15 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool] Set `unicode_filenames: true` in config to preserve the characters of the original language (for example Korean, Japanese, or Chinese) instead of transliterating them to ASCII equivalents. Pass ``unicode`` to decide that - for one call instead of the config value. + for one call instead of the config value. The name goes to NFC first, so + composed and decomposed spellings of a title give the same name. """ - keep_unicode = config.unicode_filenames if unicode is None else unicode - if not keep_unicode: + filename = unicodedata.normalize("NFC", filename) + if not (config.unicode_filenames if unicode is None else unicode): filename = unidecode(filename) filename = re.sub(r"\[\(+", "[", filename) filename = re.sub(r"\)+\]", "]", filename) - filename = _strip_hidden_characters(filename, keep_letter_marks=keep_unicode) + filename = "".join(c for c in filename if unicodedata.category(c) != "Cc") filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames if spacer == ".": filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern) @@ -163,7 +145,8 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool] filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s filename = filename.strip(" .") # strip leading and trailing spaces and dots for OS path safety - return filename + # A removed character can leave a letter next to a mark it did not have before. + return unicodedata.normalize("NFC", filename) def is_close_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: