fix(sanitize): normalise filenames to NFC and keep combining marks

Remove only control characters in unicode mode. A mark after a zero width joiner or a digit was still dropped (#161). In ASCII mode a decomposed title now transliterates like the composed one.

Names change for titles with combining marks, compatibility ideographs or decomposed spellings, so an existing library can get a second folder.

docs/guide/output-and-naming.md

@@ -380,7 +380,9 @@ unicode_filenames: true
 | Value | Result |
 |---|---|
 | `false` (default) | `기생충` → transliterated ASCII; accents stripped. |
-| `true` | Native characters preserved in filenames and folders. |
+| `true` | Native characters preserved in filenames and folders, with the vowel and tone marks of languages such as Thai, Hindi and Bengali. |
+
+With each value, unshackle first normalises the name to Unicode NFC. A title that a service sends in composed form and in decomposed form gives the same name.
 
 Regardless of this setting, unshackle always removes or replaces a set of filesystem-unsafe and structural characters during sanitisation. For example, `/` and `;` become ` & `, and unshackle removes characters like `\ * ! ? , ' " < > | $ # ~`. This is why you must never put path-unsafe characters directly in a template.
 

tests/core/test_sanitize_filename.py

@@ -1,47 +1,75 @@
-"""Nonspacing marks in sanitize_filename.
+"""Combining marks and Unicode normalisation in sanitize_filename.
 
-Thai (and Devanagari, Khmer, ...) write vowels and tone marks as nonspacing marks (Mn) on a
+Thai, Devanagari, Bengali and other scripts write vowels and tone marks as combining marks on
-base letter. With ``unicode_filenames`` those marks are part of the word and must survive,
+a base letter. With ``unicode_filenames`` those marks are part of the word and must survive.
-while hidden marks with no letter to sit on are still removed.
+In both modes, composed and decomposed spellings of a title must give the same name.
+Combining characters are written as escapes so an editor that normalises the file cannot
+change what a case tests.
 """
 
 from __future__ import annotations
 
 import pytest
 
+from unshackle.core.config import config
 from unshackle.core.utilities import sanitize_filename
 
+THAI = "\u0e40\u0e1b\u0e47\u0e19\u0e15\u0e48\u0e2d"  # a Thai title with vowel and tone marks
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        THAI,
+        "\u0e01\u0e31\u0e49\u0e19",  # Thai: a tone mark stacked on an upper vowel
+        "\u0939\u093f\u0902\u0926\u0940",  # Devanagari
+        "\u09b0\u200d\u09cd\u09af\u09be\u09ac",  # Bengali: the virama follows a zero width joiner
+        "\u1040\u102d\u102f\u1004\u103a\u1038",  # Burmese: a mark on digit zero, typed for the letter wa
+        "\u0e19\u0e49\u0e33",  # Thai sara am, which NFKC would split into two characters
+    ],
+)
+def test_unicode_filenames_keep_marks(name: str) -> None:
+    assert sanitize_filename(name, unicode=True) == name
+
 
 @pytest.mark.parametrize(
     ("name", "expected"),
     [
-        # Thai vowel and tone marks stay on their letters ("เป็นต่อ" used to become "เปนตอ").
+        # Decomposed and composed spellings give the same name.
-        ("เป็นต่อ 2026", "เป็นต่อ.2026"),
+        ("Cafe\u0301", "Caf\u00e9"),
-        ("บัญชีดำอาชญากรรมซ่อนเงื่อน", "บัญชีดำอาชญากรรมซ่อนเงื่อน"),
+        ("Caf\u00e9", "Caf\u00e9"),
-        # Stacked marks: a tone mark on top of an upper vowel.
+        ("Show \u2764\ufe0f", "Show.\u2764\ufe0f"),
-        ("กั้น", "กั้น"),
-        ("हिंदी", "हिंदी"),
-        # A decomposed accent keeps its letter's accent.
-        ("Café", "Café"),
-        # Hidden marks with nothing to attach to are still removed.
-        ("́Title", "Title"),
-        ("Title ́x", "Title.x"),
-        # A variation selector on an emoji is not a letter mark.
-        ("Show ❤️", "Show.❤"),
-        # Control characters are always removed.
         ("Some\x07Title", "SomeTitle"),
     ],
 )
-def test_unicode_filenames_keep_marks_on_letters(name: str, expected: str) -> None:
+def test_unicode_filenames_use_nfc_and_drop_control_characters(name: str, expected: str) -> None:
     assert sanitize_filename(name, unicode=True) == expected
 
 
 @pytest.mark.parametrize(
     ("name", "expected"),
     [
-        ("Café Name", "Cafe.Name"),
+        ("Cafe\u0301 Name", "Cafe.Name"),
+        ("Caf\u00e9 Name", "Cafe.Name"),
+        # Katakana with separate voicing marks transliterates like the composed spelling.
+        ("\u30ab\u3099\u30f3\u30bf\u3099\u30e0", "gandamu"),
+        ("\u30ac\u30f3\u30c0\u30e0", "gandamu"),
         ("Some\x07Title", "SomeTitle"),
     ],
 )
-def test_ascii_filenames_drop_all_marks(name: str, expected: str) -> None:
+def test_ascii_filenames_transliterate_both_spellings(name: str, expected: str) -> None:
     assert sanitize_filename(name, unicode=False) == expected
+
+
+def test_config_option_decides_when_unicode_is_not_passed(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(config, "unicode_filenames", True)
+    assert sanitize_filename(THAI) == THAI
+    monkeypatch.setattr(config, "unicode_filenames", False)
+    assert sanitize_filename(THAI).isascii()
+
+
+def test_sanitising_a_name_again_does_not_change_it() -> None:
+    # The removed "?" sits between the letter and its mark.
+    once = sanitize_filename("e?\u0301", unicode=True)
+    assert once == "\u00e9"
+    assert sanitize_filename(once, unicode=True) == once

unshackle/core/utilities.py

@@ -118,25 +118,6 @@ def import_module_by_path(path: Path) -> ModuleType:
     return module
 
 
-def _strip_hidden_characters(text: str, keep_letter_marks: bool) -> str:
-    """Drop control characters (Cc) and hidden nonspacing marks (Mn).
-
-    Several scripts write vowels and tone marks as nonspacing marks on a base letter, e.g.
-    Thai "เป็นต่อ" or Devanagari "हिंदी"; dropping those changes the word ("เปนตอ"). With
-    ``keep_letter_marks`` a mark that sits on a letter (or on another such mark) is kept, while
-    one with nothing to attach to, or on a symbol (a variation selector after an emoji), is not.
-    """
-    kept: list[str] = []
-    for c in text:
-        category = unicodedata.category(c)
-        if category == "Cc":
-            continue
-        if category == "Mn" and not (keep_letter_marks and kept and unicodedata.category(kept[-1])[0] in "LM"):
-            continue
-        kept.append(c)
-    return "".join(kept)
-
-
 def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool] = None) -> str:
     """
     Sanitise a string to be filename safe.
@@ -147,14 +128,15 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
     Set `unicode_filenames: true` in config to preserve the characters of the
     original language (for example Korean, Japanese, or Chinese) instead of
     transliterating them to ASCII equivalents. Pass ``unicode`` to decide that
-    for one call instead of the config value.
+    for one call instead of the config value. The name goes to NFC first, so
+    composed and decomposed spellings of a title give the same name.
     """
-    keep_unicode = config.unicode_filenames if unicode is None else unicode
+    filename = unicodedata.normalize("NFC", filename)
-    if not keep_unicode:
+    if not (config.unicode_filenames if unicode is None else unicode):
         filename = unidecode(filename)
         filename = re.sub(r"\[\(+", "[", filename)
         filename = re.sub(r"\)+\]", "]", filename)
-    filename = _strip_hidden_characters(filename, keep_letter_marks=keep_unicode)
+    filename = "".join(c for c in filename if unicodedata.category(c) != "Cc")
     filename = filename.replace("/", " & ").replace(";", " & ")  # e.g. multi-episode filenames
     if spacer == ".":
         filename = re.sub(r" - ", spacer, filename)  # title separators to spacer (avoids .-. pattern)
@@ -163,7 +145,8 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
     filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename)  # remove extra neighbouring (spacer)s
     filename = filename.strip(" .")  # strip leading and trailing spaces and dots for OS path safety
 
-    return filename
+    # A removed character can leave a letter next to a mark it did not have before.
+    return unicodedata.normalize("NFC", filename)
 
 
 def is_close_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: