fix(sanitize): keep nonspacing marks on letters in unicode filenames (#162)

tests/core/test_sanitize_filename.py

@@ -0,0 +1,47 @@
+"""Nonspacing marks in sanitize_filename.
+
+Thai (and Devanagari, Khmer, ...) write vowels and tone marks as nonspacing marks (Mn) on a
+base letter. With ``unicode_filenames`` those marks are part of the word and must survive,
+while hidden marks with no letter to sit on are still removed.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from unshackle.core.utilities import sanitize_filename
+
+
+@pytest.mark.parametrize(
+    ("name", "expected"),
+    [
+        # Thai vowel and tone marks stay on their letters ("เป็นต่อ" used to become "เปนตอ").
+        ("เป็นต่อ 2026", "เป็นต่อ.2026"),
+        ("บัญชีดำอาชญากรรมซ่อนเงื่อน", "บัญชีดำอาชญากรรมซ่อนเงื่อน"),
+        # Stacked marks: a tone mark on top of an upper vowel.
+        ("กั้น", "กั้น"),
+        ("हिंदी", "हिंदी"),
+        # A decomposed accent keeps its letter's accent.
+        ("Café", "Café"),
+        # Hidden marks with nothing to attach to are still removed.
+        ("́Title", "Title"),
+        ("Title ́x", "Title.x"),
+        # A variation selector on an emoji is not a letter mark.
+        ("Show ❤️", "Show.❤"),
+        # Control characters are always removed.
+        ("Some\x07Title", "SomeTitle"),
+    ],
+)
+def test_unicode_filenames_keep_marks_on_letters(name: str, expected: str) -> None:
+    assert sanitize_filename(name, unicode=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("name", "expected"),
+    [
+        ("Café Name", "Cafe.Name"),
+        ("Some\x07Title", "SomeTitle"),
+    ],
+)
+def test_ascii_filenames_drop_all_marks(name: str, expected: str) -> None:
+    assert sanitize_filename(name, unicode=False) == expected

unshackle/core/utilities.py

@@ -118,6 +118,25 @@ def import_module_by_path(path: Path) -> ModuleType:
     return module
 
 
+def _strip_hidden_characters(text: str, keep_letter_marks: bool) -> str:
+    """Drop control characters (Cc) and hidden nonspacing marks (Mn).
+
+    Several scripts write vowels and tone marks as nonspacing marks on a base letter, e.g.
+    Thai "เป็นต่อ" or Devanagari "हिंदी"; dropping those changes the word ("เปนตอ"). With
+    ``keep_letter_marks`` a mark that sits on a letter (or on another such mark) is kept, while
+    one with nothing to attach to, or on a symbol (a variation selector after an emoji), is not.
+    """
+    kept: list[str] = []
+    for c in text:
+        category = unicodedata.category(c)
+        if category == "Cc":
+            continue
+        if category == "Mn" and not (keep_letter_marks and kept and unicodedata.category(kept[-1])[0] in "LM"):
+            continue
+        kept.append(c)
+    return "".join(kept)
+
+
 def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool] = None) -> str:
     """
     Sanitise a string to be filename safe.
@@ -130,11 +149,12 @@ def sanitize_filename(filename: str, spacer: str = ".", unicode: Optional[bool]
     transliterating them to ASCII equivalents. Pass ``unicode`` to decide that
     for one call instead of the config value.
     """
-    if not (config.unicode_filenames if unicode is None else unicode):
+    keep_unicode = config.unicode_filenames if unicode is None else unicode
+    if not keep_unicode:
         filename = unidecode(filename)
         filename = re.sub(r"\[\(+", "[", filename)
         filename = re.sub(r"\)+\]", "]", filename)
-    filename = "".join(c for c in filename if unicodedata.category(c) not in ("Mn", "Cc"))
+    filename = _strip_hidden_characters(filename, keep_letter_marks=keep_unicode)
     filename = filename.replace("/", " & ").replace(";", " & ")  # e.g. multi-episode filenames
     if spacer == ".":
         filename = re.sub(r" - ", spacer, filename)  # title separators to spacer (avoids .-. pattern)