mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 03:55:05 +00:00
imSp4rky
34eb94b55d
Remove only control characters in unicode mode. A mark after a zero width joiner or a digit was still dropped (#161). In ASCII mode a decomposed title now transliterates like the composed one. Names change for titles with combining marks, compatibility ideographs or decomposed spellings, so an existing library can get a second folder.
76 lines
2.8 KiB
Python
76 lines
2.8 KiB
Python
"""Combining marks and Unicode normalisation in sanitize_filename.
|
|
|
|
Thai, Devanagari, Bengali and other scripts write vowels and tone marks as combining marks on
|
|
a base letter. With ``unicode_filenames`` those marks are part of the word and must survive.
|
|
In both modes, composed and decomposed spellings of a title must give the same name.
|
|
Combining characters are written as escapes so an editor that normalises the file cannot
|
|
change what a case tests.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from unshackle.core.config import config
|
|
from unshackle.core.utilities import sanitize_filename
|
|
|
|
THAI = "\u0e40\u0e1b\u0e47\u0e19\u0e15\u0e48\u0e2d" # a Thai title with vowel and tone marks
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name",
|
|
[
|
|
THAI,
|
|
"\u0e01\u0e31\u0e49\u0e19", # Thai: a tone mark stacked on an upper vowel
|
|
"\u0939\u093f\u0902\u0926\u0940", # Devanagari
|
|
"\u09b0\u200d\u09cd\u09af\u09be\u09ac", # Bengali: the virama follows a zero width joiner
|
|
"\u1040\u102d\u102f\u1004\u103a\u1038", # Burmese: a mark on digit zero, typed for the letter wa
|
|
"\u0e19\u0e49\u0e33", # Thai sara am, which NFKC would split into two characters
|
|
],
|
|
)
|
|
def test_unicode_filenames_keep_marks(name: str) -> None:
|
|
assert sanitize_filename(name, unicode=True) == name
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("name", "expected"),
|
|
[
|
|
# Decomposed and composed spellings give the same name.
|
|
("Cafe\u0301", "Caf\u00e9"),
|
|
("Caf\u00e9", "Caf\u00e9"),
|
|
("Show \u2764\ufe0f", "Show.\u2764\ufe0f"),
|
|
("Some\x07Title", "SomeTitle"),
|
|
],
|
|
)
|
|
def test_unicode_filenames_use_nfc_and_drop_control_characters(name: str, expected: str) -> None:
|
|
assert sanitize_filename(name, unicode=True) == expected
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("name", "expected"),
|
|
[
|
|
("Cafe\u0301 Name", "Cafe.Name"),
|
|
("Caf\u00e9 Name", "Cafe.Name"),
|
|
# Katakana with separate voicing marks transliterates like the composed spelling.
|
|
("\u30ab\u3099\u30f3\u30bf\u3099\u30e0", "gandamu"),
|
|
("\u30ac\u30f3\u30c0\u30e0", "gandamu"),
|
|
("Some\x07Title", "SomeTitle"),
|
|
],
|
|
)
|
|
def test_ascii_filenames_transliterate_both_spellings(name: str, expected: str) -> None:
|
|
assert sanitize_filename(name, unicode=False) == expected
|
|
|
|
|
|
def test_config_option_decides_when_unicode_is_not_passed(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(config, "unicode_filenames", True)
|
|
assert sanitize_filename(THAI) == THAI
|
|
monkeypatch.setattr(config, "unicode_filenames", False)
|
|
assert sanitize_filename(THAI).isascii()
|
|
|
|
|
|
def test_sanitising_a_name_again_does_not_change_it() -> None:
|
|
# The removed "?" sits between the letter and its mark.
|
|
once = sanitize_filename("e?\u0301", unicode=True)
|
|
assert once == "\u00e9"
|
|
assert sanitize_filename(once, unicode=True) == once
|