diff --git a/Cargo.lock b/Cargo.lock index fa5e7b52236..3b7364c5327 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3548,6 +3548,7 @@ dependencies = [ "optional", "rustpython-unicode", "rustpython-wtf8", + "unic-ucd-category", ] [[package]] diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 5cc1b04b9fc..4d71be5ef4e 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -448,7 +448,6 @@ pub(crate) const fn is_uni_linebreak(ch: u32) -> bool { #[inline] pub(crate) fn is_uni_alnum(ch: u32) -> bool { - // TODO: check with cpython char::try_from(ch).is_ok_and(rustpython_unicode::classify::is_alnum) } diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index 684bd66a1ff..611f89c95a5 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -902,6 +902,15 @@ class MyString(str): assert id(b) != id(b * 2) +# Regression tests for isalpha/isalnum Unicode General Category correctness. +# These characters are in letter categories (Ll/Lo) and should return True, +# but were missed in older Unicode tables used by unic-ucd-category. +# See: https://github.com/RustPython/RustPython/pull/7520#issuecomment-4148322294 +for _cp in [1376, 1416, 1519, 2160, 2161, 2162, 2163, 2164, 2165, 2166]: + _c = chr(_cp) + assert _c.isalpha(), f"U+{_cp:04X} should be isalpha" + assert _c.isalnum(), f"U+{_cp:04X} should be isalnum" + def test_huge_width(): # A width that cannot be allocated is a MemoryError, not an aborted # process, and a tabsize wider than a C int does not fit at all. diff --git a/extra_tests/snippets/builtin_str_unicode.py b/extra_tests/snippets/builtin_str_unicode.py index ae2701d05ea..c83ff853523 100644 --- a/extra_tests/snippets/builtin_str_unicode.py +++ b/extra_tests/snippets/builtin_str_unicode.py @@ -11,6 +11,7 @@ assert c == "👋👋👋" +import re import unicodedata assert unicodedata.category("a") == "Ll" @@ -44,3 +45,10 @@ assert unicodedata.ucd_3_2_0.east_asian_width("\u231a") == "N" assert not unicodedata.ucd_3_2_0.mirrored("\u0f3a") + +# U+0345 COMBINING GREEK YPOGEGRAMMENI (category Mn) should not be alphanumeric. +# CPython's isalpha/isalnum use Unicode letter categories (Lu/Ll/Lt/Lm/Lo), +# not the broader Unicode Alphabetic derived property. +assert not "\u0345".isalpha(), "isalpha should not match Mn category characters" +assert not "\u0345".isalnum(), "isalnum should not match Mn category characters" +assert not re.match(r"\w", "\u0345"), r"\w should not match U+0345 (category Mn)"