Strict UCS-2 character classification (Unicode 18.0.0) functions (locale-independent).

3 files changed, 125 insertions, 0 deletions

diff --git a/src/ctype/unicode/README.unicode b/src/ctype/unicode/README.unicode
new file mode 100644
index 0000000..48863b1
--- /dev/null
+++ b/src/ctype/unicode/README.unicode
@@ -0,0 +1,21 @@
+LIBMPUIO strict UCS-2 character classification
+==============================================
+
+The distributed mpu-ctype-data.c is a pre-generated table for Unicode 18.0.0.
+The ordinary LIBMPUIO build is completely offline and never downloads Unicode
+files or regenerates this table.
+
+For a maintainer regeneration download, in advance, the official files:
+
+  https://www.unicode.org/Public/18.0.0/ucd/DerivedCoreProperties.txt
+  https://www.unicode.org/Public/18.0.0/ucd/extracted/DerivedGeneralCategory.txt
+
+Then run gen-ctype-table.py explicitly. (see: ./gen-ctype-table.py --help)
+
+The generator reads Alphabetic, Uppercase, Lowercase, XID_Start and XID_Continue
+from DerivedCoreProperties.txt and Nd from DerivedGeneralCategory.txt.  Only the
+BMP U+0000..U+FFFF is emitted.  U+D800..U+DFFF is forced to zero because
+LIBMPUIO uses strict UCS-2 and rejects surrogate code units.
+
+Unicode data and documentation are subject to the Unicode License:
+  https://www.unicode.org/license.txt
diff --git a/src/ctype/unicode/UNICODE-LICENSE b/src/ctype/unicode/UNICODE-LICENSE
new file mode 100644
index 0000000..b946276
--- /dev/null
+++ b/src/ctype/unicode/UNICODE-LICENSE
@@ -0,0 +1,35 @@
+UNICODE LICENSE V3
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright © 1991-2026 Unicode, Inc.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in
+the Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
diff --git a/src/ctype/unicode/gen-ctype-table.py b/src/ctype/unicode/gen-ctype-table.py
new file mode 100755
index 0000000..dc2ece5
--- /dev/null
+++ b/src/ctype/unicode/gen-ctype-table.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Generate mpu-ctype-data.c from official Unicode 18.0.0 UCD files.
+
+This script is a maintainer tool.  It is never run by Makefile/configure.
+Place these already-downloaded files beside the script or pass their paths:
+  DerivedCoreProperties.txt
+  DerivedGeneralCategory.txt
+"""
+import argparse
+import re
+from pathlib import Path
+
+BITS = {
+    "Alphabetic": 0x01,
+    "Uppercase": 0x02,
+    "Lowercase": 0x04,
+    "XID_Start": 0x10,
+    "XID_Continue": 0x20,
+}
+DIGIT = 0x08
+
+def parse_range(s):
+    if ".." in s:
+        a,b=s.split("..",1); return int(a,16),int(b,16)
+    n=int(s,16); return n,n
+
+def apply(path, flags, wanted, general=False):
+    version_ok=False
+    with path.open(encoding="utf-8") as f:
+        for raw in f:
+            if "18.0.0" in raw[:80]: version_ok=True
+            line=raw.split("#",1)[0].strip()
+            if not line or ";" not in line: continue
+            field, prop=(x.strip() for x in line.split(";",1))
+            bit = DIGIT if general and prop == "Nd" else wanted.get(prop)
+            if bit is None: continue
+            a,b=parse_range(field)
+            if a > 0xffff: continue
+            b=min(b,0xffff)
+            for cp in range(a,b+1): flags[cp] |= bit
+    if not version_ok:
+        raise SystemExit(f"{path}: expected Unicode 18.0.0 data")
+
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument("--core",default="DerivedCoreProperties.txt")
+    ap.add_argument("--category",default="DerivedGeneralCategory.txt")
+    ap.add_argument("--output",default="mpu-ctype-data.c")
+    ns=ap.parse_args()
+    flags=[0]*0x10000
+    apply(Path(ns.core),flags,BITS)
+    apply(Path(ns.category),flags,{},general=True)
+    for cp in range(0xd800,0xe000): flags[cp]=0
+    out=Path(ns.output)
+    with out.open("w",encoding="utf-8") as f:
+        f.write("/***************************************************************\n")
+        f.write("  MPU-CTYPE-DATA.C\n\n")
+        f.write("       Generated strict UCS-2 classification table.\n")
+        f.write("       Unicode Character Database version: 18.0.0.\n")
+        f.write(" ***************************************************************/\n\n")
+        f.write("#include <libmpuio.h>\n#include <mpu-symbols.h>\n#include <mpu-ctype.h>\n\n")
+        f.write("__mpu_hidden_decl( __mpu_ucs2_ctype );\n\n")
+        f.write("const unsigned char __mpu_ucs2_ctype[65536] =\n{\n")
+        for base in range(0,0x10000,16):
+            vals=", ".join("0x%02x"%flags[base+i] for i in range(16))
+            f.write("  %s, /* U+%04X..U+%04X */\n"%(vals,base,base+15))
+        f.write("};\n")
+
+if __name__ == "__main__": main()