#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 # # Leverage Python's unicodedata module to generate ucs_width_table.h import unicodedata import sys import argparse # This script's file name from pathlib import Path this_file = Path(__file__).name # Default output file name DEFAULT_OUT_FILE = "ucs_width_table.h" # --- Global Constants for Width Assignments --- # Known zero-width characters KNOWN_ZERO_WIDTH = ( 0x200B, # ZERO WIDTH SPACE 0x200C, # ZERO WIDTH NON-JOINER 0x200D, # ZERO WIDTH JOINER 0x2060, # WORD JOINER 0xFEFF # ZERO WIDTH NO-BREAK SPACE (BOM) ) # Zero-width emoji modifiers and components # NOTE: Some of these characters would normally be single-width according to # East Asian Width properties, but we deliberately override them to be # zero-width because they function as modifiers in emoji sequences. EMOJI_ZERO_WIDTH = [ # Skin tone modifiers (0x1F3FB, 0x1F3FF), # Emoji modifiers (skin tones) # Variation selectors (note: VS16 is treated specially in vt.c) (0xFE00, 0xFE0F), # Variation Selectors 1-16 # Gender and hair style modifiers # These would be single-width by Unicode properties, but are zero-width # when part of emoji (0x2640, 0x2640), # Female sign (0x2642, 0x2642), # Male sign (0x26A7, 0x26A7), # Transgender symbol (0x1F9B0, 0x1F9B3), # Hair components (red, curly, white, bald) # Tag characters (0xE0020, 0xE007E), # Tags ] # Regional indicators (flag components) REGIONAL_INDICATORS = (0x1F1E6, 0x1F1FF) # Regional indicator symbols A-Z # Double-width emoji ranges # # Many emoji characters are classified as single-width according to Unicode # Standard Annex #11 East Asian Width property (N or Neutral), but we # deliberately override them to be double-width. References: # 1. Unicode Technical Standard #51: Unicode Emoji # (https://www.unicode.org/reports/tr51/) # 2. Principle of "emoji presentation" in WHATWG CSS Text specification # (https://drafts.csswg.org/css-text-3/#character-properties) # 3. Terminal emulator implementations (iTerm2, Windows Terminal, etc.) which # universally render emoji as double-width characters regardless of their # Unicode EAW property # 4. W3C Work Item: Requirements for Japanese Text Layout - Section 3.8.1 # Emoji width (https://www.w3.org/TR/jlreq/) EMOJI_RANGES = [ (0x1F000, 0x1F02F), # Mahjong Tiles (EAW: N, but displayed as double-width) (0x1F0A0, 0x1F0FF), # Playing Cards (EAW: N, but displayed as double-width) (0x1F300, 0x1F5FF), # Miscellaneous Symbols and Pictographs (0x1F600, 0x1F64F), # Emoticons (0x1F680, 0x1F6FF), # Transport and Map Symbols (0x1F700, 0x1F77F), # Alchemical Symbols (0x1F780, 0x1F7FF), # Geometric Shapes Extended (0x1F800, 0x1F8FF), # Supplemental Arrows-C (0x1F900, 0x1F9FF), # Supplemental Symbols and Pictographs (0x1FA00, 0x1FA6F), # Chess Symbols (0x1FA70, 0x1FAFF), # Symbols and Pictographs Extended-A ] def create_width_tables(): """ Creates Unicode character width tables and returns the data structures. Returns: tuple: (zero_width_ranges, double_width_ranges) """ # Width data mapping width_map = {} # Maps code points to width (0, 1, 2) # Mark emoji modifiers as zero-width for start, end in EMOJI_ZERO_WIDTH: for cp in range(start, end + 1): width_map[cp] = 0 # Mark all regional indicators as single-width as they are usually paired # providing a combined width of 2 when displayed together. start, end = REGIONAL_INDICATORS for cp in range(start, end + 1): width_map[cp] = 1 # Process all assigned Unicode code points (Basic Multilingual Plane + # Supplementary Planes) Range 0x0 to 0x10FFFF (the full Unicode range) for block_start in range(0, 0x110000, 0x1000): block_end = block_start + 0x1000 for cp in range(block_start, block_end): try: char = chr(cp) # Skip if already processed if cp in width_map: continue # Check for combining marks and a format characters category = unicodedata.category(char) # Combining marks if category.startswith('M'): width_map[cp] = 0 continue # Format characters # Since we have no support for bidirectional text, all format # characters (category Cf) can be treated with width 0 (zero) # for simplicity, as they don't need to occupy visual space # in a non-bidirectional text environment. if category == 'Cf': width_map[cp] = 0 continue # Known zero-width characters if cp in KNOWN_ZERO_WIDTH: width_map[cp] = 0 continue # Use East Asian Width property eaw = unicodedata.east_asian_width(char) if eaw in ('F', 'W'): # Fullwidth or Wide width_map[cp] = 2 elif eaw in ('Na', 'H', 'N', 'A'): # Narrow, Halfwidth, Neutral, Ambiguous width_map[cp] = 1 else: # Default to single-width for unknown width_map[cp] = 1 except (ValueError, OverflowError): # Skip invalid code points continue # Process Emoji - generally double-width for start, end in EMOJI_RANGES: for cp in range(start, end + 1): if cp not in width_map or width_map[cp] != 0: # Don't override zero-width try: char = chr(cp) width_map[cp] = 2 except (ValueError, OverflowError): continue # Optimize to create range tables def ranges_optimize(width_data, target_width): points = sorted([cp for cp, width in width_data.items() if width == target_width]) if not points: return [] # Group consecutive code points into ranges ranges = [] start = points[0] prev = start for cp in points[1:]: if cp > prev + 1: ranges.append((start, prev)) start = cp prev = cp # Add the last range ranges.append((start, prev)) return ranges # Extract ranges for each width zero_width_ranges = ranges_optimize(width_map, 0) double_width_ranges = ranges_optimize(width_map, 2) return zero_width_ranges, double_width_ranges def write_tables(zero_width_ranges, double_width_ranges, out_file=DEFAULT_OUT_FILE): """ Write the generated tables to C header file. The output uses a single sorted-by-`first` table per region (BMP and non-BMP), with zero-width and double-width ranges merged together. The non-BMP table also hosts the BMP double-width bitmap in spare bits of `last`. See the encoding comment at the top of ucs.c for the layout. Args: zero_width_ranges: List of (start, end) ranges for zero-width characters double_width_ranges: List of (start, end) ranges for double-width characters out_file: Output file name (default: DEFAULT_OUT_FILE) """ # Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field. # 8 bits makes `idx / BITS_PER_CHUNK` / `idx % BITS_PER_CHUNK` compile to # a cheap shift+mask in the lookup. The chunk size is also emitted as # UCS_NONBMP_BMP_BITS in the generated header so ucs.c stays in sync. BITS_PER_CHUNK = 8 # Function to split ranges into BMP (16-bit) and non-BMP (above 16-bit) def split_ranges_by_size(ranges): bmp_ranges = [] non_bmp_ranges = [] for start, end in ranges: if end <= 0xFFFF: bmp_ranges.append((start, end)) elif start > 0xFFFF: non_bmp_ranges.append((start, end)) else: # Split the range at 0xFFFF bmp_ranges.append((start, 0xFFFF)) non_bmp_ranges.append((0x10000, end)) return bmp_ranges, non_bmp_ranges # Split ranges into BMP and non-BMP zero_width_bmp, zero_width_non_bmp = split_ranges_by_size(zero_width_ranges) double_width_bmp, double_width_non_bmp = split_ranges_by_size(double_width_ranges) # Merge zero- and double-width ranges per region, tagging each with its # width, then sort by `first` so binary search works on the union. bmp_entries = sorted( [(s, e, 0) for s, e in zero_width_bmp] + [(s, e, 2) for s, e in double_width_bmp], key=lambda t: t[0]) nonbmp_entries = sorted( [(s, e, 0) for s, e in zero_width_non_bmp] + [(s, e, 2) for s, e in double_width_non_bmp], key=lambda t: t[0]) # Build the BMP double-width bitmap: one bit per BMP entry (in sort # order), set iff that entry is double-width. Pack into BITS_PER_CHUNK- # wide chunks, with bit j of the chunk corresponding to entry # (chunk_index * BITS_PER_CHUNK + j). bmp_w2_bits = [1 if w == 2 else 0 for _, _, w in bmp_entries] n_chunks = (len(bmp_w2_bits) + BITS_PER_CHUNK - 1) // BITS_PER_CHUNK if n_chunks > len(nonbmp_entries): raise RuntimeError( f"BMP bitmap needs {n_chunks} host entries, " f"but only {len(nonbmp_entries)} non-BMP entries are available") chunks = [] # list of (base_index, end_index, packed_value) for c in range(n_chunks): base = c * BITS_PER_CHUNK end_idx = min(base + BITS_PER_CHUNK - 1, len(bmp_w2_bits) - 1) value = 0 for j in range(BITS_PER_CHUNK): k = base + j if k < len(bmp_w2_bits) and bmp_w2_bits[k]: value |= 1 << j chunks.append((base, end_idx, value)) # Function to generate code point description comments def get_code_point_comment(start, end): try: start_char_desc = unicodedata.name(chr(start)) if start == end: return f"/* {start_char_desc} */" else: end_char_desc = unicodedata.name(chr(end)) return f"/* {start_char_desc} - {end_char_desc} */" except: if start == end: return f"/* U+{start:04X} */" else: return f"/* U+{start:04X} - U+{end:04X} */" # Generate C tables with open(out_file, 'w') as f: f.write(f"""\ /* SPDX-License-Identifier: GPL-2.0 */ /* * {out_file} - Unicode character width * * Auto-generated by {this_file} * * Unicode Version: {unicodedata.unidata_version} * * Zero-width and double-width ranges are merged into one sorted-by-`first` * table per region. The non-BMP table additionally hosts the BMP * double-width bitmap in the low {BITS_PER_CHUNK} bits of `last` of its * first {n_chunks} entries (covering {len(bmp_w2_bits)} BMP entries). * See ucs.c for the encoding details and the lookup code. */ /* Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field. */ #define UCS_NONBMP_BMP_BITS {BITS_PER_CHUNK} /* Combined zero- and double-width ranges * (BMP - Basic Multilingual Plane, U+0000 to U+FFFF). */ static const struct ucs_width16 ucs_bmp_ranges[] = {{ """) for s, e, w in bmp_entries: macro = "BMP_0WIDTH" if w == 0 else "BMP_2WIDTH" comment = get_code_point_comment(s, e) f.write(f"\t{{ {macro}(0x{s:04X}, 0x{e:04X}) }}, {comment}\n") f.write(f"""\ }}; /* Combined zero- and double-width ranges (non-BMP, U+10000 and above). * The first {n_chunks} entries host the BMP double-width bitmap in the low * {BITS_PER_CHUNK} bits of `last`. */ static const struct ucs_width32 ucs_nonbmp_ranges[] = {{ """) for i, (s, e, w) in enumerate(nonbmp_entries): macro = "RANGE_0WIDTH" if w == 0 else "RANGE_2WIDTH" comment = get_code_point_comment(s, e) if i < len(chunks): base, end_idx, value = chunks[i] f.write( f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) {comment}\n" f"\t | BMP_2W_BITS(0b{value:0{BITS_PER_CHUNK}b}) }}," f" /* BMP entries [{base:>3}..{end_idx:>3}] */\n") else: f.write(f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) }}, {comment}\n") f.write("};\n") if __name__ == "__main__": # Parse command line arguments parser = argparse.ArgumentParser(description="Generate Unicode width tables") parser.add_argument("-o", "--output", dest="output_file", default=DEFAULT_OUT_FILE, help=f"Output file name (default: {DEFAULT_OUT_FILE})") args = parser.parse_args() # Write tables to header file zero_width_ranges, double_width_ranges = create_width_tables() write_tables(zero_width_ranges, double_width_ranges, out_file=args.output_file) # Print summary zero_width_count = sum(end - start + 1 for start, end in zero_width_ranges) double_width_count = sum(end - start + 1 for start, end in double_width_ranges) n_zero = len(zero_width_ranges) n_double = len(double_width_ranges) print(f"Generated {args.output_file} with:") print(f"- {n_zero} zero-width ranges covering ~{zero_width_count} code points") print(f"- {n_double} double-width ranges covering ~{double_width_count} code points") print(f"- {n_zero + n_double} merged ranges total") print(f"- Unicode Version: {unicodedata.unidata_version}")