1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# Leverage Python's unicodedata module to generate ucs_width_table.h 5 6import unicodedata 7import sys 8import argparse 9 10# This script's file name 11from pathlib import Path 12this_file = Path(__file__).name 13 14# Default output file name 15DEFAULT_OUT_FILE = "ucs_width_table.h" 16 17# --- Global Constants for Width Assignments --- 18 19# Known zero-width characters 20KNOWN_ZERO_WIDTH = ( 21 0x200B, # ZERO WIDTH SPACE 22 0x200C, # ZERO WIDTH NON-JOINER 23 0x200D, # ZERO WIDTH JOINER 24 0x2060, # WORD JOINER 25 0xFEFF # ZERO WIDTH NO-BREAK SPACE (BOM) 26) 27 28# Zero-width emoji modifiers and components 29# NOTE: Some of these characters would normally be single-width according to 30# East Asian Width properties, but we deliberately override them to be 31# zero-width because they function as modifiers in emoji sequences. 32EMOJI_ZERO_WIDTH = [ 33 # Skin tone modifiers 34 (0x1F3FB, 0x1F3FF), # Emoji modifiers (skin tones) 35 36 # Variation selectors (note: VS16 is treated specially in vt.c) 37 (0xFE00, 0xFE0F), # Variation Selectors 1-16 38 39 # Gender and hair style modifiers 40 # These would be single-width by Unicode properties, but are zero-width 41 # when part of emoji 42 (0x2640, 0x2640), # Female sign 43 (0x2642, 0x2642), # Male sign 44 (0x26A7, 0x26A7), # Transgender symbol 45 (0x1F9B0, 0x1F9B3), # Hair components (red, curly, white, bald) 46 47 # Tag characters 48 (0xE0020, 0xE007E), # Tags 49] 50 51# Regional indicators (flag components) 52REGIONAL_INDICATORS = (0x1F1E6, 0x1F1FF) # Regional indicator symbols A-Z 53 54# Double-width emoji ranges 55# 56# Many emoji characters are classified as single-width according to Unicode 57# Standard Annex #11 East Asian Width property (N or Neutral), but we 58# deliberately override them to be double-width. References: 59# 1. Unicode Technical Standard #51: Unicode Emoji 60# (https://www.unicode.org/reports/tr51/) 61# 2. Principle of "emoji presentation" in WHATWG CSS Text specification 62# (https://drafts.csswg.org/css-text-3/#character-properties) 63# 3. Terminal emulator implementations (iTerm2, Windows Terminal, etc.) which 64# universally render emoji as double-width characters regardless of their 65# Unicode EAW property 66# 4. W3C Work Item: Requirements for Japanese Text Layout - Section 3.8.1 67# Emoji width (https://www.w3.org/TR/jlreq/) 68EMOJI_RANGES = [ 69 (0x1F000, 0x1F02F), # Mahjong Tiles (EAW: N, but displayed as double-width) 70 (0x1F0A0, 0x1F0FF), # Playing Cards (EAW: N, but displayed as double-width) 71 (0x1F300, 0x1F5FF), # Miscellaneous Symbols and Pictographs 72 (0x1F600, 0x1F64F), # Emoticons 73 (0x1F680, 0x1F6FF), # Transport and Map Symbols 74 (0x1F700, 0x1F77F), # Alchemical Symbols 75 (0x1F780, 0x1F7FF), # Geometric Shapes Extended 76 (0x1F800, 0x1F8FF), # Supplemental Arrows-C 77 (0x1F900, 0x1F9FF), # Supplemental Symbols and Pictographs 78 (0x1FA00, 0x1FA6F), # Chess Symbols 79 (0x1FA70, 0x1FAFF), # Symbols and Pictographs Extended-A 80] 81 82def create_width_tables(): 83 """ 84 Creates Unicode character width tables and returns the data structures. 85 86 Returns: 87 tuple: (zero_width_ranges, double_width_ranges) 88 """ 89 90 # Width data mapping 91 width_map = {} # Maps code points to width (0, 1, 2) 92 93 # Mark emoji modifiers as zero-width 94 for start, end in EMOJI_ZERO_WIDTH: 95 for cp in range(start, end + 1): 96 width_map[cp] = 0 97 98 # Mark all regional indicators as single-width as they are usually paired 99 # providing a combined width of 2 when displayed together. 100 start, end = REGIONAL_INDICATORS 101 for cp in range(start, end + 1): 102 width_map[cp] = 1 103 104 # Process all assigned Unicode code points (Basic Multilingual Plane + 105 # Supplementary Planes) Range 0x0 to 0x10FFFF (the full Unicode range) 106 for block_start in range(0, 0x110000, 0x1000): 107 block_end = block_start + 0x1000 108 for cp in range(block_start, block_end): 109 try: 110 char = chr(cp) 111 112 # Skip if already processed 113 if cp in width_map: 114 continue 115 116 # Check for combining marks and a format characters 117 category = unicodedata.category(char) 118 119 # Combining marks 120 if category.startswith('M'): 121 width_map[cp] = 0 122 continue 123 124 # Format characters 125 # Since we have no support for bidirectional text, all format 126 # characters (category Cf) can be treated with width 0 (zero) 127 # for simplicity, as they don't need to occupy visual space 128 # in a non-bidirectional text environment. 129 if category == 'Cf': 130 width_map[cp] = 0 131 continue 132 133 # Known zero-width characters 134 if cp in KNOWN_ZERO_WIDTH: 135 width_map[cp] = 0 136 continue 137 138 # Use East Asian Width property 139 eaw = unicodedata.east_asian_width(char) 140 if eaw in ('F', 'W'): # Fullwidth or Wide 141 width_map[cp] = 2 142 elif eaw in ('Na', 'H', 'N', 'A'): # Narrow, Halfwidth, Neutral, Ambiguous 143 width_map[cp] = 1 144 else: 145 # Default to single-width for unknown 146 width_map[cp] = 1 147 148 except (ValueError, OverflowError): 149 # Skip invalid code points 150 continue 151 152 # Process Emoji - generally double-width 153 for start, end in EMOJI_RANGES: 154 for cp in range(start, end + 1): 155 if cp not in width_map or width_map[cp] != 0: # Don't override zero-width 156 try: 157 char = chr(cp) 158 width_map[cp] = 2 159 except (ValueError, OverflowError): 160 continue 161 162 # Optimize to create range tables 163 def ranges_optimize(width_data, target_width): 164 points = sorted([cp for cp, width in width_data.items() if width == target_width]) 165 if not points: 166 return [] 167 168 # Group consecutive code points into ranges 169 ranges = [] 170 start = points[0] 171 prev = start 172 173 for cp in points[1:]: 174 if cp > prev + 1: 175 ranges.append((start, prev)) 176 start = cp 177 prev = cp 178 179 # Add the last range 180 ranges.append((start, prev)) 181 return ranges 182 183 # Extract ranges for each width 184 zero_width_ranges = ranges_optimize(width_map, 0) 185 double_width_ranges = ranges_optimize(width_map, 2) 186 187 return zero_width_ranges, double_width_ranges 188 189def write_tables(zero_width_ranges, double_width_ranges, out_file=DEFAULT_OUT_FILE): 190 """ 191 Write the generated tables to C header file. 192 193 The output uses a single sorted-by-`first` table per region (BMP and 194 non-BMP), with zero-width and double-width ranges merged together. The 195 non-BMP table also hosts the BMP double-width bitmap in spare bits of 196 `last`. See the encoding comment at the top of ucs.c for the layout. 197 198 Args: 199 zero_width_ranges: List of (start, end) ranges for zero-width characters 200 double_width_ranges: List of (start, end) ranges for double-width characters 201 out_file: Output file name (default: DEFAULT_OUT_FILE) 202 """ 203 204 # Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field. 205 # 8 bits makes `idx / BITS_PER_CHUNK` / `idx % BITS_PER_CHUNK` compile to 206 # a cheap shift+mask in the lookup. The chunk size is also emitted as 207 # UCS_NONBMP_BMP_BITS in the generated header so ucs.c stays in sync. 208 BITS_PER_CHUNK = 8 209 210 # Function to split ranges into BMP (16-bit) and non-BMP (above 16-bit) 211 def split_ranges_by_size(ranges): 212 bmp_ranges = [] 213 non_bmp_ranges = [] 214 215 for start, end in ranges: 216 if end <= 0xFFFF: 217 bmp_ranges.append((start, end)) 218 elif start > 0xFFFF: 219 non_bmp_ranges.append((start, end)) 220 else: 221 # Split the range at 0xFFFF 222 bmp_ranges.append((start, 0xFFFF)) 223 non_bmp_ranges.append((0x10000, end)) 224 225 return bmp_ranges, non_bmp_ranges 226 227 # Split ranges into BMP and non-BMP 228 zero_width_bmp, zero_width_non_bmp = split_ranges_by_size(zero_width_ranges) 229 double_width_bmp, double_width_non_bmp = split_ranges_by_size(double_width_ranges) 230 231 # Merge zero- and double-width ranges per region, tagging each with its 232 # width, then sort by `first` so binary search works on the union. 233 bmp_entries = sorted( 234 [(s, e, 0) for s, e in zero_width_bmp] + 235 [(s, e, 2) for s, e in double_width_bmp], 236 key=lambda t: t[0]) 237 nonbmp_entries = sorted( 238 [(s, e, 0) for s, e in zero_width_non_bmp] + 239 [(s, e, 2) for s, e in double_width_non_bmp], 240 key=lambda t: t[0]) 241 242 # Build the BMP double-width bitmap: one bit per BMP entry (in sort 243 # order), set iff that entry is double-width. Pack into BITS_PER_CHUNK- 244 # wide chunks, with bit j of the chunk corresponding to entry 245 # (chunk_index * BITS_PER_CHUNK + j). 246 bmp_w2_bits = [1 if w == 2 else 0 for _, _, w in bmp_entries] 247 n_chunks = (len(bmp_w2_bits) + BITS_PER_CHUNK - 1) // BITS_PER_CHUNK 248 249 if n_chunks > len(nonbmp_entries): 250 raise RuntimeError( 251 f"BMP bitmap needs {n_chunks} host entries, " 252 f"but only {len(nonbmp_entries)} non-BMP entries are available") 253 254 chunks = [] # list of (base_index, end_index, packed_value) 255 for c in range(n_chunks): 256 base = c * BITS_PER_CHUNK 257 end_idx = min(base + BITS_PER_CHUNK - 1, len(bmp_w2_bits) - 1) 258 value = 0 259 for j in range(BITS_PER_CHUNK): 260 k = base + j 261 if k < len(bmp_w2_bits) and bmp_w2_bits[k]: 262 value |= 1 << j 263 chunks.append((base, end_idx, value)) 264 265 # Function to generate code point description comments 266 def get_code_point_comment(start, end): 267 try: 268 start_char_desc = unicodedata.name(chr(start)) 269 if start == end: 270 return f"/* {start_char_desc} */" 271 else: 272 end_char_desc = unicodedata.name(chr(end)) 273 return f"/* {start_char_desc} - {end_char_desc} */" 274 except: 275 if start == end: 276 return f"/* U+{start:04X} */" 277 else: 278 return f"/* U+{start:04X} - U+{end:04X} */" 279 280 # Generate C tables 281 with open(out_file, 'w') as f: 282 f.write(f"""\ 283/* SPDX-License-Identifier: GPL-2.0 */ 284/* 285 * {out_file} - Unicode character width 286 * 287 * Auto-generated by {this_file} 288 * 289 * Unicode Version: {unicodedata.unidata_version} 290 * 291 * Zero-width and double-width ranges are merged into one sorted-by-`first` 292 * table per region. The non-BMP table additionally hosts the BMP 293 * double-width bitmap in the low {BITS_PER_CHUNK} bits of `last` of its 294 * first {n_chunks} entries (covering {len(bmp_w2_bits)} BMP entries). 295 * See ucs.c for the encoding details and the lookup code. 296 */ 297 298/* Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field. */ 299#define UCS_NONBMP_BMP_BITS {BITS_PER_CHUNK} 300 301/* Combined zero- and double-width ranges 302 * (BMP - Basic Multilingual Plane, U+0000 to U+FFFF). */ 303static const struct ucs_width16 ucs_bmp_ranges[] = {{ 304""") 305 306 for s, e, w in bmp_entries: 307 macro = "BMP_0WIDTH" if w == 0 else "BMP_2WIDTH" 308 comment = get_code_point_comment(s, e) 309 f.write(f"\t{{ {macro}(0x{s:04X}, 0x{e:04X}) }}, {comment}\n") 310 311 f.write(f"""\ 312}}; 313 314/* Combined zero- and double-width ranges (non-BMP, U+10000 and above). 315 * The first {n_chunks} entries host the BMP double-width bitmap in the low 316 * {BITS_PER_CHUNK} bits of `last`. */ 317static const struct ucs_width32 ucs_nonbmp_ranges[] = {{ 318""") 319 320 for i, (s, e, w) in enumerate(nonbmp_entries): 321 macro = "RANGE_0WIDTH" if w == 0 else "RANGE_2WIDTH" 322 comment = get_code_point_comment(s, e) 323 if i < len(chunks): 324 base, end_idx, value = chunks[i] 325 f.write( 326 f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) {comment}\n" 327 f"\t | BMP_2W_BITS(0b{value:0{BITS_PER_CHUNK}b}) }}," 328 f" /* BMP entries [{base:>3}..{end_idx:>3}] */\n") 329 else: 330 f.write(f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) }}, {comment}\n") 331 332 f.write("};\n") 333 334if __name__ == "__main__": 335 # Parse command line arguments 336 parser = argparse.ArgumentParser(description="Generate Unicode width tables") 337 parser.add_argument("-o", "--output", dest="output_file", default=DEFAULT_OUT_FILE, 338 help=f"Output file name (default: {DEFAULT_OUT_FILE})") 339 args = parser.parse_args() 340 341 # Write tables to header file 342 zero_width_ranges, double_width_ranges = create_width_tables() 343 write_tables(zero_width_ranges, double_width_ranges, out_file=args.output_file) 344 345 # Print summary 346 zero_width_count = sum(end - start + 1 for start, end in zero_width_ranges) 347 double_width_count = sum(end - start + 1 for start, end in double_width_ranges) 348 n_zero = len(zero_width_ranges) 349 n_double = len(double_width_ranges) 350 print(f"Generated {args.output_file} with:") 351 print(f"- {n_zero} zero-width ranges covering ~{zero_width_count} code points") 352 print(f"- {n_double} double-width ranges covering ~{double_width_count} code points") 353 print(f"- {n_zero + n_double} merged ranges total") 354 print(f"- Unicode Version: {unicodedata.unidata_version}") 355