1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 27 28 29 /* 30 * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458). 31 * 32 * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F), 33 * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also 34 * the section 3C man pages. 35 * Interface stability: Committed. 36 */ 37 38 #include <sys/types.h> 39 #include <sys/strings.h> 40 #include <sys/param.h> 41 #include <sys/sysmacros.h> 42 #include <sys/debug.h> 43 #include <sys/kmem.h> 44 #include <sys/sunddi.h> 45 #include <sys/u8_textprep.h> 46 #include <sys/byteorder.h> 47 #include <sys/errno.h> 48 #include <sys/u8_textprep_data.h> 49 #include <sys/mod.h> 50 51 /* The maximum possible number of bytes in a UTF-8 character. */ 52 #define U8_MB_CUR_MAX (4) 53 54 /* 55 * The maximum number of bytes needed for a UTF-8 character to cover 56 * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2. 57 */ 58 #define U8_MAX_BYTES_UCS2 (3) 59 60 /* The maximum possible number of bytes in a Stream-Safe Text. */ 61 #define U8_STREAM_SAFE_TEXT_MAX (128) 62 63 /* 64 * The maximum number of characters in a combining/conjoining sequence and 65 * the actual upperbound limit of a combining/conjoining sequence. 66 */ 67 #define U8_MAX_CHARS_A_SEQ (32) 68 #define U8_UPPER_LIMIT_IN_A_SEQ (31) 69 70 /* The combining class value for Starter. */ 71 #define U8_COMBINING_CLASS_STARTER (0) 72 73 /* 74 * Some Hangul related macros at below. 75 * 76 * The first and the last of Hangul syllables, Hangul Jamo Leading consonants, 77 * Vowels, and optional Trailing consonants in Unicode scalar values. 78 * 79 * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not 80 * the actual U+11A8. This is due to that the trailing consonant is optional 81 * and thus we are doing a pre-calculation of subtracting one. 82 * 83 * Each of 19 modern leading consonants has total 588 possible syllables since 84 * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for 85 * no trailing consonant case, i.e., 21 x 28 = 588. 86 * 87 * We also have bunch of Hangul related macros at below. Please bear in mind 88 * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is 89 * a Hangul Jamo or not but the value does not guarantee that it is a Hangul 90 * Jamo; it just guarantee that it will be most likely. 91 */ 92 #define U8_HANGUL_SYL_FIRST (0xAC00U) 93 #define U8_HANGUL_SYL_LAST (0xD7A3U) 94 95 #define U8_HANGUL_JAMO_L_FIRST (0x1100U) 96 #define U8_HANGUL_JAMO_L_LAST (0x1112U) 97 #define U8_HANGUL_JAMO_V_FIRST (0x1161U) 98 #define U8_HANGUL_JAMO_V_LAST (0x1175U) 99 #define U8_HANGUL_JAMO_T_FIRST (0x11A7U) 100 #define U8_HANGUL_JAMO_T_LAST (0x11C2U) 101 102 #define U8_HANGUL_V_COUNT (21) 103 #define U8_HANGUL_VT_COUNT (588) 104 #define U8_HANGUL_T_COUNT (28) 105 106 #define U8_HANGUL_JAMO_1ST_BYTE (0xE1U) 107 108 #define U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \ 109 (s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \ 110 (s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \ 111 (s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU)); 112 113 #define U8_HANGUL_JAMO_L(u) \ 114 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST) 115 116 #define U8_HANGUL_JAMO_V(u) \ 117 ((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST) 118 119 #define U8_HANGUL_JAMO_T(u) \ 120 ((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST) 121 122 #define U8_HANGUL_JAMO(u) \ 123 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST) 124 125 #define U8_HANGUL_SYLLABLE(u) \ 126 ((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST) 127 128 #define U8_HANGUL_COMPOSABLE_L_V(s, u) \ 129 ((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u))) 130 131 #define U8_HANGUL_COMPOSABLE_LV_T(s, u) \ 132 ((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u))) 133 134 /* The types of decomposition mappings. */ 135 #define U8_DECOMP_BOTH (0xF5U) 136 #define U8_DECOMP_CANONICAL (0xF6U) 137 138 /* The indicator for 16-bit table. */ 139 #define U8_16BIT_TABLE_INDICATOR (0x8000U) 140 141 /* The following are some convenience macros. */ 142 #define U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3) \ 143 (u) = ((((uint32_t)(b1) & 0x0F) << 12) | \ 144 (((uint32_t)(b2) & 0x3F) << 6) | \ 145 ((uint32_t)(b3) & 0x3F)); 146 147 #define U8_SIMPLE_SWAP(a, b, t) \ 148 (t) = (a); \ 149 (a) = (b); \ 150 (b) = (t); 151 152 #define U8_ASCII_TOUPPER(c) \ 153 (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c)) 154 155 #define U8_ASCII_TOLOWER(c) \ 156 (((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c)) 157 158 #define U8_ISASCII(c) (((uchar_t)(c)) < 0x80U) 159 /* 160 * The following macro assumes that the two characters that are to be 161 * swapped are adjacent to each other and 'a' comes before 'b'. 162 * 163 * If the assumptions are not met, then, the macro will fail. 164 */ 165 #define U8_SWAP_COMB_MARKS(a, b) \ 166 for (k = 0; k < disp[(a)]; k++) \ 167 u8t[k] = u8s[start[(a)] + k]; \ 168 for (k = 0; k < disp[(b)]; k++) \ 169 u8s[start[(a)] + k] = u8s[start[(b)] + k]; \ 170 start[(b)] = start[(a)] + disp[(b)]; \ 171 for (k = 0; k < disp[(a)]; k++) \ 172 u8s[start[(b)] + k] = u8t[k]; \ 173 U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \ 174 U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc); 175 176 /* The possible states during normalization. */ 177 typedef enum { 178 U8_STATE_START = 0, 179 U8_STATE_HANGUL_L = 1, 180 U8_STATE_HANGUL_LV = 2, 181 U8_STATE_HANGUL_LVT = 3, 182 U8_STATE_HANGUL_V = 4, 183 U8_STATE_HANGUL_T = 5, 184 U8_STATE_COMBINING_MARK = 6 185 } u8_normalization_states_t; 186 187 /* 188 * The three vectors at below are used to check bytes of a given UTF-8 189 * character are valid and not containing any malformed byte values. 190 * 191 * We used to have a quite relaxed UTF-8 binary representation but then there 192 * was some security related issues and so the Unicode Consortium defined 193 * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it 194 * one more time at the Unicode 3.2. The following three tables are based on 195 * that. 196 */ 197 198 #define U8_ILLEGAL_NEXT_BYTE_COMMON(c) ((c) < 0x80 || (c) > 0xBF) 199 200 #define I_ U8_ILLEGAL_CHAR 201 #define O_ U8_OUT_OF_RANGE_CHAR 202 203 const int8_t u8_number_of_bytes[0x100] = { 204 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 205 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 206 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 207 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 208 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 209 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 210 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 211 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 212 213 /* 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F */ 214 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 215 216 /* 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F */ 217 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 218 219 /* A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF */ 220 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 221 222 /* B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF */ 223 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 224 225 /* C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF */ 226 I_, I_, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 227 228 /* D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF */ 229 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 230 231 /* E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF */ 232 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 233 234 /* F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF */ 235 4, 4, 4, 4, 4, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, 236 }; 237 238 #undef I_ 239 #undef O_ 240 241 const uint8_t u8_valid_min_2nd_byte[0x100] = { 242 0, 0, 0, 0, 0, 0, 0, 0, 243 0, 0, 0, 0, 0, 0, 0, 0, 244 0, 0, 0, 0, 0, 0, 0, 0, 245 0, 0, 0, 0, 0, 0, 0, 0, 246 0, 0, 0, 0, 0, 0, 0, 0, 247 0, 0, 0, 0, 0, 0, 0, 0, 248 0, 0, 0, 0, 0, 0, 0, 0, 249 0, 0, 0, 0, 0, 0, 0, 0, 250 0, 0, 0, 0, 0, 0, 0, 0, 251 0, 0, 0, 0, 0, 0, 0, 0, 252 0, 0, 0, 0, 0, 0, 0, 0, 253 0, 0, 0, 0, 0, 0, 0, 0, 254 0, 0, 0, 0, 0, 0, 0, 0, 255 0, 0, 0, 0, 0, 0, 0, 0, 256 0, 0, 0, 0, 0, 0, 0, 0, 257 0, 0, 0, 0, 0, 0, 0, 0, 258 0, 0, 0, 0, 0, 0, 0, 0, 259 0, 0, 0, 0, 0, 0, 0, 0, 260 0, 0, 0, 0, 0, 0, 0, 0, 261 0, 0, 0, 0, 0, 0, 0, 0, 262 0, 0, 0, 0, 0, 0, 0, 0, 263 0, 0, 0, 0, 0, 0, 0, 0, 264 0, 0, 0, 0, 0, 0, 0, 0, 265 0, 0, 0, 0, 0, 0, 0, 0, 266 /* C0 C1 C2 C3 C4 C5 C6 C7 */ 267 0, 0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 268 /* C8 C9 CA CB CC CD CE CF */ 269 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 270 /* D0 D1 D2 D3 D4 D5 D6 D7 */ 271 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 272 /* D8 D9 DA DB DC DD DE DF */ 273 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 274 /* E0 E1 E2 E3 E4 E5 E6 E7 */ 275 0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 276 /* E8 E9 EA EB EC ED EE EF */ 277 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 278 /* F0 F1 F2 F3 F4 F5 F6 F7 */ 279 0x90, 0x80, 0x80, 0x80, 0x80, 0, 0, 0, 280 0, 0, 0, 0, 0, 0, 0, 0, 281 }; 282 283 const uint8_t u8_valid_max_2nd_byte[0x100] = { 284 0, 0, 0, 0, 0, 0, 0, 0, 285 0, 0, 0, 0, 0, 0, 0, 0, 286 0, 0, 0, 0, 0, 0, 0, 0, 287 0, 0, 0, 0, 0, 0, 0, 0, 288 0, 0, 0, 0, 0, 0, 0, 0, 289 0, 0, 0, 0, 0, 0, 0, 0, 290 0, 0, 0, 0, 0, 0, 0, 0, 291 0, 0, 0, 0, 0, 0, 0, 0, 292 0, 0, 0, 0, 0, 0, 0, 0, 293 0, 0, 0, 0, 0, 0, 0, 0, 294 0, 0, 0, 0, 0, 0, 0, 0, 295 0, 0, 0, 0, 0, 0, 0, 0, 296 0, 0, 0, 0, 0, 0, 0, 0, 297 0, 0, 0, 0, 0, 0, 0, 0, 298 0, 0, 0, 0, 0, 0, 0, 0, 299 0, 0, 0, 0, 0, 0, 0, 0, 300 0, 0, 0, 0, 0, 0, 0, 0, 301 0, 0, 0, 0, 0, 0, 0, 0, 302 0, 0, 0, 0, 0, 0, 0, 0, 303 0, 0, 0, 0, 0, 0, 0, 0, 304 0, 0, 0, 0, 0, 0, 0, 0, 305 0, 0, 0, 0, 0, 0, 0, 0, 306 0, 0, 0, 0, 0, 0, 0, 0, 307 0, 0, 0, 0, 0, 0, 0, 0, 308 /* C0 C1 C2 C3 C4 C5 C6 C7 */ 309 0, 0, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 310 /* C8 C9 CA CB CC CD CE CF */ 311 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 312 /* D0 D1 D2 D3 D4 D5 D6 D7 */ 313 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 314 /* D8 D9 DA DB DC DD DE DF */ 315 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 316 /* E0 E1 E2 E3 E4 E5 E6 E7 */ 317 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 318 /* E8 E9 EA EB EC ED EE EF */ 319 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf, 320 /* F0 F1 F2 F3 F4 F5 F6 F7 */ 321 0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0, 0, 0, 322 0, 0, 0, 0, 0, 0, 0, 0, 323 }; 324 325 326 /* 327 * The u8_validate() validates on the given UTF-8 character string and 328 * calculate the byte length. It is quite similar to mblen(3C) except that 329 * this will validate against the list of characters if required and 330 * specific to UTF-8 and Unicode. 331 */ 332 int 333 u8_validate(const char *u8str, size_t n, char **list, int flag, int *errnum) 334 { 335 uchar_t *ib; 336 uchar_t *ibtail; 337 uchar_t **p; 338 uchar_t *s1; 339 uchar_t *s2; 340 uchar_t f; 341 int sz; 342 size_t i; 343 int ret_val; 344 boolean_t second; 345 boolean_t no_need_to_validate_entire; 346 boolean_t check_additional; 347 boolean_t validate_ucs2_range_only; 348 349 if (! u8str) 350 return (0); 351 352 ib = (uchar_t *)u8str; 353 ibtail = ib + n; 354 355 ret_val = 0; 356 357 no_need_to_validate_entire = ! (flag & U8_VALIDATE_ENTIRE); 358 check_additional = flag & U8_VALIDATE_CHECK_ADDITIONAL; 359 validate_ucs2_range_only = flag & U8_VALIDATE_UCS2_RANGE; 360 361 while (ib < ibtail) { 362 /* 363 * The first byte of a UTF-8 character tells how many 364 * bytes will follow for the character. If the first byte 365 * is an illegal byte value or out of range value, we just 366 * return -1 with an appropriate error number. 367 */ 368 sz = u8_number_of_bytes[*ib]; 369 if (sz == U8_ILLEGAL_CHAR) { 370 *errnum = EILSEQ; 371 return (-1); 372 } 373 374 if (sz == U8_OUT_OF_RANGE_CHAR || 375 (validate_ucs2_range_only && sz > U8_MAX_BYTES_UCS2)) { 376 *errnum = ERANGE; 377 return (-1); 378 } 379 380 /* 381 * If we don't have enough bytes to check on, that's also 382 * an error. As you can see, we give illegal byte sequence 383 * checking higher priority then EINVAL cases. 384 */ 385 if ((ibtail - ib) < sz) { 386 *errnum = EINVAL; 387 return (-1); 388 } 389 390 if (sz == 1) { 391 ib++; 392 ret_val++; 393 } else { 394 /* 395 * Check on the multi-byte UTF-8 character. For more 396 * details on this, see comment added for the used 397 * data structures at the beginning of the file. 398 */ 399 f = *ib++; 400 ret_val++; 401 second = B_TRUE; 402 for (i = 1; i < sz; i++) { 403 if (second) { 404 if (*ib < u8_valid_min_2nd_byte[f] || 405 *ib > u8_valid_max_2nd_byte[f]) { 406 *errnum = EILSEQ; 407 return (-1); 408 } 409 second = B_FALSE; 410 } else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib)) { 411 *errnum = EILSEQ; 412 return (-1); 413 } 414 ib++; 415 ret_val++; 416 } 417 } 418 419 if (check_additional) { 420 for (p = (uchar_t **)list, i = 0; p[i]; i++) { 421 s1 = ib - sz; 422 s2 = p[i]; 423 while (s1 < ib) { 424 if (*s1 != *s2 || *s2 == '\0') 425 break; 426 s1++; 427 s2++; 428 } 429 430 if (s1 >= ib && *s2 == '\0') { 431 *errnum = EBADF; 432 return (-1); 433 } 434 } 435 } 436 437 if (no_need_to_validate_entire) 438 break; 439 } 440 441 return (ret_val); 442 } 443 444 /* 445 * The do_case_conv() looks at the mapping tables and returns found 446 * bytes if any. If not found, the input bytes are returned. The function 447 * always terminate the return bytes with a null character assuming that 448 * there are plenty of room to do so. 449 * 450 * The case conversions are simple case conversions mapping a character to 451 * another character as specified in the Unicode data. The byte size of 452 * the mapped character could be different from that of the input character. 453 * 454 * The return value is the byte length of the returned character excluding 455 * the terminating null byte. 456 */ 457 static size_t 458 do_case_conv(int uv, uchar_t *u8s, uchar_t *s, int sz, boolean_t is_it_toupper) 459 { 460 size_t i; 461 uint16_t b1 = 0; 462 uint16_t b2 = 0; 463 uint16_t b3 = 0; 464 uint16_t b3_tbl; 465 uint16_t b3_base; 466 uint16_t b4 = 0; 467 size_t start_id; 468 size_t end_id; 469 470 /* 471 * At this point, the only possible values for sz are 2, 3, and 4. 472 * The u8s should point to a vector that is well beyond the size of 473 * 5 bytes. 474 */ 475 if (sz == 2) { 476 b3 = u8s[0] = s[0]; 477 b4 = u8s[1] = s[1]; 478 } else if (sz == 3) { 479 b2 = u8s[0] = s[0]; 480 b3 = u8s[1] = s[1]; 481 b4 = u8s[2] = s[2]; 482 } else if (sz == 4) { 483 b1 = u8s[0] = s[0]; 484 b2 = u8s[1] = s[1]; 485 b3 = u8s[2] = s[2]; 486 b4 = u8s[3] = s[3]; 487 } else { 488 /* This is not possible but just in case as a fallback. */ 489 if (is_it_toupper) 490 *u8s = U8_ASCII_TOUPPER(*s); 491 else 492 *u8s = U8_ASCII_TOLOWER(*s); 493 u8s[1] = '\0'; 494 495 return (1); 496 } 497 u8s[sz] = '\0'; 498 499 /* 500 * Let's find out if we have a corresponding character. 501 */ 502 b1 = u8_common_b1_tbl[uv][b1]; 503 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 504 return ((size_t)sz); 505 506 b2 = u8_case_common_b2_tbl[uv][b1][b2]; 507 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 508 return ((size_t)sz); 509 510 if (is_it_toupper) { 511 b3_tbl = u8_toupper_b3_tbl[uv][b2][b3].tbl_id; 512 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 513 return ((size_t)sz); 514 515 start_id = u8_toupper_b4_tbl[uv][b3_tbl][b4]; 516 end_id = u8_toupper_b4_tbl[uv][b3_tbl][b4 + 1]; 517 518 /* Either there is no match or an error at the table. */ 519 if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX) 520 return ((size_t)sz); 521 522 b3_base = u8_toupper_b3_tbl[uv][b2][b3].base; 523 524 for (i = 0; start_id < end_id; start_id++) 525 u8s[i++] = u8_toupper_final_tbl[uv][b3_base + start_id]; 526 } else { 527 b3_tbl = u8_tolower_b3_tbl[uv][b2][b3].tbl_id; 528 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 529 return ((size_t)sz); 530 531 start_id = u8_tolower_b4_tbl[uv][b3_tbl][b4]; 532 end_id = u8_tolower_b4_tbl[uv][b3_tbl][b4 + 1]; 533 534 if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX) 535 return ((size_t)sz); 536 537 b3_base = u8_tolower_b3_tbl[uv][b2][b3].base; 538 539 for (i = 0; start_id < end_id; start_id++) 540 u8s[i++] = u8_tolower_final_tbl[uv][b3_base + start_id]; 541 } 542 543 /* 544 * If i is still zero, that means there is no corresponding character. 545 */ 546 if (i == 0) 547 return ((size_t)sz); 548 549 u8s[i] = '\0'; 550 551 return (i); 552 } 553 554 /* 555 * The do_case_compare() function compares the two input strings, s1 and s2, 556 * one character at a time doing case conversions if applicable and return 557 * the comparison result as like strcmp(). 558 * 559 * Since, in empirical sense, most of text data are 7-bit ASCII characters, 560 * we treat the 7-bit ASCII characters as a special case trying to yield 561 * faster processing time. 562 */ 563 static int 564 do_case_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, 565 size_t n2, boolean_t is_it_toupper, int *errnum) 566 { 567 int f; 568 int sz1; 569 int sz2; 570 size_t j; 571 size_t i1; 572 size_t i2; 573 uchar_t u8s1[U8_MB_CUR_MAX + 1]; 574 uchar_t u8s2[U8_MB_CUR_MAX + 1]; 575 576 i1 = i2 = 0; 577 while (i1 < n1 && i2 < n2) { 578 /* 579 * Find out what would be the byte length for this UTF-8 580 * character at string s1 and also find out if this is 581 * an illegal start byte or not and if so, issue a proper 582 * error number and yet treat this byte as a character. 583 */ 584 sz1 = u8_number_of_bytes[*s1]; 585 if (sz1 < 0) { 586 *errnum = EILSEQ; 587 sz1 = 1; 588 } 589 590 /* 591 * For 7-bit ASCII characters mainly, we do a quick case 592 * conversion right at here. 593 * 594 * If we don't have enough bytes for this character, issue 595 * an EINVAL error and use what are available. 596 * 597 * If we have enough bytes, find out if there is 598 * a corresponding uppercase character and if so, copy over 599 * the bytes for a comparison later. If there is no 600 * corresponding uppercase character, then, use what we have 601 * for the comparison. 602 */ 603 if (sz1 == 1) { 604 if (is_it_toupper) 605 u8s1[0] = U8_ASCII_TOUPPER(*s1); 606 else 607 u8s1[0] = U8_ASCII_TOLOWER(*s1); 608 s1++; 609 u8s1[1] = '\0'; 610 } else if ((i1 + sz1) > n1) { 611 *errnum = EINVAL; 612 for (j = 0; (i1 + j) < n1; ) 613 u8s1[j++] = *s1++; 614 u8s1[j] = '\0'; 615 } else { 616 (void) do_case_conv(uv, u8s1, s1, sz1, is_it_toupper); 617 s1 += sz1; 618 } 619 620 /* Do the same for the string s2. */ 621 sz2 = u8_number_of_bytes[*s2]; 622 if (sz2 < 0) { 623 *errnum = EILSEQ; 624 sz2 = 1; 625 } 626 627 if (sz2 == 1) { 628 if (is_it_toupper) 629 u8s2[0] = U8_ASCII_TOUPPER(*s2); 630 else 631 u8s2[0] = U8_ASCII_TOLOWER(*s2); 632 s2++; 633 u8s2[1] = '\0'; 634 } else if ((i2 + sz2) > n2) { 635 *errnum = EINVAL; 636 for (j = 0; (i2 + j) < n2; ) 637 u8s2[j++] = *s2++; 638 u8s2[j] = '\0'; 639 } else { 640 (void) do_case_conv(uv, u8s2, s2, sz2, is_it_toupper); 641 s2 += sz2; 642 } 643 644 /* Now compare the two characters. */ 645 if (sz1 == 1 && sz2 == 1) { 646 if (*u8s1 > *u8s2) 647 return (1); 648 if (*u8s1 < *u8s2) 649 return (-1); 650 } else { 651 f = strcmp((const char *)u8s1, (const char *)u8s2); 652 if (f != 0) 653 return (f); 654 } 655 656 /* 657 * They were the same. Let's move on to the next 658 * characters then. 659 */ 660 i1 += sz1; 661 i2 += sz2; 662 } 663 664 /* 665 * We compared until the end of either or both strings. 666 * 667 * If we reached to or went over the ends for the both, that means 668 * they are the same. 669 * 670 * If we reached only one of the two ends, that means the other string 671 * has something which then the fact can be used to determine 672 * the return value. 673 */ 674 if (i1 >= n1) { 675 if (i2 >= n2) 676 return (0); 677 return (-1); 678 } 679 return (1); 680 } 681 682 /* 683 * The combining_class() function checks on the given bytes and find out 684 * the corresponding Unicode combining class value. The return value 0 means 685 * it is a Starter. Any illegal UTF-8 character will also be treated as 686 * a Starter. 687 */ 688 static uchar_t 689 combining_class(size_t uv, uchar_t *s, size_t sz) 690 { 691 uint16_t b1 = 0; 692 uint16_t b2 = 0; 693 uint16_t b3 = 0; 694 uint16_t b4 = 0; 695 696 if (sz == 1 || sz > 4) 697 return (0); 698 699 if (sz == 2) { 700 b3 = s[0]; 701 b4 = s[1]; 702 } else if (sz == 3) { 703 b2 = s[0]; 704 b3 = s[1]; 705 b4 = s[2]; 706 } else if (sz == 4) { 707 b1 = s[0]; 708 b2 = s[1]; 709 b3 = s[2]; 710 b4 = s[3]; 711 } 712 713 b1 = u8_common_b1_tbl[uv][b1]; 714 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 715 return (0); 716 717 b2 = u8_combining_class_b2_tbl[uv][b1][b2]; 718 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 719 return (0); 720 721 b3 = u8_combining_class_b3_tbl[uv][b2][b3]; 722 if (b3 == U8_TBL_ELEMENT_NOT_DEF) 723 return (0); 724 725 return (u8_combining_class_b4_tbl[uv][b3][b4]); 726 } 727 728 /* 729 * The do_decomp() function finds out a matching decomposition if any 730 * and return. If there is no match, the input bytes are copied and returned. 731 * The function also checks if there is a Hangul, decomposes it if necessary 732 * and returns. 733 * 734 * To save time, a single byte 7-bit ASCII character should be handled by 735 * the caller. 736 * 737 * The function returns the number of bytes returned sans always terminating 738 * the null byte. It will also return a state that will tell if there was 739 * a Hangul character decomposed which then will be used by the caller. 740 */ 741 static size_t 742 do_decomp(size_t uv, uchar_t *u8s, uchar_t *s, int sz, 743 boolean_t canonical_decomposition, u8_normalization_states_t *state) 744 { 745 uint16_t b1 = 0; 746 uint16_t b2 = 0; 747 uint16_t b3 = 0; 748 uint16_t b3_tbl; 749 uint16_t b3_base; 750 uint16_t b4 = 0; 751 size_t start_id; 752 size_t end_id; 753 size_t i; 754 uint32_t u1; 755 756 if (sz == 2) { 757 b3 = u8s[0] = s[0]; 758 b4 = u8s[1] = s[1]; 759 u8s[2] = '\0'; 760 } else if (sz == 3) { 761 /* Convert it to a Unicode scalar value. */ 762 U8_PUT_3BYTES_INTO_UTF32(u1, s[0], s[1], s[2]); 763 764 /* 765 * If this is a Hangul syllable, we decompose it into 766 * a leading consonant, a vowel, and an optional trailing 767 * consonant and then return. 768 */ 769 if (U8_HANGUL_SYLLABLE(u1)) { 770 u1 -= U8_HANGUL_SYL_FIRST; 771 772 b1 = U8_HANGUL_JAMO_L_FIRST + u1 / U8_HANGUL_VT_COUNT; 773 b2 = U8_HANGUL_JAMO_V_FIRST + (u1 % U8_HANGUL_VT_COUNT) 774 / U8_HANGUL_T_COUNT; 775 b3 = u1 % U8_HANGUL_T_COUNT; 776 777 U8_SAVE_HANGUL_AS_UTF8(u8s, 0, 1, 2, b1); 778 U8_SAVE_HANGUL_AS_UTF8(u8s, 3, 4, 5, b2); 779 if (b3) { 780 b3 += U8_HANGUL_JAMO_T_FIRST; 781 U8_SAVE_HANGUL_AS_UTF8(u8s, 6, 7, 8, b3); 782 783 u8s[9] = '\0'; 784 *state = U8_STATE_HANGUL_LVT; 785 return (9); 786 } 787 788 u8s[6] = '\0'; 789 *state = U8_STATE_HANGUL_LV; 790 return (6); 791 } 792 793 b2 = u8s[0] = s[0]; 794 b3 = u8s[1] = s[1]; 795 b4 = u8s[2] = s[2]; 796 u8s[3] = '\0'; 797 798 /* 799 * If this is a Hangul Jamo, we know there is nothing 800 * further that we can decompose. 801 */ 802 if (U8_HANGUL_JAMO_L(u1)) { 803 *state = U8_STATE_HANGUL_L; 804 return (3); 805 } 806 807 if (U8_HANGUL_JAMO_V(u1)) { 808 if (*state == U8_STATE_HANGUL_L) 809 *state = U8_STATE_HANGUL_LV; 810 else 811 *state = U8_STATE_HANGUL_V; 812 return (3); 813 } 814 815 if (U8_HANGUL_JAMO_T(u1)) { 816 if (*state == U8_STATE_HANGUL_LV) 817 *state = U8_STATE_HANGUL_LVT; 818 else 819 *state = U8_STATE_HANGUL_T; 820 return (3); 821 } 822 } else if (sz == 4) { 823 b1 = u8s[0] = s[0]; 824 b2 = u8s[1] = s[1]; 825 b3 = u8s[2] = s[2]; 826 b4 = u8s[3] = s[3]; 827 u8s[4] = '\0'; 828 } else { 829 /* 830 * This is a fallback and should not happen if the function 831 * was called properly. 832 */ 833 u8s[0] = s[0]; 834 u8s[1] = '\0'; 835 *state = U8_STATE_START; 836 return (1); 837 } 838 839 /* 840 * At this point, this routine does not know what it would get. 841 * The caller should sort it out if the state isn't a Hangul one. 842 */ 843 *state = U8_STATE_START; 844 845 /* Try to find matching decomposition mapping byte sequence. */ 846 b1 = u8_common_b1_tbl[uv][b1]; 847 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 848 return ((size_t)sz); 849 850 b2 = u8_decomp_b2_tbl[uv][b1][b2]; 851 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 852 return ((size_t)sz); 853 854 b3_tbl = u8_decomp_b3_tbl[uv][b2][b3].tbl_id; 855 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 856 return ((size_t)sz); 857 858 /* 859 * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR 860 * which is 0x8000, this means we couldn't fit the mappings into 861 * the cardinality of a unsigned byte. 862 */ 863 if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) { 864 b3_tbl -= U8_16BIT_TABLE_INDICATOR; 865 start_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4]; 866 end_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4 + 1]; 867 } else { 868 start_id = u8_decomp_b4_tbl[uv][b3_tbl][b4]; 869 end_id = u8_decomp_b4_tbl[uv][b3_tbl][b4 + 1]; 870 } 871 872 /* This also means there wasn't any matching decomposition. */ 873 if (start_id >= end_id) 874 return ((size_t)sz); 875 876 /* 877 * The final table for decomposition mappings has three types of 878 * byte sequences depending on whether a mapping is for compatibility 879 * decomposition, canonical decomposition, or both like the following: 880 * 881 * (1) Compatibility decomposition mappings: 882 * 883 * +---+---+-...-+---+ 884 * | B0| B1| ... | Bm| 885 * +---+---+-...-+---+ 886 * 887 * The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH). 888 * 889 * (2) Canonical decomposition mappings: 890 * 891 * +---+---+---+-...-+---+ 892 * | T | b0| b1| ... | bn| 893 * +---+---+---+-...-+---+ 894 * 895 * where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL). 896 * 897 * (3) Both mappings: 898 * 899 * +---+---+---+---+-...-+---+---+---+-...-+---+ 900 * | T | D | b0| b1| ... | bn| B0| B1| ... | Bm| 901 * +---+---+---+---+-...-+---+---+---+-...-+---+ 902 * 903 * where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement 904 * byte, b0 to bn are canonical mapping bytes and B0 to Bm are 905 * compatibility mapping bytes. 906 * 907 * Note that compatibility decomposition means doing recursive 908 * decompositions using both compatibility decomposition mappings and 909 * canonical decomposition mappings. On the other hand, canonical 910 * decomposition means doing recursive decompositions using only 911 * canonical decomposition mappings. Since the table we have has gone 912 * through the recursions already, we do not need to do so during 913 * runtime, i.e., the table has been completely flattened out 914 * already. 915 */ 916 917 b3_base = u8_decomp_b3_tbl[uv][b2][b3].base; 918 919 /* Get the type, T, of the byte sequence. */ 920 b1 = u8_decomp_final_tbl[uv][b3_base + start_id]; 921 922 /* 923 * If necessary, adjust start_id, end_id, or both. Note that if 924 * this is compatibility decomposition mapping, there is no 925 * adjustment. 926 */ 927 if (canonical_decomposition) { 928 /* Is the mapping only for compatibility decomposition? */ 929 if (b1 < U8_DECOMP_BOTH) 930 return ((size_t)sz); 931 932 start_id++; 933 934 if (b1 == U8_DECOMP_BOTH) { 935 end_id = start_id + 936 u8_decomp_final_tbl[uv][b3_base + start_id]; 937 start_id++; 938 } 939 } else { 940 /* 941 * Unless this is a compatibility decomposition mapping, 942 * we adjust the start_id. 943 */ 944 if (b1 == U8_DECOMP_BOTH) { 945 start_id++; 946 start_id += u8_decomp_final_tbl[uv][b3_base + start_id]; 947 } else if (b1 == U8_DECOMP_CANONICAL) { 948 start_id++; 949 } 950 } 951 952 for (i = 0; start_id < end_id; start_id++) 953 u8s[i++] = u8_decomp_final_tbl[uv][b3_base + start_id]; 954 u8s[i] = '\0'; 955 956 return (i); 957 } 958 959 /* 960 * The find_composition_start() function uses the character bytes given and 961 * find out the matching composition mappings if any and return the address 962 * to the composition mappings as explained in the do_composition(). 963 */ 964 static uchar_t * 965 find_composition_start(size_t uv, uchar_t *s, size_t sz) 966 { 967 uint16_t b1 = 0; 968 uint16_t b2 = 0; 969 uint16_t b3 = 0; 970 uint16_t b3_tbl; 971 uint16_t b3_base; 972 uint16_t b4 = 0; 973 size_t start_id; 974 size_t end_id; 975 976 if (sz == 1) { 977 b4 = s[0]; 978 } else if (sz == 2) { 979 b3 = s[0]; 980 b4 = s[1]; 981 } else if (sz == 3) { 982 b2 = s[0]; 983 b3 = s[1]; 984 b4 = s[2]; 985 } else if (sz == 4) { 986 b1 = s[0]; 987 b2 = s[1]; 988 b3 = s[2]; 989 b4 = s[3]; 990 } else { 991 /* 992 * This is a fallback and should not happen if the function 993 * was called properly. 994 */ 995 return (NULL); 996 } 997 998 b1 = u8_composition_b1_tbl[uv][b1]; 999 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 1000 return (NULL); 1001 1002 b2 = u8_composition_b2_tbl[uv][b1][b2]; 1003 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 1004 return (NULL); 1005 1006 b3_tbl = u8_composition_b3_tbl[uv][b2][b3].tbl_id; 1007 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 1008 return (NULL); 1009 1010 if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) { 1011 b3_tbl -= U8_16BIT_TABLE_INDICATOR; 1012 start_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4]; 1013 end_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4 + 1]; 1014 } else { 1015 start_id = u8_composition_b4_tbl[uv][b3_tbl][b4]; 1016 end_id = u8_composition_b4_tbl[uv][b3_tbl][b4 + 1]; 1017 } 1018 1019 if (start_id >= end_id) 1020 return (NULL); 1021 1022 b3_base = u8_composition_b3_tbl[uv][b2][b3].base; 1023 1024 return ((uchar_t *)&(u8_composition_final_tbl[uv][b3_base + start_id])); 1025 } 1026 1027 /* 1028 * The blocked() function checks on the combining class values of previous 1029 * characters in this sequence and return whether it is blocked or not. 1030 */ 1031 static boolean_t 1032 blocked(uchar_t *comb_class, size_t last) 1033 { 1034 uchar_t my_comb_class; 1035 size_t i; 1036 1037 my_comb_class = comb_class[last]; 1038 for (i = 1; i < last; i++) 1039 if (comb_class[i] >= my_comb_class || 1040 comb_class[i] == U8_COMBINING_CLASS_STARTER) 1041 return (B_TRUE); 1042 1043 return (B_FALSE); 1044 } 1045 1046 /* 1047 * The do_composition() reads the character string pointed by 's' and 1048 * do necessary canonical composition and then copy over the result back to 1049 * the 's'. 1050 * 1051 * The input argument 's' cannot contain more than 32 characters. 1052 */ 1053 static size_t 1054 do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start, 1055 uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast) 1056 { 1057 uchar_t t[U8_STREAM_SAFE_TEXT_MAX + 1]; 1058 uchar_t tc[U8_MB_CUR_MAX] = { '\0' }; 1059 uint8_t saved_marks[U8_MAX_CHARS_A_SEQ]; 1060 size_t saved_marks_count; 1061 uchar_t *p; 1062 uchar_t *saved_p; 1063 uchar_t *q; 1064 size_t i; 1065 size_t saved_i; 1066 size_t j; 1067 size_t k; 1068 size_t l; 1069 size_t C; 1070 size_t saved_l; 1071 size_t size; 1072 uint32_t u1; 1073 uint32_t u2; 1074 boolean_t match_not_found = B_TRUE; 1075 1076 /* 1077 * This should never happen unless the callers are doing some strange 1078 * and unexpected things. 1079 * 1080 * The "last" is the index pointing to the last character not last + 1. 1081 */ 1082 if (last >= U8_MAX_CHARS_A_SEQ) 1083 last = U8_UPPER_LIMIT_IN_A_SEQ; 1084 1085 for (i = l = 0; i <= last; i++) { 1086 /* 1087 * The last or any non-Starters at the beginning, we don't 1088 * have any chance to do composition and so we just copy them 1089 * to the temporary buffer. 1090 */ 1091 if (i >= last || comb_class[i] != U8_COMBINING_CLASS_STARTER) { 1092 SAVE_THE_CHAR: 1093 p = s + start[i]; 1094 size = disp[i]; 1095 for (k = 0; k < size; k++) 1096 t[l++] = *p++; 1097 continue; 1098 } 1099 1100 /* 1101 * If this could be a start of Hangul Jamos, then, we try to 1102 * conjoin them. 1103 */ 1104 if (s[start[i]] == U8_HANGUL_JAMO_1ST_BYTE) { 1105 U8_PUT_3BYTES_INTO_UTF32(u1, s[start[i]], 1106 s[start[i] + 1], s[start[i] + 2]); 1107 U8_PUT_3BYTES_INTO_UTF32(u2, s[start[i] + 3], 1108 s[start[i] + 4], s[start[i] + 5]); 1109 1110 if (U8_HANGUL_JAMO_L(u1) && U8_HANGUL_JAMO_V(u2)) { 1111 u1 -= U8_HANGUL_JAMO_L_FIRST; 1112 u2 -= U8_HANGUL_JAMO_V_FIRST; 1113 u1 = U8_HANGUL_SYL_FIRST + 1114 (u1 * U8_HANGUL_V_COUNT + u2) * 1115 U8_HANGUL_T_COUNT; 1116 1117 i += 2; 1118 if (i <= last) { 1119 U8_PUT_3BYTES_INTO_UTF32(u2, 1120 s[start[i]], s[start[i] + 1], 1121 s[start[i] + 2]); 1122 1123 if (U8_HANGUL_JAMO_T(u2)) { 1124 u1 += u2 - 1125 U8_HANGUL_JAMO_T_FIRST; 1126 i++; 1127 } 1128 } 1129 1130 U8_SAVE_HANGUL_AS_UTF8(t + l, 0, 1, 2, u1); 1131 i--; 1132 l += 3; 1133 continue; 1134 } 1135 } 1136 1137 /* 1138 * Let's then find out if this Starter has composition 1139 * mapping. 1140 */ 1141 p = find_composition_start(uv, s + start[i], disp[i]); 1142 if (p == NULL) 1143 goto SAVE_THE_CHAR; 1144 1145 /* 1146 * We have a Starter with composition mapping and the next 1147 * character is a non-Starter. Let's try to find out if 1148 * we can do composition. 1149 */ 1150 1151 saved_p = p; 1152 saved_i = i; 1153 saved_l = l; 1154 saved_marks_count = 0; 1155 1156 TRY_THE_NEXT_MARK: 1157 q = s + start[++i]; 1158 size = disp[i]; 1159 1160 /* 1161 * The next for() loop compares the non-Starter pointed by 1162 * 'q' with the possible (joinable) characters pointed by 'p'. 1163 * 1164 * The composition final table entry pointed by the 'p' 1165 * looks like the following: 1166 * 1167 * +---+---+---+-...-+---+---+---+---+-...-+---+---+ 1168 * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F | 1169 * +---+---+---+-...-+---+---+---+---+-...-+---+---+ 1170 * 1171 * where C is the count byte indicating the number of 1172 * mapping pairs where each pair would be look like 1173 * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second 1174 * character of a canonical decomposition and the B0-Bm are 1175 * the bytes of a matching composite character. The F is 1176 * a filler byte after each character as the separator. 1177 */ 1178 1179 match_not_found = B_TRUE; 1180 1181 for (C = *p++; C > 0; C--) { 1182 for (k = 0; k < size; p++, k++) 1183 if (*p != q[k]) 1184 break; 1185 1186 /* Have we found it? */ 1187 if (k >= size && *p == U8_TBL_ELEMENT_FILLER) { 1188 match_not_found = B_FALSE; 1189 1190 l = saved_l; 1191 1192 while (*++p != U8_TBL_ELEMENT_FILLER) 1193 t[l++] = *p; 1194 1195 break; 1196 } 1197 1198 /* We didn't find; skip to the next pair. */ 1199 if (*p != U8_TBL_ELEMENT_FILLER) 1200 while (*++p != U8_TBL_ELEMENT_FILLER) 1201 ; 1202 while (*++p != U8_TBL_ELEMENT_FILLER) 1203 ; 1204 p++; 1205 } 1206 1207 /* 1208 * If there was no match, we will need to save the combining 1209 * mark for later appending. After that, if the next one 1210 * is a non-Starter and not blocked, then, we try once 1211 * again to do composition with the next non-Starter. 1212 * 1213 * If there was no match and this was a Starter, then, 1214 * this is a new start. 1215 * 1216 * If there was a match and a composition done and we have 1217 * more to check on, then, we retrieve a new composition final 1218 * table entry for the composite and then try to do the 1219 * composition again. 1220 */ 1221 1222 if (match_not_found) { 1223 if (comb_class[i] == U8_COMBINING_CLASS_STARTER) { 1224 i--; 1225 goto SAVE_THE_CHAR; 1226 } 1227 1228 saved_marks[saved_marks_count++] = i; 1229 } 1230 1231 if (saved_l == l) { 1232 while (i < last) { 1233 if (blocked(comb_class, i + 1)) 1234 saved_marks[saved_marks_count++] = ++i; 1235 else 1236 break; 1237 } 1238 if (i < last) { 1239 p = saved_p; 1240 goto TRY_THE_NEXT_MARK; 1241 } 1242 } else if (i < last) { 1243 p = find_composition_start(uv, t + saved_l, 1244 l - saved_l); 1245 if (p != NULL) { 1246 saved_p = p; 1247 goto TRY_THE_NEXT_MARK; 1248 } 1249 } 1250 1251 /* 1252 * There is no more composition possible. 1253 * 1254 * If there was no composition what so ever then we copy 1255 * over the original Starter and then append any non-Starters 1256 * remaining at the target string sequentially after that. 1257 */ 1258 1259 if (saved_l == l) { 1260 p = s + start[saved_i]; 1261 size = disp[saved_i]; 1262 for (j = 0; j < size; j++) 1263 t[l++] = *p++; 1264 } 1265 1266 for (k = 0; k < saved_marks_count; k++) { 1267 p = s + start[saved_marks[k]]; 1268 size = disp[saved_marks[k]]; 1269 for (j = 0; j < size; j++) 1270 t[l++] = *p++; 1271 } 1272 } 1273 1274 /* 1275 * If the last character is a Starter and if we have a character 1276 * (possibly another Starter) that can be turned into a composite, 1277 * we do so and we do so until there is no more of composition 1278 * possible. 1279 */ 1280 if (comb_class[last] == U8_COMBINING_CLASS_STARTER) { 1281 p = *os; 1282 saved_l = l - disp[last]; 1283 1284 while (p < oslast) { 1285 size = u8_number_of_bytes[*p]; 1286 if (size <= 1 || (p + size) > oslast) 1287 break; 1288 1289 saved_p = p; 1290 1291 for (i = 0; i < size; i++) 1292 tc[i] = *p++; 1293 1294 q = find_composition_start(uv, t + saved_l, 1295 l - saved_l); 1296 if (q == NULL) { 1297 p = saved_p; 1298 break; 1299 } 1300 1301 match_not_found = B_TRUE; 1302 1303 for (C = *q++; C > 0; C--) { 1304 for (k = 0; k < size; q++, k++) 1305 if (*q != tc[k]) 1306 break; 1307 1308 if (k >= size && *q == U8_TBL_ELEMENT_FILLER) { 1309 match_not_found = B_FALSE; 1310 1311 l = saved_l; 1312 1313 while (*++q != U8_TBL_ELEMENT_FILLER) { 1314 /* 1315 * This is practically 1316 * impossible but we don't 1317 * want to take any chances. 1318 */ 1319 if (l >= 1320 U8_STREAM_SAFE_TEXT_MAX) { 1321 p = saved_p; 1322 goto SAFE_RETURN; 1323 } 1324 t[l++] = *q; 1325 } 1326 1327 break; 1328 } 1329 1330 if (*q != U8_TBL_ELEMENT_FILLER) 1331 while (*++q != U8_TBL_ELEMENT_FILLER) 1332 ; 1333 while (*++q != U8_TBL_ELEMENT_FILLER) 1334 ; 1335 q++; 1336 } 1337 1338 if (match_not_found) { 1339 p = saved_p; 1340 break; 1341 } 1342 } 1343 SAFE_RETURN: 1344 *os = p; 1345 } 1346 1347 /* 1348 * Now we copy over the temporary string to the target string. 1349 * Since composition always reduces the number of characters or 1350 * the number of characters stay, we don't need to worry about 1351 * the buffer overflow here. 1352 */ 1353 for (i = 0; i < l; i++) 1354 s[i] = t[i]; 1355 s[l] = '\0'; 1356 1357 return (l); 1358 } 1359 1360 /* 1361 * The collect_a_seq() function checks on the given string s, collect 1362 * a sequence of characters at u8s, and return the sequence. While it collects 1363 * a sequence, it also applies case conversion, canonical or compatibility 1364 * decomposition, canonical decomposition, or some or all of them and 1365 * in that order. 1366 * 1367 * The collected sequence cannot be bigger than 32 characters since if 1368 * it is having more than 31 characters, the sequence will be terminated 1369 * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into 1370 * a Stream-Safe Text. The collected sequence is always terminated with 1371 * a null byte and the return value is the byte length of the sequence 1372 * including 0. The return value does not include the terminating 1373 * null byte. 1374 */ 1375 static size_t 1376 collect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast, 1377 boolean_t is_it_toupper, boolean_t is_it_tolower, 1378 boolean_t canonical_decomposition, boolean_t compatibility_decomposition, 1379 boolean_t canonical_composition, 1380 int *errnum, u8_normalization_states_t *state) 1381 { 1382 uchar_t *s; 1383 int sz; 1384 int saved_sz; 1385 size_t i; 1386 size_t j; 1387 size_t k; 1388 size_t l; 1389 uchar_t comb_class[U8_MAX_CHARS_A_SEQ]; 1390 uchar_t disp[U8_MAX_CHARS_A_SEQ]; 1391 uchar_t start[U8_MAX_CHARS_A_SEQ]; 1392 uchar_t u8t[U8_MB_CUR_MAX] = { '\0' }; 1393 uchar_t uts[U8_STREAM_SAFE_TEXT_MAX + 1]; 1394 uchar_t tc; 1395 size_t last; 1396 size_t saved_last; 1397 uint32_t u1; 1398 1399 /* 1400 * Save the source string pointer which we will return a changed 1401 * pointer if we do processing. 1402 */ 1403 s = *source; 1404 1405 /* 1406 * The following is a fallback for just in case callers are not 1407 * checking the string boundaries before the calling. 1408 */ 1409 if (s >= slast) { 1410 u8s[0] = '\0'; 1411 1412 return (0); 1413 } 1414 1415 /* 1416 * As the first thing, let's collect a character and do case 1417 * conversion if necessary. 1418 */ 1419 1420 sz = u8_number_of_bytes[*s]; 1421 1422 if (sz < 0) { 1423 *errnum = EILSEQ; 1424 1425 u8s[0] = *s++; 1426 u8s[1] = '\0'; 1427 1428 *source = s; 1429 1430 return (1); 1431 } 1432 1433 if (sz == 1) { 1434 if (is_it_toupper) 1435 u8s[0] = U8_ASCII_TOUPPER(*s); 1436 else if (is_it_tolower) 1437 u8s[0] = U8_ASCII_TOLOWER(*s); 1438 else 1439 u8s[0] = *s; 1440 s++; 1441 u8s[1] = '\0'; 1442 } else if ((s + sz) > slast) { 1443 *errnum = EINVAL; 1444 1445 for (i = 0; s < slast; ) 1446 u8s[i++] = *s++; 1447 u8s[i] = '\0'; 1448 1449 *source = s; 1450 1451 return (i); 1452 } else { 1453 if (is_it_toupper || is_it_tolower) { 1454 i = do_case_conv(uv, u8s, s, sz, is_it_toupper); 1455 s += sz; 1456 sz = i; 1457 } else { 1458 for (i = 0; i < sz; ) 1459 u8s[i++] = *s++; 1460 u8s[i] = '\0'; 1461 } 1462 } 1463 1464 /* 1465 * And then canonical/compatibility decomposition followed by 1466 * an optional canonical composition. Please be noted that 1467 * canonical composition is done only when a decomposition is 1468 * done. 1469 */ 1470 if (canonical_decomposition || compatibility_decomposition) { 1471 if (sz == 1) { 1472 *state = U8_STATE_START; 1473 1474 saved_sz = 1; 1475 1476 comb_class[0] = 0; 1477 start[0] = 0; 1478 disp[0] = 1; 1479 1480 last = 1; 1481 } else { 1482 saved_sz = do_decomp(uv, u8s, u8s, sz, 1483 canonical_decomposition, state); 1484 1485 last = 0; 1486 1487 for (i = 0; i < saved_sz; ) { 1488 sz = u8_number_of_bytes[u8s[i]]; 1489 1490 comb_class[last] = combining_class(uv, 1491 u8s + i, sz); 1492 start[last] = i; 1493 disp[last] = sz; 1494 1495 last++; 1496 i += sz; 1497 } 1498 1499 /* 1500 * Decomposition yields various Hangul related 1501 * states but not on combining marks. We need to 1502 * find out at here by checking on the last 1503 * character. 1504 */ 1505 if (*state == U8_STATE_START) { 1506 if (comb_class[last - 1]) 1507 *state = U8_STATE_COMBINING_MARK; 1508 } 1509 } 1510 1511 saved_last = last; 1512 1513 while (s < slast) { 1514 sz = u8_number_of_bytes[*s]; 1515 1516 /* 1517 * If this is an illegal character, an incomplete 1518 * character, or an 7-bit ASCII Starter character, 1519 * then we have collected a sequence; break and let 1520 * the next call deal with the two cases. 1521 * 1522 * Note that this is okay only if you are using this 1523 * function with a fixed length string, not on 1524 * a buffer with multiple calls of one chunk at a time. 1525 */ 1526 if (sz <= 1) { 1527 break; 1528 } else if ((s + sz) > slast) { 1529 break; 1530 } else { 1531 /* 1532 * If the previous character was a Hangul Jamo 1533 * and this character is a Hangul Jamo that 1534 * can be conjoined, we collect the Jamo. 1535 */ 1536 if (*s == U8_HANGUL_JAMO_1ST_BYTE) { 1537 U8_PUT_3BYTES_INTO_UTF32(u1, 1538 *s, *(s + 1), *(s + 2)); 1539 1540 if (U8_HANGUL_COMPOSABLE_L_V(*state, 1541 u1)) { 1542 i = 0; 1543 *state = U8_STATE_HANGUL_LV; 1544 goto COLLECT_A_HANGUL; 1545 } 1546 1547 if (U8_HANGUL_COMPOSABLE_LV_T(*state, 1548 u1)) { 1549 i = 0; 1550 *state = U8_STATE_HANGUL_LVT; 1551 goto COLLECT_A_HANGUL; 1552 } 1553 } 1554 1555 /* 1556 * Regardless of whatever it was, if this is 1557 * a Starter, we don't collect the character 1558 * since that's a new start and we will deal 1559 * with it at the next time. 1560 */ 1561 i = combining_class(uv, s, sz); 1562 if (i == U8_COMBINING_CLASS_STARTER) 1563 break; 1564 1565 /* 1566 * We know the current character is a combining 1567 * mark. If the previous character wasn't 1568 * a Starter (not Hangul) or a combining mark, 1569 * then, we don't collect this combining mark. 1570 */ 1571 if (*state != U8_STATE_START && 1572 *state != U8_STATE_COMBINING_MARK) 1573 break; 1574 1575 *state = U8_STATE_COMBINING_MARK; 1576 COLLECT_A_HANGUL: 1577 /* 1578 * If we collected a Starter and combining 1579 * marks up to 30, i.e., total 31 characters, 1580 * then, we terminate this degenerately long 1581 * combining sequence with a U+034F COMBINING 1582 * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in 1583 * UTF-8 and turn this into a Stream-Safe 1584 * Text. This will be extremely rare but 1585 * possible. 1586 * 1587 * The following will also guarantee that 1588 * we are not writing more than 32 characters 1589 * plus a NULL at u8s[]. 1590 */ 1591 if (last >= U8_UPPER_LIMIT_IN_A_SEQ) { 1592 TURN_STREAM_SAFE: 1593 *state = U8_STATE_START; 1594 comb_class[last] = 0; 1595 start[last] = saved_sz; 1596 disp[last] = 2; 1597 last++; 1598 1599 u8s[saved_sz++] = 0xCD; 1600 u8s[saved_sz++] = 0x8F; 1601 1602 break; 1603 } 1604 1605 /* 1606 * Some combining marks also do decompose into 1607 * another combining mark or marks. 1608 */ 1609 if (*state == U8_STATE_COMBINING_MARK) { 1610 k = last; 1611 l = sz; 1612 i = do_decomp(uv, uts, s, sz, 1613 canonical_decomposition, state); 1614 for (j = 0; j < i; ) { 1615 sz = u8_number_of_bytes[uts[j]]; 1616 1617 comb_class[last] = 1618 combining_class(uv, 1619 uts + j, sz); 1620 start[last] = saved_sz + j; 1621 disp[last] = sz; 1622 1623 last++; 1624 if (last >= 1625 U8_UPPER_LIMIT_IN_A_SEQ) { 1626 last = k; 1627 goto TURN_STREAM_SAFE; 1628 } 1629 j += sz; 1630 } 1631 1632 *state = U8_STATE_COMBINING_MARK; 1633 sz = i; 1634 s += l; 1635 1636 for (i = 0; i < sz; i++) 1637 u8s[saved_sz++] = uts[i]; 1638 } else { 1639 comb_class[last] = i; 1640 start[last] = saved_sz; 1641 disp[last] = sz; 1642 last++; 1643 1644 for (i = 0; i < sz; i++) 1645 u8s[saved_sz++] = *s++; 1646 } 1647 1648 /* 1649 * If this is U+0345 COMBINING GREEK 1650 * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a., 1651 * iota subscript, and need to be converted to 1652 * uppercase letter, convert it to U+0399 GREEK 1653 * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8), 1654 * i.e., convert to capital adscript form as 1655 * specified in the Unicode standard. 1656 * 1657 * This is the only special case of (ambiguous) 1658 * case conversion at combining marks and 1659 * probably the standard will never have 1660 * anything similar like this in future. 1661 */ 1662 if (is_it_toupper && sz >= 2 && 1663 u8s[saved_sz - 2] == 0xCD && 1664 u8s[saved_sz - 1] == 0x85) { 1665 u8s[saved_sz - 2] = 0xCE; 1666 u8s[saved_sz - 1] = 0x99; 1667 } 1668 } 1669 } 1670 1671 /* 1672 * Let's try to ensure a canonical ordering for the collected 1673 * combining marks. We do this only if we have collected 1674 * at least one more non-Starter. (The decomposition mapping 1675 * data tables have fully (and recursively) expanded and 1676 * canonically ordered decompositions.) 1677 * 1678 * The U8_SWAP_COMB_MARKS() convenience macro has some 1679 * assumptions and we are meeting the assumptions. 1680 */ 1681 last--; 1682 if (last >= saved_last) { 1683 for (i = 0; i < last; i++) 1684 for (j = last; j > i; j--) 1685 if (comb_class[j] && 1686 comb_class[j - 1] > comb_class[j]) { 1687 U8_SWAP_COMB_MARKS(j - 1, j); 1688 } 1689 } 1690 1691 *source = s; 1692 1693 if (! canonical_composition) { 1694 u8s[saved_sz] = '\0'; 1695 return (saved_sz); 1696 } 1697 1698 /* 1699 * Now do the canonical composition. Note that we do this 1700 * only after a canonical or compatibility decomposition to 1701 * finish up NFC or NFKC. 1702 */ 1703 sz = do_composition(uv, u8s, comb_class, start, disp, last, 1704 &s, slast); 1705 } 1706 1707 *source = s; 1708 1709 return ((size_t)sz); 1710 } 1711 1712 /* 1713 * The do_norm_compare() function does string comparison based on Unicode 1714 * simple case mappings and Unicode Normalization definitions. 1715 * 1716 * It does so by collecting a sequence of character at a time and comparing 1717 * the collected sequences from the strings. 1718 * 1719 * The meanings on the return values are the same as the usual strcmp(). 1720 */ 1721 static int 1722 do_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2, 1723 int flag, int *errnum) 1724 { 1725 int result; 1726 size_t sz1; 1727 size_t sz2; 1728 uchar_t u8s1[U8_STREAM_SAFE_TEXT_MAX + 1]; 1729 uchar_t u8s2[U8_STREAM_SAFE_TEXT_MAX + 1]; 1730 uchar_t *s1last; 1731 uchar_t *s2last; 1732 boolean_t is_it_toupper; 1733 boolean_t is_it_tolower; 1734 boolean_t canonical_decomposition; 1735 boolean_t compatibility_decomposition; 1736 boolean_t canonical_composition; 1737 u8_normalization_states_t state; 1738 1739 s1last = s1 + n1; 1740 s2last = s2 + n2; 1741 1742 is_it_toupper = flag & U8_TEXTPREP_TOUPPER; 1743 is_it_tolower = flag & U8_TEXTPREP_TOLOWER; 1744 canonical_decomposition = flag & U8_CANON_DECOMP; 1745 compatibility_decomposition = flag & U8_COMPAT_DECOMP; 1746 canonical_composition = flag & U8_CANON_COMP; 1747 1748 while (s1 < s1last && s2 < s2last) { 1749 /* 1750 * If the current character is a 7-bit ASCII and the last 1751 * character, or, if the current character and the next 1752 * character are both some 7-bit ASCII characters then 1753 * we treat the current character as a sequence. 1754 * 1755 * In any other cases, we need to call collect_a_seq(). 1756 */ 1757 1758 if (U8_ISASCII(*s1) && ((s1 + 1) >= s1last || 1759 ((s1 + 1) < s1last && U8_ISASCII(*(s1 + 1))))) { 1760 if (is_it_toupper) 1761 u8s1[0] = U8_ASCII_TOUPPER(*s1); 1762 else if (is_it_tolower) 1763 u8s1[0] = U8_ASCII_TOLOWER(*s1); 1764 else 1765 u8s1[0] = *s1; 1766 u8s1[1] = '\0'; 1767 sz1 = 1; 1768 s1++; 1769 } else { 1770 state = U8_STATE_START; 1771 sz1 = collect_a_seq(uv, u8s1, &s1, s1last, 1772 is_it_toupper, is_it_tolower, 1773 canonical_decomposition, 1774 compatibility_decomposition, 1775 canonical_composition, errnum, &state); 1776 } 1777 1778 if (U8_ISASCII(*s2) && ((s2 + 1) >= s2last || 1779 ((s2 + 1) < s2last && U8_ISASCII(*(s2 + 1))))) { 1780 if (is_it_toupper) 1781 u8s2[0] = U8_ASCII_TOUPPER(*s2); 1782 else if (is_it_tolower) 1783 u8s2[0] = U8_ASCII_TOLOWER(*s2); 1784 else 1785 u8s2[0] = *s2; 1786 u8s2[1] = '\0'; 1787 sz2 = 1; 1788 s2++; 1789 } else { 1790 state = U8_STATE_START; 1791 sz2 = collect_a_seq(uv, u8s2, &s2, s2last, 1792 is_it_toupper, is_it_tolower, 1793 canonical_decomposition, 1794 compatibility_decomposition, 1795 canonical_composition, errnum, &state); 1796 } 1797 1798 /* 1799 * Now compare the two characters. If they are the same, 1800 * we move on to the next character sequences. 1801 */ 1802 if (sz1 == 1 && sz2 == 1) { 1803 if (*u8s1 > *u8s2) 1804 return (1); 1805 if (*u8s1 < *u8s2) 1806 return (-1); 1807 } else { 1808 result = strcmp((const char *)u8s1, (const char *)u8s2); 1809 if (result != 0) 1810 return (result); 1811 } 1812 } 1813 1814 /* 1815 * We compared until the end of either or both strings. 1816 * 1817 * If we reached to or went over the ends for the both, that means 1818 * they are the same. 1819 * 1820 * If we reached only one end, that means the other string has 1821 * something which then can be used to determine the return value. 1822 */ 1823 if (s1 >= s1last) { 1824 if (s2 >= s2last) 1825 return (0); 1826 return (-1); 1827 } 1828 return (1); 1829 } 1830 1831 /* 1832 * The u8_strcmp() function compares two UTF-8 strings quite similar to 1833 * the strcmp(). For the comparison, however, Unicode Normalization specific 1834 * equivalency and Unicode simple case conversion mappings based equivalency 1835 * can be requested and checked against. 1836 */ 1837 int 1838 u8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv, 1839 int *errnum) 1840 { 1841 int f; 1842 size_t n1; 1843 size_t n2; 1844 1845 *errnum = 0; 1846 1847 /* 1848 * Check on the requested Unicode version, case conversion, and 1849 * normalization flag values. 1850 */ 1851 1852 if (uv > U8_UNICODE_LATEST) { 1853 *errnum = ERANGE; 1854 uv = U8_UNICODE_LATEST; 1855 } 1856 1857 if (flag == 0) { 1858 flag = U8_STRCMP_CS; 1859 } else { 1860 f = flag & (U8_STRCMP_CS | U8_STRCMP_CI_UPPER | 1861 U8_STRCMP_CI_LOWER); 1862 if (f == 0) { 1863 flag |= U8_STRCMP_CS; 1864 } else if (f != U8_STRCMP_CS && f != U8_STRCMP_CI_UPPER && 1865 f != U8_STRCMP_CI_LOWER) { 1866 *errnum = EBADF; 1867 flag = U8_STRCMP_CS; 1868 } 1869 1870 f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP); 1871 if (f && f != U8_STRCMP_NFD && f != U8_STRCMP_NFC && 1872 f != U8_STRCMP_NFKD && f != U8_STRCMP_NFKC) { 1873 *errnum = EBADF; 1874 flag = U8_STRCMP_CS; 1875 } 1876 } 1877 1878 if (flag == U8_STRCMP_CS) { 1879 return (n == 0 ? strcmp(s1, s2) : strncmp(s1, s2, n)); 1880 } 1881 1882 n1 = strlen(s1); 1883 n2 = strlen(s2); 1884 if (n != 0) { 1885 if (n < n1) 1886 n1 = n; 1887 if (n < n2) 1888 n2 = n; 1889 } 1890 1891 /* 1892 * Simple case conversion can be done much faster and so we do 1893 * them separately here. 1894 */ 1895 if (flag == U8_STRCMP_CI_UPPER) { 1896 return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2, 1897 n1, n2, B_TRUE, errnum)); 1898 } else if (flag == U8_STRCMP_CI_LOWER) { 1899 return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2, 1900 n1, n2, B_FALSE, errnum)); 1901 } 1902 1903 return (do_norm_compare(uv, (uchar_t *)s1, (uchar_t *)s2, n1, n2, 1904 flag, errnum)); 1905 } 1906 1907 size_t 1908 u8_textprep_str(char *inarray, size_t *inlen, char *outarray, size_t *outlen, 1909 int flag, size_t unicode_version, int *errnum) 1910 { 1911 int f; 1912 int sz; 1913 uchar_t *ib; 1914 uchar_t *ibtail; 1915 uchar_t *ob; 1916 uchar_t *obtail; 1917 boolean_t do_not_ignore_null; 1918 boolean_t do_not_ignore_invalid; 1919 boolean_t is_it_toupper; 1920 boolean_t is_it_tolower; 1921 boolean_t canonical_decomposition; 1922 boolean_t compatibility_decomposition; 1923 boolean_t canonical_composition; 1924 size_t ret_val; 1925 size_t i; 1926 size_t j; 1927 uchar_t u8s[U8_STREAM_SAFE_TEXT_MAX + 1]; 1928 u8_normalization_states_t state; 1929 1930 if (unicode_version > U8_UNICODE_LATEST) { 1931 *errnum = ERANGE; 1932 return ((size_t)-1); 1933 } 1934 1935 f = flag & (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER); 1936 if (f == (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER)) { 1937 *errnum = EBADF; 1938 return ((size_t)-1); 1939 } 1940 1941 f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP); 1942 if (f && f != U8_TEXTPREP_NFD && f != U8_TEXTPREP_NFC && 1943 f != U8_TEXTPREP_NFKD && f != U8_TEXTPREP_NFKC) { 1944 *errnum = EBADF; 1945 return ((size_t)-1); 1946 } 1947 1948 if (inarray == NULL || *inlen == 0) 1949 return (0); 1950 1951 if (outarray == NULL) { 1952 *errnum = E2BIG; 1953 return ((size_t)-1); 1954 } 1955 1956 ib = (uchar_t *)inarray; 1957 ob = (uchar_t *)outarray; 1958 ibtail = ib + *inlen; 1959 obtail = ob + *outlen; 1960 1961 do_not_ignore_null = !(flag & U8_TEXTPREP_IGNORE_NULL); 1962 do_not_ignore_invalid = !(flag & U8_TEXTPREP_IGNORE_INVALID); 1963 is_it_toupper = flag & U8_TEXTPREP_TOUPPER; 1964 is_it_tolower = flag & U8_TEXTPREP_TOLOWER; 1965 1966 ret_val = 0; 1967 1968 /* 1969 * If we don't have a normalization flag set, we do the simple case 1970 * conversion based text preparation separately below. Text 1971 * preparation involving Normalization will be done in the false task 1972 * block, again, separately since it will take much more time and 1973 * resource than doing simple case conversions. 1974 */ 1975 if (f == 0) { 1976 while (ib < ibtail) { 1977 if (*ib == '\0' && do_not_ignore_null) 1978 break; 1979 1980 sz = u8_number_of_bytes[*ib]; 1981 1982 if (sz < 0) { 1983 if (do_not_ignore_invalid) { 1984 *errnum = EILSEQ; 1985 ret_val = (size_t)-1; 1986 break; 1987 } 1988 1989 sz = 1; 1990 ret_val++; 1991 } 1992 1993 if (sz == 1) { 1994 if (ob >= obtail) { 1995 *errnum = E2BIG; 1996 ret_val = (size_t)-1; 1997 break; 1998 } 1999 2000 if (is_it_toupper) 2001 *ob = U8_ASCII_TOUPPER(*ib); 2002 else if (is_it_tolower) 2003 *ob = U8_ASCII_TOLOWER(*ib); 2004 else 2005 *ob = *ib; 2006 ib++; 2007 ob++; 2008 } else if ((ib + sz) > ibtail) { 2009 if (do_not_ignore_invalid) { 2010 *errnum = EINVAL; 2011 ret_val = (size_t)-1; 2012 break; 2013 } 2014 2015 if ((obtail - ob) < (ibtail - ib)) { 2016 *errnum = E2BIG; 2017 ret_val = (size_t)-1; 2018 break; 2019 } 2020 2021 /* 2022 * We treat the remaining incomplete character 2023 * bytes as a character. 2024 */ 2025 ret_val++; 2026 2027 while (ib < ibtail) 2028 *ob++ = *ib++; 2029 } else { 2030 if (is_it_toupper || is_it_tolower) { 2031 i = do_case_conv(unicode_version, u8s, 2032 ib, sz, is_it_toupper); 2033 2034 if ((obtail - ob) < i) { 2035 *errnum = E2BIG; 2036 ret_val = (size_t)-1; 2037 break; 2038 } 2039 2040 ib += sz; 2041 2042 for (sz = 0; sz < i; sz++) 2043 *ob++ = u8s[sz]; 2044 } else { 2045 if ((obtail - ob) < sz) { 2046 *errnum = E2BIG; 2047 ret_val = (size_t)-1; 2048 break; 2049 } 2050 2051 for (i = 0; i < sz; i++) 2052 *ob++ = *ib++; 2053 } 2054 } 2055 } 2056 } else { 2057 canonical_decomposition = flag & U8_CANON_DECOMP; 2058 compatibility_decomposition = flag & U8_COMPAT_DECOMP; 2059 canonical_composition = flag & U8_CANON_COMP; 2060 2061 while (ib < ibtail) { 2062 if (*ib == '\0' && do_not_ignore_null) 2063 break; 2064 2065 /* 2066 * If the current character is a 7-bit ASCII 2067 * character and it is the last character, or, 2068 * if the current character is a 7-bit ASCII 2069 * character and the next character is also a 7-bit 2070 * ASCII character, then, we copy over this 2071 * character without going through collect_a_seq(). 2072 * 2073 * In any other cases, we need to look further with 2074 * the collect_a_seq() function. 2075 */ 2076 if (U8_ISASCII(*ib) && ((ib + 1) >= ibtail || 2077 ((ib + 1) < ibtail && U8_ISASCII(*(ib + 1))))) { 2078 if (ob >= obtail) { 2079 *errnum = E2BIG; 2080 ret_val = (size_t)-1; 2081 break; 2082 } 2083 2084 if (is_it_toupper) 2085 *ob = U8_ASCII_TOUPPER(*ib); 2086 else if (is_it_tolower) 2087 *ob = U8_ASCII_TOLOWER(*ib); 2088 else 2089 *ob = *ib; 2090 ib++; 2091 ob++; 2092 } else { 2093 *errnum = 0; 2094 state = U8_STATE_START; 2095 2096 j = collect_a_seq(unicode_version, u8s, 2097 &ib, ibtail, 2098 is_it_toupper, 2099 is_it_tolower, 2100 canonical_decomposition, 2101 compatibility_decomposition, 2102 canonical_composition, 2103 errnum, &state); 2104 2105 if (*errnum && do_not_ignore_invalid) { 2106 ret_val = (size_t)-1; 2107 break; 2108 } 2109 2110 if ((obtail - ob) < j) { 2111 *errnum = E2BIG; 2112 ret_val = (size_t)-1; 2113 break; 2114 } 2115 2116 for (i = 0; i < j; i++) 2117 *ob++ = u8s[i]; 2118 } 2119 } 2120 } 2121 2122 *inlen = ibtail - ib; 2123 *outlen = obtail - ob; 2124 2125 return (ret_val); 2126 } 2127 2128 #if defined(_KERNEL) 2129 static int __init 2130 unicode_init(void) 2131 { 2132 return (0); 2133 } 2134 2135 static void __exit 2136 unicode_fini(void) 2137 { 2138 } 2139 2140 module_init(unicode_init); 2141 module_exit(unicode_fini); 2142 #endif 2143 2144 ZFS_MODULE_DESCRIPTION("Unicode implementation"); 2145 ZFS_MODULE_AUTHOR(ZFS_META_AUTHOR); 2146 ZFS_MODULE_LICENSE(ZFS_META_LICENSE); 2147 ZFS_MODULE_VERSION(ZFS_META_VERSION "-" ZFS_META_RELEASE); 2148 2149 EXPORT_SYMBOL(u8_validate); 2150 EXPORT_SYMBOL(u8_strcmp); 2151 EXPORT_SYMBOL(u8_textprep_str); 2152