1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 14 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved. 15 */ 16 17 /* 18 * The functions in this file convert from the standard multibyte forms 19 * to the wide character forms used internally by libc. Unfortunately, 20 * this approach means that we need a method for each and every encoding. 21 */ 22 23 #include <stdlib.h> 24 #include <wchar.h> 25 #include <string.h> 26 #include <note.h> 27 #include <sys/types.h> 28 #include "localedef.h" 29 30 static int towide_none(wchar_t *, const char *, unsigned); 31 static int towide_utf8(wchar_t *, const char *, unsigned); 32 static int towide_big5(wchar_t *, const char *, unsigned); 33 static int towide_gbk(wchar_t *, const char *, unsigned); 34 static int towide_gb2312(wchar_t *, const char *, unsigned); 35 static int towide_gb18030(wchar_t *, const char *, unsigned); 36 static int towide_mskanji(wchar_t *, const char *, unsigned); 37 static int towide_euccn(wchar_t *, const char *, unsigned); 38 static int towide_eucjp(wchar_t *, const char *, unsigned); 39 static int towide_euckr(wchar_t *, const char *, unsigned); 40 static int towide_euctw(wchar_t *, const char *, unsigned); 41 42 static int tomb_none(char *, wchar_t); 43 static int tomb_utf8(char *, wchar_t); 44 static int tomb_mbs(char *, wchar_t); 45 46 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none; 47 static int (*_tomb)(char *, wchar_t) = tomb_none; 48 static const char *_encoding = "NONE"; 49 static int _nbits = 7; 50 51 /* 52 * Table of supported encodings. We only bother to list the multibyte 53 * encodings here, because single byte locales are handed by "NONE". 54 */ 55 static struct { 56 const char *name; 57 /* the name that the underlying libc implemenation uses */ 58 const char *cname; 59 /* the maximum number of bits required for priorities */ 60 int nbits; 61 int (*towide)(wchar_t *, const char *, unsigned); 62 int (*tomb)(char *, wchar_t); 63 } mb_encodings[] = { 64 /* 65 * UTF8 values max out at 0x1fffff (although in theory there could 66 * be later extensions, but it won't happen.) This means we only need 67 * 21 bits to be able to encode the entire range of priorities. 68 */ 69 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 70 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 71 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 72 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 73 74 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs }, 75 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs }, 76 /* 77 * Becuase the 3-byte form of EUC-JP use the same leading byte, 78 * only 17 bits required to provide unique priorities. (The low 79 * bit of that first byte is set.) By setting this value low, 80 * we can get by with only 3 bytes in the strxfrm expansion. 81 */ 82 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, 83 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, 84 85 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs }, 86 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs }, 87 /* 88 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the 89 * high order byte is 0x8E. However, with 4 byte encodings, 90 * the third byte will be A0-B0. So we only need to consider 91 * the lower order 24 bits for collation. 92 */ 93 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs }, 94 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs }, 95 96 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, 97 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, 98 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs }, 99 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, 100 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, 101 102 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs }, 103 { "big5", "BIG5", 16, towide_big5, tomb_mbs }, 104 { "Big5", "BIG5", 16, towide_big5, tomb_mbs }, 105 106 { "GBK", "GBK", 16, towide_gbk, tomb_mbs }, 107 108 /* 109 * GB18030 can get away with just 31 bits. This is because the 110 * high order bit is always set for 4 byte values, and the 111 * at least one of the other bits in that 4 byte value will 112 * be non-zero. 113 */ 114 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs }, 115 116 /* 117 * This should probably be an aliase for euc-cn, or vice versa. 118 */ 119 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs }, 120 121 { NULL, NULL }, 122 }; 123 124 static char * 125 show_mb(const char *mb) 126 { 127 static char buf[64]; 128 129 /* ASCII stuff we just print */ 130 if (isascii(*mb) && isgraph(*mb)) { 131 buf[0] = *mb; 132 buf[1] = 0; 133 return (buf); 134 } 135 buf[0] = 0; 136 while (*mb != 0) { 137 char scr[8]; 138 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb); 139 (void) strlcat(buf, scr, sizeof (buf)); 140 mb++; 141 } 142 return (buf); 143 } 144 145 static char *widemsg; 146 147 void 148 werr(const char *fmt, ...) 149 { 150 char *msg; 151 152 va_list va; 153 va_start(va, fmt); 154 (void) vasprintf(&msg, fmt, va); 155 va_end(va); 156 157 free(widemsg); 158 widemsg = msg; 159 } 160 161 /* 162 * This is used for 8-bit encodings. 163 */ 164 int 165 towide_none(wchar_t *c, const char *mb, unsigned n) 166 { 167 _NOTE(ARGUNUSED(n)); 168 169 if (mb_cur_max != 1) { 170 werr("invalid or unsupported multibyte locale"); 171 return (-1); 172 } 173 *c = (uint8_t)*mb; 174 return (1); 175 } 176 177 int 178 tomb_none(char *mb, wchar_t wc) 179 { 180 if (mb_cur_max != 1) { 181 werr("invalid or unsupported multibyte locale"); 182 return (-1); 183 } 184 *(uint8_t *)mb = (wc & 0xff); 185 mb[1] = 0; 186 return (1); 187 } 188 189 /* 190 * UTF-8 stores wide characters in UTF-32 form. 191 */ 192 int 193 towide_utf8(wchar_t *wc, const char *mb, unsigned n) 194 { 195 wchar_t c; 196 int nb; 197 int lv; /* lowest legal value */ 198 int i; 199 const uint8_t *s = (const uint8_t *)mb; 200 201 c = *s; 202 203 if ((c & 0x80) == 0) { 204 /* 7-bit ASCII */ 205 *wc = c; 206 return (1); 207 } else if ((c & 0xe0) == 0xc0) { 208 /* u80-u7ff - two bytes encoded */ 209 nb = 2; 210 lv = 0x80; 211 c &= ~0xe0; 212 } else if ((c & 0xf0) == 0xe0) { 213 /* u800-uffff - three bytes encoded */ 214 nb = 3; 215 lv = 0x800; 216 c &= ~0xf0; 217 } else if ((c & 0xf8) == 0xf0) { 218 /* u1000-u1fffff - four bytes encoded */ 219 nb = 4; 220 lv = 0x1000; 221 c &= ~0xf8; 222 } else { 223 /* 5 and 6 byte encodings are not legal unicode */ 224 werr("utf8 encoding too large (%s)", show_mb(mb)); 225 return (-1); 226 } 227 if (nb > n) { 228 werr("incomplete utf8 sequence (%s)", show_mb(mb)); 229 return (-1); 230 } 231 232 for (i = 1; i < nb; i++) { 233 if (((s[i]) & 0xc0) != 0x80) { 234 werr("illegal utf8 byte (%x)", s[i]); 235 return (-1); 236 } 237 c <<= 6; 238 c |= (s[i] & 0x3f); 239 } 240 241 if (c < lv) { 242 werr("illegal redundant utf8 encoding (%s)", show_mb(mb)); 243 return (-1); 244 } 245 *wc = c; 246 return (nb); 247 } 248 249 int 250 tomb_utf8(char *mb, wchar_t wc) 251 { 252 uint8_t *s = (uint8_t *)mb; 253 uint8_t msk; 254 int cnt; 255 int i; 256 257 if (wc <= 0x7f) { 258 s[0] = wc & 0x7f; 259 s[1] = 0; 260 return (1); 261 } 262 if (wc <= 0x7ff) { 263 cnt = 2; 264 msk = 0xc0; 265 } else if (wc <= 0xffff) { 266 cnt = 3; 267 msk = 0xe0; 268 } else if (wc <= 0x1fffff) { 269 cnt = 4; 270 msk = 0xf0; 271 } else { 272 werr("illegal uf8 char (%x)", wc); 273 return (-1); 274 } 275 for (i = cnt - 1; i; i--) { 276 s[i] = (wc & 0x3f) | 0x80; 277 wc >>= 6; 278 } 279 s[0] = (msk) | wc; 280 s[cnt] = 0; 281 return (cnt); 282 } 283 284 /* 285 * Several encodings share a simplistic dual byte encoding. In these 286 * forms, they all indicate that a two byte sequence is to be used if 287 * the first byte has its high bit set. They all store this simple 288 * encoding as a 16-bit value, although a great many of the possible 289 * code points are not used in most character sets. This gives a possible 290 * set of just over 32,000 valid code points. 291 * 292 * 0x00 - 0x7f - 1 byte encoding 293 * 0x80 - 0x7fff - illegal 294 * 0x8000 - 0xffff - 2 byte encoding 295 */ 296 static int 297 towide_dbcs(wchar_t *wc, const char *mb, unsigned n) 298 { 299 wchar_t c; 300 301 c = *(uint8_t *)mb; 302 303 if ((c & 0x80) == 0) { 304 /* 7-bit */ 305 *wc = c; 306 return (1); 307 } 308 if (n < 2) { 309 werr("incomplete character sequence (%s)", show_mb(mb)); 310 return (-1); 311 } 312 313 /* Store both bytes as a single 16-bit wide. */ 314 c <<= 8; 315 c |= (uint8_t)(mb[1]); 316 *wc = c; 317 return (2); 318 } 319 320 /* 321 * Most multibyte locales just convert the wide character to the multibyte 322 * form by stripping leading null bytes, and writing the 32-bit quantity 323 * in big-endian order. 324 */ 325 int 326 tomb_mbs(char *mb, wchar_t wc) 327 { 328 uint8_t *s = (uint8_t *)mb; 329 int n = 0, c; 330 331 if ((wc & 0xff000000U) != 0) { 332 n = 4; 333 } else if ((wc & 0x00ff0000U) != 0) { 334 n = 3; 335 } else if ((wc & 0x0000ff00U) != 0) { 336 n = 2; 337 } else { 338 n = 1; 339 } 340 c = n; 341 while (n) { 342 n--; 343 s[n] = wc & 0xff; 344 wc >>= 8; 345 } 346 /* ensure null termination */ 347 s[c] = 0; 348 return (c); 349 } 350 351 352 /* 353 * big5 is a simple dual byte character set. 354 */ 355 int 356 towide_big5(wchar_t *wc, const char *mb, unsigned n) 357 { 358 return (towide_dbcs(wc, mb, n)); 359 } 360 361 /* 362 * GBK encodes wides in the same way that big5 does, the high order 363 * bit of the first byte indicates a double byte character. 364 */ 365 int 366 towide_gbk(wchar_t *wc, const char *mb, unsigned n) 367 { 368 return (towide_dbcs(wc, mb, n)); 369 } 370 371 /* 372 * GB2312 is another DBCS. Its cleaner than others in that the second 373 * byte does not encode ASCII, but it supports characters. 374 */ 375 int 376 towide_gb2312(wchar_t *wc, const char *mb, unsigned n) 377 { 378 return (towide_dbcs(wc, mb, n)); 379 } 380 381 /* 382 * GB18030. This encodes as 8, 16, or 32-bits. 383 * 7-bit values are in 1 byte, 4 byte sequences are used when 384 * the second byte encodes 0x30-39 and all other sequences are 2 bytes. 385 */ 386 int 387 towide_gb18030(wchar_t *wc, const char *mb, unsigned n) 388 { 389 wchar_t c; 390 391 c = *(uint8_t *)mb; 392 393 if ((c & 0x80) == 0) { 394 /* 7-bit */ 395 *wc = c; 396 return (1); 397 } 398 if (n < 2) { 399 werr("incomplete character sequence (%s)", show_mb(mb)); 400 return (-1); 401 } 402 403 /* pull in the second byte */ 404 c <<= 8; 405 c |= (uint8_t)(mb[1]); 406 407 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) { 408 if (n < 4) { 409 werr("incomplete 4-byte character sequence (%s)", 410 show_mb(mb)); 411 return (-1); 412 } 413 c <<= 8; 414 c |= (uint8_t)(mb[2]); 415 c <<= 8; 416 c |= (uint8_t)(mb[3]); 417 *wc = c; 418 return (4); 419 } 420 421 *wc = c; 422 return (2); 423 } 424 425 /* 426 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it 427 * also has a range of single byte characters above 0x80. (0xa1-0xdf). 428 */ 429 int 430 towide_mskanji(wchar_t *wc, const char *mb, unsigned n) 431 { 432 wchar_t c; 433 434 c = *(uint8_t *)mb; 435 436 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) { 437 /* 7-bit */ 438 *wc = c; 439 return (1); 440 } 441 442 if (n < 2) { 443 werr("incomplete character sequence (%s)", show_mb(mb)); 444 return (-1); 445 } 446 447 /* Store both bytes as a single 16-bit wide. */ 448 c <<= 8; 449 c |= (uint8_t)(mb[1]); 450 *wc = c; 451 return (2); 452 } 453 454 /* 455 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional 456 * variable data to encode these, but we're going to treat each as independent 457 * instead. Its the only way we can sensibly move forward. 458 * 459 * Note that the way in which the different EUC forms vary is how wide 460 * CS2 and CS3 are and what the first byte of them is. 461 */ 462 static int 463 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n, 464 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width) 465 { 466 int i; 467 int width = 2; 468 wchar_t c; 469 470 c = *(uint8_t *)mb; 471 472 /* 473 * All variations of EUC encode 7-bit ASCII as one byte, and use 474 * additional bytes for more than that. 475 */ 476 if ((c & 0x80) == 0) { 477 /* 7-bit */ 478 *wc = c; 479 return (1); 480 } 481 482 /* 483 * All EUC variants reserve 0xa1-0xff to identify CS1, which 484 * is always two bytes wide. Note that unused CS will be zero, 485 * and that cannot be true because we know that the high order 486 * bit must be set. 487 */ 488 if (c >= 0xa1) { 489 width = 2; 490 } else if (c == cs2) { 491 width = cs2width; 492 } else if (c == cs3) { 493 width = cs3width; 494 } 495 496 if (n < width) { 497 werr("incomplete character sequence (%s)", show_mb(mb)); 498 return (-1); 499 } 500 501 for (i = 1; i < width; i++) { 502 /* pull in the next byte */ 503 c <<= 8; 504 c |= (uint8_t)(mb[i]); 505 } 506 507 *wc = c; 508 return (width); 509 } 510 511 /* 512 * EUC-CN encodes as follows: 513 * 514 * Code set 0 (ASCII): 0x21-0x7E 515 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE 516 * Code set 2: unused 517 * Code set 3: unused 518 */ 519 int 520 towide_euccn(wchar_t *wc, const char *mb, unsigned n) 521 { 522 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); 523 } 524 525 /* 526 * EUC-JP encodes as follows: 527 * 528 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E 529 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE 530 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF 531 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE 532 */ 533 int 534 towide_eucjp(wchar_t *wc, const char *mb, unsigned n) 535 { 536 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3)); 537 } 538 539 /* 540 * EUC-KR encodes as follows: 541 * 542 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E 543 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE 544 * Code set 2: unused 545 * Code set 3: unused 546 */ 547 int 548 towide_euckr(wchar_t *wc, const char *mb, unsigned n) 549 { 550 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0)); 551 } 552 553 /* 554 * EUC-TW encodes as follows: 555 * 556 * Code set 0 (ASCII): 0x21-0x7E 557 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE 558 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE 559 * Code set 3: unused 560 */ 561 int 562 towide_euctw(wchar_t *wc, const char *mb, unsigned n) 563 { 564 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); 565 } 566 567 /* 568 * Public entry points. 569 */ 570 571 int 572 to_wide(wchar_t *wc, const char *mb) 573 { 574 /* this won't fail hard */ 575 return (_towide(wc, mb, strlen(mb))); 576 } 577 578 int 579 to_mb(char *mb, wchar_t wc) 580 { 581 int rv; 582 583 if ((rv = _tomb(mb, wc)) < 0) { 584 errf(widemsg); 585 free(widemsg); 586 widemsg = NULL; 587 } 588 return (rv); 589 } 590 591 char * 592 to_mb_string(const wchar_t *wcs) 593 { 594 char *mbs; 595 char *ptr; 596 int len; 597 598 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1); 599 if (mbs == NULL) { 600 errf("out of memory"); 601 return (NULL); 602 } 603 ptr = mbs; 604 while (*wcs) { 605 if ((len = to_mb(ptr, *wcs)) < 0) { 606 INTERR; 607 free(mbs); 608 return (NULL); 609 } 610 wcs++; 611 ptr += len; 612 } 613 *ptr = 0; 614 return (mbs); 615 } 616 617 void 618 set_wide_encoding(const char *encoding) 619 { 620 int i; 621 622 _towide = towide_none; 623 _tomb = tomb_none; 624 _encoding = "NONE"; 625 _nbits = 8; 626 627 for (i = 0; mb_encodings[i].name; i++) { 628 if (strcasecmp(encoding, mb_encodings[i].name) == 0) { 629 _towide = mb_encodings[i].towide; 630 _tomb = mb_encodings[i].tomb; 631 _encoding = mb_encodings[i].cname; 632 _nbits = mb_encodings[i].nbits; 633 break; 634 } 635 } 636 } 637 638 const char * 639 get_wide_encoding(void) 640 { 641 return (_encoding); 642 } 643 644 int 645 max_wide(void) 646 { 647 return ((int)((1U << _nbits) - 1)); 648 } 649