1 /*- 2 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved. 4 * Copyright 2015 John Marino <draco@marino.st> 5 * 6 * This source code is derived from the illumos localedef command, and 7 * provided under BSD-style license terms by Nexenta Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * The functions in this file convert from the standard multibyte forms 34 * to the wide character forms used internally by libc. Unfortunately, 35 * this approach means that we need a method for each and every encoding. 36 */ 37 #include <sys/cdefs.h> 38 #include <ctype.h> 39 #include <stdlib.h> 40 #include <wchar.h> 41 #include <string.h> 42 #include <sys/types.h> 43 #include "localedef.h" 44 45 static int towide_none(wchar_t *, const char *, unsigned); 46 static int towide_utf8(wchar_t *, const char *, unsigned); 47 static int towide_big5(wchar_t *, const char *, unsigned); 48 static int towide_gbk(wchar_t *, const char *, unsigned); 49 static int towide_gb2312(wchar_t *, const char *, unsigned); 50 static int towide_gb18030(wchar_t *, const char *, unsigned); 51 static int towide_mskanji(wchar_t *, const char *, unsigned); 52 static int towide_euccn(wchar_t *, const char *, unsigned); 53 static int towide_eucjp(wchar_t *, const char *, unsigned); 54 static int towide_euckr(wchar_t *, const char *, unsigned); 55 static int towide_euctw(wchar_t *, const char *, unsigned); 56 57 static int tomb_none(char *, wchar_t); 58 static int tomb_utf8(char *, wchar_t); 59 static int tomb_mbs(char *, wchar_t); 60 61 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none; 62 static int (*_tomb)(char *, wchar_t) = tomb_none; 63 static char _encoding_buffer[20] = {'N','O','N','E'}; 64 static const char *_encoding = _encoding_buffer; 65 static int _nbits = 7; 66 67 /* 68 * Table of supported encodings. We only bother to list the multibyte 69 * encodings here, because single byte locales are handed by "NONE". 70 */ 71 static struct { 72 const char *name; 73 /* the name that the underlying libc implemenation uses */ 74 const char *cname; 75 /* the maximum number of bits required for priorities */ 76 int nbits; 77 int (*towide)(wchar_t *, const char *, unsigned); 78 int (*tomb)(char *, wchar_t); 79 } mb_encodings[] = { 80 /* 81 * UTF8 values max out at 0x1fffff (although in theory there could 82 * be later extensions, but it won't happen.) This means we only need 83 * 21 bits to be able to encode the entire range of priorities. 84 */ 85 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 86 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 87 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 88 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 89 90 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs }, 91 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs }, 92 /* 93 * Because the 3-byte form of EUC-JP use the same leading byte, 94 * only 17 bits required to provide unique priorities. (The low 95 * bit of that first byte is set.) By setting this value low, 96 * we can get by with only 3 bytes in the strxfrm expansion. 97 */ 98 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, 99 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, 100 101 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs }, 102 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs }, 103 /* 104 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the 105 * high order byte is 0x8E. However, with 4 byte encodings, 106 * the third byte will be A0-B0. So we only need to consider 107 * the lower order 24 bits for collation. 108 */ 109 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs }, 110 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs }, 111 112 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, 113 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, 114 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs }, 115 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, 116 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, 117 118 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs }, 119 { "big5", "BIG5", 16, towide_big5, tomb_mbs }, 120 { "Big5", "BIG5", 16, towide_big5, tomb_mbs }, 121 122 { "GBK", "GBK", 16, towide_gbk, tomb_mbs }, 123 124 /* 125 * GB18030 can get away with just 31 bits. This is because the 126 * high order bit is always set for 4 byte values, and the 127 * at least one of the other bits in that 4 byte value will 128 * be non-zero. 129 */ 130 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs }, 131 132 /* 133 * This should probably be an aliase for euc-cn, or vice versa. 134 */ 135 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs }, 136 137 { NULL, NULL, 0, 0, 0 }, 138 }; 139 140 static char * 141 show_mb(const char *mb) 142 { 143 static char buf[64]; 144 145 /* ASCII stuff we just print */ 146 if (isascii(*mb) && isgraph(*mb)) { 147 buf[0] = *mb; 148 buf[1] = 0; 149 return (buf); 150 } 151 buf[0] = 0; 152 while (*mb != 0) { 153 char scr[8]; 154 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb); 155 (void) strlcat(buf, scr, sizeof (buf)); 156 mb++; 157 } 158 return (buf); 159 } 160 161 static char *widemsg; 162 163 void 164 werr(const char *fmt, ...) 165 { 166 char *msg; 167 168 va_list va; 169 va_start(va, fmt); 170 (void) vasprintf(&msg, fmt, va); 171 va_end(va); 172 173 free(widemsg); 174 widemsg = msg; 175 } 176 177 /* 178 * This is used for 8-bit encodings. 179 */ 180 int 181 towide_none(wchar_t *c, const char *mb, unsigned n __unused) 182 { 183 if (mb_cur_max != 1) { 184 werr("invalid or unsupported multibyte locale"); 185 return (-1); 186 } 187 *c = (uint8_t)*mb; 188 return (1); 189 } 190 191 int 192 tomb_none(char *mb, wchar_t wc) 193 { 194 if (mb_cur_max != 1) { 195 werr("invalid or unsupported multibyte locale"); 196 return (-1); 197 } 198 *(uint8_t *)mb = (wc & 0xff); 199 mb[1] = 0; 200 return (1); 201 } 202 203 /* 204 * UTF-8 stores wide characters in UTF-32 form. 205 */ 206 int 207 towide_utf8(wchar_t *wc, const char *mb, unsigned n) 208 { 209 wchar_t c; 210 int nb; 211 wchar_t lv; /* lowest legal value */ 212 int i; 213 const uint8_t *s = (const uint8_t *)mb; 214 215 c = *s; 216 217 if ((c & 0x80) == 0) { 218 /* 7-bit ASCII */ 219 *wc = c; 220 return (1); 221 } else if ((c & 0xe0) == 0xc0) { 222 /* u80-u7ff - two bytes encoded */ 223 nb = 2; 224 lv = 0x80; 225 c &= ~0xe0; 226 } else if ((c & 0xf0) == 0xe0) { 227 /* u800-uffff - three bytes encoded */ 228 nb = 3; 229 lv = 0x800; 230 c &= ~0xf0; 231 } else if ((c & 0xf8) == 0xf0) { 232 /* u1000-u1fffff - four bytes encoded */ 233 nb = 4; 234 lv = 0x1000; 235 c &= ~0xf8; 236 } else { 237 /* 5 and 6 byte encodings are not legal unicode */ 238 werr("utf8 encoding too large (%s)", show_mb(mb)); 239 return (-1); 240 } 241 if (nb > (int)n) { 242 werr("incomplete utf8 sequence (%s)", show_mb(mb)); 243 return (-1); 244 } 245 246 for (i = 1; i < nb; i++) { 247 if (((s[i]) & 0xc0) != 0x80) { 248 werr("illegal utf8 byte (%x)", s[i]); 249 return (-1); 250 } 251 c <<= 6; 252 c |= (s[i] & 0x3f); 253 } 254 255 if (c < lv) { 256 werr("illegal redundant utf8 encoding (%s)", show_mb(mb)); 257 return (-1); 258 } 259 *wc = c; 260 return (nb); 261 } 262 263 int 264 tomb_utf8(char *mb, wchar_t wc) 265 { 266 uint8_t *s = (uint8_t *)mb; 267 uint8_t msk; 268 int cnt; 269 int i; 270 271 if (wc <= 0x7f) { 272 s[0] = wc & 0x7f; 273 s[1] = 0; 274 return (1); 275 } 276 if (wc <= 0x7ff) { 277 cnt = 2; 278 msk = 0xc0; 279 } else if (wc <= 0xffff) { 280 cnt = 3; 281 msk = 0xe0; 282 } else if (wc <= 0x1fffff) { 283 cnt = 4; 284 msk = 0xf0; 285 } else { 286 werr("illegal uf8 char (%x)", wc); 287 return (-1); 288 } 289 for (i = cnt - 1; i; i--) { 290 s[i] = (wc & 0x3f) | 0x80; 291 wc >>= 6; 292 } 293 s[0] = (msk) | wc; 294 s[cnt] = 0; 295 return (cnt); 296 } 297 298 /* 299 * Several encodings share a simplistic dual byte encoding. In these 300 * forms, they all indicate that a two byte sequence is to be used if 301 * the first byte has its high bit set. They all store this simple 302 * encoding as a 16-bit value, although a great many of the possible 303 * code points are not used in most character sets. This gives a possible 304 * set of just over 32,000 valid code points. 305 * 306 * 0x00 - 0x7f - 1 byte encoding 307 * 0x80 - 0x7fff - illegal 308 * 0x8000 - 0xffff - 2 byte encoding 309 */ 310 311 static int 312 towide_dbcs(wchar_t *wc, const char *mb, unsigned n) 313 { 314 wchar_t c; 315 316 c = *(const uint8_t *)mb; 317 318 if ((c & 0x80) == 0) { 319 /* 7-bit */ 320 *wc = c; 321 return (1); 322 } 323 if (n < 2) { 324 werr("incomplete character sequence (%s)", show_mb(mb)); 325 return (-1); 326 } 327 328 /* Store both bytes as a single 16-bit wide. */ 329 c <<= 8; 330 c |= (uint8_t)(mb[1]); 331 *wc = c; 332 return (2); 333 } 334 335 /* 336 * Most multibyte locales just convert the wide character to the multibyte 337 * form by stripping leading null bytes, and writing the 32-bit quantity 338 * in big-endian order. 339 */ 340 int 341 tomb_mbs(char *mb, wchar_t wc) 342 { 343 uint8_t *s = (uint8_t *)mb; 344 int n = 0, c; 345 346 if ((wc & 0xff000000U) != 0) { 347 n = 4; 348 } else if ((wc & 0x00ff0000U) != 0) { 349 n = 3; 350 } else if ((wc & 0x0000ff00U) != 0) { 351 n = 2; 352 } else { 353 n = 1; 354 } 355 c = n; 356 while (n) { 357 n--; 358 s[n] = wc & 0xff; 359 wc >>= 8; 360 } 361 /* ensure null termination */ 362 s[c] = 0; 363 return (c); 364 } 365 366 367 /* 368 * big5 is a simple dual byte character set. 369 */ 370 int 371 towide_big5(wchar_t *wc, const char *mb, unsigned n) 372 { 373 return (towide_dbcs(wc, mb, n)); 374 } 375 376 /* 377 * GBK encodes wides in the same way that big5 does, the high order 378 * bit of the first byte indicates a double byte character. 379 */ 380 int 381 towide_gbk(wchar_t *wc, const char *mb, unsigned n) 382 { 383 return (towide_dbcs(wc, mb, n)); 384 } 385 386 /* 387 * GB2312 is another DBCS. Its cleaner than others in that the second 388 * byte does not encode ASCII, but it supports characters. 389 */ 390 int 391 towide_gb2312(wchar_t *wc, const char *mb, unsigned n) 392 { 393 return (towide_dbcs(wc, mb, n)); 394 } 395 396 /* 397 * GB18030. This encodes as 8, 16, or 32-bits. 398 * 7-bit values are in 1 byte, 4 byte sequences are used when 399 * the second byte encodes 0x30-39 and all other sequences are 2 bytes. 400 */ 401 int 402 towide_gb18030(wchar_t *wc, const char *mb, unsigned n) 403 { 404 wchar_t c; 405 406 c = *(const uint8_t *)mb; 407 408 if ((c & 0x80) == 0) { 409 /* 7-bit */ 410 *wc = c; 411 return (1); 412 } 413 if (n < 2) { 414 werr("incomplete character sequence (%s)", show_mb(mb)); 415 return (-1); 416 } 417 418 /* pull in the second byte */ 419 c <<= 8; 420 c |= (uint8_t)(mb[1]); 421 422 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) { 423 if (n < 4) { 424 werr("incomplete 4-byte character sequence (%s)", 425 show_mb(mb)); 426 return (-1); 427 } 428 c <<= 8; 429 c |= (uint8_t)(mb[2]); 430 c <<= 8; 431 c |= (uint8_t)(mb[3]); 432 *wc = c; 433 return (4); 434 } 435 436 *wc = c; 437 return (2); 438 } 439 440 /* 441 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it 442 * also has a range of single byte characters above 0x80. (0xa1-0xdf). 443 */ 444 int 445 towide_mskanji(wchar_t *wc, const char *mb, unsigned n) 446 { 447 wchar_t c; 448 449 c = *(const uint8_t *)mb; 450 451 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) { 452 /* 7-bit */ 453 *wc = c; 454 return (1); 455 } 456 457 if (n < 2) { 458 werr("incomplete character sequence (%s)", show_mb(mb)); 459 return (-1); 460 } 461 462 /* Store both bytes as a single 16-bit wide. */ 463 c <<= 8; 464 c |= (uint8_t)(mb[1]); 465 *wc = c; 466 return (2); 467 } 468 469 /* 470 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional 471 * variable data to encode these, but we're going to treat each as independent 472 * instead. Its the only way we can sensibly move forward. 473 * 474 * Note that the way in which the different EUC forms vary is how wide 475 * CS2 and CS3 are and what the first byte of them is. 476 */ 477 static int 478 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n, 479 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width) 480 { 481 int i; 482 int width = 2; 483 wchar_t c; 484 485 c = *(const uint8_t *)mb; 486 487 /* 488 * All variations of EUC encode 7-bit ASCII as one byte, and use 489 * additional bytes for more than that. 490 */ 491 if ((c & 0x80) == 0) { 492 /* 7-bit */ 493 *wc = c; 494 return (1); 495 } 496 497 /* 498 * All EUC variants reserve 0xa1-0xff to identify CS1, which 499 * is always two bytes wide. Note that unused CS will be zero, 500 * and that cannot be true because we know that the high order 501 * bit must be set. 502 */ 503 if (c >= 0xa1) { 504 width = 2; 505 } else if (c == cs2) { 506 width = cs2width; 507 } else if (c == cs3) { 508 width = cs3width; 509 } 510 511 if ((int)n < width) { 512 werr("incomplete character sequence (%s)", show_mb(mb)); 513 return (-1); 514 } 515 516 for (i = 1; i < width; i++) { 517 /* pull in the next byte */ 518 c <<= 8; 519 c |= (uint8_t)(mb[i]); 520 } 521 522 *wc = c; 523 return (width); 524 } 525 526 /* 527 * EUC-CN encodes as follows: 528 * 529 * Code set 0 (ASCII): 0x21-0x7E 530 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE 531 * Code set 2: unused 532 * Code set 3: unused 533 */ 534 int 535 towide_euccn(wchar_t *wc, const char *mb, unsigned n) 536 { 537 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); 538 } 539 540 /* 541 * EUC-JP encodes as follows: 542 * 543 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E 544 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE 545 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF 546 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE 547 */ 548 int 549 towide_eucjp(wchar_t *wc, const char *mb, unsigned n) 550 { 551 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3)); 552 } 553 554 /* 555 * EUC-KR encodes as follows: 556 * 557 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E 558 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE 559 * Code set 2: unused 560 * Code set 3: unused 561 */ 562 int 563 towide_euckr(wchar_t *wc, const char *mb, unsigned n) 564 { 565 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0)); 566 } 567 568 /* 569 * EUC-TW encodes as follows: 570 * 571 * Code set 0 (ASCII): 0x21-0x7E 572 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE 573 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE 574 * Code set 3: unused 575 */ 576 int 577 towide_euctw(wchar_t *wc, const char *mb, unsigned n) 578 { 579 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); 580 } 581 582 /* 583 * Public entry points. 584 */ 585 586 int 587 to_wide(wchar_t *wc, const char *mb) 588 { 589 /* this won't fail hard */ 590 return (_towide(wc, mb, strlen(mb))); 591 } 592 593 int 594 to_mb(char *mb, wchar_t wc) 595 { 596 int rv; 597 598 if ((rv = _tomb(mb, wc)) < 0) { 599 warn("%s", widemsg); 600 free(widemsg); 601 widemsg = NULL; 602 } 603 return (rv); 604 } 605 606 char * 607 to_mb_string(const wchar_t *wcs) 608 { 609 char *mbs; 610 char *ptr; 611 int len; 612 613 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1); 614 if (mbs == NULL) { 615 warn("out of memory"); 616 return (NULL); 617 } 618 ptr = mbs; 619 while (*wcs) { 620 if ((len = to_mb(ptr, *wcs)) < 0) { 621 INTERR; 622 free(mbs); 623 return (NULL); 624 } 625 wcs++; 626 ptr += len; 627 } 628 *ptr = 0; 629 return (mbs); 630 } 631 632 void 633 set_wide_encoding(const char *encoding) 634 { 635 int i; 636 637 _towide = towide_none; 638 _tomb = tomb_none; 639 _nbits = 8; 640 641 snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s", 642 encoding); 643 for (i = 0; mb_encodings[i].name; i++) { 644 if (strcasecmp(encoding, mb_encodings[i].name) == 0) { 645 _towide = mb_encodings[i].towide; 646 _tomb = mb_encodings[i].tomb; 647 _encoding = mb_encodings[i].cname; 648 _nbits = mb_encodings[i].nbits; 649 break; 650 } 651 } 652 } 653 654 const char * 655 get_wide_encoding(void) 656 { 657 return (_encoding); 658 } 659 660 int 661 max_wide(void) 662 { 663 return ((int)((1U << _nbits) - 1)); 664 } 665