1 /*-
2 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
5 *
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * The functions in this file convert from the standard multibyte forms
34 * to the wide character forms used internally by libc. Unfortunately,
35 * this approach means that we need a method for each and every encoding.
36 */
37 #include <sys/cdefs.h>
38 #include <ctype.h>
39 #include <stdlib.h>
40 #include <wchar.h>
41 #include <string.h>
42 #include <sys/types.h>
43 #include "localedef.h"
44
45 static int towide_none(wchar_t *, const char *, unsigned);
46 static int towide_utf8(wchar_t *, const char *, unsigned);
47 static int towide_big5(wchar_t *, const char *, unsigned);
48 static int towide_gbk(wchar_t *, const char *, unsigned);
49 static int towide_gb2312(wchar_t *, const char *, unsigned);
50 static int towide_gb18030(wchar_t *, const char *, unsigned);
51 static int towide_mskanji(wchar_t *, const char *, unsigned);
52 static int towide_euccn(wchar_t *, const char *, unsigned);
53 static int towide_eucjp(wchar_t *, const char *, unsigned);
54 static int towide_euckr(wchar_t *, const char *, unsigned);
55 static int towide_euctw(wchar_t *, const char *, unsigned);
56
57 static int tomb_none(char *, wchar_t);
58 static int tomb_utf8(char *, wchar_t);
59 static int tomb_mbs(char *, wchar_t);
60
61 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
62 static int (*_tomb)(char *, wchar_t) = tomb_none;
63 static char _encoding_buffer[20] = {'N','O','N','E'};
64 static const char *_encoding = _encoding_buffer;
65 static int _nbits = 7;
66
67 /*
68 * Table of supported encodings. We only bother to list the multibyte
69 * encodings here, because single byte locales are handed by "NONE".
70 */
71 static struct {
72 const char *name;
73 /* the name that the underlying libc implemenation uses */
74 const char *cname;
75 /* the maximum number of bits required for priorities */
76 int nbits;
77 int (*towide)(wchar_t *, const char *, unsigned);
78 int (*tomb)(char *, wchar_t);
79 } mb_encodings[] = {
80 /*
81 * UTF8 values max out at 0x1fffff (although in theory there could
82 * be later extensions, but it won't happen.) This means we only need
83 * 21 bits to be able to encode the entire range of priorities.
84 */
85 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
86 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 },
87 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 },
88 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
89
90 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs },
91 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs },
92 /*
93 * Because the 3-byte form of EUC-JP use the same leading byte,
94 * only 17 bits required to provide unique priorities. (The low
95 * bit of that first byte is set.) By setting this value low,
96 * we can get by with only 3 bytes in the strxfrm expansion.
97 */
98 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
99 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
100
101 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs },
102 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs },
103 /*
104 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
105 * high order byte is 0x8E. However, with 4 byte encodings,
106 * the third byte will be A0-B0. So we only need to consider
107 * the lower order 24 bits for collation.
108 */
109 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs },
110 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs },
111
112 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
113 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
114 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs },
115 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
116 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
117
118 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs },
119 { "big5", "BIG5", 16, towide_big5, tomb_mbs },
120 { "Big5", "BIG5", 16, towide_big5, tomb_mbs },
121
122 { "GBK", "GBK", 16, towide_gbk, tomb_mbs },
123
124 /*
125 * GB18030 can get away with just 31 bits. This is because the
126 * high order bit is always set for 4 byte values, and the
127 * at least one of the other bits in that 4 byte value will
128 * be non-zero.
129 */
130 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs },
131
132 /*
133 * This should probably be an aliase for euc-cn, or vice versa.
134 */
135 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs },
136
137 { NULL, NULL, 0, 0, 0 },
138 };
139
140 static char *
show_mb(const char * mb)141 show_mb(const char *mb)
142 {
143 static char buf[64];
144
145 /* ASCII stuff we just print */
146 if (isascii(*mb) && isgraph(*mb)) {
147 buf[0] = *mb;
148 buf[1] = 0;
149 return (buf);
150 }
151 buf[0] = 0;
152 while (*mb != 0) {
153 char scr[8];
154 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
155 (void) strlcat(buf, scr, sizeof (buf));
156 mb++;
157 }
158 return (buf);
159 }
160
161 static char *widemsg;
162
163 void
werr(const char * fmt,...)164 werr(const char *fmt, ...)
165 {
166 char *msg;
167
168 va_list va;
169 va_start(va, fmt);
170 (void) vasprintf(&msg, fmt, va);
171 va_end(va);
172
173 free(widemsg);
174 widemsg = msg;
175 }
176
177 /*
178 * This is used for 8-bit encodings.
179 */
180 int
towide_none(wchar_t * c,const char * mb,unsigned n __unused)181 towide_none(wchar_t *c, const char *mb, unsigned n __unused)
182 {
183 if (mb_cur_max != 1) {
184 werr("invalid or unsupported multibyte locale");
185 return (-1);
186 }
187 *c = (uint8_t)*mb;
188 return (1);
189 }
190
191 int
tomb_none(char * mb,wchar_t wc)192 tomb_none(char *mb, wchar_t wc)
193 {
194 if (mb_cur_max != 1) {
195 werr("invalid or unsupported multibyte locale");
196 return (-1);
197 }
198 *(uint8_t *)mb = (wc & 0xff);
199 mb[1] = 0;
200 return (1);
201 }
202
203 /*
204 * UTF-8 stores wide characters in UTF-32 form.
205 */
206 int
towide_utf8(wchar_t * wc,const char * mb,unsigned n)207 towide_utf8(wchar_t *wc, const char *mb, unsigned n)
208 {
209 wchar_t c;
210 int nb;
211 wchar_t lv; /* lowest legal value */
212 int i;
213 const uint8_t *s = (const uint8_t *)mb;
214
215 c = *s;
216
217 if ((c & 0x80) == 0) {
218 /* 7-bit ASCII */
219 *wc = c;
220 return (1);
221 } else if ((c & 0xe0) == 0xc0) {
222 /* u80-u7ff - two bytes encoded */
223 nb = 2;
224 lv = 0x80;
225 c &= ~0xe0;
226 } else if ((c & 0xf0) == 0xe0) {
227 /* u800-uffff - three bytes encoded */
228 nb = 3;
229 lv = 0x800;
230 c &= ~0xf0;
231 } else if ((c & 0xf8) == 0xf0) {
232 /* u1000-u1fffff - four bytes encoded */
233 nb = 4;
234 lv = 0x1000;
235 c &= ~0xf8;
236 } else {
237 /* 5 and 6 byte encodings are not legal unicode */
238 werr("utf8 encoding too large (%s)", show_mb(mb));
239 return (-1);
240 }
241 if (nb > (int)n) {
242 werr("incomplete utf8 sequence (%s)", show_mb(mb));
243 return (-1);
244 }
245
246 for (i = 1; i < nb; i++) {
247 if (((s[i]) & 0xc0) != 0x80) {
248 werr("illegal utf8 byte (%x)", s[i]);
249 return (-1);
250 }
251 c <<= 6;
252 c |= (s[i] & 0x3f);
253 }
254
255 if (c < lv) {
256 werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
257 return (-1);
258 }
259 *wc = c;
260 return (nb);
261 }
262
263 int
tomb_utf8(char * mb,wchar_t wc)264 tomb_utf8(char *mb, wchar_t wc)
265 {
266 uint8_t *s = (uint8_t *)mb;
267 uint8_t msk;
268 int cnt;
269 int i;
270
271 if (wc <= 0x7f) {
272 s[0] = wc & 0x7f;
273 s[1] = 0;
274 return (1);
275 }
276 if (wc <= 0x7ff) {
277 cnt = 2;
278 msk = 0xc0;
279 } else if (wc <= 0xffff) {
280 cnt = 3;
281 msk = 0xe0;
282 } else if (wc <= 0x1fffff) {
283 cnt = 4;
284 msk = 0xf0;
285 } else {
286 werr("illegal uf8 char (%x)", wc);
287 return (-1);
288 }
289 for (i = cnt - 1; i; i--) {
290 s[i] = (wc & 0x3f) | 0x80;
291 wc >>= 6;
292 }
293 s[0] = (msk) | wc;
294 s[cnt] = 0;
295 return (cnt);
296 }
297
298 /*
299 * Several encodings share a simplistic dual byte encoding. In these
300 * forms, they all indicate that a two byte sequence is to be used if
301 * the first byte has its high bit set. They all store this simple
302 * encoding as a 16-bit value, although a great many of the possible
303 * code points are not used in most character sets. This gives a possible
304 * set of just over 32,000 valid code points.
305 *
306 * 0x00 - 0x7f - 1 byte encoding
307 * 0x80 - 0x7fff - illegal
308 * 0x8000 - 0xffff - 2 byte encoding
309 */
310
311 static int
towide_dbcs(wchar_t * wc,const char * mb,unsigned n)312 towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
313 {
314 wchar_t c;
315
316 c = *(const uint8_t *)mb;
317
318 if ((c & 0x80) == 0) {
319 /* 7-bit */
320 *wc = c;
321 return (1);
322 }
323 if (n < 2) {
324 werr("incomplete character sequence (%s)", show_mb(mb));
325 return (-1);
326 }
327
328 /* Store both bytes as a single 16-bit wide. */
329 c <<= 8;
330 c |= (uint8_t)(mb[1]);
331 *wc = c;
332 return (2);
333 }
334
335 /*
336 * Most multibyte locales just convert the wide character to the multibyte
337 * form by stripping leading null bytes, and writing the 32-bit quantity
338 * in big-endian order.
339 */
340 int
tomb_mbs(char * mb,wchar_t wc)341 tomb_mbs(char *mb, wchar_t wc)
342 {
343 uint8_t *s = (uint8_t *)mb;
344 int n = 0, c;
345
346 if ((wc & 0xff000000U) != 0) {
347 n = 4;
348 } else if ((wc & 0x00ff0000U) != 0) {
349 n = 3;
350 } else if ((wc & 0x0000ff00U) != 0) {
351 n = 2;
352 } else {
353 n = 1;
354 }
355 c = n;
356 while (n) {
357 n--;
358 s[n] = wc & 0xff;
359 wc >>= 8;
360 }
361 /* ensure null termination */
362 s[c] = 0;
363 return (c);
364 }
365
366
367 /*
368 * big5 is a simple dual byte character set.
369 */
370 int
towide_big5(wchar_t * wc,const char * mb,unsigned n)371 towide_big5(wchar_t *wc, const char *mb, unsigned n)
372 {
373 return (towide_dbcs(wc, mb, n));
374 }
375
376 /*
377 * GBK encodes wides in the same way that big5 does, the high order
378 * bit of the first byte indicates a double byte character.
379 */
380 int
towide_gbk(wchar_t * wc,const char * mb,unsigned n)381 towide_gbk(wchar_t *wc, const char *mb, unsigned n)
382 {
383 return (towide_dbcs(wc, mb, n));
384 }
385
386 /*
387 * GB2312 is another DBCS. Its cleaner than others in that the second
388 * byte does not encode ASCII, but it supports characters.
389 */
390 int
towide_gb2312(wchar_t * wc,const char * mb,unsigned n)391 towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
392 {
393 return (towide_dbcs(wc, mb, n));
394 }
395
396 /*
397 * GB18030. This encodes as 8, 16, or 32-bits.
398 * 7-bit values are in 1 byte, 4 byte sequences are used when
399 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
400 */
401 int
towide_gb18030(wchar_t * wc,const char * mb,unsigned n)402 towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
403 {
404 wchar_t c;
405
406 c = *(const uint8_t *)mb;
407
408 if ((c & 0x80) == 0) {
409 /* 7-bit */
410 *wc = c;
411 return (1);
412 }
413 if (n < 2) {
414 werr("incomplete character sequence (%s)", show_mb(mb));
415 return (-1);
416 }
417
418 /* pull in the second byte */
419 c <<= 8;
420 c |= (uint8_t)(mb[1]);
421
422 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
423 if (n < 4) {
424 werr("incomplete 4-byte character sequence (%s)",
425 show_mb(mb));
426 return (-1);
427 }
428 c <<= 8;
429 c |= (uint8_t)(mb[2]);
430 c <<= 8;
431 c |= (uint8_t)(mb[3]);
432 *wc = c;
433 return (4);
434 }
435
436 *wc = c;
437 return (2);
438 }
439
440 /*
441 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
442 * also has a range of single byte characters above 0x80. (0xa1-0xdf).
443 */
444 int
towide_mskanji(wchar_t * wc,const char * mb,unsigned n)445 towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
446 {
447 wchar_t c;
448
449 c = *(const uint8_t *)mb;
450
451 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
452 /* 7-bit */
453 *wc = c;
454 return (1);
455 }
456
457 if (n < 2) {
458 werr("incomplete character sequence (%s)", show_mb(mb));
459 return (-1);
460 }
461
462 /* Store both bytes as a single 16-bit wide. */
463 c <<= 8;
464 c |= (uint8_t)(mb[1]);
465 *wc = c;
466 return (2);
467 }
468
469 /*
470 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional
471 * variable data to encode these, but we're going to treat each as independent
472 * instead. Its the only way we can sensibly move forward.
473 *
474 * Note that the way in which the different EUC forms vary is how wide
475 * CS2 and CS3 are and what the first byte of them is.
476 */
477 static int
towide_euc_impl(wchar_t * wc,const char * mb,unsigned n,uint8_t cs2,uint8_t cs2width,uint8_t cs3,uint8_t cs3width)478 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
479 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
480 {
481 int i;
482 int width = 2;
483 wchar_t c;
484
485 c = *(const uint8_t *)mb;
486
487 /*
488 * All variations of EUC encode 7-bit ASCII as one byte, and use
489 * additional bytes for more than that.
490 */
491 if ((c & 0x80) == 0) {
492 /* 7-bit */
493 *wc = c;
494 return (1);
495 }
496
497 /*
498 * All EUC variants reserve 0xa1-0xff to identify CS1, which
499 * is always two bytes wide. Note that unused CS will be zero,
500 * and that cannot be true because we know that the high order
501 * bit must be set.
502 */
503 if (c >= 0xa1) {
504 width = 2;
505 } else if (c == cs2) {
506 width = cs2width;
507 } else if (c == cs3) {
508 width = cs3width;
509 }
510
511 if ((int)n < width) {
512 werr("incomplete character sequence (%s)", show_mb(mb));
513 return (-1);
514 }
515
516 for (i = 1; i < width; i++) {
517 /* pull in the next byte */
518 c <<= 8;
519 c |= (uint8_t)(mb[i]);
520 }
521
522 *wc = c;
523 return (width);
524 }
525
526 /*
527 * EUC-CN encodes as follows:
528 *
529 * Code set 0 (ASCII): 0x21-0x7E
530 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
531 * Code set 2: unused
532 * Code set 3: unused
533 */
534 int
towide_euccn(wchar_t * wc,const char * mb,unsigned n)535 towide_euccn(wchar_t *wc, const char *mb, unsigned n)
536 {
537 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
538 }
539
540 /*
541 * EUC-JP encodes as follows:
542 *
543 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E
544 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE
545 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF
546 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE
547 */
548 int
towide_eucjp(wchar_t * wc,const char * mb,unsigned n)549 towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
550 {
551 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
552 }
553
554 /*
555 * EUC-KR encodes as follows:
556 *
557 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E
558 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE
559 * Code set 2: unused
560 * Code set 3: unused
561 */
562 int
towide_euckr(wchar_t * wc,const char * mb,unsigned n)563 towide_euckr(wchar_t *wc, const char *mb, unsigned n)
564 {
565 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
566 }
567
568 /*
569 * EUC-TW encodes as follows:
570 *
571 * Code set 0 (ASCII): 0x21-0x7E
572 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
573 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE
574 * Code set 3: unused
575 */
576 int
towide_euctw(wchar_t * wc,const char * mb,unsigned n)577 towide_euctw(wchar_t *wc, const char *mb, unsigned n)
578 {
579 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
580 }
581
582 /*
583 * Public entry points.
584 */
585
586 int
to_wide(wchar_t * wc,const char * mb)587 to_wide(wchar_t *wc, const char *mb)
588 {
589 /* this won't fail hard */
590 return (_towide(wc, mb, strlen(mb)));
591 }
592
593 int
to_mb(char * mb,wchar_t wc)594 to_mb(char *mb, wchar_t wc)
595 {
596 int rv;
597
598 if ((rv = _tomb(mb, wc)) < 0) {
599 warn("%s", widemsg);
600 free(widemsg);
601 widemsg = NULL;
602 }
603 return (rv);
604 }
605
606 char *
to_mb_string(const wchar_t * wcs)607 to_mb_string(const wchar_t *wcs)
608 {
609 char *mbs;
610 char *ptr;
611 int len;
612
613 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
614 if (mbs == NULL) {
615 warn("out of memory");
616 return (NULL);
617 }
618 ptr = mbs;
619 while (*wcs) {
620 if ((len = to_mb(ptr, *wcs)) < 0) {
621 INTERR;
622 free(mbs);
623 return (NULL);
624 }
625 wcs++;
626 ptr += len;
627 }
628 *ptr = 0;
629 return (mbs);
630 }
631
632 void
set_wide_encoding(const char * encoding)633 set_wide_encoding(const char *encoding)
634 {
635 int i;
636
637 _towide = towide_none;
638 _tomb = tomb_none;
639 _nbits = 8;
640
641 snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s",
642 encoding);
643 for (i = 0; mb_encodings[i].name; i++) {
644 if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
645 _towide = mb_encodings[i].towide;
646 _tomb = mb_encodings[i].tomb;
647 _encoding = mb_encodings[i].cname;
648 _nbits = mb_encodings[i].nbits;
649 break;
650 }
651 }
652 }
653
654 const char *
get_wide_encoding(void)655 get_wide_encoding(void)
656 {
657 return (_encoding);
658 }
659
660 int
max_wide(void)661 max_wide(void)
662 {
663 return ((int)((1U << _nbits) - 1));
664 }
665