xref: /freebsd/usr.bin/localedef/wide.c (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1 /*-
2  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
3  * Copyright 2012 Garrett D'Amore <garrett@damore.org>  All rights reserved.
4  * Copyright 2015 John Marino <draco@marino.st>
5  *
6  * This source code is derived from the illumos localedef command, and
7  * provided under BSD-style license terms by Nexenta Systems, Inc.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * The functions in this file convert from the standard multibyte forms
34  * to the wide character forms used internally by libc.  Unfortunately,
35  * this approach means that we need a method for each and every encoding.
36  */
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include <ctype.h>
41 #include <stdlib.h>
42 #include <wchar.h>
43 #include <string.h>
44 #include <sys/types.h>
45 #include "localedef.h"
46 
47 static int towide_none(wchar_t *, const char *, unsigned);
48 static int towide_utf8(wchar_t *, const char *, unsigned);
49 static int towide_big5(wchar_t *, const char *, unsigned);
50 static int towide_gbk(wchar_t *, const char *, unsigned);
51 static int towide_gb2312(wchar_t *, const char *, unsigned);
52 static int towide_gb18030(wchar_t *, const char *, unsigned);
53 static int towide_mskanji(wchar_t *, const char *, unsigned);
54 static int towide_euccn(wchar_t *, const char *, unsigned);
55 static int towide_eucjp(wchar_t *, const char *, unsigned);
56 static int towide_euckr(wchar_t *, const char *, unsigned);
57 static int towide_euctw(wchar_t *, const char *, unsigned);
58 
59 static int tomb_none(char *, wchar_t);
60 static int tomb_utf8(char *, wchar_t);
61 static int tomb_mbs(char *, wchar_t);
62 
63 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
64 static int (*_tomb)(char *, wchar_t) = tomb_none;
65 static char _encoding_buffer[20] = {'N','O','N','E'};
66 static const char *_encoding = _encoding_buffer;
67 static int _nbits = 7;
68 
69 /*
70  * Table of supported encodings.  We only bother to list the multibyte
71  * encodings here, because single byte locales are handed by "NONE".
72  */
73 static struct {
74 	const char *name;
75 	/* the name that the underlying libc implemenation uses */
76 	const char *cname;
77 	/* the maximum number of bits required for priorities */
78 	int nbits;
79 	int (*towide)(wchar_t *, const char *, unsigned);
80 	int (*tomb)(char *, wchar_t);
81 } mb_encodings[] = {
82 	/*
83 	 * UTF8 values max out at 0x1fffff (although in theory there could
84 	 * be later extensions, but it won't happen.)  This means we only need
85 	 * 21 bits to be able to encode the entire range of priorities.
86 	 */
87 	{ "UTF-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
88 	{ "UTF8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
89 	{ "utf8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
90 	{ "utf-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
91 
92 	{ "EUC-CN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
93 	{ "eucCN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
94 	/*
95 	 * Because the 3-byte form of EUC-JP use the same leading byte,
96 	 * only 17 bits required to provide unique priorities.  (The low
97 	 * bit of that first byte is set.)  By setting this value low,
98 	 * we can get by with only 3 bytes in the strxfrm expansion.
99 	 */
100 	{ "EUC-JP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
101 	{ "eucJP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
102 
103 	{ "EUC-KR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
104 	{ "eucKR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
105 	/*
106 	 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
107 	 * high order byte is 0x8E.  However, with 4 byte encodings,
108 	 * the third byte will be A0-B0.  So we only need to consider
109 	 * the lower order 24 bits for collation.
110 	 */
111 	{ "EUC-TW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
112 	{ "eucTW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
113 
114 	{ "MS_Kanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
115 	{ "MSKanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
116 	{ "PCK",	"MSKanji",	16, towide_mskanji, tomb_mbs },
117 	{ "SJIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
118 	{ "Shift_JIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
119 
120 	{ "BIG5",	"BIG5",		16, towide_big5, tomb_mbs },
121 	{ "big5",	"BIG5",		16, towide_big5, tomb_mbs },
122 	{ "Big5",	"BIG5",		16, towide_big5, tomb_mbs },
123 
124 	{ "GBK",	"GBK",		16, towide_gbk,	tomb_mbs },
125 
126 	/*
127 	 * GB18030 can get away with just 31 bits.  This is because the
128 	 * high order bit is always set for 4 byte values, and the
129 	 * at least one of the other bits in that 4 byte value will
130 	 * be non-zero.
131 	 */
132 	{ "GB18030",	"GB18030",	31, towide_gb18030, tomb_mbs },
133 
134 	/*
135 	 * This should probably be an aliase for euc-cn, or vice versa.
136 	 */
137 	{ "GB2312",	"GB2312",	16, towide_gb2312, tomb_mbs },
138 
139 	{ NULL, NULL, 0, 0, 0 },
140 };
141 
142 static char *
143 show_mb(const char *mb)
144 {
145 	static char buf[64];
146 
147 	/* ASCII stuff we just print */
148 	if (isascii(*mb) && isgraph(*mb)) {
149 		buf[0] = *mb;
150 		buf[1] = 0;
151 		return (buf);
152 	}
153 	buf[0] = 0;
154 	while (*mb != 0) {
155 		char scr[8];
156 		(void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
157 		(void) strlcat(buf, scr, sizeof (buf));
158 		mb++;
159 	}
160 	return (buf);
161 }
162 
163 static char	*widemsg;
164 
165 void
166 werr(const char *fmt, ...)
167 {
168 	char	*msg;
169 
170 	va_list	va;
171 	va_start(va, fmt);
172 	(void) vasprintf(&msg, fmt, va);
173 	va_end(va);
174 
175 	free(widemsg);
176 	widemsg = msg;
177 }
178 
179 /*
180  * This is used for 8-bit encodings.
181  */
182 int
183 towide_none(wchar_t *c, const char *mb, unsigned n __unused)
184 {
185 	if (mb_cur_max != 1) {
186 		werr("invalid or unsupported multibyte locale");
187 		return (-1);
188 	}
189 	*c = (uint8_t)*mb;
190 	return (1);
191 }
192 
193 int
194 tomb_none(char *mb, wchar_t wc)
195 {
196 	if (mb_cur_max != 1) {
197 		werr("invalid or unsupported multibyte locale");
198 		return (-1);
199 	}
200 	*(uint8_t *)mb = (wc & 0xff);
201 	mb[1] = 0;
202 	return (1);
203 }
204 
205 /*
206  * UTF-8 stores wide characters in UTF-32 form.
207  */
208 int
209 towide_utf8(wchar_t *wc, const char *mb, unsigned n)
210 {
211 	wchar_t	c;
212 	int	nb;
213 	wchar_t	lv;	/* lowest legal value */
214 	int	i;
215 	const uint8_t *s = (const uint8_t *)mb;
216 
217 	c = *s;
218 
219 	if ((c & 0x80) == 0) {
220 		/* 7-bit ASCII */
221 		*wc = c;
222 		return (1);
223 	} else if ((c & 0xe0) == 0xc0) {
224 		/* u80-u7ff - two bytes encoded */
225 		nb = 2;
226 		lv = 0x80;
227 		c &= ~0xe0;
228 	} else if ((c & 0xf0) == 0xe0) {
229 		/* u800-uffff - three bytes encoded */
230 		nb = 3;
231 		lv = 0x800;
232 		c &= ~0xf0;
233 	} else if ((c & 0xf8) == 0xf0) {
234 		/* u1000-u1fffff - four bytes encoded */
235 		nb = 4;
236 		lv = 0x1000;
237 		c &= ~0xf8;
238 	} else {
239 		/* 5 and 6 byte encodings are not legal unicode */
240 		werr("utf8 encoding too large (%s)", show_mb(mb));
241 		return (-1);
242 	}
243 	if (nb > (int)n) {
244 		werr("incomplete utf8 sequence (%s)", show_mb(mb));
245 		return (-1);
246 	}
247 
248 	for (i = 1; i < nb; i++) {
249 		if (((s[i]) & 0xc0) != 0x80) {
250 			werr("illegal utf8 byte (%x)", s[i]);
251 			return (-1);
252 		}
253 		c <<= 6;
254 		c |= (s[i] & 0x3f);
255 	}
256 
257 	if (c < lv) {
258 		werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
259 		return (-1);
260 	}
261 	*wc = c;
262 	return (nb);
263 }
264 
265 int
266 tomb_utf8(char *mb, wchar_t wc)
267 {
268 	uint8_t *s = (uint8_t *)mb;
269 	uint8_t msk;
270 	int cnt;
271 	int i;
272 
273 	if (wc <= 0x7f) {
274 		s[0] = wc & 0x7f;
275 		s[1] = 0;
276 		return (1);
277 	}
278 	if (wc <= 0x7ff) {
279 		cnt = 2;
280 		msk = 0xc0;
281 	} else if (wc <= 0xffff) {
282 		cnt = 3;
283 		msk = 0xe0;
284 	} else if (wc <= 0x1fffff) {
285 		cnt = 4;
286 		msk = 0xf0;
287 	} else {
288 		werr("illegal uf8 char (%x)", wc);
289 		return (-1);
290 	}
291 	for (i = cnt - 1; i; i--) {
292 		s[i] = (wc & 0x3f) | 0x80;
293 		wc >>= 6;
294 	}
295 	s[0] = (msk) | wc;
296 	s[cnt] = 0;
297 	return (cnt);
298 }
299 
300 /*
301  * Several encodings share a simplistic dual byte encoding.  In these
302  * forms, they all indicate that a two byte sequence is to be used if
303  * the first byte has its high bit set.  They all store this simple
304  * encoding as a 16-bit value, although a great many of the possible
305  * code points are not used in most character sets.  This gives a possible
306  * set of just over 32,000 valid code points.
307  *
308  * 0x00 - 0x7f		- 1 byte encoding
309  * 0x80 - 0x7fff	- illegal
310  * 0x8000 - 0xffff	- 2 byte encoding
311  */
312 
313 static int
314 towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
315 {
316 	wchar_t	c;
317 
318 	c = *(const uint8_t *)mb;
319 
320 	if ((c & 0x80) == 0) {
321 		/* 7-bit */
322 		*wc = c;
323 		return (1);
324 	}
325 	if (n < 2) {
326 		werr("incomplete character sequence (%s)", show_mb(mb));
327 		return (-1);
328 	}
329 
330 	/* Store both bytes as a single 16-bit wide. */
331 	c <<= 8;
332 	c |= (uint8_t)(mb[1]);
333 	*wc = c;
334 	return (2);
335 }
336 
337 /*
338  * Most multibyte locales just convert the wide character to the multibyte
339  * form by stripping leading null bytes, and writing the 32-bit quantity
340  * in big-endian order.
341  */
342 int
343 tomb_mbs(char *mb, wchar_t wc)
344 {
345 	uint8_t *s = (uint8_t *)mb;
346 	int 	n = 0, c;
347 
348 	if ((wc & 0xff000000U) != 0) {
349 		n = 4;
350 	} else if ((wc & 0x00ff0000U) != 0) {
351 		n = 3;
352 	} else if ((wc & 0x0000ff00U) != 0) {
353 		n = 2;
354 	} else {
355 		n = 1;
356 	}
357 	c = n;
358 	while (n) {
359 		n--;
360 		s[n] = wc & 0xff;
361 		wc >>= 8;
362 	}
363 	/* ensure null termination */
364 	s[c] = 0;
365 	return (c);
366 }
367 
368 
369 /*
370  * big5 is a simple dual byte character set.
371  */
372 int
373 towide_big5(wchar_t *wc, const char *mb, unsigned n)
374 {
375 	return (towide_dbcs(wc, mb, n));
376 }
377 
378 /*
379  * GBK encodes wides in the same way that big5 does, the high order
380  * bit of the first byte indicates a double byte character.
381  */
382 int
383 towide_gbk(wchar_t *wc, const char *mb, unsigned n)
384 {
385 	return (towide_dbcs(wc, mb, n));
386 }
387 
388 /*
389  * GB2312 is another DBCS.  Its cleaner than others in that the second
390  * byte does not encode ASCII, but it supports characters.
391  */
392 int
393 towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
394 {
395 	return (towide_dbcs(wc, mb, n));
396 }
397 
398 /*
399  * GB18030.  This encodes as 8, 16, or 32-bits.
400  * 7-bit values are in 1 byte,  4 byte sequences are used when
401  * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
402  */
403 int
404 towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
405 {
406 	wchar_t	c;
407 
408 	c = *(const uint8_t *)mb;
409 
410 	if ((c & 0x80) == 0) {
411 		/* 7-bit */
412 		*wc = c;
413 		return (1);
414 	}
415 	if (n < 2) {
416 		werr("incomplete character sequence (%s)", show_mb(mb));
417 		return (-1);
418 	}
419 
420 	/* pull in the second byte */
421 	c <<= 8;
422 	c |= (uint8_t)(mb[1]);
423 
424 	if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
425 		if (n < 4) {
426 			werr("incomplete 4-byte character sequence (%s)",
427 			    show_mb(mb));
428 			return (-1);
429 		}
430 		c <<= 8;
431 		c |= (uint8_t)(mb[2]);
432 		c <<= 8;
433 		c |= (uint8_t)(mb[3]);
434 		*wc = c;
435 		return (4);
436 	}
437 
438 	*wc = c;
439 	return (2);
440 }
441 
442 /*
443  * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
444  * also has a range of single byte characters above 0x80.  (0xa1-0xdf).
445  */
446 int
447 towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
448 {
449 	wchar_t	c;
450 
451 	c = *(const uint8_t *)mb;
452 
453 	if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
454 		/* 7-bit */
455 		*wc = c;
456 		return (1);
457 	}
458 
459 	if (n < 2) {
460 		werr("incomplete character sequence (%s)", show_mb(mb));
461 		return (-1);
462 	}
463 
464 	/* Store both bytes as a single 16-bit wide. */
465 	c <<= 8;
466 	c |= (uint8_t)(mb[1]);
467 	*wc = c;
468 	return (2);
469 }
470 
471 /*
472  * EUC forms.  EUC encodings are "variable".  FreeBSD carries some additional
473  * variable data to encode these, but we're going to treat each as independent
474  * instead.  Its the only way we can sensibly move forward.
475  *
476  * Note that the way in which the different EUC forms vary is how wide
477  * CS2 and CS3 are and what the first byte of them is.
478  */
479 static int
480 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
481     uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
482 {
483 	int i;
484 	int width = 2;
485 	wchar_t	c;
486 
487 	c = *(const uint8_t *)mb;
488 
489 	/*
490 	 * All variations of EUC encode 7-bit ASCII as one byte, and use
491 	 * additional bytes for more than that.
492 	 */
493 	if ((c & 0x80) == 0) {
494 		/* 7-bit */
495 		*wc = c;
496 		return (1);
497 	}
498 
499 	/*
500 	 * All EUC variants reserve 0xa1-0xff to identify CS1, which
501 	 * is always two bytes wide.  Note that unused CS will be zero,
502 	 * and that cannot be true because we know that the high order
503 	 * bit must be set.
504 	 */
505 	if (c >= 0xa1) {
506 		width = 2;
507 	} else if (c == cs2) {
508 		width = cs2width;
509 	} else if (c == cs3) {
510 		width = cs3width;
511 	}
512 
513 	if ((int)n < width) {
514 		werr("incomplete character sequence (%s)", show_mb(mb));
515 		return (-1);
516 	}
517 
518 	for (i = 1; i < width; i++) {
519 		/* pull in the next byte */
520 		c <<= 8;
521 		c |= (uint8_t)(mb[i]);
522 	}
523 
524 	*wc = c;
525 	return (width);
526 }
527 
528 /*
529  * EUC-CN encodes as follows:
530  *
531  * Code set 0 (ASCII):				0x21-0x7E
532  * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
533  * Code set 2:					unused
534  * Code set 3:					unused
535  */
536 int
537 towide_euccn(wchar_t *wc, const char *mb, unsigned n)
538 {
539 	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
540 }
541 
542 /*
543  * EUC-JP encodes as follows:
544  *
545  * Code set 0 (ASCII or JIS X 0201-1976 Roman):	0x21-0x7E
546  * Code set 1 (JIS X 0208):			0xA1A1-0xFEFE
547  * Code set 2 (half-width katakana):		0x8EA1-0x8EDF
548  * Code set 3 (JIS X 0212-1990):		0x8FA1A1-0x8FFEFE
549  */
550 int
551 towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
552 {
553 	return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
554 }
555 
556 /*
557  * EUC-KR encodes as follows:
558  *
559  * Code set 0 (ASCII or KS C 5636-1993):	0x21-0x7E
560  * Code set 1 (KS C 5601-1992):			0xA1A1-0xFEFE
561  * Code set 2:					unused
562  * Code set 3:					unused
563  */
564 int
565 towide_euckr(wchar_t *wc, const char *mb, unsigned n)
566 {
567 	return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
568 }
569 
570 /*
571  * EUC-TW encodes as follows:
572  *
573  * Code set 0 (ASCII):				0x21-0x7E
574  * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
575  * Code set 2 (CNS 11643-1992 Planes 1-16):	0x8EA1A1A1-0x8EB0FEFE
576  * Code set 3:					unused
577  */
578 int
579 towide_euctw(wchar_t *wc, const char *mb, unsigned n)
580 {
581 	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
582 }
583 
584 /*
585  * Public entry points.
586  */
587 
588 int
589 to_wide(wchar_t *wc, const char *mb)
590 {
591 	/* this won't fail hard */
592 	return (_towide(wc, mb, strlen(mb)));
593 }
594 
595 int
596 to_mb(char *mb, wchar_t wc)
597 {
598 	int	rv;
599 
600 	if ((rv = _tomb(mb, wc)) < 0) {
601 		warn("%s", widemsg);
602 		free(widemsg);
603 		widemsg = NULL;
604 	}
605 	return (rv);
606 }
607 
608 char *
609 to_mb_string(const wchar_t *wcs)
610 {
611 	char	*mbs;
612 	char	*ptr;
613 	int	len;
614 
615 	mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
616 	if (mbs == NULL) {
617 		warn("out of memory");
618 		return (NULL);
619 	}
620 	ptr = mbs;
621 	while (*wcs) {
622 		if ((len = to_mb(ptr, *wcs)) < 0) {
623 			INTERR;
624 			free(mbs);
625 			return (NULL);
626 		}
627 		wcs++;
628 		ptr += len;
629 	}
630 	*ptr = 0;
631 	return (mbs);
632 }
633 
634 void
635 set_wide_encoding(const char *encoding)
636 {
637 	int i;
638 
639 	_towide = towide_none;
640 	_tomb = tomb_none;
641 	_nbits = 8;
642 
643 	snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s",
644 	    encoding);
645 	for (i = 0; mb_encodings[i].name; i++) {
646 		if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
647 			_towide = mb_encodings[i].towide;
648 			_tomb = mb_encodings[i].tomb;
649 			_encoding = mb_encodings[i].cname;
650 			_nbits = mb_encodings[i].nbits;
651 			break;
652 		}
653 	}
654 }
655 
656 const char *
657 get_wide_encoding(void)
658 {
659 	return (_encoding);
660 }
661 
662 int
663 max_wide(void)
664 {
665 	return ((int)((1U << _nbits) - 1));
666 }
667