xref: /freebsd/contrib/less/charset.c (revision cfd6422a5217410fbd66f7a7a8a64d9d85e61229)
1 /*
2  * Copyright (C) 1984-2020  Mark Nudelman
3  *
4  * You may distribute under the terms of either the GNU General Public
5  * License or the Less License, as specified in the README file.
6  *
7  * For more information, see the README file.
8  */
9 
10 
11 /*
12  * Functions to define the character set
13  * and do things specific to the character set.
14  */
15 
16 #include "less.h"
17 #if HAVE_LOCALE
18 #include <locale.h>
19 #include <ctype.h>
20 #include <langinfo.h>
21 #endif
22 
23 #include "charset.h"
24 
25 #if MSDOS_COMPILER==WIN32C
26 #define WIN32_LEAN_AND_MEAN
27 #include <windows.h>
28 #endif
29 
30 extern int bs_mode;
31 
32 public int utf_mode = 0;
33 
34 /*
35  * Predefined character sets,
36  * selected by the LESSCHARSET environment variable.
37  */
38 struct charset {
39 	char *name;
40 	int *p_flag;
41 	char *desc;
42 } charsets[] = {
43 	{ "ascii",		NULL,       "8bcccbcc18b95.b" },
44 	{ "utf-8",		&utf_mode,  "8bcccbcc18b95.b126.bb" },
45 	{ "iso8859",		NULL,       "8bcccbcc18b95.33b." },
46 	{ "latin3",		NULL,       "8bcccbcc18b95.33b5.b8.b15.b4.b12.b18.b12.b." },
47 	{ "arabic",		NULL,       "8bcccbcc18b95.33b.3b.7b2.13b.3b.b26.5b19.b" },
48 	{ "greek",		NULL,       "8bcccbcc18b95.33b4.2b4.b3.b35.b44.b" },
49 	{ "greek2005",		NULL,       "8bcccbcc18b95.33b14.b35.b44.b" },
50 	{ "hebrew",		NULL,       "8bcccbcc18b95.33b.b29.32b28.2b2.b" },
51 	{ "koi8-r",		NULL,       "8bcccbcc18b95.b." },
52 	{ "KOI8-T",		NULL,       "8bcccbcc18b95.b8.b6.b8.b.b.5b7.3b4.b4.b3.b.b.3b." },
53 	{ "georgianps",		NULL,       "8bcccbcc18b95.3b11.4b12.2b." },
54 	{ "tcvn",		NULL,       "b..b...bcccbccbbb7.8b95.b48.5b." },
55 	{ "TIS-620",		NULL,       "8bcccbcc18b95.b.4b.11b7.8b." },
56 	{ "next",		NULL,       "8bcccbcc18b95.bb125.bb" },
57 	{ "dos",		NULL,       "8bcccbcc12bc5b95.b." },
58 	{ "windows-1251",	NULL,       "8bcccbcc12bc5b95.b24.b." },
59 	{ "windows-1252",	NULL,       "8bcccbcc12bc5b95.b.b11.b.2b12.b." },
60 	{ "windows-1255",	NULL,       "8bcccbcc12bc5b95.b.b8.b.5b9.b.4b." },
61 	{ "ebcdic",		NULL,       "5bc6bcc7bcc41b.9b7.9b5.b..8b6.10b6.b9.7b9.8b8.17b3.3b9.7b9.8b8.6b10.b.b.b." },
62 	{ "IBM-1047",		NULL,       "4cbcbc3b9cbccbccbb4c6bcc5b3cbbc4bc4bccbc191.b" },
63 	{ NULL, NULL, NULL }
64 };
65 
66 /*
67  * Support "locale charmap"/nl_langinfo(CODESET) values, as well as others.
68  */
69 struct cs_alias {
70 	char *name;
71 	char *oname;
72 } cs_aliases[] = {
73 	{ "UTF-8",		"utf-8" },
74 	{ "utf8",		"utf-8" },
75 	{ "UTF8",		"utf-8" },
76 	{ "ANSI_X3.4-1968",	"ascii" },
77 	{ "US-ASCII",		"ascii" },
78 	{ "latin1",		"iso8859" },
79 	{ "ISO-8859-1",		"iso8859" },
80 	{ "latin9",		"iso8859" },
81 	{ "ISO-8859-15",	"iso8859" },
82 	{ "latin2",		"iso8859" },
83 	{ "ISO-8859-2",		"iso8859" },
84 	{ "ISO-8859-3",		"latin3" },
85 	{ "latin4",		"iso8859" },
86 	{ "ISO-8859-4",		"iso8859" },
87 	{ "cyrillic",		"iso8859" },
88 	{ "ISO-8859-5",		"iso8859" },
89 	{ "ISO-8859-6",		"arabic" },
90 	{ "ISO-8859-7",		"greek" },
91 	{ "IBM9005",		"greek2005" },
92 	{ "ISO-8859-8",		"hebrew" },
93 	{ "latin5",		"iso8859" },
94 	{ "ISO-8859-9",		"iso8859" },
95 	{ "latin6",		"iso8859" },
96 	{ "ISO-8859-10",	"iso8859" },
97 	{ "latin7",		"iso8859" },
98 	{ "ISO-8859-13",	"iso8859" },
99 	{ "latin8",		"iso8859" },
100 	{ "ISO-8859-14",	"iso8859" },
101 	{ "latin10",		"iso8859" },
102 	{ "ISO-8859-16",	"iso8859" },
103 	{ "IBM437",		"dos" },
104 	{ "EBCDIC-US",		"ebcdic" },
105 	{ "IBM1047",		"IBM-1047" },
106 	{ "KOI8-R",		"koi8-r" },
107 	{ "KOI8-U",		"koi8-r" },
108 	{ "GEORGIAN-PS",	"georgianps" },
109 	{ "TCVN5712-1", 	"tcvn" },
110 	{ "NEXTSTEP",		"next" },
111 	{ "windows",		"windows-1252" }, /* backward compatibility */
112 	{ "CP1251",		"windows-1251" },
113 	{ "CP1252",		"windows-1252" },
114 	{ "CP1255",		"windows-1255" },
115 	{ NULL, NULL }
116 };
117 
118 #define	IS_BINARY_CHAR	01
119 #define	IS_CONTROL_CHAR	02
120 
121 static char chardef[256];
122 static char *binfmt = NULL;
123 static char *utfbinfmt = NULL;
124 public int binattr = AT_STANDOUT;
125 
126 
127 /*
128  * Define a charset, given a description string.
129  * The string consists of 256 letters,
130  * one for each character in the charset.
131  * If the string is shorter than 256 letters, missing letters
132  * are taken to be identical to the last one.
133  * A decimal number followed by a letter is taken to be a
134  * repetition of the letter.
135  *
136  * Each letter is one of:
137  *	. normal character
138  *	b binary character
139  *	c control character
140  */
141 	static void
142 ichardef(s)
143 	char *s;
144 {
145 	char *cp;
146 	int n;
147 	char v;
148 
149 	n = 0;
150 	v = 0;
151 	cp = chardef;
152 	while (*s != '\0')
153 	{
154 		switch (*s++)
155 		{
156 		case '.':
157 			v = 0;
158 			break;
159 		case 'c':
160 			v = IS_CONTROL_CHAR;
161 			break;
162 		case 'b':
163 			v = IS_BINARY_CHAR|IS_CONTROL_CHAR;
164 			break;
165 
166 		case '0': case '1': case '2': case '3': case '4':
167 		case '5': case '6': case '7': case '8': case '9':
168 			n = (10 * n) + (s[-1] - '0');
169 			continue;
170 
171 		default:
172 			error("invalid chardef", NULL_PARG);
173 			quit(QUIT_ERROR);
174 			/*NOTREACHED*/
175 		}
176 
177 		do
178 		{
179 			if (cp >= chardef + sizeof(chardef))
180 			{
181 				error("chardef longer than 256", NULL_PARG);
182 				quit(QUIT_ERROR);
183 				/*NOTREACHED*/
184 			}
185 			*cp++ = v;
186 		} while (--n > 0);
187 		n = 0;
188 	}
189 
190 	while (cp < chardef + sizeof(chardef))
191 		*cp++ = v;
192 }
193 
194 /*
195  * Define a charset, given a charset name.
196  * The valid charset names are listed in the "charsets" array.
197  */
198 	static int
199 icharset(name, no_error)
200 	char *name;
201 	int no_error;
202 {
203 	struct charset *p;
204 	struct cs_alias *a;
205 
206 	if (name == NULL || *name == '\0')
207 		return (0);
208 
209 	/* First see if the name is an alias. */
210 	for (a = cs_aliases;  a->name != NULL;  a++)
211 	{
212 		if (strcmp(name, a->name) == 0)
213 		{
214 			name = a->oname;
215 			break;
216 		}
217 	}
218 
219 	for (p = charsets;  p->name != NULL;  p++)
220 	{
221 		if (strcmp(name, p->name) == 0)
222 		{
223 			ichardef(p->desc);
224 			if (p->p_flag != NULL)
225 			{
226 #if MSDOS_COMPILER==WIN32C
227 				*(p->p_flag) = 1 + (GetConsoleOutputCP() != CP_UTF8);
228 #else
229 				*(p->p_flag) = 1;
230 #endif
231 			}
232 			return (1);
233 		}
234 	}
235 
236 	if (!no_error) {
237 		error("invalid charset name", NULL_PARG);
238 		quit(QUIT_ERROR);
239 	}
240 	return (0);
241 }
242 
243 #if HAVE_LOCALE
244 /*
245  * Define a charset, given a locale name.
246  */
247 	static void
248 ilocale(VOID_PARAM)
249 {
250 	int c;
251 
252 	for (c = 0;  c < (int) sizeof(chardef);  c++)
253 	{
254 		if (isprint(c))
255 			chardef[c] = 0;
256 		else if (iscntrl(c))
257 			chardef[c] = IS_CONTROL_CHAR;
258 		else
259 			chardef[c] = IS_BINARY_CHAR|IS_CONTROL_CHAR;
260 	}
261 }
262 #endif
263 
264 /*
265  * Define the printing format for control (or binary utf) chars.
266  */
267 	public void
268 setfmt(s, fmtvarptr, attrptr, default_fmt)
269 	char *s;
270 	char **fmtvarptr;
271 	int *attrptr;
272 	char *default_fmt;
273 {
274 	if (s && utf_mode)
275 	{
276 		/* It would be too hard to account for width otherwise.  */
277 		char constant *t = s;
278 		while (*t)
279 		{
280 			if (*t < ' ' || *t > '~')
281 			{
282 				s = default_fmt;
283 				goto attr;
284 			}
285 			t++;
286 		}
287 	}
288 
289 	/* %n is evil */
290 	if (s == NULL || *s == '\0' ||
291 	    (*s == '*' && (s[1] == '\0' || s[2] == '\0' || strchr(s + 2, 'n'))) ||
292 	    (*s != '*' && strchr(s, 'n')))
293 		s = default_fmt;
294 
295 	/*
296 	 * Select the attributes if it starts with "*".
297 	 */
298  attr:
299 	if (*s == '*' && s[1] != '\0')
300 	{
301 		switch (s[1])
302 		{
303 		case 'd':  *attrptr = AT_BOLD;      break;
304 		case 'k':  *attrptr = AT_BLINK;     break;
305 		case 's':  *attrptr = AT_STANDOUT;  break;
306 		case 'u':  *attrptr = AT_UNDERLINE; break;
307 		default:   *attrptr = AT_NORMAL;    break;
308 		}
309 		s += 2;
310 	}
311 	*fmtvarptr = s;
312 }
313 
314 /*
315  *
316  */
317 	static void
318 set_charset(VOID_PARAM)
319 {
320 	char *s;
321 
322 #if MSDOS_COMPILER==WIN32C
323 	/*
324 	 * If the Windows console is using UTF-8, we'll use it too.
325 	 */
326 	if (GetConsoleOutputCP() == CP_UTF8)
327 		if (icharset("utf-8", 1))
328 			return;
329 #endif
330 	/*
331 	 * See if environment variable LESSCHARSET is defined.
332 	 */
333 	s = lgetenv("LESSCHARSET");
334 	if (icharset(s, 0))
335 		return;
336 
337 	/*
338 	 * LESSCHARSET is not defined: try LESSCHARDEF.
339 	 */
340 	s = lgetenv("LESSCHARDEF");
341 	if (!isnullenv(s))
342 	{
343 		ichardef(s);
344 		return;
345 	}
346 
347 #if HAVE_LOCALE
348 #ifdef CODESET
349 	/*
350 	 * Try using the codeset name as the charset name.
351 	 */
352 	s = nl_langinfo(CODESET);
353 	if (icharset(s, 1))
354 		return;
355 #endif
356 #endif
357 
358 #if HAVE_STRSTR
359 	/*
360 	 * Check whether LC_ALL, LC_CTYPE or LANG look like UTF-8 is used.
361 	 */
362 	if ((s = lgetenv("LC_ALL")) != NULL ||
363 	    (s = lgetenv("LC_CTYPE")) != NULL ||
364 	    (s = lgetenv("LANG")) != NULL)
365 	{
366 		if (   strstr(s, "UTF-8") != NULL || strstr(s, "utf-8") != NULL
367 		    || strstr(s, "UTF8")  != NULL || strstr(s, "utf8")  != NULL)
368 			if (icharset("utf-8", 1))
369 				return;
370 	}
371 #endif
372 
373 #if HAVE_LOCALE
374 	/*
375 	 * Get character definitions from locale functions,
376 	 * rather than from predefined charset entry.
377 	 */
378 	ilocale();
379 #else
380 #if MSDOS_COMPILER
381 	/*
382 	 * Default to "dos".
383 	 */
384 	(void) icharset("dos", 1);
385 #else
386 	/*
387 	 * Default to "latin1".
388 	 */
389 	(void) icharset("latin1", 1);
390 #endif
391 #endif
392 }
393 
394 /*
395  * Initialize charset data structures.
396  */
397 	public void
398 init_charset(VOID_PARAM)
399 {
400 	char *s;
401 
402 #if HAVE_LOCALE
403 	setlocale(LC_ALL, "");
404 #endif
405 
406 	set_charset();
407 
408 	s = lgetenv("LESSBINFMT");
409 	setfmt(s, &binfmt, &binattr, "*s<%02X>");
410 
411 	s = lgetenv("LESSUTFBINFMT");
412 	setfmt(s, &utfbinfmt, &binattr, "<U+%04lX>");
413 }
414 
415 /*
416  * Is a given character a "binary" character?
417  */
418 	public int
419 binary_char(c)
420 	LWCHAR c;
421 {
422 	if (utf_mode)
423 		return (is_ubin_char(c));
424 	c &= 0377;
425 	return (chardef[c] & IS_BINARY_CHAR);
426 }
427 
428 /*
429  * Is a given character a "control" character?
430  */
431 	public int
432 control_char(c)
433 	LWCHAR c;
434 {
435 	c &= 0377;
436 	return (chardef[c] & IS_CONTROL_CHAR);
437 }
438 
439 /*
440  * Return the printable form of a character.
441  * For example, in the "ascii" charset '\3' is printed as "^C".
442  */
443 	public char *
444 prchar(c)
445 	LWCHAR c;
446 {
447 	/* {{ This buffer can be overrun if LESSBINFMT is a long string. }} */
448 	static char buf[32];
449 
450 	c &= 0377;
451 	if ((c < 128 || !utf_mode) && !control_char(c))
452 		SNPRINTF1(buf, sizeof(buf), "%c", (int) c);
453 	else if (c == ESC)
454 		strcpy(buf, "ESC");
455 #if IS_EBCDIC_HOST
456 	else if (!binary_char(c) && c < 64)
457 		SNPRINTF1(buf, sizeof(buf), "^%c",
458 		/*
459 		 * This array roughly inverts CONTROL() #defined in less.h,
460 	 	 * and should be kept in sync with CONTROL() and IBM-1047.
461  	 	 */
462 		"@ABC.I.?...KLMNO"
463 		"PQRS.JH.XY.."
464 		"\\]^_"
465 		"......W[.....EFG"
466 		"..V....D....TU.Z"[c]);
467 #else
468   	else if (c < 128 && !control_char(c ^ 0100))
469   		SNPRINTF1(buf, sizeof(buf), "^%c", (int) (c ^ 0100));
470 #endif
471 	else
472 		SNPRINTF1(buf, sizeof(buf), binfmt, c);
473 	return (buf);
474 }
475 
476 /*
477  * Return the printable form of a UTF-8 character.
478  */
479 	public char *
480 prutfchar(ch)
481 	LWCHAR ch;
482 {
483 	static char buf[32];
484 
485 	if (ch == ESC)
486 		strcpy(buf, "ESC");
487   	else if (ch < 128 && control_char(ch))
488 	{
489 		if (!control_char(ch ^ 0100))
490 			SNPRINTF1(buf, sizeof(buf), "^%c", ((char) ch) ^ 0100);
491 		else
492 			SNPRINTF1(buf, sizeof(buf), binfmt, (char) ch);
493 	} else if (is_ubin_char(ch))
494 	{
495 		SNPRINTF1(buf, sizeof(buf), utfbinfmt, ch);
496 	} else
497 	{
498 		char *p = buf;
499 		if (ch >= 0x80000000)
500 			ch = 0xFFFD; /* REPLACEMENT CHARACTER */
501 		put_wchar(&p, ch);
502 		*p = '\0';
503 	}
504 	return (buf);
505 }
506 
507 /*
508  * Get the length of a UTF-8 character in bytes.
509  */
510 	public int
511 utf_len(ch)
512 	int ch;
513 {
514 	if ((ch & 0x80) == 0)
515 		return 1;
516 	if ((ch & 0xE0) == 0xC0)
517 		return 2;
518 	if ((ch & 0xF0) == 0xE0)
519 		return 3;
520 	if ((ch & 0xF8) == 0xF0)
521 		return 4;
522 	if ((ch & 0xFC) == 0xF8)
523 		return 5;
524 	if ((ch & 0xFE) == 0xFC)
525 		return 6;
526 	/* Invalid UTF-8 encoding. */
527 	return 1;
528 }
529 
530 /*
531  * Does the parameter point to the lead byte of a well-formed UTF-8 character?
532  */
533 	public int
534 is_utf8_well_formed(ss, slen)
535 	char *ss;
536 	int slen;
537 {
538 	int i;
539 	int len;
540 	unsigned char *s = (unsigned char *) ss;
541 
542 	if (IS_UTF8_INVALID(s[0]))
543 		return (0);
544 
545 	len = utf_len(s[0]);
546 	if (len > slen)
547 		return (0);
548 	if (len == 1)
549 		return (1);
550 	if (len == 2)
551 	{
552 		if (s[0] < 0xC2)
553 		    return (0);
554 	} else
555 	{
556 		unsigned char mask;
557 		mask = (~((1 << (8-len)) - 1)) & 0xFF;
558 		if (s[0] == mask && (s[1] & mask) == 0x80)
559 			return (0);
560 	}
561 
562 	for (i = 1;  i < len;  i++)
563 		if (!IS_UTF8_TRAIL(s[i]))
564 			return (0);
565 	return (1);
566 }
567 
568 /*
569  * Skip bytes until a UTF-8 lead byte (11xxxxxx) or ASCII byte (0xxxxxxx) is found.
570  */
571 	public void
572 utf_skip_to_lead(pp, limit)
573 	char **pp;
574 	char *limit;
575 {
576 	do {
577 		++(*pp);
578 	} while (*pp < limit && !IS_UTF8_LEAD((*pp)[0] & 0377) && !IS_ASCII_OCTET((*pp)[0]));
579 }
580 
581 
582 /*
583  * Get the value of a UTF-8 character.
584  */
585 	public LWCHAR
586 get_wchar(p)
587 	constant char *p;
588 {
589 	switch (utf_len(p[0]))
590 	{
591 	case 1:
592 	default:
593 		/* 0xxxxxxx */
594 		return (LWCHAR)
595 			(p[0] & 0xFF);
596 	case 2:
597 		/* 110xxxxx 10xxxxxx */
598 		return (LWCHAR) (
599 			((p[0] & 0x1F) << 6) |
600 			(p[1] & 0x3F));
601 	case 3:
602 		/* 1110xxxx 10xxxxxx 10xxxxxx */
603 		return (LWCHAR) (
604 			((p[0] & 0x0F) << 12) |
605 			((p[1] & 0x3F) << 6) |
606 			(p[2] & 0x3F));
607 	case 4:
608 		/* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
609 		return (LWCHAR) (
610 			((p[0] & 0x07) << 18) |
611 			((p[1] & 0x3F) << 12) |
612 			((p[2] & 0x3F) << 6) |
613 			(p[3] & 0x3F));
614 	case 5:
615 		/* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
616 		return (LWCHAR) (
617 			((p[0] & 0x03) << 24) |
618 			((p[1] & 0x3F) << 18) |
619 			((p[2] & 0x3F) << 12) |
620 			((p[3] & 0x3F) << 6) |
621 			(p[4] & 0x3F));
622 	case 6:
623 		/* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
624 		return (LWCHAR) (
625 			((p[0] & 0x01) << 30) |
626 			((p[1] & 0x3F) << 24) |
627 			((p[2] & 0x3F) << 18) |
628 			((p[3] & 0x3F) << 12) |
629 			((p[4] & 0x3F) << 6) |
630 			(p[5] & 0x3F));
631 	}
632 }
633 
634 /*
635  * Store a character into a UTF-8 string.
636  */
637 	public void
638 put_wchar(pp, ch)
639 	char **pp;
640 	LWCHAR ch;
641 {
642 	if (!utf_mode || ch < 0x80)
643 	{
644 		/* 0xxxxxxx */
645 		*(*pp)++ = (char) ch;
646 	} else if (ch < 0x800)
647 	{
648 		/* 110xxxxx 10xxxxxx */
649 		*(*pp)++ = (char) (0xC0 | ((ch >> 6) & 0x1F));
650 		*(*pp)++ = (char) (0x80 | (ch & 0x3F));
651 	} else if (ch < 0x10000)
652 	{
653 		/* 1110xxxx 10xxxxxx 10xxxxxx */
654 		*(*pp)++ = (char) (0xE0 | ((ch >> 12) & 0x0F));
655 		*(*pp)++ = (char) (0x80 | ((ch >> 6) & 0x3F));
656 		*(*pp)++ = (char) (0x80 | (ch & 0x3F));
657 	} else if (ch < 0x200000)
658 	{
659 		/* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
660 		*(*pp)++ = (char) (0xF0 | ((ch >> 18) & 0x07));
661 		*(*pp)++ = (char) (0x80 | ((ch >> 12) & 0x3F));
662 		*(*pp)++ = (char) (0x80 | ((ch >> 6) & 0x3F));
663 		*(*pp)++ = (char) (0x80 | (ch & 0x3F));
664 	} else if (ch < 0x4000000)
665 	{
666 		/* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
667 		*(*pp)++ = (char) (0xF0 | ((ch >> 24) & 0x03));
668 		*(*pp)++ = (char) (0x80 | ((ch >> 18) & 0x3F));
669 		*(*pp)++ = (char) (0x80 | ((ch >> 12) & 0x3F));
670 		*(*pp)++ = (char) (0x80 | ((ch >> 6) & 0x3F));
671 		*(*pp)++ = (char) (0x80 | (ch & 0x3F));
672 	} else
673 	{
674 		/* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
675 		*(*pp)++ = (char) (0xF0 | ((ch >> 30) & 0x01));
676 		*(*pp)++ = (char) (0x80 | ((ch >> 24) & 0x3F));
677 		*(*pp)++ = (char) (0x80 | ((ch >> 18) & 0x3F));
678 		*(*pp)++ = (char) (0x80 | ((ch >> 12) & 0x3F));
679 		*(*pp)++ = (char) (0x80 | ((ch >> 6) & 0x3F));
680 		*(*pp)++ = (char) (0x80 | (ch & 0x3F));
681 	}
682 }
683 
684 /*
685  * Step forward or backward one character in a string.
686  */
687 	public LWCHAR
688 step_char(pp, dir, limit)
689 	char **pp;
690 	signed int dir;
691 	constant char *limit;
692 {
693 	LWCHAR ch;
694 	int len;
695 	char *p = *pp;
696 
697 	if (!utf_mode)
698 	{
699 		/* It's easy if chars are one byte. */
700 		if (dir > 0)
701 			ch = (LWCHAR) (unsigned char) ((p < limit) ? *p++ : 0);
702 		else
703 			ch = (LWCHAR) (unsigned char) ((p > limit) ? *--p : 0);
704 	} else if (dir > 0)
705 	{
706 		len = utf_len(*p);
707 		if (p + len > limit)
708 		{
709 			ch = 0;
710 			p = (char *) limit;
711 		} else
712 		{
713 			ch = get_wchar(p);
714 			p += len;
715 		}
716 	} else
717 	{
718 		while (p > limit && IS_UTF8_TRAIL(p[-1]))
719 			p--;
720 		if (p > limit)
721 			ch = get_wchar(--p);
722 		else
723 			ch = 0;
724 	}
725 	*pp = p;
726 	return ch;
727 }
728 
729 /*
730  * Unicode characters data
731  * Actual data is in the generated *.uni files.
732  */
733 
734 #define DECLARE_RANGE_TABLE_START(name) \
735     static struct wchar_range name##_array[] = {
736 #define DECLARE_RANGE_TABLE_END(name) \
737     }; struct wchar_range_table name##_table = { name##_array, sizeof(name##_array)/sizeof(*name##_array) };
738 
739 DECLARE_RANGE_TABLE_START(compose)
740 #include "compose.uni"
741 DECLARE_RANGE_TABLE_END(compose)
742 
743 DECLARE_RANGE_TABLE_START(ubin)
744 #include "ubin.uni"
745 DECLARE_RANGE_TABLE_END(ubin)
746 
747 DECLARE_RANGE_TABLE_START(wide)
748 #include "wide.uni"
749 DECLARE_RANGE_TABLE_END(wide)
750 
751 DECLARE_RANGE_TABLE_START(fmt)
752 #include "fmt.uni"
753 DECLARE_RANGE_TABLE_END(fmt)
754 
755 /* comb_table is special pairs, not ranges. */
756 static struct wchar_range comb_table[] = {
757 	{0x0644,0x0622}, {0x0644,0x0623}, {0x0644,0x0625}, {0x0644,0x0627},
758 };
759 
760 
761 	static int
762 is_in_table(ch, table)
763 	LWCHAR ch;
764 	struct wchar_range_table *table;
765 {
766 	int hi;
767 	int lo;
768 
769 	/* Binary search in the table. */
770 	if (ch < table->table[0].first)
771 		return 0;
772 	lo = 0;
773 	hi = table->count - 1;
774 	while (lo <= hi)
775 	{
776 		int mid = (lo + hi) / 2;
777 		if (ch > table->table[mid].last)
778 			lo = mid + 1;
779 		else if (ch < table->table[mid].first)
780 			hi = mid - 1;
781 		else
782 			return 1;
783 	}
784 	return 0;
785 }
786 
787 /*
788  * Is a character a UTF-8 composing character?
789  * If a composing character follows any char, the two combine into one glyph.
790  */
791 	public int
792 is_composing_char(ch)
793 	LWCHAR ch;
794 {
795 	return is_in_table(ch, &compose_table) ||
796 	       (bs_mode != BS_CONTROL && is_in_table(ch, &fmt_table));
797 }
798 
799 /*
800  * Should this UTF-8 character be treated as binary?
801  */
802 	public int
803 is_ubin_char(ch)
804 	LWCHAR ch;
805 {
806 	int ubin = is_in_table(ch, &ubin_table) ||
807 	           (bs_mode == BS_CONTROL && is_in_table(ch, &fmt_table));
808 #if MSDOS_COMPILER==WIN32C
809 	if (!ubin && utf_mode == 2 && ch < 0x10000)
810 	{
811 		/*
812 		 * Consider it binary if it can't be converted.
813 		 */
814 		BOOL used_default = TRUE;
815 		WideCharToMultiByte(GetConsoleOutputCP(), WC_NO_BEST_FIT_CHARS, (LPCWSTR) &ch, 1, NULL, 0, NULL, &used_default);
816 		if (used_default)
817 			ubin = 1;
818 	}
819 #endif
820 	return ubin;
821 }
822 
823 /*
824  * Is this a double width UTF-8 character?
825  */
826 	public int
827 is_wide_char(ch)
828 	LWCHAR ch;
829 {
830 	return is_in_table(ch, &wide_table);
831 }
832 
833 /*
834  * Is a character a UTF-8 combining character?
835  * A combining char acts like an ordinary char, but if it follows
836  * a specific char (not any char), the two combine into one glyph.
837  */
838 	public int
839 is_combining_char(ch1, ch2)
840 	LWCHAR ch1;
841 	LWCHAR ch2;
842 {
843 	/* The table is small; use linear search. */
844 	int i;
845 	for (i = 0;  i < sizeof(comb_table)/sizeof(*comb_table);  i++)
846 	{
847 		if (ch1 == comb_table[i].first &&
848 		    ch2 == comb_table[i].last)
849 			return 1;
850 	}
851 	return 0;
852 }
853 
854