1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
14 */
15
16 /*
17 * CHARMAP file handling for iconv.
18 */
19
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <errno.h>
24 #include <limits.h>
25 #include <unistd.h>
26 #include <alloca.h>
27 #include <sys/avl.h>
28 #include <stddef.h>
29 #include <unistd.h>
30 #include "charmap.h"
31 #include "parser.tab.h"
32 #include <assert.h>
33
34 enum cmap_pass cmap_pass;
35 static avl_tree_t cmap_sym;
36 static avl_tree_t cmap_mbs;
37
38 typedef struct charmap {
39 const char *cm_name;
40 struct charmap *cm_alias_of;
41 avl_node_t cm_avl_sym;
42 avl_node_t cm_avl_mbs;
43 int cm_warned;
44 int cm_frmbs_len;
45 int cm_tombs_len;
46 char cm_frmbs[MB_LEN_MAX + 1]; /* input */
47 char cm_tombs[MB_LEN_MAX + 1]; /* output */
48 } charmap_t;
49
50 static void add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups);
51 static void add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups);
52
53 /*
54 * Array of POSIX specific portable characters.
55 */
56 static const struct {
57 char *name;
58 int ch;
59 } portable_chars[] = {
60 { "NUL", '\0' },
61 { "alert", '\a' },
62 { "backspace", '\b' },
63 { "tab", '\t' },
64 { "carriage-return", '\r' },
65 { "newline", '\n' },
66 { "vertical-tab", '\v' },
67 { "form-feed", '\f' },
68 { "space", ' ' },
69 { "exclamation-mark", '!' },
70 { "quotation-mark", '"' },
71 { "number-sign", '#' },
72 { "dollar-sign", '$' },
73 { "percent-sign", '%' },
74 { "ampersand", '&' },
75 { "apostrophe", '\'' },
76 { "left-parenthesis", '(' },
77 { "right-parenthesis", '(' },
78 { "asterisk", '*' },
79 { "plus-sign", '+' },
80 { "comma", ','},
81 { "hyphen-minus", '-' },
82 { "hyphen", '-' },
83 { "full-stop", '.' },
84 { "period", '.' },
85 { "slash", '/' },
86 { "solidus", '/' },
87 { "zero", '0' },
88 { "one", '1' },
89 { "two", '2' },
90 { "three", '3' },
91 { "four", '4' },
92 { "five", '5' },
93 { "six", '6' },
94 { "seven", '7' },
95 { "eight", '8' },
96 { "nine", '9' },
97 { "colon", ':' },
98 { "semicolon", ';' },
99 { "less-than-sign", '<' },
100 { "equals-sign", '=' },
101 { "greater-than-sign", '>' },
102 { "question-mark", '?' },
103 { "commercial-at", '@' },
104 { "left-square-bracket", '[' },
105 { "backslash", '\\' },
106 { "reverse-solidus", '\\' },
107 { "right-square-bracket", ']' },
108 { "circumflex", '^' },
109 { "circumflex-accent", '^' },
110 { "low-line", '_' },
111 { "underscore", '_' },
112 { "grave-accent", '`' },
113 { "left-brace", '{' },
114 { "left-curly-bracket", '{' },
115 { "vertical-line", '|' },
116 { "right-brace", '}' },
117 { "right-curly-bracket", '}' },
118 { "tilde", '~' },
119 { "A", 'A' },
120 { "B", 'B' },
121 { "C", 'C' },
122 { "D", 'D' },
123 { "E", 'E' },
124 { "F", 'F' },
125 { "G", 'G' },
126 { "H", 'H' },
127 { "I", 'I' },
128 { "J", 'J' },
129 { "K", 'K' },
130 { "L", 'L' },
131 { "M", 'M' },
132 { "N", 'N' },
133 { "O", 'O' },
134 { "P", 'P' },
135 { "Q", 'Q' },
136 { "R", 'R' },
137 { "S", 'S' },
138 { "T", 'T' },
139 { "U", 'U' },
140 { "V", 'V' },
141 { "W", 'W' },
142 { "X", 'X' },
143 { "Y", 'Y' },
144 { "Z", 'Z' },
145 { "a", 'a' },
146 { "b", 'b' },
147 { "c", 'c' },
148 { "d", 'd' },
149 { "e", 'e' },
150 { "f", 'f' },
151 { "g", 'g' },
152 { "h", 'h' },
153 { "i", 'i' },
154 { "j", 'j' },
155 { "k", 'k' },
156 { "l", 'l' },
157 { "m", 'm' },
158 { "n", 'n' },
159 { "o", 'o' },
160 { "p", 'p' },
161 { "q", 'q' },
162 { "r", 'r' },
163 { "s", 's' },
164 { "t", 't' },
165 { "u", 'u' },
166 { "v", 'v' },
167 { "w", 'w' },
168 { "x", 'x' },
169 { "y", 'y' },
170 { "z", 'z' },
171 { NULL, 0 }
172 };
173
174 static int
cmap_compare_sym(const void * n1,const void * n2)175 cmap_compare_sym(const void *n1, const void *n2)
176 {
177 const charmap_t *c1 = n1;
178 const charmap_t *c2 = n2;
179 int rv;
180
181 rv = strcmp(c1->cm_name, c2->cm_name);
182 return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0);
183 }
184
185 /*
186 * In order for partial match searches to work,
187 * we need these sorted by mbs contents.
188 */
189 static int
cmap_compare_mbs(const void * n1,const void * n2)190 cmap_compare_mbs(const void *n1, const void *n2)
191 {
192 const charmap_t *c1 = n1;
193 const charmap_t *c2 = n2;
194 int len, rv;
195
196 len = c1->cm_frmbs_len;
197 if (len < c2->cm_frmbs_len)
198 len = c2->cm_frmbs_len;
199 rv = memcmp(c1->cm_frmbs, c2->cm_frmbs, len);
200 if (rv < 0)
201 return (-1);
202 if (rv > 0)
203 return (1);
204 /* they match through length */
205 if (c1->cm_frmbs_len < c2->cm_frmbs_len)
206 return (-1);
207 if (c2->cm_frmbs_len < c1->cm_frmbs_len)
208 return (1);
209 return (0);
210 }
211
212 void
charmap_init(char * to_map,char * from_map)213 charmap_init(char *to_map, char *from_map)
214 {
215 avl_create(&cmap_sym, cmap_compare_sym, sizeof (charmap_t),
216 offsetof(charmap_t, cm_avl_sym));
217
218 avl_create(&cmap_mbs, cmap_compare_mbs, sizeof (charmap_t),
219 offsetof(charmap_t, cm_avl_mbs));
220
221 cmap_pass = CMAP_PASS_FROM;
222 reset_scanner(from_map);
223 (void) yyparse();
224 add_charmap_posix();
225
226 cmap_pass = CMAP_PASS_TO;
227 reset_scanner(to_map);
228 (void) yyparse();
229 }
230
231 void
charmap_dump()232 charmap_dump()
233 {
234 charmap_t *cm;
235 int i;
236
237 cm = avl_first(&cmap_mbs);
238 while (cm != NULL) {
239 (void) printf("name=\"%s\"\n", cm->cm_name);
240
241 (void) printf("\timbs=\"");
242 for (i = 0; i < cm->cm_frmbs_len; i++)
243 (void) printf("\\x%02x", cm->cm_frmbs[i] & 0xFF);
244 (void) printf("\"\n");
245
246 (void) printf("\tombs=\"");
247 for (i = 0; i < cm->cm_tombs_len; i++)
248 (void) printf("\\x%02x", cm->cm_tombs[i] & 0xFF);
249 (void) printf("\"\n");
250
251 cm = AVL_NEXT(&cmap_mbs, cm);
252 }
253 }
254
255 /*
256 * We parse two charmap files: First the "from" map, where we build
257 * cmap_mbs and cmap_sym which we'll later use to translate the input
258 * stream (mbs encodings) to symbols. Second, we parse the "to" map,
259 * where we fill in the tombs members of entries in cmap_sym, (which
260 * must alread exist) used later to write the output encoding.
261 */
262 static void
add_charmap_impl(char * sym,char * mbs,int mbs_len,int nodups)263 add_charmap_impl(char *sym, char *mbs, int mbs_len, int nodups)
264 {
265
266 /*
267 * While parsing both the "from" and "to" cmaps,
268 * require both the symbol and encoding.
269 */
270 if (sym == NULL || mbs == NULL) {
271 errf(_("invalid charmap entry"));
272 return;
273 }
274
275 switch (cmap_pass) {
276 case CMAP_PASS_FROM:
277 add_charmap_impl_fr(sym, mbs, mbs_len, nodups);
278 break;
279 case CMAP_PASS_TO:
280 add_charmap_impl_to(sym, mbs, mbs_len, nodups);
281 break;
282 default:
283 abort();
284 break;
285 }
286 }
287
288 static void
add_charmap_impl_fr(char * sym,char * mbs,int mbs_len,int nodups)289 add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups)
290 {
291 charmap_t *m, *n, *s;
292 avl_index_t where_sym, where_mbs;
293
294 if ((n = calloc(1, sizeof (*n))) == NULL) {
295 errf(_("out of memory"));
296 return;
297 }
298 n->cm_name = sym;
299
300 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
301 (void) memcpy(n->cm_frmbs, mbs, mbs_len);
302 n->cm_frmbs_len = mbs_len;
303
304 m = avl_find(&cmap_mbs, n, &where_mbs);
305 s = avl_find(&cmap_sym, n, &where_sym);
306
307 /*
308 * If we found the symbol, this is a dup.
309 */
310 if (s != NULL) {
311 if (nodups) {
312 warn(_("%s: duplicate character symbol"), sym);
313 }
314 free(n);
315 return;
316 }
317
318 /*
319 * If we found the mbs, the new one is an alias,
320 * which we'll add _only_ to the symbol AVL.
321 */
322 if (m != NULL) {
323 /* The new one is an alias of the original. */
324 n->cm_alias_of = m;
325 avl_insert(&cmap_sym, n, where_sym);
326 return;
327 }
328
329 avl_insert(&cmap_sym, n, where_sym);
330 avl_insert(&cmap_mbs, n, where_mbs);
331 }
332
333 static void
add_charmap_impl_to(char * sym,char * mbs,int mbs_len,int nodups)334 add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups)
335 {
336 charmap_t srch = {0};
337 charmap_t *m;
338
339 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
340
341 srch.cm_name = sym;
342
343 m = avl_find(&cmap_sym, &srch, NULL);
344 if (m == NULL) {
345 if (sflag == 0)
346 warn(_("%s: symbol not found"), sym);
347 return;
348 }
349 if (m->cm_alias_of != NULL) {
350 m = m->cm_alias_of;
351
352 /* don't warn for dups with aliases */
353 if (m->cm_tombs_len != 0)
354 return;
355 }
356
357 if (m->cm_tombs_len != 0) {
358 if (nodups) {
359 warn(_("%s: duplicate encoding for"), sym);
360 }
361 return;
362 }
363
364 (void) memcpy(m->cm_tombs, mbs, mbs_len);
365 m->cm_tombs_len = mbs_len;
366 }
367
368 void
add_charmap(char * sym,char * mbs)369 add_charmap(char *sym, char *mbs)
370 {
371 /* mbs[0] is the length */
372 int mbs_len = *mbs++;
373 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
374 add_charmap_impl(sym, mbs, mbs_len, 1);
375 }
376
377
378 /*
379 * This is called by the parser with start/end symbol strings (ssym, esym),
380 * which are allocated in the scanner (T_SYMBOL) and free'd here.
381 */
382 void
add_charmap_range(char * ssym,char * esym,char * mbs)383 add_charmap_range(char *ssym, char *esym, char *mbs)
384 {
385 int ls, le;
386 int si;
387 int sn, en;
388 int i;
389 int mbs_len;
390 char tmbs[MB_LEN_MAX+1];
391 char *mb_last;
392
393 static const char *digits = "0123456789";
394
395 /* mbs[0] is the length */
396 mbs_len = *mbs++;
397 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
398 (void) memcpy(tmbs, mbs, mbs_len);
399 mb_last = tmbs + mbs_len - 1;
400
401 ls = strlen(ssym);
402 le = strlen(esym);
403
404 if (((si = strcspn(ssym, digits)) == 0) || (si == ls) ||
405 (strncmp(ssym, esym, si) != 0) ||
406 (strspn(ssym + si, digits) != (ls - si)) ||
407 (strspn(esym + si, digits) != (le - si)) ||
408 ((sn = atoi(ssym + si)) > ((en = atoi(esym + si))))) {
409 errf(_("malformed charmap range"));
410 return;
411 }
412
413 ssym[si] = 0;
414 for (i = sn; i <= en; i++) {
415 char *nn;
416 (void) asprintf(&nn, "%s%0*u", ssym, ls - si, i);
417 if (nn == NULL) {
418 errf(_("out of memory"));
419 return;
420 }
421
422 add_charmap_impl(nn, tmbs, mbs_len, 1);
423 (*mb_last)++;
424 }
425 free(ssym);
426 free(esym);
427 }
428
429 void
add_charmap_char(char * name,int c)430 add_charmap_char(char *name, int c)
431 {
432 char mbs[MB_LEN_MAX+1];
433
434 mbs[0] = c;
435 mbs[1] = '\0';
436 add_charmap_impl(name, mbs, 1, 0);
437 }
438
439 /*
440 * POSIX insists that certain entries be present, even when not in the
441 * orginal charmap file.
442 */
443 void
add_charmap_posix(void)444 add_charmap_posix(void)
445 {
446 int i;
447
448 for (i = 0; portable_chars[i].name; i++) {
449 add_charmap_char(portable_chars[i].name, portable_chars[i].ch);
450 }
451 }
452
453 /*
454 * This is called with a buffer of (typically) MB_LEN_MAX bytes,
455 * which is potentially a multi-byte symbol, but often contains
456 * extra bytes. Find and return the longest match in the charmap.
457 */
458 static charmap_t *
find_mbs(const char * mbs,int len)459 find_mbs(const char *mbs, int len)
460 {
461 charmap_t srch = {0};
462 charmap_t *cm = NULL;
463
464 while (len > 0) {
465 (void) memcpy(srch.cm_frmbs, mbs, len);
466 srch.cm_frmbs_len = len;
467 cm = avl_find(&cmap_mbs, &srch, NULL);
468 if (cm != NULL)
469 break;
470 len--;
471 }
472
473 return (cm);
474 }
475
476 /*
477 * Return true if this sequence matches the initial part
478 * of any sequence known in this charmap.
479 */
480 static boolean_t
find_mbs_partial(const char * mbs,int len)481 find_mbs_partial(const char *mbs, int len)
482 {
483 charmap_t srch = {0};
484 charmap_t *cm;
485 avl_index_t where;
486
487 (void) memcpy(srch.cm_frmbs, mbs, len);
488 srch.cm_frmbs_len = len;
489 cm = avl_find(&cmap_mbs, &srch, &where);
490 if (cm != NULL) {
491 /* full match - not expected, but OK */
492 return (B_TRUE);
493 }
494 cm = avl_nearest(&cmap_mbs, where, AVL_AFTER);
495 if (cm != NULL && 0 == memcmp(cm->cm_frmbs, mbs, len))
496 return (B_TRUE);
497
498 return (B_FALSE);
499 }
500
501 /*
502 * Do like iconv(3), but with charmaps.
503 */
504 size_t
cm_iconv(const char ** iptr,size_t * ileft,char ** optr,size_t * oleft)505 cm_iconv(const char **iptr, size_t *ileft, char **optr, size_t *oleft)
506 {
507 charmap_t *cm;
508 int mbs_len;
509
510 /* Ignore state reset requests. */
511 if (iptr == NULL || *iptr == NULL)
512 return (0);
513
514 if (*oleft < MB_LEN_MAX) {
515 errno = E2BIG;
516 return ((size_t)-1);
517 }
518
519 while (*ileft > 0 && *oleft >= MB_LEN_MAX) {
520 mbs_len = MB_LEN_MAX;
521 if (mbs_len > *ileft)
522 mbs_len = *ileft;
523 cm = find_mbs(*iptr, mbs_len);
524 if (cm == NULL) {
525 if (mbs_len < MB_LEN_MAX &&
526 find_mbs_partial(*iptr, mbs_len)) {
527 /* incomplete sequence */
528 errno = EINVAL;
529 } else {
530 errno = EILSEQ;
531 }
532 return ((size_t)-1);
533 }
534 assert(cm->cm_frmbs_len > 0);
535 if (cm->cm_tombs_len == 0) {
536 if (sflag == 0 && cm->cm_warned == 0) {
537 cm->cm_warned = 1;
538 warn(_("To-map does not encode <%s>\n"),
539 cm->cm_name);
540 }
541 if (cflag == 0) {
542 errno = EILSEQ;
543 return ((size_t)-1);
544 }
545 /* just skip this input seq. */
546 *iptr += cm->cm_frmbs_len;
547 *ileft -= cm->cm_frmbs_len;
548 continue;
549 }
550
551 *iptr += cm->cm_frmbs_len;
552 *ileft -= cm->cm_frmbs_len;
553 (void) memcpy(*optr, cm->cm_tombs, cm->cm_tombs_len);
554 *optr += cm->cm_tombs_len;
555 *oleft -= cm->cm_tombs_len;
556 }
557
558 return (0);
559 }
560