1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 14 */ 15 16 /* 17 * CHARMAP file handling for iconv. 18 */ 19 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <errno.h> 24 #include <limits.h> 25 #include <unistd.h> 26 #include <alloca.h> 27 #include <sys/avl.h> 28 #include <stddef.h> 29 #include <unistd.h> 30 #include "charmap.h" 31 #include "parser.tab.h" 32 #include <assert.h> 33 34 enum cmap_pass cmap_pass; 35 static avl_tree_t cmap_sym; 36 static avl_tree_t cmap_mbs; 37 38 typedef struct charmap { 39 const char *cm_name; 40 struct charmap *cm_alias_of; 41 avl_node_t cm_avl_sym; 42 avl_node_t cm_avl_mbs; 43 int cm_warned; 44 int cm_frmbs_len; 45 int cm_tombs_len; 46 char cm_frmbs[MB_LEN_MAX + 1]; /* input */ 47 char cm_tombs[MB_LEN_MAX + 1]; /* output */ 48 } charmap_t; 49 50 static void add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups); 51 static void add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups); 52 53 /* 54 * Array of POSIX specific portable characters. 55 */ 56 static const struct { 57 char *name; 58 int ch; 59 } portable_chars[] = { 60 { "NUL", '\0' }, 61 { "alert", '\a' }, 62 { "backspace", '\b' }, 63 { "tab", '\t' }, 64 { "carriage-return", '\r' }, 65 { "newline", '\n' }, 66 { "vertical-tab", '\v' }, 67 { "form-feed", '\f' }, 68 { "space", ' ' }, 69 { "exclamation-mark", '!' }, 70 { "quotation-mark", '"' }, 71 { "number-sign", '#' }, 72 { "dollar-sign", '$' }, 73 { "percent-sign", '%' }, 74 { "ampersand", '&' }, 75 { "apostrophe", '\'' }, 76 { "left-parenthesis", '(' }, 77 { "right-parenthesis", '(' }, 78 { "asterisk", '*' }, 79 { "plus-sign", '+' }, 80 { "comma", ','}, 81 { "hyphen-minus", '-' }, 82 { "hyphen", '-' }, 83 { "full-stop", '.' }, 84 { "period", '.' }, 85 { "slash", '/' }, 86 { "solidus", '/' }, 87 { "zero", '0' }, 88 { "one", '1' }, 89 { "two", '2' }, 90 { "three", '3' }, 91 { "four", '4' }, 92 { "five", '5' }, 93 { "six", '6' }, 94 { "seven", '7' }, 95 { "eight", '8' }, 96 { "nine", '9' }, 97 { "colon", ':' }, 98 { "semicolon", ';' }, 99 { "less-than-sign", '<' }, 100 { "equals-sign", '=' }, 101 { "greater-than-sign", '>' }, 102 { "question-mark", '?' }, 103 { "commercial-at", '@' }, 104 { "left-square-bracket", '[' }, 105 { "backslash", '\\' }, 106 { "reverse-solidus", '\\' }, 107 { "right-square-bracket", ']' }, 108 { "circumflex", '^' }, 109 { "circumflex-accent", '^' }, 110 { "low-line", '_' }, 111 { "underscore", '_' }, 112 { "grave-accent", '`' }, 113 { "left-brace", '{' }, 114 { "left-curly-bracket", '{' }, 115 { "vertical-line", '|' }, 116 { "right-brace", '}' }, 117 { "right-curly-bracket", '}' }, 118 { "tilde", '~' }, 119 { "A", 'A' }, 120 { "B", 'B' }, 121 { "C", 'C' }, 122 { "D", 'D' }, 123 { "E", 'E' }, 124 { "F", 'F' }, 125 { "G", 'G' }, 126 { "H", 'H' }, 127 { "I", 'I' }, 128 { "J", 'J' }, 129 { "K", 'K' }, 130 { "L", 'L' }, 131 { "M", 'M' }, 132 { "N", 'N' }, 133 { "O", 'O' }, 134 { "P", 'P' }, 135 { "Q", 'Q' }, 136 { "R", 'R' }, 137 { "S", 'S' }, 138 { "T", 'T' }, 139 { "U", 'U' }, 140 { "V", 'V' }, 141 { "W", 'W' }, 142 { "X", 'X' }, 143 { "Y", 'Y' }, 144 { "Z", 'Z' }, 145 { "a", 'a' }, 146 { "b", 'b' }, 147 { "c", 'c' }, 148 { "d", 'd' }, 149 { "e", 'e' }, 150 { "f", 'f' }, 151 { "g", 'g' }, 152 { "h", 'h' }, 153 { "i", 'i' }, 154 { "j", 'j' }, 155 { "k", 'k' }, 156 { "l", 'l' }, 157 { "m", 'm' }, 158 { "n", 'n' }, 159 { "o", 'o' }, 160 { "p", 'p' }, 161 { "q", 'q' }, 162 { "r", 'r' }, 163 { "s", 's' }, 164 { "t", 't' }, 165 { "u", 'u' }, 166 { "v", 'v' }, 167 { "w", 'w' }, 168 { "x", 'x' }, 169 { "y", 'y' }, 170 { "z", 'z' }, 171 { NULL, 0 } 172 }; 173 174 static int 175 cmap_compare_sym(const void *n1, const void *n2) 176 { 177 const charmap_t *c1 = n1; 178 const charmap_t *c2 = n2; 179 int rv; 180 181 rv = strcmp(c1->cm_name, c2->cm_name); 182 return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0); 183 } 184 185 /* 186 * In order for partial match searches to work, 187 * we need these sorted by mbs contents. 188 */ 189 static int 190 cmap_compare_mbs(const void *n1, const void *n2) 191 { 192 const charmap_t *c1 = n1; 193 const charmap_t *c2 = n2; 194 int len, rv; 195 196 len = c1->cm_frmbs_len; 197 if (len < c2->cm_frmbs_len) 198 len = c2->cm_frmbs_len; 199 rv = memcmp(c1->cm_frmbs, c2->cm_frmbs, len); 200 if (rv < 0) 201 return (-1); 202 if (rv > 0) 203 return (1); 204 /* they match through length */ 205 if (c1->cm_frmbs_len < c2->cm_frmbs_len) 206 return (-1); 207 if (c2->cm_frmbs_len < c1->cm_frmbs_len) 208 return (1); 209 return (0); 210 } 211 212 void 213 charmap_init(char *to_map, char *from_map) 214 { 215 avl_create(&cmap_sym, cmap_compare_sym, sizeof (charmap_t), 216 offsetof(charmap_t, cm_avl_sym)); 217 218 avl_create(&cmap_mbs, cmap_compare_mbs, sizeof (charmap_t), 219 offsetof(charmap_t, cm_avl_mbs)); 220 221 cmap_pass = CMAP_PASS_FROM; 222 reset_scanner(from_map); 223 (void) yyparse(); 224 add_charmap_posix(); 225 226 cmap_pass = CMAP_PASS_TO; 227 reset_scanner(to_map); 228 (void) yyparse(); 229 } 230 231 void 232 charmap_dump() 233 { 234 charmap_t *cm; 235 int i; 236 237 cm = avl_first(&cmap_mbs); 238 while (cm != NULL) { 239 (void) printf("name=\"%s\"\n", cm->cm_name); 240 241 (void) printf("\timbs=\""); 242 for (i = 0; i < cm->cm_frmbs_len; i++) 243 (void) printf("\\x%02x", cm->cm_frmbs[i] & 0xFF); 244 (void) printf("\"\n"); 245 246 (void) printf("\tombs=\""); 247 for (i = 0; i < cm->cm_tombs_len; i++) 248 (void) printf("\\x%02x", cm->cm_tombs[i] & 0xFF); 249 (void) printf("\"\n"); 250 251 cm = AVL_NEXT(&cmap_mbs, cm); 252 } 253 } 254 255 /* 256 * We parse two charmap files: First the "from" map, where we build 257 * cmap_mbs and cmap_sym which we'll later use to translate the input 258 * stream (mbs encodings) to symbols. Second, we parse the "to" map, 259 * where we fill in the tombs members of entries in cmap_sym, (which 260 * must alread exist) used later to write the output encoding. 261 */ 262 static void 263 add_charmap_impl(char *sym, char *mbs, int mbs_len, int nodups) 264 { 265 266 /* 267 * While parsing both the "from" and "to" cmaps, 268 * require both the symbol and encoding. 269 */ 270 if (sym == NULL || mbs == NULL) { 271 errf(_("invalid charmap entry")); 272 return; 273 } 274 275 switch (cmap_pass) { 276 case CMAP_PASS_FROM: 277 add_charmap_impl_fr(sym, mbs, mbs_len, nodups); 278 break; 279 case CMAP_PASS_TO: 280 add_charmap_impl_to(sym, mbs, mbs_len, nodups); 281 break; 282 default: 283 abort(); 284 break; 285 } 286 } 287 288 static void 289 add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups) 290 { 291 charmap_t *m, *n, *s; 292 avl_index_t where_sym, where_mbs; 293 294 if ((n = calloc(1, sizeof (*n))) == NULL) { 295 errf(_("out of memory")); 296 return; 297 } 298 n->cm_name = sym; 299 300 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX); 301 (void) memcpy(n->cm_frmbs, mbs, mbs_len); 302 n->cm_frmbs_len = mbs_len; 303 304 m = avl_find(&cmap_mbs, n, &where_mbs); 305 s = avl_find(&cmap_sym, n, &where_sym); 306 307 /* 308 * If we found the symbol, this is a dup. 309 */ 310 if (s != NULL) { 311 if (nodups) { 312 warn(_("%s: duplicate character symbol"), sym); 313 } 314 free(n); 315 return; 316 } 317 318 /* 319 * If we found the mbs, the new one is an alias, 320 * which we'll add _only_ to the symbol AVL. 321 */ 322 if (m != NULL) { 323 /* The new one is an alias of the original. */ 324 n->cm_alias_of = m; 325 avl_insert(&cmap_sym, n, where_sym); 326 return; 327 } 328 329 avl_insert(&cmap_sym, n, where_sym); 330 avl_insert(&cmap_mbs, n, where_mbs); 331 } 332 333 static void 334 add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups) 335 { 336 charmap_t srch = {0}; 337 charmap_t *m; 338 339 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX); 340 341 srch.cm_name = sym; 342 343 m = avl_find(&cmap_sym, &srch, NULL); 344 if (m == NULL) { 345 if (sflag == 0) 346 warn(_("%s: symbol not found"), sym); 347 return; 348 } 349 if (m->cm_alias_of != NULL) { 350 m = m->cm_alias_of; 351 352 /* don't warn for dups with aliases */ 353 if (m->cm_tombs_len != 0) 354 return; 355 } 356 357 if (m->cm_tombs_len != 0) { 358 if (nodups) { 359 warn(_("%s: duplicate encoding for"), sym); 360 } 361 return; 362 } 363 364 (void) memcpy(m->cm_tombs, mbs, mbs_len); 365 m->cm_tombs_len = mbs_len; 366 } 367 368 void 369 add_charmap(char *sym, char *mbs) 370 { 371 /* mbs[0] is the length */ 372 int mbs_len = *mbs++; 373 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX); 374 add_charmap_impl(sym, mbs, mbs_len, 1); 375 } 376 377 378 /* 379 * This is called by the parser with start/end symbol strings (ssym, esym), 380 * which are allocated in the scanner (T_SYMBOL) and free'd here. 381 */ 382 void 383 add_charmap_range(char *ssym, char *esym, char *mbs) 384 { 385 int ls, le; 386 int si; 387 int sn, en; 388 int i; 389 int mbs_len; 390 char tmbs[MB_LEN_MAX+1]; 391 char *mb_last; 392 393 static const char *digits = "0123456789"; 394 395 /* mbs[0] is the length */ 396 mbs_len = *mbs++; 397 assert(0 < mbs_len && mbs_len <= MB_LEN_MAX); 398 (void) memcpy(tmbs, mbs, mbs_len); 399 mb_last = tmbs + mbs_len - 1; 400 401 ls = strlen(ssym); 402 le = strlen(esym); 403 404 if (((si = strcspn(ssym, digits)) == 0) || (si == ls) || 405 (strncmp(ssym, esym, si) != 0) || 406 (strspn(ssym + si, digits) != (ls - si)) || 407 (strspn(esym + si, digits) != (le - si)) || 408 ((sn = atoi(ssym + si)) > ((en = atoi(esym + si))))) { 409 errf(_("malformed charmap range")); 410 return; 411 } 412 413 ssym[si] = 0; 414 for (i = sn; i <= en; i++) { 415 char *nn; 416 (void) asprintf(&nn, "%s%0*u", ssym, ls - si, i); 417 if (nn == NULL) { 418 errf(_("out of memory")); 419 return; 420 } 421 422 add_charmap_impl(nn, tmbs, mbs_len, 1); 423 (*mb_last)++; 424 } 425 free(ssym); 426 free(esym); 427 } 428 429 void 430 add_charmap_char(char *name, int c) 431 { 432 char mbs[MB_LEN_MAX+1]; 433 434 mbs[0] = c; 435 mbs[1] = '\0'; 436 add_charmap_impl(name, mbs, 1, 0); 437 } 438 439 /* 440 * POSIX insists that certain entries be present, even when not in the 441 * orginal charmap file. 442 */ 443 void 444 add_charmap_posix(void) 445 { 446 int i; 447 448 for (i = 0; portable_chars[i].name; i++) { 449 add_charmap_char(portable_chars[i].name, portable_chars[i].ch); 450 } 451 } 452 453 /* 454 * This is called with a buffer of (typically) MB_LEN_MAX bytes, 455 * which is potentially a multi-byte symbol, but often contains 456 * extra bytes. Find and return the longest match in the charmap. 457 */ 458 static charmap_t * 459 find_mbs(const char *mbs, int len) 460 { 461 charmap_t srch = {0}; 462 charmap_t *cm = NULL; 463 464 while (len > 0) { 465 (void) memcpy(srch.cm_frmbs, mbs, len); 466 srch.cm_frmbs_len = len; 467 cm = avl_find(&cmap_mbs, &srch, NULL); 468 if (cm != NULL) 469 break; 470 len--; 471 } 472 473 return (cm); 474 } 475 476 /* 477 * Return true if this sequence matches the initial part 478 * of any sequence known in this charmap. 479 */ 480 static boolean_t 481 find_mbs_partial(const char *mbs, int len) 482 { 483 charmap_t srch = {0}; 484 charmap_t *cm; 485 avl_index_t where; 486 487 (void) memcpy(srch.cm_frmbs, mbs, len); 488 srch.cm_frmbs_len = len; 489 cm = avl_find(&cmap_mbs, &srch, &where); 490 if (cm != NULL) { 491 /* full match - not expected, but OK */ 492 return (B_TRUE); 493 } 494 cm = avl_nearest(&cmap_mbs, where, AVL_AFTER); 495 if (cm != NULL && 0 == memcmp(cm->cm_frmbs, mbs, len)) 496 return (B_TRUE); 497 498 return (B_FALSE); 499 } 500 501 /* 502 * Do like iconv(3), but with charmaps. 503 */ 504 size_t 505 cm_iconv(const char **iptr, size_t *ileft, char **optr, size_t *oleft) 506 { 507 charmap_t *cm; 508 int mbs_len; 509 510 /* Ignore state reset requests. */ 511 if (iptr == NULL || *iptr == NULL) 512 return (0); 513 514 if (*oleft < MB_LEN_MAX) { 515 errno = E2BIG; 516 return ((size_t)-1); 517 } 518 519 while (*ileft > 0 && *oleft >= MB_LEN_MAX) { 520 mbs_len = MB_LEN_MAX; 521 if (mbs_len > *ileft) 522 mbs_len = *ileft; 523 cm = find_mbs(*iptr, mbs_len); 524 if (cm == NULL) { 525 if (mbs_len < MB_LEN_MAX && 526 find_mbs_partial(*iptr, mbs_len)) { 527 /* incomplete sequence */ 528 errno = EINVAL; 529 } else { 530 errno = EILSEQ; 531 } 532 return ((size_t)-1); 533 } 534 assert(cm->cm_frmbs_len > 0); 535 if (cm->cm_tombs_len == 0) { 536 if (sflag == 0 && cm->cm_warned == 0) { 537 cm->cm_warned = 1; 538 warn(_("To-map does not encode <%s>\n"), 539 cm->cm_name); 540 } 541 if (cflag == 0) { 542 errno = EILSEQ; 543 return ((size_t)-1); 544 } 545 /* just skip this input seq. */ 546 *iptr += cm->cm_frmbs_len; 547 *ileft -= cm->cm_frmbs_len; 548 continue; 549 } 550 551 *iptr += cm->cm_frmbs_len; 552 *ileft -= cm->cm_frmbs_len; 553 (void) memcpy(*optr, cm->cm_tombs, cm->cm_tombs_len); 554 *optr += cm->cm_tombs_len; 555 *oleft -= cm->cm_tombs_len; 556 } 557 558 return (0); 559 } 560