1 /*- 2 * Copyright (c) 2003-2011 Tim Kientzle 3 * Copyright (c) 2011-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 __FBSDID("$FreeBSD$"); 29 30 /* 31 * Basic resizable string support, to simplify manipulating arbitrary-sized 32 * strings while minimizing heap activity. 33 * 34 * In particular, the buffer used by a string object is only grown, it 35 * never shrinks, so you can clear and reuse the same string object 36 * without incurring additional memory allocations. 37 */ 38 39 #ifdef HAVE_ERRNO_H 40 #include <errno.h> 41 #endif 42 #ifdef HAVE_ICONV_H 43 #include <iconv.h> 44 #endif 45 #ifdef HAVE_LANGINFO_H 46 #include <langinfo.h> 47 #endif 48 #ifdef HAVE_LOCALCHARSET_H 49 #include <localcharset.h> 50 #endif 51 #ifdef HAVE_STDLIB_H 52 #include <stdlib.h> 53 #endif 54 #ifdef HAVE_STRING_H 55 #include <string.h> 56 #endif 57 #ifdef HAVE_WCHAR_H 58 #include <wchar.h> 59 #endif 60 #if defined(_WIN32) && !defined(__CYGWIN__) 61 #include <windows.h> 62 #include <locale.h> 63 #endif 64 65 #include "archive_endian.h" 66 #include "archive_private.h" 67 #include "archive_string.h" 68 #include "archive_string_composition.h" 69 70 #if !defined(HAVE_WMEMCPY) && !defined(wmemcpy) 71 #define wmemcpy(a,b,i) (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t)) 72 #endif 73 74 struct archive_string_conv { 75 struct archive_string_conv *next; 76 char *from_charset; 77 char *to_charset; 78 unsigned from_cp; 79 unsigned to_cp; 80 /* Set 1 if from_charset and to_charset are the same. */ 81 int same; 82 int flag; 83 #define SCONV_TO_CHARSET 1 /* MBS is being converted to specified 84 * charset. */ 85 #define SCONV_FROM_CHARSET (1<<1) /* MBS is being converted from 86 * specified charset. */ 87 #define SCONV_BEST_EFFORT (1<<2) /* Copy at least ASCII code. */ 88 #define SCONV_WIN_CP (1<<3) /* Use Windows API for converting 89 * MBS. */ 90 #define SCONV_UTF8_LIBARCHIVE_2 (1<<4) /* Incorrect UTF-8 made by libarchive 91 * 2.x in the wrong assumption. */ 92 #define SCONV_NORMALIZATION_C (1<<6) /* Need normalization to be Form C. 93 * Before UTF-8 characters are actually 94 * processed. */ 95 #define SCONV_NORMALIZATION_D (1<<7) /* Need normalization to be Form D. 96 * Before UTF-8 characters are actually 97 * processed. 98 * Currently this only for MAC OS X. */ 99 #define SCONV_TO_UTF8 (1<<8) /* "to charset" side is UTF-8. */ 100 #define SCONV_FROM_UTF8 (1<<9) /* "from charset" side is UTF-8. */ 101 #define SCONV_TO_UTF16BE (1<<10) /* "to charset" side is UTF-16BE. */ 102 #define SCONV_FROM_UTF16BE (1<<11) /* "from charset" side is UTF-16BE. */ 103 #define SCONV_TO_UTF16LE (1<<12) /* "to charset" side is UTF-16LE. */ 104 #define SCONV_FROM_UTF16LE (1<<13) /* "from charset" side is UTF-16LE. */ 105 #define SCONV_TO_UTF16 (SCONV_TO_UTF16BE | SCONV_TO_UTF16LE) 106 #define SCONV_FROM_UTF16 (SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE) 107 108 #if HAVE_ICONV 109 iconv_t cd; 110 iconv_t cd_w;/* Use at archive_mstring on 111 * Windows. */ 112 #endif 113 /* A temporary buffer for normalization. */ 114 struct archive_string utftmp; 115 int (*converter[2])(struct archive_string *, const void *, size_t, 116 struct archive_string_conv *); 117 int nconverter; 118 }; 119 120 #define CP_C_LOCALE 0 /* "C" locale only for this file. */ 121 #define CP_UTF16LE 1200 122 #define CP_UTF16BE 1201 123 124 #define IS_HIGH_SURROGATE_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDBFF) 125 #define IS_LOW_SURROGATE_LA(uc) ((uc) >= 0xDC00 && (uc) <= 0xDFFF) 126 #define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF) 127 #define UNICODE_MAX 0x10FFFF 128 #define UNICODE_R_CHAR 0xFFFD /* Replacement character. */ 129 /* Set U+FFFD(Replacement character) in UTF-8. */ 130 #define UTF8_SET_R_CHAR(outp) do { \ 131 (outp)[0] = 0xef; \ 132 (outp)[1] = 0xbf; \ 133 (outp)[2] = 0xbd; \ 134 } while (0) 135 #define UTF8_R_CHAR_SIZE 3 136 137 static struct archive_string_conv *find_sconv_object(struct archive *, 138 const char *, const char *); 139 static void add_sconv_object(struct archive *, struct archive_string_conv *); 140 static struct archive_string_conv *create_sconv_object(const char *, 141 const char *, unsigned, int); 142 static void free_sconv_object(struct archive_string_conv *); 143 static struct archive_string_conv *get_sconv_object(struct archive *, 144 const char *, const char *, int); 145 static unsigned make_codepage_from_charset(const char *); 146 static unsigned get_current_codepage(void); 147 static unsigned get_current_oemcp(void); 148 static size_t mbsnbytes(const void *, size_t); 149 static size_t utf16nbytes(const void *, size_t); 150 #if defined(_WIN32) && !defined(__CYGWIN__) 151 static int archive_wstring_append_from_mbs_in_codepage( 152 struct archive_wstring *, const char *, size_t, 153 struct archive_string_conv *); 154 static int archive_string_append_from_wcs_in_codepage(struct archive_string *, 155 const wchar_t *, size_t, struct archive_string_conv *); 156 static int is_big_endian(void); 157 static int strncat_in_codepage(struct archive_string *, const void *, 158 size_t, struct archive_string_conv *); 159 static int win_strncat_from_utf16be(struct archive_string *, const void *, 160 size_t, struct archive_string_conv *); 161 static int win_strncat_from_utf16le(struct archive_string *, const void *, 162 size_t, struct archive_string_conv *); 163 static int win_strncat_to_utf16be(struct archive_string *, const void *, 164 size_t, struct archive_string_conv *); 165 static int win_strncat_to_utf16le(struct archive_string *, const void *, 166 size_t, struct archive_string_conv *); 167 #endif 168 static int best_effort_strncat_from_utf16be(struct archive_string *, 169 const void *, size_t, struct archive_string_conv *); 170 static int best_effort_strncat_from_utf16le(struct archive_string *, 171 const void *, size_t, struct archive_string_conv *); 172 static int best_effort_strncat_to_utf16be(struct archive_string *, 173 const void *, size_t, struct archive_string_conv *); 174 static int best_effort_strncat_to_utf16le(struct archive_string *, 175 const void *, size_t, struct archive_string_conv *); 176 #if defined(HAVE_ICONV) 177 static int iconv_strncat_in_locale(struct archive_string *, const void *, 178 size_t, struct archive_string_conv *); 179 #endif 180 static int best_effort_strncat_in_locale(struct archive_string *, 181 const void *, size_t, struct archive_string_conv *); 182 static int _utf8_to_unicode(uint32_t *, const char *, size_t); 183 static int utf8_to_unicode(uint32_t *, const char *, size_t); 184 static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t); 185 static int cesu8_to_unicode(uint32_t *, const char *, size_t); 186 static size_t unicode_to_utf8(char *, size_t, uint32_t); 187 static int utf16_to_unicode(uint32_t *, const char *, size_t, int); 188 static size_t unicode_to_utf16be(char *, size_t, uint32_t); 189 static size_t unicode_to_utf16le(char *, size_t, uint32_t); 190 static int strncat_from_utf8_libarchive2(struct archive_string *, 191 const void *, size_t, struct archive_string_conv *); 192 static int strncat_from_utf8_to_utf8(struct archive_string *, const void *, 193 size_t, struct archive_string_conv *); 194 static int archive_string_normalize_C(struct archive_string *, const void *, 195 size_t, struct archive_string_conv *); 196 static int archive_string_normalize_D(struct archive_string *, const void *, 197 size_t, struct archive_string_conv *); 198 static int archive_string_append_unicode(struct archive_string *, 199 const void *, size_t, struct archive_string_conv *); 200 201 static struct archive_string * 202 archive_string_append(struct archive_string *as, const char *p, size_t s) 203 { 204 if (archive_string_ensure(as, as->length + s + 1) == NULL) 205 return (NULL); 206 memcpy(as->s + as->length, p, s); 207 as->length += s; 208 as->s[as->length] = 0; 209 return (as); 210 } 211 212 static struct archive_wstring * 213 archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) 214 { 215 if (archive_wstring_ensure(as, as->length + s + 1) == NULL) 216 return (NULL); 217 wmemcpy(as->s + as->length, p, s); 218 as->length += s; 219 as->s[as->length] = 0; 220 return (as); 221 } 222 223 void 224 archive_string_concat(struct archive_string *dest, struct archive_string *src) 225 { 226 if (archive_string_append(dest, src->s, src->length) == NULL) 227 __archive_errx(1, "Out of memory"); 228 } 229 230 void 231 archive_wstring_concat(struct archive_wstring *dest, 232 struct archive_wstring *src) 233 { 234 if (archive_wstring_append(dest, src->s, src->length) == NULL) 235 __archive_errx(1, "Out of memory"); 236 } 237 238 void 239 archive_string_free(struct archive_string *as) 240 { 241 as->length = 0; 242 as->buffer_length = 0; 243 free(as->s); 244 as->s = NULL; 245 } 246 247 void 248 archive_wstring_free(struct archive_wstring *as) 249 { 250 as->length = 0; 251 as->buffer_length = 0; 252 free(as->s); 253 as->s = NULL; 254 } 255 256 struct archive_wstring * 257 archive_wstring_ensure(struct archive_wstring *as, size_t s) 258 { 259 return (struct archive_wstring *) 260 archive_string_ensure((struct archive_string *)as, 261 s * sizeof(wchar_t)); 262 } 263 264 /* Returns NULL on any allocation failure. */ 265 struct archive_string * 266 archive_string_ensure(struct archive_string *as, size_t s) 267 { 268 char *p; 269 size_t new_length; 270 271 /* If buffer is already big enough, don't reallocate. */ 272 if (as->s && (s <= as->buffer_length)) 273 return (as); 274 275 /* 276 * Growing the buffer at least exponentially ensures that 277 * append operations are always linear in the number of 278 * characters appended. Using a smaller growth rate for 279 * larger buffers reduces memory waste somewhat at the cost of 280 * a larger constant factor. 281 */ 282 if (as->buffer_length < 32) 283 /* Start with a minimum 32-character buffer. */ 284 new_length = 32; 285 else if (as->buffer_length < 8192) 286 /* Buffers under 8k are doubled for speed. */ 287 new_length = as->buffer_length + as->buffer_length; 288 else { 289 /* Buffers 8k and over grow by at least 25% each time. */ 290 new_length = as->buffer_length + as->buffer_length / 4; 291 /* Be safe: If size wraps, fail. */ 292 if (new_length < as->buffer_length) { 293 /* On failure, wipe the string and return NULL. */ 294 archive_string_free(as); 295 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 296 return (NULL); 297 } 298 } 299 /* 300 * The computation above is a lower limit to how much we'll 301 * grow the buffer. In any case, we have to grow it enough to 302 * hold the request. 303 */ 304 if (new_length < s) 305 new_length = s; 306 /* Now we can reallocate the buffer. */ 307 p = (char *)realloc(as->s, new_length); 308 if (p == NULL) { 309 /* On failure, wipe the string and return NULL. */ 310 archive_string_free(as); 311 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 312 return (NULL); 313 } 314 315 as->s = p; 316 as->buffer_length = new_length; 317 return (as); 318 } 319 320 /* 321 * TODO: See if there's a way to avoid scanning 322 * the source string twice. Then test to see 323 * if it actually helps (remember that we're almost 324 * always called with pretty short arguments, so 325 * such an optimization might not help). 326 */ 327 struct archive_string * 328 archive_strncat(struct archive_string *as, const void *_p, size_t n) 329 { 330 size_t s; 331 const char *p, *pp; 332 333 p = (const char *)_p; 334 335 /* Like strlen(p), except won't examine positions beyond p[n]. */ 336 s = 0; 337 pp = p; 338 while (s < n && *pp) { 339 pp++; 340 s++; 341 } 342 if ((as = archive_string_append(as, p, s)) == NULL) 343 __archive_errx(1, "Out of memory"); 344 return (as); 345 } 346 347 struct archive_wstring * 348 archive_wstrncat(struct archive_wstring *as, const wchar_t *p, size_t n) 349 { 350 size_t s; 351 const wchar_t *pp; 352 353 /* Like strlen(p), except won't examine positions beyond p[n]. */ 354 s = 0; 355 pp = p; 356 while (s < n && *pp) { 357 pp++; 358 s++; 359 } 360 if ((as = archive_wstring_append(as, p, s)) == NULL) 361 __archive_errx(1, "Out of memory"); 362 return (as); 363 } 364 365 struct archive_string * 366 archive_strcat(struct archive_string *as, const void *p) 367 { 368 /* strcat is just strncat without an effective limit. 369 * Assert that we'll never get called with a source 370 * string over 16MB. 371 * TODO: Review all uses of strcat in the source 372 * and try to replace them with strncat(). 373 */ 374 return archive_strncat(as, p, 0x1000000); 375 } 376 377 struct archive_wstring * 378 archive_wstrcat(struct archive_wstring *as, const wchar_t *p) 379 { 380 /* Ditto. */ 381 return archive_wstrncat(as, p, 0x1000000); 382 } 383 384 struct archive_string * 385 archive_strappend_char(struct archive_string *as, char c) 386 { 387 if ((as = archive_string_append(as, &c, 1)) == NULL) 388 __archive_errx(1, "Out of memory"); 389 return (as); 390 } 391 392 struct archive_wstring * 393 archive_wstrappend_wchar(struct archive_wstring *as, wchar_t c) 394 { 395 if ((as = archive_wstring_append(as, &c, 1)) == NULL) 396 __archive_errx(1, "Out of memory"); 397 return (as); 398 } 399 400 /* 401 * Get the "current character set" name to use with iconv. 402 * On FreeBSD, the empty character set name "" chooses 403 * the correct character encoding for the current locale, 404 * so this isn't necessary. 405 * But iconv on Mac OS 10.6 doesn't seem to handle this correctly; 406 * on that system, we have to explicitly call nl_langinfo() 407 * to get the right name. Not sure about other platforms. 408 * 409 * NOTE: GNU libiconv does not recognize the character-set name 410 * which some platform nl_langinfo(CODESET) returns, so we should 411 * use locale_charset() instead of nl_langinfo(CODESET) for GNU libiconv. 412 */ 413 static const char * 414 default_iconv_charset(const char *charset) { 415 if (charset != NULL && charset[0] != '\0') 416 return charset; 417 #if HAVE_LOCALE_CHARSET && !defined(__APPLE__) 418 /* locale_charset() is broken on Mac OS */ 419 return locale_charset(); 420 #elif HAVE_NL_LANGINFO 421 return nl_langinfo(CODESET); 422 #else 423 return ""; 424 #endif 425 } 426 427 #if defined(_WIN32) && !defined(__CYGWIN__) 428 429 /* 430 * Convert MBS to WCS. 431 * Note: returns -1 if conversion fails. 432 */ 433 int 434 archive_wstring_append_from_mbs(struct archive_wstring *dest, 435 const char *p, size_t len) 436 { 437 return archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL); 438 } 439 440 static int 441 archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest, 442 const char *s, size_t length, struct archive_string_conv *sc) 443 { 444 int count, ret = 0; 445 UINT from_cp; 446 447 if (sc != NULL) 448 from_cp = sc->from_cp; 449 else 450 from_cp = get_current_codepage(); 451 452 if (from_cp == CP_C_LOCALE) { 453 /* 454 * "C" locale special process. 455 */ 456 wchar_t *ws; 457 const unsigned char *mp; 458 459 if (NULL == archive_wstring_ensure(dest, 460 dest->length + length + 1)) 461 return (-1); 462 463 ws = dest->s + dest->length; 464 mp = (const unsigned char *)s; 465 count = 0; 466 while (count < (int)length && *mp) { 467 *ws++ = (wchar_t)*mp++; 468 count++; 469 } 470 } else if (sc != NULL && 471 (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) { 472 /* 473 * Normalize UTF-8 and UTF-16BE and convert it directly 474 * to UTF-16 as wchar_t. 475 */ 476 struct archive_string u16; 477 int saved_flag = sc->flag;/* save current flag. */ 478 479 if (is_big_endian()) 480 sc->flag |= SCONV_TO_UTF16BE; 481 else 482 sc->flag |= SCONV_TO_UTF16LE; 483 484 if (sc->flag & SCONV_FROM_UTF16) { 485 /* 486 * UTF-16BE/LE NFD ===> UTF-16 NFC 487 * UTF-16BE/LE NFC ===> UTF-16 NFD 488 */ 489 count = (int)utf16nbytes(s, length); 490 } else { 491 /* 492 * UTF-8 NFD ===> UTF-16 NFC 493 * UTF-8 NFC ===> UTF-16 NFD 494 */ 495 count = (int)mbsnbytes(s, length); 496 } 497 u16.s = (char *)dest->s; 498 u16.length = dest->length << 1;; 499 u16.buffer_length = dest->buffer_length; 500 if (sc->flag & SCONV_NORMALIZATION_C) 501 ret = archive_string_normalize_C(&u16, s, count, sc); 502 else 503 ret = archive_string_normalize_D(&u16, s, count, sc); 504 dest->s = (wchar_t *)u16.s; 505 dest->length = u16.length >> 1; 506 dest->buffer_length = u16.buffer_length; 507 sc->flag = saved_flag;/* restore the saved flag. */ 508 return (ret); 509 } else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) { 510 count = (int)utf16nbytes(s, length); 511 count >>= 1; /* to be WCS length */ 512 /* Allocate memory for WCS. */ 513 if (NULL == archive_wstring_ensure(dest, 514 dest->length + count + 1)) 515 return (-1); 516 wmemcpy(dest->s + dest->length, (const wchar_t *)s, count); 517 if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) { 518 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 519 int b; 520 for (b = 0; b < count; b++) { 521 uint16_t val = archive_le16dec(u16+b); 522 archive_be16enc(u16+b, val); 523 } 524 } else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) { 525 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 526 int b; 527 for (b = 0; b < count; b++) { 528 uint16_t val = archive_be16dec(u16+b); 529 archive_le16enc(u16+b, val); 530 } 531 } 532 } else { 533 DWORD mbflag; 534 size_t buffsize; 535 536 if (sc == NULL) 537 mbflag = 0; 538 else if (sc->flag & SCONV_FROM_CHARSET) { 539 /* Do not trust the length which comes from 540 * an archive file. */ 541 length = mbsnbytes(s, length); 542 mbflag = 0; 543 } else 544 mbflag = MB_PRECOMPOSED; 545 546 buffsize = dest->length + length + 1; 547 do { 548 /* Allocate memory for WCS. */ 549 if (NULL == archive_wstring_ensure(dest, buffsize)) 550 return (-1); 551 /* Convert MBS to WCS. */ 552 count = MultiByteToWideChar(from_cp, 553 mbflag, s, (int)length, dest->s + dest->length, 554 (int)(dest->buffer_length >> 1) -1); 555 if (count == 0 && 556 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 557 /* Expand the WCS buffer. */ 558 buffsize = dest->buffer_length << 1; 559 continue; 560 } 561 if (count == 0 && length != 0) 562 ret = -1; 563 } while (0); 564 } 565 dest->length += count; 566 dest->s[dest->length] = L'\0'; 567 return (ret); 568 } 569 570 #else 571 572 /* 573 * Convert MBS to WCS. 574 * Note: returns -1 if conversion fails. 575 */ 576 int 577 archive_wstring_append_from_mbs(struct archive_wstring *dest, 578 const char *p, size_t len) 579 { 580 size_t r; 581 int ret_val = 0; 582 /* 583 * No single byte will be more than one wide character, 584 * so this length estimate will always be big enough. 585 */ 586 size_t wcs_length = len; 587 size_t mbs_length = len; 588 const char *mbs = p; 589 wchar_t *wcs; 590 #if HAVE_MBRTOWC 591 mbstate_t shift_state; 592 593 memset(&shift_state, 0, sizeof(shift_state)); 594 #endif 595 if (NULL == archive_wstring_ensure(dest, dest->length + wcs_length + 1)) 596 return (-1); 597 wcs = dest->s + dest->length; 598 /* 599 * We cannot use mbsrtowcs/mbstowcs here because those may convert 600 * extra MBS when strlen(p) > len and one wide character consis of 601 * multi bytes. 602 */ 603 while (*mbs && mbs_length > 0) { 604 if (wcs_length == 0) { 605 dest->length = wcs - dest->s; 606 dest->s[dest->length] = L'\0'; 607 wcs_length = mbs_length; 608 if (NULL == archive_wstring_ensure(dest, 609 dest->length + wcs_length + 1)) 610 return (-1); 611 wcs = dest->s + dest->length; 612 } 613 #if HAVE_MBRTOWC 614 r = mbrtowc(wcs, mbs, wcs_length, &shift_state); 615 #else 616 r = mbtowc(wcs, mbs, wcs_length); 617 #endif 618 if (r == (size_t)-1 || r == (size_t)-2) { 619 ret_val = -1; 620 if (errno == EILSEQ) { 621 ++mbs; 622 --mbs_length; 623 continue; 624 } else 625 break; 626 } 627 if (r == 0 || r > mbs_length) 628 break; 629 wcs++; 630 wcs_length--; 631 mbs += r; 632 mbs_length -= r; 633 } 634 dest->length = wcs - dest->s; 635 dest->s[dest->length] = L'\0'; 636 return (ret_val); 637 } 638 639 #endif 640 641 #if defined(_WIN32) && !defined(__CYGWIN__) 642 643 /* 644 * WCS ==> MBS. 645 * Note: returns -1 if conversion fails. 646 * 647 * Win32 builds use WideCharToMultiByte from the Windows API. 648 * (Maybe Cygwin should too? WideCharToMultiByte will know a 649 * lot more about local character encodings than the wcrtomb() 650 * wrapper is going to know.) 651 */ 652 int 653 archive_string_append_from_wcs(struct archive_string *as, 654 const wchar_t *w, size_t len) 655 { 656 return archive_string_append_from_wcs_in_codepage(as, w, len, NULL); 657 } 658 659 static int 660 archive_string_append_from_wcs_in_codepage(struct archive_string *as, 661 const wchar_t *ws, size_t len, struct archive_string_conv *sc) 662 { 663 BOOL defchar_used, *dp; 664 int count, ret = 0; 665 UINT to_cp; 666 int wslen = (int)len; 667 668 if (sc != NULL) 669 to_cp = sc->to_cp; 670 else 671 to_cp = get_current_codepage(); 672 673 if (to_cp == CP_C_LOCALE) { 674 /* 675 * "C" locale special process. 676 */ 677 const wchar_t *wp = ws; 678 char *p; 679 680 if (NULL == archive_string_ensure(as, 681 as->length + wslen +1)) 682 return (-1); 683 p = as->s + as->length; 684 count = 0; 685 defchar_used = 0; 686 while (count < wslen && *wp) { 687 if (*wp > 255) { 688 *p++ = '?'; 689 wp++; 690 defchar_used = 1; 691 } else 692 *p++ = (char)*wp++; 693 count++; 694 } 695 } else if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) { 696 uint16_t *u16; 697 698 if (NULL == 699 archive_string_ensure(as, as->length + len * 2 + 2)) 700 return (-1); 701 u16 = (uint16_t *)(as->s + as->length); 702 count = 0; 703 defchar_used = 0; 704 if (sc->flag & SCONV_TO_UTF16BE) { 705 while (count < (int)len && *ws) { 706 archive_be16enc(u16+count, *ws); 707 ws++; 708 count++; 709 } 710 } else { 711 while (count < (int)len && *ws) { 712 archive_le16enc(u16+count, *ws); 713 ws++; 714 count++; 715 } 716 } 717 count <<= 1; /* to be byte size */ 718 } else { 719 /* Make sure the MBS buffer has plenty to set. */ 720 if (NULL == 721 archive_string_ensure(as, as->length + len * 2 + 1)) 722 return (-1); 723 do { 724 defchar_used = 0; 725 if (to_cp == CP_UTF8 || sc == NULL) 726 dp = NULL; 727 else 728 dp = &defchar_used; 729 count = WideCharToMultiByte(to_cp, 0, ws, wslen, 730 as->s + as->length, (int)as->buffer_length-1, NULL, dp); 731 if (count == 0 && 732 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 733 /* Expand the MBS buffer and retry. */ 734 if (NULL == archive_string_ensure(as, 735 as->buffer_length + len)) 736 return (-1); 737 continue; 738 } 739 if (count == 0) 740 ret = -1; 741 } while (0); 742 } 743 as->length += count; 744 as->s[as->length] = '\0'; 745 return (defchar_used?-1:ret); 746 } 747 748 #elif defined(HAVE_WCTOMB) || defined(HAVE_WCRTOMB) 749 750 /* 751 * Translates a wide character string into current locale character set 752 * and appends to the archive_string. Note: returns -1 if conversion 753 * fails. 754 */ 755 int 756 archive_string_append_from_wcs(struct archive_string *as, 757 const wchar_t *w, size_t len) 758 { 759 /* We cannot use the standard wcstombs() here because it 760 * cannot tell us how big the output buffer should be. So 761 * I've built a loop around wcrtomb() or wctomb() that 762 * converts a character at a time and resizes the string as 763 * needed. We prefer wcrtomb() when it's available because 764 * it's thread-safe. */ 765 int n, ret_val = 0; 766 char *p; 767 char *end; 768 #if HAVE_WCRTOMB 769 mbstate_t shift_state; 770 771 memset(&shift_state, 0, sizeof(shift_state)); 772 #else 773 /* Clear the shift state before starting. */ 774 wctomb(NULL, L'\0'); 775 #endif 776 /* 777 * Allocate buffer for MBS. 778 * We need this allocation here since it is possible that 779 * as->s is still NULL. 780 */ 781 if (archive_string_ensure(as, as->length + len + 1) == NULL) 782 return (-1); 783 784 p = as->s + as->length; 785 end = as->s + as->buffer_length - MB_CUR_MAX -1; 786 while (*w != L'\0' && len > 0) { 787 if (p >= end) { 788 as->length = p - as->s; 789 as->s[as->length] = '\0'; 790 /* Re-allocate buffer for MBS. */ 791 if (archive_string_ensure(as, 792 as->length + len * 2 + 1) == NULL) 793 return (-1); 794 p = as->s + as->length; 795 end = as->s + as->buffer_length - MB_CUR_MAX -1; 796 } 797 #if HAVE_WCRTOMB 798 n = wcrtomb(p, *w++, &shift_state); 799 #else 800 n = wctomb(p, *w++); 801 #endif 802 if (n == -1) { 803 if (errno == EILSEQ) { 804 /* Skip an illegal wide char. */ 805 *p++ = '?'; 806 ret_val = -1; 807 } else { 808 ret_val = -1; 809 break; 810 } 811 } else 812 p += n; 813 len--; 814 } 815 as->length = p - as->s; 816 as->s[as->length] = '\0'; 817 return (ret_val); 818 } 819 820 #else /* HAVE_WCTOMB || HAVE_WCRTOMB */ 821 822 /* 823 * TODO: Test if __STDC_ISO_10646__ is defined. 824 * Non-Windows uses ISO C wcrtomb() or wctomb() to perform the conversion 825 * one character at a time. If a non-Windows platform doesn't have 826 * either of these, fall back to the built-in UTF8 conversion. 827 */ 828 int 829 archive_string_append_from_wcs(struct archive_string *as, 830 const wchar_t *w, size_t len) 831 { 832 (void)as;/* UNUSED */ 833 (void)w;/* UNUSED */ 834 (void)len;/* UNUSED */ 835 errno = ENOSYS; 836 return (-1); 837 } 838 839 #endif /* HAVE_WCTOMB || HAVE_WCRTOMB */ 840 841 /* 842 * Find a string conversion object by a pair of 'from' charset name 843 * and 'to' charset name from an archive object. 844 * Return NULL if not found. 845 */ 846 static struct archive_string_conv * 847 find_sconv_object(struct archive *a, const char *fc, const char *tc) 848 { 849 struct archive_string_conv *sc; 850 851 if (a == NULL) 852 return (NULL); 853 854 for (sc = a->sconv; sc != NULL; sc = sc->next) { 855 if (strcmp(sc->from_charset, fc) == 0 && 856 strcmp(sc->to_charset, tc) == 0) 857 break; 858 } 859 return (sc); 860 } 861 862 /* 863 * Register a string object to an archive object. 864 */ 865 static void 866 add_sconv_object(struct archive *a, struct archive_string_conv *sc) 867 { 868 struct archive_string_conv **psc; 869 870 /* Add a new sconv to sconv list. */ 871 psc = &(a->sconv); 872 while (*psc != NULL) 873 psc = &((*psc)->next); 874 *psc = sc; 875 } 876 877 static void 878 add_converter(struct archive_string_conv *sc, int (*converter) 879 (struct archive_string *, const void *, size_t, 880 struct archive_string_conv *)) 881 { 882 if (sc == NULL || sc->nconverter >= 2) 883 __archive_errx(1, "Programing error"); 884 sc->converter[sc->nconverter++] = converter; 885 } 886 887 static void 888 setup_converter(struct archive_string_conv *sc) 889 { 890 891 /* Reset. */ 892 sc->nconverter = 0; 893 894 /* 895 * Perform special sequence for the incorrect UTF-8 filenames 896 * made by libarchive2.x. 897 */ 898 if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) { 899 add_converter(sc, strncat_from_utf8_libarchive2); 900 return; 901 } 902 903 /* 904 * Convert a string to UTF-16BE/LE. 905 */ 906 if (sc->flag & SCONV_TO_UTF16) { 907 /* 908 * If the current locale is UTF-8, we can translate 909 * a UTF-8 string into a UTF-16BE string. 910 */ 911 if (sc->flag & SCONV_FROM_UTF8) { 912 add_converter(sc, archive_string_append_unicode); 913 return; 914 } 915 916 #if defined(_WIN32) && !defined(__CYGWIN__) 917 if (sc->flag & SCONV_WIN_CP) { 918 if (sc->flag & SCONV_TO_UTF16BE) 919 add_converter(sc, win_strncat_to_utf16be); 920 else 921 add_converter(sc, win_strncat_to_utf16le); 922 return; 923 } 924 #endif 925 926 #if defined(HAVE_ICONV) 927 if (sc->cd != (iconv_t)-1) { 928 add_converter(sc, iconv_strncat_in_locale); 929 return; 930 } 931 #endif 932 933 if (sc->flag & SCONV_BEST_EFFORT) { 934 if (sc->flag & SCONV_TO_UTF16BE) 935 add_converter(sc, 936 best_effort_strncat_to_utf16be); 937 else 938 add_converter(sc, 939 best_effort_strncat_to_utf16le); 940 } else 941 /* Make sure we have no converter. */ 942 sc->nconverter = 0; 943 return; 944 } 945 946 /* 947 * Convert a string from UTF-16BE/LE. 948 */ 949 if (sc->flag & SCONV_FROM_UTF16) { 950 /* 951 * At least we should normalize a UTF-16BE string. 952 */ 953 if (sc->flag & SCONV_NORMALIZATION_D) 954 add_converter(sc,archive_string_normalize_D); 955 else if (sc->flag & SCONV_NORMALIZATION_C) 956 add_converter(sc, archive_string_normalize_C); 957 958 if (sc->flag & SCONV_TO_UTF8) { 959 /* 960 * If the current locale is UTF-8, we can translate 961 * a UTF-16BE/LE string into a UTF-8 string directly. 962 */ 963 if (!(sc->flag & 964 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 965 add_converter(sc, 966 archive_string_append_unicode); 967 return; 968 } 969 970 #if defined(_WIN32) && !defined(__CYGWIN__) 971 if (sc->flag & SCONV_WIN_CP) { 972 if (sc->flag & SCONV_FROM_UTF16BE) 973 add_converter(sc, win_strncat_from_utf16be); 974 else 975 add_converter(sc, win_strncat_from_utf16le); 976 return; 977 } 978 #endif 979 980 #if defined(HAVE_ICONV) 981 if (sc->cd != (iconv_t)-1) { 982 add_converter(sc, iconv_strncat_in_locale); 983 return; 984 } 985 #endif 986 987 if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 988 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 989 add_converter(sc, best_effort_strncat_from_utf16be); 990 else if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 991 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 992 add_converter(sc, best_effort_strncat_from_utf16le); 993 else 994 /* Make sure we have no converter. */ 995 sc->nconverter = 0; 996 return; 997 } 998 999 if (sc->flag & SCONV_FROM_UTF8) { 1000 /* 1001 * At least we should normalize a UTF-8 string. 1002 */ 1003 if (sc->flag & SCONV_NORMALIZATION_D) 1004 add_converter(sc,archive_string_normalize_D); 1005 else if (sc->flag & SCONV_NORMALIZATION_C) 1006 add_converter(sc, archive_string_normalize_C); 1007 1008 /* 1009 * Copy UTF-8 string with a check of CESU-8. 1010 * Apparently, iconv does not check surrogate pairs in UTF-8 1011 * when both from-charset and to-charset are UTF-8, and then 1012 * we use our UTF-8 copy code. 1013 */ 1014 if (sc->flag & SCONV_TO_UTF8) { 1015 /* 1016 * If the current locale is UTF-8, we can translate 1017 * a UTF-16BE string into a UTF-8 string directly. 1018 */ 1019 if (!(sc->flag & 1020 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 1021 add_converter(sc, strncat_from_utf8_to_utf8); 1022 return; 1023 } 1024 } 1025 1026 #if defined(_WIN32) && !defined(__CYGWIN__) 1027 /* 1028 * On Windows we can use Windows API for a string conversion. 1029 */ 1030 if (sc->flag & SCONV_WIN_CP) { 1031 add_converter(sc, strncat_in_codepage); 1032 return; 1033 } 1034 #endif 1035 1036 #if HAVE_ICONV 1037 if (sc->cd != (iconv_t)-1) { 1038 add_converter(sc, iconv_strncat_in_locale); 1039 /* 1040 * iconv generally does not support UTF-8-MAC and so 1041 * we have to the output of iconv from NFC to NFD if 1042 * need. 1043 */ 1044 if ((sc->flag & SCONV_FROM_CHARSET) && 1045 (sc->flag & SCONV_TO_UTF8)) { 1046 if (sc->flag & SCONV_NORMALIZATION_D) 1047 add_converter(sc, archive_string_normalize_D); 1048 } 1049 return; 1050 } 1051 #endif 1052 1053 /* 1054 * Try conversion in the best effort or no conversion. 1055 */ 1056 if ((sc->flag & SCONV_BEST_EFFORT) || sc->same) 1057 add_converter(sc, best_effort_strncat_in_locale); 1058 else 1059 /* Make sure we have no converter. */ 1060 sc->nconverter = 0; 1061 } 1062 1063 /* 1064 * Return canonicalized charset-name but this supports just UTF-8, UTF-16BE 1065 * and CP932 which are referenced in create_sconv_object(). 1066 */ 1067 static const char * 1068 canonical_charset_name(const char *charset) 1069 { 1070 char cs[16]; 1071 char *p; 1072 const char *s; 1073 1074 if (charset == NULL || charset[0] == '\0' 1075 || strlen(charset) > 15) 1076 return (charset); 1077 1078 /* Copy name to uppercase. */ 1079 p = cs; 1080 s = charset; 1081 while (*s) { 1082 char c = *s++; 1083 if (c >= 'a' && c <= 'z') 1084 c -= 'a' - 'A'; 1085 *p++ = c; 1086 } 1087 *p++ = '\0'; 1088 1089 if (strcmp(cs, "UTF-8") == 0 || 1090 strcmp(cs, "UTF8") == 0) 1091 return ("UTF-8"); 1092 if (strcmp(cs, "UTF-16BE") == 0 || 1093 strcmp(cs, "UTF16BE") == 0) 1094 return ("UTF-16BE"); 1095 if (strcmp(cs, "UTF-16LE") == 0 || 1096 strcmp(cs, "UTF16LE") == 0) 1097 return ("UTF-16LE"); 1098 if (strcmp(cs, "CP932") == 0) 1099 return ("CP932"); 1100 return (charset); 1101 } 1102 1103 /* 1104 * Create a string conversion object. 1105 */ 1106 static struct archive_string_conv * 1107 create_sconv_object(const char *fc, const char *tc, 1108 unsigned current_codepage, int flag) 1109 { 1110 struct archive_string_conv *sc; 1111 1112 sc = calloc(1, sizeof(*sc)); 1113 if (sc == NULL) 1114 return (NULL); 1115 sc->next = NULL; 1116 sc->from_charset = strdup(fc); 1117 if (sc->from_charset == NULL) { 1118 free(sc); 1119 return (NULL); 1120 } 1121 sc->to_charset = strdup(tc); 1122 if (sc->to_charset == NULL) { 1123 free(sc->from_charset); 1124 free(sc); 1125 return (NULL); 1126 } 1127 archive_string_init(&sc->utftmp); 1128 1129 if (flag & SCONV_TO_CHARSET) { 1130 /* 1131 * Convert characters from the current locale charset to 1132 * a specified charset. 1133 */ 1134 sc->from_cp = current_codepage; 1135 sc->to_cp = make_codepage_from_charset(tc); 1136 #if defined(_WIN32) && !defined(__CYGWIN__) 1137 if (IsValidCodePage(sc->to_cp)) 1138 flag |= SCONV_WIN_CP; 1139 #endif 1140 } else if (flag & SCONV_FROM_CHARSET) { 1141 /* 1142 * Convert characters from a specified charset to 1143 * the current locale charset. 1144 */ 1145 sc->to_cp = current_codepage; 1146 sc->from_cp = make_codepage_from_charset(fc); 1147 #if defined(_WIN32) && !defined(__CYGWIN__) 1148 if (IsValidCodePage(sc->from_cp)) 1149 flag |= SCONV_WIN_CP; 1150 #endif 1151 } 1152 1153 /* 1154 * Check if "from charset" and "to charset" are the same. 1155 */ 1156 if (strcmp(fc, tc) == 0 || 1157 (sc->from_cp != (unsigned)-1 && sc->from_cp == sc->to_cp)) 1158 sc->same = 1; 1159 else 1160 sc->same = 0; 1161 1162 /* 1163 * Mark if "from charset" or "to charset" are UTF-8 or UTF-16BE/LE. 1164 */ 1165 if (strcmp(tc, "UTF-8") == 0) 1166 flag |= SCONV_TO_UTF8; 1167 else if (strcmp(tc, "UTF-16BE") == 0) 1168 flag |= SCONV_TO_UTF16BE; 1169 else if (strcmp(tc, "UTF-16LE") == 0) 1170 flag |= SCONV_TO_UTF16LE; 1171 if (strcmp(fc, "UTF-8") == 0) 1172 flag |= SCONV_FROM_UTF8; 1173 else if (strcmp(fc, "UTF-16BE") == 0) 1174 flag |= SCONV_FROM_UTF16BE; 1175 else if (strcmp(fc, "UTF-16LE") == 0) 1176 flag |= SCONV_FROM_UTF16LE; 1177 #if defined(_WIN32) && !defined(__CYGWIN__) 1178 if (sc->to_cp == CP_UTF8) 1179 flag |= SCONV_TO_UTF8; 1180 else if (sc->to_cp == CP_UTF16BE) 1181 flag |= SCONV_TO_UTF16BE | SCONV_WIN_CP; 1182 else if (sc->to_cp == CP_UTF16LE) 1183 flag |= SCONV_TO_UTF16LE | SCONV_WIN_CP; 1184 if (sc->from_cp == CP_UTF8) 1185 flag |= SCONV_FROM_UTF8; 1186 else if (sc->from_cp == CP_UTF16BE) 1187 flag |= SCONV_FROM_UTF16BE | SCONV_WIN_CP; 1188 else if (sc->from_cp == CP_UTF16LE) 1189 flag |= SCONV_FROM_UTF16LE | SCONV_WIN_CP; 1190 #endif 1191 1192 /* 1193 * Set a flag for Unicode NFD. Usually iconv cannot correctly 1194 * handle it. So we have to translate NFD characters to NFC ones 1195 * ourselves before iconv handles. Another reason is to prevent 1196 * that the same sight of two filenames, one is NFC and other 1197 * is NFD, would be in its directory. 1198 * On Mac OS X, although its filesystem layer automatically 1199 * convert filenames to NFD, it would be useful for filename 1200 * comparing to find out the same filenames that we normalize 1201 * that to be NFD ourselves. 1202 */ 1203 if ((flag & SCONV_FROM_CHARSET) && 1204 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8))) { 1205 #if defined(__APPLE__) 1206 if (flag & SCONV_TO_UTF8) 1207 flag |= SCONV_NORMALIZATION_D; 1208 else 1209 #endif 1210 flag |= SCONV_NORMALIZATION_C; 1211 } 1212 #if defined(__APPLE__) 1213 /* 1214 * In case writing an archive file, make sure that a filename 1215 * going to be passed to iconv is a Unicode NFC string since 1216 * a filename in HFS Plus filesystem is a Unicode NFD one and 1217 * iconv cannot handle it with "UTF-8" charset. It is simpler 1218 * than a use of "UTF-8-MAC" charset. 1219 */ 1220 if ((flag & SCONV_TO_CHARSET) && 1221 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1222 !(flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1223 flag |= SCONV_NORMALIZATION_C; 1224 /* 1225 * In case reading an archive file. make sure that a filename 1226 * will be passed to users is a Unicode NFD string in order to 1227 * correctly compare the filename with other one which comes 1228 * from HFS Plus filesystem. 1229 */ 1230 if ((flag & SCONV_FROM_CHARSET) && 1231 !(flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1232 (flag & SCONV_TO_UTF8)) 1233 flag |= SCONV_NORMALIZATION_D; 1234 #endif 1235 1236 #if defined(HAVE_ICONV) 1237 sc->cd_w = (iconv_t)-1; 1238 /* 1239 * Create an iconv object. 1240 */ 1241 if (((flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) && 1242 (flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16))) || 1243 (flag & SCONV_WIN_CP)) { 1244 /* This case we won't use iconv. */ 1245 sc->cd = (iconv_t)-1; 1246 } else { 1247 sc->cd = iconv_open(tc, fc); 1248 if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT)) { 1249 /* 1250 * Unfortunaly, all of iconv implements do support 1251 * "CP932" character-set, so we should use "SJIS" 1252 * instead if iconv_open failed. 1253 */ 1254 if (strcmp(tc, "CP932") == 0) 1255 sc->cd = iconv_open("SJIS", fc); 1256 else if (strcmp(fc, "CP932") == 0) 1257 sc->cd = iconv_open(tc, "SJIS"); 1258 } 1259 #if defined(_WIN32) && !defined(__CYGWIN__) 1260 /* 1261 * archive_mstring on Windows directly convert multi-bytes 1262 * into archive_wstring in order not to depend on locale 1263 * so that you can do a I18N programing. This will be 1264 * used only in archive_mstring_copy_mbs_len_l so far. 1265 */ 1266 if (flag & SCONV_FROM_CHARSET) { 1267 sc->cd_w = iconv_open("UTF-8", fc); 1268 if (sc->cd_w == (iconv_t)-1 && 1269 (sc->flag & SCONV_BEST_EFFORT)) { 1270 if (strcmp(fc, "CP932") == 0) 1271 sc->cd_w = iconv_open("UTF-8", "SJIS"); 1272 } 1273 } 1274 #endif /* _WIN32 && !__CYGWIN__ */ 1275 } 1276 #endif /* HAVE_ICONV */ 1277 1278 sc->flag = flag; 1279 1280 /* 1281 * Set up converters. 1282 */ 1283 setup_converter(sc); 1284 1285 return (sc); 1286 } 1287 1288 /* 1289 * Free a string conversion object. 1290 */ 1291 static void 1292 free_sconv_object(struct archive_string_conv *sc) 1293 { 1294 free(sc->from_charset); 1295 free(sc->to_charset); 1296 archive_string_free(&sc->utftmp); 1297 #if HAVE_ICONV 1298 if (sc->cd != (iconv_t)-1) 1299 iconv_close(sc->cd); 1300 if (sc->cd_w != (iconv_t)-1) 1301 iconv_close(sc->cd_w); 1302 #endif 1303 free(sc); 1304 } 1305 1306 #if defined(_WIN32) && !defined(__CYGWIN__) 1307 static unsigned 1308 my_atoi(const char *p) 1309 { 1310 unsigned cp; 1311 1312 cp = 0; 1313 while (*p) { 1314 if (*p >= '0' && *p <= '9') 1315 cp = cp * 10 + (*p - '0'); 1316 else 1317 return (-1); 1318 p++; 1319 } 1320 return (cp); 1321 } 1322 1323 /* 1324 * Translate Charset name (as used by iconv) into CodePage (as used by Windows) 1325 * Return -1 if failed. 1326 * 1327 * Note: This translation code may be insufficient. 1328 */ 1329 static struct charset { 1330 const char *name; 1331 unsigned cp; 1332 } charsets[] = { 1333 /* MUST BE SORTED! */ 1334 {"ASCII", 1252}, 1335 {"ASMO-708", 708}, 1336 {"BIG5", 950}, 1337 {"CHINESE", 936}, 1338 {"CP367", 1252}, 1339 {"CP819", 1252}, 1340 {"CP1025", 21025}, 1341 {"DOS-720", 720}, 1342 {"DOS-862", 862}, 1343 {"EUC-CN", 51936}, 1344 {"EUC-JP", 51932}, 1345 {"EUC-KR", 949}, 1346 {"EUCCN", 51936}, 1347 {"EUCJP", 51932}, 1348 {"EUCKR", 949}, 1349 {"GB18030", 54936}, 1350 {"GB2312", 936}, 1351 {"HEBREW", 1255}, 1352 {"HZ-GB-2312", 52936}, 1353 {"IBM273", 20273}, 1354 {"IBM277", 20277}, 1355 {"IBM278", 20278}, 1356 {"IBM280", 20280}, 1357 {"IBM284", 20284}, 1358 {"IBM285", 20285}, 1359 {"IBM290", 20290}, 1360 {"IBM297", 20297}, 1361 {"IBM367", 1252}, 1362 {"IBM420", 20420}, 1363 {"IBM423", 20423}, 1364 {"IBM424", 20424}, 1365 {"IBM819", 1252}, 1366 {"IBM871", 20871}, 1367 {"IBM880", 20880}, 1368 {"IBM905", 20905}, 1369 {"IBM924", 20924}, 1370 {"ISO-8859-1", 28591}, 1371 {"ISO-8859-13", 28603}, 1372 {"ISO-8859-15", 28605}, 1373 {"ISO-8859-2", 28592}, 1374 {"ISO-8859-3", 28593}, 1375 {"ISO-8859-4", 28594}, 1376 {"ISO-8859-5", 28595}, 1377 {"ISO-8859-6", 28596}, 1378 {"ISO-8859-7", 28597}, 1379 {"ISO-8859-8", 28598}, 1380 {"ISO-8859-9", 28599}, 1381 {"ISO8859-1", 28591}, 1382 {"ISO8859-13", 28603}, 1383 {"ISO8859-15", 28605}, 1384 {"ISO8859-2", 28592}, 1385 {"ISO8859-3", 28593}, 1386 {"ISO8859-4", 28594}, 1387 {"ISO8859-5", 28595}, 1388 {"ISO8859-6", 28596}, 1389 {"ISO8859-7", 28597}, 1390 {"ISO8859-8", 28598}, 1391 {"ISO8859-9", 28599}, 1392 {"JOHAB", 1361}, 1393 {"KOI8-R", 20866}, 1394 {"KOI8-U", 21866}, 1395 {"KS_C_5601-1987", 949}, 1396 {"LATIN1", 1252}, 1397 {"LATIN2", 28592}, 1398 {"MACINTOSH", 10000}, 1399 {"SHIFT-JIS", 932}, 1400 {"SHIFT_JIS", 932}, 1401 {"SJIS", 932}, 1402 {"US", 1252}, 1403 {"US-ASCII", 1252}, 1404 {"UTF-16", 1200}, 1405 {"UTF-16BE", 1201}, 1406 {"UTF-16LE", 1200}, 1407 {"UTF-8", CP_UTF8}, 1408 {"X-EUROPA", 29001}, 1409 {"X-MAC-ARABIC", 10004}, 1410 {"X-MAC-CE", 10029}, 1411 {"X-MAC-CHINESEIMP", 10008}, 1412 {"X-MAC-CHINESETRAD", 10002}, 1413 {"X-MAC-CROATIAN", 10082}, 1414 {"X-MAC-CYRILLIC", 10007}, 1415 {"X-MAC-GREEK", 10006}, 1416 {"X-MAC-HEBREW", 10005}, 1417 {"X-MAC-ICELANDIC", 10079}, 1418 {"X-MAC-JAPANESE", 10001}, 1419 {"X-MAC-KOREAN", 10003}, 1420 {"X-MAC-ROMANIAN", 10010}, 1421 {"X-MAC-THAI", 10021}, 1422 {"X-MAC-TURKISH", 10081}, 1423 {"X-MAC-UKRAINIAN", 10017}, 1424 }; 1425 static unsigned 1426 make_codepage_from_charset(const char *charset) 1427 { 1428 char cs[16]; 1429 char *p; 1430 unsigned cp; 1431 int a, b; 1432 1433 if (charset == NULL || strlen(charset) > 15) 1434 return -1; 1435 1436 /* Copy name to uppercase. */ 1437 p = cs; 1438 while (*charset) { 1439 char c = *charset++; 1440 if (c >= 'a' && c <= 'z') 1441 c -= 'a' - 'A'; 1442 *p++ = c; 1443 } 1444 *p++ = '\0'; 1445 cp = -1; 1446 1447 /* Look it up in the table first, so that we can easily 1448 * override CP367, which we map to 1252 instead of 367. */ 1449 a = 0; 1450 b = sizeof(charsets)/sizeof(charsets[0]); 1451 while (b > a) { 1452 int c = (b + a) / 2; 1453 int r = strcmp(charsets[c].name, cs); 1454 if (r < 0) 1455 a = c + 1; 1456 else if (r > 0) 1457 b = c; 1458 else 1459 return charsets[c].cp; 1460 } 1461 1462 /* If it's not in the table, try to parse it. */ 1463 switch (*cs) { 1464 case 'C': 1465 if (cs[1] == 'P' && cs[2] >= '0' && cs[2] <= '9') { 1466 cp = my_atoi(cs + 2); 1467 } else if (strcmp(cs, "CP_ACP") == 0) 1468 cp = get_current_codepage(); 1469 else if (strcmp(cs, "CP_OEMCP") == 0) 1470 cp = get_current_oemcp(); 1471 break; 1472 case 'I': 1473 if (cs[1] == 'B' && cs[2] == 'M' && 1474 cs[3] >= '0' && cs[3] <= '9') { 1475 cp = my_atoi(cs + 3); 1476 } 1477 break; 1478 case 'W': 1479 if (strncmp(cs, "WINDOWS-", 8) == 0) { 1480 cp = my_atoi(cs + 8); 1481 if (cp != 874 && (cp < 1250 || cp > 1258)) 1482 cp = -1;/* This may invalid code. */ 1483 } 1484 break; 1485 } 1486 return (cp); 1487 } 1488 1489 /* 1490 * Return ANSI Code Page of current locale set by setlocale(). 1491 */ 1492 static unsigned 1493 get_current_codepage(void) 1494 { 1495 char *locale, *p; 1496 unsigned cp; 1497 1498 locale = setlocale(LC_CTYPE, NULL); 1499 if (locale == NULL) 1500 return (GetACP()); 1501 if (locale[0] == 'C' && locale[1] == '\0') 1502 return (CP_C_LOCALE); 1503 p = strrchr(locale, '.'); 1504 if (p == NULL) 1505 return (GetACP()); 1506 cp = my_atoi(p+1); 1507 if (cp <= 0) 1508 return (GetACP()); 1509 return (cp); 1510 } 1511 1512 /* 1513 * Translation table between Locale Name and ACP/OEMCP. 1514 */ 1515 static struct { 1516 unsigned acp; 1517 unsigned ocp; 1518 const char *locale; 1519 } acp_ocp_map[] = { 1520 { 950, 950, "Chinese_Taiwan" }, 1521 { 936, 936, "Chinese_People's Republic of China" }, 1522 { 950, 950, "Chinese_Taiwan" }, 1523 { 1250, 852, "Czech_Czech Republic" }, 1524 { 1252, 850, "Danish_Denmark" }, 1525 { 1252, 850, "Dutch_Netherlands" }, 1526 { 1252, 850, "Dutch_Belgium" }, 1527 { 1252, 437, "English_United States" }, 1528 { 1252, 850, "English_Australia" }, 1529 { 1252, 850, "English_Canada" }, 1530 { 1252, 850, "English_New Zealand" }, 1531 { 1252, 850, "English_United Kingdom" }, 1532 { 1252, 437, "English_United States" }, 1533 { 1252, 850, "Finnish_Finland" }, 1534 { 1252, 850, "French_France" }, 1535 { 1252, 850, "French_Belgium" }, 1536 { 1252, 850, "French_Canada" }, 1537 { 1252, 850, "French_Switzerland" }, 1538 { 1252, 850, "German_Germany" }, 1539 { 1252, 850, "German_Austria" }, 1540 { 1252, 850, "German_Switzerland" }, 1541 { 1253, 737, "Greek_Greece" }, 1542 { 1250, 852, "Hungarian_Hungary" }, 1543 { 1252, 850, "Icelandic_Iceland" }, 1544 { 1252, 850, "Italian_Italy" }, 1545 { 1252, 850, "Italian_Switzerland" }, 1546 { 932, 932, "Japanese_Japan" }, 1547 { 949, 949, "Korean_Korea" }, 1548 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1549 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1550 { 1252, 850, "Norwegian-Nynorsk_Norway" }, 1551 { 1250, 852, "Polish_Poland" }, 1552 { 1252, 850, "Portuguese_Portugal" }, 1553 { 1252, 850, "Portuguese_Brazil" }, 1554 { 1251, 866, "Russian_Russia" }, 1555 { 1250, 852, "Slovak_Slovakia" }, 1556 { 1252, 850, "Spanish_Spain" }, 1557 { 1252, 850, "Spanish_Mexico" }, 1558 { 1252, 850, "Spanish_Spain" }, 1559 { 1252, 850, "Swedish_Sweden" }, 1560 { 1254, 857, "Turkish_Turkey" }, 1561 { 0, 0, NULL} 1562 }; 1563 1564 /* 1565 * Return OEM Code Page of current locale set by setlocale(). 1566 */ 1567 static unsigned 1568 get_current_oemcp(void) 1569 { 1570 int i; 1571 char *locale, *p; 1572 size_t len; 1573 1574 locale = setlocale(LC_CTYPE, NULL); 1575 if (locale == NULL) 1576 return (GetOEMCP()); 1577 if (locale[0] == 'C' && locale[1] == '\0') 1578 return (CP_C_LOCALE); 1579 1580 p = strrchr(locale, '.'); 1581 if (p == NULL) 1582 return (GetOEMCP()); 1583 len = p - locale; 1584 for (i = 0; acp_ocp_map[i].acp; i++) { 1585 if (strncmp(acp_ocp_map[i].locale, locale, len) == 0) 1586 return (acp_ocp_map[i].ocp); 1587 } 1588 return (GetOEMCP()); 1589 } 1590 #else 1591 1592 /* 1593 * POSIX platform does not use CodePage. 1594 */ 1595 1596 static unsigned 1597 get_current_codepage(void) 1598 { 1599 return (-1);/* Unknown */ 1600 } 1601 static unsigned 1602 make_codepage_from_charset(const char *charset) 1603 { 1604 (void)charset; /* UNUSED */ 1605 return (-1);/* Unknown */ 1606 } 1607 static unsigned 1608 get_current_oemcp(void) 1609 { 1610 return (-1);/* Unknown */ 1611 } 1612 1613 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 1614 1615 /* 1616 * Return a string conversion object. 1617 */ 1618 static struct archive_string_conv * 1619 get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag) 1620 { 1621 struct archive_string_conv *sc; 1622 unsigned current_codepage; 1623 1624 /* Check if we have made the sconv object. */ 1625 sc = find_sconv_object(a, fc, tc); 1626 if (sc != NULL) 1627 return (sc); 1628 1629 if (a == NULL) 1630 current_codepage = get_current_codepage(); 1631 else 1632 current_codepage = a->current_codepage; 1633 1634 sc = create_sconv_object(canonical_charset_name(fc), 1635 canonical_charset_name(tc), current_codepage, flag); 1636 if (sc == NULL) { 1637 if (a != NULL) 1638 archive_set_error(a, ENOMEM, 1639 "Could not allocate memory for " 1640 "a string conversion object"); 1641 return (NULL); 1642 } 1643 1644 /* 1645 * If there is no converter for current string conversion object, 1646 * we cannot handle this conversion. 1647 */ 1648 if (sc->nconverter == 0) { 1649 if (a != NULL) { 1650 #if HAVE_ICONV 1651 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1652 "iconv_open failed : Cannot handle ``%s''", 1653 (flag & SCONV_TO_CHARSET)?tc:fc); 1654 #else 1655 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1656 "A character-set conversion not fully supported " 1657 "on this platform"); 1658 #endif 1659 } 1660 /* Failed; free a sconv object. */ 1661 free_sconv_object(sc); 1662 return (NULL); 1663 } 1664 1665 /* 1666 * Success! 1667 */ 1668 if (a != NULL) 1669 add_sconv_object(a, sc); 1670 return (sc); 1671 } 1672 1673 static const char * 1674 get_current_charset(struct archive *a) 1675 { 1676 const char *cur_charset; 1677 1678 if (a == NULL) 1679 cur_charset = default_iconv_charset(""); 1680 else { 1681 cur_charset = default_iconv_charset(a->current_code); 1682 if (a->current_code == NULL) { 1683 a->current_code = strdup(cur_charset); 1684 a->current_codepage = get_current_codepage(); 1685 a->current_oemcp = get_current_oemcp(); 1686 } 1687 } 1688 return (cur_charset); 1689 } 1690 1691 /* 1692 * Make and Return a string conversion object. 1693 * Return NULL if the platform does not support the specified conversion 1694 * and best_effort is 0. 1695 * If best_effort is set, A string conversion object must be returned 1696 * unless memory allocation for the object fails, but the conversion 1697 * might fail when non-ASCII code is found. 1698 */ 1699 struct archive_string_conv * 1700 archive_string_conversion_to_charset(struct archive *a, const char *charset, 1701 int best_effort) 1702 { 1703 int flag = SCONV_TO_CHARSET; 1704 1705 if (best_effort) 1706 flag |= SCONV_BEST_EFFORT; 1707 return (get_sconv_object(a, get_current_charset(a), charset, flag)); 1708 } 1709 1710 struct archive_string_conv * 1711 archive_string_conversion_from_charset(struct archive *a, const char *charset, 1712 int best_effort) 1713 { 1714 int flag = SCONV_FROM_CHARSET; 1715 1716 if (best_effort) 1717 flag |= SCONV_BEST_EFFORT; 1718 return (get_sconv_object(a, charset, get_current_charset(a), flag)); 1719 } 1720 1721 /* 1722 * archive_string_default_conversion_*_archive() are provided for Windows 1723 * platform because other archiver application use CP_OEMCP for 1724 * MultiByteToWideChar() and WideCharToMultiByte() for the filenames 1725 * in tar or zip files. But mbstowcs/wcstombs(CRT) usually use CP_ACP 1726 * unless you use setlocale(LC_ALL, ".OCP")(specify CP_OEMCP). 1727 * So we should make a string conversion between CP_ACP and CP_OEMCP 1728 * for compatibillty. 1729 */ 1730 #if defined(_WIN32) && !defined(__CYGWIN__) 1731 struct archive_string_conv * 1732 archive_string_default_conversion_for_read(struct archive *a) 1733 { 1734 const char *cur_charset = get_current_charset(a); 1735 char oemcp[16]; 1736 1737 /* NOTE: a check of cur_charset is unneeded but we need 1738 * that get_current_charset() has been surely called at 1739 * this time whatever C compiler optimized. */ 1740 if (cur_charset != NULL && 1741 (a->current_codepage == CP_C_LOCALE || 1742 a->current_codepage == a->current_oemcp)) 1743 return (NULL);/* no conversion. */ 1744 1745 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1746 /* Make sure a null termination must be set. */ 1747 oemcp[sizeof(oemcp)-1] = '\0'; 1748 return (get_sconv_object(a, oemcp, cur_charset, 1749 SCONV_FROM_CHARSET)); 1750 } 1751 1752 struct archive_string_conv * 1753 archive_string_default_conversion_for_write(struct archive *a) 1754 { 1755 const char *cur_charset = get_current_charset(a); 1756 char oemcp[16]; 1757 1758 /* NOTE: a check of cur_charset is unneeded but we need 1759 * that get_current_charset() has been surely called at 1760 * this time whatever C compiler optimized. */ 1761 if (cur_charset != NULL && 1762 (a->current_codepage == CP_C_LOCALE || 1763 a->current_codepage == a->current_oemcp)) 1764 return (NULL);/* no conversion. */ 1765 1766 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1767 /* Make sure a null termination must be set. */ 1768 oemcp[sizeof(oemcp)-1] = '\0'; 1769 return (get_sconv_object(a, cur_charset, oemcp, 1770 SCONV_TO_CHARSET)); 1771 } 1772 #else 1773 struct archive_string_conv * 1774 archive_string_default_conversion_for_read(struct archive *a) 1775 { 1776 (void)a; /* UNUSED */ 1777 return (NULL); 1778 } 1779 1780 struct archive_string_conv * 1781 archive_string_default_conversion_for_write(struct archive *a) 1782 { 1783 (void)a; /* UNUSED */ 1784 return (NULL); 1785 } 1786 #endif 1787 1788 /* 1789 * Dispose of all character conversion objects in the archive object. 1790 */ 1791 void 1792 archive_string_conversion_free(struct archive *a) 1793 { 1794 struct archive_string_conv *sc; 1795 struct archive_string_conv *sc_next; 1796 1797 for (sc = a->sconv; sc != NULL; sc = sc_next) { 1798 sc_next = sc->next; 1799 free_sconv_object(sc); 1800 } 1801 a->sconv = NULL; 1802 free(a->current_code); 1803 a->current_code = NULL; 1804 } 1805 1806 /* 1807 * Return a conversion charset name. 1808 */ 1809 const char * 1810 archive_string_conversion_charset_name(struct archive_string_conv *sc) 1811 { 1812 if (sc->flag & SCONV_TO_CHARSET) 1813 return (sc->to_charset); 1814 else 1815 return (sc->from_charset); 1816 } 1817 1818 /* 1819 * Change the behavior of a string conversion. 1820 */ 1821 void 1822 archive_string_conversion_set_opt(struct archive_string_conv *sc, int opt) 1823 { 1824 switch (opt) { 1825 /* 1826 * A filename in UTF-8 was made with libarchive 2.x in a wrong 1827 * assumption that wchar_t was Unicode. 1828 * This option enables simulating the assumption in order to read 1829 * that filname correctly. 1830 */ 1831 case SCONV_SET_OPT_UTF8_LIBARCHIVE2X: 1832 #if (defined(_WIN32) && !defined(__CYGWIN__)) \ 1833 || defined(__STDC_ISO_10646__) || defined(__APPLE__) 1834 /* 1835 * Nothing to do for it since wchar_t on these platforms 1836 * is really Unicode. 1837 */ 1838 (void)sc; /* UNUSED */ 1839 #else 1840 if ((sc->flag & SCONV_UTF8_LIBARCHIVE_2) == 0) { 1841 sc->flag |= SCONV_UTF8_LIBARCHIVE_2; 1842 /* Set up string converters. */ 1843 setup_converter(sc); 1844 } 1845 #endif 1846 break; 1847 case SCONV_SET_OPT_NORMALIZATION_C: 1848 if ((sc->flag & SCONV_NORMALIZATION_C) == 0) { 1849 sc->flag |= SCONV_NORMALIZATION_C; 1850 sc->flag &= ~SCONV_NORMALIZATION_D; 1851 /* Set up string converters. */ 1852 setup_converter(sc); 1853 } 1854 break; 1855 case SCONV_SET_OPT_NORMALIZATION_D: 1856 #if defined(HAVE_ICONV) 1857 /* 1858 * If iconv will take the string, do not change the 1859 * setting of the normalization. 1860 */ 1861 if (!(sc->flag & SCONV_WIN_CP) && 1862 (sc->flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1863 !(sc->flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1864 break; 1865 #endif 1866 if ((sc->flag & SCONV_NORMALIZATION_D) == 0) { 1867 sc->flag |= SCONV_NORMALIZATION_D; 1868 sc->flag &= ~SCONV_NORMALIZATION_C; 1869 /* Set up string converters. */ 1870 setup_converter(sc); 1871 } 1872 break; 1873 default: 1874 break; 1875 } 1876 } 1877 1878 /* 1879 * 1880 * Copy one archive_string to another in locale conversion. 1881 * 1882 * archive_strncat_l(); 1883 * archive_strncpy_l(); 1884 * 1885 */ 1886 1887 static size_t 1888 mbsnbytes(const void *_p, size_t n) 1889 { 1890 size_t s; 1891 const char *p, *pp; 1892 1893 if (_p == NULL) 1894 return (0); 1895 p = (const char *)_p; 1896 1897 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1898 s = 0; 1899 pp = p; 1900 while (s < n && *pp) { 1901 pp++; 1902 s++; 1903 } 1904 return (s); 1905 } 1906 1907 static size_t 1908 utf16nbytes(const void *_p, size_t n) 1909 { 1910 size_t s; 1911 const char *p, *pp; 1912 1913 if (_p == NULL) 1914 return (0); 1915 p = (const char *)_p; 1916 1917 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1918 s = 0; 1919 pp = p; 1920 n >>= 1; 1921 while (s < n && (pp[0] || pp[1])) { 1922 pp += 2; 1923 s++; 1924 } 1925 return (s<<1); 1926 } 1927 1928 int 1929 archive_strncpy_l(struct archive_string *as, const void *_p, size_t n, 1930 struct archive_string_conv *sc) 1931 { 1932 as->length = 0; 1933 return (archive_strncat_l(as, _p, n, sc)); 1934 } 1935 1936 int 1937 archive_strncat_l(struct archive_string *as, const void *_p, size_t n, 1938 struct archive_string_conv *sc) 1939 { 1940 const void *s; 1941 size_t length; 1942 int i, r = 0, r2; 1943 1944 /* We must allocate memory even if there is no data for conversion 1945 * or copy. This simulates archive_string_append behavior. */ 1946 if (_p == NULL || n == 0) { 1947 int tn = 1; 1948 if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) 1949 tn = 2; 1950 if (archive_string_ensure(as, as->length + tn) == NULL) 1951 return (-1); 1952 as->s[as->length] = 0; 1953 if (tn == 2) 1954 as->s[as->length+1] = 0; 1955 return (0); 1956 } 1957 1958 /* 1959 * If sc is NULL, we just make a copy. 1960 */ 1961 if (sc == NULL) { 1962 length = mbsnbytes(_p, n); 1963 if (archive_string_append(as, _p, length) == NULL) 1964 return (-1);/* No memory */ 1965 return (0); 1966 } 1967 1968 if (sc->flag & SCONV_FROM_UTF16) 1969 length = utf16nbytes(_p, n); 1970 else 1971 length = mbsnbytes(_p, n); 1972 s = _p; 1973 i = 0; 1974 if (sc->nconverter > 1) { 1975 sc->utftmp.length = 0; 1976 r2 = sc->converter[0](&(sc->utftmp), s, length, sc); 1977 if (r2 != 0 && errno == ENOMEM) 1978 return (r2); 1979 if (r > r2) 1980 r = r2; 1981 s = sc->utftmp.s; 1982 length = sc->utftmp.length; 1983 ++i; 1984 } 1985 r2 = sc->converter[i](as, s, length, sc); 1986 if (r > r2) 1987 r = r2; 1988 return (r); 1989 } 1990 1991 #if HAVE_ICONV 1992 1993 /* 1994 * Return -1 if conversion failes. 1995 */ 1996 static int 1997 iconv_strncat_in_locale(struct archive_string *as, const void *_p, 1998 size_t length, struct archive_string_conv *sc) 1999 { 2000 ICONV_CONST char *itp; 2001 size_t remaining; 2002 iconv_t cd; 2003 char *outp; 2004 size_t avail, bs; 2005 int return_value = 0; /* success */ 2006 int to_size, from_size; 2007 2008 if (sc->flag & SCONV_TO_UTF16) 2009 to_size = 2; 2010 else 2011 to_size = 1; 2012 if (sc->flag & SCONV_FROM_UTF16) 2013 from_size = 2; 2014 else 2015 from_size = 1; 2016 2017 if (archive_string_ensure(as, as->length + length*2+to_size) == NULL) 2018 return (-1); 2019 2020 cd = sc->cd; 2021 itp = (char *)(uintptr_t)_p; 2022 remaining = length; 2023 outp = as->s + as->length; 2024 avail = as->buffer_length - as->length - to_size; 2025 while (remaining >= (size_t)from_size) { 2026 size_t result = iconv(cd, &itp, &remaining, &outp, &avail); 2027 2028 if (result != (size_t)-1) 2029 break; /* Conversion completed. */ 2030 2031 if (errno == EILSEQ || errno == EINVAL) { 2032 /* 2033 * If an output charset is UTF-8 or UTF-16BE/LE, 2034 * unknown character should be U+FFFD 2035 * (replacement character). 2036 */ 2037 if (sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) { 2038 size_t rbytes; 2039 if (sc->flag & SCONV_TO_UTF8) 2040 rbytes = UTF8_R_CHAR_SIZE; 2041 else 2042 rbytes = 2; 2043 2044 if (avail < rbytes) { 2045 as->length = outp - as->s; 2046 bs = as->buffer_length + 2047 (remaining * to_size) + rbytes; 2048 if (NULL == 2049 archive_string_ensure(as, bs)) 2050 return (-1); 2051 outp = as->s + as->length; 2052 avail = as->buffer_length 2053 - as->length - to_size; 2054 } 2055 if (sc->flag & SCONV_TO_UTF8) 2056 UTF8_SET_R_CHAR(outp); 2057 else if (sc->flag & SCONV_TO_UTF16BE) 2058 archive_be16enc(outp, UNICODE_R_CHAR); 2059 else 2060 archive_le16enc(outp, UNICODE_R_CHAR); 2061 outp += rbytes; 2062 avail -= rbytes; 2063 } else { 2064 /* Skip the illegal input bytes. */ 2065 *outp++ = '?'; 2066 avail--; 2067 } 2068 itp += from_size; 2069 remaining -= from_size; 2070 return_value = -1; /* failure */ 2071 } else { 2072 /* E2BIG no output buffer, 2073 * Increase an output buffer. */ 2074 as->length = outp - as->s; 2075 bs = as->buffer_length + remaining * 2; 2076 if (NULL == archive_string_ensure(as, bs)) 2077 return (-1); 2078 outp = as->s + as->length; 2079 avail = as->buffer_length - as->length - to_size; 2080 } 2081 } 2082 as->length = outp - as->s; 2083 as->s[as->length] = 0; 2084 if (to_size == 2) 2085 as->s[as->length+1] = 0; 2086 return (return_value); 2087 } 2088 2089 #endif /* HAVE_ICONV */ 2090 2091 2092 #if defined(_WIN32) && !defined(__CYGWIN__) 2093 2094 /* 2095 * Translate a string from a some CodePage to an another CodePage by 2096 * Windows APIs, and copy the result. Return -1 if conversion failes. 2097 */ 2098 static int 2099 strncat_in_codepage(struct archive_string *as, 2100 const void *_p, size_t length, struct archive_string_conv *sc) 2101 { 2102 const char *s = (const char *)_p; 2103 struct archive_wstring aws; 2104 size_t l; 2105 int r, saved_flag; 2106 2107 archive_string_init(&aws); 2108 saved_flag = sc->flag; 2109 sc->flag &= ~(SCONV_NORMALIZATION_D | SCONV_NORMALIZATION_C); 2110 r = archive_wstring_append_from_mbs_in_codepage(&aws, s, length, sc); 2111 sc->flag = saved_flag; 2112 if (r != 0) { 2113 archive_wstring_free(&aws); 2114 if (errno != ENOMEM) 2115 archive_string_append(as, s, length); 2116 return (-1); 2117 } 2118 2119 l = as->length; 2120 r = archive_string_append_from_wcs_in_codepage( 2121 as, aws.s, aws.length, sc); 2122 if (r != 0 && errno != ENOMEM && l == as->length) 2123 archive_string_append(as, s, length); 2124 archive_wstring_free(&aws); 2125 return (r); 2126 } 2127 2128 /* 2129 * Test whether MBS ==> WCS is okay. 2130 */ 2131 static int 2132 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2133 { 2134 const char *p = (const char *)_p; 2135 unsigned codepage; 2136 DWORD mbflag = MB_ERR_INVALID_CHARS; 2137 2138 if (sc->flag & SCONV_FROM_CHARSET) 2139 codepage = sc->to_cp; 2140 else 2141 codepage = sc->from_cp; 2142 2143 if (codepage == CP_C_LOCALE) 2144 return (0); 2145 if (codepage != CP_UTF8) 2146 mbflag |= MB_PRECOMPOSED; 2147 2148 if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0) 2149 return (-1); /* Invalid */ 2150 return (0); /* Okay */ 2151 } 2152 2153 #else 2154 2155 /* 2156 * Test whether MBS ==> WCS is okay. 2157 */ 2158 static int 2159 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2160 { 2161 const char *p = (const char *)_p; 2162 size_t r; 2163 2164 #if HAVE_MBRTOWC 2165 mbstate_t shift_state; 2166 2167 memset(&shift_state, 0, sizeof(shift_state)); 2168 #else 2169 /* Clear the shift state before starting. */ 2170 mbtowc(NULL, NULL, 0); 2171 #endif 2172 while (n) { 2173 wchar_t wc; 2174 2175 #if HAVE_MBRTOWC 2176 r = mbrtowc(&wc, p, n, &shift_state); 2177 #else 2178 r = mbtowc(&wc, p, n); 2179 #endif 2180 if (r == (size_t)-1 || r == (size_t)-2) 2181 return (-1);/* Invalid. */ 2182 if (r == 0) 2183 break; 2184 p += r; 2185 n -= r; 2186 } 2187 (void)sc; /* UNUSED */ 2188 return (0); /* All Okey. */ 2189 } 2190 2191 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 2192 2193 /* 2194 * Basically returns -1 because we cannot make a conversion of charset 2195 * without iconv but in some cases this would return 0. 2196 * Returns 0 if all copied characters are ASCII. 2197 * Returns 0 if both from-locale and to-locale are the same and those 2198 * can be WCS with no error. 2199 */ 2200 static int 2201 best_effort_strncat_in_locale(struct archive_string *as, const void *_p, 2202 size_t length, struct archive_string_conv *sc) 2203 { 2204 size_t remaining; 2205 char *otp; 2206 const uint8_t *itp; 2207 size_t avail; 2208 int return_value = 0; /* success */ 2209 2210 /* 2211 * If both from-locale and to-locale is the same, this makes a copy. 2212 * And then this checks all copied MBS can be WCS if so returns 0. 2213 */ 2214 if (sc->same) { 2215 if (archive_string_append(as, _p, length) == NULL) 2216 return (-1);/* No memory */ 2217 return (invalid_mbs(_p, length, sc)); 2218 } 2219 2220 /* 2221 * If a character is ASCII, this just copies it. If not, this 2222 * assigns '?' charater instead but in UTF-8 locale this assigns 2223 * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD, 2224 * a Replacement Character in Unicode. 2225 */ 2226 if (archive_string_ensure(as, as->length + length + 1) == NULL) 2227 return (-1); 2228 2229 remaining = length; 2230 itp = (const uint8_t *)_p; 2231 otp = as->s + as->length; 2232 avail = as->buffer_length - as->length -1; 2233 while (*itp && remaining > 0) { 2234 if (*itp > 127 && (sc->flag & SCONV_TO_UTF8)) { 2235 if (avail < UTF8_R_CHAR_SIZE) { 2236 as->length = otp - as->s; 2237 if (NULL == archive_string_ensure(as, 2238 as->buffer_length + remaining + 2239 UTF8_R_CHAR_SIZE)) 2240 return (-1); 2241 otp = as->s + as->length; 2242 avail = as->buffer_length - as->length -1; 2243 } 2244 /* 2245 * When coping a string in UTF-8, unknown character 2246 * should be U+FFFD (replacement character). 2247 */ 2248 UTF8_SET_R_CHAR(otp); 2249 otp += UTF8_R_CHAR_SIZE; 2250 avail -= UTF8_R_CHAR_SIZE; 2251 itp++; 2252 remaining--; 2253 return_value = -1; 2254 } else if (*itp > 127) { 2255 *otp++ = '?'; 2256 itp++; 2257 remaining--; 2258 return_value = -1; 2259 } else { 2260 *otp++ = (char)*itp++; 2261 remaining--; 2262 } 2263 } 2264 as->length = otp - as->s; 2265 as->s[as->length] = '\0'; 2266 return (return_value); 2267 } 2268 2269 2270 /* 2271 * Unicode conversion functions. 2272 * - UTF-8 <===> UTF-8 in removing surrogate pairs. 2273 * - UTF-8 NFD ===> UTF-8 NFC in removing surrogate pairs. 2274 * - UTF-8 made by libarchive 2.x ===> UTF-8. 2275 * - UTF-16BE <===> UTF-8. 2276 * 2277 */ 2278 2279 /* 2280 * Utility to convert a single UTF-8 sequence. 2281 * 2282 * Usually return used bytes, return used byte in negative value when 2283 * a unicode character is replaced with U+FFFD. 2284 * See also http://unicode.org/review/pr-121.html Public Review Issue #121 2285 * Recommended Practice for Replacement Characters. 2286 */ 2287 static int 2288 _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2289 { 2290 static const char utf8_count[256] = { 2291 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 00 - 0F */ 2292 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 10 - 1F */ 2293 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 20 - 2F */ 2294 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 30 - 3F */ 2295 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 40 - 4F */ 2296 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 50 - 5F */ 2297 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 60 - 6F */ 2298 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 70 - 7F */ 2299 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 80 - 8F */ 2300 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 90 - 9F */ 2301 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* A0 - AF */ 2302 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* B0 - BF */ 2303 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* C0 - CF */ 2304 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* D0 - DF */ 2305 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,/* E0 - EF */ 2306 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0 - FF */ 2307 }; 2308 int ch, i; 2309 int cnt; 2310 uint32_t wc; 2311 2312 /* Sanity check. */ 2313 if (n == 0) 2314 return (0); 2315 /* 2316 * Decode 1-4 bytes depending on the value of the first byte. 2317 */ 2318 ch = (unsigned char)*s; 2319 if (ch == 0) 2320 return (0); /* Standard: return 0 for end-of-string. */ 2321 cnt = utf8_count[ch]; 2322 2323 /* Invalide sequence or there are not plenty bytes. */ 2324 if ((int)n < cnt) { 2325 cnt = (int)n; 2326 for (i = 1; i < cnt; i++) { 2327 if ((s[i] & 0xc0) != 0x80) { 2328 cnt = i; 2329 break; 2330 } 2331 } 2332 goto invalid_sequence; 2333 } 2334 2335 /* Make a Unicode code point from a single UTF-8 sequence. */ 2336 switch (cnt) { 2337 case 1: /* 1 byte sequence. */ 2338 *pwc = ch & 0x7f; 2339 return (cnt); 2340 case 2: /* 2 bytes sequence. */ 2341 if ((s[1] & 0xc0) != 0x80) { 2342 cnt = 1; 2343 goto invalid_sequence; 2344 } 2345 *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f); 2346 return (cnt); 2347 case 3: /* 3 bytes sequence. */ 2348 if ((s[1] & 0xc0) != 0x80) { 2349 cnt = 1; 2350 goto invalid_sequence; 2351 } 2352 if ((s[2] & 0xc0) != 0x80) { 2353 cnt = 2; 2354 goto invalid_sequence; 2355 } 2356 wc = ((ch & 0x0f) << 12) 2357 | ((s[1] & 0x3f) << 6) 2358 | (s[2] & 0x3f); 2359 if (wc < 0x800) 2360 goto invalid_sequence;/* Overlong sequence. */ 2361 break; 2362 case 4: /* 4 bytes sequence. */ 2363 if ((s[1] & 0xc0) != 0x80) { 2364 cnt = 1; 2365 goto invalid_sequence; 2366 } 2367 if ((s[2] & 0xc0) != 0x80) { 2368 cnt = 2; 2369 goto invalid_sequence; 2370 } 2371 if ((s[3] & 0xc0) != 0x80) { 2372 cnt = 3; 2373 goto invalid_sequence; 2374 } 2375 wc = ((ch & 0x07) << 18) 2376 | ((s[1] & 0x3f) << 12) 2377 | ((s[2] & 0x3f) << 6) 2378 | (s[3] & 0x3f); 2379 if (wc < 0x10000) 2380 goto invalid_sequence;/* Overlong sequence. */ 2381 break; 2382 default: /* Others are all invalid sequence. */ 2383 if (ch == 0xc0 || ch == 0xc1) 2384 cnt = 2; 2385 else if (ch >= 0xf5 && ch <= 0xf7) 2386 cnt = 4; 2387 else if (ch >= 0xf8 && ch <= 0xfb) 2388 cnt = 5; 2389 else if (ch == 0xfc || ch == 0xfd) 2390 cnt = 6; 2391 else 2392 cnt = 1; 2393 if ((int)n < cnt) 2394 cnt = (int)n; 2395 for (i = 1; i < cnt; i++) { 2396 if ((s[i] & 0xc0) != 0x80) { 2397 cnt = i; 2398 break; 2399 } 2400 } 2401 goto invalid_sequence; 2402 } 2403 2404 /* The code point larger than 0x10FFFF is not leagal 2405 * Unicode values. */ 2406 if (wc > UNICODE_MAX) 2407 goto invalid_sequence; 2408 /* Correctly gets a Unicode, returns used bytes. */ 2409 *pwc = wc; 2410 return (cnt); 2411 invalid_sequence: 2412 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2413 return (cnt * -1); 2414 } 2415 2416 static int 2417 utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2418 { 2419 int cnt; 2420 2421 cnt = _utf8_to_unicode(pwc, s, n); 2422 /* Any of Surrogate pair is not leagal Unicode values. */ 2423 if (cnt == 3 && IS_SURROGATE_PAIR_LA(*pwc)) 2424 return (-3); 2425 return (cnt); 2426 } 2427 2428 static inline uint32_t 2429 combine_surrogate_pair(uint32_t uc, uint32_t uc2) 2430 { 2431 uc -= 0xD800; 2432 uc *= 0x400; 2433 uc += uc2 - 0xDC00; 2434 uc += 0x10000; 2435 return (uc); 2436 } 2437 2438 /* 2439 * Convert a single UTF-8/CESU-8 sequence to a Unicode code point in 2440 * removing surrogate pairs. 2441 * 2442 * CESU-8: The Compatibility Encoding Scheme for UTF-16. 2443 * 2444 * Usually return used bytes, return used byte in negative value when 2445 * a unicode character is replaced with U+FFFD. 2446 */ 2447 static int 2448 cesu8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2449 { 2450 uint32_t wc = 0; 2451 int cnt; 2452 2453 cnt = _utf8_to_unicode(&wc, s, n); 2454 if (cnt == 3 && IS_HIGH_SURROGATE_LA(wc)) { 2455 uint32_t wc2 = 0; 2456 if (n - 3 < 3) { 2457 /* Invalid byte sequence. */ 2458 goto invalid_sequence; 2459 } 2460 cnt = _utf8_to_unicode(&wc2, s+3, n-3); 2461 if (cnt != 3 || !IS_LOW_SURROGATE_LA(wc2)) { 2462 /* Invalid byte sequence. */ 2463 goto invalid_sequence; 2464 } 2465 wc = combine_surrogate_pair(wc, wc2); 2466 cnt = 6; 2467 } else if (cnt == 3 && IS_LOW_SURROGATE_LA(wc)) { 2468 /* Invalid byte sequence. */ 2469 goto invalid_sequence; 2470 } 2471 *pwc = wc; 2472 return (cnt); 2473 invalid_sequence: 2474 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2475 if (cnt > 0) 2476 cnt *= -1; 2477 return (cnt); 2478 } 2479 2480 /* 2481 * Convert a Unicode code point to a single UTF-8 sequence. 2482 * 2483 * NOTE:This function does not check if the Unicode is leagal or not. 2484 * Please you definitely check it before calling this. 2485 */ 2486 static size_t 2487 unicode_to_utf8(char *p, size_t remaining, uint32_t uc) 2488 { 2489 char *_p = p; 2490 2491 /* Translate code point to UTF8 */ 2492 if (uc <= 0x7f) { 2493 if (remaining == 0) 2494 return (0); 2495 *p++ = (char)uc; 2496 } else if (uc <= 0x7ff) { 2497 if (remaining < 2) 2498 return (0); 2499 *p++ = 0xc0 | ((uc >> 6) & 0x1f); 2500 *p++ = 0x80 | (uc & 0x3f); 2501 } else if (uc <= 0xffff) { 2502 if (remaining < 3) 2503 return (0); 2504 *p++ = 0xe0 | ((uc >> 12) & 0x0f); 2505 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2506 *p++ = 0x80 | (uc & 0x3f); 2507 } else if (uc <= UNICODE_MAX) { 2508 if (remaining < 4) 2509 return (0); 2510 *p++ = 0xf0 | ((uc >> 18) & 0x07); 2511 *p++ = 0x80 | ((uc >> 12) & 0x3f); 2512 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2513 *p++ = 0x80 | (uc & 0x3f); 2514 } else { 2515 /* 2516 * Undescribed code point should be U+FFFD 2517 * (replacement character). 2518 */ 2519 if (remaining < UTF8_R_CHAR_SIZE) 2520 return (0); 2521 UTF8_SET_R_CHAR(p); 2522 p += UTF8_R_CHAR_SIZE; 2523 } 2524 return (p - _p); 2525 } 2526 2527 static int 2528 utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n) 2529 { 2530 return (utf16_to_unicode(pwc, s, n, 1)); 2531 } 2532 2533 static int 2534 utf16le_to_unicode(uint32_t *pwc, const char *s, size_t n) 2535 { 2536 return (utf16_to_unicode(pwc, s, n, 0)); 2537 } 2538 2539 static int 2540 utf16_to_unicode(uint32_t *pwc, const char *s, size_t n, int be) 2541 { 2542 const char *utf16 = s; 2543 unsigned uc; 2544 2545 if (n == 0) 2546 return (0); 2547 if (n == 1) { 2548 /* set the Replacement Character instead. */ 2549 *pwc = UNICODE_R_CHAR; 2550 return (-1); 2551 } 2552 2553 if (be) 2554 uc = archive_be16dec(utf16); 2555 else 2556 uc = archive_le16dec(utf16); 2557 utf16 += 2; 2558 2559 /* If this is a surrogate pair, assemble the full code point.*/ 2560 if (IS_HIGH_SURROGATE_LA(uc)) { 2561 unsigned uc2; 2562 2563 if (n >= 4) { 2564 if (be) 2565 uc2 = archive_be16dec(utf16); 2566 else 2567 uc2 = archive_le16dec(utf16); 2568 } else 2569 uc2 = 0; 2570 if (IS_LOW_SURROGATE_LA(uc2)) { 2571 uc = combine_surrogate_pair(uc, uc2); 2572 utf16 += 2; 2573 } else { 2574 /* Undescribed code point should be U+FFFD 2575 * (replacement character). */ 2576 *pwc = UNICODE_R_CHAR; 2577 return (-2); 2578 } 2579 } 2580 2581 /* 2582 * Surrogate pair values(0xd800 through 0xdfff) are only 2583 * used by UTF-16, so, after above culculation, the code 2584 * must not be surrogate values, and Unicode has no codes 2585 * larger than 0x10ffff. Thus, those are not leagal Unicode 2586 * values. 2587 */ 2588 if (IS_SURROGATE_PAIR_LA(uc) || uc > UNICODE_MAX) { 2589 /* Undescribed code point should be U+FFFD 2590 * (replacement character). */ 2591 *pwc = UNICODE_R_CHAR; 2592 return (((int)(utf16 - s)) * -1); 2593 } 2594 *pwc = uc; 2595 return ((int)(utf16 - s)); 2596 } 2597 2598 static size_t 2599 unicode_to_utf16be(char *p, size_t remaining, uint32_t uc) 2600 { 2601 char *utf16 = p; 2602 2603 if (uc > 0xffff) { 2604 /* We have a code point that won't fit into a 2605 * wchar_t; convert it to a surrogate pair. */ 2606 if (remaining < 4) 2607 return (0); 2608 uc -= 0x10000; 2609 archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2610 archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2611 return (4); 2612 } else { 2613 if (remaining < 2) 2614 return (0); 2615 archive_be16enc(utf16, uc); 2616 return (2); 2617 } 2618 } 2619 2620 static size_t 2621 unicode_to_utf16le(char *p, size_t remaining, uint32_t uc) 2622 { 2623 char *utf16 = p; 2624 2625 if (uc > 0xffff) { 2626 /* We have a code point that won't fit into a 2627 * wchar_t; convert it to a surrogate pair. */ 2628 if (remaining < 4) 2629 return (0); 2630 uc -= 0x10000; 2631 archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2632 archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2633 return (4); 2634 } else { 2635 if (remaining < 2) 2636 return (0); 2637 archive_le16enc(utf16, uc); 2638 return (2); 2639 } 2640 } 2641 2642 /* 2643 * Copy UTF-8 string in checking surrogate pair. 2644 * If any surrogate pair are found, it would be canonicalized. 2645 */ 2646 static int 2647 strncat_from_utf8_to_utf8(struct archive_string *as, const void *_p, 2648 size_t len, struct archive_string_conv *sc) 2649 { 2650 const char *s; 2651 char *p, *endp; 2652 int n, ret = 0; 2653 2654 (void)sc; /* UNUSED */ 2655 2656 if (archive_string_ensure(as, as->length + len + 1) == NULL) 2657 return (-1); 2658 2659 s = (const char *)_p; 2660 p = as->s + as->length; 2661 endp = as->s + as->buffer_length -1; 2662 do { 2663 uint32_t uc; 2664 const char *ss = s; 2665 size_t w; 2666 2667 /* 2668 * Forward byte sequence until a conversion of that is needed. 2669 */ 2670 while ((n = utf8_to_unicode(&uc, s, len)) > 0) { 2671 s += n; 2672 len -= n; 2673 } 2674 if (ss < s) { 2675 if (p + (s - ss) > endp) { 2676 as->length = p - as->s; 2677 if (archive_string_ensure(as, 2678 as->buffer_length + len + 1) == NULL) 2679 return (-1); 2680 p = as->s + as->length; 2681 endp = as->s + as->buffer_length -1; 2682 } 2683 2684 memcpy(p, ss, s - ss); 2685 p += s - ss; 2686 } 2687 2688 /* 2689 * If n is negative, current byte sequence needs a replacement. 2690 */ 2691 if (n < 0) { 2692 if (n == -3 && IS_SURROGATE_PAIR_LA(uc)) { 2693 /* Current byte sequence may be CESU-8. */ 2694 n = cesu8_to_unicode(&uc, s, len); 2695 } 2696 if (n < 0) { 2697 ret = -1; 2698 n *= -1;/* Use a replaced unicode character. */ 2699 } 2700 2701 /* Rebuild UTF-8 byte sequence. */ 2702 while ((w = unicode_to_utf8(p, endp - p, uc)) == 0) { 2703 as->length = p - as->s; 2704 if (archive_string_ensure(as, 2705 as->buffer_length + len + 1) == NULL) 2706 return (-1); 2707 p = as->s + as->length; 2708 endp = as->s + as->buffer_length -1; 2709 } 2710 p += w; 2711 s += n; 2712 len -= n; 2713 } 2714 } while (n > 0); 2715 as->length = p - as->s; 2716 as->s[as->length] = '\0'; 2717 return (ret); 2718 } 2719 2720 static int 2721 archive_string_append_unicode(struct archive_string *as, const void *_p, 2722 size_t len, struct archive_string_conv *sc) 2723 { 2724 const char *s; 2725 char *p, *endp; 2726 uint32_t uc; 2727 size_t w; 2728 int n, ret = 0, ts, tm; 2729 int (*parse)(uint32_t *, const char *, size_t); 2730 size_t (*unparse)(char *, size_t, uint32_t); 2731 2732 if (sc->flag & SCONV_TO_UTF16BE) { 2733 unparse = unicode_to_utf16be; 2734 ts = 2; 2735 } else if (sc->flag & SCONV_TO_UTF16LE) { 2736 unparse = unicode_to_utf16le; 2737 ts = 2; 2738 } else if (sc->flag & SCONV_TO_UTF8) { 2739 unparse = unicode_to_utf8; 2740 ts = 1; 2741 } else { 2742 /* 2743 * This case is going to be converted to another 2744 * character-set through iconv. 2745 */ 2746 if (sc->flag & SCONV_FROM_UTF16BE) { 2747 unparse = unicode_to_utf16be; 2748 ts = 2; 2749 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2750 unparse = unicode_to_utf16le; 2751 ts = 2; 2752 } else { 2753 unparse = unicode_to_utf8; 2754 ts = 1; 2755 } 2756 } 2757 2758 if (sc->flag & SCONV_FROM_UTF16BE) { 2759 parse = utf16be_to_unicode; 2760 tm = 1; 2761 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2762 parse = utf16le_to_unicode; 2763 tm = 1; 2764 } else { 2765 parse = cesu8_to_unicode; 2766 tm = ts; 2767 } 2768 2769 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2770 return (-1); 2771 2772 s = (const char *)_p; 2773 p = as->s + as->length; 2774 endp = as->s + as->buffer_length - ts; 2775 while ((n = parse(&uc, s, len)) != 0) { 2776 if (n < 0) { 2777 /* Use a replaced unicode character. */ 2778 n *= -1; 2779 ret = -1; 2780 } 2781 s += n; 2782 len -= n; 2783 while ((w = unparse(p, endp - p, uc)) == 0) { 2784 /* There is not enough output buffer so 2785 * we have to expand it. */ 2786 as->length = p - as->s; 2787 if (archive_string_ensure(as, 2788 as->buffer_length + len * tm + ts) == NULL) 2789 return (-1); 2790 p = as->s + as->length; 2791 endp = as->s + as->buffer_length - ts; 2792 } 2793 p += w; 2794 } 2795 as->length = p - as->s; 2796 as->s[as->length] = '\0'; 2797 if (ts == 2) 2798 as->s[as->length+1] = '\0'; 2799 return (ret); 2800 } 2801 2802 /* 2803 * Following Constants for Hangul compositions this information comes from 2804 * Unicode Standard Annex #15 http://unicode.org/reports/tr15/ 2805 */ 2806 #define HC_SBASE 0xAC00 2807 #define HC_LBASE 0x1100 2808 #define HC_VBASE 0x1161 2809 #define HC_TBASE 0x11A7 2810 #define HC_LCOUNT 19 2811 #define HC_VCOUNT 21 2812 #define HC_TCOUNT 28 2813 #define HC_NCOUNT (HC_VCOUNT * HC_TCOUNT) 2814 #define HC_SCOUNT (HC_LCOUNT * HC_NCOUNT) 2815 2816 static uint32_t 2817 get_nfc(uint32_t uc, uint32_t uc2) 2818 { 2819 int t, b; 2820 2821 t = 0; 2822 b = sizeof(u_composition_table)/sizeof(u_composition_table[0]) -1; 2823 while (b >= t) { 2824 int m = (t + b) / 2; 2825 if (u_composition_table[m].cp1 < uc) 2826 t = m + 1; 2827 else if (u_composition_table[m].cp1 > uc) 2828 b = m - 1; 2829 else if (u_composition_table[m].cp2 < uc2) 2830 t = m + 1; 2831 else if (u_composition_table[m].cp2 > uc2) 2832 b = m - 1; 2833 else 2834 return (u_composition_table[m].nfc); 2835 } 2836 return (0); 2837 } 2838 2839 #define FDC_MAX 10 /* The maximum number of Following Decomposable 2840 * Characters. */ 2841 2842 /* 2843 * Update first code point. 2844 */ 2845 #define UPDATE_UC(new_uc) do { \ 2846 uc = new_uc; \ 2847 ucptr = NULL; \ 2848 } while (0) 2849 2850 /* 2851 * Replace first code point with second code point. 2852 */ 2853 #define REPLACE_UC_WITH_UC2() do { \ 2854 uc = uc2; \ 2855 ucptr = uc2ptr; \ 2856 n = n2; \ 2857 } while (0) 2858 2859 #define EXPAND_BUFFER() do { \ 2860 as->length = p - as->s; \ 2861 if (archive_string_ensure(as, \ 2862 as->buffer_length + len * tm + ts) == NULL)\ 2863 return (-1); \ 2864 p = as->s + as->length; \ 2865 endp = as->s + as->buffer_length - ts; \ 2866 } while (0) 2867 2868 #define UNPARSE(p, endp, uc) do { \ 2869 while ((w = unparse(p, (endp) - (p), uc)) == 0) {\ 2870 EXPAND_BUFFER(); \ 2871 } \ 2872 p += w; \ 2873 } while (0) 2874 2875 /* 2876 * Write first code point. 2877 * If the code point has not be changed from its original code, 2878 * this just copies it from its original buffer pointer. 2879 * If not, this converts it to UTF-8 byte sequence and copies it. 2880 */ 2881 #define WRITE_UC() do { \ 2882 if (ucptr) { \ 2883 if (p + n > endp) \ 2884 EXPAND_BUFFER(); \ 2885 switch (n) { \ 2886 case 4: \ 2887 *p++ = *ucptr++; \ 2888 /* FALL THROUGH */ \ 2889 case 3: \ 2890 *p++ = *ucptr++; \ 2891 /* FALL THROUGH */ \ 2892 case 2: \ 2893 *p++ = *ucptr++; \ 2894 /* FALL THROUGH */ \ 2895 case 1: \ 2896 *p++ = *ucptr; \ 2897 break; \ 2898 } \ 2899 ucptr = NULL; \ 2900 } else { \ 2901 UNPARSE(p, endp, uc); \ 2902 } \ 2903 } while (0) 2904 2905 /* 2906 * Collect following decomposable code points. 2907 */ 2908 #define COLLECT_CPS(start) do { \ 2909 int _i; \ 2910 for (_i = start; _i < FDC_MAX ; _i++) { \ 2911 nx = parse(&ucx[_i], s, len); \ 2912 if (nx <= 0) \ 2913 break; \ 2914 cx = CCC(ucx[_i]); \ 2915 if (cl >= cx && cl != 228 && cx != 228)\ 2916 break; \ 2917 s += nx; \ 2918 len -= nx; \ 2919 cl = cx; \ 2920 ccx[_i] = cx; \ 2921 } \ 2922 if (_i >= FDC_MAX) { \ 2923 ret = -1; \ 2924 ucx_size = FDC_MAX; \ 2925 } else \ 2926 ucx_size = _i; \ 2927 } while (0) 2928 2929 /* 2930 * Normalize UTF-8/UTF-16BE characters to Form C and copy the result. 2931 * 2932 * TODO: Convert composition exclusions,which are never converted 2933 * from NFC,NFD,NFKC and NFKD, to Form C. 2934 */ 2935 static int 2936 archive_string_normalize_C(struct archive_string *as, const void *_p, 2937 size_t len, struct archive_string_conv *sc) 2938 { 2939 const char *s = (const char *)_p; 2940 char *p, *endp; 2941 uint32_t uc, uc2; 2942 size_t w; 2943 int always_replace, n, n2, ret = 0, spair, ts, tm; 2944 int (*parse)(uint32_t *, const char *, size_t); 2945 size_t (*unparse)(char *, size_t, uint32_t); 2946 2947 always_replace = 1; 2948 ts = 1;/* text size. */ 2949 if (sc->flag & SCONV_TO_UTF16BE) { 2950 unparse = unicode_to_utf16be; 2951 ts = 2; 2952 if (sc->flag & SCONV_FROM_UTF16BE) 2953 always_replace = 0; 2954 } else if (sc->flag & SCONV_TO_UTF16LE) { 2955 unparse = unicode_to_utf16le; 2956 ts = 2; 2957 if (sc->flag & SCONV_FROM_UTF16LE) 2958 always_replace = 0; 2959 } else if (sc->flag & SCONV_TO_UTF8) { 2960 unparse = unicode_to_utf8; 2961 if (sc->flag & SCONV_FROM_UTF8) 2962 always_replace = 0; 2963 } else { 2964 /* 2965 * This case is going to be converted to another 2966 * character-set through iconv. 2967 */ 2968 always_replace = 0; 2969 if (sc->flag & SCONV_FROM_UTF16BE) { 2970 unparse = unicode_to_utf16be; 2971 ts = 2; 2972 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2973 unparse = unicode_to_utf16le; 2974 ts = 2; 2975 } else { 2976 unparse = unicode_to_utf8; 2977 } 2978 } 2979 2980 if (sc->flag & SCONV_FROM_UTF16BE) { 2981 parse = utf16be_to_unicode; 2982 tm = 1; 2983 spair = 4;/* surrogate pair size in UTF-16. */ 2984 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2985 parse = utf16le_to_unicode; 2986 tm = 1; 2987 spair = 4;/* surrogate pair size in UTF-16. */ 2988 } else { 2989 parse = cesu8_to_unicode; 2990 tm = ts; 2991 spair = 6;/* surrogate pair size in UTF-8. */ 2992 } 2993 2994 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2995 return (-1); 2996 2997 p = as->s + as->length; 2998 endp = as->s + as->buffer_length - ts; 2999 while ((n = parse(&uc, s, len)) != 0) { 3000 const char *ucptr, *uc2ptr; 3001 3002 if (n < 0) { 3003 /* Use a replaced unicode character. */ 3004 UNPARSE(p, endp, uc); 3005 s += n*-1; 3006 len -= n*-1; 3007 ret = -1; 3008 continue; 3009 } else if (n == spair || always_replace) 3010 /* uc is converted from a surrogate pair. 3011 * this should be treated as a changed code. */ 3012 ucptr = NULL; 3013 else 3014 ucptr = s; 3015 s += n; 3016 len -= n; 3017 3018 /* Read second code point. */ 3019 while ((n2 = parse(&uc2, s, len)) > 0) { 3020 uint32_t ucx[FDC_MAX]; 3021 int ccx[FDC_MAX]; 3022 int cl, cx, i, nx, ucx_size; 3023 int LIndex,SIndex; 3024 uint32_t nfc; 3025 3026 if (n2 == spair || always_replace) 3027 /* uc2 is converted from a surrogate pair. 3028 * this should be treated as a changed code. */ 3029 uc2ptr = NULL; 3030 else 3031 uc2ptr = s; 3032 s += n2; 3033 len -= n2; 3034 3035 /* 3036 * If current second code point is out of decomposable 3037 * code points, finding compositions is unneeded. 3038 */ 3039 if (!IS_DECOMPOSABLE_BLOCK(uc2)) { 3040 WRITE_UC(); 3041 REPLACE_UC_WITH_UC2(); 3042 continue; 3043 } 3044 3045 /* 3046 * Try to combine current code points. 3047 */ 3048 /* 3049 * We have to combine Hangul characters according to 3050 * http://uniicode.org/reports/tr15/#Hangul 3051 */ 3052 if (0 <= (LIndex = uc - HC_LBASE) && 3053 LIndex < HC_LCOUNT) { 3054 /* 3055 * Hangul Composition. 3056 * 1. Two current code points are L and V. 3057 */ 3058 int VIndex = uc2 - HC_VBASE; 3059 if (0 <= VIndex && VIndex < HC_VCOUNT) { 3060 /* Make syllable of form LV. */ 3061 UPDATE_UC(HC_SBASE + 3062 (LIndex * HC_VCOUNT + VIndex) * 3063 HC_TCOUNT); 3064 } else { 3065 WRITE_UC(); 3066 REPLACE_UC_WITH_UC2(); 3067 } 3068 continue; 3069 } else if (0 <= (SIndex = uc - HC_SBASE) && 3070 SIndex < HC_SCOUNT && (SIndex % HC_TCOUNT) == 0) { 3071 /* 3072 * Hangul Composition. 3073 * 2. Two current code points are LV and T. 3074 */ 3075 int TIndex = uc2 - HC_TBASE; 3076 if (0 < TIndex && TIndex < HC_TCOUNT) { 3077 /* Make syllable of form LVT. */ 3078 UPDATE_UC(uc + TIndex); 3079 } else { 3080 WRITE_UC(); 3081 REPLACE_UC_WITH_UC2(); 3082 } 3083 continue; 3084 } else if ((nfc = get_nfc(uc, uc2)) != 0) { 3085 /* A composition to current code points 3086 * is found. */ 3087 UPDATE_UC(nfc); 3088 continue; 3089 } else if ((cl = CCC(uc2)) == 0) { 3090 /* Clearly 'uc2' the second code point is not 3091 * a decomposable code. */ 3092 WRITE_UC(); 3093 REPLACE_UC_WITH_UC2(); 3094 continue; 3095 } 3096 3097 /* 3098 * Collect following decomposable code points. 3099 */ 3100 cx = 0; 3101 ucx[0] = uc2; 3102 ccx[0] = cl; 3103 COLLECT_CPS(1); 3104 3105 /* 3106 * Find a composed code in the collected code points. 3107 */ 3108 i = 1; 3109 while (i < ucx_size) { 3110 int j; 3111 3112 if ((nfc = get_nfc(uc, ucx[i])) == 0) { 3113 i++; 3114 continue; 3115 } 3116 3117 /* 3118 * nfc is composed of uc and ucx[i]. 3119 */ 3120 UPDATE_UC(nfc); 3121 3122 /* 3123 * Remove ucx[i] by shifting 3124 * following code points. 3125 */ 3126 for (j = i; j+1 < ucx_size; j++) { 3127 ucx[j] = ucx[j+1]; 3128 ccx[j] = ccx[j+1]; 3129 } 3130 ucx_size --; 3131 3132 /* 3133 * Collect following code points blocked 3134 * by ucx[i] the removed code point. 3135 */ 3136 if (ucx_size > 0 && i == ucx_size && 3137 nx > 0 && cx == cl) { 3138 cl = ccx[ucx_size-1]; 3139 COLLECT_CPS(ucx_size); 3140 } 3141 /* 3142 * Restart finding a composed code with 3143 * the updated uc from the top of the 3144 * collected code points. 3145 */ 3146 i = 0; 3147 } 3148 3149 /* 3150 * Apparently the current code points are not 3151 * decomposed characters or already composed. 3152 */ 3153 WRITE_UC(); 3154 for (i = 0; i < ucx_size; i++) 3155 UNPARSE(p, endp, ucx[i]); 3156 3157 /* 3158 * Flush out remaining canonical combining characters. 3159 */ 3160 if (nx > 0 && cx == cl && len > 0) { 3161 while ((nx = parse(&ucx[0], s, len)) 3162 > 0) { 3163 cx = CCC(ucx[0]); 3164 if (cl > cx) 3165 break; 3166 s += nx; 3167 len -= nx; 3168 cl = cx; 3169 UNPARSE(p, endp, ucx[0]); 3170 } 3171 } 3172 break; 3173 } 3174 if (n2 < 0) { 3175 WRITE_UC(); 3176 /* Use a replaced unicode character. */ 3177 UNPARSE(p, endp, uc2); 3178 s += n2*-1; 3179 len -= n2*-1; 3180 ret = -1; 3181 continue; 3182 } else if (n2 == 0) { 3183 WRITE_UC(); 3184 break; 3185 } 3186 } 3187 as->length = p - as->s; 3188 as->s[as->length] = '\0'; 3189 if (ts == 2) 3190 as->s[as->length+1] = '\0'; 3191 return (ret); 3192 } 3193 3194 static int 3195 get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc) 3196 { 3197 int t, b; 3198 3199 /* 3200 * These are not converted to NFD on Mac OS. 3201 */ 3202 if ((uc >= 0x2000 && uc <= 0x2FFF) || 3203 (uc >= 0xF900 && uc <= 0xFAFF) || 3204 (uc >= 0x2F800 && uc <= 0x2FAFF)) 3205 return (0); 3206 /* 3207 * Those code points are not converted to NFD on Mac OS. 3208 * I do not know the reason because it is undocumented. 3209 * NFC NFD 3210 * 1109A ==> 11099 110BA 3211 * 1109C ==> 1109B 110BA 3212 * 110AB ==> 110A5 110BA 3213 */ 3214 if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB) 3215 return (0); 3216 3217 t = 0; 3218 b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1; 3219 while (b >= t) { 3220 int m = (t + b) / 2; 3221 if (u_decomposition_table[m].nfc < uc) 3222 t = m + 1; 3223 else if (u_decomposition_table[m].nfc > uc) 3224 b = m - 1; 3225 else { 3226 *cp1 = u_decomposition_table[m].cp1; 3227 *cp2 = u_decomposition_table[m].cp2; 3228 return (1); 3229 } 3230 } 3231 return (0); 3232 } 3233 3234 #define REPLACE_UC_WITH(cp) do { \ 3235 uc = cp; \ 3236 ucptr = NULL; \ 3237 } while (0) 3238 3239 /* 3240 * Normalize UTF-8 characters to Form D and copy the result. 3241 */ 3242 static int 3243 archive_string_normalize_D(struct archive_string *as, const void *_p, 3244 size_t len, struct archive_string_conv *sc) 3245 { 3246 const char *s = (const char *)_p; 3247 char *p, *endp; 3248 uint32_t uc, uc2; 3249 size_t w; 3250 int always_replace, n, n2, ret = 0, spair, ts, tm; 3251 int (*parse)(uint32_t *, const char *, size_t); 3252 size_t (*unparse)(char *, size_t, uint32_t); 3253 3254 always_replace = 1; 3255 ts = 1;/* text size. */ 3256 if (sc->flag & SCONV_TO_UTF16BE) { 3257 unparse = unicode_to_utf16be; 3258 ts = 2; 3259 if (sc->flag & SCONV_FROM_UTF16BE) 3260 always_replace = 0; 3261 } else if (sc->flag & SCONV_TO_UTF16LE) { 3262 unparse = unicode_to_utf16le; 3263 ts = 2; 3264 if (sc->flag & SCONV_FROM_UTF16LE) 3265 always_replace = 0; 3266 } else if (sc->flag & SCONV_TO_UTF8) { 3267 unparse = unicode_to_utf8; 3268 if (sc->flag & SCONV_FROM_UTF8) 3269 always_replace = 0; 3270 } else { 3271 /* 3272 * This case is going to be converted to another 3273 * character-set through iconv. 3274 */ 3275 always_replace = 0; 3276 if (sc->flag & SCONV_FROM_UTF16BE) { 3277 unparse = unicode_to_utf16be; 3278 ts = 2; 3279 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3280 unparse = unicode_to_utf16le; 3281 ts = 2; 3282 } else { 3283 unparse = unicode_to_utf8; 3284 } 3285 } 3286 3287 if (sc->flag & SCONV_FROM_UTF16BE) { 3288 parse = utf16be_to_unicode; 3289 tm = 1; 3290 spair = 4;/* surrogate pair size in UTF-16. */ 3291 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3292 parse = utf16le_to_unicode; 3293 tm = 1; 3294 spair = 4;/* surrogate pair size in UTF-16. */ 3295 } else { 3296 parse = cesu8_to_unicode; 3297 tm = ts; 3298 spair = 6;/* surrogate pair size in UTF-8. */ 3299 } 3300 3301 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 3302 return (-1); 3303 3304 p = as->s + as->length; 3305 endp = as->s + as->buffer_length - ts; 3306 while ((n = parse(&uc, s, len)) != 0) { 3307 const char *ucptr; 3308 uint32_t cp1, cp2; 3309 int SIndex; 3310 struct { 3311 uint32_t uc; 3312 int ccc; 3313 } fdc[FDC_MAX]; 3314 int fdi, fdj; 3315 int ccc; 3316 3317 check_first_code: 3318 if (n < 0) { 3319 /* Use a replaced unicode character. */ 3320 UNPARSE(p, endp, uc); 3321 s += n*-1; 3322 len -= n*-1; 3323 ret = -1; 3324 continue; 3325 } else if (n == spair || always_replace) 3326 /* uc is converted from a surrogate pair. 3327 * this should be treated as a changed code. */ 3328 ucptr = NULL; 3329 else 3330 ucptr = s; 3331 s += n; 3332 len -= n; 3333 3334 /* Hangul Decomposition. */ 3335 if ((SIndex = uc - HC_SBASE) >= 0 && SIndex < HC_SCOUNT) { 3336 int L = HC_LBASE + SIndex / HC_NCOUNT; 3337 int V = HC_VBASE + (SIndex % HC_NCOUNT) / HC_TCOUNT; 3338 int T = HC_TBASE + SIndex % HC_TCOUNT; 3339 3340 REPLACE_UC_WITH(L); 3341 WRITE_UC(); 3342 REPLACE_UC_WITH(V); 3343 WRITE_UC(); 3344 if (T != HC_TBASE) { 3345 REPLACE_UC_WITH(T); 3346 WRITE_UC(); 3347 } 3348 continue; 3349 } 3350 if (IS_DECOMPOSABLE_BLOCK(uc) && CCC(uc) != 0) { 3351 WRITE_UC(); 3352 continue; 3353 } 3354 3355 fdi = 0; 3356 while (get_nfd(&cp1, &cp2, uc) && fdi < FDC_MAX) { 3357 int k; 3358 3359 for (k = fdi; k > 0; k--) 3360 fdc[k] = fdc[k-1]; 3361 fdc[0].ccc = CCC(cp2); 3362 fdc[0].uc = cp2; 3363 fdi++; 3364 REPLACE_UC_WITH(cp1); 3365 } 3366 3367 /* Read following code points. */ 3368 while ((n2 = parse(&uc2, s, len)) > 0 && 3369 (ccc = CCC(uc2)) != 0 && fdi < FDC_MAX) { 3370 int j, k; 3371 3372 s += n2; 3373 len -= n2; 3374 for (j = 0; j < fdi; j++) { 3375 if (fdc[j].ccc > ccc) 3376 break; 3377 } 3378 if (j < fdi) { 3379 for (k = fdi; k > j; k--) 3380 fdc[k] = fdc[k-1]; 3381 fdc[j].ccc = ccc; 3382 fdc[j].uc = uc2; 3383 } else { 3384 fdc[fdi].ccc = ccc; 3385 fdc[fdi].uc = uc2; 3386 } 3387 fdi++; 3388 } 3389 3390 WRITE_UC(); 3391 for (fdj = 0; fdj < fdi; fdj++) { 3392 REPLACE_UC_WITH(fdc[fdj].uc); 3393 WRITE_UC(); 3394 } 3395 3396 if (n2 == 0) 3397 break; 3398 REPLACE_UC_WITH(uc2); 3399 n = n2; 3400 goto check_first_code; 3401 } 3402 as->length = p - as->s; 3403 as->s[as->length] = '\0'; 3404 if (ts == 2) 3405 as->s[as->length+1] = '\0'; 3406 return (ret); 3407 } 3408 3409 /* 3410 * libarchive 2.x made incorrect UTF-8 strings in the wrong assumption 3411 * that WCS is Unicode. It is true for several platforms but some are false. 3412 * And then people who did not use UTF-8 locale on the non Unicode WCS 3413 * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those 3414 * now cannot get right filename from libarchive 3.x and later since we 3415 * fixed the wrong assumption and it is incompatible to older its versions. 3416 * So we provide special option, "compat-2x.x", for resolving it. 3417 * That option enable the string conversion of libarchive 2.x. 3418 * 3419 * Translates the wrong UTF-8 string made by libarchive 2.x into current 3420 * locale character set and appends to the archive_string. 3421 * Note: returns -1 if conversion fails. 3422 */ 3423 static int 3424 strncat_from_utf8_libarchive2(struct archive_string *as, 3425 const void *_p, size_t len, struct archive_string_conv *sc) 3426 { 3427 const char *s; 3428 int n; 3429 char *p; 3430 char *end; 3431 uint32_t unicode; 3432 #if HAVE_WCRTOMB 3433 mbstate_t shift_state; 3434 3435 memset(&shift_state, 0, sizeof(shift_state)); 3436 #else 3437 /* Clear the shift state before starting. */ 3438 wctomb(NULL, L'\0'); 3439 #endif 3440 (void)sc; /* UNUSED */ 3441 /* 3442 * Allocate buffer for MBS. 3443 * We need this allocation here since it is possible that 3444 * as->s is still NULL. 3445 */ 3446 if (archive_string_ensure(as, as->length + len + 1) == NULL) 3447 return (-1); 3448 3449 s = (const char *)_p; 3450 p = as->s + as->length; 3451 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3452 while ((n = _utf8_to_unicode(&unicode, s, len)) != 0) { 3453 wchar_t wc; 3454 3455 if (p >= end) { 3456 as->length = p - as->s; 3457 /* Re-allocate buffer for MBS. */ 3458 if (archive_string_ensure(as, 3459 as->length + len * 2 + 1) == NULL) 3460 return (-1); 3461 p = as->s + as->length; 3462 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3463 } 3464 3465 /* 3466 * As libarchie 2.x, translates the UTF-8 characters into 3467 * wide-characters in the assumption that WCS is Unicode. 3468 */ 3469 if (n < 0) { 3470 n *= -1; 3471 wc = L'?'; 3472 } else 3473 wc = (wchar_t)unicode; 3474 3475 s += n; 3476 len -= n; 3477 /* 3478 * Translates the wide-character into the current locale MBS. 3479 */ 3480 #if HAVE_WCRTOMB 3481 n = (int)wcrtomb(p, wc, &shift_state); 3482 #else 3483 n = (int)wctomb(p, wc); 3484 #endif 3485 if (n == -1) 3486 return (-1); 3487 p += n; 3488 } 3489 as->length = p - as->s; 3490 as->s[as->length] = '\0'; 3491 return (0); 3492 } 3493 3494 3495 /* 3496 * Conversion functions between current locale dependent MBS and UTF-16BE. 3497 * strncat_from_utf16be() : UTF-16BE --> MBS 3498 * strncat_to_utf16be() : MBS --> UTF16BE 3499 */ 3500 3501 #if defined(_WIN32) && !defined(__CYGWIN__) 3502 3503 /* 3504 * Convert a UTF-16BE/LE string to current locale and copy the result. 3505 * Return -1 if conversion failes. 3506 */ 3507 static int 3508 win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes, 3509 struct archive_string_conv *sc, int be) 3510 { 3511 struct archive_string tmp; 3512 const char *u16; 3513 int ll; 3514 BOOL defchar; 3515 char *mbs; 3516 size_t mbs_size, b; 3517 int ret = 0; 3518 3519 bytes &= ~1; 3520 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3521 return (-1); 3522 3523 mbs = as->s + as->length; 3524 mbs_size = as->buffer_length - as->length -1; 3525 3526 if (sc->to_cp == CP_C_LOCALE) { 3527 /* 3528 * "C" locale special process. 3529 */ 3530 u16 = _p; 3531 ll = 0; 3532 for (b = 0; b < bytes; b += 2) { 3533 uint16_t val; 3534 if (be) 3535 val = archive_be16dec(u16+b); 3536 else 3537 val = archive_le16dec(u16+b); 3538 if (val > 255) { 3539 *mbs++ = '?'; 3540 ret = -1; 3541 } else 3542 *mbs++ = (char)(val&0xff); 3543 ll++; 3544 } 3545 as->length += ll; 3546 as->s[as->length] = '\0'; 3547 return (ret); 3548 } 3549 3550 archive_string_init(&tmp); 3551 if (be) { 3552 if (is_big_endian()) { 3553 u16 = _p; 3554 } else { 3555 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3556 return (-1); 3557 memcpy(tmp.s, _p, bytes); 3558 for (b = 0; b < bytes; b += 2) { 3559 uint16_t val = archive_be16dec(tmp.s+b); 3560 archive_le16enc(tmp.s+b, val); 3561 } 3562 u16 = tmp.s; 3563 } 3564 } else { 3565 if (!is_big_endian()) { 3566 u16 = _p; 3567 } else { 3568 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3569 return (-1); 3570 memcpy(tmp.s, _p, bytes); 3571 for (b = 0; b < bytes; b += 2) { 3572 uint16_t val = archive_le16dec(tmp.s+b); 3573 archive_be16enc(tmp.s+b, val); 3574 } 3575 u16 = tmp.s; 3576 } 3577 } 3578 3579 do { 3580 defchar = 0; 3581 ll = WideCharToMultiByte(sc->to_cp, 0, 3582 (LPCWSTR)u16, (int)bytes>>1, mbs, (int)mbs_size, 3583 NULL, &defchar); 3584 if (ll == 0 && 3585 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 3586 /* Need more buffer for MBS. */ 3587 ll = WideCharToMultiByte(sc->to_cp, 0, 3588 (LPCWSTR)u16, (int)bytes, NULL, 0, NULL, NULL); 3589 if (archive_string_ensure(as, ll +1) == NULL) 3590 return (-1); 3591 mbs = as->s + as->length; 3592 mbs_size = as->buffer_length - as->length -1; 3593 continue; 3594 } 3595 } while (0); 3596 archive_string_free(&tmp); 3597 as->length += ll; 3598 as->s[as->length] = '\0'; 3599 if (ll == 0 || defchar) 3600 ret = -1; 3601 return (ret); 3602 } 3603 3604 static int 3605 win_strncat_from_utf16be(struct archive_string *as, const void *_p, 3606 size_t bytes, struct archive_string_conv *sc) 3607 { 3608 return (win_strncat_from_utf16(as, _p, bytes, sc, 1)); 3609 } 3610 3611 static int 3612 win_strncat_from_utf16le(struct archive_string *as, const void *_p, 3613 size_t bytes, struct archive_string_conv *sc) 3614 { 3615 return (win_strncat_from_utf16(as, _p, bytes, sc, 0)); 3616 } 3617 3618 static int 3619 is_big_endian(void) 3620 { 3621 uint16_t d = 1; 3622 3623 return (archive_be16dec(&d) == 1); 3624 } 3625 3626 /* 3627 * Convert a current locale string to UTF-16BE/LE and copy the result. 3628 * Return -1 if conversion failes. 3629 */ 3630 static int 3631 win_strncat_to_utf16(struct archive_string *as16, const void *_p, 3632 size_t length, struct archive_string_conv *sc, int bigendian) 3633 { 3634 const char *s = (const char *)_p; 3635 char *u16; 3636 size_t count, avail; 3637 3638 if (archive_string_ensure(as16, 3639 as16->length + (length + 1) * 2) == NULL) 3640 return (-1); 3641 3642 u16 = as16->s + as16->length; 3643 avail = as16->buffer_length - 2; 3644 if (sc->from_cp == CP_C_LOCALE) { 3645 /* 3646 * "C" locale special process. 3647 */ 3648 count = 0; 3649 while (count < length && *s) { 3650 if (bigendian) 3651 archive_be16enc(u16, *s); 3652 else 3653 archive_le16enc(u16, *s); 3654 u16 += 2; 3655 s++; 3656 count++; 3657 } 3658 as16->length += count << 1; 3659 as16->s[as16->length] = 0; 3660 as16->s[as16->length+1] = 0; 3661 return (0); 3662 } 3663 do { 3664 count = MultiByteToWideChar(sc->from_cp, 3665 MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1); 3666 if (count == 0 && 3667 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 3668 /* Need more buffer for UTF-16 string */ 3669 count = MultiByteToWideChar(sc->from_cp, 3670 MB_PRECOMPOSED, s, (int)length, NULL, 0); 3671 if (archive_string_ensure(as16, (count +1) * 2) 3672 == NULL) 3673 return (-1); 3674 u16 = as16->s + as16->length; 3675 avail = as16->buffer_length - 2; 3676 continue; 3677 } 3678 } while (0); 3679 as16->length += count * 2; 3680 as16->s[as16->length] = 0; 3681 as16->s[as16->length+1] = 0; 3682 if (count == 0) 3683 return (-1); 3684 3685 if (is_big_endian()) { 3686 if (!bigendian) { 3687 while (count > 0) { 3688 uint16_t v = archive_be16dec(u16); 3689 archive_le16enc(u16, v); 3690 u16 += 2; 3691 count--; 3692 } 3693 } 3694 } else { 3695 if (bigendian) { 3696 while (count > 0) { 3697 uint16_t v = archive_le16dec(u16); 3698 archive_be16enc(u16, v); 3699 u16 += 2; 3700 count--; 3701 } 3702 } 3703 } 3704 return (0); 3705 } 3706 3707 static int 3708 win_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3709 size_t length, struct archive_string_conv *sc) 3710 { 3711 return (win_strncat_to_utf16(as16, _p, length, sc, 1)); 3712 } 3713 3714 static int 3715 win_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3716 size_t length, struct archive_string_conv *sc) 3717 { 3718 return (win_strncat_to_utf16(as16, _p, length, sc, 0)); 3719 } 3720 3721 #endif /* _WIN32 && !__CYGWIN__ */ 3722 3723 /* 3724 * Do the best effort for conversions. 3725 * We cannot handle UTF-16BE character-set without such iconv, 3726 * but there is a chance if a string consists just ASCII code or 3727 * a current locale is UTF-8. 3728 */ 3729 3730 /* 3731 * Convert a UTF-16BE string to current locale and copy the result. 3732 * Return -1 if conversion failes. 3733 */ 3734 static int 3735 best_effort_strncat_from_utf16(struct archive_string *as, const void *_p, 3736 size_t bytes, struct archive_string_conv *sc, int be) 3737 { 3738 const char *utf16 = (const char *)_p; 3739 char *mbs; 3740 uint32_t uc; 3741 int n, ret; 3742 3743 (void)sc; /* UNUSED */ 3744 /* 3745 * Other case, we should do the best effort. 3746 * If all character are ASCII(<0x7f), we can convert it. 3747 * if not , we set a alternative character and return -1. 3748 */ 3749 ret = 0; 3750 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3751 return (-1); 3752 mbs = as->s + as->length; 3753 3754 while ((n = utf16_to_unicode(&uc, utf16, bytes, be)) != 0) { 3755 if (n < 0) { 3756 n *= -1; 3757 ret = -1; 3758 } 3759 bytes -= n; 3760 utf16 += n; 3761 3762 if (uc > 127) { 3763 /* We cannot handle it. */ 3764 *mbs++ = '?'; 3765 ret = -1; 3766 } else 3767 *mbs++ = (char)uc; 3768 } 3769 as->length = mbs - as->s; 3770 as->s[as->length] = '\0'; 3771 return (ret); 3772 } 3773 3774 static int 3775 best_effort_strncat_from_utf16be(struct archive_string *as, const void *_p, 3776 size_t bytes, struct archive_string_conv *sc) 3777 { 3778 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 1)); 3779 } 3780 3781 static int 3782 best_effort_strncat_from_utf16le(struct archive_string *as, const void *_p, 3783 size_t bytes, struct archive_string_conv *sc) 3784 { 3785 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 0)); 3786 } 3787 3788 /* 3789 * Convert a current locale string to UTF-16BE/LE and copy the result. 3790 * Return -1 if conversion failes. 3791 */ 3792 static int 3793 best_effort_strncat_to_utf16(struct archive_string *as16, const void *_p, 3794 size_t length, struct archive_string_conv *sc, int bigendian) 3795 { 3796 const char *s = (const char *)_p; 3797 char *utf16; 3798 size_t remaining; 3799 int ret; 3800 3801 (void)sc; /* UNUSED */ 3802 /* 3803 * Other case, we should do the best effort. 3804 * If all character are ASCII(<0x7f), we can convert it. 3805 * if not , we set a alternative character and return -1. 3806 */ 3807 ret = 0; 3808 remaining = length; 3809 3810 if (archive_string_ensure(as16, 3811 as16->length + (length + 1) * 2) == NULL) 3812 return (-1); 3813 3814 utf16 = as16->s + as16->length; 3815 while (remaining--) { 3816 unsigned c = *s++; 3817 if (c > 127) { 3818 /* We cannot handle it. */ 3819 c = UNICODE_R_CHAR; 3820 ret = -1; 3821 } 3822 if (bigendian) 3823 archive_be16enc(utf16, c); 3824 else 3825 archive_le16enc(utf16, c); 3826 utf16 += 2; 3827 } 3828 as16->length = utf16 - as16->s; 3829 as16->s[as16->length] = 0; 3830 as16->s[as16->length+1] = 0; 3831 return (ret); 3832 } 3833 3834 static int 3835 best_effort_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3836 size_t length, struct archive_string_conv *sc) 3837 { 3838 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 1)); 3839 } 3840 3841 static int 3842 best_effort_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3843 size_t length, struct archive_string_conv *sc) 3844 { 3845 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 0)); 3846 } 3847 3848 3849 /* 3850 * Multistring operations. 3851 */ 3852 3853 void 3854 archive_mstring_clean(struct archive_mstring *aes) 3855 { 3856 archive_wstring_free(&(aes->aes_wcs)); 3857 archive_string_free(&(aes->aes_mbs)); 3858 archive_string_free(&(aes->aes_utf8)); 3859 archive_string_free(&(aes->aes_mbs_in_locale)); 3860 aes->aes_set = 0; 3861 } 3862 3863 void 3864 archive_mstring_copy(struct archive_mstring *dest, struct archive_mstring *src) 3865 { 3866 dest->aes_set = src->aes_set; 3867 archive_string_copy(&(dest->aes_mbs), &(src->aes_mbs)); 3868 archive_string_copy(&(dest->aes_utf8), &(src->aes_utf8)); 3869 archive_wstring_copy(&(dest->aes_wcs), &(src->aes_wcs)); 3870 } 3871 3872 int 3873 archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes, 3874 const char **p) 3875 { 3876 struct archive_string_conv *sc; 3877 int r; 3878 3879 /* If we already have a UTF8 form, return that immediately. */ 3880 if (aes->aes_set & AES_SET_UTF8) { 3881 *p = aes->aes_utf8.s; 3882 return (0); 3883 } 3884 3885 *p = NULL; 3886 if (aes->aes_set & AES_SET_MBS) { 3887 sc = archive_string_conversion_to_charset(a, "UTF-8", 1); 3888 if (sc == NULL) 3889 return (-1);/* Couldn't allocate memory for sc. */ 3890 r = archive_strncpy_l(&(aes->aes_mbs), aes->aes_mbs.s, 3891 aes->aes_mbs.length, sc); 3892 if (a == NULL) 3893 free_sconv_object(sc); 3894 if (r == 0) { 3895 aes->aes_set |= AES_SET_UTF8; 3896 *p = aes->aes_utf8.s; 3897 return (0);/* success. */ 3898 } else 3899 return (-1);/* failure. */ 3900 } 3901 return (0);/* success. */ 3902 } 3903 3904 int 3905 archive_mstring_get_mbs(struct archive *a, struct archive_mstring *aes, 3906 const char **p) 3907 { 3908 int r, ret = 0; 3909 3910 (void)a; /* UNUSED */ 3911 /* If we already have an MBS form, return that immediately. */ 3912 if (aes->aes_set & AES_SET_MBS) { 3913 *p = aes->aes_mbs.s; 3914 return (ret); 3915 } 3916 3917 *p = NULL; 3918 /* If there's a WCS form, try converting with the native locale. */ 3919 if (aes->aes_set & AES_SET_WCS) { 3920 archive_string_empty(&(aes->aes_mbs)); 3921 r = archive_string_append_from_wcs(&(aes->aes_mbs), 3922 aes->aes_wcs.s, aes->aes_wcs.length); 3923 *p = aes->aes_mbs.s; 3924 if (r == 0) { 3925 aes->aes_set |= AES_SET_MBS; 3926 return (ret); 3927 } else 3928 ret = -1; 3929 } 3930 3931 /* 3932 * Only a UTF-8 form cannot avail because its conversion already 3933 * failed at archive_mstring_update_utf8(). 3934 */ 3935 return (ret); 3936 } 3937 3938 int 3939 archive_mstring_get_wcs(struct archive *a, struct archive_mstring *aes, 3940 const wchar_t **wp) 3941 { 3942 int r, ret = 0; 3943 3944 (void)a;/* UNUSED */ 3945 /* Return WCS form if we already have it. */ 3946 if (aes->aes_set & AES_SET_WCS) { 3947 *wp = aes->aes_wcs.s; 3948 return (ret); 3949 } 3950 3951 *wp = NULL; 3952 /* Try converting MBS to WCS using native locale. */ 3953 if (aes->aes_set & AES_SET_MBS) { 3954 archive_wstring_empty(&(aes->aes_wcs)); 3955 r = archive_wstring_append_from_mbs(&(aes->aes_wcs), 3956 aes->aes_mbs.s, aes->aes_mbs.length); 3957 if (r == 0) { 3958 aes->aes_set |= AES_SET_WCS; 3959 *wp = aes->aes_wcs.s; 3960 } else 3961 ret = -1;/* failure. */ 3962 } 3963 return (ret); 3964 } 3965 3966 int 3967 archive_mstring_get_mbs_l(struct archive_mstring *aes, 3968 const char **p, size_t *length, struct archive_string_conv *sc) 3969 { 3970 int r, ret = 0; 3971 3972 #if defined(_WIN32) && !defined(__CYGWIN__) 3973 /* 3974 * Internationalization programing on Windows must use Wide 3975 * characters because Windows platform cannot make locale UTF-8. 3976 */ 3977 if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) { 3978 archive_string_empty(&(aes->aes_mbs_in_locale)); 3979 r = archive_string_append_from_wcs_in_codepage( 3980 &(aes->aes_mbs_in_locale), aes->aes_wcs.s, 3981 aes->aes_wcs.length, sc); 3982 if (r == 0) { 3983 *p = aes->aes_mbs_in_locale.s; 3984 if (length != NULL) 3985 *length = aes->aes_mbs_in_locale.length; 3986 return (0); 3987 } else if (errno == ENOMEM) 3988 return (-1); 3989 else 3990 ret = -1; 3991 } 3992 #endif 3993 3994 /* If there is not an MBS form but is a WCS form, try converting 3995 * with the native locale to be used for translating it to specified 3996 * character-set. */ 3997 if ((aes->aes_set & AES_SET_MBS) == 0 && 3998 (aes->aes_set & AES_SET_WCS) != 0) { 3999 archive_string_empty(&(aes->aes_mbs)); 4000 r = archive_string_append_from_wcs(&(aes->aes_mbs), 4001 aes->aes_wcs.s, aes->aes_wcs.length); 4002 if (r == 0) 4003 aes->aes_set |= AES_SET_MBS; 4004 else if (errno == ENOMEM) 4005 return (-1); 4006 else 4007 ret = -1; 4008 } 4009 /* If we already have an MBS form, use it to be translated to 4010 * specified character-set. */ 4011 if (aes->aes_set & AES_SET_MBS) { 4012 if (sc == NULL) { 4013 /* Conversion is unneeded. */ 4014 *p = aes->aes_mbs.s; 4015 if (length != NULL) 4016 *length = aes->aes_mbs.length; 4017 return (0); 4018 } 4019 ret = archive_strncpy_l(&(aes->aes_mbs_in_locale), 4020 aes->aes_mbs.s, aes->aes_mbs.length, sc); 4021 *p = aes->aes_mbs_in_locale.s; 4022 if (length != NULL) 4023 *length = aes->aes_mbs_in_locale.length; 4024 } else { 4025 *p = NULL; 4026 if (length != NULL) 4027 *length = 0; 4028 } 4029 return (ret); 4030 } 4031 4032 int 4033 archive_mstring_copy_mbs(struct archive_mstring *aes, const char *mbs) 4034 { 4035 if (mbs == NULL) { 4036 aes->aes_set = 0; 4037 return (0); 4038 } 4039 return (archive_mstring_copy_mbs_len(aes, mbs, strlen(mbs))); 4040 } 4041 4042 int 4043 archive_mstring_copy_mbs_len(struct archive_mstring *aes, const char *mbs, 4044 size_t len) 4045 { 4046 if (mbs == NULL) { 4047 aes->aes_set = 0; 4048 return (0); 4049 } 4050 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4051 archive_strncpy(&(aes->aes_mbs), mbs, len); 4052 archive_string_empty(&(aes->aes_utf8)); 4053 archive_wstring_empty(&(aes->aes_wcs)); 4054 return (0); 4055 } 4056 4057 int 4058 archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs) 4059 { 4060 return archive_mstring_copy_wcs_len(aes, wcs, 4061 wcs == NULL ? 0 : wcslen(wcs)); 4062 } 4063 4064 int 4065 archive_mstring_copy_wcs_len(struct archive_mstring *aes, const wchar_t *wcs, 4066 size_t len) 4067 { 4068 if (wcs == NULL) { 4069 aes->aes_set = 0; 4070 } 4071 aes->aes_set = AES_SET_WCS; /* Only WCS form set. */ 4072 archive_string_empty(&(aes->aes_mbs)); 4073 archive_string_empty(&(aes->aes_utf8)); 4074 archive_wstrncpy(&(aes->aes_wcs), wcs, len); 4075 return (0); 4076 } 4077 4078 int 4079 archive_mstring_copy_mbs_len_l(struct archive_mstring *aes, 4080 const char *mbs, size_t len, struct archive_string_conv *sc) 4081 { 4082 int r; 4083 4084 if (mbs == NULL) { 4085 aes->aes_set = 0; 4086 return (0); 4087 } 4088 archive_string_empty(&(aes->aes_mbs)); 4089 archive_wstring_empty(&(aes->aes_wcs)); 4090 archive_string_empty(&(aes->aes_utf8)); 4091 #if defined(_WIN32) && !defined(__CYGWIN__) 4092 /* 4093 * Internationalization programing on Windows must use Wide 4094 * characters because Windows platform cannot make locale UTF-8. 4095 */ 4096 if (sc == NULL) { 4097 if (archive_string_append(&(aes->aes_mbs), 4098 mbs, mbsnbytes(mbs, len)) == NULL) { 4099 aes->aes_set = 0; 4100 r = -1; 4101 } else { 4102 aes->aes_set = AES_SET_MBS; 4103 r = 0; 4104 } 4105 #if defined(HAVE_ICONV) 4106 } else if (sc != NULL && sc->cd_w != (iconv_t)-1) { 4107 /* 4108 * This case happens only when MultiByteToWideChar() cannot 4109 * handle sc->from_cp, and we have to iconv in order to 4110 * translate character-set to wchar_t,UTF-16. 4111 */ 4112 iconv_t cd = sc->cd; 4113 unsigned from_cp; 4114 int flag; 4115 4116 /* 4117 * Translate multi-bytes from some character-set to UTF-8. 4118 */ 4119 sc->cd = sc->cd_w; 4120 r = archive_strncpy_l(&(aes->aes_utf8), mbs, len, sc); 4121 sc->cd = cd; 4122 if (r != 0) { 4123 aes->aes_set = 0; 4124 return (r); 4125 } 4126 aes->aes_set = AES_SET_UTF8; 4127 4128 /* 4129 * Append the UTF-8 string into wstring. 4130 */ 4131 flag = sc->flag; 4132 sc->flag &= ~(SCONV_NORMALIZATION_C 4133 | SCONV_TO_UTF16| SCONV_FROM_UTF16); 4134 from_cp = sc->from_cp; 4135 sc->from_cp = CP_UTF8; 4136 r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4137 aes->aes_utf8.s, aes->aes_utf8.length, sc); 4138 sc->flag = flag; 4139 sc->from_cp = from_cp; 4140 if (r == 0) 4141 aes->aes_set |= AES_SET_WCS; 4142 #endif 4143 } else { 4144 r = archive_wstring_append_from_mbs_in_codepage( 4145 &(aes->aes_wcs), mbs, len, sc); 4146 if (r == 0) 4147 aes->aes_set = AES_SET_WCS; 4148 else 4149 aes->aes_set = 0; 4150 } 4151 #else 4152 r = archive_strncpy_l(&(aes->aes_mbs), mbs, len, sc); 4153 if (r == 0) 4154 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4155 else 4156 aes->aes_set = 0; 4157 #endif 4158 return (r); 4159 } 4160 4161 /* 4162 * The 'update' form tries to proactively update all forms of 4163 * this string (WCS and MBS) and returns an error if any of 4164 * them fail. This is used by the 'pax' handler, for instance, 4165 * to detect and report character-conversion failures early while 4166 * still allowing clients to get potentially useful values from 4167 * the more tolerant lazy conversions. (get_mbs and get_wcs will 4168 * strive to give the user something useful, so you can get hopefully 4169 * usable values even if some of the character conversions are failing.) 4170 */ 4171 int 4172 archive_mstring_update_utf8(struct archive *a, struct archive_mstring *aes, 4173 const char *utf8) 4174 { 4175 struct archive_string_conv *sc; 4176 int r; 4177 4178 if (utf8 == NULL) { 4179 aes->aes_set = 0; 4180 return (0); /* Succeeded in clearing everything. */ 4181 } 4182 4183 /* Save the UTF8 string. */ 4184 archive_strcpy(&(aes->aes_utf8), utf8); 4185 4186 /* Empty the mbs and wcs strings. */ 4187 archive_string_empty(&(aes->aes_mbs)); 4188 archive_wstring_empty(&(aes->aes_wcs)); 4189 4190 aes->aes_set = AES_SET_UTF8; /* Only UTF8 is set now. */ 4191 4192 /* Try converting UTF-8 to MBS, return false on failure. */ 4193 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 4194 if (sc == NULL) 4195 return (-1);/* Couldn't allocate memory for sc. */ 4196 r = archive_strcpy_l(&(aes->aes_mbs), utf8, sc); 4197 if (a == NULL) 4198 free_sconv_object(sc); 4199 if (r != 0) 4200 return (-1); 4201 aes->aes_set = AES_SET_UTF8 | AES_SET_MBS; /* Both UTF8 and MBS set. */ 4202 4203 /* Try converting MBS to WCS, return false on failure. */ 4204 if (archive_wstring_append_from_mbs(&(aes->aes_wcs), aes->aes_mbs.s, 4205 aes->aes_mbs.length)) 4206 return (-1); 4207 aes->aes_set = AES_SET_UTF8 | AES_SET_WCS | AES_SET_MBS; 4208 4209 /* All conversions succeeded. */ 4210 return (0); 4211 } 4212