1 /*- 2 * Copyright (c) 2003-2011 Tim Kientzle 3 * Copyright (c) 2011-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 29 /* 30 * Basic resizable string support, to simplify manipulating arbitrary-sized 31 * strings while minimizing heap activity. 32 * 33 * In particular, the buffer used by a string object is only grown, it 34 * never shrinks, so you can clear and reuse the same string object 35 * without incurring additional memory allocations. 36 */ 37 38 #ifdef HAVE_ERRNO_H 39 #include <errno.h> 40 #endif 41 #ifdef HAVE_ICONV_H 42 #include <iconv.h> 43 #endif 44 #ifdef HAVE_LANGINFO_H 45 #include <langinfo.h> 46 #endif 47 #ifdef HAVE_LOCALCHARSET_H 48 #include <localcharset.h> 49 #endif 50 #ifdef HAVE_STDLIB_H 51 #include <stdlib.h> 52 #endif 53 #ifdef HAVE_STRING_H 54 #include <string.h> 55 #endif 56 #ifdef HAVE_WCHAR_H 57 #include <wchar.h> 58 #endif 59 #if defined(_WIN32) && !defined(__CYGWIN__) 60 #include <windows.h> 61 #include <locale.h> 62 #endif 63 64 #include "archive_endian.h" 65 #include "archive_private.h" 66 #include "archive_string.h" 67 #include "archive_string_composition.h" 68 69 #if !defined(HAVE_WMEMCPY) && !defined(wmemcpy) 70 #define wmemcpy(a,b,i) (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t)) 71 #endif 72 73 #if !defined(HAVE_WMEMMOVE) && !defined(wmemmove) 74 #define wmemmove(a,b,i) (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t)) 75 #endif 76 77 #undef max 78 #define max(a, b) ((a)>(b)?(a):(b)) 79 80 struct archive_string_conv { 81 struct archive_string_conv *next; 82 char *from_charset; 83 char *to_charset; 84 unsigned from_cp; 85 unsigned to_cp; 86 /* Set 1 if from_charset and to_charset are the same. */ 87 int same; 88 int flag; 89 #define SCONV_TO_CHARSET 1 /* MBS is being converted to specified 90 * charset. */ 91 #define SCONV_FROM_CHARSET (1<<1) /* MBS is being converted from 92 * specified charset. */ 93 #define SCONV_BEST_EFFORT (1<<2) /* Copy at least ASCII code. */ 94 #define SCONV_WIN_CP (1<<3) /* Use Windows API for converting 95 * MBS. */ 96 #define SCONV_UTF8_LIBARCHIVE_2 (1<<4) /* Incorrect UTF-8 made by libarchive 97 * 2.x in the wrong assumption. */ 98 #define SCONV_NORMALIZATION_C (1<<6) /* Need normalization to be Form C. 99 * Before UTF-8 characters are actually 100 * processed. */ 101 #define SCONV_NORMALIZATION_D (1<<7) /* Need normalization to be Form D. 102 * Before UTF-8 characters are actually 103 * processed. 104 * Currently this only for MAC OS X. */ 105 #define SCONV_TO_UTF8 (1<<8) /* "to charset" side is UTF-8. */ 106 #define SCONV_FROM_UTF8 (1<<9) /* "from charset" side is UTF-8. */ 107 #define SCONV_TO_UTF16BE (1<<10) /* "to charset" side is UTF-16BE. */ 108 #define SCONV_FROM_UTF16BE (1<<11) /* "from charset" side is UTF-16BE. */ 109 #define SCONV_TO_UTF16LE (1<<12) /* "to charset" side is UTF-16LE. */ 110 #define SCONV_FROM_UTF16LE (1<<13) /* "from charset" side is UTF-16LE. */ 111 #define SCONV_TO_UTF16 (SCONV_TO_UTF16BE | SCONV_TO_UTF16LE) 112 #define SCONV_FROM_UTF16 (SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE) 113 114 #if HAVE_ICONV 115 iconv_t cd; 116 iconv_t cd_w;/* Use at archive_mstring on 117 * Windows. */ 118 #endif 119 /* A temporary buffer for normalization. */ 120 struct archive_string utftmp; 121 int (*converter[2])(struct archive_string *, const void *, size_t, 122 struct archive_string_conv *); 123 int nconverter; 124 }; 125 126 #define CP_C_LOCALE 0 /* "C" locale only for this file. */ 127 #define CP_UTF16LE 1200 128 #define CP_UTF16BE 1201 129 130 #define IS_HIGH_SURROGATE_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDBFF) 131 #define IS_LOW_SURROGATE_LA(uc) ((uc) >= 0xDC00 && (uc) <= 0xDFFF) 132 #define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF) 133 #define UNICODE_MAX 0x10FFFF 134 #define UNICODE_R_CHAR 0xFFFD /* Replacement character. */ 135 /* Set U+FFFD(Replacement character) in UTF-8. */ 136 static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd}; 137 138 static struct archive_string_conv *find_sconv_object(struct archive *, 139 const char *, const char *); 140 static void add_sconv_object(struct archive *, struct archive_string_conv *); 141 static struct archive_string_conv *create_sconv_object(const char *, 142 const char *, unsigned, int); 143 static void free_sconv_object(struct archive_string_conv *); 144 static struct archive_string_conv *get_sconv_object(struct archive *, 145 const char *, const char *, int); 146 static unsigned make_codepage_from_charset(const char *); 147 static unsigned get_current_codepage(void); 148 static unsigned get_current_oemcp(void); 149 static size_t mbsnbytes(const void *, size_t); 150 static size_t utf16nbytes(const void *, size_t); 151 #if defined(_WIN32) && !defined(__CYGWIN__) 152 static int archive_wstring_append_from_mbs_in_codepage( 153 struct archive_wstring *, const char *, size_t, 154 struct archive_string_conv *); 155 static int archive_string_append_from_wcs_in_codepage(struct archive_string *, 156 const wchar_t *, size_t, struct archive_string_conv *); 157 static int is_big_endian(void); 158 static int strncat_in_codepage(struct archive_string *, const void *, 159 size_t, struct archive_string_conv *); 160 static int win_strncat_from_utf16be(struct archive_string *, const void *, 161 size_t, struct archive_string_conv *); 162 static int win_strncat_from_utf16le(struct archive_string *, const void *, 163 size_t, struct archive_string_conv *); 164 static int win_strncat_to_utf16be(struct archive_string *, const void *, 165 size_t, struct archive_string_conv *); 166 static int win_strncat_to_utf16le(struct archive_string *, const void *, 167 size_t, struct archive_string_conv *); 168 #endif 169 static int best_effort_strncat_from_utf16be(struct archive_string *, 170 const void *, size_t, struct archive_string_conv *); 171 static int best_effort_strncat_from_utf16le(struct archive_string *, 172 const void *, size_t, struct archive_string_conv *); 173 static int best_effort_strncat_to_utf16be(struct archive_string *, 174 const void *, size_t, struct archive_string_conv *); 175 static int best_effort_strncat_to_utf16le(struct archive_string *, 176 const void *, size_t, struct archive_string_conv *); 177 #if defined(HAVE_ICONV) 178 static int iconv_strncat_in_locale(struct archive_string *, const void *, 179 size_t, struct archive_string_conv *); 180 #endif 181 static int best_effort_strncat_in_locale(struct archive_string *, 182 const void *, size_t, struct archive_string_conv *); 183 static int _utf8_to_unicode(uint32_t *, const char *, size_t); 184 static int utf8_to_unicode(uint32_t *, const char *, size_t); 185 static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t); 186 static int cesu8_to_unicode(uint32_t *, const char *, size_t); 187 static size_t unicode_to_utf8(char *, size_t, uint32_t); 188 static int utf16_to_unicode(uint32_t *, const char *, size_t, int); 189 static size_t unicode_to_utf16be(char *, size_t, uint32_t); 190 static size_t unicode_to_utf16le(char *, size_t, uint32_t); 191 static int strncat_from_utf8_libarchive2(struct archive_string *, 192 const void *, size_t, struct archive_string_conv *); 193 static int strncat_from_utf8_to_utf8(struct archive_string *, const void *, 194 size_t, struct archive_string_conv *); 195 static int archive_string_normalize_C(struct archive_string *, const void *, 196 size_t, struct archive_string_conv *); 197 static int archive_string_normalize_D(struct archive_string *, const void *, 198 size_t, struct archive_string_conv *); 199 static int archive_string_append_unicode(struct archive_string *, 200 const void *, size_t, struct archive_string_conv *); 201 202 static struct archive_string * 203 archive_string_append(struct archive_string *as, const char *p, size_t s) 204 { 205 if (archive_string_ensure(as, as->length + s + 1) == NULL) 206 return (NULL); 207 if (s) 208 memmove(as->s + as->length, p, s); 209 as->length += s; 210 as->s[as->length] = 0; 211 return (as); 212 } 213 214 static struct archive_wstring * 215 archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) 216 { 217 if (archive_wstring_ensure(as, as->length + s + 1) == NULL) 218 return (NULL); 219 if (s) 220 wmemmove(as->s + as->length, p, s); 221 as->length += s; 222 as->s[as->length] = 0; 223 return (as); 224 } 225 226 struct archive_string * 227 archive_array_append(struct archive_string *as, const char *p, size_t s) 228 { 229 return archive_string_append(as, p, s); 230 } 231 232 void 233 archive_string_concat(struct archive_string *dest, struct archive_string *src) 234 { 235 if (archive_string_append(dest, src->s, src->length) == NULL) 236 __archive_errx(1, "Out of memory"); 237 } 238 239 void 240 archive_wstring_concat(struct archive_wstring *dest, 241 struct archive_wstring *src) 242 { 243 if (archive_wstring_append(dest, src->s, src->length) == NULL) 244 __archive_errx(1, "Out of memory"); 245 } 246 247 void 248 archive_string_free(struct archive_string *as) 249 { 250 as->length = 0; 251 as->buffer_length = 0; 252 free(as->s); 253 as->s = NULL; 254 } 255 256 void 257 archive_wstring_free(struct archive_wstring *as) 258 { 259 as->length = 0; 260 as->buffer_length = 0; 261 free(as->s); 262 as->s = NULL; 263 } 264 265 struct archive_wstring * 266 archive_wstring_ensure(struct archive_wstring *as, size_t s) 267 { 268 return (struct archive_wstring *) 269 archive_string_ensure((struct archive_string *)as, 270 s * sizeof(wchar_t)); 271 } 272 273 /* Returns NULL on any allocation failure. */ 274 struct archive_string * 275 archive_string_ensure(struct archive_string *as, size_t s) 276 { 277 char *p; 278 size_t new_length; 279 280 /* If buffer is already big enough, don't reallocate. */ 281 if (as->s && (s <= as->buffer_length)) 282 return (as); 283 284 /* 285 * Growing the buffer at least exponentially ensures that 286 * append operations are always linear in the number of 287 * characters appended. Using a smaller growth rate for 288 * larger buffers reduces memory waste somewhat at the cost of 289 * a larger constant factor. 290 */ 291 if (as->buffer_length < 32) 292 /* Start with a minimum 32-character buffer. */ 293 new_length = 32; 294 else if (as->buffer_length < 8192) 295 /* Buffers under 8k are doubled for speed. */ 296 new_length = as->buffer_length + as->buffer_length; 297 else { 298 /* Buffers 8k and over grow by at least 25% each time. */ 299 new_length = as->buffer_length + as->buffer_length / 4; 300 /* Be safe: If size wraps, fail. */ 301 if (new_length < as->buffer_length) { 302 /* On failure, wipe the string and return NULL. */ 303 archive_string_free(as); 304 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 305 return (NULL); 306 } 307 } 308 /* 309 * The computation above is a lower limit to how much we'll 310 * grow the buffer. In any case, we have to grow it enough to 311 * hold the request. 312 */ 313 if (new_length < s) 314 new_length = s; 315 /* Now we can reallocate the buffer. */ 316 p = realloc(as->s, new_length); 317 if (p == NULL) { 318 /* On failure, wipe the string and return NULL. */ 319 archive_string_free(as); 320 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 321 return (NULL); 322 } 323 324 as->s = p; 325 as->buffer_length = new_length; 326 return (as); 327 } 328 329 /* 330 * TODO: See if there's a way to avoid scanning 331 * the source string twice. Then test to see 332 * if it actually helps (remember that we're almost 333 * always called with pretty short arguments, so 334 * such an optimization might not help). 335 */ 336 struct archive_string * 337 archive_strncat(struct archive_string *as, const void *_p, size_t n) 338 { 339 size_t s; 340 const char *p, *pp; 341 342 p = (const char *)_p; 343 344 /* Like strlen(p), except won't examine positions beyond p[n]. */ 345 s = 0; 346 pp = p; 347 while (s < n && *pp) { 348 pp++; 349 s++; 350 } 351 if ((as = archive_string_append(as, p, s)) == NULL) 352 __archive_errx(1, "Out of memory"); 353 return (as); 354 } 355 356 struct archive_wstring * 357 archive_wstrncat(struct archive_wstring *as, const wchar_t *p, size_t n) 358 { 359 size_t s; 360 const wchar_t *pp; 361 362 /* Like strlen(p), except won't examine positions beyond p[n]. */ 363 s = 0; 364 pp = p; 365 while (s < n && *pp) { 366 pp++; 367 s++; 368 } 369 if ((as = archive_wstring_append(as, p, s)) == NULL) 370 __archive_errx(1, "Out of memory"); 371 return (as); 372 } 373 374 struct archive_string * 375 archive_strcat(struct archive_string *as, const void *p) 376 { 377 /* strcat is just strncat without an effective limit. 378 * Assert that we'll never get called with a source 379 * string over 16MB. 380 * TODO: Review all uses of strcat in the source 381 * and try to replace them with strncat(). 382 */ 383 return archive_strncat(as, p, 0x1000000); 384 } 385 386 struct archive_wstring * 387 archive_wstrcat(struct archive_wstring *as, const wchar_t *p) 388 { 389 /* Ditto. */ 390 return archive_wstrncat(as, p, 0x1000000); 391 } 392 393 struct archive_string * 394 archive_strappend_char(struct archive_string *as, char c) 395 { 396 if ((as = archive_string_append(as, &c, 1)) == NULL) 397 __archive_errx(1, "Out of memory"); 398 return (as); 399 } 400 401 struct archive_wstring * 402 archive_wstrappend_wchar(struct archive_wstring *as, wchar_t c) 403 { 404 if ((as = archive_wstring_append(as, &c, 1)) == NULL) 405 __archive_errx(1, "Out of memory"); 406 return (as); 407 } 408 409 /* 410 * Get the "current character set" name to use with iconv. 411 * On FreeBSD, the empty character set name "" chooses 412 * the correct character encoding for the current locale, 413 * so this isn't necessary. 414 * But iconv on Mac OS 10.6 doesn't seem to handle this correctly; 415 * on that system, we have to explicitly call nl_langinfo() 416 * to get the right name. Not sure about other platforms. 417 * 418 * NOTE: GNU libiconv does not recognize the character-set name 419 * which some platform nl_langinfo(CODESET) returns, so we should 420 * use locale_charset() instead of nl_langinfo(CODESET) for GNU libiconv. 421 */ 422 static const char * 423 default_iconv_charset(const char *charset) { 424 if (charset != NULL && charset[0] != '\0') 425 return charset; 426 #if HAVE_LOCALE_CHARSET && !defined(__APPLE__) 427 /* locale_charset() is broken on Mac OS */ 428 return locale_charset(); 429 #elif HAVE_NL_LANGINFO 430 return nl_langinfo(CODESET); 431 #else 432 return ""; 433 #endif 434 } 435 436 #if defined(_WIN32) && !defined(__CYGWIN__) 437 438 /* 439 * Convert MBS to WCS. 440 * Note: returns -1 if conversion fails. 441 */ 442 int 443 archive_wstring_append_from_mbs(struct archive_wstring *dest, 444 const char *p, size_t len) 445 { 446 return archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL); 447 } 448 449 static int 450 archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest, 451 const char *s, size_t length, struct archive_string_conv *sc) 452 { 453 int count, ret = 0; 454 UINT from_cp; 455 456 if (sc != NULL) 457 from_cp = sc->from_cp; 458 else 459 from_cp = get_current_codepage(); 460 461 if (from_cp == CP_C_LOCALE) { 462 /* 463 * "C" locale special processing. 464 */ 465 wchar_t *ws; 466 const unsigned char *mp; 467 468 if (NULL == archive_wstring_ensure(dest, 469 dest->length + length + 1)) 470 return (-1); 471 472 ws = dest->s + dest->length; 473 mp = (const unsigned char *)s; 474 count = 0; 475 while (count < (int)length && *mp) { 476 *ws++ = (wchar_t)*mp++; 477 count++; 478 } 479 } else if (sc != NULL && 480 (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) { 481 /* 482 * Normalize UTF-8 and UTF-16BE and convert it directly 483 * to UTF-16 as wchar_t. 484 */ 485 struct archive_string u16; 486 int saved_flag = sc->flag;/* save current flag. */ 487 488 if (is_big_endian()) 489 sc->flag |= SCONV_TO_UTF16BE; 490 else 491 sc->flag |= SCONV_TO_UTF16LE; 492 493 if (sc->flag & SCONV_FROM_UTF16) { 494 /* 495 * UTF-16BE/LE NFD ===> UTF-16 NFC 496 * UTF-16BE/LE NFC ===> UTF-16 NFD 497 */ 498 count = (int)utf16nbytes(s, length); 499 } else { 500 /* 501 * UTF-8 NFD ===> UTF-16 NFC 502 * UTF-8 NFC ===> UTF-16 NFD 503 */ 504 count = (int)mbsnbytes(s, length); 505 } 506 u16.s = (char *)dest->s; 507 u16.length = dest->length << 1;; 508 u16.buffer_length = dest->buffer_length; 509 if (sc->flag & SCONV_NORMALIZATION_C) 510 ret = archive_string_normalize_C(&u16, s, count, sc); 511 else 512 ret = archive_string_normalize_D(&u16, s, count, sc); 513 dest->s = (wchar_t *)u16.s; 514 dest->length = u16.length >> 1; 515 dest->buffer_length = u16.buffer_length; 516 sc->flag = saved_flag;/* restore the saved flag. */ 517 return (ret); 518 } else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) { 519 count = (int)utf16nbytes(s, length); 520 count >>= 1; /* to be WCS length */ 521 /* Allocate memory for WCS. */ 522 if (NULL == archive_wstring_ensure(dest, 523 dest->length + count + 1)) 524 return (-1); 525 wmemcpy(dest->s + dest->length, (const wchar_t *)s, count); 526 if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) { 527 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 528 int b; 529 for (b = 0; b < count; b++) { 530 uint16_t val = archive_le16dec(u16+b); 531 archive_be16enc(u16+b, val); 532 } 533 } else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) { 534 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 535 int b; 536 for (b = 0; b < count; b++) { 537 uint16_t val = archive_be16dec(u16+b); 538 archive_le16enc(u16+b, val); 539 } 540 } 541 } else { 542 DWORD mbflag; 543 size_t buffsize; 544 545 if (sc == NULL) 546 mbflag = 0; 547 else if (sc->flag & SCONV_FROM_CHARSET) { 548 /* Do not trust the length which comes from 549 * an archive file. */ 550 length = mbsnbytes(s, length); 551 mbflag = 0; 552 } else 553 mbflag = MB_PRECOMPOSED; 554 555 mbflag |= MB_ERR_INVALID_CHARS; 556 557 buffsize = dest->length + length + 1; 558 do { 559 /* Allocate memory for WCS. */ 560 if (NULL == archive_wstring_ensure(dest, buffsize)) 561 return (-1); 562 /* Convert MBS to WCS. */ 563 count = MultiByteToWideChar(from_cp, 564 mbflag, s, (int)length, dest->s + dest->length, 565 (int)(dest->buffer_length >> 1) -1); 566 if (count == 0 && 567 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 568 /* Expand the WCS buffer. */ 569 buffsize = dest->buffer_length << 1; 570 continue; 571 } 572 if (count == 0 && length != 0) 573 ret = -1; 574 break; 575 } while (1); 576 } 577 dest->length += count; 578 dest->s[dest->length] = L'\0'; 579 return (ret); 580 } 581 582 #else 583 584 /* 585 * Convert MBS to WCS. 586 * Note: returns -1 if conversion fails. 587 */ 588 int 589 archive_wstring_append_from_mbs(struct archive_wstring *dest, 590 const char *p, size_t len) 591 { 592 size_t r; 593 int ret_val = 0; 594 /* 595 * No single byte will be more than one wide character, 596 * so this length estimate will always be big enough. 597 */ 598 // size_t wcs_length = len; 599 size_t mbs_length = len; 600 const char *mbs = p; 601 wchar_t *wcs; 602 #if HAVE_MBRTOWC 603 mbstate_t shift_state; 604 605 memset(&shift_state, 0, sizeof(shift_state)); 606 #endif 607 /* 608 * As we decided to have wcs_length == mbs_length == len 609 * we can use len here instead of wcs_length 610 */ 611 if (NULL == archive_wstring_ensure(dest, dest->length + len + 1)) 612 return (-1); 613 wcs = dest->s + dest->length; 614 /* 615 * We cannot use mbsrtowcs/mbstowcs here because those may convert 616 * extra MBS when strlen(p) > len and one wide character consists of 617 * multi bytes. 618 */ 619 while (*mbs && mbs_length > 0) { 620 /* 621 * The buffer we allocated is always big enough. 622 * Keep this code path in a comment if we decide to choose 623 * smaller wcs_length in the future 624 */ 625 /* 626 if (wcs_length == 0) { 627 dest->length = wcs - dest->s; 628 dest->s[dest->length] = L'\0'; 629 wcs_length = mbs_length; 630 if (NULL == archive_wstring_ensure(dest, 631 dest->length + wcs_length + 1)) 632 return (-1); 633 wcs = dest->s + dest->length; 634 } 635 */ 636 #if HAVE_MBRTOWC 637 r = mbrtowc(wcs, mbs, mbs_length, &shift_state); 638 #else 639 r = mbtowc(wcs, mbs, mbs_length); 640 #endif 641 if (r == (size_t)-1 || r == (size_t)-2) { 642 ret_val = -1; 643 break; 644 } 645 if (r == 0 || r > mbs_length) 646 break; 647 wcs++; 648 // wcs_length--; 649 mbs += r; 650 mbs_length -= r; 651 } 652 dest->length = wcs - dest->s; 653 dest->s[dest->length] = L'\0'; 654 return (ret_val); 655 } 656 657 #endif 658 659 #if defined(_WIN32) && !defined(__CYGWIN__) 660 661 /* 662 * WCS ==> MBS. 663 * Note: returns -1 if conversion fails. 664 * 665 * Win32 builds use WideCharToMultiByte from the Windows API. 666 * (Maybe Cygwin should too? WideCharToMultiByte will know a 667 * lot more about local character encodings than the wcrtomb() 668 * wrapper is going to know.) 669 */ 670 int 671 archive_string_append_from_wcs(struct archive_string *as, 672 const wchar_t *w, size_t len) 673 { 674 return archive_string_append_from_wcs_in_codepage(as, w, len, NULL); 675 } 676 677 static int 678 archive_string_append_from_wcs_in_codepage(struct archive_string *as, 679 const wchar_t *ws, size_t len, struct archive_string_conv *sc) 680 { 681 BOOL defchar_used, *dp; 682 int count, ret = 0; 683 UINT to_cp; 684 int wslen = (int)len; 685 686 if (sc != NULL) 687 to_cp = sc->to_cp; 688 else 689 to_cp = get_current_codepage(); 690 691 if (to_cp == CP_C_LOCALE) { 692 /* 693 * "C" locale special processing. 694 */ 695 const wchar_t *wp = ws; 696 char *p; 697 698 if (NULL == archive_string_ensure(as, 699 as->length + wslen +1)) 700 return (-1); 701 p = as->s + as->length; 702 count = 0; 703 defchar_used = 0; 704 while (count < wslen && *wp) { 705 if (*wp > 255) { 706 *p++ = '?'; 707 wp++; 708 defchar_used = 1; 709 } else 710 *p++ = (char)*wp++; 711 count++; 712 } 713 } else if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) { 714 uint16_t *u16; 715 716 if (NULL == 717 archive_string_ensure(as, as->length + len * 2 + 2)) 718 return (-1); 719 u16 = (uint16_t *)(as->s + as->length); 720 count = 0; 721 defchar_used = 0; 722 if (sc->flag & SCONV_TO_UTF16BE) { 723 while (count < (int)len && *ws) { 724 archive_be16enc(u16+count, *ws); 725 ws++; 726 count++; 727 } 728 } else { 729 while (count < (int)len && *ws) { 730 archive_le16enc(u16+count, *ws); 731 ws++; 732 count++; 733 } 734 } 735 count <<= 1; /* to be byte size */ 736 } else { 737 /* Make sure the MBS buffer has plenty to set. */ 738 if (NULL == 739 archive_string_ensure(as, as->length + len * 2 + 1)) 740 return (-1); 741 do { 742 defchar_used = 0; 743 if (to_cp == CP_UTF8 || sc == NULL) 744 dp = NULL; 745 else 746 dp = &defchar_used; 747 count = WideCharToMultiByte(to_cp, 0, ws, wslen, 748 as->s + as->length, 749 (int)as->buffer_length - (int)as->length - 1, NULL, dp); 750 if (count == 0 && 751 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 752 /* Expand the MBS buffer and retry. */ 753 if (NULL == archive_string_ensure(as, 754 as->buffer_length + len)) 755 return (-1); 756 continue; 757 } 758 if (count == 0) 759 ret = -1; 760 break; 761 } while (1); 762 } 763 as->length += count; 764 as->s[as->length] = '\0'; 765 return (defchar_used?-1:ret); 766 } 767 768 #elif defined(HAVE_WCTOMB) || defined(HAVE_WCRTOMB) 769 770 /* 771 * Translates a wide character string into current locale character set 772 * and appends to the archive_string. Note: returns -1 if conversion 773 * fails. 774 */ 775 int 776 archive_string_append_from_wcs(struct archive_string *as, 777 const wchar_t *w, size_t len) 778 { 779 /* We cannot use the standard wcstombs() here because it 780 * cannot tell us how big the output buffer should be. So 781 * I've built a loop around wcrtomb() or wctomb() that 782 * converts a character at a time and resizes the string as 783 * needed. We prefer wcrtomb() when it's available because 784 * it's thread-safe. */ 785 int n, ret_val = 0; 786 char *p; 787 char *end; 788 #if HAVE_WCRTOMB 789 mbstate_t shift_state; 790 791 memset(&shift_state, 0, sizeof(shift_state)); 792 #else 793 /* Clear the shift state before starting. */ 794 wctomb(NULL, L'\0'); 795 #endif 796 /* 797 * Allocate buffer for MBS. 798 * We need this allocation here since it is possible that 799 * as->s is still NULL. 800 */ 801 if (archive_string_ensure(as, as->length + len + 1) == NULL) 802 return (-1); 803 804 p = as->s + as->length; 805 end = as->s + as->buffer_length - MB_CUR_MAX -1; 806 while (*w != L'\0' && len > 0) { 807 if (p >= end) { 808 as->length = p - as->s; 809 as->s[as->length] = '\0'; 810 /* Re-allocate buffer for MBS. */ 811 if (archive_string_ensure(as, 812 as->length + max(len * 2, 813 (size_t)MB_CUR_MAX) + 1) == NULL) 814 return (-1); 815 p = as->s + as->length; 816 end = as->s + as->buffer_length - MB_CUR_MAX -1; 817 } 818 #if HAVE_WCRTOMB 819 n = wcrtomb(p, *w++, &shift_state); 820 #else 821 n = wctomb(p, *w++); 822 #endif 823 if (n == -1) { 824 if (errno == EILSEQ) { 825 /* Skip an illegal wide char. */ 826 *p++ = '?'; 827 ret_val = -1; 828 } else { 829 ret_val = -1; 830 break; 831 } 832 } else 833 p += n; 834 len--; 835 } 836 as->length = p - as->s; 837 as->s[as->length] = '\0'; 838 return (ret_val); 839 } 840 841 #else /* HAVE_WCTOMB || HAVE_WCRTOMB */ 842 843 /* 844 * TODO: Test if __STDC_ISO_10646__ is defined. 845 * Non-Windows uses ISO C wcrtomb() or wctomb() to perform the conversion 846 * one character at a time. If a non-Windows platform doesn't have 847 * either of these, fall back to the built-in UTF8 conversion. 848 */ 849 int 850 archive_string_append_from_wcs(struct archive_string *as, 851 const wchar_t *w, size_t len) 852 { 853 (void)as;/* UNUSED */ 854 (void)w;/* UNUSED */ 855 (void)len;/* UNUSED */ 856 errno = ENOSYS; 857 return (-1); 858 } 859 860 #endif /* HAVE_WCTOMB || HAVE_WCRTOMB */ 861 862 /* 863 * Find a string conversion object by a pair of 'from' charset name 864 * and 'to' charset name from an archive object. 865 * Return NULL if not found. 866 */ 867 static struct archive_string_conv * 868 find_sconv_object(struct archive *a, const char *fc, const char *tc) 869 { 870 struct archive_string_conv *sc; 871 872 if (a == NULL) 873 return (NULL); 874 875 for (sc = a->sconv; sc != NULL; sc = sc->next) { 876 if (strcmp(sc->from_charset, fc) == 0 && 877 strcmp(sc->to_charset, tc) == 0) 878 break; 879 } 880 return (sc); 881 } 882 883 /* 884 * Register a string object to an archive object. 885 */ 886 static void 887 add_sconv_object(struct archive *a, struct archive_string_conv *sc) 888 { 889 struct archive_string_conv **psc; 890 891 /* Add a new sconv to sconv list. */ 892 psc = &(a->sconv); 893 while (*psc != NULL) 894 psc = &((*psc)->next); 895 *psc = sc; 896 } 897 898 static void 899 add_converter(struct archive_string_conv *sc, int (*converter) 900 (struct archive_string *, const void *, size_t, 901 struct archive_string_conv *)) 902 { 903 if (sc == NULL || sc->nconverter >= 2) 904 __archive_errx(1, "Programming error"); 905 sc->converter[sc->nconverter++] = converter; 906 } 907 908 static void 909 setup_converter(struct archive_string_conv *sc) 910 { 911 912 /* Reset. */ 913 sc->nconverter = 0; 914 915 /* 916 * Perform special sequence for the incorrect UTF-8 filenames 917 * made by libarchive2.x. 918 */ 919 if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) { 920 add_converter(sc, strncat_from_utf8_libarchive2); 921 return; 922 } 923 924 /* 925 * Convert a string to UTF-16BE/LE. 926 */ 927 if (sc->flag & SCONV_TO_UTF16) { 928 /* 929 * If the current locale is UTF-8, we can translate 930 * a UTF-8 string into a UTF-16BE string. 931 */ 932 if (sc->flag & SCONV_FROM_UTF8) { 933 add_converter(sc, archive_string_append_unicode); 934 return; 935 } 936 937 #if defined(_WIN32) && !defined(__CYGWIN__) 938 if (sc->flag & SCONV_WIN_CP) { 939 if (sc->flag & SCONV_TO_UTF16BE) 940 add_converter(sc, win_strncat_to_utf16be); 941 else 942 add_converter(sc, win_strncat_to_utf16le); 943 return; 944 } 945 #endif 946 947 #if defined(HAVE_ICONV) 948 if (sc->cd != (iconv_t)-1) { 949 add_converter(sc, iconv_strncat_in_locale); 950 return; 951 } 952 #endif 953 954 if (sc->flag & SCONV_BEST_EFFORT) { 955 if (sc->flag & SCONV_TO_UTF16BE) 956 add_converter(sc, 957 best_effort_strncat_to_utf16be); 958 else 959 add_converter(sc, 960 best_effort_strncat_to_utf16le); 961 } else 962 /* Make sure we have no converter. */ 963 sc->nconverter = 0; 964 return; 965 } 966 967 /* 968 * Convert a string from UTF-16BE/LE. 969 */ 970 if (sc->flag & SCONV_FROM_UTF16) { 971 /* 972 * At least we should normalize a UTF-16BE string. 973 */ 974 if (sc->flag & SCONV_NORMALIZATION_D) 975 add_converter(sc,archive_string_normalize_D); 976 else if (sc->flag & SCONV_NORMALIZATION_C) 977 add_converter(sc, archive_string_normalize_C); 978 979 if (sc->flag & SCONV_TO_UTF8) { 980 /* 981 * If the current locale is UTF-8, we can translate 982 * a UTF-16BE/LE string into a UTF-8 string directly. 983 */ 984 if (!(sc->flag & 985 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 986 add_converter(sc, 987 archive_string_append_unicode); 988 return; 989 } 990 991 #if defined(_WIN32) && !defined(__CYGWIN__) 992 if (sc->flag & SCONV_WIN_CP) { 993 if (sc->flag & SCONV_FROM_UTF16BE) 994 add_converter(sc, win_strncat_from_utf16be); 995 else 996 add_converter(sc, win_strncat_from_utf16le); 997 return; 998 } 999 #endif 1000 1001 #if defined(HAVE_ICONV) 1002 if (sc->cd != (iconv_t)-1) { 1003 add_converter(sc, iconv_strncat_in_locale); 1004 return; 1005 } 1006 #endif 1007 1008 if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 1009 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 1010 add_converter(sc, best_effort_strncat_from_utf16be); 1011 else if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 1012 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 1013 add_converter(sc, best_effort_strncat_from_utf16le); 1014 else 1015 /* Make sure we have no converter. */ 1016 sc->nconverter = 0; 1017 return; 1018 } 1019 1020 if (sc->flag & SCONV_FROM_UTF8) { 1021 /* 1022 * At least we should normalize a UTF-8 string. 1023 */ 1024 if (sc->flag & SCONV_NORMALIZATION_D) 1025 add_converter(sc,archive_string_normalize_D); 1026 else if (sc->flag & SCONV_NORMALIZATION_C) 1027 add_converter(sc, archive_string_normalize_C); 1028 1029 /* 1030 * Copy UTF-8 string with a check of CESU-8. 1031 * Apparently, iconv does not check surrogate pairs in UTF-8 1032 * when both from-charset and to-charset are UTF-8, and then 1033 * we use our UTF-8 copy code. 1034 */ 1035 if (sc->flag & SCONV_TO_UTF8) { 1036 /* 1037 * If the current locale is UTF-8, we can translate 1038 * a UTF-16BE string into a UTF-8 string directly. 1039 */ 1040 if (!(sc->flag & 1041 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 1042 add_converter(sc, strncat_from_utf8_to_utf8); 1043 return; 1044 } 1045 } 1046 1047 #if defined(_WIN32) && !defined(__CYGWIN__) 1048 /* 1049 * On Windows we can use Windows API for a string conversion. 1050 */ 1051 if (sc->flag & SCONV_WIN_CP) { 1052 add_converter(sc, strncat_in_codepage); 1053 return; 1054 } 1055 #endif 1056 1057 #if HAVE_ICONV 1058 if (sc->cd != (iconv_t)-1) { 1059 add_converter(sc, iconv_strncat_in_locale); 1060 /* 1061 * iconv generally does not support UTF-8-MAC and so 1062 * we have to the output of iconv from NFC to NFD if 1063 * need. 1064 */ 1065 if ((sc->flag & SCONV_FROM_CHARSET) && 1066 (sc->flag & SCONV_TO_UTF8)) { 1067 if (sc->flag & SCONV_NORMALIZATION_D) 1068 add_converter(sc, archive_string_normalize_D); 1069 } 1070 return; 1071 } 1072 #endif 1073 1074 /* 1075 * Try conversion in the best effort or no conversion. 1076 */ 1077 if ((sc->flag & SCONV_BEST_EFFORT) || sc->same) 1078 add_converter(sc, best_effort_strncat_in_locale); 1079 else 1080 /* Make sure we have no converter. */ 1081 sc->nconverter = 0; 1082 } 1083 1084 /* 1085 * Return canonicalized charset-name but this supports just UTF-8, UTF-16BE 1086 * and CP932 which are referenced in create_sconv_object(). 1087 */ 1088 static const char * 1089 canonical_charset_name(const char *charset) 1090 { 1091 char cs[16]; 1092 char *p; 1093 const char *s; 1094 1095 if (charset == NULL || charset[0] == '\0' 1096 || strlen(charset) > 15) 1097 return (charset); 1098 1099 /* Copy name to uppercase. */ 1100 p = cs; 1101 s = charset; 1102 while (*s) { 1103 char c = *s++; 1104 if (c >= 'a' && c <= 'z') 1105 c -= 'a' - 'A'; 1106 *p++ = c; 1107 } 1108 *p++ = '\0'; 1109 1110 if (strcmp(cs, "UTF-8") == 0 || 1111 strcmp(cs, "UTF8") == 0) 1112 return ("UTF-8"); 1113 if (strcmp(cs, "UTF-16BE") == 0 || 1114 strcmp(cs, "UTF16BE") == 0) 1115 return ("UTF-16BE"); 1116 if (strcmp(cs, "UTF-16LE") == 0 || 1117 strcmp(cs, "UTF16LE") == 0) 1118 return ("UTF-16LE"); 1119 if (strcmp(cs, "CP932") == 0) 1120 return ("CP932"); 1121 return (charset); 1122 } 1123 1124 /* 1125 * Create a string conversion object. 1126 */ 1127 static struct archive_string_conv * 1128 create_sconv_object(const char *fc, const char *tc, 1129 unsigned current_codepage, int flag) 1130 { 1131 struct archive_string_conv *sc; 1132 1133 sc = calloc(1, sizeof(*sc)); 1134 if (sc == NULL) 1135 return (NULL); 1136 sc->next = NULL; 1137 sc->from_charset = strdup(fc); 1138 if (sc->from_charset == NULL) { 1139 free(sc); 1140 return (NULL); 1141 } 1142 sc->to_charset = strdup(tc); 1143 if (sc->to_charset == NULL) { 1144 free(sc->from_charset); 1145 free(sc); 1146 return (NULL); 1147 } 1148 archive_string_init(&sc->utftmp); 1149 1150 if (flag & SCONV_TO_CHARSET) { 1151 /* 1152 * Convert characters from the current locale charset to 1153 * a specified charset. 1154 */ 1155 sc->from_cp = current_codepage; 1156 sc->to_cp = make_codepage_from_charset(tc); 1157 #if defined(_WIN32) && !defined(__CYGWIN__) 1158 if (IsValidCodePage(sc->to_cp)) 1159 flag |= SCONV_WIN_CP; 1160 #endif 1161 } else if (flag & SCONV_FROM_CHARSET) { 1162 /* 1163 * Convert characters from a specified charset to 1164 * the current locale charset. 1165 */ 1166 sc->to_cp = current_codepage; 1167 sc->from_cp = make_codepage_from_charset(fc); 1168 #if defined(_WIN32) && !defined(__CYGWIN__) 1169 if (IsValidCodePage(sc->from_cp)) 1170 flag |= SCONV_WIN_CP; 1171 #endif 1172 } 1173 1174 /* 1175 * Check if "from charset" and "to charset" are the same. 1176 */ 1177 if (strcmp(fc, tc) == 0 || 1178 (sc->from_cp != (unsigned)-1 && sc->from_cp == sc->to_cp)) 1179 sc->same = 1; 1180 else 1181 sc->same = 0; 1182 1183 /* 1184 * Mark if "from charset" or "to charset" are UTF-8 or UTF-16BE/LE. 1185 */ 1186 if (strcmp(tc, "UTF-8") == 0) 1187 flag |= SCONV_TO_UTF8; 1188 else if (strcmp(tc, "UTF-16BE") == 0) 1189 flag |= SCONV_TO_UTF16BE; 1190 else if (strcmp(tc, "UTF-16LE") == 0) 1191 flag |= SCONV_TO_UTF16LE; 1192 if (strcmp(fc, "UTF-8") == 0) 1193 flag |= SCONV_FROM_UTF8; 1194 else if (strcmp(fc, "UTF-16BE") == 0) 1195 flag |= SCONV_FROM_UTF16BE; 1196 else if (strcmp(fc, "UTF-16LE") == 0) 1197 flag |= SCONV_FROM_UTF16LE; 1198 #if defined(_WIN32) && !defined(__CYGWIN__) 1199 if (sc->to_cp == CP_UTF8) 1200 flag |= SCONV_TO_UTF8; 1201 else if (sc->to_cp == CP_UTF16BE) 1202 flag |= SCONV_TO_UTF16BE | SCONV_WIN_CP; 1203 else if (sc->to_cp == CP_UTF16LE) 1204 flag |= SCONV_TO_UTF16LE | SCONV_WIN_CP; 1205 if (sc->from_cp == CP_UTF8) 1206 flag |= SCONV_FROM_UTF8; 1207 else if (sc->from_cp == CP_UTF16BE) 1208 flag |= SCONV_FROM_UTF16BE | SCONV_WIN_CP; 1209 else if (sc->from_cp == CP_UTF16LE) 1210 flag |= SCONV_FROM_UTF16LE | SCONV_WIN_CP; 1211 #endif 1212 1213 /* 1214 * Set a flag for Unicode NFD. Usually iconv cannot correctly 1215 * handle it. So we have to translate NFD characters to NFC ones 1216 * ourselves before iconv handles. Another reason is to prevent 1217 * that the same sight of two filenames, one is NFC and other 1218 * is NFD, would be in its directory. 1219 * On Mac OS X, although its filesystem layer automatically 1220 * convert filenames to NFD, it would be useful for filename 1221 * comparing to find out the same filenames that we normalize 1222 * that to be NFD ourselves. 1223 */ 1224 if ((flag & SCONV_FROM_CHARSET) && 1225 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8))) { 1226 #if defined(__APPLE__) 1227 if (flag & SCONV_TO_UTF8) 1228 flag |= SCONV_NORMALIZATION_D; 1229 else 1230 #endif 1231 flag |= SCONV_NORMALIZATION_C; 1232 } 1233 #if defined(__APPLE__) 1234 /* 1235 * In case writing an archive file, make sure that a filename 1236 * going to be passed to iconv is a Unicode NFC string since 1237 * a filename in HFS Plus filesystem is a Unicode NFD one and 1238 * iconv cannot handle it with "UTF-8" charset. It is simpler 1239 * than a use of "UTF-8-MAC" charset. 1240 */ 1241 if ((flag & SCONV_TO_CHARSET) && 1242 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1243 !(flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1244 flag |= SCONV_NORMALIZATION_C; 1245 /* 1246 * In case reading an archive file. make sure that a filename 1247 * will be passed to users is a Unicode NFD string in order to 1248 * correctly compare the filename with other one which comes 1249 * from HFS Plus filesystem. 1250 */ 1251 if ((flag & SCONV_FROM_CHARSET) && 1252 !(flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1253 (flag & SCONV_TO_UTF8)) 1254 flag |= SCONV_NORMALIZATION_D; 1255 #endif 1256 1257 #if defined(HAVE_ICONV) 1258 sc->cd_w = (iconv_t)-1; 1259 /* 1260 * Create an iconv object. 1261 */ 1262 if (((flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) && 1263 (flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16))) || 1264 (flag & SCONV_WIN_CP)) { 1265 /* This case we won't use iconv. */ 1266 sc->cd = (iconv_t)-1; 1267 } else { 1268 sc->cd = iconv_open(tc, fc); 1269 if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT)) { 1270 /* 1271 * Unfortunately, all of iconv implements do support 1272 * "CP932" character-set, so we should use "SJIS" 1273 * instead if iconv_open failed. 1274 */ 1275 if (strcmp(tc, "CP932") == 0) 1276 sc->cd = iconv_open("SJIS", fc); 1277 else if (strcmp(fc, "CP932") == 0) 1278 sc->cd = iconv_open(tc, "SJIS"); 1279 } 1280 #if defined(_WIN32) && !defined(__CYGWIN__) 1281 /* 1282 * archive_mstring on Windows directly convert multi-bytes 1283 * into archive_wstring in order not to depend on locale 1284 * so that you can do a I18N programming. This will be 1285 * used only in archive_mstring_copy_mbs_len_l so far. 1286 */ 1287 if (flag & SCONV_FROM_CHARSET) { 1288 sc->cd_w = iconv_open("UTF-8", fc); 1289 if (sc->cd_w == (iconv_t)-1 && 1290 (sc->flag & SCONV_BEST_EFFORT)) { 1291 if (strcmp(fc, "CP932") == 0) 1292 sc->cd_w = iconv_open("UTF-8", "SJIS"); 1293 } 1294 } 1295 #endif /* _WIN32 && !__CYGWIN__ */ 1296 } 1297 #endif /* HAVE_ICONV */ 1298 1299 sc->flag = flag; 1300 1301 /* 1302 * Set up converters. 1303 */ 1304 setup_converter(sc); 1305 1306 return (sc); 1307 } 1308 1309 /* 1310 * Free a string conversion object. 1311 */ 1312 static void 1313 free_sconv_object(struct archive_string_conv *sc) 1314 { 1315 free(sc->from_charset); 1316 free(sc->to_charset); 1317 archive_string_free(&sc->utftmp); 1318 #if HAVE_ICONV 1319 if (sc->cd != (iconv_t)-1) 1320 iconv_close(sc->cd); 1321 if (sc->cd_w != (iconv_t)-1) 1322 iconv_close(sc->cd_w); 1323 #endif 1324 free(sc); 1325 } 1326 1327 #if defined(_WIN32) && !defined(__CYGWIN__) 1328 # if defined(WINAPI_FAMILY_PARTITION) && !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) 1329 # define GetOEMCP() CP_OEMCP 1330 # endif 1331 1332 static unsigned 1333 my_atoi(const char *p) 1334 { 1335 unsigned cp; 1336 1337 cp = 0; 1338 while (*p) { 1339 if (*p >= '0' && *p <= '9') 1340 cp = cp * 10 + (*p - '0'); 1341 else 1342 return (-1); 1343 p++; 1344 } 1345 return (cp); 1346 } 1347 1348 /* 1349 * Translate Charset name (as used by iconv) into CodePage (as used by Windows) 1350 * Return -1 if failed. 1351 * 1352 * Note: This translation code may be insufficient. 1353 */ 1354 static struct charset { 1355 const char *name; 1356 unsigned cp; 1357 } charsets[] = { 1358 /* MUST BE SORTED! */ 1359 {"ASCII", 1252}, 1360 {"ASMO-708", 708}, 1361 {"BIG5", 950}, 1362 {"CHINESE", 936}, 1363 {"CP367", 1252}, 1364 {"CP819", 1252}, 1365 {"CP1025", 21025}, 1366 {"DOS-720", 720}, 1367 {"DOS-862", 862}, 1368 {"EUC-CN", 51936}, 1369 {"EUC-JP", 51932}, 1370 {"EUC-KR", 949}, 1371 {"EUCCN", 51936}, 1372 {"EUCJP", 51932}, 1373 {"EUCKR", 949}, 1374 {"GB18030", 54936}, 1375 {"GB2312", 936}, 1376 {"HEBREW", 1255}, 1377 {"HZ-GB-2312", 52936}, 1378 {"IBM273", 20273}, 1379 {"IBM277", 20277}, 1380 {"IBM278", 20278}, 1381 {"IBM280", 20280}, 1382 {"IBM284", 20284}, 1383 {"IBM285", 20285}, 1384 {"IBM290", 20290}, 1385 {"IBM297", 20297}, 1386 {"IBM367", 1252}, 1387 {"IBM420", 20420}, 1388 {"IBM423", 20423}, 1389 {"IBM424", 20424}, 1390 {"IBM819", 1252}, 1391 {"IBM871", 20871}, 1392 {"IBM880", 20880}, 1393 {"IBM905", 20905}, 1394 {"IBM924", 20924}, 1395 {"ISO-8859-1", 28591}, 1396 {"ISO-8859-13", 28603}, 1397 {"ISO-8859-15", 28605}, 1398 {"ISO-8859-2", 28592}, 1399 {"ISO-8859-3", 28593}, 1400 {"ISO-8859-4", 28594}, 1401 {"ISO-8859-5", 28595}, 1402 {"ISO-8859-6", 28596}, 1403 {"ISO-8859-7", 28597}, 1404 {"ISO-8859-8", 28598}, 1405 {"ISO-8859-9", 28599}, 1406 {"ISO8859-1", 28591}, 1407 {"ISO8859-13", 28603}, 1408 {"ISO8859-15", 28605}, 1409 {"ISO8859-2", 28592}, 1410 {"ISO8859-3", 28593}, 1411 {"ISO8859-4", 28594}, 1412 {"ISO8859-5", 28595}, 1413 {"ISO8859-6", 28596}, 1414 {"ISO8859-7", 28597}, 1415 {"ISO8859-8", 28598}, 1416 {"ISO8859-9", 28599}, 1417 {"JOHAB", 1361}, 1418 {"KOI8-R", 20866}, 1419 {"KOI8-U", 21866}, 1420 {"KS_C_5601-1987", 949}, 1421 {"LATIN1", 1252}, 1422 {"LATIN2", 28592}, 1423 {"MACINTOSH", 10000}, 1424 {"SHIFT-JIS", 932}, 1425 {"SHIFT_JIS", 932}, 1426 {"SJIS", 932}, 1427 {"US", 1252}, 1428 {"US-ASCII", 1252}, 1429 {"UTF-16", 1200}, 1430 {"UTF-16BE", 1201}, 1431 {"UTF-16LE", 1200}, 1432 {"UTF-8", CP_UTF8}, 1433 {"X-EUROPA", 29001}, 1434 {"X-MAC-ARABIC", 10004}, 1435 {"X-MAC-CE", 10029}, 1436 {"X-MAC-CHINESEIMP", 10008}, 1437 {"X-MAC-CHINESETRAD", 10002}, 1438 {"X-MAC-CROATIAN", 10082}, 1439 {"X-MAC-CYRILLIC", 10007}, 1440 {"X-MAC-GREEK", 10006}, 1441 {"X-MAC-HEBREW", 10005}, 1442 {"X-MAC-ICELANDIC", 10079}, 1443 {"X-MAC-JAPANESE", 10001}, 1444 {"X-MAC-KOREAN", 10003}, 1445 {"X-MAC-ROMANIAN", 10010}, 1446 {"X-MAC-THAI", 10021}, 1447 {"X-MAC-TURKISH", 10081}, 1448 {"X-MAC-UKRAINIAN", 10017}, 1449 }; 1450 static unsigned 1451 make_codepage_from_charset(const char *charset) 1452 { 1453 char cs[16]; 1454 char *p; 1455 unsigned cp; 1456 int a, b; 1457 1458 if (charset == NULL || strlen(charset) > 15) 1459 return -1; 1460 1461 /* Copy name to uppercase. */ 1462 p = cs; 1463 while (*charset) { 1464 char c = *charset++; 1465 if (c >= 'a' && c <= 'z') 1466 c -= 'a' - 'A'; 1467 *p++ = c; 1468 } 1469 *p++ = '\0'; 1470 cp = -1; 1471 1472 /* Look it up in the table first, so that we can easily 1473 * override CP367, which we map to 1252 instead of 367. */ 1474 a = 0; 1475 b = sizeof(charsets)/sizeof(charsets[0]); 1476 while (b > a) { 1477 int c = (b + a) / 2; 1478 int r = strcmp(charsets[c].name, cs); 1479 if (r < 0) 1480 a = c + 1; 1481 else if (r > 0) 1482 b = c; 1483 else 1484 return charsets[c].cp; 1485 } 1486 1487 /* If it's not in the table, try to parse it. */ 1488 switch (*cs) { 1489 case 'C': 1490 if (cs[1] == 'P' && cs[2] >= '0' && cs[2] <= '9') { 1491 cp = my_atoi(cs + 2); 1492 } else if (strcmp(cs, "CP_ACP") == 0) 1493 cp = get_current_codepage(); 1494 else if (strcmp(cs, "CP_OEMCP") == 0) 1495 cp = get_current_oemcp(); 1496 break; 1497 case 'I': 1498 if (cs[1] == 'B' && cs[2] == 'M' && 1499 cs[3] >= '0' && cs[3] <= '9') { 1500 cp = my_atoi(cs + 3); 1501 } 1502 break; 1503 case 'W': 1504 if (strncmp(cs, "WINDOWS-", 8) == 0) { 1505 cp = my_atoi(cs + 8); 1506 if (cp != 874 && (cp < 1250 || cp > 1258)) 1507 cp = -1;/* This may invalid code. */ 1508 } 1509 break; 1510 } 1511 return (cp); 1512 } 1513 1514 /* 1515 * Return ANSI Code Page of current locale set by setlocale(). 1516 */ 1517 static unsigned 1518 get_current_codepage(void) 1519 { 1520 char *locale, *p; 1521 unsigned cp; 1522 1523 locale = setlocale(LC_CTYPE, NULL); 1524 if (locale == NULL) 1525 return (GetACP()); 1526 if (locale[0] == 'C' && locale[1] == '\0') 1527 return (CP_C_LOCALE); 1528 p = strrchr(locale, '.'); 1529 if (p == NULL) 1530 return (GetACP()); 1531 if ((strcmp(p+1, "utf8") == 0) || (strcmp(p+1, "UTF-8") == 0)) 1532 return CP_UTF8; 1533 cp = my_atoi(p+1); 1534 if ((int)cp <= 0) 1535 return (GetACP()); 1536 return (cp); 1537 } 1538 1539 /* 1540 * Translation table between Locale Name and ACP/OEMCP. 1541 */ 1542 static struct { 1543 unsigned acp; 1544 unsigned ocp; 1545 const char *locale; 1546 } acp_ocp_map[] = { 1547 { 950, 950, "Chinese_Taiwan" }, 1548 { 936, 936, "Chinese_People's Republic of China" }, 1549 { 950, 950, "Chinese_Taiwan" }, 1550 { 1250, 852, "Czech_Czech Republic" }, 1551 { 1252, 850, "Danish_Denmark" }, 1552 { 1252, 850, "Dutch_Netherlands" }, 1553 { 1252, 850, "Dutch_Belgium" }, 1554 { 1252, 437, "English_United States" }, 1555 { 1252, 850, "English_Australia" }, 1556 { 1252, 850, "English_Canada" }, 1557 { 1252, 850, "English_New Zealand" }, 1558 { 1252, 850, "English_United Kingdom" }, 1559 { 1252, 437, "English_United States" }, 1560 { 1252, 850, "Finnish_Finland" }, 1561 { 1252, 850, "French_France" }, 1562 { 1252, 850, "French_Belgium" }, 1563 { 1252, 850, "French_Canada" }, 1564 { 1252, 850, "French_Switzerland" }, 1565 { 1252, 850, "German_Germany" }, 1566 { 1252, 850, "German_Austria" }, 1567 { 1252, 850, "German_Switzerland" }, 1568 { 1253, 737, "Greek_Greece" }, 1569 { 1250, 852, "Hungarian_Hungary" }, 1570 { 1252, 850, "Icelandic_Iceland" }, 1571 { 1252, 850, "Italian_Italy" }, 1572 { 1252, 850, "Italian_Switzerland" }, 1573 { 932, 932, "Japanese_Japan" }, 1574 { 949, 949, "Korean_Korea" }, 1575 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1576 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1577 { 1252, 850, "Norwegian-Nynorsk_Norway" }, 1578 { 1250, 852, "Polish_Poland" }, 1579 { 1252, 850, "Portuguese_Portugal" }, 1580 { 1252, 850, "Portuguese_Brazil" }, 1581 { 1251, 866, "Russian_Russia" }, 1582 { 1250, 852, "Slovak_Slovakia" }, 1583 { 1252, 850, "Spanish_Spain" }, 1584 { 1252, 850, "Spanish_Mexico" }, 1585 { 1252, 850, "Spanish_Spain" }, 1586 { 1252, 850, "Swedish_Sweden" }, 1587 { 1254, 857, "Turkish_Turkey" }, 1588 { 0, 0, NULL} 1589 }; 1590 1591 /* 1592 * Return OEM Code Page of current locale set by setlocale(). 1593 */ 1594 static unsigned 1595 get_current_oemcp(void) 1596 { 1597 int i; 1598 char *locale, *p; 1599 size_t len; 1600 1601 locale = setlocale(LC_CTYPE, NULL); 1602 if (locale == NULL) 1603 return (GetOEMCP()); 1604 if (locale[0] == 'C' && locale[1] == '\0') 1605 return (CP_C_LOCALE); 1606 1607 p = strrchr(locale, '.'); 1608 if (p == NULL) 1609 return (GetOEMCP()); 1610 len = p - locale; 1611 for (i = 0; acp_ocp_map[i].acp; i++) { 1612 if (strncmp(acp_ocp_map[i].locale, locale, len) == 0) 1613 return (acp_ocp_map[i].ocp); 1614 } 1615 return (GetOEMCP()); 1616 } 1617 #else 1618 1619 /* 1620 * POSIX platform does not use CodePage. 1621 */ 1622 1623 static unsigned 1624 get_current_codepage(void) 1625 { 1626 return (-1);/* Unknown */ 1627 } 1628 static unsigned 1629 make_codepage_from_charset(const char *charset) 1630 { 1631 (void)charset; /* UNUSED */ 1632 return (-1);/* Unknown */ 1633 } 1634 static unsigned 1635 get_current_oemcp(void) 1636 { 1637 return (-1);/* Unknown */ 1638 } 1639 1640 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 1641 1642 /* 1643 * Return a string conversion object. 1644 */ 1645 static struct archive_string_conv * 1646 get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag) 1647 { 1648 struct archive_string_conv *sc; 1649 unsigned current_codepage; 1650 1651 /* Check if we have made the sconv object. */ 1652 sc = find_sconv_object(a, fc, tc); 1653 if (sc != NULL) 1654 return (sc); 1655 1656 if (a == NULL) 1657 current_codepage = get_current_codepage(); 1658 else 1659 current_codepage = a->current_codepage; 1660 1661 sc = create_sconv_object(canonical_charset_name(fc), 1662 canonical_charset_name(tc), current_codepage, flag); 1663 if (sc == NULL) { 1664 if (a != NULL) 1665 archive_set_error(a, ENOMEM, 1666 "Could not allocate memory for " 1667 "a string conversion object"); 1668 return (NULL); 1669 } 1670 1671 /* 1672 * If there is no converter for current string conversion object, 1673 * we cannot handle this conversion. 1674 */ 1675 if (sc->nconverter == 0) { 1676 if (a != NULL) { 1677 #if HAVE_ICONV 1678 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1679 "iconv_open failed : Cannot handle ``%s''", 1680 (flag & SCONV_TO_CHARSET)?tc:fc); 1681 #else 1682 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1683 "A character-set conversion not fully supported " 1684 "on this platform"); 1685 #endif 1686 } 1687 /* Failed; free a sconv object. */ 1688 free_sconv_object(sc); 1689 return (NULL); 1690 } 1691 1692 /* 1693 * Success! 1694 */ 1695 if (a != NULL) 1696 add_sconv_object(a, sc); 1697 return (sc); 1698 } 1699 1700 static const char * 1701 get_current_charset(struct archive *a) 1702 { 1703 const char *cur_charset; 1704 1705 if (a == NULL) 1706 cur_charset = default_iconv_charset(""); 1707 else { 1708 cur_charset = default_iconv_charset(a->current_code); 1709 if (a->current_code == NULL) { 1710 a->current_code = strdup(cur_charset); 1711 a->current_codepage = get_current_codepage(); 1712 a->current_oemcp = get_current_oemcp(); 1713 } 1714 } 1715 return (cur_charset); 1716 } 1717 1718 /* 1719 * Make and Return a string conversion object. 1720 * Return NULL if the platform does not support the specified conversion 1721 * and best_effort is 0. 1722 * If best_effort is set, A string conversion object must be returned 1723 * unless memory allocation for the object fails, but the conversion 1724 * might fail when non-ASCII code is found. 1725 */ 1726 struct archive_string_conv * 1727 archive_string_conversion_to_charset(struct archive *a, const char *charset, 1728 int best_effort) 1729 { 1730 int flag = SCONV_TO_CHARSET; 1731 1732 if (best_effort) 1733 flag |= SCONV_BEST_EFFORT; 1734 return (get_sconv_object(a, get_current_charset(a), charset, flag)); 1735 } 1736 1737 struct archive_string_conv * 1738 archive_string_conversion_from_charset(struct archive *a, const char *charset, 1739 int best_effort) 1740 { 1741 int flag = SCONV_FROM_CHARSET; 1742 1743 if (best_effort) 1744 flag |= SCONV_BEST_EFFORT; 1745 return (get_sconv_object(a, charset, get_current_charset(a), flag)); 1746 } 1747 1748 /* 1749 * archive_string_default_conversion_*_archive() are provided for Windows 1750 * platform because other archiver application use CP_OEMCP for 1751 * MultiByteToWideChar() and WideCharToMultiByte() for the filenames 1752 * in tar or zip files. But mbstowcs/wcstombs(CRT) usually use CP_ACP 1753 * unless you use setlocale(LC_ALL, ".OCP")(specify CP_OEMCP). 1754 * So we should make a string conversion between CP_ACP and CP_OEMCP 1755 * for compatibility. 1756 */ 1757 #if defined(_WIN32) && !defined(__CYGWIN__) 1758 struct archive_string_conv * 1759 archive_string_default_conversion_for_read(struct archive *a) 1760 { 1761 const char *cur_charset = get_current_charset(a); 1762 char oemcp[16]; 1763 1764 /* NOTE: a check of cur_charset is unneeded but we need 1765 * that get_current_charset() has been surely called at 1766 * this time whatever C compiler optimized. */ 1767 if (cur_charset != NULL && 1768 (a->current_codepage == CP_C_LOCALE || 1769 a->current_codepage == a->current_oemcp)) 1770 return (NULL);/* no conversion. */ 1771 1772 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1773 /* Make sure a null termination must be set. */ 1774 oemcp[sizeof(oemcp)-1] = '\0'; 1775 return (get_sconv_object(a, oemcp, cur_charset, 1776 SCONV_FROM_CHARSET)); 1777 } 1778 1779 struct archive_string_conv * 1780 archive_string_default_conversion_for_write(struct archive *a) 1781 { 1782 const char *cur_charset = get_current_charset(a); 1783 char oemcp[16]; 1784 1785 /* NOTE: a check of cur_charset is unneeded but we need 1786 * that get_current_charset() has been surely called at 1787 * this time whatever C compiler optimized. */ 1788 if (cur_charset != NULL && 1789 (a->current_codepage == CP_C_LOCALE || 1790 a->current_codepage == a->current_oemcp)) 1791 return (NULL);/* no conversion. */ 1792 1793 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1794 /* Make sure a null termination must be set. */ 1795 oemcp[sizeof(oemcp)-1] = '\0'; 1796 return (get_sconv_object(a, cur_charset, oemcp, 1797 SCONV_TO_CHARSET)); 1798 } 1799 #else 1800 struct archive_string_conv * 1801 archive_string_default_conversion_for_read(struct archive *a) 1802 { 1803 (void)a; /* UNUSED */ 1804 return (NULL); 1805 } 1806 1807 struct archive_string_conv * 1808 archive_string_default_conversion_for_write(struct archive *a) 1809 { 1810 (void)a; /* UNUSED */ 1811 return (NULL); 1812 } 1813 #endif 1814 1815 /* 1816 * Dispose of all character conversion objects in the archive object. 1817 */ 1818 void 1819 archive_string_conversion_free(struct archive *a) 1820 { 1821 struct archive_string_conv *sc; 1822 struct archive_string_conv *sc_next; 1823 1824 for (sc = a->sconv; sc != NULL; sc = sc_next) { 1825 sc_next = sc->next; 1826 free_sconv_object(sc); 1827 } 1828 a->sconv = NULL; 1829 free(a->current_code); 1830 a->current_code = NULL; 1831 } 1832 1833 /* 1834 * Return a conversion charset name. 1835 */ 1836 const char * 1837 archive_string_conversion_charset_name(struct archive_string_conv *sc) 1838 { 1839 if (sc->flag & SCONV_TO_CHARSET) 1840 return (sc->to_charset); 1841 else 1842 return (sc->from_charset); 1843 } 1844 1845 /* 1846 * Change the behavior of a string conversion. 1847 */ 1848 void 1849 archive_string_conversion_set_opt(struct archive_string_conv *sc, int opt) 1850 { 1851 switch (opt) { 1852 /* 1853 * A filename in UTF-8 was made with libarchive 2.x in a wrong 1854 * assumption that wchar_t was Unicode. 1855 * This option enables simulating the assumption in order to read 1856 * that filename correctly. 1857 */ 1858 case SCONV_SET_OPT_UTF8_LIBARCHIVE2X: 1859 #if (defined(_WIN32) && !defined(__CYGWIN__)) \ 1860 || defined(__STDC_ISO_10646__) || defined(__APPLE__) 1861 /* 1862 * Nothing to do for it since wchar_t on these platforms 1863 * is really Unicode. 1864 */ 1865 (void)sc; /* UNUSED */ 1866 #else 1867 if ((sc->flag & SCONV_UTF8_LIBARCHIVE_2) == 0) { 1868 sc->flag |= SCONV_UTF8_LIBARCHIVE_2; 1869 /* Set up string converters. */ 1870 setup_converter(sc); 1871 } 1872 #endif 1873 break; 1874 case SCONV_SET_OPT_NORMALIZATION_C: 1875 if ((sc->flag & SCONV_NORMALIZATION_C) == 0) { 1876 sc->flag |= SCONV_NORMALIZATION_C; 1877 sc->flag &= ~SCONV_NORMALIZATION_D; 1878 /* Set up string converters. */ 1879 setup_converter(sc); 1880 } 1881 break; 1882 case SCONV_SET_OPT_NORMALIZATION_D: 1883 #if defined(HAVE_ICONV) 1884 /* 1885 * If iconv will take the string, do not change the 1886 * setting of the normalization. 1887 */ 1888 if (!(sc->flag & SCONV_WIN_CP) && 1889 (sc->flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1890 !(sc->flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1891 break; 1892 #endif 1893 if ((sc->flag & SCONV_NORMALIZATION_D) == 0) { 1894 sc->flag |= SCONV_NORMALIZATION_D; 1895 sc->flag &= ~SCONV_NORMALIZATION_C; 1896 /* Set up string converters. */ 1897 setup_converter(sc); 1898 } 1899 break; 1900 default: 1901 break; 1902 } 1903 } 1904 1905 /* 1906 * 1907 * Copy one archive_string to another in locale conversion. 1908 * 1909 * archive_strncat_l(); 1910 * archive_strncpy_l(); 1911 * 1912 */ 1913 1914 static size_t 1915 mbsnbytes(const void *_p, size_t n) 1916 { 1917 size_t s; 1918 const char *p, *pp; 1919 1920 if (_p == NULL) 1921 return (0); 1922 p = (const char *)_p; 1923 1924 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1925 s = 0; 1926 pp = p; 1927 while (s < n && *pp) { 1928 pp++; 1929 s++; 1930 } 1931 return (s); 1932 } 1933 1934 static size_t 1935 utf16nbytes(const void *_p, size_t n) 1936 { 1937 size_t s; 1938 const char *p, *pp; 1939 1940 if (_p == NULL) 1941 return (0); 1942 p = (const char *)_p; 1943 1944 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1945 s = 0; 1946 pp = p; 1947 n >>= 1; 1948 while (s < n && (pp[0] || pp[1])) { 1949 pp += 2; 1950 s++; 1951 } 1952 return (s<<1); 1953 } 1954 1955 int 1956 archive_strncpy_l(struct archive_string *as, const void *_p, size_t n, 1957 struct archive_string_conv *sc) 1958 { 1959 as->length = 0; 1960 return (archive_strncat_l(as, _p, n, sc)); 1961 } 1962 1963 int 1964 archive_strncat_l(struct archive_string *as, const void *_p, size_t n, 1965 struct archive_string_conv *sc) 1966 { 1967 const void *s; 1968 size_t length = 0; 1969 int i, r = 0, r2; 1970 1971 if (_p != NULL && n > 0) { 1972 if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) 1973 length = utf16nbytes(_p, n); 1974 else 1975 length = mbsnbytes(_p, n); 1976 } 1977 1978 /* We must allocate memory even if there is no data for conversion 1979 * or copy. This simulates archive_string_append behavior. */ 1980 if (length == 0) { 1981 int tn = 1; 1982 if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) 1983 tn = 2; 1984 if (archive_string_ensure(as, as->length + tn) == NULL) 1985 return (-1); 1986 as->s[as->length] = 0; 1987 if (tn == 2) 1988 as->s[as->length+1] = 0; 1989 return (0); 1990 } 1991 1992 /* 1993 * If sc is NULL, we just make a copy. 1994 */ 1995 if (sc == NULL) { 1996 if (archive_string_append(as, _p, length) == NULL) 1997 return (-1);/* No memory */ 1998 return (0); 1999 } 2000 2001 s = _p; 2002 i = 0; 2003 if (sc->nconverter > 1) { 2004 sc->utftmp.length = 0; 2005 r2 = sc->converter[0](&(sc->utftmp), s, length, sc); 2006 if (r2 != 0 && errno == ENOMEM) 2007 return (r2); 2008 if (r > r2) 2009 r = r2; 2010 s = sc->utftmp.s; 2011 length = sc->utftmp.length; 2012 ++i; 2013 } 2014 r2 = sc->converter[i](as, s, length, sc); 2015 if (r > r2) 2016 r = r2; 2017 return (r); 2018 } 2019 2020 #if HAVE_ICONV 2021 2022 /* 2023 * Return -1 if conversion fails. 2024 */ 2025 static int 2026 iconv_strncat_in_locale(struct archive_string *as, const void *_p, 2027 size_t length, struct archive_string_conv *sc) 2028 { 2029 ICONV_CONST char *itp; 2030 size_t remaining; 2031 iconv_t cd; 2032 char *outp; 2033 size_t avail, bs; 2034 int return_value = 0; /* success */ 2035 int to_size, from_size; 2036 2037 if (sc->flag & SCONV_TO_UTF16) 2038 to_size = 2; 2039 else 2040 to_size = 1; 2041 if (sc->flag & SCONV_FROM_UTF16) 2042 from_size = 2; 2043 else 2044 from_size = 1; 2045 2046 if (archive_string_ensure(as, as->length + length*2+to_size) == NULL) 2047 return (-1); 2048 2049 cd = sc->cd; 2050 itp = (char *)(uintptr_t)_p; 2051 remaining = length; 2052 outp = as->s + as->length; 2053 avail = as->buffer_length - as->length - to_size; 2054 while (remaining >= (size_t)from_size) { 2055 size_t result = iconv(cd, &itp, &remaining, &outp, &avail); 2056 2057 if (result != (size_t)-1) 2058 break; /* Conversion completed. */ 2059 2060 if (errno == EILSEQ || errno == EINVAL) { 2061 /* 2062 * If an output charset is UTF-8 or UTF-16BE/LE, 2063 * unknown character should be U+FFFD 2064 * (replacement character). 2065 */ 2066 if (sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) { 2067 size_t rbytes; 2068 if (sc->flag & SCONV_TO_UTF8) 2069 rbytes = sizeof(utf8_replacement_char); 2070 else 2071 rbytes = 2; 2072 2073 if (avail < rbytes) { 2074 as->length = outp - as->s; 2075 bs = as->buffer_length + 2076 (remaining * to_size) + rbytes; 2077 if (NULL == 2078 archive_string_ensure(as, bs)) 2079 return (-1); 2080 outp = as->s + as->length; 2081 avail = as->buffer_length 2082 - as->length - to_size; 2083 } 2084 if (sc->flag & SCONV_TO_UTF8) 2085 memcpy(outp, utf8_replacement_char, sizeof(utf8_replacement_char)); 2086 else if (sc->flag & SCONV_TO_UTF16BE) 2087 archive_be16enc(outp, UNICODE_R_CHAR); 2088 else 2089 archive_le16enc(outp, UNICODE_R_CHAR); 2090 outp += rbytes; 2091 avail -= rbytes; 2092 } else { 2093 /* Skip the illegal input bytes. */ 2094 *outp++ = '?'; 2095 avail--; 2096 } 2097 itp += from_size; 2098 remaining -= from_size; 2099 return_value = -1; /* failure */ 2100 } else { 2101 /* E2BIG no output buffer, 2102 * Increase an output buffer. */ 2103 as->length = outp - as->s; 2104 bs = as->buffer_length + remaining * 2; 2105 if (NULL == archive_string_ensure(as, bs)) 2106 return (-1); 2107 outp = as->s + as->length; 2108 avail = as->buffer_length - as->length - to_size; 2109 } 2110 } 2111 as->length = outp - as->s; 2112 as->s[as->length] = 0; 2113 if (to_size == 2) 2114 as->s[as->length+1] = 0; 2115 return (return_value); 2116 } 2117 2118 #endif /* HAVE_ICONV */ 2119 2120 2121 #if defined(_WIN32) && !defined(__CYGWIN__) 2122 2123 /* 2124 * Translate a string from a some CodePage to an another CodePage by 2125 * Windows APIs, and copy the result. Return -1 if conversion fails. 2126 */ 2127 static int 2128 strncat_in_codepage(struct archive_string *as, 2129 const void *_p, size_t length, struct archive_string_conv *sc) 2130 { 2131 const char *s = (const char *)_p; 2132 struct archive_wstring aws; 2133 size_t l; 2134 int r, saved_flag; 2135 2136 archive_string_init(&aws); 2137 saved_flag = sc->flag; 2138 sc->flag &= ~(SCONV_NORMALIZATION_D | SCONV_NORMALIZATION_C); 2139 r = archive_wstring_append_from_mbs_in_codepage(&aws, s, length, sc); 2140 sc->flag = saved_flag; 2141 if (r != 0) { 2142 archive_wstring_free(&aws); 2143 if (errno != ENOMEM) 2144 archive_string_append(as, s, length); 2145 return (-1); 2146 } 2147 2148 l = as->length; 2149 r = archive_string_append_from_wcs_in_codepage( 2150 as, aws.s, aws.length, sc); 2151 if (r != 0 && errno != ENOMEM && l == as->length) 2152 archive_string_append(as, s, length); 2153 archive_wstring_free(&aws); 2154 return (r); 2155 } 2156 2157 /* 2158 * Test whether MBS ==> WCS is okay. 2159 */ 2160 static int 2161 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2162 { 2163 const char *p = (const char *)_p; 2164 unsigned codepage; 2165 DWORD mbflag = MB_ERR_INVALID_CHARS; 2166 2167 if (sc->flag & SCONV_FROM_CHARSET) 2168 codepage = sc->to_cp; 2169 else 2170 codepage = sc->from_cp; 2171 2172 if (codepage == CP_C_LOCALE) 2173 return (0); 2174 if (codepage != CP_UTF8) 2175 mbflag |= MB_PRECOMPOSED; 2176 2177 if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0) 2178 return (-1); /* Invalid */ 2179 return (0); /* Okay */ 2180 } 2181 2182 #else 2183 2184 /* 2185 * Test whether MBS ==> WCS is okay. 2186 */ 2187 static int 2188 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2189 { 2190 const char *p = (const char *)_p; 2191 size_t r; 2192 2193 #if HAVE_MBRTOWC 2194 mbstate_t shift_state; 2195 2196 memset(&shift_state, 0, sizeof(shift_state)); 2197 #else 2198 /* Clear the shift state before starting. */ 2199 mbtowc(NULL, NULL, 0); 2200 #endif 2201 while (n) { 2202 wchar_t wc; 2203 2204 #if HAVE_MBRTOWC 2205 r = mbrtowc(&wc, p, n, &shift_state); 2206 #else 2207 r = mbtowc(&wc, p, n); 2208 #endif 2209 if (r == (size_t)-1 || r == (size_t)-2) 2210 return (-1);/* Invalid. */ 2211 if (r == 0) 2212 break; 2213 p += r; 2214 n -= r; 2215 } 2216 (void)sc; /* UNUSED */ 2217 return (0); /* All Okey. */ 2218 } 2219 2220 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 2221 2222 /* 2223 * Basically returns -1 because we cannot make a conversion of charset 2224 * without iconv but in some cases this would return 0. 2225 * Returns 0 if all copied characters are ASCII. 2226 * Returns 0 if both from-locale and to-locale are the same and those 2227 * can be WCS with no error. 2228 */ 2229 static int 2230 best_effort_strncat_in_locale(struct archive_string *as, const void *_p, 2231 size_t length, struct archive_string_conv *sc) 2232 { 2233 size_t remaining; 2234 const uint8_t *itp; 2235 int return_value = 0; /* success */ 2236 2237 /* 2238 * If both from-locale and to-locale is the same, this makes a copy. 2239 * And then this checks all copied MBS can be WCS if so returns 0. 2240 */ 2241 if (sc->same) { 2242 if (archive_string_append(as, _p, length) == NULL) 2243 return (-1);/* No memory */ 2244 return (invalid_mbs(_p, length, sc)); 2245 } 2246 2247 /* 2248 * If a character is ASCII, this just copies it. If not, this 2249 * assigns '?' character instead but in UTF-8 locale this assigns 2250 * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD, 2251 * a Replacement Character in Unicode. 2252 */ 2253 2254 remaining = length; 2255 itp = (const uint8_t *)_p; 2256 while (*itp && remaining > 0) { 2257 if (*itp > 127) { 2258 // Non-ASCII: Substitute with suitable replacement 2259 if (sc->flag & SCONV_TO_UTF8) { 2260 if (archive_string_append(as, utf8_replacement_char, sizeof(utf8_replacement_char)) == NULL) { 2261 __archive_errx(1, "Out of memory"); 2262 } 2263 } else { 2264 archive_strappend_char(as, '?'); 2265 } 2266 return_value = -1; 2267 } else { 2268 archive_strappend_char(as, *itp); 2269 } 2270 ++itp; 2271 } 2272 return (return_value); 2273 } 2274 2275 2276 /* 2277 * Unicode conversion functions. 2278 * - UTF-8 <===> UTF-8 in removing surrogate pairs. 2279 * - UTF-8 NFD ===> UTF-8 NFC in removing surrogate pairs. 2280 * - UTF-8 made by libarchive 2.x ===> UTF-8. 2281 * - UTF-16BE <===> UTF-8. 2282 * 2283 */ 2284 2285 /* 2286 * Utility to convert a single UTF-8 sequence. 2287 * 2288 * Usually return used bytes, return used byte in negative value when 2289 * a unicode character is replaced with U+FFFD. 2290 * See also http://unicode.org/review/pr-121.html Public Review Issue #121 2291 * Recommended Practice for Replacement Characters. 2292 */ 2293 static int 2294 _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2295 { 2296 static const char utf8_count[256] = { 2297 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 00 - 0F */ 2298 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 10 - 1F */ 2299 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 20 - 2F */ 2300 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 30 - 3F */ 2301 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 40 - 4F */ 2302 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 50 - 5F */ 2303 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 60 - 6F */ 2304 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 70 - 7F */ 2305 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 80 - 8F */ 2306 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 90 - 9F */ 2307 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* A0 - AF */ 2308 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* B0 - BF */ 2309 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* C0 - CF */ 2310 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* D0 - DF */ 2311 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,/* E0 - EF */ 2312 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0 - FF */ 2313 }; 2314 int ch, i; 2315 int cnt; 2316 uint32_t wc; 2317 2318 /* Sanity check. */ 2319 if (n == 0) 2320 return (0); 2321 /* 2322 * Decode 1-4 bytes depending on the value of the first byte. 2323 */ 2324 ch = (unsigned char)*s; 2325 if (ch == 0) 2326 return (0); /* Standard: return 0 for end-of-string. */ 2327 cnt = utf8_count[ch]; 2328 2329 /* Invalid sequence or there are not plenty bytes. */ 2330 if ((int)n < cnt) { 2331 cnt = (int)n; 2332 for (i = 1; i < cnt; i++) { 2333 if ((s[i] & 0xc0) != 0x80) { 2334 cnt = i; 2335 break; 2336 } 2337 } 2338 goto invalid_sequence; 2339 } 2340 2341 /* Make a Unicode code point from a single UTF-8 sequence. */ 2342 switch (cnt) { 2343 case 1: /* 1 byte sequence. */ 2344 *pwc = ch & 0x7f; 2345 return (cnt); 2346 case 2: /* 2 bytes sequence. */ 2347 if ((s[1] & 0xc0) != 0x80) { 2348 cnt = 1; 2349 goto invalid_sequence; 2350 } 2351 *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f); 2352 return (cnt); 2353 case 3: /* 3 bytes sequence. */ 2354 if ((s[1] & 0xc0) != 0x80) { 2355 cnt = 1; 2356 goto invalid_sequence; 2357 } 2358 if ((s[2] & 0xc0) != 0x80) { 2359 cnt = 2; 2360 goto invalid_sequence; 2361 } 2362 wc = ((ch & 0x0f) << 12) 2363 | ((s[1] & 0x3f) << 6) 2364 | (s[2] & 0x3f); 2365 if (wc < 0x800) 2366 goto invalid_sequence;/* Overlong sequence. */ 2367 break; 2368 case 4: /* 4 bytes sequence. */ 2369 if ((s[1] & 0xc0) != 0x80) { 2370 cnt = 1; 2371 goto invalid_sequence; 2372 } 2373 if ((s[2] & 0xc0) != 0x80) { 2374 cnt = 2; 2375 goto invalid_sequence; 2376 } 2377 if ((s[3] & 0xc0) != 0x80) { 2378 cnt = 3; 2379 goto invalid_sequence; 2380 } 2381 wc = ((ch & 0x07) << 18) 2382 | ((s[1] & 0x3f) << 12) 2383 | ((s[2] & 0x3f) << 6) 2384 | (s[3] & 0x3f); 2385 if (wc < 0x10000) 2386 goto invalid_sequence;/* Overlong sequence. */ 2387 break; 2388 default: /* Others are all invalid sequence. */ 2389 if (ch == 0xc0 || ch == 0xc1) 2390 cnt = 2; 2391 else if (ch >= 0xf5 && ch <= 0xf7) 2392 cnt = 4; 2393 else if (ch >= 0xf8 && ch <= 0xfb) 2394 cnt = 5; 2395 else if (ch == 0xfc || ch == 0xfd) 2396 cnt = 6; 2397 else 2398 cnt = 1; 2399 if ((int)n < cnt) 2400 cnt = (int)n; 2401 for (i = 1; i < cnt; i++) { 2402 if ((s[i] & 0xc0) != 0x80) { 2403 cnt = i; 2404 break; 2405 } 2406 } 2407 goto invalid_sequence; 2408 } 2409 2410 /* The code point larger than 0x10FFFF is not legal 2411 * Unicode values. */ 2412 if (wc > UNICODE_MAX) 2413 goto invalid_sequence; 2414 /* Correctly gets a Unicode, returns used bytes. */ 2415 *pwc = wc; 2416 return (cnt); 2417 invalid_sequence: 2418 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2419 return (cnt * -1); 2420 } 2421 2422 static int 2423 utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2424 { 2425 int cnt; 2426 2427 cnt = _utf8_to_unicode(pwc, s, n); 2428 /* Any of Surrogate pair is not legal Unicode values. */ 2429 if (cnt == 3 && IS_SURROGATE_PAIR_LA(*pwc)) 2430 return (-3); 2431 return (cnt); 2432 } 2433 2434 static inline uint32_t 2435 combine_surrogate_pair(uint32_t uc, uint32_t uc2) 2436 { 2437 uc -= 0xD800; 2438 uc *= 0x400; 2439 uc += uc2 - 0xDC00; 2440 uc += 0x10000; 2441 return (uc); 2442 } 2443 2444 /* 2445 * Convert a single UTF-8/CESU-8 sequence to a Unicode code point in 2446 * removing surrogate pairs. 2447 * 2448 * CESU-8: The Compatibility Encoding Scheme for UTF-16. 2449 * 2450 * Usually return used bytes, return used byte in negative value when 2451 * a unicode character is replaced with U+FFFD. 2452 */ 2453 static int 2454 cesu8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2455 { 2456 uint32_t wc = 0; 2457 int cnt; 2458 2459 cnt = _utf8_to_unicode(&wc, s, n); 2460 if (cnt == 3 && IS_HIGH_SURROGATE_LA(wc)) { 2461 uint32_t wc2 = 0; 2462 if (n - 3 < 3) { 2463 /* Invalid byte sequence. */ 2464 goto invalid_sequence; 2465 } 2466 cnt = _utf8_to_unicode(&wc2, s+3, n-3); 2467 if (cnt != 3 || !IS_LOW_SURROGATE_LA(wc2)) { 2468 /* Invalid byte sequence. */ 2469 goto invalid_sequence; 2470 } 2471 wc = combine_surrogate_pair(wc, wc2); 2472 cnt = 6; 2473 } else if (cnt == 3 && IS_LOW_SURROGATE_LA(wc)) { 2474 /* Invalid byte sequence. */ 2475 goto invalid_sequence; 2476 } 2477 *pwc = wc; 2478 return (cnt); 2479 invalid_sequence: 2480 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2481 if (cnt > 0) 2482 cnt *= -1; 2483 return (cnt); 2484 } 2485 2486 /* 2487 * Convert a Unicode code point to a single UTF-8 sequence. 2488 * 2489 * NOTE:This function does not check if the Unicode is legal or not. 2490 * Please you definitely check it before calling this. 2491 */ 2492 static size_t 2493 unicode_to_utf8(char *p, size_t remaining, uint32_t uc) 2494 { 2495 char *_p = p; 2496 2497 /* Invalid Unicode char maps to Replacement character */ 2498 if (uc > UNICODE_MAX) 2499 uc = UNICODE_R_CHAR; 2500 /* Translate code point to UTF8 */ 2501 if (uc <= 0x7f) { 2502 if (remaining == 0) 2503 return (0); 2504 *p++ = (char)uc; 2505 } else if (uc <= 0x7ff) { 2506 if (remaining < 2) 2507 return (0); 2508 *p++ = 0xc0 | ((uc >> 6) & 0x1f); 2509 *p++ = 0x80 | (uc & 0x3f); 2510 } else if (uc <= 0xffff) { 2511 if (remaining < 3) 2512 return (0); 2513 *p++ = 0xe0 | ((uc >> 12) & 0x0f); 2514 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2515 *p++ = 0x80 | (uc & 0x3f); 2516 } else { 2517 if (remaining < 4) 2518 return (0); 2519 *p++ = 0xf0 | ((uc >> 18) & 0x07); 2520 *p++ = 0x80 | ((uc >> 12) & 0x3f); 2521 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2522 *p++ = 0x80 | (uc & 0x3f); 2523 } 2524 return (p - _p); 2525 } 2526 2527 static int 2528 utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n) 2529 { 2530 return (utf16_to_unicode(pwc, s, n, 1)); 2531 } 2532 2533 static int 2534 utf16le_to_unicode(uint32_t *pwc, const char *s, size_t n) 2535 { 2536 return (utf16_to_unicode(pwc, s, n, 0)); 2537 } 2538 2539 static int 2540 utf16_to_unicode(uint32_t *pwc, const char *s, size_t n, int be) 2541 { 2542 const char *utf16 = s; 2543 unsigned uc; 2544 2545 if (n == 0) 2546 return (0); 2547 if (n == 1) { 2548 /* set the Replacement Character instead. */ 2549 *pwc = UNICODE_R_CHAR; 2550 return (-1); 2551 } 2552 2553 if (be) 2554 uc = archive_be16dec(utf16); 2555 else 2556 uc = archive_le16dec(utf16); 2557 utf16 += 2; 2558 2559 /* If this is a surrogate pair, assemble the full code point.*/ 2560 if (IS_HIGH_SURROGATE_LA(uc)) { 2561 unsigned uc2; 2562 2563 if (n >= 4) { 2564 if (be) 2565 uc2 = archive_be16dec(utf16); 2566 else 2567 uc2 = archive_le16dec(utf16); 2568 } else 2569 uc2 = 0; 2570 if (IS_LOW_SURROGATE_LA(uc2)) { 2571 uc = combine_surrogate_pair(uc, uc2); 2572 utf16 += 2; 2573 } else { 2574 /* Undescribed code point should be U+FFFD 2575 * (replacement character). */ 2576 *pwc = UNICODE_R_CHAR; 2577 return (-2); 2578 } 2579 } 2580 2581 /* 2582 * Surrogate pair values(0xd800 through 0xdfff) are only 2583 * used by UTF-16, so, after above calculation, the code 2584 * must not be surrogate values, and Unicode has no codes 2585 * larger than 0x10ffff. Thus, those are not legal Unicode 2586 * values. 2587 */ 2588 if (IS_SURROGATE_PAIR_LA(uc) || uc > UNICODE_MAX) { 2589 /* Undescribed code point should be U+FFFD 2590 * (replacement character). */ 2591 *pwc = UNICODE_R_CHAR; 2592 return (((int)(utf16 - s)) * -1); 2593 } 2594 *pwc = uc; 2595 return ((int)(utf16 - s)); 2596 } 2597 2598 static size_t 2599 unicode_to_utf16be(char *p, size_t remaining, uint32_t uc) 2600 { 2601 char *utf16 = p; 2602 2603 if (uc > 0xffff) { 2604 /* We have a code point that won't fit into a 2605 * wchar_t; convert it to a surrogate pair. */ 2606 if (remaining < 4) 2607 return (0); 2608 uc -= 0x10000; 2609 archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2610 archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2611 return (4); 2612 } else { 2613 if (remaining < 2) 2614 return (0); 2615 archive_be16enc(utf16, uc); 2616 return (2); 2617 } 2618 } 2619 2620 static size_t 2621 unicode_to_utf16le(char *p, size_t remaining, uint32_t uc) 2622 { 2623 char *utf16 = p; 2624 2625 if (uc > 0xffff) { 2626 /* We have a code point that won't fit into a 2627 * wchar_t; convert it to a surrogate pair. */ 2628 if (remaining < 4) 2629 return (0); 2630 uc -= 0x10000; 2631 archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2632 archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2633 return (4); 2634 } else { 2635 if (remaining < 2) 2636 return (0); 2637 archive_le16enc(utf16, uc); 2638 return (2); 2639 } 2640 } 2641 2642 /* 2643 * Append new UTF-8 string to existing UTF-8 string. 2644 * Existing string is assumed to already be in proper form; 2645 * the new string will have invalid sequences replaced and 2646 * surrogate pairs canonicalized. 2647 */ 2648 static int 2649 strncat_from_utf8_to_utf8(struct archive_string *as, const void *_src, 2650 size_t len, struct archive_string_conv *sc) 2651 { 2652 int ret = 0; 2653 const char *src = _src; 2654 (void)sc; /* UNUSED */ 2655 2656 /* Pre-extend the destination */ 2657 if (archive_string_ensure(as, as->length + len + 1) == NULL) 2658 return (-1); 2659 2660 /* Invariant: src points to the first UTF8 byte that hasn't 2661 * been copied to the destination `as`. */ 2662 for (;;) { 2663 int n; 2664 uint32_t uc; 2665 const char *e = src; 2666 2667 /* Skip UTF-8 sequences until we reach end-of-string or 2668 * a code point that needs conversion. */ 2669 while ((n = utf8_to_unicode(&uc, e, len)) > 0) { 2670 e += n; 2671 len -= n; 2672 } 2673 /* Copy the part that doesn't need conversion */ 2674 if (e > src) { 2675 if (archive_string_append(as, src, e - src) == NULL) 2676 return (-1); 2677 src = e; 2678 } 2679 2680 if (n == 0) { 2681 /* We reached end-of-string */ 2682 return (ret); 2683 } else { 2684 /* Next code point needs conversion */ 2685 char t[4]; 2686 size_t w; 2687 2688 /* Try decoding a surrogate pair */ 2689 if (n == -3 && IS_SURROGATE_PAIR_LA(uc)) { 2690 n = cesu8_to_unicode(&uc, src, len); 2691 } 2692 /* Not a (valid) surrogate, so use a replacement char */ 2693 if (n < 0) { 2694 ret = -1; /* Return -1 if we used any replacement */ 2695 n *= -1; 2696 } 2697 /* Consume converted code point */ 2698 src += n; 2699 len -= n; 2700 /* Convert and append new UTF-8 sequence. */ 2701 w = unicode_to_utf8(t, sizeof(t), uc); 2702 if (archive_string_append(as, t, w) == NULL) 2703 return (-1); 2704 } 2705 } 2706 } 2707 2708 static int 2709 archive_string_append_unicode(struct archive_string *as, const void *_p, 2710 size_t len, struct archive_string_conv *sc) 2711 { 2712 const char *s; 2713 char *p, *endp; 2714 uint32_t uc; 2715 size_t w; 2716 int n, ret = 0, ts, tm; 2717 int (*parse)(uint32_t *, const char *, size_t); 2718 size_t (*unparse)(char *, size_t, uint32_t); 2719 2720 if (sc->flag & SCONV_TO_UTF16BE) { 2721 unparse = unicode_to_utf16be; 2722 ts = 2; 2723 } else if (sc->flag & SCONV_TO_UTF16LE) { 2724 unparse = unicode_to_utf16le; 2725 ts = 2; 2726 } else if (sc->flag & SCONV_TO_UTF8) { 2727 unparse = unicode_to_utf8; 2728 ts = 1; 2729 } else { 2730 /* 2731 * This case is going to be converted to another 2732 * character-set through iconv. 2733 */ 2734 if (sc->flag & SCONV_FROM_UTF16BE) { 2735 unparse = unicode_to_utf16be; 2736 ts = 2; 2737 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2738 unparse = unicode_to_utf16le; 2739 ts = 2; 2740 } else { 2741 unparse = unicode_to_utf8; 2742 ts = 1; 2743 } 2744 } 2745 2746 if (sc->flag & SCONV_FROM_UTF16BE) { 2747 parse = utf16be_to_unicode; 2748 tm = 1; 2749 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2750 parse = utf16le_to_unicode; 2751 tm = 1; 2752 } else { 2753 parse = cesu8_to_unicode; 2754 tm = ts; 2755 } 2756 2757 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2758 return (-1); 2759 2760 s = (const char *)_p; 2761 p = as->s + as->length; 2762 endp = as->s + as->buffer_length - ts; 2763 while ((n = parse(&uc, s, len)) != 0) { 2764 if (n < 0) { 2765 /* Use a replaced unicode character. */ 2766 n *= -1; 2767 ret = -1; 2768 } 2769 s += n; 2770 len -= n; 2771 while ((w = unparse(p, endp - p, uc)) == 0) { 2772 /* There is not enough output buffer so 2773 * we have to expand it. */ 2774 as->length = p - as->s; 2775 if (archive_string_ensure(as, 2776 as->buffer_length + len * tm + ts) == NULL) 2777 return (-1); 2778 p = as->s + as->length; 2779 endp = as->s + as->buffer_length - ts; 2780 } 2781 p += w; 2782 } 2783 as->length = p - as->s; 2784 as->s[as->length] = '\0'; 2785 if (ts == 2) 2786 as->s[as->length+1] = '\0'; 2787 return (ret); 2788 } 2789 2790 /* 2791 * Following Constants for Hangul compositions this information comes from 2792 * Unicode Standard Annex #15 http://unicode.org/reports/tr15/ 2793 */ 2794 #define HC_SBASE 0xAC00 2795 #define HC_LBASE 0x1100 2796 #define HC_VBASE 0x1161 2797 #define HC_TBASE 0x11A7 2798 #define HC_LCOUNT 19 2799 #define HC_VCOUNT 21 2800 #define HC_TCOUNT 28 2801 #define HC_NCOUNT (HC_VCOUNT * HC_TCOUNT) 2802 #define HC_SCOUNT (HC_LCOUNT * HC_NCOUNT) 2803 2804 static uint32_t 2805 get_nfc(uint32_t uc, uint32_t uc2) 2806 { 2807 int t, b; 2808 2809 t = 0; 2810 b = sizeof(u_composition_table)/sizeof(u_composition_table[0]) -1; 2811 while (b >= t) { 2812 int m = (t + b) / 2; 2813 if (u_composition_table[m].cp1 < uc) 2814 t = m + 1; 2815 else if (u_composition_table[m].cp1 > uc) 2816 b = m - 1; 2817 else if (u_composition_table[m].cp2 < uc2) 2818 t = m + 1; 2819 else if (u_composition_table[m].cp2 > uc2) 2820 b = m - 1; 2821 else 2822 return (u_composition_table[m].nfc); 2823 } 2824 return (0); 2825 } 2826 2827 #define FDC_MAX 10 /* The maximum number of Following Decomposable 2828 * Characters. */ 2829 2830 /* 2831 * Update first code point. 2832 */ 2833 #define UPDATE_UC(new_uc) do { \ 2834 uc = new_uc; \ 2835 ucptr = NULL; \ 2836 } while (0) 2837 2838 /* 2839 * Replace first code point with second code point. 2840 */ 2841 #define REPLACE_UC_WITH_UC2() do { \ 2842 uc = uc2; \ 2843 ucptr = uc2ptr; \ 2844 n = n2; \ 2845 } while (0) 2846 2847 #define EXPAND_BUFFER() do { \ 2848 as->length = p - as->s; \ 2849 if (archive_string_ensure(as, \ 2850 as->buffer_length + len * tm + ts) == NULL)\ 2851 return (-1); \ 2852 p = as->s + as->length; \ 2853 endp = as->s + as->buffer_length - ts; \ 2854 } while (0) 2855 2856 #define UNPARSE(p, endp, uc) do { \ 2857 while ((w = unparse(p, (endp) - (p), uc)) == 0) {\ 2858 EXPAND_BUFFER(); \ 2859 } \ 2860 p += w; \ 2861 } while (0) 2862 2863 /* 2864 * Write first code point. 2865 * If the code point has not be changed from its original code, 2866 * this just copies it from its original buffer pointer. 2867 * If not, this converts it to UTF-8 byte sequence and copies it. 2868 */ 2869 #define WRITE_UC() do { \ 2870 if (ucptr) { \ 2871 if (p + n > endp) \ 2872 EXPAND_BUFFER(); \ 2873 switch (n) { \ 2874 case 4: \ 2875 *p++ = *ucptr++; \ 2876 /* FALL THROUGH */ \ 2877 case 3: \ 2878 *p++ = *ucptr++; \ 2879 /* FALL THROUGH */ \ 2880 case 2: \ 2881 *p++ = *ucptr++; \ 2882 /* FALL THROUGH */ \ 2883 case 1: \ 2884 *p++ = *ucptr; \ 2885 break; \ 2886 } \ 2887 ucptr = NULL; \ 2888 } else { \ 2889 UNPARSE(p, endp, uc); \ 2890 } \ 2891 } while (0) 2892 2893 /* 2894 * Collect following decomposable code points. 2895 */ 2896 #define COLLECT_CPS(start) do { \ 2897 int _i; \ 2898 for (_i = start; _i < FDC_MAX ; _i++) { \ 2899 nx = parse(&ucx[_i], s, len); \ 2900 if (nx <= 0) \ 2901 break; \ 2902 cx = CCC(ucx[_i]); \ 2903 if (cl >= cx && cl != 228 && cx != 228)\ 2904 break; \ 2905 s += nx; \ 2906 len -= nx; \ 2907 cl = cx; \ 2908 ccx[_i] = cx; \ 2909 } \ 2910 if (_i >= FDC_MAX) { \ 2911 ret = -1; \ 2912 ucx_size = FDC_MAX; \ 2913 } else \ 2914 ucx_size = _i; \ 2915 } while (0) 2916 2917 /* 2918 * Normalize UTF-8/UTF-16BE characters to Form C and copy the result. 2919 * 2920 * TODO: Convert composition exclusions, which are never converted 2921 * from NFC,NFD,NFKC and NFKD, to Form C. 2922 */ 2923 static int 2924 archive_string_normalize_C(struct archive_string *as, const void *_p, 2925 size_t len, struct archive_string_conv *sc) 2926 { 2927 const char *s = (const char *)_p; 2928 char *p, *endp; 2929 uint32_t uc, uc2; 2930 size_t w; 2931 int always_replace, n, n2, ret = 0, spair, ts, tm; 2932 int (*parse)(uint32_t *, const char *, size_t); 2933 size_t (*unparse)(char *, size_t, uint32_t); 2934 2935 always_replace = 1; 2936 ts = 1;/* text size. */ 2937 if (sc->flag & SCONV_TO_UTF16BE) { 2938 unparse = unicode_to_utf16be; 2939 ts = 2; 2940 if (sc->flag & SCONV_FROM_UTF16BE) 2941 always_replace = 0; 2942 } else if (sc->flag & SCONV_TO_UTF16LE) { 2943 unparse = unicode_to_utf16le; 2944 ts = 2; 2945 if (sc->flag & SCONV_FROM_UTF16LE) 2946 always_replace = 0; 2947 } else if (sc->flag & SCONV_TO_UTF8) { 2948 unparse = unicode_to_utf8; 2949 if (sc->flag & SCONV_FROM_UTF8) 2950 always_replace = 0; 2951 } else { 2952 /* 2953 * This case is going to be converted to another 2954 * character-set through iconv. 2955 */ 2956 always_replace = 0; 2957 if (sc->flag & SCONV_FROM_UTF16BE) { 2958 unparse = unicode_to_utf16be; 2959 ts = 2; 2960 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2961 unparse = unicode_to_utf16le; 2962 ts = 2; 2963 } else { 2964 unparse = unicode_to_utf8; 2965 } 2966 } 2967 2968 if (sc->flag & SCONV_FROM_UTF16BE) { 2969 parse = utf16be_to_unicode; 2970 tm = 1; 2971 spair = 4;/* surrogate pair size in UTF-16. */ 2972 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2973 parse = utf16le_to_unicode; 2974 tm = 1; 2975 spair = 4;/* surrogate pair size in UTF-16. */ 2976 } else { 2977 parse = cesu8_to_unicode; 2978 tm = ts; 2979 spair = 6;/* surrogate pair size in UTF-8. */ 2980 } 2981 2982 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2983 return (-1); 2984 2985 p = as->s + as->length; 2986 endp = as->s + as->buffer_length - ts; 2987 while ((n = parse(&uc, s, len)) != 0) { 2988 const char *ucptr, *uc2ptr; 2989 2990 if (n < 0) { 2991 /* Use a replaced unicode character. */ 2992 UNPARSE(p, endp, uc); 2993 s += n*-1; 2994 len -= n*-1; 2995 ret = -1; 2996 continue; 2997 } else if (n == spair || always_replace) 2998 /* uc is converted from a surrogate pair. 2999 * this should be treated as a changed code. */ 3000 ucptr = NULL; 3001 else 3002 ucptr = s; 3003 s += n; 3004 len -= n; 3005 3006 /* Read second code point. */ 3007 while ((n2 = parse(&uc2, s, len)) > 0) { 3008 uint32_t ucx[FDC_MAX]; 3009 int ccx[FDC_MAX]; 3010 int cl, cx, i, nx, ucx_size; 3011 int LIndex,SIndex; 3012 uint32_t nfc; 3013 3014 if (n2 == spair || always_replace) 3015 /* uc2 is converted from a surrogate pair. 3016 * this should be treated as a changed code. */ 3017 uc2ptr = NULL; 3018 else 3019 uc2ptr = s; 3020 s += n2; 3021 len -= n2; 3022 3023 /* 3024 * If current second code point is out of decomposable 3025 * code points, finding compositions is unneeded. 3026 */ 3027 if (!IS_DECOMPOSABLE_BLOCK(uc2)) { 3028 WRITE_UC(); 3029 REPLACE_UC_WITH_UC2(); 3030 continue; 3031 } 3032 3033 /* 3034 * Try to combine current code points. 3035 */ 3036 /* 3037 * We have to combine Hangul characters according to 3038 * http://uniicode.org/reports/tr15/#Hangul 3039 */ 3040 if (0 <= (LIndex = uc - HC_LBASE) && 3041 LIndex < HC_LCOUNT) { 3042 /* 3043 * Hangul Composition. 3044 * 1. Two current code points are L and V. 3045 */ 3046 int VIndex = uc2 - HC_VBASE; 3047 if (0 <= VIndex && VIndex < HC_VCOUNT) { 3048 /* Make syllable of form LV. */ 3049 UPDATE_UC(HC_SBASE + 3050 (LIndex * HC_VCOUNT + VIndex) * 3051 HC_TCOUNT); 3052 } else { 3053 WRITE_UC(); 3054 REPLACE_UC_WITH_UC2(); 3055 } 3056 continue; 3057 } else if (0 <= (SIndex = uc - HC_SBASE) && 3058 SIndex < HC_SCOUNT && (SIndex % HC_TCOUNT) == 0) { 3059 /* 3060 * Hangul Composition. 3061 * 2. Two current code points are LV and T. 3062 */ 3063 int TIndex = uc2 - HC_TBASE; 3064 if (0 < TIndex && TIndex < HC_TCOUNT) { 3065 /* Make syllable of form LVT. */ 3066 UPDATE_UC(uc + TIndex); 3067 } else { 3068 WRITE_UC(); 3069 REPLACE_UC_WITH_UC2(); 3070 } 3071 continue; 3072 } else if ((nfc = get_nfc(uc, uc2)) != 0) { 3073 /* A composition to current code points 3074 * is found. */ 3075 UPDATE_UC(nfc); 3076 continue; 3077 } else if ((cl = CCC(uc2)) == 0) { 3078 /* Clearly 'uc2' the second code point is not 3079 * a decomposable code. */ 3080 WRITE_UC(); 3081 REPLACE_UC_WITH_UC2(); 3082 continue; 3083 } 3084 3085 /* 3086 * Collect following decomposable code points. 3087 */ 3088 cx = 0; 3089 ucx[0] = uc2; 3090 ccx[0] = cl; 3091 COLLECT_CPS(1); 3092 3093 /* 3094 * Find a composed code in the collected code points. 3095 */ 3096 i = 1; 3097 while (i < ucx_size) { 3098 int j; 3099 3100 if ((nfc = get_nfc(uc, ucx[i])) == 0) { 3101 i++; 3102 continue; 3103 } 3104 3105 /* 3106 * nfc is composed of uc and ucx[i]. 3107 */ 3108 UPDATE_UC(nfc); 3109 3110 /* 3111 * Remove ucx[i] by shifting 3112 * following code points. 3113 */ 3114 for (j = i; j+1 < ucx_size; j++) { 3115 ucx[j] = ucx[j+1]; 3116 ccx[j] = ccx[j+1]; 3117 } 3118 ucx_size --; 3119 3120 /* 3121 * Collect following code points blocked 3122 * by ucx[i] the removed code point. 3123 */ 3124 if (ucx_size > 0 && i == ucx_size && 3125 nx > 0 && cx == cl) { 3126 cl = ccx[ucx_size-1]; 3127 COLLECT_CPS(ucx_size); 3128 } 3129 /* 3130 * Restart finding a composed code with 3131 * the updated uc from the top of the 3132 * collected code points. 3133 */ 3134 i = 0; 3135 } 3136 3137 /* 3138 * Apparently the current code points are not 3139 * decomposed characters or already composed. 3140 */ 3141 WRITE_UC(); 3142 for (i = 0; i < ucx_size; i++) 3143 UNPARSE(p, endp, ucx[i]); 3144 3145 /* 3146 * Flush out remaining canonical combining characters. 3147 */ 3148 if (nx > 0 && cx == cl && len > 0) { 3149 while ((nx = parse(&ucx[0], s, len)) 3150 > 0) { 3151 cx = CCC(ucx[0]); 3152 if (cl > cx) 3153 break; 3154 s += nx; 3155 len -= nx; 3156 cl = cx; 3157 UNPARSE(p, endp, ucx[0]); 3158 } 3159 } 3160 break; 3161 } 3162 if (n2 < 0) { 3163 WRITE_UC(); 3164 /* Use a replaced unicode character. */ 3165 UNPARSE(p, endp, uc2); 3166 s += n2*-1; 3167 len -= n2*-1; 3168 ret = -1; 3169 continue; 3170 } else if (n2 == 0) { 3171 WRITE_UC(); 3172 break; 3173 } 3174 } 3175 as->length = p - as->s; 3176 as->s[as->length] = '\0'; 3177 if (ts == 2) 3178 as->s[as->length+1] = '\0'; 3179 return (ret); 3180 } 3181 3182 static int 3183 get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc) 3184 { 3185 int t, b; 3186 3187 /* 3188 * These are not converted to NFD on Mac OS. 3189 */ 3190 if ((uc >= 0x2000 && uc <= 0x2FFF) || 3191 (uc >= 0xF900 && uc <= 0xFAFF) || 3192 (uc >= 0x2F800 && uc <= 0x2FAFF)) 3193 return (0); 3194 /* 3195 * Those code points are not converted to NFD on Mac OS. 3196 * I do not know the reason because it is undocumented. 3197 * NFC NFD 3198 * 1109A ==> 11099 110BA 3199 * 1109C ==> 1109B 110BA 3200 * 110AB ==> 110A5 110BA 3201 */ 3202 if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB) 3203 return (0); 3204 3205 t = 0; 3206 b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1; 3207 while (b >= t) { 3208 int m = (t + b) / 2; 3209 if (u_decomposition_table[m].nfc < uc) 3210 t = m + 1; 3211 else if (u_decomposition_table[m].nfc > uc) 3212 b = m - 1; 3213 else { 3214 *cp1 = u_decomposition_table[m].cp1; 3215 *cp2 = u_decomposition_table[m].cp2; 3216 return (1); 3217 } 3218 } 3219 return (0); 3220 } 3221 3222 #define REPLACE_UC_WITH(cp) do { \ 3223 uc = cp; \ 3224 ucptr = NULL; \ 3225 } while (0) 3226 3227 /* 3228 * Normalize UTF-8 characters to Form D and copy the result. 3229 */ 3230 static int 3231 archive_string_normalize_D(struct archive_string *as, const void *_p, 3232 size_t len, struct archive_string_conv *sc) 3233 { 3234 const char *s = (const char *)_p; 3235 char *p, *endp; 3236 uint32_t uc, uc2; 3237 size_t w; 3238 int always_replace, n, n2, ret = 0, spair, ts, tm; 3239 int (*parse)(uint32_t *, const char *, size_t); 3240 size_t (*unparse)(char *, size_t, uint32_t); 3241 3242 always_replace = 1; 3243 ts = 1;/* text size. */ 3244 if (sc->flag & SCONV_TO_UTF16BE) { 3245 unparse = unicode_to_utf16be; 3246 ts = 2; 3247 if (sc->flag & SCONV_FROM_UTF16BE) 3248 always_replace = 0; 3249 } else if (sc->flag & SCONV_TO_UTF16LE) { 3250 unparse = unicode_to_utf16le; 3251 ts = 2; 3252 if (sc->flag & SCONV_FROM_UTF16LE) 3253 always_replace = 0; 3254 } else if (sc->flag & SCONV_TO_UTF8) { 3255 unparse = unicode_to_utf8; 3256 if (sc->flag & SCONV_FROM_UTF8) 3257 always_replace = 0; 3258 } else { 3259 /* 3260 * This case is going to be converted to another 3261 * character-set through iconv. 3262 */ 3263 always_replace = 0; 3264 if (sc->flag & SCONV_FROM_UTF16BE) { 3265 unparse = unicode_to_utf16be; 3266 ts = 2; 3267 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3268 unparse = unicode_to_utf16le; 3269 ts = 2; 3270 } else { 3271 unparse = unicode_to_utf8; 3272 } 3273 } 3274 3275 if (sc->flag & SCONV_FROM_UTF16BE) { 3276 parse = utf16be_to_unicode; 3277 tm = 1; 3278 spair = 4;/* surrogate pair size in UTF-16. */ 3279 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3280 parse = utf16le_to_unicode; 3281 tm = 1; 3282 spair = 4;/* surrogate pair size in UTF-16. */ 3283 } else { 3284 parse = cesu8_to_unicode; 3285 tm = ts; 3286 spair = 6;/* surrogate pair size in UTF-8. */ 3287 } 3288 3289 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 3290 return (-1); 3291 3292 p = as->s + as->length; 3293 endp = as->s + as->buffer_length - ts; 3294 while ((n = parse(&uc, s, len)) != 0) { 3295 const char *ucptr; 3296 uint32_t cp1, cp2; 3297 int SIndex; 3298 struct { 3299 uint32_t uc; 3300 int ccc; 3301 } fdc[FDC_MAX]; 3302 int fdi, fdj; 3303 int ccc; 3304 3305 check_first_code: 3306 if (n < 0) { 3307 /* Use a replaced unicode character. */ 3308 UNPARSE(p, endp, uc); 3309 s += n*-1; 3310 len -= n*-1; 3311 ret = -1; 3312 continue; 3313 } else if (n == spair || always_replace) 3314 /* uc is converted from a surrogate pair. 3315 * this should be treated as a changed code. */ 3316 ucptr = NULL; 3317 else 3318 ucptr = s; 3319 s += n; 3320 len -= n; 3321 3322 /* Hangul Decomposition. */ 3323 if ((SIndex = uc - HC_SBASE) >= 0 && SIndex < HC_SCOUNT) { 3324 int L = HC_LBASE + SIndex / HC_NCOUNT; 3325 int V = HC_VBASE + (SIndex % HC_NCOUNT) / HC_TCOUNT; 3326 int T = HC_TBASE + SIndex % HC_TCOUNT; 3327 3328 REPLACE_UC_WITH(L); 3329 WRITE_UC(); 3330 REPLACE_UC_WITH(V); 3331 WRITE_UC(); 3332 if (T != HC_TBASE) { 3333 REPLACE_UC_WITH(T); 3334 WRITE_UC(); 3335 } 3336 continue; 3337 } 3338 if (IS_DECOMPOSABLE_BLOCK(uc) && CCC(uc) != 0) { 3339 WRITE_UC(); 3340 continue; 3341 } 3342 3343 fdi = 0; 3344 while (get_nfd(&cp1, &cp2, uc) && fdi < FDC_MAX) { 3345 int k; 3346 3347 for (k = fdi; k > 0; k--) 3348 fdc[k] = fdc[k-1]; 3349 fdc[0].ccc = CCC(cp2); 3350 fdc[0].uc = cp2; 3351 fdi++; 3352 REPLACE_UC_WITH(cp1); 3353 } 3354 3355 /* Read following code points. */ 3356 while ((n2 = parse(&uc2, s, len)) > 0 && 3357 (ccc = CCC(uc2)) != 0 && fdi < FDC_MAX) { 3358 int j, k; 3359 3360 s += n2; 3361 len -= n2; 3362 for (j = 0; j < fdi; j++) { 3363 if (fdc[j].ccc > ccc) 3364 break; 3365 } 3366 if (j < fdi) { 3367 for (k = fdi; k > j; k--) 3368 fdc[k] = fdc[k-1]; 3369 fdc[j].ccc = ccc; 3370 fdc[j].uc = uc2; 3371 } else { 3372 fdc[fdi].ccc = ccc; 3373 fdc[fdi].uc = uc2; 3374 } 3375 fdi++; 3376 } 3377 3378 WRITE_UC(); 3379 for (fdj = 0; fdj < fdi; fdj++) { 3380 REPLACE_UC_WITH(fdc[fdj].uc); 3381 WRITE_UC(); 3382 } 3383 3384 if (n2 == 0) 3385 break; 3386 REPLACE_UC_WITH(uc2); 3387 n = n2; 3388 goto check_first_code; 3389 } 3390 as->length = p - as->s; 3391 as->s[as->length] = '\0'; 3392 if (ts == 2) 3393 as->s[as->length+1] = '\0'; 3394 return (ret); 3395 } 3396 3397 /* 3398 * libarchive 2.x made incorrect UTF-8 strings in the wrong assumption 3399 * that WCS is Unicode. It is true for several platforms but some are false. 3400 * And then people who did not use UTF-8 locale on the non Unicode WCS 3401 * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those 3402 * now cannot get right filename from libarchive 3.x and later since we 3403 * fixed the wrong assumption and it is incompatible to older its versions. 3404 * So we provide special option, "compat-2x.x", for resolving it. 3405 * That option enable the string conversion of libarchive 2.x. 3406 * 3407 * Translates the wrong UTF-8 string made by libarchive 2.x into current 3408 * locale character set and appends to the archive_string. 3409 * Note: returns -1 if conversion fails. 3410 */ 3411 static int 3412 strncat_from_utf8_libarchive2(struct archive_string *as, 3413 const void *_p, size_t len, struct archive_string_conv *sc) 3414 { 3415 const char *s; 3416 int n; 3417 char *p; 3418 char *end; 3419 uint32_t unicode; 3420 #if HAVE_WCRTOMB 3421 mbstate_t shift_state; 3422 3423 memset(&shift_state, 0, sizeof(shift_state)); 3424 #else 3425 /* Clear the shift state before starting. */ 3426 wctomb(NULL, L'\0'); 3427 #endif 3428 (void)sc; /* UNUSED */ 3429 /* 3430 * Allocate buffer for MBS. 3431 * We need this allocation here since it is possible that 3432 * as->s is still NULL. 3433 */ 3434 if (archive_string_ensure(as, as->length + len + 1) == NULL) 3435 return (-1); 3436 3437 s = (const char *)_p; 3438 p = as->s + as->length; 3439 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3440 while ((n = _utf8_to_unicode(&unicode, s, len)) != 0) { 3441 wchar_t wc; 3442 3443 if (p >= end) { 3444 as->length = p - as->s; 3445 /* Re-allocate buffer for MBS. */ 3446 if (archive_string_ensure(as, 3447 as->length + max(len * 2, 3448 (size_t)MB_CUR_MAX) + 1) == NULL) 3449 return (-1); 3450 p = as->s + as->length; 3451 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3452 } 3453 3454 /* 3455 * As libarchive 2.x, translates the UTF-8 characters into 3456 * wide-characters in the assumption that WCS is Unicode. 3457 */ 3458 if (n < 0) { 3459 n *= -1; 3460 wc = L'?'; 3461 } else 3462 wc = (wchar_t)unicode; 3463 3464 s += n; 3465 len -= n; 3466 /* 3467 * Translates the wide-character into the current locale MBS. 3468 */ 3469 #if HAVE_WCRTOMB 3470 n = (int)wcrtomb(p, wc, &shift_state); 3471 #else 3472 n = (int)wctomb(p, wc); 3473 #endif 3474 if (n == -1) 3475 return (-1); 3476 p += n; 3477 } 3478 as->length = p - as->s; 3479 as->s[as->length] = '\0'; 3480 return (0); 3481 } 3482 3483 3484 /* 3485 * Conversion functions between current locale dependent MBS and UTF-16BE. 3486 * strncat_from_utf16be() : UTF-16BE --> MBS 3487 * strncat_to_utf16be() : MBS --> UTF16BE 3488 */ 3489 3490 #if defined(_WIN32) && !defined(__CYGWIN__) 3491 3492 /* 3493 * Convert a UTF-16BE/LE string to current locale and copy the result. 3494 * Return -1 if conversion fails. 3495 */ 3496 static int 3497 win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes, 3498 struct archive_string_conv *sc, int be) 3499 { 3500 struct archive_string tmp; 3501 const char *u16; 3502 int ll; 3503 BOOL defchar; 3504 char *mbs; 3505 size_t mbs_size, b; 3506 int ret = 0; 3507 3508 bytes &= ~1; 3509 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3510 return (-1); 3511 3512 mbs = as->s + as->length; 3513 mbs_size = as->buffer_length - as->length -1; 3514 3515 if (sc->to_cp == CP_C_LOCALE) { 3516 /* 3517 * "C" locale special process. 3518 */ 3519 u16 = _p; 3520 ll = 0; 3521 for (b = 0; b < bytes; b += 2) { 3522 uint16_t val; 3523 if (be) 3524 val = archive_be16dec(u16+b); 3525 else 3526 val = archive_le16dec(u16+b); 3527 if (val > 255) { 3528 *mbs++ = '?'; 3529 ret = -1; 3530 } else 3531 *mbs++ = (char)(val&0xff); 3532 ll++; 3533 } 3534 as->length += ll; 3535 as->s[as->length] = '\0'; 3536 return (ret); 3537 } 3538 3539 archive_string_init(&tmp); 3540 if (be) { 3541 if (is_big_endian()) { 3542 u16 = _p; 3543 } else { 3544 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3545 return (-1); 3546 memcpy(tmp.s, _p, bytes); 3547 for (b = 0; b < bytes; b += 2) { 3548 uint16_t val = archive_be16dec(tmp.s+b); 3549 archive_le16enc(tmp.s+b, val); 3550 } 3551 u16 = tmp.s; 3552 } 3553 } else { 3554 if (!is_big_endian()) { 3555 u16 = _p; 3556 } else { 3557 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3558 return (-1); 3559 memcpy(tmp.s, _p, bytes); 3560 for (b = 0; b < bytes; b += 2) { 3561 uint16_t val = archive_le16dec(tmp.s+b); 3562 archive_be16enc(tmp.s+b, val); 3563 } 3564 u16 = tmp.s; 3565 } 3566 } 3567 3568 do { 3569 defchar = 0; 3570 ll = WideCharToMultiByte(sc->to_cp, 0, 3571 (LPCWSTR)u16, (int)bytes>>1, mbs, (int)mbs_size, 3572 NULL, &defchar); 3573 /* Exit loop if we succeeded */ 3574 if (ll != 0 || 3575 GetLastError() != ERROR_INSUFFICIENT_BUFFER) { 3576 break; 3577 } 3578 /* Else expand buffer and loop to try again. */ 3579 ll = WideCharToMultiByte(sc->to_cp, 0, 3580 (LPCWSTR)u16, (int)bytes, NULL, 0, NULL, NULL); 3581 if (archive_string_ensure(as, ll +1) == NULL) 3582 return (-1); 3583 mbs = as->s + as->length; 3584 mbs_size = as->buffer_length - as->length -1; 3585 } while (1); 3586 archive_string_free(&tmp); 3587 as->length += ll; 3588 as->s[as->length] = '\0'; 3589 if (ll == 0 || defchar) 3590 ret = -1; 3591 return (ret); 3592 } 3593 3594 static int 3595 win_strncat_from_utf16be(struct archive_string *as, const void *_p, 3596 size_t bytes, struct archive_string_conv *sc) 3597 { 3598 return (win_strncat_from_utf16(as, _p, bytes, sc, 1)); 3599 } 3600 3601 static int 3602 win_strncat_from_utf16le(struct archive_string *as, const void *_p, 3603 size_t bytes, struct archive_string_conv *sc) 3604 { 3605 return (win_strncat_from_utf16(as, _p, bytes, sc, 0)); 3606 } 3607 3608 static int 3609 is_big_endian(void) 3610 { 3611 uint16_t d = 1; 3612 3613 return (archive_be16dec(&d) == 1); 3614 } 3615 3616 /* 3617 * Convert a current locale string to UTF-16BE/LE and copy the result. 3618 * Return -1 if conversion fails. 3619 */ 3620 static int 3621 win_strncat_to_utf16(struct archive_string *as16, const void *_p, 3622 size_t length, struct archive_string_conv *sc, int bigendian) 3623 { 3624 const char *s = (const char *)_p; 3625 char *u16; 3626 size_t count, avail; 3627 3628 if (archive_string_ensure(as16, 3629 as16->length + (length + 1) * 2) == NULL) 3630 return (-1); 3631 3632 u16 = as16->s + as16->length; 3633 avail = as16->buffer_length - 2; 3634 if (sc->from_cp == CP_C_LOCALE) { 3635 /* 3636 * "C" locale special process. 3637 */ 3638 count = 0; 3639 while (count < length && *s) { 3640 if (bigendian) 3641 archive_be16enc(u16, *s); 3642 else 3643 archive_le16enc(u16, *s); 3644 u16 += 2; 3645 s++; 3646 count++; 3647 } 3648 as16->length += count << 1; 3649 as16->s[as16->length] = 0; 3650 as16->s[as16->length+1] = 0; 3651 return (0); 3652 } 3653 do { 3654 count = MultiByteToWideChar(sc->from_cp, 3655 MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1); 3656 /* Exit loop if we succeeded */ 3657 if (count != 0 || 3658 GetLastError() != ERROR_INSUFFICIENT_BUFFER) { 3659 break; 3660 } 3661 /* Expand buffer and try again */ 3662 count = MultiByteToWideChar(sc->from_cp, 3663 MB_PRECOMPOSED, s, (int)length, NULL, 0); 3664 if (archive_string_ensure(as16, (count +1) * 2) 3665 == NULL) 3666 return (-1); 3667 u16 = as16->s + as16->length; 3668 avail = as16->buffer_length - 2; 3669 } while (1); 3670 as16->length += count * 2; 3671 as16->s[as16->length] = 0; 3672 as16->s[as16->length+1] = 0; 3673 if (count == 0) 3674 return (-1); 3675 3676 if (is_big_endian()) { 3677 if (!bigendian) { 3678 while (count > 0) { 3679 uint16_t v = archive_be16dec(u16); 3680 archive_le16enc(u16, v); 3681 u16 += 2; 3682 count--; 3683 } 3684 } 3685 } else { 3686 if (bigendian) { 3687 while (count > 0) { 3688 uint16_t v = archive_le16dec(u16); 3689 archive_be16enc(u16, v); 3690 u16 += 2; 3691 count--; 3692 } 3693 } 3694 } 3695 return (0); 3696 } 3697 3698 static int 3699 win_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3700 size_t length, struct archive_string_conv *sc) 3701 { 3702 return (win_strncat_to_utf16(as16, _p, length, sc, 1)); 3703 } 3704 3705 static int 3706 win_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3707 size_t length, struct archive_string_conv *sc) 3708 { 3709 return (win_strncat_to_utf16(as16, _p, length, sc, 0)); 3710 } 3711 3712 #endif /* _WIN32 && !__CYGWIN__ */ 3713 3714 /* 3715 * Do the best effort for conversions. 3716 * We cannot handle UTF-16BE character-set without such iconv, 3717 * but there is a chance if a string consists just ASCII code or 3718 * a current locale is UTF-8. 3719 */ 3720 3721 /* 3722 * Convert a UTF-16BE string to current locale and copy the result. 3723 * Return -1 if conversion fails. 3724 */ 3725 static int 3726 best_effort_strncat_from_utf16(struct archive_string *as, const void *_p, 3727 size_t bytes, struct archive_string_conv *sc, int be) 3728 { 3729 const char *utf16 = (const char *)_p; 3730 char *mbs; 3731 uint32_t uc; 3732 int n, ret; 3733 3734 (void)sc; /* UNUSED */ 3735 /* 3736 * Other case, we should do the best effort. 3737 * If all character are ASCII(<0x7f), we can convert it. 3738 * if not , we set a alternative character and return -1. 3739 */ 3740 ret = 0; 3741 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3742 return (-1); 3743 mbs = as->s + as->length; 3744 3745 while ((n = utf16_to_unicode(&uc, utf16, bytes, be)) != 0) { 3746 if (n < 0) { 3747 n *= -1; 3748 ret = -1; 3749 } 3750 bytes -= n; 3751 utf16 += n; 3752 3753 if (uc > 127) { 3754 /* We cannot handle it. */ 3755 *mbs++ = '?'; 3756 ret = -1; 3757 } else 3758 *mbs++ = (char)uc; 3759 } 3760 as->length = mbs - as->s; 3761 as->s[as->length] = '\0'; 3762 return (ret); 3763 } 3764 3765 static int 3766 best_effort_strncat_from_utf16be(struct archive_string *as, const void *_p, 3767 size_t bytes, struct archive_string_conv *sc) 3768 { 3769 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 1)); 3770 } 3771 3772 static int 3773 best_effort_strncat_from_utf16le(struct archive_string *as, const void *_p, 3774 size_t bytes, struct archive_string_conv *sc) 3775 { 3776 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 0)); 3777 } 3778 3779 /* 3780 * Convert a current locale string to UTF-16BE/LE and copy the result. 3781 * Return -1 if conversion fails. 3782 */ 3783 static int 3784 best_effort_strncat_to_utf16(struct archive_string *as16, const void *_p, 3785 size_t length, struct archive_string_conv *sc, int bigendian) 3786 { 3787 const char *s = (const char *)_p; 3788 char *utf16; 3789 size_t remaining; 3790 int ret; 3791 3792 (void)sc; /* UNUSED */ 3793 /* 3794 * Other case, we should do the best effort. 3795 * If all character are ASCII(<0x7f), we can convert it. 3796 * if not , we set a alternative character and return -1. 3797 */ 3798 ret = 0; 3799 remaining = length; 3800 3801 if (archive_string_ensure(as16, 3802 as16->length + (length + 1) * 2) == NULL) 3803 return (-1); 3804 3805 utf16 = as16->s + as16->length; 3806 while (remaining--) { 3807 unsigned c = *s++; 3808 if (c > 127) { 3809 /* We cannot handle it. */ 3810 c = UNICODE_R_CHAR; 3811 ret = -1; 3812 } 3813 if (bigendian) 3814 archive_be16enc(utf16, c); 3815 else 3816 archive_le16enc(utf16, c); 3817 utf16 += 2; 3818 } 3819 as16->length = utf16 - as16->s; 3820 as16->s[as16->length] = 0; 3821 as16->s[as16->length+1] = 0; 3822 return (ret); 3823 } 3824 3825 static int 3826 best_effort_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3827 size_t length, struct archive_string_conv *sc) 3828 { 3829 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 1)); 3830 } 3831 3832 static int 3833 best_effort_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3834 size_t length, struct archive_string_conv *sc) 3835 { 3836 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 0)); 3837 } 3838 3839 3840 /* 3841 * Multistring operations. 3842 */ 3843 3844 void 3845 archive_mstring_clean(struct archive_mstring *aes) 3846 { 3847 archive_wstring_free(&(aes->aes_wcs)); 3848 archive_string_free(&(aes->aes_mbs)); 3849 archive_string_free(&(aes->aes_utf8)); 3850 archive_string_free(&(aes->aes_mbs_in_locale)); 3851 aes->aes_set = 0; 3852 } 3853 3854 void 3855 archive_mstring_copy(struct archive_mstring *dest, struct archive_mstring *src) 3856 { 3857 dest->aes_set = src->aes_set; 3858 archive_string_copy(&(dest->aes_mbs), &(src->aes_mbs)); 3859 archive_string_copy(&(dest->aes_utf8), &(src->aes_utf8)); 3860 archive_wstring_copy(&(dest->aes_wcs), &(src->aes_wcs)); 3861 } 3862 3863 int 3864 archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes, 3865 const char **p) 3866 { 3867 struct archive_string_conv *sc; 3868 int r; 3869 3870 /* If we already have a UTF8 form, return that immediately. */ 3871 if (aes->aes_set & AES_SET_UTF8) { 3872 *p = aes->aes_utf8.s; 3873 return (0); 3874 } 3875 3876 *p = NULL; 3877 #if defined(_WIN32) && !defined(__CYGWIN__) 3878 /* 3879 * On Windows, first try converting from WCS because (1) there's no 3880 * guarantee that the conversion to MBS will succeed, e.g. when using 3881 * CP_ACP, and (2) that's more efficient than converting to MBS, just to 3882 * convert back to WCS again before finally converting to UTF-8 3883 */ 3884 if ((aes->aes_set & AES_SET_WCS) != 0) { 3885 sc = archive_string_conversion_to_charset(a, "UTF-8", 1); 3886 if (sc == NULL) 3887 return (-1);/* Couldn't allocate memory for sc. */ 3888 archive_string_empty(&(aes->aes_utf8)); 3889 r = archive_string_append_from_wcs_in_codepage(&(aes->aes_utf8), 3890 aes->aes_wcs.s, aes->aes_wcs.length, sc); 3891 if (a == NULL) 3892 free_sconv_object(sc); 3893 if (r == 0) { 3894 aes->aes_set |= AES_SET_UTF8; 3895 *p = aes->aes_utf8.s; 3896 return (0);/* success. */ 3897 } else 3898 return (-1);/* failure. */ 3899 } 3900 #endif 3901 /* Try converting WCS to MBS first if MBS does not exist yet. */ 3902 if ((aes->aes_set & AES_SET_MBS) == 0) { 3903 const char *pm; /* unused */ 3904 archive_mstring_get_mbs(a, aes, &pm); /* ignore errors, we'll handle it later */ 3905 } 3906 if (aes->aes_set & AES_SET_MBS) { 3907 sc = archive_string_conversion_to_charset(a, "UTF-8", 1); 3908 if (sc == NULL) 3909 return (-1);/* Couldn't allocate memory for sc. */ 3910 r = archive_strncpy_l(&(aes->aes_utf8), aes->aes_mbs.s, 3911 aes->aes_mbs.length, sc); 3912 if (a == NULL) 3913 free_sconv_object(sc); 3914 if (r == 0) { 3915 aes->aes_set |= AES_SET_UTF8; 3916 *p = aes->aes_utf8.s; 3917 return (0);/* success. */ 3918 } else 3919 return (-1);/* failure. */ 3920 } 3921 return (0);/* success. */ 3922 } 3923 3924 int 3925 archive_mstring_get_mbs(struct archive *a, struct archive_mstring *aes, 3926 const char **p) 3927 { 3928 struct archive_string_conv *sc; 3929 int r, ret = 0; 3930 3931 /* If we already have an MBS form, return that immediately. */ 3932 if (aes->aes_set & AES_SET_MBS) { 3933 *p = aes->aes_mbs.s; 3934 return (ret); 3935 } 3936 3937 *p = NULL; 3938 /* If there's a WCS form, try converting with the native locale. */ 3939 if (aes->aes_set & AES_SET_WCS) { 3940 archive_string_empty(&(aes->aes_mbs)); 3941 r = archive_string_append_from_wcs(&(aes->aes_mbs), 3942 aes->aes_wcs.s, aes->aes_wcs.length); 3943 *p = aes->aes_mbs.s; 3944 if (r == 0) { 3945 aes->aes_set |= AES_SET_MBS; 3946 return (ret); 3947 } else 3948 ret = -1; 3949 } 3950 3951 /* If there's a UTF-8 form, try converting with the native locale. */ 3952 if (aes->aes_set & AES_SET_UTF8) { 3953 archive_string_empty(&(aes->aes_mbs)); 3954 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 3955 if (sc == NULL) 3956 return (-1);/* Couldn't allocate memory for sc. */ 3957 r = archive_strncpy_l(&(aes->aes_mbs), 3958 aes->aes_utf8.s, aes->aes_utf8.length, sc); 3959 if (a == NULL) 3960 free_sconv_object(sc); 3961 *p = aes->aes_mbs.s; 3962 if (r == 0) { 3963 aes->aes_set |= AES_SET_MBS; 3964 ret = 0;/* success; overwrite previous error. */ 3965 } else 3966 ret = -1;/* failure. */ 3967 } 3968 return (ret); 3969 } 3970 3971 int 3972 archive_mstring_get_wcs(struct archive *a, struct archive_mstring *aes, 3973 const wchar_t **wp) 3974 { 3975 int r, ret = 0; 3976 3977 (void)a;/* UNUSED */ 3978 /* Return WCS form if we already have it. */ 3979 if (aes->aes_set & AES_SET_WCS) { 3980 *wp = aes->aes_wcs.s; 3981 return (ret); 3982 } 3983 3984 *wp = NULL; 3985 #if defined(_WIN32) && !defined(__CYGWIN__) 3986 /* 3987 * On Windows, prefer converting from UTF-8 directly to WCS because: 3988 * (1) there's no guarantee that the string can be represented in MBS (e.g. 3989 * with CP_ACP), and (2) in order to convert from UTF-8 to MBS, we're going 3990 * to need to convert from UTF-8 to WCS anyway and its wasteful to throw 3991 * away that intermediate result 3992 */ 3993 if (aes->aes_set & AES_SET_UTF8) { 3994 struct archive_string_conv *sc; 3995 3996 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 3997 if (sc != NULL) { 3998 archive_wstring_empty((&aes->aes_wcs)); 3999 r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4000 aes->aes_utf8.s, aes->aes_utf8.length, sc); 4001 if (a == NULL) 4002 free_sconv_object(sc); 4003 if (r == 0) { 4004 aes->aes_set |= AES_SET_WCS; 4005 *wp = aes->aes_wcs.s; 4006 return (0); 4007 } 4008 } 4009 } 4010 #endif 4011 /* Try converting UTF8 to MBS first if MBS does not exist yet. */ 4012 if ((aes->aes_set & AES_SET_MBS) == 0) { 4013 const char *p; /* unused */ 4014 archive_mstring_get_mbs(a, aes, &p); /* ignore errors, we'll handle it later */ 4015 } 4016 /* Try converting MBS to WCS using native locale. */ 4017 if (aes->aes_set & AES_SET_MBS) { 4018 archive_wstring_empty(&(aes->aes_wcs)); 4019 r = archive_wstring_append_from_mbs(&(aes->aes_wcs), 4020 aes->aes_mbs.s, aes->aes_mbs.length); 4021 if (r == 0) { 4022 aes->aes_set |= AES_SET_WCS; 4023 *wp = aes->aes_wcs.s; 4024 } else 4025 ret = -1;/* failure. */ 4026 } 4027 return (ret); 4028 } 4029 4030 int 4031 archive_mstring_get_mbs_l(struct archive *a, struct archive_mstring *aes, 4032 const char **p, size_t *length, struct archive_string_conv *sc) 4033 { 4034 int ret = 0; 4035 #if defined(_WIN32) && !defined(__CYGWIN__) 4036 int r; 4037 4038 /* 4039 * Internationalization programming on Windows must use Wide 4040 * characters because Windows platform cannot make locale UTF-8. 4041 */ 4042 if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) { 4043 archive_string_empty(&(aes->aes_mbs_in_locale)); 4044 r = archive_string_append_from_wcs_in_codepage( 4045 &(aes->aes_mbs_in_locale), aes->aes_wcs.s, 4046 aes->aes_wcs.length, sc); 4047 if (r == 0) { 4048 *p = aes->aes_mbs_in_locale.s; 4049 if (length != NULL) 4050 *length = aes->aes_mbs_in_locale.length; 4051 return (0); 4052 } else if (errno == ENOMEM) 4053 return (-1); 4054 else 4055 ret = -1; 4056 } 4057 #endif 4058 4059 /* If there is not an MBS form but there is a WCS or UTF8 form, try converting 4060 * with the native locale to be used for translating it to specified 4061 * character-set. */ 4062 if ((aes->aes_set & AES_SET_MBS) == 0) { 4063 const char *pm; /* unused */ 4064 archive_mstring_get_mbs(a, aes, &pm); /* ignore errors, we'll handle it later */ 4065 } 4066 /* If we already have an MBS form, use it to be translated to 4067 * specified character-set. */ 4068 if (aes->aes_set & AES_SET_MBS) { 4069 if (sc == NULL) { 4070 /* Conversion is unneeded. */ 4071 *p = aes->aes_mbs.s; 4072 if (length != NULL) 4073 *length = aes->aes_mbs.length; 4074 return (0); 4075 } 4076 ret = archive_strncpy_l(&(aes->aes_mbs_in_locale), 4077 aes->aes_mbs.s, aes->aes_mbs.length, sc); 4078 *p = aes->aes_mbs_in_locale.s; 4079 if (length != NULL) 4080 *length = aes->aes_mbs_in_locale.length; 4081 } else { 4082 *p = NULL; 4083 if (length != NULL) 4084 *length = 0; 4085 } 4086 return (ret); 4087 } 4088 4089 int 4090 archive_mstring_copy_mbs(struct archive_mstring *aes, const char *mbs) 4091 { 4092 if (mbs == NULL) { 4093 aes->aes_set = 0; 4094 return (0); 4095 } 4096 return (archive_mstring_copy_mbs_len(aes, mbs, strlen(mbs))); 4097 } 4098 4099 int 4100 archive_mstring_copy_mbs_len(struct archive_mstring *aes, const char *mbs, 4101 size_t len) 4102 { 4103 if (mbs == NULL) { 4104 aes->aes_set = 0; 4105 return (0); 4106 } 4107 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4108 archive_strncpy(&(aes->aes_mbs), mbs, len); 4109 archive_string_empty(&(aes->aes_utf8)); 4110 archive_wstring_empty(&(aes->aes_wcs)); 4111 return (0); 4112 } 4113 4114 int 4115 archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs) 4116 { 4117 return archive_mstring_copy_wcs_len(aes, wcs, 4118 wcs == NULL ? 0 : wcslen(wcs)); 4119 } 4120 4121 int 4122 archive_mstring_copy_utf8(struct archive_mstring *aes, const char *utf8) 4123 { 4124 if (utf8 == NULL) { 4125 aes->aes_set = 0; 4126 return (0); 4127 } 4128 aes->aes_set = AES_SET_UTF8; 4129 archive_string_empty(&(aes->aes_mbs)); 4130 archive_string_empty(&(aes->aes_wcs)); 4131 archive_strncpy(&(aes->aes_utf8), utf8, strlen(utf8)); 4132 return (int)strlen(utf8); 4133 } 4134 4135 int 4136 archive_mstring_copy_wcs_len(struct archive_mstring *aes, const wchar_t *wcs, 4137 size_t len) 4138 { 4139 if (wcs == NULL) { 4140 aes->aes_set = 0; 4141 return (0); 4142 } 4143 aes->aes_set = AES_SET_WCS; /* Only WCS form set. */ 4144 archive_string_empty(&(aes->aes_mbs)); 4145 archive_string_empty(&(aes->aes_utf8)); 4146 archive_wstrncpy(&(aes->aes_wcs), wcs, len); 4147 return (0); 4148 } 4149 4150 int 4151 archive_mstring_copy_mbs_len_l(struct archive_mstring *aes, 4152 const char *mbs, size_t len, struct archive_string_conv *sc) 4153 { 4154 int r; 4155 4156 if (mbs == NULL) { 4157 aes->aes_set = 0; 4158 return (0); 4159 } 4160 archive_string_empty(&(aes->aes_mbs)); 4161 archive_wstring_empty(&(aes->aes_wcs)); 4162 archive_string_empty(&(aes->aes_utf8)); 4163 #if defined(_WIN32) && !defined(__CYGWIN__) 4164 /* 4165 * Internationalization programming on Windows must use Wide 4166 * characters because Windows platform cannot make locale UTF-8. 4167 */ 4168 if (sc == NULL) { 4169 if (archive_string_append(&(aes->aes_mbs), 4170 mbs, mbsnbytes(mbs, len)) == NULL) { 4171 aes->aes_set = 0; 4172 r = -1; 4173 } else { 4174 aes->aes_set = AES_SET_MBS; 4175 r = 0; 4176 } 4177 #if defined(HAVE_ICONV) 4178 } else if (sc != NULL && sc->cd_w != (iconv_t)-1) { 4179 /* 4180 * This case happens only when MultiByteToWideChar() cannot 4181 * handle sc->from_cp, and we have to iconv in order to 4182 * translate character-set to wchar_t,UTF-16. 4183 */ 4184 iconv_t cd = sc->cd; 4185 unsigned from_cp; 4186 int flag; 4187 4188 /* 4189 * Translate multi-bytes from some character-set to UTF-8. 4190 */ 4191 sc->cd = sc->cd_w; 4192 r = archive_strncpy_l(&(aes->aes_utf8), mbs, len, sc); 4193 sc->cd = cd; 4194 if (r != 0) { 4195 aes->aes_set = 0; 4196 return (r); 4197 } 4198 aes->aes_set = AES_SET_UTF8; 4199 4200 /* 4201 * Append the UTF-8 string into wstring. 4202 */ 4203 flag = sc->flag; 4204 sc->flag &= ~(SCONV_NORMALIZATION_C 4205 | SCONV_TO_UTF16| SCONV_FROM_UTF16); 4206 from_cp = sc->from_cp; 4207 sc->from_cp = CP_UTF8; 4208 r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4209 aes->aes_utf8.s, aes->aes_utf8.length, sc); 4210 sc->flag = flag; 4211 sc->from_cp = from_cp; 4212 if (r == 0) 4213 aes->aes_set |= AES_SET_WCS; 4214 #endif 4215 } else { 4216 r = archive_wstring_append_from_mbs_in_codepage( 4217 &(aes->aes_wcs), mbs, len, sc); 4218 if (r == 0) 4219 aes->aes_set = AES_SET_WCS; 4220 else 4221 aes->aes_set = 0; 4222 } 4223 #else 4224 r = archive_strncpy_l(&(aes->aes_mbs), mbs, len, sc); 4225 if (r == 0) 4226 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4227 else 4228 aes->aes_set = 0; 4229 #endif 4230 return (r); 4231 } 4232 4233 /* 4234 * The 'update' form tries to proactively update all forms of 4235 * this string (WCS and MBS) and returns an error if any of 4236 * them fail. This is used by the 'pax' handler, for instance, 4237 * to detect and report character-conversion failures early while 4238 * still allowing clients to get potentially useful values from 4239 * the more tolerant lazy conversions. (get_mbs and get_wcs will 4240 * strive to give the user something useful, so you can get hopefully 4241 * usable values even if some of the character conversions are failing.) 4242 */ 4243 int 4244 archive_mstring_update_utf8(struct archive *a, struct archive_mstring *aes, 4245 const char *utf8) 4246 { 4247 struct archive_string_conv *sc; 4248 int r; 4249 4250 if (utf8 == NULL) { 4251 aes->aes_set = 0; 4252 return (0); /* Succeeded in clearing everything. */ 4253 } 4254 4255 /* Save the UTF8 string. */ 4256 archive_strcpy(&(aes->aes_utf8), utf8); 4257 4258 /* Empty the mbs and wcs strings. */ 4259 archive_string_empty(&(aes->aes_mbs)); 4260 archive_wstring_empty(&(aes->aes_wcs)); 4261 4262 aes->aes_set = AES_SET_UTF8; /* Only UTF8 is set now. */ 4263 4264 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 4265 if (sc == NULL) 4266 return (-1);/* Couldn't allocate memory for sc. */ 4267 4268 #if defined(_WIN32) && !defined(__CYGWIN__) 4269 /* On Windows, there's no good way to convert from UTF8 -> MBS directly, so 4270 * prefer to first convert to WCS as (1) it's wasteful to throw away the 4271 * intermediate result, and (2) WCS will still be set even if we fail to 4272 * convert to MBS (e.g. with ACP that can't represent the characters) */ 4273 r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4274 aes->aes_utf8.s, aes->aes_utf8.length, sc); 4275 4276 if (a == NULL) 4277 free_sconv_object(sc); 4278 if (r != 0) 4279 return (-1); /* This will guarantee we can't convert to MBS */ 4280 aes->aes_set = AES_SET_UTF8 | AES_SET_WCS; /* Both UTF8 and WCS set. */ 4281 4282 /* Try converting WCS to MBS, return false on failure. */ 4283 if (archive_string_append_from_wcs(&(aes->aes_mbs), aes->aes_wcs.s, 4284 aes->aes_wcs.length)) 4285 return (-1); 4286 #else 4287 /* Try converting UTF-8 to MBS, return false on failure. */ 4288 r = archive_strcpy_l(&(aes->aes_mbs), utf8, sc); 4289 4290 if (a == NULL) 4291 free_sconv_object(sc); 4292 if (r != 0) 4293 return (-1); 4294 aes->aes_set = AES_SET_UTF8 | AES_SET_MBS; /* Both UTF8 and MBS set. */ 4295 4296 /* Try converting MBS to WCS, return false on failure. */ 4297 if (archive_wstring_append_from_mbs(&(aes->aes_wcs), aes->aes_mbs.s, 4298 aes->aes_mbs.length)) 4299 return (-1); 4300 #endif 4301 4302 /* All conversions succeeded. */ 4303 aes->aes_set = AES_SET_UTF8 | AES_SET_WCS | AES_SET_MBS; 4304 4305 return (0); 4306 } 4307