1 /*- 2 * Copyright (c) 2003-2011 Tim Kientzle 3 * Copyright (c) 2011-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 29 /* 30 * Basic resizable string support, to simplify manipulating arbitrary-sized 31 * strings while minimizing heap activity. 32 * 33 * In particular, the buffer used by a string object is only grown, it 34 * never shrinks, so you can clear and reuse the same string object 35 * without incurring additional memory allocations. 36 */ 37 38 #ifdef HAVE_ERRNO_H 39 #include <errno.h> 40 #endif 41 #ifdef HAVE_ICONV_H 42 #include <iconv.h> 43 #endif 44 #ifdef HAVE_LANGINFO_H 45 #include <langinfo.h> 46 #endif 47 #ifdef HAVE_LOCALCHARSET_H 48 #include <localcharset.h> 49 #endif 50 #ifdef HAVE_STDLIB_H 51 #include <stdlib.h> 52 #endif 53 #ifdef HAVE_STRING_H 54 #include <string.h> 55 #endif 56 #ifdef HAVE_WCHAR_H 57 #include <wchar.h> 58 #endif 59 #if defined(_WIN32) && !defined(__CYGWIN__) 60 #include <windows.h> 61 #include <locale.h> 62 #endif 63 64 #include "archive_endian.h" 65 #include "archive_private.h" 66 #include "archive_string.h" 67 #include "archive_string_composition.h" 68 69 #if !defined(HAVE_WMEMCPY) && !defined(wmemcpy) 70 #define wmemcpy(a,b,i) (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t)) 71 #endif 72 73 #if !defined(HAVE_WMEMMOVE) && !defined(wmemmove) 74 #define wmemmove(a,b,i) (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t)) 75 #endif 76 77 #undef max 78 #define max(a, b) ((a)>(b)?(a):(b)) 79 80 struct archive_string_conv { 81 struct archive_string_conv *next; 82 char *from_charset; 83 char *to_charset; 84 unsigned from_cp; 85 unsigned to_cp; 86 /* Set 1 if from_charset and to_charset are the same. */ 87 int same; 88 int flag; 89 #define SCONV_TO_CHARSET 1 /* MBS is being converted to specified 90 * charset. */ 91 #define SCONV_FROM_CHARSET (1<<1) /* MBS is being converted from 92 * specified charset. */ 93 #define SCONV_BEST_EFFORT (1<<2) /* Copy at least ASCII code. */ 94 #define SCONV_WIN_CP (1<<3) /* Use Windows API for converting 95 * MBS. */ 96 #define SCONV_UTF8_LIBARCHIVE_2 (1<<4) /* Incorrect UTF-8 made by libarchive 97 * 2.x in the wrong assumption. */ 98 #define SCONV_NORMALIZATION_C (1<<6) /* Need normalization to be Form C. 99 * Before UTF-8 characters are actually 100 * processed. */ 101 #define SCONV_NORMALIZATION_D (1<<7) /* Need normalization to be Form D. 102 * Before UTF-8 characters are actually 103 * processed. 104 * Currently this only for MAC OS X. */ 105 #define SCONV_TO_UTF8 (1<<8) /* "to charset" side is UTF-8. */ 106 #define SCONV_FROM_UTF8 (1<<9) /* "from charset" side is UTF-8. */ 107 #define SCONV_TO_UTF16BE (1<<10) /* "to charset" side is UTF-16BE. */ 108 #define SCONV_FROM_UTF16BE (1<<11) /* "from charset" side is UTF-16BE. */ 109 #define SCONV_TO_UTF16LE (1<<12) /* "to charset" side is UTF-16LE. */ 110 #define SCONV_FROM_UTF16LE (1<<13) /* "from charset" side is UTF-16LE. */ 111 #define SCONV_TO_UTF16 (SCONV_TO_UTF16BE | SCONV_TO_UTF16LE) 112 #define SCONV_FROM_UTF16 (SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE) 113 114 #if HAVE_ICONV 115 iconv_t cd; 116 iconv_t cd_w;/* Use at archive_mstring on 117 * Windows. */ 118 #endif 119 /* A temporary buffer for normalization. */ 120 struct archive_string utftmp; 121 int (*converter[2])(struct archive_string *, const void *, size_t, 122 struct archive_string_conv *); 123 int nconverter; 124 }; 125 126 #define CP_C_LOCALE 0 /* "C" locale only for this file. */ 127 #define CP_UTF16LE 1200 128 #define CP_UTF16BE 1201 129 130 #define IS_HIGH_SURROGATE_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDBFF) 131 #define IS_LOW_SURROGATE_LA(uc) ((uc) >= 0xDC00 && (uc) <= 0xDFFF) 132 #define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF) 133 #define UNICODE_MAX 0x10FFFF 134 #define UNICODE_R_CHAR 0xFFFD /* Replacement character. */ 135 /* Set U+FFFD(Replacement character) in UTF-8. */ 136 static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd}; 137 138 static struct archive_string_conv *find_sconv_object(struct archive *, 139 const char *, const char *); 140 static void add_sconv_object(struct archive *, struct archive_string_conv *); 141 static struct archive_string_conv *create_sconv_object(const char *, 142 const char *, unsigned, int); 143 static void free_sconv_object(struct archive_string_conv *); 144 static struct archive_string_conv *get_sconv_object(struct archive *, 145 const char *, const char *, int); 146 static unsigned make_codepage_from_charset(const char *); 147 static unsigned get_current_codepage(void); 148 static unsigned get_current_oemcp(void); 149 static size_t mbsnbytes(const void *, size_t); 150 static size_t utf16nbytes(const void *, size_t); 151 #if defined(_WIN32) && !defined(__CYGWIN__) 152 static int archive_wstring_append_from_mbs_in_codepage( 153 struct archive_wstring *, const char *, size_t, 154 struct archive_string_conv *); 155 static int archive_string_append_from_wcs_in_codepage(struct archive_string *, 156 const wchar_t *, size_t, struct archive_string_conv *); 157 static int is_big_endian(void); 158 static int strncat_in_codepage(struct archive_string *, const void *, 159 size_t, struct archive_string_conv *); 160 static int win_strncat_from_utf16be(struct archive_string *, const void *, 161 size_t, struct archive_string_conv *); 162 static int win_strncat_from_utf16le(struct archive_string *, const void *, 163 size_t, struct archive_string_conv *); 164 static int win_strncat_to_utf16be(struct archive_string *, const void *, 165 size_t, struct archive_string_conv *); 166 static int win_strncat_to_utf16le(struct archive_string *, const void *, 167 size_t, struct archive_string_conv *); 168 #endif 169 static int best_effort_strncat_from_utf16be(struct archive_string *, 170 const void *, size_t, struct archive_string_conv *); 171 static int best_effort_strncat_from_utf16le(struct archive_string *, 172 const void *, size_t, struct archive_string_conv *); 173 static int best_effort_strncat_to_utf16be(struct archive_string *, 174 const void *, size_t, struct archive_string_conv *); 175 static int best_effort_strncat_to_utf16le(struct archive_string *, 176 const void *, size_t, struct archive_string_conv *); 177 #if defined(HAVE_ICONV) 178 static int iconv_strncat_in_locale(struct archive_string *, const void *, 179 size_t, struct archive_string_conv *); 180 #endif 181 static int best_effort_strncat_in_locale(struct archive_string *, 182 const void *, size_t, struct archive_string_conv *); 183 static int _utf8_to_unicode(uint32_t *, const char *, size_t); 184 static int utf8_to_unicode(uint32_t *, const char *, size_t); 185 static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t); 186 static int cesu8_to_unicode(uint32_t *, const char *, size_t); 187 static size_t unicode_to_utf8(char *, size_t, uint32_t); 188 static int utf16_to_unicode(uint32_t *, const char *, size_t, int); 189 static size_t unicode_to_utf16be(char *, size_t, uint32_t); 190 static size_t unicode_to_utf16le(char *, size_t, uint32_t); 191 static int strncat_from_utf8_libarchive2(struct archive_string *, 192 const void *, size_t, struct archive_string_conv *); 193 static int strncat_from_utf8_to_utf8(struct archive_string *, const void *, 194 size_t, struct archive_string_conv *); 195 static int archive_string_normalize_C(struct archive_string *, const void *, 196 size_t, struct archive_string_conv *); 197 static int archive_string_normalize_D(struct archive_string *, const void *, 198 size_t, struct archive_string_conv *); 199 static int archive_string_append_unicode(struct archive_string *, 200 const void *, size_t, struct archive_string_conv *); 201 202 static struct archive_string * 203 archive_string_append(struct archive_string *as, const char *p, size_t s) 204 { 205 if (archive_string_ensure(as, as->length + s + 1) == NULL) 206 return (NULL); 207 if (s) 208 memmove(as->s + as->length, p, s); 209 as->length += s; 210 as->s[as->length] = 0; 211 return (as); 212 } 213 214 static struct archive_wstring * 215 archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) 216 { 217 if (archive_wstring_ensure(as, as->length + s + 1) == NULL) 218 return (NULL); 219 if (s) 220 wmemmove(as->s + as->length, p, s); 221 as->length += s; 222 as->s[as->length] = 0; 223 return (as); 224 } 225 226 struct archive_string * 227 archive_array_append(struct archive_string *as, const char *p, size_t s) 228 { 229 return archive_string_append(as, p, s); 230 } 231 232 void 233 archive_string_concat(struct archive_string *dest, struct archive_string *src) 234 { 235 if (archive_string_append(dest, src->s, src->length) == NULL) 236 __archive_errx(1, "Out of memory"); 237 } 238 239 void 240 archive_wstring_concat(struct archive_wstring *dest, 241 struct archive_wstring *src) 242 { 243 if (archive_wstring_append(dest, src->s, src->length) == NULL) 244 __archive_errx(1, "Out of memory"); 245 } 246 247 void 248 archive_string_free(struct archive_string *as) 249 { 250 as->length = 0; 251 as->buffer_length = 0; 252 free(as->s); 253 as->s = NULL; 254 } 255 256 void 257 archive_wstring_free(struct archive_wstring *as) 258 { 259 as->length = 0; 260 as->buffer_length = 0; 261 free(as->s); 262 as->s = NULL; 263 } 264 265 struct archive_wstring * 266 archive_wstring_ensure(struct archive_wstring *as, size_t s) 267 { 268 return (struct archive_wstring *) 269 archive_string_ensure((struct archive_string *)as, 270 s * sizeof(wchar_t)); 271 } 272 273 /* Returns NULL on any allocation failure. */ 274 struct archive_string * 275 archive_string_ensure(struct archive_string *as, size_t s) 276 { 277 char *p; 278 size_t new_length; 279 280 /* If buffer is already big enough, don't reallocate. */ 281 if (as->s && (s <= as->buffer_length)) 282 return (as); 283 284 /* 285 * Growing the buffer at least exponentially ensures that 286 * append operations are always linear in the number of 287 * characters appended. Using a smaller growth rate for 288 * larger buffers reduces memory waste somewhat at the cost of 289 * a larger constant factor. 290 */ 291 if (as->buffer_length < 32) 292 /* Start with a minimum 32-character buffer. */ 293 new_length = 32; 294 else if (as->buffer_length < 8192) 295 /* Buffers under 8k are doubled for speed. */ 296 new_length = as->buffer_length + as->buffer_length; 297 else { 298 /* Buffers 8k and over grow by at least 25% each time. */ 299 new_length = as->buffer_length + as->buffer_length / 4; 300 /* Be safe: If size wraps, fail. */ 301 if (new_length < as->buffer_length) { 302 /* On failure, wipe the string and return NULL. */ 303 archive_string_free(as); 304 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 305 return (NULL); 306 } 307 } 308 /* 309 * The computation above is a lower limit to how much we'll 310 * grow the buffer. In any case, we have to grow it enough to 311 * hold the request. 312 */ 313 if (new_length < s) 314 new_length = s; 315 /* Now we can reallocate the buffer. */ 316 p = (char *)realloc(as->s, new_length); 317 if (p == NULL) { 318 /* On failure, wipe the string and return NULL. */ 319 archive_string_free(as); 320 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 321 return (NULL); 322 } 323 324 as->s = p; 325 as->buffer_length = new_length; 326 return (as); 327 } 328 329 /* 330 * TODO: See if there's a way to avoid scanning 331 * the source string twice. Then test to see 332 * if it actually helps (remember that we're almost 333 * always called with pretty short arguments, so 334 * such an optimization might not help). 335 */ 336 struct archive_string * 337 archive_strncat(struct archive_string *as, const void *_p, size_t n) 338 { 339 size_t s; 340 const char *p, *pp; 341 342 p = (const char *)_p; 343 344 /* Like strlen(p), except won't examine positions beyond p[n]. */ 345 s = 0; 346 pp = p; 347 while (s < n && *pp) { 348 pp++; 349 s++; 350 } 351 if ((as = archive_string_append(as, p, s)) == NULL) 352 __archive_errx(1, "Out of memory"); 353 return (as); 354 } 355 356 struct archive_wstring * 357 archive_wstrncat(struct archive_wstring *as, const wchar_t *p, size_t n) 358 { 359 size_t s; 360 const wchar_t *pp; 361 362 /* Like strlen(p), except won't examine positions beyond p[n]. */ 363 s = 0; 364 pp = p; 365 while (s < n && *pp) { 366 pp++; 367 s++; 368 } 369 if ((as = archive_wstring_append(as, p, s)) == NULL) 370 __archive_errx(1, "Out of memory"); 371 return (as); 372 } 373 374 struct archive_string * 375 archive_strcat(struct archive_string *as, const void *p) 376 { 377 /* strcat is just strncat without an effective limit. 378 * Assert that we'll never get called with a source 379 * string over 16MB. 380 * TODO: Review all uses of strcat in the source 381 * and try to replace them with strncat(). 382 */ 383 return archive_strncat(as, p, 0x1000000); 384 } 385 386 struct archive_wstring * 387 archive_wstrcat(struct archive_wstring *as, const wchar_t *p) 388 { 389 /* Ditto. */ 390 return archive_wstrncat(as, p, 0x1000000); 391 } 392 393 struct archive_string * 394 archive_strappend_char(struct archive_string *as, char c) 395 { 396 if ((as = archive_string_append(as, &c, 1)) == NULL) 397 __archive_errx(1, "Out of memory"); 398 return (as); 399 } 400 401 struct archive_wstring * 402 archive_wstrappend_wchar(struct archive_wstring *as, wchar_t c) 403 { 404 if ((as = archive_wstring_append(as, &c, 1)) == NULL) 405 __archive_errx(1, "Out of memory"); 406 return (as); 407 } 408 409 /* 410 * Get the "current character set" name to use with iconv. 411 * On FreeBSD, the empty character set name "" chooses 412 * the correct character encoding for the current locale, 413 * so this isn't necessary. 414 * But iconv on Mac OS 10.6 doesn't seem to handle this correctly; 415 * on that system, we have to explicitly call nl_langinfo() 416 * to get the right name. Not sure about other platforms. 417 * 418 * NOTE: GNU libiconv does not recognize the character-set name 419 * which some platform nl_langinfo(CODESET) returns, so we should 420 * use locale_charset() instead of nl_langinfo(CODESET) for GNU libiconv. 421 */ 422 static const char * 423 default_iconv_charset(const char *charset) { 424 if (charset != NULL && charset[0] != '\0') 425 return charset; 426 #if HAVE_LOCALE_CHARSET && !defined(__APPLE__) 427 /* locale_charset() is broken on Mac OS */ 428 return locale_charset(); 429 #elif HAVE_NL_LANGINFO 430 return nl_langinfo(CODESET); 431 #else 432 return ""; 433 #endif 434 } 435 436 #if defined(_WIN32) && !defined(__CYGWIN__) 437 438 /* 439 * Convert MBS to WCS. 440 * Note: returns -1 if conversion fails. 441 */ 442 int 443 archive_wstring_append_from_mbs(struct archive_wstring *dest, 444 const char *p, size_t len) 445 { 446 return archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL); 447 } 448 449 static int 450 archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest, 451 const char *s, size_t length, struct archive_string_conv *sc) 452 { 453 int count, ret = 0; 454 UINT from_cp; 455 456 if (sc != NULL) 457 from_cp = sc->from_cp; 458 else 459 from_cp = get_current_codepage(); 460 461 if (from_cp == CP_C_LOCALE) { 462 /* 463 * "C" locale special processing. 464 */ 465 wchar_t *ws; 466 const unsigned char *mp; 467 468 if (NULL == archive_wstring_ensure(dest, 469 dest->length + length + 1)) 470 return (-1); 471 472 ws = dest->s + dest->length; 473 mp = (const unsigned char *)s; 474 count = 0; 475 while (count < (int)length && *mp) { 476 *ws++ = (wchar_t)*mp++; 477 count++; 478 } 479 } else if (sc != NULL && 480 (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) { 481 /* 482 * Normalize UTF-8 and UTF-16BE and convert it directly 483 * to UTF-16 as wchar_t. 484 */ 485 struct archive_string u16; 486 int saved_flag = sc->flag;/* save current flag. */ 487 488 if (is_big_endian()) 489 sc->flag |= SCONV_TO_UTF16BE; 490 else 491 sc->flag |= SCONV_TO_UTF16LE; 492 493 if (sc->flag & SCONV_FROM_UTF16) { 494 /* 495 * UTF-16BE/LE NFD ===> UTF-16 NFC 496 * UTF-16BE/LE NFC ===> UTF-16 NFD 497 */ 498 count = (int)utf16nbytes(s, length); 499 } else { 500 /* 501 * UTF-8 NFD ===> UTF-16 NFC 502 * UTF-8 NFC ===> UTF-16 NFD 503 */ 504 count = (int)mbsnbytes(s, length); 505 } 506 u16.s = (char *)dest->s; 507 u16.length = dest->length << 1;; 508 u16.buffer_length = dest->buffer_length; 509 if (sc->flag & SCONV_NORMALIZATION_C) 510 ret = archive_string_normalize_C(&u16, s, count, sc); 511 else 512 ret = archive_string_normalize_D(&u16, s, count, sc); 513 dest->s = (wchar_t *)u16.s; 514 dest->length = u16.length >> 1; 515 dest->buffer_length = u16.buffer_length; 516 sc->flag = saved_flag;/* restore the saved flag. */ 517 return (ret); 518 } else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) { 519 count = (int)utf16nbytes(s, length); 520 count >>= 1; /* to be WCS length */ 521 /* Allocate memory for WCS. */ 522 if (NULL == archive_wstring_ensure(dest, 523 dest->length + count + 1)) 524 return (-1); 525 wmemcpy(dest->s + dest->length, (const wchar_t *)s, count); 526 if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) { 527 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 528 int b; 529 for (b = 0; b < count; b++) { 530 uint16_t val = archive_le16dec(u16+b); 531 archive_be16enc(u16+b, val); 532 } 533 } else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) { 534 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 535 int b; 536 for (b = 0; b < count; b++) { 537 uint16_t val = archive_be16dec(u16+b); 538 archive_le16enc(u16+b, val); 539 } 540 } 541 } else { 542 DWORD mbflag; 543 size_t buffsize; 544 545 if (sc == NULL) 546 mbflag = 0; 547 else if (sc->flag & SCONV_FROM_CHARSET) { 548 /* Do not trust the length which comes from 549 * an archive file. */ 550 length = mbsnbytes(s, length); 551 mbflag = 0; 552 } else 553 mbflag = MB_PRECOMPOSED; 554 555 mbflag |= MB_ERR_INVALID_CHARS; 556 557 buffsize = dest->length + length + 1; 558 do { 559 /* Allocate memory for WCS. */ 560 if (NULL == archive_wstring_ensure(dest, buffsize)) 561 return (-1); 562 /* Convert MBS to WCS. */ 563 count = MultiByteToWideChar(from_cp, 564 mbflag, s, (int)length, dest->s + dest->length, 565 (int)(dest->buffer_length >> 1) -1); 566 if (count == 0 && 567 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 568 /* Expand the WCS buffer. */ 569 buffsize = dest->buffer_length << 1; 570 continue; 571 } 572 if (count == 0 && length != 0) 573 ret = -1; 574 break; 575 } while (1); 576 } 577 dest->length += count; 578 dest->s[dest->length] = L'\0'; 579 return (ret); 580 } 581 582 #else 583 584 /* 585 * Convert MBS to WCS. 586 * Note: returns -1 if conversion fails. 587 */ 588 int 589 archive_wstring_append_from_mbs(struct archive_wstring *dest, 590 const char *p, size_t len) 591 { 592 size_t r; 593 int ret_val = 0; 594 /* 595 * No single byte will be more than one wide character, 596 * so this length estimate will always be big enough. 597 */ 598 // size_t wcs_length = len; 599 size_t mbs_length = len; 600 const char *mbs = p; 601 wchar_t *wcs; 602 #if HAVE_MBRTOWC 603 mbstate_t shift_state; 604 605 memset(&shift_state, 0, sizeof(shift_state)); 606 #endif 607 /* 608 * As we decided to have wcs_length == mbs_length == len 609 * we can use len here instead of wcs_length 610 */ 611 if (NULL == archive_wstring_ensure(dest, dest->length + len + 1)) 612 return (-1); 613 wcs = dest->s + dest->length; 614 /* 615 * We cannot use mbsrtowcs/mbstowcs here because those may convert 616 * extra MBS when strlen(p) > len and one wide character consists of 617 * multi bytes. 618 */ 619 while (*mbs && mbs_length > 0) { 620 /* 621 * The buffer we allocated is always big enough. 622 * Keep this code path in a comment if we decide to choose 623 * smaller wcs_length in the future 624 */ 625 /* 626 if (wcs_length == 0) { 627 dest->length = wcs - dest->s; 628 dest->s[dest->length] = L'\0'; 629 wcs_length = mbs_length; 630 if (NULL == archive_wstring_ensure(dest, 631 dest->length + wcs_length + 1)) 632 return (-1); 633 wcs = dest->s + dest->length; 634 } 635 */ 636 #if HAVE_MBRTOWC 637 r = mbrtowc(wcs, mbs, mbs_length, &shift_state); 638 #else 639 r = mbtowc(wcs, mbs, mbs_length); 640 #endif 641 if (r == (size_t)-1 || r == (size_t)-2) { 642 ret_val = -1; 643 break; 644 } 645 if (r == 0 || r > mbs_length) 646 break; 647 wcs++; 648 // wcs_length--; 649 mbs += r; 650 mbs_length -= r; 651 } 652 dest->length = wcs - dest->s; 653 dest->s[dest->length] = L'\0'; 654 return (ret_val); 655 } 656 657 #endif 658 659 #if defined(_WIN32) && !defined(__CYGWIN__) 660 661 /* 662 * WCS ==> MBS. 663 * Note: returns -1 if conversion fails. 664 * 665 * Win32 builds use WideCharToMultiByte from the Windows API. 666 * (Maybe Cygwin should too? WideCharToMultiByte will know a 667 * lot more about local character encodings than the wcrtomb() 668 * wrapper is going to know.) 669 */ 670 int 671 archive_string_append_from_wcs(struct archive_string *as, 672 const wchar_t *w, size_t len) 673 { 674 return archive_string_append_from_wcs_in_codepage(as, w, len, NULL); 675 } 676 677 static int 678 archive_string_append_from_wcs_in_codepage(struct archive_string *as, 679 const wchar_t *ws, size_t len, struct archive_string_conv *sc) 680 { 681 BOOL defchar_used, *dp; 682 int count, ret = 0; 683 UINT to_cp; 684 int wslen = (int)len; 685 686 if (sc != NULL) 687 to_cp = sc->to_cp; 688 else 689 to_cp = get_current_codepage(); 690 691 if (to_cp == CP_C_LOCALE) { 692 /* 693 * "C" locale special processing. 694 */ 695 const wchar_t *wp = ws; 696 char *p; 697 698 if (NULL == archive_string_ensure(as, 699 as->length + wslen +1)) 700 return (-1); 701 p = as->s + as->length; 702 count = 0; 703 defchar_used = 0; 704 while (count < wslen && *wp) { 705 if (*wp > 255) { 706 *p++ = '?'; 707 wp++; 708 defchar_used = 1; 709 } else 710 *p++ = (char)*wp++; 711 count++; 712 } 713 } else if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) { 714 uint16_t *u16; 715 716 if (NULL == 717 archive_string_ensure(as, as->length + len * 2 + 2)) 718 return (-1); 719 u16 = (uint16_t *)(as->s + as->length); 720 count = 0; 721 defchar_used = 0; 722 if (sc->flag & SCONV_TO_UTF16BE) { 723 while (count < (int)len && *ws) { 724 archive_be16enc(u16+count, *ws); 725 ws++; 726 count++; 727 } 728 } else { 729 while (count < (int)len && *ws) { 730 archive_le16enc(u16+count, *ws); 731 ws++; 732 count++; 733 } 734 } 735 count <<= 1; /* to be byte size */ 736 } else { 737 /* Make sure the MBS buffer has plenty to set. */ 738 if (NULL == 739 archive_string_ensure(as, as->length + len * 2 + 1)) 740 return (-1); 741 do { 742 defchar_used = 0; 743 if (to_cp == CP_UTF8 || sc == NULL) 744 dp = NULL; 745 else 746 dp = &defchar_used; 747 count = WideCharToMultiByte(to_cp, 0, ws, wslen, 748 as->s + as->length, 749 (int)as->buffer_length - (int)as->length - 1, NULL, dp); 750 if (count == 0 && 751 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 752 /* Expand the MBS buffer and retry. */ 753 if (NULL == archive_string_ensure(as, 754 as->buffer_length + len)) 755 return (-1); 756 continue; 757 } 758 if (count == 0) 759 ret = -1; 760 break; 761 } while (1); 762 } 763 as->length += count; 764 as->s[as->length] = '\0'; 765 return (defchar_used?-1:ret); 766 } 767 768 #elif defined(HAVE_WCTOMB) || defined(HAVE_WCRTOMB) 769 770 /* 771 * Translates a wide character string into current locale character set 772 * and appends to the archive_string. Note: returns -1 if conversion 773 * fails. 774 */ 775 int 776 archive_string_append_from_wcs(struct archive_string *as, 777 const wchar_t *w, size_t len) 778 { 779 /* We cannot use the standard wcstombs() here because it 780 * cannot tell us how big the output buffer should be. So 781 * I've built a loop around wcrtomb() or wctomb() that 782 * converts a character at a time and resizes the string as 783 * needed. We prefer wcrtomb() when it's available because 784 * it's thread-safe. */ 785 int n, ret_val = 0; 786 char *p; 787 char *end; 788 #if HAVE_WCRTOMB 789 mbstate_t shift_state; 790 791 memset(&shift_state, 0, sizeof(shift_state)); 792 #else 793 /* Clear the shift state before starting. */ 794 wctomb(NULL, L'\0'); 795 #endif 796 /* 797 * Allocate buffer for MBS. 798 * We need this allocation here since it is possible that 799 * as->s is still NULL. 800 */ 801 if (archive_string_ensure(as, as->length + len + 1) == NULL) 802 return (-1); 803 804 p = as->s + as->length; 805 end = as->s + as->buffer_length - MB_CUR_MAX -1; 806 while (*w != L'\0' && len > 0) { 807 if (p >= end) { 808 as->length = p - as->s; 809 as->s[as->length] = '\0'; 810 /* Re-allocate buffer for MBS. */ 811 if (archive_string_ensure(as, 812 as->length + max(len * 2, 813 (size_t)MB_CUR_MAX) + 1) == NULL) 814 return (-1); 815 p = as->s + as->length; 816 end = as->s + as->buffer_length - MB_CUR_MAX -1; 817 } 818 #if HAVE_WCRTOMB 819 n = wcrtomb(p, *w++, &shift_state); 820 #else 821 n = wctomb(p, *w++); 822 #endif 823 if (n == -1) { 824 if (errno == EILSEQ) { 825 /* Skip an illegal wide char. */ 826 *p++ = '?'; 827 ret_val = -1; 828 } else { 829 ret_val = -1; 830 break; 831 } 832 } else 833 p += n; 834 len--; 835 } 836 as->length = p - as->s; 837 as->s[as->length] = '\0'; 838 return (ret_val); 839 } 840 841 #else /* HAVE_WCTOMB || HAVE_WCRTOMB */ 842 843 /* 844 * TODO: Test if __STDC_ISO_10646__ is defined. 845 * Non-Windows uses ISO C wcrtomb() or wctomb() to perform the conversion 846 * one character at a time. If a non-Windows platform doesn't have 847 * either of these, fall back to the built-in UTF8 conversion. 848 */ 849 int 850 archive_string_append_from_wcs(struct archive_string *as, 851 const wchar_t *w, size_t len) 852 { 853 (void)as;/* UNUSED */ 854 (void)w;/* UNUSED */ 855 (void)len;/* UNUSED */ 856 errno = ENOSYS; 857 return (-1); 858 } 859 860 #endif /* HAVE_WCTOMB || HAVE_WCRTOMB */ 861 862 /* 863 * Find a string conversion object by a pair of 'from' charset name 864 * and 'to' charset name from an archive object. 865 * Return NULL if not found. 866 */ 867 static struct archive_string_conv * 868 find_sconv_object(struct archive *a, const char *fc, const char *tc) 869 { 870 struct archive_string_conv *sc; 871 872 if (a == NULL) 873 return (NULL); 874 875 for (sc = a->sconv; sc != NULL; sc = sc->next) { 876 if (strcmp(sc->from_charset, fc) == 0 && 877 strcmp(sc->to_charset, tc) == 0) 878 break; 879 } 880 return (sc); 881 } 882 883 /* 884 * Register a string object to an archive object. 885 */ 886 static void 887 add_sconv_object(struct archive *a, struct archive_string_conv *sc) 888 { 889 struct archive_string_conv **psc; 890 891 /* Add a new sconv to sconv list. */ 892 psc = &(a->sconv); 893 while (*psc != NULL) 894 psc = &((*psc)->next); 895 *psc = sc; 896 } 897 898 static void 899 add_converter(struct archive_string_conv *sc, int (*converter) 900 (struct archive_string *, const void *, size_t, 901 struct archive_string_conv *)) 902 { 903 if (sc == NULL || sc->nconverter >= 2) 904 __archive_errx(1, "Programming error"); 905 sc->converter[sc->nconverter++] = converter; 906 } 907 908 static void 909 setup_converter(struct archive_string_conv *sc) 910 { 911 912 /* Reset. */ 913 sc->nconverter = 0; 914 915 /* 916 * Perform special sequence for the incorrect UTF-8 filenames 917 * made by libarchive2.x. 918 */ 919 if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) { 920 add_converter(sc, strncat_from_utf8_libarchive2); 921 return; 922 } 923 924 /* 925 * Convert a string to UTF-16BE/LE. 926 */ 927 if (sc->flag & SCONV_TO_UTF16) { 928 /* 929 * If the current locale is UTF-8, we can translate 930 * a UTF-8 string into a UTF-16BE string. 931 */ 932 if (sc->flag & SCONV_FROM_UTF8) { 933 add_converter(sc, archive_string_append_unicode); 934 return; 935 } 936 937 #if defined(_WIN32) && !defined(__CYGWIN__) 938 if (sc->flag & SCONV_WIN_CP) { 939 if (sc->flag & SCONV_TO_UTF16BE) 940 add_converter(sc, win_strncat_to_utf16be); 941 else 942 add_converter(sc, win_strncat_to_utf16le); 943 return; 944 } 945 #endif 946 947 #if defined(HAVE_ICONV) 948 if (sc->cd != (iconv_t)-1) { 949 add_converter(sc, iconv_strncat_in_locale); 950 return; 951 } 952 #endif 953 954 if (sc->flag & SCONV_BEST_EFFORT) { 955 if (sc->flag & SCONV_TO_UTF16BE) 956 add_converter(sc, 957 best_effort_strncat_to_utf16be); 958 else 959 add_converter(sc, 960 best_effort_strncat_to_utf16le); 961 } else 962 /* Make sure we have no converter. */ 963 sc->nconverter = 0; 964 return; 965 } 966 967 /* 968 * Convert a string from UTF-16BE/LE. 969 */ 970 if (sc->flag & SCONV_FROM_UTF16) { 971 /* 972 * At least we should normalize a UTF-16BE string. 973 */ 974 if (sc->flag & SCONV_NORMALIZATION_D) 975 add_converter(sc,archive_string_normalize_D); 976 else if (sc->flag & SCONV_NORMALIZATION_C) 977 add_converter(sc, archive_string_normalize_C); 978 979 if (sc->flag & SCONV_TO_UTF8) { 980 /* 981 * If the current locale is UTF-8, we can translate 982 * a UTF-16BE/LE string into a UTF-8 string directly. 983 */ 984 if (!(sc->flag & 985 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 986 add_converter(sc, 987 archive_string_append_unicode); 988 return; 989 } 990 991 #if defined(_WIN32) && !defined(__CYGWIN__) 992 if (sc->flag & SCONV_WIN_CP) { 993 if (sc->flag & SCONV_FROM_UTF16BE) 994 add_converter(sc, win_strncat_from_utf16be); 995 else 996 add_converter(sc, win_strncat_from_utf16le); 997 return; 998 } 999 #endif 1000 1001 #if defined(HAVE_ICONV) 1002 if (sc->cd != (iconv_t)-1) { 1003 add_converter(sc, iconv_strncat_in_locale); 1004 return; 1005 } 1006 #endif 1007 1008 if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 1009 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 1010 add_converter(sc, best_effort_strncat_from_utf16be); 1011 else if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 1012 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 1013 add_converter(sc, best_effort_strncat_from_utf16le); 1014 else 1015 /* Make sure we have no converter. */ 1016 sc->nconverter = 0; 1017 return; 1018 } 1019 1020 if (sc->flag & SCONV_FROM_UTF8) { 1021 /* 1022 * At least we should normalize a UTF-8 string. 1023 */ 1024 if (sc->flag & SCONV_NORMALIZATION_D) 1025 add_converter(sc,archive_string_normalize_D); 1026 else if (sc->flag & SCONV_NORMALIZATION_C) 1027 add_converter(sc, archive_string_normalize_C); 1028 1029 /* 1030 * Copy UTF-8 string with a check of CESU-8. 1031 * Apparently, iconv does not check surrogate pairs in UTF-8 1032 * when both from-charset and to-charset are UTF-8, and then 1033 * we use our UTF-8 copy code. 1034 */ 1035 if (sc->flag & SCONV_TO_UTF8) { 1036 /* 1037 * If the current locale is UTF-8, we can translate 1038 * a UTF-16BE string into a UTF-8 string directly. 1039 */ 1040 if (!(sc->flag & 1041 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 1042 add_converter(sc, strncat_from_utf8_to_utf8); 1043 return; 1044 } 1045 } 1046 1047 #if defined(_WIN32) && !defined(__CYGWIN__) 1048 /* 1049 * On Windows we can use Windows API for a string conversion. 1050 */ 1051 if (sc->flag & SCONV_WIN_CP) { 1052 add_converter(sc, strncat_in_codepage); 1053 return; 1054 } 1055 #endif 1056 1057 #if HAVE_ICONV 1058 if (sc->cd != (iconv_t)-1) { 1059 add_converter(sc, iconv_strncat_in_locale); 1060 /* 1061 * iconv generally does not support UTF-8-MAC and so 1062 * we have to the output of iconv from NFC to NFD if 1063 * need. 1064 */ 1065 if ((sc->flag & SCONV_FROM_CHARSET) && 1066 (sc->flag & SCONV_TO_UTF8)) { 1067 if (sc->flag & SCONV_NORMALIZATION_D) 1068 add_converter(sc, archive_string_normalize_D); 1069 } 1070 return; 1071 } 1072 #endif 1073 1074 /* 1075 * Try conversion in the best effort or no conversion. 1076 */ 1077 if ((sc->flag & SCONV_BEST_EFFORT) || sc->same) 1078 add_converter(sc, best_effort_strncat_in_locale); 1079 else 1080 /* Make sure we have no converter. */ 1081 sc->nconverter = 0; 1082 } 1083 1084 /* 1085 * Return canonicalized charset-name but this supports just UTF-8, UTF-16BE 1086 * and CP932 which are referenced in create_sconv_object(). 1087 */ 1088 static const char * 1089 canonical_charset_name(const char *charset) 1090 { 1091 char cs[16]; 1092 char *p; 1093 const char *s; 1094 1095 if (charset == NULL || charset[0] == '\0' 1096 || strlen(charset) > 15) 1097 return (charset); 1098 1099 /* Copy name to uppercase. */ 1100 p = cs; 1101 s = charset; 1102 while (*s) { 1103 char c = *s++; 1104 if (c >= 'a' && c <= 'z') 1105 c -= 'a' - 'A'; 1106 *p++ = c; 1107 } 1108 *p++ = '\0'; 1109 1110 if (strcmp(cs, "UTF-8") == 0 || 1111 strcmp(cs, "UTF8") == 0) 1112 return ("UTF-8"); 1113 if (strcmp(cs, "UTF-16BE") == 0 || 1114 strcmp(cs, "UTF16BE") == 0) 1115 return ("UTF-16BE"); 1116 if (strcmp(cs, "UTF-16LE") == 0 || 1117 strcmp(cs, "UTF16LE") == 0) 1118 return ("UTF-16LE"); 1119 if (strcmp(cs, "CP932") == 0) 1120 return ("CP932"); 1121 return (charset); 1122 } 1123 1124 /* 1125 * Create a string conversion object. 1126 */ 1127 static struct archive_string_conv * 1128 create_sconv_object(const char *fc, const char *tc, 1129 unsigned current_codepage, int flag) 1130 { 1131 struct archive_string_conv *sc; 1132 1133 sc = calloc(1, sizeof(*sc)); 1134 if (sc == NULL) 1135 return (NULL); 1136 sc->next = NULL; 1137 sc->from_charset = strdup(fc); 1138 if (sc->from_charset == NULL) { 1139 free(sc); 1140 return (NULL); 1141 } 1142 sc->to_charset = strdup(tc); 1143 if (sc->to_charset == NULL) { 1144 free(sc->from_charset); 1145 free(sc); 1146 return (NULL); 1147 } 1148 archive_string_init(&sc->utftmp); 1149 1150 if (flag & SCONV_TO_CHARSET) { 1151 /* 1152 * Convert characters from the current locale charset to 1153 * a specified charset. 1154 */ 1155 sc->from_cp = current_codepage; 1156 sc->to_cp = make_codepage_from_charset(tc); 1157 #if defined(_WIN32) && !defined(__CYGWIN__) 1158 if (IsValidCodePage(sc->to_cp)) 1159 flag |= SCONV_WIN_CP; 1160 #endif 1161 } else if (flag & SCONV_FROM_CHARSET) { 1162 /* 1163 * Convert characters from a specified charset to 1164 * the current locale charset. 1165 */ 1166 sc->to_cp = current_codepage; 1167 sc->from_cp = make_codepage_from_charset(fc); 1168 #if defined(_WIN32) && !defined(__CYGWIN__) 1169 if (IsValidCodePage(sc->from_cp)) 1170 flag |= SCONV_WIN_CP; 1171 #endif 1172 } 1173 1174 /* 1175 * Check if "from charset" and "to charset" are the same. 1176 */ 1177 if (strcmp(fc, tc) == 0 || 1178 (sc->from_cp != (unsigned)-1 && sc->from_cp == sc->to_cp)) 1179 sc->same = 1; 1180 else 1181 sc->same = 0; 1182 1183 /* 1184 * Mark if "from charset" or "to charset" are UTF-8 or UTF-16BE/LE. 1185 */ 1186 if (strcmp(tc, "UTF-8") == 0) 1187 flag |= SCONV_TO_UTF8; 1188 else if (strcmp(tc, "UTF-16BE") == 0) 1189 flag |= SCONV_TO_UTF16BE; 1190 else if (strcmp(tc, "UTF-16LE") == 0) 1191 flag |= SCONV_TO_UTF16LE; 1192 if (strcmp(fc, "UTF-8") == 0) 1193 flag |= SCONV_FROM_UTF8; 1194 else if (strcmp(fc, "UTF-16BE") == 0) 1195 flag |= SCONV_FROM_UTF16BE; 1196 else if (strcmp(fc, "UTF-16LE") == 0) 1197 flag |= SCONV_FROM_UTF16LE; 1198 #if defined(_WIN32) && !defined(__CYGWIN__) 1199 if (sc->to_cp == CP_UTF8) 1200 flag |= SCONV_TO_UTF8; 1201 else if (sc->to_cp == CP_UTF16BE) 1202 flag |= SCONV_TO_UTF16BE | SCONV_WIN_CP; 1203 else if (sc->to_cp == CP_UTF16LE) 1204 flag |= SCONV_TO_UTF16LE | SCONV_WIN_CP; 1205 if (sc->from_cp == CP_UTF8) 1206 flag |= SCONV_FROM_UTF8; 1207 else if (sc->from_cp == CP_UTF16BE) 1208 flag |= SCONV_FROM_UTF16BE | SCONV_WIN_CP; 1209 else if (sc->from_cp == CP_UTF16LE) 1210 flag |= SCONV_FROM_UTF16LE | SCONV_WIN_CP; 1211 #endif 1212 1213 /* 1214 * Set a flag for Unicode NFD. Usually iconv cannot correctly 1215 * handle it. So we have to translate NFD characters to NFC ones 1216 * ourselves before iconv handles. Another reason is to prevent 1217 * that the same sight of two filenames, one is NFC and other 1218 * is NFD, would be in its directory. 1219 * On Mac OS X, although its filesystem layer automatically 1220 * convert filenames to NFD, it would be useful for filename 1221 * comparing to find out the same filenames that we normalize 1222 * that to be NFD ourselves. 1223 */ 1224 if ((flag & SCONV_FROM_CHARSET) && 1225 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8))) { 1226 #if defined(__APPLE__) 1227 if (flag & SCONV_TO_UTF8) 1228 flag |= SCONV_NORMALIZATION_D; 1229 else 1230 #endif 1231 flag |= SCONV_NORMALIZATION_C; 1232 } 1233 #if defined(__APPLE__) 1234 /* 1235 * In case writing an archive file, make sure that a filename 1236 * going to be passed to iconv is a Unicode NFC string since 1237 * a filename in HFS Plus filesystem is a Unicode NFD one and 1238 * iconv cannot handle it with "UTF-8" charset. It is simpler 1239 * than a use of "UTF-8-MAC" charset. 1240 */ 1241 if ((flag & SCONV_TO_CHARSET) && 1242 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1243 !(flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1244 flag |= SCONV_NORMALIZATION_C; 1245 /* 1246 * In case reading an archive file. make sure that a filename 1247 * will be passed to users is a Unicode NFD string in order to 1248 * correctly compare the filename with other one which comes 1249 * from HFS Plus filesystem. 1250 */ 1251 if ((flag & SCONV_FROM_CHARSET) && 1252 !(flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1253 (flag & SCONV_TO_UTF8)) 1254 flag |= SCONV_NORMALIZATION_D; 1255 #endif 1256 1257 #if defined(HAVE_ICONV) 1258 sc->cd_w = (iconv_t)-1; 1259 /* 1260 * Create an iconv object. 1261 */ 1262 if (((flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) && 1263 (flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16))) || 1264 (flag & SCONV_WIN_CP)) { 1265 /* This case we won't use iconv. */ 1266 sc->cd = (iconv_t)-1; 1267 } else { 1268 sc->cd = iconv_open(tc, fc); 1269 if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT)) { 1270 /* 1271 * Unfortunately, all of iconv implements do support 1272 * "CP932" character-set, so we should use "SJIS" 1273 * instead if iconv_open failed. 1274 */ 1275 if (strcmp(tc, "CP932") == 0) 1276 sc->cd = iconv_open("SJIS", fc); 1277 else if (strcmp(fc, "CP932") == 0) 1278 sc->cd = iconv_open(tc, "SJIS"); 1279 } 1280 #if defined(_WIN32) && !defined(__CYGWIN__) 1281 /* 1282 * archive_mstring on Windows directly convert multi-bytes 1283 * into archive_wstring in order not to depend on locale 1284 * so that you can do a I18N programming. This will be 1285 * used only in archive_mstring_copy_mbs_len_l so far. 1286 */ 1287 if (flag & SCONV_FROM_CHARSET) { 1288 sc->cd_w = iconv_open("UTF-8", fc); 1289 if (sc->cd_w == (iconv_t)-1 && 1290 (sc->flag & SCONV_BEST_EFFORT)) { 1291 if (strcmp(fc, "CP932") == 0) 1292 sc->cd_w = iconv_open("UTF-8", "SJIS"); 1293 } 1294 } 1295 #endif /* _WIN32 && !__CYGWIN__ */ 1296 } 1297 #endif /* HAVE_ICONV */ 1298 1299 sc->flag = flag; 1300 1301 /* 1302 * Set up converters. 1303 */ 1304 setup_converter(sc); 1305 1306 return (sc); 1307 } 1308 1309 /* 1310 * Free a string conversion object. 1311 */ 1312 static void 1313 free_sconv_object(struct archive_string_conv *sc) 1314 { 1315 free(sc->from_charset); 1316 free(sc->to_charset); 1317 archive_string_free(&sc->utftmp); 1318 #if HAVE_ICONV 1319 if (sc->cd != (iconv_t)-1) 1320 iconv_close(sc->cd); 1321 if (sc->cd_w != (iconv_t)-1) 1322 iconv_close(sc->cd_w); 1323 #endif 1324 free(sc); 1325 } 1326 1327 #if defined(_WIN32) && !defined(__CYGWIN__) 1328 # if defined(WINAPI_FAMILY_PARTITION) && !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) 1329 # define GetOEMCP() CP_OEMCP 1330 # endif 1331 1332 static unsigned 1333 my_atoi(const char *p) 1334 { 1335 unsigned cp; 1336 1337 cp = 0; 1338 while (*p) { 1339 if (*p >= '0' && *p <= '9') 1340 cp = cp * 10 + (*p - '0'); 1341 else 1342 return (-1); 1343 p++; 1344 } 1345 return (cp); 1346 } 1347 1348 /* 1349 * Translate Charset name (as used by iconv) into CodePage (as used by Windows) 1350 * Return -1 if failed. 1351 * 1352 * Note: This translation code may be insufficient. 1353 */ 1354 static struct charset { 1355 const char *name; 1356 unsigned cp; 1357 } charsets[] = { 1358 /* MUST BE SORTED! */ 1359 {"ASCII", 1252}, 1360 {"ASMO-708", 708}, 1361 {"BIG5", 950}, 1362 {"CHINESE", 936}, 1363 {"CP367", 1252}, 1364 {"CP819", 1252}, 1365 {"CP1025", 21025}, 1366 {"DOS-720", 720}, 1367 {"DOS-862", 862}, 1368 {"EUC-CN", 51936}, 1369 {"EUC-JP", 51932}, 1370 {"EUC-KR", 949}, 1371 {"EUCCN", 51936}, 1372 {"EUCJP", 51932}, 1373 {"EUCKR", 949}, 1374 {"GB18030", 54936}, 1375 {"GB2312", 936}, 1376 {"HEBREW", 1255}, 1377 {"HZ-GB-2312", 52936}, 1378 {"IBM273", 20273}, 1379 {"IBM277", 20277}, 1380 {"IBM278", 20278}, 1381 {"IBM280", 20280}, 1382 {"IBM284", 20284}, 1383 {"IBM285", 20285}, 1384 {"IBM290", 20290}, 1385 {"IBM297", 20297}, 1386 {"IBM367", 1252}, 1387 {"IBM420", 20420}, 1388 {"IBM423", 20423}, 1389 {"IBM424", 20424}, 1390 {"IBM819", 1252}, 1391 {"IBM871", 20871}, 1392 {"IBM880", 20880}, 1393 {"IBM905", 20905}, 1394 {"IBM924", 20924}, 1395 {"ISO-8859-1", 28591}, 1396 {"ISO-8859-13", 28603}, 1397 {"ISO-8859-15", 28605}, 1398 {"ISO-8859-2", 28592}, 1399 {"ISO-8859-3", 28593}, 1400 {"ISO-8859-4", 28594}, 1401 {"ISO-8859-5", 28595}, 1402 {"ISO-8859-6", 28596}, 1403 {"ISO-8859-7", 28597}, 1404 {"ISO-8859-8", 28598}, 1405 {"ISO-8859-9", 28599}, 1406 {"ISO8859-1", 28591}, 1407 {"ISO8859-13", 28603}, 1408 {"ISO8859-15", 28605}, 1409 {"ISO8859-2", 28592}, 1410 {"ISO8859-3", 28593}, 1411 {"ISO8859-4", 28594}, 1412 {"ISO8859-5", 28595}, 1413 {"ISO8859-6", 28596}, 1414 {"ISO8859-7", 28597}, 1415 {"ISO8859-8", 28598}, 1416 {"ISO8859-9", 28599}, 1417 {"JOHAB", 1361}, 1418 {"KOI8-R", 20866}, 1419 {"KOI8-U", 21866}, 1420 {"KS_C_5601-1987", 949}, 1421 {"LATIN1", 1252}, 1422 {"LATIN2", 28592}, 1423 {"MACINTOSH", 10000}, 1424 {"SHIFT-JIS", 932}, 1425 {"SHIFT_JIS", 932}, 1426 {"SJIS", 932}, 1427 {"US", 1252}, 1428 {"US-ASCII", 1252}, 1429 {"UTF-16", 1200}, 1430 {"UTF-16BE", 1201}, 1431 {"UTF-16LE", 1200}, 1432 {"UTF-8", CP_UTF8}, 1433 {"X-EUROPA", 29001}, 1434 {"X-MAC-ARABIC", 10004}, 1435 {"X-MAC-CE", 10029}, 1436 {"X-MAC-CHINESEIMP", 10008}, 1437 {"X-MAC-CHINESETRAD", 10002}, 1438 {"X-MAC-CROATIAN", 10082}, 1439 {"X-MAC-CYRILLIC", 10007}, 1440 {"X-MAC-GREEK", 10006}, 1441 {"X-MAC-HEBREW", 10005}, 1442 {"X-MAC-ICELANDIC", 10079}, 1443 {"X-MAC-JAPANESE", 10001}, 1444 {"X-MAC-KOREAN", 10003}, 1445 {"X-MAC-ROMANIAN", 10010}, 1446 {"X-MAC-THAI", 10021}, 1447 {"X-MAC-TURKISH", 10081}, 1448 {"X-MAC-UKRAINIAN", 10017}, 1449 }; 1450 static unsigned 1451 make_codepage_from_charset(const char *charset) 1452 { 1453 char cs[16]; 1454 char *p; 1455 unsigned cp; 1456 int a, b; 1457 1458 if (charset == NULL || strlen(charset) > 15) 1459 return -1; 1460 1461 /* Copy name to uppercase. */ 1462 p = cs; 1463 while (*charset) { 1464 char c = *charset++; 1465 if (c >= 'a' && c <= 'z') 1466 c -= 'a' - 'A'; 1467 *p++ = c; 1468 } 1469 *p++ = '\0'; 1470 cp = -1; 1471 1472 /* Look it up in the table first, so that we can easily 1473 * override CP367, which we map to 1252 instead of 367. */ 1474 a = 0; 1475 b = sizeof(charsets)/sizeof(charsets[0]); 1476 while (b > a) { 1477 int c = (b + a) / 2; 1478 int r = strcmp(charsets[c].name, cs); 1479 if (r < 0) 1480 a = c + 1; 1481 else if (r > 0) 1482 b = c; 1483 else 1484 return charsets[c].cp; 1485 } 1486 1487 /* If it's not in the table, try to parse it. */ 1488 switch (*cs) { 1489 case 'C': 1490 if (cs[1] == 'P' && cs[2] >= '0' && cs[2] <= '9') { 1491 cp = my_atoi(cs + 2); 1492 } else if (strcmp(cs, "CP_ACP") == 0) 1493 cp = get_current_codepage(); 1494 else if (strcmp(cs, "CP_OEMCP") == 0) 1495 cp = get_current_oemcp(); 1496 break; 1497 case 'I': 1498 if (cs[1] == 'B' && cs[2] == 'M' && 1499 cs[3] >= '0' && cs[3] <= '9') { 1500 cp = my_atoi(cs + 3); 1501 } 1502 break; 1503 case 'W': 1504 if (strncmp(cs, "WINDOWS-", 8) == 0) { 1505 cp = my_atoi(cs + 8); 1506 if (cp != 874 && (cp < 1250 || cp > 1258)) 1507 cp = -1;/* This may invalid code. */ 1508 } 1509 break; 1510 } 1511 return (cp); 1512 } 1513 1514 /* 1515 * Return ANSI Code Page of current locale set by setlocale(). 1516 */ 1517 static unsigned 1518 get_current_codepage(void) 1519 { 1520 char *locale, *p; 1521 unsigned cp; 1522 1523 locale = setlocale(LC_CTYPE, NULL); 1524 if (locale == NULL) 1525 return (GetACP()); 1526 if (locale[0] == 'C' && locale[1] == '\0') 1527 return (CP_C_LOCALE); 1528 p = strrchr(locale, '.'); 1529 if (p == NULL) 1530 return (GetACP()); 1531 if ((strcmp(p+1, "utf8") == 0) || (strcmp(p+1, "UTF-8") == 0)) 1532 return CP_UTF8; 1533 cp = my_atoi(p+1); 1534 if ((int)cp <= 0) 1535 return (GetACP()); 1536 return (cp); 1537 } 1538 1539 /* 1540 * Translation table between Locale Name and ACP/OEMCP. 1541 */ 1542 static struct { 1543 unsigned acp; 1544 unsigned ocp; 1545 const char *locale; 1546 } acp_ocp_map[] = { 1547 { 950, 950, "Chinese_Taiwan" }, 1548 { 936, 936, "Chinese_People's Republic of China" }, 1549 { 950, 950, "Chinese_Taiwan" }, 1550 { 1250, 852, "Czech_Czech Republic" }, 1551 { 1252, 850, "Danish_Denmark" }, 1552 { 1252, 850, "Dutch_Netherlands" }, 1553 { 1252, 850, "Dutch_Belgium" }, 1554 { 1252, 437, "English_United States" }, 1555 { 1252, 850, "English_Australia" }, 1556 { 1252, 850, "English_Canada" }, 1557 { 1252, 850, "English_New Zealand" }, 1558 { 1252, 850, "English_United Kingdom" }, 1559 { 1252, 437, "English_United States" }, 1560 { 1252, 850, "Finnish_Finland" }, 1561 { 1252, 850, "French_France" }, 1562 { 1252, 850, "French_Belgium" }, 1563 { 1252, 850, "French_Canada" }, 1564 { 1252, 850, "French_Switzerland" }, 1565 { 1252, 850, "German_Germany" }, 1566 { 1252, 850, "German_Austria" }, 1567 { 1252, 850, "German_Switzerland" }, 1568 { 1253, 737, "Greek_Greece" }, 1569 { 1250, 852, "Hungarian_Hungary" }, 1570 { 1252, 850, "Icelandic_Iceland" }, 1571 { 1252, 850, "Italian_Italy" }, 1572 { 1252, 850, "Italian_Switzerland" }, 1573 { 932, 932, "Japanese_Japan" }, 1574 { 949, 949, "Korean_Korea" }, 1575 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1576 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1577 { 1252, 850, "Norwegian-Nynorsk_Norway" }, 1578 { 1250, 852, "Polish_Poland" }, 1579 { 1252, 850, "Portuguese_Portugal" }, 1580 { 1252, 850, "Portuguese_Brazil" }, 1581 { 1251, 866, "Russian_Russia" }, 1582 { 1250, 852, "Slovak_Slovakia" }, 1583 { 1252, 850, "Spanish_Spain" }, 1584 { 1252, 850, "Spanish_Mexico" }, 1585 { 1252, 850, "Spanish_Spain" }, 1586 { 1252, 850, "Swedish_Sweden" }, 1587 { 1254, 857, "Turkish_Turkey" }, 1588 { 0, 0, NULL} 1589 }; 1590 1591 /* 1592 * Return OEM Code Page of current locale set by setlocale(). 1593 */ 1594 static unsigned 1595 get_current_oemcp(void) 1596 { 1597 int i; 1598 char *locale, *p; 1599 size_t len; 1600 1601 locale = setlocale(LC_CTYPE, NULL); 1602 if (locale == NULL) 1603 return (GetOEMCP()); 1604 if (locale[0] == 'C' && locale[1] == '\0') 1605 return (CP_C_LOCALE); 1606 1607 p = strrchr(locale, '.'); 1608 if (p == NULL) 1609 return (GetOEMCP()); 1610 len = p - locale; 1611 for (i = 0; acp_ocp_map[i].acp; i++) { 1612 if (strncmp(acp_ocp_map[i].locale, locale, len) == 0) 1613 return (acp_ocp_map[i].ocp); 1614 } 1615 return (GetOEMCP()); 1616 } 1617 #else 1618 1619 /* 1620 * POSIX platform does not use CodePage. 1621 */ 1622 1623 static unsigned 1624 get_current_codepage(void) 1625 { 1626 return (-1);/* Unknown */ 1627 } 1628 static unsigned 1629 make_codepage_from_charset(const char *charset) 1630 { 1631 (void)charset; /* UNUSED */ 1632 return (-1);/* Unknown */ 1633 } 1634 static unsigned 1635 get_current_oemcp(void) 1636 { 1637 return (-1);/* Unknown */ 1638 } 1639 1640 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 1641 1642 /* 1643 * Return a string conversion object. 1644 */ 1645 static struct archive_string_conv * 1646 get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag) 1647 { 1648 struct archive_string_conv *sc; 1649 unsigned current_codepage; 1650 1651 /* Check if we have made the sconv object. */ 1652 sc = find_sconv_object(a, fc, tc); 1653 if (sc != NULL) 1654 return (sc); 1655 1656 if (a == NULL) 1657 current_codepage = get_current_codepage(); 1658 else 1659 current_codepage = a->current_codepage; 1660 1661 sc = create_sconv_object(canonical_charset_name(fc), 1662 canonical_charset_name(tc), current_codepage, flag); 1663 if (sc == NULL) { 1664 if (a != NULL) 1665 archive_set_error(a, ENOMEM, 1666 "Could not allocate memory for " 1667 "a string conversion object"); 1668 return (NULL); 1669 } 1670 1671 /* 1672 * If there is no converter for current string conversion object, 1673 * we cannot handle this conversion. 1674 */ 1675 if (sc->nconverter == 0) { 1676 if (a != NULL) { 1677 #if HAVE_ICONV 1678 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1679 "iconv_open failed : Cannot handle ``%s''", 1680 (flag & SCONV_TO_CHARSET)?tc:fc); 1681 #else 1682 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1683 "A character-set conversion not fully supported " 1684 "on this platform"); 1685 #endif 1686 } 1687 /* Failed; free a sconv object. */ 1688 free_sconv_object(sc); 1689 return (NULL); 1690 } 1691 1692 /* 1693 * Success! 1694 */ 1695 if (a != NULL) 1696 add_sconv_object(a, sc); 1697 return (sc); 1698 } 1699 1700 static const char * 1701 get_current_charset(struct archive *a) 1702 { 1703 const char *cur_charset; 1704 1705 if (a == NULL) 1706 cur_charset = default_iconv_charset(""); 1707 else { 1708 cur_charset = default_iconv_charset(a->current_code); 1709 if (a->current_code == NULL) { 1710 a->current_code = strdup(cur_charset); 1711 a->current_codepage = get_current_codepage(); 1712 a->current_oemcp = get_current_oemcp(); 1713 } 1714 } 1715 return (cur_charset); 1716 } 1717 1718 /* 1719 * Make and Return a string conversion object. 1720 * Return NULL if the platform does not support the specified conversion 1721 * and best_effort is 0. 1722 * If best_effort is set, A string conversion object must be returned 1723 * unless memory allocation for the object fails, but the conversion 1724 * might fail when non-ASCII code is found. 1725 */ 1726 struct archive_string_conv * 1727 archive_string_conversion_to_charset(struct archive *a, const char *charset, 1728 int best_effort) 1729 { 1730 int flag = SCONV_TO_CHARSET; 1731 1732 if (best_effort) 1733 flag |= SCONV_BEST_EFFORT; 1734 return (get_sconv_object(a, get_current_charset(a), charset, flag)); 1735 } 1736 1737 struct archive_string_conv * 1738 archive_string_conversion_from_charset(struct archive *a, const char *charset, 1739 int best_effort) 1740 { 1741 int flag = SCONV_FROM_CHARSET; 1742 1743 if (best_effort) 1744 flag |= SCONV_BEST_EFFORT; 1745 return (get_sconv_object(a, charset, get_current_charset(a), flag)); 1746 } 1747 1748 /* 1749 * archive_string_default_conversion_*_archive() are provided for Windows 1750 * platform because other archiver application use CP_OEMCP for 1751 * MultiByteToWideChar() and WideCharToMultiByte() for the filenames 1752 * in tar or zip files. But mbstowcs/wcstombs(CRT) usually use CP_ACP 1753 * unless you use setlocale(LC_ALL, ".OCP")(specify CP_OEMCP). 1754 * So we should make a string conversion between CP_ACP and CP_OEMCP 1755 * for compatibility. 1756 */ 1757 #if defined(_WIN32) && !defined(__CYGWIN__) 1758 struct archive_string_conv * 1759 archive_string_default_conversion_for_read(struct archive *a) 1760 { 1761 const char *cur_charset = get_current_charset(a); 1762 char oemcp[16]; 1763 1764 /* NOTE: a check of cur_charset is unneeded but we need 1765 * that get_current_charset() has been surely called at 1766 * this time whatever C compiler optimized. */ 1767 if (cur_charset != NULL && 1768 (a->current_codepage == CP_C_LOCALE || 1769 a->current_codepage == a->current_oemcp)) 1770 return (NULL);/* no conversion. */ 1771 1772 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1773 /* Make sure a null termination must be set. */ 1774 oemcp[sizeof(oemcp)-1] = '\0'; 1775 return (get_sconv_object(a, oemcp, cur_charset, 1776 SCONV_FROM_CHARSET)); 1777 } 1778 1779 struct archive_string_conv * 1780 archive_string_default_conversion_for_write(struct archive *a) 1781 { 1782 const char *cur_charset = get_current_charset(a); 1783 char oemcp[16]; 1784 1785 /* NOTE: a check of cur_charset is unneeded but we need 1786 * that get_current_charset() has been surely called at 1787 * this time whatever C compiler optimized. */ 1788 if (cur_charset != NULL && 1789 (a->current_codepage == CP_C_LOCALE || 1790 a->current_codepage == a->current_oemcp)) 1791 return (NULL);/* no conversion. */ 1792 1793 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1794 /* Make sure a null termination must be set. */ 1795 oemcp[sizeof(oemcp)-1] = '\0'; 1796 return (get_sconv_object(a, cur_charset, oemcp, 1797 SCONV_TO_CHARSET)); 1798 } 1799 #else 1800 struct archive_string_conv * 1801 archive_string_default_conversion_for_read(struct archive *a) 1802 { 1803 (void)a; /* UNUSED */ 1804 return (NULL); 1805 } 1806 1807 struct archive_string_conv * 1808 archive_string_default_conversion_for_write(struct archive *a) 1809 { 1810 (void)a; /* UNUSED */ 1811 return (NULL); 1812 } 1813 #endif 1814 1815 /* 1816 * Dispose of all character conversion objects in the archive object. 1817 */ 1818 void 1819 archive_string_conversion_free(struct archive *a) 1820 { 1821 struct archive_string_conv *sc; 1822 struct archive_string_conv *sc_next; 1823 1824 for (sc = a->sconv; sc != NULL; sc = sc_next) { 1825 sc_next = sc->next; 1826 free_sconv_object(sc); 1827 } 1828 a->sconv = NULL; 1829 free(a->current_code); 1830 a->current_code = NULL; 1831 } 1832 1833 /* 1834 * Return a conversion charset name. 1835 */ 1836 const char * 1837 archive_string_conversion_charset_name(struct archive_string_conv *sc) 1838 { 1839 if (sc->flag & SCONV_TO_CHARSET) 1840 return (sc->to_charset); 1841 else 1842 return (sc->from_charset); 1843 } 1844 1845 /* 1846 * Change the behavior of a string conversion. 1847 */ 1848 void 1849 archive_string_conversion_set_opt(struct archive_string_conv *sc, int opt) 1850 { 1851 switch (opt) { 1852 /* 1853 * A filename in UTF-8 was made with libarchive 2.x in a wrong 1854 * assumption that wchar_t was Unicode. 1855 * This option enables simulating the assumption in order to read 1856 * that filename correctly. 1857 */ 1858 case SCONV_SET_OPT_UTF8_LIBARCHIVE2X: 1859 #if (defined(_WIN32) && !defined(__CYGWIN__)) \ 1860 || defined(__STDC_ISO_10646__) || defined(__APPLE__) 1861 /* 1862 * Nothing to do for it since wchar_t on these platforms 1863 * is really Unicode. 1864 */ 1865 (void)sc; /* UNUSED */ 1866 #else 1867 if ((sc->flag & SCONV_UTF8_LIBARCHIVE_2) == 0) { 1868 sc->flag |= SCONV_UTF8_LIBARCHIVE_2; 1869 /* Set up string converters. */ 1870 setup_converter(sc); 1871 } 1872 #endif 1873 break; 1874 case SCONV_SET_OPT_NORMALIZATION_C: 1875 if ((sc->flag & SCONV_NORMALIZATION_C) == 0) { 1876 sc->flag |= SCONV_NORMALIZATION_C; 1877 sc->flag &= ~SCONV_NORMALIZATION_D; 1878 /* Set up string converters. */ 1879 setup_converter(sc); 1880 } 1881 break; 1882 case SCONV_SET_OPT_NORMALIZATION_D: 1883 #if defined(HAVE_ICONV) 1884 /* 1885 * If iconv will take the string, do not change the 1886 * setting of the normalization. 1887 */ 1888 if (!(sc->flag & SCONV_WIN_CP) && 1889 (sc->flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1890 !(sc->flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1891 break; 1892 #endif 1893 if ((sc->flag & SCONV_NORMALIZATION_D) == 0) { 1894 sc->flag |= SCONV_NORMALIZATION_D; 1895 sc->flag &= ~SCONV_NORMALIZATION_C; 1896 /* Set up string converters. */ 1897 setup_converter(sc); 1898 } 1899 break; 1900 default: 1901 break; 1902 } 1903 } 1904 1905 /* 1906 * 1907 * Copy one archive_string to another in locale conversion. 1908 * 1909 * archive_strncat_l(); 1910 * archive_strncpy_l(); 1911 * 1912 */ 1913 1914 static size_t 1915 mbsnbytes(const void *_p, size_t n) 1916 { 1917 size_t s; 1918 const char *p, *pp; 1919 1920 if (_p == NULL) 1921 return (0); 1922 p = (const char *)_p; 1923 1924 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1925 s = 0; 1926 pp = p; 1927 while (s < n && *pp) { 1928 pp++; 1929 s++; 1930 } 1931 return (s); 1932 } 1933 1934 static size_t 1935 utf16nbytes(const void *_p, size_t n) 1936 { 1937 size_t s; 1938 const char *p, *pp; 1939 1940 if (_p == NULL) 1941 return (0); 1942 p = (const char *)_p; 1943 1944 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1945 s = 0; 1946 pp = p; 1947 n >>= 1; 1948 while (s < n && (pp[0] || pp[1])) { 1949 pp += 2; 1950 s++; 1951 } 1952 return (s<<1); 1953 } 1954 1955 int 1956 archive_strncpy_l(struct archive_string *as, const void *_p, size_t n, 1957 struct archive_string_conv *sc) 1958 { 1959 as->length = 0; 1960 return (archive_strncat_l(as, _p, n, sc)); 1961 } 1962 1963 int 1964 archive_strncat_l(struct archive_string *as, const void *_p, size_t n, 1965 struct archive_string_conv *sc) 1966 { 1967 const void *s; 1968 size_t length = 0; 1969 int i, r = 0, r2; 1970 1971 if (_p != NULL && n > 0) { 1972 if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) 1973 length = utf16nbytes(_p, n); 1974 else 1975 length = mbsnbytes(_p, n); 1976 } 1977 1978 /* We must allocate memory even if there is no data for conversion 1979 * or copy. This simulates archive_string_append behavior. */ 1980 if (length == 0) { 1981 int tn = 1; 1982 if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) 1983 tn = 2; 1984 if (archive_string_ensure(as, as->length + tn) == NULL) 1985 return (-1); 1986 as->s[as->length] = 0; 1987 if (tn == 2) 1988 as->s[as->length+1] = 0; 1989 return (0); 1990 } 1991 1992 /* 1993 * If sc is NULL, we just make a copy. 1994 */ 1995 if (sc == NULL) { 1996 if (archive_string_append(as, _p, length) == NULL) 1997 return (-1);/* No memory */ 1998 return (0); 1999 } 2000 2001 s = _p; 2002 i = 0; 2003 if (sc->nconverter > 1) { 2004 sc->utftmp.length = 0; 2005 r2 = sc->converter[0](&(sc->utftmp), s, length, sc); 2006 if (r2 != 0 && errno == ENOMEM) 2007 return (r2); 2008 if (r > r2) 2009 r = r2; 2010 s = sc->utftmp.s; 2011 length = sc->utftmp.length; 2012 ++i; 2013 } 2014 r2 = sc->converter[i](as, s, length, sc); 2015 if (r > r2) 2016 r = r2; 2017 return (r); 2018 } 2019 2020 #if HAVE_ICONV 2021 2022 /* 2023 * Return -1 if conversion fails. 2024 */ 2025 static int 2026 iconv_strncat_in_locale(struct archive_string *as, const void *_p, 2027 size_t length, struct archive_string_conv *sc) 2028 { 2029 ICONV_CONST char *itp; 2030 size_t remaining; 2031 iconv_t cd; 2032 char *outp; 2033 size_t avail, bs; 2034 int return_value = 0; /* success */ 2035 int to_size, from_size; 2036 2037 if (sc->flag & SCONV_TO_UTF16) 2038 to_size = 2; 2039 else 2040 to_size = 1; 2041 if (sc->flag & SCONV_FROM_UTF16) 2042 from_size = 2; 2043 else 2044 from_size = 1; 2045 2046 if (archive_string_ensure(as, as->length + length*2+to_size) == NULL) 2047 return (-1); 2048 2049 cd = sc->cd; 2050 itp = (char *)(uintptr_t)_p; 2051 remaining = length; 2052 outp = as->s + as->length; 2053 avail = as->buffer_length - as->length - to_size; 2054 while (remaining >= (size_t)from_size) { 2055 size_t result = iconv(cd, &itp, &remaining, &outp, &avail); 2056 2057 if (result != (size_t)-1) 2058 break; /* Conversion completed. */ 2059 2060 if (errno == EILSEQ || errno == EINVAL) { 2061 /* 2062 * If an output charset is UTF-8 or UTF-16BE/LE, 2063 * unknown character should be U+FFFD 2064 * (replacement character). 2065 */ 2066 if (sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) { 2067 size_t rbytes; 2068 if (sc->flag & SCONV_TO_UTF8) 2069 rbytes = sizeof(utf8_replacement_char); 2070 else 2071 rbytes = 2; 2072 2073 if (avail < rbytes) { 2074 as->length = outp - as->s; 2075 bs = as->buffer_length + 2076 (remaining * to_size) + rbytes; 2077 if (NULL == 2078 archive_string_ensure(as, bs)) 2079 return (-1); 2080 outp = as->s + as->length; 2081 avail = as->buffer_length 2082 - as->length - to_size; 2083 } 2084 if (sc->flag & SCONV_TO_UTF8) 2085 memcpy(outp, utf8_replacement_char, sizeof(utf8_replacement_char)); 2086 else if (sc->flag & SCONV_TO_UTF16BE) 2087 archive_be16enc(outp, UNICODE_R_CHAR); 2088 else 2089 archive_le16enc(outp, UNICODE_R_CHAR); 2090 outp += rbytes; 2091 avail -= rbytes; 2092 } else { 2093 /* Skip the illegal input bytes. */ 2094 *outp++ = '?'; 2095 avail--; 2096 } 2097 itp += from_size; 2098 remaining -= from_size; 2099 return_value = -1; /* failure */ 2100 } else { 2101 /* E2BIG no output buffer, 2102 * Increase an output buffer. */ 2103 as->length = outp - as->s; 2104 bs = as->buffer_length + remaining * 2; 2105 if (NULL == archive_string_ensure(as, bs)) 2106 return (-1); 2107 outp = as->s + as->length; 2108 avail = as->buffer_length - as->length - to_size; 2109 } 2110 } 2111 as->length = outp - as->s; 2112 as->s[as->length] = 0; 2113 if (to_size == 2) 2114 as->s[as->length+1] = 0; 2115 return (return_value); 2116 } 2117 2118 #endif /* HAVE_ICONV */ 2119 2120 2121 #if defined(_WIN32) && !defined(__CYGWIN__) 2122 2123 /* 2124 * Translate a string from a some CodePage to an another CodePage by 2125 * Windows APIs, and copy the result. Return -1 if conversion fails. 2126 */ 2127 static int 2128 strncat_in_codepage(struct archive_string *as, 2129 const void *_p, size_t length, struct archive_string_conv *sc) 2130 { 2131 const char *s = (const char *)_p; 2132 struct archive_wstring aws; 2133 size_t l; 2134 int r, saved_flag; 2135 2136 archive_string_init(&aws); 2137 saved_flag = sc->flag; 2138 sc->flag &= ~(SCONV_NORMALIZATION_D | SCONV_NORMALIZATION_C); 2139 r = archive_wstring_append_from_mbs_in_codepage(&aws, s, length, sc); 2140 sc->flag = saved_flag; 2141 if (r != 0) { 2142 archive_wstring_free(&aws); 2143 if (errno != ENOMEM) 2144 archive_string_append(as, s, length); 2145 return (-1); 2146 } 2147 2148 l = as->length; 2149 r = archive_string_append_from_wcs_in_codepage( 2150 as, aws.s, aws.length, sc); 2151 if (r != 0 && errno != ENOMEM && l == as->length) 2152 archive_string_append(as, s, length); 2153 archive_wstring_free(&aws); 2154 return (r); 2155 } 2156 2157 /* 2158 * Test whether MBS ==> WCS is okay. 2159 */ 2160 static int 2161 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2162 { 2163 const char *p = (const char *)_p; 2164 unsigned codepage; 2165 DWORD mbflag = MB_ERR_INVALID_CHARS; 2166 2167 if (sc->flag & SCONV_FROM_CHARSET) 2168 codepage = sc->to_cp; 2169 else 2170 codepage = sc->from_cp; 2171 2172 if (codepage == CP_C_LOCALE) 2173 return (0); 2174 if (codepage != CP_UTF8) 2175 mbflag |= MB_PRECOMPOSED; 2176 2177 if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0) 2178 return (-1); /* Invalid */ 2179 return (0); /* Okay */ 2180 } 2181 2182 #else 2183 2184 /* 2185 * Test whether MBS ==> WCS is okay. 2186 */ 2187 static int 2188 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2189 { 2190 const char *p = (const char *)_p; 2191 size_t r; 2192 2193 #if HAVE_MBRTOWC 2194 mbstate_t shift_state; 2195 2196 memset(&shift_state, 0, sizeof(shift_state)); 2197 #else 2198 /* Clear the shift state before starting. */ 2199 mbtowc(NULL, NULL, 0); 2200 #endif 2201 while (n) { 2202 wchar_t wc; 2203 2204 #if HAVE_MBRTOWC 2205 r = mbrtowc(&wc, p, n, &shift_state); 2206 #else 2207 r = mbtowc(&wc, p, n); 2208 #endif 2209 if (r == (size_t)-1 || r == (size_t)-2) 2210 return (-1);/* Invalid. */ 2211 if (r == 0) 2212 break; 2213 p += r; 2214 n -= r; 2215 } 2216 (void)sc; /* UNUSED */ 2217 return (0); /* All Okey. */ 2218 } 2219 2220 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 2221 2222 /* 2223 * Basically returns -1 because we cannot make a conversion of charset 2224 * without iconv but in some cases this would return 0. 2225 * Returns 0 if all copied characters are ASCII. 2226 * Returns 0 if both from-locale and to-locale are the same and those 2227 * can be WCS with no error. 2228 */ 2229 static int 2230 best_effort_strncat_in_locale(struct archive_string *as, const void *_p, 2231 size_t length, struct archive_string_conv *sc) 2232 { 2233 size_t remaining; 2234 const uint8_t *itp; 2235 int return_value = 0; /* success */ 2236 2237 /* 2238 * If both from-locale and to-locale is the same, this makes a copy. 2239 * And then this checks all copied MBS can be WCS if so returns 0. 2240 */ 2241 if (sc->same) { 2242 if (archive_string_append(as, _p, length) == NULL) 2243 return (-1);/* No memory */ 2244 return (invalid_mbs(_p, length, sc)); 2245 } 2246 2247 /* 2248 * If a character is ASCII, this just copies it. If not, this 2249 * assigns '?' character instead but in UTF-8 locale this assigns 2250 * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD, 2251 * a Replacement Character in Unicode. 2252 */ 2253 2254 remaining = length; 2255 itp = (const uint8_t *)_p; 2256 while (*itp && remaining > 0) { 2257 if (*itp > 127) { 2258 // Non-ASCII: Substitute with suitable replacement 2259 if (sc->flag & SCONV_TO_UTF8) { 2260 if (archive_string_append(as, utf8_replacement_char, sizeof(utf8_replacement_char)) == NULL) { 2261 __archive_errx(1, "Out of memory"); 2262 } 2263 } else { 2264 archive_strappend_char(as, '?'); 2265 } 2266 return_value = -1; 2267 } else { 2268 archive_strappend_char(as, *itp); 2269 } 2270 ++itp; 2271 } 2272 return (return_value); 2273 } 2274 2275 2276 /* 2277 * Unicode conversion functions. 2278 * - UTF-8 <===> UTF-8 in removing surrogate pairs. 2279 * - UTF-8 NFD ===> UTF-8 NFC in removing surrogate pairs. 2280 * - UTF-8 made by libarchive 2.x ===> UTF-8. 2281 * - UTF-16BE <===> UTF-8. 2282 * 2283 */ 2284 2285 /* 2286 * Utility to convert a single UTF-8 sequence. 2287 * 2288 * Usually return used bytes, return used byte in negative value when 2289 * a unicode character is replaced with U+FFFD. 2290 * See also http://unicode.org/review/pr-121.html Public Review Issue #121 2291 * Recommended Practice for Replacement Characters. 2292 */ 2293 static int 2294 _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2295 { 2296 static const char utf8_count[256] = { 2297 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 00 - 0F */ 2298 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 10 - 1F */ 2299 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 20 - 2F */ 2300 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 30 - 3F */ 2301 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 40 - 4F */ 2302 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 50 - 5F */ 2303 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 60 - 6F */ 2304 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 70 - 7F */ 2305 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 80 - 8F */ 2306 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 90 - 9F */ 2307 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* A0 - AF */ 2308 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* B0 - BF */ 2309 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* C0 - CF */ 2310 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* D0 - DF */ 2311 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,/* E0 - EF */ 2312 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0 - FF */ 2313 }; 2314 int ch, i; 2315 int cnt; 2316 uint32_t wc; 2317 2318 /* Sanity check. */ 2319 if (n == 0) 2320 return (0); 2321 /* 2322 * Decode 1-4 bytes depending on the value of the first byte. 2323 */ 2324 ch = (unsigned char)*s; 2325 if (ch == 0) 2326 return (0); /* Standard: return 0 for end-of-string. */ 2327 cnt = utf8_count[ch]; 2328 2329 /* Invalid sequence or there are not plenty bytes. */ 2330 if ((int)n < cnt) { 2331 cnt = (int)n; 2332 for (i = 1; i < cnt; i++) { 2333 if ((s[i] & 0xc0) != 0x80) { 2334 cnt = i; 2335 break; 2336 } 2337 } 2338 goto invalid_sequence; 2339 } 2340 2341 /* Make a Unicode code point from a single UTF-8 sequence. */ 2342 switch (cnt) { 2343 case 1: /* 1 byte sequence. */ 2344 *pwc = ch & 0x7f; 2345 return (cnt); 2346 case 2: /* 2 bytes sequence. */ 2347 if ((s[1] & 0xc0) != 0x80) { 2348 cnt = 1; 2349 goto invalid_sequence; 2350 } 2351 *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f); 2352 return (cnt); 2353 case 3: /* 3 bytes sequence. */ 2354 if ((s[1] & 0xc0) != 0x80) { 2355 cnt = 1; 2356 goto invalid_sequence; 2357 } 2358 if ((s[2] & 0xc0) != 0x80) { 2359 cnt = 2; 2360 goto invalid_sequence; 2361 } 2362 wc = ((ch & 0x0f) << 12) 2363 | ((s[1] & 0x3f) << 6) 2364 | (s[2] & 0x3f); 2365 if (wc < 0x800) 2366 goto invalid_sequence;/* Overlong sequence. */ 2367 break; 2368 case 4: /* 4 bytes sequence. */ 2369 if ((s[1] & 0xc0) != 0x80) { 2370 cnt = 1; 2371 goto invalid_sequence; 2372 } 2373 if ((s[2] & 0xc0) != 0x80) { 2374 cnt = 2; 2375 goto invalid_sequence; 2376 } 2377 if ((s[3] & 0xc0) != 0x80) { 2378 cnt = 3; 2379 goto invalid_sequence; 2380 } 2381 wc = ((ch & 0x07) << 18) 2382 | ((s[1] & 0x3f) << 12) 2383 | ((s[2] & 0x3f) << 6) 2384 | (s[3] & 0x3f); 2385 if (wc < 0x10000) 2386 goto invalid_sequence;/* Overlong sequence. */ 2387 break; 2388 default: /* Others are all invalid sequence. */ 2389 if (ch == 0xc0 || ch == 0xc1) 2390 cnt = 2; 2391 else if (ch >= 0xf5 && ch <= 0xf7) 2392 cnt = 4; 2393 else if (ch >= 0xf8 && ch <= 0xfb) 2394 cnt = 5; 2395 else if (ch == 0xfc || ch == 0xfd) 2396 cnt = 6; 2397 else 2398 cnt = 1; 2399 if ((int)n < cnt) 2400 cnt = (int)n; 2401 for (i = 1; i < cnt; i++) { 2402 if ((s[i] & 0xc0) != 0x80) { 2403 cnt = i; 2404 break; 2405 } 2406 } 2407 goto invalid_sequence; 2408 } 2409 2410 /* The code point larger than 0x10FFFF is not legal 2411 * Unicode values. */ 2412 if (wc > UNICODE_MAX) 2413 goto invalid_sequence; 2414 /* Correctly gets a Unicode, returns used bytes. */ 2415 *pwc = wc; 2416 return (cnt); 2417 invalid_sequence: 2418 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2419 return (cnt * -1); 2420 } 2421 2422 static int 2423 utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2424 { 2425 int cnt; 2426 2427 cnt = _utf8_to_unicode(pwc, s, n); 2428 /* Any of Surrogate pair is not legal Unicode values. */ 2429 if (cnt == 3 && IS_SURROGATE_PAIR_LA(*pwc)) 2430 return (-3); 2431 return (cnt); 2432 } 2433 2434 static inline uint32_t 2435 combine_surrogate_pair(uint32_t uc, uint32_t uc2) 2436 { 2437 uc -= 0xD800; 2438 uc *= 0x400; 2439 uc += uc2 - 0xDC00; 2440 uc += 0x10000; 2441 return (uc); 2442 } 2443 2444 /* 2445 * Convert a single UTF-8/CESU-8 sequence to a Unicode code point in 2446 * removing surrogate pairs. 2447 * 2448 * CESU-8: The Compatibility Encoding Scheme for UTF-16. 2449 * 2450 * Usually return used bytes, return used byte in negative value when 2451 * a unicode character is replaced with U+FFFD. 2452 */ 2453 static int 2454 cesu8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2455 { 2456 uint32_t wc = 0; 2457 int cnt; 2458 2459 cnt = _utf8_to_unicode(&wc, s, n); 2460 if (cnt == 3 && IS_HIGH_SURROGATE_LA(wc)) { 2461 uint32_t wc2 = 0; 2462 if (n - 3 < 3) { 2463 /* Invalid byte sequence. */ 2464 goto invalid_sequence; 2465 } 2466 cnt = _utf8_to_unicode(&wc2, s+3, n-3); 2467 if (cnt != 3 || !IS_LOW_SURROGATE_LA(wc2)) { 2468 /* Invalid byte sequence. */ 2469 goto invalid_sequence; 2470 } 2471 wc = combine_surrogate_pair(wc, wc2); 2472 cnt = 6; 2473 } else if (cnt == 3 && IS_LOW_SURROGATE_LA(wc)) { 2474 /* Invalid byte sequence. */ 2475 goto invalid_sequence; 2476 } 2477 *pwc = wc; 2478 return (cnt); 2479 invalid_sequence: 2480 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2481 if (cnt > 0) 2482 cnt *= -1; 2483 return (cnt); 2484 } 2485 2486 /* 2487 * Convert a Unicode code point to a single UTF-8 sequence. 2488 * 2489 * NOTE:This function does not check if the Unicode is legal or not. 2490 * Please you definitely check it before calling this. 2491 */ 2492 static size_t 2493 unicode_to_utf8(char *p, size_t remaining, uint32_t uc) 2494 { 2495 char *_p = p; 2496 2497 /* Invalid Unicode char maps to Replacement character */ 2498 if (uc > UNICODE_MAX) 2499 uc = UNICODE_R_CHAR; 2500 /* Translate code point to UTF8 */ 2501 if (uc <= 0x7f) { 2502 if (remaining == 0) 2503 return (0); 2504 *p++ = (char)uc; 2505 } else if (uc <= 0x7ff) { 2506 if (remaining < 2) 2507 return (0); 2508 *p++ = 0xc0 | ((uc >> 6) & 0x1f); 2509 *p++ = 0x80 | (uc & 0x3f); 2510 } else if (uc <= 0xffff) { 2511 if (remaining < 3) 2512 return (0); 2513 *p++ = 0xe0 | ((uc >> 12) & 0x0f); 2514 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2515 *p++ = 0x80 | (uc & 0x3f); 2516 } else { 2517 if (remaining < 4) 2518 return (0); 2519 *p++ = 0xf0 | ((uc >> 18) & 0x07); 2520 *p++ = 0x80 | ((uc >> 12) & 0x3f); 2521 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2522 *p++ = 0x80 | (uc & 0x3f); 2523 } 2524 return (p - _p); 2525 } 2526 2527 static int 2528 utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n) 2529 { 2530 return (utf16_to_unicode(pwc, s, n, 1)); 2531 } 2532 2533 static int 2534 utf16le_to_unicode(uint32_t *pwc, const char *s, size_t n) 2535 { 2536 return (utf16_to_unicode(pwc, s, n, 0)); 2537 } 2538 2539 static int 2540 utf16_to_unicode(uint32_t *pwc, const char *s, size_t n, int be) 2541 { 2542 const char *utf16 = s; 2543 unsigned uc; 2544 2545 if (n == 0) 2546 return (0); 2547 if (n == 1) { 2548 /* set the Replacement Character instead. */ 2549 *pwc = UNICODE_R_CHAR; 2550 return (-1); 2551 } 2552 2553 if (be) 2554 uc = archive_be16dec(utf16); 2555 else 2556 uc = archive_le16dec(utf16); 2557 utf16 += 2; 2558 2559 /* If this is a surrogate pair, assemble the full code point.*/ 2560 if (IS_HIGH_SURROGATE_LA(uc)) { 2561 unsigned uc2; 2562 2563 if (n >= 4) { 2564 if (be) 2565 uc2 = archive_be16dec(utf16); 2566 else 2567 uc2 = archive_le16dec(utf16); 2568 } else 2569 uc2 = 0; 2570 if (IS_LOW_SURROGATE_LA(uc2)) { 2571 uc = combine_surrogate_pair(uc, uc2); 2572 utf16 += 2; 2573 } else { 2574 /* Undescribed code point should be U+FFFD 2575 * (replacement character). */ 2576 *pwc = UNICODE_R_CHAR; 2577 return (-2); 2578 } 2579 } 2580 2581 /* 2582 * Surrogate pair values(0xd800 through 0xdfff) are only 2583 * used by UTF-16, so, after above calculation, the code 2584 * must not be surrogate values, and Unicode has no codes 2585 * larger than 0x10ffff. Thus, those are not legal Unicode 2586 * values. 2587 */ 2588 if (IS_SURROGATE_PAIR_LA(uc) || uc > UNICODE_MAX) { 2589 /* Undescribed code point should be U+FFFD 2590 * (replacement character). */ 2591 *pwc = UNICODE_R_CHAR; 2592 return (((int)(utf16 - s)) * -1); 2593 } 2594 *pwc = uc; 2595 return ((int)(utf16 - s)); 2596 } 2597 2598 static size_t 2599 unicode_to_utf16be(char *p, size_t remaining, uint32_t uc) 2600 { 2601 char *utf16 = p; 2602 2603 if (uc > 0xffff) { 2604 /* We have a code point that won't fit into a 2605 * wchar_t; convert it to a surrogate pair. */ 2606 if (remaining < 4) 2607 return (0); 2608 uc -= 0x10000; 2609 archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2610 archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2611 return (4); 2612 } else { 2613 if (remaining < 2) 2614 return (0); 2615 archive_be16enc(utf16, uc); 2616 return (2); 2617 } 2618 } 2619 2620 static size_t 2621 unicode_to_utf16le(char *p, size_t remaining, uint32_t uc) 2622 { 2623 char *utf16 = p; 2624 2625 if (uc > 0xffff) { 2626 /* We have a code point that won't fit into a 2627 * wchar_t; convert it to a surrogate pair. */ 2628 if (remaining < 4) 2629 return (0); 2630 uc -= 0x10000; 2631 archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2632 archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2633 return (4); 2634 } else { 2635 if (remaining < 2) 2636 return (0); 2637 archive_le16enc(utf16, uc); 2638 return (2); 2639 } 2640 } 2641 2642 /* 2643 * Copy UTF-8 string in checking surrogate pair. 2644 * If any surrogate pair are found, it would be canonicalized. 2645 */ 2646 static int 2647 strncat_from_utf8_to_utf8(struct archive_string *as, const void *_p, 2648 size_t len, struct archive_string_conv *sc) 2649 { 2650 const char *s; 2651 char *p, *endp; 2652 int n, ret = 0; 2653 2654 (void)sc; /* UNUSED */ 2655 2656 if (archive_string_ensure(as, as->length + len + 1) == NULL) 2657 return (-1); 2658 2659 s = (const char *)_p; 2660 p = as->s + as->length; 2661 endp = as->s + as->buffer_length -1; 2662 do { 2663 uint32_t uc; 2664 const char *ss = s; 2665 size_t w; 2666 2667 /* 2668 * Forward byte sequence until a conversion of that is needed. 2669 */ 2670 while ((n = utf8_to_unicode(&uc, s, len)) > 0) { 2671 s += n; 2672 len -= n; 2673 } 2674 if (ss < s) { 2675 if (p + (s - ss) > endp) { 2676 as->length = p - as->s; 2677 if (archive_string_ensure(as, 2678 as->buffer_length + len + 1) == NULL) 2679 return (-1); 2680 p = as->s + as->length; 2681 endp = as->s + as->buffer_length -1; 2682 } 2683 2684 memcpy(p, ss, s - ss); 2685 p += s - ss; 2686 } 2687 2688 /* 2689 * If n is negative, current byte sequence needs a replacement. 2690 */ 2691 if (n < 0) { 2692 if (n == -3 && IS_SURROGATE_PAIR_LA(uc)) { 2693 /* Current byte sequence may be CESU-8. */ 2694 n = cesu8_to_unicode(&uc, s, len); 2695 } 2696 if (n < 0) { 2697 ret = -1; 2698 n *= -1;/* Use a replaced unicode character. */ 2699 } 2700 2701 /* Rebuild UTF-8 byte sequence. */ 2702 while ((w = unicode_to_utf8(p, endp - p, uc)) == 0) { 2703 as->length = p - as->s; 2704 if (archive_string_ensure(as, 2705 as->buffer_length + len + 1) == NULL) 2706 return (-1); 2707 p = as->s + as->length; 2708 endp = as->s + as->buffer_length -1; 2709 } 2710 p += w; 2711 s += n; 2712 len -= n; 2713 } 2714 } while (n > 0); 2715 as->length = p - as->s; 2716 as->s[as->length] = '\0'; 2717 return (ret); 2718 } 2719 2720 static int 2721 archive_string_append_unicode(struct archive_string *as, const void *_p, 2722 size_t len, struct archive_string_conv *sc) 2723 { 2724 const char *s; 2725 char *p, *endp; 2726 uint32_t uc; 2727 size_t w; 2728 int n, ret = 0, ts, tm; 2729 int (*parse)(uint32_t *, const char *, size_t); 2730 size_t (*unparse)(char *, size_t, uint32_t); 2731 2732 if (sc->flag & SCONV_TO_UTF16BE) { 2733 unparse = unicode_to_utf16be; 2734 ts = 2; 2735 } else if (sc->flag & SCONV_TO_UTF16LE) { 2736 unparse = unicode_to_utf16le; 2737 ts = 2; 2738 } else if (sc->flag & SCONV_TO_UTF8) { 2739 unparse = unicode_to_utf8; 2740 ts = 1; 2741 } else { 2742 /* 2743 * This case is going to be converted to another 2744 * character-set through iconv. 2745 */ 2746 if (sc->flag & SCONV_FROM_UTF16BE) { 2747 unparse = unicode_to_utf16be; 2748 ts = 2; 2749 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2750 unparse = unicode_to_utf16le; 2751 ts = 2; 2752 } else { 2753 unparse = unicode_to_utf8; 2754 ts = 1; 2755 } 2756 } 2757 2758 if (sc->flag & SCONV_FROM_UTF16BE) { 2759 parse = utf16be_to_unicode; 2760 tm = 1; 2761 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2762 parse = utf16le_to_unicode; 2763 tm = 1; 2764 } else { 2765 parse = cesu8_to_unicode; 2766 tm = ts; 2767 } 2768 2769 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2770 return (-1); 2771 2772 s = (const char *)_p; 2773 p = as->s + as->length; 2774 endp = as->s + as->buffer_length - ts; 2775 while ((n = parse(&uc, s, len)) != 0) { 2776 if (n < 0) { 2777 /* Use a replaced unicode character. */ 2778 n *= -1; 2779 ret = -1; 2780 } 2781 s += n; 2782 len -= n; 2783 while ((w = unparse(p, endp - p, uc)) == 0) { 2784 /* There is not enough output buffer so 2785 * we have to expand it. */ 2786 as->length = p - as->s; 2787 if (archive_string_ensure(as, 2788 as->buffer_length + len * tm + ts) == NULL) 2789 return (-1); 2790 p = as->s + as->length; 2791 endp = as->s + as->buffer_length - ts; 2792 } 2793 p += w; 2794 } 2795 as->length = p - as->s; 2796 as->s[as->length] = '\0'; 2797 if (ts == 2) 2798 as->s[as->length+1] = '\0'; 2799 return (ret); 2800 } 2801 2802 /* 2803 * Following Constants for Hangul compositions this information comes from 2804 * Unicode Standard Annex #15 http://unicode.org/reports/tr15/ 2805 */ 2806 #define HC_SBASE 0xAC00 2807 #define HC_LBASE 0x1100 2808 #define HC_VBASE 0x1161 2809 #define HC_TBASE 0x11A7 2810 #define HC_LCOUNT 19 2811 #define HC_VCOUNT 21 2812 #define HC_TCOUNT 28 2813 #define HC_NCOUNT (HC_VCOUNT * HC_TCOUNT) 2814 #define HC_SCOUNT (HC_LCOUNT * HC_NCOUNT) 2815 2816 static uint32_t 2817 get_nfc(uint32_t uc, uint32_t uc2) 2818 { 2819 int t, b; 2820 2821 t = 0; 2822 b = sizeof(u_composition_table)/sizeof(u_composition_table[0]) -1; 2823 while (b >= t) { 2824 int m = (t + b) / 2; 2825 if (u_composition_table[m].cp1 < uc) 2826 t = m + 1; 2827 else if (u_composition_table[m].cp1 > uc) 2828 b = m - 1; 2829 else if (u_composition_table[m].cp2 < uc2) 2830 t = m + 1; 2831 else if (u_composition_table[m].cp2 > uc2) 2832 b = m - 1; 2833 else 2834 return (u_composition_table[m].nfc); 2835 } 2836 return (0); 2837 } 2838 2839 #define FDC_MAX 10 /* The maximum number of Following Decomposable 2840 * Characters. */ 2841 2842 /* 2843 * Update first code point. 2844 */ 2845 #define UPDATE_UC(new_uc) do { \ 2846 uc = new_uc; \ 2847 ucptr = NULL; \ 2848 } while (0) 2849 2850 /* 2851 * Replace first code point with second code point. 2852 */ 2853 #define REPLACE_UC_WITH_UC2() do { \ 2854 uc = uc2; \ 2855 ucptr = uc2ptr; \ 2856 n = n2; \ 2857 } while (0) 2858 2859 #define EXPAND_BUFFER() do { \ 2860 as->length = p - as->s; \ 2861 if (archive_string_ensure(as, \ 2862 as->buffer_length + len * tm + ts) == NULL)\ 2863 return (-1); \ 2864 p = as->s + as->length; \ 2865 endp = as->s + as->buffer_length - ts; \ 2866 } while (0) 2867 2868 #define UNPARSE(p, endp, uc) do { \ 2869 while ((w = unparse(p, (endp) - (p), uc)) == 0) {\ 2870 EXPAND_BUFFER(); \ 2871 } \ 2872 p += w; \ 2873 } while (0) 2874 2875 /* 2876 * Write first code point. 2877 * If the code point has not be changed from its original code, 2878 * this just copies it from its original buffer pointer. 2879 * If not, this converts it to UTF-8 byte sequence and copies it. 2880 */ 2881 #define WRITE_UC() do { \ 2882 if (ucptr) { \ 2883 if (p + n > endp) \ 2884 EXPAND_BUFFER(); \ 2885 switch (n) { \ 2886 case 4: \ 2887 *p++ = *ucptr++; \ 2888 /* FALL THROUGH */ \ 2889 case 3: \ 2890 *p++ = *ucptr++; \ 2891 /* FALL THROUGH */ \ 2892 case 2: \ 2893 *p++ = *ucptr++; \ 2894 /* FALL THROUGH */ \ 2895 case 1: \ 2896 *p++ = *ucptr; \ 2897 break; \ 2898 } \ 2899 ucptr = NULL; \ 2900 } else { \ 2901 UNPARSE(p, endp, uc); \ 2902 } \ 2903 } while (0) 2904 2905 /* 2906 * Collect following decomposable code points. 2907 */ 2908 #define COLLECT_CPS(start) do { \ 2909 int _i; \ 2910 for (_i = start; _i < FDC_MAX ; _i++) { \ 2911 nx = parse(&ucx[_i], s, len); \ 2912 if (nx <= 0) \ 2913 break; \ 2914 cx = CCC(ucx[_i]); \ 2915 if (cl >= cx && cl != 228 && cx != 228)\ 2916 break; \ 2917 s += nx; \ 2918 len -= nx; \ 2919 cl = cx; \ 2920 ccx[_i] = cx; \ 2921 } \ 2922 if (_i >= FDC_MAX) { \ 2923 ret = -1; \ 2924 ucx_size = FDC_MAX; \ 2925 } else \ 2926 ucx_size = _i; \ 2927 } while (0) 2928 2929 /* 2930 * Normalize UTF-8/UTF-16BE characters to Form C and copy the result. 2931 * 2932 * TODO: Convert composition exclusions, which are never converted 2933 * from NFC,NFD,NFKC and NFKD, to Form C. 2934 */ 2935 static int 2936 archive_string_normalize_C(struct archive_string *as, const void *_p, 2937 size_t len, struct archive_string_conv *sc) 2938 { 2939 const char *s = (const char *)_p; 2940 char *p, *endp; 2941 uint32_t uc, uc2; 2942 size_t w; 2943 int always_replace, n, n2, ret = 0, spair, ts, tm; 2944 int (*parse)(uint32_t *, const char *, size_t); 2945 size_t (*unparse)(char *, size_t, uint32_t); 2946 2947 always_replace = 1; 2948 ts = 1;/* text size. */ 2949 if (sc->flag & SCONV_TO_UTF16BE) { 2950 unparse = unicode_to_utf16be; 2951 ts = 2; 2952 if (sc->flag & SCONV_FROM_UTF16BE) 2953 always_replace = 0; 2954 } else if (sc->flag & SCONV_TO_UTF16LE) { 2955 unparse = unicode_to_utf16le; 2956 ts = 2; 2957 if (sc->flag & SCONV_FROM_UTF16LE) 2958 always_replace = 0; 2959 } else if (sc->flag & SCONV_TO_UTF8) { 2960 unparse = unicode_to_utf8; 2961 if (sc->flag & SCONV_FROM_UTF8) 2962 always_replace = 0; 2963 } else { 2964 /* 2965 * This case is going to be converted to another 2966 * character-set through iconv. 2967 */ 2968 always_replace = 0; 2969 if (sc->flag & SCONV_FROM_UTF16BE) { 2970 unparse = unicode_to_utf16be; 2971 ts = 2; 2972 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2973 unparse = unicode_to_utf16le; 2974 ts = 2; 2975 } else { 2976 unparse = unicode_to_utf8; 2977 } 2978 } 2979 2980 if (sc->flag & SCONV_FROM_UTF16BE) { 2981 parse = utf16be_to_unicode; 2982 tm = 1; 2983 spair = 4;/* surrogate pair size in UTF-16. */ 2984 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2985 parse = utf16le_to_unicode; 2986 tm = 1; 2987 spair = 4;/* surrogate pair size in UTF-16. */ 2988 } else { 2989 parse = cesu8_to_unicode; 2990 tm = ts; 2991 spair = 6;/* surrogate pair size in UTF-8. */ 2992 } 2993 2994 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2995 return (-1); 2996 2997 p = as->s + as->length; 2998 endp = as->s + as->buffer_length - ts; 2999 while ((n = parse(&uc, s, len)) != 0) { 3000 const char *ucptr, *uc2ptr; 3001 3002 if (n < 0) { 3003 /* Use a replaced unicode character. */ 3004 UNPARSE(p, endp, uc); 3005 s += n*-1; 3006 len -= n*-1; 3007 ret = -1; 3008 continue; 3009 } else if (n == spair || always_replace) 3010 /* uc is converted from a surrogate pair. 3011 * this should be treated as a changed code. */ 3012 ucptr = NULL; 3013 else 3014 ucptr = s; 3015 s += n; 3016 len -= n; 3017 3018 /* Read second code point. */ 3019 while ((n2 = parse(&uc2, s, len)) > 0) { 3020 uint32_t ucx[FDC_MAX]; 3021 int ccx[FDC_MAX]; 3022 int cl, cx, i, nx, ucx_size; 3023 int LIndex,SIndex; 3024 uint32_t nfc; 3025 3026 if (n2 == spair || always_replace) 3027 /* uc2 is converted from a surrogate pair. 3028 * this should be treated as a changed code. */ 3029 uc2ptr = NULL; 3030 else 3031 uc2ptr = s; 3032 s += n2; 3033 len -= n2; 3034 3035 /* 3036 * If current second code point is out of decomposable 3037 * code points, finding compositions is unneeded. 3038 */ 3039 if (!IS_DECOMPOSABLE_BLOCK(uc2)) { 3040 WRITE_UC(); 3041 REPLACE_UC_WITH_UC2(); 3042 continue; 3043 } 3044 3045 /* 3046 * Try to combine current code points. 3047 */ 3048 /* 3049 * We have to combine Hangul characters according to 3050 * http://uniicode.org/reports/tr15/#Hangul 3051 */ 3052 if (0 <= (LIndex = uc - HC_LBASE) && 3053 LIndex < HC_LCOUNT) { 3054 /* 3055 * Hangul Composition. 3056 * 1. Two current code points are L and V. 3057 */ 3058 int VIndex = uc2 - HC_VBASE; 3059 if (0 <= VIndex && VIndex < HC_VCOUNT) { 3060 /* Make syllable of form LV. */ 3061 UPDATE_UC(HC_SBASE + 3062 (LIndex * HC_VCOUNT + VIndex) * 3063 HC_TCOUNT); 3064 } else { 3065 WRITE_UC(); 3066 REPLACE_UC_WITH_UC2(); 3067 } 3068 continue; 3069 } else if (0 <= (SIndex = uc - HC_SBASE) && 3070 SIndex < HC_SCOUNT && (SIndex % HC_TCOUNT) == 0) { 3071 /* 3072 * Hangul Composition. 3073 * 2. Two current code points are LV and T. 3074 */ 3075 int TIndex = uc2 - HC_TBASE; 3076 if (0 < TIndex && TIndex < HC_TCOUNT) { 3077 /* Make syllable of form LVT. */ 3078 UPDATE_UC(uc + TIndex); 3079 } else { 3080 WRITE_UC(); 3081 REPLACE_UC_WITH_UC2(); 3082 } 3083 continue; 3084 } else if ((nfc = get_nfc(uc, uc2)) != 0) { 3085 /* A composition to current code points 3086 * is found. */ 3087 UPDATE_UC(nfc); 3088 continue; 3089 } else if ((cl = CCC(uc2)) == 0) { 3090 /* Clearly 'uc2' the second code point is not 3091 * a decomposable code. */ 3092 WRITE_UC(); 3093 REPLACE_UC_WITH_UC2(); 3094 continue; 3095 } 3096 3097 /* 3098 * Collect following decomposable code points. 3099 */ 3100 cx = 0; 3101 ucx[0] = uc2; 3102 ccx[0] = cl; 3103 COLLECT_CPS(1); 3104 3105 /* 3106 * Find a composed code in the collected code points. 3107 */ 3108 i = 1; 3109 while (i < ucx_size) { 3110 int j; 3111 3112 if ((nfc = get_nfc(uc, ucx[i])) == 0) { 3113 i++; 3114 continue; 3115 } 3116 3117 /* 3118 * nfc is composed of uc and ucx[i]. 3119 */ 3120 UPDATE_UC(nfc); 3121 3122 /* 3123 * Remove ucx[i] by shifting 3124 * following code points. 3125 */ 3126 for (j = i; j+1 < ucx_size; j++) { 3127 ucx[j] = ucx[j+1]; 3128 ccx[j] = ccx[j+1]; 3129 } 3130 ucx_size --; 3131 3132 /* 3133 * Collect following code points blocked 3134 * by ucx[i] the removed code point. 3135 */ 3136 if (ucx_size > 0 && i == ucx_size && 3137 nx > 0 && cx == cl) { 3138 cl = ccx[ucx_size-1]; 3139 COLLECT_CPS(ucx_size); 3140 } 3141 /* 3142 * Restart finding a composed code with 3143 * the updated uc from the top of the 3144 * collected code points. 3145 */ 3146 i = 0; 3147 } 3148 3149 /* 3150 * Apparently the current code points are not 3151 * decomposed characters or already composed. 3152 */ 3153 WRITE_UC(); 3154 for (i = 0; i < ucx_size; i++) 3155 UNPARSE(p, endp, ucx[i]); 3156 3157 /* 3158 * Flush out remaining canonical combining characters. 3159 */ 3160 if (nx > 0 && cx == cl && len > 0) { 3161 while ((nx = parse(&ucx[0], s, len)) 3162 > 0) { 3163 cx = CCC(ucx[0]); 3164 if (cl > cx) 3165 break; 3166 s += nx; 3167 len -= nx; 3168 cl = cx; 3169 UNPARSE(p, endp, ucx[0]); 3170 } 3171 } 3172 break; 3173 } 3174 if (n2 < 0) { 3175 WRITE_UC(); 3176 /* Use a replaced unicode character. */ 3177 UNPARSE(p, endp, uc2); 3178 s += n2*-1; 3179 len -= n2*-1; 3180 ret = -1; 3181 continue; 3182 } else if (n2 == 0) { 3183 WRITE_UC(); 3184 break; 3185 } 3186 } 3187 as->length = p - as->s; 3188 as->s[as->length] = '\0'; 3189 if (ts == 2) 3190 as->s[as->length+1] = '\0'; 3191 return (ret); 3192 } 3193 3194 static int 3195 get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc) 3196 { 3197 int t, b; 3198 3199 /* 3200 * These are not converted to NFD on Mac OS. 3201 */ 3202 if ((uc >= 0x2000 && uc <= 0x2FFF) || 3203 (uc >= 0xF900 && uc <= 0xFAFF) || 3204 (uc >= 0x2F800 && uc <= 0x2FAFF)) 3205 return (0); 3206 /* 3207 * Those code points are not converted to NFD on Mac OS. 3208 * I do not know the reason because it is undocumented. 3209 * NFC NFD 3210 * 1109A ==> 11099 110BA 3211 * 1109C ==> 1109B 110BA 3212 * 110AB ==> 110A5 110BA 3213 */ 3214 if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB) 3215 return (0); 3216 3217 t = 0; 3218 b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1; 3219 while (b >= t) { 3220 int m = (t + b) / 2; 3221 if (u_decomposition_table[m].nfc < uc) 3222 t = m + 1; 3223 else if (u_decomposition_table[m].nfc > uc) 3224 b = m - 1; 3225 else { 3226 *cp1 = u_decomposition_table[m].cp1; 3227 *cp2 = u_decomposition_table[m].cp2; 3228 return (1); 3229 } 3230 } 3231 return (0); 3232 } 3233 3234 #define REPLACE_UC_WITH(cp) do { \ 3235 uc = cp; \ 3236 ucptr = NULL; \ 3237 } while (0) 3238 3239 /* 3240 * Normalize UTF-8 characters to Form D and copy the result. 3241 */ 3242 static int 3243 archive_string_normalize_D(struct archive_string *as, const void *_p, 3244 size_t len, struct archive_string_conv *sc) 3245 { 3246 const char *s = (const char *)_p; 3247 char *p, *endp; 3248 uint32_t uc, uc2; 3249 size_t w; 3250 int always_replace, n, n2, ret = 0, spair, ts, tm; 3251 int (*parse)(uint32_t *, const char *, size_t); 3252 size_t (*unparse)(char *, size_t, uint32_t); 3253 3254 always_replace = 1; 3255 ts = 1;/* text size. */ 3256 if (sc->flag & SCONV_TO_UTF16BE) { 3257 unparse = unicode_to_utf16be; 3258 ts = 2; 3259 if (sc->flag & SCONV_FROM_UTF16BE) 3260 always_replace = 0; 3261 } else if (sc->flag & SCONV_TO_UTF16LE) { 3262 unparse = unicode_to_utf16le; 3263 ts = 2; 3264 if (sc->flag & SCONV_FROM_UTF16LE) 3265 always_replace = 0; 3266 } else if (sc->flag & SCONV_TO_UTF8) { 3267 unparse = unicode_to_utf8; 3268 if (sc->flag & SCONV_FROM_UTF8) 3269 always_replace = 0; 3270 } else { 3271 /* 3272 * This case is going to be converted to another 3273 * character-set through iconv. 3274 */ 3275 always_replace = 0; 3276 if (sc->flag & SCONV_FROM_UTF16BE) { 3277 unparse = unicode_to_utf16be; 3278 ts = 2; 3279 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3280 unparse = unicode_to_utf16le; 3281 ts = 2; 3282 } else { 3283 unparse = unicode_to_utf8; 3284 } 3285 } 3286 3287 if (sc->flag & SCONV_FROM_UTF16BE) { 3288 parse = utf16be_to_unicode; 3289 tm = 1; 3290 spair = 4;/* surrogate pair size in UTF-16. */ 3291 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3292 parse = utf16le_to_unicode; 3293 tm = 1; 3294 spair = 4;/* surrogate pair size in UTF-16. */ 3295 } else { 3296 parse = cesu8_to_unicode; 3297 tm = ts; 3298 spair = 6;/* surrogate pair size in UTF-8. */ 3299 } 3300 3301 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 3302 return (-1); 3303 3304 p = as->s + as->length; 3305 endp = as->s + as->buffer_length - ts; 3306 while ((n = parse(&uc, s, len)) != 0) { 3307 const char *ucptr; 3308 uint32_t cp1, cp2; 3309 int SIndex; 3310 struct { 3311 uint32_t uc; 3312 int ccc; 3313 } fdc[FDC_MAX]; 3314 int fdi, fdj; 3315 int ccc; 3316 3317 check_first_code: 3318 if (n < 0) { 3319 /* Use a replaced unicode character. */ 3320 UNPARSE(p, endp, uc); 3321 s += n*-1; 3322 len -= n*-1; 3323 ret = -1; 3324 continue; 3325 } else if (n == spair || always_replace) 3326 /* uc is converted from a surrogate pair. 3327 * this should be treated as a changed code. */ 3328 ucptr = NULL; 3329 else 3330 ucptr = s; 3331 s += n; 3332 len -= n; 3333 3334 /* Hangul Decomposition. */ 3335 if ((SIndex = uc - HC_SBASE) >= 0 && SIndex < HC_SCOUNT) { 3336 int L = HC_LBASE + SIndex / HC_NCOUNT; 3337 int V = HC_VBASE + (SIndex % HC_NCOUNT) / HC_TCOUNT; 3338 int T = HC_TBASE + SIndex % HC_TCOUNT; 3339 3340 REPLACE_UC_WITH(L); 3341 WRITE_UC(); 3342 REPLACE_UC_WITH(V); 3343 WRITE_UC(); 3344 if (T != HC_TBASE) { 3345 REPLACE_UC_WITH(T); 3346 WRITE_UC(); 3347 } 3348 continue; 3349 } 3350 if (IS_DECOMPOSABLE_BLOCK(uc) && CCC(uc) != 0) { 3351 WRITE_UC(); 3352 continue; 3353 } 3354 3355 fdi = 0; 3356 while (get_nfd(&cp1, &cp2, uc) && fdi < FDC_MAX) { 3357 int k; 3358 3359 for (k = fdi; k > 0; k--) 3360 fdc[k] = fdc[k-1]; 3361 fdc[0].ccc = CCC(cp2); 3362 fdc[0].uc = cp2; 3363 fdi++; 3364 REPLACE_UC_WITH(cp1); 3365 } 3366 3367 /* Read following code points. */ 3368 while ((n2 = parse(&uc2, s, len)) > 0 && 3369 (ccc = CCC(uc2)) != 0 && fdi < FDC_MAX) { 3370 int j, k; 3371 3372 s += n2; 3373 len -= n2; 3374 for (j = 0; j < fdi; j++) { 3375 if (fdc[j].ccc > ccc) 3376 break; 3377 } 3378 if (j < fdi) { 3379 for (k = fdi; k > j; k--) 3380 fdc[k] = fdc[k-1]; 3381 fdc[j].ccc = ccc; 3382 fdc[j].uc = uc2; 3383 } else { 3384 fdc[fdi].ccc = ccc; 3385 fdc[fdi].uc = uc2; 3386 } 3387 fdi++; 3388 } 3389 3390 WRITE_UC(); 3391 for (fdj = 0; fdj < fdi; fdj++) { 3392 REPLACE_UC_WITH(fdc[fdj].uc); 3393 WRITE_UC(); 3394 } 3395 3396 if (n2 == 0) 3397 break; 3398 REPLACE_UC_WITH(uc2); 3399 n = n2; 3400 goto check_first_code; 3401 } 3402 as->length = p - as->s; 3403 as->s[as->length] = '\0'; 3404 if (ts == 2) 3405 as->s[as->length+1] = '\0'; 3406 return (ret); 3407 } 3408 3409 /* 3410 * libarchive 2.x made incorrect UTF-8 strings in the wrong assumption 3411 * that WCS is Unicode. It is true for several platforms but some are false. 3412 * And then people who did not use UTF-8 locale on the non Unicode WCS 3413 * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those 3414 * now cannot get right filename from libarchive 3.x and later since we 3415 * fixed the wrong assumption and it is incompatible to older its versions. 3416 * So we provide special option, "compat-2x.x", for resolving it. 3417 * That option enable the string conversion of libarchive 2.x. 3418 * 3419 * Translates the wrong UTF-8 string made by libarchive 2.x into current 3420 * locale character set and appends to the archive_string. 3421 * Note: returns -1 if conversion fails. 3422 */ 3423 static int 3424 strncat_from_utf8_libarchive2(struct archive_string *as, 3425 const void *_p, size_t len, struct archive_string_conv *sc) 3426 { 3427 const char *s; 3428 int n; 3429 char *p; 3430 char *end; 3431 uint32_t unicode; 3432 #if HAVE_WCRTOMB 3433 mbstate_t shift_state; 3434 3435 memset(&shift_state, 0, sizeof(shift_state)); 3436 #else 3437 /* Clear the shift state before starting. */ 3438 wctomb(NULL, L'\0'); 3439 #endif 3440 (void)sc; /* UNUSED */ 3441 /* 3442 * Allocate buffer for MBS. 3443 * We need this allocation here since it is possible that 3444 * as->s is still NULL. 3445 */ 3446 if (archive_string_ensure(as, as->length + len + 1) == NULL) 3447 return (-1); 3448 3449 s = (const char *)_p; 3450 p = as->s + as->length; 3451 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3452 while ((n = _utf8_to_unicode(&unicode, s, len)) != 0) { 3453 wchar_t wc; 3454 3455 if (p >= end) { 3456 as->length = p - as->s; 3457 /* Re-allocate buffer for MBS. */ 3458 if (archive_string_ensure(as, 3459 as->length + max(len * 2, 3460 (size_t)MB_CUR_MAX) + 1) == NULL) 3461 return (-1); 3462 p = as->s + as->length; 3463 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3464 } 3465 3466 /* 3467 * As libarchive 2.x, translates the UTF-8 characters into 3468 * wide-characters in the assumption that WCS is Unicode. 3469 */ 3470 if (n < 0) { 3471 n *= -1; 3472 wc = L'?'; 3473 } else 3474 wc = (wchar_t)unicode; 3475 3476 s += n; 3477 len -= n; 3478 /* 3479 * Translates the wide-character into the current locale MBS. 3480 */ 3481 #if HAVE_WCRTOMB 3482 n = (int)wcrtomb(p, wc, &shift_state); 3483 #else 3484 n = (int)wctomb(p, wc); 3485 #endif 3486 if (n == -1) 3487 return (-1); 3488 p += n; 3489 } 3490 as->length = p - as->s; 3491 as->s[as->length] = '\0'; 3492 return (0); 3493 } 3494 3495 3496 /* 3497 * Conversion functions between current locale dependent MBS and UTF-16BE. 3498 * strncat_from_utf16be() : UTF-16BE --> MBS 3499 * strncat_to_utf16be() : MBS --> UTF16BE 3500 */ 3501 3502 #if defined(_WIN32) && !defined(__CYGWIN__) 3503 3504 /* 3505 * Convert a UTF-16BE/LE string to current locale and copy the result. 3506 * Return -1 if conversion fails. 3507 */ 3508 static int 3509 win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes, 3510 struct archive_string_conv *sc, int be) 3511 { 3512 struct archive_string tmp; 3513 const char *u16; 3514 int ll; 3515 BOOL defchar; 3516 char *mbs; 3517 size_t mbs_size, b; 3518 int ret = 0; 3519 3520 bytes &= ~1; 3521 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3522 return (-1); 3523 3524 mbs = as->s + as->length; 3525 mbs_size = as->buffer_length - as->length -1; 3526 3527 if (sc->to_cp == CP_C_LOCALE) { 3528 /* 3529 * "C" locale special process. 3530 */ 3531 u16 = _p; 3532 ll = 0; 3533 for (b = 0; b < bytes; b += 2) { 3534 uint16_t val; 3535 if (be) 3536 val = archive_be16dec(u16+b); 3537 else 3538 val = archive_le16dec(u16+b); 3539 if (val > 255) { 3540 *mbs++ = '?'; 3541 ret = -1; 3542 } else 3543 *mbs++ = (char)(val&0xff); 3544 ll++; 3545 } 3546 as->length += ll; 3547 as->s[as->length] = '\0'; 3548 return (ret); 3549 } 3550 3551 archive_string_init(&tmp); 3552 if (be) { 3553 if (is_big_endian()) { 3554 u16 = _p; 3555 } else { 3556 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3557 return (-1); 3558 memcpy(tmp.s, _p, bytes); 3559 for (b = 0; b < bytes; b += 2) { 3560 uint16_t val = archive_be16dec(tmp.s+b); 3561 archive_le16enc(tmp.s+b, val); 3562 } 3563 u16 = tmp.s; 3564 } 3565 } else { 3566 if (!is_big_endian()) { 3567 u16 = _p; 3568 } else { 3569 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3570 return (-1); 3571 memcpy(tmp.s, _p, bytes); 3572 for (b = 0; b < bytes; b += 2) { 3573 uint16_t val = archive_le16dec(tmp.s+b); 3574 archive_be16enc(tmp.s+b, val); 3575 } 3576 u16 = tmp.s; 3577 } 3578 } 3579 3580 do { 3581 defchar = 0; 3582 ll = WideCharToMultiByte(sc->to_cp, 0, 3583 (LPCWSTR)u16, (int)bytes>>1, mbs, (int)mbs_size, 3584 NULL, &defchar); 3585 /* Exit loop if we succeeded */ 3586 if (ll != 0 || 3587 GetLastError() != ERROR_INSUFFICIENT_BUFFER) { 3588 break; 3589 } 3590 /* Else expand buffer and loop to try again. */ 3591 ll = WideCharToMultiByte(sc->to_cp, 0, 3592 (LPCWSTR)u16, (int)bytes, NULL, 0, NULL, NULL); 3593 if (archive_string_ensure(as, ll +1) == NULL) 3594 return (-1); 3595 mbs = as->s + as->length; 3596 mbs_size = as->buffer_length - as->length -1; 3597 } while (1); 3598 archive_string_free(&tmp); 3599 as->length += ll; 3600 as->s[as->length] = '\0'; 3601 if (ll == 0 || defchar) 3602 ret = -1; 3603 return (ret); 3604 } 3605 3606 static int 3607 win_strncat_from_utf16be(struct archive_string *as, const void *_p, 3608 size_t bytes, struct archive_string_conv *sc) 3609 { 3610 return (win_strncat_from_utf16(as, _p, bytes, sc, 1)); 3611 } 3612 3613 static int 3614 win_strncat_from_utf16le(struct archive_string *as, const void *_p, 3615 size_t bytes, struct archive_string_conv *sc) 3616 { 3617 return (win_strncat_from_utf16(as, _p, bytes, sc, 0)); 3618 } 3619 3620 static int 3621 is_big_endian(void) 3622 { 3623 uint16_t d = 1; 3624 3625 return (archive_be16dec(&d) == 1); 3626 } 3627 3628 /* 3629 * Convert a current locale string to UTF-16BE/LE and copy the result. 3630 * Return -1 if conversion fails. 3631 */ 3632 static int 3633 win_strncat_to_utf16(struct archive_string *as16, const void *_p, 3634 size_t length, struct archive_string_conv *sc, int bigendian) 3635 { 3636 const char *s = (const char *)_p; 3637 char *u16; 3638 size_t count, avail; 3639 3640 if (archive_string_ensure(as16, 3641 as16->length + (length + 1) * 2) == NULL) 3642 return (-1); 3643 3644 u16 = as16->s + as16->length; 3645 avail = as16->buffer_length - 2; 3646 if (sc->from_cp == CP_C_LOCALE) { 3647 /* 3648 * "C" locale special process. 3649 */ 3650 count = 0; 3651 while (count < length && *s) { 3652 if (bigendian) 3653 archive_be16enc(u16, *s); 3654 else 3655 archive_le16enc(u16, *s); 3656 u16 += 2; 3657 s++; 3658 count++; 3659 } 3660 as16->length += count << 1; 3661 as16->s[as16->length] = 0; 3662 as16->s[as16->length+1] = 0; 3663 return (0); 3664 } 3665 do { 3666 count = MultiByteToWideChar(sc->from_cp, 3667 MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1); 3668 /* Exit loop if we succeeded */ 3669 if (count != 0 || 3670 GetLastError() != ERROR_INSUFFICIENT_BUFFER) { 3671 break; 3672 } 3673 /* Expand buffer and try again */ 3674 count = MultiByteToWideChar(sc->from_cp, 3675 MB_PRECOMPOSED, s, (int)length, NULL, 0); 3676 if (archive_string_ensure(as16, (count +1) * 2) 3677 == NULL) 3678 return (-1); 3679 u16 = as16->s + as16->length; 3680 avail = as16->buffer_length - 2; 3681 } while (1); 3682 as16->length += count * 2; 3683 as16->s[as16->length] = 0; 3684 as16->s[as16->length+1] = 0; 3685 if (count == 0) 3686 return (-1); 3687 3688 if (is_big_endian()) { 3689 if (!bigendian) { 3690 while (count > 0) { 3691 uint16_t v = archive_be16dec(u16); 3692 archive_le16enc(u16, v); 3693 u16 += 2; 3694 count--; 3695 } 3696 } 3697 } else { 3698 if (bigendian) { 3699 while (count > 0) { 3700 uint16_t v = archive_le16dec(u16); 3701 archive_be16enc(u16, v); 3702 u16 += 2; 3703 count--; 3704 } 3705 } 3706 } 3707 return (0); 3708 } 3709 3710 static int 3711 win_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3712 size_t length, struct archive_string_conv *sc) 3713 { 3714 return (win_strncat_to_utf16(as16, _p, length, sc, 1)); 3715 } 3716 3717 static int 3718 win_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3719 size_t length, struct archive_string_conv *sc) 3720 { 3721 return (win_strncat_to_utf16(as16, _p, length, sc, 0)); 3722 } 3723 3724 #endif /* _WIN32 && !__CYGWIN__ */ 3725 3726 /* 3727 * Do the best effort for conversions. 3728 * We cannot handle UTF-16BE character-set without such iconv, 3729 * but there is a chance if a string consists just ASCII code or 3730 * a current locale is UTF-8. 3731 */ 3732 3733 /* 3734 * Convert a UTF-16BE string to current locale and copy the result. 3735 * Return -1 if conversion fails. 3736 */ 3737 static int 3738 best_effort_strncat_from_utf16(struct archive_string *as, const void *_p, 3739 size_t bytes, struct archive_string_conv *sc, int be) 3740 { 3741 const char *utf16 = (const char *)_p; 3742 char *mbs; 3743 uint32_t uc; 3744 int n, ret; 3745 3746 (void)sc; /* UNUSED */ 3747 /* 3748 * Other case, we should do the best effort. 3749 * If all character are ASCII(<0x7f), we can convert it. 3750 * if not , we set a alternative character and return -1. 3751 */ 3752 ret = 0; 3753 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3754 return (-1); 3755 mbs = as->s + as->length; 3756 3757 while ((n = utf16_to_unicode(&uc, utf16, bytes, be)) != 0) { 3758 if (n < 0) { 3759 n *= -1; 3760 ret = -1; 3761 } 3762 bytes -= n; 3763 utf16 += n; 3764 3765 if (uc > 127) { 3766 /* We cannot handle it. */ 3767 *mbs++ = '?'; 3768 ret = -1; 3769 } else 3770 *mbs++ = (char)uc; 3771 } 3772 as->length = mbs - as->s; 3773 as->s[as->length] = '\0'; 3774 return (ret); 3775 } 3776 3777 static int 3778 best_effort_strncat_from_utf16be(struct archive_string *as, const void *_p, 3779 size_t bytes, struct archive_string_conv *sc) 3780 { 3781 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 1)); 3782 } 3783 3784 static int 3785 best_effort_strncat_from_utf16le(struct archive_string *as, const void *_p, 3786 size_t bytes, struct archive_string_conv *sc) 3787 { 3788 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 0)); 3789 } 3790 3791 /* 3792 * Convert a current locale string to UTF-16BE/LE and copy the result. 3793 * Return -1 if conversion fails. 3794 */ 3795 static int 3796 best_effort_strncat_to_utf16(struct archive_string *as16, const void *_p, 3797 size_t length, struct archive_string_conv *sc, int bigendian) 3798 { 3799 const char *s = (const char *)_p; 3800 char *utf16; 3801 size_t remaining; 3802 int ret; 3803 3804 (void)sc; /* UNUSED */ 3805 /* 3806 * Other case, we should do the best effort. 3807 * If all character are ASCII(<0x7f), we can convert it. 3808 * if not , we set a alternative character and return -1. 3809 */ 3810 ret = 0; 3811 remaining = length; 3812 3813 if (archive_string_ensure(as16, 3814 as16->length + (length + 1) * 2) == NULL) 3815 return (-1); 3816 3817 utf16 = as16->s + as16->length; 3818 while (remaining--) { 3819 unsigned c = *s++; 3820 if (c > 127) { 3821 /* We cannot handle it. */ 3822 c = UNICODE_R_CHAR; 3823 ret = -1; 3824 } 3825 if (bigendian) 3826 archive_be16enc(utf16, c); 3827 else 3828 archive_le16enc(utf16, c); 3829 utf16 += 2; 3830 } 3831 as16->length = utf16 - as16->s; 3832 as16->s[as16->length] = 0; 3833 as16->s[as16->length+1] = 0; 3834 return (ret); 3835 } 3836 3837 static int 3838 best_effort_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3839 size_t length, struct archive_string_conv *sc) 3840 { 3841 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 1)); 3842 } 3843 3844 static int 3845 best_effort_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3846 size_t length, struct archive_string_conv *sc) 3847 { 3848 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 0)); 3849 } 3850 3851 3852 /* 3853 * Multistring operations. 3854 */ 3855 3856 void 3857 archive_mstring_clean(struct archive_mstring *aes) 3858 { 3859 archive_wstring_free(&(aes->aes_wcs)); 3860 archive_string_free(&(aes->aes_mbs)); 3861 archive_string_free(&(aes->aes_utf8)); 3862 archive_string_free(&(aes->aes_mbs_in_locale)); 3863 aes->aes_set = 0; 3864 } 3865 3866 void 3867 archive_mstring_copy(struct archive_mstring *dest, struct archive_mstring *src) 3868 { 3869 dest->aes_set = src->aes_set; 3870 archive_string_copy(&(dest->aes_mbs), &(src->aes_mbs)); 3871 archive_string_copy(&(dest->aes_utf8), &(src->aes_utf8)); 3872 archive_wstring_copy(&(dest->aes_wcs), &(src->aes_wcs)); 3873 } 3874 3875 int 3876 archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes, 3877 const char **p) 3878 { 3879 struct archive_string_conv *sc; 3880 int r; 3881 3882 /* If we already have a UTF8 form, return that immediately. */ 3883 if (aes->aes_set & AES_SET_UTF8) { 3884 *p = aes->aes_utf8.s; 3885 return (0); 3886 } 3887 3888 *p = NULL; 3889 /* Try converting WCS to MBS first if MBS does not exist yet. */ 3890 if ((aes->aes_set & AES_SET_MBS) == 0) { 3891 const char *pm; /* unused */ 3892 archive_mstring_get_mbs(a, aes, &pm); /* ignore errors, we'll handle it later */ 3893 } 3894 if (aes->aes_set & AES_SET_MBS) { 3895 sc = archive_string_conversion_to_charset(a, "UTF-8", 1); 3896 if (sc == NULL) 3897 return (-1);/* Couldn't allocate memory for sc. */ 3898 r = archive_strncpy_l(&(aes->aes_utf8), aes->aes_mbs.s, 3899 aes->aes_mbs.length, sc); 3900 if (a == NULL) 3901 free_sconv_object(sc); 3902 if (r == 0) { 3903 aes->aes_set |= AES_SET_UTF8; 3904 *p = aes->aes_utf8.s; 3905 return (0);/* success. */ 3906 } else 3907 return (-1);/* failure. */ 3908 } 3909 return (0);/* success. */ 3910 } 3911 3912 int 3913 archive_mstring_get_mbs(struct archive *a, struct archive_mstring *aes, 3914 const char **p) 3915 { 3916 struct archive_string_conv *sc; 3917 int r, ret = 0; 3918 3919 /* If we already have an MBS form, return that immediately. */ 3920 if (aes->aes_set & AES_SET_MBS) { 3921 *p = aes->aes_mbs.s; 3922 return (ret); 3923 } 3924 3925 *p = NULL; 3926 /* If there's a WCS form, try converting with the native locale. */ 3927 if (aes->aes_set & AES_SET_WCS) { 3928 archive_string_empty(&(aes->aes_mbs)); 3929 r = archive_string_append_from_wcs(&(aes->aes_mbs), 3930 aes->aes_wcs.s, aes->aes_wcs.length); 3931 *p = aes->aes_mbs.s; 3932 if (r == 0) { 3933 aes->aes_set |= AES_SET_MBS; 3934 return (ret); 3935 } else 3936 ret = -1; 3937 } 3938 3939 /* If there's a UTF-8 form, try converting with the native locale. */ 3940 if (aes->aes_set & AES_SET_UTF8) { 3941 archive_string_empty(&(aes->aes_mbs)); 3942 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 3943 if (sc == NULL) 3944 return (-1);/* Couldn't allocate memory for sc. */ 3945 r = archive_strncpy_l(&(aes->aes_mbs), 3946 aes->aes_utf8.s, aes->aes_utf8.length, sc); 3947 if (a == NULL) 3948 free_sconv_object(sc); 3949 *p = aes->aes_mbs.s; 3950 if (r == 0) { 3951 aes->aes_set |= AES_SET_MBS; 3952 ret = 0;/* success; overwrite previous error. */ 3953 } else 3954 ret = -1;/* failure. */ 3955 } 3956 return (ret); 3957 } 3958 3959 int 3960 archive_mstring_get_wcs(struct archive *a, struct archive_mstring *aes, 3961 const wchar_t **wp) 3962 { 3963 int r, ret = 0; 3964 3965 (void)a;/* UNUSED */ 3966 /* Return WCS form if we already have it. */ 3967 if (aes->aes_set & AES_SET_WCS) { 3968 *wp = aes->aes_wcs.s; 3969 return (ret); 3970 } 3971 3972 *wp = NULL; 3973 /* Try converting UTF8 to MBS first if MBS does not exist yet. */ 3974 if ((aes->aes_set & AES_SET_MBS) == 0) { 3975 const char *p; /* unused */ 3976 archive_mstring_get_mbs(a, aes, &p); /* ignore errors, we'll handle it later */ 3977 } 3978 /* Try converting MBS to WCS using native locale. */ 3979 if (aes->aes_set & AES_SET_MBS) { 3980 archive_wstring_empty(&(aes->aes_wcs)); 3981 r = archive_wstring_append_from_mbs(&(aes->aes_wcs), 3982 aes->aes_mbs.s, aes->aes_mbs.length); 3983 if (r == 0) { 3984 aes->aes_set |= AES_SET_WCS; 3985 *wp = aes->aes_wcs.s; 3986 } else 3987 ret = -1;/* failure. */ 3988 } 3989 return (ret); 3990 } 3991 3992 int 3993 archive_mstring_get_mbs_l(struct archive *a, struct archive_mstring *aes, 3994 const char **p, size_t *length, struct archive_string_conv *sc) 3995 { 3996 int ret = 0; 3997 #if defined(_WIN32) && !defined(__CYGWIN__) 3998 int r; 3999 4000 /* 4001 * Internationalization programming on Windows must use Wide 4002 * characters because Windows platform cannot make locale UTF-8. 4003 */ 4004 if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) { 4005 archive_string_empty(&(aes->aes_mbs_in_locale)); 4006 r = archive_string_append_from_wcs_in_codepage( 4007 &(aes->aes_mbs_in_locale), aes->aes_wcs.s, 4008 aes->aes_wcs.length, sc); 4009 if (r == 0) { 4010 *p = aes->aes_mbs_in_locale.s; 4011 if (length != NULL) 4012 *length = aes->aes_mbs_in_locale.length; 4013 return (0); 4014 } else if (errno == ENOMEM) 4015 return (-1); 4016 else 4017 ret = -1; 4018 } 4019 #endif 4020 4021 /* If there is not an MBS form but there is a WCS or UTF8 form, try converting 4022 * with the native locale to be used for translating it to specified 4023 * character-set. */ 4024 if ((aes->aes_set & AES_SET_MBS) == 0) { 4025 const char *pm; /* unused */ 4026 archive_mstring_get_mbs(a, aes, &pm); /* ignore errors, we'll handle it later */ 4027 } 4028 /* If we already have an MBS form, use it to be translated to 4029 * specified character-set. */ 4030 if (aes->aes_set & AES_SET_MBS) { 4031 if (sc == NULL) { 4032 /* Conversion is unneeded. */ 4033 *p = aes->aes_mbs.s; 4034 if (length != NULL) 4035 *length = aes->aes_mbs.length; 4036 return (0); 4037 } 4038 ret = archive_strncpy_l(&(aes->aes_mbs_in_locale), 4039 aes->aes_mbs.s, aes->aes_mbs.length, sc); 4040 *p = aes->aes_mbs_in_locale.s; 4041 if (length != NULL) 4042 *length = aes->aes_mbs_in_locale.length; 4043 } else { 4044 *p = NULL; 4045 if (length != NULL) 4046 *length = 0; 4047 } 4048 return (ret); 4049 } 4050 4051 int 4052 archive_mstring_copy_mbs(struct archive_mstring *aes, const char *mbs) 4053 { 4054 if (mbs == NULL) { 4055 aes->aes_set = 0; 4056 return (0); 4057 } 4058 return (archive_mstring_copy_mbs_len(aes, mbs, strlen(mbs))); 4059 } 4060 4061 int 4062 archive_mstring_copy_mbs_len(struct archive_mstring *aes, const char *mbs, 4063 size_t len) 4064 { 4065 if (mbs == NULL) { 4066 aes->aes_set = 0; 4067 return (0); 4068 } 4069 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4070 archive_strncpy(&(aes->aes_mbs), mbs, len); 4071 archive_string_empty(&(aes->aes_utf8)); 4072 archive_wstring_empty(&(aes->aes_wcs)); 4073 return (0); 4074 } 4075 4076 int 4077 archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs) 4078 { 4079 return archive_mstring_copy_wcs_len(aes, wcs, 4080 wcs == NULL ? 0 : wcslen(wcs)); 4081 } 4082 4083 int 4084 archive_mstring_copy_utf8(struct archive_mstring *aes, const char *utf8) 4085 { 4086 if (utf8 == NULL) { 4087 aes->aes_set = 0; 4088 return (0); 4089 } 4090 aes->aes_set = AES_SET_UTF8; 4091 archive_string_empty(&(aes->aes_mbs)); 4092 archive_string_empty(&(aes->aes_wcs)); 4093 archive_strncpy(&(aes->aes_utf8), utf8, strlen(utf8)); 4094 return (int)strlen(utf8); 4095 } 4096 4097 int 4098 archive_mstring_copy_wcs_len(struct archive_mstring *aes, const wchar_t *wcs, 4099 size_t len) 4100 { 4101 if (wcs == NULL) { 4102 aes->aes_set = 0; 4103 return (0); 4104 } 4105 aes->aes_set = AES_SET_WCS; /* Only WCS form set. */ 4106 archive_string_empty(&(aes->aes_mbs)); 4107 archive_string_empty(&(aes->aes_utf8)); 4108 archive_wstrncpy(&(aes->aes_wcs), wcs, len); 4109 return (0); 4110 } 4111 4112 int 4113 archive_mstring_copy_mbs_len_l(struct archive_mstring *aes, 4114 const char *mbs, size_t len, struct archive_string_conv *sc) 4115 { 4116 int r; 4117 4118 if (mbs == NULL) { 4119 aes->aes_set = 0; 4120 return (0); 4121 } 4122 archive_string_empty(&(aes->aes_mbs)); 4123 archive_wstring_empty(&(aes->aes_wcs)); 4124 archive_string_empty(&(aes->aes_utf8)); 4125 #if defined(_WIN32) && !defined(__CYGWIN__) 4126 /* 4127 * Internationalization programming on Windows must use Wide 4128 * characters because Windows platform cannot make locale UTF-8. 4129 */ 4130 if (sc == NULL) { 4131 if (archive_string_append(&(aes->aes_mbs), 4132 mbs, mbsnbytes(mbs, len)) == NULL) { 4133 aes->aes_set = 0; 4134 r = -1; 4135 } else { 4136 aes->aes_set = AES_SET_MBS; 4137 r = 0; 4138 } 4139 #if defined(HAVE_ICONV) 4140 } else if (sc != NULL && sc->cd_w != (iconv_t)-1) { 4141 /* 4142 * This case happens only when MultiByteToWideChar() cannot 4143 * handle sc->from_cp, and we have to iconv in order to 4144 * translate character-set to wchar_t,UTF-16. 4145 */ 4146 iconv_t cd = sc->cd; 4147 unsigned from_cp; 4148 int flag; 4149 4150 /* 4151 * Translate multi-bytes from some character-set to UTF-8. 4152 */ 4153 sc->cd = sc->cd_w; 4154 r = archive_strncpy_l(&(aes->aes_utf8), mbs, len, sc); 4155 sc->cd = cd; 4156 if (r != 0) { 4157 aes->aes_set = 0; 4158 return (r); 4159 } 4160 aes->aes_set = AES_SET_UTF8; 4161 4162 /* 4163 * Append the UTF-8 string into wstring. 4164 */ 4165 flag = sc->flag; 4166 sc->flag &= ~(SCONV_NORMALIZATION_C 4167 | SCONV_TO_UTF16| SCONV_FROM_UTF16); 4168 from_cp = sc->from_cp; 4169 sc->from_cp = CP_UTF8; 4170 r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4171 aes->aes_utf8.s, aes->aes_utf8.length, sc); 4172 sc->flag = flag; 4173 sc->from_cp = from_cp; 4174 if (r == 0) 4175 aes->aes_set |= AES_SET_WCS; 4176 #endif 4177 } else { 4178 r = archive_wstring_append_from_mbs_in_codepage( 4179 &(aes->aes_wcs), mbs, len, sc); 4180 if (r == 0) 4181 aes->aes_set = AES_SET_WCS; 4182 else 4183 aes->aes_set = 0; 4184 } 4185 #else 4186 r = archive_strncpy_l(&(aes->aes_mbs), mbs, len, sc); 4187 if (r == 0) 4188 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4189 else 4190 aes->aes_set = 0; 4191 #endif 4192 return (r); 4193 } 4194 4195 /* 4196 * The 'update' form tries to proactively update all forms of 4197 * this string (WCS and MBS) and returns an error if any of 4198 * them fail. This is used by the 'pax' handler, for instance, 4199 * to detect and report character-conversion failures early while 4200 * still allowing clients to get potentially useful values from 4201 * the more tolerant lazy conversions. (get_mbs and get_wcs will 4202 * strive to give the user something useful, so you can get hopefully 4203 * usable values even if some of the character conversions are failing.) 4204 */ 4205 int 4206 archive_mstring_update_utf8(struct archive *a, struct archive_mstring *aes, 4207 const char *utf8) 4208 { 4209 struct archive_string_conv *sc; 4210 int r; 4211 4212 if (utf8 == NULL) { 4213 aes->aes_set = 0; 4214 return (0); /* Succeeded in clearing everything. */ 4215 } 4216 4217 /* Save the UTF8 string. */ 4218 archive_strcpy(&(aes->aes_utf8), utf8); 4219 4220 /* Empty the mbs and wcs strings. */ 4221 archive_string_empty(&(aes->aes_mbs)); 4222 archive_wstring_empty(&(aes->aes_wcs)); 4223 4224 aes->aes_set = AES_SET_UTF8; /* Only UTF8 is set now. */ 4225 4226 /* Try converting UTF-8 to MBS, return false on failure. */ 4227 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 4228 if (sc == NULL) 4229 return (-1);/* Couldn't allocate memory for sc. */ 4230 r = archive_strcpy_l(&(aes->aes_mbs), utf8, sc); 4231 4232 #if defined(_WIN32) && !defined(__CYGWIN__) 4233 /* On failure, make an effort to convert UTF8 to WCS as the active code page 4234 * may not be able to represent all characters in the string */ 4235 if (r != 0) { 4236 if (archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4237 aes->aes_utf8.s, aes->aes_utf8.length, sc) == 0) 4238 aes->aes_set = AES_SET_UTF8 | AES_SET_WCS; 4239 } 4240 #endif 4241 4242 if (a == NULL) 4243 free_sconv_object(sc); 4244 if (r != 0) 4245 return (-1); 4246 aes->aes_set = AES_SET_UTF8 | AES_SET_MBS; /* Both UTF8 and MBS set. */ 4247 4248 /* Try converting MBS to WCS, return false on failure. */ 4249 if (archive_wstring_append_from_mbs(&(aes->aes_wcs), aes->aes_mbs.s, 4250 aes->aes_mbs.length)) 4251 return (-1); 4252 aes->aes_set = AES_SET_UTF8 | AES_SET_WCS | AES_SET_MBS; 4253 4254 /* All conversions succeeded. */ 4255 return (0); 4256 } 4257