xref: /freebsd/contrib/libarchive/libarchive/archive_string.c (revision 06c3fb2749bda94cb5201f81ffdb8fa6c3161b2e)
1 /*-
2  * Copyright (c) 2003-2011 Tim Kientzle
3  * Copyright (c) 2011-2012 Michihiro NAKAJIMA
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include "archive_platform.h"
28 __FBSDID("$FreeBSD$");
29 
30 /*
31  * Basic resizable string support, to simplify manipulating arbitrary-sized
32  * strings while minimizing heap activity.
33  *
34  * In particular, the buffer used by a string object is only grown, it
35  * never shrinks, so you can clear and reuse the same string object
36  * without incurring additional memory allocations.
37  */
38 
39 #ifdef HAVE_ERRNO_H
40 #include <errno.h>
41 #endif
42 #ifdef HAVE_ICONV_H
43 #include <iconv.h>
44 #endif
45 #ifdef HAVE_LANGINFO_H
46 #include <langinfo.h>
47 #endif
48 #ifdef HAVE_LOCALCHARSET_H
49 #include <localcharset.h>
50 #endif
51 #ifdef HAVE_STDLIB_H
52 #include <stdlib.h>
53 #endif
54 #ifdef HAVE_STRING_H
55 #include <string.h>
56 #endif
57 #ifdef HAVE_WCHAR_H
58 #include <wchar.h>
59 #endif
60 #if defined(_WIN32) && !defined(__CYGWIN__)
61 #include <windows.h>
62 #include <locale.h>
63 #endif
64 
65 #include "archive_endian.h"
66 #include "archive_private.h"
67 #include "archive_string.h"
68 #include "archive_string_composition.h"
69 
70 #if !defined(HAVE_WMEMCPY) && !defined(wmemcpy)
71 #define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))
72 #endif
73 
74 #if !defined(HAVE_WMEMMOVE) && !defined(wmemmove)
75 #define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))
76 #endif
77 
78 #undef max
79 #define max(a, b)       ((a)>(b)?(a):(b))
80 
81 struct archive_string_conv {
82 	struct archive_string_conv	*next;
83 	char				*from_charset;
84 	char				*to_charset;
85 	unsigned			 from_cp;
86 	unsigned			 to_cp;
87 	/* Set 1 if from_charset and to_charset are the same. */
88 	int				 same;
89 	int				 flag;
90 #define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
91 					 * charset. */
92 #define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
93 					 * specified charset. */
94 #define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
95 #define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
96 					 * MBS. */
97 #define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
98 					 * 2.x in the wrong assumption. */
99 #define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
100 					 * Before UTF-8 characters are actually
101 					 * processed. */
102 #define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
103 					 * Before UTF-8 characters are actually
104 					 * processed.
105 					 * Currently this only for MAC OS X. */
106 #define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
107 #define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
108 #define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
109 #define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
110 #define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
111 #define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
112 #define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
113 #define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)
114 
115 #if HAVE_ICONV
116 	iconv_t				 cd;
117 	iconv_t				 cd_w;/* Use at archive_mstring on
118 				 	       * Windows. */
119 #endif
120 	/* A temporary buffer for normalization. */
121 	struct archive_string		 utftmp;
122 	int (*converter[2])(struct archive_string *, const void *, size_t,
123 	    struct archive_string_conv *);
124 	int				 nconverter;
125 };
126 
127 #define CP_C_LOCALE	0	/* "C" locale only for this file. */
128 #define CP_UTF16LE	1200
129 #define CP_UTF16BE	1201
130 
131 #define IS_HIGH_SURROGATE_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDBFF)
132 #define IS_LOW_SURROGATE_LA(uc)	 ((uc) >= 0xDC00 && (uc) <= 0xDFFF)
133 #define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF)
134 #define UNICODE_MAX		0x10FFFF
135 #define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */
136 /* Set U+FFFD(Replacement character) in UTF-8. */
137 static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};
138 
139 static struct archive_string_conv *find_sconv_object(struct archive *,
140 	const char *, const char *);
141 static void add_sconv_object(struct archive *, struct archive_string_conv *);
142 static struct archive_string_conv *create_sconv_object(const char *,
143 	const char *, unsigned, int);
144 static void free_sconv_object(struct archive_string_conv *);
145 static struct archive_string_conv *get_sconv_object(struct archive *,
146 	const char *, const char *, int);
147 static unsigned make_codepage_from_charset(const char *);
148 static unsigned get_current_codepage(void);
149 static unsigned get_current_oemcp(void);
150 static size_t mbsnbytes(const void *, size_t);
151 static size_t utf16nbytes(const void *, size_t);
152 #if defined(_WIN32) && !defined(__CYGWIN__)
153 static int archive_wstring_append_from_mbs_in_codepage(
154     struct archive_wstring *, const char *, size_t,
155     struct archive_string_conv *);
156 static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
157     const wchar_t *, size_t, struct archive_string_conv *);
158 static int is_big_endian(void);
159 static int strncat_in_codepage(struct archive_string *, const void *,
160     size_t, struct archive_string_conv *);
161 static int win_strncat_from_utf16be(struct archive_string *, const void *,
162     size_t, struct archive_string_conv *);
163 static int win_strncat_from_utf16le(struct archive_string *, const void *,
164     size_t, struct archive_string_conv *);
165 static int win_strncat_to_utf16be(struct archive_string *, const void *,
166     size_t, struct archive_string_conv *);
167 static int win_strncat_to_utf16le(struct archive_string *, const void *,
168     size_t, struct archive_string_conv *);
169 #endif
170 static int best_effort_strncat_from_utf16be(struct archive_string *,
171     const void *, size_t, struct archive_string_conv *);
172 static int best_effort_strncat_from_utf16le(struct archive_string *,
173     const void *, size_t, struct archive_string_conv *);
174 static int best_effort_strncat_to_utf16be(struct archive_string *,
175     const void *, size_t, struct archive_string_conv *);
176 static int best_effort_strncat_to_utf16le(struct archive_string *,
177     const void *, size_t, struct archive_string_conv *);
178 #if defined(HAVE_ICONV)
179 static int iconv_strncat_in_locale(struct archive_string *, const void *,
180     size_t, struct archive_string_conv *);
181 #endif
182 static int best_effort_strncat_in_locale(struct archive_string *,
183     const void *, size_t, struct archive_string_conv *);
184 static int _utf8_to_unicode(uint32_t *, const char *, size_t);
185 static int utf8_to_unicode(uint32_t *, const char *, size_t);
186 static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t);
187 static int cesu8_to_unicode(uint32_t *, const char *, size_t);
188 static size_t unicode_to_utf8(char *, size_t, uint32_t);
189 static int utf16_to_unicode(uint32_t *, const char *, size_t, int);
190 static size_t unicode_to_utf16be(char *, size_t, uint32_t);
191 static size_t unicode_to_utf16le(char *, size_t, uint32_t);
192 static int strncat_from_utf8_libarchive2(struct archive_string *,
193     const void *, size_t, struct archive_string_conv *);
194 static int strncat_from_utf8_to_utf8(struct archive_string *, const void *,
195     size_t, struct archive_string_conv *);
196 static int archive_string_normalize_C(struct archive_string *, const void *,
197     size_t, struct archive_string_conv *);
198 static int archive_string_normalize_D(struct archive_string *, const void *,
199     size_t, struct archive_string_conv *);
200 static int archive_string_append_unicode(struct archive_string *,
201     const void *, size_t, struct archive_string_conv *);
202 
203 static struct archive_string *
204 archive_string_append(struct archive_string *as, const char *p, size_t s)
205 {
206 	if (archive_string_ensure(as, as->length + s + 1) == NULL)
207 		return (NULL);
208 	if (s)
209 		memmove(as->s + as->length, p, s);
210 	as->length += s;
211 	as->s[as->length] = 0;
212 	return (as);
213 }
214 
215 static struct archive_wstring *
216 archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s)
217 {
218 	if (archive_wstring_ensure(as, as->length + s + 1) == NULL)
219 		return (NULL);
220 	if (s)
221 		wmemmove(as->s + as->length, p, s);
222 	as->length += s;
223 	as->s[as->length] = 0;
224 	return (as);
225 }
226 
227 struct archive_string *
228 archive_array_append(struct archive_string *as, const char *p, size_t s)
229 {
230 	return archive_string_append(as, p, s);
231 }
232 
233 void
234 archive_string_concat(struct archive_string *dest, struct archive_string *src)
235 {
236 	if (archive_string_append(dest, src->s, src->length) == NULL)
237 		__archive_errx(1, "Out of memory");
238 }
239 
240 void
241 archive_wstring_concat(struct archive_wstring *dest,
242     struct archive_wstring *src)
243 {
244 	if (archive_wstring_append(dest, src->s, src->length) == NULL)
245 		__archive_errx(1, "Out of memory");
246 }
247 
248 void
249 archive_string_free(struct archive_string *as)
250 {
251 	as->length = 0;
252 	as->buffer_length = 0;
253 	free(as->s);
254 	as->s = NULL;
255 }
256 
257 void
258 archive_wstring_free(struct archive_wstring *as)
259 {
260 	as->length = 0;
261 	as->buffer_length = 0;
262 	free(as->s);
263 	as->s = NULL;
264 }
265 
266 struct archive_wstring *
267 archive_wstring_ensure(struct archive_wstring *as, size_t s)
268 {
269 	return (struct archive_wstring *)
270 		archive_string_ensure((struct archive_string *)as,
271 					s * sizeof(wchar_t));
272 }
273 
274 /* Returns NULL on any allocation failure. */
275 struct archive_string *
276 archive_string_ensure(struct archive_string *as, size_t s)
277 {
278 	char *p;
279 	size_t new_length;
280 
281 	/* If buffer is already big enough, don't reallocate. */
282 	if (as->s && (s <= as->buffer_length))
283 		return (as);
284 
285 	/*
286 	 * Growing the buffer at least exponentially ensures that
287 	 * append operations are always linear in the number of
288 	 * characters appended.  Using a smaller growth rate for
289 	 * larger buffers reduces memory waste somewhat at the cost of
290 	 * a larger constant factor.
291 	 */
292 	if (as->buffer_length < 32)
293 		/* Start with a minimum 32-character buffer. */
294 		new_length = 32;
295 	else if (as->buffer_length < 8192)
296 		/* Buffers under 8k are doubled for speed. */
297 		new_length = as->buffer_length + as->buffer_length;
298 	else {
299 		/* Buffers 8k and over grow by at least 25% each time. */
300 		new_length = as->buffer_length + as->buffer_length / 4;
301 		/* Be safe: If size wraps, fail. */
302 		if (new_length < as->buffer_length) {
303 			/* On failure, wipe the string and return NULL. */
304 			archive_string_free(as);
305 			errno = ENOMEM;/* Make sure errno has ENOMEM. */
306 			return (NULL);
307 		}
308 	}
309 	/*
310 	 * The computation above is a lower limit to how much we'll
311 	 * grow the buffer.  In any case, we have to grow it enough to
312 	 * hold the request.
313 	 */
314 	if (new_length < s)
315 		new_length = s;
316 	/* Now we can reallocate the buffer. */
317 	p = (char *)realloc(as->s, new_length);
318 	if (p == NULL) {
319 		/* On failure, wipe the string and return NULL. */
320 		archive_string_free(as);
321 		errno = ENOMEM;/* Make sure errno has ENOMEM. */
322 		return (NULL);
323 	}
324 
325 	as->s = p;
326 	as->buffer_length = new_length;
327 	return (as);
328 }
329 
330 /*
331  * TODO: See if there's a way to avoid scanning
332  * the source string twice.  Then test to see
333  * if it actually helps (remember that we're almost
334  * always called with pretty short arguments, so
335  * such an optimization might not help).
336  */
337 struct archive_string *
338 archive_strncat(struct archive_string *as, const void *_p, size_t n)
339 {
340 	size_t s;
341 	const char *p, *pp;
342 
343 	p = (const char *)_p;
344 
345 	/* Like strlen(p), except won't examine positions beyond p[n]. */
346 	s = 0;
347 	pp = p;
348 	while (s < n && *pp) {
349 		pp++;
350 		s++;
351 	}
352 	if ((as = archive_string_append(as, p, s)) == NULL)
353 		__archive_errx(1, "Out of memory");
354 	return (as);
355 }
356 
357 struct archive_wstring *
358 archive_wstrncat(struct archive_wstring *as, const wchar_t *p, size_t n)
359 {
360 	size_t s;
361 	const wchar_t *pp;
362 
363 	/* Like strlen(p), except won't examine positions beyond p[n]. */
364 	s = 0;
365 	pp = p;
366 	while (s < n && *pp) {
367 		pp++;
368 		s++;
369 	}
370 	if ((as = archive_wstring_append(as, p, s)) == NULL)
371 		__archive_errx(1, "Out of memory");
372 	return (as);
373 }
374 
375 struct archive_string *
376 archive_strcat(struct archive_string *as, const void *p)
377 {
378 	/* strcat is just strncat without an effective limit.
379 	 * Assert that we'll never get called with a source
380 	 * string over 16MB.
381 	 * TODO: Review all uses of strcat in the source
382 	 * and try to replace them with strncat().
383 	 */
384 	return archive_strncat(as, p, 0x1000000);
385 }
386 
387 struct archive_wstring *
388 archive_wstrcat(struct archive_wstring *as, const wchar_t *p)
389 {
390 	/* Ditto. */
391 	return archive_wstrncat(as, p, 0x1000000);
392 }
393 
394 struct archive_string *
395 archive_strappend_char(struct archive_string *as, char c)
396 {
397 	if ((as = archive_string_append(as, &c, 1)) == NULL)
398 		__archive_errx(1, "Out of memory");
399 	return (as);
400 }
401 
402 struct archive_wstring *
403 archive_wstrappend_wchar(struct archive_wstring *as, wchar_t c)
404 {
405 	if ((as = archive_wstring_append(as, &c, 1)) == NULL)
406 		__archive_errx(1, "Out of memory");
407 	return (as);
408 }
409 
410 /*
411  * Get the "current character set" name to use with iconv.
412  * On FreeBSD, the empty character set name "" chooses
413  * the correct character encoding for the current locale,
414  * so this isn't necessary.
415  * But iconv on Mac OS 10.6 doesn't seem to handle this correctly;
416  * on that system, we have to explicitly call nl_langinfo()
417  * to get the right name.  Not sure about other platforms.
418  *
419  * NOTE: GNU libiconv does not recognize the character-set name
420  * which some platform nl_langinfo(CODESET) returns, so we should
421  * use locale_charset() instead of nl_langinfo(CODESET) for GNU libiconv.
422  */
423 static const char *
424 default_iconv_charset(const char *charset) {
425 	if (charset != NULL && charset[0] != '\0')
426 		return charset;
427 #if HAVE_LOCALE_CHARSET && !defined(__APPLE__)
428 	/* locale_charset() is broken on Mac OS */
429 	return locale_charset();
430 #elif HAVE_NL_LANGINFO
431 	return nl_langinfo(CODESET);
432 #else
433 	return "";
434 #endif
435 }
436 
437 #if defined(_WIN32) && !defined(__CYGWIN__)
438 
439 /*
440  * Convert MBS to WCS.
441  * Note: returns -1 if conversion fails.
442  */
443 int
444 archive_wstring_append_from_mbs(struct archive_wstring *dest,
445     const char *p, size_t len)
446 {
447 	return archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL);
448 }
449 
450 static int
451 archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
452     const char *s, size_t length, struct archive_string_conv *sc)
453 {
454 	int count, ret = 0;
455 	UINT from_cp;
456 
457 	if (sc != NULL)
458 		from_cp = sc->from_cp;
459 	else
460 		from_cp = get_current_codepage();
461 
462 	if (from_cp == CP_C_LOCALE) {
463 		/*
464 		 * "C" locale special processing.
465 		 */
466 		wchar_t *ws;
467 		const unsigned char *mp;
468 
469 		if (NULL == archive_wstring_ensure(dest,
470 		    dest->length + length + 1))
471 			return (-1);
472 
473 		ws = dest->s + dest->length;
474 		mp = (const unsigned char *)s;
475 		count = 0;
476 		while (count < (int)length && *mp) {
477 			*ws++ = (wchar_t)*mp++;
478 			count++;
479 		}
480 	} else if (sc != NULL &&
481 	    (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) {
482 		/*
483 		 * Normalize UTF-8 and UTF-16BE and convert it directly
484 		 * to UTF-16 as wchar_t.
485 		 */
486 		struct archive_string u16;
487 		int saved_flag = sc->flag;/* save current flag. */
488 
489 		if (is_big_endian())
490 			sc->flag |= SCONV_TO_UTF16BE;
491 		else
492 			sc->flag |= SCONV_TO_UTF16LE;
493 
494 		if (sc->flag & SCONV_FROM_UTF16) {
495 			/*
496 			 *  UTF-16BE/LE NFD ===> UTF-16 NFC
497 			 *  UTF-16BE/LE NFC ===> UTF-16 NFD
498 			 */
499 			count = (int)utf16nbytes(s, length);
500 		} else {
501 			/*
502 			 *  UTF-8 NFD ===> UTF-16 NFC
503 			 *  UTF-8 NFC ===> UTF-16 NFD
504 			 */
505 			count = (int)mbsnbytes(s, length);
506 		}
507 		u16.s = (char *)dest->s;
508 		u16.length = dest->length << 1;;
509 		u16.buffer_length = dest->buffer_length;
510 		if (sc->flag & SCONV_NORMALIZATION_C)
511 			ret = archive_string_normalize_C(&u16, s, count, sc);
512 		else
513 			ret = archive_string_normalize_D(&u16, s, count, sc);
514 		dest->s = (wchar_t *)u16.s;
515 		dest->length = u16.length >> 1;
516 		dest->buffer_length = u16.buffer_length;
517 		sc->flag = saved_flag;/* restore the saved flag. */
518 		return (ret);
519 	} else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) {
520 		count = (int)utf16nbytes(s, length);
521 		count >>= 1; /* to be WCS length */
522 		/* Allocate memory for WCS. */
523 		if (NULL == archive_wstring_ensure(dest,
524 		    dest->length + count + 1))
525 			return (-1);
526 		wmemcpy(dest->s + dest->length, (const wchar_t *)s, count);
527 		if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) {
528 			uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
529 			int b;
530 			for (b = 0; b < count; b++) {
531 				uint16_t val = archive_le16dec(u16+b);
532 				archive_be16enc(u16+b, val);
533 			}
534 		} else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) {
535 			uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
536 			int b;
537 			for (b = 0; b < count; b++) {
538 				uint16_t val = archive_be16dec(u16+b);
539 				archive_le16enc(u16+b, val);
540 			}
541 		}
542 	} else {
543 		DWORD mbflag;
544 		size_t buffsize;
545 
546 		if (sc == NULL)
547 			mbflag = 0;
548 		else if (sc->flag & SCONV_FROM_CHARSET) {
549 			/* Do not trust the length which comes from
550 			 * an archive file. */
551 			length = mbsnbytes(s, length);
552 			mbflag = 0;
553 		} else
554 			mbflag = MB_PRECOMPOSED;
555 
556 		buffsize = dest->length + length + 1;
557 		do {
558 			/* Allocate memory for WCS. */
559 			if (NULL == archive_wstring_ensure(dest, buffsize))
560 				return (-1);
561 			/* Convert MBS to WCS. */
562 			count = MultiByteToWideChar(from_cp,
563 			    mbflag, s, (int)length, dest->s + dest->length,
564 			    (int)(dest->buffer_length >> 1) -1);
565 			if (count == 0 &&
566 			    GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
567 				/* Expand the WCS buffer. */
568 				buffsize = dest->buffer_length << 1;
569 				continue;
570 			}
571 			if (count == 0 && length != 0)
572 				ret = -1;
573 			break;
574 		} while (1);
575 	}
576 	dest->length += count;
577 	dest->s[dest->length] = L'\0';
578 	return (ret);
579 }
580 
581 #else
582 
583 /*
584  * Convert MBS to WCS.
585  * Note: returns -1 if conversion fails.
586  */
587 int
588 archive_wstring_append_from_mbs(struct archive_wstring *dest,
589     const char *p, size_t len)
590 {
591 	size_t r;
592 	int ret_val = 0;
593 	/*
594 	 * No single byte will be more than one wide character,
595 	 * so this length estimate will always be big enough.
596 	 */
597 	// size_t wcs_length = len;
598 	size_t mbs_length = len;
599 	const char *mbs = p;
600 	wchar_t *wcs;
601 #if HAVE_MBRTOWC
602 	mbstate_t shift_state;
603 
604 	memset(&shift_state, 0, sizeof(shift_state));
605 #endif
606 	/*
607 	 * As we decided to have wcs_length == mbs_length == len
608 	 * we can use len here instead of wcs_length
609 	 */
610 	if (NULL == archive_wstring_ensure(dest, dest->length + len + 1))
611 		return (-1);
612 	wcs = dest->s + dest->length;
613 	/*
614 	 * We cannot use mbsrtowcs/mbstowcs here because those may convert
615 	 * extra MBS when strlen(p) > len and one wide character consists of
616 	 * multi bytes.
617 	 */
618 	while (*mbs && mbs_length > 0) {
619 		/*
620 		 * The buffer we allocated is always big enough.
621 		 * Keep this code path in a comment if we decide to choose
622 		 * smaller wcs_length in the future
623 		 */
624 /*
625 		if (wcs_length == 0) {
626 			dest->length = wcs - dest->s;
627 			dest->s[dest->length] = L'\0';
628 			wcs_length = mbs_length;
629 			if (NULL == archive_wstring_ensure(dest,
630 			    dest->length + wcs_length + 1))
631 				return (-1);
632 			wcs = dest->s + dest->length;
633 		}
634 */
635 #if HAVE_MBRTOWC
636 		r = mbrtowc(wcs, mbs, mbs_length, &shift_state);
637 #else
638 		r = mbtowc(wcs, mbs, mbs_length);
639 #endif
640 		if (r == (size_t)-1 || r == (size_t)-2) {
641 			ret_val = -1;
642 			break;
643 		}
644 		if (r == 0 || r > mbs_length)
645 			break;
646 		wcs++;
647 		// wcs_length--;
648 		mbs += r;
649 		mbs_length -= r;
650 	}
651 	dest->length = wcs - dest->s;
652 	dest->s[dest->length] = L'\0';
653 	return (ret_val);
654 }
655 
656 #endif
657 
658 #if defined(_WIN32) && !defined(__CYGWIN__)
659 
660 /*
661  * WCS ==> MBS.
662  * Note: returns -1 if conversion fails.
663  *
664  * Win32 builds use WideCharToMultiByte from the Windows API.
665  * (Maybe Cygwin should too?  WideCharToMultiByte will know a
666  * lot more about local character encodings than the wcrtomb()
667  * wrapper is going to know.)
668  */
669 int
670 archive_string_append_from_wcs(struct archive_string *as,
671     const wchar_t *w, size_t len)
672 {
673 	return archive_string_append_from_wcs_in_codepage(as, w, len, NULL);
674 }
675 
676 static int
677 archive_string_append_from_wcs_in_codepage(struct archive_string *as,
678     const wchar_t *ws, size_t len, struct archive_string_conv *sc)
679 {
680 	BOOL defchar_used, *dp;
681 	int count, ret = 0;
682 	UINT to_cp;
683 	int wslen = (int)len;
684 
685 	if (sc != NULL)
686 		to_cp = sc->to_cp;
687 	else
688 		to_cp = get_current_codepage();
689 
690 	if (to_cp == CP_C_LOCALE) {
691 		/*
692 		 * "C" locale special processing.
693 		 */
694 		const wchar_t *wp = ws;
695 		char *p;
696 
697 		if (NULL == archive_string_ensure(as,
698 		    as->length + wslen +1))
699 			return (-1);
700 		p = as->s + as->length;
701 		count = 0;
702 		defchar_used = 0;
703 		while (count < wslen && *wp) {
704 			if (*wp > 255) {
705 				*p++ = '?';
706 				wp++;
707 				defchar_used = 1;
708 			} else
709 				*p++ = (char)*wp++;
710 			count++;
711 		}
712 	} else if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) {
713 		uint16_t *u16;
714 
715 		if (NULL ==
716 		    archive_string_ensure(as, as->length + len * 2 + 2))
717 			return (-1);
718 		u16 = (uint16_t *)(as->s + as->length);
719 		count = 0;
720 		defchar_used = 0;
721 		if (sc->flag & SCONV_TO_UTF16BE) {
722 			while (count < (int)len && *ws) {
723 				archive_be16enc(u16+count, *ws);
724 				ws++;
725 				count++;
726 			}
727 		} else {
728 			while (count < (int)len && *ws) {
729 				archive_le16enc(u16+count, *ws);
730 				ws++;
731 				count++;
732 			}
733 		}
734 		count <<= 1; /* to be byte size */
735 	} else {
736 		/* Make sure the MBS buffer has plenty to set. */
737 		if (NULL ==
738 		    archive_string_ensure(as, as->length + len * 2 + 1))
739 			return (-1);
740 		do {
741 			defchar_used = 0;
742 			if (to_cp == CP_UTF8 || sc == NULL)
743 				dp = NULL;
744 			else
745 				dp = &defchar_used;
746 			count = WideCharToMultiByte(to_cp, 0, ws, wslen,
747 			    as->s + as->length,
748 			    (int)as->buffer_length - (int)as->length - 1, NULL, dp);
749 			if (count == 0 &&
750 			    GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
751 				/* Expand the MBS buffer and retry. */
752 				if (NULL == archive_string_ensure(as,
753 					as->buffer_length + len))
754 					return (-1);
755 				continue;
756 			}
757 			if (count == 0)
758 				ret = -1;
759 			break;
760 		} while (1);
761 	}
762 	as->length += count;
763 	as->s[as->length] = '\0';
764 	return (defchar_used?-1:ret);
765 }
766 
767 #elif defined(HAVE_WCTOMB) || defined(HAVE_WCRTOMB)
768 
769 /*
770  * Translates a wide character string into current locale character set
771  * and appends to the archive_string.  Note: returns -1 if conversion
772  * fails.
773  */
774 int
775 archive_string_append_from_wcs(struct archive_string *as,
776     const wchar_t *w, size_t len)
777 {
778 	/* We cannot use the standard wcstombs() here because it
779 	 * cannot tell us how big the output buffer should be.  So
780 	 * I've built a loop around wcrtomb() or wctomb() that
781 	 * converts a character at a time and resizes the string as
782 	 * needed.  We prefer wcrtomb() when it's available because
783 	 * it's thread-safe. */
784 	int n, ret_val = 0;
785 	char *p;
786 	char *end;
787 #if HAVE_WCRTOMB
788 	mbstate_t shift_state;
789 
790 	memset(&shift_state, 0, sizeof(shift_state));
791 #else
792 	/* Clear the shift state before starting. */
793 	wctomb(NULL, L'\0');
794 #endif
795 	/*
796 	 * Allocate buffer for MBS.
797 	 * We need this allocation here since it is possible that
798 	 * as->s is still NULL.
799 	 */
800 	if (archive_string_ensure(as, as->length + len + 1) == NULL)
801 		return (-1);
802 
803 	p = as->s + as->length;
804 	end = as->s + as->buffer_length - MB_CUR_MAX -1;
805 	while (*w != L'\0' && len > 0) {
806 		if (p >= end) {
807 			as->length = p - as->s;
808 			as->s[as->length] = '\0';
809 			/* Re-allocate buffer for MBS. */
810 			if (archive_string_ensure(as,
811 			    as->length + max(len * 2,
812 			    (size_t)MB_CUR_MAX) + 1) == NULL)
813 				return (-1);
814 			p = as->s + as->length;
815 			end = as->s + as->buffer_length - MB_CUR_MAX -1;
816 		}
817 #if HAVE_WCRTOMB
818 		n = wcrtomb(p, *w++, &shift_state);
819 #else
820 		n = wctomb(p, *w++);
821 #endif
822 		if (n == -1) {
823 			if (errno == EILSEQ) {
824 				/* Skip an illegal wide char. */
825 				*p++ = '?';
826 				ret_val = -1;
827 			} else {
828 				ret_val = -1;
829 				break;
830 			}
831 		} else
832 			p += n;
833 		len--;
834 	}
835 	as->length = p - as->s;
836 	as->s[as->length] = '\0';
837 	return (ret_val);
838 }
839 
840 #else /* HAVE_WCTOMB || HAVE_WCRTOMB */
841 
842 /*
843  * TODO: Test if __STDC_ISO_10646__ is defined.
844  * Non-Windows uses ISO C wcrtomb() or wctomb() to perform the conversion
845  * one character at a time.  If a non-Windows platform doesn't have
846  * either of these, fall back to the built-in UTF8 conversion.
847  */
848 int
849 archive_string_append_from_wcs(struct archive_string *as,
850     const wchar_t *w, size_t len)
851 {
852 	(void)as;/* UNUSED */
853 	(void)w;/* UNUSED */
854 	(void)len;/* UNUSED */
855 	errno = ENOSYS;
856 	return (-1);
857 }
858 
859 #endif /* HAVE_WCTOMB || HAVE_WCRTOMB */
860 
861 /*
862  * Find a string conversion object by a pair of 'from' charset name
863  * and 'to' charset name from an archive object.
864  * Return NULL if not found.
865  */
866 static struct archive_string_conv *
867 find_sconv_object(struct archive *a, const char *fc, const char *tc)
868 {
869 	struct archive_string_conv *sc;
870 
871 	if (a == NULL)
872 		return (NULL);
873 
874 	for (sc = a->sconv; sc != NULL; sc = sc->next) {
875 		if (strcmp(sc->from_charset, fc) == 0 &&
876 		    strcmp(sc->to_charset, tc) == 0)
877 			break;
878 	}
879 	return (sc);
880 }
881 
882 /*
883  * Register a string object to an archive object.
884  */
885 static void
886 add_sconv_object(struct archive *a, struct archive_string_conv *sc)
887 {
888 	struct archive_string_conv **psc;
889 
890 	/* Add a new sconv to sconv list. */
891 	psc = &(a->sconv);
892 	while (*psc != NULL)
893 		psc = &((*psc)->next);
894 	*psc = sc;
895 }
896 
897 static void
898 add_converter(struct archive_string_conv *sc, int (*converter)
899     (struct archive_string *, const void *, size_t,
900      struct archive_string_conv *))
901 {
902 	if (sc == NULL || sc->nconverter >= 2)
903 		__archive_errx(1, "Programming error");
904 	sc->converter[sc->nconverter++] = converter;
905 }
906 
907 static void
908 setup_converter(struct archive_string_conv *sc)
909 {
910 
911 	/* Reset. */
912 	sc->nconverter = 0;
913 
914 	/*
915 	 * Perform special sequence for the incorrect UTF-8 filenames
916 	 * made by libarchive2.x.
917 	 */
918 	if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) {
919 		add_converter(sc, strncat_from_utf8_libarchive2);
920 		return;
921 	}
922 
923 	/*
924 	 * Convert a string to UTF-16BE/LE.
925 	 */
926 	if (sc->flag & SCONV_TO_UTF16) {
927 		/*
928 		 * If the current locale is UTF-8, we can translate
929 		 * a UTF-8 string into a UTF-16BE string.
930 		 */
931 		if (sc->flag & SCONV_FROM_UTF8) {
932 			add_converter(sc, archive_string_append_unicode);
933 			return;
934 		}
935 
936 #if defined(_WIN32) && !defined(__CYGWIN__)
937 		if (sc->flag & SCONV_WIN_CP) {
938 			if (sc->flag & SCONV_TO_UTF16BE)
939 				add_converter(sc, win_strncat_to_utf16be);
940 			else
941 				add_converter(sc, win_strncat_to_utf16le);
942 			return;
943 		}
944 #endif
945 
946 #if defined(HAVE_ICONV)
947 		if (sc->cd != (iconv_t)-1) {
948 			add_converter(sc, iconv_strncat_in_locale);
949 			return;
950 		}
951 #endif
952 
953 		if (sc->flag & SCONV_BEST_EFFORT) {
954 			if (sc->flag & SCONV_TO_UTF16BE)
955 				add_converter(sc,
956 					best_effort_strncat_to_utf16be);
957 			else
958 				add_converter(sc,
959 					best_effort_strncat_to_utf16le);
960 		} else
961 			/* Make sure we have no converter. */
962 			sc->nconverter = 0;
963 		return;
964 	}
965 
966 	/*
967 	 * Convert a string from UTF-16BE/LE.
968 	 */
969 	if (sc->flag & SCONV_FROM_UTF16) {
970 		/*
971 		 * At least we should normalize a UTF-16BE string.
972 		 */
973 		if (sc->flag & SCONV_NORMALIZATION_D)
974 			add_converter(sc,archive_string_normalize_D);
975 		else if (sc->flag & SCONV_NORMALIZATION_C)
976 			add_converter(sc, archive_string_normalize_C);
977 
978 		if (sc->flag & SCONV_TO_UTF8) {
979 			/*
980 			 * If the current locale is UTF-8, we can translate
981 			 * a UTF-16BE/LE string into a UTF-8 string directly.
982 			 */
983 			if (!(sc->flag &
984 			    (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C)))
985 				add_converter(sc,
986 				    archive_string_append_unicode);
987 			return;
988 		}
989 
990 #if defined(_WIN32) && !defined(__CYGWIN__)
991 		if (sc->flag & SCONV_WIN_CP) {
992 			if (sc->flag & SCONV_FROM_UTF16BE)
993 				add_converter(sc, win_strncat_from_utf16be);
994 			else
995 				add_converter(sc, win_strncat_from_utf16le);
996 			return;
997 		}
998 #endif
999 
1000 #if defined(HAVE_ICONV)
1001 		if (sc->cd != (iconv_t)-1) {
1002 			add_converter(sc, iconv_strncat_in_locale);
1003 			return;
1004 		}
1005 #endif
1006 
1007 		if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE))
1008 		    == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE))
1009 			add_converter(sc, best_effort_strncat_from_utf16be);
1010 		else if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE))
1011 		    == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE))
1012 			add_converter(sc, best_effort_strncat_from_utf16le);
1013 		else
1014 			/* Make sure we have no converter. */
1015 			sc->nconverter = 0;
1016 		return;
1017 	}
1018 
1019 	if (sc->flag & SCONV_FROM_UTF8) {
1020 		/*
1021 		 * At least we should normalize a UTF-8 string.
1022 		 */
1023 		if (sc->flag & SCONV_NORMALIZATION_D)
1024 			add_converter(sc,archive_string_normalize_D);
1025 		else if (sc->flag & SCONV_NORMALIZATION_C)
1026 			add_converter(sc, archive_string_normalize_C);
1027 
1028 		/*
1029 		 * Copy UTF-8 string with a check of CESU-8.
1030 		 * Apparently, iconv does not check surrogate pairs in UTF-8
1031 		 * when both from-charset and to-charset are UTF-8, and then
1032 		 * we use our UTF-8 copy code.
1033 		 */
1034 		if (sc->flag & SCONV_TO_UTF8) {
1035 			/*
1036 			 * If the current locale is UTF-8, we can translate
1037 			 * a UTF-16BE string into a UTF-8 string directly.
1038 			 */
1039 			if (!(sc->flag &
1040 			    (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C)))
1041 				add_converter(sc, strncat_from_utf8_to_utf8);
1042 			return;
1043 		}
1044 	}
1045 
1046 #if defined(_WIN32) && !defined(__CYGWIN__)
1047 	/*
1048 	 * On Windows we can use Windows API for a string conversion.
1049 	 */
1050 	if (sc->flag & SCONV_WIN_CP) {
1051 		add_converter(sc, strncat_in_codepage);
1052 		return;
1053 	}
1054 #endif
1055 
1056 #if HAVE_ICONV
1057 	if (sc->cd != (iconv_t)-1) {
1058 		add_converter(sc, iconv_strncat_in_locale);
1059 		/*
1060 		 * iconv generally does not support UTF-8-MAC and so
1061 		 * we have to the output of iconv from NFC to NFD if
1062 		 * need.
1063 		 */
1064 		if ((sc->flag & SCONV_FROM_CHARSET) &&
1065 		    (sc->flag & SCONV_TO_UTF8)) {
1066 			if (sc->flag & SCONV_NORMALIZATION_D)
1067 				add_converter(sc, archive_string_normalize_D);
1068 		}
1069 		return;
1070 	}
1071 #endif
1072 
1073 	/*
1074 	 * Try conversion in the best effort or no conversion.
1075 	 */
1076 	if ((sc->flag & SCONV_BEST_EFFORT) || sc->same)
1077 		add_converter(sc, best_effort_strncat_in_locale);
1078 	else
1079 		/* Make sure we have no converter. */
1080 		sc->nconverter = 0;
1081 }
1082 
1083 /*
1084  * Return canonicalized charset-name but this supports just UTF-8, UTF-16BE
1085  * and CP932 which are referenced in create_sconv_object().
1086  */
1087 static const char *
1088 canonical_charset_name(const char *charset)
1089 {
1090 	char cs[16];
1091 	char *p;
1092 	const char *s;
1093 
1094 	if (charset == NULL || charset[0] == '\0'
1095 	    || strlen(charset) > 15)
1096 		return (charset);
1097 
1098 	/* Copy name to uppercase. */
1099 	p = cs;
1100 	s = charset;
1101 	while (*s) {
1102 		char c = *s++;
1103 		if (c >= 'a' && c <= 'z')
1104 			c -= 'a' - 'A';
1105 		*p++ = c;
1106 	}
1107 	*p++ = '\0';
1108 
1109 	if (strcmp(cs, "UTF-8") == 0 ||
1110 	    strcmp(cs, "UTF8") == 0)
1111 		return ("UTF-8");
1112 	if (strcmp(cs, "UTF-16BE") == 0 ||
1113 	    strcmp(cs, "UTF16BE") == 0)
1114 		return ("UTF-16BE");
1115 	if (strcmp(cs, "UTF-16LE") == 0 ||
1116 	    strcmp(cs, "UTF16LE") == 0)
1117 		return ("UTF-16LE");
1118 	if (strcmp(cs, "CP932") == 0)
1119 		return ("CP932");
1120 	return (charset);
1121 }
1122 
1123 /*
1124  * Create a string conversion object.
1125  */
1126 static struct archive_string_conv *
1127 create_sconv_object(const char *fc, const char *tc,
1128     unsigned current_codepage, int flag)
1129 {
1130 	struct archive_string_conv *sc;
1131 
1132 	sc = calloc(1, sizeof(*sc));
1133 	if (sc == NULL)
1134 		return (NULL);
1135 	sc->next = NULL;
1136 	sc->from_charset = strdup(fc);
1137 	if (sc->from_charset == NULL) {
1138 		free(sc);
1139 		return (NULL);
1140 	}
1141 	sc->to_charset = strdup(tc);
1142 	if (sc->to_charset == NULL) {
1143 		free(sc->from_charset);
1144 		free(sc);
1145 		return (NULL);
1146 	}
1147 	archive_string_init(&sc->utftmp);
1148 
1149 	if (flag & SCONV_TO_CHARSET) {
1150 		/*
1151 		 * Convert characters from the current locale charset to
1152 		 * a specified charset.
1153 		 */
1154 		sc->from_cp = current_codepage;
1155 		sc->to_cp = make_codepage_from_charset(tc);
1156 #if defined(_WIN32) && !defined(__CYGWIN__)
1157 		if (IsValidCodePage(sc->to_cp))
1158 			flag |= SCONV_WIN_CP;
1159 #endif
1160 	} else if (flag & SCONV_FROM_CHARSET) {
1161 		/*
1162 		 * Convert characters from a specified charset to
1163 		 * the current locale charset.
1164 		 */
1165 		sc->to_cp = current_codepage;
1166 		sc->from_cp = make_codepage_from_charset(fc);
1167 #if defined(_WIN32) && !defined(__CYGWIN__)
1168 		if (IsValidCodePage(sc->from_cp))
1169 			flag |= SCONV_WIN_CP;
1170 #endif
1171 	}
1172 
1173 	/*
1174 	 * Check if "from charset" and "to charset" are the same.
1175 	 */
1176 	if (strcmp(fc, tc) == 0 ||
1177 	    (sc->from_cp != (unsigned)-1 && sc->from_cp == sc->to_cp))
1178 		sc->same = 1;
1179 	else
1180 		sc->same = 0;
1181 
1182 	/*
1183 	 * Mark if "from charset" or "to charset" are UTF-8 or UTF-16BE/LE.
1184 	 */
1185 	if (strcmp(tc, "UTF-8") == 0)
1186 		flag |= SCONV_TO_UTF8;
1187 	else if (strcmp(tc, "UTF-16BE") == 0)
1188 		flag |= SCONV_TO_UTF16BE;
1189 	else if (strcmp(tc, "UTF-16LE") == 0)
1190 		flag |= SCONV_TO_UTF16LE;
1191 	if (strcmp(fc, "UTF-8") == 0)
1192 		flag |= SCONV_FROM_UTF8;
1193 	else if (strcmp(fc, "UTF-16BE") == 0)
1194 		flag |= SCONV_FROM_UTF16BE;
1195 	else if (strcmp(fc, "UTF-16LE") == 0)
1196 		flag |= SCONV_FROM_UTF16LE;
1197 #if defined(_WIN32) && !defined(__CYGWIN__)
1198 	if (sc->to_cp == CP_UTF8)
1199 		flag |= SCONV_TO_UTF8;
1200 	else if (sc->to_cp == CP_UTF16BE)
1201 		flag |= SCONV_TO_UTF16BE | SCONV_WIN_CP;
1202 	else if (sc->to_cp == CP_UTF16LE)
1203 		flag |= SCONV_TO_UTF16LE | SCONV_WIN_CP;
1204 	if (sc->from_cp == CP_UTF8)
1205 		flag |= SCONV_FROM_UTF8;
1206 	else if (sc->from_cp == CP_UTF16BE)
1207 		flag |= SCONV_FROM_UTF16BE | SCONV_WIN_CP;
1208 	else if (sc->from_cp == CP_UTF16LE)
1209 		flag |= SCONV_FROM_UTF16LE | SCONV_WIN_CP;
1210 #endif
1211 
1212 	/*
1213 	 * Set a flag for Unicode NFD. Usually iconv cannot correctly
1214 	 * handle it. So we have to translate NFD characters to NFC ones
1215 	 * ourselves before iconv handles. Another reason is to prevent
1216 	 * that the same sight of two filenames, one is NFC and other
1217 	 * is NFD, would be in its directory.
1218 	 * On Mac OS X, although its filesystem layer automatically
1219 	 * convert filenames to NFD, it would be useful for filename
1220 	 * comparing to find out the same filenames that we normalize
1221 	 * that to be NFD ourselves.
1222 	 */
1223 	if ((flag & SCONV_FROM_CHARSET) &&
1224 	    (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8))) {
1225 #if defined(__APPLE__)
1226 		if (flag & SCONV_TO_UTF8)
1227 			flag |= SCONV_NORMALIZATION_D;
1228 		else
1229 #endif
1230 			flag |= SCONV_NORMALIZATION_C;
1231 	}
1232 #if defined(__APPLE__)
1233 	/*
1234 	 * In case writing an archive file, make sure that a filename
1235 	 * going to be passed to iconv is a Unicode NFC string since
1236 	 * a filename in HFS Plus filesystem is a Unicode NFD one and
1237 	 * iconv cannot handle it with "UTF-8" charset. It is simpler
1238 	 * than a use of "UTF-8-MAC" charset.
1239 	 */
1240 	if ((flag & SCONV_TO_CHARSET) &&
1241 	    (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) &&
1242 	    !(flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8)))
1243 		flag |= SCONV_NORMALIZATION_C;
1244 	/*
1245 	 * In case reading an archive file. make sure that a filename
1246 	 * will be passed to users is a Unicode NFD string in order to
1247 	 * correctly compare the filename with other one which comes
1248 	 * from HFS Plus filesystem.
1249 	 */
1250 	if ((flag & SCONV_FROM_CHARSET) &&
1251 	   !(flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) &&
1252 	    (flag & SCONV_TO_UTF8))
1253 		flag |= SCONV_NORMALIZATION_D;
1254 #endif
1255 
1256 #if defined(HAVE_ICONV)
1257 	sc->cd_w = (iconv_t)-1;
1258 	/*
1259 	 * Create an iconv object.
1260 	 */
1261 	if (((flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) &&
1262 	    (flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16))) ||
1263 	    (flag & SCONV_WIN_CP)) {
1264 		/* This case we won't use iconv. */
1265 		sc->cd = (iconv_t)-1;
1266 	} else {
1267 		sc->cd = iconv_open(tc, fc);
1268 		if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT)) {
1269 			/*
1270 			 * Unfortunately, all of iconv implements do support
1271 			 * "CP932" character-set, so we should use "SJIS"
1272 			 * instead if iconv_open failed.
1273 			 */
1274 			if (strcmp(tc, "CP932") == 0)
1275 				sc->cd = iconv_open("SJIS", fc);
1276 			else if (strcmp(fc, "CP932") == 0)
1277 				sc->cd = iconv_open(tc, "SJIS");
1278 		}
1279 #if defined(_WIN32) && !defined(__CYGWIN__)
1280 		/*
1281 		 * archive_mstring on Windows directly convert multi-bytes
1282 		 * into archive_wstring in order not to depend on locale
1283 		 * so that you can do a I18N programming. This will be
1284 		 * used only in archive_mstring_copy_mbs_len_l so far.
1285 		 */
1286 		if (flag & SCONV_FROM_CHARSET) {
1287 			sc->cd_w = iconv_open("UTF-8", fc);
1288 			if (sc->cd_w == (iconv_t)-1 &&
1289 			    (sc->flag & SCONV_BEST_EFFORT)) {
1290 				if (strcmp(fc, "CP932") == 0)
1291 					sc->cd_w = iconv_open("UTF-8", "SJIS");
1292 			}
1293 		}
1294 #endif /* _WIN32 && !__CYGWIN__ */
1295 	}
1296 #endif	/* HAVE_ICONV */
1297 
1298 	sc->flag = flag;
1299 
1300 	/*
1301 	 * Set up converters.
1302 	 */
1303 	setup_converter(sc);
1304 
1305 	return (sc);
1306 }
1307 
1308 /*
1309  * Free a string conversion object.
1310  */
1311 static void
1312 free_sconv_object(struct archive_string_conv *sc)
1313 {
1314 	free(sc->from_charset);
1315 	free(sc->to_charset);
1316 	archive_string_free(&sc->utftmp);
1317 #if HAVE_ICONV
1318 	if (sc->cd != (iconv_t)-1)
1319 		iconv_close(sc->cd);
1320 	if (sc->cd_w != (iconv_t)-1)
1321 		iconv_close(sc->cd_w);
1322 #endif
1323 	free(sc);
1324 }
1325 
1326 #if defined(_WIN32) && !defined(__CYGWIN__)
1327 # if defined(WINAPI_FAMILY_PARTITION) && !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
1328 #  define GetOEMCP() CP_OEMCP
1329 # endif
1330 
1331 static unsigned
1332 my_atoi(const char *p)
1333 {
1334 	unsigned cp;
1335 
1336 	cp = 0;
1337 	while (*p) {
1338 		if (*p >= '0' && *p <= '9')
1339 			cp = cp * 10 + (*p - '0');
1340 		else
1341 			return (-1);
1342 		p++;
1343 	}
1344 	return (cp);
1345 }
1346 
1347 /*
1348  * Translate Charset name (as used by iconv) into CodePage (as used by Windows)
1349  * Return -1 if failed.
1350  *
1351  * Note: This translation code may be insufficient.
1352  */
1353 static struct charset {
1354 	const char *name;
1355 	unsigned cp;
1356 } charsets[] = {
1357 	/* MUST BE SORTED! */
1358 	{"ASCII", 1252},
1359 	{"ASMO-708", 708},
1360 	{"BIG5", 950},
1361 	{"CHINESE", 936},
1362 	{"CP367", 1252},
1363 	{"CP819", 1252},
1364 	{"CP1025", 21025},
1365 	{"DOS-720", 720},
1366 	{"DOS-862", 862},
1367 	{"EUC-CN", 51936},
1368 	{"EUC-JP", 51932},
1369 	{"EUC-KR", 949},
1370 	{"EUCCN", 51936},
1371 	{"EUCJP", 51932},
1372 	{"EUCKR", 949},
1373 	{"GB18030", 54936},
1374 	{"GB2312", 936},
1375 	{"HEBREW", 1255},
1376 	{"HZ-GB-2312", 52936},
1377 	{"IBM273", 20273},
1378 	{"IBM277", 20277},
1379 	{"IBM278", 20278},
1380 	{"IBM280", 20280},
1381 	{"IBM284", 20284},
1382 	{"IBM285", 20285},
1383 	{"IBM290", 20290},
1384 	{"IBM297", 20297},
1385 	{"IBM367", 1252},
1386 	{"IBM420", 20420},
1387 	{"IBM423", 20423},
1388 	{"IBM424", 20424},
1389 	{"IBM819", 1252},
1390 	{"IBM871", 20871},
1391 	{"IBM880", 20880},
1392 	{"IBM905", 20905},
1393 	{"IBM924", 20924},
1394 	{"ISO-8859-1", 28591},
1395 	{"ISO-8859-13", 28603},
1396 	{"ISO-8859-15", 28605},
1397 	{"ISO-8859-2", 28592},
1398 	{"ISO-8859-3", 28593},
1399 	{"ISO-8859-4", 28594},
1400 	{"ISO-8859-5", 28595},
1401 	{"ISO-8859-6", 28596},
1402 	{"ISO-8859-7", 28597},
1403 	{"ISO-8859-8", 28598},
1404 	{"ISO-8859-9", 28599},
1405 	{"ISO8859-1", 28591},
1406 	{"ISO8859-13", 28603},
1407 	{"ISO8859-15", 28605},
1408 	{"ISO8859-2", 28592},
1409 	{"ISO8859-3", 28593},
1410 	{"ISO8859-4", 28594},
1411 	{"ISO8859-5", 28595},
1412 	{"ISO8859-6", 28596},
1413 	{"ISO8859-7", 28597},
1414 	{"ISO8859-8", 28598},
1415 	{"ISO8859-9", 28599},
1416 	{"JOHAB", 1361},
1417 	{"KOI8-R", 20866},
1418 	{"KOI8-U", 21866},
1419 	{"KS_C_5601-1987", 949},
1420 	{"LATIN1", 1252},
1421 	{"LATIN2", 28592},
1422 	{"MACINTOSH", 10000},
1423 	{"SHIFT-JIS", 932},
1424 	{"SHIFT_JIS", 932},
1425 	{"SJIS", 932},
1426 	{"US", 1252},
1427 	{"US-ASCII", 1252},
1428 	{"UTF-16", 1200},
1429 	{"UTF-16BE", 1201},
1430 	{"UTF-16LE", 1200},
1431 	{"UTF-8", CP_UTF8},
1432 	{"X-EUROPA", 29001},
1433 	{"X-MAC-ARABIC", 10004},
1434 	{"X-MAC-CE", 10029},
1435 	{"X-MAC-CHINESEIMP", 10008},
1436 	{"X-MAC-CHINESETRAD", 10002},
1437 	{"X-MAC-CROATIAN", 10082},
1438 	{"X-MAC-CYRILLIC", 10007},
1439 	{"X-MAC-GREEK", 10006},
1440 	{"X-MAC-HEBREW", 10005},
1441 	{"X-MAC-ICELANDIC", 10079},
1442 	{"X-MAC-JAPANESE", 10001},
1443 	{"X-MAC-KOREAN", 10003},
1444 	{"X-MAC-ROMANIAN", 10010},
1445 	{"X-MAC-THAI", 10021},
1446 	{"X-MAC-TURKISH", 10081},
1447 	{"X-MAC-UKRAINIAN", 10017},
1448 };
1449 static unsigned
1450 make_codepage_from_charset(const char *charset)
1451 {
1452 	char cs[16];
1453 	char *p;
1454 	unsigned cp;
1455 	int a, b;
1456 
1457 	if (charset == NULL || strlen(charset) > 15)
1458 		return -1;
1459 
1460 	/* Copy name to uppercase. */
1461 	p = cs;
1462 	while (*charset) {
1463 		char c = *charset++;
1464 		if (c >= 'a' && c <= 'z')
1465 			c -= 'a' - 'A';
1466 		*p++ = c;
1467 	}
1468 	*p++ = '\0';
1469 	cp = -1;
1470 
1471 	/* Look it up in the table first, so that we can easily
1472 	 * override CP367, which we map to 1252 instead of 367. */
1473 	a = 0;
1474 	b = sizeof(charsets)/sizeof(charsets[0]);
1475 	while (b > a) {
1476 		int c = (b + a) / 2;
1477 		int r = strcmp(charsets[c].name, cs);
1478 		if (r < 0)
1479 			a = c + 1;
1480 		else if (r > 0)
1481 			b = c;
1482 		else
1483 			return charsets[c].cp;
1484 	}
1485 
1486 	/* If it's not in the table, try to parse it. */
1487 	switch (*cs) {
1488 	case 'C':
1489 		if (cs[1] == 'P' && cs[2] >= '0' && cs[2] <= '9') {
1490 			cp = my_atoi(cs + 2);
1491 		} else if (strcmp(cs, "CP_ACP") == 0)
1492 			cp = get_current_codepage();
1493 		else if (strcmp(cs, "CP_OEMCP") == 0)
1494 			cp = get_current_oemcp();
1495 		break;
1496 	case 'I':
1497 		if (cs[1] == 'B' && cs[2] == 'M' &&
1498 		    cs[3] >= '0' && cs[3] <= '9') {
1499 			cp = my_atoi(cs + 3);
1500 		}
1501 		break;
1502 	case 'W':
1503 		if (strncmp(cs, "WINDOWS-", 8) == 0) {
1504 			cp = my_atoi(cs + 8);
1505 			if (cp != 874 && (cp < 1250 || cp > 1258))
1506 				cp = -1;/* This may invalid code. */
1507 		}
1508 		break;
1509 	}
1510 	return (cp);
1511 }
1512 
1513 /*
1514  * Return ANSI Code Page of current locale set by setlocale().
1515  */
1516 static unsigned
1517 get_current_codepage(void)
1518 {
1519 	char *locale, *p;
1520 	unsigned cp;
1521 
1522 	locale = setlocale(LC_CTYPE, NULL);
1523 	if (locale == NULL)
1524 		return (GetACP());
1525 	if (locale[0] == 'C' && locale[1] == '\0')
1526 		return (CP_C_LOCALE);
1527 	p = strrchr(locale, '.');
1528 	if (p == NULL)
1529 		return (GetACP());
1530 	if (strcmp(p+1, "utf8") == 0)
1531 		return CP_UTF8;
1532 	cp = my_atoi(p+1);
1533 	if ((int)cp <= 0)
1534 		return (GetACP());
1535 	return (cp);
1536 }
1537 
1538 /*
1539  * Translation table between Locale Name and ACP/OEMCP.
1540  */
1541 static struct {
1542 	unsigned acp;
1543 	unsigned ocp;
1544 	const char *locale;
1545 } acp_ocp_map[] = {
1546 	{  950,  950, "Chinese_Taiwan" },
1547 	{  936,  936, "Chinese_People's Republic of China" },
1548 	{  950,  950, "Chinese_Taiwan" },
1549 	{ 1250,  852, "Czech_Czech Republic" },
1550 	{ 1252,  850, "Danish_Denmark" },
1551 	{ 1252,  850, "Dutch_Netherlands" },
1552 	{ 1252,  850, "Dutch_Belgium" },
1553 	{ 1252,  437, "English_United States" },
1554 	{ 1252,  850, "English_Australia" },
1555 	{ 1252,  850, "English_Canada" },
1556 	{ 1252,  850, "English_New Zealand" },
1557 	{ 1252,  850, "English_United Kingdom" },
1558 	{ 1252,  437, "English_United States" },
1559 	{ 1252,  850, "Finnish_Finland" },
1560 	{ 1252,  850, "French_France" },
1561 	{ 1252,  850, "French_Belgium" },
1562 	{ 1252,  850, "French_Canada" },
1563 	{ 1252,  850, "French_Switzerland" },
1564 	{ 1252,  850, "German_Germany" },
1565 	{ 1252,  850, "German_Austria" },
1566 	{ 1252,  850, "German_Switzerland" },
1567 	{ 1253,  737, "Greek_Greece" },
1568 	{ 1250,  852, "Hungarian_Hungary" },
1569 	{ 1252,  850, "Icelandic_Iceland" },
1570 	{ 1252,  850, "Italian_Italy" },
1571 	{ 1252,  850, "Italian_Switzerland" },
1572 	{  932,  932, "Japanese_Japan" },
1573 	{  949,  949, "Korean_Korea" },
1574 	{ 1252,  850, "Norwegian (BokmOl)_Norway" },
1575 	{ 1252,  850, "Norwegian (BokmOl)_Norway" },
1576 	{ 1252,  850, "Norwegian-Nynorsk_Norway" },
1577 	{ 1250,  852, "Polish_Poland" },
1578 	{ 1252,  850, "Portuguese_Portugal" },
1579 	{ 1252,  850, "Portuguese_Brazil" },
1580 	{ 1251,  866, "Russian_Russia" },
1581 	{ 1250,  852, "Slovak_Slovakia" },
1582 	{ 1252,  850, "Spanish_Spain" },
1583 	{ 1252,  850, "Spanish_Mexico" },
1584 	{ 1252,  850, "Spanish_Spain" },
1585 	{ 1252,  850, "Swedish_Sweden" },
1586 	{ 1254,  857, "Turkish_Turkey" },
1587 	{ 0, 0, NULL}
1588 };
1589 
1590 /*
1591  * Return OEM Code Page of current locale set by setlocale().
1592  */
1593 static unsigned
1594 get_current_oemcp(void)
1595 {
1596 	int i;
1597 	char *locale, *p;
1598 	size_t len;
1599 
1600 	locale = setlocale(LC_CTYPE, NULL);
1601 	if (locale == NULL)
1602 		return (GetOEMCP());
1603 	if (locale[0] == 'C' && locale[1] == '\0')
1604 		return (CP_C_LOCALE);
1605 
1606 	p = strrchr(locale, '.');
1607 	if (p == NULL)
1608 		return (GetOEMCP());
1609 	len = p - locale;
1610 	for (i = 0; acp_ocp_map[i].acp; i++) {
1611 		if (strncmp(acp_ocp_map[i].locale, locale, len) == 0)
1612 			return (acp_ocp_map[i].ocp);
1613 	}
1614 	return (GetOEMCP());
1615 }
1616 #else
1617 
1618 /*
1619  * POSIX platform does not use CodePage.
1620  */
1621 
1622 static unsigned
1623 get_current_codepage(void)
1624 {
1625 	return (-1);/* Unknown */
1626 }
1627 static unsigned
1628 make_codepage_from_charset(const char *charset)
1629 {
1630 	(void)charset; /* UNUSED */
1631 	return (-1);/* Unknown */
1632 }
1633 static unsigned
1634 get_current_oemcp(void)
1635 {
1636 	return (-1);/* Unknown */
1637 }
1638 
1639 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */
1640 
1641 /*
1642  * Return a string conversion object.
1643  */
1644 static struct archive_string_conv *
1645 get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag)
1646 {
1647 	struct archive_string_conv *sc;
1648 	unsigned current_codepage;
1649 
1650 	/* Check if we have made the sconv object. */
1651 	sc = find_sconv_object(a, fc, tc);
1652 	if (sc != NULL)
1653 		return (sc);
1654 
1655 	if (a == NULL)
1656 		current_codepage = get_current_codepage();
1657 	else
1658 		current_codepage = a->current_codepage;
1659 
1660 	sc = create_sconv_object(canonical_charset_name(fc),
1661 	    canonical_charset_name(tc), current_codepage, flag);
1662 	if (sc == NULL) {
1663 		if (a != NULL)
1664 			archive_set_error(a, ENOMEM,
1665 			    "Could not allocate memory for "
1666 			    "a string conversion object");
1667 		return (NULL);
1668 	}
1669 
1670 	/*
1671 	 * If there is no converter for current string conversion object,
1672 	 * we cannot handle this conversion.
1673 	 */
1674 	if (sc->nconverter == 0) {
1675 		if (a != NULL) {
1676 #if HAVE_ICONV
1677 			archive_set_error(a, ARCHIVE_ERRNO_MISC,
1678 			    "iconv_open failed : Cannot handle ``%s''",
1679 			    (flag & SCONV_TO_CHARSET)?tc:fc);
1680 #else
1681 			archive_set_error(a, ARCHIVE_ERRNO_MISC,
1682 			    "A character-set conversion not fully supported "
1683 			    "on this platform");
1684 #endif
1685 		}
1686 		/* Failed; free a sconv object. */
1687 		free_sconv_object(sc);
1688 		return (NULL);
1689 	}
1690 
1691 	/*
1692 	 * Success!
1693 	 */
1694 	if (a != NULL)
1695 		add_sconv_object(a, sc);
1696 	return (sc);
1697 }
1698 
1699 static const char *
1700 get_current_charset(struct archive *a)
1701 {
1702 	const char *cur_charset;
1703 
1704 	if (a == NULL)
1705 		cur_charset = default_iconv_charset("");
1706 	else {
1707 		cur_charset = default_iconv_charset(a->current_code);
1708 		if (a->current_code == NULL) {
1709 			a->current_code = strdup(cur_charset);
1710 			a->current_codepage = get_current_codepage();
1711 			a->current_oemcp = get_current_oemcp();
1712 		}
1713 	}
1714 	return (cur_charset);
1715 }
1716 
1717 /*
1718  * Make and Return a string conversion object.
1719  * Return NULL if the platform does not support the specified conversion
1720  * and best_effort is 0.
1721  * If best_effort is set, A string conversion object must be returned
1722  * unless memory allocation for the object fails, but the conversion
1723  * might fail when non-ASCII code is found.
1724  */
1725 struct archive_string_conv *
1726 archive_string_conversion_to_charset(struct archive *a, const char *charset,
1727     int best_effort)
1728 {
1729 	int flag = SCONV_TO_CHARSET;
1730 
1731 	if (best_effort)
1732 		flag |= SCONV_BEST_EFFORT;
1733 	return (get_sconv_object(a, get_current_charset(a), charset, flag));
1734 }
1735 
1736 struct archive_string_conv *
1737 archive_string_conversion_from_charset(struct archive *a, const char *charset,
1738     int best_effort)
1739 {
1740 	int flag = SCONV_FROM_CHARSET;
1741 
1742 	if (best_effort)
1743 		flag |= SCONV_BEST_EFFORT;
1744 	return (get_sconv_object(a, charset, get_current_charset(a), flag));
1745 }
1746 
1747 /*
1748  * archive_string_default_conversion_*_archive() are provided for Windows
1749  * platform because other archiver application use CP_OEMCP for
1750  * MultiByteToWideChar() and WideCharToMultiByte() for the filenames
1751  * in tar or zip files. But mbstowcs/wcstombs(CRT) usually use CP_ACP
1752  * unless you use setlocale(LC_ALL, ".OCP")(specify CP_OEMCP).
1753  * So we should make a string conversion between CP_ACP and CP_OEMCP
1754  * for compatibility.
1755  */
1756 #if defined(_WIN32) && !defined(__CYGWIN__)
1757 struct archive_string_conv *
1758 archive_string_default_conversion_for_read(struct archive *a)
1759 {
1760 	const char *cur_charset = get_current_charset(a);
1761 	char oemcp[16];
1762 
1763 	/* NOTE: a check of cur_charset is unneeded but we need
1764 	 * that get_current_charset() has been surely called at
1765 	 * this time whatever C compiler optimized. */
1766 	if (cur_charset != NULL &&
1767 	    (a->current_codepage == CP_C_LOCALE ||
1768 	     a->current_codepage == a->current_oemcp))
1769 		return (NULL);/* no conversion. */
1770 
1771 	_snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp);
1772 	/* Make sure a null termination must be set. */
1773 	oemcp[sizeof(oemcp)-1] = '\0';
1774 	return (get_sconv_object(a, oemcp, cur_charset,
1775 	    SCONV_FROM_CHARSET));
1776 }
1777 
1778 struct archive_string_conv *
1779 archive_string_default_conversion_for_write(struct archive *a)
1780 {
1781 	const char *cur_charset = get_current_charset(a);
1782 	char oemcp[16];
1783 
1784 	/* NOTE: a check of cur_charset is unneeded but we need
1785 	 * that get_current_charset() has been surely called at
1786 	 * this time whatever C compiler optimized. */
1787 	if (cur_charset != NULL &&
1788 	    (a->current_codepage == CP_C_LOCALE ||
1789 	     a->current_codepage == a->current_oemcp))
1790 		return (NULL);/* no conversion. */
1791 
1792 	_snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp);
1793 	/* Make sure a null termination must be set. */
1794 	oemcp[sizeof(oemcp)-1] = '\0';
1795 	return (get_sconv_object(a, cur_charset, oemcp,
1796 	    SCONV_TO_CHARSET));
1797 }
1798 #else
1799 struct archive_string_conv *
1800 archive_string_default_conversion_for_read(struct archive *a)
1801 {
1802 	(void)a; /* UNUSED */
1803 	return (NULL);
1804 }
1805 
1806 struct archive_string_conv *
1807 archive_string_default_conversion_for_write(struct archive *a)
1808 {
1809 	(void)a; /* UNUSED */
1810 	return (NULL);
1811 }
1812 #endif
1813 
1814 /*
1815  * Dispose of all character conversion objects in the archive object.
1816  */
1817 void
1818 archive_string_conversion_free(struct archive *a)
1819 {
1820 	struct archive_string_conv *sc;
1821 	struct archive_string_conv *sc_next;
1822 
1823 	for (sc = a->sconv; sc != NULL; sc = sc_next) {
1824 		sc_next = sc->next;
1825 		free_sconv_object(sc);
1826 	}
1827 	a->sconv = NULL;
1828 	free(a->current_code);
1829 	a->current_code = NULL;
1830 }
1831 
1832 /*
1833  * Return a conversion charset name.
1834  */
1835 const char *
1836 archive_string_conversion_charset_name(struct archive_string_conv *sc)
1837 {
1838 	if (sc->flag & SCONV_TO_CHARSET)
1839 		return (sc->to_charset);
1840 	else
1841 		return (sc->from_charset);
1842 }
1843 
1844 /*
1845  * Change the behavior of a string conversion.
1846  */
1847 void
1848 archive_string_conversion_set_opt(struct archive_string_conv *sc, int opt)
1849 {
1850 	switch (opt) {
1851 	/*
1852 	 * A filename in UTF-8 was made with libarchive 2.x in a wrong
1853 	 * assumption that wchar_t was Unicode.
1854 	 * This option enables simulating the assumption in order to read
1855 	 * that filename correctly.
1856 	 */
1857 	case SCONV_SET_OPT_UTF8_LIBARCHIVE2X:
1858 #if (defined(_WIN32) && !defined(__CYGWIN__)) \
1859 	 || defined(__STDC_ISO_10646__) || defined(__APPLE__)
1860 		/*
1861 		 * Nothing to do for it since wchar_t on these platforms
1862 		 * is really Unicode.
1863 		 */
1864 		(void)sc; /* UNUSED */
1865 #else
1866 		if ((sc->flag & SCONV_UTF8_LIBARCHIVE_2) == 0) {
1867 			sc->flag |= SCONV_UTF8_LIBARCHIVE_2;
1868 			/* Set up string converters. */
1869 			setup_converter(sc);
1870 		}
1871 #endif
1872 		break;
1873 	case SCONV_SET_OPT_NORMALIZATION_C:
1874 		if ((sc->flag & SCONV_NORMALIZATION_C) == 0) {
1875 			sc->flag |= SCONV_NORMALIZATION_C;
1876 			sc->flag &= ~SCONV_NORMALIZATION_D;
1877 			/* Set up string converters. */
1878 			setup_converter(sc);
1879 		}
1880 		break;
1881 	case SCONV_SET_OPT_NORMALIZATION_D:
1882 #if defined(HAVE_ICONV)
1883 		/*
1884 		 * If iconv will take the string, do not change the
1885 		 * setting of the normalization.
1886 		 */
1887 		if (!(sc->flag & SCONV_WIN_CP) &&
1888 		     (sc->flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) &&
1889 		    !(sc->flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8)))
1890 			break;
1891 #endif
1892 		if ((sc->flag & SCONV_NORMALIZATION_D) == 0) {
1893 			sc->flag |= SCONV_NORMALIZATION_D;
1894 			sc->flag &= ~SCONV_NORMALIZATION_C;
1895 			/* Set up string converters. */
1896 			setup_converter(sc);
1897 		}
1898 		break;
1899 	default:
1900 		break;
1901 	}
1902 }
1903 
1904 /*
1905  *
1906  * Copy one archive_string to another in locale conversion.
1907  *
1908  *	archive_strncat_l();
1909  *	archive_strncpy_l();
1910  *
1911  */
1912 
1913 static size_t
1914 mbsnbytes(const void *_p, size_t n)
1915 {
1916 	size_t s;
1917 	const char *p, *pp;
1918 
1919 	if (_p == NULL)
1920 		return (0);
1921 	p = (const char *)_p;
1922 
1923 	/* Like strlen(p), except won't examine positions beyond p[n]. */
1924 	s = 0;
1925 	pp = p;
1926 	while (s < n && *pp) {
1927 		pp++;
1928 		s++;
1929 	}
1930 	return (s);
1931 }
1932 
1933 static size_t
1934 utf16nbytes(const void *_p, size_t n)
1935 {
1936 	size_t s;
1937 	const char *p, *pp;
1938 
1939 	if (_p == NULL)
1940 		return (0);
1941 	p = (const char *)_p;
1942 
1943 	/* Like strlen(p), except won't examine positions beyond p[n]. */
1944 	s = 0;
1945 	pp = p;
1946 	n >>= 1;
1947 	while (s < n && (pp[0] || pp[1])) {
1948 		pp += 2;
1949 		s++;
1950 	}
1951 	return (s<<1);
1952 }
1953 
1954 int
1955 archive_strncpy_l(struct archive_string *as, const void *_p, size_t n,
1956     struct archive_string_conv *sc)
1957 {
1958 	as->length = 0;
1959 	return (archive_strncat_l(as, _p, n, sc));
1960 }
1961 
1962 int
1963 archive_strncat_l(struct archive_string *as, const void *_p, size_t n,
1964     struct archive_string_conv *sc)
1965 {
1966 	const void *s;
1967 	size_t length = 0;
1968 	int i, r = 0, r2;
1969 
1970 	if (_p != NULL && n > 0) {
1971 		if (sc != NULL && (sc->flag & SCONV_FROM_UTF16))
1972 			length = utf16nbytes(_p, n);
1973 		else
1974 			length = mbsnbytes(_p, n);
1975 	}
1976 
1977 	/* We must allocate memory even if there is no data for conversion
1978 	 * or copy. This simulates archive_string_append behavior. */
1979 	if (length == 0) {
1980 		int tn = 1;
1981 		if (sc != NULL && (sc->flag & SCONV_TO_UTF16))
1982 			tn = 2;
1983 		if (archive_string_ensure(as, as->length + tn) == NULL)
1984 			return (-1);
1985 		as->s[as->length] = 0;
1986 		if (tn == 2)
1987 			as->s[as->length+1] = 0;
1988 		return (0);
1989 	}
1990 
1991 	/*
1992 	 * If sc is NULL, we just make a copy.
1993 	 */
1994 	if (sc == NULL) {
1995 		if (archive_string_append(as, _p, length) == NULL)
1996 			return (-1);/* No memory */
1997 		return (0);
1998 	}
1999 
2000 	s = _p;
2001 	i = 0;
2002 	if (sc->nconverter > 1) {
2003 		sc->utftmp.length = 0;
2004 		r2 = sc->converter[0](&(sc->utftmp), s, length, sc);
2005 		if (r2 != 0 && errno == ENOMEM)
2006 			return (r2);
2007 		if (r > r2)
2008 			r = r2;
2009 		s = sc->utftmp.s;
2010 		length = sc->utftmp.length;
2011 		++i;
2012 	}
2013 	r2 = sc->converter[i](as, s, length, sc);
2014 	if (r > r2)
2015 		r = r2;
2016 	return (r);
2017 }
2018 
2019 #if HAVE_ICONV
2020 
2021 /*
2022  * Return -1 if conversion fails.
2023  */
2024 static int
2025 iconv_strncat_in_locale(struct archive_string *as, const void *_p,
2026     size_t length, struct archive_string_conv *sc)
2027 {
2028 	ICONV_CONST char *itp;
2029 	size_t remaining;
2030 	iconv_t cd;
2031 	char *outp;
2032 	size_t avail, bs;
2033 	int return_value = 0; /* success */
2034 	int to_size, from_size;
2035 
2036 	if (sc->flag & SCONV_TO_UTF16)
2037 		to_size = 2;
2038 	else
2039 		to_size = 1;
2040 	if (sc->flag & SCONV_FROM_UTF16)
2041 		from_size = 2;
2042 	else
2043 		from_size = 1;
2044 
2045 	if (archive_string_ensure(as, as->length + length*2+to_size) == NULL)
2046 		return (-1);
2047 
2048 	cd = sc->cd;
2049 	itp = (char *)(uintptr_t)_p;
2050 	remaining = length;
2051 	outp = as->s + as->length;
2052 	avail = as->buffer_length - as->length - to_size;
2053 	while (remaining >= (size_t)from_size) {
2054 		size_t result = iconv(cd, &itp, &remaining, &outp, &avail);
2055 
2056 		if (result != (size_t)-1)
2057 			break; /* Conversion completed. */
2058 
2059 		if (errno == EILSEQ || errno == EINVAL) {
2060 			/*
2061 		 	 * If an output charset is UTF-8 or UTF-16BE/LE,
2062 			 * unknown character should be U+FFFD
2063 			 * (replacement character).
2064 			 */
2065 			if (sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) {
2066 				size_t rbytes;
2067 				if (sc->flag & SCONV_TO_UTF8)
2068 					rbytes = sizeof(utf8_replacement_char);
2069 				else
2070 					rbytes = 2;
2071 
2072 				if (avail < rbytes) {
2073 					as->length = outp - as->s;
2074 					bs = as->buffer_length +
2075 					    (remaining * to_size) + rbytes;
2076 					if (NULL ==
2077 					    archive_string_ensure(as, bs))
2078 						return (-1);
2079 					outp = as->s + as->length;
2080 					avail = as->buffer_length
2081 					    - as->length - to_size;
2082 				}
2083 				if (sc->flag & SCONV_TO_UTF8)
2084 					memcpy(outp, utf8_replacement_char, sizeof(utf8_replacement_char));
2085 				else if (sc->flag & SCONV_TO_UTF16BE)
2086 					archive_be16enc(outp, UNICODE_R_CHAR);
2087 				else
2088 					archive_le16enc(outp, UNICODE_R_CHAR);
2089 				outp += rbytes;
2090 				avail -= rbytes;
2091 			} else {
2092 				/* Skip the illegal input bytes. */
2093 				*outp++ = '?';
2094 				avail--;
2095 			}
2096 			itp += from_size;
2097 			remaining -= from_size;
2098 			return_value = -1; /* failure */
2099 		} else {
2100 			/* E2BIG no output buffer,
2101 			 * Increase an output buffer.  */
2102 			as->length = outp - as->s;
2103 			bs = as->buffer_length + remaining * 2;
2104 			if (NULL == archive_string_ensure(as, bs))
2105 				return (-1);
2106 			outp = as->s + as->length;
2107 			avail = as->buffer_length - as->length - to_size;
2108 		}
2109 	}
2110 	as->length = outp - as->s;
2111 	as->s[as->length] = 0;
2112 	if (to_size == 2)
2113 		as->s[as->length+1] = 0;
2114 	return (return_value);
2115 }
2116 
2117 #endif /* HAVE_ICONV */
2118 
2119 
2120 #if defined(_WIN32) && !defined(__CYGWIN__)
2121 
2122 /*
2123  * Translate a string from a some CodePage to an another CodePage by
2124  * Windows APIs, and copy the result. Return -1 if conversion fails.
2125  */
2126 static int
2127 strncat_in_codepage(struct archive_string *as,
2128     const void *_p, size_t length, struct archive_string_conv *sc)
2129 {
2130 	const char *s = (const char *)_p;
2131 	struct archive_wstring aws;
2132 	size_t l;
2133 	int r, saved_flag;
2134 
2135 	archive_string_init(&aws);
2136 	saved_flag = sc->flag;
2137 	sc->flag &= ~(SCONV_NORMALIZATION_D | SCONV_NORMALIZATION_C);
2138 	r = archive_wstring_append_from_mbs_in_codepage(&aws, s, length, sc);
2139 	sc->flag = saved_flag;
2140 	if (r != 0) {
2141 		archive_wstring_free(&aws);
2142 		if (errno != ENOMEM)
2143 			archive_string_append(as, s, length);
2144 		return (-1);
2145 	}
2146 
2147 	l = as->length;
2148 	r = archive_string_append_from_wcs_in_codepage(
2149 	    as, aws.s, aws.length, sc);
2150 	if (r != 0 && errno != ENOMEM && l == as->length)
2151 		archive_string_append(as, s, length);
2152 	archive_wstring_free(&aws);
2153 	return (r);
2154 }
2155 
2156 /*
2157  * Test whether MBS ==> WCS is okay.
2158  */
2159 static int
2160 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc)
2161 {
2162 	const char *p = (const char *)_p;
2163 	unsigned codepage;
2164 	DWORD mbflag = MB_ERR_INVALID_CHARS;
2165 
2166 	if (sc->flag & SCONV_FROM_CHARSET)
2167 		codepage = sc->to_cp;
2168 	else
2169 		codepage = sc->from_cp;
2170 
2171 	if (codepage == CP_C_LOCALE)
2172 		return (0);
2173 	if (codepage != CP_UTF8)
2174 		mbflag |= MB_PRECOMPOSED;
2175 
2176 	if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0)
2177 		return (-1); /* Invalid */
2178 	return (0); /* Okay */
2179 }
2180 
2181 #else
2182 
2183 /*
2184  * Test whether MBS ==> WCS is okay.
2185  */
2186 static int
2187 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc)
2188 {
2189 	const char *p = (const char *)_p;
2190 	size_t r;
2191 
2192 #if HAVE_MBRTOWC
2193 	mbstate_t shift_state;
2194 
2195 	memset(&shift_state, 0, sizeof(shift_state));
2196 #else
2197 	/* Clear the shift state before starting. */
2198 	mbtowc(NULL, NULL, 0);
2199 #endif
2200 	while (n) {
2201 		wchar_t wc;
2202 
2203 #if HAVE_MBRTOWC
2204 		r = mbrtowc(&wc, p, n, &shift_state);
2205 #else
2206 		r = mbtowc(&wc, p, n);
2207 #endif
2208 		if (r == (size_t)-1 || r == (size_t)-2)
2209 			return (-1);/* Invalid. */
2210 		if (r == 0)
2211 			break;
2212 		p += r;
2213 		n -= r;
2214 	}
2215 	(void)sc; /* UNUSED */
2216 	return (0); /* All Okey. */
2217 }
2218 
2219 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */
2220 
2221 /*
2222  * Basically returns -1 because we cannot make a conversion of charset
2223  * without iconv but in some cases this would return 0.
2224  * Returns 0 if all copied characters are ASCII.
2225  * Returns 0 if both from-locale and to-locale are the same and those
2226  * can be WCS with no error.
2227  */
2228 static int
2229 best_effort_strncat_in_locale(struct archive_string *as, const void *_p,
2230     size_t length, struct archive_string_conv *sc)
2231 {
2232 	size_t remaining;
2233 	const uint8_t *itp;
2234 	int return_value = 0; /* success */
2235 
2236 	/*
2237 	 * If both from-locale and to-locale is the same, this makes a copy.
2238 	 * And then this checks all copied MBS can be WCS if so returns 0.
2239 	 */
2240 	if (sc->same) {
2241 		if (archive_string_append(as, _p, length) == NULL)
2242 			return (-1);/* No memory */
2243 		return (invalid_mbs(_p, length, sc));
2244 	}
2245 
2246 	/*
2247 	 * If a character is ASCII, this just copies it. If not, this
2248 	 * assigns '?' character instead but in UTF-8 locale this assigns
2249 	 * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD,
2250 	 * a Replacement Character in Unicode.
2251 	 */
2252 
2253 	remaining = length;
2254 	itp = (const uint8_t *)_p;
2255 	while (*itp && remaining > 0) {
2256 		if (*itp > 127) {
2257 			// Non-ASCII: Substitute with suitable replacement
2258 			if (sc->flag & SCONV_TO_UTF8) {
2259 				if (archive_string_append(as, utf8_replacement_char, sizeof(utf8_replacement_char)) == NULL) {
2260 					__archive_errx(1, "Out of memory");
2261 				}
2262 			} else {
2263 				archive_strappend_char(as, '?');
2264 			}
2265 			return_value = -1;
2266 		} else {
2267 			archive_strappend_char(as, *itp);
2268 		}
2269 		++itp;
2270 	}
2271 	return (return_value);
2272 }
2273 
2274 
2275 /*
2276  * Unicode conversion functions.
2277  *   - UTF-8 <===> UTF-8 in removing surrogate pairs.
2278  *   - UTF-8 NFD ===> UTF-8 NFC in removing surrogate pairs.
2279  *   - UTF-8 made by libarchive 2.x ===> UTF-8.
2280  *   - UTF-16BE <===> UTF-8.
2281  *
2282  */
2283 
2284 /*
2285  * Utility to convert a single UTF-8 sequence.
2286  *
2287  * Usually return used bytes, return used byte in negative value when
2288  * a unicode character is replaced with U+FFFD.
2289  * See also http://unicode.org/review/pr-121.html Public Review Issue #121
2290  * Recommended Practice for Replacement Characters.
2291  */
2292 static int
2293 _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n)
2294 {
2295 	static const char utf8_count[256] = {
2296 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 00 - 0F */
2297 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 10 - 1F */
2298 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 20 - 2F */
2299 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 30 - 3F */
2300 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 40 - 4F */
2301 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 50 - 5F */
2302 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 60 - 6F */
2303 		 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 70 - 7F */
2304 		 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 80 - 8F */
2305 		 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 90 - 9F */
2306 		 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* A0 - AF */
2307 		 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* B0 - BF */
2308 		 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* C0 - CF */
2309 		 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* D0 - DF */
2310 		 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,/* E0 - EF */
2311 		 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0 - FF */
2312 	};
2313 	int ch, i;
2314 	int cnt;
2315 	uint32_t wc;
2316 
2317 	/* Sanity check. */
2318 	if (n == 0)
2319 		return (0);
2320 	/*
2321 	 * Decode 1-4 bytes depending on the value of the first byte.
2322 	 */
2323 	ch = (unsigned char)*s;
2324 	if (ch == 0)
2325 		return (0); /* Standard:  return 0 for end-of-string. */
2326 	cnt = utf8_count[ch];
2327 
2328 	/* Invalid sequence or there are not plenty bytes. */
2329 	if ((int)n < cnt) {
2330 		cnt = (int)n;
2331 		for (i = 1; i < cnt; i++) {
2332 			if ((s[i] & 0xc0) != 0x80) {
2333 				cnt = i;
2334 				break;
2335 			}
2336 		}
2337 		goto invalid_sequence;
2338 	}
2339 
2340 	/* Make a Unicode code point from a single UTF-8 sequence. */
2341 	switch (cnt) {
2342 	case 1:	/* 1 byte sequence. */
2343 		*pwc = ch & 0x7f;
2344 		return (cnt);
2345 	case 2:	/* 2 bytes sequence. */
2346 		if ((s[1] & 0xc0) != 0x80) {
2347 			cnt = 1;
2348 			goto invalid_sequence;
2349 		}
2350 		*pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f);
2351 		return (cnt);
2352 	case 3:	/* 3 bytes sequence. */
2353 		if ((s[1] & 0xc0) != 0x80) {
2354 			cnt = 1;
2355 			goto invalid_sequence;
2356 		}
2357 		if ((s[2] & 0xc0) != 0x80) {
2358 			cnt = 2;
2359 			goto invalid_sequence;
2360 		}
2361 		wc = ((ch & 0x0f) << 12)
2362 		    | ((s[1] & 0x3f) << 6)
2363 		    | (s[2] & 0x3f);
2364 		if (wc < 0x800)
2365 			goto invalid_sequence;/* Overlong sequence. */
2366 		break;
2367 	case 4:	/* 4 bytes sequence. */
2368 		if ((s[1] & 0xc0) != 0x80) {
2369 			cnt = 1;
2370 			goto invalid_sequence;
2371 		}
2372 		if ((s[2] & 0xc0) != 0x80) {
2373 			cnt = 2;
2374 			goto invalid_sequence;
2375 		}
2376 		if ((s[3] & 0xc0) != 0x80) {
2377 			cnt = 3;
2378 			goto invalid_sequence;
2379 		}
2380 		wc = ((ch & 0x07) << 18)
2381 		    | ((s[1] & 0x3f) << 12)
2382 		    | ((s[2] & 0x3f) << 6)
2383 		    | (s[3] & 0x3f);
2384 		if (wc < 0x10000)
2385 			goto invalid_sequence;/* Overlong sequence. */
2386 		break;
2387 	default: /* Others are all invalid sequence. */
2388 		if (ch == 0xc0 || ch == 0xc1)
2389 			cnt = 2;
2390 		else if (ch >= 0xf5 && ch <= 0xf7)
2391 			cnt = 4;
2392 		else if (ch >= 0xf8 && ch <= 0xfb)
2393 			cnt = 5;
2394 		else if (ch == 0xfc || ch == 0xfd)
2395 			cnt = 6;
2396 		else
2397 			cnt = 1;
2398 		if ((int)n < cnt)
2399 			cnt = (int)n;
2400 		for (i = 1; i < cnt; i++) {
2401 			if ((s[i] & 0xc0) != 0x80) {
2402 				cnt = i;
2403 				break;
2404 			}
2405 		}
2406 		goto invalid_sequence;
2407 	}
2408 
2409 	/* The code point larger than 0x10FFFF is not legal
2410 	 * Unicode values. */
2411 	if (wc > UNICODE_MAX)
2412 		goto invalid_sequence;
2413 	/* Correctly gets a Unicode, returns used bytes. */
2414 	*pwc = wc;
2415 	return (cnt);
2416 invalid_sequence:
2417 	*pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */
2418 	return (cnt * -1);
2419 }
2420 
2421 static int
2422 utf8_to_unicode(uint32_t *pwc, const char *s, size_t n)
2423 {
2424 	int cnt;
2425 
2426 	cnt = _utf8_to_unicode(pwc, s, n);
2427 	/* Any of Surrogate pair is not legal Unicode values. */
2428 	if (cnt == 3 && IS_SURROGATE_PAIR_LA(*pwc))
2429 		return (-3);
2430 	return (cnt);
2431 }
2432 
2433 static inline uint32_t
2434 combine_surrogate_pair(uint32_t uc, uint32_t uc2)
2435 {
2436 	uc -= 0xD800;
2437 	uc *= 0x400;
2438 	uc += uc2 - 0xDC00;
2439 	uc += 0x10000;
2440 	return (uc);
2441 }
2442 
2443 /*
2444  * Convert a single UTF-8/CESU-8 sequence to a Unicode code point in
2445  * removing surrogate pairs.
2446  *
2447  * CESU-8: The Compatibility Encoding Scheme for UTF-16.
2448  *
2449  * Usually return used bytes, return used byte in negative value when
2450  * a unicode character is replaced with U+FFFD.
2451  */
2452 static int
2453 cesu8_to_unicode(uint32_t *pwc, const char *s, size_t n)
2454 {
2455 	uint32_t wc = 0;
2456 	int cnt;
2457 
2458 	cnt = _utf8_to_unicode(&wc, s, n);
2459 	if (cnt == 3 && IS_HIGH_SURROGATE_LA(wc)) {
2460 		uint32_t wc2 = 0;
2461 		if (n - 3 < 3) {
2462 			/* Invalid byte sequence. */
2463 			goto invalid_sequence;
2464 		}
2465 		cnt = _utf8_to_unicode(&wc2, s+3, n-3);
2466 		if (cnt != 3 || !IS_LOW_SURROGATE_LA(wc2)) {
2467 			/* Invalid byte sequence. */
2468 			goto invalid_sequence;
2469 		}
2470 		wc = combine_surrogate_pair(wc, wc2);
2471 		cnt = 6;
2472 	} else if (cnt == 3 && IS_LOW_SURROGATE_LA(wc)) {
2473 		/* Invalid byte sequence. */
2474 		goto invalid_sequence;
2475 	}
2476 	*pwc = wc;
2477 	return (cnt);
2478 invalid_sequence:
2479 	*pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */
2480 	if (cnt > 0)
2481 		cnt *= -1;
2482 	return (cnt);
2483 }
2484 
2485 /*
2486  * Convert a Unicode code point to a single UTF-8 sequence.
2487  *
2488  * NOTE:This function does not check if the Unicode is legal or not.
2489  * Please you definitely check it before calling this.
2490  */
2491 static size_t
2492 unicode_to_utf8(char *p, size_t remaining, uint32_t uc)
2493 {
2494 	char *_p = p;
2495 
2496 	/* Invalid Unicode char maps to Replacement character */
2497 	if (uc > UNICODE_MAX)
2498 		uc = UNICODE_R_CHAR;
2499 	/* Translate code point to UTF8 */
2500 	if (uc <= 0x7f) {
2501 		if (remaining == 0)
2502 			return (0);
2503 		*p++ = (char)uc;
2504 	} else if (uc <= 0x7ff) {
2505 		if (remaining < 2)
2506 			return (0);
2507 		*p++ = 0xc0 | ((uc >> 6) & 0x1f);
2508 		*p++ = 0x80 | (uc & 0x3f);
2509 	} else if (uc <= 0xffff) {
2510 		if (remaining < 3)
2511 			return (0);
2512 		*p++ = 0xe0 | ((uc >> 12) & 0x0f);
2513 		*p++ = 0x80 | ((uc >> 6) & 0x3f);
2514 		*p++ = 0x80 | (uc & 0x3f);
2515 	} else {
2516 		if (remaining < 4)
2517 			return (0);
2518 		*p++ = 0xf0 | ((uc >> 18) & 0x07);
2519 		*p++ = 0x80 | ((uc >> 12) & 0x3f);
2520 		*p++ = 0x80 | ((uc >> 6) & 0x3f);
2521 		*p++ = 0x80 | (uc & 0x3f);
2522 	}
2523 	return (p - _p);
2524 }
2525 
2526 static int
2527 utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n)
2528 {
2529 	return (utf16_to_unicode(pwc, s, n, 1));
2530 }
2531 
2532 static int
2533 utf16le_to_unicode(uint32_t *pwc, const char *s, size_t n)
2534 {
2535 	return (utf16_to_unicode(pwc, s, n, 0));
2536 }
2537 
2538 static int
2539 utf16_to_unicode(uint32_t *pwc, const char *s, size_t n, int be)
2540 {
2541 	const char *utf16 = s;
2542 	unsigned uc;
2543 
2544 	if (n == 0)
2545 		return (0);
2546 	if (n == 1) {
2547 		/* set the Replacement Character instead. */
2548 		*pwc = UNICODE_R_CHAR;
2549 		return (-1);
2550 	}
2551 
2552 	if (be)
2553 		uc = archive_be16dec(utf16);
2554 	else
2555 		uc = archive_le16dec(utf16);
2556 	utf16 += 2;
2557 
2558 	/* If this is a surrogate pair, assemble the full code point.*/
2559 	if (IS_HIGH_SURROGATE_LA(uc)) {
2560 		unsigned uc2;
2561 
2562 		if (n >= 4) {
2563 			if (be)
2564 				uc2 = archive_be16dec(utf16);
2565 			else
2566 				uc2 = archive_le16dec(utf16);
2567 		} else
2568 			uc2 = 0;
2569 		if (IS_LOW_SURROGATE_LA(uc2)) {
2570 			uc = combine_surrogate_pair(uc, uc2);
2571 			utf16 += 2;
2572 		} else {
2573 	 		/* Undescribed code point should be U+FFFD
2574 		 	* (replacement character). */
2575 			*pwc = UNICODE_R_CHAR;
2576 			return (-2);
2577 		}
2578 	}
2579 
2580 	/*
2581 	 * Surrogate pair values(0xd800 through 0xdfff) are only
2582 	 * used by UTF-16, so, after above calculation, the code
2583 	 * must not be surrogate values, and Unicode has no codes
2584 	 * larger than 0x10ffff. Thus, those are not legal Unicode
2585 	 * values.
2586 	 */
2587 	if (IS_SURROGATE_PAIR_LA(uc) || uc > UNICODE_MAX) {
2588 	 	/* Undescribed code point should be U+FFFD
2589 	 	* (replacement character). */
2590 		*pwc = UNICODE_R_CHAR;
2591 		return (((int)(utf16 - s)) * -1);
2592 	}
2593 	*pwc = uc;
2594 	return ((int)(utf16 - s));
2595 }
2596 
2597 static size_t
2598 unicode_to_utf16be(char *p, size_t remaining, uint32_t uc)
2599 {
2600 	char *utf16 = p;
2601 
2602 	if (uc > 0xffff) {
2603 		/* We have a code point that won't fit into a
2604 		 * wchar_t; convert it to a surrogate pair. */
2605 		if (remaining < 4)
2606 			return (0);
2607 		uc -= 0x10000;
2608 		archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
2609 		archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
2610 		return (4);
2611 	} else {
2612 		if (remaining < 2)
2613 			return (0);
2614 		archive_be16enc(utf16, uc);
2615 		return (2);
2616 	}
2617 }
2618 
2619 static size_t
2620 unicode_to_utf16le(char *p, size_t remaining, uint32_t uc)
2621 {
2622 	char *utf16 = p;
2623 
2624 	if (uc > 0xffff) {
2625 		/* We have a code point that won't fit into a
2626 		 * wchar_t; convert it to a surrogate pair. */
2627 		if (remaining < 4)
2628 			return (0);
2629 		uc -= 0x10000;
2630 		archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
2631 		archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
2632 		return (4);
2633 	} else {
2634 		if (remaining < 2)
2635 			return (0);
2636 		archive_le16enc(utf16, uc);
2637 		return (2);
2638 	}
2639 }
2640 
2641 /*
2642  * Copy UTF-8 string in checking surrogate pair.
2643  * If any surrogate pair are found, it would be canonicalized.
2644  */
2645 static int
2646 strncat_from_utf8_to_utf8(struct archive_string *as, const void *_p,
2647     size_t len, struct archive_string_conv *sc)
2648 {
2649 	const char *s;
2650 	char *p, *endp;
2651 	int n, ret = 0;
2652 
2653 	(void)sc; /* UNUSED */
2654 
2655 	if (archive_string_ensure(as, as->length + len + 1) == NULL)
2656 		return (-1);
2657 
2658 	s = (const char *)_p;
2659 	p = as->s + as->length;
2660 	endp = as->s + as->buffer_length -1;
2661 	do {
2662 		uint32_t uc;
2663 		const char *ss = s;
2664 		size_t w;
2665 
2666 		/*
2667 		 * Forward byte sequence until a conversion of that is needed.
2668 		 */
2669 		while ((n = utf8_to_unicode(&uc, s, len)) > 0) {
2670 			s += n;
2671 			len -= n;
2672 		}
2673 		if (ss < s) {
2674 			if (p + (s - ss) > endp) {
2675 				as->length = p - as->s;
2676 				if (archive_string_ensure(as,
2677 				    as->buffer_length + len + 1) == NULL)
2678 					return (-1);
2679 				p = as->s + as->length;
2680 				endp = as->s + as->buffer_length -1;
2681 			}
2682 
2683 			memcpy(p, ss, s - ss);
2684 			p += s - ss;
2685 		}
2686 
2687 		/*
2688 		 * If n is negative, current byte sequence needs a replacement.
2689 		 */
2690 		if (n < 0) {
2691 			if (n == -3 && IS_SURROGATE_PAIR_LA(uc)) {
2692 				/* Current byte sequence may be CESU-8. */
2693 				n = cesu8_to_unicode(&uc, s, len);
2694 			}
2695 			if (n < 0) {
2696 				ret = -1;
2697 				n *= -1;/* Use a replaced unicode character. */
2698 			}
2699 
2700 			/* Rebuild UTF-8 byte sequence. */
2701 			while ((w = unicode_to_utf8(p, endp - p, uc)) == 0) {
2702 				as->length = p - as->s;
2703 				if (archive_string_ensure(as,
2704 				    as->buffer_length + len + 1) == NULL)
2705 					return (-1);
2706 				p = as->s + as->length;
2707 				endp = as->s + as->buffer_length -1;
2708 			}
2709 			p += w;
2710 			s += n;
2711 			len -= n;
2712 		}
2713 	} while (n > 0);
2714 	as->length = p - as->s;
2715 	as->s[as->length] = '\0';
2716 	return (ret);
2717 }
2718 
2719 static int
2720 archive_string_append_unicode(struct archive_string *as, const void *_p,
2721     size_t len, struct archive_string_conv *sc)
2722 {
2723 	const char *s;
2724 	char *p, *endp;
2725 	uint32_t uc;
2726 	size_t w;
2727 	int n, ret = 0, ts, tm;
2728 	int (*parse)(uint32_t *, const char *, size_t);
2729 	size_t (*unparse)(char *, size_t, uint32_t);
2730 
2731 	if (sc->flag & SCONV_TO_UTF16BE) {
2732 		unparse = unicode_to_utf16be;
2733 		ts = 2;
2734 	} else if (sc->flag & SCONV_TO_UTF16LE) {
2735 		unparse = unicode_to_utf16le;
2736 		ts = 2;
2737 	} else if (sc->flag & SCONV_TO_UTF8) {
2738 		unparse = unicode_to_utf8;
2739 		ts = 1;
2740 	} else {
2741 		/*
2742 		 * This case is going to be converted to another
2743 		 * character-set through iconv.
2744 		 */
2745 		if (sc->flag & SCONV_FROM_UTF16BE) {
2746 			unparse = unicode_to_utf16be;
2747 			ts = 2;
2748 		} else if (sc->flag & SCONV_FROM_UTF16LE) {
2749 			unparse = unicode_to_utf16le;
2750 			ts = 2;
2751 		} else {
2752 			unparse = unicode_to_utf8;
2753 			ts = 1;
2754 		}
2755 	}
2756 
2757 	if (sc->flag & SCONV_FROM_UTF16BE) {
2758 		parse = utf16be_to_unicode;
2759 		tm = 1;
2760 	} else if (sc->flag & SCONV_FROM_UTF16LE) {
2761 		parse = utf16le_to_unicode;
2762 		tm = 1;
2763 	} else {
2764 		parse = cesu8_to_unicode;
2765 		tm = ts;
2766 	}
2767 
2768 	if (archive_string_ensure(as, as->length + len * tm + ts) == NULL)
2769 		return (-1);
2770 
2771 	s = (const char *)_p;
2772 	p = as->s + as->length;
2773 	endp = as->s + as->buffer_length - ts;
2774 	while ((n = parse(&uc, s, len)) != 0) {
2775 		if (n < 0) {
2776 			/* Use a replaced unicode character. */
2777 			n *= -1;
2778 			ret = -1;
2779 		}
2780 		s += n;
2781 		len -= n;
2782 		while ((w = unparse(p, endp - p, uc)) == 0) {
2783 			/* There is not enough output buffer so
2784 			 * we have to expand it. */
2785 			as->length = p - as->s;
2786 			if (archive_string_ensure(as,
2787 			    as->buffer_length + len * tm + ts) == NULL)
2788 				return (-1);
2789 			p = as->s + as->length;
2790 			endp = as->s + as->buffer_length - ts;
2791 		}
2792 		p += w;
2793 	}
2794 	as->length = p - as->s;
2795 	as->s[as->length] = '\0';
2796 	if (ts == 2)
2797 		as->s[as->length+1] = '\0';
2798 	return (ret);
2799 }
2800 
2801 /*
2802  * Following Constants for Hangul compositions this information comes from
2803  * Unicode Standard Annex #15  http://unicode.org/reports/tr15/
2804  */
2805 #define HC_SBASE	0xAC00
2806 #define HC_LBASE	0x1100
2807 #define HC_VBASE	0x1161
2808 #define HC_TBASE	0x11A7
2809 #define HC_LCOUNT	19
2810 #define HC_VCOUNT	21
2811 #define HC_TCOUNT	28
2812 #define HC_NCOUNT	(HC_VCOUNT * HC_TCOUNT)
2813 #define HC_SCOUNT	(HC_LCOUNT * HC_NCOUNT)
2814 
2815 static uint32_t
2816 get_nfc(uint32_t uc, uint32_t uc2)
2817 {
2818 	int t, b;
2819 
2820 	t = 0;
2821 	b = sizeof(u_composition_table)/sizeof(u_composition_table[0]) -1;
2822 	while (b >= t) {
2823 		int m = (t + b) / 2;
2824 		if (u_composition_table[m].cp1 < uc)
2825 			t = m + 1;
2826 		else if (u_composition_table[m].cp1 > uc)
2827 			b = m - 1;
2828 		else if (u_composition_table[m].cp2 < uc2)
2829 			t = m + 1;
2830 		else if (u_composition_table[m].cp2 > uc2)
2831 			b = m - 1;
2832 		else
2833 			return (u_composition_table[m].nfc);
2834 	}
2835 	return (0);
2836 }
2837 
2838 #define FDC_MAX 10	/* The maximum number of Following Decomposable
2839 			 * Characters. */
2840 
2841 /*
2842  * Update first code point.
2843  */
2844 #define UPDATE_UC(new_uc)	do {		\
2845 	uc = new_uc;				\
2846 	ucptr = NULL;				\
2847 } while (0)
2848 
2849 /*
2850  * Replace first code point with second code point.
2851  */
2852 #define REPLACE_UC_WITH_UC2() do {		\
2853 	uc = uc2;				\
2854 	ucptr = uc2ptr;				\
2855 	n = n2;					\
2856 } while (0)
2857 
2858 #define EXPAND_BUFFER() do {			\
2859 	as->length = p - as->s;			\
2860 	if (archive_string_ensure(as,		\
2861 	    as->buffer_length + len * tm + ts) == NULL)\
2862 		return (-1);			\
2863 	p = as->s + as->length;			\
2864 	endp = as->s + as->buffer_length - ts;	\
2865 } while (0)
2866 
2867 #define UNPARSE(p, endp, uc)	do {		\
2868 	while ((w = unparse(p, (endp) - (p), uc)) == 0) {\
2869 		EXPAND_BUFFER();		\
2870 	}					\
2871 	p += w;					\
2872 } while (0)
2873 
2874 /*
2875  * Write first code point.
2876  * If the code point has not be changed from its original code,
2877  * this just copies it from its original buffer pointer.
2878  * If not, this converts it to UTF-8 byte sequence and copies it.
2879  */
2880 #define WRITE_UC()	do {			\
2881 	if (ucptr) {				\
2882 		if (p + n > endp)		\
2883 			EXPAND_BUFFER();	\
2884 		switch (n) {			\
2885 		case 4:				\
2886 			*p++ = *ucptr++;	\
2887 			/* FALL THROUGH */	\
2888 		case 3:				\
2889 			*p++ = *ucptr++;	\
2890 			/* FALL THROUGH */	\
2891 		case 2:				\
2892 			*p++ = *ucptr++;	\
2893 			/* FALL THROUGH */	\
2894 		case 1:				\
2895 			*p++ = *ucptr;		\
2896 			break;			\
2897 		}				\
2898 		ucptr = NULL;			\
2899 	} else {				\
2900 		UNPARSE(p, endp, uc);		\
2901 	}					\
2902 } while (0)
2903 
2904 /*
2905  * Collect following decomposable code points.
2906  */
2907 #define COLLECT_CPS(start)	do {		\
2908 	int _i;					\
2909 	for (_i = start; _i < FDC_MAX ; _i++) {	\
2910 		nx = parse(&ucx[_i], s, len);	\
2911 		if (nx <= 0)			\
2912 			break;			\
2913 		cx = CCC(ucx[_i]);		\
2914 		if (cl >= cx && cl != 228 && cx != 228)\
2915 			break;			\
2916 		s += nx;			\
2917 		len -= nx;			\
2918 		cl = cx;			\
2919 		ccx[_i] = cx;			\
2920 	}					\
2921 	if (_i >= FDC_MAX) {			\
2922 		ret = -1;			\
2923 		ucx_size = FDC_MAX;		\
2924 	} else					\
2925 		ucx_size = _i;			\
2926 } while (0)
2927 
2928 /*
2929  * Normalize UTF-8/UTF-16BE characters to Form C and copy the result.
2930  *
2931  * TODO: Convert composition exclusions, which are never converted
2932  * from NFC,NFD,NFKC and NFKD, to Form C.
2933  */
2934 static int
2935 archive_string_normalize_C(struct archive_string *as, const void *_p,
2936     size_t len, struct archive_string_conv *sc)
2937 {
2938 	const char *s = (const char *)_p;
2939 	char *p, *endp;
2940 	uint32_t uc, uc2;
2941 	size_t w;
2942 	int always_replace, n, n2, ret = 0, spair, ts, tm;
2943 	int (*parse)(uint32_t *, const char *, size_t);
2944 	size_t (*unparse)(char *, size_t, uint32_t);
2945 
2946 	always_replace = 1;
2947 	ts = 1;/* text size. */
2948 	if (sc->flag & SCONV_TO_UTF16BE) {
2949 		unparse = unicode_to_utf16be;
2950 		ts = 2;
2951 		if (sc->flag & SCONV_FROM_UTF16BE)
2952 			always_replace = 0;
2953 	} else if (sc->flag & SCONV_TO_UTF16LE) {
2954 		unparse = unicode_to_utf16le;
2955 		ts = 2;
2956 		if (sc->flag & SCONV_FROM_UTF16LE)
2957 			always_replace = 0;
2958 	} else if (sc->flag & SCONV_TO_UTF8) {
2959 		unparse = unicode_to_utf8;
2960 		if (sc->flag & SCONV_FROM_UTF8)
2961 			always_replace = 0;
2962 	} else {
2963 		/*
2964 		 * This case is going to be converted to another
2965 		 * character-set through iconv.
2966 		 */
2967 		always_replace = 0;
2968 		if (sc->flag & SCONV_FROM_UTF16BE) {
2969 			unparse = unicode_to_utf16be;
2970 			ts = 2;
2971 		} else if (sc->flag & SCONV_FROM_UTF16LE) {
2972 			unparse = unicode_to_utf16le;
2973 			ts = 2;
2974 		} else {
2975 			unparse = unicode_to_utf8;
2976 		}
2977 	}
2978 
2979 	if (sc->flag & SCONV_FROM_UTF16BE) {
2980 		parse = utf16be_to_unicode;
2981 		tm = 1;
2982 		spair = 4;/* surrogate pair size in UTF-16. */
2983 	} else if (sc->flag & SCONV_FROM_UTF16LE) {
2984 		parse = utf16le_to_unicode;
2985 		tm = 1;
2986 		spair = 4;/* surrogate pair size in UTF-16. */
2987 	} else {
2988 		parse = cesu8_to_unicode;
2989 		tm = ts;
2990 		spair = 6;/* surrogate pair size in UTF-8. */
2991 	}
2992 
2993 	if (archive_string_ensure(as, as->length + len * tm + ts) == NULL)
2994 		return (-1);
2995 
2996 	p = as->s + as->length;
2997 	endp = as->s + as->buffer_length - ts;
2998 	while ((n = parse(&uc, s, len)) != 0) {
2999 		const char *ucptr, *uc2ptr;
3000 
3001 		if (n < 0) {
3002 			/* Use a replaced unicode character. */
3003 			UNPARSE(p, endp, uc);
3004 			s += n*-1;
3005 			len -= n*-1;
3006 			ret = -1;
3007 			continue;
3008 		} else if (n == spair || always_replace)
3009 			/* uc is converted from a surrogate pair.
3010 			 * this should be treated as a changed code. */
3011 			ucptr = NULL;
3012 		else
3013 			ucptr = s;
3014 		s += n;
3015 		len -= n;
3016 
3017 		/* Read second code point. */
3018 		while ((n2 = parse(&uc2, s, len)) > 0) {
3019 			uint32_t ucx[FDC_MAX];
3020 			int ccx[FDC_MAX];
3021 			int cl, cx, i, nx, ucx_size;
3022 			int LIndex,SIndex;
3023 			uint32_t nfc;
3024 
3025 			if (n2 == spair || always_replace)
3026 				/* uc2 is converted from a surrogate pair.
3027 			 	 * this should be treated as a changed code. */
3028 				uc2ptr = NULL;
3029 			else
3030 				uc2ptr = s;
3031 			s += n2;
3032 			len -= n2;
3033 
3034 			/*
3035 			 * If current second code point is out of decomposable
3036 			 * code points, finding compositions is unneeded.
3037 			 */
3038 			if (!IS_DECOMPOSABLE_BLOCK(uc2)) {
3039 				WRITE_UC();
3040 				REPLACE_UC_WITH_UC2();
3041 				continue;
3042 			}
3043 
3044 			/*
3045 			 * Try to combine current code points.
3046 			 */
3047 			/*
3048 			 * We have to combine Hangul characters according to
3049 			 * http://uniicode.org/reports/tr15/#Hangul
3050 			 */
3051 			if (0 <= (LIndex = uc - HC_LBASE) &&
3052 			    LIndex < HC_LCOUNT) {
3053 				/*
3054 				 * Hangul Composition.
3055 				 * 1. Two current code points are L and V.
3056 				 */
3057 				int VIndex = uc2 - HC_VBASE;
3058 				if (0 <= VIndex && VIndex < HC_VCOUNT) {
3059 					/* Make syllable of form LV. */
3060 					UPDATE_UC(HC_SBASE +
3061 					    (LIndex * HC_VCOUNT + VIndex) *
3062 					     HC_TCOUNT);
3063 				} else {
3064 					WRITE_UC();
3065 					REPLACE_UC_WITH_UC2();
3066 				}
3067 				continue;
3068 			} else if (0 <= (SIndex = uc - HC_SBASE) &&
3069 			    SIndex < HC_SCOUNT && (SIndex % HC_TCOUNT) == 0) {
3070 				/*
3071 				 * Hangul Composition.
3072 				 * 2. Two current code points are LV and T.
3073 				 */
3074 				int TIndex = uc2 - HC_TBASE;
3075 				if (0 < TIndex && TIndex < HC_TCOUNT) {
3076 					/* Make syllable of form LVT. */
3077 					UPDATE_UC(uc + TIndex);
3078 				} else {
3079 					WRITE_UC();
3080 					REPLACE_UC_WITH_UC2();
3081 				}
3082 				continue;
3083 			} else if ((nfc = get_nfc(uc, uc2)) != 0) {
3084 				/* A composition to current code points
3085 				 * is found. */
3086 				UPDATE_UC(nfc);
3087 				continue;
3088 			} else if ((cl = CCC(uc2)) == 0) {
3089 				/* Clearly 'uc2' the second code point is not
3090 				 * a decomposable code. */
3091 				WRITE_UC();
3092 				REPLACE_UC_WITH_UC2();
3093 				continue;
3094 			}
3095 
3096 			/*
3097 			 * Collect following decomposable code points.
3098 			 */
3099 			cx = 0;
3100 			ucx[0] = uc2;
3101 			ccx[0] = cl;
3102 			COLLECT_CPS(1);
3103 
3104 			/*
3105 			 * Find a composed code in the collected code points.
3106 			 */
3107 			i = 1;
3108 			while (i < ucx_size) {
3109 				int j;
3110 
3111 				if ((nfc = get_nfc(uc, ucx[i])) == 0) {
3112 					i++;
3113 					continue;
3114 				}
3115 
3116 				/*
3117 				 * nfc is composed of uc and ucx[i].
3118 				 */
3119 				UPDATE_UC(nfc);
3120 
3121 				/*
3122 				 * Remove ucx[i] by shifting
3123 				 * following code points.
3124 				 */
3125 				for (j = i; j+1 < ucx_size; j++) {
3126 					ucx[j] = ucx[j+1];
3127 					ccx[j] = ccx[j+1];
3128 				}
3129 				ucx_size --;
3130 
3131 				/*
3132 				 * Collect following code points blocked
3133 				 * by ucx[i] the removed code point.
3134 				 */
3135 				if (ucx_size > 0 && i == ucx_size &&
3136 				    nx > 0 && cx == cl) {
3137 					cl =  ccx[ucx_size-1];
3138 					COLLECT_CPS(ucx_size);
3139 				}
3140 				/*
3141 				 * Restart finding a composed code with
3142 				 * the updated uc from the top of the
3143 				 * collected code points.
3144 				 */
3145 				i = 0;
3146 			}
3147 
3148 			/*
3149 			 * Apparently the current code points are not
3150 			 * decomposed characters or already composed.
3151 			 */
3152 			WRITE_UC();
3153 			for (i = 0; i < ucx_size; i++)
3154 				UNPARSE(p, endp, ucx[i]);
3155 
3156 			/*
3157 			 * Flush out remaining canonical combining characters.
3158 			 */
3159 			if (nx > 0 && cx == cl && len > 0) {
3160 				while ((nx = parse(&ucx[0], s, len))
3161 				    > 0) {
3162 					cx = CCC(ucx[0]);
3163 					if (cl > cx)
3164 						break;
3165 					s += nx;
3166 					len -= nx;
3167 					cl = cx;
3168 					UNPARSE(p, endp, ucx[0]);
3169 				}
3170 			}
3171 			break;
3172 		}
3173 		if (n2 < 0) {
3174 			WRITE_UC();
3175 			/* Use a replaced unicode character. */
3176 			UNPARSE(p, endp, uc2);
3177 			s += n2*-1;
3178 			len -= n2*-1;
3179 			ret = -1;
3180 			continue;
3181 		} else if (n2 == 0) {
3182 			WRITE_UC();
3183 			break;
3184 		}
3185 	}
3186 	as->length = p - as->s;
3187 	as->s[as->length] = '\0';
3188 	if (ts == 2)
3189 		as->s[as->length+1] = '\0';
3190 	return (ret);
3191 }
3192 
3193 static int
3194 get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc)
3195 {
3196 	int t, b;
3197 
3198 	/*
3199 	 * These are not converted to NFD on Mac OS.
3200 	 */
3201 	if ((uc >= 0x2000 && uc <= 0x2FFF) ||
3202 	    (uc >= 0xF900 && uc <= 0xFAFF) ||
3203 	    (uc >= 0x2F800 && uc <= 0x2FAFF))
3204 		return (0);
3205 	/*
3206 	 * Those code points are not converted to NFD on Mac OS.
3207 	 * I do not know the reason because it is undocumented.
3208 	 *   NFC        NFD
3209 	 *   1109A  ==> 11099 110BA
3210 	 *   1109C  ==> 1109B 110BA
3211 	 *   110AB  ==> 110A5 110BA
3212 	 */
3213 	if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB)
3214 		return (0);
3215 
3216 	t = 0;
3217 	b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1;
3218 	while (b >= t) {
3219 		int m = (t + b) / 2;
3220 		if (u_decomposition_table[m].nfc < uc)
3221 			t = m + 1;
3222 		else if (u_decomposition_table[m].nfc > uc)
3223 			b = m - 1;
3224 		else {
3225 			*cp1 = u_decomposition_table[m].cp1;
3226 			*cp2 = u_decomposition_table[m].cp2;
3227 			return (1);
3228 		}
3229 	}
3230 	return (0);
3231 }
3232 
3233 #define REPLACE_UC_WITH(cp) do {		\
3234 	uc = cp;				\
3235 	ucptr = NULL;				\
3236 } while (0)
3237 
3238 /*
3239  * Normalize UTF-8 characters to Form D and copy the result.
3240  */
3241 static int
3242 archive_string_normalize_D(struct archive_string *as, const void *_p,
3243     size_t len, struct archive_string_conv *sc)
3244 {
3245 	const char *s = (const char *)_p;
3246 	char *p, *endp;
3247 	uint32_t uc, uc2;
3248 	size_t w;
3249 	int always_replace, n, n2, ret = 0, spair, ts, tm;
3250 	int (*parse)(uint32_t *, const char *, size_t);
3251 	size_t (*unparse)(char *, size_t, uint32_t);
3252 
3253 	always_replace = 1;
3254 	ts = 1;/* text size. */
3255 	if (sc->flag & SCONV_TO_UTF16BE) {
3256 		unparse = unicode_to_utf16be;
3257 		ts = 2;
3258 		if (sc->flag & SCONV_FROM_UTF16BE)
3259 			always_replace = 0;
3260 	} else if (sc->flag & SCONV_TO_UTF16LE) {
3261 		unparse = unicode_to_utf16le;
3262 		ts = 2;
3263 		if (sc->flag & SCONV_FROM_UTF16LE)
3264 			always_replace = 0;
3265 	} else if (sc->flag & SCONV_TO_UTF8) {
3266 		unparse = unicode_to_utf8;
3267 		if (sc->flag & SCONV_FROM_UTF8)
3268 			always_replace = 0;
3269 	} else {
3270 		/*
3271 		 * This case is going to be converted to another
3272 		 * character-set through iconv.
3273 		 */
3274 		always_replace = 0;
3275 		if (sc->flag & SCONV_FROM_UTF16BE) {
3276 			unparse = unicode_to_utf16be;
3277 			ts = 2;
3278 		} else if (sc->flag & SCONV_FROM_UTF16LE) {
3279 			unparse = unicode_to_utf16le;
3280 			ts = 2;
3281 		} else {
3282 			unparse = unicode_to_utf8;
3283 		}
3284 	}
3285 
3286 	if (sc->flag & SCONV_FROM_UTF16BE) {
3287 		parse = utf16be_to_unicode;
3288 		tm = 1;
3289 		spair = 4;/* surrogate pair size in UTF-16. */
3290 	} else if (sc->flag & SCONV_FROM_UTF16LE) {
3291 		parse = utf16le_to_unicode;
3292 		tm = 1;
3293 		spair = 4;/* surrogate pair size in UTF-16. */
3294 	} else {
3295 		parse = cesu8_to_unicode;
3296 		tm = ts;
3297 		spair = 6;/* surrogate pair size in UTF-8. */
3298 	}
3299 
3300 	if (archive_string_ensure(as, as->length + len * tm + ts) == NULL)
3301 		return (-1);
3302 
3303 	p = as->s + as->length;
3304 	endp = as->s + as->buffer_length - ts;
3305 	while ((n = parse(&uc, s, len)) != 0) {
3306 		const char *ucptr;
3307 		uint32_t cp1, cp2;
3308 		int SIndex;
3309 		struct {
3310 			uint32_t uc;
3311 			int ccc;
3312 		} fdc[FDC_MAX];
3313 		int fdi, fdj;
3314 		int ccc;
3315 
3316 check_first_code:
3317 		if (n < 0) {
3318 			/* Use a replaced unicode character. */
3319 			UNPARSE(p, endp, uc);
3320 			s += n*-1;
3321 			len -= n*-1;
3322 			ret = -1;
3323 			continue;
3324 		} else if (n == spair || always_replace)
3325 			/* uc is converted from a surrogate pair.
3326 			 * this should be treated as a changed code. */
3327 			ucptr = NULL;
3328 		else
3329 			ucptr = s;
3330 		s += n;
3331 		len -= n;
3332 
3333 		/* Hangul Decomposition. */
3334 		if ((SIndex = uc - HC_SBASE) >= 0 && SIndex < HC_SCOUNT) {
3335 			int L = HC_LBASE + SIndex / HC_NCOUNT;
3336 			int V = HC_VBASE + (SIndex % HC_NCOUNT) / HC_TCOUNT;
3337 			int T = HC_TBASE + SIndex % HC_TCOUNT;
3338 
3339 			REPLACE_UC_WITH(L);
3340 			WRITE_UC();
3341 			REPLACE_UC_WITH(V);
3342 			WRITE_UC();
3343 			if (T != HC_TBASE) {
3344 				REPLACE_UC_WITH(T);
3345 				WRITE_UC();
3346 			}
3347 			continue;
3348 		}
3349 		if (IS_DECOMPOSABLE_BLOCK(uc) && CCC(uc) != 0) {
3350 			WRITE_UC();
3351 			continue;
3352 		}
3353 
3354 		fdi = 0;
3355 		while (get_nfd(&cp1, &cp2, uc) && fdi < FDC_MAX) {
3356 			int k;
3357 
3358 			for (k = fdi; k > 0; k--)
3359 				fdc[k] = fdc[k-1];
3360 			fdc[0].ccc = CCC(cp2);
3361 			fdc[0].uc = cp2;
3362 			fdi++;
3363 			REPLACE_UC_WITH(cp1);
3364 		}
3365 
3366 		/* Read following code points. */
3367 		while ((n2 = parse(&uc2, s, len)) > 0 &&
3368 		    (ccc = CCC(uc2)) != 0 && fdi < FDC_MAX) {
3369 			int j, k;
3370 
3371 			s += n2;
3372 			len -= n2;
3373 			for (j = 0; j < fdi; j++) {
3374 				if (fdc[j].ccc > ccc)
3375 					break;
3376 			}
3377 			if (j < fdi) {
3378 				for (k = fdi; k > j; k--)
3379 					fdc[k] = fdc[k-1];
3380 				fdc[j].ccc = ccc;
3381 				fdc[j].uc = uc2;
3382 			} else {
3383 				fdc[fdi].ccc = ccc;
3384 				fdc[fdi].uc = uc2;
3385 			}
3386 			fdi++;
3387 		}
3388 
3389 		WRITE_UC();
3390 		for (fdj = 0; fdj < fdi; fdj++) {
3391 			REPLACE_UC_WITH(fdc[fdj].uc);
3392 			WRITE_UC();
3393 		}
3394 
3395 		if (n2 == 0)
3396 			break;
3397 		REPLACE_UC_WITH(uc2);
3398 		n = n2;
3399 		goto check_first_code;
3400 	}
3401 	as->length = p - as->s;
3402 	as->s[as->length] = '\0';
3403 	if (ts == 2)
3404 		as->s[as->length+1] = '\0';
3405 	return (ret);
3406 }
3407 
3408 /*
3409  * libarchive 2.x made incorrect UTF-8 strings in the wrong assumption
3410  * that WCS is Unicode. It is true for several platforms but some are false.
3411  * And then people who did not use UTF-8 locale on the non Unicode WCS
3412  * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those
3413  * now cannot get right filename from libarchive 3.x and later since we
3414  * fixed the wrong assumption and it is incompatible to older its versions.
3415  * So we provide special option, "compat-2x.x", for resolving it.
3416  * That option enable the string conversion of libarchive 2.x.
3417  *
3418  * Translates the wrong UTF-8 string made by libarchive 2.x into current
3419  * locale character set and appends to the archive_string.
3420  * Note: returns -1 if conversion fails.
3421  */
3422 static int
3423 strncat_from_utf8_libarchive2(struct archive_string *as,
3424     const void *_p, size_t len, struct archive_string_conv *sc)
3425 {
3426 	const char *s;
3427 	int n;
3428 	char *p;
3429 	char *end;
3430 	uint32_t unicode;
3431 #if HAVE_WCRTOMB
3432 	mbstate_t shift_state;
3433 
3434 	memset(&shift_state, 0, sizeof(shift_state));
3435 #else
3436 	/* Clear the shift state before starting. */
3437 	wctomb(NULL, L'\0');
3438 #endif
3439 	(void)sc; /* UNUSED */
3440 	/*
3441 	 * Allocate buffer for MBS.
3442 	 * We need this allocation here since it is possible that
3443 	 * as->s is still NULL.
3444 	 */
3445 	if (archive_string_ensure(as, as->length + len + 1) == NULL)
3446 		return (-1);
3447 
3448 	s = (const char *)_p;
3449 	p = as->s + as->length;
3450 	end = as->s + as->buffer_length - MB_CUR_MAX -1;
3451 	while ((n = _utf8_to_unicode(&unicode, s, len)) != 0) {
3452 		wchar_t wc;
3453 
3454 		if (p >= end) {
3455 			as->length = p - as->s;
3456 			/* Re-allocate buffer for MBS. */
3457 			if (archive_string_ensure(as,
3458 			    as->length + max(len * 2,
3459 			    (size_t)MB_CUR_MAX) + 1) == NULL)
3460 				return (-1);
3461 			p = as->s + as->length;
3462 			end = as->s + as->buffer_length - MB_CUR_MAX -1;
3463 		}
3464 
3465 		/*
3466 		 * As libarchive 2.x, translates the UTF-8 characters into
3467 		 * wide-characters in the assumption that WCS is Unicode.
3468 		 */
3469 		if (n < 0) {
3470 			n *= -1;
3471 			wc = L'?';
3472 		} else
3473 			wc = (wchar_t)unicode;
3474 
3475 		s += n;
3476 		len -= n;
3477 		/*
3478 		 * Translates the wide-character into the current locale MBS.
3479 		 */
3480 #if HAVE_WCRTOMB
3481 		n = (int)wcrtomb(p, wc, &shift_state);
3482 #else
3483 		n = (int)wctomb(p, wc);
3484 #endif
3485 		if (n == -1)
3486 			return (-1);
3487 		p += n;
3488 	}
3489 	as->length = p - as->s;
3490 	as->s[as->length] = '\0';
3491 	return (0);
3492 }
3493 
3494 
3495 /*
3496  * Conversion functions between current locale dependent MBS and UTF-16BE.
3497  *   strncat_from_utf16be() : UTF-16BE --> MBS
3498  *   strncat_to_utf16be()   : MBS --> UTF16BE
3499  */
3500 
3501 #if defined(_WIN32) && !defined(__CYGWIN__)
3502 
3503 /*
3504  * Convert a UTF-16BE/LE string to current locale and copy the result.
3505  * Return -1 if conversion fails.
3506  */
3507 static int
3508 win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes,
3509     struct archive_string_conv *sc, int be)
3510 {
3511 	struct archive_string tmp;
3512 	const char *u16;
3513 	int ll;
3514 	BOOL defchar;
3515 	char *mbs;
3516 	size_t mbs_size, b;
3517 	int ret = 0;
3518 
3519 	bytes &= ~1;
3520 	if (archive_string_ensure(as, as->length + bytes +1) == NULL)
3521 		return (-1);
3522 
3523 	mbs = as->s + as->length;
3524 	mbs_size = as->buffer_length - as->length -1;
3525 
3526 	if (sc->to_cp == CP_C_LOCALE) {
3527 		/*
3528 		 * "C" locale special process.
3529 		 */
3530 		u16 = _p;
3531 		ll = 0;
3532 		for (b = 0; b < bytes; b += 2) {
3533 			uint16_t val;
3534 			if (be)
3535 				val = archive_be16dec(u16+b);
3536 			else
3537 				val = archive_le16dec(u16+b);
3538 			if (val > 255) {
3539 				*mbs++ = '?';
3540 				ret = -1;
3541 			} else
3542 				*mbs++ = (char)(val&0xff);
3543 			ll++;
3544 		}
3545 		as->length += ll;
3546 		as->s[as->length] = '\0';
3547 		return (ret);
3548 	}
3549 
3550 	archive_string_init(&tmp);
3551 	if (be) {
3552 		if (is_big_endian()) {
3553 			u16 = _p;
3554 		} else {
3555 			if (archive_string_ensure(&tmp, bytes+2) == NULL)
3556 				return (-1);
3557 			memcpy(tmp.s, _p, bytes);
3558 			for (b = 0; b < bytes; b += 2) {
3559 				uint16_t val = archive_be16dec(tmp.s+b);
3560 				archive_le16enc(tmp.s+b, val);
3561 			}
3562 			u16 = tmp.s;
3563 		}
3564 	} else {
3565 		if (!is_big_endian()) {
3566 			u16 = _p;
3567 		} else {
3568 			if (archive_string_ensure(&tmp, bytes+2) == NULL)
3569 				return (-1);
3570 			memcpy(tmp.s, _p, bytes);
3571 			for (b = 0; b < bytes; b += 2) {
3572 				uint16_t val = archive_le16dec(tmp.s+b);
3573 				archive_be16enc(tmp.s+b, val);
3574 			}
3575 			u16 = tmp.s;
3576 		}
3577 	}
3578 
3579 	do {
3580 		defchar = 0;
3581 		ll = WideCharToMultiByte(sc->to_cp, 0,
3582 		    (LPCWSTR)u16, (int)bytes>>1, mbs, (int)mbs_size,
3583 			NULL, &defchar);
3584 		/* Exit loop if we succeeded */
3585 		if (ll != 0 ||
3586 		    GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
3587 			break;
3588 		}
3589 		/* Else expand buffer and loop to try again. */
3590 		ll = WideCharToMultiByte(sc->to_cp, 0,
3591 		    (LPCWSTR)u16, (int)bytes, NULL, 0, NULL, NULL);
3592 		if (archive_string_ensure(as, ll +1) == NULL)
3593 			return (-1);
3594 		mbs = as->s + as->length;
3595 		mbs_size = as->buffer_length - as->length -1;
3596 	} while (1);
3597 	archive_string_free(&tmp);
3598 	as->length += ll;
3599 	as->s[as->length] = '\0';
3600 	if (ll == 0 || defchar)
3601 		ret = -1;
3602 	return (ret);
3603 }
3604 
3605 static int
3606 win_strncat_from_utf16be(struct archive_string *as, const void *_p,
3607     size_t bytes, struct archive_string_conv *sc)
3608 {
3609 	return (win_strncat_from_utf16(as, _p, bytes, sc, 1));
3610 }
3611 
3612 static int
3613 win_strncat_from_utf16le(struct archive_string *as, const void *_p,
3614     size_t bytes, struct archive_string_conv *sc)
3615 {
3616 	return (win_strncat_from_utf16(as, _p, bytes, sc, 0));
3617 }
3618 
3619 static int
3620 is_big_endian(void)
3621 {
3622 	uint16_t d = 1;
3623 
3624 	return (archive_be16dec(&d) == 1);
3625 }
3626 
3627 /*
3628  * Convert a current locale string to UTF-16BE/LE and copy the result.
3629  * Return -1 if conversion fails.
3630  */
3631 static int
3632 win_strncat_to_utf16(struct archive_string *as16, const void *_p,
3633     size_t length, struct archive_string_conv *sc, int bigendian)
3634 {
3635 	const char *s = (const char *)_p;
3636 	char *u16;
3637 	size_t count, avail;
3638 
3639 	if (archive_string_ensure(as16,
3640 	    as16->length + (length + 1) * 2) == NULL)
3641 		return (-1);
3642 
3643 	u16 = as16->s + as16->length;
3644 	avail = as16->buffer_length - 2;
3645 	if (sc->from_cp == CP_C_LOCALE) {
3646 		/*
3647 		 * "C" locale special process.
3648 		 */
3649 		count = 0;
3650 		while (count < length && *s) {
3651 			if (bigendian)
3652 				archive_be16enc(u16, *s);
3653 			else
3654 				archive_le16enc(u16, *s);
3655 			u16 += 2;
3656 			s++;
3657 			count++;
3658 		}
3659 		as16->length += count << 1;
3660 		as16->s[as16->length] = 0;
3661 		as16->s[as16->length+1] = 0;
3662 		return (0);
3663 	}
3664 	do {
3665 		count = MultiByteToWideChar(sc->from_cp,
3666 		    MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1);
3667 		/* Exit loop if we succeeded */
3668 		if (count != 0 ||
3669 		    GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
3670 			break;
3671 		}
3672 		/* Expand buffer and try again */
3673 		count = MultiByteToWideChar(sc->from_cp,
3674 		    MB_PRECOMPOSED, s, (int)length, NULL, 0);
3675 		if (archive_string_ensure(as16, (count +1) * 2)
3676 		    == NULL)
3677 			return (-1);
3678 		u16 = as16->s + as16->length;
3679 		avail = as16->buffer_length - 2;
3680 	} while (1);
3681 	as16->length += count * 2;
3682 	as16->s[as16->length] = 0;
3683 	as16->s[as16->length+1] = 0;
3684 	if (count == 0)
3685 		return (-1);
3686 
3687 	if (is_big_endian()) {
3688 		if (!bigendian) {
3689 			while (count > 0) {
3690 				uint16_t v = archive_be16dec(u16);
3691 				archive_le16enc(u16, v);
3692 				u16 += 2;
3693 				count--;
3694 			}
3695 		}
3696 	} else {
3697 		if (bigendian) {
3698 			while (count > 0) {
3699 				uint16_t v = archive_le16dec(u16);
3700 				archive_be16enc(u16, v);
3701 				u16 += 2;
3702 				count--;
3703 			}
3704 		}
3705 	}
3706 	return (0);
3707 }
3708 
3709 static int
3710 win_strncat_to_utf16be(struct archive_string *as16, const void *_p,
3711     size_t length, struct archive_string_conv *sc)
3712 {
3713 	return (win_strncat_to_utf16(as16, _p, length, sc, 1));
3714 }
3715 
3716 static int
3717 win_strncat_to_utf16le(struct archive_string *as16, const void *_p,
3718     size_t length, struct archive_string_conv *sc)
3719 {
3720 	return (win_strncat_to_utf16(as16, _p, length, sc, 0));
3721 }
3722 
3723 #endif /* _WIN32 && !__CYGWIN__ */
3724 
3725 /*
3726  * Do the best effort for conversions.
3727  * We cannot handle UTF-16BE character-set without such iconv,
3728  * but there is a chance if a string consists just ASCII code or
3729  * a current locale is UTF-8.
3730  */
3731 
3732 /*
3733  * Convert a UTF-16BE string to current locale and copy the result.
3734  * Return -1 if conversion fails.
3735  */
3736 static int
3737 best_effort_strncat_from_utf16(struct archive_string *as, const void *_p,
3738     size_t bytes, struct archive_string_conv *sc, int be)
3739 {
3740 	const char *utf16 = (const char *)_p;
3741 	char *mbs;
3742 	uint32_t uc;
3743 	int n, ret;
3744 
3745 	(void)sc; /* UNUSED */
3746 	/*
3747 	 * Other case, we should do the best effort.
3748 	 * If all character are ASCII(<0x7f), we can convert it.
3749 	 * if not , we set a alternative character and return -1.
3750 	 */
3751 	ret = 0;
3752 	if (archive_string_ensure(as, as->length + bytes +1) == NULL)
3753 		return (-1);
3754 	mbs = as->s + as->length;
3755 
3756 	while ((n = utf16_to_unicode(&uc, utf16, bytes, be)) != 0) {
3757 		if (n < 0) {
3758 			n *= -1;
3759 			ret =  -1;
3760 		}
3761 		bytes -= n;
3762 		utf16 += n;
3763 
3764 		if (uc > 127) {
3765 			/* We cannot handle it. */
3766 			*mbs++ = '?';
3767 			ret =  -1;
3768 		} else
3769 			*mbs++ = (char)uc;
3770 	}
3771 	as->length = mbs - as->s;
3772 	as->s[as->length] = '\0';
3773 	return (ret);
3774 }
3775 
3776 static int
3777 best_effort_strncat_from_utf16be(struct archive_string *as, const void *_p,
3778     size_t bytes, struct archive_string_conv *sc)
3779 {
3780 	return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 1));
3781 }
3782 
3783 static int
3784 best_effort_strncat_from_utf16le(struct archive_string *as, const void *_p,
3785     size_t bytes, struct archive_string_conv *sc)
3786 {
3787 	return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 0));
3788 }
3789 
3790 /*
3791  * Convert a current locale string to UTF-16BE/LE and copy the result.
3792  * Return -1 if conversion fails.
3793  */
3794 static int
3795 best_effort_strncat_to_utf16(struct archive_string *as16, const void *_p,
3796     size_t length, struct archive_string_conv *sc, int bigendian)
3797 {
3798 	const char *s = (const char *)_p;
3799 	char *utf16;
3800 	size_t remaining;
3801 	int ret;
3802 
3803 	(void)sc; /* UNUSED */
3804 	/*
3805 	 * Other case, we should do the best effort.
3806 	 * If all character are ASCII(<0x7f), we can convert it.
3807 	 * if not , we set a alternative character and return -1.
3808 	 */
3809 	ret = 0;
3810 	remaining = length;
3811 
3812 	if (archive_string_ensure(as16,
3813 	    as16->length + (length + 1) * 2) == NULL)
3814 		return (-1);
3815 
3816 	utf16 = as16->s + as16->length;
3817 	while (remaining--) {
3818 		unsigned c = *s++;
3819 		if (c > 127) {
3820 			/* We cannot handle it. */
3821 			c = UNICODE_R_CHAR;
3822 			ret = -1;
3823 		}
3824 		if (bigendian)
3825 			archive_be16enc(utf16, c);
3826 		else
3827 			archive_le16enc(utf16, c);
3828 		utf16 += 2;
3829 	}
3830 	as16->length = utf16 - as16->s;
3831 	as16->s[as16->length] = 0;
3832 	as16->s[as16->length+1] = 0;
3833 	return (ret);
3834 }
3835 
3836 static int
3837 best_effort_strncat_to_utf16be(struct archive_string *as16, const void *_p,
3838     size_t length, struct archive_string_conv *sc)
3839 {
3840 	return (best_effort_strncat_to_utf16(as16, _p, length, sc, 1));
3841 }
3842 
3843 static int
3844 best_effort_strncat_to_utf16le(struct archive_string *as16, const void *_p,
3845     size_t length, struct archive_string_conv *sc)
3846 {
3847 	return (best_effort_strncat_to_utf16(as16, _p, length, sc, 0));
3848 }
3849 
3850 
3851 /*
3852  * Multistring operations.
3853  */
3854 
3855 void
3856 archive_mstring_clean(struct archive_mstring *aes)
3857 {
3858 	archive_wstring_free(&(aes->aes_wcs));
3859 	archive_string_free(&(aes->aes_mbs));
3860 	archive_string_free(&(aes->aes_utf8));
3861 	archive_string_free(&(aes->aes_mbs_in_locale));
3862 	aes->aes_set = 0;
3863 }
3864 
3865 void
3866 archive_mstring_copy(struct archive_mstring *dest, struct archive_mstring *src)
3867 {
3868 	dest->aes_set = src->aes_set;
3869 	archive_string_copy(&(dest->aes_mbs), &(src->aes_mbs));
3870 	archive_string_copy(&(dest->aes_utf8), &(src->aes_utf8));
3871 	archive_wstring_copy(&(dest->aes_wcs), &(src->aes_wcs));
3872 }
3873 
3874 int
3875 archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes,
3876   const char **p)
3877 {
3878 	struct archive_string_conv *sc;
3879 	int r;
3880 
3881 	/* If we already have a UTF8 form, return that immediately. */
3882 	if (aes->aes_set & AES_SET_UTF8) {
3883 		*p = aes->aes_utf8.s;
3884 		return (0);
3885 	}
3886 
3887 	*p = NULL;
3888 	/* Try converting WCS to MBS first if MBS does not exist yet. */
3889 	if ((aes->aes_set & AES_SET_MBS) == 0) {
3890 		const char *pm; /* unused */
3891 		archive_mstring_get_mbs(a, aes, &pm); /* ignore errors, we'll handle it later */
3892 	}
3893 	if (aes->aes_set & AES_SET_MBS) {
3894 		sc = archive_string_conversion_to_charset(a, "UTF-8", 1);
3895 		if (sc == NULL)
3896 			return (-1);/* Couldn't allocate memory for sc. */
3897 		r = archive_strncpy_l(&(aes->aes_utf8), aes->aes_mbs.s,
3898 		    aes->aes_mbs.length, sc);
3899 		if (a == NULL)
3900 			free_sconv_object(sc);
3901 		if (r == 0) {
3902 			aes->aes_set |= AES_SET_UTF8;
3903 			*p = aes->aes_utf8.s;
3904 			return (0);/* success. */
3905 		} else
3906 			return (-1);/* failure. */
3907 	}
3908 	return (0);/* success. */
3909 }
3910 
3911 int
3912 archive_mstring_get_mbs(struct archive *a, struct archive_mstring *aes,
3913     const char **p)
3914 {
3915 	struct archive_string_conv *sc;
3916 	int r, ret = 0;
3917 
3918 	/* If we already have an MBS form, return that immediately. */
3919 	if (aes->aes_set & AES_SET_MBS) {
3920 		*p = aes->aes_mbs.s;
3921 		return (ret);
3922 	}
3923 
3924 	*p = NULL;
3925 	/* If there's a WCS form, try converting with the native locale. */
3926 	if (aes->aes_set & AES_SET_WCS) {
3927 		archive_string_empty(&(aes->aes_mbs));
3928 		r = archive_string_append_from_wcs(&(aes->aes_mbs),
3929 		    aes->aes_wcs.s, aes->aes_wcs.length);
3930 		*p = aes->aes_mbs.s;
3931 		if (r == 0) {
3932 			aes->aes_set |= AES_SET_MBS;
3933 			return (ret);
3934 		} else
3935 			ret = -1;
3936 	}
3937 
3938 	/* If there's a UTF-8 form, try converting with the native locale. */
3939 	if (aes->aes_set & AES_SET_UTF8) {
3940 		archive_string_empty(&(aes->aes_mbs));
3941 		sc = archive_string_conversion_from_charset(a, "UTF-8", 1);
3942 		if (sc == NULL)
3943 			return (-1);/* Couldn't allocate memory for sc. */
3944 		r = archive_strncpy_l(&(aes->aes_mbs),
3945 			aes->aes_utf8.s, aes->aes_utf8.length, sc);
3946 		if (a == NULL)
3947 			free_sconv_object(sc);
3948 		*p = aes->aes_mbs.s;
3949 		if (r == 0) {
3950 			aes->aes_set |= AES_SET_MBS;
3951 			ret = 0;/* success; overwrite previous error. */
3952 		} else
3953 			ret = -1;/* failure. */
3954 	}
3955 	return (ret);
3956 }
3957 
3958 int
3959 archive_mstring_get_wcs(struct archive *a, struct archive_mstring *aes,
3960     const wchar_t **wp)
3961 {
3962 	int r, ret = 0;
3963 
3964 	(void)a;/* UNUSED */
3965 	/* Return WCS form if we already have it. */
3966 	if (aes->aes_set & AES_SET_WCS) {
3967 		*wp = aes->aes_wcs.s;
3968 		return (ret);
3969 	}
3970 
3971 	*wp = NULL;
3972 	/* Try converting UTF8 to MBS first if MBS does not exist yet. */
3973 	if ((aes->aes_set & AES_SET_MBS) == 0) {
3974 		const char *p; /* unused */
3975 		archive_mstring_get_mbs(a, aes, &p); /* ignore errors, we'll handle it later */
3976 	}
3977 	/* Try converting MBS to WCS using native locale. */
3978 	if (aes->aes_set & AES_SET_MBS) {
3979 		archive_wstring_empty(&(aes->aes_wcs));
3980 		r = archive_wstring_append_from_mbs(&(aes->aes_wcs),
3981 		    aes->aes_mbs.s, aes->aes_mbs.length);
3982 		if (r == 0) {
3983 			aes->aes_set |= AES_SET_WCS;
3984 			*wp = aes->aes_wcs.s;
3985 		} else
3986 			ret = -1;/* failure. */
3987 	}
3988 	return (ret);
3989 }
3990 
3991 int
3992 archive_mstring_get_mbs_l(struct archive *a, struct archive_mstring *aes,
3993     const char **p, size_t *length, struct archive_string_conv *sc)
3994 {
3995 	int ret = 0;
3996 #if defined(_WIN32) && !defined(__CYGWIN__)
3997 	int r;
3998 
3999 	/*
4000 	 * Internationalization programming on Windows must use Wide
4001 	 * characters because Windows platform cannot make locale UTF-8.
4002 	 */
4003 	if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) {
4004 		archive_string_empty(&(aes->aes_mbs_in_locale));
4005 		r = archive_string_append_from_wcs_in_codepage(
4006 		    &(aes->aes_mbs_in_locale), aes->aes_wcs.s,
4007 		    aes->aes_wcs.length, sc);
4008 		if (r == 0) {
4009 			*p = aes->aes_mbs_in_locale.s;
4010 			if (length != NULL)
4011 				*length = aes->aes_mbs_in_locale.length;
4012 			return (0);
4013 		} else if (errno == ENOMEM)
4014 			return (-1);
4015 		else
4016 			ret = -1;
4017 	}
4018 #endif
4019 
4020 	/* If there is not an MBS form but there is a WCS or UTF8 form, try converting
4021 	 * with the native locale to be used for translating it to specified
4022 	 * character-set. */
4023 	if ((aes->aes_set & AES_SET_MBS) == 0) {
4024 		const char *pm; /* unused */
4025 		archive_mstring_get_mbs(a, aes, &pm); /* ignore errors, we'll handle it later */
4026 	}
4027 	/* If we already have an MBS form, use it to be translated to
4028 	 * specified character-set. */
4029 	if (aes->aes_set & AES_SET_MBS) {
4030 		if (sc == NULL) {
4031 			/* Conversion is unneeded. */
4032 			*p = aes->aes_mbs.s;
4033 			if (length != NULL)
4034 				*length = aes->aes_mbs.length;
4035 			return (0);
4036 		}
4037 		ret = archive_strncpy_l(&(aes->aes_mbs_in_locale),
4038 		    aes->aes_mbs.s, aes->aes_mbs.length, sc);
4039 		*p = aes->aes_mbs_in_locale.s;
4040 		if (length != NULL)
4041 			*length = aes->aes_mbs_in_locale.length;
4042 	} else {
4043 		*p = NULL;
4044 		if (length != NULL)
4045 			*length = 0;
4046 	}
4047 	return (ret);
4048 }
4049 
4050 int
4051 archive_mstring_copy_mbs(struct archive_mstring *aes, const char *mbs)
4052 {
4053 	if (mbs == NULL) {
4054 		aes->aes_set = 0;
4055 		return (0);
4056 	}
4057 	return (archive_mstring_copy_mbs_len(aes, mbs, strlen(mbs)));
4058 }
4059 
4060 int
4061 archive_mstring_copy_mbs_len(struct archive_mstring *aes, const char *mbs,
4062     size_t len)
4063 {
4064 	if (mbs == NULL) {
4065 		aes->aes_set = 0;
4066 		return (0);
4067 	}
4068 	aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */
4069 	archive_strncpy(&(aes->aes_mbs), mbs, len);
4070 	archive_string_empty(&(aes->aes_utf8));
4071 	archive_wstring_empty(&(aes->aes_wcs));
4072 	return (0);
4073 }
4074 
4075 int
4076 archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs)
4077 {
4078 	return archive_mstring_copy_wcs_len(aes, wcs,
4079 				wcs == NULL ? 0 : wcslen(wcs));
4080 }
4081 
4082 int
4083 archive_mstring_copy_utf8(struct archive_mstring *aes, const char *utf8)
4084 {
4085   if (utf8 == NULL) {
4086     aes->aes_set = 0;
4087     return (0);
4088   }
4089   aes->aes_set = AES_SET_UTF8;
4090   archive_string_empty(&(aes->aes_mbs));
4091   archive_string_empty(&(aes->aes_wcs));
4092   archive_strncpy(&(aes->aes_utf8), utf8, strlen(utf8));
4093   return (int)strlen(utf8);
4094 }
4095 
4096 int
4097 archive_mstring_copy_wcs_len(struct archive_mstring *aes, const wchar_t *wcs,
4098     size_t len)
4099 {
4100 	if (wcs == NULL) {
4101 		aes->aes_set = 0;
4102 		return (0);
4103 	}
4104 	aes->aes_set = AES_SET_WCS; /* Only WCS form set. */
4105 	archive_string_empty(&(aes->aes_mbs));
4106 	archive_string_empty(&(aes->aes_utf8));
4107 	archive_wstrncpy(&(aes->aes_wcs), wcs, len);
4108 	return (0);
4109 }
4110 
4111 int
4112 archive_mstring_copy_mbs_len_l(struct archive_mstring *aes,
4113     const char *mbs, size_t len, struct archive_string_conv *sc)
4114 {
4115 	int r;
4116 
4117 	if (mbs == NULL) {
4118 		aes->aes_set = 0;
4119 		return (0);
4120 	}
4121 	archive_string_empty(&(aes->aes_mbs));
4122 	archive_wstring_empty(&(aes->aes_wcs));
4123 	archive_string_empty(&(aes->aes_utf8));
4124 #if defined(_WIN32) && !defined(__CYGWIN__)
4125 	/*
4126 	 * Internationalization programming on Windows must use Wide
4127 	 * characters because Windows platform cannot make locale UTF-8.
4128 	 */
4129 	if (sc == NULL) {
4130 		if (archive_string_append(&(aes->aes_mbs),
4131 			mbs, mbsnbytes(mbs, len)) == NULL) {
4132 			aes->aes_set = 0;
4133 			r = -1;
4134 		} else {
4135 			aes->aes_set = AES_SET_MBS;
4136 			r = 0;
4137 		}
4138 #if defined(HAVE_ICONV)
4139 	} else if (sc != NULL && sc->cd_w != (iconv_t)-1) {
4140 		/*
4141 		 * This case happens only when MultiByteToWideChar() cannot
4142 		 * handle sc->from_cp, and we have to iconv in order to
4143 		 * translate character-set to wchar_t,UTF-16.
4144 		 */
4145 		iconv_t cd = sc->cd;
4146 		unsigned from_cp;
4147 		int flag;
4148 
4149 		/*
4150 		 * Translate multi-bytes from some character-set to UTF-8.
4151 		 */
4152 		sc->cd = sc->cd_w;
4153 		r = archive_strncpy_l(&(aes->aes_utf8), mbs, len, sc);
4154 		sc->cd = cd;
4155 		if (r != 0) {
4156 			aes->aes_set = 0;
4157 			return (r);
4158 		}
4159 		aes->aes_set = AES_SET_UTF8;
4160 
4161 		/*
4162 		 * Append the UTF-8 string into wstring.
4163 		 */
4164 		flag = sc->flag;
4165 		sc->flag &= ~(SCONV_NORMALIZATION_C
4166 				| SCONV_TO_UTF16| SCONV_FROM_UTF16);
4167 		from_cp = sc->from_cp;
4168 		sc->from_cp = CP_UTF8;
4169 		r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs),
4170 			aes->aes_utf8.s, aes->aes_utf8.length, sc);
4171 		sc->flag = flag;
4172 		sc->from_cp = from_cp;
4173 		if (r == 0)
4174 			aes->aes_set |= AES_SET_WCS;
4175 #endif
4176 	} else {
4177 		r = archive_wstring_append_from_mbs_in_codepage(
4178 		    &(aes->aes_wcs), mbs, len, sc);
4179 		if (r == 0)
4180 			aes->aes_set = AES_SET_WCS;
4181 		else
4182 			aes->aes_set = 0;
4183 	}
4184 #else
4185 	r = archive_strncpy_l(&(aes->aes_mbs), mbs, len, sc);
4186 	if (r == 0)
4187 		aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */
4188 	else
4189 		aes->aes_set = 0;
4190 #endif
4191 	return (r);
4192 }
4193 
4194 /*
4195  * The 'update' form tries to proactively update all forms of
4196  * this string (WCS and MBS) and returns an error if any of
4197  * them fail.  This is used by the 'pax' handler, for instance,
4198  * to detect and report character-conversion failures early while
4199  * still allowing clients to get potentially useful values from
4200  * the more tolerant lazy conversions.  (get_mbs and get_wcs will
4201  * strive to give the user something useful, so you can get hopefully
4202  * usable values even if some of the character conversions are failing.)
4203  */
4204 int
4205 archive_mstring_update_utf8(struct archive *a, struct archive_mstring *aes,
4206     const char *utf8)
4207 {
4208 	struct archive_string_conv *sc;
4209 	int r;
4210 
4211 	if (utf8 == NULL) {
4212 		aes->aes_set = 0;
4213 		return (0); /* Succeeded in clearing everything. */
4214 	}
4215 
4216 	/* Save the UTF8 string. */
4217 	archive_strcpy(&(aes->aes_utf8), utf8);
4218 
4219 	/* Empty the mbs and wcs strings. */
4220 	archive_string_empty(&(aes->aes_mbs));
4221 	archive_wstring_empty(&(aes->aes_wcs));
4222 
4223 	aes->aes_set = AES_SET_UTF8;	/* Only UTF8 is set now. */
4224 
4225 	/* Try converting UTF-8 to MBS, return false on failure. */
4226 	sc = archive_string_conversion_from_charset(a, "UTF-8", 1);
4227 	if (sc == NULL)
4228 		return (-1);/* Couldn't allocate memory for sc. */
4229 	r = archive_strcpy_l(&(aes->aes_mbs), utf8, sc);
4230 	if (a == NULL)
4231 		free_sconv_object(sc);
4232 	if (r != 0)
4233 		return (-1);
4234 	aes->aes_set = AES_SET_UTF8 | AES_SET_MBS; /* Both UTF8 and MBS set. */
4235 
4236 	/* Try converting MBS to WCS, return false on failure. */
4237 	if (archive_wstring_append_from_mbs(&(aes->aes_wcs), aes->aes_mbs.s,
4238 	    aes->aes_mbs.length))
4239 		return (-1);
4240 	aes->aes_set = AES_SET_UTF8 | AES_SET_WCS | AES_SET_MBS;
4241 
4242 	/* All conversions succeeded. */
4243 	return (0);
4244 }
4245