1 /* $NetBSD$ */
2
3 /*-
4 * Copyright (c) 2025 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * iconv(3)
31 */
32
33 #include <sys/cdefs.h>
34 __RCSID("$NetBSD: t_mbrtoc8.c,v 1.3 2024/08/20 17:43:09 riastradh Exp $");
35
36 #include <atf-c.h>
37 #include <errno.h>
38 #include <iconv.h>
39 #include <signal.h>
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43
44 #include "h_macros.h"
45
46 static const struct sample {
47 const char *src_codeset;
48 size_t nsrc;
49 const char *src;
50 const char *dst_codeset;
51 size_t ndst;
52 const char *dst;
53 size_t ninval;
54 size_t ninval_rev;
55 const char *xfail;
56 #define IRREVERSIBLE ((size_t)-1)
57 } samples[] = {
58 [0] = { "us-ascii", 6, "hello",
59 "us-ascii", 6, "hello",
60 0, 0, NULL },
61 /* LATIN SMALL LETTER E WITH ACUTE ACCENT */
62 [1] = { "iso-8859-1", 1, (const char[1]){0xe9},
63 "UTF-8", 2, (const char[2]){0xc3,0xa9},
64 0, 0, NULL },
65 /* LEFT CURLY BRACKET */
66 [2] = { "UTF-8", 2, (const char[2]){0x00,0x7b},
67 "ZW", 6, (const char[6]){0x00,0x7a,0x57,0x20,0x7b,0x0a},
68 0, 0, NULL },
69 /* LATIN SMALL LETTER O, TILDE */
70 [3] = { "UTF-8", 2, (const char[2]){0x4f,0x7e},
71 "VIQR", 3, (const char[3]){0x4f,0x5c,0x7e},
72 0, 0, NULL },
73 /* LEFT CURLY BRACKET (optionally encoded in UTF-7) */
74 [4] = { "UTF-7", 1, (const char[1]){0x7b},
75 "UTF-8", 1, (const char[1]){0x7b},
76 0, IRREVERSIBLE, NULL },
77 [5] = { "UTF-7", 4, "+AHs-",
78 "UTF-8", 1, (const char[1]){0x7b},
79 0, 0, NULL },
80 /* éclair */
81 [6] = { "UTF-7", 10, "+AOk-clair",
82 "UTF-8", 7, (const char[7]){
83 0xc3,0xa9,0x63,0x6c,0x61,0x69,0x72,
84 },
85 0, 0, NULL },
86 [7] = { "HZ", 55, /* RFC 1843, Sec. 4, Example 1 */
87 "The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye.",
88 "UTF-8", 61, (const char[61]) {
89 0x54,0x68,0x65,0x20, 0x6e,0x65,0x78,0x74,
90 0x20,0x73,0x65,0x6e, 0x74,0x65,0x6e,0x63,
91 0x65,0x20,0x69,0x73, 0x20,0x69,0x6e,0x20,
92 0x47,0x42,0x2e,0xe5, 0xb7,0xb1,0xe6,0x89,
93 0x80,0xe4,0xb8,0x8d, 0xe6,0xac,0xb2,0xef,
94 0xbc,0x8c,0xe5,0x8b, 0xbf,0xe6,0x96,0xbd,
95 0xe6,0x96,0xbc,0xe4, 0xba,0xba,0xe3,0x80,
96 0x82,0x42,0x79,0x65, 0x2e
97 },
98 0, 0, NULL },
99 /* Same as above but with HZ8 instead of HZ. */
100 [8] = { "HZ8", 55, (const char[55]) {
101 0x54,0x68,0x65,0x20,0x6e,0x65,0x78,0x74,
102 0x20,0x73,0x65,0x6e,0x74,0x65,0x6e,0x63,
103 0x65,0x20,0x69,0x73,0x20,0x69,0x6e,0x20,
104 0x47,0x42,0x2e,0x7e,0x7b,0xbc,0xba,0xcb,
105 0xf9,0xb2,0xbb,0xd3,0xfb,0xa3,0xac,0xce,
106 0xf0,0xca,0xa9,0xec,0xb6,0xc8,0xcb,0xa1,
107 0xa3,0x7e,0x7d,0x42,0x79,0x65,0x2e
108 },
109 "UTF-8", 61, (const char[61]) {
110 0x54,0x68,0x65,0x20, 0x6e,0x65,0x78,0x74,
111 0x20,0x73,0x65,0x6e, 0x74,0x65,0x6e,0x63,
112 0x65,0x20,0x69,0x73, 0x20,0x69,0x6e,0x20,
113 0x47,0x42,0x2e,0xe5, 0xb7,0xb1,0xe6,0x89,
114 0x80,0xe4,0xb8,0x8d, 0xe6,0xac,0xb2,0xef,
115 0xbc,0x8c,0xe5,0x8b, 0xbf,0xe6,0x96,0xbd,
116 0xe6,0x96,0xbc,0xe4, 0xba,0xba,0xe3,0x80,
117 0x82,0x42,0x79,0x65, 0x2e
118 },
119 0, 0, NULL },
120 /* 馬 from CNS 11643-1 */
121 [9] = { "UTF-8", 3, (const char[3]){0xe9,0xa6,0xac},
122 "ISO-2022-CN", 8,
123 (const char[8]){0x1b,0x24,0x29,0x47,0x0e,0x58,0x6b,0x0f},
124 0, 0, NULL },
125 /* 馬毦 shifting from CNS 11643-1 to CNS 11643-2 */
126 [10] = { "UTF-8", 6, (const char[6]){0xe9,0xa6,0xac,0xe6,0xaf,0xa6},
127 "ISO-2022-CN", 16,
128 (const char[16]){
129 /* ESC $ ) G (shift G1 to CNS 11643 plane 1) */
130 0x1b,0x24,0x29,0x47,
131 0x0e, /* GL is G1 from now on */
132 0x58,0x6b, /* 馬 */
133 /* ESC $ * H (shift G2 to CNS 11643 plane 2) */
134 0x1b,0x24,0x2a,0x48,
135 0x1b,0x4e, /* GL is G2 for next char */
136 0x30,0x21, /* 毦 */
137 0x0f, /* GL is G0 from now on */
138 },
139 0, 0, "PR lib/59019: various iconv issues ([case 10: ISO-2022-CN to UTF-8 14/6] iconv: Illegal byte sequence (85))" },
140 };
141
142 #define MIN(x, y) ((x) < (y) ? (x) : (y))
143
144 static void
test_sample_bounded(const char * title,const struct sample * S,size_t nsrc,size_t ndst)145 test_sample_bounded(const char *title, const struct sample *S,
146 size_t nsrc, size_t ndst)
147 {
148 char inbuf[4096], inguard = arc4random();
149 char outbuf[4096], outguard = arc4random();
150 iconv_t C;
151 char *src0, *src, *dst0, *dst;
152 size_t srcleft0, srcleft, dstleft0, dstleft;
153 size_t ninval;
154 int error;
155
156 ATF_REQUIRE_MSG(S->nsrc < sizeof(inbuf) - 2, "[%s]", title);
157 ATF_REQUIRE_MSG(S->ndst < sizeof(outbuf) - 2, "[%s]", title);
158
159 memset(inbuf, inguard, sizeof(inbuf));
160 memset(outbuf, outguard, sizeof(outbuf));
161 src = src0 = inbuf + 1;
162 memcpy(src, S->src, nsrc);
163 dst = dst0 = outbuf + 1;
164 srcleft = srcleft0 = nsrc;
165 dstleft = dstleft0 = ndst;
166
167 C = iconv_open(S->dst_codeset, S->src_codeset);
168 if (C == (iconv_t)-1) {
169 error = errno;
170 atf_tc_fail_nonfatal("[%s] iconv_open: %s (%d)", title,
171 strerror(error), error);
172 return;
173 }
174
175 ninval = iconv(C, &src, &srcleft, &dst, &dstleft);
176 if (ninval == (size_t)-1) {
177 error = errno;
178 /*
179 * Incomplete input manifests as EINVAL -- ignore that,
180 * unless we're trying the full-size input.
181 *
182 * Truncated output manifests as E2BIG -- ignore that,
183 * unless we're trying the full-size output.
184 */
185 if ((error != EINVAL || nsrc == S->nsrc) &&
186 (error != E2BIG || ndst == S->ndst)) {
187 atf_tc_fail_nonfatal("[%s] iconv: %s (%d)",
188 title, strerror(error), error);
189 goto next;
190 }
191 } else if (ninval != S->ninval) {
192 error = errno;
193 atf_tc_fail_nonfatal("[%s] iconv:"
194 " %zu invalid, expected %zu",
195 title, ninval, S->ninval);
196 }
197
198 ATF_CHECK_MSG(src0 <= src && src <= src0 + srcleft0, "[%s] iconv:"
199 " src went from %p to %p, expected [%p,%p]",
200 title, src0, src, src0, src0 + srcleft0);
201 ATF_CHECK_MSG(srcleft <= srcleft0, "[%s] iconv:"
202 " srcleft went from %zu to %zu",
203 title, srcleft0, srcleft);
204 ATF_CHECK_MSG(dst0 <= dst && dst <= dst0 + dstleft0, "[%s] iconv:"
205 " dst went from %p to %p, expected [%p,%p]",
206 title, dst0, dst, dst0, dst0 + dstleft0);
207 ATF_CHECK_MSG(dstleft <= dstleft0, "[%s] iconv:"
208 " dstleft went from %zu to %zu",
209 title, dstleft0, dstleft);
210 if (memcmp(dst0, S->dst, MIN(ndst, dstleft0 - dstleft)) != 0) {
211 size_t k;
212
213 atf_tc_fail_nonfatal("[%s] iconv: bad conv", title);
214 fprintf(stderr, "[%s] input: ", title);
215 for (k = 0; k < nsrc; k++)
216 fprintf(stderr, " %02x", (unsigned char)S->src[k]);
217 fprintf(stderr, "\n");
218 fprintf(stderr, "[%s] expected: ", title);
219 for (k = 0; k < ndst; k++)
220 fprintf(stderr, " %02x", (unsigned char)S->dst[k]);
221 fprintf(stderr, "\n");
222 fprintf(stderr, "[%s] got: ", title);
223 for (k = 0; k < ndst; k++)
224 fprintf(stderr, " %02x", (unsigned char)dst0[k]);
225 fprintf(stderr, "\n");
226 }
227
228 next: if (dst0[-1] != outguard) {
229 atf_tc_fail_nonfatal("[%s] iconv overran before buffer:"
230 " dst0[-1] = 0x%02x, expected 0x%02x",
231 title, dst0[-1], outguard);
232 }
233 if (dst0[ndst] != outguard) {
234 atf_tc_fail_nonfatal("[%s] iconv overran after buffer:"
235 " dst0[ndst=%zu] = 0x%02x, expected 0x%02x",
236 title, ndst, dst0[ndst], outguard);
237 }
238 if (dst[0] != outguard) {
239 atf_tc_fail_nonfatal("[%s] iconv overran past updated dst:"
240 " dst[0] = 0x%02x, expected 0x%02x",
241 title, dst[0], outguard);
242 }
243
244 if (iconv_close(C) == -1) {
245 error = errno;
246 atf_tc_fail_nonfatal("[%s] iconv_close: %s (%d)",
247 title, strerror(error), error);
248 }
249 }
250
251 static void
test_sample(const char * title,const struct sample * S)252 test_sample(const char *title, const struct sample *S)
253 {
254 char buf[128];
255 size_t nsrc, ndst;
256
257 fprintf(stderr, "test %s from %s to %s\n",
258 title, S->src_codeset, S->dst_codeset);
259
260 for (nsrc = 0; nsrc <= S->nsrc; nsrc++) {
261 snprintf(buf, sizeof(buf), "%s %zu/%zu", title, nsrc, S->ndst);
262 test_sample_bounded(buf, S, nsrc, S->ndst);
263 }
264
265 for (ndst = 0; ndst <= S->ndst; ndst++) {
266 snprintf(buf, sizeof(buf), "%s %zu/%zu", title, S->nsrc, ndst);
267 test_sample_bounded(buf, S, S->nsrc, ndst);
268 }
269 }
270
271 ATF_TC(iconv_samples);
ATF_TC_HEAD(iconv_samples,tc)272 ATF_TC_HEAD(iconv_samples, tc)
273 {
274
275 atf_tc_set_md_var(tc, "descr", "Test iconv on various fixed samples");
276 }
ATF_TC_BODY(iconv_samples,tc)277 ATF_TC_BODY(iconv_samples, tc)
278 {
279 unsigned i;
280
281 for (i = 0; i < __arraycount(samples); i++) {
282 const struct sample *S = &samples[i];
283 struct sample reverse = {
284 .src_codeset = S->dst_codeset,
285 .dst_codeset = S->src_codeset,
286 .nsrc = S->ndst,
287 .src = S->dst,
288 .ndst = S->nsrc,
289 .dst = S->src,
290 .ninval = S->ninval_rev,
291 };
292 char buf[128];
293
294 if (S->xfail)
295 atf_tc_expect_fail("%s", S->xfail);
296
297 snprintf(buf, sizeof(buf), "case %u: %s to %s", i,
298 S->src_codeset, S->dst_codeset);
299 test_sample(buf, S);
300 if (S->ninval_rev != IRREVERSIBLE) {
301 snprintf(buf, sizeof(buf), "case %u: %s to %s", i,
302 S->dst_codeset, S->src_codeset);
303 test_sample(buf, &reverse);
304 }
305
306 if (S->xfail)
307 atf_tc_expect_pass();
308 }
309 }
310
311 ATF_TC(iconv_pr59019_hz8);
ATF_TC_HEAD(iconv_pr59019_hz8,tc)312 ATF_TC_HEAD(iconv_pr59019_hz8, tc)
313 {
314
315 atf_tc_set_md_var(tc, "descr", "Truncated HZ8 input from PR 59019");
316 }
ATF_TC_BODY(iconv_pr59019_hz8,tc)317 ATF_TC_BODY(iconv_pr59019_hz8, tc)
318 {
319 const char title[] = "iconv_pr59019_hz8";
320 char in[4] = "\x7e\x7b\x7e\x7e"; /* ~{~~ */
321 char out[4096], guard = arc4random();
322 char *src0, *src, *dst0, *dst;
323 size_t srcleft0, srcleft, dstleft0, dstleft;
324 iconv_t C;
325 size_t ninval;
326
327 memset(out, guard, sizeof(out));
328
329 REQUIRE_LIBC((C = iconv_open(/*to*/"UTF-8", /*from*/"HZ8")),
330 (iconv_t)-1);
331
332 src = src0 = in;
333 srcleft = srcleft0 = sizeof(in);
334 dst = dst0 = out + 1;
335 dstleft = dstleft0 = sizeof(out) - 2;
336
337 ninval = iconv(C, &src, &srcleft, &dst, &dstleft);
338
339 /*
340 * XXX Not actually 100% sure this is the correct result -- I
341 * can't find a reference for the HZ8 encoding.
342 */
343 ATF_CHECK_EQ_MSG(ninval, (size_t)-1, "[%s] iconv: ninval=%zu", title,
344 ninval);
345
346 ATF_CHECK_MSG(src0 <= src && src <= src0 + srcleft0, "[%s] iconv:"
347 " src went from %p to %p, expected [%p,%p)",
348 title, src0, src, src0, src0 + srcleft0);
349 ATF_CHECK_MSG(srcleft <= srcleft0, "[%s] iconv:"
350 " srcleft went from %zu to %zu",
351 title, srcleft0, srcleft);
352 ATF_CHECK_MSG(dst0 <= dst && dst <= dst0 + dstleft0, "[%s] iconv:"
353 " dst went from %p to %p, expected [%p,%p)",
354 title, dst0, dst, dst0, dst0 + dstleft0);
355 ATF_CHECK_MSG(dstleft <= dstleft0, "[%s] iconv:"
356 " dstleft went from %zu to %zu",
357 title, dstleft0, dstleft);
358
359 ATF_CHECK_EQ(dst0[-1], guard);
360 ATF_CHECK_EQ(dst0[dstleft0], guard);
361
362 RL(iconv_close(C));
363 }
364
ATF_TP_ADD_TCS(tp)365 ATF_TP_ADD_TCS(tp)
366 {
367
368 ATF_TP_ADD_TC(tp, iconv_pr59019_hz8);
369 ATF_TP_ADD_TC(tp, iconv_samples);
370
371 return atf_no_error();
372 }
373