1 /*-
2 * Copyright (c) 2003-2007,2013 Tim Kientzle
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25 #include "test.h"
26
27 #include <errno.h>
28 #include <stdlib.h>
29 #include <string.h>
30
31 /*
32 * This is a somewhat tricky test that verifies the ability to
33 * write and read very large entries to zip archives.
34 *
35 * See test_tar_large.c for more information about the machinery
36 * being used here.
37 */
38
39 static size_t nullsize;
40 static void *nulldata;
41
42 struct fileblock {
43 struct fileblock *next;
44 int size;
45 void *buff;
46 int64_t gap_size; /* Size of following gap */
47 };
48
49 struct fileblocks {
50 int64_t filesize;
51 int64_t fileposition;
52 int64_t gap_remaining;
53 void *buff;
54 struct fileblock *first;
55 struct fileblock *current;
56 struct fileblock *last;
57 };
58
59 /* The following size definitions simplify things below. */
60 #define KB ((int64_t)1024)
61 #define MB ((int64_t)1024 * KB)
62 #define GB ((int64_t)1024 * MB)
63 #define TB ((int64_t)1024 * GB)
64
65 static int64_t memory_read_skip(struct archive *, void *, int64_t request);
66 static ssize_t memory_read(struct archive *, void *, const void **buff);
67 static ssize_t memory_write(struct archive *, void *, const void *, size_t);
68
le16(const void * _p)69 static uint16_t le16(const void *_p) {
70 const uint8_t *p = _p;
71 return p[0] | (p[1] << 8);
72 }
73
le32(const void * _p)74 static uint32_t le32(const void *_p) {
75 const uint8_t *p = _p;
76 return le16(p) | ((uint32_t)le16(p + 2) << 16);
77 }
78
le64(const void * _p)79 static uint64_t le64(const void *_p) {
80 const uint8_t *p = _p;
81 return le32(p) | ((uint64_t)le32(p + 4) << 32);
82 }
83
84 static ssize_t
memory_write(struct archive * a,void * _private,const void * buff,size_t size)85 memory_write(struct archive *a, void *_private, const void *buff, size_t size)
86 {
87 struct fileblocks *private = _private;
88 struct fileblock *block;
89
90 (void)a;
91
92 if ((const char *)nulldata <= (const char *)buff
93 && (const char *)buff < (const char *)nulldata + nullsize) {
94 /* We don't need to store a block of gap data. */
95 private->last->gap_size += (int64_t)size;
96 } else {
97 /* Yes, we're assuming the very first write is metadata. */
98 /* It's header or metadata, copy and save it. */
99 block = malloc(sizeof(*block));
100 memset(block, 0, sizeof(*block));
101 block->size = (int)size;
102 block->buff = malloc(size);
103 memcpy(block->buff, buff, size);
104 if (private->last == NULL) {
105 private->first = private->last = block;
106 } else {
107 private->last->next = block;
108 private->last = block;
109 }
110 block->next = NULL;
111 }
112 private->filesize += size;
113 return ((long)size);
114 }
115
116 static ssize_t
memory_read(struct archive * a,void * _private,const void ** buff)117 memory_read(struct archive *a, void *_private, const void **buff)
118 {
119 struct fileblocks *private = _private;
120 ssize_t size;
121
122 (void)a;
123
124 while (private->current != NULL && private->buff == NULL && private->gap_remaining == 0) {
125 private->current = private->current->next;
126 if (private->current != NULL) {
127 private->buff = private->current->buff;
128 private->gap_remaining = private->current->gap_size;
129 }
130 }
131
132 if (private->current == NULL)
133 return (0);
134
135 /* If there's real data, return that. */
136 if (private->buff != NULL) {
137 *buff = private->buff;
138 size = ((char *)private->current->buff + private->current->size)
139 - (char *)private->buff;
140 private->buff = NULL;
141 private->fileposition += size;
142 return (size);
143 }
144
145 /* Big gap: too big to return all at once, so just return some. */
146 if (private->gap_remaining > (int64_t)nullsize) {
147 private->gap_remaining -= nullsize;
148 *buff = nulldata;
149 private->fileposition += nullsize;
150 return (nullsize);
151 }
152
153 /* Small gap: finish the gap and prep for next block. */
154 if (private->gap_remaining > 0) {
155 size = (ssize_t)private->gap_remaining;
156 *buff = nulldata;
157 private->gap_remaining = 0;
158 private->fileposition += size;
159
160 private->current = private->current->next;
161 if (private->current != NULL) {
162 private->buff = private->current->buff;
163 private->gap_remaining = private->current->gap_size;
164 }
165
166 return (size);
167 }
168 fprintf(stderr, "\n\n\nInternal failure\n\n\n");
169 exit(1);
170 }
171
172 static int
memory_read_open(struct archive * a,void * _private)173 memory_read_open(struct archive *a, void *_private)
174 {
175 struct fileblocks *private = _private;
176
177 (void)a; /* UNUSED */
178
179 private->current = private->first;
180 private->fileposition = 0;
181 if (private->current != NULL) {
182 private->buff = private->current->buff;
183 private->gap_remaining = private->current->gap_size;
184 }
185 return (ARCHIVE_OK);
186 }
187
188 static int64_t
memory_read_seek(struct archive * a,void * _private,int64_t offset,int whence)189 memory_read_seek(struct archive *a, void *_private, int64_t offset, int whence)
190 {
191 struct fileblocks *private = _private;
192
193 (void)a;
194 if (whence == SEEK_END) {
195 offset = private->filesize + offset;
196 } else if (whence == SEEK_CUR) {
197 offset = private->fileposition + offset;
198 }
199
200 if (offset < 0) {
201 fprintf(stderr, "\n\n\nInternal failure: negative seek\n\n\n");
202 exit(1);
203 }
204
205 /* We've converted the request into a SEEK_SET. */
206 private->fileposition = offset;
207
208 /* Walk the block list to find the new position. */
209 offset = 0;
210 private->current = private->first;
211 while (private->current != NULL) {
212 if (offset + private->current->size > private->fileposition) {
213 /* Position is in this block. */
214 private->buff = (char *)private->current->buff
215 + private->fileposition - offset;
216 private->gap_remaining = private->current->gap_size;
217 return private->fileposition;
218 }
219 offset += private->current->size;
220 if (offset + private->current->gap_size > private->fileposition) {
221 /* Position is in this gap. */
222 private->buff = NULL;
223 private->gap_remaining = private->current->gap_size
224 - (private->fileposition - offset);
225 return private->fileposition;
226 }
227 offset += private->current->gap_size;
228 /* Skip to next block. */
229 private->current = private->current->next;
230 }
231 if (private->fileposition == private->filesize) {
232 return private->fileposition;
233 }
234 fprintf(stderr, "\n\n\nInternal failure: over-sized seek\n\n\n");
235 exit(1);
236 }
237
238 static int64_t
memory_read_skip(struct archive * a,void * _private,int64_t skip)239 memory_read_skip(struct archive *a, void *_private, int64_t skip)
240 {
241 struct fileblocks *private = _private;
242 int64_t old_position = private->fileposition;
243 int64_t new_position = memory_read_seek(a, _private, skip, SEEK_CUR);
244 return (new_position - old_position);
245 }
246
247 static struct fileblocks *
fileblocks_new(void)248 fileblocks_new(void)
249 {
250 struct fileblocks *fileblocks;
251
252 fileblocks = calloc(1, sizeof(struct fileblocks));
253 return fileblocks;
254 }
255
256 static void
fileblocks_free(struct fileblocks * fileblocks)257 fileblocks_free(struct fileblocks *fileblocks)
258 {
259 while (fileblocks->first != NULL) {
260 struct fileblock *b = fileblocks->first;
261 fileblocks->first = fileblocks->first->next;
262 free(b->buff);
263 free(b);
264 }
265 free(fileblocks);
266 }
267
268
269 /* The sizes of the entries we're going to generate. */
270 static int64_t test_sizes[] = {
271 /* Test for 32-bit signed overflow. */
272 2 * GB - 1, 2 * GB, 2 * GB + 1,
273 /* Test for 32-bit unsigned overflow. */
274 4 * GB - 1, 4 * GB, 4 * GB + 1,
275 /* And one larger sample */
276 5 * GB,
277 0
278 };
279
280
281 static void
verify_large_zip(struct archive * a,struct fileblocks * fileblocks,int seeking)282 verify_large_zip(struct archive *a, struct fileblocks *fileblocks, int seeking)
283 {
284 char namebuff[64];
285 struct archive_entry *ae;
286 int i;
287
288 (void)seeking; /* UNUSED */
289
290 assertEqualIntA(a, ARCHIVE_OK,
291 archive_read_set_options(a, "zip:ignorecrc32"));
292 assertEqualIntA(a, ARCHIVE_OK,
293 archive_read_set_open_callback(a, memory_read_open));
294 assertEqualIntA(a, ARCHIVE_OK,
295 archive_read_set_read_callback(a, memory_read));
296 assertEqualIntA(a, ARCHIVE_OK,
297 archive_read_set_skip_callback(a, memory_read_skip));
298 assertEqualIntA(a, ARCHIVE_OK,
299 archive_read_set_seek_callback(a, memory_read_seek));
300 assertEqualIntA(a, ARCHIVE_OK,
301 archive_read_set_callback_data(a, fileblocks));
302 assertEqualIntA(a, ARCHIVE_OK, archive_read_open1(a));
303
304 /*
305 * Read entries back.
306 */
307 for (i = 0; test_sizes[i] > 0; i++) {
308 assertEqualIntA(a, ARCHIVE_OK,
309 archive_read_next_header(a, &ae));
310 snprintf(namebuff, sizeof(namebuff), "file_%d", i);
311 assertEqualString(namebuff, archive_entry_pathname(ae));
312 if (seeking) {
313 assert(archive_entry_size_is_set(ae));
314 assertEqualInt(test_sizes[i], archive_entry_size(ae));
315 } else {
316 assertEqualInt(0, archive_entry_size_is_set(ae));
317 }
318 /* TODO: Read to end of data, verify length */
319 }
320 assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
321 assertEqualString("lastfile", archive_entry_pathname(ae));
322
323 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
324
325 /* Close out the archive. */
326 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
327 }
328
DEFINE_TEST(test_write_format_zip_large)329 DEFINE_TEST(test_write_format_zip_large)
330 {
331 int i;
332 char namebuff[64];
333 struct fileblocks *fileblocks = fileblocks_new();
334 struct archive_entry *ae;
335 struct archive *a;
336 const char *p;
337 const char *cd_start, *zip64_eocd, *zip64_locator, *eocd;
338 int64_t cd_size;
339 char *buff;
340 int64_t filesize;
341 size_t writesize, buffsize, s;
342
343 nullsize = (size_t)(1 * MB);
344 nulldata = malloc(nullsize);
345 memset(nulldata, 0xAA, nullsize);
346
347 /*
348 * Open an archive for writing.
349 */
350 a = archive_write_new();
351 archive_write_set_format_zip(a);
352 /* TODO: Repeat this entire test suite with default compression */
353 assertEqualIntA(a, ARCHIVE_OK,
354 archive_write_set_options(a, "zip:compression=store"));
355 assertEqualIntA(a, ARCHIVE_OK,
356 archive_write_set_options(a, "zip:fakecrc32"));
357 assertEqualIntA(a, ARCHIVE_OK,
358 archive_write_set_bytes_per_block(a, 0)); /* No buffering. */
359 assertEqualIntA(a, ARCHIVE_OK,
360 archive_write_open(a, fileblocks, NULL, memory_write, NULL));
361
362 /*
363 * Write a series of large files to it.
364 */
365 for (i = 0; test_sizes[i] != 0; i++) {
366 assert((ae = archive_entry_new()) != NULL);
367 snprintf(namebuff, sizeof(namebuff), "file_%d", i);
368 archive_entry_copy_pathname(ae, namebuff);
369 archive_entry_set_mode(ae, S_IFREG | 0755);
370 filesize = test_sizes[i];
371 archive_entry_set_size(ae, filesize);
372
373 assertEqualIntA(a, ARCHIVE_OK,
374 archive_write_header(a, ae));
375 archive_entry_free(ae);
376
377 /*
378 * Write the actual data to the archive.
379 */
380 while (filesize > 0) {
381 writesize = nullsize;
382 if ((int64_t)writesize > filesize)
383 writesize = (size_t)filesize;
384 assertEqualIntA(a, (int)writesize,
385 (int)archive_write_data(a, nulldata, writesize));
386 filesize -= writesize;
387 }
388 }
389
390 assert((ae = archive_entry_new()) != NULL);
391 archive_entry_copy_pathname(ae, "lastfile");
392 archive_entry_set_mode(ae, S_IFREG | 0755);
393 assertA(0 == archive_write_header(a, ae));
394 archive_entry_free(ae);
395
396 /* Close out the archive. */
397 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
398 assertEqualInt(ARCHIVE_OK, archive_write_free(a));
399
400 /*
401 * Read back with seeking reader:
402 */
403 a = archive_read_new();
404 assertEqualIntA(a, ARCHIVE_OK,
405 archive_read_support_format_zip_seekable(a));
406 verify_large_zip(a, fileblocks, 1);
407 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
408
409 /*
410 * Read back with streaming reader:
411 */
412 a = archive_read_new();
413 assertEqualIntA(a, ARCHIVE_OK,
414 archive_read_support_format_zip_streamable(a));
415 verify_large_zip(a, fileblocks, 0);
416 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
417
418 /*
419 * Manually verify some of the final bytes of the archives.
420 */
421 /* Collect the final bytes together */
422 #define FINAL_SIZE 8192
423 buff = malloc(FINAL_SIZE);
424 buffsize = 0;
425 memory_read_open(NULL, fileblocks);
426 memory_read_seek(NULL, fileblocks, -FINAL_SIZE, SEEK_END);
427 while ((s = memory_read(NULL, fileblocks, (const void **)&p)) > 0) {
428 memcpy(buff + buffsize, p, s);
429 buffsize += s;
430 }
431 assertEqualInt(buffsize, FINAL_SIZE);
432
433 p = buff + buffsize;
434
435 /* Verify regular end-of-central-directory record */
436 eocd = p - 22;
437 assertEqualMem(eocd, "PK\005\006\0\0\0\0", 8);
438 assertEqualMem(eocd + 8, "\010\0\010\0", 4); /* 8 entries total */
439 cd_size = le32(eocd + 12);
440 /* Start of CD offset should be 0xffffffff */
441 assertEqualMem(eocd + 16, "\xff\xff\xff\xff", 4);
442 assertEqualMem(eocd + 20, "\0\0", 2); /* No Zip comment */
443
444 /* Verify Zip64 locator */
445 zip64_locator = p - 42;
446 assertEqualMem(zip64_locator, "PK\006\007\0\0\0\0", 8);
447 zip64_eocd = p - (fileblocks->filesize - le64(zip64_locator + 8));
448 assertEqualMem(zip64_locator + 16, "\001\0\0\0", 4);
449
450 /* Verify Zip64 end-of-cd record. */
451 assert(zip64_eocd == p - 98);
452 assertEqualMem(zip64_eocd, "PK\006\006", 4);
453 assertEqualInt(44, le64(zip64_eocd + 4)); // Size of EoCD record - 12
454 assertEqualMem(zip64_eocd + 12, "\055\0", 2); // Made by version: 45
455 assertEqualMem(zip64_eocd + 14, "\055\0", 2); // Requires version: 45
456 assertEqualMem(zip64_eocd + 16, "\0\0\0\0", 4); // This disk
457 assertEqualMem(zip64_eocd + 20, "\0\0\0\0", 4); // Total disks
458 assertEqualInt(8, le64(zip64_eocd + 24)); // Entries on this disk
459 assertEqualInt(8, le64(zip64_eocd + 32)); // Total entries
460 cd_size = le64(zip64_eocd + 40);
461 cd_start = p - (fileblocks->filesize - le64(zip64_eocd + 48));
462
463 assert(cd_start + cd_size == zip64_eocd);
464
465 assertEqualInt(le64(zip64_eocd + 48) // Start of CD
466 + cd_size
467 + 56 // Size of Zip64 EOCD
468 + 20 // Size of Zip64 locator
469 + 22, // Size of EOCD
470 fileblocks->filesize);
471
472 // TODO: Scan entire Central Directory, sanity-check all data
473 assertEqualMem(cd_start, "PK\001\002", 4);
474
475 fileblocks_free(fileblocks);
476 free(buff);
477 free(nulldata);
478 }
479