1 /*-
2 * Copyright (c) 2003-2007,2013 Tim Kientzle
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25 #include "test.h"
26
27 #include <errno.h>
28 #include <stdlib.h>
29 #include <string.h>
30
31 /*
32 * This is a somewhat tricky test that verifies the ability to
33 * write and read very large entries to zip archives.
34 *
35 * See test_tar_large.c for more information about the machinery
36 * being used here.
37 */
38
39 static size_t nullsize;
40 static void *nulldata;
41
42 struct fileblock {
43 struct fileblock *next;
44 int size;
45 void *buff;
46 int64_t gap_size; /* Size of following gap */
47 };
48
49 struct fileblocks {
50 int64_t filesize;
51 int64_t fileposition;
52 int64_t gap_remaining;
53 void *buff;
54 struct fileblock *first;
55 struct fileblock *current;
56 struct fileblock *last;
57 };
58
59 /* The following size definitions simplify things below. */
60 #define KB ((int64_t)1024)
61 #define MB ((int64_t)1024 * KB)
62 #define GB ((int64_t)1024 * MB)
63 #define TB ((int64_t)1024 * GB)
64
65 static ssize_t
memory_write(struct archive * a,void * _private,const void * buff,size_t size)66 memory_write(struct archive *a, void *_private, const void *buff, size_t size)
67 {
68 struct fileblocks *private = _private;
69 struct fileblock *block;
70
71 (void)a;
72
73 if ((const char *)nulldata <= (const char *)buff
74 && (const char *)buff < (const char *)nulldata + nullsize) {
75 /* We don't need to store a block of gap data. */
76 private->last->gap_size += (int64_t)size;
77 } else {
78 /* Yes, we're assuming the very first write is metadata. */
79 /* It's header or metadata, copy and save it. */
80 block = malloc(sizeof(*block));
81 memset(block, 0, sizeof(*block));
82 block->size = (int)size;
83 block->buff = malloc(size);
84 memcpy(block->buff, buff, size);
85 if (private->last == NULL) {
86 private->first = private->last = block;
87 } else {
88 private->last->next = block;
89 private->last = block;
90 }
91 block->next = NULL;
92 }
93 private->filesize += size;
94 return ((long)size);
95 }
96
97 static ssize_t
memory_read(struct archive * a,void * _private,const void ** buff)98 memory_read(struct archive *a, void *_private, const void **buff)
99 {
100 struct fileblocks *private = _private;
101 ssize_t size;
102
103 (void)a;
104
105 while (private->current != NULL && private->buff == NULL && private->gap_remaining == 0) {
106 private->current = private->current->next;
107 if (private->current != NULL) {
108 private->buff = private->current->buff;
109 private->gap_remaining = private->current->gap_size;
110 }
111 }
112
113 if (private->current == NULL)
114 return (0);
115
116 /* If there's real data, return that. */
117 if (private->buff != NULL) {
118 *buff = private->buff;
119 size = ((char *)private->current->buff + private->current->size)
120 - (char *)private->buff;
121 private->buff = NULL;
122 private->fileposition += size;
123 return (size);
124 }
125
126 /* Big gap: too big to return all at once, so just return some. */
127 if (private->gap_remaining > (int64_t)nullsize) {
128 private->gap_remaining -= nullsize;
129 *buff = nulldata;
130 private->fileposition += nullsize;
131 return (nullsize);
132 }
133
134 /* Small gap: finish the gap and prep for next block. */
135 if (private->gap_remaining > 0) {
136 size = (ssize_t)private->gap_remaining;
137 *buff = nulldata;
138 private->gap_remaining = 0;
139 private->fileposition += size;
140
141 private->current = private->current->next;
142 if (private->current != NULL) {
143 private->buff = private->current->buff;
144 private->gap_remaining = private->current->gap_size;
145 }
146
147 return (size);
148 }
149 fprintf(stderr, "\n\n\nInternal failure\n\n\n");
150 exit(1);
151 }
152
153 static int
memory_read_open(struct archive * a,void * _private)154 memory_read_open(struct archive *a, void *_private)
155 {
156 struct fileblocks *private = _private;
157
158 (void)a; /* UNUSED */
159
160 private->current = private->first;
161 private->fileposition = 0;
162 if (private->current != NULL) {
163 private->buff = private->current->buff;
164 private->gap_remaining = private->current->gap_size;
165 }
166 return (ARCHIVE_OK);
167 }
168
169 static int64_t
memory_read_seek(struct archive * a,void * _private,int64_t offset,int whence)170 memory_read_seek(struct archive *a, void *_private, int64_t offset, int whence)
171 {
172 struct fileblocks *private = _private;
173
174 (void)a;
175 if (whence == SEEK_END) {
176 offset = private->filesize + offset;
177 } else if (whence == SEEK_CUR) {
178 offset = private->fileposition + offset;
179 }
180
181 if (offset < 0) {
182 fprintf(stderr, "\n\n\nInternal failure: negative seek\n\n\n");
183 exit(1);
184 }
185
186 /* We've converted the request into a SEEK_SET. */
187 private->fileposition = offset;
188
189 /* Walk the block list to find the new position. */
190 offset = 0;
191 private->current = private->first;
192 while (private->current != NULL) {
193 if (offset + private->current->size > private->fileposition) {
194 /* Position is in this block. */
195 private->buff = (char *)private->current->buff
196 + private->fileposition - offset;
197 private->gap_remaining = private->current->gap_size;
198 return private->fileposition;
199 }
200 offset += private->current->size;
201 if (offset + private->current->gap_size > private->fileposition) {
202 /* Position is in this gap. */
203 private->buff = NULL;
204 private->gap_remaining = private->current->gap_size
205 - (private->fileposition - offset);
206 return private->fileposition;
207 }
208 offset += private->current->gap_size;
209 /* Skip to next block. */
210 private->current = private->current->next;
211 }
212 if (private->fileposition == private->filesize) {
213 return private->fileposition;
214 }
215 fprintf(stderr, "\n\n\nInternal failure: over-sized seek\n\n\n");
216 exit(1);
217 }
218
219 static int64_t
memory_read_skip(struct archive * a,void * _private,int64_t skip)220 memory_read_skip(struct archive *a, void *_private, int64_t skip)
221 {
222 struct fileblocks *private = _private;
223 int64_t old_position = private->fileposition;
224 int64_t new_position = memory_read_seek(a, _private, skip, SEEK_CUR);
225 return (new_position - old_position);
226 }
227
228 static struct fileblocks *
fileblocks_new(void)229 fileblocks_new(void)
230 {
231 struct fileblocks *fileblocks;
232
233 fileblocks = calloc(1, sizeof(struct fileblocks));
234 return fileblocks;
235 }
236
237 static void
fileblocks_free(struct fileblocks * fileblocks)238 fileblocks_free(struct fileblocks *fileblocks)
239 {
240 while (fileblocks->first != NULL) {
241 struct fileblock *b = fileblocks->first;
242 fileblocks->first = fileblocks->first->next;
243 free(b->buff);
244 free(b);
245 }
246 free(fileblocks);
247 }
248
249
250 /* The sizes of the entries we're going to generate. */
251 static int64_t test_sizes[] = {
252 /* Test for 32-bit signed overflow. */
253 2 * GB - 1, 2 * GB, 2 * GB + 1,
254 /* Test for 32-bit unsigned overflow. */
255 4 * GB - 1, 4 * GB, 4 * GB + 1,
256 /* And one larger sample */
257 5 * GB,
258 0
259 };
260
261
262 static void
verify_large_zip(struct archive * a,struct fileblocks * fileblocks,int seeking)263 verify_large_zip(struct archive *a, struct fileblocks *fileblocks, int seeking)
264 {
265 char namebuff[64];
266 struct archive_entry *ae;
267 int i;
268
269 (void)seeking; /* UNUSED */
270
271 assertEqualIntA(a, ARCHIVE_OK,
272 archive_read_set_options(a, "zip:ignorecrc32"));
273 assertEqualIntA(a, ARCHIVE_OK,
274 archive_read_set_open_callback(a, memory_read_open));
275 assertEqualIntA(a, ARCHIVE_OK,
276 archive_read_set_read_callback(a, memory_read));
277 assertEqualIntA(a, ARCHIVE_OK,
278 archive_read_set_skip_callback(a, memory_read_skip));
279 assertEqualIntA(a, ARCHIVE_OK,
280 archive_read_set_seek_callback(a, memory_read_seek));
281 assertEqualIntA(a, ARCHIVE_OK,
282 archive_read_set_callback_data(a, fileblocks));
283 assertEqualIntA(a, ARCHIVE_OK, archive_read_open1(a));
284
285 /*
286 * Read entries back.
287 */
288 for (i = 0; test_sizes[i] > 0; i++) {
289 assertEqualIntA(a, ARCHIVE_OK,
290 archive_read_next_header(a, &ae));
291 snprintf(namebuff, sizeof(namebuff), "file_%d", i);
292 assertEqualString(namebuff, archive_entry_pathname(ae));
293 if (seeking) {
294 assert(archive_entry_size_is_set(ae));
295 assertEqualInt(test_sizes[i], archive_entry_size(ae));
296 } else {
297 assertEqualInt(0, archive_entry_size_is_set(ae));
298 }
299 /* TODO: Read to end of data, verify length */
300 }
301 assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
302 assertEqualString("lastfile", archive_entry_pathname(ae));
303
304 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
305
306 /* Close out the archive. */
307 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
308 }
309
DEFINE_TEST(test_write_format_zip_large)310 DEFINE_TEST(test_write_format_zip_large)
311 {
312 int i;
313 char namebuff[64];
314 struct fileblocks *fileblocks = fileblocks_new();
315 struct archive_entry *ae;
316 struct archive *a;
317 const char *p;
318 const char *cd_start, *zip64_eocd, *zip64_locator, *eocd;
319 int64_t cd_size;
320 char *buff;
321 int64_t filesize;
322 size_t writesize, buffsize, s;
323
324 nullsize = (size_t)(1 * MB);
325 nulldata = malloc(nullsize);
326 memset(nulldata, 0xAA, nullsize);
327
328 /*
329 * Open an archive for writing.
330 */
331 a = archive_write_new();
332 archive_write_set_format_zip(a);
333 /* TODO: Repeat this entire test suite with default compression */
334 assertEqualIntA(a, ARCHIVE_OK,
335 archive_write_set_options(a, "zip:compression=store"));
336 assertEqualIntA(a, ARCHIVE_OK,
337 archive_write_set_options(a, "zip:fakecrc32"));
338 assertEqualIntA(a, ARCHIVE_OK,
339 archive_write_set_bytes_per_block(a, 0)); /* No buffering. */
340 assertEqualIntA(a, ARCHIVE_OK,
341 archive_write_open(a, fileblocks, NULL, memory_write, NULL));
342
343 /*
344 * Write a series of large files to it.
345 */
346 for (i = 0; test_sizes[i] != 0; i++) {
347 assert((ae = archive_entry_new()) != NULL);
348 snprintf(namebuff, sizeof(namebuff), "file_%d", i);
349 archive_entry_copy_pathname(ae, namebuff);
350 archive_entry_set_mode(ae, S_IFREG | 0755);
351 filesize = test_sizes[i];
352 archive_entry_set_size(ae, filesize);
353
354 assertEqualIntA(a, ARCHIVE_OK,
355 archive_write_header(a, ae));
356 archive_entry_free(ae);
357
358 /*
359 * Write the actual data to the archive.
360 */
361 while (filesize > 0) {
362 writesize = nullsize;
363 if ((int64_t)writesize > filesize)
364 writesize = (size_t)filesize;
365 assertEqualIntA(a, (int)writesize,
366 (int)archive_write_data(a, nulldata, writesize));
367 filesize -= writesize;
368 }
369 }
370
371 assert((ae = archive_entry_new()) != NULL);
372 archive_entry_copy_pathname(ae, "lastfile");
373 archive_entry_set_mode(ae, S_IFREG | 0755);
374 assertA(0 == archive_write_header(a, ae));
375 archive_entry_free(ae);
376
377 /* Close out the archive. */
378 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
379 assertEqualInt(ARCHIVE_OK, archive_write_free(a));
380
381 /*
382 * Read back with seeking reader:
383 */
384 a = archive_read_new();
385 assertEqualIntA(a, ARCHIVE_OK,
386 archive_read_support_format_zip_seekable(a));
387 verify_large_zip(a, fileblocks, 1);
388 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
389
390 /*
391 * Read back with streaming reader:
392 */
393 a = archive_read_new();
394 assertEqualIntA(a, ARCHIVE_OK,
395 archive_read_support_format_zip_streamable(a));
396 verify_large_zip(a, fileblocks, 0);
397 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
398
399 /*
400 * Manually verify some of the final bytes of the archives.
401 */
402 /* Collect the final bytes together */
403 #define FINAL_SIZE 8192
404 buff = malloc(FINAL_SIZE);
405 buffsize = 0;
406 memory_read_open(NULL, fileblocks);
407 memory_read_seek(NULL, fileblocks, -FINAL_SIZE, SEEK_END);
408 while ((s = memory_read(NULL, fileblocks, (const void **)&p)) > 0) {
409 memcpy(buff + buffsize, p, s);
410 buffsize += s;
411 }
412 assertEqualInt(buffsize, FINAL_SIZE);
413
414 p = buff + buffsize;
415
416 /* Verify regular end-of-central-directory record */
417 eocd = p - 22;
418 assertEqualMem(eocd, "PK\005\006\0\0\0\0", 8);
419 assertEqualMem(eocd + 8, "\010\0\010\0", 4); /* 8 entries total */
420 cd_size = i4le(eocd + 12);
421 /* Start of CD offset should be 0xffffffff */
422 assertEqualMem(eocd + 16, "\xff\xff\xff\xff", 4);
423 assertEqualMem(eocd + 20, "\0\0", 2); /* No Zip comment */
424
425 /* Verify Zip64 locator */
426 zip64_locator = p - 42;
427 assertEqualMem(zip64_locator, "PK\006\007\0\0\0\0", 8);
428 zip64_eocd = p - (fileblocks->filesize - i8le(zip64_locator + 8));
429 assertEqualMem(zip64_locator + 16, "\001\0\0\0", 4);
430
431 /* Verify Zip64 end-of-cd record. */
432 assert(zip64_eocd == p - 98);
433 assertEqualMem(zip64_eocd, "PK\006\006", 4);
434 assertEqualInt(44, i8le(zip64_eocd + 4)); // Size of EoCD record - 12
435 assertEqualMem(zip64_eocd + 12, "\055\0", 2); // Made by version: 45
436 assertEqualMem(zip64_eocd + 14, "\055\0", 2); // Requires version: 45
437 assertEqualMem(zip64_eocd + 16, "\0\0\0\0", 4); // This disk
438 assertEqualMem(zip64_eocd + 20, "\0\0\0\0", 4); // Total disks
439 assertEqualInt(8, i8le(zip64_eocd + 24)); // Entries on this disk
440 assertEqualInt(8, i8le(zip64_eocd + 32)); // Total entries
441 cd_size = i8le(zip64_eocd + 40);
442 cd_start = p - (fileblocks->filesize - i8le(zip64_eocd + 48));
443
444 assert(cd_start + cd_size == zip64_eocd);
445
446 assertEqualInt(i8le(zip64_eocd + 48) // Start of CD
447 + cd_size
448 + 56 // Size of Zip64 EOCD
449 + 20 // Size of Zip64 locator
450 + 22, // Size of EOCD
451 fileblocks->filesize);
452
453 // TODO: Scan entire Central Directory, sanity-check all data
454 assertEqualMem(cd_start, "PK\001\002", 4);
455
456 fileblocks_free(fileblocks);
457 free(buff);
458 free(nulldata);
459 }
460