1 /*-
2 * Copyright (c) 2003-2007 Tim Kientzle
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25 #include "test.h"
26
27 #include <errno.h>
28 #include <stdlib.h>
29 #include <string.h>
30
31 /*
32 * This is a somewhat tricky test that verifies the ability to
33 * write and read very large entries to tar archives. It
34 * writes entries from 2GB up to 1TB to an archive in memory.
35 * The memory storage here carefully avoids actually storing
36 * any part of the file bodies, so it runs very quickly and requires
37 * very little memory. If you're willing to wait a few minutes,
38 * you should be able to exercise petabyte entries with this code.
39 */
40
41 /*
42 * Each file is built up by duplicating the following block.
43 */
44 static size_t filedatasize;
45 static void *filedata;
46
47 /*
48 * We store the archive as blocks of data generated by libarchive,
49 * each possibly followed by bytes of file data.
50 */
51 struct memblock {
52 struct memblock *next;
53 size_t size;
54 void *buff;
55 int64_t filebytes;
56 };
57
58 /*
59 * The total memory store is just a list of memblocks plus
60 * some accounting overhead.
61 */
62 struct memdata {
63 int64_t filebytes;
64 void *buff;
65 struct memblock *first;
66 struct memblock *last;
67 };
68
69 /* The following size definitions simplify things below. */
70 #define KB ((int64_t)1024)
71 #define MB ((int64_t)1024 * KB)
72 #define GB ((int64_t)1024 * MB)
73 #define TB ((int64_t)1024 * GB)
74
75 static int64_t memory_read_skip(struct archive *, void *, int64_t request);
76 static ssize_t memory_read(struct archive *, void *, const void **buff);
77 static ssize_t memory_write(struct archive *, void *, const void *, size_t);
78
79
80 static ssize_t
memory_write(struct archive * a,void * _private,const void * buff,size_t size)81 memory_write(struct archive *a, void *_private, const void *buff, size_t size)
82 {
83 struct memdata *private = _private;
84 struct memblock *block;
85
86 (void)a;
87
88 /*
89 * Since libarchive tries to behave in a zero-copy manner, if
90 * you give a pointer to filedata to the library, a pointer
91 * into that data will (usually) pop out here. This way, we
92 * can tell the difference between filedata and library header
93 * and metadata.
94 */
95 if ((const char *)filedata <= (const char *)buff
96 && (const char *)buff < (const char *)filedata + filedatasize) {
97 /* We don't need to store a block of file data. */
98 private->last->filebytes += (int64_t)size;
99 } else {
100 /* Yes, we're assuming the very first write is metadata. */
101 /* It's header or metadata, copy and save it. */
102 block = malloc(sizeof(*block));
103 memset(block, 0, sizeof(*block));
104 block->size = size;
105 block->buff = malloc(size);
106 memcpy(block->buff, buff, size);
107 if (private->last == NULL) {
108 private->first = private->last = block;
109 } else {
110 private->last->next = block;
111 private->last = block;
112 }
113 block->next = NULL;
114 }
115 return ((long)size);
116 }
117
118 static ssize_t
memory_read(struct archive * a,void * _private,const void ** buff)119 memory_read(struct archive *a, void *_private, const void **buff)
120 {
121 struct memdata *private = _private;
122 struct memblock *block;
123 ssize_t size;
124
125 (void)a;
126
127 free(private->buff);
128 private->buff = NULL;
129 if (private->first == NULL) {
130 private->last = NULL;
131 return (ARCHIVE_EOF);
132 }
133 if (private->filebytes > 0) {
134 /*
135 * We're returning file bytes, simulate it by
136 * passing blocks from the template data.
137 */
138 if (private->filebytes > (int64_t)filedatasize)
139 size = (ssize_t)filedatasize;
140 else
141 size = (ssize_t)private->filebytes;
142 private->filebytes -= size;
143 *buff = filedata;
144 } else {
145 /*
146 * We need to get some real data to return.
147 */
148 block = private->first;
149 private->first = block->next;
150 size = (ssize_t)block->size;
151 if (block->buff != NULL) {
152 private->buff = block->buff;
153 *buff = block->buff;
154 } else {
155 private->buff = NULL;
156 *buff = filedata;
157 }
158 private->filebytes = block->filebytes;
159 free(block);
160 }
161 return (size);
162 }
163
164
165 static int64_t
memory_read_skip(struct archive * a,void * _private,int64_t skip)166 memory_read_skip(struct archive *a, void *_private, int64_t skip)
167 {
168 struct memdata *private = _private;
169
170 (void)a;
171
172 if (private->first == NULL) {
173 private->last = NULL;
174 return (0);
175 }
176 if (private->filebytes > 0) {
177 if (private->filebytes < skip)
178 skip = (off_t)private->filebytes;
179 private->filebytes -= skip;
180 } else {
181 skip = 0;
182 }
183 return (skip);
184 }
185
DEFINE_TEST(test_tar_large)186 DEFINE_TEST(test_tar_large)
187 {
188 /* The sizes of the entries we're going to generate. */
189 static int64_t tests[] = {
190 /* Test for 32-bit signed overflow. */
191 2 * GB - 1, 2 * GB, 2 * GB + 1,
192 /* Test for 32-bit unsigned overflow. */
193 4 * GB - 1, 4 * GB, 4 * GB + 1,
194 /* 8GB is the "official" max for ustar. */
195 8 * GB - 1, 8 * GB, 8 * GB + 1,
196 /* Bend ustar a tad and you can get 64GB (12 octal digits). */
197 64 * GB - 1, 64 * GB,
198 /* And larger entries that require non-ustar extensions. */
199 256 * GB, 1 * TB, 0 };
200 int i;
201 char namebuff[64];
202 struct memdata memdata;
203 struct archive_entry *ae;
204 struct archive *a;
205 int64_t filesize;
206 size_t writesize;
207
208 filedatasize = (size_t)(1 * MB);
209 filedata = malloc(filedatasize);
210 memset(filedata, 0xAA, filedatasize);
211 memset(&memdata, 0, sizeof(memdata));
212
213 /*
214 * Open an archive for writing.
215 */
216 a = archive_write_new();
217 archive_write_set_format_pax_restricted(a);
218 archive_write_set_bytes_per_block(a, 0); /* No buffering. */
219 archive_write_open(a, &memdata, NULL, memory_write, NULL);
220
221 /*
222 * Write a series of large files to it.
223 */
224 for (i = 0; tests[i] != 0; i++) {
225 assert((ae = archive_entry_new()) != NULL);
226 snprintf(namebuff, sizeof(namebuff), "file_%d", i);
227 archive_entry_copy_pathname(ae, namebuff);
228 archive_entry_set_mode(ae, S_IFREG | 0755);
229 filesize = tests[i];
230
231 archive_entry_set_size(ae, filesize);
232
233 assertA(0 == archive_write_header(a, ae));
234 archive_entry_free(ae);
235
236 /*
237 * Write the actual data to the archive.
238 */
239 while (filesize > 0) {
240 writesize = filedatasize;
241 if ((int64_t)writesize > filesize)
242 writesize = (size_t)filesize;
243 assertA((int)writesize
244 == archive_write_data(a, filedata, writesize));
245 filesize -= writesize;
246 }
247 }
248
249 assert((ae = archive_entry_new()) != NULL);
250 archive_entry_copy_pathname(ae, "lastfile");
251 archive_entry_set_mode(ae, S_IFREG | 0755);
252 assertA(0 == archive_write_header(a, ae));
253 archive_entry_free(ae);
254
255
256 /* Close out the archive. */
257 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
258 assertEqualInt(ARCHIVE_OK, archive_write_free(a));
259
260 /*
261 * Open the same archive for reading.
262 */
263 a = archive_read_new();
264 archive_read_support_format_tar(a);
265 archive_read_open2(a, &memdata, NULL,
266 memory_read, memory_read_skip, NULL);
267
268 /*
269 * Read entries back.
270 */
271 for (i = 0; tests[i] > 0; i++) {
272 assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
273 snprintf(namebuff, sizeof(namebuff), "file_%d", i);
274 assertEqualString(namebuff, archive_entry_pathname(ae));
275 assert(tests[i] == archive_entry_size(ae));
276 }
277 assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
278 assertEqualString("lastfile", archive_entry_pathname(ae));
279
280 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
281
282 /* Close out the archive. */
283 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
284 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
285
286 free(memdata.buff);
287 free(filedata);
288 }
289