1 /*-
2 * Copyright (c) 2014 Sebastian Freundt
3 * Author: Sebastian Freundt <devel@fresse.org>
4 *
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "archive_platform.h"
29
30 #ifdef HAVE_ERRNO_H
31 #include <errno.h>
32 #endif
33 #include <stdio.h>
34 #ifdef HAVE_STDLIB_H
35 #include <stdlib.h>
36 #endif
37 #ifdef HAVE_STRING_H
38 #include <string.h>
39 #endif
40 #ifdef HAVE_TIME_H
41 #include <time.h>
42 #endif
43
44 #include "archive.h"
45 #include "archive_entry.h"
46 #include "archive_entry_locale.h"
47 #include "archive_private.h"
48 #include "archive_random_private.h"
49 #include "archive_write_private.h"
50 #include "archive_write_set_format_private.h"
51
52 /*
53 * Overview of the WARC writer:
54 *
55 * This writer emits WARC/1.0 resource records for regular files. It
56 * writes a warcinfo record once by default, unless the omit-warcinfo
57 * option is used. Each resource record gets a WARC-Target-URI from
58 * the entry pathname, a generated WARC-Record-ID, and a mandatory
59 * Content-Length header.
60 */
61
62 struct warc {
63 unsigned int omit_warcinfo:1;
64
65 time_t now;
66 mode_t filetype;
67 /* Remaining bytes to write for the current entry */
68 uint64_t entry_bytes_remaining;
69 };
70
71 #define WARC_HEADER_MAX_SIZE 512
72
73 static const char warcinfo_payload[] =
74 "software: libarchive/" ARCHIVE_VERSION_ONLY_STRING "\r\n"
75 "format: WARC file version 1.0\r\n";
76
77 enum warc_type {
78 WARC_TYPE_NONE,
79 /* WARC info */
80 WARC_TYPE_INFO,
81 /* Metadata */
82 WARC_TYPE_METADATA,
83 /* Resource */
84 WARC_TYPE_RESOURCE,
85 /* Request, unsupported */
86 WARC_TYPE_REQUEST,
87 /* Response, unsupported by this writer */
88 WARC_TYPE_RESPONSE,
89 /* Revisit, unsupported */
90 WARC_TYPE_REVISIT,
91 /* Conversion, unsupported */
92 WARC_TYPE_CONVERSION,
93 /* Continuation, currently unsupported */
94 WARC_TYPE_CONTINUATION,
95 WARC_TYPE_LAST
96 };
97
98 struct warc_header {
99 enum warc_type type;
100 const char *target_uri;
101 const char *record_id;
102 time_t record_time;
103 time_t modification_time;
104 const char *content_type;
105 uint64_t content_length;
106 };
107
108 struct warc_uuid {
109 unsigned int value[4U];
110 };
111
112 static int archive_write_warc_options(struct archive_write *,
113 const char *, const char *);
114 static int archive_write_warc_header(struct archive_write *,
115 struct archive_entry *);
116 static ssize_t archive_write_warc_data(struct archive_write *, const void *,
117 size_t);
118 static int archive_write_warc_finish_entry(struct archive_write *);
119 static int archive_write_warc_close(struct archive_write *);
120 static int archive_write_warc_free(struct archive_write *);
121
122 static void warc_format_time(struct archive_string *, const char *, time_t);
123 static ssize_t warc_populate_header(struct archive_string *, size_t,
124 struct warc_header);
125 static void warc_generate_uuid(struct warc_uuid *);
126
127 /*
128 * Set output format to ISO 28500 (aka WARC) format.
129 */
130 int
archive_write_set_format_warc(struct archive * _a)131 archive_write_set_format_warc(struct archive *_a)
132 {
133 struct archive_write *a = (struct archive_write *)_a;
134 struct warc *warc;
135
136 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC,
137 ARCHIVE_STATE_NEW, "archive_write_set_format_warc");
138
139 /* If another format was already registered, unregister it. */
140 (void)__archive_write_unregister_format(a);
141
142 warc = malloc(sizeof(*warc));
143 if (warc == NULL) {
144 archive_set_error(&a->archive, ENOMEM,
145 "Can't allocate warc data");
146 return (ARCHIVE_FATAL);
147 }
148 /* Emit a warcinfo record by default. */
149 warc->omit_warcinfo = 0U;
150 /* Use the current time for WARC-Date values. */
151 warc->now = time(NULL);
152 /* Reset file type information. */
153 warc->filetype = 0;
154
155 a->format_data = warc;
156 a->format_name = "WARC/1.0";
157 a->format_options = archive_write_warc_options;
158 a->format_write_header = archive_write_warc_header;
159 a->format_write_data = archive_write_warc_data;
160 a->format_close = archive_write_warc_close;
161 a->format_free = archive_write_warc_free;
162 a->format_finish_entry = archive_write_warc_finish_entry;
163 a->archive.archive_format = ARCHIVE_FORMAT_WARC;
164 a->archive.archive_format_name = "WARC/1.0";
165 return (ARCHIVE_OK);
166 }
167
168 static int
archive_write_warc_options(struct archive_write * a,const char * key,const char * val)169 archive_write_warc_options(struct archive_write *a, const char *key,
170 const char *val)
171 {
172 struct warc *warc = a->format_data;
173
174 if (strcmp(key, "omit-warcinfo") == 0) {
175 if (val == NULL || strcmp(val, "true") == 0) {
176 /* Option accepted. */
177 warc->omit_warcinfo = 1U;
178 return (ARCHIVE_OK);
179 }
180 }
181
182 /* ARCHIVE_WARN tells the options supervisor that this option was not
183 * handled here. It will report an error if no module uses it. */
184 return (ARCHIVE_WARN);
185 }
186
187 static int
archive_write_warc_header(struct archive_write * a,struct archive_entry * entry)188 archive_write_warc_header(struct archive_write *a, struct archive_entry *entry)
189 {
190 struct warc *warc = a->format_data;
191 struct archive_string header;
192
193 /* Emit the warcinfo record if needed. */
194 if (!warc->omit_warcinfo) {
195 ssize_t header_size;
196 int ret;
197 struct warc_header warcinfo_header = {
198 WARC_TYPE_INFO,
199 /* URI */NULL,
200 /* Record ID */NULL,
201 /* Record time */0,
202 /* Modified time */0,
203 /* Content type */"application/warc-fields",
204 /* Content length */sizeof(warcinfo_payload) - 1U,
205 };
206 warcinfo_header.record_time = warc->now;
207 warcinfo_header.modification_time = warc->now;
208
209 archive_string_init(&header);
210 header_size = warc_populate_header(&header,
211 WARC_HEADER_MAX_SIZE, warcinfo_header);
212 if (header_size < 0) {
213 archive_string_free(&header);
214 archive_set_error(&a->archive,
215 ARCHIVE_ERRNO_FILE_FORMAT,
216 "Cannot archive warcinfo record");
217 return (ARCHIVE_FAILED);
218 }
219
220 /* Reuse the header buffer for the warcinfo payload. */
221 archive_strncat(&header, warcinfo_payload,
222 sizeof(warcinfo_payload) - 1);
223
224 /* Append the end-of-record indicator. */
225 archive_strncat(&header, "\r\n\r\n", 4);
226
227 /* Write the warcinfo record to the output stream. */
228 ret = __archive_write_output(a, header.s,
229 archive_strlen(&header));
230 if (ret != ARCHIVE_OK) {
231 archive_string_free(&header);
232 return (ret);
233 }
234
235 /* Mark the file header as written. */
236 warc->omit_warcinfo = 1U;
237 archive_string_free(&header);
238 }
239
240 if (archive_entry_pathname(entry) == NULL) {
241 archive_set_error(&a->archive, EINVAL,
242 "Invalid filename");
243 return (ARCHIVE_WARN);
244 }
245
246 warc->filetype = archive_entry_filetype(entry);
247 warc->entry_bytes_remaining = 0U;
248 if (warc->filetype == AE_IFREG) {
249 struct warc_header resource_header = {
250 WARC_TYPE_RESOURCE,
251 /* URI */NULL,
252 /* Record ID */NULL,
253 /* Record time */0,
254 /* Modified time */0,
255 /* Content type */NULL,
256 /* Content length */0,
257 };
258 ssize_t header_size;
259 int ret;
260 int64_t size;
261
262 resource_header.target_uri = archive_entry_pathname(entry);
263 resource_header.record_time = warc->now;
264 resource_header.modification_time = archive_entry_mtime(entry);
265 if (!archive_entry_size_is_set(entry)) {
266 archive_set_error(&a->archive, -1,
267 "Size required");
268 return (ARCHIVE_FAILED);
269 }
270 size = archive_entry_size(entry);
271 if (size < 0) {
272 archive_set_error(&a->archive, -1,
273 "Size required");
274 return (ARCHIVE_FAILED);
275 }
276 resource_header.content_length = (uint64_t)size;
277
278 archive_string_init(&header);
279 header_size = warc_populate_header(&header,
280 WARC_HEADER_MAX_SIZE, resource_header);
281 if (header_size < 0) {
282 /* Header generation failed. */
283 archive_string_free(&header);
284 archive_set_error(&a->archive,
285 ARCHIVE_ERRNO_FILE_FORMAT,
286 "WARC resource header is too large");
287 return (ARCHIVE_FATAL);
288 }
289 /* Append the header to the output stream. */
290 ret = __archive_write_output(a, header.s, header_size);
291 if (ret != ARCHIVE_OK) {
292 archive_string_free(&header);
293 return (ret);
294 }
295 /* Save the remaining size for subsequent data callbacks. */
296 warc->entry_bytes_remaining = resource_header.content_length;
297 archive_string_free(&header);
298 return (ARCHIVE_OK);
299 }
300 /* Report unsupported file types through the common helper. */
301 __archive_write_entry_filetype_unsupported(
302 &a->archive, entry, "WARC");
303 return (ARCHIVE_FAILED);
304 }
305
306 static ssize_t
archive_write_warc_data(struct archive_write * a,const void * buf,size_t len)307 archive_write_warc_data(struct archive_write *a, const void *buf, size_t len)
308 {
309 struct warc *warc = a->format_data;
310
311 if (warc->filetype == AE_IFREG) {
312 int ret;
313
314 /* Never write more bytes than announced. */
315 if ((uint64_t)len > warc->entry_bytes_remaining) {
316 len = (size_t)warc->entry_bytes_remaining;
317 }
318
319 /* Write the entry data. */
320 ret = __archive_write_output(a, buf, len);
321 if (ret != ARCHIVE_OK) {
322 return (ret);
323 }
324 warc->entry_bytes_remaining -= len;
325 }
326 return (len);
327 }
328
329 static int
archive_write_warc_finish_entry(struct archive_write * a)330 archive_write_warc_finish_entry(struct archive_write *a)
331 {
332 static const char end_of_record[] = "\r\n\r\n";
333 struct warc *warc = a->format_data;
334
335 if (warc->filetype == AE_IFREG) {
336 int ret;
337
338 if (warc->entry_bytes_remaining != 0U) {
339 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
340 "WARC entry is shorter than Content-Length");
341 return (ARCHIVE_FATAL);
342 }
343
344 ret = __archive_write_output(a, end_of_record,
345 sizeof(end_of_record) - 1U);
346 if (ret != ARCHIVE_OK) {
347 return (ret);
348 }
349 }
350 /* Reset file type information. */
351 warc->filetype = 0;
352 return (ARCHIVE_OK);
353 }
354
355 static int
archive_write_warc_close(struct archive_write * a)356 archive_write_warc_close(struct archive_write *a)
357 {
358 (void)a; /* UNUSED */
359 return (ARCHIVE_OK);
360 }
361
362 static int
archive_write_warc_free(struct archive_write * a)363 archive_write_warc_free(struct archive_write *a)
364 {
365 struct warc *warc = a->format_data;
366
367 free(warc);
368 a->format_data = NULL;
369 return (ARCHIVE_OK);
370 }
371
372 /* Like strftime(3), but for time_t objects. */
373 static void
warc_format_time(struct archive_string * str,const char * format,time_t t)374 warc_format_time(struct archive_string *str, const char *format, time_t t)
375 {
376 struct tm *tm;
377 #if defined(HAVE_GMTIME_R) || defined(HAVE_GMTIME_S)
378 struct tm tm_storage;
379 #endif
380 char time_string[100];
381 size_t len;
382
383 #if defined(HAVE_GMTIME_S)
384 tm = gmtime_s(&tm_storage, &t) ? NULL : &tm_storage;
385 #elif defined(HAVE_GMTIME_R)
386 tm = gmtime_r(&t, &tm_storage);
387 #else
388 tm = gmtime(&t);
389 #endif
390 if (tm == NULL)
391 return;
392 /* Let strftime() handle the actual formatting. */
393 len = strftime(time_string, sizeof(time_string) - 1, format, tm);
394 archive_strncat(str, time_string, len);
395 }
396
397 static ssize_t
warc_populate_header(struct archive_string * header_string,size_t max_size,struct warc_header header)398 warc_populate_header(struct archive_string *header_string, size_t max_size,
399 struct warc_header header)
400 {
401 static const char version[] = "WARC/1.0\r\n";
402 static const char * const record_types[WARC_TYPE_LAST] = {
403 NULL, "warcinfo", "metadata", "resource", NULL
404 };
405 char generated_record_id[48U];
406
407 if (header.type == WARC_TYPE_NONE || header.type > WARC_TYPE_RESOURCE) {
408 /* Invalid record type for this writer. */
409 return (-1);
410 }
411
412 archive_strcpy(header_string, version);
413
414 archive_string_sprintf(header_string, "WARC-Type: %s\r\n",
415 record_types[header.type]);
416
417 if (header.target_uri != NULL) {
418 const char *uri_prefix;
419 const char *scheme = strchr(header.target_uri, ':');
420
421 /* Check whether the value already contains ://. */
422 if (scheme != NULL && scheme[1U] == '/' && scheme[2U] == '/') {
423 /* Already has a scheme-style :// prefix. */
424 uri_prefix = "";
425 } else {
426 /* Prepend file:// for local paths. */
427 uri_prefix = "file://";
428 }
429 archive_string_sprintf(header_string,
430 "WARC-Target-URI: %s%s\r\n", uri_prefix,
431 header.target_uri);
432 }
433
434 /* Write WARC-Date from header.record_time. */
435 warc_format_time(header_string,
436 "WARC-Date: %Y-%m-%dT%H:%M:%SZ\r\n", header.record_time);
437
438 /* Also write Last-Modified from header.modification_time. */
439 warc_format_time(header_string,
440 "Last-Modified: %Y-%m-%dT%H:%M:%SZ\r\n",
441 header.modification_time);
442
443 if (header.record_id == NULL) {
444 /* Generate a record ID when one was not provided. */
445 struct warc_uuid uuid;
446
447 warc_generate_uuid(&uuid);
448 /* archive_string_sprintf() does not support minimum field widths, so
449 * use snprintf() for UUID formatting. */
450 #if defined(_WIN32) && !defined(__CYGWIN__) && \
451 !(defined(_MSC_VER) && _MSC_VER >= 1900)
452 #define snprintf _snprintf
453 #endif
454 snprintf(generated_record_id, sizeof(generated_record_id),
455 "<urn:uuid:%08x-%04x-%04x-%04x-%04x%08x>",
456 uuid.value[0U],
457 uuid.value[1U] >> 16U, uuid.value[1U] & 0xffffU,
458 uuid.value[2U] >> 16U, uuid.value[2U] & 0xffffU,
459 uuid.value[3U]);
460 header.record_id = generated_record_id;
461 }
462
463 /* WARC-Record-ID is mandatory. */
464 archive_string_sprintf(header_string, "WARC-Record-ID: %s\r\n",
465 header.record_id);
466
467 if (header.content_type != NULL) {
468 archive_string_sprintf(header_string, "Content-Type: %s\r\n",
469 header.content_type);
470 }
471
472 /* Content-Length is mandatory. */
473 archive_string_sprintf(header_string, "Content-Length: %ju\r\n",
474 (uintmax_t)header.content_length);
475 /* End of header. */
476 archive_strncat(header_string, "\r\n", 2);
477
478 return (archive_strlen(header_string) >= max_size) ?
479 -1 : (ssize_t)archive_strlen(header_string);
480 }
481
482 static void
warc_generate_uuid(struct warc_uuid * uuid)483 warc_generate_uuid(struct warc_uuid *uuid)
484 {
485 archive_random(uuid->value, sizeof(uuid->value));
486 /* Apply UUID version 4 rules. */
487 uuid->value[1U] &= 0xffff0fffU;
488 uuid->value[1U] |= 0x4000U;
489 uuid->value[2U] &= 0x3fffffffU;
490 uuid->value[2U] |= 0x80000000U;
491 }
492