1 /*-
2 * Copyright (c) 2014 Sebastian Freundt
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 #include "archive_platform.h"
27
28 /*
29 * An overview of WARC format:
30 *
31 * WARC files are laid out as a sequence of records. Each record has
32 * a text header followed by a content block whose size is given by
33 * Content-Length. This reader supports WARC/0.12 through WARC/1.0
34 * and was written using the final draft that became ISO 28500:2009:
35 * http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf
36 *
37 * This reader exposes resource and response records as regular files
38 * when they have a usable WARC-Target-URI. WARC-Date is exposed as
39 * ctime, and a Last-Modified record header is exposed as mtime when
40 * present.
41 *
42 * TODO: Real-world WARCs can contain resources at endpoints ending in
43 * a slash, for example http://bibnum.bnf.fr/warc/. Some responses
44 * include a Content-Location header that points to a Unix-compatible
45 * filename such as http://bibnum.bnf.fr/warc/index.html, but WARC does
46 * not require that convention and some sites do not follow it. Until
47 * archive options exist to control these entries, this reader skips
48 * them instead of creating directory endpoints as files.
49 */
50
51 #ifdef HAVE_SYS_STAT_H
52 #include <sys/stat.h>
53 #endif
54 #ifdef HAVE_CTYPE_H
55 #include <ctype.h>
56 #endif
57 #ifdef HAVE_ERRNO_H
58 #include <errno.h>
59 #endif
60 #ifdef HAVE_STDLIB_H
61 #include <stdlib.h>
62 #endif
63 #ifdef HAVE_STRING_H
64 #include <string.h>
65 #endif
66 #ifdef HAVE_TIME_H
67 #include <time.h>
68 #endif
69
70 #include "archive.h"
71 #include "archive_entry.h"
72 #include "archive_integer.h"
73 #include "archive_private.h"
74 #include "archive_read_private.h"
75
76 typedef enum {
77 WT_NONE,
78 /* WARC info */
79 WT_INFO,
80 /* Metadata */
81 WT_META,
82 /* Resource */
83 WT_RSRC,
84 /* Request, unsupported */
85 WT_REQ,
86 /* Response */
87 WT_RSP,
88 /* Revisit, unsupported */
89 WT_RVIS,
90 /* Conversion, unsupported */
91 WT_CONV,
92 /* Continuation, currently unsupported */
93 WT_CONT,
94 /* Invalid type */
95 LAST_WT
96 } warc_type_t;
97
98 typedef struct {
99 size_t len;
100 const char *str;
101 } warc_string_t;
102
103 typedef struct {
104 size_t len;
105 char *str;
106 } warc_strbuf_t;
107
108 struct warc {
109 /* Content length of the current record */
110 int64_t cntlen;
111 /* Bytes processed from the current record */
112 int64_t cntoff;
113 /* Bytes to consume before the next read */
114 int64_t unconsumed;
115
116 /* String pool */
117 warc_strbuf_t pool;
118 /* Previous version */
119 unsigned int pver;
120 /* Stringified format name */
121 struct archive_string sver;
122 };
123
124 static int archive_read_format_warc_bid(struct archive_read *, int);
125 static int archive_read_format_warc_cleanup(struct archive_read *);
126 static int archive_read_format_warc_read_data(struct archive_read *,
127 const void **, size_t *, int64_t *);
128 static int archive_read_format_warc_skip(struct archive_read *);
129 static int archive_read_format_warc_read_header(struct archive_read *,
130 struct archive_entry *);
131
132 /* Private routines */
133 static unsigned int warc_read_version(const char *, size_t);
134 static unsigned int warc_read_type(const char *, size_t);
135 static warc_string_t warc_read_uri(const char *, size_t);
136 static int64_t warc_read_length(const char *, size_t);
137 static time_t warc_read_date(const char *, size_t);
138 static time_t warc_read_last_modified(const char *, size_t);
139 static const char *warc_find_eoh(const char *, size_t);
140 static const char *warc_find_eol(const char *, size_t);
141
142 int
archive_read_support_format_warc(struct archive * _a)143 archive_read_support_format_warc(struct archive *_a)
144 {
145 struct archive_read *a = (struct archive_read *)_a;
146 struct warc *warc;
147 int r;
148
149 archive_check_magic(_a, ARCHIVE_READ_MAGIC,
150 ARCHIVE_STATE_NEW, "archive_read_support_format_warc");
151
152 if ((warc = calloc(1, sizeof(*warc))) == NULL) {
153 archive_set_error(&a->archive, ENOMEM,
154 "Can't allocate warc data");
155 return (ARCHIVE_FATAL);
156 }
157
158 r = __archive_read_register_format(a,
159 warc,
160 "warc",
161 archive_read_format_warc_bid,
162 NULL,
163 archive_read_format_warc_read_header,
164 archive_read_format_warc_read_data,
165 archive_read_format_warc_skip,
166 NULL,
167 archive_read_format_warc_cleanup,
168 NULL,
169 NULL);
170
171 if (r != ARCHIVE_OK) {
172 free(warc);
173 return (r);
174 }
175 return (ARCHIVE_OK);
176 }
177
178 static int
archive_read_format_warc_cleanup(struct archive_read * a)179 archive_read_format_warc_cleanup(struct archive_read *a)
180 {
181 struct warc *warc = a->format->data;
182
183 if (warc->pool.len > 0U) {
184 free(warc->pool.str);
185 }
186 archive_string_free(&warc->sver);
187 free(warc);
188 a->format->data = NULL;
189 return (ARCHIVE_OK);
190 }
191
192 static int
archive_read_format_warc_bid(struct archive_read * a,int best_bid)193 archive_read_format_warc_bid(struct archive_read *a, int best_bid)
194 {
195 const char *hdr;
196 ssize_t nrd;
197 unsigned int ver;
198
199 (void)best_bid; /* UNUSED */
200
201 /* Check the first line, which should already be a record header. */
202 if ((hdr = __archive_read_ahead(a, 12, &nrd)) == NULL) {
203 /* Not enough data to identify this format. */
204 return -1;
205 }
206
207 /* Parse the record version number. */
208 ver = warc_read_version(hdr, nrd);
209 if (ver < 1200U || ver > 10000U) {
210 /* Only WARC 0.12 through WARC 1.0 are supported. */
211 return -1;
212 }
213
214 /* WARC magic and version checks passed. */
215 return (64);
216 }
217
218 static int
archive_read_format_warc_read_header(struct archive_read * a,struct archive_entry * entry)219 archive_read_format_warc_read_header(struct archive_read *a,
220 struct archive_entry *entry)
221 {
222 #define HDR_PROBE_LEN (12U)
223 struct warc *warc = a->format->data;
224 unsigned int ver;
225 const char *buf;
226 ssize_t nrd;
227 const char *eoh;
228 char *tmp;
229 /* Reuse the header buffer while parsing the file name. */
230 warc_string_t fnam;
231 /* WARC record type */
232 warc_type_t ftyp;
233 /* Content length, or a negative error indicator */
234 int64_t cntlen;
235 /* WARC-Date is exposed as the entry ctime. */
236 time_t rtime;
237 /* A Last-Modified record header is exposed as the entry mtime. */
238 time_t mtime;
239
240 start_over:
241 /* Use read_ahead(); it already tracks unconsumed bytes, so this
242 * reader does not need a separate shift buffer. */
243 buf = __archive_read_ahead(a, HDR_PROBE_LEN, &nrd);
244
245 if (nrd < 0) {
246 /* I/O or stream error. */
247 archive_set_error(
248 &a->archive, ARCHIVE_ERRNO_MISC,
249 "Bad record header");
250 return (ARCHIVE_FATAL);
251 } else if (buf == NULL) {
252 /* there should be room for at least WARC/bla\r\n
253 * must be EOF therefore */
254 return (ARCHIVE_EOF);
255 }
256 /* Locate the end of the record header. */
257 eoh = warc_find_eoh(buf, nrd);
258 if (eoh == NULL) {
259 /* The header terminator was not found in the probed data. */
260 archive_set_error(
261 &a->archive, ARCHIVE_ERRNO_MISC,
262 "Bad record header");
263 return (ARCHIVE_FATAL);
264 }
265 ver = warc_read_version(buf, eoh - buf);
266 /* Only WARC 0.12 through WARC 1.0 are supported. */
267 if (ver == 0U) {
268 archive_set_error(
269 &a->archive, ARCHIVE_ERRNO_MISC,
270 "Invalid record version");
271 return (ARCHIVE_FATAL);
272 } else if (ver < 1200U || ver > 10000U) {
273 archive_set_error(
274 &a->archive, ARCHIVE_ERRNO_MISC,
275 "Unsupported record version: %u.%u",
276 ver / 10000, (ver % 10000) / 100);
277 return (ARCHIVE_FATAL);
278 }
279 cntlen = warc_read_length(buf, eoh - buf);
280 if (cntlen < 0) {
281 /* This reader requires Content-Length before processing a record. */
282 archive_set_error(
283 &a->archive, EINVAL,
284 "Bad content length");
285 return (ARCHIVE_FATAL);
286 }
287 rtime = warc_read_date(buf, eoh - buf);
288 if (rtime == (time_t)-1) {
289 /* This reader requires WARC-Date before processing a record. */
290 archive_set_error(
291 &a->archive, EINVAL,
292 "Bad record time");
293 return (ARCHIVE_FATAL);
294 }
295
296 /* Report this archive as WARC. */
297 a->archive.archive_format = ARCHIVE_FORMAT_WARC;
298 if (ver != warc->pver) {
299 /* Format this entry's WARC version. */
300 archive_string_sprintf(&warc->sver,
301 "WARC/%u.%u", ver / 10000, (ver % 10000) / 100);
302 /* Remember the version for later entries. */
303 warc->pver = ver;
304 }
305 /* Parse the record type. */
306 ftyp = warc_read_type(buf, eoh - buf);
307 /* Save content state for subsequent read calls. */
308 warc->cntlen = cntlen;
309 warc->cntoff = 0;
310 mtime = 0;/* Avoid compiler warnings on some platforms. */
311
312 switch (ftyp) {
313 case WT_RSRC:
314 case WT_RSP:
315 /* Read the filename only for record types that are expected to
316 * have a target URI. */
317 fnam = warc_read_uri(buf, eoh - buf);
318 /* Avoid creating directory endpoints as files. */
319 if (fnam.len == 0 || fnam.str[fnam.len - 1] == '/') {
320 /* Skip this record. */
321 fnam.len = 0U;
322 fnam.str = NULL;
323 break;
324 }
325 /* Copy the name into the reusable string pool to avoid a malloc/free
326 * roundtrip for each entry. */
327 if (fnam.len + 1U > warc->pool.len) {
328 warc->pool.len = ((fnam.len + 64U) / 64U) * 64U;
329 tmp = realloc(warc->pool.str, warc->pool.len);
330 if (tmp == NULL) {
331 archive_set_error(
332 &a->archive, ENOMEM,
333 "Out of memory");
334 return (ARCHIVE_FATAL);
335 }
336 warc->pool.str = tmp;
337 }
338 memcpy(warc->pool.str, fnam.str, fnam.len);
339 warc->pool.str[fnam.len] = '\0';
340 /* Hide the pool implementation behind the parsed string. */
341 fnam.str = warc->pool.str;
342
343 /* Use a Last-Modified record header when present; otherwise fall back
344 * to WARC-Date. */
345 if ((mtime = warc_read_last_modified(buf, eoh - buf)) == (time_t)-1) {
346 mtime = rtime;
347 }
348 break;
349 case WT_NONE:
350 case WT_INFO:
351 case WT_META:
352 case WT_REQ:
353 case WT_RVIS:
354 case WT_CONV:
355 case WT_CONT:
356 case LAST_WT:
357 default:
358 fnam.len = 0U;
359 fnam.str = NULL;
360 break;
361 }
362
363 /* Consume the record header. */
364 __archive_read_consume(a, eoh - buf);
365
366 switch (ftyp) {
367 case WT_RSRC:
368 case WT_RSP:
369 if (fnam.len > 0U) {
370 /* Populate the entry object. */
371 archive_entry_set_filetype(entry, AE_IFREG);
372 archive_entry_copy_pathname(entry, fnam.str);
373 archive_entry_set_size(entry, cntlen);
374 archive_entry_set_perm(entry, 0644);
375 /* WARC-Date becomes ctime; mtime comes from Last-Modified or WARC-Date. */
376 archive_entry_set_ctime(entry, rtime, 0L);
377 archive_entry_set_mtime(entry, mtime, 0L);
378 break;
379 }
380 /* FALLTHROUGH */
381 case WT_NONE:
382 case WT_INFO:
383 case WT_META:
384 case WT_REQ:
385 case WT_RVIS:
386 case WT_CONV:
387 case WT_CONT:
388 case LAST_WT:
389 default:
390 /* Skip this record body and look for the next one. */
391 if (archive_read_format_warc_skip(a) < 0)
392 return (ARCHIVE_FATAL);
393 goto start_over;
394 }
395 return (ARCHIVE_OK);
396 }
397
398 static int
archive_read_format_warc_read_data(struct archive_read * a,const void ** buf,size_t * bsz,int64_t * off)399 archive_read_format_warc_read_data(struct archive_read *a, const void **buf,
400 size_t *bsz, int64_t *off)
401 {
402 struct warc *warc = a->format->data;
403 const char *rab;
404 ssize_t nrd;
405
406 if (warc->unconsumed) {
407 __archive_read_consume(a, warc->unconsumed);
408 warc->unconsumed = 0;
409 }
410
411 if (warc->cntoff >= warc->cntlen) {
412 /* No data is available to return for this entry. */
413 *buf = NULL;
414 *bsz = 0U;
415 *off = warc->cntoff;
416 return (ARCHIVE_EOF);
417 }
418
419 rab = __archive_read_ahead(a, 1U, &nrd);
420 if (nrd < 0) {
421 *bsz = 0U;
422 /* Propagate the read error. */
423 return (int)nrd;
424 } else if (nrd == 0) {
425 archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
426 "Truncated WARC file data");
427 return (ARCHIVE_FATAL);
428 } else if ((int64_t)nrd > warc->cntlen - warc->cntoff) {
429 /* Clamp reads to Content-Length. */
430 nrd = warc->cntlen - warc->cntoff;
431 }
432 *off = warc->cntoff;
433 *bsz = nrd;
434 *buf = rab;
435
436 warc->cntoff += nrd;
437 warc->unconsumed = nrd;
438 return (ARCHIVE_OK);
439 }
440
441 static int
archive_read_format_warc_skip(struct archive_read * a)442 archive_read_format_warc_skip(struct archive_read *a)
443 {
444 struct warc *warc = a->format->data;
445
446 if (warc->cntoff > warc->cntlen)
447 return (ARCHIVE_FATAL);
448 if (warc->unconsumed) {
449 __archive_read_consume(a, warc->unconsumed);
450 warc->unconsumed = 0;
451 }
452 if (__archive_read_consume(a, warc->cntlen - warc->cntoff) < 0 ||
453 __archive_read_consume(a, 4U/*\r\n\r\n separator*/) < 0)
454 return (ARCHIVE_FATAL);
455 warc->cntlen = 0;
456 warc->cntoff = 0;
457 return (ARCHIVE_OK);
458 }
459
460
461 /* Private routines */
462 static void*
deconst(const void * c)463 deconst(const void *c)
464 {
465 return (void *)(uintptr_t)c;
466 }
467
468 static char*
xmemmem(const char * hay,const size_t haysize,const char * needle,const size_t needlesize)469 xmemmem(const char *hay, const size_t haysize,
470 const char *needle, const size_t needlesize)
471 {
472 const char *const eoh = hay + haysize;
473 const char *const eon = needle + needlesize;
474 const char *hp;
475 const char *np;
476 const char *cand;
477 unsigned int hsum;
478 unsigned int nsum;
479 unsigned int eqp;
480
481 /* Handle trivial cases first. A zero-sized needle is defined to be
482 * found anywhere in the haystack; otherwise find the first candidate
483 * that begins with *NEEDLE. */
484 if (needlesize == 0UL) {
485 return deconst(hay);
486 } else if ((hay = memchr(hay, *needle, haysize)) == NULL) {
487 /* No candidate match remains. */
488 return NULL;
489 }
490
491 /* The first characters of haystack and needle already match, and both
492 * strings are at least one character long. Compute the rolling XOR
493 * values for the needle and the first NEEDLESIZE characters of haystack. */
494 for (hp = hay + 1U, np = needle + 1U, hsum = *hay, nsum = *hay, eqp = 1U;
495 hp < eoh && np < eon;
496 hsum ^= *hp, nsum ^= *np, eqp &= *hp == *np, hp++, np++);
497
498 /* HP now references the (NEEDLESIZE + 1)-th character. */
499 if (np < eon) {
500 /* The haystack is smaller than the needle. */
501 return NULL;
502 } else if (eqp) {
503 /* Found a match. */
504 return deconst(hay);
505 }
506
507 /* Loop through the rest of the haystack and update the rolling XOR
508 * iteratively. */
509 for (cand = hay; hp < eoh; hp++) {
510 hsum ^= *cand++;
511 hsum ^= *hp;
512
513 /* When the rolling XOR values match, it is enough to check
514 * NEEDLESIZE - 1 characters for equality. CAND is always before
515 * HP by design, so no range check is needed. */
516 if (hsum == nsum && memcmp(cand, needle, needlesize - 1U) == 0) {
517 return deconst(cand);
518 }
519 }
520 return NULL;
521 }
522
523 static int
strtoi_lim(const char * str,const char ** ep,int llim,int ulim)524 strtoi_lim(const char *str, const char **ep, int llim, int ulim)
525 {
526 int res = 0;
527 const char *sp;
528 /* Track the number of digits with rulim. */
529 int rulim;
530
531 for (sp = str, rulim = ulim > 10 ? ulim : 10;
532 res * 10 <= ulim && rulim && *sp >= '0' && *sp <= '9';
533 sp++, rulim /= 10) {
534 res *= 10;
535 res += *sp - '0';
536 }
537 if (sp == str) {
538 res = -1;
539 } else if (res < llim || res > ulim) {
540 res = -2;
541 }
542 *ep = (const char*)sp;
543 return res;
544 }
545
546 static time_t
time_from_tm(struct tm * t)547 time_from_tm(struct tm *t)
548 {
549 #if HAVE__MKGMTIME
550 return _mkgmtime(t);
551 #elif HAVE_TIMEGM
552 /* Use platform timegm() if available. */
553 return (timegm(t));
554 #else
555 /* Otherwise, calculate directly using POSIX assumptions. */
556 /* First, fix up tm_yday based on the year, month, and day. */
557 if (mktime(t) == (time_t)-1)
558 return ((time_t)-1);
559 /* Then compute timegm() from first principles. */
560 return (t->tm_sec
561 + t->tm_min * 60
562 + t->tm_hour * 3600
563 + t->tm_yday * 86400
564 + (t->tm_year - 70) * 31536000
565 + ((t->tm_year - 69) / 4) * 86400
566 - ((t->tm_year - 1) / 100) * 86400
567 + ((t->tm_year + 299) / 400) * 86400);
568 #endif
569 }
570
571 static time_t
xstrpisotime(const char * s,char ** endptr)572 xstrpisotime(const char *s, char **endptr)
573 {
574 /* Like strptime(), but only for ISO 8601 Zulu strings. */
575 struct tm tm;
576 time_t res = (time_t)-1;
577
578 /* Clear the tm structure. */
579 memset(&tm, 0, sizeof(tm));
580
581 /* This is a non-standard routine, so skip leading whitespace for
582 * caller convenience. */
583 while (*s == ' ' || *s == '\t')
584 ++s;
585
586 /* Read the year. */
587 if ((tm.tm_year = strtoi_lim(s, &s, 1583, 4095)) < 0 || *s++ != '-') {
588 goto out;
589 }
590 /* Read the month. */
591 if ((tm.tm_mon = strtoi_lim(s, &s, 1, 12)) < 0 || *s++ != '-') {
592 goto out;
593 }
594 /* Read the day of the month. */
595 if ((tm.tm_mday = strtoi_lim(s, &s, 1, 31)) < 0 || *s++ != 'T') {
596 goto out;
597 }
598 /* Read the hour. */
599 if ((tm.tm_hour = strtoi_lim(s, &s, 0, 23)) < 0 || *s++ != ':') {
600 goto out;
601 }
602 /* Read the minute. */
603 if ((tm.tm_min = strtoi_lim(s, &s, 0, 59)) < 0 || *s++ != ':') {
604 goto out;
605 }
606 /* Read the second. */
607 if ((tm.tm_sec = strtoi_lim(s, &s, 0, 60)) < 0 || *s++ != 'Z') {
608 goto out;
609 }
610
611 /* Adjust tm fields to satisfy POSIX constraints. */
612 tm.tm_year -= 1900;
613 tm.tm_mon--;
614
615 /* Convert the tm structure to a Unix timestamp in UTC. */
616 res = time_from_tm(&tm);
617
618 out:
619 if (endptr != NULL) {
620 *endptr = deconst(s);
621 }
622 return res;
623 }
624
625 static int
warc_isdigit(const char c)626 warc_isdigit(const char c)
627 {
628 return c >= '0' && c <= '9';
629 }
630
631 static unsigned int
warc_read_version(const char * buf,size_t bsz)632 warc_read_version(const char *buf, size_t bsz)
633 {
634 static const char magic[] = "WARC/";
635 const char *c;
636 unsigned int ver = 0U;
637 unsigned int end = 0U;
638
639 if (bsz < 12 || memcmp(buf, magic, sizeof(magic) - 1U) != 0) {
640 /* Buffer too small or invalid magic. */
641 return ver;
642 }
643 /* Parse the version number. */
644 buf += sizeof(magic) - 1U;
645
646 if (warc_isdigit(buf[0]) && buf[1] == '.' && warc_isdigit(buf[2])) {
647 /* Support at most two digits in the minor version. */
648 if (warc_isdigit(buf[3]))
649 end = 1U;
650 /* Set up the major version. */
651 ver = (buf[0U] - '0') * 10000U;
652 /* Set up the minor version. */
653 if (end == 1U) {
654 ver += (buf[2U] - '0') * 1000U;
655 ver += (buf[3U] - '0') * 100U;
656 } else
657 ver += (buf[2U] - '0') * 100U;
658 /*
659 * WARC versions before 0.12 use a space-separated header.
660 * WARC 0.12 and later terminate the version with CRLF.
661 */
662 c = buf + 3U + end;
663 if (ver >= 1200U) {
664 if (memcmp(c, "\r\n", 2U) != 0)
665 ver = 0U;
666 } else {
667 /* Version is below WARC 0.12. */
668 if (*c != ' ' && *c != '\t')
669 ver = 0U;
670 }
671 }
672 return ver;
673 }
674
675 static unsigned int
warc_read_type(const char * buf,size_t bsz)676 warc_read_type(const char *buf, size_t bsz)
677 {
678 static const char _key[] = "\r\nWARC-Type:";
679 const char *val, *eol;
680
681 if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) {
682 /* Header field is absent. */
683 return WT_NONE;
684 }
685 val += sizeof(_key) - 1U;
686 if ((eol = warc_find_eol(val, buf + bsz - val)) == NULL) {
687 /* Header field has no end of line. */
688 return WT_NONE;
689 }
690
691 /* Skip leading whitespace. */
692 while (val < eol && (*val == ' ' || *val == '\t'))
693 ++val;
694
695 if (val + 8U == eol) {
696 if (memcmp(val, "resource", 8U) == 0)
697 return WT_RSRC;
698 else if (memcmp(val, "response", 8U) == 0)
699 return WT_RSP;
700 }
701 return WT_NONE;
702 }
703
704 static warc_string_t
warc_read_uri(const char * buf,size_t bsz)705 warc_read_uri(const char *buf, size_t bsz)
706 {
707 static const char _key[] = "\r\nWARC-Target-URI:";
708 const char *val, *uri, *eol, *p;
709 warc_string_t res = {0U, NULL};
710
711 if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) {
712 /* Header field is absent. */
713 return res;
714 }
715 /* Skip leading whitespace. */
716 val += sizeof(_key) - 1U;
717 if ((eol = warc_find_eol(val, buf + bsz - val)) == NULL) {
718 /* Header field has no end of line. */
719 return res;
720 }
721
722 while (val < eol && (*val == ' ' || *val == '\t'))
723 ++val;
724
725 /* Locate the :// separator. */
726 if ((uri = xmemmem(val, eol - val, "://", 3U)) == NULL) {
727 /* Ignore values without a :// separator. */
728 return res;
729 }
730
731 /* Spaces inside a URI are not allowed; CRLF should follow. */
732 for (p = val; p < eol; p++) {
733 if (isspace((unsigned char)*p))
734 return res;
735 }
736
737 /* Require enough room for the shortest supported scheme. */
738 if (uri < (val + 3U))
739 return res;
740
741 /* Move uri past the :// separator. */
742 uri += 3U;
743
744 /* Inspect the scheme prefix. */
745 if (memcmp(val, "file", 4U) == 0) {
746 /* Keep file:// paths as-is. */
747
748 } else if (memcmp(val, "http", 4U) == 0 ||
749 memcmp(val, "ftp", 3U) == 0) {
750 /* Skip the domain and the first slash. */
751 while (uri < eol && *uri++ != '/');
752 } else {
753 /* Unsupported URI scheme. */
754 return res;
755 }
756 res.str = uri;
757 res.len = eol - uri;
758 return res;
759 }
760
761 static int64_t
warc_read_length(const char * buf,size_t bsz)762 warc_read_length(const char *buf, size_t bsz)
763 {
764 static const char _key[] = "\r\nContent-Length:";
765 const char *val, *eol, *p;
766 int64_t len;
767
768 if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) {
769 /* Header field is absent. */
770 return -1;
771 }
772 val += sizeof(_key) - 1U;
773
774 if ((eol = warc_find_eol(val, buf + bsz - val)) == NULL) {
775 /* Malformed field with no end of line. */
776 return -1;
777 }
778
779 /* Skip leading whitespace. */
780 while (val < eol && (*val == ' ' || *val == '\t'))
781 val++;
782
783 /* Require at least one digit. */
784 if (val >= eol || *val < '0' || *val > '9')
785 return -1;
786
787 len = 0;
788 for (p = val; p < eol; p++) {
789 int64_t digit;
790
791 if (*p < '0' || *p > '9')
792 return -1;
793 digit = *p - '0';
794 if (archive_ckd_mul_i64(&len, len, 10) ||
795 archive_ckd_add_i64(&len, len, digit))
796 return -1;
797 }
798
799 return len;
800 }
801
802 static time_t
warc_read_date(const char * buf,size_t bsz)803 warc_read_date(const char *buf, size_t bsz)
804 {
805 static const char _key[] = "\r\nWARC-Date:";
806 const char *val, *eol;
807 char *on = NULL;
808 time_t res;
809
810 if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) {
811 /* Header field is absent. */
812 return (time_t)-1;
813 }
814 val += sizeof(_key) - 1U;
815 if ((eol = warc_find_eol(val, buf + bsz - val)) == NULL ) {
816 /* Header field has no end of line. */
817 return -1;
818 }
819
820 /* xstrpisotime() skips leading whitespace. */
821 res = xstrpisotime(val, &on);
822 if (on != eol) {
823 /* The field must end here. */
824 return -1;
825 }
826 return res;
827 }
828
829 static time_t
warc_read_last_modified(const char * buf,size_t bsz)830 warc_read_last_modified(const char *buf, size_t bsz)
831 {
832 static const char _key[] = "\r\nLast-Modified:";
833 const char *val, *eol;
834 char *on = NULL;
835 time_t res;
836
837 if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) {
838 /* Header field is absent. */
839 return (time_t)-1;
840 }
841 val += sizeof(_key) - 1U;
842 if ((eol = warc_find_eol(val, buf + bsz - val)) == NULL ) {
843 /* Header field has no end of line. */
844 return -1;
845 }
846
847 /* xstrpisotime() skips leading whitespace. */
848 res = xstrpisotime(val, &on);
849 if (on != eol) {
850 /* The field must end here. */
851 return -1;
852 }
853 return res;
854 }
855
856 static const char *
warc_find_eoh(const char * buf,size_t bsz)857 warc_find_eoh(const char *buf, size_t bsz)
858 {
859 static const char _marker[] = "\r\n\r\n";
860 const char *hit = xmemmem(buf, bsz, _marker, sizeof(_marker) - 1U);
861
862 if (hit != NULL) {
863 hit += sizeof(_marker) - 1U;
864 }
865 return hit;
866 }
867
868 static const char *
warc_find_eol(const char * buf,size_t bsz)869 warc_find_eol(const char *buf, size_t bsz)
870 {
871 static const char _marker[] = "\r\n";
872 const char *hit = xmemmem(buf, bsz, _marker, sizeof(_marker) - 1U);
873
874 return hit;
875 }
876