1 /*- 2 * Copyright (c) 1998 Dag-Erling Co�dan Sm�rgrav 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer 10 * in this position and unchanged. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 /* 32 * The base64 code in this file is based on code from MIT fetch, which 33 * has the following copyright and license: 34 * 35 *- 36 * Copyright 1997 Massachusetts Institute of Technology 37 * 38 * Permission to use, copy, modify, and distribute this software and 39 * its documentation for any purpose and without fee is hereby 40 * granted, provided that both the above copyright notice and this 41 * permission notice appear in all copies, that both the above 42 * copyright notice and this permission notice appear in all 43 * supporting documentation, and that the name of M.I.T. not be used 44 * in advertising or publicity pertaining to distribution of the 45 * software without specific, written prior permission. M.I.T. makes 46 * no representations about the suitability of this software for any 47 * purpose. It is provided "as is" without express or implied 48 * warranty. 49 * 50 * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS 51 * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, 52 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 53 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT 54 * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 55 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 56 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF 57 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 58 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 59 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 60 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. */ 62 63 #include <sys/param.h> 64 #include <sys/socket.h> 65 66 #include <err.h> 67 #include <ctype.h> 68 #include <locale.h> 69 #include <netdb.h> 70 #include <stdarg.h> 71 #include <stdio.h> 72 #include <stdlib.h> 73 #include <string.h> 74 #include <time.h> 75 #include <unistd.h> 76 77 #include "fetch.h" 78 #include "common.h" 79 #include "httperr.h" 80 81 extern char *__progname; 82 83 #define ENDL "\r\n" 84 85 #define HTTP_OK 200 86 #define HTTP_PARTIAL 206 87 #define HTTP_MOVED 302 88 89 struct cookie 90 { 91 FILE *real_f; 92 #define ENC_NONE 0 93 #define ENC_CHUNKED 1 94 int encoding; /* 1 = chunked, 0 = none */ 95 #define HTTPCTYPELEN 59 96 char content_type[HTTPCTYPELEN+1]; 97 char *buf; 98 int b_cur, eof; 99 unsigned b_len, chunksize; 100 }; 101 102 /* 103 * Send a formatted line; optionally echo to terminal 104 */ 105 static int 106 _http_cmd(FILE *f, char *fmt, ...) 107 { 108 va_list ap; 109 110 va_start(ap, fmt); 111 vfprintf(f, fmt, ap); 112 #ifndef NDEBUG 113 fprintf(stderr, "\033[1m>>> "); 114 vfprintf(stderr, fmt, ap); 115 fprintf(stderr, "\033[m"); 116 #endif 117 va_end(ap); 118 119 return 0; /* XXX */ 120 } 121 122 /* 123 * Fill the input buffer, do chunk decoding on the fly 124 */ 125 static char * 126 _http_fillbuf(struct cookie *c) 127 { 128 char *ln; 129 unsigned int len; 130 131 if (c->eof) 132 return NULL; 133 134 if (c->encoding == ENC_NONE) { 135 c->buf = fgetln(c->real_f, &(c->b_len)); 136 c->b_cur = 0; 137 } else if (c->encoding == ENC_CHUNKED) { 138 if (c->chunksize == 0) { 139 ln = fgetln(c->real_f, &len); 140 if (len <= 2) 141 return NULL; 142 DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): new chunk: " 143 "%*.*s\033[m\n", (int)len-2, (int)len-2, ln)); 144 sscanf(ln, "%x", &(c->chunksize)); 145 if (!c->chunksize) { 146 DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): " 147 "end of last chunk\033[m\n")); 148 c->eof = 1; 149 return NULL; 150 } 151 DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): " 152 "new chunk: %X\033[m\n", c->chunksize)); 153 } 154 c->buf = fgetln(c->real_f, &(c->b_len)); 155 if (c->b_len > c->chunksize) 156 c->b_len = c->chunksize; 157 c->chunksize -= c->b_len; 158 c->b_cur = 0; 159 } 160 else return NULL; /* unknown encoding */ 161 return c->buf; 162 } 163 164 /* 165 * Read function 166 */ 167 static int 168 _http_readfn(struct cookie *c, char *buf, int len) 169 { 170 int l, pos = 0; 171 while (len) { 172 /* empty buffer */ 173 if (!c->buf || (c->b_cur == c->b_len)) 174 if (!_http_fillbuf(c)) 175 break; 176 177 l = c->b_len - c->b_cur; 178 if (len < l) l = len; 179 memcpy(buf + pos, c->buf + c->b_cur, l); 180 c->b_cur += l; 181 pos += l; 182 len -= l; 183 } 184 185 if (ferror(c->real_f)) 186 return -1; 187 else return pos; 188 } 189 190 /* 191 * Write function 192 */ 193 static int 194 _http_writefn(struct cookie *c, const char *buf, int len) 195 { 196 size_t r = fwrite(buf, 1, (size_t)len, c->real_f); 197 return r ? r : -1; 198 } 199 200 /* 201 * Close function 202 */ 203 static int 204 _http_closefn(struct cookie *c) 205 { 206 int r = fclose(c->real_f); 207 free(c); 208 return (r == EOF) ? -1 : 0; 209 } 210 211 /* 212 * Extract content type from cookie 213 */ 214 char * 215 fetchContentType(FILE *f) 216 { 217 /* 218 * We have no way of making sure this really *is* one of our cookies, 219 * so just check for a null pointer and hope for the best. 220 */ 221 return f->_cookie ? (((struct cookie *)f->_cookie)->content_type) : NULL; 222 } 223 224 /* 225 * Base64 encoding 226 */ 227 int 228 _http_base64(char *dst, char *src, int l) 229 { 230 static const char base64[] = 231 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 232 "abcdefghijklmnopqrstuvwxyz" 233 "0123456789+/"; 234 int t, r = 0; 235 236 while (l >= 3) { 237 t = (src[0] << 16) | (src[1] << 8) | src[2]; 238 dst[0] = base64[(t >> 18) & 0x3f]; 239 dst[1] = base64[(t >> 12) & 0x3f]; 240 dst[2] = base64[(t >> 6) & 0x3f]; 241 dst[3] = base64[(t >> 0) & 0x3f]; 242 src += 3; l -= 3; 243 dst += 4; r += 4; 244 } 245 246 switch (l) { 247 case 2: 248 t = (src[0] << 16) | (src[1] << 8); 249 dst[0] = base64[(t >> 18) & 0x3f]; 250 dst[1] = base64[(t >> 12) & 0x3f]; 251 dst[2] = base64[(t >> 6) & 0x3f]; 252 dst[3] = '='; 253 dst += 4; 254 r += 4; 255 break; 256 case 1: 257 t = src[0] << 16; 258 dst[0] = base64[(t >> 18) & 0x3f]; 259 dst[1] = base64[(t >> 12) & 0x3f]; 260 dst[2] = dst[3] = '='; 261 dst += 4; 262 r += 4; 263 break; 264 case 0: 265 break; 266 } 267 268 *dst = 0; 269 return r; 270 } 271 272 /* 273 * Encode username and password 274 */ 275 char * 276 _http_auth(char *usr, char *pwd) 277 { 278 int len, lu, lp; 279 char *str, *s; 280 281 lu = strlen(usr); 282 lp = strlen(pwd); 283 284 len = (lu * 4 + 2) / 3 /* user name, round up */ 285 + 1 /* colon */ 286 + (lp * 4 + 2) / 3 /* password, round up */ 287 + 1; /* null */ 288 289 if ((s = str = (char *)malloc(len)) == NULL) 290 return NULL; 291 292 s += _http_base64(s, usr, lu); 293 *s++ = ':'; 294 s += _http_base64(s, pwd, lp); 295 *s = 0; 296 297 return str; 298 } 299 300 /* 301 * Connect to server or proxy 302 */ 303 FILE * 304 _http_connect(struct url *URL, char *flags) 305 { 306 int direct, sd = -1, verbose; 307 #ifdef INET6 308 int af = AF_UNSPEC; 309 #else 310 int af = AF_INET; 311 #endif 312 size_t len; 313 char *px; 314 FILE *f; 315 316 direct = (flags && strchr(flags, 'd')); 317 verbose = (flags && strchr(flags, 'v')); 318 if ((flags && strchr(flags, '4'))) 319 af = AF_INET; 320 else if ((flags && strchr(flags, '6'))) 321 af = AF_INET6; 322 323 /* check port */ 324 if (!URL->port) { 325 struct servent *se; 326 327 if (strcasecmp(URL->scheme, "ftp") == 0) 328 if ((se = getservbyname("ftp", "tcp")) != NULL) 329 URL->port = ntohs(se->s_port); 330 else 331 URL->port = 21; 332 else 333 if ((se = getservbyname("http", "tcp")) != NULL) 334 URL->port = ntohs(se->s_port); 335 else 336 URL->port = 80; 337 } 338 339 /* attempt to connect to proxy server */ 340 if (!direct && (px = getenv("HTTP_PROXY")) != NULL) { 341 char host[MAXHOSTNAMELEN]; 342 int port = 0; 343 344 /* measure length */ 345 #ifdef INET6 346 if (px[0] != '[' || 347 (len = strcspn(px, "]")) >= strlen(px) || 348 (px[++len] != '\0' && px[len] != ':')) 349 #endif 350 len = strcspn(px, ":"); 351 352 /* get port (XXX atoi is a little too tolerant perhaps?) */ 353 if (px[len] == ':') { 354 if (strspn(px+len+1, "0123456789") != strlen(px+len+1) 355 || strlen(px+len+1) > 5) { 356 /* XXX we should emit some kind of warning */ 357 } 358 port = atoi(px+len+1); 359 if (port < 1 || port > 65535) { 360 /* XXX we should emit some kind of warning */ 361 } 362 } 363 if (!port) { 364 #if 0 365 /* 366 * commented out, since there is currently no service name 367 * for HTTP proxies 368 */ 369 struct servent *se; 370 371 if ((se = getservbyname("xxxx", "tcp")) != NULL) 372 port = ntohs(se->s_port); 373 else 374 #endif 375 port = 3128; 376 } 377 378 /* get host name */ 379 #ifdef INET6 380 if (len > 1 && px[0] == '[' && px[len - 1] == ']') { 381 px++; 382 len -= 2; 383 } 384 #endif 385 if (len >= MAXHOSTNAMELEN) 386 len = MAXHOSTNAMELEN - 1; 387 strncpy(host, px, len); 388 host[len] = 0; 389 390 /* connect */ 391 sd = _fetch_connect(host, port, af, verbose); 392 } 393 394 /* if no proxy is configured or could be contacted, try direct */ 395 if (sd == -1) { 396 if (strcasecmp(URL->scheme, "ftp") == 0) 397 goto ouch; 398 if ((sd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1) 399 goto ouch; 400 } 401 402 /* reopen as stream */ 403 if ((f = fdopen(sd, "r+")) == NULL) 404 goto ouch; 405 406 return f; 407 408 ouch: 409 if (sd >= 0) 410 close(sd); 411 _http_seterr(999); /* XXX do this properly RSN */ 412 return NULL; 413 } 414 415 /* 416 * Check a header line 417 */ 418 char * 419 _http_match(char *str, char *hdr) 420 { 421 while (*str && *hdr && tolower(*str++) == tolower(*hdr++)) 422 /* nothing */; 423 if (*str || *hdr != ':') 424 return NULL; 425 while (*hdr && isspace(*++hdr)) 426 /* nothing */; 427 return hdr; 428 } 429 430 /* 431 * Send a HEAD or GET request 432 */ 433 int 434 _http_request(FILE *f, char *op, struct url *URL, char *flags) 435 { 436 int e, verbose; 437 char *ln, *p; 438 size_t len; 439 char *host; 440 #ifdef INET6 441 char hbuf[MAXHOSTNAMELEN + 1]; 442 #endif 443 444 verbose = (flags && strchr(flags, 'v')); 445 446 host = URL->host; 447 #ifdef INET6 448 if (strchr(URL->host, ':')) { 449 snprintf(hbuf, sizeof(hbuf), "[%s]", URL->host); 450 host = hbuf; 451 } 452 #endif 453 454 /* send request (proxies require absolute form, so use that) */ 455 if (verbose) 456 _fetch_info("requesting %s://%s:%d%s", 457 URL->scheme, host, URL->port, URL->doc); 458 _http_cmd(f, "%s %s://%s:%d%s HTTP/1.1" ENDL, 459 op, URL->scheme, host, URL->port, URL->doc); 460 461 /* start sending headers away */ 462 if (URL->user[0] || URL->pwd[0]) { 463 char *auth_str = _http_auth(URL->user, URL->pwd); 464 if (!auth_str) 465 return 999; /* XXX wrong */ 466 _http_cmd(f, "Authorization: Basic %s" ENDL, auth_str); 467 free(auth_str); 468 } 469 _http_cmd(f, "Host: %s:%d" ENDL, host, URL->port); 470 _http_cmd(f, "User-Agent: %s " _LIBFETCH_VER ENDL, __progname); 471 if (URL->offset) 472 _http_cmd(f, "Range: bytes=%lld-" ENDL, URL->offset); 473 _http_cmd(f, "Connection: close" ENDL ENDL); 474 475 /* get response */ 476 if ((ln = fgetln(f, &len)) == NULL) 477 return 999; 478 DEBUG(fprintf(stderr, "response: [\033[1m%*.*s\033[m]\n", 479 (int)len-2, (int)len-2, ln)); 480 481 /* we can't use strchr() and friends since ln isn't NUL-terminated */ 482 p = ln; 483 while ((p < ln + len) && !isspace(*p)) 484 p++; 485 while ((p < ln + len) && !isdigit(*p)) 486 p++; 487 if (!isdigit(*p)) 488 return 999; 489 490 e = atoi(p); 491 DEBUG(fprintf(stderr, "code: [\033[1m%d\033[m]\n", e)); 492 return e; 493 } 494 495 /* 496 * Retrieve a file by HTTP 497 */ 498 FILE * 499 fetchGetHTTP(struct url *URL, char *flags) 500 { 501 int e, enc = ENC_NONE, i, noredirect; 502 struct cookie *c; 503 char *ln, *p, *q; 504 FILE *f, *cf; 505 size_t len; 506 off_t pos = 0; 507 508 noredirect = (flags && strchr(flags, 'A')); 509 510 /* allocate cookie */ 511 if ((c = calloc(1, sizeof *c)) == NULL) 512 return NULL; 513 514 /* connect */ 515 if ((f = _http_connect(URL, flags)) == NULL) { 516 free(c); 517 return NULL; 518 } 519 c->real_f = f; 520 521 e = _http_request(f, "GET", URL, flags); 522 if (e != (URL->offset ? HTTP_PARTIAL : HTTP_OK) 523 && (e != HTTP_MOVED || noredirect)) { 524 _http_seterr(e); 525 goto fouch; 526 } 527 528 /* browse through header */ 529 while (1) { 530 if ((ln = fgetln(f, &len)) == NULL) 531 goto fouch; 532 if ((ln[0] == '\r') || (ln[0] == '\n')) 533 break; 534 while (isspace(ln[len-1])) 535 --len; 536 ln[len] = '\0'; /* XXX */ 537 DEBUG(fprintf(stderr, "header: [\033[1m%s\033[m]\n", ln)); 538 if ((p = _http_match("Location", ln)) != NULL) { 539 struct url *url; 540 541 for (q = p; *q && !isspace(*q); q++) 542 /* VOID */ ; 543 *q = 0; 544 if ((url = fetchParseURL(p)) == NULL) 545 goto fouch; 546 url->offset = URL->offset; 547 url->length = URL->length; 548 DEBUG(fprintf(stderr, "location: [\033[1m%s\033[m]\n", p)); 549 cf = fetchGetHTTP(url, flags); 550 fetchFreeURL(url); 551 fclose(f); 552 return cf; 553 } else if ((p = _http_match("Transfer-Encoding", ln)) != NULL) { 554 for (q = p; *q && !isspace(*q); q++) 555 /* VOID */ ; 556 *q = 0; 557 if (strcasecmp(p, "chunked") == 0) 558 enc = ENC_CHUNKED; 559 DEBUG(fprintf(stderr, "transfer encoding: [\033[1m%s\033[m]\n", p)); 560 } else if ((p = _http_match("Content-Type", ln)) != NULL) { 561 for (i = 0; *p && i < HTTPCTYPELEN; p++, i++) 562 c->content_type[i] = *p; 563 do c->content_type[i--] = 0; while (isspace(c->content_type[i])); 564 DEBUG(fprintf(stderr, "content type: [\033[1m%s\033[m]\n", 565 c->content_type)); 566 } else if ((p = _http_match("Content-Range", ln)) != NULL) { 567 if (strncasecmp(p, "bytes ", 6) != 0) 568 goto fouch; 569 p += 6; 570 while (*p && isdigit(*p)) 571 pos = pos * 10 + (*p++ - '0'); 572 /* XXX wouldn't hurt to be slightly more paranoid here */ 573 DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", pos)); 574 if (pos > URL->offset) 575 goto fouch; 576 } 577 } 578 579 /* only body remains */ 580 c->encoding = enc; 581 cf = funopen(c, 582 (int (*)(void *, char *, int))_http_readfn, 583 (int (*)(void *, const char *, int))_http_writefn, 584 (fpos_t (*)(void *, fpos_t, int))NULL, 585 (int (*)(void *))_http_closefn); 586 if (cf == NULL) 587 goto fouch; 588 589 while (pos < URL->offset) 590 if (fgetc(cf) == EOF) 591 goto cfouch; 592 593 return cf; 594 595 fouch: 596 fclose(f); 597 free(c); 598 _http_seterr(999); /* XXX do this properly RSN */ 599 return NULL; 600 cfouch: 601 fclose(cf); 602 _http_seterr(999); /* XXX do this properly RSN */ 603 return NULL; 604 } 605 606 FILE * 607 fetchPutHTTP(struct url *URL, char *flags) 608 { 609 warnx("fetchPutHTTP(): not implemented"); 610 return NULL; 611 } 612 613 /* 614 * Get an HTTP document's metadata 615 */ 616 int 617 fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags) 618 { 619 int e, noredirect; 620 size_t len; 621 char *ln, *p, *q; 622 FILE *f; 623 624 noredirect = (flags && strchr(flags, 'A')); 625 626 us->size = -1; 627 us->atime = us->mtime = 0; 628 629 /* connect */ 630 if ((f = _http_connect(URL, flags)) == NULL) 631 return -1; 632 633 e = _http_request(f, "HEAD", URL, flags); 634 if (e != HTTP_OK && (e != HTTP_MOVED || noredirect)) { 635 _http_seterr(e); 636 goto ouch; 637 } 638 639 while (1) { 640 if ((ln = fgetln(f, &len)) == NULL) 641 goto fouch; 642 if ((ln[0] == '\r') || (ln[0] == '\n')) 643 break; 644 while (isspace(ln[len-1])) 645 --len; 646 ln[len] = '\0'; /* XXX */ 647 DEBUG(fprintf(stderr, "header: [\033[1m%s\033[m]\n", ln)); 648 if ((p = _http_match("Location", ln)) != NULL) { 649 struct url *url; 650 651 for (q = p; *q && !isspace(*q); q++) 652 /* VOID */ ; 653 *q = 0; 654 if ((url = fetchParseURL(p)) == NULL) 655 goto ouch; 656 url->offset = URL->offset; 657 url->length = URL->length; 658 DEBUG(fprintf(stderr, "location: [\033[1m%s\033[m]\n", p)); 659 e = fetchStatHTTP(url, us, flags); 660 fetchFreeURL(url); 661 fclose(f); 662 return e; 663 } else if ((p = _http_match("Last-Modified", ln)) != NULL) { 664 struct tm tm; 665 char locale[64]; 666 667 strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale); 668 setlocale(LC_TIME, "C"); 669 strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); 670 /* XXX should add support for date-2 and date-3 */ 671 setlocale(LC_TIME, locale); 672 us->atime = us->mtime = timegm(&tm); 673 DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " 674 "%02d:%02d:%02d\033[m]\n", 675 tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, 676 tm.tm_hour, tm.tm_min, tm.tm_sec)); 677 } else if ((p = _http_match("Content-Length", ln)) != NULL) { 678 us->size = 0; 679 while (*p && isdigit(*p)) 680 us->size = us->size * 10 + (*p++ - '0'); 681 DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", us->size)); 682 } 683 } 684 685 fclose(f); 686 return 0; 687 ouch: 688 _http_seterr(999); /* XXX do this properly RSN */ 689 fouch: 690 fclose(f); 691 return -1; 692 } 693 694 /* 695 * List a directory 696 */ 697 struct url_ent * 698 fetchListHTTP(struct url *url, char *flags) 699 { 700 warnx("fetchListHTTP(): not implemented"); 701 return NULL; 702 } 703