1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2005 Colin Percival 5 * All rights reserved 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted providing that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/types.h> 33 #include <sys/time.h> 34 #include <sys/socket.h> 35 36 #include <ctype.h> 37 #include <err.h> 38 #include <errno.h> 39 #include <fcntl.h> 40 #include <limits.h> 41 #include <netdb.h> 42 #include <stdint.h> 43 #include <stdio.h> 44 #include <stdlib.h> 45 #include <string.h> 46 #include <sysexits.h> 47 #include <unistd.h> 48 49 static const char * env_HTTP_PROXY; 50 static char * env_HTTP_PROXY_AUTH; 51 static const char * env_HTTP_USER_AGENT; 52 static char * env_HTTP_TIMEOUT; 53 static const char * proxyport; 54 static char * proxyauth; 55 56 static struct timeval timo = { 15, 0}; 57 58 static void 59 usage(void) 60 { 61 62 fprintf(stderr, "usage: phttpget server [file ...]\n"); 63 exit(EX_USAGE); 64 } 65 66 /* 67 * Base64 encode a string; the string returned, if non-NULL, is 68 * allocated using malloc() and must be freed by the caller. 69 */ 70 static char * 71 b64enc(const char *ptext) 72 { 73 static const char base64[] = 74 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 75 "abcdefghijklmnopqrstuvwxyz" 76 "0123456789+/"; 77 const char *pt; 78 char *ctext, *pc; 79 size_t ptlen, ctlen; 80 uint32_t t; 81 unsigned int j; 82 83 /* 84 * Encoded length is 4 characters per 3-byte block or partial 85 * block of plaintext, plus one byte for the terminating NUL 86 */ 87 ptlen = strlen(ptext); 88 if (ptlen > ((SIZE_MAX - 1) / 4) * 3 - 2) 89 return NULL; /* Possible integer overflow */ 90 ctlen = 4 * ((ptlen + 2) / 3) + 1; 91 if ((ctext = malloc(ctlen)) == NULL) 92 return NULL; 93 ctext[ctlen - 1] = 0; 94 95 /* 96 * Scan through ptext, reading up to 3 bytes from ptext and 97 * writing 4 bytes to ctext, until we run out of input. 98 */ 99 for (pt = ptext, pc = ctext; ptlen; ptlen -= 3, pc += 4) { 100 /* Read 3 bytes */ 101 for (t = j = 0; j < 3; j++) { 102 t <<= 8; 103 if (j < ptlen) 104 t += *pt++; 105 } 106 107 /* Write 4 bytes */ 108 for (j = 0; j < 4; j++) { 109 if (j <= ptlen + 1) 110 pc[j] = base64[(t >> 18) & 0x3f]; 111 else 112 pc[j] = '='; 113 t <<= 6; 114 } 115 116 /* If we're done, exit the loop */ 117 if (ptlen <= 3) 118 break; 119 } 120 121 return (ctext); 122 } 123 124 static void 125 readenv(void) 126 { 127 char *proxy_auth_userpass, *proxy_auth_userpass64, *p; 128 char *proxy_auth_user = NULL; 129 char *proxy_auth_pass = NULL; 130 long http_timeout; 131 132 env_HTTP_PROXY = getenv("HTTP_PROXY"); 133 if (env_HTTP_PROXY == NULL) 134 env_HTTP_PROXY = getenv("http_proxy"); 135 if (env_HTTP_PROXY != NULL) { 136 if (strncmp(env_HTTP_PROXY, "http://", 7) == 0) 137 env_HTTP_PROXY += 7; 138 p = strchr(env_HTTP_PROXY, '/'); 139 if (p != NULL) 140 *p = 0; 141 p = strchr(env_HTTP_PROXY, ':'); 142 if (p != NULL) { 143 *p = 0; 144 proxyport = p + 1; 145 } else 146 proxyport = "3128"; 147 } 148 149 env_HTTP_PROXY_AUTH = getenv("HTTP_PROXY_AUTH"); 150 if ((env_HTTP_PROXY != NULL) && 151 (env_HTTP_PROXY_AUTH != NULL) && 152 (strncasecmp(env_HTTP_PROXY_AUTH, "basic:" , 6) == 0)) { 153 /* Ignore authentication scheme */ 154 (void) strsep(&env_HTTP_PROXY_AUTH, ":"); 155 156 /* Ignore realm */ 157 (void) strsep(&env_HTTP_PROXY_AUTH, ":"); 158 159 /* Obtain username and password */ 160 proxy_auth_user = strsep(&env_HTTP_PROXY_AUTH, ":"); 161 proxy_auth_pass = env_HTTP_PROXY_AUTH; 162 } 163 164 if ((proxy_auth_user != NULL) && (proxy_auth_pass != NULL)) { 165 asprintf(&proxy_auth_userpass, "%s:%s", 166 proxy_auth_user, proxy_auth_pass); 167 if (proxy_auth_userpass == NULL) 168 err(1, "asprintf"); 169 170 proxy_auth_userpass64 = b64enc(proxy_auth_userpass); 171 if (proxy_auth_userpass64 == NULL) 172 err(1, "malloc"); 173 174 asprintf(&proxyauth, "Proxy-Authorization: Basic %s\r\n", 175 proxy_auth_userpass64); 176 if (proxyauth == NULL) 177 err(1, "asprintf"); 178 179 free(proxy_auth_userpass); 180 free(proxy_auth_userpass64); 181 } else 182 proxyauth = NULL; 183 184 env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT"); 185 if (env_HTTP_USER_AGENT == NULL) 186 env_HTTP_USER_AGENT = "phttpget/0.1"; 187 188 env_HTTP_TIMEOUT = getenv("HTTP_TIMEOUT"); 189 if (env_HTTP_TIMEOUT != NULL) { 190 http_timeout = strtol(env_HTTP_TIMEOUT, &p, 10); 191 if ((*env_HTTP_TIMEOUT == '\0') || (*p != '\0') || 192 (http_timeout < 0)) 193 warnx("HTTP_TIMEOUT (%s) is not a positive integer", 194 env_HTTP_TIMEOUT); 195 else 196 timo.tv_sec = http_timeout; 197 } 198 } 199 200 static int 201 makerequest(char ** buf, char * path, char * server, int connclose) 202 { 203 int buflen; 204 205 buflen = asprintf(buf, 206 "GET %s%s/%s HTTP/1.1\r\n" 207 "Host: %s\r\n" 208 "User-Agent: %s\r\n" 209 "%s" 210 "%s" 211 "\r\n", 212 env_HTTP_PROXY ? "http://" : "", 213 env_HTTP_PROXY ? server : "", 214 path, server, env_HTTP_USER_AGENT, 215 proxyauth ? proxyauth : "", 216 connclose ? "Connection: Close\r\n" : "Connection: Keep-Alive\r\n"); 217 if (buflen == -1) 218 err(1, "asprintf"); 219 return(buflen); 220 } 221 222 static int 223 readln(int sd, char * resbuf, int * resbuflen, int * resbufpos) 224 { 225 ssize_t len; 226 227 while (strnstr(resbuf + *resbufpos, "\r\n", 228 *resbuflen - *resbufpos) == NULL) { 229 /* Move buffered data to the start of the buffer */ 230 if (*resbufpos != 0) { 231 memmove(resbuf, resbuf + *resbufpos, 232 *resbuflen - *resbufpos); 233 *resbuflen -= *resbufpos; 234 *resbufpos = 0; 235 } 236 237 /* If the buffer is full, complain */ 238 if (*resbuflen == BUFSIZ) 239 return -1; 240 241 /* Read more data into the buffer */ 242 len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0); 243 if ((len == 0) || 244 ((len == -1) && (errno != EINTR))) 245 return -1; 246 247 if (len != -1) 248 *resbuflen += len; 249 } 250 251 return 0; 252 } 253 254 static int 255 copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen, 256 int * resbufpos) 257 { 258 ssize_t len; 259 260 while (copylen) { 261 /* Write data from resbuf to fd */ 262 len = *resbuflen - *resbufpos; 263 if (copylen < len) 264 len = copylen; 265 if (len > 0) { 266 if (fd != -1) 267 len = write(fd, resbuf + *resbufpos, len); 268 if (len == -1) 269 err(1, "write"); 270 *resbufpos += len; 271 copylen -= len; 272 continue; 273 } 274 275 /* Read more data into buffer */ 276 len = recv(sd, resbuf, BUFSIZ, 0); 277 if (len == -1) { 278 if (errno == EINTR) 279 continue; 280 return -1; 281 } else if (len == 0) { 282 return -2; 283 } else { 284 *resbuflen = len; 285 *resbufpos = 0; 286 } 287 } 288 289 return 0; 290 } 291 292 int 293 main(int argc, char *argv[]) 294 { 295 struct addrinfo hints; /* Hints to getaddrinfo */ 296 struct addrinfo *res; /* Pointer to server address being used */ 297 struct addrinfo *res0; /* Pointer to server addresses */ 298 char * resbuf = NULL; /* Response buffer */ 299 int resbufpos = 0; /* Response buffer position */ 300 int resbuflen = 0; /* Response buffer length */ 301 char * eolp; /* Pointer to "\r\n" within resbuf */ 302 char * hln; /* Pointer within header line */ 303 char * servername; /* Name of server */ 304 char * fname = NULL; /* Name of downloaded file */ 305 char * reqbuf = NULL; /* Request buffer */ 306 int reqbufpos = 0; /* Request buffer position */ 307 int reqbuflen = 0; /* Request buffer length */ 308 ssize_t len; /* Length sent or received */ 309 int nreq = 0; /* Number of next request to send */ 310 int nres = 0; /* Number of next reply to receive */ 311 int pipelined = 0; /* != 0 if connection in pipelined mode. */ 312 int keepalive; /* != 0 if HTTP/1.0 keep-alive rcvd. */ 313 int sd = -1; /* Socket descriptor */ 314 int sdflags = 0; /* Flags on the socket sd */ 315 int fd = -1; /* Descriptor for download target file */ 316 int error; /* Error code */ 317 int statuscode; /* HTTP Status code */ 318 off_t contentlength; /* Value from Content-Length header */ 319 int chunked; /* != if transfer-encoding is chunked */ 320 off_t clen; /* Chunk length */ 321 int firstreq = 0; /* # of first request for this connection */ 322 int val; /* Value used for setsockopt call */ 323 324 /* Check that the arguments are sensible */ 325 if (argc < 2) 326 usage(); 327 328 /* Read important environment variables */ 329 readenv(); 330 331 /* Get server name and adjust arg[cv] to point at file names */ 332 servername = argv[1]; 333 argv += 2; 334 argc -= 2; 335 336 /* Allocate response buffer */ 337 resbuf = malloc(BUFSIZ); 338 if (resbuf == NULL) 339 err(1, "malloc"); 340 341 /* Look up server */ 342 memset(&hints, 0, sizeof(hints)); 343 hints.ai_family = PF_UNSPEC; 344 hints.ai_socktype = SOCK_STREAM; 345 error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername, 346 env_HTTP_PROXY ? proxyport : "http", &hints, &res0); 347 if (error) 348 errx(1, "host = %s, port = %s: %s", 349 env_HTTP_PROXY ? env_HTTP_PROXY : servername, 350 env_HTTP_PROXY ? proxyport : "http", 351 gai_strerror(error)); 352 if (res0 == NULL) 353 errx(1, "could not look up %s", servername); 354 res = res0; 355 356 /* Do the fetching */ 357 while (nres < argc) { 358 /* Make sure we have a connected socket */ 359 for (; sd == -1; res = res->ai_next) { 360 /* No addresses left to try :-( */ 361 if (res == NULL) 362 errx(1, "Could not connect to %s", servername); 363 364 /* Create a socket... */ 365 sd = socket(res->ai_family, res->ai_socktype, 366 res->ai_protocol); 367 if (sd == -1) 368 continue; 369 370 /* ... set 15-second timeouts ... */ 371 setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO, 372 (void *)&timo, (socklen_t)sizeof(timo)); 373 setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, 374 (void *)&timo, (socklen_t)sizeof(timo)); 375 376 /* ... disable SIGPIPE generation ... */ 377 val = 1; 378 setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, 379 (void *)&val, sizeof(int)); 380 381 /* ... and connect to the server. */ 382 if(connect(sd, res->ai_addr, res->ai_addrlen)) { 383 close(sd); 384 sd = -1; 385 continue; 386 } 387 388 firstreq = nres; 389 } 390 391 /* 392 * If in pipelined HTTP mode, put socket into non-blocking 393 * mode, since we're probably going to want to try to send 394 * several HTTP requests. 395 */ 396 if (pipelined) { 397 sdflags = fcntl(sd, F_GETFL); 398 if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1) 399 err(1, "fcntl"); 400 } 401 402 /* Construct requests and/or send them without blocking */ 403 while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) { 404 /* If not in the middle of a request, make one */ 405 if (reqbuf == NULL) { 406 reqbuflen = makerequest(&reqbuf, argv[nreq], 407 servername, (nreq == argc - 1)); 408 reqbufpos = 0; 409 } 410 411 /* If in pipelined mode, try to send the request */ 412 if (pipelined) { 413 while (reqbufpos < reqbuflen) { 414 len = send(sd, reqbuf + reqbufpos, 415 reqbuflen - reqbufpos, 0); 416 if (len == -1) 417 break; 418 reqbufpos += len; 419 } 420 if (reqbufpos < reqbuflen) { 421 if (errno != EAGAIN) 422 goto conndied; 423 break; 424 } else { 425 free(reqbuf); 426 reqbuf = NULL; 427 nreq++; 428 } 429 } 430 } 431 432 /* Put connection back into blocking mode */ 433 if (pipelined) { 434 if (fcntl(sd, F_SETFL, sdflags) == -1) 435 err(1, "fcntl"); 436 } 437 438 /* Do we need to blocking-send a request? */ 439 if (nres == nreq) { 440 while (reqbufpos < reqbuflen) { 441 len = send(sd, reqbuf + reqbufpos, 442 reqbuflen - reqbufpos, 0); 443 if (len == -1) 444 goto conndied; 445 reqbufpos += len; 446 } 447 free(reqbuf); 448 reqbuf = NULL; 449 nreq++; 450 } 451 452 /* Scan through the response processing headers. */ 453 statuscode = 0; 454 contentlength = -1; 455 chunked = 0; 456 keepalive = 0; 457 do { 458 /* Get a header line */ 459 error = readln(sd, resbuf, &resbuflen, &resbufpos); 460 if (error) 461 goto conndied; 462 hln = resbuf + resbufpos; 463 eolp = strnstr(hln, "\r\n", resbuflen - resbufpos); 464 resbufpos = (eolp - resbuf) + 2; 465 *eolp = '\0'; 466 467 /* Make sure it doesn't contain a NUL character */ 468 if (strchr(hln, '\0') != eolp) 469 goto conndied; 470 471 if (statuscode == 0) { 472 /* The first line MUST be HTTP/1.x xxx ... */ 473 if ((strncmp(hln, "HTTP/1.", 7) != 0) || 474 ! isdigit(hln[7])) 475 goto conndied; 476 477 /* 478 * If the minor version number isn't zero, 479 * then we can assume that pipelining our 480 * requests is OK -- as long as we don't 481 * see a "Connection: close" line later 482 * and we either have a Content-Length or 483 * Transfer-Encoding: chunked header to 484 * tell us the length. 485 */ 486 if (hln[7] != '0') 487 pipelined = 1; 488 489 /* Skip over the minor version number */ 490 hln = strchr(hln + 7, ' '); 491 if (hln == NULL) 492 goto conndied; 493 else 494 hln++; 495 496 /* Read the status code */ 497 while (isdigit(*hln)) { 498 statuscode = statuscode * 10 + 499 *hln - '0'; 500 hln++; 501 } 502 503 if (statuscode < 100 || statuscode > 599) 504 goto conndied; 505 506 /* Ignore the rest of the line */ 507 continue; 508 } 509 510 /* 511 * Check for "Connection: close" or 512 * "Connection: Keep-Alive" header 513 */ 514 if (strncasecmp(hln, "Connection:", 11) == 0) { 515 hln += 11; 516 if (strcasestr(hln, "close") != NULL) 517 pipelined = 0; 518 if (strcasestr(hln, "Keep-Alive") != NULL) 519 keepalive = 1; 520 521 /* Next header... */ 522 continue; 523 } 524 525 /* Check for "Content-Length:" header */ 526 if (strncasecmp(hln, "Content-Length:", 15) == 0) { 527 hln += 15; 528 contentlength = 0; 529 530 /* Find the start of the length */ 531 while (!isdigit(*hln) && (*hln != '\0')) 532 hln++; 533 534 /* Compute the length */ 535 while (isdigit(*hln)) { 536 if (contentlength >= OFF_MAX / 10) { 537 /* Nasty people... */ 538 goto conndied; 539 } 540 contentlength = contentlength * 10 + 541 *hln - '0'; 542 hln++; 543 } 544 545 /* Next header... */ 546 continue; 547 } 548 549 /* Check for "Transfer-Encoding: chunked" header */ 550 if (strncasecmp(hln, "Transfer-Encoding:", 18) == 0) { 551 hln += 18; 552 if (strcasestr(hln, "chunked") != NULL) 553 chunked = 1; 554 555 /* Next header... */ 556 continue; 557 } 558 559 /* We blithely ignore any other header lines */ 560 561 /* No more header lines */ 562 if (strlen(hln) == 0) { 563 /* 564 * If the status code was 1xx, then there will 565 * be a real header later. Servers may emit 566 * 1xx header blocks at will, but since we 567 * don't expect one, we should just ignore it. 568 */ 569 if (100 <= statuscode && statuscode <= 199) { 570 statuscode = 0; 571 continue; 572 } 573 574 /* End of header; message body follows */ 575 break; 576 } 577 } while (1); 578 579 /* No message body for 204 or 304 */ 580 if (statuscode == 204 || statuscode == 304) { 581 nres++; 582 continue; 583 } 584 585 /* 586 * There should be a message body coming, but we only want 587 * to send it to a file if the status code is 200 588 */ 589 if (statuscode == 200) { 590 /* Generate a file name for the download */ 591 fname = strrchr(argv[nres], '/'); 592 if (fname == NULL) 593 fname = argv[nres]; 594 else 595 fname++; 596 if (strlen(fname) == 0) 597 errx(1, "Cannot obtain file name from %s\n", 598 argv[nres]); 599 600 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 601 if (fd == -1) 602 errx(1, "open(%s)", fname); 603 } 604 605 /* Read the message and send data to fd if appropriate */ 606 if (chunked) { 607 /* Handle a chunked-encoded entity */ 608 609 /* Read chunks */ 610 do { 611 error = readln(sd, resbuf, &resbuflen, 612 &resbufpos); 613 if (error) 614 goto conndied; 615 hln = resbuf + resbufpos; 616 eolp = strstr(hln, "\r\n"); 617 resbufpos = (eolp - resbuf) + 2; 618 619 clen = 0; 620 while (isxdigit(*hln)) { 621 if (clen >= OFF_MAX / 16) { 622 /* Nasty people... */ 623 goto conndied; 624 } 625 if (isdigit(*hln)) 626 clen = clen * 16 + *hln - '0'; 627 else 628 clen = clen * 16 + 10 + 629 tolower(*hln) - 'a'; 630 hln++; 631 } 632 633 error = copybytes(sd, fd, clen, resbuf, 634 &resbuflen, &resbufpos); 635 if (error) { 636 goto conndied; 637 } 638 } while (clen != 0); 639 640 /* Read trailer and final CRLF */ 641 do { 642 error = readln(sd, resbuf, &resbuflen, 643 &resbufpos); 644 if (error) 645 goto conndied; 646 hln = resbuf + resbufpos; 647 eolp = strstr(hln, "\r\n"); 648 resbufpos = (eolp - resbuf) + 2; 649 } while (hln != eolp); 650 } else if (contentlength != -1) { 651 error = copybytes(sd, fd, contentlength, resbuf, 652 &resbuflen, &resbufpos); 653 if (error) 654 goto conndied; 655 } else { 656 /* 657 * Not chunked, and no content length header. 658 * Read everything until the server closes the 659 * socket. 660 */ 661 error = copybytes(sd, fd, OFF_MAX, resbuf, 662 &resbuflen, &resbufpos); 663 if (error == -1) 664 goto conndied; 665 pipelined = 0; 666 } 667 668 if (fd != -1) { 669 close(fd); 670 fd = -1; 671 } 672 673 fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres], 674 statuscode); 675 if (statuscode == 200) 676 fprintf(stderr, "OK\n"); 677 else if (statuscode < 300) 678 fprintf(stderr, "Successful (ignored)\n"); 679 else if (statuscode < 400) 680 fprintf(stderr, "Redirection (ignored)\n"); 681 else 682 fprintf(stderr, "Error (ignored)\n"); 683 684 /* We've finished this file! */ 685 nres++; 686 687 /* 688 * If necessary, clean up this connection so that we 689 * can start a new one. 690 */ 691 if (pipelined == 0 && keepalive == 0) 692 goto cleanupconn; 693 continue; 694 695 conndied: 696 /* 697 * Something went wrong -- our connection died, the server 698 * sent us garbage, etc. If this happened on the first 699 * request we sent over this connection, give up. Otherwise, 700 * close this connection, open a new one, and reissue the 701 * request. 702 */ 703 if (nres == firstreq) 704 errx(1, "Connection failure"); 705 706 cleanupconn: 707 /* 708 * Clean up our connection and keep on going 709 */ 710 shutdown(sd, SHUT_RDWR); 711 close(sd); 712 sd = -1; 713 if (fd != -1) { 714 close(fd); 715 fd = -1; 716 } 717 if (reqbuf != NULL) { 718 free(reqbuf); 719 reqbuf = NULL; 720 } 721 nreq = nres; 722 res = res0; 723 pipelined = 0; 724 resbufpos = resbuflen = 0; 725 continue; 726 } 727 728 free(resbuf); 729 freeaddrinfo(res0); 730 731 return 0; 732 } 733