1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright 2005 Colin Percival 5 * All rights reserved 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted providing that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/time.h> 31 #include <sys/socket.h> 32 33 #include <ctype.h> 34 #include <err.h> 35 #include <errno.h> 36 #include <fcntl.h> 37 #include <limits.h> 38 #include <netdb.h> 39 #include <stdint.h> 40 #include <stdio.h> 41 #include <stdlib.h> 42 #include <string.h> 43 #include <sysexits.h> 44 #include <unistd.h> 45 46 static const char * env_HTTP_PROXY; 47 static char * env_HTTP_PROXY_AUTH; 48 static const char * env_HTTP_USER_AGENT; 49 static char * env_HTTP_TIMEOUT; 50 static const char * proxyport; 51 static char * proxyauth; 52 53 static struct timeval timo = { 15, 0}; 54 55 static void 56 usage(void) 57 { 58 59 fprintf(stderr, "usage: phttpget server [file ...]\n"); 60 exit(EX_USAGE); 61 } 62 63 /* 64 * Base64 encode a string; the string returned, if non-NULL, is 65 * allocated using malloc() and must be freed by the caller. 66 */ 67 static char * 68 b64enc(const char *ptext) 69 { 70 static const char base64[] = 71 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 72 "abcdefghijklmnopqrstuvwxyz" 73 "0123456789+/"; 74 const char *pt; 75 char *ctext, *pc; 76 size_t ptlen, ctlen; 77 uint32_t t; 78 unsigned int j; 79 80 /* 81 * Encoded length is 4 characters per 3-byte block or partial 82 * block of plaintext, plus one byte for the terminating NUL 83 */ 84 ptlen = strlen(ptext); 85 if (ptlen > ((SIZE_MAX - 1) / 4) * 3 - 2) 86 return NULL; /* Possible integer overflow */ 87 ctlen = 4 * ((ptlen + 2) / 3) + 1; 88 if ((ctext = malloc(ctlen)) == NULL) 89 return NULL; 90 ctext[ctlen - 1] = 0; 91 92 /* 93 * Scan through ptext, reading up to 3 bytes from ptext and 94 * writing 4 bytes to ctext, until we run out of input. 95 */ 96 for (pt = ptext, pc = ctext; ptlen; ptlen -= 3, pc += 4) { 97 /* Read 3 bytes */ 98 for (t = j = 0; j < 3; j++) { 99 t <<= 8; 100 if (j < ptlen) 101 t += *pt++; 102 } 103 104 /* Write 4 bytes */ 105 for (j = 0; j < 4; j++) { 106 if (j <= ptlen + 1) 107 pc[j] = base64[(t >> 18) & 0x3f]; 108 else 109 pc[j] = '='; 110 t <<= 6; 111 } 112 113 /* If we're done, exit the loop */ 114 if (ptlen <= 3) 115 break; 116 } 117 118 return (ctext); 119 } 120 121 static void 122 readenv(void) 123 { 124 char *proxy_auth_userpass, *proxy_auth_userpass64, *p; 125 char *proxy_auth_user = NULL; 126 char *proxy_auth_pass = NULL; 127 long http_timeout; 128 129 env_HTTP_PROXY = getenv("HTTP_PROXY"); 130 if (env_HTTP_PROXY == NULL) 131 env_HTTP_PROXY = getenv("http_proxy"); 132 if (env_HTTP_PROXY != NULL) { 133 if (strncmp(env_HTTP_PROXY, "http://", 7) == 0) 134 env_HTTP_PROXY += 7; 135 p = strchr(env_HTTP_PROXY, '/'); 136 if (p != NULL) 137 *p = 0; 138 p = strchr(env_HTTP_PROXY, ':'); 139 if (p != NULL) { 140 *p = 0; 141 proxyport = p + 1; 142 } else 143 proxyport = "3128"; 144 } 145 146 env_HTTP_PROXY_AUTH = getenv("HTTP_PROXY_AUTH"); 147 if ((env_HTTP_PROXY != NULL) && 148 (env_HTTP_PROXY_AUTH != NULL) && 149 (strncasecmp(env_HTTP_PROXY_AUTH, "basic:" , 6) == 0)) { 150 /* Ignore authentication scheme */ 151 (void) strsep(&env_HTTP_PROXY_AUTH, ":"); 152 153 /* Ignore realm */ 154 (void) strsep(&env_HTTP_PROXY_AUTH, ":"); 155 156 /* Obtain username and password */ 157 proxy_auth_user = strsep(&env_HTTP_PROXY_AUTH, ":"); 158 proxy_auth_pass = env_HTTP_PROXY_AUTH; 159 } 160 161 if ((proxy_auth_user != NULL) && (proxy_auth_pass != NULL)) { 162 asprintf(&proxy_auth_userpass, "%s:%s", 163 proxy_auth_user, proxy_auth_pass); 164 if (proxy_auth_userpass == NULL) 165 err(1, "asprintf"); 166 167 proxy_auth_userpass64 = b64enc(proxy_auth_userpass); 168 if (proxy_auth_userpass64 == NULL) 169 err(1, "malloc"); 170 171 asprintf(&proxyauth, "Proxy-Authorization: Basic %s\r\n", 172 proxy_auth_userpass64); 173 if (proxyauth == NULL) 174 err(1, "asprintf"); 175 176 free(proxy_auth_userpass); 177 free(proxy_auth_userpass64); 178 } else 179 proxyauth = NULL; 180 181 env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT"); 182 if (env_HTTP_USER_AGENT == NULL) 183 env_HTTP_USER_AGENT = "phttpget/0.1"; 184 185 env_HTTP_TIMEOUT = getenv("HTTP_TIMEOUT"); 186 if (env_HTTP_TIMEOUT != NULL) { 187 http_timeout = strtol(env_HTTP_TIMEOUT, &p, 10); 188 if ((*env_HTTP_TIMEOUT == '\0') || (*p != '\0') || 189 (http_timeout < 0)) 190 warnx("HTTP_TIMEOUT (%s) is not a positive integer", 191 env_HTTP_TIMEOUT); 192 else 193 timo.tv_sec = http_timeout; 194 } 195 } 196 197 static int 198 makerequest(char ** buf, char * path, char * server, int connclose) 199 { 200 int buflen; 201 202 buflen = asprintf(buf, 203 "GET %s%s/%s HTTP/1.1\r\n" 204 "Host: %s\r\n" 205 "User-Agent: %s\r\n" 206 "%s" 207 "%s" 208 "\r\n", 209 env_HTTP_PROXY ? "http://" : "", 210 env_HTTP_PROXY ? server : "", 211 path, server, env_HTTP_USER_AGENT, 212 proxyauth ? proxyauth : "", 213 connclose ? "Connection: Close\r\n" : "Connection: Keep-Alive\r\n"); 214 if (buflen == -1) 215 err(1, "asprintf"); 216 return(buflen); 217 } 218 219 static int 220 readln(int sd, char * resbuf, int * resbuflen, int * resbufpos) 221 { 222 ssize_t len; 223 224 while (strnstr(resbuf + *resbufpos, "\r\n", 225 *resbuflen - *resbufpos) == NULL) { 226 /* Move buffered data to the start of the buffer */ 227 if (*resbufpos != 0) { 228 memmove(resbuf, resbuf + *resbufpos, 229 *resbuflen - *resbufpos); 230 *resbuflen -= *resbufpos; 231 *resbufpos = 0; 232 } 233 234 /* If the buffer is full, complain */ 235 if (*resbuflen == BUFSIZ) 236 return -1; 237 238 /* Read more data into the buffer */ 239 len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0); 240 if ((len == 0) || 241 ((len == -1) && (errno != EINTR))) 242 return -1; 243 244 if (len != -1) 245 *resbuflen += len; 246 } 247 248 return 0; 249 } 250 251 static int 252 copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen, 253 int * resbufpos) 254 { 255 ssize_t len; 256 257 while (copylen) { 258 /* Write data from resbuf to fd */ 259 len = *resbuflen - *resbufpos; 260 if (copylen < len) 261 len = copylen; 262 if (len > 0) { 263 if (fd != -1) 264 len = write(fd, resbuf + *resbufpos, len); 265 if (len == -1) 266 err(1, "write"); 267 *resbufpos += len; 268 copylen -= len; 269 continue; 270 } 271 272 /* Read more data into buffer */ 273 len = recv(sd, resbuf, BUFSIZ, 0); 274 if (len == -1) { 275 if (errno == EINTR) 276 continue; 277 return -1; 278 } else if (len == 0) { 279 return -2; 280 } else { 281 *resbuflen = len; 282 *resbufpos = 0; 283 } 284 } 285 286 return 0; 287 } 288 289 int 290 main(int argc, char *argv[]) 291 { 292 struct addrinfo hints; /* Hints to getaddrinfo */ 293 struct addrinfo *res; /* Pointer to server address being used */ 294 struct addrinfo *res0; /* Pointer to server addresses */ 295 char * resbuf = NULL; /* Response buffer */ 296 int resbufpos = 0; /* Response buffer position */ 297 int resbuflen = 0; /* Response buffer length */ 298 char * eolp; /* Pointer to "\r\n" within resbuf */ 299 char * hln; /* Pointer within header line */ 300 char * servername; /* Name of server */ 301 char * fname = NULL; /* Name of downloaded file */ 302 char * reqbuf = NULL; /* Request buffer */ 303 int reqbufpos = 0; /* Request buffer position */ 304 int reqbuflen = 0; /* Request buffer length */ 305 ssize_t len; /* Length sent or received */ 306 int nreq = 0; /* Number of next request to send */ 307 int nres = 0; /* Number of next reply to receive */ 308 int pipelined = 0; /* != 0 if connection in pipelined mode. */ 309 int keepalive; /* != 0 if HTTP/1.0 keep-alive rcvd. */ 310 int sd = -1; /* Socket descriptor */ 311 int sdflags = 0; /* Flags on the socket sd */ 312 int fd = -1; /* Descriptor for download target file */ 313 int error; /* Error code */ 314 int statuscode; /* HTTP Status code */ 315 off_t contentlength; /* Value from Content-Length header */ 316 int chunked; /* != if transfer-encoding is chunked */ 317 off_t clen; /* Chunk length */ 318 int firstreq = 0; /* # of first request for this connection */ 319 int val; /* Value used for setsockopt call */ 320 321 /* Check that the arguments are sensible */ 322 if (argc < 2) 323 usage(); 324 325 /* Read important environment variables */ 326 readenv(); 327 328 /* Get server name and adjust arg[cv] to point at file names */ 329 servername = argv[1]; 330 argv += 2; 331 argc -= 2; 332 333 /* Allocate response buffer */ 334 resbuf = malloc(BUFSIZ); 335 if (resbuf == NULL) 336 err(1, "malloc"); 337 338 /* Look up server */ 339 memset(&hints, 0, sizeof(hints)); 340 hints.ai_family = PF_UNSPEC; 341 hints.ai_socktype = SOCK_STREAM; 342 error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername, 343 env_HTTP_PROXY ? proxyport : "http", &hints, &res0); 344 if (error) 345 errx(1, "host = %s, port = %s: %s", 346 env_HTTP_PROXY ? env_HTTP_PROXY : servername, 347 env_HTTP_PROXY ? proxyport : "http", 348 gai_strerror(error)); 349 if (res0 == NULL) 350 errx(1, "could not look up %s", servername); 351 res = res0; 352 353 /* Do the fetching */ 354 while (nres < argc) { 355 /* Make sure we have a connected socket */ 356 for (; sd == -1; res = res->ai_next) { 357 /* No addresses left to try :-( */ 358 if (res == NULL) 359 errx(1, "Could not connect to %s", servername); 360 361 /* Create a socket... */ 362 sd = socket(res->ai_family, res->ai_socktype, 363 res->ai_protocol); 364 if (sd == -1) 365 continue; 366 367 /* ... set 15-second timeouts ... */ 368 setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO, 369 (void *)&timo, (socklen_t)sizeof(timo)); 370 setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, 371 (void *)&timo, (socklen_t)sizeof(timo)); 372 373 /* ... disable SIGPIPE generation ... */ 374 val = 1; 375 setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, 376 (void *)&val, sizeof(int)); 377 378 /* ... and connect to the server. */ 379 if(connect(sd, res->ai_addr, res->ai_addrlen)) { 380 close(sd); 381 sd = -1; 382 continue; 383 } 384 385 firstreq = nres; 386 } 387 388 /* 389 * If in pipelined HTTP mode, put socket into non-blocking 390 * mode, since we're probably going to want to try to send 391 * several HTTP requests. 392 */ 393 if (pipelined) { 394 sdflags = fcntl(sd, F_GETFL); 395 if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1) 396 err(1, "fcntl"); 397 } 398 399 /* Construct requests and/or send them without blocking */ 400 while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) { 401 /* If not in the middle of a request, make one */ 402 if (reqbuf == NULL) { 403 reqbuflen = makerequest(&reqbuf, argv[nreq], 404 servername, (nreq == argc - 1)); 405 reqbufpos = 0; 406 } 407 408 /* If in pipelined mode, try to send the request */ 409 if (pipelined) { 410 while (reqbufpos < reqbuflen) { 411 len = send(sd, reqbuf + reqbufpos, 412 reqbuflen - reqbufpos, 0); 413 if (len == -1) 414 break; 415 reqbufpos += len; 416 } 417 if (reqbufpos < reqbuflen) { 418 if (errno != EAGAIN) 419 goto conndied; 420 break; 421 } else { 422 free(reqbuf); 423 reqbuf = NULL; 424 nreq++; 425 } 426 } 427 } 428 429 /* Put connection back into blocking mode */ 430 if (pipelined) { 431 if (fcntl(sd, F_SETFL, sdflags) == -1) 432 err(1, "fcntl"); 433 } 434 435 /* Do we need to blocking-send a request? */ 436 if (nres == nreq) { 437 while (reqbufpos < reqbuflen) { 438 len = send(sd, reqbuf + reqbufpos, 439 reqbuflen - reqbufpos, 0); 440 if (len == -1) 441 goto conndied; 442 reqbufpos += len; 443 } 444 free(reqbuf); 445 reqbuf = NULL; 446 nreq++; 447 } 448 449 /* Scan through the response processing headers. */ 450 statuscode = 0; 451 contentlength = -1; 452 chunked = 0; 453 keepalive = 0; 454 do { 455 /* Get a header line */ 456 error = readln(sd, resbuf, &resbuflen, &resbufpos); 457 if (error) 458 goto conndied; 459 hln = resbuf + resbufpos; 460 eolp = strnstr(hln, "\r\n", resbuflen - resbufpos); 461 resbufpos = (eolp - resbuf) + 2; 462 *eolp = '\0'; 463 464 /* Make sure it doesn't contain a NUL character */ 465 if (strchr(hln, '\0') != eolp) 466 goto conndied; 467 468 if (statuscode == 0) { 469 /* The first line MUST be HTTP/1.x xxx ... */ 470 if ((strncmp(hln, "HTTP/1.", 7) != 0) || 471 ! isdigit(hln[7])) 472 goto conndied; 473 474 /* 475 * If the minor version number isn't zero, 476 * then we can assume that pipelining our 477 * requests is OK -- as long as we don't 478 * see a "Connection: close" line later 479 * and we either have a Content-Length or 480 * Transfer-Encoding: chunked header to 481 * tell us the length. 482 */ 483 if (hln[7] != '0') 484 pipelined = 1; 485 486 /* Skip over the minor version number */ 487 hln = strchr(hln + 7, ' '); 488 if (hln == NULL) 489 goto conndied; 490 else 491 hln++; 492 493 /* Read the status code */ 494 while (isdigit(*hln)) { 495 statuscode = statuscode * 10 + 496 *hln - '0'; 497 hln++; 498 } 499 500 if (statuscode < 100 || statuscode > 599) 501 goto conndied; 502 503 /* Ignore the rest of the line */ 504 continue; 505 } 506 507 /* 508 * Check for "Connection: close" or 509 * "Connection: Keep-Alive" header 510 */ 511 if (strncasecmp(hln, "Connection:", 11) == 0) { 512 hln += 11; 513 if (strcasestr(hln, "close") != NULL) 514 pipelined = 0; 515 if (strcasestr(hln, "Keep-Alive") != NULL) 516 keepalive = 1; 517 518 /* Next header... */ 519 continue; 520 } 521 522 /* Check for "Content-Length:" header */ 523 if (strncasecmp(hln, "Content-Length:", 15) == 0) { 524 hln += 15; 525 contentlength = 0; 526 527 /* Find the start of the length */ 528 while (!isdigit(*hln) && (*hln != '\0')) 529 hln++; 530 531 /* Compute the length */ 532 while (isdigit(*hln)) { 533 if (contentlength >= OFF_MAX / 10) { 534 /* Nasty people... */ 535 goto conndied; 536 } 537 contentlength = contentlength * 10 + 538 *hln - '0'; 539 hln++; 540 } 541 542 /* Next header... */ 543 continue; 544 } 545 546 /* Check for "Transfer-Encoding: chunked" header */ 547 if (strncasecmp(hln, "Transfer-Encoding:", 18) == 0) { 548 hln += 18; 549 if (strcasestr(hln, "chunked") != NULL) 550 chunked = 1; 551 552 /* Next header... */ 553 continue; 554 } 555 556 /* We blithely ignore any other header lines */ 557 558 /* No more header lines */ 559 if (strlen(hln) == 0) { 560 /* 561 * If the status code was 1xx, then there will 562 * be a real header later. Servers may emit 563 * 1xx header blocks at will, but since we 564 * don't expect one, we should just ignore it. 565 */ 566 if (100 <= statuscode && statuscode <= 199) { 567 statuscode = 0; 568 continue; 569 } 570 571 /* End of header; message body follows */ 572 break; 573 } 574 } while (1); 575 576 /* No message body for 204 or 304 */ 577 if (statuscode == 204 || statuscode == 304) { 578 nres++; 579 continue; 580 } 581 582 /* 583 * There should be a message body coming, but we only want 584 * to send it to a file if the status code is 200 585 */ 586 if (statuscode == 200) { 587 /* Generate a file name for the download */ 588 fname = strrchr(argv[nres], '/'); 589 if (fname == NULL) 590 fname = argv[nres]; 591 else 592 fname++; 593 if (strlen(fname) == 0) 594 errx(1, "Cannot obtain file name from %s\n", 595 argv[nres]); 596 597 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 598 if (fd == -1) 599 errx(1, "open(%s)", fname); 600 } 601 602 /* Read the message and send data to fd if appropriate */ 603 if (chunked) { 604 /* Handle a chunked-encoded entity */ 605 606 /* Read chunks */ 607 do { 608 error = readln(sd, resbuf, &resbuflen, 609 &resbufpos); 610 if (error) 611 goto conndied; 612 hln = resbuf + resbufpos; 613 eolp = strstr(hln, "\r\n"); 614 resbufpos = (eolp - resbuf) + 2; 615 616 clen = 0; 617 while (isxdigit(*hln)) { 618 if (clen >= OFF_MAX / 16) { 619 /* Nasty people... */ 620 goto conndied; 621 } 622 if (isdigit(*hln)) 623 clen = clen * 16 + *hln - '0'; 624 else 625 clen = clen * 16 + 10 + 626 tolower(*hln) - 'a'; 627 hln++; 628 } 629 630 error = copybytes(sd, fd, clen, resbuf, 631 &resbuflen, &resbufpos); 632 if (error) { 633 goto conndied; 634 } 635 } while (clen != 0); 636 637 /* Read trailer and final CRLF */ 638 do { 639 error = readln(sd, resbuf, &resbuflen, 640 &resbufpos); 641 if (error) 642 goto conndied; 643 hln = resbuf + resbufpos; 644 eolp = strstr(hln, "\r\n"); 645 resbufpos = (eolp - resbuf) + 2; 646 } while (hln != eolp); 647 } else if (contentlength != -1) { 648 error = copybytes(sd, fd, contentlength, resbuf, 649 &resbuflen, &resbufpos); 650 if (error) 651 goto conndied; 652 } else { 653 /* 654 * Not chunked, and no content length header. 655 * Read everything until the server closes the 656 * socket. 657 */ 658 error = copybytes(sd, fd, OFF_MAX, resbuf, 659 &resbuflen, &resbufpos); 660 if (error == -1) 661 goto conndied; 662 pipelined = 0; 663 } 664 665 if (fd != -1) { 666 close(fd); 667 fd = -1; 668 } 669 670 fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres], 671 statuscode); 672 if (statuscode == 200) 673 fprintf(stderr, "OK\n"); 674 else if (statuscode < 300) 675 fprintf(stderr, "Successful (ignored)\n"); 676 else if (statuscode < 400) 677 fprintf(stderr, "Redirection (ignored)\n"); 678 else 679 fprintf(stderr, "Error (ignored)\n"); 680 681 /* We've finished this file! */ 682 nres++; 683 684 /* 685 * If necessary, clean up this connection so that we 686 * can start a new one. 687 */ 688 if (pipelined == 0 && keepalive == 0) 689 goto cleanupconn; 690 continue; 691 692 conndied: 693 /* 694 * Something went wrong -- our connection died, the server 695 * sent us garbage, etc. If this happened on the first 696 * request we sent over this connection, give up. Otherwise, 697 * close this connection, open a new one, and reissue the 698 * request. 699 */ 700 if (nres == firstreq) 701 errx(1, "Connection failure"); 702 703 cleanupconn: 704 /* 705 * Clean up our connection and keep on going 706 */ 707 shutdown(sd, SHUT_RDWR); 708 close(sd); 709 sd = -1; 710 if (fd != -1) { 711 close(fd); 712 fd = -1; 713 } 714 if (reqbuf != NULL) { 715 free(reqbuf); 716 reqbuf = NULL; 717 } 718 nreq = nres; 719 res = res0; 720 pipelined = 0; 721 resbufpos = resbuflen = 0; 722 continue; 723 } 724 725 free(resbuf); 726 freeaddrinfo(res0); 727 728 return 0; 729 } 730