1 /*- 2 * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer 10 * in this position and unchanged. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/param.h> 32 #include <sys/stat.h> 33 #include <sys/socket.h> 34 35 #include <ctype.h> 36 #include <err.h> 37 #include <errno.h> 38 #include <signal.h> 39 #include <stdio.h> 40 #include <stdlib.h> 41 #include <string.h> 42 #include <sysexits.h> 43 #include <unistd.h> 44 45 #include <fetch.h> 46 47 #define MINBUFSIZE 4096 48 49 /* Option flags */ 50 int A_flag; /* -A: do not follow 302 redirects */ 51 int a_flag; /* -a: auto retry */ 52 size_t B_size; /* -B: buffer size */ 53 int b_flag; /*! -b: workaround TCP bug */ 54 char *c_dirname; /* -c: remote directory */ 55 int d_flag; /* -d: direct connection */ 56 int F_flag; /* -F: restart without checking mtime */ 57 char *f_filename; /* -f: file to fetch */ 58 char *h_hostname; /* -h: host to fetch from */ 59 int l_flag; /* -l: link rather than copy file: URLs */ 60 int m_flag; /* -[Mm]: mirror mode */ 61 int n_flag; /* -n: do not preserve modification time */ 62 int o_flag; /* -o: specify output file */ 63 int o_directory; /* output file is a directory */ 64 char *o_filename; /* name of output file */ 65 int o_stdout; /* output file is stdout */ 66 int once_flag; /* -1: stop at first successful file */ 67 int p_flag; /* -[Pp]: use passive FTP */ 68 int R_flag; /* -R: don't delete partially transferred files */ 69 int r_flag; /* -r: restart previously interrupted transfer */ 70 off_t S_size; /* -S: require size to match */ 71 int s_flag; /* -s: show size, don't fetch */ 72 u_int T_secs = 0; /* -T: transfer timeout in seconds */ 73 int t_flag; /*! -t: workaround TCP bug */ 74 int U_flag; /* -U: do not use high ports */ 75 int v_level = 1; /* -v: verbosity level */ 76 int v_tty; /* stdout is a tty */ 77 u_int w_secs; /* -w: retry delay */ 78 int family = PF_UNSPEC; /* -[46]: address family to use */ 79 80 int sigalrm; /* SIGALRM received */ 81 int siginfo; /* SIGINFO received */ 82 int sigint; /* SIGINT received */ 83 84 u_int ftp_timeout; /* default timeout for FTP transfers */ 85 u_int http_timeout; /* default timeout for HTTP transfers */ 86 u_char *buf; /* transfer buffer */ 87 88 89 void 90 sig_handler(int sig) 91 { 92 switch (sig) { 93 case SIGALRM: 94 sigalrm = 1; 95 break; 96 case SIGINFO: 97 siginfo = 1; 98 break; 99 case SIGINT: 100 sigint = 1; 101 break; 102 } 103 } 104 105 struct xferstat { 106 char name[40]; 107 struct timeval start; 108 struct timeval end; 109 struct timeval last; 110 off_t size; 111 off_t offset; 112 off_t rcvd; 113 }; 114 115 void 116 stat_display(struct xferstat *xs, int force) 117 { 118 struct timeval now; 119 120 if (!v_tty || !v_level) 121 return; 122 123 gettimeofday(&now, NULL); 124 if (!force && now.tv_sec <= xs->last.tv_sec) 125 return; 126 xs->last = now; 127 128 fprintf(stderr, "\rReceiving %s", xs->name); 129 if (xs->size == -1) 130 fprintf(stderr, ": %lld bytes", xs->rcvd); 131 else 132 fprintf(stderr, " (%lld bytes): %d%%", xs->size, 133 (int)((100.0 * xs->rcvd) / xs->size)); 134 } 135 136 void 137 stat_start(struct xferstat *xs, char *name, off_t size, off_t offset) 138 { 139 snprintf(xs->name, sizeof xs->name, "%s", name); 140 gettimeofday(&xs->start, NULL); 141 xs->last.tv_sec = xs->last.tv_usec = 0; 142 xs->end = xs->last; 143 xs->size = size; 144 xs->offset = offset; 145 xs->rcvd = offset; 146 stat_display(xs, 1); 147 } 148 149 void 150 stat_update(struct xferstat *xs, off_t rcvd, int force) 151 { 152 xs->rcvd = rcvd; 153 stat_display(xs, 0); 154 } 155 156 void 157 stat_end(struct xferstat *xs) 158 { 159 double delta; 160 double bps; 161 162 if (!v_level) 163 return; 164 165 gettimeofday(&xs->end, NULL); 166 167 stat_display(xs, 1); 168 fputc('\n', stderr); 169 delta = (xs->end.tv_sec + (xs->end.tv_usec / 1.e6)) 170 - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 171 fprintf(stderr, "%lld bytes transferred in %.1f seconds ", 172 xs->rcvd - xs->offset, delta); 173 bps = (xs->rcvd - xs->offset) / delta; 174 if (bps > 1024*1024) 175 fprintf(stderr, "(%.2f MBps)\n", bps / (1024*1024)); 176 else if (bps > 1024) 177 fprintf(stderr, "(%.2f kBps)\n", bps / 1024); 178 else 179 fprintf(stderr, "(%.2f Bps)\n", bps); 180 } 181 182 int 183 fetch(char *URL, char *path) 184 { 185 struct url *url; 186 struct url_stat us; 187 struct stat sb; 188 struct xferstat xs; 189 FILE *f, *of; 190 size_t size, wr; 191 off_t count; 192 char flags[8]; 193 int n, r; 194 u_int timeout; 195 u_char *ptr; 196 197 f = of = NULL; 198 199 /* parse URL */ 200 if ((url = fetchParseURL(URL)) == NULL) { 201 warnx("%s: parse error", URL); 202 goto failure; 203 } 204 205 /* if no scheme was specified, take a guess */ 206 if (!*url->scheme) { 207 if (!*url->host) 208 strcpy(url->scheme, SCHEME_FILE); 209 else if (strncasecmp(url->host, "ftp.", 4)) 210 strcpy(url->scheme, SCHEME_FTP); 211 else if (strncasecmp(url->host, "www.", 4)) 212 strcpy(url->scheme, SCHEME_HTTP); 213 } 214 215 timeout = 0; 216 *flags = 0; 217 count = 0; 218 219 /* common flags */ 220 if (v_level > 1) 221 strcat(flags, "v"); 222 switch (family) { 223 case PF_INET: 224 strcat(flags, "4"); 225 break; 226 case PF_INET6: 227 strcat(flags, "6"); 228 break; 229 } 230 231 /* FTP specific flags */ 232 if (strcmp(url->scheme, "ftp") == 0) { 233 if (p_flag) 234 strcat(flags, "p"); 235 if (d_flag) 236 strcat(flags, "d"); 237 if (U_flag) 238 strcat(flags, "l"); 239 timeout = T_secs ? T_secs : ftp_timeout; 240 } 241 242 /* HTTP specific flags */ 243 if (strcmp(url->scheme, "http") == 0) { 244 if (d_flag) 245 strcat(flags, "d"); 246 if (A_flag) 247 strcat(flags, "A"); 248 timeout = T_secs ? T_secs : http_timeout; 249 } 250 251 /* set the protocol timeout. */ 252 fetchTimeout = timeout; 253 254 /* just print size */ 255 if (s_flag) { 256 if (fetchStat(url, &us, flags) == -1) 257 goto failure; 258 if (us.size == -1) 259 printf("Unknown\n"); 260 else 261 printf("%lld\n", us.size); 262 goto success; 263 } 264 265 /* 266 * If the -r flag was specified, we have to compare the local and 267 * remote files, so we should really do a fetchStat() first, but I 268 * know of at least one HTTP server that only sends the content 269 * size in response to GET requests, and leaves it out of replies 270 * to HEAD requests. Also, in the (frequent) case that the local 271 * and remote files match but the local file is truncated, we have 272 * sufficient information *before* the compare to issue a correct 273 * request. Therefore, we always issue a GET request as if we were 274 * sure the local file was a truncated copy of the remote file; we 275 * can drop the connection later if we change our minds. 276 */ 277 if ((r_flag || m_flag) && !o_stdout && stat(path, &sb) != -1) { 278 if (r_flag) 279 url->offset = sb.st_size; 280 } else { 281 sb.st_size = -1; 282 } 283 284 /* start the transfer */ 285 if ((f = fetchXGet(url, &us, flags)) == NULL) { 286 warnx("%s: %s", path, fetchLastErrString); 287 goto failure; 288 } 289 if (sigint) 290 goto signal; 291 292 /* check that size is as expected */ 293 if (S_size) { 294 if (us.size == -1) { 295 warnx("%s: size unknown", path); 296 goto failure; 297 } else if (us.size != S_size) { 298 warnx("%s: size mismatch: expected %lld, actual %lld", 299 path, S_size, us.size); 300 goto failure; 301 } 302 } 303 304 /* symlink instead of copy */ 305 if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 306 if (symlink(url->doc, path) == -1) { 307 warn("%s: symlink()", path); 308 goto failure; 309 } 310 goto success; 311 } 312 313 if (us.size == -1) 314 warnx("%s: size of remote file is not known", path); 315 if (v_level > 1) { 316 if (sb.st_size != -1) 317 fprintf(stderr, "local size / mtime: %lld / %ld\n", 318 sb.st_size, sb.st_mtime); 319 if (us.size != -1) 320 fprintf(stderr, "remote size / mtime: %lld / %ld\n", 321 us.size, us.mtime); 322 } 323 324 /* open output file */ 325 if (o_stdout) { 326 /* output to stdout */ 327 of = stdout; 328 } else if (sb.st_size != -1) { 329 /* resume mode, local file exists */ 330 if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 331 /* no match! have to refetch */ 332 fclose(f); 333 /* if precious, warn the user and give up */ 334 if (R_flag) { 335 warnx("%s: local modification time does not match remote", 336 path); 337 goto failure_keep; 338 } 339 url->offset = 0; 340 if ((f = fetchXGet(url, &us, flags)) == NULL) { 341 warnx("%s: %s", path, fetchLastErrString); 342 goto failure; 343 } 344 if (sigint) 345 goto signal; 346 } else { 347 if (us.size == sb.st_size) 348 /* nothing to do */ 349 goto success; 350 if (sb.st_size > us.size) { 351 /* local file too long! */ 352 warnx("%s: local file (%lld bytes) is longer " 353 "than remote file (%lld bytes)", 354 path, sb.st_size, us.size); 355 goto failure; 356 } 357 /* we got through, open local file and seek to offset */ 358 /* 359 * XXX there's a race condition here - the file we open is not 360 * necessarily the same as the one we stat()'ed earlier... 361 */ 362 if ((of = fopen(path, "a")) == NULL) { 363 warn("%s: fopen()", path); 364 goto failure; 365 } 366 if (fseek(of, url->offset, SEEK_SET) == -1) { 367 warn("%s: fseek()", path); 368 goto failure; 369 } 370 } 371 } 372 if (m_flag && sb.st_size != -1) { 373 /* mirror mode, local file exists */ 374 if (sb.st_size == us.size && sb.st_mtime == us.mtime) 375 goto success; 376 } 377 if (!of) { 378 /* 379 * We don't yet have an output file; either this is a vanilla 380 * run with no special flags, or the local and remote files 381 * didn't match. 382 */ 383 if ((of = fopen(path, "w")) == NULL) { 384 warn("%s: open()", path); 385 goto failure; 386 } 387 } 388 count = url->offset; 389 390 /* start the counter */ 391 stat_start(&xs, path, us.size, count); 392 393 sigalrm = siginfo = sigint = 0; 394 395 /* suck in the data */ 396 signal(SIGINFO, sig_handler); 397 for (n = 0; !sigint && !sigalrm; ++n) { 398 if (us.size != -1 && us.size - count < B_size) 399 size = us.size - count; 400 else 401 size = B_size; 402 if (timeout) 403 alarm(timeout); 404 if ((size = fread(buf, 1, size, f)) == 0) { 405 if (ferror(f) && errno == EINTR && !sigalrm && !sigint) 406 clearerr(f); 407 else 408 break; 409 } 410 if (timeout) 411 alarm(0); 412 if (siginfo) { 413 stat_end(&xs); 414 siginfo = 0; 415 } 416 stat_update(&xs, count += size, 0); 417 for (ptr = buf; size > 0; ptr += wr, size -= wr) 418 if ((wr = fwrite(ptr, 1, size, of)) < size) { 419 if (ferror(of) && errno == EINTR && !sigalrm && !sigint) 420 clearerr(of); 421 else 422 break; 423 } 424 if (size != 0) 425 break; 426 } 427 signal(SIGINFO, SIG_DFL); 428 429 if (timeout) 430 alarm(0); 431 432 stat_end(&xs); 433 434 /* set mtime of local file */ 435 if (!n_flag && us.mtime && !o_stdout 436 && (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 437 struct timeval tv[2]; 438 439 fflush(of); 440 tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 441 tv[1].tv_sec = (long)us.mtime; 442 tv[0].tv_usec = tv[1].tv_usec = 0; 443 if (utimes(path, tv)) 444 warn("%s: utimes()", path); 445 } 446 447 /* timed out or interrupted? */ 448 signal: 449 if (sigalrm) 450 warnx("transfer timed out"); 451 if (sigint) { 452 warnx("transfer interrupted"); 453 goto failure; 454 } 455 456 if (!sigalrm) { 457 /* check the status of our files */ 458 if (ferror(f)) 459 warn("%s", URL); 460 if (ferror(of)) 461 warn("%s", path); 462 if (ferror(f) || ferror(of)) 463 goto failure; 464 } 465 466 /* did the transfer complete normally? */ 467 if (us.size != -1 && count < us.size) { 468 warnx("%s appears to be truncated: %lld/%lld bytes", 469 path, count, us.size); 470 goto failure_keep; 471 } 472 473 /* 474 * If the transfer timed out and we didn't know how much to 475 * expect, assume the worst (i.e. we didn't get all of it) 476 */ 477 if (sigalrm && us.size == -1) { 478 warnx("%s may be truncated", path); 479 goto failure_keep; 480 } 481 482 success: 483 r = 0; 484 goto done; 485 failure: 486 if (of && of != stdout && !R_flag && !r_flag) 487 if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 488 unlink(path); 489 failure_keep: 490 r = -1; 491 goto done; 492 done: 493 if (f) 494 fclose(f); 495 if (of && of != stdout) 496 fclose(of); 497 if (url) 498 fetchFreeURL(url); 499 return r; 500 } 501 502 void 503 usage(void) 504 { 505 fprintf(stderr, 506 "Usage: fetch [-146AFMPRUadlmnpqrsv] [-o outputfile] [-S bytes]\n" 507 " [-B bytes] [-T seconds] [-w seconds]\n" 508 " [-h host -f file [-c dir] | URL ...]\n" 509 ); 510 } 511 512 513 #define PARSENUM(NAME, TYPE) \ 514 int \ 515 NAME(char *s, TYPE *v) \ 516 { \ 517 *v = 0; \ 518 for (*v = 0; *s; s++) \ 519 if (isdigit(*s)) \ 520 *v = *v * 10 + *s - '0'; \ 521 else \ 522 return -1; \ 523 return 0; \ 524 } 525 526 PARSENUM(parseint, u_int) 527 PARSENUM(parsesize, size_t) 528 PARSENUM(parseoff, off_t) 529 530 int 531 main(int argc, char *argv[]) 532 { 533 struct stat sb; 534 struct sigaction sa; 535 char *p, *q, *s; 536 int c, e, r; 537 538 while ((c = getopt(argc, argv, 539 "146AaB:bc:dFf:Hh:lMmnPpo:qRrS:sT:tUvw:")) != EOF) 540 switch (c) { 541 case '1': 542 once_flag = 1; 543 break; 544 case '4': 545 family = PF_INET; 546 break; 547 case '6': 548 family = PF_INET6; 549 break; 550 case 'A': 551 A_flag = 1; 552 break; 553 case 'a': 554 a_flag = 1; 555 break; 556 case 'B': 557 if (parsesize(optarg, &B_size) == -1) 558 errx(1, "invalid buffer size"); 559 break; 560 case 'b': 561 warnx("warning: the -b option is deprecated"); 562 b_flag = 1; 563 break; 564 case 'c': 565 c_dirname = optarg; 566 break; 567 case 'd': 568 d_flag = 1; 569 break; 570 case 'F': 571 F_flag = 1; 572 break; 573 case 'f': 574 f_filename = optarg; 575 break; 576 case 'H': 577 warnx("The -H option is now implicit, use -U to disable\n"); 578 break; 579 case 'h': 580 h_hostname = optarg; 581 break; 582 case 'l': 583 l_flag = 1; 584 break; 585 case 'o': 586 o_flag = 1; 587 o_filename = optarg; 588 break; 589 case 'M': 590 case 'm': 591 if (r_flag) 592 errx(1, "the -m and -r flags are mutually exclusive"); 593 m_flag = 1; 594 break; 595 case 'n': 596 n_flag = 1; 597 break; 598 case 'P': 599 case 'p': 600 p_flag = 1; 601 break; 602 case 'q': 603 v_level = 0; 604 break; 605 case 'R': 606 R_flag = 1; 607 break; 608 case 'r': 609 if (m_flag) 610 errx(1, "the -m and -r flags are mutually exclusive"); 611 r_flag = 1; 612 break; 613 case 'S': 614 if (parseoff(optarg, &S_size) == -1) 615 errx(1, "invalid size"); 616 break; 617 case 's': 618 s_flag = 1; 619 break; 620 case 'T': 621 if (parseint(optarg, &T_secs) == -1) 622 errx(1, "invalid timeout"); 623 break; 624 case 't': 625 t_flag = 1; 626 warnx("warning: the -t option is deprecated"); 627 break; 628 case 'U': 629 U_flag = 1; 630 break; 631 case 'v': 632 v_level++; 633 break; 634 case 'w': 635 a_flag = 1; 636 if (parseint(optarg, &w_secs) == -1) 637 errx(1, "invalid delay"); 638 break; 639 default: 640 usage(); 641 exit(EX_USAGE); 642 } 643 644 argc -= optind; 645 argv += optind; 646 647 if (h_hostname || f_filename || c_dirname) { 648 if (!h_hostname || !f_filename || argc) { 649 usage(); 650 exit(EX_USAGE); 651 } 652 /* XXX this is a hack. */ 653 if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 654 errx(1, "invalid hostname"); 655 if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 656 c_dirname ? c_dirname : "", f_filename) == -1) 657 errx(1, "%s", strerror(ENOMEM)); 658 argc++; 659 } 660 661 if (!argc) { 662 usage(); 663 exit(EX_USAGE); 664 } 665 666 /* allocate buffer */ 667 if (B_size < MINBUFSIZE) 668 B_size = MINBUFSIZE; 669 if ((buf = malloc(B_size)) == NULL) 670 errx(1, "%s", strerror(ENOMEM)); 671 672 /* timeouts */ 673 if ((s = getenv("FTP_TIMEOUT")) != NULL) { 674 if (parseint(s, &ftp_timeout) == -1) { 675 warnx("FTP_TIMEOUT is not a positive integer"); 676 ftp_timeout = 0; 677 } 678 } 679 if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 680 if (parseint(s, &http_timeout) == -1) { 681 warnx("HTTP_TIMEOUT is not a positive integer"); 682 http_timeout = 0; 683 } 684 } 685 686 /* signal handling */ 687 sa.sa_flags = 0; 688 sa.sa_handler = sig_handler; 689 sigemptyset(&sa.sa_mask); 690 sigaction(SIGALRM, &sa, NULL); 691 sa.sa_flags = SA_RESETHAND; 692 sigaction(SIGINT, &sa, NULL); 693 fetchRestartCalls = 0; 694 695 /* output file */ 696 if (o_flag) { 697 if (strcmp(o_filename, "-") == 0) { 698 o_stdout = 1; 699 } else if (stat(o_filename, &sb) == -1) { 700 if (errno == ENOENT) { 701 if (argc > 1) 702 errx(EX_USAGE, "%s is not a directory", o_filename); 703 } else { 704 err(EX_IOERR, "%s", o_filename); 705 } 706 } else { 707 if (sb.st_mode & S_IFDIR) 708 o_directory = 1; 709 } 710 } 711 712 /* check if output is to a tty (for progress report) */ 713 v_tty = isatty(STDERR_FILENO); 714 r = 0; 715 716 while (argc) { 717 if ((p = strrchr(*argv, '/')) == NULL) 718 p = *argv; 719 else 720 p++; 721 722 if (!*p) 723 p = "fetch.out"; 724 725 fetchLastErrCode = 0; 726 727 if (o_flag) { 728 if (o_stdout) { 729 e = fetch(*argv, "-"); 730 } else if (o_directory) { 731 asprintf(&q, "%s/%s", o_filename, p); 732 e = fetch(*argv, q); 733 free(q); 734 } else { 735 e = fetch(*argv, o_filename); 736 } 737 } else { 738 e = fetch(*argv, p); 739 } 740 741 if (sigint) 742 kill(getpid(), SIGINT); 743 744 if (e == 0 && once_flag) 745 exit(0); 746 747 if (e) { 748 r = 1; 749 if ((fetchLastErrCode 750 && fetchLastErrCode != FETCH_UNAVAIL 751 && fetchLastErrCode != FETCH_MOVED 752 && fetchLastErrCode != FETCH_URL 753 && fetchLastErrCode != FETCH_RESOLV 754 && fetchLastErrCode != FETCH_UNKNOWN)) { 755 if (w_secs) { 756 if (v_level) 757 fprintf(stderr, "Waiting %d seconds before retrying\n", 758 w_secs); 759 sleep(w_secs); 760 } 761 if (a_flag) 762 continue; 763 } 764 } 765 766 argc--, argv++; 767 } 768 769 exit(r); 770 } 771