1 /* 2 * Run a set of tests, reporting results. 3 * 4 * Test suite driver that runs a set of tests implementing a subset of the 5 * Test Anything Protocol (TAP) and reports the results. 6 * 7 * Any bug reports, bug fixes, and improvements are very much welcome and 8 * should be sent to the e-mail address below. This program is part of C TAP 9 * Harness <https://www.eyrie.org/~eagle/software/c-tap-harness/>. 10 * 11 * Copyright 2000-2001, 2004, 2006-2019 Russ Allbery <eagle@eyrie.org> 12 * 13 * Permission is hereby granted, free of charge, to any person obtaining a 14 * copy of this software and associated documentation files (the "Software"), 15 * to deal in the Software without restriction, including without limitation 16 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 17 * and/or sell copies of the Software, and to permit persons to whom the 18 * Software is furnished to do so, subject to the following conditions: 19 * 20 * The above copyright notice and this permission notice shall be included in 21 * all copies or substantial portions of the Software. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 26 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 29 * DEALINGS IN THE SOFTWARE. 30 * 31 * SPDX-License-Identifier: MIT 32 */ 33 34 /* 35 * Usage: 36 * 37 * runtests [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list> 38 * runtests [-hv] [-b <build-dir>] [-s <source-dir>] <test> [<test> ...] 39 * runtests -o [-h] [-b <build-dir>] [-s <source-dir>] <test> 40 * 41 * In the first case, expects a list of executables located in the given file, 42 * one line per executable, possibly followed by a space-separated list of 43 * options. For each one, runs it as part of a test suite, reporting results. 44 * In the second case, use the same infrastructure, but run only the tests 45 * listed on the command line. 46 * 47 * Test output should start with a line containing the number of tests 48 * (numbered from 1 to this number), optionally preceded by "1..", although 49 * that line may be given anywhere in the output. Each additional line should 50 * be in the following format: 51 * 52 * ok <number> 53 * not ok <number> 54 * ok <number> # skip 55 * not ok <number> # todo 56 * 57 * where <number> is the number of the test. An optional comment is permitted 58 * after the number if preceded by whitespace. ok indicates success, not ok 59 * indicates failure. "# skip" and "# todo" are a special cases of a comment, 60 * and must start with exactly that formatting. They indicate the test was 61 * skipped for some reason (maybe because it doesn't apply to this platform) 62 * or is testing something known to currently fail. The text following either 63 * "# skip" or "# todo" and whitespace is the reason. 64 * 65 * As a special case, the first line of the output may be in the form: 66 * 67 * 1..0 # skip some reason 68 * 69 * which indicates that this entire test case should be skipped and gives a 70 * reason. 71 * 72 * Any other lines are ignored, although for compliance with the TAP protocol 73 * all lines other than the ones in the above format should be sent to 74 * standard error rather than standard output and start with #. 75 * 76 * This is a subset of TAP as documented in Test::Harness::TAP or 77 * TAP::Parser::Grammar, which comes with Perl. 78 * 79 * If the -o option is given, instead run a single test and display all of its 80 * output. This is intended for use with failing tests so that the person 81 * running the test suite can get more details about what failed. 82 * 83 * If built with the C preprocessor symbols C_TAP_SOURCE and C_TAP_BUILD 84 * defined, C TAP Harness will export those values in the environment so that 85 * tests can find the source and build directory and will look for tests under 86 * both directories. These paths can also be set with the -b and -s 87 * command-line options, which will override anything set at build time. 88 * 89 * If the -v option is given, or the C_TAP_VERBOSE environment variable is set, 90 * display the full output of each test as it runs rather than showing a 91 * summary of the results of each test. 92 */ 93 94 /* Required for fdopen(), getopt(), and putenv(). */ 95 #if defined(__STRICT_ANSI__) || defined(PEDANTIC) 96 # ifndef _XOPEN_SOURCE 97 # define _XOPEN_SOURCE 500 98 # endif 99 #endif 100 101 #include <ctype.h> 102 #include <errno.h> 103 #include <fcntl.h> 104 #include <limits.h> 105 #include <stdarg.h> 106 #include <stddef.h> 107 #include <stdio.h> 108 #include <stdlib.h> 109 #include <string.h> 110 #include <strings.h> 111 #include <sys/stat.h> 112 #include <sys/time.h> 113 #include <sys/types.h> 114 #include <sys/wait.h> 115 #include <time.h> 116 #include <unistd.h> 117 118 /* sys/time.h must be included before sys/resource.h on some platforms. */ 119 #include <sys/resource.h> 120 121 /* AIX 6.1 (and possibly later) doesn't have WCOREDUMP. */ 122 #ifndef WCOREDUMP 123 # define WCOREDUMP(status) ((unsigned) (status) &0x80) 124 #endif 125 126 /* 127 * POSIX requires that these be defined in <unistd.h>, but they're not always 128 * available. If one of them has been defined, all the rest almost certainly 129 * have. 130 */ 131 #ifndef STDIN_FILENO 132 # define STDIN_FILENO 0 133 # define STDOUT_FILENO 1 134 # define STDERR_FILENO 2 135 #endif 136 137 /* 138 * Used for iterating through arrays. Returns the number of elements in the 139 * array (useful for a < upper bound in a for loop). 140 */ 141 #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) 142 143 /* 144 * The source and build versions of the tests directory. This is used to set 145 * the C_TAP_SOURCE and C_TAP_BUILD environment variables (and the SOURCE and 146 * BUILD environment variables set for backward compatibility) and find test 147 * programs, if set. Normally, this should be set as part of the build 148 * process to the test subdirectories of $(abs_top_srcdir) and 149 * $(abs_top_builddir) respectively. 150 */ 151 #ifndef C_TAP_SOURCE 152 # define C_TAP_SOURCE NULL 153 #endif 154 #ifndef C_TAP_BUILD 155 # define C_TAP_BUILD NULL 156 #endif 157 158 /* Test status codes. */ 159 enum test_status 160 { 161 TEST_FAIL, 162 TEST_PASS, 163 TEST_SKIP, 164 TEST_INVALID 165 }; 166 167 /* Really, just a boolean, but this is more self-documenting. */ 168 enum test_verbose 169 { 170 CONCISE = 0, 171 VERBOSE = 1 172 }; 173 174 /* Indicates the state of our plan. */ 175 enum plan_status 176 { 177 PLAN_INIT, /* Nothing seen yet. */ 178 PLAN_FIRST, /* Plan seen before any tests. */ 179 PLAN_PENDING, /* Test seen and no plan yet. */ 180 PLAN_FINAL /* Plan seen after some tests. */ 181 }; 182 183 /* Error exit statuses for test processes. */ 184 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */ 185 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */ 186 #define CHILDERR_STDIN 102 /* Couldn't open stdin file. */ 187 #define CHILDERR_STDERR 103 /* Couldn't open stderr file. */ 188 189 /* Structure to hold data for a set of tests. */ 190 struct testset { 191 char *file; /* The file name of the test. */ 192 char **command; /* The argv vector to run the command. */ 193 enum plan_status plan; /* The status of our plan. */ 194 unsigned long count; /* Expected count of tests. */ 195 unsigned long current; /* The last seen test number. */ 196 unsigned int length; /* The length of the last status message. */ 197 unsigned long passed; /* Count of passing tests. */ 198 unsigned long failed; /* Count of failing lists. */ 199 unsigned long skipped; /* Count of skipped tests (passed). */ 200 unsigned long allocated; /* The size of the results table. */ 201 enum test_status *results; /* Table of results by test number. */ 202 unsigned int aborted; /* Whether the set was aborted. */ 203 unsigned int reported; /* Whether the results were reported. */ 204 int status; /* The exit status of the test. */ 205 unsigned int all_skipped; /* Whether all tests were skipped. */ 206 char *reason; /* Why all tests were skipped. */ 207 }; 208 209 /* Structure to hold a linked list of test sets. */ 210 struct testlist { 211 struct testset *ts; 212 struct testlist *next; 213 }; 214 215 /* 216 * Usage message. Should be used as a printf format with four arguments: the 217 * path to runtests, given three times, and the usage_description. This is 218 * split into variables to satisfy the pedantic ISO C90 limit on strings. 219 */ 220 static const char usage_message[] = "\ 221 Usage: %s [-hv] [-b <build-dir>] [-s <source-dir>] <test> ...\n\ 222 %s [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>\n\ 223 %s -o [-h] [-b <build-dir>] [-s <source-dir>] <test>\n\ 224 \n\ 225 Options:\n\ 226 -b <build-dir> Set the build directory to <build-dir>\n\ 227 %s"; 228 static const char usage_extra[] = "\ 229 -l <list> Take the list of tests to run from <test-list>\n\ 230 -o Run a single test rather than a list of tests\n\ 231 -s <source-dir> Set the source directory to <source-dir>\n\ 232 -v Show the full output of each test\n\ 233 \n\ 234 runtests normally runs each test listed on the command line. With the -l\n\ 235 option, it instead runs every test listed in a file. With the -o option,\n\ 236 it instead runs a single test and shows its complete output.\n"; 237 238 /* 239 * Header used for test output. %s is replaced by the file name of the list 240 * of tests. 241 */ 242 static const char banner[] = "\n\ 243 Running all tests listed in %s. If any tests fail, run the failing\n\ 244 test program with runtests -o to see more details.\n\n"; 245 246 /* Header for reports of failed tests. */ 247 static const char header[] = "\n\ 248 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\ 249 -------------------------- -------------- ---- ---- ------------------------"; 250 251 /* Include the file name and line number in malloc failures. */ 252 #define xcalloc(n, type) \ 253 ((type *) x_calloc((n), sizeof(type), __FILE__, __LINE__)) 254 #define xmalloc(size) ((char *) x_malloc((size), __FILE__, __LINE__)) 255 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__) 256 #define xstrndup(p, size) x_strndup((p), (size), __FILE__, __LINE__) 257 #define xreallocarray(p, n, type) \ 258 ((type *) x_reallocarray((p), (n), sizeof(type), __FILE__, __LINE__)) 259 260 /* 261 * __attribute__ is available in gcc 2.5 and later, but only with gcc 2.7 262 * could you use the __format__ form of the attributes, which is what we use 263 * (to avoid confusion with other macros). 264 */ 265 #ifndef __attribute__ 266 # if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 7) 267 # define __attribute__(spec) /* empty */ 268 # endif 269 #endif 270 271 /* 272 * We use __alloc_size__, but it was only available in fairly recent versions 273 * of GCC. Suppress warnings about the unknown attribute if GCC is too old. 274 * We know that we're GCC at this point, so we can use the GCC variadic macro 275 * extension, which will still work with versions of GCC too old to have C99 276 * variadic macro support. 277 */ 278 #if !defined(__attribute__) && !defined(__alloc_size__) 279 # if defined(__GNUC__) && !defined(__clang__) 280 # if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 3) 281 # define __alloc_size__(spec, args...) /* empty */ 282 # endif 283 # endif 284 #endif 285 286 /* 287 * LLVM and Clang pretend to be GCC but don't support all of the __attribute__ 288 * settings that GCC does. For them, suppress warnings about unknown 289 * attributes on declarations. This unfortunately will affect the entire 290 * compilation context, but there's no push and pop available. 291 */ 292 #if !defined(__attribute__) && (defined(__llvm__) || defined(__clang__)) 293 # pragma GCC diagnostic ignored "-Wattributes" 294 #endif 295 296 /* Declare internal functions that benefit from compiler attributes. */ 297 static void die(const char *, ...) 298 __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2))); 299 static void sysdie(const char *, ...) 300 __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2))); 301 static void *x_calloc(size_t, size_t, const char *, int) 302 __attribute__((__alloc_size__(1, 2), __malloc__, __nonnull__)); 303 static void *x_malloc(size_t, const char *, int) 304 __attribute__((__alloc_size__(1), __malloc__, __nonnull__)); 305 static void *x_reallocarray(void *, size_t, size_t, const char *, int) 306 __attribute__((__alloc_size__(2, 3), __malloc__, __nonnull__(4))); 307 static char *x_strdup(const char *, const char *, int) 308 __attribute__((__malloc__, __nonnull__)); 309 static char *x_strndup(const char *, size_t, const char *, int) 310 __attribute__((__malloc__, __nonnull__)); 311 312 313 /* 314 * Report a fatal error and exit. 315 */ 316 static void 317 die(const char *format, ...) 318 { 319 va_list args; 320 321 fflush(stdout); 322 fprintf(stderr, "runtests: "); 323 va_start(args, format); 324 vfprintf(stderr, format, args); 325 va_end(args); 326 fprintf(stderr, "\n"); 327 exit(1); 328 } 329 330 331 /* 332 * Report a fatal error, including the results of strerror, and exit. 333 */ 334 static void 335 sysdie(const char *format, ...) 336 { 337 int oerrno; 338 va_list args; 339 340 oerrno = errno; 341 fflush(stdout); 342 fprintf(stderr, "runtests: "); 343 va_start(args, format); 344 vfprintf(stderr, format, args); 345 va_end(args); 346 fprintf(stderr, ": %s\n", strerror(oerrno)); 347 exit(1); 348 } 349 350 351 /* 352 * Allocate zeroed memory, reporting a fatal error and exiting on failure. 353 */ 354 static void * 355 x_calloc(size_t n, size_t size, const char *file, int line) 356 { 357 void *p; 358 359 n = (n > 0) ? n : 1; 360 size = (size > 0) ? size : 1; 361 p = calloc(n, size); 362 if (p == NULL) 363 sysdie("failed to calloc %lu bytes at %s line %d", 364 (unsigned long) size, file, line); 365 return p; 366 } 367 368 369 /* 370 * Allocate memory, reporting a fatal error and exiting on failure. 371 */ 372 static void * 373 x_malloc(size_t size, const char *file, int line) 374 { 375 void *p; 376 377 p = malloc(size); 378 if (p == NULL) 379 sysdie("failed to malloc %lu bytes at %s line %d", 380 (unsigned long) size, file, line); 381 return p; 382 } 383 384 385 /* 386 * Reallocate memory, reporting a fatal error and exiting on failure. 387 * 388 * We should technically use SIZE_MAX here for the overflow check, but 389 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not 390 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we 391 * can assume that UINT_MAX <= SIZE_MAX. And we should not be allocating 392 * anything anywhere near that large. 393 * 394 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but 395 * I disbelieve in the existence of such systems and they will have to cope 396 * without overflow checks.) 397 */ 398 static void * 399 x_reallocarray(void *p, size_t n, size_t size, const char *file, int line) 400 { 401 n = (n > 0) ? n : 1; 402 size = (size > 0) ? size : 1; 403 404 if (n > 0 && UINT_MAX / n <= size) 405 sysdie("realloc too large at %s line %d", file, line); 406 p = realloc(p, n * size); 407 if (p == NULL) 408 sysdie("failed to realloc %lu bytes at %s line %d", 409 (unsigned long) (n * size), file, line); 410 return p; 411 } 412 413 414 /* 415 * Copy a string, reporting a fatal error and exiting on failure. 416 */ 417 static char * 418 x_strdup(const char *s, const char *file, int line) 419 { 420 char *p; 421 size_t len; 422 423 len = strlen(s) + 1; 424 p = (char *) malloc(len); 425 if (p == NULL) 426 sysdie("failed to strdup %lu bytes at %s line %d", (unsigned long) len, 427 file, line); 428 memcpy(p, s, len); 429 return p; 430 } 431 432 433 /* 434 * Copy the first n characters of a string, reporting a fatal error and 435 * existing on failure. 436 * 437 * Avoid using the system strndup function since it may not exist (on Mac OS 438 * X, for example), and there's no need to introduce another portability 439 * requirement. 440 */ 441 char * 442 x_strndup(const char *s, size_t size, const char *file, int line) 443 { 444 const char *p; 445 size_t len; 446 char *copy; 447 448 /* Don't assume that the source string is nul-terminated. */ 449 for (p = s; (size_t)(p - s) < size && *p != '\0'; p++) 450 ; 451 len = (size_t)(p - s); 452 copy = (char *) malloc(len + 1); 453 if (copy == NULL) 454 sysdie("failed to strndup %lu bytes at %s line %d", 455 (unsigned long) len, file, line); 456 memcpy(copy, s, len); 457 copy[len] = '\0'; 458 return copy; 459 } 460 461 462 /* 463 * Form a new string by concatenating multiple strings. The arguments must be 464 * terminated by (const char *) 0. 465 * 466 * This function only exists because we can't assume asprintf. We can't 467 * simulate asprintf with snprintf because we're only assuming SUSv3, which 468 * does not require that snprintf with a NULL buffer return the required 469 * length. When those constraints are relaxed, this should be ripped out and 470 * replaced with asprintf or a more trivial replacement with snprintf. 471 */ 472 static char * 473 concat(const char *first, ...) 474 { 475 va_list args; 476 char *result; 477 const char *string; 478 size_t offset; 479 size_t length = 0; 480 481 /* 482 * Find the total memory required. Ensure we don't overflow length. We 483 * aren't guaranteed to have SIZE_MAX, so use UINT_MAX as an acceptable 484 * substitute (see the x_nrealloc comments). 485 */ 486 va_start(args, first); 487 for (string = first; string != NULL; string = va_arg(args, const char *)) { 488 if (length >= UINT_MAX - strlen(string)) { 489 errno = EINVAL; 490 sysdie("strings too long in concat"); 491 } 492 length += strlen(string); 493 } 494 va_end(args); 495 length++; 496 497 /* Create the string. */ 498 result = xmalloc(length); 499 va_start(args, first); 500 offset = 0; 501 for (string = first; string != NULL; string = va_arg(args, const char *)) { 502 memcpy(result + offset, string, strlen(string)); 503 offset += strlen(string); 504 } 505 va_end(args); 506 result[offset] = '\0'; 507 return result; 508 } 509 510 511 /* 512 * Given a struct timeval, return the number of seconds it represents as a 513 * double. Use difftime() to convert a time_t to a double. 514 */ 515 static double 516 tv_seconds(const struct timeval *tv) 517 { 518 return difftime(tv->tv_sec, 0) + (double) tv->tv_usec * 1e-6; 519 } 520 521 522 /* 523 * Given two struct timevals, return the difference in seconds. 524 */ 525 static double 526 tv_diff(const struct timeval *tv1, const struct timeval *tv0) 527 { 528 return tv_seconds(tv1) - tv_seconds(tv0); 529 } 530 531 532 /* 533 * Given two struct timevals, return the sum in seconds as a double. 534 */ 535 static double 536 tv_sum(const struct timeval *tv1, const struct timeval *tv2) 537 { 538 return tv_seconds(tv1) + tv_seconds(tv2); 539 } 540 541 542 /* 543 * Given a pointer to a string, skip any leading whitespace and return a 544 * pointer to the first non-whitespace character. 545 */ 546 static const char * 547 skip_whitespace(const char *p) 548 { 549 while (isspace((unsigned char) (*p))) 550 p++; 551 return p; 552 } 553 554 555 /* 556 * Given a pointer to a string, skip any non-whitespace characters and return 557 * a pointer to the first whitespace character, or to the end of the string. 558 */ 559 static const char * 560 skip_non_whitespace(const char *p) 561 { 562 while (*p != '\0' && !isspace((unsigned char) (*p))) 563 p++; 564 return p; 565 } 566 567 568 /* 569 * Start a program, connecting its stdout to a pipe on our end and its stderr 570 * to /dev/null, and storing the file descriptor to read from in the two 571 * argument. Returns the PID of the new process. Errors are fatal. 572 */ 573 static pid_t 574 test_start(char *const *command, int *fd) 575 { 576 int fds[2], infd, errfd; 577 pid_t child; 578 579 /* Create a pipe used to capture the output from the test program. */ 580 if (pipe(fds) == -1) { 581 puts("ABORTED"); 582 fflush(stdout); 583 sysdie("can't create pipe"); 584 } 585 586 /* Fork a child process, massage the file descriptors, and exec. */ 587 child = fork(); 588 switch (child) { 589 case -1: 590 puts("ABORTED"); 591 fflush(stdout); 592 sysdie("can't fork"); 593 594 /* In the child. Set up our standard output. */ 595 case 0: 596 close(fds[0]); 597 close(STDOUT_FILENO); 598 if (dup2(fds[1], STDOUT_FILENO) < 0) 599 _exit(CHILDERR_DUP); 600 close(fds[1]); 601 602 /* Point standard input at /dev/null. */ 603 close(STDIN_FILENO); 604 infd = open("/dev/null", O_RDONLY); 605 if (infd < 0) 606 _exit(CHILDERR_STDIN); 607 if (infd != STDIN_FILENO) { 608 if (dup2(infd, STDIN_FILENO) < 0) 609 _exit(CHILDERR_DUP); 610 close(infd); 611 } 612 613 /* Point standard error at /dev/null. */ 614 close(STDERR_FILENO); 615 errfd = open("/dev/null", O_WRONLY); 616 if (errfd < 0) 617 _exit(CHILDERR_STDERR); 618 if (errfd != STDERR_FILENO) { 619 if (dup2(errfd, STDERR_FILENO) < 0) 620 _exit(CHILDERR_DUP); 621 close(errfd); 622 } 623 624 /* Now, exec our process. */ 625 if (execv(command[0], command) == -1) 626 _exit(CHILDERR_EXEC); 627 break; 628 629 /* In parent. Close the extra file descriptor. */ 630 default: 631 close(fds[1]); 632 break; 633 } 634 *fd = fds[0]; 635 return child; 636 } 637 638 639 /* 640 * Back up over the output saying what test we were executing. 641 */ 642 static void 643 test_backspace(struct testset *ts) 644 { 645 unsigned int i; 646 647 if (!isatty(STDOUT_FILENO)) 648 return; 649 for (i = 0; i < ts->length; i++) 650 putchar('\b'); 651 for (i = 0; i < ts->length; i++) 652 putchar(' '); 653 for (i = 0; i < ts->length; i++) 654 putchar('\b'); 655 ts->length = 0; 656 } 657 658 659 /* 660 * Allocate or resize the array of test results to be large enough to contain 661 * the test number in. 662 */ 663 static void 664 resize_results(struct testset *ts, unsigned long n) 665 { 666 unsigned long i; 667 size_t s; 668 669 /* If there's already enough space, return quickly. */ 670 if (n <= ts->allocated) 671 return; 672 673 /* 674 * If no space has been allocated, do the initial allocation. Otherwise, 675 * resize. Start with 32 test cases and then add 1024 with each resize to 676 * try to reduce the number of reallocations. 677 */ 678 if (ts->allocated == 0) { 679 s = (n > 32) ? n : 32; 680 ts->results = xcalloc(s, enum test_status); 681 } else { 682 s = (n > ts->allocated + 1024) ? n : ts->allocated + 1024; 683 ts->results = xreallocarray(ts->results, s, enum test_status); 684 } 685 686 /* Set the results for the newly-allocated test array. */ 687 for (i = ts->allocated; i < s; i++) 688 ts->results[i] = TEST_INVALID; 689 ts->allocated = s; 690 } 691 692 693 /* 694 * Report an invalid test number and set the appropriate flags. Pulled into a 695 * separate function since we do this in several places. 696 */ 697 static void 698 invalid_test_number(struct testset *ts, long n, enum test_verbose verbose) 699 { 700 if (!verbose) 701 test_backspace(ts); 702 printf("ABORTED (invalid test number %ld)\n", n); 703 ts->aborted = 1; 704 ts->reported = 1; 705 } 706 707 708 /* 709 * Read the plan line of test output, which should contain the range of test 710 * numbers. We may initialize the testset structure here if we haven't yet 711 * seen a test. Return true if initialization succeeded and the test should 712 * continue, false otherwise. 713 */ 714 static int 715 test_plan(const char *line, struct testset *ts, enum test_verbose verbose) 716 { 717 long n; 718 719 /* 720 * Accept a plan without the leading 1.. for compatibility with older 721 * versions of runtests. This will only be allowed if we've not yet seen 722 * a test result. 723 */ 724 line = skip_whitespace(line); 725 if (strncmp(line, "1..", 3) == 0) 726 line += 3; 727 728 /* 729 * Get the count and check it for validity. 730 * 731 * If we have something of the form "1..0 # skip foo", the whole file was 732 * skipped; record that. If we do skip the whole file, zero out all of 733 * our statistics, since they're no longer relevant. 734 * 735 * strtol is called with a second argument to advance the line pointer 736 * past the count to make it simpler to detect the # skip case. 737 */ 738 n = strtol(line, (char **) &line, 10); 739 if (n == 0) { 740 line = skip_whitespace(line); 741 if (*line == '#') { 742 line = skip_whitespace(line + 1); 743 if (strncasecmp(line, "skip", 4) == 0) { 744 line = skip_whitespace(line + 4); 745 if (*line != '\0') { 746 ts->reason = xstrdup(line); 747 ts->reason[strlen(ts->reason) - 1] = '\0'; 748 } 749 ts->all_skipped = 1; 750 ts->aborted = 1; 751 ts->count = 0; 752 ts->passed = 0; 753 ts->skipped = 0; 754 ts->failed = 0; 755 return 0; 756 } 757 } 758 } 759 if (n <= 0) { 760 puts("ABORTED (invalid test count)"); 761 ts->aborted = 1; 762 ts->reported = 1; 763 return 0; 764 } 765 766 /* 767 * If we are doing lazy planning, check the plan against the largest test 768 * number that we saw and fail now if we saw a check outside the plan 769 * range. 770 */ 771 if (ts->plan == PLAN_PENDING && (unsigned long) n < ts->count) { 772 invalid_test_number(ts, (long) ts->count, verbose); 773 return 0; 774 } 775 776 /* 777 * Otherwise, allocated or resize the results if needed and update count, 778 * and then record that we've seen a plan. 779 */ 780 resize_results(ts, (unsigned long) n); 781 ts->count = (unsigned long) n; 782 if (ts->plan == PLAN_INIT) 783 ts->plan = PLAN_FIRST; 784 else if (ts->plan == PLAN_PENDING) 785 ts->plan = PLAN_FINAL; 786 return 1; 787 } 788 789 790 /* 791 * Given a single line of output from a test, parse it and return the success 792 * status of that test. Anything printed to stdout not matching the form 793 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just 794 * reported status. 795 */ 796 static void 797 test_checkline(const char *line, struct testset *ts, enum test_verbose verbose) 798 { 799 enum test_status status = TEST_PASS; 800 const char *bail; 801 char *end; 802 long number; 803 unsigned long current; 804 int outlen; 805 806 /* Before anything, check for a test abort. */ 807 bail = strstr(line, "Bail out!"); 808 if (bail != NULL) { 809 bail = skip_whitespace(bail + strlen("Bail out!")); 810 if (*bail != '\0') { 811 size_t length; 812 813 length = strlen(bail); 814 if (bail[length - 1] == '\n') 815 length--; 816 if (!verbose) 817 test_backspace(ts); 818 printf("ABORTED (%.*s)\n", (int) length, bail); 819 ts->reported = 1; 820 } 821 ts->aborted = 1; 822 return; 823 } 824 825 /* 826 * If the given line isn't newline-terminated, it was too big for an 827 * fgets(), which means ignore it. 828 */ 829 if (line[strlen(line) - 1] != '\n') 830 return; 831 832 /* If the line begins with a hash mark, ignore it. */ 833 if (line[0] == '#') 834 return; 835 836 /* If we haven't yet seen a plan, look for one. */ 837 if (ts->plan == PLAN_INIT && isdigit((unsigned char) (*line))) { 838 if (!test_plan(line, ts, verbose)) 839 return; 840 } else if (strncmp(line, "1..", 3) == 0) { 841 if (ts->plan == PLAN_PENDING) { 842 if (!test_plan(line, ts, verbose)) 843 return; 844 } else { 845 if (!verbose) 846 test_backspace(ts); 847 puts("ABORTED (multiple plans)"); 848 ts->aborted = 1; 849 ts->reported = 1; 850 return; 851 } 852 } 853 854 /* Parse the line, ignoring something we can't parse. */ 855 if (strncmp(line, "not ", 4) == 0) { 856 status = TEST_FAIL; 857 line += 4; 858 } 859 if (strncmp(line, "ok", 2) != 0) 860 return; 861 line = skip_whitespace(line + 2); 862 errno = 0; 863 number = strtol(line, &end, 10); 864 if (errno != 0 || end == line) 865 current = ts->current + 1; 866 else if (number <= 0) { 867 invalid_test_number(ts, number, verbose); 868 return; 869 } else 870 current = (unsigned long) number; 871 if (current > ts->count && ts->plan == PLAN_FIRST) { 872 invalid_test_number(ts, (long) current, verbose); 873 return; 874 } 875 876 /* We have a valid test result. Tweak the results array if needed. */ 877 if (ts->plan == PLAN_INIT || ts->plan == PLAN_PENDING) { 878 ts->plan = PLAN_PENDING; 879 resize_results(ts, current); 880 if (current > ts->count) 881 ts->count = current; 882 } 883 884 /* 885 * Handle directives. We should probably do something more interesting 886 * with unexpected passes of todo tests. 887 */ 888 while (isdigit((unsigned char) (*line))) 889 line++; 890 line = skip_whitespace(line); 891 if (*line == '#') { 892 line = skip_whitespace(line + 1); 893 if (strncasecmp(line, "skip", 4) == 0) 894 status = TEST_SKIP; 895 if (strncasecmp(line, "todo", 4) == 0) 896 status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL; 897 } 898 899 /* Make sure that the test number is in range and not a duplicate. */ 900 if (ts->results[current - 1] != TEST_INVALID) { 901 if (!verbose) 902 test_backspace(ts); 903 printf("ABORTED (duplicate test number %lu)\n", current); 904 ts->aborted = 1; 905 ts->reported = 1; 906 return; 907 } 908 909 /* Good results. Increment our various counters. */ 910 switch (status) { 911 case TEST_PASS: 912 ts->passed++; 913 break; 914 case TEST_FAIL: 915 ts->failed++; 916 break; 917 case TEST_SKIP: 918 ts->skipped++; 919 break; 920 case TEST_INVALID: 921 break; 922 } 923 ts->current = current; 924 ts->results[current - 1] = status; 925 if (!verbose && isatty(STDOUT_FILENO)) { 926 test_backspace(ts); 927 if (ts->plan == PLAN_PENDING) 928 outlen = printf("%lu/?", current); 929 else 930 outlen = printf("%lu/%lu", current, ts->count); 931 ts->length = (outlen >= 0) ? (unsigned int) outlen : 0; 932 fflush(stdout); 933 } 934 } 935 936 937 /* 938 * Print out a range of test numbers, returning the number of characters it 939 * took up. Takes the first number, the last number, the number of characters 940 * already printed on the line, and the limit of number of characters the line 941 * can hold. Add a comma and a space before the range if chars indicates that 942 * something has already been printed on the line, and print ... instead if 943 * chars plus the space needed would go over the limit (use a limit of 0 to 944 * disable this). 945 */ 946 static unsigned int 947 test_print_range(unsigned long first, unsigned long last, unsigned long chars, 948 unsigned int limit) 949 { 950 unsigned int needed = 0; 951 unsigned long n; 952 953 for (n = first; n > 0; n /= 10) 954 needed++; 955 if (last > first) { 956 for (n = last; n > 0; n /= 10) 957 needed++; 958 needed++; 959 } 960 if (chars > 0) 961 needed += 2; 962 if (limit > 0 && chars + needed > limit) { 963 needed = 0; 964 if (chars <= limit) { 965 if (chars > 0) { 966 printf(", "); 967 needed += 2; 968 } 969 printf("..."); 970 needed += 3; 971 } 972 } else { 973 if (chars > 0) 974 printf(", "); 975 if (last > first) 976 printf("%lu-", first); 977 printf("%lu", last); 978 } 979 return needed; 980 } 981 982 983 /* 984 * Summarize a single test set. The second argument is 0 if the set exited 985 * cleanly, a positive integer representing the exit status if it exited 986 * with a non-zero status, and a negative integer representing the signal 987 * that terminated it if it was killed by a signal. 988 */ 989 static void 990 test_summarize(struct testset *ts, int status) 991 { 992 unsigned long i; 993 unsigned long missing = 0; 994 unsigned long failed = 0; 995 unsigned long first = 0; 996 unsigned long last = 0; 997 998 if (ts->aborted) { 999 fputs("ABORTED", stdout); 1000 if (ts->count > 0) 1001 printf(" (passed %lu/%lu)", ts->passed, ts->count - ts->skipped); 1002 } else { 1003 for (i = 0; i < ts->count; i++) { 1004 if (ts->results[i] == TEST_INVALID) { 1005 if (missing == 0) 1006 fputs("MISSED ", stdout); 1007 if (first && i == last) 1008 last = i + 1; 1009 else { 1010 if (first) 1011 test_print_range(first, last, missing - 1, 0); 1012 missing++; 1013 first = i + 1; 1014 last = i + 1; 1015 } 1016 } 1017 } 1018 if (first) 1019 test_print_range(first, last, missing - 1, 0); 1020 first = 0; 1021 last = 0; 1022 for (i = 0; i < ts->count; i++) { 1023 if (ts->results[i] == TEST_FAIL) { 1024 if (missing && !failed) 1025 fputs("; ", stdout); 1026 if (failed == 0) 1027 fputs("FAILED ", stdout); 1028 if (first && i == last) 1029 last = i + 1; 1030 else { 1031 if (first) 1032 test_print_range(first, last, failed - 1, 0); 1033 failed++; 1034 first = i + 1; 1035 last = i + 1; 1036 } 1037 } 1038 } 1039 if (first) 1040 test_print_range(first, last, failed - 1, 0); 1041 if (!missing && !failed) { 1042 fputs(!status ? "ok" : "dubious", stdout); 1043 if (ts->skipped > 0) { 1044 if (ts->skipped == 1) 1045 printf(" (skipped %lu test)", ts->skipped); 1046 else 1047 printf(" (skipped %lu tests)", ts->skipped); 1048 } 1049 } 1050 } 1051 if (status > 0) 1052 printf(" (exit status %d)", status); 1053 else if (status < 0) 1054 printf(" (killed by signal %d%s)", -status, 1055 WCOREDUMP(ts->status) ? ", core dumped" : ""); 1056 putchar('\n'); 1057 } 1058 1059 1060 /* 1061 * Given a test set, analyze the results, classify the exit status, handle a 1062 * few special error messages, and then pass it along to test_summarize() for 1063 * the regular output. Returns true if the test set ran successfully and all 1064 * tests passed or were skipped, false otherwise. 1065 */ 1066 static int 1067 test_analyze(struct testset *ts) 1068 { 1069 if (ts->reported) 1070 return 0; 1071 if (ts->all_skipped) { 1072 if (ts->reason == NULL) 1073 puts("skipped"); 1074 else 1075 printf("skipped (%s)\n", ts->reason); 1076 return 1; 1077 } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) { 1078 switch (WEXITSTATUS(ts->status)) { 1079 case CHILDERR_DUP: 1080 if (!ts->reported) 1081 puts("ABORTED (can't dup file descriptors)"); 1082 break; 1083 case CHILDERR_EXEC: 1084 if (!ts->reported) 1085 puts("ABORTED (execution failed -- not found?)"); 1086 break; 1087 case CHILDERR_STDIN: 1088 case CHILDERR_STDERR: 1089 if (!ts->reported) 1090 puts("ABORTED (can't open /dev/null)"); 1091 break; 1092 default: 1093 test_summarize(ts, WEXITSTATUS(ts->status)); 1094 break; 1095 } 1096 return 0; 1097 } else if (WIFSIGNALED(ts->status)) { 1098 test_summarize(ts, -WTERMSIG(ts->status)); 1099 return 0; 1100 } else if (ts->plan != PLAN_FIRST && ts->plan != PLAN_FINAL) { 1101 puts("ABORTED (no valid test plan)"); 1102 ts->aborted = 1; 1103 return 0; 1104 } else { 1105 test_summarize(ts, 0); 1106 return (ts->failed == 0); 1107 } 1108 } 1109 1110 1111 /* 1112 * Runs a single test set, accumulating and then reporting the results. 1113 * Returns true if the test set was successfully run and all tests passed, 1114 * false otherwise. 1115 */ 1116 static int 1117 test_run(struct testset *ts, enum test_verbose verbose) 1118 { 1119 pid_t testpid, child; 1120 int outfd, status; 1121 unsigned long i; 1122 FILE *output; 1123 char buffer[BUFSIZ]; 1124 1125 /* Run the test program. */ 1126 testpid = test_start(ts->command, &outfd); 1127 output = fdopen(outfd, "r"); 1128 if (!output) { 1129 puts("ABORTED"); 1130 fflush(stdout); 1131 sysdie("fdopen failed"); 1132 } 1133 1134 /* 1135 * Pass each line of output to test_checkline(), and print the line if 1136 * verbosity is requested. 1137 */ 1138 while (!ts->aborted && fgets(buffer, sizeof(buffer), output)) { 1139 if (verbose) 1140 printf("%s", buffer); 1141 test_checkline(buffer, ts, verbose); 1142 } 1143 if (ferror(output) || ts->plan == PLAN_INIT) 1144 ts->aborted = 1; 1145 if (!verbose) 1146 test_backspace(ts); 1147 1148 /* 1149 * Consume the rest of the test output, close the output descriptor, 1150 * retrieve the exit status, and pass that information to test_analyze() 1151 * for eventual output. 1152 */ 1153 while (fgets(buffer, sizeof(buffer), output)) 1154 if (verbose) 1155 printf("%s", buffer); 1156 fclose(output); 1157 child = waitpid(testpid, &ts->status, 0); 1158 if (child == (pid_t) -1) { 1159 if (!ts->reported) { 1160 puts("ABORTED"); 1161 fflush(stdout); 1162 } 1163 sysdie("waitpid for %u failed", (unsigned int) testpid); 1164 } 1165 if (ts->all_skipped) 1166 ts->aborted = 0; 1167 status = test_analyze(ts); 1168 1169 /* Convert missing tests to failed tests. */ 1170 for (i = 0; i < ts->count; i++) { 1171 if (ts->results[i] == TEST_INVALID) { 1172 ts->failed++; 1173 ts->results[i] = TEST_FAIL; 1174 status = 0; 1175 } 1176 } 1177 return status; 1178 } 1179 1180 1181 /* Summarize a list of test failures. */ 1182 static void 1183 test_fail_summary(const struct testlist *fails) 1184 { 1185 struct testset *ts; 1186 unsigned int chars; 1187 unsigned long i, first, last, total; 1188 double failed; 1189 1190 puts(header); 1191 1192 /* Failed Set Fail/Total (%) Skip Stat Failing (25) 1193 -------------------------- -------------- ---- ---- -------------- */ 1194 for (; fails; fails = fails->next) { 1195 ts = fails->ts; 1196 total = ts->count - ts->skipped; 1197 failed = (double) ts->failed; 1198 printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed, 1199 total, total ? (failed * 100.0) / (double) total : 0, 1200 ts->skipped); 1201 if (WIFEXITED(ts->status)) 1202 printf("%4d ", WEXITSTATUS(ts->status)); 1203 else 1204 printf(" -- "); 1205 if (ts->aborted) { 1206 puts("aborted"); 1207 continue; 1208 } 1209 chars = 0; 1210 first = 0; 1211 last = 0; 1212 for (i = 0; i < ts->count; i++) { 1213 if (ts->results[i] == TEST_FAIL) { 1214 if (first != 0 && i == last) 1215 last = i + 1; 1216 else { 1217 if (first != 0) 1218 chars += test_print_range(first, last, chars, 19); 1219 first = i + 1; 1220 last = i + 1; 1221 } 1222 } 1223 } 1224 if (first != 0) 1225 test_print_range(first, last, chars, 19); 1226 putchar('\n'); 1227 } 1228 } 1229 1230 1231 /* 1232 * Check whether a given file path is a valid test. Currently, this checks 1233 * whether it is executable and is a regular file. Returns true or false. 1234 */ 1235 static int 1236 is_valid_test(const char *path) 1237 { 1238 struct stat st; 1239 1240 if (access(path, X_OK) < 0) 1241 return 0; 1242 if (stat(path, &st) < 0) 1243 return 0; 1244 if (!S_ISREG(st.st_mode)) 1245 return 0; 1246 return 1; 1247 } 1248 1249 1250 /* 1251 * Given the name of a test, a pointer to the testset struct, and the source 1252 * and build directories, find the test. We try first relative to the current 1253 * directory, then in the build directory (if not NULL), then in the source 1254 * directory. In each of those directories, we first try a "-t" extension and 1255 * then a ".t" extension. When we find an executable program, we return the 1256 * path to that program. If none of those paths are executable, just fill in 1257 * the name of the test as is. 1258 * 1259 * The caller is responsible for freeing the path member of the testset 1260 * struct. 1261 */ 1262 static char * 1263 find_test(const char *name, const char *source, const char *build) 1264 { 1265 char *path = NULL; 1266 const char *bases[3], *suffix, *base; 1267 unsigned int i, j; 1268 const char *suffixes[3] = {"-t", ".t", ""}; 1269 1270 /* Possible base directories. */ 1271 bases[0] = "."; 1272 bases[1] = build; 1273 bases[2] = source; 1274 1275 /* Try each suffix with each base. */ 1276 for (i = 0; i < ARRAY_SIZE(suffixes); i++) { 1277 suffix = suffixes[i]; 1278 for (j = 0; j < ARRAY_SIZE(bases); j++) { 1279 base = bases[j]; 1280 if (base == NULL) 1281 continue; 1282 path = concat(base, "/", name, suffix, (const char *) 0); 1283 if (is_valid_test(path)) 1284 return path; 1285 free(path); 1286 path = NULL; 1287 } 1288 } 1289 if (path == NULL) 1290 path = xstrdup(name); 1291 return path; 1292 } 1293 1294 1295 /* 1296 * Parse a single line of a test list and store the test name and command to 1297 * execute it in the given testset struct. 1298 * 1299 * Normally, each line is just the name of the test, which is located in the 1300 * test directory and turned into a command to run. However, each line may 1301 * have whitespace-separated options, which change the command that's run. 1302 * Current supported options are: 1303 * 1304 * valgrind 1305 * Run the test under valgrind if C_TAP_VALGRIND is set. The contents 1306 * of that environment variable are taken as the valgrind command (with 1307 * options) to run. The command is parsed with a simple split on 1308 * whitespace and no quoting is supported. 1309 * 1310 * libtool 1311 * If running under valgrind, use libtool to invoke valgrind. This avoids 1312 * running valgrind on the wrapper shell script generated by libtool. If 1313 * set, C_TAP_LIBTOOL must be set to the full path to the libtool program 1314 * to use to run valgrind and thus the test. Ignored if the test isn't 1315 * being run under valgrind. 1316 */ 1317 static void 1318 parse_test_list_line(const char *line, struct testset *ts, const char *source, 1319 const char *build) 1320 { 1321 const char *p, *end, *option, *libtool; 1322 const char *valgrind = NULL; 1323 unsigned int use_libtool = 0; 1324 unsigned int use_valgrind = 0; 1325 size_t len, i; 1326 1327 /* Determine the name of the test. */ 1328 p = skip_non_whitespace(line); 1329 ts->file = xstrndup(line, p - line); 1330 1331 /* Check if any test options are set. */ 1332 p = skip_whitespace(p); 1333 while (*p != '\0') { 1334 end = skip_non_whitespace(p); 1335 if (strncmp(p, "libtool", end - p) == 0) { 1336 use_libtool = 1; 1337 } else if (strncmp(p, "valgrind", end - p) == 0) { 1338 valgrind = getenv("C_TAP_VALGRIND"); 1339 use_valgrind = (valgrind != NULL); 1340 } else { 1341 option = xstrndup(p, end - p); 1342 die("unknown test list option %s", option); 1343 } 1344 p = skip_whitespace(end); 1345 } 1346 1347 /* Construct the argv to run the test. First, find the length. */ 1348 len = 1; 1349 if (use_valgrind && valgrind != NULL) { 1350 p = skip_whitespace(valgrind); 1351 while (*p != '\0') { 1352 len++; 1353 p = skip_whitespace(skip_non_whitespace(p)); 1354 } 1355 if (use_libtool) 1356 len += 2; 1357 } 1358 1359 /* Now, build the command. */ 1360 ts->command = xcalloc(len + 1, char *); 1361 i = 0; 1362 if (use_valgrind && valgrind != NULL) { 1363 if (use_libtool) { 1364 libtool = getenv("C_TAP_LIBTOOL"); 1365 if (libtool == NULL) 1366 die("valgrind with libtool requested, but C_TAP_LIBTOOL is not" 1367 " set"); 1368 ts->command[i++] = xstrdup(libtool); 1369 ts->command[i++] = xstrdup("--mode=execute"); 1370 } 1371 p = skip_whitespace(valgrind); 1372 while (*p != '\0') { 1373 end = skip_non_whitespace(p); 1374 ts->command[i++] = xstrndup(p, end - p); 1375 p = skip_whitespace(end); 1376 } 1377 } 1378 if (i != len - 1) 1379 die("internal error while constructing command line"); 1380 ts->command[i++] = find_test(ts->file, source, build); 1381 ts->command[i] = NULL; 1382 } 1383 1384 1385 /* 1386 * Read a list of tests from a file, returning the list of tests as a struct 1387 * testlist, or NULL if there were no tests (such as a file containing only 1388 * comments). Reports an error to standard error and exits if the list of 1389 * tests cannot be read. 1390 */ 1391 static struct testlist * 1392 read_test_list(const char *filename, const char *source, const char *build) 1393 { 1394 FILE *file; 1395 unsigned int line; 1396 size_t length; 1397 char buffer[BUFSIZ]; 1398 const char *start; 1399 struct testlist *listhead, *current; 1400 1401 /* Create the initial container list that will hold our results. */ 1402 listhead = xcalloc(1, struct testlist); 1403 current = NULL; 1404 1405 /* 1406 * Open our file of tests to run and read it line by line, creating a new 1407 * struct testlist and struct testset for each line. 1408 */ 1409 file = fopen(filename, "r"); 1410 if (file == NULL) 1411 sysdie("can't open %s", filename); 1412 line = 0; 1413 while (fgets(buffer, sizeof(buffer), file)) { 1414 line++; 1415 length = strlen(buffer) - 1; 1416 if (buffer[length] != '\n') { 1417 fprintf(stderr, "%s:%u: line too long\n", filename, line); 1418 exit(1); 1419 } 1420 buffer[length] = '\0'; 1421 1422 /* Skip comments, leading spaces, and blank lines. */ 1423 start = skip_whitespace(buffer); 1424 if (strlen(start) == 0) 1425 continue; 1426 if (start[0] == '#') 1427 continue; 1428 1429 /* Allocate the new testset structure. */ 1430 if (current == NULL) 1431 current = listhead; 1432 else { 1433 current->next = xcalloc(1, struct testlist); 1434 current = current->next; 1435 } 1436 current->ts = xcalloc(1, struct testset); 1437 current->ts->plan = PLAN_INIT; 1438 1439 /* Parse the line and store the results in the testset struct. */ 1440 parse_test_list_line(start, current->ts, source, build); 1441 } 1442 fclose(file); 1443 1444 /* If there were no tests, current is still NULL. */ 1445 if (current == NULL) { 1446 free(listhead); 1447 return NULL; 1448 } 1449 1450 /* Return the results. */ 1451 return listhead; 1452 } 1453 1454 1455 /* 1456 * Build a list of tests from command line arguments. Takes the argv and argc 1457 * representing the command line arguments and returns a newly allocated test 1458 * list, or NULL if there were no tests. The caller is responsible for 1459 * freeing. 1460 */ 1461 static struct testlist * 1462 build_test_list(char *argv[], int argc, const char *source, const char *build) 1463 { 1464 int i; 1465 struct testlist *listhead, *current; 1466 1467 /* Create the initial container list that will hold our results. */ 1468 listhead = xcalloc(1, struct testlist); 1469 current = NULL; 1470 1471 /* Walk the list of arguments and create test sets for them. */ 1472 for (i = 0; i < argc; i++) { 1473 if (current == NULL) 1474 current = listhead; 1475 else { 1476 current->next = xcalloc(1, struct testlist); 1477 current = current->next; 1478 } 1479 current->ts = xcalloc(1, struct testset); 1480 current->ts->plan = PLAN_INIT; 1481 current->ts->file = xstrdup(argv[i]); 1482 current->ts->command = xcalloc(2, char *); 1483 current->ts->command[0] = find_test(current->ts->file, source, build); 1484 current->ts->command[1] = NULL; 1485 } 1486 1487 /* If there were no tests, current is still NULL. */ 1488 if (current == NULL) { 1489 free(listhead); 1490 return NULL; 1491 } 1492 1493 /* Return the results. */ 1494 return listhead; 1495 } 1496 1497 1498 /* Free a struct testset. */ 1499 static void 1500 free_testset(struct testset *ts) 1501 { 1502 size_t i; 1503 1504 free(ts->file); 1505 for (i = 0; ts->command[i] != NULL; i++) 1506 free(ts->command[i]); 1507 free(ts->command); 1508 free(ts->results); 1509 free(ts->reason); 1510 free(ts); 1511 } 1512 1513 1514 /* 1515 * Run a batch of tests. Takes two additional parameters: the root of the 1516 * source directory and the root of the build directory. Test programs will 1517 * be first searched for in the current directory, then the build directory, 1518 * then the source directory. Returns true iff all tests passed, and always 1519 * frees the test list that's passed in. 1520 */ 1521 static int 1522 test_batch(struct testlist *tests, enum test_verbose verbose) 1523 { 1524 size_t length, i; 1525 size_t longest = 0; 1526 unsigned int count = 0; 1527 struct testset *ts; 1528 struct timeval start, end; 1529 struct rusage stats; 1530 struct testlist *failhead = NULL; 1531 struct testlist *failtail = NULL; 1532 struct testlist *current, *next; 1533 int succeeded; 1534 unsigned long total = 0; 1535 unsigned long passed = 0; 1536 unsigned long skipped = 0; 1537 unsigned long failed = 0; 1538 unsigned long aborted = 0; 1539 1540 /* Walk the list of tests to find the longest name. */ 1541 for (current = tests; current != NULL; current = current->next) { 1542 length = strlen(current->ts->file); 1543 if (length > longest) 1544 longest = length; 1545 } 1546 1547 /* 1548 * Add two to longest and round up to the nearest tab stop. This is how 1549 * wide the column for printing the current test name will be. 1550 */ 1551 longest += 2; 1552 if (longest % 8) 1553 longest += 8 - (longest % 8); 1554 1555 /* Start the wall clock timer. */ 1556 gettimeofday(&start, NULL); 1557 1558 /* Now, plow through our tests again, running each one. */ 1559 for (current = tests; current != NULL; current = current->next) { 1560 ts = current->ts; 1561 1562 /* Print out the name of the test file. */ 1563 fputs(ts->file, stdout); 1564 if (verbose) 1565 fputs("\n\n", stdout); 1566 else 1567 for (i = strlen(ts->file); i < longest; i++) 1568 putchar('.'); 1569 if (isatty(STDOUT_FILENO)) 1570 fflush(stdout); 1571 1572 /* Run the test. */ 1573 succeeded = test_run(ts, verbose); 1574 fflush(stdout); 1575 if (verbose) 1576 putchar('\n'); 1577 1578 /* Record cumulative statistics. */ 1579 aborted += ts->aborted; 1580 total += ts->count + ts->all_skipped; 1581 passed += ts->passed; 1582 skipped += ts->skipped + ts->all_skipped; 1583 failed += ts->failed; 1584 count++; 1585 1586 /* If the test fails, we shuffle it over to the fail list. */ 1587 if (!succeeded) { 1588 if (failhead == NULL) { 1589 failhead = xcalloc(1, struct testlist); 1590 failtail = failhead; 1591 } else { 1592 failtail->next = xcalloc(1, struct testlist); 1593 failtail = failtail->next; 1594 } 1595 failtail->ts = ts; 1596 failtail->next = NULL; 1597 } 1598 } 1599 total -= skipped; 1600 1601 /* Stop the timer and get our child resource statistics. */ 1602 gettimeofday(&end, NULL); 1603 getrusage(RUSAGE_CHILDREN, &stats); 1604 1605 /* Summarize the failures and free the failure list. */ 1606 if (failhead != NULL) { 1607 test_fail_summary(failhead); 1608 while (failhead != NULL) { 1609 next = failhead->next; 1610 free(failhead); 1611 failhead = next; 1612 } 1613 } 1614 1615 /* Free the memory used by the test lists. */ 1616 while (tests != NULL) { 1617 next = tests->next; 1618 free_testset(tests->ts); 1619 free(tests); 1620 tests = next; 1621 } 1622 1623 /* Print out the final test summary. */ 1624 putchar('\n'); 1625 if (aborted != 0) { 1626 if (aborted == 1) 1627 printf("Aborted %lu test set", aborted); 1628 else 1629 printf("Aborted %lu test sets", aborted); 1630 printf(", passed %lu/%lu tests", passed, total); 1631 } else if (failed == 0) 1632 fputs("All tests successful", stdout); 1633 else 1634 printf("Failed %lu/%lu tests, %.2f%% okay", failed, total, 1635 (double) (total - failed) * 100.0 / (double) total); 1636 if (skipped != 0) { 1637 if (skipped == 1) 1638 printf(", %lu test skipped", skipped); 1639 else 1640 printf(", %lu tests skipped", skipped); 1641 } 1642 puts("."); 1643 printf("Files=%u, Tests=%lu", count, total); 1644 printf(", %.2f seconds", tv_diff(&end, &start)); 1645 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n", tv_seconds(&stats.ru_utime), 1646 tv_seconds(&stats.ru_stime), 1647 tv_sum(&stats.ru_utime, &stats.ru_stime)); 1648 return (failed == 0 && aborted == 0); 1649 } 1650 1651 1652 /* 1653 * Run a single test case. This involves just running the test program after 1654 * having done the environment setup and finding the test program. 1655 */ 1656 static void 1657 test_single(const char *program, const char *source, const char *build) 1658 { 1659 char *path; 1660 1661 path = find_test(program, source, build); 1662 if (execl(path, path, (char *) 0) == -1) 1663 sysdie("cannot exec %s", path); 1664 } 1665 1666 1667 /* 1668 * Main routine. Set the C_TAP_SOURCE, C_TAP_BUILD, SOURCE, and BUILD 1669 * environment variables and then, given a file listing tests, run each test 1670 * listed. 1671 */ 1672 int 1673 main(int argc, char *argv[]) 1674 { 1675 int option; 1676 int status = 0; 1677 int single = 0; 1678 enum test_verbose verbose = CONCISE; 1679 char *c_tap_source_env = NULL; 1680 char *c_tap_build_env = NULL; 1681 char *source_env = NULL; 1682 char *build_env = NULL; 1683 const char *program; 1684 const char *shortlist; 1685 const char *list = NULL; 1686 const char *source = C_TAP_SOURCE; 1687 const char *build = C_TAP_BUILD; 1688 struct testlist *tests; 1689 1690 program = argv[0]; 1691 while ((option = getopt(argc, argv, "b:hl:os:v")) != EOF) { 1692 switch (option) { 1693 case 'b': 1694 build = optarg; 1695 break; 1696 case 'h': 1697 printf(usage_message, program, program, program, usage_extra); 1698 exit(0); 1699 case 'l': 1700 list = optarg; 1701 break; 1702 case 'o': 1703 single = 1; 1704 break; 1705 case 's': 1706 source = optarg; 1707 break; 1708 case 'v': 1709 verbose = VERBOSE; 1710 break; 1711 default: 1712 exit(1); 1713 } 1714 } 1715 argv += optind; 1716 argc -= optind; 1717 if ((list == NULL && argc < 1) || (list != NULL && argc > 0)) { 1718 fprintf(stderr, usage_message, program, program, program, usage_extra); 1719 exit(1); 1720 } 1721 1722 /* 1723 * If C_TAP_VERBOSE is set in the environment, that also turns on verbose 1724 * mode. 1725 */ 1726 if (getenv("C_TAP_VERBOSE") != NULL) 1727 verbose = VERBOSE; 1728 1729 /* 1730 * Set C_TAP_SOURCE and C_TAP_BUILD environment variables. Also set 1731 * SOURCE and BUILD for backward compatibility, although we're trying to 1732 * migrate to the ones with a C_TAP_* prefix. 1733 */ 1734 if (source != NULL) { 1735 c_tap_source_env = concat("C_TAP_SOURCE=", source, (const char *) 0); 1736 if (putenv(c_tap_source_env) != 0) 1737 sysdie("cannot set C_TAP_SOURCE in the environment"); 1738 source_env = concat("SOURCE=", source, (const char *) 0); 1739 if (putenv(source_env) != 0) 1740 sysdie("cannot set SOURCE in the environment"); 1741 } 1742 if (build != NULL) { 1743 c_tap_build_env = concat("C_TAP_BUILD=", build, (const char *) 0); 1744 if (putenv(c_tap_build_env) != 0) 1745 sysdie("cannot set C_TAP_BUILD in the environment"); 1746 build_env = concat("BUILD=", build, (const char *) 0); 1747 if (putenv(build_env) != 0) 1748 sysdie("cannot set BUILD in the environment"); 1749 } 1750 1751 /* Run the tests as instructed. */ 1752 if (single) 1753 test_single(argv[0], source, build); 1754 else if (list != NULL) { 1755 shortlist = strrchr(list, '/'); 1756 if (shortlist == NULL) 1757 shortlist = list; 1758 else 1759 shortlist++; 1760 printf(banner, shortlist); 1761 tests = read_test_list(list, source, build); 1762 status = test_batch(tests, verbose) ? 0 : 1; 1763 } else { 1764 tests = build_test_list(argv, argc, source, build); 1765 status = test_batch(tests, verbose) ? 0 : 1; 1766 } 1767 1768 /* For valgrind cleanliness, free all our memory. */ 1769 if (source_env != NULL) { 1770 putenv((char *) "C_TAP_SOURCE="); 1771 putenv((char *) "SOURCE="); 1772 free(c_tap_source_env); 1773 free(source_env); 1774 } 1775 if (build_env != NULL) { 1776 putenv((char *) "C_TAP_BUILD="); 1777 putenv((char *) "BUILD="); 1778 free(c_tap_build_env); 1779 free(build_env); 1780 } 1781 exit(status); 1782 } 1783