1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 #ifdef TINYTEST_LOCAL 26 #include "tinytest_local.h" 27 #endif 28 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <assert.h> 33 34 #ifndef NO_FORKING 35 36 #ifdef _WIN32 37 #include <windows.h> 38 #else 39 #include <sys/types.h> 40 #include <sys/wait.h> 41 #include <unistd.h> 42 #endif 43 44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 46 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 47 /* Workaround for a stupid bug in OSX 10.6 */ 48 #define FORK_BREAKS_GCOV 49 #include <vproc.h> 50 #endif 51 #endif 52 53 #endif /* !NO_FORKING */ 54 55 #ifndef __GNUC__ 56 #define __attribute__(x) 57 #endif 58 59 #include "tinytest.h" 60 #include "tinytest_macros.h" 61 62 #define LONGEST_TEST_NAME 16384 63 64 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 65 static int n_ok = 0; /**< Number of tests that have passed */ 66 static int n_bad = 0; /**< Number of tests that have failed. */ 67 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 68 69 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 70 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 71 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 72 const char *verbosity_flag = ""; 73 74 const struct testlist_alias_t *cfg_aliases=NULL; 75 76 enum outcome { SKIP=2, OK=1, FAIL=0 }; 77 static enum outcome cur_test_outcome = 0; 78 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 79 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 80 const char *cur_test_name = NULL; 81 82 #ifdef _WIN32 83 /* Copy of argv[0] for win32. */ 84 static char commandname[MAX_PATH+1]; 85 #endif 86 87 static void usage(struct testgroup_t *groups, int list_groups) 88 __attribute__((noreturn)); 89 static int process_test_option(struct testgroup_t *groups, const char *test); 90 91 static enum outcome 92 testcase_run_bare_(const struct testcase_t *testcase) 93 { 94 void *env = NULL; 95 int outcome; 96 if (testcase->setup) { 97 env = testcase->setup->setup_fn(testcase); 98 if (!env) 99 return FAIL; 100 else if (env == (void*)TT_SKIP) 101 return SKIP; 102 } 103 104 cur_test_outcome = OK; 105 testcase->fn(env); 106 outcome = cur_test_outcome; 107 108 if (testcase->setup) { 109 if (testcase->setup->cleanup_fn(testcase, env) == 0) 110 outcome = FAIL; 111 } 112 113 return outcome; 114 } 115 116 #define MAGIC_EXITCODE 42 117 118 #ifndef NO_FORKING 119 120 static enum outcome 121 testcase_run_forked_(const struct testgroup_t *group, 122 const struct testcase_t *testcase) 123 { 124 #ifdef _WIN32 125 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 126 we'll invoke our own exe (whose name we recall from the command 127 line) with a command line that tells it to run just the test we 128 want, and this time without forking. 129 130 (No, threads aren't an option. The whole point of forking is to 131 share no state between tests.) 132 */ 133 int ok; 134 char buffer[LONGEST_TEST_NAME+256]; 135 STARTUPINFOA si; 136 PROCESS_INFORMATION info; 137 DWORD exitcode; 138 139 if (!in_tinytest_main) { 140 printf("\nERROR. On Windows, testcase_run_forked_ must be" 141 " called from within tinytest_main.\n"); 142 abort(); 143 } 144 if (opt_verbosity>0) 145 printf("[forking] "); 146 147 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 148 commandname, verbosity_flag, group->prefix, testcase->name); 149 150 memset(&si, 0, sizeof(si)); 151 memset(&info, 0, sizeof(info)); 152 si.cb = sizeof(si); 153 154 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 155 0, NULL, NULL, &si, &info); 156 if (!ok) { 157 printf("CreateProcess failed!\n"); 158 return 0; 159 } 160 WaitForSingleObject(info.hProcess, INFINITE); 161 GetExitCodeProcess(info.hProcess, &exitcode); 162 CloseHandle(info.hProcess); 163 CloseHandle(info.hThread); 164 if (exitcode == 0) 165 return OK; 166 else if (exitcode == MAGIC_EXITCODE) 167 return SKIP; 168 else 169 return FAIL; 170 #else 171 int outcome_pipe[2]; 172 pid_t pid; 173 (void)group; 174 175 if (pipe(outcome_pipe)) 176 perror("opening pipe"); 177 178 if (opt_verbosity>0) 179 printf("[forking] "); 180 pid = fork(); 181 #ifdef FORK_BREAKS_GCOV 182 vproc_transaction_begin(0); 183 #endif 184 if (!pid) { 185 /* child. */ 186 int test_r, write_r; 187 char b[1]; 188 close(outcome_pipe[0]); 189 test_r = testcase_run_bare_(testcase); 190 assert(0<=(int)test_r && (int)test_r<=2); 191 b[0] = "NYS"[test_r]; 192 write_r = (int)write(outcome_pipe[1], b, 1); 193 if (write_r != 1) { 194 perror("write outcome to pipe"); 195 exit(1); 196 } 197 exit(0); 198 return FAIL; /* unreachable */ 199 } else { 200 /* parent */ 201 int status, r; 202 char b[1]; 203 /* Close this now, so that if the other side closes it, 204 * our read fails. */ 205 close(outcome_pipe[1]); 206 r = (int)read(outcome_pipe[0], b, 1); 207 if (r == 0) { 208 printf("[Lost connection!] "); 209 return 0; 210 } else if (r != 1) { 211 perror("read outcome from pipe"); 212 } 213 waitpid(pid, &status, 0); 214 close(outcome_pipe[0]); 215 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 216 } 217 #endif 218 } 219 220 #endif /* !NO_FORKING */ 221 222 int 223 testcase_run_one(const struct testgroup_t *group, 224 const struct testcase_t *testcase) 225 { 226 enum outcome outcome; 227 228 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 229 if (opt_verbosity>0) 230 printf("%s%s: %s\n", 231 group->prefix, testcase->name, 232 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 233 ++n_skipped; 234 return SKIP; 235 } 236 237 if (opt_verbosity>0 && !opt_forked) { 238 printf("%s%s: ", group->prefix, testcase->name); 239 } else { 240 if (opt_verbosity==0) printf("."); 241 cur_test_prefix = group->prefix; 242 cur_test_name = testcase->name; 243 } 244 245 #ifndef NO_FORKING 246 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 247 outcome = testcase_run_forked_(group, testcase); 248 } else { 249 #else 250 { 251 #endif 252 outcome = testcase_run_bare_(testcase); 253 } 254 255 if (outcome == OK) { 256 ++n_ok; 257 if (opt_verbosity>0 && !opt_forked) 258 puts(opt_verbosity==1?"OK":""); 259 } else if (outcome == SKIP) { 260 ++n_skipped; 261 if (opt_verbosity>0 && !opt_forked) 262 puts("SKIPPED"); 263 } else { 264 ++n_bad; 265 if (!opt_forked) 266 printf("\n [%s FAILED]\n", testcase->name); 267 } 268 269 if (opt_forked) { 270 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 271 return 1; /* unreachable */ 272 } else { 273 return (int)outcome; 274 } 275 } 276 277 int 278 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 279 { 280 int i, j; 281 size_t length = LONGEST_TEST_NAME; 282 char fullname[LONGEST_TEST_NAME]; 283 int found=0; 284 if (strstr(arg, "..")) 285 length = strstr(arg,"..")-arg; 286 for (i=0; groups[i].prefix; ++i) { 287 for (j=0; groups[i].cases[j].name; ++j) { 288 struct testcase_t *testcase = &groups[i].cases[j]; 289 snprintf(fullname, sizeof(fullname), "%s%s", 290 groups[i].prefix, testcase->name); 291 if (!flag) { /* Hack! */ 292 printf(" %s", fullname); 293 if (testcase->flags & TT_OFF_BY_DEFAULT) 294 puts(" (Off by default)"); 295 else if (testcase->flags & TT_SKIP) 296 puts(" (DISABLED)"); 297 else 298 puts(""); 299 } 300 if (!strncmp(fullname, arg, length)) { 301 if (set) 302 testcase->flags |= flag; 303 else 304 testcase->flags &= ~flag; 305 ++found; 306 } 307 } 308 } 309 return found; 310 } 311 312 static void 313 usage(struct testgroup_t *groups, int list_groups) 314 { 315 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 316 puts(" Specify tests by name, or using a prefix ending with '..'"); 317 puts(" To skip a test, prefix its name with a colon."); 318 puts(" To enable a disabled test, prefix its name with a plus."); 319 puts(" Use --list-tests for a list of tests."); 320 if (list_groups) { 321 puts("Known tests are:"); 322 tinytest_set_flag_(groups, "..", 1, 0); 323 } 324 exit(0); 325 } 326 327 static int 328 process_test_alias(struct testgroup_t *groups, const char *test) 329 { 330 int i, j, n, r; 331 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 332 if (!strcmp(cfg_aliases[i].name, test)) { 333 n = 0; 334 for (j = 0; cfg_aliases[i].tests[j]; ++j) { 335 r = process_test_option(groups, cfg_aliases[i].tests[j]); 336 if (r<0) 337 return -1; 338 n += r; 339 } 340 return n; 341 } 342 } 343 printf("No such test alias as @%s!",test); 344 return -1; 345 } 346 347 static int 348 process_test_option(struct testgroup_t *groups, const char *test) 349 { 350 int flag = TT_ENABLED_; 351 int n = 0; 352 if (test[0] == '@') { 353 return process_test_alias(groups, test + 1); 354 } else if (test[0] == ':') { 355 ++test; 356 flag = TT_SKIP; 357 } else if (test[0] == '+') { 358 ++test; 359 ++n; 360 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 361 printf("No such test as %s!\n", test); 362 return -1; 363 } 364 } else { 365 ++n; 366 } 367 if (!tinytest_set_flag_(groups, test, 1, flag)) { 368 printf("No such test as %s!\n", test); 369 return -1; 370 } 371 return n; 372 } 373 374 void 375 tinytest_set_aliases(const struct testlist_alias_t *aliases) 376 { 377 cfg_aliases = aliases; 378 } 379 380 int 381 tinytest_main(int c, const char **v, struct testgroup_t *groups) 382 { 383 int i, j, n=0; 384 385 #ifdef _WIN32 386 const char *sp = strrchr(v[0], '.'); 387 const char *extension = ""; 388 if (!sp || stricmp(sp, ".exe")) 389 extension = ".exe"; /* Add an exe so CreateProcess will work */ 390 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 391 commandname[MAX_PATH]='\0'; 392 #endif 393 for (i=1; i<c; ++i) { 394 if (v[i][0] == '-') { 395 if (!strcmp(v[i], "--RUNNING-FORKED")) { 396 opt_forked = 1; 397 } else if (!strcmp(v[i], "--no-fork")) { 398 opt_nofork = 1; 399 } else if (!strcmp(v[i], "--quiet")) { 400 opt_verbosity = -1; 401 verbosity_flag = "--quiet"; 402 } else if (!strcmp(v[i], "--verbose")) { 403 opt_verbosity = 2; 404 verbosity_flag = "--verbose"; 405 } else if (!strcmp(v[i], "--terse")) { 406 opt_verbosity = 0; 407 verbosity_flag = "--terse"; 408 } else if (!strcmp(v[i], "--help")) { 409 usage(groups, 0); 410 } else if (!strcmp(v[i], "--list-tests")) { 411 usage(groups, 1); 412 } else { 413 printf("Unknown option %s. Try --help\n",v[i]); 414 return -1; 415 } 416 } else { 417 int r = process_test_option(groups, v[i]); 418 if (r<0) 419 return -1; 420 n += r; 421 } 422 } 423 if (!n) 424 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 425 426 #ifdef _IONBF 427 setvbuf(stdout, NULL, _IONBF, 0); 428 #endif 429 430 ++in_tinytest_main; 431 for (i=0; groups[i].prefix; ++i) 432 for (j=0; groups[i].cases[j].name; ++j) 433 if (groups[i].cases[j].flags & TT_ENABLED_) 434 testcase_run_one(&groups[i], 435 &groups[i].cases[j]); 436 437 --in_tinytest_main; 438 439 if (opt_verbosity==0) 440 puts(""); 441 442 if (n_bad) 443 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 444 n_bad+n_ok,n_skipped); 445 else if (opt_verbosity >= 1) 446 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 447 448 return (n_bad == 0) ? 0 : 1; 449 } 450 451 int 452 tinytest_get_verbosity_(void) 453 { 454 return opt_verbosity; 455 } 456 457 void 458 tinytest_set_test_failed_(void) 459 { 460 if (opt_verbosity <= 0 && cur_test_name) { 461 if (opt_verbosity==0) puts(""); 462 printf("%s%s: ", cur_test_prefix, cur_test_name); 463 cur_test_name = NULL; 464 } 465 cur_test_outcome = 0; 466 } 467 468 void 469 tinytest_set_test_skipped_(void) 470 { 471 if (cur_test_outcome==OK) 472 cur_test_outcome = SKIP; 473 } 474 475 char * 476 tinytest_format_hex_(const void *val_, unsigned long len) 477 { 478 const unsigned char *val = val_; 479 char *result, *cp; 480 size_t i; 481 482 if (!val) 483 return strdup("null"); 484 if (!(result = malloc(len*2+1))) 485 return strdup("<allocation failure>"); 486 cp = result; 487 for (i=0;i<len;++i) { 488 *cp++ = "0123456789ABCDEF"[val[i] >> 4]; 489 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; 490 } 491 *cp = 0; 492 return result; 493 } 494