1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 #ifdef TINYTEST_LOCAL 26 #include "tinytest_local.h" 27 #endif 28 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <assert.h> 33 34 #ifdef _WIN32 35 #include <windows.h> 36 #else 37 #include <sys/types.h> 38 #include <sys/wait.h> 39 #include <unistd.h> 40 #endif 41 42 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 43 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 44 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 45 /* Workaround for a stupid bug in OSX 10.6 */ 46 #define FORK_BREAKS_GCOV 47 #include <vproc.h> 48 #endif 49 #endif 50 51 #ifndef __GNUC__ 52 #define __attribute__(x) 53 #endif 54 55 #include "tinytest.h" 56 #include "tinytest_macros.h" 57 58 #define LONGEST_TEST_NAME 16384 59 60 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 61 static int n_ok = 0; /**< Number of tests that have passed */ 62 static int n_bad = 0; /**< Number of tests that have failed. */ 63 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 64 65 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 66 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 67 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 68 const char *verbosity_flag = ""; 69 70 const struct testlist_alias_t *cfg_aliases=NULL; 71 72 enum outcome { SKIP=2, OK=1, FAIL=0 }; 73 static enum outcome cur_test_outcome = 0; 74 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 75 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 76 const char *cur_test_name = NULL; 77 78 #ifdef _WIN32 79 /* Copy of argv[0] for win32. */ 80 static char commandname[MAX_PATH+1]; 81 #endif 82 83 static void usage(struct testgroup_t *groups, int list_groups) 84 __attribute__((noreturn)); 85 static int process_test_option(struct testgroup_t *groups, const char *test); 86 87 static enum outcome 88 testcase_run_bare_(const struct testcase_t *testcase) 89 { 90 void *env = NULL; 91 int outcome; 92 if (testcase->setup) { 93 env = testcase->setup->setup_fn(testcase); 94 if (!env) 95 return FAIL; 96 else if (env == (void*)TT_SKIP) 97 return SKIP; 98 } 99 100 cur_test_outcome = OK; 101 testcase->fn(env); 102 outcome = cur_test_outcome; 103 104 if (testcase->setup) { 105 if (testcase->setup->cleanup_fn(testcase, env) == 0) 106 outcome = FAIL; 107 } 108 109 return outcome; 110 } 111 112 #define MAGIC_EXITCODE 42 113 114 static enum outcome 115 testcase_run_forked_(const struct testgroup_t *group, 116 const struct testcase_t *testcase) 117 { 118 #ifdef _WIN32 119 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 120 we'll invoke our own exe (whose name we recall from the command 121 line) with a command line that tells it to run just the test we 122 want, and this time without forking. 123 124 (No, threads aren't an option. The whole point of forking is to 125 share no state between tests.) 126 */ 127 int ok; 128 char buffer[LONGEST_TEST_NAME+256]; 129 STARTUPINFOA si; 130 PROCESS_INFORMATION info; 131 DWORD exitcode; 132 133 if (!in_tinytest_main) { 134 printf("\nERROR. On Windows, testcase_run_forked_ must be" 135 " called from within tinytest_main.\n"); 136 abort(); 137 } 138 if (opt_verbosity>0) 139 printf("[forking] "); 140 141 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 142 commandname, verbosity_flag, group->prefix, testcase->name); 143 144 memset(&si, 0, sizeof(si)); 145 memset(&info, 0, sizeof(info)); 146 si.cb = sizeof(si); 147 148 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 149 0, NULL, NULL, &si, &info); 150 if (!ok) { 151 printf("CreateProcess failed!\n"); 152 return 0; 153 } 154 WaitForSingleObject(info.hProcess, INFINITE); 155 GetExitCodeProcess(info.hProcess, &exitcode); 156 CloseHandle(info.hProcess); 157 CloseHandle(info.hThread); 158 if (exitcode == 0) 159 return OK; 160 else if (exitcode == MAGIC_EXITCODE) 161 return SKIP; 162 else 163 return FAIL; 164 #else 165 int outcome_pipe[2]; 166 pid_t pid; 167 (void)group; 168 169 if (pipe(outcome_pipe)) 170 perror("opening pipe"); 171 172 if (opt_verbosity>0) 173 printf("[forking] "); 174 pid = fork(); 175 #ifdef FORK_BREAKS_GCOV 176 vproc_transaction_begin(0); 177 #endif 178 if (!pid) { 179 /* child. */ 180 int test_r, write_r; 181 char b[1]; 182 close(outcome_pipe[0]); 183 test_r = testcase_run_bare_(testcase); 184 assert(0<=(int)test_r && (int)test_r<=2); 185 b[0] = "NYS"[test_r]; 186 write_r = (int)write(outcome_pipe[1], b, 1); 187 if (write_r != 1) { 188 perror("write outcome to pipe"); 189 exit(1); 190 } 191 exit(0); 192 return FAIL; /* unreachable */ 193 } else { 194 /* parent */ 195 int status, r; 196 char b[1]; 197 /* Close this now, so that if the other side closes it, 198 * our read fails. */ 199 close(outcome_pipe[1]); 200 r = (int)read(outcome_pipe[0], b, 1); 201 if (r == 0) { 202 printf("[Lost connection!] "); 203 return 0; 204 } else if (r != 1) { 205 perror("read outcome from pipe"); 206 } 207 waitpid(pid, &status, 0); 208 close(outcome_pipe[0]); 209 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 210 } 211 #endif 212 } 213 214 int 215 testcase_run_one(const struct testgroup_t *group, 216 const struct testcase_t *testcase) 217 { 218 enum outcome outcome; 219 220 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 221 if (opt_verbosity>0) 222 printf("%s%s: %s\n", 223 group->prefix, testcase->name, 224 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 225 ++n_skipped; 226 return SKIP; 227 } 228 229 if (opt_verbosity>0 && !opt_forked) { 230 printf("%s%s: ", group->prefix, testcase->name); 231 } else { 232 if (opt_verbosity==0) printf("."); 233 cur_test_prefix = group->prefix; 234 cur_test_name = testcase->name; 235 } 236 237 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 238 outcome = testcase_run_forked_(group, testcase); 239 } else { 240 outcome = testcase_run_bare_(testcase); 241 } 242 243 if (outcome == OK) { 244 ++n_ok; 245 if (opt_verbosity>0 && !opt_forked) 246 puts(opt_verbosity==1?"OK":""); 247 } else if (outcome == SKIP) { 248 ++n_skipped; 249 if (opt_verbosity>0 && !opt_forked) 250 puts("SKIPPED"); 251 } else { 252 ++n_bad; 253 if (!opt_forked) 254 printf("\n [%s FAILED]\n", testcase->name); 255 } 256 257 if (opt_forked) { 258 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 259 return 1; /* unreachable */ 260 } else { 261 return (int)outcome; 262 } 263 } 264 265 int 266 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 267 { 268 int i, j; 269 size_t length = LONGEST_TEST_NAME; 270 char fullname[LONGEST_TEST_NAME]; 271 int found=0; 272 if (strstr(arg, "..")) 273 length = strstr(arg,"..")-arg; 274 for (i=0; groups[i].prefix; ++i) { 275 for (j=0; groups[i].cases[j].name; ++j) { 276 struct testcase_t *testcase = &groups[i].cases[j]; 277 snprintf(fullname, sizeof(fullname), "%s%s", 278 groups[i].prefix, testcase->name); 279 if (!flag) { /* Hack! */ 280 printf(" %s", fullname); 281 if (testcase->flags & TT_OFF_BY_DEFAULT) 282 puts(" (Off by default)"); 283 else if (testcase->flags & TT_SKIP) 284 puts(" (DISABLED)"); 285 else 286 puts(""); 287 } 288 if (!strncmp(fullname, arg, length)) { 289 if (set) 290 testcase->flags |= flag; 291 else 292 testcase->flags &= ~flag; 293 ++found; 294 } 295 } 296 } 297 return found; 298 } 299 300 static void 301 usage(struct testgroup_t *groups, int list_groups) 302 { 303 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 304 puts(" Specify tests by name, or using a prefix ending with '..'"); 305 puts(" To skip a test, prefix its name with a colon."); 306 puts(" To enable a disabled test, prefix its name with a plus."); 307 puts(" Use --list-tests for a list of tests."); 308 if (list_groups) { 309 puts("Known tests are:"); 310 tinytest_set_flag_(groups, "..", 1, 0); 311 } 312 exit(0); 313 } 314 315 static int 316 process_test_alias(struct testgroup_t *groups, const char *test) 317 { 318 int i, j, n, r; 319 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 320 if (!strcmp(cfg_aliases[i].name, test)) { 321 n = 0; 322 for (j = 0; cfg_aliases[i].tests[j]; ++j) { 323 r = process_test_option(groups, cfg_aliases[i].tests[j]); 324 if (r<0) 325 return -1; 326 n += r; 327 } 328 return n; 329 } 330 } 331 printf("No such test alias as @%s!",test); 332 return -1; 333 } 334 335 static int 336 process_test_option(struct testgroup_t *groups, const char *test) 337 { 338 int flag = TT_ENABLED_; 339 int n = 0; 340 if (test[0] == '@') { 341 return process_test_alias(groups, test + 1); 342 } else if (test[0] == ':') { 343 ++test; 344 flag = TT_SKIP; 345 } else if (test[0] == '+') { 346 ++test; 347 ++n; 348 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 349 printf("No such test as %s!\n", test); 350 return -1; 351 } 352 } else { 353 ++n; 354 } 355 if (!tinytest_set_flag_(groups, test, 1, flag)) { 356 printf("No such test as %s!\n", test); 357 return -1; 358 } 359 return n; 360 } 361 362 void 363 tinytest_set_aliases(const struct testlist_alias_t *aliases) 364 { 365 cfg_aliases = aliases; 366 } 367 368 int 369 tinytest_main(int c, const char **v, struct testgroup_t *groups) 370 { 371 int i, j, n=0; 372 373 #ifdef _WIN32 374 const char *sp = strrchr(v[0], '.'); 375 const char *extension = ""; 376 if (!sp || stricmp(sp, ".exe")) 377 extension = ".exe"; /* Add an exe so CreateProcess will work */ 378 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 379 commandname[MAX_PATH]='\0'; 380 #endif 381 for (i=1; i<c; ++i) { 382 if (v[i][0] == '-') { 383 if (!strcmp(v[i], "--RUNNING-FORKED")) { 384 opt_forked = 1; 385 } else if (!strcmp(v[i], "--no-fork")) { 386 opt_nofork = 1; 387 } else if (!strcmp(v[i], "--quiet")) { 388 opt_verbosity = -1; 389 verbosity_flag = "--quiet"; 390 } else if (!strcmp(v[i], "--verbose")) { 391 opt_verbosity = 2; 392 verbosity_flag = "--verbose"; 393 } else if (!strcmp(v[i], "--terse")) { 394 opt_verbosity = 0; 395 verbosity_flag = "--terse"; 396 } else if (!strcmp(v[i], "--help")) { 397 usage(groups, 0); 398 } else if (!strcmp(v[i], "--list-tests")) { 399 usage(groups, 1); 400 } else { 401 printf("Unknown option %s. Try --help\n",v[i]); 402 return -1; 403 } 404 } else { 405 int r = process_test_option(groups, v[i]); 406 if (r<0) 407 return -1; 408 n += r; 409 } 410 } 411 if (!n) 412 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 413 414 setvbuf(stdout, NULL, _IONBF, 0); 415 416 ++in_tinytest_main; 417 for (i=0; groups[i].prefix; ++i) 418 for (j=0; groups[i].cases[j].name; ++j) 419 if (groups[i].cases[j].flags & TT_ENABLED_) 420 testcase_run_one(&groups[i], 421 &groups[i].cases[j]); 422 423 --in_tinytest_main; 424 425 if (opt_verbosity==0) 426 puts(""); 427 428 if (n_bad) 429 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 430 n_bad+n_ok,n_skipped); 431 else if (opt_verbosity >= 1) 432 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 433 434 return (n_bad == 0) ? 0 : 1; 435 } 436 437 int 438 tinytest_get_verbosity_(void) 439 { 440 return opt_verbosity; 441 } 442 443 void 444 tinytest_set_test_failed_(void) 445 { 446 if (opt_verbosity <= 0 && cur_test_name) { 447 if (opt_verbosity==0) puts(""); 448 printf("%s%s: ", cur_test_prefix, cur_test_name); 449 cur_test_name = NULL; 450 } 451 cur_test_outcome = 0; 452 } 453 454 void 455 tinytest_set_test_skipped_(void) 456 { 457 if (cur_test_outcome==OK) 458 cur_test_outcome = SKIP; 459 } 460 461