1 /* 2 * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 #include "../testutil.h" 11 #include "output.h" 12 #include "tu_local.h" 13 14 #include <string.h> 15 #include <assert.h> 16 17 #include "internal/nelem.h" 18 #include <openssl/bio.h> 19 20 #include "platform.h" /* From libapps */ 21 22 #if defined(_WIN32) && !defined(__BORLANDC__) 23 # define strdup _strdup 24 #endif 25 26 27 /* 28 * Declares the structures needed to register each test case function. 29 */ 30 typedef struct test_info { 31 const char *test_case_name; 32 int (*test_fn) (void); 33 int (*param_test_fn)(int idx); 34 int num; 35 36 /* flags */ 37 int subtest:1; 38 } TEST_INFO; 39 40 static TEST_INFO all_tests[1024]; 41 static int num_tests = 0; 42 static int show_list = 0; 43 static int single_test = -1; 44 static int single_iter = -1; 45 static int level = 0; 46 static int seed = 0; 47 static int rand_order = 0; 48 49 /* 50 * A parameterised test runs a loop of test cases. 51 * |num_test_cases| counts the total number of non-subtest test cases 52 * across all tests. 53 */ 54 static int num_test_cases = 0; 55 56 static int process_shared_options(void); 57 58 59 void add_test(const char *test_case_name, int (*test_fn) (void)) 60 { 61 assert(num_tests != OSSL_NELEM(all_tests)); 62 all_tests[num_tests].test_case_name = test_case_name; 63 all_tests[num_tests].test_fn = test_fn; 64 all_tests[num_tests].num = -1; 65 ++num_tests; 66 ++num_test_cases; 67 } 68 69 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), 70 int num, int subtest) 71 { 72 assert(num_tests != OSSL_NELEM(all_tests)); 73 all_tests[num_tests].test_case_name = test_case_name; 74 all_tests[num_tests].param_test_fn = test_fn; 75 all_tests[num_tests].num = num; 76 all_tests[num_tests].subtest = subtest; 77 ++num_tests; 78 if (subtest) 79 ++num_test_cases; 80 else 81 num_test_cases += num; 82 } 83 84 static int gcd(int a, int b) 85 { 86 while (b != 0) { 87 int t = b; 88 b = a % b; 89 a = t; 90 } 91 return a; 92 } 93 94 static void set_seed(int s) 95 { 96 seed = s; 97 if (seed <= 0) 98 seed = (int)time(NULL); 99 test_random_seed(seed); 100 } 101 102 103 int setup_test_framework(int argc, char *argv[]) 104 { 105 char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER"); 106 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); 107 108 if (TAP_levels != NULL) 109 level = 4 * atoi(TAP_levels); 110 test_adjust_streams_tap_level(level); 111 if (test_seed != NULL) { 112 rand_order = 1; 113 set_seed(atoi(test_seed)); 114 } else { 115 set_seed(0); 116 } 117 118 #if defined(OPENSSL_SYS_VMS) && defined(__DECC) 119 argv = copy_argv(&argc, argv); 120 #elif defined(_WIN32) 121 /* 122 * Replace argv[] with UTF-8 encoded strings. 123 */ 124 win32_utf8argv(&argc, &argv); 125 #endif 126 127 if (!opt_init(argc, argv, test_get_options())) 128 return 0; 129 return 1; 130 } 131 132 133 /* 134 * This can only be called after setup() has run, since num_tests and 135 * all_tests[] are setup at this point 136 */ 137 static int check_single_test_params(char *name, char *testname, char *itname) 138 { 139 if (name != NULL) { 140 int i; 141 for (i = 0; i < num_tests; ++i) { 142 if (strcmp(name, all_tests[i].test_case_name) == 0) { 143 single_test = 1 + i; 144 break; 145 } 146 } 147 if (i >= num_tests) 148 single_test = atoi(name); 149 } 150 151 152 /* if only iteration is specified, assume we want the first test */ 153 if (single_test == -1 && single_iter != -1) 154 single_test = 1; 155 156 if (single_test != -1) { 157 if (single_test < 1 || single_test > num_tests) { 158 test_printf_stderr("Invalid -%s value " 159 "(Value must be a valid test name OR a value between %d..%d)\n", 160 testname, 1, num_tests); 161 return 0; 162 } 163 } 164 if (single_iter != -1) { 165 if (all_tests[single_test - 1].num == -1) { 166 test_printf_stderr("-%s option is not valid for test %d:%s\n", 167 itname, 168 single_test, 169 all_tests[single_test - 1].test_case_name); 170 return 0; 171 } else if (single_iter < 1 172 || single_iter > all_tests[single_test - 1].num) { 173 test_printf_stderr("Invalid -%s value for test %d:%s\t" 174 "(Value must be in the range %d..%d)\n", 175 itname, single_test, 176 all_tests[single_test - 1].test_case_name, 177 1, all_tests[single_test - 1].num); 178 return 0; 179 } 180 } 181 return 1; 182 } 183 184 static int process_shared_options(void) 185 { 186 OPTION_CHOICE_DEFAULT o; 187 int value; 188 int ret = -1; 189 char *flag_test = ""; 190 char *flag_iter = ""; 191 char *testname = NULL; 192 193 opt_begin(); 194 while ((o = opt_next()) != OPT_EOF) { 195 switch (o) { 196 /* Ignore any test options at this level */ 197 default: 198 break; 199 case OPT_ERR: 200 return ret; 201 case OPT_TEST_HELP: 202 opt_help(test_get_options()); 203 return 0; 204 case OPT_TEST_LIST: 205 show_list = 1; 206 break; 207 case OPT_TEST_SINGLE: 208 flag_test = opt_flag(); 209 testname = opt_arg(); 210 break; 211 case OPT_TEST_ITERATION: 212 flag_iter = opt_flag(); 213 if (!opt_int(opt_arg(), &single_iter)) 214 goto end; 215 break; 216 case OPT_TEST_INDENT: 217 if (!opt_int(opt_arg(), &value)) 218 goto end; 219 level = 4 * value; 220 test_adjust_streams_tap_level(level); 221 break; 222 case OPT_TEST_SEED: 223 if (!opt_int(opt_arg(), &value)) 224 goto end; 225 set_seed(value); 226 break; 227 } 228 } 229 if (!check_single_test_params(testname, flag_test, flag_iter)) 230 goto end; 231 ret = 1; 232 end: 233 return ret; 234 } 235 236 237 int pulldown_test_framework(int ret) 238 { 239 set_test_title(NULL); 240 return ret; 241 } 242 243 static void finalize(int success) 244 { 245 if (success) 246 ERR_clear_error(); 247 else 248 ERR_print_errors_cb(openssl_error_cb, NULL); 249 } 250 251 static char *test_title = NULL; 252 253 void set_test_title(const char *title) 254 { 255 free(test_title); 256 test_title = title == NULL ? NULL : strdup(title); 257 } 258 259 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict, 260 const char *description, ...) 261 { 262 va_list ap; 263 264 test_flush_stdout(); 265 test_flush_stderr(); 266 267 if (verdict == 0 && seed != 0) 268 test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); 269 test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok"); 270 va_start(ap, description); 271 test_vprintf_tapout(description, ap); 272 va_end(ap); 273 if (verdict == TEST_SKIP_CODE) 274 test_printf_tapout(" # skipped"); 275 test_printf_tapout("\n"); 276 test_flush_tapout(); 277 } 278 279 int run_tests(const char *test_prog_name) 280 { 281 int num_failed = 0; 282 int verdict = 1; 283 int ii, i, jj, j, jstep; 284 int test_case_count = 0; 285 int subtest_case_count = 0; 286 int permute[OSSL_NELEM(all_tests)]; 287 288 i = process_shared_options(); 289 if (i == 0) 290 return EXIT_SUCCESS; 291 if (i == -1) 292 return EXIT_FAILURE; 293 294 if (num_tests < 1) { 295 test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name); 296 } else if (show_list == 0 && single_test == -1) { 297 if (level > 0) { 298 test_printf_stdout("Subtest: %s\n", test_prog_name); 299 test_flush_stdout(); 300 } 301 test_printf_tapout("1..%d\n", num_test_cases); 302 } 303 304 test_flush_tapout(); 305 306 for (i = 0; i < num_tests; i++) 307 permute[i] = i; 308 if (rand_order != 0) 309 for (i = num_tests - 1; i >= 1; i--) { 310 j = test_random() % (1 + i); 311 ii = permute[j]; 312 permute[j] = permute[i]; 313 permute[i] = ii; 314 } 315 316 for (ii = 0; ii != num_tests; ++ii) { 317 i = permute[ii]; 318 319 if (single_test != -1 && ((i+1) != single_test)) { 320 continue; 321 } 322 else if (show_list) { 323 if (all_tests[i].num != -1) { 324 test_printf_tapout("%d - %s (%d..%d)\n", ii + 1, 325 all_tests[i].test_case_name, 1, 326 all_tests[i].num); 327 } else { 328 test_printf_tapout("%d - %s\n", ii + 1, 329 all_tests[i].test_case_name); 330 } 331 test_flush_tapout(); 332 } else if (all_tests[i].num == -1) { 333 set_test_title(all_tests[i].test_case_name); 334 ERR_clear_error(); 335 verdict = all_tests[i].test_fn(); 336 finalize(verdict != 0); 337 test_verdict(verdict, "%d - %s", test_case_count + 1, test_title); 338 if (verdict == 0) 339 num_failed++; 340 test_case_count++; 341 } else { 342 verdict = TEST_SKIP_CODE; 343 set_test_title(all_tests[i].test_case_name); 344 if (all_tests[i].subtest) { 345 level += 4; 346 test_adjust_streams_tap_level(level); 347 if (single_iter == -1) { 348 test_printf_stdout("Subtest: %s\n", test_title); 349 test_printf_tapout("%d..%d\n", 1, all_tests[i].num); 350 test_flush_stdout(); 351 test_flush_tapout(); 352 } 353 } 354 355 j = -1; 356 if (rand_order == 0 || all_tests[i].num < 3) 357 jstep = 1; 358 else 359 do 360 jstep = test_random() % all_tests[i].num; 361 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1); 362 363 for (jj = 0; jj < all_tests[i].num; jj++) { 364 int v; 365 366 j = (j + jstep) % all_tests[i].num; 367 if (single_iter != -1 && ((jj + 1) != single_iter)) 368 continue; 369 ERR_clear_error(); 370 v = all_tests[i].param_test_fn(j); 371 372 if (v == 0) { 373 verdict = 0; 374 } else if (v != TEST_SKIP_CODE && verdict != 0) { 375 verdict = 1; 376 } 377 378 finalize(v != 0); 379 380 if (all_tests[i].subtest) 381 test_verdict(v, "%d - iteration %d", 382 subtest_case_count + 1, j + 1); 383 else 384 test_verdict(v, "%d - %s - iteration %d", 385 test_case_count + subtest_case_count + 1, 386 test_title, j + 1); 387 subtest_case_count++; 388 } 389 390 if (all_tests[i].subtest) { 391 level -= 4; 392 test_adjust_streams_tap_level(level); 393 } 394 if (verdict == 0) 395 ++num_failed; 396 if (all_tests[i].num == -1 || all_tests[i].subtest) 397 test_verdict(verdict, "%d - %s", test_case_count + 1, 398 all_tests[i].test_case_name); 399 test_case_count++; 400 } 401 } 402 if (num_failed != 0) 403 return EXIT_FAILURE; 404 return EXIT_SUCCESS; 405 } 406 407 /* 408 * Glue an array of strings together and return it as an allocated string. 409 * Optionally return the whole length of this string in |out_len| 410 */ 411 char *glue_strings(const char *list[], size_t *out_len) 412 { 413 size_t len = 0; 414 char *p, *ret; 415 int i; 416 417 for (i = 0; list[i] != NULL; i++) 418 len += strlen(list[i]); 419 420 if (out_len != NULL) 421 *out_len = len; 422 423 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1))) 424 return NULL; 425 426 for (i = 0; list[i] != NULL; i++) 427 p += strlen(strcpy(p, list[i])); 428 429 return ret; 430 } 431 432 char *test_mk_file_path(const char *dir, const char *file) 433 { 434 # ifndef OPENSSL_SYS_VMS 435 const char *sep = "/"; 436 # else 437 const char *sep = ""; 438 char *dir_end; 439 char dir_end_sep; 440 # endif 441 size_t dirlen = dir != NULL ? strlen(dir) : 0; 442 size_t len = dirlen + strlen(sep) + strlen(file) + 1; 443 char *full_file = OPENSSL_zalloc(len); 444 445 if (full_file != NULL) { 446 if (dir != NULL && dirlen > 0) { 447 OPENSSL_strlcpy(full_file, dir, len); 448 # ifdef OPENSSL_SYS_VMS 449 /* 450 * If |file| contains a directory spec, we need to do some 451 * careful merging. 452 * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become 453 * "vol:[dir.dir.certs]sm2-root.crt" 454 */ 455 dir_end = &full_file[strlen(full_file) - 1]; 456 dir_end_sep = *dir_end; 457 if ((dir_end_sep == ']' || dir_end_sep == '>') 458 && (file[0] == '[' || file[0] == '<')) { 459 file++; 460 if (file[0] == '.') 461 *dir_end = '\0'; 462 else 463 *dir_end = '.'; 464 } 465 #else 466 OPENSSL_strlcat(full_file, sep, len); 467 #endif 468 } 469 OPENSSL_strlcat(full_file, file, len); 470 } 471 472 return full_file; 473 } 474