1 /* 2 * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 #include "../testutil.h" 11 #include "output.h" 12 #include "tu_local.h" 13 14 #include <string.h> 15 #include <assert.h> 16 17 #include "internal/nelem.h" 18 #include <openssl/bio.h> 19 20 #include "platform.h" /* From libapps */ 21 22 #if defined(_WIN32) && !defined(__BORLANDC__) 23 # define strdup _strdup 24 #endif 25 26 27 /* 28 * Declares the structures needed to register each test case function. 29 */ 30 typedef struct test_info { 31 const char *test_case_name; 32 int (*test_fn) (void); 33 int (*param_test_fn)(int idx); 34 int num; 35 36 /* flags */ 37 int subtest:1; 38 } TEST_INFO; 39 40 static TEST_INFO all_tests[1024]; 41 static int num_tests = 0; 42 static int show_list = 0; 43 static int single_test = -1; 44 static int single_iter = -1; 45 static int level = 0; 46 static int seed = 0; 47 static int rand_order = 0; 48 49 /* 50 * A parameterised test runs a loop of test cases. 51 * |num_test_cases| counts the total number of non-subtest test cases 52 * across all tests. 53 */ 54 static int num_test_cases = 0; 55 56 static int process_shared_options(void); 57 58 59 void add_test(const char *test_case_name, int (*test_fn) (void)) 60 { 61 assert(num_tests != OSSL_NELEM(all_tests)); 62 all_tests[num_tests].test_case_name = test_case_name; 63 all_tests[num_tests].test_fn = test_fn; 64 all_tests[num_tests].num = -1; 65 ++num_tests; 66 ++num_test_cases; 67 } 68 69 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), 70 int num, int subtest) 71 { 72 assert(num_tests != OSSL_NELEM(all_tests)); 73 all_tests[num_tests].test_case_name = test_case_name; 74 all_tests[num_tests].param_test_fn = test_fn; 75 all_tests[num_tests].num = num; 76 all_tests[num_tests].subtest = subtest; 77 ++num_tests; 78 if (subtest) 79 ++num_test_cases; 80 else 81 num_test_cases += num; 82 } 83 84 static int gcd(int a, int b) 85 { 86 while (b != 0) { 87 int t = b; 88 b = a % b; 89 a = t; 90 } 91 return a; 92 } 93 94 static void set_seed(int s) 95 { 96 seed = s; 97 if (seed <= 0) 98 seed = (int)time(NULL); 99 test_random_seed(seed); 100 } 101 102 103 int setup_test_framework(int argc, char *argv[]) 104 { 105 char *test_rand_order = getenv("OPENSSL_TEST_RAND_ORDER"); 106 char *test_rand_seed = getenv("OPENSSL_TEST_RAND_SEED"); 107 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); 108 109 if (TAP_levels != NULL) 110 level = 4 * atoi(TAP_levels); 111 test_adjust_streams_tap_level(level); 112 if (test_rand_order != NULL) { 113 rand_order = 1; 114 set_seed(atoi(test_rand_order)); 115 } else if (test_rand_seed != NULL) { 116 set_seed(atoi(test_rand_seed)); 117 } else { 118 set_seed(0); 119 } 120 121 #if defined(OPENSSL_SYS_VMS) && defined(__DECC) 122 argv = copy_argv(&argc, argv); 123 #elif defined(_WIN32) 124 /* 125 * Replace argv[] with UTF-8 encoded strings. 126 */ 127 win32_utf8argv(&argc, &argv); 128 #endif 129 130 if (!opt_init(argc, argv, test_get_options())) 131 return 0; 132 return 1; 133 } 134 135 136 /* 137 * This can only be called after setup() has run, since num_tests and 138 * all_tests[] are setup at this point 139 */ 140 static int check_single_test_params(char *name, char *testname, char *itname) 141 { 142 if (name != NULL) { 143 int i; 144 for (i = 0; i < num_tests; ++i) { 145 if (strcmp(name, all_tests[i].test_case_name) == 0) { 146 single_test = 1 + i; 147 break; 148 } 149 } 150 if (i >= num_tests) 151 single_test = atoi(name); 152 } 153 154 155 /* if only iteration is specified, assume we want the first test */ 156 if (single_test == -1 && single_iter != -1) 157 single_test = 1; 158 159 if (single_test != -1) { 160 if (single_test < 1 || single_test > num_tests) { 161 test_printf_stderr("Invalid -%s value " 162 "(Value must be a valid test name OR a value between %d..%d)\n", 163 testname, 1, num_tests); 164 return 0; 165 } 166 } 167 if (single_iter != -1) { 168 if (all_tests[single_test - 1].num == -1) { 169 test_printf_stderr("-%s option is not valid for test %d:%s\n", 170 itname, 171 single_test, 172 all_tests[single_test - 1].test_case_name); 173 return 0; 174 } else if (single_iter < 1 175 || single_iter > all_tests[single_test - 1].num) { 176 test_printf_stderr("Invalid -%s value for test %d:%s\t" 177 "(Value must be in the range %d..%d)\n", 178 itname, single_test, 179 all_tests[single_test - 1].test_case_name, 180 1, all_tests[single_test - 1].num); 181 return 0; 182 } 183 } 184 return 1; 185 } 186 187 static int process_shared_options(void) 188 { 189 OPTION_CHOICE_DEFAULT o; 190 int value; 191 int ret = -1; 192 char *flag_test = ""; 193 char *flag_iter = ""; 194 char *testname = NULL; 195 196 opt_begin(); 197 while ((o = opt_next()) != OPT_EOF) { 198 switch (o) { 199 /* Ignore any test options at this level */ 200 default: 201 break; 202 case OPT_ERR: 203 return ret; 204 case OPT_TEST_HELP: 205 opt_help(test_get_options()); 206 return 0; 207 case OPT_TEST_LIST: 208 show_list = 1; 209 break; 210 case OPT_TEST_SINGLE: 211 flag_test = opt_flag(); 212 testname = opt_arg(); 213 break; 214 case OPT_TEST_ITERATION: 215 flag_iter = opt_flag(); 216 if (!opt_int(opt_arg(), &single_iter)) 217 goto end; 218 break; 219 case OPT_TEST_INDENT: 220 if (!opt_int(opt_arg(), &value)) 221 goto end; 222 level = 4 * value; 223 test_adjust_streams_tap_level(level); 224 break; 225 case OPT_TEST_SEED: 226 if (!opt_int(opt_arg(), &value)) 227 goto end; 228 set_seed(value); 229 break; 230 } 231 } 232 if (!check_single_test_params(testname, flag_test, flag_iter)) 233 goto end; 234 ret = 1; 235 end: 236 return ret; 237 } 238 239 240 int pulldown_test_framework(int ret) 241 { 242 set_test_title(NULL); 243 return ret; 244 } 245 246 static void finalize(int success) 247 { 248 if (success) 249 ERR_clear_error(); 250 else 251 ERR_print_errors_cb(openssl_error_cb, NULL); 252 } 253 254 static char *test_title = NULL; 255 256 void set_test_title(const char *title) 257 { 258 free(test_title); 259 test_title = title == NULL ? NULL : strdup(title); 260 } 261 262 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict, 263 const char *description, ...) 264 { 265 va_list ap; 266 267 test_flush_stdout(); 268 test_flush_stderr(); 269 270 if (verdict == 0) { 271 if (rand_order) 272 test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); 273 else 274 test_printf_tapout("# OPENSSL_TEST_RAND_SEED=%d\n", seed); 275 } 276 test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok"); 277 va_start(ap, description); 278 test_vprintf_tapout(description, ap); 279 va_end(ap); 280 if (verdict == TEST_SKIP_CODE) 281 test_printf_tapout(" # skipped"); 282 test_printf_tapout("\n"); 283 test_flush_tapout(); 284 } 285 286 int run_tests(const char *test_prog_name) 287 { 288 int num_failed = 0; 289 int verdict = 1; 290 int ii, i, jj, j, jstep; 291 int test_case_count = 0; 292 int subtest_case_count = 0; 293 int permute[OSSL_NELEM(all_tests)]; 294 295 i = process_shared_options(); 296 if (i == 0) 297 return EXIT_SUCCESS; 298 if (i == -1) 299 return EXIT_FAILURE; 300 301 if (num_tests < 1) { 302 test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name); 303 } else if (show_list == 0 && single_test == -1) { 304 if (level > 0) { 305 test_printf_stdout("Subtest: %s\n", test_prog_name); 306 test_flush_stdout(); 307 } 308 test_printf_tapout("1..%d\n", num_test_cases); 309 } 310 311 test_flush_tapout(); 312 313 for (i = 0; i < num_tests; i++) 314 permute[i] = i; 315 if (rand_order != 0) 316 for (i = num_tests - 1; i >= 1; i--) { 317 j = test_random() % (1 + i); 318 ii = permute[j]; 319 permute[j] = permute[i]; 320 permute[i] = ii; 321 } 322 323 for (ii = 0; ii != num_tests; ++ii) { 324 i = permute[ii]; 325 326 if (single_test != -1 && ((i+1) != single_test)) { 327 continue; 328 } 329 else if (show_list) { 330 if (all_tests[i].num != -1) { 331 test_printf_tapout("%d - %s (%d..%d)\n", ii + 1, 332 all_tests[i].test_case_name, 1, 333 all_tests[i].num); 334 } else { 335 test_printf_tapout("%d - %s\n", ii + 1, 336 all_tests[i].test_case_name); 337 } 338 test_flush_tapout(); 339 } else if (all_tests[i].num == -1) { 340 set_test_title(all_tests[i].test_case_name); 341 ERR_clear_error(); 342 verdict = all_tests[i].test_fn(); 343 finalize(verdict != 0); 344 test_verdict(verdict, "%d - %s", test_case_count + 1, test_title); 345 if (verdict == 0) 346 num_failed++; 347 test_case_count++; 348 } else { 349 verdict = TEST_SKIP_CODE; 350 set_test_title(all_tests[i].test_case_name); 351 if (all_tests[i].subtest) { 352 level += 4; 353 test_adjust_streams_tap_level(level); 354 if (single_iter == -1) { 355 test_printf_stdout("Subtest: %s\n", test_title); 356 test_printf_tapout("%d..%d\n", 1, all_tests[i].num); 357 test_flush_stdout(); 358 test_flush_tapout(); 359 } 360 } 361 362 j = -1; 363 if (rand_order == 0 || all_tests[i].num < 3) 364 jstep = 1; 365 else 366 do 367 jstep = test_random() % all_tests[i].num; 368 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1); 369 370 for (jj = 0; jj < all_tests[i].num; jj++) { 371 int v; 372 373 j = (j + jstep) % all_tests[i].num; 374 if (single_iter != -1 && ((jj + 1) != single_iter)) 375 continue; 376 ERR_clear_error(); 377 v = all_tests[i].param_test_fn(j); 378 379 if (v == 0) { 380 verdict = 0; 381 } else if (v != TEST_SKIP_CODE && verdict != 0) { 382 verdict = 1; 383 } 384 385 finalize(v != 0); 386 387 if (all_tests[i].subtest) 388 test_verdict(v, "%d - iteration %d", 389 subtest_case_count + 1, j + 1); 390 else 391 test_verdict(v, "%d - %s - iteration %d", 392 test_case_count + subtest_case_count + 1, 393 test_title, j + 1); 394 subtest_case_count++; 395 } 396 397 if (all_tests[i].subtest) { 398 level -= 4; 399 test_adjust_streams_tap_level(level); 400 } 401 if (verdict == 0) 402 ++num_failed; 403 if (all_tests[i].num == -1 || all_tests[i].subtest) 404 test_verdict(verdict, "%d - %s", test_case_count + 1, 405 all_tests[i].test_case_name); 406 test_case_count++; 407 } 408 } 409 if (num_failed != 0) 410 return EXIT_FAILURE; 411 return EXIT_SUCCESS; 412 } 413 414 /* 415 * Glue an array of strings together and return it as an allocated string. 416 * Optionally return the whole length of this string in |out_len| 417 */ 418 char *glue_strings(const char *list[], size_t *out_len) 419 { 420 size_t len = 0; 421 char *p, *ret; 422 int i; 423 424 for (i = 0; list[i] != NULL; i++) 425 len += strlen(list[i]); 426 427 if (out_len != NULL) 428 *out_len = len; 429 430 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1))) 431 return NULL; 432 433 for (i = 0; list[i] != NULL; i++) 434 p += strlen(strcpy(p, list[i])); 435 436 return ret; 437 } 438 439 char *test_mk_file_path(const char *dir, const char *file) 440 { 441 # ifndef OPENSSL_SYS_VMS 442 const char *sep = "/"; 443 # else 444 const char *sep = ""; 445 char *dir_end; 446 char dir_end_sep; 447 # endif 448 size_t dirlen = dir != NULL ? strlen(dir) : 0; 449 size_t len = dirlen + strlen(sep) + strlen(file) + 1; 450 char *full_file = OPENSSL_zalloc(len); 451 452 if (full_file != NULL) { 453 if (dir != NULL && dirlen > 0) { 454 OPENSSL_strlcpy(full_file, dir, len); 455 # ifdef OPENSSL_SYS_VMS 456 /* 457 * If |file| contains a directory spec, we need to do some 458 * careful merging. 459 * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become 460 * "vol:[dir.dir.certs]sm2-root.crt" 461 */ 462 dir_end = &full_file[strlen(full_file) - 1]; 463 dir_end_sep = *dir_end; 464 if ((dir_end_sep == ']' || dir_end_sep == '>') 465 && (file[0] == '[' || file[0] == '<')) { 466 file++; 467 if (file[0] == '.') 468 *dir_end = '\0'; 469 else 470 *dir_end = '.'; 471 } 472 #else 473 OPENSSL_strlcat(full_file, sep, len); 474 #endif 475 } 476 OPENSSL_strlcat(full_file, file, len); 477 } 478 479 return full_file; 480 } 481