1 /* 2 * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 #include "../testutil.h" 11 #include "output.h" 12 #include "tu_local.h" 13 14 #include <string.h> 15 #include <assert.h> 16 17 #include "internal/nelem.h" 18 #include <openssl/bio.h> 19 20 #include "platform.h" /* From libapps */ 21 22 #if defined(_WIN32) && !defined(__BORLANDC__) 23 #define strdup _strdup 24 #endif 25 26 /* 27 * Declares the structures needed to register each test case function. 28 */ 29 typedef struct test_info { 30 const char *test_case_name; 31 int (*test_fn)(void); 32 int (*param_test_fn)(int idx); 33 int num; 34 35 /* flags */ 36 int subtest : 1; 37 } TEST_INFO; 38 39 static TEST_INFO all_tests[1024]; 40 static int num_tests = 0; 41 static int show_list = 0; 42 static int single_test = -1; 43 static int single_iter = -1; 44 static int level = 0; 45 static int seed = 0; 46 static int rand_order = 0; 47 48 /* 49 * A parameterised test runs a loop of test cases. 50 * |num_test_cases| counts the total number of non-subtest test cases 51 * across all tests. 52 */ 53 static int num_test_cases = 0; 54 55 static int process_shared_options(void); 56 57 void add_test(const char *test_case_name, int (*test_fn)(void)) 58 { 59 assert(num_tests != OSSL_NELEM(all_tests)); 60 all_tests[num_tests].test_case_name = test_case_name; 61 all_tests[num_tests].test_fn = test_fn; 62 all_tests[num_tests].num = -1; 63 ++num_tests; 64 ++num_test_cases; 65 } 66 67 void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), 68 int num, int subtest) 69 { 70 assert(num_tests != OSSL_NELEM(all_tests)); 71 all_tests[num_tests].test_case_name = test_case_name; 72 all_tests[num_tests].param_test_fn = test_fn; 73 all_tests[num_tests].num = num; 74 all_tests[num_tests].subtest = subtest; 75 ++num_tests; 76 if (subtest) 77 ++num_test_cases; 78 else 79 num_test_cases += num; 80 } 81 82 static int gcd(int a, int b) 83 { 84 while (b != 0) { 85 int t = b; 86 b = a % b; 87 a = t; 88 } 89 return a; 90 } 91 92 static void set_seed(int s) 93 { 94 seed = s; 95 if (seed <= 0) 96 seed = (int)time(NULL); 97 test_random_seed(seed); 98 } 99 100 int setup_test_framework(int argc, char *argv[]) 101 { 102 char *test_rand_order = getenv("OPENSSL_TEST_RAND_ORDER"); 103 char *test_rand_seed = getenv("OPENSSL_TEST_RAND_SEED"); 104 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); 105 106 if (TAP_levels != NULL) 107 level = 4 * atoi(TAP_levels); 108 test_adjust_streams_tap_level(level); 109 if (test_rand_order != NULL) { 110 rand_order = 1; 111 set_seed(atoi(test_rand_order)); 112 } else if (test_rand_seed != NULL) { 113 set_seed(atoi(test_rand_seed)); 114 } else { 115 set_seed(0); 116 } 117 118 #if defined(OPENSSL_SYS_VMS) && defined(__DECC) 119 argv = copy_argv(&argc, argv); 120 #elif defined(_WIN32) 121 /* 122 * Replace argv[] with UTF-8 encoded strings. 123 */ 124 win32_utf8argv(&argc, &argv); 125 #endif 126 127 if (!opt_init(argc, argv, test_get_options())) 128 return 0; 129 return 1; 130 } 131 132 /* 133 * This can only be called after setup() has run, since num_tests and 134 * all_tests[] are setup at this point 135 */ 136 static int check_single_test_params(char *name, char *testname, char *itname) 137 { 138 if (name != NULL) { 139 int i; 140 for (i = 0; i < num_tests; ++i) { 141 if (strcmp(name, all_tests[i].test_case_name) == 0) { 142 single_test = 1 + i; 143 break; 144 } 145 } 146 if (i >= num_tests) 147 single_test = atoi(name); 148 } 149 150 /* if only iteration is specified, assume we want the first test */ 151 if (single_test == -1 && single_iter != -1) 152 single_test = 1; 153 154 if (single_test != -1) { 155 if (single_test < 1 || single_test > num_tests) { 156 test_printf_stderr("Invalid -%s value " 157 "(Value must be a valid test name OR a value between %d..%d)\n", 158 testname, 1, num_tests); 159 return 0; 160 } 161 } 162 if (single_iter != -1) { 163 if (all_tests[single_test - 1].num == -1) { 164 test_printf_stderr("-%s option is not valid for test %d:%s\n", 165 itname, 166 single_test, 167 all_tests[single_test - 1].test_case_name); 168 return 0; 169 } else if (single_iter < 1 170 || single_iter > all_tests[single_test - 1].num) { 171 test_printf_stderr("Invalid -%s value for test %d:%s\t" 172 "(Value must be in the range %d..%d)\n", 173 itname, single_test, 174 all_tests[single_test - 1].test_case_name, 175 1, all_tests[single_test - 1].num); 176 return 0; 177 } 178 } 179 return 1; 180 } 181 182 static int process_shared_options(void) 183 { 184 OPTION_CHOICE_DEFAULT o; 185 int value; 186 int ret = -1; 187 char *flag_test = ""; 188 char *flag_iter = ""; 189 char *testname = NULL; 190 191 opt_begin(); 192 while ((o = opt_next()) != OPT_EOF) { 193 switch (o) { 194 /* Ignore any test options at this level */ 195 default: 196 break; 197 case OPT_ERR: 198 return ret; 199 case OPT_TEST_HELP: 200 opt_help(test_get_options()); 201 return 0; 202 case OPT_TEST_LIST: 203 show_list = 1; 204 break; 205 case OPT_TEST_SINGLE: 206 flag_test = opt_flag(); 207 testname = opt_arg(); 208 break; 209 case OPT_TEST_ITERATION: 210 flag_iter = opt_flag(); 211 if (!opt_int(opt_arg(), &single_iter)) 212 goto end; 213 break; 214 case OPT_TEST_INDENT: 215 if (!opt_int(opt_arg(), &value)) 216 goto end; 217 level = 4 * value; 218 test_adjust_streams_tap_level(level); 219 break; 220 case OPT_TEST_SEED: 221 if (!opt_int(opt_arg(), &value)) 222 goto end; 223 set_seed(value); 224 break; 225 } 226 } 227 if (!check_single_test_params(testname, flag_test, flag_iter)) 228 goto end; 229 ret = 1; 230 end: 231 return ret; 232 } 233 234 int pulldown_test_framework(int ret) 235 { 236 set_test_title(NULL); 237 return ret; 238 } 239 240 static void finalize(int success) 241 { 242 if (success) 243 ERR_clear_error(); 244 else 245 ERR_print_errors_cb(openssl_error_cb, NULL); 246 } 247 248 static char *test_title = NULL; 249 250 void set_test_title(const char *title) 251 { 252 free(test_title); 253 test_title = title == NULL ? NULL : strdup(title); 254 } 255 256 PRINTF_FORMAT(2, 3) 257 static void test_verdict(int verdict, 258 const char *description, ...) 259 { 260 va_list ap; 261 262 test_flush_stdout(); 263 test_flush_stderr(); 264 265 if (verdict == 0) { 266 if (rand_order) 267 test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); 268 else 269 test_printf_tapout("# OPENSSL_TEST_RAND_SEED=%d\n", seed); 270 } 271 test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok"); 272 va_start(ap, description); 273 test_vprintf_tapout(description, ap); 274 va_end(ap); 275 if (verdict == TEST_SKIP_CODE) 276 test_printf_tapout(" # skipped"); 277 test_printf_tapout("\n"); 278 test_flush_tapout(); 279 } 280 281 int run_tests(const char *test_prog_name) 282 { 283 int num_failed = 0; 284 int verdict = 1; 285 int ii, i, jj, j, jstep; 286 int test_case_count = 0; 287 int subtest_case_count = 0; 288 int permute[OSSL_NELEM(all_tests)]; 289 290 i = process_shared_options(); 291 if (i == 0) 292 return EXIT_SUCCESS; 293 if (i == -1) 294 return EXIT_FAILURE; 295 296 if (num_tests < 1) { 297 test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name); 298 } else if (show_list == 0 && single_test == -1) { 299 if (level > 0) { 300 test_printf_stdout("Subtest: %s\n", test_prog_name); 301 test_flush_stdout(); 302 } 303 test_printf_tapout("1..%d\n", num_test_cases); 304 } 305 306 test_flush_tapout(); 307 308 for (i = 0; i < num_tests; i++) 309 permute[i] = i; 310 if (rand_order != 0) 311 for (i = num_tests - 1; i >= 1; i--) { 312 j = test_random() % (1 + i); 313 ii = permute[j]; 314 permute[j] = permute[i]; 315 permute[i] = ii; 316 } 317 318 for (ii = 0; ii != num_tests; ++ii) { 319 i = permute[ii]; 320 321 if (single_test != -1 && ((i + 1) != single_test)) { 322 continue; 323 } else if (show_list) { 324 if (all_tests[i].num != -1) { 325 test_printf_tapout("%d - %s (%d..%d)\n", ii + 1, 326 all_tests[i].test_case_name, 1, 327 all_tests[i].num); 328 } else { 329 test_printf_tapout("%d - %s\n", ii + 1, 330 all_tests[i].test_case_name); 331 } 332 test_flush_tapout(); 333 } else if (all_tests[i].num == -1) { 334 set_test_title(all_tests[i].test_case_name); 335 ERR_clear_error(); 336 verdict = all_tests[i].test_fn(); 337 finalize(verdict != 0); 338 test_verdict(verdict, "%d - %s", test_case_count + 1, test_title); 339 if (verdict == 0) 340 num_failed++; 341 test_case_count++; 342 } else { 343 verdict = TEST_SKIP_CODE; 344 set_test_title(all_tests[i].test_case_name); 345 if (all_tests[i].subtest) { 346 level += 4; 347 test_adjust_streams_tap_level(level); 348 if (single_iter == -1) { 349 test_printf_stdout("Subtest: %s\n", test_title); 350 test_printf_tapout("%d..%d\n", 1, all_tests[i].num); 351 test_flush_stdout(); 352 test_flush_tapout(); 353 } 354 } 355 356 j = -1; 357 if (rand_order == 0 || all_tests[i].num < 3) 358 jstep = 1; 359 else 360 do 361 jstep = test_random() % all_tests[i].num; 362 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1); 363 364 for (jj = 0; jj < all_tests[i].num; jj++) { 365 int v; 366 367 j = (j + jstep) % all_tests[i].num; 368 if (single_iter != -1 && ((jj + 1) != single_iter)) 369 continue; 370 ERR_clear_error(); 371 v = all_tests[i].param_test_fn(j); 372 373 if (v == 0) { 374 verdict = 0; 375 } else if (v != TEST_SKIP_CODE && verdict != 0) { 376 verdict = 1; 377 } 378 379 finalize(v != 0); 380 381 if (all_tests[i].subtest) 382 test_verdict(v, "%d - iteration %d", 383 subtest_case_count + 1, j + 1); 384 else 385 test_verdict(v, "%d - %s - iteration %d", 386 test_case_count + subtest_case_count + 1, 387 test_title, j + 1); 388 subtest_case_count++; 389 } 390 391 if (all_tests[i].subtest) { 392 level -= 4; 393 test_adjust_streams_tap_level(level); 394 } 395 if (verdict == 0) 396 ++num_failed; 397 if (all_tests[i].num == -1 || all_tests[i].subtest) 398 test_verdict(verdict, "%d - %s", test_case_count + 1, 399 all_tests[i].test_case_name); 400 test_case_count++; 401 } 402 } 403 if (num_failed != 0) 404 return EXIT_FAILURE; 405 return EXIT_SUCCESS; 406 } 407 408 /* 409 * Glue an array of strings together and return it as an allocated string. 410 * Optionally return the whole length of this string in |out_len| 411 */ 412 char *glue_strings(const char *list[], size_t *out_len) 413 { 414 size_t len = 0; 415 char *p, *ret; 416 int i; 417 418 for (i = 0; list[i] != NULL; i++) 419 len += strlen(list[i]); 420 421 if (out_len != NULL) 422 *out_len = len; 423 424 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1))) 425 return NULL; 426 427 for (i = 0; list[i] != NULL; i++) 428 p += strlen(strcpy(p, list[i])); 429 430 return ret; 431 } 432 433 char *test_mk_file_path(const char *dir, const char *file) 434 { 435 #ifndef OPENSSL_SYS_VMS 436 const char *sep = "/"; 437 #else 438 const char *sep = ""; 439 char *dir_end; 440 char dir_end_sep; 441 #endif 442 size_t dirlen = dir != NULL ? strlen(dir) : 0; 443 size_t len = dirlen + strlen(sep) + strlen(file) + 1; 444 char *full_file = OPENSSL_zalloc(len); 445 446 if (full_file != NULL) { 447 if (dir != NULL && dirlen > 0) { 448 OPENSSL_strlcpy(full_file, dir, len); 449 #ifdef OPENSSL_SYS_VMS 450 /* 451 * If |file| contains a directory spec, we need to do some 452 * careful merging. 453 * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become 454 * "vol:[dir.dir.certs]sm2-root.crt" 455 */ 456 dir_end = &full_file[strlen(full_file) - 1]; 457 dir_end_sep = *dir_end; 458 if ((dir_end_sep == ']' || dir_end_sep == '>') 459 && (file[0] == '[' || file[0] == '<')) { 460 file++; 461 if (file[0] == '.') 462 *dir_end = '\0'; 463 else 464 *dir_end = '.'; 465 } 466 #else 467 OPENSSL_strlcat(full_file, sep, len); 468 #endif 469 } 470 OPENSSL_strlcat(full_file, file, len); 471 } 472 473 return full_file; 474 } 475