1 /* $OpenBSD: test_helper.c,v 1.16 2026/03/06 06:57:33 dtucker Exp $ */
2 /*
3 * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18 /* Utility functions/framework for regress tests */
19
20 #include "includes.h"
21
22 #include <sys/types.h>
23 #include <sys/uio.h>
24 #include <sys/time.h>
25
26 #include <assert.h>
27 #include <fcntl.h>
28 #include <limits.h>
29 #include <math.h>
30 #include <signal.h>
31 #include <stdarg.h>
32 #include <stdint.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <time.h>
37 #include <unistd.h>
38
39 #ifdef WITH_OPENSSL
40 #include <openssl/bn.h>
41 #include <openssl/err.h>
42 #endif
43
44 #if defined(HAVE_STRNVIS) && defined(HAVE_VIS_H) && !defined(BROKEN_STRNVIS)
45 # include <vis.h>
46 #endif
47
48 #include "entropy.h"
49 #include "test_helper.h"
50 #include "atomicio.h"
51 #include "match.h"
52 #include "misc.h"
53 #include "xmalloc.h"
54
55 #define BENCH_FAST_DEADLINE 1
56 #define BENCH_NORMAL_DEADLINE 10
57 #define BENCH_SLOW_DEADLINE 60
58 #define BENCH_SAMPLES_ALLOC 8192
59 #define BENCH_COLUMN_WIDTH 40
60
61 #define MINIMUM(a, b) (((a) < (b)) ? (a) : (b))
62
63 #define TEST_CHECK_INT(r, pred) do { \
64 switch (pred) { \
65 case TEST_EQ: \
66 if (r == 0) \
67 return; \
68 break; \
69 case TEST_NE: \
70 if (r != 0) \
71 return; \
72 break; \
73 case TEST_LT: \
74 if (r < 0) \
75 return; \
76 break; \
77 case TEST_LE: \
78 if (r <= 0) \
79 return; \
80 break; \
81 case TEST_GT: \
82 if (r > 0) \
83 return; \
84 break; \
85 case TEST_GE: \
86 if (r >= 0) \
87 return; \
88 break; \
89 default: \
90 abort(); \
91 } \
92 } while (0)
93
94 #define TEST_CHECK(x1, x2, pred) do { \
95 switch (pred) { \
96 case TEST_EQ: \
97 if (x1 == x2) \
98 return; \
99 break; \
100 case TEST_NE: \
101 if (x1 != x2) \
102 return; \
103 break; \
104 case TEST_LT: \
105 if (x1 < x2) \
106 return; \
107 break; \
108 case TEST_LE: \
109 if (x1 <= x2) \
110 return; \
111 break; \
112 case TEST_GT: \
113 if (x1 > x2) \
114 return; \
115 break; \
116 case TEST_GE: \
117 if (x1 >= x2) \
118 return; \
119 break; \
120 default: \
121 abort(); \
122 } \
123 } while (0)
124
125 extern char *__progname;
126
127 static int verbose_mode = 0;
128 static int quiet_mode = 0;
129 static char *active_test_name = NULL;
130 static u_int test_number = 0;
131 static test_onerror_func_t *test_onerror = NULL;
132 static void *onerror_ctx = NULL;
133 static const char *data_dir = NULL;
134 static char subtest_info[512];
135 static int fast = 0;
136 static int slow = 0;
137 static int benchmark_detail_statistics = 0;
138
139 static int benchmark = 0;
140 static const char *bench_name = NULL;
141 static char *benchmark_pattern = NULL;
142 static struct timespec bench_start_time, bench_finish_time;
143 static struct timespec *bench_samples;
144 static int bench_skip, bench_nruns, bench_nalloc;
145 double bench_accum_secs;
146
147 int
main(int argc,char ** argv)148 main(int argc, char **argv)
149 {
150 int ch;
151
152 seed_rng();
153 #ifdef WITH_OPENSSL
154 ERR_load_crypto_strings();
155 #endif
156
157 /* Handle systems without __progname */
158 if (__progname == NULL) {
159 __progname = strrchr(argv[0], '/');
160 if (__progname == NULL || __progname[1] == '\0')
161 __progname = argv[0];
162 else
163 __progname++;
164 if ((__progname = strdup(__progname)) == NULL) {
165 fprintf(stderr, "strdup failed\n");
166 exit(1);
167 }
168 }
169
170 while ((ch = getopt(argc, argv, "O:bBFfvqd:")) != -1) {
171 switch (ch) {
172 case 'b':
173 benchmark = 1;
174 break;
175 case 'B':
176 benchmark = benchmark_detail_statistics = 1;
177 break;
178 case 'O':
179 benchmark_pattern = xstrdup(optarg);
180 break;
181 case 'F':
182 slow = 1;
183 break;
184 case 'f':
185 fast = 1;
186 break;
187 case 'd':
188 data_dir = optarg;
189 break;
190 case 'q':
191 verbose_mode = 0;
192 quiet_mode = 1;
193 break;
194 case 'v':
195 verbose_mode = 1;
196 quiet_mode = 0;
197 break;
198 default:
199 fprintf(stderr, "Unrecognised command line option\n");
200 fprintf(stderr, "Usage: %s [-vqfFbB] [-d data_dir] "
201 "[-O pattern]\n", __progname);
202 exit(1);
203 }
204 }
205 setvbuf(stdout, NULL, _IONBF, 0);
206 if (!quiet_mode)
207 printf("%s: ", __progname);
208 if (verbose_mode)
209 printf("\n");
210
211 if (benchmark)
212 benchmarks();
213 else
214 tests();
215
216 if (!quiet_mode && !benchmark)
217 printf(" %u tests ok\n", test_number);
218 return 0;
219 }
220
221 int
test_is_verbose(void)222 test_is_verbose(void)
223 {
224 return verbose_mode;
225 }
226
227 int
test_is_quiet(void)228 test_is_quiet(void)
229 {
230 return quiet_mode;
231 }
232
233 int
test_is_fast(void)234 test_is_fast(void)
235 {
236 return fast;
237 }
238
239 int
test_is_slow(void)240 test_is_slow(void)
241 {
242 return slow;
243 }
244
245 const char *
test_data_file(const char * name)246 test_data_file(const char *name)
247 {
248 static char ret[PATH_MAX];
249
250 if (data_dir != NULL)
251 snprintf(ret, sizeof(ret), "%s/%s", data_dir, name);
252 else
253 strlcpy(ret, name, sizeof(ret));
254 if (access(ret, F_OK) != 0) {
255 fprintf(stderr, "Cannot access data file %s: %s\n",
256 ret, strerror(errno));
257 exit(1);
258 }
259 return ret;
260 }
261
262 void
test_info(char * s,size_t len)263 test_info(char *s, size_t len)
264 {
265 snprintf(s, len, "In test %u: \"%s\"%s%s\n", test_number,
266 active_test_name == NULL ? "<none>" : active_test_name,
267 *subtest_info != '\0' ? " - " : "", subtest_info);
268 }
269
270 static void
siginfo(int unused)271 siginfo(int unused __attribute__((__unused__)))
272 {
273 char buf[256];
274
275 test_info(buf, sizeof(buf));
276 atomicio(vwrite, STDERR_FILENO, buf, strlen(buf));
277 }
278
279 void
test_start(const char * n)280 test_start(const char *n)
281 {
282 assert(active_test_name == NULL);
283 assert((active_test_name = strdup(n)) != NULL);
284 *subtest_info = '\0';
285 if (verbose_mode)
286 printf("test %u - \"%s\": ", test_number, active_test_name);
287 test_number++;
288 #ifdef SIGINFO
289 signal(SIGINFO, siginfo);
290 #endif
291 signal(SIGUSR1, siginfo);
292 }
293
294 void
set_onerror_func(test_onerror_func_t * f,void * ctx)295 set_onerror_func(test_onerror_func_t *f, void *ctx)
296 {
297 test_onerror = f;
298 onerror_ctx = ctx;
299 }
300
301 void
test_done(void)302 test_done(void)
303 {
304 *subtest_info = '\0';
305 assert(active_test_name != NULL);
306 free(active_test_name);
307 active_test_name = NULL;
308 if (verbose_mode)
309 printf("OK\n");
310 else if (!quiet_mode && !benchmark) {
311 printf(".");
312 fflush(stdout);
313 }
314 }
315
316 void
test_subtest_info(const char * fmt,...)317 test_subtest_info(const char *fmt, ...)
318 {
319 va_list ap;
320
321 va_start(ap, fmt);
322 vsnprintf(subtest_info, sizeof(subtest_info), fmt, ap);
323 va_end(ap);
324 }
325
326 int
test_is_benchmark(void)327 test_is_benchmark(void)
328 {
329 return benchmark;
330 }
331
332 void
ssl_err_check(const char * file,int line)333 ssl_err_check(const char *file, int line)
334 {
335 #ifdef WITH_OPENSSL
336 long openssl_error = ERR_get_error();
337
338 if (openssl_error == 0)
339 return;
340
341 fprintf(stderr, "\n%s:%d: uncaught OpenSSL error: %s",
342 file, line, ERR_error_string(openssl_error, NULL));
343 #else /* WITH_OPENSSL */
344 fprintf(stderr, "\n%s:%d: uncaught OpenSSL error ",
345 file, line);
346 #endif /* WITH_OPENSSL */
347 abort();
348 }
349
350 static const char *
pred_name(enum test_predicate p)351 pred_name(enum test_predicate p)
352 {
353 switch (p) {
354 case TEST_EQ:
355 return "EQ";
356 case TEST_NE:
357 return "NE";
358 case TEST_LT:
359 return "LT";
360 case TEST_LE:
361 return "LE";
362 case TEST_GT:
363 return "GT";
364 case TEST_GE:
365 return "GE";
366 default:
367 return "UNKNOWN";
368 }
369 }
370
371 static void
test_die(void)372 test_die(void)
373 {
374 if (test_onerror != NULL)
375 test_onerror(onerror_ctx);
376 abort();
377 }
378
379 static void
test_header(const char * file,int line,const char * a1,const char * a2,const char * name,enum test_predicate pred)380 test_header(const char *file, int line, const char *a1, const char *a2,
381 const char *name, enum test_predicate pred)
382 {
383 fprintf(stderr, "\n%s:%d test #%u \"%s\"%s%s\n",
384 file, line, test_number, active_test_name,
385 *subtest_info != '\0' ? " - " : "", subtest_info);
386 fprintf(stderr, "ASSERT_%s_%s(%s%s%s) failed:\n",
387 name, pred_name(pred), a1,
388 a2 != NULL ? ", " : "", a2 != NULL ? a2 : "");
389 }
390
391 #ifdef WITH_OPENSSL
392 void
assert_bignum(const char * file,int line,const char * a1,const char * a2,const BIGNUM * aa1,const BIGNUM * aa2,enum test_predicate pred)393 assert_bignum(const char *file, int line, const char *a1, const char *a2,
394 const BIGNUM *aa1, const BIGNUM *aa2, enum test_predicate pred)
395 {
396 int r = BN_cmp(aa1, aa2);
397
398 TEST_CHECK_INT(r, pred);
399 test_header(file, line, a1, a2, "BIGNUM", pred);
400 fprintf(stderr, "%12s = 0x%s\n", a1, BN_bn2hex(aa1));
401 fprintf(stderr, "%12s = 0x%s\n", a2, BN_bn2hex(aa2));
402 test_die();
403 }
404 #endif
405
406 void
assert_string(const char * file,int line,const char * a1,const char * a2,const char * aa1,const char * aa2,enum test_predicate pred)407 assert_string(const char *file, int line, const char *a1, const char *a2,
408 const char *aa1, const char *aa2, enum test_predicate pred)
409 {
410 int r;
411
412 /* Verify pointers are not NULL */
413 assert_ptr(file, line, a1, "NULL", aa1, NULL, TEST_NE);
414 assert_ptr(file, line, a2, "NULL", aa2, NULL, TEST_NE);
415
416 r = strcmp(aa1, aa2);
417 TEST_CHECK_INT(r, pred);
418 test_header(file, line, a1, a2, "STRING", pred);
419 fprintf(stderr, "%12s = %s (len %zu)\n", a1, aa1, strlen(aa1));
420 fprintf(stderr, "%12s = %s (len %zu)\n", a2, aa2, strlen(aa2));
421 test_die();
422 }
423
424 void
assert_mem(const char * file,int line,const char * a1,const char * a2,const void * aa1,const void * aa2,size_t l,enum test_predicate pred)425 assert_mem(const char *file, int line, const char *a1, const char *a2,
426 const void *aa1, const void *aa2, size_t l, enum test_predicate pred)
427 {
428 int r;
429 char *aa1_tohex = NULL;
430 char *aa2_tohex = NULL;
431
432 if (l == 0)
433 return;
434 /* If length is >0, then verify pointers are not NULL */
435 assert_ptr(file, line, a1, "NULL", aa1, NULL, TEST_NE);
436 assert_ptr(file, line, a2, "NULL", aa2, NULL, TEST_NE);
437
438 r = memcmp(aa1, aa2, l);
439 TEST_CHECK_INT(r, pred);
440 test_header(file, line, a1, a2, "STRING", pred);
441 aa1_tohex = tohex(aa1, MINIMUM(l, 256));
442 aa2_tohex = tohex(aa2, MINIMUM(l, 256));
443 fprintf(stderr, "%12s = %s (len %zu)\n", a1, aa1_tohex, l);
444 fprintf(stderr, "%12s = %s (len %zu)\n", a2, aa2_tohex, l);
445 free(aa1_tohex);
446 free(aa2_tohex);
447 test_die();
448 }
449
450 static int
memvalcmp(const uint8_t * s,u_char v,size_t l,size_t * where)451 memvalcmp(const uint8_t *s, u_char v, size_t l, size_t *where)
452 {
453 size_t i;
454
455 for (i = 0; i < l; i++) {
456 if (s[i] != v) {
457 *where = i;
458 return 1;
459 }
460 }
461 return 0;
462 }
463
464 void
assert_mem_filled(const char * file,int line,const char * a1,const void * aa1,u_char v,size_t l,enum test_predicate pred)465 assert_mem_filled(const char *file, int line, const char *a1,
466 const void *aa1, u_char v, size_t l, enum test_predicate pred)
467 {
468 size_t where = -1;
469 int r;
470 char tmp[64];
471 char *aa1_tohex = NULL;
472
473 if (l == 0)
474 return;
475 /* If length is >0, then verify the pointer is not NULL */
476 assert_ptr(file, line, a1, "NULL", aa1, NULL, TEST_NE);
477
478 r = memvalcmp(aa1, v, l, &where);
479 TEST_CHECK_INT(r, pred);
480 test_header(file, line, a1, NULL, "MEM_ZERO", pred);
481 aa1_tohex = tohex(aa1, MINIMUM(l, 20));
482 fprintf(stderr, "%20s = %s%s (len %zu)\n", a1,
483 aa1_tohex, l > 20 ? "..." : "", l);
484 free(aa1_tohex);
485 snprintf(tmp, sizeof(tmp), "(%s)[%zu]", a1, where);
486 fprintf(stderr, "%20s = 0x%02x (expected 0x%02x)\n", tmp,
487 ((u_char *)aa1)[where], v);
488 test_die();
489 }
490
491 void
assert_int(const char * file,int line,const char * a1,const char * a2,int aa1,int aa2,enum test_predicate pred)492 assert_int(const char *file, int line, const char *a1, const char *a2,
493 int aa1, int aa2, enum test_predicate pred)
494 {
495 TEST_CHECK(aa1, aa2, pred);
496 test_header(file, line, a1, a2, "INT", pred);
497 fprintf(stderr, "%12s = %d\n", a1, aa1);
498 fprintf(stderr, "%12s = %d\n", a2, aa2);
499 test_die();
500 }
501
502 void
assert_size_t(const char * file,int line,const char * a1,const char * a2,size_t aa1,size_t aa2,enum test_predicate pred)503 assert_size_t(const char *file, int line, const char *a1, const char *a2,
504 size_t aa1, size_t aa2, enum test_predicate pred)
505 {
506 TEST_CHECK(aa1, aa2, pred);
507 test_header(file, line, a1, a2, "SIZE_T", pred);
508 fprintf(stderr, "%12s = %zu\n", a1, aa1);
509 fprintf(stderr, "%12s = %zu\n", a2, aa2);
510 test_die();
511 }
512
513 void
assert_u_int(const char * file,int line,const char * a1,const char * a2,u_int aa1,u_int aa2,enum test_predicate pred)514 assert_u_int(const char *file, int line, const char *a1, const char *a2,
515 u_int aa1, u_int aa2, enum test_predicate pred)
516 {
517 TEST_CHECK(aa1, aa2, pred);
518 test_header(file, line, a1, a2, "U_INT", pred);
519 fprintf(stderr, "%12s = %u / 0x%x\n", a1, aa1, aa1);
520 fprintf(stderr, "%12s = %u / 0x%x\n", a2, aa2, aa2);
521 test_die();
522 }
523
524 void
assert_long(const char * file,int line,const char * a1,const char * a2,long aa1,long aa2,enum test_predicate pred)525 assert_long(const char *file, int line, const char *a1, const char *a2,
526 long aa1, long aa2, enum test_predicate pred)
527 {
528 TEST_CHECK(aa1, aa2, pred);
529 test_header(file, line, a1, a2, "LONG", pred);
530 fprintf(stderr, "%12s = %ld / 0x%lx\n", a1, aa1, aa1);
531 fprintf(stderr, "%12s = %ld / 0x%lx\n", a2, aa2, aa2);
532 test_die();
533 }
534
535 void
assert_long_long(const char * file,int line,const char * a1,const char * a2,long long aa1,long long aa2,enum test_predicate pred)536 assert_long_long(const char *file, int line, const char *a1, const char *a2,
537 long long aa1, long long aa2, enum test_predicate pred)
538 {
539 TEST_CHECK(aa1, aa2, pred);
540 test_header(file, line, a1, a2, "LONG LONG", pred);
541 fprintf(stderr, "%12s = %lld / 0x%llx\n", a1, aa1, aa1);
542 fprintf(stderr, "%12s = %lld / 0x%llx\n", a2, aa2, aa2);
543 test_die();
544 }
545
546 void
assert_char(const char * file,int line,const char * a1,const char * a2,char aa1,char aa2,enum test_predicate pred)547 assert_char(const char *file, int line, const char *a1, const char *a2,
548 char aa1, char aa2, enum test_predicate pred)
549 {
550 char buf[8];
551
552 TEST_CHECK(aa1, aa2, pred);
553 test_header(file, line, a1, a2, "CHAR", pred);
554 fprintf(stderr, "%12s = '%s' / 0x02%x\n", a1,
555 vis(buf, aa1, VIS_SAFE|VIS_NL|VIS_TAB|VIS_OCTAL, 0), aa1);
556 fprintf(stderr, "%12s = '%s' / 0x02%x\n", a1,
557 vis(buf, aa2, VIS_SAFE|VIS_NL|VIS_TAB|VIS_OCTAL, 0), aa2);
558 test_die();
559 }
560
561 void
assert_u8(const char * file,int line,const char * a1,const char * a2,uint8_t aa1,uint8_t aa2,enum test_predicate pred)562 assert_u8(const char *file, int line, const char *a1, const char *a2,
563 uint8_t aa1, uint8_t aa2, enum test_predicate pred)
564 {
565 TEST_CHECK(aa1, aa2, pred);
566 test_header(file, line, a1, a2, "U8", pred);
567 fprintf(stderr, "%12s = 0x%02x %u\n", a1, aa1, aa1);
568 fprintf(stderr, "%12s = 0x%02x %u\n", a2, aa2, aa2);
569 test_die();
570 }
571
572 void
assert_u16(const char * file,int line,const char * a1,const char * a2,uint16_t aa1,uint16_t aa2,enum test_predicate pred)573 assert_u16(const char *file, int line, const char *a1, const char *a2,
574 uint16_t aa1, uint16_t aa2, enum test_predicate pred)
575 {
576 TEST_CHECK(aa1, aa2, pred);
577 test_header(file, line, a1, a2, "U16", pred);
578 fprintf(stderr, "%12s = 0x%04x %u\n", a1, aa1, aa1);
579 fprintf(stderr, "%12s = 0x%04x %u\n", a2, aa2, aa2);
580 test_die();
581 }
582
583 void
assert_u32(const char * file,int line,const char * a1,const char * a2,uint32_t aa1,uint32_t aa2,enum test_predicate pred)584 assert_u32(const char *file, int line, const char *a1, const char *a2,
585 uint32_t aa1, uint32_t aa2, enum test_predicate pred)
586 {
587 TEST_CHECK(aa1, aa2, pred);
588 test_header(file, line, a1, a2, "U32", pred);
589 fprintf(stderr, "%12s = 0x%08x %u\n", a1, aa1, aa1);
590 fprintf(stderr, "%12s = 0x%08x %u\n", a2, aa2, aa2);
591 test_die();
592 }
593
594 void
assert_u64(const char * file,int line,const char * a1,const char * a2,uint64_t aa1,uint64_t aa2,enum test_predicate pred)595 assert_u64(const char *file, int line, const char *a1, const char *a2,
596 uint64_t aa1, uint64_t aa2, enum test_predicate pred)
597 {
598 TEST_CHECK(aa1, aa2, pred);
599 test_header(file, line, a1, a2, "U64", pred);
600 fprintf(stderr, "%12s = 0x%016llx %llu\n", a1,
601 (unsigned long long)aa1, (unsigned long long)aa1);
602 fprintf(stderr, "%12s = 0x%016llx %llu\n", a2,
603 (unsigned long long)aa2, (unsigned long long)aa2);
604 test_die();
605 }
606
607 void
assert_double(const char * file,int line,const char * a1,const char * a2,double aa1,double aa2,enum test_predicate pred)608 assert_double(const char *file, int line, const char *a1, const char *a2,
609 double aa1, double aa2, enum test_predicate pred)
610 {
611 const double epsilon = 0.000000001;
612
613 switch (pred) {
614 case TEST_EQ:
615 if (fabs(aa1 - aa2) < epsilon)
616 return;
617 break;
618 case TEST_NE:
619 if (fabs(aa1 - aa2) >= epsilon)
620 return;
621 break;
622 case TEST_LT:
623 if (aa1 < aa2)
624 return;
625 break;
626 case TEST_LE:
627 if (aa1 <= aa2)
628 return;
629 break;
630 case TEST_GT:
631 if (aa1 > aa2)
632 return;
633 break;
634 case TEST_GE:
635 if (aa1 >= aa2)
636 return;
637 break;
638 default:
639 abort();
640 }
641
642 test_header(file, line, a1, a2, "DOUBLE", pred);
643 fprintf(stderr, "%12s = %f\n", a1, aa1);
644 fprintf(stderr, "%12s = %f\n", a2, aa2);
645 test_die();
646 }
647
648 void
assert_ptr(const char * file,int line,const char * a1,const char * a2,const void * aa1,const void * aa2,enum test_predicate pred)649 assert_ptr(const char *file, int line, const char *a1, const char *a2,
650 const void *aa1, const void *aa2, enum test_predicate pred)
651 {
652 TEST_CHECK(aa1, aa2, pred);
653 test_header(file, line, a1, a2, "PTR", pred);
654 fprintf(stderr, "%12s = %p\n", a1, aa1);
655 fprintf(stderr, "%12s = %p\n", a2, aa2);
656 test_die();
657 }
658
659 static double
tstod(const struct timespec * ts)660 tstod(const struct timespec *ts)
661 {
662 return (double)ts->tv_sec + ((double)ts->tv_nsec / 1000000000.0);
663 }
664
665 void
bench_start(const char * file,int line,const char * name)666 bench_start(const char *file, int line, const char *name)
667 {
668 char *cp;
669
670 if (bench_name != NULL) {
671 fprintf(stderr, "\n%s:%d internal error: BENCH_START() called "
672 "while previous benchmark \"%s\" incomplete",
673 file, line, bench_name);
674 abort();
675 }
676 cp = xstrdup(name);
677 lowercase(cp);
678 bench_skip = benchmark_pattern != NULL &&
679 match_pattern_list(cp, benchmark_pattern, 1) != 1;
680 free(cp);
681
682 bench_name = name;
683 bench_nruns = 0;
684 if (bench_skip)
685 return;
686 free(bench_samples);
687 bench_nalloc = BENCH_SAMPLES_ALLOC;
688 bench_samples = xcalloc(sizeof(*bench_samples), bench_nalloc);
689 bench_accum_secs = 0;
690 }
691
692 int
bench_done(void)693 bench_done(void)
694 {
695 return bench_skip || bench_accum_secs >= (fast ? BENCH_FAST_DEADLINE :
696 (slow ? BENCH_SLOW_DEADLINE : BENCH_NORMAL_DEADLINE));
697 }
698
699 void
bench_case_start(const char * file,int line)700 bench_case_start(const char *file, int line)
701 {
702 clock_gettime(CLOCK_REALTIME, &bench_start_time);
703 }
704
705 void
bench_case_finish(const char * file,int line)706 bench_case_finish(const char *file, int line)
707 {
708 struct timespec ts;
709
710 clock_gettime(CLOCK_REALTIME, &bench_finish_time);
711 timespecsub(&bench_finish_time, &bench_start_time, &ts);
712 if (bench_nruns >= bench_nalloc) {
713 if (bench_nalloc >= INT_MAX / 2) {
714 fprintf(stderr, "\n%s:%d benchmark %s too many samples",
715 __FILE__, __LINE__, bench_name);
716 abort();
717 }
718 bench_samples = xrecallocarray(bench_samples, bench_nalloc,
719 bench_nalloc * 2, sizeof(*bench_samples));
720 bench_nalloc *= 2;
721 }
722 bench_samples[bench_nruns++] = ts;
723 bench_accum_secs += tstod(&ts);
724 }
725
726 static int
tscmp(const void * aa,const void * bb)727 tscmp(const void *aa, const void *bb)
728 {
729 const struct timespec *a = (const struct timespec *)aa;
730 const struct timespec *b = (const struct timespec *)bb;
731
732 if (timespeccmp(a, b, ==))
733 return 0;
734 return timespeccmp(a, b, <) ? -1 : 1;
735 }
736
737 void
bench_finish(const char * file,int line,const char * unit)738 bench_finish(const char *file, int line, const char *unit)
739 {
740 double std_dev = 0, mean_spr, mean_rps, med_spr, med_rps;
741 int i;
742
743 if (bench_skip)
744 goto done;
745
746 if (bench_nruns < 1) {
747 fprintf(stderr, "\n%s:%d benchmark %s never ran", file, line,
748 bench_name);
749 abort();
750 }
751 /* median */
752 qsort(bench_samples, bench_nruns, sizeof(*bench_samples), tscmp);
753 i = bench_nruns / 2;
754 med_spr = tstod(&bench_samples[i]);
755 if (bench_nruns > 1 && bench_nruns & 1)
756 med_spr = (med_spr + tstod(&bench_samples[i - 1])) / 2.0;
757 med_rps = (med_spr == 0.0) ? INFINITY : 1.0/med_spr;
758 /* mean */
759 mean_spr = bench_accum_secs / (double)bench_nruns;
760 mean_rps = (mean_spr == 0.0) ? INFINITY : 1.0/mean_spr;
761 /* std. dev */
762 std_dev = 0;
763 for (i = 0; i < bench_nruns; i++) {
764 std_dev = tstod(&bench_samples[i]) - mean_spr;
765 std_dev *= std_dev;
766 }
767 std_dev /= (double)bench_nruns;
768 std_dev = sqrt(std_dev);
769 if (benchmark_detail_statistics) {
770 printf("%s: %d runs in %0.3fs, %0.03f/%0.03f ms/%s "
771 "(mean/median), std.dev %0.03f ms, "
772 "%0.2f/%0.2f %s/s (mean/median)\n",
773 bench_name, bench_nruns, bench_accum_secs,
774 mean_spr * 1000, med_spr * 1000, unit, std_dev * 1000,
775 mean_rps, med_rps, unit);
776 } else {
777 printf("%-*s %0.2f %s/s\n", BENCH_COLUMN_WIDTH,
778 bench_name, med_rps, unit);
779 }
780 done:
781 bench_name = NULL;
782 bench_nruns = 0;
783 free(bench_samples);
784 bench_samples = NULL;
785 bench_skip = 0;
786 }
787