1 /* $OpenBSD: test_helper.c,v 1.14 2025/04/15 04:00:42 djm Exp $ */
2 /*
3 * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18 /* Utility functions/framework for regress tests */
19
20 #include "includes.h"
21
22 #include <sys/types.h>
23 #include <sys/uio.h>
24 #include <sys/time.h>
25
26 #include <assert.h>
27 #include <fcntl.h>
28 #include <limits.h>
29 #include <math.h>
30 #include <signal.h>
31 #include <stdarg.h>
32 #include <stdint.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <time.h>
37 #include <unistd.h>
38
39 #ifdef WITH_OPENSSL
40 #include <openssl/bn.h>
41 #include <openssl/err.h>
42 #endif
43
44 #if defined(HAVE_STRNVIS) && defined(HAVE_VIS_H) && !defined(BROKEN_STRNVIS)
45 # include <vis.h>
46 #endif
47
48 #include "entropy.h"
49 #include "test_helper.h"
50 #include "atomicio.h"
51 #include "match.h"
52 #include "misc.h"
53 #include "xmalloc.h"
54
55 #define BENCH_FAST_DEADLINE 1
56 #define BENCH_NORMAL_DEADLINE 10
57 #define BENCH_SLOW_DEADLINE 60
58 #define BENCH_SAMPLES_ALLOC 8192
59 #define BENCH_COLUMN_WIDTH 40
60
61 #define MINIMUM(a, b) (((a) < (b)) ? (a) : (b))
62
63 #define TEST_CHECK_INT(r, pred) do { \
64 switch (pred) { \
65 case TEST_EQ: \
66 if (r == 0) \
67 return; \
68 break; \
69 case TEST_NE: \
70 if (r != 0) \
71 return; \
72 break; \
73 case TEST_LT: \
74 if (r < 0) \
75 return; \
76 break; \
77 case TEST_LE: \
78 if (r <= 0) \
79 return; \
80 break; \
81 case TEST_GT: \
82 if (r > 0) \
83 return; \
84 break; \
85 case TEST_GE: \
86 if (r >= 0) \
87 return; \
88 break; \
89 default: \
90 abort(); \
91 } \
92 } while (0)
93
94 #define TEST_CHECK(x1, x2, pred) do { \
95 switch (pred) { \
96 case TEST_EQ: \
97 if (x1 == x2) \
98 return; \
99 break; \
100 case TEST_NE: \
101 if (x1 != x2) \
102 return; \
103 break; \
104 case TEST_LT: \
105 if (x1 < x2) \
106 return; \
107 break; \
108 case TEST_LE: \
109 if (x1 <= x2) \
110 return; \
111 break; \
112 case TEST_GT: \
113 if (x1 > x2) \
114 return; \
115 break; \
116 case TEST_GE: \
117 if (x1 >= x2) \
118 return; \
119 break; \
120 default: \
121 abort(); \
122 } \
123 } while (0)
124
125 extern char *__progname;
126
127 static int verbose_mode = 0;
128 static int quiet_mode = 0;
129 static char *active_test_name = NULL;
130 static u_int test_number = 0;
131 static test_onerror_func_t *test_onerror = NULL;
132 static void *onerror_ctx = NULL;
133 static const char *data_dir = NULL;
134 static char subtest_info[512];
135 static int fast = 0;
136 static int slow = 0;
137 static int benchmark_detail_statistics = 0;
138
139 static int benchmark = 0;
140 static const char *bench_name = NULL;
141 static char *benchmark_pattern = NULL;
142 static struct timespec bench_start_time, bench_finish_time;
143 static struct timespec *bench_samples;
144 static int bench_skip, bench_nruns, bench_nalloc;
145 double bench_accum_secs;
146
147 int
main(int argc,char ** argv)148 main(int argc, char **argv)
149 {
150 int ch;
151
152 seed_rng();
153 #ifdef WITH_OPENSSL
154 ERR_load_crypto_strings();
155 #endif
156
157 /* Handle systems without __progname */
158 if (__progname == NULL) {
159 __progname = strrchr(argv[0], '/');
160 if (__progname == NULL || __progname[1] == '\0')
161 __progname = argv[0];
162 else
163 __progname++;
164 if ((__progname = strdup(__progname)) == NULL) {
165 fprintf(stderr, "strdup failed\n");
166 exit(1);
167 }
168 }
169
170 while ((ch = getopt(argc, argv, "O:bBFfvqd:")) != -1) {
171 switch (ch) {
172 case 'b':
173 benchmark = 1;
174 break;
175 case 'B':
176 benchmark = benchmark_detail_statistics = 1;
177 break;
178 case 'O':
179 benchmark_pattern = xstrdup(optarg);
180 break;
181 case 'F':
182 slow = 1;
183 break;
184 case 'f':
185 fast = 1;
186 break;
187 case 'd':
188 data_dir = optarg;
189 break;
190 case 'q':
191 verbose_mode = 0;
192 quiet_mode = 1;
193 break;
194 case 'v':
195 verbose_mode = 1;
196 quiet_mode = 0;
197 break;
198 default:
199 fprintf(stderr, "Unrecognised command line option\n");
200 fprintf(stderr, "Usage: %s [-vqfFbB] [-d data_dir] "
201 "[-O pattern]\n", __progname);
202 exit(1);
203 }
204 }
205 setvbuf(stdout, NULL, _IONBF, 0);
206 if (!quiet_mode)
207 printf("%s: ", __progname);
208 if (verbose_mode)
209 printf("\n");
210
211 if (benchmark)
212 benchmarks();
213 else
214 tests();
215
216 if (!quiet_mode && !benchmark)
217 printf(" %u tests ok\n", test_number);
218 return 0;
219 }
220
221 int
test_is_verbose(void)222 test_is_verbose(void)
223 {
224 return verbose_mode;
225 }
226
227 int
test_is_quiet(void)228 test_is_quiet(void)
229 {
230 return quiet_mode;
231 }
232
233 int
test_is_fast(void)234 test_is_fast(void)
235 {
236 return fast;
237 }
238
239 int
test_is_slow(void)240 test_is_slow(void)
241 {
242 return slow;
243 }
244
245 const char *
test_data_file(const char * name)246 test_data_file(const char *name)
247 {
248 static char ret[PATH_MAX];
249
250 if (data_dir != NULL)
251 snprintf(ret, sizeof(ret), "%s/%s", data_dir, name);
252 else
253 strlcpy(ret, name, sizeof(ret));
254 if (access(ret, F_OK) != 0) {
255 fprintf(stderr, "Cannot access data file %s: %s\n",
256 ret, strerror(errno));
257 exit(1);
258 }
259 return ret;
260 }
261
262 void
test_info(char * s,size_t len)263 test_info(char *s, size_t len)
264 {
265 snprintf(s, len, "In test %u: \"%s\"%s%s\n", test_number,
266 active_test_name == NULL ? "<none>" : active_test_name,
267 *subtest_info != '\0' ? " - " : "", subtest_info);
268 }
269
270 static void
siginfo(int unused)271 siginfo(int unused __attribute__((__unused__)))
272 {
273 char buf[256];
274
275 test_info(buf, sizeof(buf));
276 atomicio(vwrite, STDERR_FILENO, buf, strlen(buf));
277 }
278
279 void
test_start(const char * n)280 test_start(const char *n)
281 {
282 assert(active_test_name == NULL);
283 assert((active_test_name = strdup(n)) != NULL);
284 *subtest_info = '\0';
285 if (verbose_mode)
286 printf("test %u - \"%s\": ", test_number, active_test_name);
287 test_number++;
288 #ifdef SIGINFO
289 signal(SIGINFO, siginfo);
290 #endif
291 signal(SIGUSR1, siginfo);
292 }
293
294 void
set_onerror_func(test_onerror_func_t * f,void * ctx)295 set_onerror_func(test_onerror_func_t *f, void *ctx)
296 {
297 test_onerror = f;
298 onerror_ctx = ctx;
299 }
300
301 void
test_done(void)302 test_done(void)
303 {
304 *subtest_info = '\0';
305 assert(active_test_name != NULL);
306 free(active_test_name);
307 active_test_name = NULL;
308 if (verbose_mode)
309 printf("OK\n");
310 else if (!quiet_mode && !benchmark) {
311 printf(".");
312 fflush(stdout);
313 }
314 }
315
316 void
test_subtest_info(const char * fmt,...)317 test_subtest_info(const char *fmt, ...)
318 {
319 va_list ap;
320
321 va_start(ap, fmt);
322 vsnprintf(subtest_info, sizeof(subtest_info), fmt, ap);
323 va_end(ap);
324 }
325
326 int
test_is_benchmark(void)327 test_is_benchmark(void)
328 {
329 return benchmark;
330 }
331
332 void
ssl_err_check(const char * file,int line)333 ssl_err_check(const char *file, int line)
334 {
335 #ifdef WITH_OPENSSL
336 long openssl_error = ERR_get_error();
337
338 if (openssl_error == 0)
339 return;
340
341 fprintf(stderr, "\n%s:%d: uncaught OpenSSL error: %s",
342 file, line, ERR_error_string(openssl_error, NULL));
343 #else /* WITH_OPENSSL */
344 fprintf(stderr, "\n%s:%d: uncaught OpenSSL error ",
345 file, line);
346 #endif /* WITH_OPENSSL */
347 abort();
348 }
349
350 static const char *
pred_name(enum test_predicate p)351 pred_name(enum test_predicate p)
352 {
353 switch (p) {
354 case TEST_EQ:
355 return "EQ";
356 case TEST_NE:
357 return "NE";
358 case TEST_LT:
359 return "LT";
360 case TEST_LE:
361 return "LE";
362 case TEST_GT:
363 return "GT";
364 case TEST_GE:
365 return "GE";
366 default:
367 return "UNKNOWN";
368 }
369 }
370
371 static void
test_die(void)372 test_die(void)
373 {
374 if (test_onerror != NULL)
375 test_onerror(onerror_ctx);
376 abort();
377 }
378
379 static void
test_header(const char * file,int line,const char * a1,const char * a2,const char * name,enum test_predicate pred)380 test_header(const char *file, int line, const char *a1, const char *a2,
381 const char *name, enum test_predicate pred)
382 {
383 fprintf(stderr, "\n%s:%d test #%u \"%s\"%s%s\n",
384 file, line, test_number, active_test_name,
385 *subtest_info != '\0' ? " - " : "", subtest_info);
386 fprintf(stderr, "ASSERT_%s_%s(%s%s%s) failed:\n",
387 name, pred_name(pred), a1,
388 a2 != NULL ? ", " : "", a2 != NULL ? a2 : "");
389 }
390
391 #ifdef WITH_OPENSSL
392 void
assert_bignum(const char * file,int line,const char * a1,const char * a2,const BIGNUM * aa1,const BIGNUM * aa2,enum test_predicate pred)393 assert_bignum(const char *file, int line, const char *a1, const char *a2,
394 const BIGNUM *aa1, const BIGNUM *aa2, enum test_predicate pred)
395 {
396 int r = BN_cmp(aa1, aa2);
397
398 TEST_CHECK_INT(r, pred);
399 test_header(file, line, a1, a2, "BIGNUM", pred);
400 fprintf(stderr, "%12s = 0x%s\n", a1, BN_bn2hex(aa1));
401 fprintf(stderr, "%12s = 0x%s\n", a2, BN_bn2hex(aa2));
402 test_die();
403 }
404 #endif
405
406 void
assert_string(const char * file,int line,const char * a1,const char * a2,const char * aa1,const char * aa2,enum test_predicate pred)407 assert_string(const char *file, int line, const char *a1, const char *a2,
408 const char *aa1, const char *aa2, enum test_predicate pred)
409 {
410 int r;
411
412 /* Verify pointers are not NULL */
413 assert_ptr(file, line, a1, "NULL", aa1, NULL, TEST_NE);
414 assert_ptr(file, line, a2, "NULL", aa2, NULL, TEST_NE);
415
416 r = strcmp(aa1, aa2);
417 TEST_CHECK_INT(r, pred);
418 test_header(file, line, a1, a2, "STRING", pred);
419 fprintf(stderr, "%12s = %s (len %zu)\n", a1, aa1, strlen(aa1));
420 fprintf(stderr, "%12s = %s (len %zu)\n", a2, aa2, strlen(aa2));
421 test_die();
422 }
423
424 void
assert_mem(const char * file,int line,const char * a1,const char * a2,const void * aa1,const void * aa2,size_t l,enum test_predicate pred)425 assert_mem(const char *file, int line, const char *a1, const char *a2,
426 const void *aa1, const void *aa2, size_t l, enum test_predicate pred)
427 {
428 int r;
429 char *aa1_tohex = NULL;
430 char *aa2_tohex = NULL;
431
432 if (l == 0)
433 return;
434 /* If length is >0, then verify pointers are not NULL */
435 assert_ptr(file, line, a1, "NULL", aa1, NULL, TEST_NE);
436 assert_ptr(file, line, a2, "NULL", aa2, NULL, TEST_NE);
437
438 r = memcmp(aa1, aa2, l);
439 TEST_CHECK_INT(r, pred);
440 test_header(file, line, a1, a2, "STRING", pred);
441 aa1_tohex = tohex(aa1, MINIMUM(l, 256));
442 aa2_tohex = tohex(aa2, MINIMUM(l, 256));
443 fprintf(stderr, "%12s = %s (len %zu)\n", a1, aa1_tohex, l);
444 fprintf(stderr, "%12s = %s (len %zu)\n", a2, aa2_tohex, l);
445 free(aa1_tohex);
446 free(aa2_tohex);
447 test_die();
448 }
449
450 static int
memvalcmp(const u_int8_t * s,u_char v,size_t l,size_t * where)451 memvalcmp(const u_int8_t *s, u_char v, size_t l, size_t *where)
452 {
453 size_t i;
454
455 for (i = 0; i < l; i++) {
456 if (s[i] != v) {
457 *where = i;
458 return 1;
459 }
460 }
461 return 0;
462 }
463
464 void
assert_mem_filled(const char * file,int line,const char * a1,const void * aa1,u_char v,size_t l,enum test_predicate pred)465 assert_mem_filled(const char *file, int line, const char *a1,
466 const void *aa1, u_char v, size_t l, enum test_predicate pred)
467 {
468 size_t where = -1;
469 int r;
470 char tmp[64];
471 char *aa1_tohex = NULL;
472
473 if (l == 0)
474 return;
475 /* If length is >0, then verify the pointer is not NULL */
476 assert_ptr(file, line, a1, "NULL", aa1, NULL, TEST_NE);
477
478 r = memvalcmp(aa1, v, l, &where);
479 TEST_CHECK_INT(r, pred);
480 test_header(file, line, a1, NULL, "MEM_ZERO", pred);
481 aa1_tohex = tohex(aa1, MINIMUM(l, 20));
482 fprintf(stderr, "%20s = %s%s (len %zu)\n", a1,
483 aa1_tohex, l > 20 ? "..." : "", l);
484 free(aa1_tohex);
485 snprintf(tmp, sizeof(tmp), "(%s)[%zu]", a1, where);
486 fprintf(stderr, "%20s = 0x%02x (expected 0x%02x)\n", tmp,
487 ((u_char *)aa1)[where], v);
488 test_die();
489 }
490
491 void
assert_int(const char * file,int line,const char * a1,const char * a2,int aa1,int aa2,enum test_predicate pred)492 assert_int(const char *file, int line, const char *a1, const char *a2,
493 int aa1, int aa2, enum test_predicate pred)
494 {
495 TEST_CHECK(aa1, aa2, pred);
496 test_header(file, line, a1, a2, "INT", pred);
497 fprintf(stderr, "%12s = %d\n", a1, aa1);
498 fprintf(stderr, "%12s = %d\n", a2, aa2);
499 test_die();
500 }
501
502 void
assert_size_t(const char * file,int line,const char * a1,const char * a2,size_t aa1,size_t aa2,enum test_predicate pred)503 assert_size_t(const char *file, int line, const char *a1, const char *a2,
504 size_t aa1, size_t aa2, enum test_predicate pred)
505 {
506 TEST_CHECK(aa1, aa2, pred);
507 test_header(file, line, a1, a2, "SIZE_T", pred);
508 fprintf(stderr, "%12s = %zu\n", a1, aa1);
509 fprintf(stderr, "%12s = %zu\n", a2, aa2);
510 test_die();
511 }
512
513 void
assert_u_int(const char * file,int line,const char * a1,const char * a2,u_int aa1,u_int aa2,enum test_predicate pred)514 assert_u_int(const char *file, int line, const char *a1, const char *a2,
515 u_int aa1, u_int aa2, enum test_predicate pred)
516 {
517 TEST_CHECK(aa1, aa2, pred);
518 test_header(file, line, a1, a2, "U_INT", pred);
519 fprintf(stderr, "%12s = %u / 0x%x\n", a1, aa1, aa1);
520 fprintf(stderr, "%12s = %u / 0x%x\n", a2, aa2, aa2);
521 test_die();
522 }
523
524 void
assert_long(const char * file,int line,const char * a1,const char * a2,long aa1,long aa2,enum test_predicate pred)525 assert_long(const char *file, int line, const char *a1, const char *a2,
526 long aa1, long aa2, enum test_predicate pred)
527 {
528 TEST_CHECK(aa1, aa2, pred);
529 test_header(file, line, a1, a2, "LONG", pred);
530 fprintf(stderr, "%12s = %ld / 0x%lx\n", a1, aa1, aa1);
531 fprintf(stderr, "%12s = %ld / 0x%lx\n", a2, aa2, aa2);
532 test_die();
533 }
534
535 void
assert_long_long(const char * file,int line,const char * a1,const char * a2,long long aa1,long long aa2,enum test_predicate pred)536 assert_long_long(const char *file, int line, const char *a1, const char *a2,
537 long long aa1, long long aa2, enum test_predicate pred)
538 {
539 TEST_CHECK(aa1, aa2, pred);
540 test_header(file, line, a1, a2, "LONG LONG", pred);
541 fprintf(stderr, "%12s = %lld / 0x%llx\n", a1, aa1, aa1);
542 fprintf(stderr, "%12s = %lld / 0x%llx\n", a2, aa2, aa2);
543 test_die();
544 }
545
546 void
assert_char(const char * file,int line,const char * a1,const char * a2,char aa1,char aa2,enum test_predicate pred)547 assert_char(const char *file, int line, const char *a1, const char *a2,
548 char aa1, char aa2, enum test_predicate pred)
549 {
550 char buf[8];
551
552 TEST_CHECK(aa1, aa2, pred);
553 test_header(file, line, a1, a2, "CHAR", pred);
554 fprintf(stderr, "%12s = '%s' / 0x02%x\n", a1,
555 vis(buf, aa1, VIS_SAFE|VIS_NL|VIS_TAB|VIS_OCTAL, 0), aa1);
556 fprintf(stderr, "%12s = '%s' / 0x02%x\n", a1,
557 vis(buf, aa2, VIS_SAFE|VIS_NL|VIS_TAB|VIS_OCTAL, 0), aa2);
558 test_die();
559 }
560
561 void
assert_u8(const char * file,int line,const char * a1,const char * a2,u_int8_t aa1,u_int8_t aa2,enum test_predicate pred)562 assert_u8(const char *file, int line, const char *a1, const char *a2,
563 u_int8_t aa1, u_int8_t aa2, enum test_predicate pred)
564 {
565 TEST_CHECK(aa1, aa2, pred);
566 test_header(file, line, a1, a2, "U8", pred);
567 fprintf(stderr, "%12s = 0x%02x %u\n", a1, aa1, aa1);
568 fprintf(stderr, "%12s = 0x%02x %u\n", a2, aa2, aa2);
569 test_die();
570 }
571
572 void
assert_u16(const char * file,int line,const char * a1,const char * a2,u_int16_t aa1,u_int16_t aa2,enum test_predicate pred)573 assert_u16(const char *file, int line, const char *a1, const char *a2,
574 u_int16_t aa1, u_int16_t aa2, enum test_predicate pred)
575 {
576 TEST_CHECK(aa1, aa2, pred);
577 test_header(file, line, a1, a2, "U16", pred);
578 fprintf(stderr, "%12s = 0x%04x %u\n", a1, aa1, aa1);
579 fprintf(stderr, "%12s = 0x%04x %u\n", a2, aa2, aa2);
580 test_die();
581 }
582
583 void
assert_u32(const char * file,int line,const char * a1,const char * a2,u_int32_t aa1,u_int32_t aa2,enum test_predicate pred)584 assert_u32(const char *file, int line, const char *a1, const char *a2,
585 u_int32_t aa1, u_int32_t aa2, enum test_predicate pred)
586 {
587 TEST_CHECK(aa1, aa2, pred);
588 test_header(file, line, a1, a2, "U32", pred);
589 fprintf(stderr, "%12s = 0x%08x %u\n", a1, aa1, aa1);
590 fprintf(stderr, "%12s = 0x%08x %u\n", a2, aa2, aa2);
591 test_die();
592 }
593
594 void
assert_u64(const char * file,int line,const char * a1,const char * a2,u_int64_t aa1,u_int64_t aa2,enum test_predicate pred)595 assert_u64(const char *file, int line, const char *a1, const char *a2,
596 u_int64_t aa1, u_int64_t aa2, enum test_predicate pred)
597 {
598 TEST_CHECK(aa1, aa2, pred);
599 test_header(file, line, a1, a2, "U64", pred);
600 fprintf(stderr, "%12s = 0x%016llx %llu\n", a1,
601 (unsigned long long)aa1, (unsigned long long)aa1);
602 fprintf(stderr, "%12s = 0x%016llx %llu\n", a2,
603 (unsigned long long)aa2, (unsigned long long)aa2);
604 test_die();
605 }
606
607 void
assert_ptr(const char * file,int line,const char * a1,const char * a2,const void * aa1,const void * aa2,enum test_predicate pred)608 assert_ptr(const char *file, int line, const char *a1, const char *a2,
609 const void *aa1, const void *aa2, enum test_predicate pred)
610 {
611 TEST_CHECK(aa1, aa2, pred);
612 test_header(file, line, a1, a2, "PTR", pred);
613 fprintf(stderr, "%12s = %p\n", a1, aa1);
614 fprintf(stderr, "%12s = %p\n", a2, aa2);
615 test_die();
616 }
617
618 static double
tstod(const struct timespec * ts)619 tstod(const struct timespec *ts)
620 {
621 return (double)ts->tv_sec + ((double)ts->tv_nsec / 1000000000.0);
622 }
623
624 void
bench_start(const char * file,int line,const char * name)625 bench_start(const char *file, int line, const char *name)
626 {
627 char *cp;
628
629 if (bench_name != NULL) {
630 fprintf(stderr, "\n%s:%d internal error: BENCH_START() called "
631 "while previous benchmark \"%s\" incomplete",
632 file, line, bench_name);
633 abort();
634 }
635 cp = xstrdup(name);
636 lowercase(cp);
637 bench_skip = benchmark_pattern != NULL &&
638 match_pattern_list(cp, benchmark_pattern, 1) != 1;
639 free(cp);
640
641 bench_name = name;
642 bench_nruns = 0;
643 if (bench_skip)
644 return;
645 free(bench_samples);
646 bench_nalloc = BENCH_SAMPLES_ALLOC;
647 bench_samples = xcalloc(sizeof(*bench_samples), bench_nalloc);
648 bench_accum_secs = 0;
649 }
650
651 int
bench_done(void)652 bench_done(void)
653 {
654 return bench_skip || bench_accum_secs >= (fast ? BENCH_FAST_DEADLINE :
655 (slow ? BENCH_SLOW_DEADLINE : BENCH_NORMAL_DEADLINE));
656 }
657
658 void
bench_case_start(const char * file,int line)659 bench_case_start(const char *file, int line)
660 {
661 clock_gettime(CLOCK_REALTIME, &bench_start_time);
662 }
663
664 void
bench_case_finish(const char * file,int line)665 bench_case_finish(const char *file, int line)
666 {
667 struct timespec ts;
668
669 clock_gettime(CLOCK_REALTIME, &bench_finish_time);
670 timespecsub(&bench_finish_time, &bench_start_time, &ts);
671 if (bench_nruns >= bench_nalloc) {
672 if (bench_nalloc >= INT_MAX / 2) {
673 fprintf(stderr, "\n%s:%d benchmark %s too many samples",
674 __FILE__, __LINE__, bench_name);
675 abort();
676 }
677 bench_samples = xrecallocarray(bench_samples, bench_nalloc,
678 bench_nalloc * 2, sizeof(*bench_samples));
679 bench_nalloc *= 2;
680 }
681 bench_samples[bench_nruns++] = ts;
682 bench_accum_secs += tstod(&ts);
683 }
684
685 static int
tscmp(const void * aa,const void * bb)686 tscmp(const void *aa, const void *bb)
687 {
688 const struct timespec *a = (const struct timespec *)aa;
689 const struct timespec *b = (const struct timespec *)bb;
690
691 if (timespeccmp(a, b, ==))
692 return 0;
693 return timespeccmp(a, b, <) ? -1 : 1;
694 }
695
696 void
bench_finish(const char * file,int line,const char * unit)697 bench_finish(const char *file, int line, const char *unit)
698 {
699 double std_dev = 0, mean_spr, mean_rps, med_spr, med_rps;
700 int i;
701
702 if (bench_skip)
703 goto done;
704
705 if (bench_nruns < 1) {
706 fprintf(stderr, "\n%s:%d benchmark %s never ran", file, line,
707 bench_name);
708 abort();
709 }
710 /* median */
711 qsort(bench_samples, bench_nruns, sizeof(*bench_samples), tscmp);
712 i = bench_nruns / 2;
713 med_spr = tstod(&bench_samples[i]);
714 if (bench_nruns > 1 && bench_nruns & 1)
715 med_spr = (med_spr + tstod(&bench_samples[i - 1])) / 2.0;
716 med_rps = (med_spr == 0.0) ? INFINITY : 1.0/med_spr;
717 /* mean */
718 mean_spr = bench_accum_secs / (double)bench_nruns;
719 mean_rps = (mean_spr == 0.0) ? INFINITY : 1.0/mean_spr;
720 /* std. dev */
721 std_dev = 0;
722 for (i = 0; i < bench_nruns; i++) {
723 std_dev = tstod(&bench_samples[i]) - mean_spr;
724 std_dev *= std_dev;
725 }
726 std_dev /= (double)bench_nruns;
727 std_dev = sqrt(std_dev);
728 if (benchmark_detail_statistics) {
729 printf("%s: %d runs in %0.3fs, %0.03f/%0.03f ms/%s "
730 "(mean/median), std.dev %0.03f ms, "
731 "%0.2f/%0.2f %s/s (mean/median)\n",
732 bench_name, bench_nruns, bench_accum_secs,
733 mean_spr * 1000, med_spr * 1000, unit, std_dev * 1000,
734 mean_rps, med_rps, unit);
735 } else {
736 printf("%-*s %0.2f %s/s\n", BENCH_COLUMN_WIDTH,
737 bench_name, med_rps, unit);
738 }
739 done:
740 bench_name = NULL;
741 bench_nruns = 0;
742 free(bench_samples);
743 bench_samples = NULL;
744 bench_skip = 0;
745 }
746