1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2025 Oxide Computer Company
14 * Copyright 2024 Ryan Zezeski
15 */
16 #include <sys/debug.h>
17 #include <sys/ktest.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <sys/list.h>
21 #include <fcntl.h>
22 #include <unistd.h>
23 #include <stropts.h>
24 #include <stdio.h>
25 #include <stddef.h>
26 #include <stdlib.h>
27 #include <errno.h>
28 #include <err.h>
29 #include <stdarg.h>
30 #include <strings.h>
31 #include <libgen.h>
32 #include <libnvpair.h>
33 #include <regex.h>
34 #include <libcmdutils.h>
35 #include <ofmt.h>
36 #include <zone.h>
37 #include <libktest.h>
38
39 #define EXIT_USAGE 2
40 #define KTEST_CMD_SZ 24
41
42 static const char *ktest_prog;
43
44
45 /* An adapter to use errx with libofmt. */
46 void
ktest_ofmt_errx(const char * fmt,...)47 ktest_ofmt_errx(const char *fmt, ...)
48 {
49 va_list ap;
50
51 va_start(ap, fmt);
52 verrx(EXIT_FAILURE, fmt, ap);
53 }
54
55 typedef enum ktest_fmt_fields {
56 KTEST_FMT_RESULT,
57 KTEST_FMT_MODULE,
58 KTEST_FMT_SUITE,
59 KTEST_FMT_TEST,
60 KTEST_FMT_INPUT_FLAG,
61 KTEST_FMT_INPUT_PATH,
62 KTEST_FMT_LINE,
63 KTEST_FMT_REASON,
64 } ktest_fmt_fields_t;
65
66 typedef struct ktest_list_ofmt {
67 char *klof_module;
68 char *klof_suite;
69 char *klof_test;
70 boolean_t klof_input;
71 } ktest_list_ofmt_t;
72
73 static boolean_t
ktest_list_ofmt_cb(ofmt_arg_t * ofarg,char * buf,uint_t len)74 ktest_list_ofmt_cb(ofmt_arg_t *ofarg, char *buf, uint_t len)
75 {
76 ktest_entry_t *ent = ofarg->ofmt_cbarg;
77
78 switch (ofarg->ofmt_id) {
79 case KTEST_FMT_MODULE:
80 if (snprintf(buf, len, "%s", ent->ke_module) >= len) {
81 return (B_FALSE);
82 }
83 break;
84
85 case KTEST_FMT_SUITE:
86 if (snprintf(buf, len, "%s", ent->ke_suite) >= len) {
87 return (B_FALSE);
88 }
89 break;
90
91 case KTEST_FMT_TEST:
92 if (snprintf(buf, len, "%s", ent->ke_test) >= len) {
93 return (B_FALSE);
94 }
95 break;
96
97 case KTEST_FMT_INPUT_FLAG: {
98 const char *flag = ent->ke_requires_input ? "Y" : "N";
99
100 if (snprintf(buf, len, "%s", flag) >= len) {
101 return (B_FALSE);
102 }
103 }
104 }
105
106 return (B_TRUE);
107 }
108
109 #define KTEST_LIST_CMD_DEF_FIELDS "module,suite,test,input"
110
111 static const ofmt_field_t ktest_list_ofmt[] = {
112 { "MODULE", 12, KTEST_FMT_MODULE, ktest_list_ofmt_cb },
113 { "SUITE", 16, KTEST_FMT_SUITE, ktest_list_ofmt_cb },
114 { "TEST", 45, KTEST_FMT_TEST, ktest_list_ofmt_cb },
115 { "INPUT", 7, KTEST_FMT_INPUT_FLAG, ktest_list_ofmt_cb },
116 { NULL, 0, 0, NULL },
117 };
118
119 typedef struct ktest_run_output {
120 ktest_run_req_t *kro_req;
121 ktest_run_result_t *kro_result;
122 char *kro_input_path;
123 } ktest_run_output_t;
124
125 static boolean_t
ktest_run_ofmt_cb(ofmt_arg_t * ofarg,char * buf,uint_t len)126 ktest_run_ofmt_cb(ofmt_arg_t *ofarg, char *buf, uint_t len)
127 {
128 const ktest_run_output_t *kro = ofarg->ofmt_cbarg;
129 const ktest_run_req_t *req = kro->kro_req;
130 const ktest_run_result_t *result = kro->kro_result;
131 const char *input_path =
132 kro->kro_input_path != NULL ? kro->kro_input_path : "";
133
134 switch (ofarg->ofmt_id) {
135 case KTEST_FMT_RESULT:
136 if (snprintf(buf, len, "%s",
137 ktest_code_name(result->krr_code)) >= len) {
138 return (B_FALSE);
139 }
140 break;
141 case KTEST_FMT_MODULE:
142 if (snprintf(buf, len, "%s", req->krq_module) >= len) {
143 return (B_FALSE);
144 }
145 break;
146
147 case KTEST_FMT_SUITE:
148 if (snprintf(buf, len, "%s", req->krq_suite) >= len) {
149 return (B_FALSE);
150 }
151 break;
152
153 case KTEST_FMT_TEST:
154 if (snprintf(buf, len, "%s", req->krq_test) >= len) {
155 return (B_FALSE);
156 }
157 break;
158 case KTEST_FMT_INPUT_PATH:
159 if (snprintf(buf, len, "%s", input_path) >= len) {
160 return (B_FALSE);
161 }
162 break;
163 case KTEST_FMT_LINE:
164 if (snprintf(buf, len, "%u", result->krr_line) >= len) {
165 return (B_FALSE);
166 }
167 break;
168 case KTEST_FMT_REASON:
169 if (snprintf(buf, len, "%s", result->krr_msg) >= len) {
170 return (B_FALSE);
171 }
172 break;
173 }
174
175 return (B_TRUE);
176 }
177
178 #define KTEST_RUN_CMD_DEF_FIELDS "result,line,module,suite,test"
179
180 /*
181 * The input column for the run command is for displaying the path to
182 * the input file, as opposed to the list command which indicates if
183 * the test requires input or not.
184 */
185 static const ofmt_field_t ktest_run_ofmt[] = {
186 { "RESULT", 7, KTEST_FMT_RESULT, ktest_run_ofmt_cb },
187 { "MODULE", 12, KTEST_FMT_MODULE, ktest_run_ofmt_cb },
188 { "SUITE", 16, KTEST_FMT_SUITE, ktest_run_ofmt_cb },
189 { "TEST", 45, KTEST_FMT_TEST, ktest_run_ofmt_cb },
190 { "INPUT", 48, KTEST_FMT_INPUT_PATH, ktest_run_ofmt_cb },
191 { "LINE", 6, KTEST_FMT_LINE, ktest_run_ofmt_cb },
192 { "REASON", 256, KTEST_FMT_REASON, ktest_run_ofmt_cb },
193 { NULL, 0, 0, NULL },
194 };
195
196 typedef enum ktest_stat_type {
197 KTEST_STAT_MOD,
198 KTEST_STAT_SUITE,
199 } ktest_stat_type_t;
200
201 typedef struct ktest_stats {
202 list_node_t ks_node;
203 ktest_stat_type_t ks_type;
204 char *ks_name;
205 uint32_t ks_total;
206 uint32_t ks_pass;
207 uint32_t ks_fail;
208 uint32_t ks_err;
209 uint32_t ks_skip;
210 uint32_t ks_none;
211 } ktest_stats_t;
212
213 static ktest_stats_t *
ktest_stats_new(ktest_stat_type_t type,const char * name)214 ktest_stats_new(ktest_stat_type_t type, const char *name)
215 {
216 ktest_stats_t *stats;
217
218 if ((stats = malloc(sizeof (ktest_stats_t))) == NULL) {
219 err(EXIT_FAILURE, "failed to allocate stats structure");
220 }
221
222 stats->ks_type = type;
223 stats->ks_name = strndup(name, KTEST_MAX_NAME_LEN);
224
225 if (stats->ks_name == NULL) {
226 err(EXIT_FAILURE, "failed to allocate stats name");
227 }
228
229 stats->ks_total = 0;
230 stats->ks_pass = 0;
231 stats->ks_fail = 0;
232 stats->ks_err = 0;
233 stats->ks_skip = 0;
234 stats->ks_none = 0;
235 return (stats);
236 }
237
238 static void
ktest_record_stat(ktest_stats_t * mod,ktest_stats_t * suite,const ktest_run_result_t * res)239 ktest_record_stat(ktest_stats_t *mod, ktest_stats_t *suite,
240 const ktest_run_result_t *res)
241 {
242 mod->ks_total++;
243 suite->ks_total++;
244
245 switch (res->krr_code) {
246 case KTEST_CODE_NONE:
247 mod->ks_none++;
248 suite->ks_none++;
249 break;
250
251 case KTEST_CODE_PASS:
252 mod->ks_pass++;
253 suite->ks_pass++;
254 break;
255
256 case KTEST_CODE_FAIL:
257 mod->ks_fail++;
258 suite->ks_fail++;
259 break;
260
261 case KTEST_CODE_SKIP:
262 mod->ks_skip++;
263 suite->ks_skip++;
264 break;
265
266 case KTEST_CODE_ERROR:
267 mod->ks_err++;
268 suite->ks_err++;
269 break;
270 }
271 }
272
273 typedef enum ktest_fmt_stats {
274 KTEST_FMT_STATS_MS,
275 KTEST_FMT_STATS_TOTAL,
276 KTEST_FMT_STATS_PASS,
277 KTEST_FMT_STATS_FAIL,
278 KTEST_FMT_STATS_ERR,
279 KTEST_FMT_STATS_SKIP,
280 KTEST_FMT_STATS_NONE,
281 } ktest_fmt_stats_t;
282
283 static boolean_t
ktest_stats_ofmt_cb(ofmt_arg_t * ofarg,char * buf,uint_t len)284 ktest_stats_ofmt_cb(ofmt_arg_t *ofarg, char *buf, uint_t len)
285 {
286 ktest_stats_t *stats = ofarg->ofmt_cbarg;
287
288 switch (ofarg->ofmt_id) {
289 case KTEST_FMT_STATS_MS: {
290 char *pre = (stats->ks_type == KTEST_STAT_MOD) ? "" : " ";
291
292 if (snprintf(buf, len, "%s%s", pre, stats->ks_name) >= len) {
293 return (B_FALSE);
294 }
295 break;
296 }
297
298 case KTEST_FMT_STATS_TOTAL:
299 if (snprintf(buf, len, "%" PRIu32, stats->ks_total) >= len) {
300 return (B_FALSE);
301 }
302 break;
303
304 case KTEST_FMT_STATS_PASS:
305 if (snprintf(buf, len, "%" PRIu32, stats->ks_pass) >= len) {
306 return (B_FALSE);
307 }
308 break;
309
310 case KTEST_FMT_STATS_FAIL:
311 if (snprintf(buf, len, "%" PRIu32, stats->ks_fail) >= len) {
312 return (B_FALSE);
313 }
314 break;
315
316 case KTEST_FMT_STATS_ERR:
317 if (snprintf(buf, len, "%" PRIu32, stats->ks_err) >= len) {
318 return (B_FALSE);
319 }
320 break;
321
322 case KTEST_FMT_STATS_SKIP:
323 if (snprintf(buf, len, "%" PRIu32, stats->ks_skip) >= len) {
324 return (B_FALSE);
325 }
326 break;
327
328 case KTEST_FMT_STATS_NONE:
329 if (snprintf(buf, len, "%" PRIu32, stats->ks_none) >= len) {
330 return (B_FALSE);
331 }
332 break;
333 }
334
335 return (B_TRUE);
336 }
337
338 #define KTEST_STATS_FIELDS "module/suite,total,pass,fail,err,skip,none"
339
340 static const ofmt_field_t ktest_stats_ofmt[] = {
341 { "MODULE/SUITE", 40, KTEST_FMT_STATS_MS, ktest_stats_ofmt_cb },
342 { "TOTAL", 6, KTEST_FMT_STATS_TOTAL, ktest_stats_ofmt_cb },
343 { "PASS", 6, KTEST_FMT_STATS_PASS, ktest_stats_ofmt_cb },
344 { "FAIL", 6, KTEST_FMT_STATS_FAIL, ktest_stats_ofmt_cb },
345 { "ERR", 6, KTEST_FMT_STATS_ERR, ktest_stats_ofmt_cb },
346 { "SKIP", 6, KTEST_FMT_STATS_SKIP, ktest_stats_ofmt_cb },
347 { "NONE", 6, KTEST_FMT_STATS_NONE, ktest_stats_ofmt_cb },
348 };
349
350 static void
ktest_usage(const char * fmt,...)351 ktest_usage(const char *fmt, ...)
352 {
353 if (fmt != NULL) {
354 va_list ap;
355
356 va_start(ap, fmt);
357 vwarnx(fmt, ap);
358 va_end(ap);
359 }
360
361 (void) fprintf(stderr,
362 "usage: %s <subcommand> [<opts>] [<args>]\n\n"
363 "\tlist [-H] [[-p] -o field,...] [<triple> ...]: "
364 "list registered tests\n"
365 "\trun [-Hn] [[-p] -o field,...] [-i <file>] <triple> ...: "
366 "run specified tests\n"
367 "\tload <name> | -a\n"
368 "\tunload <name> | -a\n",
369 ktest_prog);
370 }
371
372 /*
373 * A user-specified test triple. The input path is set to the empty
374 * string if no input is provided. The line number provides useful
375 * error reporting when an error is encountered in the run file.
376 */
377 typedef struct ktest_triple {
378 list_node_t ktr_node;
379 char *ktr_module;
380 char *ktr_suite;
381 char *ktr_test;
382 /* Did this triple match one or more tests during a ktest-run? */
383 boolean_t ktr_was_matched;
384 } ktest_triple_t;
385
386 /* Match-all triple used as default when user provides no triples */
387 static ktest_triple_t ktest_def_triple = {
388 .ktr_module = "*",
389 .ktr_suite = "*",
390 .ktr_test = "*",
391 };
392
393 static void
ktest_free_triples(list_t * triples)394 ktest_free_triples(list_t *triples)
395 {
396 ktest_triple_t *t = NULL;
397
398 while ((t = list_remove_head(triples)) != NULL) {
399 if (t == &ktest_def_triple) {
400 /*
401 * Default triple is not heap allocated, and is used
402 * only when no other matches are specified
403 */
404 VERIFY(list_is_empty(triples));
405 continue;
406 }
407 free(t->ktr_module);
408 free(t->ktr_suite);
409 free(t->ktr_test);
410 free(t);
411 }
412
413 list_destroy(triples);
414 }
415
416 /*
417 * Does the test entry match this triple?
418 */
419 static boolean_t
ktest_match_triple(const ktest_entry_t * ent,const ktest_triple_t * triple)420 ktest_match_triple(const ktest_entry_t *ent, const ktest_triple_t *triple)
421 {
422 return (gmatch(ent->ke_module, triple->ktr_module) != 0 &&
423 gmatch(ent->ke_suite, triple->ktr_suite) != 0 &&
424 gmatch(ent->ke_test, triple->ktr_test) != 0);
425 }
426
427 /*
428 * Does the test entry match any triples in the provided list?
429 *
430 * Returns a pointer to the matching triple, if one found.
431 */
432 static ktest_triple_t *
ktest_match_triples(const ktest_entry_t * ent,list_t * triples)433 ktest_match_triples(const ktest_entry_t *ent, list_t *triples)
434 {
435 for (ktest_triple_t *triple = list_head(triples);
436 triple != NULL;
437 triple = list_next(triples, triple)) {
438 if (ktest_match_triple(ent, triple)) {
439 return (triple);
440 }
441 }
442 return (NULL);
443 }
444
445 /*
446 * Attempt to parse the test triple string and return the resulting
447 * triple struct. This leaves the original triple string untouched.
448 *
449 * This function produces a warning when failing to parse a triple,
450 * this is on purpose. This allows the run file parser to produce an
451 * error that points out the line number with the bad triple.
452 */
453 static ktest_triple_t *
ktest_parse_triple(const char * tstr)454 ktest_parse_triple(const char *tstr)
455 {
456 char *cp = NULL, *orig = NULL;
457 char *module = NULL;
458 char *suite = NULL;
459 char *test = NULL;
460 ktest_triple_t *triple = NULL;
461
462 if ((triple = calloc(1, sizeof (*triple))) == NULL) {
463 warn("failed to allocate triple");
464 return (NULL);
465 }
466
467 if (strnlen(tstr, KTEST_MAX_TRIPLE_LEN) >= KTEST_MAX_TRIPLE_LEN) {
468 warnx("triple is too long");
469 goto fail;
470 }
471
472 if ((cp = strndup(tstr, KTEST_MAX_TRIPLE_LEN)) == NULL) {
473 warn("failed to dup triple string");
474 goto fail;
475 }
476
477 orig = cp;
478 module = strsep(&cp, KTEST_SEPARATOR);
479
480 if (strnlen(module, KTEST_MAX_NAME_LEN) >= KTEST_MAX_NAME_LEN) {
481 warnx("module pattern too long: %s", module);
482 goto fail;
483 }
484
485 if (*module == '\0') {
486 module = "*";
487 }
488
489 if (cp == NULL) {
490 suite = "*";
491 test = "*";
492 goto copy;
493 }
494
495 suite = strsep(&cp, KTEST_SEPARATOR);
496
497 if (strnlen(suite, KTEST_MAX_NAME_LEN) >= KTEST_MAX_NAME_LEN) {
498 warnx("suite pattern too long: %s", suite);
499 goto fail;
500 }
501
502 if (*suite == '\0') {
503 suite = "*";
504 }
505
506 if (cp == NULL) {
507 test = "*";
508 goto copy;
509 }
510
511 test = cp;
512
513 if (strstr(cp, KTEST_SEPARATOR) != NULL) {
514 warnx("malformed triple, unexpected ':' in test pattern: %s",
515 test);
516 goto fail;
517 }
518
519 if (strnlen(test, KTEST_MAX_NAME_LEN) >= KTEST_MAX_NAME_LEN) {
520 warnx("test pattern too long: %s", test);
521 goto fail;
522 }
523
524 if (*test == '\0') {
525 test = "*";
526 }
527
528 copy:
529 triple->ktr_module = strdup(module);
530 triple->ktr_suite = strdup(suite);
531 triple->ktr_test = strdup(test);
532 free(orig);
533 return (triple);
534
535 fail:
536 free(orig);
537 free(triple);
538 return (NULL);
539 }
540
541 static void
ktest_parse_triples(list_t * triples,uint_t count,const char * tinput[])542 ktest_parse_triples(list_t *triples, uint_t count, const char *tinput[])
543 {
544 list_create(triples, sizeof (ktest_triple_t),
545 offsetof(ktest_triple_t, ktr_node));
546
547 if (count == 0) {
548 list_insert_tail(triples, &ktest_def_triple);
549 return;
550 }
551
552 for (uint_t i = 0; i < count; i++) {
553 ktest_triple_t *triple = ktest_parse_triple(tinput[i]);
554
555 if (triple == NULL) {
556 errx(EXIT_FAILURE, "failed to parse triple: %s",
557 tinput[i]);
558 }
559
560 list_insert_tail(triples, triple);
561 }
562 }
563
564
565 /*
566 * Does the test entry match any of the triples?
567 */
568 static boolean_t
ktest_match_any(const ktest_entry_t * ent,list_t * triples)569 ktest_match_any(const ktest_entry_t *ent, list_t *triples)
570 {
571 for (ktest_triple_t *triple = list_head(triples); triple != NULL;
572 triple = list_next(triples, triple)) {
573 if (ktest_match_triple(ent, triple)) {
574 return (B_TRUE);
575 }
576 }
577
578 return (B_FALSE);
579 }
580
581 static void
ktest_print_stats(list_t * stats)582 ktest_print_stats(list_t *stats)
583 {
584 ktest_stats_t *stat;
585 ofmt_handle_t stats_ofmt;
586 ofmt_status_t oferr;
587 boolean_t first = B_FALSE;
588
589 oferr = ofmt_open(KTEST_STATS_FIELDS, ktest_stats_ofmt, 0, 0,
590 &stats_ofmt);
591 ofmt_check(oferr, B_FALSE, stats_ofmt, ktest_ofmt_errx, warnx);
592
593 for (stat = list_head(stats); stat != NULL;
594 stat = list_next(stats, stat)) {
595 if (!first && stat->ks_type == KTEST_STAT_MOD) {
596 printf("\n");
597 }
598
599 ofmt_print(stats_ofmt, stat);
600
601 if (stat->ks_type == KTEST_STAT_MOD) {
602 first = B_FALSE;
603 /* Print a 72-char horizontal rule. */
604 printf("-----------------------------------"
605 "-----------------------------------\n");
606 }
607 }
608
609 ofmt_close(stats_ofmt);
610 }
611
612 /*
613 * Read file at path into the byte array. If an error occurs, `err` will be
614 * populated with an allocated string describing the problem. If the file was
615 * read successfully, the allocated buffer will be placed in `bytes` with its
616 * size recorded in `len`.
617 */
618 static boolean_t
ktest_read_file(const char * path,uchar_t ** bytes,size_t * len,char ** err)619 ktest_read_file(const char *path, uchar_t **bytes, size_t *len, char **err)
620 {
621 FILE *fp = fopen(path, "r");
622 if (fp == NULL) {
623 *err = strdup("failed to open input file");
624 return (B_FALSE);
625 }
626
627 struct stat stats;
628 if (fstat(fileno(fp), &stats) == -1) {
629 (void) fclose(fp);
630 *err = strdup("failed to stat input file");
631 return (B_FALSE);
632 }
633
634 const size_t target_sz = (size_t)stats.st_size;
635 const size_t max_sz = ktest_max_input_size();
636 if (target_sz > max_sz) {
637 (void) fclose(fp);
638 (void) asprintf(err,
639 "input size greater than max of %u bytes", max_sz);
640 return (B_FALSE);
641 } else if (target_sz == 0) {
642 (void) fclose(fp);
643 *err = strdup("input file cannot be zero-length");
644 return (B_FALSE);
645 }
646
647 uchar_t *buf = malloc(target_sz);
648 if (buf == NULL) {
649 (void) fclose(fp);
650 *err = strdup("failed to allocate byte array of size");
651 return (B_FALSE);
652 }
653
654 if (fread(buf, 1, target_sz, fp) != target_sz) {
655 (void) fclose(fp);
656 (void) asprintf(err,
657 "failed to read %u bytes from input file", target_sz);
658 return (B_FALSE);
659 }
660
661 *bytes = buf;
662 *len = target_sz;
663 return (B_TRUE);
664 }
665
666 static boolean_t
ktest_run_test(ktest_hdl_t * kthdl,const ktest_entry_t * ent,char * input_path,ktest_stats_t * mod_stats,ktest_stats_t * suite_stats,ofmt_handle_t ofmt)667 ktest_run_test(ktest_hdl_t *kthdl, const ktest_entry_t *ent,
668 char *input_path, ktest_stats_t *mod_stats, ktest_stats_t *suite_stats,
669 ofmt_handle_t ofmt)
670 {
671 ktest_run_req_t req = {
672 .krq_module = ent->ke_module,
673 .krq_suite = ent->ke_suite,
674 .krq_test = ent->ke_test,
675 };
676 ktest_run_result_t res = { 0 };
677 ktest_run_output_t kro = {
678 .kro_req = &req,
679 .kro_result = &res,
680 .kro_input_path = input_path,
681 };
682
683 /* Fail with error when lacking input for test which requires it */
684 if (ent->ke_requires_input && input_path == NULL) {
685 res.krr_msg = strdup("test requires input and none provided");
686 res.krr_code = KTEST_CODE_ERROR;
687 ktest_record_stat(mod_stats, suite_stats, &res);
688 ofmt_print(ofmt, &kro);
689 free(res.krr_msg);
690 return (B_FALSE);
691 }
692
693 if (input_path != NULL) {
694 /* We treat a failure to read the input file as a test error. */
695 if (!ktest_read_file(input_path, &req.krq_input,
696 &req.krq_input_len, &res.krr_msg)) {
697 res.krr_code = KTEST_CODE_ERROR;
698 ktest_record_stat(mod_stats, suite_stats, &res);
699 ofmt_print(ofmt, &kro);
700 free(res.krr_msg);
701 return (B_FALSE);
702 }
703 }
704
705 if (!ktest_run(kthdl, &req, &res)) {
706 if (input_path != NULL) {
707 err(EXIT_FAILURE, "failed to run test %s:%s:%s with "
708 "input %s", ent->ke_module, ent->ke_suite,
709 ent->ke_test, input_path);
710 } else {
711 err(EXIT_FAILURE, "failed to run test %s:%s:%s",
712 ent->ke_module, ent->ke_suite, ent->ke_test);
713 }
714 }
715
716 ktest_record_stat(mod_stats, suite_stats, &res);
717 ofmt_print(ofmt, &kro);
718 free(res.krr_msg);
719 return (res.krr_code == KTEST_CODE_PASS ||
720 res.krr_code == KTEST_CODE_SKIP);
721 }
722
723 typedef enum ktest_run_test_flags {
724 KRTF_PRINT_STATS = (1 << 0),
725 KRTF_SKIP_INPUT_REQ = (1 << 1),
726 } ktest_run_test_flags_t;
727
728 /*
729 * Run all tests specified in the run list and print the result of each test.
730 *
731 * Returns the number of tests which failed.
732 */
733 static uint_t
ktest_run_tests(ktest_hdl_t * kthdl,list_t * run_list,char * input_path,ofmt_handle_t ofmt,ktest_run_test_flags_t flags)734 ktest_run_tests(ktest_hdl_t *kthdl, list_t *run_list, char *input_path,
735 ofmt_handle_t ofmt, ktest_run_test_flags_t flags)
736 {
737 ktest_stats_t *mod_stats = NULL;
738 ktest_stats_t *suite_stats = NULL;
739 list_t stats;
740 ktest_stats_t *stat = NULL;
741
742 list_create(&stats, sizeof (ktest_stats_t),
743 offsetof(ktest_stats_t, ks_node));
744
745 ktest_list_iter_t *iter = ktest_list(kthdl);
746 if (iter == NULL) {
747 err(EXIT_FAILURE, "Could not list ktests");
748 }
749
750 uint_t tests_matched = 0, tests_failed = 0;
751 ktest_entry_t ent;
752 while (ktest_list_next(iter, &ent)) {
753 ktest_triple_t *triple;
754 if ((triple = ktest_match_triples(&ent, run_list)) == NULL) {
755 continue;
756 }
757
758 if (ent.ke_requires_input && input_path == NULL &&
759 (flags & KRTF_SKIP_INPUT_REQ)) {
760 /*
761 * User has provided no input and requested that
762 * input-required tests be explicitly skipped.
763 */
764 continue;
765 }
766
767 /*
768 * Since this matching test will not be skipped for input
769 * reasons, record that its corresponding triple was used.
770 *
771 * This could be inadequate if the user specifies triples which
772 * overlap in their matches. We can make it more robust to such
773 * cases later.
774 */
775 triple->ktr_was_matched |= B_TRUE;
776 tests_matched++;
777
778 /*
779 * Either this is our first matching test or we are
780 * transitioning to a new module and/or suite. In either case,
781 * create new stats structures and add them to the list.
782 */
783 if (mod_stats == NULL ||
784 strcmp(mod_stats->ks_name, ent.ke_module) != 0) {
785 mod_stats = ktest_stats_new(KTEST_STAT_MOD,
786 ent.ke_module);
787 list_insert_tail(&stats, mod_stats);
788 }
789 if (suite_stats == NULL ||
790 strcmp(suite_stats->ks_name, ent.ke_suite) != 0) {
791 suite_stats = ktest_stats_new(KTEST_STAT_SUITE,
792 ent.ke_suite);
793 list_insert_tail(&stats, suite_stats);
794 }
795
796 /* Run the test */
797 if (!ktest_run_test(kthdl, &ent, input_path,
798 mod_stats, suite_stats, ofmt)) {
799 tests_failed++;
800 }
801 }
802
803 /* Make sure we ran _something_ */
804 if (tests_matched == 0) {
805 errx(EXIT_FAILURE, "No tests matched selection triple(s)");
806 }
807
808 /* Confirm that all triples matched something */
809 boolean_t fail_match = B_FALSE;
810 for (ktest_triple_t *triple = list_head(run_list);
811 triple != NULL;
812 triple = list_next(run_list, triple)) {
813 if (!triple->ktr_was_matched) {
814 fail_match = B_TRUE;
815 break;
816 }
817 }
818 if (fail_match) {
819 (void) fprintf(stderr, "These triples failed to match "
820 "any tests, or were superseded by other matches:\n");
821 for (ktest_triple_t *triple = list_head(run_list);
822 triple != NULL;
823 triple = list_next(run_list, triple)) {
824 if (!triple->ktr_was_matched) {
825 (void) fprintf(stderr, "\t%s:%s:%s\n",
826 triple->ktr_module, triple->ktr_suite,
827 triple->ktr_test);
828 }
829 }
830 exit(EXIT_FAILURE);
831 }
832
833 if (flags & KRTF_PRINT_STATS) {
834 printf("\n");
835 ktest_print_stats(&stats);
836 }
837
838 while ((stat = list_remove_head(&stats)) != NULL) {
839 free(stat);
840 }
841 list_destroy(&stats);
842 free(iter);
843 return (tests_failed);
844 }
845
846 static void
ktest_run_cmd(int argc,char * argv[])847 ktest_run_cmd(int argc, char *argv[])
848 {
849 int c;
850 char *input_path = NULL;
851 boolean_t parsable = B_FALSE;
852 boolean_t fields_set = B_FALSE;
853 ktest_run_test_flags_t flags = KRTF_PRINT_STATS;
854 char *fields = KTEST_RUN_CMD_DEF_FIELDS;
855 uint_t oflags = 0;
856 ofmt_handle_t ofmt = NULL;
857 ofmt_status_t oferr;
858
859 while ((c = getopt(argc, argv, ":Hno:pi:")) != -1) {
860 switch (c) {
861 case 'H':
862 oflags |= OFMT_NOHEADER;
863 break;
864 case 'n':
865 flags |= KRTF_SKIP_INPUT_REQ;
866 break;
867 case 'o':
868 fields = optarg;
869 fields_set = B_TRUE;
870 break;
871 case 'p':
872 parsable = B_TRUE;
873 flags &= ~KRTF_PRINT_STATS;
874 oflags |= OFMT_PARSABLE;
875 break;
876 case 'i':
877 if (input_path != NULL) {
878 ktest_usage("cannot specify -i more than once");
879 exit(EXIT_USAGE);
880 }
881
882 input_path = optarg;
883
884 if (strnlen(input_path, MAXPATHLEN) >= MAXPATHLEN) {
885 err(EXIT_FAILURE, "input path too long");
886 }
887
888 break;
889 case ':':
890 ktest_usage("missing argument to -%c", optopt);
891 exit(EXIT_USAGE);
892
893 case '?':
894 ktest_usage("unknown run option: -%c", optopt);
895 exit(EXIT_USAGE);
896 }
897 }
898
899 if (parsable && !fields_set) {
900 ktest_usage("must specify -o with -p");
901 exit(EXIT_USAGE);
902 }
903
904 oferr = ofmt_open(fields, ktest_run_ofmt, oflags, 0, &ofmt);
905 ofmt_check(oferr, parsable, ofmt, ktest_ofmt_errx, warnx);
906
907 argc -= optind;
908 argv += optind;
909
910 /*
911 * We don't run all tests by default. We assume that as the library of
912 * test modules grows we want to be sure the user actually wants to run
913 * all tests by forcing them to at least specify the `*` glob.
914 */
915 if (argc < 1) {
916 ktest_usage("must specify at least one triple");
917 exit(EXIT_USAGE);
918 }
919 list_t triples;
920 ktest_parse_triples(&triples, argc, (const char **)argv);
921
922 ktest_hdl_t *kthdl = ktest_init();
923 if (kthdl == NULL) {
924 err(EXIT_FAILURE, "Could not open ktest");
925 }
926 uint_t failed_count =
927 ktest_run_tests(kthdl, &triples, input_path, ofmt, flags);
928
929 ofmt_close(ofmt);
930 ktest_free_triples(&triples);
931 ktest_fini(kthdl);
932
933 if (failed_count != 0) {
934 errx(EXIT_FAILURE, "%u %s did not pass",
935 failed_count, failed_count > 1 ? "tests" : "test");
936 }
937 }
938
939 static void
ktest_list_cmd(int argc,char * argv[])940 ktest_list_cmd(int argc, char *argv[])
941 {
942 int c;
943 boolean_t parsable = B_FALSE;
944 boolean_t fields_set = B_FALSE;
945 char *fields = KTEST_LIST_CMD_DEF_FIELDS;
946 uint_t oflags = 0;
947 ofmt_handle_t list_ofmt = NULL;
948 ofmt_status_t oferr;
949
950 while ((c = getopt(argc, argv, ":Ho:p")) != -1) {
951 switch (c) {
952 case 'H':
953 oflags |= OFMT_NOHEADER;
954 break;
955 case 'o':
956 fields = optarg;
957 fields_set = B_TRUE;
958 break;
959 case 'p':
960 parsable = B_TRUE;
961 oflags |= OFMT_PARSABLE;
962 break;
963 case ':':
964 ktest_usage("missing argument to -%c", optopt);
965 exit(EXIT_USAGE);
966 case '?':
967 ktest_usage("unknown option: -%c", optopt);
968 exit(EXIT_USAGE);
969 }
970 }
971
972 if (parsable && !fields_set) {
973 ktest_usage("must specify -o with -p");
974 exit(EXIT_USAGE);
975 }
976
977 oferr = ofmt_open(fields, ktest_list_ofmt, oflags, 0, &list_ofmt);
978 ofmt_check(oferr, parsable, list_ofmt, ktest_ofmt_errx, warnx);
979
980 argc -= optind;
981 argv += optind;
982
983 list_t triples;
984 ktest_parse_triples(&triples, argc, (const char **)argv);
985
986 ktest_hdl_t *kthdl = ktest_init();
987 if (kthdl == NULL) {
988 err(EXIT_FAILURE, "Could not open ktest");
989 }
990 ktest_list_iter_t *iter = ktest_list(kthdl);
991 if (iter == NULL) {
992 err(EXIT_FAILURE, "Could not list ktests");
993 }
994
995 ktest_entry_t ent;
996 while (ktest_list_next(iter, &ent)) {
997 if (!ktest_match_any(&ent, &triples)) {
998 continue;
999 }
1000 ofmt_print(list_ofmt, &ent);
1001 }
1002
1003 ktest_list_free(iter);
1004 ktest_fini(kthdl);
1005
1006 ofmt_close(list_ofmt);
1007 ktest_free_triples(&triples);
1008 }
1009
1010 static void
ktest_load_cmd(int argc,char * argv[])1011 ktest_load_cmd(int argc, char *argv[])
1012 {
1013 int c;
1014 boolean_t load_all = B_FALSE;
1015
1016 while ((c = getopt(argc, argv, "a")) != -1) {
1017 switch (c) {
1018 case 'a':
1019 load_all = B_TRUE;
1020 break;
1021 case '?':
1022 ktest_usage("unknown option: -%c", optopt);
1023 exit(EXIT_USAGE);
1024 }
1025 }
1026
1027 argc -= optind;
1028 argv += optind;
1029
1030 if (load_all) {
1031 /*
1032 * We just ignore specified module names if the user requested
1033 * that everything should be loaded.
1034 */
1035 if (!ktest_mod_load_all()) {
1036 err(EXIT_FAILURE, "Could not load all ktests");
1037 }
1038 return;
1039 }
1040 if (argc <= 0) {
1041 ktest_usage("must specify module name(s) or -a");
1042 exit(EXIT_USAGE);
1043 }
1044 boolean_t any_failed = B_FALSE;
1045 for (int i = 0; i < argc; i++) {
1046 if (!ktest_mod_load(argv[i])) {
1047 any_failed = B_TRUE;
1048 warn("Could not load module %s", argv[i]);
1049 }
1050 }
1051 if (any_failed) {
1052 errx(EXIT_FAILURE, "Some modules failed to load");
1053 }
1054 }
1055
1056 static void
ktest_unload_cmd(int argc,char * argv[])1057 ktest_unload_cmd(int argc, char *argv[])
1058 {
1059 int c;
1060 boolean_t unload_all = B_FALSE;
1061
1062 while ((c = getopt(argc, argv, "a")) != -1) {
1063 switch (c) {
1064 case 'a':
1065 unload_all = B_TRUE;
1066 break;
1067 case '?':
1068 ktest_usage("unknown option: -%c", optopt);
1069 exit(EXIT_USAGE);
1070 }
1071 }
1072
1073 argc -= optind;
1074 argv += optind;
1075
1076 if (unload_all) {
1077 /*
1078 * We just ignore specified module names if the user requested
1079 * that everything should be unloaded.
1080 */
1081 if (!ktest_mod_unload_all()) {
1082 err(EXIT_FAILURE, "Could not unload all ktests");
1083 }
1084 return;
1085 }
1086 if (argc <= 0) {
1087 ktest_usage("must specify module name(s) or -a");
1088 exit(EXIT_USAGE);
1089 }
1090 for (int i = 0; i < argc; i++) {
1091 ktest_mod_unload(argv[i]);
1092 }
1093 }
1094
1095 int
main(int argc,char * argv[])1096 main(int argc, char *argv[])
1097 {
1098 const char *cmd;
1099
1100 ktest_prog = basename(argv[0]);
1101
1102 if (getzoneid() != GLOBAL_ZONEID || getuid() != 0) {
1103 errx(EXIT_FAILURE, "can only be used by root from"
1104 " the global zone");
1105 }
1106
1107 if (argc < 2) {
1108 ktest_usage("no command specified");
1109 exit(EXIT_USAGE);
1110 }
1111
1112 /*
1113 * Peel off program name and command.
1114 */
1115 cmd = argv[1];
1116 argc -= 2;
1117 argv += 2;
1118 optind = 0;
1119
1120 if (strncasecmp("list", cmd, KTEST_CMD_SZ) == 0) {
1121 ktest_list_cmd(argc, argv);
1122 } else if (strncasecmp("run", cmd, KTEST_CMD_SZ) == 0) {
1123 ktest_run_cmd(argc, argv);
1124 } else if (strncasecmp("load", cmd, KTEST_CMD_SZ) == 0) {
1125 ktest_load_cmd(argc, argv);
1126 } else if (strncasecmp("unload", cmd, KTEST_CMD_SZ) == 0) {
1127 ktest_unload_cmd(argc, argv);
1128 } else if (strncasecmp("help", cmd, KTEST_CMD_SZ) == 0) {
1129 ktest_usage(NULL);
1130 } else {
1131 ktest_usage("unknown command: %s", cmd);
1132 exit(EXIT_USAGE);
1133 }
1134
1135 return (EXIT_SUCCESS);
1136 }
1137