xref: /illumos-gate/usr/src/cmd/raidz_test/raidz_test.c (revision 1a2d662a91cee3bf82f41cd47c7ae6f3825d9db2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
24  * Copyright 2020 Joyent, Inc.
25  */
26 
27 #include <sys/zfs_context.h>
28 #include <sys/time.h>
29 #include <sys/wait.h>
30 #include <sys/zio.h>
31 #include <umem.h>
32 #include <sys/vdev_raidz.h>
33 #include <sys/vdev_raidz_impl.h>
34 #include <assert.h>
35 #include <stdio.h>
36 #include <strings.h>
37 #include <unistd.h>
38 #include "raidz_test.h"
39 
40 static int *rand_data;
41 raidz_test_opts_t rto_opts;
42 
43 static char gdb[256];
44 static const char gdb_tmpl[] = "gdb -ex \"set pagination 0\" -p %d";
45 
46 #define	boot_ncpus	(sysconf(_SC_NPROCESSORS_ONLN))
47 
48 static size_t ilog2(size_t a)
49 {
50 	return (a > 1 ? 1 + ilog2(a >> 1) : 0);
51 }
52 
53 static void print_opts(raidz_test_opts_t *opts, boolean_t force)
54 {
55 	char *verbose;
56 	switch (opts->rto_v) {
57 		case 0:
58 			verbose = "no";
59 			break;
60 		case 1:
61 			verbose = "info";
62 			break;
63 		default:
64 			verbose = "debug";
65 			break;
66 	}
67 
68 	if (force || opts->rto_v >= D_INFO) {
69 		(void) fprintf(stdout, DBLSEP "Running with options:\n"
70 		    "  (-a) zio ashift                   : %zu\n"
71 		    "  (-o) zio offset                   : 1 << %zu\n"
72 		    "  (-d) number of raidz data columns : %zu\n"
73 		    "  (-s) size of DATA                 : 1 << %zu\n"
74 		    "  (-S) sweep parameters             : %s \n"
75 		    "  (-v) verbose                      : %s \n\n",
76 		    opts->rto_ashift,			/* -a */
77 		    ilog2(opts->rto_offset),		/* -o */
78 		    opts->rto_dcols,			/* -d */
79 		    ilog2(opts->rto_dsize),		/* -s */
80 		    opts->rto_sweep ? "yes" : "no",	/* -S */
81 		    verbose);				/* -v */
82 	}
83 }
84 
85 static void usage(boolean_t requested)
86 {
87 	const raidz_test_opts_t *o = &rto_opts_defaults;
88 
89 	FILE *fp = requested ? stdout : stderr;
90 
91 	(void) fprintf(fp, "Usage:\n"
92 	    "\t[-a zio ashift (default: %zu)]\n"
93 	    "\t[-o zio offset, exponent radix 2 (default: %zu)]\n"
94 	    "\t[-d number of raidz data columns (default: %zu)]\n"
95 	    "\t[-s zio size, exponent radix 2 (default: %zu)]\n"
96 	    "\t[-S parameter sweep (default: %s)]\n"
97 	    "\t[-t timeout for parameter sweep test]\n"
98 	    "\t[-B benchmark all raidz implementations]\n"
99 	    "\t[-v increase verbosity (default: %zu)]\n"
100 	    "\t[-h (print help)]\n"
101 	    "\t[-T test the test, see if failure would be detected]\n"
102 	    "\t[-D debug (attach gdb on SIGSEGV)]\n"
103 	    "",
104 	    o->rto_ashift,				/* -a */
105 	    ilog2(o->rto_offset),			/* -o */
106 	    o->rto_dcols,				/* -d */
107 	    ilog2(o->rto_dsize),			/* -s */
108 	    rto_opts.rto_sweep ? "yes" : "no",		/* -S */
109 	    o->rto_v);					/* -d */
110 
111 	exit(requested ? 0 : 1);
112 }
113 
114 static void process_options(int argc, char **argv)
115 {
116 	size_t value;
117 	int opt;
118 
119 	raidz_test_opts_t *o = &rto_opts;
120 
121 	bcopy(&rto_opts_defaults, o, sizeof (*o));
122 
123 	while ((opt = getopt(argc, argv, "TDBSvha:o:d:s:t:")) != -1) {
124 		value = 0;
125 
126 		switch (opt) {
127 		case 'a':
128 			value = strtoull(optarg, NULL, 0);
129 			o->rto_ashift = MIN(13, MAX(9, value));
130 			break;
131 		case 'o':
132 			value = strtoull(optarg, NULL, 0);
133 			o->rto_offset = ((1ULL << MIN(12, value)) >> 9) << 9;
134 			break;
135 		case 'd':
136 			value = strtoull(optarg, NULL, 0);
137 			o->rto_dcols = MIN(255, MAX(1, value));
138 			break;
139 		case 's':
140 			value = strtoull(optarg, NULL, 0);
141 			o->rto_dsize = 1ULL <<  MIN(SPA_MAXBLOCKSHIFT,
142 			    MAX(SPA_MINBLOCKSHIFT, value));
143 			break;
144 		case 't':
145 			value = strtoull(optarg, NULL, 0);
146 			o->rto_sweep_timeout = value;
147 			break;
148 		case 'v':
149 			o->rto_v++;
150 			break;
151 		case 'S':
152 			o->rto_sweep = 1;
153 			break;
154 		case 'B':
155 			o->rto_benchmark = 1;
156 			break;
157 		case 'D':
158 			o->rto_gdb = 1;
159 			break;
160 		case 'T':
161 			o->rto_sanity = 1;
162 			break;
163 		case 'h':
164 			usage(B_TRUE);
165 			break;
166 		case '?':
167 		default:
168 			usage(B_FALSE);
169 			break;
170 		}
171 	}
172 }
173 
174 #define	DATA_COL(rm, i) ((rm)->rm_col[raidz_parity(rm) + (i)].rc_abd)
175 #define	DATA_COL_SIZE(rm, i) ((rm)->rm_col[raidz_parity(rm) + (i)].rc_size)
176 
177 #define	CODE_COL(rm, i) ((rm)->rm_col[(i)].rc_abd)
178 #define	CODE_COL_SIZE(rm, i) ((rm)->rm_col[(i)].rc_size)
179 
180 static int
181 cmp_code(raidz_test_opts_t *opts, const raidz_map_t *rm, const int parity)
182 {
183 	int i, ret = 0;
184 
185 	VERIFY(parity >= 1 && parity <= 3);
186 
187 	for (i = 0; i < parity; i++) {
188 		if (abd_cmp(CODE_COL(rm, i), CODE_COL(opts->rm_golden, i),
189 		    CODE_COL(rm, i)->abd_size) != 0) {
190 			ret++;
191 			LOG_OPT(D_DEBUG, opts,
192 			    "\nParity block [%d] different!\n", i);
193 		}
194 	}
195 	return (ret);
196 }
197 
198 static int
199 cmp_data(raidz_test_opts_t *opts, raidz_map_t *rm)
200 {
201 	int i, ret = 0;
202 	int dcols = opts->rm_golden->rm_cols - raidz_parity(opts->rm_golden);
203 
204 	for (i = 0; i < dcols; i++) {
205 		if (abd_cmp(DATA_COL(opts->rm_golden, i), DATA_COL(rm, i),
206 		    DATA_COL(opts->rm_golden, i)->abd_size) != 0) {
207 			ret++;
208 
209 			LOG_OPT(D_DEBUG, opts,
210 			    "\nData block [%d] different!\n", i);
211 		}
212 	}
213 	return (ret);
214 }
215 
216 static int
217 init_rand(void *data, size_t size, void *private)
218 {
219 	int i;
220 	int *dst = (int *)data;
221 
222 	for (i = 0; i < size / sizeof (int); i++)
223 		dst[i] = rand_data[i];
224 
225 	return (0);
226 }
227 
228 static void
229 corrupt_colums(raidz_map_t *rm, const int *tgts, const int cnt)
230 {
231 	int i;
232 	raidz_col_t *col;
233 
234 	for (i = 0; i < cnt; i++) {
235 		col = &rm->rm_col[tgts[i]];
236 		(void) abd_iterate_func(col->rc_abd, 0, col->rc_size,
237 		    init_rand, NULL);
238 	}
239 }
240 
241 void
242 init_zio_abd(zio_t *zio)
243 {
244 	(void) abd_iterate_func(zio->io_abd, 0, zio->io_size, init_rand, NULL);
245 }
246 
247 static void
248 fini_raidz_map(zio_t **zio, raidz_map_t **rm)
249 {
250 	vdev_raidz_map_free(*rm);
251 	raidz_free((*zio)->io_abd, (*zio)->io_size);
252 	umem_free(*zio, sizeof (zio_t));
253 
254 	*zio = NULL;
255 	*rm = NULL;
256 }
257 
258 static int
259 init_raidz_golden_map(raidz_test_opts_t *opts, const int parity)
260 {
261 	int err = 0;
262 	zio_t *zio_test;
263 	raidz_map_t *rm_test;
264 	const size_t total_ncols = opts->rto_dcols + parity;
265 
266 	if (opts->rm_golden) {
267 		fini_raidz_map(&opts->zio_golden, &opts->rm_golden);
268 	}
269 
270 	opts->zio_golden = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL);
271 	zio_test = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL);
272 
273 	opts->zio_golden->io_offset = zio_test->io_offset = opts->rto_offset;
274 	opts->zio_golden->io_size = zio_test->io_size = opts->rto_dsize;
275 
276 	opts->zio_golden->io_abd = raidz_alloc(opts->rto_dsize);
277 	zio_test->io_abd = raidz_alloc(opts->rto_dsize);
278 
279 	init_zio_abd(opts->zio_golden);
280 	init_zio_abd(zio_test);
281 
282 	VERIFY0(vdev_raidz_impl_set("original"));
283 
284 	opts->rm_golden = vdev_raidz_map_alloc(opts->zio_golden,
285 	    opts->rto_ashift, total_ncols, parity);
286 	rm_test = vdev_raidz_map_alloc(zio_test,
287 	    opts->rto_ashift, total_ncols, parity);
288 
289 	VERIFY(opts->zio_golden);
290 	VERIFY(opts->rm_golden);
291 
292 	vdev_raidz_generate_parity(opts->rm_golden);
293 	vdev_raidz_generate_parity(rm_test);
294 
295 	/* sanity check */
296 	err |= cmp_data(opts, rm_test);
297 	err |= cmp_code(opts, rm_test, parity);
298 
299 	if (err)
300 		ERRMSG("initializing the golden copy ... [FAIL]!\n");
301 
302 	/* tear down raidz_map of test zio */
303 	fini_raidz_map(&zio_test, &rm_test);
304 
305 	return (err);
306 }
307 
308 static raidz_map_t *
309 init_raidz_map(raidz_test_opts_t *opts, zio_t **zio, const int parity)
310 {
311 	raidz_map_t *rm = NULL;
312 	const size_t alloc_dsize = opts->rto_dsize;
313 	const size_t total_ncols = opts->rto_dcols + parity;
314 	const int ccols[] = { 0, 1, 2 };
315 
316 	VERIFY(zio);
317 	VERIFY(parity <= 3 && parity >= 1);
318 
319 	*zio = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL);
320 
321 	(*zio)->io_offset = 0;
322 	(*zio)->io_size = alloc_dsize;
323 	(*zio)->io_abd = raidz_alloc(alloc_dsize);
324 	init_zio_abd(*zio);
325 
326 	rm = vdev_raidz_map_alloc(*zio, opts->rto_ashift,
327 	    total_ncols, parity);
328 	VERIFY(rm);
329 
330 	/* Make sure code columns are destroyed */
331 	corrupt_colums(rm, ccols, parity);
332 
333 	return (rm);
334 }
335 
336 static int
337 run_gen_check(raidz_test_opts_t *opts)
338 {
339 	char **impl_name;
340 	int fn, err = 0;
341 	zio_t *zio_test;
342 	raidz_map_t *rm_test;
343 
344 	err = init_raidz_golden_map(opts, PARITY_PQR);
345 	if (0 != err)
346 		return (err);
347 
348 	LOG(D_INFO, DBLSEP);
349 	LOG(D_INFO, "Testing parity generation...\n");
350 
351 	for (impl_name = (char **)raidz_impl_names+1; *impl_name != NULL;
352 	    impl_name++) {
353 
354 		LOG(D_INFO, SEP);
355 		LOG(D_INFO, "\tTesting [%s] implementation...", *impl_name);
356 
357 		if (0 != vdev_raidz_impl_set(*impl_name)) {
358 			LOG(D_INFO, "[SKIP]\n");
359 			continue;
360 		} else {
361 			LOG(D_INFO, "[SUPPORTED]\n");
362 		}
363 
364 		for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
365 
366 			/* Check if should stop */
367 			if (rto_opts.rto_should_stop)
368 				return (err);
369 
370 			/* create suitable raidz_map */
371 			rm_test = init_raidz_map(opts, &zio_test, fn+1);
372 			VERIFY(rm_test);
373 
374 			LOG(D_INFO, "\t\tTesting method [%s] ...",
375 			    raidz_gen_name[fn]);
376 
377 			if (!opts->rto_sanity)
378 				vdev_raidz_generate_parity(rm_test);
379 
380 			if (cmp_code(opts, rm_test, fn+1) != 0) {
381 				LOG(D_INFO, "[FAIL]\n");
382 				err++;
383 			} else
384 				LOG(D_INFO, "[PASS]\n");
385 
386 			fini_raidz_map(&zio_test, &rm_test);
387 		}
388 	}
389 
390 	fini_raidz_map(&opts->zio_golden, &opts->rm_golden);
391 
392 	return (err);
393 }
394 
395 static int
396 run_rec_check_impl(raidz_test_opts_t *opts, raidz_map_t *rm, const int fn)
397 {
398 	int x0, x1, x2;
399 	int tgtidx[3];
400 	int err = 0;
401 	static const int rec_tgts[7][3] = {
402 		{1, 2, 3},	/* rec_p:   bad QR & D[0]	*/
403 		{0, 2, 3},	/* rec_q:   bad PR & D[0]	*/
404 		{0, 1, 3},	/* rec_r:   bad PQ & D[0]	*/
405 		{2, 3, 4},	/* rec_pq:  bad R  & D[0][1]	*/
406 		{1, 3, 4},	/* rec_pr:  bad Q  & D[0][1]	*/
407 		{0, 3, 4},	/* rec_qr:  bad P  & D[0][1]	*/
408 		{3, 4, 5}	/* rec_pqr: bad    & D[0][1][2] */
409 	};
410 
411 	memcpy(tgtidx, rec_tgts[fn], sizeof (tgtidx));
412 
413 	if (fn < RAIDZ_REC_PQ) {
414 		/* can reconstruct 1 failed data disk */
415 		for (x0 = 0; x0 < opts->rto_dcols; x0++) {
416 			if (x0 >= rm->rm_cols - raidz_parity(rm))
417 				continue;
418 
419 			/* Check if should stop */
420 			if (rto_opts.rto_should_stop)
421 				return (err);
422 
423 			LOG(D_DEBUG, "[%d] ", x0);
424 
425 			tgtidx[2] = x0 + raidz_parity(rm);
426 
427 			corrupt_colums(rm, tgtidx+2, 1);
428 
429 			if (!opts->rto_sanity)
430 				(void) vdev_raidz_reconstruct(rm, tgtidx, 3);
431 
432 			if (cmp_data(opts, rm) != 0) {
433 				err++;
434 				LOG(D_DEBUG, "\nREC D[%d]... [FAIL]\n", x0);
435 			}
436 		}
437 
438 	} else if (fn < RAIDZ_REC_PQR) {
439 		/* can reconstruct 2 failed data disk */
440 		for (x0 = 0; x0 < opts->rto_dcols; x0++) {
441 			if (x0 >= rm->rm_cols - raidz_parity(rm))
442 				continue;
443 			for (x1 = x0 + 1; x1 < opts->rto_dcols; x1++) {
444 				if (x1 >= rm->rm_cols - raidz_parity(rm))
445 					continue;
446 
447 				/* Check if should stop */
448 				if (rto_opts.rto_should_stop)
449 					return (err);
450 
451 				LOG(D_DEBUG, "[%d %d] ", x0, x1);
452 
453 				tgtidx[1] = x0 + raidz_parity(rm);
454 				tgtidx[2] = x1 + raidz_parity(rm);
455 
456 				corrupt_colums(rm, tgtidx+1, 2);
457 
458 				if (!opts->rto_sanity)
459 					(void) vdev_raidz_reconstruct(rm,
460 					    tgtidx, 3);
461 
462 				if (cmp_data(opts, rm) != 0) {
463 					err++;
464 					LOG(D_DEBUG, "\nREC D[%d %d]... "
465 					    "[FAIL]\n", x0, x1);
466 				}
467 			}
468 		}
469 	} else {
470 		/* can reconstruct 3 failed data disk */
471 		for (x0 = 0; x0 < opts->rto_dcols; x0++) {
472 			if (x0 >= rm->rm_cols - raidz_parity(rm))
473 				continue;
474 			for (x1 = x0 + 1; x1 < opts->rto_dcols; x1++) {
475 				if (x1 >= rm->rm_cols - raidz_parity(rm))
476 					continue;
477 				for (x2 = x1 + 1; x2 < opts->rto_dcols; x2++) {
478 					if (x2 >=
479 					    rm->rm_cols - raidz_parity(rm))
480 						continue;
481 
482 					/* Check if should stop */
483 					if (rto_opts.rto_should_stop)
484 						return (err);
485 
486 					LOG(D_DEBUG, "[%d %d %d]", x0, x1, x2);
487 
488 					tgtidx[0] = x0 + raidz_parity(rm);
489 					tgtidx[1] = x1 + raidz_parity(rm);
490 					tgtidx[2] = x2 + raidz_parity(rm);
491 
492 					corrupt_colums(rm, tgtidx, 3);
493 
494 					if (!opts->rto_sanity)
495 						(void) vdev_raidz_reconstruct(
496 						    rm, tgtidx, 3);
497 
498 					if (cmp_data(opts, rm) != 0) {
499 						err++;
500 						LOG(D_DEBUG,
501 						    "\nREC D[%d %d %d]... "
502 						    "[FAIL]\n", x0, x1, x2);
503 					}
504 				}
505 			}
506 		}
507 	}
508 	return (err);
509 }
510 
511 static int
512 run_rec_check(raidz_test_opts_t *opts)
513 {
514 	char **impl_name;
515 	unsigned fn, err = 0;
516 	zio_t *zio_test;
517 	raidz_map_t *rm_test;
518 
519 	err = init_raidz_golden_map(opts, PARITY_PQR);
520 	if (0 != err)
521 		return (err);
522 
523 	LOG(D_INFO, DBLSEP);
524 	LOG(D_INFO, "Testing data reconstruction...\n");
525 
526 	for (impl_name = (char **)raidz_impl_names+1; *impl_name != NULL;
527 	    impl_name++) {
528 
529 		LOG(D_INFO, SEP);
530 		LOG(D_INFO, "\tTesting [%s] implementation...", *impl_name);
531 
532 		if (vdev_raidz_impl_set(*impl_name) != 0) {
533 			LOG(D_INFO, "[SKIP]\n");
534 			continue;
535 		} else
536 			LOG(D_INFO, "[SUPPORTED]\n");
537 
538 
539 		/* create suitable raidz_map */
540 		rm_test = init_raidz_map(opts, &zio_test, PARITY_PQR);
541 		/* generate parity */
542 		vdev_raidz_generate_parity(rm_test);
543 
544 		for (fn = 0; fn < RAIDZ_REC_NUM; fn++) {
545 
546 			LOG(D_INFO, "\t\tTesting method [%s] ...",
547 			    raidz_rec_name[fn]);
548 
549 			if (run_rec_check_impl(opts, rm_test, fn) != 0) {
550 				LOG(D_INFO, "[FAIL]\n");
551 				err++;
552 
553 			} else
554 				LOG(D_INFO, "[PASS]\n");
555 
556 		}
557 		/* tear down test raidz_map */
558 		fini_raidz_map(&zio_test, &rm_test);
559 	}
560 
561 	fini_raidz_map(&opts->zio_golden, &opts->rm_golden);
562 
563 	return (err);
564 }
565 
566 static int
567 run_test(raidz_test_opts_t *opts)
568 {
569 	int err = 0;
570 
571 	if (opts == NULL)
572 		opts = &rto_opts;
573 
574 	print_opts(opts, B_FALSE);
575 
576 	err |= run_gen_check(opts);
577 	err |= run_rec_check(opts);
578 
579 	return (err);
580 }
581 
582 #define	SWEEP_RUNNING	0
583 #define	SWEEP_FINISHED	1
584 #define	SWEEP_ERROR	2
585 #define	SWEEP_TIMEOUT	3
586 
587 static int sweep_state = 0;
588 static raidz_test_opts_t failed_opts;
589 
590 static kmutex_t sem_mtx;
591 static kcondvar_t sem_cv;
592 static int max_free_slots;
593 static int free_slots;
594 
595 static void
596 sweep_thread(void *arg)
597 {
598 	int err = 0;
599 	raidz_test_opts_t *opts = (raidz_test_opts_t *)arg;
600 	VERIFY(opts != NULL);
601 
602 	err = run_test(opts);
603 
604 	if (rto_opts.rto_sanity) {
605 		/* 25% chance that a sweep test fails */
606 		if (rand() < (RAND_MAX/4))
607 			err = 1;
608 	}
609 
610 	if (0 != err) {
611 		mutex_enter(&sem_mtx);
612 		memcpy(&failed_opts, opts, sizeof (raidz_test_opts_t));
613 		sweep_state = SWEEP_ERROR;
614 		mutex_exit(&sem_mtx);
615 	}
616 
617 	umem_free(opts, sizeof (raidz_test_opts_t));
618 
619 	/* signal the next thread */
620 	mutex_enter(&sem_mtx);
621 	free_slots++;
622 	cv_signal(&sem_cv);
623 	mutex_exit(&sem_mtx);
624 
625 	thread_exit();
626 }
627 
628 static int
629 run_sweep(void)
630 {
631 	static const size_t dcols_v[] = { 1, 2, 3, 4, 5, 6, 7, 8, 12, 15, 16 };
632 	static const size_t ashift_v[] = { 9, 12, 14 };
633 	static const size_t size_v[] = { 1 << 9, 21 * (1 << 9), 13 * (1 << 12),
634 		1 << 17, (1 << 20) - (1 << 12), SPA_MAXBLOCKSIZE };
635 
636 	(void) setvbuf(stdout, NULL, _IONBF, 0);
637 
638 	ulong_t total_comb = ARRAY_SIZE(size_v) * ARRAY_SIZE(ashift_v) *
639 	    ARRAY_SIZE(dcols_v);
640 	ulong_t tried_comb = 0;
641 	hrtime_t time_diff, start_time = gethrtime();
642 	raidz_test_opts_t *opts;
643 	int a, d, s;
644 
645 	max_free_slots = free_slots = MAX(2, boot_ncpus);
646 
647 	mutex_init(&sem_mtx, NULL, MUTEX_DEFAULT, NULL);
648 	cv_init(&sem_cv, NULL, CV_DEFAULT, NULL);
649 
650 	for (s = 0; s < ARRAY_SIZE(size_v); s++)
651 	for (a = 0; a < ARRAY_SIZE(ashift_v); a++)
652 	for (d = 0; d < ARRAY_SIZE(dcols_v); d++) {
653 
654 		if (size_v[s] < (1 << ashift_v[a])) {
655 			total_comb--;
656 			continue;
657 		}
658 
659 		if (++tried_comb % 20 == 0)
660 			LOG(D_ALL, "%lu/%lu... ", tried_comb, total_comb);
661 
662 		/* wait for signal to start new thread */
663 		mutex_enter(&sem_mtx);
664 		while (cv_timedwait_sig(&sem_cv, &sem_mtx,
665 		    ddi_get_lbolt() + hz)) {
666 
667 			/* check if should stop the test (timeout) */
668 			time_diff = (gethrtime() - start_time) / NANOSEC;
669 			if (rto_opts.rto_sweep_timeout > 0 &&
670 			    time_diff >= rto_opts.rto_sweep_timeout) {
671 				sweep_state = SWEEP_TIMEOUT;
672 				rto_opts.rto_should_stop = B_TRUE;
673 				mutex_exit(&sem_mtx);
674 				goto exit;
675 			}
676 
677 			/* check if should stop the test (error) */
678 			if (sweep_state != SWEEP_RUNNING) {
679 				mutex_exit(&sem_mtx);
680 				goto exit;
681 			}
682 
683 			/* exit loop if a slot is available */
684 			if (free_slots > 0) {
685 				break;
686 			}
687 		}
688 
689 		free_slots--;
690 		mutex_exit(&sem_mtx);
691 
692 		opts = umem_zalloc(sizeof (raidz_test_opts_t), UMEM_NOFAIL);
693 		opts->rto_ashift = ashift_v[a];
694 		opts->rto_dcols = dcols_v[d];
695 		opts->rto_offset = (1 << ashift_v[a]) * rand();
696 		opts->rto_dsize = size_v[s];
697 		opts->rto_v = 0; /* be quiet */
698 
699 		VERIFY3P(thread_create(NULL, 0, sweep_thread, (void *) opts,
700 		    0, NULL, TS_RUN, maxclsyspri), !=, NULL);
701 	}
702 
703 exit:
704 	LOG(D_ALL, "\nWaiting for test threads to finish...\n");
705 	mutex_enter(&sem_mtx);
706 	VERIFY(free_slots <= max_free_slots);
707 	while (free_slots < max_free_slots) {
708 		(void) cv_wait(&sem_cv, &sem_mtx);
709 	}
710 	mutex_exit(&sem_mtx);
711 
712 	if (sweep_state == SWEEP_ERROR) {
713 		ERRMSG("Sweep test failed! Failed option: \n");
714 		print_opts(&failed_opts, B_TRUE);
715 	} else {
716 		if (sweep_state == SWEEP_TIMEOUT)
717 			LOG(D_ALL, "Test timeout (%lus). Stopping...\n",
718 			    (ulong_t)rto_opts.rto_sweep_timeout);
719 
720 		LOG(D_ALL, "Sweep test succeeded on %lu raidz maps!\n",
721 		    (ulong_t)tried_comb);
722 	}
723 
724 	mutex_destroy(&sem_mtx);
725 
726 	return (sweep_state == SWEEP_ERROR ? SWEEP_ERROR : 0);
727 }
728 
729 int
730 main(int argc, char **argv)
731 {
732 	size_t i;
733 	int err = 0;
734 
735 	/* init gdb string early */
736 	(void) sprintf(gdb, gdb_tmpl, getpid());
737 
738 	(void) setvbuf(stdout, NULL, _IOLBF, 0);
739 
740 	dprintf_setup(&argc, argv);
741 
742 	process_options(argc, argv);
743 
744 	kernel_init(FREAD);
745 
746 	/* setup random data because rand() is not reentrant */
747 	rand_data = (int *)umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL);
748 	srand((unsigned)time(NULL) * getpid());
749 	for (i = 0; i < SPA_MAXBLOCKSIZE / sizeof (int); i++)
750 		rand_data[i] = rand();
751 
752 	mprotect((void *)rand_data, SPA_MAXBLOCKSIZE, PROT_READ);
753 
754 	if (rto_opts.rto_benchmark) {
755 		run_raidz_benchmark();
756 	} else if (rto_opts.rto_sweep) {
757 		err = run_sweep();
758 	} else {
759 		err = run_test(NULL);
760 	}
761 
762 	umem_free(rand_data, SPA_MAXBLOCKSIZE);
763 	kernel_fini();
764 
765 	return (err);
766 }
767