xref: /freebsd/sys/contrib/openzfs/cmd/ztest.c (revision be181ee2a28aa2b4b0e76684bce9f673ef668874)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
24  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
25  * Copyright (c) 2013 Steven Hartland. All rights reserved.
26  * Copyright (c) 2014 Integros [integros.com]
27  * Copyright 2017 Joyent, Inc.
28  * Copyright (c) 2017, Intel Corporation.
29  */
30 
31 /*
32  * The objective of this program is to provide a DMU/ZAP/SPA stress test
33  * that runs entirely in userland, is easy to use, and easy to extend.
34  *
35  * The overall design of the ztest program is as follows:
36  *
37  * (1) For each major functional area (e.g. adding vdevs to a pool,
38  *     creating and destroying datasets, reading and writing objects, etc)
39  *     we have a simple routine to test that functionality.  These
40  *     individual routines do not have to do anything "stressful".
41  *
42  * (2) We turn these simple functionality tests into a stress test by
43  *     running them all in parallel, with as many threads as desired,
44  *     and spread across as many datasets, objects, and vdevs as desired.
45  *
46  * (3) While all this is happening, we inject faults into the pool to
47  *     verify that self-healing data really works.
48  *
49  * (4) Every time we open a dataset, we change its checksum and compression
50  *     functions.  Thus even individual objects vary from block to block
51  *     in which checksum they use and whether they're compressed.
52  *
53  * (5) To verify that we never lose on-disk consistency after a crash,
54  *     we run the entire test in a child of the main process.
55  *     At random times, the child self-immolates with a SIGKILL.
56  *     This is the software equivalent of pulling the power cord.
57  *     The parent then runs the test again, using the existing
58  *     storage pool, as many times as desired. If backwards compatibility
59  *     testing is enabled ztest will sometimes run the "older" version
60  *     of ztest after a SIGKILL.
61  *
62  * (6) To verify that we don't have future leaks or temporal incursions,
63  *     many of the functional tests record the transaction group number
64  *     as part of their data.  When reading old data, they verify that
65  *     the transaction group number is less than the current, open txg.
66  *     If you add a new test, please do this if applicable.
67  *
68  * (7) Threads are created with a reduced stack size, for sanity checking.
69  *     Therefore, it's important not to allocate huge buffers on the stack.
70  *
71  * When run with no arguments, ztest runs for about five minutes and
72  * produces no output if successful.  To get a little bit of information,
73  * specify -V.  To get more information, specify -VV, and so on.
74  *
75  * To turn this into an overnight stress test, use -T to specify run time.
76  *
77  * You can ask more vdevs [-v], datasets [-d], or threads [-t]
78  * to increase the pool capacity, fanout, and overall stress level.
79  *
80  * Use the -k option to set the desired frequency of kills.
81  *
82  * When ztest invokes itself it passes all relevant information through a
83  * temporary file which is mmap-ed in the child process. This allows shared
84  * memory to survive the exec syscall. The ztest_shared_hdr_t struct is always
85  * stored at offset 0 of this file and contains information on the size and
86  * number of shared structures in the file. The information stored in this file
87  * must remain backwards compatible with older versions of ztest so that
88  * ztest can invoke them during backwards compatibility testing (-B).
89  */
90 
91 #include <sys/zfs_context.h>
92 #include <sys/spa.h>
93 #include <sys/dmu.h>
94 #include <sys/txg.h>
95 #include <sys/dbuf.h>
96 #include <sys/zap.h>
97 #include <sys/dmu_objset.h>
98 #include <sys/poll.h>
99 #include <sys/stat.h>
100 #include <sys/time.h>
101 #include <sys/wait.h>
102 #include <sys/mman.h>
103 #include <sys/resource.h>
104 #include <sys/zio.h>
105 #include <sys/zil.h>
106 #include <sys/zil_impl.h>
107 #include <sys/vdev_draid.h>
108 #include <sys/vdev_impl.h>
109 #include <sys/vdev_file.h>
110 #include <sys/vdev_initialize.h>
111 #include <sys/vdev_raidz.h>
112 #include <sys/vdev_trim.h>
113 #include <sys/spa_impl.h>
114 #include <sys/metaslab_impl.h>
115 #include <sys/dsl_prop.h>
116 #include <sys/dsl_dataset.h>
117 #include <sys/dsl_destroy.h>
118 #include <sys/dsl_scan.h>
119 #include <sys/zio_checksum.h>
120 #include <sys/zfs_refcount.h>
121 #include <sys/zfeature.h>
122 #include <sys/dsl_userhold.h>
123 #include <sys/abd.h>
124 #include <sys/blake3.h>
125 #include <stdio.h>
126 #include <stdlib.h>
127 #include <unistd.h>
128 #include <getopt.h>
129 #include <signal.h>
130 #include <umem.h>
131 #include <ctype.h>
132 #include <math.h>
133 #include <sys/fs/zfs.h>
134 #include <zfs_fletcher.h>
135 #include <libnvpair.h>
136 #include <libzutil.h>
137 #include <sys/crypto/icp.h>
138 #if (__GLIBC__ && !__UCLIBC__)
139 #include <execinfo.h> /* for backtrace() */
140 #endif
141 
142 static int ztest_fd_data = -1;
143 static int ztest_fd_rand = -1;
144 
145 typedef struct ztest_shared_hdr {
146 	uint64_t	zh_hdr_size;
147 	uint64_t	zh_opts_size;
148 	uint64_t	zh_size;
149 	uint64_t	zh_stats_size;
150 	uint64_t	zh_stats_count;
151 	uint64_t	zh_ds_size;
152 	uint64_t	zh_ds_count;
153 } ztest_shared_hdr_t;
154 
155 static ztest_shared_hdr_t *ztest_shared_hdr;
156 
157 enum ztest_class_state {
158 	ZTEST_VDEV_CLASS_OFF,
159 	ZTEST_VDEV_CLASS_ON,
160 	ZTEST_VDEV_CLASS_RND
161 };
162 
163 #define	ZO_GVARS_MAX_ARGLEN	((size_t)64)
164 #define	ZO_GVARS_MAX_COUNT	((size_t)10)
165 
166 typedef struct ztest_shared_opts {
167 	char zo_pool[ZFS_MAX_DATASET_NAME_LEN];
168 	char zo_dir[ZFS_MAX_DATASET_NAME_LEN];
169 	char zo_alt_ztest[MAXNAMELEN];
170 	char zo_alt_libpath[MAXNAMELEN];
171 	uint64_t zo_vdevs;
172 	uint64_t zo_vdevtime;
173 	size_t zo_vdev_size;
174 	int zo_ashift;
175 	int zo_mirrors;
176 	int zo_raid_children;
177 	int zo_raid_parity;
178 	char zo_raid_type[8];
179 	int zo_draid_data;
180 	int zo_draid_spares;
181 	int zo_datasets;
182 	int zo_threads;
183 	uint64_t zo_passtime;
184 	uint64_t zo_killrate;
185 	int zo_verbose;
186 	int zo_init;
187 	uint64_t zo_time;
188 	uint64_t zo_maxloops;
189 	uint64_t zo_metaslab_force_ganging;
190 	int zo_mmp_test;
191 	int zo_special_vdevs;
192 	int zo_dump_dbgmsg;
193 	int zo_gvars_count;
194 	char zo_gvars[ZO_GVARS_MAX_COUNT][ZO_GVARS_MAX_ARGLEN];
195 } ztest_shared_opts_t;
196 
197 /* Default values for command line options. */
198 #define	DEFAULT_POOL "ztest"
199 #define	DEFAULT_VDEV_DIR "/tmp"
200 #define	DEFAULT_VDEV_COUNT 5
201 #define	DEFAULT_VDEV_SIZE (SPA_MINDEVSIZE * 4)	/* 256m default size */
202 #define	DEFAULT_VDEV_SIZE_STR "256M"
203 #define	DEFAULT_ASHIFT SPA_MINBLOCKSHIFT
204 #define	DEFAULT_MIRRORS 2
205 #define	DEFAULT_RAID_CHILDREN 4
206 #define	DEFAULT_RAID_PARITY 1
207 #define	DEFAULT_DRAID_DATA 4
208 #define	DEFAULT_DRAID_SPARES 1
209 #define	DEFAULT_DATASETS_COUNT 7
210 #define	DEFAULT_THREADS 23
211 #define	DEFAULT_RUN_TIME 300 /* 300 seconds */
212 #define	DEFAULT_RUN_TIME_STR "300 sec"
213 #define	DEFAULT_PASS_TIME 60 /* 60 seconds */
214 #define	DEFAULT_PASS_TIME_STR "60 sec"
215 #define	DEFAULT_KILL_RATE 70 /* 70% kill rate */
216 #define	DEFAULT_KILLRATE_STR "70%"
217 #define	DEFAULT_INITS 1
218 #define	DEFAULT_MAX_LOOPS 50 /* 5 minutes */
219 #define	DEFAULT_FORCE_GANGING (64 << 10)
220 #define	DEFAULT_FORCE_GANGING_STR "64K"
221 
222 /* Simplifying assumption: -1 is not a valid default. */
223 #define	NO_DEFAULT -1
224 
225 static const ztest_shared_opts_t ztest_opts_defaults = {
226 	.zo_pool = DEFAULT_POOL,
227 	.zo_dir = DEFAULT_VDEV_DIR,
228 	.zo_alt_ztest = { '\0' },
229 	.zo_alt_libpath = { '\0' },
230 	.zo_vdevs = DEFAULT_VDEV_COUNT,
231 	.zo_ashift = DEFAULT_ASHIFT,
232 	.zo_mirrors = DEFAULT_MIRRORS,
233 	.zo_raid_children = DEFAULT_RAID_CHILDREN,
234 	.zo_raid_parity = DEFAULT_RAID_PARITY,
235 	.zo_raid_type = VDEV_TYPE_RAIDZ,
236 	.zo_vdev_size = DEFAULT_VDEV_SIZE,
237 	.zo_draid_data = DEFAULT_DRAID_DATA,	/* data drives */
238 	.zo_draid_spares = DEFAULT_DRAID_SPARES, /* distributed spares */
239 	.zo_datasets = DEFAULT_DATASETS_COUNT,
240 	.zo_threads = DEFAULT_THREADS,
241 	.zo_passtime = DEFAULT_PASS_TIME,
242 	.zo_killrate = DEFAULT_KILL_RATE,
243 	.zo_verbose = 0,
244 	.zo_mmp_test = 0,
245 	.zo_init = DEFAULT_INITS,
246 	.zo_time = DEFAULT_RUN_TIME,
247 	.zo_maxloops = DEFAULT_MAX_LOOPS, /* max loops during spa_freeze() */
248 	.zo_metaslab_force_ganging = DEFAULT_FORCE_GANGING,
249 	.zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
250 	.zo_gvars_count = 0,
251 };
252 
253 extern uint64_t metaslab_force_ganging;
254 extern uint64_t metaslab_df_alloc_threshold;
255 extern unsigned long zfs_deadman_synctime_ms;
256 extern uint_t metaslab_preload_limit;
257 extern int zfs_compressed_arc_enabled;
258 extern int zfs_abd_scatter_enabled;
259 extern uint_t dmu_object_alloc_chunk_shift;
260 extern boolean_t zfs_force_some_double_word_sm_entries;
261 extern unsigned long zio_decompress_fail_fraction;
262 extern unsigned long zfs_reconstruct_indirect_damage_fraction;
263 
264 
265 static ztest_shared_opts_t *ztest_shared_opts;
266 static ztest_shared_opts_t ztest_opts;
267 static const char *const ztest_wkeydata = "abcdefghijklmnopqrstuvwxyz012345";
268 
269 typedef struct ztest_shared_ds {
270 	uint64_t	zd_seq;
271 } ztest_shared_ds_t;
272 
273 static ztest_shared_ds_t *ztest_shared_ds;
274 #define	ZTEST_GET_SHARED_DS(d) (&ztest_shared_ds[d])
275 
276 #define	BT_MAGIC	0x123456789abcdefULL
277 #define	MAXFAULTS(zs) \
278 	(MAX((zs)->zs_mirrors, 1) * (ztest_opts.zo_raid_parity + 1) - 1)
279 
280 enum ztest_io_type {
281 	ZTEST_IO_WRITE_TAG,
282 	ZTEST_IO_WRITE_PATTERN,
283 	ZTEST_IO_WRITE_ZEROES,
284 	ZTEST_IO_TRUNCATE,
285 	ZTEST_IO_SETATTR,
286 	ZTEST_IO_REWRITE,
287 	ZTEST_IO_TYPES
288 };
289 
290 typedef struct ztest_block_tag {
291 	uint64_t	bt_magic;
292 	uint64_t	bt_objset;
293 	uint64_t	bt_object;
294 	uint64_t	bt_dnodesize;
295 	uint64_t	bt_offset;
296 	uint64_t	bt_gen;
297 	uint64_t	bt_txg;
298 	uint64_t	bt_crtxg;
299 } ztest_block_tag_t;
300 
301 typedef struct bufwad {
302 	uint64_t	bw_index;
303 	uint64_t	bw_txg;
304 	uint64_t	bw_data;
305 } bufwad_t;
306 
307 /*
308  * It would be better to use a rangelock_t per object.  Unfortunately
309  * the rangelock_t is not a drop-in replacement for rl_t, because we
310  * still need to map from object ID to rangelock_t.
311  */
312 typedef enum {
313 	RL_READER,
314 	RL_WRITER,
315 	RL_APPEND
316 } rl_type_t;
317 
318 typedef struct rll {
319 	void		*rll_writer;
320 	int		rll_readers;
321 	kmutex_t	rll_lock;
322 	kcondvar_t	rll_cv;
323 } rll_t;
324 
325 typedef struct rl {
326 	uint64_t	rl_object;
327 	uint64_t	rl_offset;
328 	uint64_t	rl_size;
329 	rll_t		*rl_lock;
330 } rl_t;
331 
332 #define	ZTEST_RANGE_LOCKS	64
333 #define	ZTEST_OBJECT_LOCKS	64
334 
335 /*
336  * Object descriptor.  Used as a template for object lookup/create/remove.
337  */
338 typedef struct ztest_od {
339 	uint64_t	od_dir;
340 	uint64_t	od_object;
341 	dmu_object_type_t od_type;
342 	dmu_object_type_t od_crtype;
343 	uint64_t	od_blocksize;
344 	uint64_t	od_crblocksize;
345 	uint64_t	od_crdnodesize;
346 	uint64_t	od_gen;
347 	uint64_t	od_crgen;
348 	char		od_name[ZFS_MAX_DATASET_NAME_LEN];
349 } ztest_od_t;
350 
351 /*
352  * Per-dataset state.
353  */
354 typedef struct ztest_ds {
355 	ztest_shared_ds_t *zd_shared;
356 	objset_t	*zd_os;
357 	pthread_rwlock_t zd_zilog_lock;
358 	zilog_t		*zd_zilog;
359 	ztest_od_t	*zd_od;		/* debugging aid */
360 	char		zd_name[ZFS_MAX_DATASET_NAME_LEN];
361 	kmutex_t	zd_dirobj_lock;
362 	rll_t		zd_object_lock[ZTEST_OBJECT_LOCKS];
363 	rll_t		zd_range_lock[ZTEST_RANGE_LOCKS];
364 } ztest_ds_t;
365 
366 /*
367  * Per-iteration state.
368  */
369 typedef void ztest_func_t(ztest_ds_t *zd, uint64_t id);
370 
371 typedef struct ztest_info {
372 	ztest_func_t	*zi_func;	/* test function */
373 	uint64_t	zi_iters;	/* iterations per execution */
374 	uint64_t	*zi_interval;	/* execute every <interval> seconds */
375 	const char	*zi_funcname;	/* name of test function */
376 } ztest_info_t;
377 
378 typedef struct ztest_shared_callstate {
379 	uint64_t	zc_count;	/* per-pass count */
380 	uint64_t	zc_time;	/* per-pass time */
381 	uint64_t	zc_next;	/* next time to call this function */
382 } ztest_shared_callstate_t;
383 
384 static ztest_shared_callstate_t *ztest_shared_callstate;
385 #define	ZTEST_GET_SHARED_CALLSTATE(c) (&ztest_shared_callstate[c])
386 
387 ztest_func_t ztest_dmu_read_write;
388 ztest_func_t ztest_dmu_write_parallel;
389 ztest_func_t ztest_dmu_object_alloc_free;
390 ztest_func_t ztest_dmu_object_next_chunk;
391 ztest_func_t ztest_dmu_commit_callbacks;
392 ztest_func_t ztest_zap;
393 ztest_func_t ztest_zap_parallel;
394 ztest_func_t ztest_zil_commit;
395 ztest_func_t ztest_zil_remount;
396 ztest_func_t ztest_dmu_read_write_zcopy;
397 ztest_func_t ztest_dmu_objset_create_destroy;
398 ztest_func_t ztest_dmu_prealloc;
399 ztest_func_t ztest_fzap;
400 ztest_func_t ztest_dmu_snapshot_create_destroy;
401 ztest_func_t ztest_dsl_prop_get_set;
402 ztest_func_t ztest_spa_prop_get_set;
403 ztest_func_t ztest_spa_create_destroy;
404 ztest_func_t ztest_fault_inject;
405 ztest_func_t ztest_dmu_snapshot_hold;
406 ztest_func_t ztest_mmp_enable_disable;
407 ztest_func_t ztest_scrub;
408 ztest_func_t ztest_dsl_dataset_promote_busy;
409 ztest_func_t ztest_vdev_attach_detach;
410 ztest_func_t ztest_vdev_LUN_growth;
411 ztest_func_t ztest_vdev_add_remove;
412 ztest_func_t ztest_vdev_class_add;
413 ztest_func_t ztest_vdev_aux_add_remove;
414 ztest_func_t ztest_split_pool;
415 ztest_func_t ztest_reguid;
416 ztest_func_t ztest_spa_upgrade;
417 ztest_func_t ztest_device_removal;
418 ztest_func_t ztest_spa_checkpoint_create_discard;
419 ztest_func_t ztest_initialize;
420 ztest_func_t ztest_trim;
421 ztest_func_t ztest_blake3;
422 ztest_func_t ztest_fletcher;
423 ztest_func_t ztest_fletcher_incr;
424 ztest_func_t ztest_verify_dnode_bt;
425 
426 uint64_t zopt_always = 0ULL * NANOSEC;		/* all the time */
427 uint64_t zopt_incessant = 1ULL * NANOSEC / 10;	/* every 1/10 second */
428 uint64_t zopt_often = 1ULL * NANOSEC;		/* every second */
429 uint64_t zopt_sometimes = 10ULL * NANOSEC;	/* every 10 seconds */
430 uint64_t zopt_rarely = 60ULL * NANOSEC;		/* every 60 seconds */
431 
432 #define	ZTI_INIT(func, iters, interval) \
433 	{   .zi_func = (func), \
434 	    .zi_iters = (iters), \
435 	    .zi_interval = (interval), \
436 	    .zi_funcname = # func }
437 
438 ztest_info_t ztest_info[] = {
439 	ZTI_INIT(ztest_dmu_read_write, 1, &zopt_always),
440 	ZTI_INIT(ztest_dmu_write_parallel, 10, &zopt_always),
441 	ZTI_INIT(ztest_dmu_object_alloc_free, 1, &zopt_always),
442 	ZTI_INIT(ztest_dmu_object_next_chunk, 1, &zopt_sometimes),
443 	ZTI_INIT(ztest_dmu_commit_callbacks, 1, &zopt_always),
444 	ZTI_INIT(ztest_zap, 30, &zopt_always),
445 	ZTI_INIT(ztest_zap_parallel, 100, &zopt_always),
446 	ZTI_INIT(ztest_split_pool, 1, &zopt_always),
447 	ZTI_INIT(ztest_zil_commit, 1, &zopt_incessant),
448 	ZTI_INIT(ztest_zil_remount, 1, &zopt_sometimes),
449 	ZTI_INIT(ztest_dmu_read_write_zcopy, 1, &zopt_often),
450 	ZTI_INIT(ztest_dmu_objset_create_destroy, 1, &zopt_often),
451 	ZTI_INIT(ztest_dsl_prop_get_set, 1, &zopt_often),
452 	ZTI_INIT(ztest_spa_prop_get_set, 1, &zopt_sometimes),
453 #if 0
454 	ZTI_INIT(ztest_dmu_prealloc, 1, &zopt_sometimes),
455 #endif
456 	ZTI_INIT(ztest_fzap, 1, &zopt_sometimes),
457 	ZTI_INIT(ztest_dmu_snapshot_create_destroy, 1, &zopt_sometimes),
458 	ZTI_INIT(ztest_spa_create_destroy, 1, &zopt_sometimes),
459 	ZTI_INIT(ztest_fault_inject, 1, &zopt_sometimes),
460 	ZTI_INIT(ztest_dmu_snapshot_hold, 1, &zopt_sometimes),
461 	ZTI_INIT(ztest_mmp_enable_disable, 1, &zopt_sometimes),
462 	ZTI_INIT(ztest_reguid, 1, &zopt_rarely),
463 	ZTI_INIT(ztest_scrub, 1, &zopt_rarely),
464 	ZTI_INIT(ztest_spa_upgrade, 1, &zopt_rarely),
465 	ZTI_INIT(ztest_dsl_dataset_promote_busy, 1, &zopt_rarely),
466 	ZTI_INIT(ztest_vdev_attach_detach, 1, &zopt_sometimes),
467 	ZTI_INIT(ztest_vdev_LUN_growth, 1, &zopt_rarely),
468 	ZTI_INIT(ztest_vdev_add_remove, 1, &ztest_opts.zo_vdevtime),
469 	ZTI_INIT(ztest_vdev_class_add, 1, &ztest_opts.zo_vdevtime),
470 	ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime),
471 	ZTI_INIT(ztest_device_removal, 1, &zopt_sometimes),
472 	ZTI_INIT(ztest_spa_checkpoint_create_discard, 1, &zopt_rarely),
473 	ZTI_INIT(ztest_initialize, 1, &zopt_sometimes),
474 	ZTI_INIT(ztest_trim, 1, &zopt_sometimes),
475 	ZTI_INIT(ztest_blake3, 1, &zopt_rarely),
476 	ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
477 	ZTI_INIT(ztest_fletcher_incr, 1, &zopt_rarely),
478 	ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes),
479 };
480 
481 #define	ZTEST_FUNCS	(sizeof (ztest_info) / sizeof (ztest_info_t))
482 
483 /*
484  * The following struct is used to hold a list of uncalled commit callbacks.
485  * The callbacks are ordered by txg number.
486  */
487 typedef struct ztest_cb_list {
488 	kmutex_t	zcl_callbacks_lock;
489 	list_t		zcl_callbacks;
490 } ztest_cb_list_t;
491 
492 /*
493  * Stuff we need to share writably between parent and child.
494  */
495 typedef struct ztest_shared {
496 	boolean_t	zs_do_init;
497 	hrtime_t	zs_proc_start;
498 	hrtime_t	zs_proc_stop;
499 	hrtime_t	zs_thread_start;
500 	hrtime_t	zs_thread_stop;
501 	hrtime_t	zs_thread_kill;
502 	uint64_t	zs_enospc_count;
503 	uint64_t	zs_vdev_next_leaf;
504 	uint64_t	zs_vdev_aux;
505 	uint64_t	zs_alloc;
506 	uint64_t	zs_space;
507 	uint64_t	zs_splits;
508 	uint64_t	zs_mirrors;
509 	uint64_t	zs_metaslab_sz;
510 	uint64_t	zs_metaslab_df_alloc_threshold;
511 	uint64_t	zs_guid;
512 } ztest_shared_t;
513 
514 #define	ID_PARALLEL	-1ULL
515 
516 static char ztest_dev_template[] = "%s/%s.%llua";
517 static char ztest_aux_template[] = "%s/%s.%s.%llu";
518 ztest_shared_t *ztest_shared;
519 
520 static spa_t *ztest_spa = NULL;
521 static ztest_ds_t *ztest_ds;
522 
523 static kmutex_t ztest_vdev_lock;
524 static boolean_t ztest_device_removal_active = B_FALSE;
525 static boolean_t ztest_pool_scrubbed = B_FALSE;
526 static kmutex_t ztest_checkpoint_lock;
527 
528 /*
529  * The ztest_name_lock protects the pool and dataset namespace used by
530  * the individual tests. To modify the namespace, consumers must grab
531  * this lock as writer. Grabbing the lock as reader will ensure that the
532  * namespace does not change while the lock is held.
533  */
534 static pthread_rwlock_t ztest_name_lock;
535 
536 static boolean_t ztest_dump_core = B_TRUE;
537 static boolean_t ztest_exiting;
538 
539 /* Global commit callback list */
540 static ztest_cb_list_t zcl;
541 /* Commit cb delay */
542 static uint64_t zc_min_txg_delay = UINT64_MAX;
543 static int zc_cb_counter = 0;
544 
545 /*
546  * Minimum number of commit callbacks that need to be registered for us to check
547  * whether the minimum txg delay is acceptable.
548  */
549 #define	ZTEST_COMMIT_CB_MIN_REG	100
550 
551 /*
552  * If a number of txgs equal to this threshold have been created after a commit
553  * callback has been registered but not called, then we assume there is an
554  * implementation bug.
555  */
556 #define	ZTEST_COMMIT_CB_THRESH	(TXG_CONCURRENT_STATES + 1000)
557 
558 enum ztest_object {
559 	ZTEST_META_DNODE = 0,
560 	ZTEST_DIROBJ,
561 	ZTEST_OBJECTS
562 };
563 
564 static __attribute__((noreturn)) void usage(boolean_t requested);
565 static int ztest_scrub_impl(spa_t *spa);
566 
567 /*
568  * These libumem hooks provide a reasonable set of defaults for the allocator's
569  * debugging facilities.
570  */
571 const char *
572 _umem_debug_init(void)
573 {
574 	return ("default,verbose"); /* $UMEM_DEBUG setting */
575 }
576 
577 const char *
578 _umem_logging_init(void)
579 {
580 	return ("fail,contents"); /* $UMEM_LOGGING setting */
581 }
582 
583 static void
584 dump_debug_buffer(void)
585 {
586 	ssize_t ret __attribute__((unused));
587 
588 	if (!ztest_opts.zo_dump_dbgmsg)
589 		return;
590 
591 	/*
592 	 * We use write() instead of printf() so that this function
593 	 * is safe to call from a signal handler.
594 	 */
595 	ret = write(STDOUT_FILENO, "\n", 1);
596 	zfs_dbgmsg_print("ztest");
597 }
598 
599 #define	BACKTRACE_SZ	100
600 
601 static void sig_handler(int signo)
602 {
603 	struct sigaction action;
604 #if (__GLIBC__ && !__UCLIBC__) /* backtrace() is a GNU extension */
605 	int nptrs;
606 	void *buffer[BACKTRACE_SZ];
607 
608 	nptrs = backtrace(buffer, BACKTRACE_SZ);
609 	backtrace_symbols_fd(buffer, nptrs, STDERR_FILENO);
610 #endif
611 	dump_debug_buffer();
612 
613 	/*
614 	 * Restore default action and re-raise signal so SIGSEGV and
615 	 * SIGABRT can trigger a core dump.
616 	 */
617 	action.sa_handler = SIG_DFL;
618 	sigemptyset(&action.sa_mask);
619 	action.sa_flags = 0;
620 	(void) sigaction(signo, &action, NULL);
621 	raise(signo);
622 }
623 
624 #define	FATAL_MSG_SZ	1024
625 
626 static const char *fatal_msg;
627 
628 static __attribute__((format(printf, 2, 3))) __attribute__((noreturn)) void
629 fatal(int do_perror, const char *message, ...)
630 {
631 	va_list args;
632 	int save_errno = errno;
633 	char *buf;
634 
635 	(void) fflush(stdout);
636 	buf = umem_alloc(FATAL_MSG_SZ, UMEM_NOFAIL);
637 	if (buf == NULL)
638 		goto out;
639 
640 	va_start(args, message);
641 	(void) sprintf(buf, "ztest: ");
642 	/* LINTED */
643 	(void) vsprintf(buf + strlen(buf), message, args);
644 	va_end(args);
645 	if (do_perror) {
646 		(void) snprintf(buf + strlen(buf), FATAL_MSG_SZ - strlen(buf),
647 		    ": %s", strerror(save_errno));
648 	}
649 	(void) fprintf(stderr, "%s\n", buf);
650 	fatal_msg = buf;			/* to ease debugging */
651 
652 out:
653 	if (ztest_dump_core)
654 		abort();
655 	else
656 		dump_debug_buffer();
657 
658 	exit(3);
659 }
660 
661 static int
662 str2shift(const char *buf)
663 {
664 	const char *ends = "BKMGTPEZ";
665 	int i;
666 
667 	if (buf[0] == '\0')
668 		return (0);
669 	for (i = 0; i < strlen(ends); i++) {
670 		if (toupper(buf[0]) == ends[i])
671 			break;
672 	}
673 	if (i == strlen(ends)) {
674 		(void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n",
675 		    buf);
676 		usage(B_FALSE);
677 	}
678 	if (buf[1] == '\0' || (toupper(buf[1]) == 'B' && buf[2] == '\0')) {
679 		return (10*i);
680 	}
681 	(void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n", buf);
682 	usage(B_FALSE);
683 }
684 
685 static uint64_t
686 nicenumtoull(const char *buf)
687 {
688 	char *end;
689 	uint64_t val;
690 
691 	val = strtoull(buf, &end, 0);
692 	if (end == buf) {
693 		(void) fprintf(stderr, "ztest: bad numeric value: %s\n", buf);
694 		usage(B_FALSE);
695 	} else if (end[0] == '.') {
696 		double fval = strtod(buf, &end);
697 		fval *= pow(2, str2shift(end));
698 		/*
699 		 * UINT64_MAX is not exactly representable as a double.
700 		 * The closest representation is UINT64_MAX + 1, so we
701 		 * use a >= comparison instead of > for the bounds check.
702 		 */
703 		if (fval >= (double)UINT64_MAX) {
704 			(void) fprintf(stderr, "ztest: value too large: %s\n",
705 			    buf);
706 			usage(B_FALSE);
707 		}
708 		val = (uint64_t)fval;
709 	} else {
710 		int shift = str2shift(end);
711 		if (shift >= 64 || (val << shift) >> shift != val) {
712 			(void) fprintf(stderr, "ztest: value too large: %s\n",
713 			    buf);
714 			usage(B_FALSE);
715 		}
716 		val <<= shift;
717 	}
718 	return (val);
719 }
720 
721 typedef struct ztest_option {
722 	const char	short_opt;
723 	const char	*long_opt;
724 	const char	*long_opt_param;
725 	const char	*comment;
726 	unsigned int	default_int;
727 	const char	*default_str;
728 } ztest_option_t;
729 
730 /*
731  * The following option_table is used for generating the usage info as well as
732  * the long and short option information for calling getopt_long().
733  */
734 static ztest_option_t option_table[] = {
735 	{ 'v',	"vdevs", "INTEGER", "Number of vdevs", DEFAULT_VDEV_COUNT,
736 	    NULL},
737 	{ 's',	"vdev-size", "INTEGER", "Size of each vdev",
738 	    NO_DEFAULT, DEFAULT_VDEV_SIZE_STR},
739 	{ 'a',	"alignment-shift", "INTEGER",
740 	    "Alignment shift; use 0 for random", DEFAULT_ASHIFT, NULL},
741 	{ 'm',	"mirror-copies", "INTEGER", "Number of mirror copies",
742 	    DEFAULT_MIRRORS, NULL},
743 	{ 'r',	"raid-disks", "INTEGER", "Number of raidz/draid disks",
744 	    DEFAULT_RAID_CHILDREN, NULL},
745 	{ 'R',	"raid-parity", "INTEGER", "Raid parity",
746 	    DEFAULT_RAID_PARITY, NULL},
747 	{ 'K',	"raid-kind", "raidz|draid|random", "Raid kind",
748 	    NO_DEFAULT, "random"},
749 	{ 'D',	"draid-data", "INTEGER", "Number of draid data drives",
750 	    DEFAULT_DRAID_DATA, NULL},
751 	{ 'S',	"draid-spares", "INTEGER", "Number of draid spares",
752 	    DEFAULT_DRAID_SPARES, NULL},
753 	{ 'd',	"datasets", "INTEGER", "Number of datasets",
754 	    DEFAULT_DATASETS_COUNT, NULL},
755 	{ 't',	"threads", "INTEGER", "Number of ztest threads",
756 	    DEFAULT_THREADS, NULL},
757 	{ 'g',	"gang-block-threshold", "INTEGER",
758 	    "Metaslab gang block threshold",
759 	    NO_DEFAULT, DEFAULT_FORCE_GANGING_STR},
760 	{ 'i',	"init-count", "INTEGER", "Number of times to initialize pool",
761 	    DEFAULT_INITS, NULL},
762 	{ 'k',	"kill-percentage", "INTEGER", "Kill percentage",
763 	    NO_DEFAULT, DEFAULT_KILLRATE_STR},
764 	{ 'p',	"pool-name", "STRING", "Pool name",
765 	    NO_DEFAULT, DEFAULT_POOL},
766 	{ 'f',	"vdev-file-directory", "PATH", "File directory for vdev files",
767 	    NO_DEFAULT, DEFAULT_VDEV_DIR},
768 	{ 'M',	"multi-host", NULL,
769 	    "Multi-host; simulate pool imported on remote host",
770 	    NO_DEFAULT, NULL},
771 	{ 'E',	"use-existing-pool", NULL,
772 	    "Use existing pool instead of creating new one", NO_DEFAULT, NULL},
773 	{ 'T',	"run-time", "INTEGER", "Total run time",
774 	    NO_DEFAULT, DEFAULT_RUN_TIME_STR},
775 	{ 'P',	"pass-time", "INTEGER", "Time per pass",
776 	    NO_DEFAULT, DEFAULT_PASS_TIME_STR},
777 	{ 'F',	"freeze-loops", "INTEGER", "Max loops in spa_freeze()",
778 	    DEFAULT_MAX_LOOPS, NULL},
779 	{ 'B',	"alt-ztest", "PATH", "Alternate ztest path",
780 	    NO_DEFAULT, NULL},
781 	{ 'C',	"vdev-class-state", "on|off|random", "vdev class state",
782 	    NO_DEFAULT, "random"},
783 	{ 'o',	"option", "\"OPTION=INTEGER\"",
784 	    "Set global variable to an unsigned 32-bit integer value",
785 	    NO_DEFAULT, NULL},
786 	{ 'G',	"dump-debug-msg", NULL,
787 	    "Dump zfs_dbgmsg buffer before exiting due to an error",
788 	    NO_DEFAULT, NULL},
789 	{ 'V',	"verbose", NULL,
790 	    "Verbose (use multiple times for ever more verbosity)",
791 	    NO_DEFAULT, NULL},
792 	{ 'h',	"help",	NULL, "Show this help",
793 	    NO_DEFAULT, NULL},
794 	{0, 0, 0, 0, 0, 0}
795 };
796 
797 static struct option *long_opts = NULL;
798 static char *short_opts = NULL;
799 
800 static void
801 init_options(void)
802 {
803 	ASSERT3P(long_opts, ==, NULL);
804 	ASSERT3P(short_opts, ==, NULL);
805 
806 	int count = sizeof (option_table) / sizeof (option_table[0]);
807 	long_opts = umem_alloc(sizeof (struct option) * count, UMEM_NOFAIL);
808 
809 	short_opts = umem_alloc(sizeof (char) * 2 * count, UMEM_NOFAIL);
810 	int short_opt_index = 0;
811 
812 	for (int i = 0; i < count; i++) {
813 		long_opts[i].val = option_table[i].short_opt;
814 		long_opts[i].name = option_table[i].long_opt;
815 		long_opts[i].has_arg = option_table[i].long_opt_param != NULL
816 		    ? required_argument : no_argument;
817 		long_opts[i].flag = NULL;
818 		short_opts[short_opt_index++] = option_table[i].short_opt;
819 		if (option_table[i].long_opt_param != NULL) {
820 			short_opts[short_opt_index++] = ':';
821 		}
822 	}
823 }
824 
825 static void
826 fini_options(void)
827 {
828 	int count = sizeof (option_table) / sizeof (option_table[0]);
829 
830 	umem_free(long_opts, sizeof (struct option) * count);
831 	umem_free(short_opts, sizeof (char) * 2 * count);
832 
833 	long_opts = NULL;
834 	short_opts = NULL;
835 }
836 
837 static __attribute__((noreturn)) void
838 usage(boolean_t requested)
839 {
840 	char option[80];
841 	FILE *fp = requested ? stdout : stderr;
842 
843 	(void) fprintf(fp, "Usage: %s [OPTIONS...]\n", DEFAULT_POOL);
844 	for (int i = 0; option_table[i].short_opt != 0; i++) {
845 		if (option_table[i].long_opt_param != NULL) {
846 			(void) sprintf(option, "  -%c --%s=%s",
847 			    option_table[i].short_opt,
848 			    option_table[i].long_opt,
849 			    option_table[i].long_opt_param);
850 		} else {
851 			(void) sprintf(option, "  -%c --%s",
852 			    option_table[i].short_opt,
853 			    option_table[i].long_opt);
854 		}
855 		(void) fprintf(fp, "  %-40s%s", option,
856 		    option_table[i].comment);
857 
858 		if (option_table[i].long_opt_param != NULL) {
859 			if (option_table[i].default_str != NULL) {
860 				(void) fprintf(fp, " (default: %s)",
861 				    option_table[i].default_str);
862 			} else if (option_table[i].default_int != NO_DEFAULT) {
863 				(void) fprintf(fp, " (default: %u)",
864 				    option_table[i].default_int);
865 			}
866 		}
867 		(void) fprintf(fp, "\n");
868 	}
869 	exit(requested ? 0 : 1);
870 }
871 
872 static uint64_t
873 ztest_random(uint64_t range)
874 {
875 	uint64_t r;
876 
877 	ASSERT3S(ztest_fd_rand, >=, 0);
878 
879 	if (range == 0)
880 		return (0);
881 
882 	if (read(ztest_fd_rand, &r, sizeof (r)) != sizeof (r))
883 		fatal(B_TRUE, "short read from /dev/urandom");
884 
885 	return (r % range);
886 }
887 
888 static void
889 ztest_parse_name_value(const char *input, ztest_shared_opts_t *zo)
890 {
891 	char name[32];
892 	char *value;
893 	int state = ZTEST_VDEV_CLASS_RND;
894 
895 	(void) strlcpy(name, input, sizeof (name));
896 
897 	value = strchr(name, '=');
898 	if (value == NULL) {
899 		(void) fprintf(stderr, "missing value in property=value "
900 		    "'-C' argument (%s)\n", input);
901 		usage(B_FALSE);
902 	}
903 	*(value) = '\0';
904 	value++;
905 
906 	if (strcmp(value, "on") == 0) {
907 		state = ZTEST_VDEV_CLASS_ON;
908 	} else if (strcmp(value, "off") == 0) {
909 		state = ZTEST_VDEV_CLASS_OFF;
910 	} else if (strcmp(value, "random") == 0) {
911 		state = ZTEST_VDEV_CLASS_RND;
912 	} else {
913 		(void) fprintf(stderr, "invalid property value '%s'\n", value);
914 		usage(B_FALSE);
915 	}
916 
917 	if (strcmp(name, "special") == 0) {
918 		zo->zo_special_vdevs = state;
919 	} else {
920 		(void) fprintf(stderr, "invalid property name '%s'\n", name);
921 		usage(B_FALSE);
922 	}
923 	if (zo->zo_verbose >= 3)
924 		(void) printf("%s vdev state is '%s'\n", name, value);
925 }
926 
927 static void
928 process_options(int argc, char **argv)
929 {
930 	char *path;
931 	ztest_shared_opts_t *zo = &ztest_opts;
932 
933 	int opt;
934 	uint64_t value;
935 	const char *raid_kind = "random";
936 
937 	memcpy(zo, &ztest_opts_defaults, sizeof (*zo));
938 
939 	init_options();
940 
941 	while ((opt = getopt_long(argc, argv, short_opts, long_opts,
942 	    NULL)) != EOF) {
943 		value = 0;
944 		switch (opt) {
945 		case 'v':
946 		case 's':
947 		case 'a':
948 		case 'm':
949 		case 'r':
950 		case 'R':
951 		case 'D':
952 		case 'S':
953 		case 'd':
954 		case 't':
955 		case 'g':
956 		case 'i':
957 		case 'k':
958 		case 'T':
959 		case 'P':
960 		case 'F':
961 			value = nicenumtoull(optarg);
962 		}
963 		switch (opt) {
964 		case 'v':
965 			zo->zo_vdevs = value;
966 			break;
967 		case 's':
968 			zo->zo_vdev_size = MAX(SPA_MINDEVSIZE, value);
969 			break;
970 		case 'a':
971 			zo->zo_ashift = value;
972 			break;
973 		case 'm':
974 			zo->zo_mirrors = value;
975 			break;
976 		case 'r':
977 			zo->zo_raid_children = MAX(1, value);
978 			break;
979 		case 'R':
980 			zo->zo_raid_parity = MIN(MAX(value, 1), 3);
981 			break;
982 		case 'K':
983 			raid_kind = optarg;
984 			break;
985 		case 'D':
986 			zo->zo_draid_data = MAX(1, value);
987 			break;
988 		case 'S':
989 			zo->zo_draid_spares = MAX(1, value);
990 			break;
991 		case 'd':
992 			zo->zo_datasets = MAX(1, value);
993 			break;
994 		case 't':
995 			zo->zo_threads = MAX(1, value);
996 			break;
997 		case 'g':
998 			zo->zo_metaslab_force_ganging =
999 			    MAX(SPA_MINBLOCKSIZE << 1, value);
1000 			break;
1001 		case 'i':
1002 			zo->zo_init = value;
1003 			break;
1004 		case 'k':
1005 			zo->zo_killrate = value;
1006 			break;
1007 		case 'p':
1008 			(void) strlcpy(zo->zo_pool, optarg,
1009 			    sizeof (zo->zo_pool));
1010 			break;
1011 		case 'f':
1012 			path = realpath(optarg, NULL);
1013 			if (path == NULL) {
1014 				(void) fprintf(stderr, "error: %s: %s\n",
1015 				    optarg, strerror(errno));
1016 				usage(B_FALSE);
1017 			} else {
1018 				(void) strlcpy(zo->zo_dir, path,
1019 				    sizeof (zo->zo_dir));
1020 				free(path);
1021 			}
1022 			break;
1023 		case 'M':
1024 			zo->zo_mmp_test = 1;
1025 			break;
1026 		case 'V':
1027 			zo->zo_verbose++;
1028 			break;
1029 		case 'E':
1030 			zo->zo_init = 0;
1031 			break;
1032 		case 'T':
1033 			zo->zo_time = value;
1034 			break;
1035 		case 'P':
1036 			zo->zo_passtime = MAX(1, value);
1037 			break;
1038 		case 'F':
1039 			zo->zo_maxloops = MAX(1, value);
1040 			break;
1041 		case 'B':
1042 			(void) strlcpy(zo->zo_alt_ztest, optarg,
1043 			    sizeof (zo->zo_alt_ztest));
1044 			break;
1045 		case 'C':
1046 			ztest_parse_name_value(optarg, zo);
1047 			break;
1048 		case 'o':
1049 			if (zo->zo_gvars_count >= ZO_GVARS_MAX_COUNT) {
1050 				(void) fprintf(stderr,
1051 				    "max global var count (%zu) exceeded\n",
1052 				    ZO_GVARS_MAX_COUNT);
1053 				usage(B_FALSE);
1054 			}
1055 			char *v = zo->zo_gvars[zo->zo_gvars_count];
1056 			if (strlcpy(v, optarg, ZO_GVARS_MAX_ARGLEN) >=
1057 			    ZO_GVARS_MAX_ARGLEN) {
1058 				(void) fprintf(stderr,
1059 				    "global var option '%s' is too long\n",
1060 				    optarg);
1061 				usage(B_FALSE);
1062 			}
1063 			zo->zo_gvars_count++;
1064 			break;
1065 		case 'G':
1066 			zo->zo_dump_dbgmsg = 1;
1067 			break;
1068 		case 'h':
1069 			usage(B_TRUE);
1070 			break;
1071 		case '?':
1072 		default:
1073 			usage(B_FALSE);
1074 			break;
1075 		}
1076 	}
1077 
1078 	fini_options();
1079 
1080 	/* When raid choice is 'random' add a draid pool 50% of the time */
1081 	if (strcmp(raid_kind, "random") == 0) {
1082 		raid_kind = (ztest_random(2) == 0) ? "draid" : "raidz";
1083 
1084 		if (ztest_opts.zo_verbose >= 3)
1085 			(void) printf("choosing RAID type '%s'\n", raid_kind);
1086 	}
1087 
1088 	if (strcmp(raid_kind, "draid") == 0) {
1089 		uint64_t min_devsize;
1090 
1091 		/* With fewer disk use 256M, otherwise 128M is OK */
1092 		min_devsize = (ztest_opts.zo_raid_children < 16) ?
1093 		    (256ULL << 20) : (128ULL << 20);
1094 
1095 		/* No top-level mirrors with dRAID for now */
1096 		zo->zo_mirrors = 0;
1097 
1098 		/* Use more appropriate defaults for dRAID */
1099 		if (zo->zo_vdevs == ztest_opts_defaults.zo_vdevs)
1100 			zo->zo_vdevs = 1;
1101 		if (zo->zo_raid_children ==
1102 		    ztest_opts_defaults.zo_raid_children)
1103 			zo->zo_raid_children = 16;
1104 		if (zo->zo_ashift < 12)
1105 			zo->zo_ashift = 12;
1106 		if (zo->zo_vdev_size < min_devsize)
1107 			zo->zo_vdev_size = min_devsize;
1108 
1109 		if (zo->zo_draid_data + zo->zo_raid_parity >
1110 		    zo->zo_raid_children - zo->zo_draid_spares) {
1111 			(void) fprintf(stderr, "error: too few draid "
1112 			    "children (%d) for stripe width (%d)\n",
1113 			    zo->zo_raid_children,
1114 			    zo->zo_draid_data + zo->zo_raid_parity);
1115 			usage(B_FALSE);
1116 		}
1117 
1118 		(void) strlcpy(zo->zo_raid_type, VDEV_TYPE_DRAID,
1119 		    sizeof (zo->zo_raid_type));
1120 
1121 	} else /* using raidz */ {
1122 		ASSERT0(strcmp(raid_kind, "raidz"));
1123 
1124 		zo->zo_raid_parity = MIN(zo->zo_raid_parity,
1125 		    zo->zo_raid_children - 1);
1126 	}
1127 
1128 	zo->zo_vdevtime =
1129 	    (zo->zo_vdevs > 0 ? zo->zo_time * NANOSEC / zo->zo_vdevs :
1130 	    UINT64_MAX >> 2);
1131 
1132 	if (*zo->zo_alt_ztest) {
1133 		const char *invalid_what = "ztest";
1134 		char *val = zo->zo_alt_ztest;
1135 		if (0 != access(val, X_OK) ||
1136 		    (strrchr(val, '/') == NULL && (errno = EINVAL)))
1137 			goto invalid;
1138 
1139 		int dirlen = strrchr(val, '/') - val;
1140 		strlcpy(zo->zo_alt_libpath, val,
1141 		    MIN(sizeof (zo->zo_alt_libpath), dirlen + 1));
1142 		invalid_what = "library path", val = zo->zo_alt_libpath;
1143 		if (strrchr(val, '/') == NULL && (errno = EINVAL))
1144 			goto invalid;
1145 		*strrchr(val, '/') = '\0';
1146 		strlcat(val, "/lib", sizeof (zo->zo_alt_libpath));
1147 
1148 		if (0 != access(zo->zo_alt_libpath, X_OK))
1149 			goto invalid;
1150 		return;
1151 
1152 invalid:
1153 		ztest_dump_core = B_FALSE;
1154 		fatal(B_TRUE, "invalid alternate %s %s", invalid_what, val);
1155 	}
1156 }
1157 
1158 static void
1159 ztest_kill(ztest_shared_t *zs)
1160 {
1161 	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(ztest_spa));
1162 	zs->zs_space = metaslab_class_get_space(spa_normal_class(ztest_spa));
1163 
1164 	/*
1165 	 * Before we kill ourselves, make sure that the config is updated.
1166 	 * See comment above spa_write_cachefile().
1167 	 */
1168 	mutex_enter(&spa_namespace_lock);
1169 	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE, B_FALSE);
1170 	mutex_exit(&spa_namespace_lock);
1171 
1172 	(void) raise(SIGKILL);
1173 }
1174 
1175 static void
1176 ztest_record_enospc(const char *s)
1177 {
1178 	(void) s;
1179 	ztest_shared->zs_enospc_count++;
1180 }
1181 
1182 static uint64_t
1183 ztest_get_ashift(void)
1184 {
1185 	if (ztest_opts.zo_ashift == 0)
1186 		return (SPA_MINBLOCKSHIFT + ztest_random(5));
1187 	return (ztest_opts.zo_ashift);
1188 }
1189 
1190 static boolean_t
1191 ztest_is_draid_spare(const char *name)
1192 {
1193 	uint64_t spare_id = 0, parity = 0, vdev_id = 0;
1194 
1195 	if (sscanf(name, VDEV_TYPE_DRAID "%"PRIu64"-%"PRIu64"-%"PRIu64"",
1196 	    &parity, &vdev_id, &spare_id) == 3) {
1197 		return (B_TRUE);
1198 	}
1199 
1200 	return (B_FALSE);
1201 }
1202 
1203 static nvlist_t *
1204 make_vdev_file(const char *path, const char *aux, const char *pool,
1205     size_t size, uint64_t ashift)
1206 {
1207 	char *pathbuf = NULL;
1208 	uint64_t vdev;
1209 	nvlist_t *file;
1210 	boolean_t draid_spare = B_FALSE;
1211 
1212 
1213 	if (ashift == 0)
1214 		ashift = ztest_get_ashift();
1215 
1216 	if (path == NULL) {
1217 		pathbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
1218 		path = pathbuf;
1219 
1220 		if (aux != NULL) {
1221 			vdev = ztest_shared->zs_vdev_aux;
1222 			(void) snprintf(pathbuf, MAXPATHLEN,
1223 			    ztest_aux_template, ztest_opts.zo_dir,
1224 			    pool == NULL ? ztest_opts.zo_pool : pool,
1225 			    aux, vdev);
1226 		} else {
1227 			vdev = ztest_shared->zs_vdev_next_leaf++;
1228 			(void) snprintf(pathbuf, MAXPATHLEN,
1229 			    ztest_dev_template, ztest_opts.zo_dir,
1230 			    pool == NULL ? ztest_opts.zo_pool : pool, vdev);
1231 		}
1232 	} else {
1233 		draid_spare = ztest_is_draid_spare(path);
1234 	}
1235 
1236 	if (size != 0 && !draid_spare) {
1237 		int fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
1238 		if (fd == -1)
1239 			fatal(B_TRUE, "can't open %s", path);
1240 		if (ftruncate(fd, size) != 0)
1241 			fatal(B_TRUE, "can't ftruncate %s", path);
1242 		(void) close(fd);
1243 	}
1244 
1245 	file = fnvlist_alloc();
1246 	fnvlist_add_string(file, ZPOOL_CONFIG_TYPE,
1247 	    draid_spare ? VDEV_TYPE_DRAID_SPARE : VDEV_TYPE_FILE);
1248 	fnvlist_add_string(file, ZPOOL_CONFIG_PATH, path);
1249 	fnvlist_add_uint64(file, ZPOOL_CONFIG_ASHIFT, ashift);
1250 	umem_free(pathbuf, MAXPATHLEN);
1251 
1252 	return (file);
1253 }
1254 
1255 static nvlist_t *
1256 make_vdev_raid(const char *path, const char *aux, const char *pool, size_t size,
1257     uint64_t ashift, int r)
1258 {
1259 	nvlist_t *raid, **child;
1260 	int c;
1261 
1262 	if (r < 2)
1263 		return (make_vdev_file(path, aux, pool, size, ashift));
1264 	child = umem_alloc(r * sizeof (nvlist_t *), UMEM_NOFAIL);
1265 
1266 	for (c = 0; c < r; c++)
1267 		child[c] = make_vdev_file(path, aux, pool, size, ashift);
1268 
1269 	raid = fnvlist_alloc();
1270 	fnvlist_add_string(raid, ZPOOL_CONFIG_TYPE,
1271 	    ztest_opts.zo_raid_type);
1272 	fnvlist_add_uint64(raid, ZPOOL_CONFIG_NPARITY,
1273 	    ztest_opts.zo_raid_parity);
1274 	fnvlist_add_nvlist_array(raid, ZPOOL_CONFIG_CHILDREN,
1275 	    (const nvlist_t **)child, r);
1276 
1277 	if (strcmp(ztest_opts.zo_raid_type, VDEV_TYPE_DRAID) == 0) {
1278 		uint64_t ndata = ztest_opts.zo_draid_data;
1279 		uint64_t nparity = ztest_opts.zo_raid_parity;
1280 		uint64_t nspares = ztest_opts.zo_draid_spares;
1281 		uint64_t children = ztest_opts.zo_raid_children;
1282 		uint64_t ngroups = 1;
1283 
1284 		/*
1285 		 * Calculate the minimum number of groups required to fill a
1286 		 * slice. This is the LCM of the stripe width (data + parity)
1287 		 * and the number of data drives (children - spares).
1288 		 */
1289 		while (ngroups * (ndata + nparity) % (children - nspares) != 0)
1290 			ngroups++;
1291 
1292 		/* Store the basic dRAID configuration. */
1293 		fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NDATA, ndata);
1294 		fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NSPARES, nspares);
1295 		fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NGROUPS, ngroups);
1296 	}
1297 
1298 	for (c = 0; c < r; c++)
1299 		fnvlist_free(child[c]);
1300 
1301 	umem_free(child, r * sizeof (nvlist_t *));
1302 
1303 	return (raid);
1304 }
1305 
1306 static nvlist_t *
1307 make_vdev_mirror(const char *path, const char *aux, const char *pool,
1308     size_t size, uint64_t ashift, int r, int m)
1309 {
1310 	nvlist_t *mirror, **child;
1311 	int c;
1312 
1313 	if (m < 1)
1314 		return (make_vdev_raid(path, aux, pool, size, ashift, r));
1315 
1316 	child = umem_alloc(m * sizeof (nvlist_t *), UMEM_NOFAIL);
1317 
1318 	for (c = 0; c < m; c++)
1319 		child[c] = make_vdev_raid(path, aux, pool, size, ashift, r);
1320 
1321 	mirror = fnvlist_alloc();
1322 	fnvlist_add_string(mirror, ZPOOL_CONFIG_TYPE, VDEV_TYPE_MIRROR);
1323 	fnvlist_add_nvlist_array(mirror, ZPOOL_CONFIG_CHILDREN,
1324 	    (const nvlist_t **)child, m);
1325 
1326 	for (c = 0; c < m; c++)
1327 		fnvlist_free(child[c]);
1328 
1329 	umem_free(child, m * sizeof (nvlist_t *));
1330 
1331 	return (mirror);
1332 }
1333 
1334 static nvlist_t *
1335 make_vdev_root(const char *path, const char *aux, const char *pool, size_t size,
1336     uint64_t ashift, const char *class, int r, int m, int t)
1337 {
1338 	nvlist_t *root, **child;
1339 	int c;
1340 	boolean_t log;
1341 
1342 	ASSERT3S(t, >, 0);
1343 
1344 	log = (class != NULL && strcmp(class, "log") == 0);
1345 
1346 	child = umem_alloc(t * sizeof (nvlist_t *), UMEM_NOFAIL);
1347 
1348 	for (c = 0; c < t; c++) {
1349 		child[c] = make_vdev_mirror(path, aux, pool, size, ashift,
1350 		    r, m);
1351 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_IS_LOG, log);
1352 
1353 		if (class != NULL && class[0] != '\0') {
1354 			ASSERT(m > 1 || log);   /* expecting a mirror */
1355 			fnvlist_add_string(child[c],
1356 			    ZPOOL_CONFIG_ALLOCATION_BIAS, class);
1357 		}
1358 	}
1359 
1360 	root = fnvlist_alloc();
1361 	fnvlist_add_string(root, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT);
1362 	fnvlist_add_nvlist_array(root, aux ? aux : ZPOOL_CONFIG_CHILDREN,
1363 	    (const nvlist_t **)child, t);
1364 
1365 	for (c = 0; c < t; c++)
1366 		fnvlist_free(child[c]);
1367 
1368 	umem_free(child, t * sizeof (nvlist_t *));
1369 
1370 	return (root);
1371 }
1372 
1373 /*
1374  * Find a random spa version. Returns back a random spa version in the
1375  * range [initial_version, SPA_VERSION_FEATURES].
1376  */
1377 static uint64_t
1378 ztest_random_spa_version(uint64_t initial_version)
1379 {
1380 	uint64_t version = initial_version;
1381 
1382 	if (version <= SPA_VERSION_BEFORE_FEATURES) {
1383 		version = version +
1384 		    ztest_random(SPA_VERSION_BEFORE_FEATURES - version + 1);
1385 	}
1386 
1387 	if (version > SPA_VERSION_BEFORE_FEATURES)
1388 		version = SPA_VERSION_FEATURES;
1389 
1390 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
1391 	return (version);
1392 }
1393 
1394 static int
1395 ztest_random_blocksize(void)
1396 {
1397 	ASSERT3U(ztest_spa->spa_max_ashift, !=, 0);
1398 
1399 	/*
1400 	 * Choose a block size >= the ashift.
1401 	 * If the SPA supports new MAXBLOCKSIZE, test up to 1MB blocks.
1402 	 */
1403 	int maxbs = SPA_OLD_MAXBLOCKSHIFT;
1404 	if (spa_maxblocksize(ztest_spa) == SPA_MAXBLOCKSIZE)
1405 		maxbs = 20;
1406 	uint64_t block_shift =
1407 	    ztest_random(maxbs - ztest_spa->spa_max_ashift + 1);
1408 	return (1 << (SPA_MINBLOCKSHIFT + block_shift));
1409 }
1410 
1411 static int
1412 ztest_random_dnodesize(void)
1413 {
1414 	int slots;
1415 	int max_slots = spa_maxdnodesize(ztest_spa) >> DNODE_SHIFT;
1416 
1417 	if (max_slots == DNODE_MIN_SLOTS)
1418 		return (DNODE_MIN_SIZE);
1419 
1420 	/*
1421 	 * Weight the random distribution more heavily toward smaller
1422 	 * dnode sizes since that is more likely to reflect real-world
1423 	 * usage.
1424 	 */
1425 	ASSERT3U(max_slots, >, 4);
1426 	switch (ztest_random(10)) {
1427 	case 0:
1428 		slots = 5 + ztest_random(max_slots - 4);
1429 		break;
1430 	case 1 ... 4:
1431 		slots = 2 + ztest_random(3);
1432 		break;
1433 	default:
1434 		slots = 1;
1435 		break;
1436 	}
1437 
1438 	return (slots << DNODE_SHIFT);
1439 }
1440 
1441 static int
1442 ztest_random_ibshift(void)
1443 {
1444 	return (DN_MIN_INDBLKSHIFT +
1445 	    ztest_random(DN_MAX_INDBLKSHIFT - DN_MIN_INDBLKSHIFT + 1));
1446 }
1447 
1448 static uint64_t
1449 ztest_random_vdev_top(spa_t *spa, boolean_t log_ok)
1450 {
1451 	uint64_t top;
1452 	vdev_t *rvd = spa->spa_root_vdev;
1453 	vdev_t *tvd;
1454 
1455 	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
1456 
1457 	do {
1458 		top = ztest_random(rvd->vdev_children);
1459 		tvd = rvd->vdev_child[top];
1460 	} while (!vdev_is_concrete(tvd) || (tvd->vdev_islog && !log_ok) ||
1461 	    tvd->vdev_mg == NULL || tvd->vdev_mg->mg_class == NULL);
1462 
1463 	return (top);
1464 }
1465 
1466 static uint64_t
1467 ztest_random_dsl_prop(zfs_prop_t prop)
1468 {
1469 	uint64_t value;
1470 
1471 	do {
1472 		value = zfs_prop_random_value(prop, ztest_random(-1ULL));
1473 	} while (prop == ZFS_PROP_CHECKSUM && value == ZIO_CHECKSUM_OFF);
1474 
1475 	return (value);
1476 }
1477 
1478 static int
1479 ztest_dsl_prop_set_uint64(char *osname, zfs_prop_t prop, uint64_t value,
1480     boolean_t inherit)
1481 {
1482 	const char *propname = zfs_prop_to_name(prop);
1483 	const char *valname;
1484 	char *setpoint;
1485 	uint64_t curval;
1486 	int error;
1487 
1488 	error = dsl_prop_set_int(osname, propname,
1489 	    (inherit ? ZPROP_SRC_NONE : ZPROP_SRC_LOCAL), value);
1490 
1491 	if (error == ENOSPC) {
1492 		ztest_record_enospc(FTAG);
1493 		return (error);
1494 	}
1495 	ASSERT0(error);
1496 
1497 	setpoint = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
1498 	VERIFY0(dsl_prop_get_integer(osname, propname, &curval, setpoint));
1499 
1500 	if (ztest_opts.zo_verbose >= 6) {
1501 		int err;
1502 
1503 		err = zfs_prop_index_to_string(prop, curval, &valname);
1504 		if (err)
1505 			(void) printf("%s %s = %llu at '%s'\n", osname,
1506 			    propname, (unsigned long long)curval, setpoint);
1507 		else
1508 			(void) printf("%s %s = %s at '%s'\n",
1509 			    osname, propname, valname, setpoint);
1510 	}
1511 	umem_free(setpoint, MAXPATHLEN);
1512 
1513 	return (error);
1514 }
1515 
1516 static int
1517 ztest_spa_prop_set_uint64(zpool_prop_t prop, uint64_t value)
1518 {
1519 	spa_t *spa = ztest_spa;
1520 	nvlist_t *props = NULL;
1521 	int error;
1522 
1523 	props = fnvlist_alloc();
1524 	fnvlist_add_uint64(props, zpool_prop_to_name(prop), value);
1525 
1526 	error = spa_prop_set(spa, props);
1527 
1528 	fnvlist_free(props);
1529 
1530 	if (error == ENOSPC) {
1531 		ztest_record_enospc(FTAG);
1532 		return (error);
1533 	}
1534 	ASSERT0(error);
1535 
1536 	return (error);
1537 }
1538 
1539 static int
1540 ztest_dmu_objset_own(const char *name, dmu_objset_type_t type,
1541     boolean_t readonly, boolean_t decrypt, const void *tag, objset_t **osp)
1542 {
1543 	int err;
1544 	char *cp = NULL;
1545 	char ddname[ZFS_MAX_DATASET_NAME_LEN];
1546 
1547 	strlcpy(ddname, name, sizeof (ddname));
1548 	cp = strchr(ddname, '@');
1549 	if (cp != NULL)
1550 		*cp = '\0';
1551 
1552 	err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
1553 	while (decrypt && err == EACCES) {
1554 		dsl_crypto_params_t *dcp;
1555 		nvlist_t *crypto_args = fnvlist_alloc();
1556 
1557 		fnvlist_add_uint8_array(crypto_args, "wkeydata",
1558 		    (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
1559 		VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, NULL,
1560 		    crypto_args, &dcp));
1561 		err = spa_keystore_load_wkey(ddname, dcp, B_FALSE);
1562 		/*
1563 		 * Note: if there was an error loading, the wkey was not
1564 		 * consumed, and needs to be freed.
1565 		 */
1566 		dsl_crypto_params_free(dcp, (err != 0));
1567 		fnvlist_free(crypto_args);
1568 
1569 		if (err == EINVAL) {
1570 			/*
1571 			 * We couldn't load a key for this dataset so try
1572 			 * the parent. This loop will eventually hit the
1573 			 * encryption root since ztest only makes clones
1574 			 * as children of their origin datasets.
1575 			 */
1576 			cp = strrchr(ddname, '/');
1577 			if (cp == NULL)
1578 				return (err);
1579 
1580 			*cp = '\0';
1581 			err = EACCES;
1582 			continue;
1583 		} else if (err != 0) {
1584 			break;
1585 		}
1586 
1587 		err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
1588 		break;
1589 	}
1590 
1591 	return (err);
1592 }
1593 
1594 static void
1595 ztest_rll_init(rll_t *rll)
1596 {
1597 	rll->rll_writer = NULL;
1598 	rll->rll_readers = 0;
1599 	mutex_init(&rll->rll_lock, NULL, MUTEX_DEFAULT, NULL);
1600 	cv_init(&rll->rll_cv, NULL, CV_DEFAULT, NULL);
1601 }
1602 
1603 static void
1604 ztest_rll_destroy(rll_t *rll)
1605 {
1606 	ASSERT3P(rll->rll_writer, ==, NULL);
1607 	ASSERT0(rll->rll_readers);
1608 	mutex_destroy(&rll->rll_lock);
1609 	cv_destroy(&rll->rll_cv);
1610 }
1611 
1612 static void
1613 ztest_rll_lock(rll_t *rll, rl_type_t type)
1614 {
1615 	mutex_enter(&rll->rll_lock);
1616 
1617 	if (type == RL_READER) {
1618 		while (rll->rll_writer != NULL)
1619 			(void) cv_wait(&rll->rll_cv, &rll->rll_lock);
1620 		rll->rll_readers++;
1621 	} else {
1622 		while (rll->rll_writer != NULL || rll->rll_readers)
1623 			(void) cv_wait(&rll->rll_cv, &rll->rll_lock);
1624 		rll->rll_writer = curthread;
1625 	}
1626 
1627 	mutex_exit(&rll->rll_lock);
1628 }
1629 
1630 static void
1631 ztest_rll_unlock(rll_t *rll)
1632 {
1633 	mutex_enter(&rll->rll_lock);
1634 
1635 	if (rll->rll_writer) {
1636 		ASSERT0(rll->rll_readers);
1637 		rll->rll_writer = NULL;
1638 	} else {
1639 		ASSERT3S(rll->rll_readers, >, 0);
1640 		ASSERT3P(rll->rll_writer, ==, NULL);
1641 		rll->rll_readers--;
1642 	}
1643 
1644 	if (rll->rll_writer == NULL && rll->rll_readers == 0)
1645 		cv_broadcast(&rll->rll_cv);
1646 
1647 	mutex_exit(&rll->rll_lock);
1648 }
1649 
1650 static void
1651 ztest_object_lock(ztest_ds_t *zd, uint64_t object, rl_type_t type)
1652 {
1653 	rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];
1654 
1655 	ztest_rll_lock(rll, type);
1656 }
1657 
1658 static void
1659 ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
1660 {
1661 	rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];
1662 
1663 	ztest_rll_unlock(rll);
1664 }
1665 
1666 static rl_t *
1667 ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
1668     uint64_t size, rl_type_t type)
1669 {
1670 	uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
1671 	rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
1672 	rl_t *rl;
1673 
1674 	rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
1675 	rl->rl_object = object;
1676 	rl->rl_offset = offset;
1677 	rl->rl_size = size;
1678 	rl->rl_lock = rll;
1679 
1680 	ztest_rll_lock(rll, type);
1681 
1682 	return (rl);
1683 }
1684 
1685 static void
1686 ztest_range_unlock(rl_t *rl)
1687 {
1688 	rll_t *rll = rl->rl_lock;
1689 
1690 	ztest_rll_unlock(rll);
1691 
1692 	umem_free(rl, sizeof (*rl));
1693 }
1694 
1695 static void
1696 ztest_zd_init(ztest_ds_t *zd, ztest_shared_ds_t *szd, objset_t *os)
1697 {
1698 	zd->zd_os = os;
1699 	zd->zd_zilog = dmu_objset_zil(os);
1700 	zd->zd_shared = szd;
1701 	dmu_objset_name(os, zd->zd_name);
1702 	int l;
1703 
1704 	if (zd->zd_shared != NULL)
1705 		zd->zd_shared->zd_seq = 0;
1706 
1707 	VERIFY0(pthread_rwlock_init(&zd->zd_zilog_lock, NULL));
1708 	mutex_init(&zd->zd_dirobj_lock, NULL, MUTEX_DEFAULT, NULL);
1709 
1710 	for (l = 0; l < ZTEST_OBJECT_LOCKS; l++)
1711 		ztest_rll_init(&zd->zd_object_lock[l]);
1712 
1713 	for (l = 0; l < ZTEST_RANGE_LOCKS; l++)
1714 		ztest_rll_init(&zd->zd_range_lock[l]);
1715 }
1716 
1717 static void
1718 ztest_zd_fini(ztest_ds_t *zd)
1719 {
1720 	int l;
1721 
1722 	mutex_destroy(&zd->zd_dirobj_lock);
1723 	(void) pthread_rwlock_destroy(&zd->zd_zilog_lock);
1724 
1725 	for (l = 0; l < ZTEST_OBJECT_LOCKS; l++)
1726 		ztest_rll_destroy(&zd->zd_object_lock[l]);
1727 
1728 	for (l = 0; l < ZTEST_RANGE_LOCKS; l++)
1729 		ztest_rll_destroy(&zd->zd_range_lock[l]);
1730 }
1731 
1732 #define	TXG_MIGHTWAIT	(ztest_random(10) == 0 ? TXG_NOWAIT : TXG_WAIT)
1733 
1734 static uint64_t
1735 ztest_tx_assign(dmu_tx_t *tx, uint64_t txg_how, const char *tag)
1736 {
1737 	uint64_t txg;
1738 	int error;
1739 
1740 	/*
1741 	 * Attempt to assign tx to some transaction group.
1742 	 */
1743 	error = dmu_tx_assign(tx, txg_how);
1744 	if (error) {
1745 		if (error == ERESTART) {
1746 			ASSERT3U(txg_how, ==, TXG_NOWAIT);
1747 			dmu_tx_wait(tx);
1748 		} else {
1749 			ASSERT3U(error, ==, ENOSPC);
1750 			ztest_record_enospc(tag);
1751 		}
1752 		dmu_tx_abort(tx);
1753 		return (0);
1754 	}
1755 	txg = dmu_tx_get_txg(tx);
1756 	ASSERT3U(txg, !=, 0);
1757 	return (txg);
1758 }
1759 
1760 static void
1761 ztest_bt_generate(ztest_block_tag_t *bt, objset_t *os, uint64_t object,
1762     uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
1763     uint64_t crtxg)
1764 {
1765 	bt->bt_magic = BT_MAGIC;
1766 	bt->bt_objset = dmu_objset_id(os);
1767 	bt->bt_object = object;
1768 	bt->bt_dnodesize = dnodesize;
1769 	bt->bt_offset = offset;
1770 	bt->bt_gen = gen;
1771 	bt->bt_txg = txg;
1772 	bt->bt_crtxg = crtxg;
1773 }
1774 
1775 static void
1776 ztest_bt_verify(ztest_block_tag_t *bt, objset_t *os, uint64_t object,
1777     uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
1778     uint64_t crtxg)
1779 {
1780 	ASSERT3U(bt->bt_magic, ==, BT_MAGIC);
1781 	ASSERT3U(bt->bt_objset, ==, dmu_objset_id(os));
1782 	ASSERT3U(bt->bt_object, ==, object);
1783 	ASSERT3U(bt->bt_dnodesize, ==, dnodesize);
1784 	ASSERT3U(bt->bt_offset, ==, offset);
1785 	ASSERT3U(bt->bt_gen, <=, gen);
1786 	ASSERT3U(bt->bt_txg, <=, txg);
1787 	ASSERT3U(bt->bt_crtxg, ==, crtxg);
1788 }
1789 
1790 static ztest_block_tag_t *
1791 ztest_bt_bonus(dmu_buf_t *db)
1792 {
1793 	dmu_object_info_t doi;
1794 	ztest_block_tag_t *bt;
1795 
1796 	dmu_object_info_from_db(db, &doi);
1797 	ASSERT3U(doi.doi_bonus_size, <=, db->db_size);
1798 	ASSERT3U(doi.doi_bonus_size, >=, sizeof (*bt));
1799 	bt = (void *)((char *)db->db_data + doi.doi_bonus_size - sizeof (*bt));
1800 
1801 	return (bt);
1802 }
1803 
1804 /*
1805  * Generate a token to fill up unused bonus buffer space.  Try to make
1806  * it unique to the object, generation, and offset to verify that data
1807  * is not getting overwritten by data from other dnodes.
1808  */
1809 #define	ZTEST_BONUS_FILL_TOKEN(obj, ds, gen, offset) \
1810 	(((ds) << 48) | ((gen) << 32) | ((obj) << 8) | (offset))
1811 
1812 /*
1813  * Fill up the unused bonus buffer region before the block tag with a
1814  * verifiable pattern. Filling the whole bonus area with non-zero data
1815  * helps ensure that all dnode traversal code properly skips the
1816  * interior regions of large dnodes.
1817  */
1818 static void
1819 ztest_fill_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
1820     objset_t *os, uint64_t gen)
1821 {
1822 	uint64_t *bonusp;
1823 
1824 	ASSERT(IS_P2ALIGNED((char *)end - (char *)db->db_data, 8));
1825 
1826 	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
1827 		uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
1828 		    gen, bonusp - (uint64_t *)db->db_data);
1829 		*bonusp = token;
1830 	}
1831 }
1832 
1833 /*
1834  * Verify that the unused area of a bonus buffer is filled with the
1835  * expected tokens.
1836  */
1837 static void
1838 ztest_verify_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
1839     objset_t *os, uint64_t gen)
1840 {
1841 	uint64_t *bonusp;
1842 
1843 	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
1844 		uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
1845 		    gen, bonusp - (uint64_t *)db->db_data);
1846 		VERIFY3U(*bonusp, ==, token);
1847 	}
1848 }
1849 
1850 /*
1851  * ZIL logging ops
1852  */
1853 
1854 #define	lrz_type	lr_mode
1855 #define	lrz_blocksize	lr_uid
1856 #define	lrz_ibshift	lr_gid
1857 #define	lrz_bonustype	lr_rdev
1858 #define	lrz_dnodesize	lr_crtime[1]
1859 
1860 static void
1861 ztest_log_create(ztest_ds_t *zd, dmu_tx_t *tx, lr_create_t *lr)
1862 {
1863 	char *name = (void *)(lr + 1);		/* name follows lr */
1864 	size_t namesize = strlen(name) + 1;
1865 	itx_t *itx;
1866 
1867 	if (zil_replaying(zd->zd_zilog, tx))
1868 		return;
1869 
1870 	itx = zil_itx_create(TX_CREATE, sizeof (*lr) + namesize);
1871 	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1872 	    sizeof (*lr) + namesize - sizeof (lr_t));
1873 
1874 	zil_itx_assign(zd->zd_zilog, itx, tx);
1875 }
1876 
1877 static void
1878 ztest_log_remove(ztest_ds_t *zd, dmu_tx_t *tx, lr_remove_t *lr, uint64_t object)
1879 {
1880 	char *name = (void *)(lr + 1);		/* name follows lr */
1881 	size_t namesize = strlen(name) + 1;
1882 	itx_t *itx;
1883 
1884 	if (zil_replaying(zd->zd_zilog, tx))
1885 		return;
1886 
1887 	itx = zil_itx_create(TX_REMOVE, sizeof (*lr) + namesize);
1888 	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1889 	    sizeof (*lr) + namesize - sizeof (lr_t));
1890 
1891 	itx->itx_oid = object;
1892 	zil_itx_assign(zd->zd_zilog, itx, tx);
1893 }
1894 
1895 static void
1896 ztest_log_write(ztest_ds_t *zd, dmu_tx_t *tx, lr_write_t *lr)
1897 {
1898 	itx_t *itx;
1899 	itx_wr_state_t write_state = ztest_random(WR_NUM_STATES);
1900 
1901 	if (zil_replaying(zd->zd_zilog, tx))
1902 		return;
1903 
1904 	if (lr->lr_length > zil_max_log_data(zd->zd_zilog))
1905 		write_state = WR_INDIRECT;
1906 
1907 	itx = zil_itx_create(TX_WRITE,
1908 	    sizeof (*lr) + (write_state == WR_COPIED ? lr->lr_length : 0));
1909 
1910 	if (write_state == WR_COPIED &&
1911 	    dmu_read(zd->zd_os, lr->lr_foid, lr->lr_offset, lr->lr_length,
1912 	    ((lr_write_t *)&itx->itx_lr) + 1, DMU_READ_NO_PREFETCH) != 0) {
1913 		zil_itx_destroy(itx);
1914 		itx = zil_itx_create(TX_WRITE, sizeof (*lr));
1915 		write_state = WR_NEED_COPY;
1916 	}
1917 	itx->itx_private = zd;
1918 	itx->itx_wr_state = write_state;
1919 	itx->itx_sync = (ztest_random(8) == 0);
1920 
1921 	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1922 	    sizeof (*lr) - sizeof (lr_t));
1923 
1924 	zil_itx_assign(zd->zd_zilog, itx, tx);
1925 }
1926 
1927 static void
1928 ztest_log_truncate(ztest_ds_t *zd, dmu_tx_t *tx, lr_truncate_t *lr)
1929 {
1930 	itx_t *itx;
1931 
1932 	if (zil_replaying(zd->zd_zilog, tx))
1933 		return;
1934 
1935 	itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
1936 	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1937 	    sizeof (*lr) - sizeof (lr_t));
1938 
1939 	itx->itx_sync = B_FALSE;
1940 	zil_itx_assign(zd->zd_zilog, itx, tx);
1941 }
1942 
1943 static void
1944 ztest_log_setattr(ztest_ds_t *zd, dmu_tx_t *tx, lr_setattr_t *lr)
1945 {
1946 	itx_t *itx;
1947 
1948 	if (zil_replaying(zd->zd_zilog, tx))
1949 		return;
1950 
1951 	itx = zil_itx_create(TX_SETATTR, sizeof (*lr));
1952 	memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1953 	    sizeof (*lr) - sizeof (lr_t));
1954 
1955 	itx->itx_sync = B_FALSE;
1956 	zil_itx_assign(zd->zd_zilog, itx, tx);
1957 }
1958 
1959 /*
1960  * ZIL replay ops
1961  */
1962 static int
1963 ztest_replay_create(void *arg1, void *arg2, boolean_t byteswap)
1964 {
1965 	ztest_ds_t *zd = arg1;
1966 	lr_create_t *lr = arg2;
1967 	char *name = (void *)(lr + 1);		/* name follows lr */
1968 	objset_t *os = zd->zd_os;
1969 	ztest_block_tag_t *bbt;
1970 	dmu_buf_t *db;
1971 	dmu_tx_t *tx;
1972 	uint64_t txg;
1973 	int error = 0;
1974 	int bonuslen;
1975 
1976 	if (byteswap)
1977 		byteswap_uint64_array(lr, sizeof (*lr));
1978 
1979 	ASSERT3U(lr->lr_doid, ==, ZTEST_DIROBJ);
1980 	ASSERT3S(name[0], !=, '\0');
1981 
1982 	tx = dmu_tx_create(os);
1983 
1984 	dmu_tx_hold_zap(tx, lr->lr_doid, B_TRUE, name);
1985 
1986 	if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
1987 		dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL);
1988 	} else {
1989 		dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1990 	}
1991 
1992 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1993 	if (txg == 0)
1994 		return (ENOSPC);
1995 
1996 	ASSERT3U(dmu_objset_zil(os)->zl_replay, ==, !!lr->lr_foid);
1997 	bonuslen = DN_BONUS_SIZE(lr->lrz_dnodesize);
1998 
1999 	if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
2000 		if (lr->lr_foid == 0) {
2001 			lr->lr_foid = zap_create_dnsize(os,
2002 			    lr->lrz_type, lr->lrz_bonustype,
2003 			    bonuslen, lr->lrz_dnodesize, tx);
2004 		} else {
2005 			error = zap_create_claim_dnsize(os, lr->lr_foid,
2006 			    lr->lrz_type, lr->lrz_bonustype,
2007 			    bonuslen, lr->lrz_dnodesize, tx);
2008 		}
2009 	} else {
2010 		if (lr->lr_foid == 0) {
2011 			lr->lr_foid = dmu_object_alloc_dnsize(os,
2012 			    lr->lrz_type, 0, lr->lrz_bonustype,
2013 			    bonuslen, lr->lrz_dnodesize, tx);
2014 		} else {
2015 			error = dmu_object_claim_dnsize(os, lr->lr_foid,
2016 			    lr->lrz_type, 0, lr->lrz_bonustype,
2017 			    bonuslen, lr->lrz_dnodesize, tx);
2018 		}
2019 	}
2020 
2021 	if (error) {
2022 		ASSERT3U(error, ==, EEXIST);
2023 		ASSERT(zd->zd_zilog->zl_replay);
2024 		dmu_tx_commit(tx);
2025 		return (error);
2026 	}
2027 
2028 	ASSERT3U(lr->lr_foid, !=, 0);
2029 
2030 	if (lr->lrz_type != DMU_OT_ZAP_OTHER)
2031 		VERIFY0(dmu_object_set_blocksize(os, lr->lr_foid,
2032 		    lr->lrz_blocksize, lr->lrz_ibshift, tx));
2033 
2034 	VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
2035 	bbt = ztest_bt_bonus(db);
2036 	dmu_buf_will_dirty(db, tx);
2037 	ztest_bt_generate(bbt, os, lr->lr_foid, lr->lrz_dnodesize, -1ULL,
2038 	    lr->lr_gen, txg, txg);
2039 	ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, lr->lr_gen);
2040 	dmu_buf_rele(db, FTAG);
2041 
2042 	VERIFY0(zap_add(os, lr->lr_doid, name, sizeof (uint64_t), 1,
2043 	    &lr->lr_foid, tx));
2044 
2045 	(void) ztest_log_create(zd, tx, lr);
2046 
2047 	dmu_tx_commit(tx);
2048 
2049 	return (0);
2050 }
2051 
2052 static int
2053 ztest_replay_remove(void *arg1, void *arg2, boolean_t byteswap)
2054 {
2055 	ztest_ds_t *zd = arg1;
2056 	lr_remove_t *lr = arg2;
2057 	char *name = (void *)(lr + 1);		/* name follows lr */
2058 	objset_t *os = zd->zd_os;
2059 	dmu_object_info_t doi;
2060 	dmu_tx_t *tx;
2061 	uint64_t object, txg;
2062 
2063 	if (byteswap)
2064 		byteswap_uint64_array(lr, sizeof (*lr));
2065 
2066 	ASSERT3U(lr->lr_doid, ==, ZTEST_DIROBJ);
2067 	ASSERT3S(name[0], !=, '\0');
2068 
2069 	VERIFY0(
2070 	    zap_lookup(os, lr->lr_doid, name, sizeof (object), 1, &object));
2071 	ASSERT3U(object, !=, 0);
2072 
2073 	ztest_object_lock(zd, object, RL_WRITER);
2074 
2075 	VERIFY0(dmu_object_info(os, object, &doi));
2076 
2077 	tx = dmu_tx_create(os);
2078 
2079 	dmu_tx_hold_zap(tx, lr->lr_doid, B_FALSE, name);
2080 	dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
2081 
2082 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2083 	if (txg == 0) {
2084 		ztest_object_unlock(zd, object);
2085 		return (ENOSPC);
2086 	}
2087 
2088 	if (doi.doi_type == DMU_OT_ZAP_OTHER) {
2089 		VERIFY0(zap_destroy(os, object, tx));
2090 	} else {
2091 		VERIFY0(dmu_object_free(os, object, tx));
2092 	}
2093 
2094 	VERIFY0(zap_remove(os, lr->lr_doid, name, tx));
2095 
2096 	(void) ztest_log_remove(zd, tx, lr, object);
2097 
2098 	dmu_tx_commit(tx);
2099 
2100 	ztest_object_unlock(zd, object);
2101 
2102 	return (0);
2103 }
2104 
2105 static int
2106 ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
2107 {
2108 	ztest_ds_t *zd = arg1;
2109 	lr_write_t *lr = arg2;
2110 	objset_t *os = zd->zd_os;
2111 	void *data = lr + 1;			/* data follows lr */
2112 	uint64_t offset, length;
2113 	ztest_block_tag_t *bt = data;
2114 	ztest_block_tag_t *bbt;
2115 	uint64_t gen, txg, lrtxg, crtxg;
2116 	dmu_object_info_t doi;
2117 	dmu_tx_t *tx;
2118 	dmu_buf_t *db;
2119 	arc_buf_t *abuf = NULL;
2120 	rl_t *rl;
2121 
2122 	if (byteswap)
2123 		byteswap_uint64_array(lr, sizeof (*lr));
2124 
2125 	offset = lr->lr_offset;
2126 	length = lr->lr_length;
2127 
2128 	/* If it's a dmu_sync() block, write the whole block */
2129 	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
2130 		uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
2131 		if (length < blocksize) {
2132 			offset -= offset % blocksize;
2133 			length = blocksize;
2134 		}
2135 	}
2136 
2137 	if (bt->bt_magic == BSWAP_64(BT_MAGIC))
2138 		byteswap_uint64_array(bt, sizeof (*bt));
2139 
2140 	if (bt->bt_magic != BT_MAGIC)
2141 		bt = NULL;
2142 
2143 	ztest_object_lock(zd, lr->lr_foid, RL_READER);
2144 	rl = ztest_range_lock(zd, lr->lr_foid, offset, length, RL_WRITER);
2145 
2146 	VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
2147 
2148 	dmu_object_info_from_db(db, &doi);
2149 
2150 	bbt = ztest_bt_bonus(db);
2151 	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2152 	gen = bbt->bt_gen;
2153 	crtxg = bbt->bt_crtxg;
2154 	lrtxg = lr->lr_common.lrc_txg;
2155 
2156 	tx = dmu_tx_create(os);
2157 
2158 	dmu_tx_hold_write(tx, lr->lr_foid, offset, length);
2159 
2160 	if (ztest_random(8) == 0 && length == doi.doi_data_block_size &&
2161 	    P2PHASE(offset, length) == 0)
2162 		abuf = dmu_request_arcbuf(db, length);
2163 
2164 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2165 	if (txg == 0) {
2166 		if (abuf != NULL)
2167 			dmu_return_arcbuf(abuf);
2168 		dmu_buf_rele(db, FTAG);
2169 		ztest_range_unlock(rl);
2170 		ztest_object_unlock(zd, lr->lr_foid);
2171 		return (ENOSPC);
2172 	}
2173 
2174 	if (bt != NULL) {
2175 		/*
2176 		 * Usually, verify the old data before writing new data --
2177 		 * but not always, because we also want to verify correct
2178 		 * behavior when the data was not recently read into cache.
2179 		 */
2180 		ASSERT0(offset % doi.doi_data_block_size);
2181 		if (ztest_random(4) != 0) {
2182 			int prefetch = ztest_random(2) ?
2183 			    DMU_READ_PREFETCH : DMU_READ_NO_PREFETCH;
2184 			ztest_block_tag_t rbt;
2185 
2186 			VERIFY(dmu_read(os, lr->lr_foid, offset,
2187 			    sizeof (rbt), &rbt, prefetch) == 0);
2188 			if (rbt.bt_magic == BT_MAGIC) {
2189 				ztest_bt_verify(&rbt, os, lr->lr_foid, 0,
2190 				    offset, gen, txg, crtxg);
2191 			}
2192 		}
2193 
2194 		/*
2195 		 * Writes can appear to be newer than the bonus buffer because
2196 		 * the ztest_get_data() callback does a dmu_read() of the
2197 		 * open-context data, which may be different than the data
2198 		 * as it was when the write was generated.
2199 		 */
2200 		if (zd->zd_zilog->zl_replay) {
2201 			ztest_bt_verify(bt, os, lr->lr_foid, 0, offset,
2202 			    MAX(gen, bt->bt_gen), MAX(txg, lrtxg),
2203 			    bt->bt_crtxg);
2204 		}
2205 
2206 		/*
2207 		 * Set the bt's gen/txg to the bonus buffer's gen/txg
2208 		 * so that all of the usual ASSERTs will work.
2209 		 */
2210 		ztest_bt_generate(bt, os, lr->lr_foid, 0, offset, gen, txg,
2211 		    crtxg);
2212 	}
2213 
2214 	if (abuf == NULL) {
2215 		dmu_write(os, lr->lr_foid, offset, length, data, tx);
2216 	} else {
2217 		memcpy(abuf->b_data, data, length);
2218 		VERIFY0(dmu_assign_arcbuf_by_dbuf(db, offset, abuf, tx));
2219 	}
2220 
2221 	(void) ztest_log_write(zd, tx, lr);
2222 
2223 	dmu_buf_rele(db, FTAG);
2224 
2225 	dmu_tx_commit(tx);
2226 
2227 	ztest_range_unlock(rl);
2228 	ztest_object_unlock(zd, lr->lr_foid);
2229 
2230 	return (0);
2231 }
2232 
2233 static int
2234 ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
2235 {
2236 	ztest_ds_t *zd = arg1;
2237 	lr_truncate_t *lr = arg2;
2238 	objset_t *os = zd->zd_os;
2239 	dmu_tx_t *tx;
2240 	uint64_t txg;
2241 	rl_t *rl;
2242 
2243 	if (byteswap)
2244 		byteswap_uint64_array(lr, sizeof (*lr));
2245 
2246 	ztest_object_lock(zd, lr->lr_foid, RL_READER);
2247 	rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
2248 	    RL_WRITER);
2249 
2250 	tx = dmu_tx_create(os);
2251 
2252 	dmu_tx_hold_free(tx, lr->lr_foid, lr->lr_offset, lr->lr_length);
2253 
2254 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2255 	if (txg == 0) {
2256 		ztest_range_unlock(rl);
2257 		ztest_object_unlock(zd, lr->lr_foid);
2258 		return (ENOSPC);
2259 	}
2260 
2261 	VERIFY0(dmu_free_range(os, lr->lr_foid, lr->lr_offset,
2262 	    lr->lr_length, tx));
2263 
2264 	(void) ztest_log_truncate(zd, tx, lr);
2265 
2266 	dmu_tx_commit(tx);
2267 
2268 	ztest_range_unlock(rl);
2269 	ztest_object_unlock(zd, lr->lr_foid);
2270 
2271 	return (0);
2272 }
2273 
2274 static int
2275 ztest_replay_setattr(void *arg1, void *arg2, boolean_t byteswap)
2276 {
2277 	ztest_ds_t *zd = arg1;
2278 	lr_setattr_t *lr = arg2;
2279 	objset_t *os = zd->zd_os;
2280 	dmu_tx_t *tx;
2281 	dmu_buf_t *db;
2282 	ztest_block_tag_t *bbt;
2283 	uint64_t txg, lrtxg, crtxg, dnodesize;
2284 
2285 	if (byteswap)
2286 		byteswap_uint64_array(lr, sizeof (*lr));
2287 
2288 	ztest_object_lock(zd, lr->lr_foid, RL_WRITER);
2289 
2290 	VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
2291 
2292 	tx = dmu_tx_create(os);
2293 	dmu_tx_hold_bonus(tx, lr->lr_foid);
2294 
2295 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2296 	if (txg == 0) {
2297 		dmu_buf_rele(db, FTAG);
2298 		ztest_object_unlock(zd, lr->lr_foid);
2299 		return (ENOSPC);
2300 	}
2301 
2302 	bbt = ztest_bt_bonus(db);
2303 	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2304 	crtxg = bbt->bt_crtxg;
2305 	lrtxg = lr->lr_common.lrc_txg;
2306 	dnodesize = bbt->bt_dnodesize;
2307 
2308 	if (zd->zd_zilog->zl_replay) {
2309 		ASSERT3U(lr->lr_size, !=, 0);
2310 		ASSERT3U(lr->lr_mode, !=, 0);
2311 		ASSERT3U(lrtxg, !=, 0);
2312 	} else {
2313 		/*
2314 		 * Randomly change the size and increment the generation.
2315 		 */
2316 		lr->lr_size = (ztest_random(db->db_size / sizeof (*bbt)) + 1) *
2317 		    sizeof (*bbt);
2318 		lr->lr_mode = bbt->bt_gen + 1;
2319 		ASSERT0(lrtxg);
2320 	}
2321 
2322 	/*
2323 	 * Verify that the current bonus buffer is not newer than our txg.
2324 	 */
2325 	ztest_bt_verify(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
2326 	    MAX(txg, lrtxg), crtxg);
2327 
2328 	dmu_buf_will_dirty(db, tx);
2329 
2330 	ASSERT3U(lr->lr_size, >=, sizeof (*bbt));
2331 	ASSERT3U(lr->lr_size, <=, db->db_size);
2332 	VERIFY0(dmu_set_bonus(db, lr->lr_size, tx));
2333 	bbt = ztest_bt_bonus(db);
2334 
2335 	ztest_bt_generate(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
2336 	    txg, crtxg);
2337 	ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, bbt->bt_gen);
2338 	dmu_buf_rele(db, FTAG);
2339 
2340 	(void) ztest_log_setattr(zd, tx, lr);
2341 
2342 	dmu_tx_commit(tx);
2343 
2344 	ztest_object_unlock(zd, lr->lr_foid);
2345 
2346 	return (0);
2347 }
2348 
2349 zil_replay_func_t *ztest_replay_vector[TX_MAX_TYPE] = {
2350 	NULL,			/* 0 no such transaction type */
2351 	ztest_replay_create,	/* TX_CREATE */
2352 	NULL,			/* TX_MKDIR */
2353 	NULL,			/* TX_MKXATTR */
2354 	NULL,			/* TX_SYMLINK */
2355 	ztest_replay_remove,	/* TX_REMOVE */
2356 	NULL,			/* TX_RMDIR */
2357 	NULL,			/* TX_LINK */
2358 	NULL,			/* TX_RENAME */
2359 	ztest_replay_write,	/* TX_WRITE */
2360 	ztest_replay_truncate,	/* TX_TRUNCATE */
2361 	ztest_replay_setattr,	/* TX_SETATTR */
2362 	NULL,			/* TX_ACL */
2363 	NULL,			/* TX_CREATE_ACL */
2364 	NULL,			/* TX_CREATE_ATTR */
2365 	NULL,			/* TX_CREATE_ACL_ATTR */
2366 	NULL,			/* TX_MKDIR_ACL */
2367 	NULL,			/* TX_MKDIR_ATTR */
2368 	NULL,			/* TX_MKDIR_ACL_ATTR */
2369 	NULL,			/* TX_WRITE2 */
2370 	NULL,			/* TX_SETSAXATTR */
2371 };
2372 
2373 /*
2374  * ZIL get_data callbacks
2375  */
2376 
2377 static void
2378 ztest_get_done(zgd_t *zgd, int error)
2379 {
2380 	(void) error;
2381 	ztest_ds_t *zd = zgd->zgd_private;
2382 	uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;
2383 
2384 	if (zgd->zgd_db)
2385 		dmu_buf_rele(zgd->zgd_db, zgd);
2386 
2387 	ztest_range_unlock((rl_t *)zgd->zgd_lr);
2388 	ztest_object_unlock(zd, object);
2389 
2390 	umem_free(zgd, sizeof (*zgd));
2391 }
2392 
2393 static int
2394 ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
2395     struct lwb *lwb, zio_t *zio)
2396 {
2397 	(void) arg2;
2398 	ztest_ds_t *zd = arg;
2399 	objset_t *os = zd->zd_os;
2400 	uint64_t object = lr->lr_foid;
2401 	uint64_t offset = lr->lr_offset;
2402 	uint64_t size = lr->lr_length;
2403 	uint64_t txg = lr->lr_common.lrc_txg;
2404 	uint64_t crtxg;
2405 	dmu_object_info_t doi;
2406 	dmu_buf_t *db;
2407 	zgd_t *zgd;
2408 	int error;
2409 
2410 	ASSERT3P(lwb, !=, NULL);
2411 	ASSERT3P(zio, !=, NULL);
2412 	ASSERT3U(size, !=, 0);
2413 
2414 	ztest_object_lock(zd, object, RL_READER);
2415 	error = dmu_bonus_hold(os, object, FTAG, &db);
2416 	if (error) {
2417 		ztest_object_unlock(zd, object);
2418 		return (error);
2419 	}
2420 
2421 	crtxg = ztest_bt_bonus(db)->bt_crtxg;
2422 
2423 	if (crtxg == 0 || crtxg > txg) {
2424 		dmu_buf_rele(db, FTAG);
2425 		ztest_object_unlock(zd, object);
2426 		return (ENOENT);
2427 	}
2428 
2429 	dmu_object_info_from_db(db, &doi);
2430 	dmu_buf_rele(db, FTAG);
2431 	db = NULL;
2432 
2433 	zgd = umem_zalloc(sizeof (*zgd), UMEM_NOFAIL);
2434 	zgd->zgd_lwb = lwb;
2435 	zgd->zgd_private = zd;
2436 
2437 	if (buf != NULL) {	/* immediate write */
2438 		zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2439 		    object, offset, size, RL_READER);
2440 
2441 		error = dmu_read(os, object, offset, size, buf,
2442 		    DMU_READ_NO_PREFETCH);
2443 		ASSERT0(error);
2444 	} else {
2445 		size = doi.doi_data_block_size;
2446 		if (ISP2(size)) {
2447 			offset = P2ALIGN(offset, size);
2448 		} else {
2449 			ASSERT3U(offset, <, size);
2450 			offset = 0;
2451 		}
2452 
2453 		zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2454 		    object, offset, size, RL_READER);
2455 
2456 		error = dmu_buf_hold(os, object, offset, zgd, &db,
2457 		    DMU_READ_NO_PREFETCH);
2458 
2459 		if (error == 0) {
2460 			blkptr_t *bp = &lr->lr_blkptr;
2461 
2462 			zgd->zgd_db = db;
2463 			zgd->zgd_bp = bp;
2464 
2465 			ASSERT3U(db->db_offset, ==, offset);
2466 			ASSERT3U(db->db_size, ==, size);
2467 
2468 			error = dmu_sync(zio, lr->lr_common.lrc_txg,
2469 			    ztest_get_done, zgd);
2470 
2471 			if (error == 0)
2472 				return (0);
2473 		}
2474 	}
2475 
2476 	ztest_get_done(zgd, error);
2477 
2478 	return (error);
2479 }
2480 
2481 static void *
2482 ztest_lr_alloc(size_t lrsize, char *name)
2483 {
2484 	char *lr;
2485 	size_t namesize = name ? strlen(name) + 1 : 0;
2486 
2487 	lr = umem_zalloc(lrsize + namesize, UMEM_NOFAIL);
2488 
2489 	if (name)
2490 		memcpy(lr + lrsize, name, namesize);
2491 
2492 	return (lr);
2493 }
2494 
2495 static void
2496 ztest_lr_free(void *lr, size_t lrsize, char *name)
2497 {
2498 	size_t namesize = name ? strlen(name) + 1 : 0;
2499 
2500 	umem_free(lr, lrsize + namesize);
2501 }
2502 
2503 /*
2504  * Lookup a bunch of objects.  Returns the number of objects not found.
2505  */
2506 static int
2507 ztest_lookup(ztest_ds_t *zd, ztest_od_t *od, int count)
2508 {
2509 	int missing = 0;
2510 	int error;
2511 	int i;
2512 
2513 	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2514 
2515 	for (i = 0; i < count; i++, od++) {
2516 		od->od_object = 0;
2517 		error = zap_lookup(zd->zd_os, od->od_dir, od->od_name,
2518 		    sizeof (uint64_t), 1, &od->od_object);
2519 		if (error) {
2520 			ASSERT3S(error, ==, ENOENT);
2521 			ASSERT0(od->od_object);
2522 			missing++;
2523 		} else {
2524 			dmu_buf_t *db;
2525 			ztest_block_tag_t *bbt;
2526 			dmu_object_info_t doi;
2527 
2528 			ASSERT3U(od->od_object, !=, 0);
2529 			ASSERT0(missing);	/* there should be no gaps */
2530 
2531 			ztest_object_lock(zd, od->od_object, RL_READER);
2532 			VERIFY0(dmu_bonus_hold(zd->zd_os, od->od_object,
2533 			    FTAG, &db));
2534 			dmu_object_info_from_db(db, &doi);
2535 			bbt = ztest_bt_bonus(db);
2536 			ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2537 			od->od_type = doi.doi_type;
2538 			od->od_blocksize = doi.doi_data_block_size;
2539 			od->od_gen = bbt->bt_gen;
2540 			dmu_buf_rele(db, FTAG);
2541 			ztest_object_unlock(zd, od->od_object);
2542 		}
2543 	}
2544 
2545 	return (missing);
2546 }
2547 
2548 static int
2549 ztest_create(ztest_ds_t *zd, ztest_od_t *od, int count)
2550 {
2551 	int missing = 0;
2552 	int i;
2553 
2554 	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2555 
2556 	for (i = 0; i < count; i++, od++) {
2557 		if (missing) {
2558 			od->od_object = 0;
2559 			missing++;
2560 			continue;
2561 		}
2562 
2563 		lr_create_t *lr = ztest_lr_alloc(sizeof (*lr), od->od_name);
2564 
2565 		lr->lr_doid = od->od_dir;
2566 		lr->lr_foid = 0;	/* 0 to allocate, > 0 to claim */
2567 		lr->lrz_type = od->od_crtype;
2568 		lr->lrz_blocksize = od->od_crblocksize;
2569 		lr->lrz_ibshift = ztest_random_ibshift();
2570 		lr->lrz_bonustype = DMU_OT_UINT64_OTHER;
2571 		lr->lrz_dnodesize = od->od_crdnodesize;
2572 		lr->lr_gen = od->od_crgen;
2573 		lr->lr_crtime[0] = time(NULL);
2574 
2575 		if (ztest_replay_create(zd, lr, B_FALSE) != 0) {
2576 			ASSERT0(missing);
2577 			od->od_object = 0;
2578 			missing++;
2579 		} else {
2580 			od->od_object = lr->lr_foid;
2581 			od->od_type = od->od_crtype;
2582 			od->od_blocksize = od->od_crblocksize;
2583 			od->od_gen = od->od_crgen;
2584 			ASSERT3U(od->od_object, !=, 0);
2585 		}
2586 
2587 		ztest_lr_free(lr, sizeof (*lr), od->od_name);
2588 	}
2589 
2590 	return (missing);
2591 }
2592 
2593 static int
2594 ztest_remove(ztest_ds_t *zd, ztest_od_t *od, int count)
2595 {
2596 	int missing = 0;
2597 	int error;
2598 	int i;
2599 
2600 	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2601 
2602 	od += count - 1;
2603 
2604 	for (i = count - 1; i >= 0; i--, od--) {
2605 		if (missing) {
2606 			missing++;
2607 			continue;
2608 		}
2609 
2610 		/*
2611 		 * No object was found.
2612 		 */
2613 		if (od->od_object == 0)
2614 			continue;
2615 
2616 		lr_remove_t *lr = ztest_lr_alloc(sizeof (*lr), od->od_name);
2617 
2618 		lr->lr_doid = od->od_dir;
2619 
2620 		if ((error = ztest_replay_remove(zd, lr, B_FALSE)) != 0) {
2621 			ASSERT3U(error, ==, ENOSPC);
2622 			missing++;
2623 		} else {
2624 			od->od_object = 0;
2625 		}
2626 		ztest_lr_free(lr, sizeof (*lr), od->od_name);
2627 	}
2628 
2629 	return (missing);
2630 }
2631 
2632 static int
2633 ztest_write(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size,
2634     void *data)
2635 {
2636 	lr_write_t *lr;
2637 	int error;
2638 
2639 	lr = ztest_lr_alloc(sizeof (*lr) + size, NULL);
2640 
2641 	lr->lr_foid = object;
2642 	lr->lr_offset = offset;
2643 	lr->lr_length = size;
2644 	lr->lr_blkoff = 0;
2645 	BP_ZERO(&lr->lr_blkptr);
2646 
2647 	memcpy(lr + 1, data, size);
2648 
2649 	error = ztest_replay_write(zd, lr, B_FALSE);
2650 
2651 	ztest_lr_free(lr, sizeof (*lr) + size, NULL);
2652 
2653 	return (error);
2654 }
2655 
2656 static int
2657 ztest_truncate(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
2658 {
2659 	lr_truncate_t *lr;
2660 	int error;
2661 
2662 	lr = ztest_lr_alloc(sizeof (*lr), NULL);
2663 
2664 	lr->lr_foid = object;
2665 	lr->lr_offset = offset;
2666 	lr->lr_length = size;
2667 
2668 	error = ztest_replay_truncate(zd, lr, B_FALSE);
2669 
2670 	ztest_lr_free(lr, sizeof (*lr), NULL);
2671 
2672 	return (error);
2673 }
2674 
2675 static int
2676 ztest_setattr(ztest_ds_t *zd, uint64_t object)
2677 {
2678 	lr_setattr_t *lr;
2679 	int error;
2680 
2681 	lr = ztest_lr_alloc(sizeof (*lr), NULL);
2682 
2683 	lr->lr_foid = object;
2684 	lr->lr_size = 0;
2685 	lr->lr_mode = 0;
2686 
2687 	error = ztest_replay_setattr(zd, lr, B_FALSE);
2688 
2689 	ztest_lr_free(lr, sizeof (*lr), NULL);
2690 
2691 	return (error);
2692 }
2693 
2694 static void
2695 ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
2696 {
2697 	objset_t *os = zd->zd_os;
2698 	dmu_tx_t *tx;
2699 	uint64_t txg;
2700 	rl_t *rl;
2701 
2702 	txg_wait_synced(dmu_objset_pool(os), 0);
2703 
2704 	ztest_object_lock(zd, object, RL_READER);
2705 	rl = ztest_range_lock(zd, object, offset, size, RL_WRITER);
2706 
2707 	tx = dmu_tx_create(os);
2708 
2709 	dmu_tx_hold_write(tx, object, offset, size);
2710 
2711 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2712 
2713 	if (txg != 0) {
2714 		dmu_prealloc(os, object, offset, size, tx);
2715 		dmu_tx_commit(tx);
2716 		txg_wait_synced(dmu_objset_pool(os), txg);
2717 	} else {
2718 		(void) dmu_free_long_range(os, object, offset, size);
2719 	}
2720 
2721 	ztest_range_unlock(rl);
2722 	ztest_object_unlock(zd, object);
2723 }
2724 
2725 static void
2726 ztest_io(ztest_ds_t *zd, uint64_t object, uint64_t offset)
2727 {
2728 	int err;
2729 	ztest_block_tag_t wbt;
2730 	dmu_object_info_t doi;
2731 	enum ztest_io_type io_type;
2732 	uint64_t blocksize;
2733 	void *data;
2734 
2735 	VERIFY0(dmu_object_info(zd->zd_os, object, &doi));
2736 	blocksize = doi.doi_data_block_size;
2737 	data = umem_alloc(blocksize, UMEM_NOFAIL);
2738 
2739 	/*
2740 	 * Pick an i/o type at random, biased toward writing block tags.
2741 	 */
2742 	io_type = ztest_random(ZTEST_IO_TYPES);
2743 	if (ztest_random(2) == 0)
2744 		io_type = ZTEST_IO_WRITE_TAG;
2745 
2746 	(void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);
2747 
2748 	switch (io_type) {
2749 
2750 	case ZTEST_IO_WRITE_TAG:
2751 		ztest_bt_generate(&wbt, zd->zd_os, object, doi.doi_dnodesize,
2752 		    offset, 0, 0, 0);
2753 		(void) ztest_write(zd, object, offset, sizeof (wbt), &wbt);
2754 		break;
2755 
2756 	case ZTEST_IO_WRITE_PATTERN:
2757 		(void) memset(data, 'a' + (object + offset) % 5, blocksize);
2758 		if (ztest_random(2) == 0) {
2759 			/*
2760 			 * Induce fletcher2 collisions to ensure that
2761 			 * zio_ddt_collision() detects and resolves them
2762 			 * when using fletcher2-verify for deduplication.
2763 			 */
2764 			((uint64_t *)data)[0] ^= 1ULL << 63;
2765 			((uint64_t *)data)[4] ^= 1ULL << 63;
2766 		}
2767 		(void) ztest_write(zd, object, offset, blocksize, data);
2768 		break;
2769 
2770 	case ZTEST_IO_WRITE_ZEROES:
2771 		memset(data, 0, blocksize);
2772 		(void) ztest_write(zd, object, offset, blocksize, data);
2773 		break;
2774 
2775 	case ZTEST_IO_TRUNCATE:
2776 		(void) ztest_truncate(zd, object, offset, blocksize);
2777 		break;
2778 
2779 	case ZTEST_IO_SETATTR:
2780 		(void) ztest_setattr(zd, object);
2781 		break;
2782 	default:
2783 		break;
2784 
2785 	case ZTEST_IO_REWRITE:
2786 		(void) pthread_rwlock_rdlock(&ztest_name_lock);
2787 		err = ztest_dsl_prop_set_uint64(zd->zd_name,
2788 		    ZFS_PROP_CHECKSUM, spa_dedup_checksum(ztest_spa),
2789 		    B_FALSE);
2790 		VERIFY(err == 0 || err == ENOSPC);
2791 		err = ztest_dsl_prop_set_uint64(zd->zd_name,
2792 		    ZFS_PROP_COMPRESSION,
2793 		    ztest_random_dsl_prop(ZFS_PROP_COMPRESSION),
2794 		    B_FALSE);
2795 		VERIFY(err == 0 || err == ENOSPC);
2796 		(void) pthread_rwlock_unlock(&ztest_name_lock);
2797 
2798 		VERIFY0(dmu_read(zd->zd_os, object, offset, blocksize, data,
2799 		    DMU_READ_NO_PREFETCH));
2800 
2801 		(void) ztest_write(zd, object, offset, blocksize, data);
2802 		break;
2803 	}
2804 
2805 	(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
2806 
2807 	umem_free(data, blocksize);
2808 }
2809 
2810 /*
2811  * Initialize an object description template.
2812  */
2813 static void
2814 ztest_od_init(ztest_od_t *od, uint64_t id, const char *tag, uint64_t index,
2815     dmu_object_type_t type, uint64_t blocksize, uint64_t dnodesize,
2816     uint64_t gen)
2817 {
2818 	od->od_dir = ZTEST_DIROBJ;
2819 	od->od_object = 0;
2820 
2821 	od->od_crtype = type;
2822 	od->od_crblocksize = blocksize ? blocksize : ztest_random_blocksize();
2823 	od->od_crdnodesize = dnodesize ? dnodesize : ztest_random_dnodesize();
2824 	od->od_crgen = gen;
2825 
2826 	od->od_type = DMU_OT_NONE;
2827 	od->od_blocksize = 0;
2828 	od->od_gen = 0;
2829 
2830 	(void) snprintf(od->od_name, sizeof (od->od_name),
2831 	    "%s(%"PRId64")[%"PRIu64"]",
2832 	    tag, id, index);
2833 }
2834 
2835 /*
2836  * Lookup or create the objects for a test using the od template.
2837  * If the objects do not all exist, or if 'remove' is specified,
2838  * remove any existing objects and create new ones.  Otherwise,
2839  * use the existing objects.
2840  */
2841 static int
2842 ztest_object_init(ztest_ds_t *zd, ztest_od_t *od, size_t size, boolean_t remove)
2843 {
2844 	int count = size / sizeof (*od);
2845 	int rv = 0;
2846 
2847 	mutex_enter(&zd->zd_dirobj_lock);
2848 	if ((ztest_lookup(zd, od, count) != 0 || remove) &&
2849 	    (ztest_remove(zd, od, count) != 0 ||
2850 	    ztest_create(zd, od, count) != 0))
2851 		rv = -1;
2852 	zd->zd_od = od;
2853 	mutex_exit(&zd->zd_dirobj_lock);
2854 
2855 	return (rv);
2856 }
2857 
2858 void
2859 ztest_zil_commit(ztest_ds_t *zd, uint64_t id)
2860 {
2861 	(void) id;
2862 	zilog_t *zilog = zd->zd_zilog;
2863 
2864 	(void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);
2865 
2866 	zil_commit(zilog, ztest_random(ZTEST_OBJECTS));
2867 
2868 	/*
2869 	 * Remember the committed values in zd, which is in parent/child
2870 	 * shared memory.  If we die, the next iteration of ztest_run()
2871 	 * will verify that the log really does contain this record.
2872 	 */
2873 	mutex_enter(&zilog->zl_lock);
2874 	ASSERT3P(zd->zd_shared, !=, NULL);
2875 	ASSERT3U(zd->zd_shared->zd_seq, <=, zilog->zl_commit_lr_seq);
2876 	zd->zd_shared->zd_seq = zilog->zl_commit_lr_seq;
2877 	mutex_exit(&zilog->zl_lock);
2878 
2879 	(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
2880 }
2881 
2882 /*
2883  * This function is designed to simulate the operations that occur during a
2884  * mount/unmount operation.  We hold the dataset across these operations in an
2885  * attempt to expose any implicit assumptions about ZIL management.
2886  */
2887 void
2888 ztest_zil_remount(ztest_ds_t *zd, uint64_t id)
2889 {
2890 	(void) id;
2891 	objset_t *os = zd->zd_os;
2892 
2893 	/*
2894 	 * We hold the ztest_vdev_lock so we don't cause problems with
2895 	 * other threads that wish to remove a log device, such as
2896 	 * ztest_device_removal().
2897 	 */
2898 	mutex_enter(&ztest_vdev_lock);
2899 
2900 	/*
2901 	 * We grab the zd_dirobj_lock to ensure that no other thread is
2902 	 * updating the zil (i.e. adding in-memory log records) and the
2903 	 * zd_zilog_lock to block any I/O.
2904 	 */
2905 	mutex_enter(&zd->zd_dirobj_lock);
2906 	(void) pthread_rwlock_wrlock(&zd->zd_zilog_lock);
2907 
2908 	/* zfsvfs_teardown() */
2909 	zil_close(zd->zd_zilog);
2910 
2911 	/* zfsvfs_setup() */
2912 	VERIFY3P(zil_open(os, ztest_get_data, NULL), ==, zd->zd_zilog);
2913 	zil_replay(os, zd, ztest_replay_vector);
2914 
2915 	(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
2916 	mutex_exit(&zd->zd_dirobj_lock);
2917 	mutex_exit(&ztest_vdev_lock);
2918 }
2919 
2920 /*
2921  * Verify that we can't destroy an active pool, create an existing pool,
2922  * or create a pool with a bad vdev spec.
2923  */
2924 void
2925 ztest_spa_create_destroy(ztest_ds_t *zd, uint64_t id)
2926 {
2927 	(void) zd, (void) id;
2928 	ztest_shared_opts_t *zo = &ztest_opts;
2929 	spa_t *spa;
2930 	nvlist_t *nvroot;
2931 
2932 	if (zo->zo_mmp_test)
2933 		return;
2934 
2935 	/*
2936 	 * Attempt to create using a bad file.
2937 	 */
2938 	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
2939 	VERIFY3U(ENOENT, ==,
2940 	    spa_create("ztest_bad_file", nvroot, NULL, NULL, NULL));
2941 	fnvlist_free(nvroot);
2942 
2943 	/*
2944 	 * Attempt to create using a bad mirror.
2945 	 */
2946 	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 2, 1);
2947 	VERIFY3U(ENOENT, ==,
2948 	    spa_create("ztest_bad_mirror", nvroot, NULL, NULL, NULL));
2949 	fnvlist_free(nvroot);
2950 
2951 	/*
2952 	 * Attempt to create an existing pool.  It shouldn't matter
2953 	 * what's in the nvroot; we should fail with EEXIST.
2954 	 */
2955 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
2956 	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
2957 	VERIFY3U(EEXIST, ==,
2958 	    spa_create(zo->zo_pool, nvroot, NULL, NULL, NULL));
2959 	fnvlist_free(nvroot);
2960 
2961 	/*
2962 	 * We open a reference to the spa and then we try to export it
2963 	 * expecting one of the following errors:
2964 	 *
2965 	 * EBUSY
2966 	 *	Because of the reference we just opened.
2967 	 *
2968 	 * ZFS_ERR_EXPORT_IN_PROGRESS
2969 	 *	For the case that there is another ztest thread doing
2970 	 *	an export concurrently.
2971 	 */
2972 	VERIFY0(spa_open(zo->zo_pool, &spa, FTAG));
2973 	int error = spa_destroy(zo->zo_pool);
2974 	if (error != EBUSY && error != ZFS_ERR_EXPORT_IN_PROGRESS) {
2975 		fatal(B_FALSE, "spa_destroy(%s) returned unexpected value %d",
2976 		    spa->spa_name, error);
2977 	}
2978 	spa_close(spa, FTAG);
2979 
2980 	(void) pthread_rwlock_unlock(&ztest_name_lock);
2981 }
2982 
2983 /*
2984  * Start and then stop the MMP threads to ensure the startup and shutdown code
2985  * works properly.  Actual protection and property-related code tested via ZTS.
2986  */
2987 void
2988 ztest_mmp_enable_disable(ztest_ds_t *zd, uint64_t id)
2989 {
2990 	(void) zd, (void) id;
2991 	ztest_shared_opts_t *zo = &ztest_opts;
2992 	spa_t *spa = ztest_spa;
2993 
2994 	if (zo->zo_mmp_test)
2995 		return;
2996 
2997 	/*
2998 	 * Since enabling MMP involves setting a property, it could not be done
2999 	 * while the pool is suspended.
3000 	 */
3001 	if (spa_suspended(spa))
3002 		return;
3003 
3004 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
3005 	mutex_enter(&spa->spa_props_lock);
3006 
3007 	zfs_multihost_fail_intervals = 0;
3008 
3009 	if (!spa_multihost(spa)) {
3010 		spa->spa_multihost = B_TRUE;
3011 		mmp_thread_start(spa);
3012 	}
3013 
3014 	mutex_exit(&spa->spa_props_lock);
3015 	spa_config_exit(spa, SCL_CONFIG, FTAG);
3016 
3017 	txg_wait_synced(spa_get_dsl(spa), 0);
3018 	mmp_signal_all_threads();
3019 	txg_wait_synced(spa_get_dsl(spa), 0);
3020 
3021 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
3022 	mutex_enter(&spa->spa_props_lock);
3023 
3024 	if (spa_multihost(spa)) {
3025 		mmp_thread_stop(spa);
3026 		spa->spa_multihost = B_FALSE;
3027 	}
3028 
3029 	mutex_exit(&spa->spa_props_lock);
3030 	spa_config_exit(spa, SCL_CONFIG, FTAG);
3031 }
3032 
3033 void
3034 ztest_spa_upgrade(ztest_ds_t *zd, uint64_t id)
3035 {
3036 	(void) zd, (void) id;
3037 	spa_t *spa;
3038 	uint64_t initial_version = SPA_VERSION_INITIAL;
3039 	uint64_t version, newversion;
3040 	nvlist_t *nvroot, *props;
3041 	char *name;
3042 
3043 	if (ztest_opts.zo_mmp_test)
3044 		return;
3045 
3046 	/* dRAID added after feature flags, skip upgrade test. */
3047 	if (strcmp(ztest_opts.zo_raid_type, VDEV_TYPE_DRAID) == 0)
3048 		return;
3049 
3050 	mutex_enter(&ztest_vdev_lock);
3051 	name = kmem_asprintf("%s_upgrade", ztest_opts.zo_pool);
3052 
3053 	/*
3054 	 * Clean up from previous runs.
3055 	 */
3056 	(void) spa_destroy(name);
3057 
3058 	nvroot = make_vdev_root(NULL, NULL, name, ztest_opts.zo_vdev_size, 0,
3059 	    NULL, ztest_opts.zo_raid_children, ztest_opts.zo_mirrors, 1);
3060 
3061 	/*
3062 	 * If we're configuring a RAIDZ device then make sure that the
3063 	 * initial version is capable of supporting that feature.
3064 	 */
3065 	switch (ztest_opts.zo_raid_parity) {
3066 	case 0:
3067 	case 1:
3068 		initial_version = SPA_VERSION_INITIAL;
3069 		break;
3070 	case 2:
3071 		initial_version = SPA_VERSION_RAIDZ2;
3072 		break;
3073 	case 3:
3074 		initial_version = SPA_VERSION_RAIDZ3;
3075 		break;
3076 	}
3077 
3078 	/*
3079 	 * Create a pool with a spa version that can be upgraded. Pick
3080 	 * a value between initial_version and SPA_VERSION_BEFORE_FEATURES.
3081 	 */
3082 	do {
3083 		version = ztest_random_spa_version(initial_version);
3084 	} while (version > SPA_VERSION_BEFORE_FEATURES);
3085 
3086 	props = fnvlist_alloc();
3087 	fnvlist_add_uint64(props,
3088 	    zpool_prop_to_name(ZPOOL_PROP_VERSION), version);
3089 	VERIFY0(spa_create(name, nvroot, props, NULL, NULL));
3090 	fnvlist_free(nvroot);
3091 	fnvlist_free(props);
3092 
3093 	VERIFY0(spa_open(name, &spa, FTAG));
3094 	VERIFY3U(spa_version(spa), ==, version);
3095 	newversion = ztest_random_spa_version(version + 1);
3096 
3097 	if (ztest_opts.zo_verbose >= 4) {
3098 		(void) printf("upgrading spa version from "
3099 		    "%"PRIu64" to %"PRIu64"\n",
3100 		    version, newversion);
3101 	}
3102 
3103 	spa_upgrade(spa, newversion);
3104 	VERIFY3U(spa_version(spa), >, version);
3105 	VERIFY3U(spa_version(spa), ==, fnvlist_lookup_uint64(spa->spa_config,
3106 	    zpool_prop_to_name(ZPOOL_PROP_VERSION)));
3107 	spa_close(spa, FTAG);
3108 
3109 	kmem_strfree(name);
3110 	mutex_exit(&ztest_vdev_lock);
3111 }
3112 
3113 static void
3114 ztest_spa_checkpoint(spa_t *spa)
3115 {
3116 	ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));
3117 
3118 	int error = spa_checkpoint(spa->spa_name);
3119 
3120 	switch (error) {
3121 	case 0:
3122 	case ZFS_ERR_DEVRM_IN_PROGRESS:
3123 	case ZFS_ERR_DISCARDING_CHECKPOINT:
3124 	case ZFS_ERR_CHECKPOINT_EXISTS:
3125 		break;
3126 	case ENOSPC:
3127 		ztest_record_enospc(FTAG);
3128 		break;
3129 	default:
3130 		fatal(B_FALSE, "spa_checkpoint(%s) = %d", spa->spa_name, error);
3131 	}
3132 }
3133 
3134 static void
3135 ztest_spa_discard_checkpoint(spa_t *spa)
3136 {
3137 	ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));
3138 
3139 	int error = spa_checkpoint_discard(spa->spa_name);
3140 
3141 	switch (error) {
3142 	case 0:
3143 	case ZFS_ERR_DISCARDING_CHECKPOINT:
3144 	case ZFS_ERR_NO_CHECKPOINT:
3145 		break;
3146 	default:
3147 		fatal(B_FALSE, "spa_discard_checkpoint(%s) = %d",
3148 		    spa->spa_name, error);
3149 	}
3150 
3151 }
3152 
3153 void
3154 ztest_spa_checkpoint_create_discard(ztest_ds_t *zd, uint64_t id)
3155 {
3156 	(void) zd, (void) id;
3157 	spa_t *spa = ztest_spa;
3158 
3159 	mutex_enter(&ztest_checkpoint_lock);
3160 	if (ztest_random(2) == 0) {
3161 		ztest_spa_checkpoint(spa);
3162 	} else {
3163 		ztest_spa_discard_checkpoint(spa);
3164 	}
3165 	mutex_exit(&ztest_checkpoint_lock);
3166 }
3167 
3168 
3169 static vdev_t *
3170 vdev_lookup_by_path(vdev_t *vd, const char *path)
3171 {
3172 	vdev_t *mvd;
3173 	int c;
3174 
3175 	if (vd->vdev_path != NULL && strcmp(path, vd->vdev_path) == 0)
3176 		return (vd);
3177 
3178 	for (c = 0; c < vd->vdev_children; c++)
3179 		if ((mvd = vdev_lookup_by_path(vd->vdev_child[c], path)) !=
3180 		    NULL)
3181 			return (mvd);
3182 
3183 	return (NULL);
3184 }
3185 
3186 static int
3187 spa_num_top_vdevs(spa_t *spa)
3188 {
3189 	vdev_t *rvd = spa->spa_root_vdev;
3190 	ASSERT3U(spa_config_held(spa, SCL_VDEV, RW_READER), ==, SCL_VDEV);
3191 	return (rvd->vdev_children);
3192 }
3193 
3194 /*
3195  * Verify that vdev_add() works as expected.
3196  */
3197 void
3198 ztest_vdev_add_remove(ztest_ds_t *zd, uint64_t id)
3199 {
3200 	(void) zd, (void) id;
3201 	ztest_shared_t *zs = ztest_shared;
3202 	spa_t *spa = ztest_spa;
3203 	uint64_t leaves;
3204 	uint64_t guid;
3205 	nvlist_t *nvroot;
3206 	int error;
3207 
3208 	if (ztest_opts.zo_mmp_test)
3209 		return;
3210 
3211 	mutex_enter(&ztest_vdev_lock);
3212 	leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) *
3213 	    ztest_opts.zo_raid_children;
3214 
3215 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3216 
3217 	ztest_shared->zs_vdev_next_leaf = spa_num_top_vdevs(spa) * leaves;
3218 
3219 	/*
3220 	 * If we have slogs then remove them 1/4 of the time.
3221 	 */
3222 	if (spa_has_slogs(spa) && ztest_random(4) == 0) {
3223 		metaslab_group_t *mg;
3224 
3225 		/*
3226 		 * find the first real slog in log allocation class
3227 		 */
3228 		mg =  spa_log_class(spa)->mc_allocator[0].mca_rotor;
3229 		while (!mg->mg_vd->vdev_islog)
3230 			mg = mg->mg_next;
3231 
3232 		guid = mg->mg_vd->vdev_guid;
3233 
3234 		spa_config_exit(spa, SCL_VDEV, FTAG);
3235 
3236 		/*
3237 		 * We have to grab the zs_name_lock as writer to
3238 		 * prevent a race between removing a slog (dmu_objset_find)
3239 		 * and destroying a dataset. Removing the slog will
3240 		 * grab a reference on the dataset which may cause
3241 		 * dsl_destroy_head() to fail with EBUSY thus
3242 		 * leaving the dataset in an inconsistent state.
3243 		 */
3244 		pthread_rwlock_wrlock(&ztest_name_lock);
3245 		error = spa_vdev_remove(spa, guid, B_FALSE);
3246 		pthread_rwlock_unlock(&ztest_name_lock);
3247 
3248 		switch (error) {
3249 		case 0:
3250 		case EEXIST:	/* Generic zil_reset() error */
3251 		case EBUSY:	/* Replay required */
3252 		case EACCES:	/* Crypto key not loaded */
3253 		case ZFS_ERR_CHECKPOINT_EXISTS:
3254 		case ZFS_ERR_DISCARDING_CHECKPOINT:
3255 			break;
3256 		default:
3257 			fatal(B_FALSE, "spa_vdev_remove() = %d", error);
3258 		}
3259 	} else {
3260 		spa_config_exit(spa, SCL_VDEV, FTAG);
3261 
3262 		/*
3263 		 * Make 1/4 of the devices be log devices
3264 		 */
3265 		nvroot = make_vdev_root(NULL, NULL, NULL,
3266 		    ztest_opts.zo_vdev_size, 0, (ztest_random(4) == 0) ?
3267 		    "log" : NULL, ztest_opts.zo_raid_children, zs->zs_mirrors,
3268 		    1);
3269 
3270 		error = spa_vdev_add(spa, nvroot);
3271 		fnvlist_free(nvroot);
3272 
3273 		switch (error) {
3274 		case 0:
3275 			break;
3276 		case ENOSPC:
3277 			ztest_record_enospc("spa_vdev_add");
3278 			break;
3279 		default:
3280 			fatal(B_FALSE, "spa_vdev_add() = %d", error);
3281 		}
3282 	}
3283 
3284 	mutex_exit(&ztest_vdev_lock);
3285 }
3286 
3287 void
3288 ztest_vdev_class_add(ztest_ds_t *zd, uint64_t id)
3289 {
3290 	(void) zd, (void) id;
3291 	ztest_shared_t *zs = ztest_shared;
3292 	spa_t *spa = ztest_spa;
3293 	uint64_t leaves;
3294 	nvlist_t *nvroot;
3295 	const char *class = (ztest_random(2) == 0) ?
3296 	    VDEV_ALLOC_BIAS_SPECIAL : VDEV_ALLOC_BIAS_DEDUP;
3297 	int error;
3298 
3299 	/*
3300 	 * By default add a special vdev 50% of the time
3301 	 */
3302 	if ((ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_OFF) ||
3303 	    (ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_RND &&
3304 	    ztest_random(2) == 0)) {
3305 		return;
3306 	}
3307 
3308 	mutex_enter(&ztest_vdev_lock);
3309 
3310 	/* Only test with mirrors */
3311 	if (zs->zs_mirrors < 2) {
3312 		mutex_exit(&ztest_vdev_lock);
3313 		return;
3314 	}
3315 
3316 	/* requires feature@allocation_classes */
3317 	if (!spa_feature_is_enabled(spa, SPA_FEATURE_ALLOCATION_CLASSES)) {
3318 		mutex_exit(&ztest_vdev_lock);
3319 		return;
3320 	}
3321 
3322 	leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) *
3323 	    ztest_opts.zo_raid_children;
3324 
3325 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3326 	ztest_shared->zs_vdev_next_leaf = spa_num_top_vdevs(spa) * leaves;
3327 	spa_config_exit(spa, SCL_VDEV, FTAG);
3328 
3329 	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
3330 	    class, ztest_opts.zo_raid_children, zs->zs_mirrors, 1);
3331 
3332 	error = spa_vdev_add(spa, nvroot);
3333 	fnvlist_free(nvroot);
3334 
3335 	if (error == ENOSPC)
3336 		ztest_record_enospc("spa_vdev_add");
3337 	else if (error != 0)
3338 		fatal(B_FALSE, "spa_vdev_add() = %d", error);
3339 
3340 	/*
3341 	 * 50% of the time allow small blocks in the special class
3342 	 */
3343 	if (error == 0 &&
3344 	    spa_special_class(spa)->mc_groups == 1 && ztest_random(2) == 0) {
3345 		if (ztest_opts.zo_verbose >= 3)
3346 			(void) printf("Enabling special VDEV small blocks\n");
3347 		(void) ztest_dsl_prop_set_uint64(zd->zd_name,
3348 		    ZFS_PROP_SPECIAL_SMALL_BLOCKS, 32768, B_FALSE);
3349 	}
3350 
3351 	mutex_exit(&ztest_vdev_lock);
3352 
3353 	if (ztest_opts.zo_verbose >= 3) {
3354 		metaslab_class_t *mc;
3355 
3356 		if (strcmp(class, VDEV_ALLOC_BIAS_SPECIAL) == 0)
3357 			mc = spa_special_class(spa);
3358 		else
3359 			mc = spa_dedup_class(spa);
3360 		(void) printf("Added a %s mirrored vdev (of %d)\n",
3361 		    class, (int)mc->mc_groups);
3362 	}
3363 }
3364 
3365 /*
3366  * Verify that adding/removing aux devices (l2arc, hot spare) works as expected.
3367  */
3368 void
3369 ztest_vdev_aux_add_remove(ztest_ds_t *zd, uint64_t id)
3370 {
3371 	(void) zd, (void) id;
3372 	ztest_shared_t *zs = ztest_shared;
3373 	spa_t *spa = ztest_spa;
3374 	vdev_t *rvd = spa->spa_root_vdev;
3375 	spa_aux_vdev_t *sav;
3376 	const char *aux;
3377 	char *path;
3378 	uint64_t guid = 0;
3379 	int error, ignore_err = 0;
3380 
3381 	if (ztest_opts.zo_mmp_test)
3382 		return;
3383 
3384 	path = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
3385 
3386 	if (ztest_random(2) == 0) {
3387 		sav = &spa->spa_spares;
3388 		aux = ZPOOL_CONFIG_SPARES;
3389 	} else {
3390 		sav = &spa->spa_l2cache;
3391 		aux = ZPOOL_CONFIG_L2CACHE;
3392 	}
3393 
3394 	mutex_enter(&ztest_vdev_lock);
3395 
3396 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3397 
3398 	if (sav->sav_count != 0 && ztest_random(4) == 0) {
3399 		/*
3400 		 * Pick a random device to remove.
3401 		 */
3402 		vdev_t *svd = sav->sav_vdevs[ztest_random(sav->sav_count)];
3403 
3404 		/* dRAID spares cannot be removed; try anyways to see ENOTSUP */
3405 		if (strstr(svd->vdev_path, VDEV_TYPE_DRAID) != NULL)
3406 			ignore_err = ENOTSUP;
3407 
3408 		guid = svd->vdev_guid;
3409 	} else {
3410 		/*
3411 		 * Find an unused device we can add.
3412 		 */
3413 		zs->zs_vdev_aux = 0;
3414 		for (;;) {
3415 			int c;
3416 			(void) snprintf(path, MAXPATHLEN, ztest_aux_template,
3417 			    ztest_opts.zo_dir, ztest_opts.zo_pool, aux,
3418 			    zs->zs_vdev_aux);
3419 			for (c = 0; c < sav->sav_count; c++)
3420 				if (strcmp(sav->sav_vdevs[c]->vdev_path,
3421 				    path) == 0)
3422 					break;
3423 			if (c == sav->sav_count &&
3424 			    vdev_lookup_by_path(rvd, path) == NULL)
3425 				break;
3426 			zs->zs_vdev_aux++;
3427 		}
3428 	}
3429 
3430 	spa_config_exit(spa, SCL_VDEV, FTAG);
3431 
3432 	if (guid == 0) {
3433 		/*
3434 		 * Add a new device.
3435 		 */
3436 		nvlist_t *nvroot = make_vdev_root(NULL, aux, NULL,
3437 		    (ztest_opts.zo_vdev_size * 5) / 4, 0, NULL, 0, 0, 1);
3438 		error = spa_vdev_add(spa, nvroot);
3439 
3440 		switch (error) {
3441 		case 0:
3442 			break;
3443 		default:
3444 			fatal(B_FALSE, "spa_vdev_add(%p) = %d", nvroot, error);
3445 		}
3446 		fnvlist_free(nvroot);
3447 	} else {
3448 		/*
3449 		 * Remove an existing device.  Sometimes, dirty its
3450 		 * vdev state first to make sure we handle removal
3451 		 * of devices that have pending state changes.
3452 		 */
3453 		if (ztest_random(2) == 0)
3454 			(void) vdev_online(spa, guid, 0, NULL);
3455 
3456 		error = spa_vdev_remove(spa, guid, B_FALSE);
3457 
3458 		switch (error) {
3459 		case 0:
3460 		case EBUSY:
3461 		case ZFS_ERR_CHECKPOINT_EXISTS:
3462 		case ZFS_ERR_DISCARDING_CHECKPOINT:
3463 			break;
3464 		default:
3465 			if (error != ignore_err)
3466 				fatal(B_FALSE,
3467 				    "spa_vdev_remove(%"PRIu64") = %d",
3468 				    guid, error);
3469 		}
3470 	}
3471 
3472 	mutex_exit(&ztest_vdev_lock);
3473 
3474 	umem_free(path, MAXPATHLEN);
3475 }
3476 
3477 /*
3478  * split a pool if it has mirror tlvdevs
3479  */
3480 void
3481 ztest_split_pool(ztest_ds_t *zd, uint64_t id)
3482 {
3483 	(void) zd, (void) id;
3484 	ztest_shared_t *zs = ztest_shared;
3485 	spa_t *spa = ztest_spa;
3486 	vdev_t *rvd = spa->spa_root_vdev;
3487 	nvlist_t *tree, **child, *config, *split, **schild;
3488 	uint_t c, children, schildren = 0, lastlogid = 0;
3489 	int error = 0;
3490 
3491 	if (ztest_opts.zo_mmp_test)
3492 		return;
3493 
3494 	mutex_enter(&ztest_vdev_lock);
3495 
3496 	/* ensure we have a usable config; mirrors of raidz aren't supported */
3497 	if (zs->zs_mirrors < 3 || ztest_opts.zo_raid_children > 1) {
3498 		mutex_exit(&ztest_vdev_lock);
3499 		return;
3500 	}
3501 
3502 	/* clean up the old pool, if any */
3503 	(void) spa_destroy("splitp");
3504 
3505 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3506 
3507 	/* generate a config from the existing config */
3508 	mutex_enter(&spa->spa_props_lock);
3509 	tree = fnvlist_lookup_nvlist(spa->spa_config, ZPOOL_CONFIG_VDEV_TREE);
3510 	mutex_exit(&spa->spa_props_lock);
3511 
3512 	VERIFY0(nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN,
3513 	    &child, &children));
3514 
3515 	schild = malloc(rvd->vdev_children * sizeof (nvlist_t *));
3516 	for (c = 0; c < children; c++) {
3517 		vdev_t *tvd = rvd->vdev_child[c];
3518 		nvlist_t **mchild;
3519 		uint_t mchildren;
3520 
3521 		if (tvd->vdev_islog || tvd->vdev_ops == &vdev_hole_ops) {
3522 			schild[schildren] = fnvlist_alloc();
3523 			fnvlist_add_string(schild[schildren],
3524 			    ZPOOL_CONFIG_TYPE, VDEV_TYPE_HOLE);
3525 			fnvlist_add_uint64(schild[schildren],
3526 			    ZPOOL_CONFIG_IS_HOLE, 1);
3527 			if (lastlogid == 0)
3528 				lastlogid = schildren;
3529 			++schildren;
3530 			continue;
3531 		}
3532 		lastlogid = 0;
3533 		VERIFY0(nvlist_lookup_nvlist_array(child[c],
3534 		    ZPOOL_CONFIG_CHILDREN, &mchild, &mchildren));
3535 		schild[schildren++] = fnvlist_dup(mchild[0]);
3536 	}
3537 
3538 	/* OK, create a config that can be used to split */
3539 	split = fnvlist_alloc();
3540 	fnvlist_add_string(split, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT);
3541 	fnvlist_add_nvlist_array(split, ZPOOL_CONFIG_CHILDREN,
3542 	    (const nvlist_t **)schild, lastlogid != 0 ? lastlogid : schildren);
3543 
3544 	config = fnvlist_alloc();
3545 	fnvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, split);
3546 
3547 	for (c = 0; c < schildren; c++)
3548 		fnvlist_free(schild[c]);
3549 	free(schild);
3550 	fnvlist_free(split);
3551 
3552 	spa_config_exit(spa, SCL_VDEV, FTAG);
3553 
3554 	(void) pthread_rwlock_wrlock(&ztest_name_lock);
3555 	error = spa_vdev_split_mirror(spa, "splitp", config, NULL, B_FALSE);
3556 	(void) pthread_rwlock_unlock(&ztest_name_lock);
3557 
3558 	fnvlist_free(config);
3559 
3560 	if (error == 0) {
3561 		(void) printf("successful split - results:\n");
3562 		mutex_enter(&spa_namespace_lock);
3563 		show_pool_stats(spa);
3564 		show_pool_stats(spa_lookup("splitp"));
3565 		mutex_exit(&spa_namespace_lock);
3566 		++zs->zs_splits;
3567 		--zs->zs_mirrors;
3568 	}
3569 	mutex_exit(&ztest_vdev_lock);
3570 }
3571 
3572 /*
3573  * Verify that we can attach and detach devices.
3574  */
3575 void
3576 ztest_vdev_attach_detach(ztest_ds_t *zd, uint64_t id)
3577 {
3578 	(void) zd, (void) id;
3579 	ztest_shared_t *zs = ztest_shared;
3580 	spa_t *spa = ztest_spa;
3581 	spa_aux_vdev_t *sav = &spa->spa_spares;
3582 	vdev_t *rvd = spa->spa_root_vdev;
3583 	vdev_t *oldvd, *newvd, *pvd;
3584 	nvlist_t *root;
3585 	uint64_t leaves;
3586 	uint64_t leaf, top;
3587 	uint64_t ashift = ztest_get_ashift();
3588 	uint64_t oldguid, pguid;
3589 	uint64_t oldsize, newsize;
3590 	char *oldpath, *newpath;
3591 	int replacing;
3592 	int oldvd_has_siblings = B_FALSE;
3593 	int newvd_is_spare = B_FALSE;
3594 	int newvd_is_dspare = B_FALSE;
3595 	int oldvd_is_log;
3596 	int error, expected_error;
3597 
3598 	if (ztest_opts.zo_mmp_test)
3599 		return;
3600 
3601 	oldpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
3602 	newpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
3603 
3604 	mutex_enter(&ztest_vdev_lock);
3605 	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raid_children;
3606 
3607 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3608 
3609 	/*
3610 	 * If a vdev is in the process of being removed, its removal may
3611 	 * finish while we are in progress, leading to an unexpected error
3612 	 * value.  Don't bother trying to attach while we are in the middle
3613 	 * of removal.
3614 	 */
3615 	if (ztest_device_removal_active) {
3616 		spa_config_exit(spa, SCL_ALL, FTAG);
3617 		goto out;
3618 	}
3619 
3620 	/*
3621 	 * Decide whether to do an attach or a replace.
3622 	 */
3623 	replacing = ztest_random(2);
3624 
3625 	/*
3626 	 * Pick a random top-level vdev.
3627 	 */
3628 	top = ztest_random_vdev_top(spa, B_TRUE);
3629 
3630 	/*
3631 	 * Pick a random leaf within it.
3632 	 */
3633 	leaf = ztest_random(leaves);
3634 
3635 	/*
3636 	 * Locate this vdev.
3637 	 */
3638 	oldvd = rvd->vdev_child[top];
3639 
3640 	/* pick a child from the mirror */
3641 	if (zs->zs_mirrors >= 1) {
3642 		ASSERT3P(oldvd->vdev_ops, ==, &vdev_mirror_ops);
3643 		ASSERT3U(oldvd->vdev_children, >=, zs->zs_mirrors);
3644 		oldvd = oldvd->vdev_child[leaf / ztest_opts.zo_raid_children];
3645 	}
3646 
3647 	/* pick a child out of the raidz group */
3648 	if (ztest_opts.zo_raid_children > 1) {
3649 		if (strcmp(oldvd->vdev_ops->vdev_op_type, "raidz") == 0)
3650 			ASSERT3P(oldvd->vdev_ops, ==, &vdev_raidz_ops);
3651 		else
3652 			ASSERT3P(oldvd->vdev_ops, ==, &vdev_draid_ops);
3653 		ASSERT3U(oldvd->vdev_children, ==, ztest_opts.zo_raid_children);
3654 		oldvd = oldvd->vdev_child[leaf % ztest_opts.zo_raid_children];
3655 	}
3656 
3657 	/*
3658 	 * If we're already doing an attach or replace, oldvd may be a
3659 	 * mirror vdev -- in which case, pick a random child.
3660 	 */
3661 	while (oldvd->vdev_children != 0) {
3662 		oldvd_has_siblings = B_TRUE;
3663 		ASSERT3U(oldvd->vdev_children, >=, 2);
3664 		oldvd = oldvd->vdev_child[ztest_random(oldvd->vdev_children)];
3665 	}
3666 
3667 	oldguid = oldvd->vdev_guid;
3668 	oldsize = vdev_get_min_asize(oldvd);
3669 	oldvd_is_log = oldvd->vdev_top->vdev_islog;
3670 	(void) strlcpy(oldpath, oldvd->vdev_path, MAXPATHLEN);
3671 	pvd = oldvd->vdev_parent;
3672 	pguid = pvd->vdev_guid;
3673 
3674 	/*
3675 	 * If oldvd has siblings, then half of the time, detach it.  Prior
3676 	 * to the detach the pool is scrubbed in order to prevent creating
3677 	 * unrepairable blocks as a result of the data corruption injection.
3678 	 */
3679 	if (oldvd_has_siblings && ztest_random(2) == 0) {
3680 		spa_config_exit(spa, SCL_ALL, FTAG);
3681 
3682 		error = ztest_scrub_impl(spa);
3683 		if (error)
3684 			goto out;
3685 
3686 		error = spa_vdev_detach(spa, oldguid, pguid, B_FALSE);
3687 		if (error != 0 && error != ENODEV && error != EBUSY &&
3688 		    error != ENOTSUP && error != ZFS_ERR_CHECKPOINT_EXISTS &&
3689 		    error != ZFS_ERR_DISCARDING_CHECKPOINT)
3690 			fatal(B_FALSE, "detach (%s) returned %d",
3691 			    oldpath, error);
3692 		goto out;
3693 	}
3694 
3695 	/*
3696 	 * For the new vdev, choose with equal probability between the two
3697 	 * standard paths (ending in either 'a' or 'b') or a random hot spare.
3698 	 */
3699 	if (sav->sav_count != 0 && ztest_random(3) == 0) {
3700 		newvd = sav->sav_vdevs[ztest_random(sav->sav_count)];
3701 		newvd_is_spare = B_TRUE;
3702 
3703 		if (newvd->vdev_ops == &vdev_draid_spare_ops)
3704 			newvd_is_dspare = B_TRUE;
3705 
3706 		(void) strlcpy(newpath, newvd->vdev_path, MAXPATHLEN);
3707 	} else {
3708 		(void) snprintf(newpath, MAXPATHLEN, ztest_dev_template,
3709 		    ztest_opts.zo_dir, ztest_opts.zo_pool,
3710 		    top * leaves + leaf);
3711 		if (ztest_random(2) == 0)
3712 			newpath[strlen(newpath) - 1] = 'b';
3713 		newvd = vdev_lookup_by_path(rvd, newpath);
3714 	}
3715 
3716 	if (newvd) {
3717 		/*
3718 		 * Reopen to ensure the vdev's asize field isn't stale.
3719 		 */
3720 		vdev_reopen(newvd);
3721 		newsize = vdev_get_min_asize(newvd);
3722 	} else {
3723 		/*
3724 		 * Make newsize a little bigger or smaller than oldsize.
3725 		 * If it's smaller, the attach should fail.
3726 		 * If it's larger, and we're doing a replace,
3727 		 * we should get dynamic LUN growth when we're done.
3728 		 */
3729 		newsize = 10 * oldsize / (9 + ztest_random(3));
3730 	}
3731 
3732 	/*
3733 	 * If pvd is not a mirror or root, the attach should fail with ENOTSUP,
3734 	 * unless it's a replace; in that case any non-replacing parent is OK.
3735 	 *
3736 	 * If newvd is already part of the pool, it should fail with EBUSY.
3737 	 *
3738 	 * If newvd is too small, it should fail with EOVERFLOW.
3739 	 *
3740 	 * If newvd is a distributed spare and it's being attached to a
3741 	 * dRAID which is not its parent it should fail with EINVAL.
3742 	 */
3743 	if (pvd->vdev_ops != &vdev_mirror_ops &&
3744 	    pvd->vdev_ops != &vdev_root_ops && (!replacing ||
3745 	    pvd->vdev_ops == &vdev_replacing_ops ||
3746 	    pvd->vdev_ops == &vdev_spare_ops))
3747 		expected_error = ENOTSUP;
3748 	else if (newvd_is_spare && (!replacing || oldvd_is_log))
3749 		expected_error = ENOTSUP;
3750 	else if (newvd == oldvd)
3751 		expected_error = replacing ? 0 : EBUSY;
3752 	else if (vdev_lookup_by_path(rvd, newpath) != NULL)
3753 		expected_error = EBUSY;
3754 	else if (!newvd_is_dspare && newsize < oldsize)
3755 		expected_error = EOVERFLOW;
3756 	else if (ashift > oldvd->vdev_top->vdev_ashift)
3757 		expected_error = EDOM;
3758 	else if (newvd_is_dspare && pvd != vdev_draid_spare_get_parent(newvd))
3759 		expected_error = ENOTSUP;
3760 	else
3761 		expected_error = 0;
3762 
3763 	spa_config_exit(spa, SCL_ALL, FTAG);
3764 
3765 	/*
3766 	 * Build the nvlist describing newpath.
3767 	 */
3768 	root = make_vdev_root(newpath, NULL, NULL, newvd == NULL ? newsize : 0,
3769 	    ashift, NULL, 0, 0, 1);
3770 
3771 	/*
3772 	 * When supported select either a healing or sequential resilver.
3773 	 */
3774 	boolean_t rebuilding = B_FALSE;
3775 	if (pvd->vdev_ops == &vdev_mirror_ops ||
3776 	    pvd->vdev_ops ==  &vdev_root_ops) {
3777 		rebuilding = !!ztest_random(2);
3778 	}
3779 
3780 	error = spa_vdev_attach(spa, oldguid, root, replacing, rebuilding);
3781 
3782 	fnvlist_free(root);
3783 
3784 	/*
3785 	 * If our parent was the replacing vdev, but the replace completed,
3786 	 * then instead of failing with ENOTSUP we may either succeed,
3787 	 * fail with ENODEV, or fail with EOVERFLOW.
3788 	 */
3789 	if (expected_error == ENOTSUP &&
3790 	    (error == 0 || error == ENODEV || error == EOVERFLOW))
3791 		expected_error = error;
3792 
3793 	/*
3794 	 * If someone grew the LUN, the replacement may be too small.
3795 	 */
3796 	if (error == EOVERFLOW || error == EBUSY)
3797 		expected_error = error;
3798 
3799 	if (error == ZFS_ERR_CHECKPOINT_EXISTS ||
3800 	    error == ZFS_ERR_DISCARDING_CHECKPOINT ||
3801 	    error == ZFS_ERR_RESILVER_IN_PROGRESS ||
3802 	    error == ZFS_ERR_REBUILD_IN_PROGRESS)
3803 		expected_error = error;
3804 
3805 	if (error != expected_error && expected_error != EBUSY) {
3806 		fatal(B_FALSE, "attach (%s %"PRIu64", %s %"PRIu64", %d) "
3807 		    "returned %d, expected %d",
3808 		    oldpath, oldsize, newpath,
3809 		    newsize, replacing, error, expected_error);
3810 	}
3811 out:
3812 	mutex_exit(&ztest_vdev_lock);
3813 
3814 	umem_free(oldpath, MAXPATHLEN);
3815 	umem_free(newpath, MAXPATHLEN);
3816 }
3817 
3818 void
3819 ztest_device_removal(ztest_ds_t *zd, uint64_t id)
3820 {
3821 	(void) zd, (void) id;
3822 	spa_t *spa = ztest_spa;
3823 	vdev_t *vd;
3824 	uint64_t guid;
3825 	int error;
3826 
3827 	mutex_enter(&ztest_vdev_lock);
3828 
3829 	if (ztest_device_removal_active) {
3830 		mutex_exit(&ztest_vdev_lock);
3831 		return;
3832 	}
3833 
3834 	/*
3835 	 * Remove a random top-level vdev and wait for removal to finish.
3836 	 */
3837 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3838 	vd = vdev_lookup_top(spa, ztest_random_vdev_top(spa, B_FALSE));
3839 	guid = vd->vdev_guid;
3840 	spa_config_exit(spa, SCL_VDEV, FTAG);
3841 
3842 	error = spa_vdev_remove(spa, guid, B_FALSE);
3843 	if (error == 0) {
3844 		ztest_device_removal_active = B_TRUE;
3845 		mutex_exit(&ztest_vdev_lock);
3846 
3847 		/*
3848 		 * spa->spa_vdev_removal is created in a sync task that
3849 		 * is initiated via dsl_sync_task_nowait(). Since the
3850 		 * task may not run before spa_vdev_remove() returns, we
3851 		 * must wait at least 1 txg to ensure that the removal
3852 		 * struct has been created.
3853 		 */
3854 		txg_wait_synced(spa_get_dsl(spa), 0);
3855 
3856 		while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
3857 			txg_wait_synced(spa_get_dsl(spa), 0);
3858 	} else {
3859 		mutex_exit(&ztest_vdev_lock);
3860 		return;
3861 	}
3862 
3863 	/*
3864 	 * The pool needs to be scrubbed after completing device removal.
3865 	 * Failure to do so may result in checksum errors due to the
3866 	 * strategy employed by ztest_fault_inject() when selecting which
3867 	 * offset are redundant and can be damaged.
3868 	 */
3869 	error = spa_scan(spa, POOL_SCAN_SCRUB);
3870 	if (error == 0) {
3871 		while (dsl_scan_scrubbing(spa_get_dsl(spa)))
3872 			txg_wait_synced(spa_get_dsl(spa), 0);
3873 	}
3874 
3875 	mutex_enter(&ztest_vdev_lock);
3876 	ztest_device_removal_active = B_FALSE;
3877 	mutex_exit(&ztest_vdev_lock);
3878 }
3879 
3880 /*
3881  * Callback function which expands the physical size of the vdev.
3882  */
3883 static vdev_t *
3884 grow_vdev(vdev_t *vd, void *arg)
3885 {
3886 	spa_t *spa __maybe_unused = vd->vdev_spa;
3887 	size_t *newsize = arg;
3888 	size_t fsize;
3889 	int fd;
3890 
3891 	ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), ==, SCL_STATE);
3892 	ASSERT(vd->vdev_ops->vdev_op_leaf);
3893 
3894 	if ((fd = open(vd->vdev_path, O_RDWR)) == -1)
3895 		return (vd);
3896 
3897 	fsize = lseek(fd, 0, SEEK_END);
3898 	VERIFY0(ftruncate(fd, *newsize));
3899 
3900 	if (ztest_opts.zo_verbose >= 6) {
3901 		(void) printf("%s grew from %lu to %lu bytes\n",
3902 		    vd->vdev_path, (ulong_t)fsize, (ulong_t)*newsize);
3903 	}
3904 	(void) close(fd);
3905 	return (NULL);
3906 }
3907 
3908 /*
3909  * Callback function which expands a given vdev by calling vdev_online().
3910  */
3911 static vdev_t *
3912 online_vdev(vdev_t *vd, void *arg)
3913 {
3914 	(void) arg;
3915 	spa_t *spa = vd->vdev_spa;
3916 	vdev_t *tvd = vd->vdev_top;
3917 	uint64_t guid = vd->vdev_guid;
3918 	uint64_t generation = spa->spa_config_generation + 1;
3919 	vdev_state_t newstate = VDEV_STATE_UNKNOWN;
3920 	int error;
3921 
3922 	ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), ==, SCL_STATE);
3923 	ASSERT(vd->vdev_ops->vdev_op_leaf);
3924 
3925 	/* Calling vdev_online will initialize the new metaslabs */
3926 	spa_config_exit(spa, SCL_STATE, spa);
3927 	error = vdev_online(spa, guid, ZFS_ONLINE_EXPAND, &newstate);
3928 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
3929 
3930 	/*
3931 	 * If vdev_online returned an error or the underlying vdev_open
3932 	 * failed then we abort the expand. The only way to know that
3933 	 * vdev_open fails is by checking the returned newstate.
3934 	 */
3935 	if (error || newstate != VDEV_STATE_HEALTHY) {
3936 		if (ztest_opts.zo_verbose >= 5) {
3937 			(void) printf("Unable to expand vdev, state %u, "
3938 			    "error %d\n", newstate, error);
3939 		}
3940 		return (vd);
3941 	}
3942 	ASSERT3U(newstate, ==, VDEV_STATE_HEALTHY);
3943 
3944 	/*
3945 	 * Since we dropped the lock we need to ensure that we're
3946 	 * still talking to the original vdev. It's possible this
3947 	 * vdev may have been detached/replaced while we were
3948 	 * trying to online it.
3949 	 */
3950 	if (generation != spa->spa_config_generation) {
3951 		if (ztest_opts.zo_verbose >= 5) {
3952 			(void) printf("vdev configuration has changed, "
3953 			    "guid %"PRIu64", state %"PRIu64", "
3954 			    "expected gen %"PRIu64", got gen %"PRIu64"\n",
3955 			    guid,
3956 			    tvd->vdev_state,
3957 			    generation,
3958 			    spa->spa_config_generation);
3959 		}
3960 		return (vd);
3961 	}
3962 	return (NULL);
3963 }
3964 
3965 /*
3966  * Traverse the vdev tree calling the supplied function.
3967  * We continue to walk the tree until we either have walked all
3968  * children or we receive a non-NULL return from the callback.
3969  * If a NULL callback is passed, then we just return back the first
3970  * leaf vdev we encounter.
3971  */
3972 static vdev_t *
3973 vdev_walk_tree(vdev_t *vd, vdev_t *(*func)(vdev_t *, void *), void *arg)
3974 {
3975 	uint_t c;
3976 
3977 	if (vd->vdev_ops->vdev_op_leaf) {
3978 		if (func == NULL)
3979 			return (vd);
3980 		else
3981 			return (func(vd, arg));
3982 	}
3983 
3984 	for (c = 0; c < vd->vdev_children; c++) {
3985 		vdev_t *cvd = vd->vdev_child[c];
3986 		if ((cvd = vdev_walk_tree(cvd, func, arg)) != NULL)
3987 			return (cvd);
3988 	}
3989 	return (NULL);
3990 }
3991 
3992 /*
3993  * Verify that dynamic LUN growth works as expected.
3994  */
3995 void
3996 ztest_vdev_LUN_growth(ztest_ds_t *zd, uint64_t id)
3997 {
3998 	(void) zd, (void) id;
3999 	spa_t *spa = ztest_spa;
4000 	vdev_t *vd, *tvd;
4001 	metaslab_class_t *mc;
4002 	metaslab_group_t *mg;
4003 	size_t psize, newsize;
4004 	uint64_t top;
4005 	uint64_t old_class_space, new_class_space, old_ms_count, new_ms_count;
4006 
4007 	mutex_enter(&ztest_checkpoint_lock);
4008 	mutex_enter(&ztest_vdev_lock);
4009 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
4010 
4011 	/*
4012 	 * If there is a vdev removal in progress, it could complete while
4013 	 * we are running, in which case we would not be able to verify
4014 	 * that the metaslab_class space increased (because it decreases
4015 	 * when the device removal completes).
4016 	 */
4017 	if (ztest_device_removal_active) {
4018 		spa_config_exit(spa, SCL_STATE, spa);
4019 		mutex_exit(&ztest_vdev_lock);
4020 		mutex_exit(&ztest_checkpoint_lock);
4021 		return;
4022 	}
4023 
4024 	top = ztest_random_vdev_top(spa, B_TRUE);
4025 
4026 	tvd = spa->spa_root_vdev->vdev_child[top];
4027 	mg = tvd->vdev_mg;
4028 	mc = mg->mg_class;
4029 	old_ms_count = tvd->vdev_ms_count;
4030 	old_class_space = metaslab_class_get_space(mc);
4031 
4032 	/*
4033 	 * Determine the size of the first leaf vdev associated with
4034 	 * our top-level device.
4035 	 */
4036 	vd = vdev_walk_tree(tvd, NULL, NULL);
4037 	ASSERT3P(vd, !=, NULL);
4038 	ASSERT(vd->vdev_ops->vdev_op_leaf);
4039 
4040 	psize = vd->vdev_psize;
4041 
4042 	/*
4043 	 * We only try to expand the vdev if it's healthy, less than 4x its
4044 	 * original size, and it has a valid psize.
4045 	 */
4046 	if (tvd->vdev_state != VDEV_STATE_HEALTHY ||
4047 	    psize == 0 || psize >= 4 * ztest_opts.zo_vdev_size) {
4048 		spa_config_exit(spa, SCL_STATE, spa);
4049 		mutex_exit(&ztest_vdev_lock);
4050 		mutex_exit(&ztest_checkpoint_lock);
4051 		return;
4052 	}
4053 	ASSERT3U(psize, >, 0);
4054 	newsize = psize + MAX(psize / 8, SPA_MAXBLOCKSIZE);
4055 	ASSERT3U(newsize, >, psize);
4056 
4057 	if (ztest_opts.zo_verbose >= 6) {
4058 		(void) printf("Expanding LUN %s from %lu to %lu\n",
4059 		    vd->vdev_path, (ulong_t)psize, (ulong_t)newsize);
4060 	}
4061 
4062 	/*
4063 	 * Growing the vdev is a two step process:
4064 	 *	1). expand the physical size (i.e. relabel)
4065 	 *	2). online the vdev to create the new metaslabs
4066 	 */
4067 	if (vdev_walk_tree(tvd, grow_vdev, &newsize) != NULL ||
4068 	    vdev_walk_tree(tvd, online_vdev, NULL) != NULL ||
4069 	    tvd->vdev_state != VDEV_STATE_HEALTHY) {
4070 		if (ztest_opts.zo_verbose >= 5) {
4071 			(void) printf("Could not expand LUN because "
4072 			    "the vdev configuration changed.\n");
4073 		}
4074 		spa_config_exit(spa, SCL_STATE, spa);
4075 		mutex_exit(&ztest_vdev_lock);
4076 		mutex_exit(&ztest_checkpoint_lock);
4077 		return;
4078 	}
4079 
4080 	spa_config_exit(spa, SCL_STATE, spa);
4081 
4082 	/*
4083 	 * Expanding the LUN will update the config asynchronously,
4084 	 * thus we must wait for the async thread to complete any
4085 	 * pending tasks before proceeding.
4086 	 */
4087 	for (;;) {
4088 		boolean_t done;
4089 		mutex_enter(&spa->spa_async_lock);
4090 		done = (spa->spa_async_thread == NULL && !spa->spa_async_tasks);
4091 		mutex_exit(&spa->spa_async_lock);
4092 		if (done)
4093 			break;
4094 		txg_wait_synced(spa_get_dsl(spa), 0);
4095 		(void) poll(NULL, 0, 100);
4096 	}
4097 
4098 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
4099 
4100 	tvd = spa->spa_root_vdev->vdev_child[top];
4101 	new_ms_count = tvd->vdev_ms_count;
4102 	new_class_space = metaslab_class_get_space(mc);
4103 
4104 	if (tvd->vdev_mg != mg || mg->mg_class != mc) {
4105 		if (ztest_opts.zo_verbose >= 5) {
4106 			(void) printf("Could not verify LUN expansion due to "
4107 			    "intervening vdev offline or remove.\n");
4108 		}
4109 		spa_config_exit(spa, SCL_STATE, spa);
4110 		mutex_exit(&ztest_vdev_lock);
4111 		mutex_exit(&ztest_checkpoint_lock);
4112 		return;
4113 	}
4114 
4115 	/*
4116 	 * Make sure we were able to grow the vdev.
4117 	 */
4118 	if (new_ms_count <= old_ms_count) {
4119 		fatal(B_FALSE,
4120 		    "LUN expansion failed: ms_count %"PRIu64" < %"PRIu64"\n",
4121 		    old_ms_count, new_ms_count);
4122 	}
4123 
4124 	/*
4125 	 * Make sure we were able to grow the pool.
4126 	 */
4127 	if (new_class_space <= old_class_space) {
4128 		fatal(B_FALSE,
4129 		    "LUN expansion failed: class_space %"PRIu64" < %"PRIu64"\n",
4130 		    old_class_space, new_class_space);
4131 	}
4132 
4133 	if (ztest_opts.zo_verbose >= 5) {
4134 		char oldnumbuf[NN_NUMBUF_SZ], newnumbuf[NN_NUMBUF_SZ];
4135 
4136 		nicenum(old_class_space, oldnumbuf, sizeof (oldnumbuf));
4137 		nicenum(new_class_space, newnumbuf, sizeof (newnumbuf));
4138 		(void) printf("%s grew from %s to %s\n",
4139 		    spa->spa_name, oldnumbuf, newnumbuf);
4140 	}
4141 
4142 	spa_config_exit(spa, SCL_STATE, spa);
4143 	mutex_exit(&ztest_vdev_lock);
4144 	mutex_exit(&ztest_checkpoint_lock);
4145 }
4146 
4147 /*
4148  * Verify that dmu_objset_{create,destroy,open,close} work as expected.
4149  */
4150 static void
4151 ztest_objset_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
4152 {
4153 	(void) arg, (void) cr;
4154 
4155 	/*
4156 	 * Create the objects common to all ztest datasets.
4157 	 */
4158 	VERIFY0(zap_create_claim(os, ZTEST_DIROBJ,
4159 	    DMU_OT_ZAP_OTHER, DMU_OT_NONE, 0, tx));
4160 }
4161 
4162 static int
4163 ztest_dataset_create(char *dsname)
4164 {
4165 	int err;
4166 	uint64_t rand;
4167 	dsl_crypto_params_t *dcp = NULL;
4168 
4169 	/*
4170 	 * 50% of the time, we create encrypted datasets
4171 	 * using a random cipher suite and a hard-coded
4172 	 * wrapping key.
4173 	 */
4174 	rand = ztest_random(2);
4175 	if (rand != 0) {
4176 		nvlist_t *crypto_args = fnvlist_alloc();
4177 		nvlist_t *props = fnvlist_alloc();
4178 
4179 		/* slight bias towards the default cipher suite */
4180 		rand = ztest_random(ZIO_CRYPT_FUNCTIONS);
4181 		if (rand < ZIO_CRYPT_AES_128_CCM)
4182 			rand = ZIO_CRYPT_ON;
4183 
4184 		fnvlist_add_uint64(props,
4185 		    zfs_prop_to_name(ZFS_PROP_ENCRYPTION), rand);
4186 		fnvlist_add_uint8_array(crypto_args, "wkeydata",
4187 		    (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
4188 
4189 		/*
4190 		 * These parameters aren't really used by the kernel. They
4191 		 * are simply stored so that userspace knows how to load
4192 		 * the wrapping key.
4193 		 */
4194 		fnvlist_add_uint64(props,
4195 		    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_RAW);
4196 		fnvlist_add_string(props,
4197 		    zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt");
4198 		fnvlist_add_uint64(props,
4199 		    zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 0ULL);
4200 		fnvlist_add_uint64(props,
4201 		    zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 0ULL);
4202 
4203 		VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, props,
4204 		    crypto_args, &dcp));
4205 
4206 		/*
4207 		 * Cycle through all available encryption implementations
4208 		 * to verify interoperability.
4209 		 */
4210 		VERIFY0(gcm_impl_set("cycle"));
4211 		VERIFY0(aes_impl_set("cycle"));
4212 
4213 		fnvlist_free(crypto_args);
4214 		fnvlist_free(props);
4215 	}
4216 
4217 	err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, dcp,
4218 	    ztest_objset_create_cb, NULL);
4219 	dsl_crypto_params_free(dcp, !!err);
4220 
4221 	rand = ztest_random(100);
4222 	if (err || rand < 80)
4223 		return (err);
4224 
4225 	if (ztest_opts.zo_verbose >= 5)
4226 		(void) printf("Setting dataset %s to sync always\n", dsname);
4227 	return (ztest_dsl_prop_set_uint64(dsname, ZFS_PROP_SYNC,
4228 	    ZFS_SYNC_ALWAYS, B_FALSE));
4229 }
4230 
4231 static int
4232 ztest_objset_destroy_cb(const char *name, void *arg)
4233 {
4234 	(void) arg;
4235 	objset_t *os;
4236 	dmu_object_info_t doi;
4237 	int error;
4238 
4239 	/*
4240 	 * Verify that the dataset contains a directory object.
4241 	 */
4242 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
4243 	    B_TRUE, FTAG, &os));
4244 	error = dmu_object_info(os, ZTEST_DIROBJ, &doi);
4245 	if (error != ENOENT) {
4246 		/* We could have crashed in the middle of destroying it */
4247 		ASSERT0(error);
4248 		ASSERT3U(doi.doi_type, ==, DMU_OT_ZAP_OTHER);
4249 		ASSERT3S(doi.doi_physical_blocks_512, >=, 0);
4250 	}
4251 	dmu_objset_disown(os, B_TRUE, FTAG);
4252 
4253 	/*
4254 	 * Destroy the dataset.
4255 	 */
4256 	if (strchr(name, '@') != NULL) {
4257 		error = dsl_destroy_snapshot(name, B_TRUE);
4258 		if (error != ECHRNG) {
4259 			/*
4260 			 * The program was executed, but encountered a runtime
4261 			 * error, such as insufficient slop, or a hold on the
4262 			 * dataset.
4263 			 */
4264 			ASSERT0(error);
4265 		}
4266 	} else {
4267 		error = dsl_destroy_head(name);
4268 		if (error == ENOSPC) {
4269 			/* There could be checkpoint or insufficient slop */
4270 			ztest_record_enospc(FTAG);
4271 		} else if (error != EBUSY) {
4272 			/* There could be a hold on this dataset */
4273 			ASSERT0(error);
4274 		}
4275 	}
4276 	return (0);
4277 }
4278 
4279 static boolean_t
4280 ztest_snapshot_create(char *osname, uint64_t id)
4281 {
4282 	char snapname[ZFS_MAX_DATASET_NAME_LEN];
4283 	int error;
4284 
4285 	(void) snprintf(snapname, sizeof (snapname), "%"PRIu64"", id);
4286 
4287 	error = dmu_objset_snapshot_one(osname, snapname);
4288 	if (error == ENOSPC) {
4289 		ztest_record_enospc(FTAG);
4290 		return (B_FALSE);
4291 	}
4292 	if (error != 0 && error != EEXIST) {
4293 		fatal(B_FALSE, "ztest_snapshot_create(%s@%s) = %d", osname,
4294 		    snapname, error);
4295 	}
4296 	return (B_TRUE);
4297 }
4298 
4299 static boolean_t
4300 ztest_snapshot_destroy(char *osname, uint64_t id)
4301 {
4302 	char snapname[ZFS_MAX_DATASET_NAME_LEN];
4303 	int error;
4304 
4305 	(void) snprintf(snapname, sizeof (snapname), "%s@%"PRIu64"",
4306 	    osname, id);
4307 
4308 	error = dsl_destroy_snapshot(snapname, B_FALSE);
4309 	if (error != 0 && error != ENOENT)
4310 		fatal(B_FALSE, "ztest_snapshot_destroy(%s) = %d",
4311 		    snapname, error);
4312 	return (B_TRUE);
4313 }
4314 
4315 void
4316 ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
4317 {
4318 	(void) zd;
4319 	ztest_ds_t *zdtmp;
4320 	int iters;
4321 	int error;
4322 	objset_t *os, *os2;
4323 	char name[ZFS_MAX_DATASET_NAME_LEN];
4324 	zilog_t *zilog;
4325 	int i;
4326 
4327 	zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);
4328 
4329 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
4330 
4331 	(void) snprintf(name, sizeof (name), "%s/temp_%"PRIu64"",
4332 	    ztest_opts.zo_pool, id);
4333 
4334 	/*
4335 	 * If this dataset exists from a previous run, process its replay log
4336 	 * half of the time.  If we don't replay it, then dsl_destroy_head()
4337 	 * (invoked from ztest_objset_destroy_cb()) should just throw it away.
4338 	 */
4339 	if (ztest_random(2) == 0 &&
4340 	    ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
4341 	    B_TRUE, FTAG, &os) == 0) {
4342 		ztest_zd_init(zdtmp, NULL, os);
4343 		zil_replay(os, zdtmp, ztest_replay_vector);
4344 		ztest_zd_fini(zdtmp);
4345 		dmu_objset_disown(os, B_TRUE, FTAG);
4346 	}
4347 
4348 	/*
4349 	 * There may be an old instance of the dataset we're about to
4350 	 * create lying around from a previous run.  If so, destroy it
4351 	 * and all of its snapshots.
4352 	 */
4353 	(void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
4354 	    DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
4355 
4356 	/*
4357 	 * Verify that the destroyed dataset is no longer in the namespace.
4358 	 */
4359 	VERIFY3U(ENOENT, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
4360 	    B_TRUE, FTAG, &os));
4361 
4362 	/*
4363 	 * Verify that we can create a new dataset.
4364 	 */
4365 	error = ztest_dataset_create(name);
4366 	if (error) {
4367 		if (error == ENOSPC) {
4368 			ztest_record_enospc(FTAG);
4369 			goto out;
4370 		}
4371 		fatal(B_FALSE, "dmu_objset_create(%s) = %d", name, error);
4372 	}
4373 
4374 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
4375 	    FTAG, &os));
4376 
4377 	ztest_zd_init(zdtmp, NULL, os);
4378 
4379 	/*
4380 	 * Open the intent log for it.
4381 	 */
4382 	zilog = zil_open(os, ztest_get_data, NULL);
4383 
4384 	/*
4385 	 * Put some objects in there, do a little I/O to them,
4386 	 * and randomly take a couple of snapshots along the way.
4387 	 */
4388 	iters = ztest_random(5);
4389 	for (i = 0; i < iters; i++) {
4390 		ztest_dmu_object_alloc_free(zdtmp, id);
4391 		if (ztest_random(iters) == 0)
4392 			(void) ztest_snapshot_create(name, i);
4393 	}
4394 
4395 	/*
4396 	 * Verify that we cannot create an existing dataset.
4397 	 */
4398 	VERIFY3U(EEXIST, ==,
4399 	    dmu_objset_create(name, DMU_OST_OTHER, 0, NULL, NULL, NULL));
4400 
4401 	/*
4402 	 * Verify that we can hold an objset that is also owned.
4403 	 */
4404 	VERIFY0(dmu_objset_hold(name, FTAG, &os2));
4405 	dmu_objset_rele(os2, FTAG);
4406 
4407 	/*
4408 	 * Verify that we cannot own an objset that is already owned.
4409 	 */
4410 	VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
4411 	    B_FALSE, B_TRUE, FTAG, &os2));
4412 
4413 	zil_close(zilog);
4414 	dmu_objset_disown(os, B_TRUE, FTAG);
4415 	ztest_zd_fini(zdtmp);
4416 out:
4417 	(void) pthread_rwlock_unlock(&ztest_name_lock);
4418 
4419 	umem_free(zdtmp, sizeof (ztest_ds_t));
4420 }
4421 
4422 /*
4423  * Verify that dmu_snapshot_{create,destroy,open,close} work as expected.
4424  */
4425 void
4426 ztest_dmu_snapshot_create_destroy(ztest_ds_t *zd, uint64_t id)
4427 {
4428 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
4429 	(void) ztest_snapshot_destroy(zd->zd_name, id);
4430 	(void) ztest_snapshot_create(zd->zd_name, id);
4431 	(void) pthread_rwlock_unlock(&ztest_name_lock);
4432 }
4433 
4434 /*
4435  * Cleanup non-standard snapshots and clones.
4436  */
4437 static void
4438 ztest_dsl_dataset_cleanup(char *osname, uint64_t id)
4439 {
4440 	char *snap1name;
4441 	char *clone1name;
4442 	char *snap2name;
4443 	char *clone2name;
4444 	char *snap3name;
4445 	int error;
4446 
4447 	snap1name  = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4448 	clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4449 	snap2name  = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4450 	clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4451 	snap3name  = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4452 
4453 	(void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, "%s@s1_%"PRIu64"",
4454 	    osname, id);
4455 	(void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, "%s/c1_%"PRIu64"",
4456 	    osname, id);
4457 	(void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, "%s@s2_%"PRIu64"",
4458 	    clone1name, id);
4459 	(void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, "%s/c2_%"PRIu64"",
4460 	    osname, id);
4461 	(void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, "%s@s3_%"PRIu64"",
4462 	    clone1name, id);
4463 
4464 	error = dsl_destroy_head(clone2name);
4465 	if (error && error != ENOENT)
4466 		fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clone2name, error);
4467 	error = dsl_destroy_snapshot(snap3name, B_FALSE);
4468 	if (error && error != ENOENT)
4469 		fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
4470 		    snap3name, error);
4471 	error = dsl_destroy_snapshot(snap2name, B_FALSE);
4472 	if (error && error != ENOENT)
4473 		fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
4474 		    snap2name, error);
4475 	error = dsl_destroy_head(clone1name);
4476 	if (error && error != ENOENT)
4477 		fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clone1name, error);
4478 	error = dsl_destroy_snapshot(snap1name, B_FALSE);
4479 	if (error && error != ENOENT)
4480 		fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
4481 		    snap1name, error);
4482 
4483 	umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN);
4484 	umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN);
4485 	umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN);
4486 	umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN);
4487 	umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN);
4488 }
4489 
4490 /*
4491  * Verify dsl_dataset_promote handles EBUSY
4492  */
4493 void
4494 ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id)
4495 {
4496 	objset_t *os;
4497 	char *snap1name;
4498 	char *clone1name;
4499 	char *snap2name;
4500 	char *clone2name;
4501 	char *snap3name;
4502 	char *osname = zd->zd_name;
4503 	int error;
4504 
4505 	snap1name  = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4506 	clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4507 	snap2name  = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4508 	clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4509 	snap3name  = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4510 
4511 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
4512 
4513 	ztest_dsl_dataset_cleanup(osname, id);
4514 
4515 	(void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, "%s@s1_%"PRIu64"",
4516 	    osname, id);
4517 	(void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, "%s/c1_%"PRIu64"",
4518 	    osname, id);
4519 	(void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, "%s@s2_%"PRIu64"",
4520 	    clone1name, id);
4521 	(void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, "%s/c2_%"PRIu64"",
4522 	    osname, id);
4523 	(void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, "%s@s3_%"PRIu64"",
4524 	    clone1name, id);
4525 
4526 	error = dmu_objset_snapshot_one(osname, strchr(snap1name, '@') + 1);
4527 	if (error && error != EEXIST) {
4528 		if (error == ENOSPC) {
4529 			ztest_record_enospc(FTAG);
4530 			goto out;
4531 		}
4532 		fatal(B_FALSE, "dmu_take_snapshot(%s) = %d", snap1name, error);
4533 	}
4534 
4535 	error = dmu_objset_clone(clone1name, snap1name);
4536 	if (error) {
4537 		if (error == ENOSPC) {
4538 			ztest_record_enospc(FTAG);
4539 			goto out;
4540 		}
4541 		fatal(B_FALSE, "dmu_objset_create(%s) = %d", clone1name, error);
4542 	}
4543 
4544 	error = dmu_objset_snapshot_one(clone1name, strchr(snap2name, '@') + 1);
4545 	if (error && error != EEXIST) {
4546 		if (error == ENOSPC) {
4547 			ztest_record_enospc(FTAG);
4548 			goto out;
4549 		}
4550 		fatal(B_FALSE, "dmu_open_snapshot(%s) = %d", snap2name, error);
4551 	}
4552 
4553 	error = dmu_objset_snapshot_one(clone1name, strchr(snap3name, '@') + 1);
4554 	if (error && error != EEXIST) {
4555 		if (error == ENOSPC) {
4556 			ztest_record_enospc(FTAG);
4557 			goto out;
4558 		}
4559 		fatal(B_FALSE, "dmu_open_snapshot(%s) = %d", snap3name, error);
4560 	}
4561 
4562 	error = dmu_objset_clone(clone2name, snap3name);
4563 	if (error) {
4564 		if (error == ENOSPC) {
4565 			ztest_record_enospc(FTAG);
4566 			goto out;
4567 		}
4568 		fatal(B_FALSE, "dmu_objset_create(%s) = %d", clone2name, error);
4569 	}
4570 
4571 	error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
4572 	    FTAG, &os);
4573 	if (error)
4574 		fatal(B_FALSE, "dmu_objset_own(%s) = %d", snap2name, error);
4575 	error = dsl_dataset_promote(clone2name, NULL);
4576 	if (error == ENOSPC) {
4577 		dmu_objset_disown(os, B_TRUE, FTAG);
4578 		ztest_record_enospc(FTAG);
4579 		goto out;
4580 	}
4581 	if (error != EBUSY)
4582 		fatal(B_FALSE, "dsl_dataset_promote(%s), %d, not EBUSY",
4583 		    clone2name, error);
4584 	dmu_objset_disown(os, B_TRUE, FTAG);
4585 
4586 out:
4587 	ztest_dsl_dataset_cleanup(osname, id);
4588 
4589 	(void) pthread_rwlock_unlock(&ztest_name_lock);
4590 
4591 	umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN);
4592 	umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN);
4593 	umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN);
4594 	umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN);
4595 	umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN);
4596 }
4597 
4598 #undef OD_ARRAY_SIZE
4599 #define	OD_ARRAY_SIZE	4
4600 
4601 /*
4602  * Verify that dmu_object_{alloc,free} work as expected.
4603  */
4604 void
4605 ztest_dmu_object_alloc_free(ztest_ds_t *zd, uint64_t id)
4606 {
4607 	ztest_od_t *od;
4608 	int batchsize;
4609 	int size;
4610 	int b;
4611 
4612 	size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
4613 	od = umem_alloc(size, UMEM_NOFAIL);
4614 	batchsize = OD_ARRAY_SIZE;
4615 
4616 	for (b = 0; b < batchsize; b++)
4617 		ztest_od_init(od + b, id, FTAG, b, DMU_OT_UINT64_OTHER,
4618 		    0, 0, 0);
4619 
4620 	/*
4621 	 * Destroy the previous batch of objects, create a new batch,
4622 	 * and do some I/O on the new objects.
4623 	 */
4624 	if (ztest_object_init(zd, od, size, B_TRUE) != 0)
4625 		return;
4626 
4627 	while (ztest_random(4 * batchsize) != 0)
4628 		ztest_io(zd, od[ztest_random(batchsize)].od_object,
4629 		    ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
4630 
4631 	umem_free(od, size);
4632 }
4633 
4634 /*
4635  * Rewind the global allocator to verify object allocation backfilling.
4636  */
4637 void
4638 ztest_dmu_object_next_chunk(ztest_ds_t *zd, uint64_t id)
4639 {
4640 	(void) id;
4641 	objset_t *os = zd->zd_os;
4642 	uint_t dnodes_per_chunk = 1 << dmu_object_alloc_chunk_shift;
4643 	uint64_t object;
4644 
4645 	/*
4646 	 * Rewind the global allocator randomly back to a lower object number
4647 	 * to force backfilling and reclamation of recently freed dnodes.
4648 	 */
4649 	mutex_enter(&os->os_obj_lock);
4650 	object = ztest_random(os->os_obj_next_chunk);
4651 	os->os_obj_next_chunk = P2ALIGN(object, dnodes_per_chunk);
4652 	mutex_exit(&os->os_obj_lock);
4653 }
4654 
4655 #undef OD_ARRAY_SIZE
4656 #define	OD_ARRAY_SIZE	2
4657 
4658 /*
4659  * Verify that dmu_{read,write} work as expected.
4660  */
4661 void
4662 ztest_dmu_read_write(ztest_ds_t *zd, uint64_t id)
4663 {
4664 	int size;
4665 	ztest_od_t *od;
4666 
4667 	objset_t *os = zd->zd_os;
4668 	size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
4669 	od = umem_alloc(size, UMEM_NOFAIL);
4670 	dmu_tx_t *tx;
4671 	int freeit, error;
4672 	uint64_t i, n, s, txg;
4673 	bufwad_t *packbuf, *bigbuf, *pack, *bigH, *bigT;
4674 	uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
4675 	uint64_t chunksize = (1000 + ztest_random(1000)) * sizeof (uint64_t);
4676 	uint64_t regions = 997;
4677 	uint64_t stride = 123456789ULL;
4678 	uint64_t width = 40;
4679 	int free_percent = 5;
4680 
4681 	/*
4682 	 * This test uses two objects, packobj and bigobj, that are always
4683 	 * updated together (i.e. in the same tx) so that their contents are
4684 	 * in sync and can be compared.  Their contents relate to each other
4685 	 * in a simple way: packobj is a dense array of 'bufwad' structures,
4686 	 * while bigobj is a sparse array of the same bufwads.  Specifically,
4687 	 * for any index n, there are three bufwads that should be identical:
4688 	 *
4689 	 *	packobj, at offset n * sizeof (bufwad_t)
4690 	 *	bigobj, at the head of the nth chunk
4691 	 *	bigobj, at the tail of the nth chunk
4692 	 *
4693 	 * The chunk size is arbitrary. It doesn't have to be a power of two,
4694 	 * and it doesn't have any relation to the object blocksize.
4695 	 * The only requirement is that it can hold at least two bufwads.
4696 	 *
4697 	 * Normally, we write the bufwad to each of these locations.
4698 	 * However, free_percent of the time we instead write zeroes to
4699 	 * packobj and perform a dmu_free_range() on bigobj.  By comparing
4700 	 * bigobj to packobj, we can verify that the DMU is correctly
4701 	 * tracking which parts of an object are allocated and free,
4702 	 * and that the contents of the allocated blocks are correct.
4703 	 */
4704 
4705 	/*
4706 	 * Read the directory info.  If it's the first time, set things up.
4707 	 */
4708 	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, chunksize);
4709 	ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
4710 	    chunksize);
4711 
4712 	if (ztest_object_init(zd, od, size, B_FALSE) != 0) {
4713 		umem_free(od, size);
4714 		return;
4715 	}
4716 
4717 	bigobj = od[0].od_object;
4718 	packobj = od[1].od_object;
4719 	chunksize = od[0].od_gen;
4720 	ASSERT3U(chunksize, ==, od[1].od_gen);
4721 
4722 	/*
4723 	 * Prefetch a random chunk of the big object.
4724 	 * Our aim here is to get some async reads in flight
4725 	 * for blocks that we may free below; the DMU should
4726 	 * handle this race correctly.
4727 	 */
4728 	n = ztest_random(regions) * stride + ztest_random(width);
4729 	s = 1 + ztest_random(2 * width - 1);
4730 	dmu_prefetch(os, bigobj, 0, n * chunksize, s * chunksize,
4731 	    ZIO_PRIORITY_SYNC_READ);
4732 
4733 	/*
4734 	 * Pick a random index and compute the offsets into packobj and bigobj.
4735 	 */
4736 	n = ztest_random(regions) * stride + ztest_random(width);
4737 	s = 1 + ztest_random(width - 1);
4738 
4739 	packoff = n * sizeof (bufwad_t);
4740 	packsize = s * sizeof (bufwad_t);
4741 
4742 	bigoff = n * chunksize;
4743 	bigsize = s * chunksize;
4744 
4745 	packbuf = umem_alloc(packsize, UMEM_NOFAIL);
4746 	bigbuf = umem_alloc(bigsize, UMEM_NOFAIL);
4747 
4748 	/*
4749 	 * free_percent of the time, free a range of bigobj rather than
4750 	 * overwriting it.
4751 	 */
4752 	freeit = (ztest_random(100) < free_percent);
4753 
4754 	/*
4755 	 * Read the current contents of our objects.
4756 	 */
4757 	error = dmu_read(os, packobj, packoff, packsize, packbuf,
4758 	    DMU_READ_PREFETCH);
4759 	ASSERT0(error);
4760 	error = dmu_read(os, bigobj, bigoff, bigsize, bigbuf,
4761 	    DMU_READ_PREFETCH);
4762 	ASSERT0(error);
4763 
4764 	/*
4765 	 * Get a tx for the mods to both packobj and bigobj.
4766 	 */
4767 	tx = dmu_tx_create(os);
4768 
4769 	dmu_tx_hold_write(tx, packobj, packoff, packsize);
4770 
4771 	if (freeit)
4772 		dmu_tx_hold_free(tx, bigobj, bigoff, bigsize);
4773 	else
4774 		dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);
4775 
4776 	/* This accounts for setting the checksum/compression. */
4777 	dmu_tx_hold_bonus(tx, bigobj);
4778 
4779 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4780 	if (txg == 0) {
4781 		umem_free(packbuf, packsize);
4782 		umem_free(bigbuf, bigsize);
4783 		umem_free(od, size);
4784 		return;
4785 	}
4786 
4787 	enum zio_checksum cksum;
4788 	do {
4789 		cksum = (enum zio_checksum)
4790 		    ztest_random_dsl_prop(ZFS_PROP_CHECKSUM);
4791 	} while (cksum >= ZIO_CHECKSUM_LEGACY_FUNCTIONS);
4792 	dmu_object_set_checksum(os, bigobj, cksum, tx);
4793 
4794 	enum zio_compress comp;
4795 	do {
4796 		comp = (enum zio_compress)
4797 		    ztest_random_dsl_prop(ZFS_PROP_COMPRESSION);
4798 	} while (comp >= ZIO_COMPRESS_LEGACY_FUNCTIONS);
4799 	dmu_object_set_compress(os, bigobj, comp, tx);
4800 
4801 	/*
4802 	 * For each index from n to n + s, verify that the existing bufwad
4803 	 * in packobj matches the bufwads at the head and tail of the
4804 	 * corresponding chunk in bigobj.  Then update all three bufwads
4805 	 * with the new values we want to write out.
4806 	 */
4807 	for (i = 0; i < s; i++) {
4808 		/* LINTED */
4809 		pack = (bufwad_t *)((char *)packbuf + i * sizeof (bufwad_t));
4810 		/* LINTED */
4811 		bigH = (bufwad_t *)((char *)bigbuf + i * chunksize);
4812 		/* LINTED */
4813 		bigT = (bufwad_t *)((char *)bigH + chunksize) - 1;
4814 
4815 		ASSERT3U((uintptr_t)bigH - (uintptr_t)bigbuf, <, bigsize);
4816 		ASSERT3U((uintptr_t)bigT - (uintptr_t)bigbuf, <, bigsize);
4817 
4818 		if (pack->bw_txg > txg)
4819 			fatal(B_FALSE,
4820 			    "future leak: got %"PRIx64", open txg is %"PRIx64"",
4821 			    pack->bw_txg, txg);
4822 
4823 		if (pack->bw_data != 0 && pack->bw_index != n + i)
4824 			fatal(B_FALSE, "wrong index: "
4825 			    "got %"PRIx64", wanted %"PRIx64"+%"PRIx64"",
4826 			    pack->bw_index, n, i);
4827 
4828 		if (memcmp(pack, bigH, sizeof (bufwad_t)) != 0)
4829 			fatal(B_FALSE, "pack/bigH mismatch in %p/%p",
4830 			    pack, bigH);
4831 
4832 		if (memcmp(pack, bigT, sizeof (bufwad_t)) != 0)
4833 			fatal(B_FALSE, "pack/bigT mismatch in %p/%p",
4834 			    pack, bigT);
4835 
4836 		if (freeit) {
4837 			memset(pack, 0, sizeof (bufwad_t));
4838 		} else {
4839 			pack->bw_index = n + i;
4840 			pack->bw_txg = txg;
4841 			pack->bw_data = 1 + ztest_random(-2ULL);
4842 		}
4843 		*bigH = *pack;
4844 		*bigT = *pack;
4845 	}
4846 
4847 	/*
4848 	 * We've verified all the old bufwads, and made new ones.
4849 	 * Now write them out.
4850 	 */
4851 	dmu_write(os, packobj, packoff, packsize, packbuf, tx);
4852 
4853 	if (freeit) {
4854 		if (ztest_opts.zo_verbose >= 7) {
4855 			(void) printf("freeing offset %"PRIx64" size %"PRIx64""
4856 			    " txg %"PRIx64"\n",
4857 			    bigoff, bigsize, txg);
4858 		}
4859 		VERIFY0(dmu_free_range(os, bigobj, bigoff, bigsize, tx));
4860 	} else {
4861 		if (ztest_opts.zo_verbose >= 7) {
4862 			(void) printf("writing offset %"PRIx64" size %"PRIx64""
4863 			    " txg %"PRIx64"\n",
4864 			    bigoff, bigsize, txg);
4865 		}
4866 		dmu_write(os, bigobj, bigoff, bigsize, bigbuf, tx);
4867 	}
4868 
4869 	dmu_tx_commit(tx);
4870 
4871 	/*
4872 	 * Sanity check the stuff we just wrote.
4873 	 */
4874 	{
4875 		void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
4876 		void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);
4877 
4878 		VERIFY0(dmu_read(os, packobj, packoff,
4879 		    packsize, packcheck, DMU_READ_PREFETCH));
4880 		VERIFY0(dmu_read(os, bigobj, bigoff,
4881 		    bigsize, bigcheck, DMU_READ_PREFETCH));
4882 
4883 		ASSERT0(memcmp(packbuf, packcheck, packsize));
4884 		ASSERT0(memcmp(bigbuf, bigcheck, bigsize));
4885 
4886 		umem_free(packcheck, packsize);
4887 		umem_free(bigcheck, bigsize);
4888 	}
4889 
4890 	umem_free(packbuf, packsize);
4891 	umem_free(bigbuf, bigsize);
4892 	umem_free(od, size);
4893 }
4894 
4895 static void
4896 compare_and_update_pbbufs(uint64_t s, bufwad_t *packbuf, bufwad_t *bigbuf,
4897     uint64_t bigsize, uint64_t n, uint64_t chunksize, uint64_t txg)
4898 {
4899 	uint64_t i;
4900 	bufwad_t *pack;
4901 	bufwad_t *bigH;
4902 	bufwad_t *bigT;
4903 
4904 	/*
4905 	 * For each index from n to n + s, verify that the existing bufwad
4906 	 * in packobj matches the bufwads at the head and tail of the
4907 	 * corresponding chunk in bigobj.  Then update all three bufwads
4908 	 * with the new values we want to write out.
4909 	 */
4910 	for (i = 0; i < s; i++) {
4911 		/* LINTED */
4912 		pack = (bufwad_t *)((char *)packbuf + i * sizeof (bufwad_t));
4913 		/* LINTED */
4914 		bigH = (bufwad_t *)((char *)bigbuf + i * chunksize);
4915 		/* LINTED */
4916 		bigT = (bufwad_t *)((char *)bigH + chunksize) - 1;
4917 
4918 		ASSERT3U((uintptr_t)bigH - (uintptr_t)bigbuf, <, bigsize);
4919 		ASSERT3U((uintptr_t)bigT - (uintptr_t)bigbuf, <, bigsize);
4920 
4921 		if (pack->bw_txg > txg)
4922 			fatal(B_FALSE,
4923 			    "future leak: got %"PRIx64", open txg is %"PRIx64"",
4924 			    pack->bw_txg, txg);
4925 
4926 		if (pack->bw_data != 0 && pack->bw_index != n + i)
4927 			fatal(B_FALSE, "wrong index: "
4928 			    "got %"PRIx64", wanted %"PRIx64"+%"PRIx64"",
4929 			    pack->bw_index, n, i);
4930 
4931 		if (memcmp(pack, bigH, sizeof (bufwad_t)) != 0)
4932 			fatal(B_FALSE, "pack/bigH mismatch in %p/%p",
4933 			    pack, bigH);
4934 
4935 		if (memcmp(pack, bigT, sizeof (bufwad_t)) != 0)
4936 			fatal(B_FALSE, "pack/bigT mismatch in %p/%p",
4937 			    pack, bigT);
4938 
4939 		pack->bw_index = n + i;
4940 		pack->bw_txg = txg;
4941 		pack->bw_data = 1 + ztest_random(-2ULL);
4942 
4943 		*bigH = *pack;
4944 		*bigT = *pack;
4945 	}
4946 }
4947 
4948 #undef OD_ARRAY_SIZE
4949 #define	OD_ARRAY_SIZE	2
4950 
4951 void
4952 ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
4953 {
4954 	objset_t *os = zd->zd_os;
4955 	ztest_od_t *od;
4956 	dmu_tx_t *tx;
4957 	uint64_t i;
4958 	int error;
4959 	int size;
4960 	uint64_t n, s, txg;
4961 	bufwad_t *packbuf, *bigbuf;
4962 	uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
4963 	uint64_t blocksize = ztest_random_blocksize();
4964 	uint64_t chunksize = blocksize;
4965 	uint64_t regions = 997;
4966 	uint64_t stride = 123456789ULL;
4967 	uint64_t width = 9;
4968 	dmu_buf_t *bonus_db;
4969 	arc_buf_t **bigbuf_arcbufs;
4970 	dmu_object_info_t doi;
4971 
4972 	size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
4973 	od = umem_alloc(size, UMEM_NOFAIL);
4974 
4975 	/*
4976 	 * This test uses two objects, packobj and bigobj, that are always
4977 	 * updated together (i.e. in the same tx) so that their contents are
4978 	 * in sync and can be compared.  Their contents relate to each other
4979 	 * in a simple way: packobj is a dense array of 'bufwad' structures,
4980 	 * while bigobj is a sparse array of the same bufwads.  Specifically,
4981 	 * for any index n, there are three bufwads that should be identical:
4982 	 *
4983 	 *	packobj, at offset n * sizeof (bufwad_t)
4984 	 *	bigobj, at the head of the nth chunk
4985 	 *	bigobj, at the tail of the nth chunk
4986 	 *
4987 	 * The chunk size is set equal to bigobj block size so that
4988 	 * dmu_assign_arcbuf_by_dbuf() can be tested for object updates.
4989 	 */
4990 
4991 	/*
4992 	 * Read the directory info.  If it's the first time, set things up.
4993 	 */
4994 	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);
4995 	ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
4996 	    chunksize);
4997 
4998 
4999 	if (ztest_object_init(zd, od, size, B_FALSE) != 0) {
5000 		umem_free(od, size);
5001 		return;
5002 	}
5003 
5004 	bigobj = od[0].od_object;
5005 	packobj = od[1].od_object;
5006 	blocksize = od[0].od_blocksize;
5007 	chunksize = blocksize;
5008 	ASSERT3U(chunksize, ==, od[1].od_gen);
5009 
5010 	VERIFY0(dmu_object_info(os, bigobj, &doi));
5011 	VERIFY(ISP2(doi.doi_data_block_size));
5012 	VERIFY3U(chunksize, ==, doi.doi_data_block_size);
5013 	VERIFY3U(chunksize, >=, 2 * sizeof (bufwad_t));
5014 
5015 	/*
5016 	 * Pick a random index and compute the offsets into packobj and bigobj.
5017 	 */
5018 	n = ztest_random(regions) * stride + ztest_random(width);
5019 	s = 1 + ztest_random(width - 1);
5020 
5021 	packoff = n * sizeof (bufwad_t);
5022 	packsize = s * sizeof (bufwad_t);
5023 
5024 	bigoff = n * chunksize;
5025 	bigsize = s * chunksize;
5026 
5027 	packbuf = umem_zalloc(packsize, UMEM_NOFAIL);
5028 	bigbuf = umem_zalloc(bigsize, UMEM_NOFAIL);
5029 
5030 	VERIFY0(dmu_bonus_hold(os, bigobj, FTAG, &bonus_db));
5031 
5032 	bigbuf_arcbufs = umem_zalloc(2 * s * sizeof (arc_buf_t *), UMEM_NOFAIL);
5033 
5034 	/*
5035 	 * Iteration 0 test zcopy for DB_UNCACHED dbufs.
5036 	 * Iteration 1 test zcopy to already referenced dbufs.
5037 	 * Iteration 2 test zcopy to dirty dbuf in the same txg.
5038 	 * Iteration 3 test zcopy to dbuf dirty in previous txg.
5039 	 * Iteration 4 test zcopy when dbuf is no longer dirty.
5040 	 * Iteration 5 test zcopy when it can't be done.
5041 	 * Iteration 6 one more zcopy write.
5042 	 */
5043 	for (i = 0; i < 7; i++) {
5044 		uint64_t j;
5045 		uint64_t off;
5046 
5047 		/*
5048 		 * In iteration 5 (i == 5) use arcbufs
5049 		 * that don't match bigobj blksz to test
5050 		 * dmu_assign_arcbuf_by_dbuf() when it can't directly
5051 		 * assign an arcbuf to a dbuf.
5052 		 */
5053 		for (j = 0; j < s; j++) {
5054 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
5055 				bigbuf_arcbufs[j] =
5056 				    dmu_request_arcbuf(bonus_db, chunksize);
5057 			} else {
5058 				bigbuf_arcbufs[2 * j] =
5059 				    dmu_request_arcbuf(bonus_db, chunksize / 2);
5060 				bigbuf_arcbufs[2 * j + 1] =
5061 				    dmu_request_arcbuf(bonus_db, chunksize / 2);
5062 			}
5063 		}
5064 
5065 		/*
5066 		 * Get a tx for the mods to both packobj and bigobj.
5067 		 */
5068 		tx = dmu_tx_create(os);
5069 
5070 		dmu_tx_hold_write(tx, packobj, packoff, packsize);
5071 		dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);
5072 
5073 		txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5074 		if (txg == 0) {
5075 			umem_free(packbuf, packsize);
5076 			umem_free(bigbuf, bigsize);
5077 			for (j = 0; j < s; j++) {
5078 				if (i != 5 ||
5079 				    chunksize < (SPA_MINBLOCKSIZE * 2)) {
5080 					dmu_return_arcbuf(bigbuf_arcbufs[j]);
5081 				} else {
5082 					dmu_return_arcbuf(
5083 					    bigbuf_arcbufs[2 * j]);
5084 					dmu_return_arcbuf(
5085 					    bigbuf_arcbufs[2 * j + 1]);
5086 				}
5087 			}
5088 			umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
5089 			umem_free(od, size);
5090 			dmu_buf_rele(bonus_db, FTAG);
5091 			return;
5092 		}
5093 
5094 		/*
5095 		 * 50% of the time don't read objects in the 1st iteration to
5096 		 * test dmu_assign_arcbuf_by_dbuf() for the case when there are
5097 		 * no existing dbufs for the specified offsets.
5098 		 */
5099 		if (i != 0 || ztest_random(2) != 0) {
5100 			error = dmu_read(os, packobj, packoff,
5101 			    packsize, packbuf, DMU_READ_PREFETCH);
5102 			ASSERT0(error);
5103 			error = dmu_read(os, bigobj, bigoff, bigsize,
5104 			    bigbuf, DMU_READ_PREFETCH);
5105 			ASSERT0(error);
5106 		}
5107 		compare_and_update_pbbufs(s, packbuf, bigbuf, bigsize,
5108 		    n, chunksize, txg);
5109 
5110 		/*
5111 		 * We've verified all the old bufwads, and made new ones.
5112 		 * Now write them out.
5113 		 */
5114 		dmu_write(os, packobj, packoff, packsize, packbuf, tx);
5115 		if (ztest_opts.zo_verbose >= 7) {
5116 			(void) printf("writing offset %"PRIx64" size %"PRIx64""
5117 			    " txg %"PRIx64"\n",
5118 			    bigoff, bigsize, txg);
5119 		}
5120 		for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
5121 			dmu_buf_t *dbt;
5122 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
5123 				memcpy(bigbuf_arcbufs[j]->b_data,
5124 				    (caddr_t)bigbuf + (off - bigoff),
5125 				    chunksize);
5126 			} else {
5127 				memcpy(bigbuf_arcbufs[2 * j]->b_data,
5128 				    (caddr_t)bigbuf + (off - bigoff),
5129 				    chunksize / 2);
5130 				memcpy(bigbuf_arcbufs[2 * j + 1]->b_data,
5131 				    (caddr_t)bigbuf + (off - bigoff) +
5132 				    chunksize / 2,
5133 				    chunksize / 2);
5134 			}
5135 
5136 			if (i == 1) {
5137 				VERIFY(dmu_buf_hold(os, bigobj, off,
5138 				    FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
5139 			}
5140 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
5141 				VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
5142 				    off, bigbuf_arcbufs[j], tx));
5143 			} else {
5144 				VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
5145 				    off, bigbuf_arcbufs[2 * j], tx));
5146 				VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
5147 				    off + chunksize / 2,
5148 				    bigbuf_arcbufs[2 * j + 1], tx));
5149 			}
5150 			if (i == 1) {
5151 				dmu_buf_rele(dbt, FTAG);
5152 			}
5153 		}
5154 		dmu_tx_commit(tx);
5155 
5156 		/*
5157 		 * Sanity check the stuff we just wrote.
5158 		 */
5159 		{
5160 			void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
5161 			void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);
5162 
5163 			VERIFY0(dmu_read(os, packobj, packoff,
5164 			    packsize, packcheck, DMU_READ_PREFETCH));
5165 			VERIFY0(dmu_read(os, bigobj, bigoff,
5166 			    bigsize, bigcheck, DMU_READ_PREFETCH));
5167 
5168 			ASSERT0(memcmp(packbuf, packcheck, packsize));
5169 			ASSERT0(memcmp(bigbuf, bigcheck, bigsize));
5170 
5171 			umem_free(packcheck, packsize);
5172 			umem_free(bigcheck, bigsize);
5173 		}
5174 		if (i == 2) {
5175 			txg_wait_open(dmu_objset_pool(os), 0, B_TRUE);
5176 		} else if (i == 3) {
5177 			txg_wait_synced(dmu_objset_pool(os), 0);
5178 		}
5179 	}
5180 
5181 	dmu_buf_rele(bonus_db, FTAG);
5182 	umem_free(packbuf, packsize);
5183 	umem_free(bigbuf, bigsize);
5184 	umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
5185 	umem_free(od, size);
5186 }
5187 
5188 void
5189 ztest_dmu_write_parallel(ztest_ds_t *zd, uint64_t id)
5190 {
5191 	(void) id;
5192 	ztest_od_t *od;
5193 
5194 	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5195 	uint64_t offset = (1ULL << (ztest_random(20) + 43)) +
5196 	    (ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
5197 
5198 	/*
5199 	 * Have multiple threads write to large offsets in an object
5200 	 * to verify that parallel writes to an object -- even to the
5201 	 * same blocks within the object -- doesn't cause any trouble.
5202 	 */
5203 	ztest_od_init(od, ID_PARALLEL, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
5204 
5205 	if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0)
5206 		return;
5207 
5208 	while (ztest_random(10) != 0)
5209 		ztest_io(zd, od->od_object, offset);
5210 
5211 	umem_free(od, sizeof (ztest_od_t));
5212 }
5213 
5214 void
5215 ztest_dmu_prealloc(ztest_ds_t *zd, uint64_t id)
5216 {
5217 	ztest_od_t *od;
5218 	uint64_t offset = (1ULL << (ztest_random(4) + SPA_MAXBLOCKSHIFT)) +
5219 	    (ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
5220 	uint64_t count = ztest_random(20) + 1;
5221 	uint64_t blocksize = ztest_random_blocksize();
5222 	void *data;
5223 
5224 	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5225 
5226 	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);
5227 
5228 	if (ztest_object_init(zd, od, sizeof (ztest_od_t),
5229 	    !ztest_random(2)) != 0) {
5230 		umem_free(od, sizeof (ztest_od_t));
5231 		return;
5232 	}
5233 
5234 	if (ztest_truncate(zd, od->od_object, offset, count * blocksize) != 0) {
5235 		umem_free(od, sizeof (ztest_od_t));
5236 		return;
5237 	}
5238 
5239 	ztest_prealloc(zd, od->od_object, offset, count * blocksize);
5240 
5241 	data = umem_zalloc(blocksize, UMEM_NOFAIL);
5242 
5243 	while (ztest_random(count) != 0) {
5244 		uint64_t randoff = offset + (ztest_random(count) * blocksize);
5245 		if (ztest_write(zd, od->od_object, randoff, blocksize,
5246 		    data) != 0)
5247 			break;
5248 		while (ztest_random(4) != 0)
5249 			ztest_io(zd, od->od_object, randoff);
5250 	}
5251 
5252 	umem_free(data, blocksize);
5253 	umem_free(od, sizeof (ztest_od_t));
5254 }
5255 
5256 /*
5257  * Verify that zap_{create,destroy,add,remove,update} work as expected.
5258  */
5259 #define	ZTEST_ZAP_MIN_INTS	1
5260 #define	ZTEST_ZAP_MAX_INTS	4
5261 #define	ZTEST_ZAP_MAX_PROPS	1000
5262 
5263 void
5264 ztest_zap(ztest_ds_t *zd, uint64_t id)
5265 {
5266 	objset_t *os = zd->zd_os;
5267 	ztest_od_t *od;
5268 	uint64_t object;
5269 	uint64_t txg, last_txg;
5270 	uint64_t value[ZTEST_ZAP_MAX_INTS];
5271 	uint64_t zl_ints, zl_intsize, prop;
5272 	int i, ints;
5273 	dmu_tx_t *tx;
5274 	char propname[100], txgname[100];
5275 	int error;
5276 	const char *const hc[2] = { "s.acl.h", ".s.open.h.hyLZlg" };
5277 
5278 	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5279 	ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);
5280 
5281 	if (ztest_object_init(zd, od, sizeof (ztest_od_t),
5282 	    !ztest_random(2)) != 0)
5283 		goto out;
5284 
5285 	object = od->od_object;
5286 
5287 	/*
5288 	 * Generate a known hash collision, and verify that
5289 	 * we can lookup and remove both entries.
5290 	 */
5291 	tx = dmu_tx_create(os);
5292 	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5293 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5294 	if (txg == 0)
5295 		goto out;
5296 	for (i = 0; i < 2; i++) {
5297 		value[i] = i;
5298 		VERIFY0(zap_add(os, object, hc[i], sizeof (uint64_t),
5299 		    1, &value[i], tx));
5300 	}
5301 	for (i = 0; i < 2; i++) {
5302 		VERIFY3U(EEXIST, ==, zap_add(os, object, hc[i],
5303 		    sizeof (uint64_t), 1, &value[i], tx));
5304 		VERIFY0(
5305 		    zap_length(os, object, hc[i], &zl_intsize, &zl_ints));
5306 		ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
5307 		ASSERT3U(zl_ints, ==, 1);
5308 	}
5309 	for (i = 0; i < 2; i++) {
5310 		VERIFY0(zap_remove(os, object, hc[i], tx));
5311 	}
5312 	dmu_tx_commit(tx);
5313 
5314 	/*
5315 	 * Generate a bunch of random entries.
5316 	 */
5317 	ints = MAX(ZTEST_ZAP_MIN_INTS, object % ZTEST_ZAP_MAX_INTS);
5318 
5319 	prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
5320 	(void) sprintf(propname, "prop_%"PRIu64"", prop);
5321 	(void) sprintf(txgname, "txg_%"PRIu64"", prop);
5322 	memset(value, 0, sizeof (value));
5323 	last_txg = 0;
5324 
5325 	/*
5326 	 * If these zap entries already exist, validate their contents.
5327 	 */
5328 	error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
5329 	if (error == 0) {
5330 		ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
5331 		ASSERT3U(zl_ints, ==, 1);
5332 
5333 		VERIFY0(zap_lookup(os, object, txgname, zl_intsize,
5334 		    zl_ints, &last_txg));
5335 
5336 		VERIFY0(zap_length(os, object, propname, &zl_intsize,
5337 		    &zl_ints));
5338 
5339 		ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
5340 		ASSERT3U(zl_ints, ==, ints);
5341 
5342 		VERIFY0(zap_lookup(os, object, propname, zl_intsize,
5343 		    zl_ints, value));
5344 
5345 		for (i = 0; i < ints; i++) {
5346 			ASSERT3U(value[i], ==, last_txg + object + i);
5347 		}
5348 	} else {
5349 		ASSERT3U(error, ==, ENOENT);
5350 	}
5351 
5352 	/*
5353 	 * Atomically update two entries in our zap object.
5354 	 * The first is named txg_%llu, and contains the txg
5355 	 * in which the property was last updated.  The second
5356 	 * is named prop_%llu, and the nth element of its value
5357 	 * should be txg + object + n.
5358 	 */
5359 	tx = dmu_tx_create(os);
5360 	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5361 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5362 	if (txg == 0)
5363 		goto out;
5364 
5365 	if (last_txg > txg)
5366 		fatal(B_FALSE, "zap future leak: old %"PRIu64" new %"PRIu64"",
5367 		    last_txg, txg);
5368 
5369 	for (i = 0; i < ints; i++)
5370 		value[i] = txg + object + i;
5371 
5372 	VERIFY0(zap_update(os, object, txgname, sizeof (uint64_t),
5373 	    1, &txg, tx));
5374 	VERIFY0(zap_update(os, object, propname, sizeof (uint64_t),
5375 	    ints, value, tx));
5376 
5377 	dmu_tx_commit(tx);
5378 
5379 	/*
5380 	 * Remove a random pair of entries.
5381 	 */
5382 	prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
5383 	(void) sprintf(propname, "prop_%"PRIu64"", prop);
5384 	(void) sprintf(txgname, "txg_%"PRIu64"", prop);
5385 
5386 	error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
5387 
5388 	if (error == ENOENT)
5389 		goto out;
5390 
5391 	ASSERT0(error);
5392 
5393 	tx = dmu_tx_create(os);
5394 	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5395 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5396 	if (txg == 0)
5397 		goto out;
5398 	VERIFY0(zap_remove(os, object, txgname, tx));
5399 	VERIFY0(zap_remove(os, object, propname, tx));
5400 	dmu_tx_commit(tx);
5401 out:
5402 	umem_free(od, sizeof (ztest_od_t));
5403 }
5404 
5405 /*
5406  * Test case to test the upgrading of a microzap to fatzap.
5407  */
5408 void
5409 ztest_fzap(ztest_ds_t *zd, uint64_t id)
5410 {
5411 	objset_t *os = zd->zd_os;
5412 	ztest_od_t *od;
5413 	uint64_t object, txg, value;
5414 
5415 	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5416 	ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);
5417 
5418 	if (ztest_object_init(zd, od, sizeof (ztest_od_t),
5419 	    !ztest_random(2)) != 0)
5420 		goto out;
5421 	object = od->od_object;
5422 
5423 	/*
5424 	 * Add entries to this ZAP and make sure it spills over
5425 	 * and gets upgraded to a fatzap. Also, since we are adding
5426 	 * 2050 entries we should see ptrtbl growth and leaf-block split.
5427 	 */
5428 	for (value = 0; value < 2050; value++) {
5429 		char name[ZFS_MAX_DATASET_NAME_LEN];
5430 		dmu_tx_t *tx;
5431 		int error;
5432 
5433 		(void) snprintf(name, sizeof (name), "fzap-%"PRIu64"-%"PRIu64"",
5434 		    id, value);
5435 
5436 		tx = dmu_tx_create(os);
5437 		dmu_tx_hold_zap(tx, object, B_TRUE, name);
5438 		txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5439 		if (txg == 0)
5440 			goto out;
5441 		error = zap_add(os, object, name, sizeof (uint64_t), 1,
5442 		    &value, tx);
5443 		ASSERT(error == 0 || error == EEXIST);
5444 		dmu_tx_commit(tx);
5445 	}
5446 out:
5447 	umem_free(od, sizeof (ztest_od_t));
5448 }
5449 
5450 void
5451 ztest_zap_parallel(ztest_ds_t *zd, uint64_t id)
5452 {
5453 	(void) id;
5454 	objset_t *os = zd->zd_os;
5455 	ztest_od_t *od;
5456 	uint64_t txg, object, count, wsize, wc, zl_wsize, zl_wc;
5457 	dmu_tx_t *tx;
5458 	int i, namelen, error;
5459 	int micro = ztest_random(2);
5460 	char name[20], string_value[20];
5461 	void *data;
5462 
5463 	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5464 	ztest_od_init(od, ID_PARALLEL, FTAG, micro, DMU_OT_ZAP_OTHER, 0, 0, 0);
5465 
5466 	if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
5467 		umem_free(od, sizeof (ztest_od_t));
5468 		return;
5469 	}
5470 
5471 	object = od->od_object;
5472 
5473 	/*
5474 	 * Generate a random name of the form 'xxx.....' where each
5475 	 * x is a random printable character and the dots are dots.
5476 	 * There are 94 such characters, and the name length goes from
5477 	 * 6 to 20, so there are 94^3 * 15 = 12,458,760 possible names.
5478 	 */
5479 	namelen = ztest_random(sizeof (name) - 5) + 5 + 1;
5480 
5481 	for (i = 0; i < 3; i++)
5482 		name[i] = '!' + ztest_random('~' - '!' + 1);
5483 	for (; i < namelen - 1; i++)
5484 		name[i] = '.';
5485 	name[i] = '\0';
5486 
5487 	if ((namelen & 1) || micro) {
5488 		wsize = sizeof (txg);
5489 		wc = 1;
5490 		data = &txg;
5491 	} else {
5492 		wsize = 1;
5493 		wc = namelen;
5494 		data = string_value;
5495 	}
5496 
5497 	count = -1ULL;
5498 	VERIFY0(zap_count(os, object, &count));
5499 	ASSERT3S(count, !=, -1ULL);
5500 
5501 	/*
5502 	 * Select an operation: length, lookup, add, update, remove.
5503 	 */
5504 	i = ztest_random(5);
5505 
5506 	if (i >= 2) {
5507 		tx = dmu_tx_create(os);
5508 		dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5509 		txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5510 		if (txg == 0) {
5511 			umem_free(od, sizeof (ztest_od_t));
5512 			return;
5513 		}
5514 		memcpy(string_value, name, namelen);
5515 	} else {
5516 		tx = NULL;
5517 		txg = 0;
5518 		memset(string_value, 0, namelen);
5519 	}
5520 
5521 	switch (i) {
5522 
5523 	case 0:
5524 		error = zap_length(os, object, name, &zl_wsize, &zl_wc);
5525 		if (error == 0) {
5526 			ASSERT3U(wsize, ==, zl_wsize);
5527 			ASSERT3U(wc, ==, zl_wc);
5528 		} else {
5529 			ASSERT3U(error, ==, ENOENT);
5530 		}
5531 		break;
5532 
5533 	case 1:
5534 		error = zap_lookup(os, object, name, wsize, wc, data);
5535 		if (error == 0) {
5536 			if (data == string_value &&
5537 			    memcmp(name, data, namelen) != 0)
5538 				fatal(B_FALSE, "name '%s' != val '%s' len %d",
5539 				    name, (char *)data, namelen);
5540 		} else {
5541 			ASSERT3U(error, ==, ENOENT);
5542 		}
5543 		break;
5544 
5545 	case 2:
5546 		error = zap_add(os, object, name, wsize, wc, data, tx);
5547 		ASSERT(error == 0 || error == EEXIST);
5548 		break;
5549 
5550 	case 3:
5551 		VERIFY0(zap_update(os, object, name, wsize, wc, data, tx));
5552 		break;
5553 
5554 	case 4:
5555 		error = zap_remove(os, object, name, tx);
5556 		ASSERT(error == 0 || error == ENOENT);
5557 		break;
5558 	}
5559 
5560 	if (tx != NULL)
5561 		dmu_tx_commit(tx);
5562 
5563 	umem_free(od, sizeof (ztest_od_t));
5564 }
5565 
5566 /*
5567  * Commit callback data.
5568  */
5569 typedef struct ztest_cb_data {
5570 	list_node_t		zcd_node;
5571 	uint64_t		zcd_txg;
5572 	int			zcd_expected_err;
5573 	boolean_t		zcd_added;
5574 	boolean_t		zcd_called;
5575 	spa_t			*zcd_spa;
5576 } ztest_cb_data_t;
5577 
5578 /* This is the actual commit callback function */
5579 static void
5580 ztest_commit_callback(void *arg, int error)
5581 {
5582 	ztest_cb_data_t *data = arg;
5583 	uint64_t synced_txg;
5584 
5585 	VERIFY3P(data, !=, NULL);
5586 	VERIFY3S(data->zcd_expected_err, ==, error);
5587 	VERIFY(!data->zcd_called);
5588 
5589 	synced_txg = spa_last_synced_txg(data->zcd_spa);
5590 	if (data->zcd_txg > synced_txg)
5591 		fatal(B_FALSE,
5592 		    "commit callback of txg %"PRIu64" called prematurely, "
5593 		    "last synced txg = %"PRIu64"\n",
5594 		    data->zcd_txg, synced_txg);
5595 
5596 	data->zcd_called = B_TRUE;
5597 
5598 	if (error == ECANCELED) {
5599 		ASSERT0(data->zcd_txg);
5600 		ASSERT(!data->zcd_added);
5601 
5602 		/*
5603 		 * The private callback data should be destroyed here, but
5604 		 * since we are going to check the zcd_called field after
5605 		 * dmu_tx_abort(), we will destroy it there.
5606 		 */
5607 		return;
5608 	}
5609 
5610 	ASSERT(data->zcd_added);
5611 	ASSERT3U(data->zcd_txg, !=, 0);
5612 
5613 	(void) mutex_enter(&zcl.zcl_callbacks_lock);
5614 
5615 	/* See if this cb was called more quickly */
5616 	if ((synced_txg - data->zcd_txg) < zc_min_txg_delay)
5617 		zc_min_txg_delay = synced_txg - data->zcd_txg;
5618 
5619 	/* Remove our callback from the list */
5620 	list_remove(&zcl.zcl_callbacks, data);
5621 
5622 	(void) mutex_exit(&zcl.zcl_callbacks_lock);
5623 
5624 	umem_free(data, sizeof (ztest_cb_data_t));
5625 }
5626 
5627 /* Allocate and initialize callback data structure */
5628 static ztest_cb_data_t *
5629 ztest_create_cb_data(objset_t *os, uint64_t txg)
5630 {
5631 	ztest_cb_data_t *cb_data;
5632 
5633 	cb_data = umem_zalloc(sizeof (ztest_cb_data_t), UMEM_NOFAIL);
5634 
5635 	cb_data->zcd_txg = txg;
5636 	cb_data->zcd_spa = dmu_objset_spa(os);
5637 	list_link_init(&cb_data->zcd_node);
5638 
5639 	return (cb_data);
5640 }
5641 
5642 /*
5643  * Commit callback test.
5644  */
5645 void
5646 ztest_dmu_commit_callbacks(ztest_ds_t *zd, uint64_t id)
5647 {
5648 	objset_t *os = zd->zd_os;
5649 	ztest_od_t *od;
5650 	dmu_tx_t *tx;
5651 	ztest_cb_data_t *cb_data[3], *tmp_cb;
5652 	uint64_t old_txg, txg;
5653 	int i, error = 0;
5654 
5655 	od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5656 	ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
5657 
5658 	if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
5659 		umem_free(od, sizeof (ztest_od_t));
5660 		return;
5661 	}
5662 
5663 	tx = dmu_tx_create(os);
5664 
5665 	cb_data[0] = ztest_create_cb_data(os, 0);
5666 	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[0]);
5667 
5668 	dmu_tx_hold_write(tx, od->od_object, 0, sizeof (uint64_t));
5669 
5670 	/* Every once in a while, abort the transaction on purpose */
5671 	if (ztest_random(100) == 0)
5672 		error = -1;
5673 
5674 	if (!error)
5675 		error = dmu_tx_assign(tx, TXG_NOWAIT);
5676 
5677 	txg = error ? 0 : dmu_tx_get_txg(tx);
5678 
5679 	cb_data[0]->zcd_txg = txg;
5680 	cb_data[1] = ztest_create_cb_data(os, txg);
5681 	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[1]);
5682 
5683 	if (error) {
5684 		/*
5685 		 * It's not a strict requirement to call the registered
5686 		 * callbacks from inside dmu_tx_abort(), but that's what
5687 		 * it's supposed to happen in the current implementation
5688 		 * so we will check for that.
5689 		 */
5690 		for (i = 0; i < 2; i++) {
5691 			cb_data[i]->zcd_expected_err = ECANCELED;
5692 			VERIFY(!cb_data[i]->zcd_called);
5693 		}
5694 
5695 		dmu_tx_abort(tx);
5696 
5697 		for (i = 0; i < 2; i++) {
5698 			VERIFY(cb_data[i]->zcd_called);
5699 			umem_free(cb_data[i], sizeof (ztest_cb_data_t));
5700 		}
5701 
5702 		umem_free(od, sizeof (ztest_od_t));
5703 		return;
5704 	}
5705 
5706 	cb_data[2] = ztest_create_cb_data(os, txg);
5707 	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[2]);
5708 
5709 	/*
5710 	 * Read existing data to make sure there isn't a future leak.
5711 	 */
5712 	VERIFY0(dmu_read(os, od->od_object, 0, sizeof (uint64_t),
5713 	    &old_txg, DMU_READ_PREFETCH));
5714 
5715 	if (old_txg > txg)
5716 		fatal(B_FALSE,
5717 		    "future leak: got %"PRIu64", open txg is %"PRIu64"",
5718 		    old_txg, txg);
5719 
5720 	dmu_write(os, od->od_object, 0, sizeof (uint64_t), &txg, tx);
5721 
5722 	(void) mutex_enter(&zcl.zcl_callbacks_lock);
5723 
5724 	/*
5725 	 * Since commit callbacks don't have any ordering requirement and since
5726 	 * it is theoretically possible for a commit callback to be called
5727 	 * after an arbitrary amount of time has elapsed since its txg has been
5728 	 * synced, it is difficult to reliably determine whether a commit
5729 	 * callback hasn't been called due to high load or due to a flawed
5730 	 * implementation.
5731 	 *
5732 	 * In practice, we will assume that if after a certain number of txgs a
5733 	 * commit callback hasn't been called, then most likely there's an
5734 	 * implementation bug..
5735 	 */
5736 	tmp_cb = list_head(&zcl.zcl_callbacks);
5737 	if (tmp_cb != NULL &&
5738 	    tmp_cb->zcd_txg + ZTEST_COMMIT_CB_THRESH < txg) {
5739 		fatal(B_FALSE,
5740 		    "Commit callback threshold exceeded, "
5741 		    "oldest txg: %"PRIu64", open txg: %"PRIu64"\n",
5742 		    tmp_cb->zcd_txg, txg);
5743 	}
5744 
5745 	/*
5746 	 * Let's find the place to insert our callbacks.
5747 	 *
5748 	 * Even though the list is ordered by txg, it is possible for the
5749 	 * insertion point to not be the end because our txg may already be
5750 	 * quiescing at this point and other callbacks in the open txg
5751 	 * (from other objsets) may have sneaked in.
5752 	 */
5753 	tmp_cb = list_tail(&zcl.zcl_callbacks);
5754 	while (tmp_cb != NULL && tmp_cb->zcd_txg > txg)
5755 		tmp_cb = list_prev(&zcl.zcl_callbacks, tmp_cb);
5756 
5757 	/* Add the 3 callbacks to the list */
5758 	for (i = 0; i < 3; i++) {
5759 		if (tmp_cb == NULL)
5760 			list_insert_head(&zcl.zcl_callbacks, cb_data[i]);
5761 		else
5762 			list_insert_after(&zcl.zcl_callbacks, tmp_cb,
5763 			    cb_data[i]);
5764 
5765 		cb_data[i]->zcd_added = B_TRUE;
5766 		VERIFY(!cb_data[i]->zcd_called);
5767 
5768 		tmp_cb = cb_data[i];
5769 	}
5770 
5771 	zc_cb_counter += 3;
5772 
5773 	(void) mutex_exit(&zcl.zcl_callbacks_lock);
5774 
5775 	dmu_tx_commit(tx);
5776 
5777 	umem_free(od, sizeof (ztest_od_t));
5778 }
5779 
5780 /*
5781  * Visit each object in the dataset. Verify that its properties
5782  * are consistent what was stored in the block tag when it was created,
5783  * and that its unused bonus buffer space has not been overwritten.
5784  */
5785 void
5786 ztest_verify_dnode_bt(ztest_ds_t *zd, uint64_t id)
5787 {
5788 	(void) id;
5789 	objset_t *os = zd->zd_os;
5790 	uint64_t obj;
5791 	int err = 0;
5792 
5793 	for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {
5794 		ztest_block_tag_t *bt = NULL;
5795 		dmu_object_info_t doi;
5796 		dmu_buf_t *db;
5797 
5798 		ztest_object_lock(zd, obj, RL_READER);
5799 		if (dmu_bonus_hold(os, obj, FTAG, &db) != 0) {
5800 			ztest_object_unlock(zd, obj);
5801 			continue;
5802 		}
5803 
5804 		dmu_object_info_from_db(db, &doi);
5805 		if (doi.doi_bonus_size >= sizeof (*bt))
5806 			bt = ztest_bt_bonus(db);
5807 
5808 		if (bt && bt->bt_magic == BT_MAGIC) {
5809 			ztest_bt_verify(bt, os, obj, doi.doi_dnodesize,
5810 			    bt->bt_offset, bt->bt_gen, bt->bt_txg,
5811 			    bt->bt_crtxg);
5812 			ztest_verify_unused_bonus(db, bt, obj, os, bt->bt_gen);
5813 		}
5814 
5815 		dmu_buf_rele(db, FTAG);
5816 		ztest_object_unlock(zd, obj);
5817 	}
5818 }
5819 
5820 void
5821 ztest_dsl_prop_get_set(ztest_ds_t *zd, uint64_t id)
5822 {
5823 	(void) id;
5824 	zfs_prop_t proplist[] = {
5825 		ZFS_PROP_CHECKSUM,
5826 		ZFS_PROP_COMPRESSION,
5827 		ZFS_PROP_COPIES,
5828 		ZFS_PROP_DEDUP
5829 	};
5830 
5831 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
5832 
5833 	for (int p = 0; p < sizeof (proplist) / sizeof (proplist[0]); p++)
5834 		(void) ztest_dsl_prop_set_uint64(zd->zd_name, proplist[p],
5835 		    ztest_random_dsl_prop(proplist[p]), (int)ztest_random(2));
5836 
5837 	VERIFY0(ztest_dsl_prop_set_uint64(zd->zd_name, ZFS_PROP_RECORDSIZE,
5838 	    ztest_random_blocksize(), (int)ztest_random(2)));
5839 
5840 	(void) pthread_rwlock_unlock(&ztest_name_lock);
5841 }
5842 
5843 void
5844 ztest_spa_prop_get_set(ztest_ds_t *zd, uint64_t id)
5845 {
5846 	(void) zd, (void) id;
5847 	nvlist_t *props = NULL;
5848 
5849 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
5850 
5851 	(void) ztest_spa_prop_set_uint64(ZPOOL_PROP_AUTOTRIM, ztest_random(2));
5852 
5853 	VERIFY0(spa_prop_get(ztest_spa, &props));
5854 
5855 	if (ztest_opts.zo_verbose >= 6)
5856 		dump_nvlist(props, 4);
5857 
5858 	fnvlist_free(props);
5859 
5860 	(void) pthread_rwlock_unlock(&ztest_name_lock);
5861 }
5862 
5863 static int
5864 user_release_one(const char *snapname, const char *holdname)
5865 {
5866 	nvlist_t *snaps, *holds;
5867 	int error;
5868 
5869 	snaps = fnvlist_alloc();
5870 	holds = fnvlist_alloc();
5871 	fnvlist_add_boolean(holds, holdname);
5872 	fnvlist_add_nvlist(snaps, snapname, holds);
5873 	fnvlist_free(holds);
5874 	error = dsl_dataset_user_release(snaps, NULL);
5875 	fnvlist_free(snaps);
5876 	return (error);
5877 }
5878 
5879 /*
5880  * Test snapshot hold/release and deferred destroy.
5881  */
5882 void
5883 ztest_dmu_snapshot_hold(ztest_ds_t *zd, uint64_t id)
5884 {
5885 	int error;
5886 	objset_t *os = zd->zd_os;
5887 	objset_t *origin;
5888 	char snapname[100];
5889 	char fullname[100];
5890 	char clonename[100];
5891 	char tag[100];
5892 	char osname[ZFS_MAX_DATASET_NAME_LEN];
5893 	nvlist_t *holds;
5894 
5895 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
5896 
5897 	dmu_objset_name(os, osname);
5898 
5899 	(void) snprintf(snapname, sizeof (snapname), "sh1_%"PRIu64"", id);
5900 	(void) snprintf(fullname, sizeof (fullname), "%s@%s", osname, snapname);
5901 	(void) snprintf(clonename, sizeof (clonename), "%s/ch1_%"PRIu64"",
5902 	    osname, id);
5903 	(void) snprintf(tag, sizeof (tag), "tag_%"PRIu64"", id);
5904 
5905 	/*
5906 	 * Clean up from any previous run.
5907 	 */
5908 	error = dsl_destroy_head(clonename);
5909 	if (error != ENOENT)
5910 		ASSERT0(error);
5911 	error = user_release_one(fullname, tag);
5912 	if (error != ESRCH && error != ENOENT)
5913 		ASSERT0(error);
5914 	error = dsl_destroy_snapshot(fullname, B_FALSE);
5915 	if (error != ENOENT)
5916 		ASSERT0(error);
5917 
5918 	/*
5919 	 * Create snapshot, clone it, mark snap for deferred destroy,
5920 	 * destroy clone, verify snap was also destroyed.
5921 	 */
5922 	error = dmu_objset_snapshot_one(osname, snapname);
5923 	if (error) {
5924 		if (error == ENOSPC) {
5925 			ztest_record_enospc("dmu_objset_snapshot");
5926 			goto out;
5927 		}
5928 		fatal(B_FALSE, "dmu_objset_snapshot(%s) = %d", fullname, error);
5929 	}
5930 
5931 	error = dmu_objset_clone(clonename, fullname);
5932 	if (error) {
5933 		if (error == ENOSPC) {
5934 			ztest_record_enospc("dmu_objset_clone");
5935 			goto out;
5936 		}
5937 		fatal(B_FALSE, "dmu_objset_clone(%s) = %d", clonename, error);
5938 	}
5939 
5940 	error = dsl_destroy_snapshot(fullname, B_TRUE);
5941 	if (error) {
5942 		fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
5943 		    fullname, error);
5944 	}
5945 
5946 	error = dsl_destroy_head(clonename);
5947 	if (error)
5948 		fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clonename, error);
5949 
5950 	error = dmu_objset_hold(fullname, FTAG, &origin);
5951 	if (error != ENOENT)
5952 		fatal(B_FALSE, "dmu_objset_hold(%s) = %d", fullname, error);
5953 
5954 	/*
5955 	 * Create snapshot, add temporary hold, verify that we can't
5956 	 * destroy a held snapshot, mark for deferred destroy,
5957 	 * release hold, verify snapshot was destroyed.
5958 	 */
5959 	error = dmu_objset_snapshot_one(osname, snapname);
5960 	if (error) {
5961 		if (error == ENOSPC) {
5962 			ztest_record_enospc("dmu_objset_snapshot");
5963 			goto out;
5964 		}
5965 		fatal(B_FALSE, "dmu_objset_snapshot(%s) = %d", fullname, error);
5966 	}
5967 
5968 	holds = fnvlist_alloc();
5969 	fnvlist_add_string(holds, fullname, tag);
5970 	error = dsl_dataset_user_hold(holds, 0, NULL);
5971 	fnvlist_free(holds);
5972 
5973 	if (error == ENOSPC) {
5974 		ztest_record_enospc("dsl_dataset_user_hold");
5975 		goto out;
5976 	} else if (error) {
5977 		fatal(B_FALSE, "dsl_dataset_user_hold(%s, %s) = %u",
5978 		    fullname, tag, error);
5979 	}
5980 
5981 	error = dsl_destroy_snapshot(fullname, B_FALSE);
5982 	if (error != EBUSY) {
5983 		fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_FALSE) = %d",
5984 		    fullname, error);
5985 	}
5986 
5987 	error = dsl_destroy_snapshot(fullname, B_TRUE);
5988 	if (error) {
5989 		fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
5990 		    fullname, error);
5991 	}
5992 
5993 	error = user_release_one(fullname, tag);
5994 	if (error)
5995 		fatal(B_FALSE, "user_release_one(%s, %s) = %d",
5996 		    fullname, tag, error);
5997 
5998 	VERIFY3U(dmu_objset_hold(fullname, FTAG, &origin), ==, ENOENT);
5999 
6000 out:
6001 	(void) pthread_rwlock_unlock(&ztest_name_lock);
6002 }
6003 
6004 /*
6005  * Inject random faults into the on-disk data.
6006  */
6007 void
6008 ztest_fault_inject(ztest_ds_t *zd, uint64_t id)
6009 {
6010 	(void) zd, (void) id;
6011 	ztest_shared_t *zs = ztest_shared;
6012 	spa_t *spa = ztest_spa;
6013 	int fd;
6014 	uint64_t offset;
6015 	uint64_t leaves;
6016 	uint64_t bad = 0x1990c0ffeedecadeull;
6017 	uint64_t top, leaf;
6018 	char *path0;
6019 	char *pathrand;
6020 	size_t fsize;
6021 	int bshift = SPA_MAXBLOCKSHIFT + 2;
6022 	int iters = 1000;
6023 	int maxfaults;
6024 	int mirror_save;
6025 	vdev_t *vd0 = NULL;
6026 	uint64_t guid0 = 0;
6027 	boolean_t islog = B_FALSE;
6028 
6029 	path0 = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
6030 	pathrand = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
6031 
6032 	mutex_enter(&ztest_vdev_lock);
6033 
6034 	/*
6035 	 * Device removal is in progress, fault injection must be disabled
6036 	 * until it completes and the pool is scrubbed.  The fault injection
6037 	 * strategy for damaging blocks does not take in to account evacuated
6038 	 * blocks which may have already been damaged.
6039 	 */
6040 	if (ztest_device_removal_active) {
6041 		mutex_exit(&ztest_vdev_lock);
6042 		goto out;
6043 	}
6044 
6045 	maxfaults = MAXFAULTS(zs);
6046 	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raid_children;
6047 	mirror_save = zs->zs_mirrors;
6048 	mutex_exit(&ztest_vdev_lock);
6049 
6050 	ASSERT3U(leaves, >=, 1);
6051 
6052 	/*
6053 	 * While ztest is running the number of leaves will not change.  This
6054 	 * is critical for the fault injection logic as it determines where
6055 	 * errors can be safely injected such that they are always repairable.
6056 	 *
6057 	 * When restarting ztest a different number of leaves may be requested
6058 	 * which will shift the regions to be damaged.  This is fine as long
6059 	 * as the pool has been scrubbed prior to using the new mapping.
6060 	 * Failure to do can result in non-repairable damage being injected.
6061 	 */
6062 	if (ztest_pool_scrubbed == B_FALSE)
6063 		goto out;
6064 
6065 	/*
6066 	 * Grab the name lock as reader. There are some operations
6067 	 * which don't like to have their vdevs changed while
6068 	 * they are in progress (i.e. spa_change_guid). Those
6069 	 * operations will have grabbed the name lock as writer.
6070 	 */
6071 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
6072 
6073 	/*
6074 	 * We need SCL_STATE here because we're going to look at vd0->vdev_tsd.
6075 	 */
6076 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
6077 
6078 	if (ztest_random(2) == 0) {
6079 		/*
6080 		 * Inject errors on a normal data device or slog device.
6081 		 */
6082 		top = ztest_random_vdev_top(spa, B_TRUE);
6083 		leaf = ztest_random(leaves) + zs->zs_splits;
6084 
6085 		/*
6086 		 * Generate paths to the first leaf in this top-level vdev,
6087 		 * and to the random leaf we selected.  We'll induce transient
6088 		 * write failures and random online/offline activity on leaf 0,
6089 		 * and we'll write random garbage to the randomly chosen leaf.
6090 		 */
6091 		(void) snprintf(path0, MAXPATHLEN, ztest_dev_template,
6092 		    ztest_opts.zo_dir, ztest_opts.zo_pool,
6093 		    top * leaves + zs->zs_splits);
6094 		(void) snprintf(pathrand, MAXPATHLEN, ztest_dev_template,
6095 		    ztest_opts.zo_dir, ztest_opts.zo_pool,
6096 		    top * leaves + leaf);
6097 
6098 		vd0 = vdev_lookup_by_path(spa->spa_root_vdev, path0);
6099 		if (vd0 != NULL && vd0->vdev_top->vdev_islog)
6100 			islog = B_TRUE;
6101 
6102 		/*
6103 		 * If the top-level vdev needs to be resilvered
6104 		 * then we only allow faults on the device that is
6105 		 * resilvering.
6106 		 */
6107 		if (vd0 != NULL && maxfaults != 1 &&
6108 		    (!vdev_resilver_needed(vd0->vdev_top, NULL, NULL) ||
6109 		    vd0->vdev_resilver_txg != 0)) {
6110 			/*
6111 			 * Make vd0 explicitly claim to be unreadable,
6112 			 * or unwritable, or reach behind its back
6113 			 * and close the underlying fd.  We can do this if
6114 			 * maxfaults == 0 because we'll fail and reexecute,
6115 			 * and we can do it if maxfaults >= 2 because we'll
6116 			 * have enough redundancy.  If maxfaults == 1, the
6117 			 * combination of this with injection of random data
6118 			 * corruption below exceeds the pool's fault tolerance.
6119 			 */
6120 			vdev_file_t *vf = vd0->vdev_tsd;
6121 
6122 			zfs_dbgmsg("injecting fault to vdev %llu; maxfaults=%d",
6123 			    (long long)vd0->vdev_id, (int)maxfaults);
6124 
6125 			if (vf != NULL && ztest_random(3) == 0) {
6126 				(void) close(vf->vf_file->f_fd);
6127 				vf->vf_file->f_fd = -1;
6128 			} else if (ztest_random(2) == 0) {
6129 				vd0->vdev_cant_read = B_TRUE;
6130 			} else {
6131 				vd0->vdev_cant_write = B_TRUE;
6132 			}
6133 			guid0 = vd0->vdev_guid;
6134 		}
6135 	} else {
6136 		/*
6137 		 * Inject errors on an l2cache device.
6138 		 */
6139 		spa_aux_vdev_t *sav = &spa->spa_l2cache;
6140 
6141 		if (sav->sav_count == 0) {
6142 			spa_config_exit(spa, SCL_STATE, FTAG);
6143 			(void) pthread_rwlock_unlock(&ztest_name_lock);
6144 			goto out;
6145 		}
6146 		vd0 = sav->sav_vdevs[ztest_random(sav->sav_count)];
6147 		guid0 = vd0->vdev_guid;
6148 		(void) strlcpy(path0, vd0->vdev_path, MAXPATHLEN);
6149 		(void) strlcpy(pathrand, vd0->vdev_path, MAXPATHLEN);
6150 
6151 		leaf = 0;
6152 		leaves = 1;
6153 		maxfaults = INT_MAX;	/* no limit on cache devices */
6154 	}
6155 
6156 	spa_config_exit(spa, SCL_STATE, FTAG);
6157 	(void) pthread_rwlock_unlock(&ztest_name_lock);
6158 
6159 	/*
6160 	 * If we can tolerate two or more faults, or we're dealing
6161 	 * with a slog, randomly online/offline vd0.
6162 	 */
6163 	if ((maxfaults >= 2 || islog) && guid0 != 0) {
6164 		if (ztest_random(10) < 6) {
6165 			int flags = (ztest_random(2) == 0 ?
6166 			    ZFS_OFFLINE_TEMPORARY : 0);
6167 
6168 			/*
6169 			 * We have to grab the zs_name_lock as writer to
6170 			 * prevent a race between offlining a slog and
6171 			 * destroying a dataset. Offlining the slog will
6172 			 * grab a reference on the dataset which may cause
6173 			 * dsl_destroy_head() to fail with EBUSY thus
6174 			 * leaving the dataset in an inconsistent state.
6175 			 */
6176 			if (islog)
6177 				(void) pthread_rwlock_wrlock(&ztest_name_lock);
6178 
6179 			VERIFY3U(vdev_offline(spa, guid0, flags), !=, EBUSY);
6180 
6181 			if (islog)
6182 				(void) pthread_rwlock_unlock(&ztest_name_lock);
6183 		} else {
6184 			/*
6185 			 * Ideally we would like to be able to randomly
6186 			 * call vdev_[on|off]line without holding locks
6187 			 * to force unpredictable failures but the side
6188 			 * effects of vdev_[on|off]line prevent us from
6189 			 * doing so. We grab the ztest_vdev_lock here to
6190 			 * prevent a race between injection testing and
6191 			 * aux_vdev removal.
6192 			 */
6193 			mutex_enter(&ztest_vdev_lock);
6194 			(void) vdev_online(spa, guid0, 0, NULL);
6195 			mutex_exit(&ztest_vdev_lock);
6196 		}
6197 	}
6198 
6199 	if (maxfaults == 0)
6200 		goto out;
6201 
6202 	/*
6203 	 * We have at least single-fault tolerance, so inject data corruption.
6204 	 */
6205 	fd = open(pathrand, O_RDWR);
6206 
6207 	if (fd == -1) /* we hit a gap in the device namespace */
6208 		goto out;
6209 
6210 	fsize = lseek(fd, 0, SEEK_END);
6211 
6212 	while (--iters != 0) {
6213 		/*
6214 		 * The offset must be chosen carefully to ensure that
6215 		 * we do not inject a given logical block with errors
6216 		 * on two different leaf devices, because ZFS can not
6217 		 * tolerate that (if maxfaults==1).
6218 		 *
6219 		 * To achieve this we divide each leaf device into
6220 		 * chunks of size (# leaves * SPA_MAXBLOCKSIZE * 4).
6221 		 * Each chunk is further divided into error-injection
6222 		 * ranges (can accept errors) and clear ranges (we do
6223 		 * not inject errors in those). Each error-injection
6224 		 * range can accept errors only for a single leaf vdev.
6225 		 * Error-injection ranges are separated by clear ranges.
6226 		 *
6227 		 * For example, with 3 leaves, each chunk looks like:
6228 		 *    0 to  32M: injection range for leaf 0
6229 		 *  32M to  64M: clear range - no injection allowed
6230 		 *  64M to  96M: injection range for leaf 1
6231 		 *  96M to 128M: clear range - no injection allowed
6232 		 * 128M to 160M: injection range for leaf 2
6233 		 * 160M to 192M: clear range - no injection allowed
6234 		 *
6235 		 * Each clear range must be large enough such that a
6236 		 * single block cannot straddle it. This way a block
6237 		 * can't be a target in two different injection ranges
6238 		 * (on different leaf vdevs).
6239 		 */
6240 		offset = ztest_random(fsize / (leaves << bshift)) *
6241 		    (leaves << bshift) + (leaf << bshift) +
6242 		    (ztest_random(1ULL << (bshift - 1)) & -8ULL);
6243 
6244 		/*
6245 		 * Only allow damage to the labels at one end of the vdev.
6246 		 *
6247 		 * If all labels are damaged, the device will be totally
6248 		 * inaccessible, which will result in loss of data,
6249 		 * because we also damage (parts of) the other side of
6250 		 * the mirror/raidz.
6251 		 *
6252 		 * Additionally, we will always have both an even and an
6253 		 * odd label, so that we can handle crashes in the
6254 		 * middle of vdev_config_sync().
6255 		 */
6256 		if ((leaf & 1) == 0 && offset < VDEV_LABEL_START_SIZE)
6257 			continue;
6258 
6259 		/*
6260 		 * The two end labels are stored at the "end" of the disk, but
6261 		 * the end of the disk (vdev_psize) is aligned to
6262 		 * sizeof (vdev_label_t).
6263 		 */
6264 		uint64_t psize = P2ALIGN(fsize, sizeof (vdev_label_t));
6265 		if ((leaf & 1) == 1 &&
6266 		    offset + sizeof (bad) > psize - VDEV_LABEL_END_SIZE)
6267 			continue;
6268 
6269 		mutex_enter(&ztest_vdev_lock);
6270 		if (mirror_save != zs->zs_mirrors) {
6271 			mutex_exit(&ztest_vdev_lock);
6272 			(void) close(fd);
6273 			goto out;
6274 		}
6275 
6276 		if (pwrite(fd, &bad, sizeof (bad), offset) != sizeof (bad))
6277 			fatal(B_TRUE,
6278 			    "can't inject bad word at 0x%"PRIx64" in %s",
6279 			    offset, pathrand);
6280 
6281 		mutex_exit(&ztest_vdev_lock);
6282 
6283 		if (ztest_opts.zo_verbose >= 7)
6284 			(void) printf("injected bad word into %s,"
6285 			    " offset 0x%"PRIx64"\n", pathrand, offset);
6286 	}
6287 
6288 	(void) close(fd);
6289 out:
6290 	umem_free(path0, MAXPATHLEN);
6291 	umem_free(pathrand, MAXPATHLEN);
6292 }
6293 
6294 /*
6295  * By design ztest will never inject uncorrectable damage in to the pool.
6296  * Issue a scrub, wait for it to complete, and verify there is never any
6297  * persistent damage.
6298  *
6299  * Only after a full scrub has been completed is it safe to start injecting
6300  * data corruption.  See the comment in zfs_fault_inject().
6301  */
6302 static int
6303 ztest_scrub_impl(spa_t *spa)
6304 {
6305 	int error = spa_scan(spa, POOL_SCAN_SCRUB);
6306 	if (error)
6307 		return (error);
6308 
6309 	while (dsl_scan_scrubbing(spa_get_dsl(spa)))
6310 		txg_wait_synced(spa_get_dsl(spa), 0);
6311 
6312 	if (spa_get_errlog_size(spa) > 0)
6313 		return (ECKSUM);
6314 
6315 	ztest_pool_scrubbed = B_TRUE;
6316 
6317 	return (0);
6318 }
6319 
6320 /*
6321  * Scrub the pool.
6322  */
6323 void
6324 ztest_scrub(ztest_ds_t *zd, uint64_t id)
6325 {
6326 	(void) zd, (void) id;
6327 	spa_t *spa = ztest_spa;
6328 	int error;
6329 
6330 	/*
6331 	 * Scrub in progress by device removal.
6332 	 */
6333 	if (ztest_device_removal_active)
6334 		return;
6335 
6336 	/*
6337 	 * Start a scrub, wait a moment, then force a restart.
6338 	 */
6339 	(void) spa_scan(spa, POOL_SCAN_SCRUB);
6340 	(void) poll(NULL, 0, 100);
6341 
6342 	error = ztest_scrub_impl(spa);
6343 	if (error == EBUSY)
6344 		error = 0;
6345 	ASSERT0(error);
6346 }
6347 
6348 /*
6349  * Change the guid for the pool.
6350  */
6351 void
6352 ztest_reguid(ztest_ds_t *zd, uint64_t id)
6353 {
6354 	(void) zd, (void) id;
6355 	spa_t *spa = ztest_spa;
6356 	uint64_t orig, load;
6357 	int error;
6358 
6359 	if (ztest_opts.zo_mmp_test)
6360 		return;
6361 
6362 	orig = spa_guid(spa);
6363 	load = spa_load_guid(spa);
6364 
6365 	(void) pthread_rwlock_wrlock(&ztest_name_lock);
6366 	error = spa_change_guid(spa);
6367 	(void) pthread_rwlock_unlock(&ztest_name_lock);
6368 
6369 	if (error != 0)
6370 		return;
6371 
6372 	if (ztest_opts.zo_verbose >= 4) {
6373 		(void) printf("Changed guid old %"PRIu64" -> %"PRIu64"\n",
6374 		    orig, spa_guid(spa));
6375 	}
6376 
6377 	VERIFY3U(orig, !=, spa_guid(spa));
6378 	VERIFY3U(load, ==, spa_load_guid(spa));
6379 }
6380 
6381 void
6382 ztest_blake3(ztest_ds_t *zd, uint64_t id)
6383 {
6384 	(void) zd, (void) id;
6385 	hrtime_t end = gethrtime() + NANOSEC;
6386 	zio_cksum_salt_t salt;
6387 	void *salt_ptr = &salt.zcs_bytes;
6388 	struct abd *abd_data, *abd_meta;
6389 	void *buf, *templ;
6390 	int i, *ptr;
6391 	uint32_t size;
6392 	BLAKE3_CTX ctx;
6393 
6394 	size = ztest_random_blocksize();
6395 	buf = umem_alloc(size, UMEM_NOFAIL);
6396 	abd_data = abd_alloc(size, B_FALSE);
6397 	abd_meta = abd_alloc(size, B_TRUE);
6398 
6399 	for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
6400 		*ptr = ztest_random(UINT_MAX);
6401 	memset(salt_ptr, 'A', 32);
6402 
6403 	abd_copy_from_buf_off(abd_data, buf, 0, size);
6404 	abd_copy_from_buf_off(abd_meta, buf, 0, size);
6405 
6406 	while (gethrtime() <= end) {
6407 		int run_count = 100;
6408 		zio_cksum_t zc_ref1, zc_ref2;
6409 		zio_cksum_t zc_res1, zc_res2;
6410 
6411 		void *ref1 = &zc_ref1;
6412 		void *ref2 = &zc_ref2;
6413 		void *res1 = &zc_res1;
6414 		void *res2 = &zc_res2;
6415 
6416 		/* BLAKE3_KEY_LEN = 32 */
6417 		VERIFY0(blake3_impl_setname("generic"));
6418 		templ = abd_checksum_blake3_tmpl_init(&salt);
6419 		Blake3_InitKeyed(&ctx, salt_ptr);
6420 		Blake3_Update(&ctx, buf, size);
6421 		Blake3_Final(&ctx, ref1);
6422 		zc_ref2 = zc_ref1;
6423 		ZIO_CHECKSUM_BSWAP(&zc_ref2);
6424 		abd_checksum_blake3_tmpl_free(templ);
6425 
6426 		VERIFY0(blake3_impl_setname("cycle"));
6427 		while (run_count-- > 0) {
6428 
6429 			/* Test current implementation */
6430 			Blake3_InitKeyed(&ctx, salt_ptr);
6431 			Blake3_Update(&ctx, buf, size);
6432 			Blake3_Final(&ctx, res1);
6433 			zc_res2 = zc_res1;
6434 			ZIO_CHECKSUM_BSWAP(&zc_res2);
6435 
6436 			VERIFY0(memcmp(ref1, res1, 32));
6437 			VERIFY0(memcmp(ref2, res2, 32));
6438 
6439 			/* Test ABD - data */
6440 			templ = abd_checksum_blake3_tmpl_init(&salt);
6441 			abd_checksum_blake3_native(abd_data, size,
6442 			    templ, &zc_res1);
6443 			abd_checksum_blake3_byteswap(abd_data, size,
6444 			    templ, &zc_res2);
6445 
6446 			VERIFY0(memcmp(ref1, res1, 32));
6447 			VERIFY0(memcmp(ref2, res2, 32));
6448 
6449 			/* Test ABD - metadata */
6450 			abd_checksum_blake3_native(abd_meta, size,
6451 			    templ, &zc_res1);
6452 			abd_checksum_blake3_byteswap(abd_meta, size,
6453 			    templ, &zc_res2);
6454 			abd_checksum_blake3_tmpl_free(templ);
6455 
6456 			VERIFY0(memcmp(ref1, res1, 32));
6457 			VERIFY0(memcmp(ref2, res2, 32));
6458 
6459 		}
6460 	}
6461 
6462 	abd_free(abd_data);
6463 	abd_free(abd_meta);
6464 	umem_free(buf, size);
6465 }
6466 
6467 void
6468 ztest_fletcher(ztest_ds_t *zd, uint64_t id)
6469 {
6470 	(void) zd, (void) id;
6471 	hrtime_t end = gethrtime() + NANOSEC;
6472 
6473 	while (gethrtime() <= end) {
6474 		int run_count = 100;
6475 		void *buf;
6476 		struct abd *abd_data, *abd_meta;
6477 		uint32_t size;
6478 		int *ptr;
6479 		int i;
6480 		zio_cksum_t zc_ref;
6481 		zio_cksum_t zc_ref_byteswap;
6482 
6483 		size = ztest_random_blocksize();
6484 
6485 		buf = umem_alloc(size, UMEM_NOFAIL);
6486 		abd_data = abd_alloc(size, B_FALSE);
6487 		abd_meta = abd_alloc(size, B_TRUE);
6488 
6489 		for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
6490 			*ptr = ztest_random(UINT_MAX);
6491 
6492 		abd_copy_from_buf_off(abd_data, buf, 0, size);
6493 		abd_copy_from_buf_off(abd_meta, buf, 0, size);
6494 
6495 		VERIFY0(fletcher_4_impl_set("scalar"));
6496 		fletcher_4_native(buf, size, NULL, &zc_ref);
6497 		fletcher_4_byteswap(buf, size, NULL, &zc_ref_byteswap);
6498 
6499 		VERIFY0(fletcher_4_impl_set("cycle"));
6500 		while (run_count-- > 0) {
6501 			zio_cksum_t zc;
6502 			zio_cksum_t zc_byteswap;
6503 
6504 			fletcher_4_byteswap(buf, size, NULL, &zc_byteswap);
6505 			fletcher_4_native(buf, size, NULL, &zc);
6506 
6507 			VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
6508 			VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
6509 			    sizeof (zc_byteswap)));
6510 
6511 			/* Test ABD - data */
6512 			abd_fletcher_4_byteswap(abd_data, size, NULL,
6513 			    &zc_byteswap);
6514 			abd_fletcher_4_native(abd_data, size, NULL, &zc);
6515 
6516 			VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
6517 			VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
6518 			    sizeof (zc_byteswap)));
6519 
6520 			/* Test ABD - metadata */
6521 			abd_fletcher_4_byteswap(abd_meta, size, NULL,
6522 			    &zc_byteswap);
6523 			abd_fletcher_4_native(abd_meta, size, NULL, &zc);
6524 
6525 			VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
6526 			VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
6527 			    sizeof (zc_byteswap)));
6528 
6529 		}
6530 
6531 		umem_free(buf, size);
6532 		abd_free(abd_data);
6533 		abd_free(abd_meta);
6534 	}
6535 }
6536 
6537 void
6538 ztest_fletcher_incr(ztest_ds_t *zd, uint64_t id)
6539 {
6540 	(void) zd, (void) id;
6541 	void *buf;
6542 	size_t size;
6543 	int *ptr;
6544 	int i;
6545 	zio_cksum_t zc_ref;
6546 	zio_cksum_t zc_ref_bswap;
6547 
6548 	hrtime_t end = gethrtime() + NANOSEC;
6549 
6550 	while (gethrtime() <= end) {
6551 		int run_count = 100;
6552 
6553 		size = ztest_random_blocksize();
6554 		buf = umem_alloc(size, UMEM_NOFAIL);
6555 
6556 		for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
6557 			*ptr = ztest_random(UINT_MAX);
6558 
6559 		VERIFY0(fletcher_4_impl_set("scalar"));
6560 		fletcher_4_native(buf, size, NULL, &zc_ref);
6561 		fletcher_4_byteswap(buf, size, NULL, &zc_ref_bswap);
6562 
6563 		VERIFY0(fletcher_4_impl_set("cycle"));
6564 
6565 		while (run_count-- > 0) {
6566 			zio_cksum_t zc;
6567 			zio_cksum_t zc_bswap;
6568 			size_t pos = 0;
6569 
6570 			ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
6571 			ZIO_SET_CHECKSUM(&zc_bswap, 0, 0, 0, 0);
6572 
6573 			while (pos < size) {
6574 				size_t inc = 64 * ztest_random(size / 67);
6575 				/* sometimes add few bytes to test non-simd */
6576 				if (ztest_random(100) < 10)
6577 					inc += P2ALIGN(ztest_random(64),
6578 					    sizeof (uint32_t));
6579 
6580 				if (inc > (size - pos))
6581 					inc = size - pos;
6582 
6583 				fletcher_4_incremental_native(buf + pos, inc,
6584 				    &zc);
6585 				fletcher_4_incremental_byteswap(buf + pos, inc,
6586 				    &zc_bswap);
6587 
6588 				pos += inc;
6589 			}
6590 
6591 			VERIFY3U(pos, ==, size);
6592 
6593 			VERIFY(ZIO_CHECKSUM_EQUAL(zc, zc_ref));
6594 			VERIFY(ZIO_CHECKSUM_EQUAL(zc_bswap, zc_ref_bswap));
6595 
6596 			/*
6597 			 * verify if incremental on the whole buffer is
6598 			 * equivalent to non-incremental version
6599 			 */
6600 			ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
6601 			ZIO_SET_CHECKSUM(&zc_bswap, 0, 0, 0, 0);
6602 
6603 			fletcher_4_incremental_native(buf, size, &zc);
6604 			fletcher_4_incremental_byteswap(buf, size, &zc_bswap);
6605 
6606 			VERIFY(ZIO_CHECKSUM_EQUAL(zc, zc_ref));
6607 			VERIFY(ZIO_CHECKSUM_EQUAL(zc_bswap, zc_ref_bswap));
6608 		}
6609 
6610 		umem_free(buf, size);
6611 	}
6612 }
6613 
6614 static int
6615 ztest_set_global_vars(void)
6616 {
6617 	for (size_t i = 0; i < ztest_opts.zo_gvars_count; i++) {
6618 		char *kv = ztest_opts.zo_gvars[i];
6619 		VERIFY3U(strlen(kv), <=, ZO_GVARS_MAX_ARGLEN);
6620 		VERIFY3U(strlen(kv), >, 0);
6621 		int err = set_global_var(kv);
6622 		if (ztest_opts.zo_verbose > 0) {
6623 			(void) printf("setting global var %s ... %s\n", kv,
6624 			    err ? "failed" : "ok");
6625 		}
6626 		if (err != 0) {
6627 			(void) fprintf(stderr,
6628 			    "failed to set global var '%s'\n", kv);
6629 			return (err);
6630 		}
6631 	}
6632 	return (0);
6633 }
6634 
6635 static char **
6636 ztest_global_vars_to_zdb_args(void)
6637 {
6638 	char **args = calloc(2*ztest_opts.zo_gvars_count + 1, sizeof (char *));
6639 	char **cur = args;
6640 	if (args == NULL)
6641 		return (NULL);
6642 	for (size_t i = 0; i < ztest_opts.zo_gvars_count; i++) {
6643 		*cur++ = (char *)"-o";
6644 		*cur++ = ztest_opts.zo_gvars[i];
6645 	}
6646 	ASSERT3P(cur, ==, &args[2*ztest_opts.zo_gvars_count]);
6647 	*cur = NULL;
6648 	return (args);
6649 }
6650 
6651 /* The end of strings is indicated by a NULL element */
6652 static char *
6653 join_strings(char **strings, const char *sep)
6654 {
6655 	size_t totallen = 0;
6656 	for (char **sp = strings; *sp != NULL; sp++) {
6657 		totallen += strlen(*sp);
6658 		totallen += strlen(sep);
6659 	}
6660 	if (totallen > 0) {
6661 		ASSERT(totallen >= strlen(sep));
6662 		totallen -= strlen(sep);
6663 	}
6664 
6665 	size_t buflen = totallen + 1;
6666 	char *o = malloc(buflen); /* trailing 0 byte */
6667 	o[0] = '\0';
6668 	for (char **sp = strings; *sp != NULL; sp++) {
6669 		size_t would;
6670 		would = strlcat(o, *sp, buflen);
6671 		VERIFY3U(would, <, buflen);
6672 		if (*(sp+1) == NULL) {
6673 			break;
6674 		}
6675 		would = strlcat(o, sep, buflen);
6676 		VERIFY3U(would, <, buflen);
6677 	}
6678 	ASSERT3S(strlen(o), ==, totallen);
6679 	return (o);
6680 }
6681 
6682 static int
6683 ztest_check_path(char *path)
6684 {
6685 	struct stat s;
6686 	/* return true on success */
6687 	return (!stat(path, &s));
6688 }
6689 
6690 static void
6691 ztest_get_zdb_bin(char *bin, int len)
6692 {
6693 	char *zdb_path;
6694 	/*
6695 	 * Try to use $ZDB and in-tree zdb path. If not successful, just
6696 	 * let popen to search through PATH.
6697 	 */
6698 	if ((zdb_path = getenv("ZDB"))) {
6699 		strlcpy(bin, zdb_path, len); /* In env */
6700 		if (!ztest_check_path(bin)) {
6701 			ztest_dump_core = 0;
6702 			fatal(B_TRUE, "invalid ZDB '%s'", bin);
6703 		}
6704 		return;
6705 	}
6706 
6707 	VERIFY3P(realpath(getexecname(), bin), !=, NULL);
6708 	if (strstr(bin, ".libs/ztest")) {
6709 		strstr(bin, ".libs/ztest")[0] = '\0'; /* In-tree */
6710 		strcat(bin, "zdb");
6711 		if (ztest_check_path(bin))
6712 			return;
6713 	}
6714 	strcpy(bin, "zdb");
6715 }
6716 
6717 static vdev_t *
6718 ztest_random_concrete_vdev_leaf(vdev_t *vd)
6719 {
6720 	if (vd == NULL)
6721 		return (NULL);
6722 
6723 	if (vd->vdev_children == 0)
6724 		return (vd);
6725 
6726 	vdev_t *eligible[vd->vdev_children];
6727 	int eligible_idx = 0, i;
6728 	for (i = 0; i < vd->vdev_children; i++) {
6729 		vdev_t *cvd = vd->vdev_child[i];
6730 		if (cvd->vdev_top->vdev_removing)
6731 			continue;
6732 		if (cvd->vdev_children > 0 ||
6733 		    (vdev_is_concrete(cvd) && !cvd->vdev_detached)) {
6734 			eligible[eligible_idx++] = cvd;
6735 		}
6736 	}
6737 	VERIFY3S(eligible_idx, >, 0);
6738 
6739 	uint64_t child_no = ztest_random(eligible_idx);
6740 	return (ztest_random_concrete_vdev_leaf(eligible[child_no]));
6741 }
6742 
6743 void
6744 ztest_initialize(ztest_ds_t *zd, uint64_t id)
6745 {
6746 	(void) zd, (void) id;
6747 	spa_t *spa = ztest_spa;
6748 	int error = 0;
6749 
6750 	mutex_enter(&ztest_vdev_lock);
6751 
6752 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
6753 
6754 	/* Random leaf vdev */
6755 	vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
6756 	if (rand_vd == NULL) {
6757 		spa_config_exit(spa, SCL_VDEV, FTAG);
6758 		mutex_exit(&ztest_vdev_lock);
6759 		return;
6760 	}
6761 
6762 	/*
6763 	 * The random vdev we've selected may change as soon as we
6764 	 * drop the spa_config_lock. We create local copies of things
6765 	 * we're interested in.
6766 	 */
6767 	uint64_t guid = rand_vd->vdev_guid;
6768 	char *path = strdup(rand_vd->vdev_path);
6769 	boolean_t active = rand_vd->vdev_initialize_thread != NULL;
6770 
6771 	zfs_dbgmsg("vd %px, guid %llu", rand_vd, (u_longlong_t)guid);
6772 	spa_config_exit(spa, SCL_VDEV, FTAG);
6773 
6774 	uint64_t cmd = ztest_random(POOL_INITIALIZE_FUNCS);
6775 
6776 	nvlist_t *vdev_guids = fnvlist_alloc();
6777 	nvlist_t *vdev_errlist = fnvlist_alloc();
6778 	fnvlist_add_uint64(vdev_guids, path, guid);
6779 	error = spa_vdev_initialize(spa, vdev_guids, cmd, vdev_errlist);
6780 	fnvlist_free(vdev_guids);
6781 	fnvlist_free(vdev_errlist);
6782 
6783 	switch (cmd) {
6784 	case POOL_INITIALIZE_CANCEL:
6785 		if (ztest_opts.zo_verbose >= 4) {
6786 			(void) printf("Cancel initialize %s", path);
6787 			if (!active)
6788 				(void) printf(" failed (no initialize active)");
6789 			(void) printf("\n");
6790 		}
6791 		break;
6792 	case POOL_INITIALIZE_START:
6793 		if (ztest_opts.zo_verbose >= 4) {
6794 			(void) printf("Start initialize %s", path);
6795 			if (active && error == 0)
6796 				(void) printf(" failed (already active)");
6797 			else if (error != 0)
6798 				(void) printf(" failed (error %d)", error);
6799 			(void) printf("\n");
6800 		}
6801 		break;
6802 	case POOL_INITIALIZE_SUSPEND:
6803 		if (ztest_opts.zo_verbose >= 4) {
6804 			(void) printf("Suspend initialize %s", path);
6805 			if (!active)
6806 				(void) printf(" failed (no initialize active)");
6807 			(void) printf("\n");
6808 		}
6809 		break;
6810 	}
6811 	free(path);
6812 	mutex_exit(&ztest_vdev_lock);
6813 }
6814 
6815 void
6816 ztest_trim(ztest_ds_t *zd, uint64_t id)
6817 {
6818 	(void) zd, (void) id;
6819 	spa_t *spa = ztest_spa;
6820 	int error = 0;
6821 
6822 	mutex_enter(&ztest_vdev_lock);
6823 
6824 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
6825 
6826 	/* Random leaf vdev */
6827 	vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
6828 	if (rand_vd == NULL) {
6829 		spa_config_exit(spa, SCL_VDEV, FTAG);
6830 		mutex_exit(&ztest_vdev_lock);
6831 		return;
6832 	}
6833 
6834 	/*
6835 	 * The random vdev we've selected may change as soon as we
6836 	 * drop the spa_config_lock. We create local copies of things
6837 	 * we're interested in.
6838 	 */
6839 	uint64_t guid = rand_vd->vdev_guid;
6840 	char *path = strdup(rand_vd->vdev_path);
6841 	boolean_t active = rand_vd->vdev_trim_thread != NULL;
6842 
6843 	zfs_dbgmsg("vd %p, guid %llu", rand_vd, (u_longlong_t)guid);
6844 	spa_config_exit(spa, SCL_VDEV, FTAG);
6845 
6846 	uint64_t cmd = ztest_random(POOL_TRIM_FUNCS);
6847 	uint64_t rate = 1 << ztest_random(30);
6848 	boolean_t partial = (ztest_random(5) > 0);
6849 	boolean_t secure = (ztest_random(5) > 0);
6850 
6851 	nvlist_t *vdev_guids = fnvlist_alloc();
6852 	nvlist_t *vdev_errlist = fnvlist_alloc();
6853 	fnvlist_add_uint64(vdev_guids, path, guid);
6854 	error = spa_vdev_trim(spa, vdev_guids, cmd, rate, partial,
6855 	    secure, vdev_errlist);
6856 	fnvlist_free(vdev_guids);
6857 	fnvlist_free(vdev_errlist);
6858 
6859 	switch (cmd) {
6860 	case POOL_TRIM_CANCEL:
6861 		if (ztest_opts.zo_verbose >= 4) {
6862 			(void) printf("Cancel TRIM %s", path);
6863 			if (!active)
6864 				(void) printf(" failed (no TRIM active)");
6865 			(void) printf("\n");
6866 		}
6867 		break;
6868 	case POOL_TRIM_START:
6869 		if (ztest_opts.zo_verbose >= 4) {
6870 			(void) printf("Start TRIM %s", path);
6871 			if (active && error == 0)
6872 				(void) printf(" failed (already active)");
6873 			else if (error != 0)
6874 				(void) printf(" failed (error %d)", error);
6875 			(void) printf("\n");
6876 		}
6877 		break;
6878 	case POOL_TRIM_SUSPEND:
6879 		if (ztest_opts.zo_verbose >= 4) {
6880 			(void) printf("Suspend TRIM %s", path);
6881 			if (!active)
6882 				(void) printf(" failed (no TRIM active)");
6883 			(void) printf("\n");
6884 		}
6885 		break;
6886 	}
6887 	free(path);
6888 	mutex_exit(&ztest_vdev_lock);
6889 }
6890 
6891 /*
6892  * Verify pool integrity by running zdb.
6893  */
6894 static void
6895 ztest_run_zdb(const char *pool)
6896 {
6897 	int status;
6898 	char *bin;
6899 	char *zdb;
6900 	char *zbuf;
6901 	const int len = MAXPATHLEN + MAXNAMELEN + 20;
6902 	FILE *fp;
6903 
6904 	bin = umem_alloc(len, UMEM_NOFAIL);
6905 	zdb = umem_alloc(len, UMEM_NOFAIL);
6906 	zbuf = umem_alloc(1024, UMEM_NOFAIL);
6907 
6908 	ztest_get_zdb_bin(bin, len);
6909 
6910 	char **set_gvars_args = ztest_global_vars_to_zdb_args();
6911 	if (set_gvars_args == NULL) {
6912 		fatal(B_FALSE, "Failed to allocate memory in "
6913 		    "ztest_global_vars_to_zdb_args(). Cannot run zdb.\n");
6914 	}
6915 	char *set_gvars_args_joined = join_strings(set_gvars_args, " ");
6916 	free(set_gvars_args);
6917 
6918 	size_t would = snprintf(zdb, len,
6919 	    "%s -bcc%s%s -G -d -Y -e -y %s -p %s %s",
6920 	    bin,
6921 	    ztest_opts.zo_verbose >= 3 ? "s" : "",
6922 	    ztest_opts.zo_verbose >= 4 ? "v" : "",
6923 	    set_gvars_args_joined,
6924 	    ztest_opts.zo_dir,
6925 	    pool);
6926 	ASSERT3U(would, <, len);
6927 
6928 	free(set_gvars_args_joined);
6929 
6930 	if (ztest_opts.zo_verbose >= 5)
6931 		(void) printf("Executing %s\n", zdb);
6932 
6933 	fp = popen(zdb, "r");
6934 
6935 	while (fgets(zbuf, 1024, fp) != NULL)
6936 		if (ztest_opts.zo_verbose >= 3)
6937 			(void) printf("%s", zbuf);
6938 
6939 	status = pclose(fp);
6940 
6941 	if (status == 0)
6942 		goto out;
6943 
6944 	ztest_dump_core = 0;
6945 	if (WIFEXITED(status))
6946 		fatal(B_FALSE, "'%s' exit code %d", zdb, WEXITSTATUS(status));
6947 	else
6948 		fatal(B_FALSE, "'%s' died with signal %d",
6949 		    zdb, WTERMSIG(status));
6950 out:
6951 	umem_free(bin, len);
6952 	umem_free(zdb, len);
6953 	umem_free(zbuf, 1024);
6954 }
6955 
6956 static void
6957 ztest_walk_pool_directory(const char *header)
6958 {
6959 	spa_t *spa = NULL;
6960 
6961 	if (ztest_opts.zo_verbose >= 6)
6962 		(void) puts(header);
6963 
6964 	mutex_enter(&spa_namespace_lock);
6965 	while ((spa = spa_next(spa)) != NULL)
6966 		if (ztest_opts.zo_verbose >= 6)
6967 			(void) printf("\t%s\n", spa_name(spa));
6968 	mutex_exit(&spa_namespace_lock);
6969 }
6970 
6971 static void
6972 ztest_spa_import_export(char *oldname, char *newname)
6973 {
6974 	nvlist_t *config, *newconfig;
6975 	uint64_t pool_guid;
6976 	spa_t *spa;
6977 	int error;
6978 
6979 	if (ztest_opts.zo_verbose >= 4) {
6980 		(void) printf("import/export: old = %s, new = %s\n",
6981 		    oldname, newname);
6982 	}
6983 
6984 	/*
6985 	 * Clean up from previous runs.
6986 	 */
6987 	(void) spa_destroy(newname);
6988 
6989 	/*
6990 	 * Get the pool's configuration and guid.
6991 	 */
6992 	VERIFY0(spa_open(oldname, &spa, FTAG));
6993 
6994 	/*
6995 	 * Kick off a scrub to tickle scrub/export races.
6996 	 */
6997 	if (ztest_random(2) == 0)
6998 		(void) spa_scan(spa, POOL_SCAN_SCRUB);
6999 
7000 	pool_guid = spa_guid(spa);
7001 	spa_close(spa, FTAG);
7002 
7003 	ztest_walk_pool_directory("pools before export");
7004 
7005 	/*
7006 	 * Export it.
7007 	 */
7008 	VERIFY0(spa_export(oldname, &config, B_FALSE, B_FALSE));
7009 
7010 	ztest_walk_pool_directory("pools after export");
7011 
7012 	/*
7013 	 * Try to import it.
7014 	 */
7015 	newconfig = spa_tryimport(config);
7016 	ASSERT3P(newconfig, !=, NULL);
7017 	fnvlist_free(newconfig);
7018 
7019 	/*
7020 	 * Import it under the new name.
7021 	 */
7022 	error = spa_import(newname, config, NULL, 0);
7023 	if (error != 0) {
7024 		dump_nvlist(config, 0);
7025 		fatal(B_FALSE, "couldn't import pool %s as %s: error %u",
7026 		    oldname, newname, error);
7027 	}
7028 
7029 	ztest_walk_pool_directory("pools after import");
7030 
7031 	/*
7032 	 * Try to import it again -- should fail with EEXIST.
7033 	 */
7034 	VERIFY3U(EEXIST, ==, spa_import(newname, config, NULL, 0));
7035 
7036 	/*
7037 	 * Try to import it under a different name -- should fail with EEXIST.
7038 	 */
7039 	VERIFY3U(EEXIST, ==, spa_import(oldname, config, NULL, 0));
7040 
7041 	/*
7042 	 * Verify that the pool is no longer visible under the old name.
7043 	 */
7044 	VERIFY3U(ENOENT, ==, spa_open(oldname, &spa, FTAG));
7045 
7046 	/*
7047 	 * Verify that we can open and close the pool using the new name.
7048 	 */
7049 	VERIFY0(spa_open(newname, &spa, FTAG));
7050 	ASSERT3U(pool_guid, ==, spa_guid(spa));
7051 	spa_close(spa, FTAG);
7052 
7053 	fnvlist_free(config);
7054 }
7055 
7056 static void
7057 ztest_resume(spa_t *spa)
7058 {
7059 	if (spa_suspended(spa) && ztest_opts.zo_verbose >= 6)
7060 		(void) printf("resuming from suspended state\n");
7061 	spa_vdev_state_enter(spa, SCL_NONE);
7062 	vdev_clear(spa, NULL);
7063 	(void) spa_vdev_state_exit(spa, NULL, 0);
7064 	(void) zio_resume(spa);
7065 }
7066 
7067 static __attribute__((noreturn)) void
7068 ztest_resume_thread(void *arg)
7069 {
7070 	spa_t *spa = arg;
7071 
7072 	while (!ztest_exiting) {
7073 		if (spa_suspended(spa))
7074 			ztest_resume(spa);
7075 		(void) poll(NULL, 0, 100);
7076 
7077 		/*
7078 		 * Periodically change the zfs_compressed_arc_enabled setting.
7079 		 */
7080 		if (ztest_random(10) == 0)
7081 			zfs_compressed_arc_enabled = ztest_random(2);
7082 
7083 		/*
7084 		 * Periodically change the zfs_abd_scatter_enabled setting.
7085 		 */
7086 		if (ztest_random(10) == 0)
7087 			zfs_abd_scatter_enabled = ztest_random(2);
7088 	}
7089 
7090 	thread_exit();
7091 }
7092 
7093 static __attribute__((noreturn)) void
7094 ztest_deadman_thread(void *arg)
7095 {
7096 	ztest_shared_t *zs = arg;
7097 	spa_t *spa = ztest_spa;
7098 	hrtime_t delay, overdue, last_run = gethrtime();
7099 
7100 	delay = (zs->zs_thread_stop - zs->zs_thread_start) +
7101 	    MSEC2NSEC(zfs_deadman_synctime_ms);
7102 
7103 	while (!ztest_exiting) {
7104 		/*
7105 		 * Wait for the delay timer while checking occasionally
7106 		 * if we should stop.
7107 		 */
7108 		if (gethrtime() < last_run + delay) {
7109 			(void) poll(NULL, 0, 1000);
7110 			continue;
7111 		}
7112 
7113 		/*
7114 		 * If the pool is suspended then fail immediately. Otherwise,
7115 		 * check to see if the pool is making any progress. If
7116 		 * vdev_deadman() discovers that there hasn't been any recent
7117 		 * I/Os then it will end up aborting the tests.
7118 		 */
7119 		if (spa_suspended(spa) || spa->spa_root_vdev == NULL) {
7120 			fatal(B_FALSE,
7121 			    "aborting test after %lu seconds because "
7122 			    "pool has transitioned to a suspended state.",
7123 			    zfs_deadman_synctime_ms / 1000);
7124 		}
7125 		vdev_deadman(spa->spa_root_vdev, FTAG);
7126 
7127 		/*
7128 		 * If the process doesn't complete within a grace period of
7129 		 * zfs_deadman_synctime_ms over the expected finish time,
7130 		 * then it may be hung and is terminated.
7131 		 */
7132 		overdue = zs->zs_proc_stop + MSEC2NSEC(zfs_deadman_synctime_ms);
7133 		if (gethrtime() > overdue) {
7134 			fatal(B_FALSE,
7135 			    "aborting test after %llu seconds because "
7136 			    "the process is overdue for termination.",
7137 			    (gethrtime() - zs->zs_proc_start) / NANOSEC);
7138 		}
7139 
7140 		(void) printf("ztest has been running for %lld seconds\n",
7141 		    (gethrtime() - zs->zs_proc_start) / NANOSEC);
7142 
7143 		last_run = gethrtime();
7144 		delay = MSEC2NSEC(zfs_deadman_checktime_ms);
7145 	}
7146 
7147 	thread_exit();
7148 }
7149 
7150 static void
7151 ztest_execute(int test, ztest_info_t *zi, uint64_t id)
7152 {
7153 	ztest_ds_t *zd = &ztest_ds[id % ztest_opts.zo_datasets];
7154 	ztest_shared_callstate_t *zc = ZTEST_GET_SHARED_CALLSTATE(test);
7155 	hrtime_t functime = gethrtime();
7156 	int i;
7157 
7158 	for (i = 0; i < zi->zi_iters; i++)
7159 		zi->zi_func(zd, id);
7160 
7161 	functime = gethrtime() - functime;
7162 
7163 	atomic_add_64(&zc->zc_count, 1);
7164 	atomic_add_64(&zc->zc_time, functime);
7165 
7166 	if (ztest_opts.zo_verbose >= 4)
7167 		(void) printf("%6.2f sec in %s\n",
7168 		    (double)functime / NANOSEC, zi->zi_funcname);
7169 }
7170 
7171 static __attribute__((noreturn)) void
7172 ztest_thread(void *arg)
7173 {
7174 	int rand;
7175 	uint64_t id = (uintptr_t)arg;
7176 	ztest_shared_t *zs = ztest_shared;
7177 	uint64_t call_next;
7178 	hrtime_t now;
7179 	ztest_info_t *zi;
7180 	ztest_shared_callstate_t *zc;
7181 
7182 	while ((now = gethrtime()) < zs->zs_thread_stop) {
7183 		/*
7184 		 * See if it's time to force a crash.
7185 		 */
7186 		if (now > zs->zs_thread_kill)
7187 			ztest_kill(zs);
7188 
7189 		/*
7190 		 * If we're getting ENOSPC with some regularity, stop.
7191 		 */
7192 		if (zs->zs_enospc_count > 10)
7193 			break;
7194 
7195 		/*
7196 		 * Pick a random function to execute.
7197 		 */
7198 		rand = ztest_random(ZTEST_FUNCS);
7199 		zi = &ztest_info[rand];
7200 		zc = ZTEST_GET_SHARED_CALLSTATE(rand);
7201 		call_next = zc->zc_next;
7202 
7203 		if (now >= call_next &&
7204 		    atomic_cas_64(&zc->zc_next, call_next, call_next +
7205 		    ztest_random(2 * zi->zi_interval[0] + 1)) == call_next) {
7206 			ztest_execute(rand, zi, id);
7207 		}
7208 	}
7209 
7210 	thread_exit();
7211 }
7212 
7213 static void
7214 ztest_dataset_name(char *dsname, const char *pool, int d)
7215 {
7216 	(void) snprintf(dsname, ZFS_MAX_DATASET_NAME_LEN, "%s/ds_%d", pool, d);
7217 }
7218 
7219 static void
7220 ztest_dataset_destroy(int d)
7221 {
7222 	char name[ZFS_MAX_DATASET_NAME_LEN];
7223 	int t;
7224 
7225 	ztest_dataset_name(name, ztest_opts.zo_pool, d);
7226 
7227 	if (ztest_opts.zo_verbose >= 3)
7228 		(void) printf("Destroying %s to free up space\n", name);
7229 
7230 	/*
7231 	 * Cleanup any non-standard clones and snapshots.  In general,
7232 	 * ztest thread t operates on dataset (t % zopt_datasets),
7233 	 * so there may be more than one thing to clean up.
7234 	 */
7235 	for (t = d; t < ztest_opts.zo_threads;
7236 	    t += ztest_opts.zo_datasets)
7237 		ztest_dsl_dataset_cleanup(name, t);
7238 
7239 	(void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
7240 	    DS_FIND_SNAPSHOTS | DS_FIND_CHILDREN);
7241 }
7242 
7243 static void
7244 ztest_dataset_dirobj_verify(ztest_ds_t *zd)
7245 {
7246 	uint64_t usedobjs, dirobjs, scratch;
7247 
7248 	/*
7249 	 * ZTEST_DIROBJ is the object directory for the entire dataset.
7250 	 * Therefore, the number of objects in use should equal the
7251 	 * number of ZTEST_DIROBJ entries, +1 for ZTEST_DIROBJ itself.
7252 	 * If not, we have an object leak.
7253 	 *
7254 	 * Note that we can only check this in ztest_dataset_open(),
7255 	 * when the open-context and syncing-context values agree.
7256 	 * That's because zap_count() returns the open-context value,
7257 	 * while dmu_objset_space() returns the rootbp fill count.
7258 	 */
7259 	VERIFY0(zap_count(zd->zd_os, ZTEST_DIROBJ, &dirobjs));
7260 	dmu_objset_space(zd->zd_os, &scratch, &scratch, &usedobjs, &scratch);
7261 	ASSERT3U(dirobjs + 1, ==, usedobjs);
7262 }
7263 
7264 static int
7265 ztest_dataset_open(int d)
7266 {
7267 	ztest_ds_t *zd = &ztest_ds[d];
7268 	uint64_t committed_seq = ZTEST_GET_SHARED_DS(d)->zd_seq;
7269 	objset_t *os;
7270 	zilog_t *zilog;
7271 	char name[ZFS_MAX_DATASET_NAME_LEN];
7272 	int error;
7273 
7274 	ztest_dataset_name(name, ztest_opts.zo_pool, d);
7275 
7276 	(void) pthread_rwlock_rdlock(&ztest_name_lock);
7277 
7278 	error = ztest_dataset_create(name);
7279 	if (error == ENOSPC) {
7280 		(void) pthread_rwlock_unlock(&ztest_name_lock);
7281 		ztest_record_enospc(FTAG);
7282 		return (error);
7283 	}
7284 	ASSERT(error == 0 || error == EEXIST);
7285 
7286 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
7287 	    B_TRUE, zd, &os));
7288 	(void) pthread_rwlock_unlock(&ztest_name_lock);
7289 
7290 	ztest_zd_init(zd, ZTEST_GET_SHARED_DS(d), os);
7291 
7292 	zilog = zd->zd_zilog;
7293 
7294 	if (zilog->zl_header->zh_claim_lr_seq != 0 &&
7295 	    zilog->zl_header->zh_claim_lr_seq < committed_seq)
7296 		fatal(B_FALSE, "missing log records: "
7297 		    "claimed %"PRIu64" < committed %"PRIu64"",
7298 		    zilog->zl_header->zh_claim_lr_seq, committed_seq);
7299 
7300 	ztest_dataset_dirobj_verify(zd);
7301 
7302 	zil_replay(os, zd, ztest_replay_vector);
7303 
7304 	ztest_dataset_dirobj_verify(zd);
7305 
7306 	if (ztest_opts.zo_verbose >= 6)
7307 		(void) printf("%s replay %"PRIu64" blocks, "
7308 		    "%"PRIu64" records, seq %"PRIu64"\n",
7309 		    zd->zd_name,
7310 		    zilog->zl_parse_blk_count,
7311 		    zilog->zl_parse_lr_count,
7312 		    zilog->zl_replaying_seq);
7313 
7314 	zilog = zil_open(os, ztest_get_data, NULL);
7315 
7316 	if (zilog->zl_replaying_seq != 0 &&
7317 	    zilog->zl_replaying_seq < committed_seq)
7318 		fatal(B_FALSE, "missing log records: "
7319 		    "replayed %"PRIu64" < committed %"PRIu64"",
7320 		    zilog->zl_replaying_seq, committed_seq);
7321 
7322 	return (0);
7323 }
7324 
7325 static void
7326 ztest_dataset_close(int d)
7327 {
7328 	ztest_ds_t *zd = &ztest_ds[d];
7329 
7330 	zil_close(zd->zd_zilog);
7331 	dmu_objset_disown(zd->zd_os, B_TRUE, zd);
7332 
7333 	ztest_zd_fini(zd);
7334 }
7335 
7336 static int
7337 ztest_replay_zil_cb(const char *name, void *arg)
7338 {
7339 	(void) arg;
7340 	objset_t *os;
7341 	ztest_ds_t *zdtmp;
7342 
7343 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_ANY, B_TRUE,
7344 	    B_TRUE, FTAG, &os));
7345 
7346 	zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);
7347 
7348 	ztest_zd_init(zdtmp, NULL, os);
7349 	zil_replay(os, zdtmp, ztest_replay_vector);
7350 	ztest_zd_fini(zdtmp);
7351 
7352 	if (dmu_objset_zil(os)->zl_parse_lr_count != 0 &&
7353 	    ztest_opts.zo_verbose >= 6) {
7354 		zilog_t *zilog = dmu_objset_zil(os);
7355 
7356 		(void) printf("%s replay %"PRIu64" blocks, "
7357 		    "%"PRIu64" records, seq %"PRIu64"\n",
7358 		    name,
7359 		    zilog->zl_parse_blk_count,
7360 		    zilog->zl_parse_lr_count,
7361 		    zilog->zl_replaying_seq);
7362 	}
7363 
7364 	umem_free(zdtmp, sizeof (ztest_ds_t));
7365 
7366 	dmu_objset_disown(os, B_TRUE, FTAG);
7367 	return (0);
7368 }
7369 
7370 static void
7371 ztest_freeze(void)
7372 {
7373 	ztest_ds_t *zd = &ztest_ds[0];
7374 	spa_t *spa;
7375 	int numloops = 0;
7376 
7377 	if (ztest_opts.zo_verbose >= 3)
7378 		(void) printf("testing spa_freeze()...\n");
7379 
7380 	kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7381 	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7382 	VERIFY0(ztest_dataset_open(0));
7383 	ztest_spa = spa;
7384 
7385 	/*
7386 	 * Force the first log block to be transactionally allocated.
7387 	 * We have to do this before we freeze the pool -- otherwise
7388 	 * the log chain won't be anchored.
7389 	 */
7390 	while (BP_IS_HOLE(&zd->zd_zilog->zl_header->zh_log)) {
7391 		ztest_dmu_object_alloc_free(zd, 0);
7392 		zil_commit(zd->zd_zilog, 0);
7393 	}
7394 
7395 	txg_wait_synced(spa_get_dsl(spa), 0);
7396 
7397 	/*
7398 	 * Freeze the pool.  This stops spa_sync() from doing anything,
7399 	 * so that the only way to record changes from now on is the ZIL.
7400 	 */
7401 	spa_freeze(spa);
7402 
7403 	/*
7404 	 * Because it is hard to predict how much space a write will actually
7405 	 * require beforehand, we leave ourselves some fudge space to write over
7406 	 * capacity.
7407 	 */
7408 	uint64_t capacity = metaslab_class_get_space(spa_normal_class(spa)) / 2;
7409 
7410 	/*
7411 	 * Run tests that generate log records but don't alter the pool config
7412 	 * or depend on DSL sync tasks (snapshots, objset create/destroy, etc).
7413 	 * We do a txg_wait_synced() after each iteration to force the txg
7414 	 * to increase well beyond the last synced value in the uberblock.
7415 	 * The ZIL should be OK with that.
7416 	 *
7417 	 * Run a random number of times less than zo_maxloops and ensure we do
7418 	 * not run out of space on the pool.
7419 	 */
7420 	while (ztest_random(10) != 0 &&
7421 	    numloops++ < ztest_opts.zo_maxloops &&
7422 	    metaslab_class_get_alloc(spa_normal_class(spa)) < capacity) {
7423 		ztest_od_t od;
7424 		ztest_od_init(&od, 0, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
7425 		VERIFY0(ztest_object_init(zd, &od, sizeof (od), B_FALSE));
7426 		ztest_io(zd, od.od_object,
7427 		    ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
7428 		txg_wait_synced(spa_get_dsl(spa), 0);
7429 	}
7430 
7431 	/*
7432 	 * Commit all of the changes we just generated.
7433 	 */
7434 	zil_commit(zd->zd_zilog, 0);
7435 	txg_wait_synced(spa_get_dsl(spa), 0);
7436 
7437 	/*
7438 	 * Close our dataset and close the pool.
7439 	 */
7440 	ztest_dataset_close(0);
7441 	spa_close(spa, FTAG);
7442 	kernel_fini();
7443 
7444 	/*
7445 	 * Open and close the pool and dataset to induce log replay.
7446 	 */
7447 	kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7448 	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7449 	ASSERT3U(spa_freeze_txg(spa), ==, UINT64_MAX);
7450 	VERIFY0(ztest_dataset_open(0));
7451 	ztest_spa = spa;
7452 	txg_wait_synced(spa_get_dsl(spa), 0);
7453 	ztest_dataset_close(0);
7454 	ztest_reguid(NULL, 0);
7455 
7456 	spa_close(spa, FTAG);
7457 	kernel_fini();
7458 }
7459 
7460 static void
7461 ztest_import_impl(void)
7462 {
7463 	importargs_t args = { 0 };
7464 	nvlist_t *cfg = NULL;
7465 	int nsearch = 1;
7466 	char *searchdirs[nsearch];
7467 	int flags = ZFS_IMPORT_MISSING_LOG;
7468 
7469 	searchdirs[0] = ztest_opts.zo_dir;
7470 	args.paths = nsearch;
7471 	args.path = searchdirs;
7472 	args.can_be_active = B_FALSE;
7473 
7474 	VERIFY0(zpool_find_config(NULL, ztest_opts.zo_pool, &cfg, &args,
7475 	    &libzpool_config_ops));
7476 	VERIFY0(spa_import(ztest_opts.zo_pool, cfg, NULL, flags));
7477 	fnvlist_free(cfg);
7478 }
7479 
7480 /*
7481  * Import a storage pool with the given name.
7482  */
7483 static void
7484 ztest_import(ztest_shared_t *zs)
7485 {
7486 	spa_t *spa;
7487 
7488 	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
7489 	mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
7490 	VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
7491 
7492 	kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7493 
7494 	ztest_import_impl();
7495 
7496 	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7497 	zs->zs_metaslab_sz =
7498 	    1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
7499 	spa_close(spa, FTAG);
7500 
7501 	kernel_fini();
7502 
7503 	if (!ztest_opts.zo_mmp_test) {
7504 		ztest_run_zdb(ztest_opts.zo_pool);
7505 		ztest_freeze();
7506 		ztest_run_zdb(ztest_opts.zo_pool);
7507 	}
7508 
7509 	(void) pthread_rwlock_destroy(&ztest_name_lock);
7510 	mutex_destroy(&ztest_vdev_lock);
7511 	mutex_destroy(&ztest_checkpoint_lock);
7512 }
7513 
7514 /*
7515  * Kick off threads to run tests on all datasets in parallel.
7516  */
7517 static void
7518 ztest_run(ztest_shared_t *zs)
7519 {
7520 	spa_t *spa;
7521 	objset_t *os;
7522 	kthread_t *resume_thread, *deadman_thread;
7523 	kthread_t **run_threads;
7524 	uint64_t object;
7525 	int error;
7526 	int t, d;
7527 
7528 	ztest_exiting = B_FALSE;
7529 
7530 	/*
7531 	 * Initialize parent/child shared state.
7532 	 */
7533 	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
7534 	mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
7535 	VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
7536 
7537 	zs->zs_thread_start = gethrtime();
7538 	zs->zs_thread_stop =
7539 	    zs->zs_thread_start + ztest_opts.zo_passtime * NANOSEC;
7540 	zs->zs_thread_stop = MIN(zs->zs_thread_stop, zs->zs_proc_stop);
7541 	zs->zs_thread_kill = zs->zs_thread_stop;
7542 	if (ztest_random(100) < ztest_opts.zo_killrate) {
7543 		zs->zs_thread_kill -=
7544 		    ztest_random(ztest_opts.zo_passtime * NANOSEC);
7545 	}
7546 
7547 	mutex_init(&zcl.zcl_callbacks_lock, NULL, MUTEX_DEFAULT, NULL);
7548 
7549 	list_create(&zcl.zcl_callbacks, sizeof (ztest_cb_data_t),
7550 	    offsetof(ztest_cb_data_t, zcd_node));
7551 
7552 	/*
7553 	 * Open our pool.  It may need to be imported first depending on
7554 	 * what tests were running when the previous pass was terminated.
7555 	 */
7556 	kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7557 	error = spa_open(ztest_opts.zo_pool, &spa, FTAG);
7558 	if (error) {
7559 		VERIFY3S(error, ==, ENOENT);
7560 		ztest_import_impl();
7561 		VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7562 		zs->zs_metaslab_sz =
7563 		    1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
7564 	}
7565 
7566 	metaslab_preload_limit = ztest_random(20) + 1;
7567 	ztest_spa = spa;
7568 
7569 	VERIFY0(vdev_raidz_impl_set("cycle"));
7570 
7571 	dmu_objset_stats_t dds;
7572 	VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
7573 	    DMU_OST_ANY, B_TRUE, B_TRUE, FTAG, &os));
7574 	dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
7575 	dmu_objset_fast_stat(os, &dds);
7576 	dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
7577 	zs->zs_guid = dds.dds_guid;
7578 	dmu_objset_disown(os, B_TRUE, FTAG);
7579 
7580 	/*
7581 	 * Create a thread to periodically resume suspended I/O.
7582 	 */
7583 	resume_thread = thread_create(NULL, 0, ztest_resume_thread,
7584 	    spa, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
7585 
7586 	/*
7587 	 * Create a deadman thread and set to panic if we hang.
7588 	 */
7589 	deadman_thread = thread_create(NULL, 0, ztest_deadman_thread,
7590 	    zs, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
7591 
7592 	spa->spa_deadman_failmode = ZIO_FAILURE_MODE_PANIC;
7593 
7594 	/*
7595 	 * Verify that we can safely inquire about any object,
7596 	 * whether it's allocated or not.  To make it interesting,
7597 	 * we probe a 5-wide window around each power of two.
7598 	 * This hits all edge cases, including zero and the max.
7599 	 */
7600 	for (t = 0; t < 64; t++) {
7601 		for (d = -5; d <= 5; d++) {
7602 			error = dmu_object_info(spa->spa_meta_objset,
7603 			    (1ULL << t) + d, NULL);
7604 			ASSERT(error == 0 || error == ENOENT ||
7605 			    error == EINVAL);
7606 		}
7607 	}
7608 
7609 	/*
7610 	 * If we got any ENOSPC errors on the previous run, destroy something.
7611 	 */
7612 	if (zs->zs_enospc_count != 0) {
7613 		int d = ztest_random(ztest_opts.zo_datasets);
7614 		ztest_dataset_destroy(d);
7615 	}
7616 	zs->zs_enospc_count = 0;
7617 
7618 	/*
7619 	 * If we were in the middle of ztest_device_removal() and were killed
7620 	 * we need to ensure the removal and scrub complete before running
7621 	 * any tests that check ztest_device_removal_active. The removal will
7622 	 * be restarted automatically when the spa is opened, but we need to
7623 	 * initiate the scrub manually if it is not already in progress. Note
7624 	 * that we always run the scrub whenever an indirect vdev exists
7625 	 * because we have no way of knowing for sure if ztest_device_removal()
7626 	 * fully completed its scrub before the pool was reimported.
7627 	 */
7628 	if (spa->spa_removing_phys.sr_state == DSS_SCANNING ||
7629 	    spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
7630 		while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
7631 			txg_wait_synced(spa_get_dsl(spa), 0);
7632 
7633 		error = ztest_scrub_impl(spa);
7634 		if (error == EBUSY)
7635 			error = 0;
7636 		ASSERT0(error);
7637 	}
7638 
7639 	run_threads = umem_zalloc(ztest_opts.zo_threads * sizeof (kthread_t *),
7640 	    UMEM_NOFAIL);
7641 
7642 	if (ztest_opts.zo_verbose >= 4)
7643 		(void) printf("starting main threads...\n");
7644 
7645 	/*
7646 	 * Replay all logs of all datasets in the pool. This is primarily for
7647 	 * temporary datasets which wouldn't otherwise get replayed, which
7648 	 * can trigger failures when attempting to offline a SLOG in
7649 	 * ztest_fault_inject().
7650 	 */
7651 	(void) dmu_objset_find(ztest_opts.zo_pool, ztest_replay_zil_cb,
7652 	    NULL, DS_FIND_CHILDREN);
7653 
7654 	/*
7655 	 * Kick off all the tests that run in parallel.
7656 	 */
7657 	for (t = 0; t < ztest_opts.zo_threads; t++) {
7658 		if (t < ztest_opts.zo_datasets && ztest_dataset_open(t) != 0) {
7659 			umem_free(run_threads, ztest_opts.zo_threads *
7660 			    sizeof (kthread_t *));
7661 			return;
7662 		}
7663 
7664 		run_threads[t] = thread_create(NULL, 0, ztest_thread,
7665 		    (void *)(uintptr_t)t, 0, NULL, TS_RUN | TS_JOINABLE,
7666 		    defclsyspri);
7667 	}
7668 
7669 	/*
7670 	 * Wait for all of the tests to complete.
7671 	 */
7672 	for (t = 0; t < ztest_opts.zo_threads; t++)
7673 		VERIFY0(thread_join(run_threads[t]));
7674 
7675 	/*
7676 	 * Close all datasets. This must be done after all the threads
7677 	 * are joined so we can be sure none of the datasets are in-use
7678 	 * by any of the threads.
7679 	 */
7680 	for (t = 0; t < ztest_opts.zo_threads; t++) {
7681 		if (t < ztest_opts.zo_datasets)
7682 			ztest_dataset_close(t);
7683 	}
7684 
7685 	txg_wait_synced(spa_get_dsl(spa), 0);
7686 
7687 	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(spa));
7688 	zs->zs_space = metaslab_class_get_space(spa_normal_class(spa));
7689 
7690 	umem_free(run_threads, ztest_opts.zo_threads * sizeof (kthread_t *));
7691 
7692 	/* Kill the resume and deadman threads */
7693 	ztest_exiting = B_TRUE;
7694 	VERIFY0(thread_join(resume_thread));
7695 	VERIFY0(thread_join(deadman_thread));
7696 	ztest_resume(spa);
7697 
7698 	/*
7699 	 * Right before closing the pool, kick off a bunch of async I/O;
7700 	 * spa_close() should wait for it to complete.
7701 	 */
7702 	for (object = 1; object < 50; object++) {
7703 		dmu_prefetch(spa->spa_meta_objset, object, 0, 0, 1ULL << 20,
7704 		    ZIO_PRIORITY_SYNC_READ);
7705 	}
7706 
7707 	/* Verify that at least one commit cb was called in a timely fashion */
7708 	if (zc_cb_counter >= ZTEST_COMMIT_CB_MIN_REG)
7709 		VERIFY0(zc_min_txg_delay);
7710 
7711 	spa_close(spa, FTAG);
7712 
7713 	/*
7714 	 * Verify that we can loop over all pools.
7715 	 */
7716 	mutex_enter(&spa_namespace_lock);
7717 	for (spa = spa_next(NULL); spa != NULL; spa = spa_next(spa))
7718 		if (ztest_opts.zo_verbose > 3)
7719 			(void) printf("spa_next: found %s\n", spa_name(spa));
7720 	mutex_exit(&spa_namespace_lock);
7721 
7722 	/*
7723 	 * Verify that we can export the pool and reimport it under a
7724 	 * different name.
7725 	 */
7726 	if ((ztest_random(2) == 0) && !ztest_opts.zo_mmp_test) {
7727 		char name[ZFS_MAX_DATASET_NAME_LEN];
7728 		(void) snprintf(name, sizeof (name), "%s_import",
7729 		    ztest_opts.zo_pool);
7730 		ztest_spa_import_export(ztest_opts.zo_pool, name);
7731 		ztest_spa_import_export(name, ztest_opts.zo_pool);
7732 	}
7733 
7734 	kernel_fini();
7735 
7736 	list_destroy(&zcl.zcl_callbacks);
7737 	mutex_destroy(&zcl.zcl_callbacks_lock);
7738 	(void) pthread_rwlock_destroy(&ztest_name_lock);
7739 	mutex_destroy(&ztest_vdev_lock);
7740 	mutex_destroy(&ztest_checkpoint_lock);
7741 }
7742 
7743 static void
7744 print_time(hrtime_t t, char *timebuf)
7745 {
7746 	hrtime_t s = t / NANOSEC;
7747 	hrtime_t m = s / 60;
7748 	hrtime_t h = m / 60;
7749 	hrtime_t d = h / 24;
7750 
7751 	s -= m * 60;
7752 	m -= h * 60;
7753 	h -= d * 24;
7754 
7755 	timebuf[0] = '\0';
7756 
7757 	if (d)
7758 		(void) sprintf(timebuf,
7759 		    "%llud%02lluh%02llum%02llus", d, h, m, s);
7760 	else if (h)
7761 		(void) sprintf(timebuf, "%lluh%02llum%02llus", h, m, s);
7762 	else if (m)
7763 		(void) sprintf(timebuf, "%llum%02llus", m, s);
7764 	else
7765 		(void) sprintf(timebuf, "%llus", s);
7766 }
7767 
7768 static nvlist_t *
7769 make_random_props(void)
7770 {
7771 	nvlist_t *props;
7772 
7773 	props = fnvlist_alloc();
7774 
7775 	if (ztest_random(2) == 0)
7776 		return (props);
7777 
7778 	fnvlist_add_uint64(props,
7779 	    zpool_prop_to_name(ZPOOL_PROP_AUTOREPLACE), 1);
7780 
7781 	return (props);
7782 }
7783 
7784 /*
7785  * Create a storage pool with the given name and initial vdev size.
7786  * Then test spa_freeze() functionality.
7787  */
7788 static void
7789 ztest_init(ztest_shared_t *zs)
7790 {
7791 	spa_t *spa;
7792 	nvlist_t *nvroot, *props;
7793 	int i;
7794 
7795 	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
7796 	mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
7797 	VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
7798 
7799 	kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7800 
7801 	/*
7802 	 * Create the storage pool.
7803 	 */
7804 	(void) spa_destroy(ztest_opts.zo_pool);
7805 	ztest_shared->zs_vdev_next_leaf = 0;
7806 	zs->zs_splits = 0;
7807 	zs->zs_mirrors = ztest_opts.zo_mirrors;
7808 	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
7809 	    NULL, ztest_opts.zo_raid_children, zs->zs_mirrors, 1);
7810 	props = make_random_props();
7811 
7812 	/*
7813 	 * We don't expect the pool to suspend unless maxfaults == 0,
7814 	 * in which case ztest_fault_inject() temporarily takes away
7815 	 * the only valid replica.
7816 	 */
7817 	fnvlist_add_uint64(props,
7818 	    zpool_prop_to_name(ZPOOL_PROP_FAILUREMODE),
7819 	    MAXFAULTS(zs) ? ZIO_FAILURE_MODE_PANIC : ZIO_FAILURE_MODE_WAIT);
7820 
7821 	for (i = 0; i < SPA_FEATURES; i++) {
7822 		char *buf;
7823 
7824 		if (!spa_feature_table[i].fi_zfs_mod_supported)
7825 			continue;
7826 
7827 		/*
7828 		 * 75% chance of using the log space map feature. We want ztest
7829 		 * to exercise both the code paths that use the log space map
7830 		 * feature and the ones that don't.
7831 		 */
7832 		if (i == SPA_FEATURE_LOG_SPACEMAP && ztest_random(4) == 0)
7833 			continue;
7834 
7835 		VERIFY3S(-1, !=, asprintf(&buf, "feature@%s",
7836 		    spa_feature_table[i].fi_uname));
7837 		fnvlist_add_uint64(props, buf, 0);
7838 		free(buf);
7839 	}
7840 
7841 	VERIFY0(spa_create(ztest_opts.zo_pool, nvroot, props, NULL, NULL));
7842 	fnvlist_free(nvroot);
7843 	fnvlist_free(props);
7844 
7845 	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7846 	zs->zs_metaslab_sz =
7847 	    1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
7848 	spa_close(spa, FTAG);
7849 
7850 	kernel_fini();
7851 
7852 	if (!ztest_opts.zo_mmp_test) {
7853 		ztest_run_zdb(ztest_opts.zo_pool);
7854 		ztest_freeze();
7855 		ztest_run_zdb(ztest_opts.zo_pool);
7856 	}
7857 
7858 	(void) pthread_rwlock_destroy(&ztest_name_lock);
7859 	mutex_destroy(&ztest_vdev_lock);
7860 	mutex_destroy(&ztest_checkpoint_lock);
7861 }
7862 
7863 static void
7864 setup_data_fd(void)
7865 {
7866 	static char ztest_name_data[] = "/tmp/ztest.data.XXXXXX";
7867 
7868 	ztest_fd_data = mkstemp(ztest_name_data);
7869 	ASSERT3S(ztest_fd_data, >=, 0);
7870 	(void) unlink(ztest_name_data);
7871 }
7872 
7873 static int
7874 shared_data_size(ztest_shared_hdr_t *hdr)
7875 {
7876 	int size;
7877 
7878 	size = hdr->zh_hdr_size;
7879 	size += hdr->zh_opts_size;
7880 	size += hdr->zh_size;
7881 	size += hdr->zh_stats_size * hdr->zh_stats_count;
7882 	size += hdr->zh_ds_size * hdr->zh_ds_count;
7883 
7884 	return (size);
7885 }
7886 
7887 static void
7888 setup_hdr(void)
7889 {
7890 	int size;
7891 	ztest_shared_hdr_t *hdr;
7892 
7893 	hdr = (void *)mmap(0, P2ROUNDUP(sizeof (*hdr), getpagesize()),
7894 	    PROT_READ | PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
7895 	ASSERT3P(hdr, !=, MAP_FAILED);
7896 
7897 	VERIFY0(ftruncate(ztest_fd_data, sizeof (ztest_shared_hdr_t)));
7898 
7899 	hdr->zh_hdr_size = sizeof (ztest_shared_hdr_t);
7900 	hdr->zh_opts_size = sizeof (ztest_shared_opts_t);
7901 	hdr->zh_size = sizeof (ztest_shared_t);
7902 	hdr->zh_stats_size = sizeof (ztest_shared_callstate_t);
7903 	hdr->zh_stats_count = ZTEST_FUNCS;
7904 	hdr->zh_ds_size = sizeof (ztest_shared_ds_t);
7905 	hdr->zh_ds_count = ztest_opts.zo_datasets;
7906 
7907 	size = shared_data_size(hdr);
7908 	VERIFY0(ftruncate(ztest_fd_data, size));
7909 
7910 	(void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
7911 }
7912 
7913 static void
7914 setup_data(void)
7915 {
7916 	int size, offset;
7917 	ztest_shared_hdr_t *hdr;
7918 	uint8_t *buf;
7919 
7920 	hdr = (void *)mmap(0, P2ROUNDUP(sizeof (*hdr), getpagesize()),
7921 	    PROT_READ, MAP_SHARED, ztest_fd_data, 0);
7922 	ASSERT3P(hdr, !=, MAP_FAILED);
7923 
7924 	size = shared_data_size(hdr);
7925 
7926 	(void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
7927 	hdr = ztest_shared_hdr = (void *)mmap(0, P2ROUNDUP(size, getpagesize()),
7928 	    PROT_READ | PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
7929 	ASSERT3P(hdr, !=, MAP_FAILED);
7930 	buf = (uint8_t *)hdr;
7931 
7932 	offset = hdr->zh_hdr_size;
7933 	ztest_shared_opts = (void *)&buf[offset];
7934 	offset += hdr->zh_opts_size;
7935 	ztest_shared = (void *)&buf[offset];
7936 	offset += hdr->zh_size;
7937 	ztest_shared_callstate = (void *)&buf[offset];
7938 	offset += hdr->zh_stats_size * hdr->zh_stats_count;
7939 	ztest_shared_ds = (void *)&buf[offset];
7940 }
7941 
7942 static boolean_t
7943 exec_child(char *cmd, char *libpath, boolean_t ignorekill, int *statusp)
7944 {
7945 	pid_t pid;
7946 	int status;
7947 	char *cmdbuf = NULL;
7948 
7949 	pid = fork();
7950 
7951 	if (cmd == NULL) {
7952 		cmdbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
7953 		(void) strlcpy(cmdbuf, getexecname(), MAXPATHLEN);
7954 		cmd = cmdbuf;
7955 	}
7956 
7957 	if (pid == -1)
7958 		fatal(B_TRUE, "fork failed");
7959 
7960 	if (pid == 0) {	/* child */
7961 		char fd_data_str[12];
7962 
7963 		VERIFY3S(11, >=,
7964 		    snprintf(fd_data_str, 12, "%d", ztest_fd_data));
7965 		VERIFY0(setenv("ZTEST_FD_DATA", fd_data_str, 1));
7966 
7967 		if (libpath != NULL) {
7968 			const char *curlp = getenv("LD_LIBRARY_PATH");
7969 			if (curlp == NULL)
7970 				VERIFY0(setenv("LD_LIBRARY_PATH", libpath, 1));
7971 			else {
7972 				char *newlp = NULL;
7973 				VERIFY3S(-1, !=,
7974 				    asprintf(&newlp, "%s:%s", libpath, curlp));
7975 				VERIFY0(setenv("LD_LIBRARY_PATH", newlp, 1));
7976 				free(newlp);
7977 			}
7978 		}
7979 		(void) execl(cmd, cmd, (char *)NULL);
7980 		ztest_dump_core = B_FALSE;
7981 		fatal(B_TRUE, "exec failed: %s", cmd);
7982 	}
7983 
7984 	if (cmdbuf != NULL) {
7985 		umem_free(cmdbuf, MAXPATHLEN);
7986 		cmd = NULL;
7987 	}
7988 
7989 	while (waitpid(pid, &status, 0) != pid)
7990 		continue;
7991 	if (statusp != NULL)
7992 		*statusp = status;
7993 
7994 	if (WIFEXITED(status)) {
7995 		if (WEXITSTATUS(status) != 0) {
7996 			(void) fprintf(stderr, "child exited with code %d\n",
7997 			    WEXITSTATUS(status));
7998 			exit(2);
7999 		}
8000 		return (B_FALSE);
8001 	} else if (WIFSIGNALED(status)) {
8002 		if (!ignorekill || WTERMSIG(status) != SIGKILL) {
8003 			(void) fprintf(stderr, "child died with signal %d\n",
8004 			    WTERMSIG(status));
8005 			exit(3);
8006 		}
8007 		return (B_TRUE);
8008 	} else {
8009 		(void) fprintf(stderr, "something strange happened to child\n");
8010 		exit(4);
8011 	}
8012 }
8013 
8014 static void
8015 ztest_run_init(void)
8016 {
8017 	int i;
8018 
8019 	ztest_shared_t *zs = ztest_shared;
8020 
8021 	/*
8022 	 * Blow away any existing copy of zpool.cache
8023 	 */
8024 	(void) remove(spa_config_path);
8025 
8026 	if (ztest_opts.zo_init == 0) {
8027 		if (ztest_opts.zo_verbose >= 1)
8028 			(void) printf("Importing pool %s\n",
8029 			    ztest_opts.zo_pool);
8030 		ztest_import(zs);
8031 		return;
8032 	}
8033 
8034 	/*
8035 	 * Create and initialize our storage pool.
8036 	 */
8037 	for (i = 1; i <= ztest_opts.zo_init; i++) {
8038 		memset(zs, 0, sizeof (*zs));
8039 		if (ztest_opts.zo_verbose >= 3 &&
8040 		    ztest_opts.zo_init != 1) {
8041 			(void) printf("ztest_init(), pass %d\n", i);
8042 		}
8043 		ztest_init(zs);
8044 	}
8045 }
8046 
8047 int
8048 main(int argc, char **argv)
8049 {
8050 	int kills = 0;
8051 	int iters = 0;
8052 	int older = 0;
8053 	int newer = 0;
8054 	ztest_shared_t *zs;
8055 	ztest_info_t *zi;
8056 	ztest_shared_callstate_t *zc;
8057 	char timebuf[100];
8058 	char numbuf[NN_NUMBUF_SZ];
8059 	char *cmd;
8060 	boolean_t hasalt;
8061 	int f, err;
8062 	char *fd_data_str = getenv("ZTEST_FD_DATA");
8063 	struct sigaction action;
8064 
8065 	(void) setvbuf(stdout, NULL, _IOLBF, 0);
8066 
8067 	dprintf_setup(&argc, argv);
8068 	zfs_deadman_synctime_ms = 300000;
8069 	zfs_deadman_checktime_ms = 30000;
8070 	/*
8071 	 * As two-word space map entries may not come up often (especially
8072 	 * if pool and vdev sizes are small) we want to force at least some
8073 	 * of them so the feature get tested.
8074 	 */
8075 	zfs_force_some_double_word_sm_entries = B_TRUE;
8076 
8077 	/*
8078 	 * Verify that even extensively damaged split blocks with many
8079 	 * segments can be reconstructed in a reasonable amount of time
8080 	 * when reconstruction is known to be possible.
8081 	 *
8082 	 * Note: the lower this value is, the more damage we inflict, and
8083 	 * the more time ztest spends in recovering that damage. We chose
8084 	 * to induce damage 1/100th of the time so recovery is tested but
8085 	 * not so frequently that ztest doesn't get to test other code paths.
8086 	 */
8087 	zfs_reconstruct_indirect_damage_fraction = 100;
8088 
8089 	action.sa_handler = sig_handler;
8090 	sigemptyset(&action.sa_mask);
8091 	action.sa_flags = 0;
8092 
8093 	if (sigaction(SIGSEGV, &action, NULL) < 0) {
8094 		(void) fprintf(stderr, "ztest: cannot catch SIGSEGV: %s.\n",
8095 		    strerror(errno));
8096 		exit(EXIT_FAILURE);
8097 	}
8098 
8099 	if (sigaction(SIGABRT, &action, NULL) < 0) {
8100 		(void) fprintf(stderr, "ztest: cannot catch SIGABRT: %s.\n",
8101 		    strerror(errno));
8102 		exit(EXIT_FAILURE);
8103 	}
8104 
8105 	/*
8106 	 * Force random_get_bytes() to use /dev/urandom in order to prevent
8107 	 * ztest from needlessly depleting the system entropy pool.
8108 	 */
8109 	random_path = "/dev/urandom";
8110 	ztest_fd_rand = open(random_path, O_RDONLY | O_CLOEXEC);
8111 	ASSERT3S(ztest_fd_rand, >=, 0);
8112 
8113 	if (!fd_data_str) {
8114 		process_options(argc, argv);
8115 
8116 		setup_data_fd();
8117 		setup_hdr();
8118 		setup_data();
8119 		memcpy(ztest_shared_opts, &ztest_opts,
8120 		    sizeof (*ztest_shared_opts));
8121 	} else {
8122 		ztest_fd_data = atoi(fd_data_str);
8123 		setup_data();
8124 		memcpy(&ztest_opts, ztest_shared_opts, sizeof (ztest_opts));
8125 	}
8126 	ASSERT3U(ztest_opts.zo_datasets, ==, ztest_shared_hdr->zh_ds_count);
8127 
8128 	err = ztest_set_global_vars();
8129 	if (err != 0 && !fd_data_str) {
8130 		/* error message done by ztest_set_global_vars */
8131 		exit(EXIT_FAILURE);
8132 	} else {
8133 		/* children should not be spawned if setting gvars fails */
8134 		VERIFY3S(err, ==, 0);
8135 	}
8136 
8137 	/* Override location of zpool.cache */
8138 	VERIFY3S(asprintf((char **)&spa_config_path, "%s/zpool.cache",
8139 	    ztest_opts.zo_dir), !=, -1);
8140 
8141 	ztest_ds = umem_alloc(ztest_opts.zo_datasets * sizeof (ztest_ds_t),
8142 	    UMEM_NOFAIL);
8143 	zs = ztest_shared;
8144 
8145 	if (fd_data_str) {
8146 		metaslab_force_ganging = ztest_opts.zo_metaslab_force_ganging;
8147 		metaslab_df_alloc_threshold =
8148 		    zs->zs_metaslab_df_alloc_threshold;
8149 
8150 		if (zs->zs_do_init)
8151 			ztest_run_init();
8152 		else
8153 			ztest_run(zs);
8154 		exit(0);
8155 	}
8156 
8157 	hasalt = (strlen(ztest_opts.zo_alt_ztest) != 0);
8158 
8159 	if (ztest_opts.zo_verbose >= 1) {
8160 		(void) printf("%"PRIu64" vdevs, %d datasets, %d threads,"
8161 		    "%d %s disks, %"PRIu64" seconds...\n\n",
8162 		    ztest_opts.zo_vdevs,
8163 		    ztest_opts.zo_datasets,
8164 		    ztest_opts.zo_threads,
8165 		    ztest_opts.zo_raid_children,
8166 		    ztest_opts.zo_raid_type,
8167 		    ztest_opts.zo_time);
8168 	}
8169 
8170 	cmd = umem_alloc(MAXNAMELEN, UMEM_NOFAIL);
8171 	(void) strlcpy(cmd, getexecname(), MAXNAMELEN);
8172 
8173 	zs->zs_do_init = B_TRUE;
8174 	if (strlen(ztest_opts.zo_alt_ztest) != 0) {
8175 		if (ztest_opts.zo_verbose >= 1) {
8176 			(void) printf("Executing older ztest for "
8177 			    "initialization: %s\n", ztest_opts.zo_alt_ztest);
8178 		}
8179 		VERIFY(!exec_child(ztest_opts.zo_alt_ztest,
8180 		    ztest_opts.zo_alt_libpath, B_FALSE, NULL));
8181 	} else {
8182 		VERIFY(!exec_child(NULL, NULL, B_FALSE, NULL));
8183 	}
8184 	zs->zs_do_init = B_FALSE;
8185 
8186 	zs->zs_proc_start = gethrtime();
8187 	zs->zs_proc_stop = zs->zs_proc_start + ztest_opts.zo_time * NANOSEC;
8188 
8189 	for (f = 0; f < ZTEST_FUNCS; f++) {
8190 		zi = &ztest_info[f];
8191 		zc = ZTEST_GET_SHARED_CALLSTATE(f);
8192 		if (zs->zs_proc_start + zi->zi_interval[0] > zs->zs_proc_stop)
8193 			zc->zc_next = UINT64_MAX;
8194 		else
8195 			zc->zc_next = zs->zs_proc_start +
8196 			    ztest_random(2 * zi->zi_interval[0] + 1);
8197 	}
8198 
8199 	/*
8200 	 * Run the tests in a loop.  These tests include fault injection
8201 	 * to verify that self-healing data works, and forced crashes
8202 	 * to verify that we never lose on-disk consistency.
8203 	 */
8204 	while (gethrtime() < zs->zs_proc_stop) {
8205 		int status;
8206 		boolean_t killed;
8207 
8208 		/*
8209 		 * Initialize the workload counters for each function.
8210 		 */
8211 		for (f = 0; f < ZTEST_FUNCS; f++) {
8212 			zc = ZTEST_GET_SHARED_CALLSTATE(f);
8213 			zc->zc_count = 0;
8214 			zc->zc_time = 0;
8215 		}
8216 
8217 		/* Set the allocation switch size */
8218 		zs->zs_metaslab_df_alloc_threshold =
8219 		    ztest_random(zs->zs_metaslab_sz / 4) + 1;
8220 
8221 		if (!hasalt || ztest_random(2) == 0) {
8222 			if (hasalt && ztest_opts.zo_verbose >= 1) {
8223 				(void) printf("Executing newer ztest: %s\n",
8224 				    cmd);
8225 			}
8226 			newer++;
8227 			killed = exec_child(cmd, NULL, B_TRUE, &status);
8228 		} else {
8229 			if (hasalt && ztest_opts.zo_verbose >= 1) {
8230 				(void) printf("Executing older ztest: %s\n",
8231 				    ztest_opts.zo_alt_ztest);
8232 			}
8233 			older++;
8234 			killed = exec_child(ztest_opts.zo_alt_ztest,
8235 			    ztest_opts.zo_alt_libpath, B_TRUE, &status);
8236 		}
8237 
8238 		if (killed)
8239 			kills++;
8240 		iters++;
8241 
8242 		if (ztest_opts.zo_verbose >= 1) {
8243 			hrtime_t now = gethrtime();
8244 
8245 			now = MIN(now, zs->zs_proc_stop);
8246 			print_time(zs->zs_proc_stop - now, timebuf);
8247 			nicenum(zs->zs_space, numbuf, sizeof (numbuf));
8248 
8249 			(void) printf("Pass %3d, %8s, %3"PRIu64" ENOSPC, "
8250 			    "%4.1f%% of %5s used, %3.0f%% done, %8s to go\n",
8251 			    iters,
8252 			    WIFEXITED(status) ? "Complete" : "SIGKILL",
8253 			    zs->zs_enospc_count,
8254 			    100.0 * zs->zs_alloc / zs->zs_space,
8255 			    numbuf,
8256 			    100.0 * (now - zs->zs_proc_start) /
8257 			    (ztest_opts.zo_time * NANOSEC), timebuf);
8258 		}
8259 
8260 		if (ztest_opts.zo_verbose >= 2) {
8261 			(void) printf("\nWorkload summary:\n\n");
8262 			(void) printf("%7s %9s   %s\n",
8263 			    "Calls", "Time", "Function");
8264 			(void) printf("%7s %9s   %s\n",
8265 			    "-----", "----", "--------");
8266 			for (f = 0; f < ZTEST_FUNCS; f++) {
8267 				zi = &ztest_info[f];
8268 				zc = ZTEST_GET_SHARED_CALLSTATE(f);
8269 				print_time(zc->zc_time, timebuf);
8270 				(void) printf("%7"PRIu64" %9s   %s\n",
8271 				    zc->zc_count, timebuf,
8272 				    zi->zi_funcname);
8273 			}
8274 			(void) printf("\n");
8275 		}
8276 
8277 		if (!ztest_opts.zo_mmp_test)
8278 			ztest_run_zdb(ztest_opts.zo_pool);
8279 	}
8280 
8281 	if (ztest_opts.zo_verbose >= 1) {
8282 		if (hasalt) {
8283 			(void) printf("%d runs of older ztest: %s\n", older,
8284 			    ztest_opts.zo_alt_ztest);
8285 			(void) printf("%d runs of newer ztest: %s\n", newer,
8286 			    cmd);
8287 		}
8288 		(void) printf("%d killed, %d completed, %.0f%% kill rate\n",
8289 		    kills, iters - kills, (100.0 * kills) / MAX(1, iters));
8290 	}
8291 
8292 	umem_free(cmd, MAXNAMELEN);
8293 
8294 	return (0);
8295 }
8296