xref: /freebsd/sys/contrib/openzfs/module/zfs/spa.c (revision 7c43148a974877188a930e4078a164f83da8e652)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
25  * Copyright (c) 2018, Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27  * Copyright 2013 Saso Kiselkov. All rights reserved.
28  * Copyright (c) 2014 Integros [integros.com]
29  * Copyright 2016 Toomas Soome <tsoome@me.com>
30  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
31  * Copyright 2018 Joyent, Inc.
32  * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
33  * Copyright 2017 Joyent, Inc.
34  * Copyright (c) 2017, Intel Corporation.
35  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
36  * Copyright (c) 2023 Hewlett Packard Enterprise Development LP.
37  */
38 
39 /*
40  * SPA: Storage Pool Allocator
41  *
42  * This file contains all the routines used when modifying on-disk SPA state.
43  * This includes opening, importing, destroying, exporting a pool, and syncing a
44  * pool.
45  */
46 
47 #include <sys/zfs_context.h>
48 #include <sys/fm/fs/zfs.h>
49 #include <sys/spa_impl.h>
50 #include <sys/zio.h>
51 #include <sys/zio_checksum.h>
52 #include <sys/dmu.h>
53 #include <sys/dmu_tx.h>
54 #include <sys/zap.h>
55 #include <sys/zil.h>
56 #include <sys/brt.h>
57 #include <sys/ddt.h>
58 #include <sys/vdev_impl.h>
59 #include <sys/vdev_removal.h>
60 #include <sys/vdev_indirect_mapping.h>
61 #include <sys/vdev_indirect_births.h>
62 #include <sys/vdev_initialize.h>
63 #include <sys/vdev_rebuild.h>
64 #include <sys/vdev_trim.h>
65 #include <sys/vdev_disk.h>
66 #include <sys/vdev_raidz.h>
67 #include <sys/vdev_draid.h>
68 #include <sys/metaslab.h>
69 #include <sys/metaslab_impl.h>
70 #include <sys/mmp.h>
71 #include <sys/uberblock_impl.h>
72 #include <sys/txg.h>
73 #include <sys/avl.h>
74 #include <sys/bpobj.h>
75 #include <sys/dmu_traverse.h>
76 #include <sys/dmu_objset.h>
77 #include <sys/unique.h>
78 #include <sys/dsl_pool.h>
79 #include <sys/dsl_dataset.h>
80 #include <sys/dsl_dir.h>
81 #include <sys/dsl_prop.h>
82 #include <sys/dsl_synctask.h>
83 #include <sys/fs/zfs.h>
84 #include <sys/arc.h>
85 #include <sys/callb.h>
86 #include <sys/systeminfo.h>
87 #include <sys/zfs_ioctl.h>
88 #include <sys/dsl_scan.h>
89 #include <sys/zfeature.h>
90 #include <sys/dsl_destroy.h>
91 #include <sys/zvol.h>
92 
93 #ifdef	_KERNEL
94 #include <sys/fm/protocol.h>
95 #include <sys/fm/util.h>
96 #include <sys/callb.h>
97 #include <sys/zone.h>
98 #include <sys/vmsystm.h>
99 #endif	/* _KERNEL */
100 
101 #include "zfs_prop.h"
102 #include "zfs_comutil.h"
103 #include <cityhash.h>
104 
105 /*
106  * spa_thread() existed on Illumos as a parent thread for the various worker
107  * threads that actually run the pool, as a way to both reference the entire
108  * pool work as a single object, and to share properties like scheduling
109  * options. It has not yet been adapted to Linux or FreeBSD. This define is
110  * used to mark related parts of the code to make things easier for the reader,
111  * and to compile this code out. It can be removed when someone implements it,
112  * moves it to some Illumos-specific place, or removes it entirely.
113  */
114 #undef HAVE_SPA_THREAD
115 
116 /*
117  * The "System Duty Cycle" scheduling class is an Illumos feature to help
118  * prevent CPU-intensive kernel threads from affecting latency on interactive
119  * threads. It doesn't exist on Linux or FreeBSD, so the supporting code is
120  * gated behind a define. On Illumos SDC depends on spa_thread(), but
121  * spa_thread() also has other uses, so this is a separate define.
122  */
123 #undef HAVE_SYSDC
124 
125 /*
126  * The interval, in seconds, at which failed configuration cache file writes
127  * should be retried.
128  */
129 int zfs_ccw_retry_interval = 300;
130 
131 typedef enum zti_modes {
132 	ZTI_MODE_FIXED,			/* value is # of threads (min 1) */
133 	ZTI_MODE_SCALE,			/* Taskqs scale with CPUs. */
134 	ZTI_MODE_SYNC,			/* sync thread assigned */
135 	ZTI_MODE_NULL,			/* don't create a taskq */
136 	ZTI_NMODES
137 } zti_modes_t;
138 
139 #define	ZTI_P(n, q)	{ ZTI_MODE_FIXED, (n), (q) }
140 #define	ZTI_PCT(n)	{ ZTI_MODE_ONLINE_PERCENT, (n), 1 }
141 #define	ZTI_SCALE	{ ZTI_MODE_SCALE, 0, 1 }
142 #define	ZTI_SYNC	{ ZTI_MODE_SYNC, 0, 1 }
143 #define	ZTI_NULL	{ ZTI_MODE_NULL, 0, 0 }
144 
145 #define	ZTI_N(n)	ZTI_P(n, 1)
146 #define	ZTI_ONE		ZTI_N(1)
147 
148 typedef struct zio_taskq_info {
149 	zti_modes_t zti_mode;
150 	uint_t zti_value;
151 	uint_t zti_count;
152 } zio_taskq_info_t;
153 
154 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = {
155 	"iss", "iss_h", "int", "int_h"
156 };
157 
158 /*
159  * This table defines the taskq settings for each ZFS I/O type. When
160  * initializing a pool, we use this table to create an appropriately sized
161  * taskq. Some operations are low volume and therefore have a small, static
162  * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE
163  * macros. Other operations process a large amount of data; the ZTI_SCALE
164  * macro causes us to create a taskq oriented for throughput. Some operations
165  * are so high frequency and short-lived that the taskq itself can become a
166  * point of lock contention. The ZTI_P(#, #) macro indicates that we need an
167  * additional degree of parallelism specified by the number of threads per-
168  * taskq and the number of taskqs; when dispatching an event in this case, the
169  * particular taskq is chosen at random. ZTI_SCALE uses a number of taskqs
170  * that scales with the number of CPUs.
171  *
172  * The different taskq priorities are to handle the different contexts (issue
173  * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that
174  * need to be handled with minimum delay.
175  */
176 static const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = {
177 	/* ISSUE	ISSUE_HIGH	INTR		INTR_HIGH */
178 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* NULL */
179 	{ ZTI_N(8),	ZTI_NULL,	ZTI_SCALE,	ZTI_NULL }, /* READ */
180 	{ ZTI_SYNC,	ZTI_N(5),	ZTI_SCALE,	ZTI_N(5) }, /* WRITE */
181 	{ ZTI_SCALE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* FREE */
182 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* CLAIM */
183 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* IOCTL */
184 	{ ZTI_N(4),	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* TRIM */
185 };
186 
187 static void spa_sync_version(void *arg, dmu_tx_t *tx);
188 static void spa_sync_props(void *arg, dmu_tx_t *tx);
189 static boolean_t spa_has_active_shared_spare(spa_t *spa);
190 static int spa_load_impl(spa_t *spa, spa_import_type_t type,
191     const char **ereport);
192 static void spa_vdev_resilver_done(spa_t *spa);
193 
194 /*
195  * Percentage of all CPUs that can be used by the metaslab preload taskq.
196  */
197 static uint_t metaslab_preload_pct = 50;
198 
199 static uint_t	zio_taskq_batch_pct = 80;	  /* 1 thread per cpu in pset */
200 static uint_t	zio_taskq_batch_tpq;		  /* threads per taskq */
201 
202 #ifdef HAVE_SYSDC
203 static const boolean_t	zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */
204 static const uint_t	zio_taskq_basedc = 80;	  /* base duty cycle */
205 #endif
206 
207 #ifdef HAVE_SPA_THREAD
208 static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */
209 #endif
210 
211 static uint_t	zio_taskq_wr_iss_ncpus = 0;
212 
213 /*
214  * Report any spa_load_verify errors found, but do not fail spa_load.
215  * This is used by zdb to analyze non-idle pools.
216  */
217 boolean_t	spa_load_verify_dryrun = B_FALSE;
218 
219 /*
220  * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ).
221  * This is used by zdb for spacemaps verification.
222  */
223 boolean_t	spa_mode_readable_spacemaps = B_FALSE;
224 
225 /*
226  * This (illegal) pool name is used when temporarily importing a spa_t in order
227  * to get the vdev stats associated with the imported devices.
228  */
229 #define	TRYIMPORT_NAME	"$import"
230 
231 /*
232  * For debugging purposes: print out vdev tree during pool import.
233  */
234 static int		spa_load_print_vdev_tree = B_FALSE;
235 
236 /*
237  * A non-zero value for zfs_max_missing_tvds means that we allow importing
238  * pools with missing top-level vdevs. This is strictly intended for advanced
239  * pool recovery cases since missing data is almost inevitable. Pools with
240  * missing devices can only be imported read-only for safety reasons, and their
241  * fail-mode will be automatically set to "continue".
242  *
243  * With 1 missing vdev we should be able to import the pool and mount all
244  * datasets. User data that was not modified after the missing device has been
245  * added should be recoverable. This means that snapshots created prior to the
246  * addition of that device should be completely intact.
247  *
248  * With 2 missing vdevs, some datasets may fail to mount since there are
249  * dataset statistics that are stored as regular metadata. Some data might be
250  * recoverable if those vdevs were added recently.
251  *
252  * With 3 or more missing vdevs, the pool is severely damaged and MOS entries
253  * may be missing entirely. Chances of data recovery are very low. Note that
254  * there are also risks of performing an inadvertent rewind as we might be
255  * missing all the vdevs with the latest uberblocks.
256  */
257 uint64_t	zfs_max_missing_tvds = 0;
258 
259 /*
260  * The parameters below are similar to zfs_max_missing_tvds but are only
261  * intended for a preliminary open of the pool with an untrusted config which
262  * might be incomplete or out-dated.
263  *
264  * We are more tolerant for pools opened from a cachefile since we could have
265  * an out-dated cachefile where a device removal was not registered.
266  * We could have set the limit arbitrarily high but in the case where devices
267  * are really missing we would want to return the proper error codes; we chose
268  * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available
269  * and we get a chance to retrieve the trusted config.
270  */
271 uint64_t	zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1;
272 
273 /*
274  * In the case where config was assembled by scanning device paths (/dev/dsks
275  * by default) we are less tolerant since all the existing devices should have
276  * been detected and we want spa_load to return the right error codes.
277  */
278 uint64_t	zfs_max_missing_tvds_scan = 0;
279 
280 /*
281  * Debugging aid that pauses spa_sync() towards the end.
282  */
283 static const boolean_t	zfs_pause_spa_sync = B_FALSE;
284 
285 /*
286  * Variables to indicate the livelist condense zthr func should wait at certain
287  * points for the livelist to be removed - used to test condense/destroy races
288  */
289 static int zfs_livelist_condense_zthr_pause = 0;
290 static int zfs_livelist_condense_sync_pause = 0;
291 
292 /*
293  * Variables to track whether or not condense cancellation has been
294  * triggered in testing.
295  */
296 static int zfs_livelist_condense_sync_cancel = 0;
297 static int zfs_livelist_condense_zthr_cancel = 0;
298 
299 /*
300  * Variable to track whether or not extra ALLOC blkptrs were added to a
301  * livelist entry while it was being condensed (caused by the way we track
302  * remapped blkptrs in dbuf_remap_impl)
303  */
304 static int zfs_livelist_condense_new_alloc = 0;
305 
306 /*
307  * ==========================================================================
308  * SPA properties routines
309  * ==========================================================================
310  */
311 
312 /*
313  * Add a (source=src, propname=propval) list to an nvlist.
314  */
315 static void
316 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval,
317     uint64_t intval, zprop_source_t src)
318 {
319 	const char *propname = zpool_prop_to_name(prop);
320 	nvlist_t *propval;
321 
322 	propval = fnvlist_alloc();
323 	fnvlist_add_uint64(propval, ZPROP_SOURCE, src);
324 
325 	if (strval != NULL)
326 		fnvlist_add_string(propval, ZPROP_VALUE, strval);
327 	else
328 		fnvlist_add_uint64(propval, ZPROP_VALUE, intval);
329 
330 	fnvlist_add_nvlist(nvl, propname, propval);
331 	nvlist_free(propval);
332 }
333 
334 /*
335  * Add a user property (source=src, propname=propval) to an nvlist.
336  */
337 static void
338 spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval,
339     zprop_source_t src)
340 {
341 	nvlist_t *propval;
342 
343 	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
344 	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
345 	VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
346 	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
347 	nvlist_free(propval);
348 }
349 
350 /*
351  * Get property values from the spa configuration.
352  */
353 static void
354 spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
355 {
356 	vdev_t *rvd = spa->spa_root_vdev;
357 	dsl_pool_t *pool = spa->spa_dsl_pool;
358 	uint64_t size, alloc, cap, version;
359 	const zprop_source_t src = ZPROP_SRC_NONE;
360 	spa_config_dirent_t *dp;
361 	metaslab_class_t *mc = spa_normal_class(spa);
362 
363 	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
364 
365 	if (rvd != NULL) {
366 		alloc = metaslab_class_get_alloc(mc);
367 		alloc += metaslab_class_get_alloc(spa_special_class(spa));
368 		alloc += metaslab_class_get_alloc(spa_dedup_class(spa));
369 		alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa));
370 
371 		size = metaslab_class_get_space(mc);
372 		size += metaslab_class_get_space(spa_special_class(spa));
373 		size += metaslab_class_get_space(spa_dedup_class(spa));
374 		size += metaslab_class_get_space(spa_embedded_log_class(spa));
375 
376 		spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
377 		spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src);
378 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
379 		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
380 		    size - alloc, src);
381 		spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL,
382 		    spa->spa_checkpoint_info.sci_dspace, src);
383 
384 		spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
385 		    metaslab_class_fragmentation(mc), src);
386 		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
387 		    metaslab_class_expandable_space(mc), src);
388 		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
389 		    (spa_mode(spa) == SPA_MODE_READ), src);
390 
391 		cap = (size == 0) ? 0 : (alloc * 100 / size);
392 		spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src);
393 
394 		spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
395 		    ddt_get_pool_dedup_ratio(spa), src);
396 		spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONEUSED, NULL,
397 		    brt_get_used(spa), src);
398 		spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONESAVED, NULL,
399 		    brt_get_saved(spa), src);
400 		spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONERATIO, NULL,
401 		    brt_get_ratio(spa), src);
402 
403 		spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
404 		    rvd->vdev_state, src);
405 
406 		version = spa_version(spa);
407 		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) {
408 			spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
409 			    version, ZPROP_SRC_DEFAULT);
410 		} else {
411 			spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
412 			    version, ZPROP_SRC_LOCAL);
413 		}
414 		spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID,
415 		    NULL, spa_load_guid(spa), src);
416 	}
417 
418 	if (pool != NULL) {
419 		/*
420 		 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
421 		 * when opening pools before this version freedir will be NULL.
422 		 */
423 		if (pool->dp_free_dir != NULL) {
424 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
425 			    dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
426 			    src);
427 		} else {
428 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
429 			    NULL, 0, src);
430 		}
431 
432 		if (pool->dp_leak_dir != NULL) {
433 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
434 			    dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
435 			    src);
436 		} else {
437 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
438 			    NULL, 0, src);
439 		}
440 	}
441 
442 	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);
443 
444 	if (spa->spa_comment != NULL) {
445 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment,
446 		    0, ZPROP_SRC_LOCAL);
447 	}
448 
449 	if (spa->spa_compatibility != NULL) {
450 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY,
451 		    spa->spa_compatibility, 0, ZPROP_SRC_LOCAL);
452 	}
453 
454 	if (spa->spa_root != NULL)
455 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
456 		    0, ZPROP_SRC_LOCAL);
457 
458 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
459 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
460 		    MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
461 	} else {
462 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
463 		    SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
464 	}
465 
466 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) {
467 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
468 		    DNODE_MAX_SIZE, ZPROP_SRC_NONE);
469 	} else {
470 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
471 		    DNODE_MIN_SIZE, ZPROP_SRC_NONE);
472 	}
473 
474 	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
475 		if (dp->scd_path == NULL) {
476 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
477 			    "none", 0, ZPROP_SRC_LOCAL);
478 		} else if (strcmp(dp->scd_path, spa_config_path) != 0) {
479 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
480 			    dp->scd_path, 0, ZPROP_SRC_LOCAL);
481 		}
482 	}
483 }
484 
485 /*
486  * Get zpool property values.
487  */
488 int
489 spa_prop_get(spa_t *spa, nvlist_t **nvp)
490 {
491 	objset_t *mos = spa->spa_meta_objset;
492 	zap_cursor_t zc;
493 	zap_attribute_t za;
494 	dsl_pool_t *dp;
495 	int err;
496 
497 	err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP);
498 	if (err)
499 		return (err);
500 
501 	dp = spa_get_dsl(spa);
502 	dsl_pool_config_enter(dp, FTAG);
503 	mutex_enter(&spa->spa_props_lock);
504 
505 	/*
506 	 * Get properties from the spa config.
507 	 */
508 	spa_prop_get_config(spa, nvp);
509 
510 	/* If no pool property object, no more prop to get. */
511 	if (mos == NULL || spa->spa_pool_props_object == 0)
512 		goto out;
513 
514 	/*
515 	 * Get properties from the MOS pool property object.
516 	 */
517 	for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object);
518 	    (err = zap_cursor_retrieve(&zc, &za)) == 0;
519 	    zap_cursor_advance(&zc)) {
520 		uint64_t intval = 0;
521 		char *strval = NULL;
522 		zprop_source_t src = ZPROP_SRC_DEFAULT;
523 		zpool_prop_t prop;
524 
525 		if ((prop = zpool_name_to_prop(za.za_name)) ==
526 		    ZPOOL_PROP_INVAL && !zfs_prop_user(za.za_name))
527 			continue;
528 
529 		switch (za.za_integer_length) {
530 		case 8:
531 			/* integer property */
532 			if (za.za_first_integer !=
533 			    zpool_prop_default_numeric(prop))
534 				src = ZPROP_SRC_LOCAL;
535 
536 			if (prop == ZPOOL_PROP_BOOTFS) {
537 				dsl_dataset_t *ds = NULL;
538 
539 				err = dsl_dataset_hold_obj(dp,
540 				    za.za_first_integer, FTAG, &ds);
541 				if (err != 0)
542 					break;
543 
544 				strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN,
545 				    KM_SLEEP);
546 				dsl_dataset_name(ds, strval);
547 				dsl_dataset_rele(ds, FTAG);
548 			} else {
549 				strval = NULL;
550 				intval = za.za_first_integer;
551 			}
552 
553 			spa_prop_add_list(*nvp, prop, strval, intval, src);
554 
555 			if (strval != NULL)
556 				kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN);
557 
558 			break;
559 
560 		case 1:
561 			/* string property */
562 			strval = kmem_alloc(za.za_num_integers, KM_SLEEP);
563 			err = zap_lookup(mos, spa->spa_pool_props_object,
564 			    za.za_name, 1, za.za_num_integers, strval);
565 			if (err) {
566 				kmem_free(strval, za.za_num_integers);
567 				break;
568 			}
569 			if (prop != ZPOOL_PROP_INVAL) {
570 				spa_prop_add_list(*nvp, prop, strval, 0, src);
571 			} else {
572 				src = ZPROP_SRC_LOCAL;
573 				spa_prop_add_user(*nvp, za.za_name, strval,
574 				    src);
575 			}
576 			kmem_free(strval, za.za_num_integers);
577 			break;
578 
579 		default:
580 			break;
581 		}
582 	}
583 	zap_cursor_fini(&zc);
584 out:
585 	mutex_exit(&spa->spa_props_lock);
586 	dsl_pool_config_exit(dp, FTAG);
587 	if (err && err != ENOENT) {
588 		nvlist_free(*nvp);
589 		*nvp = NULL;
590 		return (err);
591 	}
592 
593 	return (0);
594 }
595 
596 /*
597  * Validate the given pool properties nvlist and modify the list
598  * for the property values to be set.
599  */
600 static int
601 spa_prop_validate(spa_t *spa, nvlist_t *props)
602 {
603 	nvpair_t *elem;
604 	int error = 0, reset_bootfs = 0;
605 	uint64_t objnum = 0;
606 	boolean_t has_feature = B_FALSE;
607 
608 	elem = NULL;
609 	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
610 		uint64_t intval;
611 		const char *strval, *slash, *check, *fname;
612 		const char *propname = nvpair_name(elem);
613 		zpool_prop_t prop = zpool_name_to_prop(propname);
614 
615 		switch (prop) {
616 		case ZPOOL_PROP_INVAL:
617 			/*
618 			 * Sanitize the input.
619 			 */
620 			if (zfs_prop_user(propname)) {
621 				if (strlen(propname) >= ZAP_MAXNAMELEN) {
622 					error = SET_ERROR(ENAMETOOLONG);
623 					break;
624 				}
625 
626 				if (strlen(fnvpair_value_string(elem)) >=
627 				    ZAP_MAXVALUELEN) {
628 					error = SET_ERROR(E2BIG);
629 					break;
630 				}
631 			} else if (zpool_prop_feature(propname)) {
632 				if (nvpair_type(elem) != DATA_TYPE_UINT64) {
633 					error = SET_ERROR(EINVAL);
634 					break;
635 				}
636 
637 				if (nvpair_value_uint64(elem, &intval) != 0) {
638 					error = SET_ERROR(EINVAL);
639 					break;
640 				}
641 
642 				if (intval != 0) {
643 					error = SET_ERROR(EINVAL);
644 					break;
645 				}
646 
647 				fname = strchr(propname, '@') + 1;
648 				if (zfeature_lookup_name(fname, NULL) != 0) {
649 					error = SET_ERROR(EINVAL);
650 					break;
651 				}
652 
653 				has_feature = B_TRUE;
654 			} else {
655 				error = SET_ERROR(EINVAL);
656 				break;
657 			}
658 			break;
659 
660 		case ZPOOL_PROP_VERSION:
661 			error = nvpair_value_uint64(elem, &intval);
662 			if (!error &&
663 			    (intval < spa_version(spa) ||
664 			    intval > SPA_VERSION_BEFORE_FEATURES ||
665 			    has_feature))
666 				error = SET_ERROR(EINVAL);
667 			break;
668 
669 		case ZPOOL_PROP_DELEGATION:
670 		case ZPOOL_PROP_AUTOREPLACE:
671 		case ZPOOL_PROP_LISTSNAPS:
672 		case ZPOOL_PROP_AUTOEXPAND:
673 		case ZPOOL_PROP_AUTOTRIM:
674 			error = nvpair_value_uint64(elem, &intval);
675 			if (!error && intval > 1)
676 				error = SET_ERROR(EINVAL);
677 			break;
678 
679 		case ZPOOL_PROP_MULTIHOST:
680 			error = nvpair_value_uint64(elem, &intval);
681 			if (!error && intval > 1)
682 				error = SET_ERROR(EINVAL);
683 
684 			if (!error) {
685 				uint32_t hostid = zone_get_hostid(NULL);
686 				if (hostid)
687 					spa->spa_hostid = hostid;
688 				else
689 					error = SET_ERROR(ENOTSUP);
690 			}
691 
692 			break;
693 
694 		case ZPOOL_PROP_BOOTFS:
695 			/*
696 			 * If the pool version is less than SPA_VERSION_BOOTFS,
697 			 * or the pool is still being created (version == 0),
698 			 * the bootfs property cannot be set.
699 			 */
700 			if (spa_version(spa) < SPA_VERSION_BOOTFS) {
701 				error = SET_ERROR(ENOTSUP);
702 				break;
703 			}
704 
705 			/*
706 			 * Make sure the vdev config is bootable
707 			 */
708 			if (!vdev_is_bootable(spa->spa_root_vdev)) {
709 				error = SET_ERROR(ENOTSUP);
710 				break;
711 			}
712 
713 			reset_bootfs = 1;
714 
715 			error = nvpair_value_string(elem, &strval);
716 
717 			if (!error) {
718 				objset_t *os;
719 
720 				if (strval == NULL || strval[0] == '\0') {
721 					objnum = zpool_prop_default_numeric(
722 					    ZPOOL_PROP_BOOTFS);
723 					break;
724 				}
725 
726 				error = dmu_objset_hold(strval, FTAG, &os);
727 				if (error != 0)
728 					break;
729 
730 				/* Must be ZPL. */
731 				if (dmu_objset_type(os) != DMU_OST_ZFS) {
732 					error = SET_ERROR(ENOTSUP);
733 				} else {
734 					objnum = dmu_objset_id(os);
735 				}
736 				dmu_objset_rele(os, FTAG);
737 			}
738 			break;
739 
740 		case ZPOOL_PROP_FAILUREMODE:
741 			error = nvpair_value_uint64(elem, &intval);
742 			if (!error && intval > ZIO_FAILURE_MODE_PANIC)
743 				error = SET_ERROR(EINVAL);
744 
745 			/*
746 			 * This is a special case which only occurs when
747 			 * the pool has completely failed. This allows
748 			 * the user to change the in-core failmode property
749 			 * without syncing it out to disk (I/Os might
750 			 * currently be blocked). We do this by returning
751 			 * EIO to the caller (spa_prop_set) to trick it
752 			 * into thinking we encountered a property validation
753 			 * error.
754 			 */
755 			if (!error && spa_suspended(spa)) {
756 				spa->spa_failmode = intval;
757 				error = SET_ERROR(EIO);
758 			}
759 			break;
760 
761 		case ZPOOL_PROP_CACHEFILE:
762 			if ((error = nvpair_value_string(elem, &strval)) != 0)
763 				break;
764 
765 			if (strval[0] == '\0')
766 				break;
767 
768 			if (strcmp(strval, "none") == 0)
769 				break;
770 
771 			if (strval[0] != '/') {
772 				error = SET_ERROR(EINVAL);
773 				break;
774 			}
775 
776 			slash = strrchr(strval, '/');
777 			ASSERT(slash != NULL);
778 
779 			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
780 			    strcmp(slash, "/..") == 0)
781 				error = SET_ERROR(EINVAL);
782 			break;
783 
784 		case ZPOOL_PROP_COMMENT:
785 			if ((error = nvpair_value_string(elem, &strval)) != 0)
786 				break;
787 			for (check = strval; *check != '\0'; check++) {
788 				if (!isprint(*check)) {
789 					error = SET_ERROR(EINVAL);
790 					break;
791 				}
792 			}
793 			if (strlen(strval) > ZPROP_MAX_COMMENT)
794 				error = SET_ERROR(E2BIG);
795 			break;
796 
797 		default:
798 			break;
799 		}
800 
801 		if (error)
802 			break;
803 	}
804 
805 	(void) nvlist_remove_all(props,
806 	    zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO));
807 
808 	if (!error && reset_bootfs) {
809 		error = nvlist_remove(props,
810 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING);
811 
812 		if (!error) {
813 			error = nvlist_add_uint64(props,
814 			    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum);
815 		}
816 	}
817 
818 	return (error);
819 }
820 
821 void
822 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync)
823 {
824 	const char *cachefile;
825 	spa_config_dirent_t *dp;
826 
827 	if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
828 	    &cachefile) != 0)
829 		return;
830 
831 	dp = kmem_alloc(sizeof (spa_config_dirent_t),
832 	    KM_SLEEP);
833 
834 	if (cachefile[0] == '\0')
835 		dp->scd_path = spa_strdup(spa_config_path);
836 	else if (strcmp(cachefile, "none") == 0)
837 		dp->scd_path = NULL;
838 	else
839 		dp->scd_path = spa_strdup(cachefile);
840 
841 	list_insert_head(&spa->spa_config_list, dp);
842 	if (need_sync)
843 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
844 }
845 
846 int
847 spa_prop_set(spa_t *spa, nvlist_t *nvp)
848 {
849 	int error;
850 	nvpair_t *elem = NULL;
851 	boolean_t need_sync = B_FALSE;
852 
853 	if ((error = spa_prop_validate(spa, nvp)) != 0)
854 		return (error);
855 
856 	while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) {
857 		zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem));
858 
859 		if (prop == ZPOOL_PROP_CACHEFILE ||
860 		    prop == ZPOOL_PROP_ALTROOT ||
861 		    prop == ZPOOL_PROP_READONLY)
862 			continue;
863 
864 		if (prop == ZPOOL_PROP_INVAL &&
865 		    zfs_prop_user(nvpair_name(elem))) {
866 			need_sync = B_TRUE;
867 			break;
868 		}
869 
870 		if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) {
871 			uint64_t ver = 0;
872 
873 			if (prop == ZPOOL_PROP_VERSION) {
874 				VERIFY(nvpair_value_uint64(elem, &ver) == 0);
875 			} else {
876 				ASSERT(zpool_prop_feature(nvpair_name(elem)));
877 				ver = SPA_VERSION_FEATURES;
878 				need_sync = B_TRUE;
879 			}
880 
881 			/* Save time if the version is already set. */
882 			if (ver == spa_version(spa))
883 				continue;
884 
885 			/*
886 			 * In addition to the pool directory object, we might
887 			 * create the pool properties object, the features for
888 			 * read object, the features for write object, or the
889 			 * feature descriptions object.
890 			 */
891 			error = dsl_sync_task(spa->spa_name, NULL,
892 			    spa_sync_version, &ver,
893 			    6, ZFS_SPACE_CHECK_RESERVED);
894 			if (error)
895 				return (error);
896 			continue;
897 		}
898 
899 		need_sync = B_TRUE;
900 		break;
901 	}
902 
903 	if (need_sync) {
904 		return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
905 		    nvp, 6, ZFS_SPACE_CHECK_RESERVED));
906 	}
907 
908 	return (0);
909 }
910 
911 /*
912  * If the bootfs property value is dsobj, clear it.
913  */
914 void
915 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx)
916 {
917 	if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) {
918 		VERIFY(zap_remove(spa->spa_meta_objset,
919 		    spa->spa_pool_props_object,
920 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0);
921 		spa->spa_bootfs = 0;
922 	}
923 }
924 
925 static int
926 spa_change_guid_check(void *arg, dmu_tx_t *tx)
927 {
928 	uint64_t *newguid __maybe_unused = arg;
929 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
930 	vdev_t *rvd = spa->spa_root_vdev;
931 	uint64_t vdev_state;
932 
933 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
934 		int error = (spa_has_checkpoint(spa)) ?
935 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
936 		return (SET_ERROR(error));
937 	}
938 
939 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
940 	vdev_state = rvd->vdev_state;
941 	spa_config_exit(spa, SCL_STATE, FTAG);
942 
943 	if (vdev_state != VDEV_STATE_HEALTHY)
944 		return (SET_ERROR(ENXIO));
945 
946 	ASSERT3U(spa_guid(spa), !=, *newguid);
947 
948 	return (0);
949 }
950 
951 static void
952 spa_change_guid_sync(void *arg, dmu_tx_t *tx)
953 {
954 	uint64_t *newguid = arg;
955 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
956 	uint64_t oldguid;
957 	vdev_t *rvd = spa->spa_root_vdev;
958 
959 	oldguid = spa_guid(spa);
960 
961 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
962 	rvd->vdev_guid = *newguid;
963 	rvd->vdev_guid_sum += (*newguid - oldguid);
964 	vdev_config_dirty(rvd);
965 	spa_config_exit(spa, SCL_STATE, FTAG);
966 
967 	spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu",
968 	    (u_longlong_t)oldguid, (u_longlong_t)*newguid);
969 }
970 
971 /*
972  * Change the GUID for the pool.  This is done so that we can later
973  * re-import a pool built from a clone of our own vdevs.  We will modify
974  * the root vdev's guid, our own pool guid, and then mark all of our
975  * vdevs dirty.  Note that we must make sure that all our vdevs are
976  * online when we do this, or else any vdevs that weren't present
977  * would be orphaned from our pool.  We are also going to issue a
978  * sysevent to update any watchers.
979  */
980 int
981 spa_change_guid(spa_t *spa)
982 {
983 	int error;
984 	uint64_t guid;
985 
986 	mutex_enter(&spa->spa_vdev_top_lock);
987 	mutex_enter(&spa_namespace_lock);
988 	guid = spa_generate_guid(NULL);
989 
990 	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
991 	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);
992 
993 	if (error == 0) {
994 		/*
995 		 * Clear the kobj flag from all the vdevs to allow
996 		 * vdev_cache_process_kobj_evt() to post events to all the
997 		 * vdevs since GUID is updated.
998 		 */
999 		vdev_clear_kobj_evt(spa->spa_root_vdev);
1000 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
1001 			vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]);
1002 
1003 		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
1004 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID);
1005 	}
1006 
1007 	mutex_exit(&spa_namespace_lock);
1008 	mutex_exit(&spa->spa_vdev_top_lock);
1009 
1010 	return (error);
1011 }
1012 
1013 /*
1014  * ==========================================================================
1015  * SPA state manipulation (open/create/destroy/import/export)
1016  * ==========================================================================
1017  */
1018 
1019 static int
1020 spa_error_entry_compare(const void *a, const void *b)
1021 {
1022 	const spa_error_entry_t *sa = (const spa_error_entry_t *)a;
1023 	const spa_error_entry_t *sb = (const spa_error_entry_t *)b;
1024 	int ret;
1025 
1026 	ret = memcmp(&sa->se_bookmark, &sb->se_bookmark,
1027 	    sizeof (zbookmark_phys_t));
1028 
1029 	return (TREE_ISIGN(ret));
1030 }
1031 
1032 /*
1033  * Utility function which retrieves copies of the current logs and
1034  * re-initializes them in the process.
1035  */
1036 void
1037 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub)
1038 {
1039 	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));
1040 
1041 	memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t));
1042 	memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t));
1043 
1044 	avl_create(&spa->spa_errlist_scrub,
1045 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1046 	    offsetof(spa_error_entry_t, se_avl));
1047 	avl_create(&spa->spa_errlist_last,
1048 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1049 	    offsetof(spa_error_entry_t, se_avl));
1050 }
1051 
1052 static void
1053 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
1054 {
1055 	const zio_taskq_info_t *ztip = &zio_taskqs[t][q];
1056 	enum zti_modes mode = ztip->zti_mode;
1057 	uint_t value = ztip->zti_value;
1058 	uint_t count = ztip->zti_count;
1059 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1060 	uint_t cpus, flags = TASKQ_DYNAMIC;
1061 
1062 	switch (mode) {
1063 	case ZTI_MODE_FIXED:
1064 		ASSERT3U(value, >, 0);
1065 		break;
1066 
1067 	case ZTI_MODE_SYNC:
1068 
1069 		/*
1070 		 * Create one wr_iss taskq for every 'zio_taskq_wr_iss_ncpus',
1071 		 * not to exceed the number of spa allocators.
1072 		 */
1073 		if (zio_taskq_wr_iss_ncpus == 0) {
1074 			count = MAX(boot_ncpus / spa->spa_alloc_count, 1);
1075 		} else {
1076 			count = MAX(1,
1077 			    boot_ncpus / MAX(1, zio_taskq_wr_iss_ncpus));
1078 		}
1079 		count = MAX(count, (zio_taskq_batch_pct + 99) / 100);
1080 		count = MIN(count, spa->spa_alloc_count);
1081 
1082 		/*
1083 		 * zio_taskq_batch_pct is unbounded and may exceed 100%, but no
1084 		 * single taskq may have more threads than 100% of online cpus.
1085 		 */
1086 		value = (zio_taskq_batch_pct + count / 2) / count;
1087 		value = MIN(value, 100);
1088 		flags |= TASKQ_THREADS_CPU_PCT;
1089 		break;
1090 
1091 	case ZTI_MODE_SCALE:
1092 		flags |= TASKQ_THREADS_CPU_PCT;
1093 		/*
1094 		 * We want more taskqs to reduce lock contention, but we want
1095 		 * less for better request ordering and CPU utilization.
1096 		 */
1097 		cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100);
1098 		if (zio_taskq_batch_tpq > 0) {
1099 			count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) /
1100 			    zio_taskq_batch_tpq);
1101 		} else {
1102 			/*
1103 			 * Prefer 6 threads per taskq, but no more taskqs
1104 			 * than threads in them on large systems. For 80%:
1105 			 *
1106 			 *                 taskq   taskq   total
1107 			 * cpus    taskqs  percent threads threads
1108 			 * ------- ------- ------- ------- -------
1109 			 * 1       1       80%     1       1
1110 			 * 2       1       80%     1       1
1111 			 * 4       1       80%     3       3
1112 			 * 8       2       40%     3       6
1113 			 * 16      3       27%     4       12
1114 			 * 32      5       16%     5       25
1115 			 * 64      7       11%     7       49
1116 			 * 128     10      8%      10      100
1117 			 * 256     14      6%      15      210
1118 			 */
1119 			count = 1 + cpus / 6;
1120 			while (count * count > cpus)
1121 				count--;
1122 		}
1123 		/* Limit each taskq within 100% to not trigger assertion. */
1124 		count = MAX(count, (zio_taskq_batch_pct + 99) / 100);
1125 		value = (zio_taskq_batch_pct + count / 2) / count;
1126 		break;
1127 
1128 	case ZTI_MODE_NULL:
1129 		tqs->stqs_count = 0;
1130 		tqs->stqs_taskq = NULL;
1131 		return;
1132 
1133 	default:
1134 		panic("unrecognized mode for %s_%s taskq (%u:%u) in "
1135 		    "spa_taskqs_init()",
1136 		    zio_type_name[t], zio_taskq_types[q], mode, value);
1137 		break;
1138 	}
1139 
1140 	ASSERT3U(count, >, 0);
1141 	tqs->stqs_count = count;
1142 	tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP);
1143 
1144 	for (uint_t i = 0; i < count; i++) {
1145 		taskq_t *tq;
1146 		char name[32];
1147 
1148 		if (count > 1)
1149 			(void) snprintf(name, sizeof (name), "%s_%s_%u",
1150 			    zio_type_name[t], zio_taskq_types[q], i);
1151 		else
1152 			(void) snprintf(name, sizeof (name), "%s_%s",
1153 			    zio_type_name[t], zio_taskq_types[q]);
1154 
1155 #ifdef HAVE_SYSDC
1156 		if (zio_taskq_sysdc && spa->spa_proc != &p0) {
1157 			(void) zio_taskq_basedc;
1158 			tq = taskq_create_sysdc(name, value, 50, INT_MAX,
1159 			    spa->spa_proc, zio_taskq_basedc, flags);
1160 		} else {
1161 #endif
1162 			pri_t pri = maxclsyspri;
1163 			/*
1164 			 * The write issue taskq can be extremely CPU
1165 			 * intensive.  Run it at slightly less important
1166 			 * priority than the other taskqs.
1167 			 *
1168 			 * Under Linux and FreeBSD this means incrementing
1169 			 * the priority value as opposed to platforms like
1170 			 * illumos where it should be decremented.
1171 			 *
1172 			 * On FreeBSD, if priorities divided by four (RQ_PPQ)
1173 			 * are equal then a difference between them is
1174 			 * insignificant.
1175 			 */
1176 			if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) {
1177 #if defined(__linux__)
1178 				pri++;
1179 #elif defined(__FreeBSD__)
1180 				pri += 4;
1181 #else
1182 #error "unknown OS"
1183 #endif
1184 			}
1185 			tq = taskq_create_proc(name, value, pri, 50,
1186 			    INT_MAX, spa->spa_proc, flags);
1187 #ifdef HAVE_SYSDC
1188 		}
1189 #endif
1190 
1191 		tqs->stqs_taskq[i] = tq;
1192 	}
1193 }
1194 
1195 static void
1196 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
1197 {
1198 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1199 
1200 	if (tqs->stqs_taskq == NULL) {
1201 		ASSERT3U(tqs->stqs_count, ==, 0);
1202 		return;
1203 	}
1204 
1205 	for (uint_t i = 0; i < tqs->stqs_count; i++) {
1206 		ASSERT3P(tqs->stqs_taskq[i], !=, NULL);
1207 		taskq_destroy(tqs->stqs_taskq[i]);
1208 	}
1209 
1210 	kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *));
1211 	tqs->stqs_taskq = NULL;
1212 }
1213 
1214 /*
1215  * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority.
1216  * Note that a type may have multiple discrete taskqs to avoid lock contention
1217  * on the taskq itself.
1218  */
1219 static taskq_t *
1220 spa_taskq_dispatch_select(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1221     zio_t *zio)
1222 {
1223 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1224 	taskq_t *tq;
1225 
1226 	ASSERT3P(tqs->stqs_taskq, !=, NULL);
1227 	ASSERT3U(tqs->stqs_count, !=, 0);
1228 
1229 	if ((t == ZIO_TYPE_WRITE) && (q == ZIO_TASKQ_ISSUE) &&
1230 	    (zio != NULL) && (zio->io_wr_iss_tq != NULL)) {
1231 		/* dispatch to assigned write issue taskq */
1232 		tq = zio->io_wr_iss_tq;
1233 		return (tq);
1234 	}
1235 
1236 	if (tqs->stqs_count == 1) {
1237 		tq = tqs->stqs_taskq[0];
1238 	} else {
1239 		tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count];
1240 	}
1241 	return (tq);
1242 }
1243 
1244 void
1245 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1246     task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent,
1247     zio_t *zio)
1248 {
1249 	taskq_t *tq = spa_taskq_dispatch_select(spa, t, q, zio);
1250 	taskq_dispatch_ent(tq, func, arg, flags, ent);
1251 }
1252 
1253 /*
1254  * Same as spa_taskq_dispatch_ent() but block on the task until completion.
1255  */
1256 void
1257 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1258     task_func_t *func, void *arg, uint_t flags)
1259 {
1260 	taskq_t *tq = spa_taskq_dispatch_select(spa, t, q, NULL);
1261 	taskqid_t id = taskq_dispatch(tq, func, arg, flags);
1262 	if (id)
1263 		taskq_wait_id(tq, id);
1264 }
1265 
1266 static void
1267 spa_create_zio_taskqs(spa_t *spa)
1268 {
1269 	for (int t = 0; t < ZIO_TYPES; t++) {
1270 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1271 			spa_taskqs_init(spa, t, q);
1272 		}
1273 	}
1274 }
1275 
1276 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD)
1277 static void
1278 spa_thread(void *arg)
1279 {
1280 	psetid_t zio_taskq_psrset_bind = PS_NONE;
1281 	callb_cpr_t cprinfo;
1282 
1283 	spa_t *spa = arg;
1284 	user_t *pu = PTOU(curproc);
1285 
1286 	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
1287 	    spa->spa_name);
1288 
1289 	ASSERT(curproc != &p0);
1290 	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
1291 	    "zpool-%s", spa->spa_name);
1292 	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
1293 
1294 	/* bind this thread to the requested psrset */
1295 	if (zio_taskq_psrset_bind != PS_NONE) {
1296 		pool_lock();
1297 		mutex_enter(&cpu_lock);
1298 		mutex_enter(&pidlock);
1299 		mutex_enter(&curproc->p_lock);
1300 
1301 		if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind,
1302 		    0, NULL, NULL) == 0)  {
1303 			curthread->t_bind_pset = zio_taskq_psrset_bind;
1304 		} else {
1305 			cmn_err(CE_WARN,
1306 			    "Couldn't bind process for zfs pool \"%s\" to "
1307 			    "pset %d\n", spa->spa_name, zio_taskq_psrset_bind);
1308 		}
1309 
1310 		mutex_exit(&curproc->p_lock);
1311 		mutex_exit(&pidlock);
1312 		mutex_exit(&cpu_lock);
1313 		pool_unlock();
1314 	}
1315 
1316 #ifdef HAVE_SYSDC
1317 	if (zio_taskq_sysdc) {
1318 		sysdc_thread_enter(curthread, 100, 0);
1319 	}
1320 #endif
1321 
1322 	spa->spa_proc = curproc;
1323 	spa->spa_did = curthread->t_did;
1324 
1325 	spa_create_zio_taskqs(spa);
1326 
1327 	mutex_enter(&spa->spa_proc_lock);
1328 	ASSERT(spa->spa_proc_state == SPA_PROC_CREATED);
1329 
1330 	spa->spa_proc_state = SPA_PROC_ACTIVE;
1331 	cv_broadcast(&spa->spa_proc_cv);
1332 
1333 	CALLB_CPR_SAFE_BEGIN(&cprinfo);
1334 	while (spa->spa_proc_state == SPA_PROC_ACTIVE)
1335 		cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1336 	CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock);
1337 
1338 	ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE);
1339 	spa->spa_proc_state = SPA_PROC_GONE;
1340 	spa->spa_proc = &p0;
1341 	cv_broadcast(&spa->spa_proc_cv);
1342 	CALLB_CPR_EXIT(&cprinfo);	/* drops spa_proc_lock */
1343 
1344 	mutex_enter(&curproc->p_lock);
1345 	lwp_exit();
1346 }
1347 #endif
1348 
1349 extern metaslab_ops_t *metaslab_allocator(spa_t *spa);
1350 
1351 /*
1352  * Activate an uninitialized pool.
1353  */
1354 static void
1355 spa_activate(spa_t *spa, spa_mode_t mode)
1356 {
1357 	metaslab_ops_t *msp = metaslab_allocator(spa);
1358 	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
1359 
1360 	spa->spa_state = POOL_STATE_ACTIVE;
1361 	spa->spa_mode = mode;
1362 	spa->spa_read_spacemaps = spa_mode_readable_spacemaps;
1363 
1364 	spa->spa_normal_class = metaslab_class_create(spa, msp);
1365 	spa->spa_log_class = metaslab_class_create(spa, msp);
1366 	spa->spa_embedded_log_class = metaslab_class_create(spa, msp);
1367 	spa->spa_special_class = metaslab_class_create(spa, msp);
1368 	spa->spa_dedup_class = metaslab_class_create(spa, msp);
1369 
1370 	/* Try to create a covering process */
1371 	mutex_enter(&spa->spa_proc_lock);
1372 	ASSERT(spa->spa_proc_state == SPA_PROC_NONE);
1373 	ASSERT(spa->spa_proc == &p0);
1374 	spa->spa_did = 0;
1375 
1376 #ifdef HAVE_SPA_THREAD
1377 	/* Only create a process if we're going to be around a while. */
1378 	if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) {
1379 		if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri,
1380 		    NULL, 0) == 0) {
1381 			spa->spa_proc_state = SPA_PROC_CREATED;
1382 			while (spa->spa_proc_state == SPA_PROC_CREATED) {
1383 				cv_wait(&spa->spa_proc_cv,
1384 				    &spa->spa_proc_lock);
1385 			}
1386 			ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1387 			ASSERT(spa->spa_proc != &p0);
1388 			ASSERT(spa->spa_did != 0);
1389 		} else {
1390 #ifdef _KERNEL
1391 			cmn_err(CE_WARN,
1392 			    "Couldn't create process for zfs pool \"%s\"\n",
1393 			    spa->spa_name);
1394 #endif
1395 		}
1396 	}
1397 #endif /* HAVE_SPA_THREAD */
1398 	mutex_exit(&spa->spa_proc_lock);
1399 
1400 	/* If we didn't create a process, we need to create our taskqs. */
1401 	if (spa->spa_proc == &p0) {
1402 		spa_create_zio_taskqs(spa);
1403 	}
1404 
1405 	for (size_t i = 0; i < TXG_SIZE; i++) {
1406 		spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL,
1407 		    ZIO_FLAG_CANFAIL);
1408 	}
1409 
1410 	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
1411 	    offsetof(vdev_t, vdev_config_dirty_node));
1412 	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
1413 	    offsetof(objset_t, os_evicting_node));
1414 	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
1415 	    offsetof(vdev_t, vdev_state_dirty_node));
1416 
1417 	txg_list_create(&spa->spa_vdev_txg_list, spa,
1418 	    offsetof(struct vdev, vdev_txg_node));
1419 
1420 	avl_create(&spa->spa_errlist_scrub,
1421 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1422 	    offsetof(spa_error_entry_t, se_avl));
1423 	avl_create(&spa->spa_errlist_last,
1424 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1425 	    offsetof(spa_error_entry_t, se_avl));
1426 	avl_create(&spa->spa_errlist_healed,
1427 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1428 	    offsetof(spa_error_entry_t, se_avl));
1429 
1430 	spa_activate_os(spa);
1431 
1432 	spa_keystore_init(&spa->spa_keystore);
1433 
1434 	/*
1435 	 * This taskq is used to perform zvol-minor-related tasks
1436 	 * asynchronously. This has several advantages, including easy
1437 	 * resolution of various deadlocks.
1438 	 *
1439 	 * The taskq must be single threaded to ensure tasks are always
1440 	 * processed in the order in which they were dispatched.
1441 	 *
1442 	 * A taskq per pool allows one to keep the pools independent.
1443 	 * This way if one pool is suspended, it will not impact another.
1444 	 *
1445 	 * The preferred location to dispatch a zvol minor task is a sync
1446 	 * task. In this context, there is easy access to the spa_t and minimal
1447 	 * error handling is required because the sync task must succeed.
1448 	 */
1449 	spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri,
1450 	    1, INT_MAX, 0);
1451 
1452 	/*
1453 	 * The taskq to preload metaslabs.
1454 	 */
1455 	spa->spa_metaslab_taskq = taskq_create("z_metaslab",
1456 	    metaslab_preload_pct, maxclsyspri, 1, INT_MAX,
1457 	    TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1458 
1459 	/*
1460 	 * Taskq dedicated to prefetcher threads: this is used to prevent the
1461 	 * pool traverse code from monopolizing the global (and limited)
1462 	 * system_taskq by inappropriately scheduling long running tasks on it.
1463 	 */
1464 	spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100,
1465 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1466 
1467 	/*
1468 	 * The taskq to upgrade datasets in this pool. Currently used by
1469 	 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA.
1470 	 */
1471 	spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100,
1472 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1473 }
1474 
1475 /*
1476  * Opposite of spa_activate().
1477  */
1478 static void
1479 spa_deactivate(spa_t *spa)
1480 {
1481 	ASSERT(spa->spa_sync_on == B_FALSE);
1482 	ASSERT(spa->spa_dsl_pool == NULL);
1483 	ASSERT(spa->spa_root_vdev == NULL);
1484 	ASSERT(spa->spa_async_zio_root == NULL);
1485 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
1486 
1487 	spa_evicting_os_wait(spa);
1488 
1489 	if (spa->spa_zvol_taskq) {
1490 		taskq_destroy(spa->spa_zvol_taskq);
1491 		spa->spa_zvol_taskq = NULL;
1492 	}
1493 
1494 	if (spa->spa_metaslab_taskq) {
1495 		taskq_destroy(spa->spa_metaslab_taskq);
1496 		spa->spa_metaslab_taskq = NULL;
1497 	}
1498 
1499 	if (spa->spa_prefetch_taskq) {
1500 		taskq_destroy(spa->spa_prefetch_taskq);
1501 		spa->spa_prefetch_taskq = NULL;
1502 	}
1503 
1504 	if (spa->spa_upgrade_taskq) {
1505 		taskq_destroy(spa->spa_upgrade_taskq);
1506 		spa->spa_upgrade_taskq = NULL;
1507 	}
1508 
1509 	txg_list_destroy(&spa->spa_vdev_txg_list);
1510 
1511 	list_destroy(&spa->spa_config_dirty_list);
1512 	list_destroy(&spa->spa_evicting_os_list);
1513 	list_destroy(&spa->spa_state_dirty_list);
1514 
1515 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
1516 
1517 	for (int t = 0; t < ZIO_TYPES; t++) {
1518 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1519 			spa_taskqs_fini(spa, t, q);
1520 		}
1521 	}
1522 
1523 	for (size_t i = 0; i < TXG_SIZE; i++) {
1524 		ASSERT3P(spa->spa_txg_zio[i], !=, NULL);
1525 		VERIFY0(zio_wait(spa->spa_txg_zio[i]));
1526 		spa->spa_txg_zio[i] = NULL;
1527 	}
1528 
1529 	metaslab_class_destroy(spa->spa_normal_class);
1530 	spa->spa_normal_class = NULL;
1531 
1532 	metaslab_class_destroy(spa->spa_log_class);
1533 	spa->spa_log_class = NULL;
1534 
1535 	metaslab_class_destroy(spa->spa_embedded_log_class);
1536 	spa->spa_embedded_log_class = NULL;
1537 
1538 	metaslab_class_destroy(spa->spa_special_class);
1539 	spa->spa_special_class = NULL;
1540 
1541 	metaslab_class_destroy(spa->spa_dedup_class);
1542 	spa->spa_dedup_class = NULL;
1543 
1544 	/*
1545 	 * If this was part of an import or the open otherwise failed, we may
1546 	 * still have errors left in the queues.  Empty them just in case.
1547 	 */
1548 	spa_errlog_drain(spa);
1549 	avl_destroy(&spa->spa_errlist_scrub);
1550 	avl_destroy(&spa->spa_errlist_last);
1551 	avl_destroy(&spa->spa_errlist_healed);
1552 
1553 	spa_keystore_fini(&spa->spa_keystore);
1554 
1555 	spa->spa_state = POOL_STATE_UNINITIALIZED;
1556 
1557 	mutex_enter(&spa->spa_proc_lock);
1558 	if (spa->spa_proc_state != SPA_PROC_NONE) {
1559 		ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1560 		spa->spa_proc_state = SPA_PROC_DEACTIVATE;
1561 		cv_broadcast(&spa->spa_proc_cv);
1562 		while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) {
1563 			ASSERT(spa->spa_proc != &p0);
1564 			cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1565 		}
1566 		ASSERT(spa->spa_proc_state == SPA_PROC_GONE);
1567 		spa->spa_proc_state = SPA_PROC_NONE;
1568 	}
1569 	ASSERT(spa->spa_proc == &p0);
1570 	mutex_exit(&spa->spa_proc_lock);
1571 
1572 	/*
1573 	 * We want to make sure spa_thread() has actually exited the ZFS
1574 	 * module, so that the module can't be unloaded out from underneath
1575 	 * it.
1576 	 */
1577 	if (spa->spa_did != 0) {
1578 		thread_join(spa->spa_did);
1579 		spa->spa_did = 0;
1580 	}
1581 
1582 	spa_deactivate_os(spa);
1583 
1584 }
1585 
1586 /*
1587  * Verify a pool configuration, and construct the vdev tree appropriately.  This
1588  * will create all the necessary vdevs in the appropriate layout, with each vdev
1589  * in the CLOSED state.  This will prep the pool before open/creation/import.
1590  * All vdev validation is done by the vdev_alloc() routine.
1591  */
1592 int
1593 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent,
1594     uint_t id, int atype)
1595 {
1596 	nvlist_t **child;
1597 	uint_t children;
1598 	int error;
1599 
1600 	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
1601 		return (error);
1602 
1603 	if ((*vdp)->vdev_ops->vdev_op_leaf)
1604 		return (0);
1605 
1606 	error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1607 	    &child, &children);
1608 
1609 	if (error == ENOENT)
1610 		return (0);
1611 
1612 	if (error) {
1613 		vdev_free(*vdp);
1614 		*vdp = NULL;
1615 		return (SET_ERROR(EINVAL));
1616 	}
1617 
1618 	for (int c = 0; c < children; c++) {
1619 		vdev_t *vd;
1620 		if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
1621 		    atype)) != 0) {
1622 			vdev_free(*vdp);
1623 			*vdp = NULL;
1624 			return (error);
1625 		}
1626 	}
1627 
1628 	ASSERT(*vdp != NULL);
1629 
1630 	return (0);
1631 }
1632 
1633 static boolean_t
1634 spa_should_flush_logs_on_unload(spa_t *spa)
1635 {
1636 	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))
1637 		return (B_FALSE);
1638 
1639 	if (!spa_writeable(spa))
1640 		return (B_FALSE);
1641 
1642 	if (!spa->spa_sync_on)
1643 		return (B_FALSE);
1644 
1645 	if (spa_state(spa) != POOL_STATE_EXPORTED)
1646 		return (B_FALSE);
1647 
1648 	if (zfs_keep_log_spacemaps_at_export)
1649 		return (B_FALSE);
1650 
1651 	return (B_TRUE);
1652 }
1653 
1654 /*
1655  * Opens a transaction that will set the flag that will instruct
1656  * spa_sync to attempt to flush all the metaslabs for that txg.
1657  */
1658 static void
1659 spa_unload_log_sm_flush_all(spa_t *spa)
1660 {
1661 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
1662 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
1663 
1664 	ASSERT3U(spa->spa_log_flushall_txg, ==, 0);
1665 	spa->spa_log_flushall_txg = dmu_tx_get_txg(tx);
1666 
1667 	dmu_tx_commit(tx);
1668 	txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg);
1669 }
1670 
1671 static void
1672 spa_unload_log_sm_metadata(spa_t *spa)
1673 {
1674 	void *cookie = NULL;
1675 	spa_log_sm_t *sls;
1676 	log_summary_entry_t *e;
1677 
1678 	while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg,
1679 	    &cookie)) != NULL) {
1680 		VERIFY0(sls->sls_mscount);
1681 		kmem_free(sls, sizeof (spa_log_sm_t));
1682 	}
1683 
1684 	while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) {
1685 		VERIFY0(e->lse_mscount);
1686 		kmem_free(e, sizeof (log_summary_entry_t));
1687 	}
1688 
1689 	spa->spa_unflushed_stats.sus_nblocks = 0;
1690 	spa->spa_unflushed_stats.sus_memused = 0;
1691 	spa->spa_unflushed_stats.sus_blocklimit = 0;
1692 }
1693 
1694 static void
1695 spa_destroy_aux_threads(spa_t *spa)
1696 {
1697 	if (spa->spa_condense_zthr != NULL) {
1698 		zthr_destroy(spa->spa_condense_zthr);
1699 		spa->spa_condense_zthr = NULL;
1700 	}
1701 	if (spa->spa_checkpoint_discard_zthr != NULL) {
1702 		zthr_destroy(spa->spa_checkpoint_discard_zthr);
1703 		spa->spa_checkpoint_discard_zthr = NULL;
1704 	}
1705 	if (spa->spa_livelist_delete_zthr != NULL) {
1706 		zthr_destroy(spa->spa_livelist_delete_zthr);
1707 		spa->spa_livelist_delete_zthr = NULL;
1708 	}
1709 	if (spa->spa_livelist_condense_zthr != NULL) {
1710 		zthr_destroy(spa->spa_livelist_condense_zthr);
1711 		spa->spa_livelist_condense_zthr = NULL;
1712 	}
1713 	if (spa->spa_raidz_expand_zthr != NULL) {
1714 		zthr_destroy(spa->spa_raidz_expand_zthr);
1715 		spa->spa_raidz_expand_zthr = NULL;
1716 	}
1717 }
1718 
1719 /*
1720  * Opposite of spa_load().
1721  */
1722 static void
1723 spa_unload(spa_t *spa)
1724 {
1725 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1726 	ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED);
1727 
1728 	spa_import_progress_remove(spa_guid(spa));
1729 	spa_load_note(spa, "UNLOADING");
1730 
1731 	spa_wake_waiters(spa);
1732 
1733 	/*
1734 	 * If we have set the spa_final_txg, we have already performed the
1735 	 * tasks below in spa_export_common(). We should not redo it here since
1736 	 * we delay the final TXGs beyond what spa_final_txg is set at.
1737 	 */
1738 	if (spa->spa_final_txg == UINT64_MAX) {
1739 		/*
1740 		 * If the log space map feature is enabled and the pool is
1741 		 * getting exported (but not destroyed), we want to spend some
1742 		 * time flushing as many metaslabs as we can in an attempt to
1743 		 * destroy log space maps and save import time.
1744 		 */
1745 		if (spa_should_flush_logs_on_unload(spa))
1746 			spa_unload_log_sm_flush_all(spa);
1747 
1748 		/*
1749 		 * Stop async tasks.
1750 		 */
1751 		spa_async_suspend(spa);
1752 
1753 		if (spa->spa_root_vdev) {
1754 			vdev_t *root_vdev = spa->spa_root_vdev;
1755 			vdev_initialize_stop_all(root_vdev,
1756 			    VDEV_INITIALIZE_ACTIVE);
1757 			vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE);
1758 			vdev_autotrim_stop_all(spa);
1759 			vdev_rebuild_stop_all(spa);
1760 		}
1761 	}
1762 
1763 	/*
1764 	 * Stop syncing.
1765 	 */
1766 	if (spa->spa_sync_on) {
1767 		txg_sync_stop(spa->spa_dsl_pool);
1768 		spa->spa_sync_on = B_FALSE;
1769 	}
1770 
1771 	/*
1772 	 * This ensures that there is no async metaslab prefetching
1773 	 * while we attempt to unload the spa.
1774 	 */
1775 	taskq_wait(spa->spa_metaslab_taskq);
1776 
1777 	if (spa->spa_mmp.mmp_thread)
1778 		mmp_thread_stop(spa);
1779 
1780 	/*
1781 	 * Wait for any outstanding async I/O to complete.
1782 	 */
1783 	if (spa->spa_async_zio_root != NULL) {
1784 		for (int i = 0; i < max_ncpus; i++)
1785 			(void) zio_wait(spa->spa_async_zio_root[i]);
1786 		kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
1787 		spa->spa_async_zio_root = NULL;
1788 	}
1789 
1790 	if (spa->spa_vdev_removal != NULL) {
1791 		spa_vdev_removal_destroy(spa->spa_vdev_removal);
1792 		spa->spa_vdev_removal = NULL;
1793 	}
1794 
1795 	spa_destroy_aux_threads(spa);
1796 
1797 	spa_condense_fini(spa);
1798 
1799 	bpobj_close(&spa->spa_deferred_bpobj);
1800 
1801 	spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);
1802 
1803 	/*
1804 	 * Close all vdevs.
1805 	 */
1806 	if (spa->spa_root_vdev)
1807 		vdev_free(spa->spa_root_vdev);
1808 	ASSERT(spa->spa_root_vdev == NULL);
1809 
1810 	/*
1811 	 * Close the dsl pool.
1812 	 */
1813 	if (spa->spa_dsl_pool) {
1814 		dsl_pool_close(spa->spa_dsl_pool);
1815 		spa->spa_dsl_pool = NULL;
1816 		spa->spa_meta_objset = NULL;
1817 	}
1818 
1819 	ddt_unload(spa);
1820 	brt_unload(spa);
1821 	spa_unload_log_sm_metadata(spa);
1822 
1823 	/*
1824 	 * Drop and purge level 2 cache
1825 	 */
1826 	spa_l2cache_drop(spa);
1827 
1828 	if (spa->spa_spares.sav_vdevs) {
1829 		for (int i = 0; i < spa->spa_spares.sav_count; i++)
1830 			vdev_free(spa->spa_spares.sav_vdevs[i]);
1831 		kmem_free(spa->spa_spares.sav_vdevs,
1832 		    spa->spa_spares.sav_count * sizeof (void *));
1833 		spa->spa_spares.sav_vdevs = NULL;
1834 	}
1835 	if (spa->spa_spares.sav_config) {
1836 		nvlist_free(spa->spa_spares.sav_config);
1837 		spa->spa_spares.sav_config = NULL;
1838 	}
1839 	spa->spa_spares.sav_count = 0;
1840 
1841 	if (spa->spa_l2cache.sav_vdevs) {
1842 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1843 			vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]);
1844 			vdev_free(spa->spa_l2cache.sav_vdevs[i]);
1845 		}
1846 		kmem_free(spa->spa_l2cache.sav_vdevs,
1847 		    spa->spa_l2cache.sav_count * sizeof (void *));
1848 		spa->spa_l2cache.sav_vdevs = NULL;
1849 	}
1850 	if (spa->spa_l2cache.sav_config) {
1851 		nvlist_free(spa->spa_l2cache.sav_config);
1852 		spa->spa_l2cache.sav_config = NULL;
1853 	}
1854 	spa->spa_l2cache.sav_count = 0;
1855 
1856 	spa->spa_async_suspended = 0;
1857 
1858 	spa->spa_indirect_vdevs_loaded = B_FALSE;
1859 
1860 	if (spa->spa_comment != NULL) {
1861 		spa_strfree(spa->spa_comment);
1862 		spa->spa_comment = NULL;
1863 	}
1864 	if (spa->spa_compatibility != NULL) {
1865 		spa_strfree(spa->spa_compatibility);
1866 		spa->spa_compatibility = NULL;
1867 	}
1868 
1869 	spa->spa_raidz_expand = NULL;
1870 
1871 	spa_config_exit(spa, SCL_ALL, spa);
1872 }
1873 
1874 /*
1875  * Load (or re-load) the current list of vdevs describing the active spares for
1876  * this pool.  When this is called, we have some form of basic information in
1877  * 'spa_spares.sav_config'.  We parse this into vdevs, try to open them, and
1878  * then re-generate a more complete list including status information.
1879  */
1880 void
1881 spa_load_spares(spa_t *spa)
1882 {
1883 	nvlist_t **spares;
1884 	uint_t nspares;
1885 	int i;
1886 	vdev_t *vd, *tvd;
1887 
1888 #ifndef _KERNEL
1889 	/*
1890 	 * zdb opens both the current state of the pool and the
1891 	 * checkpointed state (if present), with a different spa_t.
1892 	 *
1893 	 * As spare vdevs are shared among open pools, we skip loading
1894 	 * them when we load the checkpointed state of the pool.
1895 	 */
1896 	if (!spa_writeable(spa))
1897 		return;
1898 #endif
1899 
1900 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1901 
1902 	/*
1903 	 * First, close and free any existing spare vdevs.
1904 	 */
1905 	if (spa->spa_spares.sav_vdevs) {
1906 		for (i = 0; i < spa->spa_spares.sav_count; i++) {
1907 			vd = spa->spa_spares.sav_vdevs[i];
1908 
1909 			/* Undo the call to spa_activate() below */
1910 			if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1911 			    B_FALSE)) != NULL && tvd->vdev_isspare)
1912 				spa_spare_remove(tvd);
1913 			vdev_close(vd);
1914 			vdev_free(vd);
1915 		}
1916 
1917 		kmem_free(spa->spa_spares.sav_vdevs,
1918 		    spa->spa_spares.sav_count * sizeof (void *));
1919 	}
1920 
1921 	if (spa->spa_spares.sav_config == NULL)
1922 		nspares = 0;
1923 	else
1924 		VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
1925 		    ZPOOL_CONFIG_SPARES, &spares, &nspares));
1926 
1927 	spa->spa_spares.sav_count = (int)nspares;
1928 	spa->spa_spares.sav_vdevs = NULL;
1929 
1930 	if (nspares == 0)
1931 		return;
1932 
1933 	/*
1934 	 * Construct the array of vdevs, opening them to get status in the
1935 	 * process.   For each spare, there is potentially two different vdev_t
1936 	 * structures associated with it: one in the list of spares (used only
1937 	 * for basic validation purposes) and one in the active vdev
1938 	 * configuration (if it's spared in).  During this phase we open and
1939 	 * validate each vdev on the spare list.  If the vdev also exists in the
1940 	 * active configuration, then we also mark this vdev as an active spare.
1941 	 */
1942 	spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *),
1943 	    KM_SLEEP);
1944 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1945 		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
1946 		    VDEV_ALLOC_SPARE) == 0);
1947 		ASSERT(vd != NULL);
1948 
1949 		spa->spa_spares.sav_vdevs[i] = vd;
1950 
1951 		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1952 		    B_FALSE)) != NULL) {
1953 			if (!tvd->vdev_isspare)
1954 				spa_spare_add(tvd);
1955 
1956 			/*
1957 			 * We only mark the spare active if we were successfully
1958 			 * able to load the vdev.  Otherwise, importing a pool
1959 			 * with a bad active spare would result in strange
1960 			 * behavior, because multiple pool would think the spare
1961 			 * is actively in use.
1962 			 *
1963 			 * There is a vulnerability here to an equally bizarre
1964 			 * circumstance, where a dead active spare is later
1965 			 * brought back to life (onlined or otherwise).  Given
1966 			 * the rarity of this scenario, and the extra complexity
1967 			 * it adds, we ignore the possibility.
1968 			 */
1969 			if (!vdev_is_dead(tvd))
1970 				spa_spare_activate(tvd);
1971 		}
1972 
1973 		vd->vdev_top = vd;
1974 		vd->vdev_aux = &spa->spa_spares;
1975 
1976 		if (vdev_open(vd) != 0)
1977 			continue;
1978 
1979 		if (vdev_validate_aux(vd) == 0)
1980 			spa_spare_add(vd);
1981 	}
1982 
1983 	/*
1984 	 * Recompute the stashed list of spares, with status information
1985 	 * this time.
1986 	 */
1987 	fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES);
1988 
1989 	spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *),
1990 	    KM_SLEEP);
1991 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1992 		spares[i] = vdev_config_generate(spa,
1993 		    spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE);
1994 	fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
1995 	    ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
1996 	    spa->spa_spares.sav_count);
1997 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1998 		nvlist_free(spares[i]);
1999 	kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *));
2000 }
2001 
2002 /*
2003  * Load (or re-load) the current list of vdevs describing the active l2cache for
2004  * this pool.  When this is called, we have some form of basic information in
2005  * 'spa_l2cache.sav_config'.  We parse this into vdevs, try to open them, and
2006  * then re-generate a more complete list including status information.
2007  * Devices which are already active have their details maintained, and are
2008  * not re-opened.
2009  */
2010 void
2011 spa_load_l2cache(spa_t *spa)
2012 {
2013 	nvlist_t **l2cache = NULL;
2014 	uint_t nl2cache;
2015 	int i, j, oldnvdevs;
2016 	uint64_t guid;
2017 	vdev_t *vd, **oldvdevs, **newvdevs;
2018 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
2019 
2020 #ifndef _KERNEL
2021 	/*
2022 	 * zdb opens both the current state of the pool and the
2023 	 * checkpointed state (if present), with a different spa_t.
2024 	 *
2025 	 * As L2 caches are part of the ARC which is shared among open
2026 	 * pools, we skip loading them when we load the checkpointed
2027 	 * state of the pool.
2028 	 */
2029 	if (!spa_writeable(spa))
2030 		return;
2031 #endif
2032 
2033 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
2034 
2035 	oldvdevs = sav->sav_vdevs;
2036 	oldnvdevs = sav->sav_count;
2037 	sav->sav_vdevs = NULL;
2038 	sav->sav_count = 0;
2039 
2040 	if (sav->sav_config == NULL) {
2041 		nl2cache = 0;
2042 		newvdevs = NULL;
2043 		goto out;
2044 	}
2045 
2046 	VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config,
2047 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache));
2048 	newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP);
2049 
2050 	/*
2051 	 * Process new nvlist of vdevs.
2052 	 */
2053 	for (i = 0; i < nl2cache; i++) {
2054 		guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID);
2055 
2056 		newvdevs[i] = NULL;
2057 		for (j = 0; j < oldnvdevs; j++) {
2058 			vd = oldvdevs[j];
2059 			if (vd != NULL && guid == vd->vdev_guid) {
2060 				/*
2061 				 * Retain previous vdev for add/remove ops.
2062 				 */
2063 				newvdevs[i] = vd;
2064 				oldvdevs[j] = NULL;
2065 				break;
2066 			}
2067 		}
2068 
2069 		if (newvdevs[i] == NULL) {
2070 			/*
2071 			 * Create new vdev
2072 			 */
2073 			VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
2074 			    VDEV_ALLOC_L2CACHE) == 0);
2075 			ASSERT(vd != NULL);
2076 			newvdevs[i] = vd;
2077 
2078 			/*
2079 			 * Commit this vdev as an l2cache device,
2080 			 * even if it fails to open.
2081 			 */
2082 			spa_l2cache_add(vd);
2083 
2084 			vd->vdev_top = vd;
2085 			vd->vdev_aux = sav;
2086 
2087 			spa_l2cache_activate(vd);
2088 
2089 			if (vdev_open(vd) != 0)
2090 				continue;
2091 
2092 			(void) vdev_validate_aux(vd);
2093 
2094 			if (!vdev_is_dead(vd))
2095 				l2arc_add_vdev(spa, vd);
2096 
2097 			/*
2098 			 * Upon cache device addition to a pool or pool
2099 			 * creation with a cache device or if the header
2100 			 * of the device is invalid we issue an async
2101 			 * TRIM command for the whole device which will
2102 			 * execute if l2arc_trim_ahead > 0.
2103 			 */
2104 			spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM);
2105 		}
2106 	}
2107 
2108 	sav->sav_vdevs = newvdevs;
2109 	sav->sav_count = (int)nl2cache;
2110 
2111 	/*
2112 	 * Recompute the stashed list of l2cache devices, with status
2113 	 * information this time.
2114 	 */
2115 	fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE);
2116 
2117 	if (sav->sav_count > 0)
2118 		l2cache = kmem_alloc(sav->sav_count * sizeof (void *),
2119 		    KM_SLEEP);
2120 	for (i = 0; i < sav->sav_count; i++)
2121 		l2cache[i] = vdev_config_generate(spa,
2122 		    sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE);
2123 	fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE,
2124 	    (const nvlist_t * const *)l2cache, sav->sav_count);
2125 
2126 out:
2127 	/*
2128 	 * Purge vdevs that were dropped
2129 	 */
2130 	if (oldvdevs) {
2131 		for (i = 0; i < oldnvdevs; i++) {
2132 			uint64_t pool;
2133 
2134 			vd = oldvdevs[i];
2135 			if (vd != NULL) {
2136 				ASSERT(vd->vdev_isl2cache);
2137 
2138 				if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
2139 				    pool != 0ULL && l2arc_vdev_present(vd))
2140 					l2arc_remove_vdev(vd);
2141 				vdev_clear_stats(vd);
2142 				vdev_free(vd);
2143 			}
2144 		}
2145 
2146 		kmem_free(oldvdevs, oldnvdevs * sizeof (void *));
2147 	}
2148 
2149 	for (i = 0; i < sav->sav_count; i++)
2150 		nvlist_free(l2cache[i]);
2151 	if (sav->sav_count)
2152 		kmem_free(l2cache, sav->sav_count * sizeof (void *));
2153 }
2154 
2155 static int
2156 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
2157 {
2158 	dmu_buf_t *db;
2159 	char *packed = NULL;
2160 	size_t nvsize = 0;
2161 	int error;
2162 	*value = NULL;
2163 
2164 	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
2165 	if (error)
2166 		return (error);
2167 
2168 	nvsize = *(uint64_t *)db->db_data;
2169 	dmu_buf_rele(db, FTAG);
2170 
2171 	packed = vmem_alloc(nvsize, KM_SLEEP);
2172 	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed,
2173 	    DMU_READ_PREFETCH);
2174 	if (error == 0)
2175 		error = nvlist_unpack(packed, nvsize, value, 0);
2176 	vmem_free(packed, nvsize);
2177 
2178 	return (error);
2179 }
2180 
2181 /*
2182  * Concrete top-level vdevs that are not missing and are not logs. At every
2183  * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds.
2184  */
2185 static uint64_t
2186 spa_healthy_core_tvds(spa_t *spa)
2187 {
2188 	vdev_t *rvd = spa->spa_root_vdev;
2189 	uint64_t tvds = 0;
2190 
2191 	for (uint64_t i = 0; i < rvd->vdev_children; i++) {
2192 		vdev_t *vd = rvd->vdev_child[i];
2193 		if (vd->vdev_islog)
2194 			continue;
2195 		if (vdev_is_concrete(vd) && !vdev_is_dead(vd))
2196 			tvds++;
2197 	}
2198 
2199 	return (tvds);
2200 }
2201 
2202 /*
2203  * Checks to see if the given vdev could not be opened, in which case we post a
2204  * sysevent to notify the autoreplace code that the device has been removed.
2205  */
2206 static void
2207 spa_check_removed(vdev_t *vd)
2208 {
2209 	for (uint64_t c = 0; c < vd->vdev_children; c++)
2210 		spa_check_removed(vd->vdev_child[c]);
2211 
2212 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) &&
2213 	    vdev_is_concrete(vd)) {
2214 		zfs_post_autoreplace(vd->vdev_spa, vd);
2215 		spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK);
2216 	}
2217 }
2218 
2219 static int
2220 spa_check_for_missing_logs(spa_t *spa)
2221 {
2222 	vdev_t *rvd = spa->spa_root_vdev;
2223 
2224 	/*
2225 	 * If we're doing a normal import, then build up any additional
2226 	 * diagnostic information about missing log devices.
2227 	 * We'll pass this up to the user for further processing.
2228 	 */
2229 	if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) {
2230 		nvlist_t **child, *nv;
2231 		uint64_t idx = 0;
2232 
2233 		child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *),
2234 		    KM_SLEEP);
2235 		nv = fnvlist_alloc();
2236 
2237 		for (uint64_t c = 0; c < rvd->vdev_children; c++) {
2238 			vdev_t *tvd = rvd->vdev_child[c];
2239 
2240 			/*
2241 			 * We consider a device as missing only if it failed
2242 			 * to open (i.e. offline or faulted is not considered
2243 			 * as missing).
2244 			 */
2245 			if (tvd->vdev_islog &&
2246 			    tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
2247 				child[idx++] = vdev_config_generate(spa, tvd,
2248 				    B_FALSE, VDEV_CONFIG_MISSING);
2249 			}
2250 		}
2251 
2252 		if (idx > 0) {
2253 			fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2254 			    (const nvlist_t * const *)child, idx);
2255 			fnvlist_add_nvlist(spa->spa_load_info,
2256 			    ZPOOL_CONFIG_MISSING_DEVICES, nv);
2257 
2258 			for (uint64_t i = 0; i < idx; i++)
2259 				nvlist_free(child[i]);
2260 		}
2261 		nvlist_free(nv);
2262 		kmem_free(child, rvd->vdev_children * sizeof (char **));
2263 
2264 		if (idx > 0) {
2265 			spa_load_failed(spa, "some log devices are missing");
2266 			vdev_dbgmsg_print_tree(rvd, 2);
2267 			return (SET_ERROR(ENXIO));
2268 		}
2269 	} else {
2270 		for (uint64_t c = 0; c < rvd->vdev_children; c++) {
2271 			vdev_t *tvd = rvd->vdev_child[c];
2272 
2273 			if (tvd->vdev_islog &&
2274 			    tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
2275 				spa_set_log_state(spa, SPA_LOG_CLEAR);
2276 				spa_load_note(spa, "some log devices are "
2277 				    "missing, ZIL is dropped.");
2278 				vdev_dbgmsg_print_tree(rvd, 2);
2279 				break;
2280 			}
2281 		}
2282 	}
2283 
2284 	return (0);
2285 }
2286 
2287 /*
2288  * Check for missing log devices
2289  */
2290 static boolean_t
2291 spa_check_logs(spa_t *spa)
2292 {
2293 	boolean_t rv = B_FALSE;
2294 	dsl_pool_t *dp = spa_get_dsl(spa);
2295 
2296 	switch (spa->spa_log_state) {
2297 	default:
2298 		break;
2299 	case SPA_LOG_MISSING:
2300 		/* need to recheck in case slog has been restored */
2301 	case SPA_LOG_UNKNOWN:
2302 		rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
2303 		    zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0);
2304 		if (rv)
2305 			spa_set_log_state(spa, SPA_LOG_MISSING);
2306 		break;
2307 	}
2308 	return (rv);
2309 }
2310 
2311 /*
2312  * Passivate any log vdevs (note, does not apply to embedded log metaslabs).
2313  */
2314 static boolean_t
2315 spa_passivate_log(spa_t *spa)
2316 {
2317 	vdev_t *rvd = spa->spa_root_vdev;
2318 	boolean_t slog_found = B_FALSE;
2319 
2320 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
2321 
2322 	for (int c = 0; c < rvd->vdev_children; c++) {
2323 		vdev_t *tvd = rvd->vdev_child[c];
2324 
2325 		if (tvd->vdev_islog) {
2326 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
2327 			metaslab_group_passivate(tvd->vdev_mg);
2328 			slog_found = B_TRUE;
2329 		}
2330 	}
2331 
2332 	return (slog_found);
2333 }
2334 
2335 /*
2336  * Activate any log vdevs (note, does not apply to embedded log metaslabs).
2337  */
2338 static void
2339 spa_activate_log(spa_t *spa)
2340 {
2341 	vdev_t *rvd = spa->spa_root_vdev;
2342 
2343 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
2344 
2345 	for (int c = 0; c < rvd->vdev_children; c++) {
2346 		vdev_t *tvd = rvd->vdev_child[c];
2347 
2348 		if (tvd->vdev_islog) {
2349 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
2350 			metaslab_group_activate(tvd->vdev_mg);
2351 		}
2352 	}
2353 }
2354 
2355 int
2356 spa_reset_logs(spa_t *spa)
2357 {
2358 	int error;
2359 
2360 	error = dmu_objset_find(spa_name(spa), zil_reset,
2361 	    NULL, DS_FIND_CHILDREN);
2362 	if (error == 0) {
2363 		/*
2364 		 * We successfully offlined the log device, sync out the
2365 		 * current txg so that the "stubby" block can be removed
2366 		 * by zil_sync().
2367 		 */
2368 		txg_wait_synced(spa->spa_dsl_pool, 0);
2369 	}
2370 	return (error);
2371 }
2372 
2373 static void
2374 spa_aux_check_removed(spa_aux_vdev_t *sav)
2375 {
2376 	for (int i = 0; i < sav->sav_count; i++)
2377 		spa_check_removed(sav->sav_vdevs[i]);
2378 }
2379 
2380 void
2381 spa_claim_notify(zio_t *zio)
2382 {
2383 	spa_t *spa = zio->io_spa;
2384 
2385 	if (zio->io_error)
2386 		return;
2387 
2388 	mutex_enter(&spa->spa_props_lock);	/* any mutex will do */
2389 	if (spa->spa_claim_max_txg < zio->io_bp->blk_birth)
2390 		spa->spa_claim_max_txg = zio->io_bp->blk_birth;
2391 	mutex_exit(&spa->spa_props_lock);
2392 }
2393 
2394 typedef struct spa_load_error {
2395 	boolean_t	sle_verify_data;
2396 	uint64_t	sle_meta_count;
2397 	uint64_t	sle_data_count;
2398 } spa_load_error_t;
2399 
2400 static void
2401 spa_load_verify_done(zio_t *zio)
2402 {
2403 	blkptr_t *bp = zio->io_bp;
2404 	spa_load_error_t *sle = zio->io_private;
2405 	dmu_object_type_t type = BP_GET_TYPE(bp);
2406 	int error = zio->io_error;
2407 	spa_t *spa = zio->io_spa;
2408 
2409 	abd_free(zio->io_abd);
2410 	if (error) {
2411 		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
2412 		    type != DMU_OT_INTENT_LOG)
2413 			atomic_inc_64(&sle->sle_meta_count);
2414 		else
2415 			atomic_inc_64(&sle->sle_data_count);
2416 	}
2417 
2418 	mutex_enter(&spa->spa_scrub_lock);
2419 	spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp);
2420 	cv_broadcast(&spa->spa_scrub_io_cv);
2421 	mutex_exit(&spa->spa_scrub_lock);
2422 }
2423 
2424 /*
2425  * Maximum number of inflight bytes is the log2 fraction of the arc size.
2426  * By default, we set it to 1/16th of the arc.
2427  */
2428 static uint_t spa_load_verify_shift = 4;
2429 static int spa_load_verify_metadata = B_TRUE;
2430 static int spa_load_verify_data = B_TRUE;
2431 
2432 static int
2433 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
2434     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
2435 {
2436 	zio_t *rio = arg;
2437 	spa_load_error_t *sle = rio->io_private;
2438 
2439 	(void) zilog, (void) dnp;
2440 
2441 	/*
2442 	 * Note: normally this routine will not be called if
2443 	 * spa_load_verify_metadata is not set.  However, it may be useful
2444 	 * to manually set the flag after the traversal has begun.
2445 	 */
2446 	if (!spa_load_verify_metadata)
2447 		return (0);
2448 
2449 	/*
2450 	 * Sanity check the block pointer in order to detect obvious damage
2451 	 * before using the contents in subsequent checks or in zio_read().
2452 	 * When damaged consider it to be a metadata error since we cannot
2453 	 * trust the BP_GET_TYPE and BP_GET_LEVEL values.
2454 	 */
2455 	if (!zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
2456 		atomic_inc_64(&sle->sle_meta_count);
2457 		return (0);
2458 	}
2459 
2460 	if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) ||
2461 	    BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp))
2462 		return (0);
2463 
2464 	if (!BP_IS_METADATA(bp) &&
2465 	    (!spa_load_verify_data || !sle->sle_verify_data))
2466 		return (0);
2467 
2468 	uint64_t maxinflight_bytes =
2469 	    arc_target_bytes() >> spa_load_verify_shift;
2470 	size_t size = BP_GET_PSIZE(bp);
2471 
2472 	mutex_enter(&spa->spa_scrub_lock);
2473 	while (spa->spa_load_verify_bytes >= maxinflight_bytes)
2474 		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
2475 	spa->spa_load_verify_bytes += size;
2476 	mutex_exit(&spa->spa_scrub_lock);
2477 
2478 	zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size,
2479 	    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
2480 	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
2481 	    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
2482 	return (0);
2483 }
2484 
2485 static int
2486 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
2487 {
2488 	(void) dp, (void) arg;
2489 
2490 	if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN)
2491 		return (SET_ERROR(ENAMETOOLONG));
2492 
2493 	return (0);
2494 }
2495 
2496 static int
2497 spa_load_verify(spa_t *spa)
2498 {
2499 	zio_t *rio;
2500 	spa_load_error_t sle = { 0 };
2501 	zpool_load_policy_t policy;
2502 	boolean_t verify_ok = B_FALSE;
2503 	int error = 0;
2504 
2505 	zpool_get_load_policy(spa->spa_config, &policy);
2506 
2507 	if (policy.zlp_rewind & ZPOOL_NEVER_REWIND ||
2508 	    policy.zlp_maxmeta == UINT64_MAX)
2509 		return (0);
2510 
2511 	dsl_pool_config_enter(spa->spa_dsl_pool, FTAG);
2512 	error = dmu_objset_find_dp(spa->spa_dsl_pool,
2513 	    spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL,
2514 	    DS_FIND_CHILDREN);
2515 	dsl_pool_config_exit(spa->spa_dsl_pool, FTAG);
2516 	if (error != 0)
2517 		return (error);
2518 
2519 	/*
2520 	 * Verify data only if we are rewinding or error limit was set.
2521 	 * Otherwise nothing except dbgmsg care about it to waste time.
2522 	 */
2523 	sle.sle_verify_data = (policy.zlp_rewind & ZPOOL_REWIND_MASK) ||
2524 	    (policy.zlp_maxdata < UINT64_MAX);
2525 
2526 	rio = zio_root(spa, NULL, &sle,
2527 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
2528 
2529 	if (spa_load_verify_metadata) {
2530 		if (spa->spa_extreme_rewind) {
2531 			spa_load_note(spa, "performing a complete scan of the "
2532 			    "pool since extreme rewind is on. This may take "
2533 			    "a very long time.\n  (spa_load_verify_data=%u, "
2534 			    "spa_load_verify_metadata=%u)",
2535 			    spa_load_verify_data, spa_load_verify_metadata);
2536 		}
2537 
2538 		error = traverse_pool(spa, spa->spa_verify_min_txg,
2539 		    TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA |
2540 		    TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio);
2541 	}
2542 
2543 	(void) zio_wait(rio);
2544 	ASSERT0(spa->spa_load_verify_bytes);
2545 
2546 	spa->spa_load_meta_errors = sle.sle_meta_count;
2547 	spa->spa_load_data_errors = sle.sle_data_count;
2548 
2549 	if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) {
2550 		spa_load_note(spa, "spa_load_verify found %llu metadata errors "
2551 		    "and %llu data errors", (u_longlong_t)sle.sle_meta_count,
2552 		    (u_longlong_t)sle.sle_data_count);
2553 	}
2554 
2555 	if (spa_load_verify_dryrun ||
2556 	    (!error && sle.sle_meta_count <= policy.zlp_maxmeta &&
2557 	    sle.sle_data_count <= policy.zlp_maxdata)) {
2558 		int64_t loss = 0;
2559 
2560 		verify_ok = B_TRUE;
2561 		spa->spa_load_txg = spa->spa_uberblock.ub_txg;
2562 		spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp;
2563 
2564 		loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts;
2565 		fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME,
2566 		    spa->spa_load_txg_ts);
2567 		fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME,
2568 		    loss);
2569 		fnvlist_add_uint64(spa->spa_load_info,
2570 		    ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count);
2571 		fnvlist_add_uint64(spa->spa_load_info,
2572 		    ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count);
2573 	} else {
2574 		spa->spa_load_max_txg = spa->spa_uberblock.ub_txg;
2575 	}
2576 
2577 	if (spa_load_verify_dryrun)
2578 		return (0);
2579 
2580 	if (error) {
2581 		if (error != ENXIO && error != EIO)
2582 			error = SET_ERROR(EIO);
2583 		return (error);
2584 	}
2585 
2586 	return (verify_ok ? 0 : EIO);
2587 }
2588 
2589 /*
2590  * Find a value in the pool props object.
2591  */
2592 static void
2593 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val)
2594 {
2595 	(void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object,
2596 	    zpool_prop_to_name(prop), sizeof (uint64_t), 1, val);
2597 }
2598 
2599 /*
2600  * Find a value in the pool directory object.
2601  */
2602 static int
2603 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent)
2604 {
2605 	int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
2606 	    name, sizeof (uint64_t), 1, val);
2607 
2608 	if (error != 0 && (error != ENOENT || log_enoent)) {
2609 		spa_load_failed(spa, "couldn't get '%s' value in MOS directory "
2610 		    "[error=%d]", name, error);
2611 	}
2612 
2613 	return (error);
2614 }
2615 
2616 static int
2617 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err)
2618 {
2619 	vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux);
2620 	return (SET_ERROR(err));
2621 }
2622 
2623 boolean_t
2624 spa_livelist_delete_check(spa_t *spa)
2625 {
2626 	return (spa->spa_livelists_to_delete != 0);
2627 }
2628 
2629 static boolean_t
2630 spa_livelist_delete_cb_check(void *arg, zthr_t *z)
2631 {
2632 	(void) z;
2633 	spa_t *spa = arg;
2634 	return (spa_livelist_delete_check(spa));
2635 }
2636 
2637 static int
2638 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
2639 {
2640 	spa_t *spa = arg;
2641 	zio_free(spa, tx->tx_txg, bp);
2642 	dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD,
2643 	    -bp_get_dsize_sync(spa, bp),
2644 	    -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx);
2645 	return (0);
2646 }
2647 
2648 static int
2649 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp)
2650 {
2651 	int err;
2652 	zap_cursor_t zc;
2653 	zap_attribute_t za;
2654 	zap_cursor_init(&zc, os, zap_obj);
2655 	err = zap_cursor_retrieve(&zc, &za);
2656 	zap_cursor_fini(&zc);
2657 	if (err == 0)
2658 		*llp = za.za_first_integer;
2659 	return (err);
2660 }
2661 
2662 /*
2663  * Components of livelist deletion that must be performed in syncing
2664  * context: freeing block pointers and updating the pool-wide data
2665  * structures to indicate how much work is left to do
2666  */
2667 typedef struct sublist_delete_arg {
2668 	spa_t *spa;
2669 	dsl_deadlist_t *ll;
2670 	uint64_t key;
2671 	bplist_t *to_free;
2672 } sublist_delete_arg_t;
2673 
2674 static void
2675 sublist_delete_sync(void *arg, dmu_tx_t *tx)
2676 {
2677 	sublist_delete_arg_t *sda = arg;
2678 	spa_t *spa = sda->spa;
2679 	dsl_deadlist_t *ll = sda->ll;
2680 	uint64_t key = sda->key;
2681 	bplist_t *to_free = sda->to_free;
2682 
2683 	bplist_iterate(to_free, delete_blkptr_cb, spa, tx);
2684 	dsl_deadlist_remove_entry(ll, key, tx);
2685 }
2686 
2687 typedef struct livelist_delete_arg {
2688 	spa_t *spa;
2689 	uint64_t ll_obj;
2690 	uint64_t zap_obj;
2691 } livelist_delete_arg_t;
2692 
2693 static void
2694 livelist_delete_sync(void *arg, dmu_tx_t *tx)
2695 {
2696 	livelist_delete_arg_t *lda = arg;
2697 	spa_t *spa = lda->spa;
2698 	uint64_t ll_obj = lda->ll_obj;
2699 	uint64_t zap_obj = lda->zap_obj;
2700 	objset_t *mos = spa->spa_meta_objset;
2701 	uint64_t count;
2702 
2703 	/* free the livelist and decrement the feature count */
2704 	VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx));
2705 	dsl_deadlist_free(mos, ll_obj, tx);
2706 	spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx);
2707 	VERIFY0(zap_count(mos, zap_obj, &count));
2708 	if (count == 0) {
2709 		/* no more livelists to delete */
2710 		VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT,
2711 		    DMU_POOL_DELETED_CLONES, tx));
2712 		VERIFY0(zap_destroy(mos, zap_obj, tx));
2713 		spa->spa_livelists_to_delete = 0;
2714 		spa_notify_waiters(spa);
2715 	}
2716 }
2717 
2718 /*
2719  * Load in the value for the livelist to be removed and open it. Then,
2720  * load its first sublist and determine which block pointers should actually
2721  * be freed. Then, call a synctask which performs the actual frees and updates
2722  * the pool-wide livelist data.
2723  */
2724 static void
2725 spa_livelist_delete_cb(void *arg, zthr_t *z)
2726 {
2727 	spa_t *spa = arg;
2728 	uint64_t ll_obj = 0, count;
2729 	objset_t *mos = spa->spa_meta_objset;
2730 	uint64_t zap_obj = spa->spa_livelists_to_delete;
2731 	/*
2732 	 * Determine the next livelist to delete. This function should only
2733 	 * be called if there is at least one deleted clone.
2734 	 */
2735 	VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj));
2736 	VERIFY0(zap_count(mos, ll_obj, &count));
2737 	if (count > 0) {
2738 		dsl_deadlist_t *ll;
2739 		dsl_deadlist_entry_t *dle;
2740 		bplist_t to_free;
2741 		ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP);
2742 		dsl_deadlist_open(ll, mos, ll_obj);
2743 		dle = dsl_deadlist_first(ll);
2744 		ASSERT3P(dle, !=, NULL);
2745 		bplist_create(&to_free);
2746 		int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free,
2747 		    z, NULL);
2748 		if (err == 0) {
2749 			sublist_delete_arg_t sync_arg = {
2750 			    .spa = spa,
2751 			    .ll = ll,
2752 			    .key = dle->dle_mintxg,
2753 			    .to_free = &to_free
2754 			};
2755 			zfs_dbgmsg("deleting sublist (id %llu) from"
2756 			    " livelist %llu, %lld remaining",
2757 			    (u_longlong_t)dle->dle_bpobj.bpo_object,
2758 			    (u_longlong_t)ll_obj, (longlong_t)count - 1);
2759 			VERIFY0(dsl_sync_task(spa_name(spa), NULL,
2760 			    sublist_delete_sync, &sync_arg, 0,
2761 			    ZFS_SPACE_CHECK_DESTROY));
2762 		} else {
2763 			VERIFY3U(err, ==, EINTR);
2764 		}
2765 		bplist_clear(&to_free);
2766 		bplist_destroy(&to_free);
2767 		dsl_deadlist_close(ll);
2768 		kmem_free(ll, sizeof (dsl_deadlist_t));
2769 	} else {
2770 		livelist_delete_arg_t sync_arg = {
2771 		    .spa = spa,
2772 		    .ll_obj = ll_obj,
2773 		    .zap_obj = zap_obj
2774 		};
2775 		zfs_dbgmsg("deletion of livelist %llu completed",
2776 		    (u_longlong_t)ll_obj);
2777 		VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync,
2778 		    &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY));
2779 	}
2780 }
2781 
2782 static void
2783 spa_start_livelist_destroy_thread(spa_t *spa)
2784 {
2785 	ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL);
2786 	spa->spa_livelist_delete_zthr =
2787 	    zthr_create("z_livelist_destroy",
2788 	    spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa,
2789 	    minclsyspri);
2790 }
2791 
2792 typedef struct livelist_new_arg {
2793 	bplist_t *allocs;
2794 	bplist_t *frees;
2795 } livelist_new_arg_t;
2796 
2797 static int
2798 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
2799     dmu_tx_t *tx)
2800 {
2801 	ASSERT(tx == NULL);
2802 	livelist_new_arg_t *lna = arg;
2803 	if (bp_freed) {
2804 		bplist_append(lna->frees, bp);
2805 	} else {
2806 		bplist_append(lna->allocs, bp);
2807 		zfs_livelist_condense_new_alloc++;
2808 	}
2809 	return (0);
2810 }
2811 
2812 typedef struct livelist_condense_arg {
2813 	spa_t *spa;
2814 	bplist_t to_keep;
2815 	uint64_t first_size;
2816 	uint64_t next_size;
2817 } livelist_condense_arg_t;
2818 
2819 static void
2820 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx)
2821 {
2822 	livelist_condense_arg_t *lca = arg;
2823 	spa_t *spa = lca->spa;
2824 	bplist_t new_frees;
2825 	dsl_dataset_t *ds = spa->spa_to_condense.ds;
2826 
2827 	/* Have we been cancelled? */
2828 	if (spa->spa_to_condense.cancelled) {
2829 		zfs_livelist_condense_sync_cancel++;
2830 		goto out;
2831 	}
2832 
2833 	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
2834 	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
2835 	dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist;
2836 
2837 	/*
2838 	 * It's possible that the livelist was changed while the zthr was
2839 	 * running. Therefore, we need to check for new blkptrs in the two
2840 	 * entries being condensed and continue to track them in the livelist.
2841 	 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl),
2842 	 * it's possible that the newly added blkptrs are FREEs or ALLOCs so
2843 	 * we need to sort them into two different bplists.
2844 	 */
2845 	uint64_t first_obj = first->dle_bpobj.bpo_object;
2846 	uint64_t next_obj = next->dle_bpobj.bpo_object;
2847 	uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs;
2848 	uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs;
2849 
2850 	bplist_create(&new_frees);
2851 	livelist_new_arg_t new_bps = {
2852 	    .allocs = &lca->to_keep,
2853 	    .frees = &new_frees,
2854 	};
2855 
2856 	if (cur_first_size > lca->first_size) {
2857 		VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj,
2858 		    livelist_track_new_cb, &new_bps, lca->first_size));
2859 	}
2860 	if (cur_next_size > lca->next_size) {
2861 		VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj,
2862 		    livelist_track_new_cb, &new_bps, lca->next_size));
2863 	}
2864 
2865 	dsl_deadlist_clear_entry(first, ll, tx);
2866 	ASSERT(bpobj_is_empty(&first->dle_bpobj));
2867 	dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx);
2868 
2869 	bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx);
2870 	bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx);
2871 	bplist_destroy(&new_frees);
2872 
2873 	char dsname[ZFS_MAX_DATASET_NAME_LEN];
2874 	dsl_dataset_name(ds, dsname);
2875 	zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu "
2876 	    "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu "
2877 	    "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname,
2878 	    (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj,
2879 	    (u_longlong_t)cur_first_size, (u_longlong_t)next_obj,
2880 	    (u_longlong_t)cur_next_size,
2881 	    (u_longlong_t)first->dle_bpobj.bpo_object,
2882 	    (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs);
2883 out:
2884 	dmu_buf_rele(ds->ds_dbuf, spa);
2885 	spa->spa_to_condense.ds = NULL;
2886 	bplist_clear(&lca->to_keep);
2887 	bplist_destroy(&lca->to_keep);
2888 	kmem_free(lca, sizeof (livelist_condense_arg_t));
2889 	spa->spa_to_condense.syncing = B_FALSE;
2890 }
2891 
2892 static void
2893 spa_livelist_condense_cb(void *arg, zthr_t *t)
2894 {
2895 	while (zfs_livelist_condense_zthr_pause &&
2896 	    !(zthr_has_waiters(t) || zthr_iscancelled(t)))
2897 		delay(1);
2898 
2899 	spa_t *spa = arg;
2900 	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
2901 	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
2902 	uint64_t first_size, next_size;
2903 
2904 	livelist_condense_arg_t *lca =
2905 	    kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP);
2906 	bplist_create(&lca->to_keep);
2907 
2908 	/*
2909 	 * Process the livelists (matching FREEs and ALLOCs) in open context
2910 	 * so we have minimal work in syncing context to condense.
2911 	 *
2912 	 * We save bpobj sizes (first_size and next_size) to use later in
2913 	 * syncing context to determine if entries were added to these sublists
2914 	 * while in open context. This is possible because the clone is still
2915 	 * active and open for normal writes and we want to make sure the new,
2916 	 * unprocessed blockpointers are inserted into the livelist normally.
2917 	 *
2918 	 * Note that dsl_process_sub_livelist() both stores the size number of
2919 	 * blockpointers and iterates over them while the bpobj's lock held, so
2920 	 * the sizes returned to us are consistent which what was actually
2921 	 * processed.
2922 	 */
2923 	int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t,
2924 	    &first_size);
2925 	if (err == 0)
2926 		err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep,
2927 		    t, &next_size);
2928 
2929 	if (err == 0) {
2930 		while (zfs_livelist_condense_sync_pause &&
2931 		    !(zthr_has_waiters(t) || zthr_iscancelled(t)))
2932 			delay(1);
2933 
2934 		dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
2935 		dmu_tx_mark_netfree(tx);
2936 		dmu_tx_hold_space(tx, 1);
2937 		err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE);
2938 		if (err == 0) {
2939 			/*
2940 			 * Prevent the condense zthr restarting before
2941 			 * the synctask completes.
2942 			 */
2943 			spa->spa_to_condense.syncing = B_TRUE;
2944 			lca->spa = spa;
2945 			lca->first_size = first_size;
2946 			lca->next_size = next_size;
2947 			dsl_sync_task_nowait(spa_get_dsl(spa),
2948 			    spa_livelist_condense_sync, lca, tx);
2949 			dmu_tx_commit(tx);
2950 			return;
2951 		}
2952 	}
2953 	/*
2954 	 * Condensing can not continue: either it was externally stopped or
2955 	 * we were unable to assign to a tx because the pool has run out of
2956 	 * space. In the second case, we'll just end up trying to condense
2957 	 * again in a later txg.
2958 	 */
2959 	ASSERT(err != 0);
2960 	bplist_clear(&lca->to_keep);
2961 	bplist_destroy(&lca->to_keep);
2962 	kmem_free(lca, sizeof (livelist_condense_arg_t));
2963 	dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa);
2964 	spa->spa_to_condense.ds = NULL;
2965 	if (err == EINTR)
2966 		zfs_livelist_condense_zthr_cancel++;
2967 }
2968 
2969 /*
2970  * Check that there is something to condense but that a condense is not
2971  * already in progress and that condensing has not been cancelled.
2972  */
2973 static boolean_t
2974 spa_livelist_condense_cb_check(void *arg, zthr_t *z)
2975 {
2976 	(void) z;
2977 	spa_t *spa = arg;
2978 	if ((spa->spa_to_condense.ds != NULL) &&
2979 	    (spa->spa_to_condense.syncing == B_FALSE) &&
2980 	    (spa->spa_to_condense.cancelled == B_FALSE)) {
2981 		return (B_TRUE);
2982 	}
2983 	return (B_FALSE);
2984 }
2985 
2986 static void
2987 spa_start_livelist_condensing_thread(spa_t *spa)
2988 {
2989 	spa->spa_to_condense.ds = NULL;
2990 	spa->spa_to_condense.first = NULL;
2991 	spa->spa_to_condense.next = NULL;
2992 	spa->spa_to_condense.syncing = B_FALSE;
2993 	spa->spa_to_condense.cancelled = B_FALSE;
2994 
2995 	ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL);
2996 	spa->spa_livelist_condense_zthr =
2997 	    zthr_create("z_livelist_condense",
2998 	    spa_livelist_condense_cb_check,
2999 	    spa_livelist_condense_cb, spa, minclsyspri);
3000 }
3001 
3002 static void
3003 spa_spawn_aux_threads(spa_t *spa)
3004 {
3005 	ASSERT(spa_writeable(spa));
3006 
3007 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
3008 
3009 	spa_start_raidz_expansion_thread(spa);
3010 	spa_start_indirect_condensing_thread(spa);
3011 	spa_start_livelist_destroy_thread(spa);
3012 	spa_start_livelist_condensing_thread(spa);
3013 
3014 	ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL);
3015 	spa->spa_checkpoint_discard_zthr =
3016 	    zthr_create("z_checkpoint_discard",
3017 	    spa_checkpoint_discard_thread_check,
3018 	    spa_checkpoint_discard_thread, spa, minclsyspri);
3019 }
3020 
3021 /*
3022  * Fix up config after a partly-completed split.  This is done with the
3023  * ZPOOL_CONFIG_SPLIT nvlist.  Both the splitting pool and the split-off
3024  * pool have that entry in their config, but only the splitting one contains
3025  * a list of all the guids of the vdevs that are being split off.
3026  *
3027  * This function determines what to do with that list: either rejoin
3028  * all the disks to the pool, or complete the splitting process.  To attempt
3029  * the rejoin, each disk that is offlined is marked online again, and
3030  * we do a reopen() call.  If the vdev label for every disk that was
3031  * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL)
3032  * then we call vdev_split() on each disk, and complete the split.
3033  *
3034  * Otherwise we leave the config alone, with all the vdevs in place in
3035  * the original pool.
3036  */
3037 static void
3038 spa_try_repair(spa_t *spa, nvlist_t *config)
3039 {
3040 	uint_t extracted;
3041 	uint64_t *glist;
3042 	uint_t i, gcount;
3043 	nvlist_t *nvl;
3044 	vdev_t **vd;
3045 	boolean_t attempt_reopen;
3046 
3047 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0)
3048 		return;
3049 
3050 	/* check that the config is complete */
3051 	if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
3052 	    &glist, &gcount) != 0)
3053 		return;
3054 
3055 	vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP);
3056 
3057 	/* attempt to online all the vdevs & validate */
3058 	attempt_reopen = B_TRUE;
3059 	for (i = 0; i < gcount; i++) {
3060 		if (glist[i] == 0)	/* vdev is hole */
3061 			continue;
3062 
3063 		vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE);
3064 		if (vd[i] == NULL) {
3065 			/*
3066 			 * Don't bother attempting to reopen the disks;
3067 			 * just do the split.
3068 			 */
3069 			attempt_reopen = B_FALSE;
3070 		} else {
3071 			/* attempt to re-online it */
3072 			vd[i]->vdev_offline = B_FALSE;
3073 		}
3074 	}
3075 
3076 	if (attempt_reopen) {
3077 		vdev_reopen(spa->spa_root_vdev);
3078 
3079 		/* check each device to see what state it's in */
3080 		for (extracted = 0, i = 0; i < gcount; i++) {
3081 			if (vd[i] != NULL &&
3082 			    vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL)
3083 				break;
3084 			++extracted;
3085 		}
3086 	}
3087 
3088 	/*
3089 	 * If every disk has been moved to the new pool, or if we never
3090 	 * even attempted to look at them, then we split them off for
3091 	 * good.
3092 	 */
3093 	if (!attempt_reopen || gcount == extracted) {
3094 		for (i = 0; i < gcount; i++)
3095 			if (vd[i] != NULL)
3096 				vdev_split(vd[i]);
3097 		vdev_reopen(spa->spa_root_vdev);
3098 	}
3099 
3100 	kmem_free(vd, gcount * sizeof (vdev_t *));
3101 }
3102 
3103 static int
3104 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type)
3105 {
3106 	const char *ereport = FM_EREPORT_ZFS_POOL;
3107 	int error;
3108 
3109 	spa->spa_load_state = state;
3110 	(void) spa_import_progress_set_state(spa_guid(spa),
3111 	    spa_load_state(spa));
3112 
3113 	gethrestime(&spa->spa_loaded_ts);
3114 	error = spa_load_impl(spa, type, &ereport);
3115 
3116 	/*
3117 	 * Don't count references from objsets that are already closed
3118 	 * and are making their way through the eviction process.
3119 	 */
3120 	spa_evicting_os_wait(spa);
3121 	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
3122 	if (error) {
3123 		if (error != EEXIST) {
3124 			spa->spa_loaded_ts.tv_sec = 0;
3125 			spa->spa_loaded_ts.tv_nsec = 0;
3126 		}
3127 		if (error != EBADF) {
3128 			(void) zfs_ereport_post(ereport, spa,
3129 			    NULL, NULL, NULL, 0);
3130 		}
3131 	}
3132 	spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE;
3133 	spa->spa_ena = 0;
3134 
3135 	(void) spa_import_progress_set_state(spa_guid(spa),
3136 	    spa_load_state(spa));
3137 
3138 	return (error);
3139 }
3140 
3141 #ifdef ZFS_DEBUG
3142 /*
3143  * Count the number of per-vdev ZAPs associated with all of the vdevs in the
3144  * vdev tree rooted in the given vd, and ensure that each ZAP is present in the
3145  * spa's per-vdev ZAP list.
3146  */
3147 static uint64_t
3148 vdev_count_verify_zaps(vdev_t *vd)
3149 {
3150 	spa_t *spa = vd->vdev_spa;
3151 	uint64_t total = 0;
3152 
3153 	if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) &&
3154 	    vd->vdev_root_zap != 0) {
3155 		total++;
3156 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
3157 		    spa->spa_all_vdev_zaps, vd->vdev_root_zap));
3158 	}
3159 	if (vd->vdev_top_zap != 0) {
3160 		total++;
3161 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
3162 		    spa->spa_all_vdev_zaps, vd->vdev_top_zap));
3163 	}
3164 	if (vd->vdev_leaf_zap != 0) {
3165 		total++;
3166 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
3167 		    spa->spa_all_vdev_zaps, vd->vdev_leaf_zap));
3168 	}
3169 
3170 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
3171 		total += vdev_count_verify_zaps(vd->vdev_child[i]);
3172 	}
3173 
3174 	return (total);
3175 }
3176 #else
3177 #define	vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0)
3178 #endif
3179 
3180 /*
3181  * Determine whether the activity check is required.
3182  */
3183 static boolean_t
3184 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label,
3185     nvlist_t *config)
3186 {
3187 	uint64_t state = 0;
3188 	uint64_t hostid = 0;
3189 	uint64_t tryconfig_txg = 0;
3190 	uint64_t tryconfig_timestamp = 0;
3191 	uint16_t tryconfig_mmp_seq = 0;
3192 	nvlist_t *nvinfo;
3193 
3194 	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
3195 		nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3196 		(void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG,
3197 		    &tryconfig_txg);
3198 		(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
3199 		    &tryconfig_timestamp);
3200 		(void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ,
3201 		    &tryconfig_mmp_seq);
3202 	}
3203 
3204 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state);
3205 
3206 	/*
3207 	 * Disable the MMP activity check - This is used by zdb which
3208 	 * is intended to be used on potentially active pools.
3209 	 */
3210 	if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP)
3211 		return (B_FALSE);
3212 
3213 	/*
3214 	 * Skip the activity check when the MMP feature is disabled.
3215 	 */
3216 	if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0)
3217 		return (B_FALSE);
3218 
3219 	/*
3220 	 * If the tryconfig_ values are nonzero, they are the results of an
3221 	 * earlier tryimport.  If they all match the uberblock we just found,
3222 	 * then the pool has not changed and we return false so we do not test
3223 	 * a second time.
3224 	 */
3225 	if (tryconfig_txg && tryconfig_txg == ub->ub_txg &&
3226 	    tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp &&
3227 	    tryconfig_mmp_seq && tryconfig_mmp_seq ==
3228 	    (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0))
3229 		return (B_FALSE);
3230 
3231 	/*
3232 	 * Allow the activity check to be skipped when importing the pool
3233 	 * on the same host which last imported it.  Since the hostid from
3234 	 * configuration may be stale use the one read from the label.
3235 	 */
3236 	if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID))
3237 		hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID);
3238 
3239 	if (hostid == spa_get_hostid(spa))
3240 		return (B_FALSE);
3241 
3242 	/*
3243 	 * Skip the activity test when the pool was cleanly exported.
3244 	 */
3245 	if (state != POOL_STATE_ACTIVE)
3246 		return (B_FALSE);
3247 
3248 	return (B_TRUE);
3249 }
3250 
3251 /*
3252  * Nanoseconds the activity check must watch for changes on-disk.
3253  */
3254 static uint64_t
3255 spa_activity_check_duration(spa_t *spa, uberblock_t *ub)
3256 {
3257 	uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1);
3258 	uint64_t multihost_interval = MSEC2NSEC(
3259 	    MMP_INTERVAL_OK(zfs_multihost_interval));
3260 	uint64_t import_delay = MAX(NANOSEC, import_intervals *
3261 	    multihost_interval);
3262 
3263 	/*
3264 	 * Local tunables determine a minimum duration except for the case
3265 	 * where we know when the remote host will suspend the pool if MMP
3266 	 * writes do not land.
3267 	 *
3268 	 * See Big Theory comment at the top of mmp.c for the reasoning behind
3269 	 * these cases and times.
3270 	 */
3271 
3272 	ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100);
3273 
3274 	if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
3275 	    MMP_FAIL_INT(ub) > 0) {
3276 
3277 		/* MMP on remote host will suspend pool after failed writes */
3278 		import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) *
3279 		    MMP_IMPORT_SAFETY_FACTOR / 100;
3280 
3281 		zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp "
3282 		    "mmp_fails=%llu ub_mmp mmp_interval=%llu "
3283 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3284 		    (u_longlong_t)MMP_FAIL_INT(ub),
3285 		    (u_longlong_t)MMP_INTERVAL(ub),
3286 		    (u_longlong_t)import_intervals);
3287 
3288 	} else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
3289 	    MMP_FAIL_INT(ub) == 0) {
3290 
3291 		/* MMP on remote host will never suspend pool */
3292 		import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) +
3293 		    ub->ub_mmp_delay) * import_intervals);
3294 
3295 		zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp "
3296 		    "mmp_interval=%llu ub_mmp_delay=%llu "
3297 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3298 		    (u_longlong_t)MMP_INTERVAL(ub),
3299 		    (u_longlong_t)ub->ub_mmp_delay,
3300 		    (u_longlong_t)import_intervals);
3301 
3302 	} else if (MMP_VALID(ub)) {
3303 		/*
3304 		 * zfs-0.7 compatibility case
3305 		 */
3306 
3307 		import_delay = MAX(import_delay, (multihost_interval +
3308 		    ub->ub_mmp_delay) * import_intervals);
3309 
3310 		zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu "
3311 		    "import_intervals=%llu leaves=%u",
3312 		    (u_longlong_t)import_delay,
3313 		    (u_longlong_t)ub->ub_mmp_delay,
3314 		    (u_longlong_t)import_intervals,
3315 		    vdev_count_leaves(spa));
3316 	} else {
3317 		/* Using local tunings is the only reasonable option */
3318 		zfs_dbgmsg("pool last imported on non-MMP aware "
3319 		    "host using import_delay=%llu multihost_interval=%llu "
3320 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3321 		    (u_longlong_t)multihost_interval,
3322 		    (u_longlong_t)import_intervals);
3323 	}
3324 
3325 	return (import_delay);
3326 }
3327 
3328 /*
3329  * Perform the import activity check.  If the user canceled the import or
3330  * we detected activity then fail.
3331  */
3332 static int
3333 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config)
3334 {
3335 	uint64_t txg = ub->ub_txg;
3336 	uint64_t timestamp = ub->ub_timestamp;
3337 	uint64_t mmp_config = ub->ub_mmp_config;
3338 	uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0;
3339 	uint64_t import_delay;
3340 	hrtime_t import_expire;
3341 	nvlist_t *mmp_label = NULL;
3342 	vdev_t *rvd = spa->spa_root_vdev;
3343 	kcondvar_t cv;
3344 	kmutex_t mtx;
3345 	int error = 0;
3346 
3347 	cv_init(&cv, NULL, CV_DEFAULT, NULL);
3348 	mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL);
3349 	mutex_enter(&mtx);
3350 
3351 	/*
3352 	 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed
3353 	 * during the earlier tryimport.  If the txg recorded there is 0 then
3354 	 * the pool is known to be active on another host.
3355 	 *
3356 	 * Otherwise, the pool might be in use on another host.  Check for
3357 	 * changes in the uberblocks on disk if necessary.
3358 	 */
3359 	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
3360 		nvlist_t *nvinfo = fnvlist_lookup_nvlist(config,
3361 		    ZPOOL_CONFIG_LOAD_INFO);
3362 
3363 		if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) &&
3364 		    fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) {
3365 			vdev_uberblock_load(rvd, ub, &mmp_label);
3366 			error = SET_ERROR(EREMOTEIO);
3367 			goto out;
3368 		}
3369 	}
3370 
3371 	import_delay = spa_activity_check_duration(spa, ub);
3372 
3373 	/* Add a small random factor in case of simultaneous imports (0-25%) */
3374 	import_delay += import_delay * random_in_range(250) / 1000;
3375 
3376 	import_expire = gethrtime() + import_delay;
3377 
3378 	while (gethrtime() < import_expire) {
3379 		(void) spa_import_progress_set_mmp_check(spa_guid(spa),
3380 		    NSEC2SEC(import_expire - gethrtime()));
3381 
3382 		vdev_uberblock_load(rvd, ub, &mmp_label);
3383 
3384 		if (txg != ub->ub_txg || timestamp != ub->ub_timestamp ||
3385 		    mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) {
3386 			zfs_dbgmsg("multihost activity detected "
3387 			    "txg %llu ub_txg  %llu "
3388 			    "timestamp %llu ub_timestamp  %llu "
3389 			    "mmp_config %#llx ub_mmp_config %#llx",
3390 			    (u_longlong_t)txg, (u_longlong_t)ub->ub_txg,
3391 			    (u_longlong_t)timestamp,
3392 			    (u_longlong_t)ub->ub_timestamp,
3393 			    (u_longlong_t)mmp_config,
3394 			    (u_longlong_t)ub->ub_mmp_config);
3395 
3396 			error = SET_ERROR(EREMOTEIO);
3397 			break;
3398 		}
3399 
3400 		if (mmp_label) {
3401 			nvlist_free(mmp_label);
3402 			mmp_label = NULL;
3403 		}
3404 
3405 		error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz);
3406 		if (error != -1) {
3407 			error = SET_ERROR(EINTR);
3408 			break;
3409 		}
3410 		error = 0;
3411 	}
3412 
3413 out:
3414 	mutex_exit(&mtx);
3415 	mutex_destroy(&mtx);
3416 	cv_destroy(&cv);
3417 
3418 	/*
3419 	 * If the pool is determined to be active store the status in the
3420 	 * spa->spa_load_info nvlist.  If the remote hostname or hostid are
3421 	 * available from configuration read from disk store them as well.
3422 	 * This allows 'zpool import' to generate a more useful message.
3423 	 *
3424 	 * ZPOOL_CONFIG_MMP_STATE    - observed pool status (mandatory)
3425 	 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool
3426 	 * ZPOOL_CONFIG_MMP_HOSTID   - hostid from the active pool
3427 	 */
3428 	if (error == EREMOTEIO) {
3429 		const char *hostname = "<unknown>";
3430 		uint64_t hostid = 0;
3431 
3432 		if (mmp_label) {
3433 			if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) {
3434 				hostname = fnvlist_lookup_string(mmp_label,
3435 				    ZPOOL_CONFIG_HOSTNAME);
3436 				fnvlist_add_string(spa->spa_load_info,
3437 				    ZPOOL_CONFIG_MMP_HOSTNAME, hostname);
3438 			}
3439 
3440 			if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) {
3441 				hostid = fnvlist_lookup_uint64(mmp_label,
3442 				    ZPOOL_CONFIG_HOSTID);
3443 				fnvlist_add_uint64(spa->spa_load_info,
3444 				    ZPOOL_CONFIG_MMP_HOSTID, hostid);
3445 			}
3446 		}
3447 
3448 		fnvlist_add_uint64(spa->spa_load_info,
3449 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE);
3450 		fnvlist_add_uint64(spa->spa_load_info,
3451 		    ZPOOL_CONFIG_MMP_TXG, 0);
3452 
3453 		error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO);
3454 	}
3455 
3456 	if (mmp_label)
3457 		nvlist_free(mmp_label);
3458 
3459 	return (error);
3460 }
3461 
3462 static int
3463 spa_verify_host(spa_t *spa, nvlist_t *mos_config)
3464 {
3465 	uint64_t hostid;
3466 	const char *hostname;
3467 	uint64_t myhostid = 0;
3468 
3469 	if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config,
3470 	    ZPOOL_CONFIG_HOSTID, &hostid) == 0) {
3471 		hostname = fnvlist_lookup_string(mos_config,
3472 		    ZPOOL_CONFIG_HOSTNAME);
3473 
3474 		myhostid = zone_get_hostid(NULL);
3475 
3476 		if (hostid != 0 && myhostid != 0 && hostid != myhostid) {
3477 			cmn_err(CE_WARN, "pool '%s' could not be "
3478 			    "loaded as it was last accessed by "
3479 			    "another system (host: %s hostid: 0x%llx). "
3480 			    "See: https://openzfs.github.io/openzfs-docs/msg/"
3481 			    "ZFS-8000-EY",
3482 			    spa_name(spa), hostname, (u_longlong_t)hostid);
3483 			spa_load_failed(spa, "hostid verification failed: pool "
3484 			    "last accessed by host: %s (hostid: 0x%llx)",
3485 			    hostname, (u_longlong_t)hostid);
3486 			return (SET_ERROR(EBADF));
3487 		}
3488 	}
3489 
3490 	return (0);
3491 }
3492 
3493 static int
3494 spa_ld_parse_config(spa_t *spa, spa_import_type_t type)
3495 {
3496 	int error = 0;
3497 	nvlist_t *nvtree, *nvl, *config = spa->spa_config;
3498 	int parse;
3499 	vdev_t *rvd;
3500 	uint64_t pool_guid;
3501 	const char *comment;
3502 	const char *compatibility;
3503 
3504 	/*
3505 	 * Versioning wasn't explicitly added to the label until later, so if
3506 	 * it's not present treat it as the initial version.
3507 	 */
3508 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
3509 	    &spa->spa_ubsync.ub_version) != 0)
3510 		spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;
3511 
3512 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) {
3513 		spa_load_failed(spa, "invalid config provided: '%s' missing",
3514 		    ZPOOL_CONFIG_POOL_GUID);
3515 		return (SET_ERROR(EINVAL));
3516 	}
3517 
3518 	/*
3519 	 * If we are doing an import, ensure that the pool is not already
3520 	 * imported by checking if its pool guid already exists in the
3521 	 * spa namespace.
3522 	 *
3523 	 * The only case that we allow an already imported pool to be
3524 	 * imported again, is when the pool is checkpointed and we want to
3525 	 * look at its checkpointed state from userland tools like zdb.
3526 	 */
3527 #ifdef _KERNEL
3528 	if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
3529 	    spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
3530 	    spa_guid_exists(pool_guid, 0)) {
3531 #else
3532 	if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
3533 	    spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
3534 	    spa_guid_exists(pool_guid, 0) &&
3535 	    !spa_importing_readonly_checkpoint(spa)) {
3536 #endif
3537 		spa_load_failed(spa, "a pool with guid %llu is already open",
3538 		    (u_longlong_t)pool_guid);
3539 		return (SET_ERROR(EEXIST));
3540 	}
3541 
3542 	spa->spa_config_guid = pool_guid;
3543 
3544 	nvlist_free(spa->spa_load_info);
3545 	spa->spa_load_info = fnvlist_alloc();
3546 
3547 	ASSERT(spa->spa_comment == NULL);
3548 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
3549 		spa->spa_comment = spa_strdup(comment);
3550 
3551 	ASSERT(spa->spa_compatibility == NULL);
3552 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY,
3553 	    &compatibility) == 0)
3554 		spa->spa_compatibility = spa_strdup(compatibility);
3555 
3556 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
3557 	    &spa->spa_config_txg);
3558 
3559 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0)
3560 		spa->spa_config_splitting = fnvlist_dup(nvl);
3561 
3562 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) {
3563 		spa_load_failed(spa, "invalid config provided: '%s' missing",
3564 		    ZPOOL_CONFIG_VDEV_TREE);
3565 		return (SET_ERROR(EINVAL));
3566 	}
3567 
3568 	/*
3569 	 * Create "The Godfather" zio to hold all async IOs
3570 	 */
3571 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
3572 	    KM_SLEEP);
3573 	for (int i = 0; i < max_ncpus; i++) {
3574 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
3575 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
3576 		    ZIO_FLAG_GODFATHER);
3577 	}
3578 
3579 	/*
3580 	 * Parse the configuration into a vdev tree.  We explicitly set the
3581 	 * value that will be returned by spa_version() since parsing the
3582 	 * configuration requires knowing the version number.
3583 	 */
3584 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3585 	parse = (type == SPA_IMPORT_EXISTING ?
3586 	    VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT);
3587 	error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse);
3588 	spa_config_exit(spa, SCL_ALL, FTAG);
3589 
3590 	if (error != 0) {
3591 		spa_load_failed(spa, "unable to parse config [error=%d]",
3592 		    error);
3593 		return (error);
3594 	}
3595 
3596 	ASSERT(spa->spa_root_vdev == rvd);
3597 	ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
3598 	ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);
3599 
3600 	if (type != SPA_IMPORT_ASSEMBLE) {
3601 		ASSERT(spa_guid(spa) == pool_guid);
3602 	}
3603 
3604 	return (0);
3605 }
3606 
3607 /*
3608  * Recursively open all vdevs in the vdev tree. This function is called twice:
3609  * first with the untrusted config, then with the trusted config.
3610  */
3611 static int
3612 spa_ld_open_vdevs(spa_t *spa)
3613 {
3614 	int error = 0;
3615 
3616 	/*
3617 	 * spa_missing_tvds_allowed defines how many top-level vdevs can be
3618 	 * missing/unopenable for the root vdev to be still considered openable.
3619 	 */
3620 	if (spa->spa_trust_config) {
3621 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds;
3622 	} else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) {
3623 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile;
3624 	} else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) {
3625 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan;
3626 	} else {
3627 		spa->spa_missing_tvds_allowed = 0;
3628 	}
3629 
3630 	spa->spa_missing_tvds_allowed =
3631 	    MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed);
3632 
3633 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3634 	error = vdev_open(spa->spa_root_vdev);
3635 	spa_config_exit(spa, SCL_ALL, FTAG);
3636 
3637 	if (spa->spa_missing_tvds != 0) {
3638 		spa_load_note(spa, "vdev tree has %lld missing top-level "
3639 		    "vdevs.", (u_longlong_t)spa->spa_missing_tvds);
3640 		if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) {
3641 			/*
3642 			 * Although theoretically we could allow users to open
3643 			 * incomplete pools in RW mode, we'd need to add a lot
3644 			 * of extra logic (e.g. adjust pool space to account
3645 			 * for missing vdevs).
3646 			 * This limitation also prevents users from accidentally
3647 			 * opening the pool in RW mode during data recovery and
3648 			 * damaging it further.
3649 			 */
3650 			spa_load_note(spa, "pools with missing top-level "
3651 			    "vdevs can only be opened in read-only mode.");
3652 			error = SET_ERROR(ENXIO);
3653 		} else {
3654 			spa_load_note(spa, "current settings allow for maximum "
3655 			    "%lld missing top-level vdevs at this stage.",
3656 			    (u_longlong_t)spa->spa_missing_tvds_allowed);
3657 		}
3658 	}
3659 	if (error != 0) {
3660 		spa_load_failed(spa, "unable to open vdev tree [error=%d]",
3661 		    error);
3662 	}
3663 	if (spa->spa_missing_tvds != 0 || error != 0)
3664 		vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2);
3665 
3666 	return (error);
3667 }
3668 
3669 /*
3670  * We need to validate the vdev labels against the configuration that
3671  * we have in hand. This function is called twice: first with an untrusted
3672  * config, then with a trusted config. The validation is more strict when the
3673  * config is trusted.
3674  */
3675 static int
3676 spa_ld_validate_vdevs(spa_t *spa)
3677 {
3678 	int error = 0;
3679 	vdev_t *rvd = spa->spa_root_vdev;
3680 
3681 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3682 	error = vdev_validate(rvd);
3683 	spa_config_exit(spa, SCL_ALL, FTAG);
3684 
3685 	if (error != 0) {
3686 		spa_load_failed(spa, "vdev_validate failed [error=%d]", error);
3687 		return (error);
3688 	}
3689 
3690 	if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
3691 		spa_load_failed(spa, "cannot open vdev tree after invalidating "
3692 		    "some vdevs");
3693 		vdev_dbgmsg_print_tree(rvd, 2);
3694 		return (SET_ERROR(ENXIO));
3695 	}
3696 
3697 	return (0);
3698 }
3699 
3700 static void
3701 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub)
3702 {
3703 	spa->spa_state = POOL_STATE_ACTIVE;
3704 	spa->spa_ubsync = spa->spa_uberblock;
3705 	spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
3706 	    TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
3707 	spa->spa_first_txg = spa->spa_last_ubsync_txg ?
3708 	    spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
3709 	spa->spa_claim_max_txg = spa->spa_first_txg;
3710 	spa->spa_prev_software_version = ub->ub_software_version;
3711 }
3712 
3713 static int
3714 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type)
3715 {
3716 	vdev_t *rvd = spa->spa_root_vdev;
3717 	nvlist_t *label;
3718 	uberblock_t *ub = &spa->spa_uberblock;
3719 	boolean_t activity_check = B_FALSE;
3720 
3721 	/*
3722 	 * If we are opening the checkpointed state of the pool by
3723 	 * rewinding to it, at this point we will have written the
3724 	 * checkpointed uberblock to the vdev labels, so searching
3725 	 * the labels will find the right uberblock.  However, if
3726 	 * we are opening the checkpointed state read-only, we have
3727 	 * not modified the labels. Therefore, we must ignore the
3728 	 * labels and continue using the spa_uberblock that was set
3729 	 * by spa_ld_checkpoint_rewind.
3730 	 *
3731 	 * Note that it would be fine to ignore the labels when
3732 	 * rewinding (opening writeable) as well. However, if we
3733 	 * crash just after writing the labels, we will end up
3734 	 * searching the labels. Doing so in the common case means
3735 	 * that this code path gets exercised normally, rather than
3736 	 * just in the edge case.
3737 	 */
3738 	if (ub->ub_checkpoint_txg != 0 &&
3739 	    spa_importing_readonly_checkpoint(spa)) {
3740 		spa_ld_select_uberblock_done(spa, ub);
3741 		return (0);
3742 	}
3743 
3744 	/*
3745 	 * Find the best uberblock.
3746 	 */
3747 	vdev_uberblock_load(rvd, ub, &label);
3748 
3749 	/*
3750 	 * If we weren't able to find a single valid uberblock, return failure.
3751 	 */
3752 	if (ub->ub_txg == 0) {
3753 		nvlist_free(label);
3754 		spa_load_failed(spa, "no valid uberblock found");
3755 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO));
3756 	}
3757 
3758 	if (spa->spa_load_max_txg != UINT64_MAX) {
3759 		(void) spa_import_progress_set_max_txg(spa_guid(spa),
3760 		    (u_longlong_t)spa->spa_load_max_txg);
3761 	}
3762 	spa_load_note(spa, "using uberblock with txg=%llu",
3763 	    (u_longlong_t)ub->ub_txg);
3764 	if (ub->ub_raidz_reflow_info != 0) {
3765 		spa_load_note(spa, "uberblock raidz_reflow_info: "
3766 		    "state=%u offset=%llu",
3767 		    (int)RRSS_GET_STATE(ub),
3768 		    (u_longlong_t)RRSS_GET_OFFSET(ub));
3769 	}
3770 
3771 
3772 	/*
3773 	 * For pools which have the multihost property on determine if the
3774 	 * pool is truly inactive and can be safely imported.  Prevent
3775 	 * hosts which don't have a hostid set from importing the pool.
3776 	 */
3777 	activity_check = spa_activity_check_required(spa, ub, label,
3778 	    spa->spa_config);
3779 	if (activity_check) {
3780 		if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay &&
3781 		    spa_get_hostid(spa) == 0) {
3782 			nvlist_free(label);
3783 			fnvlist_add_uint64(spa->spa_load_info,
3784 			    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
3785 			return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
3786 		}
3787 
3788 		int error = spa_activity_check(spa, ub, spa->spa_config);
3789 		if (error) {
3790 			nvlist_free(label);
3791 			return (error);
3792 		}
3793 
3794 		fnvlist_add_uint64(spa->spa_load_info,
3795 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE);
3796 		fnvlist_add_uint64(spa->spa_load_info,
3797 		    ZPOOL_CONFIG_MMP_TXG, ub->ub_txg);
3798 		fnvlist_add_uint16(spa->spa_load_info,
3799 		    ZPOOL_CONFIG_MMP_SEQ,
3800 		    (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0));
3801 	}
3802 
3803 	/*
3804 	 * If the pool has an unsupported version we can't open it.
3805 	 */
3806 	if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
3807 		nvlist_free(label);
3808 		spa_load_failed(spa, "version %llu is not supported",
3809 		    (u_longlong_t)ub->ub_version);
3810 		return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP));
3811 	}
3812 
3813 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
3814 		nvlist_t *features;
3815 
3816 		/*
3817 		 * If we weren't able to find what's necessary for reading the
3818 		 * MOS in the label, return failure.
3819 		 */
3820 		if (label == NULL) {
3821 			spa_load_failed(spa, "label config unavailable");
3822 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
3823 			    ENXIO));
3824 		}
3825 
3826 		if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ,
3827 		    &features) != 0) {
3828 			nvlist_free(label);
3829 			spa_load_failed(spa, "invalid label: '%s' missing",
3830 			    ZPOOL_CONFIG_FEATURES_FOR_READ);
3831 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
3832 			    ENXIO));
3833 		}
3834 
3835 		/*
3836 		 * Update our in-core representation with the definitive values
3837 		 * from the label.
3838 		 */
3839 		nvlist_free(spa->spa_label_features);
3840 		spa->spa_label_features = fnvlist_dup(features);
3841 	}
3842 
3843 	nvlist_free(label);
3844 
3845 	/*
3846 	 * Look through entries in the label nvlist's features_for_read. If
3847 	 * there is a feature listed there which we don't understand then we
3848 	 * cannot open a pool.
3849 	 */
3850 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
3851 		nvlist_t *unsup_feat;
3852 
3853 		unsup_feat = fnvlist_alloc();
3854 
3855 		for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
3856 		    NULL); nvp != NULL;
3857 		    nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
3858 			if (!zfeature_is_supported(nvpair_name(nvp))) {
3859 				fnvlist_add_string(unsup_feat,
3860 				    nvpair_name(nvp), "");
3861 			}
3862 		}
3863 
3864 		if (!nvlist_empty(unsup_feat)) {
3865 			fnvlist_add_nvlist(spa->spa_load_info,
3866 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
3867 			nvlist_free(unsup_feat);
3868 			spa_load_failed(spa, "some features are unsupported");
3869 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
3870 			    ENOTSUP));
3871 		}
3872 
3873 		nvlist_free(unsup_feat);
3874 	}
3875 
3876 	if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
3877 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3878 		spa_try_repair(spa, spa->spa_config);
3879 		spa_config_exit(spa, SCL_ALL, FTAG);
3880 		nvlist_free(spa->spa_config_splitting);
3881 		spa->spa_config_splitting = NULL;
3882 	}
3883 
3884 	/*
3885 	 * Initialize internal SPA structures.
3886 	 */
3887 	spa_ld_select_uberblock_done(spa, ub);
3888 
3889 	return (0);
3890 }
3891 
3892 static int
3893 spa_ld_open_rootbp(spa_t *spa)
3894 {
3895 	int error = 0;
3896 	vdev_t *rvd = spa->spa_root_vdev;
3897 
3898 	error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
3899 	if (error != 0) {
3900 		spa_load_failed(spa, "unable to open rootbp in dsl_pool_init "
3901 		    "[error=%d]", error);
3902 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3903 	}
3904 	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
3905 
3906 	return (0);
3907 }
3908 
3909 static int
3910 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type,
3911     boolean_t reloading)
3912 {
3913 	vdev_t *mrvd, *rvd = spa->spa_root_vdev;
3914 	nvlist_t *nv, *mos_config, *policy;
3915 	int error = 0, copy_error;
3916 	uint64_t healthy_tvds, healthy_tvds_mos;
3917 	uint64_t mos_config_txg;
3918 
3919 	if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE)
3920 	    != 0)
3921 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3922 
3923 	/*
3924 	 * If we're assembling a pool from a split, the config provided is
3925 	 * already trusted so there is nothing to do.
3926 	 */
3927 	if (type == SPA_IMPORT_ASSEMBLE)
3928 		return (0);
3929 
3930 	healthy_tvds = spa_healthy_core_tvds(spa);
3931 
3932 	if (load_nvlist(spa, spa->spa_config_object, &mos_config)
3933 	    != 0) {
3934 		spa_load_failed(spa, "unable to retrieve MOS config");
3935 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3936 	}
3937 
3938 	/*
3939 	 * If we are doing an open, pool owner wasn't verified yet, thus do
3940 	 * the verification here.
3941 	 */
3942 	if (spa->spa_load_state == SPA_LOAD_OPEN) {
3943 		error = spa_verify_host(spa, mos_config);
3944 		if (error != 0) {
3945 			nvlist_free(mos_config);
3946 			return (error);
3947 		}
3948 	}
3949 
3950 	nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE);
3951 
3952 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3953 
3954 	/*
3955 	 * Build a new vdev tree from the trusted config
3956 	 */
3957 	error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD);
3958 	if (error != 0) {
3959 		nvlist_free(mos_config);
3960 		spa_config_exit(spa, SCL_ALL, FTAG);
3961 		spa_load_failed(spa, "spa_config_parse failed [error=%d]",
3962 		    error);
3963 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
3964 	}
3965 
3966 	/*
3967 	 * Vdev paths in the MOS may be obsolete. If the untrusted config was
3968 	 * obtained by scanning /dev/dsk, then it will have the right vdev
3969 	 * paths. We update the trusted MOS config with this information.
3970 	 * We first try to copy the paths with vdev_copy_path_strict, which
3971 	 * succeeds only when both configs have exactly the same vdev tree.
3972 	 * If that fails, we fall back to a more flexible method that has a
3973 	 * best effort policy.
3974 	 */
3975 	copy_error = vdev_copy_path_strict(rvd, mrvd);
3976 	if (copy_error != 0 || spa_load_print_vdev_tree) {
3977 		spa_load_note(spa, "provided vdev tree:");
3978 		vdev_dbgmsg_print_tree(rvd, 2);
3979 		spa_load_note(spa, "MOS vdev tree:");
3980 		vdev_dbgmsg_print_tree(mrvd, 2);
3981 	}
3982 	if (copy_error != 0) {
3983 		spa_load_note(spa, "vdev_copy_path_strict failed, falling "
3984 		    "back to vdev_copy_path_relaxed");
3985 		vdev_copy_path_relaxed(rvd, mrvd);
3986 	}
3987 
3988 	vdev_close(rvd);
3989 	vdev_free(rvd);
3990 	spa->spa_root_vdev = mrvd;
3991 	rvd = mrvd;
3992 	spa_config_exit(spa, SCL_ALL, FTAG);
3993 
3994 	/*
3995 	 * If 'zpool import' used a cached config, then the on-disk hostid and
3996 	 * hostname may be different to the cached config in ways that should
3997 	 * prevent import.  Userspace can't discover this without a scan, but
3998 	 * we know, so we add these values to LOAD_INFO so the caller can know
3999 	 * the difference.
4000 	 *
4001 	 * Note that we have to do this before the config is regenerated,
4002 	 * because the new config will have the hostid and hostname for this
4003 	 * host, in readiness for import.
4004 	 */
4005 	if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTID))
4006 		fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID,
4007 		    fnvlist_lookup_uint64(mos_config, ZPOOL_CONFIG_HOSTID));
4008 	if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTNAME))
4009 		fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME,
4010 		    fnvlist_lookup_string(mos_config, ZPOOL_CONFIG_HOSTNAME));
4011 
4012 	/*
4013 	 * We will use spa_config if we decide to reload the spa or if spa_load
4014 	 * fails and we rewind. We must thus regenerate the config using the
4015 	 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to
4016 	 * pass settings on how to load the pool and is not stored in the MOS.
4017 	 * We copy it over to our new, trusted config.
4018 	 */
4019 	mos_config_txg = fnvlist_lookup_uint64(mos_config,
4020 	    ZPOOL_CONFIG_POOL_TXG);
4021 	nvlist_free(mos_config);
4022 	mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE);
4023 	if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY,
4024 	    &policy) == 0)
4025 		fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy);
4026 	spa_config_set(spa, mos_config);
4027 	spa->spa_config_source = SPA_CONFIG_SRC_MOS;
4028 
4029 	/*
4030 	 * Now that we got the config from the MOS, we should be more strict
4031 	 * in checking blkptrs and can make assumptions about the consistency
4032 	 * of the vdev tree. spa_trust_config must be set to true before opening
4033 	 * vdevs in order for them to be writeable.
4034 	 */
4035 	spa->spa_trust_config = B_TRUE;
4036 
4037 	/*
4038 	 * Open and validate the new vdev tree
4039 	 */
4040 	error = spa_ld_open_vdevs(spa);
4041 	if (error != 0)
4042 		return (error);
4043 
4044 	error = spa_ld_validate_vdevs(spa);
4045 	if (error != 0)
4046 		return (error);
4047 
4048 	if (copy_error != 0 || spa_load_print_vdev_tree) {
4049 		spa_load_note(spa, "final vdev tree:");
4050 		vdev_dbgmsg_print_tree(rvd, 2);
4051 	}
4052 
4053 	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT &&
4054 	    !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) {
4055 		/*
4056 		 * Sanity check to make sure that we are indeed loading the
4057 		 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds
4058 		 * in the config provided and they happened to be the only ones
4059 		 * to have the latest uberblock, we could involuntarily perform
4060 		 * an extreme rewind.
4061 		 */
4062 		healthy_tvds_mos = spa_healthy_core_tvds(spa);
4063 		if (healthy_tvds_mos - healthy_tvds >=
4064 		    SPA_SYNC_MIN_VDEVS) {
4065 			spa_load_note(spa, "config provided misses too many "
4066 			    "top-level vdevs compared to MOS (%lld vs %lld). ",
4067 			    (u_longlong_t)healthy_tvds,
4068 			    (u_longlong_t)healthy_tvds_mos);
4069 			spa_load_note(spa, "vdev tree:");
4070 			vdev_dbgmsg_print_tree(rvd, 2);
4071 			if (reloading) {
4072 				spa_load_failed(spa, "config was already "
4073 				    "provided from MOS. Aborting.");
4074 				return (spa_vdev_err(rvd,
4075 				    VDEV_AUX_CORRUPT_DATA, EIO));
4076 			}
4077 			spa_load_note(spa, "spa must be reloaded using MOS "
4078 			    "config");
4079 			return (SET_ERROR(EAGAIN));
4080 		}
4081 	}
4082 
4083 	error = spa_check_for_missing_logs(spa);
4084 	if (error != 0)
4085 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO));
4086 
4087 	if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) {
4088 		spa_load_failed(spa, "uberblock guid sum doesn't match MOS "
4089 		    "guid sum (%llu != %llu)",
4090 		    (u_longlong_t)spa->spa_uberblock.ub_guid_sum,
4091 		    (u_longlong_t)rvd->vdev_guid_sum);
4092 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM,
4093 		    ENXIO));
4094 	}
4095 
4096 	return (0);
4097 }
4098 
4099 static int
4100 spa_ld_open_indirect_vdev_metadata(spa_t *spa)
4101 {
4102 	int error = 0;
4103 	vdev_t *rvd = spa->spa_root_vdev;
4104 
4105 	/*
4106 	 * Everything that we read before spa_remove_init() must be stored
4107 	 * on concreted vdevs.  Therefore we do this as early as possible.
4108 	 */
4109 	error = spa_remove_init(spa);
4110 	if (error != 0) {
4111 		spa_load_failed(spa, "spa_remove_init failed [error=%d]",
4112 		    error);
4113 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4114 	}
4115 
4116 	/*
4117 	 * Retrieve information needed to condense indirect vdev mappings.
4118 	 */
4119 	error = spa_condense_init(spa);
4120 	if (error != 0) {
4121 		spa_load_failed(spa, "spa_condense_init failed [error=%d]",
4122 		    error);
4123 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4124 	}
4125 
4126 	return (0);
4127 }
4128 
4129 static int
4130 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep)
4131 {
4132 	int error = 0;
4133 	vdev_t *rvd = spa->spa_root_vdev;
4134 
4135 	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
4136 		boolean_t missing_feat_read = B_FALSE;
4137 		nvlist_t *unsup_feat, *enabled_feat;
4138 
4139 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
4140 		    &spa->spa_feat_for_read_obj, B_TRUE) != 0) {
4141 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4142 		}
4143 
4144 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE,
4145 		    &spa->spa_feat_for_write_obj, B_TRUE) != 0) {
4146 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4147 		}
4148 
4149 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS,
4150 		    &spa->spa_feat_desc_obj, B_TRUE) != 0) {
4151 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4152 		}
4153 
4154 		enabled_feat = fnvlist_alloc();
4155 		unsup_feat = fnvlist_alloc();
4156 
4157 		if (!spa_features_check(spa, B_FALSE,
4158 		    unsup_feat, enabled_feat))
4159 			missing_feat_read = B_TRUE;
4160 
4161 		if (spa_writeable(spa) ||
4162 		    spa->spa_load_state == SPA_LOAD_TRYIMPORT) {
4163 			if (!spa_features_check(spa, B_TRUE,
4164 			    unsup_feat, enabled_feat)) {
4165 				*missing_feat_writep = B_TRUE;
4166 			}
4167 		}
4168 
4169 		fnvlist_add_nvlist(spa->spa_load_info,
4170 		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
4171 
4172 		if (!nvlist_empty(unsup_feat)) {
4173 			fnvlist_add_nvlist(spa->spa_load_info,
4174 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
4175 		}
4176 
4177 		fnvlist_free(enabled_feat);
4178 		fnvlist_free(unsup_feat);
4179 
4180 		if (!missing_feat_read) {
4181 			fnvlist_add_boolean(spa->spa_load_info,
4182 			    ZPOOL_CONFIG_CAN_RDONLY);
4183 		}
4184 
4185 		/*
4186 		 * If the state is SPA_LOAD_TRYIMPORT, our objective is
4187 		 * twofold: to determine whether the pool is available for
4188 		 * import in read-write mode and (if it is not) whether the
4189 		 * pool is available for import in read-only mode. If the pool
4190 		 * is available for import in read-write mode, it is displayed
4191 		 * as available in userland; if it is not available for import
4192 		 * in read-only mode, it is displayed as unavailable in
4193 		 * userland. If the pool is available for import in read-only
4194 		 * mode but not read-write mode, it is displayed as unavailable
4195 		 * in userland with a special note that the pool is actually
4196 		 * available for open in read-only mode.
4197 		 *
4198 		 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are
4199 		 * missing a feature for write, we must first determine whether
4200 		 * the pool can be opened read-only before returning to
4201 		 * userland in order to know whether to display the
4202 		 * abovementioned note.
4203 		 */
4204 		if (missing_feat_read || (*missing_feat_writep &&
4205 		    spa_writeable(spa))) {
4206 			spa_load_failed(spa, "pool uses unsupported features");
4207 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
4208 			    ENOTSUP));
4209 		}
4210 
4211 		/*
4212 		 * Load refcounts for ZFS features from disk into an in-memory
4213 		 * cache during SPA initialization.
4214 		 */
4215 		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
4216 			uint64_t refcount;
4217 
4218 			error = feature_get_refcount_from_disk(spa,
4219 			    &spa_feature_table[i], &refcount);
4220 			if (error == 0) {
4221 				spa->spa_feat_refcount_cache[i] = refcount;
4222 			} else if (error == ENOTSUP) {
4223 				spa->spa_feat_refcount_cache[i] =
4224 				    SPA_FEATURE_DISABLED;
4225 			} else {
4226 				spa_load_failed(spa, "error getting refcount "
4227 				    "for feature %s [error=%d]",
4228 				    spa_feature_table[i].fi_guid, error);
4229 				return (spa_vdev_err(rvd,
4230 				    VDEV_AUX_CORRUPT_DATA, EIO));
4231 			}
4232 		}
4233 	}
4234 
4235 	if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) {
4236 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG,
4237 		    &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0)
4238 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4239 	}
4240 
4241 	/*
4242 	 * Encryption was added before bookmark_v2, even though bookmark_v2
4243 	 * is now a dependency. If this pool has encryption enabled without
4244 	 * bookmark_v2, trigger an errata message.
4245 	 */
4246 	if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) &&
4247 	    !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) {
4248 		spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
4249 	}
4250 
4251 	return (0);
4252 }
4253 
4254 static int
4255 spa_ld_load_special_directories(spa_t *spa)
4256 {
4257 	int error = 0;
4258 	vdev_t *rvd = spa->spa_root_vdev;
4259 
4260 	spa->spa_is_initializing = B_TRUE;
4261 	error = dsl_pool_open(spa->spa_dsl_pool);
4262 	spa->spa_is_initializing = B_FALSE;
4263 	if (error != 0) {
4264 		spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error);
4265 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4266 	}
4267 
4268 	return (0);
4269 }
4270 
4271 static int
4272 spa_ld_get_props(spa_t *spa)
4273 {
4274 	int error = 0;
4275 	uint64_t obj;
4276 	vdev_t *rvd = spa->spa_root_vdev;
4277 
4278 	/* Grab the checksum salt from the MOS. */
4279 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4280 	    DMU_POOL_CHECKSUM_SALT, 1,
4281 	    sizeof (spa->spa_cksum_salt.zcs_bytes),
4282 	    spa->spa_cksum_salt.zcs_bytes);
4283 	if (error == ENOENT) {
4284 		/* Generate a new salt for subsequent use */
4285 		(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
4286 		    sizeof (spa->spa_cksum_salt.zcs_bytes));
4287 	} else if (error != 0) {
4288 		spa_load_failed(spa, "unable to retrieve checksum salt from "
4289 		    "MOS [error=%d]", error);
4290 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4291 	}
4292 
4293 	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0)
4294 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4295 	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
4296 	if (error != 0) {
4297 		spa_load_failed(spa, "error opening deferred-frees bpobj "
4298 		    "[error=%d]", error);
4299 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4300 	}
4301 
4302 	/*
4303 	 * Load the bit that tells us to use the new accounting function
4304 	 * (raid-z deflation).  If we have an older pool, this will not
4305 	 * be present.
4306 	 */
4307 	error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE);
4308 	if (error != 0 && error != ENOENT)
4309 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4310 
4311 	error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION,
4312 	    &spa->spa_creation_version, B_FALSE);
4313 	if (error != 0 && error != ENOENT)
4314 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4315 
4316 	/*
4317 	 * Load the persistent error log.  If we have an older pool, this will
4318 	 * not be present.
4319 	 */
4320 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last,
4321 	    B_FALSE);
4322 	if (error != 0 && error != ENOENT)
4323 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4324 
4325 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB,
4326 	    &spa->spa_errlog_scrub, B_FALSE);
4327 	if (error != 0 && error != ENOENT)
4328 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4329 
4330 	/*
4331 	 * Load the livelist deletion field. If a livelist is queued for
4332 	 * deletion, indicate that in the spa
4333 	 */
4334 	error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES,
4335 	    &spa->spa_livelists_to_delete, B_FALSE);
4336 	if (error != 0 && error != ENOENT)
4337 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4338 
4339 	/*
4340 	 * Load the history object.  If we have an older pool, this
4341 	 * will not be present.
4342 	 */
4343 	error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE);
4344 	if (error != 0 && error != ENOENT)
4345 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4346 
4347 	/*
4348 	 * Load the per-vdev ZAP map. If we have an older pool, this will not
4349 	 * be present; in this case, defer its creation to a later time to
4350 	 * avoid dirtying the MOS this early / out of sync context. See
4351 	 * spa_sync_config_object.
4352 	 */
4353 
4354 	/* The sentinel is only available in the MOS config. */
4355 	nvlist_t *mos_config;
4356 	if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) {
4357 		spa_load_failed(spa, "unable to retrieve MOS config");
4358 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4359 	}
4360 
4361 	error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP,
4362 	    &spa->spa_all_vdev_zaps, B_FALSE);
4363 
4364 	if (error == ENOENT) {
4365 		VERIFY(!nvlist_exists(mos_config,
4366 		    ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
4367 		spa->spa_avz_action = AVZ_ACTION_INITIALIZE;
4368 		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4369 	} else if (error != 0) {
4370 		nvlist_free(mos_config);
4371 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4372 	} else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) {
4373 		/*
4374 		 * An older version of ZFS overwrote the sentinel value, so
4375 		 * we have orphaned per-vdev ZAPs in the MOS. Defer their
4376 		 * destruction to later; see spa_sync_config_object.
4377 		 */
4378 		spa->spa_avz_action = AVZ_ACTION_DESTROY;
4379 		/*
4380 		 * We're assuming that no vdevs have had their ZAPs created
4381 		 * before this. Better be sure of it.
4382 		 */
4383 		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4384 	}
4385 	nvlist_free(mos_config);
4386 
4387 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
4388 
4389 	error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object,
4390 	    B_FALSE);
4391 	if (error && error != ENOENT)
4392 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4393 
4394 	if (error == 0) {
4395 		uint64_t autoreplace = 0;
4396 
4397 		spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
4398 		spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace);
4399 		spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
4400 		spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
4401 		spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
4402 		spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost);
4403 		spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim);
4404 		spa->spa_autoreplace = (autoreplace != 0);
4405 	}
4406 
4407 	/*
4408 	 * If we are importing a pool with missing top-level vdevs,
4409 	 * we enforce that the pool doesn't panic or get suspended on
4410 	 * error since the likelihood of missing data is extremely high.
4411 	 */
4412 	if (spa->spa_missing_tvds > 0 &&
4413 	    spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE &&
4414 	    spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4415 		spa_load_note(spa, "forcing failmode to 'continue' "
4416 		    "as some top level vdevs are missing");
4417 		spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE;
4418 	}
4419 
4420 	return (0);
4421 }
4422 
4423 static int
4424 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type)
4425 {
4426 	int error = 0;
4427 	vdev_t *rvd = spa->spa_root_vdev;
4428 
4429 	/*
4430 	 * If we're assembling the pool from the split-off vdevs of
4431 	 * an existing pool, we don't want to attach the spares & cache
4432 	 * devices.
4433 	 */
4434 
4435 	/*
4436 	 * Load any hot spares for this pool.
4437 	 */
4438 	error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object,
4439 	    B_FALSE);
4440 	if (error != 0 && error != ENOENT)
4441 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4442 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
4443 		ASSERT(spa_version(spa) >= SPA_VERSION_SPARES);
4444 		if (load_nvlist(spa, spa->spa_spares.sav_object,
4445 		    &spa->spa_spares.sav_config) != 0) {
4446 			spa_load_failed(spa, "error loading spares nvlist");
4447 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4448 		}
4449 
4450 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4451 		spa_load_spares(spa);
4452 		spa_config_exit(spa, SCL_ALL, FTAG);
4453 	} else if (error == 0) {
4454 		spa->spa_spares.sav_sync = B_TRUE;
4455 	}
4456 
4457 	/*
4458 	 * Load any level 2 ARC devices for this pool.
4459 	 */
4460 	error = spa_dir_prop(spa, DMU_POOL_L2CACHE,
4461 	    &spa->spa_l2cache.sav_object, B_FALSE);
4462 	if (error != 0 && error != ENOENT)
4463 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4464 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
4465 		ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE);
4466 		if (load_nvlist(spa, spa->spa_l2cache.sav_object,
4467 		    &spa->spa_l2cache.sav_config) != 0) {
4468 			spa_load_failed(spa, "error loading l2cache nvlist");
4469 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4470 		}
4471 
4472 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4473 		spa_load_l2cache(spa);
4474 		spa_config_exit(spa, SCL_ALL, FTAG);
4475 	} else if (error == 0) {
4476 		spa->spa_l2cache.sav_sync = B_TRUE;
4477 	}
4478 
4479 	return (0);
4480 }
4481 
4482 static int
4483 spa_ld_load_vdev_metadata(spa_t *spa)
4484 {
4485 	int error = 0;
4486 	vdev_t *rvd = spa->spa_root_vdev;
4487 
4488 	/*
4489 	 * If the 'multihost' property is set, then never allow a pool to
4490 	 * be imported when the system hostid is zero.  The exception to
4491 	 * this rule is zdb which is always allowed to access pools.
4492 	 */
4493 	if (spa_multihost(spa) && spa_get_hostid(spa) == 0 &&
4494 	    (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) {
4495 		fnvlist_add_uint64(spa->spa_load_info,
4496 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
4497 		return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
4498 	}
4499 
4500 	/*
4501 	 * If the 'autoreplace' property is set, then post a resource notifying
4502 	 * the ZFS DE that it should not issue any faults for unopenable
4503 	 * devices.  We also iterate over the vdevs, and post a sysevent for any
4504 	 * unopenable vdevs so that the normal autoreplace handler can take
4505 	 * over.
4506 	 */
4507 	if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4508 		spa_check_removed(spa->spa_root_vdev);
4509 		/*
4510 		 * For the import case, this is done in spa_import(), because
4511 		 * at this point we're using the spare definitions from
4512 		 * the MOS config, not necessarily from the userland config.
4513 		 */
4514 		if (spa->spa_load_state != SPA_LOAD_IMPORT) {
4515 			spa_aux_check_removed(&spa->spa_spares);
4516 			spa_aux_check_removed(&spa->spa_l2cache);
4517 		}
4518 	}
4519 
4520 	/*
4521 	 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc.
4522 	 */
4523 	error = vdev_load(rvd);
4524 	if (error != 0) {
4525 		spa_load_failed(spa, "vdev_load failed [error=%d]", error);
4526 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4527 	}
4528 
4529 	error = spa_ld_log_spacemaps(spa);
4530 	if (error != 0) {
4531 		spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]",
4532 		    error);
4533 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4534 	}
4535 
4536 	/*
4537 	 * Propagate the leaf DTLs we just loaded all the way up the vdev tree.
4538 	 */
4539 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4540 	vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE);
4541 	spa_config_exit(spa, SCL_ALL, FTAG);
4542 
4543 	return (0);
4544 }
4545 
4546 static int
4547 spa_ld_load_dedup_tables(spa_t *spa)
4548 {
4549 	int error = 0;
4550 	vdev_t *rvd = spa->spa_root_vdev;
4551 
4552 	error = ddt_load(spa);
4553 	if (error != 0) {
4554 		spa_load_failed(spa, "ddt_load failed [error=%d]", error);
4555 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4556 	}
4557 
4558 	return (0);
4559 }
4560 
4561 static int
4562 spa_ld_load_brt(spa_t *spa)
4563 {
4564 	int error = 0;
4565 	vdev_t *rvd = spa->spa_root_vdev;
4566 
4567 	error = brt_load(spa);
4568 	if (error != 0) {
4569 		spa_load_failed(spa, "brt_load failed [error=%d]", error);
4570 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4571 	}
4572 
4573 	return (0);
4574 }
4575 
4576 static int
4577 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport)
4578 {
4579 	vdev_t *rvd = spa->spa_root_vdev;
4580 
4581 	if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) {
4582 		boolean_t missing = spa_check_logs(spa);
4583 		if (missing) {
4584 			if (spa->spa_missing_tvds != 0) {
4585 				spa_load_note(spa, "spa_check_logs failed "
4586 				    "so dropping the logs");
4587 			} else {
4588 				*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
4589 				spa_load_failed(spa, "spa_check_logs failed");
4590 				return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG,
4591 				    ENXIO));
4592 			}
4593 		}
4594 	}
4595 
4596 	return (0);
4597 }
4598 
4599 static int
4600 spa_ld_verify_pool_data(spa_t *spa)
4601 {
4602 	int error = 0;
4603 	vdev_t *rvd = spa->spa_root_vdev;
4604 
4605 	/*
4606 	 * We've successfully opened the pool, verify that we're ready
4607 	 * to start pushing transactions.
4608 	 */
4609 	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4610 		error = spa_load_verify(spa);
4611 		if (error != 0) {
4612 			spa_load_failed(spa, "spa_load_verify failed "
4613 			    "[error=%d]", error);
4614 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
4615 			    error));
4616 		}
4617 	}
4618 
4619 	return (0);
4620 }
4621 
4622 static void
4623 spa_ld_claim_log_blocks(spa_t *spa)
4624 {
4625 	dmu_tx_t *tx;
4626 	dsl_pool_t *dp = spa_get_dsl(spa);
4627 
4628 	/*
4629 	 * Claim log blocks that haven't been committed yet.
4630 	 * This must all happen in a single txg.
4631 	 * Note: spa_claim_max_txg is updated by spa_claim_notify(),
4632 	 * invoked from zil_claim_log_block()'s i/o done callback.
4633 	 * Price of rollback is that we abandon the log.
4634 	 */
4635 	spa->spa_claiming = B_TRUE;
4636 
4637 	tx = dmu_tx_create_assigned(dp, spa_first_txg(spa));
4638 	(void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
4639 	    zil_claim, tx, DS_FIND_CHILDREN);
4640 	dmu_tx_commit(tx);
4641 
4642 	spa->spa_claiming = B_FALSE;
4643 
4644 	spa_set_log_state(spa, SPA_LOG_GOOD);
4645 }
4646 
4647 static void
4648 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg,
4649     boolean_t update_config_cache)
4650 {
4651 	vdev_t *rvd = spa->spa_root_vdev;
4652 	int need_update = B_FALSE;
4653 
4654 	/*
4655 	 * If the config cache is stale, or we have uninitialized
4656 	 * metaslabs (see spa_vdev_add()), then update the config.
4657 	 *
4658 	 * If this is a verbatim import, trust the current
4659 	 * in-core spa_config and update the disk labels.
4660 	 */
4661 	if (update_config_cache || config_cache_txg != spa->spa_config_txg ||
4662 	    spa->spa_load_state == SPA_LOAD_IMPORT ||
4663 	    spa->spa_load_state == SPA_LOAD_RECOVER ||
4664 	    (spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
4665 		need_update = B_TRUE;
4666 
4667 	for (int c = 0; c < rvd->vdev_children; c++)
4668 		if (rvd->vdev_child[c]->vdev_ms_array == 0)
4669 			need_update = B_TRUE;
4670 
4671 	/*
4672 	 * Update the config cache asynchronously in case we're the
4673 	 * root pool, in which case the config cache isn't writable yet.
4674 	 */
4675 	if (need_update)
4676 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
4677 }
4678 
4679 static void
4680 spa_ld_prepare_for_reload(spa_t *spa)
4681 {
4682 	spa_mode_t mode = spa->spa_mode;
4683 	int async_suspended = spa->spa_async_suspended;
4684 
4685 	spa_unload(spa);
4686 	spa_deactivate(spa);
4687 	spa_activate(spa, mode);
4688 
4689 	/*
4690 	 * We save the value of spa_async_suspended as it gets reset to 0 by
4691 	 * spa_unload(). We want to restore it back to the original value before
4692 	 * returning as we might be calling spa_async_resume() later.
4693 	 */
4694 	spa->spa_async_suspended = async_suspended;
4695 }
4696 
4697 static int
4698 spa_ld_read_checkpoint_txg(spa_t *spa)
4699 {
4700 	uberblock_t checkpoint;
4701 	int error = 0;
4702 
4703 	ASSERT0(spa->spa_checkpoint_txg);
4704 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4705 
4706 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4707 	    DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
4708 	    sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);
4709 
4710 	if (error == ENOENT)
4711 		return (0);
4712 
4713 	if (error != 0)
4714 		return (error);
4715 
4716 	ASSERT3U(checkpoint.ub_txg, !=, 0);
4717 	ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0);
4718 	ASSERT3U(checkpoint.ub_timestamp, !=, 0);
4719 	spa->spa_checkpoint_txg = checkpoint.ub_txg;
4720 	spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp;
4721 
4722 	return (0);
4723 }
4724 
4725 static int
4726 spa_ld_mos_init(spa_t *spa, spa_import_type_t type)
4727 {
4728 	int error = 0;
4729 
4730 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4731 	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
4732 
4733 	/*
4734 	 * Never trust the config that is provided unless we are assembling
4735 	 * a pool following a split.
4736 	 * This means don't trust blkptrs and the vdev tree in general. This
4737 	 * also effectively puts the spa in read-only mode since
4738 	 * spa_writeable() checks for spa_trust_config to be true.
4739 	 * We will later load a trusted config from the MOS.
4740 	 */
4741 	if (type != SPA_IMPORT_ASSEMBLE)
4742 		spa->spa_trust_config = B_FALSE;
4743 
4744 	/*
4745 	 * Parse the config provided to create a vdev tree.
4746 	 */
4747 	error = spa_ld_parse_config(spa, type);
4748 	if (error != 0)
4749 		return (error);
4750 
4751 	spa_import_progress_add(spa);
4752 
4753 	/*
4754 	 * Now that we have the vdev tree, try to open each vdev. This involves
4755 	 * opening the underlying physical device, retrieving its geometry and
4756 	 * probing the vdev with a dummy I/O. The state of each vdev will be set
4757 	 * based on the success of those operations. After this we'll be ready
4758 	 * to read from the vdevs.
4759 	 */
4760 	error = spa_ld_open_vdevs(spa);
4761 	if (error != 0)
4762 		return (error);
4763 
4764 	/*
4765 	 * Read the label of each vdev and make sure that the GUIDs stored
4766 	 * there match the GUIDs in the config provided.
4767 	 * If we're assembling a new pool that's been split off from an
4768 	 * existing pool, the labels haven't yet been updated so we skip
4769 	 * validation for now.
4770 	 */
4771 	if (type != SPA_IMPORT_ASSEMBLE) {
4772 		error = spa_ld_validate_vdevs(spa);
4773 		if (error != 0)
4774 			return (error);
4775 	}
4776 
4777 	/*
4778 	 * Read all vdev labels to find the best uberblock (i.e. latest,
4779 	 * unless spa_load_max_txg is set) and store it in spa_uberblock. We
4780 	 * get the list of features required to read blkptrs in the MOS from
4781 	 * the vdev label with the best uberblock and verify that our version
4782 	 * of zfs supports them all.
4783 	 */
4784 	error = spa_ld_select_uberblock(spa, type);
4785 	if (error != 0)
4786 		return (error);
4787 
4788 	/*
4789 	 * Pass that uberblock to the dsl_pool layer which will open the root
4790 	 * blkptr. This blkptr points to the latest version of the MOS and will
4791 	 * allow us to read its contents.
4792 	 */
4793 	error = spa_ld_open_rootbp(spa);
4794 	if (error != 0)
4795 		return (error);
4796 
4797 	return (0);
4798 }
4799 
4800 static int
4801 spa_ld_checkpoint_rewind(spa_t *spa)
4802 {
4803 	uberblock_t checkpoint;
4804 	int error = 0;
4805 
4806 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4807 	ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
4808 
4809 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4810 	    DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
4811 	    sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);
4812 
4813 	if (error != 0) {
4814 		spa_load_failed(spa, "unable to retrieve checkpointed "
4815 		    "uberblock from the MOS config [error=%d]", error);
4816 
4817 		if (error == ENOENT)
4818 			error = ZFS_ERR_NO_CHECKPOINT;
4819 
4820 		return (error);
4821 	}
4822 
4823 	ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg);
4824 	ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg);
4825 
4826 	/*
4827 	 * We need to update the txg and timestamp of the checkpointed
4828 	 * uberblock to be higher than the latest one. This ensures that
4829 	 * the checkpointed uberblock is selected if we were to close and
4830 	 * reopen the pool right after we've written it in the vdev labels.
4831 	 * (also see block comment in vdev_uberblock_compare)
4832 	 */
4833 	checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1;
4834 	checkpoint.ub_timestamp = gethrestime_sec();
4835 
4836 	/*
4837 	 * Set current uberblock to be the checkpointed uberblock.
4838 	 */
4839 	spa->spa_uberblock = checkpoint;
4840 
4841 	/*
4842 	 * If we are doing a normal rewind, then the pool is open for
4843 	 * writing and we sync the "updated" checkpointed uberblock to
4844 	 * disk. Once this is done, we've basically rewound the whole
4845 	 * pool and there is no way back.
4846 	 *
4847 	 * There are cases when we don't want to attempt and sync the
4848 	 * checkpointed uberblock to disk because we are opening a
4849 	 * pool as read-only. Specifically, verifying the checkpointed
4850 	 * state with zdb, and importing the checkpointed state to get
4851 	 * a "preview" of its content.
4852 	 */
4853 	if (spa_writeable(spa)) {
4854 		vdev_t *rvd = spa->spa_root_vdev;
4855 
4856 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4857 		vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
4858 		int svdcount = 0;
4859 		int children = rvd->vdev_children;
4860 		int c0 = random_in_range(children);
4861 
4862 		for (int c = 0; c < children; c++) {
4863 			vdev_t *vd = rvd->vdev_child[(c0 + c) % children];
4864 
4865 			/* Stop when revisiting the first vdev */
4866 			if (c > 0 && svd[0] == vd)
4867 				break;
4868 
4869 			if (vd->vdev_ms_array == 0 || vd->vdev_islog ||
4870 			    !vdev_is_concrete(vd))
4871 				continue;
4872 
4873 			svd[svdcount++] = vd;
4874 			if (svdcount == SPA_SYNC_MIN_VDEVS)
4875 				break;
4876 		}
4877 		error = vdev_config_sync(svd, svdcount, spa->spa_first_txg);
4878 		if (error == 0)
4879 			spa->spa_last_synced_guid = rvd->vdev_guid;
4880 		spa_config_exit(spa, SCL_ALL, FTAG);
4881 
4882 		if (error != 0) {
4883 			spa_load_failed(spa, "failed to write checkpointed "
4884 			    "uberblock to the vdev labels [error=%d]", error);
4885 			return (error);
4886 		}
4887 	}
4888 
4889 	return (0);
4890 }
4891 
4892 static int
4893 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type,
4894     boolean_t *update_config_cache)
4895 {
4896 	int error;
4897 
4898 	/*
4899 	 * Parse the config for pool, open and validate vdevs,
4900 	 * select an uberblock, and use that uberblock to open
4901 	 * the MOS.
4902 	 */
4903 	error = spa_ld_mos_init(spa, type);
4904 	if (error != 0)
4905 		return (error);
4906 
4907 	/*
4908 	 * Retrieve the trusted config stored in the MOS and use it to create
4909 	 * a new, exact version of the vdev tree, then reopen all vdevs.
4910 	 */
4911 	error = spa_ld_trusted_config(spa, type, B_FALSE);
4912 	if (error == EAGAIN) {
4913 		if (update_config_cache != NULL)
4914 			*update_config_cache = B_TRUE;
4915 
4916 		/*
4917 		 * Redo the loading process with the trusted config if it is
4918 		 * too different from the untrusted config.
4919 		 */
4920 		spa_ld_prepare_for_reload(spa);
4921 		spa_load_note(spa, "RELOADING");
4922 		error = spa_ld_mos_init(spa, type);
4923 		if (error != 0)
4924 			return (error);
4925 
4926 		error = spa_ld_trusted_config(spa, type, B_TRUE);
4927 		if (error != 0)
4928 			return (error);
4929 
4930 	} else if (error != 0) {
4931 		return (error);
4932 	}
4933 
4934 	return (0);
4935 }
4936 
4937 /*
4938  * Load an existing storage pool, using the config provided. This config
4939  * describes which vdevs are part of the pool and is later validated against
4940  * partial configs present in each vdev's label and an entire copy of the
4941  * config stored in the MOS.
4942  */
4943 static int
4944 spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport)
4945 {
4946 	int error = 0;
4947 	boolean_t missing_feat_write = B_FALSE;
4948 	boolean_t checkpoint_rewind =
4949 	    (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
4950 	boolean_t update_config_cache = B_FALSE;
4951 
4952 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4953 	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
4954 
4955 	spa_load_note(spa, "LOADING");
4956 
4957 	error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache);
4958 	if (error != 0)
4959 		return (error);
4960 
4961 	/*
4962 	 * If we are rewinding to the checkpoint then we need to repeat
4963 	 * everything we've done so far in this function but this time
4964 	 * selecting the checkpointed uberblock and using that to open
4965 	 * the MOS.
4966 	 */
4967 	if (checkpoint_rewind) {
4968 		/*
4969 		 * If we are rewinding to the checkpoint update config cache
4970 		 * anyway.
4971 		 */
4972 		update_config_cache = B_TRUE;
4973 
4974 		/*
4975 		 * Extract the checkpointed uberblock from the current MOS
4976 		 * and use this as the pool's uberblock from now on. If the
4977 		 * pool is imported as writeable we also write the checkpoint
4978 		 * uberblock to the labels, making the rewind permanent.
4979 		 */
4980 		error = spa_ld_checkpoint_rewind(spa);
4981 		if (error != 0)
4982 			return (error);
4983 
4984 		/*
4985 		 * Redo the loading process again with the
4986 		 * checkpointed uberblock.
4987 		 */
4988 		spa_ld_prepare_for_reload(spa);
4989 		spa_load_note(spa, "LOADING checkpointed uberblock");
4990 		error = spa_ld_mos_with_trusted_config(spa, type, NULL);
4991 		if (error != 0)
4992 			return (error);
4993 	}
4994 
4995 	/*
4996 	 * Retrieve the checkpoint txg if the pool has a checkpoint.
4997 	 */
4998 	error = spa_ld_read_checkpoint_txg(spa);
4999 	if (error != 0)
5000 		return (error);
5001 
5002 	/*
5003 	 * Retrieve the mapping of indirect vdevs. Those vdevs were removed
5004 	 * from the pool and their contents were re-mapped to other vdevs. Note
5005 	 * that everything that we read before this step must have been
5006 	 * rewritten on concrete vdevs after the last device removal was
5007 	 * initiated. Otherwise we could be reading from indirect vdevs before
5008 	 * we have loaded their mappings.
5009 	 */
5010 	error = spa_ld_open_indirect_vdev_metadata(spa);
5011 	if (error != 0)
5012 		return (error);
5013 
5014 	/*
5015 	 * Retrieve the full list of active features from the MOS and check if
5016 	 * they are all supported.
5017 	 */
5018 	error = spa_ld_check_features(spa, &missing_feat_write);
5019 	if (error != 0)
5020 		return (error);
5021 
5022 	/*
5023 	 * Load several special directories from the MOS needed by the dsl_pool
5024 	 * layer.
5025 	 */
5026 	error = spa_ld_load_special_directories(spa);
5027 	if (error != 0)
5028 		return (error);
5029 
5030 	/*
5031 	 * Retrieve pool properties from the MOS.
5032 	 */
5033 	error = spa_ld_get_props(spa);
5034 	if (error != 0)
5035 		return (error);
5036 
5037 	/*
5038 	 * Retrieve the list of auxiliary devices - cache devices and spares -
5039 	 * and open them.
5040 	 */
5041 	error = spa_ld_open_aux_vdevs(spa, type);
5042 	if (error != 0)
5043 		return (error);
5044 
5045 	/*
5046 	 * Load the metadata for all vdevs. Also check if unopenable devices
5047 	 * should be autoreplaced.
5048 	 */
5049 	error = spa_ld_load_vdev_metadata(spa);
5050 	if (error != 0)
5051 		return (error);
5052 
5053 	error = spa_ld_load_dedup_tables(spa);
5054 	if (error != 0)
5055 		return (error);
5056 
5057 	error = spa_ld_load_brt(spa);
5058 	if (error != 0)
5059 		return (error);
5060 
5061 	/*
5062 	 * Verify the logs now to make sure we don't have any unexpected errors
5063 	 * when we claim log blocks later.
5064 	 */
5065 	error = spa_ld_verify_logs(spa, type, ereport);
5066 	if (error != 0)
5067 		return (error);
5068 
5069 	if (missing_feat_write) {
5070 		ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT);
5071 
5072 		/*
5073 		 * At this point, we know that we can open the pool in
5074 		 * read-only mode but not read-write mode. We now have enough
5075 		 * information and can return to userland.
5076 		 */
5077 		return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT,
5078 		    ENOTSUP));
5079 	}
5080 
5081 	/*
5082 	 * Traverse the last txgs to make sure the pool was left off in a safe
5083 	 * state. When performing an extreme rewind, we verify the whole pool,
5084 	 * which can take a very long time.
5085 	 */
5086 	error = spa_ld_verify_pool_data(spa);
5087 	if (error != 0)
5088 		return (error);
5089 
5090 	/*
5091 	 * Calculate the deflated space for the pool. This must be done before
5092 	 * we write anything to the pool because we'd need to update the space
5093 	 * accounting using the deflated sizes.
5094 	 */
5095 	spa_update_dspace(spa);
5096 
5097 	/*
5098 	 * We have now retrieved all the information we needed to open the
5099 	 * pool. If we are importing the pool in read-write mode, a few
5100 	 * additional steps must be performed to finish the import.
5101 	 */
5102 	if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER ||
5103 	    spa->spa_load_max_txg == UINT64_MAX)) {
5104 		uint64_t config_cache_txg = spa->spa_config_txg;
5105 
5106 		ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT);
5107 
5108 		/*
5109 		 * Before we do any zio_write's, complete the raidz expansion
5110 		 * scratch space copying, if necessary.
5111 		 */
5112 		if (RRSS_GET_STATE(&spa->spa_uberblock) == RRSS_SCRATCH_VALID)
5113 			vdev_raidz_reflow_copy_scratch(spa);
5114 
5115 		/*
5116 		 * In case of a checkpoint rewind, log the original txg
5117 		 * of the checkpointed uberblock.
5118 		 */
5119 		if (checkpoint_rewind) {
5120 			spa_history_log_internal(spa, "checkpoint rewind",
5121 			    NULL, "rewound state to txg=%llu",
5122 			    (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg);
5123 		}
5124 
5125 		/*
5126 		 * Traverse the ZIL and claim all blocks.
5127 		 */
5128 		spa_ld_claim_log_blocks(spa);
5129 
5130 		/*
5131 		 * Kick-off the syncing thread.
5132 		 */
5133 		spa->spa_sync_on = B_TRUE;
5134 		txg_sync_start(spa->spa_dsl_pool);
5135 		mmp_thread_start(spa);
5136 
5137 		/*
5138 		 * Wait for all claims to sync.  We sync up to the highest
5139 		 * claimed log block birth time so that claimed log blocks
5140 		 * don't appear to be from the future.  spa_claim_max_txg
5141 		 * will have been set for us by ZIL traversal operations
5142 		 * performed above.
5143 		 */
5144 		txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);
5145 
5146 		/*
5147 		 * Check if we need to request an update of the config. On the
5148 		 * next sync, we would update the config stored in vdev labels
5149 		 * and the cachefile (by default /etc/zfs/zpool.cache).
5150 		 */
5151 		spa_ld_check_for_config_update(spa, config_cache_txg,
5152 		    update_config_cache);
5153 
5154 		/*
5155 		 * Check if a rebuild was in progress and if so resume it.
5156 		 * Then check all DTLs to see if anything needs resilvering.
5157 		 * The resilver will be deferred if a rebuild was started.
5158 		 */
5159 		if (vdev_rebuild_active(spa->spa_root_vdev)) {
5160 			vdev_rebuild_restart(spa);
5161 		} else if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
5162 		    vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
5163 			spa_async_request(spa, SPA_ASYNC_RESILVER);
5164 		}
5165 
5166 		/*
5167 		 * Log the fact that we booted up (so that we can detect if
5168 		 * we rebooted in the middle of an operation).
5169 		 */
5170 		spa_history_log_version(spa, "open", NULL);
5171 
5172 		spa_restart_removal(spa);
5173 		spa_spawn_aux_threads(spa);
5174 
5175 		/*
5176 		 * Delete any inconsistent datasets.
5177 		 *
5178 		 * Note:
5179 		 * Since we may be issuing deletes for clones here,
5180 		 * we make sure to do so after we've spawned all the
5181 		 * auxiliary threads above (from which the livelist
5182 		 * deletion zthr is part of).
5183 		 */
5184 		(void) dmu_objset_find(spa_name(spa),
5185 		    dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN);
5186 
5187 		/*
5188 		 * Clean up any stale temporary dataset userrefs.
5189 		 */
5190 		dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);
5191 
5192 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
5193 		vdev_initialize_restart(spa->spa_root_vdev);
5194 		vdev_trim_restart(spa->spa_root_vdev);
5195 		vdev_autotrim_restart(spa);
5196 		spa_config_exit(spa, SCL_CONFIG, FTAG);
5197 	}
5198 
5199 	spa_import_progress_remove(spa_guid(spa));
5200 	spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD);
5201 
5202 	spa_load_note(spa, "LOADED");
5203 
5204 	return (0);
5205 }
5206 
5207 static int
5208 spa_load_retry(spa_t *spa, spa_load_state_t state)
5209 {
5210 	spa_mode_t mode = spa->spa_mode;
5211 
5212 	spa_unload(spa);
5213 	spa_deactivate(spa);
5214 
5215 	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
5216 
5217 	spa_activate(spa, mode);
5218 	spa_async_suspend(spa);
5219 
5220 	spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu",
5221 	    (u_longlong_t)spa->spa_load_max_txg);
5222 
5223 	return (spa_load(spa, state, SPA_IMPORT_EXISTING));
5224 }
5225 
5226 /*
5227  * If spa_load() fails this function will try loading prior txg's. If
5228  * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool
5229  * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this
5230  * function will not rewind the pool and will return the same error as
5231  * spa_load().
5232  */
5233 static int
5234 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request,
5235     int rewind_flags)
5236 {
5237 	nvlist_t *loadinfo = NULL;
5238 	nvlist_t *config = NULL;
5239 	int load_error, rewind_error;
5240 	uint64_t safe_rewind_txg;
5241 	uint64_t min_txg;
5242 
5243 	if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
5244 		spa->spa_load_max_txg = spa->spa_load_txg;
5245 		spa_set_log_state(spa, SPA_LOG_CLEAR);
5246 	} else {
5247 		spa->spa_load_max_txg = max_request;
5248 		if (max_request != UINT64_MAX)
5249 			spa->spa_extreme_rewind = B_TRUE;
5250 	}
5251 
5252 	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING);
5253 	if (load_error == 0)
5254 		return (0);
5255 	if (load_error == ZFS_ERR_NO_CHECKPOINT) {
5256 		/*
5257 		 * When attempting checkpoint-rewind on a pool with no
5258 		 * checkpoint, we should not attempt to load uberblocks
5259 		 * from previous txgs when spa_load fails.
5260 		 */
5261 		ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5262 		spa_import_progress_remove(spa_guid(spa));
5263 		return (load_error);
5264 	}
5265 
5266 	if (spa->spa_root_vdev != NULL)
5267 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5268 
5269 	spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
5270 	spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;
5271 
5272 	if (rewind_flags & ZPOOL_NEVER_REWIND) {
5273 		nvlist_free(config);
5274 		spa_import_progress_remove(spa_guid(spa));
5275 		return (load_error);
5276 	}
5277 
5278 	if (state == SPA_LOAD_RECOVER) {
5279 		/* Price of rolling back is discarding txgs, including log */
5280 		spa_set_log_state(spa, SPA_LOG_CLEAR);
5281 	} else {
5282 		/*
5283 		 * If we aren't rolling back save the load info from our first
5284 		 * import attempt so that we can restore it after attempting
5285 		 * to rewind.
5286 		 */
5287 		loadinfo = spa->spa_load_info;
5288 		spa->spa_load_info = fnvlist_alloc();
5289 	}
5290 
5291 	spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
5292 	safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
5293 	min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ?
5294 	    TXG_INITIAL : safe_rewind_txg;
5295 
5296 	/*
5297 	 * Continue as long as we're finding errors, we're still within
5298 	 * the acceptable rewind range, and we're still finding uberblocks
5299 	 */
5300 	while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
5301 	    spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
5302 		if (spa->spa_load_max_txg < safe_rewind_txg)
5303 			spa->spa_extreme_rewind = B_TRUE;
5304 		rewind_error = spa_load_retry(spa, state);
5305 	}
5306 
5307 	spa->spa_extreme_rewind = B_FALSE;
5308 	spa->spa_load_max_txg = UINT64_MAX;
5309 
5310 	if (config && (rewind_error || state != SPA_LOAD_RECOVER))
5311 		spa_config_set(spa, config);
5312 	else
5313 		nvlist_free(config);
5314 
5315 	if (state == SPA_LOAD_RECOVER) {
5316 		ASSERT3P(loadinfo, ==, NULL);
5317 		spa_import_progress_remove(spa_guid(spa));
5318 		return (rewind_error);
5319 	} else {
5320 		/* Store the rewind info as part of the initial load info */
5321 		fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO,
5322 		    spa->spa_load_info);
5323 
5324 		/* Restore the initial load info */
5325 		fnvlist_free(spa->spa_load_info);
5326 		spa->spa_load_info = loadinfo;
5327 
5328 		spa_import_progress_remove(spa_guid(spa));
5329 		return (load_error);
5330 	}
5331 }
5332 
5333 /*
5334  * Pool Open/Import
5335  *
5336  * The import case is identical to an open except that the configuration is sent
5337  * down from userland, instead of grabbed from the configuration cache.  For the
5338  * case of an open, the pool configuration will exist in the
5339  * POOL_STATE_UNINITIALIZED state.
5340  *
5341  * The stats information (gen/count/ustats) is used to gather vdev statistics at
5342  * the same time open the pool, without having to keep around the spa_t in some
5343  * ambiguous state.
5344  */
5345 static int
5346 spa_open_common(const char *pool, spa_t **spapp, const void *tag,
5347     nvlist_t *nvpolicy, nvlist_t **config)
5348 {
5349 	spa_t *spa;
5350 	spa_load_state_t state = SPA_LOAD_OPEN;
5351 	int error;
5352 	int locked = B_FALSE;
5353 	int firstopen = B_FALSE;
5354 
5355 	*spapp = NULL;
5356 
5357 	/*
5358 	 * As disgusting as this is, we need to support recursive calls to this
5359 	 * function because dsl_dir_open() is called during spa_load(), and ends
5360 	 * up calling spa_open() again.  The real fix is to figure out how to
5361 	 * avoid dsl_dir_open() calling this in the first place.
5362 	 */
5363 	if (MUTEX_NOT_HELD(&spa_namespace_lock)) {
5364 		mutex_enter(&spa_namespace_lock);
5365 		locked = B_TRUE;
5366 	}
5367 
5368 	if ((spa = spa_lookup(pool)) == NULL) {
5369 		if (locked)
5370 			mutex_exit(&spa_namespace_lock);
5371 		return (SET_ERROR(ENOENT));
5372 	}
5373 
5374 	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
5375 		zpool_load_policy_t policy;
5376 
5377 		firstopen = B_TRUE;
5378 
5379 		zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config,
5380 		    &policy);
5381 		if (policy.zlp_rewind & ZPOOL_DO_REWIND)
5382 			state = SPA_LOAD_RECOVER;
5383 
5384 		spa_activate(spa, spa_mode_global);
5385 
5386 		if (state != SPA_LOAD_RECOVER)
5387 			spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
5388 		spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
5389 
5390 		zfs_dbgmsg("spa_open_common: opening %s", pool);
5391 		error = spa_load_best(spa, state, policy.zlp_txg,
5392 		    policy.zlp_rewind);
5393 
5394 		if (error == EBADF) {
5395 			/*
5396 			 * If vdev_validate() returns failure (indicated by
5397 			 * EBADF), it indicates that one of the vdevs indicates
5398 			 * that the pool has been exported or destroyed.  If
5399 			 * this is the case, the config cache is out of sync and
5400 			 * we should remove the pool from the namespace.
5401 			 */
5402 			spa_unload(spa);
5403 			spa_deactivate(spa);
5404 			spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
5405 			spa_remove(spa);
5406 			if (locked)
5407 				mutex_exit(&spa_namespace_lock);
5408 			return (SET_ERROR(ENOENT));
5409 		}
5410 
5411 		if (error) {
5412 			/*
5413 			 * We can't open the pool, but we still have useful
5414 			 * information: the state of each vdev after the
5415 			 * attempted vdev_open().  Return this to the user.
5416 			 */
5417 			if (config != NULL && spa->spa_config) {
5418 				*config = fnvlist_dup(spa->spa_config);
5419 				fnvlist_add_nvlist(*config,
5420 				    ZPOOL_CONFIG_LOAD_INFO,
5421 				    spa->spa_load_info);
5422 			}
5423 			spa_unload(spa);
5424 			spa_deactivate(spa);
5425 			spa->spa_last_open_failed = error;
5426 			if (locked)
5427 				mutex_exit(&spa_namespace_lock);
5428 			*spapp = NULL;
5429 			return (error);
5430 		}
5431 	}
5432 
5433 	spa_open_ref(spa, tag);
5434 
5435 	if (config != NULL)
5436 		*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5437 
5438 	/*
5439 	 * If we've recovered the pool, pass back any information we
5440 	 * gathered while doing the load.
5441 	 */
5442 	if (state == SPA_LOAD_RECOVER && config != NULL) {
5443 		fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO,
5444 		    spa->spa_load_info);
5445 	}
5446 
5447 	if (locked) {
5448 		spa->spa_last_open_failed = 0;
5449 		spa->spa_last_ubsync_txg = 0;
5450 		spa->spa_load_txg = 0;
5451 		mutex_exit(&spa_namespace_lock);
5452 	}
5453 
5454 	if (firstopen)
5455 		zvol_create_minors_recursive(spa_name(spa));
5456 
5457 	*spapp = spa;
5458 
5459 	return (0);
5460 }
5461 
5462 int
5463 spa_open_rewind(const char *name, spa_t **spapp, const void *tag,
5464     nvlist_t *policy, nvlist_t **config)
5465 {
5466 	return (spa_open_common(name, spapp, tag, policy, config));
5467 }
5468 
5469 int
5470 spa_open(const char *name, spa_t **spapp, const void *tag)
5471 {
5472 	return (spa_open_common(name, spapp, tag, NULL, NULL));
5473 }
5474 
5475 /*
5476  * Lookup the given spa_t, incrementing the inject count in the process,
5477  * preventing it from being exported or destroyed.
5478  */
5479 spa_t *
5480 spa_inject_addref(char *name)
5481 {
5482 	spa_t *spa;
5483 
5484 	mutex_enter(&spa_namespace_lock);
5485 	if ((spa = spa_lookup(name)) == NULL) {
5486 		mutex_exit(&spa_namespace_lock);
5487 		return (NULL);
5488 	}
5489 	spa->spa_inject_ref++;
5490 	mutex_exit(&spa_namespace_lock);
5491 
5492 	return (spa);
5493 }
5494 
5495 void
5496 spa_inject_delref(spa_t *spa)
5497 {
5498 	mutex_enter(&spa_namespace_lock);
5499 	spa->spa_inject_ref--;
5500 	mutex_exit(&spa_namespace_lock);
5501 }
5502 
5503 /*
5504  * Add spares device information to the nvlist.
5505  */
5506 static void
5507 spa_add_spares(spa_t *spa, nvlist_t *config)
5508 {
5509 	nvlist_t **spares;
5510 	uint_t i, nspares;
5511 	nvlist_t *nvroot;
5512 	uint64_t guid;
5513 	vdev_stat_t *vs;
5514 	uint_t vsc;
5515 	uint64_t pool;
5516 
5517 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5518 
5519 	if (spa->spa_spares.sav_count == 0)
5520 		return;
5521 
5522 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
5523 	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
5524 	    ZPOOL_CONFIG_SPARES, &spares, &nspares));
5525 	if (nspares != 0) {
5526 		fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
5527 		    (const nvlist_t * const *)spares, nspares);
5528 		VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
5529 		    &spares, &nspares));
5530 
5531 		/*
5532 		 * Go through and find any spares which have since been
5533 		 * repurposed as an active spare.  If this is the case, update
5534 		 * their status appropriately.
5535 		 */
5536 		for (i = 0; i < nspares; i++) {
5537 			guid = fnvlist_lookup_uint64(spares[i],
5538 			    ZPOOL_CONFIG_GUID);
5539 			VERIFY0(nvlist_lookup_uint64_array(spares[i],
5540 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
5541 			if (spa_spare_exists(guid, &pool, NULL) &&
5542 			    pool != 0ULL) {
5543 				vs->vs_state = VDEV_STATE_CANT_OPEN;
5544 				vs->vs_aux = VDEV_AUX_SPARED;
5545 			} else {
5546 				vs->vs_state =
5547 				    spa->spa_spares.sav_vdevs[i]->vdev_state;
5548 			}
5549 		}
5550 	}
5551 }
5552 
5553 /*
5554  * Add l2cache device information to the nvlist, including vdev stats.
5555  */
5556 static void
5557 spa_add_l2cache(spa_t *spa, nvlist_t *config)
5558 {
5559 	nvlist_t **l2cache;
5560 	uint_t i, j, nl2cache;
5561 	nvlist_t *nvroot;
5562 	uint64_t guid;
5563 	vdev_t *vd;
5564 	vdev_stat_t *vs;
5565 	uint_t vsc;
5566 
5567 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5568 
5569 	if (spa->spa_l2cache.sav_count == 0)
5570 		return;
5571 
5572 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
5573 	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
5574 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache));
5575 	if (nl2cache != 0) {
5576 		fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
5577 		    (const nvlist_t * const *)l2cache, nl2cache);
5578 		VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
5579 		    &l2cache, &nl2cache));
5580 
5581 		/*
5582 		 * Update level 2 cache device stats.
5583 		 */
5584 
5585 		for (i = 0; i < nl2cache; i++) {
5586 			guid = fnvlist_lookup_uint64(l2cache[i],
5587 			    ZPOOL_CONFIG_GUID);
5588 
5589 			vd = NULL;
5590 			for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
5591 				if (guid ==
5592 				    spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
5593 					vd = spa->spa_l2cache.sav_vdevs[j];
5594 					break;
5595 				}
5596 			}
5597 			ASSERT(vd != NULL);
5598 
5599 			VERIFY0(nvlist_lookup_uint64_array(l2cache[i],
5600 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
5601 			vdev_get_stats(vd, vs);
5602 			vdev_config_generate_stats(vd, l2cache[i]);
5603 
5604 		}
5605 	}
5606 }
5607 
5608 static void
5609 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features)
5610 {
5611 	zap_cursor_t zc;
5612 	zap_attribute_t za;
5613 
5614 	if (spa->spa_feat_for_read_obj != 0) {
5615 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
5616 		    spa->spa_feat_for_read_obj);
5617 		    zap_cursor_retrieve(&zc, &za) == 0;
5618 		    zap_cursor_advance(&zc)) {
5619 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
5620 			    za.za_num_integers == 1);
5621 			VERIFY0(nvlist_add_uint64(features, za.za_name,
5622 			    za.za_first_integer));
5623 		}
5624 		zap_cursor_fini(&zc);
5625 	}
5626 
5627 	if (spa->spa_feat_for_write_obj != 0) {
5628 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
5629 		    spa->spa_feat_for_write_obj);
5630 		    zap_cursor_retrieve(&zc, &za) == 0;
5631 		    zap_cursor_advance(&zc)) {
5632 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
5633 			    za.za_num_integers == 1);
5634 			VERIFY0(nvlist_add_uint64(features, za.za_name,
5635 			    za.za_first_integer));
5636 		}
5637 		zap_cursor_fini(&zc);
5638 	}
5639 }
5640 
5641 static void
5642 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features)
5643 {
5644 	int i;
5645 
5646 	for (i = 0; i < SPA_FEATURES; i++) {
5647 		zfeature_info_t feature = spa_feature_table[i];
5648 		uint64_t refcount;
5649 
5650 		if (feature_get_refcount(spa, &feature, &refcount) != 0)
5651 			continue;
5652 
5653 		VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount));
5654 	}
5655 }
5656 
5657 /*
5658  * Store a list of pool features and their reference counts in the
5659  * config.
5660  *
5661  * The first time this is called on a spa, allocate a new nvlist, fetch
5662  * the pool features and reference counts from disk, then save the list
5663  * in the spa. In subsequent calls on the same spa use the saved nvlist
5664  * and refresh its values from the cached reference counts.  This
5665  * ensures we don't block here on I/O on a suspended pool so 'zpool
5666  * clear' can resume the pool.
5667  */
5668 static void
5669 spa_add_feature_stats(spa_t *spa, nvlist_t *config)
5670 {
5671 	nvlist_t *features;
5672 
5673 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5674 
5675 	mutex_enter(&spa->spa_feat_stats_lock);
5676 	features = spa->spa_feat_stats;
5677 
5678 	if (features != NULL) {
5679 		spa_feature_stats_from_cache(spa, features);
5680 	} else {
5681 		VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP));
5682 		spa->spa_feat_stats = features;
5683 		spa_feature_stats_from_disk(spa, features);
5684 	}
5685 
5686 	VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS,
5687 	    features));
5688 
5689 	mutex_exit(&spa->spa_feat_stats_lock);
5690 }
5691 
5692 int
5693 spa_get_stats(const char *name, nvlist_t **config,
5694     char *altroot, size_t buflen)
5695 {
5696 	int error;
5697 	spa_t *spa;
5698 
5699 	*config = NULL;
5700 	error = spa_open_common(name, &spa, FTAG, NULL, config);
5701 
5702 	if (spa != NULL) {
5703 		/*
5704 		 * This still leaves a window of inconsistency where the spares
5705 		 * or l2cache devices could change and the config would be
5706 		 * self-inconsistent.
5707 		 */
5708 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
5709 
5710 		if (*config != NULL) {
5711 			uint64_t loadtimes[2];
5712 
5713 			loadtimes[0] = spa->spa_loaded_ts.tv_sec;
5714 			loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
5715 			fnvlist_add_uint64_array(*config,
5716 			    ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2);
5717 
5718 			fnvlist_add_uint64(*config,
5719 			    ZPOOL_CONFIG_ERRCOUNT,
5720 			    spa_approx_errlog_size(spa));
5721 
5722 			if (spa_suspended(spa)) {
5723 				fnvlist_add_uint64(*config,
5724 				    ZPOOL_CONFIG_SUSPENDED,
5725 				    spa->spa_failmode);
5726 				fnvlist_add_uint64(*config,
5727 				    ZPOOL_CONFIG_SUSPENDED_REASON,
5728 				    spa->spa_suspended);
5729 			}
5730 
5731 			spa_add_spares(spa, *config);
5732 			spa_add_l2cache(spa, *config);
5733 			spa_add_feature_stats(spa, *config);
5734 		}
5735 	}
5736 
5737 	/*
5738 	 * We want to get the alternate root even for faulted pools, so we cheat
5739 	 * and call spa_lookup() directly.
5740 	 */
5741 	if (altroot) {
5742 		if (spa == NULL) {
5743 			mutex_enter(&spa_namespace_lock);
5744 			spa = spa_lookup(name);
5745 			if (spa)
5746 				spa_altroot(spa, altroot, buflen);
5747 			else
5748 				altroot[0] = '\0';
5749 			spa = NULL;
5750 			mutex_exit(&spa_namespace_lock);
5751 		} else {
5752 			spa_altroot(spa, altroot, buflen);
5753 		}
5754 	}
5755 
5756 	if (spa != NULL) {
5757 		spa_config_exit(spa, SCL_CONFIG, FTAG);
5758 		spa_close(spa, FTAG);
5759 	}
5760 
5761 	return (error);
5762 }
5763 
5764 /*
5765  * Validate that the auxiliary device array is well formed.  We must have an
5766  * array of nvlists, each which describes a valid leaf vdev.  If this is an
5767  * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be
5768  * specified, as long as they are well-formed.
5769  */
5770 static int
5771 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode,
5772     spa_aux_vdev_t *sav, const char *config, uint64_t version,
5773     vdev_labeltype_t label)
5774 {
5775 	nvlist_t **dev;
5776 	uint_t i, ndev;
5777 	vdev_t *vd;
5778 	int error;
5779 
5780 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5781 
5782 	/*
5783 	 * It's acceptable to have no devs specified.
5784 	 */
5785 	if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0)
5786 		return (0);
5787 
5788 	if (ndev == 0)
5789 		return (SET_ERROR(EINVAL));
5790 
5791 	/*
5792 	 * Make sure the pool is formatted with a version that supports this
5793 	 * device type.
5794 	 */
5795 	if (spa_version(spa) < version)
5796 		return (SET_ERROR(ENOTSUP));
5797 
5798 	/*
5799 	 * Set the pending device list so we correctly handle device in-use
5800 	 * checking.
5801 	 */
5802 	sav->sav_pending = dev;
5803 	sav->sav_npending = ndev;
5804 
5805 	for (i = 0; i < ndev; i++) {
5806 		if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0,
5807 		    mode)) != 0)
5808 			goto out;
5809 
5810 		if (!vd->vdev_ops->vdev_op_leaf) {
5811 			vdev_free(vd);
5812 			error = SET_ERROR(EINVAL);
5813 			goto out;
5814 		}
5815 
5816 		vd->vdev_top = vd;
5817 
5818 		if ((error = vdev_open(vd)) == 0 &&
5819 		    (error = vdev_label_init(vd, crtxg, label)) == 0) {
5820 			fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID,
5821 			    vd->vdev_guid);
5822 		}
5823 
5824 		vdev_free(vd);
5825 
5826 		if (error &&
5827 		    (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE))
5828 			goto out;
5829 		else
5830 			error = 0;
5831 	}
5832 
5833 out:
5834 	sav->sav_pending = NULL;
5835 	sav->sav_npending = 0;
5836 	return (error);
5837 }
5838 
5839 static int
5840 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode)
5841 {
5842 	int error;
5843 
5844 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5845 
5846 	if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode,
5847 	    &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
5848 	    VDEV_LABEL_SPARE)) != 0) {
5849 		return (error);
5850 	}
5851 
5852 	return (spa_validate_aux_devs(spa, nvroot, crtxg, mode,
5853 	    &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
5854 	    VDEV_LABEL_L2CACHE));
5855 }
5856 
5857 static void
5858 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs,
5859     const char *config)
5860 {
5861 	int i;
5862 
5863 	if (sav->sav_config != NULL) {
5864 		nvlist_t **olddevs;
5865 		uint_t oldndevs;
5866 		nvlist_t **newdevs;
5867 
5868 		/*
5869 		 * Generate new dev list by concatenating with the
5870 		 * current dev list.
5871 		 */
5872 		VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config,
5873 		    &olddevs, &oldndevs));
5874 
5875 		newdevs = kmem_alloc(sizeof (void *) *
5876 		    (ndevs + oldndevs), KM_SLEEP);
5877 		for (i = 0; i < oldndevs; i++)
5878 			newdevs[i] = fnvlist_dup(olddevs[i]);
5879 		for (i = 0; i < ndevs; i++)
5880 			newdevs[i + oldndevs] = fnvlist_dup(devs[i]);
5881 
5882 		fnvlist_remove(sav->sav_config, config);
5883 
5884 		fnvlist_add_nvlist_array(sav->sav_config, config,
5885 		    (const nvlist_t * const *)newdevs, ndevs + oldndevs);
5886 		for (i = 0; i < oldndevs + ndevs; i++)
5887 			nvlist_free(newdevs[i]);
5888 		kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *));
5889 	} else {
5890 		/*
5891 		 * Generate a new dev list.
5892 		 */
5893 		sav->sav_config = fnvlist_alloc();
5894 		fnvlist_add_nvlist_array(sav->sav_config, config,
5895 		    (const nvlist_t * const *)devs, ndevs);
5896 	}
5897 }
5898 
5899 /*
5900  * Stop and drop level 2 ARC devices
5901  */
5902 void
5903 spa_l2cache_drop(spa_t *spa)
5904 {
5905 	vdev_t *vd;
5906 	int i;
5907 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
5908 
5909 	for (i = 0; i < sav->sav_count; i++) {
5910 		uint64_t pool;
5911 
5912 		vd = sav->sav_vdevs[i];
5913 		ASSERT(vd != NULL);
5914 
5915 		if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
5916 		    pool != 0ULL && l2arc_vdev_present(vd))
5917 			l2arc_remove_vdev(vd);
5918 	}
5919 }
5920 
5921 /*
5922  * Verify encryption parameters for spa creation. If we are encrypting, we must
5923  * have the encryption feature flag enabled.
5924  */
5925 static int
5926 spa_create_check_encryption_params(dsl_crypto_params_t *dcp,
5927     boolean_t has_encryption)
5928 {
5929 	if (dcp->cp_crypt != ZIO_CRYPT_OFF &&
5930 	    dcp->cp_crypt != ZIO_CRYPT_INHERIT &&
5931 	    !has_encryption)
5932 		return (SET_ERROR(ENOTSUP));
5933 
5934 	return (dmu_objset_create_crypt_check(NULL, dcp, NULL));
5935 }
5936 
5937 /*
5938  * Pool Creation
5939  */
5940 int
5941 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
5942     nvlist_t *zplprops, dsl_crypto_params_t *dcp)
5943 {
5944 	spa_t *spa;
5945 	const char *altroot = NULL;
5946 	vdev_t *rvd;
5947 	dsl_pool_t *dp;
5948 	dmu_tx_t *tx;
5949 	int error = 0;
5950 	uint64_t txg = TXG_INITIAL;
5951 	nvlist_t **spares, **l2cache;
5952 	uint_t nspares, nl2cache;
5953 	uint64_t version, obj, ndraid = 0;
5954 	boolean_t has_features;
5955 	boolean_t has_encryption;
5956 	boolean_t has_allocclass;
5957 	spa_feature_t feat;
5958 	const char *feat_name;
5959 	const char *poolname;
5960 	nvlist_t *nvl;
5961 
5962 	if (props == NULL ||
5963 	    nvlist_lookup_string(props, "tname", &poolname) != 0)
5964 		poolname = (char *)pool;
5965 
5966 	/*
5967 	 * If this pool already exists, return failure.
5968 	 */
5969 	mutex_enter(&spa_namespace_lock);
5970 	if (spa_lookup(poolname) != NULL) {
5971 		mutex_exit(&spa_namespace_lock);
5972 		return (SET_ERROR(EEXIST));
5973 	}
5974 
5975 	/*
5976 	 * Allocate a new spa_t structure.
5977 	 */
5978 	nvl = fnvlist_alloc();
5979 	fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool);
5980 	(void) nvlist_lookup_string(props,
5981 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
5982 	spa = spa_add(poolname, nvl, altroot);
5983 	fnvlist_free(nvl);
5984 	spa_activate(spa, spa_mode_global);
5985 
5986 	if (props && (error = spa_prop_validate(spa, props))) {
5987 		spa_deactivate(spa);
5988 		spa_remove(spa);
5989 		mutex_exit(&spa_namespace_lock);
5990 		return (error);
5991 	}
5992 
5993 	/*
5994 	 * Temporary pool names should never be written to disk.
5995 	 */
5996 	if (poolname != pool)
5997 		spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME;
5998 
5999 	has_features = B_FALSE;
6000 	has_encryption = B_FALSE;
6001 	has_allocclass = B_FALSE;
6002 	for (nvpair_t *elem = nvlist_next_nvpair(props, NULL);
6003 	    elem != NULL; elem = nvlist_next_nvpair(props, elem)) {
6004 		if (zpool_prop_feature(nvpair_name(elem))) {
6005 			has_features = B_TRUE;
6006 
6007 			feat_name = strchr(nvpair_name(elem), '@') + 1;
6008 			VERIFY0(zfeature_lookup_name(feat_name, &feat));
6009 			if (feat == SPA_FEATURE_ENCRYPTION)
6010 				has_encryption = B_TRUE;
6011 			if (feat == SPA_FEATURE_ALLOCATION_CLASSES)
6012 				has_allocclass = B_TRUE;
6013 		}
6014 	}
6015 
6016 	/* verify encryption params, if they were provided */
6017 	if (dcp != NULL) {
6018 		error = spa_create_check_encryption_params(dcp, has_encryption);
6019 		if (error != 0) {
6020 			spa_deactivate(spa);
6021 			spa_remove(spa);
6022 			mutex_exit(&spa_namespace_lock);
6023 			return (error);
6024 		}
6025 	}
6026 	if (!has_allocclass && zfs_special_devs(nvroot, NULL)) {
6027 		spa_deactivate(spa);
6028 		spa_remove(spa);
6029 		mutex_exit(&spa_namespace_lock);
6030 		return (ENOTSUP);
6031 	}
6032 
6033 	if (has_features || nvlist_lookup_uint64(props,
6034 	    zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) {
6035 		version = SPA_VERSION;
6036 	}
6037 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
6038 
6039 	spa->spa_first_txg = txg;
6040 	spa->spa_uberblock.ub_txg = txg - 1;
6041 	spa->spa_uberblock.ub_version = version;
6042 	spa->spa_ubsync = spa->spa_uberblock;
6043 	spa->spa_load_state = SPA_LOAD_CREATE;
6044 	spa->spa_removing_phys.sr_state = DSS_NONE;
6045 	spa->spa_removing_phys.sr_removing_vdev = -1;
6046 	spa->spa_removing_phys.sr_prev_indirect_vdev = -1;
6047 	spa->spa_indirect_vdevs_loaded = B_TRUE;
6048 
6049 	/*
6050 	 * Create "The Godfather" zio to hold all async IOs
6051 	 */
6052 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
6053 	    KM_SLEEP);
6054 	for (int i = 0; i < max_ncpus; i++) {
6055 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
6056 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
6057 		    ZIO_FLAG_GODFATHER);
6058 	}
6059 
6060 	/*
6061 	 * Create the root vdev.
6062 	 */
6063 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6064 
6065 	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);
6066 
6067 	ASSERT(error != 0 || rvd != NULL);
6068 	ASSERT(error != 0 || spa->spa_root_vdev == rvd);
6069 
6070 	if (error == 0 && !zfs_allocatable_devs(nvroot))
6071 		error = SET_ERROR(EINVAL);
6072 
6073 	if (error == 0 &&
6074 	    (error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
6075 	    (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 &&
6076 	    (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) {
6077 		/*
6078 		 * instantiate the metaslab groups (this will dirty the vdevs)
6079 		 * we can no longer error exit past this point
6080 		 */
6081 		for (int c = 0; error == 0 && c < rvd->vdev_children; c++) {
6082 			vdev_t *vd = rvd->vdev_child[c];
6083 
6084 			vdev_metaslab_set_size(vd);
6085 			vdev_expand(vd, txg);
6086 		}
6087 	}
6088 
6089 	spa_config_exit(spa, SCL_ALL, FTAG);
6090 
6091 	if (error != 0) {
6092 		spa_unload(spa);
6093 		spa_deactivate(spa);
6094 		spa_remove(spa);
6095 		mutex_exit(&spa_namespace_lock);
6096 		return (error);
6097 	}
6098 
6099 	/*
6100 	 * Get the list of spares, if specified.
6101 	 */
6102 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6103 	    &spares, &nspares) == 0) {
6104 		spa->spa_spares.sav_config = fnvlist_alloc();
6105 		fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
6106 		    ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
6107 		    nspares);
6108 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6109 		spa_load_spares(spa);
6110 		spa_config_exit(spa, SCL_ALL, FTAG);
6111 		spa->spa_spares.sav_sync = B_TRUE;
6112 	}
6113 
6114 	/*
6115 	 * Get the list of level 2 cache devices, if specified.
6116 	 */
6117 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6118 	    &l2cache, &nl2cache) == 0) {
6119 		VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config,
6120 		    NV_UNIQUE_NAME, KM_SLEEP));
6121 		fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
6122 		    ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache,
6123 		    nl2cache);
6124 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6125 		spa_load_l2cache(spa);
6126 		spa_config_exit(spa, SCL_ALL, FTAG);
6127 		spa->spa_l2cache.sav_sync = B_TRUE;
6128 	}
6129 
6130 	spa->spa_is_initializing = B_TRUE;
6131 	spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg);
6132 	spa->spa_is_initializing = B_FALSE;
6133 
6134 	/*
6135 	 * Create DDTs (dedup tables).
6136 	 */
6137 	ddt_create(spa);
6138 	/*
6139 	 * Create BRT table and BRT table object.
6140 	 */
6141 	brt_create(spa);
6142 
6143 	spa_update_dspace(spa);
6144 
6145 	tx = dmu_tx_create_assigned(dp, txg);
6146 
6147 	/*
6148 	 * Create the pool's history object.
6149 	 */
6150 	if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history)
6151 		spa_history_create_obj(spa, tx);
6152 
6153 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE);
6154 	spa_history_log_version(spa, "create", tx);
6155 
6156 	/*
6157 	 * Create the pool config object.
6158 	 */
6159 	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
6160 	    DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE,
6161 	    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
6162 
6163 	if (zap_add(spa->spa_meta_objset,
6164 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
6165 	    sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
6166 		cmn_err(CE_PANIC, "failed to add pool config");
6167 	}
6168 
6169 	if (zap_add(spa->spa_meta_objset,
6170 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION,
6171 	    sizeof (uint64_t), 1, &version, tx) != 0) {
6172 		cmn_err(CE_PANIC, "failed to add pool version");
6173 	}
6174 
6175 	/* Newly created pools with the right version are always deflated. */
6176 	if (version >= SPA_VERSION_RAIDZ_DEFLATE) {
6177 		spa->spa_deflate = TRUE;
6178 		if (zap_add(spa->spa_meta_objset,
6179 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
6180 		    sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
6181 			cmn_err(CE_PANIC, "failed to add deflate");
6182 		}
6183 	}
6184 
6185 	/*
6186 	 * Create the deferred-free bpobj.  Turn off compression
6187 	 * because sync-to-convergence takes longer if the blocksize
6188 	 * keeps changing.
6189 	 */
6190 	obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
6191 	dmu_object_set_compress(spa->spa_meta_objset, obj,
6192 	    ZIO_COMPRESS_OFF, tx);
6193 	if (zap_add(spa->spa_meta_objset,
6194 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ,
6195 	    sizeof (uint64_t), 1, &obj, tx) != 0) {
6196 		cmn_err(CE_PANIC, "failed to add bpobj");
6197 	}
6198 	VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
6199 	    spa->spa_meta_objset, obj));
6200 
6201 	/*
6202 	 * Generate some random noise for salted checksums to operate on.
6203 	 */
6204 	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
6205 	    sizeof (spa->spa_cksum_salt.zcs_bytes));
6206 
6207 	/*
6208 	 * Set pool properties.
6209 	 */
6210 	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
6211 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
6212 	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
6213 	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
6214 	spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST);
6215 	spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM);
6216 
6217 	if (props != NULL) {
6218 		spa_configfile_set(spa, props, B_FALSE);
6219 		spa_sync_props(props, tx);
6220 	}
6221 
6222 	for (int i = 0; i < ndraid; i++)
6223 		spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
6224 
6225 	dmu_tx_commit(tx);
6226 
6227 	spa->spa_sync_on = B_TRUE;
6228 	txg_sync_start(dp);
6229 	mmp_thread_start(spa);
6230 	txg_wait_synced(dp, txg);
6231 
6232 	spa_spawn_aux_threads(spa);
6233 
6234 	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
6235 
6236 	/*
6237 	 * Don't count references from objsets that are already closed
6238 	 * and are making their way through the eviction process.
6239 	 */
6240 	spa_evicting_os_wait(spa);
6241 	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
6242 	spa->spa_load_state = SPA_LOAD_NONE;
6243 
6244 	spa_import_os(spa);
6245 
6246 	mutex_exit(&spa_namespace_lock);
6247 
6248 	return (0);
6249 }
6250 
6251 /*
6252  * Import a non-root pool into the system.
6253  */
6254 int
6255 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
6256 {
6257 	spa_t *spa;
6258 	const char *altroot = NULL;
6259 	spa_load_state_t state = SPA_LOAD_IMPORT;
6260 	zpool_load_policy_t policy;
6261 	spa_mode_t mode = spa_mode_global;
6262 	uint64_t readonly = B_FALSE;
6263 	int error;
6264 	nvlist_t *nvroot;
6265 	nvlist_t **spares, **l2cache;
6266 	uint_t nspares, nl2cache;
6267 
6268 	/*
6269 	 * If a pool with this name exists, return failure.
6270 	 */
6271 	mutex_enter(&spa_namespace_lock);
6272 	if (spa_lookup(pool) != NULL) {
6273 		mutex_exit(&spa_namespace_lock);
6274 		return (SET_ERROR(EEXIST));
6275 	}
6276 
6277 	/*
6278 	 * Create and initialize the spa structure.
6279 	 */
6280 	(void) nvlist_lookup_string(props,
6281 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
6282 	(void) nvlist_lookup_uint64(props,
6283 	    zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
6284 	if (readonly)
6285 		mode = SPA_MODE_READ;
6286 	spa = spa_add(pool, config, altroot);
6287 	spa->spa_import_flags = flags;
6288 
6289 	/*
6290 	 * Verbatim import - Take a pool and insert it into the namespace
6291 	 * as if it had been loaded at boot.
6292 	 */
6293 	if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
6294 		if (props != NULL)
6295 			spa_configfile_set(spa, props, B_FALSE);
6296 
6297 		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
6298 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
6299 		zfs_dbgmsg("spa_import: verbatim import of %s", pool);
6300 		mutex_exit(&spa_namespace_lock);
6301 		return (0);
6302 	}
6303 
6304 	spa_activate(spa, mode);
6305 
6306 	/*
6307 	 * Don't start async tasks until we know everything is healthy.
6308 	 */
6309 	spa_async_suspend(spa);
6310 
6311 	zpool_get_load_policy(config, &policy);
6312 	if (policy.zlp_rewind & ZPOOL_DO_REWIND)
6313 		state = SPA_LOAD_RECOVER;
6314 
6315 	spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT;
6316 
6317 	if (state != SPA_LOAD_RECOVER) {
6318 		spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
6319 		zfs_dbgmsg("spa_import: importing %s", pool);
6320 	} else {
6321 		zfs_dbgmsg("spa_import: importing %s, max_txg=%lld "
6322 		    "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg);
6323 	}
6324 	error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind);
6325 
6326 	/*
6327 	 * Propagate anything learned while loading the pool and pass it
6328 	 * back to caller (i.e. rewind info, missing devices, etc).
6329 	 */
6330 	fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info);
6331 
6332 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6333 	/*
6334 	 * Toss any existing sparelist, as it doesn't have any validity
6335 	 * anymore, and conflicts with spa_has_spare().
6336 	 */
6337 	if (spa->spa_spares.sav_config) {
6338 		nvlist_free(spa->spa_spares.sav_config);
6339 		spa->spa_spares.sav_config = NULL;
6340 		spa_load_spares(spa);
6341 	}
6342 	if (spa->spa_l2cache.sav_config) {
6343 		nvlist_free(spa->spa_l2cache.sav_config);
6344 		spa->spa_l2cache.sav_config = NULL;
6345 		spa_load_l2cache(spa);
6346 	}
6347 
6348 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6349 	spa_config_exit(spa, SCL_ALL, FTAG);
6350 
6351 	if (props != NULL)
6352 		spa_configfile_set(spa, props, B_FALSE);
6353 
6354 	if (error != 0 || (props && spa_writeable(spa) &&
6355 	    (error = spa_prop_set(spa, props)))) {
6356 		spa_unload(spa);
6357 		spa_deactivate(spa);
6358 		spa_remove(spa);
6359 		mutex_exit(&spa_namespace_lock);
6360 		return (error);
6361 	}
6362 
6363 	spa_async_resume(spa);
6364 
6365 	/*
6366 	 * Override any spares and level 2 cache devices as specified by
6367 	 * the user, as these may have correct device names/devids, etc.
6368 	 */
6369 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6370 	    &spares, &nspares) == 0) {
6371 		if (spa->spa_spares.sav_config)
6372 			fnvlist_remove(spa->spa_spares.sav_config,
6373 			    ZPOOL_CONFIG_SPARES);
6374 		else
6375 			spa->spa_spares.sav_config = fnvlist_alloc();
6376 		fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
6377 		    ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
6378 		    nspares);
6379 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6380 		spa_load_spares(spa);
6381 		spa_config_exit(spa, SCL_ALL, FTAG);
6382 		spa->spa_spares.sav_sync = B_TRUE;
6383 	}
6384 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6385 	    &l2cache, &nl2cache) == 0) {
6386 		if (spa->spa_l2cache.sav_config)
6387 			fnvlist_remove(spa->spa_l2cache.sav_config,
6388 			    ZPOOL_CONFIG_L2CACHE);
6389 		else
6390 			spa->spa_l2cache.sav_config = fnvlist_alloc();
6391 		fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
6392 		    ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache,
6393 		    nl2cache);
6394 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6395 		spa_load_l2cache(spa);
6396 		spa_config_exit(spa, SCL_ALL, FTAG);
6397 		spa->spa_l2cache.sav_sync = B_TRUE;
6398 	}
6399 
6400 	/*
6401 	 * Check for any removed devices.
6402 	 */
6403 	if (spa->spa_autoreplace) {
6404 		spa_aux_check_removed(&spa->spa_spares);
6405 		spa_aux_check_removed(&spa->spa_l2cache);
6406 	}
6407 
6408 	if (spa_writeable(spa)) {
6409 		/*
6410 		 * Update the config cache to include the newly-imported pool.
6411 		 */
6412 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
6413 	}
6414 
6415 	/*
6416 	 * It's possible that the pool was expanded while it was exported.
6417 	 * We kick off an async task to handle this for us.
6418 	 */
6419 	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
6420 
6421 	spa_history_log_version(spa, "import", NULL);
6422 
6423 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
6424 
6425 	mutex_exit(&spa_namespace_lock);
6426 
6427 	zvol_create_minors_recursive(pool);
6428 
6429 	spa_import_os(spa);
6430 
6431 	return (0);
6432 }
6433 
6434 nvlist_t *
6435 spa_tryimport(nvlist_t *tryconfig)
6436 {
6437 	nvlist_t *config = NULL;
6438 	const char *poolname, *cachefile;
6439 	spa_t *spa;
6440 	uint64_t state;
6441 	int error;
6442 	zpool_load_policy_t policy;
6443 
6444 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
6445 		return (NULL);
6446 
6447 	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
6448 		return (NULL);
6449 
6450 	/*
6451 	 * Create and initialize the spa structure.
6452 	 */
6453 	mutex_enter(&spa_namespace_lock);
6454 	spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL);
6455 	spa_activate(spa, SPA_MODE_READ);
6456 
6457 	/*
6458 	 * Rewind pool if a max txg was provided.
6459 	 */
6460 	zpool_get_load_policy(spa->spa_config, &policy);
6461 	if (policy.zlp_txg != UINT64_MAX) {
6462 		spa->spa_load_max_txg = policy.zlp_txg;
6463 		spa->spa_extreme_rewind = B_TRUE;
6464 		zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld",
6465 		    poolname, (longlong_t)policy.zlp_txg);
6466 	} else {
6467 		zfs_dbgmsg("spa_tryimport: importing %s", poolname);
6468 	}
6469 
6470 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile)
6471 	    == 0) {
6472 		zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile);
6473 		spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
6474 	} else {
6475 		spa->spa_config_source = SPA_CONFIG_SRC_SCAN;
6476 	}
6477 
6478 	/*
6479 	 * spa_import() relies on a pool config fetched by spa_try_import()
6480 	 * for spare/cache devices. Import flags are not passed to
6481 	 * spa_tryimport(), which makes it return early due to a missing log
6482 	 * device and missing retrieving the cache device and spare eventually.
6483 	 * Passing ZFS_IMPORT_MISSING_LOG to spa_tryimport() makes it fetch
6484 	 * the correct configuration regardless of the missing log device.
6485 	 */
6486 	spa->spa_import_flags |= ZFS_IMPORT_MISSING_LOG;
6487 
6488 	error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING);
6489 
6490 	/*
6491 	 * If 'tryconfig' was at least parsable, return the current config.
6492 	 */
6493 	if (spa->spa_root_vdev != NULL) {
6494 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
6495 		fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname);
6496 		fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state);
6497 		fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
6498 		    spa->spa_uberblock.ub_timestamp);
6499 		fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
6500 		    spa->spa_load_info);
6501 		fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA,
6502 		    spa->spa_errata);
6503 
6504 		/*
6505 		 * If the bootfs property exists on this pool then we
6506 		 * copy it out so that external consumers can tell which
6507 		 * pools are bootable.
6508 		 */
6509 		if ((!error || error == EEXIST) && spa->spa_bootfs) {
6510 			char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
6511 
6512 			/*
6513 			 * We have to play games with the name since the
6514 			 * pool was opened as TRYIMPORT_NAME.
6515 			 */
6516 			if (dsl_dsobj_to_dsname(spa_name(spa),
6517 			    spa->spa_bootfs, tmpname) == 0) {
6518 				char *cp;
6519 				char *dsname;
6520 
6521 				dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
6522 
6523 				cp = strchr(tmpname, '/');
6524 				if (cp == NULL) {
6525 					(void) strlcpy(dsname, tmpname,
6526 					    MAXPATHLEN);
6527 				} else {
6528 					(void) snprintf(dsname, MAXPATHLEN,
6529 					    "%s/%s", poolname, ++cp);
6530 				}
6531 				fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS,
6532 				    dsname);
6533 				kmem_free(dsname, MAXPATHLEN);
6534 			}
6535 			kmem_free(tmpname, MAXPATHLEN);
6536 		}
6537 
6538 		/*
6539 		 * Add the list of hot spares and level 2 cache devices.
6540 		 */
6541 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
6542 		spa_add_spares(spa, config);
6543 		spa_add_l2cache(spa, config);
6544 		spa_config_exit(spa, SCL_CONFIG, FTAG);
6545 	}
6546 
6547 	spa_unload(spa);
6548 	spa_deactivate(spa);
6549 	spa_remove(spa);
6550 	mutex_exit(&spa_namespace_lock);
6551 
6552 	return (config);
6553 }
6554 
6555 /*
6556  * Pool export/destroy
6557  *
6558  * The act of destroying or exporting a pool is very simple.  We make sure there
6559  * is no more pending I/O and any references to the pool are gone.  Then, we
6560  * update the pool state and sync all the labels to disk, removing the
6561  * configuration from the cache afterwards. If the 'hardforce' flag is set, then
6562  * we don't sync the labels or remove the configuration cache.
6563  */
6564 static int
6565 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig,
6566     boolean_t force, boolean_t hardforce)
6567 {
6568 	int error;
6569 	spa_t *spa;
6570 
6571 	if (oldconfig)
6572 		*oldconfig = NULL;
6573 
6574 	if (!(spa_mode_global & SPA_MODE_WRITE))
6575 		return (SET_ERROR(EROFS));
6576 
6577 	mutex_enter(&spa_namespace_lock);
6578 	if ((spa = spa_lookup(pool)) == NULL) {
6579 		mutex_exit(&spa_namespace_lock);
6580 		return (SET_ERROR(ENOENT));
6581 	}
6582 
6583 	if (spa->spa_is_exporting) {
6584 		/* the pool is being exported by another thread */
6585 		mutex_exit(&spa_namespace_lock);
6586 		return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS));
6587 	}
6588 	spa->spa_is_exporting = B_TRUE;
6589 
6590 	/*
6591 	 * Put a hold on the pool, drop the namespace lock, stop async tasks,
6592 	 * reacquire the namespace lock, and see if we can export.
6593 	 */
6594 	spa_open_ref(spa, FTAG);
6595 	mutex_exit(&spa_namespace_lock);
6596 	spa_async_suspend(spa);
6597 	if (spa->spa_zvol_taskq) {
6598 		zvol_remove_minors(spa, spa_name(spa), B_TRUE);
6599 		taskq_wait(spa->spa_zvol_taskq);
6600 	}
6601 	mutex_enter(&spa_namespace_lock);
6602 	spa_close(spa, FTAG);
6603 
6604 	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
6605 		goto export_spa;
6606 	/*
6607 	 * The pool will be in core if it's openable, in which case we can
6608 	 * modify its state.  Objsets may be open only because they're dirty,
6609 	 * so we have to force it to sync before checking spa_refcnt.
6610 	 */
6611 	if (spa->spa_sync_on) {
6612 		txg_wait_synced(spa->spa_dsl_pool, 0);
6613 		spa_evicting_os_wait(spa);
6614 	}
6615 
6616 	/*
6617 	 * A pool cannot be exported or destroyed if there are active
6618 	 * references.  If we are resetting a pool, allow references by
6619 	 * fault injection handlers.
6620 	 */
6621 	if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) {
6622 		error = SET_ERROR(EBUSY);
6623 		goto fail;
6624 	}
6625 
6626 	if (spa->spa_sync_on) {
6627 		vdev_t *rvd = spa->spa_root_vdev;
6628 		/*
6629 		 * A pool cannot be exported if it has an active shared spare.
6630 		 * This is to prevent other pools stealing the active spare
6631 		 * from an exported pool. At user's own will, such pool can
6632 		 * be forcedly exported.
6633 		 */
6634 		if (!force && new_state == POOL_STATE_EXPORTED &&
6635 		    spa_has_active_shared_spare(spa)) {
6636 			error = SET_ERROR(EXDEV);
6637 			goto fail;
6638 		}
6639 
6640 		/*
6641 		 * We're about to export or destroy this pool. Make sure
6642 		 * we stop all initialization and trim activity here before
6643 		 * we set the spa_final_txg. This will ensure that all
6644 		 * dirty data resulting from the initialization is
6645 		 * committed to disk before we unload the pool.
6646 		 */
6647 		vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE);
6648 		vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE);
6649 		vdev_autotrim_stop_all(spa);
6650 		vdev_rebuild_stop_all(spa);
6651 
6652 		/*
6653 		 * We want this to be reflected on every label,
6654 		 * so mark them all dirty.  spa_unload() will do the
6655 		 * final sync that pushes these changes out.
6656 		 */
6657 		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
6658 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6659 			spa->spa_state = new_state;
6660 			vdev_config_dirty(rvd);
6661 			spa_config_exit(spa, SCL_ALL, FTAG);
6662 		}
6663 
6664 		/*
6665 		 * If the log space map feature is enabled and the pool is
6666 		 * getting exported (but not destroyed), we want to spend some
6667 		 * time flushing as many metaslabs as we can in an attempt to
6668 		 * destroy log space maps and save import time. This has to be
6669 		 * done before we set the spa_final_txg, otherwise
6670 		 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs.
6671 		 * spa_should_flush_logs_on_unload() should be called after
6672 		 * spa_state has been set to the new_state.
6673 		 */
6674 		if (spa_should_flush_logs_on_unload(spa))
6675 			spa_unload_log_sm_flush_all(spa);
6676 
6677 		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
6678 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6679 			spa->spa_final_txg = spa_last_synced_txg(spa) +
6680 			    TXG_DEFER_SIZE + 1;
6681 			spa_config_exit(spa, SCL_ALL, FTAG);
6682 		}
6683 	}
6684 
6685 export_spa:
6686 	spa_export_os(spa);
6687 
6688 	if (new_state == POOL_STATE_DESTROYED)
6689 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY);
6690 	else if (new_state == POOL_STATE_EXPORTED)
6691 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT);
6692 
6693 	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
6694 		spa_unload(spa);
6695 		spa_deactivate(spa);
6696 	}
6697 
6698 	if (oldconfig && spa->spa_config)
6699 		*oldconfig = fnvlist_dup(spa->spa_config);
6700 
6701 	if (new_state != POOL_STATE_UNINITIALIZED) {
6702 		if (!hardforce)
6703 			spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
6704 		spa_remove(spa);
6705 	} else {
6706 		/*
6707 		 * If spa_remove() is not called for this spa_t and
6708 		 * there is any possibility that it can be reused,
6709 		 * we make sure to reset the exporting flag.
6710 		 */
6711 		spa->spa_is_exporting = B_FALSE;
6712 	}
6713 
6714 	mutex_exit(&spa_namespace_lock);
6715 	return (0);
6716 
6717 fail:
6718 	spa->spa_is_exporting = B_FALSE;
6719 	spa_async_resume(spa);
6720 	mutex_exit(&spa_namespace_lock);
6721 	return (error);
6722 }
6723 
6724 /*
6725  * Destroy a storage pool.
6726  */
6727 int
6728 spa_destroy(const char *pool)
6729 {
6730 	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL,
6731 	    B_FALSE, B_FALSE));
6732 }
6733 
6734 /*
6735  * Export a storage pool.
6736  */
6737 int
6738 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force,
6739     boolean_t hardforce)
6740 {
6741 	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig,
6742 	    force, hardforce));
6743 }
6744 
6745 /*
6746  * Similar to spa_export(), this unloads the spa_t without actually removing it
6747  * from the namespace in any way.
6748  */
6749 int
6750 spa_reset(const char *pool)
6751 {
6752 	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL,
6753 	    B_FALSE, B_FALSE));
6754 }
6755 
6756 /*
6757  * ==========================================================================
6758  * Device manipulation
6759  * ==========================================================================
6760  */
6761 
6762 /*
6763  * This is called as a synctask to increment the draid feature flag
6764  */
6765 static void
6766 spa_draid_feature_incr(void *arg, dmu_tx_t *tx)
6767 {
6768 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
6769 	int draid = (int)(uintptr_t)arg;
6770 
6771 	for (int c = 0; c < draid; c++)
6772 		spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
6773 }
6774 
6775 /*
6776  * Add a device to a storage pool.
6777  */
6778 int
6779 spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
6780 {
6781 	uint64_t txg, ndraid = 0;
6782 	int error;
6783 	vdev_t *rvd = spa->spa_root_vdev;
6784 	vdev_t *vd, *tvd;
6785 	nvlist_t **spares, **l2cache;
6786 	uint_t nspares, nl2cache;
6787 
6788 	ASSERT(spa_writeable(spa));
6789 
6790 	txg = spa_vdev_enter(spa);
6791 
6792 	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
6793 	    VDEV_ALLOC_ADD)) != 0)
6794 		return (spa_vdev_exit(spa, NULL, txg, error));
6795 
6796 	spa->spa_pending_vdev = vd;	/* spa_vdev_exit() will clear this */
6797 
6798 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
6799 	    &nspares) != 0)
6800 		nspares = 0;
6801 
6802 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache,
6803 	    &nl2cache) != 0)
6804 		nl2cache = 0;
6805 
6806 	if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
6807 		return (spa_vdev_exit(spa, vd, txg, EINVAL));
6808 
6809 	if (vd->vdev_children != 0 &&
6810 	    (error = vdev_create(vd, txg, B_FALSE)) != 0) {
6811 		return (spa_vdev_exit(spa, vd, txg, error));
6812 	}
6813 
6814 	/*
6815 	 * The virtual dRAID spares must be added after vdev tree is created
6816 	 * and the vdev guids are generated.  The guid of their associated
6817 	 * dRAID is stored in the config and used when opening the spare.
6818 	 */
6819 	if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid,
6820 	    rvd->vdev_children)) == 0) {
6821 		if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot,
6822 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0)
6823 			nspares = 0;
6824 	} else {
6825 		return (spa_vdev_exit(spa, vd, txg, error));
6826 	}
6827 
6828 	/*
6829 	 * We must validate the spares and l2cache devices after checking the
6830 	 * children.  Otherwise, vdev_inuse() will blindly overwrite the spare.
6831 	 */
6832 	if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0)
6833 		return (spa_vdev_exit(spa, vd, txg, error));
6834 
6835 	/*
6836 	 * If we are in the middle of a device removal, we can only add
6837 	 * devices which match the existing devices in the pool.
6838 	 * If we are in the middle of a removal, or have some indirect
6839 	 * vdevs, we can not add raidz or dRAID top levels.
6840 	 */
6841 	if (spa->spa_vdev_removal != NULL ||
6842 	    spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
6843 		for (int c = 0; c < vd->vdev_children; c++) {
6844 			tvd = vd->vdev_child[c];
6845 			if (spa->spa_vdev_removal != NULL &&
6846 			    tvd->vdev_ashift != spa->spa_max_ashift) {
6847 				return (spa_vdev_exit(spa, vd, txg, EINVAL));
6848 			}
6849 			/* Fail if top level vdev is raidz or a dRAID */
6850 			if (vdev_get_nparity(tvd) != 0)
6851 				return (spa_vdev_exit(spa, vd, txg, EINVAL));
6852 
6853 			/*
6854 			 * Need the top level mirror to be
6855 			 * a mirror of leaf vdevs only
6856 			 */
6857 			if (tvd->vdev_ops == &vdev_mirror_ops) {
6858 				for (uint64_t cid = 0;
6859 				    cid < tvd->vdev_children; cid++) {
6860 					vdev_t *cvd = tvd->vdev_child[cid];
6861 					if (!cvd->vdev_ops->vdev_op_leaf) {
6862 						return (spa_vdev_exit(spa, vd,
6863 						    txg, EINVAL));
6864 					}
6865 				}
6866 			}
6867 		}
6868 	}
6869 
6870 	for (int c = 0; c < vd->vdev_children; c++) {
6871 		tvd = vd->vdev_child[c];
6872 		vdev_remove_child(vd, tvd);
6873 		tvd->vdev_id = rvd->vdev_children;
6874 		vdev_add_child(rvd, tvd);
6875 		vdev_config_dirty(tvd);
6876 	}
6877 
6878 	if (nspares != 0) {
6879 		spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
6880 		    ZPOOL_CONFIG_SPARES);
6881 		spa_load_spares(spa);
6882 		spa->spa_spares.sav_sync = B_TRUE;
6883 	}
6884 
6885 	if (nl2cache != 0) {
6886 		spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
6887 		    ZPOOL_CONFIG_L2CACHE);
6888 		spa_load_l2cache(spa);
6889 		spa->spa_l2cache.sav_sync = B_TRUE;
6890 	}
6891 
6892 	/*
6893 	 * We can't increment a feature while holding spa_vdev so we
6894 	 * have to do it in a synctask.
6895 	 */
6896 	if (ndraid != 0) {
6897 		dmu_tx_t *tx;
6898 
6899 		tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
6900 		dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr,
6901 		    (void *)(uintptr_t)ndraid, tx);
6902 		dmu_tx_commit(tx);
6903 	}
6904 
6905 	/*
6906 	 * We have to be careful when adding new vdevs to an existing pool.
6907 	 * If other threads start allocating from these vdevs before we
6908 	 * sync the config cache, and we lose power, then upon reboot we may
6909 	 * fail to open the pool because there are DVAs that the config cache
6910 	 * can't translate.  Therefore, we first add the vdevs without
6911 	 * initializing metaslabs; sync the config cache (via spa_vdev_exit());
6912 	 * and then let spa_config_update() initialize the new metaslabs.
6913 	 *
6914 	 * spa_load() checks for added-but-not-initialized vdevs, so that
6915 	 * if we lose power at any point in this sequence, the remaining
6916 	 * steps will be completed the next time we load the pool.
6917 	 */
6918 	(void) spa_vdev_exit(spa, vd, txg, 0);
6919 
6920 	mutex_enter(&spa_namespace_lock);
6921 	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
6922 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD);
6923 	mutex_exit(&spa_namespace_lock);
6924 
6925 	return (0);
6926 }
6927 
6928 /*
6929  * Attach a device to a vdev specified by its guid.  The vdev type can be
6930  * a mirror, a raidz, or a leaf device that is also a top-level (e.g. a
6931  * single device). When the vdev is a single device, a mirror vdev will be
6932  * automatically inserted.
6933  *
6934  * If 'replacing' is specified, the new device is intended to replace the
6935  * existing device; in this case the two devices are made into their own
6936  * mirror using the 'replacing' vdev, which is functionally identical to
6937  * the mirror vdev (it actually reuses all the same ops) but has a few
6938  * extra rules: you can't attach to it after it's been created, and upon
6939  * completion of resilvering, the first disk (the one being replaced)
6940  * is automatically detached.
6941  *
6942  * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild)
6943  * should be performed instead of traditional healing reconstruction.  From
6944  * an administrators perspective these are both resilver operations.
6945  */
6946 int
6947 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing,
6948     int rebuild)
6949 {
6950 	uint64_t txg, dtl_max_txg;
6951 	vdev_t *rvd = spa->spa_root_vdev;
6952 	vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd;
6953 	vdev_ops_t *pvops;
6954 	char *oldvdpath, *newvdpath;
6955 	int newvd_isspare = B_FALSE;
6956 	int error;
6957 
6958 	ASSERT(spa_writeable(spa));
6959 
6960 	txg = spa_vdev_enter(spa);
6961 
6962 	oldvd = spa_lookup_by_guid(spa, guid, B_FALSE);
6963 
6964 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
6965 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
6966 		error = (spa_has_checkpoint(spa)) ?
6967 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
6968 		return (spa_vdev_exit(spa, NULL, txg, error));
6969 	}
6970 
6971 	if (rebuild) {
6972 		if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD))
6973 			return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
6974 
6975 		if (dsl_scan_resilvering(spa_get_dsl(spa)) ||
6976 		    dsl_scan_resilver_scheduled(spa_get_dsl(spa))) {
6977 			return (spa_vdev_exit(spa, NULL, txg,
6978 			    ZFS_ERR_RESILVER_IN_PROGRESS));
6979 		}
6980 	} else {
6981 		if (vdev_rebuild_active(rvd))
6982 			return (spa_vdev_exit(spa, NULL, txg,
6983 			    ZFS_ERR_REBUILD_IN_PROGRESS));
6984 	}
6985 
6986 	if (spa->spa_vdev_removal != NULL) {
6987 		return (spa_vdev_exit(spa, NULL, txg,
6988 		    ZFS_ERR_DEVRM_IN_PROGRESS));
6989 	}
6990 
6991 	if (oldvd == NULL)
6992 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
6993 
6994 	boolean_t raidz = oldvd->vdev_ops == &vdev_raidz_ops;
6995 
6996 	if (raidz) {
6997 		if (!spa_feature_is_enabled(spa, SPA_FEATURE_RAIDZ_EXPANSION))
6998 			return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
6999 
7000 		/*
7001 		 * Can't expand a raidz while prior expand is in progress.
7002 		 */
7003 		if (spa->spa_raidz_expand != NULL) {
7004 			return (spa_vdev_exit(spa, NULL, txg,
7005 			    ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS));
7006 		}
7007 	} else if (!oldvd->vdev_ops->vdev_op_leaf) {
7008 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7009 	}
7010 
7011 	if (raidz)
7012 		pvd = oldvd;
7013 	else
7014 		pvd = oldvd->vdev_parent;
7015 
7016 	if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
7017 	    VDEV_ALLOC_ATTACH) != 0)
7018 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7019 
7020 	if (newrootvd->vdev_children != 1)
7021 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
7022 
7023 	newvd = newrootvd->vdev_child[0];
7024 
7025 	if (!newvd->vdev_ops->vdev_op_leaf)
7026 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
7027 
7028 	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
7029 		return (spa_vdev_exit(spa, newrootvd, txg, error));
7030 
7031 	/*
7032 	 * log, dedup and special vdevs should not be replaced by spares.
7033 	 */
7034 	if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE ||
7035 	    oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) {
7036 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7037 	}
7038 
7039 	/*
7040 	 * A dRAID spare can only replace a child of its parent dRAID vdev.
7041 	 */
7042 	if (newvd->vdev_ops == &vdev_draid_spare_ops &&
7043 	    oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) {
7044 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7045 	}
7046 
7047 	if (rebuild) {
7048 		/*
7049 		 * For rebuilds, the top vdev must support reconstruction
7050 		 * using only space maps.  This means the only allowable
7051 		 * vdevs types are the root vdev, a mirror, or dRAID.
7052 		 */
7053 		tvd = pvd;
7054 		if (pvd->vdev_top != NULL)
7055 			tvd = pvd->vdev_top;
7056 
7057 		if (tvd->vdev_ops != &vdev_mirror_ops &&
7058 		    tvd->vdev_ops != &vdev_root_ops &&
7059 		    tvd->vdev_ops != &vdev_draid_ops) {
7060 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7061 		}
7062 	}
7063 
7064 	if (!replacing) {
7065 		/*
7066 		 * For attach, the only allowable parent is a mirror or the root
7067 		 * vdev.
7068 		 */
7069 		if (pvd->vdev_ops != &vdev_mirror_ops &&
7070 		    pvd->vdev_ops != &vdev_raidz_ops &&
7071 		    pvd->vdev_ops != &vdev_root_ops)
7072 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7073 
7074 		pvops = &vdev_mirror_ops;
7075 	} else {
7076 		/*
7077 		 * Active hot spares can only be replaced by inactive hot
7078 		 * spares.
7079 		 */
7080 		if (pvd->vdev_ops == &vdev_spare_ops &&
7081 		    oldvd->vdev_isspare &&
7082 		    !spa_has_spare(spa, newvd->vdev_guid))
7083 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7084 
7085 		/*
7086 		 * If the source is a hot spare, and the parent isn't already a
7087 		 * spare, then we want to create a new hot spare.  Otherwise, we
7088 		 * want to create a replacing vdev.  The user is not allowed to
7089 		 * attach to a spared vdev child unless the 'isspare' state is
7090 		 * the same (spare replaces spare, non-spare replaces
7091 		 * non-spare).
7092 		 */
7093 		if (pvd->vdev_ops == &vdev_replacing_ops &&
7094 		    spa_version(spa) < SPA_VERSION_MULTI_REPLACE) {
7095 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7096 		} else if (pvd->vdev_ops == &vdev_spare_ops &&
7097 		    newvd->vdev_isspare != oldvd->vdev_isspare) {
7098 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7099 		}
7100 
7101 		if (newvd->vdev_isspare)
7102 			pvops = &vdev_spare_ops;
7103 		else
7104 			pvops = &vdev_replacing_ops;
7105 	}
7106 
7107 	/*
7108 	 * Make sure the new device is big enough.
7109 	 */
7110 	vdev_t *min_vdev = raidz ? oldvd->vdev_child[0] : oldvd;
7111 	if (newvd->vdev_asize < vdev_get_min_asize(min_vdev))
7112 		return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));
7113 
7114 	/*
7115 	 * The new device cannot have a higher alignment requirement
7116 	 * than the top-level vdev.
7117 	 */
7118 	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
7119 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7120 
7121 	/*
7122 	 * RAIDZ-expansion-specific checks.
7123 	 */
7124 	if (raidz) {
7125 		if (vdev_raidz_attach_check(newvd) != 0)
7126 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7127 
7128 		/*
7129 		 * Fail early if a child is not healthy or being replaced
7130 		 */
7131 		for (int i = 0; i < oldvd->vdev_children; i++) {
7132 			if (vdev_is_dead(oldvd->vdev_child[i]) ||
7133 			    !oldvd->vdev_child[i]->vdev_ops->vdev_op_leaf) {
7134 				return (spa_vdev_exit(spa, newrootvd, txg,
7135 				    ENXIO));
7136 			}
7137 			/* Also fail if reserved boot area is in-use */
7138 			if (vdev_check_boot_reserve(spa, oldvd->vdev_child[i])
7139 			    != 0) {
7140 				return (spa_vdev_exit(spa, newrootvd, txg,
7141 				    EADDRINUSE));
7142 			}
7143 		}
7144 	}
7145 
7146 	if (raidz) {
7147 		/*
7148 		 * Note: oldvdpath is freed by spa_strfree(),  but
7149 		 * kmem_asprintf() is freed by kmem_strfree(), so we have to
7150 		 * move it to a spa_strdup-ed string.
7151 		 */
7152 		char *tmp = kmem_asprintf("raidz%u-%u",
7153 		    (uint_t)vdev_get_nparity(oldvd), (uint_t)oldvd->vdev_id);
7154 		oldvdpath = spa_strdup(tmp);
7155 		kmem_strfree(tmp);
7156 	} else {
7157 		oldvdpath = spa_strdup(oldvd->vdev_path);
7158 	}
7159 	newvdpath = spa_strdup(newvd->vdev_path);
7160 
7161 	/*
7162 	 * If this is an in-place replacement, update oldvd's path and devid
7163 	 * to make it distinguishable from newvd, and unopenable from now on.
7164 	 */
7165 	if (strcmp(oldvdpath, newvdpath) == 0) {
7166 		spa_strfree(oldvd->vdev_path);
7167 		oldvd->vdev_path = kmem_alloc(strlen(newvdpath) + 5,
7168 		    KM_SLEEP);
7169 		(void) sprintf(oldvd->vdev_path, "%s/old",
7170 		    newvdpath);
7171 		if (oldvd->vdev_devid != NULL) {
7172 			spa_strfree(oldvd->vdev_devid);
7173 			oldvd->vdev_devid = NULL;
7174 		}
7175 		spa_strfree(oldvdpath);
7176 		oldvdpath = spa_strdup(oldvd->vdev_path);
7177 	}
7178 
7179 	/*
7180 	 * If the parent is not a mirror, or if we're replacing, insert the new
7181 	 * mirror/replacing/spare vdev above oldvd.
7182 	 */
7183 	if (!raidz && pvd->vdev_ops != pvops) {
7184 		pvd = vdev_add_parent(oldvd, pvops);
7185 		ASSERT(pvd->vdev_ops == pvops);
7186 		ASSERT(oldvd->vdev_parent == pvd);
7187 	}
7188 
7189 	ASSERT(pvd->vdev_top->vdev_parent == rvd);
7190 
7191 	/*
7192 	 * Extract the new device from its root and add it to pvd.
7193 	 */
7194 	vdev_remove_child(newrootvd, newvd);
7195 	newvd->vdev_id = pvd->vdev_children;
7196 	newvd->vdev_crtxg = oldvd->vdev_crtxg;
7197 	vdev_add_child(pvd, newvd);
7198 
7199 	/*
7200 	 * Reevaluate the parent vdev state.
7201 	 */
7202 	vdev_propagate_state(pvd);
7203 
7204 	tvd = newvd->vdev_top;
7205 	ASSERT(pvd->vdev_top == tvd);
7206 	ASSERT(tvd->vdev_parent == rvd);
7207 
7208 	vdev_config_dirty(tvd);
7209 
7210 	/*
7211 	 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account
7212 	 * for any dmu_sync-ed blocks.  It will propagate upward when
7213 	 * spa_vdev_exit() calls vdev_dtl_reassess().
7214 	 */
7215 	dtl_max_txg = txg + TXG_CONCURRENT_STATES;
7216 
7217 	if (raidz) {
7218 		/*
7219 		 * Wait for the youngest allocations and frees to sync,
7220 		 * and then wait for the deferral of those frees to finish.
7221 		 */
7222 		spa_vdev_config_exit(spa, NULL,
7223 		    txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG);
7224 
7225 		vdev_initialize_stop_all(tvd, VDEV_INITIALIZE_ACTIVE);
7226 		vdev_trim_stop_all(tvd, VDEV_TRIM_ACTIVE);
7227 		vdev_autotrim_stop_wait(tvd);
7228 
7229 		dtl_max_txg = spa_vdev_config_enter(spa);
7230 
7231 		tvd->vdev_rz_expanding = B_TRUE;
7232 
7233 		vdev_dirty_leaves(tvd, VDD_DTL, dtl_max_txg);
7234 		vdev_config_dirty(tvd);
7235 
7236 		dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool,
7237 		    dtl_max_txg);
7238 		dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_raidz_attach_sync,
7239 		    newvd, tx);
7240 		dmu_tx_commit(tx);
7241 	} else {
7242 		vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL,
7243 		    dtl_max_txg - TXG_INITIAL);
7244 
7245 		if (newvd->vdev_isspare) {
7246 			spa_spare_activate(newvd);
7247 			spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE);
7248 		}
7249 
7250 		newvd_isspare = newvd->vdev_isspare;
7251 
7252 		/*
7253 		 * Mark newvd's DTL dirty in this txg.
7254 		 */
7255 		vdev_dirty(tvd, VDD_DTL, newvd, txg);
7256 
7257 		/*
7258 		 * Schedule the resilver or rebuild to restart in the future.
7259 		 * We do this to ensure that dmu_sync-ed blocks have been
7260 		 * stitched into the respective datasets.
7261 		 */
7262 		if (rebuild) {
7263 			newvd->vdev_rebuild_txg = txg;
7264 
7265 			vdev_rebuild(tvd);
7266 		} else {
7267 			newvd->vdev_resilver_txg = txg;
7268 
7269 			if (dsl_scan_resilvering(spa_get_dsl(spa)) &&
7270 			    spa_feature_is_enabled(spa,
7271 			    SPA_FEATURE_RESILVER_DEFER)) {
7272 				vdev_defer_resilver(newvd);
7273 			} else {
7274 				dsl_scan_restart_resilver(spa->spa_dsl_pool,
7275 				    dtl_max_txg);
7276 			}
7277 		}
7278 	}
7279 
7280 	if (spa->spa_bootfs)
7281 		spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH);
7282 
7283 	spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH);
7284 
7285 	/*
7286 	 * Commit the config
7287 	 */
7288 	(void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0);
7289 
7290 	spa_history_log_internal(spa, "vdev attach", NULL,
7291 	    "%s vdev=%s %s vdev=%s",
7292 	    replacing && newvd_isspare ? "spare in" :
7293 	    replacing ? "replace" : "attach", newvdpath,
7294 	    replacing ? "for" : "to", oldvdpath);
7295 
7296 	spa_strfree(oldvdpath);
7297 	spa_strfree(newvdpath);
7298 
7299 	return (0);
7300 }
7301 
7302 /*
7303  * Detach a device from a mirror or replacing vdev.
7304  *
7305  * If 'replace_done' is specified, only detach if the parent
7306  * is a replacing or a spare vdev.
7307  */
7308 int
7309 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)
7310 {
7311 	uint64_t txg;
7312 	int error;
7313 	vdev_t *rvd __maybe_unused = spa->spa_root_vdev;
7314 	vdev_t *vd, *pvd, *cvd, *tvd;
7315 	boolean_t unspare = B_FALSE;
7316 	uint64_t unspare_guid = 0;
7317 	char *vdpath;
7318 
7319 	ASSERT(spa_writeable(spa));
7320 
7321 	txg = spa_vdev_detach_enter(spa, guid);
7322 
7323 	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7324 
7325 	/*
7326 	 * Besides being called directly from the userland through the
7327 	 * ioctl interface, spa_vdev_detach() can be potentially called
7328 	 * at the end of spa_vdev_resilver_done().
7329 	 *
7330 	 * In the regular case, when we have a checkpoint this shouldn't
7331 	 * happen as we never empty the DTLs of a vdev during the scrub
7332 	 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done()
7333 	 * should never get here when we have a checkpoint.
7334 	 *
7335 	 * That said, even in a case when we checkpoint the pool exactly
7336 	 * as spa_vdev_resilver_done() calls this function everything
7337 	 * should be fine as the resilver will return right away.
7338 	 */
7339 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7340 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
7341 		error = (spa_has_checkpoint(spa)) ?
7342 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
7343 		return (spa_vdev_exit(spa, NULL, txg, error));
7344 	}
7345 
7346 	if (vd == NULL)
7347 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
7348 
7349 	if (!vd->vdev_ops->vdev_op_leaf)
7350 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7351 
7352 	pvd = vd->vdev_parent;
7353 
7354 	/*
7355 	 * If the parent/child relationship is not as expected, don't do it.
7356 	 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
7357 	 * vdev that's replacing B with C.  The user's intent in replacing
7358 	 * is to go from M(A,B) to M(A,C).  If the user decides to cancel
7359 	 * the replace by detaching C, the expected behavior is to end up
7360 	 * M(A,B).  But suppose that right after deciding to detach C,
7361 	 * the replacement of B completes.  We would have M(A,C), and then
7362 	 * ask to detach C, which would leave us with just A -- not what
7363 	 * the user wanted.  To prevent this, we make sure that the
7364 	 * parent/child relationship hasn't changed -- in this example,
7365 	 * that C's parent is still the replacing vdev R.
7366 	 */
7367 	if (pvd->vdev_guid != pguid && pguid != 0)
7368 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
7369 
7370 	/*
7371 	 * Only 'replacing' or 'spare' vdevs can be replaced.
7372 	 */
7373 	if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
7374 	    pvd->vdev_ops != &vdev_spare_ops)
7375 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7376 
7377 	ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
7378 	    spa_version(spa) >= SPA_VERSION_SPARES);
7379 
7380 	/*
7381 	 * Only mirror, replacing, and spare vdevs support detach.
7382 	 */
7383 	if (pvd->vdev_ops != &vdev_replacing_ops &&
7384 	    pvd->vdev_ops != &vdev_mirror_ops &&
7385 	    pvd->vdev_ops != &vdev_spare_ops)
7386 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7387 
7388 	/*
7389 	 * If this device has the only valid copy of some data,
7390 	 * we cannot safely detach it.
7391 	 */
7392 	if (vdev_dtl_required(vd))
7393 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
7394 
7395 	ASSERT(pvd->vdev_children >= 2);
7396 
7397 	/*
7398 	 * If we are detaching the second disk from a replacing vdev, then
7399 	 * check to see if we changed the original vdev's path to have "/old"
7400 	 * at the end in spa_vdev_attach().  If so, undo that change now.
7401 	 */
7402 	if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
7403 	    vd->vdev_path != NULL) {
7404 		size_t len = strlen(vd->vdev_path);
7405 
7406 		for (int c = 0; c < pvd->vdev_children; c++) {
7407 			cvd = pvd->vdev_child[c];
7408 
7409 			if (cvd == vd || cvd->vdev_path == NULL)
7410 				continue;
7411 
7412 			if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
7413 			    strcmp(cvd->vdev_path + len, "/old") == 0) {
7414 				spa_strfree(cvd->vdev_path);
7415 				cvd->vdev_path = spa_strdup(vd->vdev_path);
7416 				break;
7417 			}
7418 		}
7419 	}
7420 
7421 	/*
7422 	 * If we are detaching the original disk from a normal spare, then it
7423 	 * implies that the spare should become a real disk, and be removed
7424 	 * from the active spare list for the pool.  dRAID spares on the
7425 	 * other hand are coupled to the pool and thus should never be removed
7426 	 * from the spares list.
7427 	 */
7428 	if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) {
7429 		vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1];
7430 
7431 		if (last_cvd->vdev_isspare &&
7432 		    last_cvd->vdev_ops != &vdev_draid_spare_ops) {
7433 			unspare = B_TRUE;
7434 		}
7435 	}
7436 
7437 	/*
7438 	 * Erase the disk labels so the disk can be used for other things.
7439 	 * This must be done after all other error cases are handled,
7440 	 * but before we disembowel vd (so we can still do I/O to it).
7441 	 * But if we can't do it, don't treat the error as fatal --
7442 	 * it may be that the unwritability of the disk is the reason
7443 	 * it's being detached!
7444 	 */
7445 	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
7446 
7447 	/*
7448 	 * Remove vd from its parent and compact the parent's children.
7449 	 */
7450 	vdev_remove_child(pvd, vd);
7451 	vdev_compact_children(pvd);
7452 
7453 	/*
7454 	 * Remember one of the remaining children so we can get tvd below.
7455 	 */
7456 	cvd = pvd->vdev_child[pvd->vdev_children - 1];
7457 
7458 	/*
7459 	 * If we need to remove the remaining child from the list of hot spares,
7460 	 * do it now, marking the vdev as no longer a spare in the process.
7461 	 * We must do this before vdev_remove_parent(), because that can
7462 	 * change the GUID if it creates a new toplevel GUID.  For a similar
7463 	 * reason, we must remove the spare now, in the same txg as the detach;
7464 	 * otherwise someone could attach a new sibling, change the GUID, and
7465 	 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail.
7466 	 */
7467 	if (unspare) {
7468 		ASSERT(cvd->vdev_isspare);
7469 		spa_spare_remove(cvd);
7470 		unspare_guid = cvd->vdev_guid;
7471 		(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
7472 		cvd->vdev_unspare = B_TRUE;
7473 	}
7474 
7475 	/*
7476 	 * If the parent mirror/replacing vdev only has one child,
7477 	 * the parent is no longer needed.  Remove it from the tree.
7478 	 */
7479 	if (pvd->vdev_children == 1) {
7480 		if (pvd->vdev_ops == &vdev_spare_ops)
7481 			cvd->vdev_unspare = B_FALSE;
7482 		vdev_remove_parent(cvd);
7483 	}
7484 
7485 	/*
7486 	 * We don't set tvd until now because the parent we just removed
7487 	 * may have been the previous top-level vdev.
7488 	 */
7489 	tvd = cvd->vdev_top;
7490 	ASSERT(tvd->vdev_parent == rvd);
7491 
7492 	/*
7493 	 * Reevaluate the parent vdev state.
7494 	 */
7495 	vdev_propagate_state(cvd);
7496 
7497 	/*
7498 	 * If the 'autoexpand' property is set on the pool then automatically
7499 	 * try to expand the size of the pool. For example if the device we
7500 	 * just detached was smaller than the others, it may be possible to
7501 	 * add metaslabs (i.e. grow the pool). We need to reopen the vdev
7502 	 * first so that we can obtain the updated sizes of the leaf vdevs.
7503 	 */
7504 	if (spa->spa_autoexpand) {
7505 		vdev_reopen(tvd);
7506 		vdev_expand(tvd, txg);
7507 	}
7508 
7509 	vdev_config_dirty(tvd);
7510 
7511 	/*
7512 	 * Mark vd's DTL as dirty in this txg.  vdev_dtl_sync() will see that
7513 	 * vd->vdev_detached is set and free vd's DTL object in syncing context.
7514 	 * But first make sure we're not on any *other* txg's DTL list, to
7515 	 * prevent vd from being accessed after it's freed.
7516 	 */
7517 	vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none");
7518 	for (int t = 0; t < TXG_SIZE; t++)
7519 		(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
7520 	vd->vdev_detached = B_TRUE;
7521 	vdev_dirty(tvd, VDD_DTL, vd, txg);
7522 
7523 	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE);
7524 	spa_notify_waiters(spa);
7525 
7526 	/* hang on to the spa before we release the lock */
7527 	spa_open_ref(spa, FTAG);
7528 
7529 	error = spa_vdev_exit(spa, vd, txg, 0);
7530 
7531 	spa_history_log_internal(spa, "detach", NULL,
7532 	    "vdev=%s", vdpath);
7533 	spa_strfree(vdpath);
7534 
7535 	/*
7536 	 * If this was the removal of the original device in a hot spare vdev,
7537 	 * then we want to go through and remove the device from the hot spare
7538 	 * list of every other pool.
7539 	 */
7540 	if (unspare) {
7541 		spa_t *altspa = NULL;
7542 
7543 		mutex_enter(&spa_namespace_lock);
7544 		while ((altspa = spa_next(altspa)) != NULL) {
7545 			if (altspa->spa_state != POOL_STATE_ACTIVE ||
7546 			    altspa == spa)
7547 				continue;
7548 
7549 			spa_open_ref(altspa, FTAG);
7550 			mutex_exit(&spa_namespace_lock);
7551 			(void) spa_vdev_remove(altspa, unspare_guid, B_TRUE);
7552 			mutex_enter(&spa_namespace_lock);
7553 			spa_close(altspa, FTAG);
7554 		}
7555 		mutex_exit(&spa_namespace_lock);
7556 
7557 		/* search the rest of the vdevs for spares to remove */
7558 		spa_vdev_resilver_done(spa);
7559 	}
7560 
7561 	/* all done with the spa; OK to release */
7562 	mutex_enter(&spa_namespace_lock);
7563 	spa_close(spa, FTAG);
7564 	mutex_exit(&spa_namespace_lock);
7565 
7566 	return (error);
7567 }
7568 
7569 static int
7570 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
7571     list_t *vd_list)
7572 {
7573 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7574 
7575 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
7576 
7577 	/* Look up vdev and ensure it's a leaf. */
7578 	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7579 	if (vd == NULL || vd->vdev_detached) {
7580 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7581 		return (SET_ERROR(ENODEV));
7582 	} else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
7583 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7584 		return (SET_ERROR(EINVAL));
7585 	} else if (!vdev_writeable(vd)) {
7586 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7587 		return (SET_ERROR(EROFS));
7588 	}
7589 	mutex_enter(&vd->vdev_initialize_lock);
7590 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7591 
7592 	/*
7593 	 * When we activate an initialize action we check to see
7594 	 * if the vdev_initialize_thread is NULL. We do this instead
7595 	 * of using the vdev_initialize_state since there might be
7596 	 * a previous initialization process which has completed but
7597 	 * the thread is not exited.
7598 	 */
7599 	if (cmd_type == POOL_INITIALIZE_START &&
7600 	    (vd->vdev_initialize_thread != NULL ||
7601 	    vd->vdev_top->vdev_removing || vd->vdev_top->vdev_rz_expanding)) {
7602 		mutex_exit(&vd->vdev_initialize_lock);
7603 		return (SET_ERROR(EBUSY));
7604 	} else if (cmd_type == POOL_INITIALIZE_CANCEL &&
7605 	    (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE &&
7606 	    vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) {
7607 		mutex_exit(&vd->vdev_initialize_lock);
7608 		return (SET_ERROR(ESRCH));
7609 	} else if (cmd_type == POOL_INITIALIZE_SUSPEND &&
7610 	    vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) {
7611 		mutex_exit(&vd->vdev_initialize_lock);
7612 		return (SET_ERROR(ESRCH));
7613 	} else if (cmd_type == POOL_INITIALIZE_UNINIT &&
7614 	    vd->vdev_initialize_thread != NULL) {
7615 		mutex_exit(&vd->vdev_initialize_lock);
7616 		return (SET_ERROR(EBUSY));
7617 	}
7618 
7619 	switch (cmd_type) {
7620 	case POOL_INITIALIZE_START:
7621 		vdev_initialize(vd);
7622 		break;
7623 	case POOL_INITIALIZE_CANCEL:
7624 		vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list);
7625 		break;
7626 	case POOL_INITIALIZE_SUSPEND:
7627 		vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list);
7628 		break;
7629 	case POOL_INITIALIZE_UNINIT:
7630 		vdev_uninitialize(vd);
7631 		break;
7632 	default:
7633 		panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
7634 	}
7635 	mutex_exit(&vd->vdev_initialize_lock);
7636 
7637 	return (0);
7638 }
7639 
7640 int
7641 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type,
7642     nvlist_t *vdev_errlist)
7643 {
7644 	int total_errors = 0;
7645 	list_t vd_list;
7646 
7647 	list_create(&vd_list, sizeof (vdev_t),
7648 	    offsetof(vdev_t, vdev_initialize_node));
7649 
7650 	/*
7651 	 * We hold the namespace lock through the whole function
7652 	 * to prevent any changes to the pool while we're starting or
7653 	 * stopping initialization. The config and state locks are held so that
7654 	 * we can properly assess the vdev state before we commit to
7655 	 * the initializing operation.
7656 	 */
7657 	mutex_enter(&spa_namespace_lock);
7658 
7659 	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
7660 	    pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
7661 		uint64_t vdev_guid = fnvpair_value_uint64(pair);
7662 
7663 		int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type,
7664 		    &vd_list);
7665 		if (error != 0) {
7666 			char guid_as_str[MAXNAMELEN];
7667 
7668 			(void) snprintf(guid_as_str, sizeof (guid_as_str),
7669 			    "%llu", (unsigned long long)vdev_guid);
7670 			fnvlist_add_int64(vdev_errlist, guid_as_str, error);
7671 			total_errors++;
7672 		}
7673 	}
7674 
7675 	/* Wait for all initialize threads to stop. */
7676 	vdev_initialize_stop_wait(spa, &vd_list);
7677 
7678 	/* Sync out the initializing state */
7679 	txg_wait_synced(spa->spa_dsl_pool, 0);
7680 	mutex_exit(&spa_namespace_lock);
7681 
7682 	list_destroy(&vd_list);
7683 
7684 	return (total_errors);
7685 }
7686 
7687 static int
7688 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
7689     uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list)
7690 {
7691 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7692 
7693 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
7694 
7695 	/* Look up vdev and ensure it's a leaf. */
7696 	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7697 	if (vd == NULL || vd->vdev_detached) {
7698 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7699 		return (SET_ERROR(ENODEV));
7700 	} else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
7701 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7702 		return (SET_ERROR(EINVAL));
7703 	} else if (!vdev_writeable(vd)) {
7704 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7705 		return (SET_ERROR(EROFS));
7706 	} else if (!vd->vdev_has_trim) {
7707 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7708 		return (SET_ERROR(EOPNOTSUPP));
7709 	} else if (secure && !vd->vdev_has_securetrim) {
7710 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7711 		return (SET_ERROR(EOPNOTSUPP));
7712 	}
7713 	mutex_enter(&vd->vdev_trim_lock);
7714 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7715 
7716 	/*
7717 	 * When we activate a TRIM action we check to see if the
7718 	 * vdev_trim_thread is NULL. We do this instead of using the
7719 	 * vdev_trim_state since there might be a previous TRIM process
7720 	 * which has completed but the thread is not exited.
7721 	 */
7722 	if (cmd_type == POOL_TRIM_START &&
7723 	    (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing ||
7724 	    vd->vdev_top->vdev_rz_expanding)) {
7725 		mutex_exit(&vd->vdev_trim_lock);
7726 		return (SET_ERROR(EBUSY));
7727 	} else if (cmd_type == POOL_TRIM_CANCEL &&
7728 	    (vd->vdev_trim_state != VDEV_TRIM_ACTIVE &&
7729 	    vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) {
7730 		mutex_exit(&vd->vdev_trim_lock);
7731 		return (SET_ERROR(ESRCH));
7732 	} else if (cmd_type == POOL_TRIM_SUSPEND &&
7733 	    vd->vdev_trim_state != VDEV_TRIM_ACTIVE) {
7734 		mutex_exit(&vd->vdev_trim_lock);
7735 		return (SET_ERROR(ESRCH));
7736 	}
7737 
7738 	switch (cmd_type) {
7739 	case POOL_TRIM_START:
7740 		vdev_trim(vd, rate, partial, secure);
7741 		break;
7742 	case POOL_TRIM_CANCEL:
7743 		vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list);
7744 		break;
7745 	case POOL_TRIM_SUSPEND:
7746 		vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list);
7747 		break;
7748 	default:
7749 		panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
7750 	}
7751 	mutex_exit(&vd->vdev_trim_lock);
7752 
7753 	return (0);
7754 }
7755 
7756 /*
7757  * Initiates a manual TRIM for the requested vdevs. This kicks off individual
7758  * TRIM threads for each child vdev.  These threads pass over all of the free
7759  * space in the vdev's metaslabs and issues TRIM commands for that space.
7760  */
7761 int
7762 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate,
7763     boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist)
7764 {
7765 	int total_errors = 0;
7766 	list_t vd_list;
7767 
7768 	list_create(&vd_list, sizeof (vdev_t),
7769 	    offsetof(vdev_t, vdev_trim_node));
7770 
7771 	/*
7772 	 * We hold the namespace lock through the whole function
7773 	 * to prevent any changes to the pool while we're starting or
7774 	 * stopping TRIM. The config and state locks are held so that
7775 	 * we can properly assess the vdev state before we commit to
7776 	 * the TRIM operation.
7777 	 */
7778 	mutex_enter(&spa_namespace_lock);
7779 
7780 	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
7781 	    pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
7782 		uint64_t vdev_guid = fnvpair_value_uint64(pair);
7783 
7784 		int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type,
7785 		    rate, partial, secure, &vd_list);
7786 		if (error != 0) {
7787 			char guid_as_str[MAXNAMELEN];
7788 
7789 			(void) snprintf(guid_as_str, sizeof (guid_as_str),
7790 			    "%llu", (unsigned long long)vdev_guid);
7791 			fnvlist_add_int64(vdev_errlist, guid_as_str, error);
7792 			total_errors++;
7793 		}
7794 	}
7795 
7796 	/* Wait for all TRIM threads to stop. */
7797 	vdev_trim_stop_wait(spa, &vd_list);
7798 
7799 	/* Sync out the TRIM state */
7800 	txg_wait_synced(spa->spa_dsl_pool, 0);
7801 	mutex_exit(&spa_namespace_lock);
7802 
7803 	list_destroy(&vd_list);
7804 
7805 	return (total_errors);
7806 }
7807 
7808 /*
7809  * Split a set of devices from their mirrors, and create a new pool from them.
7810  */
7811 int
7812 spa_vdev_split_mirror(spa_t *spa, const char *newname, nvlist_t *config,
7813     nvlist_t *props, boolean_t exp)
7814 {
7815 	int error = 0;
7816 	uint64_t txg, *glist;
7817 	spa_t *newspa;
7818 	uint_t c, children, lastlog;
7819 	nvlist_t **child, *nvl, *tmp;
7820 	dmu_tx_t *tx;
7821 	const char *altroot = NULL;
7822 	vdev_t *rvd, **vml = NULL;			/* vdev modify list */
7823 	boolean_t activate_slog;
7824 
7825 	ASSERT(spa_writeable(spa));
7826 
7827 	txg = spa_vdev_enter(spa);
7828 
7829 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7830 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
7831 		error = (spa_has_checkpoint(spa)) ?
7832 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
7833 		return (spa_vdev_exit(spa, NULL, txg, error));
7834 	}
7835 
7836 	/* clear the log and flush everything up to now */
7837 	activate_slog = spa_passivate_log(spa);
7838 	(void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
7839 	error = spa_reset_logs(spa);
7840 	txg = spa_vdev_config_enter(spa);
7841 
7842 	if (activate_slog)
7843 		spa_activate_log(spa);
7844 
7845 	if (error != 0)
7846 		return (spa_vdev_exit(spa, NULL, txg, error));
7847 
7848 	/* check new spa name before going any further */
7849 	if (spa_lookup(newname) != NULL)
7850 		return (spa_vdev_exit(spa, NULL, txg, EEXIST));
7851 
7852 	/*
7853 	 * scan through all the children to ensure they're all mirrors
7854 	 */
7855 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 ||
7856 	    nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
7857 	    &children) != 0)
7858 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7859 
7860 	/* first, check to ensure we've got the right child count */
7861 	rvd = spa->spa_root_vdev;
7862 	lastlog = 0;
7863 	for (c = 0; c < rvd->vdev_children; c++) {
7864 		vdev_t *vd = rvd->vdev_child[c];
7865 
7866 		/* don't count the holes & logs as children */
7867 		if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops &&
7868 		    !vdev_is_concrete(vd))) {
7869 			if (lastlog == 0)
7870 				lastlog = c;
7871 			continue;
7872 		}
7873 
7874 		lastlog = 0;
7875 	}
7876 	if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
7877 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7878 
7879 	/* next, ensure no spare or cache devices are part of the split */
7880 	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 ||
7881 	    nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0)
7882 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7883 
7884 	vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP);
7885 	glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP);
7886 
7887 	/* then, loop over each vdev and validate it */
7888 	for (c = 0; c < children; c++) {
7889 		uint64_t is_hole = 0;
7890 
7891 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
7892 		    &is_hole);
7893 
7894 		if (is_hole != 0) {
7895 			if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole ||
7896 			    spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
7897 				continue;
7898 			} else {
7899 				error = SET_ERROR(EINVAL);
7900 				break;
7901 			}
7902 		}
7903 
7904 		/* deal with indirect vdevs */
7905 		if (spa->spa_root_vdev->vdev_child[c]->vdev_ops ==
7906 		    &vdev_indirect_ops)
7907 			continue;
7908 
7909 		/* which disk is going to be split? */
7910 		if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
7911 		    &glist[c]) != 0) {
7912 			error = SET_ERROR(EINVAL);
7913 			break;
7914 		}
7915 
7916 		/* look it up in the spa */
7917 		vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE);
7918 		if (vml[c] == NULL) {
7919 			error = SET_ERROR(ENODEV);
7920 			break;
7921 		}
7922 
7923 		/* make sure there's nothing stopping the split */
7924 		if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops ||
7925 		    vml[c]->vdev_islog ||
7926 		    !vdev_is_concrete(vml[c]) ||
7927 		    vml[c]->vdev_isspare ||
7928 		    vml[c]->vdev_isl2cache ||
7929 		    !vdev_writeable(vml[c]) ||
7930 		    vml[c]->vdev_children != 0 ||
7931 		    vml[c]->vdev_state != VDEV_STATE_HEALTHY ||
7932 		    c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
7933 			error = SET_ERROR(EINVAL);
7934 			break;
7935 		}
7936 
7937 		if (vdev_dtl_required(vml[c]) ||
7938 		    vdev_resilver_needed(vml[c], NULL, NULL)) {
7939 			error = SET_ERROR(EBUSY);
7940 			break;
7941 		}
7942 
7943 		/* we need certain info from the top level */
7944 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY,
7945 		    vml[c]->vdev_top->vdev_ms_array);
7946 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT,
7947 		    vml[c]->vdev_top->vdev_ms_shift);
7948 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE,
7949 		    vml[c]->vdev_top->vdev_asize);
7950 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT,
7951 		    vml[c]->vdev_top->vdev_ashift);
7952 
7953 		/* transfer per-vdev ZAPs */
7954 		ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0);
7955 		VERIFY0(nvlist_add_uint64(child[c],
7956 		    ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap));
7957 
7958 		ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0);
7959 		VERIFY0(nvlist_add_uint64(child[c],
7960 		    ZPOOL_CONFIG_VDEV_TOP_ZAP,
7961 		    vml[c]->vdev_parent->vdev_top_zap));
7962 	}
7963 
7964 	if (error != 0) {
7965 		kmem_free(vml, children * sizeof (vdev_t *));
7966 		kmem_free(glist, children * sizeof (uint64_t));
7967 		return (spa_vdev_exit(spa, NULL, txg, error));
7968 	}
7969 
7970 	/* stop writers from using the disks */
7971 	for (c = 0; c < children; c++) {
7972 		if (vml[c] != NULL)
7973 			vml[c]->vdev_offline = B_TRUE;
7974 	}
7975 	vdev_reopen(spa->spa_root_vdev);
7976 
7977 	/*
7978 	 * Temporarily record the splitting vdevs in the spa config.  This
7979 	 * will disappear once the config is regenerated.
7980 	 */
7981 	nvl = fnvlist_alloc();
7982 	fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children);
7983 	kmem_free(glist, children * sizeof (uint64_t));
7984 
7985 	mutex_enter(&spa->spa_props_lock);
7986 	fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl);
7987 	mutex_exit(&spa->spa_props_lock);
7988 	spa->spa_config_splitting = nvl;
7989 	vdev_config_dirty(spa->spa_root_vdev);
7990 
7991 	/* configure and create the new pool */
7992 	fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname);
7993 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
7994 	    exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE);
7995 	fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa));
7996 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg);
7997 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID,
7998 	    spa_generate_guid(NULL));
7999 	VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
8000 	(void) nvlist_lookup_string(props,
8001 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
8002 
8003 	/* add the new pool to the namespace */
8004 	newspa = spa_add(newname, config, altroot);
8005 	newspa->spa_avz_action = AVZ_ACTION_REBUILD;
8006 	newspa->spa_config_txg = spa->spa_config_txg;
8007 	spa_set_log_state(newspa, SPA_LOG_CLEAR);
8008 
8009 	/* release the spa config lock, retaining the namespace lock */
8010 	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
8011 
8012 	if (zio_injection_enabled)
8013 		zio_handle_panic_injection(spa, FTAG, 1);
8014 
8015 	spa_activate(newspa, spa_mode_global);
8016 	spa_async_suspend(newspa);
8017 
8018 	/*
8019 	 * Temporarily stop the initializing and TRIM activity.  We set the
8020 	 * state to ACTIVE so that we know to resume initializing or TRIM
8021 	 * once the split has completed.
8022 	 */
8023 	list_t vd_initialize_list;
8024 	list_create(&vd_initialize_list, sizeof (vdev_t),
8025 	    offsetof(vdev_t, vdev_initialize_node));
8026 
8027 	list_t vd_trim_list;
8028 	list_create(&vd_trim_list, sizeof (vdev_t),
8029 	    offsetof(vdev_t, vdev_trim_node));
8030 
8031 	for (c = 0; c < children; c++) {
8032 		if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
8033 			mutex_enter(&vml[c]->vdev_initialize_lock);
8034 			vdev_initialize_stop(vml[c],
8035 			    VDEV_INITIALIZE_ACTIVE, &vd_initialize_list);
8036 			mutex_exit(&vml[c]->vdev_initialize_lock);
8037 
8038 			mutex_enter(&vml[c]->vdev_trim_lock);
8039 			vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list);
8040 			mutex_exit(&vml[c]->vdev_trim_lock);
8041 		}
8042 	}
8043 
8044 	vdev_initialize_stop_wait(spa, &vd_initialize_list);
8045 	vdev_trim_stop_wait(spa, &vd_trim_list);
8046 
8047 	list_destroy(&vd_initialize_list);
8048 	list_destroy(&vd_trim_list);
8049 
8050 	newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT;
8051 	newspa->spa_is_splitting = B_TRUE;
8052 
8053 	/* create the new pool from the disks of the original pool */
8054 	error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE);
8055 	if (error)
8056 		goto out;
8057 
8058 	/* if that worked, generate a real config for the new pool */
8059 	if (newspa->spa_root_vdev != NULL) {
8060 		newspa->spa_config_splitting = fnvlist_alloc();
8061 		fnvlist_add_uint64(newspa->spa_config_splitting,
8062 		    ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa));
8063 		spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
8064 		    B_TRUE));
8065 	}
8066 
8067 	/* set the props */
8068 	if (props != NULL) {
8069 		spa_configfile_set(newspa, props, B_FALSE);
8070 		error = spa_prop_set(newspa, props);
8071 		if (error)
8072 			goto out;
8073 	}
8074 
8075 	/* flush everything */
8076 	txg = spa_vdev_config_enter(newspa);
8077 	vdev_config_dirty(newspa->spa_root_vdev);
8078 	(void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG);
8079 
8080 	if (zio_injection_enabled)
8081 		zio_handle_panic_injection(spa, FTAG, 2);
8082 
8083 	spa_async_resume(newspa);
8084 
8085 	/* finally, update the original pool's config */
8086 	txg = spa_vdev_config_enter(spa);
8087 	tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
8088 	error = dmu_tx_assign(tx, TXG_WAIT);
8089 	if (error != 0)
8090 		dmu_tx_abort(tx);
8091 	for (c = 0; c < children; c++) {
8092 		if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
8093 			vdev_t *tvd = vml[c]->vdev_top;
8094 
8095 			/*
8096 			 * Need to be sure the detachable VDEV is not
8097 			 * on any *other* txg's DTL list to prevent it
8098 			 * from being accessed after it's freed.
8099 			 */
8100 			for (int t = 0; t < TXG_SIZE; t++) {
8101 				(void) txg_list_remove_this(
8102 				    &tvd->vdev_dtl_list, vml[c], t);
8103 			}
8104 
8105 			vdev_split(vml[c]);
8106 			if (error == 0)
8107 				spa_history_log_internal(spa, "detach", tx,
8108 				    "vdev=%s", vml[c]->vdev_path);
8109 
8110 			vdev_free(vml[c]);
8111 		}
8112 	}
8113 	spa->spa_avz_action = AVZ_ACTION_REBUILD;
8114 	vdev_config_dirty(spa->spa_root_vdev);
8115 	spa->spa_config_splitting = NULL;
8116 	nvlist_free(nvl);
8117 	if (error == 0)
8118 		dmu_tx_commit(tx);
8119 	(void) spa_vdev_exit(spa, NULL, txg, 0);
8120 
8121 	if (zio_injection_enabled)
8122 		zio_handle_panic_injection(spa, FTAG, 3);
8123 
8124 	/* split is complete; log a history record */
8125 	spa_history_log_internal(newspa, "split", NULL,
8126 	    "from pool %s", spa_name(spa));
8127 
8128 	newspa->spa_is_splitting = B_FALSE;
8129 	kmem_free(vml, children * sizeof (vdev_t *));
8130 
8131 	/* if we're not going to mount the filesystems in userland, export */
8132 	if (exp)
8133 		error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL,
8134 		    B_FALSE, B_FALSE);
8135 
8136 	return (error);
8137 
8138 out:
8139 	spa_unload(newspa);
8140 	spa_deactivate(newspa);
8141 	spa_remove(newspa);
8142 
8143 	txg = spa_vdev_config_enter(spa);
8144 
8145 	/* re-online all offlined disks */
8146 	for (c = 0; c < children; c++) {
8147 		if (vml[c] != NULL)
8148 			vml[c]->vdev_offline = B_FALSE;
8149 	}
8150 
8151 	/* restart initializing or trimming disks as necessary */
8152 	spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART);
8153 	spa_async_request(spa, SPA_ASYNC_TRIM_RESTART);
8154 	spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART);
8155 
8156 	vdev_reopen(spa->spa_root_vdev);
8157 
8158 	nvlist_free(spa->spa_config_splitting);
8159 	spa->spa_config_splitting = NULL;
8160 	(void) spa_vdev_exit(spa, NULL, txg, error);
8161 
8162 	kmem_free(vml, children * sizeof (vdev_t *));
8163 	return (error);
8164 }
8165 
8166 /*
8167  * Find any device that's done replacing, or a vdev marked 'unspare' that's
8168  * currently spared, so we can detach it.
8169  */
8170 static vdev_t *
8171 spa_vdev_resilver_done_hunt(vdev_t *vd)
8172 {
8173 	vdev_t *newvd, *oldvd;
8174 
8175 	for (int c = 0; c < vd->vdev_children; c++) {
8176 		oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
8177 		if (oldvd != NULL)
8178 			return (oldvd);
8179 	}
8180 
8181 	/*
8182 	 * Check for a completed replacement.  We always consider the first
8183 	 * vdev in the list to be the oldest vdev, and the last one to be
8184 	 * the newest (see spa_vdev_attach() for how that works).  In
8185 	 * the case where the newest vdev is faulted, we will not automatically
8186 	 * remove it after a resilver completes.  This is OK as it will require
8187 	 * user intervention to determine which disk the admin wishes to keep.
8188 	 */
8189 	if (vd->vdev_ops == &vdev_replacing_ops) {
8190 		ASSERT(vd->vdev_children > 1);
8191 
8192 		newvd = vd->vdev_child[vd->vdev_children - 1];
8193 		oldvd = vd->vdev_child[0];
8194 
8195 		if (vdev_dtl_empty(newvd, DTL_MISSING) &&
8196 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
8197 		    !vdev_dtl_required(oldvd))
8198 			return (oldvd);
8199 	}
8200 
8201 	/*
8202 	 * Check for a completed resilver with the 'unspare' flag set.
8203 	 * Also potentially update faulted state.
8204 	 */
8205 	if (vd->vdev_ops == &vdev_spare_ops) {
8206 		vdev_t *first = vd->vdev_child[0];
8207 		vdev_t *last = vd->vdev_child[vd->vdev_children - 1];
8208 
8209 		if (last->vdev_unspare) {
8210 			oldvd = first;
8211 			newvd = last;
8212 		} else if (first->vdev_unspare) {
8213 			oldvd = last;
8214 			newvd = first;
8215 		} else {
8216 			oldvd = NULL;
8217 		}
8218 
8219 		if (oldvd != NULL &&
8220 		    vdev_dtl_empty(newvd, DTL_MISSING) &&
8221 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
8222 		    !vdev_dtl_required(oldvd))
8223 			return (oldvd);
8224 
8225 		vdev_propagate_state(vd);
8226 
8227 		/*
8228 		 * If there are more than two spares attached to a disk,
8229 		 * and those spares are not required, then we want to
8230 		 * attempt to free them up now so that they can be used
8231 		 * by other pools.  Once we're back down to a single
8232 		 * disk+spare, we stop removing them.
8233 		 */
8234 		if (vd->vdev_children > 2) {
8235 			newvd = vd->vdev_child[1];
8236 
8237 			if (newvd->vdev_isspare && last->vdev_isspare &&
8238 			    vdev_dtl_empty(last, DTL_MISSING) &&
8239 			    vdev_dtl_empty(last, DTL_OUTAGE) &&
8240 			    !vdev_dtl_required(newvd))
8241 				return (newvd);
8242 		}
8243 	}
8244 
8245 	return (NULL);
8246 }
8247 
8248 static void
8249 spa_vdev_resilver_done(spa_t *spa)
8250 {
8251 	vdev_t *vd, *pvd, *ppvd;
8252 	uint64_t guid, sguid, pguid, ppguid;
8253 
8254 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
8255 
8256 	while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
8257 		pvd = vd->vdev_parent;
8258 		ppvd = pvd->vdev_parent;
8259 		guid = vd->vdev_guid;
8260 		pguid = pvd->vdev_guid;
8261 		ppguid = ppvd->vdev_guid;
8262 		sguid = 0;
8263 		/*
8264 		 * If we have just finished replacing a hot spared device, then
8265 		 * we need to detach the parent's first child (the original hot
8266 		 * spare) as well.
8267 		 */
8268 		if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
8269 		    ppvd->vdev_children == 2) {
8270 			ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
8271 			sguid = ppvd->vdev_child[1]->vdev_guid;
8272 		}
8273 		ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd));
8274 
8275 		spa_config_exit(spa, SCL_ALL, FTAG);
8276 		if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0)
8277 			return;
8278 		if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0)
8279 			return;
8280 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
8281 	}
8282 
8283 	spa_config_exit(spa, SCL_ALL, FTAG);
8284 
8285 	/*
8286 	 * If a detach was not performed above replace waiters will not have
8287 	 * been notified.  In which case we must do so now.
8288 	 */
8289 	spa_notify_waiters(spa);
8290 }
8291 
8292 /*
8293  * Update the stored path or FRU for this vdev.
8294  */
8295 static int
8296 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value,
8297     boolean_t ispath)
8298 {
8299 	vdev_t *vd;
8300 	boolean_t sync = B_FALSE;
8301 
8302 	ASSERT(spa_writeable(spa));
8303 
8304 	spa_vdev_state_enter(spa, SCL_ALL);
8305 
8306 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
8307 		return (spa_vdev_state_exit(spa, NULL, ENOENT));
8308 
8309 	if (!vd->vdev_ops->vdev_op_leaf)
8310 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
8311 
8312 	if (ispath) {
8313 		if (strcmp(value, vd->vdev_path) != 0) {
8314 			spa_strfree(vd->vdev_path);
8315 			vd->vdev_path = spa_strdup(value);
8316 			sync = B_TRUE;
8317 		}
8318 	} else {
8319 		if (vd->vdev_fru == NULL) {
8320 			vd->vdev_fru = spa_strdup(value);
8321 			sync = B_TRUE;
8322 		} else if (strcmp(value, vd->vdev_fru) != 0) {
8323 			spa_strfree(vd->vdev_fru);
8324 			vd->vdev_fru = spa_strdup(value);
8325 			sync = B_TRUE;
8326 		}
8327 	}
8328 
8329 	return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0));
8330 }
8331 
8332 int
8333 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath)
8334 {
8335 	return (spa_vdev_set_common(spa, guid, newpath, B_TRUE));
8336 }
8337 
8338 int
8339 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru)
8340 {
8341 	return (spa_vdev_set_common(spa, guid, newfru, B_FALSE));
8342 }
8343 
8344 /*
8345  * ==========================================================================
8346  * SPA Scanning
8347  * ==========================================================================
8348  */
8349 int
8350 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd)
8351 {
8352 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8353 
8354 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
8355 		return (SET_ERROR(EBUSY));
8356 
8357 	return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd));
8358 }
8359 
8360 int
8361 spa_scan_stop(spa_t *spa)
8362 {
8363 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8364 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
8365 		return (SET_ERROR(EBUSY));
8366 
8367 	return (dsl_scan_cancel(spa->spa_dsl_pool));
8368 }
8369 
8370 int
8371 spa_scan(spa_t *spa, pool_scan_func_t func)
8372 {
8373 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8374 
8375 	if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE)
8376 		return (SET_ERROR(ENOTSUP));
8377 
8378 	if (func == POOL_SCAN_RESILVER &&
8379 	    !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER))
8380 		return (SET_ERROR(ENOTSUP));
8381 
8382 	/*
8383 	 * If a resilver was requested, but there is no DTL on a
8384 	 * writeable leaf device, we have nothing to do.
8385 	 */
8386 	if (func == POOL_SCAN_RESILVER &&
8387 	    !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
8388 		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
8389 		return (0);
8390 	}
8391 
8392 	if (func == POOL_SCAN_ERRORSCRUB &&
8393 	    !spa_feature_is_enabled(spa, SPA_FEATURE_HEAD_ERRLOG))
8394 		return (SET_ERROR(ENOTSUP));
8395 
8396 	return (dsl_scan(spa->spa_dsl_pool, func));
8397 }
8398 
8399 /*
8400  * ==========================================================================
8401  * SPA async task processing
8402  * ==========================================================================
8403  */
8404 
8405 static void
8406 spa_async_remove(spa_t *spa, vdev_t *vd)
8407 {
8408 	if (vd->vdev_remove_wanted) {
8409 		vd->vdev_remove_wanted = B_FALSE;
8410 		vd->vdev_delayed_close = B_FALSE;
8411 		vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE);
8412 
8413 		/*
8414 		 * We want to clear the stats, but we don't want to do a full
8415 		 * vdev_clear() as that will cause us to throw away
8416 		 * degraded/faulted state as well as attempt to reopen the
8417 		 * device, all of which is a waste.
8418 		 */
8419 		vd->vdev_stat.vs_read_errors = 0;
8420 		vd->vdev_stat.vs_write_errors = 0;
8421 		vd->vdev_stat.vs_checksum_errors = 0;
8422 
8423 		vdev_state_dirty(vd->vdev_top);
8424 
8425 		/* Tell userspace that the vdev is gone. */
8426 		zfs_post_remove(spa, vd);
8427 	}
8428 
8429 	for (int c = 0; c < vd->vdev_children; c++)
8430 		spa_async_remove(spa, vd->vdev_child[c]);
8431 }
8432 
8433 static void
8434 spa_async_probe(spa_t *spa, vdev_t *vd)
8435 {
8436 	if (vd->vdev_probe_wanted) {
8437 		vd->vdev_probe_wanted = B_FALSE;
8438 		vdev_reopen(vd);	/* vdev_open() does the actual probe */
8439 	}
8440 
8441 	for (int c = 0; c < vd->vdev_children; c++)
8442 		spa_async_probe(spa, vd->vdev_child[c]);
8443 }
8444 
8445 static void
8446 spa_async_autoexpand(spa_t *spa, vdev_t *vd)
8447 {
8448 	if (!spa->spa_autoexpand)
8449 		return;
8450 
8451 	for (int c = 0; c < vd->vdev_children; c++) {
8452 		vdev_t *cvd = vd->vdev_child[c];
8453 		spa_async_autoexpand(spa, cvd);
8454 	}
8455 
8456 	if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL)
8457 		return;
8458 
8459 	spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND);
8460 }
8461 
8462 static __attribute__((noreturn)) void
8463 spa_async_thread(void *arg)
8464 {
8465 	spa_t *spa = (spa_t *)arg;
8466 	dsl_pool_t *dp = spa->spa_dsl_pool;
8467 	int tasks;
8468 
8469 	ASSERT(spa->spa_sync_on);
8470 
8471 	mutex_enter(&spa->spa_async_lock);
8472 	tasks = spa->spa_async_tasks;
8473 	spa->spa_async_tasks = 0;
8474 	mutex_exit(&spa->spa_async_lock);
8475 
8476 	/*
8477 	 * See if the config needs to be updated.
8478 	 */
8479 	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
8480 		uint64_t old_space, new_space;
8481 
8482 		mutex_enter(&spa_namespace_lock);
8483 		old_space = metaslab_class_get_space(spa_normal_class(spa));
8484 		old_space += metaslab_class_get_space(spa_special_class(spa));
8485 		old_space += metaslab_class_get_space(spa_dedup_class(spa));
8486 		old_space += metaslab_class_get_space(
8487 		    spa_embedded_log_class(spa));
8488 
8489 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
8490 
8491 		new_space = metaslab_class_get_space(spa_normal_class(spa));
8492 		new_space += metaslab_class_get_space(spa_special_class(spa));
8493 		new_space += metaslab_class_get_space(spa_dedup_class(spa));
8494 		new_space += metaslab_class_get_space(
8495 		    spa_embedded_log_class(spa));
8496 		mutex_exit(&spa_namespace_lock);
8497 
8498 		/*
8499 		 * If the pool grew as a result of the config update,
8500 		 * then log an internal history event.
8501 		 */
8502 		if (new_space != old_space) {
8503 			spa_history_log_internal(spa, "vdev online", NULL,
8504 			    "pool '%s' size: %llu(+%llu)",
8505 			    spa_name(spa), (u_longlong_t)new_space,
8506 			    (u_longlong_t)(new_space - old_space));
8507 		}
8508 	}
8509 
8510 	/*
8511 	 * See if any devices need to be marked REMOVED.
8512 	 */
8513 	if (tasks & SPA_ASYNC_REMOVE) {
8514 		spa_vdev_state_enter(spa, SCL_NONE);
8515 		spa_async_remove(spa, spa->spa_root_vdev);
8516 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
8517 			spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]);
8518 		for (int i = 0; i < spa->spa_spares.sav_count; i++)
8519 			spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]);
8520 		(void) spa_vdev_state_exit(spa, NULL, 0);
8521 	}
8522 
8523 	if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) {
8524 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8525 		spa_async_autoexpand(spa, spa->spa_root_vdev);
8526 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8527 	}
8528 
8529 	/*
8530 	 * See if any devices need to be probed.
8531 	 */
8532 	if (tasks & SPA_ASYNC_PROBE) {
8533 		spa_vdev_state_enter(spa, SCL_NONE);
8534 		spa_async_probe(spa, spa->spa_root_vdev);
8535 		(void) spa_vdev_state_exit(spa, NULL, 0);
8536 	}
8537 
8538 	/*
8539 	 * If any devices are done replacing, detach them.
8540 	 */
8541 	if (tasks & SPA_ASYNC_RESILVER_DONE ||
8542 	    tasks & SPA_ASYNC_REBUILD_DONE ||
8543 	    tasks & SPA_ASYNC_DETACH_SPARE) {
8544 		spa_vdev_resilver_done(spa);
8545 	}
8546 
8547 	/*
8548 	 * Kick off a resilver.
8549 	 */
8550 	if (tasks & SPA_ASYNC_RESILVER &&
8551 	    !vdev_rebuild_active(spa->spa_root_vdev) &&
8552 	    (!dsl_scan_resilvering(dp) ||
8553 	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)))
8554 		dsl_scan_restart_resilver(dp, 0);
8555 
8556 	if (tasks & SPA_ASYNC_INITIALIZE_RESTART) {
8557 		mutex_enter(&spa_namespace_lock);
8558 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8559 		vdev_initialize_restart(spa->spa_root_vdev);
8560 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8561 		mutex_exit(&spa_namespace_lock);
8562 	}
8563 
8564 	if (tasks & SPA_ASYNC_TRIM_RESTART) {
8565 		mutex_enter(&spa_namespace_lock);
8566 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8567 		vdev_trim_restart(spa->spa_root_vdev);
8568 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8569 		mutex_exit(&spa_namespace_lock);
8570 	}
8571 
8572 	if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) {
8573 		mutex_enter(&spa_namespace_lock);
8574 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8575 		vdev_autotrim_restart(spa);
8576 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8577 		mutex_exit(&spa_namespace_lock);
8578 	}
8579 
8580 	/*
8581 	 * Kick off L2 cache whole device TRIM.
8582 	 */
8583 	if (tasks & SPA_ASYNC_L2CACHE_TRIM) {
8584 		mutex_enter(&spa_namespace_lock);
8585 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8586 		vdev_trim_l2arc(spa);
8587 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8588 		mutex_exit(&spa_namespace_lock);
8589 	}
8590 
8591 	/*
8592 	 * Kick off L2 cache rebuilding.
8593 	 */
8594 	if (tasks & SPA_ASYNC_L2CACHE_REBUILD) {
8595 		mutex_enter(&spa_namespace_lock);
8596 		spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER);
8597 		l2arc_spa_rebuild_start(spa);
8598 		spa_config_exit(spa, SCL_L2ARC, FTAG);
8599 		mutex_exit(&spa_namespace_lock);
8600 	}
8601 
8602 	/*
8603 	 * Let the world know that we're done.
8604 	 */
8605 	mutex_enter(&spa->spa_async_lock);
8606 	spa->spa_async_thread = NULL;
8607 	cv_broadcast(&spa->spa_async_cv);
8608 	mutex_exit(&spa->spa_async_lock);
8609 	thread_exit();
8610 }
8611 
8612 void
8613 spa_async_suspend(spa_t *spa)
8614 {
8615 	mutex_enter(&spa->spa_async_lock);
8616 	spa->spa_async_suspended++;
8617 	while (spa->spa_async_thread != NULL)
8618 		cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
8619 	mutex_exit(&spa->spa_async_lock);
8620 
8621 	spa_vdev_remove_suspend(spa);
8622 
8623 	zthr_t *condense_thread = spa->spa_condense_zthr;
8624 	if (condense_thread != NULL)
8625 		zthr_cancel(condense_thread);
8626 
8627 	zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr;
8628 	if (raidz_expand_thread != NULL)
8629 		zthr_cancel(raidz_expand_thread);
8630 
8631 	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
8632 	if (discard_thread != NULL)
8633 		zthr_cancel(discard_thread);
8634 
8635 	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
8636 	if (ll_delete_thread != NULL)
8637 		zthr_cancel(ll_delete_thread);
8638 
8639 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
8640 	if (ll_condense_thread != NULL)
8641 		zthr_cancel(ll_condense_thread);
8642 }
8643 
8644 void
8645 spa_async_resume(spa_t *spa)
8646 {
8647 	mutex_enter(&spa->spa_async_lock);
8648 	ASSERT(spa->spa_async_suspended != 0);
8649 	spa->spa_async_suspended--;
8650 	mutex_exit(&spa->spa_async_lock);
8651 	spa_restart_removal(spa);
8652 
8653 	zthr_t *condense_thread = spa->spa_condense_zthr;
8654 	if (condense_thread != NULL)
8655 		zthr_resume(condense_thread);
8656 
8657 	zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr;
8658 	if (raidz_expand_thread != NULL)
8659 		zthr_resume(raidz_expand_thread);
8660 
8661 	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
8662 	if (discard_thread != NULL)
8663 		zthr_resume(discard_thread);
8664 
8665 	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
8666 	if (ll_delete_thread != NULL)
8667 		zthr_resume(ll_delete_thread);
8668 
8669 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
8670 	if (ll_condense_thread != NULL)
8671 		zthr_resume(ll_condense_thread);
8672 }
8673 
8674 static boolean_t
8675 spa_async_tasks_pending(spa_t *spa)
8676 {
8677 	uint_t non_config_tasks;
8678 	uint_t config_task;
8679 	boolean_t config_task_suspended;
8680 
8681 	non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
8682 	config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
8683 	if (spa->spa_ccw_fail_time == 0) {
8684 		config_task_suspended = B_FALSE;
8685 	} else {
8686 		config_task_suspended =
8687 		    (gethrtime() - spa->spa_ccw_fail_time) <
8688 		    ((hrtime_t)zfs_ccw_retry_interval * NANOSEC);
8689 	}
8690 
8691 	return (non_config_tasks || (config_task && !config_task_suspended));
8692 }
8693 
8694 static void
8695 spa_async_dispatch(spa_t *spa)
8696 {
8697 	mutex_enter(&spa->spa_async_lock);
8698 	if (spa_async_tasks_pending(spa) &&
8699 	    !spa->spa_async_suspended &&
8700 	    spa->spa_async_thread == NULL)
8701 		spa->spa_async_thread = thread_create(NULL, 0,
8702 		    spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
8703 	mutex_exit(&spa->spa_async_lock);
8704 }
8705 
8706 void
8707 spa_async_request(spa_t *spa, int task)
8708 {
8709 	zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
8710 	mutex_enter(&spa->spa_async_lock);
8711 	spa->spa_async_tasks |= task;
8712 	mutex_exit(&spa->spa_async_lock);
8713 }
8714 
8715 int
8716 spa_async_tasks(spa_t *spa)
8717 {
8718 	return (spa->spa_async_tasks);
8719 }
8720 
8721 /*
8722  * ==========================================================================
8723  * SPA syncing routines
8724  * ==========================================================================
8725  */
8726 
8727 
8728 static int
8729 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
8730     dmu_tx_t *tx)
8731 {
8732 	bpobj_t *bpo = arg;
8733 	bpobj_enqueue(bpo, bp, bp_freed, tx);
8734 	return (0);
8735 }
8736 
8737 int
8738 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
8739 {
8740 	return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx));
8741 }
8742 
8743 int
8744 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
8745 {
8746 	return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx));
8747 }
8748 
8749 static int
8750 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
8751 {
8752 	zio_t *pio = arg;
8753 
8754 	zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp,
8755 	    pio->io_flags));
8756 	return (0);
8757 }
8758 
8759 static int
8760 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
8761     dmu_tx_t *tx)
8762 {
8763 	ASSERT(!bp_freed);
8764 	return (spa_free_sync_cb(arg, bp, tx));
8765 }
8766 
8767 /*
8768  * Note: this simple function is not inlined to make it easier to dtrace the
8769  * amount of time spent syncing frees.
8770  */
8771 static void
8772 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx)
8773 {
8774 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
8775 	bplist_iterate(bpl, spa_free_sync_cb, zio, tx);
8776 	VERIFY(zio_wait(zio) == 0);
8777 }
8778 
8779 /*
8780  * Note: this simple function is not inlined to make it easier to dtrace the
8781  * amount of time spent syncing deferred frees.
8782  */
8783 static void
8784 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx)
8785 {
8786 	if (spa_sync_pass(spa) != 1)
8787 		return;
8788 
8789 	/*
8790 	 * Note:
8791 	 * If the log space map feature is active, we stop deferring
8792 	 * frees to the next TXG and therefore running this function
8793 	 * would be considered a no-op as spa_deferred_bpobj should
8794 	 * not have any entries.
8795 	 *
8796 	 * That said we run this function anyway (instead of returning
8797 	 * immediately) for the edge-case scenario where we just
8798 	 * activated the log space map feature in this TXG but we have
8799 	 * deferred frees from the previous TXG.
8800 	 */
8801 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
8802 	VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj,
8803 	    bpobj_spa_free_sync_cb, zio, tx), ==, 0);
8804 	VERIFY0(zio_wait(zio));
8805 }
8806 
8807 static void
8808 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
8809 {
8810 	char *packed = NULL;
8811 	size_t bufsize;
8812 	size_t nvsize = 0;
8813 	dmu_buf_t *db;
8814 
8815 	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);
8816 
8817 	/*
8818 	 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration
8819 	 * information.  This avoids the dmu_buf_will_dirty() path and
8820 	 * saves us a pre-read to get data we don't actually care about.
8821 	 */
8822 	bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE);
8823 	packed = vmem_alloc(bufsize, KM_SLEEP);
8824 
8825 	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
8826 	    KM_SLEEP) == 0);
8827 	memset(packed + nvsize, 0, bufsize - nvsize);
8828 
8829 	dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx);
8830 
8831 	vmem_free(packed, bufsize);
8832 
8833 	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
8834 	dmu_buf_will_dirty(db, tx);
8835 	*(uint64_t *)db->db_data = nvsize;
8836 	dmu_buf_rele(db, FTAG);
8837 }
8838 
8839 static void
8840 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx,
8841     const char *config, const char *entry)
8842 {
8843 	nvlist_t *nvroot;
8844 	nvlist_t **list;
8845 	int i;
8846 
8847 	if (!sav->sav_sync)
8848 		return;
8849 
8850 	/*
8851 	 * Update the MOS nvlist describing the list of available devices.
8852 	 * spa_validate_aux() will have already made sure this nvlist is
8853 	 * valid and the vdevs are labeled appropriately.
8854 	 */
8855 	if (sav->sav_object == 0) {
8856 		sav->sav_object = dmu_object_alloc(spa->spa_meta_objset,
8857 		    DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE,
8858 		    sizeof (uint64_t), tx);
8859 		VERIFY(zap_update(spa->spa_meta_objset,
8860 		    DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1,
8861 		    &sav->sav_object, tx) == 0);
8862 	}
8863 
8864 	nvroot = fnvlist_alloc();
8865 	if (sav->sav_count == 0) {
8866 		fnvlist_add_nvlist_array(nvroot, config,
8867 		    (const nvlist_t * const *)NULL, 0);
8868 	} else {
8869 		list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP);
8870 		for (i = 0; i < sav->sav_count; i++)
8871 			list[i] = vdev_config_generate(spa, sav->sav_vdevs[i],
8872 			    B_FALSE, VDEV_CONFIG_L2CACHE);
8873 		fnvlist_add_nvlist_array(nvroot, config,
8874 		    (const nvlist_t * const *)list, sav->sav_count);
8875 		for (i = 0; i < sav->sav_count; i++)
8876 			nvlist_free(list[i]);
8877 		kmem_free(list, sav->sav_count * sizeof (void *));
8878 	}
8879 
8880 	spa_sync_nvlist(spa, sav->sav_object, nvroot, tx);
8881 	nvlist_free(nvroot);
8882 
8883 	sav->sav_sync = B_FALSE;
8884 }
8885 
8886 /*
8887  * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t.
8888  * The all-vdev ZAP must be empty.
8889  */
8890 static void
8891 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx)
8892 {
8893 	spa_t *spa = vd->vdev_spa;
8894 
8895 	if (vd->vdev_root_zap != 0 &&
8896 	    spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) {
8897 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
8898 		    vd->vdev_root_zap, tx));
8899 	}
8900 	if (vd->vdev_top_zap != 0) {
8901 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
8902 		    vd->vdev_top_zap, tx));
8903 	}
8904 	if (vd->vdev_leaf_zap != 0) {
8905 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
8906 		    vd->vdev_leaf_zap, tx));
8907 	}
8908 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
8909 		spa_avz_build(vd->vdev_child[i], avz, tx);
8910 	}
8911 }
8912 
8913 static void
8914 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx)
8915 {
8916 	nvlist_t *config;
8917 
8918 	/*
8919 	 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS,
8920 	 * its config may not be dirty but we still need to build per-vdev ZAPs.
8921 	 * Similarly, if the pool is being assembled (e.g. after a split), we
8922 	 * need to rebuild the AVZ although the config may not be dirty.
8923 	 */
8924 	if (list_is_empty(&spa->spa_config_dirty_list) &&
8925 	    spa->spa_avz_action == AVZ_ACTION_NONE)
8926 		return;
8927 
8928 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
8929 
8930 	ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE ||
8931 	    spa->spa_avz_action == AVZ_ACTION_INITIALIZE ||
8932 	    spa->spa_all_vdev_zaps != 0);
8933 
8934 	if (spa->spa_avz_action == AVZ_ACTION_REBUILD) {
8935 		/* Make and build the new AVZ */
8936 		uint64_t new_avz = zap_create(spa->spa_meta_objset,
8937 		    DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx);
8938 		spa_avz_build(spa->spa_root_vdev, new_avz, tx);
8939 
8940 		/* Diff old AVZ with new one */
8941 		zap_cursor_t zc;
8942 		zap_attribute_t za;
8943 
8944 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
8945 		    spa->spa_all_vdev_zaps);
8946 		    zap_cursor_retrieve(&zc, &za) == 0;
8947 		    zap_cursor_advance(&zc)) {
8948 			uint64_t vdzap = za.za_first_integer;
8949 			if (zap_lookup_int(spa->spa_meta_objset, new_avz,
8950 			    vdzap) == ENOENT) {
8951 				/*
8952 				 * ZAP is listed in old AVZ but not in new one;
8953 				 * destroy it
8954 				 */
8955 				VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap,
8956 				    tx));
8957 			}
8958 		}
8959 
8960 		zap_cursor_fini(&zc);
8961 
8962 		/* Destroy the old AVZ */
8963 		VERIFY0(zap_destroy(spa->spa_meta_objset,
8964 		    spa->spa_all_vdev_zaps, tx));
8965 
8966 		/* Replace the old AVZ in the dir obj with the new one */
8967 		VERIFY0(zap_update(spa->spa_meta_objset,
8968 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP,
8969 		    sizeof (new_avz), 1, &new_avz, tx));
8970 
8971 		spa->spa_all_vdev_zaps = new_avz;
8972 	} else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) {
8973 		zap_cursor_t zc;
8974 		zap_attribute_t za;
8975 
8976 		/* Walk through the AVZ and destroy all listed ZAPs */
8977 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
8978 		    spa->spa_all_vdev_zaps);
8979 		    zap_cursor_retrieve(&zc, &za) == 0;
8980 		    zap_cursor_advance(&zc)) {
8981 			uint64_t zap = za.za_first_integer;
8982 			VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx));
8983 		}
8984 
8985 		zap_cursor_fini(&zc);
8986 
8987 		/* Destroy and unlink the AVZ itself */
8988 		VERIFY0(zap_destroy(spa->spa_meta_objset,
8989 		    spa->spa_all_vdev_zaps, tx));
8990 		VERIFY0(zap_remove(spa->spa_meta_objset,
8991 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx));
8992 		spa->spa_all_vdev_zaps = 0;
8993 	}
8994 
8995 	if (spa->spa_all_vdev_zaps == 0) {
8996 		spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset,
8997 		    DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
8998 		    DMU_POOL_VDEV_ZAP_MAP, tx);
8999 	}
9000 	spa->spa_avz_action = AVZ_ACTION_NONE;
9001 
9002 	/* Create ZAPs for vdevs that don't have them. */
9003 	vdev_construct_zaps(spa->spa_root_vdev, tx);
9004 
9005 	config = spa_config_generate(spa, spa->spa_root_vdev,
9006 	    dmu_tx_get_txg(tx), B_FALSE);
9007 
9008 	/*
9009 	 * If we're upgrading the spa version then make sure that
9010 	 * the config object gets updated with the correct version.
9011 	 */
9012 	if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version)
9013 		fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
9014 		    spa->spa_uberblock.ub_version);
9015 
9016 	spa_config_exit(spa, SCL_STATE, FTAG);
9017 
9018 	nvlist_free(spa->spa_config_syncing);
9019 	spa->spa_config_syncing = config;
9020 
9021 	spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
9022 }
9023 
9024 static void
9025 spa_sync_version(void *arg, dmu_tx_t *tx)
9026 {
9027 	uint64_t *versionp = arg;
9028 	uint64_t version = *versionp;
9029 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
9030 
9031 	/*
9032 	 * Setting the version is special cased when first creating the pool.
9033 	 */
9034 	ASSERT(tx->tx_txg != TXG_INITIAL);
9035 
9036 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
9037 	ASSERT(version >= spa_version(spa));
9038 
9039 	spa->spa_uberblock.ub_version = version;
9040 	vdev_config_dirty(spa->spa_root_vdev);
9041 	spa_history_log_internal(spa, "set", tx, "version=%lld",
9042 	    (longlong_t)version);
9043 }
9044 
9045 /*
9046  * Set zpool properties.
9047  */
9048 static void
9049 spa_sync_props(void *arg, dmu_tx_t *tx)
9050 {
9051 	nvlist_t *nvp = arg;
9052 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
9053 	objset_t *mos = spa->spa_meta_objset;
9054 	nvpair_t *elem = NULL;
9055 
9056 	mutex_enter(&spa->spa_props_lock);
9057 
9058 	while ((elem = nvlist_next_nvpair(nvp, elem))) {
9059 		uint64_t intval;
9060 		const char *strval, *fname;
9061 		zpool_prop_t prop;
9062 		const char *propname;
9063 		const char *elemname = nvpair_name(elem);
9064 		zprop_type_t proptype;
9065 		spa_feature_t fid;
9066 
9067 		switch (prop = zpool_name_to_prop(elemname)) {
9068 		case ZPOOL_PROP_VERSION:
9069 			intval = fnvpair_value_uint64(elem);
9070 			/*
9071 			 * The version is synced separately before other
9072 			 * properties and should be correct by now.
9073 			 */
9074 			ASSERT3U(spa_version(spa), >=, intval);
9075 			break;
9076 
9077 		case ZPOOL_PROP_ALTROOT:
9078 			/*
9079 			 * 'altroot' is a non-persistent property. It should
9080 			 * have been set temporarily at creation or import time.
9081 			 */
9082 			ASSERT(spa->spa_root != NULL);
9083 			break;
9084 
9085 		case ZPOOL_PROP_READONLY:
9086 		case ZPOOL_PROP_CACHEFILE:
9087 			/*
9088 			 * 'readonly' and 'cachefile' are also non-persistent
9089 			 * properties.
9090 			 */
9091 			break;
9092 		case ZPOOL_PROP_COMMENT:
9093 			strval = fnvpair_value_string(elem);
9094 			if (spa->spa_comment != NULL)
9095 				spa_strfree(spa->spa_comment);
9096 			spa->spa_comment = spa_strdup(strval);
9097 			/*
9098 			 * We need to dirty the configuration on all the vdevs
9099 			 * so that their labels get updated.  We also need to
9100 			 * update the cache file to keep it in sync with the
9101 			 * MOS version. It's unnecessary to do this for pool
9102 			 * creation since the vdev's configuration has already
9103 			 * been dirtied.
9104 			 */
9105 			if (tx->tx_txg != TXG_INITIAL) {
9106 				vdev_config_dirty(spa->spa_root_vdev);
9107 				spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
9108 			}
9109 			spa_history_log_internal(spa, "set", tx,
9110 			    "%s=%s", elemname, strval);
9111 			break;
9112 		case ZPOOL_PROP_COMPATIBILITY:
9113 			strval = fnvpair_value_string(elem);
9114 			if (spa->spa_compatibility != NULL)
9115 				spa_strfree(spa->spa_compatibility);
9116 			spa->spa_compatibility = spa_strdup(strval);
9117 			/*
9118 			 * Dirty the configuration on vdevs as above.
9119 			 */
9120 			if (tx->tx_txg != TXG_INITIAL) {
9121 				vdev_config_dirty(spa->spa_root_vdev);
9122 				spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
9123 			}
9124 
9125 			spa_history_log_internal(spa, "set", tx,
9126 			    "%s=%s", nvpair_name(elem), strval);
9127 			break;
9128 
9129 		case ZPOOL_PROP_INVAL:
9130 			if (zpool_prop_feature(elemname)) {
9131 				fname = strchr(elemname, '@') + 1;
9132 				VERIFY0(zfeature_lookup_name(fname, &fid));
9133 
9134 				spa_feature_enable(spa, fid, tx);
9135 				spa_history_log_internal(spa, "set", tx,
9136 				    "%s=enabled", elemname);
9137 				break;
9138 			} else if (!zfs_prop_user(elemname)) {
9139 				ASSERT(zpool_prop_feature(elemname));
9140 				break;
9141 			}
9142 			zfs_fallthrough;
9143 		default:
9144 			/*
9145 			 * Set pool property values in the poolprops mos object.
9146 			 */
9147 			if (spa->spa_pool_props_object == 0) {
9148 				spa->spa_pool_props_object =
9149 				    zap_create_link(mos, DMU_OT_POOL_PROPS,
9150 				    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS,
9151 				    tx);
9152 			}
9153 
9154 			/* normalize the property name */
9155 			if (prop == ZPOOL_PROP_INVAL) {
9156 				propname = elemname;
9157 				proptype = PROP_TYPE_STRING;
9158 			} else {
9159 				propname = zpool_prop_to_name(prop);
9160 				proptype = zpool_prop_get_type(prop);
9161 			}
9162 
9163 			if (nvpair_type(elem) == DATA_TYPE_STRING) {
9164 				ASSERT(proptype == PROP_TYPE_STRING);
9165 				strval = fnvpair_value_string(elem);
9166 				VERIFY0(zap_update(mos,
9167 				    spa->spa_pool_props_object, propname,
9168 				    1, strlen(strval) + 1, strval, tx));
9169 				spa_history_log_internal(spa, "set", tx,
9170 				    "%s=%s", elemname, strval);
9171 			} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
9172 				intval = fnvpair_value_uint64(elem);
9173 
9174 				if (proptype == PROP_TYPE_INDEX) {
9175 					const char *unused;
9176 					VERIFY0(zpool_prop_index_to_string(
9177 					    prop, intval, &unused));
9178 				}
9179 				VERIFY0(zap_update(mos,
9180 				    spa->spa_pool_props_object, propname,
9181 				    8, 1, &intval, tx));
9182 				spa_history_log_internal(spa, "set", tx,
9183 				    "%s=%lld", elemname,
9184 				    (longlong_t)intval);
9185 
9186 				switch (prop) {
9187 				case ZPOOL_PROP_DELEGATION:
9188 					spa->spa_delegation = intval;
9189 					break;
9190 				case ZPOOL_PROP_BOOTFS:
9191 					spa->spa_bootfs = intval;
9192 					break;
9193 				case ZPOOL_PROP_FAILUREMODE:
9194 					spa->spa_failmode = intval;
9195 					break;
9196 				case ZPOOL_PROP_AUTOTRIM:
9197 					spa->spa_autotrim = intval;
9198 					spa_async_request(spa,
9199 					    SPA_ASYNC_AUTOTRIM_RESTART);
9200 					break;
9201 				case ZPOOL_PROP_AUTOEXPAND:
9202 					spa->spa_autoexpand = intval;
9203 					if (tx->tx_txg != TXG_INITIAL)
9204 						spa_async_request(spa,
9205 						    SPA_ASYNC_AUTOEXPAND);
9206 					break;
9207 				case ZPOOL_PROP_MULTIHOST:
9208 					spa->spa_multihost = intval;
9209 					break;
9210 				default:
9211 					break;
9212 				}
9213 			} else {
9214 				ASSERT(0); /* not allowed */
9215 			}
9216 		}
9217 
9218 	}
9219 
9220 	mutex_exit(&spa->spa_props_lock);
9221 }
9222 
9223 /*
9224  * Perform one-time upgrade on-disk changes.  spa_version() does not
9225  * reflect the new version this txg, so there must be no changes this
9226  * txg to anything that the upgrade code depends on after it executes.
9227  * Therefore this must be called after dsl_pool_sync() does the sync
9228  * tasks.
9229  */
9230 static void
9231 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx)
9232 {
9233 	if (spa_sync_pass(spa) != 1)
9234 		return;
9235 
9236 	dsl_pool_t *dp = spa->spa_dsl_pool;
9237 	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
9238 
9239 	if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN &&
9240 	    spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) {
9241 		dsl_pool_create_origin(dp, tx);
9242 
9243 		/* Keeping the origin open increases spa_minref */
9244 		spa->spa_minref += 3;
9245 	}
9246 
9247 	if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES &&
9248 	    spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) {
9249 		dsl_pool_upgrade_clones(dp, tx);
9250 	}
9251 
9252 	if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES &&
9253 	    spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) {
9254 		dsl_pool_upgrade_dir_clones(dp, tx);
9255 
9256 		/* Keeping the freedir open increases spa_minref */
9257 		spa->spa_minref += 3;
9258 	}
9259 
9260 	if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES &&
9261 	    spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
9262 		spa_feature_create_zap_objects(spa, tx);
9263 	}
9264 
9265 	/*
9266 	 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable
9267 	 * when possibility to use lz4 compression for metadata was added
9268 	 * Old pools that have this feature enabled must be upgraded to have
9269 	 * this feature active
9270 	 */
9271 	if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
9272 		boolean_t lz4_en = spa_feature_is_enabled(spa,
9273 		    SPA_FEATURE_LZ4_COMPRESS);
9274 		boolean_t lz4_ac = spa_feature_is_active(spa,
9275 		    SPA_FEATURE_LZ4_COMPRESS);
9276 
9277 		if (lz4_en && !lz4_ac)
9278 			spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
9279 	}
9280 
9281 	/*
9282 	 * If we haven't written the salt, do so now.  Note that the
9283 	 * feature may not be activated yet, but that's fine since
9284 	 * the presence of this ZAP entry is backwards compatible.
9285 	 */
9286 	if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
9287 	    DMU_POOL_CHECKSUM_SALT) == ENOENT) {
9288 		VERIFY0(zap_add(spa->spa_meta_objset,
9289 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1,
9290 		    sizeof (spa->spa_cksum_salt.zcs_bytes),
9291 		    spa->spa_cksum_salt.zcs_bytes, tx));
9292 	}
9293 
9294 	rrw_exit(&dp->dp_config_rwlock, FTAG);
9295 }
9296 
9297 static void
9298 vdev_indirect_state_sync_verify(vdev_t *vd)
9299 {
9300 	vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping;
9301 	vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births;
9302 
9303 	if (vd->vdev_ops == &vdev_indirect_ops) {
9304 		ASSERT(vim != NULL);
9305 		ASSERT(vib != NULL);
9306 	}
9307 
9308 	uint64_t obsolete_sm_object = 0;
9309 	ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object));
9310 	if (obsolete_sm_object != 0) {
9311 		ASSERT(vd->vdev_obsolete_sm != NULL);
9312 		ASSERT(vd->vdev_removing ||
9313 		    vd->vdev_ops == &vdev_indirect_ops);
9314 		ASSERT(vdev_indirect_mapping_num_entries(vim) > 0);
9315 		ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0);
9316 		ASSERT3U(obsolete_sm_object, ==,
9317 		    space_map_object(vd->vdev_obsolete_sm));
9318 		ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=,
9319 		    space_map_allocated(vd->vdev_obsolete_sm));
9320 	}
9321 	ASSERT(vd->vdev_obsolete_segments != NULL);
9322 
9323 	/*
9324 	 * Since frees / remaps to an indirect vdev can only
9325 	 * happen in syncing context, the obsolete segments
9326 	 * tree must be empty when we start syncing.
9327 	 */
9328 	ASSERT0(range_tree_space(vd->vdev_obsolete_segments));
9329 }
9330 
9331 /*
9332  * Set the top-level vdev's max queue depth. Evaluate each top-level's
9333  * async write queue depth in case it changed. The max queue depth will
9334  * not change in the middle of syncing out this txg.
9335  */
9336 static void
9337 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa)
9338 {
9339 	ASSERT(spa_writeable(spa));
9340 
9341 	vdev_t *rvd = spa->spa_root_vdev;
9342 	uint32_t max_queue_depth = zfs_vdev_async_write_max_active *
9343 	    zfs_vdev_queue_depth_pct / 100;
9344 	metaslab_class_t *normal = spa_normal_class(spa);
9345 	metaslab_class_t *special = spa_special_class(spa);
9346 	metaslab_class_t *dedup = spa_dedup_class(spa);
9347 
9348 	uint64_t slots_per_allocator = 0;
9349 	for (int c = 0; c < rvd->vdev_children; c++) {
9350 		vdev_t *tvd = rvd->vdev_child[c];
9351 
9352 		metaslab_group_t *mg = tvd->vdev_mg;
9353 		if (mg == NULL || !metaslab_group_initialized(mg))
9354 			continue;
9355 
9356 		metaslab_class_t *mc = mg->mg_class;
9357 		if (mc != normal && mc != special && mc != dedup)
9358 			continue;
9359 
9360 		/*
9361 		 * It is safe to do a lock-free check here because only async
9362 		 * allocations look at mg_max_alloc_queue_depth, and async
9363 		 * allocations all happen from spa_sync().
9364 		 */
9365 		for (int i = 0; i < mg->mg_allocators; i++) {
9366 			ASSERT0(zfs_refcount_count(
9367 			    &(mg->mg_allocator[i].mga_alloc_queue_depth)));
9368 		}
9369 		mg->mg_max_alloc_queue_depth = max_queue_depth;
9370 
9371 		for (int i = 0; i < mg->mg_allocators; i++) {
9372 			mg->mg_allocator[i].mga_cur_max_alloc_queue_depth =
9373 			    zfs_vdev_def_queue_depth;
9374 		}
9375 		slots_per_allocator += zfs_vdev_def_queue_depth;
9376 	}
9377 
9378 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9379 		ASSERT0(zfs_refcount_count(&normal->mc_allocator[i].
9380 		    mca_alloc_slots));
9381 		ASSERT0(zfs_refcount_count(&special->mc_allocator[i].
9382 		    mca_alloc_slots));
9383 		ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i].
9384 		    mca_alloc_slots));
9385 		normal->mc_allocator[i].mca_alloc_max_slots =
9386 		    slots_per_allocator;
9387 		special->mc_allocator[i].mca_alloc_max_slots =
9388 		    slots_per_allocator;
9389 		dedup->mc_allocator[i].mca_alloc_max_slots =
9390 		    slots_per_allocator;
9391 	}
9392 	normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9393 	special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9394 	dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9395 }
9396 
9397 static void
9398 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx)
9399 {
9400 	ASSERT(spa_writeable(spa));
9401 
9402 	vdev_t *rvd = spa->spa_root_vdev;
9403 	for (int c = 0; c < rvd->vdev_children; c++) {
9404 		vdev_t *vd = rvd->vdev_child[c];
9405 		vdev_indirect_state_sync_verify(vd);
9406 
9407 		if (vdev_indirect_should_condense(vd)) {
9408 			spa_condense_indirect_start_sync(vd, tx);
9409 			break;
9410 		}
9411 	}
9412 }
9413 
9414 static void
9415 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx)
9416 {
9417 	objset_t *mos = spa->spa_meta_objset;
9418 	dsl_pool_t *dp = spa->spa_dsl_pool;
9419 	uint64_t txg = tx->tx_txg;
9420 	bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK];
9421 
9422 	do {
9423 		int pass = ++spa->spa_sync_pass;
9424 
9425 		spa_sync_config_object(spa, tx);
9426 		spa_sync_aux_dev(spa, &spa->spa_spares, tx,
9427 		    ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES);
9428 		spa_sync_aux_dev(spa, &spa->spa_l2cache, tx,
9429 		    ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE);
9430 		spa_errlog_sync(spa, txg);
9431 		dsl_pool_sync(dp, txg);
9432 
9433 		if (pass < zfs_sync_pass_deferred_free ||
9434 		    spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) {
9435 			/*
9436 			 * If the log space map feature is active we don't
9437 			 * care about deferred frees and the deferred bpobj
9438 			 * as the log space map should effectively have the
9439 			 * same results (i.e. appending only to one object).
9440 			 */
9441 			spa_sync_frees(spa, free_bpl, tx);
9442 		} else {
9443 			/*
9444 			 * We can not defer frees in pass 1, because
9445 			 * we sync the deferred frees later in pass 1.
9446 			 */
9447 			ASSERT3U(pass, >, 1);
9448 			bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb,
9449 			    &spa->spa_deferred_bpobj, tx);
9450 		}
9451 
9452 		brt_sync(spa, txg);
9453 		ddt_sync(spa, txg);
9454 		dsl_scan_sync(dp, tx);
9455 		dsl_errorscrub_sync(dp, tx);
9456 		svr_sync(spa, tx);
9457 		spa_sync_upgrades(spa, tx);
9458 
9459 		spa_flush_metaslabs(spa, tx);
9460 
9461 		vdev_t *vd = NULL;
9462 		while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
9463 		    != NULL)
9464 			vdev_sync(vd, txg);
9465 
9466 		if (pass == 1) {
9467 			/*
9468 			 * dsl_pool_sync() -> dp_sync_tasks may have dirtied
9469 			 * the config. If that happens, this txg should not
9470 			 * be a no-op. So we must sync the config to the MOS
9471 			 * before checking for no-op.
9472 			 *
9473 			 * Note that when the config is dirty, it will
9474 			 * be written to the MOS (i.e. the MOS will be
9475 			 * dirtied) every time we call spa_sync_config_object()
9476 			 * in this txg.  Therefore we can't call this after
9477 			 * dsl_pool_sync() every pass, because it would
9478 			 * prevent us from converging, since we'd dirty
9479 			 * the MOS every pass.
9480 			 *
9481 			 * Sync tasks can only be processed in pass 1, so
9482 			 * there's no need to do this in later passes.
9483 			 */
9484 			spa_sync_config_object(spa, tx);
9485 		}
9486 
9487 		/*
9488 		 * Note: We need to check if the MOS is dirty because we could
9489 		 * have marked the MOS dirty without updating the uberblock
9490 		 * (e.g. if we have sync tasks but no dirty user data). We need
9491 		 * to check the uberblock's rootbp because it is updated if we
9492 		 * have synced out dirty data (though in this case the MOS will
9493 		 * most likely also be dirty due to second order effects, we
9494 		 * don't want to rely on that here).
9495 		 */
9496 		if (pass == 1 &&
9497 		    spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
9498 		    !dmu_objset_is_dirty(mos, txg)) {
9499 			/*
9500 			 * Nothing changed on the first pass, therefore this
9501 			 * TXG is a no-op. Avoid syncing deferred frees, so
9502 			 * that we can keep this TXG as a no-op.
9503 			 */
9504 			ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
9505 			ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
9506 			ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
9507 			ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg));
9508 			break;
9509 		}
9510 
9511 		spa_sync_deferred_frees(spa, tx);
9512 	} while (dmu_objset_is_dirty(mos, txg));
9513 }
9514 
9515 /*
9516  * Rewrite the vdev configuration (which includes the uberblock) to
9517  * commit the transaction group.
9518  *
9519  * If there are no dirty vdevs, we sync the uberblock to a few random
9520  * top-level vdevs that are known to be visible in the config cache
9521  * (see spa_vdev_add() for a complete description). If there *are* dirty
9522  * vdevs, sync the uberblock to all vdevs.
9523  */
9524 static void
9525 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx)
9526 {
9527 	vdev_t *rvd = spa->spa_root_vdev;
9528 	uint64_t txg = tx->tx_txg;
9529 
9530 	for (;;) {
9531 		int error = 0;
9532 
9533 		/*
9534 		 * We hold SCL_STATE to prevent vdev open/close/etc.
9535 		 * while we're attempting to write the vdev labels.
9536 		 */
9537 		spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9538 
9539 		if (list_is_empty(&spa->spa_config_dirty_list)) {
9540 			vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
9541 			int svdcount = 0;
9542 			int children = rvd->vdev_children;
9543 			int c0 = random_in_range(children);
9544 
9545 			for (int c = 0; c < children; c++) {
9546 				vdev_t *vd =
9547 				    rvd->vdev_child[(c0 + c) % children];
9548 
9549 				/* Stop when revisiting the first vdev */
9550 				if (c > 0 && svd[0] == vd)
9551 					break;
9552 
9553 				if (vd->vdev_ms_array == 0 ||
9554 				    vd->vdev_islog ||
9555 				    !vdev_is_concrete(vd))
9556 					continue;
9557 
9558 				svd[svdcount++] = vd;
9559 				if (svdcount == SPA_SYNC_MIN_VDEVS)
9560 					break;
9561 			}
9562 			error = vdev_config_sync(svd, svdcount, txg);
9563 		} else {
9564 			error = vdev_config_sync(rvd->vdev_child,
9565 			    rvd->vdev_children, txg);
9566 		}
9567 
9568 		if (error == 0)
9569 			spa->spa_last_synced_guid = rvd->vdev_guid;
9570 
9571 		spa_config_exit(spa, SCL_STATE, FTAG);
9572 
9573 		if (error == 0)
9574 			break;
9575 		zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR);
9576 		zio_resume_wait(spa);
9577 	}
9578 }
9579 
9580 /*
9581  * Sync the specified transaction group.  New blocks may be dirtied as
9582  * part of the process, so we iterate until it converges.
9583  */
9584 void
9585 spa_sync(spa_t *spa, uint64_t txg)
9586 {
9587 	vdev_t *vd = NULL;
9588 
9589 	VERIFY(spa_writeable(spa));
9590 
9591 	/*
9592 	 * Wait for i/os issued in open context that need to complete
9593 	 * before this txg syncs.
9594 	 */
9595 	(void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]);
9596 	spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL,
9597 	    ZIO_FLAG_CANFAIL);
9598 
9599 	/*
9600 	 * Now that there can be no more cloning in this transaction group,
9601 	 * but we are still before issuing frees, we can process pending BRT
9602 	 * updates.
9603 	 */
9604 	brt_pending_apply(spa, txg);
9605 
9606 	/*
9607 	 * Lock out configuration changes.
9608 	 */
9609 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
9610 
9611 	spa->spa_syncing_txg = txg;
9612 	spa->spa_sync_pass = 0;
9613 
9614 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9615 		mutex_enter(&spa->spa_allocs[i].spaa_lock);
9616 		VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
9617 		mutex_exit(&spa->spa_allocs[i].spaa_lock);
9618 	}
9619 
9620 	/*
9621 	 * If there are any pending vdev state changes, convert them
9622 	 * into config changes that go out with this transaction group.
9623 	 */
9624 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9625 	while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
9626 		/* Avoid holding the write lock unless actually necessary */
9627 		if (vd->vdev_aux == NULL) {
9628 			vdev_state_clean(vd);
9629 			vdev_config_dirty(vd);
9630 			continue;
9631 		}
9632 		/*
9633 		 * We need the write lock here because, for aux vdevs,
9634 		 * calling vdev_config_dirty() modifies sav_config.
9635 		 * This is ugly and will become unnecessary when we
9636 		 * eliminate the aux vdev wart by integrating all vdevs
9637 		 * into the root vdev tree.
9638 		 */
9639 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9640 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER);
9641 		while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
9642 			vdev_state_clean(vd);
9643 			vdev_config_dirty(vd);
9644 		}
9645 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9646 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
9647 	}
9648 	spa_config_exit(spa, SCL_STATE, FTAG);
9649 
9650 	dsl_pool_t *dp = spa->spa_dsl_pool;
9651 	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);
9652 
9653 	spa->spa_sync_starttime = gethrtime();
9654 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
9655 	spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq,
9656 	    spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() +
9657 	    NSEC_TO_TICK(spa->spa_deadman_synctime));
9658 
9659 	/*
9660 	 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg,
9661 	 * set spa_deflate if we have no raid-z vdevs.
9662 	 */
9663 	if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE &&
9664 	    spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) {
9665 		vdev_t *rvd = spa->spa_root_vdev;
9666 
9667 		int i;
9668 		for (i = 0; i < rvd->vdev_children; i++) {
9669 			vd = rvd->vdev_child[i];
9670 			if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
9671 				break;
9672 		}
9673 		if (i == rvd->vdev_children) {
9674 			spa->spa_deflate = TRUE;
9675 			VERIFY0(zap_add(spa->spa_meta_objset,
9676 			    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
9677 			    sizeof (uint64_t), 1, &spa->spa_deflate, tx));
9678 		}
9679 	}
9680 
9681 	spa_sync_adjust_vdev_max_queue_depth(spa);
9682 
9683 	spa_sync_condense_indirect(spa, tx);
9684 
9685 	spa_sync_iterate_to_convergence(spa, tx);
9686 
9687 #ifdef ZFS_DEBUG
9688 	if (!list_is_empty(&spa->spa_config_dirty_list)) {
9689 	/*
9690 	 * Make sure that the number of ZAPs for all the vdevs matches
9691 	 * the number of ZAPs in the per-vdev ZAP list. This only gets
9692 	 * called if the config is dirty; otherwise there may be
9693 	 * outstanding AVZ operations that weren't completed in
9694 	 * spa_sync_config_object.
9695 	 */
9696 		uint64_t all_vdev_zap_entry_count;
9697 		ASSERT0(zap_count(spa->spa_meta_objset,
9698 		    spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count));
9699 		ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==,
9700 		    all_vdev_zap_entry_count);
9701 	}
9702 #endif
9703 
9704 	if (spa->spa_vdev_removal != NULL) {
9705 		ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]);
9706 	}
9707 
9708 	spa_sync_rewrite_vdev_config(spa, tx);
9709 	dmu_tx_commit(tx);
9710 
9711 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
9712 	spa->spa_deadman_tqid = 0;
9713 
9714 	/*
9715 	 * Clear the dirty config list.
9716 	 */
9717 	while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL)
9718 		vdev_config_clean(vd);
9719 
9720 	/*
9721 	 * Now that the new config has synced transactionally,
9722 	 * let it become visible to the config cache.
9723 	 */
9724 	if (spa->spa_config_syncing != NULL) {
9725 		spa_config_set(spa, spa->spa_config_syncing);
9726 		spa->spa_config_txg = txg;
9727 		spa->spa_config_syncing = NULL;
9728 	}
9729 
9730 	dsl_pool_sync_done(dp, txg);
9731 
9732 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9733 		mutex_enter(&spa->spa_allocs[i].spaa_lock);
9734 		VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
9735 		mutex_exit(&spa->spa_allocs[i].spaa_lock);
9736 	}
9737 
9738 	/*
9739 	 * Update usable space statistics.
9740 	 */
9741 	while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
9742 	    != NULL)
9743 		vdev_sync_done(vd, txg);
9744 
9745 	metaslab_class_evict_old(spa->spa_normal_class, txg);
9746 	metaslab_class_evict_old(spa->spa_log_class, txg);
9747 
9748 	spa_sync_close_syncing_log_sm(spa);
9749 
9750 	spa_update_dspace(spa);
9751 
9752 	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON)
9753 		vdev_autotrim_kick(spa);
9754 
9755 	/*
9756 	 * It had better be the case that we didn't dirty anything
9757 	 * since vdev_config_sync().
9758 	 */
9759 	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
9760 	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
9761 	ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));
9762 
9763 	while (zfs_pause_spa_sync)
9764 		delay(1);
9765 
9766 	spa->spa_sync_pass = 0;
9767 
9768 	/*
9769 	 * Update the last synced uberblock here. We want to do this at
9770 	 * the end of spa_sync() so that consumers of spa_last_synced_txg()
9771 	 * will be guaranteed that all the processing associated with
9772 	 * that txg has been completed.
9773 	 */
9774 	spa->spa_ubsync = spa->spa_uberblock;
9775 	spa_config_exit(spa, SCL_CONFIG, FTAG);
9776 
9777 	spa_handle_ignored_writes(spa);
9778 
9779 	/*
9780 	 * If any async tasks have been requested, kick them off.
9781 	 */
9782 	spa_async_dispatch(spa);
9783 }
9784 
9785 /*
9786  * Sync all pools.  We don't want to hold the namespace lock across these
9787  * operations, so we take a reference on the spa_t and drop the lock during the
9788  * sync.
9789  */
9790 void
9791 spa_sync_allpools(void)
9792 {
9793 	spa_t *spa = NULL;
9794 	mutex_enter(&spa_namespace_lock);
9795 	while ((spa = spa_next(spa)) != NULL) {
9796 		if (spa_state(spa) != POOL_STATE_ACTIVE ||
9797 		    !spa_writeable(spa) || spa_suspended(spa))
9798 			continue;
9799 		spa_open_ref(spa, FTAG);
9800 		mutex_exit(&spa_namespace_lock);
9801 		txg_wait_synced(spa_get_dsl(spa), 0);
9802 		mutex_enter(&spa_namespace_lock);
9803 		spa_close(spa, FTAG);
9804 	}
9805 	mutex_exit(&spa_namespace_lock);
9806 }
9807 
9808 taskq_t *
9809 spa_sync_tq_create(spa_t *spa, const char *name)
9810 {
9811 	kthread_t **kthreads;
9812 
9813 	ASSERT(spa->spa_sync_tq == NULL);
9814 	ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus);
9815 
9816 	/*
9817 	 * - do not allow more allocators than cpus.
9818 	 * - there may be more cpus than allocators.
9819 	 * - do not allow more sync taskq threads than allocators or cpus.
9820 	 */
9821 	int nthreads = spa->spa_alloc_count;
9822 	spa->spa_syncthreads = kmem_zalloc(sizeof (spa_syncthread_info_t) *
9823 	    nthreads, KM_SLEEP);
9824 
9825 	spa->spa_sync_tq = taskq_create_synced(name, nthreads, minclsyspri,
9826 	    nthreads, INT_MAX, TASKQ_PREPOPULATE, &kthreads);
9827 	VERIFY(spa->spa_sync_tq != NULL);
9828 	VERIFY(kthreads != NULL);
9829 
9830 	spa_taskqs_t *tqs =
9831 	    &spa->spa_zio_taskq[ZIO_TYPE_WRITE][ZIO_TASKQ_ISSUE];
9832 
9833 	spa_syncthread_info_t *ti = spa->spa_syncthreads;
9834 	for (int i = 0, w = 0; i < nthreads; i++, w++, ti++) {
9835 		ti->sti_thread = kthreads[i];
9836 		if (w == tqs->stqs_count) {
9837 			w = 0;
9838 		}
9839 		ti->sti_wr_iss_tq = tqs->stqs_taskq[w];
9840 	}
9841 
9842 	kmem_free(kthreads, sizeof (*kthreads) * nthreads);
9843 	return (spa->spa_sync_tq);
9844 }
9845 
9846 void
9847 spa_sync_tq_destroy(spa_t *spa)
9848 {
9849 	ASSERT(spa->spa_sync_tq != NULL);
9850 
9851 	taskq_wait(spa->spa_sync_tq);
9852 	taskq_destroy(spa->spa_sync_tq);
9853 	kmem_free(spa->spa_syncthreads,
9854 	    sizeof (spa_syncthread_info_t) * spa->spa_alloc_count);
9855 	spa->spa_sync_tq = NULL;
9856 }
9857 
9858 void
9859 spa_select_allocator(zio_t *zio)
9860 {
9861 	zbookmark_phys_t *bm = &zio->io_bookmark;
9862 	spa_t *spa = zio->io_spa;
9863 
9864 	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
9865 
9866 	/*
9867 	 * A gang block (for example) may have inherited its parent's
9868 	 * allocator, in which case there is nothing further to do here.
9869 	 */
9870 	if (ZIO_HAS_ALLOCATOR(zio))
9871 		return;
9872 
9873 	ASSERT(spa != NULL);
9874 	ASSERT(bm != NULL);
9875 
9876 	/*
9877 	 * First try to use an allocator assigned to the syncthread, and set
9878 	 * the corresponding write issue taskq for the allocator.
9879 	 * Note, we must have an open pool to do this.
9880 	 */
9881 	if (spa->spa_sync_tq != NULL) {
9882 		spa_syncthread_info_t *ti = spa->spa_syncthreads;
9883 		for (int i = 0; i < spa->spa_alloc_count; i++, ti++) {
9884 			if (ti->sti_thread == curthread) {
9885 				zio->io_allocator = i;
9886 				zio->io_wr_iss_tq = ti->sti_wr_iss_tq;
9887 				return;
9888 			}
9889 		}
9890 	}
9891 
9892 	/*
9893 	 * We want to try to use as many allocators as possible to help improve
9894 	 * performance, but we also want logically adjacent IOs to be physically
9895 	 * adjacent to improve sequential read performance. We chunk each object
9896 	 * into 2^20 block regions, and then hash based on the objset, object,
9897 	 * level, and region to accomplish both of these goals.
9898 	 */
9899 	uint64_t hv = cityhash4(bm->zb_objset, bm->zb_object, bm->zb_level,
9900 	    bm->zb_blkid >> 20);
9901 
9902 	zio->io_allocator = (uint_t)hv % spa->spa_alloc_count;
9903 	zio->io_wr_iss_tq = NULL;
9904 }
9905 
9906 /*
9907  * ==========================================================================
9908  * Miscellaneous routines
9909  * ==========================================================================
9910  */
9911 
9912 /*
9913  * Remove all pools in the system.
9914  */
9915 void
9916 spa_evict_all(void)
9917 {
9918 	spa_t *spa;
9919 
9920 	/*
9921 	 * Remove all cached state.  All pools should be closed now,
9922 	 * so every spa in the AVL tree should be unreferenced.
9923 	 */
9924 	mutex_enter(&spa_namespace_lock);
9925 	while ((spa = spa_next(NULL)) != NULL) {
9926 		/*
9927 		 * Stop async tasks.  The async thread may need to detach
9928 		 * a device that's been replaced, which requires grabbing
9929 		 * spa_namespace_lock, so we must drop it here.
9930 		 */
9931 		spa_open_ref(spa, FTAG);
9932 		mutex_exit(&spa_namespace_lock);
9933 		spa_async_suspend(spa);
9934 		mutex_enter(&spa_namespace_lock);
9935 		spa_close(spa, FTAG);
9936 
9937 		if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
9938 			spa_unload(spa);
9939 			spa_deactivate(spa);
9940 		}
9941 		spa_remove(spa);
9942 	}
9943 	mutex_exit(&spa_namespace_lock);
9944 }
9945 
9946 vdev_t *
9947 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux)
9948 {
9949 	vdev_t *vd;
9950 	int i;
9951 
9952 	if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL)
9953 		return (vd);
9954 
9955 	if (aux) {
9956 		for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
9957 			vd = spa->spa_l2cache.sav_vdevs[i];
9958 			if (vd->vdev_guid == guid)
9959 				return (vd);
9960 		}
9961 
9962 		for (i = 0; i < spa->spa_spares.sav_count; i++) {
9963 			vd = spa->spa_spares.sav_vdevs[i];
9964 			if (vd->vdev_guid == guid)
9965 				return (vd);
9966 		}
9967 	}
9968 
9969 	return (NULL);
9970 }
9971 
9972 void
9973 spa_upgrade(spa_t *spa, uint64_t version)
9974 {
9975 	ASSERT(spa_writeable(spa));
9976 
9977 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
9978 
9979 	/*
9980 	 * This should only be called for a non-faulted pool, and since a
9981 	 * future version would result in an unopenable pool, this shouldn't be
9982 	 * possible.
9983 	 */
9984 	ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version));
9985 	ASSERT3U(version, >=, spa->spa_uberblock.ub_version);
9986 
9987 	spa->spa_uberblock.ub_version = version;
9988 	vdev_config_dirty(spa->spa_root_vdev);
9989 
9990 	spa_config_exit(spa, SCL_ALL, FTAG);
9991 
9992 	txg_wait_synced(spa_get_dsl(spa), 0);
9993 }
9994 
9995 static boolean_t
9996 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav)
9997 {
9998 	(void) spa;
9999 	int i;
10000 	uint64_t vdev_guid;
10001 
10002 	for (i = 0; i < sav->sav_count; i++)
10003 		if (sav->sav_vdevs[i]->vdev_guid == guid)
10004 			return (B_TRUE);
10005 
10006 	for (i = 0; i < sav->sav_npending; i++) {
10007 		if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID,
10008 		    &vdev_guid) == 0 && vdev_guid == guid)
10009 			return (B_TRUE);
10010 	}
10011 
10012 	return (B_FALSE);
10013 }
10014 
10015 boolean_t
10016 spa_has_l2cache(spa_t *spa, uint64_t guid)
10017 {
10018 	return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache));
10019 }
10020 
10021 boolean_t
10022 spa_has_spare(spa_t *spa, uint64_t guid)
10023 {
10024 	return (spa_has_aux_vdev(spa, guid, &spa->spa_spares));
10025 }
10026 
10027 /*
10028  * Check if a pool has an active shared spare device.
10029  * Note: reference count of an active spare is 2, as a spare and as a replace
10030  */
10031 static boolean_t
10032 spa_has_active_shared_spare(spa_t *spa)
10033 {
10034 	int i, refcnt;
10035 	uint64_t pool;
10036 	spa_aux_vdev_t *sav = &spa->spa_spares;
10037 
10038 	for (i = 0; i < sav->sav_count; i++) {
10039 		if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool,
10040 		    &refcnt) && pool != 0ULL && pool == spa_guid(spa) &&
10041 		    refcnt > 2)
10042 			return (B_TRUE);
10043 	}
10044 
10045 	return (B_FALSE);
10046 }
10047 
10048 uint64_t
10049 spa_total_metaslabs(spa_t *spa)
10050 {
10051 	vdev_t *rvd = spa->spa_root_vdev;
10052 
10053 	uint64_t m = 0;
10054 	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
10055 		vdev_t *vd = rvd->vdev_child[c];
10056 		if (!vdev_is_concrete(vd))
10057 			continue;
10058 		m += vd->vdev_ms_count;
10059 	}
10060 	return (m);
10061 }
10062 
10063 /*
10064  * Notify any waiting threads that some activity has switched from being in-
10065  * progress to not-in-progress so that the thread can wake up and determine
10066  * whether it is finished waiting.
10067  */
10068 void
10069 spa_notify_waiters(spa_t *spa)
10070 {
10071 	/*
10072 	 * Acquiring spa_activities_lock here prevents the cv_broadcast from
10073 	 * happening between the waiting thread's check and cv_wait.
10074 	 */
10075 	mutex_enter(&spa->spa_activities_lock);
10076 	cv_broadcast(&spa->spa_activities_cv);
10077 	mutex_exit(&spa->spa_activities_lock);
10078 }
10079 
10080 /*
10081  * Notify any waiting threads that the pool is exporting, and then block until
10082  * they are finished using the spa_t.
10083  */
10084 void
10085 spa_wake_waiters(spa_t *spa)
10086 {
10087 	mutex_enter(&spa->spa_activities_lock);
10088 	spa->spa_waiters_cancel = B_TRUE;
10089 	cv_broadcast(&spa->spa_activities_cv);
10090 	while (spa->spa_waiters != 0)
10091 		cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock);
10092 	spa->spa_waiters_cancel = B_FALSE;
10093 	mutex_exit(&spa->spa_activities_lock);
10094 }
10095 
10096 /* Whether the vdev or any of its descendants are being initialized/trimmed. */
10097 static boolean_t
10098 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity)
10099 {
10100 	spa_t *spa = vd->vdev_spa;
10101 
10102 	ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER));
10103 	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
10104 	ASSERT(activity == ZPOOL_WAIT_INITIALIZE ||
10105 	    activity == ZPOOL_WAIT_TRIM);
10106 
10107 	kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ?
10108 	    &vd->vdev_initialize_lock : &vd->vdev_trim_lock;
10109 
10110 	mutex_exit(&spa->spa_activities_lock);
10111 	mutex_enter(lock);
10112 	mutex_enter(&spa->spa_activities_lock);
10113 
10114 	boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ?
10115 	    (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) :
10116 	    (vd->vdev_trim_state == VDEV_TRIM_ACTIVE);
10117 	mutex_exit(lock);
10118 
10119 	if (in_progress)
10120 		return (B_TRUE);
10121 
10122 	for (int i = 0; i < vd->vdev_children; i++) {
10123 		if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i],
10124 		    activity))
10125 			return (B_TRUE);
10126 	}
10127 
10128 	return (B_FALSE);
10129 }
10130 
10131 /*
10132  * If use_guid is true, this checks whether the vdev specified by guid is
10133  * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool
10134  * is being initialized/trimmed. The caller must hold the config lock and
10135  * spa_activities_lock.
10136  */
10137 static int
10138 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid,
10139     zpool_wait_activity_t activity, boolean_t *in_progress)
10140 {
10141 	mutex_exit(&spa->spa_activities_lock);
10142 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
10143 	mutex_enter(&spa->spa_activities_lock);
10144 
10145 	vdev_t *vd;
10146 	if (use_guid) {
10147 		vd = spa_lookup_by_guid(spa, guid, B_FALSE);
10148 		if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) {
10149 			spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
10150 			return (EINVAL);
10151 		}
10152 	} else {
10153 		vd = spa->spa_root_vdev;
10154 	}
10155 
10156 	*in_progress = spa_vdev_activity_in_progress_impl(vd, activity);
10157 
10158 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
10159 	return (0);
10160 }
10161 
10162 /*
10163  * Locking for waiting threads
10164  * ---------------------------
10165  *
10166  * Waiting threads need a way to check whether a given activity is in progress,
10167  * and then, if it is, wait for it to complete. Each activity will have some
10168  * in-memory representation of the relevant on-disk state which can be used to
10169  * determine whether or not the activity is in progress. The in-memory state and
10170  * the locking used to protect it will be different for each activity, and may
10171  * not be suitable for use with a cvar (e.g., some state is protected by the
10172  * config lock). To allow waiting threads to wait without any races, another
10173  * lock, spa_activities_lock, is used.
10174  *
10175  * When the state is checked, both the activity-specific lock (if there is one)
10176  * and spa_activities_lock are held. In some cases, the activity-specific lock
10177  * is acquired explicitly (e.g. the config lock). In others, the locking is
10178  * internal to some check (e.g. bpobj_is_empty). After checking, the waiting
10179  * thread releases the activity-specific lock and, if the activity is in
10180  * progress, then cv_waits using spa_activities_lock.
10181  *
10182  * The waiting thread is woken when another thread, one completing some
10183  * activity, updates the state of the activity and then calls
10184  * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only
10185  * needs to hold its activity-specific lock when updating the state, and this
10186  * lock can (but doesn't have to) be dropped before calling spa_notify_waiters.
10187  *
10188  * Because spa_notify_waiters acquires spa_activities_lock before broadcasting,
10189  * and because it is held when the waiting thread checks the state of the
10190  * activity, it can never be the case that the completing thread both updates
10191  * the activity state and cv_broadcasts in between the waiting thread's check
10192  * and cv_wait. Thus, a waiting thread can never miss a wakeup.
10193  *
10194  * In order to prevent deadlock, when the waiting thread does its check, in some
10195  * cases it will temporarily drop spa_activities_lock in order to acquire the
10196  * activity-specific lock. The order in which spa_activities_lock and the
10197  * activity specific lock are acquired in the waiting thread is determined by
10198  * the order in which they are acquired in the completing thread; if the
10199  * completing thread calls spa_notify_waiters with the activity-specific lock
10200  * held, then the waiting thread must also acquire the activity-specific lock
10201  * first.
10202  */
10203 
10204 static int
10205 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity,
10206     boolean_t use_tag, uint64_t tag, boolean_t *in_progress)
10207 {
10208 	int error = 0;
10209 
10210 	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
10211 
10212 	switch (activity) {
10213 	case ZPOOL_WAIT_CKPT_DISCARD:
10214 		*in_progress =
10215 		    (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) &&
10216 		    zap_contains(spa_meta_objset(spa),
10217 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) ==
10218 		    ENOENT);
10219 		break;
10220 	case ZPOOL_WAIT_FREE:
10221 		*in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS &&
10222 		    !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) ||
10223 		    spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) ||
10224 		    spa_livelist_delete_check(spa));
10225 		break;
10226 	case ZPOOL_WAIT_INITIALIZE:
10227 	case ZPOOL_WAIT_TRIM:
10228 		error = spa_vdev_activity_in_progress(spa, use_tag, tag,
10229 		    activity, in_progress);
10230 		break;
10231 	case ZPOOL_WAIT_REPLACE:
10232 		mutex_exit(&spa->spa_activities_lock);
10233 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
10234 		mutex_enter(&spa->spa_activities_lock);
10235 
10236 		*in_progress = vdev_replace_in_progress(spa->spa_root_vdev);
10237 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
10238 		break;
10239 	case ZPOOL_WAIT_REMOVE:
10240 		*in_progress = (spa->spa_removing_phys.sr_state ==
10241 		    DSS_SCANNING);
10242 		break;
10243 	case ZPOOL_WAIT_RESILVER:
10244 		*in_progress = vdev_rebuild_active(spa->spa_root_vdev);
10245 		if (*in_progress)
10246 			break;
10247 		zfs_fallthrough;
10248 	case ZPOOL_WAIT_SCRUB:
10249 	{
10250 		boolean_t scanning, paused, is_scrub;
10251 		dsl_scan_t *scn =  spa->spa_dsl_pool->dp_scan;
10252 
10253 		is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB);
10254 		scanning = (scn->scn_phys.scn_state == DSS_SCANNING);
10255 		paused = dsl_scan_is_paused_scrub(scn);
10256 		*in_progress = (scanning && !paused &&
10257 		    is_scrub == (activity == ZPOOL_WAIT_SCRUB));
10258 		break;
10259 	}
10260 	case ZPOOL_WAIT_RAIDZ_EXPAND:
10261 	{
10262 		vdev_raidz_expand_t *vre = spa->spa_raidz_expand;
10263 		*in_progress = (vre != NULL && vre->vre_state == DSS_SCANNING);
10264 		break;
10265 	}
10266 	default:
10267 		panic("unrecognized value for activity %d", activity);
10268 	}
10269 
10270 	return (error);
10271 }
10272 
10273 static int
10274 spa_wait_common(const char *pool, zpool_wait_activity_t activity,
10275     boolean_t use_tag, uint64_t tag, boolean_t *waited)
10276 {
10277 	/*
10278 	 * The tag is used to distinguish between instances of an activity.
10279 	 * 'initialize' and 'trim' are the only activities that we use this for.
10280 	 * The other activities can only have a single instance in progress in a
10281 	 * pool at one time, making the tag unnecessary.
10282 	 *
10283 	 * There can be multiple devices being replaced at once, but since they
10284 	 * all finish once resilvering finishes, we don't bother keeping track
10285 	 * of them individually, we just wait for them all to finish.
10286 	 */
10287 	if (use_tag && activity != ZPOOL_WAIT_INITIALIZE &&
10288 	    activity != ZPOOL_WAIT_TRIM)
10289 		return (EINVAL);
10290 
10291 	if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES)
10292 		return (EINVAL);
10293 
10294 	spa_t *spa;
10295 	int error = spa_open(pool, &spa, FTAG);
10296 	if (error != 0)
10297 		return (error);
10298 
10299 	/*
10300 	 * Increment the spa's waiter count so that we can call spa_close and
10301 	 * still ensure that the spa_t doesn't get freed before this thread is
10302 	 * finished with it when the pool is exported. We want to call spa_close
10303 	 * before we start waiting because otherwise the additional ref would
10304 	 * prevent the pool from being exported or destroyed throughout the
10305 	 * potentially long wait.
10306 	 */
10307 	mutex_enter(&spa->spa_activities_lock);
10308 	spa->spa_waiters++;
10309 	spa_close(spa, FTAG);
10310 
10311 	*waited = B_FALSE;
10312 	for (;;) {
10313 		boolean_t in_progress;
10314 		error = spa_activity_in_progress(spa, activity, use_tag, tag,
10315 		    &in_progress);
10316 
10317 		if (error || !in_progress || spa->spa_waiters_cancel)
10318 			break;
10319 
10320 		*waited = B_TRUE;
10321 
10322 		if (cv_wait_sig(&spa->spa_activities_cv,
10323 		    &spa->spa_activities_lock) == 0) {
10324 			error = EINTR;
10325 			break;
10326 		}
10327 	}
10328 
10329 	spa->spa_waiters--;
10330 	cv_signal(&spa->spa_waiters_cv);
10331 	mutex_exit(&spa->spa_activities_lock);
10332 
10333 	return (error);
10334 }
10335 
10336 /*
10337  * Wait for a particular instance of the specified activity to complete, where
10338  * the instance is identified by 'tag'
10339  */
10340 int
10341 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag,
10342     boolean_t *waited)
10343 {
10344 	return (spa_wait_common(pool, activity, B_TRUE, tag, waited));
10345 }
10346 
10347 /*
10348  * Wait for all instances of the specified activity complete
10349  */
10350 int
10351 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited)
10352 {
10353 
10354 	return (spa_wait_common(pool, activity, B_FALSE, 0, waited));
10355 }
10356 
10357 sysevent_t *
10358 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name)
10359 {
10360 	sysevent_t *ev = NULL;
10361 #ifdef _KERNEL
10362 	nvlist_t *resource;
10363 
10364 	resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl);
10365 	if (resource) {
10366 		ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP);
10367 		ev->resource = resource;
10368 	}
10369 #else
10370 	(void) spa, (void) vd, (void) hist_nvl, (void) name;
10371 #endif
10372 	return (ev);
10373 }
10374 
10375 void
10376 spa_event_post(sysevent_t *ev)
10377 {
10378 #ifdef _KERNEL
10379 	if (ev) {
10380 		zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb);
10381 		kmem_free(ev, sizeof (*ev));
10382 	}
10383 #else
10384 	(void) ev;
10385 #endif
10386 }
10387 
10388 /*
10389  * Post a zevent corresponding to the given sysevent.   The 'name' must be one
10390  * of the event definitions in sys/sysevent/eventdefs.h.  The payload will be
10391  * filled in from the spa and (optionally) the vdev.  This doesn't do anything
10392  * in the userland libzpool, as we don't want consumers to misinterpret ztest
10393  * or zdb as real changes.
10394  */
10395 void
10396 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name)
10397 {
10398 	spa_event_post(spa_event_create(spa, vd, hist_nvl, name));
10399 }
10400 
10401 /* state manipulation functions */
10402 EXPORT_SYMBOL(spa_open);
10403 EXPORT_SYMBOL(spa_open_rewind);
10404 EXPORT_SYMBOL(spa_get_stats);
10405 EXPORT_SYMBOL(spa_create);
10406 EXPORT_SYMBOL(spa_import);
10407 EXPORT_SYMBOL(spa_tryimport);
10408 EXPORT_SYMBOL(spa_destroy);
10409 EXPORT_SYMBOL(spa_export);
10410 EXPORT_SYMBOL(spa_reset);
10411 EXPORT_SYMBOL(spa_async_request);
10412 EXPORT_SYMBOL(spa_async_suspend);
10413 EXPORT_SYMBOL(spa_async_resume);
10414 EXPORT_SYMBOL(spa_inject_addref);
10415 EXPORT_SYMBOL(spa_inject_delref);
10416 EXPORT_SYMBOL(spa_scan_stat_init);
10417 EXPORT_SYMBOL(spa_scan_get_stats);
10418 
10419 /* device manipulation */
10420 EXPORT_SYMBOL(spa_vdev_add);
10421 EXPORT_SYMBOL(spa_vdev_attach);
10422 EXPORT_SYMBOL(spa_vdev_detach);
10423 EXPORT_SYMBOL(spa_vdev_setpath);
10424 EXPORT_SYMBOL(spa_vdev_setfru);
10425 EXPORT_SYMBOL(spa_vdev_split_mirror);
10426 
10427 /* spare statech is global across all pools) */
10428 EXPORT_SYMBOL(spa_spare_add);
10429 EXPORT_SYMBOL(spa_spare_remove);
10430 EXPORT_SYMBOL(spa_spare_exists);
10431 EXPORT_SYMBOL(spa_spare_activate);
10432 
10433 /* L2ARC statech is global across all pools) */
10434 EXPORT_SYMBOL(spa_l2cache_add);
10435 EXPORT_SYMBOL(spa_l2cache_remove);
10436 EXPORT_SYMBOL(spa_l2cache_exists);
10437 EXPORT_SYMBOL(spa_l2cache_activate);
10438 EXPORT_SYMBOL(spa_l2cache_drop);
10439 
10440 /* scanning */
10441 EXPORT_SYMBOL(spa_scan);
10442 EXPORT_SYMBOL(spa_scan_stop);
10443 
10444 /* spa syncing */
10445 EXPORT_SYMBOL(spa_sync); /* only for DMU use */
10446 EXPORT_SYMBOL(spa_sync_allpools);
10447 
10448 /* properties */
10449 EXPORT_SYMBOL(spa_prop_set);
10450 EXPORT_SYMBOL(spa_prop_get);
10451 EXPORT_SYMBOL(spa_prop_clear_bootfs);
10452 
10453 /* asynchronous event notification */
10454 EXPORT_SYMBOL(spa_event_notify);
10455 
10456 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_pct, UINT, ZMOD_RW,
10457 	"Percentage of CPUs to run a metaslab preload taskq");
10458 
10459 /* BEGIN CSTYLED */
10460 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW,
10461 	"log2 fraction of arc that can be used by inflight I/Os when "
10462 	"verifying pool during import");
10463 /* END CSTYLED */
10464 
10465 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW,
10466 	"Set to traverse metadata on pool import");
10467 
10468 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW,
10469 	"Set to traverse data on pool import");
10470 
10471 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW,
10472 	"Print vdev tree to zfs_dbgmsg during pool import");
10473 
10474 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD,
10475 	"Percentage of CPUs to run an IO worker thread");
10476 
10477 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD,
10478 	"Number of threads per IO worker taskqueue");
10479 
10480 /* BEGIN CSTYLED */
10481 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW,
10482 	"Allow importing pool with up to this number of missing top-level "
10483 	"vdevs (in read-only mode)");
10484 /* END CSTYLED */
10485 
10486 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT,
10487 	ZMOD_RW, "Set the livelist condense zthr to pause");
10488 
10489 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT,
10490 	ZMOD_RW, "Set the livelist condense synctask to pause");
10491 
10492 /* BEGIN CSTYLED */
10493 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel,
10494 	INT, ZMOD_RW,
10495 	"Whether livelist condensing was canceled in the synctask");
10496 
10497 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel,
10498 	INT, ZMOD_RW,
10499 	"Whether livelist condensing was canceled in the zthr function");
10500 
10501 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT,
10502 	ZMOD_RW,
10503 	"Whether extra ALLOC blkptrs were added to a livelist entry while it "
10504 	"was being condensed");
10505 /* END CSTYLED */
10506 
10507 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_wr_iss_ncpus, UINT, ZMOD_RW,
10508 	"Number of CPUs to run write issue taskqs");
10509