xref: /freebsd/sys/contrib/openzfs/module/zfs/spa.c (revision 6ba2210ee039f2f12878c217bcf058e9c8b26b29)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
25  * Copyright (c) 2018, Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27  * Copyright 2013 Saso Kiselkov. All rights reserved.
28  * Copyright (c) 2014 Integros [integros.com]
29  * Copyright 2016 Toomas Soome <tsoome@me.com>
30  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
31  * Copyright 2018 Joyent, Inc.
32  * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
33  * Copyright 2017 Joyent, Inc.
34  * Copyright (c) 2017, Intel Corporation.
35  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
36  */
37 
38 /*
39  * SPA: Storage Pool Allocator
40  *
41  * This file contains all the routines used when modifying on-disk SPA state.
42  * This includes opening, importing, destroying, exporting a pool, and syncing a
43  * pool.
44  */
45 
46 #include <sys/zfs_context.h>
47 #include <sys/fm/fs/zfs.h>
48 #include <sys/spa_impl.h>
49 #include <sys/zio.h>
50 #include <sys/zio_checksum.h>
51 #include <sys/dmu.h>
52 #include <sys/dmu_tx.h>
53 #include <sys/zap.h>
54 #include <sys/zil.h>
55 #include <sys/ddt.h>
56 #include <sys/vdev_impl.h>
57 #include <sys/vdev_removal.h>
58 #include <sys/vdev_indirect_mapping.h>
59 #include <sys/vdev_indirect_births.h>
60 #include <sys/vdev_initialize.h>
61 #include <sys/vdev_rebuild.h>
62 #include <sys/vdev_trim.h>
63 #include <sys/vdev_disk.h>
64 #include <sys/vdev_draid.h>
65 #include <sys/metaslab.h>
66 #include <sys/metaslab_impl.h>
67 #include <sys/mmp.h>
68 #include <sys/uberblock_impl.h>
69 #include <sys/txg.h>
70 #include <sys/avl.h>
71 #include <sys/bpobj.h>
72 #include <sys/dmu_traverse.h>
73 #include <sys/dmu_objset.h>
74 #include <sys/unique.h>
75 #include <sys/dsl_pool.h>
76 #include <sys/dsl_dataset.h>
77 #include <sys/dsl_dir.h>
78 #include <sys/dsl_prop.h>
79 #include <sys/dsl_synctask.h>
80 #include <sys/fs/zfs.h>
81 #include <sys/arc.h>
82 #include <sys/callb.h>
83 #include <sys/systeminfo.h>
84 #include <sys/spa_boot.h>
85 #include <sys/zfs_ioctl.h>
86 #include <sys/dsl_scan.h>
87 #include <sys/zfeature.h>
88 #include <sys/dsl_destroy.h>
89 #include <sys/zvol.h>
90 
91 #ifdef	_KERNEL
92 #include <sys/fm/protocol.h>
93 #include <sys/fm/util.h>
94 #include <sys/callb.h>
95 #include <sys/zone.h>
96 #include <sys/vmsystm.h>
97 #endif	/* _KERNEL */
98 
99 #include "zfs_prop.h"
100 #include "zfs_comutil.h"
101 
102 /*
103  * The interval, in seconds, at which failed configuration cache file writes
104  * should be retried.
105  */
106 int zfs_ccw_retry_interval = 300;
107 
108 typedef enum zti_modes {
109 	ZTI_MODE_FIXED,			/* value is # of threads (min 1) */
110 	ZTI_MODE_BATCH,			/* cpu-intensive; value is ignored */
111 	ZTI_MODE_SCALE,			/* Taskqs scale with CPUs. */
112 	ZTI_MODE_NULL,			/* don't create a taskq */
113 	ZTI_NMODES
114 } zti_modes_t;
115 
116 #define	ZTI_P(n, q)	{ ZTI_MODE_FIXED, (n), (q) }
117 #define	ZTI_PCT(n)	{ ZTI_MODE_ONLINE_PERCENT, (n), 1 }
118 #define	ZTI_BATCH	{ ZTI_MODE_BATCH, 0, 1 }
119 #define	ZTI_SCALE	{ ZTI_MODE_SCALE, 0, 1 }
120 #define	ZTI_NULL	{ ZTI_MODE_NULL, 0, 0 }
121 
122 #define	ZTI_N(n)	ZTI_P(n, 1)
123 #define	ZTI_ONE		ZTI_N(1)
124 
125 typedef struct zio_taskq_info {
126 	zti_modes_t zti_mode;
127 	uint_t zti_value;
128 	uint_t zti_count;
129 } zio_taskq_info_t;
130 
131 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = {
132 	"iss", "iss_h", "int", "int_h"
133 };
134 
135 /*
136  * This table defines the taskq settings for each ZFS I/O type. When
137  * initializing a pool, we use this table to create an appropriately sized
138  * taskq. Some operations are low volume and therefore have a small, static
139  * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE
140  * macros. Other operations process a large amount of data; the ZTI_BATCH
141  * macro causes us to create a taskq oriented for throughput. Some operations
142  * are so high frequency and short-lived that the taskq itself can become a
143  * point of lock contention. The ZTI_P(#, #) macro indicates that we need an
144  * additional degree of parallelism specified by the number of threads per-
145  * taskq and the number of taskqs; when dispatching an event in this case, the
146  * particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH,
147  * but with number of taskqs also scaling with number of CPUs.
148  *
149  * The different taskq priorities are to handle the different contexts (issue
150  * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that
151  * need to be handled with minimum delay.
152  */
153 const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = {
154 	/* ISSUE	ISSUE_HIGH	INTR		INTR_HIGH */
155 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* NULL */
156 	{ ZTI_N(8),	ZTI_NULL,	ZTI_SCALE,	ZTI_NULL }, /* READ */
157 	{ ZTI_BATCH,	ZTI_N(5),	ZTI_SCALE,	ZTI_N(5) }, /* WRITE */
158 	{ ZTI_SCALE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* FREE */
159 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* CLAIM */
160 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* IOCTL */
161 	{ ZTI_N(4),	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* TRIM */
162 };
163 
164 static void spa_sync_version(void *arg, dmu_tx_t *tx);
165 static void spa_sync_props(void *arg, dmu_tx_t *tx);
166 static boolean_t spa_has_active_shared_spare(spa_t *spa);
167 static int spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport);
168 static void spa_vdev_resilver_done(spa_t *spa);
169 
170 uint_t		zio_taskq_batch_pct = 80;	/* 1 thread per cpu in pset */
171 uint_t		zio_taskq_batch_tpq;		/* threads per taskq */
172 boolean_t	zio_taskq_sysdc = B_TRUE;	/* use SDC scheduling class */
173 uint_t		zio_taskq_basedc = 80;		/* base duty cycle */
174 
175 boolean_t	spa_create_process = B_TRUE;	/* no process ==> no sysdc */
176 
177 /*
178  * Report any spa_load_verify errors found, but do not fail spa_load.
179  * This is used by zdb to analyze non-idle pools.
180  */
181 boolean_t	spa_load_verify_dryrun = B_FALSE;
182 
183 /*
184  * This (illegal) pool name is used when temporarily importing a spa_t in order
185  * to get the vdev stats associated with the imported devices.
186  */
187 #define	TRYIMPORT_NAME	"$import"
188 
189 /*
190  * For debugging purposes: print out vdev tree during pool import.
191  */
192 int		spa_load_print_vdev_tree = B_FALSE;
193 
194 /*
195  * A non-zero value for zfs_max_missing_tvds means that we allow importing
196  * pools with missing top-level vdevs. This is strictly intended for advanced
197  * pool recovery cases since missing data is almost inevitable. Pools with
198  * missing devices can only be imported read-only for safety reasons, and their
199  * fail-mode will be automatically set to "continue".
200  *
201  * With 1 missing vdev we should be able to import the pool and mount all
202  * datasets. User data that was not modified after the missing device has been
203  * added should be recoverable. This means that snapshots created prior to the
204  * addition of that device should be completely intact.
205  *
206  * With 2 missing vdevs, some datasets may fail to mount since there are
207  * dataset statistics that are stored as regular metadata. Some data might be
208  * recoverable if those vdevs were added recently.
209  *
210  * With 3 or more missing vdevs, the pool is severely damaged and MOS entries
211  * may be missing entirely. Chances of data recovery are very low. Note that
212  * there are also risks of performing an inadvertent rewind as we might be
213  * missing all the vdevs with the latest uberblocks.
214  */
215 unsigned long	zfs_max_missing_tvds = 0;
216 
217 /*
218  * The parameters below are similar to zfs_max_missing_tvds but are only
219  * intended for a preliminary open of the pool with an untrusted config which
220  * might be incomplete or out-dated.
221  *
222  * We are more tolerant for pools opened from a cachefile since we could have
223  * an out-dated cachefile where a device removal was not registered.
224  * We could have set the limit arbitrarily high but in the case where devices
225  * are really missing we would want to return the proper error codes; we chose
226  * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available
227  * and we get a chance to retrieve the trusted config.
228  */
229 uint64_t	zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1;
230 
231 /*
232  * In the case where config was assembled by scanning device paths (/dev/dsks
233  * by default) we are less tolerant since all the existing devices should have
234  * been detected and we want spa_load to return the right error codes.
235  */
236 uint64_t	zfs_max_missing_tvds_scan = 0;
237 
238 /*
239  * Debugging aid that pauses spa_sync() towards the end.
240  */
241 boolean_t	zfs_pause_spa_sync = B_FALSE;
242 
243 /*
244  * Variables to indicate the livelist condense zthr func should wait at certain
245  * points for the livelist to be removed - used to test condense/destroy races
246  */
247 int zfs_livelist_condense_zthr_pause = 0;
248 int zfs_livelist_condense_sync_pause = 0;
249 
250 /*
251  * Variables to track whether or not condense cancellation has been
252  * triggered in testing.
253  */
254 int zfs_livelist_condense_sync_cancel = 0;
255 int zfs_livelist_condense_zthr_cancel = 0;
256 
257 /*
258  * Variable to track whether or not extra ALLOC blkptrs were added to a
259  * livelist entry while it was being condensed (caused by the way we track
260  * remapped blkptrs in dbuf_remap_impl)
261  */
262 int zfs_livelist_condense_new_alloc = 0;
263 
264 /*
265  * ==========================================================================
266  * SPA properties routines
267  * ==========================================================================
268  */
269 
270 /*
271  * Add a (source=src, propname=propval) list to an nvlist.
272  */
273 static void
274 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval,
275     uint64_t intval, zprop_source_t src)
276 {
277 	const char *propname = zpool_prop_to_name(prop);
278 	nvlist_t *propval;
279 
280 	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
281 	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
282 
283 	if (strval != NULL)
284 		VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
285 	else
286 		VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0);
287 
288 	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
289 	nvlist_free(propval);
290 }
291 
292 /*
293  * Get property values from the spa configuration.
294  */
295 static void
296 spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
297 {
298 	vdev_t *rvd = spa->spa_root_vdev;
299 	dsl_pool_t *pool = spa->spa_dsl_pool;
300 	uint64_t size, alloc, cap, version;
301 	const zprop_source_t src = ZPROP_SRC_NONE;
302 	spa_config_dirent_t *dp;
303 	metaslab_class_t *mc = spa_normal_class(spa);
304 
305 	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
306 
307 	if (rvd != NULL) {
308 		alloc = metaslab_class_get_alloc(mc);
309 		alloc += metaslab_class_get_alloc(spa_special_class(spa));
310 		alloc += metaslab_class_get_alloc(spa_dedup_class(spa));
311 		alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa));
312 
313 		size = metaslab_class_get_space(mc);
314 		size += metaslab_class_get_space(spa_special_class(spa));
315 		size += metaslab_class_get_space(spa_dedup_class(spa));
316 		size += metaslab_class_get_space(spa_embedded_log_class(spa));
317 
318 		spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
319 		spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src);
320 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
321 		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
322 		    size - alloc, src);
323 		spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL,
324 		    spa->spa_checkpoint_info.sci_dspace, src);
325 
326 		spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
327 		    metaslab_class_fragmentation(mc), src);
328 		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
329 		    metaslab_class_expandable_space(mc), src);
330 		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
331 		    (spa_mode(spa) == SPA_MODE_READ), src);
332 
333 		cap = (size == 0) ? 0 : (alloc * 100 / size);
334 		spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src);
335 
336 		spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
337 		    ddt_get_pool_dedup_ratio(spa), src);
338 
339 		spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
340 		    rvd->vdev_state, src);
341 
342 		version = spa_version(spa);
343 		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) {
344 			spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
345 			    version, ZPROP_SRC_DEFAULT);
346 		} else {
347 			spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
348 			    version, ZPROP_SRC_LOCAL);
349 		}
350 		spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID,
351 		    NULL, spa_load_guid(spa), src);
352 	}
353 
354 	if (pool != NULL) {
355 		/*
356 		 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
357 		 * when opening pools before this version freedir will be NULL.
358 		 */
359 		if (pool->dp_free_dir != NULL) {
360 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
361 			    dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
362 			    src);
363 		} else {
364 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
365 			    NULL, 0, src);
366 		}
367 
368 		if (pool->dp_leak_dir != NULL) {
369 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
370 			    dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
371 			    src);
372 		} else {
373 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
374 			    NULL, 0, src);
375 		}
376 	}
377 
378 	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);
379 
380 	if (spa->spa_comment != NULL) {
381 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment,
382 		    0, ZPROP_SRC_LOCAL);
383 	}
384 
385 	if (spa->spa_compatibility != NULL) {
386 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY,
387 		    spa->spa_compatibility, 0, ZPROP_SRC_LOCAL);
388 	}
389 
390 	if (spa->spa_root != NULL)
391 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
392 		    0, ZPROP_SRC_LOCAL);
393 
394 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
395 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
396 		    MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
397 	} else {
398 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
399 		    SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
400 	}
401 
402 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) {
403 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
404 		    DNODE_MAX_SIZE, ZPROP_SRC_NONE);
405 	} else {
406 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
407 		    DNODE_MIN_SIZE, ZPROP_SRC_NONE);
408 	}
409 
410 	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
411 		if (dp->scd_path == NULL) {
412 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
413 			    "none", 0, ZPROP_SRC_LOCAL);
414 		} else if (strcmp(dp->scd_path, spa_config_path) != 0) {
415 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
416 			    dp->scd_path, 0, ZPROP_SRC_LOCAL);
417 		}
418 	}
419 }
420 
421 /*
422  * Get zpool property values.
423  */
424 int
425 spa_prop_get(spa_t *spa, nvlist_t **nvp)
426 {
427 	objset_t *mos = spa->spa_meta_objset;
428 	zap_cursor_t zc;
429 	zap_attribute_t za;
430 	dsl_pool_t *dp;
431 	int err;
432 
433 	err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP);
434 	if (err)
435 		return (err);
436 
437 	dp = spa_get_dsl(spa);
438 	dsl_pool_config_enter(dp, FTAG);
439 	mutex_enter(&spa->spa_props_lock);
440 
441 	/*
442 	 * Get properties from the spa config.
443 	 */
444 	spa_prop_get_config(spa, nvp);
445 
446 	/* If no pool property object, no more prop to get. */
447 	if (mos == NULL || spa->spa_pool_props_object == 0)
448 		goto out;
449 
450 	/*
451 	 * Get properties from the MOS pool property object.
452 	 */
453 	for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object);
454 	    (err = zap_cursor_retrieve(&zc, &za)) == 0;
455 	    zap_cursor_advance(&zc)) {
456 		uint64_t intval = 0;
457 		char *strval = NULL;
458 		zprop_source_t src = ZPROP_SRC_DEFAULT;
459 		zpool_prop_t prop;
460 
461 		if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL)
462 			continue;
463 
464 		switch (za.za_integer_length) {
465 		case 8:
466 			/* integer property */
467 			if (za.za_first_integer !=
468 			    zpool_prop_default_numeric(prop))
469 				src = ZPROP_SRC_LOCAL;
470 
471 			if (prop == ZPOOL_PROP_BOOTFS) {
472 				dsl_dataset_t *ds = NULL;
473 
474 				err = dsl_dataset_hold_obj(dp,
475 				    za.za_first_integer, FTAG, &ds);
476 				if (err != 0)
477 					break;
478 
479 				strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN,
480 				    KM_SLEEP);
481 				dsl_dataset_name(ds, strval);
482 				dsl_dataset_rele(ds, FTAG);
483 			} else {
484 				strval = NULL;
485 				intval = za.za_first_integer;
486 			}
487 
488 			spa_prop_add_list(*nvp, prop, strval, intval, src);
489 
490 			if (strval != NULL)
491 				kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN);
492 
493 			break;
494 
495 		case 1:
496 			/* string property */
497 			strval = kmem_alloc(za.za_num_integers, KM_SLEEP);
498 			err = zap_lookup(mos, spa->spa_pool_props_object,
499 			    za.za_name, 1, za.za_num_integers, strval);
500 			if (err) {
501 				kmem_free(strval, za.za_num_integers);
502 				break;
503 			}
504 			spa_prop_add_list(*nvp, prop, strval, 0, src);
505 			kmem_free(strval, za.za_num_integers);
506 			break;
507 
508 		default:
509 			break;
510 		}
511 	}
512 	zap_cursor_fini(&zc);
513 out:
514 	mutex_exit(&spa->spa_props_lock);
515 	dsl_pool_config_exit(dp, FTAG);
516 	if (err && err != ENOENT) {
517 		nvlist_free(*nvp);
518 		*nvp = NULL;
519 		return (err);
520 	}
521 
522 	return (0);
523 }
524 
525 /*
526  * Validate the given pool properties nvlist and modify the list
527  * for the property values to be set.
528  */
529 static int
530 spa_prop_validate(spa_t *spa, nvlist_t *props)
531 {
532 	nvpair_t *elem;
533 	int error = 0, reset_bootfs = 0;
534 	uint64_t objnum = 0;
535 	boolean_t has_feature = B_FALSE;
536 
537 	elem = NULL;
538 	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
539 		uint64_t intval;
540 		char *strval, *slash, *check, *fname;
541 		const char *propname = nvpair_name(elem);
542 		zpool_prop_t prop = zpool_name_to_prop(propname);
543 
544 		switch (prop) {
545 		case ZPOOL_PROP_INVAL:
546 			if (!zpool_prop_feature(propname)) {
547 				error = SET_ERROR(EINVAL);
548 				break;
549 			}
550 
551 			/*
552 			 * Sanitize the input.
553 			 */
554 			if (nvpair_type(elem) != DATA_TYPE_UINT64) {
555 				error = SET_ERROR(EINVAL);
556 				break;
557 			}
558 
559 			if (nvpair_value_uint64(elem, &intval) != 0) {
560 				error = SET_ERROR(EINVAL);
561 				break;
562 			}
563 
564 			if (intval != 0) {
565 				error = SET_ERROR(EINVAL);
566 				break;
567 			}
568 
569 			fname = strchr(propname, '@') + 1;
570 			if (zfeature_lookup_name(fname, NULL) != 0) {
571 				error = SET_ERROR(EINVAL);
572 				break;
573 			}
574 
575 			has_feature = B_TRUE;
576 			break;
577 
578 		case ZPOOL_PROP_VERSION:
579 			error = nvpair_value_uint64(elem, &intval);
580 			if (!error &&
581 			    (intval < spa_version(spa) ||
582 			    intval > SPA_VERSION_BEFORE_FEATURES ||
583 			    has_feature))
584 				error = SET_ERROR(EINVAL);
585 			break;
586 
587 		case ZPOOL_PROP_DELEGATION:
588 		case ZPOOL_PROP_AUTOREPLACE:
589 		case ZPOOL_PROP_LISTSNAPS:
590 		case ZPOOL_PROP_AUTOEXPAND:
591 		case ZPOOL_PROP_AUTOTRIM:
592 			error = nvpair_value_uint64(elem, &intval);
593 			if (!error && intval > 1)
594 				error = SET_ERROR(EINVAL);
595 			break;
596 
597 		case ZPOOL_PROP_MULTIHOST:
598 			error = nvpair_value_uint64(elem, &intval);
599 			if (!error && intval > 1)
600 				error = SET_ERROR(EINVAL);
601 
602 			if (!error) {
603 				uint32_t hostid = zone_get_hostid(NULL);
604 				if (hostid)
605 					spa->spa_hostid = hostid;
606 				else
607 					error = SET_ERROR(ENOTSUP);
608 			}
609 
610 			break;
611 
612 		case ZPOOL_PROP_BOOTFS:
613 			/*
614 			 * If the pool version is less than SPA_VERSION_BOOTFS,
615 			 * or the pool is still being created (version == 0),
616 			 * the bootfs property cannot be set.
617 			 */
618 			if (spa_version(spa) < SPA_VERSION_BOOTFS) {
619 				error = SET_ERROR(ENOTSUP);
620 				break;
621 			}
622 
623 			/*
624 			 * Make sure the vdev config is bootable
625 			 */
626 			if (!vdev_is_bootable(spa->spa_root_vdev)) {
627 				error = SET_ERROR(ENOTSUP);
628 				break;
629 			}
630 
631 			reset_bootfs = 1;
632 
633 			error = nvpair_value_string(elem, &strval);
634 
635 			if (!error) {
636 				objset_t *os;
637 
638 				if (strval == NULL || strval[0] == '\0') {
639 					objnum = zpool_prop_default_numeric(
640 					    ZPOOL_PROP_BOOTFS);
641 					break;
642 				}
643 
644 				error = dmu_objset_hold(strval, FTAG, &os);
645 				if (error != 0)
646 					break;
647 
648 				/* Must be ZPL. */
649 				if (dmu_objset_type(os) != DMU_OST_ZFS) {
650 					error = SET_ERROR(ENOTSUP);
651 				} else {
652 					objnum = dmu_objset_id(os);
653 				}
654 				dmu_objset_rele(os, FTAG);
655 			}
656 			break;
657 
658 		case ZPOOL_PROP_FAILUREMODE:
659 			error = nvpair_value_uint64(elem, &intval);
660 			if (!error && intval > ZIO_FAILURE_MODE_PANIC)
661 				error = SET_ERROR(EINVAL);
662 
663 			/*
664 			 * This is a special case which only occurs when
665 			 * the pool has completely failed. This allows
666 			 * the user to change the in-core failmode property
667 			 * without syncing it out to disk (I/Os might
668 			 * currently be blocked). We do this by returning
669 			 * EIO to the caller (spa_prop_set) to trick it
670 			 * into thinking we encountered a property validation
671 			 * error.
672 			 */
673 			if (!error && spa_suspended(spa)) {
674 				spa->spa_failmode = intval;
675 				error = SET_ERROR(EIO);
676 			}
677 			break;
678 
679 		case ZPOOL_PROP_CACHEFILE:
680 			if ((error = nvpair_value_string(elem, &strval)) != 0)
681 				break;
682 
683 			if (strval[0] == '\0')
684 				break;
685 
686 			if (strcmp(strval, "none") == 0)
687 				break;
688 
689 			if (strval[0] != '/') {
690 				error = SET_ERROR(EINVAL);
691 				break;
692 			}
693 
694 			slash = strrchr(strval, '/');
695 			ASSERT(slash != NULL);
696 
697 			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
698 			    strcmp(slash, "/..") == 0)
699 				error = SET_ERROR(EINVAL);
700 			break;
701 
702 		case ZPOOL_PROP_COMMENT:
703 			if ((error = nvpair_value_string(elem, &strval)) != 0)
704 				break;
705 			for (check = strval; *check != '\0'; check++) {
706 				if (!isprint(*check)) {
707 					error = SET_ERROR(EINVAL);
708 					break;
709 				}
710 			}
711 			if (strlen(strval) > ZPROP_MAX_COMMENT)
712 				error = SET_ERROR(E2BIG);
713 			break;
714 
715 		default:
716 			break;
717 		}
718 
719 		if (error)
720 			break;
721 	}
722 
723 	(void) nvlist_remove_all(props,
724 	    zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO));
725 
726 	if (!error && reset_bootfs) {
727 		error = nvlist_remove(props,
728 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING);
729 
730 		if (!error) {
731 			error = nvlist_add_uint64(props,
732 			    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum);
733 		}
734 	}
735 
736 	return (error);
737 }
738 
739 void
740 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync)
741 {
742 	char *cachefile;
743 	spa_config_dirent_t *dp;
744 
745 	if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
746 	    &cachefile) != 0)
747 		return;
748 
749 	dp = kmem_alloc(sizeof (spa_config_dirent_t),
750 	    KM_SLEEP);
751 
752 	if (cachefile[0] == '\0')
753 		dp->scd_path = spa_strdup(spa_config_path);
754 	else if (strcmp(cachefile, "none") == 0)
755 		dp->scd_path = NULL;
756 	else
757 		dp->scd_path = spa_strdup(cachefile);
758 
759 	list_insert_head(&spa->spa_config_list, dp);
760 	if (need_sync)
761 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
762 }
763 
764 int
765 spa_prop_set(spa_t *spa, nvlist_t *nvp)
766 {
767 	int error;
768 	nvpair_t *elem = NULL;
769 	boolean_t need_sync = B_FALSE;
770 
771 	if ((error = spa_prop_validate(spa, nvp)) != 0)
772 		return (error);
773 
774 	while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) {
775 		zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem));
776 
777 		if (prop == ZPOOL_PROP_CACHEFILE ||
778 		    prop == ZPOOL_PROP_ALTROOT ||
779 		    prop == ZPOOL_PROP_READONLY)
780 			continue;
781 
782 		if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) {
783 			uint64_t ver;
784 
785 			if (prop == ZPOOL_PROP_VERSION) {
786 				VERIFY(nvpair_value_uint64(elem, &ver) == 0);
787 			} else {
788 				ASSERT(zpool_prop_feature(nvpair_name(elem)));
789 				ver = SPA_VERSION_FEATURES;
790 				need_sync = B_TRUE;
791 			}
792 
793 			/* Save time if the version is already set. */
794 			if (ver == spa_version(spa))
795 				continue;
796 
797 			/*
798 			 * In addition to the pool directory object, we might
799 			 * create the pool properties object, the features for
800 			 * read object, the features for write object, or the
801 			 * feature descriptions object.
802 			 */
803 			error = dsl_sync_task(spa->spa_name, NULL,
804 			    spa_sync_version, &ver,
805 			    6, ZFS_SPACE_CHECK_RESERVED);
806 			if (error)
807 				return (error);
808 			continue;
809 		}
810 
811 		need_sync = B_TRUE;
812 		break;
813 	}
814 
815 	if (need_sync) {
816 		return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
817 		    nvp, 6, ZFS_SPACE_CHECK_RESERVED));
818 	}
819 
820 	return (0);
821 }
822 
823 /*
824  * If the bootfs property value is dsobj, clear it.
825  */
826 void
827 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx)
828 {
829 	if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) {
830 		VERIFY(zap_remove(spa->spa_meta_objset,
831 		    spa->spa_pool_props_object,
832 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0);
833 		spa->spa_bootfs = 0;
834 	}
835 }
836 
837 /*ARGSUSED*/
838 static int
839 spa_change_guid_check(void *arg, dmu_tx_t *tx)
840 {
841 	uint64_t *newguid __maybe_unused = arg;
842 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
843 	vdev_t *rvd = spa->spa_root_vdev;
844 	uint64_t vdev_state;
845 
846 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
847 		int error = (spa_has_checkpoint(spa)) ?
848 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
849 		return (SET_ERROR(error));
850 	}
851 
852 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
853 	vdev_state = rvd->vdev_state;
854 	spa_config_exit(spa, SCL_STATE, FTAG);
855 
856 	if (vdev_state != VDEV_STATE_HEALTHY)
857 		return (SET_ERROR(ENXIO));
858 
859 	ASSERT3U(spa_guid(spa), !=, *newguid);
860 
861 	return (0);
862 }
863 
864 static void
865 spa_change_guid_sync(void *arg, dmu_tx_t *tx)
866 {
867 	uint64_t *newguid = arg;
868 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
869 	uint64_t oldguid;
870 	vdev_t *rvd = spa->spa_root_vdev;
871 
872 	oldguid = spa_guid(spa);
873 
874 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
875 	rvd->vdev_guid = *newguid;
876 	rvd->vdev_guid_sum += (*newguid - oldguid);
877 	vdev_config_dirty(rvd);
878 	spa_config_exit(spa, SCL_STATE, FTAG);
879 
880 	spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu",
881 	    (u_longlong_t)oldguid, (u_longlong_t)*newguid);
882 }
883 
884 /*
885  * Change the GUID for the pool.  This is done so that we can later
886  * re-import a pool built from a clone of our own vdevs.  We will modify
887  * the root vdev's guid, our own pool guid, and then mark all of our
888  * vdevs dirty.  Note that we must make sure that all our vdevs are
889  * online when we do this, or else any vdevs that weren't present
890  * would be orphaned from our pool.  We are also going to issue a
891  * sysevent to update any watchers.
892  */
893 int
894 spa_change_guid(spa_t *spa)
895 {
896 	int error;
897 	uint64_t guid;
898 
899 	mutex_enter(&spa->spa_vdev_top_lock);
900 	mutex_enter(&spa_namespace_lock);
901 	guid = spa_generate_guid(NULL);
902 
903 	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
904 	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);
905 
906 	if (error == 0) {
907 		spa_write_cachefile(spa, B_FALSE, B_TRUE);
908 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID);
909 	}
910 
911 	mutex_exit(&spa_namespace_lock);
912 	mutex_exit(&spa->spa_vdev_top_lock);
913 
914 	return (error);
915 }
916 
917 /*
918  * ==========================================================================
919  * SPA state manipulation (open/create/destroy/import/export)
920  * ==========================================================================
921  */
922 
923 static int
924 spa_error_entry_compare(const void *a, const void *b)
925 {
926 	const spa_error_entry_t *sa = (const spa_error_entry_t *)a;
927 	const spa_error_entry_t *sb = (const spa_error_entry_t *)b;
928 	int ret;
929 
930 	ret = memcmp(&sa->se_bookmark, &sb->se_bookmark,
931 	    sizeof (zbookmark_phys_t));
932 
933 	return (TREE_ISIGN(ret));
934 }
935 
936 /*
937  * Utility function which retrieves copies of the current logs and
938  * re-initializes them in the process.
939  */
940 void
941 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub)
942 {
943 	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));
944 
945 	bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t));
946 	bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t));
947 
948 	avl_create(&spa->spa_errlist_scrub,
949 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
950 	    offsetof(spa_error_entry_t, se_avl));
951 	avl_create(&spa->spa_errlist_last,
952 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
953 	    offsetof(spa_error_entry_t, se_avl));
954 }
955 
956 static void
957 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
958 {
959 	const zio_taskq_info_t *ztip = &zio_taskqs[t][q];
960 	enum zti_modes mode = ztip->zti_mode;
961 	uint_t value = ztip->zti_value;
962 	uint_t count = ztip->zti_count;
963 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
964 	uint_t cpus, flags = TASKQ_DYNAMIC;
965 	boolean_t batch = B_FALSE;
966 
967 	switch (mode) {
968 	case ZTI_MODE_FIXED:
969 		ASSERT3U(value, >, 0);
970 		break;
971 
972 	case ZTI_MODE_BATCH:
973 		batch = B_TRUE;
974 		flags |= TASKQ_THREADS_CPU_PCT;
975 		value = MIN(zio_taskq_batch_pct, 100);
976 		break;
977 
978 	case ZTI_MODE_SCALE:
979 		flags |= TASKQ_THREADS_CPU_PCT;
980 		/*
981 		 * We want more taskqs to reduce lock contention, but we want
982 		 * less for better request ordering and CPU utilization.
983 		 */
984 		cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100);
985 		if (zio_taskq_batch_tpq > 0) {
986 			count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) /
987 			    zio_taskq_batch_tpq);
988 		} else {
989 			/*
990 			 * Prefer 6 threads per taskq, but no more taskqs
991 			 * than threads in them on large systems. For 80%:
992 			 *
993 			 *                 taskq   taskq   total
994 			 * cpus    taskqs  percent threads threads
995 			 * ------- ------- ------- ------- -------
996 			 * 1       1       80%     1       1
997 			 * 2       1       80%     1       1
998 			 * 4       1       80%     3       3
999 			 * 8       2       40%     3       6
1000 			 * 16      3       27%     4       12
1001 			 * 32      5       16%     5       25
1002 			 * 64      7       11%     7       49
1003 			 * 128     10      8%      10      100
1004 			 * 256     14      6%      15      210
1005 			 */
1006 			count = 1 + cpus / 6;
1007 			while (count * count > cpus)
1008 				count--;
1009 		}
1010 		/* Limit each taskq within 100% to not trigger assertion. */
1011 		count = MAX(count, (zio_taskq_batch_pct + 99) / 100);
1012 		value = (zio_taskq_batch_pct + count / 2) / count;
1013 		break;
1014 
1015 	case ZTI_MODE_NULL:
1016 		tqs->stqs_count = 0;
1017 		tqs->stqs_taskq = NULL;
1018 		return;
1019 
1020 	default:
1021 		panic("unrecognized mode for %s_%s taskq (%u:%u) in "
1022 		    "spa_activate()",
1023 		    zio_type_name[t], zio_taskq_types[q], mode, value);
1024 		break;
1025 	}
1026 
1027 	ASSERT3U(count, >, 0);
1028 	tqs->stqs_count = count;
1029 	tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP);
1030 
1031 	for (uint_t i = 0; i < count; i++) {
1032 		taskq_t *tq;
1033 		char name[32];
1034 
1035 		if (count > 1)
1036 			(void) snprintf(name, sizeof (name), "%s_%s_%u",
1037 			    zio_type_name[t], zio_taskq_types[q], i);
1038 		else
1039 			(void) snprintf(name, sizeof (name), "%s_%s",
1040 			    zio_type_name[t], zio_taskq_types[q]);
1041 
1042 		if (zio_taskq_sysdc && spa->spa_proc != &p0) {
1043 			if (batch)
1044 				flags |= TASKQ_DC_BATCH;
1045 
1046 			tq = taskq_create_sysdc(name, value, 50, INT_MAX,
1047 			    spa->spa_proc, zio_taskq_basedc, flags);
1048 		} else {
1049 			pri_t pri = maxclsyspri;
1050 			/*
1051 			 * The write issue taskq can be extremely CPU
1052 			 * intensive.  Run it at slightly less important
1053 			 * priority than the other taskqs.
1054 			 *
1055 			 * Under Linux and FreeBSD this means incrementing
1056 			 * the priority value as opposed to platforms like
1057 			 * illumos where it should be decremented.
1058 			 *
1059 			 * On FreeBSD, if priorities divided by four (RQ_PPQ)
1060 			 * are equal then a difference between them is
1061 			 * insignificant.
1062 			 */
1063 			if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) {
1064 #if defined(__linux__)
1065 				pri++;
1066 #elif defined(__FreeBSD__)
1067 				pri += 4;
1068 #else
1069 #error "unknown OS"
1070 #endif
1071 			}
1072 			tq = taskq_create_proc(name, value, pri, 50,
1073 			    INT_MAX, spa->spa_proc, flags);
1074 		}
1075 
1076 		tqs->stqs_taskq[i] = tq;
1077 	}
1078 }
1079 
1080 static void
1081 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
1082 {
1083 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1084 
1085 	if (tqs->stqs_taskq == NULL) {
1086 		ASSERT3U(tqs->stqs_count, ==, 0);
1087 		return;
1088 	}
1089 
1090 	for (uint_t i = 0; i < tqs->stqs_count; i++) {
1091 		ASSERT3P(tqs->stqs_taskq[i], !=, NULL);
1092 		taskq_destroy(tqs->stqs_taskq[i]);
1093 	}
1094 
1095 	kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *));
1096 	tqs->stqs_taskq = NULL;
1097 }
1098 
1099 /*
1100  * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority.
1101  * Note that a type may have multiple discrete taskqs to avoid lock contention
1102  * on the taskq itself. In that case we choose which taskq at random by using
1103  * the low bits of gethrtime().
1104  */
1105 void
1106 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1107     task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent)
1108 {
1109 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1110 	taskq_t *tq;
1111 
1112 	ASSERT3P(tqs->stqs_taskq, !=, NULL);
1113 	ASSERT3U(tqs->stqs_count, !=, 0);
1114 
1115 	if (tqs->stqs_count == 1) {
1116 		tq = tqs->stqs_taskq[0];
1117 	} else {
1118 		tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count];
1119 	}
1120 
1121 	taskq_dispatch_ent(tq, func, arg, flags, ent);
1122 }
1123 
1124 /*
1125  * Same as spa_taskq_dispatch_ent() but block on the task until completion.
1126  */
1127 void
1128 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1129     task_func_t *func, void *arg, uint_t flags)
1130 {
1131 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1132 	taskq_t *tq;
1133 	taskqid_t id;
1134 
1135 	ASSERT3P(tqs->stqs_taskq, !=, NULL);
1136 	ASSERT3U(tqs->stqs_count, !=, 0);
1137 
1138 	if (tqs->stqs_count == 1) {
1139 		tq = tqs->stqs_taskq[0];
1140 	} else {
1141 		tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count];
1142 	}
1143 
1144 	id = taskq_dispatch(tq, func, arg, flags);
1145 	if (id)
1146 		taskq_wait_id(tq, id);
1147 }
1148 
1149 static void
1150 spa_create_zio_taskqs(spa_t *spa)
1151 {
1152 	for (int t = 0; t < ZIO_TYPES; t++) {
1153 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1154 			spa_taskqs_init(spa, t, q);
1155 		}
1156 	}
1157 }
1158 
1159 /*
1160  * Disabled until spa_thread() can be adapted for Linux.
1161  */
1162 #undef HAVE_SPA_THREAD
1163 
1164 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD)
1165 static void
1166 spa_thread(void *arg)
1167 {
1168 	psetid_t zio_taskq_psrset_bind = PS_NONE;
1169 	callb_cpr_t cprinfo;
1170 
1171 	spa_t *spa = arg;
1172 	user_t *pu = PTOU(curproc);
1173 
1174 	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
1175 	    spa->spa_name);
1176 
1177 	ASSERT(curproc != &p0);
1178 	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
1179 	    "zpool-%s", spa->spa_name);
1180 	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
1181 
1182 	/* bind this thread to the requested psrset */
1183 	if (zio_taskq_psrset_bind != PS_NONE) {
1184 		pool_lock();
1185 		mutex_enter(&cpu_lock);
1186 		mutex_enter(&pidlock);
1187 		mutex_enter(&curproc->p_lock);
1188 
1189 		if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind,
1190 		    0, NULL, NULL) == 0)  {
1191 			curthread->t_bind_pset = zio_taskq_psrset_bind;
1192 		} else {
1193 			cmn_err(CE_WARN,
1194 			    "Couldn't bind process for zfs pool \"%s\" to "
1195 			    "pset %d\n", spa->spa_name, zio_taskq_psrset_bind);
1196 		}
1197 
1198 		mutex_exit(&curproc->p_lock);
1199 		mutex_exit(&pidlock);
1200 		mutex_exit(&cpu_lock);
1201 		pool_unlock();
1202 	}
1203 
1204 	if (zio_taskq_sysdc) {
1205 		sysdc_thread_enter(curthread, 100, 0);
1206 	}
1207 
1208 	spa->spa_proc = curproc;
1209 	spa->spa_did = curthread->t_did;
1210 
1211 	spa_create_zio_taskqs(spa);
1212 
1213 	mutex_enter(&spa->spa_proc_lock);
1214 	ASSERT(spa->spa_proc_state == SPA_PROC_CREATED);
1215 
1216 	spa->spa_proc_state = SPA_PROC_ACTIVE;
1217 	cv_broadcast(&spa->spa_proc_cv);
1218 
1219 	CALLB_CPR_SAFE_BEGIN(&cprinfo);
1220 	while (spa->spa_proc_state == SPA_PROC_ACTIVE)
1221 		cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1222 	CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock);
1223 
1224 	ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE);
1225 	spa->spa_proc_state = SPA_PROC_GONE;
1226 	spa->spa_proc = &p0;
1227 	cv_broadcast(&spa->spa_proc_cv);
1228 	CALLB_CPR_EXIT(&cprinfo);	/* drops spa_proc_lock */
1229 
1230 	mutex_enter(&curproc->p_lock);
1231 	lwp_exit();
1232 }
1233 #endif
1234 
1235 /*
1236  * Activate an uninitialized pool.
1237  */
1238 static void
1239 spa_activate(spa_t *spa, spa_mode_t mode)
1240 {
1241 	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
1242 
1243 	spa->spa_state = POOL_STATE_ACTIVE;
1244 	spa->spa_mode = mode;
1245 
1246 	spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops);
1247 	spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops);
1248 	spa->spa_embedded_log_class =
1249 	    metaslab_class_create(spa, zfs_metaslab_ops);
1250 	spa->spa_special_class = metaslab_class_create(spa, zfs_metaslab_ops);
1251 	spa->spa_dedup_class = metaslab_class_create(spa, zfs_metaslab_ops);
1252 
1253 	/* Try to create a covering process */
1254 	mutex_enter(&spa->spa_proc_lock);
1255 	ASSERT(spa->spa_proc_state == SPA_PROC_NONE);
1256 	ASSERT(spa->spa_proc == &p0);
1257 	spa->spa_did = 0;
1258 
1259 #ifdef HAVE_SPA_THREAD
1260 	/* Only create a process if we're going to be around a while. */
1261 	if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) {
1262 		if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri,
1263 		    NULL, 0) == 0) {
1264 			spa->spa_proc_state = SPA_PROC_CREATED;
1265 			while (spa->spa_proc_state == SPA_PROC_CREATED) {
1266 				cv_wait(&spa->spa_proc_cv,
1267 				    &spa->spa_proc_lock);
1268 			}
1269 			ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1270 			ASSERT(spa->spa_proc != &p0);
1271 			ASSERT(spa->spa_did != 0);
1272 		} else {
1273 #ifdef _KERNEL
1274 			cmn_err(CE_WARN,
1275 			    "Couldn't create process for zfs pool \"%s\"\n",
1276 			    spa->spa_name);
1277 #endif
1278 		}
1279 	}
1280 #endif /* HAVE_SPA_THREAD */
1281 	mutex_exit(&spa->spa_proc_lock);
1282 
1283 	/* If we didn't create a process, we need to create our taskqs. */
1284 	if (spa->spa_proc == &p0) {
1285 		spa_create_zio_taskqs(spa);
1286 	}
1287 
1288 	for (size_t i = 0; i < TXG_SIZE; i++) {
1289 		spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL,
1290 		    ZIO_FLAG_CANFAIL);
1291 	}
1292 
1293 	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
1294 	    offsetof(vdev_t, vdev_config_dirty_node));
1295 	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
1296 	    offsetof(objset_t, os_evicting_node));
1297 	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
1298 	    offsetof(vdev_t, vdev_state_dirty_node));
1299 
1300 	txg_list_create(&spa->spa_vdev_txg_list, spa,
1301 	    offsetof(struct vdev, vdev_txg_node));
1302 
1303 	avl_create(&spa->spa_errlist_scrub,
1304 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1305 	    offsetof(spa_error_entry_t, se_avl));
1306 	avl_create(&spa->spa_errlist_last,
1307 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1308 	    offsetof(spa_error_entry_t, se_avl));
1309 
1310 	spa_keystore_init(&spa->spa_keystore);
1311 
1312 	/*
1313 	 * This taskq is used to perform zvol-minor-related tasks
1314 	 * asynchronously. This has several advantages, including easy
1315 	 * resolution of various deadlocks.
1316 	 *
1317 	 * The taskq must be single threaded to ensure tasks are always
1318 	 * processed in the order in which they were dispatched.
1319 	 *
1320 	 * A taskq per pool allows one to keep the pools independent.
1321 	 * This way if one pool is suspended, it will not impact another.
1322 	 *
1323 	 * The preferred location to dispatch a zvol minor task is a sync
1324 	 * task. In this context, there is easy access to the spa_t and minimal
1325 	 * error handling is required because the sync task must succeed.
1326 	 */
1327 	spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri,
1328 	    1, INT_MAX, 0);
1329 
1330 	/*
1331 	 * Taskq dedicated to prefetcher threads: this is used to prevent the
1332 	 * pool traverse code from monopolizing the global (and limited)
1333 	 * system_taskq by inappropriately scheduling long running tasks on it.
1334 	 */
1335 	spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100,
1336 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1337 
1338 	/*
1339 	 * The taskq to upgrade datasets in this pool. Currently used by
1340 	 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA.
1341 	 */
1342 	spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100,
1343 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1344 }
1345 
1346 /*
1347  * Opposite of spa_activate().
1348  */
1349 static void
1350 spa_deactivate(spa_t *spa)
1351 {
1352 	ASSERT(spa->spa_sync_on == B_FALSE);
1353 	ASSERT(spa->spa_dsl_pool == NULL);
1354 	ASSERT(spa->spa_root_vdev == NULL);
1355 	ASSERT(spa->spa_async_zio_root == NULL);
1356 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
1357 
1358 	spa_evicting_os_wait(spa);
1359 
1360 	if (spa->spa_zvol_taskq) {
1361 		taskq_destroy(spa->spa_zvol_taskq);
1362 		spa->spa_zvol_taskq = NULL;
1363 	}
1364 
1365 	if (spa->spa_prefetch_taskq) {
1366 		taskq_destroy(spa->spa_prefetch_taskq);
1367 		spa->spa_prefetch_taskq = NULL;
1368 	}
1369 
1370 	if (spa->spa_upgrade_taskq) {
1371 		taskq_destroy(spa->spa_upgrade_taskq);
1372 		spa->spa_upgrade_taskq = NULL;
1373 	}
1374 
1375 	txg_list_destroy(&spa->spa_vdev_txg_list);
1376 
1377 	list_destroy(&spa->spa_config_dirty_list);
1378 	list_destroy(&spa->spa_evicting_os_list);
1379 	list_destroy(&spa->spa_state_dirty_list);
1380 
1381 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
1382 
1383 	for (int t = 0; t < ZIO_TYPES; t++) {
1384 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1385 			spa_taskqs_fini(spa, t, q);
1386 		}
1387 	}
1388 
1389 	for (size_t i = 0; i < TXG_SIZE; i++) {
1390 		ASSERT3P(spa->spa_txg_zio[i], !=, NULL);
1391 		VERIFY0(zio_wait(spa->spa_txg_zio[i]));
1392 		spa->spa_txg_zio[i] = NULL;
1393 	}
1394 
1395 	metaslab_class_destroy(spa->spa_normal_class);
1396 	spa->spa_normal_class = NULL;
1397 
1398 	metaslab_class_destroy(spa->spa_log_class);
1399 	spa->spa_log_class = NULL;
1400 
1401 	metaslab_class_destroy(spa->spa_embedded_log_class);
1402 	spa->spa_embedded_log_class = NULL;
1403 
1404 	metaslab_class_destroy(spa->spa_special_class);
1405 	spa->spa_special_class = NULL;
1406 
1407 	metaslab_class_destroy(spa->spa_dedup_class);
1408 	spa->spa_dedup_class = NULL;
1409 
1410 	/*
1411 	 * If this was part of an import or the open otherwise failed, we may
1412 	 * still have errors left in the queues.  Empty them just in case.
1413 	 */
1414 	spa_errlog_drain(spa);
1415 	avl_destroy(&spa->spa_errlist_scrub);
1416 	avl_destroy(&spa->spa_errlist_last);
1417 
1418 	spa_keystore_fini(&spa->spa_keystore);
1419 
1420 	spa->spa_state = POOL_STATE_UNINITIALIZED;
1421 
1422 	mutex_enter(&spa->spa_proc_lock);
1423 	if (spa->spa_proc_state != SPA_PROC_NONE) {
1424 		ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1425 		spa->spa_proc_state = SPA_PROC_DEACTIVATE;
1426 		cv_broadcast(&spa->spa_proc_cv);
1427 		while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) {
1428 			ASSERT(spa->spa_proc != &p0);
1429 			cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1430 		}
1431 		ASSERT(spa->spa_proc_state == SPA_PROC_GONE);
1432 		spa->spa_proc_state = SPA_PROC_NONE;
1433 	}
1434 	ASSERT(spa->spa_proc == &p0);
1435 	mutex_exit(&spa->spa_proc_lock);
1436 
1437 	/*
1438 	 * We want to make sure spa_thread() has actually exited the ZFS
1439 	 * module, so that the module can't be unloaded out from underneath
1440 	 * it.
1441 	 */
1442 	if (spa->spa_did != 0) {
1443 		thread_join(spa->spa_did);
1444 		spa->spa_did = 0;
1445 	}
1446 }
1447 
1448 /*
1449  * Verify a pool configuration, and construct the vdev tree appropriately.  This
1450  * will create all the necessary vdevs in the appropriate layout, with each vdev
1451  * in the CLOSED state.  This will prep the pool before open/creation/import.
1452  * All vdev validation is done by the vdev_alloc() routine.
1453  */
1454 int
1455 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent,
1456     uint_t id, int atype)
1457 {
1458 	nvlist_t **child;
1459 	uint_t children;
1460 	int error;
1461 
1462 	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
1463 		return (error);
1464 
1465 	if ((*vdp)->vdev_ops->vdev_op_leaf)
1466 		return (0);
1467 
1468 	error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1469 	    &child, &children);
1470 
1471 	if (error == ENOENT)
1472 		return (0);
1473 
1474 	if (error) {
1475 		vdev_free(*vdp);
1476 		*vdp = NULL;
1477 		return (SET_ERROR(EINVAL));
1478 	}
1479 
1480 	for (int c = 0; c < children; c++) {
1481 		vdev_t *vd;
1482 		if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
1483 		    atype)) != 0) {
1484 			vdev_free(*vdp);
1485 			*vdp = NULL;
1486 			return (error);
1487 		}
1488 	}
1489 
1490 	ASSERT(*vdp != NULL);
1491 
1492 	return (0);
1493 }
1494 
1495 static boolean_t
1496 spa_should_flush_logs_on_unload(spa_t *spa)
1497 {
1498 	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))
1499 		return (B_FALSE);
1500 
1501 	if (!spa_writeable(spa))
1502 		return (B_FALSE);
1503 
1504 	if (!spa->spa_sync_on)
1505 		return (B_FALSE);
1506 
1507 	if (spa_state(spa) != POOL_STATE_EXPORTED)
1508 		return (B_FALSE);
1509 
1510 	if (zfs_keep_log_spacemaps_at_export)
1511 		return (B_FALSE);
1512 
1513 	return (B_TRUE);
1514 }
1515 
1516 /*
1517  * Opens a transaction that will set the flag that will instruct
1518  * spa_sync to attempt to flush all the metaslabs for that txg.
1519  */
1520 static void
1521 spa_unload_log_sm_flush_all(spa_t *spa)
1522 {
1523 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
1524 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
1525 
1526 	ASSERT3U(spa->spa_log_flushall_txg, ==, 0);
1527 	spa->spa_log_flushall_txg = dmu_tx_get_txg(tx);
1528 
1529 	dmu_tx_commit(tx);
1530 	txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg);
1531 }
1532 
1533 static void
1534 spa_unload_log_sm_metadata(spa_t *spa)
1535 {
1536 	void *cookie = NULL;
1537 	spa_log_sm_t *sls;
1538 	while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg,
1539 	    &cookie)) != NULL) {
1540 		VERIFY0(sls->sls_mscount);
1541 		kmem_free(sls, sizeof (spa_log_sm_t));
1542 	}
1543 
1544 	for (log_summary_entry_t *e = list_head(&spa->spa_log_summary);
1545 	    e != NULL; e = list_head(&spa->spa_log_summary)) {
1546 		VERIFY0(e->lse_mscount);
1547 		list_remove(&spa->spa_log_summary, e);
1548 		kmem_free(e, sizeof (log_summary_entry_t));
1549 	}
1550 
1551 	spa->spa_unflushed_stats.sus_nblocks = 0;
1552 	spa->spa_unflushed_stats.sus_memused = 0;
1553 	spa->spa_unflushed_stats.sus_blocklimit = 0;
1554 }
1555 
1556 static void
1557 spa_destroy_aux_threads(spa_t *spa)
1558 {
1559 	if (spa->spa_condense_zthr != NULL) {
1560 		zthr_destroy(spa->spa_condense_zthr);
1561 		spa->spa_condense_zthr = NULL;
1562 	}
1563 	if (spa->spa_checkpoint_discard_zthr != NULL) {
1564 		zthr_destroy(spa->spa_checkpoint_discard_zthr);
1565 		spa->spa_checkpoint_discard_zthr = NULL;
1566 	}
1567 	if (spa->spa_livelist_delete_zthr != NULL) {
1568 		zthr_destroy(spa->spa_livelist_delete_zthr);
1569 		spa->spa_livelist_delete_zthr = NULL;
1570 	}
1571 	if (spa->spa_livelist_condense_zthr != NULL) {
1572 		zthr_destroy(spa->spa_livelist_condense_zthr);
1573 		spa->spa_livelist_condense_zthr = NULL;
1574 	}
1575 }
1576 
1577 /*
1578  * Opposite of spa_load().
1579  */
1580 static void
1581 spa_unload(spa_t *spa)
1582 {
1583 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1584 	ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED);
1585 
1586 	spa_import_progress_remove(spa_guid(spa));
1587 	spa_load_note(spa, "UNLOADING");
1588 
1589 	spa_wake_waiters(spa);
1590 
1591 	/*
1592 	 * If the log space map feature is enabled and the pool is getting
1593 	 * exported (but not destroyed), we want to spend some time flushing
1594 	 * as many metaslabs as we can in an attempt to destroy log space
1595 	 * maps and save import time.
1596 	 */
1597 	if (spa_should_flush_logs_on_unload(spa))
1598 		spa_unload_log_sm_flush_all(spa);
1599 
1600 	/*
1601 	 * Stop async tasks.
1602 	 */
1603 	spa_async_suspend(spa);
1604 
1605 	if (spa->spa_root_vdev) {
1606 		vdev_t *root_vdev = spa->spa_root_vdev;
1607 		vdev_initialize_stop_all(root_vdev, VDEV_INITIALIZE_ACTIVE);
1608 		vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE);
1609 		vdev_autotrim_stop_all(spa);
1610 		vdev_rebuild_stop_all(spa);
1611 	}
1612 
1613 	/*
1614 	 * Stop syncing.
1615 	 */
1616 	if (spa->spa_sync_on) {
1617 		txg_sync_stop(spa->spa_dsl_pool);
1618 		spa->spa_sync_on = B_FALSE;
1619 	}
1620 
1621 	/*
1622 	 * This ensures that there is no async metaslab prefetching
1623 	 * while we attempt to unload the spa.
1624 	 */
1625 	if (spa->spa_root_vdev != NULL) {
1626 		for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) {
1627 			vdev_t *vc = spa->spa_root_vdev->vdev_child[c];
1628 			if (vc->vdev_mg != NULL)
1629 				taskq_wait(vc->vdev_mg->mg_taskq);
1630 		}
1631 	}
1632 
1633 	if (spa->spa_mmp.mmp_thread)
1634 		mmp_thread_stop(spa);
1635 
1636 	/*
1637 	 * Wait for any outstanding async I/O to complete.
1638 	 */
1639 	if (spa->spa_async_zio_root != NULL) {
1640 		for (int i = 0; i < max_ncpus; i++)
1641 			(void) zio_wait(spa->spa_async_zio_root[i]);
1642 		kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
1643 		spa->spa_async_zio_root = NULL;
1644 	}
1645 
1646 	if (spa->spa_vdev_removal != NULL) {
1647 		spa_vdev_removal_destroy(spa->spa_vdev_removal);
1648 		spa->spa_vdev_removal = NULL;
1649 	}
1650 
1651 	spa_destroy_aux_threads(spa);
1652 
1653 	spa_condense_fini(spa);
1654 
1655 	bpobj_close(&spa->spa_deferred_bpobj);
1656 
1657 	spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);
1658 
1659 	/*
1660 	 * Close all vdevs.
1661 	 */
1662 	if (spa->spa_root_vdev)
1663 		vdev_free(spa->spa_root_vdev);
1664 	ASSERT(spa->spa_root_vdev == NULL);
1665 
1666 	/*
1667 	 * Close the dsl pool.
1668 	 */
1669 	if (spa->spa_dsl_pool) {
1670 		dsl_pool_close(spa->spa_dsl_pool);
1671 		spa->spa_dsl_pool = NULL;
1672 		spa->spa_meta_objset = NULL;
1673 	}
1674 
1675 	ddt_unload(spa);
1676 	spa_unload_log_sm_metadata(spa);
1677 
1678 	/*
1679 	 * Drop and purge level 2 cache
1680 	 */
1681 	spa_l2cache_drop(spa);
1682 
1683 	for (int i = 0; i < spa->spa_spares.sav_count; i++)
1684 		vdev_free(spa->spa_spares.sav_vdevs[i]);
1685 	if (spa->spa_spares.sav_vdevs) {
1686 		kmem_free(spa->spa_spares.sav_vdevs,
1687 		    spa->spa_spares.sav_count * sizeof (void *));
1688 		spa->spa_spares.sav_vdevs = NULL;
1689 	}
1690 	if (spa->spa_spares.sav_config) {
1691 		nvlist_free(spa->spa_spares.sav_config);
1692 		spa->spa_spares.sav_config = NULL;
1693 	}
1694 	spa->spa_spares.sav_count = 0;
1695 
1696 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1697 		vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]);
1698 		vdev_free(spa->spa_l2cache.sav_vdevs[i]);
1699 	}
1700 	if (spa->spa_l2cache.sav_vdevs) {
1701 		kmem_free(spa->spa_l2cache.sav_vdevs,
1702 		    spa->spa_l2cache.sav_count * sizeof (void *));
1703 		spa->spa_l2cache.sav_vdevs = NULL;
1704 	}
1705 	if (spa->spa_l2cache.sav_config) {
1706 		nvlist_free(spa->spa_l2cache.sav_config);
1707 		spa->spa_l2cache.sav_config = NULL;
1708 	}
1709 	spa->spa_l2cache.sav_count = 0;
1710 
1711 	spa->spa_async_suspended = 0;
1712 
1713 	spa->spa_indirect_vdevs_loaded = B_FALSE;
1714 
1715 	if (spa->spa_comment != NULL) {
1716 		spa_strfree(spa->spa_comment);
1717 		spa->spa_comment = NULL;
1718 	}
1719 	if (spa->spa_compatibility != NULL) {
1720 		spa_strfree(spa->spa_compatibility);
1721 		spa->spa_compatibility = NULL;
1722 	}
1723 
1724 	spa_config_exit(spa, SCL_ALL, spa);
1725 }
1726 
1727 /*
1728  * Load (or re-load) the current list of vdevs describing the active spares for
1729  * this pool.  When this is called, we have some form of basic information in
1730  * 'spa_spares.sav_config'.  We parse this into vdevs, try to open them, and
1731  * then re-generate a more complete list including status information.
1732  */
1733 void
1734 spa_load_spares(spa_t *spa)
1735 {
1736 	nvlist_t **spares;
1737 	uint_t nspares;
1738 	int i;
1739 	vdev_t *vd, *tvd;
1740 
1741 #ifndef _KERNEL
1742 	/*
1743 	 * zdb opens both the current state of the pool and the
1744 	 * checkpointed state (if present), with a different spa_t.
1745 	 *
1746 	 * As spare vdevs are shared among open pools, we skip loading
1747 	 * them when we load the checkpointed state of the pool.
1748 	 */
1749 	if (!spa_writeable(spa))
1750 		return;
1751 #endif
1752 
1753 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1754 
1755 	/*
1756 	 * First, close and free any existing spare vdevs.
1757 	 */
1758 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1759 		vd = spa->spa_spares.sav_vdevs[i];
1760 
1761 		/* Undo the call to spa_activate() below */
1762 		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1763 		    B_FALSE)) != NULL && tvd->vdev_isspare)
1764 			spa_spare_remove(tvd);
1765 		vdev_close(vd);
1766 		vdev_free(vd);
1767 	}
1768 
1769 	if (spa->spa_spares.sav_vdevs)
1770 		kmem_free(spa->spa_spares.sav_vdevs,
1771 		    spa->spa_spares.sav_count * sizeof (void *));
1772 
1773 	if (spa->spa_spares.sav_config == NULL)
1774 		nspares = 0;
1775 	else
1776 		VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
1777 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
1778 
1779 	spa->spa_spares.sav_count = (int)nspares;
1780 	spa->spa_spares.sav_vdevs = NULL;
1781 
1782 	if (nspares == 0)
1783 		return;
1784 
1785 	/*
1786 	 * Construct the array of vdevs, opening them to get status in the
1787 	 * process.   For each spare, there is potentially two different vdev_t
1788 	 * structures associated with it: one in the list of spares (used only
1789 	 * for basic validation purposes) and one in the active vdev
1790 	 * configuration (if it's spared in).  During this phase we open and
1791 	 * validate each vdev on the spare list.  If the vdev also exists in the
1792 	 * active configuration, then we also mark this vdev as an active spare.
1793 	 */
1794 	spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *),
1795 	    KM_SLEEP);
1796 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1797 		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
1798 		    VDEV_ALLOC_SPARE) == 0);
1799 		ASSERT(vd != NULL);
1800 
1801 		spa->spa_spares.sav_vdevs[i] = vd;
1802 
1803 		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1804 		    B_FALSE)) != NULL) {
1805 			if (!tvd->vdev_isspare)
1806 				spa_spare_add(tvd);
1807 
1808 			/*
1809 			 * We only mark the spare active if we were successfully
1810 			 * able to load the vdev.  Otherwise, importing a pool
1811 			 * with a bad active spare would result in strange
1812 			 * behavior, because multiple pool would think the spare
1813 			 * is actively in use.
1814 			 *
1815 			 * There is a vulnerability here to an equally bizarre
1816 			 * circumstance, where a dead active spare is later
1817 			 * brought back to life (onlined or otherwise).  Given
1818 			 * the rarity of this scenario, and the extra complexity
1819 			 * it adds, we ignore the possibility.
1820 			 */
1821 			if (!vdev_is_dead(tvd))
1822 				spa_spare_activate(tvd);
1823 		}
1824 
1825 		vd->vdev_top = vd;
1826 		vd->vdev_aux = &spa->spa_spares;
1827 
1828 		if (vdev_open(vd) != 0)
1829 			continue;
1830 
1831 		if (vdev_validate_aux(vd) == 0)
1832 			spa_spare_add(vd);
1833 	}
1834 
1835 	/*
1836 	 * Recompute the stashed list of spares, with status information
1837 	 * this time.
1838 	 */
1839 	VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES,
1840 	    DATA_TYPE_NVLIST_ARRAY) == 0);
1841 
1842 	spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *),
1843 	    KM_SLEEP);
1844 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1845 		spares[i] = vdev_config_generate(spa,
1846 		    spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE);
1847 	VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
1848 	    ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0);
1849 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1850 		nvlist_free(spares[i]);
1851 	kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *));
1852 }
1853 
1854 /*
1855  * Load (or re-load) the current list of vdevs describing the active l2cache for
1856  * this pool.  When this is called, we have some form of basic information in
1857  * 'spa_l2cache.sav_config'.  We parse this into vdevs, try to open them, and
1858  * then re-generate a more complete list including status information.
1859  * Devices which are already active have their details maintained, and are
1860  * not re-opened.
1861  */
1862 void
1863 spa_load_l2cache(spa_t *spa)
1864 {
1865 	nvlist_t **l2cache = NULL;
1866 	uint_t nl2cache;
1867 	int i, j, oldnvdevs;
1868 	uint64_t guid;
1869 	vdev_t *vd, **oldvdevs, **newvdevs;
1870 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
1871 
1872 #ifndef _KERNEL
1873 	/*
1874 	 * zdb opens both the current state of the pool and the
1875 	 * checkpointed state (if present), with a different spa_t.
1876 	 *
1877 	 * As L2 caches are part of the ARC which is shared among open
1878 	 * pools, we skip loading them when we load the checkpointed
1879 	 * state of the pool.
1880 	 */
1881 	if (!spa_writeable(spa))
1882 		return;
1883 #endif
1884 
1885 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1886 
1887 	oldvdevs = sav->sav_vdevs;
1888 	oldnvdevs = sav->sav_count;
1889 	sav->sav_vdevs = NULL;
1890 	sav->sav_count = 0;
1891 
1892 	if (sav->sav_config == NULL) {
1893 		nl2cache = 0;
1894 		newvdevs = NULL;
1895 		goto out;
1896 	}
1897 
1898 	VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
1899 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
1900 	newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP);
1901 
1902 	/*
1903 	 * Process new nvlist of vdevs.
1904 	 */
1905 	for (i = 0; i < nl2cache; i++) {
1906 		VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID,
1907 		    &guid) == 0);
1908 
1909 		newvdevs[i] = NULL;
1910 		for (j = 0; j < oldnvdevs; j++) {
1911 			vd = oldvdevs[j];
1912 			if (vd != NULL && guid == vd->vdev_guid) {
1913 				/*
1914 				 * Retain previous vdev for add/remove ops.
1915 				 */
1916 				newvdevs[i] = vd;
1917 				oldvdevs[j] = NULL;
1918 				break;
1919 			}
1920 		}
1921 
1922 		if (newvdevs[i] == NULL) {
1923 			/*
1924 			 * Create new vdev
1925 			 */
1926 			VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
1927 			    VDEV_ALLOC_L2CACHE) == 0);
1928 			ASSERT(vd != NULL);
1929 			newvdevs[i] = vd;
1930 
1931 			/*
1932 			 * Commit this vdev as an l2cache device,
1933 			 * even if it fails to open.
1934 			 */
1935 			spa_l2cache_add(vd);
1936 
1937 			vd->vdev_top = vd;
1938 			vd->vdev_aux = sav;
1939 
1940 			spa_l2cache_activate(vd);
1941 
1942 			if (vdev_open(vd) != 0)
1943 				continue;
1944 
1945 			(void) vdev_validate_aux(vd);
1946 
1947 			if (!vdev_is_dead(vd))
1948 				l2arc_add_vdev(spa, vd);
1949 
1950 			/*
1951 			 * Upon cache device addition to a pool or pool
1952 			 * creation with a cache device or if the header
1953 			 * of the device is invalid we issue an async
1954 			 * TRIM command for the whole device which will
1955 			 * execute if l2arc_trim_ahead > 0.
1956 			 */
1957 			spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM);
1958 		}
1959 	}
1960 
1961 	sav->sav_vdevs = newvdevs;
1962 	sav->sav_count = (int)nl2cache;
1963 
1964 	/*
1965 	 * Recompute the stashed list of l2cache devices, with status
1966 	 * information this time.
1967 	 */
1968 	VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE,
1969 	    DATA_TYPE_NVLIST_ARRAY) == 0);
1970 
1971 	if (sav->sav_count > 0)
1972 		l2cache = kmem_alloc(sav->sav_count * sizeof (void *),
1973 		    KM_SLEEP);
1974 	for (i = 0; i < sav->sav_count; i++)
1975 		l2cache[i] = vdev_config_generate(spa,
1976 		    sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE);
1977 	VERIFY(nvlist_add_nvlist_array(sav->sav_config,
1978 	    ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0);
1979 
1980 out:
1981 	/*
1982 	 * Purge vdevs that were dropped
1983 	 */
1984 	for (i = 0; i < oldnvdevs; i++) {
1985 		uint64_t pool;
1986 
1987 		vd = oldvdevs[i];
1988 		if (vd != NULL) {
1989 			ASSERT(vd->vdev_isl2cache);
1990 
1991 			if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
1992 			    pool != 0ULL && l2arc_vdev_present(vd))
1993 				l2arc_remove_vdev(vd);
1994 			vdev_clear_stats(vd);
1995 			vdev_free(vd);
1996 		}
1997 	}
1998 
1999 	if (oldvdevs)
2000 		kmem_free(oldvdevs, oldnvdevs * sizeof (void *));
2001 
2002 	for (i = 0; i < sav->sav_count; i++)
2003 		nvlist_free(l2cache[i]);
2004 	if (sav->sav_count)
2005 		kmem_free(l2cache, sav->sav_count * sizeof (void *));
2006 }
2007 
2008 static int
2009 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
2010 {
2011 	dmu_buf_t *db;
2012 	char *packed = NULL;
2013 	size_t nvsize = 0;
2014 	int error;
2015 	*value = NULL;
2016 
2017 	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
2018 	if (error)
2019 		return (error);
2020 
2021 	nvsize = *(uint64_t *)db->db_data;
2022 	dmu_buf_rele(db, FTAG);
2023 
2024 	packed = vmem_alloc(nvsize, KM_SLEEP);
2025 	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed,
2026 	    DMU_READ_PREFETCH);
2027 	if (error == 0)
2028 		error = nvlist_unpack(packed, nvsize, value, 0);
2029 	vmem_free(packed, nvsize);
2030 
2031 	return (error);
2032 }
2033 
2034 /*
2035  * Concrete top-level vdevs that are not missing and are not logs. At every
2036  * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds.
2037  */
2038 static uint64_t
2039 spa_healthy_core_tvds(spa_t *spa)
2040 {
2041 	vdev_t *rvd = spa->spa_root_vdev;
2042 	uint64_t tvds = 0;
2043 
2044 	for (uint64_t i = 0; i < rvd->vdev_children; i++) {
2045 		vdev_t *vd = rvd->vdev_child[i];
2046 		if (vd->vdev_islog)
2047 			continue;
2048 		if (vdev_is_concrete(vd) && !vdev_is_dead(vd))
2049 			tvds++;
2050 	}
2051 
2052 	return (tvds);
2053 }
2054 
2055 /*
2056  * Checks to see if the given vdev could not be opened, in which case we post a
2057  * sysevent to notify the autoreplace code that the device has been removed.
2058  */
2059 static void
2060 spa_check_removed(vdev_t *vd)
2061 {
2062 	for (uint64_t c = 0; c < vd->vdev_children; c++)
2063 		spa_check_removed(vd->vdev_child[c]);
2064 
2065 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) &&
2066 	    vdev_is_concrete(vd)) {
2067 		zfs_post_autoreplace(vd->vdev_spa, vd);
2068 		spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK);
2069 	}
2070 }
2071 
2072 static int
2073 spa_check_for_missing_logs(spa_t *spa)
2074 {
2075 	vdev_t *rvd = spa->spa_root_vdev;
2076 
2077 	/*
2078 	 * If we're doing a normal import, then build up any additional
2079 	 * diagnostic information about missing log devices.
2080 	 * We'll pass this up to the user for further processing.
2081 	 */
2082 	if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) {
2083 		nvlist_t **child, *nv;
2084 		uint64_t idx = 0;
2085 
2086 		child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *),
2087 		    KM_SLEEP);
2088 		VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0);
2089 
2090 		for (uint64_t c = 0; c < rvd->vdev_children; c++) {
2091 			vdev_t *tvd = rvd->vdev_child[c];
2092 
2093 			/*
2094 			 * We consider a device as missing only if it failed
2095 			 * to open (i.e. offline or faulted is not considered
2096 			 * as missing).
2097 			 */
2098 			if (tvd->vdev_islog &&
2099 			    tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
2100 				child[idx++] = vdev_config_generate(spa, tvd,
2101 				    B_FALSE, VDEV_CONFIG_MISSING);
2102 			}
2103 		}
2104 
2105 		if (idx > 0) {
2106 			fnvlist_add_nvlist_array(nv,
2107 			    ZPOOL_CONFIG_CHILDREN, child, idx);
2108 			fnvlist_add_nvlist(spa->spa_load_info,
2109 			    ZPOOL_CONFIG_MISSING_DEVICES, nv);
2110 
2111 			for (uint64_t i = 0; i < idx; i++)
2112 				nvlist_free(child[i]);
2113 		}
2114 		nvlist_free(nv);
2115 		kmem_free(child, rvd->vdev_children * sizeof (char **));
2116 
2117 		if (idx > 0) {
2118 			spa_load_failed(spa, "some log devices are missing");
2119 			vdev_dbgmsg_print_tree(rvd, 2);
2120 			return (SET_ERROR(ENXIO));
2121 		}
2122 	} else {
2123 		for (uint64_t c = 0; c < rvd->vdev_children; c++) {
2124 			vdev_t *tvd = rvd->vdev_child[c];
2125 
2126 			if (tvd->vdev_islog &&
2127 			    tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
2128 				spa_set_log_state(spa, SPA_LOG_CLEAR);
2129 				spa_load_note(spa, "some log devices are "
2130 				    "missing, ZIL is dropped.");
2131 				vdev_dbgmsg_print_tree(rvd, 2);
2132 				break;
2133 			}
2134 		}
2135 	}
2136 
2137 	return (0);
2138 }
2139 
2140 /*
2141  * Check for missing log devices
2142  */
2143 static boolean_t
2144 spa_check_logs(spa_t *spa)
2145 {
2146 	boolean_t rv = B_FALSE;
2147 	dsl_pool_t *dp = spa_get_dsl(spa);
2148 
2149 	switch (spa->spa_log_state) {
2150 	default:
2151 		break;
2152 	case SPA_LOG_MISSING:
2153 		/* need to recheck in case slog has been restored */
2154 	case SPA_LOG_UNKNOWN:
2155 		rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
2156 		    zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0);
2157 		if (rv)
2158 			spa_set_log_state(spa, SPA_LOG_MISSING);
2159 		break;
2160 	}
2161 	return (rv);
2162 }
2163 
2164 /*
2165  * Passivate any log vdevs (note, does not apply to embedded log metaslabs).
2166  */
2167 static boolean_t
2168 spa_passivate_log(spa_t *spa)
2169 {
2170 	vdev_t *rvd = spa->spa_root_vdev;
2171 	boolean_t slog_found = B_FALSE;
2172 
2173 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
2174 
2175 	for (int c = 0; c < rvd->vdev_children; c++) {
2176 		vdev_t *tvd = rvd->vdev_child[c];
2177 
2178 		if (tvd->vdev_islog) {
2179 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
2180 			metaslab_group_passivate(tvd->vdev_mg);
2181 			slog_found = B_TRUE;
2182 		}
2183 	}
2184 
2185 	return (slog_found);
2186 }
2187 
2188 /*
2189  * Activate any log vdevs (note, does not apply to embedded log metaslabs).
2190  */
2191 static void
2192 spa_activate_log(spa_t *spa)
2193 {
2194 	vdev_t *rvd = spa->spa_root_vdev;
2195 
2196 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
2197 
2198 	for (int c = 0; c < rvd->vdev_children; c++) {
2199 		vdev_t *tvd = rvd->vdev_child[c];
2200 
2201 		if (tvd->vdev_islog) {
2202 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
2203 			metaslab_group_activate(tvd->vdev_mg);
2204 		}
2205 	}
2206 }
2207 
2208 int
2209 spa_reset_logs(spa_t *spa)
2210 {
2211 	int error;
2212 
2213 	error = dmu_objset_find(spa_name(spa), zil_reset,
2214 	    NULL, DS_FIND_CHILDREN);
2215 	if (error == 0) {
2216 		/*
2217 		 * We successfully offlined the log device, sync out the
2218 		 * current txg so that the "stubby" block can be removed
2219 		 * by zil_sync().
2220 		 */
2221 		txg_wait_synced(spa->spa_dsl_pool, 0);
2222 	}
2223 	return (error);
2224 }
2225 
2226 static void
2227 spa_aux_check_removed(spa_aux_vdev_t *sav)
2228 {
2229 	for (int i = 0; i < sav->sav_count; i++)
2230 		spa_check_removed(sav->sav_vdevs[i]);
2231 }
2232 
2233 void
2234 spa_claim_notify(zio_t *zio)
2235 {
2236 	spa_t *spa = zio->io_spa;
2237 
2238 	if (zio->io_error)
2239 		return;
2240 
2241 	mutex_enter(&spa->spa_props_lock);	/* any mutex will do */
2242 	if (spa->spa_claim_max_txg < zio->io_bp->blk_birth)
2243 		spa->spa_claim_max_txg = zio->io_bp->blk_birth;
2244 	mutex_exit(&spa->spa_props_lock);
2245 }
2246 
2247 typedef struct spa_load_error {
2248 	uint64_t	sle_meta_count;
2249 	uint64_t	sle_data_count;
2250 } spa_load_error_t;
2251 
2252 static void
2253 spa_load_verify_done(zio_t *zio)
2254 {
2255 	blkptr_t *bp = zio->io_bp;
2256 	spa_load_error_t *sle = zio->io_private;
2257 	dmu_object_type_t type = BP_GET_TYPE(bp);
2258 	int error = zio->io_error;
2259 	spa_t *spa = zio->io_spa;
2260 
2261 	abd_free(zio->io_abd);
2262 	if (error) {
2263 		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
2264 		    type != DMU_OT_INTENT_LOG)
2265 			atomic_inc_64(&sle->sle_meta_count);
2266 		else
2267 			atomic_inc_64(&sle->sle_data_count);
2268 	}
2269 
2270 	mutex_enter(&spa->spa_scrub_lock);
2271 	spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp);
2272 	cv_broadcast(&spa->spa_scrub_io_cv);
2273 	mutex_exit(&spa->spa_scrub_lock);
2274 }
2275 
2276 /*
2277  * Maximum number of inflight bytes is the log2 fraction of the arc size.
2278  * By default, we set it to 1/16th of the arc.
2279  */
2280 int spa_load_verify_shift = 4;
2281 int spa_load_verify_metadata = B_TRUE;
2282 int spa_load_verify_data = B_TRUE;
2283 
2284 /*ARGSUSED*/
2285 static int
2286 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
2287     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
2288 {
2289 	if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) ||
2290 	    BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp))
2291 		return (0);
2292 	/*
2293 	 * Note: normally this routine will not be called if
2294 	 * spa_load_verify_metadata is not set.  However, it may be useful
2295 	 * to manually set the flag after the traversal has begun.
2296 	 */
2297 	if (!spa_load_verify_metadata)
2298 		return (0);
2299 	if (!BP_IS_METADATA(bp) && !spa_load_verify_data)
2300 		return (0);
2301 
2302 	uint64_t maxinflight_bytes =
2303 	    arc_target_bytes() >> spa_load_verify_shift;
2304 	zio_t *rio = arg;
2305 	size_t size = BP_GET_PSIZE(bp);
2306 
2307 	mutex_enter(&spa->spa_scrub_lock);
2308 	while (spa->spa_load_verify_bytes >= maxinflight_bytes)
2309 		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
2310 	spa->spa_load_verify_bytes += size;
2311 	mutex_exit(&spa->spa_scrub_lock);
2312 
2313 	zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size,
2314 	    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
2315 	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
2316 	    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
2317 	return (0);
2318 }
2319 
2320 /* ARGSUSED */
2321 static int
2322 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
2323 {
2324 	if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN)
2325 		return (SET_ERROR(ENAMETOOLONG));
2326 
2327 	return (0);
2328 }
2329 
2330 static int
2331 spa_load_verify(spa_t *spa)
2332 {
2333 	zio_t *rio;
2334 	spa_load_error_t sle = { 0 };
2335 	zpool_load_policy_t policy;
2336 	boolean_t verify_ok = B_FALSE;
2337 	int error = 0;
2338 
2339 	zpool_get_load_policy(spa->spa_config, &policy);
2340 
2341 	if (policy.zlp_rewind & ZPOOL_NEVER_REWIND)
2342 		return (0);
2343 
2344 	dsl_pool_config_enter(spa->spa_dsl_pool, FTAG);
2345 	error = dmu_objset_find_dp(spa->spa_dsl_pool,
2346 	    spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL,
2347 	    DS_FIND_CHILDREN);
2348 	dsl_pool_config_exit(spa->spa_dsl_pool, FTAG);
2349 	if (error != 0)
2350 		return (error);
2351 
2352 	rio = zio_root(spa, NULL, &sle,
2353 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
2354 
2355 	if (spa_load_verify_metadata) {
2356 		if (spa->spa_extreme_rewind) {
2357 			spa_load_note(spa, "performing a complete scan of the "
2358 			    "pool since extreme rewind is on. This may take "
2359 			    "a very long time.\n  (spa_load_verify_data=%u, "
2360 			    "spa_load_verify_metadata=%u)",
2361 			    spa_load_verify_data, spa_load_verify_metadata);
2362 		}
2363 
2364 		error = traverse_pool(spa, spa->spa_verify_min_txg,
2365 		    TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA |
2366 		    TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio);
2367 	}
2368 
2369 	(void) zio_wait(rio);
2370 	ASSERT0(spa->spa_load_verify_bytes);
2371 
2372 	spa->spa_load_meta_errors = sle.sle_meta_count;
2373 	spa->spa_load_data_errors = sle.sle_data_count;
2374 
2375 	if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) {
2376 		spa_load_note(spa, "spa_load_verify found %llu metadata errors "
2377 		    "and %llu data errors", (u_longlong_t)sle.sle_meta_count,
2378 		    (u_longlong_t)sle.sle_data_count);
2379 	}
2380 
2381 	if (spa_load_verify_dryrun ||
2382 	    (!error && sle.sle_meta_count <= policy.zlp_maxmeta &&
2383 	    sle.sle_data_count <= policy.zlp_maxdata)) {
2384 		int64_t loss = 0;
2385 
2386 		verify_ok = B_TRUE;
2387 		spa->spa_load_txg = spa->spa_uberblock.ub_txg;
2388 		spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp;
2389 
2390 		loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts;
2391 		VERIFY(nvlist_add_uint64(spa->spa_load_info,
2392 		    ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0);
2393 		VERIFY(nvlist_add_int64(spa->spa_load_info,
2394 		    ZPOOL_CONFIG_REWIND_TIME, loss) == 0);
2395 		VERIFY(nvlist_add_uint64(spa->spa_load_info,
2396 		    ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0);
2397 	} else {
2398 		spa->spa_load_max_txg = spa->spa_uberblock.ub_txg;
2399 	}
2400 
2401 	if (spa_load_verify_dryrun)
2402 		return (0);
2403 
2404 	if (error) {
2405 		if (error != ENXIO && error != EIO)
2406 			error = SET_ERROR(EIO);
2407 		return (error);
2408 	}
2409 
2410 	return (verify_ok ? 0 : EIO);
2411 }
2412 
2413 /*
2414  * Find a value in the pool props object.
2415  */
2416 static void
2417 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val)
2418 {
2419 	(void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object,
2420 	    zpool_prop_to_name(prop), sizeof (uint64_t), 1, val);
2421 }
2422 
2423 /*
2424  * Find a value in the pool directory object.
2425  */
2426 static int
2427 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent)
2428 {
2429 	int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
2430 	    name, sizeof (uint64_t), 1, val);
2431 
2432 	if (error != 0 && (error != ENOENT || log_enoent)) {
2433 		spa_load_failed(spa, "couldn't get '%s' value in MOS directory "
2434 		    "[error=%d]", name, error);
2435 	}
2436 
2437 	return (error);
2438 }
2439 
2440 static int
2441 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err)
2442 {
2443 	vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux);
2444 	return (SET_ERROR(err));
2445 }
2446 
2447 boolean_t
2448 spa_livelist_delete_check(spa_t *spa)
2449 {
2450 	return (spa->spa_livelists_to_delete != 0);
2451 }
2452 
2453 /* ARGSUSED */
2454 static boolean_t
2455 spa_livelist_delete_cb_check(void *arg, zthr_t *z)
2456 {
2457 	spa_t *spa = arg;
2458 	return (spa_livelist_delete_check(spa));
2459 }
2460 
2461 static int
2462 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
2463 {
2464 	spa_t *spa = arg;
2465 	zio_free(spa, tx->tx_txg, bp);
2466 	dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD,
2467 	    -bp_get_dsize_sync(spa, bp),
2468 	    -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx);
2469 	return (0);
2470 }
2471 
2472 static int
2473 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp)
2474 {
2475 	int err;
2476 	zap_cursor_t zc;
2477 	zap_attribute_t za;
2478 	zap_cursor_init(&zc, os, zap_obj);
2479 	err = zap_cursor_retrieve(&zc, &za);
2480 	zap_cursor_fini(&zc);
2481 	if (err == 0)
2482 		*llp = za.za_first_integer;
2483 	return (err);
2484 }
2485 
2486 /*
2487  * Components of livelist deletion that must be performed in syncing
2488  * context: freeing block pointers and updating the pool-wide data
2489  * structures to indicate how much work is left to do
2490  */
2491 typedef struct sublist_delete_arg {
2492 	spa_t *spa;
2493 	dsl_deadlist_t *ll;
2494 	uint64_t key;
2495 	bplist_t *to_free;
2496 } sublist_delete_arg_t;
2497 
2498 static void
2499 sublist_delete_sync(void *arg, dmu_tx_t *tx)
2500 {
2501 	sublist_delete_arg_t *sda = arg;
2502 	spa_t *spa = sda->spa;
2503 	dsl_deadlist_t *ll = sda->ll;
2504 	uint64_t key = sda->key;
2505 	bplist_t *to_free = sda->to_free;
2506 
2507 	bplist_iterate(to_free, delete_blkptr_cb, spa, tx);
2508 	dsl_deadlist_remove_entry(ll, key, tx);
2509 }
2510 
2511 typedef struct livelist_delete_arg {
2512 	spa_t *spa;
2513 	uint64_t ll_obj;
2514 	uint64_t zap_obj;
2515 } livelist_delete_arg_t;
2516 
2517 static void
2518 livelist_delete_sync(void *arg, dmu_tx_t *tx)
2519 {
2520 	livelist_delete_arg_t *lda = arg;
2521 	spa_t *spa = lda->spa;
2522 	uint64_t ll_obj = lda->ll_obj;
2523 	uint64_t zap_obj = lda->zap_obj;
2524 	objset_t *mos = spa->spa_meta_objset;
2525 	uint64_t count;
2526 
2527 	/* free the livelist and decrement the feature count */
2528 	VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx));
2529 	dsl_deadlist_free(mos, ll_obj, tx);
2530 	spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx);
2531 	VERIFY0(zap_count(mos, zap_obj, &count));
2532 	if (count == 0) {
2533 		/* no more livelists to delete */
2534 		VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT,
2535 		    DMU_POOL_DELETED_CLONES, tx));
2536 		VERIFY0(zap_destroy(mos, zap_obj, tx));
2537 		spa->spa_livelists_to_delete = 0;
2538 		spa_notify_waiters(spa);
2539 	}
2540 }
2541 
2542 /*
2543  * Load in the value for the livelist to be removed and open it. Then,
2544  * load its first sublist and determine which block pointers should actually
2545  * be freed. Then, call a synctask which performs the actual frees and updates
2546  * the pool-wide livelist data.
2547  */
2548 /* ARGSUSED */
2549 static void
2550 spa_livelist_delete_cb(void *arg, zthr_t *z)
2551 {
2552 	spa_t *spa = arg;
2553 	uint64_t ll_obj = 0, count;
2554 	objset_t *mos = spa->spa_meta_objset;
2555 	uint64_t zap_obj = spa->spa_livelists_to_delete;
2556 	/*
2557 	 * Determine the next livelist to delete. This function should only
2558 	 * be called if there is at least one deleted clone.
2559 	 */
2560 	VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj));
2561 	VERIFY0(zap_count(mos, ll_obj, &count));
2562 	if (count > 0) {
2563 		dsl_deadlist_t *ll;
2564 		dsl_deadlist_entry_t *dle;
2565 		bplist_t to_free;
2566 		ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP);
2567 		dsl_deadlist_open(ll, mos, ll_obj);
2568 		dle = dsl_deadlist_first(ll);
2569 		ASSERT3P(dle, !=, NULL);
2570 		bplist_create(&to_free);
2571 		int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free,
2572 		    z, NULL);
2573 		if (err == 0) {
2574 			sublist_delete_arg_t sync_arg = {
2575 			    .spa = spa,
2576 			    .ll = ll,
2577 			    .key = dle->dle_mintxg,
2578 			    .to_free = &to_free
2579 			};
2580 			zfs_dbgmsg("deleting sublist (id %llu) from"
2581 			    " livelist %llu, %lld remaining",
2582 			    (u_longlong_t)dle->dle_bpobj.bpo_object,
2583 			    (u_longlong_t)ll_obj, (longlong_t)count - 1);
2584 			VERIFY0(dsl_sync_task(spa_name(spa), NULL,
2585 			    sublist_delete_sync, &sync_arg, 0,
2586 			    ZFS_SPACE_CHECK_DESTROY));
2587 		} else {
2588 			VERIFY3U(err, ==, EINTR);
2589 		}
2590 		bplist_clear(&to_free);
2591 		bplist_destroy(&to_free);
2592 		dsl_deadlist_close(ll);
2593 		kmem_free(ll, sizeof (dsl_deadlist_t));
2594 	} else {
2595 		livelist_delete_arg_t sync_arg = {
2596 		    .spa = spa,
2597 		    .ll_obj = ll_obj,
2598 		    .zap_obj = zap_obj
2599 		};
2600 		zfs_dbgmsg("deletion of livelist %llu completed",
2601 		    (u_longlong_t)ll_obj);
2602 		VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync,
2603 		    &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY));
2604 	}
2605 }
2606 
2607 static void
2608 spa_start_livelist_destroy_thread(spa_t *spa)
2609 {
2610 	ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL);
2611 	spa->spa_livelist_delete_zthr =
2612 	    zthr_create("z_livelist_destroy",
2613 	    spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa,
2614 	    minclsyspri);
2615 }
2616 
2617 typedef struct livelist_new_arg {
2618 	bplist_t *allocs;
2619 	bplist_t *frees;
2620 } livelist_new_arg_t;
2621 
2622 static int
2623 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
2624     dmu_tx_t *tx)
2625 {
2626 	ASSERT(tx == NULL);
2627 	livelist_new_arg_t *lna = arg;
2628 	if (bp_freed) {
2629 		bplist_append(lna->frees, bp);
2630 	} else {
2631 		bplist_append(lna->allocs, bp);
2632 		zfs_livelist_condense_new_alloc++;
2633 	}
2634 	return (0);
2635 }
2636 
2637 typedef struct livelist_condense_arg {
2638 	spa_t *spa;
2639 	bplist_t to_keep;
2640 	uint64_t first_size;
2641 	uint64_t next_size;
2642 } livelist_condense_arg_t;
2643 
2644 static void
2645 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx)
2646 {
2647 	livelist_condense_arg_t *lca = arg;
2648 	spa_t *spa = lca->spa;
2649 	bplist_t new_frees;
2650 	dsl_dataset_t *ds = spa->spa_to_condense.ds;
2651 
2652 	/* Have we been cancelled? */
2653 	if (spa->spa_to_condense.cancelled) {
2654 		zfs_livelist_condense_sync_cancel++;
2655 		goto out;
2656 	}
2657 
2658 	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
2659 	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
2660 	dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist;
2661 
2662 	/*
2663 	 * It's possible that the livelist was changed while the zthr was
2664 	 * running. Therefore, we need to check for new blkptrs in the two
2665 	 * entries being condensed and continue to track them in the livelist.
2666 	 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl),
2667 	 * it's possible that the newly added blkptrs are FREEs or ALLOCs so
2668 	 * we need to sort them into two different bplists.
2669 	 */
2670 	uint64_t first_obj = first->dle_bpobj.bpo_object;
2671 	uint64_t next_obj = next->dle_bpobj.bpo_object;
2672 	uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs;
2673 	uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs;
2674 
2675 	bplist_create(&new_frees);
2676 	livelist_new_arg_t new_bps = {
2677 	    .allocs = &lca->to_keep,
2678 	    .frees = &new_frees,
2679 	};
2680 
2681 	if (cur_first_size > lca->first_size) {
2682 		VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj,
2683 		    livelist_track_new_cb, &new_bps, lca->first_size));
2684 	}
2685 	if (cur_next_size > lca->next_size) {
2686 		VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj,
2687 		    livelist_track_new_cb, &new_bps, lca->next_size));
2688 	}
2689 
2690 	dsl_deadlist_clear_entry(first, ll, tx);
2691 	ASSERT(bpobj_is_empty(&first->dle_bpobj));
2692 	dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx);
2693 
2694 	bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx);
2695 	bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx);
2696 	bplist_destroy(&new_frees);
2697 
2698 	char dsname[ZFS_MAX_DATASET_NAME_LEN];
2699 	dsl_dataset_name(ds, dsname);
2700 	zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu "
2701 	    "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu "
2702 	    "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname,
2703 	    (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj,
2704 	    (u_longlong_t)cur_first_size, (u_longlong_t)next_obj,
2705 	    (u_longlong_t)cur_next_size,
2706 	    (u_longlong_t)first->dle_bpobj.bpo_object,
2707 	    (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs);
2708 out:
2709 	dmu_buf_rele(ds->ds_dbuf, spa);
2710 	spa->spa_to_condense.ds = NULL;
2711 	bplist_clear(&lca->to_keep);
2712 	bplist_destroy(&lca->to_keep);
2713 	kmem_free(lca, sizeof (livelist_condense_arg_t));
2714 	spa->spa_to_condense.syncing = B_FALSE;
2715 }
2716 
2717 static void
2718 spa_livelist_condense_cb(void *arg, zthr_t *t)
2719 {
2720 	while (zfs_livelist_condense_zthr_pause &&
2721 	    !(zthr_has_waiters(t) || zthr_iscancelled(t)))
2722 		delay(1);
2723 
2724 	spa_t *spa = arg;
2725 	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
2726 	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
2727 	uint64_t first_size, next_size;
2728 
2729 	livelist_condense_arg_t *lca =
2730 	    kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP);
2731 	bplist_create(&lca->to_keep);
2732 
2733 	/*
2734 	 * Process the livelists (matching FREEs and ALLOCs) in open context
2735 	 * so we have minimal work in syncing context to condense.
2736 	 *
2737 	 * We save bpobj sizes (first_size and next_size) to use later in
2738 	 * syncing context to determine if entries were added to these sublists
2739 	 * while in open context. This is possible because the clone is still
2740 	 * active and open for normal writes and we want to make sure the new,
2741 	 * unprocessed blockpointers are inserted into the livelist normally.
2742 	 *
2743 	 * Note that dsl_process_sub_livelist() both stores the size number of
2744 	 * blockpointers and iterates over them while the bpobj's lock held, so
2745 	 * the sizes returned to us are consistent which what was actually
2746 	 * processed.
2747 	 */
2748 	int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t,
2749 	    &first_size);
2750 	if (err == 0)
2751 		err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep,
2752 		    t, &next_size);
2753 
2754 	if (err == 0) {
2755 		while (zfs_livelist_condense_sync_pause &&
2756 		    !(zthr_has_waiters(t) || zthr_iscancelled(t)))
2757 			delay(1);
2758 
2759 		dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
2760 		dmu_tx_mark_netfree(tx);
2761 		dmu_tx_hold_space(tx, 1);
2762 		err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE);
2763 		if (err == 0) {
2764 			/*
2765 			 * Prevent the condense zthr restarting before
2766 			 * the synctask completes.
2767 			 */
2768 			spa->spa_to_condense.syncing = B_TRUE;
2769 			lca->spa = spa;
2770 			lca->first_size = first_size;
2771 			lca->next_size = next_size;
2772 			dsl_sync_task_nowait(spa_get_dsl(spa),
2773 			    spa_livelist_condense_sync, lca, tx);
2774 			dmu_tx_commit(tx);
2775 			return;
2776 		}
2777 	}
2778 	/*
2779 	 * Condensing can not continue: either it was externally stopped or
2780 	 * we were unable to assign to a tx because the pool has run out of
2781 	 * space. In the second case, we'll just end up trying to condense
2782 	 * again in a later txg.
2783 	 */
2784 	ASSERT(err != 0);
2785 	bplist_clear(&lca->to_keep);
2786 	bplist_destroy(&lca->to_keep);
2787 	kmem_free(lca, sizeof (livelist_condense_arg_t));
2788 	dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa);
2789 	spa->spa_to_condense.ds = NULL;
2790 	if (err == EINTR)
2791 		zfs_livelist_condense_zthr_cancel++;
2792 }
2793 
2794 /* ARGSUSED */
2795 /*
2796  * Check that there is something to condense but that a condense is not
2797  * already in progress and that condensing has not been cancelled.
2798  */
2799 static boolean_t
2800 spa_livelist_condense_cb_check(void *arg, zthr_t *z)
2801 {
2802 	spa_t *spa = arg;
2803 	if ((spa->spa_to_condense.ds != NULL) &&
2804 	    (spa->spa_to_condense.syncing == B_FALSE) &&
2805 	    (spa->spa_to_condense.cancelled == B_FALSE)) {
2806 		return (B_TRUE);
2807 	}
2808 	return (B_FALSE);
2809 }
2810 
2811 static void
2812 spa_start_livelist_condensing_thread(spa_t *spa)
2813 {
2814 	spa->spa_to_condense.ds = NULL;
2815 	spa->spa_to_condense.first = NULL;
2816 	spa->spa_to_condense.next = NULL;
2817 	spa->spa_to_condense.syncing = B_FALSE;
2818 	spa->spa_to_condense.cancelled = B_FALSE;
2819 
2820 	ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL);
2821 	spa->spa_livelist_condense_zthr =
2822 	    zthr_create("z_livelist_condense",
2823 	    spa_livelist_condense_cb_check,
2824 	    spa_livelist_condense_cb, spa, minclsyspri);
2825 }
2826 
2827 static void
2828 spa_spawn_aux_threads(spa_t *spa)
2829 {
2830 	ASSERT(spa_writeable(spa));
2831 
2832 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
2833 
2834 	spa_start_indirect_condensing_thread(spa);
2835 	spa_start_livelist_destroy_thread(spa);
2836 	spa_start_livelist_condensing_thread(spa);
2837 
2838 	ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL);
2839 	spa->spa_checkpoint_discard_zthr =
2840 	    zthr_create("z_checkpoint_discard",
2841 	    spa_checkpoint_discard_thread_check,
2842 	    spa_checkpoint_discard_thread, spa, minclsyspri);
2843 }
2844 
2845 /*
2846  * Fix up config after a partly-completed split.  This is done with the
2847  * ZPOOL_CONFIG_SPLIT nvlist.  Both the splitting pool and the split-off
2848  * pool have that entry in their config, but only the splitting one contains
2849  * a list of all the guids of the vdevs that are being split off.
2850  *
2851  * This function determines what to do with that list: either rejoin
2852  * all the disks to the pool, or complete the splitting process.  To attempt
2853  * the rejoin, each disk that is offlined is marked online again, and
2854  * we do a reopen() call.  If the vdev label for every disk that was
2855  * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL)
2856  * then we call vdev_split() on each disk, and complete the split.
2857  *
2858  * Otherwise we leave the config alone, with all the vdevs in place in
2859  * the original pool.
2860  */
2861 static void
2862 spa_try_repair(spa_t *spa, nvlist_t *config)
2863 {
2864 	uint_t extracted;
2865 	uint64_t *glist;
2866 	uint_t i, gcount;
2867 	nvlist_t *nvl;
2868 	vdev_t **vd;
2869 	boolean_t attempt_reopen;
2870 
2871 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0)
2872 		return;
2873 
2874 	/* check that the config is complete */
2875 	if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
2876 	    &glist, &gcount) != 0)
2877 		return;
2878 
2879 	vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP);
2880 
2881 	/* attempt to online all the vdevs & validate */
2882 	attempt_reopen = B_TRUE;
2883 	for (i = 0; i < gcount; i++) {
2884 		if (glist[i] == 0)	/* vdev is hole */
2885 			continue;
2886 
2887 		vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE);
2888 		if (vd[i] == NULL) {
2889 			/*
2890 			 * Don't bother attempting to reopen the disks;
2891 			 * just do the split.
2892 			 */
2893 			attempt_reopen = B_FALSE;
2894 		} else {
2895 			/* attempt to re-online it */
2896 			vd[i]->vdev_offline = B_FALSE;
2897 		}
2898 	}
2899 
2900 	if (attempt_reopen) {
2901 		vdev_reopen(spa->spa_root_vdev);
2902 
2903 		/* check each device to see what state it's in */
2904 		for (extracted = 0, i = 0; i < gcount; i++) {
2905 			if (vd[i] != NULL &&
2906 			    vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL)
2907 				break;
2908 			++extracted;
2909 		}
2910 	}
2911 
2912 	/*
2913 	 * If every disk has been moved to the new pool, or if we never
2914 	 * even attempted to look at them, then we split them off for
2915 	 * good.
2916 	 */
2917 	if (!attempt_reopen || gcount == extracted) {
2918 		for (i = 0; i < gcount; i++)
2919 			if (vd[i] != NULL)
2920 				vdev_split(vd[i]);
2921 		vdev_reopen(spa->spa_root_vdev);
2922 	}
2923 
2924 	kmem_free(vd, gcount * sizeof (vdev_t *));
2925 }
2926 
2927 static int
2928 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type)
2929 {
2930 	char *ereport = FM_EREPORT_ZFS_POOL;
2931 	int error;
2932 
2933 	spa->spa_load_state = state;
2934 	(void) spa_import_progress_set_state(spa_guid(spa),
2935 	    spa_load_state(spa));
2936 
2937 	gethrestime(&spa->spa_loaded_ts);
2938 	error = spa_load_impl(spa, type, &ereport);
2939 
2940 	/*
2941 	 * Don't count references from objsets that are already closed
2942 	 * and are making their way through the eviction process.
2943 	 */
2944 	spa_evicting_os_wait(spa);
2945 	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
2946 	if (error) {
2947 		if (error != EEXIST) {
2948 			spa->spa_loaded_ts.tv_sec = 0;
2949 			spa->spa_loaded_ts.tv_nsec = 0;
2950 		}
2951 		if (error != EBADF) {
2952 			(void) zfs_ereport_post(ereport, spa,
2953 			    NULL, NULL, NULL, 0);
2954 		}
2955 	}
2956 	spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE;
2957 	spa->spa_ena = 0;
2958 
2959 	(void) spa_import_progress_set_state(spa_guid(spa),
2960 	    spa_load_state(spa));
2961 
2962 	return (error);
2963 }
2964 
2965 #ifdef ZFS_DEBUG
2966 /*
2967  * Count the number of per-vdev ZAPs associated with all of the vdevs in the
2968  * vdev tree rooted in the given vd, and ensure that each ZAP is present in the
2969  * spa's per-vdev ZAP list.
2970  */
2971 static uint64_t
2972 vdev_count_verify_zaps(vdev_t *vd)
2973 {
2974 	spa_t *spa = vd->vdev_spa;
2975 	uint64_t total = 0;
2976 
2977 	if (vd->vdev_top_zap != 0) {
2978 		total++;
2979 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
2980 		    spa->spa_all_vdev_zaps, vd->vdev_top_zap));
2981 	}
2982 	if (vd->vdev_leaf_zap != 0) {
2983 		total++;
2984 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
2985 		    spa->spa_all_vdev_zaps, vd->vdev_leaf_zap));
2986 	}
2987 
2988 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
2989 		total += vdev_count_verify_zaps(vd->vdev_child[i]);
2990 	}
2991 
2992 	return (total);
2993 }
2994 #endif
2995 
2996 /*
2997  * Determine whether the activity check is required.
2998  */
2999 static boolean_t
3000 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label,
3001     nvlist_t *config)
3002 {
3003 	uint64_t state = 0;
3004 	uint64_t hostid = 0;
3005 	uint64_t tryconfig_txg = 0;
3006 	uint64_t tryconfig_timestamp = 0;
3007 	uint16_t tryconfig_mmp_seq = 0;
3008 	nvlist_t *nvinfo;
3009 
3010 	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
3011 		nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3012 		(void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG,
3013 		    &tryconfig_txg);
3014 		(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
3015 		    &tryconfig_timestamp);
3016 		(void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ,
3017 		    &tryconfig_mmp_seq);
3018 	}
3019 
3020 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state);
3021 
3022 	/*
3023 	 * Disable the MMP activity check - This is used by zdb which
3024 	 * is intended to be used on potentially active pools.
3025 	 */
3026 	if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP)
3027 		return (B_FALSE);
3028 
3029 	/*
3030 	 * Skip the activity check when the MMP feature is disabled.
3031 	 */
3032 	if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0)
3033 		return (B_FALSE);
3034 
3035 	/*
3036 	 * If the tryconfig_ values are nonzero, they are the results of an
3037 	 * earlier tryimport.  If they all match the uberblock we just found,
3038 	 * then the pool has not changed and we return false so we do not test
3039 	 * a second time.
3040 	 */
3041 	if (tryconfig_txg && tryconfig_txg == ub->ub_txg &&
3042 	    tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp &&
3043 	    tryconfig_mmp_seq && tryconfig_mmp_seq ==
3044 	    (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0))
3045 		return (B_FALSE);
3046 
3047 	/*
3048 	 * Allow the activity check to be skipped when importing the pool
3049 	 * on the same host which last imported it.  Since the hostid from
3050 	 * configuration may be stale use the one read from the label.
3051 	 */
3052 	if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID))
3053 		hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID);
3054 
3055 	if (hostid == spa_get_hostid(spa))
3056 		return (B_FALSE);
3057 
3058 	/*
3059 	 * Skip the activity test when the pool was cleanly exported.
3060 	 */
3061 	if (state != POOL_STATE_ACTIVE)
3062 		return (B_FALSE);
3063 
3064 	return (B_TRUE);
3065 }
3066 
3067 /*
3068  * Nanoseconds the activity check must watch for changes on-disk.
3069  */
3070 static uint64_t
3071 spa_activity_check_duration(spa_t *spa, uberblock_t *ub)
3072 {
3073 	uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1);
3074 	uint64_t multihost_interval = MSEC2NSEC(
3075 	    MMP_INTERVAL_OK(zfs_multihost_interval));
3076 	uint64_t import_delay = MAX(NANOSEC, import_intervals *
3077 	    multihost_interval);
3078 
3079 	/*
3080 	 * Local tunables determine a minimum duration except for the case
3081 	 * where we know when the remote host will suspend the pool if MMP
3082 	 * writes do not land.
3083 	 *
3084 	 * See Big Theory comment at the top of mmp.c for the reasoning behind
3085 	 * these cases and times.
3086 	 */
3087 
3088 	ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100);
3089 
3090 	if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
3091 	    MMP_FAIL_INT(ub) > 0) {
3092 
3093 		/* MMP on remote host will suspend pool after failed writes */
3094 		import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) *
3095 		    MMP_IMPORT_SAFETY_FACTOR / 100;
3096 
3097 		zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp "
3098 		    "mmp_fails=%llu ub_mmp mmp_interval=%llu "
3099 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3100 		    (u_longlong_t)MMP_FAIL_INT(ub),
3101 		    (u_longlong_t)MMP_INTERVAL(ub),
3102 		    (u_longlong_t)import_intervals);
3103 
3104 	} else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
3105 	    MMP_FAIL_INT(ub) == 0) {
3106 
3107 		/* MMP on remote host will never suspend pool */
3108 		import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) +
3109 		    ub->ub_mmp_delay) * import_intervals);
3110 
3111 		zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp "
3112 		    "mmp_interval=%llu ub_mmp_delay=%llu "
3113 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3114 		    (u_longlong_t)MMP_INTERVAL(ub),
3115 		    (u_longlong_t)ub->ub_mmp_delay,
3116 		    (u_longlong_t)import_intervals);
3117 
3118 	} else if (MMP_VALID(ub)) {
3119 		/*
3120 		 * zfs-0.7 compatibility case
3121 		 */
3122 
3123 		import_delay = MAX(import_delay, (multihost_interval +
3124 		    ub->ub_mmp_delay) * import_intervals);
3125 
3126 		zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu "
3127 		    "import_intervals=%llu leaves=%u",
3128 		    (u_longlong_t)import_delay,
3129 		    (u_longlong_t)ub->ub_mmp_delay,
3130 		    (u_longlong_t)import_intervals,
3131 		    vdev_count_leaves(spa));
3132 	} else {
3133 		/* Using local tunings is the only reasonable option */
3134 		zfs_dbgmsg("pool last imported on non-MMP aware "
3135 		    "host using import_delay=%llu multihost_interval=%llu "
3136 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3137 		    (u_longlong_t)multihost_interval,
3138 		    (u_longlong_t)import_intervals);
3139 	}
3140 
3141 	return (import_delay);
3142 }
3143 
3144 /*
3145  * Perform the import activity check.  If the user canceled the import or
3146  * we detected activity then fail.
3147  */
3148 static int
3149 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config)
3150 {
3151 	uint64_t txg = ub->ub_txg;
3152 	uint64_t timestamp = ub->ub_timestamp;
3153 	uint64_t mmp_config = ub->ub_mmp_config;
3154 	uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0;
3155 	uint64_t import_delay;
3156 	hrtime_t import_expire;
3157 	nvlist_t *mmp_label = NULL;
3158 	vdev_t *rvd = spa->spa_root_vdev;
3159 	kcondvar_t cv;
3160 	kmutex_t mtx;
3161 	int error = 0;
3162 
3163 	cv_init(&cv, NULL, CV_DEFAULT, NULL);
3164 	mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL);
3165 	mutex_enter(&mtx);
3166 
3167 	/*
3168 	 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed
3169 	 * during the earlier tryimport.  If the txg recorded there is 0 then
3170 	 * the pool is known to be active on another host.
3171 	 *
3172 	 * Otherwise, the pool might be in use on another host.  Check for
3173 	 * changes in the uberblocks on disk if necessary.
3174 	 */
3175 	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
3176 		nvlist_t *nvinfo = fnvlist_lookup_nvlist(config,
3177 		    ZPOOL_CONFIG_LOAD_INFO);
3178 
3179 		if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) &&
3180 		    fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) {
3181 			vdev_uberblock_load(rvd, ub, &mmp_label);
3182 			error = SET_ERROR(EREMOTEIO);
3183 			goto out;
3184 		}
3185 	}
3186 
3187 	import_delay = spa_activity_check_duration(spa, ub);
3188 
3189 	/* Add a small random factor in case of simultaneous imports (0-25%) */
3190 	import_delay += import_delay * random_in_range(250) / 1000;
3191 
3192 	import_expire = gethrtime() + import_delay;
3193 
3194 	while (gethrtime() < import_expire) {
3195 		(void) spa_import_progress_set_mmp_check(spa_guid(spa),
3196 		    NSEC2SEC(import_expire - gethrtime()));
3197 
3198 		vdev_uberblock_load(rvd, ub, &mmp_label);
3199 
3200 		if (txg != ub->ub_txg || timestamp != ub->ub_timestamp ||
3201 		    mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) {
3202 			zfs_dbgmsg("multihost activity detected "
3203 			    "txg %llu ub_txg  %llu "
3204 			    "timestamp %llu ub_timestamp  %llu "
3205 			    "mmp_config %#llx ub_mmp_config %#llx",
3206 			    (u_longlong_t)txg, (u_longlong_t)ub->ub_txg,
3207 			    (u_longlong_t)timestamp,
3208 			    (u_longlong_t)ub->ub_timestamp,
3209 			    (u_longlong_t)mmp_config,
3210 			    (u_longlong_t)ub->ub_mmp_config);
3211 
3212 			error = SET_ERROR(EREMOTEIO);
3213 			break;
3214 		}
3215 
3216 		if (mmp_label) {
3217 			nvlist_free(mmp_label);
3218 			mmp_label = NULL;
3219 		}
3220 
3221 		error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz);
3222 		if (error != -1) {
3223 			error = SET_ERROR(EINTR);
3224 			break;
3225 		}
3226 		error = 0;
3227 	}
3228 
3229 out:
3230 	mutex_exit(&mtx);
3231 	mutex_destroy(&mtx);
3232 	cv_destroy(&cv);
3233 
3234 	/*
3235 	 * If the pool is determined to be active store the status in the
3236 	 * spa->spa_load_info nvlist.  If the remote hostname or hostid are
3237 	 * available from configuration read from disk store them as well.
3238 	 * This allows 'zpool import' to generate a more useful message.
3239 	 *
3240 	 * ZPOOL_CONFIG_MMP_STATE    - observed pool status (mandatory)
3241 	 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool
3242 	 * ZPOOL_CONFIG_MMP_HOSTID   - hostid from the active pool
3243 	 */
3244 	if (error == EREMOTEIO) {
3245 		char *hostname = "<unknown>";
3246 		uint64_t hostid = 0;
3247 
3248 		if (mmp_label) {
3249 			if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) {
3250 				hostname = fnvlist_lookup_string(mmp_label,
3251 				    ZPOOL_CONFIG_HOSTNAME);
3252 				fnvlist_add_string(spa->spa_load_info,
3253 				    ZPOOL_CONFIG_MMP_HOSTNAME, hostname);
3254 			}
3255 
3256 			if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) {
3257 				hostid = fnvlist_lookup_uint64(mmp_label,
3258 				    ZPOOL_CONFIG_HOSTID);
3259 				fnvlist_add_uint64(spa->spa_load_info,
3260 				    ZPOOL_CONFIG_MMP_HOSTID, hostid);
3261 			}
3262 		}
3263 
3264 		fnvlist_add_uint64(spa->spa_load_info,
3265 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE);
3266 		fnvlist_add_uint64(spa->spa_load_info,
3267 		    ZPOOL_CONFIG_MMP_TXG, 0);
3268 
3269 		error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO);
3270 	}
3271 
3272 	if (mmp_label)
3273 		nvlist_free(mmp_label);
3274 
3275 	return (error);
3276 }
3277 
3278 static int
3279 spa_verify_host(spa_t *spa, nvlist_t *mos_config)
3280 {
3281 	uint64_t hostid;
3282 	char *hostname;
3283 	uint64_t myhostid = 0;
3284 
3285 	if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config,
3286 	    ZPOOL_CONFIG_HOSTID, &hostid) == 0) {
3287 		hostname = fnvlist_lookup_string(mos_config,
3288 		    ZPOOL_CONFIG_HOSTNAME);
3289 
3290 		myhostid = zone_get_hostid(NULL);
3291 
3292 		if (hostid != 0 && myhostid != 0 && hostid != myhostid) {
3293 			cmn_err(CE_WARN, "pool '%s' could not be "
3294 			    "loaded as it was last accessed by "
3295 			    "another system (host: %s hostid: 0x%llx). "
3296 			    "See: https://openzfs.github.io/openzfs-docs/msg/"
3297 			    "ZFS-8000-EY",
3298 			    spa_name(spa), hostname, (u_longlong_t)hostid);
3299 			spa_load_failed(spa, "hostid verification failed: pool "
3300 			    "last accessed by host: %s (hostid: 0x%llx)",
3301 			    hostname, (u_longlong_t)hostid);
3302 			return (SET_ERROR(EBADF));
3303 		}
3304 	}
3305 
3306 	return (0);
3307 }
3308 
3309 static int
3310 spa_ld_parse_config(spa_t *spa, spa_import_type_t type)
3311 {
3312 	int error = 0;
3313 	nvlist_t *nvtree, *nvl, *config = spa->spa_config;
3314 	int parse;
3315 	vdev_t *rvd;
3316 	uint64_t pool_guid;
3317 	char *comment;
3318 	char *compatibility;
3319 
3320 	/*
3321 	 * Versioning wasn't explicitly added to the label until later, so if
3322 	 * it's not present treat it as the initial version.
3323 	 */
3324 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
3325 	    &spa->spa_ubsync.ub_version) != 0)
3326 		spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;
3327 
3328 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) {
3329 		spa_load_failed(spa, "invalid config provided: '%s' missing",
3330 		    ZPOOL_CONFIG_POOL_GUID);
3331 		return (SET_ERROR(EINVAL));
3332 	}
3333 
3334 	/*
3335 	 * If we are doing an import, ensure that the pool is not already
3336 	 * imported by checking if its pool guid already exists in the
3337 	 * spa namespace.
3338 	 *
3339 	 * The only case that we allow an already imported pool to be
3340 	 * imported again, is when the pool is checkpointed and we want to
3341 	 * look at its checkpointed state from userland tools like zdb.
3342 	 */
3343 #ifdef _KERNEL
3344 	if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
3345 	    spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
3346 	    spa_guid_exists(pool_guid, 0)) {
3347 #else
3348 	if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
3349 	    spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
3350 	    spa_guid_exists(pool_guid, 0) &&
3351 	    !spa_importing_readonly_checkpoint(spa)) {
3352 #endif
3353 		spa_load_failed(spa, "a pool with guid %llu is already open",
3354 		    (u_longlong_t)pool_guid);
3355 		return (SET_ERROR(EEXIST));
3356 	}
3357 
3358 	spa->spa_config_guid = pool_guid;
3359 
3360 	nvlist_free(spa->spa_load_info);
3361 	spa->spa_load_info = fnvlist_alloc();
3362 
3363 	ASSERT(spa->spa_comment == NULL);
3364 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
3365 		spa->spa_comment = spa_strdup(comment);
3366 
3367 	ASSERT(spa->spa_compatibility == NULL);
3368 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY,
3369 	    &compatibility) == 0)
3370 		spa->spa_compatibility = spa_strdup(compatibility);
3371 
3372 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
3373 	    &spa->spa_config_txg);
3374 
3375 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0)
3376 		spa->spa_config_splitting = fnvlist_dup(nvl);
3377 
3378 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) {
3379 		spa_load_failed(spa, "invalid config provided: '%s' missing",
3380 		    ZPOOL_CONFIG_VDEV_TREE);
3381 		return (SET_ERROR(EINVAL));
3382 	}
3383 
3384 	/*
3385 	 * Create "The Godfather" zio to hold all async IOs
3386 	 */
3387 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
3388 	    KM_SLEEP);
3389 	for (int i = 0; i < max_ncpus; i++) {
3390 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
3391 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
3392 		    ZIO_FLAG_GODFATHER);
3393 	}
3394 
3395 	/*
3396 	 * Parse the configuration into a vdev tree.  We explicitly set the
3397 	 * value that will be returned by spa_version() since parsing the
3398 	 * configuration requires knowing the version number.
3399 	 */
3400 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3401 	parse = (type == SPA_IMPORT_EXISTING ?
3402 	    VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT);
3403 	error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse);
3404 	spa_config_exit(spa, SCL_ALL, FTAG);
3405 
3406 	if (error != 0) {
3407 		spa_load_failed(spa, "unable to parse config [error=%d]",
3408 		    error);
3409 		return (error);
3410 	}
3411 
3412 	ASSERT(spa->spa_root_vdev == rvd);
3413 	ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
3414 	ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);
3415 
3416 	if (type != SPA_IMPORT_ASSEMBLE) {
3417 		ASSERT(spa_guid(spa) == pool_guid);
3418 	}
3419 
3420 	return (0);
3421 }
3422 
3423 /*
3424  * Recursively open all vdevs in the vdev tree. This function is called twice:
3425  * first with the untrusted config, then with the trusted config.
3426  */
3427 static int
3428 spa_ld_open_vdevs(spa_t *spa)
3429 {
3430 	int error = 0;
3431 
3432 	/*
3433 	 * spa_missing_tvds_allowed defines how many top-level vdevs can be
3434 	 * missing/unopenable for the root vdev to be still considered openable.
3435 	 */
3436 	if (spa->spa_trust_config) {
3437 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds;
3438 	} else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) {
3439 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile;
3440 	} else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) {
3441 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan;
3442 	} else {
3443 		spa->spa_missing_tvds_allowed = 0;
3444 	}
3445 
3446 	spa->spa_missing_tvds_allowed =
3447 	    MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed);
3448 
3449 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3450 	error = vdev_open(spa->spa_root_vdev);
3451 	spa_config_exit(spa, SCL_ALL, FTAG);
3452 
3453 	if (spa->spa_missing_tvds != 0) {
3454 		spa_load_note(spa, "vdev tree has %lld missing top-level "
3455 		    "vdevs.", (u_longlong_t)spa->spa_missing_tvds);
3456 		if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) {
3457 			/*
3458 			 * Although theoretically we could allow users to open
3459 			 * incomplete pools in RW mode, we'd need to add a lot
3460 			 * of extra logic (e.g. adjust pool space to account
3461 			 * for missing vdevs).
3462 			 * This limitation also prevents users from accidentally
3463 			 * opening the pool in RW mode during data recovery and
3464 			 * damaging it further.
3465 			 */
3466 			spa_load_note(spa, "pools with missing top-level "
3467 			    "vdevs can only be opened in read-only mode.");
3468 			error = SET_ERROR(ENXIO);
3469 		} else {
3470 			spa_load_note(spa, "current settings allow for maximum "
3471 			    "%lld missing top-level vdevs at this stage.",
3472 			    (u_longlong_t)spa->spa_missing_tvds_allowed);
3473 		}
3474 	}
3475 	if (error != 0) {
3476 		spa_load_failed(spa, "unable to open vdev tree [error=%d]",
3477 		    error);
3478 	}
3479 	if (spa->spa_missing_tvds != 0 || error != 0)
3480 		vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2);
3481 
3482 	return (error);
3483 }
3484 
3485 /*
3486  * We need to validate the vdev labels against the configuration that
3487  * we have in hand. This function is called twice: first with an untrusted
3488  * config, then with a trusted config. The validation is more strict when the
3489  * config is trusted.
3490  */
3491 static int
3492 spa_ld_validate_vdevs(spa_t *spa)
3493 {
3494 	int error = 0;
3495 	vdev_t *rvd = spa->spa_root_vdev;
3496 
3497 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3498 	error = vdev_validate(rvd);
3499 	spa_config_exit(spa, SCL_ALL, FTAG);
3500 
3501 	if (error != 0) {
3502 		spa_load_failed(spa, "vdev_validate failed [error=%d]", error);
3503 		return (error);
3504 	}
3505 
3506 	if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
3507 		spa_load_failed(spa, "cannot open vdev tree after invalidating "
3508 		    "some vdevs");
3509 		vdev_dbgmsg_print_tree(rvd, 2);
3510 		return (SET_ERROR(ENXIO));
3511 	}
3512 
3513 	return (0);
3514 }
3515 
3516 static void
3517 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub)
3518 {
3519 	spa->spa_state = POOL_STATE_ACTIVE;
3520 	spa->spa_ubsync = spa->spa_uberblock;
3521 	spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
3522 	    TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
3523 	spa->spa_first_txg = spa->spa_last_ubsync_txg ?
3524 	    spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
3525 	spa->spa_claim_max_txg = spa->spa_first_txg;
3526 	spa->spa_prev_software_version = ub->ub_software_version;
3527 }
3528 
3529 static int
3530 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type)
3531 {
3532 	vdev_t *rvd = spa->spa_root_vdev;
3533 	nvlist_t *label;
3534 	uberblock_t *ub = &spa->spa_uberblock;
3535 	boolean_t activity_check = B_FALSE;
3536 
3537 	/*
3538 	 * If we are opening the checkpointed state of the pool by
3539 	 * rewinding to it, at this point we will have written the
3540 	 * checkpointed uberblock to the vdev labels, so searching
3541 	 * the labels will find the right uberblock.  However, if
3542 	 * we are opening the checkpointed state read-only, we have
3543 	 * not modified the labels. Therefore, we must ignore the
3544 	 * labels and continue using the spa_uberblock that was set
3545 	 * by spa_ld_checkpoint_rewind.
3546 	 *
3547 	 * Note that it would be fine to ignore the labels when
3548 	 * rewinding (opening writeable) as well. However, if we
3549 	 * crash just after writing the labels, we will end up
3550 	 * searching the labels. Doing so in the common case means
3551 	 * that this code path gets exercised normally, rather than
3552 	 * just in the edge case.
3553 	 */
3554 	if (ub->ub_checkpoint_txg != 0 &&
3555 	    spa_importing_readonly_checkpoint(spa)) {
3556 		spa_ld_select_uberblock_done(spa, ub);
3557 		return (0);
3558 	}
3559 
3560 	/*
3561 	 * Find the best uberblock.
3562 	 */
3563 	vdev_uberblock_load(rvd, ub, &label);
3564 
3565 	/*
3566 	 * If we weren't able to find a single valid uberblock, return failure.
3567 	 */
3568 	if (ub->ub_txg == 0) {
3569 		nvlist_free(label);
3570 		spa_load_failed(spa, "no valid uberblock found");
3571 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO));
3572 	}
3573 
3574 	if (spa->spa_load_max_txg != UINT64_MAX) {
3575 		(void) spa_import_progress_set_max_txg(spa_guid(spa),
3576 		    (u_longlong_t)spa->spa_load_max_txg);
3577 	}
3578 	spa_load_note(spa, "using uberblock with txg=%llu",
3579 	    (u_longlong_t)ub->ub_txg);
3580 
3581 
3582 	/*
3583 	 * For pools which have the multihost property on determine if the
3584 	 * pool is truly inactive and can be safely imported.  Prevent
3585 	 * hosts which don't have a hostid set from importing the pool.
3586 	 */
3587 	activity_check = spa_activity_check_required(spa, ub, label,
3588 	    spa->spa_config);
3589 	if (activity_check) {
3590 		if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay &&
3591 		    spa_get_hostid(spa) == 0) {
3592 			nvlist_free(label);
3593 			fnvlist_add_uint64(spa->spa_load_info,
3594 			    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
3595 			return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
3596 		}
3597 
3598 		int error = spa_activity_check(spa, ub, spa->spa_config);
3599 		if (error) {
3600 			nvlist_free(label);
3601 			return (error);
3602 		}
3603 
3604 		fnvlist_add_uint64(spa->spa_load_info,
3605 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE);
3606 		fnvlist_add_uint64(spa->spa_load_info,
3607 		    ZPOOL_CONFIG_MMP_TXG, ub->ub_txg);
3608 		fnvlist_add_uint16(spa->spa_load_info,
3609 		    ZPOOL_CONFIG_MMP_SEQ,
3610 		    (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0));
3611 	}
3612 
3613 	/*
3614 	 * If the pool has an unsupported version we can't open it.
3615 	 */
3616 	if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
3617 		nvlist_free(label);
3618 		spa_load_failed(spa, "version %llu is not supported",
3619 		    (u_longlong_t)ub->ub_version);
3620 		return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP));
3621 	}
3622 
3623 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
3624 		nvlist_t *features;
3625 
3626 		/*
3627 		 * If we weren't able to find what's necessary for reading the
3628 		 * MOS in the label, return failure.
3629 		 */
3630 		if (label == NULL) {
3631 			spa_load_failed(spa, "label config unavailable");
3632 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
3633 			    ENXIO));
3634 		}
3635 
3636 		if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ,
3637 		    &features) != 0) {
3638 			nvlist_free(label);
3639 			spa_load_failed(spa, "invalid label: '%s' missing",
3640 			    ZPOOL_CONFIG_FEATURES_FOR_READ);
3641 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
3642 			    ENXIO));
3643 		}
3644 
3645 		/*
3646 		 * Update our in-core representation with the definitive values
3647 		 * from the label.
3648 		 */
3649 		nvlist_free(spa->spa_label_features);
3650 		VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0);
3651 	}
3652 
3653 	nvlist_free(label);
3654 
3655 	/*
3656 	 * Look through entries in the label nvlist's features_for_read. If
3657 	 * there is a feature listed there which we don't understand then we
3658 	 * cannot open a pool.
3659 	 */
3660 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
3661 		nvlist_t *unsup_feat;
3662 
3663 		VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) ==
3664 		    0);
3665 
3666 		for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
3667 		    NULL); nvp != NULL;
3668 		    nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
3669 			if (!zfeature_is_supported(nvpair_name(nvp))) {
3670 				VERIFY(nvlist_add_string(unsup_feat,
3671 				    nvpair_name(nvp), "") == 0);
3672 			}
3673 		}
3674 
3675 		if (!nvlist_empty(unsup_feat)) {
3676 			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
3677 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0);
3678 			nvlist_free(unsup_feat);
3679 			spa_load_failed(spa, "some features are unsupported");
3680 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
3681 			    ENOTSUP));
3682 		}
3683 
3684 		nvlist_free(unsup_feat);
3685 	}
3686 
3687 	if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
3688 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3689 		spa_try_repair(spa, spa->spa_config);
3690 		spa_config_exit(spa, SCL_ALL, FTAG);
3691 		nvlist_free(spa->spa_config_splitting);
3692 		spa->spa_config_splitting = NULL;
3693 	}
3694 
3695 	/*
3696 	 * Initialize internal SPA structures.
3697 	 */
3698 	spa_ld_select_uberblock_done(spa, ub);
3699 
3700 	return (0);
3701 }
3702 
3703 static int
3704 spa_ld_open_rootbp(spa_t *spa)
3705 {
3706 	int error = 0;
3707 	vdev_t *rvd = spa->spa_root_vdev;
3708 
3709 	error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
3710 	if (error != 0) {
3711 		spa_load_failed(spa, "unable to open rootbp in dsl_pool_init "
3712 		    "[error=%d]", error);
3713 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3714 	}
3715 	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
3716 
3717 	return (0);
3718 }
3719 
3720 static int
3721 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type,
3722     boolean_t reloading)
3723 {
3724 	vdev_t *mrvd, *rvd = spa->spa_root_vdev;
3725 	nvlist_t *nv, *mos_config, *policy;
3726 	int error = 0, copy_error;
3727 	uint64_t healthy_tvds, healthy_tvds_mos;
3728 	uint64_t mos_config_txg;
3729 
3730 	if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE)
3731 	    != 0)
3732 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3733 
3734 	/*
3735 	 * If we're assembling a pool from a split, the config provided is
3736 	 * already trusted so there is nothing to do.
3737 	 */
3738 	if (type == SPA_IMPORT_ASSEMBLE)
3739 		return (0);
3740 
3741 	healthy_tvds = spa_healthy_core_tvds(spa);
3742 
3743 	if (load_nvlist(spa, spa->spa_config_object, &mos_config)
3744 	    != 0) {
3745 		spa_load_failed(spa, "unable to retrieve MOS config");
3746 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3747 	}
3748 
3749 	/*
3750 	 * If we are doing an open, pool owner wasn't verified yet, thus do
3751 	 * the verification here.
3752 	 */
3753 	if (spa->spa_load_state == SPA_LOAD_OPEN) {
3754 		error = spa_verify_host(spa, mos_config);
3755 		if (error != 0) {
3756 			nvlist_free(mos_config);
3757 			return (error);
3758 		}
3759 	}
3760 
3761 	nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE);
3762 
3763 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3764 
3765 	/*
3766 	 * Build a new vdev tree from the trusted config
3767 	 */
3768 	error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD);
3769 	if (error != 0) {
3770 		nvlist_free(mos_config);
3771 		spa_config_exit(spa, SCL_ALL, FTAG);
3772 		spa_load_failed(spa, "spa_config_parse failed [error=%d]",
3773 		    error);
3774 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
3775 	}
3776 
3777 	/*
3778 	 * Vdev paths in the MOS may be obsolete. If the untrusted config was
3779 	 * obtained by scanning /dev/dsk, then it will have the right vdev
3780 	 * paths. We update the trusted MOS config with this information.
3781 	 * We first try to copy the paths with vdev_copy_path_strict, which
3782 	 * succeeds only when both configs have exactly the same vdev tree.
3783 	 * If that fails, we fall back to a more flexible method that has a
3784 	 * best effort policy.
3785 	 */
3786 	copy_error = vdev_copy_path_strict(rvd, mrvd);
3787 	if (copy_error != 0 || spa_load_print_vdev_tree) {
3788 		spa_load_note(spa, "provided vdev tree:");
3789 		vdev_dbgmsg_print_tree(rvd, 2);
3790 		spa_load_note(spa, "MOS vdev tree:");
3791 		vdev_dbgmsg_print_tree(mrvd, 2);
3792 	}
3793 	if (copy_error != 0) {
3794 		spa_load_note(spa, "vdev_copy_path_strict failed, falling "
3795 		    "back to vdev_copy_path_relaxed");
3796 		vdev_copy_path_relaxed(rvd, mrvd);
3797 	}
3798 
3799 	vdev_close(rvd);
3800 	vdev_free(rvd);
3801 	spa->spa_root_vdev = mrvd;
3802 	rvd = mrvd;
3803 	spa_config_exit(spa, SCL_ALL, FTAG);
3804 
3805 	/*
3806 	 * We will use spa_config if we decide to reload the spa or if spa_load
3807 	 * fails and we rewind. We must thus regenerate the config using the
3808 	 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to
3809 	 * pass settings on how to load the pool and is not stored in the MOS.
3810 	 * We copy it over to our new, trusted config.
3811 	 */
3812 	mos_config_txg = fnvlist_lookup_uint64(mos_config,
3813 	    ZPOOL_CONFIG_POOL_TXG);
3814 	nvlist_free(mos_config);
3815 	mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE);
3816 	if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY,
3817 	    &policy) == 0)
3818 		fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy);
3819 	spa_config_set(spa, mos_config);
3820 	spa->spa_config_source = SPA_CONFIG_SRC_MOS;
3821 
3822 	/*
3823 	 * Now that we got the config from the MOS, we should be more strict
3824 	 * in checking blkptrs and can make assumptions about the consistency
3825 	 * of the vdev tree. spa_trust_config must be set to true before opening
3826 	 * vdevs in order for them to be writeable.
3827 	 */
3828 	spa->spa_trust_config = B_TRUE;
3829 
3830 	/*
3831 	 * Open and validate the new vdev tree
3832 	 */
3833 	error = spa_ld_open_vdevs(spa);
3834 	if (error != 0)
3835 		return (error);
3836 
3837 	error = spa_ld_validate_vdevs(spa);
3838 	if (error != 0)
3839 		return (error);
3840 
3841 	if (copy_error != 0 || spa_load_print_vdev_tree) {
3842 		spa_load_note(spa, "final vdev tree:");
3843 		vdev_dbgmsg_print_tree(rvd, 2);
3844 	}
3845 
3846 	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT &&
3847 	    !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) {
3848 		/*
3849 		 * Sanity check to make sure that we are indeed loading the
3850 		 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds
3851 		 * in the config provided and they happened to be the only ones
3852 		 * to have the latest uberblock, we could involuntarily perform
3853 		 * an extreme rewind.
3854 		 */
3855 		healthy_tvds_mos = spa_healthy_core_tvds(spa);
3856 		if (healthy_tvds_mos - healthy_tvds >=
3857 		    SPA_SYNC_MIN_VDEVS) {
3858 			spa_load_note(spa, "config provided misses too many "
3859 			    "top-level vdevs compared to MOS (%lld vs %lld). ",
3860 			    (u_longlong_t)healthy_tvds,
3861 			    (u_longlong_t)healthy_tvds_mos);
3862 			spa_load_note(spa, "vdev tree:");
3863 			vdev_dbgmsg_print_tree(rvd, 2);
3864 			if (reloading) {
3865 				spa_load_failed(spa, "config was already "
3866 				    "provided from MOS. Aborting.");
3867 				return (spa_vdev_err(rvd,
3868 				    VDEV_AUX_CORRUPT_DATA, EIO));
3869 			}
3870 			spa_load_note(spa, "spa must be reloaded using MOS "
3871 			    "config");
3872 			return (SET_ERROR(EAGAIN));
3873 		}
3874 	}
3875 
3876 	error = spa_check_for_missing_logs(spa);
3877 	if (error != 0)
3878 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO));
3879 
3880 	if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) {
3881 		spa_load_failed(spa, "uberblock guid sum doesn't match MOS "
3882 		    "guid sum (%llu != %llu)",
3883 		    (u_longlong_t)spa->spa_uberblock.ub_guid_sum,
3884 		    (u_longlong_t)rvd->vdev_guid_sum);
3885 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM,
3886 		    ENXIO));
3887 	}
3888 
3889 	return (0);
3890 }
3891 
3892 static int
3893 spa_ld_open_indirect_vdev_metadata(spa_t *spa)
3894 {
3895 	int error = 0;
3896 	vdev_t *rvd = spa->spa_root_vdev;
3897 
3898 	/*
3899 	 * Everything that we read before spa_remove_init() must be stored
3900 	 * on concreted vdevs.  Therefore we do this as early as possible.
3901 	 */
3902 	error = spa_remove_init(spa);
3903 	if (error != 0) {
3904 		spa_load_failed(spa, "spa_remove_init failed [error=%d]",
3905 		    error);
3906 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3907 	}
3908 
3909 	/*
3910 	 * Retrieve information needed to condense indirect vdev mappings.
3911 	 */
3912 	error = spa_condense_init(spa);
3913 	if (error != 0) {
3914 		spa_load_failed(spa, "spa_condense_init failed [error=%d]",
3915 		    error);
3916 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
3917 	}
3918 
3919 	return (0);
3920 }
3921 
3922 static int
3923 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep)
3924 {
3925 	int error = 0;
3926 	vdev_t *rvd = spa->spa_root_vdev;
3927 
3928 	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
3929 		boolean_t missing_feat_read = B_FALSE;
3930 		nvlist_t *unsup_feat, *enabled_feat;
3931 
3932 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
3933 		    &spa->spa_feat_for_read_obj, B_TRUE) != 0) {
3934 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3935 		}
3936 
3937 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE,
3938 		    &spa->spa_feat_for_write_obj, B_TRUE) != 0) {
3939 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3940 		}
3941 
3942 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS,
3943 		    &spa->spa_feat_desc_obj, B_TRUE) != 0) {
3944 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
3945 		}
3946 
3947 		enabled_feat = fnvlist_alloc();
3948 		unsup_feat = fnvlist_alloc();
3949 
3950 		if (!spa_features_check(spa, B_FALSE,
3951 		    unsup_feat, enabled_feat))
3952 			missing_feat_read = B_TRUE;
3953 
3954 		if (spa_writeable(spa) ||
3955 		    spa->spa_load_state == SPA_LOAD_TRYIMPORT) {
3956 			if (!spa_features_check(spa, B_TRUE,
3957 			    unsup_feat, enabled_feat)) {
3958 				*missing_feat_writep = B_TRUE;
3959 			}
3960 		}
3961 
3962 		fnvlist_add_nvlist(spa->spa_load_info,
3963 		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
3964 
3965 		if (!nvlist_empty(unsup_feat)) {
3966 			fnvlist_add_nvlist(spa->spa_load_info,
3967 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
3968 		}
3969 
3970 		fnvlist_free(enabled_feat);
3971 		fnvlist_free(unsup_feat);
3972 
3973 		if (!missing_feat_read) {
3974 			fnvlist_add_boolean(spa->spa_load_info,
3975 			    ZPOOL_CONFIG_CAN_RDONLY);
3976 		}
3977 
3978 		/*
3979 		 * If the state is SPA_LOAD_TRYIMPORT, our objective is
3980 		 * twofold: to determine whether the pool is available for
3981 		 * import in read-write mode and (if it is not) whether the
3982 		 * pool is available for import in read-only mode. If the pool
3983 		 * is available for import in read-write mode, it is displayed
3984 		 * as available in userland; if it is not available for import
3985 		 * in read-only mode, it is displayed as unavailable in
3986 		 * userland. If the pool is available for import in read-only
3987 		 * mode but not read-write mode, it is displayed as unavailable
3988 		 * in userland with a special note that the pool is actually
3989 		 * available for open in read-only mode.
3990 		 *
3991 		 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are
3992 		 * missing a feature for write, we must first determine whether
3993 		 * the pool can be opened read-only before returning to
3994 		 * userland in order to know whether to display the
3995 		 * abovementioned note.
3996 		 */
3997 		if (missing_feat_read || (*missing_feat_writep &&
3998 		    spa_writeable(spa))) {
3999 			spa_load_failed(spa, "pool uses unsupported features");
4000 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
4001 			    ENOTSUP));
4002 		}
4003 
4004 		/*
4005 		 * Load refcounts for ZFS features from disk into an in-memory
4006 		 * cache during SPA initialization.
4007 		 */
4008 		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
4009 			uint64_t refcount;
4010 
4011 			error = feature_get_refcount_from_disk(spa,
4012 			    &spa_feature_table[i], &refcount);
4013 			if (error == 0) {
4014 				spa->spa_feat_refcount_cache[i] = refcount;
4015 			} else if (error == ENOTSUP) {
4016 				spa->spa_feat_refcount_cache[i] =
4017 				    SPA_FEATURE_DISABLED;
4018 			} else {
4019 				spa_load_failed(spa, "error getting refcount "
4020 				    "for feature %s [error=%d]",
4021 				    spa_feature_table[i].fi_guid, error);
4022 				return (spa_vdev_err(rvd,
4023 				    VDEV_AUX_CORRUPT_DATA, EIO));
4024 			}
4025 		}
4026 	}
4027 
4028 	if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) {
4029 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG,
4030 		    &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0)
4031 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4032 	}
4033 
4034 	/*
4035 	 * Encryption was added before bookmark_v2, even though bookmark_v2
4036 	 * is now a dependency. If this pool has encryption enabled without
4037 	 * bookmark_v2, trigger an errata message.
4038 	 */
4039 	if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) &&
4040 	    !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) {
4041 		spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
4042 	}
4043 
4044 	return (0);
4045 }
4046 
4047 static int
4048 spa_ld_load_special_directories(spa_t *spa)
4049 {
4050 	int error = 0;
4051 	vdev_t *rvd = spa->spa_root_vdev;
4052 
4053 	spa->spa_is_initializing = B_TRUE;
4054 	error = dsl_pool_open(spa->spa_dsl_pool);
4055 	spa->spa_is_initializing = B_FALSE;
4056 	if (error != 0) {
4057 		spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error);
4058 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4059 	}
4060 
4061 	return (0);
4062 }
4063 
4064 static int
4065 spa_ld_get_props(spa_t *spa)
4066 {
4067 	int error = 0;
4068 	uint64_t obj;
4069 	vdev_t *rvd = spa->spa_root_vdev;
4070 
4071 	/* Grab the checksum salt from the MOS. */
4072 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4073 	    DMU_POOL_CHECKSUM_SALT, 1,
4074 	    sizeof (spa->spa_cksum_salt.zcs_bytes),
4075 	    spa->spa_cksum_salt.zcs_bytes);
4076 	if (error == ENOENT) {
4077 		/* Generate a new salt for subsequent use */
4078 		(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
4079 		    sizeof (spa->spa_cksum_salt.zcs_bytes));
4080 	} else if (error != 0) {
4081 		spa_load_failed(spa, "unable to retrieve checksum salt from "
4082 		    "MOS [error=%d]", error);
4083 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4084 	}
4085 
4086 	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0)
4087 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4088 	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
4089 	if (error != 0) {
4090 		spa_load_failed(spa, "error opening deferred-frees bpobj "
4091 		    "[error=%d]", error);
4092 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4093 	}
4094 
4095 	/*
4096 	 * Load the bit that tells us to use the new accounting function
4097 	 * (raid-z deflation).  If we have an older pool, this will not
4098 	 * be present.
4099 	 */
4100 	error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE);
4101 	if (error != 0 && error != ENOENT)
4102 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4103 
4104 	error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION,
4105 	    &spa->spa_creation_version, B_FALSE);
4106 	if (error != 0 && error != ENOENT)
4107 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4108 
4109 	/*
4110 	 * Load the persistent error log.  If we have an older pool, this will
4111 	 * not be present.
4112 	 */
4113 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last,
4114 	    B_FALSE);
4115 	if (error != 0 && error != ENOENT)
4116 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4117 
4118 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB,
4119 	    &spa->spa_errlog_scrub, B_FALSE);
4120 	if (error != 0 && error != ENOENT)
4121 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4122 
4123 	/*
4124 	 * Load the livelist deletion field. If a livelist is queued for
4125 	 * deletion, indicate that in the spa
4126 	 */
4127 	error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES,
4128 	    &spa->spa_livelists_to_delete, B_FALSE);
4129 	if (error != 0 && error != ENOENT)
4130 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4131 
4132 	/*
4133 	 * Load the history object.  If we have an older pool, this
4134 	 * will not be present.
4135 	 */
4136 	error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE);
4137 	if (error != 0 && error != ENOENT)
4138 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4139 
4140 	/*
4141 	 * Load the per-vdev ZAP map. If we have an older pool, this will not
4142 	 * be present; in this case, defer its creation to a later time to
4143 	 * avoid dirtying the MOS this early / out of sync context. See
4144 	 * spa_sync_config_object.
4145 	 */
4146 
4147 	/* The sentinel is only available in the MOS config. */
4148 	nvlist_t *mos_config;
4149 	if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) {
4150 		spa_load_failed(spa, "unable to retrieve MOS config");
4151 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4152 	}
4153 
4154 	error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP,
4155 	    &spa->spa_all_vdev_zaps, B_FALSE);
4156 
4157 	if (error == ENOENT) {
4158 		VERIFY(!nvlist_exists(mos_config,
4159 		    ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
4160 		spa->spa_avz_action = AVZ_ACTION_INITIALIZE;
4161 		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4162 	} else if (error != 0) {
4163 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4164 	} else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) {
4165 		/*
4166 		 * An older version of ZFS overwrote the sentinel value, so
4167 		 * we have orphaned per-vdev ZAPs in the MOS. Defer their
4168 		 * destruction to later; see spa_sync_config_object.
4169 		 */
4170 		spa->spa_avz_action = AVZ_ACTION_DESTROY;
4171 		/*
4172 		 * We're assuming that no vdevs have had their ZAPs created
4173 		 * before this. Better be sure of it.
4174 		 */
4175 		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4176 	}
4177 	nvlist_free(mos_config);
4178 
4179 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
4180 
4181 	error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object,
4182 	    B_FALSE);
4183 	if (error && error != ENOENT)
4184 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4185 
4186 	if (error == 0) {
4187 		uint64_t autoreplace = 0;
4188 
4189 		spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
4190 		spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace);
4191 		spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
4192 		spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
4193 		spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
4194 		spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost);
4195 		spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim);
4196 		spa->spa_autoreplace = (autoreplace != 0);
4197 	}
4198 
4199 	/*
4200 	 * If we are importing a pool with missing top-level vdevs,
4201 	 * we enforce that the pool doesn't panic or get suspended on
4202 	 * error since the likelihood of missing data is extremely high.
4203 	 */
4204 	if (spa->spa_missing_tvds > 0 &&
4205 	    spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE &&
4206 	    spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4207 		spa_load_note(spa, "forcing failmode to 'continue' "
4208 		    "as some top level vdevs are missing");
4209 		spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE;
4210 	}
4211 
4212 	return (0);
4213 }
4214 
4215 static int
4216 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type)
4217 {
4218 	int error = 0;
4219 	vdev_t *rvd = spa->spa_root_vdev;
4220 
4221 	/*
4222 	 * If we're assembling the pool from the split-off vdevs of
4223 	 * an existing pool, we don't want to attach the spares & cache
4224 	 * devices.
4225 	 */
4226 
4227 	/*
4228 	 * Load any hot spares for this pool.
4229 	 */
4230 	error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object,
4231 	    B_FALSE);
4232 	if (error != 0 && error != ENOENT)
4233 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4234 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
4235 		ASSERT(spa_version(spa) >= SPA_VERSION_SPARES);
4236 		if (load_nvlist(spa, spa->spa_spares.sav_object,
4237 		    &spa->spa_spares.sav_config) != 0) {
4238 			spa_load_failed(spa, "error loading spares nvlist");
4239 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4240 		}
4241 
4242 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4243 		spa_load_spares(spa);
4244 		spa_config_exit(spa, SCL_ALL, FTAG);
4245 	} else if (error == 0) {
4246 		spa->spa_spares.sav_sync = B_TRUE;
4247 	}
4248 
4249 	/*
4250 	 * Load any level 2 ARC devices for this pool.
4251 	 */
4252 	error = spa_dir_prop(spa, DMU_POOL_L2CACHE,
4253 	    &spa->spa_l2cache.sav_object, B_FALSE);
4254 	if (error != 0 && error != ENOENT)
4255 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4256 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
4257 		ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE);
4258 		if (load_nvlist(spa, spa->spa_l2cache.sav_object,
4259 		    &spa->spa_l2cache.sav_config) != 0) {
4260 			spa_load_failed(spa, "error loading l2cache nvlist");
4261 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4262 		}
4263 
4264 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4265 		spa_load_l2cache(spa);
4266 		spa_config_exit(spa, SCL_ALL, FTAG);
4267 	} else if (error == 0) {
4268 		spa->spa_l2cache.sav_sync = B_TRUE;
4269 	}
4270 
4271 	return (0);
4272 }
4273 
4274 static int
4275 spa_ld_load_vdev_metadata(spa_t *spa)
4276 {
4277 	int error = 0;
4278 	vdev_t *rvd = spa->spa_root_vdev;
4279 
4280 	/*
4281 	 * If the 'multihost' property is set, then never allow a pool to
4282 	 * be imported when the system hostid is zero.  The exception to
4283 	 * this rule is zdb which is always allowed to access pools.
4284 	 */
4285 	if (spa_multihost(spa) && spa_get_hostid(spa) == 0 &&
4286 	    (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) {
4287 		fnvlist_add_uint64(spa->spa_load_info,
4288 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
4289 		return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
4290 	}
4291 
4292 	/*
4293 	 * If the 'autoreplace' property is set, then post a resource notifying
4294 	 * the ZFS DE that it should not issue any faults for unopenable
4295 	 * devices.  We also iterate over the vdevs, and post a sysevent for any
4296 	 * unopenable vdevs so that the normal autoreplace handler can take
4297 	 * over.
4298 	 */
4299 	if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4300 		spa_check_removed(spa->spa_root_vdev);
4301 		/*
4302 		 * For the import case, this is done in spa_import(), because
4303 		 * at this point we're using the spare definitions from
4304 		 * the MOS config, not necessarily from the userland config.
4305 		 */
4306 		if (spa->spa_load_state != SPA_LOAD_IMPORT) {
4307 			spa_aux_check_removed(&spa->spa_spares);
4308 			spa_aux_check_removed(&spa->spa_l2cache);
4309 		}
4310 	}
4311 
4312 	/*
4313 	 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc.
4314 	 */
4315 	error = vdev_load(rvd);
4316 	if (error != 0) {
4317 		spa_load_failed(spa, "vdev_load failed [error=%d]", error);
4318 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4319 	}
4320 
4321 	error = spa_ld_log_spacemaps(spa);
4322 	if (error != 0) {
4323 		spa_load_failed(spa, "spa_ld_log_sm_data failed [error=%d]",
4324 		    error);
4325 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4326 	}
4327 
4328 	/*
4329 	 * Propagate the leaf DTLs we just loaded all the way up the vdev tree.
4330 	 */
4331 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4332 	vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE);
4333 	spa_config_exit(spa, SCL_ALL, FTAG);
4334 
4335 	return (0);
4336 }
4337 
4338 static int
4339 spa_ld_load_dedup_tables(spa_t *spa)
4340 {
4341 	int error = 0;
4342 	vdev_t *rvd = spa->spa_root_vdev;
4343 
4344 	error = ddt_load(spa);
4345 	if (error != 0) {
4346 		spa_load_failed(spa, "ddt_load failed [error=%d]", error);
4347 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4348 	}
4349 
4350 	return (0);
4351 }
4352 
4353 static int
4354 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, char **ereport)
4355 {
4356 	vdev_t *rvd = spa->spa_root_vdev;
4357 
4358 	if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) {
4359 		boolean_t missing = spa_check_logs(spa);
4360 		if (missing) {
4361 			if (spa->spa_missing_tvds != 0) {
4362 				spa_load_note(spa, "spa_check_logs failed "
4363 				    "so dropping the logs");
4364 			} else {
4365 				*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
4366 				spa_load_failed(spa, "spa_check_logs failed");
4367 				return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG,
4368 				    ENXIO));
4369 			}
4370 		}
4371 	}
4372 
4373 	return (0);
4374 }
4375 
4376 static int
4377 spa_ld_verify_pool_data(spa_t *spa)
4378 {
4379 	int error = 0;
4380 	vdev_t *rvd = spa->spa_root_vdev;
4381 
4382 	/*
4383 	 * We've successfully opened the pool, verify that we're ready
4384 	 * to start pushing transactions.
4385 	 */
4386 	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4387 		error = spa_load_verify(spa);
4388 		if (error != 0) {
4389 			spa_load_failed(spa, "spa_load_verify failed "
4390 			    "[error=%d]", error);
4391 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
4392 			    error));
4393 		}
4394 	}
4395 
4396 	return (0);
4397 }
4398 
4399 static void
4400 spa_ld_claim_log_blocks(spa_t *spa)
4401 {
4402 	dmu_tx_t *tx;
4403 	dsl_pool_t *dp = spa_get_dsl(spa);
4404 
4405 	/*
4406 	 * Claim log blocks that haven't been committed yet.
4407 	 * This must all happen in a single txg.
4408 	 * Note: spa_claim_max_txg is updated by spa_claim_notify(),
4409 	 * invoked from zil_claim_log_block()'s i/o done callback.
4410 	 * Price of rollback is that we abandon the log.
4411 	 */
4412 	spa->spa_claiming = B_TRUE;
4413 
4414 	tx = dmu_tx_create_assigned(dp, spa_first_txg(spa));
4415 	(void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
4416 	    zil_claim, tx, DS_FIND_CHILDREN);
4417 	dmu_tx_commit(tx);
4418 
4419 	spa->spa_claiming = B_FALSE;
4420 
4421 	spa_set_log_state(spa, SPA_LOG_GOOD);
4422 }
4423 
4424 static void
4425 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg,
4426     boolean_t update_config_cache)
4427 {
4428 	vdev_t *rvd = spa->spa_root_vdev;
4429 	int need_update = B_FALSE;
4430 
4431 	/*
4432 	 * If the config cache is stale, or we have uninitialized
4433 	 * metaslabs (see spa_vdev_add()), then update the config.
4434 	 *
4435 	 * If this is a verbatim import, trust the current
4436 	 * in-core spa_config and update the disk labels.
4437 	 */
4438 	if (update_config_cache || config_cache_txg != spa->spa_config_txg ||
4439 	    spa->spa_load_state == SPA_LOAD_IMPORT ||
4440 	    spa->spa_load_state == SPA_LOAD_RECOVER ||
4441 	    (spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
4442 		need_update = B_TRUE;
4443 
4444 	for (int c = 0; c < rvd->vdev_children; c++)
4445 		if (rvd->vdev_child[c]->vdev_ms_array == 0)
4446 			need_update = B_TRUE;
4447 
4448 	/*
4449 	 * Update the config cache asynchronously in case we're the
4450 	 * root pool, in which case the config cache isn't writable yet.
4451 	 */
4452 	if (need_update)
4453 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
4454 }
4455 
4456 static void
4457 spa_ld_prepare_for_reload(spa_t *spa)
4458 {
4459 	spa_mode_t mode = spa->spa_mode;
4460 	int async_suspended = spa->spa_async_suspended;
4461 
4462 	spa_unload(spa);
4463 	spa_deactivate(spa);
4464 	spa_activate(spa, mode);
4465 
4466 	/*
4467 	 * We save the value of spa_async_suspended as it gets reset to 0 by
4468 	 * spa_unload(). We want to restore it back to the original value before
4469 	 * returning as we might be calling spa_async_resume() later.
4470 	 */
4471 	spa->spa_async_suspended = async_suspended;
4472 }
4473 
4474 static int
4475 spa_ld_read_checkpoint_txg(spa_t *spa)
4476 {
4477 	uberblock_t checkpoint;
4478 	int error = 0;
4479 
4480 	ASSERT0(spa->spa_checkpoint_txg);
4481 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4482 
4483 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4484 	    DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
4485 	    sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);
4486 
4487 	if (error == ENOENT)
4488 		return (0);
4489 
4490 	if (error != 0)
4491 		return (error);
4492 
4493 	ASSERT3U(checkpoint.ub_txg, !=, 0);
4494 	ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0);
4495 	ASSERT3U(checkpoint.ub_timestamp, !=, 0);
4496 	spa->spa_checkpoint_txg = checkpoint.ub_txg;
4497 	spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp;
4498 
4499 	return (0);
4500 }
4501 
4502 static int
4503 spa_ld_mos_init(spa_t *spa, spa_import_type_t type)
4504 {
4505 	int error = 0;
4506 
4507 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4508 	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
4509 
4510 	/*
4511 	 * Never trust the config that is provided unless we are assembling
4512 	 * a pool following a split.
4513 	 * This means don't trust blkptrs and the vdev tree in general. This
4514 	 * also effectively puts the spa in read-only mode since
4515 	 * spa_writeable() checks for spa_trust_config to be true.
4516 	 * We will later load a trusted config from the MOS.
4517 	 */
4518 	if (type != SPA_IMPORT_ASSEMBLE)
4519 		spa->spa_trust_config = B_FALSE;
4520 
4521 	/*
4522 	 * Parse the config provided to create a vdev tree.
4523 	 */
4524 	error = spa_ld_parse_config(spa, type);
4525 	if (error != 0)
4526 		return (error);
4527 
4528 	spa_import_progress_add(spa);
4529 
4530 	/*
4531 	 * Now that we have the vdev tree, try to open each vdev. This involves
4532 	 * opening the underlying physical device, retrieving its geometry and
4533 	 * probing the vdev with a dummy I/O. The state of each vdev will be set
4534 	 * based on the success of those operations. After this we'll be ready
4535 	 * to read from the vdevs.
4536 	 */
4537 	error = spa_ld_open_vdevs(spa);
4538 	if (error != 0)
4539 		return (error);
4540 
4541 	/*
4542 	 * Read the label of each vdev and make sure that the GUIDs stored
4543 	 * there match the GUIDs in the config provided.
4544 	 * If we're assembling a new pool that's been split off from an
4545 	 * existing pool, the labels haven't yet been updated so we skip
4546 	 * validation for now.
4547 	 */
4548 	if (type != SPA_IMPORT_ASSEMBLE) {
4549 		error = spa_ld_validate_vdevs(spa);
4550 		if (error != 0)
4551 			return (error);
4552 	}
4553 
4554 	/*
4555 	 * Read all vdev labels to find the best uberblock (i.e. latest,
4556 	 * unless spa_load_max_txg is set) and store it in spa_uberblock. We
4557 	 * get the list of features required to read blkptrs in the MOS from
4558 	 * the vdev label with the best uberblock and verify that our version
4559 	 * of zfs supports them all.
4560 	 */
4561 	error = spa_ld_select_uberblock(spa, type);
4562 	if (error != 0)
4563 		return (error);
4564 
4565 	/*
4566 	 * Pass that uberblock to the dsl_pool layer which will open the root
4567 	 * blkptr. This blkptr points to the latest version of the MOS and will
4568 	 * allow us to read its contents.
4569 	 */
4570 	error = spa_ld_open_rootbp(spa);
4571 	if (error != 0)
4572 		return (error);
4573 
4574 	return (0);
4575 }
4576 
4577 static int
4578 spa_ld_checkpoint_rewind(spa_t *spa)
4579 {
4580 	uberblock_t checkpoint;
4581 	int error = 0;
4582 
4583 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4584 	ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
4585 
4586 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4587 	    DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
4588 	    sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);
4589 
4590 	if (error != 0) {
4591 		spa_load_failed(spa, "unable to retrieve checkpointed "
4592 		    "uberblock from the MOS config [error=%d]", error);
4593 
4594 		if (error == ENOENT)
4595 			error = ZFS_ERR_NO_CHECKPOINT;
4596 
4597 		return (error);
4598 	}
4599 
4600 	ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg);
4601 	ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg);
4602 
4603 	/*
4604 	 * We need to update the txg and timestamp of the checkpointed
4605 	 * uberblock to be higher than the latest one. This ensures that
4606 	 * the checkpointed uberblock is selected if we were to close and
4607 	 * reopen the pool right after we've written it in the vdev labels.
4608 	 * (also see block comment in vdev_uberblock_compare)
4609 	 */
4610 	checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1;
4611 	checkpoint.ub_timestamp = gethrestime_sec();
4612 
4613 	/*
4614 	 * Set current uberblock to be the checkpointed uberblock.
4615 	 */
4616 	spa->spa_uberblock = checkpoint;
4617 
4618 	/*
4619 	 * If we are doing a normal rewind, then the pool is open for
4620 	 * writing and we sync the "updated" checkpointed uberblock to
4621 	 * disk. Once this is done, we've basically rewound the whole
4622 	 * pool and there is no way back.
4623 	 *
4624 	 * There are cases when we don't want to attempt and sync the
4625 	 * checkpointed uberblock to disk because we are opening a
4626 	 * pool as read-only. Specifically, verifying the checkpointed
4627 	 * state with zdb, and importing the checkpointed state to get
4628 	 * a "preview" of its content.
4629 	 */
4630 	if (spa_writeable(spa)) {
4631 		vdev_t *rvd = spa->spa_root_vdev;
4632 
4633 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4634 		vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
4635 		int svdcount = 0;
4636 		int children = rvd->vdev_children;
4637 		int c0 = random_in_range(children);
4638 
4639 		for (int c = 0; c < children; c++) {
4640 			vdev_t *vd = rvd->vdev_child[(c0 + c) % children];
4641 
4642 			/* Stop when revisiting the first vdev */
4643 			if (c > 0 && svd[0] == vd)
4644 				break;
4645 
4646 			if (vd->vdev_ms_array == 0 || vd->vdev_islog ||
4647 			    !vdev_is_concrete(vd))
4648 				continue;
4649 
4650 			svd[svdcount++] = vd;
4651 			if (svdcount == SPA_SYNC_MIN_VDEVS)
4652 				break;
4653 		}
4654 		error = vdev_config_sync(svd, svdcount, spa->spa_first_txg);
4655 		if (error == 0)
4656 			spa->spa_last_synced_guid = rvd->vdev_guid;
4657 		spa_config_exit(spa, SCL_ALL, FTAG);
4658 
4659 		if (error != 0) {
4660 			spa_load_failed(spa, "failed to write checkpointed "
4661 			    "uberblock to the vdev labels [error=%d]", error);
4662 			return (error);
4663 		}
4664 	}
4665 
4666 	return (0);
4667 }
4668 
4669 static int
4670 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type,
4671     boolean_t *update_config_cache)
4672 {
4673 	int error;
4674 
4675 	/*
4676 	 * Parse the config for pool, open and validate vdevs,
4677 	 * select an uberblock, and use that uberblock to open
4678 	 * the MOS.
4679 	 */
4680 	error = spa_ld_mos_init(spa, type);
4681 	if (error != 0)
4682 		return (error);
4683 
4684 	/*
4685 	 * Retrieve the trusted config stored in the MOS and use it to create
4686 	 * a new, exact version of the vdev tree, then reopen all vdevs.
4687 	 */
4688 	error = spa_ld_trusted_config(spa, type, B_FALSE);
4689 	if (error == EAGAIN) {
4690 		if (update_config_cache != NULL)
4691 			*update_config_cache = B_TRUE;
4692 
4693 		/*
4694 		 * Redo the loading process with the trusted config if it is
4695 		 * too different from the untrusted config.
4696 		 */
4697 		spa_ld_prepare_for_reload(spa);
4698 		spa_load_note(spa, "RELOADING");
4699 		error = spa_ld_mos_init(spa, type);
4700 		if (error != 0)
4701 			return (error);
4702 
4703 		error = spa_ld_trusted_config(spa, type, B_TRUE);
4704 		if (error != 0)
4705 			return (error);
4706 
4707 	} else if (error != 0) {
4708 		return (error);
4709 	}
4710 
4711 	return (0);
4712 }
4713 
4714 /*
4715  * Load an existing storage pool, using the config provided. This config
4716  * describes which vdevs are part of the pool and is later validated against
4717  * partial configs present in each vdev's label and an entire copy of the
4718  * config stored in the MOS.
4719  */
4720 static int
4721 spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport)
4722 {
4723 	int error = 0;
4724 	boolean_t missing_feat_write = B_FALSE;
4725 	boolean_t checkpoint_rewind =
4726 	    (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
4727 	boolean_t update_config_cache = B_FALSE;
4728 
4729 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4730 	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
4731 
4732 	spa_load_note(spa, "LOADING");
4733 
4734 	error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache);
4735 	if (error != 0)
4736 		return (error);
4737 
4738 	/*
4739 	 * If we are rewinding to the checkpoint then we need to repeat
4740 	 * everything we've done so far in this function but this time
4741 	 * selecting the checkpointed uberblock and using that to open
4742 	 * the MOS.
4743 	 */
4744 	if (checkpoint_rewind) {
4745 		/*
4746 		 * If we are rewinding to the checkpoint update config cache
4747 		 * anyway.
4748 		 */
4749 		update_config_cache = B_TRUE;
4750 
4751 		/*
4752 		 * Extract the checkpointed uberblock from the current MOS
4753 		 * and use this as the pool's uberblock from now on. If the
4754 		 * pool is imported as writeable we also write the checkpoint
4755 		 * uberblock to the labels, making the rewind permanent.
4756 		 */
4757 		error = spa_ld_checkpoint_rewind(spa);
4758 		if (error != 0)
4759 			return (error);
4760 
4761 		/*
4762 		 * Redo the loading process again with the
4763 		 * checkpointed uberblock.
4764 		 */
4765 		spa_ld_prepare_for_reload(spa);
4766 		spa_load_note(spa, "LOADING checkpointed uberblock");
4767 		error = spa_ld_mos_with_trusted_config(spa, type, NULL);
4768 		if (error != 0)
4769 			return (error);
4770 	}
4771 
4772 	/*
4773 	 * Retrieve the checkpoint txg if the pool has a checkpoint.
4774 	 */
4775 	error = spa_ld_read_checkpoint_txg(spa);
4776 	if (error != 0)
4777 		return (error);
4778 
4779 	/*
4780 	 * Retrieve the mapping of indirect vdevs. Those vdevs were removed
4781 	 * from the pool and their contents were re-mapped to other vdevs. Note
4782 	 * that everything that we read before this step must have been
4783 	 * rewritten on concrete vdevs after the last device removal was
4784 	 * initiated. Otherwise we could be reading from indirect vdevs before
4785 	 * we have loaded their mappings.
4786 	 */
4787 	error = spa_ld_open_indirect_vdev_metadata(spa);
4788 	if (error != 0)
4789 		return (error);
4790 
4791 	/*
4792 	 * Retrieve the full list of active features from the MOS and check if
4793 	 * they are all supported.
4794 	 */
4795 	error = spa_ld_check_features(spa, &missing_feat_write);
4796 	if (error != 0)
4797 		return (error);
4798 
4799 	/*
4800 	 * Load several special directories from the MOS needed by the dsl_pool
4801 	 * layer.
4802 	 */
4803 	error = spa_ld_load_special_directories(spa);
4804 	if (error != 0)
4805 		return (error);
4806 
4807 	/*
4808 	 * Retrieve pool properties from the MOS.
4809 	 */
4810 	error = spa_ld_get_props(spa);
4811 	if (error != 0)
4812 		return (error);
4813 
4814 	/*
4815 	 * Retrieve the list of auxiliary devices - cache devices and spares -
4816 	 * and open them.
4817 	 */
4818 	error = spa_ld_open_aux_vdevs(spa, type);
4819 	if (error != 0)
4820 		return (error);
4821 
4822 	/*
4823 	 * Load the metadata for all vdevs. Also check if unopenable devices
4824 	 * should be autoreplaced.
4825 	 */
4826 	error = spa_ld_load_vdev_metadata(spa);
4827 	if (error != 0)
4828 		return (error);
4829 
4830 	error = spa_ld_load_dedup_tables(spa);
4831 	if (error != 0)
4832 		return (error);
4833 
4834 	/*
4835 	 * Verify the logs now to make sure we don't have any unexpected errors
4836 	 * when we claim log blocks later.
4837 	 */
4838 	error = spa_ld_verify_logs(spa, type, ereport);
4839 	if (error != 0)
4840 		return (error);
4841 
4842 	if (missing_feat_write) {
4843 		ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT);
4844 
4845 		/*
4846 		 * At this point, we know that we can open the pool in
4847 		 * read-only mode but not read-write mode. We now have enough
4848 		 * information and can return to userland.
4849 		 */
4850 		return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT,
4851 		    ENOTSUP));
4852 	}
4853 
4854 	/*
4855 	 * Traverse the last txgs to make sure the pool was left off in a safe
4856 	 * state. When performing an extreme rewind, we verify the whole pool,
4857 	 * which can take a very long time.
4858 	 */
4859 	error = spa_ld_verify_pool_data(spa);
4860 	if (error != 0)
4861 		return (error);
4862 
4863 	/*
4864 	 * Calculate the deflated space for the pool. This must be done before
4865 	 * we write anything to the pool because we'd need to update the space
4866 	 * accounting using the deflated sizes.
4867 	 */
4868 	spa_update_dspace(spa);
4869 
4870 	/*
4871 	 * We have now retrieved all the information we needed to open the
4872 	 * pool. If we are importing the pool in read-write mode, a few
4873 	 * additional steps must be performed to finish the import.
4874 	 */
4875 	if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER ||
4876 	    spa->spa_load_max_txg == UINT64_MAX)) {
4877 		uint64_t config_cache_txg = spa->spa_config_txg;
4878 
4879 		ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT);
4880 
4881 		/*
4882 		 * In case of a checkpoint rewind, log the original txg
4883 		 * of the checkpointed uberblock.
4884 		 */
4885 		if (checkpoint_rewind) {
4886 			spa_history_log_internal(spa, "checkpoint rewind",
4887 			    NULL, "rewound state to txg=%llu",
4888 			    (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg);
4889 		}
4890 
4891 		/*
4892 		 * Traverse the ZIL and claim all blocks.
4893 		 */
4894 		spa_ld_claim_log_blocks(spa);
4895 
4896 		/*
4897 		 * Kick-off the syncing thread.
4898 		 */
4899 		spa->spa_sync_on = B_TRUE;
4900 		txg_sync_start(spa->spa_dsl_pool);
4901 		mmp_thread_start(spa);
4902 
4903 		/*
4904 		 * Wait for all claims to sync.  We sync up to the highest
4905 		 * claimed log block birth time so that claimed log blocks
4906 		 * don't appear to be from the future.  spa_claim_max_txg
4907 		 * will have been set for us by ZIL traversal operations
4908 		 * performed above.
4909 		 */
4910 		txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);
4911 
4912 		/*
4913 		 * Check if we need to request an update of the config. On the
4914 		 * next sync, we would update the config stored in vdev labels
4915 		 * and the cachefile (by default /etc/zfs/zpool.cache).
4916 		 */
4917 		spa_ld_check_for_config_update(spa, config_cache_txg,
4918 		    update_config_cache);
4919 
4920 		/*
4921 		 * Check if a rebuild was in progress and if so resume it.
4922 		 * Then check all DTLs to see if anything needs resilvering.
4923 		 * The resilver will be deferred if a rebuild was started.
4924 		 */
4925 		if (vdev_rebuild_active(spa->spa_root_vdev)) {
4926 			vdev_rebuild_restart(spa);
4927 		} else if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
4928 		    vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
4929 			spa_async_request(spa, SPA_ASYNC_RESILVER);
4930 		}
4931 
4932 		/*
4933 		 * Log the fact that we booted up (so that we can detect if
4934 		 * we rebooted in the middle of an operation).
4935 		 */
4936 		spa_history_log_version(spa, "open", NULL);
4937 
4938 		spa_restart_removal(spa);
4939 		spa_spawn_aux_threads(spa);
4940 
4941 		/*
4942 		 * Delete any inconsistent datasets.
4943 		 *
4944 		 * Note:
4945 		 * Since we may be issuing deletes for clones here,
4946 		 * we make sure to do so after we've spawned all the
4947 		 * auxiliary threads above (from which the livelist
4948 		 * deletion zthr is part of).
4949 		 */
4950 		(void) dmu_objset_find(spa_name(spa),
4951 		    dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN);
4952 
4953 		/*
4954 		 * Clean up any stale temporary dataset userrefs.
4955 		 */
4956 		dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);
4957 
4958 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
4959 		vdev_initialize_restart(spa->spa_root_vdev);
4960 		vdev_trim_restart(spa->spa_root_vdev);
4961 		vdev_autotrim_restart(spa);
4962 		spa_config_exit(spa, SCL_CONFIG, FTAG);
4963 	}
4964 
4965 	spa_import_progress_remove(spa_guid(spa));
4966 	spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD);
4967 
4968 	spa_load_note(spa, "LOADED");
4969 
4970 	return (0);
4971 }
4972 
4973 static int
4974 spa_load_retry(spa_t *spa, spa_load_state_t state)
4975 {
4976 	spa_mode_t mode = spa->spa_mode;
4977 
4978 	spa_unload(spa);
4979 	spa_deactivate(spa);
4980 
4981 	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
4982 
4983 	spa_activate(spa, mode);
4984 	spa_async_suspend(spa);
4985 
4986 	spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu",
4987 	    (u_longlong_t)spa->spa_load_max_txg);
4988 
4989 	return (spa_load(spa, state, SPA_IMPORT_EXISTING));
4990 }
4991 
4992 /*
4993  * If spa_load() fails this function will try loading prior txg's. If
4994  * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool
4995  * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this
4996  * function will not rewind the pool and will return the same error as
4997  * spa_load().
4998  */
4999 static int
5000 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request,
5001     int rewind_flags)
5002 {
5003 	nvlist_t *loadinfo = NULL;
5004 	nvlist_t *config = NULL;
5005 	int load_error, rewind_error;
5006 	uint64_t safe_rewind_txg;
5007 	uint64_t min_txg;
5008 
5009 	if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
5010 		spa->spa_load_max_txg = spa->spa_load_txg;
5011 		spa_set_log_state(spa, SPA_LOG_CLEAR);
5012 	} else {
5013 		spa->spa_load_max_txg = max_request;
5014 		if (max_request != UINT64_MAX)
5015 			spa->spa_extreme_rewind = B_TRUE;
5016 	}
5017 
5018 	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING);
5019 	if (load_error == 0)
5020 		return (0);
5021 	if (load_error == ZFS_ERR_NO_CHECKPOINT) {
5022 		/*
5023 		 * When attempting checkpoint-rewind on a pool with no
5024 		 * checkpoint, we should not attempt to load uberblocks
5025 		 * from previous txgs when spa_load fails.
5026 		 */
5027 		ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5028 		spa_import_progress_remove(spa_guid(spa));
5029 		return (load_error);
5030 	}
5031 
5032 	if (spa->spa_root_vdev != NULL)
5033 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5034 
5035 	spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
5036 	spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;
5037 
5038 	if (rewind_flags & ZPOOL_NEVER_REWIND) {
5039 		nvlist_free(config);
5040 		spa_import_progress_remove(spa_guid(spa));
5041 		return (load_error);
5042 	}
5043 
5044 	if (state == SPA_LOAD_RECOVER) {
5045 		/* Price of rolling back is discarding txgs, including log */
5046 		spa_set_log_state(spa, SPA_LOG_CLEAR);
5047 	} else {
5048 		/*
5049 		 * If we aren't rolling back save the load info from our first
5050 		 * import attempt so that we can restore it after attempting
5051 		 * to rewind.
5052 		 */
5053 		loadinfo = spa->spa_load_info;
5054 		spa->spa_load_info = fnvlist_alloc();
5055 	}
5056 
5057 	spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
5058 	safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
5059 	min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ?
5060 	    TXG_INITIAL : safe_rewind_txg;
5061 
5062 	/*
5063 	 * Continue as long as we're finding errors, we're still within
5064 	 * the acceptable rewind range, and we're still finding uberblocks
5065 	 */
5066 	while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
5067 	    spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
5068 		if (spa->spa_load_max_txg < safe_rewind_txg)
5069 			spa->spa_extreme_rewind = B_TRUE;
5070 		rewind_error = spa_load_retry(spa, state);
5071 	}
5072 
5073 	spa->spa_extreme_rewind = B_FALSE;
5074 	spa->spa_load_max_txg = UINT64_MAX;
5075 
5076 	if (config && (rewind_error || state != SPA_LOAD_RECOVER))
5077 		spa_config_set(spa, config);
5078 	else
5079 		nvlist_free(config);
5080 
5081 	if (state == SPA_LOAD_RECOVER) {
5082 		ASSERT3P(loadinfo, ==, NULL);
5083 		spa_import_progress_remove(spa_guid(spa));
5084 		return (rewind_error);
5085 	} else {
5086 		/* Store the rewind info as part of the initial load info */
5087 		fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO,
5088 		    spa->spa_load_info);
5089 
5090 		/* Restore the initial load info */
5091 		fnvlist_free(spa->spa_load_info);
5092 		spa->spa_load_info = loadinfo;
5093 
5094 		spa_import_progress_remove(spa_guid(spa));
5095 		return (load_error);
5096 	}
5097 }
5098 
5099 /*
5100  * Pool Open/Import
5101  *
5102  * The import case is identical to an open except that the configuration is sent
5103  * down from userland, instead of grabbed from the configuration cache.  For the
5104  * case of an open, the pool configuration will exist in the
5105  * POOL_STATE_UNINITIALIZED state.
5106  *
5107  * The stats information (gen/count/ustats) is used to gather vdev statistics at
5108  * the same time open the pool, without having to keep around the spa_t in some
5109  * ambiguous state.
5110  */
5111 static int
5112 spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy,
5113     nvlist_t **config)
5114 {
5115 	spa_t *spa;
5116 	spa_load_state_t state = SPA_LOAD_OPEN;
5117 	int error;
5118 	int locked = B_FALSE;
5119 	int firstopen = B_FALSE;
5120 
5121 	*spapp = NULL;
5122 
5123 	/*
5124 	 * As disgusting as this is, we need to support recursive calls to this
5125 	 * function because dsl_dir_open() is called during spa_load(), and ends
5126 	 * up calling spa_open() again.  The real fix is to figure out how to
5127 	 * avoid dsl_dir_open() calling this in the first place.
5128 	 */
5129 	if (MUTEX_NOT_HELD(&spa_namespace_lock)) {
5130 		mutex_enter(&spa_namespace_lock);
5131 		locked = B_TRUE;
5132 	}
5133 
5134 	if ((spa = spa_lookup(pool)) == NULL) {
5135 		if (locked)
5136 			mutex_exit(&spa_namespace_lock);
5137 		return (SET_ERROR(ENOENT));
5138 	}
5139 
5140 	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
5141 		zpool_load_policy_t policy;
5142 
5143 		firstopen = B_TRUE;
5144 
5145 		zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config,
5146 		    &policy);
5147 		if (policy.zlp_rewind & ZPOOL_DO_REWIND)
5148 			state = SPA_LOAD_RECOVER;
5149 
5150 		spa_activate(spa, spa_mode_global);
5151 
5152 		if (state != SPA_LOAD_RECOVER)
5153 			spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
5154 		spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
5155 
5156 		zfs_dbgmsg("spa_open_common: opening %s", pool);
5157 		error = spa_load_best(spa, state, policy.zlp_txg,
5158 		    policy.zlp_rewind);
5159 
5160 		if (error == EBADF) {
5161 			/*
5162 			 * If vdev_validate() returns failure (indicated by
5163 			 * EBADF), it indicates that one of the vdevs indicates
5164 			 * that the pool has been exported or destroyed.  If
5165 			 * this is the case, the config cache is out of sync and
5166 			 * we should remove the pool from the namespace.
5167 			 */
5168 			spa_unload(spa);
5169 			spa_deactivate(spa);
5170 			spa_write_cachefile(spa, B_TRUE, B_TRUE);
5171 			spa_remove(spa);
5172 			if (locked)
5173 				mutex_exit(&spa_namespace_lock);
5174 			return (SET_ERROR(ENOENT));
5175 		}
5176 
5177 		if (error) {
5178 			/*
5179 			 * We can't open the pool, but we still have useful
5180 			 * information: the state of each vdev after the
5181 			 * attempted vdev_open().  Return this to the user.
5182 			 */
5183 			if (config != NULL && spa->spa_config) {
5184 				VERIFY(nvlist_dup(spa->spa_config, config,
5185 				    KM_SLEEP) == 0);
5186 				VERIFY(nvlist_add_nvlist(*config,
5187 				    ZPOOL_CONFIG_LOAD_INFO,
5188 				    spa->spa_load_info) == 0);
5189 			}
5190 			spa_unload(spa);
5191 			spa_deactivate(spa);
5192 			spa->spa_last_open_failed = error;
5193 			if (locked)
5194 				mutex_exit(&spa_namespace_lock);
5195 			*spapp = NULL;
5196 			return (error);
5197 		}
5198 	}
5199 
5200 	spa_open_ref(spa, tag);
5201 
5202 	if (config != NULL)
5203 		*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5204 
5205 	/*
5206 	 * If we've recovered the pool, pass back any information we
5207 	 * gathered while doing the load.
5208 	 */
5209 	if (state == SPA_LOAD_RECOVER) {
5210 		VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO,
5211 		    spa->spa_load_info) == 0);
5212 	}
5213 
5214 	if (locked) {
5215 		spa->spa_last_open_failed = 0;
5216 		spa->spa_last_ubsync_txg = 0;
5217 		spa->spa_load_txg = 0;
5218 		mutex_exit(&spa_namespace_lock);
5219 	}
5220 
5221 	if (firstopen)
5222 		zvol_create_minors_recursive(spa_name(spa));
5223 
5224 	*spapp = spa;
5225 
5226 	return (0);
5227 }
5228 
5229 int
5230 spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy,
5231     nvlist_t **config)
5232 {
5233 	return (spa_open_common(name, spapp, tag, policy, config));
5234 }
5235 
5236 int
5237 spa_open(const char *name, spa_t **spapp, void *tag)
5238 {
5239 	return (spa_open_common(name, spapp, tag, NULL, NULL));
5240 }
5241 
5242 /*
5243  * Lookup the given spa_t, incrementing the inject count in the process,
5244  * preventing it from being exported or destroyed.
5245  */
5246 spa_t *
5247 spa_inject_addref(char *name)
5248 {
5249 	spa_t *spa;
5250 
5251 	mutex_enter(&spa_namespace_lock);
5252 	if ((spa = spa_lookup(name)) == NULL) {
5253 		mutex_exit(&spa_namespace_lock);
5254 		return (NULL);
5255 	}
5256 	spa->spa_inject_ref++;
5257 	mutex_exit(&spa_namespace_lock);
5258 
5259 	return (spa);
5260 }
5261 
5262 void
5263 spa_inject_delref(spa_t *spa)
5264 {
5265 	mutex_enter(&spa_namespace_lock);
5266 	spa->spa_inject_ref--;
5267 	mutex_exit(&spa_namespace_lock);
5268 }
5269 
5270 /*
5271  * Add spares device information to the nvlist.
5272  */
5273 static void
5274 spa_add_spares(spa_t *spa, nvlist_t *config)
5275 {
5276 	nvlist_t **spares;
5277 	uint_t i, nspares;
5278 	nvlist_t *nvroot;
5279 	uint64_t guid;
5280 	vdev_stat_t *vs;
5281 	uint_t vsc;
5282 	uint64_t pool;
5283 
5284 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5285 
5286 	if (spa->spa_spares.sav_count == 0)
5287 		return;
5288 
5289 	VERIFY(nvlist_lookup_nvlist(config,
5290 	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
5291 	VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
5292 	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
5293 	if (nspares != 0) {
5294 		VERIFY(nvlist_add_nvlist_array(nvroot,
5295 		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
5296 		VERIFY(nvlist_lookup_nvlist_array(nvroot,
5297 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
5298 
5299 		/*
5300 		 * Go through and find any spares which have since been
5301 		 * repurposed as an active spare.  If this is the case, update
5302 		 * their status appropriately.
5303 		 */
5304 		for (i = 0; i < nspares; i++) {
5305 			VERIFY(nvlist_lookup_uint64(spares[i],
5306 			    ZPOOL_CONFIG_GUID, &guid) == 0);
5307 			if (spa_spare_exists(guid, &pool, NULL) &&
5308 			    pool != 0ULL) {
5309 				VERIFY(nvlist_lookup_uint64_array(
5310 				    spares[i], ZPOOL_CONFIG_VDEV_STATS,
5311 				    (uint64_t **)&vs, &vsc) == 0);
5312 				vs->vs_state = VDEV_STATE_CANT_OPEN;
5313 				vs->vs_aux = VDEV_AUX_SPARED;
5314 			}
5315 		}
5316 	}
5317 }
5318 
5319 /*
5320  * Add l2cache device information to the nvlist, including vdev stats.
5321  */
5322 static void
5323 spa_add_l2cache(spa_t *spa, nvlist_t *config)
5324 {
5325 	nvlist_t **l2cache;
5326 	uint_t i, j, nl2cache;
5327 	nvlist_t *nvroot;
5328 	uint64_t guid;
5329 	vdev_t *vd;
5330 	vdev_stat_t *vs;
5331 	uint_t vsc;
5332 
5333 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5334 
5335 	if (spa->spa_l2cache.sav_count == 0)
5336 		return;
5337 
5338 	VERIFY(nvlist_lookup_nvlist(config,
5339 	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
5340 	VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
5341 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
5342 	if (nl2cache != 0) {
5343 		VERIFY(nvlist_add_nvlist_array(nvroot,
5344 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
5345 		VERIFY(nvlist_lookup_nvlist_array(nvroot,
5346 		    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
5347 
5348 		/*
5349 		 * Update level 2 cache device stats.
5350 		 */
5351 
5352 		for (i = 0; i < nl2cache; i++) {
5353 			VERIFY(nvlist_lookup_uint64(l2cache[i],
5354 			    ZPOOL_CONFIG_GUID, &guid) == 0);
5355 
5356 			vd = NULL;
5357 			for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
5358 				if (guid ==
5359 				    spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
5360 					vd = spa->spa_l2cache.sav_vdevs[j];
5361 					break;
5362 				}
5363 			}
5364 			ASSERT(vd != NULL);
5365 
5366 			VERIFY(nvlist_lookup_uint64_array(l2cache[i],
5367 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)
5368 			    == 0);
5369 			vdev_get_stats(vd, vs);
5370 			vdev_config_generate_stats(vd, l2cache[i]);
5371 
5372 		}
5373 	}
5374 }
5375 
5376 static void
5377 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features)
5378 {
5379 	zap_cursor_t zc;
5380 	zap_attribute_t za;
5381 
5382 	if (spa->spa_feat_for_read_obj != 0) {
5383 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
5384 		    spa->spa_feat_for_read_obj);
5385 		    zap_cursor_retrieve(&zc, &za) == 0;
5386 		    zap_cursor_advance(&zc)) {
5387 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
5388 			    za.za_num_integers == 1);
5389 			VERIFY0(nvlist_add_uint64(features, za.za_name,
5390 			    za.za_first_integer));
5391 		}
5392 		zap_cursor_fini(&zc);
5393 	}
5394 
5395 	if (spa->spa_feat_for_write_obj != 0) {
5396 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
5397 		    spa->spa_feat_for_write_obj);
5398 		    zap_cursor_retrieve(&zc, &za) == 0;
5399 		    zap_cursor_advance(&zc)) {
5400 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
5401 			    za.za_num_integers == 1);
5402 			VERIFY0(nvlist_add_uint64(features, za.za_name,
5403 			    za.za_first_integer));
5404 		}
5405 		zap_cursor_fini(&zc);
5406 	}
5407 }
5408 
5409 static void
5410 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features)
5411 {
5412 	int i;
5413 
5414 	for (i = 0; i < SPA_FEATURES; i++) {
5415 		zfeature_info_t feature = spa_feature_table[i];
5416 		uint64_t refcount;
5417 
5418 		if (feature_get_refcount(spa, &feature, &refcount) != 0)
5419 			continue;
5420 
5421 		VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount));
5422 	}
5423 }
5424 
5425 /*
5426  * Store a list of pool features and their reference counts in the
5427  * config.
5428  *
5429  * The first time this is called on a spa, allocate a new nvlist, fetch
5430  * the pool features and reference counts from disk, then save the list
5431  * in the spa. In subsequent calls on the same spa use the saved nvlist
5432  * and refresh its values from the cached reference counts.  This
5433  * ensures we don't block here on I/O on a suspended pool so 'zpool
5434  * clear' can resume the pool.
5435  */
5436 static void
5437 spa_add_feature_stats(spa_t *spa, nvlist_t *config)
5438 {
5439 	nvlist_t *features;
5440 
5441 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5442 
5443 	mutex_enter(&spa->spa_feat_stats_lock);
5444 	features = spa->spa_feat_stats;
5445 
5446 	if (features != NULL) {
5447 		spa_feature_stats_from_cache(spa, features);
5448 	} else {
5449 		VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP));
5450 		spa->spa_feat_stats = features;
5451 		spa_feature_stats_from_disk(spa, features);
5452 	}
5453 
5454 	VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS,
5455 	    features));
5456 
5457 	mutex_exit(&spa->spa_feat_stats_lock);
5458 }
5459 
5460 int
5461 spa_get_stats(const char *name, nvlist_t **config,
5462     char *altroot, size_t buflen)
5463 {
5464 	int error;
5465 	spa_t *spa;
5466 
5467 	*config = NULL;
5468 	error = spa_open_common(name, &spa, FTAG, NULL, config);
5469 
5470 	if (spa != NULL) {
5471 		/*
5472 		 * This still leaves a window of inconsistency where the spares
5473 		 * or l2cache devices could change and the config would be
5474 		 * self-inconsistent.
5475 		 */
5476 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
5477 
5478 		if (*config != NULL) {
5479 			uint64_t loadtimes[2];
5480 
5481 			loadtimes[0] = spa->spa_loaded_ts.tv_sec;
5482 			loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
5483 			VERIFY(nvlist_add_uint64_array(*config,
5484 			    ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0);
5485 
5486 			VERIFY(nvlist_add_uint64(*config,
5487 			    ZPOOL_CONFIG_ERRCOUNT,
5488 			    spa_get_errlog_size(spa)) == 0);
5489 
5490 			if (spa_suspended(spa)) {
5491 				VERIFY(nvlist_add_uint64(*config,
5492 				    ZPOOL_CONFIG_SUSPENDED,
5493 				    spa->spa_failmode) == 0);
5494 				VERIFY(nvlist_add_uint64(*config,
5495 				    ZPOOL_CONFIG_SUSPENDED_REASON,
5496 				    spa->spa_suspended) == 0);
5497 			}
5498 
5499 			spa_add_spares(spa, *config);
5500 			spa_add_l2cache(spa, *config);
5501 			spa_add_feature_stats(spa, *config);
5502 		}
5503 	}
5504 
5505 	/*
5506 	 * We want to get the alternate root even for faulted pools, so we cheat
5507 	 * and call spa_lookup() directly.
5508 	 */
5509 	if (altroot) {
5510 		if (spa == NULL) {
5511 			mutex_enter(&spa_namespace_lock);
5512 			spa = spa_lookup(name);
5513 			if (spa)
5514 				spa_altroot(spa, altroot, buflen);
5515 			else
5516 				altroot[0] = '\0';
5517 			spa = NULL;
5518 			mutex_exit(&spa_namespace_lock);
5519 		} else {
5520 			spa_altroot(spa, altroot, buflen);
5521 		}
5522 	}
5523 
5524 	if (spa != NULL) {
5525 		spa_config_exit(spa, SCL_CONFIG, FTAG);
5526 		spa_close(spa, FTAG);
5527 	}
5528 
5529 	return (error);
5530 }
5531 
5532 /*
5533  * Validate that the auxiliary device array is well formed.  We must have an
5534  * array of nvlists, each which describes a valid leaf vdev.  If this is an
5535  * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be
5536  * specified, as long as they are well-formed.
5537  */
5538 static int
5539 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode,
5540     spa_aux_vdev_t *sav, const char *config, uint64_t version,
5541     vdev_labeltype_t label)
5542 {
5543 	nvlist_t **dev;
5544 	uint_t i, ndev;
5545 	vdev_t *vd;
5546 	int error;
5547 
5548 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5549 
5550 	/*
5551 	 * It's acceptable to have no devs specified.
5552 	 */
5553 	if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0)
5554 		return (0);
5555 
5556 	if (ndev == 0)
5557 		return (SET_ERROR(EINVAL));
5558 
5559 	/*
5560 	 * Make sure the pool is formatted with a version that supports this
5561 	 * device type.
5562 	 */
5563 	if (spa_version(spa) < version)
5564 		return (SET_ERROR(ENOTSUP));
5565 
5566 	/*
5567 	 * Set the pending device list so we correctly handle device in-use
5568 	 * checking.
5569 	 */
5570 	sav->sav_pending = dev;
5571 	sav->sav_npending = ndev;
5572 
5573 	for (i = 0; i < ndev; i++) {
5574 		if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0,
5575 		    mode)) != 0)
5576 			goto out;
5577 
5578 		if (!vd->vdev_ops->vdev_op_leaf) {
5579 			vdev_free(vd);
5580 			error = SET_ERROR(EINVAL);
5581 			goto out;
5582 		}
5583 
5584 		vd->vdev_top = vd;
5585 
5586 		if ((error = vdev_open(vd)) == 0 &&
5587 		    (error = vdev_label_init(vd, crtxg, label)) == 0) {
5588 			VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID,
5589 			    vd->vdev_guid) == 0);
5590 		}
5591 
5592 		vdev_free(vd);
5593 
5594 		if (error &&
5595 		    (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE))
5596 			goto out;
5597 		else
5598 			error = 0;
5599 	}
5600 
5601 out:
5602 	sav->sav_pending = NULL;
5603 	sav->sav_npending = 0;
5604 	return (error);
5605 }
5606 
5607 static int
5608 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode)
5609 {
5610 	int error;
5611 
5612 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5613 
5614 	if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode,
5615 	    &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
5616 	    VDEV_LABEL_SPARE)) != 0) {
5617 		return (error);
5618 	}
5619 
5620 	return (spa_validate_aux_devs(spa, nvroot, crtxg, mode,
5621 	    &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
5622 	    VDEV_LABEL_L2CACHE));
5623 }
5624 
5625 static void
5626 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs,
5627     const char *config)
5628 {
5629 	int i;
5630 
5631 	if (sav->sav_config != NULL) {
5632 		nvlist_t **olddevs;
5633 		uint_t oldndevs;
5634 		nvlist_t **newdevs;
5635 
5636 		/*
5637 		 * Generate new dev list by concatenating with the
5638 		 * current dev list.
5639 		 */
5640 		VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config,
5641 		    &olddevs, &oldndevs) == 0);
5642 
5643 		newdevs = kmem_alloc(sizeof (void *) *
5644 		    (ndevs + oldndevs), KM_SLEEP);
5645 		for (i = 0; i < oldndevs; i++)
5646 			VERIFY(nvlist_dup(olddevs[i], &newdevs[i],
5647 			    KM_SLEEP) == 0);
5648 		for (i = 0; i < ndevs; i++)
5649 			VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs],
5650 			    KM_SLEEP) == 0);
5651 
5652 		VERIFY(nvlist_remove(sav->sav_config, config,
5653 		    DATA_TYPE_NVLIST_ARRAY) == 0);
5654 
5655 		VERIFY(nvlist_add_nvlist_array(sav->sav_config,
5656 		    config, newdevs, ndevs + oldndevs) == 0);
5657 		for (i = 0; i < oldndevs + ndevs; i++)
5658 			nvlist_free(newdevs[i]);
5659 		kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *));
5660 	} else {
5661 		/*
5662 		 * Generate a new dev list.
5663 		 */
5664 		VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME,
5665 		    KM_SLEEP) == 0);
5666 		VERIFY(nvlist_add_nvlist_array(sav->sav_config, config,
5667 		    devs, ndevs) == 0);
5668 	}
5669 }
5670 
5671 /*
5672  * Stop and drop level 2 ARC devices
5673  */
5674 void
5675 spa_l2cache_drop(spa_t *spa)
5676 {
5677 	vdev_t *vd;
5678 	int i;
5679 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
5680 
5681 	for (i = 0; i < sav->sav_count; i++) {
5682 		uint64_t pool;
5683 
5684 		vd = sav->sav_vdevs[i];
5685 		ASSERT(vd != NULL);
5686 
5687 		if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
5688 		    pool != 0ULL && l2arc_vdev_present(vd))
5689 			l2arc_remove_vdev(vd);
5690 	}
5691 }
5692 
5693 /*
5694  * Verify encryption parameters for spa creation. If we are encrypting, we must
5695  * have the encryption feature flag enabled.
5696  */
5697 static int
5698 spa_create_check_encryption_params(dsl_crypto_params_t *dcp,
5699     boolean_t has_encryption)
5700 {
5701 	if (dcp->cp_crypt != ZIO_CRYPT_OFF &&
5702 	    dcp->cp_crypt != ZIO_CRYPT_INHERIT &&
5703 	    !has_encryption)
5704 		return (SET_ERROR(ENOTSUP));
5705 
5706 	return (dmu_objset_create_crypt_check(NULL, dcp, NULL));
5707 }
5708 
5709 /*
5710  * Pool Creation
5711  */
5712 int
5713 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
5714     nvlist_t *zplprops, dsl_crypto_params_t *dcp)
5715 {
5716 	spa_t *spa;
5717 	char *altroot = NULL;
5718 	vdev_t *rvd;
5719 	dsl_pool_t *dp;
5720 	dmu_tx_t *tx;
5721 	int error = 0;
5722 	uint64_t txg = TXG_INITIAL;
5723 	nvlist_t **spares, **l2cache;
5724 	uint_t nspares, nl2cache;
5725 	uint64_t version, obj, ndraid = 0;
5726 	boolean_t has_features;
5727 	boolean_t has_encryption;
5728 	boolean_t has_allocclass;
5729 	spa_feature_t feat;
5730 	char *feat_name;
5731 	char *poolname;
5732 	nvlist_t *nvl;
5733 
5734 	if (props == NULL ||
5735 	    nvlist_lookup_string(props, "tname", &poolname) != 0)
5736 		poolname = (char *)pool;
5737 
5738 	/*
5739 	 * If this pool already exists, return failure.
5740 	 */
5741 	mutex_enter(&spa_namespace_lock);
5742 	if (spa_lookup(poolname) != NULL) {
5743 		mutex_exit(&spa_namespace_lock);
5744 		return (SET_ERROR(EEXIST));
5745 	}
5746 
5747 	/*
5748 	 * Allocate a new spa_t structure.
5749 	 */
5750 	nvl = fnvlist_alloc();
5751 	fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool);
5752 	(void) nvlist_lookup_string(props,
5753 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
5754 	spa = spa_add(poolname, nvl, altroot);
5755 	fnvlist_free(nvl);
5756 	spa_activate(spa, spa_mode_global);
5757 
5758 	if (props && (error = spa_prop_validate(spa, props))) {
5759 		spa_deactivate(spa);
5760 		spa_remove(spa);
5761 		mutex_exit(&spa_namespace_lock);
5762 		return (error);
5763 	}
5764 
5765 	/*
5766 	 * Temporary pool names should never be written to disk.
5767 	 */
5768 	if (poolname != pool)
5769 		spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME;
5770 
5771 	has_features = B_FALSE;
5772 	has_encryption = B_FALSE;
5773 	has_allocclass = B_FALSE;
5774 	for (nvpair_t *elem = nvlist_next_nvpair(props, NULL);
5775 	    elem != NULL; elem = nvlist_next_nvpair(props, elem)) {
5776 		if (zpool_prop_feature(nvpair_name(elem))) {
5777 			has_features = B_TRUE;
5778 
5779 			feat_name = strchr(nvpair_name(elem), '@') + 1;
5780 			VERIFY0(zfeature_lookup_name(feat_name, &feat));
5781 			if (feat == SPA_FEATURE_ENCRYPTION)
5782 				has_encryption = B_TRUE;
5783 			if (feat == SPA_FEATURE_ALLOCATION_CLASSES)
5784 				has_allocclass = B_TRUE;
5785 		}
5786 	}
5787 
5788 	/* verify encryption params, if they were provided */
5789 	if (dcp != NULL) {
5790 		error = spa_create_check_encryption_params(dcp, has_encryption);
5791 		if (error != 0) {
5792 			spa_deactivate(spa);
5793 			spa_remove(spa);
5794 			mutex_exit(&spa_namespace_lock);
5795 			return (error);
5796 		}
5797 	}
5798 	if (!has_allocclass && zfs_special_devs(nvroot, NULL)) {
5799 		spa_deactivate(spa);
5800 		spa_remove(spa);
5801 		mutex_exit(&spa_namespace_lock);
5802 		return (ENOTSUP);
5803 	}
5804 
5805 	if (has_features || nvlist_lookup_uint64(props,
5806 	    zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) {
5807 		version = SPA_VERSION;
5808 	}
5809 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
5810 
5811 	spa->spa_first_txg = txg;
5812 	spa->spa_uberblock.ub_txg = txg - 1;
5813 	spa->spa_uberblock.ub_version = version;
5814 	spa->spa_ubsync = spa->spa_uberblock;
5815 	spa->spa_load_state = SPA_LOAD_CREATE;
5816 	spa->spa_removing_phys.sr_state = DSS_NONE;
5817 	spa->spa_removing_phys.sr_removing_vdev = -1;
5818 	spa->spa_removing_phys.sr_prev_indirect_vdev = -1;
5819 	spa->spa_indirect_vdevs_loaded = B_TRUE;
5820 
5821 	/*
5822 	 * Create "The Godfather" zio to hold all async IOs
5823 	 */
5824 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
5825 	    KM_SLEEP);
5826 	for (int i = 0; i < max_ncpus; i++) {
5827 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
5828 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
5829 		    ZIO_FLAG_GODFATHER);
5830 	}
5831 
5832 	/*
5833 	 * Create the root vdev.
5834 	 */
5835 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5836 
5837 	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);
5838 
5839 	ASSERT(error != 0 || rvd != NULL);
5840 	ASSERT(error != 0 || spa->spa_root_vdev == rvd);
5841 
5842 	if (error == 0 && !zfs_allocatable_devs(nvroot))
5843 		error = SET_ERROR(EINVAL);
5844 
5845 	if (error == 0 &&
5846 	    (error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
5847 	    (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 &&
5848 	    (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) {
5849 		/*
5850 		 * instantiate the metaslab groups (this will dirty the vdevs)
5851 		 * we can no longer error exit past this point
5852 		 */
5853 		for (int c = 0; error == 0 && c < rvd->vdev_children; c++) {
5854 			vdev_t *vd = rvd->vdev_child[c];
5855 
5856 			vdev_metaslab_set_size(vd);
5857 			vdev_expand(vd, txg);
5858 		}
5859 	}
5860 
5861 	spa_config_exit(spa, SCL_ALL, FTAG);
5862 
5863 	if (error != 0) {
5864 		spa_unload(spa);
5865 		spa_deactivate(spa);
5866 		spa_remove(spa);
5867 		mutex_exit(&spa_namespace_lock);
5868 		return (error);
5869 	}
5870 
5871 	/*
5872 	 * Get the list of spares, if specified.
5873 	 */
5874 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
5875 	    &spares, &nspares) == 0) {
5876 		VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME,
5877 		    KM_SLEEP) == 0);
5878 		VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
5879 		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
5880 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5881 		spa_load_spares(spa);
5882 		spa_config_exit(spa, SCL_ALL, FTAG);
5883 		spa->spa_spares.sav_sync = B_TRUE;
5884 	}
5885 
5886 	/*
5887 	 * Get the list of level 2 cache devices, if specified.
5888 	 */
5889 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
5890 	    &l2cache, &nl2cache) == 0) {
5891 		VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config,
5892 		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
5893 		VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
5894 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
5895 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5896 		spa_load_l2cache(spa);
5897 		spa_config_exit(spa, SCL_ALL, FTAG);
5898 		spa->spa_l2cache.sav_sync = B_TRUE;
5899 	}
5900 
5901 	spa->spa_is_initializing = B_TRUE;
5902 	spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg);
5903 	spa->spa_is_initializing = B_FALSE;
5904 
5905 	/*
5906 	 * Create DDTs (dedup tables).
5907 	 */
5908 	ddt_create(spa);
5909 
5910 	spa_update_dspace(spa);
5911 
5912 	tx = dmu_tx_create_assigned(dp, txg);
5913 
5914 	/*
5915 	 * Create the pool's history object.
5916 	 */
5917 	if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history)
5918 		spa_history_create_obj(spa, tx);
5919 
5920 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE);
5921 	spa_history_log_version(spa, "create", tx);
5922 
5923 	/*
5924 	 * Create the pool config object.
5925 	 */
5926 	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
5927 	    DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE,
5928 	    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
5929 
5930 	if (zap_add(spa->spa_meta_objset,
5931 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
5932 	    sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
5933 		cmn_err(CE_PANIC, "failed to add pool config");
5934 	}
5935 
5936 	if (zap_add(spa->spa_meta_objset,
5937 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION,
5938 	    sizeof (uint64_t), 1, &version, tx) != 0) {
5939 		cmn_err(CE_PANIC, "failed to add pool version");
5940 	}
5941 
5942 	/* Newly created pools with the right version are always deflated. */
5943 	if (version >= SPA_VERSION_RAIDZ_DEFLATE) {
5944 		spa->spa_deflate = TRUE;
5945 		if (zap_add(spa->spa_meta_objset,
5946 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
5947 		    sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
5948 			cmn_err(CE_PANIC, "failed to add deflate");
5949 		}
5950 	}
5951 
5952 	/*
5953 	 * Create the deferred-free bpobj.  Turn off compression
5954 	 * because sync-to-convergence takes longer if the blocksize
5955 	 * keeps changing.
5956 	 */
5957 	obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
5958 	dmu_object_set_compress(spa->spa_meta_objset, obj,
5959 	    ZIO_COMPRESS_OFF, tx);
5960 	if (zap_add(spa->spa_meta_objset,
5961 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ,
5962 	    sizeof (uint64_t), 1, &obj, tx) != 0) {
5963 		cmn_err(CE_PANIC, "failed to add bpobj");
5964 	}
5965 	VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
5966 	    spa->spa_meta_objset, obj));
5967 
5968 	/*
5969 	 * Generate some random noise for salted checksums to operate on.
5970 	 */
5971 	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
5972 	    sizeof (spa->spa_cksum_salt.zcs_bytes));
5973 
5974 	/*
5975 	 * Set pool properties.
5976 	 */
5977 	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
5978 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
5979 	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
5980 	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
5981 	spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST);
5982 	spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM);
5983 
5984 	if (props != NULL) {
5985 		spa_configfile_set(spa, props, B_FALSE);
5986 		spa_sync_props(props, tx);
5987 	}
5988 
5989 	for (int i = 0; i < ndraid; i++)
5990 		spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
5991 
5992 	dmu_tx_commit(tx);
5993 
5994 	spa->spa_sync_on = B_TRUE;
5995 	txg_sync_start(dp);
5996 	mmp_thread_start(spa);
5997 	txg_wait_synced(dp, txg);
5998 
5999 	spa_spawn_aux_threads(spa);
6000 
6001 	spa_write_cachefile(spa, B_FALSE, B_TRUE);
6002 
6003 	/*
6004 	 * Don't count references from objsets that are already closed
6005 	 * and are making their way through the eviction process.
6006 	 */
6007 	spa_evicting_os_wait(spa);
6008 	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
6009 	spa->spa_load_state = SPA_LOAD_NONE;
6010 
6011 	mutex_exit(&spa_namespace_lock);
6012 
6013 	return (0);
6014 }
6015 
6016 /*
6017  * Import a non-root pool into the system.
6018  */
6019 int
6020 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
6021 {
6022 	spa_t *spa;
6023 	char *altroot = NULL;
6024 	spa_load_state_t state = SPA_LOAD_IMPORT;
6025 	zpool_load_policy_t policy;
6026 	spa_mode_t mode = spa_mode_global;
6027 	uint64_t readonly = B_FALSE;
6028 	int error;
6029 	nvlist_t *nvroot;
6030 	nvlist_t **spares, **l2cache;
6031 	uint_t nspares, nl2cache;
6032 
6033 	/*
6034 	 * If a pool with this name exists, return failure.
6035 	 */
6036 	mutex_enter(&spa_namespace_lock);
6037 	if (spa_lookup(pool) != NULL) {
6038 		mutex_exit(&spa_namespace_lock);
6039 		return (SET_ERROR(EEXIST));
6040 	}
6041 
6042 	/*
6043 	 * Create and initialize the spa structure.
6044 	 */
6045 	(void) nvlist_lookup_string(props,
6046 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
6047 	(void) nvlist_lookup_uint64(props,
6048 	    zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
6049 	if (readonly)
6050 		mode = SPA_MODE_READ;
6051 	spa = spa_add(pool, config, altroot);
6052 	spa->spa_import_flags = flags;
6053 
6054 	/*
6055 	 * Verbatim import - Take a pool and insert it into the namespace
6056 	 * as if it had been loaded at boot.
6057 	 */
6058 	if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
6059 		if (props != NULL)
6060 			spa_configfile_set(spa, props, B_FALSE);
6061 
6062 		spa_write_cachefile(spa, B_FALSE, B_TRUE);
6063 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
6064 		zfs_dbgmsg("spa_import: verbatim import of %s", pool);
6065 		mutex_exit(&spa_namespace_lock);
6066 		return (0);
6067 	}
6068 
6069 	spa_activate(spa, mode);
6070 
6071 	/*
6072 	 * Don't start async tasks until we know everything is healthy.
6073 	 */
6074 	spa_async_suspend(spa);
6075 
6076 	zpool_get_load_policy(config, &policy);
6077 	if (policy.zlp_rewind & ZPOOL_DO_REWIND)
6078 		state = SPA_LOAD_RECOVER;
6079 
6080 	spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT;
6081 
6082 	if (state != SPA_LOAD_RECOVER) {
6083 		spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
6084 		zfs_dbgmsg("spa_import: importing %s", pool);
6085 	} else {
6086 		zfs_dbgmsg("spa_import: importing %s, max_txg=%lld "
6087 		    "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg);
6088 	}
6089 	error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind);
6090 
6091 	/*
6092 	 * Propagate anything learned while loading the pool and pass it
6093 	 * back to caller (i.e. rewind info, missing devices, etc).
6094 	 */
6095 	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
6096 	    spa->spa_load_info) == 0);
6097 
6098 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6099 	/*
6100 	 * Toss any existing sparelist, as it doesn't have any validity
6101 	 * anymore, and conflicts with spa_has_spare().
6102 	 */
6103 	if (spa->spa_spares.sav_config) {
6104 		nvlist_free(spa->spa_spares.sav_config);
6105 		spa->spa_spares.sav_config = NULL;
6106 		spa_load_spares(spa);
6107 	}
6108 	if (spa->spa_l2cache.sav_config) {
6109 		nvlist_free(spa->spa_l2cache.sav_config);
6110 		spa->spa_l2cache.sav_config = NULL;
6111 		spa_load_l2cache(spa);
6112 	}
6113 
6114 	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
6115 	    &nvroot) == 0);
6116 	spa_config_exit(spa, SCL_ALL, FTAG);
6117 
6118 	if (props != NULL)
6119 		spa_configfile_set(spa, props, B_FALSE);
6120 
6121 	if (error != 0 || (props && spa_writeable(spa) &&
6122 	    (error = spa_prop_set(spa, props)))) {
6123 		spa_unload(spa);
6124 		spa_deactivate(spa);
6125 		spa_remove(spa);
6126 		mutex_exit(&spa_namespace_lock);
6127 		return (error);
6128 	}
6129 
6130 	spa_async_resume(spa);
6131 
6132 	/*
6133 	 * Override any spares and level 2 cache devices as specified by
6134 	 * the user, as these may have correct device names/devids, etc.
6135 	 */
6136 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6137 	    &spares, &nspares) == 0) {
6138 		if (spa->spa_spares.sav_config)
6139 			VERIFY(nvlist_remove(spa->spa_spares.sav_config,
6140 			    ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0);
6141 		else
6142 			VERIFY(nvlist_alloc(&spa->spa_spares.sav_config,
6143 			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
6144 		VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
6145 		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
6146 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6147 		spa_load_spares(spa);
6148 		spa_config_exit(spa, SCL_ALL, FTAG);
6149 		spa->spa_spares.sav_sync = B_TRUE;
6150 	}
6151 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6152 	    &l2cache, &nl2cache) == 0) {
6153 		if (spa->spa_l2cache.sav_config)
6154 			VERIFY(nvlist_remove(spa->spa_l2cache.sav_config,
6155 			    ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0);
6156 		else
6157 			VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config,
6158 			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
6159 		VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
6160 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
6161 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6162 		spa_load_l2cache(spa);
6163 		spa_config_exit(spa, SCL_ALL, FTAG);
6164 		spa->spa_l2cache.sav_sync = B_TRUE;
6165 	}
6166 
6167 	/*
6168 	 * Check for any removed devices.
6169 	 */
6170 	if (spa->spa_autoreplace) {
6171 		spa_aux_check_removed(&spa->spa_spares);
6172 		spa_aux_check_removed(&spa->spa_l2cache);
6173 	}
6174 
6175 	if (spa_writeable(spa)) {
6176 		/*
6177 		 * Update the config cache to include the newly-imported pool.
6178 		 */
6179 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
6180 	}
6181 
6182 	/*
6183 	 * It's possible that the pool was expanded while it was exported.
6184 	 * We kick off an async task to handle this for us.
6185 	 */
6186 	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
6187 
6188 	spa_history_log_version(spa, "import", NULL);
6189 
6190 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
6191 
6192 	mutex_exit(&spa_namespace_lock);
6193 
6194 	zvol_create_minors_recursive(pool);
6195 
6196 	return (0);
6197 }
6198 
6199 nvlist_t *
6200 spa_tryimport(nvlist_t *tryconfig)
6201 {
6202 	nvlist_t *config = NULL;
6203 	char *poolname, *cachefile;
6204 	spa_t *spa;
6205 	uint64_t state;
6206 	int error;
6207 	zpool_load_policy_t policy;
6208 
6209 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
6210 		return (NULL);
6211 
6212 	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
6213 		return (NULL);
6214 
6215 	/*
6216 	 * Create and initialize the spa structure.
6217 	 */
6218 	mutex_enter(&spa_namespace_lock);
6219 	spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL);
6220 	spa_activate(spa, SPA_MODE_READ);
6221 
6222 	/*
6223 	 * Rewind pool if a max txg was provided.
6224 	 */
6225 	zpool_get_load_policy(spa->spa_config, &policy);
6226 	if (policy.zlp_txg != UINT64_MAX) {
6227 		spa->spa_load_max_txg = policy.zlp_txg;
6228 		spa->spa_extreme_rewind = B_TRUE;
6229 		zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld",
6230 		    poolname, (longlong_t)policy.zlp_txg);
6231 	} else {
6232 		zfs_dbgmsg("spa_tryimport: importing %s", poolname);
6233 	}
6234 
6235 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile)
6236 	    == 0) {
6237 		zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile);
6238 		spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
6239 	} else {
6240 		spa->spa_config_source = SPA_CONFIG_SRC_SCAN;
6241 	}
6242 
6243 	error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING);
6244 
6245 	/*
6246 	 * If 'tryconfig' was at least parsable, return the current config.
6247 	 */
6248 	if (spa->spa_root_vdev != NULL) {
6249 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
6250 		VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME,
6251 		    poolname) == 0);
6252 		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
6253 		    state) == 0);
6254 		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
6255 		    spa->spa_uberblock.ub_timestamp) == 0);
6256 		VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
6257 		    spa->spa_load_info) == 0);
6258 		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA,
6259 		    spa->spa_errata) == 0);
6260 
6261 		/*
6262 		 * If the bootfs property exists on this pool then we
6263 		 * copy it out so that external consumers can tell which
6264 		 * pools are bootable.
6265 		 */
6266 		if ((!error || error == EEXIST) && spa->spa_bootfs) {
6267 			char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
6268 
6269 			/*
6270 			 * We have to play games with the name since the
6271 			 * pool was opened as TRYIMPORT_NAME.
6272 			 */
6273 			if (dsl_dsobj_to_dsname(spa_name(spa),
6274 			    spa->spa_bootfs, tmpname) == 0) {
6275 				char *cp;
6276 				char *dsname;
6277 
6278 				dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
6279 
6280 				cp = strchr(tmpname, '/');
6281 				if (cp == NULL) {
6282 					(void) strlcpy(dsname, tmpname,
6283 					    MAXPATHLEN);
6284 				} else {
6285 					(void) snprintf(dsname, MAXPATHLEN,
6286 					    "%s/%s", poolname, ++cp);
6287 				}
6288 				VERIFY(nvlist_add_string(config,
6289 				    ZPOOL_CONFIG_BOOTFS, dsname) == 0);
6290 				kmem_free(dsname, MAXPATHLEN);
6291 			}
6292 			kmem_free(tmpname, MAXPATHLEN);
6293 		}
6294 
6295 		/*
6296 		 * Add the list of hot spares and level 2 cache devices.
6297 		 */
6298 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
6299 		spa_add_spares(spa, config);
6300 		spa_add_l2cache(spa, config);
6301 		spa_config_exit(spa, SCL_CONFIG, FTAG);
6302 	}
6303 
6304 	spa_unload(spa);
6305 	spa_deactivate(spa);
6306 	spa_remove(spa);
6307 	mutex_exit(&spa_namespace_lock);
6308 
6309 	return (config);
6310 }
6311 
6312 /*
6313  * Pool export/destroy
6314  *
6315  * The act of destroying or exporting a pool is very simple.  We make sure there
6316  * is no more pending I/O and any references to the pool are gone.  Then, we
6317  * update the pool state and sync all the labels to disk, removing the
6318  * configuration from the cache afterwards. If the 'hardforce' flag is set, then
6319  * we don't sync the labels or remove the configuration cache.
6320  */
6321 static int
6322 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig,
6323     boolean_t force, boolean_t hardforce)
6324 {
6325 	int error;
6326 	spa_t *spa;
6327 
6328 	if (oldconfig)
6329 		*oldconfig = NULL;
6330 
6331 	if (!(spa_mode_global & SPA_MODE_WRITE))
6332 		return (SET_ERROR(EROFS));
6333 
6334 	mutex_enter(&spa_namespace_lock);
6335 	if ((spa = spa_lookup(pool)) == NULL) {
6336 		mutex_exit(&spa_namespace_lock);
6337 		return (SET_ERROR(ENOENT));
6338 	}
6339 
6340 	if (spa->spa_is_exporting) {
6341 		/* the pool is being exported by another thread */
6342 		mutex_exit(&spa_namespace_lock);
6343 		return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS));
6344 	}
6345 	spa->spa_is_exporting = B_TRUE;
6346 
6347 	/*
6348 	 * Put a hold on the pool, drop the namespace lock, stop async tasks,
6349 	 * reacquire the namespace lock, and see if we can export.
6350 	 */
6351 	spa_open_ref(spa, FTAG);
6352 	mutex_exit(&spa_namespace_lock);
6353 	spa_async_suspend(spa);
6354 	if (spa->spa_zvol_taskq) {
6355 		zvol_remove_minors(spa, spa_name(spa), B_TRUE);
6356 		taskq_wait(spa->spa_zvol_taskq);
6357 	}
6358 	mutex_enter(&spa_namespace_lock);
6359 	spa_close(spa, FTAG);
6360 
6361 	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
6362 		goto export_spa;
6363 	/*
6364 	 * The pool will be in core if it's openable, in which case we can
6365 	 * modify its state.  Objsets may be open only because they're dirty,
6366 	 * so we have to force it to sync before checking spa_refcnt.
6367 	 */
6368 	if (spa->spa_sync_on) {
6369 		txg_wait_synced(spa->spa_dsl_pool, 0);
6370 		spa_evicting_os_wait(spa);
6371 	}
6372 
6373 	/*
6374 	 * A pool cannot be exported or destroyed if there are active
6375 	 * references.  If we are resetting a pool, allow references by
6376 	 * fault injection handlers.
6377 	 */
6378 	if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) {
6379 		error = SET_ERROR(EBUSY);
6380 		goto fail;
6381 	}
6382 
6383 	if (spa->spa_sync_on) {
6384 		/*
6385 		 * A pool cannot be exported if it has an active shared spare.
6386 		 * This is to prevent other pools stealing the active spare
6387 		 * from an exported pool. At user's own will, such pool can
6388 		 * be forcedly exported.
6389 		 */
6390 		if (!force && new_state == POOL_STATE_EXPORTED &&
6391 		    spa_has_active_shared_spare(spa)) {
6392 			error = SET_ERROR(EXDEV);
6393 			goto fail;
6394 		}
6395 
6396 		/*
6397 		 * We're about to export or destroy this pool. Make sure
6398 		 * we stop all initialization and trim activity here before
6399 		 * we set the spa_final_txg. This will ensure that all
6400 		 * dirty data resulting from the initialization is
6401 		 * committed to disk before we unload the pool.
6402 		 */
6403 		if (spa->spa_root_vdev != NULL) {
6404 			vdev_t *rvd = spa->spa_root_vdev;
6405 			vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE);
6406 			vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE);
6407 			vdev_autotrim_stop_all(spa);
6408 			vdev_rebuild_stop_all(spa);
6409 		}
6410 
6411 		/*
6412 		 * We want this to be reflected on every label,
6413 		 * so mark them all dirty.  spa_unload() will do the
6414 		 * final sync that pushes these changes out.
6415 		 */
6416 		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
6417 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6418 			spa->spa_state = new_state;
6419 			spa->spa_final_txg = spa_last_synced_txg(spa) +
6420 			    TXG_DEFER_SIZE + 1;
6421 			vdev_config_dirty(spa->spa_root_vdev);
6422 			spa_config_exit(spa, SCL_ALL, FTAG);
6423 		}
6424 	}
6425 
6426 export_spa:
6427 	if (new_state == POOL_STATE_DESTROYED)
6428 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY);
6429 	else if (new_state == POOL_STATE_EXPORTED)
6430 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT);
6431 
6432 	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
6433 		spa_unload(spa);
6434 		spa_deactivate(spa);
6435 	}
6436 
6437 	if (oldconfig && spa->spa_config)
6438 		VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0);
6439 
6440 	if (new_state != POOL_STATE_UNINITIALIZED) {
6441 		if (!hardforce)
6442 			spa_write_cachefile(spa, B_TRUE, B_TRUE);
6443 		spa_remove(spa);
6444 	} else {
6445 		/*
6446 		 * If spa_remove() is not called for this spa_t and
6447 		 * there is any possibility that it can be reused,
6448 		 * we make sure to reset the exporting flag.
6449 		 */
6450 		spa->spa_is_exporting = B_FALSE;
6451 	}
6452 
6453 	mutex_exit(&spa_namespace_lock);
6454 	return (0);
6455 
6456 fail:
6457 	spa->spa_is_exporting = B_FALSE;
6458 	spa_async_resume(spa);
6459 	mutex_exit(&spa_namespace_lock);
6460 	return (error);
6461 }
6462 
6463 /*
6464  * Destroy a storage pool.
6465  */
6466 int
6467 spa_destroy(const char *pool)
6468 {
6469 	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL,
6470 	    B_FALSE, B_FALSE));
6471 }
6472 
6473 /*
6474  * Export a storage pool.
6475  */
6476 int
6477 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force,
6478     boolean_t hardforce)
6479 {
6480 	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig,
6481 	    force, hardforce));
6482 }
6483 
6484 /*
6485  * Similar to spa_export(), this unloads the spa_t without actually removing it
6486  * from the namespace in any way.
6487  */
6488 int
6489 spa_reset(const char *pool)
6490 {
6491 	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL,
6492 	    B_FALSE, B_FALSE));
6493 }
6494 
6495 /*
6496  * ==========================================================================
6497  * Device manipulation
6498  * ==========================================================================
6499  */
6500 
6501 /*
6502  * This is called as a synctask to increment the draid feature flag
6503  */
6504 static void
6505 spa_draid_feature_incr(void *arg, dmu_tx_t *tx)
6506 {
6507 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
6508 	int draid = (int)(uintptr_t)arg;
6509 
6510 	for (int c = 0; c < draid; c++)
6511 		spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
6512 }
6513 
6514 /*
6515  * Add a device to a storage pool.
6516  */
6517 int
6518 spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
6519 {
6520 	uint64_t txg, ndraid = 0;
6521 	int error;
6522 	vdev_t *rvd = spa->spa_root_vdev;
6523 	vdev_t *vd, *tvd;
6524 	nvlist_t **spares, **l2cache;
6525 	uint_t nspares, nl2cache;
6526 
6527 	ASSERT(spa_writeable(spa));
6528 
6529 	txg = spa_vdev_enter(spa);
6530 
6531 	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
6532 	    VDEV_ALLOC_ADD)) != 0)
6533 		return (spa_vdev_exit(spa, NULL, txg, error));
6534 
6535 	spa->spa_pending_vdev = vd;	/* spa_vdev_exit() will clear this */
6536 
6537 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
6538 	    &nspares) != 0)
6539 		nspares = 0;
6540 
6541 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache,
6542 	    &nl2cache) != 0)
6543 		nl2cache = 0;
6544 
6545 	if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
6546 		return (spa_vdev_exit(spa, vd, txg, EINVAL));
6547 
6548 	if (vd->vdev_children != 0 &&
6549 	    (error = vdev_create(vd, txg, B_FALSE)) != 0) {
6550 		return (spa_vdev_exit(spa, vd, txg, error));
6551 	}
6552 
6553 	/*
6554 	 * The virtual dRAID spares must be added after vdev tree is created
6555 	 * and the vdev guids are generated.  The guid of their associated
6556 	 * dRAID is stored in the config and used when opening the spare.
6557 	 */
6558 	if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid,
6559 	    rvd->vdev_children)) == 0) {
6560 		if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot,
6561 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0)
6562 			nspares = 0;
6563 	} else {
6564 		return (spa_vdev_exit(spa, vd, txg, error));
6565 	}
6566 
6567 	/*
6568 	 * We must validate the spares and l2cache devices after checking the
6569 	 * children.  Otherwise, vdev_inuse() will blindly overwrite the spare.
6570 	 */
6571 	if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0)
6572 		return (spa_vdev_exit(spa, vd, txg, error));
6573 
6574 	/*
6575 	 * If we are in the middle of a device removal, we can only add
6576 	 * devices which match the existing devices in the pool.
6577 	 * If we are in the middle of a removal, or have some indirect
6578 	 * vdevs, we can not add raidz or dRAID top levels.
6579 	 */
6580 	if (spa->spa_vdev_removal != NULL ||
6581 	    spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
6582 		for (int c = 0; c < vd->vdev_children; c++) {
6583 			tvd = vd->vdev_child[c];
6584 			if (spa->spa_vdev_removal != NULL &&
6585 			    tvd->vdev_ashift != spa->spa_max_ashift) {
6586 				return (spa_vdev_exit(spa, vd, txg, EINVAL));
6587 			}
6588 			/* Fail if top level vdev is raidz or a dRAID */
6589 			if (vdev_get_nparity(tvd) != 0)
6590 				return (spa_vdev_exit(spa, vd, txg, EINVAL));
6591 
6592 			/*
6593 			 * Need the top level mirror to be
6594 			 * a mirror of leaf vdevs only
6595 			 */
6596 			if (tvd->vdev_ops == &vdev_mirror_ops) {
6597 				for (uint64_t cid = 0;
6598 				    cid < tvd->vdev_children; cid++) {
6599 					vdev_t *cvd = tvd->vdev_child[cid];
6600 					if (!cvd->vdev_ops->vdev_op_leaf) {
6601 						return (spa_vdev_exit(spa, vd,
6602 						    txg, EINVAL));
6603 					}
6604 				}
6605 			}
6606 		}
6607 	}
6608 
6609 	for (int c = 0; c < vd->vdev_children; c++) {
6610 		tvd = vd->vdev_child[c];
6611 		vdev_remove_child(vd, tvd);
6612 		tvd->vdev_id = rvd->vdev_children;
6613 		vdev_add_child(rvd, tvd);
6614 		vdev_config_dirty(tvd);
6615 	}
6616 
6617 	if (nspares != 0) {
6618 		spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
6619 		    ZPOOL_CONFIG_SPARES);
6620 		spa_load_spares(spa);
6621 		spa->spa_spares.sav_sync = B_TRUE;
6622 	}
6623 
6624 	if (nl2cache != 0) {
6625 		spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
6626 		    ZPOOL_CONFIG_L2CACHE);
6627 		spa_load_l2cache(spa);
6628 		spa->spa_l2cache.sav_sync = B_TRUE;
6629 	}
6630 
6631 	/*
6632 	 * We can't increment a feature while holding spa_vdev so we
6633 	 * have to do it in a synctask.
6634 	 */
6635 	if (ndraid != 0) {
6636 		dmu_tx_t *tx;
6637 
6638 		tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
6639 		dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr,
6640 		    (void *)(uintptr_t)ndraid, tx);
6641 		dmu_tx_commit(tx);
6642 	}
6643 
6644 	/*
6645 	 * We have to be careful when adding new vdevs to an existing pool.
6646 	 * If other threads start allocating from these vdevs before we
6647 	 * sync the config cache, and we lose power, then upon reboot we may
6648 	 * fail to open the pool because there are DVAs that the config cache
6649 	 * can't translate.  Therefore, we first add the vdevs without
6650 	 * initializing metaslabs; sync the config cache (via spa_vdev_exit());
6651 	 * and then let spa_config_update() initialize the new metaslabs.
6652 	 *
6653 	 * spa_load() checks for added-but-not-initialized vdevs, so that
6654 	 * if we lose power at any point in this sequence, the remaining
6655 	 * steps will be completed the next time we load the pool.
6656 	 */
6657 	(void) spa_vdev_exit(spa, vd, txg, 0);
6658 
6659 	mutex_enter(&spa_namespace_lock);
6660 	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
6661 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD);
6662 	mutex_exit(&spa_namespace_lock);
6663 
6664 	return (0);
6665 }
6666 
6667 /*
6668  * Attach a device to a mirror.  The arguments are the path to any device
6669  * in the mirror, and the nvroot for the new device.  If the path specifies
6670  * a device that is not mirrored, we automatically insert the mirror vdev.
6671  *
6672  * If 'replacing' is specified, the new device is intended to replace the
6673  * existing device; in this case the two devices are made into their own
6674  * mirror using the 'replacing' vdev, which is functionally identical to
6675  * the mirror vdev (it actually reuses all the same ops) but has a few
6676  * extra rules: you can't attach to it after it's been created, and upon
6677  * completion of resilvering, the first disk (the one being replaced)
6678  * is automatically detached.
6679  *
6680  * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild)
6681  * should be performed instead of traditional healing reconstruction.  From
6682  * an administrators perspective these are both resilver operations.
6683  */
6684 int
6685 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing,
6686     int rebuild)
6687 {
6688 	uint64_t txg, dtl_max_txg;
6689 	vdev_t *rvd = spa->spa_root_vdev;
6690 	vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd;
6691 	vdev_ops_t *pvops;
6692 	char *oldvdpath, *newvdpath;
6693 	int newvd_isspare;
6694 	int error;
6695 
6696 	ASSERT(spa_writeable(spa));
6697 
6698 	txg = spa_vdev_enter(spa);
6699 
6700 	oldvd = spa_lookup_by_guid(spa, guid, B_FALSE);
6701 
6702 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
6703 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
6704 		error = (spa_has_checkpoint(spa)) ?
6705 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
6706 		return (spa_vdev_exit(spa, NULL, txg, error));
6707 	}
6708 
6709 	if (rebuild) {
6710 		if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD))
6711 			return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
6712 
6713 		if (dsl_scan_resilvering(spa_get_dsl(spa)))
6714 			return (spa_vdev_exit(spa, NULL, txg,
6715 			    ZFS_ERR_RESILVER_IN_PROGRESS));
6716 	} else {
6717 		if (vdev_rebuild_active(rvd))
6718 			return (spa_vdev_exit(spa, NULL, txg,
6719 			    ZFS_ERR_REBUILD_IN_PROGRESS));
6720 	}
6721 
6722 	if (spa->spa_vdev_removal != NULL)
6723 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
6724 
6725 	if (oldvd == NULL)
6726 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
6727 
6728 	if (!oldvd->vdev_ops->vdev_op_leaf)
6729 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
6730 
6731 	pvd = oldvd->vdev_parent;
6732 
6733 	if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
6734 	    VDEV_ALLOC_ATTACH)) != 0)
6735 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
6736 
6737 	if (newrootvd->vdev_children != 1)
6738 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
6739 
6740 	newvd = newrootvd->vdev_child[0];
6741 
6742 	if (!newvd->vdev_ops->vdev_op_leaf)
6743 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
6744 
6745 	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
6746 		return (spa_vdev_exit(spa, newrootvd, txg, error));
6747 
6748 	/*
6749 	 * Spares can't replace logs
6750 	 */
6751 	if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare)
6752 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6753 
6754 	/*
6755 	 * A dRAID spare can only replace a child of its parent dRAID vdev.
6756 	 */
6757 	if (newvd->vdev_ops == &vdev_draid_spare_ops &&
6758 	    oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) {
6759 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6760 	}
6761 
6762 	if (rebuild) {
6763 		/*
6764 		 * For rebuilds, the top vdev must support reconstruction
6765 		 * using only space maps.  This means the only allowable
6766 		 * vdevs types are the root vdev, a mirror, or dRAID.
6767 		 */
6768 		tvd = pvd;
6769 		if (pvd->vdev_top != NULL)
6770 			tvd = pvd->vdev_top;
6771 
6772 		if (tvd->vdev_ops != &vdev_mirror_ops &&
6773 		    tvd->vdev_ops != &vdev_root_ops &&
6774 		    tvd->vdev_ops != &vdev_draid_ops) {
6775 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6776 		}
6777 	}
6778 
6779 	if (!replacing) {
6780 		/*
6781 		 * For attach, the only allowable parent is a mirror or the root
6782 		 * vdev.
6783 		 */
6784 		if (pvd->vdev_ops != &vdev_mirror_ops &&
6785 		    pvd->vdev_ops != &vdev_root_ops)
6786 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6787 
6788 		pvops = &vdev_mirror_ops;
6789 	} else {
6790 		/*
6791 		 * Active hot spares can only be replaced by inactive hot
6792 		 * spares.
6793 		 */
6794 		if (pvd->vdev_ops == &vdev_spare_ops &&
6795 		    oldvd->vdev_isspare &&
6796 		    !spa_has_spare(spa, newvd->vdev_guid))
6797 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6798 
6799 		/*
6800 		 * If the source is a hot spare, and the parent isn't already a
6801 		 * spare, then we want to create a new hot spare.  Otherwise, we
6802 		 * want to create a replacing vdev.  The user is not allowed to
6803 		 * attach to a spared vdev child unless the 'isspare' state is
6804 		 * the same (spare replaces spare, non-spare replaces
6805 		 * non-spare).
6806 		 */
6807 		if (pvd->vdev_ops == &vdev_replacing_ops &&
6808 		    spa_version(spa) < SPA_VERSION_MULTI_REPLACE) {
6809 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6810 		} else if (pvd->vdev_ops == &vdev_spare_ops &&
6811 		    newvd->vdev_isspare != oldvd->vdev_isspare) {
6812 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6813 		}
6814 
6815 		if (newvd->vdev_isspare)
6816 			pvops = &vdev_spare_ops;
6817 		else
6818 			pvops = &vdev_replacing_ops;
6819 	}
6820 
6821 	/*
6822 	 * Make sure the new device is big enough.
6823 	 */
6824 	if (newvd->vdev_asize < vdev_get_min_asize(oldvd))
6825 		return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));
6826 
6827 	/*
6828 	 * The new device cannot have a higher alignment requirement
6829 	 * than the top-level vdev.
6830 	 */
6831 	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
6832 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
6833 
6834 	/*
6835 	 * If this is an in-place replacement, update oldvd's path and devid
6836 	 * to make it distinguishable from newvd, and unopenable from now on.
6837 	 */
6838 	if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) {
6839 		spa_strfree(oldvd->vdev_path);
6840 		oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5,
6841 		    KM_SLEEP);
6842 		(void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5,
6843 		    "%s/%s", newvd->vdev_path, "old");
6844 		if (oldvd->vdev_devid != NULL) {
6845 			spa_strfree(oldvd->vdev_devid);
6846 			oldvd->vdev_devid = NULL;
6847 		}
6848 	}
6849 
6850 	/*
6851 	 * If the parent is not a mirror, or if we're replacing, insert the new
6852 	 * mirror/replacing/spare vdev above oldvd.
6853 	 */
6854 	if (pvd->vdev_ops != pvops)
6855 		pvd = vdev_add_parent(oldvd, pvops);
6856 
6857 	ASSERT(pvd->vdev_top->vdev_parent == rvd);
6858 	ASSERT(pvd->vdev_ops == pvops);
6859 	ASSERT(oldvd->vdev_parent == pvd);
6860 
6861 	/*
6862 	 * Extract the new device from its root and add it to pvd.
6863 	 */
6864 	vdev_remove_child(newrootvd, newvd);
6865 	newvd->vdev_id = pvd->vdev_children;
6866 	newvd->vdev_crtxg = oldvd->vdev_crtxg;
6867 	vdev_add_child(pvd, newvd);
6868 
6869 	/*
6870 	 * Reevaluate the parent vdev state.
6871 	 */
6872 	vdev_propagate_state(pvd);
6873 
6874 	tvd = newvd->vdev_top;
6875 	ASSERT(pvd->vdev_top == tvd);
6876 	ASSERT(tvd->vdev_parent == rvd);
6877 
6878 	vdev_config_dirty(tvd);
6879 
6880 	/*
6881 	 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account
6882 	 * for any dmu_sync-ed blocks.  It will propagate upward when
6883 	 * spa_vdev_exit() calls vdev_dtl_reassess().
6884 	 */
6885 	dtl_max_txg = txg + TXG_CONCURRENT_STATES;
6886 
6887 	vdev_dtl_dirty(newvd, DTL_MISSING,
6888 	    TXG_INITIAL, dtl_max_txg - TXG_INITIAL);
6889 
6890 	if (newvd->vdev_isspare) {
6891 		spa_spare_activate(newvd);
6892 		spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE);
6893 	}
6894 
6895 	oldvdpath = spa_strdup(oldvd->vdev_path);
6896 	newvdpath = spa_strdup(newvd->vdev_path);
6897 	newvd_isspare = newvd->vdev_isspare;
6898 
6899 	/*
6900 	 * Mark newvd's DTL dirty in this txg.
6901 	 */
6902 	vdev_dirty(tvd, VDD_DTL, newvd, txg);
6903 
6904 	/*
6905 	 * Schedule the resilver or rebuild to restart in the future. We do
6906 	 * this to ensure that dmu_sync-ed blocks have been stitched into the
6907 	 * respective datasets.
6908 	 */
6909 	if (rebuild) {
6910 		newvd->vdev_rebuild_txg = txg;
6911 
6912 		vdev_rebuild(tvd);
6913 	} else {
6914 		newvd->vdev_resilver_txg = txg;
6915 
6916 		if (dsl_scan_resilvering(spa_get_dsl(spa)) &&
6917 		    spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) {
6918 			vdev_defer_resilver(newvd);
6919 		} else {
6920 			dsl_scan_restart_resilver(spa->spa_dsl_pool,
6921 			    dtl_max_txg);
6922 		}
6923 	}
6924 
6925 	if (spa->spa_bootfs)
6926 		spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH);
6927 
6928 	spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH);
6929 
6930 	/*
6931 	 * Commit the config
6932 	 */
6933 	(void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0);
6934 
6935 	spa_history_log_internal(spa, "vdev attach", NULL,
6936 	    "%s vdev=%s %s vdev=%s",
6937 	    replacing && newvd_isspare ? "spare in" :
6938 	    replacing ? "replace" : "attach", newvdpath,
6939 	    replacing ? "for" : "to", oldvdpath);
6940 
6941 	spa_strfree(oldvdpath);
6942 	spa_strfree(newvdpath);
6943 
6944 	return (0);
6945 }
6946 
6947 /*
6948  * Detach a device from a mirror or replacing vdev.
6949  *
6950  * If 'replace_done' is specified, only detach if the parent
6951  * is a replacing vdev.
6952  */
6953 int
6954 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)
6955 {
6956 	uint64_t txg;
6957 	int error;
6958 	vdev_t *rvd __maybe_unused = spa->spa_root_vdev;
6959 	vdev_t *vd, *pvd, *cvd, *tvd;
6960 	boolean_t unspare = B_FALSE;
6961 	uint64_t unspare_guid = 0;
6962 	char *vdpath;
6963 
6964 	ASSERT(spa_writeable(spa));
6965 
6966 	txg = spa_vdev_detach_enter(spa, guid);
6967 
6968 	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
6969 
6970 	/*
6971 	 * Besides being called directly from the userland through the
6972 	 * ioctl interface, spa_vdev_detach() can be potentially called
6973 	 * at the end of spa_vdev_resilver_done().
6974 	 *
6975 	 * In the regular case, when we have a checkpoint this shouldn't
6976 	 * happen as we never empty the DTLs of a vdev during the scrub
6977 	 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done()
6978 	 * should never get here when we have a checkpoint.
6979 	 *
6980 	 * That said, even in a case when we checkpoint the pool exactly
6981 	 * as spa_vdev_resilver_done() calls this function everything
6982 	 * should be fine as the resilver will return right away.
6983 	 */
6984 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
6985 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
6986 		error = (spa_has_checkpoint(spa)) ?
6987 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
6988 		return (spa_vdev_exit(spa, NULL, txg, error));
6989 	}
6990 
6991 	if (vd == NULL)
6992 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
6993 
6994 	if (!vd->vdev_ops->vdev_op_leaf)
6995 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
6996 
6997 	pvd = vd->vdev_parent;
6998 
6999 	/*
7000 	 * If the parent/child relationship is not as expected, don't do it.
7001 	 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
7002 	 * vdev that's replacing B with C.  The user's intent in replacing
7003 	 * is to go from M(A,B) to M(A,C).  If the user decides to cancel
7004 	 * the replace by detaching C, the expected behavior is to end up
7005 	 * M(A,B).  But suppose that right after deciding to detach C,
7006 	 * the replacement of B completes.  We would have M(A,C), and then
7007 	 * ask to detach C, which would leave us with just A -- not what
7008 	 * the user wanted.  To prevent this, we make sure that the
7009 	 * parent/child relationship hasn't changed -- in this example,
7010 	 * that C's parent is still the replacing vdev R.
7011 	 */
7012 	if (pvd->vdev_guid != pguid && pguid != 0)
7013 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
7014 
7015 	/*
7016 	 * Only 'replacing' or 'spare' vdevs can be replaced.
7017 	 */
7018 	if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
7019 	    pvd->vdev_ops != &vdev_spare_ops)
7020 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7021 
7022 	ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
7023 	    spa_version(spa) >= SPA_VERSION_SPARES);
7024 
7025 	/*
7026 	 * Only mirror, replacing, and spare vdevs support detach.
7027 	 */
7028 	if (pvd->vdev_ops != &vdev_replacing_ops &&
7029 	    pvd->vdev_ops != &vdev_mirror_ops &&
7030 	    pvd->vdev_ops != &vdev_spare_ops)
7031 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7032 
7033 	/*
7034 	 * If this device has the only valid copy of some data,
7035 	 * we cannot safely detach it.
7036 	 */
7037 	if (vdev_dtl_required(vd))
7038 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
7039 
7040 	ASSERT(pvd->vdev_children >= 2);
7041 
7042 	/*
7043 	 * If we are detaching the second disk from a replacing vdev, then
7044 	 * check to see if we changed the original vdev's path to have "/old"
7045 	 * at the end in spa_vdev_attach().  If so, undo that change now.
7046 	 */
7047 	if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
7048 	    vd->vdev_path != NULL) {
7049 		size_t len = strlen(vd->vdev_path);
7050 
7051 		for (int c = 0; c < pvd->vdev_children; c++) {
7052 			cvd = pvd->vdev_child[c];
7053 
7054 			if (cvd == vd || cvd->vdev_path == NULL)
7055 				continue;
7056 
7057 			if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
7058 			    strcmp(cvd->vdev_path + len, "/old") == 0) {
7059 				spa_strfree(cvd->vdev_path);
7060 				cvd->vdev_path = spa_strdup(vd->vdev_path);
7061 				break;
7062 			}
7063 		}
7064 	}
7065 
7066 	/*
7067 	 * If we are detaching the original disk from a normal spare, then it
7068 	 * implies that the spare should become a real disk, and be removed
7069 	 * from the active spare list for the pool.  dRAID spares on the
7070 	 * other hand are coupled to the pool and thus should never be removed
7071 	 * from the spares list.
7072 	 */
7073 	if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) {
7074 		vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1];
7075 
7076 		if (last_cvd->vdev_isspare &&
7077 		    last_cvd->vdev_ops != &vdev_draid_spare_ops) {
7078 			unspare = B_TRUE;
7079 		}
7080 	}
7081 
7082 	/*
7083 	 * Erase the disk labels so the disk can be used for other things.
7084 	 * This must be done after all other error cases are handled,
7085 	 * but before we disembowel vd (so we can still do I/O to it).
7086 	 * But if we can't do it, don't treat the error as fatal --
7087 	 * it may be that the unwritability of the disk is the reason
7088 	 * it's being detached!
7089 	 */
7090 	error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
7091 
7092 	/*
7093 	 * Remove vd from its parent and compact the parent's children.
7094 	 */
7095 	vdev_remove_child(pvd, vd);
7096 	vdev_compact_children(pvd);
7097 
7098 	/*
7099 	 * Remember one of the remaining children so we can get tvd below.
7100 	 */
7101 	cvd = pvd->vdev_child[pvd->vdev_children - 1];
7102 
7103 	/*
7104 	 * If we need to remove the remaining child from the list of hot spares,
7105 	 * do it now, marking the vdev as no longer a spare in the process.
7106 	 * We must do this before vdev_remove_parent(), because that can
7107 	 * change the GUID if it creates a new toplevel GUID.  For a similar
7108 	 * reason, we must remove the spare now, in the same txg as the detach;
7109 	 * otherwise someone could attach a new sibling, change the GUID, and
7110 	 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail.
7111 	 */
7112 	if (unspare) {
7113 		ASSERT(cvd->vdev_isspare);
7114 		spa_spare_remove(cvd);
7115 		unspare_guid = cvd->vdev_guid;
7116 		(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
7117 		cvd->vdev_unspare = B_TRUE;
7118 	}
7119 
7120 	/*
7121 	 * If the parent mirror/replacing vdev only has one child,
7122 	 * the parent is no longer needed.  Remove it from the tree.
7123 	 */
7124 	if (pvd->vdev_children == 1) {
7125 		if (pvd->vdev_ops == &vdev_spare_ops)
7126 			cvd->vdev_unspare = B_FALSE;
7127 		vdev_remove_parent(cvd);
7128 	}
7129 
7130 	/*
7131 	 * We don't set tvd until now because the parent we just removed
7132 	 * may have been the previous top-level vdev.
7133 	 */
7134 	tvd = cvd->vdev_top;
7135 	ASSERT(tvd->vdev_parent == rvd);
7136 
7137 	/*
7138 	 * Reevaluate the parent vdev state.
7139 	 */
7140 	vdev_propagate_state(cvd);
7141 
7142 	/*
7143 	 * If the 'autoexpand' property is set on the pool then automatically
7144 	 * try to expand the size of the pool. For example if the device we
7145 	 * just detached was smaller than the others, it may be possible to
7146 	 * add metaslabs (i.e. grow the pool). We need to reopen the vdev
7147 	 * first so that we can obtain the updated sizes of the leaf vdevs.
7148 	 */
7149 	if (spa->spa_autoexpand) {
7150 		vdev_reopen(tvd);
7151 		vdev_expand(tvd, txg);
7152 	}
7153 
7154 	vdev_config_dirty(tvd);
7155 
7156 	/*
7157 	 * Mark vd's DTL as dirty in this txg.  vdev_dtl_sync() will see that
7158 	 * vd->vdev_detached is set and free vd's DTL object in syncing context.
7159 	 * But first make sure we're not on any *other* txg's DTL list, to
7160 	 * prevent vd from being accessed after it's freed.
7161 	 */
7162 	vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none");
7163 	for (int t = 0; t < TXG_SIZE; t++)
7164 		(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
7165 	vd->vdev_detached = B_TRUE;
7166 	vdev_dirty(tvd, VDD_DTL, vd, txg);
7167 
7168 	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE);
7169 	spa_notify_waiters(spa);
7170 
7171 	/* hang on to the spa before we release the lock */
7172 	spa_open_ref(spa, FTAG);
7173 
7174 	error = spa_vdev_exit(spa, vd, txg, 0);
7175 
7176 	spa_history_log_internal(spa, "detach", NULL,
7177 	    "vdev=%s", vdpath);
7178 	spa_strfree(vdpath);
7179 
7180 	/*
7181 	 * If this was the removal of the original device in a hot spare vdev,
7182 	 * then we want to go through and remove the device from the hot spare
7183 	 * list of every other pool.
7184 	 */
7185 	if (unspare) {
7186 		spa_t *altspa = NULL;
7187 
7188 		mutex_enter(&spa_namespace_lock);
7189 		while ((altspa = spa_next(altspa)) != NULL) {
7190 			if (altspa->spa_state != POOL_STATE_ACTIVE ||
7191 			    altspa == spa)
7192 				continue;
7193 
7194 			spa_open_ref(altspa, FTAG);
7195 			mutex_exit(&spa_namespace_lock);
7196 			(void) spa_vdev_remove(altspa, unspare_guid, B_TRUE);
7197 			mutex_enter(&spa_namespace_lock);
7198 			spa_close(altspa, FTAG);
7199 		}
7200 		mutex_exit(&spa_namespace_lock);
7201 
7202 		/* search the rest of the vdevs for spares to remove */
7203 		spa_vdev_resilver_done(spa);
7204 	}
7205 
7206 	/* all done with the spa; OK to release */
7207 	mutex_enter(&spa_namespace_lock);
7208 	spa_close(spa, FTAG);
7209 	mutex_exit(&spa_namespace_lock);
7210 
7211 	return (error);
7212 }
7213 
7214 static int
7215 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
7216     list_t *vd_list)
7217 {
7218 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7219 
7220 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
7221 
7222 	/* Look up vdev and ensure it's a leaf. */
7223 	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7224 	if (vd == NULL || vd->vdev_detached) {
7225 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7226 		return (SET_ERROR(ENODEV));
7227 	} else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
7228 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7229 		return (SET_ERROR(EINVAL));
7230 	} else if (!vdev_writeable(vd)) {
7231 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7232 		return (SET_ERROR(EROFS));
7233 	}
7234 	mutex_enter(&vd->vdev_initialize_lock);
7235 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7236 
7237 	/*
7238 	 * When we activate an initialize action we check to see
7239 	 * if the vdev_initialize_thread is NULL. We do this instead
7240 	 * of using the vdev_initialize_state since there might be
7241 	 * a previous initialization process which has completed but
7242 	 * the thread is not exited.
7243 	 */
7244 	if (cmd_type == POOL_INITIALIZE_START &&
7245 	    (vd->vdev_initialize_thread != NULL ||
7246 	    vd->vdev_top->vdev_removing)) {
7247 		mutex_exit(&vd->vdev_initialize_lock);
7248 		return (SET_ERROR(EBUSY));
7249 	} else if (cmd_type == POOL_INITIALIZE_CANCEL &&
7250 	    (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE &&
7251 	    vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) {
7252 		mutex_exit(&vd->vdev_initialize_lock);
7253 		return (SET_ERROR(ESRCH));
7254 	} else if (cmd_type == POOL_INITIALIZE_SUSPEND &&
7255 	    vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) {
7256 		mutex_exit(&vd->vdev_initialize_lock);
7257 		return (SET_ERROR(ESRCH));
7258 	}
7259 
7260 	switch (cmd_type) {
7261 	case POOL_INITIALIZE_START:
7262 		vdev_initialize(vd);
7263 		break;
7264 	case POOL_INITIALIZE_CANCEL:
7265 		vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list);
7266 		break;
7267 	case POOL_INITIALIZE_SUSPEND:
7268 		vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list);
7269 		break;
7270 	default:
7271 		panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
7272 	}
7273 	mutex_exit(&vd->vdev_initialize_lock);
7274 
7275 	return (0);
7276 }
7277 
7278 int
7279 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type,
7280     nvlist_t *vdev_errlist)
7281 {
7282 	int total_errors = 0;
7283 	list_t vd_list;
7284 
7285 	list_create(&vd_list, sizeof (vdev_t),
7286 	    offsetof(vdev_t, vdev_initialize_node));
7287 
7288 	/*
7289 	 * We hold the namespace lock through the whole function
7290 	 * to prevent any changes to the pool while we're starting or
7291 	 * stopping initialization. The config and state locks are held so that
7292 	 * we can properly assess the vdev state before we commit to
7293 	 * the initializing operation.
7294 	 */
7295 	mutex_enter(&spa_namespace_lock);
7296 
7297 	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
7298 	    pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
7299 		uint64_t vdev_guid = fnvpair_value_uint64(pair);
7300 
7301 		int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type,
7302 		    &vd_list);
7303 		if (error != 0) {
7304 			char guid_as_str[MAXNAMELEN];
7305 
7306 			(void) snprintf(guid_as_str, sizeof (guid_as_str),
7307 			    "%llu", (unsigned long long)vdev_guid);
7308 			fnvlist_add_int64(vdev_errlist, guid_as_str, error);
7309 			total_errors++;
7310 		}
7311 	}
7312 
7313 	/* Wait for all initialize threads to stop. */
7314 	vdev_initialize_stop_wait(spa, &vd_list);
7315 
7316 	/* Sync out the initializing state */
7317 	txg_wait_synced(spa->spa_dsl_pool, 0);
7318 	mutex_exit(&spa_namespace_lock);
7319 
7320 	list_destroy(&vd_list);
7321 
7322 	return (total_errors);
7323 }
7324 
7325 static int
7326 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
7327     uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list)
7328 {
7329 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7330 
7331 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
7332 
7333 	/* Look up vdev and ensure it's a leaf. */
7334 	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7335 	if (vd == NULL || vd->vdev_detached) {
7336 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7337 		return (SET_ERROR(ENODEV));
7338 	} else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
7339 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7340 		return (SET_ERROR(EINVAL));
7341 	} else if (!vdev_writeable(vd)) {
7342 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7343 		return (SET_ERROR(EROFS));
7344 	} else if (!vd->vdev_has_trim) {
7345 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7346 		return (SET_ERROR(EOPNOTSUPP));
7347 	} else if (secure && !vd->vdev_has_securetrim) {
7348 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7349 		return (SET_ERROR(EOPNOTSUPP));
7350 	}
7351 	mutex_enter(&vd->vdev_trim_lock);
7352 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7353 
7354 	/*
7355 	 * When we activate a TRIM action we check to see if the
7356 	 * vdev_trim_thread is NULL. We do this instead of using the
7357 	 * vdev_trim_state since there might be a previous TRIM process
7358 	 * which has completed but the thread is not exited.
7359 	 */
7360 	if (cmd_type == POOL_TRIM_START &&
7361 	    (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing)) {
7362 		mutex_exit(&vd->vdev_trim_lock);
7363 		return (SET_ERROR(EBUSY));
7364 	} else if (cmd_type == POOL_TRIM_CANCEL &&
7365 	    (vd->vdev_trim_state != VDEV_TRIM_ACTIVE &&
7366 	    vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) {
7367 		mutex_exit(&vd->vdev_trim_lock);
7368 		return (SET_ERROR(ESRCH));
7369 	} else if (cmd_type == POOL_TRIM_SUSPEND &&
7370 	    vd->vdev_trim_state != VDEV_TRIM_ACTIVE) {
7371 		mutex_exit(&vd->vdev_trim_lock);
7372 		return (SET_ERROR(ESRCH));
7373 	}
7374 
7375 	switch (cmd_type) {
7376 	case POOL_TRIM_START:
7377 		vdev_trim(vd, rate, partial, secure);
7378 		break;
7379 	case POOL_TRIM_CANCEL:
7380 		vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list);
7381 		break;
7382 	case POOL_TRIM_SUSPEND:
7383 		vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list);
7384 		break;
7385 	default:
7386 		panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
7387 	}
7388 	mutex_exit(&vd->vdev_trim_lock);
7389 
7390 	return (0);
7391 }
7392 
7393 /*
7394  * Initiates a manual TRIM for the requested vdevs. This kicks off individual
7395  * TRIM threads for each child vdev.  These threads pass over all of the free
7396  * space in the vdev's metaslabs and issues TRIM commands for that space.
7397  */
7398 int
7399 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate,
7400     boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist)
7401 {
7402 	int total_errors = 0;
7403 	list_t vd_list;
7404 
7405 	list_create(&vd_list, sizeof (vdev_t),
7406 	    offsetof(vdev_t, vdev_trim_node));
7407 
7408 	/*
7409 	 * We hold the namespace lock through the whole function
7410 	 * to prevent any changes to the pool while we're starting or
7411 	 * stopping TRIM. The config and state locks are held so that
7412 	 * we can properly assess the vdev state before we commit to
7413 	 * the TRIM operation.
7414 	 */
7415 	mutex_enter(&spa_namespace_lock);
7416 
7417 	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
7418 	    pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
7419 		uint64_t vdev_guid = fnvpair_value_uint64(pair);
7420 
7421 		int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type,
7422 		    rate, partial, secure, &vd_list);
7423 		if (error != 0) {
7424 			char guid_as_str[MAXNAMELEN];
7425 
7426 			(void) snprintf(guid_as_str, sizeof (guid_as_str),
7427 			    "%llu", (unsigned long long)vdev_guid);
7428 			fnvlist_add_int64(vdev_errlist, guid_as_str, error);
7429 			total_errors++;
7430 		}
7431 	}
7432 
7433 	/* Wait for all TRIM threads to stop. */
7434 	vdev_trim_stop_wait(spa, &vd_list);
7435 
7436 	/* Sync out the TRIM state */
7437 	txg_wait_synced(spa->spa_dsl_pool, 0);
7438 	mutex_exit(&spa_namespace_lock);
7439 
7440 	list_destroy(&vd_list);
7441 
7442 	return (total_errors);
7443 }
7444 
7445 /*
7446  * Split a set of devices from their mirrors, and create a new pool from them.
7447  */
7448 int
7449 spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config,
7450     nvlist_t *props, boolean_t exp)
7451 {
7452 	int error = 0;
7453 	uint64_t txg, *glist;
7454 	spa_t *newspa;
7455 	uint_t c, children, lastlog;
7456 	nvlist_t **child, *nvl, *tmp;
7457 	dmu_tx_t *tx;
7458 	char *altroot = NULL;
7459 	vdev_t *rvd, **vml = NULL;			/* vdev modify list */
7460 	boolean_t activate_slog;
7461 
7462 	ASSERT(spa_writeable(spa));
7463 
7464 	txg = spa_vdev_enter(spa);
7465 
7466 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7467 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
7468 		error = (spa_has_checkpoint(spa)) ?
7469 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
7470 		return (spa_vdev_exit(spa, NULL, txg, error));
7471 	}
7472 
7473 	/* clear the log and flush everything up to now */
7474 	activate_slog = spa_passivate_log(spa);
7475 	(void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
7476 	error = spa_reset_logs(spa);
7477 	txg = spa_vdev_config_enter(spa);
7478 
7479 	if (activate_slog)
7480 		spa_activate_log(spa);
7481 
7482 	if (error != 0)
7483 		return (spa_vdev_exit(spa, NULL, txg, error));
7484 
7485 	/* check new spa name before going any further */
7486 	if (spa_lookup(newname) != NULL)
7487 		return (spa_vdev_exit(spa, NULL, txg, EEXIST));
7488 
7489 	/*
7490 	 * scan through all the children to ensure they're all mirrors
7491 	 */
7492 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 ||
7493 	    nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
7494 	    &children) != 0)
7495 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7496 
7497 	/* first, check to ensure we've got the right child count */
7498 	rvd = spa->spa_root_vdev;
7499 	lastlog = 0;
7500 	for (c = 0; c < rvd->vdev_children; c++) {
7501 		vdev_t *vd = rvd->vdev_child[c];
7502 
7503 		/* don't count the holes & logs as children */
7504 		if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops &&
7505 		    !vdev_is_concrete(vd))) {
7506 			if (lastlog == 0)
7507 				lastlog = c;
7508 			continue;
7509 		}
7510 
7511 		lastlog = 0;
7512 	}
7513 	if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
7514 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7515 
7516 	/* next, ensure no spare or cache devices are part of the split */
7517 	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 ||
7518 	    nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0)
7519 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7520 
7521 	vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP);
7522 	glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP);
7523 
7524 	/* then, loop over each vdev and validate it */
7525 	for (c = 0; c < children; c++) {
7526 		uint64_t is_hole = 0;
7527 
7528 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
7529 		    &is_hole);
7530 
7531 		if (is_hole != 0) {
7532 			if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole ||
7533 			    spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
7534 				continue;
7535 			} else {
7536 				error = SET_ERROR(EINVAL);
7537 				break;
7538 			}
7539 		}
7540 
7541 		/* deal with indirect vdevs */
7542 		if (spa->spa_root_vdev->vdev_child[c]->vdev_ops ==
7543 		    &vdev_indirect_ops)
7544 			continue;
7545 
7546 		/* which disk is going to be split? */
7547 		if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
7548 		    &glist[c]) != 0) {
7549 			error = SET_ERROR(EINVAL);
7550 			break;
7551 		}
7552 
7553 		/* look it up in the spa */
7554 		vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE);
7555 		if (vml[c] == NULL) {
7556 			error = SET_ERROR(ENODEV);
7557 			break;
7558 		}
7559 
7560 		/* make sure there's nothing stopping the split */
7561 		if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops ||
7562 		    vml[c]->vdev_islog ||
7563 		    !vdev_is_concrete(vml[c]) ||
7564 		    vml[c]->vdev_isspare ||
7565 		    vml[c]->vdev_isl2cache ||
7566 		    !vdev_writeable(vml[c]) ||
7567 		    vml[c]->vdev_children != 0 ||
7568 		    vml[c]->vdev_state != VDEV_STATE_HEALTHY ||
7569 		    c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
7570 			error = SET_ERROR(EINVAL);
7571 			break;
7572 		}
7573 
7574 		if (vdev_dtl_required(vml[c]) ||
7575 		    vdev_resilver_needed(vml[c], NULL, NULL)) {
7576 			error = SET_ERROR(EBUSY);
7577 			break;
7578 		}
7579 
7580 		/* we need certain info from the top level */
7581 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY,
7582 		    vml[c]->vdev_top->vdev_ms_array) == 0);
7583 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT,
7584 		    vml[c]->vdev_top->vdev_ms_shift) == 0);
7585 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE,
7586 		    vml[c]->vdev_top->vdev_asize) == 0);
7587 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT,
7588 		    vml[c]->vdev_top->vdev_ashift) == 0);
7589 
7590 		/* transfer per-vdev ZAPs */
7591 		ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0);
7592 		VERIFY0(nvlist_add_uint64(child[c],
7593 		    ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap));
7594 
7595 		ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0);
7596 		VERIFY0(nvlist_add_uint64(child[c],
7597 		    ZPOOL_CONFIG_VDEV_TOP_ZAP,
7598 		    vml[c]->vdev_parent->vdev_top_zap));
7599 	}
7600 
7601 	if (error != 0) {
7602 		kmem_free(vml, children * sizeof (vdev_t *));
7603 		kmem_free(glist, children * sizeof (uint64_t));
7604 		return (spa_vdev_exit(spa, NULL, txg, error));
7605 	}
7606 
7607 	/* stop writers from using the disks */
7608 	for (c = 0; c < children; c++) {
7609 		if (vml[c] != NULL)
7610 			vml[c]->vdev_offline = B_TRUE;
7611 	}
7612 	vdev_reopen(spa->spa_root_vdev);
7613 
7614 	/*
7615 	 * Temporarily record the splitting vdevs in the spa config.  This
7616 	 * will disappear once the config is regenerated.
7617 	 */
7618 	VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0);
7619 	VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
7620 	    glist, children) == 0);
7621 	kmem_free(glist, children * sizeof (uint64_t));
7622 
7623 	mutex_enter(&spa->spa_props_lock);
7624 	VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT,
7625 	    nvl) == 0);
7626 	mutex_exit(&spa->spa_props_lock);
7627 	spa->spa_config_splitting = nvl;
7628 	vdev_config_dirty(spa->spa_root_vdev);
7629 
7630 	/* configure and create the new pool */
7631 	VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0);
7632 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
7633 	    exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0);
7634 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
7635 	    spa_version(spa)) == 0);
7636 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG,
7637 	    spa->spa_config_txg) == 0);
7638 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID,
7639 	    spa_generate_guid(NULL)) == 0);
7640 	VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
7641 	(void) nvlist_lookup_string(props,
7642 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
7643 
7644 	/* add the new pool to the namespace */
7645 	newspa = spa_add(newname, config, altroot);
7646 	newspa->spa_avz_action = AVZ_ACTION_REBUILD;
7647 	newspa->spa_config_txg = spa->spa_config_txg;
7648 	spa_set_log_state(newspa, SPA_LOG_CLEAR);
7649 
7650 	/* release the spa config lock, retaining the namespace lock */
7651 	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
7652 
7653 	if (zio_injection_enabled)
7654 		zio_handle_panic_injection(spa, FTAG, 1);
7655 
7656 	spa_activate(newspa, spa_mode_global);
7657 	spa_async_suspend(newspa);
7658 
7659 	/*
7660 	 * Temporarily stop the initializing and TRIM activity.  We set the
7661 	 * state to ACTIVE so that we know to resume initializing or TRIM
7662 	 * once the split has completed.
7663 	 */
7664 	list_t vd_initialize_list;
7665 	list_create(&vd_initialize_list, sizeof (vdev_t),
7666 	    offsetof(vdev_t, vdev_initialize_node));
7667 
7668 	list_t vd_trim_list;
7669 	list_create(&vd_trim_list, sizeof (vdev_t),
7670 	    offsetof(vdev_t, vdev_trim_node));
7671 
7672 	for (c = 0; c < children; c++) {
7673 		if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
7674 			mutex_enter(&vml[c]->vdev_initialize_lock);
7675 			vdev_initialize_stop(vml[c],
7676 			    VDEV_INITIALIZE_ACTIVE, &vd_initialize_list);
7677 			mutex_exit(&vml[c]->vdev_initialize_lock);
7678 
7679 			mutex_enter(&vml[c]->vdev_trim_lock);
7680 			vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list);
7681 			mutex_exit(&vml[c]->vdev_trim_lock);
7682 		}
7683 	}
7684 
7685 	vdev_initialize_stop_wait(spa, &vd_initialize_list);
7686 	vdev_trim_stop_wait(spa, &vd_trim_list);
7687 
7688 	list_destroy(&vd_initialize_list);
7689 	list_destroy(&vd_trim_list);
7690 
7691 	newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT;
7692 	newspa->spa_is_splitting = B_TRUE;
7693 
7694 	/* create the new pool from the disks of the original pool */
7695 	error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE);
7696 	if (error)
7697 		goto out;
7698 
7699 	/* if that worked, generate a real config for the new pool */
7700 	if (newspa->spa_root_vdev != NULL) {
7701 		VERIFY(nvlist_alloc(&newspa->spa_config_splitting,
7702 		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
7703 		VERIFY(nvlist_add_uint64(newspa->spa_config_splitting,
7704 		    ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0);
7705 		spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
7706 		    B_TRUE));
7707 	}
7708 
7709 	/* set the props */
7710 	if (props != NULL) {
7711 		spa_configfile_set(newspa, props, B_FALSE);
7712 		error = spa_prop_set(newspa, props);
7713 		if (error)
7714 			goto out;
7715 	}
7716 
7717 	/* flush everything */
7718 	txg = spa_vdev_config_enter(newspa);
7719 	vdev_config_dirty(newspa->spa_root_vdev);
7720 	(void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG);
7721 
7722 	if (zio_injection_enabled)
7723 		zio_handle_panic_injection(spa, FTAG, 2);
7724 
7725 	spa_async_resume(newspa);
7726 
7727 	/* finally, update the original pool's config */
7728 	txg = spa_vdev_config_enter(spa);
7729 	tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
7730 	error = dmu_tx_assign(tx, TXG_WAIT);
7731 	if (error != 0)
7732 		dmu_tx_abort(tx);
7733 	for (c = 0; c < children; c++) {
7734 		if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
7735 			vdev_t *tvd = vml[c]->vdev_top;
7736 
7737 			/*
7738 			 * Need to be sure the detachable VDEV is not
7739 			 * on any *other* txg's DTL list to prevent it
7740 			 * from being accessed after it's freed.
7741 			 */
7742 			for (int t = 0; t < TXG_SIZE; t++) {
7743 				(void) txg_list_remove_this(
7744 				    &tvd->vdev_dtl_list, vml[c], t);
7745 			}
7746 
7747 			vdev_split(vml[c]);
7748 			if (error == 0)
7749 				spa_history_log_internal(spa, "detach", tx,
7750 				    "vdev=%s", vml[c]->vdev_path);
7751 
7752 			vdev_free(vml[c]);
7753 		}
7754 	}
7755 	spa->spa_avz_action = AVZ_ACTION_REBUILD;
7756 	vdev_config_dirty(spa->spa_root_vdev);
7757 	spa->spa_config_splitting = NULL;
7758 	nvlist_free(nvl);
7759 	if (error == 0)
7760 		dmu_tx_commit(tx);
7761 	(void) spa_vdev_exit(spa, NULL, txg, 0);
7762 
7763 	if (zio_injection_enabled)
7764 		zio_handle_panic_injection(spa, FTAG, 3);
7765 
7766 	/* split is complete; log a history record */
7767 	spa_history_log_internal(newspa, "split", NULL,
7768 	    "from pool %s", spa_name(spa));
7769 
7770 	newspa->spa_is_splitting = B_FALSE;
7771 	kmem_free(vml, children * sizeof (vdev_t *));
7772 
7773 	/* if we're not going to mount the filesystems in userland, export */
7774 	if (exp)
7775 		error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL,
7776 		    B_FALSE, B_FALSE);
7777 
7778 	return (error);
7779 
7780 out:
7781 	spa_unload(newspa);
7782 	spa_deactivate(newspa);
7783 	spa_remove(newspa);
7784 
7785 	txg = spa_vdev_config_enter(spa);
7786 
7787 	/* re-online all offlined disks */
7788 	for (c = 0; c < children; c++) {
7789 		if (vml[c] != NULL)
7790 			vml[c]->vdev_offline = B_FALSE;
7791 	}
7792 
7793 	/* restart initializing or trimming disks as necessary */
7794 	spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART);
7795 	spa_async_request(spa, SPA_ASYNC_TRIM_RESTART);
7796 	spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART);
7797 
7798 	vdev_reopen(spa->spa_root_vdev);
7799 
7800 	nvlist_free(spa->spa_config_splitting);
7801 	spa->spa_config_splitting = NULL;
7802 	(void) spa_vdev_exit(spa, NULL, txg, error);
7803 
7804 	kmem_free(vml, children * sizeof (vdev_t *));
7805 	return (error);
7806 }
7807 
7808 /*
7809  * Find any device that's done replacing, or a vdev marked 'unspare' that's
7810  * currently spared, so we can detach it.
7811  */
7812 static vdev_t *
7813 spa_vdev_resilver_done_hunt(vdev_t *vd)
7814 {
7815 	vdev_t *newvd, *oldvd;
7816 
7817 	for (int c = 0; c < vd->vdev_children; c++) {
7818 		oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
7819 		if (oldvd != NULL)
7820 			return (oldvd);
7821 	}
7822 
7823 	/*
7824 	 * Check for a completed replacement.  We always consider the first
7825 	 * vdev in the list to be the oldest vdev, and the last one to be
7826 	 * the newest (see spa_vdev_attach() for how that works).  In
7827 	 * the case where the newest vdev is faulted, we will not automatically
7828 	 * remove it after a resilver completes.  This is OK as it will require
7829 	 * user intervention to determine which disk the admin wishes to keep.
7830 	 */
7831 	if (vd->vdev_ops == &vdev_replacing_ops) {
7832 		ASSERT(vd->vdev_children > 1);
7833 
7834 		newvd = vd->vdev_child[vd->vdev_children - 1];
7835 		oldvd = vd->vdev_child[0];
7836 
7837 		if (vdev_dtl_empty(newvd, DTL_MISSING) &&
7838 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
7839 		    !vdev_dtl_required(oldvd))
7840 			return (oldvd);
7841 	}
7842 
7843 	/*
7844 	 * Check for a completed resilver with the 'unspare' flag set.
7845 	 * Also potentially update faulted state.
7846 	 */
7847 	if (vd->vdev_ops == &vdev_spare_ops) {
7848 		vdev_t *first = vd->vdev_child[0];
7849 		vdev_t *last = vd->vdev_child[vd->vdev_children - 1];
7850 
7851 		if (last->vdev_unspare) {
7852 			oldvd = first;
7853 			newvd = last;
7854 		} else if (first->vdev_unspare) {
7855 			oldvd = last;
7856 			newvd = first;
7857 		} else {
7858 			oldvd = NULL;
7859 		}
7860 
7861 		if (oldvd != NULL &&
7862 		    vdev_dtl_empty(newvd, DTL_MISSING) &&
7863 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
7864 		    !vdev_dtl_required(oldvd))
7865 			return (oldvd);
7866 
7867 		vdev_propagate_state(vd);
7868 
7869 		/*
7870 		 * If there are more than two spares attached to a disk,
7871 		 * and those spares are not required, then we want to
7872 		 * attempt to free them up now so that they can be used
7873 		 * by other pools.  Once we're back down to a single
7874 		 * disk+spare, we stop removing them.
7875 		 */
7876 		if (vd->vdev_children > 2) {
7877 			newvd = vd->vdev_child[1];
7878 
7879 			if (newvd->vdev_isspare && last->vdev_isspare &&
7880 			    vdev_dtl_empty(last, DTL_MISSING) &&
7881 			    vdev_dtl_empty(last, DTL_OUTAGE) &&
7882 			    !vdev_dtl_required(newvd))
7883 				return (newvd);
7884 		}
7885 	}
7886 
7887 	return (NULL);
7888 }
7889 
7890 static void
7891 spa_vdev_resilver_done(spa_t *spa)
7892 {
7893 	vdev_t *vd, *pvd, *ppvd;
7894 	uint64_t guid, sguid, pguid, ppguid;
7895 
7896 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
7897 
7898 	while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
7899 		pvd = vd->vdev_parent;
7900 		ppvd = pvd->vdev_parent;
7901 		guid = vd->vdev_guid;
7902 		pguid = pvd->vdev_guid;
7903 		ppguid = ppvd->vdev_guid;
7904 		sguid = 0;
7905 		/*
7906 		 * If we have just finished replacing a hot spared device, then
7907 		 * we need to detach the parent's first child (the original hot
7908 		 * spare) as well.
7909 		 */
7910 		if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
7911 		    ppvd->vdev_children == 2) {
7912 			ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
7913 			sguid = ppvd->vdev_child[1]->vdev_guid;
7914 		}
7915 		ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd));
7916 
7917 		spa_config_exit(spa, SCL_ALL, FTAG);
7918 		if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0)
7919 			return;
7920 		if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0)
7921 			return;
7922 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
7923 	}
7924 
7925 	spa_config_exit(spa, SCL_ALL, FTAG);
7926 
7927 	/*
7928 	 * If a detach was not performed above replace waiters will not have
7929 	 * been notified.  In which case we must do so now.
7930 	 */
7931 	spa_notify_waiters(spa);
7932 }
7933 
7934 /*
7935  * Update the stored path or FRU for this vdev.
7936  */
7937 static int
7938 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value,
7939     boolean_t ispath)
7940 {
7941 	vdev_t *vd;
7942 	boolean_t sync = B_FALSE;
7943 
7944 	ASSERT(spa_writeable(spa));
7945 
7946 	spa_vdev_state_enter(spa, SCL_ALL);
7947 
7948 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
7949 		return (spa_vdev_state_exit(spa, NULL, ENOENT));
7950 
7951 	if (!vd->vdev_ops->vdev_op_leaf)
7952 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
7953 
7954 	if (ispath) {
7955 		if (strcmp(value, vd->vdev_path) != 0) {
7956 			spa_strfree(vd->vdev_path);
7957 			vd->vdev_path = spa_strdup(value);
7958 			sync = B_TRUE;
7959 		}
7960 	} else {
7961 		if (vd->vdev_fru == NULL) {
7962 			vd->vdev_fru = spa_strdup(value);
7963 			sync = B_TRUE;
7964 		} else if (strcmp(value, vd->vdev_fru) != 0) {
7965 			spa_strfree(vd->vdev_fru);
7966 			vd->vdev_fru = spa_strdup(value);
7967 			sync = B_TRUE;
7968 		}
7969 	}
7970 
7971 	return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0));
7972 }
7973 
7974 int
7975 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath)
7976 {
7977 	return (spa_vdev_set_common(spa, guid, newpath, B_TRUE));
7978 }
7979 
7980 int
7981 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru)
7982 {
7983 	return (spa_vdev_set_common(spa, guid, newfru, B_FALSE));
7984 }
7985 
7986 /*
7987  * ==========================================================================
7988  * SPA Scanning
7989  * ==========================================================================
7990  */
7991 int
7992 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd)
7993 {
7994 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
7995 
7996 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
7997 		return (SET_ERROR(EBUSY));
7998 
7999 	return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd));
8000 }
8001 
8002 int
8003 spa_scan_stop(spa_t *spa)
8004 {
8005 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8006 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
8007 		return (SET_ERROR(EBUSY));
8008 	return (dsl_scan_cancel(spa->spa_dsl_pool));
8009 }
8010 
8011 int
8012 spa_scan(spa_t *spa, pool_scan_func_t func)
8013 {
8014 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8015 
8016 	if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE)
8017 		return (SET_ERROR(ENOTSUP));
8018 
8019 	if (func == POOL_SCAN_RESILVER &&
8020 	    !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER))
8021 		return (SET_ERROR(ENOTSUP));
8022 
8023 	/*
8024 	 * If a resilver was requested, but there is no DTL on a
8025 	 * writeable leaf device, we have nothing to do.
8026 	 */
8027 	if (func == POOL_SCAN_RESILVER &&
8028 	    !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
8029 		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
8030 		return (0);
8031 	}
8032 
8033 	return (dsl_scan(spa->spa_dsl_pool, func));
8034 }
8035 
8036 /*
8037  * ==========================================================================
8038  * SPA async task processing
8039  * ==========================================================================
8040  */
8041 
8042 static void
8043 spa_async_remove(spa_t *spa, vdev_t *vd)
8044 {
8045 	if (vd->vdev_remove_wanted) {
8046 		vd->vdev_remove_wanted = B_FALSE;
8047 		vd->vdev_delayed_close = B_FALSE;
8048 		vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE);
8049 
8050 		/*
8051 		 * We want to clear the stats, but we don't want to do a full
8052 		 * vdev_clear() as that will cause us to throw away
8053 		 * degraded/faulted state as well as attempt to reopen the
8054 		 * device, all of which is a waste.
8055 		 */
8056 		vd->vdev_stat.vs_read_errors = 0;
8057 		vd->vdev_stat.vs_write_errors = 0;
8058 		vd->vdev_stat.vs_checksum_errors = 0;
8059 
8060 		vdev_state_dirty(vd->vdev_top);
8061 
8062 		/* Tell userspace that the vdev is gone. */
8063 		zfs_post_remove(spa, vd);
8064 	}
8065 
8066 	for (int c = 0; c < vd->vdev_children; c++)
8067 		spa_async_remove(spa, vd->vdev_child[c]);
8068 }
8069 
8070 static void
8071 spa_async_probe(spa_t *spa, vdev_t *vd)
8072 {
8073 	if (vd->vdev_probe_wanted) {
8074 		vd->vdev_probe_wanted = B_FALSE;
8075 		vdev_reopen(vd);	/* vdev_open() does the actual probe */
8076 	}
8077 
8078 	for (int c = 0; c < vd->vdev_children; c++)
8079 		spa_async_probe(spa, vd->vdev_child[c]);
8080 }
8081 
8082 static void
8083 spa_async_autoexpand(spa_t *spa, vdev_t *vd)
8084 {
8085 	if (!spa->spa_autoexpand)
8086 		return;
8087 
8088 	for (int c = 0; c < vd->vdev_children; c++) {
8089 		vdev_t *cvd = vd->vdev_child[c];
8090 		spa_async_autoexpand(spa, cvd);
8091 	}
8092 
8093 	if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL)
8094 		return;
8095 
8096 	spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND);
8097 }
8098 
8099 static void
8100 spa_async_thread(void *arg)
8101 {
8102 	spa_t *spa = (spa_t *)arg;
8103 	dsl_pool_t *dp = spa->spa_dsl_pool;
8104 	int tasks;
8105 
8106 	ASSERT(spa->spa_sync_on);
8107 
8108 	mutex_enter(&spa->spa_async_lock);
8109 	tasks = spa->spa_async_tasks;
8110 	spa->spa_async_tasks = 0;
8111 	mutex_exit(&spa->spa_async_lock);
8112 
8113 	/*
8114 	 * See if the config needs to be updated.
8115 	 */
8116 	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
8117 		uint64_t old_space, new_space;
8118 
8119 		mutex_enter(&spa_namespace_lock);
8120 		old_space = metaslab_class_get_space(spa_normal_class(spa));
8121 		old_space += metaslab_class_get_space(spa_special_class(spa));
8122 		old_space += metaslab_class_get_space(spa_dedup_class(spa));
8123 		old_space += metaslab_class_get_space(
8124 		    spa_embedded_log_class(spa));
8125 
8126 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
8127 
8128 		new_space = metaslab_class_get_space(spa_normal_class(spa));
8129 		new_space += metaslab_class_get_space(spa_special_class(spa));
8130 		new_space += metaslab_class_get_space(spa_dedup_class(spa));
8131 		new_space += metaslab_class_get_space(
8132 		    spa_embedded_log_class(spa));
8133 		mutex_exit(&spa_namespace_lock);
8134 
8135 		/*
8136 		 * If the pool grew as a result of the config update,
8137 		 * then log an internal history event.
8138 		 */
8139 		if (new_space != old_space) {
8140 			spa_history_log_internal(spa, "vdev online", NULL,
8141 			    "pool '%s' size: %llu(+%llu)",
8142 			    spa_name(spa), (u_longlong_t)new_space,
8143 			    (u_longlong_t)(new_space - old_space));
8144 		}
8145 	}
8146 
8147 	/*
8148 	 * See if any devices need to be marked REMOVED.
8149 	 */
8150 	if (tasks & SPA_ASYNC_REMOVE) {
8151 		spa_vdev_state_enter(spa, SCL_NONE);
8152 		spa_async_remove(spa, spa->spa_root_vdev);
8153 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
8154 			spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]);
8155 		for (int i = 0; i < spa->spa_spares.sav_count; i++)
8156 			spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]);
8157 		(void) spa_vdev_state_exit(spa, NULL, 0);
8158 	}
8159 
8160 	if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) {
8161 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8162 		spa_async_autoexpand(spa, spa->spa_root_vdev);
8163 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8164 	}
8165 
8166 	/*
8167 	 * See if any devices need to be probed.
8168 	 */
8169 	if (tasks & SPA_ASYNC_PROBE) {
8170 		spa_vdev_state_enter(spa, SCL_NONE);
8171 		spa_async_probe(spa, spa->spa_root_vdev);
8172 		(void) spa_vdev_state_exit(spa, NULL, 0);
8173 	}
8174 
8175 	/*
8176 	 * If any devices are done replacing, detach them.
8177 	 */
8178 	if (tasks & SPA_ASYNC_RESILVER_DONE ||
8179 	    tasks & SPA_ASYNC_REBUILD_DONE) {
8180 		spa_vdev_resilver_done(spa);
8181 	}
8182 
8183 	/*
8184 	 * Kick off a resilver.
8185 	 */
8186 	if (tasks & SPA_ASYNC_RESILVER &&
8187 	    !vdev_rebuild_active(spa->spa_root_vdev) &&
8188 	    (!dsl_scan_resilvering(dp) ||
8189 	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)))
8190 		dsl_scan_restart_resilver(dp, 0);
8191 
8192 	if (tasks & SPA_ASYNC_INITIALIZE_RESTART) {
8193 		mutex_enter(&spa_namespace_lock);
8194 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8195 		vdev_initialize_restart(spa->spa_root_vdev);
8196 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8197 		mutex_exit(&spa_namespace_lock);
8198 	}
8199 
8200 	if (tasks & SPA_ASYNC_TRIM_RESTART) {
8201 		mutex_enter(&spa_namespace_lock);
8202 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8203 		vdev_trim_restart(spa->spa_root_vdev);
8204 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8205 		mutex_exit(&spa_namespace_lock);
8206 	}
8207 
8208 	if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) {
8209 		mutex_enter(&spa_namespace_lock);
8210 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8211 		vdev_autotrim_restart(spa);
8212 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8213 		mutex_exit(&spa_namespace_lock);
8214 	}
8215 
8216 	/*
8217 	 * Kick off L2 cache whole device TRIM.
8218 	 */
8219 	if (tasks & SPA_ASYNC_L2CACHE_TRIM) {
8220 		mutex_enter(&spa_namespace_lock);
8221 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8222 		vdev_trim_l2arc(spa);
8223 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8224 		mutex_exit(&spa_namespace_lock);
8225 	}
8226 
8227 	/*
8228 	 * Kick off L2 cache rebuilding.
8229 	 */
8230 	if (tasks & SPA_ASYNC_L2CACHE_REBUILD) {
8231 		mutex_enter(&spa_namespace_lock);
8232 		spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER);
8233 		l2arc_spa_rebuild_start(spa);
8234 		spa_config_exit(spa, SCL_L2ARC, FTAG);
8235 		mutex_exit(&spa_namespace_lock);
8236 	}
8237 
8238 	/*
8239 	 * Let the world know that we're done.
8240 	 */
8241 	mutex_enter(&spa->spa_async_lock);
8242 	spa->spa_async_thread = NULL;
8243 	cv_broadcast(&spa->spa_async_cv);
8244 	mutex_exit(&spa->spa_async_lock);
8245 	thread_exit();
8246 }
8247 
8248 void
8249 spa_async_suspend(spa_t *spa)
8250 {
8251 	mutex_enter(&spa->spa_async_lock);
8252 	spa->spa_async_suspended++;
8253 	while (spa->spa_async_thread != NULL)
8254 		cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
8255 	mutex_exit(&spa->spa_async_lock);
8256 
8257 	spa_vdev_remove_suspend(spa);
8258 
8259 	zthr_t *condense_thread = spa->spa_condense_zthr;
8260 	if (condense_thread != NULL)
8261 		zthr_cancel(condense_thread);
8262 
8263 	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
8264 	if (discard_thread != NULL)
8265 		zthr_cancel(discard_thread);
8266 
8267 	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
8268 	if (ll_delete_thread != NULL)
8269 		zthr_cancel(ll_delete_thread);
8270 
8271 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
8272 	if (ll_condense_thread != NULL)
8273 		zthr_cancel(ll_condense_thread);
8274 }
8275 
8276 void
8277 spa_async_resume(spa_t *spa)
8278 {
8279 	mutex_enter(&spa->spa_async_lock);
8280 	ASSERT(spa->spa_async_suspended != 0);
8281 	spa->spa_async_suspended--;
8282 	mutex_exit(&spa->spa_async_lock);
8283 	spa_restart_removal(spa);
8284 
8285 	zthr_t *condense_thread = spa->spa_condense_zthr;
8286 	if (condense_thread != NULL)
8287 		zthr_resume(condense_thread);
8288 
8289 	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
8290 	if (discard_thread != NULL)
8291 		zthr_resume(discard_thread);
8292 
8293 	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
8294 	if (ll_delete_thread != NULL)
8295 		zthr_resume(ll_delete_thread);
8296 
8297 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
8298 	if (ll_condense_thread != NULL)
8299 		zthr_resume(ll_condense_thread);
8300 }
8301 
8302 static boolean_t
8303 spa_async_tasks_pending(spa_t *spa)
8304 {
8305 	uint_t non_config_tasks;
8306 	uint_t config_task;
8307 	boolean_t config_task_suspended;
8308 
8309 	non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
8310 	config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
8311 	if (spa->spa_ccw_fail_time == 0) {
8312 		config_task_suspended = B_FALSE;
8313 	} else {
8314 		config_task_suspended =
8315 		    (gethrtime() - spa->spa_ccw_fail_time) <
8316 		    ((hrtime_t)zfs_ccw_retry_interval * NANOSEC);
8317 	}
8318 
8319 	return (non_config_tasks || (config_task && !config_task_suspended));
8320 }
8321 
8322 static void
8323 spa_async_dispatch(spa_t *spa)
8324 {
8325 	mutex_enter(&spa->spa_async_lock);
8326 	if (spa_async_tasks_pending(spa) &&
8327 	    !spa->spa_async_suspended &&
8328 	    spa->spa_async_thread == NULL)
8329 		spa->spa_async_thread = thread_create(NULL, 0,
8330 		    spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
8331 	mutex_exit(&spa->spa_async_lock);
8332 }
8333 
8334 void
8335 spa_async_request(spa_t *spa, int task)
8336 {
8337 	zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
8338 	mutex_enter(&spa->spa_async_lock);
8339 	spa->spa_async_tasks |= task;
8340 	mutex_exit(&spa->spa_async_lock);
8341 }
8342 
8343 int
8344 spa_async_tasks(spa_t *spa)
8345 {
8346 	return (spa->spa_async_tasks);
8347 }
8348 
8349 /*
8350  * ==========================================================================
8351  * SPA syncing routines
8352  * ==========================================================================
8353  */
8354 
8355 
8356 static int
8357 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
8358     dmu_tx_t *tx)
8359 {
8360 	bpobj_t *bpo = arg;
8361 	bpobj_enqueue(bpo, bp, bp_freed, tx);
8362 	return (0);
8363 }
8364 
8365 int
8366 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
8367 {
8368 	return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx));
8369 }
8370 
8371 int
8372 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
8373 {
8374 	return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx));
8375 }
8376 
8377 static int
8378 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
8379 {
8380 	zio_t *pio = arg;
8381 
8382 	zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp,
8383 	    pio->io_flags));
8384 	return (0);
8385 }
8386 
8387 static int
8388 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
8389     dmu_tx_t *tx)
8390 {
8391 	ASSERT(!bp_freed);
8392 	return (spa_free_sync_cb(arg, bp, tx));
8393 }
8394 
8395 /*
8396  * Note: this simple function is not inlined to make it easier to dtrace the
8397  * amount of time spent syncing frees.
8398  */
8399 static void
8400 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx)
8401 {
8402 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
8403 	bplist_iterate(bpl, spa_free_sync_cb, zio, tx);
8404 	VERIFY(zio_wait(zio) == 0);
8405 }
8406 
8407 /*
8408  * Note: this simple function is not inlined to make it easier to dtrace the
8409  * amount of time spent syncing deferred frees.
8410  */
8411 static void
8412 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx)
8413 {
8414 	if (spa_sync_pass(spa) != 1)
8415 		return;
8416 
8417 	/*
8418 	 * Note:
8419 	 * If the log space map feature is active, we stop deferring
8420 	 * frees to the next TXG and therefore running this function
8421 	 * would be considered a no-op as spa_deferred_bpobj should
8422 	 * not have any entries.
8423 	 *
8424 	 * That said we run this function anyway (instead of returning
8425 	 * immediately) for the edge-case scenario where we just
8426 	 * activated the log space map feature in this TXG but we have
8427 	 * deferred frees from the previous TXG.
8428 	 */
8429 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
8430 	VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj,
8431 	    bpobj_spa_free_sync_cb, zio, tx), ==, 0);
8432 	VERIFY0(zio_wait(zio));
8433 }
8434 
8435 static void
8436 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
8437 {
8438 	char *packed = NULL;
8439 	size_t bufsize;
8440 	size_t nvsize = 0;
8441 	dmu_buf_t *db;
8442 
8443 	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);
8444 
8445 	/*
8446 	 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration
8447 	 * information.  This avoids the dmu_buf_will_dirty() path and
8448 	 * saves us a pre-read to get data we don't actually care about.
8449 	 */
8450 	bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE);
8451 	packed = vmem_alloc(bufsize, KM_SLEEP);
8452 
8453 	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
8454 	    KM_SLEEP) == 0);
8455 	bzero(packed + nvsize, bufsize - nvsize);
8456 
8457 	dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx);
8458 
8459 	vmem_free(packed, bufsize);
8460 
8461 	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
8462 	dmu_buf_will_dirty(db, tx);
8463 	*(uint64_t *)db->db_data = nvsize;
8464 	dmu_buf_rele(db, FTAG);
8465 }
8466 
8467 static void
8468 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx,
8469     const char *config, const char *entry)
8470 {
8471 	nvlist_t *nvroot;
8472 	nvlist_t **list;
8473 	int i;
8474 
8475 	if (!sav->sav_sync)
8476 		return;
8477 
8478 	/*
8479 	 * Update the MOS nvlist describing the list of available devices.
8480 	 * spa_validate_aux() will have already made sure this nvlist is
8481 	 * valid and the vdevs are labeled appropriately.
8482 	 */
8483 	if (sav->sav_object == 0) {
8484 		sav->sav_object = dmu_object_alloc(spa->spa_meta_objset,
8485 		    DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE,
8486 		    sizeof (uint64_t), tx);
8487 		VERIFY(zap_update(spa->spa_meta_objset,
8488 		    DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1,
8489 		    &sav->sav_object, tx) == 0);
8490 	}
8491 
8492 	VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0);
8493 	if (sav->sav_count == 0) {
8494 		VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0);
8495 	} else {
8496 		list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP);
8497 		for (i = 0; i < sav->sav_count; i++)
8498 			list[i] = vdev_config_generate(spa, sav->sav_vdevs[i],
8499 			    B_FALSE, VDEV_CONFIG_L2CACHE);
8500 		VERIFY(nvlist_add_nvlist_array(nvroot, config, list,
8501 		    sav->sav_count) == 0);
8502 		for (i = 0; i < sav->sav_count; i++)
8503 			nvlist_free(list[i]);
8504 		kmem_free(list, sav->sav_count * sizeof (void *));
8505 	}
8506 
8507 	spa_sync_nvlist(spa, sav->sav_object, nvroot, tx);
8508 	nvlist_free(nvroot);
8509 
8510 	sav->sav_sync = B_FALSE;
8511 }
8512 
8513 /*
8514  * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t.
8515  * The all-vdev ZAP must be empty.
8516  */
8517 static void
8518 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx)
8519 {
8520 	spa_t *spa = vd->vdev_spa;
8521 
8522 	if (vd->vdev_top_zap != 0) {
8523 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
8524 		    vd->vdev_top_zap, tx));
8525 	}
8526 	if (vd->vdev_leaf_zap != 0) {
8527 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
8528 		    vd->vdev_leaf_zap, tx));
8529 	}
8530 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
8531 		spa_avz_build(vd->vdev_child[i], avz, tx);
8532 	}
8533 }
8534 
8535 static void
8536 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx)
8537 {
8538 	nvlist_t *config;
8539 
8540 	/*
8541 	 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS,
8542 	 * its config may not be dirty but we still need to build per-vdev ZAPs.
8543 	 * Similarly, if the pool is being assembled (e.g. after a split), we
8544 	 * need to rebuild the AVZ although the config may not be dirty.
8545 	 */
8546 	if (list_is_empty(&spa->spa_config_dirty_list) &&
8547 	    spa->spa_avz_action == AVZ_ACTION_NONE)
8548 		return;
8549 
8550 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
8551 
8552 	ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE ||
8553 	    spa->spa_avz_action == AVZ_ACTION_INITIALIZE ||
8554 	    spa->spa_all_vdev_zaps != 0);
8555 
8556 	if (spa->spa_avz_action == AVZ_ACTION_REBUILD) {
8557 		/* Make and build the new AVZ */
8558 		uint64_t new_avz = zap_create(spa->spa_meta_objset,
8559 		    DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx);
8560 		spa_avz_build(spa->spa_root_vdev, new_avz, tx);
8561 
8562 		/* Diff old AVZ with new one */
8563 		zap_cursor_t zc;
8564 		zap_attribute_t za;
8565 
8566 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
8567 		    spa->spa_all_vdev_zaps);
8568 		    zap_cursor_retrieve(&zc, &za) == 0;
8569 		    zap_cursor_advance(&zc)) {
8570 			uint64_t vdzap = za.za_first_integer;
8571 			if (zap_lookup_int(spa->spa_meta_objset, new_avz,
8572 			    vdzap) == ENOENT) {
8573 				/*
8574 				 * ZAP is listed in old AVZ but not in new one;
8575 				 * destroy it
8576 				 */
8577 				VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap,
8578 				    tx));
8579 			}
8580 		}
8581 
8582 		zap_cursor_fini(&zc);
8583 
8584 		/* Destroy the old AVZ */
8585 		VERIFY0(zap_destroy(spa->spa_meta_objset,
8586 		    spa->spa_all_vdev_zaps, tx));
8587 
8588 		/* Replace the old AVZ in the dir obj with the new one */
8589 		VERIFY0(zap_update(spa->spa_meta_objset,
8590 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP,
8591 		    sizeof (new_avz), 1, &new_avz, tx));
8592 
8593 		spa->spa_all_vdev_zaps = new_avz;
8594 	} else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) {
8595 		zap_cursor_t zc;
8596 		zap_attribute_t za;
8597 
8598 		/* Walk through the AVZ and destroy all listed ZAPs */
8599 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
8600 		    spa->spa_all_vdev_zaps);
8601 		    zap_cursor_retrieve(&zc, &za) == 0;
8602 		    zap_cursor_advance(&zc)) {
8603 			uint64_t zap = za.za_first_integer;
8604 			VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx));
8605 		}
8606 
8607 		zap_cursor_fini(&zc);
8608 
8609 		/* Destroy and unlink the AVZ itself */
8610 		VERIFY0(zap_destroy(spa->spa_meta_objset,
8611 		    spa->spa_all_vdev_zaps, tx));
8612 		VERIFY0(zap_remove(spa->spa_meta_objset,
8613 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx));
8614 		spa->spa_all_vdev_zaps = 0;
8615 	}
8616 
8617 	if (spa->spa_all_vdev_zaps == 0) {
8618 		spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset,
8619 		    DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
8620 		    DMU_POOL_VDEV_ZAP_MAP, tx);
8621 	}
8622 	spa->spa_avz_action = AVZ_ACTION_NONE;
8623 
8624 	/* Create ZAPs for vdevs that don't have them. */
8625 	vdev_construct_zaps(spa->spa_root_vdev, tx);
8626 
8627 	config = spa_config_generate(spa, spa->spa_root_vdev,
8628 	    dmu_tx_get_txg(tx), B_FALSE);
8629 
8630 	/*
8631 	 * If we're upgrading the spa version then make sure that
8632 	 * the config object gets updated with the correct version.
8633 	 */
8634 	if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version)
8635 		fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
8636 		    spa->spa_uberblock.ub_version);
8637 
8638 	spa_config_exit(spa, SCL_STATE, FTAG);
8639 
8640 	nvlist_free(spa->spa_config_syncing);
8641 	spa->spa_config_syncing = config;
8642 
8643 	spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
8644 }
8645 
8646 static void
8647 spa_sync_version(void *arg, dmu_tx_t *tx)
8648 {
8649 	uint64_t *versionp = arg;
8650 	uint64_t version = *versionp;
8651 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
8652 
8653 	/*
8654 	 * Setting the version is special cased when first creating the pool.
8655 	 */
8656 	ASSERT(tx->tx_txg != TXG_INITIAL);
8657 
8658 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
8659 	ASSERT(version >= spa_version(spa));
8660 
8661 	spa->spa_uberblock.ub_version = version;
8662 	vdev_config_dirty(spa->spa_root_vdev);
8663 	spa_history_log_internal(spa, "set", tx, "version=%lld",
8664 	    (longlong_t)version);
8665 }
8666 
8667 /*
8668  * Set zpool properties.
8669  */
8670 static void
8671 spa_sync_props(void *arg, dmu_tx_t *tx)
8672 {
8673 	nvlist_t *nvp = arg;
8674 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
8675 	objset_t *mos = spa->spa_meta_objset;
8676 	nvpair_t *elem = NULL;
8677 
8678 	mutex_enter(&spa->spa_props_lock);
8679 
8680 	while ((elem = nvlist_next_nvpair(nvp, elem))) {
8681 		uint64_t intval;
8682 		char *strval, *fname;
8683 		zpool_prop_t prop;
8684 		const char *propname;
8685 		zprop_type_t proptype;
8686 		spa_feature_t fid;
8687 
8688 		switch (prop = zpool_name_to_prop(nvpair_name(elem))) {
8689 		case ZPOOL_PROP_INVAL:
8690 			/*
8691 			 * We checked this earlier in spa_prop_validate().
8692 			 */
8693 			ASSERT(zpool_prop_feature(nvpair_name(elem)));
8694 
8695 			fname = strchr(nvpair_name(elem), '@') + 1;
8696 			VERIFY0(zfeature_lookup_name(fname, &fid));
8697 
8698 			spa_feature_enable(spa, fid, tx);
8699 			spa_history_log_internal(spa, "set", tx,
8700 			    "%s=enabled", nvpair_name(elem));
8701 			break;
8702 
8703 		case ZPOOL_PROP_VERSION:
8704 			intval = fnvpair_value_uint64(elem);
8705 			/*
8706 			 * The version is synced separately before other
8707 			 * properties and should be correct by now.
8708 			 */
8709 			ASSERT3U(spa_version(spa), >=, intval);
8710 			break;
8711 
8712 		case ZPOOL_PROP_ALTROOT:
8713 			/*
8714 			 * 'altroot' is a non-persistent property. It should
8715 			 * have been set temporarily at creation or import time.
8716 			 */
8717 			ASSERT(spa->spa_root != NULL);
8718 			break;
8719 
8720 		case ZPOOL_PROP_READONLY:
8721 		case ZPOOL_PROP_CACHEFILE:
8722 			/*
8723 			 * 'readonly' and 'cachefile' are also non-persistent
8724 			 * properties.
8725 			 */
8726 			break;
8727 		case ZPOOL_PROP_COMMENT:
8728 			strval = fnvpair_value_string(elem);
8729 			if (spa->spa_comment != NULL)
8730 				spa_strfree(spa->spa_comment);
8731 			spa->spa_comment = spa_strdup(strval);
8732 			/*
8733 			 * We need to dirty the configuration on all the vdevs
8734 			 * so that their labels get updated.  We also need to
8735 			 * update the cache file to keep it in sync with the
8736 			 * MOS version. It's unnecessary to do this for pool
8737 			 * creation since the vdev's configuration has already
8738 			 * been dirtied.
8739 			 */
8740 			if (tx->tx_txg != TXG_INITIAL) {
8741 				vdev_config_dirty(spa->spa_root_vdev);
8742 				spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
8743 			}
8744 			spa_history_log_internal(spa, "set", tx,
8745 			    "%s=%s", nvpair_name(elem), strval);
8746 			break;
8747 		case ZPOOL_PROP_COMPATIBILITY:
8748 			strval = fnvpair_value_string(elem);
8749 			if (spa->spa_compatibility != NULL)
8750 				spa_strfree(spa->spa_compatibility);
8751 			spa->spa_compatibility = spa_strdup(strval);
8752 			/*
8753 			 * Dirty the configuration on vdevs as above.
8754 			 */
8755 			if (tx->tx_txg != TXG_INITIAL) {
8756 				vdev_config_dirty(spa->spa_root_vdev);
8757 				spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
8758 			}
8759 
8760 			spa_history_log_internal(spa, "set", tx,
8761 			    "%s=%s", nvpair_name(elem), strval);
8762 			break;
8763 
8764 		default:
8765 			/*
8766 			 * Set pool property values in the poolprops mos object.
8767 			 */
8768 			if (spa->spa_pool_props_object == 0) {
8769 				spa->spa_pool_props_object =
8770 				    zap_create_link(mos, DMU_OT_POOL_PROPS,
8771 				    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS,
8772 				    tx);
8773 			}
8774 
8775 			/* normalize the property name */
8776 			propname = zpool_prop_to_name(prop);
8777 			proptype = zpool_prop_get_type(prop);
8778 
8779 			if (nvpair_type(elem) == DATA_TYPE_STRING) {
8780 				ASSERT(proptype == PROP_TYPE_STRING);
8781 				strval = fnvpair_value_string(elem);
8782 				VERIFY0(zap_update(mos,
8783 				    spa->spa_pool_props_object, propname,
8784 				    1, strlen(strval) + 1, strval, tx));
8785 				spa_history_log_internal(spa, "set", tx,
8786 				    "%s=%s", nvpair_name(elem), strval);
8787 			} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
8788 				intval = fnvpair_value_uint64(elem);
8789 
8790 				if (proptype == PROP_TYPE_INDEX) {
8791 					const char *unused;
8792 					VERIFY0(zpool_prop_index_to_string(
8793 					    prop, intval, &unused));
8794 				}
8795 				VERIFY0(zap_update(mos,
8796 				    spa->spa_pool_props_object, propname,
8797 				    8, 1, &intval, tx));
8798 				spa_history_log_internal(spa, "set", tx,
8799 				    "%s=%lld", nvpair_name(elem),
8800 				    (longlong_t)intval);
8801 			} else {
8802 				ASSERT(0); /* not allowed */
8803 			}
8804 
8805 			switch (prop) {
8806 			case ZPOOL_PROP_DELEGATION:
8807 				spa->spa_delegation = intval;
8808 				break;
8809 			case ZPOOL_PROP_BOOTFS:
8810 				spa->spa_bootfs = intval;
8811 				break;
8812 			case ZPOOL_PROP_FAILUREMODE:
8813 				spa->spa_failmode = intval;
8814 				break;
8815 			case ZPOOL_PROP_AUTOTRIM:
8816 				spa->spa_autotrim = intval;
8817 				spa_async_request(spa,
8818 				    SPA_ASYNC_AUTOTRIM_RESTART);
8819 				break;
8820 			case ZPOOL_PROP_AUTOEXPAND:
8821 				spa->spa_autoexpand = intval;
8822 				if (tx->tx_txg != TXG_INITIAL)
8823 					spa_async_request(spa,
8824 					    SPA_ASYNC_AUTOEXPAND);
8825 				break;
8826 			case ZPOOL_PROP_MULTIHOST:
8827 				spa->spa_multihost = intval;
8828 				break;
8829 			default:
8830 				break;
8831 			}
8832 		}
8833 
8834 	}
8835 
8836 	mutex_exit(&spa->spa_props_lock);
8837 }
8838 
8839 /*
8840  * Perform one-time upgrade on-disk changes.  spa_version() does not
8841  * reflect the new version this txg, so there must be no changes this
8842  * txg to anything that the upgrade code depends on after it executes.
8843  * Therefore this must be called after dsl_pool_sync() does the sync
8844  * tasks.
8845  */
8846 static void
8847 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx)
8848 {
8849 	if (spa_sync_pass(spa) != 1)
8850 		return;
8851 
8852 	dsl_pool_t *dp = spa->spa_dsl_pool;
8853 	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
8854 
8855 	if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN &&
8856 	    spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) {
8857 		dsl_pool_create_origin(dp, tx);
8858 
8859 		/* Keeping the origin open increases spa_minref */
8860 		spa->spa_minref += 3;
8861 	}
8862 
8863 	if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES &&
8864 	    spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) {
8865 		dsl_pool_upgrade_clones(dp, tx);
8866 	}
8867 
8868 	if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES &&
8869 	    spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) {
8870 		dsl_pool_upgrade_dir_clones(dp, tx);
8871 
8872 		/* Keeping the freedir open increases spa_minref */
8873 		spa->spa_minref += 3;
8874 	}
8875 
8876 	if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES &&
8877 	    spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
8878 		spa_feature_create_zap_objects(spa, tx);
8879 	}
8880 
8881 	/*
8882 	 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable
8883 	 * when possibility to use lz4 compression for metadata was added
8884 	 * Old pools that have this feature enabled must be upgraded to have
8885 	 * this feature active
8886 	 */
8887 	if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
8888 		boolean_t lz4_en = spa_feature_is_enabled(spa,
8889 		    SPA_FEATURE_LZ4_COMPRESS);
8890 		boolean_t lz4_ac = spa_feature_is_active(spa,
8891 		    SPA_FEATURE_LZ4_COMPRESS);
8892 
8893 		if (lz4_en && !lz4_ac)
8894 			spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
8895 	}
8896 
8897 	/*
8898 	 * If we haven't written the salt, do so now.  Note that the
8899 	 * feature may not be activated yet, but that's fine since
8900 	 * the presence of this ZAP entry is backwards compatible.
8901 	 */
8902 	if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
8903 	    DMU_POOL_CHECKSUM_SALT) == ENOENT) {
8904 		VERIFY0(zap_add(spa->spa_meta_objset,
8905 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1,
8906 		    sizeof (spa->spa_cksum_salt.zcs_bytes),
8907 		    spa->spa_cksum_salt.zcs_bytes, tx));
8908 	}
8909 
8910 	rrw_exit(&dp->dp_config_rwlock, FTAG);
8911 }
8912 
8913 static void
8914 vdev_indirect_state_sync_verify(vdev_t *vd)
8915 {
8916 	vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping;
8917 	vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births;
8918 
8919 	if (vd->vdev_ops == &vdev_indirect_ops) {
8920 		ASSERT(vim != NULL);
8921 		ASSERT(vib != NULL);
8922 	}
8923 
8924 	uint64_t obsolete_sm_object = 0;
8925 	ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object));
8926 	if (obsolete_sm_object != 0) {
8927 		ASSERT(vd->vdev_obsolete_sm != NULL);
8928 		ASSERT(vd->vdev_removing ||
8929 		    vd->vdev_ops == &vdev_indirect_ops);
8930 		ASSERT(vdev_indirect_mapping_num_entries(vim) > 0);
8931 		ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0);
8932 		ASSERT3U(obsolete_sm_object, ==,
8933 		    space_map_object(vd->vdev_obsolete_sm));
8934 		ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=,
8935 		    space_map_allocated(vd->vdev_obsolete_sm));
8936 	}
8937 	ASSERT(vd->vdev_obsolete_segments != NULL);
8938 
8939 	/*
8940 	 * Since frees / remaps to an indirect vdev can only
8941 	 * happen in syncing context, the obsolete segments
8942 	 * tree must be empty when we start syncing.
8943 	 */
8944 	ASSERT0(range_tree_space(vd->vdev_obsolete_segments));
8945 }
8946 
8947 /*
8948  * Set the top-level vdev's max queue depth. Evaluate each top-level's
8949  * async write queue depth in case it changed. The max queue depth will
8950  * not change in the middle of syncing out this txg.
8951  */
8952 static void
8953 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa)
8954 {
8955 	ASSERT(spa_writeable(spa));
8956 
8957 	vdev_t *rvd = spa->spa_root_vdev;
8958 	uint32_t max_queue_depth = zfs_vdev_async_write_max_active *
8959 	    zfs_vdev_queue_depth_pct / 100;
8960 	metaslab_class_t *normal = spa_normal_class(spa);
8961 	metaslab_class_t *special = spa_special_class(spa);
8962 	metaslab_class_t *dedup = spa_dedup_class(spa);
8963 
8964 	uint64_t slots_per_allocator = 0;
8965 	for (int c = 0; c < rvd->vdev_children; c++) {
8966 		vdev_t *tvd = rvd->vdev_child[c];
8967 
8968 		metaslab_group_t *mg = tvd->vdev_mg;
8969 		if (mg == NULL || !metaslab_group_initialized(mg))
8970 			continue;
8971 
8972 		metaslab_class_t *mc = mg->mg_class;
8973 		if (mc != normal && mc != special && mc != dedup)
8974 			continue;
8975 
8976 		/*
8977 		 * It is safe to do a lock-free check here because only async
8978 		 * allocations look at mg_max_alloc_queue_depth, and async
8979 		 * allocations all happen from spa_sync().
8980 		 */
8981 		for (int i = 0; i < mg->mg_allocators; i++) {
8982 			ASSERT0(zfs_refcount_count(
8983 			    &(mg->mg_allocator[i].mga_alloc_queue_depth)));
8984 		}
8985 		mg->mg_max_alloc_queue_depth = max_queue_depth;
8986 
8987 		for (int i = 0; i < mg->mg_allocators; i++) {
8988 			mg->mg_allocator[i].mga_cur_max_alloc_queue_depth =
8989 			    zfs_vdev_def_queue_depth;
8990 		}
8991 		slots_per_allocator += zfs_vdev_def_queue_depth;
8992 	}
8993 
8994 	for (int i = 0; i < spa->spa_alloc_count; i++) {
8995 		ASSERT0(zfs_refcount_count(&normal->mc_allocator[i].
8996 		    mca_alloc_slots));
8997 		ASSERT0(zfs_refcount_count(&special->mc_allocator[i].
8998 		    mca_alloc_slots));
8999 		ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i].
9000 		    mca_alloc_slots));
9001 		normal->mc_allocator[i].mca_alloc_max_slots =
9002 		    slots_per_allocator;
9003 		special->mc_allocator[i].mca_alloc_max_slots =
9004 		    slots_per_allocator;
9005 		dedup->mc_allocator[i].mca_alloc_max_slots =
9006 		    slots_per_allocator;
9007 	}
9008 	normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9009 	special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9010 	dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9011 }
9012 
9013 static void
9014 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx)
9015 {
9016 	ASSERT(spa_writeable(spa));
9017 
9018 	vdev_t *rvd = spa->spa_root_vdev;
9019 	for (int c = 0; c < rvd->vdev_children; c++) {
9020 		vdev_t *vd = rvd->vdev_child[c];
9021 		vdev_indirect_state_sync_verify(vd);
9022 
9023 		if (vdev_indirect_should_condense(vd)) {
9024 			spa_condense_indirect_start_sync(vd, tx);
9025 			break;
9026 		}
9027 	}
9028 }
9029 
9030 static void
9031 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx)
9032 {
9033 	objset_t *mos = spa->spa_meta_objset;
9034 	dsl_pool_t *dp = spa->spa_dsl_pool;
9035 	uint64_t txg = tx->tx_txg;
9036 	bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK];
9037 
9038 	do {
9039 		int pass = ++spa->spa_sync_pass;
9040 
9041 		spa_sync_config_object(spa, tx);
9042 		spa_sync_aux_dev(spa, &spa->spa_spares, tx,
9043 		    ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES);
9044 		spa_sync_aux_dev(spa, &spa->spa_l2cache, tx,
9045 		    ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE);
9046 		spa_errlog_sync(spa, txg);
9047 		dsl_pool_sync(dp, txg);
9048 
9049 		if (pass < zfs_sync_pass_deferred_free ||
9050 		    spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) {
9051 			/*
9052 			 * If the log space map feature is active we don't
9053 			 * care about deferred frees and the deferred bpobj
9054 			 * as the log space map should effectively have the
9055 			 * same results (i.e. appending only to one object).
9056 			 */
9057 			spa_sync_frees(spa, free_bpl, tx);
9058 		} else {
9059 			/*
9060 			 * We can not defer frees in pass 1, because
9061 			 * we sync the deferred frees later in pass 1.
9062 			 */
9063 			ASSERT3U(pass, >, 1);
9064 			bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb,
9065 			    &spa->spa_deferred_bpobj, tx);
9066 		}
9067 
9068 		ddt_sync(spa, txg);
9069 		dsl_scan_sync(dp, tx);
9070 		svr_sync(spa, tx);
9071 		spa_sync_upgrades(spa, tx);
9072 
9073 		spa_flush_metaslabs(spa, tx);
9074 
9075 		vdev_t *vd = NULL;
9076 		while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
9077 		    != NULL)
9078 			vdev_sync(vd, txg);
9079 
9080 		/*
9081 		 * Note: We need to check if the MOS is dirty because we could
9082 		 * have marked the MOS dirty without updating the uberblock
9083 		 * (e.g. if we have sync tasks but no dirty user data). We need
9084 		 * to check the uberblock's rootbp because it is updated if we
9085 		 * have synced out dirty data (though in this case the MOS will
9086 		 * most likely also be dirty due to second order effects, we
9087 		 * don't want to rely on that here).
9088 		 */
9089 		if (pass == 1 &&
9090 		    spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
9091 		    !dmu_objset_is_dirty(mos, txg)) {
9092 			/*
9093 			 * Nothing changed on the first pass, therefore this
9094 			 * TXG is a no-op. Avoid syncing deferred frees, so
9095 			 * that we can keep this TXG as a no-op.
9096 			 */
9097 			ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
9098 			ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
9099 			ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
9100 			ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg));
9101 			break;
9102 		}
9103 
9104 		spa_sync_deferred_frees(spa, tx);
9105 	} while (dmu_objset_is_dirty(mos, txg));
9106 }
9107 
9108 /*
9109  * Rewrite the vdev configuration (which includes the uberblock) to
9110  * commit the transaction group.
9111  *
9112  * If there are no dirty vdevs, we sync the uberblock to a few random
9113  * top-level vdevs that are known to be visible in the config cache
9114  * (see spa_vdev_add() for a complete description). If there *are* dirty
9115  * vdevs, sync the uberblock to all vdevs.
9116  */
9117 static void
9118 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx)
9119 {
9120 	vdev_t *rvd = spa->spa_root_vdev;
9121 	uint64_t txg = tx->tx_txg;
9122 
9123 	for (;;) {
9124 		int error = 0;
9125 
9126 		/*
9127 		 * We hold SCL_STATE to prevent vdev open/close/etc.
9128 		 * while we're attempting to write the vdev labels.
9129 		 */
9130 		spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9131 
9132 		if (list_is_empty(&spa->spa_config_dirty_list)) {
9133 			vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
9134 			int svdcount = 0;
9135 			int children = rvd->vdev_children;
9136 			int c0 = random_in_range(children);
9137 
9138 			for (int c = 0; c < children; c++) {
9139 				vdev_t *vd =
9140 				    rvd->vdev_child[(c0 + c) % children];
9141 
9142 				/* Stop when revisiting the first vdev */
9143 				if (c > 0 && svd[0] == vd)
9144 					break;
9145 
9146 				if (vd->vdev_ms_array == 0 ||
9147 				    vd->vdev_islog ||
9148 				    !vdev_is_concrete(vd))
9149 					continue;
9150 
9151 				svd[svdcount++] = vd;
9152 				if (svdcount == SPA_SYNC_MIN_VDEVS)
9153 					break;
9154 			}
9155 			error = vdev_config_sync(svd, svdcount, txg);
9156 		} else {
9157 			error = vdev_config_sync(rvd->vdev_child,
9158 			    rvd->vdev_children, txg);
9159 		}
9160 
9161 		if (error == 0)
9162 			spa->spa_last_synced_guid = rvd->vdev_guid;
9163 
9164 		spa_config_exit(spa, SCL_STATE, FTAG);
9165 
9166 		if (error == 0)
9167 			break;
9168 		zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR);
9169 		zio_resume_wait(spa);
9170 	}
9171 }
9172 
9173 /*
9174  * Sync the specified transaction group.  New blocks may be dirtied as
9175  * part of the process, so we iterate until it converges.
9176  */
9177 void
9178 spa_sync(spa_t *spa, uint64_t txg)
9179 {
9180 	vdev_t *vd = NULL;
9181 
9182 	VERIFY(spa_writeable(spa));
9183 
9184 	/*
9185 	 * Wait for i/os issued in open context that need to complete
9186 	 * before this txg syncs.
9187 	 */
9188 	(void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]);
9189 	spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL,
9190 	    ZIO_FLAG_CANFAIL);
9191 
9192 	/*
9193 	 * Lock out configuration changes.
9194 	 */
9195 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
9196 
9197 	spa->spa_syncing_txg = txg;
9198 	spa->spa_sync_pass = 0;
9199 
9200 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9201 		mutex_enter(&spa->spa_allocs[i].spaa_lock);
9202 		VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
9203 		mutex_exit(&spa->spa_allocs[i].spaa_lock);
9204 	}
9205 
9206 	/*
9207 	 * If there are any pending vdev state changes, convert them
9208 	 * into config changes that go out with this transaction group.
9209 	 */
9210 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9211 	while (list_head(&spa->spa_state_dirty_list) != NULL) {
9212 		/*
9213 		 * We need the write lock here because, for aux vdevs,
9214 		 * calling vdev_config_dirty() modifies sav_config.
9215 		 * This is ugly and will become unnecessary when we
9216 		 * eliminate the aux vdev wart by integrating all vdevs
9217 		 * into the root vdev tree.
9218 		 */
9219 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9220 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER);
9221 		while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
9222 			vdev_state_clean(vd);
9223 			vdev_config_dirty(vd);
9224 		}
9225 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9226 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
9227 	}
9228 	spa_config_exit(spa, SCL_STATE, FTAG);
9229 
9230 	dsl_pool_t *dp = spa->spa_dsl_pool;
9231 	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);
9232 
9233 	spa->spa_sync_starttime = gethrtime();
9234 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
9235 	spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq,
9236 	    spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() +
9237 	    NSEC_TO_TICK(spa->spa_deadman_synctime));
9238 
9239 	/*
9240 	 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg,
9241 	 * set spa_deflate if we have no raid-z vdevs.
9242 	 */
9243 	if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE &&
9244 	    spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) {
9245 		vdev_t *rvd = spa->spa_root_vdev;
9246 
9247 		int i;
9248 		for (i = 0; i < rvd->vdev_children; i++) {
9249 			vd = rvd->vdev_child[i];
9250 			if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
9251 				break;
9252 		}
9253 		if (i == rvd->vdev_children) {
9254 			spa->spa_deflate = TRUE;
9255 			VERIFY0(zap_add(spa->spa_meta_objset,
9256 			    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
9257 			    sizeof (uint64_t), 1, &spa->spa_deflate, tx));
9258 		}
9259 	}
9260 
9261 	spa_sync_adjust_vdev_max_queue_depth(spa);
9262 
9263 	spa_sync_condense_indirect(spa, tx);
9264 
9265 	spa_sync_iterate_to_convergence(spa, tx);
9266 
9267 #ifdef ZFS_DEBUG
9268 	if (!list_is_empty(&spa->spa_config_dirty_list)) {
9269 	/*
9270 	 * Make sure that the number of ZAPs for all the vdevs matches
9271 	 * the number of ZAPs in the per-vdev ZAP list. This only gets
9272 	 * called if the config is dirty; otherwise there may be
9273 	 * outstanding AVZ operations that weren't completed in
9274 	 * spa_sync_config_object.
9275 	 */
9276 		uint64_t all_vdev_zap_entry_count;
9277 		ASSERT0(zap_count(spa->spa_meta_objset,
9278 		    spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count));
9279 		ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==,
9280 		    all_vdev_zap_entry_count);
9281 	}
9282 #endif
9283 
9284 	if (spa->spa_vdev_removal != NULL) {
9285 		ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]);
9286 	}
9287 
9288 	spa_sync_rewrite_vdev_config(spa, tx);
9289 	dmu_tx_commit(tx);
9290 
9291 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
9292 	spa->spa_deadman_tqid = 0;
9293 
9294 	/*
9295 	 * Clear the dirty config list.
9296 	 */
9297 	while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL)
9298 		vdev_config_clean(vd);
9299 
9300 	/*
9301 	 * Now that the new config has synced transactionally,
9302 	 * let it become visible to the config cache.
9303 	 */
9304 	if (spa->spa_config_syncing != NULL) {
9305 		spa_config_set(spa, spa->spa_config_syncing);
9306 		spa->spa_config_txg = txg;
9307 		spa->spa_config_syncing = NULL;
9308 	}
9309 
9310 	dsl_pool_sync_done(dp, txg);
9311 
9312 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9313 		mutex_enter(&spa->spa_allocs[i].spaa_lock);
9314 		VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
9315 		mutex_exit(&spa->spa_allocs[i].spaa_lock);
9316 	}
9317 
9318 	/*
9319 	 * Update usable space statistics.
9320 	 */
9321 	while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
9322 	    != NULL)
9323 		vdev_sync_done(vd, txg);
9324 
9325 	metaslab_class_evict_old(spa->spa_normal_class, txg);
9326 	metaslab_class_evict_old(spa->spa_log_class, txg);
9327 
9328 	spa_sync_close_syncing_log_sm(spa);
9329 
9330 	spa_update_dspace(spa);
9331 
9332 	/*
9333 	 * It had better be the case that we didn't dirty anything
9334 	 * since vdev_config_sync().
9335 	 */
9336 	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
9337 	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
9338 	ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));
9339 
9340 	while (zfs_pause_spa_sync)
9341 		delay(1);
9342 
9343 	spa->spa_sync_pass = 0;
9344 
9345 	/*
9346 	 * Update the last synced uberblock here. We want to do this at
9347 	 * the end of spa_sync() so that consumers of spa_last_synced_txg()
9348 	 * will be guaranteed that all the processing associated with
9349 	 * that txg has been completed.
9350 	 */
9351 	spa->spa_ubsync = spa->spa_uberblock;
9352 	spa_config_exit(spa, SCL_CONFIG, FTAG);
9353 
9354 	spa_handle_ignored_writes(spa);
9355 
9356 	/*
9357 	 * If any async tasks have been requested, kick them off.
9358 	 */
9359 	spa_async_dispatch(spa);
9360 }
9361 
9362 /*
9363  * Sync all pools.  We don't want to hold the namespace lock across these
9364  * operations, so we take a reference on the spa_t and drop the lock during the
9365  * sync.
9366  */
9367 void
9368 spa_sync_allpools(void)
9369 {
9370 	spa_t *spa = NULL;
9371 	mutex_enter(&spa_namespace_lock);
9372 	while ((spa = spa_next(spa)) != NULL) {
9373 		if (spa_state(spa) != POOL_STATE_ACTIVE ||
9374 		    !spa_writeable(spa) || spa_suspended(spa))
9375 			continue;
9376 		spa_open_ref(spa, FTAG);
9377 		mutex_exit(&spa_namespace_lock);
9378 		txg_wait_synced(spa_get_dsl(spa), 0);
9379 		mutex_enter(&spa_namespace_lock);
9380 		spa_close(spa, FTAG);
9381 	}
9382 	mutex_exit(&spa_namespace_lock);
9383 }
9384 
9385 /*
9386  * ==========================================================================
9387  * Miscellaneous routines
9388  * ==========================================================================
9389  */
9390 
9391 /*
9392  * Remove all pools in the system.
9393  */
9394 void
9395 spa_evict_all(void)
9396 {
9397 	spa_t *spa;
9398 
9399 	/*
9400 	 * Remove all cached state.  All pools should be closed now,
9401 	 * so every spa in the AVL tree should be unreferenced.
9402 	 */
9403 	mutex_enter(&spa_namespace_lock);
9404 	while ((spa = spa_next(NULL)) != NULL) {
9405 		/*
9406 		 * Stop async tasks.  The async thread may need to detach
9407 		 * a device that's been replaced, which requires grabbing
9408 		 * spa_namespace_lock, so we must drop it here.
9409 		 */
9410 		spa_open_ref(spa, FTAG);
9411 		mutex_exit(&spa_namespace_lock);
9412 		spa_async_suspend(spa);
9413 		mutex_enter(&spa_namespace_lock);
9414 		spa_close(spa, FTAG);
9415 
9416 		if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
9417 			spa_unload(spa);
9418 			spa_deactivate(spa);
9419 		}
9420 		spa_remove(spa);
9421 	}
9422 	mutex_exit(&spa_namespace_lock);
9423 }
9424 
9425 vdev_t *
9426 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux)
9427 {
9428 	vdev_t *vd;
9429 	int i;
9430 
9431 	if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL)
9432 		return (vd);
9433 
9434 	if (aux) {
9435 		for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
9436 			vd = spa->spa_l2cache.sav_vdevs[i];
9437 			if (vd->vdev_guid == guid)
9438 				return (vd);
9439 		}
9440 
9441 		for (i = 0; i < spa->spa_spares.sav_count; i++) {
9442 			vd = spa->spa_spares.sav_vdevs[i];
9443 			if (vd->vdev_guid == guid)
9444 				return (vd);
9445 		}
9446 	}
9447 
9448 	return (NULL);
9449 }
9450 
9451 void
9452 spa_upgrade(spa_t *spa, uint64_t version)
9453 {
9454 	ASSERT(spa_writeable(spa));
9455 
9456 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
9457 
9458 	/*
9459 	 * This should only be called for a non-faulted pool, and since a
9460 	 * future version would result in an unopenable pool, this shouldn't be
9461 	 * possible.
9462 	 */
9463 	ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version));
9464 	ASSERT3U(version, >=, spa->spa_uberblock.ub_version);
9465 
9466 	spa->spa_uberblock.ub_version = version;
9467 	vdev_config_dirty(spa->spa_root_vdev);
9468 
9469 	spa_config_exit(spa, SCL_ALL, FTAG);
9470 
9471 	txg_wait_synced(spa_get_dsl(spa), 0);
9472 }
9473 
9474 boolean_t
9475 spa_has_spare(spa_t *spa, uint64_t guid)
9476 {
9477 	int i;
9478 	uint64_t spareguid;
9479 	spa_aux_vdev_t *sav = &spa->spa_spares;
9480 
9481 	for (i = 0; i < sav->sav_count; i++)
9482 		if (sav->sav_vdevs[i]->vdev_guid == guid)
9483 			return (B_TRUE);
9484 
9485 	for (i = 0; i < sav->sav_npending; i++) {
9486 		if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID,
9487 		    &spareguid) == 0 && spareguid == guid)
9488 			return (B_TRUE);
9489 	}
9490 
9491 	return (B_FALSE);
9492 }
9493 
9494 /*
9495  * Check if a pool has an active shared spare device.
9496  * Note: reference count of an active spare is 2, as a spare and as a replace
9497  */
9498 static boolean_t
9499 spa_has_active_shared_spare(spa_t *spa)
9500 {
9501 	int i, refcnt;
9502 	uint64_t pool;
9503 	spa_aux_vdev_t *sav = &spa->spa_spares;
9504 
9505 	for (i = 0; i < sav->sav_count; i++) {
9506 		if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool,
9507 		    &refcnt) && pool != 0ULL && pool == spa_guid(spa) &&
9508 		    refcnt > 2)
9509 			return (B_TRUE);
9510 	}
9511 
9512 	return (B_FALSE);
9513 }
9514 
9515 uint64_t
9516 spa_total_metaslabs(spa_t *spa)
9517 {
9518 	vdev_t *rvd = spa->spa_root_vdev;
9519 
9520 	uint64_t m = 0;
9521 	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
9522 		vdev_t *vd = rvd->vdev_child[c];
9523 		if (!vdev_is_concrete(vd))
9524 			continue;
9525 		m += vd->vdev_ms_count;
9526 	}
9527 	return (m);
9528 }
9529 
9530 /*
9531  * Notify any waiting threads that some activity has switched from being in-
9532  * progress to not-in-progress so that the thread can wake up and determine
9533  * whether it is finished waiting.
9534  */
9535 void
9536 spa_notify_waiters(spa_t *spa)
9537 {
9538 	/*
9539 	 * Acquiring spa_activities_lock here prevents the cv_broadcast from
9540 	 * happening between the waiting thread's check and cv_wait.
9541 	 */
9542 	mutex_enter(&spa->spa_activities_lock);
9543 	cv_broadcast(&spa->spa_activities_cv);
9544 	mutex_exit(&spa->spa_activities_lock);
9545 }
9546 
9547 /*
9548  * Notify any waiting threads that the pool is exporting, and then block until
9549  * they are finished using the spa_t.
9550  */
9551 void
9552 spa_wake_waiters(spa_t *spa)
9553 {
9554 	mutex_enter(&spa->spa_activities_lock);
9555 	spa->spa_waiters_cancel = B_TRUE;
9556 	cv_broadcast(&spa->spa_activities_cv);
9557 	while (spa->spa_waiters != 0)
9558 		cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock);
9559 	spa->spa_waiters_cancel = B_FALSE;
9560 	mutex_exit(&spa->spa_activities_lock);
9561 }
9562 
9563 /* Whether the vdev or any of its descendants are being initialized/trimmed. */
9564 static boolean_t
9565 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity)
9566 {
9567 	spa_t *spa = vd->vdev_spa;
9568 
9569 	ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER));
9570 	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
9571 	ASSERT(activity == ZPOOL_WAIT_INITIALIZE ||
9572 	    activity == ZPOOL_WAIT_TRIM);
9573 
9574 	kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ?
9575 	    &vd->vdev_initialize_lock : &vd->vdev_trim_lock;
9576 
9577 	mutex_exit(&spa->spa_activities_lock);
9578 	mutex_enter(lock);
9579 	mutex_enter(&spa->spa_activities_lock);
9580 
9581 	boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ?
9582 	    (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) :
9583 	    (vd->vdev_trim_state == VDEV_TRIM_ACTIVE);
9584 	mutex_exit(lock);
9585 
9586 	if (in_progress)
9587 		return (B_TRUE);
9588 
9589 	for (int i = 0; i < vd->vdev_children; i++) {
9590 		if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i],
9591 		    activity))
9592 			return (B_TRUE);
9593 	}
9594 
9595 	return (B_FALSE);
9596 }
9597 
9598 /*
9599  * If use_guid is true, this checks whether the vdev specified by guid is
9600  * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool
9601  * is being initialized/trimmed. The caller must hold the config lock and
9602  * spa_activities_lock.
9603  */
9604 static int
9605 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid,
9606     zpool_wait_activity_t activity, boolean_t *in_progress)
9607 {
9608 	mutex_exit(&spa->spa_activities_lock);
9609 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
9610 	mutex_enter(&spa->spa_activities_lock);
9611 
9612 	vdev_t *vd;
9613 	if (use_guid) {
9614 		vd = spa_lookup_by_guid(spa, guid, B_FALSE);
9615 		if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) {
9616 			spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9617 			return (EINVAL);
9618 		}
9619 	} else {
9620 		vd = spa->spa_root_vdev;
9621 	}
9622 
9623 	*in_progress = spa_vdev_activity_in_progress_impl(vd, activity);
9624 
9625 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9626 	return (0);
9627 }
9628 
9629 /*
9630  * Locking for waiting threads
9631  * ---------------------------
9632  *
9633  * Waiting threads need a way to check whether a given activity is in progress,
9634  * and then, if it is, wait for it to complete. Each activity will have some
9635  * in-memory representation of the relevant on-disk state which can be used to
9636  * determine whether or not the activity is in progress. The in-memory state and
9637  * the locking used to protect it will be different for each activity, and may
9638  * not be suitable for use with a cvar (e.g., some state is protected by the
9639  * config lock). To allow waiting threads to wait without any races, another
9640  * lock, spa_activities_lock, is used.
9641  *
9642  * When the state is checked, both the activity-specific lock (if there is one)
9643  * and spa_activities_lock are held. In some cases, the activity-specific lock
9644  * is acquired explicitly (e.g. the config lock). In others, the locking is
9645  * internal to some check (e.g. bpobj_is_empty). After checking, the waiting
9646  * thread releases the activity-specific lock and, if the activity is in
9647  * progress, then cv_waits using spa_activities_lock.
9648  *
9649  * The waiting thread is woken when another thread, one completing some
9650  * activity, updates the state of the activity and then calls
9651  * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only
9652  * needs to hold its activity-specific lock when updating the state, and this
9653  * lock can (but doesn't have to) be dropped before calling spa_notify_waiters.
9654  *
9655  * Because spa_notify_waiters acquires spa_activities_lock before broadcasting,
9656  * and because it is held when the waiting thread checks the state of the
9657  * activity, it can never be the case that the completing thread both updates
9658  * the activity state and cv_broadcasts in between the waiting thread's check
9659  * and cv_wait. Thus, a waiting thread can never miss a wakeup.
9660  *
9661  * In order to prevent deadlock, when the waiting thread does its check, in some
9662  * cases it will temporarily drop spa_activities_lock in order to acquire the
9663  * activity-specific lock. The order in which spa_activities_lock and the
9664  * activity specific lock are acquired in the waiting thread is determined by
9665  * the order in which they are acquired in the completing thread; if the
9666  * completing thread calls spa_notify_waiters with the activity-specific lock
9667  * held, then the waiting thread must also acquire the activity-specific lock
9668  * first.
9669  */
9670 
9671 static int
9672 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity,
9673     boolean_t use_tag, uint64_t tag, boolean_t *in_progress)
9674 {
9675 	int error = 0;
9676 
9677 	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
9678 
9679 	switch (activity) {
9680 	case ZPOOL_WAIT_CKPT_DISCARD:
9681 		*in_progress =
9682 		    (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) &&
9683 		    zap_contains(spa_meta_objset(spa),
9684 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) ==
9685 		    ENOENT);
9686 		break;
9687 	case ZPOOL_WAIT_FREE:
9688 		*in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS &&
9689 		    !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) ||
9690 		    spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) ||
9691 		    spa_livelist_delete_check(spa));
9692 		break;
9693 	case ZPOOL_WAIT_INITIALIZE:
9694 	case ZPOOL_WAIT_TRIM:
9695 		error = spa_vdev_activity_in_progress(spa, use_tag, tag,
9696 		    activity, in_progress);
9697 		break;
9698 	case ZPOOL_WAIT_REPLACE:
9699 		mutex_exit(&spa->spa_activities_lock);
9700 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
9701 		mutex_enter(&spa->spa_activities_lock);
9702 
9703 		*in_progress = vdev_replace_in_progress(spa->spa_root_vdev);
9704 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9705 		break;
9706 	case ZPOOL_WAIT_REMOVE:
9707 		*in_progress = (spa->spa_removing_phys.sr_state ==
9708 		    DSS_SCANNING);
9709 		break;
9710 	case ZPOOL_WAIT_RESILVER:
9711 		if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev)))
9712 			break;
9713 		fallthrough;
9714 	case ZPOOL_WAIT_SCRUB:
9715 	{
9716 		boolean_t scanning, paused, is_scrub;
9717 		dsl_scan_t *scn =  spa->spa_dsl_pool->dp_scan;
9718 
9719 		is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB);
9720 		scanning = (scn->scn_phys.scn_state == DSS_SCANNING);
9721 		paused = dsl_scan_is_paused_scrub(scn);
9722 		*in_progress = (scanning && !paused &&
9723 		    is_scrub == (activity == ZPOOL_WAIT_SCRUB));
9724 		break;
9725 	}
9726 	default:
9727 		panic("unrecognized value for activity %d", activity);
9728 	}
9729 
9730 	return (error);
9731 }
9732 
9733 static int
9734 spa_wait_common(const char *pool, zpool_wait_activity_t activity,
9735     boolean_t use_tag, uint64_t tag, boolean_t *waited)
9736 {
9737 	/*
9738 	 * The tag is used to distinguish between instances of an activity.
9739 	 * 'initialize' and 'trim' are the only activities that we use this for.
9740 	 * The other activities can only have a single instance in progress in a
9741 	 * pool at one time, making the tag unnecessary.
9742 	 *
9743 	 * There can be multiple devices being replaced at once, but since they
9744 	 * all finish once resilvering finishes, we don't bother keeping track
9745 	 * of them individually, we just wait for them all to finish.
9746 	 */
9747 	if (use_tag && activity != ZPOOL_WAIT_INITIALIZE &&
9748 	    activity != ZPOOL_WAIT_TRIM)
9749 		return (EINVAL);
9750 
9751 	if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES)
9752 		return (EINVAL);
9753 
9754 	spa_t *spa;
9755 	int error = spa_open(pool, &spa, FTAG);
9756 	if (error != 0)
9757 		return (error);
9758 
9759 	/*
9760 	 * Increment the spa's waiter count so that we can call spa_close and
9761 	 * still ensure that the spa_t doesn't get freed before this thread is
9762 	 * finished with it when the pool is exported. We want to call spa_close
9763 	 * before we start waiting because otherwise the additional ref would
9764 	 * prevent the pool from being exported or destroyed throughout the
9765 	 * potentially long wait.
9766 	 */
9767 	mutex_enter(&spa->spa_activities_lock);
9768 	spa->spa_waiters++;
9769 	spa_close(spa, FTAG);
9770 
9771 	*waited = B_FALSE;
9772 	for (;;) {
9773 		boolean_t in_progress;
9774 		error = spa_activity_in_progress(spa, activity, use_tag, tag,
9775 		    &in_progress);
9776 
9777 		if (error || !in_progress || spa->spa_waiters_cancel)
9778 			break;
9779 
9780 		*waited = B_TRUE;
9781 
9782 		if (cv_wait_sig(&spa->spa_activities_cv,
9783 		    &spa->spa_activities_lock) == 0) {
9784 			error = EINTR;
9785 			break;
9786 		}
9787 	}
9788 
9789 	spa->spa_waiters--;
9790 	cv_signal(&spa->spa_waiters_cv);
9791 	mutex_exit(&spa->spa_activities_lock);
9792 
9793 	return (error);
9794 }
9795 
9796 /*
9797  * Wait for a particular instance of the specified activity to complete, where
9798  * the instance is identified by 'tag'
9799  */
9800 int
9801 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag,
9802     boolean_t *waited)
9803 {
9804 	return (spa_wait_common(pool, activity, B_TRUE, tag, waited));
9805 }
9806 
9807 /*
9808  * Wait for all instances of the specified activity complete
9809  */
9810 int
9811 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited)
9812 {
9813 
9814 	return (spa_wait_common(pool, activity, B_FALSE, 0, waited));
9815 }
9816 
9817 sysevent_t *
9818 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name)
9819 {
9820 	sysevent_t *ev = NULL;
9821 #ifdef _KERNEL
9822 	nvlist_t *resource;
9823 
9824 	resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl);
9825 	if (resource) {
9826 		ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP);
9827 		ev->resource = resource;
9828 	}
9829 #endif
9830 	return (ev);
9831 }
9832 
9833 void
9834 spa_event_post(sysevent_t *ev)
9835 {
9836 #ifdef _KERNEL
9837 	if (ev) {
9838 		zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb);
9839 		kmem_free(ev, sizeof (*ev));
9840 	}
9841 #endif
9842 }
9843 
9844 /*
9845  * Post a zevent corresponding to the given sysevent.   The 'name' must be one
9846  * of the event definitions in sys/sysevent/eventdefs.h.  The payload will be
9847  * filled in from the spa and (optionally) the vdev.  This doesn't do anything
9848  * in the userland libzpool, as we don't want consumers to misinterpret ztest
9849  * or zdb as real changes.
9850  */
9851 void
9852 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name)
9853 {
9854 	spa_event_post(spa_event_create(spa, vd, hist_nvl, name));
9855 }
9856 
9857 /* state manipulation functions */
9858 EXPORT_SYMBOL(spa_open);
9859 EXPORT_SYMBOL(spa_open_rewind);
9860 EXPORT_SYMBOL(spa_get_stats);
9861 EXPORT_SYMBOL(spa_create);
9862 EXPORT_SYMBOL(spa_import);
9863 EXPORT_SYMBOL(spa_tryimport);
9864 EXPORT_SYMBOL(spa_destroy);
9865 EXPORT_SYMBOL(spa_export);
9866 EXPORT_SYMBOL(spa_reset);
9867 EXPORT_SYMBOL(spa_async_request);
9868 EXPORT_SYMBOL(spa_async_suspend);
9869 EXPORT_SYMBOL(spa_async_resume);
9870 EXPORT_SYMBOL(spa_inject_addref);
9871 EXPORT_SYMBOL(spa_inject_delref);
9872 EXPORT_SYMBOL(spa_scan_stat_init);
9873 EXPORT_SYMBOL(spa_scan_get_stats);
9874 
9875 /* device manipulation */
9876 EXPORT_SYMBOL(spa_vdev_add);
9877 EXPORT_SYMBOL(spa_vdev_attach);
9878 EXPORT_SYMBOL(spa_vdev_detach);
9879 EXPORT_SYMBOL(spa_vdev_setpath);
9880 EXPORT_SYMBOL(spa_vdev_setfru);
9881 EXPORT_SYMBOL(spa_vdev_split_mirror);
9882 
9883 /* spare statech is global across all pools) */
9884 EXPORT_SYMBOL(spa_spare_add);
9885 EXPORT_SYMBOL(spa_spare_remove);
9886 EXPORT_SYMBOL(spa_spare_exists);
9887 EXPORT_SYMBOL(spa_spare_activate);
9888 
9889 /* L2ARC statech is global across all pools) */
9890 EXPORT_SYMBOL(spa_l2cache_add);
9891 EXPORT_SYMBOL(spa_l2cache_remove);
9892 EXPORT_SYMBOL(spa_l2cache_exists);
9893 EXPORT_SYMBOL(spa_l2cache_activate);
9894 EXPORT_SYMBOL(spa_l2cache_drop);
9895 
9896 /* scanning */
9897 EXPORT_SYMBOL(spa_scan);
9898 EXPORT_SYMBOL(spa_scan_stop);
9899 
9900 /* spa syncing */
9901 EXPORT_SYMBOL(spa_sync); /* only for DMU use */
9902 EXPORT_SYMBOL(spa_sync_allpools);
9903 
9904 /* properties */
9905 EXPORT_SYMBOL(spa_prop_set);
9906 EXPORT_SYMBOL(spa_prop_get);
9907 EXPORT_SYMBOL(spa_prop_clear_bootfs);
9908 
9909 /* asynchronous event notification */
9910 EXPORT_SYMBOL(spa_event_notify);
9911 
9912 /* BEGIN CSTYLED */
9913 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, INT, ZMOD_RW,
9914 	"log2 fraction of arc that can be used by inflight I/Os when "
9915 	"verifying pool during import");
9916 
9917 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW,
9918 	"Set to traverse metadata on pool import");
9919 
9920 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW,
9921 	"Set to traverse data on pool import");
9922 
9923 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW,
9924 	"Print vdev tree to zfs_dbgmsg during pool import");
9925 
9926 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD,
9927 	"Percentage of CPUs to run an IO worker thread");
9928 
9929 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD,
9930 	"Number of threads per IO worker taskqueue");
9931 
9932 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, ULONG, ZMOD_RW,
9933 	"Allow importing pool with up to this number of missing top-level "
9934 	"vdevs (in read-only mode)");
9935 
9936 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, ZMOD_RW,
9937 	"Set the livelist condense zthr to pause");
9938 
9939 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, ZMOD_RW,
9940 	"Set the livelist condense synctask to pause");
9941 
9942 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, INT, ZMOD_RW,
9943 	"Whether livelist condensing was canceled in the synctask");
9944 
9945 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, INT, ZMOD_RW,
9946 	"Whether livelist condensing was canceled in the zthr function");
9947 
9948 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, ZMOD_RW,
9949 	"Whether extra ALLOC blkptrs were added to a livelist entry while it "
9950 	"was being condensed");
9951 /* END CSTYLED */
9952