xref: /freebsd/sys/contrib/openzfs/module/zfs/spa.c (revision f126890ac5386406dadf7c4cfa9566cbb56537c5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
25  * Copyright (c) 2018, Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27  * Copyright 2013 Saso Kiselkov. All rights reserved.
28  * Copyright (c) 2014 Integros [integros.com]
29  * Copyright 2016 Toomas Soome <tsoome@me.com>
30  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
31  * Copyright 2018 Joyent, Inc.
32  * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
33  * Copyright 2017 Joyent, Inc.
34  * Copyright (c) 2017, Intel Corporation.
35  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
36  * Copyright (c) 2023 Hewlett Packard Enterprise Development LP.
37  */
38 
39 /*
40  * SPA: Storage Pool Allocator
41  *
42  * This file contains all the routines used when modifying on-disk SPA state.
43  * This includes opening, importing, destroying, exporting a pool, and syncing a
44  * pool.
45  */
46 
47 #include <sys/zfs_context.h>
48 #include <sys/fm/fs/zfs.h>
49 #include <sys/spa_impl.h>
50 #include <sys/zio.h>
51 #include <sys/zio_checksum.h>
52 #include <sys/dmu.h>
53 #include <sys/dmu_tx.h>
54 #include <sys/zap.h>
55 #include <sys/zil.h>
56 #include <sys/brt.h>
57 #include <sys/ddt.h>
58 #include <sys/vdev_impl.h>
59 #include <sys/vdev_removal.h>
60 #include <sys/vdev_indirect_mapping.h>
61 #include <sys/vdev_indirect_births.h>
62 #include <sys/vdev_initialize.h>
63 #include <sys/vdev_rebuild.h>
64 #include <sys/vdev_trim.h>
65 #include <sys/vdev_disk.h>
66 #include <sys/vdev_raidz.h>
67 #include <sys/vdev_draid.h>
68 #include <sys/metaslab.h>
69 #include <sys/metaslab_impl.h>
70 #include <sys/mmp.h>
71 #include <sys/uberblock_impl.h>
72 #include <sys/txg.h>
73 #include <sys/avl.h>
74 #include <sys/bpobj.h>
75 #include <sys/dmu_traverse.h>
76 #include <sys/dmu_objset.h>
77 #include <sys/unique.h>
78 #include <sys/dsl_pool.h>
79 #include <sys/dsl_dataset.h>
80 #include <sys/dsl_dir.h>
81 #include <sys/dsl_prop.h>
82 #include <sys/dsl_synctask.h>
83 #include <sys/fs/zfs.h>
84 #include <sys/arc.h>
85 #include <sys/callb.h>
86 #include <sys/systeminfo.h>
87 #include <sys/zfs_ioctl.h>
88 #include <sys/dsl_scan.h>
89 #include <sys/zfeature.h>
90 #include <sys/dsl_destroy.h>
91 #include <sys/zvol.h>
92 
93 #ifdef	_KERNEL
94 #include <sys/fm/protocol.h>
95 #include <sys/fm/util.h>
96 #include <sys/callb.h>
97 #include <sys/zone.h>
98 #include <sys/vmsystm.h>
99 #endif	/* _KERNEL */
100 
101 #include "zfs_prop.h"
102 #include "zfs_comutil.h"
103 #include <cityhash.h>
104 
105 /*
106  * spa_thread() existed on Illumos as a parent thread for the various worker
107  * threads that actually run the pool, as a way to both reference the entire
108  * pool work as a single object, and to share properties like scheduling
109  * options. It has not yet been adapted to Linux or FreeBSD. This define is
110  * used to mark related parts of the code to make things easier for the reader,
111  * and to compile this code out. It can be removed when someone implements it,
112  * moves it to some Illumos-specific place, or removes it entirely.
113  */
114 #undef HAVE_SPA_THREAD
115 
116 /*
117  * The "System Duty Cycle" scheduling class is an Illumos feature to help
118  * prevent CPU-intensive kernel threads from affecting latency on interactive
119  * threads. It doesn't exist on Linux or FreeBSD, so the supporting code is
120  * gated behind a define. On Illumos SDC depends on spa_thread(), but
121  * spa_thread() also has other uses, so this is a separate define.
122  */
123 #undef HAVE_SYSDC
124 
125 /*
126  * The interval, in seconds, at which failed configuration cache file writes
127  * should be retried.
128  */
129 int zfs_ccw_retry_interval = 300;
130 
131 typedef enum zti_modes {
132 	ZTI_MODE_FIXED,			/* value is # of threads (min 1) */
133 	ZTI_MODE_SCALE,			/* Taskqs scale with CPUs. */
134 	ZTI_MODE_SYNC,			/* sync thread assigned */
135 	ZTI_MODE_NULL,			/* don't create a taskq */
136 	ZTI_NMODES
137 } zti_modes_t;
138 
139 #define	ZTI_P(n, q)	{ ZTI_MODE_FIXED, (n), (q) }
140 #define	ZTI_PCT(n)	{ ZTI_MODE_ONLINE_PERCENT, (n), 1 }
141 #define	ZTI_SCALE	{ ZTI_MODE_SCALE, 0, 1 }
142 #define	ZTI_SYNC	{ ZTI_MODE_SYNC, 0, 1 }
143 #define	ZTI_NULL	{ ZTI_MODE_NULL, 0, 0 }
144 
145 #define	ZTI_N(n)	ZTI_P(n, 1)
146 #define	ZTI_ONE		ZTI_N(1)
147 
148 typedef struct zio_taskq_info {
149 	zti_modes_t zti_mode;
150 	uint_t zti_value;
151 	uint_t zti_count;
152 } zio_taskq_info_t;
153 
154 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = {
155 	"iss", "iss_h", "int", "int_h"
156 };
157 
158 /*
159  * This table defines the taskq settings for each ZFS I/O type. When
160  * initializing a pool, we use this table to create an appropriately sized
161  * taskq. Some operations are low volume and therefore have a small, static
162  * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE
163  * macros. Other operations process a large amount of data; the ZTI_SCALE
164  * macro causes us to create a taskq oriented for throughput. Some operations
165  * are so high frequency and short-lived that the taskq itself can become a
166  * point of lock contention. The ZTI_P(#, #) macro indicates that we need an
167  * additional degree of parallelism specified by the number of threads per-
168  * taskq and the number of taskqs; when dispatching an event in this case, the
169  * particular taskq is chosen at random. ZTI_SCALE uses a number of taskqs
170  * that scales with the number of CPUs.
171  *
172  * The different taskq priorities are to handle the different contexts (issue
173  * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that
174  * need to be handled with minimum delay.
175  */
176 static zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = {
177 	/* ISSUE	ISSUE_HIGH	INTR		INTR_HIGH */
178 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* NULL */
179 	{ ZTI_N(8),	ZTI_NULL,	ZTI_SCALE,	ZTI_NULL }, /* READ */
180 	{ ZTI_SYNC,	ZTI_N(5),	ZTI_SCALE,	ZTI_N(5) }, /* WRITE */
181 	{ ZTI_SCALE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* FREE */
182 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* CLAIM */
183 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* IOCTL */
184 	{ ZTI_N(4),	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* TRIM */
185 };
186 
187 static void spa_sync_version(void *arg, dmu_tx_t *tx);
188 static void spa_sync_props(void *arg, dmu_tx_t *tx);
189 static boolean_t spa_has_active_shared_spare(spa_t *spa);
190 static int spa_load_impl(spa_t *spa, spa_import_type_t type,
191     const char **ereport);
192 static void spa_vdev_resilver_done(spa_t *spa);
193 
194 /*
195  * Percentage of all CPUs that can be used by the metaslab preload taskq.
196  */
197 static uint_t metaslab_preload_pct = 50;
198 
199 static uint_t	zio_taskq_batch_pct = 80;	  /* 1 thread per cpu in pset */
200 static uint_t	zio_taskq_batch_tpq;		  /* threads per taskq */
201 
202 #ifdef HAVE_SYSDC
203 static const boolean_t	zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */
204 static const uint_t	zio_taskq_basedc = 80;	  /* base duty cycle */
205 #endif
206 
207 #ifdef HAVE_SPA_THREAD
208 static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */
209 #endif
210 
211 static uint_t	zio_taskq_wr_iss_ncpus = 0;
212 
213 /*
214  * Report any spa_load_verify errors found, but do not fail spa_load.
215  * This is used by zdb to analyze non-idle pools.
216  */
217 boolean_t	spa_load_verify_dryrun = B_FALSE;
218 
219 /*
220  * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ).
221  * This is used by zdb for spacemaps verification.
222  */
223 boolean_t	spa_mode_readable_spacemaps = B_FALSE;
224 
225 /*
226  * This (illegal) pool name is used when temporarily importing a spa_t in order
227  * to get the vdev stats associated with the imported devices.
228  */
229 #define	TRYIMPORT_NAME	"$import"
230 
231 /*
232  * For debugging purposes: print out vdev tree during pool import.
233  */
234 static int		spa_load_print_vdev_tree = B_FALSE;
235 
236 /*
237  * A non-zero value for zfs_max_missing_tvds means that we allow importing
238  * pools with missing top-level vdevs. This is strictly intended for advanced
239  * pool recovery cases since missing data is almost inevitable. Pools with
240  * missing devices can only be imported read-only for safety reasons, and their
241  * fail-mode will be automatically set to "continue".
242  *
243  * With 1 missing vdev we should be able to import the pool and mount all
244  * datasets. User data that was not modified after the missing device has been
245  * added should be recoverable. This means that snapshots created prior to the
246  * addition of that device should be completely intact.
247  *
248  * With 2 missing vdevs, some datasets may fail to mount since there are
249  * dataset statistics that are stored as regular metadata. Some data might be
250  * recoverable if those vdevs were added recently.
251  *
252  * With 3 or more missing vdevs, the pool is severely damaged and MOS entries
253  * may be missing entirely. Chances of data recovery are very low. Note that
254  * there are also risks of performing an inadvertent rewind as we might be
255  * missing all the vdevs with the latest uberblocks.
256  */
257 uint64_t	zfs_max_missing_tvds = 0;
258 
259 /*
260  * The parameters below are similar to zfs_max_missing_tvds but are only
261  * intended for a preliminary open of the pool with an untrusted config which
262  * might be incomplete or out-dated.
263  *
264  * We are more tolerant for pools opened from a cachefile since we could have
265  * an out-dated cachefile where a device removal was not registered.
266  * We could have set the limit arbitrarily high but in the case where devices
267  * are really missing we would want to return the proper error codes; we chose
268  * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available
269  * and we get a chance to retrieve the trusted config.
270  */
271 uint64_t	zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1;
272 
273 /*
274  * In the case where config was assembled by scanning device paths (/dev/dsks
275  * by default) we are less tolerant since all the existing devices should have
276  * been detected and we want spa_load to return the right error codes.
277  */
278 uint64_t	zfs_max_missing_tvds_scan = 0;
279 
280 /*
281  * Debugging aid that pauses spa_sync() towards the end.
282  */
283 static const boolean_t	zfs_pause_spa_sync = B_FALSE;
284 
285 /*
286  * Variables to indicate the livelist condense zthr func should wait at certain
287  * points for the livelist to be removed - used to test condense/destroy races
288  */
289 static int zfs_livelist_condense_zthr_pause = 0;
290 static int zfs_livelist_condense_sync_pause = 0;
291 
292 /*
293  * Variables to track whether or not condense cancellation has been
294  * triggered in testing.
295  */
296 static int zfs_livelist_condense_sync_cancel = 0;
297 static int zfs_livelist_condense_zthr_cancel = 0;
298 
299 /*
300  * Variable to track whether or not extra ALLOC blkptrs were added to a
301  * livelist entry while it was being condensed (caused by the way we track
302  * remapped blkptrs in dbuf_remap_impl)
303  */
304 static int zfs_livelist_condense_new_alloc = 0;
305 
306 /*
307  * ==========================================================================
308  * SPA properties routines
309  * ==========================================================================
310  */
311 
312 /*
313  * Add a (source=src, propname=propval) list to an nvlist.
314  */
315 static void
316 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval,
317     uint64_t intval, zprop_source_t src)
318 {
319 	const char *propname = zpool_prop_to_name(prop);
320 	nvlist_t *propval;
321 
322 	propval = fnvlist_alloc();
323 	fnvlist_add_uint64(propval, ZPROP_SOURCE, src);
324 
325 	if (strval != NULL)
326 		fnvlist_add_string(propval, ZPROP_VALUE, strval);
327 	else
328 		fnvlist_add_uint64(propval, ZPROP_VALUE, intval);
329 
330 	fnvlist_add_nvlist(nvl, propname, propval);
331 	nvlist_free(propval);
332 }
333 
334 /*
335  * Add a user property (source=src, propname=propval) to an nvlist.
336  */
337 static void
338 spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval,
339     zprop_source_t src)
340 {
341 	nvlist_t *propval;
342 
343 	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
344 	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
345 	VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
346 	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
347 	nvlist_free(propval);
348 }
349 
350 /*
351  * Get property values from the spa configuration.
352  */
353 static void
354 spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
355 {
356 	vdev_t *rvd = spa->spa_root_vdev;
357 	dsl_pool_t *pool = spa->spa_dsl_pool;
358 	uint64_t size, alloc, cap, version;
359 	const zprop_source_t src = ZPROP_SRC_NONE;
360 	spa_config_dirent_t *dp;
361 	metaslab_class_t *mc = spa_normal_class(spa);
362 
363 	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
364 
365 	if (rvd != NULL) {
366 		alloc = metaslab_class_get_alloc(mc);
367 		alloc += metaslab_class_get_alloc(spa_special_class(spa));
368 		alloc += metaslab_class_get_alloc(spa_dedup_class(spa));
369 		alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa));
370 
371 		size = metaslab_class_get_space(mc);
372 		size += metaslab_class_get_space(spa_special_class(spa));
373 		size += metaslab_class_get_space(spa_dedup_class(spa));
374 		size += metaslab_class_get_space(spa_embedded_log_class(spa));
375 
376 		spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
377 		spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src);
378 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
379 		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
380 		    size - alloc, src);
381 		spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL,
382 		    spa->spa_checkpoint_info.sci_dspace, src);
383 
384 		spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
385 		    metaslab_class_fragmentation(mc), src);
386 		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
387 		    metaslab_class_expandable_space(mc), src);
388 		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
389 		    (spa_mode(spa) == SPA_MODE_READ), src);
390 
391 		cap = (size == 0) ? 0 : (alloc * 100 / size);
392 		spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src);
393 
394 		spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
395 		    ddt_get_pool_dedup_ratio(spa), src);
396 		spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONEUSED, NULL,
397 		    brt_get_used(spa), src);
398 		spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONESAVED, NULL,
399 		    brt_get_saved(spa), src);
400 		spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONERATIO, NULL,
401 		    brt_get_ratio(spa), src);
402 
403 		spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
404 		    rvd->vdev_state, src);
405 
406 		version = spa_version(spa);
407 		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) {
408 			spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
409 			    version, ZPROP_SRC_DEFAULT);
410 		} else {
411 			spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL,
412 			    version, ZPROP_SRC_LOCAL);
413 		}
414 		spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID,
415 		    NULL, spa_load_guid(spa), src);
416 	}
417 
418 	if (pool != NULL) {
419 		/*
420 		 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
421 		 * when opening pools before this version freedir will be NULL.
422 		 */
423 		if (pool->dp_free_dir != NULL) {
424 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
425 			    dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
426 			    src);
427 		} else {
428 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
429 			    NULL, 0, src);
430 		}
431 
432 		if (pool->dp_leak_dir != NULL) {
433 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
434 			    dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
435 			    src);
436 		} else {
437 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
438 			    NULL, 0, src);
439 		}
440 	}
441 
442 	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);
443 
444 	if (spa->spa_comment != NULL) {
445 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment,
446 		    0, ZPROP_SRC_LOCAL);
447 	}
448 
449 	if (spa->spa_compatibility != NULL) {
450 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY,
451 		    spa->spa_compatibility, 0, ZPROP_SRC_LOCAL);
452 	}
453 
454 	if (spa->spa_root != NULL)
455 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
456 		    0, ZPROP_SRC_LOCAL);
457 
458 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
459 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
460 		    MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
461 	} else {
462 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
463 		    SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
464 	}
465 
466 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) {
467 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
468 		    DNODE_MAX_SIZE, ZPROP_SRC_NONE);
469 	} else {
470 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL,
471 		    DNODE_MIN_SIZE, ZPROP_SRC_NONE);
472 	}
473 
474 	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
475 		if (dp->scd_path == NULL) {
476 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
477 			    "none", 0, ZPROP_SRC_LOCAL);
478 		} else if (strcmp(dp->scd_path, spa_config_path) != 0) {
479 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
480 			    dp->scd_path, 0, ZPROP_SRC_LOCAL);
481 		}
482 	}
483 }
484 
485 /*
486  * Get zpool property values.
487  */
488 int
489 spa_prop_get(spa_t *spa, nvlist_t **nvp)
490 {
491 	objset_t *mos = spa->spa_meta_objset;
492 	zap_cursor_t zc;
493 	zap_attribute_t za;
494 	dsl_pool_t *dp;
495 	int err;
496 
497 	err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP);
498 	if (err)
499 		return (err);
500 
501 	dp = spa_get_dsl(spa);
502 	dsl_pool_config_enter(dp, FTAG);
503 	mutex_enter(&spa->spa_props_lock);
504 
505 	/*
506 	 * Get properties from the spa config.
507 	 */
508 	spa_prop_get_config(spa, nvp);
509 
510 	/* If no pool property object, no more prop to get. */
511 	if (mos == NULL || spa->spa_pool_props_object == 0)
512 		goto out;
513 
514 	/*
515 	 * Get properties from the MOS pool property object.
516 	 */
517 	for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object);
518 	    (err = zap_cursor_retrieve(&zc, &za)) == 0;
519 	    zap_cursor_advance(&zc)) {
520 		uint64_t intval = 0;
521 		char *strval = NULL;
522 		zprop_source_t src = ZPROP_SRC_DEFAULT;
523 		zpool_prop_t prop;
524 
525 		if ((prop = zpool_name_to_prop(za.za_name)) ==
526 		    ZPOOL_PROP_INVAL && !zfs_prop_user(za.za_name))
527 			continue;
528 
529 		switch (za.za_integer_length) {
530 		case 8:
531 			/* integer property */
532 			if (za.za_first_integer !=
533 			    zpool_prop_default_numeric(prop))
534 				src = ZPROP_SRC_LOCAL;
535 
536 			if (prop == ZPOOL_PROP_BOOTFS) {
537 				dsl_dataset_t *ds = NULL;
538 
539 				err = dsl_dataset_hold_obj(dp,
540 				    za.za_first_integer, FTAG, &ds);
541 				if (err != 0)
542 					break;
543 
544 				strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN,
545 				    KM_SLEEP);
546 				dsl_dataset_name(ds, strval);
547 				dsl_dataset_rele(ds, FTAG);
548 			} else {
549 				strval = NULL;
550 				intval = za.za_first_integer;
551 			}
552 
553 			spa_prop_add_list(*nvp, prop, strval, intval, src);
554 
555 			if (strval != NULL)
556 				kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN);
557 
558 			break;
559 
560 		case 1:
561 			/* string property */
562 			strval = kmem_alloc(za.za_num_integers, KM_SLEEP);
563 			err = zap_lookup(mos, spa->spa_pool_props_object,
564 			    za.za_name, 1, za.za_num_integers, strval);
565 			if (err) {
566 				kmem_free(strval, za.za_num_integers);
567 				break;
568 			}
569 			if (prop != ZPOOL_PROP_INVAL) {
570 				spa_prop_add_list(*nvp, prop, strval, 0, src);
571 			} else {
572 				src = ZPROP_SRC_LOCAL;
573 				spa_prop_add_user(*nvp, za.za_name, strval,
574 				    src);
575 			}
576 			kmem_free(strval, za.za_num_integers);
577 			break;
578 
579 		default:
580 			break;
581 		}
582 	}
583 	zap_cursor_fini(&zc);
584 out:
585 	mutex_exit(&spa->spa_props_lock);
586 	dsl_pool_config_exit(dp, FTAG);
587 	if (err && err != ENOENT) {
588 		nvlist_free(*nvp);
589 		*nvp = NULL;
590 		return (err);
591 	}
592 
593 	return (0);
594 }
595 
596 /*
597  * Validate the given pool properties nvlist and modify the list
598  * for the property values to be set.
599  */
600 static int
601 spa_prop_validate(spa_t *spa, nvlist_t *props)
602 {
603 	nvpair_t *elem;
604 	int error = 0, reset_bootfs = 0;
605 	uint64_t objnum = 0;
606 	boolean_t has_feature = B_FALSE;
607 
608 	elem = NULL;
609 	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
610 		uint64_t intval;
611 		const char *strval, *slash, *check, *fname;
612 		const char *propname = nvpair_name(elem);
613 		zpool_prop_t prop = zpool_name_to_prop(propname);
614 
615 		switch (prop) {
616 		case ZPOOL_PROP_INVAL:
617 			/*
618 			 * Sanitize the input.
619 			 */
620 			if (zfs_prop_user(propname)) {
621 				if (strlen(propname) >= ZAP_MAXNAMELEN) {
622 					error = SET_ERROR(ENAMETOOLONG);
623 					break;
624 				}
625 
626 				if (strlen(fnvpair_value_string(elem)) >=
627 				    ZAP_MAXVALUELEN) {
628 					error = SET_ERROR(E2BIG);
629 					break;
630 				}
631 			} else if (zpool_prop_feature(propname)) {
632 				if (nvpair_type(elem) != DATA_TYPE_UINT64) {
633 					error = SET_ERROR(EINVAL);
634 					break;
635 				}
636 
637 				if (nvpair_value_uint64(elem, &intval) != 0) {
638 					error = SET_ERROR(EINVAL);
639 					break;
640 				}
641 
642 				if (intval != 0) {
643 					error = SET_ERROR(EINVAL);
644 					break;
645 				}
646 
647 				fname = strchr(propname, '@') + 1;
648 				if (zfeature_lookup_name(fname, NULL) != 0) {
649 					error = SET_ERROR(EINVAL);
650 					break;
651 				}
652 
653 				has_feature = B_TRUE;
654 			} else {
655 				error = SET_ERROR(EINVAL);
656 				break;
657 			}
658 			break;
659 
660 		case ZPOOL_PROP_VERSION:
661 			error = nvpair_value_uint64(elem, &intval);
662 			if (!error &&
663 			    (intval < spa_version(spa) ||
664 			    intval > SPA_VERSION_BEFORE_FEATURES ||
665 			    has_feature))
666 				error = SET_ERROR(EINVAL);
667 			break;
668 
669 		case ZPOOL_PROP_DELEGATION:
670 		case ZPOOL_PROP_AUTOREPLACE:
671 		case ZPOOL_PROP_LISTSNAPS:
672 		case ZPOOL_PROP_AUTOEXPAND:
673 		case ZPOOL_PROP_AUTOTRIM:
674 			error = nvpair_value_uint64(elem, &intval);
675 			if (!error && intval > 1)
676 				error = SET_ERROR(EINVAL);
677 			break;
678 
679 		case ZPOOL_PROP_MULTIHOST:
680 			error = nvpair_value_uint64(elem, &intval);
681 			if (!error && intval > 1)
682 				error = SET_ERROR(EINVAL);
683 
684 			if (!error) {
685 				uint32_t hostid = zone_get_hostid(NULL);
686 				if (hostid)
687 					spa->spa_hostid = hostid;
688 				else
689 					error = SET_ERROR(ENOTSUP);
690 			}
691 
692 			break;
693 
694 		case ZPOOL_PROP_BOOTFS:
695 			/*
696 			 * If the pool version is less than SPA_VERSION_BOOTFS,
697 			 * or the pool is still being created (version == 0),
698 			 * the bootfs property cannot be set.
699 			 */
700 			if (spa_version(spa) < SPA_VERSION_BOOTFS) {
701 				error = SET_ERROR(ENOTSUP);
702 				break;
703 			}
704 
705 			/*
706 			 * Make sure the vdev config is bootable
707 			 */
708 			if (!vdev_is_bootable(spa->spa_root_vdev)) {
709 				error = SET_ERROR(ENOTSUP);
710 				break;
711 			}
712 
713 			reset_bootfs = 1;
714 
715 			error = nvpair_value_string(elem, &strval);
716 
717 			if (!error) {
718 				objset_t *os;
719 
720 				if (strval == NULL || strval[0] == '\0') {
721 					objnum = zpool_prop_default_numeric(
722 					    ZPOOL_PROP_BOOTFS);
723 					break;
724 				}
725 
726 				error = dmu_objset_hold(strval, FTAG, &os);
727 				if (error != 0)
728 					break;
729 
730 				/* Must be ZPL. */
731 				if (dmu_objset_type(os) != DMU_OST_ZFS) {
732 					error = SET_ERROR(ENOTSUP);
733 				} else {
734 					objnum = dmu_objset_id(os);
735 				}
736 				dmu_objset_rele(os, FTAG);
737 			}
738 			break;
739 
740 		case ZPOOL_PROP_FAILUREMODE:
741 			error = nvpair_value_uint64(elem, &intval);
742 			if (!error && intval > ZIO_FAILURE_MODE_PANIC)
743 				error = SET_ERROR(EINVAL);
744 
745 			/*
746 			 * This is a special case which only occurs when
747 			 * the pool has completely failed. This allows
748 			 * the user to change the in-core failmode property
749 			 * without syncing it out to disk (I/Os might
750 			 * currently be blocked). We do this by returning
751 			 * EIO to the caller (spa_prop_set) to trick it
752 			 * into thinking we encountered a property validation
753 			 * error.
754 			 */
755 			if (!error && spa_suspended(spa)) {
756 				spa->spa_failmode = intval;
757 				error = SET_ERROR(EIO);
758 			}
759 			break;
760 
761 		case ZPOOL_PROP_CACHEFILE:
762 			if ((error = nvpair_value_string(elem, &strval)) != 0)
763 				break;
764 
765 			if (strval[0] == '\0')
766 				break;
767 
768 			if (strcmp(strval, "none") == 0)
769 				break;
770 
771 			if (strval[0] != '/') {
772 				error = SET_ERROR(EINVAL);
773 				break;
774 			}
775 
776 			slash = strrchr(strval, '/');
777 			ASSERT(slash != NULL);
778 
779 			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
780 			    strcmp(slash, "/..") == 0)
781 				error = SET_ERROR(EINVAL);
782 			break;
783 
784 		case ZPOOL_PROP_COMMENT:
785 			if ((error = nvpair_value_string(elem, &strval)) != 0)
786 				break;
787 			for (check = strval; *check != '\0'; check++) {
788 				if (!isprint(*check)) {
789 					error = SET_ERROR(EINVAL);
790 					break;
791 				}
792 			}
793 			if (strlen(strval) > ZPROP_MAX_COMMENT)
794 				error = SET_ERROR(E2BIG);
795 			break;
796 
797 		default:
798 			break;
799 		}
800 
801 		if (error)
802 			break;
803 	}
804 
805 	(void) nvlist_remove_all(props,
806 	    zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO));
807 
808 	if (!error && reset_bootfs) {
809 		error = nvlist_remove(props,
810 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING);
811 
812 		if (!error) {
813 			error = nvlist_add_uint64(props,
814 			    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum);
815 		}
816 	}
817 
818 	return (error);
819 }
820 
821 void
822 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync)
823 {
824 	const char *cachefile;
825 	spa_config_dirent_t *dp;
826 
827 	if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
828 	    &cachefile) != 0)
829 		return;
830 
831 	dp = kmem_alloc(sizeof (spa_config_dirent_t),
832 	    KM_SLEEP);
833 
834 	if (cachefile[0] == '\0')
835 		dp->scd_path = spa_strdup(spa_config_path);
836 	else if (strcmp(cachefile, "none") == 0)
837 		dp->scd_path = NULL;
838 	else
839 		dp->scd_path = spa_strdup(cachefile);
840 
841 	list_insert_head(&spa->spa_config_list, dp);
842 	if (need_sync)
843 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
844 }
845 
846 int
847 spa_prop_set(spa_t *spa, nvlist_t *nvp)
848 {
849 	int error;
850 	nvpair_t *elem = NULL;
851 	boolean_t need_sync = B_FALSE;
852 
853 	if ((error = spa_prop_validate(spa, nvp)) != 0)
854 		return (error);
855 
856 	while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) {
857 		zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem));
858 
859 		if (prop == ZPOOL_PROP_CACHEFILE ||
860 		    prop == ZPOOL_PROP_ALTROOT ||
861 		    prop == ZPOOL_PROP_READONLY)
862 			continue;
863 
864 		if (prop == ZPOOL_PROP_INVAL &&
865 		    zfs_prop_user(nvpair_name(elem))) {
866 			need_sync = B_TRUE;
867 			break;
868 		}
869 
870 		if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) {
871 			uint64_t ver = 0;
872 
873 			if (prop == ZPOOL_PROP_VERSION) {
874 				VERIFY(nvpair_value_uint64(elem, &ver) == 0);
875 			} else {
876 				ASSERT(zpool_prop_feature(nvpair_name(elem)));
877 				ver = SPA_VERSION_FEATURES;
878 				need_sync = B_TRUE;
879 			}
880 
881 			/* Save time if the version is already set. */
882 			if (ver == spa_version(spa))
883 				continue;
884 
885 			/*
886 			 * In addition to the pool directory object, we might
887 			 * create the pool properties object, the features for
888 			 * read object, the features for write object, or the
889 			 * feature descriptions object.
890 			 */
891 			error = dsl_sync_task(spa->spa_name, NULL,
892 			    spa_sync_version, &ver,
893 			    6, ZFS_SPACE_CHECK_RESERVED);
894 			if (error)
895 				return (error);
896 			continue;
897 		}
898 
899 		need_sync = B_TRUE;
900 		break;
901 	}
902 
903 	if (need_sync) {
904 		return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
905 		    nvp, 6, ZFS_SPACE_CHECK_RESERVED));
906 	}
907 
908 	return (0);
909 }
910 
911 /*
912  * If the bootfs property value is dsobj, clear it.
913  */
914 void
915 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx)
916 {
917 	if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) {
918 		VERIFY(zap_remove(spa->spa_meta_objset,
919 		    spa->spa_pool_props_object,
920 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0);
921 		spa->spa_bootfs = 0;
922 	}
923 }
924 
925 static int
926 spa_change_guid_check(void *arg, dmu_tx_t *tx)
927 {
928 	uint64_t *newguid __maybe_unused = arg;
929 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
930 	vdev_t *rvd = spa->spa_root_vdev;
931 	uint64_t vdev_state;
932 
933 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
934 		int error = (spa_has_checkpoint(spa)) ?
935 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
936 		return (SET_ERROR(error));
937 	}
938 
939 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
940 	vdev_state = rvd->vdev_state;
941 	spa_config_exit(spa, SCL_STATE, FTAG);
942 
943 	if (vdev_state != VDEV_STATE_HEALTHY)
944 		return (SET_ERROR(ENXIO));
945 
946 	ASSERT3U(spa_guid(spa), !=, *newguid);
947 
948 	return (0);
949 }
950 
951 static void
952 spa_change_guid_sync(void *arg, dmu_tx_t *tx)
953 {
954 	uint64_t *newguid = arg;
955 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
956 	uint64_t oldguid;
957 	vdev_t *rvd = spa->spa_root_vdev;
958 
959 	oldguid = spa_guid(spa);
960 
961 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
962 	rvd->vdev_guid = *newguid;
963 	rvd->vdev_guid_sum += (*newguid - oldguid);
964 	vdev_config_dirty(rvd);
965 	spa_config_exit(spa, SCL_STATE, FTAG);
966 
967 	spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu",
968 	    (u_longlong_t)oldguid, (u_longlong_t)*newguid);
969 }
970 
971 /*
972  * Change the GUID for the pool.  This is done so that we can later
973  * re-import a pool built from a clone of our own vdevs.  We will modify
974  * the root vdev's guid, our own pool guid, and then mark all of our
975  * vdevs dirty.  Note that we must make sure that all our vdevs are
976  * online when we do this, or else any vdevs that weren't present
977  * would be orphaned from our pool.  We are also going to issue a
978  * sysevent to update any watchers.
979  */
980 int
981 spa_change_guid(spa_t *spa)
982 {
983 	int error;
984 	uint64_t guid;
985 
986 	mutex_enter(&spa->spa_vdev_top_lock);
987 	mutex_enter(&spa_namespace_lock);
988 	guid = spa_generate_guid(NULL);
989 
990 	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
991 	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);
992 
993 	if (error == 0) {
994 		/*
995 		 * Clear the kobj flag from all the vdevs to allow
996 		 * vdev_cache_process_kobj_evt() to post events to all the
997 		 * vdevs since GUID is updated.
998 		 */
999 		vdev_clear_kobj_evt(spa->spa_root_vdev);
1000 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
1001 			vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]);
1002 
1003 		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
1004 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID);
1005 	}
1006 
1007 	mutex_exit(&spa_namespace_lock);
1008 	mutex_exit(&spa->spa_vdev_top_lock);
1009 
1010 	return (error);
1011 }
1012 
1013 /*
1014  * ==========================================================================
1015  * SPA state manipulation (open/create/destroy/import/export)
1016  * ==========================================================================
1017  */
1018 
1019 static int
1020 spa_error_entry_compare(const void *a, const void *b)
1021 {
1022 	const spa_error_entry_t *sa = (const spa_error_entry_t *)a;
1023 	const spa_error_entry_t *sb = (const spa_error_entry_t *)b;
1024 	int ret;
1025 
1026 	ret = memcmp(&sa->se_bookmark, &sb->se_bookmark,
1027 	    sizeof (zbookmark_phys_t));
1028 
1029 	return (TREE_ISIGN(ret));
1030 }
1031 
1032 /*
1033  * Utility function which retrieves copies of the current logs and
1034  * re-initializes them in the process.
1035  */
1036 void
1037 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub)
1038 {
1039 	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));
1040 
1041 	memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t));
1042 	memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t));
1043 
1044 	avl_create(&spa->spa_errlist_scrub,
1045 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1046 	    offsetof(spa_error_entry_t, se_avl));
1047 	avl_create(&spa->spa_errlist_last,
1048 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1049 	    offsetof(spa_error_entry_t, se_avl));
1050 }
1051 
1052 static void
1053 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
1054 {
1055 	const zio_taskq_info_t *ztip = &zio_taskqs[t][q];
1056 	enum zti_modes mode = ztip->zti_mode;
1057 	uint_t value = ztip->zti_value;
1058 	uint_t count = ztip->zti_count;
1059 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1060 	uint_t cpus, flags = TASKQ_DYNAMIC;
1061 
1062 	switch (mode) {
1063 	case ZTI_MODE_FIXED:
1064 		ASSERT3U(value, >, 0);
1065 		break;
1066 
1067 	case ZTI_MODE_SYNC:
1068 
1069 		/*
1070 		 * Create one wr_iss taskq for every 'zio_taskq_wr_iss_ncpus',
1071 		 * not to exceed the number of spa allocators.
1072 		 */
1073 		if (zio_taskq_wr_iss_ncpus == 0) {
1074 			count = MAX(boot_ncpus / spa->spa_alloc_count, 1);
1075 		} else {
1076 			count = MAX(1,
1077 			    boot_ncpus / MAX(1, zio_taskq_wr_iss_ncpus));
1078 		}
1079 		count = MAX(count, (zio_taskq_batch_pct + 99) / 100);
1080 		count = MIN(count, spa->spa_alloc_count);
1081 
1082 		/*
1083 		 * zio_taskq_batch_pct is unbounded and may exceed 100%, but no
1084 		 * single taskq may have more threads than 100% of online cpus.
1085 		 */
1086 		value = (zio_taskq_batch_pct + count / 2) / count;
1087 		value = MIN(value, 100);
1088 		flags |= TASKQ_THREADS_CPU_PCT;
1089 		break;
1090 
1091 	case ZTI_MODE_SCALE:
1092 		flags |= TASKQ_THREADS_CPU_PCT;
1093 		/*
1094 		 * We want more taskqs to reduce lock contention, but we want
1095 		 * less for better request ordering and CPU utilization.
1096 		 */
1097 		cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100);
1098 		if (zio_taskq_batch_tpq > 0) {
1099 			count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) /
1100 			    zio_taskq_batch_tpq);
1101 		} else {
1102 			/*
1103 			 * Prefer 6 threads per taskq, but no more taskqs
1104 			 * than threads in them on large systems. For 80%:
1105 			 *
1106 			 *                 taskq   taskq   total
1107 			 * cpus    taskqs  percent threads threads
1108 			 * ------- ------- ------- ------- -------
1109 			 * 1       1       80%     1       1
1110 			 * 2       1       80%     1       1
1111 			 * 4       1       80%     3       3
1112 			 * 8       2       40%     3       6
1113 			 * 16      3       27%     4       12
1114 			 * 32      5       16%     5       25
1115 			 * 64      7       11%     7       49
1116 			 * 128     10      8%      10      100
1117 			 * 256     14      6%      15      210
1118 			 */
1119 			count = 1 + cpus / 6;
1120 			while (count * count > cpus)
1121 				count--;
1122 		}
1123 		/* Limit each taskq within 100% to not trigger assertion. */
1124 		count = MAX(count, (zio_taskq_batch_pct + 99) / 100);
1125 		value = (zio_taskq_batch_pct + count / 2) / count;
1126 		break;
1127 
1128 	case ZTI_MODE_NULL:
1129 		tqs->stqs_count = 0;
1130 		tqs->stqs_taskq = NULL;
1131 		return;
1132 
1133 	default:
1134 		panic("unrecognized mode for %s_%s taskq (%u:%u) in "
1135 		    "spa_taskqs_init()",
1136 		    zio_type_name[t], zio_taskq_types[q], mode, value);
1137 		break;
1138 	}
1139 
1140 	ASSERT3U(count, >, 0);
1141 	tqs->stqs_count = count;
1142 	tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP);
1143 
1144 	for (uint_t i = 0; i < count; i++) {
1145 		taskq_t *tq;
1146 		char name[32];
1147 
1148 		if (count > 1)
1149 			(void) snprintf(name, sizeof (name), "%s_%s_%u",
1150 			    zio_type_name[t], zio_taskq_types[q], i);
1151 		else
1152 			(void) snprintf(name, sizeof (name), "%s_%s",
1153 			    zio_type_name[t], zio_taskq_types[q]);
1154 
1155 #ifdef HAVE_SYSDC
1156 		if (zio_taskq_sysdc && spa->spa_proc != &p0) {
1157 			(void) zio_taskq_basedc;
1158 			tq = taskq_create_sysdc(name, value, 50, INT_MAX,
1159 			    spa->spa_proc, zio_taskq_basedc, flags);
1160 		} else {
1161 #endif
1162 			pri_t pri = maxclsyspri;
1163 			/*
1164 			 * The write issue taskq can be extremely CPU
1165 			 * intensive.  Run it at slightly less important
1166 			 * priority than the other taskqs.
1167 			 *
1168 			 * Under Linux and FreeBSD this means incrementing
1169 			 * the priority value as opposed to platforms like
1170 			 * illumos where it should be decremented.
1171 			 *
1172 			 * On FreeBSD, if priorities divided by four (RQ_PPQ)
1173 			 * are equal then a difference between them is
1174 			 * insignificant.
1175 			 */
1176 			if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) {
1177 #if defined(__linux__)
1178 				pri++;
1179 #elif defined(__FreeBSD__)
1180 				pri += 4;
1181 #else
1182 #error "unknown OS"
1183 #endif
1184 			}
1185 			tq = taskq_create_proc(name, value, pri, 50,
1186 			    INT_MAX, spa->spa_proc, flags);
1187 #ifdef HAVE_SYSDC
1188 		}
1189 #endif
1190 
1191 		tqs->stqs_taskq[i] = tq;
1192 	}
1193 }
1194 
1195 static void
1196 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
1197 {
1198 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1199 
1200 	if (tqs->stqs_taskq == NULL) {
1201 		ASSERT3U(tqs->stqs_count, ==, 0);
1202 		return;
1203 	}
1204 
1205 	for (uint_t i = 0; i < tqs->stqs_count; i++) {
1206 		ASSERT3P(tqs->stqs_taskq[i], !=, NULL);
1207 		taskq_destroy(tqs->stqs_taskq[i]);
1208 	}
1209 
1210 	kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *));
1211 	tqs->stqs_taskq = NULL;
1212 }
1213 
1214 #ifdef _KERNEL
1215 /*
1216  * The READ and WRITE rows of zio_taskqs are configurable at module load time
1217  * by setting zio_taskq_read or zio_taskq_write.
1218  *
1219  * Example (the defaults for READ and WRITE)
1220  *   zio_taskq_read='fixed,1,8 null scale null'
1221  *   zio_taskq_write='sync fixed,1,5 scale fixed,1,5'
1222  *
1223  * Each sets the entire row at a time.
1224  *
1225  * 'fixed' is parameterised: fixed,Q,T where Q is number of taskqs, T is number
1226  * of threads per taskq.
1227  *
1228  * 'null' can only be set on the high-priority queues (queue selection for
1229  * high-priority queues will fall back to the regular queue if the high-pri
1230  * is NULL.
1231  */
1232 static const char *const modes[ZTI_NMODES] = {
1233 	"fixed", "scale", "sync", "null"
1234 };
1235 
1236 /* Parse the incoming config string. Modifies cfg */
1237 static int
1238 spa_taskq_param_set(zio_type_t t, char *cfg)
1239 {
1240 	int err = 0;
1241 
1242 	zio_taskq_info_t row[ZIO_TASKQ_TYPES] = {{0}};
1243 
1244 	char *next = cfg, *tok, *c;
1245 
1246 	/*
1247 	 * Parse out each element from the string and fill `row`. The entire
1248 	 * row has to be set at once, so any errors are flagged by just
1249 	 * breaking out of this loop early.
1250 	 */
1251 	uint_t q;
1252 	for (q = 0; q < ZIO_TASKQ_TYPES; q++) {
1253 		/* `next` is the start of the config */
1254 		if (next == NULL)
1255 			break;
1256 
1257 		/* Eat up leading space */
1258 		while (isspace(*next))
1259 			next++;
1260 		if (*next == '\0')
1261 			break;
1262 
1263 		/* Mode ends at space or end of string */
1264 		tok = next;
1265 		next = strchr(tok, ' ');
1266 		if (next != NULL) *next++ = '\0';
1267 
1268 		/* Parameters start after a comma */
1269 		c = strchr(tok, ',');
1270 		if (c != NULL) *c++ = '\0';
1271 
1272 		/* Match mode string */
1273 		uint_t mode;
1274 		for (mode = 0; mode < ZTI_NMODES; mode++)
1275 			if (strcmp(tok, modes[mode]) == 0)
1276 				break;
1277 		if (mode == ZTI_NMODES)
1278 			break;
1279 
1280 		/* Invalid canary */
1281 		row[q].zti_mode = ZTI_NMODES;
1282 
1283 		/* Per-mode setup */
1284 		switch (mode) {
1285 
1286 		/*
1287 		 * FIXED is parameterised: number of queues, and number of
1288 		 * threads per queue.
1289 		 */
1290 		case ZTI_MODE_FIXED: {
1291 			/* No parameters? */
1292 			if (c == NULL || *c == '\0')
1293 				break;
1294 
1295 			/* Find next parameter */
1296 			tok = c;
1297 			c = strchr(tok, ',');
1298 			if (c == NULL)
1299 				break;
1300 
1301 			/* Take digits and convert */
1302 			unsigned long long nq;
1303 			if (!(isdigit(*tok)))
1304 				break;
1305 			err = ddi_strtoull(tok, &tok, 10, &nq);
1306 			/* Must succeed and also end at the next param sep */
1307 			if (err != 0 || tok != c)
1308 				break;
1309 
1310 			/* Move past the comma */
1311 			tok++;
1312 			/* Need another number */
1313 			if (!(isdigit(*tok)))
1314 				break;
1315 			/* Remember start to make sure we moved */
1316 			c = tok;
1317 
1318 			/* Take digits */
1319 			unsigned long long ntpq;
1320 			err = ddi_strtoull(tok, &tok, 10, &ntpq);
1321 			/* Must succeed, and moved forward */
1322 			if (err != 0 || tok == c || *tok != '\0')
1323 				break;
1324 
1325 			/*
1326 			 * sanity; zero queues/threads make no sense, and
1327 			 * 16K is almost certainly more than anyone will ever
1328 			 * need and avoids silly numbers like UINT32_MAX
1329 			 */
1330 			if (nq == 0 || nq >= 16384 ||
1331 			    ntpq == 0 || ntpq >= 16384)
1332 				break;
1333 
1334 			const zio_taskq_info_t zti = ZTI_P(ntpq, nq);
1335 			row[q] = zti;
1336 			break;
1337 		}
1338 
1339 		case ZTI_MODE_SCALE: {
1340 			const zio_taskq_info_t zti = ZTI_SCALE;
1341 			row[q] = zti;
1342 			break;
1343 		}
1344 
1345 		case ZTI_MODE_SYNC: {
1346 			const zio_taskq_info_t zti = ZTI_SYNC;
1347 			row[q] = zti;
1348 			break;
1349 		}
1350 
1351 		case ZTI_MODE_NULL: {
1352 			/*
1353 			 * Can only null the high-priority queues; the general-
1354 			 * purpose ones have to exist.
1355 			 */
1356 			if (q != ZIO_TASKQ_ISSUE_HIGH &&
1357 			    q != ZIO_TASKQ_INTERRUPT_HIGH)
1358 				break;
1359 
1360 			const zio_taskq_info_t zti = ZTI_NULL;
1361 			row[q] = zti;
1362 			break;
1363 		}
1364 
1365 		default:
1366 			break;
1367 		}
1368 
1369 		/* Ensure we set a mode */
1370 		if (row[q].zti_mode == ZTI_NMODES)
1371 			break;
1372 	}
1373 
1374 	/* Didn't get a full row, fail */
1375 	if (q < ZIO_TASKQ_TYPES)
1376 		return (SET_ERROR(EINVAL));
1377 
1378 	/* Eat trailing space */
1379 	if (next != NULL)
1380 		while (isspace(*next))
1381 			next++;
1382 
1383 	/* If there's anything left over then fail */
1384 	if (next != NULL && *next != '\0')
1385 		return (SET_ERROR(EINVAL));
1386 
1387 	/* Success! Copy it into the real config */
1388 	for (q = 0; q < ZIO_TASKQ_TYPES; q++)
1389 		zio_taskqs[t][q] = row[q];
1390 
1391 	return (0);
1392 }
1393 
1394 static int
1395 spa_taskq_param_get(zio_type_t t, char *buf, boolean_t add_newline)
1396 {
1397 	int pos = 0;
1398 
1399 	/* Build paramater string from live config */
1400 	const char *sep = "";
1401 	for (uint_t q = 0; q < ZIO_TASKQ_TYPES; q++) {
1402 		const zio_taskq_info_t *zti = &zio_taskqs[t][q];
1403 		if (zti->zti_mode == ZTI_MODE_FIXED)
1404 			pos += sprintf(&buf[pos], "%s%s,%u,%u", sep,
1405 			    modes[zti->zti_mode], zti->zti_count,
1406 			    zti->zti_value);
1407 		else
1408 			pos += sprintf(&buf[pos], "%s%s", sep,
1409 			    modes[zti->zti_mode]);
1410 		sep = " ";
1411 	}
1412 
1413 	if (add_newline)
1414 		buf[pos++] = '\n';
1415 	buf[pos] = '\0';
1416 
1417 	return (pos);
1418 }
1419 
1420 #ifdef __linux__
1421 static int
1422 spa_taskq_read_param_set(const char *val, zfs_kernel_param_t *kp)
1423 {
1424 	char *cfg = kmem_strdup(val);
1425 	int err = spa_taskq_param_set(ZIO_TYPE_READ, cfg);
1426 	kmem_free(cfg, strlen(val)+1);
1427 	return (-err);
1428 }
1429 static int
1430 spa_taskq_read_param_get(char *buf, zfs_kernel_param_t *kp)
1431 {
1432 	return (spa_taskq_param_get(ZIO_TYPE_READ, buf, TRUE));
1433 }
1434 
1435 static int
1436 spa_taskq_write_param_set(const char *val, zfs_kernel_param_t *kp)
1437 {
1438 	char *cfg = kmem_strdup(val);
1439 	int err = spa_taskq_param_set(ZIO_TYPE_WRITE, cfg);
1440 	kmem_free(cfg, strlen(val)+1);
1441 	return (-err);
1442 }
1443 static int
1444 spa_taskq_write_param_get(char *buf, zfs_kernel_param_t *kp)
1445 {
1446 	return (spa_taskq_param_get(ZIO_TYPE_WRITE, buf, TRUE));
1447 }
1448 #else
1449 /*
1450  * On FreeBSD load-time parameters can be set up before malloc() is available,
1451  * so we have to do all the parsing work on the stack.
1452  */
1453 #define	SPA_TASKQ_PARAM_MAX	(128)
1454 
1455 static int
1456 spa_taskq_read_param(ZFS_MODULE_PARAM_ARGS)
1457 {
1458 	char buf[SPA_TASKQ_PARAM_MAX];
1459 	int err;
1460 
1461 	(void) spa_taskq_param_get(ZIO_TYPE_READ, buf, FALSE);
1462 	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
1463 	if (err || req->newptr == NULL)
1464 		return (err);
1465 	return (spa_taskq_param_set(ZIO_TYPE_READ, buf));
1466 }
1467 
1468 static int
1469 spa_taskq_write_param(ZFS_MODULE_PARAM_ARGS)
1470 {
1471 	char buf[SPA_TASKQ_PARAM_MAX];
1472 	int err;
1473 
1474 	(void) spa_taskq_param_get(ZIO_TYPE_WRITE, buf, FALSE);
1475 	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
1476 	if (err || req->newptr == NULL)
1477 		return (err);
1478 	return (spa_taskq_param_set(ZIO_TYPE_WRITE, buf));
1479 }
1480 #endif
1481 #endif /* _KERNEL */
1482 
1483 /*
1484  * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority.
1485  * Note that a type may have multiple discrete taskqs to avoid lock contention
1486  * on the taskq itself.
1487  */
1488 static taskq_t *
1489 spa_taskq_dispatch_select(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1490     zio_t *zio)
1491 {
1492 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1493 	taskq_t *tq;
1494 
1495 	ASSERT3P(tqs->stqs_taskq, !=, NULL);
1496 	ASSERT3U(tqs->stqs_count, !=, 0);
1497 
1498 	if ((t == ZIO_TYPE_WRITE) && (q == ZIO_TASKQ_ISSUE) &&
1499 	    (zio != NULL) && (zio->io_wr_iss_tq != NULL)) {
1500 		/* dispatch to assigned write issue taskq */
1501 		tq = zio->io_wr_iss_tq;
1502 		return (tq);
1503 	}
1504 
1505 	if (tqs->stqs_count == 1) {
1506 		tq = tqs->stqs_taskq[0];
1507 	} else {
1508 		tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count];
1509 	}
1510 	return (tq);
1511 }
1512 
1513 void
1514 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1515     task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent,
1516     zio_t *zio)
1517 {
1518 	taskq_t *tq = spa_taskq_dispatch_select(spa, t, q, zio);
1519 	taskq_dispatch_ent(tq, func, arg, flags, ent);
1520 }
1521 
1522 /*
1523  * Same as spa_taskq_dispatch_ent() but block on the task until completion.
1524  */
1525 void
1526 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
1527     task_func_t *func, void *arg, uint_t flags)
1528 {
1529 	taskq_t *tq = spa_taskq_dispatch_select(spa, t, q, NULL);
1530 	taskqid_t id = taskq_dispatch(tq, func, arg, flags);
1531 	if (id)
1532 		taskq_wait_id(tq, id);
1533 }
1534 
1535 static void
1536 spa_create_zio_taskqs(spa_t *spa)
1537 {
1538 	for (int t = 0; t < ZIO_TYPES; t++) {
1539 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1540 			spa_taskqs_init(spa, t, q);
1541 		}
1542 	}
1543 }
1544 
1545 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD)
1546 static void
1547 spa_thread(void *arg)
1548 {
1549 	psetid_t zio_taskq_psrset_bind = PS_NONE;
1550 	callb_cpr_t cprinfo;
1551 
1552 	spa_t *spa = arg;
1553 	user_t *pu = PTOU(curproc);
1554 
1555 	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
1556 	    spa->spa_name);
1557 
1558 	ASSERT(curproc != &p0);
1559 	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
1560 	    "zpool-%s", spa->spa_name);
1561 	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
1562 
1563 	/* bind this thread to the requested psrset */
1564 	if (zio_taskq_psrset_bind != PS_NONE) {
1565 		pool_lock();
1566 		mutex_enter(&cpu_lock);
1567 		mutex_enter(&pidlock);
1568 		mutex_enter(&curproc->p_lock);
1569 
1570 		if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind,
1571 		    0, NULL, NULL) == 0)  {
1572 			curthread->t_bind_pset = zio_taskq_psrset_bind;
1573 		} else {
1574 			cmn_err(CE_WARN,
1575 			    "Couldn't bind process for zfs pool \"%s\" to "
1576 			    "pset %d\n", spa->spa_name, zio_taskq_psrset_bind);
1577 		}
1578 
1579 		mutex_exit(&curproc->p_lock);
1580 		mutex_exit(&pidlock);
1581 		mutex_exit(&cpu_lock);
1582 		pool_unlock();
1583 	}
1584 
1585 #ifdef HAVE_SYSDC
1586 	if (zio_taskq_sysdc) {
1587 		sysdc_thread_enter(curthread, 100, 0);
1588 	}
1589 #endif
1590 
1591 	spa->spa_proc = curproc;
1592 	spa->spa_did = curthread->t_did;
1593 
1594 	spa_create_zio_taskqs(spa);
1595 
1596 	mutex_enter(&spa->spa_proc_lock);
1597 	ASSERT(spa->spa_proc_state == SPA_PROC_CREATED);
1598 
1599 	spa->spa_proc_state = SPA_PROC_ACTIVE;
1600 	cv_broadcast(&spa->spa_proc_cv);
1601 
1602 	CALLB_CPR_SAFE_BEGIN(&cprinfo);
1603 	while (spa->spa_proc_state == SPA_PROC_ACTIVE)
1604 		cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1605 	CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock);
1606 
1607 	ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE);
1608 	spa->spa_proc_state = SPA_PROC_GONE;
1609 	spa->spa_proc = &p0;
1610 	cv_broadcast(&spa->spa_proc_cv);
1611 	CALLB_CPR_EXIT(&cprinfo);	/* drops spa_proc_lock */
1612 
1613 	mutex_enter(&curproc->p_lock);
1614 	lwp_exit();
1615 }
1616 #endif
1617 
1618 extern metaslab_ops_t *metaslab_allocator(spa_t *spa);
1619 
1620 /*
1621  * Activate an uninitialized pool.
1622  */
1623 static void
1624 spa_activate(spa_t *spa, spa_mode_t mode)
1625 {
1626 	metaslab_ops_t *msp = metaslab_allocator(spa);
1627 	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
1628 
1629 	spa->spa_state = POOL_STATE_ACTIVE;
1630 	spa->spa_mode = mode;
1631 	spa->spa_read_spacemaps = spa_mode_readable_spacemaps;
1632 
1633 	spa->spa_normal_class = metaslab_class_create(spa, msp);
1634 	spa->spa_log_class = metaslab_class_create(spa, msp);
1635 	spa->spa_embedded_log_class = metaslab_class_create(spa, msp);
1636 	spa->spa_special_class = metaslab_class_create(spa, msp);
1637 	spa->spa_dedup_class = metaslab_class_create(spa, msp);
1638 
1639 	/* Try to create a covering process */
1640 	mutex_enter(&spa->spa_proc_lock);
1641 	ASSERT(spa->spa_proc_state == SPA_PROC_NONE);
1642 	ASSERT(spa->spa_proc == &p0);
1643 	spa->spa_did = 0;
1644 
1645 #ifdef HAVE_SPA_THREAD
1646 	/* Only create a process if we're going to be around a while. */
1647 	if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) {
1648 		if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri,
1649 		    NULL, 0) == 0) {
1650 			spa->spa_proc_state = SPA_PROC_CREATED;
1651 			while (spa->spa_proc_state == SPA_PROC_CREATED) {
1652 				cv_wait(&spa->spa_proc_cv,
1653 				    &spa->spa_proc_lock);
1654 			}
1655 			ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1656 			ASSERT(spa->spa_proc != &p0);
1657 			ASSERT(spa->spa_did != 0);
1658 		} else {
1659 #ifdef _KERNEL
1660 			cmn_err(CE_WARN,
1661 			    "Couldn't create process for zfs pool \"%s\"\n",
1662 			    spa->spa_name);
1663 #endif
1664 		}
1665 	}
1666 #endif /* HAVE_SPA_THREAD */
1667 	mutex_exit(&spa->spa_proc_lock);
1668 
1669 	/* If we didn't create a process, we need to create our taskqs. */
1670 	if (spa->spa_proc == &p0) {
1671 		spa_create_zio_taskqs(spa);
1672 	}
1673 
1674 	for (size_t i = 0; i < TXG_SIZE; i++) {
1675 		spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL,
1676 		    ZIO_FLAG_CANFAIL);
1677 	}
1678 
1679 	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
1680 	    offsetof(vdev_t, vdev_config_dirty_node));
1681 	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
1682 	    offsetof(objset_t, os_evicting_node));
1683 	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
1684 	    offsetof(vdev_t, vdev_state_dirty_node));
1685 
1686 	txg_list_create(&spa->spa_vdev_txg_list, spa,
1687 	    offsetof(struct vdev, vdev_txg_node));
1688 
1689 	avl_create(&spa->spa_errlist_scrub,
1690 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1691 	    offsetof(spa_error_entry_t, se_avl));
1692 	avl_create(&spa->spa_errlist_last,
1693 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1694 	    offsetof(spa_error_entry_t, se_avl));
1695 	avl_create(&spa->spa_errlist_healed,
1696 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1697 	    offsetof(spa_error_entry_t, se_avl));
1698 
1699 	spa_activate_os(spa);
1700 
1701 	spa_keystore_init(&spa->spa_keystore);
1702 
1703 	/*
1704 	 * This taskq is used to perform zvol-minor-related tasks
1705 	 * asynchronously. This has several advantages, including easy
1706 	 * resolution of various deadlocks.
1707 	 *
1708 	 * The taskq must be single threaded to ensure tasks are always
1709 	 * processed in the order in which they were dispatched.
1710 	 *
1711 	 * A taskq per pool allows one to keep the pools independent.
1712 	 * This way if one pool is suspended, it will not impact another.
1713 	 *
1714 	 * The preferred location to dispatch a zvol minor task is a sync
1715 	 * task. In this context, there is easy access to the spa_t and minimal
1716 	 * error handling is required because the sync task must succeed.
1717 	 */
1718 	spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri,
1719 	    1, INT_MAX, 0);
1720 
1721 	/*
1722 	 * The taskq to preload metaslabs.
1723 	 */
1724 	spa->spa_metaslab_taskq = taskq_create("z_metaslab",
1725 	    metaslab_preload_pct, maxclsyspri, 1, INT_MAX,
1726 	    TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1727 
1728 	/*
1729 	 * Taskq dedicated to prefetcher threads: this is used to prevent the
1730 	 * pool traverse code from monopolizing the global (and limited)
1731 	 * system_taskq by inappropriately scheduling long running tasks on it.
1732 	 */
1733 	spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100,
1734 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1735 
1736 	/*
1737 	 * The taskq to upgrade datasets in this pool. Currently used by
1738 	 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA.
1739 	 */
1740 	spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100,
1741 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT);
1742 }
1743 
1744 /*
1745  * Opposite of spa_activate().
1746  */
1747 static void
1748 spa_deactivate(spa_t *spa)
1749 {
1750 	ASSERT(spa->spa_sync_on == B_FALSE);
1751 	ASSERT(spa->spa_dsl_pool == NULL);
1752 	ASSERT(spa->spa_root_vdev == NULL);
1753 	ASSERT(spa->spa_async_zio_root == NULL);
1754 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
1755 
1756 	spa_evicting_os_wait(spa);
1757 
1758 	if (spa->spa_zvol_taskq) {
1759 		taskq_destroy(spa->spa_zvol_taskq);
1760 		spa->spa_zvol_taskq = NULL;
1761 	}
1762 
1763 	if (spa->spa_metaslab_taskq) {
1764 		taskq_destroy(spa->spa_metaslab_taskq);
1765 		spa->spa_metaslab_taskq = NULL;
1766 	}
1767 
1768 	if (spa->spa_prefetch_taskq) {
1769 		taskq_destroy(spa->spa_prefetch_taskq);
1770 		spa->spa_prefetch_taskq = NULL;
1771 	}
1772 
1773 	if (spa->spa_upgrade_taskq) {
1774 		taskq_destroy(spa->spa_upgrade_taskq);
1775 		spa->spa_upgrade_taskq = NULL;
1776 	}
1777 
1778 	txg_list_destroy(&spa->spa_vdev_txg_list);
1779 
1780 	list_destroy(&spa->spa_config_dirty_list);
1781 	list_destroy(&spa->spa_evicting_os_list);
1782 	list_destroy(&spa->spa_state_dirty_list);
1783 
1784 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
1785 
1786 	for (int t = 0; t < ZIO_TYPES; t++) {
1787 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1788 			spa_taskqs_fini(spa, t, q);
1789 		}
1790 	}
1791 
1792 	for (size_t i = 0; i < TXG_SIZE; i++) {
1793 		ASSERT3P(spa->spa_txg_zio[i], !=, NULL);
1794 		VERIFY0(zio_wait(spa->spa_txg_zio[i]));
1795 		spa->spa_txg_zio[i] = NULL;
1796 	}
1797 
1798 	metaslab_class_destroy(spa->spa_normal_class);
1799 	spa->spa_normal_class = NULL;
1800 
1801 	metaslab_class_destroy(spa->spa_log_class);
1802 	spa->spa_log_class = NULL;
1803 
1804 	metaslab_class_destroy(spa->spa_embedded_log_class);
1805 	spa->spa_embedded_log_class = NULL;
1806 
1807 	metaslab_class_destroy(spa->spa_special_class);
1808 	spa->spa_special_class = NULL;
1809 
1810 	metaslab_class_destroy(spa->spa_dedup_class);
1811 	spa->spa_dedup_class = NULL;
1812 
1813 	/*
1814 	 * If this was part of an import or the open otherwise failed, we may
1815 	 * still have errors left in the queues.  Empty them just in case.
1816 	 */
1817 	spa_errlog_drain(spa);
1818 	avl_destroy(&spa->spa_errlist_scrub);
1819 	avl_destroy(&spa->spa_errlist_last);
1820 	avl_destroy(&spa->spa_errlist_healed);
1821 
1822 	spa_keystore_fini(&spa->spa_keystore);
1823 
1824 	spa->spa_state = POOL_STATE_UNINITIALIZED;
1825 
1826 	mutex_enter(&spa->spa_proc_lock);
1827 	if (spa->spa_proc_state != SPA_PROC_NONE) {
1828 		ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1829 		spa->spa_proc_state = SPA_PROC_DEACTIVATE;
1830 		cv_broadcast(&spa->spa_proc_cv);
1831 		while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) {
1832 			ASSERT(spa->spa_proc != &p0);
1833 			cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1834 		}
1835 		ASSERT(spa->spa_proc_state == SPA_PROC_GONE);
1836 		spa->spa_proc_state = SPA_PROC_NONE;
1837 	}
1838 	ASSERT(spa->spa_proc == &p0);
1839 	mutex_exit(&spa->spa_proc_lock);
1840 
1841 	/*
1842 	 * We want to make sure spa_thread() has actually exited the ZFS
1843 	 * module, so that the module can't be unloaded out from underneath
1844 	 * it.
1845 	 */
1846 	if (spa->spa_did != 0) {
1847 		thread_join(spa->spa_did);
1848 		spa->spa_did = 0;
1849 	}
1850 
1851 	spa_deactivate_os(spa);
1852 
1853 }
1854 
1855 /*
1856  * Verify a pool configuration, and construct the vdev tree appropriately.  This
1857  * will create all the necessary vdevs in the appropriate layout, with each vdev
1858  * in the CLOSED state.  This will prep the pool before open/creation/import.
1859  * All vdev validation is done by the vdev_alloc() routine.
1860  */
1861 int
1862 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent,
1863     uint_t id, int atype)
1864 {
1865 	nvlist_t **child;
1866 	uint_t children;
1867 	int error;
1868 
1869 	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
1870 		return (error);
1871 
1872 	if ((*vdp)->vdev_ops->vdev_op_leaf)
1873 		return (0);
1874 
1875 	error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1876 	    &child, &children);
1877 
1878 	if (error == ENOENT)
1879 		return (0);
1880 
1881 	if (error) {
1882 		vdev_free(*vdp);
1883 		*vdp = NULL;
1884 		return (SET_ERROR(EINVAL));
1885 	}
1886 
1887 	for (int c = 0; c < children; c++) {
1888 		vdev_t *vd;
1889 		if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
1890 		    atype)) != 0) {
1891 			vdev_free(*vdp);
1892 			*vdp = NULL;
1893 			return (error);
1894 		}
1895 	}
1896 
1897 	ASSERT(*vdp != NULL);
1898 
1899 	return (0);
1900 }
1901 
1902 static boolean_t
1903 spa_should_flush_logs_on_unload(spa_t *spa)
1904 {
1905 	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))
1906 		return (B_FALSE);
1907 
1908 	if (!spa_writeable(spa))
1909 		return (B_FALSE);
1910 
1911 	if (!spa->spa_sync_on)
1912 		return (B_FALSE);
1913 
1914 	if (spa_state(spa) != POOL_STATE_EXPORTED)
1915 		return (B_FALSE);
1916 
1917 	if (zfs_keep_log_spacemaps_at_export)
1918 		return (B_FALSE);
1919 
1920 	return (B_TRUE);
1921 }
1922 
1923 /*
1924  * Opens a transaction that will set the flag that will instruct
1925  * spa_sync to attempt to flush all the metaslabs for that txg.
1926  */
1927 static void
1928 spa_unload_log_sm_flush_all(spa_t *spa)
1929 {
1930 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
1931 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
1932 
1933 	ASSERT3U(spa->spa_log_flushall_txg, ==, 0);
1934 	spa->spa_log_flushall_txg = dmu_tx_get_txg(tx);
1935 
1936 	dmu_tx_commit(tx);
1937 	txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg);
1938 }
1939 
1940 static void
1941 spa_unload_log_sm_metadata(spa_t *spa)
1942 {
1943 	void *cookie = NULL;
1944 	spa_log_sm_t *sls;
1945 	log_summary_entry_t *e;
1946 
1947 	while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg,
1948 	    &cookie)) != NULL) {
1949 		VERIFY0(sls->sls_mscount);
1950 		kmem_free(sls, sizeof (spa_log_sm_t));
1951 	}
1952 
1953 	while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) {
1954 		VERIFY0(e->lse_mscount);
1955 		kmem_free(e, sizeof (log_summary_entry_t));
1956 	}
1957 
1958 	spa->spa_unflushed_stats.sus_nblocks = 0;
1959 	spa->spa_unflushed_stats.sus_memused = 0;
1960 	spa->spa_unflushed_stats.sus_blocklimit = 0;
1961 }
1962 
1963 static void
1964 spa_destroy_aux_threads(spa_t *spa)
1965 {
1966 	if (spa->spa_condense_zthr != NULL) {
1967 		zthr_destroy(spa->spa_condense_zthr);
1968 		spa->spa_condense_zthr = NULL;
1969 	}
1970 	if (spa->spa_checkpoint_discard_zthr != NULL) {
1971 		zthr_destroy(spa->spa_checkpoint_discard_zthr);
1972 		spa->spa_checkpoint_discard_zthr = NULL;
1973 	}
1974 	if (spa->spa_livelist_delete_zthr != NULL) {
1975 		zthr_destroy(spa->spa_livelist_delete_zthr);
1976 		spa->spa_livelist_delete_zthr = NULL;
1977 	}
1978 	if (spa->spa_livelist_condense_zthr != NULL) {
1979 		zthr_destroy(spa->spa_livelist_condense_zthr);
1980 		spa->spa_livelist_condense_zthr = NULL;
1981 	}
1982 	if (spa->spa_raidz_expand_zthr != NULL) {
1983 		zthr_destroy(spa->spa_raidz_expand_zthr);
1984 		spa->spa_raidz_expand_zthr = NULL;
1985 	}
1986 }
1987 
1988 /*
1989  * Opposite of spa_load().
1990  */
1991 static void
1992 spa_unload(spa_t *spa)
1993 {
1994 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1995 	ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED);
1996 
1997 	spa_import_progress_remove(spa_guid(spa));
1998 	spa_load_note(spa, "UNLOADING");
1999 
2000 	spa_wake_waiters(spa);
2001 
2002 	/*
2003 	 * If we have set the spa_final_txg, we have already performed the
2004 	 * tasks below in spa_export_common(). We should not redo it here since
2005 	 * we delay the final TXGs beyond what spa_final_txg is set at.
2006 	 */
2007 	if (spa->spa_final_txg == UINT64_MAX) {
2008 		/*
2009 		 * If the log space map feature is enabled and the pool is
2010 		 * getting exported (but not destroyed), we want to spend some
2011 		 * time flushing as many metaslabs as we can in an attempt to
2012 		 * destroy log space maps and save import time.
2013 		 */
2014 		if (spa_should_flush_logs_on_unload(spa))
2015 			spa_unload_log_sm_flush_all(spa);
2016 
2017 		/*
2018 		 * Stop async tasks.
2019 		 */
2020 		spa_async_suspend(spa);
2021 
2022 		if (spa->spa_root_vdev) {
2023 			vdev_t *root_vdev = spa->spa_root_vdev;
2024 			vdev_initialize_stop_all(root_vdev,
2025 			    VDEV_INITIALIZE_ACTIVE);
2026 			vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE);
2027 			vdev_autotrim_stop_all(spa);
2028 			vdev_rebuild_stop_all(spa);
2029 		}
2030 	}
2031 
2032 	/*
2033 	 * Stop syncing.
2034 	 */
2035 	if (spa->spa_sync_on) {
2036 		txg_sync_stop(spa->spa_dsl_pool);
2037 		spa->spa_sync_on = B_FALSE;
2038 	}
2039 
2040 	/*
2041 	 * This ensures that there is no async metaslab prefetching
2042 	 * while we attempt to unload the spa.
2043 	 */
2044 	taskq_wait(spa->spa_metaslab_taskq);
2045 
2046 	if (spa->spa_mmp.mmp_thread)
2047 		mmp_thread_stop(spa);
2048 
2049 	/*
2050 	 * Wait for any outstanding async I/O to complete.
2051 	 */
2052 	if (spa->spa_async_zio_root != NULL) {
2053 		for (int i = 0; i < max_ncpus; i++)
2054 			(void) zio_wait(spa->spa_async_zio_root[i]);
2055 		kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
2056 		spa->spa_async_zio_root = NULL;
2057 	}
2058 
2059 	if (spa->spa_vdev_removal != NULL) {
2060 		spa_vdev_removal_destroy(spa->spa_vdev_removal);
2061 		spa->spa_vdev_removal = NULL;
2062 	}
2063 
2064 	spa_destroy_aux_threads(spa);
2065 
2066 	spa_condense_fini(spa);
2067 
2068 	bpobj_close(&spa->spa_deferred_bpobj);
2069 
2070 	spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);
2071 
2072 	/*
2073 	 * Close all vdevs.
2074 	 */
2075 	if (spa->spa_root_vdev)
2076 		vdev_free(spa->spa_root_vdev);
2077 	ASSERT(spa->spa_root_vdev == NULL);
2078 
2079 	/*
2080 	 * Close the dsl pool.
2081 	 */
2082 	if (spa->spa_dsl_pool) {
2083 		dsl_pool_close(spa->spa_dsl_pool);
2084 		spa->spa_dsl_pool = NULL;
2085 		spa->spa_meta_objset = NULL;
2086 	}
2087 
2088 	ddt_unload(spa);
2089 	brt_unload(spa);
2090 	spa_unload_log_sm_metadata(spa);
2091 
2092 	/*
2093 	 * Drop and purge level 2 cache
2094 	 */
2095 	spa_l2cache_drop(spa);
2096 
2097 	if (spa->spa_spares.sav_vdevs) {
2098 		for (int i = 0; i < spa->spa_spares.sav_count; i++)
2099 			vdev_free(spa->spa_spares.sav_vdevs[i]);
2100 		kmem_free(spa->spa_spares.sav_vdevs,
2101 		    spa->spa_spares.sav_count * sizeof (void *));
2102 		spa->spa_spares.sav_vdevs = NULL;
2103 	}
2104 	if (spa->spa_spares.sav_config) {
2105 		nvlist_free(spa->spa_spares.sav_config);
2106 		spa->spa_spares.sav_config = NULL;
2107 	}
2108 	spa->spa_spares.sav_count = 0;
2109 
2110 	if (spa->spa_l2cache.sav_vdevs) {
2111 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
2112 			vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]);
2113 			vdev_free(spa->spa_l2cache.sav_vdevs[i]);
2114 		}
2115 		kmem_free(spa->spa_l2cache.sav_vdevs,
2116 		    spa->spa_l2cache.sav_count * sizeof (void *));
2117 		spa->spa_l2cache.sav_vdevs = NULL;
2118 	}
2119 	if (spa->spa_l2cache.sav_config) {
2120 		nvlist_free(spa->spa_l2cache.sav_config);
2121 		spa->spa_l2cache.sav_config = NULL;
2122 	}
2123 	spa->spa_l2cache.sav_count = 0;
2124 
2125 	spa->spa_async_suspended = 0;
2126 
2127 	spa->spa_indirect_vdevs_loaded = B_FALSE;
2128 
2129 	if (spa->spa_comment != NULL) {
2130 		spa_strfree(spa->spa_comment);
2131 		spa->spa_comment = NULL;
2132 	}
2133 	if (spa->spa_compatibility != NULL) {
2134 		spa_strfree(spa->spa_compatibility);
2135 		spa->spa_compatibility = NULL;
2136 	}
2137 
2138 	spa->spa_raidz_expand = NULL;
2139 
2140 	spa_config_exit(spa, SCL_ALL, spa);
2141 }
2142 
2143 /*
2144  * Load (or re-load) the current list of vdevs describing the active spares for
2145  * this pool.  When this is called, we have some form of basic information in
2146  * 'spa_spares.sav_config'.  We parse this into vdevs, try to open them, and
2147  * then re-generate a more complete list including status information.
2148  */
2149 void
2150 spa_load_spares(spa_t *spa)
2151 {
2152 	nvlist_t **spares;
2153 	uint_t nspares;
2154 	int i;
2155 	vdev_t *vd, *tvd;
2156 
2157 #ifndef _KERNEL
2158 	/*
2159 	 * zdb opens both the current state of the pool and the
2160 	 * checkpointed state (if present), with a different spa_t.
2161 	 *
2162 	 * As spare vdevs are shared among open pools, we skip loading
2163 	 * them when we load the checkpointed state of the pool.
2164 	 */
2165 	if (!spa_writeable(spa))
2166 		return;
2167 #endif
2168 
2169 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
2170 
2171 	/*
2172 	 * First, close and free any existing spare vdevs.
2173 	 */
2174 	if (spa->spa_spares.sav_vdevs) {
2175 		for (i = 0; i < spa->spa_spares.sav_count; i++) {
2176 			vd = spa->spa_spares.sav_vdevs[i];
2177 
2178 			/* Undo the call to spa_activate() below */
2179 			if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
2180 			    B_FALSE)) != NULL && tvd->vdev_isspare)
2181 				spa_spare_remove(tvd);
2182 			vdev_close(vd);
2183 			vdev_free(vd);
2184 		}
2185 
2186 		kmem_free(spa->spa_spares.sav_vdevs,
2187 		    spa->spa_spares.sav_count * sizeof (void *));
2188 	}
2189 
2190 	if (spa->spa_spares.sav_config == NULL)
2191 		nspares = 0;
2192 	else
2193 		VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
2194 		    ZPOOL_CONFIG_SPARES, &spares, &nspares));
2195 
2196 	spa->spa_spares.sav_count = (int)nspares;
2197 	spa->spa_spares.sav_vdevs = NULL;
2198 
2199 	if (nspares == 0)
2200 		return;
2201 
2202 	/*
2203 	 * Construct the array of vdevs, opening them to get status in the
2204 	 * process.   For each spare, there is potentially two different vdev_t
2205 	 * structures associated with it: one in the list of spares (used only
2206 	 * for basic validation purposes) and one in the active vdev
2207 	 * configuration (if it's spared in).  During this phase we open and
2208 	 * validate each vdev on the spare list.  If the vdev also exists in the
2209 	 * active configuration, then we also mark this vdev as an active spare.
2210 	 */
2211 	spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *),
2212 	    KM_SLEEP);
2213 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
2214 		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
2215 		    VDEV_ALLOC_SPARE) == 0);
2216 		ASSERT(vd != NULL);
2217 
2218 		spa->spa_spares.sav_vdevs[i] = vd;
2219 
2220 		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
2221 		    B_FALSE)) != NULL) {
2222 			if (!tvd->vdev_isspare)
2223 				spa_spare_add(tvd);
2224 
2225 			/*
2226 			 * We only mark the spare active if we were successfully
2227 			 * able to load the vdev.  Otherwise, importing a pool
2228 			 * with a bad active spare would result in strange
2229 			 * behavior, because multiple pool would think the spare
2230 			 * is actively in use.
2231 			 *
2232 			 * There is a vulnerability here to an equally bizarre
2233 			 * circumstance, where a dead active spare is later
2234 			 * brought back to life (onlined or otherwise).  Given
2235 			 * the rarity of this scenario, and the extra complexity
2236 			 * it adds, we ignore the possibility.
2237 			 */
2238 			if (!vdev_is_dead(tvd))
2239 				spa_spare_activate(tvd);
2240 		}
2241 
2242 		vd->vdev_top = vd;
2243 		vd->vdev_aux = &spa->spa_spares;
2244 
2245 		if (vdev_open(vd) != 0)
2246 			continue;
2247 
2248 		if (vdev_validate_aux(vd) == 0)
2249 			spa_spare_add(vd);
2250 	}
2251 
2252 	/*
2253 	 * Recompute the stashed list of spares, with status information
2254 	 * this time.
2255 	 */
2256 	fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES);
2257 
2258 	spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *),
2259 	    KM_SLEEP);
2260 	for (i = 0; i < spa->spa_spares.sav_count; i++)
2261 		spares[i] = vdev_config_generate(spa,
2262 		    spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE);
2263 	fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
2264 	    ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
2265 	    spa->spa_spares.sav_count);
2266 	for (i = 0; i < spa->spa_spares.sav_count; i++)
2267 		nvlist_free(spares[i]);
2268 	kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *));
2269 }
2270 
2271 /*
2272  * Load (or re-load) the current list of vdevs describing the active l2cache for
2273  * this pool.  When this is called, we have some form of basic information in
2274  * 'spa_l2cache.sav_config'.  We parse this into vdevs, try to open them, and
2275  * then re-generate a more complete list including status information.
2276  * Devices which are already active have their details maintained, and are
2277  * not re-opened.
2278  */
2279 void
2280 spa_load_l2cache(spa_t *spa)
2281 {
2282 	nvlist_t **l2cache = NULL;
2283 	uint_t nl2cache;
2284 	int i, j, oldnvdevs;
2285 	uint64_t guid;
2286 	vdev_t *vd, **oldvdevs, **newvdevs;
2287 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
2288 
2289 #ifndef _KERNEL
2290 	/*
2291 	 * zdb opens both the current state of the pool and the
2292 	 * checkpointed state (if present), with a different spa_t.
2293 	 *
2294 	 * As L2 caches are part of the ARC which is shared among open
2295 	 * pools, we skip loading them when we load the checkpointed
2296 	 * state of the pool.
2297 	 */
2298 	if (!spa_writeable(spa))
2299 		return;
2300 #endif
2301 
2302 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
2303 
2304 	oldvdevs = sav->sav_vdevs;
2305 	oldnvdevs = sav->sav_count;
2306 	sav->sav_vdevs = NULL;
2307 	sav->sav_count = 0;
2308 
2309 	if (sav->sav_config == NULL) {
2310 		nl2cache = 0;
2311 		newvdevs = NULL;
2312 		goto out;
2313 	}
2314 
2315 	VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config,
2316 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache));
2317 	newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP);
2318 
2319 	/*
2320 	 * Process new nvlist of vdevs.
2321 	 */
2322 	for (i = 0; i < nl2cache; i++) {
2323 		guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID);
2324 
2325 		newvdevs[i] = NULL;
2326 		for (j = 0; j < oldnvdevs; j++) {
2327 			vd = oldvdevs[j];
2328 			if (vd != NULL && guid == vd->vdev_guid) {
2329 				/*
2330 				 * Retain previous vdev for add/remove ops.
2331 				 */
2332 				newvdevs[i] = vd;
2333 				oldvdevs[j] = NULL;
2334 				break;
2335 			}
2336 		}
2337 
2338 		if (newvdevs[i] == NULL) {
2339 			/*
2340 			 * Create new vdev
2341 			 */
2342 			VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
2343 			    VDEV_ALLOC_L2CACHE) == 0);
2344 			ASSERT(vd != NULL);
2345 			newvdevs[i] = vd;
2346 
2347 			/*
2348 			 * Commit this vdev as an l2cache device,
2349 			 * even if it fails to open.
2350 			 */
2351 			spa_l2cache_add(vd);
2352 
2353 			vd->vdev_top = vd;
2354 			vd->vdev_aux = sav;
2355 
2356 			spa_l2cache_activate(vd);
2357 
2358 			if (vdev_open(vd) != 0)
2359 				continue;
2360 
2361 			(void) vdev_validate_aux(vd);
2362 
2363 			if (!vdev_is_dead(vd))
2364 				l2arc_add_vdev(spa, vd);
2365 
2366 			/*
2367 			 * Upon cache device addition to a pool or pool
2368 			 * creation with a cache device or if the header
2369 			 * of the device is invalid we issue an async
2370 			 * TRIM command for the whole device which will
2371 			 * execute if l2arc_trim_ahead > 0.
2372 			 */
2373 			spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM);
2374 		}
2375 	}
2376 
2377 	sav->sav_vdevs = newvdevs;
2378 	sav->sav_count = (int)nl2cache;
2379 
2380 	/*
2381 	 * Recompute the stashed list of l2cache devices, with status
2382 	 * information this time.
2383 	 */
2384 	fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE);
2385 
2386 	if (sav->sav_count > 0)
2387 		l2cache = kmem_alloc(sav->sav_count * sizeof (void *),
2388 		    KM_SLEEP);
2389 	for (i = 0; i < sav->sav_count; i++)
2390 		l2cache[i] = vdev_config_generate(spa,
2391 		    sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE);
2392 	fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE,
2393 	    (const nvlist_t * const *)l2cache, sav->sav_count);
2394 
2395 out:
2396 	/*
2397 	 * Purge vdevs that were dropped
2398 	 */
2399 	if (oldvdevs) {
2400 		for (i = 0; i < oldnvdevs; i++) {
2401 			uint64_t pool;
2402 
2403 			vd = oldvdevs[i];
2404 			if (vd != NULL) {
2405 				ASSERT(vd->vdev_isl2cache);
2406 
2407 				if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
2408 				    pool != 0ULL && l2arc_vdev_present(vd))
2409 					l2arc_remove_vdev(vd);
2410 				vdev_clear_stats(vd);
2411 				vdev_free(vd);
2412 			}
2413 		}
2414 
2415 		kmem_free(oldvdevs, oldnvdevs * sizeof (void *));
2416 	}
2417 
2418 	for (i = 0; i < sav->sav_count; i++)
2419 		nvlist_free(l2cache[i]);
2420 	if (sav->sav_count)
2421 		kmem_free(l2cache, sav->sav_count * sizeof (void *));
2422 }
2423 
2424 static int
2425 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
2426 {
2427 	dmu_buf_t *db;
2428 	char *packed = NULL;
2429 	size_t nvsize = 0;
2430 	int error;
2431 	*value = NULL;
2432 
2433 	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
2434 	if (error)
2435 		return (error);
2436 
2437 	nvsize = *(uint64_t *)db->db_data;
2438 	dmu_buf_rele(db, FTAG);
2439 
2440 	packed = vmem_alloc(nvsize, KM_SLEEP);
2441 	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed,
2442 	    DMU_READ_PREFETCH);
2443 	if (error == 0)
2444 		error = nvlist_unpack(packed, nvsize, value, 0);
2445 	vmem_free(packed, nvsize);
2446 
2447 	return (error);
2448 }
2449 
2450 /*
2451  * Concrete top-level vdevs that are not missing and are not logs. At every
2452  * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds.
2453  */
2454 static uint64_t
2455 spa_healthy_core_tvds(spa_t *spa)
2456 {
2457 	vdev_t *rvd = spa->spa_root_vdev;
2458 	uint64_t tvds = 0;
2459 
2460 	for (uint64_t i = 0; i < rvd->vdev_children; i++) {
2461 		vdev_t *vd = rvd->vdev_child[i];
2462 		if (vd->vdev_islog)
2463 			continue;
2464 		if (vdev_is_concrete(vd) && !vdev_is_dead(vd))
2465 			tvds++;
2466 	}
2467 
2468 	return (tvds);
2469 }
2470 
2471 /*
2472  * Checks to see if the given vdev could not be opened, in which case we post a
2473  * sysevent to notify the autoreplace code that the device has been removed.
2474  */
2475 static void
2476 spa_check_removed(vdev_t *vd)
2477 {
2478 	for (uint64_t c = 0; c < vd->vdev_children; c++)
2479 		spa_check_removed(vd->vdev_child[c]);
2480 
2481 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) &&
2482 	    vdev_is_concrete(vd)) {
2483 		zfs_post_autoreplace(vd->vdev_spa, vd);
2484 		spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK);
2485 	}
2486 }
2487 
2488 static int
2489 spa_check_for_missing_logs(spa_t *spa)
2490 {
2491 	vdev_t *rvd = spa->spa_root_vdev;
2492 
2493 	/*
2494 	 * If we're doing a normal import, then build up any additional
2495 	 * diagnostic information about missing log devices.
2496 	 * We'll pass this up to the user for further processing.
2497 	 */
2498 	if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) {
2499 		nvlist_t **child, *nv;
2500 		uint64_t idx = 0;
2501 
2502 		child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *),
2503 		    KM_SLEEP);
2504 		nv = fnvlist_alloc();
2505 
2506 		for (uint64_t c = 0; c < rvd->vdev_children; c++) {
2507 			vdev_t *tvd = rvd->vdev_child[c];
2508 
2509 			/*
2510 			 * We consider a device as missing only if it failed
2511 			 * to open (i.e. offline or faulted is not considered
2512 			 * as missing).
2513 			 */
2514 			if (tvd->vdev_islog &&
2515 			    tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
2516 				child[idx++] = vdev_config_generate(spa, tvd,
2517 				    B_FALSE, VDEV_CONFIG_MISSING);
2518 			}
2519 		}
2520 
2521 		if (idx > 0) {
2522 			fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2523 			    (const nvlist_t * const *)child, idx);
2524 			fnvlist_add_nvlist(spa->spa_load_info,
2525 			    ZPOOL_CONFIG_MISSING_DEVICES, nv);
2526 
2527 			for (uint64_t i = 0; i < idx; i++)
2528 				nvlist_free(child[i]);
2529 		}
2530 		nvlist_free(nv);
2531 		kmem_free(child, rvd->vdev_children * sizeof (char **));
2532 
2533 		if (idx > 0) {
2534 			spa_load_failed(spa, "some log devices are missing");
2535 			vdev_dbgmsg_print_tree(rvd, 2);
2536 			return (SET_ERROR(ENXIO));
2537 		}
2538 	} else {
2539 		for (uint64_t c = 0; c < rvd->vdev_children; c++) {
2540 			vdev_t *tvd = rvd->vdev_child[c];
2541 
2542 			if (tvd->vdev_islog &&
2543 			    tvd->vdev_state == VDEV_STATE_CANT_OPEN) {
2544 				spa_set_log_state(spa, SPA_LOG_CLEAR);
2545 				spa_load_note(spa, "some log devices are "
2546 				    "missing, ZIL is dropped.");
2547 				vdev_dbgmsg_print_tree(rvd, 2);
2548 				break;
2549 			}
2550 		}
2551 	}
2552 
2553 	return (0);
2554 }
2555 
2556 /*
2557  * Check for missing log devices
2558  */
2559 static boolean_t
2560 spa_check_logs(spa_t *spa)
2561 {
2562 	boolean_t rv = B_FALSE;
2563 	dsl_pool_t *dp = spa_get_dsl(spa);
2564 
2565 	switch (spa->spa_log_state) {
2566 	default:
2567 		break;
2568 	case SPA_LOG_MISSING:
2569 		/* need to recheck in case slog has been restored */
2570 	case SPA_LOG_UNKNOWN:
2571 		rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
2572 		    zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0);
2573 		if (rv)
2574 			spa_set_log_state(spa, SPA_LOG_MISSING);
2575 		break;
2576 	}
2577 	return (rv);
2578 }
2579 
2580 /*
2581  * Passivate any log vdevs (note, does not apply to embedded log metaslabs).
2582  */
2583 static boolean_t
2584 spa_passivate_log(spa_t *spa)
2585 {
2586 	vdev_t *rvd = spa->spa_root_vdev;
2587 	boolean_t slog_found = B_FALSE;
2588 
2589 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
2590 
2591 	for (int c = 0; c < rvd->vdev_children; c++) {
2592 		vdev_t *tvd = rvd->vdev_child[c];
2593 
2594 		if (tvd->vdev_islog) {
2595 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
2596 			metaslab_group_passivate(tvd->vdev_mg);
2597 			slog_found = B_TRUE;
2598 		}
2599 	}
2600 
2601 	return (slog_found);
2602 }
2603 
2604 /*
2605  * Activate any log vdevs (note, does not apply to embedded log metaslabs).
2606  */
2607 static void
2608 spa_activate_log(spa_t *spa)
2609 {
2610 	vdev_t *rvd = spa->spa_root_vdev;
2611 
2612 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
2613 
2614 	for (int c = 0; c < rvd->vdev_children; c++) {
2615 		vdev_t *tvd = rvd->vdev_child[c];
2616 
2617 		if (tvd->vdev_islog) {
2618 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
2619 			metaslab_group_activate(tvd->vdev_mg);
2620 		}
2621 	}
2622 }
2623 
2624 int
2625 spa_reset_logs(spa_t *spa)
2626 {
2627 	int error;
2628 
2629 	error = dmu_objset_find(spa_name(spa), zil_reset,
2630 	    NULL, DS_FIND_CHILDREN);
2631 	if (error == 0) {
2632 		/*
2633 		 * We successfully offlined the log device, sync out the
2634 		 * current txg so that the "stubby" block can be removed
2635 		 * by zil_sync().
2636 		 */
2637 		txg_wait_synced(spa->spa_dsl_pool, 0);
2638 	}
2639 	return (error);
2640 }
2641 
2642 static void
2643 spa_aux_check_removed(spa_aux_vdev_t *sav)
2644 {
2645 	for (int i = 0; i < sav->sav_count; i++)
2646 		spa_check_removed(sav->sav_vdevs[i]);
2647 }
2648 
2649 void
2650 spa_claim_notify(zio_t *zio)
2651 {
2652 	spa_t *spa = zio->io_spa;
2653 
2654 	if (zio->io_error)
2655 		return;
2656 
2657 	mutex_enter(&spa->spa_props_lock);	/* any mutex will do */
2658 	if (spa->spa_claim_max_txg < zio->io_bp->blk_birth)
2659 		spa->spa_claim_max_txg = zio->io_bp->blk_birth;
2660 	mutex_exit(&spa->spa_props_lock);
2661 }
2662 
2663 typedef struct spa_load_error {
2664 	boolean_t	sle_verify_data;
2665 	uint64_t	sle_meta_count;
2666 	uint64_t	sle_data_count;
2667 } spa_load_error_t;
2668 
2669 static void
2670 spa_load_verify_done(zio_t *zio)
2671 {
2672 	blkptr_t *bp = zio->io_bp;
2673 	spa_load_error_t *sle = zio->io_private;
2674 	dmu_object_type_t type = BP_GET_TYPE(bp);
2675 	int error = zio->io_error;
2676 	spa_t *spa = zio->io_spa;
2677 
2678 	abd_free(zio->io_abd);
2679 	if (error) {
2680 		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
2681 		    type != DMU_OT_INTENT_LOG)
2682 			atomic_inc_64(&sle->sle_meta_count);
2683 		else
2684 			atomic_inc_64(&sle->sle_data_count);
2685 	}
2686 
2687 	mutex_enter(&spa->spa_scrub_lock);
2688 	spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp);
2689 	cv_broadcast(&spa->spa_scrub_io_cv);
2690 	mutex_exit(&spa->spa_scrub_lock);
2691 }
2692 
2693 /*
2694  * Maximum number of inflight bytes is the log2 fraction of the arc size.
2695  * By default, we set it to 1/16th of the arc.
2696  */
2697 static uint_t spa_load_verify_shift = 4;
2698 static int spa_load_verify_metadata = B_TRUE;
2699 static int spa_load_verify_data = B_TRUE;
2700 
2701 static int
2702 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
2703     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
2704 {
2705 	zio_t *rio = arg;
2706 	spa_load_error_t *sle = rio->io_private;
2707 
2708 	(void) zilog, (void) dnp;
2709 
2710 	/*
2711 	 * Note: normally this routine will not be called if
2712 	 * spa_load_verify_metadata is not set.  However, it may be useful
2713 	 * to manually set the flag after the traversal has begun.
2714 	 */
2715 	if (!spa_load_verify_metadata)
2716 		return (0);
2717 
2718 	/*
2719 	 * Sanity check the block pointer in order to detect obvious damage
2720 	 * before using the contents in subsequent checks or in zio_read().
2721 	 * When damaged consider it to be a metadata error since we cannot
2722 	 * trust the BP_GET_TYPE and BP_GET_LEVEL values.
2723 	 */
2724 	if (!zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
2725 		atomic_inc_64(&sle->sle_meta_count);
2726 		return (0);
2727 	}
2728 
2729 	if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) ||
2730 	    BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp))
2731 		return (0);
2732 
2733 	if (!BP_IS_METADATA(bp) &&
2734 	    (!spa_load_verify_data || !sle->sle_verify_data))
2735 		return (0);
2736 
2737 	uint64_t maxinflight_bytes =
2738 	    arc_target_bytes() >> spa_load_verify_shift;
2739 	size_t size = BP_GET_PSIZE(bp);
2740 
2741 	mutex_enter(&spa->spa_scrub_lock);
2742 	while (spa->spa_load_verify_bytes >= maxinflight_bytes)
2743 		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
2744 	spa->spa_load_verify_bytes += size;
2745 	mutex_exit(&spa->spa_scrub_lock);
2746 
2747 	zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size,
2748 	    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
2749 	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
2750 	    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
2751 	return (0);
2752 }
2753 
2754 static int
2755 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
2756 {
2757 	(void) dp, (void) arg;
2758 
2759 	if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN)
2760 		return (SET_ERROR(ENAMETOOLONG));
2761 
2762 	return (0);
2763 }
2764 
2765 static int
2766 spa_load_verify(spa_t *spa)
2767 {
2768 	zio_t *rio;
2769 	spa_load_error_t sle = { 0 };
2770 	zpool_load_policy_t policy;
2771 	boolean_t verify_ok = B_FALSE;
2772 	int error = 0;
2773 
2774 	zpool_get_load_policy(spa->spa_config, &policy);
2775 
2776 	if (policy.zlp_rewind & ZPOOL_NEVER_REWIND ||
2777 	    policy.zlp_maxmeta == UINT64_MAX)
2778 		return (0);
2779 
2780 	dsl_pool_config_enter(spa->spa_dsl_pool, FTAG);
2781 	error = dmu_objset_find_dp(spa->spa_dsl_pool,
2782 	    spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL,
2783 	    DS_FIND_CHILDREN);
2784 	dsl_pool_config_exit(spa->spa_dsl_pool, FTAG);
2785 	if (error != 0)
2786 		return (error);
2787 
2788 	/*
2789 	 * Verify data only if we are rewinding or error limit was set.
2790 	 * Otherwise nothing except dbgmsg care about it to waste time.
2791 	 */
2792 	sle.sle_verify_data = (policy.zlp_rewind & ZPOOL_REWIND_MASK) ||
2793 	    (policy.zlp_maxdata < UINT64_MAX);
2794 
2795 	rio = zio_root(spa, NULL, &sle,
2796 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
2797 
2798 	if (spa_load_verify_metadata) {
2799 		if (spa->spa_extreme_rewind) {
2800 			spa_load_note(spa, "performing a complete scan of the "
2801 			    "pool since extreme rewind is on. This may take "
2802 			    "a very long time.\n  (spa_load_verify_data=%u, "
2803 			    "spa_load_verify_metadata=%u)",
2804 			    spa_load_verify_data, spa_load_verify_metadata);
2805 		}
2806 
2807 		error = traverse_pool(spa, spa->spa_verify_min_txg,
2808 		    TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA |
2809 		    TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio);
2810 	}
2811 
2812 	(void) zio_wait(rio);
2813 	ASSERT0(spa->spa_load_verify_bytes);
2814 
2815 	spa->spa_load_meta_errors = sle.sle_meta_count;
2816 	spa->spa_load_data_errors = sle.sle_data_count;
2817 
2818 	if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) {
2819 		spa_load_note(spa, "spa_load_verify found %llu metadata errors "
2820 		    "and %llu data errors", (u_longlong_t)sle.sle_meta_count,
2821 		    (u_longlong_t)sle.sle_data_count);
2822 	}
2823 
2824 	if (spa_load_verify_dryrun ||
2825 	    (!error && sle.sle_meta_count <= policy.zlp_maxmeta &&
2826 	    sle.sle_data_count <= policy.zlp_maxdata)) {
2827 		int64_t loss = 0;
2828 
2829 		verify_ok = B_TRUE;
2830 		spa->spa_load_txg = spa->spa_uberblock.ub_txg;
2831 		spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp;
2832 
2833 		loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts;
2834 		fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME,
2835 		    spa->spa_load_txg_ts);
2836 		fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME,
2837 		    loss);
2838 		fnvlist_add_uint64(spa->spa_load_info,
2839 		    ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count);
2840 		fnvlist_add_uint64(spa->spa_load_info,
2841 		    ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count);
2842 	} else {
2843 		spa->spa_load_max_txg = spa->spa_uberblock.ub_txg;
2844 	}
2845 
2846 	if (spa_load_verify_dryrun)
2847 		return (0);
2848 
2849 	if (error) {
2850 		if (error != ENXIO && error != EIO)
2851 			error = SET_ERROR(EIO);
2852 		return (error);
2853 	}
2854 
2855 	return (verify_ok ? 0 : EIO);
2856 }
2857 
2858 /*
2859  * Find a value in the pool props object.
2860  */
2861 static void
2862 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val)
2863 {
2864 	(void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object,
2865 	    zpool_prop_to_name(prop), sizeof (uint64_t), 1, val);
2866 }
2867 
2868 /*
2869  * Find a value in the pool directory object.
2870  */
2871 static int
2872 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent)
2873 {
2874 	int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
2875 	    name, sizeof (uint64_t), 1, val);
2876 
2877 	if (error != 0 && (error != ENOENT || log_enoent)) {
2878 		spa_load_failed(spa, "couldn't get '%s' value in MOS directory "
2879 		    "[error=%d]", name, error);
2880 	}
2881 
2882 	return (error);
2883 }
2884 
2885 static int
2886 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err)
2887 {
2888 	vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux);
2889 	return (SET_ERROR(err));
2890 }
2891 
2892 boolean_t
2893 spa_livelist_delete_check(spa_t *spa)
2894 {
2895 	return (spa->spa_livelists_to_delete != 0);
2896 }
2897 
2898 static boolean_t
2899 spa_livelist_delete_cb_check(void *arg, zthr_t *z)
2900 {
2901 	(void) z;
2902 	spa_t *spa = arg;
2903 	return (spa_livelist_delete_check(spa));
2904 }
2905 
2906 static int
2907 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
2908 {
2909 	spa_t *spa = arg;
2910 	zio_free(spa, tx->tx_txg, bp);
2911 	dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD,
2912 	    -bp_get_dsize_sync(spa, bp),
2913 	    -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx);
2914 	return (0);
2915 }
2916 
2917 static int
2918 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp)
2919 {
2920 	int err;
2921 	zap_cursor_t zc;
2922 	zap_attribute_t za;
2923 	zap_cursor_init(&zc, os, zap_obj);
2924 	err = zap_cursor_retrieve(&zc, &za);
2925 	zap_cursor_fini(&zc);
2926 	if (err == 0)
2927 		*llp = za.za_first_integer;
2928 	return (err);
2929 }
2930 
2931 /*
2932  * Components of livelist deletion that must be performed in syncing
2933  * context: freeing block pointers and updating the pool-wide data
2934  * structures to indicate how much work is left to do
2935  */
2936 typedef struct sublist_delete_arg {
2937 	spa_t *spa;
2938 	dsl_deadlist_t *ll;
2939 	uint64_t key;
2940 	bplist_t *to_free;
2941 } sublist_delete_arg_t;
2942 
2943 static void
2944 sublist_delete_sync(void *arg, dmu_tx_t *tx)
2945 {
2946 	sublist_delete_arg_t *sda = arg;
2947 	spa_t *spa = sda->spa;
2948 	dsl_deadlist_t *ll = sda->ll;
2949 	uint64_t key = sda->key;
2950 	bplist_t *to_free = sda->to_free;
2951 
2952 	bplist_iterate(to_free, delete_blkptr_cb, spa, tx);
2953 	dsl_deadlist_remove_entry(ll, key, tx);
2954 }
2955 
2956 typedef struct livelist_delete_arg {
2957 	spa_t *spa;
2958 	uint64_t ll_obj;
2959 	uint64_t zap_obj;
2960 } livelist_delete_arg_t;
2961 
2962 static void
2963 livelist_delete_sync(void *arg, dmu_tx_t *tx)
2964 {
2965 	livelist_delete_arg_t *lda = arg;
2966 	spa_t *spa = lda->spa;
2967 	uint64_t ll_obj = lda->ll_obj;
2968 	uint64_t zap_obj = lda->zap_obj;
2969 	objset_t *mos = spa->spa_meta_objset;
2970 	uint64_t count;
2971 
2972 	/* free the livelist and decrement the feature count */
2973 	VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx));
2974 	dsl_deadlist_free(mos, ll_obj, tx);
2975 	spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx);
2976 	VERIFY0(zap_count(mos, zap_obj, &count));
2977 	if (count == 0) {
2978 		/* no more livelists to delete */
2979 		VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT,
2980 		    DMU_POOL_DELETED_CLONES, tx));
2981 		VERIFY0(zap_destroy(mos, zap_obj, tx));
2982 		spa->spa_livelists_to_delete = 0;
2983 		spa_notify_waiters(spa);
2984 	}
2985 }
2986 
2987 /*
2988  * Load in the value for the livelist to be removed and open it. Then,
2989  * load its first sublist and determine which block pointers should actually
2990  * be freed. Then, call a synctask which performs the actual frees and updates
2991  * the pool-wide livelist data.
2992  */
2993 static void
2994 spa_livelist_delete_cb(void *arg, zthr_t *z)
2995 {
2996 	spa_t *spa = arg;
2997 	uint64_t ll_obj = 0, count;
2998 	objset_t *mos = spa->spa_meta_objset;
2999 	uint64_t zap_obj = spa->spa_livelists_to_delete;
3000 	/*
3001 	 * Determine the next livelist to delete. This function should only
3002 	 * be called if there is at least one deleted clone.
3003 	 */
3004 	VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj));
3005 	VERIFY0(zap_count(mos, ll_obj, &count));
3006 	if (count > 0) {
3007 		dsl_deadlist_t *ll;
3008 		dsl_deadlist_entry_t *dle;
3009 		bplist_t to_free;
3010 		ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP);
3011 		dsl_deadlist_open(ll, mos, ll_obj);
3012 		dle = dsl_deadlist_first(ll);
3013 		ASSERT3P(dle, !=, NULL);
3014 		bplist_create(&to_free);
3015 		int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free,
3016 		    z, NULL);
3017 		if (err == 0) {
3018 			sublist_delete_arg_t sync_arg = {
3019 			    .spa = spa,
3020 			    .ll = ll,
3021 			    .key = dle->dle_mintxg,
3022 			    .to_free = &to_free
3023 			};
3024 			zfs_dbgmsg("deleting sublist (id %llu) from"
3025 			    " livelist %llu, %lld remaining",
3026 			    (u_longlong_t)dle->dle_bpobj.bpo_object,
3027 			    (u_longlong_t)ll_obj, (longlong_t)count - 1);
3028 			VERIFY0(dsl_sync_task(spa_name(spa), NULL,
3029 			    sublist_delete_sync, &sync_arg, 0,
3030 			    ZFS_SPACE_CHECK_DESTROY));
3031 		} else {
3032 			VERIFY3U(err, ==, EINTR);
3033 		}
3034 		bplist_clear(&to_free);
3035 		bplist_destroy(&to_free);
3036 		dsl_deadlist_close(ll);
3037 		kmem_free(ll, sizeof (dsl_deadlist_t));
3038 	} else {
3039 		livelist_delete_arg_t sync_arg = {
3040 		    .spa = spa,
3041 		    .ll_obj = ll_obj,
3042 		    .zap_obj = zap_obj
3043 		};
3044 		zfs_dbgmsg("deletion of livelist %llu completed",
3045 		    (u_longlong_t)ll_obj);
3046 		VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync,
3047 		    &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY));
3048 	}
3049 }
3050 
3051 static void
3052 spa_start_livelist_destroy_thread(spa_t *spa)
3053 {
3054 	ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL);
3055 	spa->spa_livelist_delete_zthr =
3056 	    zthr_create("z_livelist_destroy",
3057 	    spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa,
3058 	    minclsyspri);
3059 }
3060 
3061 typedef struct livelist_new_arg {
3062 	bplist_t *allocs;
3063 	bplist_t *frees;
3064 } livelist_new_arg_t;
3065 
3066 static int
3067 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
3068     dmu_tx_t *tx)
3069 {
3070 	ASSERT(tx == NULL);
3071 	livelist_new_arg_t *lna = arg;
3072 	if (bp_freed) {
3073 		bplist_append(lna->frees, bp);
3074 	} else {
3075 		bplist_append(lna->allocs, bp);
3076 		zfs_livelist_condense_new_alloc++;
3077 	}
3078 	return (0);
3079 }
3080 
3081 typedef struct livelist_condense_arg {
3082 	spa_t *spa;
3083 	bplist_t to_keep;
3084 	uint64_t first_size;
3085 	uint64_t next_size;
3086 } livelist_condense_arg_t;
3087 
3088 static void
3089 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx)
3090 {
3091 	livelist_condense_arg_t *lca = arg;
3092 	spa_t *spa = lca->spa;
3093 	bplist_t new_frees;
3094 	dsl_dataset_t *ds = spa->spa_to_condense.ds;
3095 
3096 	/* Have we been cancelled? */
3097 	if (spa->spa_to_condense.cancelled) {
3098 		zfs_livelist_condense_sync_cancel++;
3099 		goto out;
3100 	}
3101 
3102 	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
3103 	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
3104 	dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist;
3105 
3106 	/*
3107 	 * It's possible that the livelist was changed while the zthr was
3108 	 * running. Therefore, we need to check for new blkptrs in the two
3109 	 * entries being condensed and continue to track them in the livelist.
3110 	 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl),
3111 	 * it's possible that the newly added blkptrs are FREEs or ALLOCs so
3112 	 * we need to sort them into two different bplists.
3113 	 */
3114 	uint64_t first_obj = first->dle_bpobj.bpo_object;
3115 	uint64_t next_obj = next->dle_bpobj.bpo_object;
3116 	uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs;
3117 	uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs;
3118 
3119 	bplist_create(&new_frees);
3120 	livelist_new_arg_t new_bps = {
3121 	    .allocs = &lca->to_keep,
3122 	    .frees = &new_frees,
3123 	};
3124 
3125 	if (cur_first_size > lca->first_size) {
3126 		VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj,
3127 		    livelist_track_new_cb, &new_bps, lca->first_size));
3128 	}
3129 	if (cur_next_size > lca->next_size) {
3130 		VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj,
3131 		    livelist_track_new_cb, &new_bps, lca->next_size));
3132 	}
3133 
3134 	dsl_deadlist_clear_entry(first, ll, tx);
3135 	ASSERT(bpobj_is_empty(&first->dle_bpobj));
3136 	dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx);
3137 
3138 	bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx);
3139 	bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx);
3140 	bplist_destroy(&new_frees);
3141 
3142 	char dsname[ZFS_MAX_DATASET_NAME_LEN];
3143 	dsl_dataset_name(ds, dsname);
3144 	zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu "
3145 	    "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu "
3146 	    "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname,
3147 	    (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj,
3148 	    (u_longlong_t)cur_first_size, (u_longlong_t)next_obj,
3149 	    (u_longlong_t)cur_next_size,
3150 	    (u_longlong_t)first->dle_bpobj.bpo_object,
3151 	    (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs);
3152 out:
3153 	dmu_buf_rele(ds->ds_dbuf, spa);
3154 	spa->spa_to_condense.ds = NULL;
3155 	bplist_clear(&lca->to_keep);
3156 	bplist_destroy(&lca->to_keep);
3157 	kmem_free(lca, sizeof (livelist_condense_arg_t));
3158 	spa->spa_to_condense.syncing = B_FALSE;
3159 }
3160 
3161 static void
3162 spa_livelist_condense_cb(void *arg, zthr_t *t)
3163 {
3164 	while (zfs_livelist_condense_zthr_pause &&
3165 	    !(zthr_has_waiters(t) || zthr_iscancelled(t)))
3166 		delay(1);
3167 
3168 	spa_t *spa = arg;
3169 	dsl_deadlist_entry_t *first = spa->spa_to_condense.first;
3170 	dsl_deadlist_entry_t *next = spa->spa_to_condense.next;
3171 	uint64_t first_size, next_size;
3172 
3173 	livelist_condense_arg_t *lca =
3174 	    kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP);
3175 	bplist_create(&lca->to_keep);
3176 
3177 	/*
3178 	 * Process the livelists (matching FREEs and ALLOCs) in open context
3179 	 * so we have minimal work in syncing context to condense.
3180 	 *
3181 	 * We save bpobj sizes (first_size and next_size) to use later in
3182 	 * syncing context to determine if entries were added to these sublists
3183 	 * while in open context. This is possible because the clone is still
3184 	 * active and open for normal writes and we want to make sure the new,
3185 	 * unprocessed blockpointers are inserted into the livelist normally.
3186 	 *
3187 	 * Note that dsl_process_sub_livelist() both stores the size number of
3188 	 * blockpointers and iterates over them while the bpobj's lock held, so
3189 	 * the sizes returned to us are consistent which what was actually
3190 	 * processed.
3191 	 */
3192 	int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t,
3193 	    &first_size);
3194 	if (err == 0)
3195 		err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep,
3196 		    t, &next_size);
3197 
3198 	if (err == 0) {
3199 		while (zfs_livelist_condense_sync_pause &&
3200 		    !(zthr_has_waiters(t) || zthr_iscancelled(t)))
3201 			delay(1);
3202 
3203 		dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
3204 		dmu_tx_mark_netfree(tx);
3205 		dmu_tx_hold_space(tx, 1);
3206 		err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE);
3207 		if (err == 0) {
3208 			/*
3209 			 * Prevent the condense zthr restarting before
3210 			 * the synctask completes.
3211 			 */
3212 			spa->spa_to_condense.syncing = B_TRUE;
3213 			lca->spa = spa;
3214 			lca->first_size = first_size;
3215 			lca->next_size = next_size;
3216 			dsl_sync_task_nowait(spa_get_dsl(spa),
3217 			    spa_livelist_condense_sync, lca, tx);
3218 			dmu_tx_commit(tx);
3219 			return;
3220 		}
3221 	}
3222 	/*
3223 	 * Condensing can not continue: either it was externally stopped or
3224 	 * we were unable to assign to a tx because the pool has run out of
3225 	 * space. In the second case, we'll just end up trying to condense
3226 	 * again in a later txg.
3227 	 */
3228 	ASSERT(err != 0);
3229 	bplist_clear(&lca->to_keep);
3230 	bplist_destroy(&lca->to_keep);
3231 	kmem_free(lca, sizeof (livelist_condense_arg_t));
3232 	dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa);
3233 	spa->spa_to_condense.ds = NULL;
3234 	if (err == EINTR)
3235 		zfs_livelist_condense_zthr_cancel++;
3236 }
3237 
3238 /*
3239  * Check that there is something to condense but that a condense is not
3240  * already in progress and that condensing has not been cancelled.
3241  */
3242 static boolean_t
3243 spa_livelist_condense_cb_check(void *arg, zthr_t *z)
3244 {
3245 	(void) z;
3246 	spa_t *spa = arg;
3247 	if ((spa->spa_to_condense.ds != NULL) &&
3248 	    (spa->spa_to_condense.syncing == B_FALSE) &&
3249 	    (spa->spa_to_condense.cancelled == B_FALSE)) {
3250 		return (B_TRUE);
3251 	}
3252 	return (B_FALSE);
3253 }
3254 
3255 static void
3256 spa_start_livelist_condensing_thread(spa_t *spa)
3257 {
3258 	spa->spa_to_condense.ds = NULL;
3259 	spa->spa_to_condense.first = NULL;
3260 	spa->spa_to_condense.next = NULL;
3261 	spa->spa_to_condense.syncing = B_FALSE;
3262 	spa->spa_to_condense.cancelled = B_FALSE;
3263 
3264 	ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL);
3265 	spa->spa_livelist_condense_zthr =
3266 	    zthr_create("z_livelist_condense",
3267 	    spa_livelist_condense_cb_check,
3268 	    spa_livelist_condense_cb, spa, minclsyspri);
3269 }
3270 
3271 static void
3272 spa_spawn_aux_threads(spa_t *spa)
3273 {
3274 	ASSERT(spa_writeable(spa));
3275 
3276 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
3277 
3278 	spa_start_raidz_expansion_thread(spa);
3279 	spa_start_indirect_condensing_thread(spa);
3280 	spa_start_livelist_destroy_thread(spa);
3281 	spa_start_livelist_condensing_thread(spa);
3282 
3283 	ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL);
3284 	spa->spa_checkpoint_discard_zthr =
3285 	    zthr_create("z_checkpoint_discard",
3286 	    spa_checkpoint_discard_thread_check,
3287 	    spa_checkpoint_discard_thread, spa, minclsyspri);
3288 }
3289 
3290 /*
3291  * Fix up config after a partly-completed split.  This is done with the
3292  * ZPOOL_CONFIG_SPLIT nvlist.  Both the splitting pool and the split-off
3293  * pool have that entry in their config, but only the splitting one contains
3294  * a list of all the guids of the vdevs that are being split off.
3295  *
3296  * This function determines what to do with that list: either rejoin
3297  * all the disks to the pool, or complete the splitting process.  To attempt
3298  * the rejoin, each disk that is offlined is marked online again, and
3299  * we do a reopen() call.  If the vdev label for every disk that was
3300  * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL)
3301  * then we call vdev_split() on each disk, and complete the split.
3302  *
3303  * Otherwise we leave the config alone, with all the vdevs in place in
3304  * the original pool.
3305  */
3306 static void
3307 spa_try_repair(spa_t *spa, nvlist_t *config)
3308 {
3309 	uint_t extracted;
3310 	uint64_t *glist;
3311 	uint_t i, gcount;
3312 	nvlist_t *nvl;
3313 	vdev_t **vd;
3314 	boolean_t attempt_reopen;
3315 
3316 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0)
3317 		return;
3318 
3319 	/* check that the config is complete */
3320 	if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
3321 	    &glist, &gcount) != 0)
3322 		return;
3323 
3324 	vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP);
3325 
3326 	/* attempt to online all the vdevs & validate */
3327 	attempt_reopen = B_TRUE;
3328 	for (i = 0; i < gcount; i++) {
3329 		if (glist[i] == 0)	/* vdev is hole */
3330 			continue;
3331 
3332 		vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE);
3333 		if (vd[i] == NULL) {
3334 			/*
3335 			 * Don't bother attempting to reopen the disks;
3336 			 * just do the split.
3337 			 */
3338 			attempt_reopen = B_FALSE;
3339 		} else {
3340 			/* attempt to re-online it */
3341 			vd[i]->vdev_offline = B_FALSE;
3342 		}
3343 	}
3344 
3345 	if (attempt_reopen) {
3346 		vdev_reopen(spa->spa_root_vdev);
3347 
3348 		/* check each device to see what state it's in */
3349 		for (extracted = 0, i = 0; i < gcount; i++) {
3350 			if (vd[i] != NULL &&
3351 			    vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL)
3352 				break;
3353 			++extracted;
3354 		}
3355 	}
3356 
3357 	/*
3358 	 * If every disk has been moved to the new pool, or if we never
3359 	 * even attempted to look at them, then we split them off for
3360 	 * good.
3361 	 */
3362 	if (!attempt_reopen || gcount == extracted) {
3363 		for (i = 0; i < gcount; i++)
3364 			if (vd[i] != NULL)
3365 				vdev_split(vd[i]);
3366 		vdev_reopen(spa->spa_root_vdev);
3367 	}
3368 
3369 	kmem_free(vd, gcount * sizeof (vdev_t *));
3370 }
3371 
3372 static int
3373 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type)
3374 {
3375 	const char *ereport = FM_EREPORT_ZFS_POOL;
3376 	int error;
3377 
3378 	spa->spa_load_state = state;
3379 	(void) spa_import_progress_set_state(spa_guid(spa),
3380 	    spa_load_state(spa));
3381 	spa_import_progress_set_notes(spa, "spa_load()");
3382 
3383 	gethrestime(&spa->spa_loaded_ts);
3384 	error = spa_load_impl(spa, type, &ereport);
3385 
3386 	/*
3387 	 * Don't count references from objsets that are already closed
3388 	 * and are making their way through the eviction process.
3389 	 */
3390 	spa_evicting_os_wait(spa);
3391 	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
3392 	if (error) {
3393 		if (error != EEXIST) {
3394 			spa->spa_loaded_ts.tv_sec = 0;
3395 			spa->spa_loaded_ts.tv_nsec = 0;
3396 		}
3397 		if (error != EBADF) {
3398 			(void) zfs_ereport_post(ereport, spa,
3399 			    NULL, NULL, NULL, 0);
3400 		}
3401 	}
3402 	spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE;
3403 	spa->spa_ena = 0;
3404 
3405 	(void) spa_import_progress_set_state(spa_guid(spa),
3406 	    spa_load_state(spa));
3407 
3408 	return (error);
3409 }
3410 
3411 #ifdef ZFS_DEBUG
3412 /*
3413  * Count the number of per-vdev ZAPs associated with all of the vdevs in the
3414  * vdev tree rooted in the given vd, and ensure that each ZAP is present in the
3415  * spa's per-vdev ZAP list.
3416  */
3417 static uint64_t
3418 vdev_count_verify_zaps(vdev_t *vd)
3419 {
3420 	spa_t *spa = vd->vdev_spa;
3421 	uint64_t total = 0;
3422 
3423 	if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) &&
3424 	    vd->vdev_root_zap != 0) {
3425 		total++;
3426 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
3427 		    spa->spa_all_vdev_zaps, vd->vdev_root_zap));
3428 	}
3429 	if (vd->vdev_top_zap != 0) {
3430 		total++;
3431 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
3432 		    spa->spa_all_vdev_zaps, vd->vdev_top_zap));
3433 	}
3434 	if (vd->vdev_leaf_zap != 0) {
3435 		total++;
3436 		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
3437 		    spa->spa_all_vdev_zaps, vd->vdev_leaf_zap));
3438 	}
3439 
3440 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
3441 		total += vdev_count_verify_zaps(vd->vdev_child[i]);
3442 	}
3443 
3444 	return (total);
3445 }
3446 #else
3447 #define	vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0)
3448 #endif
3449 
3450 /*
3451  * Determine whether the activity check is required.
3452  */
3453 static boolean_t
3454 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label,
3455     nvlist_t *config)
3456 {
3457 	uint64_t state = 0;
3458 	uint64_t hostid = 0;
3459 	uint64_t tryconfig_txg = 0;
3460 	uint64_t tryconfig_timestamp = 0;
3461 	uint16_t tryconfig_mmp_seq = 0;
3462 	nvlist_t *nvinfo;
3463 
3464 	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
3465 		nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3466 		(void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG,
3467 		    &tryconfig_txg);
3468 		(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
3469 		    &tryconfig_timestamp);
3470 		(void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ,
3471 		    &tryconfig_mmp_seq);
3472 	}
3473 
3474 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state);
3475 
3476 	/*
3477 	 * Disable the MMP activity check - This is used by zdb which
3478 	 * is intended to be used on potentially active pools.
3479 	 */
3480 	if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP)
3481 		return (B_FALSE);
3482 
3483 	/*
3484 	 * Skip the activity check when the MMP feature is disabled.
3485 	 */
3486 	if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0)
3487 		return (B_FALSE);
3488 
3489 	/*
3490 	 * If the tryconfig_ values are nonzero, they are the results of an
3491 	 * earlier tryimport.  If they all match the uberblock we just found,
3492 	 * then the pool has not changed and we return false so we do not test
3493 	 * a second time.
3494 	 */
3495 	if (tryconfig_txg && tryconfig_txg == ub->ub_txg &&
3496 	    tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp &&
3497 	    tryconfig_mmp_seq && tryconfig_mmp_seq ==
3498 	    (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0))
3499 		return (B_FALSE);
3500 
3501 	/*
3502 	 * Allow the activity check to be skipped when importing the pool
3503 	 * on the same host which last imported it.  Since the hostid from
3504 	 * configuration may be stale use the one read from the label.
3505 	 */
3506 	if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID))
3507 		hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID);
3508 
3509 	if (hostid == spa_get_hostid(spa))
3510 		return (B_FALSE);
3511 
3512 	/*
3513 	 * Skip the activity test when the pool was cleanly exported.
3514 	 */
3515 	if (state != POOL_STATE_ACTIVE)
3516 		return (B_FALSE);
3517 
3518 	return (B_TRUE);
3519 }
3520 
3521 /*
3522  * Nanoseconds the activity check must watch for changes on-disk.
3523  */
3524 static uint64_t
3525 spa_activity_check_duration(spa_t *spa, uberblock_t *ub)
3526 {
3527 	uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1);
3528 	uint64_t multihost_interval = MSEC2NSEC(
3529 	    MMP_INTERVAL_OK(zfs_multihost_interval));
3530 	uint64_t import_delay = MAX(NANOSEC, import_intervals *
3531 	    multihost_interval);
3532 
3533 	/*
3534 	 * Local tunables determine a minimum duration except for the case
3535 	 * where we know when the remote host will suspend the pool if MMP
3536 	 * writes do not land.
3537 	 *
3538 	 * See Big Theory comment at the top of mmp.c for the reasoning behind
3539 	 * these cases and times.
3540 	 */
3541 
3542 	ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100);
3543 
3544 	if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
3545 	    MMP_FAIL_INT(ub) > 0) {
3546 
3547 		/* MMP on remote host will suspend pool after failed writes */
3548 		import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) *
3549 		    MMP_IMPORT_SAFETY_FACTOR / 100;
3550 
3551 		zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp "
3552 		    "mmp_fails=%llu ub_mmp mmp_interval=%llu "
3553 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3554 		    (u_longlong_t)MMP_FAIL_INT(ub),
3555 		    (u_longlong_t)MMP_INTERVAL(ub),
3556 		    (u_longlong_t)import_intervals);
3557 
3558 	} else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) &&
3559 	    MMP_FAIL_INT(ub) == 0) {
3560 
3561 		/* MMP on remote host will never suspend pool */
3562 		import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) +
3563 		    ub->ub_mmp_delay) * import_intervals);
3564 
3565 		zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp "
3566 		    "mmp_interval=%llu ub_mmp_delay=%llu "
3567 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3568 		    (u_longlong_t)MMP_INTERVAL(ub),
3569 		    (u_longlong_t)ub->ub_mmp_delay,
3570 		    (u_longlong_t)import_intervals);
3571 
3572 	} else if (MMP_VALID(ub)) {
3573 		/*
3574 		 * zfs-0.7 compatibility case
3575 		 */
3576 
3577 		import_delay = MAX(import_delay, (multihost_interval +
3578 		    ub->ub_mmp_delay) * import_intervals);
3579 
3580 		zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu "
3581 		    "import_intervals=%llu leaves=%u",
3582 		    (u_longlong_t)import_delay,
3583 		    (u_longlong_t)ub->ub_mmp_delay,
3584 		    (u_longlong_t)import_intervals,
3585 		    vdev_count_leaves(spa));
3586 	} else {
3587 		/* Using local tunings is the only reasonable option */
3588 		zfs_dbgmsg("pool last imported on non-MMP aware "
3589 		    "host using import_delay=%llu multihost_interval=%llu "
3590 		    "import_intervals=%llu", (u_longlong_t)import_delay,
3591 		    (u_longlong_t)multihost_interval,
3592 		    (u_longlong_t)import_intervals);
3593 	}
3594 
3595 	return (import_delay);
3596 }
3597 
3598 /*
3599  * Perform the import activity check.  If the user canceled the import or
3600  * we detected activity then fail.
3601  */
3602 static int
3603 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config)
3604 {
3605 	uint64_t txg = ub->ub_txg;
3606 	uint64_t timestamp = ub->ub_timestamp;
3607 	uint64_t mmp_config = ub->ub_mmp_config;
3608 	uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0;
3609 	uint64_t import_delay;
3610 	hrtime_t import_expire, now;
3611 	nvlist_t *mmp_label = NULL;
3612 	vdev_t *rvd = spa->spa_root_vdev;
3613 	kcondvar_t cv;
3614 	kmutex_t mtx;
3615 	int error = 0;
3616 
3617 	cv_init(&cv, NULL, CV_DEFAULT, NULL);
3618 	mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL);
3619 	mutex_enter(&mtx);
3620 
3621 	/*
3622 	 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed
3623 	 * during the earlier tryimport.  If the txg recorded there is 0 then
3624 	 * the pool is known to be active on another host.
3625 	 *
3626 	 * Otherwise, the pool might be in use on another host.  Check for
3627 	 * changes in the uberblocks on disk if necessary.
3628 	 */
3629 	if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) {
3630 		nvlist_t *nvinfo = fnvlist_lookup_nvlist(config,
3631 		    ZPOOL_CONFIG_LOAD_INFO);
3632 
3633 		if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) &&
3634 		    fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) {
3635 			vdev_uberblock_load(rvd, ub, &mmp_label);
3636 			error = SET_ERROR(EREMOTEIO);
3637 			goto out;
3638 		}
3639 	}
3640 
3641 	import_delay = spa_activity_check_duration(spa, ub);
3642 
3643 	/* Add a small random factor in case of simultaneous imports (0-25%) */
3644 	import_delay += import_delay * random_in_range(250) / 1000;
3645 
3646 	import_expire = gethrtime() + import_delay;
3647 
3648 	spa_import_progress_set_notes(spa, "Checking MMP activity, waiting "
3649 	    "%llu ms", (u_longlong_t)NSEC2MSEC(import_delay));
3650 
3651 	int interations = 0;
3652 	while ((now = gethrtime()) < import_expire) {
3653 		if (interations++ % 30 == 0) {
3654 			spa_import_progress_set_notes(spa, "Checking MMP "
3655 			    "activity, %llu ms remaining",
3656 			    (u_longlong_t)NSEC2MSEC(import_expire - now));
3657 		}
3658 
3659 		(void) spa_import_progress_set_mmp_check(spa_guid(spa),
3660 		    NSEC2SEC(import_expire - gethrtime()));
3661 
3662 		vdev_uberblock_load(rvd, ub, &mmp_label);
3663 
3664 		if (txg != ub->ub_txg || timestamp != ub->ub_timestamp ||
3665 		    mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) {
3666 			zfs_dbgmsg("multihost activity detected "
3667 			    "txg %llu ub_txg  %llu "
3668 			    "timestamp %llu ub_timestamp  %llu "
3669 			    "mmp_config %#llx ub_mmp_config %#llx",
3670 			    (u_longlong_t)txg, (u_longlong_t)ub->ub_txg,
3671 			    (u_longlong_t)timestamp,
3672 			    (u_longlong_t)ub->ub_timestamp,
3673 			    (u_longlong_t)mmp_config,
3674 			    (u_longlong_t)ub->ub_mmp_config);
3675 
3676 			error = SET_ERROR(EREMOTEIO);
3677 			break;
3678 		}
3679 
3680 		if (mmp_label) {
3681 			nvlist_free(mmp_label);
3682 			mmp_label = NULL;
3683 		}
3684 
3685 		error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz);
3686 		if (error != -1) {
3687 			error = SET_ERROR(EINTR);
3688 			break;
3689 		}
3690 		error = 0;
3691 	}
3692 
3693 out:
3694 	mutex_exit(&mtx);
3695 	mutex_destroy(&mtx);
3696 	cv_destroy(&cv);
3697 
3698 	/*
3699 	 * If the pool is determined to be active store the status in the
3700 	 * spa->spa_load_info nvlist.  If the remote hostname or hostid are
3701 	 * available from configuration read from disk store them as well.
3702 	 * This allows 'zpool import' to generate a more useful message.
3703 	 *
3704 	 * ZPOOL_CONFIG_MMP_STATE    - observed pool status (mandatory)
3705 	 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool
3706 	 * ZPOOL_CONFIG_MMP_HOSTID   - hostid from the active pool
3707 	 */
3708 	if (error == EREMOTEIO) {
3709 		const char *hostname = "<unknown>";
3710 		uint64_t hostid = 0;
3711 
3712 		if (mmp_label) {
3713 			if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) {
3714 				hostname = fnvlist_lookup_string(mmp_label,
3715 				    ZPOOL_CONFIG_HOSTNAME);
3716 				fnvlist_add_string(spa->spa_load_info,
3717 				    ZPOOL_CONFIG_MMP_HOSTNAME, hostname);
3718 			}
3719 
3720 			if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) {
3721 				hostid = fnvlist_lookup_uint64(mmp_label,
3722 				    ZPOOL_CONFIG_HOSTID);
3723 				fnvlist_add_uint64(spa->spa_load_info,
3724 				    ZPOOL_CONFIG_MMP_HOSTID, hostid);
3725 			}
3726 		}
3727 
3728 		fnvlist_add_uint64(spa->spa_load_info,
3729 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE);
3730 		fnvlist_add_uint64(spa->spa_load_info,
3731 		    ZPOOL_CONFIG_MMP_TXG, 0);
3732 
3733 		error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO);
3734 	}
3735 
3736 	if (mmp_label)
3737 		nvlist_free(mmp_label);
3738 
3739 	return (error);
3740 }
3741 
3742 static int
3743 spa_verify_host(spa_t *spa, nvlist_t *mos_config)
3744 {
3745 	uint64_t hostid;
3746 	const char *hostname;
3747 	uint64_t myhostid = 0;
3748 
3749 	if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config,
3750 	    ZPOOL_CONFIG_HOSTID, &hostid) == 0) {
3751 		hostname = fnvlist_lookup_string(mos_config,
3752 		    ZPOOL_CONFIG_HOSTNAME);
3753 
3754 		myhostid = zone_get_hostid(NULL);
3755 
3756 		if (hostid != 0 && myhostid != 0 && hostid != myhostid) {
3757 			cmn_err(CE_WARN, "pool '%s' could not be "
3758 			    "loaded as it was last accessed by "
3759 			    "another system (host: %s hostid: 0x%llx). "
3760 			    "See: https://openzfs.github.io/openzfs-docs/msg/"
3761 			    "ZFS-8000-EY",
3762 			    spa_name(spa), hostname, (u_longlong_t)hostid);
3763 			spa_load_failed(spa, "hostid verification failed: pool "
3764 			    "last accessed by host: %s (hostid: 0x%llx)",
3765 			    hostname, (u_longlong_t)hostid);
3766 			return (SET_ERROR(EBADF));
3767 		}
3768 	}
3769 
3770 	return (0);
3771 }
3772 
3773 static int
3774 spa_ld_parse_config(spa_t *spa, spa_import_type_t type)
3775 {
3776 	int error = 0;
3777 	nvlist_t *nvtree, *nvl, *config = spa->spa_config;
3778 	int parse;
3779 	vdev_t *rvd;
3780 	uint64_t pool_guid;
3781 	const char *comment;
3782 	const char *compatibility;
3783 
3784 	/*
3785 	 * Versioning wasn't explicitly added to the label until later, so if
3786 	 * it's not present treat it as the initial version.
3787 	 */
3788 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
3789 	    &spa->spa_ubsync.ub_version) != 0)
3790 		spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;
3791 
3792 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) {
3793 		spa_load_failed(spa, "invalid config provided: '%s' missing",
3794 		    ZPOOL_CONFIG_POOL_GUID);
3795 		return (SET_ERROR(EINVAL));
3796 	}
3797 
3798 	/*
3799 	 * If we are doing an import, ensure that the pool is not already
3800 	 * imported by checking if its pool guid already exists in the
3801 	 * spa namespace.
3802 	 *
3803 	 * The only case that we allow an already imported pool to be
3804 	 * imported again, is when the pool is checkpointed and we want to
3805 	 * look at its checkpointed state from userland tools like zdb.
3806 	 */
3807 #ifdef _KERNEL
3808 	if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
3809 	    spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
3810 	    spa_guid_exists(pool_guid, 0)) {
3811 #else
3812 	if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
3813 	    spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
3814 	    spa_guid_exists(pool_guid, 0) &&
3815 	    !spa_importing_readonly_checkpoint(spa)) {
3816 #endif
3817 		spa_load_failed(spa, "a pool with guid %llu is already open",
3818 		    (u_longlong_t)pool_guid);
3819 		return (SET_ERROR(EEXIST));
3820 	}
3821 
3822 	spa->spa_config_guid = pool_guid;
3823 
3824 	nvlist_free(spa->spa_load_info);
3825 	spa->spa_load_info = fnvlist_alloc();
3826 
3827 	ASSERT(spa->spa_comment == NULL);
3828 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
3829 		spa->spa_comment = spa_strdup(comment);
3830 
3831 	ASSERT(spa->spa_compatibility == NULL);
3832 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY,
3833 	    &compatibility) == 0)
3834 		spa->spa_compatibility = spa_strdup(compatibility);
3835 
3836 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
3837 	    &spa->spa_config_txg);
3838 
3839 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0)
3840 		spa->spa_config_splitting = fnvlist_dup(nvl);
3841 
3842 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) {
3843 		spa_load_failed(spa, "invalid config provided: '%s' missing",
3844 		    ZPOOL_CONFIG_VDEV_TREE);
3845 		return (SET_ERROR(EINVAL));
3846 	}
3847 
3848 	/*
3849 	 * Create "The Godfather" zio to hold all async IOs
3850 	 */
3851 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
3852 	    KM_SLEEP);
3853 	for (int i = 0; i < max_ncpus; i++) {
3854 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
3855 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
3856 		    ZIO_FLAG_GODFATHER);
3857 	}
3858 
3859 	/*
3860 	 * Parse the configuration into a vdev tree.  We explicitly set the
3861 	 * value that will be returned by spa_version() since parsing the
3862 	 * configuration requires knowing the version number.
3863 	 */
3864 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3865 	parse = (type == SPA_IMPORT_EXISTING ?
3866 	    VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT);
3867 	error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse);
3868 	spa_config_exit(spa, SCL_ALL, FTAG);
3869 
3870 	if (error != 0) {
3871 		spa_load_failed(spa, "unable to parse config [error=%d]",
3872 		    error);
3873 		return (error);
3874 	}
3875 
3876 	ASSERT(spa->spa_root_vdev == rvd);
3877 	ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
3878 	ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);
3879 
3880 	if (type != SPA_IMPORT_ASSEMBLE) {
3881 		ASSERT(spa_guid(spa) == pool_guid);
3882 	}
3883 
3884 	return (0);
3885 }
3886 
3887 /*
3888  * Recursively open all vdevs in the vdev tree. This function is called twice:
3889  * first with the untrusted config, then with the trusted config.
3890  */
3891 static int
3892 spa_ld_open_vdevs(spa_t *spa)
3893 {
3894 	int error = 0;
3895 
3896 	/*
3897 	 * spa_missing_tvds_allowed defines how many top-level vdevs can be
3898 	 * missing/unopenable for the root vdev to be still considered openable.
3899 	 */
3900 	if (spa->spa_trust_config) {
3901 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds;
3902 	} else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) {
3903 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile;
3904 	} else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) {
3905 		spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan;
3906 	} else {
3907 		spa->spa_missing_tvds_allowed = 0;
3908 	}
3909 
3910 	spa->spa_missing_tvds_allowed =
3911 	    MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed);
3912 
3913 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3914 	error = vdev_open(spa->spa_root_vdev);
3915 	spa_config_exit(spa, SCL_ALL, FTAG);
3916 
3917 	if (spa->spa_missing_tvds != 0) {
3918 		spa_load_note(spa, "vdev tree has %lld missing top-level "
3919 		    "vdevs.", (u_longlong_t)spa->spa_missing_tvds);
3920 		if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) {
3921 			/*
3922 			 * Although theoretically we could allow users to open
3923 			 * incomplete pools in RW mode, we'd need to add a lot
3924 			 * of extra logic (e.g. adjust pool space to account
3925 			 * for missing vdevs).
3926 			 * This limitation also prevents users from accidentally
3927 			 * opening the pool in RW mode during data recovery and
3928 			 * damaging it further.
3929 			 */
3930 			spa_load_note(spa, "pools with missing top-level "
3931 			    "vdevs can only be opened in read-only mode.");
3932 			error = SET_ERROR(ENXIO);
3933 		} else {
3934 			spa_load_note(spa, "current settings allow for maximum "
3935 			    "%lld missing top-level vdevs at this stage.",
3936 			    (u_longlong_t)spa->spa_missing_tvds_allowed);
3937 		}
3938 	}
3939 	if (error != 0) {
3940 		spa_load_failed(spa, "unable to open vdev tree [error=%d]",
3941 		    error);
3942 	}
3943 	if (spa->spa_missing_tvds != 0 || error != 0)
3944 		vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2);
3945 
3946 	return (error);
3947 }
3948 
3949 /*
3950  * We need to validate the vdev labels against the configuration that
3951  * we have in hand. This function is called twice: first with an untrusted
3952  * config, then with a trusted config. The validation is more strict when the
3953  * config is trusted.
3954  */
3955 static int
3956 spa_ld_validate_vdevs(spa_t *spa)
3957 {
3958 	int error = 0;
3959 	vdev_t *rvd = spa->spa_root_vdev;
3960 
3961 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3962 	error = vdev_validate(rvd);
3963 	spa_config_exit(spa, SCL_ALL, FTAG);
3964 
3965 	if (error != 0) {
3966 		spa_load_failed(spa, "vdev_validate failed [error=%d]", error);
3967 		return (error);
3968 	}
3969 
3970 	if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
3971 		spa_load_failed(spa, "cannot open vdev tree after invalidating "
3972 		    "some vdevs");
3973 		vdev_dbgmsg_print_tree(rvd, 2);
3974 		return (SET_ERROR(ENXIO));
3975 	}
3976 
3977 	return (0);
3978 }
3979 
3980 static void
3981 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub)
3982 {
3983 	spa->spa_state = POOL_STATE_ACTIVE;
3984 	spa->spa_ubsync = spa->spa_uberblock;
3985 	spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
3986 	    TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
3987 	spa->spa_first_txg = spa->spa_last_ubsync_txg ?
3988 	    spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
3989 	spa->spa_claim_max_txg = spa->spa_first_txg;
3990 	spa->spa_prev_software_version = ub->ub_software_version;
3991 }
3992 
3993 static int
3994 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type)
3995 {
3996 	vdev_t *rvd = spa->spa_root_vdev;
3997 	nvlist_t *label;
3998 	uberblock_t *ub = &spa->spa_uberblock;
3999 	boolean_t activity_check = B_FALSE;
4000 
4001 	/*
4002 	 * If we are opening the checkpointed state of the pool by
4003 	 * rewinding to it, at this point we will have written the
4004 	 * checkpointed uberblock to the vdev labels, so searching
4005 	 * the labels will find the right uberblock.  However, if
4006 	 * we are opening the checkpointed state read-only, we have
4007 	 * not modified the labels. Therefore, we must ignore the
4008 	 * labels and continue using the spa_uberblock that was set
4009 	 * by spa_ld_checkpoint_rewind.
4010 	 *
4011 	 * Note that it would be fine to ignore the labels when
4012 	 * rewinding (opening writeable) as well. However, if we
4013 	 * crash just after writing the labels, we will end up
4014 	 * searching the labels. Doing so in the common case means
4015 	 * that this code path gets exercised normally, rather than
4016 	 * just in the edge case.
4017 	 */
4018 	if (ub->ub_checkpoint_txg != 0 &&
4019 	    spa_importing_readonly_checkpoint(spa)) {
4020 		spa_ld_select_uberblock_done(spa, ub);
4021 		return (0);
4022 	}
4023 
4024 	/*
4025 	 * Find the best uberblock.
4026 	 */
4027 	vdev_uberblock_load(rvd, ub, &label);
4028 
4029 	/*
4030 	 * If we weren't able to find a single valid uberblock, return failure.
4031 	 */
4032 	if (ub->ub_txg == 0) {
4033 		nvlist_free(label);
4034 		spa_load_failed(spa, "no valid uberblock found");
4035 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO));
4036 	}
4037 
4038 	if (spa->spa_load_max_txg != UINT64_MAX) {
4039 		(void) spa_import_progress_set_max_txg(spa_guid(spa),
4040 		    (u_longlong_t)spa->spa_load_max_txg);
4041 	}
4042 	spa_load_note(spa, "using uberblock with txg=%llu",
4043 	    (u_longlong_t)ub->ub_txg);
4044 	if (ub->ub_raidz_reflow_info != 0) {
4045 		spa_load_note(spa, "uberblock raidz_reflow_info: "
4046 		    "state=%u offset=%llu",
4047 		    (int)RRSS_GET_STATE(ub),
4048 		    (u_longlong_t)RRSS_GET_OFFSET(ub));
4049 	}
4050 
4051 
4052 	/*
4053 	 * For pools which have the multihost property on determine if the
4054 	 * pool is truly inactive and can be safely imported.  Prevent
4055 	 * hosts which don't have a hostid set from importing the pool.
4056 	 */
4057 	activity_check = spa_activity_check_required(spa, ub, label,
4058 	    spa->spa_config);
4059 	if (activity_check) {
4060 		if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay &&
4061 		    spa_get_hostid(spa) == 0) {
4062 			nvlist_free(label);
4063 			fnvlist_add_uint64(spa->spa_load_info,
4064 			    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
4065 			return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
4066 		}
4067 
4068 		int error = spa_activity_check(spa, ub, spa->spa_config);
4069 		if (error) {
4070 			nvlist_free(label);
4071 			return (error);
4072 		}
4073 
4074 		fnvlist_add_uint64(spa->spa_load_info,
4075 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE);
4076 		fnvlist_add_uint64(spa->spa_load_info,
4077 		    ZPOOL_CONFIG_MMP_TXG, ub->ub_txg);
4078 		fnvlist_add_uint16(spa->spa_load_info,
4079 		    ZPOOL_CONFIG_MMP_SEQ,
4080 		    (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0));
4081 	}
4082 
4083 	/*
4084 	 * If the pool has an unsupported version we can't open it.
4085 	 */
4086 	if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
4087 		nvlist_free(label);
4088 		spa_load_failed(spa, "version %llu is not supported",
4089 		    (u_longlong_t)ub->ub_version);
4090 		return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP));
4091 	}
4092 
4093 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
4094 		nvlist_t *features;
4095 
4096 		/*
4097 		 * If we weren't able to find what's necessary for reading the
4098 		 * MOS in the label, return failure.
4099 		 */
4100 		if (label == NULL) {
4101 			spa_load_failed(spa, "label config unavailable");
4102 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
4103 			    ENXIO));
4104 		}
4105 
4106 		if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ,
4107 		    &features) != 0) {
4108 			nvlist_free(label);
4109 			spa_load_failed(spa, "invalid label: '%s' missing",
4110 			    ZPOOL_CONFIG_FEATURES_FOR_READ);
4111 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
4112 			    ENXIO));
4113 		}
4114 
4115 		/*
4116 		 * Update our in-core representation with the definitive values
4117 		 * from the label.
4118 		 */
4119 		nvlist_free(spa->spa_label_features);
4120 		spa->spa_label_features = fnvlist_dup(features);
4121 	}
4122 
4123 	nvlist_free(label);
4124 
4125 	/*
4126 	 * Look through entries in the label nvlist's features_for_read. If
4127 	 * there is a feature listed there which we don't understand then we
4128 	 * cannot open a pool.
4129 	 */
4130 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
4131 		nvlist_t *unsup_feat;
4132 
4133 		unsup_feat = fnvlist_alloc();
4134 
4135 		for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
4136 		    NULL); nvp != NULL;
4137 		    nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
4138 			if (!zfeature_is_supported(nvpair_name(nvp))) {
4139 				fnvlist_add_string(unsup_feat,
4140 				    nvpair_name(nvp), "");
4141 			}
4142 		}
4143 
4144 		if (!nvlist_empty(unsup_feat)) {
4145 			fnvlist_add_nvlist(spa->spa_load_info,
4146 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
4147 			nvlist_free(unsup_feat);
4148 			spa_load_failed(spa, "some features are unsupported");
4149 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
4150 			    ENOTSUP));
4151 		}
4152 
4153 		nvlist_free(unsup_feat);
4154 	}
4155 
4156 	if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
4157 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4158 		spa_try_repair(spa, spa->spa_config);
4159 		spa_config_exit(spa, SCL_ALL, FTAG);
4160 		nvlist_free(spa->spa_config_splitting);
4161 		spa->spa_config_splitting = NULL;
4162 	}
4163 
4164 	/*
4165 	 * Initialize internal SPA structures.
4166 	 */
4167 	spa_ld_select_uberblock_done(spa, ub);
4168 
4169 	return (0);
4170 }
4171 
4172 static int
4173 spa_ld_open_rootbp(spa_t *spa)
4174 {
4175 	int error = 0;
4176 	vdev_t *rvd = spa->spa_root_vdev;
4177 
4178 	error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
4179 	if (error != 0) {
4180 		spa_load_failed(spa, "unable to open rootbp in dsl_pool_init "
4181 		    "[error=%d]", error);
4182 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4183 	}
4184 	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
4185 
4186 	return (0);
4187 }
4188 
4189 static int
4190 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type,
4191     boolean_t reloading)
4192 {
4193 	vdev_t *mrvd, *rvd = spa->spa_root_vdev;
4194 	nvlist_t *nv, *mos_config, *policy;
4195 	int error = 0, copy_error;
4196 	uint64_t healthy_tvds, healthy_tvds_mos;
4197 	uint64_t mos_config_txg;
4198 
4199 	if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE)
4200 	    != 0)
4201 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4202 
4203 	/*
4204 	 * If we're assembling a pool from a split, the config provided is
4205 	 * already trusted so there is nothing to do.
4206 	 */
4207 	if (type == SPA_IMPORT_ASSEMBLE)
4208 		return (0);
4209 
4210 	healthy_tvds = spa_healthy_core_tvds(spa);
4211 
4212 	if (load_nvlist(spa, spa->spa_config_object, &mos_config)
4213 	    != 0) {
4214 		spa_load_failed(spa, "unable to retrieve MOS config");
4215 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4216 	}
4217 
4218 	/*
4219 	 * If we are doing an open, pool owner wasn't verified yet, thus do
4220 	 * the verification here.
4221 	 */
4222 	if (spa->spa_load_state == SPA_LOAD_OPEN) {
4223 		error = spa_verify_host(spa, mos_config);
4224 		if (error != 0) {
4225 			nvlist_free(mos_config);
4226 			return (error);
4227 		}
4228 	}
4229 
4230 	nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE);
4231 
4232 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4233 
4234 	/*
4235 	 * Build a new vdev tree from the trusted config
4236 	 */
4237 	error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD);
4238 	if (error != 0) {
4239 		nvlist_free(mos_config);
4240 		spa_config_exit(spa, SCL_ALL, FTAG);
4241 		spa_load_failed(spa, "spa_config_parse failed [error=%d]",
4242 		    error);
4243 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4244 	}
4245 
4246 	/*
4247 	 * Vdev paths in the MOS may be obsolete. If the untrusted config was
4248 	 * obtained by scanning /dev/dsk, then it will have the right vdev
4249 	 * paths. We update the trusted MOS config with this information.
4250 	 * We first try to copy the paths with vdev_copy_path_strict, which
4251 	 * succeeds only when both configs have exactly the same vdev tree.
4252 	 * If that fails, we fall back to a more flexible method that has a
4253 	 * best effort policy.
4254 	 */
4255 	copy_error = vdev_copy_path_strict(rvd, mrvd);
4256 	if (copy_error != 0 || spa_load_print_vdev_tree) {
4257 		spa_load_note(spa, "provided vdev tree:");
4258 		vdev_dbgmsg_print_tree(rvd, 2);
4259 		spa_load_note(spa, "MOS vdev tree:");
4260 		vdev_dbgmsg_print_tree(mrvd, 2);
4261 	}
4262 	if (copy_error != 0) {
4263 		spa_load_note(spa, "vdev_copy_path_strict failed, falling "
4264 		    "back to vdev_copy_path_relaxed");
4265 		vdev_copy_path_relaxed(rvd, mrvd);
4266 	}
4267 
4268 	vdev_close(rvd);
4269 	vdev_free(rvd);
4270 	spa->spa_root_vdev = mrvd;
4271 	rvd = mrvd;
4272 	spa_config_exit(spa, SCL_ALL, FTAG);
4273 
4274 	/*
4275 	 * If 'zpool import' used a cached config, then the on-disk hostid and
4276 	 * hostname may be different to the cached config in ways that should
4277 	 * prevent import.  Userspace can't discover this without a scan, but
4278 	 * we know, so we add these values to LOAD_INFO so the caller can know
4279 	 * the difference.
4280 	 *
4281 	 * Note that we have to do this before the config is regenerated,
4282 	 * because the new config will have the hostid and hostname for this
4283 	 * host, in readiness for import.
4284 	 */
4285 	if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTID))
4286 		fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID,
4287 		    fnvlist_lookup_uint64(mos_config, ZPOOL_CONFIG_HOSTID));
4288 	if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTNAME))
4289 		fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME,
4290 		    fnvlist_lookup_string(mos_config, ZPOOL_CONFIG_HOSTNAME));
4291 
4292 	/*
4293 	 * We will use spa_config if we decide to reload the spa or if spa_load
4294 	 * fails and we rewind. We must thus regenerate the config using the
4295 	 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to
4296 	 * pass settings on how to load the pool and is not stored in the MOS.
4297 	 * We copy it over to our new, trusted config.
4298 	 */
4299 	mos_config_txg = fnvlist_lookup_uint64(mos_config,
4300 	    ZPOOL_CONFIG_POOL_TXG);
4301 	nvlist_free(mos_config);
4302 	mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE);
4303 	if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY,
4304 	    &policy) == 0)
4305 		fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy);
4306 	spa_config_set(spa, mos_config);
4307 	spa->spa_config_source = SPA_CONFIG_SRC_MOS;
4308 
4309 	/*
4310 	 * Now that we got the config from the MOS, we should be more strict
4311 	 * in checking blkptrs and can make assumptions about the consistency
4312 	 * of the vdev tree. spa_trust_config must be set to true before opening
4313 	 * vdevs in order for them to be writeable.
4314 	 */
4315 	spa->spa_trust_config = B_TRUE;
4316 
4317 	/*
4318 	 * Open and validate the new vdev tree
4319 	 */
4320 	error = spa_ld_open_vdevs(spa);
4321 	if (error != 0)
4322 		return (error);
4323 
4324 	error = spa_ld_validate_vdevs(spa);
4325 	if (error != 0)
4326 		return (error);
4327 
4328 	if (copy_error != 0 || spa_load_print_vdev_tree) {
4329 		spa_load_note(spa, "final vdev tree:");
4330 		vdev_dbgmsg_print_tree(rvd, 2);
4331 	}
4332 
4333 	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT &&
4334 	    !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) {
4335 		/*
4336 		 * Sanity check to make sure that we are indeed loading the
4337 		 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds
4338 		 * in the config provided and they happened to be the only ones
4339 		 * to have the latest uberblock, we could involuntarily perform
4340 		 * an extreme rewind.
4341 		 */
4342 		healthy_tvds_mos = spa_healthy_core_tvds(spa);
4343 		if (healthy_tvds_mos - healthy_tvds >=
4344 		    SPA_SYNC_MIN_VDEVS) {
4345 			spa_load_note(spa, "config provided misses too many "
4346 			    "top-level vdevs compared to MOS (%lld vs %lld). ",
4347 			    (u_longlong_t)healthy_tvds,
4348 			    (u_longlong_t)healthy_tvds_mos);
4349 			spa_load_note(spa, "vdev tree:");
4350 			vdev_dbgmsg_print_tree(rvd, 2);
4351 			if (reloading) {
4352 				spa_load_failed(spa, "config was already "
4353 				    "provided from MOS. Aborting.");
4354 				return (spa_vdev_err(rvd,
4355 				    VDEV_AUX_CORRUPT_DATA, EIO));
4356 			}
4357 			spa_load_note(spa, "spa must be reloaded using MOS "
4358 			    "config");
4359 			return (SET_ERROR(EAGAIN));
4360 		}
4361 	}
4362 
4363 	error = spa_check_for_missing_logs(spa);
4364 	if (error != 0)
4365 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO));
4366 
4367 	if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) {
4368 		spa_load_failed(spa, "uberblock guid sum doesn't match MOS "
4369 		    "guid sum (%llu != %llu)",
4370 		    (u_longlong_t)spa->spa_uberblock.ub_guid_sum,
4371 		    (u_longlong_t)rvd->vdev_guid_sum);
4372 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM,
4373 		    ENXIO));
4374 	}
4375 
4376 	return (0);
4377 }
4378 
4379 static int
4380 spa_ld_open_indirect_vdev_metadata(spa_t *spa)
4381 {
4382 	int error = 0;
4383 	vdev_t *rvd = spa->spa_root_vdev;
4384 
4385 	/*
4386 	 * Everything that we read before spa_remove_init() must be stored
4387 	 * on concreted vdevs.  Therefore we do this as early as possible.
4388 	 */
4389 	error = spa_remove_init(spa);
4390 	if (error != 0) {
4391 		spa_load_failed(spa, "spa_remove_init failed [error=%d]",
4392 		    error);
4393 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4394 	}
4395 
4396 	/*
4397 	 * Retrieve information needed to condense indirect vdev mappings.
4398 	 */
4399 	error = spa_condense_init(spa);
4400 	if (error != 0) {
4401 		spa_load_failed(spa, "spa_condense_init failed [error=%d]",
4402 		    error);
4403 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4404 	}
4405 
4406 	return (0);
4407 }
4408 
4409 static int
4410 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep)
4411 {
4412 	int error = 0;
4413 	vdev_t *rvd = spa->spa_root_vdev;
4414 
4415 	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
4416 		boolean_t missing_feat_read = B_FALSE;
4417 		nvlist_t *unsup_feat, *enabled_feat;
4418 
4419 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
4420 		    &spa->spa_feat_for_read_obj, B_TRUE) != 0) {
4421 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4422 		}
4423 
4424 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE,
4425 		    &spa->spa_feat_for_write_obj, B_TRUE) != 0) {
4426 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4427 		}
4428 
4429 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS,
4430 		    &spa->spa_feat_desc_obj, B_TRUE) != 0) {
4431 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4432 		}
4433 
4434 		enabled_feat = fnvlist_alloc();
4435 		unsup_feat = fnvlist_alloc();
4436 
4437 		if (!spa_features_check(spa, B_FALSE,
4438 		    unsup_feat, enabled_feat))
4439 			missing_feat_read = B_TRUE;
4440 
4441 		if (spa_writeable(spa) ||
4442 		    spa->spa_load_state == SPA_LOAD_TRYIMPORT) {
4443 			if (!spa_features_check(spa, B_TRUE,
4444 			    unsup_feat, enabled_feat)) {
4445 				*missing_feat_writep = B_TRUE;
4446 			}
4447 		}
4448 
4449 		fnvlist_add_nvlist(spa->spa_load_info,
4450 		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
4451 
4452 		if (!nvlist_empty(unsup_feat)) {
4453 			fnvlist_add_nvlist(spa->spa_load_info,
4454 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
4455 		}
4456 
4457 		fnvlist_free(enabled_feat);
4458 		fnvlist_free(unsup_feat);
4459 
4460 		if (!missing_feat_read) {
4461 			fnvlist_add_boolean(spa->spa_load_info,
4462 			    ZPOOL_CONFIG_CAN_RDONLY);
4463 		}
4464 
4465 		/*
4466 		 * If the state is SPA_LOAD_TRYIMPORT, our objective is
4467 		 * twofold: to determine whether the pool is available for
4468 		 * import in read-write mode and (if it is not) whether the
4469 		 * pool is available for import in read-only mode. If the pool
4470 		 * is available for import in read-write mode, it is displayed
4471 		 * as available in userland; if it is not available for import
4472 		 * in read-only mode, it is displayed as unavailable in
4473 		 * userland. If the pool is available for import in read-only
4474 		 * mode but not read-write mode, it is displayed as unavailable
4475 		 * in userland with a special note that the pool is actually
4476 		 * available for open in read-only mode.
4477 		 *
4478 		 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are
4479 		 * missing a feature for write, we must first determine whether
4480 		 * the pool can be opened read-only before returning to
4481 		 * userland in order to know whether to display the
4482 		 * abovementioned note.
4483 		 */
4484 		if (missing_feat_read || (*missing_feat_writep &&
4485 		    spa_writeable(spa))) {
4486 			spa_load_failed(spa, "pool uses unsupported features");
4487 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
4488 			    ENOTSUP));
4489 		}
4490 
4491 		/*
4492 		 * Load refcounts for ZFS features from disk into an in-memory
4493 		 * cache during SPA initialization.
4494 		 */
4495 		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
4496 			uint64_t refcount;
4497 
4498 			error = feature_get_refcount_from_disk(spa,
4499 			    &spa_feature_table[i], &refcount);
4500 			if (error == 0) {
4501 				spa->spa_feat_refcount_cache[i] = refcount;
4502 			} else if (error == ENOTSUP) {
4503 				spa->spa_feat_refcount_cache[i] =
4504 				    SPA_FEATURE_DISABLED;
4505 			} else {
4506 				spa_load_failed(spa, "error getting refcount "
4507 				    "for feature %s [error=%d]",
4508 				    spa_feature_table[i].fi_guid, error);
4509 				return (spa_vdev_err(rvd,
4510 				    VDEV_AUX_CORRUPT_DATA, EIO));
4511 			}
4512 		}
4513 	}
4514 
4515 	if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) {
4516 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG,
4517 		    &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0)
4518 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4519 	}
4520 
4521 	/*
4522 	 * Encryption was added before bookmark_v2, even though bookmark_v2
4523 	 * is now a dependency. If this pool has encryption enabled without
4524 	 * bookmark_v2, trigger an errata message.
4525 	 */
4526 	if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) &&
4527 	    !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) {
4528 		spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
4529 	}
4530 
4531 	return (0);
4532 }
4533 
4534 static int
4535 spa_ld_load_special_directories(spa_t *spa)
4536 {
4537 	int error = 0;
4538 	vdev_t *rvd = spa->spa_root_vdev;
4539 
4540 	spa->spa_is_initializing = B_TRUE;
4541 	error = dsl_pool_open(spa->spa_dsl_pool);
4542 	spa->spa_is_initializing = B_FALSE;
4543 	if (error != 0) {
4544 		spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error);
4545 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4546 	}
4547 
4548 	return (0);
4549 }
4550 
4551 static int
4552 spa_ld_get_props(spa_t *spa)
4553 {
4554 	int error = 0;
4555 	uint64_t obj;
4556 	vdev_t *rvd = spa->spa_root_vdev;
4557 
4558 	/* Grab the checksum salt from the MOS. */
4559 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4560 	    DMU_POOL_CHECKSUM_SALT, 1,
4561 	    sizeof (spa->spa_cksum_salt.zcs_bytes),
4562 	    spa->spa_cksum_salt.zcs_bytes);
4563 	if (error == ENOENT) {
4564 		/* Generate a new salt for subsequent use */
4565 		(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
4566 		    sizeof (spa->spa_cksum_salt.zcs_bytes));
4567 	} else if (error != 0) {
4568 		spa_load_failed(spa, "unable to retrieve checksum salt from "
4569 		    "MOS [error=%d]", error);
4570 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4571 	}
4572 
4573 	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0)
4574 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4575 	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
4576 	if (error != 0) {
4577 		spa_load_failed(spa, "error opening deferred-frees bpobj "
4578 		    "[error=%d]", error);
4579 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4580 	}
4581 
4582 	/*
4583 	 * Load the bit that tells us to use the new accounting function
4584 	 * (raid-z deflation).  If we have an older pool, this will not
4585 	 * be present.
4586 	 */
4587 	error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE);
4588 	if (error != 0 && error != ENOENT)
4589 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4590 
4591 	error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION,
4592 	    &spa->spa_creation_version, B_FALSE);
4593 	if (error != 0 && error != ENOENT)
4594 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4595 
4596 	/*
4597 	 * Load the persistent error log.  If we have an older pool, this will
4598 	 * not be present.
4599 	 */
4600 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last,
4601 	    B_FALSE);
4602 	if (error != 0 && error != ENOENT)
4603 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4604 
4605 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB,
4606 	    &spa->spa_errlog_scrub, B_FALSE);
4607 	if (error != 0 && error != ENOENT)
4608 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4609 
4610 	/*
4611 	 * Load the livelist deletion field. If a livelist is queued for
4612 	 * deletion, indicate that in the spa
4613 	 */
4614 	error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES,
4615 	    &spa->spa_livelists_to_delete, B_FALSE);
4616 	if (error != 0 && error != ENOENT)
4617 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4618 
4619 	/*
4620 	 * Load the history object.  If we have an older pool, this
4621 	 * will not be present.
4622 	 */
4623 	error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE);
4624 	if (error != 0 && error != ENOENT)
4625 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4626 
4627 	/*
4628 	 * Load the per-vdev ZAP map. If we have an older pool, this will not
4629 	 * be present; in this case, defer its creation to a later time to
4630 	 * avoid dirtying the MOS this early / out of sync context. See
4631 	 * spa_sync_config_object.
4632 	 */
4633 
4634 	/* The sentinel is only available in the MOS config. */
4635 	nvlist_t *mos_config;
4636 	if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) {
4637 		spa_load_failed(spa, "unable to retrieve MOS config");
4638 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4639 	}
4640 
4641 	error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP,
4642 	    &spa->spa_all_vdev_zaps, B_FALSE);
4643 
4644 	if (error == ENOENT) {
4645 		VERIFY(!nvlist_exists(mos_config,
4646 		    ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
4647 		spa->spa_avz_action = AVZ_ACTION_INITIALIZE;
4648 		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4649 	} else if (error != 0) {
4650 		nvlist_free(mos_config);
4651 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4652 	} else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) {
4653 		/*
4654 		 * An older version of ZFS overwrote the sentinel value, so
4655 		 * we have orphaned per-vdev ZAPs in the MOS. Defer their
4656 		 * destruction to later; see spa_sync_config_object.
4657 		 */
4658 		spa->spa_avz_action = AVZ_ACTION_DESTROY;
4659 		/*
4660 		 * We're assuming that no vdevs have had their ZAPs created
4661 		 * before this. Better be sure of it.
4662 		 */
4663 		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4664 	}
4665 	nvlist_free(mos_config);
4666 
4667 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
4668 
4669 	error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object,
4670 	    B_FALSE);
4671 	if (error && error != ENOENT)
4672 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4673 
4674 	if (error == 0) {
4675 		uint64_t autoreplace = 0;
4676 
4677 		spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
4678 		spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace);
4679 		spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
4680 		spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
4681 		spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
4682 		spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost);
4683 		spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim);
4684 		spa->spa_autoreplace = (autoreplace != 0);
4685 	}
4686 
4687 	/*
4688 	 * If we are importing a pool with missing top-level vdevs,
4689 	 * we enforce that the pool doesn't panic or get suspended on
4690 	 * error since the likelihood of missing data is extremely high.
4691 	 */
4692 	if (spa->spa_missing_tvds > 0 &&
4693 	    spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE &&
4694 	    spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4695 		spa_load_note(spa, "forcing failmode to 'continue' "
4696 		    "as some top level vdevs are missing");
4697 		spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE;
4698 	}
4699 
4700 	return (0);
4701 }
4702 
4703 static int
4704 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type)
4705 {
4706 	int error = 0;
4707 	vdev_t *rvd = spa->spa_root_vdev;
4708 
4709 	/*
4710 	 * If we're assembling the pool from the split-off vdevs of
4711 	 * an existing pool, we don't want to attach the spares & cache
4712 	 * devices.
4713 	 */
4714 
4715 	/*
4716 	 * Load any hot spares for this pool.
4717 	 */
4718 	error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object,
4719 	    B_FALSE);
4720 	if (error != 0 && error != ENOENT)
4721 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4722 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
4723 		ASSERT(spa_version(spa) >= SPA_VERSION_SPARES);
4724 		if (load_nvlist(spa, spa->spa_spares.sav_object,
4725 		    &spa->spa_spares.sav_config) != 0) {
4726 			spa_load_failed(spa, "error loading spares nvlist");
4727 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4728 		}
4729 
4730 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4731 		spa_load_spares(spa);
4732 		spa_config_exit(spa, SCL_ALL, FTAG);
4733 	} else if (error == 0) {
4734 		spa->spa_spares.sav_sync = B_TRUE;
4735 	}
4736 
4737 	/*
4738 	 * Load any level 2 ARC devices for this pool.
4739 	 */
4740 	error = spa_dir_prop(spa, DMU_POOL_L2CACHE,
4741 	    &spa->spa_l2cache.sav_object, B_FALSE);
4742 	if (error != 0 && error != ENOENT)
4743 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4744 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
4745 		ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE);
4746 		if (load_nvlist(spa, spa->spa_l2cache.sav_object,
4747 		    &spa->spa_l2cache.sav_config) != 0) {
4748 			spa_load_failed(spa, "error loading l2cache nvlist");
4749 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4750 		}
4751 
4752 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4753 		spa_load_l2cache(spa);
4754 		spa_config_exit(spa, SCL_ALL, FTAG);
4755 	} else if (error == 0) {
4756 		spa->spa_l2cache.sav_sync = B_TRUE;
4757 	}
4758 
4759 	return (0);
4760 }
4761 
4762 static int
4763 spa_ld_load_vdev_metadata(spa_t *spa)
4764 {
4765 	int error = 0;
4766 	vdev_t *rvd = spa->spa_root_vdev;
4767 
4768 	/*
4769 	 * If the 'multihost' property is set, then never allow a pool to
4770 	 * be imported when the system hostid is zero.  The exception to
4771 	 * this rule is zdb which is always allowed to access pools.
4772 	 */
4773 	if (spa_multihost(spa) && spa_get_hostid(spa) == 0 &&
4774 	    (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) {
4775 		fnvlist_add_uint64(spa->spa_load_info,
4776 		    ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID);
4777 		return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO));
4778 	}
4779 
4780 	/*
4781 	 * If the 'autoreplace' property is set, then post a resource notifying
4782 	 * the ZFS DE that it should not issue any faults for unopenable
4783 	 * devices.  We also iterate over the vdevs, and post a sysevent for any
4784 	 * unopenable vdevs so that the normal autoreplace handler can take
4785 	 * over.
4786 	 */
4787 	if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4788 		spa_check_removed(spa->spa_root_vdev);
4789 		/*
4790 		 * For the import case, this is done in spa_import(), because
4791 		 * at this point we're using the spare definitions from
4792 		 * the MOS config, not necessarily from the userland config.
4793 		 */
4794 		if (spa->spa_load_state != SPA_LOAD_IMPORT) {
4795 			spa_aux_check_removed(&spa->spa_spares);
4796 			spa_aux_check_removed(&spa->spa_l2cache);
4797 		}
4798 	}
4799 
4800 	/*
4801 	 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc.
4802 	 */
4803 	error = vdev_load(rvd);
4804 	if (error != 0) {
4805 		spa_load_failed(spa, "vdev_load failed [error=%d]", error);
4806 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4807 	}
4808 
4809 	error = spa_ld_log_spacemaps(spa);
4810 	if (error != 0) {
4811 		spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]",
4812 		    error);
4813 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error));
4814 	}
4815 
4816 	/*
4817 	 * Propagate the leaf DTLs we just loaded all the way up the vdev tree.
4818 	 */
4819 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4820 	vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE);
4821 	spa_config_exit(spa, SCL_ALL, FTAG);
4822 
4823 	return (0);
4824 }
4825 
4826 static int
4827 spa_ld_load_dedup_tables(spa_t *spa)
4828 {
4829 	int error = 0;
4830 	vdev_t *rvd = spa->spa_root_vdev;
4831 
4832 	error = ddt_load(spa);
4833 	if (error != 0) {
4834 		spa_load_failed(spa, "ddt_load failed [error=%d]", error);
4835 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4836 	}
4837 
4838 	return (0);
4839 }
4840 
4841 static int
4842 spa_ld_load_brt(spa_t *spa)
4843 {
4844 	int error = 0;
4845 	vdev_t *rvd = spa->spa_root_vdev;
4846 
4847 	error = brt_load(spa);
4848 	if (error != 0) {
4849 		spa_load_failed(spa, "brt_load failed [error=%d]", error);
4850 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
4851 	}
4852 
4853 	return (0);
4854 }
4855 
4856 static int
4857 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport)
4858 {
4859 	vdev_t *rvd = spa->spa_root_vdev;
4860 
4861 	if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) {
4862 		boolean_t missing = spa_check_logs(spa);
4863 		if (missing) {
4864 			if (spa->spa_missing_tvds != 0) {
4865 				spa_load_note(spa, "spa_check_logs failed "
4866 				    "so dropping the logs");
4867 			} else {
4868 				*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
4869 				spa_load_failed(spa, "spa_check_logs failed");
4870 				return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG,
4871 				    ENXIO));
4872 			}
4873 		}
4874 	}
4875 
4876 	return (0);
4877 }
4878 
4879 static int
4880 spa_ld_verify_pool_data(spa_t *spa)
4881 {
4882 	int error = 0;
4883 	vdev_t *rvd = spa->spa_root_vdev;
4884 
4885 	/*
4886 	 * We've successfully opened the pool, verify that we're ready
4887 	 * to start pushing transactions.
4888 	 */
4889 	if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
4890 		error = spa_load_verify(spa);
4891 		if (error != 0) {
4892 			spa_load_failed(spa, "spa_load_verify failed "
4893 			    "[error=%d]", error);
4894 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
4895 			    error));
4896 		}
4897 	}
4898 
4899 	return (0);
4900 }
4901 
4902 static void
4903 spa_ld_claim_log_blocks(spa_t *spa)
4904 {
4905 	dmu_tx_t *tx;
4906 	dsl_pool_t *dp = spa_get_dsl(spa);
4907 
4908 	/*
4909 	 * Claim log blocks that haven't been committed yet.
4910 	 * This must all happen in a single txg.
4911 	 * Note: spa_claim_max_txg is updated by spa_claim_notify(),
4912 	 * invoked from zil_claim_log_block()'s i/o done callback.
4913 	 * Price of rollback is that we abandon the log.
4914 	 */
4915 	spa->spa_claiming = B_TRUE;
4916 
4917 	tx = dmu_tx_create_assigned(dp, spa_first_txg(spa));
4918 	(void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
4919 	    zil_claim, tx, DS_FIND_CHILDREN);
4920 	dmu_tx_commit(tx);
4921 
4922 	spa->spa_claiming = B_FALSE;
4923 
4924 	spa_set_log_state(spa, SPA_LOG_GOOD);
4925 }
4926 
4927 static void
4928 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg,
4929     boolean_t update_config_cache)
4930 {
4931 	vdev_t *rvd = spa->spa_root_vdev;
4932 	int need_update = B_FALSE;
4933 
4934 	/*
4935 	 * If the config cache is stale, or we have uninitialized
4936 	 * metaslabs (see spa_vdev_add()), then update the config.
4937 	 *
4938 	 * If this is a verbatim import, trust the current
4939 	 * in-core spa_config and update the disk labels.
4940 	 */
4941 	if (update_config_cache || config_cache_txg != spa->spa_config_txg ||
4942 	    spa->spa_load_state == SPA_LOAD_IMPORT ||
4943 	    spa->spa_load_state == SPA_LOAD_RECOVER ||
4944 	    (spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
4945 		need_update = B_TRUE;
4946 
4947 	for (int c = 0; c < rvd->vdev_children; c++)
4948 		if (rvd->vdev_child[c]->vdev_ms_array == 0)
4949 			need_update = B_TRUE;
4950 
4951 	/*
4952 	 * Update the config cache asynchronously in case we're the
4953 	 * root pool, in which case the config cache isn't writable yet.
4954 	 */
4955 	if (need_update)
4956 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
4957 }
4958 
4959 static void
4960 spa_ld_prepare_for_reload(spa_t *spa)
4961 {
4962 	spa_mode_t mode = spa->spa_mode;
4963 	int async_suspended = spa->spa_async_suspended;
4964 
4965 	spa_unload(spa);
4966 	spa_deactivate(spa);
4967 	spa_activate(spa, mode);
4968 
4969 	/*
4970 	 * We save the value of spa_async_suspended as it gets reset to 0 by
4971 	 * spa_unload(). We want to restore it back to the original value before
4972 	 * returning as we might be calling spa_async_resume() later.
4973 	 */
4974 	spa->spa_async_suspended = async_suspended;
4975 }
4976 
4977 static int
4978 spa_ld_read_checkpoint_txg(spa_t *spa)
4979 {
4980 	uberblock_t checkpoint;
4981 	int error = 0;
4982 
4983 	ASSERT0(spa->spa_checkpoint_txg);
4984 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
4985 
4986 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4987 	    DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
4988 	    sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);
4989 
4990 	if (error == ENOENT)
4991 		return (0);
4992 
4993 	if (error != 0)
4994 		return (error);
4995 
4996 	ASSERT3U(checkpoint.ub_txg, !=, 0);
4997 	ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0);
4998 	ASSERT3U(checkpoint.ub_timestamp, !=, 0);
4999 	spa->spa_checkpoint_txg = checkpoint.ub_txg;
5000 	spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp;
5001 
5002 	return (0);
5003 }
5004 
5005 static int
5006 spa_ld_mos_init(spa_t *spa, spa_import_type_t type)
5007 {
5008 	int error = 0;
5009 
5010 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5011 	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
5012 
5013 	/*
5014 	 * Never trust the config that is provided unless we are assembling
5015 	 * a pool following a split.
5016 	 * This means don't trust blkptrs and the vdev tree in general. This
5017 	 * also effectively puts the spa in read-only mode since
5018 	 * spa_writeable() checks for spa_trust_config to be true.
5019 	 * We will later load a trusted config from the MOS.
5020 	 */
5021 	if (type != SPA_IMPORT_ASSEMBLE)
5022 		spa->spa_trust_config = B_FALSE;
5023 
5024 	/*
5025 	 * Parse the config provided to create a vdev tree.
5026 	 */
5027 	error = spa_ld_parse_config(spa, type);
5028 	if (error != 0)
5029 		return (error);
5030 
5031 	spa_import_progress_add(spa);
5032 
5033 	/*
5034 	 * Now that we have the vdev tree, try to open each vdev. This involves
5035 	 * opening the underlying physical device, retrieving its geometry and
5036 	 * probing the vdev with a dummy I/O. The state of each vdev will be set
5037 	 * based on the success of those operations. After this we'll be ready
5038 	 * to read from the vdevs.
5039 	 */
5040 	error = spa_ld_open_vdevs(spa);
5041 	if (error != 0)
5042 		return (error);
5043 
5044 	/*
5045 	 * Read the label of each vdev and make sure that the GUIDs stored
5046 	 * there match the GUIDs in the config provided.
5047 	 * If we're assembling a new pool that's been split off from an
5048 	 * existing pool, the labels haven't yet been updated so we skip
5049 	 * validation for now.
5050 	 */
5051 	if (type != SPA_IMPORT_ASSEMBLE) {
5052 		error = spa_ld_validate_vdevs(spa);
5053 		if (error != 0)
5054 			return (error);
5055 	}
5056 
5057 	/*
5058 	 * Read all vdev labels to find the best uberblock (i.e. latest,
5059 	 * unless spa_load_max_txg is set) and store it in spa_uberblock. We
5060 	 * get the list of features required to read blkptrs in the MOS from
5061 	 * the vdev label with the best uberblock and verify that our version
5062 	 * of zfs supports them all.
5063 	 */
5064 	error = spa_ld_select_uberblock(spa, type);
5065 	if (error != 0)
5066 		return (error);
5067 
5068 	/*
5069 	 * Pass that uberblock to the dsl_pool layer which will open the root
5070 	 * blkptr. This blkptr points to the latest version of the MOS and will
5071 	 * allow us to read its contents.
5072 	 */
5073 	error = spa_ld_open_rootbp(spa);
5074 	if (error != 0)
5075 		return (error);
5076 
5077 	return (0);
5078 }
5079 
5080 static int
5081 spa_ld_checkpoint_rewind(spa_t *spa)
5082 {
5083 	uberblock_t checkpoint;
5084 	int error = 0;
5085 
5086 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5087 	ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5088 
5089 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
5090 	    DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t),
5091 	    sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint);
5092 
5093 	if (error != 0) {
5094 		spa_load_failed(spa, "unable to retrieve checkpointed "
5095 		    "uberblock from the MOS config [error=%d]", error);
5096 
5097 		if (error == ENOENT)
5098 			error = ZFS_ERR_NO_CHECKPOINT;
5099 
5100 		return (error);
5101 	}
5102 
5103 	ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg);
5104 	ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg);
5105 
5106 	/*
5107 	 * We need to update the txg and timestamp of the checkpointed
5108 	 * uberblock to be higher than the latest one. This ensures that
5109 	 * the checkpointed uberblock is selected if we were to close and
5110 	 * reopen the pool right after we've written it in the vdev labels.
5111 	 * (also see block comment in vdev_uberblock_compare)
5112 	 */
5113 	checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1;
5114 	checkpoint.ub_timestamp = gethrestime_sec();
5115 
5116 	/*
5117 	 * Set current uberblock to be the checkpointed uberblock.
5118 	 */
5119 	spa->spa_uberblock = checkpoint;
5120 
5121 	/*
5122 	 * If we are doing a normal rewind, then the pool is open for
5123 	 * writing and we sync the "updated" checkpointed uberblock to
5124 	 * disk. Once this is done, we've basically rewound the whole
5125 	 * pool and there is no way back.
5126 	 *
5127 	 * There are cases when we don't want to attempt and sync the
5128 	 * checkpointed uberblock to disk because we are opening a
5129 	 * pool as read-only. Specifically, verifying the checkpointed
5130 	 * state with zdb, and importing the checkpointed state to get
5131 	 * a "preview" of its content.
5132 	 */
5133 	if (spa_writeable(spa)) {
5134 		vdev_t *rvd = spa->spa_root_vdev;
5135 
5136 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5137 		vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
5138 		int svdcount = 0;
5139 		int children = rvd->vdev_children;
5140 		int c0 = random_in_range(children);
5141 
5142 		for (int c = 0; c < children; c++) {
5143 			vdev_t *vd = rvd->vdev_child[(c0 + c) % children];
5144 
5145 			/* Stop when revisiting the first vdev */
5146 			if (c > 0 && svd[0] == vd)
5147 				break;
5148 
5149 			if (vd->vdev_ms_array == 0 || vd->vdev_islog ||
5150 			    !vdev_is_concrete(vd))
5151 				continue;
5152 
5153 			svd[svdcount++] = vd;
5154 			if (svdcount == SPA_SYNC_MIN_VDEVS)
5155 				break;
5156 		}
5157 		error = vdev_config_sync(svd, svdcount, spa->spa_first_txg);
5158 		if (error == 0)
5159 			spa->spa_last_synced_guid = rvd->vdev_guid;
5160 		spa_config_exit(spa, SCL_ALL, FTAG);
5161 
5162 		if (error != 0) {
5163 			spa_load_failed(spa, "failed to write checkpointed "
5164 			    "uberblock to the vdev labels [error=%d]", error);
5165 			return (error);
5166 		}
5167 	}
5168 
5169 	return (0);
5170 }
5171 
5172 static int
5173 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type,
5174     boolean_t *update_config_cache)
5175 {
5176 	int error;
5177 
5178 	/*
5179 	 * Parse the config for pool, open and validate vdevs,
5180 	 * select an uberblock, and use that uberblock to open
5181 	 * the MOS.
5182 	 */
5183 	error = spa_ld_mos_init(spa, type);
5184 	if (error != 0)
5185 		return (error);
5186 
5187 	/*
5188 	 * Retrieve the trusted config stored in the MOS and use it to create
5189 	 * a new, exact version of the vdev tree, then reopen all vdevs.
5190 	 */
5191 	error = spa_ld_trusted_config(spa, type, B_FALSE);
5192 	if (error == EAGAIN) {
5193 		if (update_config_cache != NULL)
5194 			*update_config_cache = B_TRUE;
5195 
5196 		/*
5197 		 * Redo the loading process with the trusted config if it is
5198 		 * too different from the untrusted config.
5199 		 */
5200 		spa_ld_prepare_for_reload(spa);
5201 		spa_load_note(spa, "RELOADING");
5202 		error = spa_ld_mos_init(spa, type);
5203 		if (error != 0)
5204 			return (error);
5205 
5206 		error = spa_ld_trusted_config(spa, type, B_TRUE);
5207 		if (error != 0)
5208 			return (error);
5209 
5210 	} else if (error != 0) {
5211 		return (error);
5212 	}
5213 
5214 	return (0);
5215 }
5216 
5217 /*
5218  * Load an existing storage pool, using the config provided. This config
5219  * describes which vdevs are part of the pool and is later validated against
5220  * partial configs present in each vdev's label and an entire copy of the
5221  * config stored in the MOS.
5222  */
5223 static int
5224 spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport)
5225 {
5226 	int error = 0;
5227 	boolean_t missing_feat_write = B_FALSE;
5228 	boolean_t checkpoint_rewind =
5229 	    (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5230 	boolean_t update_config_cache = B_FALSE;
5231 
5232 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5233 	ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
5234 
5235 	spa_load_note(spa, "LOADING");
5236 
5237 	error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache);
5238 	if (error != 0)
5239 		return (error);
5240 
5241 	/*
5242 	 * If we are rewinding to the checkpoint then we need to repeat
5243 	 * everything we've done so far in this function but this time
5244 	 * selecting the checkpointed uberblock and using that to open
5245 	 * the MOS.
5246 	 */
5247 	if (checkpoint_rewind) {
5248 		/*
5249 		 * If we are rewinding to the checkpoint update config cache
5250 		 * anyway.
5251 		 */
5252 		update_config_cache = B_TRUE;
5253 
5254 		/*
5255 		 * Extract the checkpointed uberblock from the current MOS
5256 		 * and use this as the pool's uberblock from now on. If the
5257 		 * pool is imported as writeable we also write the checkpoint
5258 		 * uberblock to the labels, making the rewind permanent.
5259 		 */
5260 		error = spa_ld_checkpoint_rewind(spa);
5261 		if (error != 0)
5262 			return (error);
5263 
5264 		/*
5265 		 * Redo the loading process again with the
5266 		 * checkpointed uberblock.
5267 		 */
5268 		spa_ld_prepare_for_reload(spa);
5269 		spa_load_note(spa, "LOADING checkpointed uberblock");
5270 		error = spa_ld_mos_with_trusted_config(spa, type, NULL);
5271 		if (error != 0)
5272 			return (error);
5273 	}
5274 
5275 	/*
5276 	 * Retrieve the checkpoint txg if the pool has a checkpoint.
5277 	 */
5278 	spa_import_progress_set_notes(spa, "Loading checkpoint txg");
5279 	error = spa_ld_read_checkpoint_txg(spa);
5280 	if (error != 0)
5281 		return (error);
5282 
5283 	/*
5284 	 * Retrieve the mapping of indirect vdevs. Those vdevs were removed
5285 	 * from the pool and their contents were re-mapped to other vdevs. Note
5286 	 * that everything that we read before this step must have been
5287 	 * rewritten on concrete vdevs after the last device removal was
5288 	 * initiated. Otherwise we could be reading from indirect vdevs before
5289 	 * we have loaded their mappings.
5290 	 */
5291 	spa_import_progress_set_notes(spa, "Loading indirect vdev metadata");
5292 	error = spa_ld_open_indirect_vdev_metadata(spa);
5293 	if (error != 0)
5294 		return (error);
5295 
5296 	/*
5297 	 * Retrieve the full list of active features from the MOS and check if
5298 	 * they are all supported.
5299 	 */
5300 	spa_import_progress_set_notes(spa, "Checking feature flags");
5301 	error = spa_ld_check_features(spa, &missing_feat_write);
5302 	if (error != 0)
5303 		return (error);
5304 
5305 	/*
5306 	 * Load several special directories from the MOS needed by the dsl_pool
5307 	 * layer.
5308 	 */
5309 	spa_import_progress_set_notes(spa, "Loading special MOS directories");
5310 	error = spa_ld_load_special_directories(spa);
5311 	if (error != 0)
5312 		return (error);
5313 
5314 	/*
5315 	 * Retrieve pool properties from the MOS.
5316 	 */
5317 	spa_import_progress_set_notes(spa, "Loading properties");
5318 	error = spa_ld_get_props(spa);
5319 	if (error != 0)
5320 		return (error);
5321 
5322 	/*
5323 	 * Retrieve the list of auxiliary devices - cache devices and spares -
5324 	 * and open them.
5325 	 */
5326 	spa_import_progress_set_notes(spa, "Loading AUX vdevs");
5327 	error = spa_ld_open_aux_vdevs(spa, type);
5328 	if (error != 0)
5329 		return (error);
5330 
5331 	/*
5332 	 * Load the metadata for all vdevs. Also check if unopenable devices
5333 	 * should be autoreplaced.
5334 	 */
5335 	spa_import_progress_set_notes(spa, "Loading vdev metadata");
5336 	error = spa_ld_load_vdev_metadata(spa);
5337 	if (error != 0)
5338 		return (error);
5339 
5340 	spa_import_progress_set_notes(spa, "Loading dedup tables");
5341 	error = spa_ld_load_dedup_tables(spa);
5342 	if (error != 0)
5343 		return (error);
5344 
5345 	spa_import_progress_set_notes(spa, "Loading BRT");
5346 	error = spa_ld_load_brt(spa);
5347 	if (error != 0)
5348 		return (error);
5349 
5350 	/*
5351 	 * Verify the logs now to make sure we don't have any unexpected errors
5352 	 * when we claim log blocks later.
5353 	 */
5354 	spa_import_progress_set_notes(spa, "Verifying Log Devices");
5355 	error = spa_ld_verify_logs(spa, type, ereport);
5356 	if (error != 0)
5357 		return (error);
5358 
5359 	if (missing_feat_write) {
5360 		ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT);
5361 
5362 		/*
5363 		 * At this point, we know that we can open the pool in
5364 		 * read-only mode but not read-write mode. We now have enough
5365 		 * information and can return to userland.
5366 		 */
5367 		return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT,
5368 		    ENOTSUP));
5369 	}
5370 
5371 	/*
5372 	 * Traverse the last txgs to make sure the pool was left off in a safe
5373 	 * state. When performing an extreme rewind, we verify the whole pool,
5374 	 * which can take a very long time.
5375 	 */
5376 	spa_import_progress_set_notes(spa, "Verifying pool data");
5377 	error = spa_ld_verify_pool_data(spa);
5378 	if (error != 0)
5379 		return (error);
5380 
5381 	/*
5382 	 * Calculate the deflated space for the pool. This must be done before
5383 	 * we write anything to the pool because we'd need to update the space
5384 	 * accounting using the deflated sizes.
5385 	 */
5386 	spa_import_progress_set_notes(spa, "Calculating deflated space");
5387 	spa_update_dspace(spa);
5388 
5389 	/*
5390 	 * We have now retrieved all the information we needed to open the
5391 	 * pool. If we are importing the pool in read-write mode, a few
5392 	 * additional steps must be performed to finish the import.
5393 	 */
5394 	spa_import_progress_set_notes(spa, "Starting import");
5395 	if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER ||
5396 	    spa->spa_load_max_txg == UINT64_MAX)) {
5397 		uint64_t config_cache_txg = spa->spa_config_txg;
5398 
5399 		ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT);
5400 
5401 		/*
5402 		 * Before we do any zio_write's, complete the raidz expansion
5403 		 * scratch space copying, if necessary.
5404 		 */
5405 		if (RRSS_GET_STATE(&spa->spa_uberblock) == RRSS_SCRATCH_VALID)
5406 			vdev_raidz_reflow_copy_scratch(spa);
5407 
5408 		/*
5409 		 * In case of a checkpoint rewind, log the original txg
5410 		 * of the checkpointed uberblock.
5411 		 */
5412 		if (checkpoint_rewind) {
5413 			spa_history_log_internal(spa, "checkpoint rewind",
5414 			    NULL, "rewound state to txg=%llu",
5415 			    (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg);
5416 		}
5417 
5418 		spa_import_progress_set_notes(spa, "Claiming ZIL blocks");
5419 		/*
5420 		 * Traverse the ZIL and claim all blocks.
5421 		 */
5422 		spa_ld_claim_log_blocks(spa);
5423 
5424 		/*
5425 		 * Kick-off the syncing thread.
5426 		 */
5427 		spa->spa_sync_on = B_TRUE;
5428 		txg_sync_start(spa->spa_dsl_pool);
5429 		mmp_thread_start(spa);
5430 
5431 		/*
5432 		 * Wait for all claims to sync.  We sync up to the highest
5433 		 * claimed log block birth time so that claimed log blocks
5434 		 * don't appear to be from the future.  spa_claim_max_txg
5435 		 * will have been set for us by ZIL traversal operations
5436 		 * performed above.
5437 		 */
5438 		spa_import_progress_set_notes(spa, "Syncing ZIL claims");
5439 		txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);
5440 
5441 		/*
5442 		 * Check if we need to request an update of the config. On the
5443 		 * next sync, we would update the config stored in vdev labels
5444 		 * and the cachefile (by default /etc/zfs/zpool.cache).
5445 		 */
5446 		spa_import_progress_set_notes(spa, "Updating configs");
5447 		spa_ld_check_for_config_update(spa, config_cache_txg,
5448 		    update_config_cache);
5449 
5450 		/*
5451 		 * Check if a rebuild was in progress and if so resume it.
5452 		 * Then check all DTLs to see if anything needs resilvering.
5453 		 * The resilver will be deferred if a rebuild was started.
5454 		 */
5455 		spa_import_progress_set_notes(spa, "Starting resilvers");
5456 		if (vdev_rebuild_active(spa->spa_root_vdev)) {
5457 			vdev_rebuild_restart(spa);
5458 		} else if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
5459 		    vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
5460 			spa_async_request(spa, SPA_ASYNC_RESILVER);
5461 		}
5462 
5463 		/*
5464 		 * Log the fact that we booted up (so that we can detect if
5465 		 * we rebooted in the middle of an operation).
5466 		 */
5467 		spa_history_log_version(spa, "open", NULL);
5468 
5469 		spa_import_progress_set_notes(spa,
5470 		    "Restarting device removals");
5471 		spa_restart_removal(spa);
5472 		spa_spawn_aux_threads(spa);
5473 
5474 		/*
5475 		 * Delete any inconsistent datasets.
5476 		 *
5477 		 * Note:
5478 		 * Since we may be issuing deletes for clones here,
5479 		 * we make sure to do so after we've spawned all the
5480 		 * auxiliary threads above (from which the livelist
5481 		 * deletion zthr is part of).
5482 		 */
5483 		spa_import_progress_set_notes(spa,
5484 		    "Cleaning up inconsistent objsets");
5485 		(void) dmu_objset_find(spa_name(spa),
5486 		    dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN);
5487 
5488 		/*
5489 		 * Clean up any stale temporary dataset userrefs.
5490 		 */
5491 		spa_import_progress_set_notes(spa,
5492 		    "Cleaning up temporary userrefs");
5493 		dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);
5494 
5495 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
5496 		spa_import_progress_set_notes(spa, "Restarting initialize");
5497 		vdev_initialize_restart(spa->spa_root_vdev);
5498 		spa_import_progress_set_notes(spa, "Restarting TRIM");
5499 		vdev_trim_restart(spa->spa_root_vdev);
5500 		vdev_autotrim_restart(spa);
5501 		spa_config_exit(spa, SCL_CONFIG, FTAG);
5502 		spa_import_progress_set_notes(spa, "Finished importing");
5503 	}
5504 
5505 	spa_import_progress_remove(spa_guid(spa));
5506 	spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD);
5507 
5508 	spa_load_note(spa, "LOADED");
5509 
5510 	return (0);
5511 }
5512 
5513 static int
5514 spa_load_retry(spa_t *spa, spa_load_state_t state)
5515 {
5516 	spa_mode_t mode = spa->spa_mode;
5517 
5518 	spa_unload(spa);
5519 	spa_deactivate(spa);
5520 
5521 	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
5522 
5523 	spa_activate(spa, mode);
5524 	spa_async_suspend(spa);
5525 
5526 	spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu",
5527 	    (u_longlong_t)spa->spa_load_max_txg);
5528 
5529 	return (spa_load(spa, state, SPA_IMPORT_EXISTING));
5530 }
5531 
5532 /*
5533  * If spa_load() fails this function will try loading prior txg's. If
5534  * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool
5535  * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this
5536  * function will not rewind the pool and will return the same error as
5537  * spa_load().
5538  */
5539 static int
5540 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request,
5541     int rewind_flags)
5542 {
5543 	nvlist_t *loadinfo = NULL;
5544 	nvlist_t *config = NULL;
5545 	int load_error, rewind_error;
5546 	uint64_t safe_rewind_txg;
5547 	uint64_t min_txg;
5548 
5549 	if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
5550 		spa->spa_load_max_txg = spa->spa_load_txg;
5551 		spa_set_log_state(spa, SPA_LOG_CLEAR);
5552 	} else {
5553 		spa->spa_load_max_txg = max_request;
5554 		if (max_request != UINT64_MAX)
5555 			spa->spa_extreme_rewind = B_TRUE;
5556 	}
5557 
5558 	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING);
5559 	if (load_error == 0)
5560 		return (0);
5561 	if (load_error == ZFS_ERR_NO_CHECKPOINT) {
5562 		/*
5563 		 * When attempting checkpoint-rewind on a pool with no
5564 		 * checkpoint, we should not attempt to load uberblocks
5565 		 * from previous txgs when spa_load fails.
5566 		 */
5567 		ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5568 		spa_import_progress_remove(spa_guid(spa));
5569 		return (load_error);
5570 	}
5571 
5572 	if (spa->spa_root_vdev != NULL)
5573 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5574 
5575 	spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
5576 	spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;
5577 
5578 	if (rewind_flags & ZPOOL_NEVER_REWIND) {
5579 		nvlist_free(config);
5580 		spa_import_progress_remove(spa_guid(spa));
5581 		return (load_error);
5582 	}
5583 
5584 	if (state == SPA_LOAD_RECOVER) {
5585 		/* Price of rolling back is discarding txgs, including log */
5586 		spa_set_log_state(spa, SPA_LOG_CLEAR);
5587 	} else {
5588 		/*
5589 		 * If we aren't rolling back save the load info from our first
5590 		 * import attempt so that we can restore it after attempting
5591 		 * to rewind.
5592 		 */
5593 		loadinfo = spa->spa_load_info;
5594 		spa->spa_load_info = fnvlist_alloc();
5595 	}
5596 
5597 	spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
5598 	safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
5599 	min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ?
5600 	    TXG_INITIAL : safe_rewind_txg;
5601 
5602 	/*
5603 	 * Continue as long as we're finding errors, we're still within
5604 	 * the acceptable rewind range, and we're still finding uberblocks
5605 	 */
5606 	while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
5607 	    spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
5608 		if (spa->spa_load_max_txg < safe_rewind_txg)
5609 			spa->spa_extreme_rewind = B_TRUE;
5610 		rewind_error = spa_load_retry(spa, state);
5611 	}
5612 
5613 	spa->spa_extreme_rewind = B_FALSE;
5614 	spa->spa_load_max_txg = UINT64_MAX;
5615 
5616 	if (config && (rewind_error || state != SPA_LOAD_RECOVER))
5617 		spa_config_set(spa, config);
5618 	else
5619 		nvlist_free(config);
5620 
5621 	if (state == SPA_LOAD_RECOVER) {
5622 		ASSERT3P(loadinfo, ==, NULL);
5623 		spa_import_progress_remove(spa_guid(spa));
5624 		return (rewind_error);
5625 	} else {
5626 		/* Store the rewind info as part of the initial load info */
5627 		fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO,
5628 		    spa->spa_load_info);
5629 
5630 		/* Restore the initial load info */
5631 		fnvlist_free(spa->spa_load_info);
5632 		spa->spa_load_info = loadinfo;
5633 
5634 		spa_import_progress_remove(spa_guid(spa));
5635 		return (load_error);
5636 	}
5637 }
5638 
5639 /*
5640  * Pool Open/Import
5641  *
5642  * The import case is identical to an open except that the configuration is sent
5643  * down from userland, instead of grabbed from the configuration cache.  For the
5644  * case of an open, the pool configuration will exist in the
5645  * POOL_STATE_UNINITIALIZED state.
5646  *
5647  * The stats information (gen/count/ustats) is used to gather vdev statistics at
5648  * the same time open the pool, without having to keep around the spa_t in some
5649  * ambiguous state.
5650  */
5651 static int
5652 spa_open_common(const char *pool, spa_t **spapp, const void *tag,
5653     nvlist_t *nvpolicy, nvlist_t **config)
5654 {
5655 	spa_t *spa;
5656 	spa_load_state_t state = SPA_LOAD_OPEN;
5657 	int error;
5658 	int locked = B_FALSE;
5659 	int firstopen = B_FALSE;
5660 
5661 	*spapp = NULL;
5662 
5663 	/*
5664 	 * As disgusting as this is, we need to support recursive calls to this
5665 	 * function because dsl_dir_open() is called during spa_load(), and ends
5666 	 * up calling spa_open() again.  The real fix is to figure out how to
5667 	 * avoid dsl_dir_open() calling this in the first place.
5668 	 */
5669 	if (MUTEX_NOT_HELD(&spa_namespace_lock)) {
5670 		mutex_enter(&spa_namespace_lock);
5671 		locked = B_TRUE;
5672 	}
5673 
5674 	if ((spa = spa_lookup(pool)) == NULL) {
5675 		if (locked)
5676 			mutex_exit(&spa_namespace_lock);
5677 		return (SET_ERROR(ENOENT));
5678 	}
5679 
5680 	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
5681 		zpool_load_policy_t policy;
5682 
5683 		firstopen = B_TRUE;
5684 
5685 		zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config,
5686 		    &policy);
5687 		if (policy.zlp_rewind & ZPOOL_DO_REWIND)
5688 			state = SPA_LOAD_RECOVER;
5689 
5690 		spa_activate(spa, spa_mode_global);
5691 
5692 		if (state != SPA_LOAD_RECOVER)
5693 			spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
5694 		spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
5695 
5696 		zfs_dbgmsg("spa_open_common: opening %s", pool);
5697 		error = spa_load_best(spa, state, policy.zlp_txg,
5698 		    policy.zlp_rewind);
5699 
5700 		if (error == EBADF) {
5701 			/*
5702 			 * If vdev_validate() returns failure (indicated by
5703 			 * EBADF), it indicates that one of the vdevs indicates
5704 			 * that the pool has been exported or destroyed.  If
5705 			 * this is the case, the config cache is out of sync and
5706 			 * we should remove the pool from the namespace.
5707 			 */
5708 			spa_unload(spa);
5709 			spa_deactivate(spa);
5710 			spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
5711 			spa_remove(spa);
5712 			if (locked)
5713 				mutex_exit(&spa_namespace_lock);
5714 			return (SET_ERROR(ENOENT));
5715 		}
5716 
5717 		if (error) {
5718 			/*
5719 			 * We can't open the pool, but we still have useful
5720 			 * information: the state of each vdev after the
5721 			 * attempted vdev_open().  Return this to the user.
5722 			 */
5723 			if (config != NULL && spa->spa_config) {
5724 				*config = fnvlist_dup(spa->spa_config);
5725 				fnvlist_add_nvlist(*config,
5726 				    ZPOOL_CONFIG_LOAD_INFO,
5727 				    spa->spa_load_info);
5728 			}
5729 			spa_unload(spa);
5730 			spa_deactivate(spa);
5731 			spa->spa_last_open_failed = error;
5732 			if (locked)
5733 				mutex_exit(&spa_namespace_lock);
5734 			*spapp = NULL;
5735 			return (error);
5736 		}
5737 	}
5738 
5739 	spa_open_ref(spa, tag);
5740 
5741 	if (config != NULL)
5742 		*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5743 
5744 	/*
5745 	 * If we've recovered the pool, pass back any information we
5746 	 * gathered while doing the load.
5747 	 */
5748 	if (state == SPA_LOAD_RECOVER && config != NULL) {
5749 		fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO,
5750 		    spa->spa_load_info);
5751 	}
5752 
5753 	if (locked) {
5754 		spa->spa_last_open_failed = 0;
5755 		spa->spa_last_ubsync_txg = 0;
5756 		spa->spa_load_txg = 0;
5757 		mutex_exit(&spa_namespace_lock);
5758 	}
5759 
5760 	if (firstopen)
5761 		zvol_create_minors_recursive(spa_name(spa));
5762 
5763 	*spapp = spa;
5764 
5765 	return (0);
5766 }
5767 
5768 int
5769 spa_open_rewind(const char *name, spa_t **spapp, const void *tag,
5770     nvlist_t *policy, nvlist_t **config)
5771 {
5772 	return (spa_open_common(name, spapp, tag, policy, config));
5773 }
5774 
5775 int
5776 spa_open(const char *name, spa_t **spapp, const void *tag)
5777 {
5778 	return (spa_open_common(name, spapp, tag, NULL, NULL));
5779 }
5780 
5781 /*
5782  * Lookup the given spa_t, incrementing the inject count in the process,
5783  * preventing it from being exported or destroyed.
5784  */
5785 spa_t *
5786 spa_inject_addref(char *name)
5787 {
5788 	spa_t *spa;
5789 
5790 	mutex_enter(&spa_namespace_lock);
5791 	if ((spa = spa_lookup(name)) == NULL) {
5792 		mutex_exit(&spa_namespace_lock);
5793 		return (NULL);
5794 	}
5795 	spa->spa_inject_ref++;
5796 	mutex_exit(&spa_namespace_lock);
5797 
5798 	return (spa);
5799 }
5800 
5801 void
5802 spa_inject_delref(spa_t *spa)
5803 {
5804 	mutex_enter(&spa_namespace_lock);
5805 	spa->spa_inject_ref--;
5806 	mutex_exit(&spa_namespace_lock);
5807 }
5808 
5809 /*
5810  * Add spares device information to the nvlist.
5811  */
5812 static void
5813 spa_add_spares(spa_t *spa, nvlist_t *config)
5814 {
5815 	nvlist_t **spares;
5816 	uint_t i, nspares;
5817 	nvlist_t *nvroot;
5818 	uint64_t guid;
5819 	vdev_stat_t *vs;
5820 	uint_t vsc;
5821 	uint64_t pool;
5822 
5823 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5824 
5825 	if (spa->spa_spares.sav_count == 0)
5826 		return;
5827 
5828 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
5829 	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
5830 	    ZPOOL_CONFIG_SPARES, &spares, &nspares));
5831 	if (nspares != 0) {
5832 		fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
5833 		    (const nvlist_t * const *)spares, nspares);
5834 		VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
5835 		    &spares, &nspares));
5836 
5837 		/*
5838 		 * Go through and find any spares which have since been
5839 		 * repurposed as an active spare.  If this is the case, update
5840 		 * their status appropriately.
5841 		 */
5842 		for (i = 0; i < nspares; i++) {
5843 			guid = fnvlist_lookup_uint64(spares[i],
5844 			    ZPOOL_CONFIG_GUID);
5845 			VERIFY0(nvlist_lookup_uint64_array(spares[i],
5846 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
5847 			if (spa_spare_exists(guid, &pool, NULL) &&
5848 			    pool != 0ULL) {
5849 				vs->vs_state = VDEV_STATE_CANT_OPEN;
5850 				vs->vs_aux = VDEV_AUX_SPARED;
5851 			} else {
5852 				vs->vs_state =
5853 				    spa->spa_spares.sav_vdevs[i]->vdev_state;
5854 			}
5855 		}
5856 	}
5857 }
5858 
5859 /*
5860  * Add l2cache device information to the nvlist, including vdev stats.
5861  */
5862 static void
5863 spa_add_l2cache(spa_t *spa, nvlist_t *config)
5864 {
5865 	nvlist_t **l2cache;
5866 	uint_t i, j, nl2cache;
5867 	nvlist_t *nvroot;
5868 	uint64_t guid;
5869 	vdev_t *vd;
5870 	vdev_stat_t *vs;
5871 	uint_t vsc;
5872 
5873 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5874 
5875 	if (spa->spa_l2cache.sav_count == 0)
5876 		return;
5877 
5878 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
5879 	VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
5880 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache));
5881 	if (nl2cache != 0) {
5882 		fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
5883 		    (const nvlist_t * const *)l2cache, nl2cache);
5884 		VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
5885 		    &l2cache, &nl2cache));
5886 
5887 		/*
5888 		 * Update level 2 cache device stats.
5889 		 */
5890 
5891 		for (i = 0; i < nl2cache; i++) {
5892 			guid = fnvlist_lookup_uint64(l2cache[i],
5893 			    ZPOOL_CONFIG_GUID);
5894 
5895 			vd = NULL;
5896 			for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
5897 				if (guid ==
5898 				    spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
5899 					vd = spa->spa_l2cache.sav_vdevs[j];
5900 					break;
5901 				}
5902 			}
5903 			ASSERT(vd != NULL);
5904 
5905 			VERIFY0(nvlist_lookup_uint64_array(l2cache[i],
5906 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
5907 			vdev_get_stats(vd, vs);
5908 			vdev_config_generate_stats(vd, l2cache[i]);
5909 
5910 		}
5911 	}
5912 }
5913 
5914 static void
5915 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features)
5916 {
5917 	zap_cursor_t zc;
5918 	zap_attribute_t za;
5919 
5920 	if (spa->spa_feat_for_read_obj != 0) {
5921 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
5922 		    spa->spa_feat_for_read_obj);
5923 		    zap_cursor_retrieve(&zc, &za) == 0;
5924 		    zap_cursor_advance(&zc)) {
5925 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
5926 			    za.za_num_integers == 1);
5927 			VERIFY0(nvlist_add_uint64(features, za.za_name,
5928 			    za.za_first_integer));
5929 		}
5930 		zap_cursor_fini(&zc);
5931 	}
5932 
5933 	if (spa->spa_feat_for_write_obj != 0) {
5934 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
5935 		    spa->spa_feat_for_write_obj);
5936 		    zap_cursor_retrieve(&zc, &za) == 0;
5937 		    zap_cursor_advance(&zc)) {
5938 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
5939 			    za.za_num_integers == 1);
5940 			VERIFY0(nvlist_add_uint64(features, za.za_name,
5941 			    za.za_first_integer));
5942 		}
5943 		zap_cursor_fini(&zc);
5944 	}
5945 }
5946 
5947 static void
5948 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features)
5949 {
5950 	int i;
5951 
5952 	for (i = 0; i < SPA_FEATURES; i++) {
5953 		zfeature_info_t feature = spa_feature_table[i];
5954 		uint64_t refcount;
5955 
5956 		if (feature_get_refcount(spa, &feature, &refcount) != 0)
5957 			continue;
5958 
5959 		VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount));
5960 	}
5961 }
5962 
5963 /*
5964  * Store a list of pool features and their reference counts in the
5965  * config.
5966  *
5967  * The first time this is called on a spa, allocate a new nvlist, fetch
5968  * the pool features and reference counts from disk, then save the list
5969  * in the spa. In subsequent calls on the same spa use the saved nvlist
5970  * and refresh its values from the cached reference counts.  This
5971  * ensures we don't block here on I/O on a suspended pool so 'zpool
5972  * clear' can resume the pool.
5973  */
5974 static void
5975 spa_add_feature_stats(spa_t *spa, nvlist_t *config)
5976 {
5977 	nvlist_t *features;
5978 
5979 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
5980 
5981 	mutex_enter(&spa->spa_feat_stats_lock);
5982 	features = spa->spa_feat_stats;
5983 
5984 	if (features != NULL) {
5985 		spa_feature_stats_from_cache(spa, features);
5986 	} else {
5987 		VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP));
5988 		spa->spa_feat_stats = features;
5989 		spa_feature_stats_from_disk(spa, features);
5990 	}
5991 
5992 	VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS,
5993 	    features));
5994 
5995 	mutex_exit(&spa->spa_feat_stats_lock);
5996 }
5997 
5998 int
5999 spa_get_stats(const char *name, nvlist_t **config,
6000     char *altroot, size_t buflen)
6001 {
6002 	int error;
6003 	spa_t *spa;
6004 
6005 	*config = NULL;
6006 	error = spa_open_common(name, &spa, FTAG, NULL, config);
6007 
6008 	if (spa != NULL) {
6009 		/*
6010 		 * This still leaves a window of inconsistency where the spares
6011 		 * or l2cache devices could change and the config would be
6012 		 * self-inconsistent.
6013 		 */
6014 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
6015 
6016 		if (*config != NULL) {
6017 			uint64_t loadtimes[2];
6018 
6019 			loadtimes[0] = spa->spa_loaded_ts.tv_sec;
6020 			loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
6021 			fnvlist_add_uint64_array(*config,
6022 			    ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2);
6023 
6024 			fnvlist_add_uint64(*config,
6025 			    ZPOOL_CONFIG_ERRCOUNT,
6026 			    spa_approx_errlog_size(spa));
6027 
6028 			if (spa_suspended(spa)) {
6029 				fnvlist_add_uint64(*config,
6030 				    ZPOOL_CONFIG_SUSPENDED,
6031 				    spa->spa_failmode);
6032 				fnvlist_add_uint64(*config,
6033 				    ZPOOL_CONFIG_SUSPENDED_REASON,
6034 				    spa->spa_suspended);
6035 			}
6036 
6037 			spa_add_spares(spa, *config);
6038 			spa_add_l2cache(spa, *config);
6039 			spa_add_feature_stats(spa, *config);
6040 		}
6041 	}
6042 
6043 	/*
6044 	 * We want to get the alternate root even for faulted pools, so we cheat
6045 	 * and call spa_lookup() directly.
6046 	 */
6047 	if (altroot) {
6048 		if (spa == NULL) {
6049 			mutex_enter(&spa_namespace_lock);
6050 			spa = spa_lookup(name);
6051 			if (spa)
6052 				spa_altroot(spa, altroot, buflen);
6053 			else
6054 				altroot[0] = '\0';
6055 			spa = NULL;
6056 			mutex_exit(&spa_namespace_lock);
6057 		} else {
6058 			spa_altroot(spa, altroot, buflen);
6059 		}
6060 	}
6061 
6062 	if (spa != NULL) {
6063 		spa_config_exit(spa, SCL_CONFIG, FTAG);
6064 		spa_close(spa, FTAG);
6065 	}
6066 
6067 	return (error);
6068 }
6069 
6070 /*
6071  * Validate that the auxiliary device array is well formed.  We must have an
6072  * array of nvlists, each which describes a valid leaf vdev.  If this is an
6073  * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be
6074  * specified, as long as they are well-formed.
6075  */
6076 static int
6077 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode,
6078     spa_aux_vdev_t *sav, const char *config, uint64_t version,
6079     vdev_labeltype_t label)
6080 {
6081 	nvlist_t **dev;
6082 	uint_t i, ndev;
6083 	vdev_t *vd;
6084 	int error;
6085 
6086 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
6087 
6088 	/*
6089 	 * It's acceptable to have no devs specified.
6090 	 */
6091 	if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0)
6092 		return (0);
6093 
6094 	if (ndev == 0)
6095 		return (SET_ERROR(EINVAL));
6096 
6097 	/*
6098 	 * Make sure the pool is formatted with a version that supports this
6099 	 * device type.
6100 	 */
6101 	if (spa_version(spa) < version)
6102 		return (SET_ERROR(ENOTSUP));
6103 
6104 	/*
6105 	 * Set the pending device list so we correctly handle device in-use
6106 	 * checking.
6107 	 */
6108 	sav->sav_pending = dev;
6109 	sav->sav_npending = ndev;
6110 
6111 	for (i = 0; i < ndev; i++) {
6112 		if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0,
6113 		    mode)) != 0)
6114 			goto out;
6115 
6116 		if (!vd->vdev_ops->vdev_op_leaf) {
6117 			vdev_free(vd);
6118 			error = SET_ERROR(EINVAL);
6119 			goto out;
6120 		}
6121 
6122 		vd->vdev_top = vd;
6123 
6124 		if ((error = vdev_open(vd)) == 0 &&
6125 		    (error = vdev_label_init(vd, crtxg, label)) == 0) {
6126 			fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID,
6127 			    vd->vdev_guid);
6128 		}
6129 
6130 		vdev_free(vd);
6131 
6132 		if (error &&
6133 		    (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE))
6134 			goto out;
6135 		else
6136 			error = 0;
6137 	}
6138 
6139 out:
6140 	sav->sav_pending = NULL;
6141 	sav->sav_npending = 0;
6142 	return (error);
6143 }
6144 
6145 static int
6146 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode)
6147 {
6148 	int error;
6149 
6150 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
6151 
6152 	if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode,
6153 	    &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
6154 	    VDEV_LABEL_SPARE)) != 0) {
6155 		return (error);
6156 	}
6157 
6158 	return (spa_validate_aux_devs(spa, nvroot, crtxg, mode,
6159 	    &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
6160 	    VDEV_LABEL_L2CACHE));
6161 }
6162 
6163 static void
6164 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs,
6165     const char *config)
6166 {
6167 	int i;
6168 
6169 	if (sav->sav_config != NULL) {
6170 		nvlist_t **olddevs;
6171 		uint_t oldndevs;
6172 		nvlist_t **newdevs;
6173 
6174 		/*
6175 		 * Generate new dev list by concatenating with the
6176 		 * current dev list.
6177 		 */
6178 		VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config,
6179 		    &olddevs, &oldndevs));
6180 
6181 		newdevs = kmem_alloc(sizeof (void *) *
6182 		    (ndevs + oldndevs), KM_SLEEP);
6183 		for (i = 0; i < oldndevs; i++)
6184 			newdevs[i] = fnvlist_dup(olddevs[i]);
6185 		for (i = 0; i < ndevs; i++)
6186 			newdevs[i + oldndevs] = fnvlist_dup(devs[i]);
6187 
6188 		fnvlist_remove(sav->sav_config, config);
6189 
6190 		fnvlist_add_nvlist_array(sav->sav_config, config,
6191 		    (const nvlist_t * const *)newdevs, ndevs + oldndevs);
6192 		for (i = 0; i < oldndevs + ndevs; i++)
6193 			nvlist_free(newdevs[i]);
6194 		kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *));
6195 	} else {
6196 		/*
6197 		 * Generate a new dev list.
6198 		 */
6199 		sav->sav_config = fnvlist_alloc();
6200 		fnvlist_add_nvlist_array(sav->sav_config, config,
6201 		    (const nvlist_t * const *)devs, ndevs);
6202 	}
6203 }
6204 
6205 /*
6206  * Stop and drop level 2 ARC devices
6207  */
6208 void
6209 spa_l2cache_drop(spa_t *spa)
6210 {
6211 	vdev_t *vd;
6212 	int i;
6213 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
6214 
6215 	for (i = 0; i < sav->sav_count; i++) {
6216 		uint64_t pool;
6217 
6218 		vd = sav->sav_vdevs[i];
6219 		ASSERT(vd != NULL);
6220 
6221 		if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
6222 		    pool != 0ULL && l2arc_vdev_present(vd))
6223 			l2arc_remove_vdev(vd);
6224 	}
6225 }
6226 
6227 /*
6228  * Verify encryption parameters for spa creation. If we are encrypting, we must
6229  * have the encryption feature flag enabled.
6230  */
6231 static int
6232 spa_create_check_encryption_params(dsl_crypto_params_t *dcp,
6233     boolean_t has_encryption)
6234 {
6235 	if (dcp->cp_crypt != ZIO_CRYPT_OFF &&
6236 	    dcp->cp_crypt != ZIO_CRYPT_INHERIT &&
6237 	    !has_encryption)
6238 		return (SET_ERROR(ENOTSUP));
6239 
6240 	return (dmu_objset_create_crypt_check(NULL, dcp, NULL));
6241 }
6242 
6243 /*
6244  * Pool Creation
6245  */
6246 int
6247 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
6248     nvlist_t *zplprops, dsl_crypto_params_t *dcp)
6249 {
6250 	spa_t *spa;
6251 	const char *altroot = NULL;
6252 	vdev_t *rvd;
6253 	dsl_pool_t *dp;
6254 	dmu_tx_t *tx;
6255 	int error = 0;
6256 	uint64_t txg = TXG_INITIAL;
6257 	nvlist_t **spares, **l2cache;
6258 	uint_t nspares, nl2cache;
6259 	uint64_t version, obj, ndraid = 0;
6260 	boolean_t has_features;
6261 	boolean_t has_encryption;
6262 	boolean_t has_allocclass;
6263 	spa_feature_t feat;
6264 	const char *feat_name;
6265 	const char *poolname;
6266 	nvlist_t *nvl;
6267 
6268 	if (props == NULL ||
6269 	    nvlist_lookup_string(props, "tname", &poolname) != 0)
6270 		poolname = (char *)pool;
6271 
6272 	/*
6273 	 * If this pool already exists, return failure.
6274 	 */
6275 	mutex_enter(&spa_namespace_lock);
6276 	if (spa_lookup(poolname) != NULL) {
6277 		mutex_exit(&spa_namespace_lock);
6278 		return (SET_ERROR(EEXIST));
6279 	}
6280 
6281 	/*
6282 	 * Allocate a new spa_t structure.
6283 	 */
6284 	nvl = fnvlist_alloc();
6285 	fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool);
6286 	(void) nvlist_lookup_string(props,
6287 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
6288 	spa = spa_add(poolname, nvl, altroot);
6289 	fnvlist_free(nvl);
6290 	spa_activate(spa, spa_mode_global);
6291 
6292 	if (props && (error = spa_prop_validate(spa, props))) {
6293 		spa_deactivate(spa);
6294 		spa_remove(spa);
6295 		mutex_exit(&spa_namespace_lock);
6296 		return (error);
6297 	}
6298 
6299 	/*
6300 	 * Temporary pool names should never be written to disk.
6301 	 */
6302 	if (poolname != pool)
6303 		spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME;
6304 
6305 	has_features = B_FALSE;
6306 	has_encryption = B_FALSE;
6307 	has_allocclass = B_FALSE;
6308 	for (nvpair_t *elem = nvlist_next_nvpair(props, NULL);
6309 	    elem != NULL; elem = nvlist_next_nvpair(props, elem)) {
6310 		if (zpool_prop_feature(nvpair_name(elem))) {
6311 			has_features = B_TRUE;
6312 
6313 			feat_name = strchr(nvpair_name(elem), '@') + 1;
6314 			VERIFY0(zfeature_lookup_name(feat_name, &feat));
6315 			if (feat == SPA_FEATURE_ENCRYPTION)
6316 				has_encryption = B_TRUE;
6317 			if (feat == SPA_FEATURE_ALLOCATION_CLASSES)
6318 				has_allocclass = B_TRUE;
6319 		}
6320 	}
6321 
6322 	/* verify encryption params, if they were provided */
6323 	if (dcp != NULL) {
6324 		error = spa_create_check_encryption_params(dcp, has_encryption);
6325 		if (error != 0) {
6326 			spa_deactivate(spa);
6327 			spa_remove(spa);
6328 			mutex_exit(&spa_namespace_lock);
6329 			return (error);
6330 		}
6331 	}
6332 	if (!has_allocclass && zfs_special_devs(nvroot, NULL)) {
6333 		spa_deactivate(spa);
6334 		spa_remove(spa);
6335 		mutex_exit(&spa_namespace_lock);
6336 		return (ENOTSUP);
6337 	}
6338 
6339 	if (has_features || nvlist_lookup_uint64(props,
6340 	    zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) {
6341 		version = SPA_VERSION;
6342 	}
6343 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
6344 
6345 	spa->spa_first_txg = txg;
6346 	spa->spa_uberblock.ub_txg = txg - 1;
6347 	spa->spa_uberblock.ub_version = version;
6348 	spa->spa_ubsync = spa->spa_uberblock;
6349 	spa->spa_load_state = SPA_LOAD_CREATE;
6350 	spa->spa_removing_phys.sr_state = DSS_NONE;
6351 	spa->spa_removing_phys.sr_removing_vdev = -1;
6352 	spa->spa_removing_phys.sr_prev_indirect_vdev = -1;
6353 	spa->spa_indirect_vdevs_loaded = B_TRUE;
6354 
6355 	/*
6356 	 * Create "The Godfather" zio to hold all async IOs
6357 	 */
6358 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
6359 	    KM_SLEEP);
6360 	for (int i = 0; i < max_ncpus; i++) {
6361 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
6362 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
6363 		    ZIO_FLAG_GODFATHER);
6364 	}
6365 
6366 	/*
6367 	 * Create the root vdev.
6368 	 */
6369 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6370 
6371 	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);
6372 
6373 	ASSERT(error != 0 || rvd != NULL);
6374 	ASSERT(error != 0 || spa->spa_root_vdev == rvd);
6375 
6376 	if (error == 0 && !zfs_allocatable_devs(nvroot))
6377 		error = SET_ERROR(EINVAL);
6378 
6379 	if (error == 0 &&
6380 	    (error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
6381 	    (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 &&
6382 	    (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) {
6383 		/*
6384 		 * instantiate the metaslab groups (this will dirty the vdevs)
6385 		 * we can no longer error exit past this point
6386 		 */
6387 		for (int c = 0; error == 0 && c < rvd->vdev_children; c++) {
6388 			vdev_t *vd = rvd->vdev_child[c];
6389 
6390 			vdev_metaslab_set_size(vd);
6391 			vdev_expand(vd, txg);
6392 		}
6393 	}
6394 
6395 	spa_config_exit(spa, SCL_ALL, FTAG);
6396 
6397 	if (error != 0) {
6398 		spa_unload(spa);
6399 		spa_deactivate(spa);
6400 		spa_remove(spa);
6401 		mutex_exit(&spa_namespace_lock);
6402 		return (error);
6403 	}
6404 
6405 	/*
6406 	 * Get the list of spares, if specified.
6407 	 */
6408 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6409 	    &spares, &nspares) == 0) {
6410 		spa->spa_spares.sav_config = fnvlist_alloc();
6411 		fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
6412 		    ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
6413 		    nspares);
6414 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6415 		spa_load_spares(spa);
6416 		spa_config_exit(spa, SCL_ALL, FTAG);
6417 		spa->spa_spares.sav_sync = B_TRUE;
6418 	}
6419 
6420 	/*
6421 	 * Get the list of level 2 cache devices, if specified.
6422 	 */
6423 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6424 	    &l2cache, &nl2cache) == 0) {
6425 		VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config,
6426 		    NV_UNIQUE_NAME, KM_SLEEP));
6427 		fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
6428 		    ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache,
6429 		    nl2cache);
6430 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6431 		spa_load_l2cache(spa);
6432 		spa_config_exit(spa, SCL_ALL, FTAG);
6433 		spa->spa_l2cache.sav_sync = B_TRUE;
6434 	}
6435 
6436 	spa->spa_is_initializing = B_TRUE;
6437 	spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg);
6438 	spa->spa_is_initializing = B_FALSE;
6439 
6440 	/*
6441 	 * Create DDTs (dedup tables).
6442 	 */
6443 	ddt_create(spa);
6444 	/*
6445 	 * Create BRT table and BRT table object.
6446 	 */
6447 	brt_create(spa);
6448 
6449 	spa_update_dspace(spa);
6450 
6451 	tx = dmu_tx_create_assigned(dp, txg);
6452 
6453 	/*
6454 	 * Create the pool's history object.
6455 	 */
6456 	if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history)
6457 		spa_history_create_obj(spa, tx);
6458 
6459 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE);
6460 	spa_history_log_version(spa, "create", tx);
6461 
6462 	/*
6463 	 * Create the pool config object.
6464 	 */
6465 	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
6466 	    DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE,
6467 	    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
6468 
6469 	if (zap_add(spa->spa_meta_objset,
6470 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
6471 	    sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
6472 		cmn_err(CE_PANIC, "failed to add pool config");
6473 	}
6474 
6475 	if (zap_add(spa->spa_meta_objset,
6476 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION,
6477 	    sizeof (uint64_t), 1, &version, tx) != 0) {
6478 		cmn_err(CE_PANIC, "failed to add pool version");
6479 	}
6480 
6481 	/* Newly created pools with the right version are always deflated. */
6482 	if (version >= SPA_VERSION_RAIDZ_DEFLATE) {
6483 		spa->spa_deflate = TRUE;
6484 		if (zap_add(spa->spa_meta_objset,
6485 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
6486 		    sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
6487 			cmn_err(CE_PANIC, "failed to add deflate");
6488 		}
6489 	}
6490 
6491 	/*
6492 	 * Create the deferred-free bpobj.  Turn off compression
6493 	 * because sync-to-convergence takes longer if the blocksize
6494 	 * keeps changing.
6495 	 */
6496 	obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
6497 	dmu_object_set_compress(spa->spa_meta_objset, obj,
6498 	    ZIO_COMPRESS_OFF, tx);
6499 	if (zap_add(spa->spa_meta_objset,
6500 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ,
6501 	    sizeof (uint64_t), 1, &obj, tx) != 0) {
6502 		cmn_err(CE_PANIC, "failed to add bpobj");
6503 	}
6504 	VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
6505 	    spa->spa_meta_objset, obj));
6506 
6507 	/*
6508 	 * Generate some random noise for salted checksums to operate on.
6509 	 */
6510 	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
6511 	    sizeof (spa->spa_cksum_salt.zcs_bytes));
6512 
6513 	/*
6514 	 * Set pool properties.
6515 	 */
6516 	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
6517 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
6518 	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
6519 	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
6520 	spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST);
6521 	spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM);
6522 
6523 	if (props != NULL) {
6524 		spa_configfile_set(spa, props, B_FALSE);
6525 		spa_sync_props(props, tx);
6526 	}
6527 
6528 	for (int i = 0; i < ndraid; i++)
6529 		spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
6530 
6531 	dmu_tx_commit(tx);
6532 
6533 	spa->spa_sync_on = B_TRUE;
6534 	txg_sync_start(dp);
6535 	mmp_thread_start(spa);
6536 	txg_wait_synced(dp, txg);
6537 
6538 	spa_spawn_aux_threads(spa);
6539 
6540 	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
6541 
6542 	/*
6543 	 * Don't count references from objsets that are already closed
6544 	 * and are making their way through the eviction process.
6545 	 */
6546 	spa_evicting_os_wait(spa);
6547 	spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
6548 	spa->spa_load_state = SPA_LOAD_NONE;
6549 
6550 	spa_import_os(spa);
6551 
6552 	mutex_exit(&spa_namespace_lock);
6553 
6554 	return (0);
6555 }
6556 
6557 /*
6558  * Import a non-root pool into the system.
6559  */
6560 int
6561 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
6562 {
6563 	spa_t *spa;
6564 	const char *altroot = NULL;
6565 	spa_load_state_t state = SPA_LOAD_IMPORT;
6566 	zpool_load_policy_t policy;
6567 	spa_mode_t mode = spa_mode_global;
6568 	uint64_t readonly = B_FALSE;
6569 	int error;
6570 	nvlist_t *nvroot;
6571 	nvlist_t **spares, **l2cache;
6572 	uint_t nspares, nl2cache;
6573 
6574 	/*
6575 	 * If a pool with this name exists, return failure.
6576 	 */
6577 	mutex_enter(&spa_namespace_lock);
6578 	if (spa_lookup(pool) != NULL) {
6579 		mutex_exit(&spa_namespace_lock);
6580 		return (SET_ERROR(EEXIST));
6581 	}
6582 
6583 	/*
6584 	 * Create and initialize the spa structure.
6585 	 */
6586 	(void) nvlist_lookup_string(props,
6587 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
6588 	(void) nvlist_lookup_uint64(props,
6589 	    zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
6590 	if (readonly)
6591 		mode = SPA_MODE_READ;
6592 	spa = spa_add(pool, config, altroot);
6593 	spa->spa_import_flags = flags;
6594 
6595 	/*
6596 	 * Verbatim import - Take a pool and insert it into the namespace
6597 	 * as if it had been loaded at boot.
6598 	 */
6599 	if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
6600 		if (props != NULL)
6601 			spa_configfile_set(spa, props, B_FALSE);
6602 
6603 		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
6604 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
6605 		zfs_dbgmsg("spa_import: verbatim import of %s", pool);
6606 		mutex_exit(&spa_namespace_lock);
6607 		return (0);
6608 	}
6609 
6610 	spa_activate(spa, mode);
6611 
6612 	/*
6613 	 * Don't start async tasks until we know everything is healthy.
6614 	 */
6615 	spa_async_suspend(spa);
6616 
6617 	zpool_get_load_policy(config, &policy);
6618 	if (policy.zlp_rewind & ZPOOL_DO_REWIND)
6619 		state = SPA_LOAD_RECOVER;
6620 
6621 	spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT;
6622 
6623 	if (state != SPA_LOAD_RECOVER) {
6624 		spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
6625 		zfs_dbgmsg("spa_import: importing %s", pool);
6626 	} else {
6627 		zfs_dbgmsg("spa_import: importing %s, max_txg=%lld "
6628 		    "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg);
6629 	}
6630 	error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind);
6631 
6632 	/*
6633 	 * Propagate anything learned while loading the pool and pass it
6634 	 * back to caller (i.e. rewind info, missing devices, etc).
6635 	 */
6636 	fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info);
6637 
6638 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6639 	/*
6640 	 * Toss any existing sparelist, as it doesn't have any validity
6641 	 * anymore, and conflicts with spa_has_spare().
6642 	 */
6643 	if (spa->spa_spares.sav_config) {
6644 		nvlist_free(spa->spa_spares.sav_config);
6645 		spa->spa_spares.sav_config = NULL;
6646 		spa_load_spares(spa);
6647 	}
6648 	if (spa->spa_l2cache.sav_config) {
6649 		nvlist_free(spa->spa_l2cache.sav_config);
6650 		spa->spa_l2cache.sav_config = NULL;
6651 		spa_load_l2cache(spa);
6652 	}
6653 
6654 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6655 	spa_config_exit(spa, SCL_ALL, FTAG);
6656 
6657 	if (props != NULL)
6658 		spa_configfile_set(spa, props, B_FALSE);
6659 
6660 	if (error != 0 || (props && spa_writeable(spa) &&
6661 	    (error = spa_prop_set(spa, props)))) {
6662 		spa_unload(spa);
6663 		spa_deactivate(spa);
6664 		spa_remove(spa);
6665 		mutex_exit(&spa_namespace_lock);
6666 		return (error);
6667 	}
6668 
6669 	spa_async_resume(spa);
6670 
6671 	/*
6672 	 * Override any spares and level 2 cache devices as specified by
6673 	 * the user, as these may have correct device names/devids, etc.
6674 	 */
6675 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6676 	    &spares, &nspares) == 0) {
6677 		if (spa->spa_spares.sav_config)
6678 			fnvlist_remove(spa->spa_spares.sav_config,
6679 			    ZPOOL_CONFIG_SPARES);
6680 		else
6681 			spa->spa_spares.sav_config = fnvlist_alloc();
6682 		fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
6683 		    ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares,
6684 		    nspares);
6685 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6686 		spa_load_spares(spa);
6687 		spa_config_exit(spa, SCL_ALL, FTAG);
6688 		spa->spa_spares.sav_sync = B_TRUE;
6689 	}
6690 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6691 	    &l2cache, &nl2cache) == 0) {
6692 		if (spa->spa_l2cache.sav_config)
6693 			fnvlist_remove(spa->spa_l2cache.sav_config,
6694 			    ZPOOL_CONFIG_L2CACHE);
6695 		else
6696 			spa->spa_l2cache.sav_config = fnvlist_alloc();
6697 		fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
6698 		    ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache,
6699 		    nl2cache);
6700 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6701 		spa_load_l2cache(spa);
6702 		spa_config_exit(spa, SCL_ALL, FTAG);
6703 		spa->spa_l2cache.sav_sync = B_TRUE;
6704 	}
6705 
6706 	/*
6707 	 * Check for any removed devices.
6708 	 */
6709 	if (spa->spa_autoreplace) {
6710 		spa_aux_check_removed(&spa->spa_spares);
6711 		spa_aux_check_removed(&spa->spa_l2cache);
6712 	}
6713 
6714 	if (spa_writeable(spa)) {
6715 		/*
6716 		 * Update the config cache to include the newly-imported pool.
6717 		 */
6718 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
6719 	}
6720 
6721 	/*
6722 	 * It's possible that the pool was expanded while it was exported.
6723 	 * We kick off an async task to handle this for us.
6724 	 */
6725 	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
6726 
6727 	spa_history_log_version(spa, "import", NULL);
6728 
6729 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
6730 
6731 	mutex_exit(&spa_namespace_lock);
6732 
6733 	zvol_create_minors_recursive(pool);
6734 
6735 	spa_import_os(spa);
6736 
6737 	return (0);
6738 }
6739 
6740 nvlist_t *
6741 spa_tryimport(nvlist_t *tryconfig)
6742 {
6743 	nvlist_t *config = NULL;
6744 	const char *poolname, *cachefile;
6745 	spa_t *spa;
6746 	uint64_t state;
6747 	int error;
6748 	zpool_load_policy_t policy;
6749 
6750 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
6751 		return (NULL);
6752 
6753 	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
6754 		return (NULL);
6755 
6756 	/*
6757 	 * Create and initialize the spa structure.
6758 	 */
6759 	mutex_enter(&spa_namespace_lock);
6760 	spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL);
6761 	spa_activate(spa, SPA_MODE_READ);
6762 
6763 	/*
6764 	 * Rewind pool if a max txg was provided.
6765 	 */
6766 	zpool_get_load_policy(spa->spa_config, &policy);
6767 	if (policy.zlp_txg != UINT64_MAX) {
6768 		spa->spa_load_max_txg = policy.zlp_txg;
6769 		spa->spa_extreme_rewind = B_TRUE;
6770 		zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld",
6771 		    poolname, (longlong_t)policy.zlp_txg);
6772 	} else {
6773 		zfs_dbgmsg("spa_tryimport: importing %s", poolname);
6774 	}
6775 
6776 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile)
6777 	    == 0) {
6778 		zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile);
6779 		spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
6780 	} else {
6781 		spa->spa_config_source = SPA_CONFIG_SRC_SCAN;
6782 	}
6783 
6784 	/*
6785 	 * spa_import() relies on a pool config fetched by spa_try_import()
6786 	 * for spare/cache devices. Import flags are not passed to
6787 	 * spa_tryimport(), which makes it return early due to a missing log
6788 	 * device and missing retrieving the cache device and spare eventually.
6789 	 * Passing ZFS_IMPORT_MISSING_LOG to spa_tryimport() makes it fetch
6790 	 * the correct configuration regardless of the missing log device.
6791 	 */
6792 	spa->spa_import_flags |= ZFS_IMPORT_MISSING_LOG;
6793 
6794 	error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING);
6795 
6796 	/*
6797 	 * If 'tryconfig' was at least parsable, return the current config.
6798 	 */
6799 	if (spa->spa_root_vdev != NULL) {
6800 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
6801 		fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname);
6802 		fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state);
6803 		fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
6804 		    spa->spa_uberblock.ub_timestamp);
6805 		fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
6806 		    spa->spa_load_info);
6807 		fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA,
6808 		    spa->spa_errata);
6809 
6810 		/*
6811 		 * If the bootfs property exists on this pool then we
6812 		 * copy it out so that external consumers can tell which
6813 		 * pools are bootable.
6814 		 */
6815 		if ((!error || error == EEXIST) && spa->spa_bootfs) {
6816 			char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
6817 
6818 			/*
6819 			 * We have to play games with the name since the
6820 			 * pool was opened as TRYIMPORT_NAME.
6821 			 */
6822 			if (dsl_dsobj_to_dsname(spa_name(spa),
6823 			    spa->spa_bootfs, tmpname) == 0) {
6824 				char *cp;
6825 				char *dsname;
6826 
6827 				dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
6828 
6829 				cp = strchr(tmpname, '/');
6830 				if (cp == NULL) {
6831 					(void) strlcpy(dsname, tmpname,
6832 					    MAXPATHLEN);
6833 				} else {
6834 					(void) snprintf(dsname, MAXPATHLEN,
6835 					    "%s/%s", poolname, ++cp);
6836 				}
6837 				fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS,
6838 				    dsname);
6839 				kmem_free(dsname, MAXPATHLEN);
6840 			}
6841 			kmem_free(tmpname, MAXPATHLEN);
6842 		}
6843 
6844 		/*
6845 		 * Add the list of hot spares and level 2 cache devices.
6846 		 */
6847 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
6848 		spa_add_spares(spa, config);
6849 		spa_add_l2cache(spa, config);
6850 		spa_config_exit(spa, SCL_CONFIG, FTAG);
6851 	}
6852 
6853 	spa_unload(spa);
6854 	spa_deactivate(spa);
6855 	spa_remove(spa);
6856 	mutex_exit(&spa_namespace_lock);
6857 
6858 	return (config);
6859 }
6860 
6861 /*
6862  * Pool export/destroy
6863  *
6864  * The act of destroying or exporting a pool is very simple.  We make sure there
6865  * is no more pending I/O and any references to the pool are gone.  Then, we
6866  * update the pool state and sync all the labels to disk, removing the
6867  * configuration from the cache afterwards. If the 'hardforce' flag is set, then
6868  * we don't sync the labels or remove the configuration cache.
6869  */
6870 static int
6871 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig,
6872     boolean_t force, boolean_t hardforce)
6873 {
6874 	int error;
6875 	spa_t *spa;
6876 
6877 	if (oldconfig)
6878 		*oldconfig = NULL;
6879 
6880 	if (!(spa_mode_global & SPA_MODE_WRITE))
6881 		return (SET_ERROR(EROFS));
6882 
6883 	mutex_enter(&spa_namespace_lock);
6884 	if ((spa = spa_lookup(pool)) == NULL) {
6885 		mutex_exit(&spa_namespace_lock);
6886 		return (SET_ERROR(ENOENT));
6887 	}
6888 
6889 	if (spa->spa_is_exporting) {
6890 		/* the pool is being exported by another thread */
6891 		mutex_exit(&spa_namespace_lock);
6892 		return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS));
6893 	}
6894 	spa->spa_is_exporting = B_TRUE;
6895 
6896 	/*
6897 	 * Put a hold on the pool, drop the namespace lock, stop async tasks,
6898 	 * reacquire the namespace lock, and see if we can export.
6899 	 */
6900 	spa_open_ref(spa, FTAG);
6901 	mutex_exit(&spa_namespace_lock);
6902 	spa_async_suspend(spa);
6903 	if (spa->spa_zvol_taskq) {
6904 		zvol_remove_minors(spa, spa_name(spa), B_TRUE);
6905 		taskq_wait(spa->spa_zvol_taskq);
6906 	}
6907 	mutex_enter(&spa_namespace_lock);
6908 	spa_close(spa, FTAG);
6909 
6910 	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
6911 		goto export_spa;
6912 	/*
6913 	 * The pool will be in core if it's openable, in which case we can
6914 	 * modify its state.  Objsets may be open only because they're dirty,
6915 	 * so we have to force it to sync before checking spa_refcnt.
6916 	 */
6917 	if (spa->spa_sync_on) {
6918 		txg_wait_synced(spa->spa_dsl_pool, 0);
6919 		spa_evicting_os_wait(spa);
6920 	}
6921 
6922 	/*
6923 	 * A pool cannot be exported or destroyed if there are active
6924 	 * references.  If we are resetting a pool, allow references by
6925 	 * fault injection handlers.
6926 	 */
6927 	if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) {
6928 		error = SET_ERROR(EBUSY);
6929 		goto fail;
6930 	}
6931 
6932 	if (spa->spa_sync_on) {
6933 		vdev_t *rvd = spa->spa_root_vdev;
6934 		/*
6935 		 * A pool cannot be exported if it has an active shared spare.
6936 		 * This is to prevent other pools stealing the active spare
6937 		 * from an exported pool. At user's own will, such pool can
6938 		 * be forcedly exported.
6939 		 */
6940 		if (!force && new_state == POOL_STATE_EXPORTED &&
6941 		    spa_has_active_shared_spare(spa)) {
6942 			error = SET_ERROR(EXDEV);
6943 			goto fail;
6944 		}
6945 
6946 		/*
6947 		 * We're about to export or destroy this pool. Make sure
6948 		 * we stop all initialization and trim activity here before
6949 		 * we set the spa_final_txg. This will ensure that all
6950 		 * dirty data resulting from the initialization is
6951 		 * committed to disk before we unload the pool.
6952 		 */
6953 		vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE);
6954 		vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE);
6955 		vdev_autotrim_stop_all(spa);
6956 		vdev_rebuild_stop_all(spa);
6957 
6958 		/*
6959 		 * We want this to be reflected on every label,
6960 		 * so mark them all dirty.  spa_unload() will do the
6961 		 * final sync that pushes these changes out.
6962 		 */
6963 		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
6964 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6965 			spa->spa_state = new_state;
6966 			vdev_config_dirty(rvd);
6967 			spa_config_exit(spa, SCL_ALL, FTAG);
6968 		}
6969 
6970 		/*
6971 		 * If the log space map feature is enabled and the pool is
6972 		 * getting exported (but not destroyed), we want to spend some
6973 		 * time flushing as many metaslabs as we can in an attempt to
6974 		 * destroy log space maps and save import time. This has to be
6975 		 * done before we set the spa_final_txg, otherwise
6976 		 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs.
6977 		 * spa_should_flush_logs_on_unload() should be called after
6978 		 * spa_state has been set to the new_state.
6979 		 */
6980 		if (spa_should_flush_logs_on_unload(spa))
6981 			spa_unload_log_sm_flush_all(spa);
6982 
6983 		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
6984 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6985 			spa->spa_final_txg = spa_last_synced_txg(spa) +
6986 			    TXG_DEFER_SIZE + 1;
6987 			spa_config_exit(spa, SCL_ALL, FTAG);
6988 		}
6989 	}
6990 
6991 export_spa:
6992 	spa_export_os(spa);
6993 
6994 	if (new_state == POOL_STATE_DESTROYED)
6995 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY);
6996 	else if (new_state == POOL_STATE_EXPORTED)
6997 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT);
6998 
6999 	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
7000 		spa_unload(spa);
7001 		spa_deactivate(spa);
7002 	}
7003 
7004 	if (oldconfig && spa->spa_config)
7005 		*oldconfig = fnvlist_dup(spa->spa_config);
7006 
7007 	if (new_state != POOL_STATE_UNINITIALIZED) {
7008 		if (!hardforce)
7009 			spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
7010 		spa_remove(spa);
7011 	} else {
7012 		/*
7013 		 * If spa_remove() is not called for this spa_t and
7014 		 * there is any possibility that it can be reused,
7015 		 * we make sure to reset the exporting flag.
7016 		 */
7017 		spa->spa_is_exporting = B_FALSE;
7018 	}
7019 
7020 	mutex_exit(&spa_namespace_lock);
7021 	return (0);
7022 
7023 fail:
7024 	spa->spa_is_exporting = B_FALSE;
7025 	spa_async_resume(spa);
7026 	mutex_exit(&spa_namespace_lock);
7027 	return (error);
7028 }
7029 
7030 /*
7031  * Destroy a storage pool.
7032  */
7033 int
7034 spa_destroy(const char *pool)
7035 {
7036 	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL,
7037 	    B_FALSE, B_FALSE));
7038 }
7039 
7040 /*
7041  * Export a storage pool.
7042  */
7043 int
7044 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force,
7045     boolean_t hardforce)
7046 {
7047 	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig,
7048 	    force, hardforce));
7049 }
7050 
7051 /*
7052  * Similar to spa_export(), this unloads the spa_t without actually removing it
7053  * from the namespace in any way.
7054  */
7055 int
7056 spa_reset(const char *pool)
7057 {
7058 	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL,
7059 	    B_FALSE, B_FALSE));
7060 }
7061 
7062 /*
7063  * ==========================================================================
7064  * Device manipulation
7065  * ==========================================================================
7066  */
7067 
7068 /*
7069  * This is called as a synctask to increment the draid feature flag
7070  */
7071 static void
7072 spa_draid_feature_incr(void *arg, dmu_tx_t *tx)
7073 {
7074 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
7075 	int draid = (int)(uintptr_t)arg;
7076 
7077 	for (int c = 0; c < draid; c++)
7078 		spa_feature_incr(spa, SPA_FEATURE_DRAID, tx);
7079 }
7080 
7081 /*
7082  * Add a device to a storage pool.
7083  */
7084 int
7085 spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
7086 {
7087 	uint64_t txg, ndraid = 0;
7088 	int error;
7089 	vdev_t *rvd = spa->spa_root_vdev;
7090 	vdev_t *vd, *tvd;
7091 	nvlist_t **spares, **l2cache;
7092 	uint_t nspares, nl2cache;
7093 
7094 	ASSERT(spa_writeable(spa));
7095 
7096 	txg = spa_vdev_enter(spa);
7097 
7098 	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
7099 	    VDEV_ALLOC_ADD)) != 0)
7100 		return (spa_vdev_exit(spa, NULL, txg, error));
7101 
7102 	spa->spa_pending_vdev = vd;	/* spa_vdev_exit() will clear this */
7103 
7104 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
7105 	    &nspares) != 0)
7106 		nspares = 0;
7107 
7108 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache,
7109 	    &nl2cache) != 0)
7110 		nl2cache = 0;
7111 
7112 	if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
7113 		return (spa_vdev_exit(spa, vd, txg, EINVAL));
7114 
7115 	if (vd->vdev_children != 0 &&
7116 	    (error = vdev_create(vd, txg, B_FALSE)) != 0) {
7117 		return (spa_vdev_exit(spa, vd, txg, error));
7118 	}
7119 
7120 	/*
7121 	 * The virtual dRAID spares must be added after vdev tree is created
7122 	 * and the vdev guids are generated.  The guid of their associated
7123 	 * dRAID is stored in the config and used when opening the spare.
7124 	 */
7125 	if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid,
7126 	    rvd->vdev_children)) == 0) {
7127 		if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot,
7128 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0)
7129 			nspares = 0;
7130 	} else {
7131 		return (spa_vdev_exit(spa, vd, txg, error));
7132 	}
7133 
7134 	/*
7135 	 * We must validate the spares and l2cache devices after checking the
7136 	 * children.  Otherwise, vdev_inuse() will blindly overwrite the spare.
7137 	 */
7138 	if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0)
7139 		return (spa_vdev_exit(spa, vd, txg, error));
7140 
7141 	/*
7142 	 * If we are in the middle of a device removal, we can only add
7143 	 * devices which match the existing devices in the pool.
7144 	 * If we are in the middle of a removal, or have some indirect
7145 	 * vdevs, we can not add raidz or dRAID top levels.
7146 	 */
7147 	if (spa->spa_vdev_removal != NULL ||
7148 	    spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
7149 		for (int c = 0; c < vd->vdev_children; c++) {
7150 			tvd = vd->vdev_child[c];
7151 			if (spa->spa_vdev_removal != NULL &&
7152 			    tvd->vdev_ashift != spa->spa_max_ashift) {
7153 				return (spa_vdev_exit(spa, vd, txg, EINVAL));
7154 			}
7155 			/* Fail if top level vdev is raidz or a dRAID */
7156 			if (vdev_get_nparity(tvd) != 0)
7157 				return (spa_vdev_exit(spa, vd, txg, EINVAL));
7158 
7159 			/*
7160 			 * Need the top level mirror to be
7161 			 * a mirror of leaf vdevs only
7162 			 */
7163 			if (tvd->vdev_ops == &vdev_mirror_ops) {
7164 				for (uint64_t cid = 0;
7165 				    cid < tvd->vdev_children; cid++) {
7166 					vdev_t *cvd = tvd->vdev_child[cid];
7167 					if (!cvd->vdev_ops->vdev_op_leaf) {
7168 						return (spa_vdev_exit(spa, vd,
7169 						    txg, EINVAL));
7170 					}
7171 				}
7172 			}
7173 		}
7174 	}
7175 
7176 	for (int c = 0; c < vd->vdev_children; c++) {
7177 		tvd = vd->vdev_child[c];
7178 		vdev_remove_child(vd, tvd);
7179 		tvd->vdev_id = rvd->vdev_children;
7180 		vdev_add_child(rvd, tvd);
7181 		vdev_config_dirty(tvd);
7182 	}
7183 
7184 	if (nspares != 0) {
7185 		spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
7186 		    ZPOOL_CONFIG_SPARES);
7187 		spa_load_spares(spa);
7188 		spa->spa_spares.sav_sync = B_TRUE;
7189 	}
7190 
7191 	if (nl2cache != 0) {
7192 		spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
7193 		    ZPOOL_CONFIG_L2CACHE);
7194 		spa_load_l2cache(spa);
7195 		spa->spa_l2cache.sav_sync = B_TRUE;
7196 	}
7197 
7198 	/*
7199 	 * We can't increment a feature while holding spa_vdev so we
7200 	 * have to do it in a synctask.
7201 	 */
7202 	if (ndraid != 0) {
7203 		dmu_tx_t *tx;
7204 
7205 		tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
7206 		dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr,
7207 		    (void *)(uintptr_t)ndraid, tx);
7208 		dmu_tx_commit(tx);
7209 	}
7210 
7211 	/*
7212 	 * We have to be careful when adding new vdevs to an existing pool.
7213 	 * If other threads start allocating from these vdevs before we
7214 	 * sync the config cache, and we lose power, then upon reboot we may
7215 	 * fail to open the pool because there are DVAs that the config cache
7216 	 * can't translate.  Therefore, we first add the vdevs without
7217 	 * initializing metaslabs; sync the config cache (via spa_vdev_exit());
7218 	 * and then let spa_config_update() initialize the new metaslabs.
7219 	 *
7220 	 * spa_load() checks for added-but-not-initialized vdevs, so that
7221 	 * if we lose power at any point in this sequence, the remaining
7222 	 * steps will be completed the next time we load the pool.
7223 	 */
7224 	(void) spa_vdev_exit(spa, vd, txg, 0);
7225 
7226 	mutex_enter(&spa_namespace_lock);
7227 	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
7228 	spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD);
7229 	mutex_exit(&spa_namespace_lock);
7230 
7231 	return (0);
7232 }
7233 
7234 /*
7235  * Attach a device to a vdev specified by its guid.  The vdev type can be
7236  * a mirror, a raidz, or a leaf device that is also a top-level (e.g. a
7237  * single device). When the vdev is a single device, a mirror vdev will be
7238  * automatically inserted.
7239  *
7240  * If 'replacing' is specified, the new device is intended to replace the
7241  * existing device; in this case the two devices are made into their own
7242  * mirror using the 'replacing' vdev, which is functionally identical to
7243  * the mirror vdev (it actually reuses all the same ops) but has a few
7244  * extra rules: you can't attach to it after it's been created, and upon
7245  * completion of resilvering, the first disk (the one being replaced)
7246  * is automatically detached.
7247  *
7248  * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild)
7249  * should be performed instead of traditional healing reconstruction.  From
7250  * an administrators perspective these are both resilver operations.
7251  */
7252 int
7253 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing,
7254     int rebuild)
7255 {
7256 	uint64_t txg, dtl_max_txg;
7257 	vdev_t *rvd = spa->spa_root_vdev;
7258 	vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd;
7259 	vdev_ops_t *pvops;
7260 	char *oldvdpath, *newvdpath;
7261 	int newvd_isspare = B_FALSE;
7262 	int error;
7263 
7264 	ASSERT(spa_writeable(spa));
7265 
7266 	txg = spa_vdev_enter(spa);
7267 
7268 	oldvd = spa_lookup_by_guid(spa, guid, B_FALSE);
7269 
7270 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7271 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
7272 		error = (spa_has_checkpoint(spa)) ?
7273 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
7274 		return (spa_vdev_exit(spa, NULL, txg, error));
7275 	}
7276 
7277 	if (rebuild) {
7278 		if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD))
7279 			return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7280 
7281 		if (dsl_scan_resilvering(spa_get_dsl(spa)) ||
7282 		    dsl_scan_resilver_scheduled(spa_get_dsl(spa))) {
7283 			return (spa_vdev_exit(spa, NULL, txg,
7284 			    ZFS_ERR_RESILVER_IN_PROGRESS));
7285 		}
7286 	} else {
7287 		if (vdev_rebuild_active(rvd))
7288 			return (spa_vdev_exit(spa, NULL, txg,
7289 			    ZFS_ERR_REBUILD_IN_PROGRESS));
7290 	}
7291 
7292 	if (spa->spa_vdev_removal != NULL) {
7293 		return (spa_vdev_exit(spa, NULL, txg,
7294 		    ZFS_ERR_DEVRM_IN_PROGRESS));
7295 	}
7296 
7297 	if (oldvd == NULL)
7298 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
7299 
7300 	boolean_t raidz = oldvd->vdev_ops == &vdev_raidz_ops;
7301 
7302 	if (raidz) {
7303 		if (!spa_feature_is_enabled(spa, SPA_FEATURE_RAIDZ_EXPANSION))
7304 			return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7305 
7306 		/*
7307 		 * Can't expand a raidz while prior expand is in progress.
7308 		 */
7309 		if (spa->spa_raidz_expand != NULL) {
7310 			return (spa_vdev_exit(spa, NULL, txg,
7311 			    ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS));
7312 		}
7313 	} else if (!oldvd->vdev_ops->vdev_op_leaf) {
7314 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7315 	}
7316 
7317 	if (raidz)
7318 		pvd = oldvd;
7319 	else
7320 		pvd = oldvd->vdev_parent;
7321 
7322 	if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
7323 	    VDEV_ALLOC_ATTACH) != 0)
7324 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
7325 
7326 	if (newrootvd->vdev_children != 1)
7327 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
7328 
7329 	newvd = newrootvd->vdev_child[0];
7330 
7331 	if (!newvd->vdev_ops->vdev_op_leaf)
7332 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
7333 
7334 	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
7335 		return (spa_vdev_exit(spa, newrootvd, txg, error));
7336 
7337 	/*
7338 	 * log, dedup and special vdevs should not be replaced by spares.
7339 	 */
7340 	if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE ||
7341 	    oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) {
7342 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7343 	}
7344 
7345 	/*
7346 	 * A dRAID spare can only replace a child of its parent dRAID vdev.
7347 	 */
7348 	if (newvd->vdev_ops == &vdev_draid_spare_ops &&
7349 	    oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) {
7350 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7351 	}
7352 
7353 	if (rebuild) {
7354 		/*
7355 		 * For rebuilds, the top vdev must support reconstruction
7356 		 * using only space maps.  This means the only allowable
7357 		 * vdevs types are the root vdev, a mirror, or dRAID.
7358 		 */
7359 		tvd = pvd;
7360 		if (pvd->vdev_top != NULL)
7361 			tvd = pvd->vdev_top;
7362 
7363 		if (tvd->vdev_ops != &vdev_mirror_ops &&
7364 		    tvd->vdev_ops != &vdev_root_ops &&
7365 		    tvd->vdev_ops != &vdev_draid_ops) {
7366 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7367 		}
7368 	}
7369 
7370 	if (!replacing) {
7371 		/*
7372 		 * For attach, the only allowable parent is a mirror or
7373 		 * the root vdev. A raidz vdev can be attached to, but
7374 		 * you cannot attach to a raidz child.
7375 		 */
7376 		if (pvd->vdev_ops != &vdev_mirror_ops &&
7377 		    pvd->vdev_ops != &vdev_root_ops &&
7378 		    !raidz)
7379 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7380 
7381 		pvops = &vdev_mirror_ops;
7382 	} else {
7383 		/*
7384 		 * Active hot spares can only be replaced by inactive hot
7385 		 * spares.
7386 		 */
7387 		if (pvd->vdev_ops == &vdev_spare_ops &&
7388 		    oldvd->vdev_isspare &&
7389 		    !spa_has_spare(spa, newvd->vdev_guid))
7390 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7391 
7392 		/*
7393 		 * If the source is a hot spare, and the parent isn't already a
7394 		 * spare, then we want to create a new hot spare.  Otherwise, we
7395 		 * want to create a replacing vdev.  The user is not allowed to
7396 		 * attach to a spared vdev child unless the 'isspare' state is
7397 		 * the same (spare replaces spare, non-spare replaces
7398 		 * non-spare).
7399 		 */
7400 		if (pvd->vdev_ops == &vdev_replacing_ops &&
7401 		    spa_version(spa) < SPA_VERSION_MULTI_REPLACE) {
7402 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7403 		} else if (pvd->vdev_ops == &vdev_spare_ops &&
7404 		    newvd->vdev_isspare != oldvd->vdev_isspare) {
7405 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7406 		}
7407 
7408 		if (newvd->vdev_isspare)
7409 			pvops = &vdev_spare_ops;
7410 		else
7411 			pvops = &vdev_replacing_ops;
7412 	}
7413 
7414 	/*
7415 	 * Make sure the new device is big enough.
7416 	 */
7417 	vdev_t *min_vdev = raidz ? oldvd->vdev_child[0] : oldvd;
7418 	if (newvd->vdev_asize < vdev_get_min_asize(min_vdev))
7419 		return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));
7420 
7421 	/*
7422 	 * The new device cannot have a higher alignment requirement
7423 	 * than the top-level vdev.
7424 	 */
7425 	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
7426 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7427 
7428 	/*
7429 	 * RAIDZ-expansion-specific checks.
7430 	 */
7431 	if (raidz) {
7432 		if (vdev_raidz_attach_check(newvd) != 0)
7433 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
7434 
7435 		/*
7436 		 * Fail early if a child is not healthy or being replaced
7437 		 */
7438 		for (int i = 0; i < oldvd->vdev_children; i++) {
7439 			if (vdev_is_dead(oldvd->vdev_child[i]) ||
7440 			    !oldvd->vdev_child[i]->vdev_ops->vdev_op_leaf) {
7441 				return (spa_vdev_exit(spa, newrootvd, txg,
7442 				    ENXIO));
7443 			}
7444 			/* Also fail if reserved boot area is in-use */
7445 			if (vdev_check_boot_reserve(spa, oldvd->vdev_child[i])
7446 			    != 0) {
7447 				return (spa_vdev_exit(spa, newrootvd, txg,
7448 				    EADDRINUSE));
7449 			}
7450 		}
7451 	}
7452 
7453 	if (raidz) {
7454 		/*
7455 		 * Note: oldvdpath is freed by spa_strfree(),  but
7456 		 * kmem_asprintf() is freed by kmem_strfree(), so we have to
7457 		 * move it to a spa_strdup-ed string.
7458 		 */
7459 		char *tmp = kmem_asprintf("raidz%u-%u",
7460 		    (uint_t)vdev_get_nparity(oldvd), (uint_t)oldvd->vdev_id);
7461 		oldvdpath = spa_strdup(tmp);
7462 		kmem_strfree(tmp);
7463 	} else {
7464 		oldvdpath = spa_strdup(oldvd->vdev_path);
7465 	}
7466 	newvdpath = spa_strdup(newvd->vdev_path);
7467 
7468 	/*
7469 	 * If this is an in-place replacement, update oldvd's path and devid
7470 	 * to make it distinguishable from newvd, and unopenable from now on.
7471 	 */
7472 	if (strcmp(oldvdpath, newvdpath) == 0) {
7473 		spa_strfree(oldvd->vdev_path);
7474 		oldvd->vdev_path = kmem_alloc(strlen(newvdpath) + 5,
7475 		    KM_SLEEP);
7476 		(void) sprintf(oldvd->vdev_path, "%s/old",
7477 		    newvdpath);
7478 		if (oldvd->vdev_devid != NULL) {
7479 			spa_strfree(oldvd->vdev_devid);
7480 			oldvd->vdev_devid = NULL;
7481 		}
7482 		spa_strfree(oldvdpath);
7483 		oldvdpath = spa_strdup(oldvd->vdev_path);
7484 	}
7485 
7486 	/*
7487 	 * If the parent is not a mirror, or if we're replacing, insert the new
7488 	 * mirror/replacing/spare vdev above oldvd.
7489 	 */
7490 	if (!raidz && pvd->vdev_ops != pvops) {
7491 		pvd = vdev_add_parent(oldvd, pvops);
7492 		ASSERT(pvd->vdev_ops == pvops);
7493 		ASSERT(oldvd->vdev_parent == pvd);
7494 	}
7495 
7496 	ASSERT(pvd->vdev_top->vdev_parent == rvd);
7497 
7498 	/*
7499 	 * Extract the new device from its root and add it to pvd.
7500 	 */
7501 	vdev_remove_child(newrootvd, newvd);
7502 	newvd->vdev_id = pvd->vdev_children;
7503 	newvd->vdev_crtxg = oldvd->vdev_crtxg;
7504 	vdev_add_child(pvd, newvd);
7505 
7506 	/*
7507 	 * Reevaluate the parent vdev state.
7508 	 */
7509 	vdev_propagate_state(pvd);
7510 
7511 	tvd = newvd->vdev_top;
7512 	ASSERT(pvd->vdev_top == tvd);
7513 	ASSERT(tvd->vdev_parent == rvd);
7514 
7515 	vdev_config_dirty(tvd);
7516 
7517 	/*
7518 	 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account
7519 	 * for any dmu_sync-ed blocks.  It will propagate upward when
7520 	 * spa_vdev_exit() calls vdev_dtl_reassess().
7521 	 */
7522 	dtl_max_txg = txg + TXG_CONCURRENT_STATES;
7523 
7524 	if (raidz) {
7525 		/*
7526 		 * Wait for the youngest allocations and frees to sync,
7527 		 * and then wait for the deferral of those frees to finish.
7528 		 */
7529 		spa_vdev_config_exit(spa, NULL,
7530 		    txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG);
7531 
7532 		vdev_initialize_stop_all(tvd, VDEV_INITIALIZE_ACTIVE);
7533 		vdev_trim_stop_all(tvd, VDEV_TRIM_ACTIVE);
7534 		vdev_autotrim_stop_wait(tvd);
7535 
7536 		dtl_max_txg = spa_vdev_config_enter(spa);
7537 
7538 		tvd->vdev_rz_expanding = B_TRUE;
7539 
7540 		vdev_dirty_leaves(tvd, VDD_DTL, dtl_max_txg);
7541 		vdev_config_dirty(tvd);
7542 
7543 		dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool,
7544 		    dtl_max_txg);
7545 		dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_raidz_attach_sync,
7546 		    newvd, tx);
7547 		dmu_tx_commit(tx);
7548 	} else {
7549 		vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL,
7550 		    dtl_max_txg - TXG_INITIAL);
7551 
7552 		if (newvd->vdev_isspare) {
7553 			spa_spare_activate(newvd);
7554 			spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE);
7555 		}
7556 
7557 		newvd_isspare = newvd->vdev_isspare;
7558 
7559 		/*
7560 		 * Mark newvd's DTL dirty in this txg.
7561 		 */
7562 		vdev_dirty(tvd, VDD_DTL, newvd, txg);
7563 
7564 		/*
7565 		 * Schedule the resilver or rebuild to restart in the future.
7566 		 * We do this to ensure that dmu_sync-ed blocks have been
7567 		 * stitched into the respective datasets.
7568 		 */
7569 		if (rebuild) {
7570 			newvd->vdev_rebuild_txg = txg;
7571 
7572 			vdev_rebuild(tvd);
7573 		} else {
7574 			newvd->vdev_resilver_txg = txg;
7575 
7576 			if (dsl_scan_resilvering(spa_get_dsl(spa)) &&
7577 			    spa_feature_is_enabled(spa,
7578 			    SPA_FEATURE_RESILVER_DEFER)) {
7579 				vdev_defer_resilver(newvd);
7580 			} else {
7581 				dsl_scan_restart_resilver(spa->spa_dsl_pool,
7582 				    dtl_max_txg);
7583 			}
7584 		}
7585 	}
7586 
7587 	if (spa->spa_bootfs)
7588 		spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH);
7589 
7590 	spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH);
7591 
7592 	/*
7593 	 * Commit the config
7594 	 */
7595 	(void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0);
7596 
7597 	spa_history_log_internal(spa, "vdev attach", NULL,
7598 	    "%s vdev=%s %s vdev=%s",
7599 	    replacing && newvd_isspare ? "spare in" :
7600 	    replacing ? "replace" : "attach", newvdpath,
7601 	    replacing ? "for" : "to", oldvdpath);
7602 
7603 	spa_strfree(oldvdpath);
7604 	spa_strfree(newvdpath);
7605 
7606 	return (0);
7607 }
7608 
7609 /*
7610  * Detach a device from a mirror or replacing vdev.
7611  *
7612  * If 'replace_done' is specified, only detach if the parent
7613  * is a replacing or a spare vdev.
7614  */
7615 int
7616 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)
7617 {
7618 	uint64_t txg;
7619 	int error;
7620 	vdev_t *rvd __maybe_unused = spa->spa_root_vdev;
7621 	vdev_t *vd, *pvd, *cvd, *tvd;
7622 	boolean_t unspare = B_FALSE;
7623 	uint64_t unspare_guid = 0;
7624 	char *vdpath;
7625 
7626 	ASSERT(spa_writeable(spa));
7627 
7628 	txg = spa_vdev_detach_enter(spa, guid);
7629 
7630 	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7631 
7632 	/*
7633 	 * Besides being called directly from the userland through the
7634 	 * ioctl interface, spa_vdev_detach() can be potentially called
7635 	 * at the end of spa_vdev_resilver_done().
7636 	 *
7637 	 * In the regular case, when we have a checkpoint this shouldn't
7638 	 * happen as we never empty the DTLs of a vdev during the scrub
7639 	 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done()
7640 	 * should never get here when we have a checkpoint.
7641 	 *
7642 	 * That said, even in a case when we checkpoint the pool exactly
7643 	 * as spa_vdev_resilver_done() calls this function everything
7644 	 * should be fine as the resilver will return right away.
7645 	 */
7646 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7647 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
7648 		error = (spa_has_checkpoint(spa)) ?
7649 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
7650 		return (spa_vdev_exit(spa, NULL, txg, error));
7651 	}
7652 
7653 	if (vd == NULL)
7654 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
7655 
7656 	if (!vd->vdev_ops->vdev_op_leaf)
7657 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7658 
7659 	pvd = vd->vdev_parent;
7660 
7661 	/*
7662 	 * If the parent/child relationship is not as expected, don't do it.
7663 	 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
7664 	 * vdev that's replacing B with C.  The user's intent in replacing
7665 	 * is to go from M(A,B) to M(A,C).  If the user decides to cancel
7666 	 * the replace by detaching C, the expected behavior is to end up
7667 	 * M(A,B).  But suppose that right after deciding to detach C,
7668 	 * the replacement of B completes.  We would have M(A,C), and then
7669 	 * ask to detach C, which would leave us with just A -- not what
7670 	 * the user wanted.  To prevent this, we make sure that the
7671 	 * parent/child relationship hasn't changed -- in this example,
7672 	 * that C's parent is still the replacing vdev R.
7673 	 */
7674 	if (pvd->vdev_guid != pguid && pguid != 0)
7675 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
7676 
7677 	/*
7678 	 * Only 'replacing' or 'spare' vdevs can be replaced.
7679 	 */
7680 	if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
7681 	    pvd->vdev_ops != &vdev_spare_ops)
7682 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7683 
7684 	ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
7685 	    spa_version(spa) >= SPA_VERSION_SPARES);
7686 
7687 	/*
7688 	 * Only mirror, replacing, and spare vdevs support detach.
7689 	 */
7690 	if (pvd->vdev_ops != &vdev_replacing_ops &&
7691 	    pvd->vdev_ops != &vdev_mirror_ops &&
7692 	    pvd->vdev_ops != &vdev_spare_ops)
7693 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
7694 
7695 	/*
7696 	 * If this device has the only valid copy of some data,
7697 	 * we cannot safely detach it.
7698 	 */
7699 	if (vdev_dtl_required(vd))
7700 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
7701 
7702 	ASSERT(pvd->vdev_children >= 2);
7703 
7704 	/*
7705 	 * If we are detaching the second disk from a replacing vdev, then
7706 	 * check to see if we changed the original vdev's path to have "/old"
7707 	 * at the end in spa_vdev_attach().  If so, undo that change now.
7708 	 */
7709 	if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
7710 	    vd->vdev_path != NULL) {
7711 		size_t len = strlen(vd->vdev_path);
7712 
7713 		for (int c = 0; c < pvd->vdev_children; c++) {
7714 			cvd = pvd->vdev_child[c];
7715 
7716 			if (cvd == vd || cvd->vdev_path == NULL)
7717 				continue;
7718 
7719 			if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
7720 			    strcmp(cvd->vdev_path + len, "/old") == 0) {
7721 				spa_strfree(cvd->vdev_path);
7722 				cvd->vdev_path = spa_strdup(vd->vdev_path);
7723 				break;
7724 			}
7725 		}
7726 	}
7727 
7728 	/*
7729 	 * If we are detaching the original disk from a normal spare, then it
7730 	 * implies that the spare should become a real disk, and be removed
7731 	 * from the active spare list for the pool.  dRAID spares on the
7732 	 * other hand are coupled to the pool and thus should never be removed
7733 	 * from the spares list.
7734 	 */
7735 	if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) {
7736 		vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1];
7737 
7738 		if (last_cvd->vdev_isspare &&
7739 		    last_cvd->vdev_ops != &vdev_draid_spare_ops) {
7740 			unspare = B_TRUE;
7741 		}
7742 	}
7743 
7744 	/*
7745 	 * Erase the disk labels so the disk can be used for other things.
7746 	 * This must be done after all other error cases are handled,
7747 	 * but before we disembowel vd (so we can still do I/O to it).
7748 	 * But if we can't do it, don't treat the error as fatal --
7749 	 * it may be that the unwritability of the disk is the reason
7750 	 * it's being detached!
7751 	 */
7752 	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
7753 
7754 	/*
7755 	 * Remove vd from its parent and compact the parent's children.
7756 	 */
7757 	vdev_remove_child(pvd, vd);
7758 	vdev_compact_children(pvd);
7759 
7760 	/*
7761 	 * Remember one of the remaining children so we can get tvd below.
7762 	 */
7763 	cvd = pvd->vdev_child[pvd->vdev_children - 1];
7764 
7765 	/*
7766 	 * If we need to remove the remaining child from the list of hot spares,
7767 	 * do it now, marking the vdev as no longer a spare in the process.
7768 	 * We must do this before vdev_remove_parent(), because that can
7769 	 * change the GUID if it creates a new toplevel GUID.  For a similar
7770 	 * reason, we must remove the spare now, in the same txg as the detach;
7771 	 * otherwise someone could attach a new sibling, change the GUID, and
7772 	 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail.
7773 	 */
7774 	if (unspare) {
7775 		ASSERT(cvd->vdev_isspare);
7776 		spa_spare_remove(cvd);
7777 		unspare_guid = cvd->vdev_guid;
7778 		(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
7779 		cvd->vdev_unspare = B_TRUE;
7780 	}
7781 
7782 	/*
7783 	 * If the parent mirror/replacing vdev only has one child,
7784 	 * the parent is no longer needed.  Remove it from the tree.
7785 	 */
7786 	if (pvd->vdev_children == 1) {
7787 		if (pvd->vdev_ops == &vdev_spare_ops)
7788 			cvd->vdev_unspare = B_FALSE;
7789 		vdev_remove_parent(cvd);
7790 	}
7791 
7792 	/*
7793 	 * We don't set tvd until now because the parent we just removed
7794 	 * may have been the previous top-level vdev.
7795 	 */
7796 	tvd = cvd->vdev_top;
7797 	ASSERT(tvd->vdev_parent == rvd);
7798 
7799 	/*
7800 	 * Reevaluate the parent vdev state.
7801 	 */
7802 	vdev_propagate_state(cvd);
7803 
7804 	/*
7805 	 * If the 'autoexpand' property is set on the pool then automatically
7806 	 * try to expand the size of the pool. For example if the device we
7807 	 * just detached was smaller than the others, it may be possible to
7808 	 * add metaslabs (i.e. grow the pool). We need to reopen the vdev
7809 	 * first so that we can obtain the updated sizes of the leaf vdevs.
7810 	 */
7811 	if (spa->spa_autoexpand) {
7812 		vdev_reopen(tvd);
7813 		vdev_expand(tvd, txg);
7814 	}
7815 
7816 	vdev_config_dirty(tvd);
7817 
7818 	/*
7819 	 * Mark vd's DTL as dirty in this txg.  vdev_dtl_sync() will see that
7820 	 * vd->vdev_detached is set and free vd's DTL object in syncing context.
7821 	 * But first make sure we're not on any *other* txg's DTL list, to
7822 	 * prevent vd from being accessed after it's freed.
7823 	 */
7824 	vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none");
7825 	for (int t = 0; t < TXG_SIZE; t++)
7826 		(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
7827 	vd->vdev_detached = B_TRUE;
7828 	vdev_dirty(tvd, VDD_DTL, vd, txg);
7829 
7830 	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE);
7831 	spa_notify_waiters(spa);
7832 
7833 	/* hang on to the spa before we release the lock */
7834 	spa_open_ref(spa, FTAG);
7835 
7836 	error = spa_vdev_exit(spa, vd, txg, 0);
7837 
7838 	spa_history_log_internal(spa, "detach", NULL,
7839 	    "vdev=%s", vdpath);
7840 	spa_strfree(vdpath);
7841 
7842 	/*
7843 	 * If this was the removal of the original device in a hot spare vdev,
7844 	 * then we want to go through and remove the device from the hot spare
7845 	 * list of every other pool.
7846 	 */
7847 	if (unspare) {
7848 		spa_t *altspa = NULL;
7849 
7850 		mutex_enter(&spa_namespace_lock);
7851 		while ((altspa = spa_next(altspa)) != NULL) {
7852 			if (altspa->spa_state != POOL_STATE_ACTIVE ||
7853 			    altspa == spa)
7854 				continue;
7855 
7856 			spa_open_ref(altspa, FTAG);
7857 			mutex_exit(&spa_namespace_lock);
7858 			(void) spa_vdev_remove(altspa, unspare_guid, B_TRUE);
7859 			mutex_enter(&spa_namespace_lock);
7860 			spa_close(altspa, FTAG);
7861 		}
7862 		mutex_exit(&spa_namespace_lock);
7863 
7864 		/* search the rest of the vdevs for spares to remove */
7865 		spa_vdev_resilver_done(spa);
7866 	}
7867 
7868 	/* all done with the spa; OK to release */
7869 	mutex_enter(&spa_namespace_lock);
7870 	spa_close(spa, FTAG);
7871 	mutex_exit(&spa_namespace_lock);
7872 
7873 	return (error);
7874 }
7875 
7876 static int
7877 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
7878     list_t *vd_list)
7879 {
7880 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7881 
7882 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
7883 
7884 	/* Look up vdev and ensure it's a leaf. */
7885 	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
7886 	if (vd == NULL || vd->vdev_detached) {
7887 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7888 		return (SET_ERROR(ENODEV));
7889 	} else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
7890 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7891 		return (SET_ERROR(EINVAL));
7892 	} else if (!vdev_writeable(vd)) {
7893 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7894 		return (SET_ERROR(EROFS));
7895 	}
7896 	mutex_enter(&vd->vdev_initialize_lock);
7897 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
7898 
7899 	/*
7900 	 * When we activate an initialize action we check to see
7901 	 * if the vdev_initialize_thread is NULL. We do this instead
7902 	 * of using the vdev_initialize_state since there might be
7903 	 * a previous initialization process which has completed but
7904 	 * the thread is not exited.
7905 	 */
7906 	if (cmd_type == POOL_INITIALIZE_START &&
7907 	    (vd->vdev_initialize_thread != NULL ||
7908 	    vd->vdev_top->vdev_removing || vd->vdev_top->vdev_rz_expanding)) {
7909 		mutex_exit(&vd->vdev_initialize_lock);
7910 		return (SET_ERROR(EBUSY));
7911 	} else if (cmd_type == POOL_INITIALIZE_CANCEL &&
7912 	    (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE &&
7913 	    vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) {
7914 		mutex_exit(&vd->vdev_initialize_lock);
7915 		return (SET_ERROR(ESRCH));
7916 	} else if (cmd_type == POOL_INITIALIZE_SUSPEND &&
7917 	    vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) {
7918 		mutex_exit(&vd->vdev_initialize_lock);
7919 		return (SET_ERROR(ESRCH));
7920 	} else if (cmd_type == POOL_INITIALIZE_UNINIT &&
7921 	    vd->vdev_initialize_thread != NULL) {
7922 		mutex_exit(&vd->vdev_initialize_lock);
7923 		return (SET_ERROR(EBUSY));
7924 	}
7925 
7926 	switch (cmd_type) {
7927 	case POOL_INITIALIZE_START:
7928 		vdev_initialize(vd);
7929 		break;
7930 	case POOL_INITIALIZE_CANCEL:
7931 		vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list);
7932 		break;
7933 	case POOL_INITIALIZE_SUSPEND:
7934 		vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list);
7935 		break;
7936 	case POOL_INITIALIZE_UNINIT:
7937 		vdev_uninitialize(vd);
7938 		break;
7939 	default:
7940 		panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
7941 	}
7942 	mutex_exit(&vd->vdev_initialize_lock);
7943 
7944 	return (0);
7945 }
7946 
7947 int
7948 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type,
7949     nvlist_t *vdev_errlist)
7950 {
7951 	int total_errors = 0;
7952 	list_t vd_list;
7953 
7954 	list_create(&vd_list, sizeof (vdev_t),
7955 	    offsetof(vdev_t, vdev_initialize_node));
7956 
7957 	/*
7958 	 * We hold the namespace lock through the whole function
7959 	 * to prevent any changes to the pool while we're starting or
7960 	 * stopping initialization. The config and state locks are held so that
7961 	 * we can properly assess the vdev state before we commit to
7962 	 * the initializing operation.
7963 	 */
7964 	mutex_enter(&spa_namespace_lock);
7965 
7966 	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
7967 	    pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
7968 		uint64_t vdev_guid = fnvpair_value_uint64(pair);
7969 
7970 		int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type,
7971 		    &vd_list);
7972 		if (error != 0) {
7973 			char guid_as_str[MAXNAMELEN];
7974 
7975 			(void) snprintf(guid_as_str, sizeof (guid_as_str),
7976 			    "%llu", (unsigned long long)vdev_guid);
7977 			fnvlist_add_int64(vdev_errlist, guid_as_str, error);
7978 			total_errors++;
7979 		}
7980 	}
7981 
7982 	/* Wait for all initialize threads to stop. */
7983 	vdev_initialize_stop_wait(spa, &vd_list);
7984 
7985 	/* Sync out the initializing state */
7986 	txg_wait_synced(spa->spa_dsl_pool, 0);
7987 	mutex_exit(&spa_namespace_lock);
7988 
7989 	list_destroy(&vd_list);
7990 
7991 	return (total_errors);
7992 }
7993 
7994 static int
7995 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type,
7996     uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list)
7997 {
7998 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
7999 
8000 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
8001 
8002 	/* Look up vdev and ensure it's a leaf. */
8003 	vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE);
8004 	if (vd == NULL || vd->vdev_detached) {
8005 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
8006 		return (SET_ERROR(ENODEV));
8007 	} else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
8008 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
8009 		return (SET_ERROR(EINVAL));
8010 	} else if (!vdev_writeable(vd)) {
8011 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
8012 		return (SET_ERROR(EROFS));
8013 	} else if (!vd->vdev_has_trim) {
8014 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
8015 		return (SET_ERROR(EOPNOTSUPP));
8016 	} else if (secure && !vd->vdev_has_securetrim) {
8017 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
8018 		return (SET_ERROR(EOPNOTSUPP));
8019 	}
8020 	mutex_enter(&vd->vdev_trim_lock);
8021 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
8022 
8023 	/*
8024 	 * When we activate a TRIM action we check to see if the
8025 	 * vdev_trim_thread is NULL. We do this instead of using the
8026 	 * vdev_trim_state since there might be a previous TRIM process
8027 	 * which has completed but the thread is not exited.
8028 	 */
8029 	if (cmd_type == POOL_TRIM_START &&
8030 	    (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing ||
8031 	    vd->vdev_top->vdev_rz_expanding)) {
8032 		mutex_exit(&vd->vdev_trim_lock);
8033 		return (SET_ERROR(EBUSY));
8034 	} else if (cmd_type == POOL_TRIM_CANCEL &&
8035 	    (vd->vdev_trim_state != VDEV_TRIM_ACTIVE &&
8036 	    vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) {
8037 		mutex_exit(&vd->vdev_trim_lock);
8038 		return (SET_ERROR(ESRCH));
8039 	} else if (cmd_type == POOL_TRIM_SUSPEND &&
8040 	    vd->vdev_trim_state != VDEV_TRIM_ACTIVE) {
8041 		mutex_exit(&vd->vdev_trim_lock);
8042 		return (SET_ERROR(ESRCH));
8043 	}
8044 
8045 	switch (cmd_type) {
8046 	case POOL_TRIM_START:
8047 		vdev_trim(vd, rate, partial, secure);
8048 		break;
8049 	case POOL_TRIM_CANCEL:
8050 		vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list);
8051 		break;
8052 	case POOL_TRIM_SUSPEND:
8053 		vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list);
8054 		break;
8055 	default:
8056 		panic("invalid cmd_type %llu", (unsigned long long)cmd_type);
8057 	}
8058 	mutex_exit(&vd->vdev_trim_lock);
8059 
8060 	return (0);
8061 }
8062 
8063 /*
8064  * Initiates a manual TRIM for the requested vdevs. This kicks off individual
8065  * TRIM threads for each child vdev.  These threads pass over all of the free
8066  * space in the vdev's metaslabs and issues TRIM commands for that space.
8067  */
8068 int
8069 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate,
8070     boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist)
8071 {
8072 	int total_errors = 0;
8073 	list_t vd_list;
8074 
8075 	list_create(&vd_list, sizeof (vdev_t),
8076 	    offsetof(vdev_t, vdev_trim_node));
8077 
8078 	/*
8079 	 * We hold the namespace lock through the whole function
8080 	 * to prevent any changes to the pool while we're starting or
8081 	 * stopping TRIM. The config and state locks are held so that
8082 	 * we can properly assess the vdev state before we commit to
8083 	 * the TRIM operation.
8084 	 */
8085 	mutex_enter(&spa_namespace_lock);
8086 
8087 	for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL);
8088 	    pair != NULL; pair = nvlist_next_nvpair(nv, pair)) {
8089 		uint64_t vdev_guid = fnvpair_value_uint64(pair);
8090 
8091 		int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type,
8092 		    rate, partial, secure, &vd_list);
8093 		if (error != 0) {
8094 			char guid_as_str[MAXNAMELEN];
8095 
8096 			(void) snprintf(guid_as_str, sizeof (guid_as_str),
8097 			    "%llu", (unsigned long long)vdev_guid);
8098 			fnvlist_add_int64(vdev_errlist, guid_as_str, error);
8099 			total_errors++;
8100 		}
8101 	}
8102 
8103 	/* Wait for all TRIM threads to stop. */
8104 	vdev_trim_stop_wait(spa, &vd_list);
8105 
8106 	/* Sync out the TRIM state */
8107 	txg_wait_synced(spa->spa_dsl_pool, 0);
8108 	mutex_exit(&spa_namespace_lock);
8109 
8110 	list_destroy(&vd_list);
8111 
8112 	return (total_errors);
8113 }
8114 
8115 /*
8116  * Split a set of devices from their mirrors, and create a new pool from them.
8117  */
8118 int
8119 spa_vdev_split_mirror(spa_t *spa, const char *newname, nvlist_t *config,
8120     nvlist_t *props, boolean_t exp)
8121 {
8122 	int error = 0;
8123 	uint64_t txg, *glist;
8124 	spa_t *newspa;
8125 	uint_t c, children, lastlog;
8126 	nvlist_t **child, *nvl, *tmp;
8127 	dmu_tx_t *tx;
8128 	const char *altroot = NULL;
8129 	vdev_t *rvd, **vml = NULL;			/* vdev modify list */
8130 	boolean_t activate_slog;
8131 
8132 	ASSERT(spa_writeable(spa));
8133 
8134 	txg = spa_vdev_enter(spa);
8135 
8136 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
8137 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
8138 		error = (spa_has_checkpoint(spa)) ?
8139 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
8140 		return (spa_vdev_exit(spa, NULL, txg, error));
8141 	}
8142 
8143 	/* clear the log and flush everything up to now */
8144 	activate_slog = spa_passivate_log(spa);
8145 	(void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
8146 	error = spa_reset_logs(spa);
8147 	txg = spa_vdev_config_enter(spa);
8148 
8149 	if (activate_slog)
8150 		spa_activate_log(spa);
8151 
8152 	if (error != 0)
8153 		return (spa_vdev_exit(spa, NULL, txg, error));
8154 
8155 	/* check new spa name before going any further */
8156 	if (spa_lookup(newname) != NULL)
8157 		return (spa_vdev_exit(spa, NULL, txg, EEXIST));
8158 
8159 	/*
8160 	 * scan through all the children to ensure they're all mirrors
8161 	 */
8162 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 ||
8163 	    nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
8164 	    &children) != 0)
8165 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
8166 
8167 	/* first, check to ensure we've got the right child count */
8168 	rvd = spa->spa_root_vdev;
8169 	lastlog = 0;
8170 	for (c = 0; c < rvd->vdev_children; c++) {
8171 		vdev_t *vd = rvd->vdev_child[c];
8172 
8173 		/* don't count the holes & logs as children */
8174 		if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops &&
8175 		    !vdev_is_concrete(vd))) {
8176 			if (lastlog == 0)
8177 				lastlog = c;
8178 			continue;
8179 		}
8180 
8181 		lastlog = 0;
8182 	}
8183 	if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
8184 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
8185 
8186 	/* next, ensure no spare or cache devices are part of the split */
8187 	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 ||
8188 	    nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0)
8189 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
8190 
8191 	vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP);
8192 	glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP);
8193 
8194 	/* then, loop over each vdev and validate it */
8195 	for (c = 0; c < children; c++) {
8196 		uint64_t is_hole = 0;
8197 
8198 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
8199 		    &is_hole);
8200 
8201 		if (is_hole != 0) {
8202 			if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole ||
8203 			    spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
8204 				continue;
8205 			} else {
8206 				error = SET_ERROR(EINVAL);
8207 				break;
8208 			}
8209 		}
8210 
8211 		/* deal with indirect vdevs */
8212 		if (spa->spa_root_vdev->vdev_child[c]->vdev_ops ==
8213 		    &vdev_indirect_ops)
8214 			continue;
8215 
8216 		/* which disk is going to be split? */
8217 		if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
8218 		    &glist[c]) != 0) {
8219 			error = SET_ERROR(EINVAL);
8220 			break;
8221 		}
8222 
8223 		/* look it up in the spa */
8224 		vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE);
8225 		if (vml[c] == NULL) {
8226 			error = SET_ERROR(ENODEV);
8227 			break;
8228 		}
8229 
8230 		/* make sure there's nothing stopping the split */
8231 		if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops ||
8232 		    vml[c]->vdev_islog ||
8233 		    !vdev_is_concrete(vml[c]) ||
8234 		    vml[c]->vdev_isspare ||
8235 		    vml[c]->vdev_isl2cache ||
8236 		    !vdev_writeable(vml[c]) ||
8237 		    vml[c]->vdev_children != 0 ||
8238 		    vml[c]->vdev_state != VDEV_STATE_HEALTHY ||
8239 		    c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
8240 			error = SET_ERROR(EINVAL);
8241 			break;
8242 		}
8243 
8244 		if (vdev_dtl_required(vml[c]) ||
8245 		    vdev_resilver_needed(vml[c], NULL, NULL)) {
8246 			error = SET_ERROR(EBUSY);
8247 			break;
8248 		}
8249 
8250 		/* we need certain info from the top level */
8251 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY,
8252 		    vml[c]->vdev_top->vdev_ms_array);
8253 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT,
8254 		    vml[c]->vdev_top->vdev_ms_shift);
8255 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE,
8256 		    vml[c]->vdev_top->vdev_asize);
8257 		fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT,
8258 		    vml[c]->vdev_top->vdev_ashift);
8259 
8260 		/* transfer per-vdev ZAPs */
8261 		ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0);
8262 		VERIFY0(nvlist_add_uint64(child[c],
8263 		    ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap));
8264 
8265 		ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0);
8266 		VERIFY0(nvlist_add_uint64(child[c],
8267 		    ZPOOL_CONFIG_VDEV_TOP_ZAP,
8268 		    vml[c]->vdev_parent->vdev_top_zap));
8269 	}
8270 
8271 	if (error != 0) {
8272 		kmem_free(vml, children * sizeof (vdev_t *));
8273 		kmem_free(glist, children * sizeof (uint64_t));
8274 		return (spa_vdev_exit(spa, NULL, txg, error));
8275 	}
8276 
8277 	/* stop writers from using the disks */
8278 	for (c = 0; c < children; c++) {
8279 		if (vml[c] != NULL)
8280 			vml[c]->vdev_offline = B_TRUE;
8281 	}
8282 	vdev_reopen(spa->spa_root_vdev);
8283 
8284 	/*
8285 	 * Temporarily record the splitting vdevs in the spa config.  This
8286 	 * will disappear once the config is regenerated.
8287 	 */
8288 	nvl = fnvlist_alloc();
8289 	fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children);
8290 	kmem_free(glist, children * sizeof (uint64_t));
8291 
8292 	mutex_enter(&spa->spa_props_lock);
8293 	fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl);
8294 	mutex_exit(&spa->spa_props_lock);
8295 	spa->spa_config_splitting = nvl;
8296 	vdev_config_dirty(spa->spa_root_vdev);
8297 
8298 	/* configure and create the new pool */
8299 	fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname);
8300 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
8301 	    exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE);
8302 	fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa));
8303 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg);
8304 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID,
8305 	    spa_generate_guid(NULL));
8306 	VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
8307 	(void) nvlist_lookup_string(props,
8308 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
8309 
8310 	/* add the new pool to the namespace */
8311 	newspa = spa_add(newname, config, altroot);
8312 	newspa->spa_avz_action = AVZ_ACTION_REBUILD;
8313 	newspa->spa_config_txg = spa->spa_config_txg;
8314 	spa_set_log_state(newspa, SPA_LOG_CLEAR);
8315 
8316 	/* release the spa config lock, retaining the namespace lock */
8317 	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
8318 
8319 	if (zio_injection_enabled)
8320 		zio_handle_panic_injection(spa, FTAG, 1);
8321 
8322 	spa_activate(newspa, spa_mode_global);
8323 	spa_async_suspend(newspa);
8324 
8325 	/*
8326 	 * Temporarily stop the initializing and TRIM activity.  We set the
8327 	 * state to ACTIVE so that we know to resume initializing or TRIM
8328 	 * once the split has completed.
8329 	 */
8330 	list_t vd_initialize_list;
8331 	list_create(&vd_initialize_list, sizeof (vdev_t),
8332 	    offsetof(vdev_t, vdev_initialize_node));
8333 
8334 	list_t vd_trim_list;
8335 	list_create(&vd_trim_list, sizeof (vdev_t),
8336 	    offsetof(vdev_t, vdev_trim_node));
8337 
8338 	for (c = 0; c < children; c++) {
8339 		if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
8340 			mutex_enter(&vml[c]->vdev_initialize_lock);
8341 			vdev_initialize_stop(vml[c],
8342 			    VDEV_INITIALIZE_ACTIVE, &vd_initialize_list);
8343 			mutex_exit(&vml[c]->vdev_initialize_lock);
8344 
8345 			mutex_enter(&vml[c]->vdev_trim_lock);
8346 			vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list);
8347 			mutex_exit(&vml[c]->vdev_trim_lock);
8348 		}
8349 	}
8350 
8351 	vdev_initialize_stop_wait(spa, &vd_initialize_list);
8352 	vdev_trim_stop_wait(spa, &vd_trim_list);
8353 
8354 	list_destroy(&vd_initialize_list);
8355 	list_destroy(&vd_trim_list);
8356 
8357 	newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT;
8358 	newspa->spa_is_splitting = B_TRUE;
8359 
8360 	/* create the new pool from the disks of the original pool */
8361 	error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE);
8362 	if (error)
8363 		goto out;
8364 
8365 	/* if that worked, generate a real config for the new pool */
8366 	if (newspa->spa_root_vdev != NULL) {
8367 		newspa->spa_config_splitting = fnvlist_alloc();
8368 		fnvlist_add_uint64(newspa->spa_config_splitting,
8369 		    ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa));
8370 		spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
8371 		    B_TRUE));
8372 	}
8373 
8374 	/* set the props */
8375 	if (props != NULL) {
8376 		spa_configfile_set(newspa, props, B_FALSE);
8377 		error = spa_prop_set(newspa, props);
8378 		if (error)
8379 			goto out;
8380 	}
8381 
8382 	/* flush everything */
8383 	txg = spa_vdev_config_enter(newspa);
8384 	vdev_config_dirty(newspa->spa_root_vdev);
8385 	(void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG);
8386 
8387 	if (zio_injection_enabled)
8388 		zio_handle_panic_injection(spa, FTAG, 2);
8389 
8390 	spa_async_resume(newspa);
8391 
8392 	/* finally, update the original pool's config */
8393 	txg = spa_vdev_config_enter(spa);
8394 	tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
8395 	error = dmu_tx_assign(tx, TXG_WAIT);
8396 	if (error != 0)
8397 		dmu_tx_abort(tx);
8398 	for (c = 0; c < children; c++) {
8399 		if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
8400 			vdev_t *tvd = vml[c]->vdev_top;
8401 
8402 			/*
8403 			 * Need to be sure the detachable VDEV is not
8404 			 * on any *other* txg's DTL list to prevent it
8405 			 * from being accessed after it's freed.
8406 			 */
8407 			for (int t = 0; t < TXG_SIZE; t++) {
8408 				(void) txg_list_remove_this(
8409 				    &tvd->vdev_dtl_list, vml[c], t);
8410 			}
8411 
8412 			vdev_split(vml[c]);
8413 			if (error == 0)
8414 				spa_history_log_internal(spa, "detach", tx,
8415 				    "vdev=%s", vml[c]->vdev_path);
8416 
8417 			vdev_free(vml[c]);
8418 		}
8419 	}
8420 	spa->spa_avz_action = AVZ_ACTION_REBUILD;
8421 	vdev_config_dirty(spa->spa_root_vdev);
8422 	spa->spa_config_splitting = NULL;
8423 	nvlist_free(nvl);
8424 	if (error == 0)
8425 		dmu_tx_commit(tx);
8426 	(void) spa_vdev_exit(spa, NULL, txg, 0);
8427 
8428 	if (zio_injection_enabled)
8429 		zio_handle_panic_injection(spa, FTAG, 3);
8430 
8431 	/* split is complete; log a history record */
8432 	spa_history_log_internal(newspa, "split", NULL,
8433 	    "from pool %s", spa_name(spa));
8434 
8435 	newspa->spa_is_splitting = B_FALSE;
8436 	kmem_free(vml, children * sizeof (vdev_t *));
8437 
8438 	/* if we're not going to mount the filesystems in userland, export */
8439 	if (exp)
8440 		error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL,
8441 		    B_FALSE, B_FALSE);
8442 
8443 	return (error);
8444 
8445 out:
8446 	spa_unload(newspa);
8447 	spa_deactivate(newspa);
8448 	spa_remove(newspa);
8449 
8450 	txg = spa_vdev_config_enter(spa);
8451 
8452 	/* re-online all offlined disks */
8453 	for (c = 0; c < children; c++) {
8454 		if (vml[c] != NULL)
8455 			vml[c]->vdev_offline = B_FALSE;
8456 	}
8457 
8458 	/* restart initializing or trimming disks as necessary */
8459 	spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART);
8460 	spa_async_request(spa, SPA_ASYNC_TRIM_RESTART);
8461 	spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART);
8462 
8463 	vdev_reopen(spa->spa_root_vdev);
8464 
8465 	nvlist_free(spa->spa_config_splitting);
8466 	spa->spa_config_splitting = NULL;
8467 	(void) spa_vdev_exit(spa, NULL, txg, error);
8468 
8469 	kmem_free(vml, children * sizeof (vdev_t *));
8470 	return (error);
8471 }
8472 
8473 /*
8474  * Find any device that's done replacing, or a vdev marked 'unspare' that's
8475  * currently spared, so we can detach it.
8476  */
8477 static vdev_t *
8478 spa_vdev_resilver_done_hunt(vdev_t *vd)
8479 {
8480 	vdev_t *newvd, *oldvd;
8481 
8482 	for (int c = 0; c < vd->vdev_children; c++) {
8483 		oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
8484 		if (oldvd != NULL)
8485 			return (oldvd);
8486 	}
8487 
8488 	/*
8489 	 * Check for a completed replacement.  We always consider the first
8490 	 * vdev in the list to be the oldest vdev, and the last one to be
8491 	 * the newest (see spa_vdev_attach() for how that works).  In
8492 	 * the case where the newest vdev is faulted, we will not automatically
8493 	 * remove it after a resilver completes.  This is OK as it will require
8494 	 * user intervention to determine which disk the admin wishes to keep.
8495 	 */
8496 	if (vd->vdev_ops == &vdev_replacing_ops) {
8497 		ASSERT(vd->vdev_children > 1);
8498 
8499 		newvd = vd->vdev_child[vd->vdev_children - 1];
8500 		oldvd = vd->vdev_child[0];
8501 
8502 		if (vdev_dtl_empty(newvd, DTL_MISSING) &&
8503 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
8504 		    !vdev_dtl_required(oldvd))
8505 			return (oldvd);
8506 	}
8507 
8508 	/*
8509 	 * Check for a completed resilver with the 'unspare' flag set.
8510 	 * Also potentially update faulted state.
8511 	 */
8512 	if (vd->vdev_ops == &vdev_spare_ops) {
8513 		vdev_t *first = vd->vdev_child[0];
8514 		vdev_t *last = vd->vdev_child[vd->vdev_children - 1];
8515 
8516 		if (last->vdev_unspare) {
8517 			oldvd = first;
8518 			newvd = last;
8519 		} else if (first->vdev_unspare) {
8520 			oldvd = last;
8521 			newvd = first;
8522 		} else {
8523 			oldvd = NULL;
8524 		}
8525 
8526 		if (oldvd != NULL &&
8527 		    vdev_dtl_empty(newvd, DTL_MISSING) &&
8528 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
8529 		    !vdev_dtl_required(oldvd))
8530 			return (oldvd);
8531 
8532 		vdev_propagate_state(vd);
8533 
8534 		/*
8535 		 * If there are more than two spares attached to a disk,
8536 		 * and those spares are not required, then we want to
8537 		 * attempt to free them up now so that they can be used
8538 		 * by other pools.  Once we're back down to a single
8539 		 * disk+spare, we stop removing them.
8540 		 */
8541 		if (vd->vdev_children > 2) {
8542 			newvd = vd->vdev_child[1];
8543 
8544 			if (newvd->vdev_isspare && last->vdev_isspare &&
8545 			    vdev_dtl_empty(last, DTL_MISSING) &&
8546 			    vdev_dtl_empty(last, DTL_OUTAGE) &&
8547 			    !vdev_dtl_required(newvd))
8548 				return (newvd);
8549 		}
8550 	}
8551 
8552 	return (NULL);
8553 }
8554 
8555 static void
8556 spa_vdev_resilver_done(spa_t *spa)
8557 {
8558 	vdev_t *vd, *pvd, *ppvd;
8559 	uint64_t guid, sguid, pguid, ppguid;
8560 
8561 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
8562 
8563 	while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
8564 		pvd = vd->vdev_parent;
8565 		ppvd = pvd->vdev_parent;
8566 		guid = vd->vdev_guid;
8567 		pguid = pvd->vdev_guid;
8568 		ppguid = ppvd->vdev_guid;
8569 		sguid = 0;
8570 		/*
8571 		 * If we have just finished replacing a hot spared device, then
8572 		 * we need to detach the parent's first child (the original hot
8573 		 * spare) as well.
8574 		 */
8575 		if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
8576 		    ppvd->vdev_children == 2) {
8577 			ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
8578 			sguid = ppvd->vdev_child[1]->vdev_guid;
8579 		}
8580 		ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd));
8581 
8582 		spa_config_exit(spa, SCL_ALL, FTAG);
8583 		if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0)
8584 			return;
8585 		if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0)
8586 			return;
8587 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
8588 	}
8589 
8590 	spa_config_exit(spa, SCL_ALL, FTAG);
8591 
8592 	/*
8593 	 * If a detach was not performed above replace waiters will not have
8594 	 * been notified.  In which case we must do so now.
8595 	 */
8596 	spa_notify_waiters(spa);
8597 }
8598 
8599 /*
8600  * Update the stored path or FRU for this vdev.
8601  */
8602 static int
8603 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value,
8604     boolean_t ispath)
8605 {
8606 	vdev_t *vd;
8607 	boolean_t sync = B_FALSE;
8608 
8609 	ASSERT(spa_writeable(spa));
8610 
8611 	spa_vdev_state_enter(spa, SCL_ALL);
8612 
8613 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
8614 		return (spa_vdev_state_exit(spa, NULL, ENOENT));
8615 
8616 	if (!vd->vdev_ops->vdev_op_leaf)
8617 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
8618 
8619 	if (ispath) {
8620 		if (strcmp(value, vd->vdev_path) != 0) {
8621 			spa_strfree(vd->vdev_path);
8622 			vd->vdev_path = spa_strdup(value);
8623 			sync = B_TRUE;
8624 		}
8625 	} else {
8626 		if (vd->vdev_fru == NULL) {
8627 			vd->vdev_fru = spa_strdup(value);
8628 			sync = B_TRUE;
8629 		} else if (strcmp(value, vd->vdev_fru) != 0) {
8630 			spa_strfree(vd->vdev_fru);
8631 			vd->vdev_fru = spa_strdup(value);
8632 			sync = B_TRUE;
8633 		}
8634 	}
8635 
8636 	return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0));
8637 }
8638 
8639 int
8640 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath)
8641 {
8642 	return (spa_vdev_set_common(spa, guid, newpath, B_TRUE));
8643 }
8644 
8645 int
8646 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru)
8647 {
8648 	return (spa_vdev_set_common(spa, guid, newfru, B_FALSE));
8649 }
8650 
8651 /*
8652  * ==========================================================================
8653  * SPA Scanning
8654  * ==========================================================================
8655  */
8656 int
8657 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd)
8658 {
8659 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8660 
8661 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
8662 		return (SET_ERROR(EBUSY));
8663 
8664 	return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd));
8665 }
8666 
8667 int
8668 spa_scan_stop(spa_t *spa)
8669 {
8670 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8671 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
8672 		return (SET_ERROR(EBUSY));
8673 
8674 	return (dsl_scan_cancel(spa->spa_dsl_pool));
8675 }
8676 
8677 int
8678 spa_scan(spa_t *spa, pool_scan_func_t func)
8679 {
8680 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
8681 
8682 	if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE)
8683 		return (SET_ERROR(ENOTSUP));
8684 
8685 	if (func == POOL_SCAN_RESILVER &&
8686 	    !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER))
8687 		return (SET_ERROR(ENOTSUP));
8688 
8689 	/*
8690 	 * If a resilver was requested, but there is no DTL on a
8691 	 * writeable leaf device, we have nothing to do.
8692 	 */
8693 	if (func == POOL_SCAN_RESILVER &&
8694 	    !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
8695 		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
8696 		return (0);
8697 	}
8698 
8699 	if (func == POOL_SCAN_ERRORSCRUB &&
8700 	    !spa_feature_is_enabled(spa, SPA_FEATURE_HEAD_ERRLOG))
8701 		return (SET_ERROR(ENOTSUP));
8702 
8703 	return (dsl_scan(spa->spa_dsl_pool, func));
8704 }
8705 
8706 /*
8707  * ==========================================================================
8708  * SPA async task processing
8709  * ==========================================================================
8710  */
8711 
8712 static void
8713 spa_async_remove(spa_t *spa, vdev_t *vd)
8714 {
8715 	if (vd->vdev_remove_wanted) {
8716 		vd->vdev_remove_wanted = B_FALSE;
8717 		vd->vdev_delayed_close = B_FALSE;
8718 		vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE);
8719 
8720 		/*
8721 		 * We want to clear the stats, but we don't want to do a full
8722 		 * vdev_clear() as that will cause us to throw away
8723 		 * degraded/faulted state as well as attempt to reopen the
8724 		 * device, all of which is a waste.
8725 		 */
8726 		vd->vdev_stat.vs_read_errors = 0;
8727 		vd->vdev_stat.vs_write_errors = 0;
8728 		vd->vdev_stat.vs_checksum_errors = 0;
8729 
8730 		vdev_state_dirty(vd->vdev_top);
8731 
8732 		/* Tell userspace that the vdev is gone. */
8733 		zfs_post_remove(spa, vd);
8734 	}
8735 
8736 	for (int c = 0; c < vd->vdev_children; c++)
8737 		spa_async_remove(spa, vd->vdev_child[c]);
8738 }
8739 
8740 static void
8741 spa_async_probe(spa_t *spa, vdev_t *vd)
8742 {
8743 	if (vd->vdev_probe_wanted) {
8744 		vd->vdev_probe_wanted = B_FALSE;
8745 		vdev_reopen(vd);	/* vdev_open() does the actual probe */
8746 	}
8747 
8748 	for (int c = 0; c < vd->vdev_children; c++)
8749 		spa_async_probe(spa, vd->vdev_child[c]);
8750 }
8751 
8752 static void
8753 spa_async_autoexpand(spa_t *spa, vdev_t *vd)
8754 {
8755 	if (!spa->spa_autoexpand)
8756 		return;
8757 
8758 	for (int c = 0; c < vd->vdev_children; c++) {
8759 		vdev_t *cvd = vd->vdev_child[c];
8760 		spa_async_autoexpand(spa, cvd);
8761 	}
8762 
8763 	if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL)
8764 		return;
8765 
8766 	spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND);
8767 }
8768 
8769 static __attribute__((noreturn)) void
8770 spa_async_thread(void *arg)
8771 {
8772 	spa_t *spa = (spa_t *)arg;
8773 	dsl_pool_t *dp = spa->spa_dsl_pool;
8774 	int tasks;
8775 
8776 	ASSERT(spa->spa_sync_on);
8777 
8778 	mutex_enter(&spa->spa_async_lock);
8779 	tasks = spa->spa_async_tasks;
8780 	spa->spa_async_tasks = 0;
8781 	mutex_exit(&spa->spa_async_lock);
8782 
8783 	/*
8784 	 * See if the config needs to be updated.
8785 	 */
8786 	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
8787 		uint64_t old_space, new_space;
8788 
8789 		mutex_enter(&spa_namespace_lock);
8790 		old_space = metaslab_class_get_space(spa_normal_class(spa));
8791 		old_space += metaslab_class_get_space(spa_special_class(spa));
8792 		old_space += metaslab_class_get_space(spa_dedup_class(spa));
8793 		old_space += metaslab_class_get_space(
8794 		    spa_embedded_log_class(spa));
8795 
8796 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
8797 
8798 		new_space = metaslab_class_get_space(spa_normal_class(spa));
8799 		new_space += metaslab_class_get_space(spa_special_class(spa));
8800 		new_space += metaslab_class_get_space(spa_dedup_class(spa));
8801 		new_space += metaslab_class_get_space(
8802 		    spa_embedded_log_class(spa));
8803 		mutex_exit(&spa_namespace_lock);
8804 
8805 		/*
8806 		 * If the pool grew as a result of the config update,
8807 		 * then log an internal history event.
8808 		 */
8809 		if (new_space != old_space) {
8810 			spa_history_log_internal(spa, "vdev online", NULL,
8811 			    "pool '%s' size: %llu(+%llu)",
8812 			    spa_name(spa), (u_longlong_t)new_space,
8813 			    (u_longlong_t)(new_space - old_space));
8814 		}
8815 	}
8816 
8817 	/*
8818 	 * See if any devices need to be marked REMOVED.
8819 	 */
8820 	if (tasks & SPA_ASYNC_REMOVE) {
8821 		spa_vdev_state_enter(spa, SCL_NONE);
8822 		spa_async_remove(spa, spa->spa_root_vdev);
8823 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
8824 			spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]);
8825 		for (int i = 0; i < spa->spa_spares.sav_count; i++)
8826 			spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]);
8827 		(void) spa_vdev_state_exit(spa, NULL, 0);
8828 	}
8829 
8830 	if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) {
8831 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8832 		spa_async_autoexpand(spa, spa->spa_root_vdev);
8833 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8834 	}
8835 
8836 	/*
8837 	 * See if any devices need to be probed.
8838 	 */
8839 	if (tasks & SPA_ASYNC_PROBE) {
8840 		spa_vdev_state_enter(spa, SCL_NONE);
8841 		spa_async_probe(spa, spa->spa_root_vdev);
8842 		(void) spa_vdev_state_exit(spa, NULL, 0);
8843 	}
8844 
8845 	/*
8846 	 * If any devices are done replacing, detach them.
8847 	 */
8848 	if (tasks & SPA_ASYNC_RESILVER_DONE ||
8849 	    tasks & SPA_ASYNC_REBUILD_DONE ||
8850 	    tasks & SPA_ASYNC_DETACH_SPARE) {
8851 		spa_vdev_resilver_done(spa);
8852 	}
8853 
8854 	/*
8855 	 * Kick off a resilver.
8856 	 */
8857 	if (tasks & SPA_ASYNC_RESILVER &&
8858 	    !vdev_rebuild_active(spa->spa_root_vdev) &&
8859 	    (!dsl_scan_resilvering(dp) ||
8860 	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)))
8861 		dsl_scan_restart_resilver(dp, 0);
8862 
8863 	if (tasks & SPA_ASYNC_INITIALIZE_RESTART) {
8864 		mutex_enter(&spa_namespace_lock);
8865 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8866 		vdev_initialize_restart(spa->spa_root_vdev);
8867 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8868 		mutex_exit(&spa_namespace_lock);
8869 	}
8870 
8871 	if (tasks & SPA_ASYNC_TRIM_RESTART) {
8872 		mutex_enter(&spa_namespace_lock);
8873 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8874 		vdev_trim_restart(spa->spa_root_vdev);
8875 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8876 		mutex_exit(&spa_namespace_lock);
8877 	}
8878 
8879 	if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) {
8880 		mutex_enter(&spa_namespace_lock);
8881 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8882 		vdev_autotrim_restart(spa);
8883 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8884 		mutex_exit(&spa_namespace_lock);
8885 	}
8886 
8887 	/*
8888 	 * Kick off L2 cache whole device TRIM.
8889 	 */
8890 	if (tasks & SPA_ASYNC_L2CACHE_TRIM) {
8891 		mutex_enter(&spa_namespace_lock);
8892 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
8893 		vdev_trim_l2arc(spa);
8894 		spa_config_exit(spa, SCL_CONFIG, FTAG);
8895 		mutex_exit(&spa_namespace_lock);
8896 	}
8897 
8898 	/*
8899 	 * Kick off L2 cache rebuilding.
8900 	 */
8901 	if (tasks & SPA_ASYNC_L2CACHE_REBUILD) {
8902 		mutex_enter(&spa_namespace_lock);
8903 		spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER);
8904 		l2arc_spa_rebuild_start(spa);
8905 		spa_config_exit(spa, SCL_L2ARC, FTAG);
8906 		mutex_exit(&spa_namespace_lock);
8907 	}
8908 
8909 	/*
8910 	 * Let the world know that we're done.
8911 	 */
8912 	mutex_enter(&spa->spa_async_lock);
8913 	spa->spa_async_thread = NULL;
8914 	cv_broadcast(&spa->spa_async_cv);
8915 	mutex_exit(&spa->spa_async_lock);
8916 	thread_exit();
8917 }
8918 
8919 void
8920 spa_async_suspend(spa_t *spa)
8921 {
8922 	mutex_enter(&spa->spa_async_lock);
8923 	spa->spa_async_suspended++;
8924 	while (spa->spa_async_thread != NULL)
8925 		cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
8926 	mutex_exit(&spa->spa_async_lock);
8927 
8928 	spa_vdev_remove_suspend(spa);
8929 
8930 	zthr_t *condense_thread = spa->spa_condense_zthr;
8931 	if (condense_thread != NULL)
8932 		zthr_cancel(condense_thread);
8933 
8934 	zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr;
8935 	if (raidz_expand_thread != NULL)
8936 		zthr_cancel(raidz_expand_thread);
8937 
8938 	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
8939 	if (discard_thread != NULL)
8940 		zthr_cancel(discard_thread);
8941 
8942 	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
8943 	if (ll_delete_thread != NULL)
8944 		zthr_cancel(ll_delete_thread);
8945 
8946 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
8947 	if (ll_condense_thread != NULL)
8948 		zthr_cancel(ll_condense_thread);
8949 }
8950 
8951 void
8952 spa_async_resume(spa_t *spa)
8953 {
8954 	mutex_enter(&spa->spa_async_lock);
8955 	ASSERT(spa->spa_async_suspended != 0);
8956 	spa->spa_async_suspended--;
8957 	mutex_exit(&spa->spa_async_lock);
8958 	spa_restart_removal(spa);
8959 
8960 	zthr_t *condense_thread = spa->spa_condense_zthr;
8961 	if (condense_thread != NULL)
8962 		zthr_resume(condense_thread);
8963 
8964 	zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr;
8965 	if (raidz_expand_thread != NULL)
8966 		zthr_resume(raidz_expand_thread);
8967 
8968 	zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
8969 	if (discard_thread != NULL)
8970 		zthr_resume(discard_thread);
8971 
8972 	zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
8973 	if (ll_delete_thread != NULL)
8974 		zthr_resume(ll_delete_thread);
8975 
8976 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
8977 	if (ll_condense_thread != NULL)
8978 		zthr_resume(ll_condense_thread);
8979 }
8980 
8981 static boolean_t
8982 spa_async_tasks_pending(spa_t *spa)
8983 {
8984 	uint_t non_config_tasks;
8985 	uint_t config_task;
8986 	boolean_t config_task_suspended;
8987 
8988 	non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
8989 	config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
8990 	if (spa->spa_ccw_fail_time == 0) {
8991 		config_task_suspended = B_FALSE;
8992 	} else {
8993 		config_task_suspended =
8994 		    (gethrtime() - spa->spa_ccw_fail_time) <
8995 		    ((hrtime_t)zfs_ccw_retry_interval * NANOSEC);
8996 	}
8997 
8998 	return (non_config_tasks || (config_task && !config_task_suspended));
8999 }
9000 
9001 static void
9002 spa_async_dispatch(spa_t *spa)
9003 {
9004 	mutex_enter(&spa->spa_async_lock);
9005 	if (spa_async_tasks_pending(spa) &&
9006 	    !spa->spa_async_suspended &&
9007 	    spa->spa_async_thread == NULL)
9008 		spa->spa_async_thread = thread_create(NULL, 0,
9009 		    spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
9010 	mutex_exit(&spa->spa_async_lock);
9011 }
9012 
9013 void
9014 spa_async_request(spa_t *spa, int task)
9015 {
9016 	zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
9017 	mutex_enter(&spa->spa_async_lock);
9018 	spa->spa_async_tasks |= task;
9019 	mutex_exit(&spa->spa_async_lock);
9020 }
9021 
9022 int
9023 spa_async_tasks(spa_t *spa)
9024 {
9025 	return (spa->spa_async_tasks);
9026 }
9027 
9028 /*
9029  * ==========================================================================
9030  * SPA syncing routines
9031  * ==========================================================================
9032  */
9033 
9034 
9035 static int
9036 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
9037     dmu_tx_t *tx)
9038 {
9039 	bpobj_t *bpo = arg;
9040 	bpobj_enqueue(bpo, bp, bp_freed, tx);
9041 	return (0);
9042 }
9043 
9044 int
9045 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
9046 {
9047 	return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx));
9048 }
9049 
9050 int
9051 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
9052 {
9053 	return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx));
9054 }
9055 
9056 static int
9057 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
9058 {
9059 	zio_t *pio = arg;
9060 
9061 	zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp,
9062 	    pio->io_flags));
9063 	return (0);
9064 }
9065 
9066 static int
9067 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
9068     dmu_tx_t *tx)
9069 {
9070 	ASSERT(!bp_freed);
9071 	return (spa_free_sync_cb(arg, bp, tx));
9072 }
9073 
9074 /*
9075  * Note: this simple function is not inlined to make it easier to dtrace the
9076  * amount of time spent syncing frees.
9077  */
9078 static void
9079 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx)
9080 {
9081 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
9082 	bplist_iterate(bpl, spa_free_sync_cb, zio, tx);
9083 	VERIFY(zio_wait(zio) == 0);
9084 }
9085 
9086 /*
9087  * Note: this simple function is not inlined to make it easier to dtrace the
9088  * amount of time spent syncing deferred frees.
9089  */
9090 static void
9091 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx)
9092 {
9093 	if (spa_sync_pass(spa) != 1)
9094 		return;
9095 
9096 	/*
9097 	 * Note:
9098 	 * If the log space map feature is active, we stop deferring
9099 	 * frees to the next TXG and therefore running this function
9100 	 * would be considered a no-op as spa_deferred_bpobj should
9101 	 * not have any entries.
9102 	 *
9103 	 * That said we run this function anyway (instead of returning
9104 	 * immediately) for the edge-case scenario where we just
9105 	 * activated the log space map feature in this TXG but we have
9106 	 * deferred frees from the previous TXG.
9107 	 */
9108 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
9109 	VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj,
9110 	    bpobj_spa_free_sync_cb, zio, tx), ==, 0);
9111 	VERIFY0(zio_wait(zio));
9112 }
9113 
9114 static void
9115 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
9116 {
9117 	char *packed = NULL;
9118 	size_t bufsize;
9119 	size_t nvsize = 0;
9120 	dmu_buf_t *db;
9121 
9122 	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);
9123 
9124 	/*
9125 	 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration
9126 	 * information.  This avoids the dmu_buf_will_dirty() path and
9127 	 * saves us a pre-read to get data we don't actually care about.
9128 	 */
9129 	bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE);
9130 	packed = vmem_alloc(bufsize, KM_SLEEP);
9131 
9132 	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
9133 	    KM_SLEEP) == 0);
9134 	memset(packed + nvsize, 0, bufsize - nvsize);
9135 
9136 	dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx);
9137 
9138 	vmem_free(packed, bufsize);
9139 
9140 	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
9141 	dmu_buf_will_dirty(db, tx);
9142 	*(uint64_t *)db->db_data = nvsize;
9143 	dmu_buf_rele(db, FTAG);
9144 }
9145 
9146 static void
9147 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx,
9148     const char *config, const char *entry)
9149 {
9150 	nvlist_t *nvroot;
9151 	nvlist_t **list;
9152 	int i;
9153 
9154 	if (!sav->sav_sync)
9155 		return;
9156 
9157 	/*
9158 	 * Update the MOS nvlist describing the list of available devices.
9159 	 * spa_validate_aux() will have already made sure this nvlist is
9160 	 * valid and the vdevs are labeled appropriately.
9161 	 */
9162 	if (sav->sav_object == 0) {
9163 		sav->sav_object = dmu_object_alloc(spa->spa_meta_objset,
9164 		    DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE,
9165 		    sizeof (uint64_t), tx);
9166 		VERIFY(zap_update(spa->spa_meta_objset,
9167 		    DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1,
9168 		    &sav->sav_object, tx) == 0);
9169 	}
9170 
9171 	nvroot = fnvlist_alloc();
9172 	if (sav->sav_count == 0) {
9173 		fnvlist_add_nvlist_array(nvroot, config,
9174 		    (const nvlist_t * const *)NULL, 0);
9175 	} else {
9176 		list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP);
9177 		for (i = 0; i < sav->sav_count; i++)
9178 			list[i] = vdev_config_generate(spa, sav->sav_vdevs[i],
9179 			    B_FALSE, VDEV_CONFIG_L2CACHE);
9180 		fnvlist_add_nvlist_array(nvroot, config,
9181 		    (const nvlist_t * const *)list, sav->sav_count);
9182 		for (i = 0; i < sav->sav_count; i++)
9183 			nvlist_free(list[i]);
9184 		kmem_free(list, sav->sav_count * sizeof (void *));
9185 	}
9186 
9187 	spa_sync_nvlist(spa, sav->sav_object, nvroot, tx);
9188 	nvlist_free(nvroot);
9189 
9190 	sav->sav_sync = B_FALSE;
9191 }
9192 
9193 /*
9194  * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t.
9195  * The all-vdev ZAP must be empty.
9196  */
9197 static void
9198 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx)
9199 {
9200 	spa_t *spa = vd->vdev_spa;
9201 
9202 	if (vd->vdev_root_zap != 0 &&
9203 	    spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) {
9204 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
9205 		    vd->vdev_root_zap, tx));
9206 	}
9207 	if (vd->vdev_top_zap != 0) {
9208 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
9209 		    vd->vdev_top_zap, tx));
9210 	}
9211 	if (vd->vdev_leaf_zap != 0) {
9212 		VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
9213 		    vd->vdev_leaf_zap, tx));
9214 	}
9215 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
9216 		spa_avz_build(vd->vdev_child[i], avz, tx);
9217 	}
9218 }
9219 
9220 static void
9221 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx)
9222 {
9223 	nvlist_t *config;
9224 
9225 	/*
9226 	 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS,
9227 	 * its config may not be dirty but we still need to build per-vdev ZAPs.
9228 	 * Similarly, if the pool is being assembled (e.g. after a split), we
9229 	 * need to rebuild the AVZ although the config may not be dirty.
9230 	 */
9231 	if (list_is_empty(&spa->spa_config_dirty_list) &&
9232 	    spa->spa_avz_action == AVZ_ACTION_NONE)
9233 		return;
9234 
9235 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9236 
9237 	ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE ||
9238 	    spa->spa_avz_action == AVZ_ACTION_INITIALIZE ||
9239 	    spa->spa_all_vdev_zaps != 0);
9240 
9241 	if (spa->spa_avz_action == AVZ_ACTION_REBUILD) {
9242 		/* Make and build the new AVZ */
9243 		uint64_t new_avz = zap_create(spa->spa_meta_objset,
9244 		    DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx);
9245 		spa_avz_build(spa->spa_root_vdev, new_avz, tx);
9246 
9247 		/* Diff old AVZ with new one */
9248 		zap_cursor_t zc;
9249 		zap_attribute_t za;
9250 
9251 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
9252 		    spa->spa_all_vdev_zaps);
9253 		    zap_cursor_retrieve(&zc, &za) == 0;
9254 		    zap_cursor_advance(&zc)) {
9255 			uint64_t vdzap = za.za_first_integer;
9256 			if (zap_lookup_int(spa->spa_meta_objset, new_avz,
9257 			    vdzap) == ENOENT) {
9258 				/*
9259 				 * ZAP is listed in old AVZ but not in new one;
9260 				 * destroy it
9261 				 */
9262 				VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap,
9263 				    tx));
9264 			}
9265 		}
9266 
9267 		zap_cursor_fini(&zc);
9268 
9269 		/* Destroy the old AVZ */
9270 		VERIFY0(zap_destroy(spa->spa_meta_objset,
9271 		    spa->spa_all_vdev_zaps, tx));
9272 
9273 		/* Replace the old AVZ in the dir obj with the new one */
9274 		VERIFY0(zap_update(spa->spa_meta_objset,
9275 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP,
9276 		    sizeof (new_avz), 1, &new_avz, tx));
9277 
9278 		spa->spa_all_vdev_zaps = new_avz;
9279 	} else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) {
9280 		zap_cursor_t zc;
9281 		zap_attribute_t za;
9282 
9283 		/* Walk through the AVZ and destroy all listed ZAPs */
9284 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
9285 		    spa->spa_all_vdev_zaps);
9286 		    zap_cursor_retrieve(&zc, &za) == 0;
9287 		    zap_cursor_advance(&zc)) {
9288 			uint64_t zap = za.za_first_integer;
9289 			VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx));
9290 		}
9291 
9292 		zap_cursor_fini(&zc);
9293 
9294 		/* Destroy and unlink the AVZ itself */
9295 		VERIFY0(zap_destroy(spa->spa_meta_objset,
9296 		    spa->spa_all_vdev_zaps, tx));
9297 		VERIFY0(zap_remove(spa->spa_meta_objset,
9298 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx));
9299 		spa->spa_all_vdev_zaps = 0;
9300 	}
9301 
9302 	if (spa->spa_all_vdev_zaps == 0) {
9303 		spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset,
9304 		    DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
9305 		    DMU_POOL_VDEV_ZAP_MAP, tx);
9306 	}
9307 	spa->spa_avz_action = AVZ_ACTION_NONE;
9308 
9309 	/* Create ZAPs for vdevs that don't have them. */
9310 	vdev_construct_zaps(spa->spa_root_vdev, tx);
9311 
9312 	config = spa_config_generate(spa, spa->spa_root_vdev,
9313 	    dmu_tx_get_txg(tx), B_FALSE);
9314 
9315 	/*
9316 	 * If we're upgrading the spa version then make sure that
9317 	 * the config object gets updated with the correct version.
9318 	 */
9319 	if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version)
9320 		fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
9321 		    spa->spa_uberblock.ub_version);
9322 
9323 	spa_config_exit(spa, SCL_STATE, FTAG);
9324 
9325 	nvlist_free(spa->spa_config_syncing);
9326 	spa->spa_config_syncing = config;
9327 
9328 	spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
9329 }
9330 
9331 static void
9332 spa_sync_version(void *arg, dmu_tx_t *tx)
9333 {
9334 	uint64_t *versionp = arg;
9335 	uint64_t version = *versionp;
9336 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
9337 
9338 	/*
9339 	 * Setting the version is special cased when first creating the pool.
9340 	 */
9341 	ASSERT(tx->tx_txg != TXG_INITIAL);
9342 
9343 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
9344 	ASSERT(version >= spa_version(spa));
9345 
9346 	spa->spa_uberblock.ub_version = version;
9347 	vdev_config_dirty(spa->spa_root_vdev);
9348 	spa_history_log_internal(spa, "set", tx, "version=%lld",
9349 	    (longlong_t)version);
9350 }
9351 
9352 /*
9353  * Set zpool properties.
9354  */
9355 static void
9356 spa_sync_props(void *arg, dmu_tx_t *tx)
9357 {
9358 	nvlist_t *nvp = arg;
9359 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
9360 	objset_t *mos = spa->spa_meta_objset;
9361 	nvpair_t *elem = NULL;
9362 
9363 	mutex_enter(&spa->spa_props_lock);
9364 
9365 	while ((elem = nvlist_next_nvpair(nvp, elem))) {
9366 		uint64_t intval;
9367 		const char *strval, *fname;
9368 		zpool_prop_t prop;
9369 		const char *propname;
9370 		const char *elemname = nvpair_name(elem);
9371 		zprop_type_t proptype;
9372 		spa_feature_t fid;
9373 
9374 		switch (prop = zpool_name_to_prop(elemname)) {
9375 		case ZPOOL_PROP_VERSION:
9376 			intval = fnvpair_value_uint64(elem);
9377 			/*
9378 			 * The version is synced separately before other
9379 			 * properties and should be correct by now.
9380 			 */
9381 			ASSERT3U(spa_version(spa), >=, intval);
9382 			break;
9383 
9384 		case ZPOOL_PROP_ALTROOT:
9385 			/*
9386 			 * 'altroot' is a non-persistent property. It should
9387 			 * have been set temporarily at creation or import time.
9388 			 */
9389 			ASSERT(spa->spa_root != NULL);
9390 			break;
9391 
9392 		case ZPOOL_PROP_READONLY:
9393 		case ZPOOL_PROP_CACHEFILE:
9394 			/*
9395 			 * 'readonly' and 'cachefile' are also non-persistent
9396 			 * properties.
9397 			 */
9398 			break;
9399 		case ZPOOL_PROP_COMMENT:
9400 			strval = fnvpair_value_string(elem);
9401 			if (spa->spa_comment != NULL)
9402 				spa_strfree(spa->spa_comment);
9403 			spa->spa_comment = spa_strdup(strval);
9404 			/*
9405 			 * We need to dirty the configuration on all the vdevs
9406 			 * so that their labels get updated.  We also need to
9407 			 * update the cache file to keep it in sync with the
9408 			 * MOS version. It's unnecessary to do this for pool
9409 			 * creation since the vdev's configuration has already
9410 			 * been dirtied.
9411 			 */
9412 			if (tx->tx_txg != TXG_INITIAL) {
9413 				vdev_config_dirty(spa->spa_root_vdev);
9414 				spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
9415 			}
9416 			spa_history_log_internal(spa, "set", tx,
9417 			    "%s=%s", elemname, strval);
9418 			break;
9419 		case ZPOOL_PROP_COMPATIBILITY:
9420 			strval = fnvpair_value_string(elem);
9421 			if (spa->spa_compatibility != NULL)
9422 				spa_strfree(spa->spa_compatibility);
9423 			spa->spa_compatibility = spa_strdup(strval);
9424 			/*
9425 			 * Dirty the configuration on vdevs as above.
9426 			 */
9427 			if (tx->tx_txg != TXG_INITIAL) {
9428 				vdev_config_dirty(spa->spa_root_vdev);
9429 				spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
9430 			}
9431 
9432 			spa_history_log_internal(spa, "set", tx,
9433 			    "%s=%s", nvpair_name(elem), strval);
9434 			break;
9435 
9436 		case ZPOOL_PROP_INVAL:
9437 			if (zpool_prop_feature(elemname)) {
9438 				fname = strchr(elemname, '@') + 1;
9439 				VERIFY0(zfeature_lookup_name(fname, &fid));
9440 
9441 				spa_feature_enable(spa, fid, tx);
9442 				spa_history_log_internal(spa, "set", tx,
9443 				    "%s=enabled", elemname);
9444 				break;
9445 			} else if (!zfs_prop_user(elemname)) {
9446 				ASSERT(zpool_prop_feature(elemname));
9447 				break;
9448 			}
9449 			zfs_fallthrough;
9450 		default:
9451 			/*
9452 			 * Set pool property values in the poolprops mos object.
9453 			 */
9454 			if (spa->spa_pool_props_object == 0) {
9455 				spa->spa_pool_props_object =
9456 				    zap_create_link(mos, DMU_OT_POOL_PROPS,
9457 				    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS,
9458 				    tx);
9459 			}
9460 
9461 			/* normalize the property name */
9462 			if (prop == ZPOOL_PROP_INVAL) {
9463 				propname = elemname;
9464 				proptype = PROP_TYPE_STRING;
9465 			} else {
9466 				propname = zpool_prop_to_name(prop);
9467 				proptype = zpool_prop_get_type(prop);
9468 			}
9469 
9470 			if (nvpair_type(elem) == DATA_TYPE_STRING) {
9471 				ASSERT(proptype == PROP_TYPE_STRING);
9472 				strval = fnvpair_value_string(elem);
9473 				VERIFY0(zap_update(mos,
9474 				    spa->spa_pool_props_object, propname,
9475 				    1, strlen(strval) + 1, strval, tx));
9476 				spa_history_log_internal(spa, "set", tx,
9477 				    "%s=%s", elemname, strval);
9478 			} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
9479 				intval = fnvpair_value_uint64(elem);
9480 
9481 				if (proptype == PROP_TYPE_INDEX) {
9482 					const char *unused;
9483 					VERIFY0(zpool_prop_index_to_string(
9484 					    prop, intval, &unused));
9485 				}
9486 				VERIFY0(zap_update(mos,
9487 				    spa->spa_pool_props_object, propname,
9488 				    8, 1, &intval, tx));
9489 				spa_history_log_internal(spa, "set", tx,
9490 				    "%s=%lld", elemname,
9491 				    (longlong_t)intval);
9492 
9493 				switch (prop) {
9494 				case ZPOOL_PROP_DELEGATION:
9495 					spa->spa_delegation = intval;
9496 					break;
9497 				case ZPOOL_PROP_BOOTFS:
9498 					spa->spa_bootfs = intval;
9499 					break;
9500 				case ZPOOL_PROP_FAILUREMODE:
9501 					spa->spa_failmode = intval;
9502 					break;
9503 				case ZPOOL_PROP_AUTOTRIM:
9504 					spa->spa_autotrim = intval;
9505 					spa_async_request(spa,
9506 					    SPA_ASYNC_AUTOTRIM_RESTART);
9507 					break;
9508 				case ZPOOL_PROP_AUTOEXPAND:
9509 					spa->spa_autoexpand = intval;
9510 					if (tx->tx_txg != TXG_INITIAL)
9511 						spa_async_request(spa,
9512 						    SPA_ASYNC_AUTOEXPAND);
9513 					break;
9514 				case ZPOOL_PROP_MULTIHOST:
9515 					spa->spa_multihost = intval;
9516 					break;
9517 				default:
9518 					break;
9519 				}
9520 			} else {
9521 				ASSERT(0); /* not allowed */
9522 			}
9523 		}
9524 
9525 	}
9526 
9527 	mutex_exit(&spa->spa_props_lock);
9528 }
9529 
9530 /*
9531  * Perform one-time upgrade on-disk changes.  spa_version() does not
9532  * reflect the new version this txg, so there must be no changes this
9533  * txg to anything that the upgrade code depends on after it executes.
9534  * Therefore this must be called after dsl_pool_sync() does the sync
9535  * tasks.
9536  */
9537 static void
9538 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx)
9539 {
9540 	if (spa_sync_pass(spa) != 1)
9541 		return;
9542 
9543 	dsl_pool_t *dp = spa->spa_dsl_pool;
9544 	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
9545 
9546 	if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN &&
9547 	    spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) {
9548 		dsl_pool_create_origin(dp, tx);
9549 
9550 		/* Keeping the origin open increases spa_minref */
9551 		spa->spa_minref += 3;
9552 	}
9553 
9554 	if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES &&
9555 	    spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) {
9556 		dsl_pool_upgrade_clones(dp, tx);
9557 	}
9558 
9559 	if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES &&
9560 	    spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) {
9561 		dsl_pool_upgrade_dir_clones(dp, tx);
9562 
9563 		/* Keeping the freedir open increases spa_minref */
9564 		spa->spa_minref += 3;
9565 	}
9566 
9567 	if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES &&
9568 	    spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
9569 		spa_feature_create_zap_objects(spa, tx);
9570 	}
9571 
9572 	/*
9573 	 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable
9574 	 * when possibility to use lz4 compression for metadata was added
9575 	 * Old pools that have this feature enabled must be upgraded to have
9576 	 * this feature active
9577 	 */
9578 	if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
9579 		boolean_t lz4_en = spa_feature_is_enabled(spa,
9580 		    SPA_FEATURE_LZ4_COMPRESS);
9581 		boolean_t lz4_ac = spa_feature_is_active(spa,
9582 		    SPA_FEATURE_LZ4_COMPRESS);
9583 
9584 		if (lz4_en && !lz4_ac)
9585 			spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
9586 	}
9587 
9588 	/*
9589 	 * If we haven't written the salt, do so now.  Note that the
9590 	 * feature may not be activated yet, but that's fine since
9591 	 * the presence of this ZAP entry is backwards compatible.
9592 	 */
9593 	if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
9594 	    DMU_POOL_CHECKSUM_SALT) == ENOENT) {
9595 		VERIFY0(zap_add(spa->spa_meta_objset,
9596 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1,
9597 		    sizeof (spa->spa_cksum_salt.zcs_bytes),
9598 		    spa->spa_cksum_salt.zcs_bytes, tx));
9599 	}
9600 
9601 	rrw_exit(&dp->dp_config_rwlock, FTAG);
9602 }
9603 
9604 static void
9605 vdev_indirect_state_sync_verify(vdev_t *vd)
9606 {
9607 	vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping;
9608 	vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births;
9609 
9610 	if (vd->vdev_ops == &vdev_indirect_ops) {
9611 		ASSERT(vim != NULL);
9612 		ASSERT(vib != NULL);
9613 	}
9614 
9615 	uint64_t obsolete_sm_object = 0;
9616 	ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object));
9617 	if (obsolete_sm_object != 0) {
9618 		ASSERT(vd->vdev_obsolete_sm != NULL);
9619 		ASSERT(vd->vdev_removing ||
9620 		    vd->vdev_ops == &vdev_indirect_ops);
9621 		ASSERT(vdev_indirect_mapping_num_entries(vim) > 0);
9622 		ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0);
9623 		ASSERT3U(obsolete_sm_object, ==,
9624 		    space_map_object(vd->vdev_obsolete_sm));
9625 		ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=,
9626 		    space_map_allocated(vd->vdev_obsolete_sm));
9627 	}
9628 	ASSERT(vd->vdev_obsolete_segments != NULL);
9629 
9630 	/*
9631 	 * Since frees / remaps to an indirect vdev can only
9632 	 * happen in syncing context, the obsolete segments
9633 	 * tree must be empty when we start syncing.
9634 	 */
9635 	ASSERT0(range_tree_space(vd->vdev_obsolete_segments));
9636 }
9637 
9638 /*
9639  * Set the top-level vdev's max queue depth. Evaluate each top-level's
9640  * async write queue depth in case it changed. The max queue depth will
9641  * not change in the middle of syncing out this txg.
9642  */
9643 static void
9644 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa)
9645 {
9646 	ASSERT(spa_writeable(spa));
9647 
9648 	vdev_t *rvd = spa->spa_root_vdev;
9649 	uint32_t max_queue_depth = zfs_vdev_async_write_max_active *
9650 	    zfs_vdev_queue_depth_pct / 100;
9651 	metaslab_class_t *normal = spa_normal_class(spa);
9652 	metaslab_class_t *special = spa_special_class(spa);
9653 	metaslab_class_t *dedup = spa_dedup_class(spa);
9654 
9655 	uint64_t slots_per_allocator = 0;
9656 	for (int c = 0; c < rvd->vdev_children; c++) {
9657 		vdev_t *tvd = rvd->vdev_child[c];
9658 
9659 		metaslab_group_t *mg = tvd->vdev_mg;
9660 		if (mg == NULL || !metaslab_group_initialized(mg))
9661 			continue;
9662 
9663 		metaslab_class_t *mc = mg->mg_class;
9664 		if (mc != normal && mc != special && mc != dedup)
9665 			continue;
9666 
9667 		/*
9668 		 * It is safe to do a lock-free check here because only async
9669 		 * allocations look at mg_max_alloc_queue_depth, and async
9670 		 * allocations all happen from spa_sync().
9671 		 */
9672 		for (int i = 0; i < mg->mg_allocators; i++) {
9673 			ASSERT0(zfs_refcount_count(
9674 			    &(mg->mg_allocator[i].mga_alloc_queue_depth)));
9675 		}
9676 		mg->mg_max_alloc_queue_depth = max_queue_depth;
9677 
9678 		for (int i = 0; i < mg->mg_allocators; i++) {
9679 			mg->mg_allocator[i].mga_cur_max_alloc_queue_depth =
9680 			    zfs_vdev_def_queue_depth;
9681 		}
9682 		slots_per_allocator += zfs_vdev_def_queue_depth;
9683 	}
9684 
9685 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9686 		ASSERT0(zfs_refcount_count(&normal->mc_allocator[i].
9687 		    mca_alloc_slots));
9688 		ASSERT0(zfs_refcount_count(&special->mc_allocator[i].
9689 		    mca_alloc_slots));
9690 		ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i].
9691 		    mca_alloc_slots));
9692 		normal->mc_allocator[i].mca_alloc_max_slots =
9693 		    slots_per_allocator;
9694 		special->mc_allocator[i].mca_alloc_max_slots =
9695 		    slots_per_allocator;
9696 		dedup->mc_allocator[i].mca_alloc_max_slots =
9697 		    slots_per_allocator;
9698 	}
9699 	normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9700 	special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9701 	dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled;
9702 }
9703 
9704 static void
9705 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx)
9706 {
9707 	ASSERT(spa_writeable(spa));
9708 
9709 	vdev_t *rvd = spa->spa_root_vdev;
9710 	for (int c = 0; c < rvd->vdev_children; c++) {
9711 		vdev_t *vd = rvd->vdev_child[c];
9712 		vdev_indirect_state_sync_verify(vd);
9713 
9714 		if (vdev_indirect_should_condense(vd)) {
9715 			spa_condense_indirect_start_sync(vd, tx);
9716 			break;
9717 		}
9718 	}
9719 }
9720 
9721 static void
9722 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx)
9723 {
9724 	objset_t *mos = spa->spa_meta_objset;
9725 	dsl_pool_t *dp = spa->spa_dsl_pool;
9726 	uint64_t txg = tx->tx_txg;
9727 	bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK];
9728 
9729 	do {
9730 		int pass = ++spa->spa_sync_pass;
9731 
9732 		spa_sync_config_object(spa, tx);
9733 		spa_sync_aux_dev(spa, &spa->spa_spares, tx,
9734 		    ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES);
9735 		spa_sync_aux_dev(spa, &spa->spa_l2cache, tx,
9736 		    ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE);
9737 		spa_errlog_sync(spa, txg);
9738 		dsl_pool_sync(dp, txg);
9739 
9740 		if (pass < zfs_sync_pass_deferred_free ||
9741 		    spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) {
9742 			/*
9743 			 * If the log space map feature is active we don't
9744 			 * care about deferred frees and the deferred bpobj
9745 			 * as the log space map should effectively have the
9746 			 * same results (i.e. appending only to one object).
9747 			 */
9748 			spa_sync_frees(spa, free_bpl, tx);
9749 		} else {
9750 			/*
9751 			 * We can not defer frees in pass 1, because
9752 			 * we sync the deferred frees later in pass 1.
9753 			 */
9754 			ASSERT3U(pass, >, 1);
9755 			bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb,
9756 			    &spa->spa_deferred_bpobj, tx);
9757 		}
9758 
9759 		brt_sync(spa, txg);
9760 		ddt_sync(spa, txg);
9761 		dsl_scan_sync(dp, tx);
9762 		dsl_errorscrub_sync(dp, tx);
9763 		svr_sync(spa, tx);
9764 		spa_sync_upgrades(spa, tx);
9765 
9766 		spa_flush_metaslabs(spa, tx);
9767 
9768 		vdev_t *vd = NULL;
9769 		while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
9770 		    != NULL)
9771 			vdev_sync(vd, txg);
9772 
9773 		if (pass == 1) {
9774 			/*
9775 			 * dsl_pool_sync() -> dp_sync_tasks may have dirtied
9776 			 * the config. If that happens, this txg should not
9777 			 * be a no-op. So we must sync the config to the MOS
9778 			 * before checking for no-op.
9779 			 *
9780 			 * Note that when the config is dirty, it will
9781 			 * be written to the MOS (i.e. the MOS will be
9782 			 * dirtied) every time we call spa_sync_config_object()
9783 			 * in this txg.  Therefore we can't call this after
9784 			 * dsl_pool_sync() every pass, because it would
9785 			 * prevent us from converging, since we'd dirty
9786 			 * the MOS every pass.
9787 			 *
9788 			 * Sync tasks can only be processed in pass 1, so
9789 			 * there's no need to do this in later passes.
9790 			 */
9791 			spa_sync_config_object(spa, tx);
9792 		}
9793 
9794 		/*
9795 		 * Note: We need to check if the MOS is dirty because we could
9796 		 * have marked the MOS dirty without updating the uberblock
9797 		 * (e.g. if we have sync tasks but no dirty user data). We need
9798 		 * to check the uberblock's rootbp because it is updated if we
9799 		 * have synced out dirty data (though in this case the MOS will
9800 		 * most likely also be dirty due to second order effects, we
9801 		 * don't want to rely on that here).
9802 		 */
9803 		if (pass == 1 &&
9804 		    spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
9805 		    !dmu_objset_is_dirty(mos, txg)) {
9806 			/*
9807 			 * Nothing changed on the first pass, therefore this
9808 			 * TXG is a no-op. Avoid syncing deferred frees, so
9809 			 * that we can keep this TXG as a no-op.
9810 			 */
9811 			ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
9812 			ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
9813 			ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
9814 			ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg));
9815 			break;
9816 		}
9817 
9818 		spa_sync_deferred_frees(spa, tx);
9819 	} while (dmu_objset_is_dirty(mos, txg));
9820 }
9821 
9822 /*
9823  * Rewrite the vdev configuration (which includes the uberblock) to
9824  * commit the transaction group.
9825  *
9826  * If there are no dirty vdevs, we sync the uberblock to a few random
9827  * top-level vdevs that are known to be visible in the config cache
9828  * (see spa_vdev_add() for a complete description). If there *are* dirty
9829  * vdevs, sync the uberblock to all vdevs.
9830  */
9831 static void
9832 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx)
9833 {
9834 	vdev_t *rvd = spa->spa_root_vdev;
9835 	uint64_t txg = tx->tx_txg;
9836 
9837 	for (;;) {
9838 		int error = 0;
9839 
9840 		/*
9841 		 * We hold SCL_STATE to prevent vdev open/close/etc.
9842 		 * while we're attempting to write the vdev labels.
9843 		 */
9844 		spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9845 
9846 		if (list_is_empty(&spa->spa_config_dirty_list)) {
9847 			vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
9848 			int svdcount = 0;
9849 			int children = rvd->vdev_children;
9850 			int c0 = random_in_range(children);
9851 
9852 			for (int c = 0; c < children; c++) {
9853 				vdev_t *vd =
9854 				    rvd->vdev_child[(c0 + c) % children];
9855 
9856 				/* Stop when revisiting the first vdev */
9857 				if (c > 0 && svd[0] == vd)
9858 					break;
9859 
9860 				if (vd->vdev_ms_array == 0 ||
9861 				    vd->vdev_islog ||
9862 				    !vdev_is_concrete(vd))
9863 					continue;
9864 
9865 				svd[svdcount++] = vd;
9866 				if (svdcount == SPA_SYNC_MIN_VDEVS)
9867 					break;
9868 			}
9869 			error = vdev_config_sync(svd, svdcount, txg);
9870 		} else {
9871 			error = vdev_config_sync(rvd->vdev_child,
9872 			    rvd->vdev_children, txg);
9873 		}
9874 
9875 		if (error == 0)
9876 			spa->spa_last_synced_guid = rvd->vdev_guid;
9877 
9878 		spa_config_exit(spa, SCL_STATE, FTAG);
9879 
9880 		if (error == 0)
9881 			break;
9882 		zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR);
9883 		zio_resume_wait(spa);
9884 	}
9885 }
9886 
9887 /*
9888  * Sync the specified transaction group.  New blocks may be dirtied as
9889  * part of the process, so we iterate until it converges.
9890  */
9891 void
9892 spa_sync(spa_t *spa, uint64_t txg)
9893 {
9894 	vdev_t *vd = NULL;
9895 
9896 	VERIFY(spa_writeable(spa));
9897 
9898 	/*
9899 	 * Wait for i/os issued in open context that need to complete
9900 	 * before this txg syncs.
9901 	 */
9902 	(void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]);
9903 	spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL,
9904 	    ZIO_FLAG_CANFAIL);
9905 
9906 	/*
9907 	 * Now that there can be no more cloning in this transaction group,
9908 	 * but we are still before issuing frees, we can process pending BRT
9909 	 * updates.
9910 	 */
9911 	brt_pending_apply(spa, txg);
9912 
9913 	/*
9914 	 * Lock out configuration changes.
9915 	 */
9916 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
9917 
9918 	spa->spa_syncing_txg = txg;
9919 	spa->spa_sync_pass = 0;
9920 
9921 	for (int i = 0; i < spa->spa_alloc_count; i++) {
9922 		mutex_enter(&spa->spa_allocs[i].spaa_lock);
9923 		VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
9924 		mutex_exit(&spa->spa_allocs[i].spaa_lock);
9925 	}
9926 
9927 	/*
9928 	 * If there are any pending vdev state changes, convert them
9929 	 * into config changes that go out with this transaction group.
9930 	 */
9931 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
9932 	while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
9933 		/* Avoid holding the write lock unless actually necessary */
9934 		if (vd->vdev_aux == NULL) {
9935 			vdev_state_clean(vd);
9936 			vdev_config_dirty(vd);
9937 			continue;
9938 		}
9939 		/*
9940 		 * We need the write lock here because, for aux vdevs,
9941 		 * calling vdev_config_dirty() modifies sav_config.
9942 		 * This is ugly and will become unnecessary when we
9943 		 * eliminate the aux vdev wart by integrating all vdevs
9944 		 * into the root vdev tree.
9945 		 */
9946 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9947 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER);
9948 		while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
9949 			vdev_state_clean(vd);
9950 			vdev_config_dirty(vd);
9951 		}
9952 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
9953 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
9954 	}
9955 	spa_config_exit(spa, SCL_STATE, FTAG);
9956 
9957 	dsl_pool_t *dp = spa->spa_dsl_pool;
9958 	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);
9959 
9960 	spa->spa_sync_starttime = gethrtime();
9961 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
9962 	spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq,
9963 	    spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() +
9964 	    NSEC_TO_TICK(spa->spa_deadman_synctime));
9965 
9966 	/*
9967 	 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg,
9968 	 * set spa_deflate if we have no raid-z vdevs.
9969 	 */
9970 	if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE &&
9971 	    spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) {
9972 		vdev_t *rvd = spa->spa_root_vdev;
9973 
9974 		int i;
9975 		for (i = 0; i < rvd->vdev_children; i++) {
9976 			vd = rvd->vdev_child[i];
9977 			if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
9978 				break;
9979 		}
9980 		if (i == rvd->vdev_children) {
9981 			spa->spa_deflate = TRUE;
9982 			VERIFY0(zap_add(spa->spa_meta_objset,
9983 			    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
9984 			    sizeof (uint64_t), 1, &spa->spa_deflate, tx));
9985 		}
9986 	}
9987 
9988 	spa_sync_adjust_vdev_max_queue_depth(spa);
9989 
9990 	spa_sync_condense_indirect(spa, tx);
9991 
9992 	spa_sync_iterate_to_convergence(spa, tx);
9993 
9994 #ifdef ZFS_DEBUG
9995 	if (!list_is_empty(&spa->spa_config_dirty_list)) {
9996 	/*
9997 	 * Make sure that the number of ZAPs for all the vdevs matches
9998 	 * the number of ZAPs in the per-vdev ZAP list. This only gets
9999 	 * called if the config is dirty; otherwise there may be
10000 	 * outstanding AVZ operations that weren't completed in
10001 	 * spa_sync_config_object.
10002 	 */
10003 		uint64_t all_vdev_zap_entry_count;
10004 		ASSERT0(zap_count(spa->spa_meta_objset,
10005 		    spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count));
10006 		ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==,
10007 		    all_vdev_zap_entry_count);
10008 	}
10009 #endif
10010 
10011 	if (spa->spa_vdev_removal != NULL) {
10012 		ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]);
10013 	}
10014 
10015 	spa_sync_rewrite_vdev_config(spa, tx);
10016 	dmu_tx_commit(tx);
10017 
10018 	taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
10019 	spa->spa_deadman_tqid = 0;
10020 
10021 	/*
10022 	 * Clear the dirty config list.
10023 	 */
10024 	while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL)
10025 		vdev_config_clean(vd);
10026 
10027 	/*
10028 	 * Now that the new config has synced transactionally,
10029 	 * let it become visible to the config cache.
10030 	 */
10031 	if (spa->spa_config_syncing != NULL) {
10032 		spa_config_set(spa, spa->spa_config_syncing);
10033 		spa->spa_config_txg = txg;
10034 		spa->spa_config_syncing = NULL;
10035 	}
10036 
10037 	dsl_pool_sync_done(dp, txg);
10038 
10039 	for (int i = 0; i < spa->spa_alloc_count; i++) {
10040 		mutex_enter(&spa->spa_allocs[i].spaa_lock);
10041 		VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree));
10042 		mutex_exit(&spa->spa_allocs[i].spaa_lock);
10043 	}
10044 
10045 	/*
10046 	 * Update usable space statistics.
10047 	 */
10048 	while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
10049 	    != NULL)
10050 		vdev_sync_done(vd, txg);
10051 
10052 	metaslab_class_evict_old(spa->spa_normal_class, txg);
10053 	metaslab_class_evict_old(spa->spa_log_class, txg);
10054 
10055 	spa_sync_close_syncing_log_sm(spa);
10056 
10057 	spa_update_dspace(spa);
10058 
10059 	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON)
10060 		vdev_autotrim_kick(spa);
10061 
10062 	/*
10063 	 * It had better be the case that we didn't dirty anything
10064 	 * since vdev_config_sync().
10065 	 */
10066 	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
10067 	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
10068 	ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));
10069 
10070 	while (zfs_pause_spa_sync)
10071 		delay(1);
10072 
10073 	spa->spa_sync_pass = 0;
10074 
10075 	/*
10076 	 * Update the last synced uberblock here. We want to do this at
10077 	 * the end of spa_sync() so that consumers of spa_last_synced_txg()
10078 	 * will be guaranteed that all the processing associated with
10079 	 * that txg has been completed.
10080 	 */
10081 	spa->spa_ubsync = spa->spa_uberblock;
10082 	spa_config_exit(spa, SCL_CONFIG, FTAG);
10083 
10084 	spa_handle_ignored_writes(spa);
10085 
10086 	/*
10087 	 * If any async tasks have been requested, kick them off.
10088 	 */
10089 	spa_async_dispatch(spa);
10090 }
10091 
10092 /*
10093  * Sync all pools.  We don't want to hold the namespace lock across these
10094  * operations, so we take a reference on the spa_t and drop the lock during the
10095  * sync.
10096  */
10097 void
10098 spa_sync_allpools(void)
10099 {
10100 	spa_t *spa = NULL;
10101 	mutex_enter(&spa_namespace_lock);
10102 	while ((spa = spa_next(spa)) != NULL) {
10103 		if (spa_state(spa) != POOL_STATE_ACTIVE ||
10104 		    !spa_writeable(spa) || spa_suspended(spa))
10105 			continue;
10106 		spa_open_ref(spa, FTAG);
10107 		mutex_exit(&spa_namespace_lock);
10108 		txg_wait_synced(spa_get_dsl(spa), 0);
10109 		mutex_enter(&spa_namespace_lock);
10110 		spa_close(spa, FTAG);
10111 	}
10112 	mutex_exit(&spa_namespace_lock);
10113 }
10114 
10115 taskq_t *
10116 spa_sync_tq_create(spa_t *spa, const char *name)
10117 {
10118 	kthread_t **kthreads;
10119 
10120 	ASSERT(spa->spa_sync_tq == NULL);
10121 	ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus);
10122 
10123 	/*
10124 	 * - do not allow more allocators than cpus.
10125 	 * - there may be more cpus than allocators.
10126 	 * - do not allow more sync taskq threads than allocators or cpus.
10127 	 */
10128 	int nthreads = spa->spa_alloc_count;
10129 	spa->spa_syncthreads = kmem_zalloc(sizeof (spa_syncthread_info_t) *
10130 	    nthreads, KM_SLEEP);
10131 
10132 	spa->spa_sync_tq = taskq_create_synced(name, nthreads, minclsyspri,
10133 	    nthreads, INT_MAX, TASKQ_PREPOPULATE, &kthreads);
10134 	VERIFY(spa->spa_sync_tq != NULL);
10135 	VERIFY(kthreads != NULL);
10136 
10137 	spa_taskqs_t *tqs =
10138 	    &spa->spa_zio_taskq[ZIO_TYPE_WRITE][ZIO_TASKQ_ISSUE];
10139 
10140 	spa_syncthread_info_t *ti = spa->spa_syncthreads;
10141 	for (int i = 0, w = 0; i < nthreads; i++, w++, ti++) {
10142 		ti->sti_thread = kthreads[i];
10143 		if (w == tqs->stqs_count) {
10144 			w = 0;
10145 		}
10146 		ti->sti_wr_iss_tq = tqs->stqs_taskq[w];
10147 	}
10148 
10149 	kmem_free(kthreads, sizeof (*kthreads) * nthreads);
10150 	return (spa->spa_sync_tq);
10151 }
10152 
10153 void
10154 spa_sync_tq_destroy(spa_t *spa)
10155 {
10156 	ASSERT(spa->spa_sync_tq != NULL);
10157 
10158 	taskq_wait(spa->spa_sync_tq);
10159 	taskq_destroy(spa->spa_sync_tq);
10160 	kmem_free(spa->spa_syncthreads,
10161 	    sizeof (spa_syncthread_info_t) * spa->spa_alloc_count);
10162 	spa->spa_sync_tq = NULL;
10163 }
10164 
10165 void
10166 spa_select_allocator(zio_t *zio)
10167 {
10168 	zbookmark_phys_t *bm = &zio->io_bookmark;
10169 	spa_t *spa = zio->io_spa;
10170 
10171 	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
10172 
10173 	/*
10174 	 * A gang block (for example) may have inherited its parent's
10175 	 * allocator, in which case there is nothing further to do here.
10176 	 */
10177 	if (ZIO_HAS_ALLOCATOR(zio))
10178 		return;
10179 
10180 	ASSERT(spa != NULL);
10181 	ASSERT(bm != NULL);
10182 
10183 	/*
10184 	 * First try to use an allocator assigned to the syncthread, and set
10185 	 * the corresponding write issue taskq for the allocator.
10186 	 * Note, we must have an open pool to do this.
10187 	 */
10188 	if (spa->spa_sync_tq != NULL) {
10189 		spa_syncthread_info_t *ti = spa->spa_syncthreads;
10190 		for (int i = 0; i < spa->spa_alloc_count; i++, ti++) {
10191 			if (ti->sti_thread == curthread) {
10192 				zio->io_allocator = i;
10193 				zio->io_wr_iss_tq = ti->sti_wr_iss_tq;
10194 				return;
10195 			}
10196 		}
10197 	}
10198 
10199 	/*
10200 	 * We want to try to use as many allocators as possible to help improve
10201 	 * performance, but we also want logically adjacent IOs to be physically
10202 	 * adjacent to improve sequential read performance. We chunk each object
10203 	 * into 2^20 block regions, and then hash based on the objset, object,
10204 	 * level, and region to accomplish both of these goals.
10205 	 */
10206 	uint64_t hv = cityhash4(bm->zb_objset, bm->zb_object, bm->zb_level,
10207 	    bm->zb_blkid >> 20);
10208 
10209 	zio->io_allocator = (uint_t)hv % spa->spa_alloc_count;
10210 	zio->io_wr_iss_tq = NULL;
10211 }
10212 
10213 /*
10214  * ==========================================================================
10215  * Miscellaneous routines
10216  * ==========================================================================
10217  */
10218 
10219 /*
10220  * Remove all pools in the system.
10221  */
10222 void
10223 spa_evict_all(void)
10224 {
10225 	spa_t *spa;
10226 
10227 	/*
10228 	 * Remove all cached state.  All pools should be closed now,
10229 	 * so every spa in the AVL tree should be unreferenced.
10230 	 */
10231 	mutex_enter(&spa_namespace_lock);
10232 	while ((spa = spa_next(NULL)) != NULL) {
10233 		/*
10234 		 * Stop async tasks.  The async thread may need to detach
10235 		 * a device that's been replaced, which requires grabbing
10236 		 * spa_namespace_lock, so we must drop it here.
10237 		 */
10238 		spa_open_ref(spa, FTAG);
10239 		mutex_exit(&spa_namespace_lock);
10240 		spa_async_suspend(spa);
10241 		mutex_enter(&spa_namespace_lock);
10242 		spa_close(spa, FTAG);
10243 
10244 		if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
10245 			spa_unload(spa);
10246 			spa_deactivate(spa);
10247 		}
10248 		spa_remove(spa);
10249 	}
10250 	mutex_exit(&spa_namespace_lock);
10251 }
10252 
10253 vdev_t *
10254 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux)
10255 {
10256 	vdev_t *vd;
10257 	int i;
10258 
10259 	if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL)
10260 		return (vd);
10261 
10262 	if (aux) {
10263 		for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
10264 			vd = spa->spa_l2cache.sav_vdevs[i];
10265 			if (vd->vdev_guid == guid)
10266 				return (vd);
10267 		}
10268 
10269 		for (i = 0; i < spa->spa_spares.sav_count; i++) {
10270 			vd = spa->spa_spares.sav_vdevs[i];
10271 			if (vd->vdev_guid == guid)
10272 				return (vd);
10273 		}
10274 	}
10275 
10276 	return (NULL);
10277 }
10278 
10279 void
10280 spa_upgrade(spa_t *spa, uint64_t version)
10281 {
10282 	ASSERT(spa_writeable(spa));
10283 
10284 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
10285 
10286 	/*
10287 	 * This should only be called for a non-faulted pool, and since a
10288 	 * future version would result in an unopenable pool, this shouldn't be
10289 	 * possible.
10290 	 */
10291 	ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version));
10292 	ASSERT3U(version, >=, spa->spa_uberblock.ub_version);
10293 
10294 	spa->spa_uberblock.ub_version = version;
10295 	vdev_config_dirty(spa->spa_root_vdev);
10296 
10297 	spa_config_exit(spa, SCL_ALL, FTAG);
10298 
10299 	txg_wait_synced(spa_get_dsl(spa), 0);
10300 }
10301 
10302 static boolean_t
10303 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav)
10304 {
10305 	(void) spa;
10306 	int i;
10307 	uint64_t vdev_guid;
10308 
10309 	for (i = 0; i < sav->sav_count; i++)
10310 		if (sav->sav_vdevs[i]->vdev_guid == guid)
10311 			return (B_TRUE);
10312 
10313 	for (i = 0; i < sav->sav_npending; i++) {
10314 		if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID,
10315 		    &vdev_guid) == 0 && vdev_guid == guid)
10316 			return (B_TRUE);
10317 	}
10318 
10319 	return (B_FALSE);
10320 }
10321 
10322 boolean_t
10323 spa_has_l2cache(spa_t *spa, uint64_t guid)
10324 {
10325 	return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache));
10326 }
10327 
10328 boolean_t
10329 spa_has_spare(spa_t *spa, uint64_t guid)
10330 {
10331 	return (spa_has_aux_vdev(spa, guid, &spa->spa_spares));
10332 }
10333 
10334 /*
10335  * Check if a pool has an active shared spare device.
10336  * Note: reference count of an active spare is 2, as a spare and as a replace
10337  */
10338 static boolean_t
10339 spa_has_active_shared_spare(spa_t *spa)
10340 {
10341 	int i, refcnt;
10342 	uint64_t pool;
10343 	spa_aux_vdev_t *sav = &spa->spa_spares;
10344 
10345 	for (i = 0; i < sav->sav_count; i++) {
10346 		if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool,
10347 		    &refcnt) && pool != 0ULL && pool == spa_guid(spa) &&
10348 		    refcnt > 2)
10349 			return (B_TRUE);
10350 	}
10351 
10352 	return (B_FALSE);
10353 }
10354 
10355 uint64_t
10356 spa_total_metaslabs(spa_t *spa)
10357 {
10358 	vdev_t *rvd = spa->spa_root_vdev;
10359 
10360 	uint64_t m = 0;
10361 	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
10362 		vdev_t *vd = rvd->vdev_child[c];
10363 		if (!vdev_is_concrete(vd))
10364 			continue;
10365 		m += vd->vdev_ms_count;
10366 	}
10367 	return (m);
10368 }
10369 
10370 /*
10371  * Notify any waiting threads that some activity has switched from being in-
10372  * progress to not-in-progress so that the thread can wake up and determine
10373  * whether it is finished waiting.
10374  */
10375 void
10376 spa_notify_waiters(spa_t *spa)
10377 {
10378 	/*
10379 	 * Acquiring spa_activities_lock here prevents the cv_broadcast from
10380 	 * happening between the waiting thread's check and cv_wait.
10381 	 */
10382 	mutex_enter(&spa->spa_activities_lock);
10383 	cv_broadcast(&spa->spa_activities_cv);
10384 	mutex_exit(&spa->spa_activities_lock);
10385 }
10386 
10387 /*
10388  * Notify any waiting threads that the pool is exporting, and then block until
10389  * they are finished using the spa_t.
10390  */
10391 void
10392 spa_wake_waiters(spa_t *spa)
10393 {
10394 	mutex_enter(&spa->spa_activities_lock);
10395 	spa->spa_waiters_cancel = B_TRUE;
10396 	cv_broadcast(&spa->spa_activities_cv);
10397 	while (spa->spa_waiters != 0)
10398 		cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock);
10399 	spa->spa_waiters_cancel = B_FALSE;
10400 	mutex_exit(&spa->spa_activities_lock);
10401 }
10402 
10403 /* Whether the vdev or any of its descendants are being initialized/trimmed. */
10404 static boolean_t
10405 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity)
10406 {
10407 	spa_t *spa = vd->vdev_spa;
10408 
10409 	ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER));
10410 	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
10411 	ASSERT(activity == ZPOOL_WAIT_INITIALIZE ||
10412 	    activity == ZPOOL_WAIT_TRIM);
10413 
10414 	kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ?
10415 	    &vd->vdev_initialize_lock : &vd->vdev_trim_lock;
10416 
10417 	mutex_exit(&spa->spa_activities_lock);
10418 	mutex_enter(lock);
10419 	mutex_enter(&spa->spa_activities_lock);
10420 
10421 	boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ?
10422 	    (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) :
10423 	    (vd->vdev_trim_state == VDEV_TRIM_ACTIVE);
10424 	mutex_exit(lock);
10425 
10426 	if (in_progress)
10427 		return (B_TRUE);
10428 
10429 	for (int i = 0; i < vd->vdev_children; i++) {
10430 		if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i],
10431 		    activity))
10432 			return (B_TRUE);
10433 	}
10434 
10435 	return (B_FALSE);
10436 }
10437 
10438 /*
10439  * If use_guid is true, this checks whether the vdev specified by guid is
10440  * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool
10441  * is being initialized/trimmed. The caller must hold the config lock and
10442  * spa_activities_lock.
10443  */
10444 static int
10445 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid,
10446     zpool_wait_activity_t activity, boolean_t *in_progress)
10447 {
10448 	mutex_exit(&spa->spa_activities_lock);
10449 	spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
10450 	mutex_enter(&spa->spa_activities_lock);
10451 
10452 	vdev_t *vd;
10453 	if (use_guid) {
10454 		vd = spa_lookup_by_guid(spa, guid, B_FALSE);
10455 		if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) {
10456 			spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
10457 			return (EINVAL);
10458 		}
10459 	} else {
10460 		vd = spa->spa_root_vdev;
10461 	}
10462 
10463 	*in_progress = spa_vdev_activity_in_progress_impl(vd, activity);
10464 
10465 	spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
10466 	return (0);
10467 }
10468 
10469 /*
10470  * Locking for waiting threads
10471  * ---------------------------
10472  *
10473  * Waiting threads need a way to check whether a given activity is in progress,
10474  * and then, if it is, wait for it to complete. Each activity will have some
10475  * in-memory representation of the relevant on-disk state which can be used to
10476  * determine whether or not the activity is in progress. The in-memory state and
10477  * the locking used to protect it will be different for each activity, and may
10478  * not be suitable for use with a cvar (e.g., some state is protected by the
10479  * config lock). To allow waiting threads to wait without any races, another
10480  * lock, spa_activities_lock, is used.
10481  *
10482  * When the state is checked, both the activity-specific lock (if there is one)
10483  * and spa_activities_lock are held. In some cases, the activity-specific lock
10484  * is acquired explicitly (e.g. the config lock). In others, the locking is
10485  * internal to some check (e.g. bpobj_is_empty). After checking, the waiting
10486  * thread releases the activity-specific lock and, if the activity is in
10487  * progress, then cv_waits using spa_activities_lock.
10488  *
10489  * The waiting thread is woken when another thread, one completing some
10490  * activity, updates the state of the activity and then calls
10491  * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only
10492  * needs to hold its activity-specific lock when updating the state, and this
10493  * lock can (but doesn't have to) be dropped before calling spa_notify_waiters.
10494  *
10495  * Because spa_notify_waiters acquires spa_activities_lock before broadcasting,
10496  * and because it is held when the waiting thread checks the state of the
10497  * activity, it can never be the case that the completing thread both updates
10498  * the activity state and cv_broadcasts in between the waiting thread's check
10499  * and cv_wait. Thus, a waiting thread can never miss a wakeup.
10500  *
10501  * In order to prevent deadlock, when the waiting thread does its check, in some
10502  * cases it will temporarily drop spa_activities_lock in order to acquire the
10503  * activity-specific lock. The order in which spa_activities_lock and the
10504  * activity specific lock are acquired in the waiting thread is determined by
10505  * the order in which they are acquired in the completing thread; if the
10506  * completing thread calls spa_notify_waiters with the activity-specific lock
10507  * held, then the waiting thread must also acquire the activity-specific lock
10508  * first.
10509  */
10510 
10511 static int
10512 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity,
10513     boolean_t use_tag, uint64_t tag, boolean_t *in_progress)
10514 {
10515 	int error = 0;
10516 
10517 	ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
10518 
10519 	switch (activity) {
10520 	case ZPOOL_WAIT_CKPT_DISCARD:
10521 		*in_progress =
10522 		    (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) &&
10523 		    zap_contains(spa_meta_objset(spa),
10524 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) ==
10525 		    ENOENT);
10526 		break;
10527 	case ZPOOL_WAIT_FREE:
10528 		*in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS &&
10529 		    !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) ||
10530 		    spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) ||
10531 		    spa_livelist_delete_check(spa));
10532 		break;
10533 	case ZPOOL_WAIT_INITIALIZE:
10534 	case ZPOOL_WAIT_TRIM:
10535 		error = spa_vdev_activity_in_progress(spa, use_tag, tag,
10536 		    activity, in_progress);
10537 		break;
10538 	case ZPOOL_WAIT_REPLACE:
10539 		mutex_exit(&spa->spa_activities_lock);
10540 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
10541 		mutex_enter(&spa->spa_activities_lock);
10542 
10543 		*in_progress = vdev_replace_in_progress(spa->spa_root_vdev);
10544 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
10545 		break;
10546 	case ZPOOL_WAIT_REMOVE:
10547 		*in_progress = (spa->spa_removing_phys.sr_state ==
10548 		    DSS_SCANNING);
10549 		break;
10550 	case ZPOOL_WAIT_RESILVER:
10551 		*in_progress = vdev_rebuild_active(spa->spa_root_vdev);
10552 		if (*in_progress)
10553 			break;
10554 		zfs_fallthrough;
10555 	case ZPOOL_WAIT_SCRUB:
10556 	{
10557 		boolean_t scanning, paused, is_scrub;
10558 		dsl_scan_t *scn =  spa->spa_dsl_pool->dp_scan;
10559 
10560 		is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB);
10561 		scanning = (scn->scn_phys.scn_state == DSS_SCANNING);
10562 		paused = dsl_scan_is_paused_scrub(scn);
10563 		*in_progress = (scanning && !paused &&
10564 		    is_scrub == (activity == ZPOOL_WAIT_SCRUB));
10565 		break;
10566 	}
10567 	case ZPOOL_WAIT_RAIDZ_EXPAND:
10568 	{
10569 		vdev_raidz_expand_t *vre = spa->spa_raidz_expand;
10570 		*in_progress = (vre != NULL && vre->vre_state == DSS_SCANNING);
10571 		break;
10572 	}
10573 	default:
10574 		panic("unrecognized value for activity %d", activity);
10575 	}
10576 
10577 	return (error);
10578 }
10579 
10580 static int
10581 spa_wait_common(const char *pool, zpool_wait_activity_t activity,
10582     boolean_t use_tag, uint64_t tag, boolean_t *waited)
10583 {
10584 	/*
10585 	 * The tag is used to distinguish between instances of an activity.
10586 	 * 'initialize' and 'trim' are the only activities that we use this for.
10587 	 * The other activities can only have a single instance in progress in a
10588 	 * pool at one time, making the tag unnecessary.
10589 	 *
10590 	 * There can be multiple devices being replaced at once, but since they
10591 	 * all finish once resilvering finishes, we don't bother keeping track
10592 	 * of them individually, we just wait for them all to finish.
10593 	 */
10594 	if (use_tag && activity != ZPOOL_WAIT_INITIALIZE &&
10595 	    activity != ZPOOL_WAIT_TRIM)
10596 		return (EINVAL);
10597 
10598 	if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES)
10599 		return (EINVAL);
10600 
10601 	spa_t *spa;
10602 	int error = spa_open(pool, &spa, FTAG);
10603 	if (error != 0)
10604 		return (error);
10605 
10606 	/*
10607 	 * Increment the spa's waiter count so that we can call spa_close and
10608 	 * still ensure that the spa_t doesn't get freed before this thread is
10609 	 * finished with it when the pool is exported. We want to call spa_close
10610 	 * before we start waiting because otherwise the additional ref would
10611 	 * prevent the pool from being exported or destroyed throughout the
10612 	 * potentially long wait.
10613 	 */
10614 	mutex_enter(&spa->spa_activities_lock);
10615 	spa->spa_waiters++;
10616 	spa_close(spa, FTAG);
10617 
10618 	*waited = B_FALSE;
10619 	for (;;) {
10620 		boolean_t in_progress;
10621 		error = spa_activity_in_progress(spa, activity, use_tag, tag,
10622 		    &in_progress);
10623 
10624 		if (error || !in_progress || spa->spa_waiters_cancel)
10625 			break;
10626 
10627 		*waited = B_TRUE;
10628 
10629 		if (cv_wait_sig(&spa->spa_activities_cv,
10630 		    &spa->spa_activities_lock) == 0) {
10631 			error = EINTR;
10632 			break;
10633 		}
10634 	}
10635 
10636 	spa->spa_waiters--;
10637 	cv_signal(&spa->spa_waiters_cv);
10638 	mutex_exit(&spa->spa_activities_lock);
10639 
10640 	return (error);
10641 }
10642 
10643 /*
10644  * Wait for a particular instance of the specified activity to complete, where
10645  * the instance is identified by 'tag'
10646  */
10647 int
10648 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag,
10649     boolean_t *waited)
10650 {
10651 	return (spa_wait_common(pool, activity, B_TRUE, tag, waited));
10652 }
10653 
10654 /*
10655  * Wait for all instances of the specified activity complete
10656  */
10657 int
10658 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited)
10659 {
10660 
10661 	return (spa_wait_common(pool, activity, B_FALSE, 0, waited));
10662 }
10663 
10664 sysevent_t *
10665 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name)
10666 {
10667 	sysevent_t *ev = NULL;
10668 #ifdef _KERNEL
10669 	nvlist_t *resource;
10670 
10671 	resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl);
10672 	if (resource) {
10673 		ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP);
10674 		ev->resource = resource;
10675 	}
10676 #else
10677 	(void) spa, (void) vd, (void) hist_nvl, (void) name;
10678 #endif
10679 	return (ev);
10680 }
10681 
10682 void
10683 spa_event_post(sysevent_t *ev)
10684 {
10685 #ifdef _KERNEL
10686 	if (ev) {
10687 		zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb);
10688 		kmem_free(ev, sizeof (*ev));
10689 	}
10690 #else
10691 	(void) ev;
10692 #endif
10693 }
10694 
10695 /*
10696  * Post a zevent corresponding to the given sysevent.   The 'name' must be one
10697  * of the event definitions in sys/sysevent/eventdefs.h.  The payload will be
10698  * filled in from the spa and (optionally) the vdev.  This doesn't do anything
10699  * in the userland libzpool, as we don't want consumers to misinterpret ztest
10700  * or zdb as real changes.
10701  */
10702 void
10703 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name)
10704 {
10705 	spa_event_post(spa_event_create(spa, vd, hist_nvl, name));
10706 }
10707 
10708 /* state manipulation functions */
10709 EXPORT_SYMBOL(spa_open);
10710 EXPORT_SYMBOL(spa_open_rewind);
10711 EXPORT_SYMBOL(spa_get_stats);
10712 EXPORT_SYMBOL(spa_create);
10713 EXPORT_SYMBOL(spa_import);
10714 EXPORT_SYMBOL(spa_tryimport);
10715 EXPORT_SYMBOL(spa_destroy);
10716 EXPORT_SYMBOL(spa_export);
10717 EXPORT_SYMBOL(spa_reset);
10718 EXPORT_SYMBOL(spa_async_request);
10719 EXPORT_SYMBOL(spa_async_suspend);
10720 EXPORT_SYMBOL(spa_async_resume);
10721 EXPORT_SYMBOL(spa_inject_addref);
10722 EXPORT_SYMBOL(spa_inject_delref);
10723 EXPORT_SYMBOL(spa_scan_stat_init);
10724 EXPORT_SYMBOL(spa_scan_get_stats);
10725 
10726 /* device manipulation */
10727 EXPORT_SYMBOL(spa_vdev_add);
10728 EXPORT_SYMBOL(spa_vdev_attach);
10729 EXPORT_SYMBOL(spa_vdev_detach);
10730 EXPORT_SYMBOL(spa_vdev_setpath);
10731 EXPORT_SYMBOL(spa_vdev_setfru);
10732 EXPORT_SYMBOL(spa_vdev_split_mirror);
10733 
10734 /* spare statech is global across all pools) */
10735 EXPORT_SYMBOL(spa_spare_add);
10736 EXPORT_SYMBOL(spa_spare_remove);
10737 EXPORT_SYMBOL(spa_spare_exists);
10738 EXPORT_SYMBOL(spa_spare_activate);
10739 
10740 /* L2ARC statech is global across all pools) */
10741 EXPORT_SYMBOL(spa_l2cache_add);
10742 EXPORT_SYMBOL(spa_l2cache_remove);
10743 EXPORT_SYMBOL(spa_l2cache_exists);
10744 EXPORT_SYMBOL(spa_l2cache_activate);
10745 EXPORT_SYMBOL(spa_l2cache_drop);
10746 
10747 /* scanning */
10748 EXPORT_SYMBOL(spa_scan);
10749 EXPORT_SYMBOL(spa_scan_stop);
10750 
10751 /* spa syncing */
10752 EXPORT_SYMBOL(spa_sync); /* only for DMU use */
10753 EXPORT_SYMBOL(spa_sync_allpools);
10754 
10755 /* properties */
10756 EXPORT_SYMBOL(spa_prop_set);
10757 EXPORT_SYMBOL(spa_prop_get);
10758 EXPORT_SYMBOL(spa_prop_clear_bootfs);
10759 
10760 /* asynchronous event notification */
10761 EXPORT_SYMBOL(spa_event_notify);
10762 
10763 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_pct, UINT, ZMOD_RW,
10764 	"Percentage of CPUs to run a metaslab preload taskq");
10765 
10766 /* BEGIN CSTYLED */
10767 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW,
10768 	"log2 fraction of arc that can be used by inflight I/Os when "
10769 	"verifying pool during import");
10770 /* END CSTYLED */
10771 
10772 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW,
10773 	"Set to traverse metadata on pool import");
10774 
10775 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW,
10776 	"Set to traverse data on pool import");
10777 
10778 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW,
10779 	"Print vdev tree to zfs_dbgmsg during pool import");
10780 
10781 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD,
10782 	"Percentage of CPUs to run an IO worker thread");
10783 
10784 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD,
10785 	"Number of threads per IO worker taskqueue");
10786 
10787 /* BEGIN CSTYLED */
10788 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW,
10789 	"Allow importing pool with up to this number of missing top-level "
10790 	"vdevs (in read-only mode)");
10791 /* END CSTYLED */
10792 
10793 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT,
10794 	ZMOD_RW, "Set the livelist condense zthr to pause");
10795 
10796 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT,
10797 	ZMOD_RW, "Set the livelist condense synctask to pause");
10798 
10799 /* BEGIN CSTYLED */
10800 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel,
10801 	INT, ZMOD_RW,
10802 	"Whether livelist condensing was canceled in the synctask");
10803 
10804 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel,
10805 	INT, ZMOD_RW,
10806 	"Whether livelist condensing was canceled in the zthr function");
10807 
10808 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT,
10809 	ZMOD_RW,
10810 	"Whether extra ALLOC blkptrs were added to a livelist entry while it "
10811 	"was being condensed");
10812 
10813 #ifdef _KERNEL
10814 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_read,
10815 	spa_taskq_read_param_set, spa_taskq_read_param_get, ZMOD_RD,
10816 	"Configure IO queues for read IO");
10817 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_write,
10818 	spa_taskq_write_param_set, spa_taskq_write_param_get, ZMOD_RD,
10819 	"Configure IO queues for write IO");
10820 #endif
10821 /* END CSTYLED */
10822 
10823 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_wr_iss_ncpus, UINT, ZMOD_RW,
10824 	"Number of CPUs to run write issue taskqs");
10825