xref: /freebsd/sys/contrib/openzfs/module/zfs/vdev.c (revision 87b759f0fa1f7554d50ce640c40138512bbded44)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2021 by Delphix. All rights reserved.
25  * Copyright 2017 Nexenta Systems, Inc.
26  * Copyright (c) 2014 Integros [integros.com]
27  * Copyright 2016 Toomas Soome <tsoome@me.com>
28  * Copyright 2017 Joyent, Inc.
29  * Copyright (c) 2017, Intel Corporation.
30  * Copyright (c) 2019, Datto Inc. All rights reserved.
31  * Copyright (c) 2021, Klara Inc.
32  * Copyright (c) 2021, 2023 Hewlett Packard Enterprise Development LP.
33  */
34 
35 #include <sys/zfs_context.h>
36 #include <sys/fm/fs/zfs.h>
37 #include <sys/spa.h>
38 #include <sys/spa_impl.h>
39 #include <sys/bpobj.h>
40 #include <sys/dmu.h>
41 #include <sys/dmu_tx.h>
42 #include <sys/dsl_dir.h>
43 #include <sys/vdev_impl.h>
44 #include <sys/vdev_rebuild.h>
45 #include <sys/vdev_draid.h>
46 #include <sys/uberblock_impl.h>
47 #include <sys/metaslab.h>
48 #include <sys/metaslab_impl.h>
49 #include <sys/space_map.h>
50 #include <sys/space_reftree.h>
51 #include <sys/zio.h>
52 #include <sys/zap.h>
53 #include <sys/fs/zfs.h>
54 #include <sys/arc.h>
55 #include <sys/zil.h>
56 #include <sys/dsl_scan.h>
57 #include <sys/vdev_raidz.h>
58 #include <sys/abd.h>
59 #include <sys/vdev_initialize.h>
60 #include <sys/vdev_trim.h>
61 #include <sys/vdev_raidz.h>
62 #include <sys/zvol.h>
63 #include <sys/zfs_ratelimit.h>
64 #include "zfs_prop.h"
65 
66 /*
67  * One metaslab from each (normal-class) vdev is used by the ZIL.  These are
68  * called "embedded slog metaslabs", are referenced by vdev_log_mg, and are
69  * part of the spa_embedded_log_class.  The metaslab with the most free space
70  * in each vdev is selected for this purpose when the pool is opened (or a
71  * vdev is added).  See vdev_metaslab_init().
72  *
73  * Log blocks can be allocated from the following locations.  Each one is tried
74  * in order until the allocation succeeds:
75  * 1. dedicated log vdevs, aka "slog" (spa_log_class)
76  * 2. embedded slog metaslabs (spa_embedded_log_class)
77  * 3. other metaslabs in normal vdevs (spa_normal_class)
78  *
79  * zfs_embedded_slog_min_ms disables the embedded slog if there are fewer
80  * than this number of metaslabs in the vdev.  This ensures that we don't set
81  * aside an unreasonable amount of space for the ZIL.  If set to less than
82  * 1 << (spa_slop_shift + 1), on small pools the usable space may be reduced
83  * (by more than 1<<spa_slop_shift) due to the embedded slog metaslab.
84  */
85 static uint_t zfs_embedded_slog_min_ms = 64;
86 
87 /* default target for number of metaslabs per top-level vdev */
88 static uint_t zfs_vdev_default_ms_count = 200;
89 
90 /* minimum number of metaslabs per top-level vdev */
91 static uint_t zfs_vdev_min_ms_count = 16;
92 
93 /* practical upper limit of total metaslabs per top-level vdev */
94 static uint_t zfs_vdev_ms_count_limit = 1ULL << 17;
95 
96 /* lower limit for metaslab size (512M) */
97 static uint_t zfs_vdev_default_ms_shift = 29;
98 
99 /* upper limit for metaslab size (16G) */
100 static uint_t zfs_vdev_max_ms_shift = 34;
101 
102 int vdev_validate_skip = B_FALSE;
103 
104 /*
105  * Since the DTL space map of a vdev is not expected to have a lot of
106  * entries, we default its block size to 4K.
107  */
108 int zfs_vdev_dtl_sm_blksz = (1 << 12);
109 
110 /*
111  * Rate limit slow IO (delay) events to this many per second.
112  */
113 static unsigned int zfs_slow_io_events_per_second = 20;
114 
115 /*
116  * Rate limit deadman "hung IO" events to this many per second.
117  */
118 static unsigned int zfs_deadman_events_per_second = 1;
119 
120 /*
121  * Rate limit direct write IO verify failures to this many per scond.
122  */
123 static unsigned int zfs_dio_write_verify_events_per_second = 20;
124 
125 /*
126  * Rate limit checksum events after this many checksum errors per second.
127  */
128 static unsigned int zfs_checksum_events_per_second = 20;
129 
130 /*
131  * Ignore errors during scrub/resilver.  Allows to work around resilver
132  * upon import when there are pool errors.
133  */
134 static int zfs_scan_ignore_errors = 0;
135 
136 /*
137  * vdev-wide space maps that have lots of entries written to them at
138  * the end of each transaction can benefit from a higher I/O bandwidth
139  * (e.g. vdev_obsolete_sm), thus we default their block size to 128K.
140  */
141 int zfs_vdev_standard_sm_blksz = (1 << 17);
142 
143 /*
144  * Tunable parameter for debugging or performance analysis. Setting this
145  * will cause pool corruption on power loss if a volatile out-of-order
146  * write cache is enabled.
147  */
148 int zfs_nocacheflush = 0;
149 
150 /*
151  * Maximum and minimum ashift values that can be automatically set based on
152  * vdev's physical ashift (disk's physical sector size).  While ASHIFT_MAX
153  * is higher than the maximum value, it is intentionally limited here to not
154  * excessively impact pool space efficiency.  Higher ashift values may still
155  * be forced by vdev logical ashift or by user via ashift property, but won't
156  * be set automatically as a performance optimization.
157  */
158 uint_t zfs_vdev_max_auto_ashift = 14;
159 uint_t zfs_vdev_min_auto_ashift = ASHIFT_MIN;
160 
161 /*
162  * VDEV checksum verification for Direct I/O writes. This is neccessary for
163  * Linux, because anonymous pages can not be placed under write protection
164  * during Direct I/O writes.
165  */
166 #if !defined(__FreeBSD__)
167 uint_t zfs_vdev_direct_write_verify = 1;
168 #else
169 uint_t zfs_vdev_direct_write_verify = 0;
170 #endif
171 
172 void
173 vdev_dbgmsg(vdev_t *vd, const char *fmt, ...)
174 {
175 	va_list adx;
176 	char buf[256];
177 
178 	va_start(adx, fmt);
179 	(void) vsnprintf(buf, sizeof (buf), fmt, adx);
180 	va_end(adx);
181 
182 	if (vd->vdev_path != NULL) {
183 		zfs_dbgmsg("%s vdev '%s': %s", vd->vdev_ops->vdev_op_type,
184 		    vd->vdev_path, buf);
185 	} else {
186 		zfs_dbgmsg("%s-%llu vdev (guid %llu): %s",
187 		    vd->vdev_ops->vdev_op_type,
188 		    (u_longlong_t)vd->vdev_id,
189 		    (u_longlong_t)vd->vdev_guid, buf);
190 	}
191 }
192 
193 void
194 vdev_dbgmsg_print_tree(vdev_t *vd, int indent)
195 {
196 	char state[20];
197 
198 	if (vd->vdev_ishole || vd->vdev_ops == &vdev_missing_ops) {
199 		zfs_dbgmsg("%*svdev %llu: %s", indent, "",
200 		    (u_longlong_t)vd->vdev_id,
201 		    vd->vdev_ops->vdev_op_type);
202 		return;
203 	}
204 
205 	switch (vd->vdev_state) {
206 	case VDEV_STATE_UNKNOWN:
207 		(void) snprintf(state, sizeof (state), "unknown");
208 		break;
209 	case VDEV_STATE_CLOSED:
210 		(void) snprintf(state, sizeof (state), "closed");
211 		break;
212 	case VDEV_STATE_OFFLINE:
213 		(void) snprintf(state, sizeof (state), "offline");
214 		break;
215 	case VDEV_STATE_REMOVED:
216 		(void) snprintf(state, sizeof (state), "removed");
217 		break;
218 	case VDEV_STATE_CANT_OPEN:
219 		(void) snprintf(state, sizeof (state), "can't open");
220 		break;
221 	case VDEV_STATE_FAULTED:
222 		(void) snprintf(state, sizeof (state), "faulted");
223 		break;
224 	case VDEV_STATE_DEGRADED:
225 		(void) snprintf(state, sizeof (state), "degraded");
226 		break;
227 	case VDEV_STATE_HEALTHY:
228 		(void) snprintf(state, sizeof (state), "healthy");
229 		break;
230 	default:
231 		(void) snprintf(state, sizeof (state), "<state %u>",
232 		    (uint_t)vd->vdev_state);
233 	}
234 
235 	zfs_dbgmsg("%*svdev %u: %s%s, guid: %llu, path: %s, %s", indent,
236 	    "", (int)vd->vdev_id, vd->vdev_ops->vdev_op_type,
237 	    vd->vdev_islog ? " (log)" : "",
238 	    (u_longlong_t)vd->vdev_guid,
239 	    vd->vdev_path ? vd->vdev_path : "N/A", state);
240 
241 	for (uint64_t i = 0; i < vd->vdev_children; i++)
242 		vdev_dbgmsg_print_tree(vd->vdev_child[i], indent + 2);
243 }
244 
245 /*
246  * Virtual device management.
247  */
248 
249 static vdev_ops_t *const vdev_ops_table[] = {
250 	&vdev_root_ops,
251 	&vdev_raidz_ops,
252 	&vdev_draid_ops,
253 	&vdev_draid_spare_ops,
254 	&vdev_mirror_ops,
255 	&vdev_replacing_ops,
256 	&vdev_spare_ops,
257 	&vdev_disk_ops,
258 	&vdev_file_ops,
259 	&vdev_missing_ops,
260 	&vdev_hole_ops,
261 	&vdev_indirect_ops,
262 	NULL
263 };
264 
265 /*
266  * Given a vdev type, return the appropriate ops vector.
267  */
268 static vdev_ops_t *
269 vdev_getops(const char *type)
270 {
271 	vdev_ops_t *ops, *const *opspp;
272 
273 	for (opspp = vdev_ops_table; (ops = *opspp) != NULL; opspp++)
274 		if (strcmp(ops->vdev_op_type, type) == 0)
275 			break;
276 
277 	return (ops);
278 }
279 
280 /*
281  * Given a vdev and a metaslab class, find which metaslab group we're
282  * interested in. All vdevs may belong to two different metaslab classes.
283  * Dedicated slog devices use only the primary metaslab group, rather than a
284  * separate log group. For embedded slogs, the vdev_log_mg will be non-NULL.
285  */
286 metaslab_group_t *
287 vdev_get_mg(vdev_t *vd, metaslab_class_t *mc)
288 {
289 	if (mc == spa_embedded_log_class(vd->vdev_spa) &&
290 	    vd->vdev_log_mg != NULL)
291 		return (vd->vdev_log_mg);
292 	else
293 		return (vd->vdev_mg);
294 }
295 
296 void
297 vdev_default_xlate(vdev_t *vd, const range_seg64_t *logical_rs,
298     range_seg64_t *physical_rs, range_seg64_t *remain_rs)
299 {
300 	(void) vd, (void) remain_rs;
301 
302 	physical_rs->rs_start = logical_rs->rs_start;
303 	physical_rs->rs_end = logical_rs->rs_end;
304 }
305 
306 /*
307  * Derive the enumerated allocation bias from string input.
308  * String origin is either the per-vdev zap or zpool(8).
309  */
310 static vdev_alloc_bias_t
311 vdev_derive_alloc_bias(const char *bias)
312 {
313 	vdev_alloc_bias_t alloc_bias = VDEV_BIAS_NONE;
314 
315 	if (strcmp(bias, VDEV_ALLOC_BIAS_LOG) == 0)
316 		alloc_bias = VDEV_BIAS_LOG;
317 	else if (strcmp(bias, VDEV_ALLOC_BIAS_SPECIAL) == 0)
318 		alloc_bias = VDEV_BIAS_SPECIAL;
319 	else if (strcmp(bias, VDEV_ALLOC_BIAS_DEDUP) == 0)
320 		alloc_bias = VDEV_BIAS_DEDUP;
321 
322 	return (alloc_bias);
323 }
324 
325 /*
326  * Default asize function: return the MAX of psize with the asize of
327  * all children.  This is what's used by anything other than RAID-Z.
328  */
329 uint64_t
330 vdev_default_asize(vdev_t *vd, uint64_t psize, uint64_t txg)
331 {
332 	uint64_t asize = P2ROUNDUP(psize, 1ULL << vd->vdev_top->vdev_ashift);
333 	uint64_t csize;
334 
335 	for (int c = 0; c < vd->vdev_children; c++) {
336 		csize = vdev_psize_to_asize_txg(vd->vdev_child[c], psize, txg);
337 		asize = MAX(asize, csize);
338 	}
339 
340 	return (asize);
341 }
342 
343 uint64_t
344 vdev_default_min_asize(vdev_t *vd)
345 {
346 	return (vd->vdev_min_asize);
347 }
348 
349 /*
350  * Get the minimum allocatable size. We define the allocatable size as
351  * the vdev's asize rounded to the nearest metaslab. This allows us to
352  * replace or attach devices which don't have the same physical size but
353  * can still satisfy the same number of allocations.
354  */
355 uint64_t
356 vdev_get_min_asize(vdev_t *vd)
357 {
358 	vdev_t *pvd = vd->vdev_parent;
359 
360 	/*
361 	 * If our parent is NULL (inactive spare or cache) or is the root,
362 	 * just return our own asize.
363 	 */
364 	if (pvd == NULL)
365 		return (vd->vdev_asize);
366 
367 	/*
368 	 * The top-level vdev just returns the allocatable size rounded
369 	 * to the nearest metaslab.
370 	 */
371 	if (vd == vd->vdev_top)
372 		return (P2ALIGN_TYPED(vd->vdev_asize, 1ULL << vd->vdev_ms_shift,
373 		    uint64_t));
374 
375 	return (pvd->vdev_ops->vdev_op_min_asize(pvd));
376 }
377 
378 void
379 vdev_set_min_asize(vdev_t *vd)
380 {
381 	vd->vdev_min_asize = vdev_get_min_asize(vd);
382 
383 	for (int c = 0; c < vd->vdev_children; c++)
384 		vdev_set_min_asize(vd->vdev_child[c]);
385 }
386 
387 /*
388  * Get the minimal allocation size for the top-level vdev.
389  */
390 uint64_t
391 vdev_get_min_alloc(vdev_t *vd)
392 {
393 	uint64_t min_alloc = 1ULL << vd->vdev_ashift;
394 
395 	if (vd->vdev_ops->vdev_op_min_alloc != NULL)
396 		min_alloc = vd->vdev_ops->vdev_op_min_alloc(vd);
397 
398 	return (min_alloc);
399 }
400 
401 /*
402  * Get the parity level for a top-level vdev.
403  */
404 uint64_t
405 vdev_get_nparity(vdev_t *vd)
406 {
407 	uint64_t nparity = 0;
408 
409 	if (vd->vdev_ops->vdev_op_nparity != NULL)
410 		nparity = vd->vdev_ops->vdev_op_nparity(vd);
411 
412 	return (nparity);
413 }
414 
415 static int
416 vdev_prop_get_int(vdev_t *vd, vdev_prop_t prop, uint64_t *value)
417 {
418 	spa_t *spa = vd->vdev_spa;
419 	objset_t *mos = spa->spa_meta_objset;
420 	uint64_t objid;
421 	int err;
422 
423 	if (vd->vdev_root_zap != 0) {
424 		objid = vd->vdev_root_zap;
425 	} else if (vd->vdev_top_zap != 0) {
426 		objid = vd->vdev_top_zap;
427 	} else if (vd->vdev_leaf_zap != 0) {
428 		objid = vd->vdev_leaf_zap;
429 	} else {
430 		return (EINVAL);
431 	}
432 
433 	err = zap_lookup(mos, objid, vdev_prop_to_name(prop),
434 	    sizeof (uint64_t), 1, value);
435 
436 	if (err == ENOENT)
437 		*value = vdev_prop_default_numeric(prop);
438 
439 	return (err);
440 }
441 
442 /*
443  * Get the number of data disks for a top-level vdev.
444  */
445 uint64_t
446 vdev_get_ndisks(vdev_t *vd)
447 {
448 	uint64_t ndisks = 1;
449 
450 	if (vd->vdev_ops->vdev_op_ndisks != NULL)
451 		ndisks = vd->vdev_ops->vdev_op_ndisks(vd);
452 
453 	return (ndisks);
454 }
455 
456 vdev_t *
457 vdev_lookup_top(spa_t *spa, uint64_t vdev)
458 {
459 	vdev_t *rvd = spa->spa_root_vdev;
460 
461 	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
462 
463 	if (vdev < rvd->vdev_children) {
464 		ASSERT(rvd->vdev_child[vdev] != NULL);
465 		return (rvd->vdev_child[vdev]);
466 	}
467 
468 	return (NULL);
469 }
470 
471 vdev_t *
472 vdev_lookup_by_guid(vdev_t *vd, uint64_t guid)
473 {
474 	vdev_t *mvd;
475 
476 	if (vd->vdev_guid == guid)
477 		return (vd);
478 
479 	for (int c = 0; c < vd->vdev_children; c++)
480 		if ((mvd = vdev_lookup_by_guid(vd->vdev_child[c], guid)) !=
481 		    NULL)
482 			return (mvd);
483 
484 	return (NULL);
485 }
486 
487 static int
488 vdev_count_leaves_impl(vdev_t *vd)
489 {
490 	int n = 0;
491 
492 	if (vd->vdev_ops->vdev_op_leaf)
493 		return (1);
494 
495 	for (int c = 0; c < vd->vdev_children; c++)
496 		n += vdev_count_leaves_impl(vd->vdev_child[c]);
497 
498 	return (n);
499 }
500 
501 int
502 vdev_count_leaves(spa_t *spa)
503 {
504 	int rc;
505 
506 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
507 	rc = vdev_count_leaves_impl(spa->spa_root_vdev);
508 	spa_config_exit(spa, SCL_VDEV, FTAG);
509 
510 	return (rc);
511 }
512 
513 void
514 vdev_add_child(vdev_t *pvd, vdev_t *cvd)
515 {
516 	size_t oldsize, newsize;
517 	uint64_t id = cvd->vdev_id;
518 	vdev_t **newchild;
519 
520 	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
521 	ASSERT(cvd->vdev_parent == NULL);
522 
523 	cvd->vdev_parent = pvd;
524 
525 	if (pvd == NULL)
526 		return;
527 
528 	ASSERT(id >= pvd->vdev_children || pvd->vdev_child[id] == NULL);
529 
530 	oldsize = pvd->vdev_children * sizeof (vdev_t *);
531 	pvd->vdev_children = MAX(pvd->vdev_children, id + 1);
532 	newsize = pvd->vdev_children * sizeof (vdev_t *);
533 
534 	newchild = kmem_alloc(newsize, KM_SLEEP);
535 	if (pvd->vdev_child != NULL) {
536 		memcpy(newchild, pvd->vdev_child, oldsize);
537 		kmem_free(pvd->vdev_child, oldsize);
538 	}
539 
540 	pvd->vdev_child = newchild;
541 	pvd->vdev_child[id] = cvd;
542 
543 	cvd->vdev_top = (pvd->vdev_top ? pvd->vdev_top: cvd);
544 	ASSERT(cvd->vdev_top->vdev_parent->vdev_parent == NULL);
545 
546 	/*
547 	 * Walk up all ancestors to update guid sum.
548 	 */
549 	for (; pvd != NULL; pvd = pvd->vdev_parent)
550 		pvd->vdev_guid_sum += cvd->vdev_guid_sum;
551 
552 	if (cvd->vdev_ops->vdev_op_leaf) {
553 		list_insert_head(&cvd->vdev_spa->spa_leaf_list, cvd);
554 		cvd->vdev_spa->spa_leaf_list_gen++;
555 	}
556 }
557 
558 void
559 vdev_remove_child(vdev_t *pvd, vdev_t *cvd)
560 {
561 	int c;
562 	uint_t id = cvd->vdev_id;
563 
564 	ASSERT(cvd->vdev_parent == pvd);
565 
566 	if (pvd == NULL)
567 		return;
568 
569 	ASSERT(id < pvd->vdev_children);
570 	ASSERT(pvd->vdev_child[id] == cvd);
571 
572 	pvd->vdev_child[id] = NULL;
573 	cvd->vdev_parent = NULL;
574 
575 	for (c = 0; c < pvd->vdev_children; c++)
576 		if (pvd->vdev_child[c])
577 			break;
578 
579 	if (c == pvd->vdev_children) {
580 		kmem_free(pvd->vdev_child, c * sizeof (vdev_t *));
581 		pvd->vdev_child = NULL;
582 		pvd->vdev_children = 0;
583 	}
584 
585 	if (cvd->vdev_ops->vdev_op_leaf) {
586 		spa_t *spa = cvd->vdev_spa;
587 		list_remove(&spa->spa_leaf_list, cvd);
588 		spa->spa_leaf_list_gen++;
589 	}
590 
591 	/*
592 	 * Walk up all ancestors to update guid sum.
593 	 */
594 	for (; pvd != NULL; pvd = pvd->vdev_parent)
595 		pvd->vdev_guid_sum -= cvd->vdev_guid_sum;
596 }
597 
598 /*
599  * Remove any holes in the child array.
600  */
601 void
602 vdev_compact_children(vdev_t *pvd)
603 {
604 	vdev_t **newchild, *cvd;
605 	int oldc = pvd->vdev_children;
606 	int newc;
607 
608 	ASSERT(spa_config_held(pvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
609 
610 	if (oldc == 0)
611 		return;
612 
613 	for (int c = newc = 0; c < oldc; c++)
614 		if (pvd->vdev_child[c])
615 			newc++;
616 
617 	if (newc > 0) {
618 		newchild = kmem_zalloc(newc * sizeof (vdev_t *), KM_SLEEP);
619 
620 		for (int c = newc = 0; c < oldc; c++) {
621 			if ((cvd = pvd->vdev_child[c]) != NULL) {
622 				newchild[newc] = cvd;
623 				cvd->vdev_id = newc++;
624 			}
625 		}
626 	} else {
627 		newchild = NULL;
628 	}
629 
630 	kmem_free(pvd->vdev_child, oldc * sizeof (vdev_t *));
631 	pvd->vdev_child = newchild;
632 	pvd->vdev_children = newc;
633 }
634 
635 /*
636  * Allocate and minimally initialize a vdev_t.
637  */
638 vdev_t *
639 vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops)
640 {
641 	vdev_t *vd;
642 	vdev_indirect_config_t *vic;
643 
644 	vd = kmem_zalloc(sizeof (vdev_t), KM_SLEEP);
645 	vic = &vd->vdev_indirect_config;
646 
647 	if (spa->spa_root_vdev == NULL) {
648 		ASSERT(ops == &vdev_root_ops);
649 		spa->spa_root_vdev = vd;
650 		spa->spa_load_guid = spa_generate_guid(NULL);
651 	}
652 
653 	if (guid == 0 && ops != &vdev_hole_ops) {
654 		if (spa->spa_root_vdev == vd) {
655 			/*
656 			 * The root vdev's guid will also be the pool guid,
657 			 * which must be unique among all pools.
658 			 */
659 			guid = spa_generate_guid(NULL);
660 		} else {
661 			/*
662 			 * Any other vdev's guid must be unique within the pool.
663 			 */
664 			guid = spa_generate_guid(spa);
665 		}
666 		ASSERT(!spa_guid_exists(spa_guid(spa), guid));
667 	}
668 
669 	vd->vdev_spa = spa;
670 	vd->vdev_id = id;
671 	vd->vdev_guid = guid;
672 	vd->vdev_guid_sum = guid;
673 	vd->vdev_ops = ops;
674 	vd->vdev_state = VDEV_STATE_CLOSED;
675 	vd->vdev_ishole = (ops == &vdev_hole_ops);
676 	vic->vic_prev_indirect_vdev = UINT64_MAX;
677 
678 	rw_init(&vd->vdev_indirect_rwlock, NULL, RW_DEFAULT, NULL);
679 	mutex_init(&vd->vdev_obsolete_lock, NULL, MUTEX_DEFAULT, NULL);
680 	vd->vdev_obsolete_segments = range_tree_create(NULL, RANGE_SEG64, NULL,
681 	    0, 0);
682 
683 	/*
684 	 * Initialize rate limit structs for events.  We rate limit ZIO delay
685 	 * and checksum events so that we don't overwhelm ZED with thousands
686 	 * of events when a disk is acting up.
687 	 */
688 	zfs_ratelimit_init(&vd->vdev_delay_rl, &zfs_slow_io_events_per_second,
689 	    1);
690 	zfs_ratelimit_init(&vd->vdev_deadman_rl, &zfs_deadman_events_per_second,
691 	    1);
692 	zfs_ratelimit_init(&vd->vdev_dio_verify_rl,
693 	    &zfs_dio_write_verify_events_per_second, 1);
694 	zfs_ratelimit_init(&vd->vdev_checksum_rl,
695 	    &zfs_checksum_events_per_second, 1);
696 
697 	/*
698 	 * Default Thresholds for tuning ZED
699 	 */
700 	vd->vdev_checksum_n = vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_N);
701 	vd->vdev_checksum_t = vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_T);
702 	vd->vdev_io_n = vdev_prop_default_numeric(VDEV_PROP_IO_N);
703 	vd->vdev_io_t = vdev_prop_default_numeric(VDEV_PROP_IO_T);
704 	vd->vdev_slow_io_n = vdev_prop_default_numeric(VDEV_PROP_SLOW_IO_N);
705 	vd->vdev_slow_io_t = vdev_prop_default_numeric(VDEV_PROP_SLOW_IO_T);
706 
707 	list_link_init(&vd->vdev_config_dirty_node);
708 	list_link_init(&vd->vdev_state_dirty_node);
709 	list_link_init(&vd->vdev_initialize_node);
710 	list_link_init(&vd->vdev_leaf_node);
711 	list_link_init(&vd->vdev_trim_node);
712 
713 	mutex_init(&vd->vdev_dtl_lock, NULL, MUTEX_NOLOCKDEP, NULL);
714 	mutex_init(&vd->vdev_stat_lock, NULL, MUTEX_DEFAULT, NULL);
715 	mutex_init(&vd->vdev_probe_lock, NULL, MUTEX_DEFAULT, NULL);
716 	mutex_init(&vd->vdev_scan_io_queue_lock, NULL, MUTEX_DEFAULT, NULL);
717 
718 	mutex_init(&vd->vdev_initialize_lock, NULL, MUTEX_DEFAULT, NULL);
719 	mutex_init(&vd->vdev_initialize_io_lock, NULL, MUTEX_DEFAULT, NULL);
720 	cv_init(&vd->vdev_initialize_cv, NULL, CV_DEFAULT, NULL);
721 	cv_init(&vd->vdev_initialize_io_cv, NULL, CV_DEFAULT, NULL);
722 
723 	mutex_init(&vd->vdev_trim_lock, NULL, MUTEX_DEFAULT, NULL);
724 	mutex_init(&vd->vdev_autotrim_lock, NULL, MUTEX_DEFAULT, NULL);
725 	mutex_init(&vd->vdev_trim_io_lock, NULL, MUTEX_DEFAULT, NULL);
726 	cv_init(&vd->vdev_trim_cv, NULL, CV_DEFAULT, NULL);
727 	cv_init(&vd->vdev_autotrim_cv, NULL, CV_DEFAULT, NULL);
728 	cv_init(&vd->vdev_autotrim_kick_cv, NULL, CV_DEFAULT, NULL);
729 	cv_init(&vd->vdev_trim_io_cv, NULL, CV_DEFAULT, NULL);
730 
731 	mutex_init(&vd->vdev_rebuild_lock, NULL, MUTEX_DEFAULT, NULL);
732 	cv_init(&vd->vdev_rebuild_cv, NULL, CV_DEFAULT, NULL);
733 
734 	for (int t = 0; t < DTL_TYPES; t++) {
735 		vd->vdev_dtl[t] = range_tree_create(NULL, RANGE_SEG64, NULL, 0,
736 		    0);
737 	}
738 
739 	txg_list_create(&vd->vdev_ms_list, spa,
740 	    offsetof(struct metaslab, ms_txg_node));
741 	txg_list_create(&vd->vdev_dtl_list, spa,
742 	    offsetof(struct vdev, vdev_dtl_node));
743 	vd->vdev_stat.vs_timestamp = gethrtime();
744 	vdev_queue_init(vd);
745 
746 	return (vd);
747 }
748 
749 /*
750  * Allocate a new vdev.  The 'alloctype' is used to control whether we are
751  * creating a new vdev or loading an existing one - the behavior is slightly
752  * different for each case.
753  */
754 int
755 vdev_alloc(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id,
756     int alloctype)
757 {
758 	vdev_ops_t *ops;
759 	const char *type;
760 	uint64_t guid = 0, islog;
761 	vdev_t *vd;
762 	vdev_indirect_config_t *vic;
763 	const char *tmp = NULL;
764 	int rc;
765 	vdev_alloc_bias_t alloc_bias = VDEV_BIAS_NONE;
766 	boolean_t top_level = (parent && !parent->vdev_parent);
767 
768 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
769 
770 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0)
771 		return (SET_ERROR(EINVAL));
772 
773 	if ((ops = vdev_getops(type)) == NULL)
774 		return (SET_ERROR(EINVAL));
775 
776 	/*
777 	 * If this is a load, get the vdev guid from the nvlist.
778 	 * Otherwise, vdev_alloc_common() will generate one for us.
779 	 */
780 	if (alloctype == VDEV_ALLOC_LOAD) {
781 		uint64_t label_id;
782 
783 		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ID, &label_id) ||
784 		    label_id != id)
785 			return (SET_ERROR(EINVAL));
786 
787 		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
788 			return (SET_ERROR(EINVAL));
789 	} else if (alloctype == VDEV_ALLOC_SPARE) {
790 		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
791 			return (SET_ERROR(EINVAL));
792 	} else if (alloctype == VDEV_ALLOC_L2CACHE) {
793 		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
794 			return (SET_ERROR(EINVAL));
795 	} else if (alloctype == VDEV_ALLOC_ROOTPOOL) {
796 		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
797 			return (SET_ERROR(EINVAL));
798 	}
799 
800 	/*
801 	 * The first allocated vdev must be of type 'root'.
802 	 */
803 	if (ops != &vdev_root_ops && spa->spa_root_vdev == NULL)
804 		return (SET_ERROR(EINVAL));
805 
806 	/*
807 	 * Determine whether we're a log vdev.
808 	 */
809 	islog = 0;
810 	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG, &islog);
811 	if (islog && spa_version(spa) < SPA_VERSION_SLOGS)
812 		return (SET_ERROR(ENOTSUP));
813 
814 	if (ops == &vdev_hole_ops && spa_version(spa) < SPA_VERSION_HOLES)
815 		return (SET_ERROR(ENOTSUP));
816 
817 	if (top_level && alloctype == VDEV_ALLOC_ADD) {
818 		const char *bias;
819 
820 		/*
821 		 * If creating a top-level vdev, check for allocation
822 		 * classes input.
823 		 */
824 		if (nvlist_lookup_string(nv, ZPOOL_CONFIG_ALLOCATION_BIAS,
825 		    &bias) == 0) {
826 			alloc_bias = vdev_derive_alloc_bias(bias);
827 
828 			/* spa_vdev_add() expects feature to be enabled */
829 			if (spa->spa_load_state != SPA_LOAD_CREATE &&
830 			    !spa_feature_is_enabled(spa,
831 			    SPA_FEATURE_ALLOCATION_CLASSES)) {
832 				return (SET_ERROR(ENOTSUP));
833 			}
834 		}
835 
836 		/* spa_vdev_add() expects feature to be enabled */
837 		if (ops == &vdev_draid_ops &&
838 		    spa->spa_load_state != SPA_LOAD_CREATE &&
839 		    !spa_feature_is_enabled(spa, SPA_FEATURE_DRAID)) {
840 			return (SET_ERROR(ENOTSUP));
841 		}
842 	}
843 
844 	/*
845 	 * Initialize the vdev specific data.  This is done before calling
846 	 * vdev_alloc_common() since it may fail and this simplifies the
847 	 * error reporting and cleanup code paths.
848 	 */
849 	void *tsd = NULL;
850 	if (ops->vdev_op_init != NULL) {
851 		rc = ops->vdev_op_init(spa, nv, &tsd);
852 		if (rc != 0) {
853 			return (rc);
854 		}
855 	}
856 
857 	vd = vdev_alloc_common(spa, id, guid, ops);
858 	vd->vdev_tsd = tsd;
859 	vd->vdev_islog = islog;
860 
861 	if (top_level && alloc_bias != VDEV_BIAS_NONE)
862 		vd->vdev_alloc_bias = alloc_bias;
863 
864 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &tmp) == 0)
865 		vd->vdev_path = spa_strdup(tmp);
866 
867 	/*
868 	 * ZPOOL_CONFIG_AUX_STATE = "external" means we previously forced a
869 	 * fault on a vdev and want it to persist across imports (like with
870 	 * zpool offline -f).
871 	 */
872 	rc = nvlist_lookup_string(nv, ZPOOL_CONFIG_AUX_STATE, &tmp);
873 	if (rc == 0 && tmp != NULL && strcmp(tmp, "external") == 0) {
874 		vd->vdev_stat.vs_aux = VDEV_AUX_EXTERNAL;
875 		vd->vdev_faulted = 1;
876 		vd->vdev_label_aux = VDEV_AUX_EXTERNAL;
877 	}
878 
879 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_DEVID, &tmp) == 0)
880 		vd->vdev_devid = spa_strdup(tmp);
881 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PHYS_PATH, &tmp) == 0)
882 		vd->vdev_physpath = spa_strdup(tmp);
883 
884 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
885 	    &tmp) == 0)
886 		vd->vdev_enc_sysfs_path = spa_strdup(tmp);
887 
888 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_FRU, &tmp) == 0)
889 		vd->vdev_fru = spa_strdup(tmp);
890 
891 	/*
892 	 * Set the whole_disk property.  If it's not specified, leave the value
893 	 * as -1.
894 	 */
895 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
896 	    &vd->vdev_wholedisk) != 0)
897 		vd->vdev_wholedisk = -1ULL;
898 
899 	vic = &vd->vdev_indirect_config;
900 
901 	ASSERT0(vic->vic_mapping_object);
902 	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_INDIRECT_OBJECT,
903 	    &vic->vic_mapping_object);
904 	ASSERT0(vic->vic_births_object);
905 	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_INDIRECT_BIRTHS,
906 	    &vic->vic_births_object);
907 	ASSERT3U(vic->vic_prev_indirect_vdev, ==, UINT64_MAX);
908 	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
909 	    &vic->vic_prev_indirect_vdev);
910 
911 	/*
912 	 * Look for the 'not present' flag.  This will only be set if the device
913 	 * was not present at the time of import.
914 	 */
915 	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
916 	    &vd->vdev_not_present);
917 
918 	/*
919 	 * Get the alignment requirement. Ignore pool ashift for vdev
920 	 * attach case.
921 	 */
922 	if (alloctype != VDEV_ALLOC_ATTACH) {
923 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASHIFT,
924 		    &vd->vdev_ashift);
925 	} else {
926 		vd->vdev_attaching = B_TRUE;
927 	}
928 
929 	/*
930 	 * Retrieve the vdev creation time.
931 	 */
932 	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_CREATE_TXG,
933 	    &vd->vdev_crtxg);
934 
935 	if (vd->vdev_ops == &vdev_root_ops &&
936 	    (alloctype == VDEV_ALLOC_LOAD ||
937 	    alloctype == VDEV_ALLOC_SPLIT ||
938 	    alloctype == VDEV_ALLOC_ROOTPOOL)) {
939 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_VDEV_ROOT_ZAP,
940 		    &vd->vdev_root_zap);
941 	}
942 
943 	/*
944 	 * If we're a top-level vdev, try to load the allocation parameters.
945 	 */
946 	if (top_level &&
947 	    (alloctype == VDEV_ALLOC_LOAD || alloctype == VDEV_ALLOC_SPLIT)) {
948 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_ARRAY,
949 		    &vd->vdev_ms_array);
950 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_SHIFT,
951 		    &vd->vdev_ms_shift);
952 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASIZE,
953 		    &vd->vdev_asize);
954 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NONALLOCATING,
955 		    &vd->vdev_noalloc);
956 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVING,
957 		    &vd->vdev_removing);
958 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_VDEV_TOP_ZAP,
959 		    &vd->vdev_top_zap);
960 		vd->vdev_rz_expanding = nvlist_exists(nv,
961 		    ZPOOL_CONFIG_RAIDZ_EXPANDING);
962 	} else {
963 		ASSERT0(vd->vdev_top_zap);
964 	}
965 
966 	if (top_level && alloctype != VDEV_ALLOC_ATTACH) {
967 		ASSERT(alloctype == VDEV_ALLOC_LOAD ||
968 		    alloctype == VDEV_ALLOC_ADD ||
969 		    alloctype == VDEV_ALLOC_SPLIT ||
970 		    alloctype == VDEV_ALLOC_ROOTPOOL);
971 		/* Note: metaslab_group_create() is now deferred */
972 	}
973 
974 	if (vd->vdev_ops->vdev_op_leaf &&
975 	    (alloctype == VDEV_ALLOC_LOAD || alloctype == VDEV_ALLOC_SPLIT)) {
976 		(void) nvlist_lookup_uint64(nv,
977 		    ZPOOL_CONFIG_VDEV_LEAF_ZAP, &vd->vdev_leaf_zap);
978 	} else {
979 		ASSERT0(vd->vdev_leaf_zap);
980 	}
981 
982 	/*
983 	 * If we're a leaf vdev, try to load the DTL object and other state.
984 	 */
985 
986 	if (vd->vdev_ops->vdev_op_leaf &&
987 	    (alloctype == VDEV_ALLOC_LOAD || alloctype == VDEV_ALLOC_L2CACHE ||
988 	    alloctype == VDEV_ALLOC_ROOTPOOL)) {
989 		if (alloctype == VDEV_ALLOC_LOAD) {
990 			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DTL,
991 			    &vd->vdev_dtl_object);
992 			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_UNSPARE,
993 			    &vd->vdev_unspare);
994 		}
995 
996 		if (alloctype == VDEV_ALLOC_ROOTPOOL) {
997 			uint64_t spare = 0;
998 
999 			if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_SPARE,
1000 			    &spare) == 0 && spare)
1001 				spa_spare_add(vd);
1002 		}
1003 
1004 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_OFFLINE,
1005 		    &vd->vdev_offline);
1006 
1007 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_RESILVER_TXG,
1008 		    &vd->vdev_resilver_txg);
1009 
1010 		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REBUILD_TXG,
1011 		    &vd->vdev_rebuild_txg);
1012 
1013 		if (nvlist_exists(nv, ZPOOL_CONFIG_RESILVER_DEFER))
1014 			vdev_defer_resilver(vd);
1015 
1016 		/*
1017 		 * In general, when importing a pool we want to ignore the
1018 		 * persistent fault state, as the diagnosis made on another
1019 		 * system may not be valid in the current context.  The only
1020 		 * exception is if we forced a vdev to a persistently faulted
1021 		 * state with 'zpool offline -f'.  The persistent fault will
1022 		 * remain across imports until cleared.
1023 		 *
1024 		 * Local vdevs will remain in the faulted state.
1025 		 */
1026 		if (spa_load_state(spa) == SPA_LOAD_OPEN ||
1027 		    spa_load_state(spa) == SPA_LOAD_IMPORT) {
1028 			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_FAULTED,
1029 			    &vd->vdev_faulted);
1030 			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DEGRADED,
1031 			    &vd->vdev_degraded);
1032 			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVED,
1033 			    &vd->vdev_removed);
1034 
1035 			if (vd->vdev_faulted || vd->vdev_degraded) {
1036 				const char *aux;
1037 
1038 				vd->vdev_label_aux =
1039 				    VDEV_AUX_ERR_EXCEEDED;
1040 				if (nvlist_lookup_string(nv,
1041 				    ZPOOL_CONFIG_AUX_STATE, &aux) == 0 &&
1042 				    strcmp(aux, "external") == 0)
1043 					vd->vdev_label_aux = VDEV_AUX_EXTERNAL;
1044 				else
1045 					vd->vdev_faulted = 0ULL;
1046 			}
1047 		}
1048 	}
1049 
1050 	/*
1051 	 * Add ourselves to the parent's list of children.
1052 	 */
1053 	vdev_add_child(parent, vd);
1054 
1055 	*vdp = vd;
1056 
1057 	return (0);
1058 }
1059 
1060 void
1061 vdev_free(vdev_t *vd)
1062 {
1063 	spa_t *spa = vd->vdev_spa;
1064 
1065 	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
1066 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
1067 	ASSERT3P(vd->vdev_autotrim_thread, ==, NULL);
1068 	ASSERT3P(vd->vdev_rebuild_thread, ==, NULL);
1069 
1070 	/*
1071 	 * Scan queues are normally destroyed at the end of a scan. If the
1072 	 * queue exists here, that implies the vdev is being removed while
1073 	 * the scan is still running.
1074 	 */
1075 	if (vd->vdev_scan_io_queue != NULL) {
1076 		mutex_enter(&vd->vdev_scan_io_queue_lock);
1077 		dsl_scan_io_queue_destroy(vd->vdev_scan_io_queue);
1078 		vd->vdev_scan_io_queue = NULL;
1079 		mutex_exit(&vd->vdev_scan_io_queue_lock);
1080 	}
1081 
1082 	/*
1083 	 * vdev_free() implies closing the vdev first.  This is simpler than
1084 	 * trying to ensure complicated semantics for all callers.
1085 	 */
1086 	vdev_close(vd);
1087 
1088 	ASSERT(!list_link_active(&vd->vdev_config_dirty_node));
1089 	ASSERT(!list_link_active(&vd->vdev_state_dirty_node));
1090 
1091 	/*
1092 	 * Free all children.
1093 	 */
1094 	for (int c = 0; c < vd->vdev_children; c++)
1095 		vdev_free(vd->vdev_child[c]);
1096 
1097 	ASSERT(vd->vdev_child == NULL);
1098 	ASSERT(vd->vdev_guid_sum == vd->vdev_guid);
1099 
1100 	if (vd->vdev_ops->vdev_op_fini != NULL)
1101 		vd->vdev_ops->vdev_op_fini(vd);
1102 
1103 	/*
1104 	 * Discard allocation state.
1105 	 */
1106 	if (vd->vdev_mg != NULL) {
1107 		vdev_metaslab_fini(vd);
1108 		metaslab_group_destroy(vd->vdev_mg);
1109 		vd->vdev_mg = NULL;
1110 	}
1111 	if (vd->vdev_log_mg != NULL) {
1112 		ASSERT0(vd->vdev_ms_count);
1113 		metaslab_group_destroy(vd->vdev_log_mg);
1114 		vd->vdev_log_mg = NULL;
1115 	}
1116 
1117 	ASSERT0(vd->vdev_stat.vs_space);
1118 	ASSERT0(vd->vdev_stat.vs_dspace);
1119 	ASSERT0(vd->vdev_stat.vs_alloc);
1120 
1121 	/*
1122 	 * Remove this vdev from its parent's child list.
1123 	 */
1124 	vdev_remove_child(vd->vdev_parent, vd);
1125 
1126 	ASSERT(vd->vdev_parent == NULL);
1127 	ASSERT(!list_link_active(&vd->vdev_leaf_node));
1128 
1129 	/*
1130 	 * Clean up vdev structure.
1131 	 */
1132 	vdev_queue_fini(vd);
1133 
1134 	if (vd->vdev_path)
1135 		spa_strfree(vd->vdev_path);
1136 	if (vd->vdev_devid)
1137 		spa_strfree(vd->vdev_devid);
1138 	if (vd->vdev_physpath)
1139 		spa_strfree(vd->vdev_physpath);
1140 
1141 	if (vd->vdev_enc_sysfs_path)
1142 		spa_strfree(vd->vdev_enc_sysfs_path);
1143 
1144 	if (vd->vdev_fru)
1145 		spa_strfree(vd->vdev_fru);
1146 
1147 	if (vd->vdev_isspare)
1148 		spa_spare_remove(vd);
1149 	if (vd->vdev_isl2cache)
1150 		spa_l2cache_remove(vd);
1151 
1152 	txg_list_destroy(&vd->vdev_ms_list);
1153 	txg_list_destroy(&vd->vdev_dtl_list);
1154 
1155 	mutex_enter(&vd->vdev_dtl_lock);
1156 	space_map_close(vd->vdev_dtl_sm);
1157 	for (int t = 0; t < DTL_TYPES; t++) {
1158 		range_tree_vacate(vd->vdev_dtl[t], NULL, NULL);
1159 		range_tree_destroy(vd->vdev_dtl[t]);
1160 	}
1161 	mutex_exit(&vd->vdev_dtl_lock);
1162 
1163 	EQUIV(vd->vdev_indirect_births != NULL,
1164 	    vd->vdev_indirect_mapping != NULL);
1165 	if (vd->vdev_indirect_births != NULL) {
1166 		vdev_indirect_mapping_close(vd->vdev_indirect_mapping);
1167 		vdev_indirect_births_close(vd->vdev_indirect_births);
1168 	}
1169 
1170 	if (vd->vdev_obsolete_sm != NULL) {
1171 		ASSERT(vd->vdev_removing ||
1172 		    vd->vdev_ops == &vdev_indirect_ops);
1173 		space_map_close(vd->vdev_obsolete_sm);
1174 		vd->vdev_obsolete_sm = NULL;
1175 	}
1176 	range_tree_destroy(vd->vdev_obsolete_segments);
1177 	rw_destroy(&vd->vdev_indirect_rwlock);
1178 	mutex_destroy(&vd->vdev_obsolete_lock);
1179 
1180 	mutex_destroy(&vd->vdev_dtl_lock);
1181 	mutex_destroy(&vd->vdev_stat_lock);
1182 	mutex_destroy(&vd->vdev_probe_lock);
1183 	mutex_destroy(&vd->vdev_scan_io_queue_lock);
1184 
1185 	mutex_destroy(&vd->vdev_initialize_lock);
1186 	mutex_destroy(&vd->vdev_initialize_io_lock);
1187 	cv_destroy(&vd->vdev_initialize_io_cv);
1188 	cv_destroy(&vd->vdev_initialize_cv);
1189 
1190 	mutex_destroy(&vd->vdev_trim_lock);
1191 	mutex_destroy(&vd->vdev_autotrim_lock);
1192 	mutex_destroy(&vd->vdev_trim_io_lock);
1193 	cv_destroy(&vd->vdev_trim_cv);
1194 	cv_destroy(&vd->vdev_autotrim_cv);
1195 	cv_destroy(&vd->vdev_autotrim_kick_cv);
1196 	cv_destroy(&vd->vdev_trim_io_cv);
1197 
1198 	mutex_destroy(&vd->vdev_rebuild_lock);
1199 	cv_destroy(&vd->vdev_rebuild_cv);
1200 
1201 	zfs_ratelimit_fini(&vd->vdev_delay_rl);
1202 	zfs_ratelimit_fini(&vd->vdev_deadman_rl);
1203 	zfs_ratelimit_fini(&vd->vdev_dio_verify_rl);
1204 	zfs_ratelimit_fini(&vd->vdev_checksum_rl);
1205 
1206 	if (vd == spa->spa_root_vdev)
1207 		spa->spa_root_vdev = NULL;
1208 
1209 	kmem_free(vd, sizeof (vdev_t));
1210 }
1211 
1212 /*
1213  * Transfer top-level vdev state from svd to tvd.
1214  */
1215 static void
1216 vdev_top_transfer(vdev_t *svd, vdev_t *tvd)
1217 {
1218 	spa_t *spa = svd->vdev_spa;
1219 	metaslab_t *msp;
1220 	vdev_t *vd;
1221 	int t;
1222 
1223 	ASSERT(tvd == tvd->vdev_top);
1224 
1225 	tvd->vdev_ms_array = svd->vdev_ms_array;
1226 	tvd->vdev_ms_shift = svd->vdev_ms_shift;
1227 	tvd->vdev_ms_count = svd->vdev_ms_count;
1228 	tvd->vdev_top_zap = svd->vdev_top_zap;
1229 
1230 	svd->vdev_ms_array = 0;
1231 	svd->vdev_ms_shift = 0;
1232 	svd->vdev_ms_count = 0;
1233 	svd->vdev_top_zap = 0;
1234 
1235 	if (tvd->vdev_mg)
1236 		ASSERT3P(tvd->vdev_mg, ==, svd->vdev_mg);
1237 	if (tvd->vdev_log_mg)
1238 		ASSERT3P(tvd->vdev_log_mg, ==, svd->vdev_log_mg);
1239 	tvd->vdev_mg = svd->vdev_mg;
1240 	tvd->vdev_log_mg = svd->vdev_log_mg;
1241 	tvd->vdev_ms = svd->vdev_ms;
1242 
1243 	svd->vdev_mg = NULL;
1244 	svd->vdev_log_mg = NULL;
1245 	svd->vdev_ms = NULL;
1246 
1247 	if (tvd->vdev_mg != NULL)
1248 		tvd->vdev_mg->mg_vd = tvd;
1249 	if (tvd->vdev_log_mg != NULL)
1250 		tvd->vdev_log_mg->mg_vd = tvd;
1251 
1252 	tvd->vdev_checkpoint_sm = svd->vdev_checkpoint_sm;
1253 	svd->vdev_checkpoint_sm = NULL;
1254 
1255 	tvd->vdev_alloc_bias = svd->vdev_alloc_bias;
1256 	svd->vdev_alloc_bias = VDEV_BIAS_NONE;
1257 
1258 	tvd->vdev_stat.vs_alloc = svd->vdev_stat.vs_alloc;
1259 	tvd->vdev_stat.vs_space = svd->vdev_stat.vs_space;
1260 	tvd->vdev_stat.vs_dspace = svd->vdev_stat.vs_dspace;
1261 
1262 	svd->vdev_stat.vs_alloc = 0;
1263 	svd->vdev_stat.vs_space = 0;
1264 	svd->vdev_stat.vs_dspace = 0;
1265 
1266 	/*
1267 	 * State which may be set on a top-level vdev that's in the
1268 	 * process of being removed.
1269 	 */
1270 	ASSERT0(tvd->vdev_indirect_config.vic_births_object);
1271 	ASSERT0(tvd->vdev_indirect_config.vic_mapping_object);
1272 	ASSERT3U(tvd->vdev_indirect_config.vic_prev_indirect_vdev, ==, -1ULL);
1273 	ASSERT3P(tvd->vdev_indirect_mapping, ==, NULL);
1274 	ASSERT3P(tvd->vdev_indirect_births, ==, NULL);
1275 	ASSERT3P(tvd->vdev_obsolete_sm, ==, NULL);
1276 	ASSERT0(tvd->vdev_noalloc);
1277 	ASSERT0(tvd->vdev_removing);
1278 	ASSERT0(tvd->vdev_rebuilding);
1279 	tvd->vdev_noalloc = svd->vdev_noalloc;
1280 	tvd->vdev_removing = svd->vdev_removing;
1281 	tvd->vdev_rebuilding = svd->vdev_rebuilding;
1282 	tvd->vdev_rebuild_config = svd->vdev_rebuild_config;
1283 	tvd->vdev_indirect_config = svd->vdev_indirect_config;
1284 	tvd->vdev_indirect_mapping = svd->vdev_indirect_mapping;
1285 	tvd->vdev_indirect_births = svd->vdev_indirect_births;
1286 	range_tree_swap(&svd->vdev_obsolete_segments,
1287 	    &tvd->vdev_obsolete_segments);
1288 	tvd->vdev_obsolete_sm = svd->vdev_obsolete_sm;
1289 	svd->vdev_indirect_config.vic_mapping_object = 0;
1290 	svd->vdev_indirect_config.vic_births_object = 0;
1291 	svd->vdev_indirect_config.vic_prev_indirect_vdev = -1ULL;
1292 	svd->vdev_indirect_mapping = NULL;
1293 	svd->vdev_indirect_births = NULL;
1294 	svd->vdev_obsolete_sm = NULL;
1295 	svd->vdev_noalloc = 0;
1296 	svd->vdev_removing = 0;
1297 	svd->vdev_rebuilding = 0;
1298 
1299 	for (t = 0; t < TXG_SIZE; t++) {
1300 		while ((msp = txg_list_remove(&svd->vdev_ms_list, t)) != NULL)
1301 			(void) txg_list_add(&tvd->vdev_ms_list, msp, t);
1302 		while ((vd = txg_list_remove(&svd->vdev_dtl_list, t)) != NULL)
1303 			(void) txg_list_add(&tvd->vdev_dtl_list, vd, t);
1304 		if (txg_list_remove_this(&spa->spa_vdev_txg_list, svd, t))
1305 			(void) txg_list_add(&spa->spa_vdev_txg_list, tvd, t);
1306 	}
1307 
1308 	if (list_link_active(&svd->vdev_config_dirty_node)) {
1309 		vdev_config_clean(svd);
1310 		vdev_config_dirty(tvd);
1311 	}
1312 
1313 	if (list_link_active(&svd->vdev_state_dirty_node)) {
1314 		vdev_state_clean(svd);
1315 		vdev_state_dirty(tvd);
1316 	}
1317 
1318 	tvd->vdev_deflate_ratio = svd->vdev_deflate_ratio;
1319 	svd->vdev_deflate_ratio = 0;
1320 
1321 	tvd->vdev_islog = svd->vdev_islog;
1322 	svd->vdev_islog = 0;
1323 
1324 	dsl_scan_io_queue_vdev_xfer(svd, tvd);
1325 }
1326 
1327 static void
1328 vdev_top_update(vdev_t *tvd, vdev_t *vd)
1329 {
1330 	if (vd == NULL)
1331 		return;
1332 
1333 	vd->vdev_top = tvd;
1334 
1335 	for (int c = 0; c < vd->vdev_children; c++)
1336 		vdev_top_update(tvd, vd->vdev_child[c]);
1337 }
1338 
1339 /*
1340  * Add a mirror/replacing vdev above an existing vdev.  There is no need to
1341  * call .vdev_op_init() since mirror/replacing vdevs do not have private state.
1342  */
1343 vdev_t *
1344 vdev_add_parent(vdev_t *cvd, vdev_ops_t *ops)
1345 {
1346 	spa_t *spa = cvd->vdev_spa;
1347 	vdev_t *pvd = cvd->vdev_parent;
1348 	vdev_t *mvd;
1349 
1350 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1351 
1352 	mvd = vdev_alloc_common(spa, cvd->vdev_id, 0, ops);
1353 
1354 	mvd->vdev_asize = cvd->vdev_asize;
1355 	mvd->vdev_min_asize = cvd->vdev_min_asize;
1356 	mvd->vdev_max_asize = cvd->vdev_max_asize;
1357 	mvd->vdev_psize = cvd->vdev_psize;
1358 	mvd->vdev_ashift = cvd->vdev_ashift;
1359 	mvd->vdev_logical_ashift = cvd->vdev_logical_ashift;
1360 	mvd->vdev_physical_ashift = cvd->vdev_physical_ashift;
1361 	mvd->vdev_state = cvd->vdev_state;
1362 	mvd->vdev_crtxg = cvd->vdev_crtxg;
1363 
1364 	vdev_remove_child(pvd, cvd);
1365 	vdev_add_child(pvd, mvd);
1366 	cvd->vdev_id = mvd->vdev_children;
1367 	vdev_add_child(mvd, cvd);
1368 	vdev_top_update(cvd->vdev_top, cvd->vdev_top);
1369 
1370 	if (mvd == mvd->vdev_top)
1371 		vdev_top_transfer(cvd, mvd);
1372 
1373 	return (mvd);
1374 }
1375 
1376 /*
1377  * Remove a 1-way mirror/replacing vdev from the tree.
1378  */
1379 void
1380 vdev_remove_parent(vdev_t *cvd)
1381 {
1382 	vdev_t *mvd = cvd->vdev_parent;
1383 	vdev_t *pvd = mvd->vdev_parent;
1384 
1385 	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1386 
1387 	ASSERT(mvd->vdev_children == 1);
1388 	ASSERT(mvd->vdev_ops == &vdev_mirror_ops ||
1389 	    mvd->vdev_ops == &vdev_replacing_ops ||
1390 	    mvd->vdev_ops == &vdev_spare_ops);
1391 	cvd->vdev_ashift = mvd->vdev_ashift;
1392 	cvd->vdev_logical_ashift = mvd->vdev_logical_ashift;
1393 	cvd->vdev_physical_ashift = mvd->vdev_physical_ashift;
1394 	vdev_remove_child(mvd, cvd);
1395 	vdev_remove_child(pvd, mvd);
1396 
1397 	/*
1398 	 * If cvd will replace mvd as a top-level vdev, preserve mvd's guid.
1399 	 * Otherwise, we could have detached an offline device, and when we
1400 	 * go to import the pool we'll think we have two top-level vdevs,
1401 	 * instead of a different version of the same top-level vdev.
1402 	 */
1403 	if (mvd->vdev_top == mvd) {
1404 		uint64_t guid_delta = mvd->vdev_guid - cvd->vdev_guid;
1405 		cvd->vdev_orig_guid = cvd->vdev_guid;
1406 		cvd->vdev_guid += guid_delta;
1407 		cvd->vdev_guid_sum += guid_delta;
1408 
1409 		/*
1410 		 * If pool not set for autoexpand, we need to also preserve
1411 		 * mvd's asize to prevent automatic expansion of cvd.
1412 		 * Otherwise if we are adjusting the mirror by attaching and
1413 		 * detaching children of non-uniform sizes, the mirror could
1414 		 * autoexpand, unexpectedly requiring larger devices to
1415 		 * re-establish the mirror.
1416 		 */
1417 		if (!cvd->vdev_spa->spa_autoexpand)
1418 			cvd->vdev_asize = mvd->vdev_asize;
1419 	}
1420 	cvd->vdev_id = mvd->vdev_id;
1421 	vdev_add_child(pvd, cvd);
1422 	vdev_top_update(cvd->vdev_top, cvd->vdev_top);
1423 
1424 	if (cvd == cvd->vdev_top)
1425 		vdev_top_transfer(mvd, cvd);
1426 
1427 	ASSERT(mvd->vdev_children == 0);
1428 	vdev_free(mvd);
1429 }
1430 
1431 /*
1432  * Choose GCD for spa_gcd_alloc.
1433  */
1434 static uint64_t
1435 vdev_gcd(uint64_t a, uint64_t b)
1436 {
1437 	while (b != 0) {
1438 		uint64_t t = b;
1439 		b = a % b;
1440 		a = t;
1441 	}
1442 	return (a);
1443 }
1444 
1445 /*
1446  * Set spa_min_alloc and spa_gcd_alloc.
1447  */
1448 static void
1449 vdev_spa_set_alloc(spa_t *spa, uint64_t min_alloc)
1450 {
1451 	if (min_alloc < spa->spa_min_alloc)
1452 		spa->spa_min_alloc = min_alloc;
1453 	if (spa->spa_gcd_alloc == INT_MAX) {
1454 		spa->spa_gcd_alloc = min_alloc;
1455 	} else {
1456 		spa->spa_gcd_alloc = vdev_gcd(min_alloc,
1457 		    spa->spa_gcd_alloc);
1458 	}
1459 }
1460 
1461 void
1462 vdev_metaslab_group_create(vdev_t *vd)
1463 {
1464 	spa_t *spa = vd->vdev_spa;
1465 
1466 	/*
1467 	 * metaslab_group_create was delayed until allocation bias was available
1468 	 */
1469 	if (vd->vdev_mg == NULL) {
1470 		metaslab_class_t *mc;
1471 
1472 		if (vd->vdev_islog && vd->vdev_alloc_bias == VDEV_BIAS_NONE)
1473 			vd->vdev_alloc_bias = VDEV_BIAS_LOG;
1474 
1475 		ASSERT3U(vd->vdev_islog, ==,
1476 		    (vd->vdev_alloc_bias == VDEV_BIAS_LOG));
1477 
1478 		switch (vd->vdev_alloc_bias) {
1479 		case VDEV_BIAS_LOG:
1480 			mc = spa_log_class(spa);
1481 			break;
1482 		case VDEV_BIAS_SPECIAL:
1483 			mc = spa_special_class(spa);
1484 			break;
1485 		case VDEV_BIAS_DEDUP:
1486 			mc = spa_dedup_class(spa);
1487 			break;
1488 		default:
1489 			mc = spa_normal_class(spa);
1490 		}
1491 
1492 		vd->vdev_mg = metaslab_group_create(mc, vd,
1493 		    spa->spa_alloc_count);
1494 
1495 		if (!vd->vdev_islog) {
1496 			vd->vdev_log_mg = metaslab_group_create(
1497 			    spa_embedded_log_class(spa), vd, 1);
1498 		}
1499 
1500 		/*
1501 		 * The spa ashift min/max only apply for the normal metaslab
1502 		 * class. Class destination is late binding so ashift boundary
1503 		 * setting had to wait until now.
1504 		 */
1505 		if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
1506 		    mc == spa_normal_class(spa) && vd->vdev_aux == NULL) {
1507 			if (vd->vdev_ashift > spa->spa_max_ashift)
1508 				spa->spa_max_ashift = vd->vdev_ashift;
1509 			if (vd->vdev_ashift < spa->spa_min_ashift)
1510 				spa->spa_min_ashift = vd->vdev_ashift;
1511 
1512 			uint64_t min_alloc = vdev_get_min_alloc(vd);
1513 			vdev_spa_set_alloc(spa, min_alloc);
1514 		}
1515 	}
1516 }
1517 
1518 int
1519 vdev_metaslab_init(vdev_t *vd, uint64_t txg)
1520 {
1521 	spa_t *spa = vd->vdev_spa;
1522 	uint64_t oldc = vd->vdev_ms_count;
1523 	uint64_t newc = vd->vdev_asize >> vd->vdev_ms_shift;
1524 	metaslab_t **mspp;
1525 	int error;
1526 	boolean_t expanding = (oldc != 0);
1527 
1528 	ASSERT(txg == 0 || spa_config_held(spa, SCL_ALLOC, RW_WRITER));
1529 
1530 	/*
1531 	 * This vdev is not being allocated from yet or is a hole.
1532 	 */
1533 	if (vd->vdev_ms_shift == 0)
1534 		return (0);
1535 
1536 	ASSERT(!vd->vdev_ishole);
1537 
1538 	ASSERT(oldc <= newc);
1539 
1540 	mspp = vmem_zalloc(newc * sizeof (*mspp), KM_SLEEP);
1541 
1542 	if (expanding) {
1543 		memcpy(mspp, vd->vdev_ms, oldc * sizeof (*mspp));
1544 		vmem_free(vd->vdev_ms, oldc * sizeof (*mspp));
1545 	}
1546 
1547 	vd->vdev_ms = mspp;
1548 	vd->vdev_ms_count = newc;
1549 
1550 	for (uint64_t m = oldc; m < newc; m++) {
1551 		uint64_t object = 0;
1552 		/*
1553 		 * vdev_ms_array may be 0 if we are creating the "fake"
1554 		 * metaslabs for an indirect vdev for zdb's leak detection.
1555 		 * See zdb_leak_init().
1556 		 */
1557 		if (txg == 0 && vd->vdev_ms_array != 0) {
1558 			error = dmu_read(spa->spa_meta_objset,
1559 			    vd->vdev_ms_array,
1560 			    m * sizeof (uint64_t), sizeof (uint64_t), &object,
1561 			    DMU_READ_PREFETCH);
1562 			if (error != 0) {
1563 				vdev_dbgmsg(vd, "unable to read the metaslab "
1564 				    "array [error=%d]", error);
1565 				return (error);
1566 			}
1567 		}
1568 
1569 		error = metaslab_init(vd->vdev_mg, m, object, txg,
1570 		    &(vd->vdev_ms[m]));
1571 		if (error != 0) {
1572 			vdev_dbgmsg(vd, "metaslab_init failed [error=%d]",
1573 			    error);
1574 			return (error);
1575 		}
1576 	}
1577 
1578 	/*
1579 	 * Find the emptiest metaslab on the vdev and mark it for use for
1580 	 * embedded slog by moving it from the regular to the log metaslab
1581 	 * group.
1582 	 */
1583 	if (vd->vdev_mg->mg_class == spa_normal_class(spa) &&
1584 	    vd->vdev_ms_count > zfs_embedded_slog_min_ms &&
1585 	    avl_is_empty(&vd->vdev_log_mg->mg_metaslab_tree)) {
1586 		uint64_t slog_msid = 0;
1587 		uint64_t smallest = UINT64_MAX;
1588 
1589 		/*
1590 		 * Note, we only search the new metaslabs, because the old
1591 		 * (pre-existing) ones may be active (e.g. have non-empty
1592 		 * range_tree's), and we don't move them to the new
1593 		 * metaslab_t.
1594 		 */
1595 		for (uint64_t m = oldc; m < newc; m++) {
1596 			uint64_t alloc =
1597 			    space_map_allocated(vd->vdev_ms[m]->ms_sm);
1598 			if (alloc < smallest) {
1599 				slog_msid = m;
1600 				smallest = alloc;
1601 			}
1602 		}
1603 		metaslab_t *slog_ms = vd->vdev_ms[slog_msid];
1604 		/*
1605 		 * The metaslab was marked as dirty at the end of
1606 		 * metaslab_init(). Remove it from the dirty list so that we
1607 		 * can uninitialize and reinitialize it to the new class.
1608 		 */
1609 		if (txg != 0) {
1610 			(void) txg_list_remove_this(&vd->vdev_ms_list,
1611 			    slog_ms, txg);
1612 		}
1613 		uint64_t sm_obj = space_map_object(slog_ms->ms_sm);
1614 		metaslab_fini(slog_ms);
1615 		VERIFY0(metaslab_init(vd->vdev_log_mg, slog_msid, sm_obj, txg,
1616 		    &vd->vdev_ms[slog_msid]));
1617 	}
1618 
1619 	if (txg == 0)
1620 		spa_config_enter(spa, SCL_ALLOC, FTAG, RW_WRITER);
1621 
1622 	/*
1623 	 * If the vdev is marked as non-allocating then don't
1624 	 * activate the metaslabs since we want to ensure that
1625 	 * no allocations are performed on this device.
1626 	 */
1627 	if (vd->vdev_noalloc) {
1628 		/* track non-allocating vdev space */
1629 		spa->spa_nonallocating_dspace += spa_deflate(spa) ?
1630 		    vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space;
1631 	} else if (!expanding) {
1632 		metaslab_group_activate(vd->vdev_mg);
1633 		if (vd->vdev_log_mg != NULL)
1634 			metaslab_group_activate(vd->vdev_log_mg);
1635 	}
1636 
1637 	if (txg == 0)
1638 		spa_config_exit(spa, SCL_ALLOC, FTAG);
1639 
1640 	return (0);
1641 }
1642 
1643 void
1644 vdev_metaslab_fini(vdev_t *vd)
1645 {
1646 	if (vd->vdev_checkpoint_sm != NULL) {
1647 		ASSERT(spa_feature_is_active(vd->vdev_spa,
1648 		    SPA_FEATURE_POOL_CHECKPOINT));
1649 		space_map_close(vd->vdev_checkpoint_sm);
1650 		/*
1651 		 * Even though we close the space map, we need to set its
1652 		 * pointer to NULL. The reason is that vdev_metaslab_fini()
1653 		 * may be called multiple times for certain operations
1654 		 * (i.e. when destroying a pool) so we need to ensure that
1655 		 * this clause never executes twice. This logic is similar
1656 		 * to the one used for the vdev_ms clause below.
1657 		 */
1658 		vd->vdev_checkpoint_sm = NULL;
1659 	}
1660 
1661 	if (vd->vdev_ms != NULL) {
1662 		metaslab_group_t *mg = vd->vdev_mg;
1663 
1664 		metaslab_group_passivate(mg);
1665 		if (vd->vdev_log_mg != NULL) {
1666 			ASSERT(!vd->vdev_islog);
1667 			metaslab_group_passivate(vd->vdev_log_mg);
1668 		}
1669 
1670 		uint64_t count = vd->vdev_ms_count;
1671 		for (uint64_t m = 0; m < count; m++) {
1672 			metaslab_t *msp = vd->vdev_ms[m];
1673 			if (msp != NULL)
1674 				metaslab_fini(msp);
1675 		}
1676 		vmem_free(vd->vdev_ms, count * sizeof (metaslab_t *));
1677 		vd->vdev_ms = NULL;
1678 		vd->vdev_ms_count = 0;
1679 
1680 		for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
1681 			ASSERT0(mg->mg_histogram[i]);
1682 			if (vd->vdev_log_mg != NULL)
1683 				ASSERT0(vd->vdev_log_mg->mg_histogram[i]);
1684 		}
1685 	}
1686 	ASSERT0(vd->vdev_ms_count);
1687 }
1688 
1689 typedef struct vdev_probe_stats {
1690 	boolean_t	vps_readable;
1691 	boolean_t	vps_writeable;
1692 	boolean_t	vps_zio_done_probe;
1693 	int		vps_flags;
1694 } vdev_probe_stats_t;
1695 
1696 static void
1697 vdev_probe_done(zio_t *zio)
1698 {
1699 	spa_t *spa = zio->io_spa;
1700 	vdev_t *vd = zio->io_vd;
1701 	vdev_probe_stats_t *vps = zio->io_private;
1702 
1703 	ASSERT(vd->vdev_probe_zio != NULL);
1704 
1705 	if (zio->io_type == ZIO_TYPE_READ) {
1706 		if (zio->io_error == 0)
1707 			vps->vps_readable = 1;
1708 		if (zio->io_error == 0 && spa_writeable(spa)) {
1709 			zio_nowait(zio_write_phys(vd->vdev_probe_zio, vd,
1710 			    zio->io_offset, zio->io_size, zio->io_abd,
1711 			    ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
1712 			    ZIO_PRIORITY_SYNC_WRITE, vps->vps_flags, B_TRUE));
1713 		} else {
1714 			abd_free(zio->io_abd);
1715 		}
1716 	} else if (zio->io_type == ZIO_TYPE_WRITE) {
1717 		if (zio->io_error == 0)
1718 			vps->vps_writeable = 1;
1719 		abd_free(zio->io_abd);
1720 	} else if (zio->io_type == ZIO_TYPE_NULL) {
1721 		zio_t *pio;
1722 		zio_link_t *zl;
1723 
1724 		vd->vdev_cant_read |= !vps->vps_readable;
1725 		vd->vdev_cant_write |= !vps->vps_writeable;
1726 		vdev_dbgmsg(vd, "probe done, cant_read=%u cant_write=%u",
1727 		    vd->vdev_cant_read, vd->vdev_cant_write);
1728 
1729 		if (vdev_readable(vd) &&
1730 		    (vdev_writeable(vd) || !spa_writeable(spa))) {
1731 			zio->io_error = 0;
1732 		} else {
1733 			ASSERT(zio->io_error != 0);
1734 			vdev_dbgmsg(vd, "failed probe");
1735 			(void) zfs_ereport_post(FM_EREPORT_ZFS_PROBE_FAILURE,
1736 			    spa, vd, NULL, NULL, 0);
1737 			zio->io_error = SET_ERROR(ENXIO);
1738 
1739 			/*
1740 			 * If this probe was initiated from zio pipeline, then
1741 			 * change the state in a spa_async_request. Probes that
1742 			 * were initiated from a vdev_open can change the state
1743 			 * as part of the open call.
1744 			 */
1745 			if (vps->vps_zio_done_probe) {
1746 				vd->vdev_fault_wanted = B_TRUE;
1747 				spa_async_request(spa, SPA_ASYNC_FAULT_VDEV);
1748 			}
1749 		}
1750 
1751 		mutex_enter(&vd->vdev_probe_lock);
1752 		ASSERT(vd->vdev_probe_zio == zio);
1753 		vd->vdev_probe_zio = NULL;
1754 		mutex_exit(&vd->vdev_probe_lock);
1755 
1756 		zl = NULL;
1757 		while ((pio = zio_walk_parents(zio, &zl)) != NULL)
1758 			if (!vdev_accessible(vd, pio))
1759 				pio->io_error = SET_ERROR(ENXIO);
1760 
1761 		kmem_free(vps, sizeof (*vps));
1762 	}
1763 }
1764 
1765 /*
1766  * Determine whether this device is accessible.
1767  *
1768  * Read and write to several known locations: the pad regions of each
1769  * vdev label but the first, which we leave alone in case it contains
1770  * a VTOC.
1771  */
1772 zio_t *
1773 vdev_probe(vdev_t *vd, zio_t *zio)
1774 {
1775 	spa_t *spa = vd->vdev_spa;
1776 	vdev_probe_stats_t *vps = NULL;
1777 	zio_t *pio;
1778 
1779 	ASSERT(vd->vdev_ops->vdev_op_leaf);
1780 
1781 	/*
1782 	 * Don't probe the probe.
1783 	 */
1784 	if (zio && (zio->io_flags & ZIO_FLAG_PROBE))
1785 		return (NULL);
1786 
1787 	/*
1788 	 * To prevent 'probe storms' when a device fails, we create
1789 	 * just one probe i/o at a time.  All zios that want to probe
1790 	 * this vdev will become parents of the probe io.
1791 	 */
1792 	mutex_enter(&vd->vdev_probe_lock);
1793 
1794 	if ((pio = vd->vdev_probe_zio) == NULL) {
1795 		vps = kmem_zalloc(sizeof (*vps), KM_SLEEP);
1796 
1797 		vps->vps_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_PROBE |
1798 		    ZIO_FLAG_DONT_AGGREGATE | ZIO_FLAG_TRYHARD;
1799 		vps->vps_zio_done_probe = (zio != NULL);
1800 
1801 		if (spa_config_held(spa, SCL_ZIO, RW_WRITER)) {
1802 			/*
1803 			 * vdev_cant_read and vdev_cant_write can only
1804 			 * transition from TRUE to FALSE when we have the
1805 			 * SCL_ZIO lock as writer; otherwise they can only
1806 			 * transition from FALSE to TRUE.  This ensures that
1807 			 * any zio looking at these values can assume that
1808 			 * failures persist for the life of the I/O.  That's
1809 			 * important because when a device has intermittent
1810 			 * connectivity problems, we want to ensure that
1811 			 * they're ascribed to the device (ENXIO) and not
1812 			 * the zio (EIO).
1813 			 *
1814 			 * Since we hold SCL_ZIO as writer here, clear both
1815 			 * values so the probe can reevaluate from first
1816 			 * principles.
1817 			 */
1818 			vps->vps_flags |= ZIO_FLAG_CONFIG_WRITER;
1819 			vd->vdev_cant_read = B_FALSE;
1820 			vd->vdev_cant_write = B_FALSE;
1821 		}
1822 
1823 		vd->vdev_probe_zio = pio = zio_null(NULL, spa, vd,
1824 		    vdev_probe_done, vps,
1825 		    vps->vps_flags | ZIO_FLAG_DONT_PROPAGATE);
1826 	}
1827 
1828 	if (zio != NULL)
1829 		zio_add_child(zio, pio);
1830 
1831 	mutex_exit(&vd->vdev_probe_lock);
1832 
1833 	if (vps == NULL) {
1834 		ASSERT(zio != NULL);
1835 		return (NULL);
1836 	}
1837 
1838 	for (int l = 1; l < VDEV_LABELS; l++) {
1839 		zio_nowait(zio_read_phys(pio, vd,
1840 		    vdev_label_offset(vd->vdev_psize, l,
1841 		    offsetof(vdev_label_t, vl_be)), VDEV_PAD_SIZE,
1842 		    abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE),
1843 		    ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
1844 		    ZIO_PRIORITY_SYNC_READ, vps->vps_flags, B_TRUE));
1845 	}
1846 
1847 	if (zio == NULL)
1848 		return (pio);
1849 
1850 	zio_nowait(pio);
1851 	return (NULL);
1852 }
1853 
1854 static void
1855 vdev_load_child(void *arg)
1856 {
1857 	vdev_t *vd = arg;
1858 
1859 	vd->vdev_load_error = vdev_load(vd);
1860 }
1861 
1862 static void
1863 vdev_open_child(void *arg)
1864 {
1865 	vdev_t *vd = arg;
1866 
1867 	vd->vdev_open_thread = curthread;
1868 	vd->vdev_open_error = vdev_open(vd);
1869 	vd->vdev_open_thread = NULL;
1870 }
1871 
1872 static boolean_t
1873 vdev_uses_zvols(vdev_t *vd)
1874 {
1875 #ifdef _KERNEL
1876 	if (zvol_is_zvol(vd->vdev_path))
1877 		return (B_TRUE);
1878 #endif
1879 
1880 	for (int c = 0; c < vd->vdev_children; c++)
1881 		if (vdev_uses_zvols(vd->vdev_child[c]))
1882 			return (B_TRUE);
1883 
1884 	return (B_FALSE);
1885 }
1886 
1887 /*
1888  * Returns B_TRUE if the passed child should be opened.
1889  */
1890 static boolean_t
1891 vdev_default_open_children_func(vdev_t *vd)
1892 {
1893 	(void) vd;
1894 	return (B_TRUE);
1895 }
1896 
1897 /*
1898  * Open the requested child vdevs.  If any of the leaf vdevs are using
1899  * a ZFS volume then do the opens in a single thread.  This avoids a
1900  * deadlock when the current thread is holding the spa_namespace_lock.
1901  */
1902 static void
1903 vdev_open_children_impl(vdev_t *vd, vdev_open_children_func_t *open_func)
1904 {
1905 	int children = vd->vdev_children;
1906 
1907 	taskq_t *tq = taskq_create("vdev_open", children, minclsyspri,
1908 	    children, children, TASKQ_PREPOPULATE);
1909 	vd->vdev_nonrot = B_TRUE;
1910 
1911 	for (int c = 0; c < children; c++) {
1912 		vdev_t *cvd = vd->vdev_child[c];
1913 
1914 		if (open_func(cvd) == B_FALSE)
1915 			continue;
1916 
1917 		if (tq == NULL || vdev_uses_zvols(vd)) {
1918 			cvd->vdev_open_error = vdev_open(cvd);
1919 		} else {
1920 			VERIFY(taskq_dispatch(tq, vdev_open_child,
1921 			    cvd, TQ_SLEEP) != TASKQID_INVALID);
1922 		}
1923 
1924 		vd->vdev_nonrot &= cvd->vdev_nonrot;
1925 	}
1926 
1927 	if (tq != NULL) {
1928 		taskq_wait(tq);
1929 		taskq_destroy(tq);
1930 	}
1931 }
1932 
1933 /*
1934  * Open all child vdevs.
1935  */
1936 void
1937 vdev_open_children(vdev_t *vd)
1938 {
1939 	vdev_open_children_impl(vd, vdev_default_open_children_func);
1940 }
1941 
1942 /*
1943  * Conditionally open a subset of child vdevs.
1944  */
1945 void
1946 vdev_open_children_subset(vdev_t *vd, vdev_open_children_func_t *open_func)
1947 {
1948 	vdev_open_children_impl(vd, open_func);
1949 }
1950 
1951 /*
1952  * Compute the raidz-deflation ratio.  Note, we hard-code 128k (1 << 17)
1953  * because it is the "typical" blocksize.  Even though SPA_MAXBLOCKSIZE
1954  * changed, this algorithm can not change, otherwise it would inconsistently
1955  * account for existing bp's.  We also hard-code txg 0 for the same reason
1956  * since expanded RAIDZ vdevs can use a different asize for different birth
1957  * txg's.
1958  */
1959 static void
1960 vdev_set_deflate_ratio(vdev_t *vd)
1961 {
1962 	if (vd == vd->vdev_top && !vd->vdev_ishole && vd->vdev_ashift != 0) {
1963 		vd->vdev_deflate_ratio = (1 << 17) /
1964 		    (vdev_psize_to_asize_txg(vd, 1 << 17, 0) >>
1965 		    SPA_MINBLOCKSHIFT);
1966 	}
1967 }
1968 
1969 /*
1970  * Choose the best of two ashifts, preferring one between logical ashift
1971  * (absolute minimum) and administrator defined maximum, otherwise take
1972  * the biggest of the two.
1973  */
1974 uint64_t
1975 vdev_best_ashift(uint64_t logical, uint64_t a, uint64_t b)
1976 {
1977 	if (a > logical && a <= zfs_vdev_max_auto_ashift) {
1978 		if (b <= logical || b > zfs_vdev_max_auto_ashift)
1979 			return (a);
1980 		else
1981 			return (MAX(a, b));
1982 	} else if (b <= logical || b > zfs_vdev_max_auto_ashift)
1983 		return (MAX(a, b));
1984 	return (b);
1985 }
1986 
1987 /*
1988  * Maximize performance by inflating the configured ashift for top level
1989  * vdevs to be as close to the physical ashift as possible while maintaining
1990  * administrator defined limits and ensuring it doesn't go below the
1991  * logical ashift.
1992  */
1993 static void
1994 vdev_ashift_optimize(vdev_t *vd)
1995 {
1996 	ASSERT(vd == vd->vdev_top);
1997 
1998 	if (vd->vdev_ashift < vd->vdev_physical_ashift &&
1999 	    vd->vdev_physical_ashift <= zfs_vdev_max_auto_ashift) {
2000 		vd->vdev_ashift = MIN(
2001 		    MAX(zfs_vdev_max_auto_ashift, vd->vdev_ashift),
2002 		    MAX(zfs_vdev_min_auto_ashift,
2003 		    vd->vdev_physical_ashift));
2004 	} else {
2005 		/*
2006 		 * If the logical and physical ashifts are the same, then
2007 		 * we ensure that the top-level vdev's ashift is not smaller
2008 		 * than our minimum ashift value. For the unusual case
2009 		 * where logical ashift > physical ashift, we can't cap
2010 		 * the calculated ashift based on max ashift as that
2011 		 * would cause failures.
2012 		 * We still check if we need to increase it to match
2013 		 * the min ashift.
2014 		 */
2015 		vd->vdev_ashift = MAX(zfs_vdev_min_auto_ashift,
2016 		    vd->vdev_ashift);
2017 	}
2018 }
2019 
2020 /*
2021  * Prepare a virtual device for access.
2022  */
2023 int
2024 vdev_open(vdev_t *vd)
2025 {
2026 	spa_t *spa = vd->vdev_spa;
2027 	int error;
2028 	uint64_t osize = 0;
2029 	uint64_t max_osize = 0;
2030 	uint64_t asize, max_asize, psize;
2031 	uint64_t logical_ashift = 0;
2032 	uint64_t physical_ashift = 0;
2033 
2034 	ASSERT(vd->vdev_open_thread == curthread ||
2035 	    spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
2036 	ASSERT(vd->vdev_state == VDEV_STATE_CLOSED ||
2037 	    vd->vdev_state == VDEV_STATE_CANT_OPEN ||
2038 	    vd->vdev_state == VDEV_STATE_OFFLINE);
2039 
2040 	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
2041 	vd->vdev_cant_read = B_FALSE;
2042 	vd->vdev_cant_write = B_FALSE;
2043 	vd->vdev_fault_wanted = B_FALSE;
2044 	vd->vdev_min_asize = vdev_get_min_asize(vd);
2045 
2046 	/*
2047 	 * If this vdev is not removed, check its fault status.  If it's
2048 	 * faulted, bail out of the open.
2049 	 */
2050 	if (!vd->vdev_removed && vd->vdev_faulted) {
2051 		ASSERT(vd->vdev_children == 0);
2052 		ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED ||
2053 		    vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
2054 		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
2055 		    vd->vdev_label_aux);
2056 		return (SET_ERROR(ENXIO));
2057 	} else if (vd->vdev_offline) {
2058 		ASSERT(vd->vdev_children == 0);
2059 		vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE, VDEV_AUX_NONE);
2060 		return (SET_ERROR(ENXIO));
2061 	}
2062 
2063 	error = vd->vdev_ops->vdev_op_open(vd, &osize, &max_osize,
2064 	    &logical_ashift, &physical_ashift);
2065 
2066 	/* Keep the device in removed state if unplugged */
2067 	if (error == ENOENT && vd->vdev_removed) {
2068 		vdev_set_state(vd, B_TRUE, VDEV_STATE_REMOVED,
2069 		    VDEV_AUX_NONE);
2070 		return (error);
2071 	}
2072 
2073 	/*
2074 	 * Physical volume size should never be larger than its max size, unless
2075 	 * the disk has shrunk while we were reading it or the device is buggy
2076 	 * or damaged: either way it's not safe for use, bail out of the open.
2077 	 */
2078 	if (osize > max_osize) {
2079 		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2080 		    VDEV_AUX_OPEN_FAILED);
2081 		return (SET_ERROR(ENXIO));
2082 	}
2083 
2084 	/*
2085 	 * Reset the vdev_reopening flag so that we actually close
2086 	 * the vdev on error.
2087 	 */
2088 	vd->vdev_reopening = B_FALSE;
2089 	if (zio_injection_enabled && error == 0)
2090 		error = zio_handle_device_injection(vd, NULL, SET_ERROR(ENXIO));
2091 
2092 	if (error) {
2093 		if (vd->vdev_removed &&
2094 		    vd->vdev_stat.vs_aux != VDEV_AUX_OPEN_FAILED)
2095 			vd->vdev_removed = B_FALSE;
2096 
2097 		if (vd->vdev_stat.vs_aux == VDEV_AUX_CHILDREN_OFFLINE) {
2098 			vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE,
2099 			    vd->vdev_stat.vs_aux);
2100 		} else {
2101 			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2102 			    vd->vdev_stat.vs_aux);
2103 		}
2104 		return (error);
2105 	}
2106 
2107 	vd->vdev_removed = B_FALSE;
2108 
2109 	/*
2110 	 * Recheck the faulted flag now that we have confirmed that
2111 	 * the vdev is accessible.  If we're faulted, bail.
2112 	 */
2113 	if (vd->vdev_faulted) {
2114 		ASSERT(vd->vdev_children == 0);
2115 		ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED ||
2116 		    vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
2117 		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
2118 		    vd->vdev_label_aux);
2119 		return (SET_ERROR(ENXIO));
2120 	}
2121 
2122 	if (vd->vdev_degraded) {
2123 		ASSERT(vd->vdev_children == 0);
2124 		vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
2125 		    VDEV_AUX_ERR_EXCEEDED);
2126 	} else {
2127 		vdev_set_state(vd, B_TRUE, VDEV_STATE_HEALTHY, 0);
2128 	}
2129 
2130 	/*
2131 	 * For hole or missing vdevs we just return success.
2132 	 */
2133 	if (vd->vdev_ishole || vd->vdev_ops == &vdev_missing_ops)
2134 		return (0);
2135 
2136 	for (int c = 0; c < vd->vdev_children; c++) {
2137 		if (vd->vdev_child[c]->vdev_state != VDEV_STATE_HEALTHY) {
2138 			vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
2139 			    VDEV_AUX_NONE);
2140 			break;
2141 		}
2142 	}
2143 
2144 	osize = P2ALIGN_TYPED(osize, sizeof (vdev_label_t), uint64_t);
2145 	max_osize = P2ALIGN_TYPED(max_osize, sizeof (vdev_label_t), uint64_t);
2146 
2147 	if (vd->vdev_children == 0) {
2148 		if (osize < SPA_MINDEVSIZE) {
2149 			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2150 			    VDEV_AUX_TOO_SMALL);
2151 			return (SET_ERROR(EOVERFLOW));
2152 		}
2153 		psize = osize;
2154 		asize = osize - (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE);
2155 		max_asize = max_osize - (VDEV_LABEL_START_SIZE +
2156 		    VDEV_LABEL_END_SIZE);
2157 	} else {
2158 		if (vd->vdev_parent != NULL && osize < SPA_MINDEVSIZE -
2159 		    (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE)) {
2160 			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2161 			    VDEV_AUX_TOO_SMALL);
2162 			return (SET_ERROR(EOVERFLOW));
2163 		}
2164 		psize = 0;
2165 		asize = osize;
2166 		max_asize = max_osize;
2167 	}
2168 
2169 	/*
2170 	 * If the vdev was expanded, record this so that we can re-create the
2171 	 * uberblock rings in labels {2,3}, during the next sync.
2172 	 */
2173 	if ((psize > vd->vdev_psize) && (vd->vdev_psize != 0))
2174 		vd->vdev_copy_uberblocks = B_TRUE;
2175 
2176 	vd->vdev_psize = psize;
2177 
2178 	/*
2179 	 * Make sure the allocatable size hasn't shrunk too much.
2180 	 */
2181 	if (asize < vd->vdev_min_asize) {
2182 		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2183 		    VDEV_AUX_BAD_LABEL);
2184 		return (SET_ERROR(EINVAL));
2185 	}
2186 
2187 	/*
2188 	 * We can always set the logical/physical ashift members since
2189 	 * their values are only used to calculate the vdev_ashift when
2190 	 * the device is first added to the config. These values should
2191 	 * not be used for anything else since they may change whenever
2192 	 * the device is reopened and we don't store them in the label.
2193 	 */
2194 	vd->vdev_physical_ashift =
2195 	    MAX(physical_ashift, vd->vdev_physical_ashift);
2196 	vd->vdev_logical_ashift = MAX(logical_ashift,
2197 	    vd->vdev_logical_ashift);
2198 
2199 	if (vd->vdev_asize == 0) {
2200 		/*
2201 		 * This is the first-ever open, so use the computed values.
2202 		 * For compatibility, a different ashift can be requested.
2203 		 */
2204 		vd->vdev_asize = asize;
2205 		vd->vdev_max_asize = max_asize;
2206 
2207 		/*
2208 		 * If the vdev_ashift was not overridden at creation time,
2209 		 * then set it the logical ashift and optimize the ashift.
2210 		 */
2211 		if (vd->vdev_ashift == 0) {
2212 			vd->vdev_ashift = vd->vdev_logical_ashift;
2213 
2214 			if (vd->vdev_logical_ashift > ASHIFT_MAX) {
2215 				vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2216 				    VDEV_AUX_ASHIFT_TOO_BIG);
2217 				return (SET_ERROR(EDOM));
2218 			}
2219 
2220 			if (vd->vdev_top == vd && vd->vdev_attaching == B_FALSE)
2221 				vdev_ashift_optimize(vd);
2222 			vd->vdev_attaching = B_FALSE;
2223 		}
2224 		if (vd->vdev_ashift != 0 && (vd->vdev_ashift < ASHIFT_MIN ||
2225 		    vd->vdev_ashift > ASHIFT_MAX)) {
2226 			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2227 			    VDEV_AUX_BAD_ASHIFT);
2228 			return (SET_ERROR(EDOM));
2229 		}
2230 	} else {
2231 		/*
2232 		 * Make sure the alignment required hasn't increased.
2233 		 */
2234 		if (vd->vdev_ashift > vd->vdev_top->vdev_ashift &&
2235 		    vd->vdev_ops->vdev_op_leaf) {
2236 			(void) zfs_ereport_post(
2237 			    FM_EREPORT_ZFS_DEVICE_BAD_ASHIFT,
2238 			    spa, vd, NULL, NULL, 0);
2239 			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2240 			    VDEV_AUX_BAD_LABEL);
2241 			return (SET_ERROR(EDOM));
2242 		}
2243 		vd->vdev_max_asize = max_asize;
2244 	}
2245 
2246 	/*
2247 	 * If all children are healthy we update asize if either:
2248 	 * The asize has increased, due to a device expansion caused by dynamic
2249 	 * LUN growth or vdev replacement, and automatic expansion is enabled;
2250 	 * making the additional space available.
2251 	 *
2252 	 * The asize has decreased, due to a device shrink usually caused by a
2253 	 * vdev replace with a smaller device. This ensures that calculations
2254 	 * based of max_asize and asize e.g. esize are always valid. It's safe
2255 	 * to do this as we've already validated that asize is greater than
2256 	 * vdev_min_asize.
2257 	 */
2258 	if (vd->vdev_state == VDEV_STATE_HEALTHY &&
2259 	    ((asize > vd->vdev_asize &&
2260 	    (vd->vdev_expanding || spa->spa_autoexpand)) ||
2261 	    (asize < vd->vdev_asize)))
2262 		vd->vdev_asize = asize;
2263 
2264 	vdev_set_min_asize(vd);
2265 
2266 	/*
2267 	 * Ensure we can issue some IO before declaring the
2268 	 * vdev open for business.
2269 	 */
2270 	if (vd->vdev_ops->vdev_op_leaf &&
2271 	    (error = zio_wait(vdev_probe(vd, NULL))) != 0) {
2272 		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
2273 		    VDEV_AUX_ERR_EXCEEDED);
2274 		return (error);
2275 	}
2276 
2277 	/*
2278 	 * Track the minimum allocation size.
2279 	 */
2280 	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
2281 	    vd->vdev_islog == 0 && vd->vdev_aux == NULL) {
2282 		uint64_t min_alloc = vdev_get_min_alloc(vd);
2283 		vdev_spa_set_alloc(spa, min_alloc);
2284 	}
2285 
2286 	/*
2287 	 * If this is a leaf vdev, assess whether a resilver is needed.
2288 	 * But don't do this if we are doing a reopen for a scrub, since
2289 	 * this would just restart the scrub we are already doing.
2290 	 */
2291 	if (vd->vdev_ops->vdev_op_leaf && !spa->spa_scrub_reopen)
2292 		dsl_scan_assess_vdev(spa->spa_dsl_pool, vd);
2293 
2294 	return (0);
2295 }
2296 
2297 static void
2298 vdev_validate_child(void *arg)
2299 {
2300 	vdev_t *vd = arg;
2301 
2302 	vd->vdev_validate_thread = curthread;
2303 	vd->vdev_validate_error = vdev_validate(vd);
2304 	vd->vdev_validate_thread = NULL;
2305 }
2306 
2307 /*
2308  * Called once the vdevs are all opened, this routine validates the label
2309  * contents. This needs to be done before vdev_load() so that we don't
2310  * inadvertently do repair I/Os to the wrong device.
2311  *
2312  * This function will only return failure if one of the vdevs indicates that it
2313  * has since been destroyed or exported.  This is only possible if
2314  * /etc/zfs/zpool.cache was readonly at the time.  Otherwise, the vdev state
2315  * will be updated but the function will return 0.
2316  */
2317 int
2318 vdev_validate(vdev_t *vd)
2319 {
2320 	spa_t *spa = vd->vdev_spa;
2321 	taskq_t *tq = NULL;
2322 	nvlist_t *label;
2323 	uint64_t guid = 0, aux_guid = 0, top_guid;
2324 	uint64_t state;
2325 	nvlist_t *nvl;
2326 	uint64_t txg;
2327 	int children = vd->vdev_children;
2328 
2329 	if (vdev_validate_skip)
2330 		return (0);
2331 
2332 	if (children > 0) {
2333 		tq = taskq_create("vdev_validate", children, minclsyspri,
2334 		    children, children, TASKQ_PREPOPULATE);
2335 	}
2336 
2337 	for (uint64_t c = 0; c < children; c++) {
2338 		vdev_t *cvd = vd->vdev_child[c];
2339 
2340 		if (tq == NULL || vdev_uses_zvols(cvd)) {
2341 			vdev_validate_child(cvd);
2342 		} else {
2343 			VERIFY(taskq_dispatch(tq, vdev_validate_child, cvd,
2344 			    TQ_SLEEP) != TASKQID_INVALID);
2345 		}
2346 	}
2347 	if (tq != NULL) {
2348 		taskq_wait(tq);
2349 		taskq_destroy(tq);
2350 	}
2351 	for (int c = 0; c < children; c++) {
2352 		int error = vd->vdev_child[c]->vdev_validate_error;
2353 
2354 		if (error != 0)
2355 			return (SET_ERROR(EBADF));
2356 	}
2357 
2358 
2359 	/*
2360 	 * If the device has already failed, or was marked offline, don't do
2361 	 * any further validation.  Otherwise, label I/O will fail and we will
2362 	 * overwrite the previous state.
2363 	 */
2364 	if (!vd->vdev_ops->vdev_op_leaf || !vdev_readable(vd))
2365 		return (0);
2366 
2367 	/*
2368 	 * If we are performing an extreme rewind, we allow for a label that
2369 	 * was modified at a point after the current txg.
2370 	 * If config lock is not held do not check for the txg. spa_sync could
2371 	 * be updating the vdev's label before updating spa_last_synced_txg.
2372 	 */
2373 	if (spa->spa_extreme_rewind || spa_last_synced_txg(spa) == 0 ||
2374 	    spa_config_held(spa, SCL_CONFIG, RW_WRITER) != SCL_CONFIG)
2375 		txg = UINT64_MAX;
2376 	else
2377 		txg = spa_last_synced_txg(spa);
2378 
2379 	if ((label = vdev_label_read_config(vd, txg)) == NULL) {
2380 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2381 		    VDEV_AUX_BAD_LABEL);
2382 		vdev_dbgmsg(vd, "vdev_validate: failed reading config for "
2383 		    "txg %llu", (u_longlong_t)txg);
2384 		return (0);
2385 	}
2386 
2387 	/*
2388 	 * Determine if this vdev has been split off into another
2389 	 * pool.  If so, then refuse to open it.
2390 	 */
2391 	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_SPLIT_GUID,
2392 	    &aux_guid) == 0 && aux_guid == spa_guid(spa)) {
2393 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2394 		    VDEV_AUX_SPLIT_POOL);
2395 		nvlist_free(label);
2396 		vdev_dbgmsg(vd, "vdev_validate: vdev split into other pool");
2397 		return (0);
2398 	}
2399 
2400 	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &guid) != 0) {
2401 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2402 		    VDEV_AUX_CORRUPT_DATA);
2403 		nvlist_free(label);
2404 		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
2405 		    ZPOOL_CONFIG_POOL_GUID);
2406 		return (0);
2407 	}
2408 
2409 	/*
2410 	 * If config is not trusted then ignore the spa guid check. This is
2411 	 * necessary because if the machine crashed during a re-guid the new
2412 	 * guid might have been written to all of the vdev labels, but not the
2413 	 * cached config. The check will be performed again once we have the
2414 	 * trusted config from the MOS.
2415 	 */
2416 	if (spa->spa_trust_config && guid != spa_guid(spa)) {
2417 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2418 		    VDEV_AUX_CORRUPT_DATA);
2419 		nvlist_free(label);
2420 		vdev_dbgmsg(vd, "vdev_validate: vdev label pool_guid doesn't "
2421 		    "match config (%llu != %llu)", (u_longlong_t)guid,
2422 		    (u_longlong_t)spa_guid(spa));
2423 		return (0);
2424 	}
2425 
2426 	if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_VDEV_TREE, &nvl)
2427 	    != 0 || nvlist_lookup_uint64(nvl, ZPOOL_CONFIG_ORIG_GUID,
2428 	    &aux_guid) != 0)
2429 		aux_guid = 0;
2430 
2431 	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0) {
2432 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2433 		    VDEV_AUX_CORRUPT_DATA);
2434 		nvlist_free(label);
2435 		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
2436 		    ZPOOL_CONFIG_GUID);
2437 		return (0);
2438 	}
2439 
2440 	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_TOP_GUID, &top_guid)
2441 	    != 0) {
2442 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2443 		    VDEV_AUX_CORRUPT_DATA);
2444 		nvlist_free(label);
2445 		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
2446 		    ZPOOL_CONFIG_TOP_GUID);
2447 		return (0);
2448 	}
2449 
2450 	/*
2451 	 * If this vdev just became a top-level vdev because its sibling was
2452 	 * detached, it will have adopted the parent's vdev guid -- but the
2453 	 * label may or may not be on disk yet. Fortunately, either version
2454 	 * of the label will have the same top guid, so if we're a top-level
2455 	 * vdev, we can safely compare to that instead.
2456 	 * However, if the config comes from a cachefile that failed to update
2457 	 * after the detach, a top-level vdev will appear as a non top-level
2458 	 * vdev in the config. Also relax the constraints if we perform an
2459 	 * extreme rewind.
2460 	 *
2461 	 * If we split this vdev off instead, then we also check the
2462 	 * original pool's guid. We don't want to consider the vdev
2463 	 * corrupt if it is partway through a split operation.
2464 	 */
2465 	if (vd->vdev_guid != guid && vd->vdev_guid != aux_guid) {
2466 		boolean_t mismatch = B_FALSE;
2467 		if (spa->spa_trust_config && !spa->spa_extreme_rewind) {
2468 			if (vd != vd->vdev_top || vd->vdev_guid != top_guid)
2469 				mismatch = B_TRUE;
2470 		} else {
2471 			if (vd->vdev_guid != top_guid &&
2472 			    vd->vdev_top->vdev_guid != guid)
2473 				mismatch = B_TRUE;
2474 		}
2475 
2476 		if (mismatch) {
2477 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2478 			    VDEV_AUX_CORRUPT_DATA);
2479 			nvlist_free(label);
2480 			vdev_dbgmsg(vd, "vdev_validate: config guid "
2481 			    "doesn't match label guid");
2482 			vdev_dbgmsg(vd, "CONFIG: guid %llu, top_guid %llu",
2483 			    (u_longlong_t)vd->vdev_guid,
2484 			    (u_longlong_t)vd->vdev_top->vdev_guid);
2485 			vdev_dbgmsg(vd, "LABEL: guid %llu, top_guid %llu, "
2486 			    "aux_guid %llu", (u_longlong_t)guid,
2487 			    (u_longlong_t)top_guid, (u_longlong_t)aux_guid);
2488 			return (0);
2489 		}
2490 	}
2491 
2492 	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE,
2493 	    &state) != 0) {
2494 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2495 		    VDEV_AUX_CORRUPT_DATA);
2496 		nvlist_free(label);
2497 		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
2498 		    ZPOOL_CONFIG_POOL_STATE);
2499 		return (0);
2500 	}
2501 
2502 	nvlist_free(label);
2503 
2504 	/*
2505 	 * If this is a verbatim import, no need to check the
2506 	 * state of the pool.
2507 	 */
2508 	if (!(spa->spa_import_flags & ZFS_IMPORT_VERBATIM) &&
2509 	    spa_load_state(spa) == SPA_LOAD_OPEN &&
2510 	    state != POOL_STATE_ACTIVE) {
2511 		vdev_dbgmsg(vd, "vdev_validate: invalid pool state (%llu) "
2512 		    "for spa %s", (u_longlong_t)state, spa->spa_name);
2513 		return (SET_ERROR(EBADF));
2514 	}
2515 
2516 	/*
2517 	 * If we were able to open and validate a vdev that was
2518 	 * previously marked permanently unavailable, clear that state
2519 	 * now.
2520 	 */
2521 	if (vd->vdev_not_present)
2522 		vd->vdev_not_present = 0;
2523 
2524 	return (0);
2525 }
2526 
2527 static void
2528 vdev_update_path(const char *prefix, char *svd, char **dvd, uint64_t guid)
2529 {
2530 	if (svd != NULL && *dvd != NULL) {
2531 		if (strcmp(svd, *dvd) != 0) {
2532 			zfs_dbgmsg("vdev_copy_path: vdev %llu: %s changed "
2533 			    "from '%s' to '%s'", (u_longlong_t)guid, prefix,
2534 			    *dvd, svd);
2535 			spa_strfree(*dvd);
2536 			*dvd = spa_strdup(svd);
2537 		}
2538 	} else if (svd != NULL) {
2539 		*dvd = spa_strdup(svd);
2540 		zfs_dbgmsg("vdev_copy_path: vdev %llu: path set to '%s'",
2541 		    (u_longlong_t)guid, *dvd);
2542 	}
2543 }
2544 
2545 static void
2546 vdev_copy_path_impl(vdev_t *svd, vdev_t *dvd)
2547 {
2548 	char *old, *new;
2549 
2550 	vdev_update_path("vdev_path", svd->vdev_path, &dvd->vdev_path,
2551 	    dvd->vdev_guid);
2552 
2553 	vdev_update_path("vdev_devid", svd->vdev_devid, &dvd->vdev_devid,
2554 	    dvd->vdev_guid);
2555 
2556 	vdev_update_path("vdev_physpath", svd->vdev_physpath,
2557 	    &dvd->vdev_physpath, dvd->vdev_guid);
2558 
2559 	/*
2560 	 * Our enclosure sysfs path may have changed between imports
2561 	 */
2562 	old = dvd->vdev_enc_sysfs_path;
2563 	new = svd->vdev_enc_sysfs_path;
2564 	if ((old != NULL && new == NULL) ||
2565 	    (old == NULL && new != NULL) ||
2566 	    ((old != NULL && new != NULL) && strcmp(new, old) != 0)) {
2567 		zfs_dbgmsg("vdev_copy_path: vdev %llu: vdev_enc_sysfs_path "
2568 		    "changed from '%s' to '%s'", (u_longlong_t)dvd->vdev_guid,
2569 		    old, new);
2570 
2571 		if (dvd->vdev_enc_sysfs_path)
2572 			spa_strfree(dvd->vdev_enc_sysfs_path);
2573 
2574 		if (svd->vdev_enc_sysfs_path) {
2575 			dvd->vdev_enc_sysfs_path = spa_strdup(
2576 			    svd->vdev_enc_sysfs_path);
2577 		} else {
2578 			dvd->vdev_enc_sysfs_path = NULL;
2579 		}
2580 	}
2581 }
2582 
2583 /*
2584  * Recursively copy vdev paths from one vdev to another. Source and destination
2585  * vdev trees must have same geometry otherwise return error. Intended to copy
2586  * paths from userland config into MOS config.
2587  */
2588 int
2589 vdev_copy_path_strict(vdev_t *svd, vdev_t *dvd)
2590 {
2591 	if ((svd->vdev_ops == &vdev_missing_ops) ||
2592 	    (svd->vdev_ishole && dvd->vdev_ishole) ||
2593 	    (dvd->vdev_ops == &vdev_indirect_ops))
2594 		return (0);
2595 
2596 	if (svd->vdev_ops != dvd->vdev_ops) {
2597 		vdev_dbgmsg(svd, "vdev_copy_path: vdev type mismatch: %s != %s",
2598 		    svd->vdev_ops->vdev_op_type, dvd->vdev_ops->vdev_op_type);
2599 		return (SET_ERROR(EINVAL));
2600 	}
2601 
2602 	if (svd->vdev_guid != dvd->vdev_guid) {
2603 		vdev_dbgmsg(svd, "vdev_copy_path: guids mismatch (%llu != "
2604 		    "%llu)", (u_longlong_t)svd->vdev_guid,
2605 		    (u_longlong_t)dvd->vdev_guid);
2606 		return (SET_ERROR(EINVAL));
2607 	}
2608 
2609 	if (svd->vdev_children != dvd->vdev_children) {
2610 		vdev_dbgmsg(svd, "vdev_copy_path: children count mismatch: "
2611 		    "%llu != %llu", (u_longlong_t)svd->vdev_children,
2612 		    (u_longlong_t)dvd->vdev_children);
2613 		return (SET_ERROR(EINVAL));
2614 	}
2615 
2616 	for (uint64_t i = 0; i < svd->vdev_children; i++) {
2617 		int error = vdev_copy_path_strict(svd->vdev_child[i],
2618 		    dvd->vdev_child[i]);
2619 		if (error != 0)
2620 			return (error);
2621 	}
2622 
2623 	if (svd->vdev_ops->vdev_op_leaf)
2624 		vdev_copy_path_impl(svd, dvd);
2625 
2626 	return (0);
2627 }
2628 
2629 static void
2630 vdev_copy_path_search(vdev_t *stvd, vdev_t *dvd)
2631 {
2632 	ASSERT(stvd->vdev_top == stvd);
2633 	ASSERT3U(stvd->vdev_id, ==, dvd->vdev_top->vdev_id);
2634 
2635 	for (uint64_t i = 0; i < dvd->vdev_children; i++) {
2636 		vdev_copy_path_search(stvd, dvd->vdev_child[i]);
2637 	}
2638 
2639 	if (!dvd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(dvd))
2640 		return;
2641 
2642 	/*
2643 	 * The idea here is that while a vdev can shift positions within
2644 	 * a top vdev (when replacing, attaching mirror, etc.) it cannot
2645 	 * step outside of it.
2646 	 */
2647 	vdev_t *vd = vdev_lookup_by_guid(stvd, dvd->vdev_guid);
2648 
2649 	if (vd == NULL || vd->vdev_ops != dvd->vdev_ops)
2650 		return;
2651 
2652 	ASSERT(vd->vdev_ops->vdev_op_leaf);
2653 
2654 	vdev_copy_path_impl(vd, dvd);
2655 }
2656 
2657 /*
2658  * Recursively copy vdev paths from one root vdev to another. Source and
2659  * destination vdev trees may differ in geometry. For each destination leaf
2660  * vdev, search a vdev with the same guid and top vdev id in the source.
2661  * Intended to copy paths from userland config into MOS config.
2662  */
2663 void
2664 vdev_copy_path_relaxed(vdev_t *srvd, vdev_t *drvd)
2665 {
2666 	uint64_t children = MIN(srvd->vdev_children, drvd->vdev_children);
2667 	ASSERT(srvd->vdev_ops == &vdev_root_ops);
2668 	ASSERT(drvd->vdev_ops == &vdev_root_ops);
2669 
2670 	for (uint64_t i = 0; i < children; i++) {
2671 		vdev_copy_path_search(srvd->vdev_child[i],
2672 		    drvd->vdev_child[i]);
2673 	}
2674 }
2675 
2676 /*
2677  * Close a virtual device.
2678  */
2679 void
2680 vdev_close(vdev_t *vd)
2681 {
2682 	vdev_t *pvd = vd->vdev_parent;
2683 	spa_t *spa __maybe_unused = vd->vdev_spa;
2684 
2685 	ASSERT(vd != NULL);
2686 	ASSERT(vd->vdev_open_thread == curthread ||
2687 	    spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
2688 
2689 	/*
2690 	 * If our parent is reopening, then we are as well, unless we are
2691 	 * going offline.
2692 	 */
2693 	if (pvd != NULL && pvd->vdev_reopening)
2694 		vd->vdev_reopening = (pvd->vdev_reopening && !vd->vdev_offline);
2695 
2696 	vd->vdev_ops->vdev_op_close(vd);
2697 
2698 	/*
2699 	 * We record the previous state before we close it, so that if we are
2700 	 * doing a reopen(), we don't generate FMA ereports if we notice that
2701 	 * it's still faulted.
2702 	 */
2703 	vd->vdev_prevstate = vd->vdev_state;
2704 
2705 	if (vd->vdev_offline)
2706 		vd->vdev_state = VDEV_STATE_OFFLINE;
2707 	else
2708 		vd->vdev_state = VDEV_STATE_CLOSED;
2709 	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
2710 }
2711 
2712 void
2713 vdev_hold(vdev_t *vd)
2714 {
2715 	spa_t *spa = vd->vdev_spa;
2716 
2717 	ASSERT(spa_is_root(spa));
2718 	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
2719 		return;
2720 
2721 	for (int c = 0; c < vd->vdev_children; c++)
2722 		vdev_hold(vd->vdev_child[c]);
2723 
2724 	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_ops->vdev_op_hold != NULL)
2725 		vd->vdev_ops->vdev_op_hold(vd);
2726 }
2727 
2728 void
2729 vdev_rele(vdev_t *vd)
2730 {
2731 	ASSERT(spa_is_root(vd->vdev_spa));
2732 	for (int c = 0; c < vd->vdev_children; c++)
2733 		vdev_rele(vd->vdev_child[c]);
2734 
2735 	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_ops->vdev_op_rele != NULL)
2736 		vd->vdev_ops->vdev_op_rele(vd);
2737 }
2738 
2739 /*
2740  * Reopen all interior vdevs and any unopened leaves.  We don't actually
2741  * reopen leaf vdevs which had previously been opened as they might deadlock
2742  * on the spa_config_lock.  Instead we only obtain the leaf's physical size.
2743  * If the leaf has never been opened then open it, as usual.
2744  */
2745 void
2746 vdev_reopen(vdev_t *vd)
2747 {
2748 	spa_t *spa = vd->vdev_spa;
2749 
2750 	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
2751 
2752 	/* set the reopening flag unless we're taking the vdev offline */
2753 	vd->vdev_reopening = !vd->vdev_offline;
2754 	vdev_close(vd);
2755 	(void) vdev_open(vd);
2756 
2757 	/*
2758 	 * Call vdev_validate() here to make sure we have the same device.
2759 	 * Otherwise, a device with an invalid label could be successfully
2760 	 * opened in response to vdev_reopen().
2761 	 */
2762 	if (vd->vdev_aux) {
2763 		(void) vdev_validate_aux(vd);
2764 		if (vdev_readable(vd) && vdev_writeable(vd) &&
2765 		    vd->vdev_aux == &spa->spa_l2cache) {
2766 			/*
2767 			 * In case the vdev is present we should evict all ARC
2768 			 * buffers and pointers to log blocks and reclaim their
2769 			 * space before restoring its contents to L2ARC.
2770 			 */
2771 			if (l2arc_vdev_present(vd)) {
2772 				l2arc_rebuild_vdev(vd, B_TRUE);
2773 			} else {
2774 				l2arc_add_vdev(spa, vd);
2775 			}
2776 			spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD);
2777 			spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM);
2778 		}
2779 	} else {
2780 		(void) vdev_validate(vd);
2781 	}
2782 
2783 	/*
2784 	 * Recheck if resilver is still needed and cancel any
2785 	 * scheduled resilver if resilver is unneeded.
2786 	 */
2787 	if (!vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL) &&
2788 	    spa->spa_async_tasks & SPA_ASYNC_RESILVER) {
2789 		mutex_enter(&spa->spa_async_lock);
2790 		spa->spa_async_tasks &= ~SPA_ASYNC_RESILVER;
2791 		mutex_exit(&spa->spa_async_lock);
2792 	}
2793 
2794 	/*
2795 	 * Reassess parent vdev's health.
2796 	 */
2797 	vdev_propagate_state(vd);
2798 }
2799 
2800 int
2801 vdev_create(vdev_t *vd, uint64_t txg, boolean_t isreplacing)
2802 {
2803 	int error;
2804 
2805 	/*
2806 	 * Normally, partial opens (e.g. of a mirror) are allowed.
2807 	 * For a create, however, we want to fail the request if
2808 	 * there are any components we can't open.
2809 	 */
2810 	error = vdev_open(vd);
2811 
2812 	if (error || vd->vdev_state != VDEV_STATE_HEALTHY) {
2813 		vdev_close(vd);
2814 		return (error ? error : SET_ERROR(ENXIO));
2815 	}
2816 
2817 	/*
2818 	 * Recursively load DTLs and initialize all labels.
2819 	 */
2820 	if ((error = vdev_dtl_load(vd)) != 0 ||
2821 	    (error = vdev_label_init(vd, txg, isreplacing ?
2822 	    VDEV_LABEL_REPLACE : VDEV_LABEL_CREATE)) != 0) {
2823 		vdev_close(vd);
2824 		return (error);
2825 	}
2826 
2827 	return (0);
2828 }
2829 
2830 void
2831 vdev_metaslab_set_size(vdev_t *vd)
2832 {
2833 	uint64_t asize = vd->vdev_asize;
2834 	uint64_t ms_count = asize >> zfs_vdev_default_ms_shift;
2835 	uint64_t ms_shift;
2836 
2837 	/*
2838 	 * There are two dimensions to the metaslab sizing calculation:
2839 	 * the size of the metaslab and the count of metaslabs per vdev.
2840 	 *
2841 	 * The default values used below are a good balance between memory
2842 	 * usage (larger metaslab size means more memory needed for loaded
2843 	 * metaslabs; more metaslabs means more memory needed for the
2844 	 * metaslab_t structs), metaslab load time (larger metaslabs take
2845 	 * longer to load), and metaslab sync time (more metaslabs means
2846 	 * more time spent syncing all of them).
2847 	 *
2848 	 * In general, we aim for zfs_vdev_default_ms_count (200) metaslabs.
2849 	 * The range of the dimensions are as follows:
2850 	 *
2851 	 *	2^29 <= ms_size  <= 2^34
2852 	 *	  16 <= ms_count <= 131,072
2853 	 *
2854 	 * On the lower end of vdev sizes, we aim for metaslabs sizes of
2855 	 * at least 512MB (2^29) to minimize fragmentation effects when
2856 	 * testing with smaller devices.  However, the count constraint
2857 	 * of at least 16 metaslabs will override this minimum size goal.
2858 	 *
2859 	 * On the upper end of vdev sizes, we aim for a maximum metaslab
2860 	 * size of 16GB.  However, we will cap the total count to 2^17
2861 	 * metaslabs to keep our memory footprint in check and let the
2862 	 * metaslab size grow from there if that limit is hit.
2863 	 *
2864 	 * The net effect of applying above constrains is summarized below.
2865 	 *
2866 	 *   vdev size       metaslab count
2867 	 *  --------------|-----------------
2868 	 *      < 8GB        ~16
2869 	 *  8GB   - 100GB   one per 512MB
2870 	 *  100GB - 3TB     ~200
2871 	 *  3TB   - 2PB     one per 16GB
2872 	 *      > 2PB       ~131,072
2873 	 *  --------------------------------
2874 	 *
2875 	 *  Finally, note that all of the above calculate the initial
2876 	 *  number of metaslabs. Expanding a top-level vdev will result
2877 	 *  in additional metaslabs being allocated making it possible
2878 	 *  to exceed the zfs_vdev_ms_count_limit.
2879 	 */
2880 
2881 	if (ms_count < zfs_vdev_min_ms_count)
2882 		ms_shift = highbit64(asize / zfs_vdev_min_ms_count);
2883 	else if (ms_count > zfs_vdev_default_ms_count)
2884 		ms_shift = highbit64(asize / zfs_vdev_default_ms_count);
2885 	else
2886 		ms_shift = zfs_vdev_default_ms_shift;
2887 
2888 	if (ms_shift < SPA_MAXBLOCKSHIFT) {
2889 		ms_shift = SPA_MAXBLOCKSHIFT;
2890 	} else if (ms_shift > zfs_vdev_max_ms_shift) {
2891 		ms_shift = zfs_vdev_max_ms_shift;
2892 		/* cap the total count to constrain memory footprint */
2893 		if ((asize >> ms_shift) > zfs_vdev_ms_count_limit)
2894 			ms_shift = highbit64(asize / zfs_vdev_ms_count_limit);
2895 	}
2896 
2897 	vd->vdev_ms_shift = ms_shift;
2898 	ASSERT3U(vd->vdev_ms_shift, >=, SPA_MAXBLOCKSHIFT);
2899 }
2900 
2901 void
2902 vdev_dirty(vdev_t *vd, int flags, void *arg, uint64_t txg)
2903 {
2904 	ASSERT(vd == vd->vdev_top);
2905 	/* indirect vdevs don't have metaslabs or dtls */
2906 	ASSERT(vdev_is_concrete(vd) || flags == 0);
2907 	ASSERT(ISP2(flags));
2908 	ASSERT(spa_writeable(vd->vdev_spa));
2909 
2910 	if (flags & VDD_METASLAB)
2911 		(void) txg_list_add(&vd->vdev_ms_list, arg, txg);
2912 
2913 	if (flags & VDD_DTL)
2914 		(void) txg_list_add(&vd->vdev_dtl_list, arg, txg);
2915 
2916 	(void) txg_list_add(&vd->vdev_spa->spa_vdev_txg_list, vd, txg);
2917 }
2918 
2919 void
2920 vdev_dirty_leaves(vdev_t *vd, int flags, uint64_t txg)
2921 {
2922 	for (int c = 0; c < vd->vdev_children; c++)
2923 		vdev_dirty_leaves(vd->vdev_child[c], flags, txg);
2924 
2925 	if (vd->vdev_ops->vdev_op_leaf)
2926 		vdev_dirty(vd->vdev_top, flags, vd, txg);
2927 }
2928 
2929 /*
2930  * DTLs.
2931  *
2932  * A vdev's DTL (dirty time log) is the set of transaction groups for which
2933  * the vdev has less than perfect replication.  There are four kinds of DTL:
2934  *
2935  * DTL_MISSING: txgs for which the vdev has no valid copies of the data
2936  *
2937  * DTL_PARTIAL: txgs for which data is available, but not fully replicated
2938  *
2939  * DTL_SCRUB: the txgs that could not be repaired by the last scrub; upon
2940  *	scrub completion, DTL_SCRUB replaces DTL_MISSING in the range of
2941  *	txgs that was scrubbed.
2942  *
2943  * DTL_OUTAGE: txgs which cannot currently be read, whether due to
2944  *	persistent errors or just some device being offline.
2945  *	Unlike the other three, the DTL_OUTAGE map is not generally
2946  *	maintained; it's only computed when needed, typically to
2947  *	determine whether a device can be detached.
2948  *
2949  * For leaf vdevs, DTL_MISSING and DTL_PARTIAL are identical: the device
2950  * either has the data or it doesn't.
2951  *
2952  * For interior vdevs such as mirror and RAID-Z the picture is more complex.
2953  * A vdev's DTL_PARTIAL is the union of its children's DTL_PARTIALs, because
2954  * if any child is less than fully replicated, then so is its parent.
2955  * A vdev's DTL_MISSING is a modified union of its children's DTL_MISSINGs,
2956  * comprising only those txgs which appear in 'maxfaults' or more children;
2957  * those are the txgs we don't have enough replication to read.  For example,
2958  * double-parity RAID-Z can tolerate up to two missing devices (maxfaults == 2);
2959  * thus, its DTL_MISSING consists of the set of txgs that appear in more than
2960  * two child DTL_MISSING maps.
2961  *
2962  * It should be clear from the above that to compute the DTLs and outage maps
2963  * for all vdevs, it suffices to know just the leaf vdevs' DTL_MISSING maps.
2964  * Therefore, that is all we keep on disk.  When loading the pool, or after
2965  * a configuration change, we generate all other DTLs from first principles.
2966  */
2967 void
2968 vdev_dtl_dirty(vdev_t *vd, vdev_dtl_type_t t, uint64_t txg, uint64_t size)
2969 {
2970 	range_tree_t *rt = vd->vdev_dtl[t];
2971 
2972 	ASSERT(t < DTL_TYPES);
2973 	ASSERT(vd != vd->vdev_spa->spa_root_vdev);
2974 	ASSERT(spa_writeable(vd->vdev_spa));
2975 
2976 	mutex_enter(&vd->vdev_dtl_lock);
2977 	if (!range_tree_contains(rt, txg, size))
2978 		range_tree_add(rt, txg, size);
2979 	mutex_exit(&vd->vdev_dtl_lock);
2980 }
2981 
2982 boolean_t
2983 vdev_dtl_contains(vdev_t *vd, vdev_dtl_type_t t, uint64_t txg, uint64_t size)
2984 {
2985 	range_tree_t *rt = vd->vdev_dtl[t];
2986 	boolean_t dirty = B_FALSE;
2987 
2988 	ASSERT(t < DTL_TYPES);
2989 	ASSERT(vd != vd->vdev_spa->spa_root_vdev);
2990 
2991 	/*
2992 	 * While we are loading the pool, the DTLs have not been loaded yet.
2993 	 * This isn't a problem but it can result in devices being tried
2994 	 * which are known to not have the data.  In which case, the import
2995 	 * is relying on the checksum to ensure that we get the right data.
2996 	 * Note that while importing we are only reading the MOS, which is
2997 	 * always checksummed.
2998 	 */
2999 	mutex_enter(&vd->vdev_dtl_lock);
3000 	if (!range_tree_is_empty(rt))
3001 		dirty = range_tree_contains(rt, txg, size);
3002 	mutex_exit(&vd->vdev_dtl_lock);
3003 
3004 	return (dirty);
3005 }
3006 
3007 boolean_t
3008 vdev_dtl_empty(vdev_t *vd, vdev_dtl_type_t t)
3009 {
3010 	range_tree_t *rt = vd->vdev_dtl[t];
3011 	boolean_t empty;
3012 
3013 	mutex_enter(&vd->vdev_dtl_lock);
3014 	empty = range_tree_is_empty(rt);
3015 	mutex_exit(&vd->vdev_dtl_lock);
3016 
3017 	return (empty);
3018 }
3019 
3020 /*
3021  * Check if the txg falls within the range which must be
3022  * resilvered.  DVAs outside this range can always be skipped.
3023  */
3024 boolean_t
3025 vdev_default_need_resilver(vdev_t *vd, const dva_t *dva, size_t psize,
3026     uint64_t phys_birth)
3027 {
3028 	(void) dva, (void) psize;
3029 
3030 	/* Set by sequential resilver. */
3031 	if (phys_birth == TXG_UNKNOWN)
3032 		return (B_TRUE);
3033 
3034 	return (vdev_dtl_contains(vd, DTL_PARTIAL, phys_birth, 1));
3035 }
3036 
3037 /*
3038  * Returns B_TRUE if the vdev determines the DVA needs to be resilvered.
3039  */
3040 boolean_t
3041 vdev_dtl_need_resilver(vdev_t *vd, const dva_t *dva, size_t psize,
3042     uint64_t phys_birth)
3043 {
3044 	ASSERT(vd != vd->vdev_spa->spa_root_vdev);
3045 
3046 	if (vd->vdev_ops->vdev_op_need_resilver == NULL ||
3047 	    vd->vdev_ops->vdev_op_leaf)
3048 		return (B_TRUE);
3049 
3050 	return (vd->vdev_ops->vdev_op_need_resilver(vd, dva, psize,
3051 	    phys_birth));
3052 }
3053 
3054 /*
3055  * Returns the lowest txg in the DTL range.
3056  */
3057 static uint64_t
3058 vdev_dtl_min(vdev_t *vd)
3059 {
3060 	ASSERT(MUTEX_HELD(&vd->vdev_dtl_lock));
3061 	ASSERT3U(range_tree_space(vd->vdev_dtl[DTL_MISSING]), !=, 0);
3062 	ASSERT0(vd->vdev_children);
3063 
3064 	return (range_tree_min(vd->vdev_dtl[DTL_MISSING]) - 1);
3065 }
3066 
3067 /*
3068  * Returns the highest txg in the DTL.
3069  */
3070 static uint64_t
3071 vdev_dtl_max(vdev_t *vd)
3072 {
3073 	ASSERT(MUTEX_HELD(&vd->vdev_dtl_lock));
3074 	ASSERT3U(range_tree_space(vd->vdev_dtl[DTL_MISSING]), !=, 0);
3075 	ASSERT0(vd->vdev_children);
3076 
3077 	return (range_tree_max(vd->vdev_dtl[DTL_MISSING]));
3078 }
3079 
3080 /*
3081  * Determine if a resilvering vdev should remove any DTL entries from
3082  * its range. If the vdev was resilvering for the entire duration of the
3083  * scan then it should excise that range from its DTLs. Otherwise, this
3084  * vdev is considered partially resilvered and should leave its DTL
3085  * entries intact. The comment in vdev_dtl_reassess() describes how we
3086  * excise the DTLs.
3087  */
3088 static boolean_t
3089 vdev_dtl_should_excise(vdev_t *vd, boolean_t rebuild_done)
3090 {
3091 	ASSERT0(vd->vdev_children);
3092 
3093 	if (vd->vdev_state < VDEV_STATE_DEGRADED)
3094 		return (B_FALSE);
3095 
3096 	if (vd->vdev_resilver_deferred)
3097 		return (B_FALSE);
3098 
3099 	if (range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]))
3100 		return (B_TRUE);
3101 
3102 	if (rebuild_done) {
3103 		vdev_rebuild_t *vr = &vd->vdev_top->vdev_rebuild_config;
3104 		vdev_rebuild_phys_t *vrp = &vr->vr_rebuild_phys;
3105 
3106 		/* Rebuild not initiated by attach */
3107 		if (vd->vdev_rebuild_txg == 0)
3108 			return (B_TRUE);
3109 
3110 		/*
3111 		 * When a rebuild completes without error then all missing data
3112 		 * up to the rebuild max txg has been reconstructed and the DTL
3113 		 * is eligible for excision.
3114 		 */
3115 		if (vrp->vrp_rebuild_state == VDEV_REBUILD_COMPLETE &&
3116 		    vdev_dtl_max(vd) <= vrp->vrp_max_txg) {
3117 			ASSERT3U(vrp->vrp_min_txg, <=, vdev_dtl_min(vd));
3118 			ASSERT3U(vrp->vrp_min_txg, <, vd->vdev_rebuild_txg);
3119 			ASSERT3U(vd->vdev_rebuild_txg, <=, vrp->vrp_max_txg);
3120 			return (B_TRUE);
3121 		}
3122 	} else {
3123 		dsl_scan_t *scn = vd->vdev_spa->spa_dsl_pool->dp_scan;
3124 		dsl_scan_phys_t *scnp __maybe_unused = &scn->scn_phys;
3125 
3126 		/* Resilver not initiated by attach */
3127 		if (vd->vdev_resilver_txg == 0)
3128 			return (B_TRUE);
3129 
3130 		/*
3131 		 * When a resilver is initiated the scan will assign the
3132 		 * scn_max_txg value to the highest txg value that exists
3133 		 * in all DTLs. If this device's max DTL is not part of this
3134 		 * scan (i.e. it is not in the range (scn_min_txg, scn_max_txg]
3135 		 * then it is not eligible for excision.
3136 		 */
3137 		if (vdev_dtl_max(vd) <= scn->scn_phys.scn_max_txg) {
3138 			ASSERT3U(scnp->scn_min_txg, <=, vdev_dtl_min(vd));
3139 			ASSERT3U(scnp->scn_min_txg, <, vd->vdev_resilver_txg);
3140 			ASSERT3U(vd->vdev_resilver_txg, <=, scnp->scn_max_txg);
3141 			return (B_TRUE);
3142 		}
3143 	}
3144 
3145 	return (B_FALSE);
3146 }
3147 
3148 /*
3149  * Reassess DTLs after a config change or scrub completion. If txg == 0 no
3150  * write operations will be issued to the pool.
3151  */
3152 static void
3153 vdev_dtl_reassess_impl(vdev_t *vd, uint64_t txg, uint64_t scrub_txg,
3154     boolean_t scrub_done, boolean_t rebuild_done, boolean_t faulting)
3155 {
3156 	spa_t *spa = vd->vdev_spa;
3157 	avl_tree_t reftree;
3158 	int minref;
3159 
3160 	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
3161 
3162 	for (int c = 0; c < vd->vdev_children; c++)
3163 		vdev_dtl_reassess_impl(vd->vdev_child[c], txg,
3164 		    scrub_txg, scrub_done, rebuild_done, faulting);
3165 
3166 	if (vd == spa->spa_root_vdev || !vdev_is_concrete(vd) || vd->vdev_aux)
3167 		return;
3168 
3169 	if (vd->vdev_ops->vdev_op_leaf) {
3170 		dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
3171 		vdev_rebuild_t *vr = &vd->vdev_top->vdev_rebuild_config;
3172 		boolean_t check_excise = B_FALSE;
3173 		boolean_t wasempty = B_TRUE;
3174 
3175 		mutex_enter(&vd->vdev_dtl_lock);
3176 
3177 		/*
3178 		 * If requested, pretend the scan or rebuild completed cleanly.
3179 		 */
3180 		if (zfs_scan_ignore_errors) {
3181 			if (scn != NULL)
3182 				scn->scn_phys.scn_errors = 0;
3183 			if (vr != NULL)
3184 				vr->vr_rebuild_phys.vrp_errors = 0;
3185 		}
3186 
3187 		if (scrub_txg != 0 &&
3188 		    !range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) {
3189 			wasempty = B_FALSE;
3190 			zfs_dbgmsg("guid:%llu txg:%llu scrub:%llu started:%d "
3191 			    "dtl:%llu/%llu errors:%llu",
3192 			    (u_longlong_t)vd->vdev_guid, (u_longlong_t)txg,
3193 			    (u_longlong_t)scrub_txg, spa->spa_scrub_started,
3194 			    (u_longlong_t)vdev_dtl_min(vd),
3195 			    (u_longlong_t)vdev_dtl_max(vd),
3196 			    (u_longlong_t)(scn ? scn->scn_phys.scn_errors : 0));
3197 		}
3198 
3199 		/*
3200 		 * If we've completed a scrub/resilver or a rebuild cleanly
3201 		 * then determine if this vdev should remove any DTLs. We
3202 		 * only want to excise regions on vdevs that were available
3203 		 * during the entire duration of this scan.
3204 		 */
3205 		if (rebuild_done &&
3206 		    vr != NULL && vr->vr_rebuild_phys.vrp_errors == 0) {
3207 			check_excise = B_TRUE;
3208 		} else {
3209 			if (spa->spa_scrub_started ||
3210 			    (scn != NULL && scn->scn_phys.scn_errors == 0)) {
3211 				check_excise = B_TRUE;
3212 			}
3213 		}
3214 
3215 		if (scrub_txg && check_excise &&
3216 		    vdev_dtl_should_excise(vd, rebuild_done)) {
3217 			/*
3218 			 * We completed a scrub, resilver or rebuild up to
3219 			 * scrub_txg.  If we did it without rebooting, then
3220 			 * the scrub dtl will be valid, so excise the old
3221 			 * region and fold in the scrub dtl.  Otherwise,
3222 			 * leave the dtl as-is if there was an error.
3223 			 *
3224 			 * There's little trick here: to excise the beginning
3225 			 * of the DTL_MISSING map, we put it into a reference
3226 			 * tree and then add a segment with refcnt -1 that
3227 			 * covers the range [0, scrub_txg).  This means
3228 			 * that each txg in that range has refcnt -1 or 0.
3229 			 * We then add DTL_SCRUB with a refcnt of 2, so that
3230 			 * entries in the range [0, scrub_txg) will have a
3231 			 * positive refcnt -- either 1 or 2.  We then convert
3232 			 * the reference tree into the new DTL_MISSING map.
3233 			 */
3234 			space_reftree_create(&reftree);
3235 			space_reftree_add_map(&reftree,
3236 			    vd->vdev_dtl[DTL_MISSING], 1);
3237 			space_reftree_add_seg(&reftree, 0, scrub_txg, -1);
3238 			space_reftree_add_map(&reftree,
3239 			    vd->vdev_dtl[DTL_SCRUB], 2);
3240 			space_reftree_generate_map(&reftree,
3241 			    vd->vdev_dtl[DTL_MISSING], 1);
3242 			space_reftree_destroy(&reftree);
3243 
3244 			if (!range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) {
3245 				zfs_dbgmsg("update DTL_MISSING:%llu/%llu",
3246 				    (u_longlong_t)vdev_dtl_min(vd),
3247 				    (u_longlong_t)vdev_dtl_max(vd));
3248 			} else if (!wasempty) {
3249 				zfs_dbgmsg("DTL_MISSING is now empty");
3250 			}
3251 		}
3252 		range_tree_vacate(vd->vdev_dtl[DTL_PARTIAL], NULL, NULL);
3253 		range_tree_walk(vd->vdev_dtl[DTL_MISSING],
3254 		    range_tree_add, vd->vdev_dtl[DTL_PARTIAL]);
3255 		if (scrub_done)
3256 			range_tree_vacate(vd->vdev_dtl[DTL_SCRUB], NULL, NULL);
3257 		range_tree_vacate(vd->vdev_dtl[DTL_OUTAGE], NULL, NULL);
3258 
3259 		/*
3260 		 * For the faulting case, treat members of a replacing vdev
3261 		 * as if they are not available. It's more likely than not that
3262 		 * a vdev in a replacing vdev could encounter read errors so
3263 		 * treat it as not being able to contribute.
3264 		 */
3265 		if (!vdev_readable(vd) ||
3266 		    (faulting && vd->vdev_parent != NULL &&
3267 		    vd->vdev_parent->vdev_ops == &vdev_replacing_ops)) {
3268 			range_tree_add(vd->vdev_dtl[DTL_OUTAGE], 0, -1ULL);
3269 		} else {
3270 			range_tree_walk(vd->vdev_dtl[DTL_MISSING],
3271 			    range_tree_add, vd->vdev_dtl[DTL_OUTAGE]);
3272 		}
3273 
3274 		/*
3275 		 * If the vdev was resilvering or rebuilding and no longer
3276 		 * has any DTLs then reset the appropriate flag and dirty
3277 		 * the top level so that we persist the change.
3278 		 */
3279 		if (txg != 0 &&
3280 		    range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]) &&
3281 		    range_tree_is_empty(vd->vdev_dtl[DTL_OUTAGE])) {
3282 			if (vd->vdev_rebuild_txg != 0) {
3283 				vd->vdev_rebuild_txg = 0;
3284 				vdev_config_dirty(vd->vdev_top);
3285 			} else if (vd->vdev_resilver_txg != 0) {
3286 				vd->vdev_resilver_txg = 0;
3287 				vdev_config_dirty(vd->vdev_top);
3288 			}
3289 		}
3290 
3291 		mutex_exit(&vd->vdev_dtl_lock);
3292 
3293 		if (txg != 0)
3294 			vdev_dirty(vd->vdev_top, VDD_DTL, vd, txg);
3295 	} else {
3296 		mutex_enter(&vd->vdev_dtl_lock);
3297 		for (int t = 0; t < DTL_TYPES; t++) {
3298 			/* account for child's outage in parent's missing map */
3299 			int s = (t == DTL_MISSING) ? DTL_OUTAGE: t;
3300 			if (t == DTL_SCRUB) {
3301 				/* leaf vdevs only */
3302 				continue;
3303 			}
3304 			if (t == DTL_PARTIAL) {
3305 				/* i.e. non-zero */
3306 				minref = 1;
3307 			} else if (vdev_get_nparity(vd) != 0) {
3308 				/* RAIDZ, DRAID */
3309 				minref = vdev_get_nparity(vd) + 1;
3310 			} else {
3311 				/* any kind of mirror */
3312 				minref = vd->vdev_children;
3313 			}
3314 			space_reftree_create(&reftree);
3315 			for (int c = 0; c < vd->vdev_children; c++) {
3316 				vdev_t *cvd = vd->vdev_child[c];
3317 				mutex_enter(&cvd->vdev_dtl_lock);
3318 				space_reftree_add_map(&reftree,
3319 				    cvd->vdev_dtl[s], 1);
3320 				mutex_exit(&cvd->vdev_dtl_lock);
3321 			}
3322 			space_reftree_generate_map(&reftree,
3323 			    vd->vdev_dtl[t], minref);
3324 			space_reftree_destroy(&reftree);
3325 		}
3326 		mutex_exit(&vd->vdev_dtl_lock);
3327 	}
3328 
3329 	if (vd->vdev_top->vdev_ops == &vdev_raidz_ops) {
3330 		raidz_dtl_reassessed(vd);
3331 	}
3332 }
3333 
3334 void
3335 vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg,
3336     boolean_t scrub_done, boolean_t rebuild_done)
3337 {
3338 	return (vdev_dtl_reassess_impl(vd, txg, scrub_txg, scrub_done,
3339 	    rebuild_done, B_FALSE));
3340 }
3341 
3342 /*
3343  * Iterate over all the vdevs except spare, and post kobj events
3344  */
3345 void
3346 vdev_post_kobj_evt(vdev_t *vd)
3347 {
3348 	if (vd->vdev_ops->vdev_op_kobj_evt_post &&
3349 	    vd->vdev_kobj_flag == B_FALSE) {
3350 		vd->vdev_kobj_flag = B_TRUE;
3351 		vd->vdev_ops->vdev_op_kobj_evt_post(vd);
3352 	}
3353 
3354 	for (int c = 0; c < vd->vdev_children; c++)
3355 		vdev_post_kobj_evt(vd->vdev_child[c]);
3356 }
3357 
3358 /*
3359  * Iterate over all the vdevs except spare, and clear kobj events
3360  */
3361 void
3362 vdev_clear_kobj_evt(vdev_t *vd)
3363 {
3364 	vd->vdev_kobj_flag = B_FALSE;
3365 
3366 	for (int c = 0; c < vd->vdev_children; c++)
3367 		vdev_clear_kobj_evt(vd->vdev_child[c]);
3368 }
3369 
3370 int
3371 vdev_dtl_load(vdev_t *vd)
3372 {
3373 	spa_t *spa = vd->vdev_spa;
3374 	objset_t *mos = spa->spa_meta_objset;
3375 	range_tree_t *rt;
3376 	int error = 0;
3377 
3378 	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_dtl_object != 0) {
3379 		ASSERT(vdev_is_concrete(vd));
3380 
3381 		/*
3382 		 * If the dtl cannot be sync'd there is no need to open it.
3383 		 */
3384 		if (spa->spa_mode == SPA_MODE_READ && !spa->spa_read_spacemaps)
3385 			return (0);
3386 
3387 		error = space_map_open(&vd->vdev_dtl_sm, mos,
3388 		    vd->vdev_dtl_object, 0, -1ULL, 0);
3389 		if (error)
3390 			return (error);
3391 		ASSERT(vd->vdev_dtl_sm != NULL);
3392 
3393 		rt = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
3394 		error = space_map_load(vd->vdev_dtl_sm, rt, SM_ALLOC);
3395 		if (error == 0) {
3396 			mutex_enter(&vd->vdev_dtl_lock);
3397 			range_tree_walk(rt, range_tree_add,
3398 			    vd->vdev_dtl[DTL_MISSING]);
3399 			mutex_exit(&vd->vdev_dtl_lock);
3400 		}
3401 
3402 		range_tree_vacate(rt, NULL, NULL);
3403 		range_tree_destroy(rt);
3404 
3405 		return (error);
3406 	}
3407 
3408 	for (int c = 0; c < vd->vdev_children; c++) {
3409 		error = vdev_dtl_load(vd->vdev_child[c]);
3410 		if (error != 0)
3411 			break;
3412 	}
3413 
3414 	return (error);
3415 }
3416 
3417 static void
3418 vdev_zap_allocation_data(vdev_t *vd, dmu_tx_t *tx)
3419 {
3420 	spa_t *spa = vd->vdev_spa;
3421 	objset_t *mos = spa->spa_meta_objset;
3422 	vdev_alloc_bias_t alloc_bias = vd->vdev_alloc_bias;
3423 	const char *string;
3424 
3425 	ASSERT(alloc_bias != VDEV_BIAS_NONE);
3426 
3427 	string =
3428 	    (alloc_bias == VDEV_BIAS_LOG) ? VDEV_ALLOC_BIAS_LOG :
3429 	    (alloc_bias == VDEV_BIAS_SPECIAL) ? VDEV_ALLOC_BIAS_SPECIAL :
3430 	    (alloc_bias == VDEV_BIAS_DEDUP) ? VDEV_ALLOC_BIAS_DEDUP : NULL;
3431 
3432 	ASSERT(string != NULL);
3433 	VERIFY0(zap_add(mos, vd->vdev_top_zap, VDEV_TOP_ZAP_ALLOCATION_BIAS,
3434 	    1, strlen(string) + 1, string, tx));
3435 
3436 	if (alloc_bias == VDEV_BIAS_SPECIAL || alloc_bias == VDEV_BIAS_DEDUP) {
3437 		spa_activate_allocation_classes(spa, tx);
3438 	}
3439 }
3440 
3441 void
3442 vdev_destroy_unlink_zap(vdev_t *vd, uint64_t zapobj, dmu_tx_t *tx)
3443 {
3444 	spa_t *spa = vd->vdev_spa;
3445 
3446 	VERIFY0(zap_destroy(spa->spa_meta_objset, zapobj, tx));
3447 	VERIFY0(zap_remove_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps,
3448 	    zapobj, tx));
3449 }
3450 
3451 uint64_t
3452 vdev_create_link_zap(vdev_t *vd, dmu_tx_t *tx)
3453 {
3454 	spa_t *spa = vd->vdev_spa;
3455 	uint64_t zap = zap_create(spa->spa_meta_objset, DMU_OTN_ZAP_METADATA,
3456 	    DMU_OT_NONE, 0, tx);
3457 
3458 	ASSERT(zap != 0);
3459 	VERIFY0(zap_add_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps,
3460 	    zap, tx));
3461 
3462 	return (zap);
3463 }
3464 
3465 void
3466 vdev_construct_zaps(vdev_t *vd, dmu_tx_t *tx)
3467 {
3468 	if (vd->vdev_ops != &vdev_hole_ops &&
3469 	    vd->vdev_ops != &vdev_missing_ops &&
3470 	    vd->vdev_ops != &vdev_root_ops &&
3471 	    !vd->vdev_top->vdev_removing) {
3472 		if (vd->vdev_ops->vdev_op_leaf && vd->vdev_leaf_zap == 0) {
3473 			vd->vdev_leaf_zap = vdev_create_link_zap(vd, tx);
3474 		}
3475 		if (vd == vd->vdev_top && vd->vdev_top_zap == 0) {
3476 			vd->vdev_top_zap = vdev_create_link_zap(vd, tx);
3477 			if (vd->vdev_alloc_bias != VDEV_BIAS_NONE)
3478 				vdev_zap_allocation_data(vd, tx);
3479 		}
3480 	}
3481 	if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap == 0 &&
3482 	    spa_feature_is_enabled(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) {
3483 		if (!spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2))
3484 			spa_feature_incr(vd->vdev_spa, SPA_FEATURE_AVZ_V2, tx);
3485 		vd->vdev_root_zap = vdev_create_link_zap(vd, tx);
3486 	}
3487 
3488 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
3489 		vdev_construct_zaps(vd->vdev_child[i], tx);
3490 	}
3491 }
3492 
3493 static void
3494 vdev_dtl_sync(vdev_t *vd, uint64_t txg)
3495 {
3496 	spa_t *spa = vd->vdev_spa;
3497 	range_tree_t *rt = vd->vdev_dtl[DTL_MISSING];
3498 	objset_t *mos = spa->spa_meta_objset;
3499 	range_tree_t *rtsync;
3500 	dmu_tx_t *tx;
3501 	uint64_t object = space_map_object(vd->vdev_dtl_sm);
3502 
3503 	ASSERT(vdev_is_concrete(vd));
3504 	ASSERT(vd->vdev_ops->vdev_op_leaf);
3505 
3506 	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
3507 
3508 	if (vd->vdev_detached || vd->vdev_top->vdev_removing) {
3509 		mutex_enter(&vd->vdev_dtl_lock);
3510 		space_map_free(vd->vdev_dtl_sm, tx);
3511 		space_map_close(vd->vdev_dtl_sm);
3512 		vd->vdev_dtl_sm = NULL;
3513 		mutex_exit(&vd->vdev_dtl_lock);
3514 
3515 		/*
3516 		 * We only destroy the leaf ZAP for detached leaves or for
3517 		 * removed log devices. Removed data devices handle leaf ZAP
3518 		 * cleanup later, once cancellation is no longer possible.
3519 		 */
3520 		if (vd->vdev_leaf_zap != 0 && (vd->vdev_detached ||
3521 		    vd->vdev_top->vdev_islog)) {
3522 			vdev_destroy_unlink_zap(vd, vd->vdev_leaf_zap, tx);
3523 			vd->vdev_leaf_zap = 0;
3524 		}
3525 
3526 		dmu_tx_commit(tx);
3527 		return;
3528 	}
3529 
3530 	if (vd->vdev_dtl_sm == NULL) {
3531 		uint64_t new_object;
3532 
3533 		new_object = space_map_alloc(mos, zfs_vdev_dtl_sm_blksz, tx);
3534 		VERIFY3U(new_object, !=, 0);
3535 
3536 		VERIFY0(space_map_open(&vd->vdev_dtl_sm, mos, new_object,
3537 		    0, -1ULL, 0));
3538 		ASSERT(vd->vdev_dtl_sm != NULL);
3539 	}
3540 
3541 	rtsync = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
3542 
3543 	mutex_enter(&vd->vdev_dtl_lock);
3544 	range_tree_walk(rt, range_tree_add, rtsync);
3545 	mutex_exit(&vd->vdev_dtl_lock);
3546 
3547 	space_map_truncate(vd->vdev_dtl_sm, zfs_vdev_dtl_sm_blksz, tx);
3548 	space_map_write(vd->vdev_dtl_sm, rtsync, SM_ALLOC, SM_NO_VDEVID, tx);
3549 	range_tree_vacate(rtsync, NULL, NULL);
3550 
3551 	range_tree_destroy(rtsync);
3552 
3553 	/*
3554 	 * If the object for the space map has changed then dirty
3555 	 * the top level so that we update the config.
3556 	 */
3557 	if (object != space_map_object(vd->vdev_dtl_sm)) {
3558 		vdev_dbgmsg(vd, "txg %llu, spa %s, DTL old object %llu, "
3559 		    "new object %llu", (u_longlong_t)txg, spa_name(spa),
3560 		    (u_longlong_t)object,
3561 		    (u_longlong_t)space_map_object(vd->vdev_dtl_sm));
3562 		vdev_config_dirty(vd->vdev_top);
3563 	}
3564 
3565 	dmu_tx_commit(tx);
3566 }
3567 
3568 /*
3569  * Determine whether the specified vdev can be
3570  * - offlined
3571  * - detached
3572  * - removed
3573  * - faulted
3574  * without losing data.
3575  */
3576 boolean_t
3577 vdev_dtl_required(vdev_t *vd)
3578 {
3579 	spa_t *spa = vd->vdev_spa;
3580 	vdev_t *tvd = vd->vdev_top;
3581 	uint8_t cant_read = vd->vdev_cant_read;
3582 	boolean_t required;
3583 	boolean_t faulting = vd->vdev_state == VDEV_STATE_FAULTED;
3584 
3585 	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
3586 
3587 	if (vd == spa->spa_root_vdev || vd == tvd)
3588 		return (B_TRUE);
3589 
3590 	/*
3591 	 * Temporarily mark the device as unreadable, and then determine
3592 	 * whether this results in any DTL outages in the top-level vdev.
3593 	 * If not, we can safely offline/detach/remove the device.
3594 	 */
3595 	vd->vdev_cant_read = B_TRUE;
3596 	vdev_dtl_reassess_impl(tvd, 0, 0, B_FALSE, B_FALSE, faulting);
3597 	required = !vdev_dtl_empty(tvd, DTL_OUTAGE);
3598 	vd->vdev_cant_read = cant_read;
3599 	vdev_dtl_reassess_impl(tvd, 0, 0, B_FALSE, B_FALSE, faulting);
3600 
3601 	if (!required && zio_injection_enabled) {
3602 		required = !!zio_handle_device_injection(vd, NULL,
3603 		    SET_ERROR(ECHILD));
3604 	}
3605 
3606 	return (required);
3607 }
3608 
3609 /*
3610  * Determine if resilver is needed, and if so the txg range.
3611  */
3612 boolean_t
3613 vdev_resilver_needed(vdev_t *vd, uint64_t *minp, uint64_t *maxp)
3614 {
3615 	boolean_t needed = B_FALSE;
3616 	uint64_t thismin = UINT64_MAX;
3617 	uint64_t thismax = 0;
3618 
3619 	if (vd->vdev_children == 0) {
3620 		mutex_enter(&vd->vdev_dtl_lock);
3621 		if (!range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]) &&
3622 		    vdev_writeable(vd)) {
3623 
3624 			thismin = vdev_dtl_min(vd);
3625 			thismax = vdev_dtl_max(vd);
3626 			needed = B_TRUE;
3627 		}
3628 		mutex_exit(&vd->vdev_dtl_lock);
3629 	} else {
3630 		for (int c = 0; c < vd->vdev_children; c++) {
3631 			vdev_t *cvd = vd->vdev_child[c];
3632 			uint64_t cmin, cmax;
3633 
3634 			if (vdev_resilver_needed(cvd, &cmin, &cmax)) {
3635 				thismin = MIN(thismin, cmin);
3636 				thismax = MAX(thismax, cmax);
3637 				needed = B_TRUE;
3638 			}
3639 		}
3640 	}
3641 
3642 	if (needed && minp) {
3643 		*minp = thismin;
3644 		*maxp = thismax;
3645 	}
3646 	return (needed);
3647 }
3648 
3649 /*
3650  * Gets the checkpoint space map object from the vdev's ZAP.  On success sm_obj
3651  * will contain either the checkpoint spacemap object or zero if none exists.
3652  * All other errors are returned to the caller.
3653  */
3654 int
3655 vdev_checkpoint_sm_object(vdev_t *vd, uint64_t *sm_obj)
3656 {
3657 	ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
3658 
3659 	if (vd->vdev_top_zap == 0) {
3660 		*sm_obj = 0;
3661 		return (0);
3662 	}
3663 
3664 	int error = zap_lookup(spa_meta_objset(vd->vdev_spa), vd->vdev_top_zap,
3665 	    VDEV_TOP_ZAP_POOL_CHECKPOINT_SM, sizeof (uint64_t), 1, sm_obj);
3666 	if (error == ENOENT) {
3667 		*sm_obj = 0;
3668 		error = 0;
3669 	}
3670 
3671 	return (error);
3672 }
3673 
3674 int
3675 vdev_load(vdev_t *vd)
3676 {
3677 	int children = vd->vdev_children;
3678 	int error = 0;
3679 	taskq_t *tq = NULL;
3680 
3681 	/*
3682 	 * It's only worthwhile to use the taskq for the root vdev, because the
3683 	 * slow part is metaslab_init, and that only happens for top-level
3684 	 * vdevs.
3685 	 */
3686 	if (vd->vdev_ops == &vdev_root_ops && vd->vdev_children > 0) {
3687 		tq = taskq_create("vdev_load", children, minclsyspri,
3688 		    children, children, TASKQ_PREPOPULATE);
3689 	}
3690 
3691 	/*
3692 	 * Recursively load all children.
3693 	 */
3694 	for (int c = 0; c < vd->vdev_children; c++) {
3695 		vdev_t *cvd = vd->vdev_child[c];
3696 
3697 		if (tq == NULL || vdev_uses_zvols(cvd)) {
3698 			cvd->vdev_load_error = vdev_load(cvd);
3699 		} else {
3700 			VERIFY(taskq_dispatch(tq, vdev_load_child,
3701 			    cvd, TQ_SLEEP) != TASKQID_INVALID);
3702 		}
3703 	}
3704 
3705 	if (tq != NULL) {
3706 		taskq_wait(tq);
3707 		taskq_destroy(tq);
3708 	}
3709 
3710 	for (int c = 0; c < vd->vdev_children; c++) {
3711 		int error = vd->vdev_child[c]->vdev_load_error;
3712 
3713 		if (error != 0)
3714 			return (error);
3715 	}
3716 
3717 	vdev_set_deflate_ratio(vd);
3718 
3719 	if (vd->vdev_ops == &vdev_raidz_ops) {
3720 		error = vdev_raidz_load(vd);
3721 		if (error != 0)
3722 			return (error);
3723 	}
3724 
3725 	/*
3726 	 * On spa_load path, grab the allocation bias from our zap
3727 	 */
3728 	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
3729 		spa_t *spa = vd->vdev_spa;
3730 		char bias_str[64];
3731 
3732 		error = zap_lookup(spa->spa_meta_objset, vd->vdev_top_zap,
3733 		    VDEV_TOP_ZAP_ALLOCATION_BIAS, 1, sizeof (bias_str),
3734 		    bias_str);
3735 		if (error == 0) {
3736 			ASSERT(vd->vdev_alloc_bias == VDEV_BIAS_NONE);
3737 			vd->vdev_alloc_bias = vdev_derive_alloc_bias(bias_str);
3738 		} else if (error != ENOENT) {
3739 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
3740 			    VDEV_AUX_CORRUPT_DATA);
3741 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(top_zap=%llu) "
3742 			    "failed [error=%d]",
3743 			    (u_longlong_t)vd->vdev_top_zap, error);
3744 			return (error);
3745 		}
3746 	}
3747 
3748 	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
3749 		spa_t *spa = vd->vdev_spa;
3750 		uint64_t failfast;
3751 
3752 		error = zap_lookup(spa->spa_meta_objset, vd->vdev_top_zap,
3753 		    vdev_prop_to_name(VDEV_PROP_FAILFAST), sizeof (failfast),
3754 		    1, &failfast);
3755 		if (error == 0) {
3756 			vd->vdev_failfast = failfast & 1;
3757 		} else if (error == ENOENT) {
3758 			vd->vdev_failfast = vdev_prop_default_numeric(
3759 			    VDEV_PROP_FAILFAST);
3760 		} else {
3761 			vdev_dbgmsg(vd,
3762 			    "vdev_load: zap_lookup(top_zap=%llu) "
3763 			    "failed [error=%d]",
3764 			    (u_longlong_t)vd->vdev_top_zap, error);
3765 		}
3766 	}
3767 
3768 	/*
3769 	 * Load any rebuild state from the top-level vdev zap.
3770 	 */
3771 	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
3772 		error = vdev_rebuild_load(vd);
3773 		if (error && error != ENOTSUP) {
3774 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
3775 			    VDEV_AUX_CORRUPT_DATA);
3776 			vdev_dbgmsg(vd, "vdev_load: vdev_rebuild_load "
3777 			    "failed [error=%d]", error);
3778 			return (error);
3779 		}
3780 	}
3781 
3782 	if (vd->vdev_top_zap != 0 || vd->vdev_leaf_zap != 0) {
3783 		uint64_t zapobj;
3784 
3785 		if (vd->vdev_top_zap != 0)
3786 			zapobj = vd->vdev_top_zap;
3787 		else
3788 			zapobj = vd->vdev_leaf_zap;
3789 
3790 		error = vdev_prop_get_int(vd, VDEV_PROP_CHECKSUM_N,
3791 		    &vd->vdev_checksum_n);
3792 		if (error && error != ENOENT)
3793 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
3794 			    "failed [error=%d]", (u_longlong_t)zapobj, error);
3795 
3796 		error = vdev_prop_get_int(vd, VDEV_PROP_CHECKSUM_T,
3797 		    &vd->vdev_checksum_t);
3798 		if (error && error != ENOENT)
3799 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
3800 			    "failed [error=%d]", (u_longlong_t)zapobj, error);
3801 
3802 		error = vdev_prop_get_int(vd, VDEV_PROP_IO_N,
3803 		    &vd->vdev_io_n);
3804 		if (error && error != ENOENT)
3805 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
3806 			    "failed [error=%d]", (u_longlong_t)zapobj, error);
3807 
3808 		error = vdev_prop_get_int(vd, VDEV_PROP_IO_T,
3809 		    &vd->vdev_io_t);
3810 		if (error && error != ENOENT)
3811 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
3812 			    "failed [error=%d]", (u_longlong_t)zapobj, error);
3813 
3814 		error = vdev_prop_get_int(vd, VDEV_PROP_SLOW_IO_N,
3815 		    &vd->vdev_slow_io_n);
3816 		if (error && error != ENOENT)
3817 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
3818 			    "failed [error=%d]", (u_longlong_t)zapobj, error);
3819 
3820 		error = vdev_prop_get_int(vd, VDEV_PROP_SLOW_IO_T,
3821 		    &vd->vdev_slow_io_t);
3822 		if (error && error != ENOENT)
3823 			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
3824 			    "failed [error=%d]", (u_longlong_t)zapobj, error);
3825 	}
3826 
3827 	/*
3828 	 * If this is a top-level vdev, initialize its metaslabs.
3829 	 */
3830 	if (vd == vd->vdev_top && vdev_is_concrete(vd)) {
3831 		vdev_metaslab_group_create(vd);
3832 
3833 		if (vd->vdev_ashift == 0 || vd->vdev_asize == 0) {
3834 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
3835 			    VDEV_AUX_CORRUPT_DATA);
3836 			vdev_dbgmsg(vd, "vdev_load: invalid size. ashift=%llu, "
3837 			    "asize=%llu", (u_longlong_t)vd->vdev_ashift,
3838 			    (u_longlong_t)vd->vdev_asize);
3839 			return (SET_ERROR(ENXIO));
3840 		}
3841 
3842 		error = vdev_metaslab_init(vd, 0);
3843 		if (error != 0) {
3844 			vdev_dbgmsg(vd, "vdev_load: metaslab_init failed "
3845 			    "[error=%d]", error);
3846 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
3847 			    VDEV_AUX_CORRUPT_DATA);
3848 			return (error);
3849 		}
3850 
3851 		uint64_t checkpoint_sm_obj;
3852 		error = vdev_checkpoint_sm_object(vd, &checkpoint_sm_obj);
3853 		if (error == 0 && checkpoint_sm_obj != 0) {
3854 			objset_t *mos = spa_meta_objset(vd->vdev_spa);
3855 			ASSERT(vd->vdev_asize != 0);
3856 			ASSERT3P(vd->vdev_checkpoint_sm, ==, NULL);
3857 
3858 			error = space_map_open(&vd->vdev_checkpoint_sm,
3859 			    mos, checkpoint_sm_obj, 0, vd->vdev_asize,
3860 			    vd->vdev_ashift);
3861 			if (error != 0) {
3862 				vdev_dbgmsg(vd, "vdev_load: space_map_open "
3863 				    "failed for checkpoint spacemap (obj %llu) "
3864 				    "[error=%d]",
3865 				    (u_longlong_t)checkpoint_sm_obj, error);
3866 				return (error);
3867 			}
3868 			ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);
3869 
3870 			/*
3871 			 * Since the checkpoint_sm contains free entries
3872 			 * exclusively we can use space_map_allocated() to
3873 			 * indicate the cumulative checkpointed space that
3874 			 * has been freed.
3875 			 */
3876 			vd->vdev_stat.vs_checkpoint_space =
3877 			    -space_map_allocated(vd->vdev_checkpoint_sm);
3878 			vd->vdev_spa->spa_checkpoint_info.sci_dspace +=
3879 			    vd->vdev_stat.vs_checkpoint_space;
3880 		} else if (error != 0) {
3881 			vdev_dbgmsg(vd, "vdev_load: failed to retrieve "
3882 			    "checkpoint space map object from vdev ZAP "
3883 			    "[error=%d]", error);
3884 			return (error);
3885 		}
3886 	}
3887 
3888 	/*
3889 	 * If this is a leaf vdev, load its DTL.
3890 	 */
3891 	if (vd->vdev_ops->vdev_op_leaf && (error = vdev_dtl_load(vd)) != 0) {
3892 		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
3893 		    VDEV_AUX_CORRUPT_DATA);
3894 		vdev_dbgmsg(vd, "vdev_load: vdev_dtl_load failed "
3895 		    "[error=%d]", error);
3896 		return (error);
3897 	}
3898 
3899 	uint64_t obsolete_sm_object;
3900 	error = vdev_obsolete_sm_object(vd, &obsolete_sm_object);
3901 	if (error == 0 && obsolete_sm_object != 0) {
3902 		objset_t *mos = vd->vdev_spa->spa_meta_objset;
3903 		ASSERT(vd->vdev_asize != 0);
3904 		ASSERT3P(vd->vdev_obsolete_sm, ==, NULL);
3905 
3906 		if ((error = space_map_open(&vd->vdev_obsolete_sm, mos,
3907 		    obsolete_sm_object, 0, vd->vdev_asize, 0))) {
3908 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
3909 			    VDEV_AUX_CORRUPT_DATA);
3910 			vdev_dbgmsg(vd, "vdev_load: space_map_open failed for "
3911 			    "obsolete spacemap (obj %llu) [error=%d]",
3912 			    (u_longlong_t)obsolete_sm_object, error);
3913 			return (error);
3914 		}
3915 	} else if (error != 0) {
3916 		vdev_dbgmsg(vd, "vdev_load: failed to retrieve obsolete "
3917 		    "space map object from vdev ZAP [error=%d]", error);
3918 		return (error);
3919 	}
3920 
3921 	return (0);
3922 }
3923 
3924 /*
3925  * The special vdev case is used for hot spares and l2cache devices.  Its
3926  * sole purpose it to set the vdev state for the associated vdev.  To do this,
3927  * we make sure that we can open the underlying device, then try to read the
3928  * label, and make sure that the label is sane and that it hasn't been
3929  * repurposed to another pool.
3930  */
3931 int
3932 vdev_validate_aux(vdev_t *vd)
3933 {
3934 	nvlist_t *label;
3935 	uint64_t guid, version;
3936 	uint64_t state;
3937 
3938 	if (!vdev_readable(vd))
3939 		return (0);
3940 
3941 	if ((label = vdev_label_read_config(vd, -1ULL)) == NULL) {
3942 		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
3943 		    VDEV_AUX_CORRUPT_DATA);
3944 		return (-1);
3945 	}
3946 
3947 	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_VERSION, &version) != 0 ||
3948 	    !SPA_VERSION_IS_SUPPORTED(version) ||
3949 	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0 ||
3950 	    guid != vd->vdev_guid ||
3951 	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE, &state) != 0) {
3952 		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
3953 		    VDEV_AUX_CORRUPT_DATA);
3954 		nvlist_free(label);
3955 		return (-1);
3956 	}
3957 
3958 	/*
3959 	 * We don't actually check the pool state here.  If it's in fact in
3960 	 * use by another pool, we update this fact on the fly when requested.
3961 	 */
3962 	nvlist_free(label);
3963 	return (0);
3964 }
3965 
3966 static void
3967 vdev_destroy_ms_flush_data(vdev_t *vd, dmu_tx_t *tx)
3968 {
3969 	objset_t *mos = spa_meta_objset(vd->vdev_spa);
3970 
3971 	if (vd->vdev_top_zap == 0)
3972 		return;
3973 
3974 	uint64_t object = 0;
3975 	int err = zap_lookup(mos, vd->vdev_top_zap,
3976 	    VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1, &object);
3977 	if (err == ENOENT)
3978 		return;
3979 	VERIFY0(err);
3980 
3981 	VERIFY0(dmu_object_free(mos, object, tx));
3982 	VERIFY0(zap_remove(mos, vd->vdev_top_zap,
3983 	    VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, tx));
3984 }
3985 
3986 /*
3987  * Free the objects used to store this vdev's spacemaps, and the array
3988  * that points to them.
3989  */
3990 void
3991 vdev_destroy_spacemaps(vdev_t *vd, dmu_tx_t *tx)
3992 {
3993 	if (vd->vdev_ms_array == 0)
3994 		return;
3995 
3996 	objset_t *mos = vd->vdev_spa->spa_meta_objset;
3997 	uint64_t array_count = vd->vdev_asize >> vd->vdev_ms_shift;
3998 	size_t array_bytes = array_count * sizeof (uint64_t);
3999 	uint64_t *smobj_array = kmem_alloc(array_bytes, KM_SLEEP);
4000 	VERIFY0(dmu_read(mos, vd->vdev_ms_array, 0,
4001 	    array_bytes, smobj_array, 0));
4002 
4003 	for (uint64_t i = 0; i < array_count; i++) {
4004 		uint64_t smobj = smobj_array[i];
4005 		if (smobj == 0)
4006 			continue;
4007 
4008 		space_map_free_obj(mos, smobj, tx);
4009 	}
4010 
4011 	kmem_free(smobj_array, array_bytes);
4012 	VERIFY0(dmu_object_free(mos, vd->vdev_ms_array, tx));
4013 	vdev_destroy_ms_flush_data(vd, tx);
4014 	vd->vdev_ms_array = 0;
4015 }
4016 
4017 static void
4018 vdev_remove_empty_log(vdev_t *vd, uint64_t txg)
4019 {
4020 	spa_t *spa = vd->vdev_spa;
4021 
4022 	ASSERT(vd->vdev_islog);
4023 	ASSERT(vd == vd->vdev_top);
4024 	ASSERT3U(txg, ==, spa_syncing_txg(spa));
4025 
4026 	dmu_tx_t *tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg);
4027 
4028 	vdev_destroy_spacemaps(vd, tx);
4029 	if (vd->vdev_top_zap != 0) {
4030 		vdev_destroy_unlink_zap(vd, vd->vdev_top_zap, tx);
4031 		vd->vdev_top_zap = 0;
4032 	}
4033 
4034 	dmu_tx_commit(tx);
4035 }
4036 
4037 void
4038 vdev_sync_done(vdev_t *vd, uint64_t txg)
4039 {
4040 	metaslab_t *msp;
4041 	boolean_t reassess = !txg_list_empty(&vd->vdev_ms_list, TXG_CLEAN(txg));
4042 
4043 	ASSERT(vdev_is_concrete(vd));
4044 
4045 	while ((msp = txg_list_remove(&vd->vdev_ms_list, TXG_CLEAN(txg)))
4046 	    != NULL)
4047 		metaslab_sync_done(msp, txg);
4048 
4049 	if (reassess) {
4050 		metaslab_sync_reassess(vd->vdev_mg);
4051 		if (vd->vdev_log_mg != NULL)
4052 			metaslab_sync_reassess(vd->vdev_log_mg);
4053 	}
4054 }
4055 
4056 void
4057 vdev_sync(vdev_t *vd, uint64_t txg)
4058 {
4059 	spa_t *spa = vd->vdev_spa;
4060 	vdev_t *lvd;
4061 	metaslab_t *msp;
4062 
4063 	ASSERT3U(txg, ==, spa->spa_syncing_txg);
4064 	dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
4065 	if (range_tree_space(vd->vdev_obsolete_segments) > 0) {
4066 		ASSERT(vd->vdev_removing ||
4067 		    vd->vdev_ops == &vdev_indirect_ops);
4068 
4069 		vdev_indirect_sync_obsolete(vd, tx);
4070 
4071 		/*
4072 		 * If the vdev is indirect, it can't have dirty
4073 		 * metaslabs or DTLs.
4074 		 */
4075 		if (vd->vdev_ops == &vdev_indirect_ops) {
4076 			ASSERT(txg_list_empty(&vd->vdev_ms_list, txg));
4077 			ASSERT(txg_list_empty(&vd->vdev_dtl_list, txg));
4078 			dmu_tx_commit(tx);
4079 			return;
4080 		}
4081 	}
4082 
4083 	ASSERT(vdev_is_concrete(vd));
4084 
4085 	if (vd->vdev_ms_array == 0 && vd->vdev_ms_shift != 0 &&
4086 	    !vd->vdev_removing) {
4087 		ASSERT(vd == vd->vdev_top);
4088 		ASSERT0(vd->vdev_indirect_config.vic_mapping_object);
4089 		vd->vdev_ms_array = dmu_object_alloc(spa->spa_meta_objset,
4090 		    DMU_OT_OBJECT_ARRAY, 0, DMU_OT_NONE, 0, tx);
4091 		ASSERT(vd->vdev_ms_array != 0);
4092 		vdev_config_dirty(vd);
4093 	}
4094 
4095 	while ((msp = txg_list_remove(&vd->vdev_ms_list, txg)) != NULL) {
4096 		metaslab_sync(msp, txg);
4097 		(void) txg_list_add(&vd->vdev_ms_list, msp, TXG_CLEAN(txg));
4098 	}
4099 
4100 	while ((lvd = txg_list_remove(&vd->vdev_dtl_list, txg)) != NULL)
4101 		vdev_dtl_sync(lvd, txg);
4102 
4103 	/*
4104 	 * If this is an empty log device being removed, destroy the
4105 	 * metadata associated with it.
4106 	 */
4107 	if (vd->vdev_islog && vd->vdev_stat.vs_alloc == 0 && vd->vdev_removing)
4108 		vdev_remove_empty_log(vd, txg);
4109 
4110 	(void) txg_list_add(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg));
4111 	dmu_tx_commit(tx);
4112 }
4113 
4114 /*
4115  * Return the amount of space that should be (or was) allocated for the given
4116  * psize (compressed block size) in the given TXG. Note that for expanded
4117  * RAIDZ vdevs, the size allocated for older BP's may be larger. See
4118  * vdev_raidz_asize().
4119  */
4120 uint64_t
4121 vdev_psize_to_asize_txg(vdev_t *vd, uint64_t psize, uint64_t txg)
4122 {
4123 	return (vd->vdev_ops->vdev_op_asize(vd, psize, txg));
4124 }
4125 
4126 uint64_t
4127 vdev_psize_to_asize(vdev_t *vd, uint64_t psize)
4128 {
4129 	return (vdev_psize_to_asize_txg(vd, psize, 0));
4130 }
4131 
4132 /*
4133  * Mark the given vdev faulted.  A faulted vdev behaves as if the device could
4134  * not be opened, and no I/O is attempted.
4135  */
4136 int
4137 vdev_fault(spa_t *spa, uint64_t guid, vdev_aux_t aux)
4138 {
4139 	vdev_t *vd, *tvd;
4140 
4141 	spa_vdev_state_enter(spa, SCL_NONE);
4142 
4143 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
4144 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));
4145 
4146 	if (!vd->vdev_ops->vdev_op_leaf)
4147 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));
4148 
4149 	tvd = vd->vdev_top;
4150 
4151 	/*
4152 	 * If user did a 'zpool offline -f' then make the fault persist across
4153 	 * reboots.
4154 	 */
4155 	if (aux == VDEV_AUX_EXTERNAL_PERSIST) {
4156 		/*
4157 		 * There are two kinds of forced faults: temporary and
4158 		 * persistent.  Temporary faults go away at pool import, while
4159 		 * persistent faults stay set.  Both types of faults can be
4160 		 * cleared with a zpool clear.
4161 		 *
4162 		 * We tell if a vdev is persistently faulted by looking at the
4163 		 * ZPOOL_CONFIG_AUX_STATE nvpair.  If it's set to "external" at
4164 		 * import then it's a persistent fault.  Otherwise, it's
4165 		 * temporary.  We get ZPOOL_CONFIG_AUX_STATE set to "external"
4166 		 * by setting vd.vdev_stat.vs_aux to VDEV_AUX_EXTERNAL.  This
4167 		 * tells vdev_config_generate() (which gets run later) to set
4168 		 * ZPOOL_CONFIG_AUX_STATE to "external" in the nvlist.
4169 		 */
4170 		vd->vdev_stat.vs_aux = VDEV_AUX_EXTERNAL;
4171 		vd->vdev_tmpoffline = B_FALSE;
4172 		aux = VDEV_AUX_EXTERNAL;
4173 	} else {
4174 		vd->vdev_tmpoffline = B_TRUE;
4175 	}
4176 
4177 	/*
4178 	 * We don't directly use the aux state here, but if we do a
4179 	 * vdev_reopen(), we need this value to be present to remember why we
4180 	 * were faulted.
4181 	 */
4182 	vd->vdev_label_aux = aux;
4183 
4184 	/*
4185 	 * Faulted state takes precedence over degraded.
4186 	 */
4187 	vd->vdev_delayed_close = B_FALSE;
4188 	vd->vdev_faulted = 1ULL;
4189 	vd->vdev_degraded = 0ULL;
4190 	vdev_set_state(vd, B_FALSE, VDEV_STATE_FAULTED, aux);
4191 
4192 	/*
4193 	 * If this device has the only valid copy of the data, then
4194 	 * back off and simply mark the vdev as degraded instead.
4195 	 */
4196 	if (!tvd->vdev_islog && vd->vdev_aux == NULL && vdev_dtl_required(vd)) {
4197 		vd->vdev_degraded = 1ULL;
4198 		vd->vdev_faulted = 0ULL;
4199 
4200 		/*
4201 		 * If we reopen the device and it's not dead, only then do we
4202 		 * mark it degraded.
4203 		 */
4204 		vdev_reopen(tvd);
4205 
4206 		if (vdev_readable(vd))
4207 			vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, aux);
4208 	}
4209 
4210 	return (spa_vdev_state_exit(spa, vd, 0));
4211 }
4212 
4213 /*
4214  * Mark the given vdev degraded.  A degraded vdev is purely an indication to the
4215  * user that something is wrong.  The vdev continues to operate as normal as far
4216  * as I/O is concerned.
4217  */
4218 int
4219 vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux)
4220 {
4221 	vdev_t *vd;
4222 
4223 	spa_vdev_state_enter(spa, SCL_NONE);
4224 
4225 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
4226 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));
4227 
4228 	if (!vd->vdev_ops->vdev_op_leaf)
4229 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));
4230 
4231 	/*
4232 	 * If the vdev is already faulted, then don't do anything.
4233 	 */
4234 	if (vd->vdev_faulted || vd->vdev_degraded)
4235 		return (spa_vdev_state_exit(spa, NULL, 0));
4236 
4237 	vd->vdev_degraded = 1ULL;
4238 	if (!vdev_is_dead(vd))
4239 		vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED,
4240 		    aux);
4241 
4242 	return (spa_vdev_state_exit(spa, vd, 0));
4243 }
4244 
4245 int
4246 vdev_remove_wanted(spa_t *spa, uint64_t guid)
4247 {
4248 	vdev_t *vd;
4249 
4250 	spa_vdev_state_enter(spa, SCL_NONE);
4251 
4252 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
4253 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));
4254 
4255 	/*
4256 	 * If the vdev is already removed, or expanding which can trigger
4257 	 * repartition add/remove events, then don't do anything.
4258 	 */
4259 	if (vd->vdev_removed || vd->vdev_expanding)
4260 		return (spa_vdev_state_exit(spa, NULL, 0));
4261 
4262 	/*
4263 	 * Confirm the vdev has been removed, otherwise don't do anything.
4264 	 */
4265 	if (vd->vdev_ops->vdev_op_leaf && !zio_wait(vdev_probe(vd, NULL)))
4266 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(EEXIST)));
4267 
4268 	vd->vdev_remove_wanted = B_TRUE;
4269 	spa_async_request(spa, SPA_ASYNC_REMOVE);
4270 
4271 	return (spa_vdev_state_exit(spa, vd, 0));
4272 }
4273 
4274 
4275 /*
4276  * Online the given vdev.
4277  *
4278  * If 'ZFS_ONLINE_UNSPARE' is set, it implies two things.  First, any attached
4279  * spare device should be detached when the device finishes resilvering.
4280  * Second, the online should be treated like a 'test' online case, so no FMA
4281  * events are generated if the device fails to open.
4282  */
4283 int
4284 vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
4285 {
4286 	vdev_t *vd, *tvd, *pvd, *rvd = spa->spa_root_vdev;
4287 	boolean_t wasoffline;
4288 	vdev_state_t oldstate;
4289 
4290 	spa_vdev_state_enter(spa, SCL_NONE);
4291 
4292 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
4293 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));
4294 
4295 	wasoffline = (vd->vdev_offline || vd->vdev_tmpoffline);
4296 	oldstate = vd->vdev_state;
4297 
4298 	tvd = vd->vdev_top;
4299 	vd->vdev_offline = B_FALSE;
4300 	vd->vdev_tmpoffline = B_FALSE;
4301 	vd->vdev_checkremove = !!(flags & ZFS_ONLINE_CHECKREMOVE);
4302 	vd->vdev_forcefault = !!(flags & ZFS_ONLINE_FORCEFAULT);
4303 
4304 	/* XXX - L2ARC 1.0 does not support expansion */
4305 	if (!vd->vdev_aux) {
4306 		for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
4307 			pvd->vdev_expanding = !!((flags & ZFS_ONLINE_EXPAND) ||
4308 			    spa->spa_autoexpand);
4309 		vd->vdev_expansion_time = gethrestime_sec();
4310 	}
4311 
4312 	vdev_reopen(tvd);
4313 	vd->vdev_checkremove = vd->vdev_forcefault = B_FALSE;
4314 
4315 	if (!vd->vdev_aux) {
4316 		for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
4317 			pvd->vdev_expanding = B_FALSE;
4318 	}
4319 
4320 	if (newstate)
4321 		*newstate = vd->vdev_state;
4322 	if ((flags & ZFS_ONLINE_UNSPARE) &&
4323 	    !vdev_is_dead(vd) && vd->vdev_parent &&
4324 	    vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
4325 	    vd->vdev_parent->vdev_child[0] == vd)
4326 		vd->vdev_unspare = B_TRUE;
4327 
4328 	if ((flags & ZFS_ONLINE_EXPAND) || spa->spa_autoexpand) {
4329 
4330 		/* XXX - L2ARC 1.0 does not support expansion */
4331 		if (vd->vdev_aux)
4332 			return (spa_vdev_state_exit(spa, vd, ENOTSUP));
4333 		spa->spa_ccw_fail_time = 0;
4334 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
4335 	}
4336 
4337 	/* Restart initializing if necessary */
4338 	mutex_enter(&vd->vdev_initialize_lock);
4339 	if (vdev_writeable(vd) &&
4340 	    vd->vdev_initialize_thread == NULL &&
4341 	    vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) {
4342 		(void) vdev_initialize(vd);
4343 	}
4344 	mutex_exit(&vd->vdev_initialize_lock);
4345 
4346 	/*
4347 	 * Restart trimming if necessary. We do not restart trimming for cache
4348 	 * devices here. This is triggered by l2arc_rebuild_vdev()
4349 	 * asynchronously for the whole device or in l2arc_evict() as it evicts
4350 	 * space for upcoming writes.
4351 	 */
4352 	mutex_enter(&vd->vdev_trim_lock);
4353 	if (vdev_writeable(vd) && !vd->vdev_isl2cache &&
4354 	    vd->vdev_trim_thread == NULL &&
4355 	    vd->vdev_trim_state == VDEV_TRIM_ACTIVE) {
4356 		(void) vdev_trim(vd, vd->vdev_trim_rate, vd->vdev_trim_partial,
4357 		    vd->vdev_trim_secure);
4358 	}
4359 	mutex_exit(&vd->vdev_trim_lock);
4360 
4361 	if (wasoffline ||
4362 	    (oldstate < VDEV_STATE_DEGRADED &&
4363 	    vd->vdev_state >= VDEV_STATE_DEGRADED)) {
4364 		spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_ONLINE);
4365 
4366 		/*
4367 		 * Asynchronously detach spare vdev if resilver or
4368 		 * rebuild is not required
4369 		 */
4370 		if (vd->vdev_unspare &&
4371 		    !dsl_scan_resilvering(spa->spa_dsl_pool) &&
4372 		    !dsl_scan_resilver_scheduled(spa->spa_dsl_pool) &&
4373 		    !vdev_rebuild_active(tvd))
4374 			spa_async_request(spa, SPA_ASYNC_DETACH_SPARE);
4375 	}
4376 	return (spa_vdev_state_exit(spa, vd, 0));
4377 }
4378 
4379 static int
4380 vdev_offline_locked(spa_t *spa, uint64_t guid, uint64_t flags)
4381 {
4382 	vdev_t *vd, *tvd;
4383 	int error = 0;
4384 	uint64_t generation;
4385 	metaslab_group_t *mg;
4386 
4387 top:
4388 	spa_vdev_state_enter(spa, SCL_ALLOC);
4389 
4390 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
4391 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));
4392 
4393 	if (!vd->vdev_ops->vdev_op_leaf)
4394 		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENOTSUP)));
4395 
4396 	if (vd->vdev_ops == &vdev_draid_spare_ops)
4397 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
4398 
4399 	tvd = vd->vdev_top;
4400 	mg = tvd->vdev_mg;
4401 	generation = spa->spa_config_generation + 1;
4402 
4403 	/*
4404 	 * If the device isn't already offline, try to offline it.
4405 	 */
4406 	if (!vd->vdev_offline) {
4407 		/*
4408 		 * If this device has the only valid copy of some data,
4409 		 * don't allow it to be offlined. Log devices are always
4410 		 * expendable.
4411 		 */
4412 		if (!tvd->vdev_islog && vd->vdev_aux == NULL &&
4413 		    vdev_dtl_required(vd))
4414 			return (spa_vdev_state_exit(spa, NULL,
4415 			    SET_ERROR(EBUSY)));
4416 
4417 		/*
4418 		 * If the top-level is a slog and it has had allocations
4419 		 * then proceed.  We check that the vdev's metaslab group
4420 		 * is not NULL since it's possible that we may have just
4421 		 * added this vdev but not yet initialized its metaslabs.
4422 		 */
4423 		if (tvd->vdev_islog && mg != NULL) {
4424 			/*
4425 			 * Prevent any future allocations.
4426 			 */
4427 			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
4428 			metaslab_group_passivate(mg);
4429 			(void) spa_vdev_state_exit(spa, vd, 0);
4430 
4431 			error = spa_reset_logs(spa);
4432 
4433 			/*
4434 			 * If the log device was successfully reset but has
4435 			 * checkpointed data, do not offline it.
4436 			 */
4437 			if (error == 0 &&
4438 			    tvd->vdev_checkpoint_sm != NULL) {
4439 				ASSERT3U(space_map_allocated(
4440 				    tvd->vdev_checkpoint_sm), !=, 0);
4441 				error = ZFS_ERR_CHECKPOINT_EXISTS;
4442 			}
4443 
4444 			spa_vdev_state_enter(spa, SCL_ALLOC);
4445 
4446 			/*
4447 			 * Check to see if the config has changed.
4448 			 */
4449 			if (error || generation != spa->spa_config_generation) {
4450 				metaslab_group_activate(mg);
4451 				if (error)
4452 					return (spa_vdev_state_exit(spa,
4453 					    vd, error));
4454 				(void) spa_vdev_state_exit(spa, vd, 0);
4455 				goto top;
4456 			}
4457 			ASSERT0(tvd->vdev_stat.vs_alloc);
4458 		}
4459 
4460 		/*
4461 		 * Offline this device and reopen its top-level vdev.
4462 		 * If the top-level vdev is a log device then just offline
4463 		 * it. Otherwise, if this action results in the top-level
4464 		 * vdev becoming unusable, undo it and fail the request.
4465 		 */
4466 		vd->vdev_offline = B_TRUE;
4467 		vdev_reopen(tvd);
4468 
4469 		if (!tvd->vdev_islog && vd->vdev_aux == NULL &&
4470 		    vdev_is_dead(tvd)) {
4471 			vd->vdev_offline = B_FALSE;
4472 			vdev_reopen(tvd);
4473 			return (spa_vdev_state_exit(spa, NULL,
4474 			    SET_ERROR(EBUSY)));
4475 		}
4476 
4477 		/*
4478 		 * Add the device back into the metaslab rotor so that
4479 		 * once we online the device it's open for business.
4480 		 */
4481 		if (tvd->vdev_islog && mg != NULL)
4482 			metaslab_group_activate(mg);
4483 	}
4484 
4485 	vd->vdev_tmpoffline = !!(flags & ZFS_OFFLINE_TEMPORARY);
4486 
4487 	return (spa_vdev_state_exit(spa, vd, 0));
4488 }
4489 
4490 int
4491 vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags)
4492 {
4493 	int error;
4494 
4495 	mutex_enter(&spa->spa_vdev_top_lock);
4496 	error = vdev_offline_locked(spa, guid, flags);
4497 	mutex_exit(&spa->spa_vdev_top_lock);
4498 
4499 	return (error);
4500 }
4501 
4502 /*
4503  * Clear the error counts associated with this vdev.  Unlike vdev_online() and
4504  * vdev_offline(), we assume the spa config is locked.  We also clear all
4505  * children.  If 'vd' is NULL, then the user wants to clear all vdevs.
4506  */
4507 void
4508 vdev_clear(spa_t *spa, vdev_t *vd)
4509 {
4510 	vdev_t *rvd = spa->spa_root_vdev;
4511 
4512 	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
4513 
4514 	if (vd == NULL)
4515 		vd = rvd;
4516 
4517 	vd->vdev_stat.vs_read_errors = 0;
4518 	vd->vdev_stat.vs_write_errors = 0;
4519 	vd->vdev_stat.vs_checksum_errors = 0;
4520 	vd->vdev_stat.vs_dio_verify_errors = 0;
4521 	vd->vdev_stat.vs_slow_ios = 0;
4522 
4523 	for (int c = 0; c < vd->vdev_children; c++)
4524 		vdev_clear(spa, vd->vdev_child[c]);
4525 
4526 	/*
4527 	 * It makes no sense to "clear" an indirect  or removed vdev.
4528 	 */
4529 	if (!vdev_is_concrete(vd) || vd->vdev_removed)
4530 		return;
4531 
4532 	/*
4533 	 * If we're in the FAULTED state or have experienced failed I/O, then
4534 	 * clear the persistent state and attempt to reopen the device.  We
4535 	 * also mark the vdev config dirty, so that the new faulted state is
4536 	 * written out to disk.
4537 	 */
4538 	if (vd->vdev_faulted || vd->vdev_degraded ||
4539 	    !vdev_readable(vd) || !vdev_writeable(vd)) {
4540 		/*
4541 		 * When reopening in response to a clear event, it may be due to
4542 		 * a fmadm repair request.  In this case, if the device is
4543 		 * still broken, we want to still post the ereport again.
4544 		 */
4545 		vd->vdev_forcefault = B_TRUE;
4546 
4547 		vd->vdev_faulted = vd->vdev_degraded = 0ULL;
4548 		vd->vdev_cant_read = B_FALSE;
4549 		vd->vdev_cant_write = B_FALSE;
4550 		vd->vdev_stat.vs_aux = 0;
4551 
4552 		vdev_reopen(vd == rvd ? rvd : vd->vdev_top);
4553 
4554 		vd->vdev_forcefault = B_FALSE;
4555 
4556 		if (vd != rvd && vdev_writeable(vd->vdev_top))
4557 			vdev_state_dirty(vd->vdev_top);
4558 
4559 		/* If a resilver isn't required, check if vdevs can be culled */
4560 		if (vd->vdev_aux == NULL && !vdev_is_dead(vd) &&
4561 		    !dsl_scan_resilvering(spa->spa_dsl_pool) &&
4562 		    !dsl_scan_resilver_scheduled(spa->spa_dsl_pool))
4563 			spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
4564 
4565 		spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_CLEAR);
4566 	}
4567 
4568 	/*
4569 	 * When clearing a FMA-diagnosed fault, we always want to
4570 	 * unspare the device, as we assume that the original spare was
4571 	 * done in response to the FMA fault.
4572 	 */
4573 	if (!vdev_is_dead(vd) && vd->vdev_parent != NULL &&
4574 	    vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
4575 	    vd->vdev_parent->vdev_child[0] == vd)
4576 		vd->vdev_unspare = B_TRUE;
4577 
4578 	/* Clear recent error events cache (i.e. duplicate events tracking) */
4579 	zfs_ereport_clear(spa, vd);
4580 }
4581 
4582 boolean_t
4583 vdev_is_dead(vdev_t *vd)
4584 {
4585 	/*
4586 	 * Holes and missing devices are always considered "dead".
4587 	 * This simplifies the code since we don't have to check for
4588 	 * these types of devices in the various code paths.
4589 	 * Instead we rely on the fact that we skip over dead devices
4590 	 * before issuing I/O to them.
4591 	 */
4592 	return (vd->vdev_state < VDEV_STATE_DEGRADED ||
4593 	    vd->vdev_ops == &vdev_hole_ops ||
4594 	    vd->vdev_ops == &vdev_missing_ops);
4595 }
4596 
4597 boolean_t
4598 vdev_readable(vdev_t *vd)
4599 {
4600 	return (!vdev_is_dead(vd) && !vd->vdev_cant_read);
4601 }
4602 
4603 boolean_t
4604 vdev_writeable(vdev_t *vd)
4605 {
4606 	return (!vdev_is_dead(vd) && !vd->vdev_cant_write &&
4607 	    vdev_is_concrete(vd));
4608 }
4609 
4610 boolean_t
4611 vdev_allocatable(vdev_t *vd)
4612 {
4613 	uint64_t state = vd->vdev_state;
4614 
4615 	/*
4616 	 * We currently allow allocations from vdevs which may be in the
4617 	 * process of reopening (i.e. VDEV_STATE_CLOSED). If the device
4618 	 * fails to reopen then we'll catch it later when we're holding
4619 	 * the proper locks.  Note that we have to get the vdev state
4620 	 * in a local variable because although it changes atomically,
4621 	 * we're asking two separate questions about it.
4622 	 */
4623 	return (!(state < VDEV_STATE_DEGRADED && state != VDEV_STATE_CLOSED) &&
4624 	    !vd->vdev_cant_write && vdev_is_concrete(vd) &&
4625 	    vd->vdev_mg->mg_initialized);
4626 }
4627 
4628 boolean_t
4629 vdev_accessible(vdev_t *vd, zio_t *zio)
4630 {
4631 	ASSERT(zio->io_vd == vd);
4632 
4633 	if (vdev_is_dead(vd) || vd->vdev_remove_wanted)
4634 		return (B_FALSE);
4635 
4636 	if (zio->io_type == ZIO_TYPE_READ)
4637 		return (!vd->vdev_cant_read);
4638 
4639 	if (zio->io_type == ZIO_TYPE_WRITE)
4640 		return (!vd->vdev_cant_write);
4641 
4642 	return (B_TRUE);
4643 }
4644 
4645 static void
4646 vdev_get_child_stat(vdev_t *cvd, vdev_stat_t *vs, vdev_stat_t *cvs)
4647 {
4648 	/*
4649 	 * Exclude the dRAID spare when aggregating to avoid double counting
4650 	 * the ops and bytes.  These IOs are counted by the physical leaves.
4651 	 */
4652 	if (cvd->vdev_ops == &vdev_draid_spare_ops)
4653 		return;
4654 
4655 	for (int t = 0; t < VS_ZIO_TYPES; t++) {
4656 		vs->vs_ops[t] += cvs->vs_ops[t];
4657 		vs->vs_bytes[t] += cvs->vs_bytes[t];
4658 	}
4659 
4660 	cvs->vs_scan_removing = cvd->vdev_removing;
4661 }
4662 
4663 /*
4664  * Get extended stats
4665  */
4666 static void
4667 vdev_get_child_stat_ex(vdev_t *cvd, vdev_stat_ex_t *vsx, vdev_stat_ex_t *cvsx)
4668 {
4669 	(void) cvd;
4670 
4671 	int t, b;
4672 	for (t = 0; t < ZIO_TYPES; t++) {
4673 		for (b = 0; b < ARRAY_SIZE(vsx->vsx_disk_histo[0]); b++)
4674 			vsx->vsx_disk_histo[t][b] += cvsx->vsx_disk_histo[t][b];
4675 
4676 		for (b = 0; b < ARRAY_SIZE(vsx->vsx_total_histo[0]); b++) {
4677 			vsx->vsx_total_histo[t][b] +=
4678 			    cvsx->vsx_total_histo[t][b];
4679 		}
4680 	}
4681 
4682 	for (t = 0; t < ZIO_PRIORITY_NUM_QUEUEABLE; t++) {
4683 		for (b = 0; b < ARRAY_SIZE(vsx->vsx_queue_histo[0]); b++) {
4684 			vsx->vsx_queue_histo[t][b] +=
4685 			    cvsx->vsx_queue_histo[t][b];
4686 		}
4687 		vsx->vsx_active_queue[t] += cvsx->vsx_active_queue[t];
4688 		vsx->vsx_pend_queue[t] += cvsx->vsx_pend_queue[t];
4689 
4690 		for (b = 0; b < ARRAY_SIZE(vsx->vsx_ind_histo[0]); b++)
4691 			vsx->vsx_ind_histo[t][b] += cvsx->vsx_ind_histo[t][b];
4692 
4693 		for (b = 0; b < ARRAY_SIZE(vsx->vsx_agg_histo[0]); b++)
4694 			vsx->vsx_agg_histo[t][b] += cvsx->vsx_agg_histo[t][b];
4695 	}
4696 
4697 }
4698 
4699 boolean_t
4700 vdev_is_spacemap_addressable(vdev_t *vd)
4701 {
4702 	if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_SPACEMAP_V2))
4703 		return (B_TRUE);
4704 
4705 	/*
4706 	 * If double-word space map entries are not enabled we assume
4707 	 * 47 bits of the space map entry are dedicated to the entry's
4708 	 * offset (see SM_OFFSET_BITS in space_map.h). We then use that
4709 	 * to calculate the maximum address that can be described by a
4710 	 * space map entry for the given device.
4711 	 */
4712 	uint64_t shift = vd->vdev_ashift + SM_OFFSET_BITS;
4713 
4714 	if (shift >= 63) /* detect potential overflow */
4715 		return (B_TRUE);
4716 
4717 	return (vd->vdev_asize < (1ULL << shift));
4718 }
4719 
4720 /*
4721  * Get statistics for the given vdev.
4722  */
4723 static void
4724 vdev_get_stats_ex_impl(vdev_t *vd, vdev_stat_t *vs, vdev_stat_ex_t *vsx)
4725 {
4726 	int t;
4727 	/*
4728 	 * If we're getting stats on the root vdev, aggregate the I/O counts
4729 	 * over all top-level vdevs (i.e. the direct children of the root).
4730 	 */
4731 	if (!vd->vdev_ops->vdev_op_leaf) {
4732 		if (vs) {
4733 			memset(vs->vs_ops, 0, sizeof (vs->vs_ops));
4734 			memset(vs->vs_bytes, 0, sizeof (vs->vs_bytes));
4735 		}
4736 		if (vsx)
4737 			memset(vsx, 0, sizeof (*vsx));
4738 
4739 		for (int c = 0; c < vd->vdev_children; c++) {
4740 			vdev_t *cvd = vd->vdev_child[c];
4741 			vdev_stat_t *cvs = &cvd->vdev_stat;
4742 			vdev_stat_ex_t *cvsx = &cvd->vdev_stat_ex;
4743 
4744 			vdev_get_stats_ex_impl(cvd, cvs, cvsx);
4745 			if (vs)
4746 				vdev_get_child_stat(cvd, vs, cvs);
4747 			if (vsx)
4748 				vdev_get_child_stat_ex(cvd, vsx, cvsx);
4749 		}
4750 	} else {
4751 		/*
4752 		 * We're a leaf.  Just copy our ZIO active queue stats in.  The
4753 		 * other leaf stats are updated in vdev_stat_update().
4754 		 */
4755 		if (!vsx)
4756 			return;
4757 
4758 		memcpy(vsx, &vd->vdev_stat_ex, sizeof (vd->vdev_stat_ex));
4759 
4760 		for (t = 0; t < ZIO_PRIORITY_NUM_QUEUEABLE; t++) {
4761 			vsx->vsx_active_queue[t] = vd->vdev_queue.vq_cactive[t];
4762 			vsx->vsx_pend_queue[t] = vdev_queue_class_length(vd, t);
4763 		}
4764 	}
4765 }
4766 
4767 void
4768 vdev_get_stats_ex(vdev_t *vd, vdev_stat_t *vs, vdev_stat_ex_t *vsx)
4769 {
4770 	vdev_t *tvd = vd->vdev_top;
4771 	mutex_enter(&vd->vdev_stat_lock);
4772 	if (vs) {
4773 		memcpy(vs, &vd->vdev_stat, sizeof (*vs));
4774 		vs->vs_timestamp = gethrtime() - vs->vs_timestamp;
4775 		vs->vs_state = vd->vdev_state;
4776 		vs->vs_rsize = vdev_get_min_asize(vd);
4777 
4778 		if (vd->vdev_ops->vdev_op_leaf) {
4779 			vs->vs_pspace = vd->vdev_psize;
4780 			vs->vs_rsize += VDEV_LABEL_START_SIZE +
4781 			    VDEV_LABEL_END_SIZE;
4782 			/*
4783 			 * Report initializing progress. Since we don't
4784 			 * have the initializing locks held, this is only
4785 			 * an estimate (although a fairly accurate one).
4786 			 */
4787 			vs->vs_initialize_bytes_done =
4788 			    vd->vdev_initialize_bytes_done;
4789 			vs->vs_initialize_bytes_est =
4790 			    vd->vdev_initialize_bytes_est;
4791 			vs->vs_initialize_state = vd->vdev_initialize_state;
4792 			vs->vs_initialize_action_time =
4793 			    vd->vdev_initialize_action_time;
4794 
4795 			/*
4796 			 * Report manual TRIM progress. Since we don't have
4797 			 * the manual TRIM locks held, this is only an
4798 			 * estimate (although fairly accurate one).
4799 			 */
4800 			vs->vs_trim_notsup = !vd->vdev_has_trim;
4801 			vs->vs_trim_bytes_done = vd->vdev_trim_bytes_done;
4802 			vs->vs_trim_bytes_est = vd->vdev_trim_bytes_est;
4803 			vs->vs_trim_state = vd->vdev_trim_state;
4804 			vs->vs_trim_action_time = vd->vdev_trim_action_time;
4805 
4806 			/* Set when there is a deferred resilver. */
4807 			vs->vs_resilver_deferred = vd->vdev_resilver_deferred;
4808 		}
4809 
4810 		/*
4811 		 * Report expandable space on top-level, non-auxiliary devices
4812 		 * only. The expandable space is reported in terms of metaslab
4813 		 * sized units since that determines how much space the pool
4814 		 * can expand.
4815 		 */
4816 		if (vd->vdev_aux == NULL && tvd != NULL) {
4817 			vs->vs_esize = P2ALIGN_TYPED(
4818 			    vd->vdev_max_asize - vd->vdev_asize,
4819 			    1ULL << tvd->vdev_ms_shift, uint64_t);
4820 		}
4821 
4822 		vs->vs_configured_ashift = vd->vdev_top != NULL
4823 		    ? vd->vdev_top->vdev_ashift : vd->vdev_ashift;
4824 		vs->vs_logical_ashift = vd->vdev_logical_ashift;
4825 		if (vd->vdev_physical_ashift <= ASHIFT_MAX)
4826 			vs->vs_physical_ashift = vd->vdev_physical_ashift;
4827 		else
4828 			vs->vs_physical_ashift = 0;
4829 
4830 		/*
4831 		 * Report fragmentation and rebuild progress for top-level,
4832 		 * non-auxiliary, concrete devices.
4833 		 */
4834 		if (vd->vdev_aux == NULL && vd == vd->vdev_top &&
4835 		    vdev_is_concrete(vd)) {
4836 			/*
4837 			 * The vdev fragmentation rating doesn't take into
4838 			 * account the embedded slog metaslab (vdev_log_mg).
4839 			 * Since it's only one metaslab, it would have a tiny
4840 			 * impact on the overall fragmentation.
4841 			 */
4842 			vs->vs_fragmentation = (vd->vdev_mg != NULL) ?
4843 			    vd->vdev_mg->mg_fragmentation : 0;
4844 		}
4845 		vs->vs_noalloc = MAX(vd->vdev_noalloc,
4846 		    tvd ? tvd->vdev_noalloc : 0);
4847 	}
4848 
4849 	vdev_get_stats_ex_impl(vd, vs, vsx);
4850 	mutex_exit(&vd->vdev_stat_lock);
4851 }
4852 
4853 void
4854 vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
4855 {
4856 	return (vdev_get_stats_ex(vd, vs, NULL));
4857 }
4858 
4859 void
4860 vdev_clear_stats(vdev_t *vd)
4861 {
4862 	mutex_enter(&vd->vdev_stat_lock);
4863 	vd->vdev_stat.vs_space = 0;
4864 	vd->vdev_stat.vs_dspace = 0;
4865 	vd->vdev_stat.vs_alloc = 0;
4866 	mutex_exit(&vd->vdev_stat_lock);
4867 }
4868 
4869 void
4870 vdev_scan_stat_init(vdev_t *vd)
4871 {
4872 	vdev_stat_t *vs = &vd->vdev_stat;
4873 
4874 	for (int c = 0; c < vd->vdev_children; c++)
4875 		vdev_scan_stat_init(vd->vdev_child[c]);
4876 
4877 	mutex_enter(&vd->vdev_stat_lock);
4878 	vs->vs_scan_processed = 0;
4879 	mutex_exit(&vd->vdev_stat_lock);
4880 }
4881 
4882 void
4883 vdev_stat_update(zio_t *zio, uint64_t psize)
4884 {
4885 	spa_t *spa = zio->io_spa;
4886 	vdev_t *rvd = spa->spa_root_vdev;
4887 	vdev_t *vd = zio->io_vd ? zio->io_vd : rvd;
4888 	vdev_t *pvd;
4889 	uint64_t txg = zio->io_txg;
4890 /* Suppress ASAN false positive */
4891 #ifdef __SANITIZE_ADDRESS__
4892 	vdev_stat_t *vs = vd ? &vd->vdev_stat : NULL;
4893 	vdev_stat_ex_t *vsx = vd ? &vd->vdev_stat_ex : NULL;
4894 #else
4895 	vdev_stat_t *vs = &vd->vdev_stat;
4896 	vdev_stat_ex_t *vsx = &vd->vdev_stat_ex;
4897 #endif
4898 	zio_type_t type = zio->io_type;
4899 	int flags = zio->io_flags;
4900 
4901 	/*
4902 	 * If this i/o is a gang leader, it didn't do any actual work.
4903 	 */
4904 	if (zio->io_gang_tree)
4905 		return;
4906 
4907 	if (zio->io_error == 0) {
4908 		/*
4909 		 * If this is a root i/o, don't count it -- we've already
4910 		 * counted the top-level vdevs, and vdev_get_stats() will
4911 		 * aggregate them when asked.  This reduces contention on
4912 		 * the root vdev_stat_lock and implicitly handles blocks
4913 		 * that compress away to holes, for which there is no i/o.
4914 		 * (Holes never create vdev children, so all the counters
4915 		 * remain zero, which is what we want.)
4916 		 *
4917 		 * Note: this only applies to successful i/o (io_error == 0)
4918 		 * because unlike i/o counts, errors are not additive.
4919 		 * When reading a ditto block, for example, failure of
4920 		 * one top-level vdev does not imply a root-level error.
4921 		 */
4922 		if (vd == rvd)
4923 			return;
4924 
4925 		ASSERT(vd == zio->io_vd);
4926 
4927 		if (flags & ZIO_FLAG_IO_BYPASS)
4928 			return;
4929 
4930 		mutex_enter(&vd->vdev_stat_lock);
4931 
4932 		if (flags & ZIO_FLAG_IO_REPAIR) {
4933 			/*
4934 			 * Repair is the result of a resilver issued by the
4935 			 * scan thread (spa_sync).
4936 			 */
4937 			if (flags & ZIO_FLAG_SCAN_THREAD) {
4938 				dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
4939 				dsl_scan_phys_t *scn_phys = &scn->scn_phys;
4940 				uint64_t *processed = &scn_phys->scn_processed;
4941 
4942 				if (vd->vdev_ops->vdev_op_leaf)
4943 					atomic_add_64(processed, psize);
4944 				vs->vs_scan_processed += psize;
4945 			}
4946 
4947 			/*
4948 			 * Repair is the result of a rebuild issued by the
4949 			 * rebuild thread (vdev_rebuild_thread).  To avoid
4950 			 * double counting repaired bytes the virtual dRAID
4951 			 * spare vdev is excluded from the processed bytes.
4952 			 */
4953 			if (zio->io_priority == ZIO_PRIORITY_REBUILD) {
4954 				vdev_t *tvd = vd->vdev_top;
4955 				vdev_rebuild_t *vr = &tvd->vdev_rebuild_config;
4956 				vdev_rebuild_phys_t *vrp = &vr->vr_rebuild_phys;
4957 				uint64_t *rebuilt = &vrp->vrp_bytes_rebuilt;
4958 
4959 				if (vd->vdev_ops->vdev_op_leaf &&
4960 				    vd->vdev_ops != &vdev_draid_spare_ops) {
4961 					atomic_add_64(rebuilt, psize);
4962 				}
4963 				vs->vs_rebuild_processed += psize;
4964 			}
4965 
4966 			if (flags & ZIO_FLAG_SELF_HEAL)
4967 				vs->vs_self_healed += psize;
4968 		}
4969 
4970 		/*
4971 		 * The bytes/ops/histograms are recorded at the leaf level and
4972 		 * aggregated into the higher level vdevs in vdev_get_stats().
4973 		 */
4974 		if (vd->vdev_ops->vdev_op_leaf &&
4975 		    (zio->io_priority < ZIO_PRIORITY_NUM_QUEUEABLE)) {
4976 			zio_type_t vs_type = type;
4977 			zio_priority_t priority = zio->io_priority;
4978 
4979 			/*
4980 			 * TRIM ops and bytes are reported to user space as
4981 			 * ZIO_TYPE_FLUSH.  This is done to preserve the
4982 			 * vdev_stat_t structure layout for user space.
4983 			 */
4984 			if (type == ZIO_TYPE_TRIM)
4985 				vs_type = ZIO_TYPE_FLUSH;
4986 
4987 			/*
4988 			 * Solely for the purposes of 'zpool iostat -lqrw'
4989 			 * reporting use the priority to categorize the IO.
4990 			 * Only the following are reported to user space:
4991 			 *
4992 			 *   ZIO_PRIORITY_SYNC_READ,
4993 			 *   ZIO_PRIORITY_SYNC_WRITE,
4994 			 *   ZIO_PRIORITY_ASYNC_READ,
4995 			 *   ZIO_PRIORITY_ASYNC_WRITE,
4996 			 *   ZIO_PRIORITY_SCRUB,
4997 			 *   ZIO_PRIORITY_TRIM,
4998 			 *   ZIO_PRIORITY_REBUILD.
4999 			 */
5000 			if (priority == ZIO_PRIORITY_INITIALIZING) {
5001 				ASSERT3U(type, ==, ZIO_TYPE_WRITE);
5002 				priority = ZIO_PRIORITY_ASYNC_WRITE;
5003 			} else if (priority == ZIO_PRIORITY_REMOVAL) {
5004 				priority = ((type == ZIO_TYPE_WRITE) ?
5005 				    ZIO_PRIORITY_ASYNC_WRITE :
5006 				    ZIO_PRIORITY_ASYNC_READ);
5007 			}
5008 
5009 			vs->vs_ops[vs_type]++;
5010 			vs->vs_bytes[vs_type] += psize;
5011 
5012 			if (flags & ZIO_FLAG_DELEGATED) {
5013 				vsx->vsx_agg_histo[priority]
5014 				    [RQ_HISTO(zio->io_size)]++;
5015 			} else {
5016 				vsx->vsx_ind_histo[priority]
5017 				    [RQ_HISTO(zio->io_size)]++;
5018 			}
5019 
5020 			if (zio->io_delta && zio->io_delay) {
5021 				vsx->vsx_queue_histo[priority]
5022 				    [L_HISTO(zio->io_delta - zio->io_delay)]++;
5023 				vsx->vsx_disk_histo[type]
5024 				    [L_HISTO(zio->io_delay)]++;
5025 				vsx->vsx_total_histo[type]
5026 				    [L_HISTO(zio->io_delta)]++;
5027 			}
5028 		}
5029 
5030 		mutex_exit(&vd->vdev_stat_lock);
5031 		return;
5032 	}
5033 
5034 	if (flags & ZIO_FLAG_SPECULATIVE)
5035 		return;
5036 
5037 	/*
5038 	 * If this is an I/O error that is going to be retried, then ignore the
5039 	 * error.  Otherwise, the user may interpret B_FAILFAST I/O errors as
5040 	 * hard errors, when in reality they can happen for any number of
5041 	 * innocuous reasons (bus resets, MPxIO link failure, etc).
5042 	 */
5043 	if (zio->io_error == EIO &&
5044 	    !(zio->io_flags & ZIO_FLAG_IO_RETRY))
5045 		return;
5046 
5047 	/*
5048 	 * Intent logs writes won't propagate their error to the root
5049 	 * I/O so don't mark these types of failures as pool-level
5050 	 * errors.
5051 	 */
5052 	if (zio->io_vd == NULL && (zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
5053 		return;
5054 
5055 	if (type == ZIO_TYPE_WRITE && txg != 0 &&
5056 	    (!(flags & ZIO_FLAG_IO_REPAIR) ||
5057 	    (flags & ZIO_FLAG_SCAN_THREAD) ||
5058 	    spa->spa_claiming)) {
5059 		/*
5060 		 * This is either a normal write (not a repair), or it's
5061 		 * a repair induced by the scrub thread, or it's a repair
5062 		 * made by zil_claim() during spa_load() in the first txg.
5063 		 * In the normal case, we commit the DTL change in the same
5064 		 * txg as the block was born.  In the scrub-induced repair
5065 		 * case, we know that scrubs run in first-pass syncing context,
5066 		 * so we commit the DTL change in spa_syncing_txg(spa).
5067 		 * In the zil_claim() case, we commit in spa_first_txg(spa).
5068 		 *
5069 		 * We currently do not make DTL entries for failed spontaneous
5070 		 * self-healing writes triggered by normal (non-scrubbing)
5071 		 * reads, because we have no transactional context in which to
5072 		 * do so -- and it's not clear that it'd be desirable anyway.
5073 		 */
5074 		if (vd->vdev_ops->vdev_op_leaf) {
5075 			uint64_t commit_txg = txg;
5076 			if (flags & ZIO_FLAG_SCAN_THREAD) {
5077 				ASSERT(flags & ZIO_FLAG_IO_REPAIR);
5078 				ASSERT(spa_sync_pass(spa) == 1);
5079 				vdev_dtl_dirty(vd, DTL_SCRUB, txg, 1);
5080 				commit_txg = spa_syncing_txg(spa);
5081 			} else if (spa->spa_claiming) {
5082 				ASSERT(flags & ZIO_FLAG_IO_REPAIR);
5083 				commit_txg = spa_first_txg(spa);
5084 			}
5085 			ASSERT(commit_txg >= spa_syncing_txg(spa));
5086 			if (vdev_dtl_contains(vd, DTL_MISSING, txg, 1))
5087 				return;
5088 			for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
5089 				vdev_dtl_dirty(pvd, DTL_PARTIAL, txg, 1);
5090 			vdev_dirty(vd->vdev_top, VDD_DTL, vd, commit_txg);
5091 		}
5092 		if (vd != rvd)
5093 			vdev_dtl_dirty(vd, DTL_MISSING, txg, 1);
5094 	}
5095 }
5096 
5097 int64_t
5098 vdev_deflated_space(vdev_t *vd, int64_t space)
5099 {
5100 	ASSERT((space & (SPA_MINBLOCKSIZE-1)) == 0);
5101 	ASSERT(vd->vdev_deflate_ratio != 0 || vd->vdev_isl2cache);
5102 
5103 	return ((space >> SPA_MINBLOCKSHIFT) * vd->vdev_deflate_ratio);
5104 }
5105 
5106 /*
5107  * Update the in-core space usage stats for this vdev, its metaslab class,
5108  * and the root vdev.
5109  */
5110 void
5111 vdev_space_update(vdev_t *vd, int64_t alloc_delta, int64_t defer_delta,
5112     int64_t space_delta)
5113 {
5114 	(void) defer_delta;
5115 	int64_t dspace_delta;
5116 	spa_t *spa = vd->vdev_spa;
5117 	vdev_t *rvd = spa->spa_root_vdev;
5118 
5119 	ASSERT(vd == vd->vdev_top);
5120 
5121 	/*
5122 	 * Apply the inverse of the psize-to-asize (ie. RAID-Z) space-expansion
5123 	 * factor.  We must calculate this here and not at the root vdev
5124 	 * because the root vdev's psize-to-asize is simply the max of its
5125 	 * children's, thus not accurate enough for us.
5126 	 */
5127 	dspace_delta = vdev_deflated_space(vd, space_delta);
5128 
5129 	mutex_enter(&vd->vdev_stat_lock);
5130 	/* ensure we won't underflow */
5131 	if (alloc_delta < 0) {
5132 		ASSERT3U(vd->vdev_stat.vs_alloc, >=, -alloc_delta);
5133 	}
5134 
5135 	vd->vdev_stat.vs_alloc += alloc_delta;
5136 	vd->vdev_stat.vs_space += space_delta;
5137 	vd->vdev_stat.vs_dspace += dspace_delta;
5138 	mutex_exit(&vd->vdev_stat_lock);
5139 
5140 	/* every class but log contributes to root space stats */
5141 	if (vd->vdev_mg != NULL && !vd->vdev_islog) {
5142 		ASSERT(!vd->vdev_isl2cache);
5143 		mutex_enter(&rvd->vdev_stat_lock);
5144 		rvd->vdev_stat.vs_alloc += alloc_delta;
5145 		rvd->vdev_stat.vs_space += space_delta;
5146 		rvd->vdev_stat.vs_dspace += dspace_delta;
5147 		mutex_exit(&rvd->vdev_stat_lock);
5148 	}
5149 	/* Note: metaslab_class_space_update moved to metaslab_space_update */
5150 }
5151 
5152 /*
5153  * Mark a top-level vdev's config as dirty, placing it on the dirty list
5154  * so that it will be written out next time the vdev configuration is synced.
5155  * If the root vdev is specified (vdev_top == NULL), dirty all top-level vdevs.
5156  */
5157 void
5158 vdev_config_dirty(vdev_t *vd)
5159 {
5160 	spa_t *spa = vd->vdev_spa;
5161 	vdev_t *rvd = spa->spa_root_vdev;
5162 	int c;
5163 
5164 	ASSERT(spa_writeable(spa));
5165 
5166 	/*
5167 	 * If this is an aux vdev (as with l2cache and spare devices), then we
5168 	 * update the vdev config manually and set the sync flag.
5169 	 */
5170 	if (vd->vdev_aux != NULL) {
5171 		spa_aux_vdev_t *sav = vd->vdev_aux;
5172 		nvlist_t **aux;
5173 		uint_t naux;
5174 
5175 		for (c = 0; c < sav->sav_count; c++) {
5176 			if (sav->sav_vdevs[c] == vd)
5177 				break;
5178 		}
5179 
5180 		if (c == sav->sav_count) {
5181 			/*
5182 			 * We're being removed.  There's nothing more to do.
5183 			 */
5184 			ASSERT(sav->sav_sync == B_TRUE);
5185 			return;
5186 		}
5187 
5188 		sav->sav_sync = B_TRUE;
5189 
5190 		if (nvlist_lookup_nvlist_array(sav->sav_config,
5191 		    ZPOOL_CONFIG_L2CACHE, &aux, &naux) != 0) {
5192 			VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
5193 			    ZPOOL_CONFIG_SPARES, &aux, &naux) == 0);
5194 		}
5195 
5196 		ASSERT(c < naux);
5197 
5198 		/*
5199 		 * Setting the nvlist in the middle if the array is a little
5200 		 * sketchy, but it will work.
5201 		 */
5202 		nvlist_free(aux[c]);
5203 		aux[c] = vdev_config_generate(spa, vd, B_TRUE, 0);
5204 
5205 		return;
5206 	}
5207 
5208 	/*
5209 	 * The dirty list is protected by the SCL_CONFIG lock.  The caller
5210 	 * must either hold SCL_CONFIG as writer, or must be the sync thread
5211 	 * (which holds SCL_CONFIG as reader).  There's only one sync thread,
5212 	 * so this is sufficient to ensure mutual exclusion.
5213 	 */
5214 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_WRITER) ||
5215 	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
5216 	    spa_config_held(spa, SCL_CONFIG, RW_READER)));
5217 
5218 	if (vd == rvd) {
5219 		for (c = 0; c < rvd->vdev_children; c++)
5220 			vdev_config_dirty(rvd->vdev_child[c]);
5221 	} else {
5222 		ASSERT(vd == vd->vdev_top);
5223 
5224 		if (!list_link_active(&vd->vdev_config_dirty_node) &&
5225 		    vdev_is_concrete(vd)) {
5226 			list_insert_head(&spa->spa_config_dirty_list, vd);
5227 		}
5228 	}
5229 }
5230 
5231 void
5232 vdev_config_clean(vdev_t *vd)
5233 {
5234 	spa_t *spa = vd->vdev_spa;
5235 
5236 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_WRITER) ||
5237 	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
5238 	    spa_config_held(spa, SCL_CONFIG, RW_READER)));
5239 
5240 	ASSERT(list_link_active(&vd->vdev_config_dirty_node));
5241 	list_remove(&spa->spa_config_dirty_list, vd);
5242 }
5243 
5244 /*
5245  * Mark a top-level vdev's state as dirty, so that the next pass of
5246  * spa_sync() can convert this into vdev_config_dirty().  We distinguish
5247  * the state changes from larger config changes because they require
5248  * much less locking, and are often needed for administrative actions.
5249  */
5250 void
5251 vdev_state_dirty(vdev_t *vd)
5252 {
5253 	spa_t *spa = vd->vdev_spa;
5254 
5255 	ASSERT(spa_writeable(spa));
5256 	ASSERT(vd == vd->vdev_top);
5257 
5258 	/*
5259 	 * The state list is protected by the SCL_STATE lock.  The caller
5260 	 * must either hold SCL_STATE as writer, or must be the sync thread
5261 	 * (which holds SCL_STATE as reader).  There's only one sync thread,
5262 	 * so this is sufficient to ensure mutual exclusion.
5263 	 */
5264 	ASSERT(spa_config_held(spa, SCL_STATE, RW_WRITER) ||
5265 	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
5266 	    spa_config_held(spa, SCL_STATE, RW_READER)));
5267 
5268 	if (!list_link_active(&vd->vdev_state_dirty_node) &&
5269 	    vdev_is_concrete(vd))
5270 		list_insert_head(&spa->spa_state_dirty_list, vd);
5271 }
5272 
5273 void
5274 vdev_state_clean(vdev_t *vd)
5275 {
5276 	spa_t *spa = vd->vdev_spa;
5277 
5278 	ASSERT(spa_config_held(spa, SCL_STATE, RW_WRITER) ||
5279 	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
5280 	    spa_config_held(spa, SCL_STATE, RW_READER)));
5281 
5282 	ASSERT(list_link_active(&vd->vdev_state_dirty_node));
5283 	list_remove(&spa->spa_state_dirty_list, vd);
5284 }
5285 
5286 /*
5287  * Propagate vdev state up from children to parent.
5288  */
5289 void
5290 vdev_propagate_state(vdev_t *vd)
5291 {
5292 	spa_t *spa = vd->vdev_spa;
5293 	vdev_t *rvd = spa->spa_root_vdev;
5294 	int degraded = 0, faulted = 0;
5295 	int corrupted = 0;
5296 	vdev_t *child;
5297 
5298 	if (vd->vdev_children > 0) {
5299 		for (int c = 0; c < vd->vdev_children; c++) {
5300 			child = vd->vdev_child[c];
5301 
5302 			/*
5303 			 * Don't factor holes or indirect vdevs into the
5304 			 * decision.
5305 			 */
5306 			if (!vdev_is_concrete(child))
5307 				continue;
5308 
5309 			if (!vdev_readable(child) ||
5310 			    (!vdev_writeable(child) && spa_writeable(spa))) {
5311 				/*
5312 				 * Root special: if there is a top-level log
5313 				 * device, treat the root vdev as if it were
5314 				 * degraded.
5315 				 */
5316 				if (child->vdev_islog && vd == rvd)
5317 					degraded++;
5318 				else
5319 					faulted++;
5320 			} else if (child->vdev_state <= VDEV_STATE_DEGRADED) {
5321 				degraded++;
5322 			}
5323 
5324 			if (child->vdev_stat.vs_aux == VDEV_AUX_CORRUPT_DATA)
5325 				corrupted++;
5326 		}
5327 
5328 		vd->vdev_ops->vdev_op_state_change(vd, faulted, degraded);
5329 
5330 		/*
5331 		 * Root special: if there is a top-level vdev that cannot be
5332 		 * opened due to corrupted metadata, then propagate the root
5333 		 * vdev's aux state as 'corrupt' rather than 'insufficient
5334 		 * replicas'.
5335 		 */
5336 		if (corrupted && vd == rvd &&
5337 		    rvd->vdev_state == VDEV_STATE_CANT_OPEN)
5338 			vdev_set_state(rvd, B_FALSE, VDEV_STATE_CANT_OPEN,
5339 			    VDEV_AUX_CORRUPT_DATA);
5340 	}
5341 
5342 	if (vd->vdev_parent)
5343 		vdev_propagate_state(vd->vdev_parent);
5344 }
5345 
5346 /*
5347  * Set a vdev's state.  If this is during an open, we don't update the parent
5348  * state, because we're in the process of opening children depth-first.
5349  * Otherwise, we propagate the change to the parent.
5350  *
5351  * If this routine places a device in a faulted state, an appropriate ereport is
5352  * generated.
5353  */
5354 void
5355 vdev_set_state(vdev_t *vd, boolean_t isopen, vdev_state_t state, vdev_aux_t aux)
5356 {
5357 	uint64_t save_state;
5358 	spa_t *spa = vd->vdev_spa;
5359 
5360 	if (state == vd->vdev_state) {
5361 		/*
5362 		 * Since vdev_offline() code path is already in an offline
5363 		 * state we can miss a statechange event to OFFLINE. Check
5364 		 * the previous state to catch this condition.
5365 		 */
5366 		if (vd->vdev_ops->vdev_op_leaf &&
5367 		    (state == VDEV_STATE_OFFLINE) &&
5368 		    (vd->vdev_prevstate >= VDEV_STATE_FAULTED)) {
5369 			/* post an offline state change */
5370 			zfs_post_state_change(spa, vd, vd->vdev_prevstate);
5371 		}
5372 		vd->vdev_stat.vs_aux = aux;
5373 		return;
5374 	}
5375 
5376 	save_state = vd->vdev_state;
5377 
5378 	vd->vdev_state = state;
5379 	vd->vdev_stat.vs_aux = aux;
5380 
5381 	/*
5382 	 * If we are setting the vdev state to anything but an open state, then
5383 	 * always close the underlying device unless the device has requested
5384 	 * a delayed close (i.e. we're about to remove or fault the device).
5385 	 * Otherwise, we keep accessible but invalid devices open forever.
5386 	 * We don't call vdev_close() itself, because that implies some extra
5387 	 * checks (offline, etc) that we don't want here.  This is limited to
5388 	 * leaf devices, because otherwise closing the device will affect other
5389 	 * children.
5390 	 */
5391 	if (!vd->vdev_delayed_close && vdev_is_dead(vd) &&
5392 	    vd->vdev_ops->vdev_op_leaf)
5393 		vd->vdev_ops->vdev_op_close(vd);
5394 
5395 	if (vd->vdev_removed &&
5396 	    state == VDEV_STATE_CANT_OPEN &&
5397 	    (aux == VDEV_AUX_OPEN_FAILED || vd->vdev_checkremove)) {
5398 		/*
5399 		 * If the previous state is set to VDEV_STATE_REMOVED, then this
5400 		 * device was previously marked removed and someone attempted to
5401 		 * reopen it.  If this failed due to a nonexistent device, then
5402 		 * keep the device in the REMOVED state.  We also let this be if
5403 		 * it is one of our special test online cases, which is only
5404 		 * attempting to online the device and shouldn't generate an FMA
5405 		 * fault.
5406 		 */
5407 		vd->vdev_state = VDEV_STATE_REMOVED;
5408 		vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
5409 	} else if (state == VDEV_STATE_REMOVED) {
5410 		vd->vdev_removed = B_TRUE;
5411 	} else if (state == VDEV_STATE_CANT_OPEN) {
5412 		/*
5413 		 * If we fail to open a vdev during an import or recovery, we
5414 		 * mark it as "not available", which signifies that it was
5415 		 * never there to begin with.  Failure to open such a device
5416 		 * is not considered an error.
5417 		 */
5418 		if ((spa_load_state(spa) == SPA_LOAD_IMPORT ||
5419 		    spa_load_state(spa) == SPA_LOAD_RECOVER) &&
5420 		    vd->vdev_ops->vdev_op_leaf)
5421 			vd->vdev_not_present = 1;
5422 
5423 		/*
5424 		 * Post the appropriate ereport.  If the 'prevstate' field is
5425 		 * set to something other than VDEV_STATE_UNKNOWN, it indicates
5426 		 * that this is part of a vdev_reopen().  In this case, we don't
5427 		 * want to post the ereport if the device was already in the
5428 		 * CANT_OPEN state beforehand.
5429 		 *
5430 		 * If the 'checkremove' flag is set, then this is an attempt to
5431 		 * online the device in response to an insertion event.  If we
5432 		 * hit this case, then we have detected an insertion event for a
5433 		 * faulted or offline device that wasn't in the removed state.
5434 		 * In this scenario, we don't post an ereport because we are
5435 		 * about to replace the device, or attempt an online with
5436 		 * vdev_forcefault, which will generate the fault for us.
5437 		 */
5438 		if ((vd->vdev_prevstate != state || vd->vdev_forcefault) &&
5439 		    !vd->vdev_not_present && !vd->vdev_checkremove &&
5440 		    vd != spa->spa_root_vdev) {
5441 			const char *class;
5442 
5443 			switch (aux) {
5444 			case VDEV_AUX_OPEN_FAILED:
5445 				class = FM_EREPORT_ZFS_DEVICE_OPEN_FAILED;
5446 				break;
5447 			case VDEV_AUX_CORRUPT_DATA:
5448 				class = FM_EREPORT_ZFS_DEVICE_CORRUPT_DATA;
5449 				break;
5450 			case VDEV_AUX_NO_REPLICAS:
5451 				class = FM_EREPORT_ZFS_DEVICE_NO_REPLICAS;
5452 				break;
5453 			case VDEV_AUX_BAD_GUID_SUM:
5454 				class = FM_EREPORT_ZFS_DEVICE_BAD_GUID_SUM;
5455 				break;
5456 			case VDEV_AUX_TOO_SMALL:
5457 				class = FM_EREPORT_ZFS_DEVICE_TOO_SMALL;
5458 				break;
5459 			case VDEV_AUX_BAD_LABEL:
5460 				class = FM_EREPORT_ZFS_DEVICE_BAD_LABEL;
5461 				break;
5462 			case VDEV_AUX_BAD_ASHIFT:
5463 				class = FM_EREPORT_ZFS_DEVICE_BAD_ASHIFT;
5464 				break;
5465 			default:
5466 				class = FM_EREPORT_ZFS_DEVICE_UNKNOWN;
5467 			}
5468 
5469 			(void) zfs_ereport_post(class, spa, vd, NULL, NULL,
5470 			    save_state);
5471 		}
5472 
5473 		/* Erase any notion of persistent removed state */
5474 		vd->vdev_removed = B_FALSE;
5475 	} else {
5476 		vd->vdev_removed = B_FALSE;
5477 	}
5478 
5479 	/*
5480 	 * Notify ZED of any significant state-change on a leaf vdev.
5481 	 *
5482 	 */
5483 	if (vd->vdev_ops->vdev_op_leaf) {
5484 		/* preserve original state from a vdev_reopen() */
5485 		if ((vd->vdev_prevstate != VDEV_STATE_UNKNOWN) &&
5486 		    (vd->vdev_prevstate != vd->vdev_state) &&
5487 		    (save_state <= VDEV_STATE_CLOSED))
5488 			save_state = vd->vdev_prevstate;
5489 
5490 		/* filter out state change due to initial vdev_open */
5491 		if (save_state > VDEV_STATE_CLOSED)
5492 			zfs_post_state_change(spa, vd, save_state);
5493 	}
5494 
5495 	if (!isopen && vd->vdev_parent)
5496 		vdev_propagate_state(vd->vdev_parent);
5497 }
5498 
5499 boolean_t
5500 vdev_children_are_offline(vdev_t *vd)
5501 {
5502 	ASSERT(!vd->vdev_ops->vdev_op_leaf);
5503 
5504 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
5505 		if (vd->vdev_child[i]->vdev_state != VDEV_STATE_OFFLINE)
5506 			return (B_FALSE);
5507 	}
5508 
5509 	return (B_TRUE);
5510 }
5511 
5512 /*
5513  * Check the vdev configuration to ensure that it's capable of supporting
5514  * a root pool. We do not support partial configuration.
5515  */
5516 boolean_t
5517 vdev_is_bootable(vdev_t *vd)
5518 {
5519 	if (!vd->vdev_ops->vdev_op_leaf) {
5520 		const char *vdev_type = vd->vdev_ops->vdev_op_type;
5521 
5522 		if (strcmp(vdev_type, VDEV_TYPE_MISSING) == 0)
5523 			return (B_FALSE);
5524 	}
5525 
5526 	for (int c = 0; c < vd->vdev_children; c++) {
5527 		if (!vdev_is_bootable(vd->vdev_child[c]))
5528 			return (B_FALSE);
5529 	}
5530 	return (B_TRUE);
5531 }
5532 
5533 boolean_t
5534 vdev_is_concrete(vdev_t *vd)
5535 {
5536 	vdev_ops_t *ops = vd->vdev_ops;
5537 	if (ops == &vdev_indirect_ops || ops == &vdev_hole_ops ||
5538 	    ops == &vdev_missing_ops || ops == &vdev_root_ops) {
5539 		return (B_FALSE);
5540 	} else {
5541 		return (B_TRUE);
5542 	}
5543 }
5544 
5545 /*
5546  * Determine if a log device has valid content.  If the vdev was
5547  * removed or faulted in the MOS config then we know that
5548  * the content on the log device has already been written to the pool.
5549  */
5550 boolean_t
5551 vdev_log_state_valid(vdev_t *vd)
5552 {
5553 	if (vd->vdev_ops->vdev_op_leaf && !vd->vdev_faulted &&
5554 	    !vd->vdev_removed)
5555 		return (B_TRUE);
5556 
5557 	for (int c = 0; c < vd->vdev_children; c++)
5558 		if (vdev_log_state_valid(vd->vdev_child[c]))
5559 			return (B_TRUE);
5560 
5561 	return (B_FALSE);
5562 }
5563 
5564 /*
5565  * Expand a vdev if possible.
5566  */
5567 void
5568 vdev_expand(vdev_t *vd, uint64_t txg)
5569 {
5570 	ASSERT(vd->vdev_top == vd);
5571 	ASSERT(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5572 	ASSERT(vdev_is_concrete(vd));
5573 
5574 	vdev_set_deflate_ratio(vd);
5575 
5576 	if ((vd->vdev_spa->spa_raidz_expand == NULL ||
5577 	    vd->vdev_spa->spa_raidz_expand->vre_vdev_id != vd->vdev_id) &&
5578 	    (vd->vdev_asize >> vd->vdev_ms_shift) > vd->vdev_ms_count &&
5579 	    vdev_is_concrete(vd)) {
5580 		vdev_metaslab_group_create(vd);
5581 		VERIFY(vdev_metaslab_init(vd, txg) == 0);
5582 		vdev_config_dirty(vd);
5583 	}
5584 }
5585 
5586 /*
5587  * Split a vdev.
5588  */
5589 void
5590 vdev_split(vdev_t *vd)
5591 {
5592 	vdev_t *cvd, *pvd = vd->vdev_parent;
5593 
5594 	VERIFY3U(pvd->vdev_children, >, 1);
5595 
5596 	vdev_remove_child(pvd, vd);
5597 	vdev_compact_children(pvd);
5598 
5599 	ASSERT3P(pvd->vdev_child, !=, NULL);
5600 
5601 	cvd = pvd->vdev_child[0];
5602 	if (pvd->vdev_children == 1) {
5603 		vdev_remove_parent(cvd);
5604 		cvd->vdev_splitting = B_TRUE;
5605 	}
5606 	vdev_propagate_state(cvd);
5607 }
5608 
5609 void
5610 vdev_deadman(vdev_t *vd, const char *tag)
5611 {
5612 	for (int c = 0; c < vd->vdev_children; c++) {
5613 		vdev_t *cvd = vd->vdev_child[c];
5614 
5615 		vdev_deadman(cvd, tag);
5616 	}
5617 
5618 	if (vd->vdev_ops->vdev_op_leaf) {
5619 		vdev_queue_t *vq = &vd->vdev_queue;
5620 
5621 		mutex_enter(&vq->vq_lock);
5622 		if (vq->vq_active > 0) {
5623 			spa_t *spa = vd->vdev_spa;
5624 			zio_t *fio;
5625 			uint64_t delta;
5626 
5627 			zfs_dbgmsg("slow vdev: %s has %u active IOs",
5628 			    vd->vdev_path, vq->vq_active);
5629 
5630 			/*
5631 			 * Look at the head of all the pending queues,
5632 			 * if any I/O has been outstanding for longer than
5633 			 * the spa_deadman_synctime invoke the deadman logic.
5634 			 */
5635 			fio = list_head(&vq->vq_active_list);
5636 			delta = gethrtime() - fio->io_timestamp;
5637 			if (delta > spa_deadman_synctime(spa))
5638 				zio_deadman(fio, tag);
5639 		}
5640 		mutex_exit(&vq->vq_lock);
5641 	}
5642 }
5643 
5644 void
5645 vdev_defer_resilver(vdev_t *vd)
5646 {
5647 	ASSERT(vd->vdev_ops->vdev_op_leaf);
5648 
5649 	vd->vdev_resilver_deferred = B_TRUE;
5650 	vd->vdev_spa->spa_resilver_deferred = B_TRUE;
5651 }
5652 
5653 /*
5654  * Clears the resilver deferred flag on all leaf devs under vd. Returns
5655  * B_TRUE if we have devices that need to be resilvered and are available to
5656  * accept resilver I/Os.
5657  */
5658 boolean_t
5659 vdev_clear_resilver_deferred(vdev_t *vd, dmu_tx_t *tx)
5660 {
5661 	boolean_t resilver_needed = B_FALSE;
5662 	spa_t *spa = vd->vdev_spa;
5663 
5664 	for (int c = 0; c < vd->vdev_children; c++) {
5665 		vdev_t *cvd = vd->vdev_child[c];
5666 		resilver_needed |= vdev_clear_resilver_deferred(cvd, tx);
5667 	}
5668 
5669 	if (vd == spa->spa_root_vdev &&
5670 	    spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) {
5671 		spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx);
5672 		vdev_config_dirty(vd);
5673 		spa->spa_resilver_deferred = B_FALSE;
5674 		return (resilver_needed);
5675 	}
5676 
5677 	if (!vdev_is_concrete(vd) || vd->vdev_aux ||
5678 	    !vd->vdev_ops->vdev_op_leaf)
5679 		return (resilver_needed);
5680 
5681 	vd->vdev_resilver_deferred = B_FALSE;
5682 
5683 	return (!vdev_is_dead(vd) && !vd->vdev_offline &&
5684 	    vdev_resilver_needed(vd, NULL, NULL));
5685 }
5686 
5687 boolean_t
5688 vdev_xlate_is_empty(range_seg64_t *rs)
5689 {
5690 	return (rs->rs_start == rs->rs_end);
5691 }
5692 
5693 /*
5694  * Translate a logical range to the first contiguous physical range for the
5695  * specified vdev_t.  This function is initially called with a leaf vdev and
5696  * will walk each parent vdev until it reaches a top-level vdev. Once the
5697  * top-level is reached the physical range is initialized and the recursive
5698  * function begins to unwind. As it unwinds it calls the parent's vdev
5699  * specific translation function to do the real conversion.
5700  */
5701 void
5702 vdev_xlate(vdev_t *vd, const range_seg64_t *logical_rs,
5703     range_seg64_t *physical_rs, range_seg64_t *remain_rs)
5704 {
5705 	/*
5706 	 * Walk up the vdev tree
5707 	 */
5708 	if (vd != vd->vdev_top) {
5709 		vdev_xlate(vd->vdev_parent, logical_rs, physical_rs,
5710 		    remain_rs);
5711 	} else {
5712 		/*
5713 		 * We've reached the top-level vdev, initialize the physical
5714 		 * range to the logical range and set an empty remaining
5715 		 * range then start to unwind.
5716 		 */
5717 		physical_rs->rs_start = logical_rs->rs_start;
5718 		physical_rs->rs_end = logical_rs->rs_end;
5719 
5720 		remain_rs->rs_start = logical_rs->rs_start;
5721 		remain_rs->rs_end = logical_rs->rs_start;
5722 
5723 		return;
5724 	}
5725 
5726 	vdev_t *pvd = vd->vdev_parent;
5727 	ASSERT3P(pvd, !=, NULL);
5728 	ASSERT3P(pvd->vdev_ops->vdev_op_xlate, !=, NULL);
5729 
5730 	/*
5731 	 * As this recursive function unwinds, translate the logical
5732 	 * range into its physical and any remaining components by calling
5733 	 * the vdev specific translate function.
5734 	 */
5735 	range_seg64_t intermediate = { 0 };
5736 	pvd->vdev_ops->vdev_op_xlate(vd, physical_rs, &intermediate, remain_rs);
5737 
5738 	physical_rs->rs_start = intermediate.rs_start;
5739 	physical_rs->rs_end = intermediate.rs_end;
5740 }
5741 
5742 void
5743 vdev_xlate_walk(vdev_t *vd, const range_seg64_t *logical_rs,
5744     vdev_xlate_func_t *func, void *arg)
5745 {
5746 	range_seg64_t iter_rs = *logical_rs;
5747 	range_seg64_t physical_rs;
5748 	range_seg64_t remain_rs;
5749 
5750 	while (!vdev_xlate_is_empty(&iter_rs)) {
5751 
5752 		vdev_xlate(vd, &iter_rs, &physical_rs, &remain_rs);
5753 
5754 		/*
5755 		 * With raidz and dRAID, it's possible that the logical range
5756 		 * does not live on this leaf vdev. Only when there is a non-
5757 		 * zero physical size call the provided function.
5758 		 */
5759 		if (!vdev_xlate_is_empty(&physical_rs))
5760 			func(arg, &physical_rs);
5761 
5762 		iter_rs = remain_rs;
5763 	}
5764 }
5765 
5766 static char *
5767 vdev_name(vdev_t *vd, char *buf, int buflen)
5768 {
5769 	if (vd->vdev_path == NULL) {
5770 		if (strcmp(vd->vdev_ops->vdev_op_type, "root") == 0) {
5771 			strlcpy(buf, vd->vdev_spa->spa_name, buflen);
5772 		} else if (!vd->vdev_ops->vdev_op_leaf) {
5773 			snprintf(buf, buflen, "%s-%llu",
5774 			    vd->vdev_ops->vdev_op_type,
5775 			    (u_longlong_t)vd->vdev_id);
5776 		}
5777 	} else {
5778 		strlcpy(buf, vd->vdev_path, buflen);
5779 	}
5780 	return (buf);
5781 }
5782 
5783 /*
5784  * Look at the vdev tree and determine whether any devices are currently being
5785  * replaced.
5786  */
5787 boolean_t
5788 vdev_replace_in_progress(vdev_t *vdev)
5789 {
5790 	ASSERT(spa_config_held(vdev->vdev_spa, SCL_ALL, RW_READER) != 0);
5791 
5792 	if (vdev->vdev_ops == &vdev_replacing_ops)
5793 		return (B_TRUE);
5794 
5795 	/*
5796 	 * A 'spare' vdev indicates that we have a replace in progress, unless
5797 	 * it has exactly two children, and the second, the hot spare, has
5798 	 * finished being resilvered.
5799 	 */
5800 	if (vdev->vdev_ops == &vdev_spare_ops && (vdev->vdev_children > 2 ||
5801 	    !vdev_dtl_empty(vdev->vdev_child[1], DTL_MISSING)))
5802 		return (B_TRUE);
5803 
5804 	for (int i = 0; i < vdev->vdev_children; i++) {
5805 		if (vdev_replace_in_progress(vdev->vdev_child[i]))
5806 			return (B_TRUE);
5807 	}
5808 
5809 	return (B_FALSE);
5810 }
5811 
5812 /*
5813  * Add a (source=src, propname=propval) list to an nvlist.
5814  */
5815 static void
5816 vdev_prop_add_list(nvlist_t *nvl, const char *propname, const char *strval,
5817     uint64_t intval, zprop_source_t src)
5818 {
5819 	nvlist_t *propval;
5820 
5821 	propval = fnvlist_alloc();
5822 	fnvlist_add_uint64(propval, ZPROP_SOURCE, src);
5823 
5824 	if (strval != NULL)
5825 		fnvlist_add_string(propval, ZPROP_VALUE, strval);
5826 	else
5827 		fnvlist_add_uint64(propval, ZPROP_VALUE, intval);
5828 
5829 	fnvlist_add_nvlist(nvl, propname, propval);
5830 	nvlist_free(propval);
5831 }
5832 
5833 static void
5834 vdev_props_set_sync(void *arg, dmu_tx_t *tx)
5835 {
5836 	vdev_t *vd;
5837 	nvlist_t *nvp = arg;
5838 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
5839 	objset_t *mos = spa->spa_meta_objset;
5840 	nvpair_t *elem = NULL;
5841 	uint64_t vdev_guid;
5842 	uint64_t objid;
5843 	nvlist_t *nvprops;
5844 
5845 	vdev_guid = fnvlist_lookup_uint64(nvp, ZPOOL_VDEV_PROPS_SET_VDEV);
5846 	nvprops = fnvlist_lookup_nvlist(nvp, ZPOOL_VDEV_PROPS_SET_PROPS);
5847 	vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE);
5848 
5849 	/* this vdev could get removed while waiting for this sync task */
5850 	if (vd == NULL)
5851 		return;
5852 
5853 	/*
5854 	 * Set vdev property values in the vdev props mos object.
5855 	 */
5856 	if (vd->vdev_root_zap != 0) {
5857 		objid = vd->vdev_root_zap;
5858 	} else if (vd->vdev_top_zap != 0) {
5859 		objid = vd->vdev_top_zap;
5860 	} else if (vd->vdev_leaf_zap != 0) {
5861 		objid = vd->vdev_leaf_zap;
5862 	} else {
5863 		panic("unexpected vdev type");
5864 	}
5865 
5866 	mutex_enter(&spa->spa_props_lock);
5867 
5868 	while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) {
5869 		uint64_t intval;
5870 		const char *strval;
5871 		vdev_prop_t prop;
5872 		const char *propname = nvpair_name(elem);
5873 		zprop_type_t proptype;
5874 
5875 		switch (prop = vdev_name_to_prop(propname)) {
5876 		case VDEV_PROP_USERPROP:
5877 			if (vdev_prop_user(propname)) {
5878 				strval = fnvpair_value_string(elem);
5879 				if (strlen(strval) == 0) {
5880 					/* remove the property if value == "" */
5881 					(void) zap_remove(mos, objid, propname,
5882 					    tx);
5883 				} else {
5884 					VERIFY0(zap_update(mos, objid, propname,
5885 					    1, strlen(strval) + 1, strval, tx));
5886 				}
5887 				spa_history_log_internal(spa, "vdev set", tx,
5888 				    "vdev_guid=%llu: %s=%s",
5889 				    (u_longlong_t)vdev_guid, nvpair_name(elem),
5890 				    strval);
5891 			}
5892 			break;
5893 		default:
5894 			/* normalize the property name */
5895 			propname = vdev_prop_to_name(prop);
5896 			proptype = vdev_prop_get_type(prop);
5897 
5898 			if (nvpair_type(elem) == DATA_TYPE_STRING) {
5899 				ASSERT(proptype == PROP_TYPE_STRING);
5900 				strval = fnvpair_value_string(elem);
5901 				VERIFY0(zap_update(mos, objid, propname,
5902 				    1, strlen(strval) + 1, strval, tx));
5903 				spa_history_log_internal(spa, "vdev set", tx,
5904 				    "vdev_guid=%llu: %s=%s",
5905 				    (u_longlong_t)vdev_guid, nvpair_name(elem),
5906 				    strval);
5907 			} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
5908 				intval = fnvpair_value_uint64(elem);
5909 
5910 				if (proptype == PROP_TYPE_INDEX) {
5911 					const char *unused;
5912 					VERIFY0(vdev_prop_index_to_string(
5913 					    prop, intval, &unused));
5914 				}
5915 				VERIFY0(zap_update(mos, objid, propname,
5916 				    sizeof (uint64_t), 1, &intval, tx));
5917 				spa_history_log_internal(spa, "vdev set", tx,
5918 				    "vdev_guid=%llu: %s=%lld",
5919 				    (u_longlong_t)vdev_guid,
5920 				    nvpair_name(elem), (longlong_t)intval);
5921 			} else {
5922 				panic("invalid vdev property type %u",
5923 				    nvpair_type(elem));
5924 			}
5925 		}
5926 
5927 	}
5928 
5929 	mutex_exit(&spa->spa_props_lock);
5930 }
5931 
5932 int
5933 vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
5934 {
5935 	spa_t *spa = vd->vdev_spa;
5936 	nvpair_t *elem = NULL;
5937 	uint64_t vdev_guid;
5938 	nvlist_t *nvprops;
5939 	int error = 0;
5940 
5941 	ASSERT(vd != NULL);
5942 
5943 	/* Check that vdev has a zap we can use */
5944 	if (vd->vdev_root_zap == 0 &&
5945 	    vd->vdev_top_zap == 0 &&
5946 	    vd->vdev_leaf_zap == 0)
5947 		return (SET_ERROR(EINVAL));
5948 
5949 	if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV,
5950 	    &vdev_guid) != 0)
5951 		return (SET_ERROR(EINVAL));
5952 
5953 	if (nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_SET_PROPS,
5954 	    &nvprops) != 0)
5955 		return (SET_ERROR(EINVAL));
5956 
5957 	if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL)
5958 		return (SET_ERROR(EINVAL));
5959 
5960 	while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) {
5961 		const char *propname = nvpair_name(elem);
5962 		vdev_prop_t prop = vdev_name_to_prop(propname);
5963 		uint64_t intval = 0;
5964 		const char *strval = NULL;
5965 
5966 		if (prop == VDEV_PROP_USERPROP && !vdev_prop_user(propname)) {
5967 			error = EINVAL;
5968 			goto end;
5969 		}
5970 
5971 		if (vdev_prop_readonly(prop)) {
5972 			error = EROFS;
5973 			goto end;
5974 		}
5975 
5976 		/* Special Processing */
5977 		switch (prop) {
5978 		case VDEV_PROP_PATH:
5979 			if (vd->vdev_path == NULL) {
5980 				error = EROFS;
5981 				break;
5982 			}
5983 			if (nvpair_value_string(elem, &strval) != 0) {
5984 				error = EINVAL;
5985 				break;
5986 			}
5987 			/* New path must start with /dev/ */
5988 			if (strncmp(strval, "/dev/", 5)) {
5989 				error = EINVAL;
5990 				break;
5991 			}
5992 			error = spa_vdev_setpath(spa, vdev_guid, strval);
5993 			break;
5994 		case VDEV_PROP_ALLOCATING:
5995 			if (nvpair_value_uint64(elem, &intval) != 0) {
5996 				error = EINVAL;
5997 				break;
5998 			}
5999 			if (intval != vd->vdev_noalloc)
6000 				break;
6001 			if (intval == 0)
6002 				error = spa_vdev_noalloc(spa, vdev_guid);
6003 			else
6004 				error = spa_vdev_alloc(spa, vdev_guid);
6005 			break;
6006 		case VDEV_PROP_FAILFAST:
6007 			if (nvpair_value_uint64(elem, &intval) != 0) {
6008 				error = EINVAL;
6009 				break;
6010 			}
6011 			vd->vdev_failfast = intval & 1;
6012 			break;
6013 		case VDEV_PROP_CHECKSUM_N:
6014 			if (nvpair_value_uint64(elem, &intval) != 0) {
6015 				error = EINVAL;
6016 				break;
6017 			}
6018 			vd->vdev_checksum_n = intval;
6019 			break;
6020 		case VDEV_PROP_CHECKSUM_T:
6021 			if (nvpair_value_uint64(elem, &intval) != 0) {
6022 				error = EINVAL;
6023 				break;
6024 			}
6025 			vd->vdev_checksum_t = intval;
6026 			break;
6027 		case VDEV_PROP_IO_N:
6028 			if (nvpair_value_uint64(elem, &intval) != 0) {
6029 				error = EINVAL;
6030 				break;
6031 			}
6032 			vd->vdev_io_n = intval;
6033 			break;
6034 		case VDEV_PROP_IO_T:
6035 			if (nvpair_value_uint64(elem, &intval) != 0) {
6036 				error = EINVAL;
6037 				break;
6038 			}
6039 			vd->vdev_io_t = intval;
6040 			break;
6041 		case VDEV_PROP_SLOW_IO_N:
6042 			if (nvpair_value_uint64(elem, &intval) != 0) {
6043 				error = EINVAL;
6044 				break;
6045 			}
6046 			vd->vdev_slow_io_n = intval;
6047 			break;
6048 		case VDEV_PROP_SLOW_IO_T:
6049 			if (nvpair_value_uint64(elem, &intval) != 0) {
6050 				error = EINVAL;
6051 				break;
6052 			}
6053 			vd->vdev_slow_io_t = intval;
6054 			break;
6055 		default:
6056 			/* Most processing is done in vdev_props_set_sync */
6057 			break;
6058 		}
6059 end:
6060 		if (error != 0) {
6061 			intval = error;
6062 			vdev_prop_add_list(outnvl, propname, strval, intval, 0);
6063 			return (error);
6064 		}
6065 	}
6066 
6067 	return (dsl_sync_task(spa->spa_name, NULL, vdev_props_set_sync,
6068 	    innvl, 6, ZFS_SPACE_CHECK_EXTRA_RESERVED));
6069 }
6070 
6071 int
6072 vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
6073 {
6074 	spa_t *spa = vd->vdev_spa;
6075 	objset_t *mos = spa->spa_meta_objset;
6076 	int err = 0;
6077 	uint64_t objid;
6078 	uint64_t vdev_guid;
6079 	nvpair_t *elem = NULL;
6080 	nvlist_t *nvprops = NULL;
6081 	uint64_t intval = 0;
6082 	char *strval = NULL;
6083 	const char *propname = NULL;
6084 	vdev_prop_t prop;
6085 
6086 	ASSERT(vd != NULL);
6087 	ASSERT(mos != NULL);
6088 
6089 	if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV,
6090 	    &vdev_guid) != 0)
6091 		return (SET_ERROR(EINVAL));
6092 
6093 	nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_GET_PROPS, &nvprops);
6094 
6095 	if (vd->vdev_root_zap != 0) {
6096 		objid = vd->vdev_root_zap;
6097 	} else if (vd->vdev_top_zap != 0) {
6098 		objid = vd->vdev_top_zap;
6099 	} else if (vd->vdev_leaf_zap != 0) {
6100 		objid = vd->vdev_leaf_zap;
6101 	} else {
6102 		return (SET_ERROR(EINVAL));
6103 	}
6104 	ASSERT(objid != 0);
6105 
6106 	mutex_enter(&spa->spa_props_lock);
6107 
6108 	if (nvprops != NULL) {
6109 		char namebuf[64] = { 0 };
6110 
6111 		while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) {
6112 			intval = 0;
6113 			strval = NULL;
6114 			propname = nvpair_name(elem);
6115 			prop = vdev_name_to_prop(propname);
6116 			zprop_source_t src = ZPROP_SRC_DEFAULT;
6117 			uint64_t integer_size, num_integers;
6118 
6119 			switch (prop) {
6120 			/* Special Read-only Properties */
6121 			case VDEV_PROP_NAME:
6122 				strval = vdev_name(vd, namebuf,
6123 				    sizeof (namebuf));
6124 				if (strval == NULL)
6125 					continue;
6126 				vdev_prop_add_list(outnvl, propname, strval, 0,
6127 				    ZPROP_SRC_NONE);
6128 				continue;
6129 			case VDEV_PROP_CAPACITY:
6130 				/* percent used */
6131 				intval = (vd->vdev_stat.vs_dspace == 0) ? 0 :
6132 				    (vd->vdev_stat.vs_alloc * 100 /
6133 				    vd->vdev_stat.vs_dspace);
6134 				vdev_prop_add_list(outnvl, propname, NULL,
6135 				    intval, ZPROP_SRC_NONE);
6136 				continue;
6137 			case VDEV_PROP_STATE:
6138 				vdev_prop_add_list(outnvl, propname, NULL,
6139 				    vd->vdev_state, ZPROP_SRC_NONE);
6140 				continue;
6141 			case VDEV_PROP_GUID:
6142 				vdev_prop_add_list(outnvl, propname, NULL,
6143 				    vd->vdev_guid, ZPROP_SRC_NONE);
6144 				continue;
6145 			case VDEV_PROP_ASIZE:
6146 				vdev_prop_add_list(outnvl, propname, NULL,
6147 				    vd->vdev_asize, ZPROP_SRC_NONE);
6148 				continue;
6149 			case VDEV_PROP_PSIZE:
6150 				vdev_prop_add_list(outnvl, propname, NULL,
6151 				    vd->vdev_psize, ZPROP_SRC_NONE);
6152 				continue;
6153 			case VDEV_PROP_ASHIFT:
6154 				vdev_prop_add_list(outnvl, propname, NULL,
6155 				    vd->vdev_ashift, ZPROP_SRC_NONE);
6156 				continue;
6157 			case VDEV_PROP_SIZE:
6158 				vdev_prop_add_list(outnvl, propname, NULL,
6159 				    vd->vdev_stat.vs_dspace, ZPROP_SRC_NONE);
6160 				continue;
6161 			case VDEV_PROP_FREE:
6162 				vdev_prop_add_list(outnvl, propname, NULL,
6163 				    vd->vdev_stat.vs_dspace -
6164 				    vd->vdev_stat.vs_alloc, ZPROP_SRC_NONE);
6165 				continue;
6166 			case VDEV_PROP_ALLOCATED:
6167 				vdev_prop_add_list(outnvl, propname, NULL,
6168 				    vd->vdev_stat.vs_alloc, ZPROP_SRC_NONE);
6169 				continue;
6170 			case VDEV_PROP_EXPANDSZ:
6171 				vdev_prop_add_list(outnvl, propname, NULL,
6172 				    vd->vdev_stat.vs_esize, ZPROP_SRC_NONE);
6173 				continue;
6174 			case VDEV_PROP_FRAGMENTATION:
6175 				vdev_prop_add_list(outnvl, propname, NULL,
6176 				    vd->vdev_stat.vs_fragmentation,
6177 				    ZPROP_SRC_NONE);
6178 				continue;
6179 			case VDEV_PROP_PARITY:
6180 				vdev_prop_add_list(outnvl, propname, NULL,
6181 				    vdev_get_nparity(vd), ZPROP_SRC_NONE);
6182 				continue;
6183 			case VDEV_PROP_PATH:
6184 				if (vd->vdev_path == NULL)
6185 					continue;
6186 				vdev_prop_add_list(outnvl, propname,
6187 				    vd->vdev_path, 0, ZPROP_SRC_NONE);
6188 				continue;
6189 			case VDEV_PROP_DEVID:
6190 				if (vd->vdev_devid == NULL)
6191 					continue;
6192 				vdev_prop_add_list(outnvl, propname,
6193 				    vd->vdev_devid, 0, ZPROP_SRC_NONE);
6194 				continue;
6195 			case VDEV_PROP_PHYS_PATH:
6196 				if (vd->vdev_physpath == NULL)
6197 					continue;
6198 				vdev_prop_add_list(outnvl, propname,
6199 				    vd->vdev_physpath, 0, ZPROP_SRC_NONE);
6200 				continue;
6201 			case VDEV_PROP_ENC_PATH:
6202 				if (vd->vdev_enc_sysfs_path == NULL)
6203 					continue;
6204 				vdev_prop_add_list(outnvl, propname,
6205 				    vd->vdev_enc_sysfs_path, 0, ZPROP_SRC_NONE);
6206 				continue;
6207 			case VDEV_PROP_FRU:
6208 				if (vd->vdev_fru == NULL)
6209 					continue;
6210 				vdev_prop_add_list(outnvl, propname,
6211 				    vd->vdev_fru, 0, ZPROP_SRC_NONE);
6212 				continue;
6213 			case VDEV_PROP_PARENT:
6214 				if (vd->vdev_parent != NULL) {
6215 					strval = vdev_name(vd->vdev_parent,
6216 					    namebuf, sizeof (namebuf));
6217 					vdev_prop_add_list(outnvl, propname,
6218 					    strval, 0, ZPROP_SRC_NONE);
6219 				}
6220 				continue;
6221 			case VDEV_PROP_CHILDREN:
6222 				if (vd->vdev_children > 0)
6223 					strval = kmem_zalloc(ZAP_MAXVALUELEN,
6224 					    KM_SLEEP);
6225 				for (uint64_t i = 0; i < vd->vdev_children;
6226 				    i++) {
6227 					const char *vname;
6228 
6229 					vname = vdev_name(vd->vdev_child[i],
6230 					    namebuf, sizeof (namebuf));
6231 					if (vname == NULL)
6232 						vname = "(unknown)";
6233 					if (strlen(strval) > 0)
6234 						strlcat(strval, ",",
6235 						    ZAP_MAXVALUELEN);
6236 					strlcat(strval, vname, ZAP_MAXVALUELEN);
6237 				}
6238 				if (strval != NULL) {
6239 					vdev_prop_add_list(outnvl, propname,
6240 					    strval, 0, ZPROP_SRC_NONE);
6241 					kmem_free(strval, ZAP_MAXVALUELEN);
6242 				}
6243 				continue;
6244 			case VDEV_PROP_NUMCHILDREN:
6245 				vdev_prop_add_list(outnvl, propname, NULL,
6246 				    vd->vdev_children, ZPROP_SRC_NONE);
6247 				continue;
6248 			case VDEV_PROP_READ_ERRORS:
6249 				vdev_prop_add_list(outnvl, propname, NULL,
6250 				    vd->vdev_stat.vs_read_errors,
6251 				    ZPROP_SRC_NONE);
6252 				continue;
6253 			case VDEV_PROP_WRITE_ERRORS:
6254 				vdev_prop_add_list(outnvl, propname, NULL,
6255 				    vd->vdev_stat.vs_write_errors,
6256 				    ZPROP_SRC_NONE);
6257 				continue;
6258 			case VDEV_PROP_CHECKSUM_ERRORS:
6259 				vdev_prop_add_list(outnvl, propname, NULL,
6260 				    vd->vdev_stat.vs_checksum_errors,
6261 				    ZPROP_SRC_NONE);
6262 				continue;
6263 			case VDEV_PROP_INITIALIZE_ERRORS:
6264 				vdev_prop_add_list(outnvl, propname, NULL,
6265 				    vd->vdev_stat.vs_initialize_errors,
6266 				    ZPROP_SRC_NONE);
6267 				continue;
6268 			case VDEV_PROP_TRIM_ERRORS:
6269 				vdev_prop_add_list(outnvl, propname, NULL,
6270 				    vd->vdev_stat.vs_trim_errors,
6271 				    ZPROP_SRC_NONE);
6272 				continue;
6273 			case VDEV_PROP_SLOW_IOS:
6274 				vdev_prop_add_list(outnvl, propname, NULL,
6275 				    vd->vdev_stat.vs_slow_ios,
6276 				    ZPROP_SRC_NONE);
6277 				continue;
6278 			case VDEV_PROP_OPS_NULL:
6279 				vdev_prop_add_list(outnvl, propname, NULL,
6280 				    vd->vdev_stat.vs_ops[ZIO_TYPE_NULL],
6281 				    ZPROP_SRC_NONE);
6282 				continue;
6283 			case VDEV_PROP_OPS_READ:
6284 				vdev_prop_add_list(outnvl, propname, NULL,
6285 				    vd->vdev_stat.vs_ops[ZIO_TYPE_READ],
6286 				    ZPROP_SRC_NONE);
6287 				continue;
6288 			case VDEV_PROP_OPS_WRITE:
6289 				vdev_prop_add_list(outnvl, propname, NULL,
6290 				    vd->vdev_stat.vs_ops[ZIO_TYPE_WRITE],
6291 				    ZPROP_SRC_NONE);
6292 				continue;
6293 			case VDEV_PROP_OPS_FREE:
6294 				vdev_prop_add_list(outnvl, propname, NULL,
6295 				    vd->vdev_stat.vs_ops[ZIO_TYPE_FREE],
6296 				    ZPROP_SRC_NONE);
6297 				continue;
6298 			case VDEV_PROP_OPS_CLAIM:
6299 				vdev_prop_add_list(outnvl, propname, NULL,
6300 				    vd->vdev_stat.vs_ops[ZIO_TYPE_CLAIM],
6301 				    ZPROP_SRC_NONE);
6302 				continue;
6303 			case VDEV_PROP_OPS_TRIM:
6304 				/*
6305 				 * TRIM ops and bytes are reported to user
6306 				 * space as ZIO_TYPE_FLUSH.  This is done to
6307 				 * preserve the vdev_stat_t structure layout
6308 				 * for user space.
6309 				 */
6310 				vdev_prop_add_list(outnvl, propname, NULL,
6311 				    vd->vdev_stat.vs_ops[ZIO_TYPE_FLUSH],
6312 				    ZPROP_SRC_NONE);
6313 				continue;
6314 			case VDEV_PROP_BYTES_NULL:
6315 				vdev_prop_add_list(outnvl, propname, NULL,
6316 				    vd->vdev_stat.vs_bytes[ZIO_TYPE_NULL],
6317 				    ZPROP_SRC_NONE);
6318 				continue;
6319 			case VDEV_PROP_BYTES_READ:
6320 				vdev_prop_add_list(outnvl, propname, NULL,
6321 				    vd->vdev_stat.vs_bytes[ZIO_TYPE_READ],
6322 				    ZPROP_SRC_NONE);
6323 				continue;
6324 			case VDEV_PROP_BYTES_WRITE:
6325 				vdev_prop_add_list(outnvl, propname, NULL,
6326 				    vd->vdev_stat.vs_bytes[ZIO_TYPE_WRITE],
6327 				    ZPROP_SRC_NONE);
6328 				continue;
6329 			case VDEV_PROP_BYTES_FREE:
6330 				vdev_prop_add_list(outnvl, propname, NULL,
6331 				    vd->vdev_stat.vs_bytes[ZIO_TYPE_FREE],
6332 				    ZPROP_SRC_NONE);
6333 				continue;
6334 			case VDEV_PROP_BYTES_CLAIM:
6335 				vdev_prop_add_list(outnvl, propname, NULL,
6336 				    vd->vdev_stat.vs_bytes[ZIO_TYPE_CLAIM],
6337 				    ZPROP_SRC_NONE);
6338 				continue;
6339 			case VDEV_PROP_BYTES_TRIM:
6340 				/*
6341 				 * TRIM ops and bytes are reported to user
6342 				 * space as ZIO_TYPE_FLUSH.  This is done to
6343 				 * preserve the vdev_stat_t structure layout
6344 				 * for user space.
6345 				 */
6346 				vdev_prop_add_list(outnvl, propname, NULL,
6347 				    vd->vdev_stat.vs_bytes[ZIO_TYPE_FLUSH],
6348 				    ZPROP_SRC_NONE);
6349 				continue;
6350 			case VDEV_PROP_REMOVING:
6351 				vdev_prop_add_list(outnvl, propname, NULL,
6352 				    vd->vdev_removing, ZPROP_SRC_NONE);
6353 				continue;
6354 			case VDEV_PROP_RAIDZ_EXPANDING:
6355 				/* Only expose this for raidz */
6356 				if (vd->vdev_ops == &vdev_raidz_ops) {
6357 					vdev_prop_add_list(outnvl, propname,
6358 					    NULL, vd->vdev_rz_expanding,
6359 					    ZPROP_SRC_NONE);
6360 				}
6361 				continue;
6362 			case VDEV_PROP_TRIM_SUPPORT:
6363 				/* only valid for leaf vdevs */
6364 				if (vd->vdev_ops->vdev_op_leaf) {
6365 					vdev_prop_add_list(outnvl, propname,
6366 					    NULL, vd->vdev_has_trim,
6367 					    ZPROP_SRC_NONE);
6368 				}
6369 				continue;
6370 			/* Numeric Properites */
6371 			case VDEV_PROP_ALLOCATING:
6372 				/* Leaf vdevs cannot have this property */
6373 				if (vd->vdev_mg == NULL &&
6374 				    vd->vdev_top != NULL) {
6375 					src = ZPROP_SRC_NONE;
6376 					intval = ZPROP_BOOLEAN_NA;
6377 				} else {
6378 					err = vdev_prop_get_int(vd, prop,
6379 					    &intval);
6380 					if (err && err != ENOENT)
6381 						break;
6382 
6383 					if (intval ==
6384 					    vdev_prop_default_numeric(prop))
6385 						src = ZPROP_SRC_DEFAULT;
6386 					else
6387 						src = ZPROP_SRC_LOCAL;
6388 				}
6389 
6390 				vdev_prop_add_list(outnvl, propname, NULL,
6391 				    intval, src);
6392 				break;
6393 			case VDEV_PROP_FAILFAST:
6394 				src = ZPROP_SRC_LOCAL;
6395 				strval = NULL;
6396 
6397 				err = zap_lookup(mos, objid, nvpair_name(elem),
6398 				    sizeof (uint64_t), 1, &intval);
6399 				if (err == ENOENT) {
6400 					intval = vdev_prop_default_numeric(
6401 					    prop);
6402 					err = 0;
6403 				} else if (err) {
6404 					break;
6405 				}
6406 				if (intval == vdev_prop_default_numeric(prop))
6407 					src = ZPROP_SRC_DEFAULT;
6408 
6409 				vdev_prop_add_list(outnvl, propname, strval,
6410 				    intval, src);
6411 				break;
6412 			case VDEV_PROP_CHECKSUM_N:
6413 			case VDEV_PROP_CHECKSUM_T:
6414 			case VDEV_PROP_IO_N:
6415 			case VDEV_PROP_IO_T:
6416 			case VDEV_PROP_SLOW_IO_N:
6417 			case VDEV_PROP_SLOW_IO_T:
6418 				err = vdev_prop_get_int(vd, prop, &intval);
6419 				if (err && err != ENOENT)
6420 					break;
6421 
6422 				if (intval == vdev_prop_default_numeric(prop))
6423 					src = ZPROP_SRC_DEFAULT;
6424 				else
6425 					src = ZPROP_SRC_LOCAL;
6426 
6427 				vdev_prop_add_list(outnvl, propname, NULL,
6428 				    intval, src);
6429 				break;
6430 			/* Text Properties */
6431 			case VDEV_PROP_COMMENT:
6432 				/* Exists in the ZAP below */
6433 				/* FALLTHRU */
6434 			case VDEV_PROP_USERPROP:
6435 				/* User Properites */
6436 				src = ZPROP_SRC_LOCAL;
6437 
6438 				err = zap_length(mos, objid, nvpair_name(elem),
6439 				    &integer_size, &num_integers);
6440 				if (err)
6441 					break;
6442 
6443 				switch (integer_size) {
6444 				case 8:
6445 					/* User properties cannot be integers */
6446 					err = EINVAL;
6447 					break;
6448 				case 1:
6449 					/* string property */
6450 					strval = kmem_alloc(num_integers,
6451 					    KM_SLEEP);
6452 					err = zap_lookup(mos, objid,
6453 					    nvpair_name(elem), 1,
6454 					    num_integers, strval);
6455 					if (err) {
6456 						kmem_free(strval,
6457 						    num_integers);
6458 						break;
6459 					}
6460 					vdev_prop_add_list(outnvl, propname,
6461 					    strval, 0, src);
6462 					kmem_free(strval, num_integers);
6463 					break;
6464 				}
6465 				break;
6466 			default:
6467 				err = ENOENT;
6468 				break;
6469 			}
6470 			if (err)
6471 				break;
6472 		}
6473 	} else {
6474 		/*
6475 		 * Get all properties from the MOS vdev property object.
6476 		 */
6477 		zap_cursor_t zc;
6478 		zap_attribute_t *za = zap_attribute_alloc();
6479 		for (zap_cursor_init(&zc, mos, objid);
6480 		    (err = zap_cursor_retrieve(&zc, za)) == 0;
6481 		    zap_cursor_advance(&zc)) {
6482 			intval = 0;
6483 			strval = NULL;
6484 			zprop_source_t src = ZPROP_SRC_DEFAULT;
6485 			propname = za->za_name;
6486 
6487 			switch (za->za_integer_length) {
6488 			case 8:
6489 				/* We do not allow integer user properties */
6490 				/* This is likely an internal value */
6491 				break;
6492 			case 1:
6493 				/* string property */
6494 				strval = kmem_alloc(za->za_num_integers,
6495 				    KM_SLEEP);
6496 				err = zap_lookup(mos, objid, za->za_name, 1,
6497 				    za->za_num_integers, strval);
6498 				if (err) {
6499 					kmem_free(strval, za->za_num_integers);
6500 					break;
6501 				}
6502 				vdev_prop_add_list(outnvl, propname, strval, 0,
6503 				    src);
6504 				kmem_free(strval, za->za_num_integers);
6505 				break;
6506 
6507 			default:
6508 				break;
6509 			}
6510 		}
6511 		zap_cursor_fini(&zc);
6512 		zap_attribute_free(za);
6513 	}
6514 
6515 	mutex_exit(&spa->spa_props_lock);
6516 	if (err && err != ENOENT) {
6517 		return (err);
6518 	}
6519 
6520 	return (0);
6521 }
6522 
6523 EXPORT_SYMBOL(vdev_fault);
6524 EXPORT_SYMBOL(vdev_degrade);
6525 EXPORT_SYMBOL(vdev_online);
6526 EXPORT_SYMBOL(vdev_offline);
6527 EXPORT_SYMBOL(vdev_clear);
6528 
6529 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, default_ms_count, UINT, ZMOD_RW,
6530 	"Target number of metaslabs per top-level vdev");
6531 
6532 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, default_ms_shift, UINT, ZMOD_RW,
6533 	"Default lower limit for metaslab size");
6534 
6535 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, max_ms_shift, UINT, ZMOD_RW,
6536 	"Default upper limit for metaslab size");
6537 
6538 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, min_ms_count, UINT, ZMOD_RW,
6539 	"Minimum number of metaslabs per top-level vdev");
6540 
6541 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, ms_count_limit, UINT, ZMOD_RW,
6542 	"Practical upper limit of total metaslabs per top-level vdev");
6543 
6544 ZFS_MODULE_PARAM(zfs, zfs_, slow_io_events_per_second, UINT, ZMOD_RW,
6545 	"Rate limit slow IO (delay) events to this many per second");
6546 
6547 ZFS_MODULE_PARAM(zfs, zfs_, deadman_events_per_second, UINT, ZMOD_RW,
6548 	"Rate limit hung IO (deadman) events to this many per second");
6549 
6550 ZFS_MODULE_PARAM(zfs, zfs_, dio_write_verify_events_per_second, UINT, ZMOD_RW,
6551 	"Rate Direct I/O write verify events to this many per second");
6552 
6553 /* BEGIN CSTYLED */
6554 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, direct_write_verify, UINT, ZMOD_RW,
6555 	"Direct I/O writes will perform for checksum verification before "
6556 	"commiting write");
6557 
6558 ZFS_MODULE_PARAM(zfs, zfs_, checksum_events_per_second, UINT, ZMOD_RW,
6559 	"Rate limit checksum events to this many checksum errors per second "
6560 	"(do not set below ZED threshold).");
6561 /* END CSTYLED */
6562 
6563 ZFS_MODULE_PARAM(zfs, zfs_, scan_ignore_errors, INT, ZMOD_RW,
6564 	"Ignore errors during resilver/scrub");
6565 
6566 ZFS_MODULE_PARAM(zfs_vdev, vdev_, validate_skip, INT, ZMOD_RW,
6567 	"Bypass vdev_validate()");
6568 
6569 ZFS_MODULE_PARAM(zfs, zfs_, nocacheflush, INT, ZMOD_RW,
6570 	"Disable cache flushes");
6571 
6572 ZFS_MODULE_PARAM(zfs, zfs_, embedded_slog_min_ms, UINT, ZMOD_RW,
6573 	"Minimum number of metaslabs required to dedicate one for log blocks");
6574 
6575 /* BEGIN CSTYLED */
6576 ZFS_MODULE_PARAM_CALL(zfs_vdev, zfs_vdev_, min_auto_ashift,
6577 	param_set_min_auto_ashift, param_get_uint, ZMOD_RW,
6578 	"Minimum ashift used when creating new top-level vdevs");
6579 
6580 ZFS_MODULE_PARAM_CALL(zfs_vdev, zfs_vdev_, max_auto_ashift,
6581 	param_set_max_auto_ashift, param_get_uint, ZMOD_RW,
6582 	"Maximum ashift used when optimizing for logical -> physical sector "
6583 	"size on new top-level vdevs");
6584 /* END CSTYLED */
6585