189 vdev_label_read(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t offset,  in vdev_label_read()  argument
197 	zio_nowait(zio_read_phys(zio, vd,  in vdev_label_read()
198 	    vdev_label_offset(vd->vdev_psize, l, offset),  in vdev_label_read()
204 vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t offset,  in vdev_label_write()  argument
212 	zio_nowait(zio_write_phys(zio, vd,  in vdev_label_write()
213 	    vdev_label_offset(vd->vdev_psize, l, offset),  in vdev_label_write()
222 vdev_config_generate_stats(vdev_t *vd, nvlist_t *nv)  in vdev_config_generate_stats()  argument
231 	vdev_get_stats_ex(vd, vs, vsx);  in vdev_config_generate_stats()
404 root_vdev_actions_getprogress(vdev_t *vd, nvlist_t *nvl)  in root_vdev_actions_getprogress()  argument
406 	spa_t *spa = vd->vdev_spa;  in root_vdev_actions_getprogress()
408 	if (vd != spa->spa_root_vdev)  in root_vdev_actions_getprogress()
442 top_vdev_actions_getprogress(vdev_t *vd, nvlist_t *nvl)  in top_vdev_actions_getprogress()  argument
444 	if (vd == vd->vdev_top) {  in top_vdev_actions_getprogress()
446 		if (vdev_rebuild_get_stats(vd, &vrs) == 0) {  in top_vdev_actions_getprogress()
458 vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats,  in vdev_config_generate()  argument
462 	vdev_indirect_config_t *vic = &vd->vdev_indirect_config;  in vdev_config_generate()
466 	fnvlist_add_string(nv, ZPOOL_CONFIG_TYPE, vd->vdev_ops->vdev_op_type);  in vdev_config_generate()
468 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_ID, vd->vdev_id);  in vdev_config_generate()
469 	fnvlist_add_uint64(nv, ZPOOL_CONFIG_GUID, vd->vdev_guid);  in vdev_config_generate()
471 	if (vd->vdev_path != NULL)  in vdev_config_generate()
472 		fnvlist_add_string(nv, ZPOOL_CONFIG_PATH, vd->vdev_path);  in vdev_config_generate()
474 	if (vd->vdev_devid != NULL)  in vdev_config_generate()
475 		fnvlist_add_string(nv, ZPOOL_CONFIG_DEVID, vd->vdev_devid);  in vdev_config_generate()
477 	if (vd->vdev_physpath != NULL)  in vdev_config_generate()
479 		    vd->vdev_physpath);  in vdev_config_generate()
481 	if (vd->vdev_enc_sysfs_path != NULL)  in vdev_config_generate()
483 		    vd->vdev_enc_sysfs_path);  in vdev_config_generate()
485 	if (vd->vdev_fru != NULL)  in vdev_config_generate()
486 		fnvlist_add_string(nv, ZPOOL_CONFIG_FRU, vd->vdev_fru);  in vdev_config_generate()
488 	if (vd->vdev_ops->vdev_op_config_generate != NULL)  in vdev_config_generate()
489 		vd->vdev_ops->vdev_op_config_generate(vd, nv);  in vdev_config_generate()
491 	if (vd->vdev_wholedisk != -1ULL) {  in vdev_config_generate()
493 		    vd->vdev_wholedisk);  in vdev_config_generate()
496 	if (vd->vdev_not_present && !(flags & VDEV_CONFIG_MISSING))  in vdev_config_generate()
499 	if (vd->vdev_isspare)  in vdev_config_generate()
503 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);  in vdev_config_generate()
506 	    vd == vd->vdev_top) {  in vdev_config_generate()
508 		    vd->vdev_ms_array);  in vdev_config_generate()
510 		    vd->vdev_ms_shift);  in vdev_config_generate()
511 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);  in vdev_config_generate()
513 		    vd->vdev_asize);  in vdev_config_generate()
514 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_LOG, vd->vdev_islog);  in vdev_config_generate()
515 		if (vd->vdev_noalloc) {  in vdev_config_generate()
517 			    vd->vdev_noalloc);  in vdev_config_generate()
524 		if (vd->vdev_removing && !vd->vdev_islog) {  in vdev_config_generate()
526 			    vd->vdev_removing);  in vdev_config_generate()
530 		if (getstats && vd->vdev_alloc_bias != VDEV_BIAS_NONE) {  in vdev_config_generate()
533 			switch (vd->vdev_alloc_bias) {  in vdev_config_generate()
544 				ASSERT3U(vd->vdev_alloc_bias, ==,  in vdev_config_generate()
552 	if (vd->vdev_dtl_sm != NULL) {  in vdev_config_generate()
554 		    space_map_object(vd->vdev_dtl_sm));  in vdev_config_generate()
572 	if (vd->vdev_crtxg)  in vdev_config_generate()
573 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_CREATE_TXG, vd->vdev_crtxg);  in vdev_config_generate()
575 	if (vd->vdev_expansion_time)  in vdev_config_generate()
577 		    vd->vdev_expansion_time);  in vdev_config_generate()
580 		if (vd->vdev_leaf_zap != 0) {  in vdev_config_generate()
581 			ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_config_generate()
583 			    vd->vdev_leaf_zap);  in vdev_config_generate()
586 		if (vd->vdev_top_zap != 0) {  in vdev_config_generate()
587 			ASSERT(vd == vd->vdev_top);  in vdev_config_generate()
589 			    vd->vdev_top_zap);  in vdev_config_generate()
592 		if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap != 0 &&  in vdev_config_generate()
593 		    spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) {  in vdev_config_generate()
595 			    vd->vdev_root_zap);  in vdev_config_generate()
598 		if (vd->vdev_resilver_deferred) {  in vdev_config_generate()
599 			ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_config_generate()
606 		vdev_config_generate_stats(vd, nv);  in vdev_config_generate()
608 		root_vdev_actions_getprogress(vd, nv);  in vdev_config_generate()
609 		top_vdev_actions_getprogress(vd, nv);  in vdev_config_generate()
616 		rw_enter(&vd->vdev_indirect_rwlock, RW_READER);  in vdev_config_generate()
617 		if (vd->vdev_indirect_mapping != NULL) {  in vdev_config_generate()
618 			ASSERT(vd->vdev_indirect_births != NULL);  in vdev_config_generate()
620 			    vd->vdev_indirect_mapping;  in vdev_config_generate()
624 		rw_exit(&vd->vdev_indirect_rwlock);  in vdev_config_generate()
625 		if (vd->vdev_mg != NULL &&  in vdev_config_generate()
626 		    vd->vdev_mg->mg_fragmentation != ZFS_FRAG_INVALID) {  in vdev_config_generate()
637 			uint64_t to_alloc = vd->vdev_stat.vs_alloc;  in vdev_config_generate()
652 					    vd->vdev_mg->mg_histogram[i] <<  in vdev_config_generate()
656 					    vd->vdev_mg->mg_histogram[i];  in vdev_config_generate()
673 	if (!vd->vdev_ops->vdev_op_leaf) {  in vdev_config_generate()
677 		ASSERT(!vd->vdev_ishole);  in vdev_config_generate()
679 		child = kmem_alloc(vd->vdev_children * sizeof (nvlist_t *),  in vdev_config_generate()
682 		for (c = 0; c < vd->vdev_children; c++) {  in vdev_config_generate()
683 			child[c] = vdev_config_generate(spa, vd->vdev_child[c],  in vdev_config_generate()
688 		    (const nvlist_t * const *)child, vd->vdev_children);  in vdev_config_generate()
690 		for (c = 0; c < vd->vdev_children; c++)  in vdev_config_generate()
693 		kmem_free(child, vd->vdev_children * sizeof (nvlist_t *));  in vdev_config_generate()
698 		if (vd->vdev_offline && !vd->vdev_tmpoffline)  in vdev_config_generate()
700 		if (vd->vdev_resilver_txg != 0)  in vdev_config_generate()
702 			    vd->vdev_resilver_txg);  in vdev_config_generate()
703 		if (vd->vdev_rebuild_txg != 0)  in vdev_config_generate()
705 			    vd->vdev_rebuild_txg);  in vdev_config_generate()
706 		if (vd->vdev_faulted)  in vdev_config_generate()
708 		if (vd->vdev_degraded)  in vdev_config_generate()
710 		if (vd->vdev_removed)  in vdev_config_generate()
712 		if (vd->vdev_unspare)  in vdev_config_generate()
714 		if (vd->vdev_ishole)  in vdev_config_generate()
718 		switch (vd->vdev_stat.vs_aux) {  in vdev_config_generate()
728 		if (aux != NULL && !vd->vdev_tmpoffline) {  in vdev_config_generate()
738 		if (vd->vdev_splitting && vd->vdev_orig_guid != 0LL) {  in vdev_config_generate()
740 			    vd->vdev_orig_guid);  in vdev_config_generate()
790 vdev_label_read_config(vdev_t *vd, uint64_t txg)  in vdev_label_read_config()  argument
792 	spa_t *spa = vd->vdev_spa;  in vdev_label_read_config()
803 	ASSERT(vd->vdev_validate_thread == curthread ||  in vdev_label_read_config()
806 	if (!vdev_readable(vd))  in vdev_label_read_config()
814 	if (vd->vdev_ops == &vdev_draid_spare_ops)  in vdev_label_read_config()
815 		return (vdev_draid_read_config_spare(vd));  in vdev_label_read_config()
826 		vdev_label_read(zio[l], vd, l, vp_abd[l],  in vdev_label_read_config()
872 		vdev_dbgmsg(vd, "label discarded as txg is too large "  in vdev_label_read_config()
889 vdev_inuse(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason,  in vdev_inuse()  argument
892 	spa_t *spa = vd->vdev_spa;  in vdev_inuse()
905 	if ((label = vdev_label_read_config(vd, -1ULL)) == NULL)  in vdev_inuse()
1017 vdev_aux_label_generate(vdev_t *vd, boolean_t reason_spare)  in vdev_aux_label_generate()  argument
1028 	    spa_version(vd->vdev_spa));  in vdev_aux_label_generate()
1031 	fnvlist_add_uint64(label, ZPOOL_CONFIG_GUID, vd->vdev_guid);  in vdev_aux_label_generate()
1041 		fnvlist_add_uint64(label, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);  in vdev_aux_label_generate()
1046 	if (vd->vdev_path != NULL)  in vdev_aux_label_generate()
1047 		fnvlist_add_string(label, ZPOOL_CONFIG_PATH, vd->vdev_path);  in vdev_aux_label_generate()
1048 	if (vd->vdev_devid != NULL)  in vdev_aux_label_generate()
1049 		fnvlist_add_string(label, ZPOOL_CONFIG_DEVID, vd->vdev_devid);  in vdev_aux_label_generate()
1050 	if (vd->vdev_physpath != NULL) {  in vdev_aux_label_generate()
1052 		    vd->vdev_physpath);  in vdev_aux_label_generate()
1066 vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)  in vdev_label_init()  argument
1068 	spa_t *spa = vd->vdev_spa;  in vdev_label_init()
1082 	    VDEV_LABEL_REMOVE && vd->vdev_isspare));  in vdev_label_init()
1084 	    VDEV_LABEL_REMOVE && vd->vdev_isl2cache));  in vdev_label_init()
1088 	for (int c = 0; c < vd->vdev_children; c++)  in vdev_label_init()
1089 		if ((error = vdev_label_init(vd->vdev_child[c],  in vdev_label_init()
1094 	vd->vdev_crtxg = crtxg;  in vdev_label_init()
1096 	if (!vd->vdev_ops->vdev_op_leaf || !spa_writeable(spa))  in vdev_label_init()
1102 	if (vdev_is_dead(vd))  in vdev_label_init()
1109 	    vdev_inuse(vd, crtxg, reason, &spare_guid, &l2cache_guid))  in vdev_label_init()
1120 		uint64_t guid_delta = spare_guid - vd->vdev_guid;  in vdev_label_init()
1122 		vd->vdev_guid += guid_delta;  in vdev_label_init()
1124 		for (vdev_t *pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)  in vdev_label_init()
1140 		uint64_t guid_delta = l2cache_guid - vd->vdev_guid;  in vdev_label_init()
1142 		vd->vdev_guid += guid_delta;  in vdev_label_init()
1144 		for (vdev_t *pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)  in vdev_label_init()
1171 		label = vdev_aux_label_generate(vd, reason_spare);  in vdev_label_init()
1185 		label = spa_config_generate(spa, vd, txg, B_FALSE);  in vdev_label_init()
1229 		vdev_label_write(zio, vd, l, vp_abd,  in vdev_label_init()
1238 		vdev_label_write(zio, vd, l, bootenv,  in vdev_label_init()
1242 		vdev_label_write(zio, vd, l, ub_abd,  in vdev_label_init()
1265 	if (error == 0 && !vd->vdev_isspare &&  in vdev_label_init()
1267 	    spa_spare_exists(vd->vdev_guid, NULL, NULL)))  in vdev_label_init()
1268 		spa_spare_add(vd);  in vdev_label_init()
1270 	if (error == 0 && !vd->vdev_isl2cache &&  in vdev_label_init()
1272 	    spa_l2cache_exists(vd->vdev_guid, NULL)))  in vdev_label_init()
1273 		spa_l2cache_add(vd);  in vdev_label_init()
1306 vdev_label_read_bootenv_impl(zio_t *zio, vdev_t *vd, int flags)  in vdev_label_read_bootenv_impl()  argument
1308 	for (int c = 0; c < vd->vdev_children; c++)  in vdev_label_read_bootenv_impl()
1309 		vdev_label_read_bootenv_impl(zio, vd->vdev_child[c], flags);  in vdev_label_read_bootenv_impl()
1317 	if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd)) {  in vdev_label_read_bootenv_impl()
1319 			vdev_label_read(zio, vd, l,  in vdev_label_read_bootenv_impl()
1394 vdev_label_write_bootenv(vdev_t *vd, nvlist_t *env)  in vdev_label_write_bootenv()  argument
1397 	spa_t *spa = vd->vdev_spa;  in vdev_label_write_bootenv()
1416 	for (int c = 0; c < vd->vdev_children; c++) {  in vdev_label_write_bootenv()
1419 		child_err = vdev_label_write_bootenv(vd->vdev_child[c], env);  in vdev_label_write_bootenv()
1428 	if (!vd->vdev_ops->vdev_op_leaf || vdev_is_dead(vd) ||  in vdev_label_write_bootenv()
1429 	    !vdev_writeable(vd)) {  in vdev_label_write_bootenv()
1470 		vdev_label_write(zio, vd, l, abd,  in vdev_label_write_bootenv()
1545 	vdev_t *vd = zio->io_vd;  in vdev_uberblock_load_done()  local
1551 	ASSERT3U(zio->io_size, ==, VDEV_UBERBLOCK_SIZE(vd));  in vdev_uberblock_load_done()
1567 			cbp->ubl_vd = vd;  in vdev_uberblock_load_done()
1576 vdev_uberblock_load_impl(zio_t *zio, vdev_t *vd, int flags,  in vdev_uberblock_load_impl()  argument
1579 	for (int c = 0; c < vd->vdev_children; c++)  in vdev_uberblock_load_impl()
1580 		vdev_uberblock_load_impl(zio, vd->vdev_child[c], flags, cbp);  in vdev_uberblock_load_impl()
1582 	if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd) &&  in vdev_uberblock_load_impl()
1583 	    vd->vdev_ops != &vdev_draid_spare_ops) {  in vdev_uberblock_load_impl()
1585 			for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {  in vdev_uberblock_load_impl()
1586 				vdev_label_read(zio, vd, l,  in vdev_uberblock_load_impl()
1587 				    abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd),  in vdev_uberblock_load_impl()
1588 				    B_TRUE), VDEV_UBERBLOCK_OFFSET(vd, n),  in vdev_uberblock_load_impl()
1589 				    VDEV_UBERBLOCK_SIZE(vd),  in vdev_uberblock_load_impl()
1676 vdev_copy_uberblocks(vdev_t *vd)  in vdev_copy_uberblocks()  argument
1684 	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_READER) ==  in vdev_copy_uberblocks()
1686 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_copy_uberblocks()
1692 	if (vd->vdev_ops == &vdev_draid_spare_ops)  in vdev_copy_uberblocks()
1695 	spa_config_enter(vd->vdev_spa, locks, FTAG, RW_READER);  in vdev_copy_uberblocks()
1697 	ub_abd = abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);  in vdev_copy_uberblocks()
1699 	write_zio = zio_root(vd->vdev_spa, NULL, NULL, flags);  in vdev_copy_uberblocks()
1700 	for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {  in vdev_copy_uberblocks()
1704 		zio = zio_root(vd->vdev_spa, NULL, NULL, flags);  in vdev_copy_uberblocks()
1705 		vdev_label_read(zio, vd, src_label, ub_abd,  in vdev_copy_uberblocks()
1706 		    VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd),  in vdev_copy_uberblocks()
1710 			abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));  in vdev_copy_uberblocks()
1713 			vdev_label_write(write_zio, vd, l, ub_abd,  in vdev_copy_uberblocks()
1714 			    VDEV_UBERBLOCK_OFFSET(vd, n),  in vdev_copy_uberblocks()
1715 			    VDEV_UBERBLOCK_SIZE(vd), NULL, NULL,  in vdev_copy_uberblocks()
1720 	spa_config_exit(vd->vdev_spa, locks, FTAG);  in vdev_copy_uberblocks()
1743     uberblock_t *ub, vdev_t *vd, int flags)  in vdev_uberblock_sync()  argument
1745 	for (uint64_t c = 0; c < vd->vdev_children; c++) {  in vdev_uberblock_sync()
1747 		    ub, vd->vdev_child[c], flags);  in vdev_uberblock_sync()
1750 	if (!vd->vdev_ops->vdev_op_leaf)  in vdev_uberblock_sync()
1753 	if (!vdev_writeable(vd))  in vdev_uberblock_sync()
1762 	if (vd->vdev_ops == &vdev_draid_spare_ops)  in vdev_uberblock_sync()
1766 	if (vd->vdev_state == VDEV_STATE_HEALTHY &&  in vdev_uberblock_sync()
1767 	    vd->vdev_copy_uberblocks == B_TRUE) {  in vdev_uberblock_sync()
1768 		vdev_copy_uberblocks(vd);  in vdev_uberblock_sync()
1769 		vd->vdev_copy_uberblocks = B_FALSE;  in vdev_uberblock_sync()
1786 	int m = spa_multihost(vd->vdev_spa) ? MMP_BLOCKS_PER_LABEL : 0;  in vdev_uberblock_sync()
1788 	    (VDEV_UBERBLOCK_COUNT(vd) - m);  in vdev_uberblock_sync()
1791 	abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);  in vdev_uberblock_sync()
1794 	    VDEV_UBERBLOCK_SIZE(vd) - sizeof (uberblock_t));  in vdev_uberblock_sync()
1797 		vdev_label_write(zio, vd, l, ub_abd,  in vdev_uberblock_sync()
1798 		    VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd),  in vdev_uberblock_sync()
1901     vdev_t *vd, int l, uint64_t txg, int flags)  in vdev_label_sync()  argument
1908 	vdev_t *pvd = vd->vdev_parent;  in vdev_label_sync()
1911 	for (int c = 0; c < vd->vdev_children; c++) {  in vdev_label_sync()
1913 		    vd->vdev_child[c], l, txg, flags);  in vdev_label_sync()
1916 	if (!vd->vdev_ops->vdev_op_leaf)  in vdev_label_sync()
1919 	if (!vdev_writeable(vd))  in vdev_label_sync()
1927 	if (vd->vdev_ops == &vdev_draid_spare_ops)  in vdev_label_sync()
1936 	if ((vd->vdev_isspare && !spare_in_use) || vd->vdev_isl2cache) {  in vdev_label_sync()
1937 		label = vdev_aux_label_generate(vd, vd->vdev_isspare);  in vdev_label_sync()
1939 		label = spa_config_generate(vd->vdev_spa, vd, txg, B_FALSE);  in vdev_label_sync()
1951 			vdev_label_write(zio, vd, l, vp_abd,  in vdev_label_sync()
1967 	vdev_t *vd;  in vdev_label_sync_list()  local
1976 	for (vd = list_head(dl); vd != NULL; vd = list_next(dl, vd)) {  in vdev_label_sync_list()
1979 		ASSERT(!vd->vdev_ishole);  in vdev_label_sync_list()
1983 		    (vd->vdev_islog || vd->vdev_aux != NULL) ?  in vdev_label_sync_list()
1986 		vdev_label_sync(vio, good_writes, vd, l, txg, flags);  in vdev_label_sync_list()
2015 	for (vd = list_head(dl); vd != NULL; vd = list_next(dl, vd))  in vdev_label_sync_list()
2016 		zio_flush(zio, vd);  in vdev_label_sync_list()
2097 	for (vdev_t *vd =  in vdev_config_sync()  local
2098 	    txg_list_head(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)); vd != NULL;  in vdev_config_sync()
2099 	    vd = txg_list_next(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg)))  in vdev_config_sync()
2100 		zio_flush(zio, vd);  in vdev_config_sync()