1 // SPDX-License-Identifier: CDDL-1.0
10  * or https://opensource.org/licenses/CDDL-1.0.
91 #define	BP_SPANB(indblkshift, level) \  argument
92 	(((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
95  * The following actions directly effect the spa's sync-to-convergence logic.
113  * lot of block's size will change and thus we have to re-allocate (not
134 #define	IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE)
181 	zio_stats_t *zs = ksp->ks_data;  in zio_kstats_update()
185 	zs->ziostat_total_allocations.value.ui64 =  in zio_kstats_update()
187 	zs->ziostat_alloc_class_fallbacks.value.ui64 =  in zio_kstats_update()
189 	zs->ziostat_gang_writes.value.ui64 =  in zio_kstats_update()
191 	zs->ziostat_gang_multilevel.value.ui64 =  in zio_kstats_update()
214 		zio_ksp->ks_data = &zio_stats;  in zio_init()
215 		zio_ksp->ks_update = zio_kstats_update;  in zio_init()
225 		 * Create cache for each half-power of 2 size, starting from  in zio_init()
232 			p2 &= p2 - 1;  in zio_init()
240 		 * kernel when modifying a non-watched buffer that shares the  in zio_init()
250 			align = 1 << (highbit64(size ^ (size - 1)) - 1);  in zio_init()
282 	while (--c != 0) {  in zio_init()
284 		if (zio_buf_cache[c - 1] == NULL)  in zio_init()
285 			zio_buf_cache[c - 1] = zio_buf_cache[c];  in zio_init()
288 		if (zio_data_buf_cache[c - 1] == NULL)  in zio_init()
289 			zio_data_buf_cache[c - 1] = zio_data_buf_cache[c];  in zio_init()
410 			    p, size, (canary - p) * sizeof (ulong_t),  in zio_buf_check_canary()
421  * excess / transient data in-core during a crashdump.
426 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;  in zio_buf_alloc()
449 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;  in zio_data_buf_alloc()
463 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;  in zio_buf_free()
479 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;  in zio_data_buf_free()
507 	zt->zt_orig_abd = zio->io_abd;  in zio_push_transform()
508 	zt->zt_orig_size = zio->io_size;  in zio_push_transform()
509 	zt->zt_bufsize = bufsize;  in zio_push_transform()
510 	zt->zt_transform = transform;  in zio_push_transform()
512 	zt->zt_next = zio->io_transform_stack;  in zio_push_transform()
513 	zio->io_transform_stack = zt;  in zio_push_transform()
515 	zio->io_abd = data;  in zio_push_transform()
516 	zio->io_size = size;  in zio_push_transform()
524 	while ((zt = zio->io_transform_stack) != NULL) {  in zio_pop_transforms()
525 		if (zt->zt_transform != NULL)  in zio_pop_transforms()
526 			zt->zt_transform(zio,  in zio_pop_transforms()
527 			    zt->zt_orig_abd, zt->zt_orig_size);  in zio_pop_transforms()
529 		if (zt->zt_bufsize != 0)  in zio_pop_transforms()
530 			abd_free(zio->io_abd);  in zio_pop_transforms()
532 		zio->io_abd = zt->zt_orig_abd;  in zio_pop_transforms()
533 		zio->io_size = zt->zt_orig_size;  in zio_pop_transforms()
534 		zio->io_transform_stack = zt->zt_next;  in zio_pop_transforms()
548 	ASSERT(zio->io_size > size);  in zio_subblock()
550 	if (zio->io_type == ZIO_TYPE_READ)  in zio_subblock()
551 		abd_copy(data, zio->io_abd, size);  in zio_subblock()
557 	if (zio->io_error == 0) {  in zio_decompress()
558 		int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),  in zio_decompress()
559 		    zio->io_abd, data, zio->io_size, size,  in zio_decompress()
560 		    &zio->io_prop.zp_complevel);  in zio_decompress()
566 			zio->io_error = SET_ERROR(EIO);  in zio_decompress()
575 	blkptr_t *bp = zio->io_bp;  in zio_decrypt()
576 	spa_t *spa = zio->io_spa;  in zio_decrypt()
577 	uint64_t dsobj = zio->io_bookmark.zb_objset;  in zio_decrypt()
579 	dmu_object_type_t ot = BP_GET_TYPE(bp);  in zio_decrypt()  local
588 	if (zio->io_error != 0)  in zio_decrypt()
609 			    zio->io_abd, abd, zio->io_size, lsize,  in zio_decrypt()
610 			    &zio->io_prop.zp_complevel);  in zio_decrypt()
621 			    zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac);  in zio_decrypt()
623 		abd_copy(data, zio->io_abd, size);  in zio_decrypt()
625 		if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) {  in zio_decrypt()
627 			    &zio->io_bookmark, ot, ECKSUM);  in zio_decrypt()
642 		if (ot == DMU_OT_OBJSET) {  in zio_decrypt()
644 			    dsobj, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp));  in zio_decrypt()
648 			    zio->io_abd, size, mac);  in zio_decrypt()
651 				    &zio->io_bookmark, ot, ECKSUM);  in zio_decrypt()
654 		abd_copy(data, zio->io_abd, size);  in zio_decrypt()
664 	if (ot == DMU_OT_INTENT_LOG) {  in zio_decrypt()
665 		tmp = abd_borrow_buf_copy(zio->io_abd, sizeof (zil_chain_t));  in zio_decrypt()
667 		abd_return_buf(zio->io_abd, tmp, sizeof (zil_chain_t));  in zio_decrypt()
672 	ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp),  in zio_decrypt()
674 	    zio->io_abd, &no_crypt);  in zio_decrypt()
676 		abd_copy(data, zio->io_abd, size);  in zio_decrypt()
685 	ASSERT(ret != EACCES || (zio->io_flags & ZIO_FLAG_SPECULATIVE));  in zio_decrypt()
692 		zio->io_error = SET_ERROR(EIO);  in zio_decrypt()
693 		if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {  in zio_decrypt()
694 			spa_log_error(spa, &zio->io_bookmark,  in zio_decrypt()
695 			    BP_GET_PHYSICAL_BIRTH(zio->io_bp));  in zio_decrypt()
697 			    spa, NULL, &zio->io_bookmark, zio, 0);  in zio_decrypt()
700 		zio->io_error = ret;  in zio_decrypt()
712 	list_t *pl = &cio->io_parent_list;  in zio_walk_parents()
718 	ASSERT((*zl)->zl_child == cio);  in zio_walk_parents()
719 	return ((*zl)->zl_parent);  in zio_walk_parents()
725 	list_t *cl = &pio->io_child_list;  in zio_walk_children()
727 	ASSERT(MUTEX_HELD(&pio->io_lock));  in zio_walk_children()
733 	ASSERT((*zl)->zl_parent == pio);  in zio_walk_children()
734 	return ((*zl)->zl_child);  in zio_walk_children()
756 	ASSERT3S(cio->io_child_type, <=, pio->io_child_type);  in zio_add_child_impl()
759 	IMPLY((cio->io_pipeline & ZIO_STAGE_READY) == 0 &&  in zio_add_child_impl()
760 	    (cio->io_child_type != ZIO_CHILD_VDEV),  in zio_add_child_impl()
761 	    (pio->io_pipeline & ZIO_STAGE_READY) == 0);  in zio_add_child_impl()
764 	zl->zl_parent = pio;  in zio_add_child_impl()
765 	zl->zl_child = cio;  in zio_add_child_impl()
767 	mutex_enter(&pio->io_lock);  in zio_add_child_impl()
770 		ASSERT(list_is_empty(&cio->io_parent_list));  in zio_add_child_impl()
772 		mutex_enter(&cio->io_lock);  in zio_add_child_impl()
774 	ASSERT0(pio->io_state[ZIO_WAIT_DONE]);  in zio_add_child_impl()
776 	uint64_t *countp = pio->io_children[cio->io_child_type];  in zio_add_child_impl()
778 		countp[w] += !cio->io_state[w];  in zio_add_child_impl()
780 	list_insert_head(&pio->io_child_list, zl);  in zio_add_child_impl()
781 	list_insert_head(&cio->io_parent_list, zl);  in zio_add_child_impl()
784 		mutex_exit(&cio->io_lock);  in zio_add_child_impl()
786 	mutex_exit(&pio->io_lock);  in zio_add_child_impl()
804 	ASSERT(zl->zl_parent == pio);  in zio_remove_child()
805 	ASSERT(zl->zl_child == cio);  in zio_remove_child()
807 	mutex_enter(&pio->io_lock);  in zio_remove_child()
808 	mutex_enter(&cio->io_lock);  in zio_remove_child()
810 	list_remove(&pio->io_child_list, zl);  in zio_remove_child()
811 	list_remove(&cio->io_parent_list, zl);  in zio_remove_child()
813 	mutex_exit(&cio->io_lock);  in zio_remove_child()
814 	mutex_exit(&pio->io_lock);  in zio_remove_child()
823 	mutex_enter(&zio->io_lock);  in zio_wait_for_children()
824 	ASSERT0P(zio->io_stall);  in zio_wait_for_children()
829 		uint64_t *countp = &zio->io_children[c][wait];  in zio_wait_for_children()
831 			zio->io_stage >>= 1;  in zio_wait_for_children()
832 			ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN);  in zio_wait_for_children()
833 			zio->io_stall = countp;  in zio_wait_for_children()
838 	mutex_exit(&zio->io_lock);  in zio_wait_for_children()
847 	uint64_t *countp = &pio->io_children[zio->io_child_type][wait];  in zio_notify_parent()
848 	int *errorp = &pio->io_child_error[zio->io_child_type];  in zio_notify_parent()
850 	mutex_enter(&pio->io_lock);  in zio_notify_parent()
851 	if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))  in zio_notify_parent()
852 		*errorp = zio_worst_error(*errorp, zio->io_error);  in zio_notify_parent()
853 	pio->io_post |= zio->io_post;  in zio_notify_parent()
856 	(*countp)--;  in zio_notify_parent()
858 	if (*countp == 0 && pio->io_stall == countp) {  in zio_notify_parent()
860 		    pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE :  in zio_notify_parent()
862 		pio->io_stall = NULL;  in zio_notify_parent()
863 		mutex_exit(&pio->io_lock);  in zio_notify_parent()
886 		 * parent-child relationships, as we do with the "mega zio"  in zio_notify_parent()
890 		    (pio->io_type == zio->io_type ||  in zio_notify_parent()
891 		    (pio->io_type == ZIO_TYPE_NULL && !pio->io_done))) {  in zio_notify_parent()
897 		mutex_exit(&pio->io_lock);  in zio_notify_parent()
904 	if (zio->io_child_error[c] != 0 && zio->io_error == 0)  in zio_inherit_child_errors()
905 		zio->io_error = zio->io_child_error[c];  in zio_inherit_child_errors()
914 	if (z1->io_bookmark.zb_objset < z2->io_bookmark.zb_objset)  in zio_bookmark_compare()
915 		return (-1);  in zio_bookmark_compare()
916 	if (z1->io_bookmark.zb_objset > z2->io_bookmark.zb_objset)  in zio_bookmark_compare()
919 	if (z1->io_bookmark.zb_object < z2->io_bookmark.zb_object)  in zio_bookmark_compare()
920 		return (-1);  in zio_bookmark_compare()
921 	if (z1->io_bookmark.zb_object > z2->io_bookmark.zb_object)  in zio_bookmark_compare()
924 	if (z1->io_bookmark.zb_level < z2->io_bookmark.zb_level)  in zio_bookmark_compare()
925 		return (-1);  in zio_bookmark_compare()
926 	if (z1->io_bookmark.zb_level > z2->io_bookmark.zb_level)  in zio_bookmark_compare()
929 	if (z1->io_bookmark.zb_blkid < z2->io_bookmark.zb_blkid)  in zio_bookmark_compare()
930 		return (-1);  in zio_bookmark_compare()
931 	if (z1->io_bookmark.zb_blkid > z2->io_bookmark.zb_blkid)  in zio_bookmark_compare()
935 		return (-1);  in zio_bookmark_compare()
970 	mutex_init(&zio->io_lock, NULL, MUTEX_NOLOCKDEP, NULL);  in zio_create()
971 	cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL);  in zio_create()
973 	list_create(&zio->io_parent_list, sizeof (zio_link_t),  in zio_create()
975 	list_create(&zio->io_child_list, sizeof (zio_link_t),  in zio_create()
977 	metaslab_trace_init(&zio->io_alloc_list);  in zio_create()
980 		zio->io_child_type = ZIO_CHILD_VDEV;  in zio_create()
982 		zio->io_child_type = ZIO_CHILD_GANG;  in zio_create()
984 		zio->io_child_type = ZIO_CHILD_DDT;  in zio_create()
986 		zio->io_child_type = ZIO_CHILD_LOGICAL;  in zio_create()
990 		    zio->io_child_type == ZIO_CHILD_DDT) {  in zio_create()
991 			zio->io_bp_copy = *bp;  in zio_create()
992 			zio->io_bp = &zio->io_bp_copy;	/* so caller can free */  in zio_create()
994 			zio->io_bp = (blkptr_t *)bp;  in zio_create()
996 		zio->io_bp_orig = *bp;  in zio_create()
997 		if (zio->io_child_type == ZIO_CHILD_LOGICAL)  in zio_create()
998 			zio->io_logical = zio;  in zio_create()
999 		if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))  in zio_create()
1002 			BP_ZERO_DVAS(zio->io_bp);  in zio_create()
1003 			BP_SET_BIRTH(zio->io_bp, 0, 0);  in zio_create()
1007 	zio->io_spa = spa;  in zio_create()
1008 	zio->io_txg = txg;  in zio_create()
1009 	zio->io_done = done;  in zio_create()
1010 	zio->io_private = private;  in zio_create()
1011 	zio->io_type = type;  in zio_create()
1012 	zio->io_priority = priority;  in zio_create()
1013 	zio->io_vd = vd;  in zio_create()
1014 	zio->io_offset = offset;  in zio_create()
1015 	zio->io_orig_abd = zio->io_abd = data;  in zio_create()
1016 	zio->io_orig_size = zio->io_size = psize;  in zio_create()
1017 	zio->io_lsize = lsize;  in zio_create()
1018 	zio->io_orig_flags = zio->io_flags = flags;  in zio_create()
1019 	zio->io_orig_stage = zio->io_stage = stage;  in zio_create()
1020 	zio->io_orig_pipeline = zio->io_pipeline = pipeline;  in zio_create()
1021 	zio->io_pipeline_trace = ZIO_STAGE_OPEN;  in zio_create()
1022 	zio->io_allocator = ZIO_ALLOCATOR_NONE;  in zio_create()
1024 	zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY) ||  in zio_create()
1026 	zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE);  in zio_create()
1029 		zio->io_bookmark = *zb;  in zio_create()
1032 		zio->io_metaslab_class = pio->io_metaslab_class;  in zio_create()
1033 		if (zio->io_logical == NULL)  in zio_create()
1034 			zio->io_logical = pio->io_logical;  in zio_create()
1035 		if (zio->io_child_type == ZIO_CHILD_GANG)  in zio_create()
1036 			zio->io_gang_leader = pio->io_gang_leader;  in zio_create()
1040 	taskq_init_ent(&zio->io_tqent);  in zio_create()
1048 	metaslab_trace_fini(&zio->io_alloc_list);  in zio_destroy()
1049 	list_destroy(&zio->io_parent_list);  in zio_destroy()
1050 	list_destroy(&zio->io_child_list);  in zio_destroy()
1051 	mutex_destroy(&zio->io_lock);  in zio_destroy()
1052 	cv_destroy(&zio->io_cv);  in zio_destroy()
1114 	    (long long)bp->blk_dva[0].dva_word[0],  in zfs_blkptr_verify_log()
1115 	    (long long)bp->blk_dva[0].dva_word[1],  in zfs_blkptr_verify_log()
1116 	    (long long)bp->blk_dva[1].dva_word[0],  in zfs_blkptr_verify_log()
1117 	    (long long)bp->blk_dva[1].dva_word[1],  in zfs_blkptr_verify_log()
1118 	    (long long)bp->blk_dva[2].dva_word[0],  in zfs_blkptr_verify_log()
1119 	    (long long)bp->blk_dva[2].dva_word[1],  in zfs_blkptr_verify_log()
1120 	    (long long)bp->blk_prop,  in zfs_blkptr_verify_log()
1121 	    (long long)bp->blk_prop2,  in zfs_blkptr_verify_log()
1122 	    (long long)bp->blk_pad,  in zfs_blkptr_verify_log()
1125 	    (long long)bp->blk_fill,  in zfs_blkptr_verify_log()
1126 	    (long long)bp->blk_cksum.zc_word[0],  in zfs_blkptr_verify_log()
1127 	    (long long)bp->blk_cksum.zc_word[1],  in zfs_blkptr_verify_log()
1128 	    (long long)bp->blk_cksum.zc_word[2],  in zfs_blkptr_verify_log()
1129 	    (long long)bp->blk_cksum.zc_word[3]);  in zfs_blkptr_verify_log()
1203 	} else if (unlikely(!DVA_IS_VALID(&bp->blk_dva[0]))) {  in zfs_blkptr_verify()
1204 		/* Non-hole, non-embedded BPs _must_ have at least one DVA */  in zfs_blkptr_verify()
1223 	if (unlikely(!spa->spa_trust_config))  in zfs_blkptr_verify()
1244 	 * Pool-specific checks.  in zfs_blkptr_verify()
1249 	 * dmu_sync()-ed blocks that are in the log) to be arbitrarily  in zfs_blkptr_verify()
1253 		const dva_t *dva = &bp->blk_dva[i];  in zfs_blkptr_verify()
1256 		if (unlikely(vdevid >= spa->spa_root_vdev->vdev_children)) {  in zfs_blkptr_verify()
1262 		vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];  in zfs_blkptr_verify()
1269 		if (unlikely(vd->vdev_ops == &vdev_hole_ops)) {  in zfs_blkptr_verify()
1275 		if (vd->vdev_ops == &vdev_missing_ops) {  in zfs_blkptr_verify()
1287 		if (unlikely(offset + asize > vd->vdev_asize)) {  in zfs_blkptr_verify()
1306 	if (vdevid >= spa->spa_root_vdev->vdev_children)  in zfs_dva_valid()
1309 	vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];  in zfs_dva_valid()
1313 	if (vd->vdev_ops == &vdev_hole_ops)  in zfs_dva_valid()
1316 	if (vd->vdev_ops == &vdev_missing_ops) {  in zfs_dva_valid()
1325 	if (offset + asize > vd->vdev_asize)  in zfs_dva_valid()
1355 	enum zio_stage pipeline = zp->zp_direct_write == B_TRUE ?  in zio_write()
1364 	zio->io_ready = ready;  in zio_write()
1365 	zio->io_children_ready = children_ready;  in zio_write()
1366 	zio->io_prop = *zp;  in zio_write()
1370 	 * provide the already-allocated BP.  But we may need the data to  in zio_write()
1372 	 * dedup (just take the already-allocated BP verbatim). Encrypted  in zio_write()
1377 	    (zio->io_prop.zp_dedup_verify || zio->io_prop.zp_encrypt)) {  in zio_write()
1378 		zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE;  in zio_write()
1402 	ASSERT(zio->io_type == ZIO_TYPE_WRITE);  in zio_write_override()
1403 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_write_override()
1404 	ASSERT(zio->io_stage == ZIO_STAGE_OPEN);  in zio_write_override()
1405 	ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));  in zio_write_override()
1413 	zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup;  in zio_write_override()
1414 	zio->io_prop.zp_nopwrite = nopwrite;  in zio_write_override()
1415 	zio->io_prop.zp_brtwrite = brtwrite;  in zio_write_override()
1416 	zio->io_prop.zp_copies = copies;  in zio_write_override()
1417 	zio->io_prop.zp_gang_copies = gang_copies;  in zio_write_override()
1418 	zio->io_bp_override = bp;  in zio_write_override()
1436 	 * Frees that are for the currently-syncing txg, are not going to be  in zio_free()
1439 	 * in-memory list for later processing.  in zio_free()
1447 	    txg != spa->spa_syncing_txg ||  in zio_free()
1452 		bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);  in zio_free()
1518 	 * All claims *must* be resolved in the first txg -- before the SPA  in zio_claim()
1519 	 * starts allocating blocks -- so that nothing is allocated twice.  in zio_claim()
1522 	ASSERT3U(BP_GET_LOGICAL_BIRTH(&spa->spa_uberblock.ub_rootbp), <,  in zio_claim()
1530 	ASSERT0(zio->io_queued_timestamp);  in zio_claim()
1542 	ASSERT0(vd->vdev_children);  in zio_trim()
1543 	ASSERT0(P2PHASE(offset, 1ULL << vd->vdev_ashift));  in zio_trim()
1544 	ASSERT0(P2PHASE(size, 1ULL << vd->vdev_ashift));  in zio_trim()
1547 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, NULL, size, size, done,  in zio_trim()
1550 	zio->io_trim_flags = trim_flags;  in zio_trim()
1562 	ASSERT0(vd->vdev_children);  in zio_read_phys()
1564 	    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);  in zio_read_phys()
1565 	ASSERT3U(offset + size, <=, vd->vdev_psize);  in zio_read_phys()
1567 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,  in zio_read_phys()
1571 	zio->io_prop.zp_checksum = checksum;  in zio_read_phys()
1583 	ASSERT0(vd->vdev_children);  in zio_write_phys()
1585 	    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);  in zio_write_phys()
1586 	ASSERT3U(offset + size, <=, vd->vdev_psize);  in zio_write_phys()
1588 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,  in zio_write_phys()
1592 	zio->io_prop.zp_checksum = checksum;  in zio_write_phys()
1596 		 * zec checksums are necessarily destructive -- they modify  in zio_write_phys()
1639 		pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;  in zio_vdev_child_io()
1647 		ASSERT0(pio->io_post & ZIO_POST_DIO_CHKSUM_ERR);  in zio_vdev_child_io()
1649 	    pio->io_prop.zp_direct_write == B_TRUE) {  in zio_vdev_child_io()
1656 		 * the top-level VDEV, which is set by this child ZIO.  in zio_vdev_child_io()
1659 		ASSERT3U(pio->io_child_type, ==, ZIO_CHILD_LOGICAL);  in zio_vdev_child_io()
1663 	if (vd->vdev_ops->vdev_op_leaf) {  in zio_vdev_child_io()
1664 		ASSERT0(vd->vdev_children);  in zio_vdev_child_io()
1671 	 * If we've decided to do a repair, the write is not speculative --  in zio_vdev_child_io()
1679 	 * top-level vdev, then the child zio is not an allocating I/O.  in zio_vdev_child_io()
1684 	    (vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) {  in zio_vdev_child_io()
1685 		ASSERT(pio->io_metaslab_class != NULL);  in zio_vdev_child_io()
1686 		ASSERT(pio->io_metaslab_class->mc_alloc_throttle_enabled);  in zio_vdev_child_io()
1690 		ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) ||  in zio_vdev_child_io()
1691 		    pio->io_child_type == ZIO_CHILD_GANG);  in zio_vdev_child_io()
1696 	zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, size,  in zio_vdev_child_io()
1697 	    done, private, type, priority, flags, vd, offset, &pio->io_bookmark,  in zio_vdev_child_io()
1699 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);  in zio_vdev_child_io()
1711 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in zio_vdev_delegated_io()
1713 	zio = zio_create(NULL, vd->vdev_spa, 0, NULL,  in zio_vdev_delegated_io()
1734 	if (vd->vdev_nowritecache)  in zio_flush()
1737 	if (vd->vdev_children == 0) {  in zio_flush()
1738 		zio_nowait(zio_create(pio, vd->vdev_spa, 0, NULL, NULL, 0, 0,  in zio_flush()
1742 		for (uint64_t c = 0; c < vd->vdev_children; c++)  in zio_flush()
1743 			zio_flush(pio, vd->vdev_child[c]);  in zio_flush()
1750 	ASSERT0P(zio->io_executor);  in zio_shrink()
1751 	ASSERT3U(zio->io_orig_size, ==, zio->io_size);  in zio_shrink()
1752 	ASSERT3U(size, <=, zio->io_size);  in zio_shrink()
1759 	ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF);  in zio_shrink()
1760 	if (!BP_IS_RAIDZ(zio->io_bp)) {  in zio_shrink()
1762 		ASSERT3U(zio->io_size, ==, zio->io_lsize);  in zio_shrink()
1763 		zio->io_orig_size = zio->io_size = zio->io_lsize = size;  in zio_shrink()
1775 	if (size > spa->spa_min_alloc)  in zio_roundup_alloc_size()
1776 		return (roundup(size, spa->spa_gcd_alloc));  in zio_roundup_alloc_size()
1777 	return (spa->spa_min_alloc);  in zio_roundup_alloc_size()
1787 	d_len = s_len - (s_len >> 3);  in zio_get_compression_max_size()
1793 	d_len = d_len - d_len % gcd_alloc;  in zio_get_compression_max_size()
1809 	blkptr_t *bp = zio->io_bp;  in zio_read_bp_init()
1813 	ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);  in zio_read_bp_init()
1816 	    zio->io_child_type == ZIO_CHILD_LOGICAL &&  in zio_read_bp_init()
1817 	    !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {  in zio_read_bp_init()
1818 		zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize),  in zio_read_bp_init()
1822 	if (((BP_IS_PROTECTED(bp) && !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) ||  in zio_read_bp_init()
1824 	    zio->io_child_type == ZIO_CHILD_LOGICAL) {  in zio_read_bp_init()
1825 		zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize),  in zio_read_bp_init()
1831 		void *data = abd_borrow_buf(zio->io_abd, psize);  in zio_read_bp_init()
1833 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_read_bp_init()
1835 		abd_return_buf_copy(zio->io_abd, data, psize);  in zio_read_bp_init()
1840 	if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)  in zio_read_bp_init()
1841 		zio->io_pipeline = ZIO_DDT_READ_PIPELINE;  in zio_read_bp_init()
1852 	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);  in zio_write_bp_init()
1854 	if (zio->io_bp_override) {  in zio_write_bp_init()
1855 		blkptr_t *bp = zio->io_bp;  in zio_write_bp_init()
1856 		zio_prop_t *zp = &zio->io_prop;  in zio_write_bp_init()
1858 		ASSERT(BP_GET_BIRTH(bp) != zio->io_txg);  in zio_write_bp_init()
1860 		*bp = *zio->io_bp_override;  in zio_write_bp_init()
1861 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_write_bp_init()
1863 		if (zp->zp_brtwrite)  in zio_write_bp_init()
1866 		ASSERT(!BP_GET_DEDUP(zio->io_bp_override));  in zio_write_bp_init()
1876 		if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) {  in zio_write_bp_init()
1877 			ASSERT(!zp->zp_dedup);  in zio_write_bp_init()
1878 			ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum);  in zio_write_bp_init()
1879 			zio->io_flags |= ZIO_FLAG_NOPWRITE;  in zio_write_bp_init()
1883 		ASSERT(!zp->zp_nopwrite);  in zio_write_bp_init()
1885 		if (BP_IS_HOLE(bp) || !zp->zp_dedup)  in zio_write_bp_init()
1888 		ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &  in zio_write_bp_init()
1889 		    ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify);  in zio_write_bp_init()
1891 		if (BP_GET_CHECKSUM(bp) == zp->zp_checksum &&  in zio_write_bp_init()
1892 		    !zp->zp_encrypt) {  in zio_write_bp_init()
1894 			zio->io_pipeline |= ZIO_STAGE_DDT_WRITE;  in zio_write_bp_init()
1902 		zio->io_bp_override = NULL;  in zio_write_bp_init()
1903 		*bp = zio->io_bp_orig;  in zio_write_bp_init()
1904 		zio->io_pipeline = zio->io_orig_pipeline;  in zio_write_bp_init()
1913 	spa_t *spa = zio->io_spa;  in zio_write_compress()
1914 	zio_prop_t *zp = &zio->io_prop;  in zio_write_compress()
1915 	enum zio_compress compress = zp->zp_compress;  in zio_write_compress()
1916 	blkptr_t *bp = zio->io_bp;  in zio_write_compress()
1917 	uint64_t lsize = zio->io_lsize;  in zio_write_compress()
1918 	uint64_t psize = zio->io_size;  in zio_write_compress()
1933 	if (zio->io_children_ready != NULL) {  in zio_write_compress()
1939 		ASSERT3U(zp->zp_level, >, 0);  in zio_write_compress()
1940 		zio->io_children_ready(zio);  in zio_write_compress()
1943 	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);  in zio_write_compress()
1944 	ASSERT0P(zio->io_bp_override);  in zio_write_compress()
1946 	if (!BP_IS_HOLE(bp) && BP_GET_BIRTH(bp) == zio->io_txg) {  in zio_write_compress()
1958 		ASSERT(zio->io_txg == spa_syncing_txg(spa));  in zio_write_compress()
1959 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_write_compress()
1967 		    MIN(zp->zp_copies, spa_max_replication(spa))  in zio_write_compress()
1973 	    !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {  in zio_write_compress()
1975 		if (abd_cmp_zero(zio->io_abd, lsize) == 0)  in zio_write_compress()
1980 			psize = zio_compress_data(compress, zio->io_abd, &cabd,  in zio_write_compress()
1983 			    spa->spa_gcd_alloc, spa->spa_min_alloc, lsize),  in zio_write_compress()
1984 			    zp->zp_complevel);  in zio_write_compress()
1991 		} else if (psize <= BPE_PAYLOAD_SIZE && !zp->zp_encrypt &&  in zio_write_compress()
1992 		    zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) &&  in zio_write_compress()
1998 			BP_SET_TYPE(bp, zio->io_prop.zp_type);  in zio_write_compress()
1999 			BP_SET_LEVEL(bp, zio->io_prop.zp_level);  in zio_write_compress()
2002 			BP_SET_LOGICAL_BIRTH(bp, zio->io_txg);  in zio_write_compress()
2003 			zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_write_compress()
2010 			 * size of the smallest-ashift device, and zero the  in zio_write_compress()
2023 				abd_zero_off(cabd, psize, rounded - psize);  in zio_write_compress()
2034 		zio->io_bp_override = NULL;  in zio_write_compress()
2035 		*bp = zio->io_bp_orig;  in zio_write_compress()
2036 		zio->io_pipeline = zio->io_orig_pipeline;  in zio_write_compress()
2038 	} else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 &&  in zio_write_compress()
2039 	    zp->zp_type == DMU_OT_DNODE) {  in zio_write_compress()
2047 		if (abd_cmp_zero(zio->io_abd, lsize) == 0) {  in zio_write_compress()
2053 	} else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS &&  in zio_write_compress()
2054 	    !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) {  in zio_write_compress()
2057 		 * take this codepath because it will change the on-disk block  in zio_write_compress()
2065 			abd_zero_off(cdata, psize, rounded - psize);  in zio_write_compress()
2066 			abd_copy_off(cdata, zio->io_abd, 0, 0, psize);  in zio_write_compress()
2077 	 * few passes offer a trade-off: allocating blocks defers convergence,  in zio_write_compress()
2083 	if (!BP_IS_HOLE(bp) && BP_GET_BIRTH(bp) == zio->io_txg &&  in zio_write_compress()
2087 		enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES;  in zio_write_compress()
2089 		zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages;  in zio_write_compress()
2090 		zio->io_flags |= ZIO_FLAG_IO_REWRITE;  in zio_write_compress()
2093 		zio->io_pipeline = ZIO_WRITE_PIPELINE;  in zio_write_compress()
2097 		if (BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig) != 0 &&  in zio_write_compress()
2100 			BP_SET_TYPE(bp, zp->zp_type);  in zio_write_compress()
2101 			BP_SET_LEVEL(bp, zp->zp_level);  in zio_write_compress()
2102 			BP_SET_BIRTH(bp, zio->io_txg, 0);  in zio_write_compress()
2104 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_write_compress()
2106 		ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER);  in zio_write_compress()
2108 		BP_SET_TYPE(bp, zp->zp_type);  in zio_write_compress()
2109 		BP_SET_LEVEL(bp, zp->zp_level);  in zio_write_compress()
2112 		BP_SET_CHECKSUM(bp, zp->zp_checksum);  in zio_write_compress()
2113 		BP_SET_DEDUP(bp, zp->zp_dedup);  in zio_write_compress()
2115 		if (zp->zp_dedup) {  in zio_write_compress()
2116 			ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_write_compress()
2117 			ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));  in zio_write_compress()
2118 			ASSERT(!zp->zp_encrypt ||  in zio_write_compress()
2119 			    DMU_OT_IS_ENCRYPTED(zp->zp_type));  in zio_write_compress()
2120 			zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE;  in zio_write_compress()
2122 		if (zp->zp_nopwrite) {  in zio_write_compress()
2123 			ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_write_compress()
2124 			ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));  in zio_write_compress()
2125 			zio->io_pipeline |= ZIO_STAGE_NOP_WRITE;  in zio_write_compress()
2134 	blkptr_t *bp = zio->io_bp;  in zio_free_bp_init()
2136 	if (zio->io_child_type == ZIO_CHILD_LOGICAL) {  in zio_free_bp_init()
2138 			zio->io_pipeline = ZIO_DDT_FREE_PIPELINE;  in zio_free_bp_init()
2141 	ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);  in zio_free_bp_init()
2155 	spa_t *spa = zio->io_spa;  in zio_taskq_dispatch()
2156 	zio_type_t t = zio->io_type;  in zio_taskq_dispatch()
2161 	 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL.  in zio_taskq_dispatch()
2163 	if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE))  in zio_taskq_dispatch()
2169 	if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux)  in zio_taskq_dispatch()
2176 	if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE) {  in zio_taskq_dispatch()
2177 		if (spa->spa_zio_taskq[t][q + 1].stqs_count != 0)  in zio_taskq_dispatch()
2191 	spa_t *spa = zio->io_spa;  in zio_taskq_member()
2196 		spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];  in zio_taskq_member()
2198 		for (i = 0; i < tqs->stqs_count; i++) {  in zio_taskq_member()
2199 			if (tqs->stqs_taskq[i] == tq)  in zio_taskq_member()
2210 	ASSERT((zio->io_type != ZIO_TYPE_WRITE) || ZIO_HAS_ALLOCATOR(zio));  in zio_issue_async()
2236 	if (zio->io_target_timestamp != 0) {  in zio_delay_interrupt()
2239 		if (now >= zio->io_target_timestamp) {  in zio_delay_interrupt()
2257 			hrtime_t diff = zio->io_target_timestamp - now;  in zio_delay_interrupt()
2286 	vdev_t *vd = pio->io_vd;  in zio_deadman_impl()
2287 	uint64_t failmode = spa_get_deadman_failmode(pio->io_spa);  in zio_deadman_impl()
2289 	if (zio_deadman_log_all || (vd != NULL && vd->vdev_ops->vdev_op_leaf)) {  in zio_deadman_impl()
2290 		vdev_queue_t *vq = vd ? &vd->vdev_queue : NULL;  in zio_deadman_impl()
2291 		zbookmark_phys_t *zb = &pio->io_bookmark;  in zio_deadman_impl()
2292 		uint64_t delta = gethrtime() - pio->io_timestamp;  in zio_deadman_impl()
2299 		    "pipeline=0x%x pipeline-trace=0x%x "  in zio_deadman_impl()
2301 		    "level=%llu blkid=%llu "  in zio_deadman_impl()
2304 		    ziodepth, pio, pio->io_timestamp,  in zio_deadman_impl()
2305 		    (u_longlong_t)delta, pio->io_delta, pio->io_delay,  in zio_deadman_impl()
2306 		    vd ? vd->vdev_path : "NULL",  in zio_deadman_impl()
2307 		    vq ? vq->vq_io_complete_ts : 0, pio->io_type,  in zio_deadman_impl()
2308 		    pio->io_priority, (u_longlong_t)pio->io_flags,  in zio_deadman_impl()
2309 		    pio->io_stage, pio->io_pipeline, pio->io_pipeline_trace,  in zio_deadman_impl()
2310 		    (u_longlong_t)zb->zb_objset, (u_longlong_t)zb->zb_object,  in zio_deadman_impl()
2311 		    (u_longlong_t)zb->zb_level, (u_longlong_t)zb->zb_blkid,  in zio_deadman_impl()
2312 		    (u_longlong_t)pio->io_offset, (u_longlong_t)pio->io_size,  in zio_deadman_impl()
2313 		    pio->io_error);  in zio_deadman_impl()
2315 		    pio->io_spa, vd, zb, pio, 0);  in zio_deadman_impl()
2318 	if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&  in zio_deadman_impl()
2319 	    list_is_empty(&pio->io_child_list) &&  in zio_deadman_impl()
2321 	    taskq_empty_ent(&pio->io_tqent) &&  in zio_deadman_impl()
2322 	    pio->io_queue_state == ZIO_QS_ACTIVE) {  in zio_deadman_impl()
2323 		pio->io_error = EINTR;  in zio_deadman_impl()
2327 	mutex_enter(&pio->io_lock);  in zio_deadman_impl()
2332 	mutex_exit(&pio->io_lock);  in zio_deadman_impl()
2342 	spa_t *spa = pio->io_spa;  in zio_deadman()
2370  * vdev-level caching or aggregation; (5) the I/O is deferred
2371  * due to vdev-level queueing; (6) the I/O is handed off to
2401  * stack size of 16K is required to avoid needing to re-dispatch the zio.
2407 	dsl_pool_t *dp = spa_get_dsl(zio->io_spa);  in zio_execute_stack_check()
2410 	if (dp && curthread == dp->dp_tx.tx_sync_thread)  in zio_execute_stack_check()
2414 	if (dp && spa_is_initializing(dp->dp_spa) &&  in zio_execute_stack_check()
2429 	ASSERT3U(zio->io_queued_timestamp, >, 0);  in __zio_execute()
2431 	while (zio->io_stage < ZIO_STAGE_DONE) {  in __zio_execute()
2432 		enum zio_stage pipeline = zio->io_pipeline;  in __zio_execute()
2433 		enum zio_stage stage = zio->io_stage;  in __zio_execute()
2435 		zio->io_executor = curthread;  in __zio_execute()
2437 		ASSERT(!MUTEX_HELD(&zio->io_lock));  in __zio_execute()
2439 		ASSERT0P(zio->io_stall);  in __zio_execute()
2456 		if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&  in __zio_execute()
2475 		zio->io_stage = stage;  in __zio_execute()
2476 		zio->io_pipeline_trace |= zio->io_stage;  in __zio_execute()
2483 		zio = zio_pipeline[highbit64(stage) - 1](zio);  in __zio_execute()
2511 	ASSERT3S(zio->io_stage, ==, ZIO_STAGE_OPEN);  in zio_wait()
2512 	ASSERT0P(zio->io_executor);  in zio_wait()
2514 	zio->io_waiter = curthread;  in zio_wait()
2515 	ASSERT0(zio->io_queued_timestamp);  in zio_wait()
2516 	zio->io_queued_timestamp = gethrtime();  in zio_wait()
2518 	if (zio->io_type == ZIO_TYPE_WRITE) {  in zio_wait()
2523 	mutex_enter(&zio->io_lock);  in zio_wait()
2524 	while (zio->io_executor != NULL) {  in zio_wait()
2525 		error = cv_timedwait_io(&zio->io_cv, &zio->io_lock,  in zio_wait()
2528 		if (zfs_deadman_enabled && error == -1 &&  in zio_wait()
2529 		    gethrtime() - zio->io_queued_timestamp >  in zio_wait()
2530 		    spa_deadman_ziotime(zio->io_spa)) {  in zio_wait()
2531 			mutex_exit(&zio->io_lock);  in zio_wait()
2534 			mutex_enter(&zio->io_lock);  in zio_wait()
2537 	mutex_exit(&zio->io_lock);  in zio_wait()
2539 	error = zio->io_error;  in zio_wait()
2554 	ASSERT0P(zio->io_executor);  in zio_nowait()
2556 	if (zio->io_child_type == ZIO_CHILD_LOGICAL &&  in zio_nowait()
2557 	    list_is_empty(&zio->io_parent_list)) {  in zio_nowait()
2565 		spa_t *spa = zio->io_spa;  in zio_nowait()
2566 		pio = spa->spa_async_zio_root[CPU_SEQID_UNSTABLE];  in zio_nowait()
2571 	ASSERT0(zio->io_queued_timestamp);  in zio_nowait()
2572 	zio->io_queued_timestamp = gethrtime();  in zio_nowait()
2573 	if (zio->io_type == ZIO_TYPE_WRITE) {  in zio_nowait()
2591 	ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_reexecute()
2592 	ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);  in zio_reexecute()
2593 	ASSERT0P(pio->io_gang_leader);  in zio_reexecute()
2594 	ASSERT0P(pio->io_gang_tree);  in zio_reexecute()
2596 	mutex_enter(&pio->io_lock);  in zio_reexecute()
2597 	pio->io_flags = pio->io_orig_flags;  in zio_reexecute()
2598 	pio->io_stage = pio->io_orig_stage;  in zio_reexecute()
2599 	pio->io_pipeline = pio->io_orig_pipeline;  in zio_reexecute()
2600 	pio->io_post = 0;  in zio_reexecute()
2601 	pio->io_flags |= ZIO_FLAG_REEXECUTED;  in zio_reexecute()
2602 	pio->io_pipeline_trace = 0;  in zio_reexecute()
2603 	pio->io_error = 0;  in zio_reexecute()
2604 	pio->io_state[ZIO_WAIT_READY] = (pio->io_stage >= ZIO_STAGE_READY) ||  in zio_reexecute()
2605 	    (pio->io_pipeline & ZIO_STAGE_READY) == 0;  in zio_reexecute()
2606 	pio->io_state[ZIO_WAIT_DONE] = (pio->io_stage >= ZIO_STAGE_DONE);  in zio_reexecute()
2618 	 * gio->io_lock before each update.  in zio_reexecute()
2622 		mutex_enter(&gio->io_lock);  in zio_reexecute()
2624 			gio->io_children[pio->io_child_type][w] +=  in zio_reexecute()
2625 			    !pio->io_state[w];  in zio_reexecute()
2627 		mutex_exit(&gio->io_lock);  in zio_reexecute()
2631 		pio->io_child_error[c] = 0;  in zio_reexecute()
2634 		BP_ZERO(pio->io_bp);  in zio_reexecute()
2646 		mutex_exit(&pio->io_lock);  in zio_reexecute()
2648 		mutex_enter(&pio->io_lock);  in zio_reexecute()
2650 	mutex_exit(&pio->io_lock);  in zio_reexecute()
2657 	if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) {  in zio_reexecute()
2658 		pio->io_queued_timestamp = gethrtime();  in zio_reexecute()
2679 	mutex_enter(&spa->spa_suspend_lock);  in zio_suspend()
2681 	if (spa->spa_suspend_zio_root == NULL)  in zio_suspend()
2682 		spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL,  in zio_suspend()
2686 	spa->spa_suspended = reason;  in zio_suspend()
2689 		ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));  in zio_suspend()
2690 		ASSERT(zio != spa->spa_suspend_zio_root);  in zio_suspend()
2691 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_suspend()
2693 		ASSERT(zio->io_stage == ZIO_STAGE_DONE);  in zio_suspend()
2694 		zio_add_child(spa->spa_suspend_zio_root, zio);  in zio_suspend()
2697 	mutex_exit(&spa->spa_suspend_lock);  in zio_suspend()
2699 	txg_wait_kick(spa->spa_dsl_pool);  in zio_suspend()
2710 	mutex_enter(&spa->spa_suspend_lock);  in zio_resume()
2711 	if (spa->spa_suspended != ZIO_SUSPEND_NONE)  in zio_resume()
2715 	spa->spa_suspended = ZIO_SUSPEND_NONE;  in zio_resume()
2716 	cv_broadcast(&spa->spa_suspend_cv);  in zio_resume()
2717 	pio = spa->spa_suspend_zio_root;  in zio_resume()
2718 	spa->spa_suspend_zio_root = NULL;  in zio_resume()
2719 	mutex_exit(&spa->spa_suspend_lock);  in zio_resume()
2731 	mutex_enter(&spa->spa_suspend_lock);  in zio_resume_wait()
2733 		cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock);  in zio_resume_wait()
2734 	mutex_exit(&spa->spa_suspend_lock);  in zio_resume_wait()
2756  * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg>
2771  * an in-core tree containing the contents of every gang header and the
2776  * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp.
2784  * The two-phase assemble/issue model solves the problem of partial failure --
2789  * is assembled, free and claim are in-memory operations that cannot fail.
2796  * Gang rewrites only happen during sync-to-convergence.  If we can't assemble
2809 	abd_free(zio->io_abd);  in zio_gang_issue_func_done()
2819 	return (zio_read(pio, pio->io_spa, bp, abd_get_offset(data, offset),  in zio_read_gang()
2821 	    NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),  in zio_read_gang()
2822 	    &pio->io_bookmark));  in zio_read_gang()
2833 		    abd_get_from_buf(gn->gn_gbh, gn->gn_gangblocksize);  in zio_rewrite_gang()
2834 		zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,  in zio_rewrite_gang()
2835 		    gbh_abd, gn->gn_gangblocksize, zio_gang_issue_func_done,  in zio_rewrite_gang()
2836 		    NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),  in zio_rewrite_gang()
2837 		    &pio->io_bookmark);  in zio_rewrite_gang()
2847 		if (gn != pio->io_gang_leader->io_gang_tree) {  in zio_rewrite_gang()
2859 		if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE)  in zio_rewrite_gang()
2860 			zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;  in zio_rewrite_gang()
2862 		zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,  in zio_rewrite_gang()
2864 		    zio_gang_issue_func_done, NULL, pio->io_priority,  in zio_rewrite_gang()
2865 		    ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);  in zio_rewrite_gang()
2877 	zio_t *zio = zio_free_sync(pio, pio->io_spa, pio->io_txg, bp,  in zio_free_gang()
2880 		zio = zio_null(pio, pio->io_spa,  in zio_free_gang()
2891 	return (zio_claim(pio, pio->io_spa, pio->io_txg, bp,  in zio_claim_gang()
2915 	gn->gn_gangblocksize = gn->gn_allocsize = gangblocksize;  in zio_gang_node_alloc()
2916 	gn->gn_gbh = zio_buf_alloc(gangblocksize);  in zio_gang_node_alloc()
2927 	for (int g = 0; g < gbh_nblkptrs(gn->gn_allocsize); g++)  in zio_gang_node_free()
2928 		ASSERT0P(gn->gn_child[g]);  in zio_gang_node_free()
2930 	zio_buf_free(gn->gn_gbh, gn->gn_allocsize);  in zio_gang_node_free()
2932 	    (gbh_nblkptrs(gn->gn_allocsize) * sizeof (gn)));  in zio_gang_node_free()
2944 	for (int g = 0; g < gbh_nblkptrs(gn->gn_allocsize); g++)  in zio_gang_tree_free()
2945 		zio_gang_tree_free(&gn->gn_child[g]);  in zio_gang_tree_free()
2954 	if (spa_feature_is_active(gio->io_spa,  in zio_gang_tree_assemble()
2956 		spa_config_enter(gio->io_spa, SCL_VDEV, FTAG, RW_READER);  in zio_gang_tree_assemble()
2958 			vdev_t *vd = vdev_lookup_top(gio->io_spa,  in zio_gang_tree_assemble()
2959 			    DVA_GET_VDEV(&bp->blk_dva[dva]));  in zio_gang_tree_assemble()
2963 		spa_config_exit(gio->io_spa, SCL_VDEV, FTAG);  in zio_gang_tree_assemble()
2969 	abd_t *gbh_abd = abd_get_from_buf(gn->gn_gbh, gangblocksize);  in zio_gang_tree_assemble()
2971 	ASSERT(gio->io_gang_leader == gio);  in zio_gang_tree_assemble()
2974 	zio_nowait(zio_read(gio, gio->io_spa, bp, gbh_abd, gangblocksize,  in zio_gang_tree_assemble()
2975 	    zio_gang_tree_assemble_done, gn, gio->io_priority,  in zio_gang_tree_assemble()
2976 	    ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark));  in zio_gang_tree_assemble()
2982 	zio_t *gio = zio->io_gang_leader;  in zio_gang_tree_assemble_done()
2983 	zio_gang_node_t *gn = zio->io_private;  in zio_gang_tree_assemble_done()
2984 	blkptr_t *bp = zio->io_bp;  in zio_gang_tree_assemble_done()
2987 	ASSERT(list_is_empty(&zio->io_child_list));  in zio_gang_tree_assemble_done()
2989 	if (zio->io_error)  in zio_gang_tree_assemble_done()
2994 		byteswap_uint64_array(abd_to_buf(zio->io_abd), zio->io_size);  in zio_gang_tree_assemble_done()
2996 	ASSERT3P(abd_to_buf(zio->io_abd), ==, gn->gn_gbh);  in zio_gang_tree_assemble_done()
2998 	 * If this was an old-style gangblock, the gangblocksize should have  in zio_gang_tree_assemble_done()
3001 	ASSERT3U(gbh_eck(gn->gn_gbh, gn->gn_gangblocksize)->zec_magic,  in zio_gang_tree_assemble_done()
3004 	abd_free(zio->io_abd);  in zio_gang_tree_assemble_done()
3006 	for (int g = 0; g < gbh_nblkptrs(gn->gn_gangblocksize); g++) {  in zio_gang_tree_assemble_done()
3007 		blkptr_t *gbp = gbh_bp(gn->gn_gbh, g);  in zio_gang_tree_assemble_done()
3010 		zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]);  in zio_gang_tree_assemble_done()
3018 	zio_t *gio = pio->io_gang_leader;  in zio_gang_tree_issue()
3022 	ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp));  in zio_gang_tree_issue()
3023 	ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree);  in zio_gang_tree_issue()
3026 	 * If you're a gang header, your data is in gn->gn_gbh.  in zio_gang_tree_issue()
3029 	zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data, offset);  in zio_gang_tree_issue()
3032 		ASSERT3U(gbh_eck(gn->gn_gbh,  in zio_gang_tree_issue()
3033 		    gn->gn_gangblocksize)->zec_magic, ==, ZEC_MAGIC);  in zio_gang_tree_issue()
3035 		for (int g = 0; g < gbh_nblkptrs(gn->gn_gangblocksize); g++) {  in zio_gang_tree_issue()
3036 			blkptr_t *gbp = gbh_bp(gn->gn_gbh, g);  in zio_gang_tree_issue()
3039 			zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data,  in zio_gang_tree_issue()
3045 	if (gn == gio->io_gang_tree)  in zio_gang_tree_issue()
3046 		ASSERT3U(gio->io_size, ==, offset);  in zio_gang_tree_issue()
3055 	blkptr_t *bp = zio->io_bp;  in zio_gang_assemble()
3057 	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL);  in zio_gang_assemble()
3058 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);  in zio_gang_assemble()
3060 	zio->io_gang_leader = zio;  in zio_gang_assemble()
3062 	zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);  in zio_gang_assemble()
3070 	blkptr_t *bp = zio->io_bp;  in zio_gang_issue()
3076 	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);  in zio_gang_issue()
3077 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);  in zio_gang_issue()
3079 	if (zio->io_child_error[ZIO_CHILD_GANG] == 0)  in zio_gang_issue()
3080 		zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_abd,  in zio_gang_issue()
3083 		zio_gang_tree_free(&zio->io_gang_tree);  in zio_gang_issue()
3085 	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_gang_issue()
3093 	cio->io_allocator = pio->io_allocator;  in zio_gang_inherit_allocator()
3100 	dva_t *cdva = zio->io_bp->blk_dva;  in zio_write_gang_member_ready()
3101 	dva_t *pdva = pio->io_bp->blk_dva;  in zio_write_gang_member_ready()
3103 	zio_t *gio __maybe_unused = zio->io_gang_leader;  in zio_write_gang_member_ready()
3105 	if (BP_IS_HOLE(zio->io_bp))  in zio_write_gang_member_ready()
3109 	 * If we're getting direct-invoked from zio_write_gang_block(),  in zio_write_gang_member_ready()
3112 	ASSERT(BP_IS_HOLE(&zio->io_bp_orig) ||  in zio_write_gang_member_ready()
3113 	    zio->io_flags & ZIO_FLAG_PREALLOCATED);  in zio_write_gang_member_ready()
3115 	ASSERT(zio->io_child_type == ZIO_CHILD_GANG);  in zio_write_gang_member_ready()
3116 	ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies);  in zio_write_gang_member_ready()
3117 	ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp));  in zio_write_gang_member_ready()
3118 	ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp));  in zio_write_gang_member_ready()
3119 	VERIFY3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp));  in zio_write_gang_member_ready()
3121 	mutex_enter(&pio->io_lock);  in zio_write_gang_member_ready()
3122 	for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) {  in zio_write_gang_member_ready()
3128 	mutex_exit(&pio->io_lock);  in zio_write_gang_member_ready()
3139 	if (zio->io_abd != NULL)  in zio_write_gang_done()
3140 		abd_free(zio->io_abd);  in zio_write_gang_done()
3146 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;  in zio_update_feature()
3153 	spa_t *spa = pio->io_spa;  in zio_write_gang_block()
3154 	blkptr_t *bp = pio->io_bp;  in zio_write_gang_block()
3155 	zio_t *gio = pio->io_gang_leader;  in zio_write_gang_block()
3160 	uint64_t txg = pio->io_txg;  in zio_write_gang_block()
3161 	uint64_t resid = pio->io_size;  in zio_write_gang_block()
3164 	boolean_t has_data = !(pio->io_flags & ZIO_FLAG_NODATA);  in zio_write_gang_block()
3171 	int gbh_copies = gio->io_prop.zp_gang_copies;  in zio_write_gang_block()
3179 		ASSERT3U(gio->io_child_type, ==, ZIO_CHILD_DDT);  in zio_write_gang_block()
3180 		pio->io_error = EAGAIN;  in zio_write_gang_block()
3188 	if (pio->io_flags & ZIO_FLAG_ALLOC_THROTTLED) {  in zio_write_gang_block()
3189 		ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);  in zio_write_gang_block()
3198 	    bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,  in zio_write_gang_block()
3199 	    &pio->io_alloc_list, pio->io_allocator, pio, &candidate);  in zio_write_gang_block()
3201 		pio->io_error = error;  in zio_write_gang_block()
3208 		gnpp = &gio->io_gang_tree;  in zio_write_gang_block()
3210 		gnpp = pio->io_private;  in zio_write_gang_block()
3211 		ASSERT(pio->io_ready == zio_write_gang_member_ready);  in zio_write_gang_block()
3215 	gbh = gn->gn_gbh;  in zio_write_gang_block()
3223 	    zio_write_gang_done, NULL, pio->io_priority,  in zio_write_gang_block()
3224 	    ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);  in zio_write_gang_block()
3227 	if (pio->io_flags & ZIO_FLAG_ALLOC_THROTTLED) {  in zio_write_gang_block()
3229 		VERIFY(metaslab_class_throttle_reserve(mc, zio->io_allocator,  in zio_write_gang_block()
3230 		    gbh_copies, zio->io_size, B_TRUE, &more));  in zio_write_gang_block()
3231 		zio->io_flags |= ZIO_FLAG_ALLOC_THROTTLED;  in zio_write_gang_block()
3244 		zp.zp_checksum = gio->io_prop.zp_checksum;  in zio_write_gang_block()
3246 		zp.zp_complevel = gio->io_prop.zp_complevel;  in zio_write_gang_block()
3249 		zp.zp_copies = gio->io_prop.zp_copies;  in zio_write_gang_block()
3250 		zp.zp_gang_copies = gio->io_prop.zp_gang_copies;  in zio_write_gang_block()
3254 		zp.zp_encrypt = gio->io_prop.zp_encrypt;  in zio_write_gang_block()
3255 		zp.zp_byteorder = gio->io_prop.zp_byteorder;  in zio_write_gang_block()
3262 		    resid / (gbh_nblkptrs(gangblocksize) - g));  in zio_write_gang_block()
3270 		    bp, gio->io_prop.zp_copies, txg, NULL,  in zio_write_gang_block()
3271 		    flags, &cio_list, zio->io_allocator, NULL, &allocated_size);  in zio_write_gang_block()
3281 		    abd_get_offset(pio->io_abd, pio->io_size - resid) : NULL,  in zio_write_gang_block()
3283 		    zio_write_gang_done, &gn->gn_child[g], pio->io_priority,  in zio_write_gang_block()
3285 		    (allocated ? ZIO_FLAG_PREALLOCATED : 0), &pio->io_bookmark);  in zio_write_gang_block()
3287 		resid -= psize;  in zio_write_gang_block()
3290 			metaslab_trace_move(&cio_list, &cio->io_alloc_list);  in zio_write_gang_block()
3292 			    &cio->io_bp_orig, zio->io_allocator, flags, psize,  in zio_write_gang_block()
3322 		    dmu_tx_create_assigned(spa->spa_dsl_pool, txg + 1);  in zio_write_gang_block()
3323 		dsl_sync_task_nowait(spa->spa_dsl_pool,  in zio_write_gang_block()
3332 	pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_write_gang_block()
3360 	blkptr_t *bp = zio->io_bp;  in zio_nop_write()
3361 	blkptr_t *bp_orig = &zio->io_bp_orig;  in zio_nop_write()
3362 	zio_prop_t *zp = &zio->io_prop;  in zio_nop_write()
3366 	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));  in zio_nop_write()
3367 	ASSERT(zp->zp_nopwrite);  in zio_nop_write()
3368 	ASSERT(!zp->zp_dedup);  in zio_nop_write()
3369 	ASSERT0P(zio->io_bp_override);  in zio_nop_write()
3385 	    zp->zp_copies != BP_GET_NDVAS(bp_orig))  in zio_nop_write()
3392 	if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) {  in zio_nop_write()
3393 		ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags &  in zio_nop_write()
3397 		ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF);  in zio_nop_write()
3398 		ASSERT3U(bp->blk_prop, ==, bp_orig->blk_prop);  in zio_nop_write()
3405 		spa_config_enter(zio->io_spa, SCL_VDEV, FTAG, RW_READER);  in zio_nop_write()
3407 			vdev_t *tvd = vdev_lookup_top(zio->io_spa,  in zio_nop_write()
3408 			    DVA_GET_VDEV(&bp_orig->blk_dva[d]));  in zio_nop_write()
3409 			if (tvd->vdev_ops == &vdev_indirect_ops) {  in zio_nop_write()
3410 				spa_config_exit(zio->io_spa, SCL_VDEV, FTAG);  in zio_nop_write()
3414 		spa_config_exit(zio->io_spa, SCL_VDEV, FTAG);  in zio_nop_write()
3417 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_nop_write()
3418 		zio->io_flags |= ZIO_FLAG_NOPWRITE;  in zio_nop_write()
3434 	bp = zio->io_bp;  in zio_brt_free()
3438 	    !brt_maybe_exists(zio->io_spa, bp)) {  in zio_brt_free()
3442 	if (!brt_entry_decref(zio->io_spa, bp)) {  in zio_brt_free()
3447 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_brt_free()
3461 	blkptr_t *bp = zio->io_bp;  in zio_ddt_child_read_done()
3463 	ddt_entry_t *dde = zio->io_private;  in zio_ddt_child_read_done()
3466 	mutex_enter(&pio->io_lock);  in zio_ddt_child_read_done()
3467 	ddt = ddt_select(zio->io_spa, bp);  in zio_ddt_child_read_done()
3469 	if (zio->io_error == 0) {  in zio_ddt_child_read_done()
3472 		ddt_phys_clear(dde->dde_phys, v);  in zio_ddt_child_read_done()
3475 	if (zio->io_error == 0 && dde->dde_io->dde_repair_abd == NULL)  in zio_ddt_child_read_done()
3476 		dde->dde_io->dde_repair_abd = zio->io_abd;  in zio_ddt_child_read_done()
3478 		abd_free(zio->io_abd);  in zio_ddt_child_read_done()
3479 	mutex_exit(&pio->io_lock);  in zio_ddt_child_read_done()
3485 	blkptr_t *bp = zio->io_bp;  in zio_ddt_read_start()
3488 	ASSERT(BP_GET_PSIZE(bp) == zio->io_size);  in zio_ddt_read_start()
3489 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_ddt_read_start()
3491 	if (zio->io_child_error[ZIO_CHILD_DDT]) {  in zio_ddt_read_start()
3492 		ddt_t *ddt = ddt_select(zio->io_spa, bp);  in zio_ddt_read_start()
3495 		ddt_univ_phys_t *ddp = dde->dde_phys;  in zio_ddt_read_start()
3498 		ASSERT0P(zio->io_vsd);  in zio_ddt_read_start()
3499 		zio->io_vsd = dde;  in zio_ddt_read_start()
3511 			ddt_bp_create(ddt->ddt_checksum, &dde->dde_key,  in zio_ddt_read_start()
3513 			zio_nowait(zio_read(zio, zio->io_spa, &blk,  in zio_ddt_read_start()
3514 			    abd_alloc_for_io(zio->io_size, B_TRUE),  in zio_ddt_read_start()
3515 			    zio->io_size, zio_ddt_child_read_done, dde,  in zio_ddt_read_start()
3516 			    zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) |  in zio_ddt_read_start()
3517 			    ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark));  in zio_ddt_read_start()
3522 	zio_nowait(zio_read(zio, zio->io_spa, bp,  in zio_ddt_read_start()
3523 	    zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority,  in zio_ddt_read_start()
3524 	    ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark));  in zio_ddt_read_start()
3532 	blkptr_t *bp = zio->io_bp;  in zio_ddt_read_done()
3539 	ASSERT(BP_GET_PSIZE(bp) == zio->io_size);  in zio_ddt_read_done()
3540 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_ddt_read_done()
3542 	if (zio->io_child_error[ZIO_CHILD_DDT]) {  in zio_ddt_read_done()
3543 		ddt_t *ddt = ddt_select(zio->io_spa, bp);  in zio_ddt_read_done()
3544 		ddt_entry_t *dde = zio->io_vsd;  in zio_ddt_read_done()
3546 			ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);  in zio_ddt_read_done()
3550 			zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1;  in zio_ddt_read_done()
3554 		if (dde->dde_io->dde_repair_abd != NULL) {  in zio_ddt_read_done()
3555 			abd_copy(zio->io_abd, dde->dde_io->dde_repair_abd,  in zio_ddt_read_done()
3556 			    zio->io_size);  in zio_ddt_read_done()
3557 			zio->io_child_error[ZIO_CHILD_DDT] = 0;  in zio_ddt_read_done()
3560 		zio->io_vsd = NULL;  in zio_ddt_read_done()
3563 	ASSERT0P(zio->io_vsd);  in zio_ddt_read_done()
3571 	spa_t *spa = zio->io_spa;  in zio_ddt_collision()
3572 	boolean_t do_raw = !!(zio->io_flags & ZIO_FLAG_RAW);  in zio_ddt_collision()
3574 	ASSERT(!(zio->io_bp_override && do_raw));  in zio_ddt_collision()
3578 	 * because when zio->io_bp is an override bp, we will not have  in zio_ddt_collision()
3592 		if (dde->dde_io == NULL)  in zio_ddt_collision()
3595 		zio_t *lio = dde->dde_io->dde_lead_zio[p];  in zio_ddt_collision()
3600 			return (lio->io_size != zio->io_size ||  in zio_ddt_collision()
3601 			    abd_cmp(zio->io_abd, lio->io_abd) != 0);  in zio_ddt_collision()
3603 		return (lio->io_orig_size != zio->io_orig_size ||  in zio_ddt_collision()
3604 		    abd_cmp(zio->io_orig_abd, lio->io_orig_abd) != 0);  in zio_ddt_collision()
3609 		uint64_t phys_birth = ddt_phys_birth(dde->dde_phys, v);  in zio_ddt_collision()
3612 			blkptr_t blk = *zio->io_bp;  in zio_ddt_collision()
3617 			ddt_bp_fill(dde->dde_phys, v, &blk, phys_birth);  in zio_ddt_collision()
3620 			if (psize != zio->io_size)  in zio_ddt_collision()
3630 			    ZIO_FLAG_RAW, &zio->io_bookmark));  in zio_ddt_collision()
3633 				if (abd_cmp(tmpabd, zio->io_abd) != 0)  in zio_ddt_collision()
3643 			blkptr_t blk = *zio->io_bp;  in zio_ddt_collision()
3646 			ddt_bp_fill(dde->dde_phys, v, &blk, phys_birth);  in zio_ddt_collision()
3648 			if (BP_GET_LSIZE(&blk) != zio->io_orig_size)  in zio_ddt_collision()
3656 			    &aflags, &zio->io_bookmark);  in zio_ddt_collision()
3659 				if (abd_cmp_buf(zio->io_orig_abd, abuf->b_data,  in zio_ddt_collision()
3660 				    zio->io_orig_size) != 0)  in zio_ddt_collision()
3676 	ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);  in zio_ddt_child_write_done()
3677 	ddt_entry_t *dde = zio->io_private;  in zio_ddt_child_write_done()
3682 	int p = DDT_PHYS_FOR_COPIES(ddt, zio->io_prop.zp_copies);  in zio_ddt_child_write_done()
3684 	ddt_univ_phys_t *ddp = dde->dde_phys;  in zio_ddt_child_write_done()
3689 	if (dde->dde_io->dde_lead_zio[p] == zio)  in zio_ddt_child_write_done()
3690 		dde->dde_io->dde_lead_zio[p] = NULL;  in zio_ddt_child_write_done()
3692 	ddt_univ_phys_t *orig = &dde->dde_io->dde_orig_phys;  in zio_ddt_child_write_done()
3694 	if (zio->io_error != 0) {  in zio_ddt_child_write_done()
3714 		if (!(pio->io_flags & ZIO_FLAG_DDT_CHILD))  in zio_ddt_child_write_done()
3723 	if (dde->dde_io->dde_lead_zio[p] == NULL)  in zio_ddt_child_write_done()
3734 	ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);  in zio_ddt_child_write_ready()
3735 	ddt_entry_t *dde = zio->io_private;  in zio_ddt_child_write_ready()
3740 	int p = DDT_PHYS_FOR_COPIES(ddt, zio->io_prop.zp_copies);  in zio_ddt_child_write_ready()
3743 	if (ddt_phys_is_gang(dde->dde_phys, v)) {  in zio_ddt_child_write_ready()
3744 		for (int i = 0; i < BP_GET_NDVAS(zio->io_bp); i++) {  in zio_ddt_child_write_ready()
3745 			dva_t *d = &zio->io_bp->blk_dva[i];  in zio_ddt_child_write_ready()
3746 			metaslab_group_alloc_decrement(zio->io_spa,  in zio_ddt_child_write_ready()
3747 			    DVA_GET_VDEV(d), zio->io_allocator,  in zio_ddt_child_write_ready()
3748 			    METASLAB_ASYNC_ALLOC, zio->io_size, zio);  in zio_ddt_child_write_ready()
3750 		zio->io_error = EAGAIN;  in zio_ddt_child_write_ready()
3753 	if (zio->io_error != 0)  in zio_ddt_child_write_ready()
3758 	ddt_phys_extend(dde->dde_phys, v, zio->io_bp);  in zio_ddt_child_write_ready()
3763 		if (!(pio->io_flags & ZIO_FLAG_DDT_CHILD))  in zio_ddt_child_write_ready()
3764 			ddt_bp_fill(dde->dde_phys, v, pio->io_bp, zio->io_txg);  in zio_ddt_child_write_ready()
3773 	spa_t *spa = zio->io_spa;  in zio_ddt_write()
3774 	blkptr_t *bp = zio->io_bp;  in zio_ddt_write()
3775 	uint64_t txg = zio->io_txg;  in zio_ddt_write()
3776 	zio_prop_t *zp = &zio->io_prop;  in zio_ddt_write()
3781 	ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum);  in zio_ddt_write()
3782 	ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override);  in zio_ddt_write()
3783 	ASSERT(!(zio->io_bp_override && (zio->io_flags & ZIO_FLAG_RAW)));  in zio_ddt_write()
3787 	 * place in the open-context. Direct I/O write can not attempt to  in zio_ddt_write()
3790 	ASSERT3B(zio->io_prop.zp_direct_write, ==, B_FALSE);  in zio_ddt_write()
3798 	IMPLY(zio->io_bp_override, !BP_GET_DEDUP(zio->io_bp_override));  in zio_ddt_write()
3802 		zp->zp_dedup = B_FALSE;  in zio_ddt_write()
3804 		if (zio->io_bp_override == NULL)  in zio_ddt_write()
3805 			zio->io_pipeline = ZIO_WRITE_PIPELINE;  in zio_ddt_write()
3810 	if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) {  in zio_ddt_write()
3815 		 * (And automatically e-mail a paper to Nature?)  in zio_ddt_write()
3817 		if (!(zio_checksum_table[zp->zp_checksum].ci_flags &  in zio_ddt_write()
3819 			zp->zp_checksum = spa_dedup_checksum(spa);  in zio_ddt_write()
3821 			zio->io_stage = ZIO_STAGE_OPEN;  in zio_ddt_write()
3824 			zp->zp_dedup = B_FALSE;  in zio_ddt_write()
3828 		zio->io_pipeline = ZIO_WRITE_PIPELINE;  in zio_ddt_write()
3833 	int p = DDT_PHYS_FOR_COPIES(ddt, zp->zp_copies);  in zio_ddt_write()
3835 	ddt_univ_phys_t *ddp = dde->dde_phys;  in zio_ddt_write()
3845 	 * - if the DDT entry has enough DVAs to satisfy the BP, we just copy  in zio_ddt_write()
3848 	 * - if the DDT entry has no DVAs (ie its brand new), then we have to  in zio_ddt_write()
3853 	 * - if the DDT entry has some DVAs, but too few, we have to issue the  in zio_ddt_write()
3875 	uint8_t need_dvas = zp->zp_copies;  in zio_ddt_write()
3883 	if (dde->dde_io == NULL || dde->dde_io->dde_lead_zio[p] == NULL) {  in zio_ddt_write()
3891 		if (zio->io_bp_override) {  in zio_ddt_write()
3893 			 * For a brand-new entry, all the work has been done  in zio_ddt_write()
3899 				ASSERT(BP_EQUAL(bp, zio->io_bp_override));  in zio_ddt_write()
3923 		 * then we can just use them as-is.  in zio_ddt_write()
3931 			if (zp->zp_rewrite) {  in zio_ddt_write()
3933 				    BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig);  in zio_ddt_write()
3935 				if (BP_EQUAL(bp, &zio->io_bp_orig)) {  in zio_ddt_write()
3937 					zio->io_flags |= ZIO_FLAG_NOPWRITE;  in zio_ddt_write()
3953 		need_dvas -= have_dvas;  in zio_ddt_write()
3956 		 * There's a write in-flight. If there's already enough DVAs on  in zio_ddt_write()
3958 		 * with, or the in-flight IO is between READY and DONE, and so  in zio_ddt_write()
3963 		if (zio->io_bp_override) {  in zio_ddt_write()
3996 			 * after the in-flight write has passed READY, and so  in zio_ddt_write()
4000 			zio_add_child(zio, dde->dde_io->dde_lead_zio[p]);  in zio_ddt_write()
4007 		 * the write in-flight and see how many DVAs it will have once  in zio_ddt_write()
4010 		 * The in-flight write has potentially had its copies request  in zio_ddt_write()
4022 		    zio_walk_parents(dde->dde_io->dde_lead_zio[p], &zl);  in zio_ddt_write()
4024 		parent_dvas = pio->io_prop.zp_copies;  in zio_ddt_write()
4027 			zio_add_child(zio, dde->dde_io->dde_lead_zio[p]);  in zio_ddt_write()
4036 		need_dvas -= parent_dvas;  in zio_ddt_write()
4040 		zp->zp_dedup = B_FALSE;  in zio_ddt_write()
4042 		zio->io_pipeline = ZIO_WRITE_PIPELINE;  in zio_ddt_write()
4060 	zio_t *cio = zio_write(zio, spa, txg, bp, zio->io_orig_abd,  in zio_ddt_write()
4061 	    zio->io_orig_size, zio->io_orig_size, &czp,  in zio_ddt_write()
4063 	    zio_ddt_child_write_done, dde, zio->io_priority,  in zio_ddt_write()
4064 	    ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark);  in zio_ddt_write()
4066 	zio_push_transform(cio, zio->io_abd, zio->io_size, 0, NULL);  in zio_ddt_write()
4073 	if (dde->dde_io->dde_lead_zio[p] == NULL) {  in zio_ddt_write()
4078 		ddt_phys_copy(&dde->dde_io->dde_orig_phys, dde->dde_phys, v);  in zio_ddt_write()
4084 		zio_add_child(cio, dde->dde_io->dde_lead_zio[p]);  in zio_ddt_write()
4086 	dde->dde_io->dde_lead_zio[p] = cio;  in zio_ddt_write()
4100 	spa_t *spa = zio->io_spa;  in zio_ddt_free()
4101 	blkptr_t *bp = zio->io_bp;  in zio_ddt_free()
4106 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_ddt_free()
4113 			ddt_phys_decref(dde->dde_phys, v);  in zio_ddt_free()
4124 		zio->io_pipeline |= ZIO_STAGE_DVA_FREE;  in zio_ddt_free()
4141 	ASSERT(MUTEX_HELD(&mca->mca_lock));  in zio_io_to_allocate()
4143 	zio = avl_first(&mca->mca_tree);  in zio_io_to_allocate()
4156 	if (!metaslab_class_throttle_reserve(zio->io_metaslab_class,  in zio_io_to_allocate()
4157 	    zio->io_allocator, zio->io_prop.zp_copies, zio->io_size,  in zio_io_to_allocate()
4161 	zio->io_flags |= ZIO_FLAG_ALLOC_THROTTLED;  in zio_io_to_allocate()
4163 	avl_remove(&mca->mca_tree, zio);  in zio_io_to_allocate()
4164 	ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);  in zio_io_to_allocate()
4166 	if (avl_is_empty(&mca->mca_tree))  in zio_io_to_allocate()
4174 	spa_t *spa = zio->io_spa;  in zio_dva_throttle()
4182 	mc = zio->io_metaslab_class;  in zio_dva_throttle()
4186 	if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE ||  in zio_dva_throttle()
4187 	    !mc->mc_alloc_throttle_enabled ||  in zio_dva_throttle()
4188 	    zio->io_child_type == ZIO_CHILD_GANG ||  in zio_dva_throttle()
4189 	    zio->io_flags & ZIO_FLAG_NODATA) {  in zio_dva_throttle()
4193 	ASSERT(zio->io_type == ZIO_TYPE_WRITE);  in zio_dva_throttle()
4195 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);  in zio_dva_throttle()
4196 	ASSERT3U(zio->io_queued_timestamp, >, 0);  in zio_dva_throttle()
4197 	ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);  in zio_dva_throttle()
4199 	zio->io_metaslab_class = mc;  in zio_dva_throttle()
4200 	metaslab_class_allocator_t *mca = &mc->mc_allocator[zio->io_allocator];  in zio_dva_throttle()
4201 	mutex_enter(&mca->mca_lock);  in zio_dva_throttle()
4202 	avl_add(&mca->mca_tree, zio);  in zio_dva_throttle()
4204 	mutex_exit(&mca->mca_lock);  in zio_dva_throttle()
4211 	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];  in zio_allocate_dispatch()
4216 		mutex_enter(&mca->mca_lock);  in zio_allocate_dispatch()
4218 		mutex_exit(&mca->mca_lock);  in zio_allocate_dispatch()
4222 		ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE);  in zio_allocate_dispatch()
4223 		ASSERT0(zio->io_error);  in zio_allocate_dispatch()
4231 	spa_t *spa = zio->io_spa;  in zio_dva_allocate()
4233 	blkptr_t *bp = zio->io_bp;  in zio_dva_allocate()
4237 	if (zio->io_gang_leader == NULL) {  in zio_dva_allocate()
4238 		ASSERT(zio->io_child_type > ZIO_CHILD_GANG);  in zio_dva_allocate()
4239 		zio->io_gang_leader = zio;  in zio_dva_allocate()
4241 	if (zio->io_flags & ZIO_FLAG_PREALLOCATED) {  in zio_dva_allocate()
4242 		ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_GANG);  in zio_dva_allocate()
4243 		memcpy(zio->io_bp->blk_dva, zio->io_bp_orig.blk_dva,  in zio_dva_allocate()
4245 		BP_SET_LOGICAL_BIRTH(zio->io_bp,  in zio_dva_allocate()
4246 		    BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig));  in zio_dva_allocate()
4247 		BP_SET_PHYSICAL_BIRTH(zio->io_bp,  in zio_dva_allocate()
4248 		    BP_GET_RAW_PHYSICAL_BIRTH(&zio->io_bp_orig));  in zio_dva_allocate()
4254 	ASSERT3U(zio->io_prop.zp_copies, >, 0);  in zio_dva_allocate()
4256 	ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa));  in zio_dva_allocate()
4257 	ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp));  in zio_dva_allocate()
4259 	if (zio->io_flags & ZIO_FLAG_GANG_CHILD)  in zio_dva_allocate()
4261 	if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE)  in zio_dva_allocate()
4267 	mc = zio->io_metaslab_class;  in zio_dva_allocate()
4270 		zio->io_metaslab_class = mc;  in zio_dva_allocate()
4287 	error = metaslab_alloc(spa, mc, zio->io_size, bp,  in zio_dva_allocate()
4288 	    zio->io_prop.zp_copies, zio->io_txg, NULL, flags,  in zio_dva_allocate()
4289 	    &zio->io_alloc_list, zio->io_allocator, zio);  in zio_dva_allocate()
4294 	 * frees that are in-flight.  We track the txg when this occurred and  in zio_dva_allocate()
4298 	if (error == ENOSPC && spa->spa_dedup_class_full_txg != zio->io_txg &&  in zio_dva_allocate()
4301 		spa->spa_dedup_class_full_txg = zio->io_txg;  in zio_dva_allocate()
4304 		    spa_name(spa), (u_longlong_t)zio->io_txg,  in zio_dva_allocate()
4306 		    (u_longlong_t)zio->io_size,  in zio_dva_allocate()
4319 		if (zio->io_flags & ZIO_FLAG_ALLOC_THROTTLED) {  in zio_dva_allocate()
4321 			    zio->io_allocator, zio->io_prop.zp_copies,  in zio_dva_allocate()
4322 			    zio->io_size)) {  in zio_dva_allocate()
4323 				zio_allocate_dispatch(zio->io_metaslab_class,  in zio_dva_allocate()
4324 				    zio->io_allocator);  in zio_dva_allocate()
4326 			zio->io_flags &= ~ZIO_FLAG_ALLOC_THROTTLED;  in zio_dva_allocate()
4335 			    zio, (u_longlong_t)zio->io_size, error);  in zio_dva_allocate()
4337 		zio->io_metaslab_class = mc = newmc;  in zio_dva_allocate()
4344 		if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE &&  in zio_dva_allocate()
4345 		    mc->mc_alloc_throttle_enabled &&  in zio_dva_allocate()
4346 		    zio->io_child_type != ZIO_CHILD_GANG &&  in zio_dva_allocate()
4347 		    !(zio->io_flags & ZIO_FLAG_NODATA)) {  in zio_dva_allocate()
4348 			zio->io_stage = ZIO_STAGE_DVA_THROTTLE >> 1;  in zio_dva_allocate()
4354 	if (error == ENOSPC && zio->io_size > spa->spa_min_alloc) {  in zio_dva_allocate()
4358 			    spa_name(spa), zio, (u_longlong_t)zio->io_size,  in zio_dva_allocate()
4371 			    spa_name(spa), zio, (u_longlong_t)zio->io_size,  in zio_dva_allocate()
4374 		zio->io_error = error;  in zio_dva_allocate()
4375 	} else if (zio->io_prop.zp_rewrite) {  in zio_dva_allocate()
4380 		uint64_t logical_birth = BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig);  in zio_dva_allocate()
4381 		ASSERT3U(logical_birth, <=, zio->io_txg);  in zio_dva_allocate()
4382 		BP_SET_BIRTH(zio->io_bp, logical_birth, zio->io_txg);  in zio_dva_allocate()
4383 		BP_SET_REWRITE(zio->io_bp, 1);  in zio_dva_allocate()
4392 	metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);  in zio_dva_free()
4402 	error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg);  in zio_dva_claim()
4404 		zio->io_error = error;  in zio_dva_claim()
4417 	ASSERT(BP_GET_BIRTH(bp) == zio->io_txg || BP_IS_HOLE(bp));  in zio_dva_unallocate()
4418 	ASSERT0P(zio->io_bp_override);  in zio_dva_unallocate()
4421 		metaslab_free(zio->io_spa, bp, BP_GET_BIRTH(bp), B_TRUE);  in zio_dva_unallocate()
4425 		for (int g = 0; g < gbh_nblkptrs(gn->gn_gangblocksize); g++) {  in zio_dva_unallocate()
4426 			zio_dva_unallocate(zio, gn->gn_child[g],  in zio_dva_unallocate()
4427 			    gbh_bp(gn->gn_gbh, g));  in zio_dva_unallocate()
4464 	int allocator = (uint_t)cityhash1(os->os_dsl_dataset->ds_object)  in zio_alloc_zil()
4465 	    % spa->spa_alloc_count;  in zio_alloc_zil()
4528 		if (os->os_encrypted) {  in zio_alloc_zil()
4567 	vdev_t *vd = zio->io_vd;  in zio_vdev_io_start()
4569 	spa_t *spa = zio->io_spa;  in zio_vdev_io_start()
4571 	zio->io_delay = 0;  in zio_vdev_io_start()
4573 	ASSERT0(zio->io_error);  in zio_vdev_io_start()
4574 	ASSERT0(zio->io_child_error[ZIO_CHILD_VDEV]);  in zio_vdev_io_start()
4577 		if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) {  in zio_vdev_io_start()
4580 			 * -> prune_candidates_sync() code path takes the  in zio_vdev_io_start()
4593 			if (spa->spa_active_ddt_prune)  in zio_vdev_io_start()
4608 	ASSERT3P(zio->io_logical, !=, zio);  in zio_vdev_io_start()
4609 	if (zio->io_type == ZIO_TYPE_WRITE) {  in zio_vdev_io_start()
4610 		ASSERT(spa->spa_trust_config);  in zio_vdev_io_start()
4616 		if (zio->io_vd->vdev_noalloc) {  in zio_vdev_io_start()
4617 			ASSERT(zio->io_flags &  in zio_vdev_io_start()
4623 	align = 1ULL << vd->vdev_top->vdev_ashift;  in zio_vdev_io_start()
4625 	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) &&  in zio_vdev_io_start()
4626 	    P2PHASE(zio->io_size, align) != 0) {  in zio_vdev_io_start()
4628 		uint64_t asize = P2ROUNDUP(zio->io_size, align);  in zio_vdev_io_start()
4629 		abd_t *abuf = abd_alloc_sametype(zio->io_abd, asize);  in zio_vdev_io_start()
4630 		ASSERT(vd == vd->vdev_top);  in zio_vdev_io_start()
4631 		if (zio->io_type == ZIO_TYPE_WRITE) {  in zio_vdev_io_start()
4632 			abd_copy(abuf, zio->io_abd, zio->io_size);  in zio_vdev_io_start()
4633 			abd_zero_off(abuf, zio->io_size, asize - zio->io_size);  in zio_vdev_io_start()
4642 	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) {  in zio_vdev_io_start()
4643 		ASSERT0(P2PHASE(zio->io_offset, align));  in zio_vdev_io_start()
4644 		ASSERT0(P2PHASE(zio->io_size, align));  in zio_vdev_io_start()
4648 		 * the device will perform a read-modify-write as necessary.  in zio_vdev_io_start()
4650 		ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE));  in zio_vdev_io_start()
4651 		ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE));  in zio_vdev_io_start()
4654 	VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa));  in zio_vdev_io_start()
4657 	 * If this is a repair I/O, and there's no self-healing involved --  in zio_vdev_io_start()
4658 	 * that is, we're just resilvering what we expect to resilver --  in zio_vdev_io_start()
4674 	 * read from C, then use the data to resilver A+B -- but we don't  in zio_vdev_io_start()
4675 	 * actually want to resilver B, just A. The top-level mirror has no  in zio_vdev_io_start()
4681 	 * + mirror, RAID-Z + replacing, etc.  in zio_vdev_io_start()
4696 	if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) &&  in zio_vdev_io_start()
4697 	    !(zio->io_flags & ZIO_FLAG_SELF_HEAL) &&  in zio_vdev_io_start()
4698 	    zio->io_txg != 0 &&	/* not a delegated i/o */  in zio_vdev_io_start()
4699 	    vd->vdev_ops != &vdev_indirect_ops &&  in zio_vdev_io_start()
4700 	    vd->vdev_top->vdev_ops != &vdev_draid_ops &&  in zio_vdev_io_start()
4701 	    !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {  in zio_vdev_io_start()
4702 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);  in zio_vdev_io_start()
4708 	 * Select the next best leaf I/O to process.  Distributed spares are  in zio_vdev_io_start()
4712 	if (vd->vdev_ops->vdev_op_leaf &&  in zio_vdev_io_start()
4713 	    vd->vdev_ops != &vdev_draid_spare_ops &&  in zio_vdev_io_start()
4714 	    (zio->io_type == ZIO_TYPE_READ ||  in zio_vdev_io_start()
4715 	    zio->io_type == ZIO_TYPE_WRITE ||  in zio_vdev_io_start()
4716 	    zio->io_type == ZIO_TYPE_TRIM)) {  in zio_vdev_io_start()
4722 			zio->io_error = SET_ERROR(ENXIO);  in zio_vdev_io_start()
4726 		zio->io_delay = gethrtime();  in zio_vdev_io_start()
4730 			 * "no-op" injections return success, but do no actual  in zio_vdev_io_start()
4738 	vd->vdev_ops->vdev_op_io_start(zio);  in zio_vdev_io_start()
4745 	vdev_t *vd = zio->io_vd;  in zio_vdev_io_done()
4746 	vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops;  in zio_vdev_io_done()
4753 	ASSERT(zio->io_type == ZIO_TYPE_READ ||  in zio_vdev_io_done()
4754 	    zio->io_type == ZIO_TYPE_WRITE ||  in zio_vdev_io_done()
4755 	    zio->io_type == ZIO_TYPE_FLUSH ||  in zio_vdev_io_done()
4756 	    zio->io_type == ZIO_TYPE_TRIM);  in zio_vdev_io_done()
4758 	if (zio->io_delay)  in zio_vdev_io_done()
4759 		zio->io_delay = gethrtime() - zio->io_delay;  in zio_vdev_io_done()
4761 	if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&  in zio_vdev_io_done()
4762 	    vd->vdev_ops != &vdev_draid_spare_ops) {  in zio_vdev_io_done()
4763 		if (zio->io_type != ZIO_TYPE_FLUSH)  in zio_vdev_io_done()
4766 		if (zio_injection_enabled && zio->io_error == 0)  in zio_vdev_io_done()
4767 			zio->io_error = zio_handle_device_injections(vd, zio,  in zio_vdev_io_done()
4770 		if (zio_injection_enabled && zio->io_error == 0)  in zio_vdev_io_done()
4771 			zio->io_error = zio_handle_label_injection(zio, EIO);  in zio_vdev_io_done()
4773 		if (zio->io_error && zio->io_type != ZIO_TYPE_FLUSH &&  in zio_vdev_io_done()
4774 		    zio->io_type != ZIO_TYPE_TRIM) {  in zio_vdev_io_done()
4776 				zio->io_error = SET_ERROR(ENXIO);  in zio_vdev_io_done()
4783 	ops->vdev_op_io_done(zio);  in zio_vdev_io_done()
4785 	if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)  in zio_vdev_io_done()
4793  * currently in-flight. This is used by the arc to upgrade priority in the
4806 	if (pio->io_vd != NULL && pio->io_vd->vdev_ops->vdev_op_leaf) {  in zio_change_priority()
4809 		pio->io_priority = priority;  in zio_change_priority()
4812 	mutex_enter(&pio->io_lock);  in zio_change_priority()
4817 	mutex_exit(&pio->io_lock);  in zio_change_priority()
4821  * For non-raidz ZIOs, we can just copy aside the bad data read from the
4829 	zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE);  in zio_vsd_default_cksum_finish()
4835 	void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size);  in zio_vsd_default_cksum_report()
4837 	abd_copy(abd, zio->io_abd, zio->io_size);  in zio_vsd_default_cksum_report()
4839 	zcr->zcr_cbinfo = zio->io_size;  in zio_vsd_default_cksum_report()
4840 	zcr->zcr_cbdata = abd;  in zio_vsd_default_cksum_report()
4841 	zcr->zcr_finish = zio_vsd_default_cksum_finish;  in zio_vsd_default_cksum_report()
4842 	zcr->zcr_free = zio_abd_free;  in zio_vsd_default_cksum_report()
4848 	vdev_t *vd = zio->io_vd;  in zio_vdev_io_assess()
4854 	if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))  in zio_vdev_io_assess()
4855 		spa_config_exit(zio->io_spa, SCL_ZIO, zio);  in zio_vdev_io_assess()
4857 	if (zio->io_vsd != NULL) {  in zio_vdev_io_assess()
4858 		zio->io_vsd_ops->vsd_free(zio);  in zio_vdev_io_assess()
4859 		zio->io_vsd = NULL;  in zio_vdev_io_assess()
4866 	if (zio->io_post & ZIO_POST_DIO_CHKSUM_ERR) {  in zio_vdev_io_assess()
4867 		if (zio->io_type == ZIO_TYPE_WRITE) {  in zio_vdev_io_assess()
4868 			ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_LOGICAL);  in zio_vdev_io_assess()
4869 			ASSERT3U(zio->io_error, ==, EIO);  in zio_vdev_io_assess()
4871 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_vdev_io_assess()
4875 	if (zio_injection_enabled && zio->io_error == 0)  in zio_vdev_io_assess()
4876 		zio->io_error = zio_handle_fault_injection(zio, EIO);  in zio_vdev_io_assess()
4884 	if (zio->io_error && vd == NULL &&  in zio_vdev_io_assess()
4885 	    !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) {  in zio_vdev_io_assess()
4886 		ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE));	/* not a leaf */  in zio_vdev_io_assess()
4887 		ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS));	/* not a leaf */  in zio_vdev_io_assess()
4888 		zio->io_error = 0;  in zio_vdev_io_assess()
4889 		zio->io_flags |= ZIO_FLAG_IO_RETRY | ZIO_FLAG_DONT_AGGREGATE;  in zio_vdev_io_assess()
4890 		zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1;  in zio_vdev_io_assess()
4900 	if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf &&  in zio_vdev_io_assess()
4902 		zio->io_error = SET_ERROR(ENXIO);  in zio_vdev_io_assess()
4905 	 * If we can't write to an interior vdev (mirror or RAID-Z),  in zio_vdev_io_assess()
4908 	if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE &&  in zio_vdev_io_assess()
4909 	    vd != NULL && !vd->vdev_ops->vdev_op_leaf) {  in zio_vdev_io_assess()
4913 		vd->vdev_cant_write = B_TRUE;  in zio_vdev_io_assess()
4922 	if (zio->io_error == ENOTSUP && zio->io_type == ZIO_TYPE_FLUSH &&  in zio_vdev_io_assess()
4924 		vd->vdev_nowritecache = B_TRUE;  in zio_vdev_io_assess()
4925 		zio->io_error = 0;  in zio_vdev_io_assess()
4928 	if (zio->io_error)  in zio_vdev_io_assess()
4929 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_vdev_io_assess()
4937 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);  in zio_vdev_io_reissue()
4938 	ASSERT0(zio->io_error);  in zio_vdev_io_reissue()
4940 	zio->io_stage >>= 1;  in zio_vdev_io_reissue()
4946 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE);  in zio_vdev_io_redone()
4948 	zio->io_stage >>= 1;  in zio_vdev_io_redone()
4954 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);  in zio_vdev_io_bypass()
4955 	ASSERT0(zio->io_error);  in zio_vdev_io_bypass()
4957 	zio->io_flags |= ZIO_FLAG_IO_BYPASS;  in zio_vdev_io_bypass()
4958 	zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1;  in zio_vdev_io_bypass()
4971  * lower-level encryption functions.
4976 	zio_prop_t *zp = &zio->io_prop;  in zio_encrypt()
4977 	spa_t *spa = zio->io_spa;  in zio_encrypt()
4978 	blkptr_t *bp = zio->io_bp;  in zio_encrypt()
4980 	uint64_t dsobj = zio->io_bookmark.zb_objset;  in zio_encrypt()
4981 	dmu_object_type_t ot = BP_GET_TYPE(bp);  in zio_encrypt()  local
4990 	if (zio->io_child_type == ZIO_CHILD_GANG)  in zio_encrypt()
4993 	/* only ZIL blocks are re-encrypted on rewrite */  in zio_encrypt()
4994 	if (!IO_IS_ALLOCATING(zio) && ot != DMU_OT_INTENT_LOG)  in zio_encrypt()
4997 	if (!(zp->zp_encrypt || BP_IS_ENCRYPTED(bp))) {  in zio_encrypt()
5003 	if (zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) {  in zio_encrypt()
5006 		BP_SET_BYTEORDER(bp, zp->zp_byteorder);  in zio_encrypt()
5007 		if (ot != DMU_OT_OBJSET)  in zio_encrypt()
5008 			zio_crypt_encode_mac_bp(bp, zp->zp_mac);  in zio_encrypt()
5011 		if (zp->zp_byteorder != ZFS_HOST_BYTEORDER &&  in zio_encrypt()
5012 		    ot == DMU_OT_DNODE) {  in zio_encrypt()
5017 			abd_copy_to_buf(bswap_buf, zio->io_abd, psize);  in zio_encrypt()
5018 			dmu_ot_byteswap[DMU_OT_BYTESWAP(ot)].ob_func(bswap_buf,  in zio_encrypt()
5025 		if (DMU_OT_IS_ENCRYPTED(ot))  in zio_encrypt()
5026 			zio_crypt_encode_params_bp(bp, zp->zp_salt, zp->zp_iv);  in zio_encrypt()
5030 	/* indirect blocks only maintain a cksum of the lower level MACs */  in zio_encrypt()
5034 		    zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp),  in zio_encrypt()
5041 	 * Objset blocks are a special case since they have 2 256-bit MACs  in zio_encrypt()
5044 	if (ot == DMU_OT_OBJSET) {  in zio_encrypt()
5045 		ASSERT0(DMU_OT_IS_ENCRYPTED(ot));  in zio_encrypt()
5049 		    zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp)));  in zio_encrypt()
5054 	if (!DMU_OT_IS_ENCRYPTED(ot)) {  in zio_encrypt()
5057 		    zio->io_abd, psize, mac));  in zio_encrypt()
5063 	 * Later passes of sync-to-convergence may decide to rewrite data  in zio_encrypt()
5071 	ASSERT(IO_IS_ALLOCATING(zio) || ot == DMU_OT_INTENT_LOG);  in zio_encrypt()
5072 	ASSERT(BP_GET_LEVEL(bp) == 0 || ot == DMU_OT_INTENT_LOG);  in zio_encrypt()
5084 	if (ot == DMU_OT_INTENT_LOG) {  in zio_encrypt()
5091 	VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark,  in zio_encrypt()
5093 	    salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt));  in zio_encrypt()
5096 	if (ot == DMU_OT_INTENT_LOG) {  in zio_encrypt()
5109 			ASSERT3U(ot, ==, DMU_OT_DNODE);  in zio_encrypt()
5127 	blkptr_t *bp = zio->io_bp;  in zio_checksum_generate()
5135 		checksum = zio->io_prop.zp_checksum;  in zio_checksum_generate()
5142 		if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) {  in zio_checksum_generate()
5150 	zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size);  in zio_checksum_generate()
5159 	blkptr_t *bp = zio->io_bp;  in zio_checksum_verify()
5162 	ASSERT(zio->io_vd != NULL);  in zio_checksum_verify()
5169 		if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)  in zio_checksum_verify()
5172 		ASSERT3U(zio->io_prop.zp_checksum, ==, ZIO_CHECKSUM_LABEL);  in zio_checksum_verify()
5175 	ASSERT0(zio->io_post & ZIO_POST_DIO_CHKSUM_ERR);  in zio_checksum_verify()
5176 	IMPLY(zio->io_flags & ZIO_FLAG_DIO_READ,  in zio_checksum_verify()
5177 	    !(zio->io_flags & ZIO_FLAG_SPECULATIVE));  in zio_checksum_verify()
5180 		zio->io_error = error;  in zio_checksum_verify()
5182 		    !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {  in zio_checksum_verify()
5183 			if (zio->io_flags & ZIO_FLAG_DIO_READ) {  in zio_checksum_verify()
5184 				zio->io_post |= ZIO_POST_DIO_CHKSUM_ERR;  in zio_checksum_verify()
5198 				if (pio->io_child_type == ZIO_CHILD_LOGICAL) {  in zio_checksum_verify()
5202 				mutex_enter(&zio->io_vd->vdev_stat_lock);  in zio_checksum_verify()
5203 				zio->io_vd->vdev_stat.vs_checksum_errors++;  in zio_checksum_verify()
5204 				mutex_exit(&zio->io_vd->vdev_stat_lock);  in zio_checksum_verify()
5205 				(void) zfs_ereport_start_checksum(zio->io_spa,  in zio_checksum_verify()
5206 				    zio->io_vd, &zio->io_bookmark, zio,  in zio_checksum_verify()
5207 				    zio->io_offset, zio->io_size, &info);  in zio_checksum_verify()
5221 	ASSERT3P(zio->io_vd, !=, NULL);  in zio_dio_checksum_verify()
5222 	ASSERT3P(zio->io_bp, !=, NULL);  in zio_dio_checksum_verify()
5223 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);  in zio_dio_checksum_verify()
5224 	ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);  in zio_dio_checksum_verify()
5225 	ASSERT3B(pio->io_prop.zp_direct_write, ==, B_TRUE);  in zio_dio_checksum_verify()
5226 	ASSERT3U(pio->io_child_type, ==, ZIO_CHILD_LOGICAL);  in zio_dio_checksum_verify()
5228 	if (zfs_vdev_direct_write_verify == 0 || zio->io_error != 0)  in zio_dio_checksum_verify()
5232 		zio->io_error = error;  in zio_dio_checksum_verify()
5234 			zio->io_post |= ZIO_POST_DIO_CHKSUM_ERR;  in zio_dio_checksum_verify()
5245  * Called by RAID-Z to ensure we don't compute the checksum twice.
5250 	zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;  in zio_checksum_verified()
5259 	ASSERT(zio->io_post & ZIO_POST_DIO_CHKSUM_ERR);  in zio_dio_chksum_verify_error_report()
5261 	if (zio->io_child_type == ZIO_CHILD_LOGICAL)  in zio_dio_chksum_verify_error_report()
5264 	mutex_enter(&zio->io_vd->vdev_stat_lock);  in zio_dio_chksum_verify_error_report()
5265 	zio->io_vd->vdev_stat.vs_dio_verify_errors++;  in zio_dio_chksum_verify_error_report()
5266 	mutex_exit(&zio->io_vd->vdev_stat_lock);  in zio_dio_chksum_verify_error_report()
5267 	if (zio->io_type == ZIO_TYPE_WRITE) {  in zio_dio_chksum_verify_error_report()
5271 		zio->io_error = SET_ERROR(EIO);  in zio_dio_chksum_verify_error_report()
5276 		    zio->io_spa,  zio->io_vd, &zio->io_bookmark, zio, 0);  in zio_dio_chksum_verify_error_report()
5282 		    zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0);  in zio_dio_chksum_verify_error_report()
5289  * An error of 0 indicates success.  ENXIO indicates whole-device failure,
5320 	blkptr_t *bp = zio->io_bp;  in zio_ready()
5331 		if (target != 0 && zio->io_target_timestamp == 0) {  in zio_ready()
5332 			zio->io_stage >>= 1;  in zio_ready()
5333 			zio->io_target_timestamp = target;  in zio_ready()
5339 	if (zio->io_ready) {  in zio_ready()
5341 		ASSERT(BP_GET_BIRTH(bp) == zio->io_txg ||  in zio_ready()
5342 		    BP_IS_HOLE(bp) || (zio->io_flags & ZIO_FLAG_NOPWRITE));  in zio_ready()
5343 		ASSERT0(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY]);  in zio_ready()
5345 		zio->io_ready(zio);  in zio_ready()
5349 	if (bp != NULL && bp != &zio->io_bp_copy)  in zio_ready()
5350 		zio->io_bp_copy = *bp;  in zio_ready()
5353 	if (zio->io_error != 0) {  in zio_ready()
5354 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;  in zio_ready()
5356 		if (zio->io_flags & ZIO_FLAG_ALLOC_THROTTLED) {  in zio_ready()
5358 			ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);  in zio_ready()
5359 			ASSERT(zio->io_metaslab_class != NULL);  in zio_ready()
5367 			    zio->io_metaslab_class, zio->io_allocator,  in zio_ready()
5368 			    zio->io_prop.zp_copies, zio->io_size)) {  in zio_ready()
5369 				zio_allocate_dispatch(zio->io_metaslab_class,  in zio_ready()
5370 				    zio->io_allocator);  in zio_ready()
5375 	mutex_enter(&zio->io_lock);  in zio_ready()
5376 	zio->io_state[ZIO_WAIT_READY] = 1;  in zio_ready()
5378 	mutex_exit(&zio->io_lock);  in zio_ready()
5392 	if (zio->io_flags & ZIO_FLAG_NODATA) {  in zio_ready()
5394 			zio->io_flags &= ~ZIO_FLAG_NODATA;  in zio_ready()
5396 			ASSERT((uintptr_t)zio->io_abd < SPA_MAXBLOCKSIZE);  in zio_ready()
5397 			zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;  in zio_ready()
5402 	    zio->io_spa->spa_syncing_txg == zio->io_txg)  in zio_ready()
5415 	vdev_t *vd = zio->io_vd;  in zio_dva_throttle_done()
5418 	uint64_t size = pio->io_size;  in zio_dva_throttle_done()
5420 	ASSERT3P(zio->io_bp, !=, NULL);  in zio_dva_throttle_done()
5421 	ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);  in zio_dva_throttle_done()
5422 	ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE);  in zio_dva_throttle_done()
5423 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);  in zio_dva_throttle_done()
5425 	ASSERT3P(vd, ==, vd->vdev_top);  in zio_dva_throttle_done()
5426 	ASSERT(zio_injection_enabled || !(zio->io_flags & ZIO_FLAG_IO_RETRY));  in zio_dva_throttle_done()
5427 	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));  in zio_dva_throttle_done()
5428 	ASSERT(zio->io_flags & ZIO_FLAG_ALLOC_THROTTLED);  in zio_dva_throttle_done()
5431 	 * Parents of gang children can have two flavors -- ones that allocated  in zio_dva_throttle_done()
5436 	if (pio->io_child_type == ZIO_CHILD_GANG &&  in zio_dva_throttle_done()
5437 	    (pio->io_flags & ZIO_FLAG_IO_REWRITE)) {  in zio_dva_throttle_done()
5442 	ASSERT(IO_IS_ALLOCATING(pio) || (pio->io_child_type == ZIO_CHILD_GANG &&  in zio_dva_throttle_done()
5443 	    (pio->io_flags & ZIO_FLAG_IO_REWRITE)));  in zio_dva_throttle_done()
5445 	ASSERT3P(zio, !=, zio->io_logical);  in zio_dva_throttle_done()
5446 	ASSERT(zio->io_logical != NULL);  in zio_dva_throttle_done()
5447 	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));  in zio_dva_throttle_done()
5448 	ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE);  in zio_dva_throttle_done()
5449 	ASSERT(zio->io_metaslab_class != NULL);  in zio_dva_throttle_done()
5450 	ASSERT(zio->io_metaslab_class->mc_alloc_throttle_enabled);  in zio_dva_throttle_done()
5452 	metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id,  in zio_dva_throttle_done()
5453 	    pio->io_allocator, flags, size, tag);  in zio_dva_throttle_done()
5455 	if (metaslab_class_throttle_unreserve(pio->io_metaslab_class,  in zio_dva_throttle_done()
5456 	    pio->io_allocator, 1, pio->io_size)) {  in zio_dva_throttle_done()
5457 		zio_allocate_dispatch(zio->io_metaslab_class,  in zio_dva_throttle_done()
5458 		    pio->io_allocator);  in zio_dva_throttle_done()
5469 	const uint64_t psize = zio->io_size;  in zio_done()
5487 	if (zio->io_flags & ZIO_FLAG_ALLOC_THROTTLED &&  in zio_done()
5488 	    zio->io_child_type == ZIO_CHILD_VDEV)  in zio_done()
5493 			ASSERT0(zio->io_children[c][w]);  in zio_done()
5495 	if (zio->io_bp != NULL && !BP_IS_EMBEDDED(zio->io_bp)) {  in zio_done()
5496 		ASSERT(memcmp(zio->io_bp, &zio->io_bp_copy,  in zio_done()
5498 		    (zio->io_bp == zio_unique_parent(zio)->io_bp));  in zio_done()
5499 		if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(zio->io_bp) &&  in zio_done()
5500 		    zio->io_bp_override == NULL &&  in zio_done()
5501 		    !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {  in zio_done()
5502 			ASSERT3U(zio->io_prop.zp_copies, <=,  in zio_done()
5503 			    BP_GET_NDVAS(zio->io_bp));  in zio_done()
5504 			ASSERT(BP_COUNT_GANG(zio->io_bp) == 0 ||  in zio_done()
5505 			    (BP_COUNT_GANG(zio->io_bp) ==  in zio_done()
5506 			    BP_GET_NDVAS(zio->io_bp)));  in zio_done()
5508 		if (zio->io_flags & ZIO_FLAG_NOPWRITE)  in zio_done()
5509 			VERIFY(BP_EQUAL(zio->io_bp, &zio->io_bp_orig));  in zio_done()
5523 	if (zio->io_error == 0) {  in zio_done()
5524 		while (zio->io_cksum_report != NULL) {  in zio_done()
5525 			zio_cksum_report_t *zcr = zio->io_cksum_report;  in zio_done()
5526 			uint64_t align = zcr->zcr_align;  in zio_done()
5528 			abd_t *adata = zio->io_abd;  in zio_done()
5532 				abd_copy(adata, zio->io_abd, psize);  in zio_done()
5533 				abd_zero_off(adata, psize, asize - psize);  in zio_done()
5536 			zio->io_cksum_report = zcr->zcr_next;  in zio_done()
5537 			zcr->zcr_next = NULL;  in zio_done()
5538 			zcr->zcr_finish(zcr, adata);  in zio_done()
5546 	zio_pop_transforms(zio);	/* note: may set zio->io_error */  in zio_done()
5555 	if (zio->io_delay >= MSEC2NSEC(zio_slow_io_ms)) {  in zio_done()
5556 		if (zio->io_vd != NULL && !vdev_is_dead(zio->io_vd)) {  in zio_done()
5567 			    zio->io_spa, zio->io_vd, zio)) {  in zio_done()
5568 				mutex_enter(&zio->io_vd->vdev_stat_lock);  in zio_done()
5569 				zio->io_vd->vdev_stat.vs_slow_ios++;  in zio_done()
5570 				mutex_exit(&zio->io_vd->vdev_stat_lock);  in zio_done()
5573 				    zio->io_spa, zio->io_vd, &zio->io_bookmark,  in zio_done()
5579 	if (zio->io_error) {  in zio_done()
5583 		 * at the block level.  We ignore these errors if the  in zio_done()
5586 		if (zio->io_error != ECKSUM && zio->io_vd != NULL &&  in zio_done()
5587 		    !vdev_is_dead(zio->io_vd) &&  in zio_done()
5588 		    !(zio->io_post & ZIO_POST_DIO_CHKSUM_ERR)) {  in zio_done()
5590 			    zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0);  in zio_done()
5592 				mutex_enter(&zio->io_vd->vdev_stat_lock);  in zio_done()
5593 				if (zio->io_type == ZIO_TYPE_READ)  in zio_done()
5594 					zio->io_vd->vdev_stat.vs_read_errors++;  in zio_done()
5595 				else if (zio->io_type == ZIO_TYPE_WRITE)  in zio_done()
5596 					zio->io_vd->vdev_stat.vs_write_errors++;  in zio_done()
5597 				mutex_exit(&zio->io_vd->vdev_stat_lock);  in zio_done()
5601 		if ((zio->io_error == EIO || !(zio->io_flags &  in zio_done()
5603 		    !(zio->io_post & ZIO_POST_DIO_CHKSUM_ERR) &&  in zio_done()
5604 		    zio == zio->io_logical) {  in zio_done()
5609 			spa_log_error(zio->io_spa, &zio->io_bookmark,  in zio_done()
5610 			    BP_GET_PHYSICAL_BIRTH(zio->io_bp));  in zio_done()
5612 			    zio->io_spa, NULL, &zio->io_bookmark, zio, 0);  in zio_done()
5616 	if (zio->io_error && zio == zio->io_logical) {  in zio_done()
5619 		 * A DDT child tried to create a mixed gang/non-gang BP. We're  in zio_done()
5620 		 * going to have to just retry as a non-dedup IO.  in zio_done()
5622 		if (zio->io_error == EAGAIN && IO_IS_ALLOCATING(zio) &&  in zio_done()
5623 		    zio->io_prop.zp_dedup) {  in zio_done()
5624 			zio->io_post |= ZIO_POST_REEXECUTE;  in zio_done()
5625 			zio->io_prop.zp_dedup = B_FALSE;  in zio_done()
5631 		ASSERT(zio->io_vd == NULL && zio->io_bp != NULL);  in zio_done()
5632 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_done()
5635 		    !(zio->io_flags & ZIO_FLAG_CANFAIL) &&  in zio_done()
5636 		    !(zio->io_post & ZIO_POST_DIO_CHKSUM_ERR)) {  in zio_done()
5637 			if (zio->io_error != ENOSPC)  in zio_done()
5638 				zio->io_post |= ZIO_POST_REEXECUTE;  in zio_done()
5640 				zio->io_post |= ZIO_POST_SUSPEND;  in zio_done()
5643 		if ((zio->io_type == ZIO_TYPE_READ ||  in zio_done()
5644 		    zio->io_type == ZIO_TYPE_FREE) &&  in zio_done()
5645 		    !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) &&  in zio_done()
5646 		    zio->io_error == ENXIO &&  in zio_done()
5647 		    spa_load_state(zio->io_spa) == SPA_LOAD_NONE &&  in zio_done()
5648 		    spa_get_failmode(zio->io_spa) != ZIO_FAILURE_MODE_CONTINUE)  in zio_done()
5649 			zio->io_post |= ZIO_POST_SUSPEND;  in zio_done()
5651 		if (!(zio->io_flags & ZIO_FLAG_CANFAIL) &&  in zio_done()
5652 		    !(zio->io_post & (ZIO_POST_REEXECUTE|ZIO_POST_SUSPEND)))  in zio_done()
5653 			zio->io_post |= ZIO_POST_SUSPEND;  in zio_done()
5672 	if ((zio->io_error ||  in zio_done()
5673 	    (zio->io_post & (ZIO_POST_REEXECUTE|ZIO_POST_SUSPEND))) &&  in zio_done()
5674 	    IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&  in zio_done()
5675 	    !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)))  in zio_done()
5676 		zio_dva_unallocate(zio, zio->io_gang_tree, zio->io_bp);  in zio_done()
5678 	zio_gang_tree_free(&zio->io_gang_tree);  in zio_done()
5683 	if ((zio->io_flags & ZIO_FLAG_GODFATHER) &&  in zio_done()
5684 	    (zio->io_post & ZIO_POST_SUSPEND))  in zio_done()
5685 		zio->io_post &= ~ZIO_POST_SUSPEND;  in zio_done()
5687 	if (zio->io_post & (ZIO_POST_REEXECUTE|ZIO_POST_SUSPEND)) {  in zio_done()
5693 		ASSERT0(zio->io_post & ZIO_POST_DIO_CHKSUM_ERR);  in zio_done()
5698 		 * Reexecute is top-down.  When an i/o fails, if it's not  in zio_done()
5708 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);  in zio_done()
5710 		zio->io_gang_leader = NULL;  in zio_done()
5712 		mutex_enter(&zio->io_lock);  in zio_done()
5713 		zio->io_state[ZIO_WAIT_DONE] = 1;  in zio_done()
5714 		mutex_exit(&zio->io_lock);  in zio_done()
5729 			if ((pio->io_flags & ZIO_FLAG_GODFATHER) &&  in zio_done()
5730 			    (zio->io_post & ZIO_POST_SUSPEND)) {  in zio_done()
5747 			ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));  in zio_done()
5748 			zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE;  in zio_done()
5754 		} else if (zio->io_post & ZIO_POST_SUSPEND) {  in zio_done()
5759 			zio_suspend(zio->io_spa, zio, ZIO_SUSPEND_IOERR);  in zio_done()
5761 			ASSERT(zio->io_post & ZIO_POST_REEXECUTE);  in zio_done()
5764 			 * Hand it off to the otherwise-unused claim taskq.  in zio_done()
5766 			spa_taskq_dispatch(zio->io_spa,  in zio_done()
5773 	ASSERT(list_is_empty(&zio->io_child_list));  in zio_done()
5774 	ASSERT0(zio->io_post & ZIO_POST_REEXECUTE);  in zio_done()
5775 	ASSERT0(zio->io_post & ZIO_POST_SUSPEND);  in zio_done()
5776 	ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL));  in zio_done()
5781 	while (zio->io_cksum_report != NULL) {  in zio_done()
5782 		zio_cksum_report_t *zcr = zio->io_cksum_report;  in zio_done()
5783 		zio->io_cksum_report = zcr->zcr_next;  in zio_done()
5784 		zcr->zcr_next = NULL;  in zio_done()
5785 		zcr->zcr_finish(zcr, NULL);  in zio_done()
5794 	if (zio->io_done)  in zio_done()
5795 		zio->io_done(zio);  in zio_done()
5797 	mutex_enter(&zio->io_lock);  in zio_done()
5798 	zio->io_state[ZIO_WAIT_DONE] = 1;  in zio_done()
5799 	mutex_exit(&zio->io_lock);  in zio_done()
5814 	if (zio->io_waiter != NULL) {  in zio_done()
5815 		mutex_enter(&zio->io_lock);  in zio_done()
5816 		zio->io_executor = NULL;  in zio_done()
5817 		cv_broadcast(&zio->io_cv);  in zio_done()
5818 		mutex_exit(&zio->io_lock);  in zio_done()
5866  * pre-order traversal of the object tree.
5868  * This is simple in every case aside from the meta-dnode object. For all other
5872  * canonical representation so we can compare meta-dnode bookmarks to
5873  * non-meta-dnode bookmarks.
5876  * zbookmarks outside of the meta-dnode use their own object and level, and
5877  * calculate the level 0 equivalent (the first L0 blkid that is contained in the
5879  * (the number of L0 blocks contained within one block at their level).
5880  * zbookmarks inside the meta-dnode calculate their object equivalent
5882  * level + 1<<31 (any value larger than a level could ever be) for their level.
5885  * compare appropriately to other bookmarks in the meta-dnode.
5894 	 * normal-object equivalents.  in zbookmark_compare()
5900 	if (zb1->zb_object == zb2->zb_object &&  in zbookmark_compare()
5901 	    zb1->zb_level == zb2->zb_level &&  in zbookmark_compare()
5902 	    zb1->zb_blkid == zb2->zb_blkid)  in zbookmark_compare()
5905 	IMPLY(zb1->zb_level > 0, ibs1 >= SPA_MINBLOCKSHIFT);  in zbookmark_compare()
5906 	IMPLY(zb2->zb_level > 0, ibs2 >= SPA_MINBLOCKSHIFT);  in zbookmark_compare()
5911 	zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level);  in zbookmark_compare()
5912 	zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level);  in zbookmark_compare()
5914 	if (zb1->zb_object == DMU_META_DNODE_OBJECT) {  in zbookmark_compare()
5915 		zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));  in zbookmark_compare()
5917 		zb1level = zb1->zb_level + COMPARE_META_LEVEL;  in zbookmark_compare()
5919 		zb1obj = zb1->zb_object;  in zbookmark_compare()
5920 		zb1level = zb1->zb_level;  in zbookmark_compare()
5923 	if (zb2->zb_object == DMU_META_DNODE_OBJECT) {  in zbookmark_compare()
5924 		zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));  in zbookmark_compare()
5926 		zb2level = zb2->zb_level + COMPARE_META_LEVEL;  in zbookmark_compare()
5928 		zb2obj = zb2->zb_object;  in zbookmark_compare()
5929 		zb2level = zb2->zb_level;  in zbookmark_compare()
5934 		return (zb1obj < zb2obj ? -1 : 1);  in zbookmark_compare()
5936 		return (zb1L0 < zb2L0 ? -1 : 1);  in zbookmark_compare()
5938 		return (zb1level > zb2level ? -1 : 1);  in zbookmark_compare()
5941 	 * and level, but different blkids, if the block sizes are not the same.  in zbookmark_compare()
5963 	ASSERT0(last_block->zb_level);  in zbookmark_subtree_completed()
5970 	 * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the  in zbookmark_subtree_completed()
5972 	 * the bookmark refers to a block in the meta-dnode.  Since we don't  in zbookmark_subtree_completed()
5977 	 * level 0.  The indirect block size is only used to calculate the span  in zbookmark_subtree_completed()
5978 	 * of the bookmark, but since the bookmark must be level 0, the span is  in zbookmark_subtree_completed()
5984 	return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,  in zbookmark_subtree_completed()
5985 	    1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb,  in zbookmark_subtree_completed()
5997 	ASSERT0(last_block->zb_level);  in zbookmark_subtree_tbd()
6000 	return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,  in zbookmark_subtree_tbd()
6001 	    1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, subtree_root,  in zbookmark_subtree_tbd()