1334 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)  in arc_cksum_is_equal()  argument
1336 	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));  in arc_cksum_is_equal()
1337 	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));  in arc_cksum_is_equal()
1352 	return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,  in arc_cksum_is_equal()
1353 	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,  in arc_cksum_is_equal()
1354 	    zio->io_offset, NULL) == 0);  in arc_cksum_is_equal()
2713 static void l2arc_log_blk_fetch_abort(zio_t *zio);
2733 static void l2arc_blk_fetch_done(zio_t *zio);
5297 arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,  in arc_bcopy_func()  argument
5309 arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,  in arc_getbuf_func()  argument
5315 		ASSERT(zio == NULL || zio->io_error != 0);  in arc_getbuf_func()
5318 		ASSERT(zio == NULL || zio->io_error == 0);  in arc_getbuf_func()
5385 arc_read_done(zio_t *zio)  in arc_read_done()  argument
5387 	blkptr_t	*bp = zio->io_bp;  in arc_read_done()
5388 	arc_buf_hdr_t	*hdr = zio->io_private;  in arc_read_done()
5403 		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));  in arc_read_done()
5405 		    BP_IDENTITY(zio->io_bp)->dva_word[0]);  in arc_read_done()
5407 		    BP_IDENTITY(zio->io_bp)->dva_word[1]);  in arc_read_done()
5409 		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,  in arc_read_done()
5413 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||  in arc_read_done()
5420 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;  in arc_read_done()
5427 			tmpbuf = abd_borrow_buf_copy(zio->io_abd,  in arc_read_done()
5431 			abd_return_buf(zio->io_abd, tmpbuf,  in arc_read_done()
5438 	if (zio->io_error == 0) {  in arc_read_done()
5440 		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {  in arc_read_done()
5441 			if (BP_GET_LEVEL(zio->io_bp) > 0) {  in arc_read_done()
5445 				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));  in arc_read_done()
5457 	if (hash_lock && zio->io_error == 0 &&  in arc_read_done()
5481 		if (zio->io_error != 0)  in arc_read_done()
5484 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,  in arc_read_done()
5493 		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||  in arc_read_done()
5503 			if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {  in arc_read_done()
5504 				spa_log_error(zio->io_spa, &acb->acb_zb);  in arc_read_done()
5507 				    zio->io_spa, NULL, &acb->acb_zb, zio, 0, 0);  in arc_read_done()
5526 			zio->io_error = error;  in arc_read_done()
5547 	if (zio->io_error == 0) {  in arc_read_done()
5548 		arc_hdr_verify(hdr, zio->io_bp);  in arc_read_done()
5582 			if (zio->io_error != 0 && acb->acb_buf != NULL) {  in arc_read_done()
5591 			acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,  in arc_read_done()
5596 			acb->acb_zio_dummy->io_error = zio->io_error;  in arc_read_done()
6402 arc_write_ready(zio_t *zio)  in arc_write_ready()  argument
6404 	arc_write_callback_t *callback = zio->io_private;  in arc_write_ready()
6407 	blkptr_t *bp = zio->io_bp;  in arc_write_ready()
6419 	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {  in arc_write_ready()
6438 	callback->awcb_ready(zio, buf, callback->awcb_private);  in arc_write_ready()
6441 		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);  in arc_write_ready()
6465 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;  in arc_write_ready()
6498 	if (zio->io_error != 0 || psize == 0)  in arc_write_ready()
6519 		abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);  in arc_write_ready()
6530 			abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);  in arc_write_ready()
6535 			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);  in arc_write_ready()
6537 			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));  in arc_write_ready()
6543 		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));  in arc_write_ready()
6544 		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));  in arc_write_ready()
6554 arc_write_children_ready(zio_t *zio)  in arc_write_children_ready()  argument
6556 	arc_write_callback_t *callback = zio->io_private;  in arc_write_children_ready()
6559 	callback->awcb_children_ready(zio, buf, callback->awcb_private);  in arc_write_children_ready()
6567 arc_write_physdone(zio_t *zio)  in arc_write_physdone()  argument
6569 	arc_write_callback_t *cb = zio->io_private;  in arc_write_physdone()
6571 		cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);  in arc_write_physdone()
6575 arc_write_done(zio_t *zio)  in arc_write_done()  argument
6577 	arc_write_callback_t *callback = zio->io_private;  in arc_write_done()
6583 	if (zio->io_error == 0) {  in arc_write_done()
6584 		arc_hdr_verify(hdr, zio->io_bp);  in arc_write_done()
6586 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {  in arc_write_done()
6589 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);  in arc_write_done()
6590 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);  in arc_write_done()
6606 		ASSERT3U(zio->io_error, ==, 0);  in arc_write_done()
6617 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {  in arc_write_done()
6618 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))  in arc_write_done()
6628 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {  in arc_write_done()
6630 				ASSERT(zio->io_prop.zp_nopwrite);  in arc_write_done()
6631 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))  in arc_write_done()
6638 				ASSERT(BP_GET_DEDUP(zio->io_bp));  in arc_write_done()
6639 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);  in arc_write_done()
6652 	callback->awcb_done(zio, buf, callback->awcb_private);  in arc_write_done()
6654 	abd_put(zio->io_abd);  in arc_write_done()
6667 	zio_t *zio;  in arc_write()  local
6741 	zio = zio_write(pio, spa, txg, bp,  in arc_write()
6748 	return (zio);  in arc_write()
7693 l2arc_write_done(zio_t *zio)  in l2arc_write_done()  argument
7705 	cb = zio->io_private;  in l2arc_write_done()
7714 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,  in l2arc_write_done()
7768 		if (zio->io_error != 0) {  in l2arc_write_done()
7802 		if (zio->io_error != 0) {  in l2arc_write_done()
7823 	if (zio->io_error != 0) {  in l2arc_write_done()
7869 l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)  in l2arc_untransform()  argument
7872 	spa_t *spa = zio->io_spa;  in l2arc_untransform()
7874 	blkptr_t *bp = zio->io_bp;  in l2arc_untransform()
7919 			zio->io_abd = eabd;  in l2arc_untransform()
7949 		zio->io_abd = cabd;  in l2arc_untransform()
7950 		zio->io_size = HDR_GET_LSIZE(hdr);  in l2arc_untransform()
7965 l2arc_read_done(zio_t *zio)  in l2arc_read_done()  argument
7968 	l2arc_read_callback_t *cb = zio->io_private;  in l2arc_read_done()
7975 	ASSERT3P(zio->io_vd, !=, NULL);  in l2arc_read_done()
7976 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);  in l2arc_read_done()
7978 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);  in l2arc_read_done()
7993 		ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);  in l2arc_read_done()
7994 		if (zio->io_error == 0) {  in l2arc_read_done()
8016 		zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);  in l2arc_read_done()
8020 			zio->io_abd = zio->io_orig_abd =  in l2arc_read_done()
8024 			zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;  in l2arc_read_done()
8028 	ASSERT3P(zio->io_abd, !=, NULL);  in l2arc_read_done()
8033 	ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||  in l2arc_read_done()
8034 	    (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));  in l2arc_read_done()
8035 	zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/  in l2arc_read_done()
8036 	zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/  in l2arc_read_done()
8038 	valid_cksum = arc_cksum_is_equal(hdr, zio);  in l2arc_read_done()
8046 		tfm_error = l2arc_untransform(zio, cb);  in l2arc_read_done()
8048 	if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&  in l2arc_read_done()
8051 		zio->io_private = hdr;  in l2arc_read_done()
8052 		arc_read_done(zio);  in l2arc_read_done()
8058 		if (zio->io_error != 0) {  in l2arc_read_done()
8061 			zio->io_error = SET_ERROR(EIO);  in l2arc_read_done()
8071 		if (zio->io_waiter == NULL) {  in l2arc_read_done()
8072 			zio_t *pio = zio_unique_parent(zio);  in l2arc_read_done()
8078 			zio = zio_read(pio, zio->io_spa, zio->io_bp,  in l2arc_read_done()
8079 			    abd, zio->io_size, arc_read_done,  in l2arc_read_done()
8080 			    hdr, zio->io_priority, cb->l2rcb_flags,  in l2arc_read_done()
8090 				acb->acb_zio_head = zio;  in l2arc_read_done()
8093 			zio_nowait(zio);  in l2arc_read_done()
8478 l2arc_blk_fetch_done(zio_t *zio)  in l2arc_blk_fetch_done()  argument
8482 	cb = zio->io_private;  in l2arc_blk_fetch_done()
9761 l2arc_log_blk_fetch_abort(zio_t *zio)  in l2arc_log_blk_fetch_abort()  argument
9763 	(void) zio_wait(zio);  in l2arc_log_blk_fetch_abort()