704 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)  argument
705 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)  argument
706 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)  argument
707 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)  argument
708 #define	HDR_PRESCIENT_PREFETCH(hdr)	\  argument
709 	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
710 #define	HDR_COMPRESSION_ENABLED(hdr)	\  argument
711 	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
713 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)  argument
714 #define	HDR_UNCACHED(hdr)	((hdr)->b_flags & ARC_FLAG_UNCACHED)  argument
715 #define	HDR_L2_READING(hdr)	\  argument
716 	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
717 	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
718 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)  argument
719 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)  argument
720 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)  argument
721 #define	HDR_PROTECTED(hdr)	((hdr)->b_flags & ARC_FLAG_PROTECTED)  argument
722 #define	HDR_NOAUTH(hdr)		((hdr)->b_flags & ARC_FLAG_NOAUTH)  argument
723 #define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)  argument
725 #define	HDR_ISTYPE_METADATA(hdr)	\  argument
726 	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
727 #define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))  argument
729 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)  argument
730 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)  argument
731 #define	HDR_HAS_RABD(hdr)	\  argument
732 	(HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) &&	\
733 	(hdr)->b_crypt_hdr.b_rabd != NULL)
734 #define	HDR_ENCRYPTED(hdr)	\  argument
735 	(HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
736 #define	HDR_AUTHENTICATED(hdr)	\  argument
737 	(HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
742 #define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \  argument
744 #define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \  argument
775 #define	HDR_LOCK(hdr) \  argument
776 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
891 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
902 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
903 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
908 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
913 #define	l2arc_hdr_arcstats_increment(hdr) \  argument
914 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
915 #define	l2arc_hdr_arcstats_decrement(hdr) \  argument
916 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
917 #define	l2arc_hdr_arcstats_increment_state(hdr) \  argument
918 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
919 #define	l2arc_hdr_arcstats_decrement_state(hdr) \  argument
920 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
1015 #define	HDR_EMPTY(hdr)						\  argument
1016 	((hdr)->b_dva.dva_word[0] == 0 &&			\
1017 	(hdr)->b_dva.dva_word[1] == 0)
1019 #define	HDR_EMPTY_OR_LOCKED(hdr)				\  argument
1020 	(HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
1022 #define	HDR_EQUAL(spa, dva, birth, hdr)				\  argument
1023 	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1024 	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1025 	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1028 buf_discard_identity(arc_buf_hdr_t *hdr)  in buf_discard_identity()  argument
1030 	hdr->b_dva.dva_word[0] = 0;  in buf_discard_identity()
1031 	hdr->b_dva.dva_word[1] = 0;  in buf_discard_identity()
1032 	hdr->b_birth = 0;  in buf_discard_identity()
1042 	arc_buf_hdr_t *hdr;  in buf_hash_find()  local
1045 	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;  in buf_hash_find()
1046 	    hdr = hdr->b_hash_next) {  in buf_hash_find()
1047 		if (HDR_EQUAL(spa, dva, birth, hdr)) {  in buf_hash_find()
1049 			return (hdr);  in buf_hash_find()
1065 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)  in buf_hash_insert()  argument
1067 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);  in buf_hash_insert()
1072 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));  in buf_hash_insert()
1073 	ASSERT(hdr->b_birth != 0);  in buf_hash_insert()
1074 	ASSERT(!HDR_IN_HASH_TABLE(hdr));  in buf_hash_insert()
1085 		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))  in buf_hash_insert()
1089 	hdr->b_hash_next = buf_hash_table.ht_table[idx];  in buf_hash_insert()
1090 	buf_hash_table.ht_table[idx] = hdr;  in buf_hash_insert()
1091 	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);  in buf_hash_insert()
1106 buf_hash_remove(arc_buf_hdr_t *hdr)  in buf_hash_remove()  argument
1109 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);  in buf_hash_remove()
1112 	ASSERT(HDR_IN_HASH_TABLE(hdr));  in buf_hash_remove()
1115 	while ((fhdr = *hdrp) != hdr) {  in buf_hash_remove()
1119 	*hdrp = hdr->b_hash_next;  in buf_hash_remove()
1120 	hdr->b_hash_next = NULL;  in buf_hash_remove()
1121 	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);  in buf_hash_remove()
1167 	arc_buf_hdr_t *hdr = vbuf;  in hdr_full_cons()  local
1169 	memset(hdr, 0, HDR_FULL_SIZE);  in hdr_full_cons()
1170 	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;  in hdr_full_cons()
1171 	zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);  in hdr_full_cons()
1173 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);  in hdr_full_cons()
1175 	multilist_link_init(&hdr->b_l1hdr.b_arc_node);  in hdr_full_cons()
1176 	list_link_init(&hdr->b_l2hdr.b_l2node);  in hdr_full_cons()
1186 	arc_buf_hdr_t *hdr = vbuf;  in hdr_l2only_cons()  local
1188 	memset(hdr, 0, HDR_L2ONLY_SIZE);  in hdr_l2only_cons()
1214 	arc_buf_hdr_t *hdr = vbuf;  in hdr_full_dest()  local
1216 	ASSERT(HDR_EMPTY(hdr));  in hdr_full_dest()
1217 	zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);  in hdr_full_dest()
1219 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);  in hdr_full_dest()
1221 	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));  in hdr_full_dest()
1229 	arc_buf_hdr_t *hdr = vbuf;  in hdr_l2only_dest()  local
1231 	ASSERT(HDR_EMPTY(hdr));  in hdr_l2only_dest()
1338 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_get_raw_params()  local
1340 	ASSERT(HDR_PROTECTED(hdr));  in arc_get_raw_params()
1342 	memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);  in arc_get_raw_params()
1343 	memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);  in arc_get_raw_params()
1344 	memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);  in arc_get_raw_params()
1345 	*byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?  in arc_get_raw_params()
1367 arc_hdr_get_compress(arc_buf_hdr_t *hdr)  in arc_hdr_get_compress()  argument
1369 	return (HDR_COMPRESSION_ENABLED(hdr) ?  in arc_hdr_get_compress()
1370 	    HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);  in arc_hdr_get_compress()
1403 arc_cksum_free(arc_buf_hdr_t *hdr)  in arc_cksum_free()  argument
1406 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_cksum_free()
1408 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_free()
1409 	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {  in arc_cksum_free()
1410 		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));  in arc_cksum_free()
1411 		hdr->b_l1hdr.b_freeze_cksum = NULL;  in arc_cksum_free()
1413 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_free()
1422 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)  in arc_hdr_has_uncompressed_buf()  argument
1424 	ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));  in arc_hdr_has_uncompressed_buf()
1426 	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {  in arc_hdr_has_uncompressed_buf()
1444 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_cksum_verify()  local
1453 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_cksum_verify()
1455 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_verify()
1457 	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {  in arc_cksum_verify()
1458 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_verify()
1463 	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))  in arc_cksum_verify()
1465 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_verify()
1475 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)  in arc_cksum_is_equal()  argument
1478 	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));  in arc_cksum_is_equal()
1511 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_cksum_compute()  local
1512 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_cksum_compute()
1513 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_compute()
1514 	if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {  in arc_cksum_compute()
1515 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_compute()
1521 	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),  in arc_cksum_compute()
1524 	    hdr->b_l1hdr.b_freeze_cksum);  in arc_cksum_compute()
1525 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);  in arc_cksum_compute()
1565 arc_buf_type(arc_buf_hdr_t *hdr)  in arc_buf_type()  argument
1568 	if (HDR_ISTYPE_METADATA(hdr)) {  in arc_buf_type()
1573 	VERIFY3U(hdr->b_type, ==, type);  in arc_buf_type()
1602 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_thaw()  local
1604 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);  in arc_buf_thaw()
1605 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_buf_thaw()
1615 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_buf_thaw()
1616 	arc_cksum_free(hdr);  in arc_buf_thaw()
1642 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)  in arc_hdr_set_flags()  argument
1644 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_hdr_set_flags()
1645 	hdr->b_flags |= flags;  in arc_hdr_set_flags()
1649 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)  in arc_hdr_clear_flags()  argument
1651 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_hdr_clear_flags()
1652 	hdr->b_flags &= ~flags;  in arc_hdr_clear_flags()
1663 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)  in arc_hdr_set_compress()  argument
1665 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_hdr_set_compress()
1672 	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {  in arc_hdr_set_compress()
1673 		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);  in arc_hdr_set_compress()
1674 		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));  in arc_hdr_set_compress()
1676 		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);  in arc_hdr_set_compress()
1677 		ASSERT(HDR_COMPRESSION_ENABLED(hdr));  in arc_hdr_set_compress()
1680 	HDR_SET_COMPRESS(hdr, cmp);  in arc_hdr_set_compress()
1681 	ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);  in arc_hdr_set_compress()
1691 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_try_copy_decompressed_data()  local
1694 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_buf_try_copy_decompressed_data()
1698 	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;  in arc_buf_try_copy_decompressed_data()
1718 		EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);  in arc_buf_try_copy_decompressed_data()
1736 	arc_buf_hdr_t	*hdr;  in arc_buf_alloc_l2only()  local
1741 	hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);  in arc_buf_alloc_l2only()
1742 	hdr->b_birth = birth;  in arc_buf_alloc_l2only()
1743 	hdr->b_type = type;  in arc_buf_alloc_l2only()
1744 	hdr->b_flags = 0;  in arc_buf_alloc_l2only()
1745 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR);  in arc_buf_alloc_l2only()
1746 	HDR_SET_LSIZE(hdr, size);  in arc_buf_alloc_l2only()
1747 	HDR_SET_PSIZE(hdr, psize);  in arc_buf_alloc_l2only()
1748 	HDR_SET_L2SIZE(hdr, asize);  in arc_buf_alloc_l2only()
1749 	arc_hdr_set_compress(hdr, compress);  in arc_buf_alloc_l2only()
1750 	hdr->b_complevel = complevel;  in arc_buf_alloc_l2only()
1752 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);  in arc_buf_alloc_l2only()
1754 		arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);  in arc_buf_alloc_l2only()
1755 	hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);  in arc_buf_alloc_l2only()
1757 	hdr->b_dva = dva;  in arc_buf_alloc_l2only()
1759 	hdr->b_l2hdr.b_dev = dev;  in arc_buf_alloc_l2only()
1760 	hdr->b_l2hdr.b_daddr = daddr;  in arc_buf_alloc_l2only()
1761 	hdr->b_l2hdr.b_arcs_state = arcs_state;  in arc_buf_alloc_l2only()
1763 	return (hdr);  in arc_buf_alloc_l2only()
1770 arc_hdr_size(arc_buf_hdr_t *hdr)  in arc_hdr_size()  argument
1774 	if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&  in arc_hdr_size()
1775 	    HDR_GET_PSIZE(hdr) > 0) {  in arc_hdr_size()
1776 		size = HDR_GET_PSIZE(hdr);  in arc_hdr_size()
1778 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);  in arc_hdr_size()
1779 		size = HDR_GET_LSIZE(hdr);  in arc_hdr_size()
1785 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)  in arc_hdr_authenticate()  argument
1789 	uint64_t lsize = HDR_GET_LSIZE(hdr);  in arc_hdr_authenticate()
1790 	uint64_t psize = HDR_GET_PSIZE(hdr);  in arc_hdr_authenticate()
1791 	abd_t *abd = hdr->b_l1hdr.b_pabd;  in arc_hdr_authenticate()
1794 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_hdr_authenticate()
1795 	ASSERT(HDR_AUTHENTICATED(hdr));  in arc_hdr_authenticate()
1806 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&  in arc_hdr_authenticate()
1807 	    !HDR_COMPRESSION_ENABLED(hdr)) {  in arc_hdr_authenticate()
1809 		csize = zio_compress_data(HDR_GET_COMPRESS(hdr),  in arc_hdr_authenticate()
1810 		    hdr->b_l1hdr.b_pabd, &abd, lsize, MIN(lsize, psize),  in arc_hdr_authenticate()
1811 		    hdr->b_complevel);  in arc_hdr_authenticate()
1825 	if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {  in arc_hdr_authenticate()
1826 		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);  in arc_hdr_authenticate()
1829 		    psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);  in arc_hdr_authenticate()
1832 		    hdr->b_crypt_hdr.b_mac);  in arc_hdr_authenticate()
1836 		arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);  in arc_hdr_authenticate()
1853 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)  in arc_hdr_decrypt()  argument
1858 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);  in arc_hdr_decrypt()
1860 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_hdr_decrypt()
1861 	ASSERT(HDR_ENCRYPTED(hdr));  in arc_hdr_decrypt()
1863 	arc_hdr_alloc_abd(hdr, 0);  in arc_hdr_decrypt()
1865 	ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,  in arc_hdr_decrypt()
1866 	    B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,  in arc_hdr_decrypt()
1867 	    hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,  in arc_hdr_decrypt()
1868 	    hdr->b_crypt_hdr.b_rabd, &no_crypt);  in arc_hdr_decrypt()
1873 		abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,  in arc_hdr_decrypt()
1874 		    HDR_GET_PSIZE(hdr));  in arc_hdr_decrypt()
1882 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&  in arc_hdr_decrypt()
1883 	    !HDR_COMPRESSION_ENABLED(hdr)) {  in arc_hdr_decrypt()
1890 		cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);  in arc_hdr_decrypt()
1892 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),  in arc_hdr_decrypt()
1893 		    hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),  in arc_hdr_decrypt()
1894 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);  in arc_hdr_decrypt()
1899 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,  in arc_hdr_decrypt()
1900 		    arc_hdr_size(hdr), hdr);  in arc_hdr_decrypt()
1901 		hdr->b_l1hdr.b_pabd = cabd;  in arc_hdr_decrypt()
1907 	arc_hdr_free_abd(hdr, B_FALSE);  in arc_hdr_decrypt()
1909 		arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);  in arc_hdr_decrypt()
1920 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,  in arc_fill_hdr_crypt()  argument
1925 	ASSERT(HDR_PROTECTED(hdr));  in arc_fill_hdr_crypt()
1930 	if (HDR_NOAUTH(hdr) && !noauth) {  in arc_fill_hdr_crypt()
1935 		ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);  in arc_fill_hdr_crypt()
1938 	} else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {  in arc_fill_hdr_crypt()
1944 		ret = arc_hdr_decrypt(hdr, spa, zb);  in arc_fill_hdr_crypt()
1949 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in arc_fill_hdr_crypt()
1972 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_untransform_in_place()  local
1974 	ASSERT(HDR_ENCRYPTED(hdr));  in arc_buf_untransform_in_place()
1975 	ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);  in arc_buf_untransform_in_place()
1976 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_buf_untransform_in_place()
1977 	ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf);  in arc_buf_untransform_in_place()
1979 	zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,  in arc_buf_untransform_in_place()
2003 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_fill()  local
2005 	    (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);  in arc_buf_fill()
2008 	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;  in arc_buf_fill()
2009 	kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);  in arc_buf_fill()
2014 	IMPLY(encrypted, HDR_ENCRYPTED(hdr));  in arc_buf_fill()
2025 		ASSERT(HDR_HAS_RABD(hdr));  in arc_buf_fill()
2026 		abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,  in arc_buf_fill()
2027 		    HDR_GET_PSIZE(hdr));  in arc_buf_fill()
2037 	if (HDR_PROTECTED(hdr)) {  in arc_buf_fill()
2038 		error = arc_fill_hdr_crypt(hdr, hash_lock, spa,  in arc_buf_fill()
2045 			arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);  in arc_buf_fill()
2068 		if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {  in arc_buf_fill()
2069 			ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);  in arc_buf_fill()
2088 			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,  in arc_buf_fill()
2106 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);  in arc_buf_fill()
2107 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);  in arc_buf_fill()
2110 			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));  in arc_buf_fill()
2115 			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),  in arc_buf_fill()
2118 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);  in arc_buf_fill()
2122 			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));  in arc_buf_fill()
2142 			    HDR_GET_LSIZE(hdr));  in arc_buf_fill()
2143 			error = zio_decompress_data(HDR_GET_COMPRESS(hdr),  in arc_buf_fill()
2144 			    hdr->b_l1hdr.b_pabd, &dabd,  in arc_buf_fill()
2145 			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),  in arc_buf_fill()
2146 			    &hdr->b_complevel);  in arc_buf_fill()
2156 				    hdr, arc_hdr_get_compress(hdr),  in arc_buf_fill()
2157 				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));  in arc_buf_fill()
2160 				arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);  in arc_buf_fill()
2171 		ASSERT(!HDR_SHARED_DATA(hdr));  in arc_buf_fill()
2173 		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));  in arc_buf_fill()
2219 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)  in arc_evictable_space_increment()  argument
2221 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_evictable_space_increment()
2223 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_evictable_space_increment()
2226 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);  in arc_evictable_space_increment()
2227 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_evictable_space_increment()
2228 		ASSERT(!HDR_HAS_RABD(hdr));  in arc_evictable_space_increment()
2230 		    HDR_GET_LSIZE(hdr), hdr);  in arc_evictable_space_increment()
2234 	if (hdr->b_l1hdr.b_pabd != NULL) {  in arc_evictable_space_increment()
2236 		    arc_hdr_size(hdr), hdr);  in arc_evictable_space_increment()
2238 	if (HDR_HAS_RABD(hdr)) {  in arc_evictable_space_increment()
2240 		    HDR_GET_PSIZE(hdr), hdr);  in arc_evictable_space_increment()
2243 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;  in arc_evictable_space_increment()
2258 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)  in arc_evictable_space_decrement()  argument
2260 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_evictable_space_decrement()
2262 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_evictable_space_decrement()
2265 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);  in arc_evictable_space_decrement()
2266 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_evictable_space_decrement()
2267 		ASSERT(!HDR_HAS_RABD(hdr));  in arc_evictable_space_decrement()
2269 		    HDR_GET_LSIZE(hdr), hdr);  in arc_evictable_space_decrement()
2273 	if (hdr->b_l1hdr.b_pabd != NULL) {  in arc_evictable_space_decrement()
2275 		    arc_hdr_size(hdr), hdr);  in arc_evictable_space_decrement()
2277 	if (HDR_HAS_RABD(hdr)) {  in arc_evictable_space_decrement()
2279 		    HDR_GET_PSIZE(hdr), hdr);  in arc_evictable_space_decrement()
2282 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;  in arc_evictable_space_decrement()
2298 add_reference(arc_buf_hdr_t *hdr, const void *tag)  in add_reference()  argument
2300 	arc_state_t *state = hdr->b_l1hdr.b_state;  in add_reference()
2302 	ASSERT(HDR_HAS_L1HDR(hdr));  in add_reference()
2303 	if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {  in add_reference()
2305 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));  in add_reference()
2306 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);  in add_reference()
2309 	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&  in add_reference()
2312 		multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);  in add_reference()
2313 		arc_evictable_space_decrement(hdr, state);  in add_reference()
2323 remove_reference(arc_buf_hdr_t *hdr, const void *tag)  in remove_reference()  argument
2326 	arc_state_t *state = hdr->b_l1hdr.b_state;  in remove_reference()
2328 	ASSERT(HDR_HAS_L1HDR(hdr));  in remove_reference()
2329 	ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr)));  in remove_reference()
2332 	if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)  in remove_reference()
2336 		arc_hdr_destroy(hdr);  in remove_reference()
2339 	if (state == arc_uncached && !HDR_PREFETCH(hdr)) {  in remove_reference()
2340 		arc_change_state(arc_anon, hdr);  in remove_reference()
2341 		arc_hdr_destroy(hdr);  in remove_reference()
2344 	multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);  in remove_reference()
2345 	arc_evictable_space_increment(hdr, state);  in remove_reference()
2360 	arc_buf_hdr_t *hdr = ab->b_hdr;  in arc_buf_info()  local
2367 	if (hdr == NULL)  in arc_buf_info()
2370 	abi->abi_flags = hdr->b_flags;  in arc_buf_info()
2372 	if (HDR_HAS_L1HDR(hdr)) {  in arc_buf_info()
2373 		l1hdr = &hdr->b_l1hdr;  in arc_buf_info()
2376 	if (HDR_HAS_L2HDR(hdr))  in arc_buf_info()
2377 		l2hdr = &hdr->b_l2hdr;  in arc_buf_info()
2397 	abi->abi_state_contents = arc_buf_type(hdr);  in arc_buf_info()
2398 	abi->abi_size = arc_hdr_size(hdr);  in arc_buf_info()
2406 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)  in arc_change_state()  argument
2411 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_change_state()
2420 	if (HDR_HAS_L1HDR(hdr)) {  in arc_change_state()
2421 		old_state = hdr->b_l1hdr.b_state;  in arc_change_state()
2422 		refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);  in arc_change_state()
2423 		update_old = (hdr->b_l1hdr.b_buf != NULL ||  in arc_change_state()
2424 		    hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));  in arc_change_state()
2426 		IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);  in arc_change_state()
2427 		IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);  in arc_change_state()
2428 		IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||  in arc_change_state()
2429 		    ARC_BUF_LAST(hdr->b_l1hdr.b_buf));  in arc_change_state()
2441 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));  in arc_change_state()
2450 			ASSERT(HDR_HAS_L1HDR(hdr));  in arc_change_state()
2452 			if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {  in arc_change_state()
2454 				    hdr);  in arc_change_state()
2455 				arc_evictable_space_decrement(hdr, old_state);  in arc_change_state()
2465 			ASSERT(HDR_HAS_L1HDR(hdr));  in arc_change_state()
2466 			multilist_insert(&new_state->arcs_list[type], hdr);  in arc_change_state()
2467 			arc_evictable_space_increment(hdr, new_state);  in arc_change_state()
2471 	ASSERT(!HDR_EMPTY(hdr));  in arc_change_state()
2472 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))  in arc_change_state()
2473 		buf_hash_remove(hdr);  in arc_change_state()
2478 		ASSERT(HDR_HAS_L1HDR(hdr));  in arc_change_state()
2489 			    HDR_GET_LSIZE(hdr), hdr);  in arc_change_state()
2490 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_change_state()
2491 			ASSERT(!HDR_HAS_RABD(hdr));  in arc_change_state()
2499 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;  in arc_change_state()
2517 			if (hdr->b_l1hdr.b_pabd != NULL) {  in arc_change_state()
2520 				    arc_hdr_size(hdr), hdr);  in arc_change_state()
2523 			if (HDR_HAS_RABD(hdr)) {  in arc_change_state()
2526 				    HDR_GET_PSIZE(hdr), hdr);  in arc_change_state()
2532 		ASSERT(HDR_HAS_L1HDR(hdr));  in arc_change_state()
2534 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_change_state()
2535 			ASSERT(!HDR_HAS_RABD(hdr));  in arc_change_state()
2547 			    HDR_GET_LSIZE(hdr), hdr);  in arc_change_state()
2555 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;  in arc_change_state()
2572 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||  in arc_change_state()
2573 			    HDR_HAS_RABD(hdr));  in arc_change_state()
2575 			if (hdr->b_l1hdr.b_pabd != NULL) {  in arc_change_state()
2578 				    arc_hdr_size(hdr), hdr);  in arc_change_state()
2581 			if (HDR_HAS_RABD(hdr)) {  in arc_change_state()
2584 				    HDR_GET_PSIZE(hdr), hdr);  in arc_change_state()
2589 	if (HDR_HAS_L1HDR(hdr)) {  in arc_change_state()
2590 		hdr->b_l1hdr.b_state = new_state;  in arc_change_state()
2592 		if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {  in arc_change_state()
2593 			l2arc_hdr_arcstats_decrement_state(hdr);  in arc_change_state()
2594 			hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;  in arc_change_state()
2595 			l2arc_hdr_arcstats_increment_state(hdr);  in arc_change_state()
2692 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)  in arc_can_share()  argument
2716 	ASSERT3P(buf->b_hdr, ==, hdr);  in arc_can_share()
2718 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;  in arc_can_share()
2722 	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&  in arc_can_share()
2723 	    !HDR_SHARED_DATA(hdr) &&  in arc_can_share()
2733 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,  in arc_buf_alloc_impl()  argument
2740 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_buf_alloc_impl()
2741 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);  in arc_buf_alloc_impl()
2742 	VERIFY(hdr->b_type == ARC_BUFC_DATA ||  in arc_buf_alloc_impl()
2743 	    hdr->b_type == ARC_BUFC_METADATA);  in arc_buf_alloc_impl()
2749 	buf->b_hdr = hdr;  in arc_buf_alloc_impl()
2751 	buf->b_next = hdr->b_l1hdr.b_buf;  in arc_buf_alloc_impl()
2754 	add_reference(hdr, tag);  in arc_buf_alloc_impl()
2760 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_buf_alloc_impl()
2772 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {  in arc_buf_alloc_impl()
2800 	boolean_t can_share = arc_can_share(hdr, buf) &&  in arc_buf_alloc_impl()
2801 	    !HDR_L2_WRITING(hdr) &&  in arc_buf_alloc_impl()
2802 	    hdr->b_l1hdr.b_pabd != NULL &&  in arc_buf_alloc_impl()
2803 	    abd_is_linear(hdr->b_l1hdr.b_pabd) &&  in arc_buf_alloc_impl()
2804 	    !abd_is_linear_page(hdr->b_l1hdr.b_pabd);  in arc_buf_alloc_impl()
2808 		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);  in arc_buf_alloc_impl()
2810 		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);  in arc_buf_alloc_impl()
2813 		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);  in arc_buf_alloc_impl()
2818 	hdr->b_l1hdr.b_buf = buf;  in arc_buf_alloc_impl()
2893 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_return_buf()  local
2896 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_return_buf()
2897 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);  in arc_return_buf()
2898 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);  in arc_return_buf()
2907 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_loan_inuse_buf()  local
2910 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_loan_inuse_buf()
2911 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);  in arc_loan_inuse_buf()
2912 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);  in arc_loan_inuse_buf()
2931 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)  in arc_hdr_free_on_write()  argument
2933 	arc_state_t *state = hdr->b_l1hdr.b_state;  in arc_hdr_free_on_write()
2934 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_hdr_free_on_write()
2935 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);  in arc_hdr_free_on_write()
2938 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {  in arc_hdr_free_on_write()
2939 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));  in arc_hdr_free_on_write()
2943 		    size, hdr);  in arc_hdr_free_on_write()
2945 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);  in arc_hdr_free_on_write()
2954 		l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);  in arc_hdr_free_on_write()
2956 		l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);  in arc_hdr_free_on_write()
2966 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)  in arc_share_buf()  argument
2968 	ASSERT(arc_can_share(hdr, buf));  in arc_share_buf()
2969 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_share_buf()
2971 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_share_buf()
2979 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],  in arc_share_buf()
2980 	    arc_hdr_size(hdr), buf, hdr);  in arc_share_buf()
2981 	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));  in arc_share_buf()
2982 	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,  in arc_share_buf()
2983 	    HDR_ISTYPE_METADATA(hdr));  in arc_share_buf()
2984 	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);  in arc_share_buf()
2992 	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));  in arc_share_buf()
2993 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));  in arc_share_buf()
2998 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)  in arc_unshare_buf()  argument
3001 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in arc_unshare_buf()
3002 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_unshare_buf()
3009 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],  in arc_unshare_buf()
3010 	    arc_hdr_size(hdr), hdr, buf);  in arc_unshare_buf()
3011 	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);  in arc_unshare_buf()
3012 	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);  in arc_unshare_buf()
3013 	abd_free(hdr->b_l1hdr.b_pabd);  in arc_unshare_buf()
3014 	hdr->b_l1hdr.b_pabd = NULL;  in arc_unshare_buf()
3021 	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));  in arc_unshare_buf()
3022 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));  in arc_unshare_buf()
3032 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)  in arc_buf_remove()  argument
3034 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_buf_remove()
3035 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_buf_remove()
3037 	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;  in arc_buf_remove()
3072 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_destroy_impl()  local
3084 		ASSERT(HDR_EMPTY_OR_LOCKED(hdr));  in arc_buf_destroy_impl()
3090 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);  in arc_buf_destroy_impl()
3094 			arc_free_data_buf(hdr, buf->b_data, size, buf);  in arc_buf_destroy_impl()
3104 		if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&  in arc_buf_destroy_impl()
3105 		    hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {  in arc_buf_destroy_impl()
3107 			for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {  in arc_buf_destroy_impl()
3112 				arc_hdr_free_abd(hdr, B_TRUE);  in arc_buf_destroy_impl()
3116 	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);  in arc_buf_destroy_impl()
3137 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in arc_buf_destroy_impl()
3138 			arc_hdr_free_abd(hdr, B_FALSE);  in arc_buf_destroy_impl()
3146 			arc_share_buf(hdr, lastbuf);  in arc_buf_destroy_impl()
3148 	} else if (HDR_SHARED_DATA(hdr)) {  in arc_buf_destroy_impl()
3159 		    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);  in arc_buf_destroy_impl()
3166 	if (!arc_hdr_has_uncompressed_buf(hdr)) {  in arc_buf_destroy_impl()
3167 		arc_cksum_free(hdr);  in arc_buf_destroy_impl()
3176 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)  in arc_hdr_alloc_abd()  argument
3181 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);  in arc_hdr_alloc_abd()
3182 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_hdr_alloc_abd()
3183 	ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata);  in arc_hdr_alloc_abd()
3184 	IMPLY(alloc_rdata, HDR_PROTECTED(hdr));  in arc_hdr_alloc_abd()
3187 		size = HDR_GET_PSIZE(hdr);  in arc_hdr_alloc_abd()
3188 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);  in arc_hdr_alloc_abd()
3189 		hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,  in arc_hdr_alloc_abd()
3191 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);  in arc_hdr_alloc_abd()
3194 		size = arc_hdr_size(hdr);  in arc_hdr_alloc_abd()
3195 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_hdr_alloc_abd()
3196 		hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,  in arc_hdr_alloc_abd()
3198 		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in arc_hdr_alloc_abd()
3202 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));  in arc_hdr_alloc_abd()
3206 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)  in arc_hdr_free_abd()  argument
3208 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);  in arc_hdr_free_abd()
3210 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_hdr_free_abd()
3211 	ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));  in arc_hdr_free_abd()
3212 	IMPLY(free_rdata, HDR_HAS_RABD(hdr));  in arc_hdr_free_abd()
3220 	if (HDR_L2_WRITING(hdr)) {  in arc_hdr_free_abd()
3221 		arc_hdr_free_on_write(hdr, free_rdata);  in arc_hdr_free_abd()
3224 		arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);  in arc_hdr_free_abd()
3226 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);  in arc_hdr_free_abd()
3230 		hdr->b_crypt_hdr.b_rabd = NULL;  in arc_hdr_free_abd()
3233 		hdr->b_l1hdr.b_pabd = NULL;  in arc_hdr_free_abd()
3236 	if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))  in arc_hdr_free_abd()
3237 		hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;  in arc_hdr_free_abd()
3240 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));  in arc_hdr_free_abd()
3270 	arc_buf_hdr_t *hdr;  in arc_hdr_alloc()  local
3273 	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);  in arc_hdr_alloc()
3275 	ASSERT(HDR_EMPTY(hdr));  in arc_hdr_alloc()
3277 	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);  in arc_hdr_alloc()
3279 	HDR_SET_PSIZE(hdr, psize);  in arc_hdr_alloc()
3280 	HDR_SET_LSIZE(hdr, lsize);  in arc_hdr_alloc()
3281 	hdr->b_spa = spa;  in arc_hdr_alloc()
3282 	hdr->b_type = type;  in arc_hdr_alloc()
3283 	hdr->b_flags = 0;  in arc_hdr_alloc()
3284 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);  in arc_hdr_alloc()
3285 	arc_hdr_set_compress(hdr, compression_type);  in arc_hdr_alloc()
3286 	hdr->b_complevel = complevel;  in arc_hdr_alloc()
3288 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);  in arc_hdr_alloc()
3290 	hdr->b_l1hdr.b_state = arc_anon;  in arc_hdr_alloc()
3291 	hdr->b_l1hdr.b_arc_access = 0;  in arc_hdr_alloc()
3292 	hdr->b_l1hdr.b_mru_hits = 0;  in arc_hdr_alloc()
3293 	hdr->b_l1hdr.b_mru_ghost_hits = 0;  in arc_hdr_alloc()
3294 	hdr->b_l1hdr.b_mfu_hits = 0;  in arc_hdr_alloc()
3295 	hdr->b_l1hdr.b_mfu_ghost_hits = 0;  in arc_hdr_alloc()
3296 	hdr->b_l1hdr.b_buf = NULL;  in arc_hdr_alloc()
3298 	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));  in arc_hdr_alloc()
3300 	return (hdr);  in arc_hdr_alloc()
3311 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)  in arc_hdr_realloc()  argument
3313 	ASSERT(HDR_HAS_L2HDR(hdr));  in arc_hdr_realloc()
3316 	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;  in arc_hdr_realloc()
3323 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));  in arc_hdr_realloc()
3324 	buf_hash_remove(hdr);  in arc_hdr_realloc()
3326 	memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);  in arc_hdr_realloc()
3339 		ASSERT(!HDR_HAS_RABD(hdr));  in arc_hdr_realloc()
3341 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);  in arc_hdr_realloc()
3343 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);  in arc_hdr_realloc()
3353 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));  in arc_hdr_realloc()
3361 		VERIFY(!HDR_L2_WRITING(hdr));  in arc_hdr_realloc()
3362 		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_hdr_realloc()
3363 		ASSERT(!HDR_HAS_RABD(hdr));  in arc_hdr_realloc()
3373 	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));  in arc_hdr_realloc()
3383 	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);  in arc_hdr_realloc()
3384 	list_remove(&dev->l2ad_buflist, hdr);  in arc_hdr_realloc()
3397 	    arc_hdr_size(hdr), hdr);  in arc_hdr_realloc()
3401 	buf_discard_identity(hdr);  in arc_hdr_realloc()
3402 	kmem_cache_free(old, hdr);  in arc_hdr_realloc()
3419 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_convert_to_raw()  local
3422 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_convert_to_raw()
3423 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);  in arc_convert_to_raw()
3426 	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);  in arc_convert_to_raw()
3427 	hdr->b_crypt_hdr.b_dsobj = dsobj;  in arc_convert_to_raw()
3428 	hdr->b_crypt_hdr.b_ot = ot;  in arc_convert_to_raw()
3429 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?  in arc_convert_to_raw()
3431 	if (!arc_hdr_has_uncompressed_buf(hdr))  in arc_convert_to_raw()
3432 		arc_cksum_free(hdr);  in arc_convert_to_raw()
3435 		memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);  in arc_convert_to_raw()
3437 		memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);  in arc_convert_to_raw()
3439 		memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);  in arc_convert_to_raw()
3450 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,  in arc_alloc_buf()  local
3454 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,  in arc_alloc_buf()
3474 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,  in arc_alloc_compressed_buf()  local
3478 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,  in arc_alloc_compressed_buf()
3487 	arc_share_buf(hdr, buf);  in arc_alloc_compressed_buf()
3498 	arc_buf_hdr_t *hdr;  in arc_alloc_raw_buf()  local
3508 	hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,  in arc_alloc_raw_buf()
3511 	hdr->b_crypt_hdr.b_dsobj = dsobj;  in arc_alloc_raw_buf()
3512 	hdr->b_crypt_hdr.b_ot = ot;  in arc_alloc_raw_buf()
3513 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?  in arc_alloc_raw_buf()
3515 	memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);  in arc_alloc_raw_buf()
3516 	memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);  in arc_alloc_raw_buf()
3517 	memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);  in arc_alloc_raw_buf()
3525 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,  in arc_alloc_raw_buf()
3533 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,  in l2arc_hdr_arcstats_update()  argument
3536 	uint64_t lsize = HDR_GET_LSIZE(hdr);  in l2arc_hdr_arcstats_update()
3537 	uint64_t psize = HDR_GET_PSIZE(hdr);  in l2arc_hdr_arcstats_update()
3538 	uint64_t asize = HDR_GET_L2SIZE(hdr);  in l2arc_hdr_arcstats_update()
3539 	arc_buf_contents_t type = hdr->b_type;  in l2arc_hdr_arcstats_update()
3558 	if (HDR_PREFETCH(hdr)) {  in l2arc_hdr_arcstats_update()
3570 		switch (hdr->b_l2hdr.b_arcs_state) {  in l2arc_hdr_arcstats_update()
3604 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)  in arc_hdr_l2hdr_destroy()  argument
3606 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;  in arc_hdr_l2hdr_destroy()
3610 	ASSERT(HDR_HAS_L2HDR(hdr));  in arc_hdr_l2hdr_destroy()
3612 	list_remove(&dev->l2ad_buflist, hdr);  in arc_hdr_l2hdr_destroy()
3614 	l2arc_hdr_arcstats_decrement(hdr);  in arc_hdr_l2hdr_destroy()
3616 		uint64_t asize = HDR_GET_L2SIZE(hdr);  in arc_hdr_l2hdr_destroy()
3620 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),  in arc_hdr_l2hdr_destroy()
3621 	    hdr);  in arc_hdr_l2hdr_destroy()
3622 	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);  in arc_hdr_l2hdr_destroy()
3626 arc_hdr_destroy(arc_buf_hdr_t *hdr)  in arc_hdr_destroy()  argument
3628 	if (HDR_HAS_L1HDR(hdr)) {  in arc_hdr_destroy()
3629 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));  in arc_hdr_destroy()
3630 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);  in arc_hdr_destroy()
3632 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_hdr_destroy()
3633 	ASSERT(!HDR_IN_HASH_TABLE(hdr));  in arc_hdr_destroy()
3635 	if (HDR_HAS_L2HDR(hdr)) {  in arc_hdr_destroy()
3636 		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;  in arc_hdr_destroy()
3651 		if (HDR_HAS_L2HDR(hdr)) {  in arc_hdr_destroy()
3653 			if (!HDR_EMPTY(hdr))  in arc_hdr_destroy()
3654 				buf_discard_identity(hdr);  in arc_hdr_destroy()
3656 			arc_hdr_l2hdr_destroy(hdr);  in arc_hdr_destroy()
3669 	if (!HDR_EMPTY(hdr))  in arc_hdr_destroy()
3670 		buf_discard_identity(hdr);  in arc_hdr_destroy()
3672 	if (HDR_HAS_L1HDR(hdr)) {  in arc_hdr_destroy()
3673 		arc_cksum_free(hdr);  in arc_hdr_destroy()
3675 		while (hdr->b_l1hdr.b_buf != NULL)  in arc_hdr_destroy()
3676 			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);  in arc_hdr_destroy()
3678 		if (hdr->b_l1hdr.b_pabd != NULL)  in arc_hdr_destroy()
3679 			arc_hdr_free_abd(hdr, B_FALSE);  in arc_hdr_destroy()
3681 		if (HDR_HAS_RABD(hdr))  in arc_hdr_destroy()
3682 			arc_hdr_free_abd(hdr, B_TRUE);  in arc_hdr_destroy()
3685 	ASSERT3P(hdr->b_hash_next, ==, NULL);  in arc_hdr_destroy()
3686 	if (HDR_HAS_L1HDR(hdr)) {  in arc_hdr_destroy()
3687 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));  in arc_hdr_destroy()
3688 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);  in arc_hdr_destroy()
3690 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);  in arc_hdr_destroy()
3692 		kmem_cache_free(hdr_full_cache, hdr);  in arc_hdr_destroy()
3694 		kmem_cache_free(hdr_l2only_cache, hdr);  in arc_hdr_destroy()
3701 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_destroy()  local
3703 	if (hdr->b_l1hdr.b_state == arc_anon) {  in arc_buf_destroy()
3704 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);  in arc_buf_destroy()
3706 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_buf_destroy()
3707 		VERIFY0(remove_reference(hdr, tag));  in arc_buf_destroy()
3711 	kmutex_t *hash_lock = HDR_LOCK(hdr);  in arc_buf_destroy()
3714 	ASSERT3P(hdr, ==, buf->b_hdr);  in arc_buf_destroy()
3715 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);  in arc_buf_destroy()
3716 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));  in arc_buf_destroy()
3717 	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);  in arc_buf_destroy()
3721 	(void) remove_reference(hdr, tag);  in arc_buf_destroy()
3748 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)  in arc_evict_hdr()  argument
3752 	uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?  in arc_evict_hdr()
3755 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));  in arc_evict_hdr()
3756 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_evict_hdr()
3757 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_evict_hdr()
3758 	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);  in arc_evict_hdr()
3759 	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));  in arc_evict_hdr()
3762 	state = hdr->b_l1hdr.b_state;  in arc_evict_hdr()
3772 		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {  in arc_evict_hdr()
3778 		bytes_evicted += HDR_GET_LSIZE(hdr);  in arc_evict_hdr()
3780 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);  in arc_evict_hdr()
3782 		if (HDR_HAS_L2HDR(hdr)) {  in arc_evict_hdr()
3783 			ASSERT(hdr->b_l1hdr.b_pabd == NULL);  in arc_evict_hdr()
3784 			ASSERT(!HDR_HAS_RABD(hdr));  in arc_evict_hdr()
3789 			arc_change_state(arc_l2c_only, hdr);  in arc_evict_hdr()
3794 			(void) arc_hdr_realloc(hdr, hdr_full_cache,  in arc_evict_hdr()
3798 			arc_change_state(arc_anon, hdr);  in arc_evict_hdr()
3799 			arc_hdr_destroy(hdr);  in arc_evict_hdr()
3810 	if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&  in arc_evict_hdr()
3811 	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <  in arc_evict_hdr()
3817 	if (HDR_HAS_L2HDR(hdr)) {  in arc_evict_hdr()
3818 		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));  in arc_evict_hdr()
3820 		if (l2arc_write_eligible(hdr->b_spa, hdr)) {  in arc_evict_hdr()
3822 			    HDR_GET_LSIZE(hdr));  in arc_evict_hdr()
3828 					    HDR_GET_LSIZE(hdr));  in arc_evict_hdr()
3833 					    HDR_GET_LSIZE(hdr));  in arc_evict_hdr()
3840 			    HDR_GET_LSIZE(hdr));  in arc_evict_hdr()
3844 	bytes_evicted += arc_hdr_size(hdr);  in arc_evict_hdr()
3845 	*real_evicted += arc_hdr_size(hdr);  in arc_evict_hdr()
3852 	if (hdr->b_l1hdr.b_pabd != NULL)  in arc_evict_hdr()
3853 		arc_hdr_free_abd(hdr, B_FALSE);  in arc_evict_hdr()
3855 	if (HDR_HAS_RABD(hdr))  in arc_evict_hdr()
3856 		arc_hdr_free_abd(hdr, B_TRUE);  in arc_evict_hdr()
3858 	arc_change_state(evicted_state, hdr);  in arc_evict_hdr()
3859 	DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);  in arc_evict_hdr()
3861 		arc_hdr_destroy(hdr);  in arc_evict_hdr()
3864 		ASSERT(HDR_IN_HASH_TABLE(hdr));  in arc_evict_hdr()
3890 	arc_buf_hdr_t *hdr;  in arc_evict_state_impl()  local
3898 	for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);  in arc_evict_state_impl()
3899 	    hdr = multilist_sublist_prev(mls, marker)) {  in arc_evict_state_impl()
3925 		if (hdr->b_spa == 0)  in arc_evict_state_impl()
3929 		if (spa != 0 && hdr->b_spa != spa) {  in arc_evict_state_impl()
3934 		hash_lock = HDR_LOCK(hdr);  in arc_evict_state_impl()
3949 			uint64_t evicted = arc_evict_hdr(hdr, &revicted);  in arc_evict_state_impl()
4926 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,  in arc_get_data_abd()  argument
4929 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_get_data_abd()
4931 	arc_get_data_impl(hdr, size, tag, alloc_flags);  in arc_get_data_abd()
4939 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)  in arc_get_data_buf()  argument
4941 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_get_data_buf()
4943 	arc_get_data_impl(hdr, size, tag, 0);  in arc_get_data_buf()
5040 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,  in arc_get_data_impl()  argument
5061 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_get_data_impl()
5072 	arc_state_t *state = hdr->b_l1hdr.b_state;  in arc_get_data_impl()
5087 		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {  in arc_get_data_impl()
5088 			ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));  in arc_get_data_impl()
5096 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size,  in arc_free_data_abd()  argument
5099 	arc_free_data_impl(hdr, size, tag);  in arc_free_data_abd()
5104 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag)  in arc_free_data_buf()  argument
5106 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_free_data_buf()
5108 	arc_free_data_impl(hdr, size, tag);  in arc_free_data_buf()
5121 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)  in arc_free_data_impl()  argument
5123 	arc_state_t *state = hdr->b_l1hdr.b_state;  in arc_free_data_impl()
5124 	arc_buf_contents_t type = arc_buf_type(hdr);  in arc_free_data_impl()
5127 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {  in arc_free_data_impl()
5128 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));  in arc_free_data_impl()
5136 	VERIFY3U(hdr->b_type, ==, type);  in arc_free_data_impl()
5149 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)  in arc_access()  argument
5151 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));  in arc_access()
5152 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_access()
5157 	boolean_t was_prefetch = HDR_PREFETCH(hdr);  in arc_access()
5162 			    HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,  in arc_access()
5165 		if (HDR_HAS_L2HDR(hdr))  in arc_access()
5166 			l2arc_hdr_arcstats_decrement_state(hdr);  in arc_access()
5168 			arc_hdr_clear_flags(hdr,  in arc_access()
5171 			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);  in arc_access()
5173 		if (HDR_HAS_L2HDR(hdr))  in arc_access()
5174 			l2arc_hdr_arcstats_increment_state(hdr);  in arc_access()
5178 			arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);  in arc_access()
5185 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);  in arc_access()
5188 	if (hdr->b_l1hdr.b_state == arc_anon) {  in arc_access()
5194 		ASSERT0(hdr->b_l1hdr.b_arc_access);  in arc_access()
5195 		hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5196 		if (HDR_UNCACHED(hdr)) {  in arc_access()
5199 			    hdr);  in arc_access()
5202 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);  in arc_access()
5204 		arc_change_state(new_state, hdr);  in arc_access()
5205 	} else if (hdr->b_l1hdr.b_state == arc_mru) {  in arc_access()
5210 		if (HDR_IO_IN_PROGRESS(hdr)) {  in arc_access()
5211 			hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5214 		hdr->b_l1hdr.b_mru_hits++;  in arc_access()
5222 			hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5230 		if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +  in arc_access()
5232 			hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5233 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);  in arc_access()
5234 			arc_change_state(arc_mfu, hdr);  in arc_access()
5236 	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {  in arc_access()
5244 		hdr->b_l1hdr.b_mru_ghost_hits++;  in arc_access()
5246 		hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5247 		wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],  in arc_access()
5248 		    arc_hdr_size(hdr));  in arc_access()
5251 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);  in arc_access()
5254 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);  in arc_access()
5256 		arc_change_state(new_state, hdr);  in arc_access()
5257 	} else if (hdr->b_l1hdr.b_state == arc_mfu) {  in arc_access()
5262 		if (!HDR_IO_IN_PROGRESS(hdr)) {  in arc_access()
5263 			hdr->b_l1hdr.b_mfu_hits++;  in arc_access()
5266 		hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5267 	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {  in arc_access()
5273 		hdr->b_l1hdr.b_mfu_ghost_hits++;  in arc_access()
5275 		hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5276 		wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],  in arc_access()
5277 		    arc_hdr_size(hdr));  in arc_access()
5278 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);  in arc_access()
5279 		arc_change_state(arc_mfu, hdr);  in arc_access()
5280 	} else if (hdr->b_l1hdr.b_state == arc_uncached) {  in arc_access()
5285 		if (!HDR_IO_IN_PROGRESS(hdr))  in arc_access()
5287 		hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5288 	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {  in arc_access()
5293 		hdr->b_l1hdr.b_arc_access = now;  in arc_access()
5294 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);  in arc_access()
5295 		arc_change_state(arc_mru, hdr);  in arc_access()
5298 		    hdr->b_l1hdr.b_state);  in arc_access()
5309 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_buf_access()  local
5316 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))  in arc_buf_access()
5319 	kmutex_t *hash_lock = HDR_LOCK(hdr);  in arc_buf_access()
5322 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {  in arc_buf_access()
5328 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||  in arc_buf_access()
5329 	    hdr->b_l1hdr.b_state == arc_mfu ||  in arc_buf_access()
5330 	    hdr->b_l1hdr.b_state == arc_uncached);  in arc_buf_access()
5332 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);  in arc_buf_access()
5333 	arc_access(hdr, 0, B_TRUE);  in arc_buf_access()
5338 	    !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);  in arc_buf_access()
5374 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)  in arc_hdr_verify()  argument
5377 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);  in arc_hdr_verify()
5378 		ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);  in arc_hdr_verify()
5380 		if (HDR_COMPRESSION_ENABLED(hdr)) {  in arc_hdr_verify()
5381 			ASSERT3U(arc_hdr_get_compress(hdr), ==,  in arc_hdr_verify()
5384 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));  in arc_hdr_verify()
5385 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));  in arc_hdr_verify()
5386 		ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));  in arc_hdr_verify()
5394 	arc_buf_hdr_t	*hdr = zio->io_private;  in arc_read_done()  local
5407 	if (HDR_IN_HASH_TABLE(hdr)) {  in arc_read_done()
5410 		ASSERT3U(hdr->b_birth, ==, BP_GET_BIRTH(zio->io_bp));  in arc_read_done()
5411 		ASSERT3U(hdr->b_dva.dva_word[0], ==,  in arc_read_done()
5413 		ASSERT3U(hdr->b_dva.dva_word[1], ==,  in arc_read_done()
5416 		found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);  in arc_read_done()
5418 		ASSERT((found == hdr &&  in arc_read_done()
5419 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||  in arc_read_done()
5420 		    (found == hdr && HDR_L2_READING(hdr)));  in arc_read_done()
5425 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);  in arc_read_done()
5426 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;  in arc_read_done()
5427 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,  in arc_read_done()
5428 		    hdr->b_crypt_hdr.b_iv);  in arc_read_done()
5437 				    hdr->b_crypt_hdr.b_mac);  in arc_read_done()
5442 				    hdr->b_crypt_hdr.b_mac);  in arc_read_done()
5451 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;  in arc_read_done()
5453 				hdr->b_l1hdr.b_byteswap =  in arc_read_done()
5457 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;  in arc_read_done()
5459 		if (!HDR_L2_READING(hdr)) {  in arc_read_done()
5460 			hdr->b_complevel = zio->io_prop.zp_complevel;  in arc_read_done()
5464 	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);  in arc_read_done()
5465 	if (l2arc_noprefetch && HDR_PREFETCH(hdr))  in arc_read_done()
5466 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);  in arc_read_done()
5468 	callback_list = hdr->b_l1hdr.b_acb;  in arc_read_done()
5470 	hdr->b_l1hdr.b_acb = NULL;  in arc_read_done()
5492 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,  in arc_read_done()
5549 		arc_hdr_verify(hdr, zio->io_bp);  in arc_read_done()
5551 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);  in arc_read_done()
5552 		if (hdr->b_l1hdr.b_state != arc_anon)  in arc_read_done()
5553 			arc_change_state(arc_anon, hdr);  in arc_read_done()
5554 		if (HDR_IN_HASH_TABLE(hdr))  in arc_read_done()
5555 			buf_hash_remove(hdr);  in arc_read_done()
5558 	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);  in arc_read_done()
5559 	(void) remove_reference(hdr, hdr);  in arc_read_done()
5607 	arc_buf_hdr_t *hdr = NULL;  in arc_cached()  local
5615 	hdr = buf_hash_find(guid, bp, &hash_lock);  in arc_cached()
5616 	if (hdr == NULL)  in arc_cached()
5619 	if (HDR_HAS_L1HDR(hdr)) {  in arc_cached()
5620 		arc_state_t *state = hdr->b_l1hdr.b_state;  in arc_cached()
5643 	if (HDR_HAS_L2HDR(hdr))  in arc_cached()
5674 	arc_buf_hdr_t *hdr = NULL;  in arc_read()  local
5711 		hdr = buf_hash_find(guid, bp, &hash_lock);  in arc_read()
5721 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||  in arc_read()
5722 	    (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {  in arc_read()
5723 		boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);  in arc_read()
5737 		if (HDR_IO_IN_PROGRESS(hdr)) {  in arc_read()
5745 			zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;  in arc_read()
5747 			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&  in arc_read()
5756 				    arc_buf_hdr_t *, hdr);  in arc_read()
5760 			DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);  in arc_read()
5761 			arc_access(hdr, *arc_flags, B_FALSE);  in arc_read()
5802 				acb->acb_next = hdr->b_l1hdr.b_acb;  in arc_read()
5803 				hdr->b_l1hdr.b_acb->acb_prev = acb;  in arc_read()
5804 				hdr->b_l1hdr.b_acb = acb;  in arc_read()
5827 		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||  in arc_read()
5828 		    hdr->b_l1hdr.b_state == arc_mfu ||  in arc_read()
5829 		    hdr->b_l1hdr.b_state == arc_uncached);  in arc_read()
5831 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);  in arc_read()
5832 		arc_access(hdr, *arc_flags, B_TRUE);  in arc_read()
5838 			rc = arc_buf_alloc_impl(hdr, spa, zb, private,  in arc_read()
5849 					spa_log_error(spa, zb, hdr->b_birth);  in arc_read()
5858 				(void) remove_reference(hdr, private);  in arc_read()
5921 		if (hdr == NULL) {  in arc_read()
5927 			hdr = arc_hdr_alloc(guid, psize, lsize,  in arc_read()
5931 				hdr->b_dva = *BP_IDENTITY(bp);  in arc_read()
5932 				hdr->b_birth = BP_GET_BIRTH(bp);  in arc_read()
5933 				exists = buf_hash_insert(hdr, &hash_lock);  in arc_read()
5938 				buf_discard_identity(hdr);  in arc_read()
5939 				arc_hdr_destroy(hdr);  in arc_read()
5949 			if (!HDR_HAS_L1HDR(hdr)) {  in arc_read()
5950 				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,  in arc_read()
5954 			if (GHOST_STATE(hdr->b_l1hdr.b_state)) {  in arc_read()
5955 				ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_read()
5956 				ASSERT(!HDR_HAS_RABD(hdr));  in arc_read()
5957 				ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_read()
5959 				    &hdr->b_l1hdr.b_refcnt));  in arc_read()
5960 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);  in arc_read()
5962 				ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);  in arc_read()
5964 			} else if (HDR_IO_IN_PROGRESS(hdr)) {  in arc_read()
5982 				    hdr->b_l1hdr.b_acb->acb_zio_head;  in arc_read()
5983 				acb->acb_next = hdr->b_l1hdr.b_acb;  in arc_read()
5984 				hdr->b_l1hdr.b_acb->acb_prev = acb;  in arc_read()
5985 				hdr->b_l1hdr.b_acb = acb;  in arc_read()
6000 			arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);  in arc_read()
6011 		add_reference(hdr, hdr);  in arc_read()
6013 			arc_access(hdr, *arc_flags, B_FALSE);  in arc_read()
6014 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);  in arc_read()
6015 		arc_hdr_alloc_abd(hdr, alloc_flags);  in arc_read()
6017 			ASSERT(HDR_HAS_RABD(hdr));  in arc_read()
6018 			size = HDR_GET_PSIZE(hdr);  in arc_read()
6019 			hdr_abd = hdr->b_crypt_hdr.b_rabd;  in arc_read()
6022 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in arc_read()
6023 			size = arc_hdr_size(hdr);  in arc_read()
6024 			hdr_abd = hdr->b_l1hdr.b_pabd;  in arc_read()
6026 			if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {  in arc_read()
6042 			arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);  in arc_read()
6044 			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);  in arc_read()
6045 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));  in arc_read()
6056 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);  in arc_read()
6057 		hdr->b_l1hdr.b_acb = acb;  in arc_read()
6059 		if (HDR_HAS_L2HDR(hdr) &&  in arc_read()
6060 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {  in arc_read()
6061 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;  in arc_read()
6062 			addr = hdr->b_l2hdr.b_daddr;  in arc_read()
6078 			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);  in arc_read()
6080 			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);  in arc_read()
6086 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);  in arc_read()
6094 			DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,  in arc_read()
6099 			    demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,  in arc_read()
6117 			if (HDR_HAS_L2HDR(hdr) &&  in arc_read()
6118 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {  in arc_read()
6123 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);  in arc_read()
6125 				hdr->b_l2hdr.b_hits++;  in arc_read()
6129 				cb->l2rcb_hdr = hdr;  in arc_read()
6139 				if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&  in arc_read()
6140 				    !HDR_COMPRESSION_ENABLED(hdr) &&  in arc_read()
6141 				    HDR_GET_PSIZE(hdr) != 0) {  in arc_read()
6142 					size = HDR_GET_PSIZE(hdr);  in arc_read()
6148 					    HDR_ISTYPE_METADATA(hdr));  in arc_read()
6164 				ASSERT3U(arc_hdr_get_compress(hdr), !=,  in arc_read()
6181 				    HDR_GET_PSIZE(hdr));  in arc_read()
6197 				    arc_buf_hdr_t *, hdr);  in arc_read()
6199 				if (HDR_L2_WRITING(hdr))  in arc_read()
6221 					    arc_buf_hdr_t *, hdr);  in arc_read()
6228 		    arc_read_done, hdr, priority, zio_flags, zb);  in arc_read()
6354 	arc_buf_hdr_t *hdr;  in arc_freed()  local
6360 	hdr = buf_hash_find(guid, bp, &hash_lock);  in arc_freed()
6361 	if (hdr == NULL)  in arc_freed()
6384 	if (!HDR_HAS_L1HDR(hdr) ||  in arc_freed()
6385 	    zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {  in arc_freed()
6386 		arc_change_state(arc_anon, hdr);  in arc_freed()
6387 		arc_hdr_destroy(hdr);  in arc_freed()
6404 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_release()  local
6412 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_release()
6419 	if (hdr->b_l1hdr.b_state == arc_anon) {  in arc_release()
6420 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_release()
6421 		ASSERT(!HDR_IN_HASH_TABLE(hdr));  in arc_release()
6422 		ASSERT(!HDR_HAS_L2HDR(hdr));  in arc_release()
6424 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);  in arc_release()
6426 		ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);  in arc_release()
6427 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));  in arc_release()
6429 		hdr->b_l1hdr.b_arc_access = 0;  in arc_release()
6435 		buf_discard_identity(hdr);  in arc_release()
6441 	kmutex_t *hash_lock = HDR_LOCK(hdr);  in arc_release()
6449 	arc_state_t *state = hdr->b_l1hdr.b_state;  in arc_release()
6450 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));  in arc_release()
6455 	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);  in arc_release()
6460 	if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {  in arc_release()
6462 		uint64_t spa = hdr->b_spa;  in arc_release()
6463 		uint64_t psize = HDR_GET_PSIZE(hdr);  in arc_release()
6464 		uint64_t lsize = HDR_GET_LSIZE(hdr);  in arc_release()
6465 		boolean_t protected = HDR_PROTECTED(hdr);  in arc_release()
6466 		enum zio_compress compress = arc_hdr_get_compress(hdr);  in arc_release()
6467 		arc_buf_contents_t type = arc_buf_type(hdr);  in arc_release()
6470 			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);  in arc_release()
6478 		VERIFY3S(remove_reference(hdr, tag), >, 0);  in arc_release()
6479 		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);  in arc_release()
6493 			arc_unshare_buf(hdr, buf);  in arc_release()
6501 			if (arc_can_share(hdr, lastbuf)) {  in arc_release()
6502 				arc_share_buf(hdr, lastbuf);  in arc_release()
6504 				arc_hdr_alloc_abd(hdr, 0);  in arc_release()
6505 				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,  in arc_release()
6508 		} else if (HDR_SHARED_DATA(hdr)) {  in arc_release()
6518 			    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);  in arc_release()
6522 		ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));  in arc_release()
6531 		if (!arc_hdr_has_uncompressed_buf(hdr))  in arc_release()
6532 			arc_cksum_free(hdr);  in arc_release()
6537 		    compress, hdr->b_complevel, type);  in arc_release()
6550 		ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);  in arc_release()
6552 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));  in arc_release()
6553 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_release()
6555 		if (HDR_HAS_L2HDR(hdr)) {  in arc_release()
6556 			mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);  in arc_release()
6558 			if (HDR_HAS_L2HDR(hdr))  in arc_release()
6559 				arc_hdr_l2hdr_destroy(hdr);  in arc_release()
6560 			mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);  in arc_release()
6563 		hdr->b_l1hdr.b_mru_hits = 0;  in arc_release()
6564 		hdr->b_l1hdr.b_mru_ghost_hits = 0;  in arc_release()
6565 		hdr->b_l1hdr.b_mfu_hits = 0;  in arc_release()
6566 		hdr->b_l1hdr.b_mfu_ghost_hits = 0;  in arc_release()
6567 		arc_change_state(arc_anon, hdr);  in arc_release()
6568 		hdr->b_l1hdr.b_arc_access = 0;  in arc_release()
6571 		buf_discard_identity(hdr);  in arc_release()
6596 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_write_ready()  local
6601 	ASSERT(HDR_HAS_L1HDR(hdr));  in arc_write_ready()
6603 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);  in arc_write_ready()
6611 		arc_cksum_free(hdr);  in arc_write_ready()
6613 		if (hdr->b_l1hdr.b_pabd != NULL) {  in arc_write_ready()
6615 				arc_unshare_buf(hdr, buf);  in arc_write_ready()
6618 				arc_hdr_free_abd(hdr, B_FALSE);  in arc_write_ready()
6622 		if (HDR_HAS_RABD(hdr))  in arc_write_ready()
6623 			arc_hdr_free_abd(hdr, B_TRUE);  in arc_write_ready()
6625 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_write_ready()
6626 	ASSERT(!HDR_HAS_RABD(hdr));  in arc_write_ready()
6627 	ASSERT(!HDR_SHARED_DATA(hdr));  in arc_write_ready()
6632 	if (HDR_IO_IN_PROGRESS(hdr)) {  in arc_write_ready()
6635 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);  in arc_write_ready()
6636 		add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */  in arc_write_ready()
6645 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;  in arc_write_ready()
6647 				hdr->b_l1hdr.b_byteswap =  in arc_write_ready()
6651 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;  in arc_write_ready()
6654 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);  in arc_write_ready()
6655 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);  in arc_write_ready()
6656 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;  in arc_write_ready()
6657 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,  in arc_write_ready()
6658 		    hdr->b_crypt_hdr.b_iv);  in arc_write_ready()
6659 		zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);  in arc_write_ready()
6661 		arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);  in arc_write_ready()
6669 		arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);  in arc_write_ready()
6685 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));  in arc_write_ready()
6688 	HDR_SET_PSIZE(hdr, psize);  in arc_write_ready()
6689 	arc_hdr_set_compress(hdr, compress);  in arc_write_ready()
6690 	hdr->b_complevel = zio->io_prop.zp_complevel;  in arc_write_ready()
6712 		arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |  in arc_write_ready()
6714 		abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);  in arc_write_ready()
6715 	} else if (!(HDR_UNCACHED(hdr) ||  in arc_write_ready()
6717 	    !arc_can_share(hdr, buf)) {  in arc_write_ready()
6725 			arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |  in arc_write_ready()
6727 			abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);  in arc_write_ready()
6728 		} else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&  in arc_write_ready()
6731 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);  in arc_write_ready()
6732 			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);  in arc_write_ready()
6734 			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));  in arc_write_ready()
6735 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);  in arc_write_ready()
6736 			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,  in arc_write_ready()
6742 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);  in arc_write_ready()
6745 		arc_share_buf(hdr, buf);  in arc_write_ready()
6749 	arc_hdr_verify(hdr, bp);  in arc_write_ready()
6767 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_write_done()  local
6769 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);  in arc_write_done()
6772 		arc_hdr_verify(hdr, zio->io_bp);  in arc_write_done()
6775 			buf_discard_identity(hdr);  in arc_write_done()
6777 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);  in arc_write_done()
6778 			hdr->b_birth = BP_GET_BIRTH(zio->io_bp);  in arc_write_done()
6781 		ASSERT(HDR_EMPTY(hdr));  in arc_write_done()
6790 	if (!HDR_EMPTY(hdr)) {  in arc_write_done()
6798 		exists = buf_hash_insert(hdr, &hash_lock);  in arc_write_done()
6808 					    (void *)hdr, (void *)exists);  in arc_write_done()
6814 				exists = buf_hash_insert(hdr, &hash_lock);  in arc_write_done()
6821 					    (void *)hdr, (void *)exists);  in arc_write_done()
6824 				ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);  in arc_write_done()
6825 				ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));  in arc_write_done()
6826 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);  in arc_write_done()
6831 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);  in arc_write_done()
6832 		VERIFY3S(remove_reference(hdr, hdr), >, 0);  in arc_write_done()
6834 		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)  in arc_write_done()
6835 			arc_access(hdr, 0, B_FALSE);  in arc_write_done()
6838 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);  in arc_write_done()
6839 		VERIFY3S(remove_reference(hdr, hdr), >, 0);  in arc_write_done()
6856 	arc_buf_hdr_t *hdr = buf->b_hdr;  in arc_write()  local
6863 	ASSERT(!HDR_IO_ERROR(hdr));  in arc_write()
6864 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));  in arc_write()
6865 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);  in arc_write()
6866 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);  in arc_write()
6868 		arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);  in arc_write()
6870 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);  in arc_write()
6875 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);  in arc_write()
6876 		localprop.zp_complevel = hdr->b_complevel;  in arc_write()
6878 		    (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?  in arc_write()
6880 		memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,  in arc_write()
6882 		memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,  in arc_write()
6884 		memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,  in arc_write()
6895 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));  in arc_write()
6896 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);  in arc_write()
6897 		localprop.zp_complevel = hdr->b_complevel;  in arc_write()
6911 	if (hdr->b_l1hdr.b_pabd != NULL) {  in arc_write()
6919 			arc_unshare_buf(hdr, buf);  in arc_write()
6922 			arc_hdr_free_abd(hdr, B_FALSE);  in arc_write()
6927 	if (HDR_HAS_RABD(hdr))  in arc_write()
6928 		arc_hdr_free_abd(hdr, B_TRUE);  in arc_write()
6931 		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);  in arc_write()
6934 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);  in arc_write()
6937 	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),  in arc_write()
6938 	    HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,  in arc_write()
7324 	arc_buf_hdr_t *hdr = obj;  in arc_state_multilist_index_func()  local
7331 	ASSERT(!HDR_EMPTY(hdr));  in arc_state_multilist_index_func()
7346 	return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %  in arc_state_multilist_index_func()
8249 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)  in l2arc_write_eligible()  argument
8258 	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||  in l2arc_write_eligible()
8259 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))  in l2arc_write_eligible()
8434 	arc_buf_hdr_t		*head, *hdr, *hdr_prev;  in l2arc_write_done()  local
8455 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {  in l2arc_write_done()
8456 		hdr_prev = list_prev(buflist, hdr);  in l2arc_write_done()
8458 		hash_lock = HDR_LOCK(hdr);  in l2arc_write_done()
8479 			list_insert_after(buflist, hdr, head);  in l2arc_write_done()
8499 		ASSERT(HDR_HAS_L1HDR(hdr));  in l2arc_write_done()
8509 			list_remove(buflist, hdr);  in l2arc_write_done()
8510 			arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);  in l2arc_write_done()
8512 			uint64_t psize = HDR_GET_PSIZE(hdr);  in l2arc_write_done()
8513 			l2arc_hdr_arcstats_decrement(hdr);  in l2arc_write_done()
8520 			    arc_hdr_size(hdr), hdr);  in l2arc_write_done()
8527 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);  in l2arc_write_done()
8614 	arc_buf_hdr_t *hdr = cb->l2rcb_hdr;  in l2arc_untransform()  local
8626 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));  in l2arc_untransform()
8627 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in l2arc_untransform()
8636 		abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,  in l2arc_untransform()
8644 		    salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,  in l2arc_untransform()
8645 		    hdr->b_l1hdr.b_pabd, &no_crypt);  in l2arc_untransform()
8647 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);  in l2arc_untransform()
8657 			arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,  in l2arc_untransform()
8658 			    arc_hdr_size(hdr), hdr);  in l2arc_untransform()
8659 			hdr->b_l1hdr.b_pabd = eabd;  in l2arc_untransform()
8662 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);  in l2arc_untransform()
8671 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&  in l2arc_untransform()
8672 	    !HDR_COMPRESSION_ENABLED(hdr)) {  in l2arc_untransform()
8673 		abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,  in l2arc_untransform()
8676 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),  in l2arc_untransform()
8677 		    hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),  in l2arc_untransform()
8678 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);  in l2arc_untransform()
8680 			arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);  in l2arc_untransform()
8684 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,  in l2arc_untransform()
8685 		    arc_hdr_size(hdr), hdr);  in l2arc_untransform()
8686 		hdr->b_l1hdr.b_pabd = cabd;  in l2arc_untransform()
8688 		zio->io_size = HDR_GET_LSIZE(hdr);  in l2arc_untransform()
8707 	arc_buf_hdr_t *hdr;  in l2arc_read_done()  local
8719 	hdr = cb->l2rcb_hdr;  in l2arc_read_done()
8720 	ASSERT3P(hdr, !=, NULL);  in l2arc_read_done()
8722 	hash_lock = HDR_LOCK(hdr);  in l2arc_read_done()
8724 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));  in l2arc_read_done()
8731 		ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);  in l2arc_read_done()
8734 				abd_copy(hdr->b_crypt_hdr.b_rabd,  in l2arc_read_done()
8735 				    cb->l2rcb_abd, arc_hdr_size(hdr));  in l2arc_read_done()
8737 				abd_copy(hdr->b_l1hdr.b_pabd,  in l2arc_read_done()
8738 				    cb->l2rcb_abd, arc_hdr_size(hdr));  in l2arc_read_done()
8754 		zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);  in l2arc_read_done()
8757 			ASSERT(HDR_HAS_RABD(hdr));  in l2arc_read_done()
8759 			    hdr->b_crypt_hdr.b_rabd;  in l2arc_read_done()
8761 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);  in l2arc_read_done()
8762 			zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;  in l2arc_read_done()
8771 	ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||  in l2arc_read_done()
8772 	    (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));  in l2arc_read_done()
8775 	zio->io_prop.zp_complevel = hdr->b_complevel;  in l2arc_read_done()
8777 	valid_cksum = arc_cksum_is_equal(hdr, zio);  in l2arc_read_done()
8788 	    !HDR_L2_EVICTED(hdr)) {  in l2arc_read_done()
8790 		zio->io_private = hdr;  in l2arc_read_done()
8813 			    hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;  in l2arc_read_done()
8819 			    hdr, zio->io_priority, cb->l2rcb_flags,  in l2arc_read_done()
8827 			for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;  in l2arc_read_done()
8921 	arc_buf_hdr_t *hdr, *hdr_prev;  in l2arc_evict()  local
9038 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {  in l2arc_evict()
9039 		hdr_prev = list_prev(buflist, hdr);  in l2arc_evict()
9041 		ASSERT(!HDR_EMPTY(hdr));  in l2arc_evict()
9042 		hash_lock = HDR_LOCK(hdr);  in l2arc_evict()
9063 		ASSERT(HDR_HAS_L2HDR(hdr));  in l2arc_evict()
9066 		ASSERT(!HDR_L2_WRITING(hdr));  in l2arc_evict()
9067 		ASSERT(!HDR_L2_WRITE_HEAD(hdr));  in l2arc_evict()
9069 		if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||  in l2arc_evict()
9070 		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {  in l2arc_evict()
9079 		if (!HDR_HAS_L1HDR(hdr)) {  in l2arc_evict()
9080 			ASSERT(!HDR_L2_READING(hdr));  in l2arc_evict()
9086 			arc_change_state(arc_anon, hdr);  in l2arc_evict()
9087 			arc_hdr_destroy(hdr);  in l2arc_evict()
9089 			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);  in l2arc_evict()
9096 			if (HDR_L2_READING(hdr)) {  in l2arc_evict()
9098 				arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);  in l2arc_evict()
9101 			arc_hdr_l2hdr_destroy(hdr);  in l2arc_evict()
9141 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,  in l2arc_apply_transforms()  argument
9145 	abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;  in l2arc_apply_transforms()
9146 	enum zio_compress compress = HDR_GET_COMPRESS(hdr);  in l2arc_apply_transforms()
9147 	uint64_t psize = HDR_GET_PSIZE(hdr);  in l2arc_apply_transforms()
9148 	uint64_t size = arc_hdr_size(hdr);  in l2arc_apply_transforms()
9149 	boolean_t ismd = HDR_ISTYPE_METADATA(hdr);  in l2arc_apply_transforms()
9150 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);  in l2arc_apply_transforms()
9155 	ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&  in l2arc_apply_transforms()
9156 	    !HDR_COMPRESSION_ENABLED(hdr)) ||  in l2arc_apply_transforms()
9157 	    HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize);  in l2arc_apply_transforms()
9165 	if (HDR_HAS_RABD(hdr)) {  in l2arc_apply_transforms()
9168 		abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);  in l2arc_apply_transforms()
9173 	if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&  in l2arc_apply_transforms()
9174 	    !HDR_ENCRYPTED(hdr)) {  in l2arc_apply_transforms()
9177 		abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);  in l2arc_apply_transforms()
9183 	if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {  in l2arc_apply_transforms()
9186 		    size, MIN(size, psize), hdr->b_complevel);  in l2arc_apply_transforms()
9201 	if (HDR_ENCRYPTED(hdr)) {  in l2arc_apply_transforms()
9210 		ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,  in l2arc_apply_transforms()
9216 		    hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,  in l2arc_apply_transforms()
9217 		    hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,  in l2arc_apply_transforms()
9229 		ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));  in l2arc_apply_transforms()
9239 	ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);  in l2arc_apply_transforms()
9281 	arc_buf_hdr_t 		*hdr, *head, *marker;  in l2arc_write_buffers()  local
9328 			hdr = multilist_sublist_head(mls);  in l2arc_write_buffers()
9330 			hdr = multilist_sublist_tail(mls);  in l2arc_write_buffers()
9332 		while (hdr != NULL) {  in l2arc_write_buffers()
9336 			hash_lock = HDR_LOCK(hdr);  in l2arc_write_buffers()
9341 					hdr = multilist_sublist_next(mls, hdr);  in l2arc_write_buffers()
9343 					hdr = multilist_sublist_prev(mls, hdr);  in l2arc_write_buffers()
9347 			passed_sz += HDR_GET_LSIZE(hdr);  in l2arc_write_buffers()
9356 			if (!l2arc_write_eligible(guid, hdr)) {  in l2arc_write_buffers()
9361 			ASSERT(HDR_HAS_L1HDR(hdr));  in l2arc_write_buffers()
9362 			ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);  in l2arc_write_buffers()
9363 			ASSERT3U(arc_hdr_size(hdr), >, 0);  in l2arc_write_buffers()
9364 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||  in l2arc_write_buffers()
9365 			    HDR_HAS_RABD(hdr));  in l2arc_write_buffers()
9366 			uint64_t psize = HDR_GET_PSIZE(hdr);  in l2arc_write_buffers()
9388 				multilist_sublist_insert_after(mls, hdr,  in l2arc_write_buffers()
9391 				multilist_sublist_insert_before(mls, hdr,  in l2arc_write_buffers()
9410 			if (HDR_HAS_RABD(hdr) && psize == asize) {  in l2arc_write_buffers()
9411 				to_write = hdr->b_crypt_hdr.b_rabd;  in l2arc_write_buffers()
9412 			} else if ((HDR_COMPRESSION_ENABLED(hdr) ||  in l2arc_write_buffers()
9413 			    HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&  in l2arc_write_buffers()
9414 			    !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&  in l2arc_write_buffers()
9416 				to_write = hdr->b_l1hdr.b_pabd;  in l2arc_write_buffers()
9419 				arc_buf_contents_t type = arc_buf_type(hdr);  in l2arc_write_buffers()
9421 				ret = l2arc_apply_transforms(spa, hdr, asize,  in l2arc_write_buffers()
9424 					arc_hdr_clear_flags(hdr,  in l2arc_write_buffers()
9433 			hdr->b_l2hdr.b_dev = dev;  in l2arc_write_buffers()
9434 			hdr->b_l2hdr.b_daddr = dev->l2ad_hand;  in l2arc_write_buffers()
9435 			hdr->b_l2hdr.b_hits = 0;  in l2arc_write_buffers()
9436 			hdr->b_l2hdr.b_arcs_state =  in l2arc_write_buffers()
9437 			    hdr->b_l1hdr.b_state->arcs_state;  in l2arc_write_buffers()
9439 			HDR_SET_L2SIZE(hdr, asize);  in l2arc_write_buffers()
9440 			arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR |  in l2arc_write_buffers()
9444 			    arc_hdr_size(hdr), hdr);  in l2arc_write_buffers()
9445 			l2arc_hdr_arcstats_increment(hdr);  in l2arc_write_buffers()
9457 			list_insert_head(&dev->l2ad_buflist, hdr);  in l2arc_write_buffers()
9460 			boolean_t commit = l2arc_log_blk_insert(dev, hdr);  in l2arc_write_buffers()
9477 			    ZIO_CHECKSUM_OFF, NULL, hdr,  in l2arc_write_buffers()
9498 				hdr = multilist_sublist_next(mls, marker);  in l2arc_write_buffers()
9500 				hdr = multilist_sublist_prev(mls, marker);  in l2arc_write_buffers()
10588 	arc_buf_hdr_t		*hdr, *exists;  in l2arc_hdr_restore()  local
10599 	hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,  in l2arc_hdr_restore()
10611 	l2arc_hdr_arcstats_increment(hdr);  in l2arc_hdr_restore()
10615 	list_insert_tail(&dev->l2ad_buflist, hdr);  in l2arc_hdr_restore()
10616 	(void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);  in l2arc_hdr_restore()
10619 	exists = buf_hash_insert(hdr, &hash_lock);  in l2arc_hdr_restore()
10622 		arc_hdr_destroy(hdr);  in l2arc_hdr_restore()
10915 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr)  in l2arc_log_blk_insert()  argument
10926 	ASSERT(HDR_HAS_L2HDR(hdr));  in l2arc_log_blk_insert()
10930 	le->le_dva = hdr->b_dva;  in l2arc_log_blk_insert()
10931 	le->le_birth = hdr->b_birth;  in l2arc_log_blk_insert()
10932 	le->le_daddr = hdr->b_l2hdr.b_daddr;  in l2arc_log_blk_insert()
10935 	L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));  in l2arc_log_blk_insert()
10936 	L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));  in l2arc_log_blk_insert()
10937 	L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));  in l2arc_log_blk_insert()
10938 	le->le_complevel = hdr->b_complevel;  in l2arc_log_blk_insert()
10939 	L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);  in l2arc_log_blk_insert()
10940 	L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));  in l2arc_log_blk_insert()
10941 	L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));  in l2arc_log_blk_insert()
10942 	L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);  in l2arc_log_blk_insert()
10945 	    HDR_GET_PSIZE(hdr));  in l2arc_log_blk_insert()