Lines Matching full:hdr
208 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
210 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
211 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
212 * hdr is compressed and the desired compression characteristics of the
215 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
216 * be anywhere in the hdr's list.
218 * The diagram below shows an example of an uncompressed ARC hdr that is
249 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
256 * buffers which have been released (and hence have their own hdr, if there
257 * were originally other readers of the buf's original hdr). This ensures that
258 * the ARC only needs to update a single buf and its hdr after a write occurs.
279 * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
703 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) argument
704 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) argument
705 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) argument
706 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) argument
707 #define HDR_PRESCIENT_PREFETCH(hdr) \ argument
708 ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
709 #define HDR_COMPRESSION_ENABLED(hdr) \ argument
710 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
712 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) argument
713 #define HDR_UNCACHED(hdr) ((hdr)->b_flags & ARC_FLAG_UNCACHED) argument
714 #define HDR_L2_READING(hdr) \ argument
715 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
716 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
717 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) argument
718 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) argument
719 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) argument
720 #define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED) argument
721 #define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH) argument
722 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA) argument
724 #define HDR_ISTYPE_METADATA(hdr) \ argument
725 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
726 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) argument
728 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) argument
729 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) argument
730 #define HDR_HAS_RABD(hdr) \ argument
731 (HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) && \
732 (hdr)->b_crypt_hdr.b_rabd != NULL)
733 #define HDR_ENCRYPTED(hdr) \ argument
734 (HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
735 #define HDR_AUTHENTICATED(hdr) \ argument
736 (HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
741 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \ argument
743 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \ argument
774 #define HDR_LOCK(hdr) \ argument
775 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
858 ARC_FILL_LOCKED = 1 << 0, /* hdr lock is held */
890 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
901 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
902 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
907 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
912 #define l2arc_hdr_arcstats_increment(hdr) \ argument
913 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
914 #define l2arc_hdr_arcstats_decrement(hdr) \ argument
915 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
916 #define l2arc_hdr_arcstats_increment_state(hdr) \ argument
917 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
918 #define l2arc_hdr_arcstats_decrement_state(hdr) \ argument
919 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
1014 #define HDR_EMPTY(hdr) \ argument
1015 ((hdr)->b_dva.dva_word[0] == 0 && \
1016 (hdr)->b_dva.dva_word[1] == 0)
1018 #define HDR_EMPTY_OR_LOCKED(hdr) \ argument
1019 (HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
1021 #define HDR_EQUAL(spa, dva, birth, hdr) \ argument
1022 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1023 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1024 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1027 buf_discard_identity(arc_buf_hdr_t *hdr) in buf_discard_identity() argument
1029 hdr->b_dva.dva_word[0] = 0; in buf_discard_identity()
1030 hdr->b_dva.dva_word[1] = 0; in buf_discard_identity()
1031 hdr->b_birth = 0; in buf_discard_identity()
1041 arc_buf_hdr_t *hdr; in buf_hash_find() local
1044 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; in buf_hash_find()
1045 hdr = hdr->b_hash_next) { in buf_hash_find()
1046 if (HDR_EQUAL(spa, dva, birth, hdr)) { in buf_hash_find()
1048 return (hdr); in buf_hash_find()
1064 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) in buf_hash_insert() argument
1066 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_insert()
1071 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); in buf_hash_insert()
1072 ASSERT(hdr->b_birth != 0); in buf_hash_insert()
1073 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in buf_hash_insert()
1084 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) in buf_hash_insert()
1088 hdr->b_hash_next = buf_hash_table.ht_table[idx]; in buf_hash_insert()
1089 buf_hash_table.ht_table[idx] = hdr; in buf_hash_insert()
1090 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in buf_hash_insert()
1105 buf_hash_remove(arc_buf_hdr_t *hdr) in buf_hash_remove() argument
1108 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_remove()
1111 ASSERT(HDR_IN_HASH_TABLE(hdr)); in buf_hash_remove()
1114 while ((fhdr = *hdrp) != hdr) { in buf_hash_remove()
1118 *hdrp = hdr->b_hash_next; in buf_hash_remove()
1119 hdr->b_hash_next = NULL; in buf_hash_remove()
1120 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in buf_hash_remove()
1166 arc_buf_hdr_t *hdr = vbuf; in hdr_full_cons() local
1168 memset(hdr, 0, HDR_FULL_SIZE); in hdr_full_cons()
1169 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in hdr_full_cons()
1170 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt); in hdr_full_cons()
1172 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); in hdr_full_cons()
1174 multilist_link_init(&hdr->b_l1hdr.b_arc_node); in hdr_full_cons()
1175 list_link_init(&hdr->b_l2hdr.b_l2node); in hdr_full_cons()
1185 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_cons() local
1187 memset(hdr, 0, HDR_L2ONLY_SIZE); in hdr_l2only_cons()
1213 arc_buf_hdr_t *hdr = vbuf; in hdr_full_dest() local
1215 ASSERT(HDR_EMPTY(hdr)); in hdr_full_dest()
1216 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt); in hdr_full_dest()
1218 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); in hdr_full_dest()
1220 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in hdr_full_dest()
1228 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_dest() local
1230 ASSERT(HDR_EMPTY(hdr)); in hdr_l2only_dest()
1337 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_get_raw_params() local
1339 ASSERT(HDR_PROTECTED(hdr)); in arc_get_raw_params()
1341 memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); in arc_get_raw_params()
1342 memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); in arc_get_raw_params()
1343 memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); in arc_get_raw_params()
1344 *byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ? in arc_get_raw_params()
1366 arc_hdr_get_compress(arc_buf_hdr_t *hdr) in arc_hdr_get_compress() argument
1368 return (HDR_COMPRESSION_ENABLED(hdr) ? in arc_hdr_get_compress()
1369 HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF); in arc_hdr_get_compress()
1390 * It would be nice to assert arc_can_share() too, but the "hdr isn't in arc_buf_is_shared()
1402 arc_cksum_free(arc_buf_hdr_t *hdr) in arc_cksum_free() argument
1405 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_free()
1407 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_free()
1408 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { in arc_cksum_free()
1409 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t)); in arc_cksum_free()
1410 hdr->b_l1hdr.b_freeze_cksum = NULL; in arc_cksum_free()
1412 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_free()
1417 * Return true iff at least one of the bufs on hdr is not compressed.
1421 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr) in arc_hdr_has_uncompressed_buf() argument
1423 ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_has_uncompressed_buf()
1425 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) { in arc_hdr_has_uncompressed_buf()
1436 * matches the checksum that is stored in the hdr. If there is no checksum,
1443 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_cksum_verify() local
1452 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_verify()
1454 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1456 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) { in arc_cksum_verify()
1457 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1462 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) in arc_cksum_verify()
1464 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1474 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio) in arc_cksum_is_equal() argument
1477 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr)); in arc_cksum_is_equal()
1499 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1501 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1510 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_cksum_compute() local
1511 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_compute()
1512 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1513 if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) { in arc_cksum_compute()
1514 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1520 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), in arc_cksum_compute()
1523 hdr->b_l1hdr.b_freeze_cksum); in arc_cksum_compute()
1524 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1564 arc_buf_type(arc_buf_hdr_t *hdr) in arc_buf_type() argument
1567 if (HDR_ISTYPE_METADATA(hdr)) { in arc_buf_type()
1572 VERIFY3U(hdr->b_type, ==, type); in arc_buf_type()
1601 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_thaw() local
1603 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_buf_thaw()
1604 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_buf_thaw()
1614 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_thaw()
1615 arc_cksum_free(hdr); in arc_buf_thaw()
1636 * the hash_lock must be held or the hdr must be undiscoverable. This
1641 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) in arc_hdr_set_flags() argument
1643 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_set_flags()
1644 hdr->b_flags |= flags; in arc_hdr_set_flags()
1648 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) in arc_hdr_clear_flags() argument
1650 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_clear_flags()
1651 hdr->b_flags &= ~flags; in arc_hdr_clear_flags()
1662 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp) in arc_hdr_set_compress() argument
1664 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_set_compress()
1671 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) { in arc_hdr_set_compress()
1672 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC); in arc_hdr_set_compress()
1673 ASSERT(!HDR_COMPRESSION_ENABLED(hdr)); in arc_hdr_set_compress()
1675 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC); in arc_hdr_set_compress()
1676 ASSERT(HDR_COMPRESSION_ENABLED(hdr)); in arc_hdr_set_compress()
1679 HDR_SET_COMPRESS(hdr, cmp); in arc_hdr_set_compress()
1680 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp); in arc_hdr_set_compress()
1684 * Looks for another buf on the same hdr which has the data decompressed, copies
1690 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_try_copy_decompressed_data() local
1693 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_try_copy_decompressed_data()
1697 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL; in arc_buf_try_copy_decompressed_data()
1714 * checksum on the hdr either. in arc_buf_try_copy_decompressed_data()
1717 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL); in arc_buf_try_copy_decompressed_data()
1735 arc_buf_hdr_t *hdr; in arc_buf_alloc_l2only() local
1740 hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP); in arc_buf_alloc_l2only()
1741 hdr->b_birth = birth; in arc_buf_alloc_l2only()
1742 hdr->b_type = type; in arc_buf_alloc_l2only()
1743 hdr->b_flags = 0; in arc_buf_alloc_l2only()
1744 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR); in arc_buf_alloc_l2only()
1745 HDR_SET_LSIZE(hdr, size); in arc_buf_alloc_l2only()
1746 HDR_SET_PSIZE(hdr, psize); in arc_buf_alloc_l2only()
1747 HDR_SET_L2SIZE(hdr, asize); in arc_buf_alloc_l2only()
1748 arc_hdr_set_compress(hdr, compress); in arc_buf_alloc_l2only()
1749 hdr->b_complevel = complevel; in arc_buf_alloc_l2only()
1751 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); in arc_buf_alloc_l2only()
1753 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_buf_alloc_l2only()
1754 hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa); in arc_buf_alloc_l2only()
1756 hdr->b_dva = dva; in arc_buf_alloc_l2only()
1758 hdr->b_l2hdr.b_dev = dev; in arc_buf_alloc_l2only()
1759 hdr->b_l2hdr.b_daddr = daddr; in arc_buf_alloc_l2only()
1760 hdr->b_l2hdr.b_arcs_state = arcs_state; in arc_buf_alloc_l2only()
1762 return (hdr); in arc_buf_alloc_l2only()
1769 arc_hdr_size(arc_buf_hdr_t *hdr) in arc_hdr_size() argument
1773 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_size()
1774 HDR_GET_PSIZE(hdr) > 0) { in arc_hdr_size()
1775 size = HDR_GET_PSIZE(hdr); in arc_hdr_size()
1777 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0); in arc_hdr_size()
1778 size = HDR_GET_LSIZE(hdr); in arc_hdr_size()
1784 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) in arc_hdr_authenticate() argument
1788 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_hdr_authenticate()
1789 uint64_t psize = HDR_GET_PSIZE(hdr); in arc_hdr_authenticate()
1790 abd_t *abd = hdr->b_l1hdr.b_pabd; in arc_hdr_authenticate()
1793 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_authenticate()
1794 ASSERT(HDR_AUTHENTICATED(hdr)); in arc_hdr_authenticate()
1805 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_authenticate()
1806 !HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_authenticate()
1808 csize = zio_compress_data(HDR_GET_COMPRESS(hdr), in arc_hdr_authenticate()
1809 hdr->b_l1hdr.b_pabd, &abd, lsize, MIN(lsize, psize), in arc_hdr_authenticate()
1810 hdr->b_complevel); in arc_hdr_authenticate()
1824 if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) { in arc_hdr_authenticate()
1825 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); in arc_hdr_authenticate()
1828 psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); in arc_hdr_authenticate()
1831 hdr->b_crypt_hdr.b_mac); in arc_hdr_authenticate()
1835 arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH); in arc_hdr_authenticate()
1852 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb) in arc_hdr_decrypt() argument
1857 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); in arc_hdr_decrypt()
1859 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_decrypt()
1860 ASSERT(HDR_ENCRYPTED(hdr)); in arc_hdr_decrypt()
1862 arc_hdr_alloc_abd(hdr, 0); in arc_hdr_decrypt()
1864 ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot, in arc_hdr_decrypt()
1865 B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv, in arc_hdr_decrypt()
1866 hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd, in arc_hdr_decrypt()
1867 hdr->b_crypt_hdr.b_rabd, &no_crypt); in arc_hdr_decrypt()
1872 abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd, in arc_hdr_decrypt()
1873 HDR_GET_PSIZE(hdr)); in arc_hdr_decrypt()
1881 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_decrypt()
1882 !HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_decrypt()
1889 cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0); in arc_hdr_decrypt()
1891 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), in arc_hdr_decrypt()
1892 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr), in arc_hdr_decrypt()
1893 HDR_GET_LSIZE(hdr), &hdr->b_complevel); in arc_hdr_decrypt()
1898 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in arc_hdr_decrypt()
1899 arc_hdr_size(hdr), hdr); in arc_hdr_decrypt()
1900 hdr->b_l1hdr.b_pabd = cabd; in arc_hdr_decrypt()
1906 arc_hdr_free_abd(hdr, B_FALSE); in arc_hdr_decrypt()
1908 arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr); in arc_hdr_decrypt()
1919 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa, in arc_fill_hdr_crypt() argument
1924 ASSERT(HDR_PROTECTED(hdr)); in arc_fill_hdr_crypt()
1929 if (HDR_NOAUTH(hdr) && !noauth) { in arc_fill_hdr_crypt()
1934 ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset); in arc_fill_hdr_crypt()
1937 } else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) { in arc_fill_hdr_crypt()
1943 ret = arc_hdr_decrypt(hdr, spa, zb); in arc_fill_hdr_crypt()
1948 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_fill_hdr_crypt()
1971 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_untransform_in_place() local
1973 ASSERT(HDR_ENCRYPTED(hdr)); in arc_buf_untransform_in_place()
1974 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE); in arc_buf_untransform_in_place()
1975 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_untransform_in_place()
1976 ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf); in arc_buf_untransform_in_place()
1978 zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data, in arc_buf_untransform_in_place()
1987 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
1992 * buf with uncompressed data. You can't request a compressed buf on a hdr with
2002 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_fill() local
2004 (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); in arc_buf_fill()
2007 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; in arc_buf_fill()
2008 kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr); in arc_buf_fill()
2013 IMPLY(encrypted, HDR_ENCRYPTED(hdr)); in arc_buf_fill()
2024 ASSERT(HDR_HAS_RABD(hdr)); in arc_buf_fill()
2025 abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd, in arc_buf_fill()
2026 HDR_GET_PSIZE(hdr)); in arc_buf_fill()
2036 if (HDR_PROTECTED(hdr)) { in arc_buf_fill()
2037 error = arc_fill_hdr_crypt(hdr, hash_lock, spa, in arc_buf_fill()
2044 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_buf_fill()
2067 if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) { in arc_buf_fill()
2068 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE); in arc_buf_fill()
2076 /* Compute the hdr's checksum if necessary */ in arc_buf_fill()
2087 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd, in arc_buf_fill()
2095 * If the buf is sharing its data with the hdr, unlink it and in arc_buf_fill()
2105 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); in arc_buf_fill()
2106 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_fill()
2109 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2114 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr), in arc_buf_fill()
2117 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); in arc_buf_fill()
2121 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr)); in arc_buf_fill()
2133 * bite the bullet and decompress the data from the hdr. in arc_buf_fill()
2141 HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2142 error = zio_decompress_data(HDR_GET_COMPRESS(hdr), in arc_buf_fill()
2143 hdr->b_l1hdr.b_pabd, &dabd, in arc_buf_fill()
2144 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr), in arc_buf_fill()
2145 &hdr->b_complevel); in arc_buf_fill()
2154 "hdr %px, compress %d, psize %d, lsize %d", in arc_buf_fill()
2155 hdr, arc_hdr_get_compress(hdr), in arc_buf_fill()
2156 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2159 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_buf_fill()
2170 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_buf_fill()
2172 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2175 /* Compute the hdr's checksum if necessary */ in arc_buf_fill()
2214 * We account for the space used by the hdr and the arc buf individually
2218 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) in arc_evictable_space_increment() argument
2220 arc_buf_contents_t type = arc_buf_type(hdr); in arc_evictable_space_increment()
2222 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evictable_space_increment()
2225 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evictable_space_increment()
2226 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_evictable_space_increment()
2227 ASSERT(!HDR_HAS_RABD(hdr)); in arc_evictable_space_increment()
2229 HDR_GET_LSIZE(hdr), hdr); in arc_evictable_space_increment()
2233 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_evictable_space_increment()
2235 arc_hdr_size(hdr), hdr); in arc_evictable_space_increment()
2237 if (HDR_HAS_RABD(hdr)) { in arc_evictable_space_increment()
2239 HDR_GET_PSIZE(hdr), hdr); in arc_evictable_space_increment()
2242 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_evictable_space_increment()
2253 * We account for the space used by the hdr and the arc buf individually
2257 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) in arc_evictable_space_decrement() argument
2259 arc_buf_contents_t type = arc_buf_type(hdr); in arc_evictable_space_decrement()
2261 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evictable_space_decrement()
2264 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evictable_space_decrement()
2265 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_evictable_space_decrement()
2266 ASSERT(!HDR_HAS_RABD(hdr)); in arc_evictable_space_decrement()
2268 HDR_GET_LSIZE(hdr), hdr); in arc_evictable_space_decrement()
2272 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_evictable_space_decrement()
2274 arc_hdr_size(hdr), hdr); in arc_evictable_space_decrement()
2276 if (HDR_HAS_RABD(hdr)) { in arc_evictable_space_decrement()
2278 HDR_GET_PSIZE(hdr), hdr); in arc_evictable_space_decrement()
2281 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_evictable_space_decrement()
2291 * Add a reference to this hdr indicating that someone is actively
2297 add_reference(arc_buf_hdr_t *hdr, const void *tag) in add_reference() argument
2299 arc_state_t *state = hdr->b_l1hdr.b_state; in add_reference()
2301 ASSERT(HDR_HAS_L1HDR(hdr)); in add_reference()
2302 if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) { in add_reference()
2304 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in add_reference()
2305 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in add_reference()
2308 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && in add_reference()
2311 multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr); in add_reference()
2312 arc_evictable_space_decrement(hdr, state); in add_reference()
2317 * Remove a reference from this hdr. When the reference transitions from
2318 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2322 remove_reference(arc_buf_hdr_t *hdr, const void *tag) in remove_reference() argument
2325 arc_state_t *state = hdr->b_l1hdr.b_state; in remove_reference()
2327 ASSERT(HDR_HAS_L1HDR(hdr)); in remove_reference()
2328 ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr))); in remove_reference()
2331 if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0) in remove_reference()
2335 arc_hdr_destroy(hdr); in remove_reference()
2338 if (state == arc_uncached && !HDR_PREFETCH(hdr)) { in remove_reference()
2339 arc_change_state(arc_anon, hdr); in remove_reference()
2340 arc_hdr_destroy(hdr); in remove_reference()
2343 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr); in remove_reference()
2344 arc_evictable_space_increment(hdr, state); in remove_reference()
2359 arc_buf_hdr_t *hdr = ab->b_hdr; in arc_buf_info() local
2366 if (hdr == NULL) in arc_buf_info()
2369 abi->abi_flags = hdr->b_flags; in arc_buf_info()
2371 if (HDR_HAS_L1HDR(hdr)) { in arc_buf_info()
2372 l1hdr = &hdr->b_l1hdr; in arc_buf_info()
2375 if (HDR_HAS_L2HDR(hdr)) in arc_buf_info()
2376 l2hdr = &hdr->b_l2hdr; in arc_buf_info()
2396 abi->abi_state_contents = arc_buf_type(hdr); in arc_buf_info()
2397 abi->abi_size = arc_hdr_size(hdr); in arc_buf_info()
2405 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr) in arc_change_state() argument
2410 arc_buf_contents_t type = arc_buf_type(hdr); in arc_change_state()
2413 * We almost always have an L1 hdr here, since we call arc_hdr_realloc() in arc_change_state()
2415 * L1 hdr doesn't always exist when we change state to arc_anon before in arc_change_state()
2416 * destroying a header, in which case reallocating to add the L1 hdr is in arc_change_state()
2419 if (HDR_HAS_L1HDR(hdr)) { in arc_change_state()
2420 old_state = hdr->b_l1hdr.b_state; in arc_change_state()
2421 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt); in arc_change_state()
2422 update_old = (hdr->b_l1hdr.b_buf != NULL || in arc_change_state()
2423 hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); in arc_change_state()
2425 IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL); in arc_change_state()
2426 IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL); in arc_change_state()
2427 IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL || in arc_change_state()
2428 ARC_BUF_LAST(hdr->b_l1hdr.b_buf)); in arc_change_state()
2440 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_change_state()
2449 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2451 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_change_state()
2453 hdr); in arc_change_state()
2454 arc_evictable_space_decrement(hdr, old_state); in arc_change_state()
2464 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2465 multilist_insert(&new_state->arcs_list[type], hdr); in arc_change_state()
2466 arc_evictable_space_increment(hdr, new_state); in arc_change_state()
2470 ASSERT(!HDR_EMPTY(hdr)); in arc_change_state()
2471 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) in arc_change_state()
2472 buf_hash_remove(hdr); in arc_change_state()
2477 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2488 HDR_GET_LSIZE(hdr), hdr); in arc_change_state()
2489 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_change_state()
2490 ASSERT(!HDR_HAS_RABD(hdr)); in arc_change_state()
2498 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2503 * block with the hdr, the owner of the in arc_change_state()
2504 * reference belongs to the hdr. Only in arc_change_state()
2516 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_change_state()
2519 arc_hdr_size(hdr), hdr); in arc_change_state()
2522 if (HDR_HAS_RABD(hdr)) { in arc_change_state()
2525 HDR_GET_PSIZE(hdr), hdr); in arc_change_state()
2531 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2533 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_change_state()
2534 ASSERT(!HDR_HAS_RABD(hdr)); in arc_change_state()
2546 HDR_GET_LSIZE(hdr), hdr); in arc_change_state()
2554 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2559 * block with the hdr, the owner of the in arc_change_state()
2560 * reference belongs to the hdr. Only in arc_change_state()
2571 ASSERT(hdr->b_l1hdr.b_pabd != NULL || in arc_change_state()
2572 HDR_HAS_RABD(hdr)); in arc_change_state()
2574 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_change_state()
2577 arc_hdr_size(hdr), hdr); in arc_change_state()
2580 if (HDR_HAS_RABD(hdr)) { in arc_change_state()
2583 HDR_GET_PSIZE(hdr), hdr); in arc_change_state()
2588 if (HDR_HAS_L1HDR(hdr)) { in arc_change_state()
2589 hdr->b_l1hdr.b_state = new_state; in arc_change_state()
2591 if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) { in arc_change_state()
2592 l2arc_hdr_arcstats_decrement_state(hdr); in arc_change_state()
2593 hdr->b_l2hdr.b_arcs_state = new_state->arcs_state; in arc_change_state()
2594 l2arc_hdr_arcstats_increment_state(hdr); in arc_change_state()
2687 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2688 * with the hdr's b_pabd.
2691 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_can_share() argument
2694 * The criteria for sharing a hdr's data are: in arc_can_share()
2696 * 2. the hdr's compression matches the buf's compression in arc_can_share()
2697 * 3. the hdr doesn't need to be byteswapped in arc_can_share()
2698 * 4. the hdr isn't already being shared in arc_can_share()
2699 * 5. the buf is either compressed or it is the last buf in the hdr list in arc_can_share()
2702 * bufs must be the final buf in the hdr's b_buf list. Reading this, you in arc_can_share()
2705 * a shared uncompressed buf anyway (because the hdr must be compressed in arc_can_share()
2715 ASSERT3P(buf->b_hdr, ==, hdr); in arc_can_share()
2717 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF; in arc_can_share()
2721 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && in arc_can_share()
2722 !HDR_SHARED_DATA(hdr) && in arc_can_share()
2727 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2732 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb, in arc_buf_alloc_impl() argument
2739 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_alloc_impl()
2740 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); in arc_buf_alloc_impl()
2741 VERIFY(hdr->b_type == ARC_BUFC_DATA || in arc_buf_alloc_impl()
2742 hdr->b_type == ARC_BUFC_METADATA); in arc_buf_alloc_impl()
2748 buf->b_hdr = hdr; in arc_buf_alloc_impl()
2750 buf->b_next = hdr->b_l1hdr.b_buf; in arc_buf_alloc_impl()
2753 add_reference(hdr, tag); in arc_buf_alloc_impl()
2756 * We're about to change the hdr's b_flags. We must either in arc_buf_alloc_impl()
2759 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_alloc_impl()
2762 * Only honor requests for compressed bufs if the hdr is actually in arc_buf_alloc_impl()
2771 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) { in arc_buf_alloc_impl()
2782 * If the hdr's data can be shared then we share the data buffer and in arc_buf_alloc_impl()
2783 * set the appropriate bit in the hdr's b_flags to indicate the hdr is in arc_buf_alloc_impl()
2788 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be in arc_buf_alloc_impl()
2790 * an arc_write() then the hdr's data buffer will be released when the in arc_buf_alloc_impl()
2792 * Second, the hdr's ABD must be linear so that the buf's user doesn't in arc_buf_alloc_impl()
2799 boolean_t can_share = arc_can_share(hdr, buf) && in arc_buf_alloc_impl()
2800 !HDR_L2_WRITING(hdr) && in arc_buf_alloc_impl()
2801 hdr->b_l1hdr.b_pabd != NULL && in arc_buf_alloc_impl()
2802 abd_is_linear(hdr->b_l1hdr.b_pabd) && in arc_buf_alloc_impl()
2803 !abd_is_linear_page(hdr->b_l1hdr.b_pabd); in arc_buf_alloc_impl()
2807 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd); in arc_buf_alloc_impl()
2809 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_alloc_impl()
2812 arc_get_data_buf(hdr, arc_buf_size(buf), buf); in arc_buf_alloc_impl()
2817 hdr->b_l1hdr.b_buf = buf; in arc_buf_alloc_impl()
2820 * If the user wants the data from the hdr, we need to either copy or in arc_buf_alloc_impl()
2892 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_return_buf() local
2895 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_return_buf()
2896 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag); in arc_return_buf()
2897 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_return_buf()
2906 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_loan_inuse_buf() local
2909 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_loan_inuse_buf()
2910 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_loan_inuse_buf()
2911 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); in arc_loan_inuse_buf()
2930 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata) in arc_hdr_free_on_write() argument
2932 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_hdr_free_on_write()
2933 arc_buf_contents_t type = arc_buf_type(hdr); in arc_hdr_free_on_write()
2934 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr); in arc_hdr_free_on_write()
2937 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_hdr_free_on_write()
2938 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_free_on_write()
2942 size, hdr); in arc_hdr_free_on_write()
2944 (void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr); in arc_hdr_free_on_write()
2953 l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type); in arc_hdr_free_on_write()
2955 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type); in arc_hdr_free_on_write()
2960 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2961 * data buffer, we transfer the refcount ownership to the hdr and update
2965 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_share_buf() argument
2967 ASSERT(arc_can_share(hdr, buf)); in arc_share_buf()
2968 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_share_buf()
2970 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_share_buf()
2974 * refcount ownership to the hdr since it always owns in arc_share_buf()
2978 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)], in arc_share_buf()
2979 arc_hdr_size(hdr), buf, hdr); in arc_share_buf()
2980 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf)); in arc_share_buf()
2981 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd, in arc_share_buf()
2982 HDR_ISTYPE_METADATA(hdr)); in arc_share_buf()
2983 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_share_buf()
2987 * Since we've transferred ownership to the hdr we need in arc_share_buf()
2991 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); in arc_share_buf()
2992 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); in arc_share_buf()
2997 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_unshare_buf() argument
3000 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_unshare_buf()
3001 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_unshare_buf()
3008 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)], in arc_unshare_buf()
3009 arc_hdr_size(hdr), hdr, buf); in arc_unshare_buf()
3010 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_unshare_buf()
3011 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); in arc_unshare_buf()
3012 abd_free(hdr->b_l1hdr.b_pabd); in arc_unshare_buf()
3013 hdr->b_l1hdr.b_pabd = NULL; in arc_unshare_buf()
3018 * the arc buf and the hdr, count it as overhead. in arc_unshare_buf()
3020 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); in arc_unshare_buf()
3021 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); in arc_unshare_buf()
3026 * Remove an arc_buf_t from the hdr's buf list and return the last
3031 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_buf_remove() argument
3033 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_remove()
3034 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_remove()
3036 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf; in arc_buf_remove()
3040 * Remove the buf from the hdr list and locate the last in arc_buf_remove()
3071 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_destroy_impl() local
3075 * sharing this with the hdr. If we are sharing it with the hdr, the in arc_buf_destroy_impl()
3076 * hdr is responsible for doing the free. in arc_buf_destroy_impl()
3080 * We're about to change the hdr's b_flags. We must either in arc_buf_destroy_impl()
3083 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_destroy_impl()
3089 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_destroy_impl()
3093 arc_free_data_buf(hdr, buf->b_data, size, buf); in arc_buf_destroy_impl()
3103 if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) && in arc_buf_destroy_impl()
3104 hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) { in arc_buf_destroy_impl()
3106 for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) { in arc_buf_destroy_impl()
3111 arc_hdr_free_abd(hdr, B_TRUE); in arc_buf_destroy_impl()
3115 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); in arc_buf_destroy_impl()
3120 * hdr, then reassign the hdr's b_pabd to share it with the new in arc_buf_destroy_impl()
3122 * the last one on the hdr's buffer list. in arc_buf_destroy_impl()
3133 /* hdr is uncompressed so can't have compressed buf */ in arc_buf_destroy_impl()
3136 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_buf_destroy_impl()
3137 arc_hdr_free_abd(hdr, B_FALSE); in arc_buf_destroy_impl()
3141 * last buffer and the hdr. The data would have in arc_buf_destroy_impl()
3143 * ownership to the hdr since it's now being shared. in arc_buf_destroy_impl()
3145 arc_share_buf(hdr, lastbuf); in arc_buf_destroy_impl()
3147 } else if (HDR_SHARED_DATA(hdr)) { in arc_buf_destroy_impl()
3153 * we rely on the hdr's compression flags to determine in arc_buf_destroy_impl()
3158 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); in arc_buf_destroy_impl()
3163 * this hdr. in arc_buf_destroy_impl()
3165 if (!arc_hdr_has_uncompressed_buf(hdr)) { in arc_buf_destroy_impl()
3166 arc_cksum_free(hdr); in arc_buf_destroy_impl()
3175 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags) in arc_hdr_alloc_abd() argument
3180 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); in arc_hdr_alloc_abd()
3181 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_alloc_abd()
3182 ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata); in arc_hdr_alloc_abd()
3183 IMPLY(alloc_rdata, HDR_PROTECTED(hdr)); in arc_hdr_alloc_abd()
3186 size = HDR_GET_PSIZE(hdr); in arc_hdr_alloc_abd()
3187 ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL); in arc_hdr_alloc_abd()
3188 hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr, in arc_hdr_alloc_abd()
3190 ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL); in arc_hdr_alloc_abd()
3193 size = arc_hdr_size(hdr); in arc_hdr_alloc_abd()
3194 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_hdr_alloc_abd()
3195 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr, in arc_hdr_alloc_abd()
3197 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_hdr_alloc_abd()
3201 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); in arc_hdr_alloc_abd()
3205 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata) in arc_hdr_free_abd() argument
3207 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr); in arc_hdr_free_abd()
3209 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_free_abd()
3210 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); in arc_hdr_free_abd()
3211 IMPLY(free_rdata, HDR_HAS_RABD(hdr)); in arc_hdr_free_abd()
3214 * If the hdr is currently being written to the l2arc then in arc_hdr_free_abd()
3219 if (HDR_L2_WRITING(hdr)) { in arc_hdr_free_abd()
3220 arc_hdr_free_on_write(hdr, free_rdata); in arc_hdr_free_abd()
3223 arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr); in arc_hdr_free_abd()
3225 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr); in arc_hdr_free_abd()
3229 hdr->b_crypt_hdr.b_rabd = NULL; in arc_hdr_free_abd()
3232 hdr->b_l1hdr.b_pabd = NULL; in arc_hdr_free_abd()
3235 if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr)) in arc_hdr_free_abd()
3236 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_hdr_free_abd()
3239 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); in arc_hdr_free_abd()
3269 arc_buf_hdr_t *hdr; in arc_hdr_alloc() local
3272 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); in arc_hdr_alloc()
3274 ASSERT(HDR_EMPTY(hdr)); in arc_hdr_alloc()
3276 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_alloc()
3278 HDR_SET_PSIZE(hdr, psize); in arc_hdr_alloc()
3279 HDR_SET_LSIZE(hdr, lsize); in arc_hdr_alloc()
3280 hdr->b_spa = spa; in arc_hdr_alloc()
3281 hdr->b_type = type; in arc_hdr_alloc()
3282 hdr->b_flags = 0; in arc_hdr_alloc()
3283 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR); in arc_hdr_alloc()
3284 arc_hdr_set_compress(hdr, compression_type); in arc_hdr_alloc()
3285 hdr->b_complevel = complevel; in arc_hdr_alloc()
3287 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); in arc_hdr_alloc()
3289 hdr->b_l1hdr.b_state = arc_anon; in arc_hdr_alloc()
3290 hdr->b_l1hdr.b_arc_access = 0; in arc_hdr_alloc()
3291 hdr->b_l1hdr.b_mru_hits = 0; in arc_hdr_alloc()
3292 hdr->b_l1hdr.b_mru_ghost_hits = 0; in arc_hdr_alloc()
3293 hdr->b_l1hdr.b_mfu_hits = 0; in arc_hdr_alloc()
3294 hdr->b_l1hdr.b_mfu_ghost_hits = 0; in arc_hdr_alloc()
3295 hdr->b_l1hdr.b_buf = NULL; in arc_hdr_alloc()
3297 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_alloc()
3299 return (hdr); in arc_hdr_alloc()
3310 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) in arc_hdr_realloc() argument
3312 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_realloc()
3315 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_realloc()
3322 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_hdr_realloc()
3323 buf_hash_remove(hdr); in arc_hdr_realloc()
3325 memcpy(nhdr, hdr, HDR_L2ONLY_SIZE); in arc_hdr_realloc()
3338 ASSERT(!HDR_HAS_RABD(hdr)); in arc_hdr_realloc()
3340 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_hdr_realloc()
3342 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_realloc()
3352 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_realloc()
3360 VERIFY(!HDR_L2_WRITING(hdr)); in arc_hdr_realloc()
3361 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_hdr_realloc()
3362 ASSERT(!HDR_HAS_RABD(hdr)); in arc_hdr_realloc()
3372 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); in arc_hdr_realloc()
3382 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); in arc_hdr_realloc()
3383 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_realloc()
3396 arc_hdr_size(hdr), hdr); in arc_hdr_realloc()
3400 buf_discard_identity(hdr); in arc_hdr_realloc()
3401 kmem_cache_free(old, hdr); in arc_hdr_realloc()
3418 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_convert_to_raw() local
3421 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_convert_to_raw()
3422 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_convert_to_raw()
3425 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); in arc_convert_to_raw()
3426 hdr->b_crypt_hdr.b_dsobj = dsobj; in arc_convert_to_raw()
3427 hdr->b_crypt_hdr.b_ot = ot; in arc_convert_to_raw()
3428 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ? in arc_convert_to_raw()
3430 if (!arc_hdr_has_uncompressed_buf(hdr)) in arc_convert_to_raw()
3431 arc_cksum_free(hdr); in arc_convert_to_raw()
3434 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN); in arc_convert_to_raw()
3436 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN); in arc_convert_to_raw()
3438 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN); in arc_convert_to_raw()
3449 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size, in arc_alloc_buf() local
3453 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE, in arc_alloc_buf()
3473 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, in arc_alloc_compressed_buf() local
3477 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, in arc_alloc_compressed_buf()
3482 * To ensure that the hdr has the correct data in it if we call in arc_alloc_compressed_buf()
3484 * it's easiest if we just set up sharing between the buf and the hdr. in arc_alloc_compressed_buf()
3486 arc_share_buf(hdr, buf); in arc_alloc_compressed_buf()
3497 arc_buf_hdr_t *hdr; in arc_alloc_raw_buf() local
3507 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE, in arc_alloc_raw_buf()
3510 hdr->b_crypt_hdr.b_dsobj = dsobj; in arc_alloc_raw_buf()
3511 hdr->b_crypt_hdr.b_ot = ot; in arc_alloc_raw_buf()
3512 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ? in arc_alloc_raw_buf()
3514 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN); in arc_alloc_raw_buf()
3515 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN); in arc_alloc_raw_buf()
3516 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN); in arc_alloc_raw_buf()
3524 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE, in arc_alloc_raw_buf()
3532 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr, in l2arc_hdr_arcstats_update() argument
3535 uint64_t lsize = HDR_GET_LSIZE(hdr); in l2arc_hdr_arcstats_update()
3536 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_hdr_arcstats_update()
3537 uint64_t asize = HDR_GET_L2SIZE(hdr); in l2arc_hdr_arcstats_update()
3538 arc_buf_contents_t type = hdr->b_type; in l2arc_hdr_arcstats_update()
3557 if (HDR_PREFETCH(hdr)) { in l2arc_hdr_arcstats_update()
3569 switch (hdr->b_l2hdr.b_arcs_state) { in l2arc_hdr_arcstats_update()
3603 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_l2hdr_destroy() argument
3605 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; in arc_hdr_l2hdr_destroy()
3609 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_l2hdr_destroy()
3611 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_l2hdr_destroy()
3613 l2arc_hdr_arcstats_decrement(hdr); in arc_hdr_l2hdr_destroy()
3615 uint64_t asize = HDR_GET_L2SIZE(hdr); in arc_hdr_l2hdr_destroy()
3619 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), in arc_hdr_l2hdr_destroy()
3620 hdr); in arc_hdr_l2hdr_destroy()
3621 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); in arc_hdr_l2hdr_destroy()
3625 arc_hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_destroy() argument
3627 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3628 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_destroy()
3629 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_hdr_destroy()
3631 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_hdr_destroy()
3632 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_hdr_destroy()
3634 if (HDR_HAS_L2HDR(hdr)) { in arc_hdr_destroy()
3635 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_destroy()
3650 if (HDR_HAS_L2HDR(hdr)) { in arc_hdr_destroy()
3652 if (!HDR_EMPTY(hdr)) in arc_hdr_destroy()
3653 buf_discard_identity(hdr); in arc_hdr_destroy()
3655 arc_hdr_l2hdr_destroy(hdr); in arc_hdr_destroy()
3668 if (!HDR_EMPTY(hdr)) in arc_hdr_destroy()
3669 buf_discard_identity(hdr); in arc_hdr_destroy()
3671 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3672 arc_cksum_free(hdr); in arc_hdr_destroy()
3674 while (hdr->b_l1hdr.b_buf != NULL) in arc_hdr_destroy()
3675 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf); in arc_hdr_destroy()
3677 if (hdr->b_l1hdr.b_pabd != NULL) in arc_hdr_destroy()
3678 arc_hdr_free_abd(hdr, B_FALSE); in arc_hdr_destroy()
3680 if (HDR_HAS_RABD(hdr)) in arc_hdr_destroy()
3681 arc_hdr_free_abd(hdr, B_TRUE); in arc_hdr_destroy()
3684 ASSERT3P(hdr->b_hash_next, ==, NULL); in arc_hdr_destroy()
3685 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3686 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_destroy()
3687 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_hdr_destroy()
3689 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_destroy()
3691 kmem_cache_free(hdr_full_cache, hdr); in arc_hdr_destroy()
3693 kmem_cache_free(hdr_l2only_cache, hdr); in arc_hdr_destroy()
3700 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_destroy() local
3702 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_buf_destroy()
3703 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf); in arc_buf_destroy()
3705 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_buf_destroy()
3706 VERIFY0(remove_reference(hdr, tag)); in arc_buf_destroy()
3710 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_destroy()
3713 ASSERT3P(hdr, ==, buf->b_hdr); in arc_buf_destroy()
3714 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL); in arc_buf_destroy()
3715 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_buf_destroy()
3716 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); in arc_buf_destroy()
3720 (void) remove_reference(hdr, tag); in arc_buf_destroy()
3747 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted) in arc_evict_hdr() argument
3751 uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ? in arc_evict_hdr()
3754 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_evict_hdr()
3755 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evict_hdr()
3756 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_evict_hdr()
3757 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evict_hdr()
3758 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_evict_hdr()
3761 state = hdr->b_l1hdr.b_state; in arc_evict_hdr()
3771 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { in arc_evict_hdr()
3777 bytes_evicted += HDR_GET_LSIZE(hdr); in arc_evict_hdr()
3779 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
3781 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
3782 ASSERT(hdr->b_l1hdr.b_pabd == NULL); in arc_evict_hdr()
3783 ASSERT(!HDR_HAS_RABD(hdr)); in arc_evict_hdr()
3788 arc_change_state(arc_l2c_only, hdr); in arc_evict_hdr()
3793 (void) arc_hdr_realloc(hdr, hdr_full_cache, in arc_evict_hdr()
3797 arc_change_state(arc_anon, hdr); in arc_evict_hdr()
3798 arc_hdr_destroy(hdr); in arc_evict_hdr()
3809 if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && in arc_evict_hdr()
3810 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < in arc_evict_hdr()
3816 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
3817 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3819 if (l2arc_write_eligible(hdr->b_spa, hdr)) { in arc_evict_hdr()
3821 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3827 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3832 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3839 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3843 bytes_evicted += arc_hdr_size(hdr); in arc_evict_hdr()
3844 *real_evicted += arc_hdr_size(hdr); in arc_evict_hdr()
3847 * If this hdr is being evicted and has a compressed buffer then we in arc_evict_hdr()
3851 if (hdr->b_l1hdr.b_pabd != NULL) in arc_evict_hdr()
3852 arc_hdr_free_abd(hdr, B_FALSE); in arc_evict_hdr()
3854 if (HDR_HAS_RABD(hdr)) in arc_evict_hdr()
3855 arc_hdr_free_abd(hdr, B_TRUE); in arc_evict_hdr()
3857 arc_change_state(evicted_state, hdr); in arc_evict_hdr()
3858 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
3860 arc_hdr_destroy(hdr); in arc_evict_hdr()
3863 ASSERT(HDR_IN_HASH_TABLE(hdr)); in arc_evict_hdr()
3889 arc_buf_hdr_t *hdr; in arc_evict_state_impl() local
3897 for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL); in arc_evict_state_impl()
3898 hdr = multilist_sublist_prev(mls, marker)) { in arc_evict_state_impl()
3904 * forward. Since we're not holding hdr's hash lock, we in arc_evict_state_impl()
3905 * must be very careful and not remove 'hdr' from the in arc_evict_state_impl()
3907 * 'hdr' as not being on a sublist when they call the in arc_evict_state_impl()
3924 if (hdr->b_spa == 0) in arc_evict_state_impl()
3928 if (spa != 0 && hdr->b_spa != spa) { in arc_evict_state_impl()
3933 hash_lock = HDR_LOCK(hdr); in arc_evict_state_impl()
3948 uint64_t evicted = arc_evict_hdr(hdr, &revicted); in arc_evict_state_impl()
4925 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag, in arc_get_data_abd() argument
4928 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_abd()
4930 arc_get_data_impl(hdr, size, tag, alloc_flags); in arc_get_data_abd()
4938 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag) in arc_get_data_buf() argument
4940 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_buf()
4942 arc_get_data_impl(hdr, size, tag, 0); in arc_get_data_buf()
5039 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag, in arc_get_data_impl() argument
5060 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_impl()
5071 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_get_data_impl()
5086 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_get_data_impl()
5087 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_get_data_impl()
5095 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, in arc_free_data_abd() argument
5098 arc_free_data_impl(hdr, size, tag); in arc_free_data_abd()
5103 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag) in arc_free_data_buf() argument
5105 arc_buf_contents_t type = arc_buf_type(hdr); in arc_free_data_buf()
5107 arc_free_data_impl(hdr, size, tag); in arc_free_data_buf()
5120 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag) in arc_free_data_impl() argument
5122 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_free_data_impl()
5123 arc_buf_contents_t type = arc_buf_type(hdr); in arc_free_data_impl()
5126 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_free_data_impl()
5127 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_free_data_impl()
5135 VERIFY3U(hdr->b_type, ==, type); in arc_free_data_impl()
5148 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit) in arc_access() argument
5150 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_access()
5151 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_access()
5156 boolean_t was_prefetch = HDR_PREFETCH(hdr); in arc_access()
5161 HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive, in arc_access()
5164 if (HDR_HAS_L2HDR(hdr)) in arc_access()
5165 l2arc_hdr_arcstats_decrement_state(hdr); in arc_access()
5167 arc_hdr_clear_flags(hdr, in arc_access()
5170 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_access()
5172 if (HDR_HAS_L2HDR(hdr)) in arc_access()
5173 l2arc_hdr_arcstats_increment_state(hdr); in arc_access()
5177 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); in arc_access()
5184 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_access()
5187 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_access()
5193 ASSERT0(hdr->b_l1hdr.b_arc_access); in arc_access()
5194 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5195 if (HDR_UNCACHED(hdr)) { in arc_access()
5198 hdr); in arc_access()
5201 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
5203 arc_change_state(new_state, hdr); in arc_access()
5204 } else if (hdr->b_l1hdr.b_state == arc_mru) { in arc_access()
5209 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_access()
5210 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5213 hdr->b_l1hdr.b_mru_hits++; in arc_access()
5221 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5229 if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access + in arc_access()
5231 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5232 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5233 arc_change_state(arc_mfu, hdr); in arc_access()
5235 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { in arc_access()
5243 hdr->b_l1hdr.b_mru_ghost_hits++; in arc_access()
5245 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5246 wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)], in arc_access()
5247 arc_hdr_size(hdr)); in arc_access()
5250 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
5253 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5255 arc_change_state(new_state, hdr); in arc_access()
5256 } else if (hdr->b_l1hdr.b_state == arc_mfu) { in arc_access()
5261 if (!HDR_IO_IN_PROGRESS(hdr)) { in arc_access()
5262 hdr->b_l1hdr.b_mfu_hits++; in arc_access()
5265 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5266 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { in arc_access()
5272 hdr->b_l1hdr.b_mfu_ghost_hits++; in arc_access()
5274 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5275 wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)], in arc_access()
5276 arc_hdr_size(hdr)); in arc_access()
5277 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5278 arc_change_state(arc_mfu, hdr); in arc_access()
5279 } else if (hdr->b_l1hdr.b_state == arc_uncached) { in arc_access()
5284 if (!HDR_IO_IN_PROGRESS(hdr)) in arc_access()
5286 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5287 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { in arc_access()
5292 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5293 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
5294 arc_change_state(arc_mru, hdr); in arc_access()
5297 hdr->b_l1hdr.b_state); in arc_access()
5308 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_access() local
5315 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) in arc_buf_access()
5318 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_access()
5321 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { in arc_buf_access()
5327 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_buf_access()
5328 hdr->b_l1hdr.b_state == arc_mfu || in arc_buf_access()
5329 hdr->b_l1hdr.b_state == arc_uncached); in arc_buf_access()
5331 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_buf_access()
5332 arc_access(hdr, 0, B_TRUE); in arc_buf_access()
5337 !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); in arc_buf_access()
5373 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp) in arc_hdr_verify() argument
5376 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0); in arc_hdr_verify()
5377 ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF); in arc_hdr_verify()
5379 if (HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_verify()
5380 ASSERT3U(arc_hdr_get_compress(hdr), ==, in arc_hdr_verify()
5383 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); in arc_hdr_verify()
5384 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp)); in arc_hdr_verify()
5385 ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp)); in arc_hdr_verify()
5393 arc_buf_hdr_t *hdr = zio->io_private; in arc_read_done() local
5399 * The hdr was inserted into hash-table and removed from lists in arc_read_done()
5406 if (HDR_IN_HASH_TABLE(hdr)) { in arc_read_done()
5409 ASSERT3U(hdr->b_birth, ==, BP_GET_BIRTH(zio->io_bp)); in arc_read_done()
5410 ASSERT3U(hdr->b_dva.dva_word[0], ==, in arc_read_done()
5412 ASSERT3U(hdr->b_dva.dva_word[1], ==, in arc_read_done()
5415 found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock); in arc_read_done()
5417 ASSERT((found == hdr && in arc_read_done()
5418 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || in arc_read_done()
5419 (found == hdr && HDR_L2_READING(hdr))); in arc_read_done()
5424 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp); in arc_read_done()
5425 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset; in arc_read_done()
5426 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt, in arc_read_done()
5427 hdr->b_crypt_hdr.b_iv); in arc_read_done()
5436 hdr->b_crypt_hdr.b_mac); in arc_read_done()
5441 hdr->b_crypt_hdr.b_mac); in arc_read_done()
5450 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; in arc_read_done()
5452 hdr->b_l1hdr.b_byteswap = in arc_read_done()
5456 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_read_done()
5458 if (!HDR_L2_READING(hdr)) { in arc_read_done()
5459 hdr->b_complevel = zio->io_prop.zp_complevel; in arc_read_done()
5463 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); in arc_read_done()
5464 if (l2arc_noprefetch && HDR_PREFETCH(hdr)) in arc_read_done()
5465 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); in arc_read_done()
5467 callback_list = hdr->b_l1hdr.b_acb; in arc_read_done()
5469 hdr->b_l1hdr.b_acb = NULL; in arc_read_done()
5491 int error = arc_buf_alloc_impl(hdr, zio->io_spa, in arc_read_done()
5529 * the hdr will not be anonymous, because in arc_read_done()
5540 * because the only way for multiple threads to find this hdr is in arc_read_done()
5542 * callbacks, the hdr is not anonymous. If it were anonymous, in arc_read_done()
5548 arc_hdr_verify(hdr, zio->io_bp); in arc_read_done()
5550 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_read_done()
5551 if (hdr->b_l1hdr.b_state != arc_anon) in arc_read_done()
5552 arc_change_state(arc_anon, hdr); in arc_read_done()
5553 if (HDR_IN_HASH_TABLE(hdr)) in arc_read_done()
5554 buf_hash_remove(hdr); in arc_read_done()
5557 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_read_done()
5558 (void) remove_reference(hdr, hdr); in arc_read_done()
5606 arc_buf_hdr_t *hdr = NULL; in arc_cached() local
5614 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_cached()
5615 if (hdr == NULL) in arc_cached()
5618 if (HDR_HAS_L1HDR(hdr)) { in arc_cached()
5619 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_cached()
5642 if (HDR_HAS_L2HDR(hdr)) in arc_cached()
5673 arc_buf_hdr_t *hdr = NULL; in arc_read() local
5709 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_read()
5719 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) || in arc_read()
5720 (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) { in arc_read()
5721 boolean_t is_data = !HDR_ISTYPE_METADATA(hdr); in arc_read()
5735 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_read()
5743 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head; in arc_read()
5745 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && in arc_read()
5754 arc_buf_hdr_t *, hdr); in arc_read()
5758 DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr); in arc_read()
5759 arc_access(hdr, *arc_flags, B_FALSE); in arc_read()
5800 acb->acb_next = hdr->b_l1hdr.b_acb; in arc_read()
5801 hdr->b_l1hdr.b_acb->acb_prev = acb; in arc_read()
5802 hdr->b_l1hdr.b_acb = acb; in arc_read()
5825 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_read()
5826 hdr->b_l1hdr.b_state == arc_mfu || in arc_read()
5827 hdr->b_l1hdr.b_state == arc_uncached); in arc_read()
5829 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
5830 arc_access(hdr, *arc_flags, B_TRUE); in arc_read()
5836 rc = arc_buf_alloc_impl(hdr, spa, zb, private, in arc_read()
5847 spa_log_error(spa, zb, hdr->b_birth); in arc_read()
5856 (void) remove_reference(hdr, private); in arc_read()
5902 if (hdr == NULL) { in arc_read()
5908 hdr = arc_hdr_alloc(guid, psize, lsize, in arc_read()
5912 hdr->b_dva = *BP_IDENTITY(bp); in arc_read()
5913 hdr->b_birth = BP_GET_BIRTH(bp); in arc_read()
5914 exists = buf_hash_insert(hdr, &hash_lock); in arc_read()
5919 buf_discard_identity(hdr); in arc_read()
5920 arc_hdr_destroy(hdr); in arc_read()
5927 * L2-only (and thus didn't have an L1 hdr), in arc_read()
5928 * we realloc the header to add an L1 hdr. in arc_read()
5930 if (!HDR_HAS_L1HDR(hdr)) { in arc_read()
5931 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, in arc_read()
5935 if (GHOST_STATE(hdr->b_l1hdr.b_state)) { in arc_read()
5936 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_read()
5937 ASSERT(!HDR_HAS_RABD(hdr)); in arc_read()
5938 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_read()
5940 &hdr->b_l1hdr.b_refcnt)); in arc_read()
5941 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_read()
5943 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_read()
5945 } else if (HDR_IO_IN_PROGRESS(hdr)) { in arc_read()
5963 hdr->b_l1hdr.b_acb->acb_zio_head; in arc_read()
5964 acb->acb_next = hdr->b_l1hdr.b_acb; in arc_read()
5965 hdr->b_l1hdr.b_acb->acb_prev = acb; in arc_read()
5966 hdr->b_l1hdr.b_acb = acb; in arc_read()
5981 arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED); in arc_read()
5992 add_reference(hdr, hdr); in arc_read()
5994 arc_access(hdr, *arc_flags, B_FALSE); in arc_read()
5995 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_read()
5996 arc_hdr_alloc_abd(hdr, alloc_flags); in arc_read()
5998 ASSERT(HDR_HAS_RABD(hdr)); in arc_read()
5999 size = HDR_GET_PSIZE(hdr); in arc_read()
6000 hdr_abd = hdr->b_crypt_hdr.b_rabd; in arc_read()
6003 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_read()
6004 size = arc_hdr_size(hdr); in arc_read()
6005 hdr_abd = hdr->b_l1hdr.b_pabd; in arc_read()
6007 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) { in arc_read()
6023 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH); in arc_read()
6025 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); in arc_read()
6026 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); in arc_read()
6036 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_read()
6037 hdr->b_l1hdr.b_acb = acb; in arc_read()
6039 if (HDR_HAS_L2HDR(hdr) && in arc_read()
6040 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { in arc_read()
6041 devw = hdr->b_l2hdr.b_dev->l2ad_writing; in arc_read()
6042 addr = hdr->b_l2hdr.b_daddr; in arc_read()
6058 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); in arc_read()
6060 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); in arc_read()
6066 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); in arc_read()
6074 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, in arc_read()
6079 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, in arc_read()
6097 if (HDR_HAS_L2HDR(hdr) && in arc_read()
6098 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) { in arc_read()
6103 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
6105 hdr->b_l2hdr.b_hits++; in arc_read()
6109 cb->l2rcb_hdr = hdr; in arc_read()
6119 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_read()
6120 !HDR_COMPRESSION_ENABLED(hdr) && in arc_read()
6121 HDR_GET_PSIZE(hdr) != 0) { in arc_read()
6122 size = HDR_GET_PSIZE(hdr); in arc_read()
6128 HDR_ISTYPE_METADATA(hdr)); in arc_read()
6144 ASSERT3U(arc_hdr_get_compress(hdr), !=, in arc_read()
6161 HDR_GET_PSIZE(hdr)); in arc_read()
6177 arc_buf_hdr_t *, hdr); in arc_read()
6179 if (HDR_L2_WRITING(hdr)) in arc_read()
6201 arc_buf_hdr_t *, hdr); in arc_read()
6208 arc_read_done, hdr, priority, zio_flags, zb); in arc_read()
6334 arc_buf_hdr_t *hdr; in arc_freed() local
6340 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_freed()
6341 if (hdr == NULL) in arc_freed()
6350 * without the dedup flag set. This would have left the hdr in the MRU in arc_freed()
6362 * this hdr, then we don't destroy the hdr. in arc_freed()
6364 if (!HDR_HAS_L1HDR(hdr) || in arc_freed()
6365 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_freed()
6366 arc_change_state(arc_anon, hdr); in arc_freed()
6367 arc_hdr_destroy(hdr); in arc_freed()
6379 * a new hdr for the buffer.
6384 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_release() local
6392 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_release()
6399 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_release()
6400 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
6401 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_release()
6402 ASSERT(!HDR_HAS_L2HDR(hdr)); in arc_release()
6404 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf); in arc_release()
6406 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); in arc_release()
6407 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
6409 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
6413 * have a hdr that is not empty. in arc_release()
6415 buf_discard_identity(hdr); in arc_release()
6421 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_release()
6429 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_release()
6430 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_release()
6434 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0); in arc_release()
6436 if (HDR_HAS_L2HDR(hdr)) { in arc_release()
6437 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
6447 if (HDR_HAS_L2HDR(hdr)) in arc_release()
6448 arc_hdr_l2hdr_destroy(hdr); in arc_release()
6450 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
6456 if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) { in arc_release()
6458 uint64_t spa = hdr->b_spa; in arc_release()
6459 uint64_t psize = HDR_GET_PSIZE(hdr); in arc_release()
6460 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_release()
6461 boolean_t protected = HDR_PROTECTED(hdr); in arc_release()
6462 enum zio_compress compress = arc_hdr_get_compress(hdr); in arc_release()
6463 arc_buf_contents_t type = arc_buf_type(hdr); in arc_release()
6464 VERIFY3U(hdr->b_type, ==, type); in arc_release()
6466 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL); in arc_release()
6467 VERIFY3S(remove_reference(hdr, tag), >, 0); in arc_release()
6470 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); in arc_release()
6475 * Pull the data off of this hdr and attach it to in arc_release()
6476 * a new anonymous hdr. Also find the last buffer in arc_release()
6477 * in the hdr's buffer list. in arc_release()
6479 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); in arc_release()
6483 * If the current arc_buf_t and the hdr are sharing their data in arc_release()
6487 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); in arc_release()
6494 arc_unshare_buf(hdr, buf); in arc_release()
6497 * Now we need to recreate the hdr's b_pabd. Since we in arc_release()
6502 if (arc_can_share(hdr, lastbuf)) { in arc_release()
6503 arc_share_buf(hdr, lastbuf); in arc_release()
6505 arc_hdr_alloc_abd(hdr, 0); in arc_release()
6506 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, in arc_release()
6510 } else if (HDR_SHARED_DATA(hdr)) { in arc_release()
6516 * we rely on the hdr's compression flags to determine in arc_release()
6520 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); in arc_release()
6524 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); in arc_release()
6530 if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_release()
6541 if (!arc_hdr_has_uncompressed_buf(hdr)) in arc_release()
6542 arc_cksum_free(hdr); in arc_release()
6547 compress, hdr->b_complevel, type); in arc_release()
6560 ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1); in arc_release()
6561 /* protected by hash lock, or hdr is on arc_anon */ in arc_release()
6562 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
6563 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
6564 hdr->b_l1hdr.b_mru_hits = 0; in arc_release()
6565 hdr->b_l1hdr.b_mru_ghost_hits = 0; in arc_release()
6566 hdr->b_l1hdr.b_mfu_hits = 0; in arc_release()
6567 hdr->b_l1hdr.b_mfu_ghost_hits = 0; in arc_release()
6568 arc_change_state(arc_anon, hdr); in arc_release()
6569 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
6572 buf_discard_identity(hdr); in arc_release()
6597 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_ready() local
6602 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_write_ready()
6604 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL); in arc_write_ready()
6612 arc_cksum_free(hdr); in arc_write_ready()
6614 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_write_ready()
6616 arc_unshare_buf(hdr, buf); in arc_write_ready()
6619 arc_hdr_free_abd(hdr, B_FALSE); in arc_write_ready()
6623 if (HDR_HAS_RABD(hdr)) in arc_write_ready()
6624 arc_hdr_free_abd(hdr, B_TRUE); in arc_write_ready()
6626 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_write_ready()
6627 ASSERT(!HDR_HAS_RABD(hdr)); in arc_write_ready()
6628 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_write_ready()
6633 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_write_ready()
6636 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_ready()
6637 add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */ in arc_write_ready()
6646 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; in arc_write_ready()
6648 hdr->b_l1hdr.b_byteswap = in arc_write_ready()
6652 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_write_ready()
6655 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); in arc_write_ready()
6656 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp); in arc_write_ready()
6657 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset; in arc_write_ready()
6658 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt, in arc_write_ready()
6659 hdr->b_crypt_hdr.b_iv); in arc_write_ready()
6660 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac); in arc_write_ready()
6662 arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED); in arc_write_ready()
6670 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH); in arc_write_ready()
6686 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); in arc_write_ready()
6689 HDR_SET_PSIZE(hdr, psize); in arc_write_ready()
6690 arc_hdr_set_compress(hdr, compress); in arc_write_ready()
6691 hdr->b_complevel = zio->io_prop.zp_complevel; in arc_write_ready()
6697 * Fill the hdr with data. If the buffer is encrypted we have no choice in arc_write_ready()
6698 * but to copy the data into b_radb. If the hdr is compressed, the data in arc_write_ready()
6702 * We might be able to share the buf's data with the hdr here. However, in arc_write_ready()
6713 arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA | in arc_write_ready()
6715 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize); in arc_write_ready()
6716 } else if (!(HDR_UNCACHED(hdr) || in arc_write_ready()
6718 !arc_can_share(hdr, buf)) { in arc_write_ready()
6722 * hdr's compression setting rather than the io_bp's. in arc_write_ready()
6726 arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA | in arc_write_ready()
6728 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize); in arc_write_ready()
6729 } else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF && in arc_write_ready()
6732 arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE); in arc_write_ready()
6733 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize); in arc_write_ready()
6735 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr)); in arc_write_ready()
6736 arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE); in arc_write_ready()
6737 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data, in arc_write_ready()
6743 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf); in arc_write_ready()
6746 arc_share_buf(hdr, buf); in arc_write_ready()
6750 arc_hdr_verify(hdr, bp); in arc_write_ready()
6768 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_done() local
6770 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_write_done()
6773 arc_hdr_verify(hdr, zio->io_bp); in arc_write_done()
6776 buf_discard_identity(hdr); in arc_write_done()
6778 hdr->b_dva = *BP_IDENTITY(zio->io_bp); in arc_write_done()
6779 hdr->b_birth = BP_GET_BIRTH(zio->io_bp); in arc_write_done()
6782 ASSERT(HDR_EMPTY(hdr)); in arc_write_done()
6791 if (!HDR_EMPTY(hdr)) { in arc_write_done()
6799 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
6808 panic("bad overwrite, hdr=%p exists=%p", in arc_write_done()
6809 (void *)hdr, (void *)exists); in arc_write_done()
6815 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
6821 panic("bad nopwrite, hdr=%p exists=%p", in arc_write_done()
6822 (void *)hdr, (void *)exists); in arc_write_done()
6825 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL); in arc_write_done()
6826 ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf)); in arc_write_done()
6827 ASSERT(hdr->b_l1hdr.b_state == arc_anon); in arc_write_done()
6832 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_done()
6833 VERIFY3S(remove_reference(hdr, hdr), >, 0); in arc_write_done()
6835 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon) in arc_write_done()
6836 arc_access(hdr, 0, B_FALSE); in arc_write_done()
6839 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_done()
6840 VERIFY3S(remove_reference(hdr, hdr), >, 0); in arc_write_done()
6857 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write() local
6864 ASSERT(!HDR_IO_ERROR(hdr)); in arc_write()
6865 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_write()
6866 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_write()
6867 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL); in arc_write()
6869 arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED); in arc_write()
6871 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_write()
6876 localprop.zp_compress = HDR_GET_COMPRESS(hdr); in arc_write()
6877 localprop.zp_complevel = hdr->b_complevel; in arc_write()
6879 (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ? in arc_write()
6881 memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt, in arc_write()
6883 memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv, in arc_write()
6885 memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac, in arc_write()
6894 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf)); in arc_write()
6895 localprop.zp_compress = HDR_GET_COMPRESS(hdr); in arc_write()
6896 localprop.zp_complevel = hdr->b_complevel; in arc_write()
6907 * The hdr's b_pabd is now stale, free it now. A new data block in arc_write()
6910 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_write()
6913 * the hdr then we need to break that relationship here. in arc_write()
6914 * The hdr will remain with a NULL data pointer and the in arc_write()
6918 arc_unshare_buf(hdr, buf); in arc_write()
6921 arc_hdr_free_abd(hdr, B_FALSE); in arc_write()
6926 if (HDR_HAS_RABD(hdr)) in arc_write()
6927 arc_hdr_free_abd(hdr, B_TRUE); in arc_write()
6930 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF); in arc_write()
6933 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_write()
6936 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)), in arc_write()
6937 HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready, in arc_write()
7323 arc_buf_hdr_t *hdr = obj; in arc_state_multilist_index_func() local
7330 ASSERT(!HDR_EMPTY(hdr)); in arc_state_multilist_index_func()
7345 return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) % in arc_state_multilist_index_func()
8184 * || hdr| ^ /^ /^ / / |
8248 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr) in l2arc_write_eligible() argument
8257 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) || in l2arc_write_eligible()
8258 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr)) in l2arc_write_eligible()
8433 arc_buf_hdr_t *head, *hdr, *hdr_prev; in l2arc_write_done() local
8454 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) { in l2arc_write_done()
8455 hdr_prev = list_prev(buflist, hdr); in l2arc_write_done()
8457 hash_lock = HDR_LOCK(hdr); in l2arc_write_done()
8478 list_insert_after(buflist, hdr, head); in l2arc_write_done()
8498 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_done()
8508 list_remove(buflist, hdr); in l2arc_write_done()
8509 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); in l2arc_write_done()
8511 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_write_done()
8512 l2arc_hdr_arcstats_decrement(hdr); in l2arc_write_done()
8519 arc_hdr_size(hdr), hdr); in l2arc_write_done()
8526 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING); in l2arc_write_done()
8613 arc_buf_hdr_t *hdr = cb->l2rcb_hdr; in l2arc_untransform() local
8625 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in l2arc_untransform()
8626 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in l2arc_untransform()
8630 * we must check the bp here and not the hdr, since the in l2arc_untransform()
8631 * hdr does not have its encryption parameters updated in l2arc_untransform()
8635 abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, in l2arc_untransform()
8643 salt, iv, mac, HDR_GET_PSIZE(hdr), eabd, in l2arc_untransform()
8644 hdr->b_l1hdr.b_pabd, &no_crypt); in l2arc_untransform()
8646 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr); in l2arc_untransform()
8656 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in l2arc_untransform()
8657 arc_hdr_size(hdr), hdr); in l2arc_untransform()
8658 hdr->b_l1hdr.b_pabd = eabd; in l2arc_untransform()
8661 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr); in l2arc_untransform()
8670 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in l2arc_untransform()
8671 !HDR_COMPRESSION_ENABLED(hdr)) { in l2arc_untransform()
8672 abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, in l2arc_untransform()
8675 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), in l2arc_untransform()
8676 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr), in l2arc_untransform()
8677 HDR_GET_LSIZE(hdr), &hdr->b_complevel); in l2arc_untransform()
8679 arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr); in l2arc_untransform()
8683 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in l2arc_untransform()
8684 arc_hdr_size(hdr), hdr); in l2arc_untransform()
8685 hdr->b_l1hdr.b_pabd = cabd; in l2arc_untransform()
8687 zio->io_size = HDR_GET_LSIZE(hdr); in l2arc_untransform()
8706 arc_buf_hdr_t *hdr; in l2arc_read_done() local
8718 hdr = cb->l2rcb_hdr; in l2arc_read_done()
8719 ASSERT3P(hdr, !=, NULL); in l2arc_read_done()
8721 hash_lock = HDR_LOCK(hdr); in l2arc_read_done()
8723 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in l2arc_read_done()
8730 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size); in l2arc_read_done()
8733 abd_copy(hdr->b_crypt_hdr.b_rabd, in l2arc_read_done()
8734 cb->l2rcb_abd, arc_hdr_size(hdr)); in l2arc_read_done()
8736 abd_copy(hdr->b_l1hdr.b_pabd, in l2arc_read_done()
8737 cb->l2rcb_abd, arc_hdr_size(hdr)); in l2arc_read_done()
8753 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr); in l2arc_read_done()
8756 ASSERT(HDR_HAS_RABD(hdr)); in l2arc_read_done()
8758 hdr->b_crypt_hdr.b_rabd; in l2arc_read_done()
8760 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in l2arc_read_done()
8761 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd; in l2arc_read_done()
8770 ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd || in l2arc_read_done()
8771 (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd)); in l2arc_read_done()
8774 zio->io_prop.zp_complevel = hdr->b_complevel; in l2arc_read_done()
8776 valid_cksum = arc_cksum_is_equal(hdr, zio); in l2arc_read_done()
8787 !HDR_L2_EVICTED(hdr)) { in l2arc_read_done()
8789 zio->io_private = hdr; in l2arc_read_done()
8812 hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd; in l2arc_read_done()
8818 hdr, zio->io_priority, cb->l2rcb_flags, in l2arc_read_done()
8826 for (struct arc_callback *acb = hdr->b_l1hdr.b_acb; in l2arc_read_done()
8920 arc_buf_hdr_t *hdr, *hdr_prev; in l2arc_evict() local
9037 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { in l2arc_evict()
9038 hdr_prev = list_prev(buflist, hdr); in l2arc_evict()
9040 ASSERT(!HDR_EMPTY(hdr)); in l2arc_evict()
9041 hash_lock = HDR_LOCK(hdr); in l2arc_evict()
9062 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_evict()
9065 ASSERT(!HDR_L2_WRITING(hdr)); in l2arc_evict()
9066 ASSERT(!HDR_L2_WRITE_HEAD(hdr)); in l2arc_evict()
9068 if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict || in l2arc_evict()
9069 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { in l2arc_evict()
9078 if (!HDR_HAS_L1HDR(hdr)) { in l2arc_evict()
9079 ASSERT(!HDR_L2_READING(hdr)); in l2arc_evict()
9085 arc_change_state(arc_anon, hdr); in l2arc_evict()
9086 arc_hdr_destroy(hdr); in l2arc_evict()
9088 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only); in l2arc_evict()
9095 if (HDR_L2_READING(hdr)) { in l2arc_evict()
9097 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED); in l2arc_evict()
9100 arc_hdr_l2hdr_destroy(hdr); in l2arc_evict()
9140 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, in l2arc_apply_transforms() argument
9144 abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd; in l2arc_apply_transforms()
9145 enum zio_compress compress = HDR_GET_COMPRESS(hdr); in l2arc_apply_transforms()
9146 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_apply_transforms()
9147 uint64_t size = arc_hdr_size(hdr); in l2arc_apply_transforms()
9148 boolean_t ismd = HDR_ISTYPE_METADATA(hdr); in l2arc_apply_transforms()
9149 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); in l2arc_apply_transforms()
9154 ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in l2arc_apply_transforms()
9155 !HDR_COMPRESSION_ENABLED(hdr)) || in l2arc_apply_transforms()
9156 HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize); in l2arc_apply_transforms()
9164 if (HDR_HAS_RABD(hdr)) { in l2arc_apply_transforms()
9167 abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize); in l2arc_apply_transforms()
9172 if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) && in l2arc_apply_transforms()
9173 !HDR_ENCRYPTED(hdr)) { in l2arc_apply_transforms()
9176 abd_copy(to_write, hdr->b_l1hdr.b_pabd, size); in l2arc_apply_transforms()
9182 if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) { in l2arc_apply_transforms()
9185 size, MIN(size, psize), hdr->b_complevel); in l2arc_apply_transforms()
9200 if (HDR_ENCRYPTED(hdr)) { in l2arc_apply_transforms()
9209 ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj, in l2arc_apply_transforms()
9215 hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt, in l2arc_apply_transforms()
9216 hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd, in l2arc_apply_transforms()
9228 ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN)); in l2arc_apply_transforms()
9238 ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd); in l2arc_apply_transforms()
9280 arc_buf_hdr_t *hdr, *head, *marker; in l2arc_write_buffers() local
9327 hdr = multilist_sublist_head(mls); in l2arc_write_buffers()
9329 hdr = multilist_sublist_tail(mls); in l2arc_write_buffers()
9331 while (hdr != NULL) { in l2arc_write_buffers()
9335 hash_lock = HDR_LOCK(hdr); in l2arc_write_buffers()
9340 hdr = multilist_sublist_next(mls, hdr); in l2arc_write_buffers()
9342 hdr = multilist_sublist_prev(mls, hdr); in l2arc_write_buffers()
9346 passed_sz += HDR_GET_LSIZE(hdr); in l2arc_write_buffers()
9355 if (!l2arc_write_eligible(guid, hdr)) { in l2arc_write_buffers()
9360 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_buffers()
9361 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0); in l2arc_write_buffers()
9362 ASSERT3U(arc_hdr_size(hdr), >, 0); in l2arc_write_buffers()
9363 ASSERT(hdr->b_l1hdr.b_pabd != NULL || in l2arc_write_buffers()
9364 HDR_HAS_RABD(hdr)); in l2arc_write_buffers()
9365 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_write_buffers()
9387 multilist_sublist_insert_after(mls, hdr, in l2arc_write_buffers()
9390 multilist_sublist_insert_before(mls, hdr, in l2arc_write_buffers()
9399 * hdr's data, but if we're sharing data between the in l2arc_write_buffers()
9400 * hdr and one of its bufs, L2ARC needs its own copy of in l2arc_write_buffers()
9409 if (HDR_HAS_RABD(hdr) && psize == asize) { in l2arc_write_buffers()
9410 to_write = hdr->b_crypt_hdr.b_rabd; in l2arc_write_buffers()
9411 } else if ((HDR_COMPRESSION_ENABLED(hdr) || in l2arc_write_buffers()
9412 HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) && in l2arc_write_buffers()
9413 !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) && in l2arc_write_buffers()
9415 to_write = hdr->b_l1hdr.b_pabd; in l2arc_write_buffers()
9418 arc_buf_contents_t type = arc_buf_type(hdr); in l2arc_write_buffers()
9420 ret = l2arc_apply_transforms(spa, hdr, asize, in l2arc_write_buffers()
9423 arc_hdr_clear_flags(hdr, in l2arc_write_buffers()
9432 hdr->b_l2hdr.b_dev = dev; in l2arc_write_buffers()
9433 hdr->b_l2hdr.b_daddr = dev->l2ad_hand; in l2arc_write_buffers()
9434 hdr->b_l2hdr.b_hits = 0; in l2arc_write_buffers()
9435 hdr->b_l2hdr.b_arcs_state = in l2arc_write_buffers()
9436 hdr->b_l1hdr.b_state->arcs_state; in l2arc_write_buffers()
9438 HDR_SET_L2SIZE(hdr, asize); in l2arc_write_buffers()
9439 arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR | in l2arc_write_buffers()
9443 arc_hdr_size(hdr), hdr); in l2arc_write_buffers()
9444 l2arc_hdr_arcstats_increment(hdr); in l2arc_write_buffers()
9456 list_insert_head(&dev->l2ad_buflist, hdr); in l2arc_write_buffers()
9459 boolean_t commit = l2arc_log_blk_insert(dev, hdr); in l2arc_write_buffers()
9476 ZIO_CHECKSUM_OFF, NULL, hdr, in l2arc_write_buffers()
9497 hdr = multilist_sublist_next(mls, marker); in l2arc_write_buffers()
9499 hdr = multilist_sublist_prev(mls, marker); in l2arc_write_buffers()
10333 * it to `hdr'. On success, this function returns 0, otherwise the appropriate
10379 * Attempt to rebuild a device containing no actual dev hdr in l2arc_dev_hdr_read()
10521 * Restores the payload of a log block to ARC. This creates empty ARC hdr
10522 * entries which only contain an l2arc hdr, essentially restoring the
10581 * Restores a single ARC buf hdr from a log entry. The ARC buffer is put
10587 arc_buf_hdr_t *hdr, *exists; in l2arc_hdr_restore() local
10598 hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type, in l2arc_hdr_restore()
10610 l2arc_hdr_arcstats_increment(hdr); in l2arc_hdr_restore()
10614 list_insert_tail(&dev->l2ad_buflist, hdr); in l2arc_hdr_restore()
10615 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr); in l2arc_hdr_restore()
10618 exists = buf_hash_insert(hdr, &hash_lock); in l2arc_hdr_restore()
10621 arc_hdr_destroy(hdr); in l2arc_hdr_restore()
10908 * Inserts ARC buffer header `hdr' into the current L2ARC log block on
10914 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr) in l2arc_log_blk_insert() argument
10925 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_log_blk_insert()
10929 le->le_dva = hdr->b_dva; in l2arc_log_blk_insert()
10930 le->le_birth = hdr->b_birth; in l2arc_log_blk_insert()
10931 le->le_daddr = hdr->b_l2hdr.b_daddr; in l2arc_log_blk_insert()
10934 L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr)); in l2arc_log_blk_insert()
10935 L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr)); in l2arc_log_blk_insert()
10936 L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr)); in l2arc_log_blk_insert()
10937 le->le_complevel = hdr->b_complevel; in l2arc_log_blk_insert()
10938 L2BLK_SET_TYPE((le)->le_prop, hdr->b_type); in l2arc_log_blk_insert()
10939 L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr))); in l2arc_log_blk_insert()
10940 L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr))); in l2arc_log_blk_insert()
10941 L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state); in l2arc_log_blk_insert()
10944 HDR_GET_PSIZE(hdr)); in l2arc_log_blk_insert()