Lines Matching refs:hdr

837 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)  argument
838 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) argument
839 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) argument
840 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) argument
841 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FLAG_FREED_IN_READ) argument
842 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE) argument
844 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) argument
845 #define HDR_L2COMPRESS(hdr) ((hdr)->b_flags & ARC_FLAG_L2COMPRESS) argument
846 #define HDR_L2_READING(hdr) \ argument
847 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
848 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
849 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) argument
850 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) argument
851 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) argument
853 #define HDR_ISTYPE_METADATA(hdr) \ argument
854 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
855 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) argument
857 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) argument
858 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) argument
893 #define HDR_LOCK(hdr) \ argument
894 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
991 arc_update_hit_stat(arc_buf_hdr_t *hdr, boolean_t hit) in arc_update_hit_stat() argument
993 boolean_t pf = !HDR_PREFETCH(hdr); in arc_update_hit_stat()
994 switch (arc_buf_type(hdr)) { in arc_update_hit_stat()
1236 static void l2arc_dev_hdr_checksum(const l2arc_dev_hdr_phys_t *hdr,
1297 buf_discard_identity(arc_buf_hdr_t *hdr) in buf_discard_identity() argument
1299 hdr->b_dva.dva_word[0] = 0; in buf_discard_identity()
1300 hdr->b_dva.dva_word[1] = 0; in buf_discard_identity()
1301 hdr->b_birth = 0; in buf_discard_identity()
1311 arc_buf_hdr_t *hdr; in buf_hash_find() local
1314 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; in buf_hash_find()
1315 hdr = hdr->b_hash_next) { in buf_hash_find()
1316 if (BUF_EQUAL(spa, dva, birth, hdr)) { in buf_hash_find()
1318 return (hdr); in buf_hash_find()
1334 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) in buf_hash_insert() argument
1336 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_insert()
1341 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); in buf_hash_insert()
1342 ASSERT(hdr->b_birth != 0); in buf_hash_insert()
1343 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in buf_hash_insert()
1354 if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) in buf_hash_insert()
1358 hdr->b_hash_next = buf_hash_table.ht_table[idx]; in buf_hash_insert()
1359 buf_hash_table.ht_table[idx] = hdr; in buf_hash_insert()
1360 hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE; in buf_hash_insert()
1378 buf_hash_remove(arc_buf_hdr_t *hdr) in buf_hash_remove() argument
1381 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_remove()
1384 ASSERT(HDR_IN_HASH_TABLE(hdr)); in buf_hash_remove()
1387 while ((fhdr = *hdrp) != hdr) { in buf_hash_remove()
1391 *hdrp = hdr->b_hash_next; in buf_hash_remove()
1392 hdr->b_hash_next = NULL; in buf_hash_remove()
1393 hdr->b_flags &= ~ARC_FLAG_IN_HASH_TABLE; in buf_hash_remove()
1432 arc_buf_hdr_t *hdr = vbuf; in hdr_full_cons() local
1434 bzero(hdr, HDR_FULL_SIZE); in hdr_full_cons()
1435 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL); in hdr_full_cons()
1436 refcount_create(&hdr->b_l1hdr.b_refcnt); in hdr_full_cons()
1437 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); in hdr_full_cons()
1438 multilist_link_init(&hdr->b_l1hdr.b_arc_node); in hdr_full_cons()
1448 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_cons() local
1450 bzero(hdr, HDR_L2ONLY_SIZE); in hdr_l2only_cons()
1477 arc_buf_hdr_t *hdr = vbuf; in hdr_full_dest() local
1479 ASSERT(BUF_EMPTY(hdr)); in hdr_full_dest()
1480 cv_destroy(&hdr->b_l1hdr.b_cv); in hdr_full_dest()
1481 refcount_destroy(&hdr->b_l1hdr.b_refcnt); in hdr_full_dest()
1482 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); in hdr_full_dest()
1483 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in hdr_full_dest()
1491 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_dest() local
1493 ASSERT(BUF_EMPTY(hdr)); in hdr_l2only_dest()
1574 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) in arc_hdr_realloc() argument
1576 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_realloc()
1579 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_realloc()
1586 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_hdr_realloc()
1587 buf_hash_remove(hdr); in arc_hdr_realloc()
1589 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE); in arc_hdr_realloc()
1603 ASSERT(hdr->b_l1hdr.b_buf == NULL); in arc_hdr_realloc()
1604 ASSERT0(hdr->b_l1hdr.b_datacnt); in arc_hdr_realloc()
1613 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_realloc()
1621 VERIFY(!HDR_L2_WRITING(hdr)); in arc_hdr_realloc()
1622 VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL); in arc_hdr_realloc()
1625 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_realloc()
1626 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_hdr_realloc()
1627 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_realloc()
1639 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); in arc_hdr_realloc()
1649 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); in arc_hdr_realloc()
1650 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_realloc()
1663 hdr->b_l2hdr.b_asize, hdr); in arc_hdr_realloc()
1668 buf_discard_identity(hdr); in arc_hdr_realloc()
1669 hdr->b_freeze_cksum = NULL; in arc_hdr_realloc()
1670 kmem_cache_free(old, hdr); in arc_hdr_realloc()
1773 arc_buf_type(arc_buf_hdr_t *hdr) in arc_buf_type() argument
1775 if (HDR_ISTYPE_METADATA(hdr)) { in arc_buf_type()
1855 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) in add_reference() argument
1857 ASSERT(HDR_HAS_L1HDR(hdr)); in add_reference()
1859 arc_state_t *state = hdr->b_l1hdr.b_state; in add_reference()
1861 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && in add_reference()
1865 arc_buf_contents_t type = arc_buf_type(hdr); in add_reference()
1866 uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt; in add_reference()
1870 multilist_remove(list, hdr); in add_reference()
1873 ASSERT0(hdr->b_l1hdr.b_datacnt); in add_reference()
1874 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in add_reference()
1875 delta = hdr->b_size; in add_reference()
1882 hdr->b_flags &= ~ARC_FLAG_PREFETCH; in add_reference()
1887 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) in remove_reference() argument
1890 arc_state_t *state = hdr->b_l1hdr.b_state; in remove_reference()
1892 ASSERT(HDR_HAS_L1HDR(hdr)); in remove_reference()
1900 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && in remove_reference()
1902 arc_buf_contents_t type = arc_buf_type(hdr); in remove_reference()
1906 multilist_insert(list, hdr); in remove_reference()
1908 ASSERT(hdr->b_l1hdr.b_datacnt > 0); in remove_reference()
1909 atomic_add_64(size, hdr->b_size * in remove_reference()
1910 hdr->b_l1hdr.b_datacnt); in remove_reference()
1920 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr, in arc_change_state() argument
1927 arc_buf_contents_t buftype = arc_buf_type(hdr); in arc_change_state()
1936 if (HDR_HAS_L1HDR(hdr)) { in arc_change_state()
1937 old_state = hdr->b_l1hdr.b_state; in arc_change_state()
1938 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt); in arc_change_state()
1939 datacnt = hdr->b_l1hdr.b_datacnt; in arc_change_state()
1952 from_delta = to_delta = datacnt * hdr->b_size; in arc_change_state()
1962 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
1963 multilist_remove(&old_state->arcs_list[buftype], hdr); in arc_change_state()
1971 ASSERT(hdr->b_l1hdr.b_buf == NULL); in arc_change_state()
1972 from_delta = hdr->b_size; in arc_change_state()
1986 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
1987 multilist_insert(&new_state->arcs_list[buftype], hdr); in arc_change_state()
1992 ASSERT(hdr->b_l1hdr.b_buf == NULL); in arc_change_state()
1993 to_delta = hdr->b_size; in arc_change_state()
1999 ASSERT(!BUF_EMPTY(hdr)); in arc_change_state()
2000 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) in arc_change_state()
2001 buf_hash_remove(hdr); in arc_change_state()
2006 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2018 hdr->b_size, hdr); in arc_change_state()
2027 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2030 hdr->b_size, buf); in arc_change_state()
2036 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2054 hdr->b_size, hdr); in arc_change_state()
2063 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2066 &old_state->arcs_size, hdr->b_size, buf); in arc_change_state()
2071 if (HDR_HAS_L1HDR(hdr)) in arc_change_state()
2072 hdr->b_l1hdr.b_state = new_state; in arc_change_state()
2148 arc_buf_hdr_t *hdr; in arc_buf_alloc() local
2152 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); in arc_buf_alloc()
2153 ASSERT(BUF_EMPTY(hdr)); in arc_buf_alloc()
2154 ASSERT3P(hdr->b_freeze_cksum, ==, NULL); in arc_buf_alloc()
2155 hdr->b_size = size; in arc_buf_alloc()
2156 hdr->b_spa = spa_load_guid(spa); in arc_buf_alloc()
2159 buf->b_hdr = hdr; in arc_buf_alloc()
2165 hdr->b_flags = arc_bufc_to_flags(type); in arc_buf_alloc()
2166 hdr->b_flags |= ARC_FLAG_HAS_L1HDR; in arc_buf_alloc()
2168 hdr->b_l1hdr.b_buf = buf; in arc_buf_alloc()
2169 hdr->b_l1hdr.b_state = arc_anon; in arc_buf_alloc()
2170 hdr->b_l1hdr.b_arc_access = 0; in arc_buf_alloc()
2171 hdr->b_l1hdr.b_datacnt = 1; in arc_buf_alloc()
2172 hdr->b_l1hdr.b_tmp_cdata = NULL; in arc_buf_alloc()
2175 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_buf_alloc()
2176 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag); in arc_buf_alloc()
2192 arc_buf_hdr_t *hdr; in arc_buf_alloc_l2only() local
2195 hdr = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE); in arc_buf_alloc_l2only()
2196 ASSERT(BUF_EMPTY(hdr)); in arc_buf_alloc_l2only()
2197 ASSERT3P(hdr->b_freeze_cksum, ==, NULL); in arc_buf_alloc_l2only()
2198 hdr->b_dva = dva; in arc_buf_alloc_l2only()
2199 hdr->b_birth = birth; in arc_buf_alloc_l2only()
2200 hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); in arc_buf_alloc_l2only()
2201 bcopy(&cksum, hdr->b_freeze_cksum, sizeof (cksum)); in arc_buf_alloc_l2only()
2202 hdr->b_flags = arc_bufc_to_flags(type); in arc_buf_alloc_l2only()
2203 hdr->b_flags |= ARC_FLAG_HAS_L2HDR; in arc_buf_alloc_l2only()
2204 hdr->b_size = size; in arc_buf_alloc_l2only()
2205 hdr->b_spa = load_guid; in arc_buf_alloc_l2only()
2207 hdr->b_l2hdr.b_compress = compress; in arc_buf_alloc_l2only()
2208 hdr->b_l2hdr.b_dev = dev; in arc_buf_alloc_l2only()
2209 hdr->b_l2hdr.b_daddr = daddr; in arc_buf_alloc_l2only()
2210 hdr->b_l2hdr.b_asize = asize; in arc_buf_alloc_l2only()
2212 return (hdr); in arc_buf_alloc_l2only()
2240 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_return_buf() local
2243 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_return_buf()
2244 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag); in arc_return_buf()
2245 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_return_buf()
2247 atomic_add_64(&arc_loaned_bytes, -hdr->b_size); in arc_return_buf()
2254 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_loan_inuse_buf() local
2257 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_loan_inuse_buf()
2258 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_loan_inuse_buf()
2259 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); in arc_loan_inuse_buf()
2263 atomic_add_64(&arc_loaned_bytes, hdr->b_size); in arc_loan_inuse_buf()
2270 arc_buf_hdr_t *hdr = from->b_hdr; in arc_buf_clone() local
2271 uint64_t size = hdr->b_size; in arc_buf_clone()
2273 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_clone()
2274 ASSERT(hdr->b_l1hdr.b_state != arc_anon); in arc_buf_clone()
2277 buf->b_hdr = hdr; in arc_buf_clone()
2281 buf->b_next = hdr->b_l1hdr.b_buf; in arc_buf_clone()
2282 hdr->b_l1hdr.b_buf = buf; in arc_buf_clone()
2292 if (HDR_ISTYPE_DATA(hdr)) { in arc_buf_clone()
2296 hdr->b_l1hdr.b_datacnt += 1; in arc_buf_clone()
2303 arc_buf_hdr_t *hdr; in arc_buf_add_ref() local
2318 hdr = buf->b_hdr; in arc_buf_add_ref()
2319 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_add_ref()
2320 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_buf_add_ref()
2323 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_buf_add_ref()
2324 hdr->b_l1hdr.b_state == arc_mfu); in arc_buf_add_ref()
2326 add_reference(hdr, hash_lock, tag); in arc_buf_add_ref()
2327 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_buf_add_ref()
2328 arc_access(hdr, hash_lock); in arc_buf_add_ref()
2331 arc_update_hit_stat(hdr, B_TRUE); in arc_buf_add_ref()
2356 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_data_free() local
2358 if (HDR_L2_WRITING(hdr)) { in arc_buf_data_free()
2359 arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func); in arc_buf_data_free()
2362 free_func(buf->b_data, hdr->b_size); in arc_buf_data_free()
2367 arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr) in arc_buf_l2_cdata_free() argument
2369 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_buf_l2_cdata_free()
2370 ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx)); in arc_buf_l2_cdata_free()
2377 if (!HDR_HAS_L1HDR(hdr)) in arc_buf_l2_cdata_free()
2384 if (!HDR_L2_WRITING(hdr)) { in arc_buf_l2_cdata_free()
2385 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL); in arc_buf_l2_cdata_free()
2398 if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_OFF) { in arc_buf_l2_cdata_free()
2399 hdr->b_l1hdr.b_tmp_cdata = NULL; in arc_buf_l2_cdata_free()
2407 if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_EMPTY) { in arc_buf_l2_cdata_free()
2408 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL); in arc_buf_l2_cdata_free()
2412 ASSERT(L2ARC_IS_VALID_COMPRESS(hdr->b_l2hdr.b_compress)); in arc_buf_l2_cdata_free()
2414 arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata, in arc_buf_l2_cdata_free()
2415 hdr->b_size, zio_data_buf_free); in arc_buf_l2_cdata_free()
2418 hdr->b_l1hdr.b_tmp_cdata = NULL; in arc_buf_l2_cdata_free()
2495 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_l2hdr_destroy() argument
2497 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; in arc_hdr_l2hdr_destroy()
2501 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_l2hdr_destroy()
2503 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_l2hdr_destroy()
2509 arc_buf_l2_cdata_free(hdr); in arc_hdr_l2hdr_destroy()
2521 IMPLY(l2hdr->b_daddr == L2ARC_ADDR_UNSET, HDR_L2_WRITING(hdr)); in arc_hdr_l2hdr_destroy()
2533 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); in arc_hdr_l2hdr_destroy()
2539 l2hdr->b_asize, hdr); in arc_hdr_l2hdr_destroy()
2542 hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR; in arc_hdr_l2hdr_destroy()
2546 arc_hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_destroy() argument
2548 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
2549 ASSERT(hdr->b_l1hdr.b_buf == NULL || in arc_hdr_destroy()
2550 hdr->b_l1hdr.b_datacnt > 0); in arc_hdr_destroy()
2551 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_destroy()
2552 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_hdr_destroy()
2554 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_hdr_destroy()
2555 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_hdr_destroy()
2557 if (HDR_HAS_L2HDR(hdr)) { in arc_hdr_destroy()
2558 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_destroy()
2573 if (HDR_HAS_L2HDR(hdr)) in arc_hdr_destroy()
2574 arc_hdr_l2hdr_destroy(hdr); in arc_hdr_destroy()
2580 if (!BUF_EMPTY(hdr)) in arc_hdr_destroy()
2581 buf_discard_identity(hdr); in arc_hdr_destroy()
2583 if (hdr->b_freeze_cksum != NULL) { in arc_hdr_destroy()
2584 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); in arc_hdr_destroy()
2585 hdr->b_freeze_cksum = NULL; in arc_hdr_destroy()
2588 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
2589 while (hdr->b_l1hdr.b_buf) { in arc_hdr_destroy()
2590 arc_buf_t *buf = hdr->b_l1hdr.b_buf; in arc_hdr_destroy()
2596 arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE); in arc_hdr_destroy()
2597 hdr->b_l1hdr.b_buf = buf->b_next; in arc_hdr_destroy()
2605 arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE); in arc_hdr_destroy()
2609 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_destroy()
2610 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_hdr_destroy()
2611 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_destroy()
2616 ASSERT3P(hdr->b_hash_next, ==, NULL); in arc_hdr_destroy()
2617 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
2618 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_destroy()
2619 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_hdr_destroy()
2620 kmem_cache_free(hdr_full_cache, hdr); in arc_hdr_destroy()
2622 kmem_cache_free(hdr_l2only_cache, hdr); in arc_hdr_destroy()
2629 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_free() local
2630 int hashed = hdr->b_l1hdr.b_state != arc_anon; in arc_buf_free()
2636 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_free()
2639 hdr = buf->b_hdr; in arc_buf_free()
2640 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_buf_free()
2642 (void) remove_reference(hdr, hash_lock, tag); in arc_buf_free()
2643 if (hdr->b_l1hdr.b_datacnt > 1) { in arc_buf_free()
2646 ASSERT(buf == hdr->b_l1hdr.b_buf); in arc_buf_free()
2648 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE; in arc_buf_free()
2651 } else if (HDR_IO_IN_PROGRESS(hdr)) { in arc_buf_free()
2659 (void) remove_reference(hdr, NULL, tag); in arc_buf_free()
2660 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_buf_free()
2661 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); in arc_buf_free()
2664 arc_hdr_destroy(hdr); in arc_buf_free()
2666 if (remove_reference(hdr, NULL, tag) > 0) in arc_buf_free()
2669 arc_hdr_destroy(hdr); in arc_buf_free()
2676 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_remove_ref() local
2677 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_remove_ref()
2680 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_buf_remove_ref()
2681 ASSERT(hdr->b_l1hdr.b_datacnt == 1); in arc_buf_remove_ref()
2687 hdr = buf->b_hdr; in arc_buf_remove_ref()
2688 ASSERT(hdr->b_l1hdr.b_datacnt > 0); in arc_buf_remove_ref()
2689 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_buf_remove_ref()
2690 ASSERT(hdr->b_l1hdr.b_state != arc_anon); in arc_buf_remove_ref()
2693 (void) remove_reference(hdr, hash_lock, tag); in arc_buf_remove_ref()
2694 if (hdr->b_l1hdr.b_datacnt > 1) { in arc_buf_remove_ref()
2698 ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL); in arc_buf_remove_ref()
2700 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE; in arc_buf_remove_ref()
2702 ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 || in arc_buf_remove_ref()
2703 refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_buf_remove_ref()
2723 arc_buf_hdr_t *hdr; in arc_buf_eviction_needed() local
2730 hdr = buf->b_hdr; in arc_buf_eviction_needed()
2731 if (hdr == NULL) { in arc_buf_eviction_needed()
2744 ASSERT3P(hdr, ==, &arc_eviction_hdr); in arc_buf_eviction_needed()
2749 if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr)) in arc_buf_eviction_needed()
2769 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) in arc_evict_hdr() argument
2775 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evict_hdr()
2777 state = hdr->b_l1hdr.b_state; in arc_evict_hdr()
2779 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_evict_hdr()
2780 ASSERT(hdr->b_l1hdr.b_buf == NULL); in arc_evict_hdr()
2789 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { in arc_evict_hdr()
2795 bytes_evicted += hdr->b_size; in arc_evict_hdr()
2797 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
2799 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
2804 arc_change_state(arc_l2c_only, hdr, hash_lock); in arc_evict_hdr()
2809 hdr = arc_hdr_realloc(hdr, hdr_full_cache, in arc_evict_hdr()
2812 arc_change_state(arc_anon, hdr, hash_lock); in arc_evict_hdr()
2813 arc_hdr_destroy(hdr); in arc_evict_hdr()
2822 if (HDR_IO_IN_PROGRESS(hdr) || in arc_evict_hdr()
2823 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && in arc_evict_hdr()
2824 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < in arc_evict_hdr()
2830 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_evict_hdr()
2831 ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0); in arc_evict_hdr()
2832 while (hdr->b_l1hdr.b_buf) { in arc_evict_hdr()
2833 arc_buf_t *buf = hdr->b_l1hdr.b_buf; in arc_evict_hdr()
2839 bytes_evicted += hdr->b_size; in arc_evict_hdr()
2843 hdr->b_l1hdr.b_buf = buf->b_next; in arc_evict_hdr()
2856 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
2857 ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size); in arc_evict_hdr()
2859 if (l2arc_write_eligible(hdr->b_spa, UINT64_MAX, hdr)) in arc_evict_hdr()
2860 ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size); in arc_evict_hdr()
2862 ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size); in arc_evict_hdr()
2865 if (hdr->b_l1hdr.b_datacnt == 0) { in arc_evict_hdr()
2866 arc_change_state(evicted_state, hdr, hash_lock); in arc_evict_hdr()
2867 ASSERT(HDR_IN_HASH_TABLE(hdr)); in arc_evict_hdr()
2868 hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE; in arc_evict_hdr()
2869 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE; in arc_evict_hdr()
2870 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
2882 arc_buf_hdr_t *hdr; in arc_evict_state_impl() local
2891 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL; in arc_evict_state_impl()
2892 hdr = multilist_sublist_prev(mls, marker)) { in arc_evict_state_impl()
2919 if (hdr->b_spa == 0) in arc_evict_state_impl()
2923 if (spa != 0 && hdr->b_spa != spa) { in arc_evict_state_impl()
2928 hash_lock = HDR_LOCK(hdr); in arc_evict_state_impl()
2942 uint64_t evicted = arc_evict_hdr(hdr, hash_lock); in arc_evict_state_impl()
3988 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_get_data_buf() local
3989 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_get_data_buf()
4002 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_get_data_buf()
4003 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_get_data_buf()
4004 atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type], in arc_get_data_buf()
4011 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon && in arc_get_data_buf()
4023 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) in arc_access() argument
4028 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_access()
4030 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_access()
4037 ASSERT0(hdr->b_l1hdr.b_arc_access); in arc_access()
4038 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4039 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
4040 arc_change_state(arc_mru, hdr, hash_lock); in arc_access()
4042 } else if (hdr->b_l1hdr.b_state == arc_mru) { in arc_access()
4053 if (HDR_PREFETCH(hdr)) { in arc_access()
4054 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { in arc_access()
4057 &hdr->b_l1hdr.b_arc_node)); in arc_access()
4059 hdr->b_flags &= ~ARC_FLAG_PREFETCH; in arc_access()
4062 hdr->b_l1hdr.b_arc_access = now; in arc_access()
4071 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) { in arc_access()
4077 hdr->b_l1hdr.b_arc_access = now; in arc_access()
4078 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4079 arc_change_state(arc_mfu, hdr, hash_lock); in arc_access()
4082 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { in arc_access()
4090 if (HDR_PREFETCH(hdr)) { in arc_access()
4092 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) in arc_access()
4093 hdr->b_flags &= ~ARC_FLAG_PREFETCH; in arc_access()
4094 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
4097 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4100 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4101 arc_change_state(new_state, hdr, hash_lock); in arc_access()
4104 } else if (hdr->b_l1hdr.b_state == arc_mfu) { in arc_access()
4114 if ((HDR_PREFETCH(hdr)) != 0) { in arc_access()
4115 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_access()
4117 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_access()
4120 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4121 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { in arc_access()
4129 if (HDR_PREFETCH(hdr)) { in arc_access()
4134 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_access()
4138 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4139 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4140 arc_change_state(new_state, hdr, hash_lock); in arc_access()
4143 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { in arc_access()
4148 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4149 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4150 arc_change_state(arc_mfu, hdr, hash_lock); in arc_access()
4183 arc_buf_hdr_t *hdr; in arc_read_done() local
4191 hdr = buf->b_hdr; in arc_read_done()
4201 if (HDR_IN_HASH_TABLE(hdr)) { in arc_read_done()
4202 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); in arc_read_done()
4203 ASSERT3U(hdr->b_dva.dva_word[0], ==, in arc_read_done()
4205 ASSERT3U(hdr->b_dva.dva_word[1], ==, in arc_read_done()
4208 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, in arc_read_done()
4211 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && in arc_read_done()
4213 (found == hdr && in arc_read_done()
4214 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || in arc_read_done()
4215 (found == hdr && HDR_L2_READING(hdr))); in arc_read_done()
4218 hdr->b_flags &= ~ARC_FLAG_L2_EVICTED; in arc_read_done()
4219 if (l2arc_noprefetch && HDR_PREFETCH(hdr)) in arc_read_done()
4220 hdr->b_flags &= ~ARC_FLAG_L2CACHE; in arc_read_done()
4223 callback_list = hdr->b_l1hdr.b_acb; in arc_read_done()
4231 func(buf->b_data, hdr->b_size); in arc_read_done()
4238 hdr->b_l1hdr.b_state == arc_anon) { in arc_read_done()
4245 arc_access(hdr, hash_lock); in arc_read_done()
4260 hdr->b_l1hdr.b_acb = NULL; in arc_read_done()
4261 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS; in arc_read_done()
4262 ASSERT(!HDR_BUF_AVAILABLE(hdr)); in arc_read_done()
4265 ASSERT(hdr->b_l1hdr.b_datacnt == 1); in arc_read_done()
4266 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE; in arc_read_done()
4269 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || in arc_read_done()
4273 hdr->b_flags |= ARC_FLAG_IO_ERROR; in arc_read_done()
4274 if (hdr->b_l1hdr.b_state != arc_anon) in arc_read_done()
4275 arc_change_state(arc_anon, hdr, hash_lock); in arc_read_done()
4276 if (HDR_IN_HASH_TABLE(hdr)) in arc_read_done()
4277 buf_hash_remove(hdr); in arc_read_done()
4278 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); in arc_read_done()
4286 cv_broadcast(&hdr->b_l1hdr.b_cv); in arc_read_done()
4297 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_read_done()
4298 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); in arc_read_done()
4316 arc_hdr_destroy(hdr); in arc_read_done()
4342 arc_buf_hdr_t *hdr = NULL; in arc_read() local
4357 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_read()
4360 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) { in arc_read()
4364 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_read()
4366 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && in arc_read()
4389 arc_buf_hdr_t *, hdr); in arc_read()
4392 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { in arc_read()
4393 hdr->b_flags &= ~ARC_FLAG_PREDICTIVE_PREFETCH; in arc_read()
4397 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); in arc_read()
4415 acb->acb_next = hdr->b_l1hdr.b_acb; in arc_read()
4416 hdr->b_l1hdr.b_acb = acb; in arc_read()
4417 add_reference(hdr, hash_lock, private); in arc_read()
4425 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_read()
4426 hdr->b_l1hdr.b_state == arc_mfu); in arc_read()
4429 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { in arc_read()
4437 arc_buf_hdr_t *, hdr); in arc_read()
4440 hdr->b_flags &= ~ARC_FLAG_PREDICTIVE_PREFETCH; in arc_read()
4442 add_reference(hdr, hash_lock, private); in arc_read()
4448 buf = hdr->b_l1hdr.b_buf; in arc_read()
4451 if (HDR_BUF_AVAILABLE(hdr)) { in arc_read()
4453 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE; in arc_read()
4459 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { in arc_read()
4460 hdr->b_flags |= ARC_FLAG_PREFETCH; in arc_read()
4462 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
4463 arc_access(hdr, hash_lock); in arc_read()
4465 hdr->b_flags |= ARC_FLAG_L2CACHE; in arc_read()
4467 hdr->b_flags |= ARC_FLAG_L2COMPRESS; in arc_read()
4470 arc_update_hit_stat(hdr, B_TRUE); in arc_read()
4483 if (hdr == NULL) { in arc_read()
4488 hdr = buf->b_hdr; in arc_read()
4490 hdr->b_dva = *BP_IDENTITY(bp); in arc_read()
4491 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); in arc_read()
4492 exists = buf_hash_insert(hdr, &hash_lock); in arc_read()
4497 buf_discard_identity(hdr); in arc_read()
4507 (void) remove_reference(hdr, hash_lock, in arc_read()
4511 hdr->b_flags |= ARC_FLAG_PREFETCH; in arc_read()
4513 hdr->b_flags |= ARC_FLAG_L2CACHE; in arc_read()
4515 hdr->b_flags |= ARC_FLAG_L2COMPRESS; in arc_read()
4517 hdr->b_flags |= ARC_FLAG_INDIRECT; in arc_read()
4524 if (!HDR_HAS_L1HDR(hdr)) { in arc_read()
4525 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, in arc_read()
4529 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state)); in arc_read()
4530 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_read()
4531 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_read()
4532 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_read()
4538 add_reference(hdr, hash_lock, private); in arc_read()
4540 hdr->b_flags |= ARC_FLAG_PREFETCH; in arc_read()
4542 hdr->b_flags |= ARC_FLAG_L2CACHE; in arc_read()
4544 hdr->b_flags |= ARC_FLAG_L2COMPRESS; in arc_read()
4546 buf->b_hdr = hdr; in arc_read()
4551 hdr->b_l1hdr.b_buf = buf; in arc_read()
4552 ASSERT0(hdr->b_l1hdr.b_datacnt); in arc_read()
4553 hdr->b_l1hdr.b_datacnt = 1; in arc_read()
4555 arc_access(hdr, hash_lock); in arc_read()
4559 hdr->b_flags |= ARC_FLAG_PREDICTIVE_PREFETCH; in arc_read()
4560 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); in arc_read()
4566 ASSERT(hdr->b_l1hdr.b_acb == NULL); in arc_read()
4567 hdr->b_l1hdr.b_acb = acb; in arc_read()
4568 hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS; in arc_read()
4570 if (HDR_HAS_L2HDR(hdr) && in arc_read()
4571 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { in arc_read()
4572 devw = hdr->b_l2hdr.b_dev->l2ad_writing; in arc_read()
4573 addr = hdr->b_l2hdr.b_daddr; in arc_read()
4574 b_compress = hdr->b_l2hdr.b_compress; in arc_read()
4575 b_asize = hdr->b_l2hdr.b_asize; in arc_read()
4591 ASSERT3U(hdr->b_size, ==, size); in arc_read()
4592 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, in arc_read()
4595 arc_update_hit_stat(hdr, B_FALSE); in arc_read()
4598 hdr->b_flags |= ARC_FLAG_PRIO_ASYNC_READ; in arc_read()
4600 hdr->b_flags &= ~ARC_FLAG_PRIO_ASYNC_READ; in arc_read()
4612 if (HDR_HAS_L2HDR(hdr) && in arc_read()
4613 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && in arc_read()
4614 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { in arc_read()
4617 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
4672 arc_buf_hdr_t *, hdr); in arc_read()
4674 if (HDR_L2_WRITING(hdr)) in arc_read()
4683 arc_buf_hdr_t *, hdr); in arc_read()
4720 arc_buf_hdr_t *hdr; in arc_freed() local
4726 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_freed()
4727 if (hdr == NULL) in arc_freed()
4729 if (HDR_BUF_AVAILABLE(hdr)) { in arc_freed()
4730 arc_buf_t *buf = hdr->b_l1hdr.b_buf; in arc_freed()
4731 add_reference(hdr, hash_lock, FTAG); in arc_freed()
4732 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE; in arc_freed()
4759 arc_buf_hdr_t *hdr; in arc_clear_callback() local
4765 hdr = buf->b_hdr; in arc_clear_callback()
4766 if (hdr == NULL) { in arc_clear_callback()
4783 hash_lock = HDR_LOCK(hdr); in arc_clear_callback()
4785 hdr = buf->b_hdr; in arc_clear_callback()
4786 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_clear_callback()
4788 ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <, in arc_clear_callback()
4789 hdr->b_l1hdr.b_datacnt); in arc_clear_callback()
4790 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_clear_callback()
4791 hdr->b_l1hdr.b_state == arc_mfu); in arc_clear_callback()
4796 if (hdr->b_l1hdr.b_datacnt > 1) { in arc_clear_callback()
4800 ASSERT(buf == hdr->b_l1hdr.b_buf); in arc_clear_callback()
4801 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE; in arc_clear_callback()
4819 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_release() local
4829 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_release()
4836 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_release()
4838 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
4839 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_release()
4840 ASSERT(!HDR_HAS_L2HDR(hdr)); in arc_release()
4841 ASSERT(BUF_EMPTY(hdr)); in arc_release()
4843 ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1); in arc_release()
4844 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); in arc_release()
4845 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
4850 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
4856 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_release()
4864 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_release()
4865 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_release()
4869 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0); in arc_release()
4871 if (HDR_HAS_L2HDR(hdr)) { in arc_release()
4872 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
4882 if (HDR_HAS_L2HDR(hdr)) in arc_release()
4883 arc_hdr_l2hdr_destroy(hdr); in arc_release()
4885 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
4891 if (hdr->b_l1hdr.b_datacnt > 1) { in arc_release()
4894 uint64_t blksz = hdr->b_size; in arc_release()
4895 uint64_t spa = hdr->b_spa; in arc_release()
4896 arc_buf_contents_t type = arc_buf_type(hdr); in arc_release()
4897 uint32_t flags = hdr->b_flags; in arc_release()
4899 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL); in arc_release()
4904 (void) remove_reference(hdr, hash_lock, tag); in arc_release()
4905 bufp = &hdr->b_l1hdr.b_buf; in arc_release()
4914 &state->arcs_size, hdr->b_size, buf); in arc_release()
4916 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_release()
4919 ASSERT3U(*size, >=, hdr->b_size); in arc_release()
4920 atomic_add_64(size, -hdr->b_size); in arc_release()
4927 if (HDR_ISTYPE_DATA(hdr)) { in arc_release()
4930 -hdr->b_size); in arc_release()
4932 hdr->b_l1hdr.b_datacnt -= 1; in arc_release()
4959 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1); in arc_release()
4961 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
4962 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
4963 arc_change_state(arc_anon, hdr, hash_lock); in arc_release()
4964 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
4967 buf_discard_identity(hdr); in arc_release()
5004 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_ready() local
5006 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_write_ready()
5008 ASSERT(hdr->b_l1hdr.b_datacnt > 0); in arc_write_ready()
5017 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_write_ready()
5018 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_write_ready()
5019 if (hdr->b_freeze_cksum != NULL) { in arc_write_ready()
5020 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); in arc_write_ready()
5021 hdr->b_freeze_cksum = NULL; in arc_write_ready()
5023 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_write_ready()
5026 hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS; in arc_write_ready()
5046 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_done() local
5048 ASSERT(hdr->b_l1hdr.b_acb == NULL); in arc_write_done()
5052 buf_discard_identity(hdr); in arc_write_done()
5054 hdr->b_dva = *BP_IDENTITY(zio->io_bp); in arc_write_done()
5055 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); in arc_write_done()
5058 ASSERT(BUF_EMPTY(hdr)); in arc_write_done()
5067 if (!BUF_EMPTY(hdr)) { in arc_write_done()
5075 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
5085 (void *)hdr, (void *)exists); in arc_write_done()
5091 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
5098 (void *)hdr, (void *)exists); in arc_write_done()
5101 ASSERT(hdr->b_l1hdr.b_datacnt == 1); in arc_write_done()
5102 ASSERT(hdr->b_l1hdr.b_state == arc_anon); in arc_write_done()
5107 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS; in arc_write_done()
5109 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon) in arc_write_done()
5110 arc_access(hdr, hash_lock); in arc_write_done()
5113 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS; in arc_write_done()
5116 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_write_done()
5129 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write() local
5135 ASSERT(!HDR_IO_ERROR(hdr)); in arc_write()
5136 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_write()
5137 ASSERT(hdr->b_l1hdr.b_acb == NULL); in arc_write()
5138 ASSERT(hdr->b_l1hdr.b_datacnt > 0); in arc_write()
5140 hdr->b_flags |= ARC_FLAG_L2CACHE; in arc_write()
5142 hdr->b_flags |= ARC_FLAG_L2COMPRESS; in arc_write()
5150 zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp, in arc_write()
5307 arc_buf_hdr_t *hdr = obj; in arc_state_multilist_index_func() local
5314 ASSERT(!BUF_EMPTY(hdr)); in arc_state_multilist_index_func()
5328 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) % in arc_state_multilist_index_func()
5836 l2arc_write_eligible(uint64_t spa_guid, uint64_t sync_txg, arc_buf_hdr_t *hdr) in l2arc_write_eligible() argument
5846 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) || in l2arc_write_eligible()
5847 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr) || in l2arc_write_eligible()
5848 hdr->b_birth >= sync_txg) in l2arc_write_eligible()
5995 arc_buf_hdr_t *head, *hdr, *hdr_prev; in l2arc_write_done() local
6019 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) { in l2arc_write_done()
6020 hdr_prev = list_prev(buflist, hdr); in l2arc_write_done()
6022 hash_lock = HDR_LOCK(hdr); in l2arc_write_done()
6043 list_insert_after(buflist, hdr, head); in l2arc_write_done()
6063 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_done()
6069 l2arc_release_cdata_buf(hdr); in l2arc_write_done()
6075 list_remove(buflist, hdr); in l2arc_write_done()
6076 hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR; in l2arc_write_done()
6078 ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize); in l2arc_write_done()
6079 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); in l2arc_write_done()
6081 bytes_dropped += hdr->b_l2hdr.b_asize; in l2arc_write_done()
6083 hdr->b_l2hdr.b_asize, hdr); in l2arc_write_done()
6090 hdr->b_flags &= ~ARC_FLAG_L2_WRITING; in l2arc_write_done()
6120 arc_buf_hdr_t *hdr; in l2arc_read_done() local
6137 hdr = buf->b_hdr; in l2arc_read_done()
6138 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in l2arc_read_done()
6144 l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress); in l2arc_read_done()
6146 ASSERT3U(zio->io_size, ==, hdr->b_size); in l2arc_read_done()
6147 ASSERT3U(BP_GET_LSIZE(&cb->l2rcb_bp), ==, hdr->b_size); in l2arc_read_done()
6153 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { in l2arc_read_done()
6184 buf->b_data, hdr->b_size, arc_read_done, buf, in l2arc_read_done()
6258 arc_buf_hdr_t *hdr, *hdr_prev; in l2arc_evict() local
6290 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { in l2arc_evict()
6291 hdr_prev = list_prev(buflist, hdr); in l2arc_evict()
6293 hash_lock = HDR_LOCK(hdr); in l2arc_evict()
6311 if (HDR_L2_WRITE_HEAD(hdr)) { in l2arc_evict()
6316 list_remove(buflist, hdr); in l2arc_evict()
6321 if (!all && HDR_HAS_L2HDR(hdr) && in l2arc_evict()
6322 (hdr->b_l2hdr.b_daddr > taddr || in l2arc_evict()
6323 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { in l2arc_evict()
6332 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_evict()
6333 if (!HDR_HAS_L1HDR(hdr)) { in l2arc_evict()
6334 ASSERT(!HDR_L2_READING(hdr)); in l2arc_evict()
6340 arc_change_state(arc_anon, hdr, hash_lock); in l2arc_evict()
6341 arc_hdr_destroy(hdr); in l2arc_evict()
6343 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only); in l2arc_evict()
6350 if (HDR_L2_READING(hdr)) { in l2arc_evict()
6352 hdr->b_flags |= ARC_FLAG_L2_EVICTED; in l2arc_evict()
6356 ASSERT(!HDR_L2_WRITING(hdr)); in l2arc_evict()
6357 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL); in l2arc_evict()
6359 arc_hdr_l2hdr_destroy(hdr); in l2arc_evict()
6381 arc_buf_hdr_t *hdr, *hdr_prev, *head; in l2arc_write_buffers() local
6426 hdr = multilist_sublist_head(mls); in l2arc_write_buffers()
6428 hdr = multilist_sublist_tail(mls); in l2arc_write_buffers()
6434 for (; hdr; hdr = hdr_prev) { in l2arc_write_buffers()
6440 hdr_prev = multilist_sublist_next(mls, hdr); in l2arc_write_buffers()
6442 hdr_prev = multilist_sublist_prev(mls, hdr); in l2arc_write_buffers()
6444 hash_lock = HDR_LOCK(hdr); in l2arc_write_buffers()
6452 passed_sz += hdr->b_size; in l2arc_write_buffers()
6461 if (!l2arc_write_eligible(guid, sync_txg, hdr)) { in l2arc_write_buffers()
6471 buf_sz = hdr->b_size; in l2arc_write_buffers()
6504 hdr->b_l2hdr.b_dev = dev; in l2arc_write_buffers()
6505 hdr->b_flags |= ARC_FLAG_L2_WRITING; in l2arc_write_buffers()
6514 hdr->b_l2hdr.b_compress = ZIO_COMPRESS_OFF; in l2arc_write_buffers()
6515 hdr->b_l2hdr.b_asize = hdr->b_size; in l2arc_write_buffers()
6516 hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data; in l2arc_write_buffers()
6539 hdr->b_l2hdr.b_daddr = L2ARC_ADDR_UNSET; in l2arc_write_buffers()
6541 hdr->b_flags |= ARC_FLAG_HAS_L2HDR; in l2arc_write_buffers()
6544 list_insert_head(&dev->l2ad_buflist, hdr); in l2arc_write_buffers()
6551 arc_cksum_verify(hdr->b_l1hdr.b_buf); in l2arc_write_buffers()
6552 arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE); in l2arc_write_buffers()
6594 for (hdr = list_prev(&dev->l2ad_buflist, head); hdr; in l2arc_write_buffers()
6595 hdr = list_prev(&dev->l2ad_buflist, hdr)) { in l2arc_write_buffers()
6604 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_buffers()
6613 hdr->b_l2hdr.b_daddr = dev->l2ad_hand; in l2arc_write_buffers()
6615 if ((HDR_L2COMPRESS(hdr)) && in l2arc_write_buffers()
6616 hdr->b_l2hdr.b_asize >= buf_compress_minsz) { in l2arc_write_buffers()
6617 if (l2arc_compress_buf(hdr)) { in l2arc_write_buffers()
6630 buf_data = hdr->b_l1hdr.b_tmp_cdata; in l2arc_write_buffers()
6631 buf_sz = hdr->b_l2hdr.b_asize; in l2arc_write_buffers()
6638 (void) refcount_add_many(&dev->l2ad_alloc, buf_sz, hdr); in l2arc_write_buffers()
6667 if (l2arc_log_blk_insert(dev, hdr)) { in l2arc_write_buffers()
6726 l2arc_compress_buf(arc_buf_hdr_t *hdr) in l2arc_compress_buf() argument
6730 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_compress_buf()
6731 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; in l2arc_compress_buf()
6733 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_compress_buf()
6735 ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL); in l2arc_compress_buf()
6740 csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata, in l2arc_compress_buf()
6754 hdr->b_l1hdr.b_tmp_cdata = NULL; in l2arc_compress_buf()
6764 hdr->b_l1hdr.b_tmp_cdata = cdata; in l2arc_compress_buf()
6790 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c) in l2arc_decompress_zio() argument
6799 zio->io_orig_size = zio->io_size = hdr->b_size; in l2arc_decompress_zio()
6809 ASSERT(hdr->b_l1hdr.b_buf != NULL); in l2arc_decompress_zio()
6810 bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size); in l2arc_decompress_zio()
6811 zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data; in l2arc_decompress_zio()
6831 hdr->b_size) != 0) in l2arc_decompress_zio()
6837 zio->io_orig_size = zio->io_size = hdr->b_size; in l2arc_decompress_zio()
6847 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr) in l2arc_release_cdata_buf() argument
6849 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_release_cdata_buf()
6850 enum zio_compress comp = hdr->b_l2hdr.b_compress; in l2arc_release_cdata_buf()
6852 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_release_cdata_buf()
6861 hdr->b_l1hdr.b_tmp_cdata = NULL; in l2arc_release_cdata_buf()
6868 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL); in l2arc_release_cdata_buf()
6874 ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL); in l2arc_release_cdata_buf()
6875 zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata, in l2arc_release_cdata_buf()
6876 hdr->b_size); in l2arc_release_cdata_buf()
6877 hdr->b_l1hdr.b_tmp_cdata = NULL; in l2arc_release_cdata_buf()
7415 l2arc_dev_hdr_phys_t *hdr = dev->l2ad_dev_hdr; in l2arc_dev_hdr_read() local
7421 VDEV_LABEL_START_SIZE, hdr_asize, hdr, in l2arc_dev_hdr_read()
7431 if (hdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC)) in l2arc_dev_hdr_read()
7432 byteswap_uint64_array(hdr, sizeof (*hdr)); in l2arc_dev_hdr_read()
7434 if (hdr->dh_magic != L2ARC_DEV_HDR_MAGIC || hdr->dh_spa_guid != guid) { in l2arc_dev_hdr_read()
7443 l2arc_dev_hdr_checksum(hdr, &cksum); in l2arc_dev_hdr_read()
7444 if (!ZIO_CHECKSUM_EQUAL(hdr->dh_self_cksum, cksum)) { in l2arc_dev_hdr_read()
7627 arc_buf_hdr_t *hdr, *exists; in l2arc_hdr_restore() local
7638 hdr = arc_buf_alloc_l2only(load_guid, LE_GET_LSIZE(le), type, in l2arc_hdr_restore()
7641 if (hdr->b_l2hdr.b_daddr != L2ARC_ADDR_UNSET) { in l2arc_hdr_restore()
7642 ARCSTAT_INCR(arcstat_l2_size, hdr->b_size); in l2arc_hdr_restore()
7643 ARCSTAT_INCR(arcstat_l2_asize, hdr->b_l2hdr.b_asize); in l2arc_hdr_restore()
7652 list_insert_tail(&dev->l2ad_buflist, hdr); in l2arc_hdr_restore()
7653 (void) refcount_add_many(&dev->l2ad_alloc, hdr->b_l2hdr.b_asize, hdr); in l2arc_hdr_restore()
7656 exists = buf_hash_insert(hdr, &hash_lock); in l2arc_hdr_restore()
7660 arc_hdr_destroy(hdr); in l2arc_hdr_restore()
7717 l2arc_dev_hdr_phys_t *hdr = dev->l2ad_dev_hdr; in l2arc_dev_hdr_update() local
7720 hdr->dh_magic = L2ARC_DEV_HDR_MAGIC; in l2arc_dev_hdr_update()
7721 hdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa); in l2arc_dev_hdr_update()
7722 hdr->dh_alloc_space = refcount_count(&dev->l2ad_alloc); in l2arc_dev_hdr_update()
7723 hdr->dh_flags = 0; in l2arc_dev_hdr_update()
7725 hdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST; in l2arc_dev_hdr_update()
7728 l2arc_dev_hdr_checksum(hdr, &hdr->dh_self_cksum); in l2arc_dev_hdr_update()
7731 hdr_asize, hdr, ZIO_CHECKSUM_OFF, NULL, NULL, in l2arc_dev_hdr_update()
7844 l2arc_dev_hdr_checksum(const l2arc_dev_hdr_phys_t *hdr, zio_cksum_t *cksum) in l2arc_dev_hdr_checksum() argument
7846 fletcher_4_native((uint8_t *)hdr + in l2arc_dev_hdr_checksum()
7848 sizeof (*hdr) - offsetof(l2arc_dev_hdr_phys_t, dh_spa_guid), in l2arc_dev_hdr_checksum()