Lines Matching full:db
292 dmu_buf_impl_t *db = vdb; in dbuf_cons() local
293 memset(db, 0, sizeof (dmu_buf_impl_t)); in dbuf_cons()
295 mutex_init(&db->db_mtx, NULL, MUTEX_NOLOCKDEP, NULL); in dbuf_cons()
296 rw_init(&db->db_rwlock, NULL, RW_NOLOCKDEP, NULL); in dbuf_cons()
297 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); in dbuf_cons()
298 multilist_link_init(&db->db_cache_link); in dbuf_cons()
299 zfs_refcount_create(&db->db_holds); in dbuf_cons()
308 dmu_buf_impl_t *db = vdb; in dbuf_dest() local
309 mutex_destroy(&db->db_mtx); in dbuf_dest()
310 rw_destroy(&db->db_rwlock); in dbuf_dest()
311 cv_destroy(&db->db_changed); in dbuf_dest()
312 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_dest()
313 zfs_refcount_destroy(&db->db_holds); in dbuf_dest()
331 #define DTRACE_SET_STATE(db, why) \ argument
332 DTRACE_PROBE2(dbuf__state_change, dmu_buf_impl_t *, db, \
336 ((dbuf)->db.db_object == (obj) && \
348 dmu_buf_impl_t *db; in dbuf_find() local
354 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { in dbuf_find()
355 if (DBUF_EQUAL(db, os, obj, level, blkid)) { in dbuf_find()
356 mutex_enter(&db->db_mtx); in dbuf_find()
357 if (db->db_state != DB_EVICTING) { in dbuf_find()
359 return (db); in dbuf_find()
361 mutex_exit(&db->db_mtx); in dbuf_find()
374 dmu_buf_impl_t *db = NULL; in dbuf_find_bonus() local
379 db = dn->dn_bonus; in dbuf_find_bonus()
380 mutex_enter(&db->db_mtx); in dbuf_find_bonus()
385 return (db); in dbuf_find_bonus()
395 dbuf_hash_insert(dmu_buf_impl_t *db) in dbuf_hash_insert() argument
398 objset_t *os = db->db_objset; in dbuf_hash_insert()
399 uint64_t obj = db->db.db_object; in dbuf_hash_insert()
400 int level = db->db_level; in dbuf_hash_insert()
405 blkid = db->db_blkid; in dbuf_hash_insert()
406 ASSERT3U(dbuf_hash(os, obj, level, blkid), ==, db->db_hash); in dbuf_hash_insert()
407 idx = db->db_hash & h->hash_table_mask; in dbuf_hash_insert()
430 mutex_enter(&db->db_mtx); in dbuf_hash_insert()
431 db->db_hash_next = h->hash_table[idx]; in dbuf_hash_insert()
432 h->hash_table[idx] = db; in dbuf_hash_insert()
445 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db) in dbuf_include_in_metadata_cache() argument
447 DB_DNODE_ENTER(db); in dbuf_include_in_metadata_cache()
448 dmu_object_type_t type = DB_DNODE(db)->dn_type; in dbuf_include_in_metadata_cache()
449 DB_DNODE_EXIT(db); in dbuf_include_in_metadata_cache()
477 dbuf_hash_remove(dmu_buf_impl_t *db) in dbuf_hash_remove() argument
483 ASSERT3U(dbuf_hash(db->db_objset, db->db.db_object, db->db_level, in dbuf_hash_remove()
484 db->db_blkid), ==, db->db_hash); in dbuf_hash_remove()
485 idx = db->db_hash & h->hash_table_mask; in dbuf_hash_remove()
491 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hash_remove()
492 ASSERT(db->db_state == DB_EVICTING); in dbuf_hash_remove()
493 ASSERT(!MUTEX_HELD(&db->db_mtx)); in dbuf_hash_remove()
497 while ((dbf = *dbp) != db) { in dbuf_hash_remove()
501 *dbp = db->db_hash_next; in dbuf_hash_remove()
502 db->db_hash_next = NULL; in dbuf_hash_remove()
516 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) in dbuf_verify_user() argument
521 if (db->db_user == NULL) in dbuf_verify_user()
525 ASSERT(db->db_level == 0); in dbuf_verify_user()
528 ASSERT(db->db.db_data != NULL); in dbuf_verify_user()
529 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_verify_user()
531 holds = zfs_refcount_count(&db->db_holds); in dbuf_verify_user()
542 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
544 if (db->db_user_immediate_evict == TRUE) in dbuf_verify_user()
545 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
553 dbuf_evict_user(dmu_buf_impl_t *db) in dbuf_evict_user() argument
555 dmu_buf_user_t *dbu = db->db_user; in dbuf_evict_user()
557 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_evict_user()
562 dbuf_verify_user(db, DBVU_EVICTING); in dbuf_evict_user()
563 db->db_user = NULL; in dbuf_evict_user()
570 if (db->db_caching_status != DB_NO_CACHE) { in dbuf_evict_user()
580 &dbuf_caches[db->db_caching_status].size, size, dbu); in dbuf_evict_user()
581 if (db->db_caching_status == DB_DBUF_CACHE) in dbuf_evict_user()
582 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size); in dbuf_evict_user()
607 dbuf_is_metadata(dmu_buf_impl_t *db) in dbuf_is_metadata() argument
612 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) { in dbuf_is_metadata()
617 DB_DNODE_ENTER(db); in dbuf_is_metadata()
618 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); in dbuf_is_metadata()
619 DB_DNODE_EXIT(db); in dbuf_is_metadata()
630 dbuf_is_l2cacheable(dmu_buf_impl_t *db, blkptr_t *bp) in dbuf_is_l2cacheable() argument
632 if (db->db_objset->os_secondary_cache == ZFS_CACHE_ALL || in dbuf_is_l2cacheable()
633 (db->db_objset->os_secondary_cache == in dbuf_is_l2cacheable()
634 ZFS_CACHE_METADATA && dbuf_is_metadata(db))) { in dbuf_is_l2cacheable()
644 blkptr_t *db_bp = bp == NULL ? db->db_blkptr : bp; in dbuf_is_l2cacheable()
649 vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev; in dbuf_is_l2cacheable()
705 dmu_buf_impl_t *db = obj; in dbuf_cache_multilist_index_func() local
720 return ((unsigned int)dbuf_hash(db->db_objset, db->db.db_object, in dbuf_cache_multilist_index_func()
721 db->db_level, db->db_blkid) % in dbuf_cache_multilist_index_func()
782 dmu_buf_impl_t *db = multilist_sublist_tail(mls); in dbuf_evict_one() local
783 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { in dbuf_evict_one()
784 db = multilist_sublist_prev(mls, db); in dbuf_evict_one()
787 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, in dbuf_evict_one()
790 if (db != NULL) { in dbuf_evict_one()
791 multilist_sublist_remove(mls, db); in dbuf_evict_one()
793 uint64_t size = db->db.db_size; in dbuf_evict_one()
794 uint64_t usize = dmu_buf_user_size(&db->db); in dbuf_evict_one()
796 &dbuf_caches[DB_DBUF_CACHE].size, size, db); in dbuf_evict_one()
798 &dbuf_caches[DB_DBUF_CACHE].size, usize, db->db_user); in dbuf_evict_one()
799 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_evict_one()
801 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size + usize); in dbuf_evict_one()
802 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE); in dbuf_evict_one()
803 db->db_caching_status = DB_NO_CACHE; in dbuf_evict_one()
804 dbuf_destroy(db); in dbuf_evict_one()
1094 dbuf_verify(dmu_buf_impl_t *db) in dbuf_verify() argument
1100 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_verify()
1105 ASSERT(db->db_objset != NULL); in dbuf_verify()
1106 DB_DNODE_ENTER(db); in dbuf_verify()
1107 dn = DB_DNODE(db); in dbuf_verify()
1109 ASSERT(db->db_parent == NULL); in dbuf_verify()
1110 ASSERT(db->db_blkptr == NULL); in dbuf_verify()
1112 ASSERT3U(db->db.db_object, ==, dn->dn_object); in dbuf_verify()
1113 ASSERT3P(db->db_objset, ==, dn->dn_objset); in dbuf_verify()
1114 ASSERT3U(db->db_level, <, dn->dn_nlevels); in dbuf_verify()
1115 ASSERT(db->db_blkid == DMU_BONUS_BLKID || in dbuf_verify()
1116 db->db_blkid == DMU_SPILL_BLKID || in dbuf_verify()
1119 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_verify()
1121 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_verify()
1122 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); in dbuf_verify()
1123 } else if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_verify()
1125 ASSERT0(db->db.db_offset); in dbuf_verify()
1127 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); in dbuf_verify()
1130 if ((dr = list_head(&db->db_dirty_records)) != NULL) { in dbuf_verify()
1131 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
1133 for (dr = list_next(&db->db_dirty_records, dr); dr != NULL; in dbuf_verify()
1134 dr = list_next(&db->db_dirty_records, dr)) { in dbuf_verify()
1135 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
1146 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_verify()
1147 dr = db->db_data_pending; in dbuf_verify()
1152 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); in dbuf_verify()
1155 /* verify db->db_blkptr */ in dbuf_verify()
1156 if (db->db_blkptr) { in dbuf_verify()
1157 if (db->db_parent == dn->dn_dbuf) { in dbuf_verify()
1158 /* db is pointed to by the dnode */ in dbuf_verify()
1159 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ in dbuf_verify()
1160 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) in dbuf_verify()
1161 ASSERT(db->db_parent == NULL); in dbuf_verify()
1163 ASSERT(db->db_parent != NULL); in dbuf_verify()
1164 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_verify()
1165 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
1166 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_verify()
1168 /* db is pointed to by an indirect block */ in dbuf_verify()
1169 int epb __maybe_unused = db->db_parent->db.db_size >> in dbuf_verify()
1171 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); in dbuf_verify()
1172 ASSERT3U(db->db_parent->db.db_object, ==, in dbuf_verify()
1173 db->db.db_object); in dbuf_verify()
1179 if (RW_LOCK_HELD(&db->db_parent->db_rwlock)) { in dbuf_verify()
1180 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
1181 ((blkptr_t *)db->db_parent->db.db_data + in dbuf_verify()
1182 db->db_blkid % epb)); in dbuf_verify()
1186 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && in dbuf_verify()
1187 (db->db_buf == NULL || db->db_buf->b_data) && in dbuf_verify()
1188 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && in dbuf_verify()
1189 db->db_state != DB_FILL && (dn == NULL || !dn->dn_free_txg)) { in dbuf_verify()
1201 if (db->db_dirtycnt == 0) { in dbuf_verify()
1202 if (db->db_level == 0) { in dbuf_verify()
1203 uint64_t *buf = db->db.db_data; in dbuf_verify()
1206 for (i = 0; i < db->db.db_size >> 3; i++) { in dbuf_verify()
1210 blkptr_t *bps = db->db.db_data; in dbuf_verify()
1211 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, in dbuf_verify()
1212 db->db.db_size); in dbuf_verify()
1221 i < db->db.db_size / sizeof (blkptr_t); in dbuf_verify()
1240 DB_DNODE_EXIT(db); in dbuf_verify()
1245 dbuf_clear_data(dmu_buf_impl_t *db) in dbuf_clear_data() argument
1247 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_clear_data()
1248 dbuf_evict_user(db); in dbuf_clear_data()
1249 ASSERT3P(db->db_buf, ==, NULL); in dbuf_clear_data()
1250 db->db.db_data = NULL; in dbuf_clear_data()
1251 if (db->db_state != DB_NOFILL) { in dbuf_clear_data()
1252 db->db_state = DB_UNCACHED; in dbuf_clear_data()
1253 DTRACE_SET_STATE(db, "clear data"); in dbuf_clear_data()
1258 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) in dbuf_set_data() argument
1260 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_set_data()
1263 db->db_buf = buf; in dbuf_set_data()
1265 db->db.db_data = buf->b_data; in dbuf_set_data()
1269 dbuf_alloc_arcbuf(dmu_buf_impl_t *db) in dbuf_alloc_arcbuf() argument
1271 spa_t *spa = db->db_objset->os_spa; in dbuf_alloc_arcbuf()
1273 return (arc_alloc_buf(spa, db, DBUF_GET_BUFC_TYPE(db), db->db.db_size)); in dbuf_alloc_arcbuf()
1280 dbuf_loan_arcbuf(dmu_buf_impl_t *db) in dbuf_loan_arcbuf() argument
1284 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_loan_arcbuf()
1285 mutex_enter(&db->db_mtx); in dbuf_loan_arcbuf()
1286 if (arc_released(db->db_buf) || zfs_refcount_count(&db->db_holds) > 1) { in dbuf_loan_arcbuf()
1287 int blksz = db->db.db_size; in dbuf_loan_arcbuf()
1288 spa_t *spa = db->db_objset->os_spa; in dbuf_loan_arcbuf()
1290 mutex_exit(&db->db_mtx); in dbuf_loan_arcbuf()
1292 memcpy(abuf->b_data, db->db.db_data, blksz); in dbuf_loan_arcbuf()
1294 abuf = db->db_buf; in dbuf_loan_arcbuf()
1295 arc_loan_inuse_buf(abuf, db); in dbuf_loan_arcbuf()
1296 db->db_buf = NULL; in dbuf_loan_arcbuf()
1297 dbuf_clear_data(db); in dbuf_loan_arcbuf()
1298 mutex_exit(&db->db_mtx); in dbuf_loan_arcbuf()
1354 dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw, const void *tag) in dmu_buf_lock_parent() argument
1357 if (db->db_parent != NULL) { in dmu_buf_lock_parent()
1358 rw_enter(&db->db_parent->db_rwlock, rw); in dmu_buf_lock_parent()
1360 } else if (dmu_objset_ds(db->db_objset) != NULL) { in dmu_buf_lock_parent()
1361 rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw, in dmu_buf_lock_parent()
1379 dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type, const void *tag) in dmu_buf_unlock_parent() argument
1382 rw_exit(&db->db_parent->db_rwlock); in dmu_buf_unlock_parent()
1384 rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag); in dmu_buf_unlock_parent()
1392 dmu_buf_impl_t *db = vdb; in dbuf_read_done() local
1394 mutex_enter(&db->db_mtx); in dbuf_read_done()
1395 ASSERT3U(db->db_state, ==, DB_READ); in dbuf_read_done()
1400 ASSERT(zfs_refcount_count(&db->db_holds) > 0); in dbuf_read_done()
1401 ASSERT(db->db_buf == NULL); in dbuf_read_done()
1402 ASSERT(db->db.db_data == NULL); in dbuf_read_done()
1406 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_read_done()
1407 ASSERT3P(db->db_buf, ==, NULL); in dbuf_read_done()
1408 db->db_state = DB_UNCACHED; in dbuf_read_done()
1409 DTRACE_SET_STATE(db, "i/o error"); in dbuf_read_done()
1410 } else if (db->db_level == 0 && db->db_freed_in_flight) { in dbuf_read_done()
1413 arc_release(buf, db); in dbuf_read_done()
1414 memset(buf->b_data, 0, db->db.db_size); in dbuf_read_done()
1416 db->db_freed_in_flight = FALSE; in dbuf_read_done()
1417 dbuf_set_data(db, buf); in dbuf_read_done()
1418 db->db_state = DB_CACHED; in dbuf_read_done()
1419 DTRACE_SET_STATE(db, "freed in flight"); in dbuf_read_done()
1423 dbuf_set_data(db, buf); in dbuf_read_done()
1424 db->db_state = DB_CACHED; in dbuf_read_done()
1425 DTRACE_SET_STATE(db, "successful read"); in dbuf_read_done()
1427 cv_broadcast(&db->db_changed); in dbuf_read_done()
1428 dbuf_rele_and_unlock(db, NULL, B_FALSE); in dbuf_read_done()
1437 dbuf_read_bonus(dmu_buf_impl_t *db, dnode_t *dn) in dbuf_read_bonus() argument
1443 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_bonus()
1444 ASSERT(DB_DNODE_HELD(db)); in dbuf_read_bonus()
1445 ASSERT3U(bonuslen, <=, db->db.db_size); in dbuf_read_bonus()
1446 db->db.db_data = kmem_alloc(max_bonuslen, KM_SLEEP); in dbuf_read_bonus()
1449 memset(db->db.db_data, 0, max_bonuslen); in dbuf_read_bonus()
1451 memcpy(db->db.db_data, DN_BONUS(dn->dn_phys), bonuslen); in dbuf_read_bonus()
1452 db->db_state = DB_CACHED; in dbuf_read_bonus()
1453 DTRACE_SET_STATE(db, "bonus buffer filled"); in dbuf_read_bonus()
1458 dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *dbbp) in dbuf_handle_indirect_hole() argument
1460 blkptr_t *bps = db->db.db_data; in dbuf_handle_indirect_hole()
1482 dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *bp) in dbuf_read_hole() argument
1484 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_hole()
1493 if (!is_hole && db->db_level == 0) in dbuf_read_hole()
1494 is_hole = dnode_block_freed(dn, db->db_blkid) || BP_IS_HOLE(bp); in dbuf_read_hole()
1497 dbuf_set_data(db, dbuf_alloc_arcbuf(db)); in dbuf_read_hole()
1498 memset(db->db.db_data, 0, db->db.db_size); in dbuf_read_hole()
1500 if (bp != NULL && db->db_level > 0 && BP_IS_HOLE(bp) && in dbuf_read_hole()
1502 dbuf_handle_indirect_hole(db, dn, bp); in dbuf_read_hole()
1504 db->db_state = DB_CACHED; in dbuf_read_hole()
1505 DTRACE_SET_STATE(db, "hole read satisfied"); in dbuf_read_hole()
1524 dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, dnode_t *dn, uint32_t flags) in dbuf_read_verify_dnode_crypt() argument
1526 objset_t *os = db->db_objset; in dbuf_read_verify_dnode_crypt()
1570 if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1572 (db->db_blkid == DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1586 dbuf_read_impl(dmu_buf_impl_t *db, dnode_t *dn, zio_t *zio, uint32_t flags, in dbuf_read_impl() argument
1593 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read_impl()
1594 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_impl()
1595 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_read_impl()
1596 ASSERT(db->db_buf == NULL); in dbuf_read_impl()
1597 ASSERT(db->db_parent == NULL || in dbuf_read_impl()
1598 RW_LOCK_HELD(&db->db_parent->db_rwlock)); in dbuf_read_impl()
1600 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_read_impl()
1601 err = dbuf_read_bonus(db, dn); in dbuf_read_impl()
1605 err = dbuf_read_hole(db, dn, bp); in dbuf_read_impl()
1618 db->db_objset->os_dsl_dataset, in dbuf_read_impl()
1624 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read_impl()
1625 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read_impl()
1631 if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bp)) { in dbuf_read_impl()
1632 spa_log_error(db->db_objset->os_spa, &zb, in dbuf_read_impl()
1638 db->db_state = DB_READ; in dbuf_read_impl()
1639 DTRACE_SET_STATE(db, "read issued"); in dbuf_read_impl()
1640 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1642 if (!DBUF_IS_CACHEABLE(db)) in dbuf_read_impl()
1644 else if (dbuf_is_l2cacheable(db, bp)) in dbuf_read_impl()
1647 dbuf_add_ref(db, NULL); in dbuf_read_impl()
1663 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1664 return (arc_read(zio, db->db_objset->os_spa, ©, in dbuf_read_impl()
1665 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags, in dbuf_read_impl()
1669 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1670 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1688 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) in dbuf_fix_old_data() argument
1690 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dbuf_fix_old_data()
1692 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_fix_old_data()
1693 ASSERT(db->db.db_data != NULL); in dbuf_fix_old_data()
1694 ASSERT(db->db_level == 0); in dbuf_fix_old_data()
1695 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dbuf_fix_old_data()
1699 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) in dbuf_fix_old_data()
1710 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_fix_old_data()
1711 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data()
1715 memcpy(dr->dt.dl.dr_data, db->db.db_data, bonuslen); in dbuf_fix_old_data()
1716 } else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) { in dbuf_fix_old_data()
1717 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data()
1718 int size = arc_buf_size(db->db_buf); in dbuf_fix_old_data()
1719 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_fix_old_data()
1720 spa_t *spa = db->db_objset->os_spa; in dbuf_fix_old_data()
1722 arc_get_compression(db->db_buf); in dbuf_fix_old_data()
1723 uint8_t complevel = arc_get_complevel(db->db_buf); in dbuf_fix_old_data()
1725 if (arc_is_encrypted(db->db_buf)) { in dbuf_fix_old_data()
1731 arc_get_raw_params(db->db_buf, &byteorder, salt, in dbuf_fix_old_data()
1733 dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db, in dbuf_fix_old_data()
1735 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf), in dbuf_fix_old_data()
1739 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db, in dbuf_fix_old_data()
1740 size, arc_buf_lsize(db->db_buf), compress_type, in dbuf_fix_old_data()
1743 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size); in dbuf_fix_old_data()
1745 memcpy(dr->dt.dl.dr_data->b_data, db->db.db_data, size); in dbuf_fix_old_data()
1747 db->db_buf = NULL; in dbuf_fix_old_data()
1748 dbuf_clear_data(db); in dbuf_fix_old_data()
1753 dbuf_read(dmu_buf_impl_t *db, zio_t *pio, uint32_t flags) in dbuf_read() argument
1759 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read()
1761 DB_DNODE_ENTER(db); in dbuf_read()
1762 dn = DB_DNODE(db); in dbuf_read()
1768 err = dbuf_read_verify_dnode_crypt(db, dn, flags); in dbuf_read()
1772 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_read()
1775 mutex_enter(&db->db_mtx); in dbuf_read()
1777 db->db_partial_read = B_TRUE; in dbuf_read()
1779 db->db_partial_read = B_FALSE; in dbuf_read()
1780 miss = (db->db_state != DB_CACHED); in dbuf_read()
1782 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_read()
1792 mutex_exit(&db->db_mtx); in dbuf_read()
1793 DB_DNODE_EXIT(db); in dbuf_read()
1797 ASSERT(db->db_state == DB_READ || in dbuf_read()
1799 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, db, in dbuf_read()
1801 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_read()
1802 } while (db->db_state == DB_READ || db->db_state == DB_FILL); in dbuf_read()
1803 if (db->db_state == DB_UNCACHED) { in dbuf_read()
1805 mutex_exit(&db->db_mtx); in dbuf_read()
1806 DB_DNODE_EXIT(db); in dbuf_read()
1811 if (db->db_state == DB_CACHED) { in dbuf_read()
1819 if ((flags & DB_RF_NO_DECRYPT) == 0 && db->db_buf != NULL && in dbuf_read()
1820 (arc_is_encrypted(db->db_buf) || in dbuf_read()
1821 arc_is_unauthenticated(db->db_buf) || in dbuf_read()
1822 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dbuf_read()
1826 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read()
1827 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read()
1828 dbuf_fix_old_data(db, spa_syncing_txg(spa)); in dbuf_read()
1829 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dbuf_read()
1830 dbuf_set_data(db, db->db_buf); in dbuf_read()
1832 mutex_exit(&db->db_mtx); in dbuf_read()
1834 ASSERT(db->db_state == DB_UNCACHED || in dbuf_read()
1835 db->db_state == DB_NOFILL); in dbuf_read()
1836 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); in dbuf_read()
1844 err = dmu_buf_get_bp_from_dbuf(db, &bp); in dbuf_read()
1847 if (pio == NULL && (db->db_state == DB_NOFILL || in dbuf_read()
1856 dbuf_read_impl(db, dn, pio, flags, dblt, bp, FTAG); in dbuf_read()
1858 mutex_exit(&db->db_mtx); in dbuf_read()
1859 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_read()
1862 miss = (db->db_state != DB_CACHED); in dbuf_read()
1866 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, miss, in dbuf_read()
1869 DB_DNODE_EXIT(db); in dbuf_read()
1899 dbuf_noread(dmu_buf_impl_t *db) in dbuf_noread() argument
1901 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_noread()
1902 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_noread()
1903 mutex_enter(&db->db_mtx); in dbuf_noread()
1904 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_noread()
1905 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_noread()
1906 if (db->db_state == DB_UNCACHED) { in dbuf_noread()
1907 ASSERT(db->db_buf == NULL); in dbuf_noread()
1908 ASSERT(db->db.db_data == NULL); in dbuf_noread()
1909 dbuf_set_data(db, dbuf_alloc_arcbuf(db)); in dbuf_noread()
1910 db->db_state = DB_FILL; in dbuf_noread()
1911 DTRACE_SET_STATE(db, "assigning filled buffer"); in dbuf_noread()
1912 } else if (db->db_state == DB_NOFILL) { in dbuf_noread()
1913 dbuf_clear_data(db); in dbuf_noread()
1915 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_noread()
1917 mutex_exit(&db->db_mtx); in dbuf_noread()
1923 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_unoverride() local
1927 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_unoverride()
1935 ASSERT(db->db_level == 0); in dbuf_unoverride()
1937 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_unoverride()
1941 ASSERT(db->db_data_pending != dr); in dbuf_unoverride()
1945 zio_free(db->db_objset->os_spa, txg, bp); in dbuf_unoverride()
1949 dr->dt.dl.dr_data = db->db_buf; in dbuf_unoverride()
1969 arc_release(dr->dt.dl.dr_data, db); in dbuf_unoverride()
1982 dmu_buf_impl_t *db, *db_next; in dbuf_free_range() local
1999 db = avl_find(&dn->dn_dbufs, db_search, &where); in dbuf_free_range()
2000 ASSERT3P(db, ==, NULL); in dbuf_free_range()
2002 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dbuf_free_range()
2004 for (; db != NULL; db = db_next) { in dbuf_free_range()
2005 db_next = AVL_NEXT(&dn->dn_dbufs, db); in dbuf_free_range()
2006 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_free_range()
2008 if (db->db_level != 0 || db->db_blkid > end_blkid) { in dbuf_free_range()
2011 ASSERT3U(db->db_blkid, >=, start_blkid); in dbuf_free_range()
2014 mutex_enter(&db->db_mtx); in dbuf_free_range()
2015 if (dbuf_undirty(db, tx)) { in dbuf_free_range()
2020 if (db->db_state == DB_UNCACHED || in dbuf_free_range()
2021 db->db_state == DB_NOFILL || in dbuf_free_range()
2022 db->db_state == DB_EVICTING) { in dbuf_free_range()
2023 ASSERT(db->db.db_data == NULL); in dbuf_free_range()
2024 mutex_exit(&db->db_mtx); in dbuf_free_range()
2027 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_free_range()
2029 db->db_freed_in_flight = TRUE; in dbuf_free_range()
2030 mutex_exit(&db->db_mtx); in dbuf_free_range()
2033 if (zfs_refcount_count(&db->db_holds) == 0) { in dbuf_free_range()
2034 ASSERT(db->db_buf); in dbuf_free_range()
2035 dbuf_destroy(db); in dbuf_free_range()
2040 dr = list_head(&db->db_dirty_records); in dbuf_free_range()
2048 if (db->db_blkid != DMU_SPILL_BLKID && in dbuf_free_range()
2049 db->db_blkid > dn->dn_maxblkid) in dbuf_free_range()
2050 dn->dn_maxblkid = db->db_blkid; in dbuf_free_range()
2059 dbuf_fix_old_data(db, txg); in dbuf_free_range()
2063 if (db->db_state == DB_CACHED) { in dbuf_free_range()
2064 ASSERT(db->db.db_data != NULL); in dbuf_free_range()
2065 arc_release(db->db_buf, db); in dbuf_free_range()
2066 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_free_range()
2067 memset(db->db.db_data, 0, db->db.db_size); in dbuf_free_range()
2068 rw_exit(&db->db_rwlock); in dbuf_free_range()
2069 arc_buf_freeze(db->db_buf); in dbuf_free_range()
2072 mutex_exit(&db->db_mtx); in dbuf_free_range()
2080 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) in dbuf_new_size() argument
2084 int osize = db->db.db_size; in dbuf_new_size()
2085 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_new_size()
2088 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_new_size()
2090 DB_DNODE_ENTER(db); in dbuf_new_size()
2091 dn = DB_DNODE(db); in dbuf_new_size()
2097 dmu_buf_will_dirty(&db->db, tx); in dbuf_new_size()
2099 VERIFY3P(db->db_buf, !=, NULL); in dbuf_new_size()
2102 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); in dbuf_new_size()
2105 old_buf = db->db_buf; in dbuf_new_size()
2111 mutex_enter(&db->db_mtx); in dbuf_new_size()
2112 dbuf_set_data(db, buf); in dbuf_new_size()
2113 arc_buf_destroy(old_buf, db); in dbuf_new_size()
2114 db->db.db_size = size; in dbuf_new_size()
2116 dr = list_head(&db->db_dirty_records); in dbuf_new_size()
2119 if (db->db_level == 0) in dbuf_new_size()
2124 mutex_exit(&db->db_mtx); in dbuf_new_size()
2127 DB_DNODE_EXIT(db); in dbuf_new_size()
2131 dbuf_release_bp(dmu_buf_impl_t *db) in dbuf_release_bp() argument
2133 objset_t *os __maybe_unused = db->db_objset; in dbuf_release_bp()
2138 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); in dbuf_release_bp()
2140 (void) arc_release(db->db_buf, db); in dbuf_release_bp()
2150 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_redirty() local
2152 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_redirty()
2154 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { in dbuf_redirty()
2160 if (db->db.db_object != DMU_META_DNODE_OBJECT && in dbuf_redirty()
2161 db->db_state != DB_NOFILL) { in dbuf_redirty()
2163 ASSERT(arc_released(db->db_buf)); in dbuf_redirty()
2164 arc_buf_thaw(db->db_buf); in dbuf_redirty()
2239 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_dirty() argument
2248 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_dirty()
2249 DMU_TX_DIRTY_BUF(tx, db); in dbuf_dirty()
2251 DB_DNODE_ENTER(db); in dbuf_dirty()
2252 dn = DB_DNODE(db); in dbuf_dirty()
2279 mutex_enter(&db->db_mtx); in dbuf_dirty()
2285 ASSERT(db->db_level != 0 || in dbuf_dirty()
2286 db->db_state == DB_CACHED || db->db_state == DB_FILL || in dbuf_dirty()
2287 db->db_state == DB_NOFILL); in dbuf_dirty()
2290 dnode_set_dirtyctx(dn, tx, db); in dbuf_dirty()
2295 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_dirty()
2301 dr_head = list_head(&db->db_dirty_records); in dbuf_dirty()
2303 db->db.db_object == DMU_META_DNODE_OBJECT); in dbuf_dirty()
2304 dr_next = dbuf_find_dirty_lte(db, tx->tx_txg); in dbuf_dirty()
2306 DB_DNODE_EXIT(db); in dbuf_dirty()
2309 mutex_exit(&db->db_mtx); in dbuf_dirty()
2320 ASSERT3U(dn->dn_nlevels, >, db->db_level); in dbuf_dirty()
2339 ASSERT(db->db.db_size != 0); in dbuf_dirty()
2341 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_dirty()
2343 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) { in dbuf_dirty()
2344 dmu_objset_willuse_space(os, db->db.db_size, tx); in dbuf_dirty()
2357 if (db->db_level == 0) { in dbuf_dirty()
2358 void *data_old = db->db_buf; in dbuf_dirty()
2360 if (db->db_state != DB_NOFILL) { in dbuf_dirty()
2361 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_dirty()
2362 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
2363 data_old = db->db.db_data; in dbuf_dirty()
2364 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { in dbuf_dirty()
2374 arc_release(db->db_buf, db); in dbuf_dirty()
2375 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
2376 data_old = db->db_buf; in dbuf_dirty()
2387 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) { in dbuf_dirty()
2388 dr->dr_accounted = db->db.db_size; in dbuf_dirty()
2390 dr->dr_dbuf = db; in dbuf_dirty()
2392 list_insert_before(&db->db_dirty_records, dr_next, dr); in dbuf_dirty()
2399 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_dirty()
2400 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_dirty()
2404 db->db_blkid, 1); in dbuf_dirty()
2407 db->db_freed_in_flight = FALSE; in dbuf_dirty()
2413 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); in dbuf_dirty()
2414 db->db_dirtycnt += 1; in dbuf_dirty()
2415 ASSERT3U(db->db_dirtycnt, <=, 3); in dbuf_dirty()
2417 mutex_exit(&db->db_mtx); in dbuf_dirty()
2419 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_dirty()
2420 db->db_blkid == DMU_SPILL_BLKID) { in dbuf_dirty()
2426 DB_DNODE_EXIT(db); in dbuf_dirty()
2441 if (db->db_blkptr != NULL) { in dbuf_dirty()
2442 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); in dbuf_dirty()
2443 ddt_prefetch(os->os_spa, db->db_blkptr); in dbuf_dirty()
2444 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_dirty()
2451 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || in dbuf_dirty()
2452 dn->dn_phys->dn_nlevels > db->db_level || in dbuf_dirty()
2453 dn->dn_next_nlevels[txgoff] > db->db_level || in dbuf_dirty()
2454 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || in dbuf_dirty()
2455 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); in dbuf_dirty()
2458 if (db->db_level == 0) { in dbuf_dirty()
2459 ASSERT(!db->db_objset->os_raw_receive || in dbuf_dirty()
2460 dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2461 dnode_new_blkid(dn, db->db_blkid, tx, in dbuf_dirty()
2463 ASSERT(dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2466 if (db->db_level+1 < dn->dn_nlevels) { in dbuf_dirty()
2467 dmu_buf_impl_t *parent = db->db_parent; in dbuf_dirty()
2471 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { in dbuf_dirty()
2473 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_dirty()
2474 db->db_blkid >> epbs, FTAG); in dbuf_dirty()
2480 ASSERT3U(db->db_level + 1, ==, parent->db_level); in dbuf_dirty()
2485 mutex_enter(&db->db_mtx); in dbuf_dirty()
2490 if (list_head(&db->db_dirty_records) == dr || in dbuf_dirty()
2499 mutex_exit(&db->db_mtx); in dbuf_dirty()
2501 ASSERT(db->db_level + 1 == dn->dn_nlevels); in dbuf_dirty()
2502 ASSERT(db->db_blkid < dn->dn_nblkptr); in dbuf_dirty()
2503 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); in dbuf_dirty()
2513 DB_DNODE_EXIT(db); in dbuf_dirty()
2520 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_undirty_bonus() local
2522 if (dr->dt.dl.dr_data != db->db.db_data) { in dbuf_undirty_bonus()
2529 db->db_data_pending = NULL; in dbuf_undirty_bonus()
2530 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); in dbuf_undirty_bonus()
2531 list_remove(&db->db_dirty_records, dr); in dbuf_undirty_bonus()
2537 ASSERT3U(db->db_dirtycnt, >, 0); in dbuf_undirty_bonus()
2538 db->db_dirtycnt -= 1; in dbuf_undirty_bonus()
2546 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_undirty() argument
2560 ASSERT(db->db_objset == in dbuf_undirty()
2561 dmu_objset_pool(db->db_objset)->dp_meta_objset || in dbuf_undirty()
2562 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); in dbuf_undirty()
2563 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_undirty()
2564 ASSERT0(db->db_level); in dbuf_undirty()
2565 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_undirty()
2570 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, txg); in dbuf_undirty()
2573 ASSERT(dr->dr_dbuf == db); in dbuf_undirty()
2585 brt_pending_remove(dmu_objset_spa(db->db_objset), in dbuf_undirty()
2592 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_undirty()
2594 ASSERT(db->db.db_size != 0); in dbuf_undirty()
2599 list_remove(&db->db_dirty_records, dr); in dbuf_undirty()
2611 } else if (db->db_blkid == DMU_SPILL_BLKID || in dbuf_undirty()
2612 db->db_level + 1 == dn->dn_nlevels) { in dbuf_undirty()
2613 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); in dbuf_undirty()
2619 if (db->db_state != DB_NOFILL && !brtwrite) { in dbuf_undirty()
2622 if (dr->dt.dl.dr_data != db->db_buf) { in dbuf_undirty()
2623 ASSERT(db->db_buf != NULL); in dbuf_undirty()
2625 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_undirty()
2631 ASSERT(db->db_dirtycnt > 0); in dbuf_undirty()
2632 db->db_dirtycnt -= 1; in dbuf_undirty()
2634 if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { in dbuf_undirty()
2635 ASSERT(db->db_state == DB_NOFILL || brtwrite || diowrite || in dbuf_undirty()
2636 arc_released(db->db_buf)); in dbuf_undirty()
2637 dbuf_destroy(db); in dbuf_undirty()
2647 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_dirty_impl() local
2651 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_dirty_impl()
2657 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_impl()
2658 if (db->db_state == DB_CACHED || db->db_state == DB_NOFILL) { in dmu_buf_will_dirty_impl()
2664 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_will_dirty_impl()
2666 if (db->db_level == 0 && in dmu_buf_will_dirty_impl()
2679 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2684 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2686 DB_DNODE_ENTER(db); in dmu_buf_will_dirty_impl()
2687 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) in dmu_buf_will_dirty_impl()
2689 DB_DNODE_EXIT(db); in dmu_buf_will_dirty_impl()
2697 (void) dbuf_read(db, NULL, flags); in dmu_buf_will_dirty_impl()
2699 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_impl()
2700 VERIFY(!dbuf_undirty(db, tx)); in dmu_buf_will_dirty_impl()
2701 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2703 (void) dbuf_dirty(db, tx); in dmu_buf_will_dirty_impl()
2716 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_is_dirty() local
2719 mutex_enter(&db->db_mtx); in dmu_buf_is_dirty()
2720 dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_is_dirty()
2721 mutex_exit(&db->db_mtx); in dmu_buf_is_dirty()
2732 dmu_buf_get_bp_from_dbuf(dmu_buf_impl_t *db, blkptr_t **bp) in dmu_buf_get_bp_from_dbuf() argument
2734 ASSERT(MUTEX_HELD(&db->db_mtx)); in dmu_buf_get_bp_from_dbuf()
2737 if (db->db_level != 0) { in dmu_buf_get_bp_from_dbuf()
2738 *bp = db->db_blkptr; in dmu_buf_get_bp_from_dbuf()
2742 *bp = db->db_blkptr; in dmu_buf_get_bp_from_dbuf()
2743 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dmu_buf_get_bp_from_dbuf()
2744 if (dr && db->db_state == DB_NOFILL) { in dmu_buf_get_bp_from_dbuf()
2750 } else if (dr && db->db_state == DB_UNCACHED) { in dmu_buf_get_bp_from_dbuf()
2764 dmu_buf_untransform_direct(dmu_buf_impl_t *db, spa_t *spa) in dmu_buf_untransform_direct() argument
2767 DB_DNODE_ENTER(db); in dmu_buf_untransform_direct()
2768 dnode_t *dn = DB_DNODE(db); in dmu_buf_untransform_direct()
2770 ASSERT3S(db->db_state, ==, DB_CACHED); in dmu_buf_untransform_direct()
2771 ASSERT(MUTEX_HELD(&db->db_mtx)); in dmu_buf_untransform_direct()
2777 err = dbuf_read_verify_dnode_crypt(db, dn, 0); in dmu_buf_untransform_direct()
2786 if (err == 0 && db->db_buf != NULL && in dmu_buf_untransform_direct()
2787 (arc_is_encrypted(db->db_buf) || in dmu_buf_untransform_direct()
2788 arc_is_unauthenticated(db->db_buf) || in dmu_buf_untransform_direct()
2789 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dmu_buf_untransform_direct()
2792 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dmu_buf_untransform_direct()
2793 db->db.db_object, db->db_level, db->db_blkid); in dmu_buf_untransform_direct()
2794 dbuf_fix_old_data(db, spa_syncing_txg(spa)); in dmu_buf_untransform_direct()
2795 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dmu_buf_untransform_direct()
2796 dbuf_set_data(db, db->db_buf); in dmu_buf_untransform_direct()
2798 DB_DNODE_EXIT(db); in dmu_buf_untransform_direct()
2810 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_clone_or_dio() local
2811 ASSERT0(db->db_level); in dmu_buf_will_clone_or_dio()
2813 ASSERT0(db->db_level); in dmu_buf_will_clone_or_dio()
2814 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_clone_or_dio()
2815 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dmu_buf_will_clone_or_dio()
2817 mutex_enter(&db->db_mtx); in dmu_buf_will_clone_or_dio()
2818 DBUF_VERIFY(db); in dmu_buf_will_clone_or_dio()
2839 * hold on the db, so it should never be evicted after calling in dmu_buf_will_clone_or_dio()
2842 VERIFY3B(dbuf_undirty(db, tx), ==, B_FALSE); in dmu_buf_will_clone_or_dio()
2843 ASSERT0P(dbuf_find_dirty_eq(db, tx->tx_txg)); in dmu_buf_will_clone_or_dio()
2845 if (db->db_buf != NULL) { in dmu_buf_will_clone_or_dio()
2851 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dmu_buf_will_clone_or_dio()
2852 if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) in dmu_buf_will_clone_or_dio()
2853 arc_buf_destroy(db->db_buf, db); in dmu_buf_will_clone_or_dio()
2860 db->db_buf = NULL; in dmu_buf_will_clone_or_dio()
2861 dbuf_clear_data(db); in dmu_buf_will_clone_or_dio()
2864 ASSERT3P(db->db_buf, ==, NULL); in dmu_buf_will_clone_or_dio()
2865 ASSERT3P(db->db.db_data, ==, NULL); in dmu_buf_will_clone_or_dio()
2867 db->db_state = DB_NOFILL; in dmu_buf_will_clone_or_dio()
2868 DTRACE_SET_STATE(db, in dmu_buf_will_clone_or_dio()
2871 DBUF_VERIFY(db); in dmu_buf_will_clone_or_dio()
2872 mutex_exit(&db->db_mtx); in dmu_buf_will_clone_or_dio()
2874 dbuf_noread(db); in dmu_buf_will_clone_or_dio()
2875 (void) dbuf_dirty(db, tx); in dmu_buf_will_clone_or_dio()
2881 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_not_fill() local
2883 mutex_enter(&db->db_mtx); in dmu_buf_will_not_fill()
2884 db->db_state = DB_NOFILL; in dmu_buf_will_not_fill()
2885 DTRACE_SET_STATE(db, "allocating NOFILL buffer"); in dmu_buf_will_not_fill()
2886 mutex_exit(&db->db_mtx); in dmu_buf_will_not_fill()
2888 dbuf_noread(db); in dmu_buf_will_not_fill()
2889 (void) dbuf_dirty(db, tx); in dmu_buf_will_not_fill()
2895 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_fill() local
2897 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_fill()
2899 ASSERT(db->db_level == 0); in dmu_buf_will_fill()
2900 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_fill()
2902 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || in dmu_buf_will_fill()
2905 mutex_enter(&db->db_mtx); in dmu_buf_will_fill()
2906 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_will_fill()
2907 if (db->db_state == DB_NOFILL || in dmu_buf_will_fill()
2908 (db->db_state == DB_UNCACHED && dr && dr->dt.dl.dr_diowrite)) { in dmu_buf_will_fill()
2914 mutex_exit(&db->db_mtx); in dmu_buf_will_fill()
2925 VERIFY(!dbuf_undirty(db, tx)); in dmu_buf_will_fill()
2926 db->db_state = DB_UNCACHED; in dmu_buf_will_fill()
2929 mutex_exit(&db->db_mtx); in dmu_buf_will_fill()
2931 dbuf_noread(db); in dmu_buf_will_fill()
2932 (void) dbuf_dirty(db, tx); in dmu_buf_will_fill()
2937 * indicates the caller expects raw encrypted data in the db, and provides
2946 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_set_crypt_params() local
2953 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dmu_buf_set_crypt_params()
2954 ASSERT0(db->db_level); in dmu_buf_set_crypt_params()
2955 ASSERT(db->db_objset->os_raw_receive); in dmu_buf_set_crypt_params()
2960 dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_set_crypt_params()
2973 dbuf_override_impl(dmu_buf_impl_t *db, const blkptr_t *bp, dmu_tx_t *tx) in dbuf_override_impl() argument
2978 ASSERT3U(db->db.db_object, !=, DMU_META_DNODE_OBJECT); in dbuf_override_impl()
2979 ASSERT0(db->db_level); in dbuf_override_impl()
2981 dr = list_head(&db->db_dirty_records); in dbuf_override_impl()
2995 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_fill_done() local
2996 mutex_enter(&db->db_mtx); in dmu_buf_fill_done()
2997 DBUF_VERIFY(db); in dmu_buf_fill_done()
2999 if (db->db_state == DB_FILL) { in dmu_buf_fill_done()
3000 if (db->db_level == 0 && db->db_freed_in_flight) { in dmu_buf_fill_done()
3001 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_fill_done()
3004 memset(db->db.db_data, 0, db->db.db_size); in dmu_buf_fill_done()
3005 db->db_freed_in_flight = FALSE; in dmu_buf_fill_done()
3006 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3007 DTRACE_SET_STATE(db, in dmu_buf_fill_done()
3011 VERIFY(!dbuf_undirty(db, tx)); in dmu_buf_fill_done()
3012 arc_buf_destroy(db->db_buf, db); in dmu_buf_fill_done()
3013 db->db_buf = NULL; in dmu_buf_fill_done()
3014 dbuf_clear_data(db); in dmu_buf_fill_done()
3015 DTRACE_SET_STATE(db, "fill failed"); in dmu_buf_fill_done()
3017 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3018 DTRACE_SET_STATE(db, "fill done"); in dmu_buf_fill_done()
3020 cv_broadcast(&db->db_changed); in dmu_buf_fill_done()
3022 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3025 mutex_exit(&db->db_mtx); in dmu_buf_fill_done()
3035 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_write_embedded() local
3041 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), in dmu_buf_write_embedded()
3045 DB_DNODE_ENTER(db); in dmu_buf_write_embedded()
3046 type = DB_DNODE(db)->dn_type; in dmu_buf_write_embedded()
3047 DB_DNODE_EXIT(db); in dmu_buf_write_embedded()
3049 ASSERT0(db->db_level); in dmu_buf_write_embedded()
3050 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_write_embedded()
3054 dr = list_head(&db->db_dirty_records); in dmu_buf_write_embedded()
3073 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_redact() local
3075 ASSERT(dsl_dataset_feature_is_active(db->db_objset->os_dsl_dataset, in dmu_buf_redact()
3078 DB_DNODE_ENTER(db); in dmu_buf_redact()
3079 type = DB_DNODE(db)->dn_type; in dmu_buf_redact()
3080 DB_DNODE_EXIT(db); in dmu_buf_redact()
3082 ASSERT0(db->db_level); in dmu_buf_redact()
3092 dbuf_override_impl(db, &bp, tx); in dmu_buf_redact()
3100 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) in dbuf_assign_arcbuf() argument
3102 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_assign_arcbuf()
3103 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_assign_arcbuf()
3104 ASSERT(db->db_level == 0); in dbuf_assign_arcbuf()
3105 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf)); in dbuf_assign_arcbuf()
3107 ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size); in dbuf_assign_arcbuf()
3110 arc_return_buf(buf, db); in dbuf_assign_arcbuf()
3113 mutex_enter(&db->db_mtx); in dbuf_assign_arcbuf()
3115 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_assign_arcbuf()
3116 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_assign_arcbuf()
3118 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED || in dbuf_assign_arcbuf()
3119 db->db_state == DB_NOFILL); in dbuf_assign_arcbuf()
3121 if (db->db_state == DB_CACHED && in dbuf_assign_arcbuf()
3122 zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { in dbuf_assign_arcbuf()
3130 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
3131 (void) dbuf_dirty(db, tx); in dbuf_assign_arcbuf()
3132 memcpy(db->db.db_data, buf->b_data, db->db.db_size); in dbuf_assign_arcbuf()
3133 arc_buf_destroy(buf, db); in dbuf_assign_arcbuf()
3137 if (db->db_state == DB_CACHED) { in dbuf_assign_arcbuf()
3138 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dbuf_assign_arcbuf()
3140 ASSERT(db->db_buf != NULL); in dbuf_assign_arcbuf()
3142 ASSERT(dr->dt.dl.dr_data == db->db_buf); in dbuf_assign_arcbuf()
3144 if (!arc_released(db->db_buf)) { in dbuf_assign_arcbuf()
3147 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
3150 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
3151 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { in dbuf_assign_arcbuf()
3152 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
3153 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
3155 db->db_buf = NULL; in dbuf_assign_arcbuf()
3156 } else if (db->db_state == DB_NOFILL) { in dbuf_assign_arcbuf()
3163 VERIFY(!dbuf_undirty(db, tx)); in dbuf_assign_arcbuf()
3164 db->db_state = DB_UNCACHED; in dbuf_assign_arcbuf()
3166 ASSERT(db->db_buf == NULL); in dbuf_assign_arcbuf()
3167 dbuf_set_data(db, buf); in dbuf_assign_arcbuf()
3168 db->db_state = DB_FILL; in dbuf_assign_arcbuf()
3169 DTRACE_SET_STATE(db, "filling assigned arcbuf"); in dbuf_assign_arcbuf()
3170 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
3171 (void) dbuf_dirty(db, tx); in dbuf_assign_arcbuf()
3172 dmu_buf_fill_done(&db->db, tx, B_FALSE); in dbuf_assign_arcbuf()
3176 dbuf_destroy(dmu_buf_impl_t *db) in dbuf_destroy() argument
3179 dmu_buf_impl_t *parent = db->db_parent; in dbuf_destroy()
3182 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_destroy()
3183 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
3185 if (db->db_buf != NULL) { in dbuf_destroy()
3186 arc_buf_destroy(db->db_buf, db); in dbuf_destroy()
3187 db->db_buf = NULL; in dbuf_destroy()
3190 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_destroy()
3191 int slots = DB_DNODE(db)->dn_num_slots; in dbuf_destroy()
3193 if (db->db.db_data != NULL) { in dbuf_destroy()
3194 kmem_free(db->db.db_data, bonuslen); in dbuf_destroy()
3196 db->db_state = DB_UNCACHED; in dbuf_destroy()
3197 DTRACE_SET_STATE(db, "buffer cleared"); in dbuf_destroy()
3201 dbuf_clear_data(db); in dbuf_destroy()
3203 if (multilist_link_active(&db->db_cache_link)) { in dbuf_destroy()
3204 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_destroy()
3205 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_destroy()
3207 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); in dbuf_destroy()
3209 ASSERT0(dmu_buf_user_size(&db->db)); in dbuf_destroy()
3211 &dbuf_caches[db->db_caching_status].size, in dbuf_destroy()
3212 db->db.db_size, db); in dbuf_destroy()
3214 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { in dbuf_destroy()
3217 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_destroy()
3219 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], in dbuf_destroy()
3220 db->db.db_size); in dbuf_destroy()
3222 db->db_caching_status = DB_NO_CACHE; in dbuf_destroy()
3225 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_destroy()
3226 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
3227 ASSERT(list_is_empty(&db->db_dirty_records)); in dbuf_destroy()
3229 db->db_state = DB_EVICTING; in dbuf_destroy()
3230 DTRACE_SET_STATE(db, "buffer eviction started"); in dbuf_destroy()
3231 db->db_blkptr = NULL; in dbuf_destroy()
3238 mutex_exit(&db->db_mtx); in dbuf_destroy()
3240 DB_DNODE_ENTER(db); in dbuf_destroy()
3241 dn = DB_DNODE(db); in dbuf_destroy()
3243 if (db->db_blkid != DMU_BONUS_BLKID) { in dbuf_destroy()
3248 avl_remove(&dn->dn_dbufs, db); in dbuf_destroy()
3250 DB_DNODE_EXIT(db); in dbuf_destroy()
3262 dnode_rele_and_unlock(dn, db, B_TRUE); in dbuf_destroy()
3264 db->db_dnode_handle = NULL; in dbuf_destroy()
3266 db->db_dnode = NULL; in dbuf_destroy()
3269 dbuf_hash_remove(db); in dbuf_destroy()
3271 DB_DNODE_EXIT(db); in dbuf_destroy()
3274 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
3276 db->db_parent = NULL; in dbuf_destroy()
3278 ASSERT(db->db_buf == NULL); in dbuf_destroy()
3279 ASSERT(db->db.db_data == NULL); in dbuf_destroy()
3280 ASSERT(db->db_hash_next == NULL); in dbuf_destroy()
3281 ASSERT(db->db_blkptr == NULL); in dbuf_destroy()
3282 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
3283 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_destroy()
3284 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_destroy()
3292 dbuf_rele_and_unlock(parent, db, B_TRUE); in dbuf_destroy()
3295 kmem_cache_free(dbuf_kmem_cache, db); in dbuf_destroy()
3376 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + in dbuf_findbp()
3401 dmu_buf_impl_t *db, *odb; in dbuf_create() local
3406 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); in dbuf_create()
3408 list_create(&db->db_dirty_records, sizeof (dbuf_dirty_record_t), in dbuf_create()
3411 db->db_objset = os; in dbuf_create()
3412 db->db.db_object = dn->dn_object; in dbuf_create()
3413 db->db_level = level; in dbuf_create()
3414 db->db_blkid = blkid; in dbuf_create()
3415 db->db_dirtycnt = 0; in dbuf_create()
3417 db->db_dnode_handle = dn->dn_handle; in dbuf_create()
3419 db->db_dnode = dn; in dbuf_create()
3421 db->db_parent = parent; in dbuf_create()
3422 db->db_blkptr = blkptr; in dbuf_create()
3423 db->db_hash = hash; in dbuf_create()
3425 db->db_user = NULL; in dbuf_create()
3426 db->db_user_immediate_evict = FALSE; in dbuf_create()
3427 db->db_freed_in_flight = FALSE; in dbuf_create()
3428 db->db_pending_evict = FALSE; in dbuf_create()
3432 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dbuf_create()
3434 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_create()
3435 db->db.db_offset = DMU_BONUS_BLKID; in dbuf_create()
3436 db->db_state = DB_UNCACHED; in dbuf_create()
3437 DTRACE_SET_STATE(db, "bonus buffer created"); in dbuf_create()
3438 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
3441 return (db); in dbuf_create()
3443 db->db.db_size = (blkptr != NULL) ? in dbuf_create()
3445 db->db.db_offset = 0; in dbuf_create()
3448 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; in dbuf_create()
3449 db->db.db_size = blocksize; in dbuf_create()
3450 db->db.db_offset = db->db_blkid * blocksize; in dbuf_create()
3461 db->db_state = DB_EVICTING; /* not worth logging this state change */ in dbuf_create()
3462 if ((odb = dbuf_hash_insert(db)) != NULL) { in dbuf_create()
3465 kmem_cache_free(dbuf_kmem_cache, db); in dbuf_create()
3469 avl_add(&dn->dn_dbufs, db); in dbuf_create()
3471 db->db_state = DB_UNCACHED; in dbuf_create()
3472 DTRACE_SET_STATE(db, "regular buffer created"); in dbuf_create()
3473 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
3478 dbuf_add_ref(parent, db); in dbuf_create()
3482 (void) zfs_refcount_add(&dn->dn_holds, db); in dbuf_create()
3484 dprintf_dbuf(db, "db=%p\n", db); in dbuf_create()
3486 return (db); in dbuf_create()
3635 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, in dbuf_prefetch_indirect_done() local
3637 if (db == NULL) { in dbuf_prefetch_indirect_done()
3642 (void) dbuf_read(db, NULL, in dbuf_prefetch_indirect_done()
3644 dbuf_rele(db, FTAG); in dbuf_prefetch_indirect_done()
3724 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, in dbuf_prefetch_impl() local
3726 if (db != NULL) { in dbuf_prefetch_impl()
3727 mutex_exit(&db->db_mtx); in dbuf_prefetch_impl()
3745 dmu_buf_impl_t *db; in dbuf_prefetch_impl() local
3748 FALSE, TRUE, FTAG, &db) == 0) { in dbuf_prefetch_impl()
3749 blkptr_t *bpp = db->db_buf->b_data; in dbuf_prefetch_impl()
3751 dbuf_rele(db, FTAG); in dbuf_prefetch_impl()
3849 dbuf_hold_copy(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_hold_copy() argument
3851 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_copy()
3863 dbuf_set_data(db, arc_alloc_raw_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3868 dbuf_set_data(db, arc_alloc_compressed_buf( in dbuf_hold_copy()
3869 dn->dn_objset->os_spa, db, arc_buf_size(data), in dbuf_hold_copy()
3872 dbuf_set_data(db, arc_alloc_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3873 DBUF_GET_BUFC_TYPE(db), db->db.db_size)); in dbuf_hold_copy()
3876 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_hold_copy()
3877 memcpy(db->db.db_data, data->b_data, arc_buf_size(data)); in dbuf_hold_copy()
3878 rw_exit(&db->db_rwlock); in dbuf_hold_copy()
3890 dmu_buf_impl_t *db, *parent = NULL; in dbuf_hold_impl() local
3907 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid, &hv); in dbuf_hold_impl()
3909 if (db == NULL) { in dbuf_hold_impl()
3929 db = dbuf_create(dn, level, blkid, parent, bp, hv); in dbuf_hold_impl()
3932 if (fail_uncached && db->db_state != DB_CACHED) { in dbuf_hold_impl()
3933 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3937 if (db->db_buf != NULL) { in dbuf_hold_impl()
3938 arc_buf_access(db->db_buf); in dbuf_hold_impl()
3939 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); in dbuf_hold_impl()
3942 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); in dbuf_hold_impl()
3949 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_hold_impl()
3951 db->db_state == DB_CACHED && db->db_data_pending) { in dbuf_hold_impl()
3952 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_impl()
3953 if (dr->dt.dl.dr_data == db->db_buf) { in dbuf_hold_impl()
3954 ASSERT3P(db->db_buf, !=, NULL); in dbuf_hold_impl()
3955 dbuf_hold_copy(dn, db); in dbuf_hold_impl()
3959 if (multilist_link_active(&db->db_cache_link)) { in dbuf_hold_impl()
3960 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hold_impl()
3961 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_hold_impl()
3962 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_hold_impl()
3964 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); in dbuf_hold_impl()
3966 uint64_t size = db->db.db_size; in dbuf_hold_impl()
3967 uint64_t usize = dmu_buf_user_size(&db->db); in dbuf_hold_impl()
3969 &dbuf_caches[db->db_caching_status].size, size, db); in dbuf_hold_impl()
3971 &dbuf_caches[db->db_caching_status].size, usize, in dbuf_hold_impl()
3972 db->db_user); in dbuf_hold_impl()
3974 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { in dbuf_hold_impl()
3977 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_hold_impl()
3979 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], in dbuf_hold_impl()
3982 db->db_caching_status = DB_NO_CACHE; in dbuf_hold_impl()
3984 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_hold_impl()
3985 DBUF_VERIFY(db); in dbuf_hold_impl()
3986 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3992 ASSERT3P(DB_DNODE(db), ==, dn); in dbuf_hold_impl()
3993 ASSERT3U(db->db_blkid, ==, blkid); in dbuf_hold_impl()
3994 ASSERT3U(db->db_level, ==, level); in dbuf_hold_impl()
3995 *dbp = db; in dbuf_hold_impl()
4009 dmu_buf_impl_t *db; in dbuf_hold_level() local
4010 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); in dbuf_hold_level()
4011 return (err ? NULL : db); in dbuf_hold_level()
4027 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dbuf_spill_set_blksz() local
4029 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_spill_set_blksz()
4033 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); in dbuf_spill_set_blksz()
4036 dbuf_new_size(db, blksz, tx); in dbuf_spill_set_blksz()
4049 dbuf_add_ref(dmu_buf_impl_t *db, const void *tag) in dbuf_add_ref() argument
4051 int64_t holds = zfs_refcount_add(&db->db_holds, tag); in dbuf_add_ref()
4060 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dbuf_try_add_ref() local
4070 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { in dbuf_try_add_ref()
4071 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_try_add_ref()
4087 dbuf_rele(dmu_buf_impl_t *db, const void *tag) in dbuf_rele() argument
4089 mutex_enter(&db->db_mtx); in dbuf_rele()
4090 dbuf_rele_and_unlock(db, tag, B_FALSE); in dbuf_rele()
4094 dmu_buf_rele(dmu_buf_t *db, const void *tag) in dmu_buf_rele() argument
4096 dbuf_rele((dmu_buf_impl_t *)db, tag); in dmu_buf_rele()
4113 dbuf_rele_and_unlock(dmu_buf_impl_t *db, const void *tag, boolean_t evicting) in dbuf_rele_and_unlock() argument
4118 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_rele_and_unlock()
4119 DBUF_VERIFY(db); in dbuf_rele_and_unlock()
4126 holds = zfs_refcount_remove(&db->db_holds, tag); in dbuf_rele_and_unlock()
4133 if (db->db_buf != NULL && in dbuf_rele_and_unlock()
4134 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { in dbuf_rele_and_unlock()
4135 arc_buf_freeze(db->db_buf); in dbuf_rele_and_unlock()
4138 if (holds == db->db_dirtycnt && in dbuf_rele_and_unlock()
4139 db->db_level == 0 && db->db_user_immediate_evict) in dbuf_rele_and_unlock()
4140 dbuf_evict_user(db); in dbuf_rele_and_unlock()
4143 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_rele_and_unlock()
4145 boolean_t evict_dbuf = db->db_pending_evict; in dbuf_rele_and_unlock()
4151 DB_DNODE_ENTER(db); in dbuf_rele_and_unlock()
4153 dn = DB_DNODE(db); in dbuf_rele_and_unlock()
4162 DB_DNODE_EXIT(db); in dbuf_rele_and_unlock()
4165 * Do not reference db after its lock is dropped. in dbuf_rele_and_unlock()
4168 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4173 dnode_rele(dn, db); in dbuf_rele_and_unlock()
4174 } else if (db->db_buf == NULL) { in dbuf_rele_and_unlock()
4179 ASSERT(db->db_state == DB_UNCACHED || in dbuf_rele_and_unlock()
4180 db->db_state == DB_NOFILL); in dbuf_rele_and_unlock()
4181 dbuf_destroy(db); in dbuf_rele_and_unlock()
4182 } else if (arc_released(db->db_buf)) { in dbuf_rele_and_unlock()
4186 dbuf_destroy(db); in dbuf_rele_and_unlock()
4187 } else if (!(DBUF_IS_CACHEABLE(db) || db->db_partial_read) || in dbuf_rele_and_unlock()
4188 db->db_pending_evict) { in dbuf_rele_and_unlock()
4189 dbuf_destroy(db); in dbuf_rele_and_unlock()
4190 } else if (!multilist_link_active(&db->db_cache_link)) { in dbuf_rele_and_unlock()
4191 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_rele_and_unlock()
4194 dbuf_include_in_metadata_cache(db) ? in dbuf_rele_and_unlock()
4196 db->db_caching_status = dcs; in dbuf_rele_and_unlock()
4198 multilist_insert(&dbuf_caches[dcs].cache, db); in dbuf_rele_and_unlock()
4199 uint64_t db_size = db->db.db_size; in dbuf_rele_and_unlock()
4200 uint64_t dbu_size = dmu_buf_user_size(&db->db); in dbuf_rele_and_unlock()
4202 &dbuf_caches[dcs].size, db_size, db); in dbuf_rele_and_unlock()
4204 &dbuf_caches[dcs].size, dbu_size, db->db_user); in dbuf_rele_and_unlock()
4205 uint8_t db_level = db->db_level; in dbuf_rele_and_unlock()
4206 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4224 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4230 dbuf_refcount(dmu_buf_impl_t *db) in dbuf_refcount() argument
4232 return (zfs_refcount_count(&db->db_holds)); in dbuf_refcount()
4239 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_user_refcount() local
4241 mutex_enter(&db->db_mtx); in dmu_buf_user_refcount()
4242 ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt); in dmu_buf_user_refcount()
4243 holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt; in dmu_buf_user_refcount()
4244 mutex_exit(&db->db_mtx); in dmu_buf_user_refcount()
4253 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_replace_user() local
4255 mutex_enter(&db->db_mtx); in dmu_buf_replace_user()
4256 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_replace_user()
4257 if (db->db_user == old_user) in dmu_buf_replace_user()
4258 db->db_user = new_user; in dmu_buf_replace_user()
4260 old_user = db->db_user; in dmu_buf_replace_user()
4261 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_replace_user()
4262 mutex_exit(&db->db_mtx); in dmu_buf_replace_user()
4276 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_set_user_ie() local
4278 db->db_user_immediate_evict = TRUE; in dmu_buf_set_user_ie()
4291 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_get_user() local
4293 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_get_user()
4294 return (db->db_user); in dmu_buf_get_user()
4300 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_user_size() local
4301 if (db->db_user == NULL) in dmu_buf_user_size()
4303 return (atomic_load_64(&db->db_user->dbu_size)); in dmu_buf_user_size()
4309 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_add_user_size() local
4310 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dmu_buf_add_user_size()
4311 ASSERT3P(db->db_user, !=, NULL); in dmu_buf_add_user_size()
4312 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), <, UINT64_MAX - nadd); in dmu_buf_add_user_size()
4313 atomic_add_64(&db->db_user->dbu_size, nadd); in dmu_buf_add_user_size()
4319 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_sub_user_size() local
4320 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dmu_buf_sub_user_size()
4321 ASSERT3P(db->db_user, !=, NULL); in dmu_buf_sub_user_size()
4322 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), >=, nsub); in dmu_buf_sub_user_size()
4323 atomic_sub_64(&db->db_user->dbu_size, nsub); in dmu_buf_sub_user_size()
4333 dmu_buf_get_blkptr(dmu_buf_t *db) in dmu_buf_get_blkptr() argument
4335 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_get_blkptr()
4340 dmu_buf_get_objset(dmu_buf_t *db) in dmu_buf_get_objset() argument
4342 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_get_objset()
4347 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_check_blkptr() argument
4350 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_check_blkptr()
4352 if (db->db_blkptr != NULL) in dbuf_check_blkptr()
4355 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_check_blkptr()
4356 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_check_blkptr()
4357 BP_ZERO(db->db_blkptr); in dbuf_check_blkptr()
4360 if (db->db_level == dn->dn_phys->dn_nlevels-1) { in dbuf_check_blkptr()
4366 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); in dbuf_check_blkptr()
4367 ASSERT(db->db_parent == NULL); in dbuf_check_blkptr()
4368 db->db_parent = dn->dn_dbuf; in dbuf_check_blkptr()
4369 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; in dbuf_check_blkptr()
4370 DBUF_VERIFY(db); in dbuf_check_blkptr()
4372 dmu_buf_impl_t *parent = db->db_parent; in dbuf_check_blkptr()
4377 mutex_exit(&db->db_mtx); in dbuf_check_blkptr()
4379 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_check_blkptr()
4380 db->db_blkid >> epbs, db); in dbuf_check_blkptr()
4382 mutex_enter(&db->db_mtx); in dbuf_check_blkptr()
4383 db->db_parent = parent; in dbuf_check_blkptr()
4385 db->db_blkptr = (blkptr_t *)parent->db.db_data + in dbuf_check_blkptr()
4386 (db->db_blkid & ((1ULL << epbs) - 1)); in dbuf_check_blkptr()
4387 DBUF_VERIFY(db); in dbuf_check_blkptr()
4394 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_bonus() local
4397 ASSERT0(db->db_level); in dbuf_sync_bonus()
4398 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_sync_bonus()
4399 ASSERT(db->db_blkid == DMU_BONUS_BLKID); in dbuf_sync_bonus()
4410 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_sync_bonus()
4424 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_prepare_encrypted_dnode_leaf() local
4426 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_prepare_encrypted_dnode_leaf()
4427 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dbuf_prepare_encrypted_dnode_leaf()
4428 ASSERT3U(db->db_level, ==, 0); in dbuf_prepare_encrypted_dnode_leaf()
4430 if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) { in dbuf_prepare_encrypted_dnode_leaf()
4440 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
4441 db->db.db_object, db->db_level, db->db_blkid); in dbuf_prepare_encrypted_dnode_leaf()
4442 err = arc_untransform(db->db_buf, db->db_objset->os_spa, in dbuf_prepare_encrypted_dnode_leaf()
4447 (void) arc_release(dr->dt.dl.dr_data, db); in dbuf_prepare_encrypted_dnode_leaf()
4449 dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
4463 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_indirect() local
4468 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_indirect()
4470 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
4472 ASSERT(db->db_level > 0); in dbuf_sync_indirect()
4473 DBUF_VERIFY(db); in dbuf_sync_indirect()
4476 if (db->db_buf == NULL) { in dbuf_sync_indirect()
4477 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
4478 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); in dbuf_sync_indirect()
4479 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
4481 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_sync_indirect()
4482 ASSERT(db->db_buf != NULL); in dbuf_sync_indirect()
4485 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_sync_indirect()
4486 dbuf_check_blkptr(dn, db); in dbuf_sync_indirect()
4489 db->db_data_pending = dr; in dbuf_sync_indirect()
4491 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
4493 dbuf_write(dr, db->db_buf, tx); in dbuf_sync_indirect()
4497 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); in dbuf_sync_indirect()
4559 blkptr_t *bp = parent_db->db.db_data; in dbuf_lightweight_bp()
4674 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_leaf() local
4681 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_leaf()
4683 mutex_enter(&db->db_mtx); in dbuf_sync_leaf()
4688 if (db->db_state == DB_UNCACHED) { in dbuf_sync_leaf()
4690 ASSERT3P(db->db.db_data, ==, NULL); in dbuf_sync_leaf()
4691 } else if (db->db_state == DB_FILL) { in dbuf_sync_leaf()
4693 ASSERT(db->db.db_data != dr->dt.dl.dr_data); in dbuf_sync_leaf()
4694 } else if (db->db_state == DB_READ) { in dbuf_sync_leaf()
4706 list_head(&db->db_dirty_records); in dbuf_sync_leaf()
4707 ASSERT3P(db->db_buf, ==, NULL); in dbuf_sync_leaf()
4708 ASSERT3P(db->db.db_data, ==, NULL); in dbuf_sync_leaf()
4712 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); in dbuf_sync_leaf()
4714 DBUF_VERIFY(db); in dbuf_sync_leaf()
4716 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_sync_leaf()
4732 db->db_blkptr = NULL; in dbuf_sync_leaf()
4744 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_sync_leaf()
4745 ASSERT(dr->dr_dbuf == db); in dbuf_sync_leaf()
4758 dbuf_check_blkptr(dn, db); in dbuf_sync_leaf()
4770 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_sync_leaf()
4780 if (*datap != NULL && *datap == db->db_buf && in dbuf_sync_leaf()
4782 zfs_refcount_count(&db->db_holds) > 1 && in dbuf_sync_leaf()
4797 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_sync_leaf()
4808 *datap = arc_alloc_raw_buf(os->os_spa, db, in dbuf_sync_leaf()
4814 *datap = arc_alloc_compressed_buf(os->os_spa, db, in dbuf_sync_leaf()
4817 *datap = arc_alloc_buf(os->os_spa, db, type, psize); in dbuf_sync_leaf()
4819 memcpy((*datap)->b_data, db->db.db_data, psize); in dbuf_sync_leaf()
4821 db->db_data_pending = dr; in dbuf_sync_leaf()
4823 mutex_exit(&db->db_mtx); in dbuf_sync_leaf()
4853 ASSERT3U(dr->dr_dbuf->db.db_object, ==, in dbuf_sync_list()
4877 dmu_buf_impl_t *db = vdb; in dbuf_write_ready() local
4886 ASSERT3P(db->db_blkptr, !=, NULL); in dbuf_write_ready()
4887 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); in dbuf_write_ready()
4889 DB_DNODE_ENTER(db); in dbuf_write_ready()
4890 dn = DB_DNODE(db); in dbuf_write_ready()
4896 ASSERT((db->db_blkid != DMU_SPILL_BLKID && in dbuf_write_ready()
4898 (db->db_blkid == DMU_SPILL_BLKID && in dbuf_write_ready()
4901 ASSERT(BP_GET_LEVEL(bp) == db->db_level); in dbuf_write_ready()
4904 mutex_enter(&db->db_mtx); in dbuf_write_ready()
4907 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_ready()
4910 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_ready()
4914 if (db->db_level == 0) { in dbuf_write_ready()
4916 if (db->db_blkid > dn->dn_phys->dn_maxblkid && in dbuf_write_ready()
4917 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_write_ready()
4918 ASSERT0(db->db_objset->os_raw_receive); in dbuf_write_ready()
4919 dn->dn_phys->dn_maxblkid = db->db_blkid; in dbuf_write_ready()
4925 while (i < db->db.db_size) { in dbuf_write_ready()
4927 (void *)(((char *)db->db.db_data) + i); in dbuf_write_ready()
4958 blkptr_t *ibp = db->db.db_data; in dbuf_write_ready()
4959 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_write_ready()
4960 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { in dbuf_write_ready()
4968 DB_DNODE_EXIT(db); in dbuf_write_ready()
4973 mutex_exit(&db->db_mtx); in dbuf_write_ready()
4975 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_WRITER, FTAG); in dbuf_write_ready()
4976 *db->db_blkptr = *bp; in dbuf_write_ready()
4977 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_write_ready()
4991 dmu_buf_impl_t *db = vdb; in dbuf_write_children_ready() local
4995 ASSERT3U(db->db_level, >, 0); in dbuf_write_children_ready()
4996 DB_DNODE_ENTER(db); in dbuf_write_children_ready()
4997 epbs = DB_DNODE(db)->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_write_children_ready()
4998 DB_DNODE_EXIT(db); in dbuf_write_children_ready()
5002 for (i = 0, bp = db->db.db_data; i < 1ULL << epbs; i++, bp++) { in dbuf_write_children_ready()
5017 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_write_children_ready()
5018 memset(db->db.db_data, 0, db->db.db_size); in dbuf_write_children_ready()
5019 rw_exit(&db->db_rwlock); in dbuf_write_children_ready()
5027 dmu_buf_impl_t *db = vdb; in dbuf_write_done() local
5029 blkptr_t *bp = db->db_blkptr; in dbuf_write_done()
5030 objset_t *os = db->db_objset; in dbuf_write_done()
5034 ASSERT(db->db_blkptr == bp); in dbuf_write_done()
5048 mutex_enter(&db->db_mtx); in dbuf_write_done()
5050 DBUF_VERIFY(db); in dbuf_write_done()
5052 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_write_done()
5055 ASSERT(dr->dr_dbuf == db); in dbuf_write_done()
5056 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); in dbuf_write_done()
5057 list_remove(&db->db_dirty_records, dr); in dbuf_write_done()
5060 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_done()
5062 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && in dbuf_write_done()
5063 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_done()
5067 if (db->db_level == 0) { in dbuf_write_done()
5068 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_write_done()
5073 dr->dt.dl.dr_data != db->db_buf) { in dbuf_write_done()
5076 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_write_done()
5080 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); in dbuf_write_done()
5081 if (!BP_IS_HOLE(db->db_blkptr)) { in dbuf_write_done()
5084 ASSERT3U(db->db_blkid, <=, in dbuf_write_done()
5085 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); in dbuf_write_done()
5086 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, in dbuf_write_done()
5087 db->db.db_size); in dbuf_write_done()
5093 cv_broadcast(&db->db_changed); in dbuf_write_done()
5094 ASSERT(db->db_dirtycnt > 0); in dbuf_write_done()
5095 db->db_dirtycnt -= 1; in dbuf_write_done()
5096 db->db_data_pending = NULL; in dbuf_write_done()
5097 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_write_done()
5121 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_ready() local
5123 dbuf_write_ready(zio, NULL, db); in dbuf_write_override_ready()
5130 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_done() local
5133 mutex_enter(&db->db_mtx); in dbuf_write_override_done()
5137 arc_release(dr->dt.dl.dr_data, db); in dbuf_write_override_done()
5139 mutex_exit(&db->db_mtx); in dbuf_write_override_done()
5141 dbuf_write_done(zio, NULL, db); in dbuf_write_override_done()
5228 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_remap() argument
5230 spa_t *spa = dmu_objset_spa(db->db_objset); in dbuf_remap()
5236 if (db->db_level > 0) { in dbuf_remap()
5237 blkptr_t *bp = db->db.db_data; in dbuf_remap()
5238 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { in dbuf_remap()
5239 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx); in dbuf_remap()
5241 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_remap()
5242 dnode_phys_t *dnp = db->db.db_data; in dbuf_remap()
5244 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; in dbuf_remap()
5264 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write() local
5267 dmu_buf_impl_t *parent = db->db_parent; in dbuf_write()
5278 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { in dbuf_write()
5285 if (BP_IS_HOLE(db->db_blkptr)) in dbuf_write()
5288 dbuf_release_bp(db); in dbuf_write()
5289 dbuf_remap(dn, db, tx); in dbuf_write()
5297 ASSERT(db->db_level == parent->db_level-1); in dbuf_write()
5306 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && in dbuf_write()
5307 db->db_blkid != DMU_SPILL_BLKID) || in dbuf_write()
5308 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); in dbuf_write()
5309 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_write()
5310 ASSERT3P(db->db_blkptr, ==, in dbuf_write()
5311 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_write()
5315 ASSERT(db->db_level == 0 || data == db->db_buf); in dbuf_write()
5316 ASSERT3U(BP_GET_LOGICAL_BIRTH(db->db_blkptr), <=, txg); in dbuf_write()
5321 db->db.db_object, db->db_level, db->db_blkid); in dbuf_write()
5323 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_write()
5327 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); in dbuf_write()
5335 dr->dr_bp_copy = *db->db_blkptr; in dbuf_write()
5337 if (db->db_level == 0 && in dbuf_write()
5348 contents, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
5352 mutex_enter(&db->db_mtx); in dbuf_write()
5357 mutex_exit(&db->db_mtx); in dbuf_write()
5362 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
5364 dbuf_write_nofill_done, db, in dbuf_write()
5376 if (db->db_level != 0) in dbuf_write()
5380 &dr->dr_bp_copy, data, !DBUF_IS_CACHEABLE(db), in dbuf_write()
5381 dbuf_is_l2cacheable(db, NULL), &zp, dbuf_write_ready, in dbuf_write()
5382 children_ready_cb, dbuf_write_done, db, in dbuf_write()