Lines Matching refs:db
293 dmu_buf_impl_t *db = vdb; in dbuf_cons() local
294 memset(db, 0, sizeof (dmu_buf_impl_t)); in dbuf_cons()
296 mutex_init(&db->db_mtx, NULL, MUTEX_NOLOCKDEP, NULL); in dbuf_cons()
297 rw_init(&db->db_rwlock, NULL, RW_NOLOCKDEP, NULL); in dbuf_cons()
298 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); in dbuf_cons()
299 multilist_link_init(&db->db_cache_link); in dbuf_cons()
300 zfs_refcount_create(&db->db_holds); in dbuf_cons()
309 dmu_buf_impl_t *db = vdb; in dbuf_dest() local
310 mutex_destroy(&db->db_mtx); in dbuf_dest()
311 rw_destroy(&db->db_rwlock); in dbuf_dest()
312 cv_destroy(&db->db_changed); in dbuf_dest()
313 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_dest()
314 zfs_refcount_destroy(&db->db_holds); in dbuf_dest()
332 #define DTRACE_SET_STATE(db, why) \ argument
333 DTRACE_PROBE2(dbuf__state_change, dmu_buf_impl_t *, db, \
337 ((dbuf)->db.db_object == (obj) && \
349 dmu_buf_impl_t *db; in dbuf_find() local
355 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { in dbuf_find()
356 if (DBUF_EQUAL(db, os, obj, level, blkid)) { in dbuf_find()
357 mutex_enter(&db->db_mtx); in dbuf_find()
358 if (db->db_state != DB_EVICTING) { in dbuf_find()
360 return (db); in dbuf_find()
362 mutex_exit(&db->db_mtx); in dbuf_find()
375 dmu_buf_impl_t *db = NULL; in dbuf_find_bonus() local
380 db = dn->dn_bonus; in dbuf_find_bonus()
381 mutex_enter(&db->db_mtx); in dbuf_find_bonus()
386 return (db); in dbuf_find_bonus()
396 dbuf_hash_insert(dmu_buf_impl_t *db) in dbuf_hash_insert() argument
399 objset_t *os = db->db_objset; in dbuf_hash_insert()
400 uint64_t obj = db->db.db_object; in dbuf_hash_insert()
401 int level = db->db_level; in dbuf_hash_insert()
406 blkid = db->db_blkid; in dbuf_hash_insert()
407 ASSERT3U(dbuf_hash(os, obj, level, blkid), ==, db->db_hash); in dbuf_hash_insert()
408 idx = db->db_hash & h->hash_table_mask; in dbuf_hash_insert()
431 mutex_enter(&db->db_mtx); in dbuf_hash_insert()
432 db->db_hash_next = h->hash_table[idx]; in dbuf_hash_insert()
433 h->hash_table[idx] = db; in dbuf_hash_insert()
446 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db) in dbuf_include_in_metadata_cache() argument
448 DB_DNODE_ENTER(db); in dbuf_include_in_metadata_cache()
449 dmu_object_type_t type = DB_DNODE(db)->dn_type; in dbuf_include_in_metadata_cache()
450 DB_DNODE_EXIT(db); in dbuf_include_in_metadata_cache()
478 dbuf_hash_remove(dmu_buf_impl_t *db) in dbuf_hash_remove() argument
484 ASSERT3U(dbuf_hash(db->db_objset, db->db.db_object, db->db_level, in dbuf_hash_remove()
485 db->db_blkid), ==, db->db_hash); in dbuf_hash_remove()
486 idx = db->db_hash & h->hash_table_mask; in dbuf_hash_remove()
492 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hash_remove()
493 ASSERT(db->db_state == DB_EVICTING); in dbuf_hash_remove()
494 ASSERT(!MUTEX_HELD(&db->db_mtx)); in dbuf_hash_remove()
498 while ((dbf = *dbp) != db) { in dbuf_hash_remove()
502 *dbp = db->db_hash_next; in dbuf_hash_remove()
503 db->db_hash_next = NULL; in dbuf_hash_remove()
517 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) in dbuf_verify_user() argument
522 if (db->db_user == NULL) in dbuf_verify_user()
526 ASSERT(db->db_level == 0); in dbuf_verify_user()
529 ASSERT(db->db.db_data != NULL); in dbuf_verify_user()
530 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_verify_user()
532 holds = zfs_refcount_count(&db->db_holds); in dbuf_verify_user()
543 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
545 if (db->db_user_immediate_evict == TRUE) in dbuf_verify_user()
546 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
554 dbuf_evict_user(dmu_buf_impl_t *db) in dbuf_evict_user() argument
556 dmu_buf_user_t *dbu = db->db_user; in dbuf_evict_user()
558 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_evict_user()
563 dbuf_verify_user(db, DBVU_EVICTING); in dbuf_evict_user()
564 db->db_user = NULL; in dbuf_evict_user()
571 if (db->db_caching_status != DB_NO_CACHE) { in dbuf_evict_user()
581 &dbuf_caches[db->db_caching_status].size, size, dbu); in dbuf_evict_user()
582 if (db->db_caching_status == DB_DBUF_CACHE) in dbuf_evict_user()
583 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size); in dbuf_evict_user()
608 dbuf_is_metadata(dmu_buf_impl_t *db) in dbuf_is_metadata() argument
613 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) { in dbuf_is_metadata()
618 DB_DNODE_ENTER(db); in dbuf_is_metadata()
619 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); in dbuf_is_metadata()
620 DB_DNODE_EXIT(db); in dbuf_is_metadata()
631 dbuf_is_l2cacheable(dmu_buf_impl_t *db, blkptr_t *bp) in dbuf_is_l2cacheable() argument
633 if (db->db_objset->os_secondary_cache == ZFS_CACHE_ALL || in dbuf_is_l2cacheable()
634 (db->db_objset->os_secondary_cache == in dbuf_is_l2cacheable()
635 ZFS_CACHE_METADATA && dbuf_is_metadata(db))) { in dbuf_is_l2cacheable()
645 blkptr_t *db_bp = bp == NULL ? db->db_blkptr : bp; in dbuf_is_l2cacheable()
650 vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev; in dbuf_is_l2cacheable()
706 dmu_buf_impl_t *db = obj; in dbuf_cache_multilist_index_func() local
721 return ((unsigned int)dbuf_hash(db->db_objset, db->db.db_object, in dbuf_cache_multilist_index_func()
722 db->db_level, db->db_blkid) % in dbuf_cache_multilist_index_func()
783 dmu_buf_impl_t *db = multilist_sublist_tail(mls); in dbuf_evict_one() local
784 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { in dbuf_evict_one()
785 db = multilist_sublist_prev(mls, db); in dbuf_evict_one()
788 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, in dbuf_evict_one()
791 if (db != NULL) { in dbuf_evict_one()
792 multilist_sublist_remove(mls, db); in dbuf_evict_one()
794 uint64_t size = db->db.db_size; in dbuf_evict_one()
795 uint64_t usize = dmu_buf_user_size(&db->db); in dbuf_evict_one()
797 &dbuf_caches[DB_DBUF_CACHE].size, size, db); in dbuf_evict_one()
799 &dbuf_caches[DB_DBUF_CACHE].size, usize, db->db_user); in dbuf_evict_one()
800 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_evict_one()
802 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size + usize); in dbuf_evict_one()
803 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE); in dbuf_evict_one()
804 db->db_caching_status = DB_NO_CACHE; in dbuf_evict_one()
805 dbuf_destroy(db); in dbuf_evict_one()
1095 dbuf_verify(dmu_buf_impl_t *db) in dbuf_verify() argument
1101 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_verify()
1106 ASSERT(db->db_objset != NULL); in dbuf_verify()
1107 DB_DNODE_ENTER(db); in dbuf_verify()
1108 dn = DB_DNODE(db); in dbuf_verify()
1110 ASSERT(db->db_parent == NULL); in dbuf_verify()
1111 ASSERT(db->db_blkptr == NULL); in dbuf_verify()
1113 ASSERT3U(db->db.db_object, ==, dn->dn_object); in dbuf_verify()
1114 ASSERT3P(db->db_objset, ==, dn->dn_objset); in dbuf_verify()
1115 ASSERT3U(db->db_level, <, dn->dn_nlevels); in dbuf_verify()
1116 ASSERT(db->db_blkid == DMU_BONUS_BLKID || in dbuf_verify()
1117 db->db_blkid == DMU_SPILL_BLKID || in dbuf_verify()
1120 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_verify()
1122 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_verify()
1123 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); in dbuf_verify()
1124 } else if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_verify()
1126 ASSERT0(db->db.db_offset); in dbuf_verify()
1128 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); in dbuf_verify()
1131 if ((dr = list_head(&db->db_dirty_records)) != NULL) { in dbuf_verify()
1132 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
1134 for (dr = list_next(&db->db_dirty_records, dr); dr != NULL; in dbuf_verify()
1135 dr = list_next(&db->db_dirty_records, dr)) { in dbuf_verify()
1136 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
1147 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_verify()
1148 dr = db->db_data_pending; in dbuf_verify()
1153 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); in dbuf_verify()
1157 if (db->db_blkptr) { in dbuf_verify()
1158 if (db->db_parent == dn->dn_dbuf) { in dbuf_verify()
1161 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) in dbuf_verify()
1162 ASSERT(db->db_parent == NULL); in dbuf_verify()
1164 ASSERT(db->db_parent != NULL); in dbuf_verify()
1165 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_verify()
1166 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
1167 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_verify()
1170 int epb __maybe_unused = db->db_parent->db.db_size >> in dbuf_verify()
1172 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); in dbuf_verify()
1173 ASSERT3U(db->db_parent->db.db_object, ==, in dbuf_verify()
1174 db->db.db_object); in dbuf_verify()
1180 if (RW_LOCK_HELD(&db->db_parent->db_rwlock)) { in dbuf_verify()
1181 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
1182 ((blkptr_t *)db->db_parent->db.db_data + in dbuf_verify()
1183 db->db_blkid % epb)); in dbuf_verify()
1187 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && in dbuf_verify()
1188 (db->db_buf == NULL || db->db_buf->b_data) && in dbuf_verify()
1189 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && in dbuf_verify()
1190 db->db_state != DB_FILL && (dn == NULL || !dn->dn_free_txg)) { in dbuf_verify()
1202 if (db->db_dirtycnt == 0) { in dbuf_verify()
1203 if (db->db_level == 0) { in dbuf_verify()
1204 uint64_t *buf = db->db.db_data; in dbuf_verify()
1207 for (i = 0; i < db->db.db_size >> 3; i++) { in dbuf_verify()
1211 blkptr_t *bps = db->db.db_data; in dbuf_verify()
1212 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, in dbuf_verify()
1213 db->db.db_size); in dbuf_verify()
1222 i < db->db.db_size / sizeof (blkptr_t); in dbuf_verify()
1241 DB_DNODE_EXIT(db); in dbuf_verify()
1246 dbuf_clear_data(dmu_buf_impl_t *db) in dbuf_clear_data() argument
1248 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_clear_data()
1249 dbuf_evict_user(db); in dbuf_clear_data()
1250 ASSERT3P(db->db_buf, ==, NULL); in dbuf_clear_data()
1251 db->db.db_data = NULL; in dbuf_clear_data()
1252 if (db->db_state != DB_NOFILL) { in dbuf_clear_data()
1253 db->db_state = DB_UNCACHED; in dbuf_clear_data()
1254 DTRACE_SET_STATE(db, "clear data"); in dbuf_clear_data()
1259 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) in dbuf_set_data() argument
1261 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_set_data()
1264 db->db_buf = buf; in dbuf_set_data()
1266 db->db.db_data = buf->b_data; in dbuf_set_data()
1270 dbuf_alloc_arcbuf(dmu_buf_impl_t *db) in dbuf_alloc_arcbuf() argument
1272 spa_t *spa = db->db_objset->os_spa; in dbuf_alloc_arcbuf()
1274 return (arc_alloc_buf(spa, db, DBUF_GET_BUFC_TYPE(db), db->db.db_size)); in dbuf_alloc_arcbuf()
1281 dbuf_loan_arcbuf(dmu_buf_impl_t *db) in dbuf_loan_arcbuf() argument
1285 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_loan_arcbuf()
1286 mutex_enter(&db->db_mtx); in dbuf_loan_arcbuf()
1287 if (arc_released(db->db_buf) || zfs_refcount_count(&db->db_holds) > 1) { in dbuf_loan_arcbuf()
1288 int blksz = db->db.db_size; in dbuf_loan_arcbuf()
1289 spa_t *spa = db->db_objset->os_spa; in dbuf_loan_arcbuf()
1291 mutex_exit(&db->db_mtx); in dbuf_loan_arcbuf()
1293 memcpy(abuf->b_data, db->db.db_data, blksz); in dbuf_loan_arcbuf()
1295 abuf = db->db_buf; in dbuf_loan_arcbuf()
1296 arc_loan_inuse_buf(abuf, db); in dbuf_loan_arcbuf()
1297 db->db_buf = NULL; in dbuf_loan_arcbuf()
1298 dbuf_clear_data(db); in dbuf_loan_arcbuf()
1299 mutex_exit(&db->db_mtx); in dbuf_loan_arcbuf()
1355 dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw, const void *tag) in dmu_buf_lock_parent() argument
1358 if (db->db_parent != NULL) { in dmu_buf_lock_parent()
1359 rw_enter(&db->db_parent->db_rwlock, rw); in dmu_buf_lock_parent()
1361 } else if (dmu_objset_ds(db->db_objset) != NULL) { in dmu_buf_lock_parent()
1362 rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw, in dmu_buf_lock_parent()
1380 dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type, const void *tag) in dmu_buf_unlock_parent() argument
1383 rw_exit(&db->db_parent->db_rwlock); in dmu_buf_unlock_parent()
1385 rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag); in dmu_buf_unlock_parent()
1393 dmu_buf_impl_t *db = vdb; in dbuf_read_done() local
1395 mutex_enter(&db->db_mtx); in dbuf_read_done()
1396 ASSERT3U(db->db_state, ==, DB_READ); in dbuf_read_done()
1401 ASSERT(zfs_refcount_count(&db->db_holds) > 0); in dbuf_read_done()
1402 ASSERT(db->db_buf == NULL); in dbuf_read_done()
1403 ASSERT(db->db.db_data == NULL); in dbuf_read_done()
1407 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_read_done()
1408 ASSERT3P(db->db_buf, ==, NULL); in dbuf_read_done()
1409 db->db_state = DB_UNCACHED; in dbuf_read_done()
1410 DTRACE_SET_STATE(db, "i/o error"); in dbuf_read_done()
1411 } else if (db->db_level == 0 && db->db_freed_in_flight) { in dbuf_read_done()
1414 arc_release(buf, db); in dbuf_read_done()
1415 memset(buf->b_data, 0, db->db.db_size); in dbuf_read_done()
1417 db->db_freed_in_flight = FALSE; in dbuf_read_done()
1418 dbuf_set_data(db, buf); in dbuf_read_done()
1419 db->db_state = DB_CACHED; in dbuf_read_done()
1420 DTRACE_SET_STATE(db, "freed in flight"); in dbuf_read_done()
1424 dbuf_set_data(db, buf); in dbuf_read_done()
1425 db->db_state = DB_CACHED; in dbuf_read_done()
1426 DTRACE_SET_STATE(db, "successful read"); in dbuf_read_done()
1428 cv_broadcast(&db->db_changed); in dbuf_read_done()
1429 dbuf_rele_and_unlock(db, NULL, B_FALSE); in dbuf_read_done()
1438 dbuf_read_bonus(dmu_buf_impl_t *db, dnode_t *dn) in dbuf_read_bonus() argument
1444 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_bonus()
1445 ASSERT(DB_DNODE_HELD(db)); in dbuf_read_bonus()
1446 ASSERT3U(bonuslen, <=, db->db.db_size); in dbuf_read_bonus()
1447 db->db.db_data = kmem_alloc(max_bonuslen, KM_SLEEP); in dbuf_read_bonus()
1450 memset(db->db.db_data, 0, max_bonuslen); in dbuf_read_bonus()
1452 memcpy(db->db.db_data, DN_BONUS(dn->dn_phys), bonuslen); in dbuf_read_bonus()
1453 db->db_state = DB_CACHED; in dbuf_read_bonus()
1454 DTRACE_SET_STATE(db, "bonus buffer filled"); in dbuf_read_bonus()
1459 dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *dbbp) in dbuf_handle_indirect_hole() argument
1461 blkptr_t *bps = db->db.db_data; in dbuf_handle_indirect_hole()
1483 dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *bp) in dbuf_read_hole() argument
1485 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_hole()
1494 if (!is_hole && db->db_level == 0) in dbuf_read_hole()
1495 is_hole = dnode_block_freed(dn, db->db_blkid) || BP_IS_HOLE(bp); in dbuf_read_hole()
1498 dbuf_set_data(db, dbuf_alloc_arcbuf(db)); in dbuf_read_hole()
1499 memset(db->db.db_data, 0, db->db.db_size); in dbuf_read_hole()
1501 if (bp != NULL && db->db_level > 0 && BP_IS_HOLE(bp) && in dbuf_read_hole()
1503 dbuf_handle_indirect_hole(db, dn, bp); in dbuf_read_hole()
1505 db->db_state = DB_CACHED; in dbuf_read_hole()
1506 DTRACE_SET_STATE(db, "hole read satisfied"); in dbuf_read_hole()
1525 dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, dnode_t *dn, uint32_t flags) in dbuf_read_verify_dnode_crypt() argument
1527 objset_t *os = db->db_objset; in dbuf_read_verify_dnode_crypt()
1571 if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1573 (db->db_blkid == DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1587 dbuf_read_impl(dmu_buf_impl_t *db, dnode_t *dn, zio_t *zio, uint32_t flags, in dbuf_read_impl() argument
1594 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read_impl()
1595 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_impl()
1596 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_read_impl()
1597 ASSERT(db->db_buf == NULL); in dbuf_read_impl()
1598 ASSERT(db->db_parent == NULL || in dbuf_read_impl()
1599 RW_LOCK_HELD(&db->db_parent->db_rwlock)); in dbuf_read_impl()
1601 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_read_impl()
1602 err = dbuf_read_bonus(db, dn); in dbuf_read_impl()
1606 err = dbuf_read_hole(db, dn, bp); in dbuf_read_impl()
1619 db->db_objset->os_dsl_dataset, in dbuf_read_impl()
1625 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read_impl()
1626 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read_impl()
1632 if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bp)) { in dbuf_read_impl()
1633 spa_log_error(db->db_objset->os_spa, &zb, in dbuf_read_impl()
1639 db->db_state = DB_READ; in dbuf_read_impl()
1640 DTRACE_SET_STATE(db, "read issued"); in dbuf_read_impl()
1641 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1643 if (!DBUF_IS_CACHEABLE(db)) in dbuf_read_impl()
1645 else if (dbuf_is_l2cacheable(db, bp)) in dbuf_read_impl()
1648 dbuf_add_ref(db, NULL); in dbuf_read_impl()
1664 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1665 return (arc_read(zio, db->db_objset->os_spa, ©, in dbuf_read_impl()
1666 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags, in dbuf_read_impl()
1670 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1671 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1689 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) in dbuf_fix_old_data() argument
1691 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dbuf_fix_old_data()
1693 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_fix_old_data()
1694 ASSERT(db->db.db_data != NULL); in dbuf_fix_old_data()
1695 ASSERT(db->db_level == 0); in dbuf_fix_old_data()
1696 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dbuf_fix_old_data()
1700 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) in dbuf_fix_old_data()
1711 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_fix_old_data()
1712 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data()
1716 memcpy(dr->dt.dl.dr_data, db->db.db_data, bonuslen); in dbuf_fix_old_data()
1717 } else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) { in dbuf_fix_old_data()
1718 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data()
1719 int size = arc_buf_size(db->db_buf); in dbuf_fix_old_data()
1720 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_fix_old_data()
1721 spa_t *spa = db->db_objset->os_spa; in dbuf_fix_old_data()
1723 arc_get_compression(db->db_buf); in dbuf_fix_old_data()
1724 uint8_t complevel = arc_get_complevel(db->db_buf); in dbuf_fix_old_data()
1726 if (arc_is_encrypted(db->db_buf)) { in dbuf_fix_old_data()
1732 arc_get_raw_params(db->db_buf, &byteorder, salt, in dbuf_fix_old_data()
1734 dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db, in dbuf_fix_old_data()
1736 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf), in dbuf_fix_old_data()
1740 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db, in dbuf_fix_old_data()
1741 size, arc_buf_lsize(db->db_buf), compress_type, in dbuf_fix_old_data()
1744 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size); in dbuf_fix_old_data()
1746 memcpy(dr->dt.dl.dr_data->b_data, db->db.db_data, size); in dbuf_fix_old_data()
1748 db->db_buf = NULL; in dbuf_fix_old_data()
1749 dbuf_clear_data(db); in dbuf_fix_old_data()
1754 dbuf_read(dmu_buf_impl_t *db, zio_t *pio, uint32_t flags) in dbuf_read() argument
1760 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read()
1762 DB_DNODE_ENTER(db); in dbuf_read()
1763 dn = DB_DNODE(db); in dbuf_read()
1769 err = dbuf_read_verify_dnode_crypt(db, dn, flags); in dbuf_read()
1773 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_read()
1776 mutex_enter(&db->db_mtx); in dbuf_read()
1778 db->db_partial_read = B_TRUE; in dbuf_read()
1780 db->db_partial_read = B_FALSE; in dbuf_read()
1781 miss = (db->db_state != DB_CACHED); in dbuf_read()
1783 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_read()
1793 mutex_exit(&db->db_mtx); in dbuf_read()
1794 DB_DNODE_EXIT(db); in dbuf_read()
1798 ASSERT(db->db_state == DB_READ || in dbuf_read()
1800 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, db, in dbuf_read()
1802 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_read()
1803 } while (db->db_state == DB_READ || db->db_state == DB_FILL); in dbuf_read()
1804 if (db->db_state == DB_UNCACHED) { in dbuf_read()
1806 mutex_exit(&db->db_mtx); in dbuf_read()
1807 DB_DNODE_EXIT(db); in dbuf_read()
1812 if (db->db_state == DB_CACHED) { in dbuf_read()
1820 if ((flags & DB_RF_NO_DECRYPT) == 0 && db->db_buf != NULL && in dbuf_read()
1821 (arc_is_encrypted(db->db_buf) || in dbuf_read()
1822 arc_is_unauthenticated(db->db_buf) || in dbuf_read()
1823 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dbuf_read()
1827 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read()
1828 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read()
1829 dbuf_fix_old_data(db, spa_syncing_txg(spa)); in dbuf_read()
1830 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dbuf_read()
1831 dbuf_set_data(db, db->db_buf); in dbuf_read()
1833 mutex_exit(&db->db_mtx); in dbuf_read()
1835 ASSERT(db->db_state == DB_UNCACHED || in dbuf_read()
1836 db->db_state == DB_NOFILL); in dbuf_read()
1837 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); in dbuf_read()
1845 err = dmu_buf_get_bp_from_dbuf(db, &bp); in dbuf_read()
1848 if (pio == NULL && (db->db_state == DB_NOFILL || in dbuf_read()
1857 dbuf_read_impl(db, dn, pio, flags, dblt, bp, FTAG); in dbuf_read()
1859 mutex_exit(&db->db_mtx); in dbuf_read()
1860 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_read()
1863 miss = (db->db_state != DB_CACHED); in dbuf_read()
1867 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, miss, in dbuf_read()
1870 DB_DNODE_EXIT(db); in dbuf_read()
1900 dbuf_noread(dmu_buf_impl_t *db) in dbuf_noread() argument
1902 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_noread()
1903 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_noread()
1904 mutex_enter(&db->db_mtx); in dbuf_noread()
1905 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_noread()
1906 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_noread()
1907 if (db->db_state == DB_UNCACHED) { in dbuf_noread()
1908 ASSERT(db->db_buf == NULL); in dbuf_noread()
1909 ASSERT(db->db.db_data == NULL); in dbuf_noread()
1910 dbuf_set_data(db, dbuf_alloc_arcbuf(db)); in dbuf_noread()
1911 db->db_state = DB_FILL; in dbuf_noread()
1912 DTRACE_SET_STATE(db, "assigning filled buffer"); in dbuf_noread()
1913 } else if (db->db_state == DB_NOFILL) { in dbuf_noread()
1914 dbuf_clear_data(db); in dbuf_noread()
1916 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_noread()
1918 mutex_exit(&db->db_mtx); in dbuf_noread()
1924 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_unoverride() local
1928 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_unoverride()
1936 ASSERT(db->db_level == 0); in dbuf_unoverride()
1938 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_unoverride()
1942 ASSERT(db->db_data_pending != dr); in dbuf_unoverride()
1946 zio_free(db->db_objset->os_spa, txg, bp); in dbuf_unoverride()
1950 dr->dt.dl.dr_data = db->db_buf; in dbuf_unoverride()
1970 arc_release(dr->dt.dl.dr_data, db); in dbuf_unoverride()
1983 dmu_buf_impl_t *db, *db_next; in dbuf_free_range() local
2000 db = avl_find(&dn->dn_dbufs, db_search, &where); in dbuf_free_range()
2001 ASSERT3P(db, ==, NULL); in dbuf_free_range()
2003 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dbuf_free_range()
2005 for (; db != NULL; db = db_next) { in dbuf_free_range()
2006 db_next = AVL_NEXT(&dn->dn_dbufs, db); in dbuf_free_range()
2007 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_free_range()
2009 if (db->db_level != 0 || db->db_blkid > end_blkid) { in dbuf_free_range()
2012 ASSERT3U(db->db_blkid, >=, start_blkid); in dbuf_free_range()
2015 mutex_enter(&db->db_mtx); in dbuf_free_range()
2016 if (dbuf_undirty(db, tx)) { in dbuf_free_range()
2021 if (db->db_state == DB_UNCACHED || in dbuf_free_range()
2022 db->db_state == DB_NOFILL || in dbuf_free_range()
2023 db->db_state == DB_EVICTING) { in dbuf_free_range()
2024 ASSERT(db->db.db_data == NULL); in dbuf_free_range()
2025 mutex_exit(&db->db_mtx); in dbuf_free_range()
2028 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_free_range()
2030 db->db_freed_in_flight = TRUE; in dbuf_free_range()
2031 mutex_exit(&db->db_mtx); in dbuf_free_range()
2034 if (zfs_refcount_count(&db->db_holds) == 0) { in dbuf_free_range()
2035 ASSERT(db->db_buf); in dbuf_free_range()
2036 dbuf_destroy(db); in dbuf_free_range()
2041 dr = list_head(&db->db_dirty_records); in dbuf_free_range()
2049 if (db->db_blkid != DMU_SPILL_BLKID && in dbuf_free_range()
2050 db->db_blkid > dn->dn_maxblkid) in dbuf_free_range()
2051 dn->dn_maxblkid = db->db_blkid; in dbuf_free_range()
2060 dbuf_fix_old_data(db, txg); in dbuf_free_range()
2064 if (db->db_state == DB_CACHED) { in dbuf_free_range()
2065 ASSERT(db->db.db_data != NULL); in dbuf_free_range()
2066 arc_release(db->db_buf, db); in dbuf_free_range()
2067 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_free_range()
2068 memset(db->db.db_data, 0, db->db.db_size); in dbuf_free_range()
2069 rw_exit(&db->db_rwlock); in dbuf_free_range()
2070 arc_buf_freeze(db->db_buf); in dbuf_free_range()
2073 mutex_exit(&db->db_mtx); in dbuf_free_range()
2081 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) in dbuf_new_size() argument
2085 int osize = db->db.db_size; in dbuf_new_size()
2086 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_new_size()
2089 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_new_size()
2091 DB_DNODE_ENTER(db); in dbuf_new_size()
2092 dn = DB_DNODE(db); in dbuf_new_size()
2098 dmu_buf_will_dirty(&db->db, tx); in dbuf_new_size()
2100 VERIFY3P(db->db_buf, !=, NULL); in dbuf_new_size()
2103 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); in dbuf_new_size()
2106 old_buf = db->db_buf; in dbuf_new_size()
2112 mutex_enter(&db->db_mtx); in dbuf_new_size()
2113 dbuf_set_data(db, buf); in dbuf_new_size()
2114 arc_buf_destroy(old_buf, db); in dbuf_new_size()
2115 db->db.db_size = size; in dbuf_new_size()
2117 dr = list_head(&db->db_dirty_records); in dbuf_new_size()
2120 if (db->db_level == 0) in dbuf_new_size()
2125 mutex_exit(&db->db_mtx); in dbuf_new_size()
2128 DB_DNODE_EXIT(db); in dbuf_new_size()
2132 dbuf_release_bp(dmu_buf_impl_t *db) in dbuf_release_bp() argument
2134 objset_t *os __maybe_unused = db->db_objset; in dbuf_release_bp()
2139 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); in dbuf_release_bp()
2141 (void) arc_release(db->db_buf, db); in dbuf_release_bp()
2151 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_redirty() local
2153 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_redirty()
2155 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { in dbuf_redirty()
2161 if (db->db.db_object != DMU_META_DNODE_OBJECT && in dbuf_redirty()
2162 db->db_state != DB_NOFILL) { in dbuf_redirty()
2164 ASSERT(arc_released(db->db_buf)); in dbuf_redirty()
2165 arc_buf_thaw(db->db_buf); in dbuf_redirty()
2240 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_dirty() argument
2249 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_dirty()
2250 DMU_TX_DIRTY_BUF(tx, db); in dbuf_dirty()
2252 DB_DNODE_ENTER(db); in dbuf_dirty()
2253 dn = DB_DNODE(db); in dbuf_dirty()
2280 mutex_enter(&db->db_mtx); in dbuf_dirty()
2286 ASSERT(db->db_level != 0 || in dbuf_dirty()
2287 db->db_state == DB_CACHED || db->db_state == DB_FILL || in dbuf_dirty()
2288 db->db_state == DB_NOFILL); in dbuf_dirty()
2291 dnode_set_dirtyctx(dn, tx, db); in dbuf_dirty()
2296 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_dirty()
2302 dr_head = list_head(&db->db_dirty_records); in dbuf_dirty()
2304 db->db.db_object == DMU_META_DNODE_OBJECT); in dbuf_dirty()
2305 dr_next = dbuf_find_dirty_lte(db, tx->tx_txg); in dbuf_dirty()
2307 DB_DNODE_EXIT(db); in dbuf_dirty()
2310 mutex_exit(&db->db_mtx); in dbuf_dirty()
2321 ASSERT3U(dn->dn_nlevels, >, db->db_level); in dbuf_dirty()
2340 ASSERT(db->db.db_size != 0); in dbuf_dirty()
2342 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_dirty()
2344 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) { in dbuf_dirty()
2345 dmu_objset_willuse_space(os, db->db.db_size, tx); in dbuf_dirty()
2358 if (db->db_level == 0) { in dbuf_dirty()
2359 void *data_old = db->db_buf; in dbuf_dirty()
2361 if (db->db_state != DB_NOFILL) { in dbuf_dirty()
2362 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_dirty()
2363 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
2364 data_old = db->db.db_data; in dbuf_dirty()
2365 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { in dbuf_dirty()
2375 arc_release(db->db_buf, db); in dbuf_dirty()
2376 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
2377 data_old = db->db_buf; in dbuf_dirty()
2388 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) { in dbuf_dirty()
2389 dr->dr_accounted = db->db.db_size; in dbuf_dirty()
2391 dr->dr_dbuf = db; in dbuf_dirty()
2393 list_insert_before(&db->db_dirty_records, dr_next, dr); in dbuf_dirty()
2400 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_dirty()
2401 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_dirty()
2405 db->db_blkid, 1); in dbuf_dirty()
2408 db->db_freed_in_flight = FALSE; in dbuf_dirty()
2414 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); in dbuf_dirty()
2415 db->db_dirtycnt += 1; in dbuf_dirty()
2416 ASSERT3U(db->db_dirtycnt, <=, 3); in dbuf_dirty()
2418 mutex_exit(&db->db_mtx); in dbuf_dirty()
2420 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_dirty()
2421 db->db_blkid == DMU_SPILL_BLKID) { in dbuf_dirty()
2427 DB_DNODE_EXIT(db); in dbuf_dirty()
2442 if (db->db_blkptr != NULL) { in dbuf_dirty()
2443 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); in dbuf_dirty()
2444 ddt_prefetch(os->os_spa, db->db_blkptr); in dbuf_dirty()
2445 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_dirty()
2452 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || in dbuf_dirty()
2453 dn->dn_phys->dn_nlevels > db->db_level || in dbuf_dirty()
2454 dn->dn_next_nlevels[txgoff] > db->db_level || in dbuf_dirty()
2455 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || in dbuf_dirty()
2456 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); in dbuf_dirty()
2459 if (db->db_level == 0) { in dbuf_dirty()
2460 ASSERT(!db->db_objset->os_raw_receive || in dbuf_dirty()
2461 dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2462 dnode_new_blkid(dn, db->db_blkid, tx, in dbuf_dirty()
2464 ASSERT(dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2467 if (db->db_level+1 < dn->dn_nlevels) { in dbuf_dirty()
2468 dmu_buf_impl_t *parent = db->db_parent; in dbuf_dirty()
2472 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { in dbuf_dirty()
2474 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_dirty()
2475 db->db_blkid >> epbs, FTAG); in dbuf_dirty()
2481 ASSERT3U(db->db_level + 1, ==, parent->db_level); in dbuf_dirty()
2486 mutex_enter(&db->db_mtx); in dbuf_dirty()
2491 if (list_head(&db->db_dirty_records) == dr || in dbuf_dirty()
2500 mutex_exit(&db->db_mtx); in dbuf_dirty()
2502 ASSERT(db->db_level + 1 == dn->dn_nlevels); in dbuf_dirty()
2503 ASSERT(db->db_blkid < dn->dn_nblkptr); in dbuf_dirty()
2504 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); in dbuf_dirty()
2514 DB_DNODE_EXIT(db); in dbuf_dirty()
2521 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_undirty_bonus() local
2523 if (dr->dt.dl.dr_data != db->db.db_data) { in dbuf_undirty_bonus()
2530 db->db_data_pending = NULL; in dbuf_undirty_bonus()
2531 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); in dbuf_undirty_bonus()
2532 list_remove(&db->db_dirty_records, dr); in dbuf_undirty_bonus()
2538 ASSERT3U(db->db_dirtycnt, >, 0); in dbuf_undirty_bonus()
2539 db->db_dirtycnt -= 1; in dbuf_undirty_bonus()
2547 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_undirty() argument
2561 ASSERT(db->db_objset == in dbuf_undirty()
2562 dmu_objset_pool(db->db_objset)->dp_meta_objset || in dbuf_undirty()
2563 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); in dbuf_undirty()
2564 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_undirty()
2565 ASSERT0(db->db_level); in dbuf_undirty()
2566 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_undirty()
2571 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, txg); in dbuf_undirty()
2574 ASSERT(dr->dr_dbuf == db); in dbuf_undirty()
2586 brt_pending_remove(dmu_objset_spa(db->db_objset), in dbuf_undirty()
2593 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_undirty()
2595 ASSERT(db->db.db_size != 0); in dbuf_undirty()
2600 list_remove(&db->db_dirty_records, dr); in dbuf_undirty()
2612 } else if (db->db_blkid == DMU_SPILL_BLKID || in dbuf_undirty()
2613 db->db_level + 1 == dn->dn_nlevels) { in dbuf_undirty()
2614 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); in dbuf_undirty()
2620 if (db->db_state != DB_NOFILL && !brtwrite) { in dbuf_undirty()
2623 if (dr->dt.dl.dr_data != db->db_buf) { in dbuf_undirty()
2624 ASSERT(db->db_buf != NULL); in dbuf_undirty()
2626 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_undirty()
2632 ASSERT(db->db_dirtycnt > 0); in dbuf_undirty()
2633 db->db_dirtycnt -= 1; in dbuf_undirty()
2635 if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { in dbuf_undirty()
2636 ASSERT(db->db_state == DB_NOFILL || brtwrite || diowrite || in dbuf_undirty()
2637 arc_released(db->db_buf)); in dbuf_undirty()
2638 dbuf_destroy(db); in dbuf_undirty()
2648 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_dirty_impl() local
2652 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_dirty_impl()
2658 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_impl()
2659 if (db->db_state == DB_CACHED || db->db_state == DB_NOFILL) { in dmu_buf_will_dirty_impl()
2665 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_will_dirty_impl()
2667 if (db->db_level == 0 && in dmu_buf_will_dirty_impl()
2680 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2685 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2687 DB_DNODE_ENTER(db); in dmu_buf_will_dirty_impl()
2688 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) in dmu_buf_will_dirty_impl()
2690 DB_DNODE_EXIT(db); in dmu_buf_will_dirty_impl()
2698 (void) dbuf_read(db, NULL, flags); in dmu_buf_will_dirty_impl()
2700 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_impl()
2701 VERIFY(!dbuf_undirty(db, tx)); in dmu_buf_will_dirty_impl()
2702 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2704 (void) dbuf_dirty(db, tx); in dmu_buf_will_dirty_impl()
2717 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_is_dirty() local
2720 mutex_enter(&db->db_mtx); in dmu_buf_is_dirty()
2721 dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_is_dirty()
2722 mutex_exit(&db->db_mtx); in dmu_buf_is_dirty()
2733 dmu_buf_get_bp_from_dbuf(dmu_buf_impl_t *db, blkptr_t **bp) in dmu_buf_get_bp_from_dbuf() argument
2735 ASSERT(MUTEX_HELD(&db->db_mtx)); in dmu_buf_get_bp_from_dbuf()
2738 if (db->db_level != 0) { in dmu_buf_get_bp_from_dbuf()
2739 *bp = db->db_blkptr; in dmu_buf_get_bp_from_dbuf()
2743 *bp = db->db_blkptr; in dmu_buf_get_bp_from_dbuf()
2744 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dmu_buf_get_bp_from_dbuf()
2745 if (dr && db->db_state == DB_NOFILL) { in dmu_buf_get_bp_from_dbuf()
2751 } else if (dr && db->db_state == DB_UNCACHED) { in dmu_buf_get_bp_from_dbuf()
2765 dmu_buf_untransform_direct(dmu_buf_impl_t *db, spa_t *spa) in dmu_buf_untransform_direct() argument
2768 DB_DNODE_ENTER(db); in dmu_buf_untransform_direct()
2769 dnode_t *dn = DB_DNODE(db); in dmu_buf_untransform_direct()
2771 ASSERT3S(db->db_state, ==, DB_CACHED); in dmu_buf_untransform_direct()
2772 ASSERT(MUTEX_HELD(&db->db_mtx)); in dmu_buf_untransform_direct()
2778 err = dbuf_read_verify_dnode_crypt(db, dn, 0); in dmu_buf_untransform_direct()
2787 if (err == 0 && db->db_buf != NULL && in dmu_buf_untransform_direct()
2788 (arc_is_encrypted(db->db_buf) || in dmu_buf_untransform_direct()
2789 arc_is_unauthenticated(db->db_buf) || in dmu_buf_untransform_direct()
2790 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dmu_buf_untransform_direct()
2793 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dmu_buf_untransform_direct()
2794 db->db.db_object, db->db_level, db->db_blkid); in dmu_buf_untransform_direct()
2795 dbuf_fix_old_data(db, spa_syncing_txg(spa)); in dmu_buf_untransform_direct()
2796 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dmu_buf_untransform_direct()
2797 dbuf_set_data(db, db->db_buf); in dmu_buf_untransform_direct()
2799 DB_DNODE_EXIT(db); in dmu_buf_untransform_direct()
2811 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_clone_or_dio() local
2812 ASSERT0(db->db_level); in dmu_buf_will_clone_or_dio()
2814 ASSERT0(db->db_level); in dmu_buf_will_clone_or_dio()
2815 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_clone_or_dio()
2816 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dmu_buf_will_clone_or_dio()
2818 mutex_enter(&db->db_mtx); in dmu_buf_will_clone_or_dio()
2819 DBUF_VERIFY(db); in dmu_buf_will_clone_or_dio()
2843 VERIFY3B(dbuf_undirty(db, tx), ==, B_FALSE); in dmu_buf_will_clone_or_dio()
2844 ASSERT0P(dbuf_find_dirty_eq(db, tx->tx_txg)); in dmu_buf_will_clone_or_dio()
2846 if (db->db_buf != NULL) { in dmu_buf_will_clone_or_dio()
2852 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dmu_buf_will_clone_or_dio()
2853 if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) in dmu_buf_will_clone_or_dio()
2854 arc_buf_destroy(db->db_buf, db); in dmu_buf_will_clone_or_dio()
2861 db->db_buf = NULL; in dmu_buf_will_clone_or_dio()
2862 dbuf_clear_data(db); in dmu_buf_will_clone_or_dio()
2865 ASSERT3P(db->db_buf, ==, NULL); in dmu_buf_will_clone_or_dio()
2866 ASSERT3P(db->db.db_data, ==, NULL); in dmu_buf_will_clone_or_dio()
2868 db->db_state = DB_NOFILL; in dmu_buf_will_clone_or_dio()
2869 DTRACE_SET_STATE(db, in dmu_buf_will_clone_or_dio()
2872 DBUF_VERIFY(db); in dmu_buf_will_clone_or_dio()
2873 mutex_exit(&db->db_mtx); in dmu_buf_will_clone_or_dio()
2875 dbuf_noread(db); in dmu_buf_will_clone_or_dio()
2876 (void) dbuf_dirty(db, tx); in dmu_buf_will_clone_or_dio()
2882 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_not_fill() local
2884 mutex_enter(&db->db_mtx); in dmu_buf_will_not_fill()
2885 db->db_state = DB_NOFILL; in dmu_buf_will_not_fill()
2886 DTRACE_SET_STATE(db, "allocating NOFILL buffer"); in dmu_buf_will_not_fill()
2887 mutex_exit(&db->db_mtx); in dmu_buf_will_not_fill()
2889 dbuf_noread(db); in dmu_buf_will_not_fill()
2890 (void) dbuf_dirty(db, tx); in dmu_buf_will_not_fill()
2896 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_fill() local
2898 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_fill()
2900 ASSERT(db->db_level == 0); in dmu_buf_will_fill()
2901 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_fill()
2903 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || in dmu_buf_will_fill()
2906 mutex_enter(&db->db_mtx); in dmu_buf_will_fill()
2907 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_will_fill()
2908 if (db->db_state == DB_NOFILL || in dmu_buf_will_fill()
2909 (db->db_state == DB_UNCACHED && dr && dr->dt.dl.dr_diowrite)) { in dmu_buf_will_fill()
2915 mutex_exit(&db->db_mtx); in dmu_buf_will_fill()
2925 if (db->db_state == DB_NOFILL) { in dmu_buf_will_fill()
2926 VERIFY(!dbuf_undirty(db, tx)); in dmu_buf_will_fill()
2927 db->db_state = DB_UNCACHED; in dmu_buf_will_fill()
2930 mutex_exit(&db->db_mtx); in dmu_buf_will_fill()
2932 dbuf_noread(db); in dmu_buf_will_fill()
2933 (void) dbuf_dirty(db, tx); in dmu_buf_will_fill()
2947 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_set_crypt_params() local
2954 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dmu_buf_set_crypt_params()
2955 ASSERT0(db->db_level); in dmu_buf_set_crypt_params()
2956 ASSERT(db->db_objset->os_raw_receive); in dmu_buf_set_crypt_params()
2961 dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_set_crypt_params()
2974 dbuf_override_impl(dmu_buf_impl_t *db, const blkptr_t *bp, dmu_tx_t *tx) in dbuf_override_impl() argument
2979 ASSERT3U(db->db.db_object, !=, DMU_META_DNODE_OBJECT); in dbuf_override_impl()
2980 ASSERT0(db->db_level); in dbuf_override_impl()
2982 dr = list_head(&db->db_dirty_records); in dbuf_override_impl()
2996 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_fill_done() local
2997 mutex_enter(&db->db_mtx); in dmu_buf_fill_done()
2998 DBUF_VERIFY(db); in dmu_buf_fill_done()
3000 if (db->db_state == DB_FILL) { in dmu_buf_fill_done()
3001 if (db->db_level == 0 && db->db_freed_in_flight) { in dmu_buf_fill_done()
3002 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_fill_done()
3005 memset(db->db.db_data, 0, db->db.db_size); in dmu_buf_fill_done()
3006 db->db_freed_in_flight = FALSE; in dmu_buf_fill_done()
3007 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3008 DTRACE_SET_STATE(db, in dmu_buf_fill_done()
3012 VERIFY(!dbuf_undirty(db, tx)); in dmu_buf_fill_done()
3013 arc_buf_destroy(db->db_buf, db); in dmu_buf_fill_done()
3014 db->db_buf = NULL; in dmu_buf_fill_done()
3015 dbuf_clear_data(db); in dmu_buf_fill_done()
3016 DTRACE_SET_STATE(db, "fill failed"); in dmu_buf_fill_done()
3018 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3019 DTRACE_SET_STATE(db, "fill done"); in dmu_buf_fill_done()
3021 cv_broadcast(&db->db_changed); in dmu_buf_fill_done()
3023 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3026 mutex_exit(&db->db_mtx); in dmu_buf_fill_done()
3036 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_write_embedded() local
3042 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), in dmu_buf_write_embedded()
3046 DB_DNODE_ENTER(db); in dmu_buf_write_embedded()
3047 type = DB_DNODE(db)->dn_type; in dmu_buf_write_embedded()
3048 DB_DNODE_EXIT(db); in dmu_buf_write_embedded()
3050 ASSERT0(db->db_level); in dmu_buf_write_embedded()
3051 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_write_embedded()
3055 dr = list_head(&db->db_dirty_records); in dmu_buf_write_embedded()
3074 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_redact() local
3076 ASSERT(dsl_dataset_feature_is_active(db->db_objset->os_dsl_dataset, in dmu_buf_redact()
3079 DB_DNODE_ENTER(db); in dmu_buf_redact()
3080 type = DB_DNODE(db)->dn_type; in dmu_buf_redact()
3081 DB_DNODE_EXIT(db); in dmu_buf_redact()
3083 ASSERT0(db->db_level); in dmu_buf_redact()
3093 dbuf_override_impl(db, &bp, tx); in dmu_buf_redact()
3101 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) in dbuf_assign_arcbuf() argument
3103 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_assign_arcbuf()
3104 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_assign_arcbuf()
3105 ASSERT(db->db_level == 0); in dbuf_assign_arcbuf()
3106 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf)); in dbuf_assign_arcbuf()
3108 ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size); in dbuf_assign_arcbuf()
3111 arc_return_buf(buf, db); in dbuf_assign_arcbuf()
3114 mutex_enter(&db->db_mtx); in dbuf_assign_arcbuf()
3116 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_assign_arcbuf()
3117 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_assign_arcbuf()
3119 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED || in dbuf_assign_arcbuf()
3120 db->db_state == DB_NOFILL); in dbuf_assign_arcbuf()
3122 if (db->db_state == DB_CACHED && in dbuf_assign_arcbuf()
3123 zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { in dbuf_assign_arcbuf()
3131 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
3132 (void) dbuf_dirty(db, tx); in dbuf_assign_arcbuf()
3133 memcpy(db->db.db_data, buf->b_data, db->db.db_size); in dbuf_assign_arcbuf()
3134 arc_buf_destroy(buf, db); in dbuf_assign_arcbuf()
3138 if (db->db_state == DB_CACHED) { in dbuf_assign_arcbuf()
3139 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dbuf_assign_arcbuf()
3141 ASSERT(db->db_buf != NULL); in dbuf_assign_arcbuf()
3143 ASSERT(dr->dt.dl.dr_data == db->db_buf); in dbuf_assign_arcbuf()
3145 if (!arc_released(db->db_buf)) { in dbuf_assign_arcbuf()
3148 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
3151 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
3152 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { in dbuf_assign_arcbuf()
3153 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
3154 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
3156 db->db_buf = NULL; in dbuf_assign_arcbuf()
3157 } else if (db->db_state == DB_NOFILL) { in dbuf_assign_arcbuf()
3164 VERIFY(!dbuf_undirty(db, tx)); in dbuf_assign_arcbuf()
3165 db->db_state = DB_UNCACHED; in dbuf_assign_arcbuf()
3167 ASSERT(db->db_buf == NULL); in dbuf_assign_arcbuf()
3168 dbuf_set_data(db, buf); in dbuf_assign_arcbuf()
3169 db->db_state = DB_FILL; in dbuf_assign_arcbuf()
3170 DTRACE_SET_STATE(db, "filling assigned arcbuf"); in dbuf_assign_arcbuf()
3171 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
3172 (void) dbuf_dirty(db, tx); in dbuf_assign_arcbuf()
3173 dmu_buf_fill_done(&db->db, tx, B_FALSE); in dbuf_assign_arcbuf()
3177 dbuf_destroy(dmu_buf_impl_t *db) in dbuf_destroy() argument
3180 dmu_buf_impl_t *parent = db->db_parent; in dbuf_destroy()
3183 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_destroy()
3184 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
3186 if (db->db_buf != NULL) { in dbuf_destroy()
3187 arc_buf_destroy(db->db_buf, db); in dbuf_destroy()
3188 db->db_buf = NULL; in dbuf_destroy()
3191 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_destroy()
3192 int slots = DB_DNODE(db)->dn_num_slots; in dbuf_destroy()
3194 if (db->db.db_data != NULL) { in dbuf_destroy()
3195 kmem_free(db->db.db_data, bonuslen); in dbuf_destroy()
3197 db->db_state = DB_UNCACHED; in dbuf_destroy()
3198 DTRACE_SET_STATE(db, "buffer cleared"); in dbuf_destroy()
3202 dbuf_clear_data(db); in dbuf_destroy()
3204 if (multilist_link_active(&db->db_cache_link)) { in dbuf_destroy()
3205 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_destroy()
3206 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_destroy()
3208 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); in dbuf_destroy()
3210 ASSERT0(dmu_buf_user_size(&db->db)); in dbuf_destroy()
3212 &dbuf_caches[db->db_caching_status].size, in dbuf_destroy()
3213 db->db.db_size, db); in dbuf_destroy()
3215 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { in dbuf_destroy()
3218 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_destroy()
3220 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], in dbuf_destroy()
3221 db->db.db_size); in dbuf_destroy()
3223 db->db_caching_status = DB_NO_CACHE; in dbuf_destroy()
3226 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_destroy()
3227 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
3228 ASSERT(list_is_empty(&db->db_dirty_records)); in dbuf_destroy()
3230 db->db_state = DB_EVICTING; in dbuf_destroy()
3231 DTRACE_SET_STATE(db, "buffer eviction started"); in dbuf_destroy()
3232 db->db_blkptr = NULL; in dbuf_destroy()
3239 mutex_exit(&db->db_mtx); in dbuf_destroy()
3241 DB_DNODE_ENTER(db); in dbuf_destroy()
3242 dn = DB_DNODE(db); in dbuf_destroy()
3244 if (db->db_blkid != DMU_BONUS_BLKID) { in dbuf_destroy()
3249 avl_remove(&dn->dn_dbufs, db); in dbuf_destroy()
3251 DB_DNODE_EXIT(db); in dbuf_destroy()
3263 dnode_rele_and_unlock(dn, db, B_TRUE); in dbuf_destroy()
3265 db->db_dnode_handle = NULL; in dbuf_destroy()
3267 db->db_dnode = NULL; in dbuf_destroy()
3270 dbuf_hash_remove(db); in dbuf_destroy()
3272 DB_DNODE_EXIT(db); in dbuf_destroy()
3275 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
3277 db->db_parent = NULL; in dbuf_destroy()
3279 ASSERT(db->db_buf == NULL); in dbuf_destroy()
3280 ASSERT(db->db.db_data == NULL); in dbuf_destroy()
3281 ASSERT(db->db_hash_next == NULL); in dbuf_destroy()
3282 ASSERT(db->db_blkptr == NULL); in dbuf_destroy()
3283 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
3284 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_destroy()
3285 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_destroy()
3293 dbuf_rele_and_unlock(parent, db, B_TRUE); in dbuf_destroy()
3296 kmem_cache_free(dbuf_kmem_cache, db); in dbuf_destroy()
3377 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + in dbuf_findbp()
3402 dmu_buf_impl_t *db, *odb; in dbuf_create() local
3407 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); in dbuf_create()
3409 list_create(&db->db_dirty_records, sizeof (dbuf_dirty_record_t), in dbuf_create()
3412 db->db_objset = os; in dbuf_create()
3413 db->db.db_object = dn->dn_object; in dbuf_create()
3414 db->db_level = level; in dbuf_create()
3415 db->db_blkid = blkid; in dbuf_create()
3416 db->db_dirtycnt = 0; in dbuf_create()
3418 db->db_dnode_handle = dn->dn_handle; in dbuf_create()
3420 db->db_dnode = dn; in dbuf_create()
3422 db->db_parent = parent; in dbuf_create()
3423 db->db_blkptr = blkptr; in dbuf_create()
3424 db->db_hash = hash; in dbuf_create()
3426 db->db_user = NULL; in dbuf_create()
3427 db->db_user_immediate_evict = FALSE; in dbuf_create()
3428 db->db_freed_in_flight = FALSE; in dbuf_create()
3429 db->db_pending_evict = FALSE; in dbuf_create()
3433 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dbuf_create()
3435 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_create()
3436 db->db.db_offset = DMU_BONUS_BLKID; in dbuf_create()
3437 db->db_state = DB_UNCACHED; in dbuf_create()
3438 DTRACE_SET_STATE(db, "bonus buffer created"); in dbuf_create()
3439 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
3442 return (db); in dbuf_create()
3444 db->db.db_size = (blkptr != NULL) ? in dbuf_create()
3446 db->db.db_offset = 0; in dbuf_create()
3449 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; in dbuf_create()
3450 db->db.db_size = blocksize; in dbuf_create()
3451 db->db.db_offset = db->db_blkid * blocksize; in dbuf_create()
3462 db->db_state = DB_EVICTING; /* not worth logging this state change */ in dbuf_create()
3463 if ((odb = dbuf_hash_insert(db)) != NULL) { in dbuf_create()
3466 kmem_cache_free(dbuf_kmem_cache, db); in dbuf_create()
3470 avl_add(&dn->dn_dbufs, db); in dbuf_create()
3472 db->db_state = DB_UNCACHED; in dbuf_create()
3473 DTRACE_SET_STATE(db, "regular buffer created"); in dbuf_create()
3474 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
3479 dbuf_add_ref(parent, db); in dbuf_create()
3483 (void) zfs_refcount_add(&dn->dn_holds, db); in dbuf_create()
3485 dprintf_dbuf(db, "db=%p\n", db); in dbuf_create()
3487 return (db); in dbuf_create()
3633 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, in dbuf_prefetch_indirect_done() local
3635 if (db == NULL) { in dbuf_prefetch_indirect_done()
3640 (void) dbuf_read(db, NULL, in dbuf_prefetch_indirect_done()
3642 dbuf_rele(db, FTAG); in dbuf_prefetch_indirect_done()
3728 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, in dbuf_prefetch_impl() local
3730 if (db != NULL) { in dbuf_prefetch_impl()
3731 mutex_exit(&db->db_mtx); in dbuf_prefetch_impl()
3749 dmu_buf_impl_t *db; in dbuf_prefetch_impl() local
3752 FALSE, TRUE, FTAG, &db) == 0) { in dbuf_prefetch_impl()
3753 blkptr_t *bpp = db->db_buf->b_data; in dbuf_prefetch_impl()
3755 dbuf_rele(db, FTAG); in dbuf_prefetch_impl()
3853 dbuf_hold_copy(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_hold_copy() argument
3855 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_copy()
3867 dbuf_set_data(db, arc_alloc_raw_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3872 dbuf_set_data(db, arc_alloc_compressed_buf( in dbuf_hold_copy()
3873 dn->dn_objset->os_spa, db, arc_buf_size(data), in dbuf_hold_copy()
3876 dbuf_set_data(db, arc_alloc_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3877 DBUF_GET_BUFC_TYPE(db), db->db.db_size)); in dbuf_hold_copy()
3880 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_hold_copy()
3881 memcpy(db->db.db_data, data->b_data, arc_buf_size(data)); in dbuf_hold_copy()
3882 rw_exit(&db->db_rwlock); in dbuf_hold_copy()
3894 dmu_buf_impl_t *db, *parent = NULL; in dbuf_hold_impl() local
3911 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid, &hv); in dbuf_hold_impl()
3913 if (db == NULL) { in dbuf_hold_impl()
3933 db = dbuf_create(dn, level, blkid, parent, bp, hv); in dbuf_hold_impl()
3936 if (fail_uncached && db->db_state != DB_CACHED) { in dbuf_hold_impl()
3937 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3941 if (db->db_buf != NULL) { in dbuf_hold_impl()
3942 arc_buf_access(db->db_buf); in dbuf_hold_impl()
3943 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); in dbuf_hold_impl()
3946 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); in dbuf_hold_impl()
3953 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_hold_impl()
3955 db->db_state == DB_CACHED && db->db_data_pending) { in dbuf_hold_impl()
3956 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_impl()
3957 if (dr->dt.dl.dr_data == db->db_buf) { in dbuf_hold_impl()
3958 ASSERT3P(db->db_buf, !=, NULL); in dbuf_hold_impl()
3959 dbuf_hold_copy(dn, db); in dbuf_hold_impl()
3963 if (multilist_link_active(&db->db_cache_link)) { in dbuf_hold_impl()
3964 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hold_impl()
3965 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_hold_impl()
3966 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_hold_impl()
3968 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); in dbuf_hold_impl()
3970 uint64_t size = db->db.db_size; in dbuf_hold_impl()
3971 uint64_t usize = dmu_buf_user_size(&db->db); in dbuf_hold_impl()
3973 &dbuf_caches[db->db_caching_status].size, size, db); in dbuf_hold_impl()
3975 &dbuf_caches[db->db_caching_status].size, usize, in dbuf_hold_impl()
3976 db->db_user); in dbuf_hold_impl()
3978 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { in dbuf_hold_impl()
3981 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_hold_impl()
3983 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], in dbuf_hold_impl()
3986 db->db_caching_status = DB_NO_CACHE; in dbuf_hold_impl()
3988 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_hold_impl()
3989 DBUF_VERIFY(db); in dbuf_hold_impl()
3990 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3996 ASSERT3P(DB_DNODE(db), ==, dn); in dbuf_hold_impl()
3997 ASSERT3U(db->db_blkid, ==, blkid); in dbuf_hold_impl()
3998 ASSERT3U(db->db_level, ==, level); in dbuf_hold_impl()
3999 *dbp = db; in dbuf_hold_impl()
4013 dmu_buf_impl_t *db; in dbuf_hold_level() local
4014 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); in dbuf_hold_level()
4015 return (err ? NULL : db); in dbuf_hold_level()
4031 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dbuf_spill_set_blksz() local
4033 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_spill_set_blksz()
4037 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); in dbuf_spill_set_blksz()
4040 dbuf_new_size(db, blksz, tx); in dbuf_spill_set_blksz()
4053 dbuf_add_ref(dmu_buf_impl_t *db, const void *tag) in dbuf_add_ref() argument
4055 int64_t holds = zfs_refcount_add(&db->db_holds, tag); in dbuf_add_ref()
4064 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dbuf_try_add_ref() local
4074 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { in dbuf_try_add_ref()
4075 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_try_add_ref()
4091 dbuf_rele(dmu_buf_impl_t *db, const void *tag) in dbuf_rele() argument
4093 mutex_enter(&db->db_mtx); in dbuf_rele()
4094 dbuf_rele_and_unlock(db, tag, B_FALSE); in dbuf_rele()
4098 dmu_buf_rele(dmu_buf_t *db, const void *tag) in dmu_buf_rele() argument
4100 dbuf_rele((dmu_buf_impl_t *)db, tag); in dmu_buf_rele()
4117 dbuf_rele_and_unlock(dmu_buf_impl_t *db, const void *tag, boolean_t evicting) in dbuf_rele_and_unlock() argument
4122 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_rele_and_unlock()
4123 DBUF_VERIFY(db); in dbuf_rele_and_unlock()
4130 holds = zfs_refcount_remove(&db->db_holds, tag); in dbuf_rele_and_unlock()
4137 if (db->db_buf != NULL && in dbuf_rele_and_unlock()
4138 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { in dbuf_rele_and_unlock()
4139 arc_buf_freeze(db->db_buf); in dbuf_rele_and_unlock()
4142 if (holds == db->db_dirtycnt && in dbuf_rele_and_unlock()
4143 db->db_level == 0 && db->db_user_immediate_evict) in dbuf_rele_and_unlock()
4144 dbuf_evict_user(db); in dbuf_rele_and_unlock()
4147 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_rele_and_unlock()
4149 boolean_t evict_dbuf = db->db_pending_evict; in dbuf_rele_and_unlock()
4155 DB_DNODE_ENTER(db); in dbuf_rele_and_unlock()
4157 dn = DB_DNODE(db); in dbuf_rele_and_unlock()
4166 DB_DNODE_EXIT(db); in dbuf_rele_and_unlock()
4172 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4177 dnode_rele(dn, db); in dbuf_rele_and_unlock()
4178 } else if (db->db_buf == NULL) { in dbuf_rele_and_unlock()
4183 ASSERT(db->db_state == DB_UNCACHED || in dbuf_rele_and_unlock()
4184 db->db_state == DB_NOFILL); in dbuf_rele_and_unlock()
4185 dbuf_destroy(db); in dbuf_rele_and_unlock()
4186 } else if (arc_released(db->db_buf)) { in dbuf_rele_and_unlock()
4190 dbuf_destroy(db); in dbuf_rele_and_unlock()
4191 } else if (!(DBUF_IS_CACHEABLE(db) || db->db_partial_read) || in dbuf_rele_and_unlock()
4192 db->db_pending_evict) { in dbuf_rele_and_unlock()
4193 dbuf_destroy(db); in dbuf_rele_and_unlock()
4194 } else if (!multilist_link_active(&db->db_cache_link)) { in dbuf_rele_and_unlock()
4195 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_rele_and_unlock()
4198 dbuf_include_in_metadata_cache(db) ? in dbuf_rele_and_unlock()
4200 db->db_caching_status = dcs; in dbuf_rele_and_unlock()
4202 multilist_insert(&dbuf_caches[dcs].cache, db); in dbuf_rele_and_unlock()
4203 uint64_t db_size = db->db.db_size; in dbuf_rele_and_unlock()
4204 uint64_t dbu_size = dmu_buf_user_size(&db->db); in dbuf_rele_and_unlock()
4206 &dbuf_caches[dcs].size, db_size, db); in dbuf_rele_and_unlock()
4208 &dbuf_caches[dcs].size, dbu_size, db->db_user); in dbuf_rele_and_unlock()
4209 uint8_t db_level = db->db_level; in dbuf_rele_and_unlock()
4210 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4228 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4234 dbuf_refcount(dmu_buf_impl_t *db) in dbuf_refcount() argument
4236 return (zfs_refcount_count(&db->db_holds)); in dbuf_refcount()
4243 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_user_refcount() local
4245 mutex_enter(&db->db_mtx); in dmu_buf_user_refcount()
4246 ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt); in dmu_buf_user_refcount()
4247 holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt; in dmu_buf_user_refcount()
4248 mutex_exit(&db->db_mtx); in dmu_buf_user_refcount()
4257 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_replace_user() local
4259 mutex_enter(&db->db_mtx); in dmu_buf_replace_user()
4260 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_replace_user()
4261 if (db->db_user == old_user) in dmu_buf_replace_user()
4262 db->db_user = new_user; in dmu_buf_replace_user()
4264 old_user = db->db_user; in dmu_buf_replace_user()
4265 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_replace_user()
4266 mutex_exit(&db->db_mtx); in dmu_buf_replace_user()
4280 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_set_user_ie() local
4282 db->db_user_immediate_evict = TRUE; in dmu_buf_set_user_ie()
4295 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_get_user() local
4297 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_get_user()
4298 return (db->db_user); in dmu_buf_get_user()
4304 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_user_size() local
4305 if (db->db_user == NULL) in dmu_buf_user_size()
4307 return (atomic_load_64(&db->db_user->dbu_size)); in dmu_buf_user_size()
4313 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_add_user_size() local
4314 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dmu_buf_add_user_size()
4315 ASSERT3P(db->db_user, !=, NULL); in dmu_buf_add_user_size()
4316 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), <, UINT64_MAX - nadd); in dmu_buf_add_user_size()
4317 atomic_add_64(&db->db_user->dbu_size, nadd); in dmu_buf_add_user_size()
4323 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_sub_user_size() local
4324 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dmu_buf_sub_user_size()
4325 ASSERT3P(db->db_user, !=, NULL); in dmu_buf_sub_user_size()
4326 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), >=, nsub); in dmu_buf_sub_user_size()
4327 atomic_sub_64(&db->db_user->dbu_size, nsub); in dmu_buf_sub_user_size()
4337 dmu_buf_get_blkptr(dmu_buf_t *db) in dmu_buf_get_blkptr() argument
4339 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_get_blkptr()
4344 dmu_buf_get_objset(dmu_buf_t *db) in dmu_buf_get_objset() argument
4346 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_get_objset()
4351 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_check_blkptr() argument
4354 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_check_blkptr()
4356 if (db->db_blkptr != NULL) in dbuf_check_blkptr()
4359 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_check_blkptr()
4360 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_check_blkptr()
4361 BP_ZERO(db->db_blkptr); in dbuf_check_blkptr()
4364 if (db->db_level == dn->dn_phys->dn_nlevels-1) { in dbuf_check_blkptr()
4370 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); in dbuf_check_blkptr()
4371 ASSERT(db->db_parent == NULL); in dbuf_check_blkptr()
4372 db->db_parent = dn->dn_dbuf; in dbuf_check_blkptr()
4373 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; in dbuf_check_blkptr()
4374 DBUF_VERIFY(db); in dbuf_check_blkptr()
4376 dmu_buf_impl_t *parent = db->db_parent; in dbuf_check_blkptr()
4381 mutex_exit(&db->db_mtx); in dbuf_check_blkptr()
4383 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_check_blkptr()
4384 db->db_blkid >> epbs, db); in dbuf_check_blkptr()
4386 mutex_enter(&db->db_mtx); in dbuf_check_blkptr()
4387 db->db_parent = parent; in dbuf_check_blkptr()
4389 db->db_blkptr = (blkptr_t *)parent->db.db_data + in dbuf_check_blkptr()
4390 (db->db_blkid & ((1ULL << epbs) - 1)); in dbuf_check_blkptr()
4391 DBUF_VERIFY(db); in dbuf_check_blkptr()
4398 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_bonus() local
4401 ASSERT0(db->db_level); in dbuf_sync_bonus()
4402 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_sync_bonus()
4403 ASSERT(db->db_blkid == DMU_BONUS_BLKID); in dbuf_sync_bonus()
4414 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_sync_bonus()
4428 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_prepare_encrypted_dnode_leaf() local
4430 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_prepare_encrypted_dnode_leaf()
4431 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dbuf_prepare_encrypted_dnode_leaf()
4432 ASSERT3U(db->db_level, ==, 0); in dbuf_prepare_encrypted_dnode_leaf()
4434 if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) { in dbuf_prepare_encrypted_dnode_leaf()
4444 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
4445 db->db.db_object, db->db_level, db->db_blkid); in dbuf_prepare_encrypted_dnode_leaf()
4446 err = arc_untransform(db->db_buf, db->db_objset->os_spa, in dbuf_prepare_encrypted_dnode_leaf()
4451 (void) arc_release(dr->dt.dl.dr_data, db); in dbuf_prepare_encrypted_dnode_leaf()
4453 dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
4467 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_indirect() local
4472 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_indirect()
4474 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
4476 ASSERT(db->db_level > 0); in dbuf_sync_indirect()
4477 DBUF_VERIFY(db); in dbuf_sync_indirect()
4480 if (db->db_buf == NULL) { in dbuf_sync_indirect()
4481 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
4482 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); in dbuf_sync_indirect()
4483 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
4485 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_sync_indirect()
4486 ASSERT(db->db_buf != NULL); in dbuf_sync_indirect()
4489 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_sync_indirect()
4490 dbuf_check_blkptr(dn, db); in dbuf_sync_indirect()
4493 db->db_data_pending = dr; in dbuf_sync_indirect()
4495 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
4497 dbuf_write(dr, db->db_buf, tx); in dbuf_sync_indirect()
4501 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); in dbuf_sync_indirect()
4563 blkptr_t *bp = parent_db->db.db_data; in dbuf_lightweight_bp()
4678 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_leaf() local
4685 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_leaf()
4687 mutex_enter(&db->db_mtx); in dbuf_sync_leaf()
4692 if (db->db_state == DB_UNCACHED) { in dbuf_sync_leaf()
4694 ASSERT3P(db->db.db_data, ==, NULL); in dbuf_sync_leaf()
4695 } else if (db->db_state == DB_FILL) { in dbuf_sync_leaf()
4697 ASSERT(db->db.db_data != dr->dt.dl.dr_data); in dbuf_sync_leaf()
4698 } else if (db->db_state == DB_READ) { in dbuf_sync_leaf()
4710 list_head(&db->db_dirty_records); in dbuf_sync_leaf()
4711 ASSERT3P(db->db_buf, ==, NULL); in dbuf_sync_leaf()
4712 ASSERT3P(db->db.db_data, ==, NULL); in dbuf_sync_leaf()
4716 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); in dbuf_sync_leaf()
4718 DBUF_VERIFY(db); in dbuf_sync_leaf()
4720 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_sync_leaf()
4736 db->db_blkptr = NULL; in dbuf_sync_leaf()
4748 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_sync_leaf()
4749 ASSERT(dr->dr_dbuf == db); in dbuf_sync_leaf()
4762 dbuf_check_blkptr(dn, db); in dbuf_sync_leaf()
4774 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_sync_leaf()
4784 if (*datap != NULL && *datap == db->db_buf && in dbuf_sync_leaf()
4786 zfs_refcount_count(&db->db_holds) > 1) { in dbuf_sync_leaf()
4800 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_sync_leaf()
4811 *datap = arc_alloc_raw_buf(os->os_spa, db, in dbuf_sync_leaf()
4817 *datap = arc_alloc_compressed_buf(os->os_spa, db, in dbuf_sync_leaf()
4820 *datap = arc_alloc_buf(os->os_spa, db, type, psize); in dbuf_sync_leaf()
4822 memcpy((*datap)->b_data, db->db.db_data, psize); in dbuf_sync_leaf()
4824 db->db_data_pending = dr; in dbuf_sync_leaf()
4826 mutex_exit(&db->db_mtx); in dbuf_sync_leaf()
4856 ASSERT3U(dr->dr_dbuf->db.db_object, ==, in dbuf_sync_list()
4880 dmu_buf_impl_t *db = vdb; in dbuf_write_ready() local
4889 ASSERT3P(db->db_blkptr, !=, NULL); in dbuf_write_ready()
4890 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); in dbuf_write_ready()
4892 DB_DNODE_ENTER(db); in dbuf_write_ready()
4893 dn = DB_DNODE(db); in dbuf_write_ready()
4899 ASSERT((db->db_blkid != DMU_SPILL_BLKID && in dbuf_write_ready()
4901 (db->db_blkid == DMU_SPILL_BLKID && in dbuf_write_ready()
4904 ASSERT(BP_GET_LEVEL(bp) == db->db_level); in dbuf_write_ready()
4907 mutex_enter(&db->db_mtx); in dbuf_write_ready()
4910 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_ready()
4913 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_ready()
4917 if (db->db_level == 0) { in dbuf_write_ready()
4919 if (db->db_blkid > dn->dn_phys->dn_maxblkid && in dbuf_write_ready()
4920 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_write_ready()
4921 ASSERT0(db->db_objset->os_raw_receive); in dbuf_write_ready()
4922 dn->dn_phys->dn_maxblkid = db->db_blkid; in dbuf_write_ready()
4928 while (i < db->db.db_size) { in dbuf_write_ready()
4930 (void *)(((char *)db->db.db_data) + i); in dbuf_write_ready()
4961 blkptr_t *ibp = db->db.db_data; in dbuf_write_ready()
4962 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_write_ready()
4963 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { in dbuf_write_ready()
4971 DB_DNODE_EXIT(db); in dbuf_write_ready()
4976 mutex_exit(&db->db_mtx); in dbuf_write_ready()
4978 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_WRITER, FTAG); in dbuf_write_ready()
4979 *db->db_blkptr = *bp; in dbuf_write_ready()
4980 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_write_ready()
4994 dmu_buf_impl_t *db = vdb; in dbuf_write_children_ready() local
4998 ASSERT3U(db->db_level, >, 0); in dbuf_write_children_ready()
4999 DB_DNODE_ENTER(db); in dbuf_write_children_ready()
5000 epbs = DB_DNODE(db)->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_write_children_ready()
5001 DB_DNODE_EXIT(db); in dbuf_write_children_ready()
5005 for (i = 0, bp = db->db.db_data; i < 1ULL << epbs; i++, bp++) { in dbuf_write_children_ready()
5020 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_write_children_ready()
5021 memset(db->db.db_data, 0, db->db.db_size); in dbuf_write_children_ready()
5022 rw_exit(&db->db_rwlock); in dbuf_write_children_ready()
5030 dmu_buf_impl_t *db = vdb; in dbuf_write_done() local
5032 blkptr_t *bp = db->db_blkptr; in dbuf_write_done()
5033 objset_t *os = db->db_objset; in dbuf_write_done()
5037 ASSERT(db->db_blkptr == bp); in dbuf_write_done()
5051 mutex_enter(&db->db_mtx); in dbuf_write_done()
5053 DBUF_VERIFY(db); in dbuf_write_done()
5055 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_write_done()
5058 ASSERT(dr->dr_dbuf == db); in dbuf_write_done()
5059 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); in dbuf_write_done()
5060 list_remove(&db->db_dirty_records, dr); in dbuf_write_done()
5063 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_done()
5065 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && in dbuf_write_done()
5066 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_done()
5070 if (db->db_level == 0) { in dbuf_write_done()
5071 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_write_done()
5076 dr->dt.dl.dr_data != db->db_buf) { in dbuf_write_done()
5079 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_write_done()
5083 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); in dbuf_write_done()
5084 if (!BP_IS_HOLE(db->db_blkptr)) { in dbuf_write_done()
5087 ASSERT3U(db->db_blkid, <=, in dbuf_write_done()
5088 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); in dbuf_write_done()
5089 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, in dbuf_write_done()
5090 db->db.db_size); in dbuf_write_done()
5096 cv_broadcast(&db->db_changed); in dbuf_write_done()
5097 ASSERT(db->db_dirtycnt > 0); in dbuf_write_done()
5098 db->db_dirtycnt -= 1; in dbuf_write_done()
5099 db->db_data_pending = NULL; in dbuf_write_done()
5100 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_write_done()
5124 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_ready() local
5126 dbuf_write_ready(zio, NULL, db); in dbuf_write_override_ready()
5133 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_done() local
5136 mutex_enter(&db->db_mtx); in dbuf_write_override_done()
5140 arc_release(dr->dt.dl.dr_data, db); in dbuf_write_override_done()
5142 mutex_exit(&db->db_mtx); in dbuf_write_override_done()
5144 dbuf_write_done(zio, NULL, db); in dbuf_write_override_done()
5231 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_remap() argument
5233 spa_t *spa = dmu_objset_spa(db->db_objset); in dbuf_remap()
5239 if (db->db_level > 0) { in dbuf_remap()
5240 blkptr_t *bp = db->db.db_data; in dbuf_remap()
5241 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { in dbuf_remap()
5242 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx); in dbuf_remap()
5244 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_remap()
5245 dnode_phys_t *dnp = db->db.db_data; in dbuf_remap()
5247 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; in dbuf_remap()
5267 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write() local
5270 dmu_buf_impl_t *parent = db->db_parent; in dbuf_write()
5281 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { in dbuf_write()
5288 if (BP_IS_HOLE(db->db_blkptr)) in dbuf_write()
5291 dbuf_release_bp(db); in dbuf_write()
5292 dbuf_remap(dn, db, tx); in dbuf_write()
5300 ASSERT(db->db_level == parent->db_level-1); in dbuf_write()
5309 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && in dbuf_write()
5310 db->db_blkid != DMU_SPILL_BLKID) || in dbuf_write()
5311 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); in dbuf_write()
5312 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_write()
5313 ASSERT3P(db->db_blkptr, ==, in dbuf_write()
5314 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_write()
5318 ASSERT(db->db_level == 0 || data == db->db_buf); in dbuf_write()
5319 ASSERT3U(BP_GET_LOGICAL_BIRTH(db->db_blkptr), <=, txg); in dbuf_write()
5324 db->db.db_object, db->db_level, db->db_blkid); in dbuf_write()
5326 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_write()
5330 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); in dbuf_write()
5338 dr->dr_bp_copy = *db->db_blkptr; in dbuf_write()
5340 if (db->db_level == 0 && in dbuf_write()
5351 contents, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
5355 mutex_enter(&db->db_mtx); in dbuf_write()
5360 mutex_exit(&db->db_mtx); in dbuf_write()
5365 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
5367 dbuf_write_nofill_done, db, in dbuf_write()
5379 if (db->db_level != 0) in dbuf_write()
5383 &dr->dr_bp_copy, data, !DBUF_IS_CACHEABLE(db), in dbuf_write()
5384 dbuf_is_l2cacheable(db, NULL), &zp, dbuf_write_ready, in dbuf_write()
5385 children_ready_cb, dbuf_write_done, db, in dbuf_write()