Lines Matching +full:io +full:- +full:hv
1 // SPDX-License-Identifier: CDDL-1.0
10 * or https://opensource.org/licenses/CDDL-1.0.
166 DBUF_STAT_INCR(stat, -(val))
170 DBUF_STAT_INCR(stat, -1)
196 * 1. Cache of metadata dbufs, to help make read-heavy administrative commands
200 * pool, to short-circuit as much I/O as possible for all administrative
217 * from the cache and later re-added to the head of the cache.
245 * The LRU dbuf cache uses a three-stage eviction policy:
246 * - A low water marker designates when the dbuf eviction thread
248 * - When we reach the maximum size (aka mid water mark), we
250 * - The high water mark indicates when the eviction thread
257 * +----------------------------------------+----------+----------+
262 * +----------------------------------------+----------+----------+
296 mutex_init(&db->db_mtx, NULL, MUTEX_NOLOCKDEP, NULL); in dbuf_cons()
297 rw_init(&db->db_rwlock, NULL, RW_NOLOCKDEP, NULL); in dbuf_cons()
298 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); in dbuf_cons()
299 multilist_link_init(&db->db_cache_link); in dbuf_cons()
300 zfs_refcount_create(&db->db_holds); in dbuf_cons()
310 mutex_destroy(&db->db_mtx); in dbuf_dest()
311 rw_destroy(&db->db_rwlock); in dbuf_dest()
312 cv_destroy(&db->db_changed); in dbuf_dest()
313 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_dest()
314 zfs_refcount_destroy(&db->db_holds); in dbuf_dest()
337 ((dbuf)->db.db_object == (obj) && \
338 (dbuf)->db_objset == (os) && \
339 (dbuf)->db_level == (level) && \
340 (dbuf)->db_blkid == (blkid))
347 uint64_t hv; in dbuf_find() local
351 hv = dbuf_hash(os, obj, level, blkid); in dbuf_find()
352 idx = hv & h->hash_table_mask; in dbuf_find()
355 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { in dbuf_find()
357 mutex_enter(&db->db_mtx); in dbuf_find()
358 if (db->db_state != DB_EVICTING) { in dbuf_find()
362 mutex_exit(&db->db_mtx); in dbuf_find()
367 *hash_out = hv; in dbuf_find()
378 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_find_bonus()
379 if (dn->dn_bonus != NULL) { in dbuf_find_bonus()
380 db = dn->dn_bonus; in dbuf_find_bonus()
381 mutex_enter(&db->db_mtx); in dbuf_find_bonus()
383 rw_exit(&dn->dn_struct_rwlock); in dbuf_find_bonus()
399 objset_t *os = db->db_objset; in dbuf_hash_insert()
400 uint64_t obj = db->db.db_object; in dbuf_hash_insert()
401 int level = db->db_level; in dbuf_hash_insert()
406 blkid = db->db_blkid; in dbuf_hash_insert()
407 ASSERT3U(dbuf_hash(os, obj, level, blkid), ==, db->db_hash); in dbuf_hash_insert()
408 idx = db->db_hash & h->hash_table_mask; in dbuf_hash_insert()
411 for (dbf = h->hash_table[idx], i = 0; dbf != NULL; in dbuf_hash_insert()
412 dbf = dbf->db_hash_next, i++) { in dbuf_hash_insert()
414 mutex_enter(&dbf->db_mtx); in dbuf_hash_insert()
415 if (dbf->db_state != DB_EVICTING) { in dbuf_hash_insert()
419 mutex_exit(&dbf->db_mtx); in dbuf_hash_insert()
431 mutex_enter(&db->db_mtx); in dbuf_hash_insert()
432 db->db_hash_next = h->hash_table[idx]; in dbuf_hash_insert()
433 h->hash_table[idx] = db; in dbuf_hash_insert()
449 dmu_object_type_t type = DB_DNODE(db)->dn_type; in dbuf_include_in_metadata_cache()
458 * Sanity check for small-memory systems: don't allocate too in dbuf_include_in_metadata_cache()
484 ASSERT3U(dbuf_hash(db->db_objset, db->db.db_object, db->db_level, in dbuf_hash_remove()
485 db->db_blkid), ==, db->db_hash); in dbuf_hash_remove()
486 idx = db->db_hash & h->hash_table_mask; in dbuf_hash_remove()
492 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hash_remove()
493 ASSERT(db->db_state == DB_EVICTING); in dbuf_hash_remove()
494 ASSERT(!MUTEX_HELD(&db->db_mtx)); in dbuf_hash_remove()
497 dbp = &h->hash_table[idx]; in dbuf_hash_remove()
499 dbp = &dbf->db_hash_next; in dbuf_hash_remove()
502 *dbp = db->db_hash_next; in dbuf_hash_remove()
503 db->db_hash_next = NULL; in dbuf_hash_remove()
504 if (h->hash_table[idx] && in dbuf_hash_remove()
505 h->hash_table[idx]->db_hash_next == NULL) in dbuf_hash_remove()
522 if (db->db_user == NULL) in dbuf_verify_user()
526 ASSERT(db->db_level == 0); in dbuf_verify_user()
529 ASSERT(db->db.db_data != NULL); in dbuf_verify_user()
530 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_verify_user()
532 holds = zfs_refcount_count(&db->db_holds); in dbuf_verify_user()
543 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
545 if (db->db_user_immediate_evict == TRUE) in dbuf_verify_user()
546 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
556 dmu_buf_user_t *dbu = db->db_user; in dbuf_evict_user()
558 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_evict_user()
564 db->db_user = NULL; in dbuf_evict_user()
567 if (dbu->dbu_clear_on_evict_dbufp != NULL) in dbuf_evict_user()
568 *dbu->dbu_clear_on_evict_dbufp = NULL; in dbuf_evict_user()
571 if (db->db_caching_status != DB_NO_CACHE) { in dbuf_evict_user()
579 uint64_t size = dbu->dbu_size; in dbuf_evict_user()
581 &dbuf_caches[db->db_caching_status].size, size, dbu); in dbuf_evict_user()
582 if (db->db_caching_status == DB_DBUF_CACHE) in dbuf_evict_user()
583 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size); in dbuf_evict_user()
587 * There are two eviction callbacks - one that we call synchronously in dbuf_evict_user()
596 boolean_t has_async = (dbu->dbu_evict_func_async != NULL); in dbuf_evict_user()
598 if (dbu->dbu_evict_func_sync != NULL) in dbuf_evict_user()
599 dbu->dbu_evict_func_sync(dbu); in dbuf_evict_user()
602 taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func_async, in dbuf_evict_user()
603 dbu, 0, &dbu->dbu_tqent); in dbuf_evict_user()
613 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) { in dbuf_is_metadata()
619 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); in dbuf_is_metadata()
633 if (db->db_objset->os_secondary_cache == ZFS_CACHE_ALL || in dbuf_is_l2cacheable()
634 (db->db_objset->os_secondary_cache == in dbuf_is_l2cacheable()
645 blkptr_t *db_bp = bp == NULL ? db->db_blkptr : bp; in dbuf_is_l2cacheable()
649 uint64_t vdev = DVA_GET_VDEV(db_bp->blk_dva); in dbuf_is_l2cacheable()
650 vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev; in dbuf_is_l2cacheable()
653 if (vdev < rvd->vdev_children) in dbuf_is_l2cacheable()
654 vd = rvd->vdev_child[vdev]; in dbuf_is_l2cacheable()
659 if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL && in dbuf_is_l2cacheable()
660 vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) in dbuf_is_l2cacheable()
669 if (dn->dn_objset->os_secondary_cache == ZFS_CACHE_ALL || in dnode_level_is_l2cacheable()
670 (dn->dn_objset->os_secondary_cache == ZFS_CACHE_METADATA && in dnode_level_is_l2cacheable()
672 DMU_OT_IS_METADATA(dn->dn_handle->dnh_dnode->dn_type)))) { in dnode_level_is_l2cacheable()
678 uint64_t vdev = DVA_GET_VDEV(bp->blk_dva); in dnode_level_is_l2cacheable()
679 vdev_t *rvd = dn->dn_objset->os_spa->spa_root_vdev; in dnode_level_is_l2cacheable()
682 if (vdev < rvd->vdev_children) in dnode_level_is_l2cacheable()
683 vd = rvd->vdev_child[vdev]; in dnode_level_is_l2cacheable()
688 if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL && in dnode_level_is_l2cacheable()
689 vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) in dnode_level_is_l2cacheable()
721 return ((unsigned int)dbuf_hash(db->db_objset, db->db.db_object, in dbuf_cache_multilist_index_func()
722 db->db_level, db->db_blkid) % in dbuf_cache_multilist_index_func()
760 return (dbuf_cache_target - in dbuf_cache_lowater_bytes()
784 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { in dbuf_evict_one()
794 uint64_t size = db->db.db_size; in dbuf_evict_one()
795 uint64_t usize = dmu_buf_user_size(&db->db); in dbuf_evict_one()
799 &dbuf_caches[DB_DBUF_CACHE].size, usize, db->db_user); in dbuf_evict_one()
800 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_evict_one()
802 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size + usize); in dbuf_evict_one()
803 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE); in dbuf_evict_one()
804 db->db_caching_status = DB_NO_CACHE; in dbuf_evict_one()
891 dbuf_stats_t *ds = ksp->ks_data; in dbuf_kstat_update()
897 ds->cache_count.value.ui64 = in dbuf_kstat_update()
899 ds->cache_size_bytes.value.ui64 = in dbuf_kstat_update()
901 ds->cache_target_bytes.value.ui64 = dbuf_cache_target_bytes(); in dbuf_kstat_update()
902 ds->cache_hiwater_bytes.value.ui64 = dbuf_cache_hiwater_bytes(); in dbuf_kstat_update()
903 ds->cache_lowater_bytes.value.ui64 = dbuf_cache_lowater_bytes(); in dbuf_kstat_update()
904 ds->cache_total_evicts.value.ui64 = in dbuf_kstat_update()
907 ds->cache_levels[i].value.ui64 = in dbuf_kstat_update()
909 ds->cache_levels_bytes[i].value.ui64 = in dbuf_kstat_update()
912 ds->hash_hits.value.ui64 = in dbuf_kstat_update()
914 ds->hash_misses.value.ui64 = in dbuf_kstat_update()
916 ds->hash_collisions.value.ui64 = in dbuf_kstat_update()
918 ds->hash_elements.value.ui64 = in dbuf_kstat_update()
920 ds->hash_chains.value.ui64 = in dbuf_kstat_update()
922 ds->hash_insert_race.value.ui64 = in dbuf_kstat_update()
924 ds->hash_table_count.value.ui64 = h->hash_table_mask + 1; in dbuf_kstat_update()
925 ds->hash_mutex_count.value.ui64 = h->hash_mutex_mask + 1; in dbuf_kstat_update()
926 ds->metadata_cache_count.value.ui64 = in dbuf_kstat_update()
928 ds->metadata_cache_size_bytes.value.ui64 = zfs_refcount_count( in dbuf_kstat_update()
930 ds->metadata_cache_overflow.value.ui64 = in dbuf_kstat_update()
945 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers). in dbuf_init()
950 h->hash_table = NULL; in dbuf_init()
951 while (h->hash_table == NULL) { in dbuf_init()
952 h->hash_table_mask = hsize - 1; in dbuf_init()
954 h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_SLEEP); in dbuf_init()
955 if (h->hash_table == NULL) in dbuf_init()
971 h->hash_mutexes = NULL; in dbuf_init()
972 while (h->hash_mutexes == NULL) { in dbuf_init()
973 h->hash_mutex_mask = hmsize - 1; in dbuf_init()
975 h->hash_mutexes = vmem_zalloc(hmsize * sizeof (kmutex_t), in dbuf_init()
977 if (h->hash_mutexes == NULL) in dbuf_init()
988 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_NOLOCKDEP, NULL); in dbuf_init()
1041 dbuf_ksp->ks_data = &dbuf_stats; in dbuf_init()
1042 dbuf_ksp->ks_update = dbuf_kstat_update; in dbuf_init()
1054 for (int i = 0; i < (h->hash_mutex_mask + 1); i++) in dbuf_fini()
1055 mutex_destroy(&h->hash_mutexes[i]); in dbuf_fini()
1057 vmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); in dbuf_fini()
1058 vmem_free(h->hash_mutexes, (h->hash_mutex_mask + 1) * in dbuf_fini()
1114 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_verify()
1119 ASSERT(db->db_objset != NULL); in dbuf_verify()
1123 ASSERT(db->db_parent == NULL); in dbuf_verify()
1124 ASSERT(db->db_blkptr == NULL); in dbuf_verify()
1126 ASSERT3U(db->db.db_object, ==, dn->dn_object); in dbuf_verify()
1127 ASSERT3P(db->db_objset, ==, dn->dn_objset); in dbuf_verify()
1128 ASSERT3U(db->db_level, <, dn->dn_nlevels); in dbuf_verify()
1129 ASSERT(db->db_blkid == DMU_BONUS_BLKID || in dbuf_verify()
1130 db->db_blkid == DMU_SPILL_BLKID || in dbuf_verify()
1131 !avl_is_empty(&dn->dn_dbufs)); in dbuf_verify()
1133 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_verify()
1135 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_verify()
1136 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); in dbuf_verify()
1137 } else if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_verify()
1139 ASSERT0(db->db.db_offset); in dbuf_verify()
1141 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); in dbuf_verify()
1144 if ((dr = list_head(&db->db_dirty_records)) != NULL) { in dbuf_verify()
1145 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
1146 txg_prev = dr->dr_txg; in dbuf_verify()
1147 for (dr = list_next(&db->db_dirty_records, dr); dr != NULL; in dbuf_verify()
1148 dr = list_next(&db->db_dirty_records, dr)) { in dbuf_verify()
1149 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
1150 ASSERT(txg_prev > dr->dr_txg); in dbuf_verify()
1151 txg_prev = dr->dr_txg; in dbuf_verify()
1160 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_verify()
1161 dr = db->db_data_pending; in dbuf_verify()
1166 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); in dbuf_verify()
1169 /* verify db->db_blkptr */ in dbuf_verify()
1170 if (db->db_blkptr) { in dbuf_verify()
1171 if (db->db_parent == dn->dn_dbuf) { in dbuf_verify()
1173 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ in dbuf_verify()
1174 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) in dbuf_verify()
1175 ASSERT(db->db_parent == NULL); in dbuf_verify()
1177 ASSERT(db->db_parent != NULL); in dbuf_verify()
1178 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_verify()
1179 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
1180 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_verify()
1183 int epb __maybe_unused = db->db_parent->db.db_size >> in dbuf_verify()
1185 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); in dbuf_verify()
1186 ASSERT3U(db->db_parent->db.db_object, ==, in dbuf_verify()
1187 db->db.db_object); in dbuf_verify()
1188 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
1189 ((blkptr_t *)db->db_parent->db.db_data + in dbuf_verify()
1190 db->db_blkid % epb)); in dbuf_verify()
1193 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && in dbuf_verify()
1194 (db->db_buf == NULL || db->db_buf->b_data) && in dbuf_verify()
1195 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && in dbuf_verify()
1196 db->db_state != DB_FILL && (dn == NULL || !dn->dn_free_txg)) { in dbuf_verify()
1208 if (db->db_dirtycnt == 0) { in dbuf_verify()
1209 if (db->db_level == 0) { in dbuf_verify()
1210 uint64_t *buf = db->db.db_data; in dbuf_verify()
1213 for (i = 0; i < db->db.db_size >> 3; i++) { in dbuf_verify()
1217 blkptr_t *bps = db->db.db_data; in dbuf_verify()
1218 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, in dbuf_verify()
1219 db->db.db_size); in dbuf_verify()
1228 i < db->db.db_size / sizeof (blkptr_t); in dbuf_verify()
1232 &bp->blk_cksum)); in dbuf_verify()
1234 DVA_IS_EMPTY(&bp->blk_dva[0]) && in dbuf_verify()
1235 DVA_IS_EMPTY(&bp->blk_dva[1]) && in dbuf_verify()
1236 DVA_IS_EMPTY(&bp->blk_dva[2])); in dbuf_verify()
1237 ASSERT0(bp->blk_fill); in dbuf_verify()
1238 ASSERT0(bp->blk_pad[0]); in dbuf_verify()
1239 ASSERT0(bp->blk_pad[1]); in dbuf_verify()
1254 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_clear_data()
1256 ASSERT3P(db->db_buf, ==, NULL); in dbuf_clear_data()
1257 db->db.db_data = NULL; in dbuf_clear_data()
1258 if (db->db_state != DB_NOFILL) { in dbuf_clear_data()
1259 db->db_state = DB_UNCACHED; in dbuf_clear_data()
1267 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_set_data()
1270 db->db_buf = buf; in dbuf_set_data()
1271 ASSERT(buf->b_data != NULL); in dbuf_set_data()
1272 db->db.db_data = buf->b_data; in dbuf_set_data()
1278 spa_t *spa = db->db_objset->os_spa; in dbuf_alloc_arcbuf()
1280 return (arc_alloc_buf(spa, db, DBUF_GET_BUFC_TYPE(db), db->db.db_size)); in dbuf_alloc_arcbuf()
1290 if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { in dbuf_whichblock()
1300 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level = in dbuf_whichblock()
1301 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)). in dbuf_whichblock()
1304 * ((2^datablkshift)*(2^(level*(indblkshift-SPA_BLKPTRSHIFT)))) in dbuf_whichblock()
1306 * (indblkshift - SPA_BLKPTRSHIFT)) in dbuf_whichblock()
1308 * (indblkshift - SPA_BLKPTRSHIFT)) in dbuf_whichblock()
1311 const unsigned exp = dn->dn_datablkshift + in dbuf_whichblock()
1312 level * (dn->dn_indblkshift - SPA_BLKPTRSHIFT); in dbuf_whichblock()
1316 ASSERT3U(level, ==, dn->dn_nlevels - 1); in dbuf_whichblock()
1324 ASSERT3U(offset, <, dn->dn_datablksz); in dbuf_whichblock()
1337 if (db->db_parent != NULL) { in dmu_buf_lock_parent()
1338 rw_enter(&db->db_parent->db_rwlock, rw); in dmu_buf_lock_parent()
1340 } else if (dmu_objset_ds(db->db_objset) != NULL) { in dmu_buf_lock_parent()
1341 rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw, in dmu_buf_lock_parent()
1346 * We only return a DLT_NONE lock when it's the top-most indirect block in dmu_buf_lock_parent()
1347 * of the meta-dnode of the MOS. in dmu_buf_lock_parent()
1355 * parent) to not the top-most via an indirection increase. This would cause a
1362 rw_exit(&db->db_parent->db_rwlock); in dmu_buf_unlock_parent()
1364 rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag); in dmu_buf_unlock_parent()
1374 mutex_enter(&db->db_mtx); in dbuf_read_done()
1375 ASSERT3U(db->db_state, ==, DB_READ); in dbuf_read_done()
1380 ASSERT(zfs_refcount_count(&db->db_holds) > 0); in dbuf_read_done()
1381 ASSERT(db->db_buf == NULL); in dbuf_read_done()
1382 ASSERT(db->db.db_data == NULL); in dbuf_read_done()
1385 ASSERT(zio == NULL || zio->io_error != 0); in dbuf_read_done()
1386 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_read_done()
1387 ASSERT3P(db->db_buf, ==, NULL); in dbuf_read_done()
1388 db->db_state = DB_UNCACHED; in dbuf_read_done()
1390 } else if (db->db_level == 0 && db->db_freed_in_flight) { in dbuf_read_done()
1392 ASSERT(zio == NULL || zio->io_error == 0); in dbuf_read_done()
1394 memset(buf->b_data, 0, db->db.db_size); in dbuf_read_done()
1396 db->db_freed_in_flight = FALSE; in dbuf_read_done()
1398 db->db_state = DB_CACHED; in dbuf_read_done()
1402 ASSERT(zio == NULL || zio->io_error == 0); in dbuf_read_done()
1404 db->db_state = DB_CACHED; in dbuf_read_done()
1407 cv_broadcast(&db->db_changed); in dbuf_read_done()
1422 bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); in dbuf_read_bonus()
1423 max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_read_bonus()
1424 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_bonus()
1426 ASSERT3U(bonuslen, <=, db->db.db_size); in dbuf_read_bonus()
1432 memcpy(db_data, DN_BONUS(dn->dn_phys), bonuslen); in dbuf_read_bonus()
1433 db->db.db_data = db_data; in dbuf_read_bonus()
1434 db->db_state = DB_CACHED; in dbuf_read_bonus()
1443 uint32_t indbs = 1ULL << dn->dn_indblkshift; in dbuf_handle_indirect_hole()
1451 dn->dn_datablksz : BP_GET_LSIZE(dbbp)); in dbuf_handle_indirect_hole()
1453 BP_SET_LEVEL(bp, BP_GET_LEVEL(dbbp) - 1); in dbuf_handle_indirect_hole()
1466 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_hole()
1476 if (!is_hole && db->db_level == 0) in dbuf_read_hole()
1477 is_hole = dnode_block_freed(dn, db->db_blkid) || BP_IS_HOLE(bp); in dbuf_read_hole()
1481 memset(db_data->b_data, 0, db->db.db_size); in dbuf_read_hole()
1483 if (bp != NULL && db->db_level > 0 && BP_IS_HOLE(bp) && in dbuf_read_hole()
1485 dbuf_handle_indirect_hole(db_data->b_data, dn, bp); in dbuf_read_hole()
1488 db->db_state = DB_CACHED; in dbuf_read_hole()
1498 * this so that we ensure we are fully authenticating the checksum-of-MACs
1500 * always verified against their secure checksum-of-MACs assuming that the
1511 objset_t *os = db->db_objset; in dbuf_read_verify_dnode_crypt()
1518 !os->os_encrypted || os->os_raw_receive || in dbuf_read_verify_dnode_crypt()
1519 (dndb = dn->dn_dbuf) == NULL) in dbuf_read_verify_dnode_crypt()
1522 dnbuf = dndb->db_buf; in dbuf_read_verify_dnode_crypt()
1526 mutex_enter(&dndb->db_mtx); in dbuf_read_verify_dnode_crypt()
1533 * plain-text reads due to txg wait, but better be safe than sorry. in dbuf_read_verify_dnode_crypt()
1537 mutex_exit(&dndb->db_mtx); in dbuf_read_verify_dnode_crypt()
1540 dbuf_dirty_record_t *dr = dndb->db_data_pending; in dbuf_read_verify_dnode_crypt()
1541 if (dr == NULL || dr->dt.dl.dr_data != dnbuf) in dbuf_read_verify_dnode_crypt()
1543 cv_wait(&dndb->db_changed, &dndb->db_mtx); in dbuf_read_verify_dnode_crypt()
1547 DMU_META_DNODE_OBJECT, 0, dndb->db_blkid); in dbuf_read_verify_dnode_crypt()
1548 err = arc_untransform(dnbuf, os->os_spa, &zb, B_TRUE); in dbuf_read_verify_dnode_crypt()
1553 * (and therefore non-encrypted) blocks. in dbuf_read_verify_dnode_crypt()
1555 if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1556 !DMU_OT_IS_ENCRYPTED(dn->dn_type)) || in dbuf_read_verify_dnode_crypt()
1557 (db->db_blkid == DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1558 !DMU_OT_IS_ENCRYPTED(dn->dn_bonustype)))) in dbuf_read_verify_dnode_crypt()
1561 mutex_exit(&dndb->db_mtx); in dbuf_read_verify_dnode_crypt()
1578 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read_impl()
1579 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_impl()
1580 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_read_impl()
1581 ASSERT(db->db_buf == NULL); in dbuf_read_impl()
1582 ASSERT(db->db_parent == NULL || in dbuf_read_impl()
1583 RW_LOCK_HELD(&db->db_parent->db_rwlock)); in dbuf_read_impl()
1585 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_read_impl()
1603 db->db_objset->os_dsl_dataset, in dbuf_read_impl()
1609 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read_impl()
1610 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read_impl()
1616 if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bp)) { in dbuf_read_impl()
1617 spa_log_error(db->db_objset->os_spa, &zb, in dbuf_read_impl()
1623 db->db_state = DB_READ; in dbuf_read_impl()
1625 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1649 return (arc_read(zio, db->db_objset->os_spa, ©, in dbuf_read_impl()
1654 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1660 * This is our just-in-time copy function. It makes a copy of buffers that
1675 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dbuf_fix_old_data()
1677 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_fix_old_data()
1678 ASSERT(db->db.db_data != NULL); in dbuf_fix_old_data()
1679 ASSERT(db->db_level == 0); in dbuf_fix_old_data()
1680 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dbuf_fix_old_data()
1683 (dr->dt.dl.dr_data != in dbuf_fix_old_data()
1684 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) in dbuf_fix_old_data()
1694 ASSERT3U(dr->dr_txg, >=, txg - 2); in dbuf_fix_old_data()
1695 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_fix_old_data()
1697 int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_fix_old_data()
1698 dr->dt.dl.dr_data = kmem_alloc(bonuslen, KM_SLEEP); in dbuf_fix_old_data()
1700 memcpy(dr->dt.dl.dr_data, db->db.db_data, bonuslen); in dbuf_fix_old_data()
1701 } else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) { in dbuf_fix_old_data()
1703 int size = arc_buf_size(db->db_buf); in dbuf_fix_old_data()
1705 spa_t *spa = db->db_objset->os_spa; in dbuf_fix_old_data()
1707 arc_get_compression(db->db_buf); in dbuf_fix_old_data()
1708 uint8_t complevel = arc_get_complevel(db->db_buf); in dbuf_fix_old_data()
1710 if (arc_is_encrypted(db->db_buf)) { in dbuf_fix_old_data()
1716 arc_get_raw_params(db->db_buf, &byteorder, salt, in dbuf_fix_old_data()
1718 dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db, in dbuf_fix_old_data()
1719 dmu_objset_id(dn->dn_objset), byteorder, salt, iv, in dbuf_fix_old_data()
1720 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf), in dbuf_fix_old_data()
1724 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db, in dbuf_fix_old_data()
1725 size, arc_buf_lsize(db->db_buf), compress_type, in dbuf_fix_old_data()
1728 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size); in dbuf_fix_old_data()
1730 memcpy(dr->dt.dl.dr_data->b_data, db->db.db_data, size); in dbuf_fix_old_data()
1732 db->db_buf = NULL; in dbuf_fix_old_data()
1744 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read()
1757 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_read()
1760 mutex_enter(&db->db_mtx); in dbuf_read()
1762 db->db_pending_evict = B_FALSE; in dbuf_read()
1764 db->db_partial_read = B_TRUE; in dbuf_read()
1766 db->db_partial_read = B_FALSE; in dbuf_read()
1767 miss = (db->db_state != DB_CACHED); in dbuf_read()
1769 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_read()
1779 mutex_exit(&db->db_mtx); in dbuf_read()
1784 ASSERT(db->db_state == DB_READ || in dbuf_read()
1788 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_read()
1789 } while (db->db_state == DB_READ || db->db_state == DB_FILL); in dbuf_read()
1790 if (db->db_state == DB_UNCACHED) { in dbuf_read()
1792 mutex_exit(&db->db_mtx); in dbuf_read()
1798 if (db->db_state == DB_CACHED) { in dbuf_read()
1806 if ((flags & DMU_READ_NO_DECRYPT) == 0 && db->db_buf != NULL && in dbuf_read()
1807 (arc_is_encrypted(db->db_buf) || in dbuf_read()
1808 arc_is_unauthenticated(db->db_buf) || in dbuf_read()
1809 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dbuf_read()
1810 spa_t *spa = dn->dn_objset->os_spa; in dbuf_read()
1813 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read()
1814 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read()
1816 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dbuf_read()
1817 dbuf_set_data(db, db->db_buf); in dbuf_read()
1819 mutex_exit(&db->db_mtx); in dbuf_read()
1821 ASSERT(db->db_state == DB_UNCACHED || in dbuf_read()
1822 db->db_state == DB_NOFILL); in dbuf_read()
1834 if (pio == NULL && (db->db_state == DB_NOFILL || in dbuf_read()
1836 spa_t *spa = dn->dn_objset->os_spa; in dbuf_read()
1845 mutex_exit(&db->db_mtx); in dbuf_read()
1849 miss = (db->db_state != DB_CACHED); in dbuf_read()
1853 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, miss, in dbuf_read()
1855 db->db_pending_evict); in dbuf_read()
1877 zio_t *zio = zio_null(pio, pio->io_spa, NULL, NULL, NULL, in dbuf_read()
1879 zio->io_error = err; in dbuf_read()
1889 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_noread()
1890 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_noread()
1891 mutex_enter(&db->db_mtx); in dbuf_noread()
1893 db->db_pending_evict = B_FALSE; in dbuf_noread()
1894 db->db_partial_read = B_FALSE; in dbuf_noread()
1895 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_noread()
1896 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_noread()
1897 if (db->db_state == DB_UNCACHED) { in dbuf_noread()
1898 ASSERT(db->db_buf == NULL); in dbuf_noread()
1899 ASSERT(db->db.db_data == NULL); in dbuf_noread()
1901 db->db_state = DB_FILL; in dbuf_noread()
1903 } else if (db->db_state == DB_NOFILL) { in dbuf_noread()
1906 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_noread()
1908 mutex_exit(&db->db_mtx); in dbuf_noread()
1914 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_unoverride()
1915 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; in dbuf_unoverride()
1916 uint64_t txg = dr->dr_txg; in dbuf_unoverride()
1918 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_unoverride()
1925 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); in dbuf_unoverride()
1926 ASSERT(db->db_level == 0); in dbuf_unoverride()
1928 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_unoverride()
1929 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) in dbuf_unoverride()
1932 ASSERT(db->db_data_pending != dr); in dbuf_unoverride()
1935 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) in dbuf_unoverride()
1936 zio_free(db->db_objset->os_spa, txg, bp); in dbuf_unoverride()
1938 if (dr->dt.dl.dr_brtwrite || dr->dt.dl.dr_diowrite) { in dbuf_unoverride()
1939 ASSERT0P(dr->dt.dl.dr_data); in dbuf_unoverride()
1940 dr->dt.dl.dr_data = db->db_buf; in dbuf_unoverride()
1942 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; in dbuf_unoverride()
1943 dr->dt.dl.dr_nopwrite = B_FALSE; in dbuf_unoverride()
1944 dr->dt.dl.dr_brtwrite = B_FALSE; in dbuf_unoverride()
1945 dr->dt.dl.dr_diowrite = B_FALSE; in dbuf_unoverride()
1946 dr->dt.dl.dr_has_raw_params = B_FALSE; in dbuf_unoverride()
1952 * Release the already-written buffer, so we leave it in in dbuf_unoverride()
1957 * immediately re-thawing it. in dbuf_unoverride()
1959 if (dr->dt.dl.dr_data) in dbuf_unoverride()
1960 arc_release(dr->dt.dl.dr_data, db); in dbuf_unoverride()
1964 * Evict (if its unreferenced) or clear (if its referenced) any level-0
1974 uint64_t txg = tx->tx_txg; in dbuf_free_range()
1978 if (end_blkid > dn->dn_maxblkid && in dbuf_free_range()
1980 end_blkid = dn->dn_maxblkid; in dbuf_free_range()
1985 db_search->db_level = 0; in dbuf_free_range()
1986 db_search->db_blkid = start_blkid; in dbuf_free_range()
1987 db_search->db_state = DB_SEARCH; in dbuf_free_range()
1989 mutex_enter(&dn->dn_dbufs_mtx); in dbuf_free_range()
1990 db = avl_find(&dn->dn_dbufs, db_search, &where); in dbuf_free_range()
1993 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dbuf_free_range()
1996 db_next = AVL_NEXT(&dn->dn_dbufs, db); in dbuf_free_range()
1997 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_free_range()
1999 if (db->db_level != 0 || db->db_blkid > end_blkid) { in dbuf_free_range()
2002 ASSERT3U(db->db_blkid, >=, start_blkid); in dbuf_free_range()
2005 mutex_enter(&db->db_mtx); in dbuf_free_range()
2011 if (db->db_state == DB_UNCACHED || in dbuf_free_range()
2012 db->db_state == DB_NOFILL || in dbuf_free_range()
2013 db->db_state == DB_EVICTING) { in dbuf_free_range()
2014 ASSERT(db->db.db_data == NULL); in dbuf_free_range()
2015 mutex_exit(&db->db_mtx); in dbuf_free_range()
2018 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_free_range()
2020 db->db_freed_in_flight = TRUE; in dbuf_free_range()
2021 mutex_exit(&db->db_mtx); in dbuf_free_range()
2024 if (zfs_refcount_count(&db->db_holds) == 0) { in dbuf_free_range()
2025 ASSERT(db->db_buf); in dbuf_free_range()
2031 dr = list_head(&db->db_dirty_records); in dbuf_free_range()
2033 if (dr->dr_txg == txg) { in dbuf_free_range()
2035 * This buffer is "in-use", re-adjust the file in dbuf_free_range()
2039 if (db->db_blkid != DMU_SPILL_BLKID && in dbuf_free_range()
2040 db->db_blkid > dn->dn_maxblkid) in dbuf_free_range()
2041 dn->dn_maxblkid = db->db_blkid; in dbuf_free_range()
2054 if (db->db_state == DB_CACHED) { in dbuf_free_range()
2055 ASSERT(db->db.db_data != NULL); in dbuf_free_range()
2056 arc_release(db->db_buf, db); in dbuf_free_range()
2057 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_free_range()
2058 memset(db->db.db_data, 0, db->db.db_size); in dbuf_free_range()
2059 rw_exit(&db->db_rwlock); in dbuf_free_range()
2060 arc_buf_freeze(db->db_buf); in dbuf_free_range()
2063 mutex_exit(&db->db_mtx); in dbuf_free_range()
2066 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_free_range()
2075 int osize = db->db.db_size; in dbuf_new_size()
2079 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_new_size()
2088 dmu_buf_will_dirty(&db->db, tx); in dbuf_new_size()
2090 VERIFY3P(db->db_buf, !=, NULL); in dbuf_new_size()
2093 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); in dbuf_new_size()
2096 old_buf = db->db_buf; in dbuf_new_size()
2097 memcpy(buf->b_data, old_buf->b_data, MIN(osize, size)); in dbuf_new_size()
2100 memset((uint8_t *)buf->b_data + osize, 0, size - osize); in dbuf_new_size()
2102 mutex_enter(&db->db_mtx); in dbuf_new_size()
2105 db->db.db_size = size; in dbuf_new_size()
2107 dr = list_head(&db->db_dirty_records); in dbuf_new_size()
2110 if (db->db_level == 0) in dbuf_new_size()
2111 dr->dt.dl.dr_data = buf; in dbuf_new_size()
2112 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); in dbuf_new_size()
2113 ASSERT3U(dr->dr_accounted, ==, osize); in dbuf_new_size()
2114 dr->dr_accounted = size; in dbuf_new_size()
2115 mutex_exit(&db->db_mtx); in dbuf_new_size()
2117 dmu_objset_willuse_space(dn->dn_objset, size - osize, tx); in dbuf_new_size()
2124 objset_t *os __maybe_unused = db->db_objset; in dbuf_release_bp()
2127 ASSERT(arc_released(os->os_phys_buf) || in dbuf_release_bp()
2128 list_link_active(&os->os_dsl_dataset->ds_synced_link)); in dbuf_release_bp()
2129 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); in dbuf_release_bp()
2131 (void) arc_release(db->db_buf, db); in dbuf_release_bp()
2141 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_redirty()
2143 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_redirty()
2145 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { in dbuf_redirty()
2151 if (db->db.db_object != DMU_META_DNODE_OBJECT && in dbuf_redirty()
2152 db->db_state != DB_NOFILL) { in dbuf_redirty()
2154 ASSERT(arc_released(db->db_buf)); in dbuf_redirty()
2155 arc_buf_thaw(db->db_buf); in dbuf_redirty()
2163 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_dirty_lightweight()
2164 IMPLY(dn->dn_objset->os_raw_receive, dn->dn_maxblkid >= blkid); in dbuf_dirty_lightweight()
2166 ASSERT(dn->dn_maxblkid >= blkid); in dbuf_dirty_lightweight()
2169 list_link_init(&dr->dr_dirty_node); in dbuf_dirty_lightweight()
2170 list_link_init(&dr->dr_dbuf_node); in dbuf_dirty_lightweight()
2171 dr->dr_dnode = dn; in dbuf_dirty_lightweight()
2172 dr->dr_txg = tx->tx_txg; in dbuf_dirty_lightweight()
2173 dr->dt.dll.dr_blkid = blkid; in dbuf_dirty_lightweight()
2174 dr->dr_accounted = dn->dn_datablksz; in dbuf_dirty_lightweight()
2181 ASSERT3P(NULL, ==, dbuf_find(dn->dn_objset, dn->dn_object, 0, blkid, in dbuf_dirty_lightweight()
2184 mutex_enter(&dn->dn_mtx); in dbuf_dirty_lightweight()
2185 int txgoff = tx->tx_txg & TXG_MASK; in dbuf_dirty_lightweight()
2186 if (dn->dn_free_ranges[txgoff] != NULL) { in dbuf_dirty_lightweight()
2187 zfs_range_tree_clear(dn->dn_free_ranges[txgoff], blkid, 1); in dbuf_dirty_lightweight()
2190 if (dn->dn_nlevels == 1) { in dbuf_dirty_lightweight()
2191 ASSERT3U(blkid, <, dn->dn_nblkptr); in dbuf_dirty_lightweight()
2192 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); in dbuf_dirty_lightweight()
2193 mutex_exit(&dn->dn_mtx); in dbuf_dirty_lightweight()
2194 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty_lightweight()
2197 mutex_exit(&dn->dn_mtx); in dbuf_dirty_lightweight()
2199 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_dirty_lightweight()
2202 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty_lightweight()
2217 mutex_enter(&parent_dr->dt.di.dr_mtx); in dbuf_dirty_lightweight()
2218 ASSERT3U(parent_dr->dr_txg, ==, tx->tx_txg); in dbuf_dirty_lightweight()
2219 list_insert_tail(&parent_dr->dt.di.dr_children, dr); in dbuf_dirty_lightweight()
2220 mutex_exit(&parent_dr->dt.di.dr_mtx); in dbuf_dirty_lightweight()
2221 dr->dr_parent = parent_dr; in dbuf_dirty_lightweight()
2224 dmu_objset_willuse_space(dn->dn_objset, dr->dr_accounted, tx); in dbuf_dirty_lightweight()
2235 int txgoff = tx->tx_txg & TXG_MASK; in dbuf_dirty()
2238 ASSERT(tx->tx_txg != 0); in dbuf_dirty()
2239 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_dirty()
2247 * were already pre-dirtied in open context. in dbuf_dirty()
2250 if (dn->dn_objset->os_dsl_dataset != NULL) { in dbuf_dirty()
2251 rrw_enter(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, in dbuf_dirty()
2255 BP_IS_HOLE(dn->dn_objset->os_rootbp) || in dbuf_dirty()
2256 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || in dbuf_dirty()
2257 dn->dn_objset->os_dsl_dataset == NULL); in dbuf_dirty()
2258 if (dn->dn_objset->os_dsl_dataset != NULL) in dbuf_dirty()
2259 rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, FTAG); in dbuf_dirty()
2263 * check if we're already dirty. They are allowed to re-dirty in dbuf_dirty()
2266 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || in dbuf_dirty()
2267 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == in dbuf_dirty()
2270 mutex_enter(&db->db_mtx); in dbuf_dirty()
2276 ASSERT(db->db_level != 0 || in dbuf_dirty()
2277 db->db_state == DB_CACHED || db->db_state == DB_FILL || in dbuf_dirty()
2278 db->db_state == DB_NOFILL); in dbuf_dirty()
2280 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2282 if (tx->tx_txg > dn->dn_dirty_txg) in dbuf_dirty()
2283 dn->dn_dirty_txg = tx->tx_txg; in dbuf_dirty()
2284 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2286 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_dirty()
2287 dn->dn_have_spill = B_TRUE; in dbuf_dirty()
2292 dr_head = list_head(&db->db_dirty_records); in dbuf_dirty()
2293 ASSERT(dr_head == NULL || dr_head->dr_txg <= tx->tx_txg || in dbuf_dirty()
2294 db->db.db_object == DMU_META_DNODE_OBJECT); in dbuf_dirty()
2295 dr_next = dbuf_find_dirty_lte(db, tx->tx_txg); in dbuf_dirty()
2296 if (dr_next && dr_next->dr_txg == tx->tx_txg) { in dbuf_dirty()
2300 mutex_exit(&db->db_mtx); in dbuf_dirty()
2307 ASSERT(dn->dn_object == 0 || in dbuf_dirty()
2308 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == in dbuf_dirty()
2311 ASSERT3U(dn->dn_nlevels, >, db->db_level); in dbuf_dirty()
2320 os = dn->dn_objset; in dbuf_dirty()
2321 VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(os->os_spa)); in dbuf_dirty()
2323 if (dn->dn_objset->os_dsl_dataset != NULL) in dbuf_dirty()
2324 rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_READER, FTAG); in dbuf_dirty()
2325 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || in dbuf_dirty()
2326 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); in dbuf_dirty()
2327 if (dn->dn_objset->os_dsl_dataset != NULL) in dbuf_dirty()
2328 rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG); in dbuf_dirty()
2330 ASSERT(db->db.db_size != 0); in dbuf_dirty()
2332 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_dirty()
2334 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) { in dbuf_dirty()
2335 dmu_objset_willuse_space(os, db->db.db_size, tx); in dbuf_dirty()
2345 list_link_init(&dr->dr_dirty_node); in dbuf_dirty()
2346 list_link_init(&dr->dr_dbuf_node); in dbuf_dirty()
2347 dr->dr_dnode = dn; in dbuf_dirty()
2348 if (db->db_level == 0) { in dbuf_dirty()
2349 void *data_old = db->db_buf; in dbuf_dirty()
2351 if (db->db_state != DB_NOFILL) { in dbuf_dirty()
2352 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_dirty()
2353 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
2354 data_old = db->db.db_data; in dbuf_dirty()
2355 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { in dbuf_dirty()
2365 arc_release(db->db_buf, db); in dbuf_dirty()
2366 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
2367 data_old = db->db_buf; in dbuf_dirty()
2371 dr->dt.dl.dr_data = data_old; in dbuf_dirty()
2373 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_NOLOCKDEP, NULL); in dbuf_dirty()
2374 list_create(&dr->dt.di.dr_children, in dbuf_dirty()
2378 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) { in dbuf_dirty()
2379 dr->dr_accounted = db->db.db_size; in dbuf_dirty()
2381 dr->dr_dbuf = db; in dbuf_dirty()
2382 dr->dr_txg = tx->tx_txg; in dbuf_dirty()
2383 list_insert_before(&db->db_dirty_records, dr_next, dr); in dbuf_dirty()
2390 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_dirty()
2391 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_dirty()
2392 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2393 if (dn->dn_free_ranges[txgoff] != NULL) { in dbuf_dirty()
2394 zfs_range_tree_clear(dn->dn_free_ranges[txgoff], in dbuf_dirty()
2395 db->db_blkid, 1); in dbuf_dirty()
2397 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2398 db->db_freed_in_flight = FALSE; in dbuf_dirty()
2404 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); in dbuf_dirty()
2405 db->db_dirtycnt += 1; in dbuf_dirty()
2406 ASSERT3U(db->db_dirtycnt, <=, 3); in dbuf_dirty()
2408 mutex_exit(&db->db_mtx); in dbuf_dirty()
2410 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_dirty()
2411 db->db_blkid == DMU_SPILL_BLKID) { in dbuf_dirty()
2412 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2413 ASSERT(!list_link_active(&dr->dr_dirty_node)); in dbuf_dirty()
2414 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); in dbuf_dirty()
2415 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2421 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { in dbuf_dirty()
2422 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_dirty()
2432 if (db->db_blkptr != NULL) { in dbuf_dirty()
2434 ddt_prefetch(os->os_spa, db->db_blkptr); in dbuf_dirty()
2442 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || in dbuf_dirty()
2443 dn->dn_phys->dn_nlevels > db->db_level || in dbuf_dirty()
2444 dn->dn_next_nlevels[txgoff] > db->db_level || in dbuf_dirty()
2445 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || in dbuf_dirty()
2446 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); in dbuf_dirty()
2449 if (db->db_level == 0) { in dbuf_dirty()
2450 ASSERT(!db->db_objset->os_raw_receive || in dbuf_dirty()
2451 dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2452 dnode_new_blkid(dn, db->db_blkid, tx, in dbuf_dirty()
2454 ASSERT(dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2457 if (db->db_level+1 < dn->dn_nlevels) { in dbuf_dirty()
2458 dmu_buf_impl_t *parent = db->db_parent; in dbuf_dirty()
2462 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { in dbuf_dirty()
2463 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_dirty()
2464 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_dirty()
2465 db->db_blkid >> epbs, FTAG); in dbuf_dirty()
2470 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty()
2471 ASSERT3U(db->db_level + 1, ==, parent->db_level); in dbuf_dirty()
2476 mutex_enter(&db->db_mtx); in dbuf_dirty()
2481 if (list_head(&db->db_dirty_records) == dr || in dbuf_dirty()
2482 dn->dn_object == DMU_META_DNODE_OBJECT) { in dbuf_dirty()
2483 mutex_enter(&di->dt.di.dr_mtx); in dbuf_dirty()
2484 ASSERT3U(di->dr_txg, ==, tx->tx_txg); in dbuf_dirty()
2485 ASSERT(!list_link_active(&dr->dr_dirty_node)); in dbuf_dirty()
2486 list_insert_tail(&di->dt.di.dr_children, dr); in dbuf_dirty()
2487 mutex_exit(&di->dt.di.dr_mtx); in dbuf_dirty()
2488 dr->dr_parent = di; in dbuf_dirty()
2490 mutex_exit(&db->db_mtx); in dbuf_dirty()
2492 ASSERT(db->db_level + 1 == dn->dn_nlevels); in dbuf_dirty()
2493 ASSERT(db->db_blkid < dn->dn_nblkptr); in dbuf_dirty()
2494 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); in dbuf_dirty()
2495 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2496 ASSERT(!list_link_active(&dr->dr_dirty_node)); in dbuf_dirty()
2497 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); in dbuf_dirty()
2498 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2500 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty()
2511 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_undirty_bonus()
2513 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_undirty_bonus()
2514 if (dr->dt.dl.dr_data != db->db.db_data) { in dbuf_undirty_bonus()
2515 struct dnode *dn = dr->dr_dnode; in dbuf_undirty_bonus()
2516 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_undirty_bonus()
2518 kmem_free(dr->dt.dl.dr_data, max_bonuslen); in dbuf_undirty_bonus()
2521 db->db_data_pending = NULL; in dbuf_undirty_bonus()
2522 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); in dbuf_undirty_bonus()
2523 list_remove(&db->db_dirty_records, dr); in dbuf_undirty_bonus()
2524 if (dr->dr_dbuf->db_level != 0) { in dbuf_undirty_bonus()
2525 mutex_destroy(&dr->dt.di.dr_mtx); in dbuf_undirty_bonus()
2526 list_destroy(&dr->dt.di.dr_children); in dbuf_undirty_bonus()
2529 ASSERT3U(db->db_dirtycnt, >, 0); in dbuf_undirty_bonus()
2530 db->db_dirtycnt -= 1; in dbuf_undirty_bonus()
2540 uint64_t txg = tx->tx_txg; in dbuf_undirty()
2552 ASSERT(db->db_objset == in dbuf_undirty()
2553 dmu_objset_pool(db->db_objset)->dp_meta_objset || in dbuf_undirty()
2554 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); in dbuf_undirty()
2555 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_undirty()
2556 ASSERT0(db->db_level); in dbuf_undirty()
2557 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_undirty()
2565 ASSERT(dr->dr_dbuf == db); in dbuf_undirty()
2567 brtwrite = dr->dt.dl.dr_brtwrite; in dbuf_undirty()
2568 diowrite = dr->dt.dl.dr_diowrite; in dbuf_undirty()
2575 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; in dbuf_undirty()
2577 brt_pending_remove(dmu_objset_spa(db->db_objset), in dbuf_undirty()
2582 dnode_t *dn = dr->dr_dnode; in dbuf_undirty()
2584 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_undirty()
2586 ASSERT(db->db.db_size != 0); in dbuf_undirty()
2588 dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), in dbuf_undirty()
2589 dr->dr_accounted, txg); in dbuf_undirty()
2591 list_remove(&db->db_dirty_records, dr); in dbuf_undirty()
2599 if (dr->dr_parent) { in dbuf_undirty()
2600 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); in dbuf_undirty()
2601 list_remove(&dr->dr_parent->dt.di.dr_children, dr); in dbuf_undirty()
2602 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); in dbuf_undirty()
2603 } else if (db->db_blkid == DMU_SPILL_BLKID || in dbuf_undirty()
2604 db->db_level + 1 == dn->dn_nlevels) { in dbuf_undirty()
2605 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); in dbuf_undirty()
2606 mutex_enter(&dn->dn_mtx); in dbuf_undirty()
2607 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); in dbuf_undirty()
2608 mutex_exit(&dn->dn_mtx); in dbuf_undirty()
2611 if (db->db_state != DB_NOFILL && !brtwrite) { in dbuf_undirty()
2614 if (dr->dt.dl.dr_data != db->db_buf) { in dbuf_undirty()
2615 ASSERT(db->db_buf != NULL); in dbuf_undirty()
2616 ASSERT(dr->dt.dl.dr_data != NULL); in dbuf_undirty()
2617 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_undirty()
2623 ASSERT(db->db_dirtycnt > 0); in dbuf_undirty()
2624 db->db_dirtycnt -= 1; in dbuf_undirty()
2626 if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { in dbuf_undirty()
2627 ASSERT(db->db_state == DB_NOFILL || brtwrite || diowrite || in dbuf_undirty()
2628 arc_released(db->db_buf)); in dbuf_undirty()
2642 ASSERT(tx->tx_txg != 0); in dmu_buf_will_dirty_flags()
2643 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_dirty_flags()
2649 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_flags()
2650 if (db->db_state == DB_CACHED || db->db_state == DB_NOFILL) { in dmu_buf_will_dirty_flags()
2656 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_will_dirty_flags()
2658 if (db->db_level == 0 && in dmu_buf_will_dirty_flags()
2659 dr->dt.dl.dr_brtwrite) { in dmu_buf_will_dirty_flags()
2671 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_flags()
2676 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_flags()
2679 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) in dmu_buf_will_dirty_flags()
2691 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_flags()
2693 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_flags()
2710 mutex_enter(&db->db_mtx); in dmu_buf_is_dirty()
2711 dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_is_dirty()
2712 mutex_exit(&db->db_mtx); in dmu_buf_is_dirty()
2717 * Normally the db_blkptr points to the most recent on-disk content for the
2725 ASSERT(MUTEX_HELD(&db->db_mtx)); in dmu_buf_get_bp_from_dbuf()
2728 if (db->db_level != 0) { in dmu_buf_get_bp_from_dbuf()
2729 *bp = db->db_blkptr; in dmu_buf_get_bp_from_dbuf()
2733 *bp = db->db_blkptr; in dmu_buf_get_bp_from_dbuf()
2734 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dmu_buf_get_bp_from_dbuf()
2735 if (dr && db->db_state == DB_NOFILL) { in dmu_buf_get_bp_from_dbuf()
2737 if (!dr->dt.dl.dr_brtwrite) in dmu_buf_get_bp_from_dbuf()
2740 *bp = &dr->dt.dl.dr_overridden_by; in dmu_buf_get_bp_from_dbuf()
2741 } else if (dr && db->db_state == DB_UNCACHED) { in dmu_buf_get_bp_from_dbuf()
2743 if (dr->dt.dl.dr_diowrite) in dmu_buf_get_bp_from_dbuf()
2744 *bp = &dr->dt.dl.dr_overridden_by; in dmu_buf_get_bp_from_dbuf()
2761 ASSERT3S(db->db_state, ==, DB_CACHED); in dmu_buf_untransform_direct()
2762 ASSERT(MUTEX_HELD(&db->db_mtx)); in dmu_buf_untransform_direct()
2777 if (err == 0 && db->db_buf != NULL && in dmu_buf_untransform_direct()
2778 (arc_is_encrypted(db->db_buf) || in dmu_buf_untransform_direct()
2779 arc_is_unauthenticated(db->db_buf) || in dmu_buf_untransform_direct()
2780 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dmu_buf_untransform_direct()
2783 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dmu_buf_untransform_direct()
2784 db->db.db_object, db->db_level, db->db_blkid); in dmu_buf_untransform_direct()
2786 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dmu_buf_untransform_direct()
2787 dbuf_set_data(db, db->db_buf); in dmu_buf_untransform_direct()
2799 * Block clones and Direct I/O writes always happen in open-context. in dmu_buf_will_clone_or_dio()
2802 ASSERT0(db->db_level); in dmu_buf_will_clone_or_dio()
2804 ASSERT0(db->db_level); in dmu_buf_will_clone_or_dio()
2805 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_clone_or_dio()
2806 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dmu_buf_will_clone_or_dio()
2808 mutex_enter(&db->db_mtx); in dmu_buf_will_clone_or_dio()
2818 * to go ahead free up the space accounting through dbuf_undirty() -> in dmu_buf_will_clone_or_dio()
2819 * dbuf_unoverride() -> zio_free(). Space accountiung for determining in dmu_buf_will_clone_or_dio()
2823 * done in open-context. Constantly allowing Direct I/O overwrites to in dmu_buf_will_clone_or_dio()
2829 * Since we are undirtying the record in open-context, we must have a in dmu_buf_will_clone_or_dio()
2834 ASSERT0P(dbuf_find_dirty_eq(db, tx->tx_txg)); in dmu_buf_will_clone_or_dio()
2836 if (db->db_buf != NULL) { in dmu_buf_will_clone_or_dio()
2842 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dmu_buf_will_clone_or_dio()
2843 if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) in dmu_buf_will_clone_or_dio()
2844 arc_buf_destroy(db->db_buf, db); in dmu_buf_will_clone_or_dio()
2851 db->db_buf = NULL; in dmu_buf_will_clone_or_dio()
2855 ASSERT3P(db->db_buf, ==, NULL); in dmu_buf_will_clone_or_dio()
2856 ASSERT3P(db->db.db_data, ==, NULL); in dmu_buf_will_clone_or_dio()
2858 db->db_state = DB_NOFILL; in dmu_buf_will_clone_or_dio()
2863 mutex_exit(&db->db_mtx); in dmu_buf_will_clone_or_dio()
2874 mutex_enter(&db->db_mtx); in dmu_buf_will_not_fill()
2875 db->db_state = DB_NOFILL; in dmu_buf_will_not_fill()
2877 mutex_exit(&db->db_mtx); in dmu_buf_will_not_fill()
2889 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_fill_flags()
2890 ASSERT(tx->tx_txg != 0); in dmu_buf_will_fill_flags()
2891 ASSERT(db->db_level == 0); in dmu_buf_will_fill_flags()
2892 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_fill_flags()
2894 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || in dmu_buf_will_fill_flags()
2897 mutex_enter(&db->db_mtx); in dmu_buf_will_fill_flags()
2898 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_will_fill_flags()
2899 if (db->db_state == DB_NOFILL || in dmu_buf_will_fill_flags()
2900 (db->db_state == DB_UNCACHED && dr && dr->dt.dl.dr_diowrite)) { in dmu_buf_will_fill_flags()
2906 mutex_exit(&db->db_mtx); in dmu_buf_will_fill_flags()
2916 if (db->db_state == DB_NOFILL) { in dmu_buf_will_fill_flags()
2918 db->db_state = DB_UNCACHED; in dmu_buf_will_fill_flags()
2921 mutex_exit(&db->db_mtx); in dmu_buf_will_fill_flags()
2951 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dmu_buf_set_crypt_params()
2952 ASSERT0(db->db_level); in dmu_buf_set_crypt_params()
2953 ASSERT(db->db_objset->os_raw_receive); in dmu_buf_set_crypt_params()
2958 dr = dbuf_find_dirty_eq(db, tx->tx_txg); in dmu_buf_set_crypt_params()
2961 ASSERT3U(dr->dt.dl.dr_override_state, ==, DR_NOT_OVERRIDDEN); in dmu_buf_set_crypt_params()
2963 dr->dt.dl.dr_has_raw_params = B_TRUE; in dmu_buf_set_crypt_params()
2964 dr->dt.dl.dr_byteorder = byteorder; in dmu_buf_set_crypt_params()
2965 memcpy(dr->dt.dl.dr_salt, salt, ZIO_DATA_SALT_LEN); in dmu_buf_set_crypt_params()
2966 memcpy(dr->dt.dl.dr_iv, iv, ZIO_DATA_IV_LEN); in dmu_buf_set_crypt_params()
2967 memcpy(dr->dt.dl.dr_mac, mac, ZIO_DATA_MAC_LEN); in dmu_buf_set_crypt_params()
2976 ASSERT3U(db->db.db_object, !=, DMU_META_DNODE_OBJECT); in dbuf_override_impl()
2977 ASSERT0(db->db_level); in dbuf_override_impl()
2979 dr = list_head(&db->db_dirty_records); in dbuf_override_impl()
2981 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); in dbuf_override_impl()
2982 dl = &dr->dt.dl; in dbuf_override_impl()
2983 ASSERT0(dl->dr_has_raw_params); in dbuf_override_impl()
2984 dl->dr_overridden_by = *bp; in dbuf_override_impl()
2985 dl->dr_override_state = DR_OVERRIDDEN; in dbuf_override_impl()
2986 BP_SET_LOGICAL_BIRTH(&dl->dr_overridden_by, dr->dr_txg); in dbuf_override_impl()
2994 mutex_enter(&db->db_mtx); in dmu_buf_fill_done()
2997 if (db->db_state == DB_FILL) { in dmu_buf_fill_done()
2998 if (db->db_level == 0 && db->db_freed_in_flight) { in dmu_buf_fill_done()
2999 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_fill_done()
3002 memset(db->db.db_data, 0, db->db.db_size); in dmu_buf_fill_done()
3003 db->db_freed_in_flight = FALSE; in dmu_buf_fill_done()
3004 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3010 arc_buf_destroy(db->db_buf, db); in dmu_buf_fill_done()
3011 db->db_buf = NULL; in dmu_buf_fill_done()
3015 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3018 cv_broadcast(&db->db_changed); in dmu_buf_fill_done()
3020 db->db_state = DB_CACHED; in dmu_buf_fill_done()
3023 mutex_exit(&db->db_mtx); in dmu_buf_fill_done()
3039 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), in dmu_buf_write_embedded()
3044 type = DB_DNODE(db)->dn_type; in dmu_buf_write_embedded()
3047 ASSERT0(db->db_level); in dmu_buf_write_embedded()
3048 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_write_embedded()
3052 dr = list_head(&db->db_dirty_records); in dmu_buf_write_embedded()
3054 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); in dmu_buf_write_embedded()
3055 dl = &dr->dt.dl; in dmu_buf_write_embedded()
3056 ASSERT0(dl->dr_has_raw_params); in dmu_buf_write_embedded()
3057 encode_embedded_bp_compressed(&dl->dr_overridden_by, in dmu_buf_write_embedded()
3059 BPE_SET_ETYPE(&dl->dr_overridden_by, etype); in dmu_buf_write_embedded()
3060 BP_SET_TYPE(&dl->dr_overridden_by, type); in dmu_buf_write_embedded()
3061 BP_SET_LEVEL(&dl->dr_overridden_by, 0); in dmu_buf_write_embedded()
3062 BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); in dmu_buf_write_embedded()
3064 dl->dr_override_state = DR_OVERRIDDEN; in dmu_buf_write_embedded()
3065 BP_SET_LOGICAL_BIRTH(&dl->dr_overridden_by, dr->dr_txg); in dmu_buf_write_embedded()
3073 ASSERT(dsl_dataset_feature_is_active(db->db_objset->os_dsl_dataset, in dmu_buf_redact()
3077 type = DB_DNODE(db)->dn_type; in dmu_buf_redact()
3080 ASSERT0(db->db_level); in dmu_buf_redact()
3086 BP_SET_BIRTH(&bp, tx->tx_txg, 0); in dmu_buf_redact()
3088 BPE_SET_LSIZE(&bp, dbuf->db_size); in dmu_buf_redact()
3101 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_assign_arcbuf()
3102 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_assign_arcbuf()
3103 ASSERT(db->db_level == 0); in dbuf_assign_arcbuf()
3106 ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size); in dbuf_assign_arcbuf()
3107 ASSERT(tx->tx_txg != 0); in dbuf_assign_arcbuf()
3112 mutex_enter(&db->db_mtx); in dbuf_assign_arcbuf()
3114 db->db_pending_evict = B_FALSE; in dbuf_assign_arcbuf()
3115 db->db_partial_read = B_FALSE; in dbuf_assign_arcbuf()
3117 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_assign_arcbuf()
3118 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_assign_arcbuf()
3120 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED || in dbuf_assign_arcbuf()
3121 db->db_state == DB_NOFILL); in dbuf_assign_arcbuf()
3123 if (db->db_state == DB_CACHED && in dbuf_assign_arcbuf()
3124 zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { in dbuf_assign_arcbuf()
3132 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
3134 memcpy(db->db.db_data, buf->b_data, db->db.db_size); in dbuf_assign_arcbuf()
3139 if (db->db_state == DB_CACHED) { in dbuf_assign_arcbuf()
3140 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); in dbuf_assign_arcbuf()
3142 ASSERT(db->db_buf != NULL); in dbuf_assign_arcbuf()
3143 if (dr != NULL && dr->dr_txg == tx->tx_txg) { in dbuf_assign_arcbuf()
3144 ASSERT(dr->dt.dl.dr_data == db->db_buf); in dbuf_assign_arcbuf()
3146 if (!arc_released(db->db_buf)) { in dbuf_assign_arcbuf()
3147 ASSERT(dr->dt.dl.dr_override_state == in dbuf_assign_arcbuf()
3149 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
3151 dr->dt.dl.dr_data = buf; in dbuf_assign_arcbuf()
3152 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
3153 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { in dbuf_assign_arcbuf()
3154 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
3155 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
3157 db->db_buf = NULL; in dbuf_assign_arcbuf()
3158 } else if (db->db_state == DB_NOFILL) { in dbuf_assign_arcbuf()
3166 db->db_state = DB_UNCACHED; in dbuf_assign_arcbuf()
3168 ASSERT(db->db_buf == NULL); in dbuf_assign_arcbuf()
3170 db->db_state = DB_FILL; in dbuf_assign_arcbuf()
3172 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
3174 dmu_buf_fill_done(&db->db, tx, B_FALSE); in dbuf_assign_arcbuf()
3181 dmu_buf_impl_t *parent = db->db_parent; in dbuf_destroy()
3184 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_destroy()
3185 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
3187 if (db->db_buf != NULL) { in dbuf_destroy()
3188 arc_buf_destroy(db->db_buf, db); in dbuf_destroy()
3189 db->db_buf = NULL; in dbuf_destroy()
3192 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_destroy()
3193 int slots = DB_DNODE(db)->dn_num_slots; in dbuf_destroy()
3195 if (db->db.db_data != NULL) { in dbuf_destroy()
3196 kmem_free(db->db.db_data, bonuslen); in dbuf_destroy()
3198 db->db_state = DB_UNCACHED; in dbuf_destroy()
3205 if (multilist_link_active(&db->db_cache_link)) { in dbuf_destroy()
3206 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_destroy()
3207 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_destroy()
3209 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); in dbuf_destroy()
3211 ASSERT0(dmu_buf_user_size(&db->db)); in dbuf_destroy()
3213 &dbuf_caches[db->db_caching_status].size, in dbuf_destroy()
3214 db->db.db_size, db); in dbuf_destroy()
3216 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { in dbuf_destroy()
3219 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_destroy()
3221 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], in dbuf_destroy()
3222 db->db.db_size); in dbuf_destroy()
3224 db->db_caching_status = DB_NO_CACHE; in dbuf_destroy()
3227 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_destroy()
3228 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
3229 ASSERT(list_is_empty(&db->db_dirty_records)); in dbuf_destroy()
3231 db->db_state = DB_EVICTING; in dbuf_destroy()
3233 db->db_blkptr = NULL; in dbuf_destroy()
3240 mutex_exit(&db->db_mtx); in dbuf_destroy()
3244 dndb = dn->dn_dbuf; in dbuf_destroy()
3245 if (db->db_blkid != DMU_BONUS_BLKID) { in dbuf_destroy()
3246 boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx); in dbuf_destroy()
3248 mutex_enter_nested(&dn->dn_dbufs_mtx, in dbuf_destroy()
3250 avl_remove(&dn->dn_dbufs, db); in dbuf_destroy()
3254 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_destroy()
3263 mutex_enter(&dn->dn_mtx); in dbuf_destroy()
3266 db->db_dnode_handle = NULL; in dbuf_destroy()
3268 db->db_dnode = NULL; in dbuf_destroy()
3276 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
3278 db->db_parent = NULL; in dbuf_destroy()
3280 ASSERT(db->db_buf == NULL); in dbuf_destroy()
3281 ASSERT(db->db.db_data == NULL); in dbuf_destroy()
3282 ASSERT(db->db_hash_next == NULL); in dbuf_destroy()
3283 ASSERT(db->db_blkptr == NULL); in dbuf_destroy()
3284 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
3285 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_destroy()
3286 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_destroy()
3293 mutex_enter(&parent->db_mtx); in dbuf_destroy()
3304 * this happens when the dnode is the meta-dnode, or {user|group|project}used
3318 mutex_enter(&dn->dn_mtx); in dbuf_findbp()
3319 if (dn->dn_have_spill && in dbuf_findbp()
3320 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) in dbuf_findbp()
3321 *bpp = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_findbp()
3324 dbuf_add_ref(dn->dn_dbuf, NULL); in dbuf_findbp()
3325 *parentp = dn->dn_dbuf; in dbuf_findbp()
3326 mutex_exit(&dn->dn_mtx); in dbuf_findbp()
3331 (dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels; in dbuf_findbp()
3332 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_findbp()
3335 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_findbp()
3341 * other words, if N * epbs + 1 > 64, then if (N-1) * epbs + 1 > 55 in dbuf_findbp()
3343 * N-1 levels and the assertion won't overflow. However, once epbs is in dbuf_findbp()
3352 ((nlevels - level - 1) * epbs) + in dbuf_findbp()
3353 highbit64(dn->dn_phys->dn_nblkptr) <= 64); in dbuf_findbp()
3355 blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr << in dbuf_findbp()
3356 ((nlevels - level - 1) * epbs)) || in dbuf_findbp()
3358 blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { in dbuf_findbp()
3361 } else if (level < nlevels-1) { in dbuf_findbp()
3377 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + in dbuf_findbp()
3378 (blkid & ((1ULL << epbs) - 1)); in dbuf_findbp()
3382 ASSERT3U(level, ==, nlevels-1); in dbuf_findbp()
3383 ASSERT(dn->dn_phys->dn_nblkptr == 0 || in dbuf_findbp()
3384 blkid < dn->dn_phys->dn_nblkptr); in dbuf_findbp()
3385 if (dn->dn_dbuf) { in dbuf_findbp()
3386 dbuf_add_ref(dn->dn_dbuf, NULL); in dbuf_findbp()
3387 *parentp = dn->dn_dbuf; in dbuf_findbp()
3389 *bpp = &dn->dn_phys->dn_blkptr[blkid]; in dbuf_findbp()
3398 objset_t *os = dn->dn_objset; in dbuf_create()
3401 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_create()
3402 ASSERT(dn->dn_type != DMU_OT_NONE); in dbuf_create()
3406 list_create(&db->db_dirty_records, sizeof (dbuf_dirty_record_t), in dbuf_create()
3409 db->db_objset = os; in dbuf_create()
3410 db->db.db_object = dn->dn_object; in dbuf_create()
3411 db->db_level = level; in dbuf_create()
3412 db->db_blkid = blkid; in dbuf_create()
3413 db->db_dirtycnt = 0; in dbuf_create()
3415 db->db_dnode_handle = dn->dn_handle; in dbuf_create()
3417 db->db_dnode = dn; in dbuf_create()
3419 db->db_parent = parent; in dbuf_create()
3420 db->db_blkptr = blkptr; in dbuf_create()
3421 db->db_hash = hash; in dbuf_create()
3423 db->db_user = NULL; in dbuf_create()
3424 db->db_user_immediate_evict = FALSE; in dbuf_create()
3425 db->db_freed_in_flight = FALSE; in dbuf_create()
3426 db->db_pending_evict = TRUE; in dbuf_create()
3427 db->db_partial_read = FALSE; in dbuf_create()
3430 ASSERT3P(parent, ==, dn->dn_dbuf); in dbuf_create()
3431 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dbuf_create()
3432 (dn->dn_nblkptr-1) * sizeof (blkptr_t); in dbuf_create()
3433 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_create()
3434 db->db.db_offset = DMU_BONUS_BLKID; in dbuf_create()
3435 db->db_state = DB_UNCACHED; in dbuf_create()
3437 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
3442 db->db.db_size = (blkptr != NULL) ? in dbuf_create()
3444 db->db.db_offset = 0; in dbuf_create()
3447 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; in dbuf_create()
3448 db->db.db_size = blocksize; in dbuf_create()
3449 db->db.db_offset = db->db_blkid * blocksize; in dbuf_create()
3459 mutex_enter(&dn->dn_dbufs_mtx); in dbuf_create()
3460 db->db_state = DB_EVICTING; /* not worth logging this state change */ in dbuf_create()
3463 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_create()
3468 avl_add(&dn->dn_dbufs, db); in dbuf_create()
3470 db->db_state = DB_UNCACHED; in dbuf_create()
3472 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
3473 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_create()
3476 if (parent && parent != dn->dn_dbuf) in dbuf_create()
3479 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || in dbuf_create()
3480 zfs_refcount_count(&dn->dn_holds) > 0); in dbuf_create()
3481 (void) zfs_refcount_add(&dn->dn_holds, db); in dbuf_create()
3502 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_dnode_findbp()
3511 *datablkszsec = dn->dn_phys->dn_datablkszsec; in dbuf_dnode_findbp()
3513 *indblkshift = dn->dn_phys->dn_indblkshift; in dbuf_dnode_findbp()
3535 if (dpa->dpa_cb != NULL) { in dbuf_prefetch_fini()
3536 dpa->dpa_cb(dpa->dpa_arg, dpa->dpa_zb.zb_level, in dbuf_prefetch_fini()
3537 dpa->dpa_zb.zb_blkid, io_done); in dbuf_prefetch_fini()
3568 dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH | in dbuf_issue_final_prefetch()
3573 dpa->dpa_curlevel == 0) in dbuf_issue_final_prefetch()
3576 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); in dbuf_issue_final_prefetch()
3577 ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level); in dbuf_issue_final_prefetch()
3578 ASSERT(dpa->dpa_zio != NULL); in dbuf_issue_final_prefetch()
3579 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, in dbuf_issue_final_prefetch()
3581 dpa->dpa_prio, zio_flags, &aflags, &dpa->dpa_zb); in dbuf_issue_final_prefetch()
3596 ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); in dbuf_prefetch_indirect_done()
3597 ASSERT3S(dpa->dpa_curlevel, >, 0); in dbuf_prefetch_indirect_done()
3600 ASSERT(zio == NULL || zio->io_error != 0); in dbuf_prefetch_indirect_done()
3604 ASSERT(zio == NULL || zio->io_error == 0); in dbuf_prefetch_indirect_done()
3618 ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel); in dbuf_prefetch_indirect_done()
3619 if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS) { in dbuf_prefetch_indirect_done()
3620 ASSERT3U(BP_GET_PSIZE(zio->io_bp), ==, zio->io_size); in dbuf_prefetch_indirect_done()
3622 ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size); in dbuf_prefetch_indirect_done()
3624 ASSERT3P(zio->io_spa, ==, dpa->dpa_spa); in dbuf_prefetch_indirect_done()
3626 dpa->dpa_dnode = NULL; in dbuf_prefetch_indirect_done()
3627 } else if (dpa->dpa_dnode != NULL) { in dbuf_prefetch_indirect_done()
3628 uint64_t curblkid = dpa->dpa_zb.zb_blkid >> in dbuf_prefetch_indirect_done()
3629 (dpa->dpa_epbs * (dpa->dpa_curlevel - in dbuf_prefetch_indirect_done()
3630 dpa->dpa_zb.zb_level)); in dbuf_prefetch_indirect_done()
3631 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, in dbuf_prefetch_indirect_done()
3632 dpa->dpa_curlevel, curblkid, FTAG); in dbuf_prefetch_indirect_done()
3643 dpa->dpa_curlevel--; in dbuf_prefetch_indirect_done()
3644 uint64_t nextblkid = dpa->dpa_zb.zb_blkid >> in dbuf_prefetch_indirect_done()
3645 (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level)); in dbuf_prefetch_indirect_done()
3646 blkptr_t *bp = ((blkptr_t *)abuf->b_data) + in dbuf_prefetch_indirect_done()
3647 P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs); in dbuf_prefetch_indirect_done()
3649 ASSERT(!BP_IS_REDACTED(bp) || dpa->dpa_dnode == NULL || in dbuf_prefetch_indirect_done()
3651 dpa->dpa_dnode->dn_objset->os_dsl_dataset, in dbuf_prefetch_indirect_done()
3657 } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) { in dbuf_prefetch_indirect_done()
3658 ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid); in dbuf_prefetch_indirect_done()
3665 if (dpa->dpa_dnode) { in dbuf_prefetch_indirect_done()
3666 if (dnode_level_is_l2cacheable(bp, dpa->dpa_dnode, in dbuf_prefetch_indirect_done()
3667 dpa->dpa_curlevel)) in dbuf_prefetch_indirect_done()
3670 if (dpa->dpa_aflags & ARC_FLAG_L2CACHE) in dbuf_prefetch_indirect_done()
3674 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); in dbuf_prefetch_indirect_done()
3676 SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset, in dbuf_prefetch_indirect_done()
3677 dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid); in dbuf_prefetch_indirect_done()
3679 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, in dbuf_prefetch_indirect_done()
3694 * the encryption key is unmapped before the IO completes.
3706 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_prefetch_impl()
3708 if (blkid > dn->dn_maxblkid) in dbuf_prefetch_impl()
3718 nlevels = dn->dn_phys->dn_nlevels; in dbuf_prefetch_impl()
3719 if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) in dbuf_prefetch_impl()
3722 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_prefetch_impl()
3723 if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) in dbuf_prefetch_impl()
3726 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, in dbuf_prefetch_impl()
3729 mutex_exit(&db->db_mtx); in dbuf_prefetch_impl()
3744 while (curlevel < nlevels - 1) { in dbuf_prefetch_impl()
3751 blkptr_t *bpp = db->db_buf->b_data; in dbuf_prefetch_impl()
3761 if (curlevel == nlevels - 1) { in dbuf_prefetch_impl()
3763 ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); in dbuf_prefetch_impl()
3764 bp = dn->dn_phys->dn_blkptr[curblkid]; in dbuf_prefetch_impl()
3767 dsl_dataset_feature_is_active(dn->dn_objset->os_dsl_dataset, in dbuf_prefetch_impl()
3774 zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, in dbuf_prefetch_impl()
3778 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; in dbuf_prefetch_impl()
3779 SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, in dbuf_prefetch_impl()
3780 dn->dn_object, level, blkid); in dbuf_prefetch_impl()
3781 dpa->dpa_curlevel = curlevel; in dbuf_prefetch_impl()
3782 dpa->dpa_prio = prio; in dbuf_prefetch_impl()
3783 dpa->dpa_aflags = aflags; in dbuf_prefetch_impl()
3784 dpa->dpa_spa = dn->dn_objset->os_spa; in dbuf_prefetch_impl()
3785 dpa->dpa_dnode = dn; in dbuf_prefetch_impl()
3786 dpa->dpa_epbs = epbs; in dbuf_prefetch_impl()
3787 dpa->dpa_zio = pio; in dbuf_prefetch_impl()
3788 dpa->dpa_cb = cb; in dbuf_prefetch_impl()
3789 dpa->dpa_arg = arg; in dbuf_prefetch_impl()
3792 dpa->dpa_aflags |= ARC_FLAG_UNCACHED; in dbuf_prefetch_impl()
3794 dpa->dpa_aflags |= ARC_FLAG_L2CACHE; in dbuf_prefetch_impl()
3814 SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, in dbuf_prefetch_impl()
3815 dn->dn_object, curlevel, curblkid); in dbuf_prefetch_impl()
3816 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, in dbuf_prefetch_impl()
3853 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_copy()
3854 arc_buf_t *data = dr->dt.dl.dr_data; in dbuf_hold_copy()
3866 db_data = arc_alloc_raw_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3867 dmu_objset_id(dn->dn_objset), byteorder, salt, iv, mac, in dbuf_hold_copy()
3868 dn->dn_type, arc_buf_size(data), arc_buf_lsize(data), in dbuf_hold_copy()
3872 dn->dn_objset->os_spa, db, arc_buf_size(data), in dbuf_hold_copy()
3875 db_data = arc_alloc_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3876 DBUF_GET_BUFC_TYPE(db), db->db.db_size); in dbuf_hold_copy()
3878 memcpy(db_data->b_data, data->b_data, arc_buf_size(data)); in dbuf_hold_copy()
3893 uint64_t hv; in dbuf_hold_impl() local
3896 spa_t *spa = dn->dn_objset->os_spa; in dbuf_hold_impl()
3897 dsl_pool_t *dp = spa->spa_dsl_pool; in dbuf_hold_impl()
3899 ASSERT(!MUTEX_HELD(&dp->dp_tx.tx_sync_lock)); in dbuf_hold_impl()
3903 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_hold_impl()
3905 ASSERT3U(dn->dn_nlevels, >, level); in dbuf_hold_impl()
3910 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid, &hv); in dbuf_hold_impl()
3932 db = dbuf_create(dn, level, blkid, parent, bp, hv); in dbuf_hold_impl()
3935 if (fail_uncached && db->db_state != DB_CACHED) { in dbuf_hold_impl()
3936 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3940 if (db->db_buf != NULL) { in dbuf_hold_impl()
3941 arc_buf_access(db->db_buf); in dbuf_hold_impl()
3942 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_hold_impl()
3943 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); in dbuf_hold_impl()
3946 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); in dbuf_hold_impl()
3953 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_hold_impl()
3954 dn->dn_object != DMU_META_DNODE_OBJECT && in dbuf_hold_impl()
3955 db->db_state == DB_CACHED && db->db_data_pending) { in dbuf_hold_impl()
3956 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_impl()
3957 if (dr->dt.dl.dr_data == db->db_buf) { in dbuf_hold_impl()
3958 ASSERT3P(db->db_buf, !=, NULL); in dbuf_hold_impl()
3963 if (multilist_link_active(&db->db_cache_link)) { in dbuf_hold_impl()
3964 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hold_impl()
3965 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_hold_impl()
3966 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_hold_impl()
3968 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); in dbuf_hold_impl()
3970 uint64_t size = db->db.db_size; in dbuf_hold_impl()
3971 uint64_t usize = dmu_buf_user_size(&db->db); in dbuf_hold_impl()
3973 &dbuf_caches[db->db_caching_status].size, size, db); in dbuf_hold_impl()
3975 &dbuf_caches[db->db_caching_status].size, usize, in dbuf_hold_impl()
3976 db->db_user); in dbuf_hold_impl()
3978 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { in dbuf_hold_impl()
3981 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); in dbuf_hold_impl()
3983 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], in dbuf_hold_impl()
3986 db->db_caching_status = DB_NO_CACHE; in dbuf_hold_impl()
3988 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_hold_impl()
3990 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3997 ASSERT3U(db->db_blkid, ==, blkid); in dbuf_hold_impl()
3998 ASSERT3U(db->db_level, ==, level); in dbuf_hold_impl()
4021 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dbuf_create_bonus()
4023 ASSERT(dn->dn_bonus == NULL); in dbuf_create_bonus()
4024 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL, in dbuf_create_bonus()
4025 dbuf_hash(dn->dn_objset, dn->dn_object, 0, DMU_BONUS_BLKID)); in dbuf_create_bonus()
4026 dn->dn_bonus->db_pending_evict = FALSE; in dbuf_create_bonus()
4034 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_spill_set_blksz()
4038 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); in dbuf_spill_set_blksz()
4056 int64_t holds = zfs_refcount_add(&db->db_holds, tag); in dbuf_add_ref()
4075 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { in dbuf_try_add_ref()
4076 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_try_add_ref()
4079 mutex_exit(&found_db->db_mtx); in dbuf_try_add_ref()
4094 mutex_enter(&db->db_mtx); in dbuf_rele()
4105 * dbuf_rele() for an already-locked dbuf. This is necessary to allow
4107 * argument should be set if we are already in the dbuf-evicting code
4111 * dbuf_rele()-->dbuf_rele_and_unlock()-->dbuf_evict_notify()
4114 * +-----dbuf_destroy()<--dbuf_evict_one()<--------+
4123 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_rele_and_unlock()
4131 holds = zfs_refcount_remove(&db->db_holds, tag); in dbuf_rele_and_unlock()
4138 if (db->db_buf != NULL && in dbuf_rele_and_unlock()
4139 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { in dbuf_rele_and_unlock()
4140 arc_buf_freeze(db->db_buf); in dbuf_rele_and_unlock()
4143 if (holds == db->db_dirtycnt && in dbuf_rele_and_unlock()
4144 db->db_level == 0 && db->db_user_immediate_evict) in dbuf_rele_and_unlock()
4148 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_rele_and_unlock()
4150 boolean_t evict_dbuf = db->db_pending_evict; in dbuf_rele_and_unlock()
4159 atomic_dec_32(&dn->dn_dbufs_count); in dbuf_rele_and_unlock()
4173 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4179 } else if (db->db_buf == NULL) { in dbuf_rele_and_unlock()
4184 ASSERT(db->db_state == DB_UNCACHED || in dbuf_rele_and_unlock()
4185 db->db_state == DB_NOFILL); in dbuf_rele_and_unlock()
4187 } else if (arc_released(db->db_buf)) { in dbuf_rele_and_unlock()
4192 } else if (!db->db_partial_read && !DBUF_IS_CACHEABLE(db)) { in dbuf_rele_and_unlock()
4198 } else if (!multilist_link_active(&db->db_cache_link)) { in dbuf_rele_and_unlock()
4199 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_rele_and_unlock()
4204 db->db_caching_status = dcs; in dbuf_rele_and_unlock()
4207 uint64_t db_size = db->db.db_size; in dbuf_rele_and_unlock()
4208 uint64_t dbu_size = dmu_buf_user_size(&db->db); in dbuf_rele_and_unlock()
4212 &dbuf_caches[dcs].size, dbu_size, db->db_user); in dbuf_rele_and_unlock()
4213 uint8_t db_level = db->db_level; in dbuf_rele_and_unlock()
4214 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4232 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
4240 return (zfs_refcount_count(&db->db_holds)); in dbuf_refcount()
4249 mutex_enter(&db->db_mtx); in dmu_buf_user_refcount()
4250 ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt); in dmu_buf_user_refcount()
4251 holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt; in dmu_buf_user_refcount()
4252 mutex_exit(&db->db_mtx); in dmu_buf_user_refcount()
4263 mutex_enter(&db->db_mtx); in dmu_buf_replace_user()
4265 if (db->db_user == old_user) in dmu_buf_replace_user()
4266 db->db_user = new_user; in dmu_buf_replace_user()
4268 old_user = db->db_user; in dmu_buf_replace_user()
4270 mutex_exit(&db->db_mtx); in dmu_buf_replace_user()
4286 db->db_user_immediate_evict = TRUE; in dmu_buf_set_user_ie()
4302 return (db->db_user); in dmu_buf_get_user()
4309 if (db->db_user == NULL) in dmu_buf_user_size()
4311 return (atomic_load_64(&db->db_user->dbu_size)); in dmu_buf_user_size()
4318 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dmu_buf_add_user_size()
4319 ASSERT3P(db->db_user, !=, NULL); in dmu_buf_add_user_size()
4320 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), <, UINT64_MAX - nadd); in dmu_buf_add_user_size()
4321 atomic_add_64(&db->db_user->dbu_size, nadd); in dmu_buf_add_user_size()
4328 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dmu_buf_sub_user_size()
4329 ASSERT3P(db->db_user, !=, NULL); in dmu_buf_sub_user_size()
4330 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), >=, nsub); in dmu_buf_sub_user_size()
4331 atomic_sub_64(&db->db_user->dbu_size, nsub); in dmu_buf_sub_user_size()
4344 return (dbi->db_blkptr); in dmu_buf_get_blkptr()
4351 return (dbi->db_objset); in dmu_buf_get_objset()
4358 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_check_blkptr()
4360 if (db->db_blkptr != NULL) in dbuf_check_blkptr()
4363 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_check_blkptr()
4364 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_check_blkptr()
4365 BP_ZERO(db->db_blkptr); in dbuf_check_blkptr()
4368 if (db->db_level == dn->dn_phys->dn_nlevels-1) { in dbuf_check_blkptr()
4374 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); in dbuf_check_blkptr()
4375 ASSERT(db->db_parent == NULL); in dbuf_check_blkptr()
4376 db->db_parent = dn->dn_dbuf; in dbuf_check_blkptr()
4377 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; in dbuf_check_blkptr()
4380 dmu_buf_impl_t *parent = db->db_parent; in dbuf_check_blkptr()
4381 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_check_blkptr()
4383 ASSERT(dn->dn_phys->dn_nlevels > 1); in dbuf_check_blkptr()
4385 mutex_exit(&db->db_mtx); in dbuf_check_blkptr()
4386 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_check_blkptr()
4387 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_check_blkptr()
4388 db->db_blkid >> epbs, db); in dbuf_check_blkptr()
4389 rw_exit(&dn->dn_struct_rwlock); in dbuf_check_blkptr()
4390 mutex_enter(&db->db_mtx); in dbuf_check_blkptr()
4391 db->db_parent = parent; in dbuf_check_blkptr()
4393 db->db_blkptr = (blkptr_t *)parent->db.db_data + in dbuf_check_blkptr()
4394 (db->db_blkid & ((1ULL << epbs) - 1)); in dbuf_check_blkptr()
4402 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_bonus()
4403 void *data = dr->dt.dl.dr_data; in dbuf_sync_bonus()
4405 ASSERT0(db->db_level); in dbuf_sync_bonus()
4406 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_sync_bonus()
4407 ASSERT(db->db_blkid == DMU_BONUS_BLKID); in dbuf_sync_bonus()
4410 dnode_t *dn = dr->dr_dnode; in dbuf_sync_bonus()
4411 ASSERT3U(DN_MAX_BONUS_LEN(dn->dn_phys), <=, in dbuf_sync_bonus()
4412 DN_SLOTS_TO_BONUSLEN(dn->dn_phys->dn_extra_slots + 1)); in dbuf_sync_bonus()
4413 memcpy(DN_BONUS(dn->dn_phys), data, DN_MAX_BONUS_LEN(dn->dn_phys)); in dbuf_sync_bonus()
4418 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_sync_bonus()
4432 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_prepare_encrypted_dnode_leaf()
4434 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_prepare_encrypted_dnode_leaf()
4435 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dbuf_prepare_encrypted_dnode_leaf()
4436 ASSERT3U(db->db_level, ==, 0); in dbuf_prepare_encrypted_dnode_leaf()
4438 if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) { in dbuf_prepare_encrypted_dnode_leaf()
4448 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
4449 db->db.db_object, db->db_level, db->db_blkid); in dbuf_prepare_encrypted_dnode_leaf()
4450 err = arc_untransform(db->db_buf, db->db_objset->os_spa, in dbuf_prepare_encrypted_dnode_leaf()
4454 } else if (dr->dt.dl.dr_has_raw_params) { in dbuf_prepare_encrypted_dnode_leaf()
4455 (void) arc_release(dr->dt.dl.dr_data, db); in dbuf_prepare_encrypted_dnode_leaf()
4456 arc_convert_to_raw(dr->dt.dl.dr_data, in dbuf_prepare_encrypted_dnode_leaf()
4457 dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
4458 dr->dt.dl.dr_byteorder, DMU_OT_DNODE, in dbuf_prepare_encrypted_dnode_leaf()
4459 dr->dt.dl.dr_salt, dr->dt.dl.dr_iv, dr->dt.dl.dr_mac); in dbuf_prepare_encrypted_dnode_leaf()
4471 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_indirect()
4472 dnode_t *dn = dr->dr_dnode; in dbuf_sync_indirect()
4476 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_indirect()
4478 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
4480 ASSERT(db->db_level > 0); in dbuf_sync_indirect()
4484 if (db->db_buf == NULL) { in dbuf_sync_indirect()
4485 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
4487 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
4489 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_sync_indirect()
4490 ASSERT(db->db_buf != NULL); in dbuf_sync_indirect()
4493 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_sync_indirect()
4497 db->db_data_pending = dr; in dbuf_sync_indirect()
4499 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
4501 dbuf_write(dr, db->db_buf, tx); in dbuf_sync_indirect()
4503 zio_t *zio = dr->dr_zio; in dbuf_sync_indirect()
4504 mutex_enter(&dr->dt.di.dr_mtx); in dbuf_sync_indirect()
4505 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); in dbuf_sync_indirect()
4506 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); in dbuf_sync_indirect()
4507 mutex_exit(&dr->dt.di.dr_mtx); in dbuf_sync_indirect()
4528 dnode_t *dn = dr->dr_dnode; in dbuf_sync_leaf_verify_bonus_dnode()
4534 if (DMU_OT_IS_ENCRYPTED(dn->dn_bonustype)) in dbuf_sync_leaf_verify_bonus_dnode()
4537 uint16_t bonuslen = dn->dn_phys->dn_bonuslen; in dbuf_sync_leaf_verify_bonus_dnode()
4538 uint16_t maxbonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_sync_leaf_verify_bonus_dnode()
4541 arc_buf_t *datap = dr->dt.dl.dr_data; in dbuf_sync_leaf_verify_bonus_dnode()
4555 ASSERT3P(dr->dr_dbuf, ==, NULL); in dbuf_lightweight_bp()
4556 dnode_t *dn = dr->dr_dnode; in dbuf_lightweight_bp()
4558 if (dn->dn_phys->dn_nlevels == 1) { in dbuf_lightweight_bp()
4559 VERIFY3U(dr->dt.dll.dr_blkid, <, dn->dn_phys->dn_nblkptr); in dbuf_lightweight_bp()
4560 return (&dn->dn_phys->dn_blkptr[dr->dt.dll.dr_blkid]); in dbuf_lightweight_bp()
4562 dmu_buf_impl_t *parent_db = dr->dr_parent->dr_dbuf; in dbuf_lightweight_bp()
4563 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_lightweight_bp()
4564 VERIFY3U(parent_db->db_level, ==, 1); in dbuf_lightweight_bp()
4566 VERIFY3U(dr->dt.dll.dr_blkid >> epbs, ==, parent_db->db_blkid); in dbuf_lightweight_bp()
4567 blkptr_t *bp = parent_db->db.db_data; in dbuf_lightweight_bp()
4568 return (&bp[dr->dt.dll.dr_blkid & ((1 << epbs) - 1)]); in dbuf_lightweight_bp()
4575 dbuf_dirty_record_t *dr = zio->io_private; in dbuf_lightweight_ready()
4576 blkptr_t *bp = zio->io_bp; in dbuf_lightweight_ready()
4578 if (zio->io_error != 0) in dbuf_lightweight_ready()
4581 dnode_t *dn = dr->dr_dnode; in dbuf_lightweight_ready()
4584 spa_t *spa = dmu_objset_spa(dn->dn_objset); in dbuf_lightweight_ready()
4585 int64_t delta = bp_get_dsize_sync(spa, bp) - in dbuf_lightweight_ready()
4589 uint64_t blkid = dr->dt.dll.dr_blkid; in dbuf_lightweight_ready()
4590 mutex_enter(&dn->dn_mtx); in dbuf_lightweight_ready()
4591 if (blkid > dn->dn_phys->dn_maxblkid) { in dbuf_lightweight_ready()
4592 ASSERT0(dn->dn_objset->os_raw_receive); in dbuf_lightweight_ready()
4593 dn->dn_phys->dn_maxblkid = blkid; in dbuf_lightweight_ready()
4595 mutex_exit(&dn->dn_mtx); in dbuf_lightweight_ready()
4603 EQUIV(dr->dr_parent == NULL, dn->dn_phys->dn_nlevels == 1); in dbuf_lightweight_ready()
4604 if (dr->dr_parent == NULL) { in dbuf_lightweight_ready()
4605 parent_db = dn->dn_dbuf; in dbuf_lightweight_ready()
4607 parent_db = dr->dr_parent->dr_dbuf; in dbuf_lightweight_ready()
4609 rw_enter(&parent_db->db_rwlock, RW_WRITER); in dbuf_lightweight_ready()
4611 rw_exit(&parent_db->db_rwlock); in dbuf_lightweight_ready()
4617 dbuf_dirty_record_t *dr = zio->io_private; in dbuf_lightweight_done()
4619 VERIFY0(zio->io_error); in dbuf_lightweight_done()
4621 objset_t *os = dr->dr_dnode->dn_objset; in dbuf_lightweight_done()
4622 dmu_tx_t *tx = os->os_synctx; in dbuf_lightweight_done()
4624 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { in dbuf_lightweight_done()
4625 ASSERT(BP_EQUAL(zio->io_bp, &zio->io_bp_orig)); in dbuf_lightweight_done()
4627 dsl_dataset_t *ds = os->os_dsl_dataset; in dbuf_lightweight_done()
4628 (void) dsl_dataset_block_kill(ds, &zio->io_bp_orig, tx, B_TRUE); in dbuf_lightweight_done()
4629 dsl_dataset_block_born(ds, zio->io_bp, tx); in dbuf_lightweight_done()
4632 dsl_pool_undirty_space(dmu_objset_pool(os), dr->dr_accounted, in dbuf_lightweight_done()
4633 zio->io_txg); in dbuf_lightweight_done()
4635 abd_free(dr->dt.dll.dr_abd); in dbuf_lightweight_done()
4642 dnode_t *dn = dr->dr_dnode; in dbuf_sync_lightweight()
4644 if (dn->dn_phys->dn_nlevels == 1) { in dbuf_sync_lightweight()
4645 pio = dn->dn_zio; in dbuf_sync_lightweight()
4647 pio = dr->dr_parent->dr_zio; in dbuf_sync_lightweight()
4651 .zb_objset = dmu_objset_id(dn->dn_objset), in dbuf_sync_lightweight()
4652 .zb_object = dn->dn_object, in dbuf_sync_lightweight()
4654 .zb_blkid = dr->dt.dll.dr_blkid, in dbuf_sync_lightweight()
4658 * See comment in dbuf_write(). This is so that zio->io_bp_orig in dbuf_sync_lightweight()
4661 dr->dr_bp_copy = *dbuf_lightweight_bp(dr); in dbuf_sync_lightweight()
4663 dr->dr_zio = zio_write(pio, dmu_objset_spa(dn->dn_objset), in dbuf_sync_lightweight()
4664 dmu_tx_get_txg(tx), &dr->dr_bp_copy, dr->dt.dll.dr_abd, in dbuf_sync_lightweight()
4665 dn->dn_datablksz, abd_get_size(dr->dt.dll.dr_abd), in dbuf_sync_lightweight()
4666 &dr->dt.dll.dr_props, dbuf_lightweight_ready, NULL, in dbuf_sync_lightweight()
4668 ZIO_FLAG_MUSTSUCCEED | dr->dt.dll.dr_flags, &zb); in dbuf_sync_lightweight()
4670 zio_nowait(dr->dr_zio); in dbuf_sync_lightweight()
4681 arc_buf_t **datap = &dr->dt.dl.dr_data; in dbuf_sync_leaf()
4682 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_leaf()
4683 dnode_t *dn = dr->dr_dnode; in dbuf_sync_leaf()
4685 uint64_t txg = tx->tx_txg; in dbuf_sync_leaf()
4689 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_leaf()
4691 mutex_enter(&db->db_mtx); in dbuf_sync_leaf()
4696 if (db->db_state == DB_UNCACHED) { in dbuf_sync_leaf()
4698 ASSERT3P(db->db.db_data, ==, NULL); in dbuf_sync_leaf()
4699 } else if (db->db_state == DB_FILL) { in dbuf_sync_leaf()
4700 /* This buffer was freed and is now being re-filled */ in dbuf_sync_leaf()
4701 ASSERT(db->db.db_data != dr->dt.dl.dr_data); in dbuf_sync_leaf()
4702 } else if (db->db_state == DB_READ) { in dbuf_sync_leaf()
4705 * occur and has an in-flgiht read on the BP. It is safe to in dbuf_sync_leaf()
4714 list_head(&db->db_dirty_records); in dbuf_sync_leaf()
4715 ASSERT3P(db->db_buf, ==, NULL); in dbuf_sync_leaf()
4716 ASSERT3P(db->db.db_data, ==, NULL); in dbuf_sync_leaf()
4717 ASSERT3P(dr_head->dt.dl.dr_data, ==, NULL); in dbuf_sync_leaf()
4718 ASSERT3U(dr_head->dt.dl.dr_override_state, ==, DR_OVERRIDDEN); in dbuf_sync_leaf()
4720 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); in dbuf_sync_leaf()
4724 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_sync_leaf()
4725 mutex_enter(&dn->dn_mtx); in dbuf_sync_leaf()
4726 if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) { in dbuf_sync_leaf()
4740 db->db_blkptr = NULL; in dbuf_sync_leaf()
4742 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; in dbuf_sync_leaf()
4743 mutex_exit(&dn->dn_mtx); in dbuf_sync_leaf()
4752 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_sync_leaf()
4753 ASSERT(dr->dr_dbuf == db); in dbuf_sync_leaf()
4758 os = dn->dn_objset; in dbuf_sync_leaf()
4770 * synchronous IO to complete. in dbuf_sync_leaf()
4774 * Direct I/O writes happen in open-context. in dbuf_sync_leaf()
4776 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { in dbuf_sync_leaf()
4777 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); in dbuf_sync_leaf()
4778 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_sync_leaf()
4785 if (os->os_encrypted && dn->dn_object == DMU_META_DNODE_OBJECT) in dbuf_sync_leaf()
4788 if (*datap != NULL && *datap == db->db_buf && in dbuf_sync_leaf()
4789 dn->dn_object != DMU_META_DNODE_OBJECT && in dbuf_sync_leaf()
4790 zfs_refcount_count(&db->db_holds) > 1) { in dbuf_sync_leaf()
4815 *datap = arc_alloc_raw_buf(os->os_spa, db, in dbuf_sync_leaf()
4817 dn->dn_type, psize, lsize, compress_type, in dbuf_sync_leaf()
4821 *datap = arc_alloc_compressed_buf(os->os_spa, db, in dbuf_sync_leaf()
4824 *datap = arc_alloc_buf(os->os_spa, db, type, psize); in dbuf_sync_leaf()
4826 memcpy((*datap)->b_data, db->db.db_data, psize); in dbuf_sync_leaf()
4828 db->db_data_pending = dr; in dbuf_sync_leaf()
4830 mutex_exit(&db->db_mtx); in dbuf_sync_leaf()
4834 ASSERT(!list_link_active(&dr->dr_dirty_node)); in dbuf_sync_leaf()
4835 if (dn->dn_object == DMU_META_DNODE_OBJECT) { in dbuf_sync_leaf()
4836 list_insert_tail(&dn->dn_dirty_records[txg & TXG_MASK], dr); in dbuf_sync_leaf()
4838 zio_nowait(dr->dr_zio); in dbuf_sync_leaf()
4852 if (dr->dr_zio != NULL) { in dbuf_sync_list()
4855 * are processing the meta-dnode, and we have finished. in dbuf_sync_list()
4860 ASSERT3U(dr->dr_dbuf->db.db_object, ==, in dbuf_sync_list()
4865 if (dr->dr_dbuf == NULL) { in dbuf_sync_list()
4868 if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && in dbuf_sync_list()
4869 dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { in dbuf_sync_list()
4870 VERIFY3U(dr->dr_dbuf->db_level, ==, level); in dbuf_sync_list()
4872 if (dr->dr_dbuf->db_level > 0) in dbuf_sync_list()
4886 blkptr_t *bp = zio->io_bp; in dbuf_write_ready()
4887 blkptr_t *bp_orig = &zio->io_bp_orig; in dbuf_write_ready()
4888 spa_t *spa = zio->io_spa; in dbuf_write_ready()
4893 ASSERT3P(db->db_blkptr, !=, NULL); in dbuf_write_ready()
4894 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); in dbuf_write_ready()
4898 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); in dbuf_write_ready()
4899 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); in dbuf_write_ready()
4900 zio->io_prev_space_delta = delta; in dbuf_write_ready()
4903 ASSERT((db->db_blkid != DMU_SPILL_BLKID && in dbuf_write_ready()
4904 BP_GET_TYPE(bp) == dn->dn_type) || in dbuf_write_ready()
4905 (db->db_blkid == DMU_SPILL_BLKID && in dbuf_write_ready()
4906 BP_GET_TYPE(bp) == dn->dn_bonustype) || in dbuf_write_ready()
4908 ASSERT(BP_GET_LEVEL(bp) == db->db_level); in dbuf_write_ready()
4911 mutex_enter(&db->db_mtx); in dbuf_write_ready()
4914 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_ready()
4915 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); in dbuf_write_ready()
4917 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_ready()
4921 if (db->db_level == 0) { in dbuf_write_ready()
4922 mutex_enter(&dn->dn_mtx); in dbuf_write_ready()
4923 if (db->db_blkid > dn->dn_phys->dn_maxblkid && in dbuf_write_ready()
4924 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_write_ready()
4925 ASSERT0(db->db_objset->os_raw_receive); in dbuf_write_ready()
4926 dn->dn_phys->dn_maxblkid = db->db_blkid; in dbuf_write_ready()
4928 mutex_exit(&dn->dn_mtx); in dbuf_write_ready()
4930 if (dn->dn_type == DMU_OT_DNODE) { in dbuf_write_ready()
4932 while (i < db->db.db_size) { in dbuf_write_ready()
4934 (void *)(((char *)db->db.db_data) + i); in dbuf_write_ready()
4937 if (dnp->dn_type != DMU_OT_NONE) { in dbuf_write_ready()
4939 for (int j = 0; j < dnp->dn_nblkptr; in dbuf_write_ready()
4942 &dnp->dn_blkptr[j], in dbuf_write_ready()
4946 if (dnp->dn_flags & in dbuf_write_ready()
4953 i += dnp->dn_extra_slots * in dbuf_write_ready()
4965 blkptr_t *ibp = db->db.db_data; in dbuf_write_ready()
4966 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_write_ready()
4967 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { in dbuf_write_ready()
4980 mutex_exit(&db->db_mtx); in dbuf_write_ready()
4983 *db->db_blkptr = *bp; in dbuf_write_ready()
5002 ASSERT3U(db->db_level, >, 0); in dbuf_write_children_ready()
5004 epbs = DB_DNODE(db)->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_write_children_ready()
5009 for (i = 0, bp = db->db.db_data; i < 1ULL << epbs; i++, bp++) { in dbuf_write_children_ready()
5024 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_write_children_ready()
5025 memset(db->db.db_data, 0, db->db.db_size); in dbuf_write_children_ready()
5026 rw_exit(&db->db_rwlock); in dbuf_write_children_ready()
5035 blkptr_t *bp_orig = &zio->io_bp_orig; in dbuf_write_done()
5036 blkptr_t *bp = db->db_blkptr; in dbuf_write_done()
5037 objset_t *os = db->db_objset; in dbuf_write_done()
5038 dmu_tx_t *tx = os->os_synctx; in dbuf_write_done()
5040 ASSERT0(zio->io_error); in dbuf_write_done()
5041 ASSERT(db->db_blkptr == bp); in dbuf_write_done()
5047 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { in dbuf_write_done()
5050 dsl_dataset_t *ds = os->os_dsl_dataset; in dbuf_write_done()
5055 mutex_enter(&db->db_mtx); in dbuf_write_done()
5059 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_write_done()
5060 dnode_t *dn = dr->dr_dnode; in dbuf_write_done()
5061 ASSERT(!list_link_active(&dr->dr_dirty_node)); in dbuf_write_done()
5062 ASSERT(dr->dr_dbuf == db); in dbuf_write_done()
5063 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); in dbuf_write_done()
5064 list_remove(&db->db_dirty_records, dr); in dbuf_write_done()
5067 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_done()
5068 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); in dbuf_write_done()
5069 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && in dbuf_write_done()
5070 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_done()
5074 if (db->db_level == 0) { in dbuf_write_done()
5075 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_write_done()
5076 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); in dbuf_write_done()
5079 if (dr->dt.dl.dr_data != NULL && in dbuf_write_done()
5080 dr->dt.dl.dr_data != db->db_buf) { in dbuf_write_done()
5081 ASSERT3B(dr->dt.dl.dr_brtwrite, ==, B_FALSE); in dbuf_write_done()
5082 ASSERT3B(dr->dt.dl.dr_diowrite, ==, B_FALSE); in dbuf_write_done()
5083 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_write_done()
5086 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); in dbuf_write_done()
5087 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); in dbuf_write_done()
5088 if (!BP_IS_HOLE(db->db_blkptr)) { in dbuf_write_done()
5089 int epbs __maybe_unused = dn->dn_phys->dn_indblkshift - in dbuf_write_done()
5091 ASSERT3U(db->db_blkid, <=, in dbuf_write_done()
5092 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); in dbuf_write_done()
5093 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, in dbuf_write_done()
5094 db->db.db_size); in dbuf_write_done()
5096 mutex_destroy(&dr->dt.di.dr_mtx); in dbuf_write_done()
5097 list_destroy(&dr->dt.di.dr_children); in dbuf_write_done()
5100 cv_broadcast(&db->db_changed); in dbuf_write_done()
5101 ASSERT(db->db_dirtycnt > 0); in dbuf_write_done()
5102 db->db_dirtycnt -= 1; in dbuf_write_done()
5103 db->db_data_pending = NULL; in dbuf_write_done()
5104 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_write_done()
5106 dsl_pool_undirty_space(dmu_objset_pool(os), dr->dr_accounted, in dbuf_write_done()
5107 zio->io_txg); in dbuf_write_done()
5115 dbuf_write_ready(zio, NULL, zio->io_private); in dbuf_write_nofill_ready()
5121 dbuf_write_done(zio, NULL, zio->io_private); in dbuf_write_nofill_done()
5127 dbuf_dirty_record_t *dr = zio->io_private; in dbuf_write_override_ready()
5128 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_ready()
5136 dbuf_dirty_record_t *dr = zio->io_private; in dbuf_write_override_done()
5137 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_done()
5138 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; in dbuf_write_override_done()
5140 mutex_enter(&db->db_mtx); in dbuf_write_override_done()
5141 if (!BP_EQUAL(zio->io_bp, obp)) { in dbuf_write_override_done()
5143 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); in dbuf_write_override_done()
5144 arc_release(dr->dt.dl.dr_data, db); in dbuf_write_override_done()
5146 mutex_exit(&db->db_mtx); in dbuf_write_override_done()
5150 if (zio->io_abd != NULL) in dbuf_write_override_done()
5151 abd_free(zio->io_abd); in dbuf_write_override_done()
5165 objset_t *os = drica->drica_os; in dbuf_remap_impl_callback()
5167 dmu_tx_t *tx = drica->drica_tx; in dbuf_remap_impl_callback()
5175 size, drica->drica_blk_birth, tx); in dbuf_remap_impl_callback()
5183 spa_t *spa = dmu_objset_spa(dn->dn_objset); in dbuf_remap_impl()
5188 drica.drica_os = dn->dn_objset; in dbuf_remap_impl()
5201 if (dn->dn_objset != spa_meta_objset(spa)) { in dbuf_remap_impl()
5202 dsl_dataset_t *ds = dmu_objset_ds(dn->dn_objset); in dbuf_remap_impl()
5203 if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) && in dbuf_remap_impl()
5205 ds->ds_dir->dd_origin_txg) { in dbuf_remap_impl()
5207 ASSERT(dsl_dir_is_clone(ds->ds_dir)); in dbuf_remap_impl()
5210 bplist_append(&ds->ds_dir->dd_pending_frees, in dbuf_remap_impl()
5212 bplist_append(&ds->ds_dir->dd_pending_allocs, in dbuf_remap_impl()
5237 spa_t *spa = dmu_objset_spa(db->db_objset); in dbuf_remap()
5243 if (db->db_level > 0) { in dbuf_remap()
5244 blkptr_t *bp = db->db.db_data; in dbuf_remap()
5245 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { in dbuf_remap()
5246 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx); in dbuf_remap()
5248 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_remap()
5249 dnode_phys_t *dnp = db->db.db_data; in dbuf_remap()
5250 ASSERT3U(dn->dn_type, ==, DMU_OT_DNODE); in dbuf_remap()
5251 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; in dbuf_remap()
5254 krwlock_t *lock = (dn->dn_dbuf == NULL ? NULL : in dbuf_remap()
5255 &dn->dn_dbuf->db_rwlock); in dbuf_remap()
5265 * Populate dr->dr_zio with a zio to commit a dirty buffer to disk.
5266 * Caller is responsible for issuing the zio_[no]wait(dr->dr_zio).
5271 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write()
5272 dnode_t *dn = dr->dr_dnode; in dbuf_write()
5274 dmu_buf_impl_t *parent = db->db_parent; in dbuf_write()
5275 uint64_t txg = tx->tx_txg; in dbuf_write()
5283 os = dn->dn_objset; in dbuf_write()
5285 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { in dbuf_write()
5292 if (BP_IS_HOLE(db->db_blkptr)) in dbuf_write()
5299 if (parent != dn->dn_dbuf) { in dbuf_write()
5302 ASSERT(parent && parent->db_data_pending); in dbuf_write()
5304 ASSERT(db->db_level == parent->db_level-1); in dbuf_write()
5309 ASSERT(arc_released(parent->db_buf)); in dbuf_write()
5310 pio = parent->db_data_pending->dr_zio; in dbuf_write()
5313 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && in dbuf_write()
5314 db->db_blkid != DMU_SPILL_BLKID) || in dbuf_write()
5315 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); in dbuf_write()
5316 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_write()
5317 ASSERT3P(db->db_blkptr, ==, in dbuf_write()
5318 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_write()
5319 pio = dn->dn_zio; in dbuf_write()
5322 ASSERT(db->db_level == 0 || data == db->db_buf); in dbuf_write()
5323 ASSERT3U(BP_GET_LOGICAL_BIRTH(db->db_blkptr), <=, txg); in dbuf_write()
5326 SET_BOOKMARK(&zb, os->os_dsl_dataset ? in dbuf_write()
5327 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, in dbuf_write()
5328 db->db.db_object, db->db_level, db->db_blkid); in dbuf_write()
5330 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_write()
5334 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); in dbuf_write()
5342 dr->dr_bp_copy = *db->db_blkptr; in dbuf_write()
5344 if (db->db_level == 0 && in dbuf_write()
5345 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { in dbuf_write()
5352 abd_get_from_buf(data->b_data, arc_buf_size(data)) : NULL; in dbuf_write()
5354 dr->dr_zio = zio_write(pio, os->os_spa, txg, &dr->dr_bp_copy, in dbuf_write()
5355 contents, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
5359 mutex_enter(&db->db_mtx); in dbuf_write()
5360 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; in dbuf_write()
5361 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, in dbuf_write()
5362 dr->dt.dl.dr_copies, dr->dt.dl.dr_gang_copies, in dbuf_write()
5363 dr->dt.dl.dr_nopwrite, dr->dt.dl.dr_brtwrite); in dbuf_write()
5364 mutex_exit(&db->db_mtx); in dbuf_write()
5368 dr->dr_zio = zio_write(pio, os->os_spa, txg, in dbuf_write()
5369 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
5383 if (db->db_level != 0) in dbuf_write()
5386 dr->dr_zio = arc_write(pio, os->os_spa, txg, in dbuf_write()
5387 &dr->dr_bp_copy, data, !DBUF_IS_CACHEABLE(db), in dbuf_write()