Lines Matching refs:dn
374 dnode_t *dn; in dbuf_find_bonus() local
377 if (dnode_hold(os, object, FTAG, &dn) == 0) { in dbuf_find_bonus()
378 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_find_bonus()
379 if (dn->dn_bonus != NULL) { in dbuf_find_bonus()
380 db = dn->dn_bonus; in dbuf_find_bonus()
383 rw_exit(&dn->dn_struct_rwlock); in dbuf_find_bonus()
384 dnode_rele(dn, FTAG); in dbuf_find_bonus()
667 dnode_level_is_l2cacheable(blkptr_t *bp, dnode_t *dn, int64_t level) in dnode_level_is_l2cacheable() argument
669 if (dn->dn_objset->os_secondary_cache == ZFS_CACHE_ALL || in dnode_level_is_l2cacheable()
670 (dn->dn_objset->os_secondary_cache == ZFS_CACHE_METADATA && in dnode_level_is_l2cacheable()
672 DMU_OT_IS_METADATA(dn->dn_handle->dnh_dnode->dn_type)))) { in dnode_level_is_l2cacheable()
679 vdev_t *rvd = dn->dn_objset->os_spa->spa_root_vdev; in dnode_level_is_l2cacheable()
1110 dnode_t *dn; in dbuf_verify() local
1121 dn = DB_DNODE(db); in dbuf_verify()
1122 if (dn == NULL) { in dbuf_verify()
1126 ASSERT3U(db->db.db_object, ==, dn->dn_object); in dbuf_verify()
1127 ASSERT3P(db->db_objset, ==, dn->dn_objset); in dbuf_verify()
1128 ASSERT3U(db->db_level, <, dn->dn_nlevels); in dbuf_verify()
1131 !avl_is_empty(&dn->dn_dbufs)); in dbuf_verify()
1134 ASSERT(dn != NULL); in dbuf_verify()
1135 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_verify()
1138 ASSERT(dn != NULL); in dbuf_verify()
1171 if (db->db_parent == dn->dn_dbuf) { in dbuf_verify()
1180 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_verify()
1196 db->db_state != DB_FILL && (dn == NULL || !dn->dn_free_txg)) { in dbuf_verify()
1288 dbuf_whichblock(const dnode_t *dn, const int64_t level, const uint64_t offset) in dbuf_whichblock() argument
1290 if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { in dbuf_whichblock()
1311 const unsigned exp = dn->dn_datablkshift + in dbuf_whichblock()
1312 level * (dn->dn_indblkshift - SPA_BLKPTRSHIFT); in dbuf_whichblock()
1316 ASSERT3U(level, ==, dn->dn_nlevels - 1); in dbuf_whichblock()
1324 ASSERT3U(offset, <, dn->dn_datablksz); in dbuf_whichblock()
1417 dbuf_read_bonus(dmu_buf_impl_t *db, dnode_t *dn) in dbuf_read_bonus() argument
1422 bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); in dbuf_read_bonus()
1423 max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_read_bonus()
1432 memcpy(db_data, DN_BONUS(dn->dn_phys), bonuslen); in dbuf_read_bonus()
1440 dbuf_handle_indirect_hole(void *data, dnode_t *dn, blkptr_t *dbbp) in dbuf_handle_indirect_hole() argument
1443 uint32_t indbs = 1ULL << dn->dn_indblkshift; in dbuf_handle_indirect_hole()
1451 dn->dn_datablksz : BP_GET_LSIZE(dbbp)); in dbuf_handle_indirect_hole()
1464 dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *bp) in dbuf_read_hole() argument
1477 is_hole = dnode_block_freed(dn, db->db_blkid) || BP_IS_HOLE(bp); in dbuf_read_hole()
1485 dbuf_handle_indirect_hole(db_data->b_data, dn, bp); in dbuf_read_hole()
1508 dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, dnode_t *dn, in dbuf_read_verify_dnode_crypt() argument
1519 (dndb = dn->dn_dbuf) == NULL) in dbuf_read_verify_dnode_crypt()
1556 !DMU_OT_IS_ENCRYPTED(dn->dn_type)) || in dbuf_read_verify_dnode_crypt()
1558 !DMU_OT_IS_ENCRYPTED(dn->dn_bonustype)))) in dbuf_read_verify_dnode_crypt()
1571 dbuf_read_impl(dmu_buf_impl_t *db, dnode_t *dn, zio_t *zio, dmu_flags_t flags, in dbuf_read_impl() argument
1586 err = dbuf_read_bonus(db, dn); in dbuf_read_impl()
1590 err = dbuf_read_hole(db, dn, bp); in dbuf_read_impl()
1696 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data() local
1697 int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_fix_old_data()
1702 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data() local
1719 dmu_objset_id(dn->dn_objset), byteorder, salt, iv, in dbuf_fix_old_data()
1720 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf), in dbuf_fix_old_data()
1740 dnode_t *dn; in dbuf_read() local
1747 dn = DB_DNODE(db); in dbuf_read()
1753 err = dbuf_read_verify_dnode_crypt(db, dn, flags); in dbuf_read()
1810 spa_t *spa = dn->dn_objset->os_spa; in dbuf_read()
1836 spa_t *spa = dn->dn_objset->os_spa; in dbuf_read()
1843 dbuf_read_impl(db, dn, pio, flags, dblt, bp, FTAG); in dbuf_read()
1853 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, miss, in dbuf_read()
1969 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, in dbuf_free_range() argument
1978 if (end_blkid > dn->dn_maxblkid && in dbuf_free_range()
1980 end_blkid = dn->dn_maxblkid; in dbuf_free_range()
1981 dprintf_dnode(dn, "start=%llu end=%llu\n", (u_longlong_t)start_blkid, in dbuf_free_range()
1989 mutex_enter(&dn->dn_dbufs_mtx); in dbuf_free_range()
1990 db = avl_find(&dn->dn_dbufs, db_search, &where); in dbuf_free_range()
1993 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dbuf_free_range()
1996 db_next = AVL_NEXT(&dn->dn_dbufs, db); in dbuf_free_range()
2040 db->db_blkid > dn->dn_maxblkid) in dbuf_free_range()
2041 dn->dn_maxblkid = db->db_blkid; in dbuf_free_range()
2066 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_free_range()
2077 dnode_t *dn; in dbuf_new_size() local
2082 dn = DB_DNODE(db); in dbuf_new_size()
2093 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); in dbuf_new_size()
2117 dmu_objset_willuse_space(dn->dn_objset, size - osize, tx); in dbuf_new_size()
2161 dbuf_dirty_lightweight(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx) in dbuf_dirty_lightweight() argument
2163 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_dirty_lightweight()
2164 IMPLY(dn->dn_objset->os_raw_receive, dn->dn_maxblkid >= blkid); in dbuf_dirty_lightweight()
2165 dnode_new_blkid(dn, blkid, tx, B_TRUE, B_FALSE); in dbuf_dirty_lightweight()
2166 ASSERT(dn->dn_maxblkid >= blkid); in dbuf_dirty_lightweight()
2171 dr->dr_dnode = dn; in dbuf_dirty_lightweight()
2174 dr->dr_accounted = dn->dn_datablksz; in dbuf_dirty_lightweight()
2181 ASSERT3P(NULL, ==, dbuf_find(dn->dn_objset, dn->dn_object, 0, blkid, in dbuf_dirty_lightweight()
2184 mutex_enter(&dn->dn_mtx); in dbuf_dirty_lightweight()
2186 if (dn->dn_free_ranges[txgoff] != NULL) { in dbuf_dirty_lightweight()
2187 zfs_range_tree_clear(dn->dn_free_ranges[txgoff], blkid, 1); in dbuf_dirty_lightweight()
2190 if (dn->dn_nlevels == 1) { in dbuf_dirty_lightweight()
2191 ASSERT3U(blkid, <, dn->dn_nblkptr); in dbuf_dirty_lightweight()
2192 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); in dbuf_dirty_lightweight()
2193 mutex_exit(&dn->dn_mtx); in dbuf_dirty_lightweight()
2194 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty_lightweight()
2195 dnode_setdirty(dn, tx); in dbuf_dirty_lightweight()
2197 mutex_exit(&dn->dn_mtx); in dbuf_dirty_lightweight()
2199 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_dirty_lightweight()
2200 dmu_buf_impl_t *parent_db = dbuf_hold_level(dn, in dbuf_dirty_lightweight()
2202 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty_lightweight()
2224 dmu_objset_willuse_space(dn->dn_objset, dr->dr_accounted, tx); in dbuf_dirty_lightweight()
2232 dnode_t *dn; in dbuf_dirty() local
2243 dn = DB_DNODE(db); in dbuf_dirty()
2250 if (dn->dn_objset->os_dsl_dataset != NULL) { in dbuf_dirty()
2251 rrw_enter(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, in dbuf_dirty()
2255 BP_IS_HOLE(dn->dn_objset->os_rootbp) || in dbuf_dirty()
2256 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || in dbuf_dirty()
2257 dn->dn_objset->os_dsl_dataset == NULL); in dbuf_dirty()
2258 if (dn->dn_objset->os_dsl_dataset != NULL) in dbuf_dirty()
2259 rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, FTAG); in dbuf_dirty()
2266 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || in dbuf_dirty()
2267 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == in dbuf_dirty()
2280 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2281 dnode_set_dirtyctx(dn, tx, db); in dbuf_dirty()
2282 if (tx->tx_txg > dn->dn_dirty_txg) in dbuf_dirty()
2283 dn->dn_dirty_txg = tx->tx_txg; in dbuf_dirty()
2284 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2287 dn->dn_have_spill = B_TRUE; in dbuf_dirty()
2307 ASSERT(dn->dn_object == 0 || in dbuf_dirty()
2308 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == in dbuf_dirty()
2311 ASSERT3U(dn->dn_nlevels, >, db->db_level); in dbuf_dirty()
2320 os = dn->dn_objset; in dbuf_dirty()
2323 if (dn->dn_objset->os_dsl_dataset != NULL) in dbuf_dirty()
2325 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || in dbuf_dirty()
2327 if (dn->dn_objset->os_dsl_dataset != NULL) in dbuf_dirty()
2347 dr->dr_dnode = dn; in dbuf_dirty()
2392 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2393 if (dn->dn_free_ranges[txgoff] != NULL) { in dbuf_dirty()
2394 zfs_range_tree_clear(dn->dn_free_ranges[txgoff], in dbuf_dirty()
2397 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2412 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2414 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); in dbuf_dirty()
2415 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2416 dnode_setdirty(dn, tx); in dbuf_dirty()
2421 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { in dbuf_dirty()
2422 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_dirty()
2442 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || in dbuf_dirty()
2443 dn->dn_phys->dn_nlevels > db->db_level || in dbuf_dirty()
2444 dn->dn_next_nlevels[txgoff] > db->db_level || in dbuf_dirty()
2445 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || in dbuf_dirty()
2446 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); in dbuf_dirty()
2451 dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2452 dnode_new_blkid(dn, db->db_blkid, tx, in dbuf_dirty()
2454 ASSERT(dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
2457 if (db->db_level+1 < dn->dn_nlevels) { in dbuf_dirty()
2462 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { in dbuf_dirty()
2463 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_dirty()
2464 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_dirty()
2470 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty()
2482 dn->dn_object == DMU_META_DNODE_OBJECT) { in dbuf_dirty()
2492 ASSERT(db->db_level + 1 == dn->dn_nlevels); in dbuf_dirty()
2493 ASSERT(db->db_blkid < dn->dn_nblkptr); in dbuf_dirty()
2494 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); in dbuf_dirty()
2495 mutex_enter(&dn->dn_mtx); in dbuf_dirty()
2497 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); in dbuf_dirty()
2498 mutex_exit(&dn->dn_mtx); in dbuf_dirty()
2500 rw_exit(&dn->dn_struct_rwlock); in dbuf_dirty()
2503 dnode_setdirty(dn, tx); in dbuf_dirty()
2515 struct dnode *dn = dr->dr_dnode; in dbuf_undirty_bonus() local
2516 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_undirty_bonus()
2582 dnode_t *dn = dr->dr_dnode; in dbuf_undirty() local
2588 dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), in dbuf_undirty()
2604 db->db_level + 1 == dn->dn_nlevels) { in dbuf_undirty()
2605 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); in dbuf_undirty()
2606 mutex_enter(&dn->dn_mtx); in dbuf_undirty()
2607 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); in dbuf_undirty()
2608 mutex_exit(&dn->dn_mtx); in dbuf_undirty()
2759 dnode_t *dn = DB_DNODE(db); in dmu_buf_untransform_direct() local
2768 err = dbuf_read_verify_dnode_crypt(db, dn, 0); in dmu_buf_untransform_direct()
3180 dnode_t *dn; in dbuf_destroy() local
3243 dn = DB_DNODE(db); in dbuf_destroy()
3244 dndb = dn->dn_dbuf; in dbuf_destroy()
3246 boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx); in dbuf_destroy()
3248 mutex_enter_nested(&dn->dn_dbufs_mtx, in dbuf_destroy()
3250 avl_remove(&dn->dn_dbufs, db); in dbuf_destroy()
3254 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_destroy()
3263 mutex_enter(&dn->dn_mtx); in dbuf_destroy()
3264 dnode_rele_and_unlock(dn, db, B_TRUE); in dbuf_destroy()
3309 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, in dbuf_findbp() argument
3318 mutex_enter(&dn->dn_mtx); in dbuf_findbp()
3319 if (dn->dn_have_spill && in dbuf_findbp()
3320 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) in dbuf_findbp()
3321 *bpp = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_findbp()
3324 dbuf_add_ref(dn->dn_dbuf, NULL); in dbuf_findbp()
3325 *parentp = dn->dn_dbuf; in dbuf_findbp()
3326 mutex_exit(&dn->dn_mtx); in dbuf_findbp()
3331 (dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels; in dbuf_findbp()
3332 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_findbp()
3335 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_findbp()
3353 highbit64(dn->dn_phys->dn_nblkptr) <= 64); in dbuf_findbp()
3355 blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr << in dbuf_findbp()
3358 blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { in dbuf_findbp()
3365 err = dbuf_hold_impl(dn, level + 1, in dbuf_findbp()
3383 ASSERT(dn->dn_phys->dn_nblkptr == 0 || in dbuf_findbp()
3384 blkid < dn->dn_phys->dn_nblkptr); in dbuf_findbp()
3385 if (dn->dn_dbuf) { in dbuf_findbp()
3386 dbuf_add_ref(dn->dn_dbuf, NULL); in dbuf_findbp()
3387 *parentp = dn->dn_dbuf; in dbuf_findbp()
3389 *bpp = &dn->dn_phys->dn_blkptr[blkid]; in dbuf_findbp()
3395 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, in dbuf_create() argument
3398 objset_t *os = dn->dn_objset; in dbuf_create()
3401 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_create()
3402 ASSERT(dn->dn_type != DMU_OT_NONE); in dbuf_create()
3410 db->db.db_object = dn->dn_object; in dbuf_create()
3415 db->db_dnode_handle = dn->dn_handle; in dbuf_create()
3417 db->db_dnode = dn; in dbuf_create()
3430 ASSERT3P(parent, ==, dn->dn_dbuf); in dbuf_create()
3431 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dbuf_create()
3432 (dn->dn_nblkptr-1) * sizeof (blkptr_t); in dbuf_create()
3433 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_create()
3447 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; in dbuf_create()
3459 mutex_enter(&dn->dn_dbufs_mtx); in dbuf_create()
3463 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_create()
3468 avl_add(&dn->dn_dbufs, db); in dbuf_create()
3473 mutex_exit(&dn->dn_dbufs_mtx); in dbuf_create()
3476 if (parent && parent != dn->dn_dbuf) in dbuf_create()
3479 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || in dbuf_create()
3480 zfs_refcount_count(&dn->dn_holds) > 0); in dbuf_create()
3481 (void) zfs_refcount_add(&dn->dn_holds, db); in dbuf_create()
3496 dbuf_dnode_findbp(dnode_t *dn, uint64_t level, uint64_t blkid, in dbuf_dnode_findbp() argument
3502 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_dnode_findbp()
3504 err = dbuf_findbp(dn, level, blkid, B_FALSE, &dbp, &bp2); in dbuf_dnode_findbp()
3511 *datablkszsec = dn->dn_phys->dn_datablkszsec; in dbuf_dnode_findbp()
3513 *indblkshift = dn->dn_phys->dn_indblkshift; in dbuf_dnode_findbp()
3697 dbuf_prefetch_impl(dnode_t *dn, int64_t level, uint64_t blkid, in dbuf_prefetch_impl() argument
3706 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_prefetch_impl()
3708 if (blkid > dn->dn_maxblkid) in dbuf_prefetch_impl()
3711 if (level == 0 && dnode_block_freed(dn, blkid)) in dbuf_prefetch_impl()
3718 nlevels = dn->dn_phys->dn_nlevels; in dbuf_prefetch_impl()
3719 if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) in dbuf_prefetch_impl()
3722 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_prefetch_impl()
3723 if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) in dbuf_prefetch_impl()
3726 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, in dbuf_prefetch_impl()
3749 if (dbuf_hold_impl(dn, parent_level, parent_blkid, in dbuf_prefetch_impl()
3763 ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); in dbuf_prefetch_impl()
3764 bp = dn->dn_phys->dn_blkptr[curblkid]; in dbuf_prefetch_impl()
3767 dsl_dataset_feature_is_active(dn->dn_objset->os_dsl_dataset, in dbuf_prefetch_impl()
3774 zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, in dbuf_prefetch_impl()
3778 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; in dbuf_prefetch_impl()
3780 dn->dn_object, level, blkid); in dbuf_prefetch_impl()
3784 dpa->dpa_spa = dn->dn_objset->os_spa; in dbuf_prefetch_impl()
3785 dpa->dpa_dnode = dn; in dbuf_prefetch_impl()
3791 if (!DNODE_LEVEL_IS_CACHEABLE(dn, level)) in dbuf_prefetch_impl()
3793 else if (dnode_level_is_l2cacheable(&bp, dn, level)) in dbuf_prefetch_impl()
3811 if (dnode_level_is_l2cacheable(&bp, dn, curlevel)) in dbuf_prefetch_impl()
3815 dn->dn_object, curlevel, curblkid); in dbuf_prefetch_impl()
3835 dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio, in dbuf_prefetch() argument
3839 return (dbuf_prefetch_impl(dn, level, blkid, prio, aflags, NULL, NULL)); in dbuf_prefetch()
3851 dbuf_hold_copy(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_hold_copy() argument
3866 db_data = arc_alloc_raw_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3867 dmu_objset_id(dn->dn_objset), byteorder, salt, iv, mac, in dbuf_hold_copy()
3868 dn->dn_type, arc_buf_size(data), arc_buf_lsize(data), in dbuf_hold_copy()
3872 dn->dn_objset->os_spa, db, arc_buf_size(data), in dbuf_hold_copy()
3875 db_data = arc_alloc_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
3888 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, in dbuf_hold_impl() argument
3896 spa_t *spa = dn->dn_objset->os_spa; in dbuf_hold_impl()
3903 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dbuf_hold_impl()
3905 ASSERT3U(dn->dn_nlevels, >, level); in dbuf_hold_impl()
3910 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid, &hv); in dbuf_hold_impl()
3920 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); in dbuf_hold_impl()
3932 db = dbuf_create(dn, level, blkid, parent, bp, hv); in dbuf_hold_impl()
3954 dn->dn_object != DMU_META_DNODE_OBJECT && in dbuf_hold_impl()
3959 dbuf_hold_copy(dn, db); in dbuf_hold_impl()
3996 ASSERT3P(DB_DNODE(db), ==, dn); in dbuf_hold_impl()
4005 dbuf_hold(dnode_t *dn, uint64_t blkid, const void *tag) in dbuf_hold() argument
4007 return (dbuf_hold_level(dn, 0, blkid, tag)); in dbuf_hold()
4011 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, const void *tag) in dbuf_hold_level() argument
4014 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); in dbuf_hold_level()
4019 dbuf_create_bonus(dnode_t *dn) in dbuf_create_bonus() argument
4021 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dbuf_create_bonus()
4023 ASSERT(dn->dn_bonus == NULL); in dbuf_create_bonus()
4024 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL, in dbuf_create_bonus()
4025 dbuf_hash(dn->dn_objset, dn->dn_object, 0, DMU_BONUS_BLKID)); in dbuf_create_bonus()
4026 dn->dn_bonus->db_pending_evict = FALSE; in dbuf_create_bonus()
4047 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) in dbuf_rm_spill() argument
4049 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); in dbuf_rm_spill()
4149 dnode_t *dn; in dbuf_rele_and_unlock() local
4158 dn = DB_DNODE(db); in dbuf_rele_and_unlock()
4159 atomic_dec_32(&dn->dn_dbufs_count); in dbuf_rele_and_unlock()
4176 dnode_evict_bonus(dn); in dbuf_rele_and_unlock()
4178 dnode_rele(dn, db); in dbuf_rele_and_unlock()
4355 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_check_blkptr() argument
4364 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_check_blkptr()
4368 if (db->db_level == dn->dn_phys->dn_nlevels-1) { in dbuf_check_blkptr()
4374 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); in dbuf_check_blkptr()
4376 db->db_parent = dn->dn_dbuf; in dbuf_check_blkptr()
4377 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; in dbuf_check_blkptr()
4381 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_check_blkptr()
4383 ASSERT(dn->dn_phys->dn_nlevels > 1); in dbuf_check_blkptr()
4386 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dbuf_check_blkptr()
4387 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_check_blkptr()
4389 rw_exit(&dn->dn_struct_rwlock); in dbuf_check_blkptr()
4410 dnode_t *dn = dr->dr_dnode; in dbuf_sync_bonus() local
4411 ASSERT3U(DN_MAX_BONUS_LEN(dn->dn_phys), <=, in dbuf_sync_bonus()
4412 DN_SLOTS_TO_BONUSLEN(dn->dn_phys->dn_extra_slots + 1)); in dbuf_sync_bonus()
4413 memcpy(DN_BONUS(dn->dn_phys), data, DN_MAX_BONUS_LEN(dn->dn_phys)); in dbuf_sync_bonus()
4472 dnode_t *dn = dr->dr_dnode; in dbuf_sync_indirect() local
4493 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_sync_indirect()
4494 dbuf_check_blkptr(dn, db); in dbuf_sync_indirect()
4528 dnode_t *dn = dr->dr_dnode; in dbuf_sync_leaf_verify_bonus_dnode() local
4534 if (DMU_OT_IS_ENCRYPTED(dn->dn_bonustype)) in dbuf_sync_leaf_verify_bonus_dnode()
4537 uint16_t bonuslen = dn->dn_phys->dn_bonuslen; in dbuf_sync_leaf_verify_bonus_dnode()
4538 uint16_t maxbonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dbuf_sync_leaf_verify_bonus_dnode()
4556 dnode_t *dn = dr->dr_dnode; in dbuf_lightweight_bp() local
4558 if (dn->dn_phys->dn_nlevels == 1) { in dbuf_lightweight_bp()
4559 VERIFY3U(dr->dt.dll.dr_blkid, <, dn->dn_phys->dn_nblkptr); in dbuf_lightweight_bp()
4560 return (&dn->dn_phys->dn_blkptr[dr->dt.dll.dr_blkid]); in dbuf_lightweight_bp()
4563 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dbuf_lightweight_bp()
4565 VERIFY3P(DB_DNODE(parent_db), ==, dn); in dbuf_lightweight_bp()
4581 dnode_t *dn = dr->dr_dnode; in dbuf_lightweight_ready() local
4584 spa_t *spa = dmu_objset_spa(dn->dn_objset); in dbuf_lightweight_ready()
4587 dnode_diduse_space(dn, delta); in dbuf_lightweight_ready()
4590 mutex_enter(&dn->dn_mtx); in dbuf_lightweight_ready()
4591 if (blkid > dn->dn_phys->dn_maxblkid) { in dbuf_lightweight_ready()
4592 ASSERT0(dn->dn_objset->os_raw_receive); in dbuf_lightweight_ready()
4593 dn->dn_phys->dn_maxblkid = blkid; in dbuf_lightweight_ready()
4595 mutex_exit(&dn->dn_mtx); in dbuf_lightweight_ready()
4603 EQUIV(dr->dr_parent == NULL, dn->dn_phys->dn_nlevels == 1); in dbuf_lightweight_ready()
4605 parent_db = dn->dn_dbuf; in dbuf_lightweight_ready()
4642 dnode_t *dn = dr->dr_dnode; in dbuf_sync_lightweight() local
4644 if (dn->dn_phys->dn_nlevels == 1) { in dbuf_sync_lightweight()
4645 pio = dn->dn_zio; in dbuf_sync_lightweight()
4651 .zb_objset = dmu_objset_id(dn->dn_objset), in dbuf_sync_lightweight()
4652 .zb_object = dn->dn_object, in dbuf_sync_lightweight()
4663 dr->dr_zio = zio_write(pio, dmu_objset_spa(dn->dn_objset), in dbuf_sync_lightweight()
4665 dn->dn_datablksz, abd_get_size(dr->dt.dll.dr_abd), in dbuf_sync_lightweight()
4683 dnode_t *dn = dr->dr_dnode; in dbuf_sync_leaf() local
4725 mutex_enter(&dn->dn_mtx); in dbuf_sync_leaf()
4726 if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) { in dbuf_sync_leaf()
4742 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; in dbuf_sync_leaf()
4743 mutex_exit(&dn->dn_mtx); in dbuf_sync_leaf()
4758 os = dn->dn_objset; in dbuf_sync_leaf()
4766 dbuf_check_blkptr(dn, db); in dbuf_sync_leaf()
4777 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); in dbuf_sync_leaf()
4785 if (os->os_encrypted && dn->dn_object == DMU_META_DNODE_OBJECT) in dbuf_sync_leaf()
4789 dn->dn_object != DMU_META_DNODE_OBJECT && in dbuf_sync_leaf()
4817 dn->dn_type, psize, lsize, compress_type, in dbuf_sync_leaf()
4835 if (dn->dn_object == DMU_META_DNODE_OBJECT) { in dbuf_sync_leaf()
4836 list_insert_tail(&dn->dn_dirty_records[txg & TXG_MASK], dr); in dbuf_sync_leaf()
4885 dnode_t *dn; in dbuf_write_ready() local
4897 dn = DB_DNODE(db); in dbuf_write_ready()
4899 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); in dbuf_write_ready()
4904 BP_GET_TYPE(bp) == dn->dn_type) || in dbuf_write_ready()
4906 BP_GET_TYPE(bp) == dn->dn_bonustype) || in dbuf_write_ready()
4915 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); in dbuf_write_ready()
4917 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_ready()
4922 mutex_enter(&dn->dn_mtx); in dbuf_write_ready()
4923 if (db->db_blkid > dn->dn_phys->dn_maxblkid && in dbuf_write_ready()
4926 dn->dn_phys->dn_maxblkid = db->db_blkid; in dbuf_write_ready()
4928 mutex_exit(&dn->dn_mtx); in dbuf_write_ready()
4930 if (dn->dn_type == DMU_OT_DNODE) { in dbuf_write_ready()
4966 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_write_ready()
5060 dnode_t *dn = dr->dr_dnode; in dbuf_write_done() local
5068 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); in dbuf_write_done()
5070 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_done()
5087 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); in dbuf_write_done()
5089 int epbs __maybe_unused = dn->dn_phys->dn_indblkshift - in dbuf_write_done()
5092 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); in dbuf_write_done()
5180 dbuf_remap_impl(dnode_t *dn, blkptr_t *bp, krwlock_t *rw, dmu_tx_t *tx) in dbuf_remap_impl() argument
5183 spa_t *spa = dmu_objset_spa(dn->dn_objset); in dbuf_remap_impl()
5188 drica.drica_os = dn->dn_objset; in dbuf_remap_impl()
5201 if (dn->dn_objset != spa_meta_objset(spa)) { in dbuf_remap_impl()
5202 dsl_dataset_t *ds = dmu_objset_ds(dn->dn_objset); in dbuf_remap_impl()
5235 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_remap() argument
5246 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx); in dbuf_remap()
5250 ASSERT3U(dn->dn_type, ==, DMU_OT_DNODE); in dbuf_remap()
5254 krwlock_t *lock = (dn->dn_dbuf == NULL ? NULL : in dbuf_remap()
5255 &dn->dn_dbuf->db_rwlock); in dbuf_remap()
5256 dbuf_remap_impl(dn, &dnp[i].dn_blkptr[j], lock, in dbuf_remap()
5272 dnode_t *dn = dr->dr_dnode; in dbuf_write() local
5283 os = dn->dn_objset; in dbuf_write()
5285 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { in dbuf_write()
5296 dbuf_remap(dn, db, tx); in dbuf_write()
5299 if (parent != dn->dn_dbuf) { in dbuf_write()
5313 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && in dbuf_write()
5318 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_write()
5319 pio = dn->dn_zio; in dbuf_write()
5334 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); in dbuf_write()