42 dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)  in dnode_increase_indirection()  argument
46 	int nblkptr = dn->dn_phys->dn_nblkptr;  in dnode_increase_indirection()
47 	int old_toplvl = dn->dn_phys->dn_nlevels - 1;  in dnode_increase_indirection()
48 	int new_level = dn->dn_next_nlevels[txgoff];  in dnode_increase_indirection()
51 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_increase_indirection()
54 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_increase_indirection()
55 	ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);  in dnode_increase_indirection()
57 	db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);  in dnode_increase_indirection()
60 	dn->dn_phys->dn_nlevels = new_level;  in dnode_increase_indirection()
61 	dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,  in dnode_increase_indirection()
62 	    dn->dn_object, dn->dn_phys->dn_nlevels);  in dnode_increase_indirection()
74 		    dbuf_find(dn->dn_objset, dn->dn_object, old_toplvl, i);  in dnode_increase_indirection()
79 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
80 		rw_enter(&dn->dn_dbuf->db_rwlock, RW_WRITER);  in dnode_increase_indirection()
85 	bcopy(dn->dn_phys->dn_blkptr, db->db.db_data,  in dnode_increase_indirection()
97 		ASSERT3P(DB_DNODE(child), ==, dn);  in dnode_increase_indirection()
100 		if (child->db_parent && child->db_parent != dn->dn_dbuf) {  in dnode_increase_indirection()
103 			    &dn->dn_phys->dn_blkptr[child->db_blkid]);  in dnode_increase_indirection()
108 		    child->db_parent == dn->dn_dbuf);  in dnode_increase_indirection()
122 	bzero(dn->dn_phys->dn_blkptr, sizeof (blkptr_t) * nblkptr);  in dnode_increase_indirection()
125 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
126 		rw_exit(&dn->dn_dbuf->db_rwlock);  in dnode_increase_indirection()
130 	rw_exit(&dn->dn_struct_rwlock);  in dnode_increase_indirection()
134 free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx)  in free_blocks()  argument
136 	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in free_blocks()
139 	dprintf("ds=%p obj=%llx num=%d\n", ds, dn->dn_object, num);  in free_blocks()
146 		ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys));  in free_blocks()
162 		if (spa_feature_is_active(dn->dn_objset->os_spa,  in free_blocks()
170 	dnode_diduse_space(dn, -bytesfreed);  in free_blocks()
180 	dnode_t *dn;  in free_verify()  local
183 	dn = DB_DNODE(db);  in free_verify()
184 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_verify()
191 	ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);  in free_verify()
203 		rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_verify()
204 		err = dbuf_hold_impl(dn, db->db_level - 1,  in free_verify()
206 		rw_exit(&dn->dn_struct_rwlock);  in free_verify()
271 	dnode_t *dn;  in free_children()  local
310 	dn = DB_DNODE(db);  in free_children()
311 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_children()
331 		free_blocks(dn, bp, end - start + 1, tx);  in free_children()
337 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_children()
338 			VERIFY0(dbuf_hold_impl(dn, db->db_level - 1,  in free_children()
340 			rw_exit(&dn->dn_struct_rwlock);  in free_children()
353 		free_blocks(dn, db->db_blkptr, 1, tx);  in free_children()
366 dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks,  in dnode_sync_free_range_impl()  argument
369 	blkptr_t *bp = dn->dn_phys->dn_blkptr;  in dnode_sync_free_range_impl()
370 	int dnlevel = dn->dn_phys->dn_nlevels;  in dnode_sync_free_range_impl()
373 	if (blkid > dn->dn_phys->dn_maxblkid)  in dnode_sync_free_range_impl()
376 	ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX);  in dnode_sync_free_range_impl()
377 	if (blkid + nblks > dn->dn_phys->dn_maxblkid) {  in dnode_sync_free_range_impl()
378 		nblks = dn->dn_phys->dn_maxblkid - blkid + 1;  in dnode_sync_free_range_impl()
384 		if (blkid >= dn->dn_phys->dn_nblkptr) {  in dnode_sync_free_range_impl()
388 		ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
389 		free_blocks(dn, bp + blkid, nblks, tx);  in dnode_sync_free_range_impl()
392 		    (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT);  in dnode_sync_free_range_impl()
397 		ASSERT(start < dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
402 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in dnode_sync_free_range_impl()
403 			VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i,  in dnode_sync_free_range_impl()
405 			rw_exit(&dn->dn_struct_rwlock);  in dnode_sync_free_range_impl()
425 	if (trunc && !dn->dn_objset->os_raw_receive) {  in dnode_sync_free_range_impl()
426 		dn->dn_phys->dn_maxblkid = blkid == 0 ? 0 : blkid - 1;  in dnode_sync_free_range_impl()
428 		uint64_t off = (dn->dn_phys->dn_maxblkid + 1) *  in dnode_sync_free_range_impl()
429 		    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);  in dnode_sync_free_range_impl()
430 		ASSERT(off < dn->dn_phys->dn_maxblkid ||  in dnode_sync_free_range_impl()
431 		    dn->dn_phys->dn_maxblkid == 0 ||  in dnode_sync_free_range_impl()
432 		    dnode_next_offset(dn, 0, &off, 1, 1, 0) != 0);  in dnode_sync_free_range_impl()
446 	dnode_t *dn = dsfra->dsfra_dnode;  in dnode_sync_free_range()  local
448 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free_range()
449 	dnode_sync_free_range_impl(dn, blkid, nblks,  in dnode_sync_free_range()
451 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free_range()
458 dnode_evict_dbufs(dnode_t *dn)  in dnode_evict_dbufs()  argument
463 	mutex_enter(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
464 	for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {  in dnode_evict_dbufs()
468 		ASSERT3P(DB_DNODE(db), ==, dn);  in dnode_evict_dbufs()
478 			avl_insert_here(&dn->dn_dbufs, &db_marker, db,  in dnode_evict_dbufs()
496 			db_next = AVL_NEXT(&dn->dn_dbufs, &db_marker);  in dnode_evict_dbufs()
497 			avl_remove(&dn->dn_dbufs, &db_marker);  in dnode_evict_dbufs()
501 			db_next = AVL_NEXT(&dn->dn_dbufs, db);  in dnode_evict_dbufs()
504 	mutex_exit(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
506 	dnode_evict_bonus(dn);  in dnode_evict_dbufs()
510 dnode_evict_bonus(dnode_t *dn)  in dnode_evict_bonus()  argument
512 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_evict_bonus()
513 	if (dn->dn_bonus != NULL) {  in dnode_evict_bonus()
514 		if (zfs_refcount_is_zero(&dn->dn_bonus->db_holds)) {  in dnode_evict_bonus()
515 			mutex_enter(&dn->dn_bonus->db_mtx);  in dnode_evict_bonus()
516 			dbuf_destroy(dn->dn_bonus);  in dnode_evict_bonus()
517 			dn->dn_bonus = NULL;  in dnode_evict_bonus()
519 			dn->dn_bonus->db_pending_evict = TRUE;  in dnode_evict_bonus()
522 	rw_exit(&dn->dn_struct_rwlock);  in dnode_evict_bonus()
557 dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync_free()  argument
567 	ASSERT0(DN_USED_BYTES(dn->dn_phys));  in dnode_sync_free()
568 	ASSERT(BP_IS_HOLE(dn->dn_phys->dn_blkptr));  in dnode_sync_free()
570 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);  in dnode_sync_free()
571 	dnode_evict_dbufs(dn);  in dnode_sync_free()
584 	dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync_free()
585 	dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync_free()
586 	dn->dn_next_blksz[txgoff] = 0;  in dnode_sync_free()
587 	dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync_free()
590 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_sync_free()
591 	ASSERT(dn->dn_type != DMU_OT_NONE);  in dnode_sync_free()
593 	ASSERT(dn->dn_free_txg > 0);  in dnode_sync_free()
594 	if (dn->dn_allocated_txg != dn->dn_free_txg)  in dnode_sync_free()
595 		dmu_buf_will_dirty(&dn->dn_dbuf->db, tx);  in dnode_sync_free()
596 	bzero(dn->dn_phys, sizeof (dnode_phys_t) * dn->dn_num_slots);  in dnode_sync_free()
597 	dnode_free_interior_slots(dn);  in dnode_sync_free()
599 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free()
600 	dn->dn_type = DMU_OT_NONE;  in dnode_sync_free()
601 	dn->dn_maxblkid = 0;  in dnode_sync_free()
602 	dn->dn_allocated_txg = 0;  in dnode_sync_free()
603 	dn->dn_free_txg = 0;  in dnode_sync_free()
604 	dn->dn_have_spill = B_FALSE;  in dnode_sync_free()
605 	dn->dn_num_slots = 1;  in dnode_sync_free()
606 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free()
608 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);  in dnode_sync_free()
610 	dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync_free()
621 dnode_sync(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync()  argument
623 	objset_t *os = dn->dn_objset;  in dnode_sync()
624 	dnode_phys_t *dnp = dn->dn_phys;  in dnode_sync()
626 	list_t *list = &dn->dn_dirty_records[txgoff];  in dnode_sync()
631 	ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg);  in dnode_sync()
634 	DNODE_VERIFY(dn);  in dnode_sync()
636 	ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf));  in dnode_sync()
643 	    !DMU_OBJECT_IS_SPECIAL(dn->dn_object) &&  in dnode_sync()
645 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
646 		dn->dn_oldused = DN_USED_BYTES(dn->dn_phys);  in dnode_sync()
647 		dn->dn_oldflags = dn->dn_phys->dn_flags;  in dnode_sync()
648 		dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED;  in dnode_sync()
649 		if (dmu_objset_userobjused_enabled(dn->dn_objset))  in dnode_sync()
650 			dn->dn_phys->dn_flags |=  in dnode_sync()
652 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
653 		dmu_objset_userquota_get_ids(dn, B_FALSE, tx);  in dnode_sync()
656 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
658 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
662 	mutex_enter(&dn->dn_mtx);  in dnode_sync()
663 	if (dn->dn_allocated_txg == tx->tx_txg) {  in dnode_sync()
668 			dnp->dn_nblkptr = dn->dn_nblkptr;  in dnode_sync()
671 		dnp->dn_type = dn->dn_type;  in dnode_sync()
672 		dnp->dn_bonustype = dn->dn_bonustype;  in dnode_sync()
673 		dnp->dn_bonuslen = dn->dn_bonuslen;  in dnode_sync()
676 	dnp->dn_extra_slots = dn->dn_num_slots - 1;  in dnode_sync()
687 	if (dn->dn_next_type[txgoff] != 0) {  in dnode_sync()
688 		dnp->dn_type = dn->dn_type;  in dnode_sync()
689 		dn->dn_next_type[txgoff] = 0;  in dnode_sync()
692 	if (dn->dn_next_blksz[txgoff] != 0) {  in dnode_sync()
693 		ASSERT(P2PHASE(dn->dn_next_blksz[txgoff],  in dnode_sync()
696 		    dn->dn_maxblkid == 0 || list_head(list) != NULL ||  in dnode_sync()
697 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT ==  in dnode_sync()
699 		    !range_tree_is_empty(dn->dn_free_ranges[txgoff]));  in dnode_sync()
701 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT;  in dnode_sync()
702 		dn->dn_next_blksz[txgoff] = 0;  in dnode_sync()
705 	if (dn->dn_next_bonuslen[txgoff] != 0) {  in dnode_sync()
706 		if (dn->dn_next_bonuslen[txgoff] == DN_ZERO_BONUSLEN)  in dnode_sync()
709 			dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff];  in dnode_sync()
712 		dn->dn_next_bonuslen[txgoff] = 0;  in dnode_sync()
715 	if (dn->dn_next_bonustype[txgoff] != 0) {  in dnode_sync()
716 		ASSERT(DMU_OT_IS_VALID(dn->dn_next_bonustype[txgoff]));  in dnode_sync()
717 		dnp->dn_bonustype = dn->dn_next_bonustype[txgoff];  in dnode_sync()
718 		dn->dn_next_bonustype[txgoff] = 0;  in dnode_sync()
721 	boolean_t freeing_dnode = dn->dn_free_txg > 0 &&  in dnode_sync()
722 	    dn->dn_free_txg <= tx->tx_txg;  in dnode_sync()
728 	if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) {  in dnode_sync()
731 		dn->dn_rm_spillblk[txgoff] = 0;  in dnode_sync()
734 	if (dn->dn_next_indblkshift[txgoff] != 0) {  in dnode_sync()
736 		dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff];  in dnode_sync()
737 		dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync()
745 	dnp->dn_checksum = dn->dn_checksum;  in dnode_sync()
746 	dnp->dn_compress = dn->dn_compress;  in dnode_sync()
748 	mutex_exit(&dn->dn_mtx);  in dnode_sync()
751 		free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx);  in dnode_sync()
752 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
754 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
758 	if (dn->dn_free_ranges[txgoff] != NULL) {  in dnode_sync()
760 		dsfra.dsfra_dnode = dn;  in dnode_sync()
763 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
765 			ASSERT(range_tree_contains(dn->dn_free_ranges[txgoff],  in dnode_sync()
766 			    0, dn->dn_maxblkid + 1));  in dnode_sync()
776 		range_tree_walk(dn->dn_free_ranges[txgoff],  in dnode_sync()
778 		range_tree_vacate(dn->dn_free_ranges[txgoff], NULL, NULL);  in dnode_sync()
779 		range_tree_destroy(dn->dn_free_ranges[txgoff]);  in dnode_sync()
780 		dn->dn_free_ranges[txgoff] = NULL;  in dnode_sync()
781 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
785 		dn->dn_objset->os_freed_dnodes++;  in dnode_sync()
786 		dnode_sync_free(dn, tx);  in dnode_sync()
790 	if (dn->dn_num_slots > DNODE_MIN_SLOTS) {  in dnode_sync()
791 		dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in dnode_sync()
798 	if (dn->dn_next_nlevels[txgoff]) {  in dnode_sync()
799 		dnode_increase_indirection(dn, tx);  in dnode_sync()
800 		dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync()
808 	if (dn->dn_next_maxblkid[txgoff]) {  in dnode_sync()
809 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
811 		    dn->dn_next_maxblkid[txgoff] & ~DMU_NEXT_MAXBLKID_SET;  in dnode_sync()
812 		dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync()
813 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
816 	if (dn->dn_next_nblkptr[txgoff]) {  in dnode_sync()
818 		ASSERT(dn->dn_allocated_txg == tx->tx_txg);  in dnode_sync()
819 		if (dn->dn_next_nblkptr[txgoff] > dnp->dn_nblkptr) {  in dnode_sync()
823 			    (dn->dn_next_nblkptr[txgoff] - dnp->dn_nblkptr));  in dnode_sync()
827 			ASSERT(dn->dn_next_nblkptr[txgoff] < dnp->dn_nblkptr);  in dnode_sync()
829 			for (i = dn->dn_next_nblkptr[txgoff];  in dnode_sync()
834 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
835 		dnp->dn_nblkptr = dn->dn_next_nblkptr[txgoff];  in dnode_sync()
836 		dn->dn_next_nblkptr[txgoff] = 0;  in dnode_sync()
837 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
840 	dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);  in dnode_sync()
842 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {  in dnode_sync()
844 		dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync()