43 dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)  in dnode_increase_indirection()  argument
47 	int nblkptr = dn->dn_phys->dn_nblkptr;  in dnode_increase_indirection()
48 	int old_toplvl = dn->dn_phys->dn_nlevels - 1;  in dnode_increase_indirection()
49 	int new_level = dn->dn_next_nlevels[txgoff];  in dnode_increase_indirection()
52 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_increase_indirection()
55 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_increase_indirection()
56 	ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);  in dnode_increase_indirection()
58 	db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);  in dnode_increase_indirection()
61 	dn->dn_phys->dn_nlevels = new_level;  in dnode_increase_indirection()
62 	dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,  in dnode_increase_indirection()
63 	    (u_longlong_t)dn->dn_object, dn->dn_phys->dn_nlevels);  in dnode_increase_indirection()
74 		children[i] = dbuf_find(dn->dn_objset, dn->dn_object,  in dnode_increase_indirection()
80 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
81 		rw_enter(&dn->dn_dbuf->db_rwlock, RW_WRITER);  in dnode_increase_indirection()
86 	memcpy(db->db.db_data, dn->dn_phys->dn_blkptr,  in dnode_increase_indirection()
98 		ASSERT3P(DB_DNODE(child), ==, dn);  in dnode_increase_indirection()
101 		if (child->db_parent && child->db_parent != dn->dn_dbuf) {  in dnode_increase_indirection()
104 			    &dn->dn_phys->dn_blkptr[child->db_blkid]);  in dnode_increase_indirection()
109 		    child->db_parent == dn->dn_dbuf);  in dnode_increase_indirection()
123 	memset(dn->dn_phys->dn_blkptr, 0, sizeof (blkptr_t) * nblkptr);  in dnode_increase_indirection()
126 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
127 		rw_exit(&dn->dn_dbuf->db_rwlock);  in dnode_increase_indirection()
131 	rw_exit(&dn->dn_struct_rwlock);  in dnode_increase_indirection()
135 free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx)  in free_blocks()  argument
137 	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in free_blocks()
140 	dprintf("ds=%p obj=%llx num=%d\n", ds, (u_longlong_t)dn->dn_object,  in free_blocks()
148 		ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys));  in free_blocks()
164 		if (spa_feature_is_active(dn->dn_objset->os_spa,  in free_blocks()
172 	dnode_diduse_space(dn, -bytesfreed);  in free_blocks()
183 	dnode_t *dn;  in free_verify()  local
186 	dn = DB_DNODE(db);  in free_verify()
187 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_verify()
191 	ASSERT3U(dn->dn_phys->dn_indblkshift, >=, SPA_BLKPTRSHIFT);  in free_verify()
195 	ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);  in free_verify()
206 		rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_verify()
207 		err = dbuf_hold_impl(dn, db->db_level - 1,  in free_verify()
209 		rw_exit(&dn->dn_struct_rwlock);  in free_verify()
277 	dnode_t *dn;  in free_children()  local
316 	dn = DB_DNODE(db);  in free_children()
317 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_children()
337 		free_blocks(dn, bp, end - start + 1, tx);  in free_children()
343 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_children()
344 			VERIFY0(dbuf_hold_impl(dn, db->db_level - 1,  in free_children()
346 			rw_exit(&dn->dn_struct_rwlock);  in free_children()
359 		free_blocks(dn, db->db_blkptr, 1, tx);  in free_children()
372 dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks,  in dnode_sync_free_range_impl()  argument
375 	blkptr_t *bp = dn->dn_phys->dn_blkptr;  in dnode_sync_free_range_impl()
376 	int dnlevel = dn->dn_phys->dn_nlevels;  in dnode_sync_free_range_impl()
379 	if (blkid > dn->dn_phys->dn_maxblkid)  in dnode_sync_free_range_impl()
382 	ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX);  in dnode_sync_free_range_impl()
383 	if (blkid + nblks > dn->dn_phys->dn_maxblkid) {  in dnode_sync_free_range_impl()
384 		nblks = dn->dn_phys->dn_maxblkid - blkid + 1;  in dnode_sync_free_range_impl()
390 		if (blkid >= dn->dn_phys->dn_nblkptr) {  in dnode_sync_free_range_impl()
394 		ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
395 		free_blocks(dn, bp + blkid, nblks, tx);  in dnode_sync_free_range_impl()
398 		    (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT);  in dnode_sync_free_range_impl()
403 		ASSERT(start < dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
408 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in dnode_sync_free_range_impl()
409 			VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i,  in dnode_sync_free_range_impl()
411 			rw_exit(&dn->dn_struct_rwlock);  in dnode_sync_free_range_impl()
431 	if (trunc && !dn->dn_objset->os_raw_receive) {  in dnode_sync_free_range_impl()
433 		dn->dn_phys->dn_maxblkid = blkid == 0 ? 0 : blkid - 1;  in dnode_sync_free_range_impl()
435 		off = (dn->dn_phys->dn_maxblkid + 1) *  in dnode_sync_free_range_impl()
436 		    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);  in dnode_sync_free_range_impl()
437 		ASSERT(off < dn->dn_phys->dn_maxblkid ||  in dnode_sync_free_range_impl()
438 		    dn->dn_phys->dn_maxblkid == 0 ||  in dnode_sync_free_range_impl()
439 		    dnode_next_offset(dn, 0, &off, 1, 1, 0) != 0);  in dnode_sync_free_range_impl()
453 	dnode_t *dn = dsfra->dsfra_dnode;  in dnode_sync_free_range()  local
455 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free_range()
456 	dnode_sync_free_range_impl(dn, blkid, nblks,  in dnode_sync_free_range()
458 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free_range()
465 dnode_evict_dbufs(dnode_t *dn)  in dnode_evict_dbufs()  argument
472 	mutex_enter(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
473 	for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {  in dnode_evict_dbufs()
477 		ASSERT3P(DB_DNODE(db), ==, dn);  in dnode_evict_dbufs()
494 			avl_insert_here(&dn->dn_dbufs, db_marker, db,  in dnode_evict_dbufs()
512 			db_next = AVL_NEXT(&dn->dn_dbufs, db_marker);  in dnode_evict_dbufs()
513 			avl_remove(&dn->dn_dbufs, db_marker);  in dnode_evict_dbufs()
518 			db_next = AVL_NEXT(&dn->dn_dbufs, db);  in dnode_evict_dbufs()
521 	mutex_exit(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
525 	dnode_evict_bonus(dn);  in dnode_evict_dbufs()
529 dnode_evict_bonus(dnode_t *dn)  in dnode_evict_bonus()  argument
531 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_evict_bonus()
532 	if (dn->dn_bonus != NULL) {  in dnode_evict_bonus()
533 		if (zfs_refcount_is_zero(&dn->dn_bonus->db_holds)) {  in dnode_evict_bonus()
534 			mutex_enter(&dn->dn_bonus->db_mtx);  in dnode_evict_bonus()
535 			dbuf_destroy(dn->dn_bonus);  in dnode_evict_bonus()
536 			dn->dn_bonus = NULL;  in dnode_evict_bonus()
538 			dn->dn_bonus->db_pending_evict = TRUE;  in dnode_evict_bonus()
541 	rw_exit(&dn->dn_struct_rwlock);  in dnode_evict_bonus()
577 dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync_free()  argument
587 	ASSERT0(DN_USED_BYTES(dn->dn_phys));  in dnode_sync_free()
588 	ASSERT(BP_IS_HOLE(dn->dn_phys->dn_blkptr));  in dnode_sync_free()
590 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);  in dnode_sync_free()
591 	dnode_evict_dbufs(dn);  in dnode_sync_free()
600 	 * ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 1);  in dnode_sync_free()
604 	dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync_free()
605 	dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync_free()
606 	dn->dn_next_blksz[txgoff] = 0;  in dnode_sync_free()
607 	dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync_free()
610 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_sync_free()
611 	ASSERT(dn->dn_type != DMU_OT_NONE);  in dnode_sync_free()
613 	ASSERT(dn->dn_free_txg > 0);  in dnode_sync_free()
614 	if (dn->dn_allocated_txg != dn->dn_free_txg)  in dnode_sync_free()
615 		dmu_buf_will_dirty(&dn->dn_dbuf->db, tx);  in dnode_sync_free()
616 	memset(dn->dn_phys, 0, sizeof (dnode_phys_t) * dn->dn_num_slots);  in dnode_sync_free()
617 	dnode_free_interior_slots(dn);  in dnode_sync_free()
619 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free()
620 	dn->dn_type = DMU_OT_NONE;  in dnode_sync_free()
621 	dn->dn_maxblkid = 0;  in dnode_sync_free()
622 	dn->dn_allocated_txg = 0;  in dnode_sync_free()
623 	dn->dn_free_txg = 0;  in dnode_sync_free()
624 	dn->dn_have_spill = B_FALSE;  in dnode_sync_free()
625 	dn->dn_num_slots = 1;  in dnode_sync_free()
626 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free()
628 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);  in dnode_sync_free()
630 	dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync_free()
642 dnode_sync(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync()  argument
644 	objset_t *os = dn->dn_objset;  in dnode_sync()
645 	dnode_phys_t *dnp = dn->dn_phys;  in dnode_sync()
647 	list_t *list = &dn->dn_dirty_records[txgoff];  in dnode_sync()
652 	ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg);  in dnode_sync()
655 	DNODE_VERIFY(dn);  in dnode_sync()
657 	ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf));  in dnode_sync()
664 	    !DMU_OBJECT_IS_SPECIAL(dn->dn_object) &&  in dnode_sync()
666 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
667 		dn->dn_oldused = DN_USED_BYTES(dn->dn_phys);  in dnode_sync()
668 		dn->dn_oldflags = dn->dn_phys->dn_flags;  in dnode_sync()
669 		dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED;  in dnode_sync()
670 		if (dmu_objset_userobjused_enabled(dn->dn_objset))  in dnode_sync()
671 			dn->dn_phys->dn_flags |=  in dnode_sync()
673 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
674 		dmu_objset_userquota_get_ids(dn, B_FALSE, tx);  in dnode_sync()
682 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
684 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
688 	mutex_enter(&dn->dn_mtx);  in dnode_sync()
689 	if (dn->dn_allocated_txg == tx->tx_txg) {  in dnode_sync()
694 			dnp->dn_nblkptr = dn->dn_nblkptr;  in dnode_sync()
697 		dnp->dn_type = dn->dn_type;  in dnode_sync()
698 		dnp->dn_bonustype = dn->dn_bonustype;  in dnode_sync()
699 		dnp->dn_bonuslen = dn->dn_bonuslen;  in dnode_sync()
702 	dnp->dn_extra_slots = dn->dn_num_slots - 1;  in dnode_sync()
713 	if (dn->dn_next_type[txgoff] != 0) {  in dnode_sync()
714 		dnp->dn_type = dn->dn_type;  in dnode_sync()
715 		dn->dn_next_type[txgoff] = 0;  in dnode_sync()
718 	if (dn->dn_next_blksz[txgoff] != 0) {  in dnode_sync()
719 		ASSERT(P2PHASE(dn->dn_next_blksz[txgoff],  in dnode_sync()
722 		    dn->dn_maxblkid == 0 || list_head(list) != NULL ||  in dnode_sync()
723 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT ==  in dnode_sync()
725 		    !zfs_range_tree_is_empty(dn->dn_free_ranges[txgoff]));  in dnode_sync()
727 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT;  in dnode_sync()
728 		dn->dn_next_blksz[txgoff] = 0;  in dnode_sync()
731 	if (dn->dn_next_bonuslen[txgoff] != 0) {  in dnode_sync()
732 		if (dn->dn_next_bonuslen[txgoff] == DN_ZERO_BONUSLEN)  in dnode_sync()
735 			dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff];  in dnode_sync()
738 		dn->dn_next_bonuslen[txgoff] = 0;  in dnode_sync()
741 	if (dn->dn_next_bonustype[txgoff] != 0) {  in dnode_sync()
742 		ASSERT(DMU_OT_IS_VALID(dn->dn_next_bonustype[txgoff]));  in dnode_sync()
743 		dnp->dn_bonustype = dn->dn_next_bonustype[txgoff];  in dnode_sync()
744 		dn->dn_next_bonustype[txgoff] = 0;  in dnode_sync()
747 	boolean_t freeing_dnode = dn->dn_free_txg > 0 &&  in dnode_sync()
748 	    dn->dn_free_txg <= tx->tx_txg;  in dnode_sync()
754 	if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) {  in dnode_sync()
757 		dn->dn_rm_spillblk[txgoff] = 0;  in dnode_sync()
760 	if (dn->dn_next_indblkshift[txgoff] != 0) {  in dnode_sync()
762 		dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff];  in dnode_sync()
763 		dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync()
771 	dnp->dn_checksum = dn->dn_checksum;  in dnode_sync()
772 	dnp->dn_compress = dn->dn_compress;  in dnode_sync()
774 	mutex_exit(&dn->dn_mtx);  in dnode_sync()
777 		free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx);  in dnode_sync()
778 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
780 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
784 	if (dn->dn_free_ranges[txgoff] != NULL) {  in dnode_sync()
786 		dsfra.dsfra_dnode = dn;  in dnode_sync()
789 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
792 			    dn->dn_free_ranges[txgoff], 0,  in dnode_sync()
793 			    dn->dn_maxblkid + 1));  in dnode_sync()
803 		zfs_range_tree_walk(dn->dn_free_ranges[txgoff],  in dnode_sync()
805 		zfs_range_tree_vacate(dn->dn_free_ranges[txgoff], NULL, NULL);  in dnode_sync()
806 		zfs_range_tree_destroy(dn->dn_free_ranges[txgoff]);  in dnode_sync()
807 		dn->dn_free_ranges[txgoff] = NULL;  in dnode_sync()
808 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
812 		dn->dn_objset->os_freed_dnodes++;  in dnode_sync()
813 		dnode_sync_free(dn, tx);  in dnode_sync()
817 	if (dn->dn_num_slots > DNODE_MIN_SLOTS) {  in dnode_sync()
818 		dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in dnode_sync()
825 	if (dn->dn_next_nlevels[txgoff]) {  in dnode_sync()
826 		dnode_increase_indirection(dn, tx);  in dnode_sync()
827 		dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync()
835 	if (dn->dn_next_maxblkid[txgoff]) {  in dnode_sync()
836 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
838 		    dn->dn_next_maxblkid[txgoff] & ~DMU_NEXT_MAXBLKID_SET;  in dnode_sync()
839 		dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync()
840 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
843 	if (dn->dn_next_nblkptr[txgoff]) {  in dnode_sync()
845 		ASSERT(dn->dn_allocated_txg == tx->tx_txg);  in dnode_sync()
846 		if (dn->dn_next_nblkptr[txgoff] > dnp->dn_nblkptr) {  in dnode_sync()
850 			    (dn->dn_next_nblkptr[txgoff] - dnp->dn_nblkptr));  in dnode_sync()
854 			ASSERT(dn->dn_next_nblkptr[txgoff] < dnp->dn_nblkptr);  in dnode_sync()
857 				if (i >= dn->dn_next_nblkptr[txgoff])  in dnode_sync()
862 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
863 		dnp->dn_nblkptr = dn->dn_next_nblkptr[txgoff];  in dnode_sync()
864 		dn->dn_next_nblkptr[txgoff] = 0;  in dnode_sync()
865 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
868 	dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);  in dnode_sync()
870 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {  in dnode_sync()
872 		dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync()