101 	bpobj_t bpo;  in bpobj_free()  local
107 	VERIFY3U(0, ==, bpobj_open(&bpo, os, obj));  in bpobj_free()
109 	mutex_enter(&bpo.bpo_lock);  in bpobj_free()
111 	if (!bpo.bpo_havesubobj || bpo.bpo_phys->bpo_subobjs == 0)  in bpobj_free()
114 	VERIFY3U(0, ==, dmu_object_info(os, bpo.bpo_phys->bpo_subobjs, &doi));  in bpobj_free()
117 	for (i = bpo.bpo_phys->bpo_num_subobjs - 1; i >= 0; i--) {  in bpobj_free()
128 			    bpo.bpo_phys->bpo_subobjs, offset, FTAG, &dbuf, 0));  in bpobj_free()
141 	VERIFY3U(0, ==, dmu_object_free(os, bpo.bpo_phys->bpo_subobjs, tx));  in bpobj_free()
144 	mutex_exit(&bpo.bpo_lock);  in bpobj_free()
145 	bpobj_close(&bpo);  in bpobj_free()
151 bpobj_open(bpobj_t *bpo, objset_t *os, uint64_t object)  in bpobj_open()  argument
160 	memset(bpo, 0, sizeof (*bpo));  in bpobj_open()
161 	mutex_init(&bpo->bpo_lock, NULL, MUTEX_DEFAULT, NULL);  in bpobj_open()
163 	ASSERT(bpo->bpo_dbuf == NULL);  in bpobj_open()
164 	ASSERT(bpo->bpo_phys == NULL);  in bpobj_open()
169 	err = dmu_bonus_hold(os, object, bpo, &bpo->bpo_dbuf);  in bpobj_open()
173 	bpo->bpo_os = os;  in bpobj_open()
174 	bpo->bpo_object = object;  in bpobj_open()
175 	bpo->bpo_epb = doi.doi_data_block_size >> SPA_BLKPTRSHIFT;  in bpobj_open()
176 	bpo->bpo_havecomp = (doi.doi_bonus_size > BPOBJ_SIZE_V0);  in bpobj_open()
177 	bpo->bpo_havesubobj = (doi.doi_bonus_size > BPOBJ_SIZE_V1);  in bpobj_open()
178 	bpo->bpo_havefreed = (doi.doi_bonus_size > BPOBJ_SIZE_V2);  in bpobj_open()
179 	bpo->bpo_phys = bpo->bpo_dbuf->db_data;  in bpobj_open()
184 bpobj_is_open(const bpobj_t *bpo)  in bpobj_is_open()  argument
186 	return (bpo->bpo_object != 0);  in bpobj_is_open()
190 bpobj_close(bpobj_t *bpo)  in bpobj_close()  argument
193 	if (bpo->bpo_object == 0)  in bpobj_close()
196 	dmu_buf_rele(bpo->bpo_dbuf, bpo);  in bpobj_close()
197 	if (bpo->bpo_cached_dbuf != NULL)  in bpobj_close()
198 		dmu_buf_rele(bpo->bpo_cached_dbuf, bpo);  in bpobj_close()
199 	bpo->bpo_dbuf = NULL;  in bpobj_close()
200 	bpo->bpo_phys = NULL;  in bpobj_close()
201 	bpo->bpo_cached_dbuf = NULL;  in bpobj_close()
202 	bpo->bpo_object = 0;  in bpobj_close()
204 	mutex_destroy(&bpo->bpo_lock);  in bpobj_close()
208 bpobj_is_empty_impl(bpobj_t *bpo)  in bpobj_is_empty_impl()  argument
210 	ASSERT(MUTEX_HELD(&bpo->bpo_lock));  in bpobj_is_empty_impl()
211 	return (bpo->bpo_phys->bpo_num_blkptrs == 0 &&  in bpobj_is_empty_impl()
212 	    (!bpo->bpo_havesubobj || bpo->bpo_phys->bpo_num_subobjs == 0));  in bpobj_is_empty_impl()
216 bpobj_is_empty(bpobj_t *bpo)  in bpobj_is_empty()  argument
218 	mutex_enter(&bpo->bpo_lock);  in bpobj_is_empty()
219 	boolean_t is_empty = bpobj_is_empty_impl(bpo);  in bpobj_is_empty()
220 	mutex_exit(&bpo->bpo_lock);  in bpobj_is_empty()
247 bpi_alloc(bpobj_t *bpo, bpobj_info_t *parent, uint64_t index)  in bpi_alloc()  argument
250 	bpi->bpi_bpo = bpo;  in bpi_alloc()
253 	if (bpo->bpo_havesubobj && bpo->bpo_phys->bpo_subobjs != 0) {  in bpi_alloc()
254 		bpi->bpi_unprocessed_subobjs = bpo->bpo_phys->bpo_num_subobjs;  in bpi_alloc()
286 	bpobj_t *bpo = bpi->bpi_bpo;  in bpobj_iterate_blkptrs()  local
288 	int64_t i = bpo->bpo_phys->bpo_num_blkptrs - 1;  in bpobj_iterate_blkptrs()
289 	uint64_t pe = P2ALIGN_TYPED(i, bpo->bpo_epb, uint64_t) *  in bpobj_iterate_blkptrs()
295 		dmu_prefetch(bpo->bpo_os, bpo->bpo_object, 0, pb, pe - pb,  in bpobj_iterate_blkptrs()
300 		uint64_t blkoff = P2PHASE(i, bpo->bpo_epb);  in bpobj_iterate_blkptrs()
305 			err = dmu_buf_hold(bpo->bpo_os, bpo->bpo_object,  in bpobj_iterate_blkptrs()
313 				dmu_prefetch(bpo->bpo_os, bpo->bpo_object, 0,  in bpobj_iterate_blkptrs()
331 			spa_t *spa = dmu_objset_spa(bpo->bpo_os);  in bpobj_iterate_blkptrs()
335 			ASSERT(dmu_buf_is_dirty(bpo->bpo_dbuf, tx));  in bpobj_iterate_blkptrs()
336 			bpo->bpo_phys->bpo_num_blkptrs--;  in bpobj_iterate_blkptrs()
337 			ASSERT3S(bpo->bpo_phys->bpo_num_blkptrs, >=, 0);  in bpobj_iterate_blkptrs()
339 				ASSERT(bpo->bpo_havefreed);  in bpobj_iterate_blkptrs()
340 				bpo->bpo_phys->bpo_num_freed--;  in bpobj_iterate_blkptrs()
341 				ASSERT3S(bpo->bpo_phys->bpo_num_freed, >=, 0);  in bpobj_iterate_blkptrs()
348 		VERIFY0(dmu_free_range(bpo->bpo_os,  in bpobj_iterate_blkptrs()
349 		    bpo->bpo_object,  in bpobj_iterate_blkptrs()
350 		    bpo->bpo_phys->bpo_num_blkptrs * sizeof (blkptr_t),  in bpobj_iterate_blkptrs()
391 		bpobj_t *bpo = bpi->bpi_bpo;  in bpobj_iterate_impl()  local
393 		ASSERT3P(bpo, !=, NULL);  in bpobj_iterate_impl()
394 		ASSERT(MUTEX_HELD(&bpo->bpo_lock));  in bpobj_iterate_impl()
395 		ASSERT(bpobj_is_open(bpo));  in bpobj_iterate_impl()
398 			dmu_buf_will_dirty(bpo->bpo_dbuf, tx);  in bpobj_iterate_impl()
418 			if (bpobj_is_empty_impl(bpo)) {  in bpobj_iterate_impl()
419 				ASSERT0(bpo->bpo_phys->bpo_bytes);  in bpobj_iterate_impl()
420 				ASSERT0(bpo->bpo_phys->bpo_comp);  in bpobj_iterate_impl()
421 				ASSERT0(bpo->bpo_phys->bpo_uncomp);  in bpobj_iterate_impl()
429 					ASSERT0(bpo->bpo_phys->bpo_num_blkptrs);  in bpobj_iterate_impl()
434 					ASSERT(dmu_buf_is_dirty(bpo->bpo_dbuf,  in bpobj_iterate_impl()
445 					if (bpo->bpo_havesubobj &&  in bpobj_iterate_impl()
446 					    bpo->bpo_phys->bpo_subobjs != 0) {  in bpobj_iterate_impl()
447 						ASSERT0(bpo->bpo_phys->  in bpobj_iterate_impl()
450 						    bpo->bpo_os,  in bpobj_iterate_impl()
451 						    bpo->bpo_phys->bpo_subobjs,  in bpobj_iterate_impl()
455 						bpo->bpo_phys->bpo_subobjs = 0;  in bpobj_iterate_impl()
458 					    bpo->bpo_object, tx);  in bpobj_iterate_impl()
463 				mutex_exit(&bpo->bpo_lock);  in bpobj_iterate_impl()
464 				bpobj_close(bpo);  in bpobj_iterate_impl()
465 				kmem_free(bpo, sizeof (bpobj_t));  in bpobj_iterate_impl()
467 				mutex_exit(&bpo->bpo_lock);  in bpobj_iterate_impl()
480 			ASSERT(bpo->bpo_havecomp);  in bpobj_iterate_impl()
488 			err = dmu_read(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs,  in bpobj_iterate_impl()
496 			err = bpobj_open(subbpo, bpo->bpo_os, subobj);  in bpobj_iterate_impl()
520 		bpobj_t *bpo = bpi->bpi_bpo;  in bpobj_iterate_impl()  local
522 		ASSERT3P(bpo, !=, NULL);  in bpobj_iterate_impl()
524 		mutex_exit(&bpo->bpo_lock);  in bpobj_iterate_impl()
544 bpobj_iterate(bpobj_t *bpo, bpobj_itor_t func, void *arg, dmu_tx_t *tx)  in bpobj_iterate()  argument
546 	return (bpobj_iterate_impl(bpo, func, arg, tx, B_TRUE, NULL));  in bpobj_iterate()
564 bpobj_iterate_nofree(bpobj_t *bpo, bpobj_itor_t func, void *arg,  in bpobj_iterate_nofree()  argument
567 	return (bpobj_iterate_impl(bpo, func, arg, NULL, B_FALSE, bpobj_size));  in bpobj_iterate_nofree()
576 livelist_bpobj_iterate_from_nofree(bpobj_t *bpo, bpobj_itor_t func, void *arg,  in livelist_bpobj_iterate_from_nofree()  argument
579 	if (bpo->bpo_havesubobj)  in livelist_bpobj_iterate_from_nofree()
580 		VERIFY0(bpo->bpo_phys->bpo_subobjs);  in livelist_bpobj_iterate_from_nofree()
581 	bpobj_info_t *bpi = bpi_alloc(bpo, NULL, 0);  in livelist_bpobj_iterate_from_nofree()
673 bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx)  in bpobj_enqueue_subobj()  argument
680 	ASSERT(bpobj_is_open(bpo));  in bpobj_enqueue_subobj()
682 	ASSERT(bpo->bpo_havesubobj);  in bpobj_enqueue_subobj()
683 	ASSERT(bpo->bpo_havecomp);  in bpobj_enqueue_subobj()
684 	ASSERT(bpo->bpo_object != dmu_objset_pool(bpo->bpo_os)->dp_empty_bpobj);  in bpobj_enqueue_subobj()
686 	if (subobj == dmu_objset_pool(bpo->bpo_os)->dp_empty_bpobj) {  in bpobj_enqueue_subobj()
687 		bpobj_decr_empty(bpo->bpo_os, tx);  in bpobj_enqueue_subobj()
691 	VERIFY3U(0, ==, bpobj_open(&subbpo, bpo->bpo_os, subobj));  in bpobj_enqueue_subobj()
695 		bpobj_free(bpo->bpo_os, subobj, tx);  in bpobj_enqueue_subobj()
700 	mutex_enter(&bpo->bpo_lock);  in bpobj_enqueue_subobj()
701 	dmu_buf_will_dirty(bpo->bpo_dbuf, tx);  in bpobj_enqueue_subobj()
705 	if (bpo->bpo_phys->bpo_subobjs != 0) {  in bpobj_enqueue_subobj()
706 		ASSERT0(dmu_object_info(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs,  in bpobj_enqueue_subobj()
718 		VERIFY0(dmu_object_info(bpo->bpo_os, subsubobjs, &doi));  in bpobj_enqueue_subobj()
730 	VERIFY3U(0, ==, dmu_object_info(bpo->bpo_os, subobj, &doi));  in bpobj_enqueue_subobj()
739 		VERIFY0(dmu_buf_hold(bpo->bpo_os, subsubobjs,  in bpobj_enqueue_subobj()
747 		if (bpo->bpo_phys->bpo_subobjs == 0) {  in bpobj_enqueue_subobj()
748 			bpo->bpo_phys->bpo_subobjs =  in bpobj_enqueue_subobj()
749 			    dmu_object_alloc(bpo->bpo_os,  in bpobj_enqueue_subobj()
753 		dmu_write(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs,  in bpobj_enqueue_subobj()
754 		    bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj),  in bpobj_enqueue_subobj()
757 		bpo->bpo_phys->bpo_num_subobjs += numsubsub;  in bpobj_enqueue_subobj()
761 		VERIFY0(dmu_object_free(bpo->bpo_os, subsubobjs, tx));  in bpobj_enqueue_subobj()
769 		VERIFY0(dmu_buf_hold(bpo->bpo_os, subobj,  in bpobj_enqueue_subobj()
777 		dmu_write(bpo->bpo_os, bpo->bpo_object,  in bpobj_enqueue_subobj()
778 		    bpo->bpo_phys->bpo_num_blkptrs * sizeof (blkptr_t),  in bpobj_enqueue_subobj()
782 		bpo->bpo_phys->bpo_num_blkptrs += numbps;  in bpobj_enqueue_subobj()
785 		VERIFY0(dmu_object_free(bpo->bpo_os, subobj, tx));  in bpobj_enqueue_subobj()
788 		if (bpo->bpo_phys->bpo_subobjs == 0) {  in bpobj_enqueue_subobj()
789 			bpo->bpo_phys->bpo_subobjs =  in bpobj_enqueue_subobj()
790 			    dmu_object_alloc(bpo->bpo_os,  in bpobj_enqueue_subobj()
795 		dmu_write(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs,  in bpobj_enqueue_subobj()
796 		    bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj),  in bpobj_enqueue_subobj()
798 		bpo->bpo_phys->bpo_num_subobjs++;  in bpobj_enqueue_subobj()
801 	bpo->bpo_phys->bpo_bytes += used;  in bpobj_enqueue_subobj()
802 	bpo->bpo_phys->bpo_comp += comp;  in bpobj_enqueue_subobj()
803 	bpo->bpo_phys->bpo_uncomp += uncomp;  in bpobj_enqueue_subobj()
804 	mutex_exit(&bpo->bpo_lock);  in bpobj_enqueue_subobj()
812 bpobj_prefetch_subobj(bpobj_t *bpo, uint64_t subobj)  in bpobj_prefetch_subobj()  argument
820 	ASSERT(bpobj_is_open(bpo));  in bpobj_prefetch_subobj()
823 	if (subobj == dmu_objset_pool(bpo->bpo_os)->dp_empty_bpobj)  in bpobj_prefetch_subobj()
826 	if (bpobj_open(&subbpo, bpo->bpo_os, subobj) != 0)  in bpobj_prefetch_subobj()
836 		if (dmu_object_info(bpo->bpo_os, subsubobjs, &doi) != 0)  in bpobj_prefetch_subobj()
842 	if (dmu_object_info(bpo->bpo_os, subobj, &doi) != 0)  in bpobj_prefetch_subobj()
848 		if (bpo->bpo_phys->bpo_subobjs) {  in bpobj_prefetch_subobj()
849 			dmu_prefetch(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, 0,  in bpobj_prefetch_subobj()
850 			    bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj), 1,  in bpobj_prefetch_subobj()
853 		dmu_prefetch(bpo->bpo_os, subsubobjs, 0, 0, 1,  in bpobj_prefetch_subobj()
858 		dmu_prefetch(bpo->bpo_os, bpo->bpo_object, 0,  in bpobj_prefetch_subobj()
859 		    bpo->bpo_phys->bpo_num_blkptrs * sizeof (blkptr_t), 1,  in bpobj_prefetch_subobj()
861 		dmu_prefetch(bpo->bpo_os, subobj, 0, 0, 1,  in bpobj_prefetch_subobj()
863 	} else if (bpo->bpo_phys->bpo_subobjs) {  in bpobj_prefetch_subobj()
864 		dmu_prefetch(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, 0,  in bpobj_prefetch_subobj()
865 		    bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj), 1,  in bpobj_prefetch_subobj()
871 bpobj_enqueue(bpobj_t *bpo, const blkptr_t *bp, boolean_t bp_freed,  in bpobj_enqueue()  argument
879 	ASSERT(bpobj_is_open(bpo));  in bpobj_enqueue()
881 	ASSERT(bpo->bpo_object != dmu_objset_pool(bpo->bpo_os)->dp_empty_bpobj);  in bpobj_enqueue()
906 	mutex_enter(&bpo->bpo_lock);  in bpobj_enqueue()
908 	offset = bpo->bpo_phys->bpo_num_blkptrs * sizeof (stored_bp);  in bpobj_enqueue()
909 	blkoff = P2PHASE(bpo->bpo_phys->bpo_num_blkptrs, bpo->bpo_epb);  in bpobj_enqueue()
911 	if (bpo->bpo_cached_dbuf == NULL ||  in bpobj_enqueue()
912 	    offset < bpo->bpo_cached_dbuf->db_offset ||  in bpobj_enqueue()
913 	    offset >= bpo->bpo_cached_dbuf->db_offset +  in bpobj_enqueue()
914 	    bpo->bpo_cached_dbuf->db_size) {  in bpobj_enqueue()
915 		if (bpo->bpo_cached_dbuf)  in bpobj_enqueue()
916 			dmu_buf_rele(bpo->bpo_cached_dbuf, bpo);  in bpobj_enqueue()
917 		VERIFY3U(0, ==, dmu_buf_hold(bpo->bpo_os, bpo->bpo_object,  in bpobj_enqueue()
918 		    offset, bpo, &bpo->bpo_cached_dbuf, 0));  in bpobj_enqueue()
919 		ASSERT3P(bpo->bpo_cached_dbuf, !=, NULL);  in bpobj_enqueue()
922 	dmu_buf_will_dirty(bpo->bpo_cached_dbuf, tx);  in bpobj_enqueue()
923 	bparray = bpo->bpo_cached_dbuf->db_data;  in bpobj_enqueue()
926 	dmu_buf_will_dirty(bpo->bpo_dbuf, tx);  in bpobj_enqueue()
927 	bpo->bpo_phys->bpo_num_blkptrs++;  in bpobj_enqueue()
929 	bpo->bpo_phys->bpo_bytes += sign *  in bpobj_enqueue()
930 	    bp_get_dsize_sync(dmu_objset_spa(bpo->bpo_os), bp);  in bpobj_enqueue()
931 	if (bpo->bpo_havecomp) {  in bpobj_enqueue()
932 		bpo->bpo_phys->bpo_comp += sign * BP_GET_PSIZE(bp);  in bpobj_enqueue()
933 		bpo->bpo_phys->bpo_uncomp += sign * BP_GET_UCSIZE(bp);  in bpobj_enqueue()
936 		ASSERT(bpo->bpo_havefreed);  in bpobj_enqueue()
937 		bpo->bpo_phys->bpo_num_freed++;  in bpobj_enqueue()
939 	mutex_exit(&bpo->bpo_lock);  in bpobj_enqueue()
970 bpobj_space(bpobj_t *bpo, uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)  in bpobj_space()  argument
972 	ASSERT(bpobj_is_open(bpo));  in bpobj_space()
973 	mutex_enter(&bpo->bpo_lock);  in bpobj_space()
975 	*usedp = bpo->bpo_phys->bpo_bytes;  in bpobj_space()
976 	if (bpo->bpo_havecomp) {  in bpobj_space()
977 		*compp = bpo->bpo_phys->bpo_comp;  in bpobj_space()
978 		*uncompp = bpo->bpo_phys->bpo_uncomp;  in bpobj_space()
979 		mutex_exit(&bpo->bpo_lock);  in bpobj_space()
982 		mutex_exit(&bpo->bpo_lock);  in bpobj_space()
983 		return (bpobj_space_range(bpo, 0, UINT64_MAX,  in bpobj_space()
993 bpobj_space_range(bpobj_t *bpo, uint64_t mintxg, uint64_t maxtxg,  in bpobj_space_range()  argument
999 	ASSERT(bpobj_is_open(bpo));  in bpobj_space_range()
1005 	if (mintxg < TXG_INITIAL && maxtxg == UINT64_MAX && bpo->bpo_havecomp)  in bpobj_space_range()
1006 		return (bpobj_space(bpo, usedp, compp, uncompp));  in bpobj_space_range()
1008 	sra.spa = dmu_objset_spa(bpo->bpo_os);  in bpobj_space_range()
1012 	err = bpobj_iterate_nofree(bpo, space_range_cb, &sra, NULL);  in bpobj_space_range()