67 zfs_refcount_create(zfs_refcount_t *rc)  in zfs_refcount_create()  argument
69 	mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);  in zfs_refcount_create()
70 	avl_create(&rc->rc_tree, zfs_refcount_compare, sizeof (reference_t),  in zfs_refcount_create()
72 	list_create(&rc->rc_removed, sizeof (reference_t),  in zfs_refcount_create()
74 	rc->rc_count = 0;  in zfs_refcount_create()
75 	rc->rc_removed_count = 0;  in zfs_refcount_create()
76 	rc->rc_tracked = reference_tracking_enable;  in zfs_refcount_create()
80 zfs_refcount_create_tracked(zfs_refcount_t *rc)  in zfs_refcount_create_tracked()  argument
82 	zfs_refcount_create(rc);  in zfs_refcount_create_tracked()
83 	rc->rc_tracked = B_TRUE;  in zfs_refcount_create_tracked()
87 zfs_refcount_create_untracked(zfs_refcount_t *rc)  in zfs_refcount_create_untracked()  argument
89 	zfs_refcount_create(rc);  in zfs_refcount_create_untracked()
90 	rc->rc_tracked = B_FALSE;  in zfs_refcount_create_untracked()
94 zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)  in zfs_refcount_destroy_many()  argument
99 	ASSERT3U(rc->rc_count, ==, number);  in zfs_refcount_destroy_many()
100 	while ((ref = avl_destroy_nodes(&rc->rc_tree, &cookie)) != NULL)  in zfs_refcount_destroy_many()
102 	avl_destroy(&rc->rc_tree);  in zfs_refcount_destroy_many()
104 	while ((ref = list_remove_head(&rc->rc_removed)))  in zfs_refcount_destroy_many()
106 	list_destroy(&rc->rc_removed);  in zfs_refcount_destroy_many()
107 	mutex_destroy(&rc->rc_mtx);  in zfs_refcount_destroy_many()
111 zfs_refcount_destroy(zfs_refcount_t *rc)  in zfs_refcount_destroy()  argument
113 	zfs_refcount_destroy_many(rc, 0);  in zfs_refcount_destroy()
117 zfs_refcount_is_zero(zfs_refcount_t *rc)  in zfs_refcount_is_zero()  argument
119 	return (zfs_refcount_count(rc) == 0);  in zfs_refcount_is_zero()
123 zfs_refcount_count(zfs_refcount_t *rc)  in zfs_refcount_count()  argument
125 	return (atomic_load_64(&rc->rc_count));  in zfs_refcount_count()
129 zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)  in zfs_refcount_add_many()  argument
134 	if (likely(!rc->rc_tracked)) {  in zfs_refcount_add_many()
135 		count = atomic_add_64_nv(&(rc)->rc_count, number);  in zfs_refcount_add_many()
144 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_add_many()
145 	avl_add(&rc->rc_tree, ref);  in zfs_refcount_add_many()
146 	rc->rc_count += number;  in zfs_refcount_add_many()
147 	count = rc->rc_count;  in zfs_refcount_add_many()
148 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_add_many()
154 zfs_refcount_add(zfs_refcount_t *rc, const void *holder)  in zfs_refcount_add()  argument
156 	return (zfs_refcount_add_many(rc, 1, holder));  in zfs_refcount_add()
160 zfs_refcount_add_few(zfs_refcount_t *rc, uint64_t number, const void *holder)  in zfs_refcount_add_few()  argument
162 	if (likely(!rc->rc_tracked))  in zfs_refcount_add_few()
163 		(void) zfs_refcount_add_many(rc, number, holder);  in zfs_refcount_add_few()
165 		(void) zfs_refcount_add(rc, holder);  in zfs_refcount_add_few()
169 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,  in zfs_refcount_remove_many()  argument
175 	if (likely(!rc->rc_tracked)) {  in zfs_refcount_remove_many()
176 		count = atomic_add_64_nv(&(rc)->rc_count, -number);  in zfs_refcount_remove_many()
184 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_remove_many()
185 	ASSERT3U(rc->rc_count, >=, number);  in zfs_refcount_remove_many()
186 	ref = avl_find(&rc->rc_tree, &s, NULL);  in zfs_refcount_remove_many()
189 		    (u_longlong_t)(uintptr_t)rc);  in zfs_refcount_remove_many()
192 	avl_remove(&rc->rc_tree, ref);  in zfs_refcount_remove_many()
194 		list_insert_head(&rc->rc_removed, ref);  in zfs_refcount_remove_many()
195 		if (rc->rc_removed_count >= reference_history) {  in zfs_refcount_remove_many()
196 			ref = list_remove_tail(&rc->rc_removed);  in zfs_refcount_remove_many()
199 			rc->rc_removed_count++;  in zfs_refcount_remove_many()
204 	rc->rc_count -= number;  in zfs_refcount_remove_many()
205 	count = rc->rc_count;  in zfs_refcount_remove_many()
206 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_remove_many()
211 zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)  in zfs_refcount_remove()  argument
213 	return (zfs_refcount_remove_many(rc, 1, holder));  in zfs_refcount_remove()
217 zfs_refcount_remove_few(zfs_refcount_t *rc, uint64_t number, const void *holder)  in zfs_refcount_remove_few()  argument
219 	if (likely(!rc->rc_tracked))  in zfs_refcount_remove_few()
220 		(void) zfs_refcount_remove_many(rc, number, holder);  in zfs_refcount_remove_few()
222 		(void) zfs_refcount_remove(rc, holder);  in zfs_refcount_remove_few()
264 zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,  in zfs_refcount_transfer_ownership_many()  argument
269 	if (likely(!rc->rc_tracked))  in zfs_refcount_transfer_ownership_many()
275 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_transfer_ownership_many()
276 	ref = avl_find(&rc->rc_tree, &s, NULL);  in zfs_refcount_transfer_ownership_many()
279 	avl_update(&rc->rc_tree, ref);  in zfs_refcount_transfer_ownership_many()
280 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_transfer_ownership_many()
284 zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder,  in zfs_refcount_transfer_ownership()  argument
287 	return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder,  in zfs_refcount_transfer_ownership()
297 zfs_refcount_held(zfs_refcount_t *rc, const void *holder)  in zfs_refcount_held()  argument
303 	if (likely(!rc->rc_tracked))  in zfs_refcount_held()
304 		return (zfs_refcount_count(rc) > 0);  in zfs_refcount_held()
309 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_held()
310 	ref = avl_find(&rc->rc_tree, &s, &idx);  in zfs_refcount_held()
312 		ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER);  in zfs_refcount_held()
314 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_held()
324 zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)  in zfs_refcount_not_held()  argument
330 	if (likely(!rc->rc_tracked))  in zfs_refcount_not_held()
333 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_not_held()
337 	ref = avl_find(&rc->rc_tree, &s, &idx);  in zfs_refcount_not_held()
339 		ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER);  in zfs_refcount_not_held()
341 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_not_held()