9  * or https://opensource.org/licenses/CDDL-1.0.
59 	int cmp1 = TREE_CMP(r1->ref_holder, r2->ref_holder);  in zfs_refcount_compare()
60 	int cmp2 = TREE_CMP(r1->ref_number, r2->ref_number);  in zfs_refcount_compare()
62 	return ((cmp || r1->ref_search) ? cmp : TREE_PCMP(r1, r2));  in zfs_refcount_compare()
68 	mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);  in zfs_refcount_create()
69 	avl_create(&rc->rc_tree, zfs_refcount_compare, sizeof (reference_t),  in zfs_refcount_create()
71 	list_create(&rc->rc_removed, sizeof (reference_t),  in zfs_refcount_create()
73 	rc->rc_count = 0;  in zfs_refcount_create()
74 	rc->rc_removed_count = 0;  in zfs_refcount_create()
75 	rc->rc_tracked = reference_tracking_enable;  in zfs_refcount_create()
82 	rc->rc_tracked = B_TRUE;  in zfs_refcount_create_tracked()
89 	rc->rc_tracked = B_FALSE;  in zfs_refcount_create_untracked()
95 	reference_t *ref;  in zfs_refcount_destroy_many()  local
98 	ASSERT3U(rc->rc_count, ==, number);  in zfs_refcount_destroy_many()
99 	while ((ref = avl_destroy_nodes(&rc->rc_tree, &cookie)) != NULL)  in zfs_refcount_destroy_many()
100 		kmem_cache_free(reference_cache, ref);  in zfs_refcount_destroy_many()
101 	avl_destroy(&rc->rc_tree);  in zfs_refcount_destroy_many()
103 	while ((ref = list_remove_head(&rc->rc_removed)))  in zfs_refcount_destroy_many()
104 		kmem_cache_free(reference_cache, ref);  in zfs_refcount_destroy_many()
105 	list_destroy(&rc->rc_removed);  in zfs_refcount_destroy_many()
106 	mutex_destroy(&rc->rc_mtx);  in zfs_refcount_destroy_many()
124 	return (atomic_load_64(&rc->rc_count));  in zfs_refcount_count()
130 	reference_t *ref;  in zfs_refcount_add_many()  local
133 	if (likely(!rc->rc_tracked)) {  in zfs_refcount_add_many()
134 		count = atomic_add_64_nv(&(rc)->rc_count, number);  in zfs_refcount_add_many()
139 	ref = kmem_cache_alloc(reference_cache, KM_SLEEP);  in zfs_refcount_add_many()
140 	ref->ref_holder = holder;  in zfs_refcount_add_many()
141 	ref->ref_number = number;  in zfs_refcount_add_many()
142 	ref->ref_search = B_FALSE;  in zfs_refcount_add_many()
143 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_add_many()
144 	avl_add(&rc->rc_tree, ref);  in zfs_refcount_add_many()
145 	rc->rc_count += number;  in zfs_refcount_add_many()
146 	count = rc->rc_count;  in zfs_refcount_add_many()
147 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_add_many()
161 	if (likely(!rc->rc_tracked))  in zfs_refcount_add_few()
163 	else for (; number > 0; number--)  in zfs_refcount_add_few()
171 	reference_t *ref, s;  in zfs_refcount_remove_many()  local
174 	if (likely(!rc->rc_tracked)) {  in zfs_refcount_remove_many()
175 		count = atomic_add_64_nv(&(rc)->rc_count, -number);  in zfs_refcount_remove_many()
183 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_remove_many()
184 	ASSERT3U(rc->rc_count, >=, number);  in zfs_refcount_remove_many()
185 	ref = avl_find(&rc->rc_tree, &s, NULL);  in zfs_refcount_remove_many()
186 	if (unlikely(ref == NULL)) {  in zfs_refcount_remove_many()
189 		return (-1);  in zfs_refcount_remove_many()
191 	avl_remove(&rc->rc_tree, ref);  in zfs_refcount_remove_many()
193 		list_insert_head(&rc->rc_removed, ref);  in zfs_refcount_remove_many()
194 		if (rc->rc_removed_count >= reference_history) {  in zfs_refcount_remove_many()
195 			ref = list_remove_tail(&rc->rc_removed);  in zfs_refcount_remove_many()
196 			kmem_cache_free(reference_cache, ref);  in zfs_refcount_remove_many()
198 			rc->rc_removed_count++;  in zfs_refcount_remove_many()
201 		kmem_cache_free(reference_cache, ref);  in zfs_refcount_remove_many()
203 	rc->rc_count -= number;  in zfs_refcount_remove_many()
204 	count = rc->rc_count;  in zfs_refcount_remove_many()
205 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_remove_many()
218 	if (likely(!rc->rc_tracked))  in zfs_refcount_remove_few()
220 	else for (; number > 0; number--)  in zfs_refcount_remove_few()
229 	reference_t *ref;  in zfs_refcount_transfer()  local
239 	mutex_enter(&src->rc_mtx);  in zfs_refcount_transfer()
240 	count = src->rc_count;  in zfs_refcount_transfer()
241 	removed_count = src->rc_removed_count;  in zfs_refcount_transfer()
242 	src->rc_count = 0;  in zfs_refcount_transfer()
243 	src->rc_removed_count = 0;  in zfs_refcount_transfer()
244 	avl_swap(&tree, &src->rc_tree);  in zfs_refcount_transfer()
245 	list_move_tail(&removed, &src->rc_removed);  in zfs_refcount_transfer()
246 	mutex_exit(&src->rc_mtx);  in zfs_refcount_transfer()
248 	mutex_enter(&dst->rc_mtx);  in zfs_refcount_transfer()
249 	dst->rc_count += count;  in zfs_refcount_transfer()
250 	dst->rc_removed_count += removed_count;  in zfs_refcount_transfer()
251 	if (avl_is_empty(&dst->rc_tree))  in zfs_refcount_transfer()
252 		avl_swap(&dst->rc_tree, &tree);  in zfs_refcount_transfer()
253 	else while ((ref = avl_destroy_nodes(&tree, &cookie)) != NULL)  in zfs_refcount_transfer()
254 		avl_add(&dst->rc_tree, ref);  in zfs_refcount_transfer()
255 	list_move_tail(&dst->rc_removed, &removed);  in zfs_refcount_transfer()
256 	mutex_exit(&dst->rc_mtx);  in zfs_refcount_transfer()
266 	reference_t *ref, s;  in zfs_refcount_transfer_ownership_many()  local
268 	if (likely(!rc->rc_tracked))  in zfs_refcount_transfer_ownership_many()
274 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_transfer_ownership_many()
275 	ref = avl_find(&rc->rc_tree, &s, NULL);  in zfs_refcount_transfer_ownership_many()
276 	ASSERT(ref);  in zfs_refcount_transfer_ownership_many()
277 	ref->ref_holder = new_holder;  in zfs_refcount_transfer_ownership_many()
278 	avl_update(&rc->rc_tree, ref);  in zfs_refcount_transfer_ownership_many()
279 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_transfer_ownership_many()
298 	reference_t *ref, s;  in zfs_refcount_held()  local
302 	if (likely(!rc->rc_tracked))  in zfs_refcount_held()
308 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_held()
309 	ref = avl_find(&rc->rc_tree, &s, &idx);  in zfs_refcount_held()
310 	if (likely(ref == NULL))  in zfs_refcount_held()
311 		ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER);  in zfs_refcount_held()
312 	res = ref && ref->ref_holder == holder;  in zfs_refcount_held()
313 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_held()
325 	reference_t *ref, s;  in zfs_refcount_not_held()  local
329 	if (likely(!rc->rc_tracked))  in zfs_refcount_not_held()
332 	mutex_enter(&rc->rc_mtx);  in zfs_refcount_not_held()
336 	ref = avl_find(&rc->rc_tree, &s, &idx);  in zfs_refcount_not_held()
337 	if (likely(ref == NULL))  in zfs_refcount_not_held()
338 		ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER);  in zfs_refcount_not_held()
339 	res = ref == NULL || ref->ref_holder != holder;  in zfs_refcount_not_held()
340 	mutex_exit(&rc->rc_mtx);  in zfs_refcount_not_held()