1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2012, 2021 by Delphix. All rights reserved. 25 */ 26 27 #include <sys/zfs_context.h> 28 #include <sys/zfs_refcount.h> 29 30 #ifdef ZFS_DEBUG 31 /* 32 * Reference count tracking is disabled by default. It's memory requirements 33 * are reasonable, however as implemented it consumes a significant amount of 34 * cpu time. Until its performance is improved it should be manually enabled. 35 */ 36 int reference_tracking_enable = B_FALSE; 37 static uint_t reference_history = 3; /* tunable */ 38 39 static kmem_cache_t *reference_cache; 40 41 void 42 zfs_refcount_init(void) 43 { 44 reference_cache = kmem_cache_create("reference_cache", 45 sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 46 } 47 48 void 49 zfs_refcount_fini(void) 50 { 51 kmem_cache_destroy(reference_cache); 52 } 53 54 static int 55 zfs_refcount_compare(const void *x1, const void *x2) 56 { 57 const reference_t *r1 = (const reference_t *)x1; 58 const reference_t *r2 = (const reference_t *)x2; 59 60 int cmp1 = TREE_CMP(r1->ref_holder, r2->ref_holder); 61 int cmp2 = TREE_CMP(r1->ref_number, r2->ref_number); 62 int cmp = cmp1 ? cmp1 : cmp2; 63 return ((cmp || r1->ref_search) ? cmp : TREE_PCMP(r1, r2)); 64 } 65 66 void 67 zfs_refcount_create(zfs_refcount_t *rc) 68 { 69 mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL); 70 avl_create(&rc->rc_tree, zfs_refcount_compare, sizeof (reference_t), 71 offsetof(reference_t, ref_link.a)); 72 list_create(&rc->rc_removed, sizeof (reference_t), 73 offsetof(reference_t, ref_link.l)); 74 rc->rc_count = 0; 75 rc->rc_removed_count = 0; 76 rc->rc_tracked = reference_tracking_enable; 77 } 78 79 void 80 zfs_refcount_create_tracked(zfs_refcount_t *rc) 81 { 82 zfs_refcount_create(rc); 83 rc->rc_tracked = B_TRUE; 84 } 85 86 void 87 zfs_refcount_create_untracked(zfs_refcount_t *rc) 88 { 89 zfs_refcount_create(rc); 90 rc->rc_tracked = B_FALSE; 91 } 92 93 void 94 zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number) 95 { 96 reference_t *ref; 97 void *cookie = NULL; 98 99 ASSERT3U(rc->rc_count, ==, number); 100 while ((ref = avl_destroy_nodes(&rc->rc_tree, &cookie)) != NULL) 101 kmem_cache_free(reference_cache, ref); 102 avl_destroy(&rc->rc_tree); 103 104 while ((ref = list_remove_head(&rc->rc_removed))) 105 kmem_cache_free(reference_cache, ref); 106 list_destroy(&rc->rc_removed); 107 mutex_destroy(&rc->rc_mtx); 108 } 109 110 void 111 zfs_refcount_destroy(zfs_refcount_t *rc) 112 { 113 zfs_refcount_destroy_many(rc, 0); 114 } 115 116 int 117 zfs_refcount_is_zero(zfs_refcount_t *rc) 118 { 119 return (zfs_refcount_count(rc) == 0); 120 } 121 122 int64_t 123 zfs_refcount_count(zfs_refcount_t *rc) 124 { 125 return (atomic_load_64(&rc->rc_count)); 126 } 127 128 int64_t 129 zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder) 130 { 131 reference_t *ref; 132 int64_t count; 133 134 if (likely(!rc->rc_tracked)) { 135 count = atomic_add_64_nv(&(rc)->rc_count, number); 136 ASSERT3U(count, >=, number); 137 return (count); 138 } 139 140 ref = kmem_cache_alloc(reference_cache, KM_SLEEP); 141 ref->ref_holder = holder; 142 ref->ref_number = number; 143 ref->ref_search = B_FALSE; 144 mutex_enter(&rc->rc_mtx); 145 avl_add(&rc->rc_tree, ref); 146 rc->rc_count += number; 147 count = rc->rc_count; 148 mutex_exit(&rc->rc_mtx); 149 150 return (count); 151 } 152 153 int64_t 154 zfs_refcount_add(zfs_refcount_t *rc, const void *holder) 155 { 156 return (zfs_refcount_add_many(rc, 1, holder)); 157 } 158 159 void 160 zfs_refcount_add_few(zfs_refcount_t *rc, uint64_t number, const void *holder) 161 { 162 if (likely(!rc->rc_tracked)) 163 (void) zfs_refcount_add_many(rc, number, holder); 164 else for (; number > 0; number--) 165 (void) zfs_refcount_add(rc, holder); 166 } 167 168 int64_t 169 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number, 170 const void *holder) 171 { 172 reference_t *ref, s; 173 int64_t count; 174 175 if (likely(!rc->rc_tracked)) { 176 count = atomic_add_64_nv(&(rc)->rc_count, -number); 177 ASSERT3S(count, >=, 0); 178 return (count); 179 } 180 181 s.ref_holder = holder; 182 s.ref_number = number; 183 s.ref_search = B_TRUE; 184 mutex_enter(&rc->rc_mtx); 185 ASSERT3U(rc->rc_count, >=, number); 186 ref = avl_find(&rc->rc_tree, &s, NULL); 187 if (unlikely(ref == NULL)) { 188 PANIC("No such hold %llx on refcount %llx", 189 (u_longlong_t)(uintptr_t)holder, 190 (u_longlong_t)(uintptr_t)rc); 191 return (-1); 192 } 193 avl_remove(&rc->rc_tree, ref); 194 if (reference_history > 0) { 195 list_insert_head(&rc->rc_removed, ref); 196 if (rc->rc_removed_count >= reference_history) { 197 ref = list_remove_tail(&rc->rc_removed); 198 kmem_cache_free(reference_cache, ref); 199 } else { 200 rc->rc_removed_count++; 201 } 202 } else { 203 kmem_cache_free(reference_cache, ref); 204 } 205 rc->rc_count -= number; 206 count = rc->rc_count; 207 mutex_exit(&rc->rc_mtx); 208 return (count); 209 } 210 211 int64_t 212 zfs_refcount_remove(zfs_refcount_t *rc, const void *holder) 213 { 214 return (zfs_refcount_remove_many(rc, 1, holder)); 215 } 216 217 void 218 zfs_refcount_remove_few(zfs_refcount_t *rc, uint64_t number, const void *holder) 219 { 220 if (likely(!rc->rc_tracked)) 221 (void) zfs_refcount_remove_many(rc, number, holder); 222 else for (; number > 0; number--) 223 (void) zfs_refcount_remove(rc, holder); 224 } 225 226 void 227 zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src) 228 { 229 avl_tree_t tree; 230 list_t removed; 231 reference_t *ref; 232 void *cookie = NULL; 233 uint64_t count; 234 uint_t removed_count; 235 236 avl_create(&tree, zfs_refcount_compare, sizeof (reference_t), 237 offsetof(reference_t, ref_link.a)); 238 list_create(&removed, sizeof (reference_t), 239 offsetof(reference_t, ref_link.l)); 240 241 mutex_enter(&src->rc_mtx); 242 count = src->rc_count; 243 removed_count = src->rc_removed_count; 244 src->rc_count = 0; 245 src->rc_removed_count = 0; 246 avl_swap(&tree, &src->rc_tree); 247 list_move_tail(&removed, &src->rc_removed); 248 mutex_exit(&src->rc_mtx); 249 250 mutex_enter(&dst->rc_mtx); 251 dst->rc_count += count; 252 dst->rc_removed_count += removed_count; 253 if (avl_is_empty(&dst->rc_tree)) 254 avl_swap(&dst->rc_tree, &tree); 255 else while ((ref = avl_destroy_nodes(&tree, &cookie)) != NULL) 256 avl_add(&dst->rc_tree, ref); 257 list_move_tail(&dst->rc_removed, &removed); 258 mutex_exit(&dst->rc_mtx); 259 260 avl_destroy(&tree); 261 list_destroy(&removed); 262 } 263 264 void 265 zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number, 266 const void *current_holder, const void *new_holder) 267 { 268 reference_t *ref, s; 269 270 if (likely(!rc->rc_tracked)) 271 return; 272 273 s.ref_holder = current_holder; 274 s.ref_number = number; 275 s.ref_search = B_TRUE; 276 mutex_enter(&rc->rc_mtx); 277 ref = avl_find(&rc->rc_tree, &s, NULL); 278 ASSERT(ref); 279 ref->ref_holder = new_holder; 280 avl_update(&rc->rc_tree, ref); 281 mutex_exit(&rc->rc_mtx); 282 } 283 284 void 285 zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder, 286 const void *new_holder) 287 { 288 return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder, 289 new_holder)); 290 } 291 292 /* 293 * If tracking is enabled, return true if a reference exists that matches 294 * the "holder" tag. If tracking is disabled, then return true if a reference 295 * might be held. 296 */ 297 boolean_t 298 zfs_refcount_held(zfs_refcount_t *rc, const void *holder) 299 { 300 reference_t *ref, s; 301 avl_index_t idx; 302 boolean_t res; 303 304 if (likely(!rc->rc_tracked)) 305 return (zfs_refcount_count(rc) > 0); 306 307 s.ref_holder = holder; 308 s.ref_number = 0; 309 s.ref_search = B_TRUE; 310 mutex_enter(&rc->rc_mtx); 311 ref = avl_find(&rc->rc_tree, &s, &idx); 312 if (likely(ref == NULL)) 313 ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER); 314 res = ref && ref->ref_holder == holder; 315 mutex_exit(&rc->rc_mtx); 316 return (res); 317 } 318 319 /* 320 * If tracking is enabled, return true if a reference does not exist that 321 * matches the "holder" tag. If tracking is disabled, always return true 322 * since the reference might not be held. 323 */ 324 boolean_t 325 zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder) 326 { 327 reference_t *ref, s; 328 avl_index_t idx; 329 boolean_t res; 330 331 if (likely(!rc->rc_tracked)) 332 return (B_TRUE); 333 334 mutex_enter(&rc->rc_mtx); 335 s.ref_holder = holder; 336 s.ref_number = 0; 337 s.ref_search = B_TRUE; 338 ref = avl_find(&rc->rc_tree, &s, &idx); 339 if (likely(ref == NULL)) 340 ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER); 341 res = ref == NULL || ref->ref_holder != holder; 342 mutex_exit(&rc->rc_mtx); 343 return (res); 344 } 345 346 EXPORT_SYMBOL(zfs_refcount_create); 347 EXPORT_SYMBOL(zfs_refcount_destroy); 348 EXPORT_SYMBOL(zfs_refcount_is_zero); 349 EXPORT_SYMBOL(zfs_refcount_count); 350 EXPORT_SYMBOL(zfs_refcount_add); 351 EXPORT_SYMBOL(zfs_refcount_remove); 352 EXPORT_SYMBOL(zfs_refcount_held); 353 354 ZFS_MODULE_PARAM(zfs, , reference_tracking_enable, INT, ZMOD_RW, 355 "Track reference holders to refcount_t objects"); 356 357 ZFS_MODULE_PARAM(zfs, , reference_history, UINT, ZMOD_RW, 358 "Maximum reference holders being tracked"); 359 #endif /* ZFS_DEBUG */ 360