1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #define pr_fmt(fmt) "ref_tracker: " fmt 4 5 #include <linux/export.h> 6 #include <linux/list_sort.h> 7 #include <linux/ref_tracker.h> 8 #include <linux/slab.h> 9 #include <linux/stacktrace.h> 10 #include <linux/stackdepot.h> 11 #include <linux/seq_file.h> 12 13 #define REF_TRACKER_STACK_ENTRIES 16 14 #define STACK_BUF_SIZE 1024 15 16 struct ref_tracker { 17 struct list_head head; /* anchor into dir->list or dir->quarantine */ 18 bool dead; 19 depot_stack_handle_t alloc_stack_handle; 20 depot_stack_handle_t free_stack_handle; 21 }; 22 23 struct ref_tracker_dir_stats { 24 int total; 25 int count; 26 struct { 27 depot_stack_handle_t stack_handle; 28 unsigned int count; 29 } stacks[]; 30 }; 31 32 #ifdef CONFIG_DEBUG_FS 33 #include <linux/xarray.h> 34 35 /* 36 * ref_tracker_dir_init() is usually called in allocation-safe contexts, but 37 * the same is not true of ref_tracker_dir_exit() which can be called from 38 * anywhere an object is freed. Removing debugfs dentries is a blocking 39 * operation, so we defer that work to the debugfs_reap_worker. 40 * 41 * Each dentry is tracked in the appropriate xarray. When 42 * ref_tracker_dir_exit() is called, its entries in the xarrays are marked and 43 * the workqueue job is scheduled. The worker then runs and deletes any marked 44 * dentries asynchronously. 45 */ 46 static struct xarray debugfs_dentries; 47 static struct xarray debugfs_symlinks; 48 static struct work_struct debugfs_reap_worker; 49 50 #define REF_TRACKER_DIR_DEAD XA_MARK_0 51 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir) 52 { 53 unsigned long flags; 54 55 xa_lock_irqsave(&debugfs_dentries, flags); 56 __xa_set_mark(&debugfs_dentries, (unsigned long)dir, REF_TRACKER_DIR_DEAD); 57 xa_unlock_irqrestore(&debugfs_dentries, flags); 58 59 xa_lock_irqsave(&debugfs_symlinks, flags); 60 __xa_set_mark(&debugfs_symlinks, (unsigned long)dir, REF_TRACKER_DIR_DEAD); 61 xa_unlock_irqrestore(&debugfs_symlinks, flags); 62 63 schedule_work(&debugfs_reap_worker); 64 } 65 #else 66 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir) 67 { 68 } 69 #endif 70 71 static struct ref_tracker_dir_stats * 72 ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit) 73 { 74 struct ref_tracker_dir_stats *stats; 75 struct ref_tracker *tracker; 76 77 stats = kmalloc(struct_size(stats, stacks, limit), 78 GFP_NOWAIT | __GFP_NOWARN); 79 if (!stats) 80 return ERR_PTR(-ENOMEM); 81 stats->total = 0; 82 stats->count = 0; 83 84 list_for_each_entry(tracker, &dir->list, head) { 85 depot_stack_handle_t stack = tracker->alloc_stack_handle; 86 int i; 87 88 ++stats->total; 89 for (i = 0; i < stats->count; ++i) 90 if (stats->stacks[i].stack_handle == stack) 91 break; 92 if (i >= limit) 93 continue; 94 if (i >= stats->count) { 95 stats->stacks[i].stack_handle = stack; 96 stats->stacks[i].count = 0; 97 ++stats->count; 98 } 99 ++stats->stacks[i].count; 100 } 101 102 return stats; 103 } 104 105 struct ostream { 106 void __ostream_printf (*func)(struct ostream *stream, char *fmt, ...); 107 char *prefix; 108 char *buf; 109 struct seq_file *seq; 110 int size, used; 111 }; 112 113 static void __ostream_printf pr_ostream_log(struct ostream *stream, char *fmt, ...) 114 { 115 va_list args; 116 117 va_start(args, fmt); 118 vprintk(fmt, args); 119 va_end(args); 120 } 121 122 static void __ostream_printf pr_ostream_buf(struct ostream *stream, char *fmt, ...) 123 { 124 int ret, len = stream->size - stream->used; 125 va_list args; 126 127 va_start(args, fmt); 128 ret = vsnprintf(stream->buf + stream->used, len, fmt, args); 129 va_end(args); 130 if (ret > 0) 131 stream->used += min(ret, len); 132 } 133 134 #define pr_ostream(stream, fmt, args...) \ 135 ({ \ 136 struct ostream *_s = (stream); \ 137 \ 138 _s->func(_s, fmt, ##args); \ 139 }) 140 141 static void 142 __ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir, 143 unsigned int display_limit, struct ostream *s) 144 { 145 struct ref_tracker_dir_stats *stats; 146 unsigned int i = 0, skipped; 147 depot_stack_handle_t stack; 148 char *sbuf; 149 150 lockdep_assert_held(&dir->lock); 151 152 if (list_empty(&dir->list)) 153 return; 154 155 stats = ref_tracker_get_stats(dir, display_limit); 156 if (IS_ERR(stats)) { 157 pr_ostream(s, "%s%s@%p: couldn't get stats, error %pe\n", 158 s->prefix, dir->class, dir, stats); 159 return; 160 } 161 162 sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT | __GFP_NOWARN); 163 164 for (i = 0, skipped = stats->total; i < stats->count; ++i) { 165 stack = stats->stacks[i].stack_handle; 166 if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4)) 167 sbuf[0] = 0; 168 pr_ostream(s, "%s%s@%p has %d/%d users at\n%s\n", s->prefix, 169 dir->class, dir, stats->stacks[i].count, 170 stats->total, sbuf); 171 skipped -= stats->stacks[i].count; 172 } 173 174 if (skipped) 175 pr_ostream(s, "%s%s@%p skipped reports about %d/%d users.\n", 176 s->prefix, dir->class, dir, skipped, stats->total); 177 178 kfree(sbuf); 179 180 kfree(stats); 181 } 182 183 void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, 184 unsigned int display_limit) 185 { 186 struct ostream os = { .func = pr_ostream_log, 187 .prefix = "ref_tracker: " }; 188 189 __ref_tracker_dir_pr_ostream(dir, display_limit, &os); 190 } 191 EXPORT_SYMBOL(ref_tracker_dir_print_locked); 192 193 void ref_tracker_dir_print(struct ref_tracker_dir *dir, 194 unsigned int display_limit) 195 { 196 unsigned long flags; 197 198 spin_lock_irqsave(&dir->lock, flags); 199 ref_tracker_dir_print_locked(dir, display_limit); 200 spin_unlock_irqrestore(&dir->lock, flags); 201 } 202 EXPORT_SYMBOL(ref_tracker_dir_print); 203 204 int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size) 205 { 206 struct ostream os = { .func = pr_ostream_buf, 207 .prefix = "ref_tracker: ", 208 .buf = buf, 209 .size = size }; 210 unsigned long flags; 211 212 spin_lock_irqsave(&dir->lock, flags); 213 __ref_tracker_dir_pr_ostream(dir, 16, &os); 214 spin_unlock_irqrestore(&dir->lock, flags); 215 216 return os.used; 217 } 218 EXPORT_SYMBOL(ref_tracker_dir_snprint); 219 220 void ref_tracker_dir_exit(struct ref_tracker_dir *dir) 221 { 222 struct ref_tracker *tracker, *n; 223 unsigned long flags; 224 bool leak = false; 225 226 dir->dead = true; 227 /* 228 * The xarray entries must be marked before the dir->lock is taken to 229 * protect simultaneous debugfs readers. 230 */ 231 ref_tracker_debugfs_mark(dir); 232 spin_lock_irqsave(&dir->lock, flags); 233 list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { 234 list_del(&tracker->head); 235 kfree(tracker); 236 dir->quarantine_avail++; 237 } 238 if (!list_empty(&dir->list)) { 239 ref_tracker_dir_print_locked(dir, 16); 240 leak = true; 241 list_for_each_entry_safe(tracker, n, &dir->list, head) { 242 list_del(&tracker->head); 243 kfree(tracker); 244 } 245 } 246 spin_unlock_irqrestore(&dir->lock, flags); 247 WARN_ON_ONCE(leak); 248 WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); 249 WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1); 250 } 251 EXPORT_SYMBOL(ref_tracker_dir_exit); 252 253 int ref_tracker_alloc(struct ref_tracker_dir *dir, 254 struct ref_tracker **trackerp, 255 gfp_t gfp) 256 { 257 unsigned long entries[REF_TRACKER_STACK_ENTRIES]; 258 struct ref_tracker *tracker; 259 unsigned int nr_entries; 260 gfp_t gfp_mask = gfp | __GFP_NOWARN; 261 unsigned long flags; 262 263 WARN_ON_ONCE(dir->dead); 264 265 if (!trackerp) { 266 refcount_inc(&dir->no_tracker); 267 return 0; 268 } 269 if (gfp & __GFP_DIRECT_RECLAIM) 270 gfp_mask |= __GFP_NOFAIL; 271 *trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask); 272 if (unlikely(!tracker)) { 273 pr_err_once("memory allocation failure, unreliable refcount tracker.\n"); 274 refcount_inc(&dir->untracked); 275 return -ENOMEM; 276 } 277 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 278 tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp); 279 280 spin_lock_irqsave(&dir->lock, flags); 281 list_add(&tracker->head, &dir->list); 282 spin_unlock_irqrestore(&dir->lock, flags); 283 return 0; 284 } 285 EXPORT_SYMBOL_GPL(ref_tracker_alloc); 286 287 int ref_tracker_free(struct ref_tracker_dir *dir, 288 struct ref_tracker **trackerp) 289 { 290 unsigned long entries[REF_TRACKER_STACK_ENTRIES]; 291 depot_stack_handle_t stack_handle; 292 struct ref_tracker *tracker; 293 unsigned int nr_entries; 294 unsigned long flags; 295 296 WARN_ON_ONCE(dir->dead); 297 298 if (!trackerp) { 299 refcount_dec(&dir->no_tracker); 300 return 0; 301 } 302 tracker = *trackerp; 303 if (!tracker) { 304 refcount_dec(&dir->untracked); 305 return -EEXIST; 306 } 307 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 308 stack_handle = stack_depot_save(entries, nr_entries, 309 GFP_NOWAIT | __GFP_NOWARN); 310 311 spin_lock_irqsave(&dir->lock, flags); 312 if (tracker->dead) { 313 pr_err("reference already released.\n"); 314 if (tracker->alloc_stack_handle) { 315 pr_err("allocated in:\n"); 316 stack_depot_print(tracker->alloc_stack_handle); 317 } 318 if (tracker->free_stack_handle) { 319 pr_err("freed in:\n"); 320 stack_depot_print(tracker->free_stack_handle); 321 } 322 spin_unlock_irqrestore(&dir->lock, flags); 323 WARN_ON_ONCE(1); 324 return -EINVAL; 325 } 326 tracker->dead = true; 327 328 tracker->free_stack_handle = stack_handle; 329 330 list_move_tail(&tracker->head, &dir->quarantine); 331 if (!dir->quarantine_avail) { 332 tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head); 333 list_del(&tracker->head); 334 } else { 335 dir->quarantine_avail--; 336 tracker = NULL; 337 } 338 spin_unlock_irqrestore(&dir->lock, flags); 339 340 kfree(tracker); 341 return 0; 342 } 343 EXPORT_SYMBOL_GPL(ref_tracker_free); 344 345 #ifdef CONFIG_DEBUG_FS 346 #include <linux/debugfs.h> 347 348 static struct dentry *ref_tracker_debug_dir = (struct dentry *)-ENOENT; 349 350 static void __ostream_printf pr_ostream_seq(struct ostream *stream, char *fmt, ...) 351 { 352 va_list args; 353 354 va_start(args, fmt); 355 seq_vprintf(stream->seq, fmt, args); 356 va_end(args); 357 } 358 359 static int ref_tracker_dir_seq_print(struct ref_tracker_dir *dir, struct seq_file *seq) 360 { 361 struct ostream os = { .func = pr_ostream_seq, 362 .prefix = "", 363 .seq = seq }; 364 365 __ref_tracker_dir_pr_ostream(dir, 16, &os); 366 367 return os.used; 368 } 369 370 static int ref_tracker_debugfs_show(struct seq_file *f, void *v) 371 { 372 struct ref_tracker_dir *dir = f->private; 373 unsigned long index = (unsigned long)dir; 374 unsigned long flags; 375 int ret; 376 377 /* 378 * "dir" may not exist at this point if ref_tracker_dir_exit() has 379 * already been called. Take care not to dereference it until its 380 * legitimacy is established. 381 * 382 * The xa_lock is necessary to ensure that "dir" doesn't disappear 383 * before its lock can be taken. If it's in the hash and not marked 384 * dead, then it's safe to take dir->lock which prevents 385 * ref_tracker_dir_exit() from completing. Once the dir->lock is 386 * acquired, the xa_lock can be released. All of this must be IRQ-safe. 387 */ 388 xa_lock_irqsave(&debugfs_dentries, flags); 389 if (!xa_load(&debugfs_dentries, index) || 390 xa_get_mark(&debugfs_dentries, index, REF_TRACKER_DIR_DEAD)) { 391 xa_unlock_irqrestore(&debugfs_dentries, flags); 392 return -ENODATA; 393 } 394 395 spin_lock(&dir->lock); 396 xa_unlock(&debugfs_dentries); 397 ret = ref_tracker_dir_seq_print(dir, f); 398 spin_unlock_irqrestore(&dir->lock, flags); 399 return ret; 400 } 401 402 static int ref_tracker_debugfs_open(struct inode *inode, struct file *filp) 403 { 404 struct ref_tracker_dir *dir = inode->i_private; 405 406 return single_open(filp, ref_tracker_debugfs_show, dir); 407 } 408 409 static const struct file_operations ref_tracker_debugfs_fops = { 410 .owner = THIS_MODULE, 411 .open = ref_tracker_debugfs_open, 412 .read = seq_read, 413 .llseek = seq_lseek, 414 .release = single_release, 415 }; 416 417 /** 418 * ref_tracker_dir_debugfs - create debugfs file for ref_tracker_dir 419 * @dir: ref_tracker_dir to be associated with debugfs file 420 * 421 * In most cases, a debugfs file will be created automatically for every 422 * ref_tracker_dir. If the object was created before debugfs is brought up 423 * then that may fail. In those cases, it is safe to call this at a later 424 * time to create the file. 425 */ 426 void ref_tracker_dir_debugfs(struct ref_tracker_dir *dir) 427 { 428 char name[NAME_MAX + 1]; 429 struct dentry *dentry; 430 int ret; 431 432 /* No-op if already created */ 433 dentry = xa_load(&debugfs_dentries, (unsigned long)dir); 434 if (dentry && !xa_is_err(dentry)) 435 return; 436 437 ret = snprintf(name, sizeof(name), "%s@%p", dir->class, dir); 438 name[sizeof(name) - 1] = '\0'; 439 440 if (ret < sizeof(name)) { 441 dentry = debugfs_create_file(name, S_IFREG | 0400, 442 ref_tracker_debug_dir, dir, 443 &ref_tracker_debugfs_fops); 444 if (!IS_ERR(dentry)) { 445 void *old; 446 447 old = xa_store_irq(&debugfs_dentries, (unsigned long)dir, 448 dentry, GFP_KERNEL); 449 450 if (xa_is_err(old)) 451 debugfs_remove(dentry); 452 else 453 WARN_ON_ONCE(old); 454 } 455 } 456 } 457 EXPORT_SYMBOL(ref_tracker_dir_debugfs); 458 459 void __ostream_printf ref_tracker_dir_symlink(struct ref_tracker_dir *dir, const char *fmt, ...) 460 { 461 char name[NAME_MAX + 1]; 462 struct dentry *symlink, *dentry; 463 va_list args; 464 int ret; 465 466 symlink = xa_load(&debugfs_symlinks, (unsigned long)dir); 467 dentry = xa_load(&debugfs_dentries, (unsigned long)dir); 468 469 /* Already created?*/ 470 if (symlink && !xa_is_err(symlink)) 471 return; 472 473 if (!dentry || xa_is_err(dentry)) 474 return; 475 476 va_start(args, fmt); 477 ret = vsnprintf(name, sizeof(name), fmt, args); 478 va_end(args); 479 name[sizeof(name) - 1] = '\0'; 480 481 if (ret < sizeof(name)) { 482 symlink = debugfs_create_symlink(name, ref_tracker_debug_dir, 483 dentry->d_name.name); 484 if (!IS_ERR(symlink)) { 485 void *old; 486 487 old = xa_store_irq(&debugfs_symlinks, (unsigned long)dir, 488 symlink, GFP_KERNEL); 489 if (xa_is_err(old)) 490 debugfs_remove(symlink); 491 else 492 WARN_ON_ONCE(old); 493 } 494 } 495 } 496 EXPORT_SYMBOL(ref_tracker_dir_symlink); 497 498 static void debugfs_reap_work(struct work_struct *work) 499 { 500 struct dentry *dentry; 501 unsigned long index; 502 bool reaped; 503 504 do { 505 reaped = false; 506 xa_for_each_marked(&debugfs_symlinks, index, dentry, REF_TRACKER_DIR_DEAD) { 507 xa_erase_irq(&debugfs_symlinks, index); 508 debugfs_remove(dentry); 509 reaped = true; 510 } 511 xa_for_each_marked(&debugfs_dentries, index, dentry, REF_TRACKER_DIR_DEAD) { 512 xa_erase_irq(&debugfs_dentries, index); 513 debugfs_remove(dentry); 514 reaped = true; 515 } 516 } while (reaped); 517 } 518 519 static int __init ref_tracker_debugfs_postcore_init(void) 520 { 521 INIT_WORK(&debugfs_reap_worker, debugfs_reap_work); 522 xa_init_flags(&debugfs_dentries, XA_FLAGS_LOCK_IRQ); 523 xa_init_flags(&debugfs_symlinks, XA_FLAGS_LOCK_IRQ); 524 return 0; 525 } 526 postcore_initcall(ref_tracker_debugfs_postcore_init); 527 528 static int __init ref_tracker_debugfs_late_init(void) 529 { 530 ref_tracker_debug_dir = debugfs_create_dir("ref_tracker", NULL); 531 return 0; 532 } 533 late_initcall(ref_tracker_debugfs_late_init); 534 #endif /* CONFIG_DEBUG_FS */ 535