1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #define pr_fmt(fmt) "ref_tracker: " fmt 4 5 #include <linux/export.h> 6 #include <linux/list_sort.h> 7 #include <linux/ref_tracker.h> 8 #include <linux/slab.h> 9 #include <linux/stacktrace.h> 10 #include <linux/stackdepot.h> 11 #include <linux/seq_file.h> 12 13 #define REF_TRACKER_STACK_ENTRIES 16 14 #define STACK_BUF_SIZE 1024 15 16 struct ref_tracker { 17 struct list_head head; /* anchor into dir->list or dir->quarantine */ 18 bool dead; 19 depot_stack_handle_t alloc_stack_handle; 20 depot_stack_handle_t free_stack_handle; 21 }; 22 23 struct ref_tracker_dir_stats { 24 int total; 25 int count; 26 struct { 27 depot_stack_handle_t stack_handle; 28 unsigned int count; 29 } stacks[]; 30 }; 31 32 #ifdef CONFIG_DEBUG_FS 33 #include <linux/xarray.h> 34 35 /* 36 * ref_tracker_dir_init() is usually called in allocation-safe contexts, but 37 * the same is not true of ref_tracker_dir_exit() which can be called from 38 * anywhere an object is freed. Removing debugfs dentries is a blocking 39 * operation, so we defer that work to the debugfs_reap_worker. 40 * 41 * Each dentry is tracked in the appropriate xarray. When 42 * ref_tracker_dir_exit() is called, its entries in the xarrays are marked and 43 * the workqueue job is scheduled. The worker then runs and deletes any marked 44 * dentries asynchronously. 45 */ 46 static struct xarray debugfs_dentries; 47 static struct xarray debugfs_symlinks; 48 static struct work_struct debugfs_reap_worker; 49 50 #define REF_TRACKER_DIR_DEAD XA_MARK_0 51 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir) 52 { 53 unsigned long flags; 54 55 xa_lock_irqsave(&debugfs_dentries, flags); 56 __xa_set_mark(&debugfs_dentries, (unsigned long)dir, REF_TRACKER_DIR_DEAD); 57 xa_unlock_irqrestore(&debugfs_dentries, flags); 58 59 xa_lock_irqsave(&debugfs_symlinks, flags); 60 __xa_set_mark(&debugfs_symlinks, (unsigned long)dir, REF_TRACKER_DIR_DEAD); 61 xa_unlock_irqrestore(&debugfs_symlinks, flags); 62 63 schedule_work(&debugfs_reap_worker); 64 } 65 #else 66 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir) 67 { 68 } 69 #endif 70 71 static struct ref_tracker_dir_stats * 72 ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit) 73 { 74 struct ref_tracker_dir_stats *stats; 75 struct ref_tracker *tracker; 76 77 stats = kmalloc_flex(*stats, stacks, limit, GFP_NOWAIT); 78 if (!stats) 79 return ERR_PTR(-ENOMEM); 80 stats->total = 0; 81 stats->count = 0; 82 83 list_for_each_entry(tracker, &dir->list, head) { 84 depot_stack_handle_t stack = tracker->alloc_stack_handle; 85 int i; 86 87 ++stats->total; 88 for (i = 0; i < stats->count; ++i) 89 if (stats->stacks[i].stack_handle == stack) 90 break; 91 if (i >= limit) 92 continue; 93 if (i >= stats->count) { 94 stats->stacks[i].stack_handle = stack; 95 stats->stacks[i].count = 0; 96 ++stats->count; 97 } 98 ++stats->stacks[i].count; 99 } 100 101 return stats; 102 } 103 104 struct ostream { 105 void __ostream_printf (*func)(struct ostream *stream, char *fmt, ...); 106 char *prefix; 107 char *buf; 108 struct seq_file *seq; 109 int size, used; 110 }; 111 112 static void __ostream_printf pr_ostream_log(struct ostream *stream, char *fmt, ...) 113 { 114 va_list args; 115 116 va_start(args, fmt); 117 vprintk(fmt, args); 118 va_end(args); 119 } 120 121 static void __ostream_printf pr_ostream_buf(struct ostream *stream, char *fmt, ...) 122 { 123 int ret, len = stream->size - stream->used; 124 va_list args; 125 126 va_start(args, fmt); 127 ret = vsnprintf(stream->buf + stream->used, len, fmt, args); 128 va_end(args); 129 if (ret > 0) 130 stream->used += min(ret, len); 131 } 132 133 #define pr_ostream(stream, fmt, args...) \ 134 ({ \ 135 struct ostream *_s = (stream); \ 136 \ 137 _s->func(_s, fmt, ##args); \ 138 }) 139 140 static void 141 __ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir, 142 unsigned int display_limit, struct ostream *s) 143 { 144 struct ref_tracker_dir_stats *stats; 145 unsigned int i = 0, skipped; 146 depot_stack_handle_t stack; 147 char *sbuf; 148 149 lockdep_assert_held(&dir->lock); 150 151 if (list_empty(&dir->list)) 152 return; 153 154 stats = ref_tracker_get_stats(dir, display_limit); 155 if (IS_ERR(stats)) { 156 pr_ostream(s, "%s%s@%p: couldn't get stats, error %pe\n", 157 s->prefix, dir->class, dir, stats); 158 return; 159 } 160 161 sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT); 162 163 for (i = 0, skipped = stats->total; i < stats->count; ++i) { 164 stack = stats->stacks[i].stack_handle; 165 if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4)) 166 sbuf[0] = 0; 167 pr_ostream(s, "%s%s@%p has %d/%d users at\n%s\n", s->prefix, 168 dir->class, dir, stats->stacks[i].count, 169 stats->total, sbuf); 170 skipped -= stats->stacks[i].count; 171 } 172 173 if (skipped) 174 pr_ostream(s, "%s%s@%p skipped reports about %d/%d users.\n", 175 s->prefix, dir->class, dir, skipped, stats->total); 176 177 kfree(sbuf); 178 179 kfree(stats); 180 } 181 182 void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, 183 unsigned int display_limit) 184 { 185 struct ostream os = { .func = pr_ostream_log, 186 .prefix = "ref_tracker: " }; 187 188 __ref_tracker_dir_pr_ostream(dir, display_limit, &os); 189 } 190 EXPORT_SYMBOL(ref_tracker_dir_print_locked); 191 192 void ref_tracker_dir_print(struct ref_tracker_dir *dir, 193 unsigned int display_limit) 194 { 195 unsigned long flags; 196 197 spin_lock_irqsave(&dir->lock, flags); 198 ref_tracker_dir_print_locked(dir, display_limit); 199 spin_unlock_irqrestore(&dir->lock, flags); 200 } 201 EXPORT_SYMBOL(ref_tracker_dir_print); 202 203 int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size) 204 { 205 struct ostream os = { .func = pr_ostream_buf, 206 .prefix = "ref_tracker: ", 207 .buf = buf, 208 .size = size }; 209 unsigned long flags; 210 211 spin_lock_irqsave(&dir->lock, flags); 212 __ref_tracker_dir_pr_ostream(dir, 16, &os); 213 spin_unlock_irqrestore(&dir->lock, flags); 214 215 return os.used; 216 } 217 EXPORT_SYMBOL(ref_tracker_dir_snprint); 218 219 void ref_tracker_dir_exit(struct ref_tracker_dir *dir) 220 { 221 struct ref_tracker *tracker, *n; 222 unsigned long flags; 223 bool leak = false; 224 225 dir->dead = true; 226 /* 227 * The xarray entries must be marked before the dir->lock is taken to 228 * protect simultaneous debugfs readers. 229 */ 230 ref_tracker_debugfs_mark(dir); 231 spin_lock_irqsave(&dir->lock, flags); 232 list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { 233 list_del(&tracker->head); 234 kfree(tracker); 235 dir->quarantine_avail++; 236 } 237 if (!list_empty(&dir->list)) { 238 ref_tracker_dir_print_locked(dir, 16); 239 leak = true; 240 list_for_each_entry_safe(tracker, n, &dir->list, head) { 241 list_del(&tracker->head); 242 kfree(tracker); 243 } 244 } 245 spin_unlock_irqrestore(&dir->lock, flags); 246 WARN_ON_ONCE(leak); 247 WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); 248 WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1); 249 } 250 EXPORT_SYMBOL(ref_tracker_dir_exit); 251 252 int ref_tracker_alloc(struct ref_tracker_dir *dir, 253 struct ref_tracker **trackerp, 254 gfp_t gfp) 255 { 256 unsigned long entries[REF_TRACKER_STACK_ENTRIES]; 257 struct ref_tracker *tracker; 258 unsigned int nr_entries; 259 gfp_t gfp_mask = gfp | __GFP_NOWARN; 260 unsigned long flags; 261 262 WARN_ON_ONCE(dir->dead); 263 264 if (!trackerp) { 265 refcount_inc(&dir->no_tracker); 266 return 0; 267 } 268 if (gfp & __GFP_DIRECT_RECLAIM) 269 gfp_mask |= __GFP_NOFAIL; 270 *trackerp = tracker = kzalloc_obj(*tracker, gfp_mask); 271 if (unlikely(!tracker)) { 272 pr_err_once("memory allocation failure, unreliable refcount tracker.\n"); 273 refcount_inc(&dir->untracked); 274 return -ENOMEM; 275 } 276 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 277 tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp); 278 279 spin_lock_irqsave(&dir->lock, flags); 280 list_add(&tracker->head, &dir->list); 281 spin_unlock_irqrestore(&dir->lock, flags); 282 return 0; 283 } 284 EXPORT_SYMBOL_GPL(ref_tracker_alloc); 285 286 int ref_tracker_free(struct ref_tracker_dir *dir, 287 struct ref_tracker **trackerp) 288 { 289 unsigned long entries[REF_TRACKER_STACK_ENTRIES]; 290 depot_stack_handle_t stack_handle; 291 struct ref_tracker *tracker; 292 unsigned int nr_entries; 293 unsigned long flags; 294 295 WARN_ON_ONCE(dir->dead); 296 297 if (!trackerp) { 298 refcount_dec(&dir->no_tracker); 299 return 0; 300 } 301 tracker = *trackerp; 302 if (!tracker) { 303 refcount_dec(&dir->untracked); 304 return -EEXIST; 305 } 306 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 307 stack_handle = stack_depot_save(entries, nr_entries, 308 GFP_NOWAIT); 309 310 spin_lock_irqsave(&dir->lock, flags); 311 if (tracker->dead) { 312 pr_err("reference already released.\n"); 313 if (tracker->alloc_stack_handle) { 314 pr_err("allocated in:\n"); 315 stack_depot_print(tracker->alloc_stack_handle); 316 } 317 if (tracker->free_stack_handle) { 318 pr_err("freed in:\n"); 319 stack_depot_print(tracker->free_stack_handle); 320 } 321 spin_unlock_irqrestore(&dir->lock, flags); 322 WARN_ON_ONCE(1); 323 return -EINVAL; 324 } 325 tracker->dead = true; 326 327 tracker->free_stack_handle = stack_handle; 328 329 list_move_tail(&tracker->head, &dir->quarantine); 330 if (!dir->quarantine_avail) { 331 tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head); 332 list_del(&tracker->head); 333 } else { 334 dir->quarantine_avail--; 335 tracker = NULL; 336 } 337 spin_unlock_irqrestore(&dir->lock, flags); 338 339 kfree(tracker); 340 return 0; 341 } 342 EXPORT_SYMBOL_GPL(ref_tracker_free); 343 344 #ifdef CONFIG_DEBUG_FS 345 #include <linux/debugfs.h> 346 347 static struct dentry *ref_tracker_debug_dir = (struct dentry *)-ENOENT; 348 349 static void __ostream_printf pr_ostream_seq(struct ostream *stream, char *fmt, ...) 350 { 351 va_list args; 352 353 va_start(args, fmt); 354 seq_vprintf(stream->seq, fmt, args); 355 va_end(args); 356 } 357 358 static int ref_tracker_dir_seq_print(struct ref_tracker_dir *dir, struct seq_file *seq) 359 { 360 struct ostream os = { .func = pr_ostream_seq, 361 .prefix = "", 362 .seq = seq }; 363 364 __ref_tracker_dir_pr_ostream(dir, 16, &os); 365 366 return os.used; 367 } 368 369 static int ref_tracker_debugfs_show(struct seq_file *f, void *v) 370 { 371 struct ref_tracker_dir *dir = f->private; 372 unsigned long index = (unsigned long)dir; 373 unsigned long flags; 374 int ret; 375 376 /* 377 * "dir" may not exist at this point if ref_tracker_dir_exit() has 378 * already been called. Take care not to dereference it until its 379 * legitimacy is established. 380 * 381 * The xa_lock is necessary to ensure that "dir" doesn't disappear 382 * before its lock can be taken. If it's in the hash and not marked 383 * dead, then it's safe to take dir->lock which prevents 384 * ref_tracker_dir_exit() from completing. Once the dir->lock is 385 * acquired, the xa_lock can be released. All of this must be IRQ-safe. 386 */ 387 xa_lock_irqsave(&debugfs_dentries, flags); 388 if (!xa_load(&debugfs_dentries, index) || 389 xa_get_mark(&debugfs_dentries, index, REF_TRACKER_DIR_DEAD)) { 390 xa_unlock_irqrestore(&debugfs_dentries, flags); 391 return -ENODATA; 392 } 393 394 spin_lock(&dir->lock); 395 xa_unlock(&debugfs_dentries); 396 ret = ref_tracker_dir_seq_print(dir, f); 397 spin_unlock_irqrestore(&dir->lock, flags); 398 return ret; 399 } 400 401 static int ref_tracker_debugfs_open(struct inode *inode, struct file *filp) 402 { 403 struct ref_tracker_dir *dir = inode->i_private; 404 405 return single_open(filp, ref_tracker_debugfs_show, dir); 406 } 407 408 static const struct file_operations ref_tracker_debugfs_fops = { 409 .owner = THIS_MODULE, 410 .open = ref_tracker_debugfs_open, 411 .read = seq_read, 412 .llseek = seq_lseek, 413 .release = single_release, 414 }; 415 416 /** 417 * ref_tracker_dir_debugfs - create debugfs file for ref_tracker_dir 418 * @dir: ref_tracker_dir to be associated with debugfs file 419 * 420 * In most cases, a debugfs file will be created automatically for every 421 * ref_tracker_dir. If the object was created before debugfs is brought up 422 * then that may fail. In those cases, it is safe to call this at a later 423 * time to create the file. 424 */ 425 void ref_tracker_dir_debugfs(struct ref_tracker_dir *dir) 426 { 427 char name[NAME_MAX + 1]; 428 struct dentry *dentry; 429 int ret; 430 431 /* No-op if already created */ 432 dentry = xa_load(&debugfs_dentries, (unsigned long)dir); 433 if (dentry && !xa_is_err(dentry)) 434 return; 435 436 ret = snprintf(name, sizeof(name), "%s@%p", dir->class, dir); 437 name[sizeof(name) - 1] = '\0'; 438 439 if (ret < sizeof(name)) { 440 dentry = debugfs_create_file(name, S_IFREG | 0400, 441 ref_tracker_debug_dir, dir, 442 &ref_tracker_debugfs_fops); 443 if (!IS_ERR(dentry)) { 444 void *old; 445 446 old = xa_store_irq(&debugfs_dentries, (unsigned long)dir, 447 dentry, GFP_KERNEL); 448 449 if (xa_is_err(old)) 450 debugfs_remove(dentry); 451 else 452 WARN_ON_ONCE(old); 453 } 454 } 455 } 456 EXPORT_SYMBOL(ref_tracker_dir_debugfs); 457 458 void __ostream_printf ref_tracker_dir_symlink(struct ref_tracker_dir *dir, const char *fmt, ...) 459 { 460 char name[NAME_MAX + 1]; 461 struct dentry *symlink, *dentry; 462 va_list args; 463 int ret; 464 465 symlink = xa_load(&debugfs_symlinks, (unsigned long)dir); 466 dentry = xa_load(&debugfs_dentries, (unsigned long)dir); 467 468 /* Already created?*/ 469 if (symlink && !xa_is_err(symlink)) 470 return; 471 472 if (!dentry || xa_is_err(dentry)) 473 return; 474 475 va_start(args, fmt); 476 ret = vsnprintf(name, sizeof(name), fmt, args); 477 va_end(args); 478 name[sizeof(name) - 1] = '\0'; 479 480 if (ret < sizeof(name)) { 481 symlink = debugfs_create_symlink(name, ref_tracker_debug_dir, 482 dentry->d_name.name); 483 if (!IS_ERR(symlink)) { 484 void *old; 485 486 old = xa_store_irq(&debugfs_symlinks, (unsigned long)dir, 487 symlink, GFP_KERNEL); 488 if (xa_is_err(old)) 489 debugfs_remove(symlink); 490 else 491 WARN_ON_ONCE(old); 492 } 493 } 494 } 495 EXPORT_SYMBOL(ref_tracker_dir_symlink); 496 497 static void debugfs_reap_work(struct work_struct *work) 498 { 499 struct dentry *dentry; 500 unsigned long index; 501 bool reaped; 502 503 do { 504 reaped = false; 505 xa_for_each_marked(&debugfs_symlinks, index, dentry, REF_TRACKER_DIR_DEAD) { 506 xa_erase_irq(&debugfs_symlinks, index); 507 debugfs_remove(dentry); 508 reaped = true; 509 } 510 xa_for_each_marked(&debugfs_dentries, index, dentry, REF_TRACKER_DIR_DEAD) { 511 xa_erase_irq(&debugfs_dentries, index); 512 debugfs_remove(dentry); 513 reaped = true; 514 } 515 } while (reaped); 516 } 517 518 static int __init ref_tracker_debugfs_postcore_init(void) 519 { 520 INIT_WORK(&debugfs_reap_worker, debugfs_reap_work); 521 xa_init_flags(&debugfs_dentries, XA_FLAGS_LOCK_IRQ); 522 xa_init_flags(&debugfs_symlinks, XA_FLAGS_LOCK_IRQ); 523 return 0; 524 } 525 postcore_initcall(ref_tracker_debugfs_postcore_init); 526 527 static int __init ref_tracker_debugfs_late_init(void) 528 { 529 ref_tracker_debug_dir = debugfs_create_dir("ref_tracker", NULL); 530 return 0; 531 } 532 late_initcall(ref_tracker_debugfs_late_init); 533 #endif /* CONFIG_DEBUG_FS */ 534