1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 #define pr_fmt(fmt) "ref_tracker: " fmt
4
5 #include <linux/export.h>
6 #include <linux/list_sort.h>
7 #include <linux/ref_tracker.h>
8 #include <linux/slab.h>
9 #include <linux/stacktrace.h>
10 #include <linux/stackdepot.h>
11 #include <linux/seq_file.h>
12
13 #define REF_TRACKER_STACK_ENTRIES 16
14 #define STACK_BUF_SIZE 1024
15
16 struct ref_tracker {
17 struct list_head head; /* anchor into dir->list or dir->quarantine */
18 bool dead;
19 depot_stack_handle_t alloc_stack_handle;
20 depot_stack_handle_t free_stack_handle;
21 };
22
23 struct ref_tracker_dir_stats {
24 int total;
25 int count;
26 struct {
27 depot_stack_handle_t stack_handle;
28 unsigned int count;
29 } stacks[];
30 };
31
32 #ifdef CONFIG_DEBUG_FS
33 #include <linux/xarray.h>
34
35 /*
36 * ref_tracker_dir_init() is usually called in allocation-safe contexts, but
37 * the same is not true of ref_tracker_dir_exit() which can be called from
38 * anywhere an object is freed. Removing debugfs dentries is a blocking
39 * operation, so we defer that work to the debugfs_reap_worker.
40 *
41 * Each dentry is tracked in the appropriate xarray. When
42 * ref_tracker_dir_exit() is called, its entries in the xarrays are marked and
43 * the workqueue job is scheduled. The worker then runs and deletes any marked
44 * dentries asynchronously.
45 */
46 static struct xarray debugfs_dentries;
47 static struct xarray debugfs_symlinks;
48 static struct work_struct debugfs_reap_worker;
49
50 #define REF_TRACKER_DIR_DEAD XA_MARK_0
ref_tracker_debugfs_mark(struct ref_tracker_dir * dir)51 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir)
52 {
53 unsigned long flags;
54
55 xa_lock_irqsave(&debugfs_dentries, flags);
56 __xa_set_mark(&debugfs_dentries, (unsigned long)dir, REF_TRACKER_DIR_DEAD);
57 xa_unlock_irqrestore(&debugfs_dentries, flags);
58
59 xa_lock_irqsave(&debugfs_symlinks, flags);
60 __xa_set_mark(&debugfs_symlinks, (unsigned long)dir, REF_TRACKER_DIR_DEAD);
61 xa_unlock_irqrestore(&debugfs_symlinks, flags);
62
63 schedule_work(&debugfs_reap_worker);
64 }
65 #else
ref_tracker_debugfs_mark(struct ref_tracker_dir * dir)66 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir)
67 {
68 }
69 #endif
70
71 static struct ref_tracker_dir_stats *
ref_tracker_get_stats(struct ref_tracker_dir * dir,unsigned int limit)72 ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
73 {
74 struct ref_tracker_dir_stats *stats;
75 struct ref_tracker *tracker;
76
77 stats = kmalloc_flex(*stats, stacks, limit, GFP_NOWAIT);
78 if (!stats)
79 return ERR_PTR(-ENOMEM);
80 stats->total = 0;
81 stats->count = 0;
82
83 list_for_each_entry(tracker, &dir->list, head) {
84 depot_stack_handle_t stack = tracker->alloc_stack_handle;
85 int i;
86
87 ++stats->total;
88 for (i = 0; i < stats->count; ++i)
89 if (stats->stacks[i].stack_handle == stack)
90 break;
91 if (i >= limit)
92 continue;
93 if (i >= stats->count) {
94 stats->stacks[i].stack_handle = stack;
95 stats->stacks[i].count = 0;
96 ++stats->count;
97 }
98 ++stats->stacks[i].count;
99 }
100
101 return stats;
102 }
103
104 struct ostream {
105 void __ostream_printf (*func)(struct ostream *stream, char *fmt, ...);
106 char *prefix;
107 char *buf;
108 struct seq_file *seq;
109 int size, used;
110 };
111
pr_ostream_log(struct ostream * stream,char * fmt,...)112 static void __ostream_printf pr_ostream_log(struct ostream *stream, char *fmt, ...)
113 {
114 va_list args;
115
116 va_start(args, fmt);
117 vprintk(fmt, args);
118 va_end(args);
119 }
120
pr_ostream_buf(struct ostream * stream,char * fmt,...)121 static void __ostream_printf pr_ostream_buf(struct ostream *stream, char *fmt, ...)
122 {
123 int ret, len = stream->size - stream->used;
124 va_list args;
125
126 va_start(args, fmt);
127 ret = vsnprintf(stream->buf + stream->used, len, fmt, args);
128 va_end(args);
129 if (ret > 0)
130 stream->used += min(ret, len);
131 }
132
133 #define pr_ostream(stream, fmt, args...) \
134 ({ \
135 struct ostream *_s = (stream); \
136 \
137 _s->func(_s, fmt, ##args); \
138 })
139
140 static void
__ref_tracker_dir_pr_ostream(struct ref_tracker_dir * dir,unsigned int display_limit,struct ostream * s)141 __ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir,
142 unsigned int display_limit, struct ostream *s)
143 {
144 struct ref_tracker_dir_stats *stats;
145 unsigned int i = 0, skipped;
146 depot_stack_handle_t stack;
147 char *sbuf;
148
149 lockdep_assert_held(&dir->lock);
150
151 if (list_empty(&dir->list))
152 return;
153
154 stats = ref_tracker_get_stats(dir, display_limit);
155 if (IS_ERR(stats)) {
156 pr_ostream(s, "%s%s@%p: couldn't get stats, error %pe\n",
157 s->prefix, dir->class, dir, stats);
158 return;
159 }
160
161 sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT);
162
163 for (i = 0, skipped = stats->total; i < stats->count; ++i) {
164 stack = stats->stacks[i].stack_handle;
165 if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4))
166 sbuf[0] = 0;
167 pr_ostream(s, "%s%s@%p has %d/%d users at\n%s\n", s->prefix,
168 dir->class, dir, stats->stacks[i].count,
169 stats->total, sbuf);
170 skipped -= stats->stacks[i].count;
171 }
172
173 if (skipped)
174 pr_ostream(s, "%s%s@%p skipped reports about %d/%d users.\n",
175 s->prefix, dir->class, dir, skipped, stats->total);
176
177 kfree(sbuf);
178
179 kfree(stats);
180 }
181
ref_tracker_dir_print_locked(struct ref_tracker_dir * dir,unsigned int display_limit)182 void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir,
183 unsigned int display_limit)
184 {
185 struct ostream os = { .func = pr_ostream_log,
186 .prefix = "ref_tracker: " };
187
188 __ref_tracker_dir_pr_ostream(dir, display_limit, &os);
189 }
190 EXPORT_SYMBOL(ref_tracker_dir_print_locked);
191
ref_tracker_dir_print(struct ref_tracker_dir * dir,unsigned int display_limit)192 void ref_tracker_dir_print(struct ref_tracker_dir *dir,
193 unsigned int display_limit)
194 {
195 unsigned long flags;
196
197 spin_lock_irqsave(&dir->lock, flags);
198 ref_tracker_dir_print_locked(dir, display_limit);
199 spin_unlock_irqrestore(&dir->lock, flags);
200 }
201 EXPORT_SYMBOL(ref_tracker_dir_print);
202
ref_tracker_dir_snprint(struct ref_tracker_dir * dir,char * buf,size_t size)203 int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size)
204 {
205 struct ostream os = { .func = pr_ostream_buf,
206 .prefix = "ref_tracker: ",
207 .buf = buf,
208 .size = size };
209 unsigned long flags;
210
211 spin_lock_irqsave(&dir->lock, flags);
212 __ref_tracker_dir_pr_ostream(dir, 16, &os);
213 spin_unlock_irqrestore(&dir->lock, flags);
214
215 return os.used;
216 }
217 EXPORT_SYMBOL(ref_tracker_dir_snprint);
218
ref_tracker_dir_exit(struct ref_tracker_dir * dir)219 void ref_tracker_dir_exit(struct ref_tracker_dir *dir)
220 {
221 struct ref_tracker *tracker, *n;
222 unsigned long flags;
223 bool leak = false;
224
225 dir->dead = true;
226 /*
227 * The xarray entries must be marked before the dir->lock is taken to
228 * protect simultaneous debugfs readers.
229 */
230 ref_tracker_debugfs_mark(dir);
231 spin_lock_irqsave(&dir->lock, flags);
232 list_for_each_entry_safe(tracker, n, &dir->quarantine, head) {
233 list_del(&tracker->head);
234 kfree(tracker);
235 dir->quarantine_avail++;
236 }
237 if (!list_empty(&dir->list)) {
238 ref_tracker_dir_print_locked(dir, 16);
239 leak = true;
240 list_for_each_entry_safe(tracker, n, &dir->list, head) {
241 list_del(&tracker->head);
242 kfree(tracker);
243 }
244 }
245 spin_unlock_irqrestore(&dir->lock, flags);
246 WARN_ON_ONCE(leak);
247 WARN_ON_ONCE(refcount_read(&dir->untracked) != 1);
248 WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1);
249 }
250 EXPORT_SYMBOL(ref_tracker_dir_exit);
251
ref_tracker_alloc(struct ref_tracker_dir * dir,struct ref_tracker ** trackerp,gfp_t gfp)252 int ref_tracker_alloc(struct ref_tracker_dir *dir,
253 struct ref_tracker **trackerp,
254 gfp_t gfp)
255 {
256 unsigned long entries[REF_TRACKER_STACK_ENTRIES];
257 struct ref_tracker *tracker;
258 unsigned int nr_entries;
259 gfp_t gfp_mask = gfp | __GFP_NOWARN;
260 unsigned long flags;
261
262 WARN_ON_ONCE(dir->dead);
263
264 if (!trackerp) {
265 refcount_inc(&dir->no_tracker);
266 return 0;
267 }
268 if (gfp & __GFP_DIRECT_RECLAIM)
269 gfp_mask |= __GFP_NOFAIL;
270 *trackerp = tracker = kzalloc_obj(*tracker, gfp_mask);
271 if (unlikely(!tracker)) {
272 pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
273 refcount_inc(&dir->untracked);
274 return -ENOMEM;
275 }
276 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
277 tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp);
278
279 spin_lock_irqsave(&dir->lock, flags);
280 list_add(&tracker->head, &dir->list);
281 spin_unlock_irqrestore(&dir->lock, flags);
282 return 0;
283 }
284 EXPORT_SYMBOL_GPL(ref_tracker_alloc);
285
ref_tracker_free(struct ref_tracker_dir * dir,struct ref_tracker ** trackerp)286 int ref_tracker_free(struct ref_tracker_dir *dir,
287 struct ref_tracker **trackerp)
288 {
289 unsigned long entries[REF_TRACKER_STACK_ENTRIES];
290 depot_stack_handle_t stack_handle;
291 struct ref_tracker *tracker;
292 unsigned int nr_entries;
293 unsigned long flags;
294
295 WARN_ON_ONCE(dir->dead);
296
297 if (!trackerp) {
298 refcount_dec(&dir->no_tracker);
299 return 0;
300 }
301 tracker = *trackerp;
302 if (!tracker) {
303 refcount_dec(&dir->untracked);
304 return -EEXIST;
305 }
306 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
307 stack_handle = stack_depot_save(entries, nr_entries,
308 GFP_NOWAIT);
309
310 spin_lock_irqsave(&dir->lock, flags);
311 if (tracker->dead) {
312 pr_err("reference already released.\n");
313 if (tracker->alloc_stack_handle) {
314 pr_err("allocated in:\n");
315 stack_depot_print(tracker->alloc_stack_handle);
316 }
317 if (tracker->free_stack_handle) {
318 pr_err("freed in:\n");
319 stack_depot_print(tracker->free_stack_handle);
320 }
321 spin_unlock_irqrestore(&dir->lock, flags);
322 WARN_ON_ONCE(1);
323 return -EINVAL;
324 }
325 tracker->dead = true;
326
327 tracker->free_stack_handle = stack_handle;
328
329 list_move_tail(&tracker->head, &dir->quarantine);
330 if (!dir->quarantine_avail) {
331 tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head);
332 list_del(&tracker->head);
333 } else {
334 dir->quarantine_avail--;
335 tracker = NULL;
336 }
337 spin_unlock_irqrestore(&dir->lock, flags);
338
339 kfree(tracker);
340 return 0;
341 }
342 EXPORT_SYMBOL_GPL(ref_tracker_free);
343
344 #ifdef CONFIG_DEBUG_FS
345 #include <linux/debugfs.h>
346
347 static struct dentry *ref_tracker_debug_dir = (struct dentry *)-ENOENT;
348
pr_ostream_seq(struct ostream * stream,char * fmt,...)349 static void __ostream_printf pr_ostream_seq(struct ostream *stream, char *fmt, ...)
350 {
351 va_list args;
352
353 va_start(args, fmt);
354 seq_vprintf(stream->seq, fmt, args);
355 va_end(args);
356 }
357
ref_tracker_dir_seq_print(struct ref_tracker_dir * dir,struct seq_file * seq)358 static int ref_tracker_dir_seq_print(struct ref_tracker_dir *dir, struct seq_file *seq)
359 {
360 struct ostream os = { .func = pr_ostream_seq,
361 .prefix = "",
362 .seq = seq };
363
364 __ref_tracker_dir_pr_ostream(dir, 16, &os);
365
366 return os.used;
367 }
368
ref_tracker_debugfs_show(struct seq_file * f,void * v)369 static int ref_tracker_debugfs_show(struct seq_file *f, void *v)
370 {
371 struct ref_tracker_dir *dir = f->private;
372 unsigned long index = (unsigned long)dir;
373 unsigned long flags;
374 int ret;
375
376 /*
377 * "dir" may not exist at this point if ref_tracker_dir_exit() has
378 * already been called. Take care not to dereference it until its
379 * legitimacy is established.
380 *
381 * The xa_lock is necessary to ensure that "dir" doesn't disappear
382 * before its lock can be taken. If it's in the hash and not marked
383 * dead, then it's safe to take dir->lock which prevents
384 * ref_tracker_dir_exit() from completing. Once the dir->lock is
385 * acquired, the xa_lock can be released. All of this must be IRQ-safe.
386 */
387 xa_lock_irqsave(&debugfs_dentries, flags);
388 if (!xa_load(&debugfs_dentries, index) ||
389 xa_get_mark(&debugfs_dentries, index, REF_TRACKER_DIR_DEAD)) {
390 xa_unlock_irqrestore(&debugfs_dentries, flags);
391 return -ENODATA;
392 }
393
394 spin_lock(&dir->lock);
395 xa_unlock(&debugfs_dentries);
396 ret = ref_tracker_dir_seq_print(dir, f);
397 spin_unlock_irqrestore(&dir->lock, flags);
398 return ret;
399 }
400
ref_tracker_debugfs_open(struct inode * inode,struct file * filp)401 static int ref_tracker_debugfs_open(struct inode *inode, struct file *filp)
402 {
403 struct ref_tracker_dir *dir = inode->i_private;
404
405 return single_open(filp, ref_tracker_debugfs_show, dir);
406 }
407
408 static const struct file_operations ref_tracker_debugfs_fops = {
409 .owner = THIS_MODULE,
410 .open = ref_tracker_debugfs_open,
411 .read = seq_read,
412 .llseek = seq_lseek,
413 .release = single_release,
414 };
415
416 /**
417 * ref_tracker_dir_debugfs - create debugfs file for ref_tracker_dir
418 * @dir: ref_tracker_dir to be associated with debugfs file
419 *
420 * In most cases, a debugfs file will be created automatically for every
421 * ref_tracker_dir. If the object was created before debugfs is brought up
422 * then that may fail. In those cases, it is safe to call this at a later
423 * time to create the file.
424 */
ref_tracker_dir_debugfs(struct ref_tracker_dir * dir)425 void ref_tracker_dir_debugfs(struct ref_tracker_dir *dir)
426 {
427 char name[NAME_MAX + 1];
428 struct dentry *dentry;
429 int ret;
430
431 /* No-op if already created */
432 dentry = xa_load(&debugfs_dentries, (unsigned long)dir);
433 if (dentry && !xa_is_err(dentry))
434 return;
435
436 ret = snprintf(name, sizeof(name), "%s@%p", dir->class, dir);
437 name[sizeof(name) - 1] = '\0';
438
439 if (ret < sizeof(name)) {
440 dentry = debugfs_create_file(name, S_IFREG | 0400,
441 ref_tracker_debug_dir, dir,
442 &ref_tracker_debugfs_fops);
443 if (!IS_ERR(dentry)) {
444 void *old;
445
446 old = xa_store_irq(&debugfs_dentries, (unsigned long)dir,
447 dentry, GFP_KERNEL);
448
449 if (xa_is_err(old))
450 debugfs_remove(dentry);
451 else
452 WARN_ON_ONCE(old);
453 }
454 }
455 }
456 EXPORT_SYMBOL(ref_tracker_dir_debugfs);
457
ref_tracker_dir_symlink(struct ref_tracker_dir * dir,const char * fmt,...)458 void __ostream_printf ref_tracker_dir_symlink(struct ref_tracker_dir *dir, const char *fmt, ...)
459 {
460 char name[NAME_MAX + 1];
461 struct dentry *symlink, *dentry;
462 va_list args;
463 int ret;
464
465 symlink = xa_load(&debugfs_symlinks, (unsigned long)dir);
466 dentry = xa_load(&debugfs_dentries, (unsigned long)dir);
467
468 /* Already created?*/
469 if (symlink && !xa_is_err(symlink))
470 return;
471
472 if (!dentry || xa_is_err(dentry))
473 return;
474
475 va_start(args, fmt);
476 ret = vsnprintf(name, sizeof(name), fmt, args);
477 va_end(args);
478 name[sizeof(name) - 1] = '\0';
479
480 if (ret < sizeof(name)) {
481 symlink = debugfs_create_symlink(name, ref_tracker_debug_dir,
482 dentry->d_name.name);
483 if (!IS_ERR(symlink)) {
484 void *old;
485
486 old = xa_store_irq(&debugfs_symlinks, (unsigned long)dir,
487 symlink, GFP_KERNEL);
488 if (xa_is_err(old))
489 debugfs_remove(symlink);
490 else
491 WARN_ON_ONCE(old);
492 }
493 }
494 }
495 EXPORT_SYMBOL(ref_tracker_dir_symlink);
496
debugfs_reap_work(struct work_struct * work)497 static void debugfs_reap_work(struct work_struct *work)
498 {
499 struct dentry *dentry;
500 unsigned long index;
501 bool reaped;
502
503 do {
504 reaped = false;
505 xa_for_each_marked(&debugfs_symlinks, index, dentry, REF_TRACKER_DIR_DEAD) {
506 xa_erase_irq(&debugfs_symlinks, index);
507 debugfs_remove(dentry);
508 reaped = true;
509 }
510 xa_for_each_marked(&debugfs_dentries, index, dentry, REF_TRACKER_DIR_DEAD) {
511 xa_erase_irq(&debugfs_dentries, index);
512 debugfs_remove(dentry);
513 reaped = true;
514 }
515 } while (reaped);
516 }
517
ref_tracker_debugfs_postcore_init(void)518 static int __init ref_tracker_debugfs_postcore_init(void)
519 {
520 INIT_WORK(&debugfs_reap_worker, debugfs_reap_work);
521 xa_init_flags(&debugfs_dentries, XA_FLAGS_LOCK_IRQ);
522 xa_init_flags(&debugfs_symlinks, XA_FLAGS_LOCK_IRQ);
523 return 0;
524 }
525 postcore_initcall(ref_tracker_debugfs_postcore_init);
526
ref_tracker_debugfs_late_init(void)527 static int __init ref_tracker_debugfs_late_init(void)
528 {
529 ref_tracker_debug_dir = debugfs_create_dir("ref_tracker", NULL);
530 return 0;
531 }
532 late_initcall(ref_tracker_debugfs_late_init);
533 #endif /* CONFIG_DEBUG_FS */
534