1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/alloc_tag.h>
3 #include <linux/execmem.h>
4 #include <linux/fs.h>
5 #include <linux/gfp.h>
6 #include <linux/kallsyms.h>
7 #include <linux/module.h>
8 #include <linux/page_ext.h>
9 #include <linux/proc_fs.h>
10 #include <linux/seq_buf.h>
11 #include <linux/seq_file.h>
12 #include <linux/vmalloc.h>
13
14 #define ALLOCINFO_FILE_NAME "allocinfo"
15 #define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
16 #define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
17 #define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
18
19 #ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
20 static bool mem_profiling_support = true;
21 #else
22 static bool mem_profiling_support;
23 #endif
24
25 static struct codetag_type *alloc_tag_cttype;
26
27 DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
28 EXPORT_SYMBOL(_shared_alloc_tag);
29
30 DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
31 mem_alloc_profiling_key);
32 EXPORT_SYMBOL(mem_alloc_profiling_key);
33
34 DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
35
36 struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
37 unsigned long alloc_tag_ref_mask;
38 int alloc_tag_ref_offs;
39
40 struct allocinfo_private {
41 struct codetag_iterator iter;
42 bool print_header;
43 };
44
allocinfo_start(struct seq_file * m,loff_t * pos)45 static void *allocinfo_start(struct seq_file *m, loff_t *pos)
46 {
47 struct allocinfo_private *priv;
48 struct codetag *ct;
49 loff_t node = *pos;
50
51 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
52 m->private = priv;
53 if (!priv)
54 return NULL;
55
56 priv->print_header = (node == 0);
57 codetag_lock_module_list(alloc_tag_cttype, true);
58 priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
59 while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
60 node--;
61
62 return ct ? priv : NULL;
63 }
64
allocinfo_next(struct seq_file * m,void * arg,loff_t * pos)65 static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
66 {
67 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
68 struct codetag *ct = codetag_next_ct(&priv->iter);
69
70 (*pos)++;
71 if (!ct)
72 return NULL;
73
74 return priv;
75 }
76
allocinfo_stop(struct seq_file * m,void * arg)77 static void allocinfo_stop(struct seq_file *m, void *arg)
78 {
79 struct allocinfo_private *priv = (struct allocinfo_private *)m->private;
80
81 if (priv) {
82 codetag_lock_module_list(alloc_tag_cttype, false);
83 kfree(priv);
84 }
85 }
86
print_allocinfo_header(struct seq_buf * buf)87 static void print_allocinfo_header(struct seq_buf *buf)
88 {
89 /* Output format version, so we can change it. */
90 seq_buf_printf(buf, "allocinfo - version: 1.0\n");
91 seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
92 }
93
alloc_tag_to_text(struct seq_buf * out,struct codetag * ct)94 static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
95 {
96 struct alloc_tag *tag = ct_to_alloc_tag(ct);
97 struct alloc_tag_counters counter = alloc_tag_read(tag);
98 s64 bytes = counter.bytes;
99
100 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
101 codetag_to_text(out, ct);
102 seq_buf_putc(out, ' ');
103 seq_buf_putc(out, '\n');
104 }
105
allocinfo_show(struct seq_file * m,void * arg)106 static int allocinfo_show(struct seq_file *m, void *arg)
107 {
108 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
109 char *bufp;
110 size_t n = seq_get_buf(m, &bufp);
111 struct seq_buf buf;
112
113 seq_buf_init(&buf, bufp, n);
114 if (priv->print_header) {
115 print_allocinfo_header(&buf);
116 priv->print_header = false;
117 }
118 alloc_tag_to_text(&buf, priv->iter.ct);
119 seq_commit(m, seq_buf_used(&buf));
120 return 0;
121 }
122
123 static const struct seq_operations allocinfo_seq_op = {
124 .start = allocinfo_start,
125 .next = allocinfo_next,
126 .stop = allocinfo_stop,
127 .show = allocinfo_show,
128 };
129
alloc_tag_top_users(struct codetag_bytes * tags,size_t count,bool can_sleep)130 size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
131 {
132 struct codetag_iterator iter;
133 struct codetag *ct;
134 struct codetag_bytes n;
135 unsigned int i, nr = 0;
136
137 if (can_sleep)
138 codetag_lock_module_list(alloc_tag_cttype, true);
139 else if (!codetag_trylock_module_list(alloc_tag_cttype))
140 return 0;
141
142 iter = codetag_get_ct_iter(alloc_tag_cttype);
143 while ((ct = codetag_next_ct(&iter))) {
144 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
145
146 n.ct = ct;
147 n.bytes = counter.bytes;
148
149 for (i = 0; i < nr; i++)
150 if (n.bytes > tags[i].bytes)
151 break;
152
153 if (i < count) {
154 nr -= nr == count;
155 memmove(&tags[i + 1],
156 &tags[i],
157 sizeof(tags[0]) * (nr - i));
158 nr++;
159 tags[i] = n;
160 }
161 }
162
163 codetag_lock_module_list(alloc_tag_cttype, false);
164
165 return nr;
166 }
167
pgalloc_tag_split(struct folio * folio,int old_order,int new_order)168 void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
169 {
170 int i;
171 struct alloc_tag *tag;
172 unsigned int nr_pages = 1 << new_order;
173
174 if (!mem_alloc_profiling_enabled())
175 return;
176
177 tag = __pgalloc_tag_get(&folio->page);
178 if (!tag)
179 return;
180
181 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
182 union pgtag_ref_handle handle;
183 union codetag_ref ref;
184
185 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
186 /* Set new reference to point to the original tag */
187 alloc_tag_ref_set(&ref, tag);
188 update_page_tag_ref(handle, &ref);
189 put_page_tag_ref(handle);
190 }
191 }
192 }
193
pgalloc_tag_swap(struct folio * new,struct folio * old)194 void pgalloc_tag_swap(struct folio *new, struct folio *old)
195 {
196 union pgtag_ref_handle handle_old, handle_new;
197 union codetag_ref ref_old, ref_new;
198 struct alloc_tag *tag_old, *tag_new;
199
200 if (!mem_alloc_profiling_enabled())
201 return;
202
203 tag_old = __pgalloc_tag_get(&old->page);
204 if (!tag_old)
205 return;
206 tag_new = __pgalloc_tag_get(&new->page);
207 if (!tag_new)
208 return;
209
210 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
211 return;
212 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
213 put_page_tag_ref(handle_old);
214 return;
215 }
216
217 /*
218 * Clear tag references to avoid debug warning when using
219 * __alloc_tag_ref_set() with non-empty reference.
220 */
221 set_codetag_empty(&ref_old);
222 set_codetag_empty(&ref_new);
223
224 /* swap tags */
225 __alloc_tag_ref_set(&ref_old, tag_new);
226 update_page_tag_ref(handle_old, &ref_old);
227 __alloc_tag_ref_set(&ref_new, tag_old);
228 update_page_tag_ref(handle_new, &ref_new);
229
230 put_page_tag_ref(handle_old);
231 put_page_tag_ref(handle_new);
232 }
233
shutdown_mem_profiling(bool remove_file)234 static void shutdown_mem_profiling(bool remove_file)
235 {
236 if (mem_alloc_profiling_enabled())
237 static_branch_disable(&mem_alloc_profiling_key);
238
239 if (!mem_profiling_support)
240 return;
241
242 if (remove_file)
243 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
244 mem_profiling_support = false;
245 }
246
alloc_tag_sec_init(void)247 void __init alloc_tag_sec_init(void)
248 {
249 struct alloc_tag *last_codetag;
250
251 if (!mem_profiling_support)
252 return;
253
254 if (!static_key_enabled(&mem_profiling_compressed))
255 return;
256
257 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
258 SECTION_START(ALLOC_TAG_SECTION_NAME));
259 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
260 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
261 kernel_tags.count = last_codetag - kernel_tags.first_tag;
262
263 /* Check if kernel tags fit into page flags */
264 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
265 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
266 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
267 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
268 return;
269 }
270
271 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
272 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
273 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
274 NR_UNUSED_PAGEFLAG_BITS);
275 }
276
277 #ifdef CONFIG_MODULES
278
279 static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
280 static struct vm_struct *vm_module_tags;
281 /* A dummy object used to indicate an unloaded module */
282 static struct module unloaded_mod;
283 /* A dummy object used to indicate a module prepended area */
284 static struct module prepend_mod;
285
286 struct alloc_tag_module_section module_tags;
287
alloc_tag_align(unsigned long val)288 static inline unsigned long alloc_tag_align(unsigned long val)
289 {
290 if (!static_key_enabled(&mem_profiling_compressed)) {
291 /* No alignment requirements when we are not indexing the tags */
292 return val;
293 }
294
295 if (val % sizeof(struct alloc_tag) == 0)
296 return val;
297 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
298 }
299
ensure_alignment(unsigned long align,unsigned int * prepend)300 static bool ensure_alignment(unsigned long align, unsigned int *prepend)
301 {
302 if (!static_key_enabled(&mem_profiling_compressed)) {
303 /* No alignment requirements when we are not indexing the tags */
304 return true;
305 }
306
307 /*
308 * If alloc_tag size is not a multiple of required alignment, tag
309 * indexing does not work.
310 */
311 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
312 return false;
313
314 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
315 if (*prepend)
316 *prepend = alloc_tag_align(*prepend);
317
318 return true;
319 }
320
tags_addressable(void)321 static inline bool tags_addressable(void)
322 {
323 unsigned long tag_idx_count;
324
325 if (!static_key_enabled(&mem_profiling_compressed))
326 return true; /* with page_ext tags are always addressable */
327
328 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
329 module_tags.size / sizeof(struct alloc_tag);
330
331 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
332 }
333
needs_section_mem(struct module * mod,unsigned long size)334 static bool needs_section_mem(struct module *mod, unsigned long size)
335 {
336 if (!mem_profiling_support)
337 return false;
338
339 return size >= sizeof(struct alloc_tag);
340 }
341
clean_unused_counters(struct alloc_tag * start_tag,struct alloc_tag * end_tag)342 static bool clean_unused_counters(struct alloc_tag *start_tag,
343 struct alloc_tag *end_tag)
344 {
345 struct alloc_tag *tag;
346 bool ret = true;
347
348 for (tag = start_tag; tag <= end_tag; tag++) {
349 struct alloc_tag_counters counter;
350
351 if (!tag->counters)
352 continue;
353
354 counter = alloc_tag_read(tag);
355 if (!counter.bytes) {
356 free_percpu(tag->counters);
357 tag->counters = NULL;
358 } else {
359 ret = false;
360 }
361 }
362
363 return ret;
364 }
365
366 /* Called with mod_area_mt locked */
clean_unused_module_areas_locked(void)367 static void clean_unused_module_areas_locked(void)
368 {
369 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
370 struct module *val;
371
372 mas_for_each(&mas, val, module_tags.size) {
373 struct alloc_tag *start_tag;
374 struct alloc_tag *end_tag;
375
376 if (val != &unloaded_mod)
377 continue;
378
379 /* Release area if all tags are unused */
380 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
381 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
382 if (clean_unused_counters(start_tag, end_tag))
383 mas_erase(&mas);
384 }
385 }
386
387 /* Called with mod_area_mt locked */
find_aligned_area(struct ma_state * mas,unsigned long section_size,unsigned long size,unsigned int prepend,unsigned long align)388 static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
389 unsigned long size, unsigned int prepend, unsigned long align)
390 {
391 bool cleanup_done = false;
392
393 repeat:
394 /* Try finding exact size and hope the start is aligned */
395 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
396 if (IS_ALIGNED(mas->index + prepend, align))
397 return true;
398
399 /* Try finding larger area to align later */
400 mas_reset(mas);
401 if (!mas_empty_area(mas, 0, section_size - 1,
402 size + prepend + align - 1))
403 return true;
404 }
405
406 /* No free area, try cleanup stale data and repeat the search once */
407 if (!cleanup_done) {
408 clean_unused_module_areas_locked();
409 cleanup_done = true;
410 mas_reset(mas);
411 goto repeat;
412 }
413
414 return false;
415 }
416
vm_module_tags_populate(void)417 static int vm_module_tags_populate(void)
418 {
419 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
420 (vm_module_tags->nr_pages << PAGE_SHIFT);
421 unsigned long new_end = module_tags.start_addr + module_tags.size;
422
423 if (phys_end < new_end) {
424 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
425 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
426 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
427 unsigned long more_pages;
428 unsigned long nr = 0;
429
430 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
431 while (nr < more_pages) {
432 unsigned long allocated;
433
434 allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
435 NUMA_NO_NODE, more_pages - nr, next_page + nr);
436
437 if (!allocated)
438 break;
439 nr += allocated;
440 }
441
442 if (nr < more_pages ||
443 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
444 next_page, PAGE_SHIFT) < 0) {
445 /* Clean up and error out */
446 for (int i = 0; i < nr; i++)
447 __free_page(next_page[i]);
448 return -ENOMEM;
449 }
450
451 vm_module_tags->nr_pages += nr;
452
453 /*
454 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
455 * When kasan_alloc_module_shadow allocates shadow memory,
456 * its unit of allocation is a page.
457 * Therefore, here we need to align to MODULE_ALIGN.
458 */
459 if (old_shadow_end < new_shadow_end)
460 kasan_alloc_module_shadow((void *)old_shadow_end,
461 new_shadow_end - old_shadow_end,
462 GFP_KERNEL);
463 }
464
465 /*
466 * Mark the pages as accessible, now that they are mapped.
467 * With hardware tag-based KASAN, marking is skipped for
468 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
469 */
470 kasan_unpoison_vmalloc((void *)module_tags.start_addr,
471 new_end - module_tags.start_addr,
472 KASAN_VMALLOC_PROT_NORMAL);
473
474 return 0;
475 }
476
reserve_module_tags(struct module * mod,unsigned long size,unsigned int prepend,unsigned long align)477 static void *reserve_module_tags(struct module *mod, unsigned long size,
478 unsigned int prepend, unsigned long align)
479 {
480 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
481 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
482 unsigned long offset;
483 void *ret = NULL;
484
485 /* If no tags return error */
486 if (size < sizeof(struct alloc_tag))
487 return ERR_PTR(-EINVAL);
488
489 /*
490 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
491 * align 0 or 1 means no alignment, to simplify set to 1.
492 */
493 if (!align)
494 align = 1;
495
496 if (!ensure_alignment(align, &prepend)) {
497 shutdown_mem_profiling(true);
498 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
499 mod->name, align);
500 return ERR_PTR(-EINVAL);
501 }
502
503 mas_lock(&mas);
504 if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
505 ret = ERR_PTR(-ENOMEM);
506 goto unlock;
507 }
508
509 /* Mark found area as reserved */
510 offset = mas.index;
511 offset += prepend;
512 offset = ALIGN(offset, align);
513 if (offset != mas.index) {
514 unsigned long pad_start = mas.index;
515
516 mas.last = offset - 1;
517 mas_store(&mas, &prepend_mod);
518 if (mas_is_err(&mas)) {
519 ret = ERR_PTR(xa_err(mas.node));
520 goto unlock;
521 }
522 mas.index = offset;
523 mas.last = offset + size - 1;
524 mas_store(&mas, mod);
525 if (mas_is_err(&mas)) {
526 mas.index = pad_start;
527 mas_erase(&mas);
528 ret = ERR_PTR(xa_err(mas.node));
529 }
530 } else {
531 mas.last = offset + size - 1;
532 mas_store(&mas, mod);
533 if (mas_is_err(&mas))
534 ret = ERR_PTR(xa_err(mas.node));
535 }
536 unlock:
537 mas_unlock(&mas);
538
539 if (IS_ERR(ret))
540 return ret;
541
542 if (module_tags.size < offset + size) {
543 int grow_res;
544
545 module_tags.size = offset + size;
546 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
547 shutdown_mem_profiling(true);
548 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
549 mod->name, NR_UNUSED_PAGEFLAG_BITS);
550 }
551
552 grow_res = vm_module_tags_populate();
553 if (grow_res) {
554 shutdown_mem_profiling(true);
555 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
556 mod->name);
557 return ERR_PTR(grow_res);
558 }
559 }
560
561 return (struct alloc_tag *)(module_tags.start_addr + offset);
562 }
563
release_module_tags(struct module * mod,bool used)564 static void release_module_tags(struct module *mod, bool used)
565 {
566 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
567 struct alloc_tag *start_tag;
568 struct alloc_tag *end_tag;
569 struct module *val;
570
571 mas_lock(&mas);
572 mas_for_each_rev(&mas, val, 0)
573 if (val == mod)
574 break;
575
576 if (!val) /* module not found */
577 goto out;
578
579 if (!used)
580 goto release_area;
581
582 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
583 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
584 if (!clean_unused_counters(start_tag, end_tag)) {
585 struct alloc_tag *tag;
586
587 for (tag = start_tag; tag <= end_tag; tag++) {
588 struct alloc_tag_counters counter;
589
590 if (!tag->counters)
591 continue;
592
593 counter = alloc_tag_read(tag);
594 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
595 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
596 tag->ct.function, counter.bytes);
597 }
598 } else {
599 used = false;
600 }
601 release_area:
602 mas_store(&mas, used ? &unloaded_mod : NULL);
603 val = mas_prev_range(&mas, 0);
604 if (val == &prepend_mod)
605 mas_store(&mas, NULL);
606 out:
607 mas_unlock(&mas);
608 }
609
load_module(struct module * mod,struct codetag * start,struct codetag * stop)610 static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
611 {
612 /* Allocate module alloc_tag percpu counters */
613 struct alloc_tag *start_tag;
614 struct alloc_tag *stop_tag;
615 struct alloc_tag *tag;
616
617 /* percpu counters for core allocations are already statically allocated */
618 if (!mod)
619 return 0;
620
621 start_tag = ct_to_alloc_tag(start);
622 stop_tag = ct_to_alloc_tag(stop);
623 for (tag = start_tag; tag < stop_tag; tag++) {
624 WARN_ON(tag->counters);
625 tag->counters = alloc_percpu(struct alloc_tag_counters);
626 if (!tag->counters) {
627 while (--tag >= start_tag) {
628 free_percpu(tag->counters);
629 tag->counters = NULL;
630 }
631 pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
632 mod->name);
633 return -ENOMEM;
634 }
635 }
636
637 return 0;
638 }
639
replace_module(struct module * mod,struct module * new_mod)640 static void replace_module(struct module *mod, struct module *new_mod)
641 {
642 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
643 struct module *val;
644
645 mas_lock(&mas);
646 mas_for_each(&mas, val, module_tags.size) {
647 if (val != mod)
648 continue;
649
650 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
651 break;
652 }
653 mas_unlock(&mas);
654 }
655
alloc_mod_tags_mem(void)656 static int __init alloc_mod_tags_mem(void)
657 {
658 /* Map space to copy allocation tags */
659 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
660 if (!vm_module_tags) {
661 pr_err("Failed to map %lu bytes for module allocation tags\n",
662 MODULE_ALLOC_TAG_VMAP_SIZE);
663 module_tags.start_addr = 0;
664 return -ENOMEM;
665 }
666
667 vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
668 sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
669 if (!vm_module_tags->pages) {
670 free_vm_area(vm_module_tags);
671 return -ENOMEM;
672 }
673
674 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
675 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
676 /* Ensure the base is alloc_tag aligned when required for indexing */
677 module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
678
679 return 0;
680 }
681
free_mod_tags_mem(void)682 static void __init free_mod_tags_mem(void)
683 {
684 int i;
685
686 module_tags.start_addr = 0;
687 for (i = 0; i < vm_module_tags->nr_pages; i++)
688 __free_page(vm_module_tags->pages[i]);
689 kfree(vm_module_tags->pages);
690 free_vm_area(vm_module_tags);
691 }
692
693 #else /* CONFIG_MODULES */
694
alloc_mod_tags_mem(void)695 static inline int alloc_mod_tags_mem(void) { return 0; }
free_mod_tags_mem(void)696 static inline void free_mod_tags_mem(void) {}
697
698 #endif /* CONFIG_MODULES */
699
700 /* See: Documentation/mm/allocation-profiling.rst */
setup_early_mem_profiling(char * str)701 static int __init setup_early_mem_profiling(char *str)
702 {
703 bool compressed = false;
704 bool enable;
705
706 if (!str || !str[0])
707 return -EINVAL;
708
709 if (!strncmp(str, "never", 5)) {
710 enable = false;
711 mem_profiling_support = false;
712 pr_info("Memory allocation profiling is disabled!\n");
713 } else {
714 char *token = strsep(&str, ",");
715
716 if (kstrtobool(token, &enable))
717 return -EINVAL;
718
719 if (str) {
720
721 if (strcmp(str, "compressed"))
722 return -EINVAL;
723
724 compressed = true;
725 }
726 mem_profiling_support = true;
727 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
728 compressed ? "with" : "without", enable ? "on" : "off");
729 }
730
731 if (enable != mem_alloc_profiling_enabled()) {
732 if (enable)
733 static_branch_enable(&mem_alloc_profiling_key);
734 else
735 static_branch_disable(&mem_alloc_profiling_key);
736 }
737 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
738 if (compressed)
739 static_branch_enable(&mem_profiling_compressed);
740 else
741 static_branch_disable(&mem_profiling_compressed);
742 }
743
744 return 0;
745 }
746 early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
747
need_page_alloc_tagging(void)748 static __init bool need_page_alloc_tagging(void)
749 {
750 if (static_key_enabled(&mem_profiling_compressed))
751 return false;
752
753 return mem_profiling_support;
754 }
755
init_page_alloc_tagging(void)756 static __init void init_page_alloc_tagging(void)
757 {
758 }
759
760 struct page_ext_operations page_alloc_tagging_ops = {
761 .size = sizeof(union codetag_ref),
762 .need = need_page_alloc_tagging,
763 .init = init_page_alloc_tagging,
764 };
765 EXPORT_SYMBOL(page_alloc_tagging_ops);
766
767 #ifdef CONFIG_SYSCTL
768 static struct ctl_table memory_allocation_profiling_sysctls[] = {
769 {
770 .procname = "mem_profiling",
771 .data = &mem_alloc_profiling_key,
772 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
773 .mode = 0444,
774 #else
775 .mode = 0644,
776 #endif
777 .proc_handler = proc_do_static_key,
778 },
779 };
780
sysctl_init(void)781 static void __init sysctl_init(void)
782 {
783 if (!mem_profiling_support)
784 memory_allocation_profiling_sysctls[0].mode = 0444;
785
786 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
787 }
788 #else /* CONFIG_SYSCTL */
sysctl_init(void)789 static inline void sysctl_init(void) {}
790 #endif /* CONFIG_SYSCTL */
791
alloc_tag_init(void)792 static int __init alloc_tag_init(void)
793 {
794 const struct codetag_type_desc desc = {
795 .section = ALLOC_TAG_SECTION_NAME,
796 .tag_size = sizeof(struct alloc_tag),
797 #ifdef CONFIG_MODULES
798 .needs_section_mem = needs_section_mem,
799 .alloc_section_mem = reserve_module_tags,
800 .free_section_mem = release_module_tags,
801 .module_load = load_module,
802 .module_replaced = replace_module,
803 #endif
804 };
805 int res;
806
807 sysctl_init();
808
809 if (!mem_profiling_support) {
810 pr_info("Memory allocation profiling is not supported!\n");
811 return 0;
812 }
813
814 if (!proc_create_seq(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op)) {
815 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
816 shutdown_mem_profiling(false);
817 return -ENOMEM;
818 }
819
820 res = alloc_mod_tags_mem();
821 if (res) {
822 pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
823 shutdown_mem_profiling(true);
824 return res;
825 }
826
827 alloc_tag_cttype = codetag_register_type(&desc);
828 if (IS_ERR(alloc_tag_cttype)) {
829 pr_err("Allocation tags registration failed, errno = %ld\n", PTR_ERR(alloc_tag_cttype));
830 free_mod_tags_mem();
831 shutdown_mem_profiling(true);
832 return PTR_ERR(alloc_tag_cttype);
833 }
834
835 return 0;
836 }
837 module_init(alloc_tag_init);
838