1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/alloc_tag.h> 3 #include <linux/execmem.h> 4 #include <linux/fs.h> 5 #include <linux/gfp.h> 6 #include <linux/kallsyms.h> 7 #include <linux/module.h> 8 #include <linux/page_ext.h> 9 #include <linux/proc_fs.h> 10 #include <linux/seq_buf.h> 11 #include <linux/seq_file.h> 12 #include <linux/vmalloc.h> 13 14 #define ALLOCINFO_FILE_NAME "allocinfo" 15 #define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) 16 #define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) 17 #define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME) 18 19 #ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT 20 static bool mem_profiling_support = true; 21 #else 22 static bool mem_profiling_support; 23 #endif 24 25 static struct codetag_type *alloc_tag_cttype; 26 27 DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); 28 EXPORT_SYMBOL(_shared_alloc_tag); 29 30 DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, 31 mem_alloc_profiling_key); 32 DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed); 33 34 struct alloc_tag_kernel_section kernel_tags = { NULL, 0 }; 35 unsigned long alloc_tag_ref_mask; 36 int alloc_tag_ref_offs; 37 38 struct allocinfo_private { 39 struct codetag_iterator iter; 40 bool print_header; 41 }; 42 43 static void *allocinfo_start(struct seq_file *m, loff_t *pos) 44 { 45 struct allocinfo_private *priv; 46 struct codetag *ct; 47 loff_t node = *pos; 48 49 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 50 m->private = priv; 51 if (!priv) 52 return NULL; 53 54 priv->print_header = (node == 0); 55 codetag_lock_module_list(alloc_tag_cttype, true); 56 priv->iter = codetag_get_ct_iter(alloc_tag_cttype); 57 while ((ct = codetag_next_ct(&priv->iter)) != NULL && node) 58 node--; 59 60 return ct ? priv : NULL; 61 } 62 63 static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) 64 { 65 struct allocinfo_private *priv = (struct allocinfo_private *)arg; 66 struct codetag *ct = codetag_next_ct(&priv->iter); 67 68 (*pos)++; 69 if (!ct) 70 return NULL; 71 72 return priv; 73 } 74 75 static void allocinfo_stop(struct seq_file *m, void *arg) 76 { 77 struct allocinfo_private *priv = (struct allocinfo_private *)m->private; 78 79 if (priv) { 80 codetag_lock_module_list(alloc_tag_cttype, false); 81 kfree(priv); 82 } 83 } 84 85 static void print_allocinfo_header(struct seq_buf *buf) 86 { 87 /* Output format version, so we can change it. */ 88 seq_buf_printf(buf, "allocinfo - version: 1.0\n"); 89 seq_buf_printf(buf, "# <size> <calls> <tag info>\n"); 90 } 91 92 static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) 93 { 94 struct alloc_tag *tag = ct_to_alloc_tag(ct); 95 struct alloc_tag_counters counter = alloc_tag_read(tag); 96 s64 bytes = counter.bytes; 97 98 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); 99 codetag_to_text(out, ct); 100 seq_buf_putc(out, ' '); 101 seq_buf_putc(out, '\n'); 102 } 103 104 static int allocinfo_show(struct seq_file *m, void *arg) 105 { 106 struct allocinfo_private *priv = (struct allocinfo_private *)arg; 107 char *bufp; 108 size_t n = seq_get_buf(m, &bufp); 109 struct seq_buf buf; 110 111 seq_buf_init(&buf, bufp, n); 112 if (priv->print_header) { 113 print_allocinfo_header(&buf); 114 priv->print_header = false; 115 } 116 alloc_tag_to_text(&buf, priv->iter.ct); 117 seq_commit(m, seq_buf_used(&buf)); 118 return 0; 119 } 120 121 static const struct seq_operations allocinfo_seq_op = { 122 .start = allocinfo_start, 123 .next = allocinfo_next, 124 .stop = allocinfo_stop, 125 .show = allocinfo_show, 126 }; 127 128 size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) 129 { 130 struct codetag_iterator iter; 131 struct codetag *ct; 132 struct codetag_bytes n; 133 unsigned int i, nr = 0; 134 135 if (can_sleep) 136 codetag_lock_module_list(alloc_tag_cttype, true); 137 else if (!codetag_trylock_module_list(alloc_tag_cttype)) 138 return 0; 139 140 iter = codetag_get_ct_iter(alloc_tag_cttype); 141 while ((ct = codetag_next_ct(&iter))) { 142 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); 143 144 n.ct = ct; 145 n.bytes = counter.bytes; 146 147 for (i = 0; i < nr; i++) 148 if (n.bytes > tags[i].bytes) 149 break; 150 151 if (i < count) { 152 nr -= nr == count; 153 memmove(&tags[i + 1], 154 &tags[i], 155 sizeof(tags[0]) * (nr - i)); 156 nr++; 157 tags[i] = n; 158 } 159 } 160 161 codetag_lock_module_list(alloc_tag_cttype, false); 162 163 return nr; 164 } 165 166 void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) 167 { 168 int i; 169 struct alloc_tag *tag; 170 unsigned int nr_pages = 1 << new_order; 171 172 if (!mem_alloc_profiling_enabled()) 173 return; 174 175 tag = pgalloc_tag_get(&folio->page); 176 if (!tag) 177 return; 178 179 for (i = nr_pages; i < (1 << old_order); i += nr_pages) { 180 union pgtag_ref_handle handle; 181 union codetag_ref ref; 182 183 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { 184 /* Set new reference to point to the original tag */ 185 alloc_tag_ref_set(&ref, tag); 186 update_page_tag_ref(handle, &ref); 187 put_page_tag_ref(handle); 188 } 189 } 190 } 191 192 void pgalloc_tag_swap(struct folio *new, struct folio *old) 193 { 194 union pgtag_ref_handle handle_old, handle_new; 195 union codetag_ref ref_old, ref_new; 196 struct alloc_tag *tag_old, *tag_new; 197 198 tag_old = pgalloc_tag_get(&old->page); 199 if (!tag_old) 200 return; 201 tag_new = pgalloc_tag_get(&new->page); 202 if (!tag_new) 203 return; 204 205 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) 206 return; 207 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { 208 put_page_tag_ref(handle_old); 209 return; 210 } 211 212 /* 213 * Clear tag references to avoid debug warning when using 214 * __alloc_tag_ref_set() with non-empty reference. 215 */ 216 set_codetag_empty(&ref_old); 217 set_codetag_empty(&ref_new); 218 219 /* swap tags */ 220 __alloc_tag_ref_set(&ref_old, tag_new); 221 update_page_tag_ref(handle_old, &ref_old); 222 __alloc_tag_ref_set(&ref_new, tag_old); 223 update_page_tag_ref(handle_new, &ref_new); 224 225 put_page_tag_ref(handle_old); 226 put_page_tag_ref(handle_new); 227 } 228 229 static void shutdown_mem_profiling(bool remove_file) 230 { 231 if (mem_alloc_profiling_enabled()) 232 static_branch_disable(&mem_alloc_profiling_key); 233 234 if (!mem_profiling_support) 235 return; 236 237 if (remove_file) 238 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); 239 mem_profiling_support = false; 240 } 241 242 static void __init procfs_init(void) 243 { 244 if (!mem_profiling_support) 245 return; 246 247 if (!proc_create_seq(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op)) { 248 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); 249 shutdown_mem_profiling(false); 250 } 251 } 252 253 void __init alloc_tag_sec_init(void) 254 { 255 struct alloc_tag *last_codetag; 256 257 if (!mem_profiling_support) 258 return; 259 260 if (!static_key_enabled(&mem_profiling_compressed)) 261 return; 262 263 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( 264 SECTION_START(ALLOC_TAG_SECTION_NAME)); 265 last_codetag = (struct alloc_tag *)kallsyms_lookup_name( 266 SECTION_STOP(ALLOC_TAG_SECTION_NAME)); 267 kernel_tags.count = last_codetag - kernel_tags.first_tag; 268 269 /* Check if kernel tags fit into page flags */ 270 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { 271 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ 272 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", 273 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); 274 return; 275 } 276 277 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); 278 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); 279 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", 280 NR_UNUSED_PAGEFLAG_BITS); 281 } 282 283 #ifdef CONFIG_MODULES 284 285 static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE); 286 static struct vm_struct *vm_module_tags; 287 /* A dummy object used to indicate an unloaded module */ 288 static struct module unloaded_mod; 289 /* A dummy object used to indicate a module prepended area */ 290 static struct module prepend_mod; 291 292 struct alloc_tag_module_section module_tags; 293 294 static inline unsigned long alloc_tag_align(unsigned long val) 295 { 296 if (!static_key_enabled(&mem_profiling_compressed)) { 297 /* No alignment requirements when we are not indexing the tags */ 298 return val; 299 } 300 301 if (val % sizeof(struct alloc_tag) == 0) 302 return val; 303 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag); 304 } 305 306 static bool ensure_alignment(unsigned long align, unsigned int *prepend) 307 { 308 if (!static_key_enabled(&mem_profiling_compressed)) { 309 /* No alignment requirements when we are not indexing the tags */ 310 return true; 311 } 312 313 /* 314 * If alloc_tag size is not a multiple of required alignment, tag 315 * indexing does not work. 316 */ 317 if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) 318 return false; 319 320 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */ 321 if (*prepend) 322 *prepend = alloc_tag_align(*prepend); 323 324 return true; 325 } 326 327 static inline bool tags_addressable(void) 328 { 329 unsigned long tag_idx_count; 330 331 if (!static_key_enabled(&mem_profiling_compressed)) 332 return true; /* with page_ext tags are always addressable */ 333 334 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + 335 module_tags.size / sizeof(struct alloc_tag); 336 337 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS); 338 } 339 340 static bool needs_section_mem(struct module *mod, unsigned long size) 341 { 342 if (!mem_profiling_support) 343 return false; 344 345 return size >= sizeof(struct alloc_tag); 346 } 347 348 static struct alloc_tag *find_used_tag(struct alloc_tag *from, struct alloc_tag *to) 349 { 350 while (from <= to) { 351 struct alloc_tag_counters counter; 352 353 counter = alloc_tag_read(from); 354 if (counter.bytes) 355 return from; 356 from++; 357 } 358 359 return NULL; 360 } 361 362 /* Called with mod_area_mt locked */ 363 static void clean_unused_module_areas_locked(void) 364 { 365 MA_STATE(mas, &mod_area_mt, 0, module_tags.size); 366 struct module *val; 367 368 mas_for_each(&mas, val, module_tags.size) { 369 if (val != &unloaded_mod) 370 continue; 371 372 /* Release area if all tags are unused */ 373 if (!find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index), 374 (struct alloc_tag *)(module_tags.start_addr + mas.last))) 375 mas_erase(&mas); 376 } 377 } 378 379 /* Called with mod_area_mt locked */ 380 static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, 381 unsigned long size, unsigned int prepend, unsigned long align) 382 { 383 bool cleanup_done = false; 384 385 repeat: 386 /* Try finding exact size and hope the start is aligned */ 387 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { 388 if (IS_ALIGNED(mas->index + prepend, align)) 389 return true; 390 391 /* Try finding larger area to align later */ 392 mas_reset(mas); 393 if (!mas_empty_area(mas, 0, section_size - 1, 394 size + prepend + align - 1)) 395 return true; 396 } 397 398 /* No free area, try cleanup stale data and repeat the search once */ 399 if (!cleanup_done) { 400 clean_unused_module_areas_locked(); 401 cleanup_done = true; 402 mas_reset(mas); 403 goto repeat; 404 } 405 406 return false; 407 } 408 409 static int vm_module_tags_populate(void) 410 { 411 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + 412 (vm_module_tags->nr_pages << PAGE_SHIFT); 413 unsigned long new_end = module_tags.start_addr + module_tags.size; 414 415 if (phys_end < new_end) { 416 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; 417 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); 418 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); 419 unsigned long more_pages; 420 unsigned long nr; 421 422 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; 423 nr = alloc_pages_bulk_array_node(GFP_KERNEL | __GFP_NOWARN, 424 NUMA_NO_NODE, more_pages, next_page); 425 if (nr < more_pages || 426 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, 427 next_page, PAGE_SHIFT) < 0) { 428 /* Clean up and error out */ 429 for (int i = 0; i < nr; i++) 430 __free_page(next_page[i]); 431 return -ENOMEM; 432 } 433 434 vm_module_tags->nr_pages += nr; 435 436 /* 437 * Kasan allocates 1 byte of shadow for every 8 bytes of data. 438 * When kasan_alloc_module_shadow allocates shadow memory, 439 * its unit of allocation is a page. 440 * Therefore, here we need to align to MODULE_ALIGN. 441 */ 442 if (old_shadow_end < new_shadow_end) 443 kasan_alloc_module_shadow((void *)old_shadow_end, 444 new_shadow_end - old_shadow_end, 445 GFP_KERNEL); 446 } 447 448 /* 449 * Mark the pages as accessible, now that they are mapped. 450 * With hardware tag-based KASAN, marking is skipped for 451 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 452 */ 453 kasan_unpoison_vmalloc((void *)module_tags.start_addr, 454 new_end - module_tags.start_addr, 455 KASAN_VMALLOC_PROT_NORMAL); 456 457 return 0; 458 } 459 460 static void *reserve_module_tags(struct module *mod, unsigned long size, 461 unsigned int prepend, unsigned long align) 462 { 463 unsigned long section_size = module_tags.end_addr - module_tags.start_addr; 464 MA_STATE(mas, &mod_area_mt, 0, section_size - 1); 465 unsigned long offset; 466 void *ret = NULL; 467 468 /* If no tags return error */ 469 if (size < sizeof(struct alloc_tag)) 470 return ERR_PTR(-EINVAL); 471 472 /* 473 * align is always power of 2, so we can use IS_ALIGNED and ALIGN. 474 * align 0 or 1 means no alignment, to simplify set to 1. 475 */ 476 if (!align) 477 align = 1; 478 479 if (!ensure_alignment(align, &prepend)) { 480 shutdown_mem_profiling(true); 481 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", 482 mod->name, align); 483 return ERR_PTR(-EINVAL); 484 } 485 486 mas_lock(&mas); 487 if (!find_aligned_area(&mas, section_size, size, prepend, align)) { 488 ret = ERR_PTR(-ENOMEM); 489 goto unlock; 490 } 491 492 /* Mark found area as reserved */ 493 offset = mas.index; 494 offset += prepend; 495 offset = ALIGN(offset, align); 496 if (offset != mas.index) { 497 unsigned long pad_start = mas.index; 498 499 mas.last = offset - 1; 500 mas_store(&mas, &prepend_mod); 501 if (mas_is_err(&mas)) { 502 ret = ERR_PTR(xa_err(mas.node)); 503 goto unlock; 504 } 505 mas.index = offset; 506 mas.last = offset + size - 1; 507 mas_store(&mas, mod); 508 if (mas_is_err(&mas)) { 509 mas.index = pad_start; 510 mas_erase(&mas); 511 ret = ERR_PTR(xa_err(mas.node)); 512 } 513 } else { 514 mas.last = offset + size - 1; 515 mas_store(&mas, mod); 516 if (mas_is_err(&mas)) 517 ret = ERR_PTR(xa_err(mas.node)); 518 } 519 unlock: 520 mas_unlock(&mas); 521 522 if (IS_ERR(ret)) 523 return ret; 524 525 if (module_tags.size < offset + size) { 526 int grow_res; 527 528 module_tags.size = offset + size; 529 if (mem_alloc_profiling_enabled() && !tags_addressable()) { 530 shutdown_mem_profiling(true); 531 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", 532 mod->name, NR_UNUSED_PAGEFLAG_BITS); 533 } 534 535 grow_res = vm_module_tags_populate(); 536 if (grow_res) { 537 shutdown_mem_profiling(true); 538 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", 539 mod->name); 540 return ERR_PTR(grow_res); 541 } 542 } 543 544 return (struct alloc_tag *)(module_tags.start_addr + offset); 545 } 546 547 static void release_module_tags(struct module *mod, bool used) 548 { 549 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); 550 struct alloc_tag *tag; 551 struct module *val; 552 553 mas_lock(&mas); 554 mas_for_each_rev(&mas, val, 0) 555 if (val == mod) 556 break; 557 558 if (!val) /* module not found */ 559 goto out; 560 561 if (!used) 562 goto release_area; 563 564 /* Find out if the area is used */ 565 tag = find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index), 566 (struct alloc_tag *)(module_tags.start_addr + mas.last)); 567 if (tag) { 568 struct alloc_tag_counters counter = alloc_tag_read(tag); 569 570 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", 571 tag->ct.filename, tag->ct.lineno, tag->ct.modname, 572 tag->ct.function, counter.bytes); 573 } else { 574 used = false; 575 } 576 release_area: 577 mas_store(&mas, used ? &unloaded_mod : NULL); 578 val = mas_prev_range(&mas, 0); 579 if (val == &prepend_mod) 580 mas_store(&mas, NULL); 581 out: 582 mas_unlock(&mas); 583 } 584 585 static void replace_module(struct module *mod, struct module *new_mod) 586 { 587 MA_STATE(mas, &mod_area_mt, 0, module_tags.size); 588 struct module *val; 589 590 mas_lock(&mas); 591 mas_for_each(&mas, val, module_tags.size) { 592 if (val != mod) 593 continue; 594 595 mas_store_gfp(&mas, new_mod, GFP_KERNEL); 596 break; 597 } 598 mas_unlock(&mas); 599 } 600 601 static int __init alloc_mod_tags_mem(void) 602 { 603 /* Map space to copy allocation tags */ 604 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); 605 if (!vm_module_tags) { 606 pr_err("Failed to map %lu bytes for module allocation tags\n", 607 MODULE_ALLOC_TAG_VMAP_SIZE); 608 module_tags.start_addr = 0; 609 return -ENOMEM; 610 } 611 612 vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, 613 sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); 614 if (!vm_module_tags->pages) { 615 free_vm_area(vm_module_tags); 616 return -ENOMEM; 617 } 618 619 module_tags.start_addr = (unsigned long)vm_module_tags->addr; 620 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; 621 /* Ensure the base is alloc_tag aligned when required for indexing */ 622 module_tags.start_addr = alloc_tag_align(module_tags.start_addr); 623 624 return 0; 625 } 626 627 static void __init free_mod_tags_mem(void) 628 { 629 int i; 630 631 module_tags.start_addr = 0; 632 for (i = 0; i < vm_module_tags->nr_pages; i++) 633 __free_page(vm_module_tags->pages[i]); 634 kfree(vm_module_tags->pages); 635 free_vm_area(vm_module_tags); 636 } 637 638 #else /* CONFIG_MODULES */ 639 640 static inline int alloc_mod_tags_mem(void) { return 0; } 641 static inline void free_mod_tags_mem(void) {} 642 643 #endif /* CONFIG_MODULES */ 644 645 /* See: Documentation/mm/allocation-profiling.rst */ 646 static int __init setup_early_mem_profiling(char *str) 647 { 648 bool compressed = false; 649 bool enable; 650 651 if (!str || !str[0]) 652 return -EINVAL; 653 654 if (!strncmp(str, "never", 5)) { 655 enable = false; 656 mem_profiling_support = false; 657 pr_info("Memory allocation profiling is disabled!\n"); 658 } else { 659 char *token = strsep(&str, ","); 660 661 if (kstrtobool(token, &enable)) 662 return -EINVAL; 663 664 if (str) { 665 666 if (strcmp(str, "compressed")) 667 return -EINVAL; 668 669 compressed = true; 670 } 671 mem_profiling_support = true; 672 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", 673 compressed ? "with" : "without", enable ? "on" : "off"); 674 } 675 676 if (enable != mem_alloc_profiling_enabled()) { 677 if (enable) 678 static_branch_enable(&mem_alloc_profiling_key); 679 else 680 static_branch_disable(&mem_alloc_profiling_key); 681 } 682 if (compressed != static_key_enabled(&mem_profiling_compressed)) { 683 if (compressed) 684 static_branch_enable(&mem_profiling_compressed); 685 else 686 static_branch_disable(&mem_profiling_compressed); 687 } 688 689 return 0; 690 } 691 early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); 692 693 static __init bool need_page_alloc_tagging(void) 694 { 695 if (static_key_enabled(&mem_profiling_compressed)) 696 return false; 697 698 return mem_profiling_support; 699 } 700 701 static __init void init_page_alloc_tagging(void) 702 { 703 } 704 705 struct page_ext_operations page_alloc_tagging_ops = { 706 .size = sizeof(union codetag_ref), 707 .need = need_page_alloc_tagging, 708 .init = init_page_alloc_tagging, 709 }; 710 EXPORT_SYMBOL(page_alloc_tagging_ops); 711 712 #ifdef CONFIG_SYSCTL 713 static struct ctl_table memory_allocation_profiling_sysctls[] = { 714 { 715 .procname = "mem_profiling", 716 .data = &mem_alloc_profiling_key, 717 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG 718 .mode = 0444, 719 #else 720 .mode = 0644, 721 #endif 722 .proc_handler = proc_do_static_key, 723 }, 724 }; 725 726 static void __init sysctl_init(void) 727 { 728 if (!mem_profiling_support) 729 memory_allocation_profiling_sysctls[0].mode = 0444; 730 731 register_sysctl_init("vm", memory_allocation_profiling_sysctls); 732 } 733 #else /* CONFIG_SYSCTL */ 734 static inline void sysctl_init(void) {} 735 #endif /* CONFIG_SYSCTL */ 736 737 static int __init alloc_tag_init(void) 738 { 739 const struct codetag_type_desc desc = { 740 .section = ALLOC_TAG_SECTION_NAME, 741 .tag_size = sizeof(struct alloc_tag), 742 #ifdef CONFIG_MODULES 743 .needs_section_mem = needs_section_mem, 744 .alloc_section_mem = reserve_module_tags, 745 .free_section_mem = release_module_tags, 746 .module_replaced = replace_module, 747 #endif 748 }; 749 int res; 750 751 res = alloc_mod_tags_mem(); 752 if (res) 753 return res; 754 755 alloc_tag_cttype = codetag_register_type(&desc); 756 if (IS_ERR(alloc_tag_cttype)) { 757 free_mod_tags_mem(); 758 return PTR_ERR(alloc_tag_cttype); 759 } 760 761 sysctl_init(); 762 procfs_init(); 763 764 return 0; 765 } 766 module_init(alloc_tag_init); 767