1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Slab allocator functions that are independent of the allocator strategy 4 * 5 * (C) 2012 Christoph Lameter <cl@linux.com> 6 */ 7 #include <linux/slab.h> 8 9 #include <linux/mm.h> 10 #include <linux/poison.h> 11 #include <linux/interrupt.h> 12 #include <linux/memory.h> 13 #include <linux/cache.h> 14 #include <linux/compiler.h> 15 #include <linux/kfence.h> 16 #include <linux/module.h> 17 #include <linux/cpu.h> 18 #include <linux/uaccess.h> 19 #include <linux/seq_file.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/swiotlb.h> 22 #include <linux/proc_fs.h> 23 #include <linux/debugfs.h> 24 #include <linux/kmemleak.h> 25 #include <linux/kasan.h> 26 #include <asm/cacheflush.h> 27 #include <asm/tlbflush.h> 28 #include <asm/page.h> 29 #include <linux/memcontrol.h> 30 #include <linux/stackdepot.h> 31 32 #include "internal.h" 33 #include "slab.h" 34 35 #define CREATE_TRACE_POINTS 36 #include <trace/events/kmem.h> 37 38 enum slab_state slab_state; 39 LIST_HEAD(slab_caches); 40 DEFINE_MUTEX(slab_mutex); 41 struct kmem_cache *kmem_cache; 42 43 static LIST_HEAD(slab_caches_to_rcu_destroy); 44 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work); 45 static DECLARE_WORK(slab_caches_to_rcu_destroy_work, 46 slab_caches_to_rcu_destroy_workfn); 47 48 /* 49 * Set of flags that will prevent slab merging 50 */ 51 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 52 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ 53 SLAB_FAILSLAB | SLAB_NO_MERGE | kasan_never_merge()) 54 55 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ 56 SLAB_CACHE_DMA32 | SLAB_ACCOUNT) 57 58 /* 59 * Merge control. If this is set then no merging of slab caches will occur. 60 */ 61 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT); 62 63 static int __init setup_slab_nomerge(char *str) 64 { 65 slab_nomerge = true; 66 return 1; 67 } 68 69 static int __init setup_slab_merge(char *str) 70 { 71 slab_nomerge = false; 72 return 1; 73 } 74 75 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0); 76 __setup_param("slub_merge", slub_merge, setup_slab_merge, 0); 77 78 __setup("slab_nomerge", setup_slab_nomerge); 79 __setup("slab_merge", setup_slab_merge); 80 81 /* 82 * Determine the size of a slab object 83 */ 84 unsigned int kmem_cache_size(struct kmem_cache *s) 85 { 86 return s->object_size; 87 } 88 EXPORT_SYMBOL(kmem_cache_size); 89 90 #ifdef CONFIG_DEBUG_VM 91 static int kmem_cache_sanity_check(const char *name, unsigned int size) 92 { 93 if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) { 94 pr_err("kmem_cache_create(%s) integrity check failed\n", name); 95 return -EINVAL; 96 } 97 98 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 99 return 0; 100 } 101 #else 102 static inline int kmem_cache_sanity_check(const char *name, unsigned int size) 103 { 104 return 0; 105 } 106 #endif 107 108 /* 109 * Figure out what the alignment of the objects will be given a set of 110 * flags, a user specified alignment and the size of the objects. 111 */ 112 static unsigned int calculate_alignment(slab_flags_t flags, 113 unsigned int align, unsigned int size) 114 { 115 /* 116 * If the user wants hardware cache aligned objects then follow that 117 * suggestion if the object is sufficiently large. 118 * 119 * The hardware cache alignment cannot override the specified 120 * alignment though. If that is greater then use it. 121 */ 122 if (flags & SLAB_HWCACHE_ALIGN) { 123 unsigned int ralign; 124 125 ralign = cache_line_size(); 126 while (size <= ralign / 2) 127 ralign /= 2; 128 align = max(align, ralign); 129 } 130 131 align = max(align, arch_slab_minalign()); 132 133 return ALIGN(align, sizeof(void *)); 134 } 135 136 /* 137 * Find a mergeable slab cache 138 */ 139 int slab_unmergeable(struct kmem_cache *s) 140 { 141 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE)) 142 return 1; 143 144 if (s->ctor) 145 return 1; 146 147 #ifdef CONFIG_HARDENED_USERCOPY 148 if (s->usersize) 149 return 1; 150 #endif 151 152 /* 153 * We may have set a slab to be unmergeable during bootstrap. 154 */ 155 if (s->refcount < 0) 156 return 1; 157 158 return 0; 159 } 160 161 struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, 162 slab_flags_t flags, const char *name, void (*ctor)(void *)) 163 { 164 struct kmem_cache *s; 165 166 if (slab_nomerge) 167 return NULL; 168 169 if (ctor) 170 return NULL; 171 172 size = ALIGN(size, sizeof(void *)); 173 align = calculate_alignment(flags, align, size); 174 size = ALIGN(size, align); 175 flags = kmem_cache_flags(size, flags, name); 176 177 if (flags & SLAB_NEVER_MERGE) 178 return NULL; 179 180 list_for_each_entry_reverse(s, &slab_caches, list) { 181 if (slab_unmergeable(s)) 182 continue; 183 184 if (size > s->size) 185 continue; 186 187 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME)) 188 continue; 189 /* 190 * Check if alignment is compatible. 191 * Courtesy of Adrian Drzewiecki 192 */ 193 if ((s->size & ~(align - 1)) != s->size) 194 continue; 195 196 if (s->size - size >= sizeof(void *)) 197 continue; 198 199 return s; 200 } 201 return NULL; 202 } 203 204 static struct kmem_cache *create_cache(const char *name, 205 unsigned int object_size, unsigned int align, 206 slab_flags_t flags, unsigned int useroffset, 207 unsigned int usersize, void (*ctor)(void *), 208 struct kmem_cache *root_cache) 209 { 210 struct kmem_cache *s; 211 int err; 212 213 if (WARN_ON(useroffset + usersize > object_size)) 214 useroffset = usersize = 0; 215 216 err = -ENOMEM; 217 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 218 if (!s) 219 goto out; 220 221 s->name = name; 222 s->size = s->object_size = object_size; 223 s->align = align; 224 s->ctor = ctor; 225 #ifdef CONFIG_HARDENED_USERCOPY 226 s->useroffset = useroffset; 227 s->usersize = usersize; 228 #endif 229 230 err = __kmem_cache_create(s, flags); 231 if (err) 232 goto out_free_cache; 233 234 s->refcount = 1; 235 list_add(&s->list, &slab_caches); 236 return s; 237 238 out_free_cache: 239 kmem_cache_free(kmem_cache, s); 240 out: 241 return ERR_PTR(err); 242 } 243 244 /** 245 * kmem_cache_create_usercopy - Create a cache with a region suitable 246 * for copying to userspace 247 * @name: A string which is used in /proc/slabinfo to identify this cache. 248 * @size: The size of objects to be created in this cache. 249 * @align: The required alignment for the objects. 250 * @flags: SLAB flags 251 * @useroffset: Usercopy region offset 252 * @usersize: Usercopy region size 253 * @ctor: A constructor for the objects. 254 * 255 * Cannot be called within a interrupt, but can be interrupted. 256 * The @ctor is run when new pages are allocated by the cache. 257 * 258 * The flags are 259 * 260 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 261 * to catch references to uninitialised memory. 262 * 263 * %SLAB_RED_ZONE - Insert `Red` zones around the allocated memory to check 264 * for buffer overruns. 265 * 266 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 267 * cacheline. This can be beneficial if you're counting cycles as closely 268 * as davem. 269 * 270 * Return: a pointer to the cache on success, NULL on failure. 271 */ 272 struct kmem_cache * 273 kmem_cache_create_usercopy(const char *name, 274 unsigned int size, unsigned int align, 275 slab_flags_t flags, 276 unsigned int useroffset, unsigned int usersize, 277 void (*ctor)(void *)) 278 { 279 struct kmem_cache *s = NULL; 280 const char *cache_name; 281 int err; 282 283 #ifdef CONFIG_SLUB_DEBUG 284 /* 285 * If no slub_debug was enabled globally, the static key is not yet 286 * enabled by setup_slub_debug(). Enable it if the cache is being 287 * created with any of the debugging flags passed explicitly. 288 * It's also possible that this is the first cache created with 289 * SLAB_STORE_USER and we should init stack_depot for it. 290 */ 291 if (flags & SLAB_DEBUG_FLAGS) 292 static_branch_enable(&slub_debug_enabled); 293 if (flags & SLAB_STORE_USER) 294 stack_depot_init(); 295 #endif 296 297 mutex_lock(&slab_mutex); 298 299 err = kmem_cache_sanity_check(name, size); 300 if (err) { 301 goto out_unlock; 302 } 303 304 /* Refuse requests with allocator specific flags */ 305 if (flags & ~SLAB_FLAGS_PERMITTED) { 306 err = -EINVAL; 307 goto out_unlock; 308 } 309 310 /* 311 * Some allocators will constraint the set of valid flags to a subset 312 * of all flags. We expect them to define CACHE_CREATE_MASK in this 313 * case, and we'll just provide them with a sanitized version of the 314 * passed flags. 315 */ 316 flags &= CACHE_CREATE_MASK; 317 318 /* Fail closed on bad usersize of useroffset values. */ 319 if (!IS_ENABLED(CONFIG_HARDENED_USERCOPY) || 320 WARN_ON(!usersize && useroffset) || 321 WARN_ON(size < usersize || size - usersize < useroffset)) 322 usersize = useroffset = 0; 323 324 if (!usersize) 325 s = __kmem_cache_alias(name, size, align, flags, ctor); 326 if (s) 327 goto out_unlock; 328 329 cache_name = kstrdup_const(name, GFP_KERNEL); 330 if (!cache_name) { 331 err = -ENOMEM; 332 goto out_unlock; 333 } 334 335 s = create_cache(cache_name, size, 336 calculate_alignment(flags, align, size), 337 flags, useroffset, usersize, ctor, NULL); 338 if (IS_ERR(s)) { 339 err = PTR_ERR(s); 340 kfree_const(cache_name); 341 } 342 343 out_unlock: 344 mutex_unlock(&slab_mutex); 345 346 if (err) { 347 if (flags & SLAB_PANIC) 348 panic("%s: Failed to create slab '%s'. Error %d\n", 349 __func__, name, err); 350 else { 351 pr_warn("%s(%s) failed with error %d\n", 352 __func__, name, err); 353 dump_stack(); 354 } 355 return NULL; 356 } 357 return s; 358 } 359 EXPORT_SYMBOL(kmem_cache_create_usercopy); 360 361 /** 362 * kmem_cache_create - Create a cache. 363 * @name: A string which is used in /proc/slabinfo to identify this cache. 364 * @size: The size of objects to be created in this cache. 365 * @align: The required alignment for the objects. 366 * @flags: SLAB flags 367 * @ctor: A constructor for the objects. 368 * 369 * Cannot be called within a interrupt, but can be interrupted. 370 * The @ctor is run when new pages are allocated by the cache. 371 * 372 * The flags are 373 * 374 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 375 * to catch references to uninitialised memory. 376 * 377 * %SLAB_RED_ZONE - Insert `Red` zones around the allocated memory to check 378 * for buffer overruns. 379 * 380 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 381 * cacheline. This can be beneficial if you're counting cycles as closely 382 * as davem. 383 * 384 * Return: a pointer to the cache on success, NULL on failure. 385 */ 386 struct kmem_cache * 387 kmem_cache_create(const char *name, unsigned int size, unsigned int align, 388 slab_flags_t flags, void (*ctor)(void *)) 389 { 390 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0, 391 ctor); 392 } 393 EXPORT_SYMBOL(kmem_cache_create); 394 395 #ifdef SLAB_SUPPORTS_SYSFS 396 /* 397 * For a given kmem_cache, kmem_cache_destroy() should only be called 398 * once or there will be a use-after-free problem. The actual deletion 399 * and release of the kobject does not need slab_mutex or cpu_hotplug_lock 400 * protection. So they are now done without holding those locks. 401 * 402 * Note that there will be a slight delay in the deletion of sysfs files 403 * if kmem_cache_release() is called indrectly from a work function. 404 */ 405 static void kmem_cache_release(struct kmem_cache *s) 406 { 407 sysfs_slab_unlink(s); 408 sysfs_slab_release(s); 409 } 410 #else 411 static void kmem_cache_release(struct kmem_cache *s) 412 { 413 slab_kmem_cache_release(s); 414 } 415 #endif 416 417 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work) 418 { 419 LIST_HEAD(to_destroy); 420 struct kmem_cache *s, *s2; 421 422 /* 423 * On destruction, SLAB_TYPESAFE_BY_RCU kmem_caches are put on the 424 * @slab_caches_to_rcu_destroy list. The slab pages are freed 425 * through RCU and the associated kmem_cache are dereferenced 426 * while freeing the pages, so the kmem_caches should be freed only 427 * after the pending RCU operations are finished. As rcu_barrier() 428 * is a pretty slow operation, we batch all pending destructions 429 * asynchronously. 430 */ 431 mutex_lock(&slab_mutex); 432 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy); 433 mutex_unlock(&slab_mutex); 434 435 if (list_empty(&to_destroy)) 436 return; 437 438 rcu_barrier(); 439 440 list_for_each_entry_safe(s, s2, &to_destroy, list) { 441 debugfs_slab_release(s); 442 kfence_shutdown_cache(s); 443 kmem_cache_release(s); 444 } 445 } 446 447 static int shutdown_cache(struct kmem_cache *s) 448 { 449 /* free asan quarantined objects */ 450 kasan_cache_shutdown(s); 451 452 if (__kmem_cache_shutdown(s) != 0) 453 return -EBUSY; 454 455 list_del(&s->list); 456 457 if (s->flags & SLAB_TYPESAFE_BY_RCU) { 458 list_add_tail(&s->list, &slab_caches_to_rcu_destroy); 459 schedule_work(&slab_caches_to_rcu_destroy_work); 460 } else { 461 kfence_shutdown_cache(s); 462 debugfs_slab_release(s); 463 } 464 465 return 0; 466 } 467 468 void slab_kmem_cache_release(struct kmem_cache *s) 469 { 470 __kmem_cache_release(s); 471 kfree_const(s->name); 472 kmem_cache_free(kmem_cache, s); 473 } 474 475 void kmem_cache_destroy(struct kmem_cache *s) 476 { 477 int err = -EBUSY; 478 bool rcu_set; 479 480 if (unlikely(!s) || !kasan_check_byte(s)) 481 return; 482 483 cpus_read_lock(); 484 mutex_lock(&slab_mutex); 485 486 rcu_set = s->flags & SLAB_TYPESAFE_BY_RCU; 487 488 s->refcount--; 489 if (s->refcount) 490 goto out_unlock; 491 492 err = shutdown_cache(s); 493 WARN(err, "%s %s: Slab cache still has objects when called from %pS", 494 __func__, s->name, (void *)_RET_IP_); 495 out_unlock: 496 mutex_unlock(&slab_mutex); 497 cpus_read_unlock(); 498 if (!err && !rcu_set) 499 kmem_cache_release(s); 500 } 501 EXPORT_SYMBOL(kmem_cache_destroy); 502 503 /** 504 * kmem_cache_shrink - Shrink a cache. 505 * @cachep: The cache to shrink. 506 * 507 * Releases as many slabs as possible for a cache. 508 * To help debugging, a zero exit status indicates all slabs were released. 509 * 510 * Return: %0 if all slabs were released, non-zero otherwise 511 */ 512 int kmem_cache_shrink(struct kmem_cache *cachep) 513 { 514 kasan_cache_shrink(cachep); 515 516 return __kmem_cache_shrink(cachep); 517 } 518 EXPORT_SYMBOL(kmem_cache_shrink); 519 520 bool slab_is_available(void) 521 { 522 return slab_state >= UP; 523 } 524 525 #ifdef CONFIG_PRINTK 526 static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 527 { 528 if (__kfence_obj_info(kpp, object, slab)) 529 return; 530 __kmem_obj_info(kpp, object, slab); 531 } 532 533 /** 534 * kmem_dump_obj - Print available slab provenance information 535 * @object: slab object for which to find provenance information. 536 * 537 * This function uses pr_cont(), so that the caller is expected to have 538 * printed out whatever preamble is appropriate. The provenance information 539 * depends on the type of object and on how much debugging is enabled. 540 * For a slab-cache object, the fact that it is a slab object is printed, 541 * and, if available, the slab name, return address, and stack trace from 542 * the allocation and last free path of that object. 543 * 544 * Return: %true if the pointer is to a not-yet-freed object from 545 * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer 546 * is to an already-freed object, and %false otherwise. 547 */ 548 bool kmem_dump_obj(void *object) 549 { 550 char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc"; 551 int i; 552 struct slab *slab; 553 unsigned long ptroffset; 554 struct kmem_obj_info kp = { }; 555 556 /* Some arches consider ZERO_SIZE_PTR to be a valid address. */ 557 if (object < (void *)PAGE_SIZE || !virt_addr_valid(object)) 558 return false; 559 slab = virt_to_slab(object); 560 if (!slab) 561 return false; 562 563 kmem_obj_info(&kp, object, slab); 564 if (kp.kp_slab_cache) 565 pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name); 566 else 567 pr_cont(" slab%s", cp); 568 if (is_kfence_address(object)) 569 pr_cont(" (kfence)"); 570 if (kp.kp_objp) 571 pr_cont(" start %px", kp.kp_objp); 572 if (kp.kp_data_offset) 573 pr_cont(" data offset %lu", kp.kp_data_offset); 574 if (kp.kp_objp) { 575 ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset; 576 pr_cont(" pointer offset %lu", ptroffset); 577 } 578 if (kp.kp_slab_cache && kp.kp_slab_cache->object_size) 579 pr_cont(" size %u", kp.kp_slab_cache->object_size); 580 if (kp.kp_ret) 581 pr_cont(" allocated at %pS\n", kp.kp_ret); 582 else 583 pr_cont("\n"); 584 for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) { 585 if (!kp.kp_stack[i]) 586 break; 587 pr_info(" %pS\n", kp.kp_stack[i]); 588 } 589 590 if (kp.kp_free_stack[0]) 591 pr_cont(" Free path:\n"); 592 593 for (i = 0; i < ARRAY_SIZE(kp.kp_free_stack); i++) { 594 if (!kp.kp_free_stack[i]) 595 break; 596 pr_info(" %pS\n", kp.kp_free_stack[i]); 597 } 598 599 return true; 600 } 601 EXPORT_SYMBOL_GPL(kmem_dump_obj); 602 #endif 603 604 /* Create a cache during boot when no slab services are available yet */ 605 void __init create_boot_cache(struct kmem_cache *s, const char *name, 606 unsigned int size, slab_flags_t flags, 607 unsigned int useroffset, unsigned int usersize) 608 { 609 int err; 610 unsigned int align = ARCH_KMALLOC_MINALIGN; 611 612 s->name = name; 613 s->size = s->object_size = size; 614 615 /* 616 * For power of two sizes, guarantee natural alignment for kmalloc 617 * caches, regardless of SL*B debugging options. 618 */ 619 if (is_power_of_2(size)) 620 align = max(align, size); 621 s->align = calculate_alignment(flags, align, size); 622 623 #ifdef CONFIG_HARDENED_USERCOPY 624 s->useroffset = useroffset; 625 s->usersize = usersize; 626 #endif 627 628 err = __kmem_cache_create(s, flags); 629 630 if (err) 631 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n", 632 name, size, err); 633 634 s->refcount = -1; /* Exempt from merging for now */ 635 } 636 637 static struct kmem_cache *__init create_kmalloc_cache(const char *name, 638 unsigned int size, 639 slab_flags_t flags) 640 { 641 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 642 643 if (!s) 644 panic("Out of memory when creating slab %s\n", name); 645 646 create_boot_cache(s, name, size, flags | SLAB_KMALLOC, 0, size); 647 list_add(&s->list, &slab_caches); 648 s->refcount = 1; 649 return s; 650 } 651 652 struct kmem_cache * 653 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init = 654 { /* initialization for https://bugs.llvm.org/show_bug.cgi?id=42570 */ }; 655 EXPORT_SYMBOL(kmalloc_caches); 656 657 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 658 unsigned long random_kmalloc_seed __ro_after_init; 659 EXPORT_SYMBOL(random_kmalloc_seed); 660 #endif 661 662 /* 663 * Conversion table for small slabs sizes / 8 to the index in the 664 * kmalloc array. This is necessary for slabs < 192 since we have non power 665 * of two cache sizes there. The size of larger slabs can be determined using 666 * fls. 667 */ 668 u8 kmalloc_size_index[24] __ro_after_init = { 669 3, /* 8 */ 670 4, /* 16 */ 671 5, /* 24 */ 672 5, /* 32 */ 673 6, /* 40 */ 674 6, /* 48 */ 675 6, /* 56 */ 676 6, /* 64 */ 677 1, /* 72 */ 678 1, /* 80 */ 679 1, /* 88 */ 680 1, /* 96 */ 681 7, /* 104 */ 682 7, /* 112 */ 683 7, /* 120 */ 684 7, /* 128 */ 685 2, /* 136 */ 686 2, /* 144 */ 687 2, /* 152 */ 688 2, /* 160 */ 689 2, /* 168 */ 690 2, /* 176 */ 691 2, /* 184 */ 692 2 /* 192 */ 693 }; 694 695 size_t kmalloc_size_roundup(size_t size) 696 { 697 if (size && size <= KMALLOC_MAX_CACHE_SIZE) { 698 /* 699 * The flags don't matter since size_index is common to all. 700 * Neither does the caller for just getting ->object_size. 701 */ 702 return kmalloc_slab(size, GFP_KERNEL, 0)->object_size; 703 } 704 705 /* Above the smaller buckets, size is a multiple of page size. */ 706 if (size && size <= KMALLOC_MAX_SIZE) 707 return PAGE_SIZE << get_order(size); 708 709 /* 710 * Return 'size' for 0 - kmalloc() returns ZERO_SIZE_PTR 711 * and very large size - kmalloc() may fail. 712 */ 713 return size; 714 715 } 716 EXPORT_SYMBOL(kmalloc_size_roundup); 717 718 #ifdef CONFIG_ZONE_DMA 719 #define KMALLOC_DMA_NAME(sz) .name[KMALLOC_DMA] = "dma-kmalloc-" #sz, 720 #else 721 #define KMALLOC_DMA_NAME(sz) 722 #endif 723 724 #ifdef CONFIG_MEMCG_KMEM 725 #define KMALLOC_CGROUP_NAME(sz) .name[KMALLOC_CGROUP] = "kmalloc-cg-" #sz, 726 #else 727 #define KMALLOC_CGROUP_NAME(sz) 728 #endif 729 730 #ifndef CONFIG_SLUB_TINY 731 #define KMALLOC_RCL_NAME(sz) .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #sz, 732 #else 733 #define KMALLOC_RCL_NAME(sz) 734 #endif 735 736 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 737 #define __KMALLOC_RANDOM_CONCAT(a, b) a ## b 738 #define KMALLOC_RANDOM_NAME(N, sz) __KMALLOC_RANDOM_CONCAT(KMA_RAND_, N)(sz) 739 #define KMA_RAND_1(sz) .name[KMALLOC_RANDOM_START + 1] = "kmalloc-rnd-01-" #sz, 740 #define KMA_RAND_2(sz) KMA_RAND_1(sz) .name[KMALLOC_RANDOM_START + 2] = "kmalloc-rnd-02-" #sz, 741 #define KMA_RAND_3(sz) KMA_RAND_2(sz) .name[KMALLOC_RANDOM_START + 3] = "kmalloc-rnd-03-" #sz, 742 #define KMA_RAND_4(sz) KMA_RAND_3(sz) .name[KMALLOC_RANDOM_START + 4] = "kmalloc-rnd-04-" #sz, 743 #define KMA_RAND_5(sz) KMA_RAND_4(sz) .name[KMALLOC_RANDOM_START + 5] = "kmalloc-rnd-05-" #sz, 744 #define KMA_RAND_6(sz) KMA_RAND_5(sz) .name[KMALLOC_RANDOM_START + 6] = "kmalloc-rnd-06-" #sz, 745 #define KMA_RAND_7(sz) KMA_RAND_6(sz) .name[KMALLOC_RANDOM_START + 7] = "kmalloc-rnd-07-" #sz, 746 #define KMA_RAND_8(sz) KMA_RAND_7(sz) .name[KMALLOC_RANDOM_START + 8] = "kmalloc-rnd-08-" #sz, 747 #define KMA_RAND_9(sz) KMA_RAND_8(sz) .name[KMALLOC_RANDOM_START + 9] = "kmalloc-rnd-09-" #sz, 748 #define KMA_RAND_10(sz) KMA_RAND_9(sz) .name[KMALLOC_RANDOM_START + 10] = "kmalloc-rnd-10-" #sz, 749 #define KMA_RAND_11(sz) KMA_RAND_10(sz) .name[KMALLOC_RANDOM_START + 11] = "kmalloc-rnd-11-" #sz, 750 #define KMA_RAND_12(sz) KMA_RAND_11(sz) .name[KMALLOC_RANDOM_START + 12] = "kmalloc-rnd-12-" #sz, 751 #define KMA_RAND_13(sz) KMA_RAND_12(sz) .name[KMALLOC_RANDOM_START + 13] = "kmalloc-rnd-13-" #sz, 752 #define KMA_RAND_14(sz) KMA_RAND_13(sz) .name[KMALLOC_RANDOM_START + 14] = "kmalloc-rnd-14-" #sz, 753 #define KMA_RAND_15(sz) KMA_RAND_14(sz) .name[KMALLOC_RANDOM_START + 15] = "kmalloc-rnd-15-" #sz, 754 #else // CONFIG_RANDOM_KMALLOC_CACHES 755 #define KMALLOC_RANDOM_NAME(N, sz) 756 #endif 757 758 #define INIT_KMALLOC_INFO(__size, __short_size) \ 759 { \ 760 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \ 761 KMALLOC_RCL_NAME(__short_size) \ 762 KMALLOC_CGROUP_NAME(__short_size) \ 763 KMALLOC_DMA_NAME(__short_size) \ 764 KMALLOC_RANDOM_NAME(RANDOM_KMALLOC_CACHES_NR, __short_size) \ 765 .size = __size, \ 766 } 767 768 /* 769 * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. 770 * kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is 771 * kmalloc-2M. 772 */ 773 const struct kmalloc_info_struct kmalloc_info[] __initconst = { 774 INIT_KMALLOC_INFO(0, 0), 775 INIT_KMALLOC_INFO(96, 96), 776 INIT_KMALLOC_INFO(192, 192), 777 INIT_KMALLOC_INFO(8, 8), 778 INIT_KMALLOC_INFO(16, 16), 779 INIT_KMALLOC_INFO(32, 32), 780 INIT_KMALLOC_INFO(64, 64), 781 INIT_KMALLOC_INFO(128, 128), 782 INIT_KMALLOC_INFO(256, 256), 783 INIT_KMALLOC_INFO(512, 512), 784 INIT_KMALLOC_INFO(1024, 1k), 785 INIT_KMALLOC_INFO(2048, 2k), 786 INIT_KMALLOC_INFO(4096, 4k), 787 INIT_KMALLOC_INFO(8192, 8k), 788 INIT_KMALLOC_INFO(16384, 16k), 789 INIT_KMALLOC_INFO(32768, 32k), 790 INIT_KMALLOC_INFO(65536, 64k), 791 INIT_KMALLOC_INFO(131072, 128k), 792 INIT_KMALLOC_INFO(262144, 256k), 793 INIT_KMALLOC_INFO(524288, 512k), 794 INIT_KMALLOC_INFO(1048576, 1M), 795 INIT_KMALLOC_INFO(2097152, 2M) 796 }; 797 798 /* 799 * Patch up the size_index table if we have strange large alignment 800 * requirements for the kmalloc array. This is only the case for 801 * MIPS it seems. The standard arches will not generate any code here. 802 * 803 * Largest permitted alignment is 256 bytes due to the way we 804 * handle the index determination for the smaller caches. 805 * 806 * Make sure that nothing crazy happens if someone starts tinkering 807 * around with ARCH_KMALLOC_MINALIGN 808 */ 809 void __init setup_kmalloc_cache_index_table(void) 810 { 811 unsigned int i; 812 813 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 || 814 !is_power_of_2(KMALLOC_MIN_SIZE)); 815 816 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) { 817 unsigned int elem = size_index_elem(i); 818 819 if (elem >= ARRAY_SIZE(kmalloc_size_index)) 820 break; 821 kmalloc_size_index[elem] = KMALLOC_SHIFT_LOW; 822 } 823 824 if (KMALLOC_MIN_SIZE >= 64) { 825 /* 826 * The 96 byte sized cache is not used if the alignment 827 * is 64 byte. 828 */ 829 for (i = 64 + 8; i <= 96; i += 8) 830 kmalloc_size_index[size_index_elem(i)] = 7; 831 832 } 833 834 if (KMALLOC_MIN_SIZE >= 128) { 835 /* 836 * The 192 byte sized cache is not used if the alignment 837 * is 128 byte. Redirect kmalloc to use the 256 byte cache 838 * instead. 839 */ 840 for (i = 128 + 8; i <= 192; i += 8) 841 kmalloc_size_index[size_index_elem(i)] = 8; 842 } 843 } 844 845 static unsigned int __kmalloc_minalign(void) 846 { 847 unsigned int minalign = dma_get_cache_alignment(); 848 849 if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && 850 is_swiotlb_allocated()) 851 minalign = ARCH_KMALLOC_MINALIGN; 852 853 return max(minalign, arch_slab_minalign()); 854 } 855 856 void __init 857 new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) 858 { 859 unsigned int minalign = __kmalloc_minalign(); 860 unsigned int aligned_size = kmalloc_info[idx].size; 861 int aligned_idx = idx; 862 863 if ((KMALLOC_RECLAIM != KMALLOC_NORMAL) && (type == KMALLOC_RECLAIM)) { 864 flags |= SLAB_RECLAIM_ACCOUNT; 865 } else if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_CGROUP)) { 866 if (mem_cgroup_kmem_disabled()) { 867 kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx]; 868 return; 869 } 870 flags |= SLAB_ACCOUNT; 871 } else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) { 872 flags |= SLAB_CACHE_DMA; 873 } 874 875 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 876 if (type >= KMALLOC_RANDOM_START && type <= KMALLOC_RANDOM_END) 877 flags |= SLAB_NO_MERGE; 878 #endif 879 880 /* 881 * If CONFIG_MEMCG_KMEM is enabled, disable cache merging for 882 * KMALLOC_NORMAL caches. 883 */ 884 if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_NORMAL)) 885 flags |= SLAB_NO_MERGE; 886 887 if (minalign > ARCH_KMALLOC_MINALIGN) { 888 aligned_size = ALIGN(aligned_size, minalign); 889 aligned_idx = __kmalloc_index(aligned_size, false); 890 } 891 892 if (!kmalloc_caches[type][aligned_idx]) 893 kmalloc_caches[type][aligned_idx] = create_kmalloc_cache( 894 kmalloc_info[aligned_idx].name[type], 895 aligned_size, flags); 896 if (idx != aligned_idx) 897 kmalloc_caches[type][idx] = kmalloc_caches[type][aligned_idx]; 898 } 899 900 /* 901 * Create the kmalloc array. Some of the regular kmalloc arrays 902 * may already have been created because they were needed to 903 * enable allocations for slab creation. 904 */ 905 void __init create_kmalloc_caches(slab_flags_t flags) 906 { 907 int i; 908 enum kmalloc_cache_type type; 909 910 /* 911 * Including KMALLOC_CGROUP if CONFIG_MEMCG_KMEM defined 912 */ 913 for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) { 914 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 915 if (!kmalloc_caches[type][i]) 916 new_kmalloc_cache(i, type, flags); 917 918 /* 919 * Caches that are not of the two-to-the-power-of size. 920 * These have to be created immediately after the 921 * earlier power of two caches 922 */ 923 if (KMALLOC_MIN_SIZE <= 32 && i == 6 && 924 !kmalloc_caches[type][1]) 925 new_kmalloc_cache(1, type, flags); 926 if (KMALLOC_MIN_SIZE <= 64 && i == 7 && 927 !kmalloc_caches[type][2]) 928 new_kmalloc_cache(2, type, flags); 929 } 930 } 931 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 932 random_kmalloc_seed = get_random_u64(); 933 #endif 934 935 /* Kmalloc array is now usable */ 936 slab_state = UP; 937 } 938 939 /** 940 * __ksize -- Report full size of underlying allocation 941 * @object: pointer to the object 942 * 943 * This should only be used internally to query the true size of allocations. 944 * It is not meant to be a way to discover the usable size of an allocation 945 * after the fact. Instead, use kmalloc_size_roundup(). Using memory beyond 946 * the originally requested allocation size may trigger KASAN, UBSAN_BOUNDS, 947 * and/or FORTIFY_SOURCE. 948 * 949 * Return: size of the actual memory used by @object in bytes 950 */ 951 size_t __ksize(const void *object) 952 { 953 struct folio *folio; 954 955 if (unlikely(object == ZERO_SIZE_PTR)) 956 return 0; 957 958 folio = virt_to_folio(object); 959 960 if (unlikely(!folio_test_slab(folio))) { 961 if (WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE)) 962 return 0; 963 if (WARN_ON(object != folio_address(folio))) 964 return 0; 965 return folio_size(folio); 966 } 967 968 #ifdef CONFIG_SLUB_DEBUG 969 skip_orig_size_check(folio_slab(folio)->slab_cache, object); 970 #endif 971 972 return slab_ksize(folio_slab(folio)->slab_cache); 973 } 974 975 gfp_t kmalloc_fix_flags(gfp_t flags) 976 { 977 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK; 978 979 flags &= ~GFP_SLAB_BUG_MASK; 980 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n", 981 invalid_mask, &invalid_mask, flags, &flags); 982 dump_stack(); 983 984 return flags; 985 } 986 987 #ifdef CONFIG_SLAB_FREELIST_RANDOM 988 /* Randomize a generic freelist */ 989 static void freelist_randomize(unsigned int *list, 990 unsigned int count) 991 { 992 unsigned int rand; 993 unsigned int i; 994 995 for (i = 0; i < count; i++) 996 list[i] = i; 997 998 /* Fisher-Yates shuffle */ 999 for (i = count - 1; i > 0; i--) { 1000 rand = get_random_u32_below(i + 1); 1001 swap(list[i], list[rand]); 1002 } 1003 } 1004 1005 /* Create a random sequence per cache */ 1006 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 1007 gfp_t gfp) 1008 { 1009 1010 if (count < 2 || cachep->random_seq) 1011 return 0; 1012 1013 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp); 1014 if (!cachep->random_seq) 1015 return -ENOMEM; 1016 1017 freelist_randomize(cachep->random_seq, count); 1018 return 0; 1019 } 1020 1021 /* Destroy the per-cache random freelist sequence */ 1022 void cache_random_seq_destroy(struct kmem_cache *cachep) 1023 { 1024 kfree(cachep->random_seq); 1025 cachep->random_seq = NULL; 1026 } 1027 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 1028 1029 #ifdef CONFIG_SLUB_DEBUG 1030 #define SLABINFO_RIGHTS (0400) 1031 1032 static void print_slabinfo_header(struct seq_file *m) 1033 { 1034 /* 1035 * Output format version, so at least we can change it 1036 * without _too_ many complaints. 1037 */ 1038 seq_puts(m, "slabinfo - version: 2.1\n"); 1039 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>"); 1040 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 1041 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 1042 seq_putc(m, '\n'); 1043 } 1044 1045 static void *slab_start(struct seq_file *m, loff_t *pos) 1046 { 1047 mutex_lock(&slab_mutex); 1048 return seq_list_start(&slab_caches, *pos); 1049 } 1050 1051 static void *slab_next(struct seq_file *m, void *p, loff_t *pos) 1052 { 1053 return seq_list_next(p, &slab_caches, pos); 1054 } 1055 1056 static void slab_stop(struct seq_file *m, void *p) 1057 { 1058 mutex_unlock(&slab_mutex); 1059 } 1060 1061 static void cache_show(struct kmem_cache *s, struct seq_file *m) 1062 { 1063 struct slabinfo sinfo; 1064 1065 memset(&sinfo, 0, sizeof(sinfo)); 1066 get_slabinfo(s, &sinfo); 1067 1068 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 1069 s->name, sinfo.active_objs, sinfo.num_objs, s->size, 1070 sinfo.objects_per_slab, (1 << sinfo.cache_order)); 1071 1072 seq_printf(m, " : tunables %4u %4u %4u", 1073 sinfo.limit, sinfo.batchcount, sinfo.shared); 1074 seq_printf(m, " : slabdata %6lu %6lu %6lu", 1075 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail); 1076 slabinfo_show_stats(m, s); 1077 seq_putc(m, '\n'); 1078 } 1079 1080 static int slab_show(struct seq_file *m, void *p) 1081 { 1082 struct kmem_cache *s = list_entry(p, struct kmem_cache, list); 1083 1084 if (p == slab_caches.next) 1085 print_slabinfo_header(m); 1086 cache_show(s, m); 1087 return 0; 1088 } 1089 1090 void dump_unreclaimable_slab(void) 1091 { 1092 struct kmem_cache *s; 1093 struct slabinfo sinfo; 1094 1095 /* 1096 * Here acquiring slab_mutex is risky since we don't prefer to get 1097 * sleep in oom path. But, without mutex hold, it may introduce a 1098 * risk of crash. 1099 * Use mutex_trylock to protect the list traverse, dump nothing 1100 * without acquiring the mutex. 1101 */ 1102 if (!mutex_trylock(&slab_mutex)) { 1103 pr_warn("excessive unreclaimable slab but cannot dump stats\n"); 1104 return; 1105 } 1106 1107 pr_info("Unreclaimable slab info:\n"); 1108 pr_info("Name Used Total\n"); 1109 1110 list_for_each_entry(s, &slab_caches, list) { 1111 if (s->flags & SLAB_RECLAIM_ACCOUNT) 1112 continue; 1113 1114 get_slabinfo(s, &sinfo); 1115 1116 if (sinfo.num_objs > 0) 1117 pr_info("%-17s %10luKB %10luKB\n", s->name, 1118 (sinfo.active_objs * s->size) / 1024, 1119 (sinfo.num_objs * s->size) / 1024); 1120 } 1121 mutex_unlock(&slab_mutex); 1122 } 1123 1124 /* 1125 * slabinfo_op - iterator that generates /proc/slabinfo 1126 * 1127 * Output layout: 1128 * cache-name 1129 * num-active-objs 1130 * total-objs 1131 * object size 1132 * num-active-slabs 1133 * total-slabs 1134 * num-pages-per-slab 1135 * + further values on SMP and with statistics enabled 1136 */ 1137 static const struct seq_operations slabinfo_op = { 1138 .start = slab_start, 1139 .next = slab_next, 1140 .stop = slab_stop, 1141 .show = slab_show, 1142 }; 1143 1144 static int slabinfo_open(struct inode *inode, struct file *file) 1145 { 1146 return seq_open(file, &slabinfo_op); 1147 } 1148 1149 static const struct proc_ops slabinfo_proc_ops = { 1150 .proc_flags = PROC_ENTRY_PERMANENT, 1151 .proc_open = slabinfo_open, 1152 .proc_read = seq_read, 1153 .proc_write = slabinfo_write, 1154 .proc_lseek = seq_lseek, 1155 .proc_release = seq_release, 1156 }; 1157 1158 static int __init slab_proc_init(void) 1159 { 1160 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, &slabinfo_proc_ops); 1161 return 0; 1162 } 1163 module_init(slab_proc_init); 1164 1165 #endif /* CONFIG_SLUB_DEBUG */ 1166 1167 static __always_inline __realloc_size(2) void * 1168 __do_krealloc(const void *p, size_t new_size, gfp_t flags) 1169 { 1170 void *ret; 1171 size_t ks; 1172 1173 /* Check for double-free before calling ksize. */ 1174 if (likely(!ZERO_OR_NULL_PTR(p))) { 1175 if (!kasan_check_byte(p)) 1176 return NULL; 1177 ks = ksize(p); 1178 } else 1179 ks = 0; 1180 1181 /* If the object still fits, repoison it precisely. */ 1182 if (ks >= new_size) { 1183 p = kasan_krealloc((void *)p, new_size, flags); 1184 return (void *)p; 1185 } 1186 1187 ret = kmalloc_track_caller(new_size, flags); 1188 if (ret && p) { 1189 /* Disable KASAN checks as the object's redzone is accessed. */ 1190 kasan_disable_current(); 1191 memcpy(ret, kasan_reset_tag(p), ks); 1192 kasan_enable_current(); 1193 } 1194 1195 return ret; 1196 } 1197 1198 /** 1199 * krealloc - reallocate memory. The contents will remain unchanged. 1200 * @p: object to reallocate memory for. 1201 * @new_size: how many bytes of memory are required. 1202 * @flags: the type of memory to allocate. 1203 * 1204 * The contents of the object pointed to are preserved up to the 1205 * lesser of the new and old sizes (__GFP_ZERO flag is effectively ignored). 1206 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 1207 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 1208 * 1209 * Return: pointer to the allocated memory or %NULL in case of error 1210 */ 1211 void *krealloc(const void *p, size_t new_size, gfp_t flags) 1212 { 1213 void *ret; 1214 1215 if (unlikely(!new_size)) { 1216 kfree(p); 1217 return ZERO_SIZE_PTR; 1218 } 1219 1220 ret = __do_krealloc(p, new_size, flags); 1221 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 1222 kfree(p); 1223 1224 return ret; 1225 } 1226 EXPORT_SYMBOL(krealloc); 1227 1228 /** 1229 * kfree_sensitive - Clear sensitive information in memory before freeing 1230 * @p: object to free memory of 1231 * 1232 * The memory of the object @p points to is zeroed before freed. 1233 * If @p is %NULL, kfree_sensitive() does nothing. 1234 * 1235 * Note: this function zeroes the whole allocated buffer which can be a good 1236 * deal bigger than the requested buffer size passed to kmalloc(). So be 1237 * careful when using this function in performance sensitive code. 1238 */ 1239 void kfree_sensitive(const void *p) 1240 { 1241 size_t ks; 1242 void *mem = (void *)p; 1243 1244 ks = ksize(mem); 1245 if (ks) { 1246 kasan_unpoison_range(mem, ks); 1247 memzero_explicit(mem, ks); 1248 } 1249 kfree(mem); 1250 } 1251 EXPORT_SYMBOL(kfree_sensitive); 1252 1253 size_t ksize(const void *objp) 1254 { 1255 /* 1256 * We need to first check that the pointer to the object is valid. 1257 * The KASAN report printed from ksize() is more useful, then when 1258 * it's printed later when the behaviour could be undefined due to 1259 * a potential use-after-free or double-free. 1260 * 1261 * We use kasan_check_byte(), which is supported for the hardware 1262 * tag-based KASAN mode, unlike kasan_check_read/write(). 1263 * 1264 * If the pointed to memory is invalid, we return 0 to avoid users of 1265 * ksize() writing to and potentially corrupting the memory region. 1266 * 1267 * We want to perform the check before __ksize(), to avoid potentially 1268 * crashing in __ksize() due to accessing invalid metadata. 1269 */ 1270 if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp)) 1271 return 0; 1272 1273 return kfence_ksize(objp) ?: __ksize(objp); 1274 } 1275 EXPORT_SYMBOL(ksize); 1276 1277 /* Tracepoints definitions. */ 1278 EXPORT_TRACEPOINT_SYMBOL(kmalloc); 1279 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); 1280 EXPORT_TRACEPOINT_SYMBOL(kfree); 1281 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 1282 1283