1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * SLUB: A slab allocator that limits cache line use instead of queuing 4 * objects in per cpu and per node lists. 5 * 6 * The allocator synchronizes using per slab locks or atomic operations 7 * and only uses a centralized lock to manage a pool of partial slabs. 8 * 9 * (C) 2007 SGI, Christoph Lameter 10 * (C) 2011 Linux Foundation, Christoph Lameter 11 */ 12 13 #include <linux/mm.h> 14 #include <linux/swap.h> /* mm_account_reclaimed_pages() */ 15 #include <linux/module.h> 16 #include <linux/bit_spinlock.h> 17 #include <linux/interrupt.h> 18 #include <linux/swab.h> 19 #include <linux/bitops.h> 20 #include <linux/slab.h> 21 #include "slab.h" 22 #include <linux/proc_fs.h> 23 #include <linux/seq_file.h> 24 #include <linux/kasan.h> 25 #include <linux/kmsan.h> 26 #include <linux/cpu.h> 27 #include <linux/cpuset.h> 28 #include <linux/mempolicy.h> 29 #include <linux/ctype.h> 30 #include <linux/stackdepot.h> 31 #include <linux/debugobjects.h> 32 #include <linux/kallsyms.h> 33 #include <linux/kfence.h> 34 #include <linux/memory.h> 35 #include <linux/math64.h> 36 #include <linux/fault-inject.h> 37 #include <linux/kmemleak.h> 38 #include <linux/stacktrace.h> 39 #include <linux/prefetch.h> 40 #include <linux/memcontrol.h> 41 #include <linux/random.h> 42 #include <kunit/test.h> 43 #include <kunit/test-bug.h> 44 #include <linux/sort.h> 45 46 #include <linux/debugfs.h> 47 #include <trace/events/kmem.h> 48 49 #include "internal.h" 50 51 /* 52 * Lock order: 53 * 1. slab_mutex (Global Mutex) 54 * 2. node->list_lock (Spinlock) 55 * 3. kmem_cache->cpu_slab->lock (Local lock) 56 * 4. slab_lock(slab) (Only on some arches) 57 * 5. object_map_lock (Only for debugging) 58 * 59 * slab_mutex 60 * 61 * The role of the slab_mutex is to protect the list of all the slabs 62 * and to synchronize major metadata changes to slab cache structures. 63 * Also synchronizes memory hotplug callbacks. 64 * 65 * slab_lock 66 * 67 * The slab_lock is a wrapper around the page lock, thus it is a bit 68 * spinlock. 69 * 70 * The slab_lock is only used on arches that do not have the ability 71 * to do a cmpxchg_double. It only protects: 72 * 73 * A. slab->freelist -> List of free objects in a slab 74 * B. slab->inuse -> Number of objects in use 75 * C. slab->objects -> Number of objects in slab 76 * D. slab->frozen -> frozen state 77 * 78 * Frozen slabs 79 * 80 * If a slab is frozen then it is exempt from list management. It is 81 * the cpu slab which is actively allocated from by the processor that 82 * froze it and it is not on any list. The processor that froze the 83 * slab is the one who can perform list operations on the slab. Other 84 * processors may put objects onto the freelist but the processor that 85 * froze the slab is the only one that can retrieve the objects from the 86 * slab's freelist. 87 * 88 * CPU partial slabs 89 * 90 * The partially empty slabs cached on the CPU partial list are used 91 * for performance reasons, which speeds up the allocation process. 92 * These slabs are not frozen, but are also exempt from list management, 93 * by clearing the PG_workingset flag when moving out of the node 94 * partial list. Please see __slab_free() for more details. 95 * 96 * To sum up, the current scheme is: 97 * - node partial slab: PG_Workingset && !frozen 98 * - cpu partial slab: !PG_Workingset && !frozen 99 * - cpu slab: !PG_Workingset && frozen 100 * - full slab: !PG_Workingset && !frozen 101 * 102 * list_lock 103 * 104 * The list_lock protects the partial and full list on each node and 105 * the partial slab counter. If taken then no new slabs may be added or 106 * removed from the lists nor make the number of partial slabs be modified. 107 * (Note that the total number of slabs is an atomic value that may be 108 * modified without taking the list lock). 109 * 110 * The list_lock is a centralized lock and thus we avoid taking it as 111 * much as possible. As long as SLUB does not have to handle partial 112 * slabs, operations can continue without any centralized lock. F.e. 113 * allocating a long series of objects that fill up slabs does not require 114 * the list lock. 115 * 116 * For debug caches, all allocations are forced to go through a list_lock 117 * protected region to serialize against concurrent validation. 118 * 119 * cpu_slab->lock local lock 120 * 121 * This locks protect slowpath manipulation of all kmem_cache_cpu fields 122 * except the stat counters. This is a percpu structure manipulated only by 123 * the local cpu, so the lock protects against being preempted or interrupted 124 * by an irq. Fast path operations rely on lockless operations instead. 125 * 126 * On PREEMPT_RT, the local lock neither disables interrupts nor preemption 127 * which means the lockless fastpath cannot be used as it might interfere with 128 * an in-progress slow path operations. In this case the local lock is always 129 * taken but it still utilizes the freelist for the common operations. 130 * 131 * lockless fastpaths 132 * 133 * The fast path allocation (slab_alloc_node()) and freeing (do_slab_free()) 134 * are fully lockless when satisfied from the percpu slab (and when 135 * cmpxchg_double is possible to use, otherwise slab_lock is taken). 136 * They also don't disable preemption or migration or irqs. They rely on 137 * the transaction id (tid) field to detect being preempted or moved to 138 * another cpu. 139 * 140 * irq, preemption, migration considerations 141 * 142 * Interrupts are disabled as part of list_lock or local_lock operations, or 143 * around the slab_lock operation, in order to make the slab allocator safe 144 * to use in the context of an irq. 145 * 146 * In addition, preemption (or migration on PREEMPT_RT) is disabled in the 147 * allocation slowpath, bulk allocation, and put_cpu_partial(), so that the 148 * local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer 149 * doesn't have to be revalidated in each section protected by the local lock. 150 * 151 * SLUB assigns one slab for allocation to each processor. 152 * Allocations only occur from these slabs called cpu slabs. 153 * 154 * Slabs with free elements are kept on a partial list and during regular 155 * operations no list for full slabs is used. If an object in a full slab is 156 * freed then the slab will show up again on the partial lists. 157 * We track full slabs for debugging purposes though because otherwise we 158 * cannot scan all objects. 159 * 160 * Slabs are freed when they become empty. Teardown and setup is 161 * minimal so we rely on the page allocators per cpu caches for 162 * fast frees and allocs. 163 * 164 * slab->frozen The slab is frozen and exempt from list processing. 165 * This means that the slab is dedicated to a purpose 166 * such as satisfying allocations for a specific 167 * processor. Objects may be freed in the slab while 168 * it is frozen but slab_free will then skip the usual 169 * list operations. It is up to the processor holding 170 * the slab to integrate the slab into the slab lists 171 * when the slab is no longer needed. 172 * 173 * One use of this flag is to mark slabs that are 174 * used for allocations. Then such a slab becomes a cpu 175 * slab. The cpu slab may be equipped with an additional 176 * freelist that allows lockless access to 177 * free objects in addition to the regular freelist 178 * that requires the slab lock. 179 * 180 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug 181 * options set. This moves slab handling out of 182 * the fast path and disables lockless freelists. 183 */ 184 185 /* 186 * We could simply use migrate_disable()/enable() but as long as it's a 187 * function call even on !PREEMPT_RT, use inline preempt_disable() there. 188 */ 189 #ifndef CONFIG_PREEMPT_RT 190 #define slub_get_cpu_ptr(var) get_cpu_ptr(var) 191 #define slub_put_cpu_ptr(var) put_cpu_ptr(var) 192 #define USE_LOCKLESS_FAST_PATH() (true) 193 #else 194 #define slub_get_cpu_ptr(var) \ 195 ({ \ 196 migrate_disable(); \ 197 this_cpu_ptr(var); \ 198 }) 199 #define slub_put_cpu_ptr(var) \ 200 do { \ 201 (void)(var); \ 202 migrate_enable(); \ 203 } while (0) 204 #define USE_LOCKLESS_FAST_PATH() (false) 205 #endif 206 207 #ifndef CONFIG_SLUB_TINY 208 #define __fastpath_inline __always_inline 209 #else 210 #define __fastpath_inline 211 #endif 212 213 #ifdef CONFIG_SLUB_DEBUG 214 #ifdef CONFIG_SLUB_DEBUG_ON 215 DEFINE_STATIC_KEY_TRUE(slub_debug_enabled); 216 #else 217 DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); 218 #endif 219 #endif /* CONFIG_SLUB_DEBUG */ 220 221 /* Structure holding parameters for get_partial() call chain */ 222 struct partial_context { 223 gfp_t flags; 224 unsigned int orig_size; 225 void *object; 226 }; 227 228 static inline bool kmem_cache_debug(struct kmem_cache *s) 229 { 230 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS); 231 } 232 233 static inline bool slub_debug_orig_size(struct kmem_cache *s) 234 { 235 return (kmem_cache_debug_flags(s, SLAB_STORE_USER) && 236 (s->flags & SLAB_KMALLOC)); 237 } 238 239 void *fixup_red_left(struct kmem_cache *s, void *p) 240 { 241 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE)) 242 p += s->red_left_pad; 243 244 return p; 245 } 246 247 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) 248 { 249 #ifdef CONFIG_SLUB_CPU_PARTIAL 250 return !kmem_cache_debug(s); 251 #else 252 return false; 253 #endif 254 } 255 256 /* 257 * Issues still to be resolved: 258 * 259 * - Support PAGE_ALLOC_DEBUG. Should be easy to do. 260 * 261 * - Variable sizing of the per node arrays 262 */ 263 264 /* Enable to log cmpxchg failures */ 265 #undef SLUB_DEBUG_CMPXCHG 266 267 #ifndef CONFIG_SLUB_TINY 268 /* 269 * Minimum number of partial slabs. These will be left on the partial 270 * lists even if they are empty. kmem_cache_shrink may reclaim them. 271 */ 272 #define MIN_PARTIAL 5 273 274 /* 275 * Maximum number of desirable partial slabs. 276 * The existence of more partial slabs makes kmem_cache_shrink 277 * sort the partial list by the number of objects in use. 278 */ 279 #define MAX_PARTIAL 10 280 #else 281 #define MIN_PARTIAL 0 282 #define MAX_PARTIAL 0 283 #endif 284 285 #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \ 286 SLAB_POISON | SLAB_STORE_USER) 287 288 /* 289 * These debug flags cannot use CMPXCHG because there might be consistency 290 * issues when checking or reading debug information 291 */ 292 #define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \ 293 SLAB_TRACE) 294 295 296 /* 297 * Debugging flags that require metadata to be stored in the slab. These get 298 * disabled when slab_debug=O is used and a cache's min order increases with 299 * metadata. 300 */ 301 #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 302 303 #define OO_SHIFT 16 304 #define OO_MASK ((1 << OO_SHIFT) - 1) 305 #define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */ 306 307 /* Internal SLUB flags */ 308 /* Poison object */ 309 #define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON) 310 /* Use cmpxchg_double */ 311 312 #ifdef system_has_freelist_aba 313 #define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE) 314 #else 315 #define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED 316 #endif 317 318 /* 319 * Tracking user of a slab. 320 */ 321 #define TRACK_ADDRS_COUNT 16 322 struct track { 323 unsigned long addr; /* Called from address */ 324 #ifdef CONFIG_STACKDEPOT 325 depot_stack_handle_t handle; 326 #endif 327 int cpu; /* Was running on cpu */ 328 int pid; /* Pid context */ 329 unsigned long when; /* When did the operation occur */ 330 }; 331 332 enum track_item { TRACK_ALLOC, TRACK_FREE }; 333 334 #ifdef SLAB_SUPPORTS_SYSFS 335 static int sysfs_slab_add(struct kmem_cache *); 336 static int sysfs_slab_alias(struct kmem_cache *, const char *); 337 #else 338 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } 339 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) 340 { return 0; } 341 #endif 342 343 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 344 static void debugfs_slab_add(struct kmem_cache *); 345 #else 346 static inline void debugfs_slab_add(struct kmem_cache *s) { } 347 #endif 348 349 enum stat_item { 350 ALLOC_FASTPATH, /* Allocation from cpu slab */ 351 ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */ 352 FREE_FASTPATH, /* Free to cpu slab */ 353 FREE_SLOWPATH, /* Freeing not to cpu slab */ 354 FREE_FROZEN, /* Freeing to frozen slab */ 355 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */ 356 FREE_REMOVE_PARTIAL, /* Freeing removes last object */ 357 ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */ 358 ALLOC_SLAB, /* Cpu slab acquired from page allocator */ 359 ALLOC_REFILL, /* Refill cpu slab from slab freelist */ 360 ALLOC_NODE_MISMATCH, /* Switching cpu slab */ 361 FREE_SLAB, /* Slab freed to the page allocator */ 362 CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ 363 DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ 364 DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */ 365 DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */ 366 DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */ 367 DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ 368 DEACTIVATE_BYPASS, /* Implicit deactivation */ 369 ORDER_FALLBACK, /* Number of times fallback was necessary */ 370 CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */ 371 CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */ 372 CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */ 373 CPU_PARTIAL_FREE, /* Refill cpu partial on free */ 374 CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */ 375 CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */ 376 NR_SLUB_STAT_ITEMS 377 }; 378 379 #ifndef CONFIG_SLUB_TINY 380 /* 381 * When changing the layout, make sure freelist and tid are still compatible 382 * with this_cpu_cmpxchg_double() alignment requirements. 383 */ 384 struct kmem_cache_cpu { 385 union { 386 struct { 387 void **freelist; /* Pointer to next available object */ 388 unsigned long tid; /* Globally unique transaction id */ 389 }; 390 freelist_aba_t freelist_tid; 391 }; 392 struct slab *slab; /* The slab from which we are allocating */ 393 #ifdef CONFIG_SLUB_CPU_PARTIAL 394 struct slab *partial; /* Partially allocated slabs */ 395 #endif 396 local_lock_t lock; /* Protects the fields above */ 397 #ifdef CONFIG_SLUB_STATS 398 unsigned int stat[NR_SLUB_STAT_ITEMS]; 399 #endif 400 }; 401 #endif /* CONFIG_SLUB_TINY */ 402 403 static inline void stat(const struct kmem_cache *s, enum stat_item si) 404 { 405 #ifdef CONFIG_SLUB_STATS 406 /* 407 * The rmw is racy on a preemptible kernel but this is acceptable, so 408 * avoid this_cpu_add()'s irq-disable overhead. 409 */ 410 raw_cpu_inc(s->cpu_slab->stat[si]); 411 #endif 412 } 413 414 static inline 415 void stat_add(const struct kmem_cache *s, enum stat_item si, int v) 416 { 417 #ifdef CONFIG_SLUB_STATS 418 raw_cpu_add(s->cpu_slab->stat[si], v); 419 #endif 420 } 421 422 /* 423 * The slab lists for all objects. 424 */ 425 struct kmem_cache_node { 426 spinlock_t list_lock; 427 unsigned long nr_partial; 428 struct list_head partial; 429 #ifdef CONFIG_SLUB_DEBUG 430 atomic_long_t nr_slabs; 431 atomic_long_t total_objects; 432 struct list_head full; 433 #endif 434 }; 435 436 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 437 { 438 return s->node[node]; 439 } 440 441 /* 442 * Iterator over all nodes. The body will be executed for each node that has 443 * a kmem_cache_node structure allocated (which is true for all online nodes) 444 */ 445 #define for_each_kmem_cache_node(__s, __node, __n) \ 446 for (__node = 0; __node < nr_node_ids; __node++) \ 447 if ((__n = get_node(__s, __node))) 448 449 /* 450 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated. 451 * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily 452 * differ during memory hotplug/hotremove operations. 453 * Protected by slab_mutex. 454 */ 455 static nodemask_t slab_nodes; 456 457 #ifndef CONFIG_SLUB_TINY 458 /* 459 * Workqueue used for flush_cpu_slab(). 460 */ 461 static struct workqueue_struct *flushwq; 462 #endif 463 464 /******************************************************************** 465 * Core slab cache functions 466 *******************************************************************/ 467 468 /* 469 * freeptr_t represents a SLUB freelist pointer, which might be encoded 470 * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled. 471 */ 472 typedef struct { unsigned long v; } freeptr_t; 473 474 /* 475 * Returns freelist pointer (ptr). With hardening, this is obfuscated 476 * with an XOR of the address where the pointer is held and a per-cache 477 * random number. 478 */ 479 static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s, 480 void *ptr, unsigned long ptr_addr) 481 { 482 unsigned long encoded; 483 484 #ifdef CONFIG_SLAB_FREELIST_HARDENED 485 encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr); 486 #else 487 encoded = (unsigned long)ptr; 488 #endif 489 return (freeptr_t){.v = encoded}; 490 } 491 492 static inline void *freelist_ptr_decode(const struct kmem_cache *s, 493 freeptr_t ptr, unsigned long ptr_addr) 494 { 495 void *decoded; 496 497 #ifdef CONFIG_SLAB_FREELIST_HARDENED 498 decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr)); 499 #else 500 decoded = (void *)ptr.v; 501 #endif 502 return decoded; 503 } 504 505 static inline void *get_freepointer(struct kmem_cache *s, void *object) 506 { 507 unsigned long ptr_addr; 508 freeptr_t p; 509 510 object = kasan_reset_tag(object); 511 ptr_addr = (unsigned long)object + s->offset; 512 p = *(freeptr_t *)(ptr_addr); 513 return freelist_ptr_decode(s, p, ptr_addr); 514 } 515 516 #ifndef CONFIG_SLUB_TINY 517 static void prefetch_freepointer(const struct kmem_cache *s, void *object) 518 { 519 prefetchw(object + s->offset); 520 } 521 #endif 522 523 /* 524 * When running under KMSAN, get_freepointer_safe() may return an uninitialized 525 * pointer value in the case the current thread loses the race for the next 526 * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in 527 * slab_alloc_node() will fail, so the uninitialized value won't be used, but 528 * KMSAN will still check all arguments of cmpxchg because of imperfect 529 * handling of inline assembly. 530 * To work around this problem, we apply __no_kmsan_checks to ensure that 531 * get_freepointer_safe() returns initialized memory. 532 */ 533 __no_kmsan_checks 534 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) 535 { 536 unsigned long freepointer_addr; 537 freeptr_t p; 538 539 if (!debug_pagealloc_enabled_static()) 540 return get_freepointer(s, object); 541 542 object = kasan_reset_tag(object); 543 freepointer_addr = (unsigned long)object + s->offset; 544 copy_from_kernel_nofault(&p, (freeptr_t *)freepointer_addr, sizeof(p)); 545 return freelist_ptr_decode(s, p, freepointer_addr); 546 } 547 548 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 549 { 550 unsigned long freeptr_addr = (unsigned long)object + s->offset; 551 552 #ifdef CONFIG_SLAB_FREELIST_HARDENED 553 BUG_ON(object == fp); /* naive detection of double free or corruption */ 554 #endif 555 556 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); 557 *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); 558 } 559 560 /* 561 * See comment in calculate_sizes(). 562 */ 563 static inline bool freeptr_outside_object(struct kmem_cache *s) 564 { 565 return s->offset >= s->inuse; 566 } 567 568 /* 569 * Return offset of the end of info block which is inuse + free pointer if 570 * not overlapping with object. 571 */ 572 static inline unsigned int get_info_end(struct kmem_cache *s) 573 { 574 if (freeptr_outside_object(s)) 575 return s->inuse + sizeof(void *); 576 else 577 return s->inuse; 578 } 579 580 /* Loop over all objects in a slab */ 581 #define for_each_object(__p, __s, __addr, __objects) \ 582 for (__p = fixup_red_left(__s, __addr); \ 583 __p < (__addr) + (__objects) * (__s)->size; \ 584 __p += (__s)->size) 585 586 static inline unsigned int order_objects(unsigned int order, unsigned int size) 587 { 588 return ((unsigned int)PAGE_SIZE << order) / size; 589 } 590 591 static inline struct kmem_cache_order_objects oo_make(unsigned int order, 592 unsigned int size) 593 { 594 struct kmem_cache_order_objects x = { 595 (order << OO_SHIFT) + order_objects(order, size) 596 }; 597 598 return x; 599 } 600 601 static inline unsigned int oo_order(struct kmem_cache_order_objects x) 602 { 603 return x.x >> OO_SHIFT; 604 } 605 606 static inline unsigned int oo_objects(struct kmem_cache_order_objects x) 607 { 608 return x.x & OO_MASK; 609 } 610 611 #ifdef CONFIG_SLUB_CPU_PARTIAL 612 static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 613 { 614 unsigned int nr_slabs; 615 616 s->cpu_partial = nr_objects; 617 618 /* 619 * We take the number of objects but actually limit the number of 620 * slabs on the per cpu partial list, in order to limit excessive 621 * growth of the list. For simplicity we assume that the slabs will 622 * be half-full. 623 */ 624 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); 625 s->cpu_partial_slabs = nr_slabs; 626 } 627 628 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 629 { 630 return s->cpu_partial_slabs; 631 } 632 #else 633 static inline void 634 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 635 { 636 } 637 638 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 639 { 640 return 0; 641 } 642 #endif /* CONFIG_SLUB_CPU_PARTIAL */ 643 644 /* 645 * Per slab locking using the pagelock 646 */ 647 static __always_inline void slab_lock(struct slab *slab) 648 { 649 bit_spin_lock(PG_locked, &slab->__page_flags); 650 } 651 652 static __always_inline void slab_unlock(struct slab *slab) 653 { 654 bit_spin_unlock(PG_locked, &slab->__page_flags); 655 } 656 657 static inline bool 658 __update_freelist_fast(struct slab *slab, 659 void *freelist_old, unsigned long counters_old, 660 void *freelist_new, unsigned long counters_new) 661 { 662 #ifdef system_has_freelist_aba 663 freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old }; 664 freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new }; 665 666 return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full); 667 #else 668 return false; 669 #endif 670 } 671 672 static inline bool 673 __update_freelist_slow(struct slab *slab, 674 void *freelist_old, unsigned long counters_old, 675 void *freelist_new, unsigned long counters_new) 676 { 677 bool ret = false; 678 679 slab_lock(slab); 680 if (slab->freelist == freelist_old && 681 slab->counters == counters_old) { 682 slab->freelist = freelist_new; 683 slab->counters = counters_new; 684 ret = true; 685 } 686 slab_unlock(slab); 687 688 return ret; 689 } 690 691 /* 692 * Interrupts must be disabled (for the fallback code to work right), typically 693 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is 694 * part of bit_spin_lock(), is sufficient because the policy is not to allow any 695 * allocation/ free operation in hardirq context. Therefore nothing can 696 * interrupt the operation. 697 */ 698 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab, 699 void *freelist_old, unsigned long counters_old, 700 void *freelist_new, unsigned long counters_new, 701 const char *n) 702 { 703 bool ret; 704 705 if (USE_LOCKLESS_FAST_PATH()) 706 lockdep_assert_irqs_disabled(); 707 708 if (s->flags & __CMPXCHG_DOUBLE) { 709 ret = __update_freelist_fast(slab, freelist_old, counters_old, 710 freelist_new, counters_new); 711 } else { 712 ret = __update_freelist_slow(slab, freelist_old, counters_old, 713 freelist_new, counters_new); 714 } 715 if (likely(ret)) 716 return true; 717 718 cpu_relax(); 719 stat(s, CMPXCHG_DOUBLE_FAIL); 720 721 #ifdef SLUB_DEBUG_CMPXCHG 722 pr_info("%s %s: cmpxchg double redo ", n, s->name); 723 #endif 724 725 return false; 726 } 727 728 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, 729 void *freelist_old, unsigned long counters_old, 730 void *freelist_new, unsigned long counters_new, 731 const char *n) 732 { 733 bool ret; 734 735 if (s->flags & __CMPXCHG_DOUBLE) { 736 ret = __update_freelist_fast(slab, freelist_old, counters_old, 737 freelist_new, counters_new); 738 } else { 739 unsigned long flags; 740 741 local_irq_save(flags); 742 ret = __update_freelist_slow(slab, freelist_old, counters_old, 743 freelist_new, counters_new); 744 local_irq_restore(flags); 745 } 746 if (likely(ret)) 747 return true; 748 749 cpu_relax(); 750 stat(s, CMPXCHG_DOUBLE_FAIL); 751 752 #ifdef SLUB_DEBUG_CMPXCHG 753 pr_info("%s %s: cmpxchg double redo ", n, s->name); 754 #endif 755 756 return false; 757 } 758 759 #ifdef CONFIG_SLUB_DEBUG 760 static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; 761 static DEFINE_SPINLOCK(object_map_lock); 762 763 static void __fill_map(unsigned long *obj_map, struct kmem_cache *s, 764 struct slab *slab) 765 { 766 void *addr = slab_address(slab); 767 void *p; 768 769 bitmap_zero(obj_map, slab->objects); 770 771 for (p = slab->freelist; p; p = get_freepointer(s, p)) 772 set_bit(__obj_to_index(s, addr, p), obj_map); 773 } 774 775 #if IS_ENABLED(CONFIG_KUNIT) 776 static bool slab_add_kunit_errors(void) 777 { 778 struct kunit_resource *resource; 779 780 if (!kunit_get_current_test()) 781 return false; 782 783 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); 784 if (!resource) 785 return false; 786 787 (*(int *)resource->data)++; 788 kunit_put_resource(resource); 789 return true; 790 } 791 792 static bool slab_in_kunit_test(void) 793 { 794 struct kunit_resource *resource; 795 796 if (!kunit_get_current_test()) 797 return false; 798 799 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); 800 if (!resource) 801 return false; 802 803 kunit_put_resource(resource); 804 return true; 805 } 806 #else 807 static inline bool slab_add_kunit_errors(void) { return false; } 808 static inline bool slab_in_kunit_test(void) { return false; } 809 #endif 810 811 static inline unsigned int size_from_object(struct kmem_cache *s) 812 { 813 if (s->flags & SLAB_RED_ZONE) 814 return s->size - s->red_left_pad; 815 816 return s->size; 817 } 818 819 static inline void *restore_red_left(struct kmem_cache *s, void *p) 820 { 821 if (s->flags & SLAB_RED_ZONE) 822 p -= s->red_left_pad; 823 824 return p; 825 } 826 827 /* 828 * Debug settings: 829 */ 830 #if defined(CONFIG_SLUB_DEBUG_ON) 831 static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS; 832 #else 833 static slab_flags_t slub_debug; 834 #endif 835 836 static char *slub_debug_string; 837 static int disable_higher_order_debug; 838 839 /* 840 * slub is about to manipulate internal object metadata. This memory lies 841 * outside the range of the allocated object, so accessing it would normally 842 * be reported by kasan as a bounds error. metadata_access_enable() is used 843 * to tell kasan that these accesses are OK. 844 */ 845 static inline void metadata_access_enable(void) 846 { 847 kasan_disable_current(); 848 kmsan_disable_current(); 849 } 850 851 static inline void metadata_access_disable(void) 852 { 853 kmsan_enable_current(); 854 kasan_enable_current(); 855 } 856 857 /* 858 * Object debugging 859 */ 860 861 /* Verify that a pointer has an address that is valid within a slab page */ 862 static inline int check_valid_pointer(struct kmem_cache *s, 863 struct slab *slab, void *object) 864 { 865 void *base; 866 867 if (!object) 868 return 1; 869 870 base = slab_address(slab); 871 object = kasan_reset_tag(object); 872 object = restore_red_left(s, object); 873 if (object < base || object >= base + slab->objects * s->size || 874 (object - base) % s->size) { 875 return 0; 876 } 877 878 return 1; 879 } 880 881 static void print_section(char *level, char *text, u8 *addr, 882 unsigned int length) 883 { 884 metadata_access_enable(); 885 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS, 886 16, 1, kasan_reset_tag((void *)addr), length, 1); 887 metadata_access_disable(); 888 } 889 890 static struct track *get_track(struct kmem_cache *s, void *object, 891 enum track_item alloc) 892 { 893 struct track *p; 894 895 p = object + get_info_end(s); 896 897 return kasan_reset_tag(p + alloc); 898 } 899 900 #ifdef CONFIG_STACKDEPOT 901 static noinline depot_stack_handle_t set_track_prepare(void) 902 { 903 depot_stack_handle_t handle; 904 unsigned long entries[TRACK_ADDRS_COUNT]; 905 unsigned int nr_entries; 906 907 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3); 908 handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT); 909 910 return handle; 911 } 912 #else 913 static inline depot_stack_handle_t set_track_prepare(void) 914 { 915 return 0; 916 } 917 #endif 918 919 static void set_track_update(struct kmem_cache *s, void *object, 920 enum track_item alloc, unsigned long addr, 921 depot_stack_handle_t handle) 922 { 923 struct track *p = get_track(s, object, alloc); 924 925 #ifdef CONFIG_STACKDEPOT 926 p->handle = handle; 927 #endif 928 p->addr = addr; 929 p->cpu = smp_processor_id(); 930 p->pid = current->pid; 931 p->when = jiffies; 932 } 933 934 static __always_inline void set_track(struct kmem_cache *s, void *object, 935 enum track_item alloc, unsigned long addr) 936 { 937 depot_stack_handle_t handle = set_track_prepare(); 938 939 set_track_update(s, object, alloc, addr, handle); 940 } 941 942 static void init_tracking(struct kmem_cache *s, void *object) 943 { 944 struct track *p; 945 946 if (!(s->flags & SLAB_STORE_USER)) 947 return; 948 949 p = get_track(s, object, TRACK_ALLOC); 950 memset(p, 0, 2*sizeof(struct track)); 951 } 952 953 static void print_track(const char *s, struct track *t, unsigned long pr_time) 954 { 955 depot_stack_handle_t handle __maybe_unused; 956 957 if (!t->addr) 958 return; 959 960 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n", 961 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid); 962 #ifdef CONFIG_STACKDEPOT 963 handle = READ_ONCE(t->handle); 964 if (handle) 965 stack_depot_print(handle); 966 else 967 pr_err("object allocation/free stack trace missing\n"); 968 #endif 969 } 970 971 void print_tracking(struct kmem_cache *s, void *object) 972 { 973 unsigned long pr_time = jiffies; 974 if (!(s->flags & SLAB_STORE_USER)) 975 return; 976 977 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time); 978 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time); 979 } 980 981 static void print_slab_info(const struct slab *slab) 982 { 983 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n", 984 slab, slab->objects, slab->inuse, slab->freelist, 985 &slab->__page_flags); 986 } 987 988 /* 989 * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API 990 * family will round up the real request size to these fixed ones, so 991 * there could be an extra area than what is requested. Save the original 992 * request size in the meta data area, for better debug and sanity check. 993 */ 994 static inline void set_orig_size(struct kmem_cache *s, 995 void *object, unsigned int orig_size) 996 { 997 void *p = kasan_reset_tag(object); 998 unsigned int kasan_meta_size; 999 1000 if (!slub_debug_orig_size(s)) 1001 return; 1002 1003 /* 1004 * KASAN can save its free meta data inside of the object at offset 0. 1005 * If this meta data size is larger than 'orig_size', it will overlap 1006 * the data redzone in [orig_size+1, object_size]. Thus, we adjust 1007 * 'orig_size' to be as at least as big as KASAN's meta data. 1008 */ 1009 kasan_meta_size = kasan_metadata_size(s, true); 1010 if (kasan_meta_size > orig_size) 1011 orig_size = kasan_meta_size; 1012 1013 p += get_info_end(s); 1014 p += sizeof(struct track) * 2; 1015 1016 *(unsigned int *)p = orig_size; 1017 } 1018 1019 static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) 1020 { 1021 void *p = kasan_reset_tag(object); 1022 1023 if (!slub_debug_orig_size(s)) 1024 return s->object_size; 1025 1026 p += get_info_end(s); 1027 p += sizeof(struct track) * 2; 1028 1029 return *(unsigned int *)p; 1030 } 1031 1032 void skip_orig_size_check(struct kmem_cache *s, const void *object) 1033 { 1034 set_orig_size(s, (void *)object, s->object_size); 1035 } 1036 1037 static void slab_bug(struct kmem_cache *s, char *fmt, ...) 1038 { 1039 struct va_format vaf; 1040 va_list args; 1041 1042 va_start(args, fmt); 1043 vaf.fmt = fmt; 1044 vaf.va = &args; 1045 pr_err("=============================================================================\n"); 1046 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); 1047 pr_err("-----------------------------------------------------------------------------\n\n"); 1048 va_end(args); 1049 } 1050 1051 __printf(2, 3) 1052 static void slab_fix(struct kmem_cache *s, char *fmt, ...) 1053 { 1054 struct va_format vaf; 1055 va_list args; 1056 1057 if (slab_add_kunit_errors()) 1058 return; 1059 1060 va_start(args, fmt); 1061 vaf.fmt = fmt; 1062 vaf.va = &args; 1063 pr_err("FIX %s: %pV\n", s->name, &vaf); 1064 va_end(args); 1065 } 1066 1067 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) 1068 { 1069 unsigned int off; /* Offset of last byte */ 1070 u8 *addr = slab_address(slab); 1071 1072 print_tracking(s, p); 1073 1074 print_slab_info(slab); 1075 1076 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n", 1077 p, p - addr, get_freepointer(s, p)); 1078 1079 if (s->flags & SLAB_RED_ZONE) 1080 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad, 1081 s->red_left_pad); 1082 else if (p > addr + 16) 1083 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16); 1084 1085 print_section(KERN_ERR, "Object ", p, 1086 min_t(unsigned int, s->object_size, PAGE_SIZE)); 1087 if (s->flags & SLAB_RED_ZONE) 1088 print_section(KERN_ERR, "Redzone ", p + s->object_size, 1089 s->inuse - s->object_size); 1090 1091 off = get_info_end(s); 1092 1093 if (s->flags & SLAB_STORE_USER) 1094 off += 2 * sizeof(struct track); 1095 1096 if (slub_debug_orig_size(s)) 1097 off += sizeof(unsigned int); 1098 1099 off += kasan_metadata_size(s, false); 1100 1101 if (off != size_from_object(s)) 1102 /* Beginning of the filler is the free pointer */ 1103 print_section(KERN_ERR, "Padding ", p + off, 1104 size_from_object(s) - off); 1105 1106 dump_stack(); 1107 } 1108 1109 static void object_err(struct kmem_cache *s, struct slab *slab, 1110 u8 *object, char *reason) 1111 { 1112 if (slab_add_kunit_errors()) 1113 return; 1114 1115 slab_bug(s, "%s", reason); 1116 print_trailer(s, slab, object); 1117 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1118 } 1119 1120 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 1121 void **freelist, void *nextfree) 1122 { 1123 if ((s->flags & SLAB_CONSISTENCY_CHECKS) && 1124 !check_valid_pointer(s, slab, nextfree) && freelist) { 1125 object_err(s, slab, *freelist, "Freechain corrupt"); 1126 *freelist = NULL; 1127 slab_fix(s, "Isolate corrupted freechain"); 1128 return true; 1129 } 1130 1131 return false; 1132 } 1133 1134 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab, 1135 const char *fmt, ...) 1136 { 1137 va_list args; 1138 char buf[100]; 1139 1140 if (slab_add_kunit_errors()) 1141 return; 1142 1143 va_start(args, fmt); 1144 vsnprintf(buf, sizeof(buf), fmt, args); 1145 va_end(args); 1146 slab_bug(s, "%s", buf); 1147 print_slab_info(slab); 1148 dump_stack(); 1149 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1150 } 1151 1152 static void init_object(struct kmem_cache *s, void *object, u8 val) 1153 { 1154 u8 *p = kasan_reset_tag(object); 1155 unsigned int poison_size = s->object_size; 1156 1157 if (s->flags & SLAB_RED_ZONE) { 1158 /* 1159 * Here and below, avoid overwriting the KMSAN shadow. Keeping 1160 * the shadow makes it possible to distinguish uninit-value 1161 * from use-after-free. 1162 */ 1163 memset_no_sanitize_memory(p - s->red_left_pad, val, 1164 s->red_left_pad); 1165 1166 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { 1167 /* 1168 * Redzone the extra allocated space by kmalloc than 1169 * requested, and the poison size will be limited to 1170 * the original request size accordingly. 1171 */ 1172 poison_size = get_orig_size(s, object); 1173 } 1174 } 1175 1176 if (s->flags & __OBJECT_POISON) { 1177 memset_no_sanitize_memory(p, POISON_FREE, poison_size - 1); 1178 memset_no_sanitize_memory(p + poison_size - 1, POISON_END, 1); 1179 } 1180 1181 if (s->flags & SLAB_RED_ZONE) 1182 memset_no_sanitize_memory(p + poison_size, val, 1183 s->inuse - poison_size); 1184 } 1185 1186 static void restore_bytes(struct kmem_cache *s, char *message, u8 data, 1187 void *from, void *to) 1188 { 1189 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data); 1190 memset(from, data, to - from); 1191 } 1192 1193 #ifdef CONFIG_KMSAN 1194 #define pad_check_attributes noinline __no_kmsan_checks 1195 #else 1196 #define pad_check_attributes 1197 #endif 1198 1199 static pad_check_attributes int 1200 check_bytes_and_report(struct kmem_cache *s, struct slab *slab, 1201 u8 *object, char *what, 1202 u8 *start, unsigned int value, unsigned int bytes) 1203 { 1204 u8 *fault; 1205 u8 *end; 1206 u8 *addr = slab_address(slab); 1207 1208 metadata_access_enable(); 1209 fault = memchr_inv(kasan_reset_tag(start), value, bytes); 1210 metadata_access_disable(); 1211 if (!fault) 1212 return 1; 1213 1214 end = start + bytes; 1215 while (end > fault && end[-1] == value) 1216 end--; 1217 1218 if (slab_add_kunit_errors()) 1219 goto skip_bug_print; 1220 1221 slab_bug(s, "%s overwritten", what); 1222 pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", 1223 fault, end - 1, fault - addr, 1224 fault[0], value); 1225 1226 skip_bug_print: 1227 restore_bytes(s, what, value, fault, end); 1228 return 0; 1229 } 1230 1231 /* 1232 * Object layout: 1233 * 1234 * object address 1235 * Bytes of the object to be managed. 1236 * If the freepointer may overlay the object then the free 1237 * pointer is at the middle of the object. 1238 * 1239 * Poisoning uses 0x6b (POISON_FREE) and the last byte is 1240 * 0xa5 (POISON_END) 1241 * 1242 * object + s->object_size 1243 * Padding to reach word boundary. This is also used for Redzoning. 1244 * Padding is extended by another word if Redzoning is enabled and 1245 * object_size == inuse. 1246 * 1247 * We fill with 0xbb (SLUB_RED_INACTIVE) for inactive objects and with 1248 * 0xcc (SLUB_RED_ACTIVE) for objects in use. 1249 * 1250 * object + s->inuse 1251 * Meta data starts here. 1252 * 1253 * A. Free pointer (if we cannot overwrite object on free) 1254 * B. Tracking data for SLAB_STORE_USER 1255 * C. Original request size for kmalloc object (SLAB_STORE_USER enabled) 1256 * D. Padding to reach required alignment boundary or at minimum 1257 * one word if debugging is on to be able to detect writes 1258 * before the word boundary. 1259 * 1260 * Padding is done using 0x5a (POISON_INUSE) 1261 * 1262 * object + s->size 1263 * Nothing is used beyond s->size. 1264 * 1265 * If slabcaches are merged then the object_size and inuse boundaries are mostly 1266 * ignored. And therefore no slab options that rely on these boundaries 1267 * may be used with merged slabcaches. 1268 */ 1269 1270 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) 1271 { 1272 unsigned long off = get_info_end(s); /* The end of info */ 1273 1274 if (s->flags & SLAB_STORE_USER) { 1275 /* We also have user information there */ 1276 off += 2 * sizeof(struct track); 1277 1278 if (s->flags & SLAB_KMALLOC) 1279 off += sizeof(unsigned int); 1280 } 1281 1282 off += kasan_metadata_size(s, false); 1283 1284 if (size_from_object(s) == off) 1285 return 1; 1286 1287 return check_bytes_and_report(s, slab, p, "Object padding", 1288 p + off, POISON_INUSE, size_from_object(s) - off); 1289 } 1290 1291 /* Check the pad bytes at the end of a slab page */ 1292 static pad_check_attributes void 1293 slab_pad_check(struct kmem_cache *s, struct slab *slab) 1294 { 1295 u8 *start; 1296 u8 *fault; 1297 u8 *end; 1298 u8 *pad; 1299 int length; 1300 int remainder; 1301 1302 if (!(s->flags & SLAB_POISON)) 1303 return; 1304 1305 start = slab_address(slab); 1306 length = slab_size(slab); 1307 end = start + length; 1308 remainder = length % s->size; 1309 if (!remainder) 1310 return; 1311 1312 pad = end - remainder; 1313 metadata_access_enable(); 1314 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder); 1315 metadata_access_disable(); 1316 if (!fault) 1317 return; 1318 while (end > fault && end[-1] == POISON_INUSE) 1319 end--; 1320 1321 slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu", 1322 fault, end - 1, fault - start); 1323 print_section(KERN_ERR, "Padding ", pad, remainder); 1324 1325 restore_bytes(s, "slab padding", POISON_INUSE, fault, end); 1326 } 1327 1328 static int check_object(struct kmem_cache *s, struct slab *slab, 1329 void *object, u8 val) 1330 { 1331 u8 *p = object; 1332 u8 *endobject = object + s->object_size; 1333 unsigned int orig_size, kasan_meta_size; 1334 int ret = 1; 1335 1336 if (s->flags & SLAB_RED_ZONE) { 1337 if (!check_bytes_and_report(s, slab, object, "Left Redzone", 1338 object - s->red_left_pad, val, s->red_left_pad)) 1339 ret = 0; 1340 1341 if (!check_bytes_and_report(s, slab, object, "Right Redzone", 1342 endobject, val, s->inuse - s->object_size)) 1343 ret = 0; 1344 1345 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { 1346 orig_size = get_orig_size(s, object); 1347 1348 if (s->object_size > orig_size && 1349 !check_bytes_and_report(s, slab, object, 1350 "kmalloc Redzone", p + orig_size, 1351 val, s->object_size - orig_size)) { 1352 ret = 0; 1353 } 1354 } 1355 } else { 1356 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 1357 if (!check_bytes_and_report(s, slab, p, "Alignment padding", 1358 endobject, POISON_INUSE, 1359 s->inuse - s->object_size)) 1360 ret = 0; 1361 } 1362 } 1363 1364 if (s->flags & SLAB_POISON) { 1365 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) { 1366 /* 1367 * KASAN can save its free meta data inside of the 1368 * object at offset 0. Thus, skip checking the part of 1369 * the redzone that overlaps with the meta data. 1370 */ 1371 kasan_meta_size = kasan_metadata_size(s, true); 1372 if (kasan_meta_size < s->object_size - 1 && 1373 !check_bytes_and_report(s, slab, p, "Poison", 1374 p + kasan_meta_size, POISON_FREE, 1375 s->object_size - kasan_meta_size - 1)) 1376 ret = 0; 1377 if (kasan_meta_size < s->object_size && 1378 !check_bytes_and_report(s, slab, p, "End Poison", 1379 p + s->object_size - 1, POISON_END, 1)) 1380 ret = 0; 1381 } 1382 /* 1383 * check_pad_bytes cleans up on its own. 1384 */ 1385 if (!check_pad_bytes(s, slab, p)) 1386 ret = 0; 1387 } 1388 1389 /* 1390 * Cannot check freepointer while object is allocated if 1391 * object and freepointer overlap. 1392 */ 1393 if ((freeptr_outside_object(s) || val != SLUB_RED_ACTIVE) && 1394 !check_valid_pointer(s, slab, get_freepointer(s, p))) { 1395 object_err(s, slab, p, "Freepointer corrupt"); 1396 /* 1397 * No choice but to zap it and thus lose the remainder 1398 * of the free objects in this slab. May cause 1399 * another error because the object count is now wrong. 1400 */ 1401 set_freepointer(s, p, NULL); 1402 ret = 0; 1403 } 1404 1405 if (!ret && !slab_in_kunit_test()) { 1406 print_trailer(s, slab, object); 1407 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1408 } 1409 1410 return ret; 1411 } 1412 1413 static int check_slab(struct kmem_cache *s, struct slab *slab) 1414 { 1415 int maxobj; 1416 1417 if (!folio_test_slab(slab_folio(slab))) { 1418 slab_err(s, slab, "Not a valid slab page"); 1419 return 0; 1420 } 1421 1422 maxobj = order_objects(slab_order(slab), s->size); 1423 if (slab->objects > maxobj) { 1424 slab_err(s, slab, "objects %u > max %u", 1425 slab->objects, maxobj); 1426 return 0; 1427 } 1428 if (slab->inuse > slab->objects) { 1429 slab_err(s, slab, "inuse %u > max %u", 1430 slab->inuse, slab->objects); 1431 return 0; 1432 } 1433 /* Slab_pad_check fixes things up after itself */ 1434 slab_pad_check(s, slab); 1435 return 1; 1436 } 1437 1438 /* 1439 * Determine if a certain object in a slab is on the freelist. Must hold the 1440 * slab lock to guarantee that the chains are in a consistent state. 1441 */ 1442 static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search) 1443 { 1444 int nr = 0; 1445 void *fp; 1446 void *object = NULL; 1447 int max_objects; 1448 1449 fp = slab->freelist; 1450 while (fp && nr <= slab->objects) { 1451 if (fp == search) 1452 return 1; 1453 if (!check_valid_pointer(s, slab, fp)) { 1454 if (object) { 1455 object_err(s, slab, object, 1456 "Freechain corrupt"); 1457 set_freepointer(s, object, NULL); 1458 } else { 1459 slab_err(s, slab, "Freepointer corrupt"); 1460 slab->freelist = NULL; 1461 slab->inuse = slab->objects; 1462 slab_fix(s, "Freelist cleared"); 1463 return 0; 1464 } 1465 break; 1466 } 1467 object = fp; 1468 fp = get_freepointer(s, object); 1469 nr++; 1470 } 1471 1472 max_objects = order_objects(slab_order(slab), s->size); 1473 if (max_objects > MAX_OBJS_PER_PAGE) 1474 max_objects = MAX_OBJS_PER_PAGE; 1475 1476 if (slab->objects != max_objects) { 1477 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d", 1478 slab->objects, max_objects); 1479 slab->objects = max_objects; 1480 slab_fix(s, "Number of objects adjusted"); 1481 } 1482 if (slab->inuse != slab->objects - nr) { 1483 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d", 1484 slab->inuse, slab->objects - nr); 1485 slab->inuse = slab->objects - nr; 1486 slab_fix(s, "Object count adjusted"); 1487 } 1488 return search == NULL; 1489 } 1490 1491 static void trace(struct kmem_cache *s, struct slab *slab, void *object, 1492 int alloc) 1493 { 1494 if (s->flags & SLAB_TRACE) { 1495 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", 1496 s->name, 1497 alloc ? "alloc" : "free", 1498 object, slab->inuse, 1499 slab->freelist); 1500 1501 if (!alloc) 1502 print_section(KERN_INFO, "Object ", (void *)object, 1503 s->object_size); 1504 1505 dump_stack(); 1506 } 1507 } 1508 1509 /* 1510 * Tracking of fully allocated slabs for debugging purposes. 1511 */ 1512 static void add_full(struct kmem_cache *s, 1513 struct kmem_cache_node *n, struct slab *slab) 1514 { 1515 if (!(s->flags & SLAB_STORE_USER)) 1516 return; 1517 1518 lockdep_assert_held(&n->list_lock); 1519 list_add(&slab->slab_list, &n->full); 1520 } 1521 1522 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) 1523 { 1524 if (!(s->flags & SLAB_STORE_USER)) 1525 return; 1526 1527 lockdep_assert_held(&n->list_lock); 1528 list_del(&slab->slab_list); 1529 } 1530 1531 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 1532 { 1533 return atomic_long_read(&n->nr_slabs); 1534 } 1535 1536 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) 1537 { 1538 struct kmem_cache_node *n = get_node(s, node); 1539 1540 atomic_long_inc(&n->nr_slabs); 1541 atomic_long_add(objects, &n->total_objects); 1542 } 1543 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) 1544 { 1545 struct kmem_cache_node *n = get_node(s, node); 1546 1547 atomic_long_dec(&n->nr_slabs); 1548 atomic_long_sub(objects, &n->total_objects); 1549 } 1550 1551 /* Object debug checks for alloc/free paths */ 1552 static void setup_object_debug(struct kmem_cache *s, void *object) 1553 { 1554 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)) 1555 return; 1556 1557 init_object(s, object, SLUB_RED_INACTIVE); 1558 init_tracking(s, object); 1559 } 1560 1561 static 1562 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) 1563 { 1564 if (!kmem_cache_debug_flags(s, SLAB_POISON)) 1565 return; 1566 1567 metadata_access_enable(); 1568 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); 1569 metadata_access_disable(); 1570 } 1571 1572 static inline int alloc_consistency_checks(struct kmem_cache *s, 1573 struct slab *slab, void *object) 1574 { 1575 if (!check_slab(s, slab)) 1576 return 0; 1577 1578 if (!check_valid_pointer(s, slab, object)) { 1579 object_err(s, slab, object, "Freelist Pointer check fails"); 1580 return 0; 1581 } 1582 1583 if (!check_object(s, slab, object, SLUB_RED_INACTIVE)) 1584 return 0; 1585 1586 return 1; 1587 } 1588 1589 static noinline bool alloc_debug_processing(struct kmem_cache *s, 1590 struct slab *slab, void *object, int orig_size) 1591 { 1592 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 1593 if (!alloc_consistency_checks(s, slab, object)) 1594 goto bad; 1595 } 1596 1597 /* Success. Perform special debug activities for allocs */ 1598 trace(s, slab, object, 1); 1599 set_orig_size(s, object, orig_size); 1600 init_object(s, object, SLUB_RED_ACTIVE); 1601 return true; 1602 1603 bad: 1604 if (folio_test_slab(slab_folio(slab))) { 1605 /* 1606 * If this is a slab page then lets do the best we can 1607 * to avoid issues in the future. Marking all objects 1608 * as used avoids touching the remaining objects. 1609 */ 1610 slab_fix(s, "Marking all objects used"); 1611 slab->inuse = slab->objects; 1612 slab->freelist = NULL; 1613 } 1614 return false; 1615 } 1616 1617 static inline int free_consistency_checks(struct kmem_cache *s, 1618 struct slab *slab, void *object, unsigned long addr) 1619 { 1620 if (!check_valid_pointer(s, slab, object)) { 1621 slab_err(s, slab, "Invalid object pointer 0x%p", object); 1622 return 0; 1623 } 1624 1625 if (on_freelist(s, slab, object)) { 1626 object_err(s, slab, object, "Object already free"); 1627 return 0; 1628 } 1629 1630 if (!check_object(s, slab, object, SLUB_RED_ACTIVE)) 1631 return 0; 1632 1633 if (unlikely(s != slab->slab_cache)) { 1634 if (!folio_test_slab(slab_folio(slab))) { 1635 slab_err(s, slab, "Attempt to free object(0x%p) outside of slab", 1636 object); 1637 } else if (!slab->slab_cache) { 1638 pr_err("SLUB <none>: no slab for object 0x%p.\n", 1639 object); 1640 dump_stack(); 1641 } else 1642 object_err(s, slab, object, 1643 "page slab pointer corrupt."); 1644 return 0; 1645 } 1646 return 1; 1647 } 1648 1649 /* 1650 * Parse a block of slab_debug options. Blocks are delimited by ';' 1651 * 1652 * @str: start of block 1653 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified 1654 * @slabs: return start of list of slabs, or NULL when there's no list 1655 * @init: assume this is initial parsing and not per-kmem-create parsing 1656 * 1657 * returns the start of next block if there's any, or NULL 1658 */ 1659 static char * 1660 parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init) 1661 { 1662 bool higher_order_disable = false; 1663 1664 /* Skip any completely empty blocks */ 1665 while (*str && *str == ';') 1666 str++; 1667 1668 if (*str == ',') { 1669 /* 1670 * No options but restriction on slabs. This means full 1671 * debugging for slabs matching a pattern. 1672 */ 1673 *flags = DEBUG_DEFAULT_FLAGS; 1674 goto check_slabs; 1675 } 1676 *flags = 0; 1677 1678 /* Determine which debug features should be switched on */ 1679 for (; *str && *str != ',' && *str != ';'; str++) { 1680 switch (tolower(*str)) { 1681 case '-': 1682 *flags = 0; 1683 break; 1684 case 'f': 1685 *flags |= SLAB_CONSISTENCY_CHECKS; 1686 break; 1687 case 'z': 1688 *flags |= SLAB_RED_ZONE; 1689 break; 1690 case 'p': 1691 *flags |= SLAB_POISON; 1692 break; 1693 case 'u': 1694 *flags |= SLAB_STORE_USER; 1695 break; 1696 case 't': 1697 *flags |= SLAB_TRACE; 1698 break; 1699 case 'a': 1700 *flags |= SLAB_FAILSLAB; 1701 break; 1702 case 'o': 1703 /* 1704 * Avoid enabling debugging on caches if its minimum 1705 * order would increase as a result. 1706 */ 1707 higher_order_disable = true; 1708 break; 1709 default: 1710 if (init) 1711 pr_err("slab_debug option '%c' unknown. skipped\n", *str); 1712 } 1713 } 1714 check_slabs: 1715 if (*str == ',') 1716 *slabs = ++str; 1717 else 1718 *slabs = NULL; 1719 1720 /* Skip over the slab list */ 1721 while (*str && *str != ';') 1722 str++; 1723 1724 /* Skip any completely empty blocks */ 1725 while (*str && *str == ';') 1726 str++; 1727 1728 if (init && higher_order_disable) 1729 disable_higher_order_debug = 1; 1730 1731 if (*str) 1732 return str; 1733 else 1734 return NULL; 1735 } 1736 1737 static int __init setup_slub_debug(char *str) 1738 { 1739 slab_flags_t flags; 1740 slab_flags_t global_flags; 1741 char *saved_str; 1742 char *slab_list; 1743 bool global_slub_debug_changed = false; 1744 bool slab_list_specified = false; 1745 1746 global_flags = DEBUG_DEFAULT_FLAGS; 1747 if (*str++ != '=' || !*str) 1748 /* 1749 * No options specified. Switch on full debugging. 1750 */ 1751 goto out; 1752 1753 saved_str = str; 1754 while (str) { 1755 str = parse_slub_debug_flags(str, &flags, &slab_list, true); 1756 1757 if (!slab_list) { 1758 global_flags = flags; 1759 global_slub_debug_changed = true; 1760 } else { 1761 slab_list_specified = true; 1762 if (flags & SLAB_STORE_USER) 1763 stack_depot_request_early_init(); 1764 } 1765 } 1766 1767 /* 1768 * For backwards compatibility, a single list of flags with list of 1769 * slabs means debugging is only changed for those slabs, so the global 1770 * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending 1771 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as 1772 * long as there is no option specifying flags without a slab list. 1773 */ 1774 if (slab_list_specified) { 1775 if (!global_slub_debug_changed) 1776 global_flags = slub_debug; 1777 slub_debug_string = saved_str; 1778 } 1779 out: 1780 slub_debug = global_flags; 1781 if (slub_debug & SLAB_STORE_USER) 1782 stack_depot_request_early_init(); 1783 if (slub_debug != 0 || slub_debug_string) 1784 static_branch_enable(&slub_debug_enabled); 1785 else 1786 static_branch_disable(&slub_debug_enabled); 1787 if ((static_branch_unlikely(&init_on_alloc) || 1788 static_branch_unlikely(&init_on_free)) && 1789 (slub_debug & SLAB_POISON)) 1790 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n"); 1791 return 1; 1792 } 1793 1794 __setup("slab_debug", setup_slub_debug); 1795 __setup_param("slub_debug", slub_debug, setup_slub_debug, 0); 1796 1797 /* 1798 * kmem_cache_flags - apply debugging options to the cache 1799 * @flags: flags to set 1800 * @name: name of the cache 1801 * 1802 * Debug option(s) are applied to @flags. In addition to the debug 1803 * option(s), if a slab name (or multiple) is specified i.e. 1804 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ... 1805 * then only the select slabs will receive the debug option(s). 1806 */ 1807 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) 1808 { 1809 char *iter; 1810 size_t len; 1811 char *next_block; 1812 slab_flags_t block_flags; 1813 slab_flags_t slub_debug_local = slub_debug; 1814 1815 if (flags & SLAB_NO_USER_FLAGS) 1816 return flags; 1817 1818 /* 1819 * If the slab cache is for debugging (e.g. kmemleak) then 1820 * don't store user (stack trace) information by default, 1821 * but let the user enable it via the command line below. 1822 */ 1823 if (flags & SLAB_NOLEAKTRACE) 1824 slub_debug_local &= ~SLAB_STORE_USER; 1825 1826 len = strlen(name); 1827 next_block = slub_debug_string; 1828 /* Go through all blocks of debug options, see if any matches our slab's name */ 1829 while (next_block) { 1830 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false); 1831 if (!iter) 1832 continue; 1833 /* Found a block that has a slab list, search it */ 1834 while (*iter) { 1835 char *end, *glob; 1836 size_t cmplen; 1837 1838 end = strchrnul(iter, ','); 1839 if (next_block && next_block < end) 1840 end = next_block - 1; 1841 1842 glob = strnchr(iter, end - iter, '*'); 1843 if (glob) 1844 cmplen = glob - iter; 1845 else 1846 cmplen = max_t(size_t, len, (end - iter)); 1847 1848 if (!strncmp(name, iter, cmplen)) { 1849 flags |= block_flags; 1850 return flags; 1851 } 1852 1853 if (!*end || *end == ';') 1854 break; 1855 iter = end + 1; 1856 } 1857 } 1858 1859 return flags | slub_debug_local; 1860 } 1861 #else /* !CONFIG_SLUB_DEBUG */ 1862 static inline void setup_object_debug(struct kmem_cache *s, void *object) {} 1863 static inline 1864 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {} 1865 1866 static inline bool alloc_debug_processing(struct kmem_cache *s, 1867 struct slab *slab, void *object, int orig_size) { return true; } 1868 1869 static inline bool free_debug_processing(struct kmem_cache *s, 1870 struct slab *slab, void *head, void *tail, int *bulk_cnt, 1871 unsigned long addr, depot_stack_handle_t handle) { return true; } 1872 1873 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {} 1874 static inline int check_object(struct kmem_cache *s, struct slab *slab, 1875 void *object, u8 val) { return 1; } 1876 static inline depot_stack_handle_t set_track_prepare(void) { return 0; } 1877 static inline void set_track(struct kmem_cache *s, void *object, 1878 enum track_item alloc, unsigned long addr) {} 1879 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, 1880 struct slab *slab) {} 1881 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, 1882 struct slab *slab) {} 1883 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) 1884 { 1885 return flags; 1886 } 1887 #define slub_debug 0 1888 1889 #define disable_higher_order_debug 0 1890 1891 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 1892 { return 0; } 1893 static inline void inc_slabs_node(struct kmem_cache *s, int node, 1894 int objects) {} 1895 static inline void dec_slabs_node(struct kmem_cache *s, int node, 1896 int objects) {} 1897 1898 #ifndef CONFIG_SLUB_TINY 1899 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 1900 void **freelist, void *nextfree) 1901 { 1902 return false; 1903 } 1904 #endif 1905 #endif /* CONFIG_SLUB_DEBUG */ 1906 1907 #ifdef CONFIG_SLAB_OBJ_EXT 1908 1909 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG 1910 1911 static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) 1912 { 1913 struct slabobj_ext *slab_exts; 1914 struct slab *obj_exts_slab; 1915 1916 obj_exts_slab = virt_to_slab(obj_exts); 1917 slab_exts = slab_obj_exts(obj_exts_slab); 1918 if (slab_exts) { 1919 unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, 1920 obj_exts_slab, obj_exts); 1921 /* codetag should be NULL */ 1922 WARN_ON(slab_exts[offs].ref.ct); 1923 set_codetag_empty(&slab_exts[offs].ref); 1924 } 1925 } 1926 1927 static inline void mark_failed_objexts_alloc(struct slab *slab) 1928 { 1929 slab->obj_exts = OBJEXTS_ALLOC_FAIL; 1930 } 1931 1932 static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 1933 struct slabobj_ext *vec, unsigned int objects) 1934 { 1935 /* 1936 * If vector previously failed to allocate then we have live 1937 * objects with no tag reference. Mark all references in this 1938 * vector as empty to avoid warnings later on. 1939 */ 1940 if (obj_exts & OBJEXTS_ALLOC_FAIL) { 1941 unsigned int i; 1942 1943 for (i = 0; i < objects; i++) 1944 set_codetag_empty(&vec[i].ref); 1945 } 1946 } 1947 1948 #else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 1949 1950 static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} 1951 static inline void mark_failed_objexts_alloc(struct slab *slab) {} 1952 static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 1953 struct slabobj_ext *vec, unsigned int objects) {} 1954 1955 #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 1956 1957 /* 1958 * The allocated objcg pointers array is not accounted directly. 1959 * Moreover, it should not come from DMA buffer and is not readily 1960 * reclaimable. So those GFP bits should be masked off. 1961 */ 1962 #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ 1963 __GFP_ACCOUNT | __GFP_NOFAIL) 1964 1965 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 1966 gfp_t gfp, bool new_slab) 1967 { 1968 unsigned int objects = objs_per_slab(s, slab); 1969 unsigned long new_exts; 1970 unsigned long old_exts; 1971 struct slabobj_ext *vec; 1972 1973 gfp &= ~OBJCGS_CLEAR_MASK; 1974 /* Prevent recursive extension vector allocation */ 1975 gfp |= __GFP_NO_OBJ_EXT; 1976 vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, 1977 slab_nid(slab)); 1978 if (!vec) { 1979 /* Mark vectors which failed to allocate */ 1980 if (new_slab) 1981 mark_failed_objexts_alloc(slab); 1982 1983 return -ENOMEM; 1984 } 1985 1986 new_exts = (unsigned long)vec; 1987 #ifdef CONFIG_MEMCG 1988 new_exts |= MEMCG_DATA_OBJEXTS; 1989 #endif 1990 old_exts = READ_ONCE(slab->obj_exts); 1991 handle_failed_objexts_alloc(old_exts, vec, objects); 1992 if (new_slab) { 1993 /* 1994 * If the slab is brand new and nobody can yet access its 1995 * obj_exts, no synchronization is required and obj_exts can 1996 * be simply assigned. 1997 */ 1998 slab->obj_exts = new_exts; 1999 } else if ((old_exts & ~OBJEXTS_FLAGS_MASK) || 2000 cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { 2001 /* 2002 * If the slab is already in use, somebody can allocate and 2003 * assign slabobj_exts in parallel. In this case the existing 2004 * objcg vector should be reused. 2005 */ 2006 mark_objexts_empty(vec); 2007 kfree(vec); 2008 return 0; 2009 } 2010 2011 kmemleak_not_leak(vec); 2012 return 0; 2013 } 2014 2015 static inline void free_slab_obj_exts(struct slab *slab) 2016 { 2017 struct slabobj_ext *obj_exts; 2018 2019 obj_exts = slab_obj_exts(slab); 2020 if (!obj_exts) 2021 return; 2022 2023 /* 2024 * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its 2025 * corresponding extension will be NULL. alloc_tag_sub() will throw a 2026 * warning if slab has extensions but the extension of an object is 2027 * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that 2028 * the extension for obj_exts is expected to be NULL. 2029 */ 2030 mark_objexts_empty(obj_exts); 2031 kfree(obj_exts); 2032 slab->obj_exts = 0; 2033 } 2034 2035 static inline bool need_slab_obj_ext(void) 2036 { 2037 if (mem_alloc_profiling_enabled()) 2038 return true; 2039 2040 /* 2041 * CONFIG_MEMCG creates vector of obj_cgroup objects conditionally 2042 * inside memcg_slab_post_alloc_hook. No other users for now. 2043 */ 2044 return false; 2045 } 2046 2047 #else /* CONFIG_SLAB_OBJ_EXT */ 2048 2049 static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 2050 gfp_t gfp, bool new_slab) 2051 { 2052 return 0; 2053 } 2054 2055 static inline void free_slab_obj_exts(struct slab *slab) 2056 { 2057 } 2058 2059 static inline bool need_slab_obj_ext(void) 2060 { 2061 return false; 2062 } 2063 2064 #endif /* CONFIG_SLAB_OBJ_EXT */ 2065 2066 #ifdef CONFIG_MEM_ALLOC_PROFILING 2067 2068 static inline struct slabobj_ext * 2069 prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) 2070 { 2071 struct slab *slab; 2072 2073 if (!p) 2074 return NULL; 2075 2076 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) 2077 return NULL; 2078 2079 if (flags & __GFP_NO_OBJ_EXT) 2080 return NULL; 2081 2082 slab = virt_to_slab(p); 2083 if (!slab_obj_exts(slab) && 2084 WARN(alloc_slab_obj_exts(slab, s, flags, false), 2085 "%s, %s: Failed to create slab extension vector!\n", 2086 __func__, s->name)) 2087 return NULL; 2088 2089 return slab_obj_exts(slab) + obj_to_index(s, slab, p); 2090 } 2091 2092 static inline void 2093 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2094 { 2095 if (need_slab_obj_ext()) { 2096 struct slabobj_ext *obj_exts; 2097 2098 obj_exts = prepare_slab_obj_exts_hook(s, flags, object); 2099 /* 2100 * Currently obj_exts is used only for allocation profiling. 2101 * If other users appear then mem_alloc_profiling_enabled() 2102 * check should be added before alloc_tag_add(). 2103 */ 2104 if (likely(obj_exts)) 2105 alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size); 2106 } 2107 } 2108 2109 static inline void 2110 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2111 int objects) 2112 { 2113 struct slabobj_ext *obj_exts; 2114 int i; 2115 2116 if (!mem_alloc_profiling_enabled()) 2117 return; 2118 2119 /* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */ 2120 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) 2121 return; 2122 2123 obj_exts = slab_obj_exts(slab); 2124 if (!obj_exts) 2125 return; 2126 2127 for (i = 0; i < objects; i++) { 2128 unsigned int off = obj_to_index(s, slab, p[i]); 2129 2130 alloc_tag_sub(&obj_exts[off].ref, s->size); 2131 } 2132 } 2133 2134 #else /* CONFIG_MEM_ALLOC_PROFILING */ 2135 2136 static inline void 2137 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2138 { 2139 } 2140 2141 static inline void 2142 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2143 int objects) 2144 { 2145 } 2146 2147 #endif /* CONFIG_MEM_ALLOC_PROFILING */ 2148 2149 2150 #ifdef CONFIG_MEMCG 2151 2152 static void memcg_alloc_abort_single(struct kmem_cache *s, void *object); 2153 2154 static __fastpath_inline 2155 bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 2156 gfp_t flags, size_t size, void **p) 2157 { 2158 if (likely(!memcg_kmem_online())) 2159 return true; 2160 2161 if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))) 2162 return true; 2163 2164 if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p))) 2165 return true; 2166 2167 if (likely(size == 1)) { 2168 memcg_alloc_abort_single(s, *p); 2169 *p = NULL; 2170 } else { 2171 kmem_cache_free_bulk(s, size, p); 2172 } 2173 2174 return false; 2175 } 2176 2177 static __fastpath_inline 2178 void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2179 int objects) 2180 { 2181 struct slabobj_ext *obj_exts; 2182 2183 if (!memcg_kmem_online()) 2184 return; 2185 2186 obj_exts = slab_obj_exts(slab); 2187 if (likely(!obj_exts)) 2188 return; 2189 2190 __memcg_slab_free_hook(s, slab, p, objects, obj_exts); 2191 } 2192 #else /* CONFIG_MEMCG */ 2193 static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s, 2194 struct list_lru *lru, 2195 gfp_t flags, size_t size, 2196 void **p) 2197 { 2198 return true; 2199 } 2200 2201 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 2202 void **p, int objects) 2203 { 2204 } 2205 #endif /* CONFIG_MEMCG */ 2206 2207 /* 2208 * Hooks for other subsystems that check memory allocations. In a typical 2209 * production configuration these hooks all should produce no code at all. 2210 * 2211 * Returns true if freeing of the object can proceed, false if its reuse 2212 * was delayed by KASAN quarantine, or it was returned to KFENCE. 2213 */ 2214 static __always_inline 2215 bool slab_free_hook(struct kmem_cache *s, void *x, bool init) 2216 { 2217 kmemleak_free_recursive(x, s->flags); 2218 kmsan_slab_free(s, x); 2219 2220 debug_check_no_locks_freed(x, s->object_size); 2221 2222 if (!(s->flags & SLAB_DEBUG_OBJECTS)) 2223 debug_check_no_obj_freed(x, s->object_size); 2224 2225 /* Use KCSAN to help debug racy use-after-free. */ 2226 if (!(s->flags & SLAB_TYPESAFE_BY_RCU)) 2227 __kcsan_check_access(x, s->object_size, 2228 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT); 2229 2230 if (kfence_free(x)) 2231 return false; 2232 2233 /* 2234 * As memory initialization might be integrated into KASAN, 2235 * kasan_slab_free and initialization memset's must be 2236 * kept together to avoid discrepancies in behavior. 2237 * 2238 * The initialization memset's clear the object and the metadata, 2239 * but don't touch the SLAB redzone. 2240 * 2241 * The object's freepointer is also avoided if stored outside the 2242 * object. 2243 */ 2244 if (unlikely(init)) { 2245 int rsize; 2246 unsigned int inuse; 2247 2248 inuse = get_info_end(s); 2249 if (!kasan_has_integrated_init()) 2250 memset(kasan_reset_tag(x), 0, s->object_size); 2251 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; 2252 memset((char *)kasan_reset_tag(x) + inuse, 0, 2253 s->size - inuse - rsize); 2254 } 2255 /* KASAN might put x into memory quarantine, delaying its reuse. */ 2256 return !kasan_slab_free(s, x, init); 2257 } 2258 2259 static __fastpath_inline 2260 bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, 2261 int *cnt) 2262 { 2263 2264 void *object; 2265 void *next = *head; 2266 void *old_tail = *tail; 2267 bool init; 2268 2269 if (is_kfence_address(next)) { 2270 slab_free_hook(s, next, false); 2271 return false; 2272 } 2273 2274 /* Head and tail of the reconstructed freelist */ 2275 *head = NULL; 2276 *tail = NULL; 2277 2278 init = slab_want_init_on_free(s); 2279 2280 do { 2281 object = next; 2282 next = get_freepointer(s, object); 2283 2284 /* If object's reuse doesn't have to be delayed */ 2285 if (likely(slab_free_hook(s, object, init))) { 2286 /* Move object to the new freelist */ 2287 set_freepointer(s, object, *head); 2288 *head = object; 2289 if (!*tail) 2290 *tail = object; 2291 } else { 2292 /* 2293 * Adjust the reconstructed freelist depth 2294 * accordingly if object's reuse is delayed. 2295 */ 2296 --(*cnt); 2297 } 2298 } while (object != old_tail); 2299 2300 return *head != NULL; 2301 } 2302 2303 static void *setup_object(struct kmem_cache *s, void *object) 2304 { 2305 setup_object_debug(s, object); 2306 object = kasan_init_slab_obj(s, object); 2307 if (unlikely(s->ctor)) { 2308 kasan_unpoison_new_object(s, object); 2309 s->ctor(object); 2310 kasan_poison_new_object(s, object); 2311 } 2312 return object; 2313 } 2314 2315 /* 2316 * Slab allocation and freeing 2317 */ 2318 static inline struct slab *alloc_slab_page(gfp_t flags, int node, 2319 struct kmem_cache_order_objects oo) 2320 { 2321 struct folio *folio; 2322 struct slab *slab; 2323 unsigned int order = oo_order(oo); 2324 2325 folio = (struct folio *)alloc_pages_node(node, flags, order); 2326 if (!folio) 2327 return NULL; 2328 2329 slab = folio_slab(folio); 2330 __folio_set_slab(folio); 2331 /* Make the flag visible before any changes to folio->mapping */ 2332 smp_wmb(); 2333 if (folio_is_pfmemalloc(folio)) 2334 slab_set_pfmemalloc(slab); 2335 2336 return slab; 2337 } 2338 2339 #ifdef CONFIG_SLAB_FREELIST_RANDOM 2340 /* Pre-initialize the random sequence cache */ 2341 static int init_cache_random_seq(struct kmem_cache *s) 2342 { 2343 unsigned int count = oo_objects(s->oo); 2344 int err; 2345 2346 /* Bailout if already initialised */ 2347 if (s->random_seq) 2348 return 0; 2349 2350 err = cache_random_seq_create(s, count, GFP_KERNEL); 2351 if (err) { 2352 pr_err("SLUB: Unable to initialize free list for %s\n", 2353 s->name); 2354 return err; 2355 } 2356 2357 /* Transform to an offset on the set of pages */ 2358 if (s->random_seq) { 2359 unsigned int i; 2360 2361 for (i = 0; i < count; i++) 2362 s->random_seq[i] *= s->size; 2363 } 2364 return 0; 2365 } 2366 2367 /* Initialize each random sequence freelist per cache */ 2368 static void __init init_freelist_randomization(void) 2369 { 2370 struct kmem_cache *s; 2371 2372 mutex_lock(&slab_mutex); 2373 2374 list_for_each_entry(s, &slab_caches, list) 2375 init_cache_random_seq(s); 2376 2377 mutex_unlock(&slab_mutex); 2378 } 2379 2380 /* Get the next entry on the pre-computed freelist randomized */ 2381 static void *next_freelist_entry(struct kmem_cache *s, 2382 unsigned long *pos, void *start, 2383 unsigned long page_limit, 2384 unsigned long freelist_count) 2385 { 2386 unsigned int idx; 2387 2388 /* 2389 * If the target page allocation failed, the number of objects on the 2390 * page might be smaller than the usual size defined by the cache. 2391 */ 2392 do { 2393 idx = s->random_seq[*pos]; 2394 *pos += 1; 2395 if (*pos >= freelist_count) 2396 *pos = 0; 2397 } while (unlikely(idx >= page_limit)); 2398 2399 return (char *)start + idx; 2400 } 2401 2402 /* Shuffle the single linked freelist based on a random pre-computed sequence */ 2403 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) 2404 { 2405 void *start; 2406 void *cur; 2407 void *next; 2408 unsigned long idx, pos, page_limit, freelist_count; 2409 2410 if (slab->objects < 2 || !s->random_seq) 2411 return false; 2412 2413 freelist_count = oo_objects(s->oo); 2414 pos = get_random_u32_below(freelist_count); 2415 2416 page_limit = slab->objects * s->size; 2417 start = fixup_red_left(s, slab_address(slab)); 2418 2419 /* First entry is used as the base of the freelist */ 2420 cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count); 2421 cur = setup_object(s, cur); 2422 slab->freelist = cur; 2423 2424 for (idx = 1; idx < slab->objects; idx++) { 2425 next = next_freelist_entry(s, &pos, start, page_limit, 2426 freelist_count); 2427 next = setup_object(s, next); 2428 set_freepointer(s, cur, next); 2429 cur = next; 2430 } 2431 set_freepointer(s, cur, NULL); 2432 2433 return true; 2434 } 2435 #else 2436 static inline int init_cache_random_seq(struct kmem_cache *s) 2437 { 2438 return 0; 2439 } 2440 static inline void init_freelist_randomization(void) { } 2441 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) 2442 { 2443 return false; 2444 } 2445 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 2446 2447 static __always_inline void account_slab(struct slab *slab, int order, 2448 struct kmem_cache *s, gfp_t gfp) 2449 { 2450 if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) 2451 alloc_slab_obj_exts(slab, s, gfp, true); 2452 2453 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 2454 PAGE_SIZE << order); 2455 } 2456 2457 static __always_inline void unaccount_slab(struct slab *slab, int order, 2458 struct kmem_cache *s) 2459 { 2460 if (memcg_kmem_online() || need_slab_obj_ext()) 2461 free_slab_obj_exts(slab); 2462 2463 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 2464 -(PAGE_SIZE << order)); 2465 } 2466 2467 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 2468 { 2469 struct slab *slab; 2470 struct kmem_cache_order_objects oo = s->oo; 2471 gfp_t alloc_gfp; 2472 void *start, *p, *next; 2473 int idx; 2474 bool shuffle; 2475 2476 flags &= gfp_allowed_mask; 2477 2478 flags |= s->allocflags; 2479 2480 /* 2481 * Let the initial higher-order allocation fail under memory pressure 2482 * so we fall-back to the minimum order allocation. 2483 */ 2484 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; 2485 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) 2486 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; 2487 2488 slab = alloc_slab_page(alloc_gfp, node, oo); 2489 if (unlikely(!slab)) { 2490 oo = s->min; 2491 alloc_gfp = flags; 2492 /* 2493 * Allocation may have failed due to fragmentation. 2494 * Try a lower order alloc if possible 2495 */ 2496 slab = alloc_slab_page(alloc_gfp, node, oo); 2497 if (unlikely(!slab)) 2498 return NULL; 2499 stat(s, ORDER_FALLBACK); 2500 } 2501 2502 slab->objects = oo_objects(oo); 2503 slab->inuse = 0; 2504 slab->frozen = 0; 2505 2506 account_slab(slab, oo_order(oo), s, flags); 2507 2508 slab->slab_cache = s; 2509 2510 kasan_poison_slab(slab); 2511 2512 start = slab_address(slab); 2513 2514 setup_slab_debug(s, slab, start); 2515 2516 shuffle = shuffle_freelist(s, slab); 2517 2518 if (!shuffle) { 2519 start = fixup_red_left(s, start); 2520 start = setup_object(s, start); 2521 slab->freelist = start; 2522 for (idx = 0, p = start; idx < slab->objects - 1; idx++) { 2523 next = p + s->size; 2524 next = setup_object(s, next); 2525 set_freepointer(s, p, next); 2526 p = next; 2527 } 2528 set_freepointer(s, p, NULL); 2529 } 2530 2531 return slab; 2532 } 2533 2534 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) 2535 { 2536 if (unlikely(flags & GFP_SLAB_BUG_MASK)) 2537 flags = kmalloc_fix_flags(flags); 2538 2539 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); 2540 2541 return allocate_slab(s, 2542 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 2543 } 2544 2545 static void __free_slab(struct kmem_cache *s, struct slab *slab) 2546 { 2547 struct folio *folio = slab_folio(slab); 2548 int order = folio_order(folio); 2549 int pages = 1 << order; 2550 2551 __slab_clear_pfmemalloc(slab); 2552 folio->mapping = NULL; 2553 /* Make the mapping reset visible before clearing the flag */ 2554 smp_wmb(); 2555 __folio_clear_slab(folio); 2556 mm_account_reclaimed_pages(pages); 2557 unaccount_slab(slab, order, s); 2558 __free_pages(&folio->page, order); 2559 } 2560 2561 static void rcu_free_slab(struct rcu_head *h) 2562 { 2563 struct slab *slab = container_of(h, struct slab, rcu_head); 2564 2565 __free_slab(slab->slab_cache, slab); 2566 } 2567 2568 static void free_slab(struct kmem_cache *s, struct slab *slab) 2569 { 2570 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) { 2571 void *p; 2572 2573 slab_pad_check(s, slab); 2574 for_each_object(p, s, slab_address(slab), slab->objects) 2575 check_object(s, slab, p, SLUB_RED_INACTIVE); 2576 } 2577 2578 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) 2579 call_rcu(&slab->rcu_head, rcu_free_slab); 2580 else 2581 __free_slab(s, slab); 2582 } 2583 2584 static void discard_slab(struct kmem_cache *s, struct slab *slab) 2585 { 2586 dec_slabs_node(s, slab_nid(slab), slab->objects); 2587 free_slab(s, slab); 2588 } 2589 2590 /* 2591 * SLUB reuses PG_workingset bit to keep track of whether it's on 2592 * the per-node partial list. 2593 */ 2594 static inline bool slab_test_node_partial(const struct slab *slab) 2595 { 2596 return folio_test_workingset(slab_folio(slab)); 2597 } 2598 2599 static inline void slab_set_node_partial(struct slab *slab) 2600 { 2601 set_bit(PG_workingset, folio_flags(slab_folio(slab), 0)); 2602 } 2603 2604 static inline void slab_clear_node_partial(struct slab *slab) 2605 { 2606 clear_bit(PG_workingset, folio_flags(slab_folio(slab), 0)); 2607 } 2608 2609 /* 2610 * Management of partially allocated slabs. 2611 */ 2612 static inline void 2613 __add_partial(struct kmem_cache_node *n, struct slab *slab, int tail) 2614 { 2615 n->nr_partial++; 2616 if (tail == DEACTIVATE_TO_TAIL) 2617 list_add_tail(&slab->slab_list, &n->partial); 2618 else 2619 list_add(&slab->slab_list, &n->partial); 2620 slab_set_node_partial(slab); 2621 } 2622 2623 static inline void add_partial(struct kmem_cache_node *n, 2624 struct slab *slab, int tail) 2625 { 2626 lockdep_assert_held(&n->list_lock); 2627 __add_partial(n, slab, tail); 2628 } 2629 2630 static inline void remove_partial(struct kmem_cache_node *n, 2631 struct slab *slab) 2632 { 2633 lockdep_assert_held(&n->list_lock); 2634 list_del(&slab->slab_list); 2635 slab_clear_node_partial(slab); 2636 n->nr_partial--; 2637 } 2638 2639 /* 2640 * Called only for kmem_cache_debug() caches instead of remove_partial(), with a 2641 * slab from the n->partial list. Remove only a single object from the slab, do 2642 * the alloc_debug_processing() checks and leave the slab on the list, or move 2643 * it to full list if it was the last free object. 2644 */ 2645 static void *alloc_single_from_partial(struct kmem_cache *s, 2646 struct kmem_cache_node *n, struct slab *slab, int orig_size) 2647 { 2648 void *object; 2649 2650 lockdep_assert_held(&n->list_lock); 2651 2652 object = slab->freelist; 2653 slab->freelist = get_freepointer(s, object); 2654 slab->inuse++; 2655 2656 if (!alloc_debug_processing(s, slab, object, orig_size)) { 2657 remove_partial(n, slab); 2658 return NULL; 2659 } 2660 2661 if (slab->inuse == slab->objects) { 2662 remove_partial(n, slab); 2663 add_full(s, n, slab); 2664 } 2665 2666 return object; 2667 } 2668 2669 /* 2670 * Called only for kmem_cache_debug() caches to allocate from a freshly 2671 * allocated slab. Allocate a single object instead of whole freelist 2672 * and put the slab to the partial (or full) list. 2673 */ 2674 static void *alloc_single_from_new_slab(struct kmem_cache *s, 2675 struct slab *slab, int orig_size) 2676 { 2677 int nid = slab_nid(slab); 2678 struct kmem_cache_node *n = get_node(s, nid); 2679 unsigned long flags; 2680 void *object; 2681 2682 2683 object = slab->freelist; 2684 slab->freelist = get_freepointer(s, object); 2685 slab->inuse = 1; 2686 2687 if (!alloc_debug_processing(s, slab, object, orig_size)) 2688 /* 2689 * It's not really expected that this would fail on a 2690 * freshly allocated slab, but a concurrent memory 2691 * corruption in theory could cause that. 2692 */ 2693 return NULL; 2694 2695 spin_lock_irqsave(&n->list_lock, flags); 2696 2697 if (slab->inuse == slab->objects) 2698 add_full(s, n, slab); 2699 else 2700 add_partial(n, slab, DEACTIVATE_TO_HEAD); 2701 2702 inc_slabs_node(s, nid, slab->objects); 2703 spin_unlock_irqrestore(&n->list_lock, flags); 2704 2705 return object; 2706 } 2707 2708 #ifdef CONFIG_SLUB_CPU_PARTIAL 2709 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain); 2710 #else 2711 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab, 2712 int drain) { } 2713 #endif 2714 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags); 2715 2716 /* 2717 * Try to allocate a partial slab from a specific node. 2718 */ 2719 static struct slab *get_partial_node(struct kmem_cache *s, 2720 struct kmem_cache_node *n, 2721 struct partial_context *pc) 2722 { 2723 struct slab *slab, *slab2, *partial = NULL; 2724 unsigned long flags; 2725 unsigned int partial_slabs = 0; 2726 2727 /* 2728 * Racy check. If we mistakenly see no partial slabs then we 2729 * just allocate an empty slab. If we mistakenly try to get a 2730 * partial slab and there is none available then get_partial() 2731 * will return NULL. 2732 */ 2733 if (!n || !n->nr_partial) 2734 return NULL; 2735 2736 spin_lock_irqsave(&n->list_lock, flags); 2737 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { 2738 if (!pfmemalloc_match(slab, pc->flags)) 2739 continue; 2740 2741 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 2742 void *object = alloc_single_from_partial(s, n, slab, 2743 pc->orig_size); 2744 if (object) { 2745 partial = slab; 2746 pc->object = object; 2747 break; 2748 } 2749 continue; 2750 } 2751 2752 remove_partial(n, slab); 2753 2754 if (!partial) { 2755 partial = slab; 2756 stat(s, ALLOC_FROM_PARTIAL); 2757 2758 if ((slub_get_cpu_partial(s) == 0)) { 2759 break; 2760 } 2761 } else { 2762 put_cpu_partial(s, slab, 0); 2763 stat(s, CPU_PARTIAL_NODE); 2764 2765 if (++partial_slabs > slub_get_cpu_partial(s) / 2) { 2766 break; 2767 } 2768 } 2769 } 2770 spin_unlock_irqrestore(&n->list_lock, flags); 2771 return partial; 2772 } 2773 2774 /* 2775 * Get a slab from somewhere. Search in increasing NUMA distances. 2776 */ 2777 static struct slab *get_any_partial(struct kmem_cache *s, 2778 struct partial_context *pc) 2779 { 2780 #ifdef CONFIG_NUMA 2781 struct zonelist *zonelist; 2782 struct zoneref *z; 2783 struct zone *zone; 2784 enum zone_type highest_zoneidx = gfp_zone(pc->flags); 2785 struct slab *slab; 2786 unsigned int cpuset_mems_cookie; 2787 2788 /* 2789 * The defrag ratio allows a configuration of the tradeoffs between 2790 * inter node defragmentation and node local allocations. A lower 2791 * defrag_ratio increases the tendency to do local allocations 2792 * instead of attempting to obtain partial slabs from other nodes. 2793 * 2794 * If the defrag_ratio is set to 0 then kmalloc() always 2795 * returns node local objects. If the ratio is higher then kmalloc() 2796 * may return off node objects because partial slabs are obtained 2797 * from other nodes and filled up. 2798 * 2799 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100 2800 * (which makes defrag_ratio = 1000) then every (well almost) 2801 * allocation will first attempt to defrag slab caches on other nodes. 2802 * This means scanning over all nodes to look for partial slabs which 2803 * may be expensive if we do it every time we are trying to find a slab 2804 * with available objects. 2805 */ 2806 if (!s->remote_node_defrag_ratio || 2807 get_cycles() % 1024 > s->remote_node_defrag_ratio) 2808 return NULL; 2809 2810 do { 2811 cpuset_mems_cookie = read_mems_allowed_begin(); 2812 zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); 2813 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { 2814 struct kmem_cache_node *n; 2815 2816 n = get_node(s, zone_to_nid(zone)); 2817 2818 if (n && cpuset_zone_allowed(zone, pc->flags) && 2819 n->nr_partial > s->min_partial) { 2820 slab = get_partial_node(s, n, pc); 2821 if (slab) { 2822 /* 2823 * Don't check read_mems_allowed_retry() 2824 * here - if mems_allowed was updated in 2825 * parallel, that was a harmless race 2826 * between allocation and the cpuset 2827 * update 2828 */ 2829 return slab; 2830 } 2831 } 2832 } 2833 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 2834 #endif /* CONFIG_NUMA */ 2835 return NULL; 2836 } 2837 2838 /* 2839 * Get a partial slab, lock it and return it. 2840 */ 2841 static struct slab *get_partial(struct kmem_cache *s, int node, 2842 struct partial_context *pc) 2843 { 2844 struct slab *slab; 2845 int searchnode = node; 2846 2847 if (node == NUMA_NO_NODE) 2848 searchnode = numa_mem_id(); 2849 2850 slab = get_partial_node(s, get_node(s, searchnode), pc); 2851 if (slab || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE))) 2852 return slab; 2853 2854 return get_any_partial(s, pc); 2855 } 2856 2857 #ifndef CONFIG_SLUB_TINY 2858 2859 #ifdef CONFIG_PREEMPTION 2860 /* 2861 * Calculate the next globally unique transaction for disambiguation 2862 * during cmpxchg. The transactions start with the cpu number and are then 2863 * incremented by CONFIG_NR_CPUS. 2864 */ 2865 #define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) 2866 #else 2867 /* 2868 * No preemption supported therefore also no need to check for 2869 * different cpus. 2870 */ 2871 #define TID_STEP 1 2872 #endif /* CONFIG_PREEMPTION */ 2873 2874 static inline unsigned long next_tid(unsigned long tid) 2875 { 2876 return tid + TID_STEP; 2877 } 2878 2879 #ifdef SLUB_DEBUG_CMPXCHG 2880 static inline unsigned int tid_to_cpu(unsigned long tid) 2881 { 2882 return tid % TID_STEP; 2883 } 2884 2885 static inline unsigned long tid_to_event(unsigned long tid) 2886 { 2887 return tid / TID_STEP; 2888 } 2889 #endif 2890 2891 static inline unsigned int init_tid(int cpu) 2892 { 2893 return cpu; 2894 } 2895 2896 static inline void note_cmpxchg_failure(const char *n, 2897 const struct kmem_cache *s, unsigned long tid) 2898 { 2899 #ifdef SLUB_DEBUG_CMPXCHG 2900 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); 2901 2902 pr_info("%s %s: cmpxchg redo ", n, s->name); 2903 2904 #ifdef CONFIG_PREEMPTION 2905 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid)) 2906 pr_warn("due to cpu change %d -> %d\n", 2907 tid_to_cpu(tid), tid_to_cpu(actual_tid)); 2908 else 2909 #endif 2910 if (tid_to_event(tid) != tid_to_event(actual_tid)) 2911 pr_warn("due to cpu running other code. Event %ld->%ld\n", 2912 tid_to_event(tid), tid_to_event(actual_tid)); 2913 else 2914 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", 2915 actual_tid, tid, next_tid(tid)); 2916 #endif 2917 stat(s, CMPXCHG_DOUBLE_CPU_FAIL); 2918 } 2919 2920 static void init_kmem_cache_cpus(struct kmem_cache *s) 2921 { 2922 int cpu; 2923 struct kmem_cache_cpu *c; 2924 2925 for_each_possible_cpu(cpu) { 2926 c = per_cpu_ptr(s->cpu_slab, cpu); 2927 local_lock_init(&c->lock); 2928 c->tid = init_tid(cpu); 2929 } 2930 } 2931 2932 /* 2933 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist, 2934 * unfreezes the slabs and puts it on the proper list. 2935 * Assumes the slab has been already safely taken away from kmem_cache_cpu 2936 * by the caller. 2937 */ 2938 static void deactivate_slab(struct kmem_cache *s, struct slab *slab, 2939 void *freelist) 2940 { 2941 struct kmem_cache_node *n = get_node(s, slab_nid(slab)); 2942 int free_delta = 0; 2943 void *nextfree, *freelist_iter, *freelist_tail; 2944 int tail = DEACTIVATE_TO_HEAD; 2945 unsigned long flags = 0; 2946 struct slab new; 2947 struct slab old; 2948 2949 if (READ_ONCE(slab->freelist)) { 2950 stat(s, DEACTIVATE_REMOTE_FREES); 2951 tail = DEACTIVATE_TO_TAIL; 2952 } 2953 2954 /* 2955 * Stage one: Count the objects on cpu's freelist as free_delta and 2956 * remember the last object in freelist_tail for later splicing. 2957 */ 2958 freelist_tail = NULL; 2959 freelist_iter = freelist; 2960 while (freelist_iter) { 2961 nextfree = get_freepointer(s, freelist_iter); 2962 2963 /* 2964 * If 'nextfree' is invalid, it is possible that the object at 2965 * 'freelist_iter' is already corrupted. So isolate all objects 2966 * starting at 'freelist_iter' by skipping them. 2967 */ 2968 if (freelist_corrupted(s, slab, &freelist_iter, nextfree)) 2969 break; 2970 2971 freelist_tail = freelist_iter; 2972 free_delta++; 2973 2974 freelist_iter = nextfree; 2975 } 2976 2977 /* 2978 * Stage two: Unfreeze the slab while splicing the per-cpu 2979 * freelist to the head of slab's freelist. 2980 */ 2981 do { 2982 old.freelist = READ_ONCE(slab->freelist); 2983 old.counters = READ_ONCE(slab->counters); 2984 VM_BUG_ON(!old.frozen); 2985 2986 /* Determine target state of the slab */ 2987 new.counters = old.counters; 2988 new.frozen = 0; 2989 if (freelist_tail) { 2990 new.inuse -= free_delta; 2991 set_freepointer(s, freelist_tail, old.freelist); 2992 new.freelist = freelist; 2993 } else { 2994 new.freelist = old.freelist; 2995 } 2996 } while (!slab_update_freelist(s, slab, 2997 old.freelist, old.counters, 2998 new.freelist, new.counters, 2999 "unfreezing slab")); 3000 3001 /* 3002 * Stage three: Manipulate the slab list based on the updated state. 3003 */ 3004 if (!new.inuse && n->nr_partial >= s->min_partial) { 3005 stat(s, DEACTIVATE_EMPTY); 3006 discard_slab(s, slab); 3007 stat(s, FREE_SLAB); 3008 } else if (new.freelist) { 3009 spin_lock_irqsave(&n->list_lock, flags); 3010 add_partial(n, slab, tail); 3011 spin_unlock_irqrestore(&n->list_lock, flags); 3012 stat(s, tail); 3013 } else { 3014 stat(s, DEACTIVATE_FULL); 3015 } 3016 } 3017 3018 #ifdef CONFIG_SLUB_CPU_PARTIAL 3019 static void __put_partials(struct kmem_cache *s, struct slab *partial_slab) 3020 { 3021 struct kmem_cache_node *n = NULL, *n2 = NULL; 3022 struct slab *slab, *slab_to_discard = NULL; 3023 unsigned long flags = 0; 3024 3025 while (partial_slab) { 3026 slab = partial_slab; 3027 partial_slab = slab->next; 3028 3029 n2 = get_node(s, slab_nid(slab)); 3030 if (n != n2) { 3031 if (n) 3032 spin_unlock_irqrestore(&n->list_lock, flags); 3033 3034 n = n2; 3035 spin_lock_irqsave(&n->list_lock, flags); 3036 } 3037 3038 if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) { 3039 slab->next = slab_to_discard; 3040 slab_to_discard = slab; 3041 } else { 3042 add_partial(n, slab, DEACTIVATE_TO_TAIL); 3043 stat(s, FREE_ADD_PARTIAL); 3044 } 3045 } 3046 3047 if (n) 3048 spin_unlock_irqrestore(&n->list_lock, flags); 3049 3050 while (slab_to_discard) { 3051 slab = slab_to_discard; 3052 slab_to_discard = slab_to_discard->next; 3053 3054 stat(s, DEACTIVATE_EMPTY); 3055 discard_slab(s, slab); 3056 stat(s, FREE_SLAB); 3057 } 3058 } 3059 3060 /* 3061 * Put all the cpu partial slabs to the node partial list. 3062 */ 3063 static void put_partials(struct kmem_cache *s) 3064 { 3065 struct slab *partial_slab; 3066 unsigned long flags; 3067 3068 local_lock_irqsave(&s->cpu_slab->lock, flags); 3069 partial_slab = this_cpu_read(s->cpu_slab->partial); 3070 this_cpu_write(s->cpu_slab->partial, NULL); 3071 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3072 3073 if (partial_slab) 3074 __put_partials(s, partial_slab); 3075 } 3076 3077 static void put_partials_cpu(struct kmem_cache *s, 3078 struct kmem_cache_cpu *c) 3079 { 3080 struct slab *partial_slab; 3081 3082 partial_slab = slub_percpu_partial(c); 3083 c->partial = NULL; 3084 3085 if (partial_slab) 3086 __put_partials(s, partial_slab); 3087 } 3088 3089 /* 3090 * Put a slab into a partial slab slot if available. 3091 * 3092 * If we did not find a slot then simply move all the partials to the 3093 * per node partial list. 3094 */ 3095 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) 3096 { 3097 struct slab *oldslab; 3098 struct slab *slab_to_put = NULL; 3099 unsigned long flags; 3100 int slabs = 0; 3101 3102 local_lock_irqsave(&s->cpu_slab->lock, flags); 3103 3104 oldslab = this_cpu_read(s->cpu_slab->partial); 3105 3106 if (oldslab) { 3107 if (drain && oldslab->slabs >= s->cpu_partial_slabs) { 3108 /* 3109 * Partial array is full. Move the existing set to the 3110 * per node partial list. Postpone the actual unfreezing 3111 * outside of the critical section. 3112 */ 3113 slab_to_put = oldslab; 3114 oldslab = NULL; 3115 } else { 3116 slabs = oldslab->slabs; 3117 } 3118 } 3119 3120 slabs++; 3121 3122 slab->slabs = slabs; 3123 slab->next = oldslab; 3124 3125 this_cpu_write(s->cpu_slab->partial, slab); 3126 3127 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3128 3129 if (slab_to_put) { 3130 __put_partials(s, slab_to_put); 3131 stat(s, CPU_PARTIAL_DRAIN); 3132 } 3133 } 3134 3135 #else /* CONFIG_SLUB_CPU_PARTIAL */ 3136 3137 static inline void put_partials(struct kmem_cache *s) { } 3138 static inline void put_partials_cpu(struct kmem_cache *s, 3139 struct kmem_cache_cpu *c) { } 3140 3141 #endif /* CONFIG_SLUB_CPU_PARTIAL */ 3142 3143 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 3144 { 3145 unsigned long flags; 3146 struct slab *slab; 3147 void *freelist; 3148 3149 local_lock_irqsave(&s->cpu_slab->lock, flags); 3150 3151 slab = c->slab; 3152 freelist = c->freelist; 3153 3154 c->slab = NULL; 3155 c->freelist = NULL; 3156 c->tid = next_tid(c->tid); 3157 3158 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3159 3160 if (slab) { 3161 deactivate_slab(s, slab, freelist); 3162 stat(s, CPUSLAB_FLUSH); 3163 } 3164 } 3165 3166 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) 3167 { 3168 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 3169 void *freelist = c->freelist; 3170 struct slab *slab = c->slab; 3171 3172 c->slab = NULL; 3173 c->freelist = NULL; 3174 c->tid = next_tid(c->tid); 3175 3176 if (slab) { 3177 deactivate_slab(s, slab, freelist); 3178 stat(s, CPUSLAB_FLUSH); 3179 } 3180 3181 put_partials_cpu(s, c); 3182 } 3183 3184 struct slub_flush_work { 3185 struct work_struct work; 3186 struct kmem_cache *s; 3187 bool skip; 3188 }; 3189 3190 /* 3191 * Flush cpu slab. 3192 * 3193 * Called from CPU work handler with migration disabled. 3194 */ 3195 static void flush_cpu_slab(struct work_struct *w) 3196 { 3197 struct kmem_cache *s; 3198 struct kmem_cache_cpu *c; 3199 struct slub_flush_work *sfw; 3200 3201 sfw = container_of(w, struct slub_flush_work, work); 3202 3203 s = sfw->s; 3204 c = this_cpu_ptr(s->cpu_slab); 3205 3206 if (c->slab) 3207 flush_slab(s, c); 3208 3209 put_partials(s); 3210 } 3211 3212 static bool has_cpu_slab(int cpu, struct kmem_cache *s) 3213 { 3214 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 3215 3216 return c->slab || slub_percpu_partial(c); 3217 } 3218 3219 static DEFINE_MUTEX(flush_lock); 3220 static DEFINE_PER_CPU(struct slub_flush_work, slub_flush); 3221 3222 static void flush_all_cpus_locked(struct kmem_cache *s) 3223 { 3224 struct slub_flush_work *sfw; 3225 unsigned int cpu; 3226 3227 lockdep_assert_cpus_held(); 3228 mutex_lock(&flush_lock); 3229 3230 for_each_online_cpu(cpu) { 3231 sfw = &per_cpu(slub_flush, cpu); 3232 if (!has_cpu_slab(cpu, s)) { 3233 sfw->skip = true; 3234 continue; 3235 } 3236 INIT_WORK(&sfw->work, flush_cpu_slab); 3237 sfw->skip = false; 3238 sfw->s = s; 3239 queue_work_on(cpu, flushwq, &sfw->work); 3240 } 3241 3242 for_each_online_cpu(cpu) { 3243 sfw = &per_cpu(slub_flush, cpu); 3244 if (sfw->skip) 3245 continue; 3246 flush_work(&sfw->work); 3247 } 3248 3249 mutex_unlock(&flush_lock); 3250 } 3251 3252 static void flush_all(struct kmem_cache *s) 3253 { 3254 cpus_read_lock(); 3255 flush_all_cpus_locked(s); 3256 cpus_read_unlock(); 3257 } 3258 3259 /* 3260 * Use the cpu notifier to insure that the cpu slabs are flushed when 3261 * necessary. 3262 */ 3263 static int slub_cpu_dead(unsigned int cpu) 3264 { 3265 struct kmem_cache *s; 3266 3267 mutex_lock(&slab_mutex); 3268 list_for_each_entry(s, &slab_caches, list) 3269 __flush_cpu_slab(s, cpu); 3270 mutex_unlock(&slab_mutex); 3271 return 0; 3272 } 3273 3274 #else /* CONFIG_SLUB_TINY */ 3275 static inline void flush_all_cpus_locked(struct kmem_cache *s) { } 3276 static inline void flush_all(struct kmem_cache *s) { } 3277 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { } 3278 static inline int slub_cpu_dead(unsigned int cpu) { return 0; } 3279 #endif /* CONFIG_SLUB_TINY */ 3280 3281 /* 3282 * Check if the objects in a per cpu structure fit numa 3283 * locality expectations. 3284 */ 3285 static inline int node_match(struct slab *slab, int node) 3286 { 3287 #ifdef CONFIG_NUMA 3288 if (node != NUMA_NO_NODE && slab_nid(slab) != node) 3289 return 0; 3290 #endif 3291 return 1; 3292 } 3293 3294 #ifdef CONFIG_SLUB_DEBUG 3295 static int count_free(struct slab *slab) 3296 { 3297 return slab->objects - slab->inuse; 3298 } 3299 3300 static inline unsigned long node_nr_objs(struct kmem_cache_node *n) 3301 { 3302 return atomic_long_read(&n->total_objects); 3303 } 3304 3305 /* Supports checking bulk free of a constructed freelist */ 3306 static inline bool free_debug_processing(struct kmem_cache *s, 3307 struct slab *slab, void *head, void *tail, int *bulk_cnt, 3308 unsigned long addr, depot_stack_handle_t handle) 3309 { 3310 bool checks_ok = false; 3311 void *object = head; 3312 int cnt = 0; 3313 3314 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 3315 if (!check_slab(s, slab)) 3316 goto out; 3317 } 3318 3319 if (slab->inuse < *bulk_cnt) { 3320 slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n", 3321 slab->inuse, *bulk_cnt); 3322 goto out; 3323 } 3324 3325 next_object: 3326 3327 if (++cnt > *bulk_cnt) 3328 goto out_cnt; 3329 3330 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 3331 if (!free_consistency_checks(s, slab, object, addr)) 3332 goto out; 3333 } 3334 3335 if (s->flags & SLAB_STORE_USER) 3336 set_track_update(s, object, TRACK_FREE, addr, handle); 3337 trace(s, slab, object, 0); 3338 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */ 3339 init_object(s, object, SLUB_RED_INACTIVE); 3340 3341 /* Reached end of constructed freelist yet? */ 3342 if (object != tail) { 3343 object = get_freepointer(s, object); 3344 goto next_object; 3345 } 3346 checks_ok = true; 3347 3348 out_cnt: 3349 if (cnt != *bulk_cnt) { 3350 slab_err(s, slab, "Bulk free expected %d objects but found %d\n", 3351 *bulk_cnt, cnt); 3352 *bulk_cnt = cnt; 3353 } 3354 3355 out: 3356 3357 if (!checks_ok) 3358 slab_fix(s, "Object at 0x%p not freed", object); 3359 3360 return checks_ok; 3361 } 3362 #endif /* CONFIG_SLUB_DEBUG */ 3363 3364 #if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS) 3365 static unsigned long count_partial(struct kmem_cache_node *n, 3366 int (*get_count)(struct slab *)) 3367 { 3368 unsigned long flags; 3369 unsigned long x = 0; 3370 struct slab *slab; 3371 3372 spin_lock_irqsave(&n->list_lock, flags); 3373 list_for_each_entry(slab, &n->partial, slab_list) 3374 x += get_count(slab); 3375 spin_unlock_irqrestore(&n->list_lock, flags); 3376 return x; 3377 } 3378 #endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */ 3379 3380 #ifdef CONFIG_SLUB_DEBUG 3381 #define MAX_PARTIAL_TO_SCAN 10000 3382 3383 static unsigned long count_partial_free_approx(struct kmem_cache_node *n) 3384 { 3385 unsigned long flags; 3386 unsigned long x = 0; 3387 struct slab *slab; 3388 3389 spin_lock_irqsave(&n->list_lock, flags); 3390 if (n->nr_partial <= MAX_PARTIAL_TO_SCAN) { 3391 list_for_each_entry(slab, &n->partial, slab_list) 3392 x += slab->objects - slab->inuse; 3393 } else { 3394 /* 3395 * For a long list, approximate the total count of objects in 3396 * it to meet the limit on the number of slabs to scan. 3397 * Scan from both the list's head and tail for better accuracy. 3398 */ 3399 unsigned long scanned = 0; 3400 3401 list_for_each_entry(slab, &n->partial, slab_list) { 3402 x += slab->objects - slab->inuse; 3403 if (++scanned == MAX_PARTIAL_TO_SCAN / 2) 3404 break; 3405 } 3406 list_for_each_entry_reverse(slab, &n->partial, slab_list) { 3407 x += slab->objects - slab->inuse; 3408 if (++scanned == MAX_PARTIAL_TO_SCAN) 3409 break; 3410 } 3411 x = mult_frac(x, n->nr_partial, scanned); 3412 x = min(x, node_nr_objs(n)); 3413 } 3414 spin_unlock_irqrestore(&n->list_lock, flags); 3415 return x; 3416 } 3417 3418 static noinline void 3419 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) 3420 { 3421 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, 3422 DEFAULT_RATELIMIT_BURST); 3423 int node; 3424 struct kmem_cache_node *n; 3425 3426 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) 3427 return; 3428 3429 pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n", 3430 nid, gfpflags, &gfpflags); 3431 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n", 3432 s->name, s->object_size, s->size, oo_order(s->oo), 3433 oo_order(s->min)); 3434 3435 if (oo_order(s->min) > get_order(s->object_size)) 3436 pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n", 3437 s->name); 3438 3439 for_each_kmem_cache_node(s, node, n) { 3440 unsigned long nr_slabs; 3441 unsigned long nr_objs; 3442 unsigned long nr_free; 3443 3444 nr_free = count_partial_free_approx(n); 3445 nr_slabs = node_nr_slabs(n); 3446 nr_objs = node_nr_objs(n); 3447 3448 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", 3449 node, nr_slabs, nr_objs, nr_free); 3450 } 3451 } 3452 #else /* CONFIG_SLUB_DEBUG */ 3453 static inline void 3454 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { } 3455 #endif 3456 3457 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) 3458 { 3459 if (unlikely(slab_test_pfmemalloc(slab))) 3460 return gfp_pfmemalloc_allowed(gfpflags); 3461 3462 return true; 3463 } 3464 3465 #ifndef CONFIG_SLUB_TINY 3466 static inline bool 3467 __update_cpu_freelist_fast(struct kmem_cache *s, 3468 void *freelist_old, void *freelist_new, 3469 unsigned long tid) 3470 { 3471 freelist_aba_t old = { .freelist = freelist_old, .counter = tid }; 3472 freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) }; 3473 3474 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full, 3475 &old.full, new.full); 3476 } 3477 3478 /* 3479 * Check the slab->freelist and either transfer the freelist to the 3480 * per cpu freelist or deactivate the slab. 3481 * 3482 * The slab is still frozen if the return value is not NULL. 3483 * 3484 * If this function returns NULL then the slab has been unfrozen. 3485 */ 3486 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) 3487 { 3488 struct slab new; 3489 unsigned long counters; 3490 void *freelist; 3491 3492 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); 3493 3494 do { 3495 freelist = slab->freelist; 3496 counters = slab->counters; 3497 3498 new.counters = counters; 3499 3500 new.inuse = slab->objects; 3501 new.frozen = freelist != NULL; 3502 3503 } while (!__slab_update_freelist(s, slab, 3504 freelist, counters, 3505 NULL, new.counters, 3506 "get_freelist")); 3507 3508 return freelist; 3509 } 3510 3511 /* 3512 * Freeze the partial slab and return the pointer to the freelist. 3513 */ 3514 static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab) 3515 { 3516 struct slab new; 3517 unsigned long counters; 3518 void *freelist; 3519 3520 do { 3521 freelist = slab->freelist; 3522 counters = slab->counters; 3523 3524 new.counters = counters; 3525 VM_BUG_ON(new.frozen); 3526 3527 new.inuse = slab->objects; 3528 new.frozen = 1; 3529 3530 } while (!slab_update_freelist(s, slab, 3531 freelist, counters, 3532 NULL, new.counters, 3533 "freeze_slab")); 3534 3535 return freelist; 3536 } 3537 3538 /* 3539 * Slow path. The lockless freelist is empty or we need to perform 3540 * debugging duties. 3541 * 3542 * Processing is still very fast if new objects have been freed to the 3543 * regular freelist. In that case we simply take over the regular freelist 3544 * as the lockless freelist and zap the regular freelist. 3545 * 3546 * If that is not working then we fall back to the partial lists. We take the 3547 * first element of the freelist as the object to allocate now and move the 3548 * rest of the freelist to the lockless freelist. 3549 * 3550 * And if we were unable to get a new slab from the partial slab lists then 3551 * we need to allocate a new slab. This is the slowest path since it involves 3552 * a call to the page allocator and the setup of a new slab. 3553 * 3554 * Version of __slab_alloc to use when we know that preemption is 3555 * already disabled (which is the case for bulk allocation). 3556 */ 3557 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 3558 unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) 3559 { 3560 void *freelist; 3561 struct slab *slab; 3562 unsigned long flags; 3563 struct partial_context pc; 3564 bool try_thisnode = true; 3565 3566 stat(s, ALLOC_SLOWPATH); 3567 3568 reread_slab: 3569 3570 slab = READ_ONCE(c->slab); 3571 if (!slab) { 3572 /* 3573 * if the node is not online or has no normal memory, just 3574 * ignore the node constraint 3575 */ 3576 if (unlikely(node != NUMA_NO_NODE && 3577 !node_isset(node, slab_nodes))) 3578 node = NUMA_NO_NODE; 3579 goto new_slab; 3580 } 3581 3582 if (unlikely(!node_match(slab, node))) { 3583 /* 3584 * same as above but node_match() being false already 3585 * implies node != NUMA_NO_NODE 3586 */ 3587 if (!node_isset(node, slab_nodes)) { 3588 node = NUMA_NO_NODE; 3589 } else { 3590 stat(s, ALLOC_NODE_MISMATCH); 3591 goto deactivate_slab; 3592 } 3593 } 3594 3595 /* 3596 * By rights, we should be searching for a slab page that was 3597 * PFMEMALLOC but right now, we are losing the pfmemalloc 3598 * information when the page leaves the per-cpu allocator 3599 */ 3600 if (unlikely(!pfmemalloc_match(slab, gfpflags))) 3601 goto deactivate_slab; 3602 3603 /* must check again c->slab in case we got preempted and it changed */ 3604 local_lock_irqsave(&s->cpu_slab->lock, flags); 3605 if (unlikely(slab != c->slab)) { 3606 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3607 goto reread_slab; 3608 } 3609 freelist = c->freelist; 3610 if (freelist) 3611 goto load_freelist; 3612 3613 freelist = get_freelist(s, slab); 3614 3615 if (!freelist) { 3616 c->slab = NULL; 3617 c->tid = next_tid(c->tid); 3618 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3619 stat(s, DEACTIVATE_BYPASS); 3620 goto new_slab; 3621 } 3622 3623 stat(s, ALLOC_REFILL); 3624 3625 load_freelist: 3626 3627 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); 3628 3629 /* 3630 * freelist is pointing to the list of objects to be used. 3631 * slab is pointing to the slab from which the objects are obtained. 3632 * That slab must be frozen for per cpu allocations to work. 3633 */ 3634 VM_BUG_ON(!c->slab->frozen); 3635 c->freelist = get_freepointer(s, freelist); 3636 c->tid = next_tid(c->tid); 3637 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3638 return freelist; 3639 3640 deactivate_slab: 3641 3642 local_lock_irqsave(&s->cpu_slab->lock, flags); 3643 if (slab != c->slab) { 3644 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3645 goto reread_slab; 3646 } 3647 freelist = c->freelist; 3648 c->slab = NULL; 3649 c->freelist = NULL; 3650 c->tid = next_tid(c->tid); 3651 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3652 deactivate_slab(s, slab, freelist); 3653 3654 new_slab: 3655 3656 #ifdef CONFIG_SLUB_CPU_PARTIAL 3657 while (slub_percpu_partial(c)) { 3658 local_lock_irqsave(&s->cpu_slab->lock, flags); 3659 if (unlikely(c->slab)) { 3660 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3661 goto reread_slab; 3662 } 3663 if (unlikely(!slub_percpu_partial(c))) { 3664 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3665 /* we were preempted and partial list got empty */ 3666 goto new_objects; 3667 } 3668 3669 slab = slub_percpu_partial(c); 3670 slub_set_percpu_partial(c, slab); 3671 3672 if (likely(node_match(slab, node) && 3673 pfmemalloc_match(slab, gfpflags))) { 3674 c->slab = slab; 3675 freelist = get_freelist(s, slab); 3676 VM_BUG_ON(!freelist); 3677 stat(s, CPU_PARTIAL_ALLOC); 3678 goto load_freelist; 3679 } 3680 3681 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3682 3683 slab->next = NULL; 3684 __put_partials(s, slab); 3685 } 3686 #endif 3687 3688 new_objects: 3689 3690 pc.flags = gfpflags; 3691 /* 3692 * When a preferred node is indicated but no __GFP_THISNODE 3693 * 3694 * 1) try to get a partial slab from target node only by having 3695 * __GFP_THISNODE in pc.flags for get_partial() 3696 * 2) if 1) failed, try to allocate a new slab from target node with 3697 * GPF_NOWAIT | __GFP_THISNODE opportunistically 3698 * 3) if 2) failed, retry with original gfpflags which will allow 3699 * get_partial() try partial lists of other nodes before potentially 3700 * allocating new page from other nodes 3701 */ 3702 if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) 3703 && try_thisnode)) 3704 pc.flags = GFP_NOWAIT | __GFP_THISNODE; 3705 3706 pc.orig_size = orig_size; 3707 slab = get_partial(s, node, &pc); 3708 if (slab) { 3709 if (kmem_cache_debug(s)) { 3710 freelist = pc.object; 3711 /* 3712 * For debug caches here we had to go through 3713 * alloc_single_from_partial() so just store the 3714 * tracking info and return the object. 3715 */ 3716 if (s->flags & SLAB_STORE_USER) 3717 set_track(s, freelist, TRACK_ALLOC, addr); 3718 3719 return freelist; 3720 } 3721 3722 freelist = freeze_slab(s, slab); 3723 goto retry_load_slab; 3724 } 3725 3726 slub_put_cpu_ptr(s->cpu_slab); 3727 slab = new_slab(s, pc.flags, node); 3728 c = slub_get_cpu_ptr(s->cpu_slab); 3729 3730 if (unlikely(!slab)) { 3731 if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) 3732 && try_thisnode) { 3733 try_thisnode = false; 3734 goto new_objects; 3735 } 3736 slab_out_of_memory(s, gfpflags, node); 3737 return NULL; 3738 } 3739 3740 stat(s, ALLOC_SLAB); 3741 3742 if (kmem_cache_debug(s)) { 3743 freelist = alloc_single_from_new_slab(s, slab, orig_size); 3744 3745 if (unlikely(!freelist)) 3746 goto new_objects; 3747 3748 if (s->flags & SLAB_STORE_USER) 3749 set_track(s, freelist, TRACK_ALLOC, addr); 3750 3751 return freelist; 3752 } 3753 3754 /* 3755 * No other reference to the slab yet so we can 3756 * muck around with it freely without cmpxchg 3757 */ 3758 freelist = slab->freelist; 3759 slab->freelist = NULL; 3760 slab->inuse = slab->objects; 3761 slab->frozen = 1; 3762 3763 inc_slabs_node(s, slab_nid(slab), slab->objects); 3764 3765 if (unlikely(!pfmemalloc_match(slab, gfpflags))) { 3766 /* 3767 * For !pfmemalloc_match() case we don't load freelist so that 3768 * we don't make further mismatched allocations easier. 3769 */ 3770 deactivate_slab(s, slab, get_freepointer(s, freelist)); 3771 return freelist; 3772 } 3773 3774 retry_load_slab: 3775 3776 local_lock_irqsave(&s->cpu_slab->lock, flags); 3777 if (unlikely(c->slab)) { 3778 void *flush_freelist = c->freelist; 3779 struct slab *flush_slab = c->slab; 3780 3781 c->slab = NULL; 3782 c->freelist = NULL; 3783 c->tid = next_tid(c->tid); 3784 3785 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3786 3787 deactivate_slab(s, flush_slab, flush_freelist); 3788 3789 stat(s, CPUSLAB_FLUSH); 3790 3791 goto retry_load_slab; 3792 } 3793 c->slab = slab; 3794 3795 goto load_freelist; 3796 } 3797 3798 /* 3799 * A wrapper for ___slab_alloc() for contexts where preemption is not yet 3800 * disabled. Compensates for possible cpu changes by refetching the per cpu area 3801 * pointer. 3802 */ 3803 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 3804 unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) 3805 { 3806 void *p; 3807 3808 #ifdef CONFIG_PREEMPT_COUNT 3809 /* 3810 * We may have been preempted and rescheduled on a different 3811 * cpu before disabling preemption. Need to reload cpu area 3812 * pointer. 3813 */ 3814 c = slub_get_cpu_ptr(s->cpu_slab); 3815 #endif 3816 3817 p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size); 3818 #ifdef CONFIG_PREEMPT_COUNT 3819 slub_put_cpu_ptr(s->cpu_slab); 3820 #endif 3821 return p; 3822 } 3823 3824 static __always_inline void *__slab_alloc_node(struct kmem_cache *s, 3825 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 3826 { 3827 struct kmem_cache_cpu *c; 3828 struct slab *slab; 3829 unsigned long tid; 3830 void *object; 3831 3832 redo: 3833 /* 3834 * Must read kmem_cache cpu data via this cpu ptr. Preemption is 3835 * enabled. We may switch back and forth between cpus while 3836 * reading from one cpu area. That does not matter as long 3837 * as we end up on the original cpu again when doing the cmpxchg. 3838 * 3839 * We must guarantee that tid and kmem_cache_cpu are retrieved on the 3840 * same cpu. We read first the kmem_cache_cpu pointer and use it to read 3841 * the tid. If we are preempted and switched to another cpu between the 3842 * two reads, it's OK as the two are still associated with the same cpu 3843 * and cmpxchg later will validate the cpu. 3844 */ 3845 c = raw_cpu_ptr(s->cpu_slab); 3846 tid = READ_ONCE(c->tid); 3847 3848 /* 3849 * Irqless object alloc/free algorithm used here depends on sequence 3850 * of fetching cpu_slab's data. tid should be fetched before anything 3851 * on c to guarantee that object and slab associated with previous tid 3852 * won't be used with current tid. If we fetch tid first, object and 3853 * slab could be one associated with next tid and our alloc/free 3854 * request will be failed. In this case, we will retry. So, no problem. 3855 */ 3856 barrier(); 3857 3858 /* 3859 * The transaction ids are globally unique per cpu and per operation on 3860 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double 3861 * occurs on the right processor and that there was no operation on the 3862 * linked list in between. 3863 */ 3864 3865 object = c->freelist; 3866 slab = c->slab; 3867 3868 if (!USE_LOCKLESS_FAST_PATH() || 3869 unlikely(!object || !slab || !node_match(slab, node))) { 3870 object = __slab_alloc(s, gfpflags, node, addr, c, orig_size); 3871 } else { 3872 void *next_object = get_freepointer_safe(s, object); 3873 3874 /* 3875 * The cmpxchg will only match if there was no additional 3876 * operation and if we are on the right processor. 3877 * 3878 * The cmpxchg does the following atomically (without lock 3879 * semantics!) 3880 * 1. Relocate first pointer to the current per cpu area. 3881 * 2. Verify that tid and freelist have not been changed 3882 * 3. If they were not changed replace tid and freelist 3883 * 3884 * Since this is without lock semantics the protection is only 3885 * against code executing on this cpu *not* from access by 3886 * other cpus. 3887 */ 3888 if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) { 3889 note_cmpxchg_failure("slab_alloc", s, tid); 3890 goto redo; 3891 } 3892 prefetch_freepointer(s, next_object); 3893 stat(s, ALLOC_FASTPATH); 3894 } 3895 3896 return object; 3897 } 3898 #else /* CONFIG_SLUB_TINY */ 3899 static void *__slab_alloc_node(struct kmem_cache *s, 3900 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 3901 { 3902 struct partial_context pc; 3903 struct slab *slab; 3904 void *object; 3905 3906 pc.flags = gfpflags; 3907 pc.orig_size = orig_size; 3908 slab = get_partial(s, node, &pc); 3909 3910 if (slab) 3911 return pc.object; 3912 3913 slab = new_slab(s, gfpflags, node); 3914 if (unlikely(!slab)) { 3915 slab_out_of_memory(s, gfpflags, node); 3916 return NULL; 3917 } 3918 3919 object = alloc_single_from_new_slab(s, slab, orig_size); 3920 3921 return object; 3922 } 3923 #endif /* CONFIG_SLUB_TINY */ 3924 3925 /* 3926 * If the object has been wiped upon free, make sure it's fully initialized by 3927 * zeroing out freelist pointer. 3928 */ 3929 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, 3930 void *obj) 3931 { 3932 if (unlikely(slab_want_init_on_free(s)) && obj && 3933 !freeptr_outside_object(s)) 3934 memset((void *)((char *)kasan_reset_tag(obj) + s->offset), 3935 0, sizeof(void *)); 3936 } 3937 3938 static __fastpath_inline 3939 struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) 3940 { 3941 flags &= gfp_allowed_mask; 3942 3943 might_alloc(flags); 3944 3945 if (unlikely(should_failslab(s, flags))) 3946 return NULL; 3947 3948 return s; 3949 } 3950 3951 static __fastpath_inline 3952 bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 3953 gfp_t flags, size_t size, void **p, bool init, 3954 unsigned int orig_size) 3955 { 3956 unsigned int zero_size = s->object_size; 3957 bool kasan_init = init; 3958 size_t i; 3959 gfp_t init_flags = flags & gfp_allowed_mask; 3960 3961 /* 3962 * For kmalloc object, the allocated memory size(object_size) is likely 3963 * larger than the requested size(orig_size). If redzone check is 3964 * enabled for the extra space, don't zero it, as it will be redzoned 3965 * soon. The redzone operation for this extra space could be seen as a 3966 * replacement of current poisoning under certain debug option, and 3967 * won't break other sanity checks. 3968 */ 3969 if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) && 3970 (s->flags & SLAB_KMALLOC)) 3971 zero_size = orig_size; 3972 3973 /* 3974 * When slab_debug is enabled, avoid memory initialization integrated 3975 * into KASAN and instead zero out the memory via the memset below with 3976 * the proper size. Otherwise, KASAN might overwrite SLUB redzones and 3977 * cause false-positive reports. This does not lead to a performance 3978 * penalty on production builds, as slab_debug is not intended to be 3979 * enabled there. 3980 */ 3981 if (__slub_debug_enabled()) 3982 kasan_init = false; 3983 3984 /* 3985 * As memory initialization might be integrated into KASAN, 3986 * kasan_slab_alloc and initialization memset must be 3987 * kept together to avoid discrepancies in behavior. 3988 * 3989 * As p[i] might get tagged, memset and kmemleak hook come after KASAN. 3990 */ 3991 for (i = 0; i < size; i++) { 3992 p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init); 3993 if (p[i] && init && (!kasan_init || 3994 !kasan_has_integrated_init())) 3995 memset(p[i], 0, zero_size); 3996 kmemleak_alloc_recursive(p[i], s->object_size, 1, 3997 s->flags, init_flags); 3998 kmsan_slab_alloc(s, p[i], init_flags); 3999 alloc_tagging_slab_alloc_hook(s, p[i], flags); 4000 } 4001 4002 return memcg_slab_post_alloc_hook(s, lru, flags, size, p); 4003 } 4004 4005 /* 4006 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) 4007 * have the fastpath folded into their functions. So no function call 4008 * overhead for requests that can be satisfied on the fastpath. 4009 * 4010 * The fastpath works by first checking if the lockless freelist can be used. 4011 * If not then __slab_alloc is called for slow processing. 4012 * 4013 * Otherwise we can simply pick the next object from the lockless free list. 4014 */ 4015 static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru, 4016 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 4017 { 4018 void *object; 4019 bool init = false; 4020 4021 s = slab_pre_alloc_hook(s, gfpflags); 4022 if (unlikely(!s)) 4023 return NULL; 4024 4025 object = kfence_alloc(s, orig_size, gfpflags); 4026 if (unlikely(object)) 4027 goto out; 4028 4029 object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); 4030 4031 maybe_wipe_obj_freeptr(s, object); 4032 init = slab_want_init_on_alloc(gfpflags, s); 4033 4034 out: 4035 /* 4036 * When init equals 'true', like for kzalloc() family, only 4037 * @orig_size bytes might be zeroed instead of s->object_size 4038 * In case this fails due to memcg_slab_post_alloc_hook(), 4039 * object is set to NULL 4040 */ 4041 slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size); 4042 4043 return object; 4044 } 4045 4046 void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) 4047 { 4048 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, 4049 s->object_size); 4050 4051 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); 4052 4053 return ret; 4054 } 4055 EXPORT_SYMBOL(kmem_cache_alloc_noprof); 4056 4057 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, 4058 gfp_t gfpflags) 4059 { 4060 void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, 4061 s->object_size); 4062 4063 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); 4064 4065 return ret; 4066 } 4067 EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof); 4068 4069 /** 4070 * kmem_cache_alloc_node - Allocate an object on the specified node 4071 * @s: The cache to allocate from. 4072 * @gfpflags: See kmalloc(). 4073 * @node: node number of the target node. 4074 * 4075 * Identical to kmem_cache_alloc but it will allocate memory on the given 4076 * node, which can improve the performance for cpu bound structures. 4077 * 4078 * Fallback to other node is possible if __GFP_THISNODE is not set. 4079 * 4080 * Return: pointer to the new object or %NULL in case of error 4081 */ 4082 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) 4083 { 4084 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); 4085 4086 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node); 4087 4088 return ret; 4089 } 4090 EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); 4091 4092 /* 4093 * To avoid unnecessary overhead, we pass through large allocation requests 4094 * directly to the page allocator. We use __GFP_COMP, because we will need to 4095 * know the allocation order to free the pages properly in kfree. 4096 */ 4097 static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) 4098 { 4099 struct folio *folio; 4100 void *ptr = NULL; 4101 unsigned int order = get_order(size); 4102 4103 if (unlikely(flags & GFP_SLAB_BUG_MASK)) 4104 flags = kmalloc_fix_flags(flags); 4105 4106 flags |= __GFP_COMP; 4107 folio = (struct folio *)alloc_pages_node_noprof(node, flags, order); 4108 if (folio) { 4109 ptr = folio_address(folio); 4110 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 4111 PAGE_SIZE << order); 4112 } 4113 4114 ptr = kasan_kmalloc_large(ptr, size, flags); 4115 /* As ptr might get tagged, call kmemleak hook after KASAN. */ 4116 kmemleak_alloc(ptr, size, 1, flags); 4117 kmsan_kmalloc_large(ptr, size, flags); 4118 4119 return ptr; 4120 } 4121 4122 void *__kmalloc_large_noprof(size_t size, gfp_t flags) 4123 { 4124 void *ret = ___kmalloc_large_node(size, flags, NUMA_NO_NODE); 4125 4126 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 4127 flags, NUMA_NO_NODE); 4128 return ret; 4129 } 4130 EXPORT_SYMBOL(__kmalloc_large_noprof); 4131 4132 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 4133 { 4134 void *ret = ___kmalloc_large_node(size, flags, node); 4135 4136 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 4137 flags, node); 4138 return ret; 4139 } 4140 EXPORT_SYMBOL(__kmalloc_large_node_noprof); 4141 4142 static __always_inline 4143 void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, 4144 unsigned long caller) 4145 { 4146 struct kmem_cache *s; 4147 void *ret; 4148 4149 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 4150 ret = __kmalloc_large_node_noprof(size, flags, node); 4151 trace_kmalloc(caller, ret, size, 4152 PAGE_SIZE << get_order(size), flags, node); 4153 return ret; 4154 } 4155 4156 if (unlikely(!size)) 4157 return ZERO_SIZE_PTR; 4158 4159 s = kmalloc_slab(size, b, flags, caller); 4160 4161 ret = slab_alloc_node(s, NULL, flags, node, caller, size); 4162 ret = kasan_kmalloc(s, ret, size, flags); 4163 trace_kmalloc(caller, ret, size, s->size, flags, node); 4164 return ret; 4165 } 4166 void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) 4167 { 4168 return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, _RET_IP_); 4169 } 4170 EXPORT_SYMBOL(__kmalloc_node_noprof); 4171 4172 void *__kmalloc_noprof(size_t size, gfp_t flags) 4173 { 4174 return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_); 4175 } 4176 EXPORT_SYMBOL(__kmalloc_noprof); 4177 4178 void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, 4179 int node, unsigned long caller) 4180 { 4181 return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, caller); 4182 4183 } 4184 EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof); 4185 4186 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) 4187 { 4188 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, 4189 _RET_IP_, size); 4190 4191 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); 4192 4193 ret = kasan_kmalloc(s, ret, size, gfpflags); 4194 return ret; 4195 } 4196 EXPORT_SYMBOL(__kmalloc_cache_noprof); 4197 4198 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 4199 int node, size_t size) 4200 { 4201 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); 4202 4203 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); 4204 4205 ret = kasan_kmalloc(s, ret, size, gfpflags); 4206 return ret; 4207 } 4208 EXPORT_SYMBOL(__kmalloc_cache_node_noprof); 4209 4210 static noinline void free_to_partial_list( 4211 struct kmem_cache *s, struct slab *slab, 4212 void *head, void *tail, int bulk_cnt, 4213 unsigned long addr) 4214 { 4215 struct kmem_cache_node *n = get_node(s, slab_nid(slab)); 4216 struct slab *slab_free = NULL; 4217 int cnt = bulk_cnt; 4218 unsigned long flags; 4219 depot_stack_handle_t handle = 0; 4220 4221 if (s->flags & SLAB_STORE_USER) 4222 handle = set_track_prepare(); 4223 4224 spin_lock_irqsave(&n->list_lock, flags); 4225 4226 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) { 4227 void *prior = slab->freelist; 4228 4229 /* Perform the actual freeing while we still hold the locks */ 4230 slab->inuse -= cnt; 4231 set_freepointer(s, tail, prior); 4232 slab->freelist = head; 4233 4234 /* 4235 * If the slab is empty, and node's partial list is full, 4236 * it should be discarded anyway no matter it's on full or 4237 * partial list. 4238 */ 4239 if (slab->inuse == 0 && n->nr_partial >= s->min_partial) 4240 slab_free = slab; 4241 4242 if (!prior) { 4243 /* was on full list */ 4244 remove_full(s, n, slab); 4245 if (!slab_free) { 4246 add_partial(n, slab, DEACTIVATE_TO_TAIL); 4247 stat(s, FREE_ADD_PARTIAL); 4248 } 4249 } else if (slab_free) { 4250 remove_partial(n, slab); 4251 stat(s, FREE_REMOVE_PARTIAL); 4252 } 4253 } 4254 4255 if (slab_free) { 4256 /* 4257 * Update the counters while still holding n->list_lock to 4258 * prevent spurious validation warnings 4259 */ 4260 dec_slabs_node(s, slab_nid(slab_free), slab_free->objects); 4261 } 4262 4263 spin_unlock_irqrestore(&n->list_lock, flags); 4264 4265 if (slab_free) { 4266 stat(s, FREE_SLAB); 4267 free_slab(s, slab_free); 4268 } 4269 } 4270 4271 /* 4272 * Slow path handling. This may still be called frequently since objects 4273 * have a longer lifetime than the cpu slabs in most processing loads. 4274 * 4275 * So we still attempt to reduce cache line usage. Just take the slab 4276 * lock and free the item. If there is no additional partial slab 4277 * handling required then we can return immediately. 4278 */ 4279 static void __slab_free(struct kmem_cache *s, struct slab *slab, 4280 void *head, void *tail, int cnt, 4281 unsigned long addr) 4282 4283 { 4284 void *prior; 4285 int was_frozen; 4286 struct slab new; 4287 unsigned long counters; 4288 struct kmem_cache_node *n = NULL; 4289 unsigned long flags; 4290 bool on_node_partial; 4291 4292 stat(s, FREE_SLOWPATH); 4293 4294 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4295 free_to_partial_list(s, slab, head, tail, cnt, addr); 4296 return; 4297 } 4298 4299 do { 4300 if (unlikely(n)) { 4301 spin_unlock_irqrestore(&n->list_lock, flags); 4302 n = NULL; 4303 } 4304 prior = slab->freelist; 4305 counters = slab->counters; 4306 set_freepointer(s, tail, prior); 4307 new.counters = counters; 4308 was_frozen = new.frozen; 4309 new.inuse -= cnt; 4310 if ((!new.inuse || !prior) && !was_frozen) { 4311 /* Needs to be taken off a list */ 4312 if (!kmem_cache_has_cpu_partial(s) || prior) { 4313 4314 n = get_node(s, slab_nid(slab)); 4315 /* 4316 * Speculatively acquire the list_lock. 4317 * If the cmpxchg does not succeed then we may 4318 * drop the list_lock without any processing. 4319 * 4320 * Otherwise the list_lock will synchronize with 4321 * other processors updating the list of slabs. 4322 */ 4323 spin_lock_irqsave(&n->list_lock, flags); 4324 4325 on_node_partial = slab_test_node_partial(slab); 4326 } 4327 } 4328 4329 } while (!slab_update_freelist(s, slab, 4330 prior, counters, 4331 head, new.counters, 4332 "__slab_free")); 4333 4334 if (likely(!n)) { 4335 4336 if (likely(was_frozen)) { 4337 /* 4338 * The list lock was not taken therefore no list 4339 * activity can be necessary. 4340 */ 4341 stat(s, FREE_FROZEN); 4342 } else if (kmem_cache_has_cpu_partial(s) && !prior) { 4343 /* 4344 * If we started with a full slab then put it onto the 4345 * per cpu partial list. 4346 */ 4347 put_cpu_partial(s, slab, 1); 4348 stat(s, CPU_PARTIAL_FREE); 4349 } 4350 4351 return; 4352 } 4353 4354 /* 4355 * This slab was partially empty but not on the per-node partial list, 4356 * in which case we shouldn't manipulate its list, just return. 4357 */ 4358 if (prior && !on_node_partial) { 4359 spin_unlock_irqrestore(&n->list_lock, flags); 4360 return; 4361 } 4362 4363 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 4364 goto slab_empty; 4365 4366 /* 4367 * Objects left in the slab. If it was not on the partial list before 4368 * then add it. 4369 */ 4370 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 4371 add_partial(n, slab, DEACTIVATE_TO_TAIL); 4372 stat(s, FREE_ADD_PARTIAL); 4373 } 4374 spin_unlock_irqrestore(&n->list_lock, flags); 4375 return; 4376 4377 slab_empty: 4378 if (prior) { 4379 /* 4380 * Slab on the partial list. 4381 */ 4382 remove_partial(n, slab); 4383 stat(s, FREE_REMOVE_PARTIAL); 4384 } 4385 4386 spin_unlock_irqrestore(&n->list_lock, flags); 4387 stat(s, FREE_SLAB); 4388 discard_slab(s, slab); 4389 } 4390 4391 #ifndef CONFIG_SLUB_TINY 4392 /* 4393 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 4394 * can perform fastpath freeing without additional function calls. 4395 * 4396 * The fastpath is only possible if we are freeing to the current cpu slab 4397 * of this processor. This typically the case if we have just allocated 4398 * the item before. 4399 * 4400 * If fastpath is not possible then fall back to __slab_free where we deal 4401 * with all sorts of special processing. 4402 * 4403 * Bulk free of a freelist with several objects (all pointing to the 4404 * same slab) possible by specifying head and tail ptr, plus objects 4405 * count (cnt). Bulk free indicated by tail pointer being set. 4406 */ 4407 static __always_inline void do_slab_free(struct kmem_cache *s, 4408 struct slab *slab, void *head, void *tail, 4409 int cnt, unsigned long addr) 4410 { 4411 struct kmem_cache_cpu *c; 4412 unsigned long tid; 4413 void **freelist; 4414 4415 redo: 4416 /* 4417 * Determine the currently cpus per cpu slab. 4418 * The cpu may change afterward. However that does not matter since 4419 * data is retrieved via this pointer. If we are on the same cpu 4420 * during the cmpxchg then the free will succeed. 4421 */ 4422 c = raw_cpu_ptr(s->cpu_slab); 4423 tid = READ_ONCE(c->tid); 4424 4425 /* Same with comment on barrier() in __slab_alloc_node() */ 4426 barrier(); 4427 4428 if (unlikely(slab != c->slab)) { 4429 __slab_free(s, slab, head, tail, cnt, addr); 4430 return; 4431 } 4432 4433 if (USE_LOCKLESS_FAST_PATH()) { 4434 freelist = READ_ONCE(c->freelist); 4435 4436 set_freepointer(s, tail, freelist); 4437 4438 if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) { 4439 note_cmpxchg_failure("slab_free", s, tid); 4440 goto redo; 4441 } 4442 } else { 4443 /* Update the free list under the local lock */ 4444 local_lock(&s->cpu_slab->lock); 4445 c = this_cpu_ptr(s->cpu_slab); 4446 if (unlikely(slab != c->slab)) { 4447 local_unlock(&s->cpu_slab->lock); 4448 goto redo; 4449 } 4450 tid = c->tid; 4451 freelist = c->freelist; 4452 4453 set_freepointer(s, tail, freelist); 4454 c->freelist = head; 4455 c->tid = next_tid(tid); 4456 4457 local_unlock(&s->cpu_slab->lock); 4458 } 4459 stat_add(s, FREE_FASTPATH, cnt); 4460 } 4461 #else /* CONFIG_SLUB_TINY */ 4462 static void do_slab_free(struct kmem_cache *s, 4463 struct slab *slab, void *head, void *tail, 4464 int cnt, unsigned long addr) 4465 { 4466 __slab_free(s, slab, head, tail, cnt, addr); 4467 } 4468 #endif /* CONFIG_SLUB_TINY */ 4469 4470 static __fastpath_inline 4471 void slab_free(struct kmem_cache *s, struct slab *slab, void *object, 4472 unsigned long addr) 4473 { 4474 memcg_slab_free_hook(s, slab, &object, 1); 4475 alloc_tagging_slab_free_hook(s, slab, &object, 1); 4476 4477 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s)))) 4478 do_slab_free(s, slab, object, object, 1, addr); 4479 } 4480 4481 #ifdef CONFIG_MEMCG 4482 /* Do not inline the rare memcg charging failed path into the allocation path */ 4483 static noinline 4484 void memcg_alloc_abort_single(struct kmem_cache *s, void *object) 4485 { 4486 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s)))) 4487 do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_); 4488 } 4489 #endif 4490 4491 static __fastpath_inline 4492 void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head, 4493 void *tail, void **p, int cnt, unsigned long addr) 4494 { 4495 memcg_slab_free_hook(s, slab, p, cnt); 4496 alloc_tagging_slab_free_hook(s, slab, p, cnt); 4497 /* 4498 * With KASAN enabled slab_free_freelist_hook modifies the freelist 4499 * to remove objects, whose reuse must be delayed. 4500 */ 4501 if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) 4502 do_slab_free(s, slab, head, tail, cnt, addr); 4503 } 4504 4505 #ifdef CONFIG_KASAN_GENERIC 4506 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) 4507 { 4508 do_slab_free(cache, virt_to_slab(x), x, x, 1, addr); 4509 } 4510 #endif 4511 4512 static inline struct kmem_cache *virt_to_cache(const void *obj) 4513 { 4514 struct slab *slab; 4515 4516 slab = virt_to_slab(obj); 4517 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__)) 4518 return NULL; 4519 return slab->slab_cache; 4520 } 4521 4522 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 4523 { 4524 struct kmem_cache *cachep; 4525 4526 if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && 4527 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) 4528 return s; 4529 4530 cachep = virt_to_cache(x); 4531 if (WARN(cachep && cachep != s, 4532 "%s: Wrong slab cache. %s but object is from %s\n", 4533 __func__, s->name, cachep->name)) 4534 print_tracking(cachep, x); 4535 return cachep; 4536 } 4537 4538 /** 4539 * kmem_cache_free - Deallocate an object 4540 * @s: The cache the allocation was from. 4541 * @x: The previously allocated object. 4542 * 4543 * Free an object which was previously allocated from this 4544 * cache. 4545 */ 4546 void kmem_cache_free(struct kmem_cache *s, void *x) 4547 { 4548 s = cache_from_obj(s, x); 4549 if (!s) 4550 return; 4551 trace_kmem_cache_free(_RET_IP_, x, s); 4552 slab_free(s, virt_to_slab(x), x, _RET_IP_); 4553 } 4554 EXPORT_SYMBOL(kmem_cache_free); 4555 4556 static void free_large_kmalloc(struct folio *folio, void *object) 4557 { 4558 unsigned int order = folio_order(folio); 4559 4560 if (WARN_ON_ONCE(order == 0)) 4561 pr_warn_once("object pointer: 0x%p\n", object); 4562 4563 kmemleak_free(object); 4564 kasan_kfree_large(object); 4565 kmsan_kfree_large(object); 4566 4567 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 4568 -(PAGE_SIZE << order)); 4569 folio_put(folio); 4570 } 4571 4572 /** 4573 * kfree - free previously allocated memory 4574 * @object: pointer returned by kmalloc() or kmem_cache_alloc() 4575 * 4576 * If @object is NULL, no operation is performed. 4577 */ 4578 void kfree(const void *object) 4579 { 4580 struct folio *folio; 4581 struct slab *slab; 4582 struct kmem_cache *s; 4583 void *x = (void *)object; 4584 4585 trace_kfree(_RET_IP_, object); 4586 4587 if (unlikely(ZERO_OR_NULL_PTR(object))) 4588 return; 4589 4590 folio = virt_to_folio(object); 4591 if (unlikely(!folio_test_slab(folio))) { 4592 free_large_kmalloc(folio, (void *)object); 4593 return; 4594 } 4595 4596 slab = folio_slab(folio); 4597 s = slab->slab_cache; 4598 slab_free(s, slab, x, _RET_IP_); 4599 } 4600 EXPORT_SYMBOL(kfree); 4601 4602 struct detached_freelist { 4603 struct slab *slab; 4604 void *tail; 4605 void *freelist; 4606 int cnt; 4607 struct kmem_cache *s; 4608 }; 4609 4610 /* 4611 * This function progressively scans the array with free objects (with 4612 * a limited look ahead) and extract objects belonging to the same 4613 * slab. It builds a detached freelist directly within the given 4614 * slab/objects. This can happen without any need for 4615 * synchronization, because the objects are owned by running process. 4616 * The freelist is build up as a single linked list in the objects. 4617 * The idea is, that this detached freelist can then be bulk 4618 * transferred to the real freelist(s), but only requiring a single 4619 * synchronization primitive. Look ahead in the array is limited due 4620 * to performance reasons. 4621 */ 4622 static inline 4623 int build_detached_freelist(struct kmem_cache *s, size_t size, 4624 void **p, struct detached_freelist *df) 4625 { 4626 int lookahead = 3; 4627 void *object; 4628 struct folio *folio; 4629 size_t same; 4630 4631 object = p[--size]; 4632 folio = virt_to_folio(object); 4633 if (!s) { 4634 /* Handle kalloc'ed objects */ 4635 if (unlikely(!folio_test_slab(folio))) { 4636 free_large_kmalloc(folio, object); 4637 df->slab = NULL; 4638 return size; 4639 } 4640 /* Derive kmem_cache from object */ 4641 df->slab = folio_slab(folio); 4642 df->s = df->slab->slab_cache; 4643 } else { 4644 df->slab = folio_slab(folio); 4645 df->s = cache_from_obj(s, object); /* Support for memcg */ 4646 } 4647 4648 /* Start new detached freelist */ 4649 df->tail = object; 4650 df->freelist = object; 4651 df->cnt = 1; 4652 4653 if (is_kfence_address(object)) 4654 return size; 4655 4656 set_freepointer(df->s, object, NULL); 4657 4658 same = size; 4659 while (size) { 4660 object = p[--size]; 4661 /* df->slab is always set at this point */ 4662 if (df->slab == virt_to_slab(object)) { 4663 /* Opportunity build freelist */ 4664 set_freepointer(df->s, object, df->freelist); 4665 df->freelist = object; 4666 df->cnt++; 4667 same--; 4668 if (size != same) 4669 swap(p[size], p[same]); 4670 continue; 4671 } 4672 4673 /* Limit look ahead search */ 4674 if (!--lookahead) 4675 break; 4676 } 4677 4678 return same; 4679 } 4680 4681 /* 4682 * Internal bulk free of objects that were not initialised by the post alloc 4683 * hooks and thus should not be processed by the free hooks 4684 */ 4685 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 4686 { 4687 if (!size) 4688 return; 4689 4690 do { 4691 struct detached_freelist df; 4692 4693 size = build_detached_freelist(s, size, p, &df); 4694 if (!df.slab) 4695 continue; 4696 4697 if (kfence_free(df.freelist)) 4698 continue; 4699 4700 do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, 4701 _RET_IP_); 4702 } while (likely(size)); 4703 } 4704 4705 /* Note that interrupts must be enabled when calling this function. */ 4706 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 4707 { 4708 if (!size) 4709 return; 4710 4711 do { 4712 struct detached_freelist df; 4713 4714 size = build_detached_freelist(s, size, p, &df); 4715 if (!df.slab) 4716 continue; 4717 4718 slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size], 4719 df.cnt, _RET_IP_); 4720 } while (likely(size)); 4721 } 4722 EXPORT_SYMBOL(kmem_cache_free_bulk); 4723 4724 #ifndef CONFIG_SLUB_TINY 4725 static inline 4726 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 4727 void **p) 4728 { 4729 struct kmem_cache_cpu *c; 4730 unsigned long irqflags; 4731 int i; 4732 4733 /* 4734 * Drain objects in the per cpu slab, while disabling local 4735 * IRQs, which protects against PREEMPT and interrupts 4736 * handlers invoking normal fastpath. 4737 */ 4738 c = slub_get_cpu_ptr(s->cpu_slab); 4739 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 4740 4741 for (i = 0; i < size; i++) { 4742 void *object = kfence_alloc(s, s->object_size, flags); 4743 4744 if (unlikely(object)) { 4745 p[i] = object; 4746 continue; 4747 } 4748 4749 object = c->freelist; 4750 if (unlikely(!object)) { 4751 /* 4752 * We may have removed an object from c->freelist using 4753 * the fastpath in the previous iteration; in that case, 4754 * c->tid has not been bumped yet. 4755 * Since ___slab_alloc() may reenable interrupts while 4756 * allocating memory, we should bump c->tid now. 4757 */ 4758 c->tid = next_tid(c->tid); 4759 4760 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); 4761 4762 /* 4763 * Invoking slow path likely have side-effect 4764 * of re-populating per CPU c->freelist 4765 */ 4766 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, 4767 _RET_IP_, c, s->object_size); 4768 if (unlikely(!p[i])) 4769 goto error; 4770 4771 c = this_cpu_ptr(s->cpu_slab); 4772 maybe_wipe_obj_freeptr(s, p[i]); 4773 4774 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 4775 4776 continue; /* goto for-loop */ 4777 } 4778 c->freelist = get_freepointer(s, object); 4779 p[i] = object; 4780 maybe_wipe_obj_freeptr(s, p[i]); 4781 stat(s, ALLOC_FASTPATH); 4782 } 4783 c->tid = next_tid(c->tid); 4784 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); 4785 slub_put_cpu_ptr(s->cpu_slab); 4786 4787 return i; 4788 4789 error: 4790 slub_put_cpu_ptr(s->cpu_slab); 4791 __kmem_cache_free_bulk(s, i, p); 4792 return 0; 4793 4794 } 4795 #else /* CONFIG_SLUB_TINY */ 4796 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, 4797 size_t size, void **p) 4798 { 4799 int i; 4800 4801 for (i = 0; i < size; i++) { 4802 void *object = kfence_alloc(s, s->object_size, flags); 4803 4804 if (unlikely(object)) { 4805 p[i] = object; 4806 continue; 4807 } 4808 4809 p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE, 4810 _RET_IP_, s->object_size); 4811 if (unlikely(!p[i])) 4812 goto error; 4813 4814 maybe_wipe_obj_freeptr(s, p[i]); 4815 } 4816 4817 return i; 4818 4819 error: 4820 __kmem_cache_free_bulk(s, i, p); 4821 return 0; 4822 } 4823 #endif /* CONFIG_SLUB_TINY */ 4824 4825 /* Note that interrupts must be enabled when calling this function. */ 4826 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, 4827 void **p) 4828 { 4829 int i; 4830 4831 if (!size) 4832 return 0; 4833 4834 s = slab_pre_alloc_hook(s, flags); 4835 if (unlikely(!s)) 4836 return 0; 4837 4838 i = __kmem_cache_alloc_bulk(s, flags, size, p); 4839 if (unlikely(i == 0)) 4840 return 0; 4841 4842 /* 4843 * memcg and kmem_cache debug support and memory initialization. 4844 * Done outside of the IRQ disabled fastpath loop. 4845 */ 4846 if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p, 4847 slab_want_init_on_alloc(flags, s), s->object_size))) { 4848 return 0; 4849 } 4850 return i; 4851 } 4852 EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); 4853 4854 4855 /* 4856 * Object placement in a slab is made very easy because we always start at 4857 * offset 0. If we tune the size of the object to the alignment then we can 4858 * get the required alignment by putting one properly sized object after 4859 * another. 4860 * 4861 * Notice that the allocation order determines the sizes of the per cpu 4862 * caches. Each processor has always one slab available for allocations. 4863 * Increasing the allocation order reduces the number of times that slabs 4864 * must be moved on and off the partial lists and is therefore a factor in 4865 * locking overhead. 4866 */ 4867 4868 /* 4869 * Minimum / Maximum order of slab pages. This influences locking overhead 4870 * and slab fragmentation. A higher order reduces the number of partial slabs 4871 * and increases the number of allocations possible without having to 4872 * take the list_lock. 4873 */ 4874 static unsigned int slub_min_order; 4875 static unsigned int slub_max_order = 4876 IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER; 4877 static unsigned int slub_min_objects; 4878 4879 /* 4880 * Calculate the order of allocation given an slab object size. 4881 * 4882 * The order of allocation has significant impact on performance and other 4883 * system components. Generally order 0 allocations should be preferred since 4884 * order 0 does not cause fragmentation in the page allocator. Larger objects 4885 * be problematic to put into order 0 slabs because there may be too much 4886 * unused space left. We go to a higher order if more than 1/16th of the slab 4887 * would be wasted. 4888 * 4889 * In order to reach satisfactory performance we must ensure that a minimum 4890 * number of objects is in one slab. Otherwise we may generate too much 4891 * activity on the partial lists which requires taking the list_lock. This is 4892 * less a concern for large slabs though which are rarely used. 4893 * 4894 * slab_max_order specifies the order where we begin to stop considering the 4895 * number of objects in a slab as critical. If we reach slab_max_order then 4896 * we try to keep the page order as low as possible. So we accept more waste 4897 * of space in favor of a small page order. 4898 * 4899 * Higher order allocations also allow the placement of more objects in a 4900 * slab and thereby reduce object handling overhead. If the user has 4901 * requested a higher minimum order then we start with that one instead of 4902 * the smallest order which will fit the object. 4903 */ 4904 static inline unsigned int calc_slab_order(unsigned int size, 4905 unsigned int min_order, unsigned int max_order, 4906 unsigned int fract_leftover) 4907 { 4908 unsigned int order; 4909 4910 for (order = min_order; order <= max_order; order++) { 4911 4912 unsigned int slab_size = (unsigned int)PAGE_SIZE << order; 4913 unsigned int rem; 4914 4915 rem = slab_size % size; 4916 4917 if (rem <= slab_size / fract_leftover) 4918 break; 4919 } 4920 4921 return order; 4922 } 4923 4924 static inline int calculate_order(unsigned int size) 4925 { 4926 unsigned int order; 4927 unsigned int min_objects; 4928 unsigned int max_objects; 4929 unsigned int min_order; 4930 4931 min_objects = slub_min_objects; 4932 if (!min_objects) { 4933 /* 4934 * Some architectures will only update present cpus when 4935 * onlining them, so don't trust the number if it's just 1. But 4936 * we also don't want to use nr_cpu_ids always, as on some other 4937 * architectures, there can be many possible cpus, but never 4938 * onlined. Here we compromise between trying to avoid too high 4939 * order on systems that appear larger than they are, and too 4940 * low order on systems that appear smaller than they are. 4941 */ 4942 unsigned int nr_cpus = num_present_cpus(); 4943 if (nr_cpus <= 1) 4944 nr_cpus = nr_cpu_ids; 4945 min_objects = 4 * (fls(nr_cpus) + 1); 4946 } 4947 /* min_objects can't be 0 because get_order(0) is undefined */ 4948 max_objects = max(order_objects(slub_max_order, size), 1U); 4949 min_objects = min(min_objects, max_objects); 4950 4951 min_order = max_t(unsigned int, slub_min_order, 4952 get_order(min_objects * size)); 4953 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE) 4954 return get_order(size * MAX_OBJS_PER_PAGE) - 1; 4955 4956 /* 4957 * Attempt to find best configuration for a slab. This works by first 4958 * attempting to generate a layout with the best possible configuration 4959 * and backing off gradually. 4960 * 4961 * We start with accepting at most 1/16 waste and try to find the 4962 * smallest order from min_objects-derived/slab_min_order up to 4963 * slab_max_order that will satisfy the constraint. Note that increasing 4964 * the order can only result in same or less fractional waste, not more. 4965 * 4966 * If that fails, we increase the acceptable fraction of waste and try 4967 * again. The last iteration with fraction of 1/2 would effectively 4968 * accept any waste and give us the order determined by min_objects, as 4969 * long as at least single object fits within slab_max_order. 4970 */ 4971 for (unsigned int fraction = 16; fraction > 1; fraction /= 2) { 4972 order = calc_slab_order(size, min_order, slub_max_order, 4973 fraction); 4974 if (order <= slub_max_order) 4975 return order; 4976 } 4977 4978 /* 4979 * Doh this slab cannot be placed using slab_max_order. 4980 */ 4981 order = get_order(size); 4982 if (order <= MAX_PAGE_ORDER) 4983 return order; 4984 return -ENOSYS; 4985 } 4986 4987 static void 4988 init_kmem_cache_node(struct kmem_cache_node *n) 4989 { 4990 n->nr_partial = 0; 4991 spin_lock_init(&n->list_lock); 4992 INIT_LIST_HEAD(&n->partial); 4993 #ifdef CONFIG_SLUB_DEBUG 4994 atomic_long_set(&n->nr_slabs, 0); 4995 atomic_long_set(&n->total_objects, 0); 4996 INIT_LIST_HEAD(&n->full); 4997 #endif 4998 } 4999 5000 #ifndef CONFIG_SLUB_TINY 5001 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 5002 { 5003 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < 5004 NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH * 5005 sizeof(struct kmem_cache_cpu)); 5006 5007 /* 5008 * Must align to double word boundary for the double cmpxchg 5009 * instructions to work; see __pcpu_double_call_return_bool(). 5010 */ 5011 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), 5012 2 * sizeof(void *)); 5013 5014 if (!s->cpu_slab) 5015 return 0; 5016 5017 init_kmem_cache_cpus(s); 5018 5019 return 1; 5020 } 5021 #else 5022 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 5023 { 5024 return 1; 5025 } 5026 #endif /* CONFIG_SLUB_TINY */ 5027 5028 static struct kmem_cache *kmem_cache_node; 5029 5030 /* 5031 * No kmalloc_node yet so do it by hand. We know that this is the first 5032 * slab on the node for this slabcache. There are no concurrent accesses 5033 * possible. 5034 * 5035 * Note that this function only works on the kmem_cache_node 5036 * when allocating for the kmem_cache_node. This is used for bootstrapping 5037 * memory on a fresh node that has no slab structures yet. 5038 */ 5039 static void early_kmem_cache_node_alloc(int node) 5040 { 5041 struct slab *slab; 5042 struct kmem_cache_node *n; 5043 5044 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); 5045 5046 slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); 5047 5048 BUG_ON(!slab); 5049 if (slab_nid(slab) != node) { 5050 pr_err("SLUB: Unable to allocate memory from node %d\n", node); 5051 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); 5052 } 5053 5054 n = slab->freelist; 5055 BUG_ON(!n); 5056 #ifdef CONFIG_SLUB_DEBUG 5057 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); 5058 #endif 5059 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false); 5060 slab->freelist = get_freepointer(kmem_cache_node, n); 5061 slab->inuse = 1; 5062 kmem_cache_node->node[node] = n; 5063 init_kmem_cache_node(n); 5064 inc_slabs_node(kmem_cache_node, node, slab->objects); 5065 5066 /* 5067 * No locks need to be taken here as it has just been 5068 * initialized and there is no concurrent access. 5069 */ 5070 __add_partial(n, slab, DEACTIVATE_TO_HEAD); 5071 } 5072 5073 static void free_kmem_cache_nodes(struct kmem_cache *s) 5074 { 5075 int node; 5076 struct kmem_cache_node *n; 5077 5078 for_each_kmem_cache_node(s, node, n) { 5079 s->node[node] = NULL; 5080 kmem_cache_free(kmem_cache_node, n); 5081 } 5082 } 5083 5084 void __kmem_cache_release(struct kmem_cache *s) 5085 { 5086 cache_random_seq_destroy(s); 5087 #ifndef CONFIG_SLUB_TINY 5088 free_percpu(s->cpu_slab); 5089 #endif 5090 free_kmem_cache_nodes(s); 5091 } 5092 5093 static int init_kmem_cache_nodes(struct kmem_cache *s) 5094 { 5095 int node; 5096 5097 for_each_node_mask(node, slab_nodes) { 5098 struct kmem_cache_node *n; 5099 5100 if (slab_state == DOWN) { 5101 early_kmem_cache_node_alloc(node); 5102 continue; 5103 } 5104 n = kmem_cache_alloc_node(kmem_cache_node, 5105 GFP_KERNEL, node); 5106 5107 if (!n) { 5108 free_kmem_cache_nodes(s); 5109 return 0; 5110 } 5111 5112 init_kmem_cache_node(n); 5113 s->node[node] = n; 5114 } 5115 return 1; 5116 } 5117 5118 static void set_cpu_partial(struct kmem_cache *s) 5119 { 5120 #ifdef CONFIG_SLUB_CPU_PARTIAL 5121 unsigned int nr_objects; 5122 5123 /* 5124 * cpu_partial determined the maximum number of objects kept in the 5125 * per cpu partial lists of a processor. 5126 * 5127 * Per cpu partial lists mainly contain slabs that just have one 5128 * object freed. If they are used for allocation then they can be 5129 * filled up again with minimal effort. The slab will never hit the 5130 * per node partial lists and therefore no locking will be required. 5131 * 5132 * For backwards compatibility reasons, this is determined as number 5133 * of objects, even though we now limit maximum number of pages, see 5134 * slub_set_cpu_partial() 5135 */ 5136 if (!kmem_cache_has_cpu_partial(s)) 5137 nr_objects = 0; 5138 else if (s->size >= PAGE_SIZE) 5139 nr_objects = 6; 5140 else if (s->size >= 1024) 5141 nr_objects = 24; 5142 else if (s->size >= 256) 5143 nr_objects = 52; 5144 else 5145 nr_objects = 120; 5146 5147 slub_set_cpu_partial(s, nr_objects); 5148 #endif 5149 } 5150 5151 /* 5152 * calculate_sizes() determines the order and the distribution of data within 5153 * a slab object. 5154 */ 5155 static int calculate_sizes(struct kmem_cache *s) 5156 { 5157 slab_flags_t flags = s->flags; 5158 unsigned int size = s->object_size; 5159 unsigned int order; 5160 5161 /* 5162 * Round up object size to the next word boundary. We can only 5163 * place the free pointer at word boundaries and this determines 5164 * the possible location of the free pointer. 5165 */ 5166 size = ALIGN(size, sizeof(void *)); 5167 5168 #ifdef CONFIG_SLUB_DEBUG 5169 /* 5170 * Determine if we can poison the object itself. If the user of 5171 * the slab may touch the object after free or before allocation 5172 * then we should never poison the object itself. 5173 */ 5174 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) && 5175 !s->ctor) 5176 s->flags |= __OBJECT_POISON; 5177 else 5178 s->flags &= ~__OBJECT_POISON; 5179 5180 5181 /* 5182 * If we are Redzoning then check if there is some space between the 5183 * end of the object and the free pointer. If not then add an 5184 * additional word to have some bytes to store Redzone information. 5185 */ 5186 if ((flags & SLAB_RED_ZONE) && size == s->object_size) 5187 size += sizeof(void *); 5188 #endif 5189 5190 /* 5191 * With that we have determined the number of bytes in actual use 5192 * by the object and redzoning. 5193 */ 5194 s->inuse = size; 5195 5196 if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) || s->ctor || 5197 ((flags & SLAB_RED_ZONE) && 5198 (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) { 5199 /* 5200 * Relocate free pointer after the object if it is not 5201 * permitted to overwrite the first word of the object on 5202 * kmem_cache_free. 5203 * 5204 * This is the case if we do RCU, have a constructor or 5205 * destructor, are poisoning the objects, or are 5206 * redzoning an object smaller than sizeof(void *) or are 5207 * redzoning an object with slub_debug_orig_size() enabled, 5208 * in which case the right redzone may be extended. 5209 * 5210 * The assumption that s->offset >= s->inuse means free 5211 * pointer is outside of the object is used in the 5212 * freeptr_outside_object() function. If that is no 5213 * longer true, the function needs to be modified. 5214 */ 5215 s->offset = size; 5216 size += sizeof(void *); 5217 } else { 5218 /* 5219 * Store freelist pointer near middle of object to keep 5220 * it away from the edges of the object to avoid small 5221 * sized over/underflows from neighboring allocations. 5222 */ 5223 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *)); 5224 } 5225 5226 #ifdef CONFIG_SLUB_DEBUG 5227 if (flags & SLAB_STORE_USER) { 5228 /* 5229 * Need to store information about allocs and frees after 5230 * the object. 5231 */ 5232 size += 2 * sizeof(struct track); 5233 5234 /* Save the original kmalloc request size */ 5235 if (flags & SLAB_KMALLOC) 5236 size += sizeof(unsigned int); 5237 } 5238 #endif 5239 5240 kasan_cache_create(s, &size, &s->flags); 5241 #ifdef CONFIG_SLUB_DEBUG 5242 if (flags & SLAB_RED_ZONE) { 5243 /* 5244 * Add some empty padding so that we can catch 5245 * overwrites from earlier objects rather than let 5246 * tracking information or the free pointer be 5247 * corrupted if a user writes before the start 5248 * of the object. 5249 */ 5250 size += sizeof(void *); 5251 5252 s->red_left_pad = sizeof(void *); 5253 s->red_left_pad = ALIGN(s->red_left_pad, s->align); 5254 size += s->red_left_pad; 5255 } 5256 #endif 5257 5258 /* 5259 * SLUB stores one object immediately after another beginning from 5260 * offset 0. In order to align the objects we have to simply size 5261 * each object to conform to the alignment. 5262 */ 5263 size = ALIGN(size, s->align); 5264 s->size = size; 5265 s->reciprocal_size = reciprocal_value(size); 5266 order = calculate_order(size); 5267 5268 if ((int)order < 0) 5269 return 0; 5270 5271 s->allocflags = __GFP_COMP; 5272 5273 if (s->flags & SLAB_CACHE_DMA) 5274 s->allocflags |= GFP_DMA; 5275 5276 if (s->flags & SLAB_CACHE_DMA32) 5277 s->allocflags |= GFP_DMA32; 5278 5279 if (s->flags & SLAB_RECLAIM_ACCOUNT) 5280 s->allocflags |= __GFP_RECLAIMABLE; 5281 5282 /* 5283 * Determine the number of objects per slab 5284 */ 5285 s->oo = oo_make(order, size); 5286 s->min = oo_make(get_order(size), size); 5287 5288 return !!oo_objects(s->oo); 5289 } 5290 5291 static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) 5292 { 5293 s->flags = kmem_cache_flags(flags, s->name); 5294 #ifdef CONFIG_SLAB_FREELIST_HARDENED 5295 s->random = get_random_long(); 5296 #endif 5297 5298 if (!calculate_sizes(s)) 5299 goto error; 5300 if (disable_higher_order_debug) { 5301 /* 5302 * Disable debugging flags that store metadata if the min slab 5303 * order increased. 5304 */ 5305 if (get_order(s->size) > get_order(s->object_size)) { 5306 s->flags &= ~DEBUG_METADATA_FLAGS; 5307 s->offset = 0; 5308 if (!calculate_sizes(s)) 5309 goto error; 5310 } 5311 } 5312 5313 #ifdef system_has_freelist_aba 5314 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) { 5315 /* Enable fast mode */ 5316 s->flags |= __CMPXCHG_DOUBLE; 5317 } 5318 #endif 5319 5320 /* 5321 * The larger the object size is, the more slabs we want on the partial 5322 * list to avoid pounding the page allocator excessively. 5323 */ 5324 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); 5325 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial); 5326 5327 set_cpu_partial(s); 5328 5329 #ifdef CONFIG_NUMA 5330 s->remote_node_defrag_ratio = 1000; 5331 #endif 5332 5333 /* Initialize the pre-computed randomized freelist if slab is up */ 5334 if (slab_state >= UP) { 5335 if (init_cache_random_seq(s)) 5336 goto error; 5337 } 5338 5339 if (!init_kmem_cache_nodes(s)) 5340 goto error; 5341 5342 if (alloc_kmem_cache_cpus(s)) 5343 return 0; 5344 5345 error: 5346 __kmem_cache_release(s); 5347 return -EINVAL; 5348 } 5349 5350 static void list_slab_objects(struct kmem_cache *s, struct slab *slab, 5351 const char *text) 5352 { 5353 #ifdef CONFIG_SLUB_DEBUG 5354 void *addr = slab_address(slab); 5355 void *p; 5356 5357 slab_err(s, slab, text, s->name); 5358 5359 spin_lock(&object_map_lock); 5360 __fill_map(object_map, s, slab); 5361 5362 for_each_object(p, s, addr, slab->objects) { 5363 5364 if (!test_bit(__obj_to_index(s, addr, p), object_map)) { 5365 pr_err("Object 0x%p @offset=%tu\n", p, p - addr); 5366 print_tracking(s, p); 5367 } 5368 } 5369 spin_unlock(&object_map_lock); 5370 #endif 5371 } 5372 5373 /* 5374 * Attempt to free all partial slabs on a node. 5375 * This is called from __kmem_cache_shutdown(). We must take list_lock 5376 * because sysfs file might still access partial list after the shutdowning. 5377 */ 5378 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) 5379 { 5380 LIST_HEAD(discard); 5381 struct slab *slab, *h; 5382 5383 BUG_ON(irqs_disabled()); 5384 spin_lock_irq(&n->list_lock); 5385 list_for_each_entry_safe(slab, h, &n->partial, slab_list) { 5386 if (!slab->inuse) { 5387 remove_partial(n, slab); 5388 list_add(&slab->slab_list, &discard); 5389 } else { 5390 list_slab_objects(s, slab, 5391 "Objects remaining in %s on __kmem_cache_shutdown()"); 5392 } 5393 } 5394 spin_unlock_irq(&n->list_lock); 5395 5396 list_for_each_entry_safe(slab, h, &discard, slab_list) 5397 discard_slab(s, slab); 5398 } 5399 5400 bool __kmem_cache_empty(struct kmem_cache *s) 5401 { 5402 int node; 5403 struct kmem_cache_node *n; 5404 5405 for_each_kmem_cache_node(s, node, n) 5406 if (n->nr_partial || node_nr_slabs(n)) 5407 return false; 5408 return true; 5409 } 5410 5411 /* 5412 * Release all resources used by a slab cache. 5413 */ 5414 int __kmem_cache_shutdown(struct kmem_cache *s) 5415 { 5416 int node; 5417 struct kmem_cache_node *n; 5418 5419 flush_all_cpus_locked(s); 5420 /* Attempt to free all objects */ 5421 for_each_kmem_cache_node(s, node, n) { 5422 free_partial(s, n); 5423 if (n->nr_partial || node_nr_slabs(n)) 5424 return 1; 5425 } 5426 return 0; 5427 } 5428 5429 #ifdef CONFIG_PRINTK 5430 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 5431 { 5432 void *base; 5433 int __maybe_unused i; 5434 unsigned int objnr; 5435 void *objp; 5436 void *objp0; 5437 struct kmem_cache *s = slab->slab_cache; 5438 struct track __maybe_unused *trackp; 5439 5440 kpp->kp_ptr = object; 5441 kpp->kp_slab = slab; 5442 kpp->kp_slab_cache = s; 5443 base = slab_address(slab); 5444 objp0 = kasan_reset_tag(object); 5445 #ifdef CONFIG_SLUB_DEBUG 5446 objp = restore_red_left(s, objp0); 5447 #else 5448 objp = objp0; 5449 #endif 5450 objnr = obj_to_index(s, slab, objp); 5451 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp); 5452 objp = base + s->size * objnr; 5453 kpp->kp_objp = objp; 5454 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size 5455 || (objp - base) % s->size) || 5456 !(s->flags & SLAB_STORE_USER)) 5457 return; 5458 #ifdef CONFIG_SLUB_DEBUG 5459 objp = fixup_red_left(s, objp); 5460 trackp = get_track(s, objp, TRACK_ALLOC); 5461 kpp->kp_ret = (void *)trackp->addr; 5462 #ifdef CONFIG_STACKDEPOT 5463 { 5464 depot_stack_handle_t handle; 5465 unsigned long *entries; 5466 unsigned int nr_entries; 5467 5468 handle = READ_ONCE(trackp->handle); 5469 if (handle) { 5470 nr_entries = stack_depot_fetch(handle, &entries); 5471 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) 5472 kpp->kp_stack[i] = (void *)entries[i]; 5473 } 5474 5475 trackp = get_track(s, objp, TRACK_FREE); 5476 handle = READ_ONCE(trackp->handle); 5477 if (handle) { 5478 nr_entries = stack_depot_fetch(handle, &entries); 5479 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) 5480 kpp->kp_free_stack[i] = (void *)entries[i]; 5481 } 5482 } 5483 #endif 5484 #endif 5485 } 5486 #endif 5487 5488 /******************************************************************** 5489 * Kmalloc subsystem 5490 *******************************************************************/ 5491 5492 static int __init setup_slub_min_order(char *str) 5493 { 5494 get_option(&str, (int *)&slub_min_order); 5495 5496 if (slub_min_order > slub_max_order) 5497 slub_max_order = slub_min_order; 5498 5499 return 1; 5500 } 5501 5502 __setup("slab_min_order=", setup_slub_min_order); 5503 __setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0); 5504 5505 5506 static int __init setup_slub_max_order(char *str) 5507 { 5508 get_option(&str, (int *)&slub_max_order); 5509 slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER); 5510 5511 if (slub_min_order > slub_max_order) 5512 slub_min_order = slub_max_order; 5513 5514 return 1; 5515 } 5516 5517 __setup("slab_max_order=", setup_slub_max_order); 5518 __setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0); 5519 5520 static int __init setup_slub_min_objects(char *str) 5521 { 5522 get_option(&str, (int *)&slub_min_objects); 5523 5524 return 1; 5525 } 5526 5527 __setup("slab_min_objects=", setup_slub_min_objects); 5528 __setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0); 5529 5530 #ifdef CONFIG_HARDENED_USERCOPY 5531 /* 5532 * Rejects incorrectly sized objects and objects that are to be copied 5533 * to/from userspace but do not fall entirely within the containing slab 5534 * cache's usercopy region. 5535 * 5536 * Returns NULL if check passes, otherwise const char * to name of cache 5537 * to indicate an error. 5538 */ 5539 void __check_heap_object(const void *ptr, unsigned long n, 5540 const struct slab *slab, bool to_user) 5541 { 5542 struct kmem_cache *s; 5543 unsigned int offset; 5544 bool is_kfence = is_kfence_address(ptr); 5545 5546 ptr = kasan_reset_tag(ptr); 5547 5548 /* Find object and usable object size. */ 5549 s = slab->slab_cache; 5550 5551 /* Reject impossible pointers. */ 5552 if (ptr < slab_address(slab)) 5553 usercopy_abort("SLUB object not in SLUB page?!", NULL, 5554 to_user, 0, n); 5555 5556 /* Find offset within object. */ 5557 if (is_kfence) 5558 offset = ptr - kfence_object_start(ptr); 5559 else 5560 offset = (ptr - slab_address(slab)) % s->size; 5561 5562 /* Adjust for redzone and reject if within the redzone. */ 5563 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) { 5564 if (offset < s->red_left_pad) 5565 usercopy_abort("SLUB object in left red zone", 5566 s->name, to_user, offset, n); 5567 offset -= s->red_left_pad; 5568 } 5569 5570 /* Allow address range falling entirely within usercopy region. */ 5571 if (offset >= s->useroffset && 5572 offset - s->useroffset <= s->usersize && 5573 n <= s->useroffset - offset + s->usersize) 5574 return; 5575 5576 usercopy_abort("SLUB object", s->name, to_user, offset, n); 5577 } 5578 #endif /* CONFIG_HARDENED_USERCOPY */ 5579 5580 #define SHRINK_PROMOTE_MAX 32 5581 5582 /* 5583 * kmem_cache_shrink discards empty slabs and promotes the slabs filled 5584 * up most to the head of the partial lists. New allocations will then 5585 * fill those up and thus they can be removed from the partial lists. 5586 * 5587 * The slabs with the least items are placed last. This results in them 5588 * being allocated from last increasing the chance that the last objects 5589 * are freed in them. 5590 */ 5591 static int __kmem_cache_do_shrink(struct kmem_cache *s) 5592 { 5593 int node; 5594 int i; 5595 struct kmem_cache_node *n; 5596 struct slab *slab; 5597 struct slab *t; 5598 struct list_head discard; 5599 struct list_head promote[SHRINK_PROMOTE_MAX]; 5600 unsigned long flags; 5601 int ret = 0; 5602 5603 for_each_kmem_cache_node(s, node, n) { 5604 INIT_LIST_HEAD(&discard); 5605 for (i = 0; i < SHRINK_PROMOTE_MAX; i++) 5606 INIT_LIST_HEAD(promote + i); 5607 5608 spin_lock_irqsave(&n->list_lock, flags); 5609 5610 /* 5611 * Build lists of slabs to discard or promote. 5612 * 5613 * Note that concurrent frees may occur while we hold the 5614 * list_lock. slab->inuse here is the upper limit. 5615 */ 5616 list_for_each_entry_safe(slab, t, &n->partial, slab_list) { 5617 int free = slab->objects - slab->inuse; 5618 5619 /* Do not reread slab->inuse */ 5620 barrier(); 5621 5622 /* We do not keep full slabs on the list */ 5623 BUG_ON(free <= 0); 5624 5625 if (free == slab->objects) { 5626 list_move(&slab->slab_list, &discard); 5627 slab_clear_node_partial(slab); 5628 n->nr_partial--; 5629 dec_slabs_node(s, node, slab->objects); 5630 } else if (free <= SHRINK_PROMOTE_MAX) 5631 list_move(&slab->slab_list, promote + free - 1); 5632 } 5633 5634 /* 5635 * Promote the slabs filled up most to the head of the 5636 * partial list. 5637 */ 5638 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) 5639 list_splice(promote + i, &n->partial); 5640 5641 spin_unlock_irqrestore(&n->list_lock, flags); 5642 5643 /* Release empty slabs */ 5644 list_for_each_entry_safe(slab, t, &discard, slab_list) 5645 free_slab(s, slab); 5646 5647 if (node_nr_slabs(n)) 5648 ret = 1; 5649 } 5650 5651 return ret; 5652 } 5653 5654 int __kmem_cache_shrink(struct kmem_cache *s) 5655 { 5656 flush_all(s); 5657 return __kmem_cache_do_shrink(s); 5658 } 5659 5660 static int slab_mem_going_offline_callback(void *arg) 5661 { 5662 struct kmem_cache *s; 5663 5664 mutex_lock(&slab_mutex); 5665 list_for_each_entry(s, &slab_caches, list) { 5666 flush_all_cpus_locked(s); 5667 __kmem_cache_do_shrink(s); 5668 } 5669 mutex_unlock(&slab_mutex); 5670 5671 return 0; 5672 } 5673 5674 static void slab_mem_offline_callback(void *arg) 5675 { 5676 struct memory_notify *marg = arg; 5677 int offline_node; 5678 5679 offline_node = marg->status_change_nid_normal; 5680 5681 /* 5682 * If the node still has available memory. we need kmem_cache_node 5683 * for it yet. 5684 */ 5685 if (offline_node < 0) 5686 return; 5687 5688 mutex_lock(&slab_mutex); 5689 node_clear(offline_node, slab_nodes); 5690 /* 5691 * We no longer free kmem_cache_node structures here, as it would be 5692 * racy with all get_node() users, and infeasible to protect them with 5693 * slab_mutex. 5694 */ 5695 mutex_unlock(&slab_mutex); 5696 } 5697 5698 static int slab_mem_going_online_callback(void *arg) 5699 { 5700 struct kmem_cache_node *n; 5701 struct kmem_cache *s; 5702 struct memory_notify *marg = arg; 5703 int nid = marg->status_change_nid_normal; 5704 int ret = 0; 5705 5706 /* 5707 * If the node's memory is already available, then kmem_cache_node is 5708 * already created. Nothing to do. 5709 */ 5710 if (nid < 0) 5711 return 0; 5712 5713 /* 5714 * We are bringing a node online. No memory is available yet. We must 5715 * allocate a kmem_cache_node structure in order to bring the node 5716 * online. 5717 */ 5718 mutex_lock(&slab_mutex); 5719 list_for_each_entry(s, &slab_caches, list) { 5720 /* 5721 * The structure may already exist if the node was previously 5722 * onlined and offlined. 5723 */ 5724 if (get_node(s, nid)) 5725 continue; 5726 /* 5727 * XXX: kmem_cache_alloc_node will fallback to other nodes 5728 * since memory is not yet available from the node that 5729 * is brought up. 5730 */ 5731 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); 5732 if (!n) { 5733 ret = -ENOMEM; 5734 goto out; 5735 } 5736 init_kmem_cache_node(n); 5737 s->node[nid] = n; 5738 } 5739 /* 5740 * Any cache created after this point will also have kmem_cache_node 5741 * initialized for the new node. 5742 */ 5743 node_set(nid, slab_nodes); 5744 out: 5745 mutex_unlock(&slab_mutex); 5746 return ret; 5747 } 5748 5749 static int slab_memory_callback(struct notifier_block *self, 5750 unsigned long action, void *arg) 5751 { 5752 int ret = 0; 5753 5754 switch (action) { 5755 case MEM_GOING_ONLINE: 5756 ret = slab_mem_going_online_callback(arg); 5757 break; 5758 case MEM_GOING_OFFLINE: 5759 ret = slab_mem_going_offline_callback(arg); 5760 break; 5761 case MEM_OFFLINE: 5762 case MEM_CANCEL_ONLINE: 5763 slab_mem_offline_callback(arg); 5764 break; 5765 case MEM_ONLINE: 5766 case MEM_CANCEL_OFFLINE: 5767 break; 5768 } 5769 if (ret) 5770 ret = notifier_from_errno(ret); 5771 else 5772 ret = NOTIFY_OK; 5773 return ret; 5774 } 5775 5776 /******************************************************************** 5777 * Basic setup of slabs 5778 *******************************************************************/ 5779 5780 /* 5781 * Used for early kmem_cache structures that were allocated using 5782 * the page allocator. Allocate them properly then fix up the pointers 5783 * that may be pointing to the wrong kmem_cache structure. 5784 */ 5785 5786 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) 5787 { 5788 int node; 5789 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 5790 struct kmem_cache_node *n; 5791 5792 memcpy(s, static_cache, kmem_cache->object_size); 5793 5794 /* 5795 * This runs very early, and only the boot processor is supposed to be 5796 * up. Even if it weren't true, IRQs are not up so we couldn't fire 5797 * IPIs around. 5798 */ 5799 __flush_cpu_slab(s, smp_processor_id()); 5800 for_each_kmem_cache_node(s, node, n) { 5801 struct slab *p; 5802 5803 list_for_each_entry(p, &n->partial, slab_list) 5804 p->slab_cache = s; 5805 5806 #ifdef CONFIG_SLUB_DEBUG 5807 list_for_each_entry(p, &n->full, slab_list) 5808 p->slab_cache = s; 5809 #endif 5810 } 5811 list_add(&s->list, &slab_caches); 5812 return s; 5813 } 5814 5815 void __init kmem_cache_init(void) 5816 { 5817 static __initdata struct kmem_cache boot_kmem_cache, 5818 boot_kmem_cache_node; 5819 int node; 5820 5821 if (debug_guardpage_minorder()) 5822 slub_max_order = 0; 5823 5824 /* Print slub debugging pointers without hashing */ 5825 if (__slub_debug_enabled()) 5826 no_hash_pointers_enable(NULL); 5827 5828 kmem_cache_node = &boot_kmem_cache_node; 5829 kmem_cache = &boot_kmem_cache; 5830 5831 /* 5832 * Initialize the nodemask for which we will allocate per node 5833 * structures. Here we don't need taking slab_mutex yet. 5834 */ 5835 for_each_node_state(node, N_NORMAL_MEMORY) 5836 node_set(node, slab_nodes); 5837 5838 create_boot_cache(kmem_cache_node, "kmem_cache_node", 5839 sizeof(struct kmem_cache_node), 5840 SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); 5841 5842 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 5843 5844 /* Able to allocate the per node structures */ 5845 slab_state = PARTIAL; 5846 5847 create_boot_cache(kmem_cache, "kmem_cache", 5848 offsetof(struct kmem_cache, node) + 5849 nr_node_ids * sizeof(struct kmem_cache_node *), 5850 SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); 5851 5852 kmem_cache = bootstrap(&boot_kmem_cache); 5853 kmem_cache_node = bootstrap(&boot_kmem_cache_node); 5854 5855 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 5856 setup_kmalloc_cache_index_table(); 5857 create_kmalloc_caches(); 5858 5859 /* Setup random freelists for each cache */ 5860 init_freelist_randomization(); 5861 5862 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL, 5863 slub_cpu_dead); 5864 5865 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n", 5866 cache_line_size(), 5867 slub_min_order, slub_max_order, slub_min_objects, 5868 nr_cpu_ids, nr_node_ids); 5869 } 5870 5871 void __init kmem_cache_init_late(void) 5872 { 5873 #ifndef CONFIG_SLUB_TINY 5874 flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); 5875 WARN_ON(!flushwq); 5876 #endif 5877 } 5878 5879 struct kmem_cache * 5880 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, 5881 slab_flags_t flags, void (*ctor)(void *)) 5882 { 5883 struct kmem_cache *s; 5884 5885 s = find_mergeable(size, align, flags, name, ctor); 5886 if (s) { 5887 if (sysfs_slab_alias(s, name)) 5888 return NULL; 5889 5890 s->refcount++; 5891 5892 /* 5893 * Adjust the object sizes so that we clear 5894 * the complete object on kzalloc. 5895 */ 5896 s->object_size = max(s->object_size, size); 5897 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); 5898 } 5899 5900 return s; 5901 } 5902 5903 int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags) 5904 { 5905 int err; 5906 5907 err = kmem_cache_open(s, flags); 5908 if (err) 5909 return err; 5910 5911 /* Mutex is not taken during early boot */ 5912 if (slab_state <= UP) 5913 return 0; 5914 5915 err = sysfs_slab_add(s); 5916 if (err) { 5917 __kmem_cache_release(s); 5918 return err; 5919 } 5920 5921 if (s->flags & SLAB_STORE_USER) 5922 debugfs_slab_add(s); 5923 5924 return 0; 5925 } 5926 5927 #ifdef SLAB_SUPPORTS_SYSFS 5928 static int count_inuse(struct slab *slab) 5929 { 5930 return slab->inuse; 5931 } 5932 5933 static int count_total(struct slab *slab) 5934 { 5935 return slab->objects; 5936 } 5937 #endif 5938 5939 #ifdef CONFIG_SLUB_DEBUG 5940 static void validate_slab(struct kmem_cache *s, struct slab *slab, 5941 unsigned long *obj_map) 5942 { 5943 void *p; 5944 void *addr = slab_address(slab); 5945 5946 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL)) 5947 return; 5948 5949 /* Now we know that a valid freelist exists */ 5950 __fill_map(obj_map, s, slab); 5951 for_each_object(p, s, addr, slab->objects) { 5952 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ? 5953 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE; 5954 5955 if (!check_object(s, slab, p, val)) 5956 break; 5957 } 5958 } 5959 5960 static int validate_slab_node(struct kmem_cache *s, 5961 struct kmem_cache_node *n, unsigned long *obj_map) 5962 { 5963 unsigned long count = 0; 5964 struct slab *slab; 5965 unsigned long flags; 5966 5967 spin_lock_irqsave(&n->list_lock, flags); 5968 5969 list_for_each_entry(slab, &n->partial, slab_list) { 5970 validate_slab(s, slab, obj_map); 5971 count++; 5972 } 5973 if (count != n->nr_partial) { 5974 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", 5975 s->name, count, n->nr_partial); 5976 slab_add_kunit_errors(); 5977 } 5978 5979 if (!(s->flags & SLAB_STORE_USER)) 5980 goto out; 5981 5982 list_for_each_entry(slab, &n->full, slab_list) { 5983 validate_slab(s, slab, obj_map); 5984 count++; 5985 } 5986 if (count != node_nr_slabs(n)) { 5987 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", 5988 s->name, count, node_nr_slabs(n)); 5989 slab_add_kunit_errors(); 5990 } 5991 5992 out: 5993 spin_unlock_irqrestore(&n->list_lock, flags); 5994 return count; 5995 } 5996 5997 long validate_slab_cache(struct kmem_cache *s) 5998 { 5999 int node; 6000 unsigned long count = 0; 6001 struct kmem_cache_node *n; 6002 unsigned long *obj_map; 6003 6004 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); 6005 if (!obj_map) 6006 return -ENOMEM; 6007 6008 flush_all(s); 6009 for_each_kmem_cache_node(s, node, n) 6010 count += validate_slab_node(s, n, obj_map); 6011 6012 bitmap_free(obj_map); 6013 6014 return count; 6015 } 6016 EXPORT_SYMBOL(validate_slab_cache); 6017 6018 #ifdef CONFIG_DEBUG_FS 6019 /* 6020 * Generate lists of code addresses where slabcache objects are allocated 6021 * and freed. 6022 */ 6023 6024 struct location { 6025 depot_stack_handle_t handle; 6026 unsigned long count; 6027 unsigned long addr; 6028 unsigned long waste; 6029 long long sum_time; 6030 long min_time; 6031 long max_time; 6032 long min_pid; 6033 long max_pid; 6034 DECLARE_BITMAP(cpus, NR_CPUS); 6035 nodemask_t nodes; 6036 }; 6037 6038 struct loc_track { 6039 unsigned long max; 6040 unsigned long count; 6041 struct location *loc; 6042 loff_t idx; 6043 }; 6044 6045 static struct dentry *slab_debugfs_root; 6046 6047 static void free_loc_track(struct loc_track *t) 6048 { 6049 if (t->max) 6050 free_pages((unsigned long)t->loc, 6051 get_order(sizeof(struct location) * t->max)); 6052 } 6053 6054 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) 6055 { 6056 struct location *l; 6057 int order; 6058 6059 order = get_order(sizeof(struct location) * max); 6060 6061 l = (void *)__get_free_pages(flags, order); 6062 if (!l) 6063 return 0; 6064 6065 if (t->count) { 6066 memcpy(l, t->loc, sizeof(struct location) * t->count); 6067 free_loc_track(t); 6068 } 6069 t->max = max; 6070 t->loc = l; 6071 return 1; 6072 } 6073 6074 static int add_location(struct loc_track *t, struct kmem_cache *s, 6075 const struct track *track, 6076 unsigned int orig_size) 6077 { 6078 long start, end, pos; 6079 struct location *l; 6080 unsigned long caddr, chandle, cwaste; 6081 unsigned long age = jiffies - track->when; 6082 depot_stack_handle_t handle = 0; 6083 unsigned int waste = s->object_size - orig_size; 6084 6085 #ifdef CONFIG_STACKDEPOT 6086 handle = READ_ONCE(track->handle); 6087 #endif 6088 start = -1; 6089 end = t->count; 6090 6091 for ( ; ; ) { 6092 pos = start + (end - start + 1) / 2; 6093 6094 /* 6095 * There is nothing at "end". If we end up there 6096 * we need to add something to before end. 6097 */ 6098 if (pos == end) 6099 break; 6100 6101 l = &t->loc[pos]; 6102 caddr = l->addr; 6103 chandle = l->handle; 6104 cwaste = l->waste; 6105 if ((track->addr == caddr) && (handle == chandle) && 6106 (waste == cwaste)) { 6107 6108 l->count++; 6109 if (track->when) { 6110 l->sum_time += age; 6111 if (age < l->min_time) 6112 l->min_time = age; 6113 if (age > l->max_time) 6114 l->max_time = age; 6115 6116 if (track->pid < l->min_pid) 6117 l->min_pid = track->pid; 6118 if (track->pid > l->max_pid) 6119 l->max_pid = track->pid; 6120 6121 cpumask_set_cpu(track->cpu, 6122 to_cpumask(l->cpus)); 6123 } 6124 node_set(page_to_nid(virt_to_page(track)), l->nodes); 6125 return 1; 6126 } 6127 6128 if (track->addr < caddr) 6129 end = pos; 6130 else if (track->addr == caddr && handle < chandle) 6131 end = pos; 6132 else if (track->addr == caddr && handle == chandle && 6133 waste < cwaste) 6134 end = pos; 6135 else 6136 start = pos; 6137 } 6138 6139 /* 6140 * Not found. Insert new tracking element. 6141 */ 6142 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) 6143 return 0; 6144 6145 l = t->loc + pos; 6146 if (pos < t->count) 6147 memmove(l + 1, l, 6148 (t->count - pos) * sizeof(struct location)); 6149 t->count++; 6150 l->count = 1; 6151 l->addr = track->addr; 6152 l->sum_time = age; 6153 l->min_time = age; 6154 l->max_time = age; 6155 l->min_pid = track->pid; 6156 l->max_pid = track->pid; 6157 l->handle = handle; 6158 l->waste = waste; 6159 cpumask_clear(to_cpumask(l->cpus)); 6160 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); 6161 nodes_clear(l->nodes); 6162 node_set(page_to_nid(virt_to_page(track)), l->nodes); 6163 return 1; 6164 } 6165 6166 static void process_slab(struct loc_track *t, struct kmem_cache *s, 6167 struct slab *slab, enum track_item alloc, 6168 unsigned long *obj_map) 6169 { 6170 void *addr = slab_address(slab); 6171 bool is_alloc = (alloc == TRACK_ALLOC); 6172 void *p; 6173 6174 __fill_map(obj_map, s, slab); 6175 6176 for_each_object(p, s, addr, slab->objects) 6177 if (!test_bit(__obj_to_index(s, addr, p), obj_map)) 6178 add_location(t, s, get_track(s, p, alloc), 6179 is_alloc ? get_orig_size(s, p) : 6180 s->object_size); 6181 } 6182 #endif /* CONFIG_DEBUG_FS */ 6183 #endif /* CONFIG_SLUB_DEBUG */ 6184 6185 #ifdef SLAB_SUPPORTS_SYSFS 6186 enum slab_stat_type { 6187 SL_ALL, /* All slabs */ 6188 SL_PARTIAL, /* Only partially allocated slabs */ 6189 SL_CPU, /* Only slabs used for cpu caches */ 6190 SL_OBJECTS, /* Determine allocated objects not slabs */ 6191 SL_TOTAL /* Determine object capacity not slabs */ 6192 }; 6193 6194 #define SO_ALL (1 << SL_ALL) 6195 #define SO_PARTIAL (1 << SL_PARTIAL) 6196 #define SO_CPU (1 << SL_CPU) 6197 #define SO_OBJECTS (1 << SL_OBJECTS) 6198 #define SO_TOTAL (1 << SL_TOTAL) 6199 6200 static ssize_t show_slab_objects(struct kmem_cache *s, 6201 char *buf, unsigned long flags) 6202 { 6203 unsigned long total = 0; 6204 int node; 6205 int x; 6206 unsigned long *nodes; 6207 int len = 0; 6208 6209 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL); 6210 if (!nodes) 6211 return -ENOMEM; 6212 6213 if (flags & SO_CPU) { 6214 int cpu; 6215 6216 for_each_possible_cpu(cpu) { 6217 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, 6218 cpu); 6219 int node; 6220 struct slab *slab; 6221 6222 slab = READ_ONCE(c->slab); 6223 if (!slab) 6224 continue; 6225 6226 node = slab_nid(slab); 6227 if (flags & SO_TOTAL) 6228 x = slab->objects; 6229 else if (flags & SO_OBJECTS) 6230 x = slab->inuse; 6231 else 6232 x = 1; 6233 6234 total += x; 6235 nodes[node] += x; 6236 6237 #ifdef CONFIG_SLUB_CPU_PARTIAL 6238 slab = slub_percpu_partial_read_once(c); 6239 if (slab) { 6240 node = slab_nid(slab); 6241 if (flags & SO_TOTAL) 6242 WARN_ON_ONCE(1); 6243 else if (flags & SO_OBJECTS) 6244 WARN_ON_ONCE(1); 6245 else 6246 x = data_race(slab->slabs); 6247 total += x; 6248 nodes[node] += x; 6249 } 6250 #endif 6251 } 6252 } 6253 6254 /* 6255 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex" 6256 * already held which will conflict with an existing lock order: 6257 * 6258 * mem_hotplug_lock->slab_mutex->kernfs_mutex 6259 * 6260 * We don't really need mem_hotplug_lock (to hold off 6261 * slab_mem_going_offline_callback) here because slab's memory hot 6262 * unplug code doesn't destroy the kmem_cache->node[] data. 6263 */ 6264 6265 #ifdef CONFIG_SLUB_DEBUG 6266 if (flags & SO_ALL) { 6267 struct kmem_cache_node *n; 6268 6269 for_each_kmem_cache_node(s, node, n) { 6270 6271 if (flags & SO_TOTAL) 6272 x = node_nr_objs(n); 6273 else if (flags & SO_OBJECTS) 6274 x = node_nr_objs(n) - count_partial(n, count_free); 6275 else 6276 x = node_nr_slabs(n); 6277 total += x; 6278 nodes[node] += x; 6279 } 6280 6281 } else 6282 #endif 6283 if (flags & SO_PARTIAL) { 6284 struct kmem_cache_node *n; 6285 6286 for_each_kmem_cache_node(s, node, n) { 6287 if (flags & SO_TOTAL) 6288 x = count_partial(n, count_total); 6289 else if (flags & SO_OBJECTS) 6290 x = count_partial(n, count_inuse); 6291 else 6292 x = n->nr_partial; 6293 total += x; 6294 nodes[node] += x; 6295 } 6296 } 6297 6298 len += sysfs_emit_at(buf, len, "%lu", total); 6299 #ifdef CONFIG_NUMA 6300 for (node = 0; node < nr_node_ids; node++) { 6301 if (nodes[node]) 6302 len += sysfs_emit_at(buf, len, " N%d=%lu", 6303 node, nodes[node]); 6304 } 6305 #endif 6306 len += sysfs_emit_at(buf, len, "\n"); 6307 kfree(nodes); 6308 6309 return len; 6310 } 6311 6312 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) 6313 #define to_slab(n) container_of(n, struct kmem_cache, kobj) 6314 6315 struct slab_attribute { 6316 struct attribute attr; 6317 ssize_t (*show)(struct kmem_cache *s, char *buf); 6318 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); 6319 }; 6320 6321 #define SLAB_ATTR_RO(_name) \ 6322 static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400) 6323 6324 #define SLAB_ATTR(_name) \ 6325 static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600) 6326 6327 static ssize_t slab_size_show(struct kmem_cache *s, char *buf) 6328 { 6329 return sysfs_emit(buf, "%u\n", s->size); 6330 } 6331 SLAB_ATTR_RO(slab_size); 6332 6333 static ssize_t align_show(struct kmem_cache *s, char *buf) 6334 { 6335 return sysfs_emit(buf, "%u\n", s->align); 6336 } 6337 SLAB_ATTR_RO(align); 6338 6339 static ssize_t object_size_show(struct kmem_cache *s, char *buf) 6340 { 6341 return sysfs_emit(buf, "%u\n", s->object_size); 6342 } 6343 SLAB_ATTR_RO(object_size); 6344 6345 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) 6346 { 6347 return sysfs_emit(buf, "%u\n", oo_objects(s->oo)); 6348 } 6349 SLAB_ATTR_RO(objs_per_slab); 6350 6351 static ssize_t order_show(struct kmem_cache *s, char *buf) 6352 { 6353 return sysfs_emit(buf, "%u\n", oo_order(s->oo)); 6354 } 6355 SLAB_ATTR_RO(order); 6356 6357 static ssize_t min_partial_show(struct kmem_cache *s, char *buf) 6358 { 6359 return sysfs_emit(buf, "%lu\n", s->min_partial); 6360 } 6361 6362 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, 6363 size_t length) 6364 { 6365 unsigned long min; 6366 int err; 6367 6368 err = kstrtoul(buf, 10, &min); 6369 if (err) 6370 return err; 6371 6372 s->min_partial = min; 6373 return length; 6374 } 6375 SLAB_ATTR(min_partial); 6376 6377 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) 6378 { 6379 unsigned int nr_partial = 0; 6380 #ifdef CONFIG_SLUB_CPU_PARTIAL 6381 nr_partial = s->cpu_partial; 6382 #endif 6383 6384 return sysfs_emit(buf, "%u\n", nr_partial); 6385 } 6386 6387 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, 6388 size_t length) 6389 { 6390 unsigned int objects; 6391 int err; 6392 6393 err = kstrtouint(buf, 10, &objects); 6394 if (err) 6395 return err; 6396 if (objects && !kmem_cache_has_cpu_partial(s)) 6397 return -EINVAL; 6398 6399 slub_set_cpu_partial(s, objects); 6400 flush_all(s); 6401 return length; 6402 } 6403 SLAB_ATTR(cpu_partial); 6404 6405 static ssize_t ctor_show(struct kmem_cache *s, char *buf) 6406 { 6407 if (!s->ctor) 6408 return 0; 6409 return sysfs_emit(buf, "%pS\n", s->ctor); 6410 } 6411 SLAB_ATTR_RO(ctor); 6412 6413 static ssize_t aliases_show(struct kmem_cache *s, char *buf) 6414 { 6415 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); 6416 } 6417 SLAB_ATTR_RO(aliases); 6418 6419 static ssize_t partial_show(struct kmem_cache *s, char *buf) 6420 { 6421 return show_slab_objects(s, buf, SO_PARTIAL); 6422 } 6423 SLAB_ATTR_RO(partial); 6424 6425 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) 6426 { 6427 return show_slab_objects(s, buf, SO_CPU); 6428 } 6429 SLAB_ATTR_RO(cpu_slabs); 6430 6431 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) 6432 { 6433 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); 6434 } 6435 SLAB_ATTR_RO(objects_partial); 6436 6437 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) 6438 { 6439 int objects = 0; 6440 int slabs = 0; 6441 int cpu __maybe_unused; 6442 int len = 0; 6443 6444 #ifdef CONFIG_SLUB_CPU_PARTIAL 6445 for_each_online_cpu(cpu) { 6446 struct slab *slab; 6447 6448 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); 6449 6450 if (slab) 6451 slabs += data_race(slab->slabs); 6452 } 6453 #endif 6454 6455 /* Approximate half-full slabs, see slub_set_cpu_partial() */ 6456 objects = (slabs * oo_objects(s->oo)) / 2; 6457 len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs); 6458 6459 #ifdef CONFIG_SLUB_CPU_PARTIAL 6460 for_each_online_cpu(cpu) { 6461 struct slab *slab; 6462 6463 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); 6464 if (slab) { 6465 slabs = data_race(slab->slabs); 6466 objects = (slabs * oo_objects(s->oo)) / 2; 6467 len += sysfs_emit_at(buf, len, " C%d=%d(%d)", 6468 cpu, objects, slabs); 6469 } 6470 } 6471 #endif 6472 len += sysfs_emit_at(buf, len, "\n"); 6473 6474 return len; 6475 } 6476 SLAB_ATTR_RO(slabs_cpu_partial); 6477 6478 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) 6479 { 6480 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); 6481 } 6482 SLAB_ATTR_RO(reclaim_account); 6483 6484 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) 6485 { 6486 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); 6487 } 6488 SLAB_ATTR_RO(hwcache_align); 6489 6490 #ifdef CONFIG_ZONE_DMA 6491 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) 6492 { 6493 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); 6494 } 6495 SLAB_ATTR_RO(cache_dma); 6496 #endif 6497 6498 #ifdef CONFIG_HARDENED_USERCOPY 6499 static ssize_t usersize_show(struct kmem_cache *s, char *buf) 6500 { 6501 return sysfs_emit(buf, "%u\n", s->usersize); 6502 } 6503 SLAB_ATTR_RO(usersize); 6504 #endif 6505 6506 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) 6507 { 6508 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU)); 6509 } 6510 SLAB_ATTR_RO(destroy_by_rcu); 6511 6512 #ifdef CONFIG_SLUB_DEBUG 6513 static ssize_t slabs_show(struct kmem_cache *s, char *buf) 6514 { 6515 return show_slab_objects(s, buf, SO_ALL); 6516 } 6517 SLAB_ATTR_RO(slabs); 6518 6519 static ssize_t total_objects_show(struct kmem_cache *s, char *buf) 6520 { 6521 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); 6522 } 6523 SLAB_ATTR_RO(total_objects); 6524 6525 static ssize_t objects_show(struct kmem_cache *s, char *buf) 6526 { 6527 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); 6528 } 6529 SLAB_ATTR_RO(objects); 6530 6531 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) 6532 { 6533 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS)); 6534 } 6535 SLAB_ATTR_RO(sanity_checks); 6536 6537 static ssize_t trace_show(struct kmem_cache *s, char *buf) 6538 { 6539 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE)); 6540 } 6541 SLAB_ATTR_RO(trace); 6542 6543 static ssize_t red_zone_show(struct kmem_cache *s, char *buf) 6544 { 6545 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); 6546 } 6547 6548 SLAB_ATTR_RO(red_zone); 6549 6550 static ssize_t poison_show(struct kmem_cache *s, char *buf) 6551 { 6552 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON)); 6553 } 6554 6555 SLAB_ATTR_RO(poison); 6556 6557 static ssize_t store_user_show(struct kmem_cache *s, char *buf) 6558 { 6559 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); 6560 } 6561 6562 SLAB_ATTR_RO(store_user); 6563 6564 static ssize_t validate_show(struct kmem_cache *s, char *buf) 6565 { 6566 return 0; 6567 } 6568 6569 static ssize_t validate_store(struct kmem_cache *s, 6570 const char *buf, size_t length) 6571 { 6572 int ret = -EINVAL; 6573 6574 if (buf[0] == '1' && kmem_cache_debug(s)) { 6575 ret = validate_slab_cache(s); 6576 if (ret >= 0) 6577 ret = length; 6578 } 6579 return ret; 6580 } 6581 SLAB_ATTR(validate); 6582 6583 #endif /* CONFIG_SLUB_DEBUG */ 6584 6585 #ifdef CONFIG_FAILSLAB 6586 static ssize_t failslab_show(struct kmem_cache *s, char *buf) 6587 { 6588 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); 6589 } 6590 6591 static ssize_t failslab_store(struct kmem_cache *s, const char *buf, 6592 size_t length) 6593 { 6594 if (s->refcount > 1) 6595 return -EINVAL; 6596 6597 if (buf[0] == '1') 6598 WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB); 6599 else 6600 WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB); 6601 6602 return length; 6603 } 6604 SLAB_ATTR(failslab); 6605 #endif 6606 6607 static ssize_t shrink_show(struct kmem_cache *s, char *buf) 6608 { 6609 return 0; 6610 } 6611 6612 static ssize_t shrink_store(struct kmem_cache *s, 6613 const char *buf, size_t length) 6614 { 6615 if (buf[0] == '1') 6616 kmem_cache_shrink(s); 6617 else 6618 return -EINVAL; 6619 return length; 6620 } 6621 SLAB_ATTR(shrink); 6622 6623 #ifdef CONFIG_NUMA 6624 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) 6625 { 6626 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10); 6627 } 6628 6629 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, 6630 const char *buf, size_t length) 6631 { 6632 unsigned int ratio; 6633 int err; 6634 6635 err = kstrtouint(buf, 10, &ratio); 6636 if (err) 6637 return err; 6638 if (ratio > 100) 6639 return -ERANGE; 6640 6641 s->remote_node_defrag_ratio = ratio * 10; 6642 6643 return length; 6644 } 6645 SLAB_ATTR(remote_node_defrag_ratio); 6646 #endif 6647 6648 #ifdef CONFIG_SLUB_STATS 6649 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) 6650 { 6651 unsigned long sum = 0; 6652 int cpu; 6653 int len = 0; 6654 int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL); 6655 6656 if (!data) 6657 return -ENOMEM; 6658 6659 for_each_online_cpu(cpu) { 6660 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; 6661 6662 data[cpu] = x; 6663 sum += x; 6664 } 6665 6666 len += sysfs_emit_at(buf, len, "%lu", sum); 6667 6668 #ifdef CONFIG_SMP 6669 for_each_online_cpu(cpu) { 6670 if (data[cpu]) 6671 len += sysfs_emit_at(buf, len, " C%d=%u", 6672 cpu, data[cpu]); 6673 } 6674 #endif 6675 kfree(data); 6676 len += sysfs_emit_at(buf, len, "\n"); 6677 6678 return len; 6679 } 6680 6681 static void clear_stat(struct kmem_cache *s, enum stat_item si) 6682 { 6683 int cpu; 6684 6685 for_each_online_cpu(cpu) 6686 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; 6687 } 6688 6689 #define STAT_ATTR(si, text) \ 6690 static ssize_t text##_show(struct kmem_cache *s, char *buf) \ 6691 { \ 6692 return show_stat(s, buf, si); \ 6693 } \ 6694 static ssize_t text##_store(struct kmem_cache *s, \ 6695 const char *buf, size_t length) \ 6696 { \ 6697 if (buf[0] != '0') \ 6698 return -EINVAL; \ 6699 clear_stat(s, si); \ 6700 return length; \ 6701 } \ 6702 SLAB_ATTR(text); \ 6703 6704 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 6705 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 6706 STAT_ATTR(FREE_FASTPATH, free_fastpath); 6707 STAT_ATTR(FREE_SLOWPATH, free_slowpath); 6708 STAT_ATTR(FREE_FROZEN, free_frozen); 6709 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 6710 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 6711 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 6712 STAT_ATTR(ALLOC_SLAB, alloc_slab); 6713 STAT_ATTR(ALLOC_REFILL, alloc_refill); 6714 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); 6715 STAT_ATTR(FREE_SLAB, free_slab); 6716 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 6717 STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 6718 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 6719 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); 6720 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); 6721 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 6722 STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); 6723 STAT_ATTR(ORDER_FALLBACK, order_fallback); 6724 STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 6725 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); 6726 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); 6727 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); 6728 STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); 6729 STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); 6730 #endif /* CONFIG_SLUB_STATS */ 6731 6732 #ifdef CONFIG_KFENCE 6733 static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf) 6734 { 6735 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE)); 6736 } 6737 6738 static ssize_t skip_kfence_store(struct kmem_cache *s, 6739 const char *buf, size_t length) 6740 { 6741 int ret = length; 6742 6743 if (buf[0] == '0') 6744 s->flags &= ~SLAB_SKIP_KFENCE; 6745 else if (buf[0] == '1') 6746 s->flags |= SLAB_SKIP_KFENCE; 6747 else 6748 ret = -EINVAL; 6749 6750 return ret; 6751 } 6752 SLAB_ATTR(skip_kfence); 6753 #endif 6754 6755 static struct attribute *slab_attrs[] = { 6756 &slab_size_attr.attr, 6757 &object_size_attr.attr, 6758 &objs_per_slab_attr.attr, 6759 &order_attr.attr, 6760 &min_partial_attr.attr, 6761 &cpu_partial_attr.attr, 6762 &objects_partial_attr.attr, 6763 &partial_attr.attr, 6764 &cpu_slabs_attr.attr, 6765 &ctor_attr.attr, 6766 &aliases_attr.attr, 6767 &align_attr.attr, 6768 &hwcache_align_attr.attr, 6769 &reclaim_account_attr.attr, 6770 &destroy_by_rcu_attr.attr, 6771 &shrink_attr.attr, 6772 &slabs_cpu_partial_attr.attr, 6773 #ifdef CONFIG_SLUB_DEBUG 6774 &total_objects_attr.attr, 6775 &objects_attr.attr, 6776 &slabs_attr.attr, 6777 &sanity_checks_attr.attr, 6778 &trace_attr.attr, 6779 &red_zone_attr.attr, 6780 &poison_attr.attr, 6781 &store_user_attr.attr, 6782 &validate_attr.attr, 6783 #endif 6784 #ifdef CONFIG_ZONE_DMA 6785 &cache_dma_attr.attr, 6786 #endif 6787 #ifdef CONFIG_NUMA 6788 &remote_node_defrag_ratio_attr.attr, 6789 #endif 6790 #ifdef CONFIG_SLUB_STATS 6791 &alloc_fastpath_attr.attr, 6792 &alloc_slowpath_attr.attr, 6793 &free_fastpath_attr.attr, 6794 &free_slowpath_attr.attr, 6795 &free_frozen_attr.attr, 6796 &free_add_partial_attr.attr, 6797 &free_remove_partial_attr.attr, 6798 &alloc_from_partial_attr.attr, 6799 &alloc_slab_attr.attr, 6800 &alloc_refill_attr.attr, 6801 &alloc_node_mismatch_attr.attr, 6802 &free_slab_attr.attr, 6803 &cpuslab_flush_attr.attr, 6804 &deactivate_full_attr.attr, 6805 &deactivate_empty_attr.attr, 6806 &deactivate_to_head_attr.attr, 6807 &deactivate_to_tail_attr.attr, 6808 &deactivate_remote_frees_attr.attr, 6809 &deactivate_bypass_attr.attr, 6810 &order_fallback_attr.attr, 6811 &cmpxchg_double_fail_attr.attr, 6812 &cmpxchg_double_cpu_fail_attr.attr, 6813 &cpu_partial_alloc_attr.attr, 6814 &cpu_partial_free_attr.attr, 6815 &cpu_partial_node_attr.attr, 6816 &cpu_partial_drain_attr.attr, 6817 #endif 6818 #ifdef CONFIG_FAILSLAB 6819 &failslab_attr.attr, 6820 #endif 6821 #ifdef CONFIG_HARDENED_USERCOPY 6822 &usersize_attr.attr, 6823 #endif 6824 #ifdef CONFIG_KFENCE 6825 &skip_kfence_attr.attr, 6826 #endif 6827 6828 NULL 6829 }; 6830 6831 static const struct attribute_group slab_attr_group = { 6832 .attrs = slab_attrs, 6833 }; 6834 6835 static ssize_t slab_attr_show(struct kobject *kobj, 6836 struct attribute *attr, 6837 char *buf) 6838 { 6839 struct slab_attribute *attribute; 6840 struct kmem_cache *s; 6841 6842 attribute = to_slab_attr(attr); 6843 s = to_slab(kobj); 6844 6845 if (!attribute->show) 6846 return -EIO; 6847 6848 return attribute->show(s, buf); 6849 } 6850 6851 static ssize_t slab_attr_store(struct kobject *kobj, 6852 struct attribute *attr, 6853 const char *buf, size_t len) 6854 { 6855 struct slab_attribute *attribute; 6856 struct kmem_cache *s; 6857 6858 attribute = to_slab_attr(attr); 6859 s = to_slab(kobj); 6860 6861 if (!attribute->store) 6862 return -EIO; 6863 6864 return attribute->store(s, buf, len); 6865 } 6866 6867 static void kmem_cache_release(struct kobject *k) 6868 { 6869 slab_kmem_cache_release(to_slab(k)); 6870 } 6871 6872 static const struct sysfs_ops slab_sysfs_ops = { 6873 .show = slab_attr_show, 6874 .store = slab_attr_store, 6875 }; 6876 6877 static const struct kobj_type slab_ktype = { 6878 .sysfs_ops = &slab_sysfs_ops, 6879 .release = kmem_cache_release, 6880 }; 6881 6882 static struct kset *slab_kset; 6883 6884 static inline struct kset *cache_kset(struct kmem_cache *s) 6885 { 6886 return slab_kset; 6887 } 6888 6889 #define ID_STR_LENGTH 32 6890 6891 /* Create a unique string id for a slab cache: 6892 * 6893 * Format :[flags-]size 6894 */ 6895 static char *create_unique_id(struct kmem_cache *s) 6896 { 6897 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); 6898 char *p = name; 6899 6900 if (!name) 6901 return ERR_PTR(-ENOMEM); 6902 6903 *p++ = ':'; 6904 /* 6905 * First flags affecting slabcache operations. We will only 6906 * get here for aliasable slabs so we do not need to support 6907 * too many flags. The flags here must cover all flags that 6908 * are matched during merging to guarantee that the id is 6909 * unique. 6910 */ 6911 if (s->flags & SLAB_CACHE_DMA) 6912 *p++ = 'd'; 6913 if (s->flags & SLAB_CACHE_DMA32) 6914 *p++ = 'D'; 6915 if (s->flags & SLAB_RECLAIM_ACCOUNT) 6916 *p++ = 'a'; 6917 if (s->flags & SLAB_CONSISTENCY_CHECKS) 6918 *p++ = 'F'; 6919 if (s->flags & SLAB_ACCOUNT) 6920 *p++ = 'A'; 6921 if (p != name + 1) 6922 *p++ = '-'; 6923 p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size); 6924 6925 if (WARN_ON(p > name + ID_STR_LENGTH - 1)) { 6926 kfree(name); 6927 return ERR_PTR(-EINVAL); 6928 } 6929 kmsan_unpoison_memory(name, p - name); 6930 return name; 6931 } 6932 6933 static int sysfs_slab_add(struct kmem_cache *s) 6934 { 6935 int err; 6936 const char *name; 6937 struct kset *kset = cache_kset(s); 6938 int unmergeable = slab_unmergeable(s); 6939 6940 if (!unmergeable && disable_higher_order_debug && 6941 (slub_debug & DEBUG_METADATA_FLAGS)) 6942 unmergeable = 1; 6943 6944 if (unmergeable) { 6945 /* 6946 * Slabcache can never be merged so we can use the name proper. 6947 * This is typically the case for debug situations. In that 6948 * case we can catch duplicate names easily. 6949 */ 6950 sysfs_remove_link(&slab_kset->kobj, s->name); 6951 name = s->name; 6952 } else { 6953 /* 6954 * Create a unique name for the slab as a target 6955 * for the symlinks. 6956 */ 6957 name = create_unique_id(s); 6958 if (IS_ERR(name)) 6959 return PTR_ERR(name); 6960 } 6961 6962 s->kobj.kset = kset; 6963 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); 6964 if (err) 6965 goto out; 6966 6967 err = sysfs_create_group(&s->kobj, &slab_attr_group); 6968 if (err) 6969 goto out_del_kobj; 6970 6971 if (!unmergeable) { 6972 /* Setup first alias */ 6973 sysfs_slab_alias(s, s->name); 6974 } 6975 out: 6976 if (!unmergeable) 6977 kfree(name); 6978 return err; 6979 out_del_kobj: 6980 kobject_del(&s->kobj); 6981 goto out; 6982 } 6983 6984 void sysfs_slab_unlink(struct kmem_cache *s) 6985 { 6986 kobject_del(&s->kobj); 6987 } 6988 6989 void sysfs_slab_release(struct kmem_cache *s) 6990 { 6991 kobject_put(&s->kobj); 6992 } 6993 6994 /* 6995 * Need to buffer aliases during bootup until sysfs becomes 6996 * available lest we lose that information. 6997 */ 6998 struct saved_alias { 6999 struct kmem_cache *s; 7000 const char *name; 7001 struct saved_alias *next; 7002 }; 7003 7004 static struct saved_alias *alias_list; 7005 7006 static int sysfs_slab_alias(struct kmem_cache *s, const char *name) 7007 { 7008 struct saved_alias *al; 7009 7010 if (slab_state == FULL) { 7011 /* 7012 * If we have a leftover link then remove it. 7013 */ 7014 sysfs_remove_link(&slab_kset->kobj, name); 7015 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 7016 } 7017 7018 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); 7019 if (!al) 7020 return -ENOMEM; 7021 7022 al->s = s; 7023 al->name = name; 7024 al->next = alias_list; 7025 alias_list = al; 7026 kmsan_unpoison_memory(al, sizeof(*al)); 7027 return 0; 7028 } 7029 7030 static int __init slab_sysfs_init(void) 7031 { 7032 struct kmem_cache *s; 7033 int err; 7034 7035 mutex_lock(&slab_mutex); 7036 7037 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj); 7038 if (!slab_kset) { 7039 mutex_unlock(&slab_mutex); 7040 pr_err("Cannot register slab subsystem.\n"); 7041 return -ENOMEM; 7042 } 7043 7044 slab_state = FULL; 7045 7046 list_for_each_entry(s, &slab_caches, list) { 7047 err = sysfs_slab_add(s); 7048 if (err) 7049 pr_err("SLUB: Unable to add boot slab %s to sysfs\n", 7050 s->name); 7051 } 7052 7053 while (alias_list) { 7054 struct saved_alias *al = alias_list; 7055 7056 alias_list = alias_list->next; 7057 err = sysfs_slab_alias(al->s, al->name); 7058 if (err) 7059 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", 7060 al->name); 7061 kfree(al); 7062 } 7063 7064 mutex_unlock(&slab_mutex); 7065 return 0; 7066 } 7067 late_initcall(slab_sysfs_init); 7068 #endif /* SLAB_SUPPORTS_SYSFS */ 7069 7070 #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS) 7071 static int slab_debugfs_show(struct seq_file *seq, void *v) 7072 { 7073 struct loc_track *t = seq->private; 7074 struct location *l; 7075 unsigned long idx; 7076 7077 idx = (unsigned long) t->idx; 7078 if (idx < t->count) { 7079 l = &t->loc[idx]; 7080 7081 seq_printf(seq, "%7ld ", l->count); 7082 7083 if (l->addr) 7084 seq_printf(seq, "%pS", (void *)l->addr); 7085 else 7086 seq_puts(seq, "<not-available>"); 7087 7088 if (l->waste) 7089 seq_printf(seq, " waste=%lu/%lu", 7090 l->count * l->waste, l->waste); 7091 7092 if (l->sum_time != l->min_time) { 7093 seq_printf(seq, " age=%ld/%llu/%ld", 7094 l->min_time, div_u64(l->sum_time, l->count), 7095 l->max_time); 7096 } else 7097 seq_printf(seq, " age=%ld", l->min_time); 7098 7099 if (l->min_pid != l->max_pid) 7100 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid); 7101 else 7102 seq_printf(seq, " pid=%ld", 7103 l->min_pid); 7104 7105 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus))) 7106 seq_printf(seq, " cpus=%*pbl", 7107 cpumask_pr_args(to_cpumask(l->cpus))); 7108 7109 if (nr_online_nodes > 1 && !nodes_empty(l->nodes)) 7110 seq_printf(seq, " nodes=%*pbl", 7111 nodemask_pr_args(&l->nodes)); 7112 7113 #ifdef CONFIG_STACKDEPOT 7114 { 7115 depot_stack_handle_t handle; 7116 unsigned long *entries; 7117 unsigned int nr_entries, j; 7118 7119 handle = READ_ONCE(l->handle); 7120 if (handle) { 7121 nr_entries = stack_depot_fetch(handle, &entries); 7122 seq_puts(seq, "\n"); 7123 for (j = 0; j < nr_entries; j++) 7124 seq_printf(seq, " %pS\n", (void *)entries[j]); 7125 } 7126 } 7127 #endif 7128 seq_puts(seq, "\n"); 7129 } 7130 7131 if (!idx && !t->count) 7132 seq_puts(seq, "No data\n"); 7133 7134 return 0; 7135 } 7136 7137 static void slab_debugfs_stop(struct seq_file *seq, void *v) 7138 { 7139 } 7140 7141 static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos) 7142 { 7143 struct loc_track *t = seq->private; 7144 7145 t->idx = ++(*ppos); 7146 if (*ppos <= t->count) 7147 return ppos; 7148 7149 return NULL; 7150 } 7151 7152 static int cmp_loc_by_count(const void *a, const void *b, const void *data) 7153 { 7154 struct location *loc1 = (struct location *)a; 7155 struct location *loc2 = (struct location *)b; 7156 7157 if (loc1->count > loc2->count) 7158 return -1; 7159 else 7160 return 1; 7161 } 7162 7163 static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos) 7164 { 7165 struct loc_track *t = seq->private; 7166 7167 t->idx = *ppos; 7168 return ppos; 7169 } 7170 7171 static const struct seq_operations slab_debugfs_sops = { 7172 .start = slab_debugfs_start, 7173 .next = slab_debugfs_next, 7174 .stop = slab_debugfs_stop, 7175 .show = slab_debugfs_show, 7176 }; 7177 7178 static int slab_debug_trace_open(struct inode *inode, struct file *filep) 7179 { 7180 7181 struct kmem_cache_node *n; 7182 enum track_item alloc; 7183 int node; 7184 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops, 7185 sizeof(struct loc_track)); 7186 struct kmem_cache *s = file_inode(filep)->i_private; 7187 unsigned long *obj_map; 7188 7189 if (!t) 7190 return -ENOMEM; 7191 7192 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); 7193 if (!obj_map) { 7194 seq_release_private(inode, filep); 7195 return -ENOMEM; 7196 } 7197 7198 if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0) 7199 alloc = TRACK_ALLOC; 7200 else 7201 alloc = TRACK_FREE; 7202 7203 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) { 7204 bitmap_free(obj_map); 7205 seq_release_private(inode, filep); 7206 return -ENOMEM; 7207 } 7208 7209 for_each_kmem_cache_node(s, node, n) { 7210 unsigned long flags; 7211 struct slab *slab; 7212 7213 if (!node_nr_slabs(n)) 7214 continue; 7215 7216 spin_lock_irqsave(&n->list_lock, flags); 7217 list_for_each_entry(slab, &n->partial, slab_list) 7218 process_slab(t, s, slab, alloc, obj_map); 7219 list_for_each_entry(slab, &n->full, slab_list) 7220 process_slab(t, s, slab, alloc, obj_map); 7221 spin_unlock_irqrestore(&n->list_lock, flags); 7222 } 7223 7224 /* Sort locations by count */ 7225 sort_r(t->loc, t->count, sizeof(struct location), 7226 cmp_loc_by_count, NULL, NULL); 7227 7228 bitmap_free(obj_map); 7229 return 0; 7230 } 7231 7232 static int slab_debug_trace_release(struct inode *inode, struct file *file) 7233 { 7234 struct seq_file *seq = file->private_data; 7235 struct loc_track *t = seq->private; 7236 7237 free_loc_track(t); 7238 return seq_release_private(inode, file); 7239 } 7240 7241 static const struct file_operations slab_debugfs_fops = { 7242 .open = slab_debug_trace_open, 7243 .read = seq_read, 7244 .llseek = seq_lseek, 7245 .release = slab_debug_trace_release, 7246 }; 7247 7248 static void debugfs_slab_add(struct kmem_cache *s) 7249 { 7250 struct dentry *slab_cache_dir; 7251 7252 if (unlikely(!slab_debugfs_root)) 7253 return; 7254 7255 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root); 7256 7257 debugfs_create_file("alloc_traces", 0400, 7258 slab_cache_dir, s, &slab_debugfs_fops); 7259 7260 debugfs_create_file("free_traces", 0400, 7261 slab_cache_dir, s, &slab_debugfs_fops); 7262 } 7263 7264 void debugfs_slab_release(struct kmem_cache *s) 7265 { 7266 debugfs_lookup_and_remove(s->name, slab_debugfs_root); 7267 } 7268 7269 static int __init slab_debugfs_init(void) 7270 { 7271 struct kmem_cache *s; 7272 7273 slab_debugfs_root = debugfs_create_dir("slab", NULL); 7274 7275 list_for_each_entry(s, &slab_caches, list) 7276 if (s->flags & SLAB_STORE_USER) 7277 debugfs_slab_add(s); 7278 7279 return 0; 7280 7281 } 7282 __initcall(slab_debugfs_init); 7283 #endif 7284 /* 7285 * The /proc/slabinfo ABI 7286 */ 7287 #ifdef CONFIG_SLUB_DEBUG 7288 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) 7289 { 7290 unsigned long nr_slabs = 0; 7291 unsigned long nr_objs = 0; 7292 unsigned long nr_free = 0; 7293 int node; 7294 struct kmem_cache_node *n; 7295 7296 for_each_kmem_cache_node(s, node, n) { 7297 nr_slabs += node_nr_slabs(n); 7298 nr_objs += node_nr_objs(n); 7299 nr_free += count_partial_free_approx(n); 7300 } 7301 7302 sinfo->active_objs = nr_objs - nr_free; 7303 sinfo->num_objs = nr_objs; 7304 sinfo->active_slabs = nr_slabs; 7305 sinfo->num_slabs = nr_slabs; 7306 sinfo->objects_per_slab = oo_objects(s->oo); 7307 sinfo->cache_order = oo_order(s->oo); 7308 } 7309 #endif /* CONFIG_SLUB_DEBUG */ 7310