xref: /linux/mm/slub.c (revision f73a058be5d70dd81a43f16b2bbff4b1576a7af8)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * SLUB: A slab allocator that limits cache line use instead of queuing
4  * objects in per cpu and per node lists.
5  *
6  * The allocator synchronizes using per slab locks or atomic operations
7  * and only uses a centralized lock to manage a pool of partial slabs.
8  *
9  * (C) 2007 SGI, Christoph Lameter
10  * (C) 2011 Linux Foundation, Christoph Lameter
11  */
12 
13 #include <linux/mm.h>
14 #include <linux/swap.h> /* mm_account_reclaimed_pages() */
15 #include <linux/module.h>
16 #include <linux/bit_spinlock.h>
17 #include <linux/interrupt.h>
18 #include <linux/swab.h>
19 #include <linux/bitops.h>
20 #include <linux/slab.h>
21 #include "slab.h"
22 #include <linux/proc_fs.h>
23 #include <linux/seq_file.h>
24 #include <linux/kasan.h>
25 #include <linux/kmsan.h>
26 #include <linux/cpu.h>
27 #include <linux/cpuset.h>
28 #include <linux/mempolicy.h>
29 #include <linux/ctype.h>
30 #include <linux/stackdepot.h>
31 #include <linux/debugobjects.h>
32 #include <linux/kallsyms.h>
33 #include <linux/kfence.h>
34 #include <linux/memory.h>
35 #include <linux/math64.h>
36 #include <linux/fault-inject.h>
37 #include <linux/kmemleak.h>
38 #include <linux/stacktrace.h>
39 #include <linux/prefetch.h>
40 #include <linux/memcontrol.h>
41 #include <linux/random.h>
42 #include <kunit/test.h>
43 #include <kunit/test-bug.h>
44 #include <linux/sort.h>
45 
46 #include <linux/debugfs.h>
47 #include <trace/events/kmem.h>
48 
49 #include "internal.h"
50 
51 /*
52  * Lock order:
53  *   1. slab_mutex (Global Mutex)
54  *   2. node->list_lock (Spinlock)
55  *   3. kmem_cache->cpu_slab->lock (Local lock)
56  *   4. slab_lock(slab) (Only on some arches)
57  *   5. object_map_lock (Only for debugging)
58  *
59  *   slab_mutex
60  *
61  *   The role of the slab_mutex is to protect the list of all the slabs
62  *   and to synchronize major metadata changes to slab cache structures.
63  *   Also synchronizes memory hotplug callbacks.
64  *
65  *   slab_lock
66  *
67  *   The slab_lock is a wrapper around the page lock, thus it is a bit
68  *   spinlock.
69  *
70  *   The slab_lock is only used on arches that do not have the ability
71  *   to do a cmpxchg_double. It only protects:
72  *
73  *	A. slab->freelist	-> List of free objects in a slab
74  *	B. slab->inuse		-> Number of objects in use
75  *	C. slab->objects	-> Number of objects in slab
76  *	D. slab->frozen		-> frozen state
77  *
78  *   Frozen slabs
79  *
80  *   If a slab is frozen then it is exempt from list management. It is
81  *   the cpu slab which is actively allocated from by the processor that
82  *   froze it and it is not on any list. The processor that froze the
83  *   slab is the one who can perform list operations on the slab. Other
84  *   processors may put objects onto the freelist but the processor that
85  *   froze the slab is the only one that can retrieve the objects from the
86  *   slab's freelist.
87  *
88  *   CPU partial slabs
89  *
90  *   The partially empty slabs cached on the CPU partial list are used
91  *   for performance reasons, which speeds up the allocation process.
92  *   These slabs are not frozen, but are also exempt from list management,
93  *   by clearing the PG_workingset flag when moving out of the node
94  *   partial list. Please see __slab_free() for more details.
95  *
96  *   To sum up, the current scheme is:
97  *   - node partial slab: PG_Workingset && !frozen
98  *   - cpu partial slab: !PG_Workingset && !frozen
99  *   - cpu slab: !PG_Workingset && frozen
100  *   - full slab: !PG_Workingset && !frozen
101  *
102  *   list_lock
103  *
104  *   The list_lock protects the partial and full list on each node and
105  *   the partial slab counter. If taken then no new slabs may be added or
106  *   removed from the lists nor make the number of partial slabs be modified.
107  *   (Note that the total number of slabs is an atomic value that may be
108  *   modified without taking the list lock).
109  *
110  *   The list_lock is a centralized lock and thus we avoid taking it as
111  *   much as possible. As long as SLUB does not have to handle partial
112  *   slabs, operations can continue without any centralized lock. F.e.
113  *   allocating a long series of objects that fill up slabs does not require
114  *   the list lock.
115  *
116  *   For debug caches, all allocations are forced to go through a list_lock
117  *   protected region to serialize against concurrent validation.
118  *
119  *   cpu_slab->lock local lock
120  *
121  *   This locks protect slowpath manipulation of all kmem_cache_cpu fields
122  *   except the stat counters. This is a percpu structure manipulated only by
123  *   the local cpu, so the lock protects against being preempted or interrupted
124  *   by an irq. Fast path operations rely on lockless operations instead.
125  *
126  *   On PREEMPT_RT, the local lock neither disables interrupts nor preemption
127  *   which means the lockless fastpath cannot be used as it might interfere with
128  *   an in-progress slow path operations. In this case the local lock is always
129  *   taken but it still utilizes the freelist for the common operations.
130  *
131  *   lockless fastpaths
132  *
133  *   The fast path allocation (slab_alloc_node()) and freeing (do_slab_free())
134  *   are fully lockless when satisfied from the percpu slab (and when
135  *   cmpxchg_double is possible to use, otherwise slab_lock is taken).
136  *   They also don't disable preemption or migration or irqs. They rely on
137  *   the transaction id (tid) field to detect being preempted or moved to
138  *   another cpu.
139  *
140  *   irq, preemption, migration considerations
141  *
142  *   Interrupts are disabled as part of list_lock or local_lock operations, or
143  *   around the slab_lock operation, in order to make the slab allocator safe
144  *   to use in the context of an irq.
145  *
146  *   In addition, preemption (or migration on PREEMPT_RT) is disabled in the
147  *   allocation slowpath, bulk allocation, and put_cpu_partial(), so that the
148  *   local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer
149  *   doesn't have to be revalidated in each section protected by the local lock.
150  *
151  * SLUB assigns one slab for allocation to each processor.
152  * Allocations only occur from these slabs called cpu slabs.
153  *
154  * Slabs with free elements are kept on a partial list and during regular
155  * operations no list for full slabs is used. If an object in a full slab is
156  * freed then the slab will show up again on the partial lists.
157  * We track full slabs for debugging purposes though because otherwise we
158  * cannot scan all objects.
159  *
160  * Slabs are freed when they become empty. Teardown and setup is
161  * minimal so we rely on the page allocators per cpu caches for
162  * fast frees and allocs.
163  *
164  * slab->frozen		The slab is frozen and exempt from list processing.
165  * 			This means that the slab is dedicated to a purpose
166  * 			such as satisfying allocations for a specific
167  * 			processor. Objects may be freed in the slab while
168  * 			it is frozen but slab_free will then skip the usual
169  * 			list operations. It is up to the processor holding
170  * 			the slab to integrate the slab into the slab lists
171  * 			when the slab is no longer needed.
172  *
173  * 			One use of this flag is to mark slabs that are
174  * 			used for allocations. Then such a slab becomes a cpu
175  * 			slab. The cpu slab may be equipped with an additional
176  * 			freelist that allows lockless access to
177  * 			free objects in addition to the regular freelist
178  * 			that requires the slab lock.
179  *
180  * SLAB_DEBUG_FLAGS	Slab requires special handling due to debug
181  * 			options set. This moves	slab handling out of
182  * 			the fast path and disables lockless freelists.
183  */
184 
185 /*
186  * We could simply use migrate_disable()/enable() but as long as it's a
187  * function call even on !PREEMPT_RT, use inline preempt_disable() there.
188  */
189 #ifndef CONFIG_PREEMPT_RT
190 #define slub_get_cpu_ptr(var)		get_cpu_ptr(var)
191 #define slub_put_cpu_ptr(var)		put_cpu_ptr(var)
192 #define USE_LOCKLESS_FAST_PATH()	(true)
193 #else
194 #define slub_get_cpu_ptr(var)		\
195 ({					\
196 	migrate_disable();		\
197 	this_cpu_ptr(var);		\
198 })
199 #define slub_put_cpu_ptr(var)		\
200 do {					\
201 	(void)(var);			\
202 	migrate_enable();		\
203 } while (0)
204 #define USE_LOCKLESS_FAST_PATH()	(false)
205 #endif
206 
207 #ifndef CONFIG_SLUB_TINY
208 #define __fastpath_inline __always_inline
209 #else
210 #define __fastpath_inline
211 #endif
212 
213 #ifdef CONFIG_SLUB_DEBUG
214 #ifdef CONFIG_SLUB_DEBUG_ON
215 DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
216 #else
217 DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
218 #endif
219 #endif		/* CONFIG_SLUB_DEBUG */
220 
221 /* Structure holding parameters for get_partial() call chain */
222 struct partial_context {
223 	gfp_t flags;
224 	unsigned int orig_size;
225 	void *object;
226 };
227 
228 static inline bool kmem_cache_debug(struct kmem_cache *s)
229 {
230 	return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
231 }
232 
233 static inline bool slub_debug_orig_size(struct kmem_cache *s)
234 {
235 	return (kmem_cache_debug_flags(s, SLAB_STORE_USER) &&
236 			(s->flags & SLAB_KMALLOC));
237 }
238 
239 void *fixup_red_left(struct kmem_cache *s, void *p)
240 {
241 	if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
242 		p += s->red_left_pad;
243 
244 	return p;
245 }
246 
247 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
248 {
249 #ifdef CONFIG_SLUB_CPU_PARTIAL
250 	return !kmem_cache_debug(s);
251 #else
252 	return false;
253 #endif
254 }
255 
256 /*
257  * Issues still to be resolved:
258  *
259  * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
260  *
261  * - Variable sizing of the per node arrays
262  */
263 
264 /* Enable to log cmpxchg failures */
265 #undef SLUB_DEBUG_CMPXCHG
266 
267 #ifndef CONFIG_SLUB_TINY
268 /*
269  * Minimum number of partial slabs. These will be left on the partial
270  * lists even if they are empty. kmem_cache_shrink may reclaim them.
271  */
272 #define MIN_PARTIAL 5
273 
274 /*
275  * Maximum number of desirable partial slabs.
276  * The existence of more partial slabs makes kmem_cache_shrink
277  * sort the partial list by the number of objects in use.
278  */
279 #define MAX_PARTIAL 10
280 #else
281 #define MIN_PARTIAL 0
282 #define MAX_PARTIAL 0
283 #endif
284 
285 #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
286 				SLAB_POISON | SLAB_STORE_USER)
287 
288 /*
289  * These debug flags cannot use CMPXCHG because there might be consistency
290  * issues when checking or reading debug information
291  */
292 #define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
293 				SLAB_TRACE)
294 
295 
296 /*
297  * Debugging flags that require metadata to be stored in the slab.  These get
298  * disabled when slab_debug=O is used and a cache's min order increases with
299  * metadata.
300  */
301 #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
302 
303 #define OO_SHIFT	16
304 #define OO_MASK		((1 << OO_SHIFT) - 1)
305 #define MAX_OBJS_PER_PAGE	32767 /* since slab.objects is u15 */
306 
307 /* Internal SLUB flags */
308 /* Poison object */
309 #define __OBJECT_POISON		__SLAB_FLAG_BIT(_SLAB_OBJECT_POISON)
310 /* Use cmpxchg_double */
311 
312 #ifdef system_has_freelist_aba
313 #define __CMPXCHG_DOUBLE	__SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE)
314 #else
315 #define __CMPXCHG_DOUBLE	__SLAB_FLAG_UNUSED
316 #endif
317 
318 /*
319  * Tracking user of a slab.
320  */
321 #define TRACK_ADDRS_COUNT 16
322 struct track {
323 	unsigned long addr;	/* Called from address */
324 #ifdef CONFIG_STACKDEPOT
325 	depot_stack_handle_t handle;
326 #endif
327 	int cpu;		/* Was running on cpu */
328 	int pid;		/* Pid context */
329 	unsigned long when;	/* When did the operation occur */
330 };
331 
332 enum track_item { TRACK_ALLOC, TRACK_FREE };
333 
334 #ifdef SLAB_SUPPORTS_SYSFS
335 static int sysfs_slab_add(struct kmem_cache *);
336 static int sysfs_slab_alias(struct kmem_cache *, const char *);
337 #else
338 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
339 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
340 							{ return 0; }
341 #endif
342 
343 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
344 static void debugfs_slab_add(struct kmem_cache *);
345 #else
346 static inline void debugfs_slab_add(struct kmem_cache *s) { }
347 #endif
348 
349 enum stat_item {
350 	ALLOC_FASTPATH,		/* Allocation from cpu slab */
351 	ALLOC_SLOWPATH,		/* Allocation by getting a new cpu slab */
352 	FREE_FASTPATH,		/* Free to cpu slab */
353 	FREE_SLOWPATH,		/* Freeing not to cpu slab */
354 	FREE_FROZEN,		/* Freeing to frozen slab */
355 	FREE_ADD_PARTIAL,	/* Freeing moves slab to partial list */
356 	FREE_REMOVE_PARTIAL,	/* Freeing removes last object */
357 	ALLOC_FROM_PARTIAL,	/* Cpu slab acquired from node partial list */
358 	ALLOC_SLAB,		/* Cpu slab acquired from page allocator */
359 	ALLOC_REFILL,		/* Refill cpu slab from slab freelist */
360 	ALLOC_NODE_MISMATCH,	/* Switching cpu slab */
361 	FREE_SLAB,		/* Slab freed to the page allocator */
362 	CPUSLAB_FLUSH,		/* Abandoning of the cpu slab */
363 	DEACTIVATE_FULL,	/* Cpu slab was full when deactivated */
364 	DEACTIVATE_EMPTY,	/* Cpu slab was empty when deactivated */
365 	DEACTIVATE_TO_HEAD,	/* Cpu slab was moved to the head of partials */
366 	DEACTIVATE_TO_TAIL,	/* Cpu slab was moved to the tail of partials */
367 	DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
368 	DEACTIVATE_BYPASS,	/* Implicit deactivation */
369 	ORDER_FALLBACK,		/* Number of times fallback was necessary */
370 	CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */
371 	CMPXCHG_DOUBLE_FAIL,	/* Failures of slab freelist update */
372 	CPU_PARTIAL_ALLOC,	/* Used cpu partial on alloc */
373 	CPU_PARTIAL_FREE,	/* Refill cpu partial on free */
374 	CPU_PARTIAL_NODE,	/* Refill cpu partial from node partial */
375 	CPU_PARTIAL_DRAIN,	/* Drain cpu partial to node partial */
376 	NR_SLUB_STAT_ITEMS
377 };
378 
379 #ifndef CONFIG_SLUB_TINY
380 /*
381  * When changing the layout, make sure freelist and tid are still compatible
382  * with this_cpu_cmpxchg_double() alignment requirements.
383  */
384 struct kmem_cache_cpu {
385 	union {
386 		struct {
387 			void **freelist;	/* Pointer to next available object */
388 			unsigned long tid;	/* Globally unique transaction id */
389 		};
390 		freelist_aba_t freelist_tid;
391 	};
392 	struct slab *slab;	/* The slab from which we are allocating */
393 #ifdef CONFIG_SLUB_CPU_PARTIAL
394 	struct slab *partial;	/* Partially allocated slabs */
395 #endif
396 	local_lock_t lock;	/* Protects the fields above */
397 #ifdef CONFIG_SLUB_STATS
398 	unsigned int stat[NR_SLUB_STAT_ITEMS];
399 #endif
400 };
401 #endif /* CONFIG_SLUB_TINY */
402 
403 static inline void stat(const struct kmem_cache *s, enum stat_item si)
404 {
405 #ifdef CONFIG_SLUB_STATS
406 	/*
407 	 * The rmw is racy on a preemptible kernel but this is acceptable, so
408 	 * avoid this_cpu_add()'s irq-disable overhead.
409 	 */
410 	raw_cpu_inc(s->cpu_slab->stat[si]);
411 #endif
412 }
413 
414 static inline
415 void stat_add(const struct kmem_cache *s, enum stat_item si, int v)
416 {
417 #ifdef CONFIG_SLUB_STATS
418 	raw_cpu_add(s->cpu_slab->stat[si], v);
419 #endif
420 }
421 
422 /*
423  * The slab lists for all objects.
424  */
425 struct kmem_cache_node {
426 	spinlock_t list_lock;
427 	unsigned long nr_partial;
428 	struct list_head partial;
429 #ifdef CONFIG_SLUB_DEBUG
430 	atomic_long_t nr_slabs;
431 	atomic_long_t total_objects;
432 	struct list_head full;
433 #endif
434 };
435 
436 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
437 {
438 	return s->node[node];
439 }
440 
441 /*
442  * Iterator over all nodes. The body will be executed for each node that has
443  * a kmem_cache_node structure allocated (which is true for all online nodes)
444  */
445 #define for_each_kmem_cache_node(__s, __node, __n) \
446 	for (__node = 0; __node < nr_node_ids; __node++) \
447 		 if ((__n = get_node(__s, __node)))
448 
449 /*
450  * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
451  * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
452  * differ during memory hotplug/hotremove operations.
453  * Protected by slab_mutex.
454  */
455 static nodemask_t slab_nodes;
456 
457 #ifndef CONFIG_SLUB_TINY
458 /*
459  * Workqueue used for flush_cpu_slab().
460  */
461 static struct workqueue_struct *flushwq;
462 #endif
463 
464 /********************************************************************
465  * 			Core slab cache functions
466  *******************************************************************/
467 
468 /*
469  * freeptr_t represents a SLUB freelist pointer, which might be encoded
470  * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
471  */
472 typedef struct { unsigned long v; } freeptr_t;
473 
474 /*
475  * Returns freelist pointer (ptr). With hardening, this is obfuscated
476  * with an XOR of the address where the pointer is held and a per-cache
477  * random number.
478  */
479 static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
480 					    void *ptr, unsigned long ptr_addr)
481 {
482 	unsigned long encoded;
483 
484 #ifdef CONFIG_SLAB_FREELIST_HARDENED
485 	encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr);
486 #else
487 	encoded = (unsigned long)ptr;
488 #endif
489 	return (freeptr_t){.v = encoded};
490 }
491 
492 static inline void *freelist_ptr_decode(const struct kmem_cache *s,
493 					freeptr_t ptr, unsigned long ptr_addr)
494 {
495 	void *decoded;
496 
497 #ifdef CONFIG_SLAB_FREELIST_HARDENED
498 	decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr));
499 #else
500 	decoded = (void *)ptr.v;
501 #endif
502 	return decoded;
503 }
504 
505 static inline void *get_freepointer(struct kmem_cache *s, void *object)
506 {
507 	unsigned long ptr_addr;
508 	freeptr_t p;
509 
510 	object = kasan_reset_tag(object);
511 	ptr_addr = (unsigned long)object + s->offset;
512 	p = *(freeptr_t *)(ptr_addr);
513 	return freelist_ptr_decode(s, p, ptr_addr);
514 }
515 
516 #ifndef CONFIG_SLUB_TINY
517 static void prefetch_freepointer(const struct kmem_cache *s, void *object)
518 {
519 	prefetchw(object + s->offset);
520 }
521 #endif
522 
523 /*
524  * When running under KMSAN, get_freepointer_safe() may return an uninitialized
525  * pointer value in the case the current thread loses the race for the next
526  * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
527  * slab_alloc_node() will fail, so the uninitialized value won't be used, but
528  * KMSAN will still check all arguments of cmpxchg because of imperfect
529  * handling of inline assembly.
530  * To work around this problem, we apply __no_kmsan_checks to ensure that
531  * get_freepointer_safe() returns initialized memory.
532  */
533 __no_kmsan_checks
534 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
535 {
536 	unsigned long freepointer_addr;
537 	freeptr_t p;
538 
539 	if (!debug_pagealloc_enabled_static())
540 		return get_freepointer(s, object);
541 
542 	object = kasan_reset_tag(object);
543 	freepointer_addr = (unsigned long)object + s->offset;
544 	copy_from_kernel_nofault(&p, (freeptr_t *)freepointer_addr, sizeof(p));
545 	return freelist_ptr_decode(s, p, freepointer_addr);
546 }
547 
548 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
549 {
550 	unsigned long freeptr_addr = (unsigned long)object + s->offset;
551 
552 #ifdef CONFIG_SLAB_FREELIST_HARDENED
553 	BUG_ON(object == fp); /* naive detection of double free or corruption */
554 #endif
555 
556 	freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
557 	*(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);
558 }
559 
560 /*
561  * See comment in calculate_sizes().
562  */
563 static inline bool freeptr_outside_object(struct kmem_cache *s)
564 {
565 	return s->offset >= s->inuse;
566 }
567 
568 /*
569  * Return offset of the end of info block which is inuse + free pointer if
570  * not overlapping with object.
571  */
572 static inline unsigned int get_info_end(struct kmem_cache *s)
573 {
574 	if (freeptr_outside_object(s))
575 		return s->inuse + sizeof(void *);
576 	else
577 		return s->inuse;
578 }
579 
580 /* Loop over all objects in a slab */
581 #define for_each_object(__p, __s, __addr, __objects) \
582 	for (__p = fixup_red_left(__s, __addr); \
583 		__p < (__addr) + (__objects) * (__s)->size; \
584 		__p += (__s)->size)
585 
586 static inline unsigned int order_objects(unsigned int order, unsigned int size)
587 {
588 	return ((unsigned int)PAGE_SIZE << order) / size;
589 }
590 
591 static inline struct kmem_cache_order_objects oo_make(unsigned int order,
592 		unsigned int size)
593 {
594 	struct kmem_cache_order_objects x = {
595 		(order << OO_SHIFT) + order_objects(order, size)
596 	};
597 
598 	return x;
599 }
600 
601 static inline unsigned int oo_order(struct kmem_cache_order_objects x)
602 {
603 	return x.x >> OO_SHIFT;
604 }
605 
606 static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
607 {
608 	return x.x & OO_MASK;
609 }
610 
611 #ifdef CONFIG_SLUB_CPU_PARTIAL
612 static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
613 {
614 	unsigned int nr_slabs;
615 
616 	s->cpu_partial = nr_objects;
617 
618 	/*
619 	 * We take the number of objects but actually limit the number of
620 	 * slabs on the per cpu partial list, in order to limit excessive
621 	 * growth of the list. For simplicity we assume that the slabs will
622 	 * be half-full.
623 	 */
624 	nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
625 	s->cpu_partial_slabs = nr_slabs;
626 }
627 
628 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s)
629 {
630 	return s->cpu_partial_slabs;
631 }
632 #else
633 static inline void
634 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
635 {
636 }
637 
638 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s)
639 {
640 	return 0;
641 }
642 #endif /* CONFIG_SLUB_CPU_PARTIAL */
643 
644 /*
645  * Per slab locking using the pagelock
646  */
647 static __always_inline void slab_lock(struct slab *slab)
648 {
649 	bit_spin_lock(PG_locked, &slab->__page_flags);
650 }
651 
652 static __always_inline void slab_unlock(struct slab *slab)
653 {
654 	bit_spin_unlock(PG_locked, &slab->__page_flags);
655 }
656 
657 static inline bool
658 __update_freelist_fast(struct slab *slab,
659 		      void *freelist_old, unsigned long counters_old,
660 		      void *freelist_new, unsigned long counters_new)
661 {
662 #ifdef system_has_freelist_aba
663 	freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old };
664 	freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new };
665 
666 	return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full);
667 #else
668 	return false;
669 #endif
670 }
671 
672 static inline bool
673 __update_freelist_slow(struct slab *slab,
674 		      void *freelist_old, unsigned long counters_old,
675 		      void *freelist_new, unsigned long counters_new)
676 {
677 	bool ret = false;
678 
679 	slab_lock(slab);
680 	if (slab->freelist == freelist_old &&
681 	    slab->counters == counters_old) {
682 		slab->freelist = freelist_new;
683 		slab->counters = counters_new;
684 		ret = true;
685 	}
686 	slab_unlock(slab);
687 
688 	return ret;
689 }
690 
691 /*
692  * Interrupts must be disabled (for the fallback code to work right), typically
693  * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is
694  * part of bit_spin_lock(), is sufficient because the policy is not to allow any
695  * allocation/ free operation in hardirq context. Therefore nothing can
696  * interrupt the operation.
697  */
698 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
699 		void *freelist_old, unsigned long counters_old,
700 		void *freelist_new, unsigned long counters_new,
701 		const char *n)
702 {
703 	bool ret;
704 
705 	if (USE_LOCKLESS_FAST_PATH())
706 		lockdep_assert_irqs_disabled();
707 
708 	if (s->flags & __CMPXCHG_DOUBLE) {
709 		ret = __update_freelist_fast(slab, freelist_old, counters_old,
710 				            freelist_new, counters_new);
711 	} else {
712 		ret = __update_freelist_slow(slab, freelist_old, counters_old,
713 				            freelist_new, counters_new);
714 	}
715 	if (likely(ret))
716 		return true;
717 
718 	cpu_relax();
719 	stat(s, CMPXCHG_DOUBLE_FAIL);
720 
721 #ifdef SLUB_DEBUG_CMPXCHG
722 	pr_info("%s %s: cmpxchg double redo ", n, s->name);
723 #endif
724 
725 	return false;
726 }
727 
728 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
729 		void *freelist_old, unsigned long counters_old,
730 		void *freelist_new, unsigned long counters_new,
731 		const char *n)
732 {
733 	bool ret;
734 
735 	if (s->flags & __CMPXCHG_DOUBLE) {
736 		ret = __update_freelist_fast(slab, freelist_old, counters_old,
737 				            freelist_new, counters_new);
738 	} else {
739 		unsigned long flags;
740 
741 		local_irq_save(flags);
742 		ret = __update_freelist_slow(slab, freelist_old, counters_old,
743 				            freelist_new, counters_new);
744 		local_irq_restore(flags);
745 	}
746 	if (likely(ret))
747 		return true;
748 
749 	cpu_relax();
750 	stat(s, CMPXCHG_DOUBLE_FAIL);
751 
752 #ifdef SLUB_DEBUG_CMPXCHG
753 	pr_info("%s %s: cmpxchg double redo ", n, s->name);
754 #endif
755 
756 	return false;
757 }
758 
759 #ifdef CONFIG_SLUB_DEBUG
760 static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
761 static DEFINE_SPINLOCK(object_map_lock);
762 
763 static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
764 		       struct slab *slab)
765 {
766 	void *addr = slab_address(slab);
767 	void *p;
768 
769 	bitmap_zero(obj_map, slab->objects);
770 
771 	for (p = slab->freelist; p; p = get_freepointer(s, p))
772 		set_bit(__obj_to_index(s, addr, p), obj_map);
773 }
774 
775 #if IS_ENABLED(CONFIG_KUNIT)
776 static bool slab_add_kunit_errors(void)
777 {
778 	struct kunit_resource *resource;
779 
780 	if (!kunit_get_current_test())
781 		return false;
782 
783 	resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
784 	if (!resource)
785 		return false;
786 
787 	(*(int *)resource->data)++;
788 	kunit_put_resource(resource);
789 	return true;
790 }
791 #else
792 static inline bool slab_add_kunit_errors(void) { return false; }
793 #endif
794 
795 static inline unsigned int size_from_object(struct kmem_cache *s)
796 {
797 	if (s->flags & SLAB_RED_ZONE)
798 		return s->size - s->red_left_pad;
799 
800 	return s->size;
801 }
802 
803 static inline void *restore_red_left(struct kmem_cache *s, void *p)
804 {
805 	if (s->flags & SLAB_RED_ZONE)
806 		p -= s->red_left_pad;
807 
808 	return p;
809 }
810 
811 /*
812  * Debug settings:
813  */
814 #if defined(CONFIG_SLUB_DEBUG_ON)
815 static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
816 #else
817 static slab_flags_t slub_debug;
818 #endif
819 
820 static char *slub_debug_string;
821 static int disable_higher_order_debug;
822 
823 /*
824  * slub is about to manipulate internal object metadata.  This memory lies
825  * outside the range of the allocated object, so accessing it would normally
826  * be reported by kasan as a bounds error.  metadata_access_enable() is used
827  * to tell kasan that these accesses are OK.
828  */
829 static inline void metadata_access_enable(void)
830 {
831 	kasan_disable_current();
832 }
833 
834 static inline void metadata_access_disable(void)
835 {
836 	kasan_enable_current();
837 }
838 
839 /*
840  * Object debugging
841  */
842 
843 /* Verify that a pointer has an address that is valid within a slab page */
844 static inline int check_valid_pointer(struct kmem_cache *s,
845 				struct slab *slab, void *object)
846 {
847 	void *base;
848 
849 	if (!object)
850 		return 1;
851 
852 	base = slab_address(slab);
853 	object = kasan_reset_tag(object);
854 	object = restore_red_left(s, object);
855 	if (object < base || object >= base + slab->objects * s->size ||
856 		(object - base) % s->size) {
857 		return 0;
858 	}
859 
860 	return 1;
861 }
862 
863 static void print_section(char *level, char *text, u8 *addr,
864 			  unsigned int length)
865 {
866 	metadata_access_enable();
867 	print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
868 			16, 1, kasan_reset_tag((void *)addr), length, 1);
869 	metadata_access_disable();
870 }
871 
872 static struct track *get_track(struct kmem_cache *s, void *object,
873 	enum track_item alloc)
874 {
875 	struct track *p;
876 
877 	p = object + get_info_end(s);
878 
879 	return kasan_reset_tag(p + alloc);
880 }
881 
882 #ifdef CONFIG_STACKDEPOT
883 static noinline depot_stack_handle_t set_track_prepare(void)
884 {
885 	depot_stack_handle_t handle;
886 	unsigned long entries[TRACK_ADDRS_COUNT];
887 	unsigned int nr_entries;
888 
889 	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
890 	handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);
891 
892 	return handle;
893 }
894 #else
895 static inline depot_stack_handle_t set_track_prepare(void)
896 {
897 	return 0;
898 }
899 #endif
900 
901 static void set_track_update(struct kmem_cache *s, void *object,
902 			     enum track_item alloc, unsigned long addr,
903 			     depot_stack_handle_t handle)
904 {
905 	struct track *p = get_track(s, object, alloc);
906 
907 #ifdef CONFIG_STACKDEPOT
908 	p->handle = handle;
909 #endif
910 	p->addr = addr;
911 	p->cpu = smp_processor_id();
912 	p->pid = current->pid;
913 	p->when = jiffies;
914 }
915 
916 static __always_inline void set_track(struct kmem_cache *s, void *object,
917 				      enum track_item alloc, unsigned long addr)
918 {
919 	depot_stack_handle_t handle = set_track_prepare();
920 
921 	set_track_update(s, object, alloc, addr, handle);
922 }
923 
924 static void init_tracking(struct kmem_cache *s, void *object)
925 {
926 	struct track *p;
927 
928 	if (!(s->flags & SLAB_STORE_USER))
929 		return;
930 
931 	p = get_track(s, object, TRACK_ALLOC);
932 	memset(p, 0, 2*sizeof(struct track));
933 }
934 
935 static void print_track(const char *s, struct track *t, unsigned long pr_time)
936 {
937 	depot_stack_handle_t handle __maybe_unused;
938 
939 	if (!t->addr)
940 		return;
941 
942 	pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
943 	       s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
944 #ifdef CONFIG_STACKDEPOT
945 	handle = READ_ONCE(t->handle);
946 	if (handle)
947 		stack_depot_print(handle);
948 	else
949 		pr_err("object allocation/free stack trace missing\n");
950 #endif
951 }
952 
953 void print_tracking(struct kmem_cache *s, void *object)
954 {
955 	unsigned long pr_time = jiffies;
956 	if (!(s->flags & SLAB_STORE_USER))
957 		return;
958 
959 	print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
960 	print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
961 }
962 
963 static void print_slab_info(const struct slab *slab)
964 {
965 	struct folio *folio = (struct folio *)slab_folio(slab);
966 
967 	pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
968 	       slab, slab->objects, slab->inuse, slab->freelist,
969 	       folio_flags(folio, 0));
970 }
971 
972 /*
973  * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API
974  * family will round up the real request size to these fixed ones, so
975  * there could be an extra area than what is requested. Save the original
976  * request size in the meta data area, for better debug and sanity check.
977  */
978 static inline void set_orig_size(struct kmem_cache *s,
979 				void *object, unsigned int orig_size)
980 {
981 	void *p = kasan_reset_tag(object);
982 	unsigned int kasan_meta_size;
983 
984 	if (!slub_debug_orig_size(s))
985 		return;
986 
987 	/*
988 	 * KASAN can save its free meta data inside of the object at offset 0.
989 	 * If this meta data size is larger than 'orig_size', it will overlap
990 	 * the data redzone in [orig_size+1, object_size]. Thus, we adjust
991 	 * 'orig_size' to be as at least as big as KASAN's meta data.
992 	 */
993 	kasan_meta_size = kasan_metadata_size(s, true);
994 	if (kasan_meta_size > orig_size)
995 		orig_size = kasan_meta_size;
996 
997 	p += get_info_end(s);
998 	p += sizeof(struct track) * 2;
999 
1000 	*(unsigned int *)p = orig_size;
1001 }
1002 
1003 static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)
1004 {
1005 	void *p = kasan_reset_tag(object);
1006 
1007 	if (!slub_debug_orig_size(s))
1008 		return s->object_size;
1009 
1010 	p += get_info_end(s);
1011 	p += sizeof(struct track) * 2;
1012 
1013 	return *(unsigned int *)p;
1014 }
1015 
1016 void skip_orig_size_check(struct kmem_cache *s, const void *object)
1017 {
1018 	set_orig_size(s, (void *)object, s->object_size);
1019 }
1020 
1021 static void slab_bug(struct kmem_cache *s, char *fmt, ...)
1022 {
1023 	struct va_format vaf;
1024 	va_list args;
1025 
1026 	va_start(args, fmt);
1027 	vaf.fmt = fmt;
1028 	vaf.va = &args;
1029 	pr_err("=============================================================================\n");
1030 	pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
1031 	pr_err("-----------------------------------------------------------------------------\n\n");
1032 	va_end(args);
1033 }
1034 
1035 __printf(2, 3)
1036 static void slab_fix(struct kmem_cache *s, char *fmt, ...)
1037 {
1038 	struct va_format vaf;
1039 	va_list args;
1040 
1041 	if (slab_add_kunit_errors())
1042 		return;
1043 
1044 	va_start(args, fmt);
1045 	vaf.fmt = fmt;
1046 	vaf.va = &args;
1047 	pr_err("FIX %s: %pV\n", s->name, &vaf);
1048 	va_end(args);
1049 }
1050 
1051 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
1052 {
1053 	unsigned int off;	/* Offset of last byte */
1054 	u8 *addr = slab_address(slab);
1055 
1056 	print_tracking(s, p);
1057 
1058 	print_slab_info(slab);
1059 
1060 	pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
1061 	       p, p - addr, get_freepointer(s, p));
1062 
1063 	if (s->flags & SLAB_RED_ZONE)
1064 		print_section(KERN_ERR, "Redzone  ", p - s->red_left_pad,
1065 			      s->red_left_pad);
1066 	else if (p > addr + 16)
1067 		print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
1068 
1069 	print_section(KERN_ERR,         "Object   ", p,
1070 		      min_t(unsigned int, s->object_size, PAGE_SIZE));
1071 	if (s->flags & SLAB_RED_ZONE)
1072 		print_section(KERN_ERR, "Redzone  ", p + s->object_size,
1073 			s->inuse - s->object_size);
1074 
1075 	off = get_info_end(s);
1076 
1077 	if (s->flags & SLAB_STORE_USER)
1078 		off += 2 * sizeof(struct track);
1079 
1080 	if (slub_debug_orig_size(s))
1081 		off += sizeof(unsigned int);
1082 
1083 	off += kasan_metadata_size(s, false);
1084 
1085 	if (off != size_from_object(s))
1086 		/* Beginning of the filler is the free pointer */
1087 		print_section(KERN_ERR, "Padding  ", p + off,
1088 			      size_from_object(s) - off);
1089 
1090 	dump_stack();
1091 }
1092 
1093 static void object_err(struct kmem_cache *s, struct slab *slab,
1094 			u8 *object, char *reason)
1095 {
1096 	if (slab_add_kunit_errors())
1097 		return;
1098 
1099 	slab_bug(s, "%s", reason);
1100 	print_trailer(s, slab, object);
1101 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1102 }
1103 
1104 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1105 			       void **freelist, void *nextfree)
1106 {
1107 	if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
1108 	    !check_valid_pointer(s, slab, nextfree) && freelist) {
1109 		object_err(s, slab, *freelist, "Freechain corrupt");
1110 		*freelist = NULL;
1111 		slab_fix(s, "Isolate corrupted freechain");
1112 		return true;
1113 	}
1114 
1115 	return false;
1116 }
1117 
1118 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
1119 			const char *fmt, ...)
1120 {
1121 	va_list args;
1122 	char buf[100];
1123 
1124 	if (slab_add_kunit_errors())
1125 		return;
1126 
1127 	va_start(args, fmt);
1128 	vsnprintf(buf, sizeof(buf), fmt, args);
1129 	va_end(args);
1130 	slab_bug(s, "%s", buf);
1131 	print_slab_info(slab);
1132 	dump_stack();
1133 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1134 }
1135 
1136 static void init_object(struct kmem_cache *s, void *object, u8 val)
1137 {
1138 	u8 *p = kasan_reset_tag(object);
1139 	unsigned int poison_size = s->object_size;
1140 
1141 	if (s->flags & SLAB_RED_ZONE) {
1142 		memset(p - s->red_left_pad, val, s->red_left_pad);
1143 
1144 		if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1145 			/*
1146 			 * Redzone the extra allocated space by kmalloc than
1147 			 * requested, and the poison size will be limited to
1148 			 * the original request size accordingly.
1149 			 */
1150 			poison_size = get_orig_size(s, object);
1151 		}
1152 	}
1153 
1154 	if (s->flags & __OBJECT_POISON) {
1155 		memset(p, POISON_FREE, poison_size - 1);
1156 		p[poison_size - 1] = POISON_END;
1157 	}
1158 
1159 	if (s->flags & SLAB_RED_ZONE)
1160 		memset(p + poison_size, val, s->inuse - poison_size);
1161 }
1162 
1163 static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
1164 						void *from, void *to)
1165 {
1166 	slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
1167 	memset(from, data, to - from);
1168 }
1169 
1170 static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
1171 			u8 *object, char *what,
1172 			u8 *start, unsigned int value, unsigned int bytes)
1173 {
1174 	u8 *fault;
1175 	u8 *end;
1176 	u8 *addr = slab_address(slab);
1177 
1178 	metadata_access_enable();
1179 	fault = memchr_inv(kasan_reset_tag(start), value, bytes);
1180 	metadata_access_disable();
1181 	if (!fault)
1182 		return 1;
1183 
1184 	end = start + bytes;
1185 	while (end > fault && end[-1] == value)
1186 		end--;
1187 
1188 	if (slab_add_kunit_errors())
1189 		goto skip_bug_print;
1190 
1191 	slab_bug(s, "%s overwritten", what);
1192 	pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
1193 					fault, end - 1, fault - addr,
1194 					fault[0], value);
1195 	print_trailer(s, slab, object);
1196 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1197 
1198 skip_bug_print:
1199 	restore_bytes(s, what, value, fault, end);
1200 	return 0;
1201 }
1202 
1203 /*
1204  * Object layout:
1205  *
1206  * object address
1207  * 	Bytes of the object to be managed.
1208  * 	If the freepointer may overlay the object then the free
1209  *	pointer is at the middle of the object.
1210  *
1211  * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
1212  * 	0xa5 (POISON_END)
1213  *
1214  * object + s->object_size
1215  * 	Padding to reach word boundary. This is also used for Redzoning.
1216  * 	Padding is extended by another word if Redzoning is enabled and
1217  * 	object_size == inuse.
1218  *
1219  * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
1220  * 	0xcc (RED_ACTIVE) for objects in use.
1221  *
1222  * object + s->inuse
1223  * 	Meta data starts here.
1224  *
1225  * 	A. Free pointer (if we cannot overwrite object on free)
1226  * 	B. Tracking data for SLAB_STORE_USER
1227  *	C. Original request size for kmalloc object (SLAB_STORE_USER enabled)
1228  *	D. Padding to reach required alignment boundary or at minimum
1229  * 		one word if debugging is on to be able to detect writes
1230  * 		before the word boundary.
1231  *
1232  *	Padding is done using 0x5a (POISON_INUSE)
1233  *
1234  * object + s->size
1235  * 	Nothing is used beyond s->size.
1236  *
1237  * If slabcaches are merged then the object_size and inuse boundaries are mostly
1238  * ignored. And therefore no slab options that rely on these boundaries
1239  * may be used with merged slabcaches.
1240  */
1241 
1242 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
1243 {
1244 	unsigned long off = get_info_end(s);	/* The end of info */
1245 
1246 	if (s->flags & SLAB_STORE_USER) {
1247 		/* We also have user information there */
1248 		off += 2 * sizeof(struct track);
1249 
1250 		if (s->flags & SLAB_KMALLOC)
1251 			off += sizeof(unsigned int);
1252 	}
1253 
1254 	off += kasan_metadata_size(s, false);
1255 
1256 	if (size_from_object(s) == off)
1257 		return 1;
1258 
1259 	return check_bytes_and_report(s, slab, p, "Object padding",
1260 			p + off, POISON_INUSE, size_from_object(s) - off);
1261 }
1262 
1263 /* Check the pad bytes at the end of a slab page */
1264 static void slab_pad_check(struct kmem_cache *s, struct slab *slab)
1265 {
1266 	u8 *start;
1267 	u8 *fault;
1268 	u8 *end;
1269 	u8 *pad;
1270 	int length;
1271 	int remainder;
1272 
1273 	if (!(s->flags & SLAB_POISON))
1274 		return;
1275 
1276 	start = slab_address(slab);
1277 	length = slab_size(slab);
1278 	end = start + length;
1279 	remainder = length % s->size;
1280 	if (!remainder)
1281 		return;
1282 
1283 	pad = end - remainder;
1284 	metadata_access_enable();
1285 	fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
1286 	metadata_access_disable();
1287 	if (!fault)
1288 		return;
1289 	while (end > fault && end[-1] == POISON_INUSE)
1290 		end--;
1291 
1292 	slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1293 			fault, end - 1, fault - start);
1294 	print_section(KERN_ERR, "Padding ", pad, remainder);
1295 
1296 	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
1297 }
1298 
1299 static int check_object(struct kmem_cache *s, struct slab *slab,
1300 					void *object, u8 val)
1301 {
1302 	u8 *p = object;
1303 	u8 *endobject = object + s->object_size;
1304 	unsigned int orig_size, kasan_meta_size;
1305 
1306 	if (s->flags & SLAB_RED_ZONE) {
1307 		if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1308 			object - s->red_left_pad, val, s->red_left_pad))
1309 			return 0;
1310 
1311 		if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1312 			endobject, val, s->inuse - s->object_size))
1313 			return 0;
1314 
1315 		if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1316 			orig_size = get_orig_size(s, object);
1317 
1318 			if (s->object_size > orig_size  &&
1319 				!check_bytes_and_report(s, slab, object,
1320 					"kmalloc Redzone", p + orig_size,
1321 					val, s->object_size - orig_size)) {
1322 				return 0;
1323 			}
1324 		}
1325 	} else {
1326 		if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
1327 			check_bytes_and_report(s, slab, p, "Alignment padding",
1328 				endobject, POISON_INUSE,
1329 				s->inuse - s->object_size);
1330 		}
1331 	}
1332 
1333 	if (s->flags & SLAB_POISON) {
1334 		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) {
1335 			/*
1336 			 * KASAN can save its free meta data inside of the
1337 			 * object at offset 0. Thus, skip checking the part of
1338 			 * the redzone that overlaps with the meta data.
1339 			 */
1340 			kasan_meta_size = kasan_metadata_size(s, true);
1341 			if (kasan_meta_size < s->object_size - 1 &&
1342 			    !check_bytes_and_report(s, slab, p, "Poison",
1343 					p + kasan_meta_size, POISON_FREE,
1344 					s->object_size - kasan_meta_size - 1))
1345 				return 0;
1346 			if (kasan_meta_size < s->object_size &&
1347 			    !check_bytes_and_report(s, slab, p, "End Poison",
1348 					p + s->object_size - 1, POISON_END, 1))
1349 				return 0;
1350 		}
1351 		/*
1352 		 * check_pad_bytes cleans up on its own.
1353 		 */
1354 		check_pad_bytes(s, slab, p);
1355 	}
1356 
1357 	if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
1358 		/*
1359 		 * Object and freepointer overlap. Cannot check
1360 		 * freepointer while object is allocated.
1361 		 */
1362 		return 1;
1363 
1364 	/* Check free pointer validity */
1365 	if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
1366 		object_err(s, slab, p, "Freepointer corrupt");
1367 		/*
1368 		 * No choice but to zap it and thus lose the remainder
1369 		 * of the free objects in this slab. May cause
1370 		 * another error because the object count is now wrong.
1371 		 */
1372 		set_freepointer(s, p, NULL);
1373 		return 0;
1374 	}
1375 	return 1;
1376 }
1377 
1378 static int check_slab(struct kmem_cache *s, struct slab *slab)
1379 {
1380 	int maxobj;
1381 
1382 	if (!folio_test_slab(slab_folio(slab))) {
1383 		slab_err(s, slab, "Not a valid slab page");
1384 		return 0;
1385 	}
1386 
1387 	maxobj = order_objects(slab_order(slab), s->size);
1388 	if (slab->objects > maxobj) {
1389 		slab_err(s, slab, "objects %u > max %u",
1390 			slab->objects, maxobj);
1391 		return 0;
1392 	}
1393 	if (slab->inuse > slab->objects) {
1394 		slab_err(s, slab, "inuse %u > max %u",
1395 			slab->inuse, slab->objects);
1396 		return 0;
1397 	}
1398 	/* Slab_pad_check fixes things up after itself */
1399 	slab_pad_check(s, slab);
1400 	return 1;
1401 }
1402 
1403 /*
1404  * Determine if a certain object in a slab is on the freelist. Must hold the
1405  * slab lock to guarantee that the chains are in a consistent state.
1406  */
1407 static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
1408 {
1409 	int nr = 0;
1410 	void *fp;
1411 	void *object = NULL;
1412 	int max_objects;
1413 
1414 	fp = slab->freelist;
1415 	while (fp && nr <= slab->objects) {
1416 		if (fp == search)
1417 			return 1;
1418 		if (!check_valid_pointer(s, slab, fp)) {
1419 			if (object) {
1420 				object_err(s, slab, object,
1421 					"Freechain corrupt");
1422 				set_freepointer(s, object, NULL);
1423 			} else {
1424 				slab_err(s, slab, "Freepointer corrupt");
1425 				slab->freelist = NULL;
1426 				slab->inuse = slab->objects;
1427 				slab_fix(s, "Freelist cleared");
1428 				return 0;
1429 			}
1430 			break;
1431 		}
1432 		object = fp;
1433 		fp = get_freepointer(s, object);
1434 		nr++;
1435 	}
1436 
1437 	max_objects = order_objects(slab_order(slab), s->size);
1438 	if (max_objects > MAX_OBJS_PER_PAGE)
1439 		max_objects = MAX_OBJS_PER_PAGE;
1440 
1441 	if (slab->objects != max_objects) {
1442 		slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1443 			 slab->objects, max_objects);
1444 		slab->objects = max_objects;
1445 		slab_fix(s, "Number of objects adjusted");
1446 	}
1447 	if (slab->inuse != slab->objects - nr) {
1448 		slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1449 			 slab->inuse, slab->objects - nr);
1450 		slab->inuse = slab->objects - nr;
1451 		slab_fix(s, "Object count adjusted");
1452 	}
1453 	return search == NULL;
1454 }
1455 
1456 static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1457 								int alloc)
1458 {
1459 	if (s->flags & SLAB_TRACE) {
1460 		pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
1461 			s->name,
1462 			alloc ? "alloc" : "free",
1463 			object, slab->inuse,
1464 			slab->freelist);
1465 
1466 		if (!alloc)
1467 			print_section(KERN_INFO, "Object ", (void *)object,
1468 					s->object_size);
1469 
1470 		dump_stack();
1471 	}
1472 }
1473 
1474 /*
1475  * Tracking of fully allocated slabs for debugging purposes.
1476  */
1477 static void add_full(struct kmem_cache *s,
1478 	struct kmem_cache_node *n, struct slab *slab)
1479 {
1480 	if (!(s->flags & SLAB_STORE_USER))
1481 		return;
1482 
1483 	lockdep_assert_held(&n->list_lock);
1484 	list_add(&slab->slab_list, &n->full);
1485 }
1486 
1487 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1488 {
1489 	if (!(s->flags & SLAB_STORE_USER))
1490 		return;
1491 
1492 	lockdep_assert_held(&n->list_lock);
1493 	list_del(&slab->slab_list);
1494 }
1495 
1496 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1497 {
1498 	return atomic_long_read(&n->nr_slabs);
1499 }
1500 
1501 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1502 {
1503 	struct kmem_cache_node *n = get_node(s, node);
1504 
1505 	atomic_long_inc(&n->nr_slabs);
1506 	atomic_long_add(objects, &n->total_objects);
1507 }
1508 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1509 {
1510 	struct kmem_cache_node *n = get_node(s, node);
1511 
1512 	atomic_long_dec(&n->nr_slabs);
1513 	atomic_long_sub(objects, &n->total_objects);
1514 }
1515 
1516 /* Object debug checks for alloc/free paths */
1517 static void setup_object_debug(struct kmem_cache *s, void *object)
1518 {
1519 	if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
1520 		return;
1521 
1522 	init_object(s, object, SLUB_RED_INACTIVE);
1523 	init_tracking(s, object);
1524 }
1525 
1526 static
1527 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1528 {
1529 	if (!kmem_cache_debug_flags(s, SLAB_POISON))
1530 		return;
1531 
1532 	metadata_access_enable();
1533 	memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
1534 	metadata_access_disable();
1535 }
1536 
1537 static inline int alloc_consistency_checks(struct kmem_cache *s,
1538 					struct slab *slab, void *object)
1539 {
1540 	if (!check_slab(s, slab))
1541 		return 0;
1542 
1543 	if (!check_valid_pointer(s, slab, object)) {
1544 		object_err(s, slab, object, "Freelist Pointer check fails");
1545 		return 0;
1546 	}
1547 
1548 	if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1549 		return 0;
1550 
1551 	return 1;
1552 }
1553 
1554 static noinline bool alloc_debug_processing(struct kmem_cache *s,
1555 			struct slab *slab, void *object, int orig_size)
1556 {
1557 	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1558 		if (!alloc_consistency_checks(s, slab, object))
1559 			goto bad;
1560 	}
1561 
1562 	/* Success. Perform special debug activities for allocs */
1563 	trace(s, slab, object, 1);
1564 	set_orig_size(s, object, orig_size);
1565 	init_object(s, object, SLUB_RED_ACTIVE);
1566 	return true;
1567 
1568 bad:
1569 	if (folio_test_slab(slab_folio(slab))) {
1570 		/*
1571 		 * If this is a slab page then lets do the best we can
1572 		 * to avoid issues in the future. Marking all objects
1573 		 * as used avoids touching the remaining objects.
1574 		 */
1575 		slab_fix(s, "Marking all objects used");
1576 		slab->inuse = slab->objects;
1577 		slab->freelist = NULL;
1578 	}
1579 	return false;
1580 }
1581 
1582 static inline int free_consistency_checks(struct kmem_cache *s,
1583 		struct slab *slab, void *object, unsigned long addr)
1584 {
1585 	if (!check_valid_pointer(s, slab, object)) {
1586 		slab_err(s, slab, "Invalid object pointer 0x%p", object);
1587 		return 0;
1588 	}
1589 
1590 	if (on_freelist(s, slab, object)) {
1591 		object_err(s, slab, object, "Object already free");
1592 		return 0;
1593 	}
1594 
1595 	if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1596 		return 0;
1597 
1598 	if (unlikely(s != slab->slab_cache)) {
1599 		if (!folio_test_slab(slab_folio(slab))) {
1600 			slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
1601 				 object);
1602 		} else if (!slab->slab_cache) {
1603 			pr_err("SLUB <none>: no slab for object 0x%p.\n",
1604 			       object);
1605 			dump_stack();
1606 		} else
1607 			object_err(s, slab, object,
1608 					"page slab pointer corrupt.");
1609 		return 0;
1610 	}
1611 	return 1;
1612 }
1613 
1614 /*
1615  * Parse a block of slab_debug options. Blocks are delimited by ';'
1616  *
1617  * @str:    start of block
1618  * @flags:  returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
1619  * @slabs:  return start of list of slabs, or NULL when there's no list
1620  * @init:   assume this is initial parsing and not per-kmem-create parsing
1621  *
1622  * returns the start of next block if there's any, or NULL
1623  */
1624 static char *
1625 parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
1626 {
1627 	bool higher_order_disable = false;
1628 
1629 	/* Skip any completely empty blocks */
1630 	while (*str && *str == ';')
1631 		str++;
1632 
1633 	if (*str == ',') {
1634 		/*
1635 		 * No options but restriction on slabs. This means full
1636 		 * debugging for slabs matching a pattern.
1637 		 */
1638 		*flags = DEBUG_DEFAULT_FLAGS;
1639 		goto check_slabs;
1640 	}
1641 	*flags = 0;
1642 
1643 	/* Determine which debug features should be switched on */
1644 	for (; *str && *str != ',' && *str != ';'; str++) {
1645 		switch (tolower(*str)) {
1646 		case '-':
1647 			*flags = 0;
1648 			break;
1649 		case 'f':
1650 			*flags |= SLAB_CONSISTENCY_CHECKS;
1651 			break;
1652 		case 'z':
1653 			*flags |= SLAB_RED_ZONE;
1654 			break;
1655 		case 'p':
1656 			*flags |= SLAB_POISON;
1657 			break;
1658 		case 'u':
1659 			*flags |= SLAB_STORE_USER;
1660 			break;
1661 		case 't':
1662 			*flags |= SLAB_TRACE;
1663 			break;
1664 		case 'a':
1665 			*flags |= SLAB_FAILSLAB;
1666 			break;
1667 		case 'o':
1668 			/*
1669 			 * Avoid enabling debugging on caches if its minimum
1670 			 * order would increase as a result.
1671 			 */
1672 			higher_order_disable = true;
1673 			break;
1674 		default:
1675 			if (init)
1676 				pr_err("slab_debug option '%c' unknown. skipped\n", *str);
1677 		}
1678 	}
1679 check_slabs:
1680 	if (*str == ',')
1681 		*slabs = ++str;
1682 	else
1683 		*slabs = NULL;
1684 
1685 	/* Skip over the slab list */
1686 	while (*str && *str != ';')
1687 		str++;
1688 
1689 	/* Skip any completely empty blocks */
1690 	while (*str && *str == ';')
1691 		str++;
1692 
1693 	if (init && higher_order_disable)
1694 		disable_higher_order_debug = 1;
1695 
1696 	if (*str)
1697 		return str;
1698 	else
1699 		return NULL;
1700 }
1701 
1702 static int __init setup_slub_debug(char *str)
1703 {
1704 	slab_flags_t flags;
1705 	slab_flags_t global_flags;
1706 	char *saved_str;
1707 	char *slab_list;
1708 	bool global_slub_debug_changed = false;
1709 	bool slab_list_specified = false;
1710 
1711 	global_flags = DEBUG_DEFAULT_FLAGS;
1712 	if (*str++ != '=' || !*str)
1713 		/*
1714 		 * No options specified. Switch on full debugging.
1715 		 */
1716 		goto out;
1717 
1718 	saved_str = str;
1719 	while (str) {
1720 		str = parse_slub_debug_flags(str, &flags, &slab_list, true);
1721 
1722 		if (!slab_list) {
1723 			global_flags = flags;
1724 			global_slub_debug_changed = true;
1725 		} else {
1726 			slab_list_specified = true;
1727 			if (flags & SLAB_STORE_USER)
1728 				stack_depot_request_early_init();
1729 		}
1730 	}
1731 
1732 	/*
1733 	 * For backwards compatibility, a single list of flags with list of
1734 	 * slabs means debugging is only changed for those slabs, so the global
1735 	 * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1736 	 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
1737 	 * long as there is no option specifying flags without a slab list.
1738 	 */
1739 	if (slab_list_specified) {
1740 		if (!global_slub_debug_changed)
1741 			global_flags = slub_debug;
1742 		slub_debug_string = saved_str;
1743 	}
1744 out:
1745 	slub_debug = global_flags;
1746 	if (slub_debug & SLAB_STORE_USER)
1747 		stack_depot_request_early_init();
1748 	if (slub_debug != 0 || slub_debug_string)
1749 		static_branch_enable(&slub_debug_enabled);
1750 	else
1751 		static_branch_disable(&slub_debug_enabled);
1752 	if ((static_branch_unlikely(&init_on_alloc) ||
1753 	     static_branch_unlikely(&init_on_free)) &&
1754 	    (slub_debug & SLAB_POISON))
1755 		pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
1756 	return 1;
1757 }
1758 
1759 __setup("slab_debug", setup_slub_debug);
1760 __setup_param("slub_debug", slub_debug, setup_slub_debug, 0);
1761 
1762 /*
1763  * kmem_cache_flags - apply debugging options to the cache
1764  * @flags:		flags to set
1765  * @name:		name of the cache
1766  *
1767  * Debug option(s) are applied to @flags. In addition to the debug
1768  * option(s), if a slab name (or multiple) is specified i.e.
1769  * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1770  * then only the select slabs will receive the debug option(s).
1771  */
1772 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1773 {
1774 	char *iter;
1775 	size_t len;
1776 	char *next_block;
1777 	slab_flags_t block_flags;
1778 	slab_flags_t slub_debug_local = slub_debug;
1779 
1780 	if (flags & SLAB_NO_USER_FLAGS)
1781 		return flags;
1782 
1783 	/*
1784 	 * If the slab cache is for debugging (e.g. kmemleak) then
1785 	 * don't store user (stack trace) information by default,
1786 	 * but let the user enable it via the command line below.
1787 	 */
1788 	if (flags & SLAB_NOLEAKTRACE)
1789 		slub_debug_local &= ~SLAB_STORE_USER;
1790 
1791 	len = strlen(name);
1792 	next_block = slub_debug_string;
1793 	/* Go through all blocks of debug options, see if any matches our slab's name */
1794 	while (next_block) {
1795 		next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
1796 		if (!iter)
1797 			continue;
1798 		/* Found a block that has a slab list, search it */
1799 		while (*iter) {
1800 			char *end, *glob;
1801 			size_t cmplen;
1802 
1803 			end = strchrnul(iter, ',');
1804 			if (next_block && next_block < end)
1805 				end = next_block - 1;
1806 
1807 			glob = strnchr(iter, end - iter, '*');
1808 			if (glob)
1809 				cmplen = glob - iter;
1810 			else
1811 				cmplen = max_t(size_t, len, (end - iter));
1812 
1813 			if (!strncmp(name, iter, cmplen)) {
1814 				flags |= block_flags;
1815 				return flags;
1816 			}
1817 
1818 			if (!*end || *end == ';')
1819 				break;
1820 			iter = end + 1;
1821 		}
1822 	}
1823 
1824 	return flags | slub_debug_local;
1825 }
1826 #else /* !CONFIG_SLUB_DEBUG */
1827 static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
1828 static inline
1829 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
1830 
1831 static inline bool alloc_debug_processing(struct kmem_cache *s,
1832 	struct slab *slab, void *object, int orig_size) { return true; }
1833 
1834 static inline bool free_debug_processing(struct kmem_cache *s,
1835 	struct slab *slab, void *head, void *tail, int *bulk_cnt,
1836 	unsigned long addr, depot_stack_handle_t handle) { return true; }
1837 
1838 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
1839 static inline int check_object(struct kmem_cache *s, struct slab *slab,
1840 			void *object, u8 val) { return 1; }
1841 static inline depot_stack_handle_t set_track_prepare(void) { return 0; }
1842 static inline void set_track(struct kmem_cache *s, void *object,
1843 			     enum track_item alloc, unsigned long addr) {}
1844 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1845 					struct slab *slab) {}
1846 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1847 					struct slab *slab) {}
1848 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1849 {
1850 	return flags;
1851 }
1852 #define slub_debug 0
1853 
1854 #define disable_higher_order_debug 0
1855 
1856 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1857 							{ return 0; }
1858 static inline void inc_slabs_node(struct kmem_cache *s, int node,
1859 							int objects) {}
1860 static inline void dec_slabs_node(struct kmem_cache *s, int node,
1861 							int objects) {}
1862 
1863 #ifndef CONFIG_SLUB_TINY
1864 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1865 			       void **freelist, void *nextfree)
1866 {
1867 	return false;
1868 }
1869 #endif
1870 #endif /* CONFIG_SLUB_DEBUG */
1871 
1872 #ifdef CONFIG_SLAB_OBJ_EXT
1873 
1874 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
1875 
1876 static inline void mark_objexts_empty(struct slabobj_ext *obj_exts)
1877 {
1878 	struct slabobj_ext *slab_exts;
1879 	struct slab *obj_exts_slab;
1880 
1881 	obj_exts_slab = virt_to_slab(obj_exts);
1882 	slab_exts = slab_obj_exts(obj_exts_slab);
1883 	if (slab_exts) {
1884 		unsigned int offs = obj_to_index(obj_exts_slab->slab_cache,
1885 						 obj_exts_slab, obj_exts);
1886 		/* codetag should be NULL */
1887 		WARN_ON(slab_exts[offs].ref.ct);
1888 		set_codetag_empty(&slab_exts[offs].ref);
1889 	}
1890 }
1891 
1892 static inline void mark_failed_objexts_alloc(struct slab *slab)
1893 {
1894 	slab->obj_exts = OBJEXTS_ALLOC_FAIL;
1895 }
1896 
1897 static inline void handle_failed_objexts_alloc(unsigned long obj_exts,
1898 			struct slabobj_ext *vec, unsigned int objects)
1899 {
1900 	/*
1901 	 * If vector previously failed to allocate then we have live
1902 	 * objects with no tag reference. Mark all references in this
1903 	 * vector as empty to avoid warnings later on.
1904 	 */
1905 	if (obj_exts & OBJEXTS_ALLOC_FAIL) {
1906 		unsigned int i;
1907 
1908 		for (i = 0; i < objects; i++)
1909 			set_codetag_empty(&vec[i].ref);
1910 	}
1911 }
1912 
1913 #else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
1914 
1915 static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {}
1916 static inline void mark_failed_objexts_alloc(struct slab *slab) {}
1917 static inline void handle_failed_objexts_alloc(unsigned long obj_exts,
1918 			struct slabobj_ext *vec, unsigned int objects) {}
1919 
1920 #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
1921 
1922 /*
1923  * The allocated objcg pointers array is not accounted directly.
1924  * Moreover, it should not come from DMA buffer and is not readily
1925  * reclaimable. So those GFP bits should be masked off.
1926  */
1927 #define OBJCGS_CLEAR_MASK	(__GFP_DMA | __GFP_RECLAIMABLE | \
1928 				__GFP_ACCOUNT | __GFP_NOFAIL)
1929 
1930 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
1931 		        gfp_t gfp, bool new_slab)
1932 {
1933 	unsigned int objects = objs_per_slab(s, slab);
1934 	unsigned long new_exts;
1935 	unsigned long old_exts;
1936 	struct slabobj_ext *vec;
1937 
1938 	gfp &= ~OBJCGS_CLEAR_MASK;
1939 	/* Prevent recursive extension vector allocation */
1940 	gfp |= __GFP_NO_OBJ_EXT;
1941 	vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp,
1942 			   slab_nid(slab));
1943 	if (!vec) {
1944 		/* Mark vectors which failed to allocate */
1945 		if (new_slab)
1946 			mark_failed_objexts_alloc(slab);
1947 
1948 		return -ENOMEM;
1949 	}
1950 
1951 	new_exts = (unsigned long)vec;
1952 #ifdef CONFIG_MEMCG
1953 	new_exts |= MEMCG_DATA_OBJEXTS;
1954 #endif
1955 	old_exts = slab->obj_exts;
1956 	handle_failed_objexts_alloc(old_exts, vec, objects);
1957 	if (new_slab) {
1958 		/*
1959 		 * If the slab is brand new and nobody can yet access its
1960 		 * obj_exts, no synchronization is required and obj_exts can
1961 		 * be simply assigned.
1962 		 */
1963 		slab->obj_exts = new_exts;
1964 	} else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) {
1965 		/*
1966 		 * If the slab is already in use, somebody can allocate and
1967 		 * assign slabobj_exts in parallel. In this case the existing
1968 		 * objcg vector should be reused.
1969 		 */
1970 		mark_objexts_empty(vec);
1971 		kfree(vec);
1972 		return 0;
1973 	}
1974 
1975 	kmemleak_not_leak(vec);
1976 	return 0;
1977 }
1978 
1979 static inline void free_slab_obj_exts(struct slab *slab)
1980 {
1981 	struct slabobj_ext *obj_exts;
1982 
1983 	obj_exts = slab_obj_exts(slab);
1984 	if (!obj_exts)
1985 		return;
1986 
1987 	/*
1988 	 * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its
1989 	 * corresponding extension will be NULL. alloc_tag_sub() will throw a
1990 	 * warning if slab has extensions but the extension of an object is
1991 	 * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
1992 	 * the extension for obj_exts is expected to be NULL.
1993 	 */
1994 	mark_objexts_empty(obj_exts);
1995 	kfree(obj_exts);
1996 	slab->obj_exts = 0;
1997 }
1998 
1999 static inline bool need_slab_obj_ext(void)
2000 {
2001 	if (mem_alloc_profiling_enabled())
2002 		return true;
2003 
2004 	/*
2005 	 * CONFIG_MEMCG_KMEM creates vector of obj_cgroup objects conditionally
2006 	 * inside memcg_slab_post_alloc_hook. No other users for now.
2007 	 */
2008 	return false;
2009 }
2010 
2011 static inline struct slabobj_ext *
2012 prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
2013 {
2014 	struct slab *slab;
2015 
2016 	if (!p)
2017 		return NULL;
2018 
2019 	if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
2020 		return NULL;
2021 
2022 	if (flags & __GFP_NO_OBJ_EXT)
2023 		return NULL;
2024 
2025 	slab = virt_to_slab(p);
2026 	if (!slab_obj_exts(slab) &&
2027 	    WARN(alloc_slab_obj_exts(slab, s, flags, false),
2028 		 "%s, %s: Failed to create slab extension vector!\n",
2029 		 __func__, s->name))
2030 		return NULL;
2031 
2032 	return slab_obj_exts(slab) + obj_to_index(s, slab, p);
2033 }
2034 
2035 static inline void
2036 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2037 			     int objects)
2038 {
2039 #ifdef CONFIG_MEM_ALLOC_PROFILING
2040 	struct slabobj_ext *obj_exts;
2041 	int i;
2042 
2043 	if (!mem_alloc_profiling_enabled())
2044 		return;
2045 
2046 	obj_exts = slab_obj_exts(slab);
2047 	if (!obj_exts)
2048 		return;
2049 
2050 	for (i = 0; i < objects; i++) {
2051 		unsigned int off = obj_to_index(s, slab, p[i]);
2052 
2053 		alloc_tag_sub(&obj_exts[off].ref, s->size);
2054 	}
2055 #endif
2056 }
2057 
2058 #else /* CONFIG_SLAB_OBJ_EXT */
2059 
2060 static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
2061 			       gfp_t gfp, bool new_slab)
2062 {
2063 	return 0;
2064 }
2065 
2066 static inline void free_slab_obj_exts(struct slab *slab)
2067 {
2068 }
2069 
2070 static inline bool need_slab_obj_ext(void)
2071 {
2072 	return false;
2073 }
2074 
2075 static inline struct slabobj_ext *
2076 prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
2077 {
2078 	return NULL;
2079 }
2080 
2081 static inline void
2082 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2083 			     int objects)
2084 {
2085 }
2086 
2087 #endif /* CONFIG_SLAB_OBJ_EXT */
2088 
2089 #ifdef CONFIG_MEMCG_KMEM
2090 
2091 static void memcg_alloc_abort_single(struct kmem_cache *s, void *object);
2092 
2093 static __fastpath_inline
2094 bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
2095 				gfp_t flags, size_t size, void **p)
2096 {
2097 	if (likely(!memcg_kmem_online()))
2098 		return true;
2099 
2100 	if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
2101 		return true;
2102 
2103 	if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p)))
2104 		return true;
2105 
2106 	if (likely(size == 1)) {
2107 		memcg_alloc_abort_single(s, *p);
2108 		*p = NULL;
2109 	} else {
2110 		kmem_cache_free_bulk(s, size, p);
2111 	}
2112 
2113 	return false;
2114 }
2115 
2116 static __fastpath_inline
2117 void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2118 			  int objects)
2119 {
2120 	struct slabobj_ext *obj_exts;
2121 
2122 	if (!memcg_kmem_online())
2123 		return;
2124 
2125 	obj_exts = slab_obj_exts(slab);
2126 	if (likely(!obj_exts))
2127 		return;
2128 
2129 	__memcg_slab_free_hook(s, slab, p, objects, obj_exts);
2130 }
2131 #else /* CONFIG_MEMCG_KMEM */
2132 static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s,
2133 					      struct list_lru *lru,
2134 					      gfp_t flags, size_t size,
2135 					      void **p)
2136 {
2137 	return true;
2138 }
2139 
2140 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
2141 					void **p, int objects)
2142 {
2143 }
2144 #endif /* CONFIG_MEMCG_KMEM */
2145 
2146 /*
2147  * Hooks for other subsystems that check memory allocations. In a typical
2148  * production configuration these hooks all should produce no code at all.
2149  *
2150  * Returns true if freeing of the object can proceed, false if its reuse
2151  * was delayed by KASAN quarantine, or it was returned to KFENCE.
2152  */
2153 static __always_inline
2154 bool slab_free_hook(struct kmem_cache *s, void *x, bool init)
2155 {
2156 	kmemleak_free_recursive(x, s->flags);
2157 	kmsan_slab_free(s, x);
2158 
2159 	debug_check_no_locks_freed(x, s->object_size);
2160 
2161 	if (!(s->flags & SLAB_DEBUG_OBJECTS))
2162 		debug_check_no_obj_freed(x, s->object_size);
2163 
2164 	/* Use KCSAN to help debug racy use-after-free. */
2165 	if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
2166 		__kcsan_check_access(x, s->object_size,
2167 				     KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
2168 
2169 	if (kfence_free(x))
2170 		return false;
2171 
2172 	/*
2173 	 * As memory initialization might be integrated into KASAN,
2174 	 * kasan_slab_free and initialization memset's must be
2175 	 * kept together to avoid discrepancies in behavior.
2176 	 *
2177 	 * The initialization memset's clear the object and the metadata,
2178 	 * but don't touch the SLAB redzone.
2179 	 *
2180 	 * The object's freepointer is also avoided if stored outside the
2181 	 * object.
2182 	 */
2183 	if (unlikely(init)) {
2184 		int rsize;
2185 		unsigned int inuse;
2186 
2187 		inuse = get_info_end(s);
2188 		if (!kasan_has_integrated_init())
2189 			memset(kasan_reset_tag(x), 0, s->object_size);
2190 		rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
2191 		memset((char *)kasan_reset_tag(x) + inuse, 0,
2192 		       s->size - inuse - rsize);
2193 	}
2194 	/* KASAN might put x into memory quarantine, delaying its reuse. */
2195 	return !kasan_slab_free(s, x, init);
2196 }
2197 
2198 static __fastpath_inline
2199 bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail,
2200 			     int *cnt)
2201 {
2202 
2203 	void *object;
2204 	void *next = *head;
2205 	void *old_tail = *tail;
2206 	bool init;
2207 
2208 	if (is_kfence_address(next)) {
2209 		slab_free_hook(s, next, false);
2210 		return false;
2211 	}
2212 
2213 	/* Head and tail of the reconstructed freelist */
2214 	*head = NULL;
2215 	*tail = NULL;
2216 
2217 	init = slab_want_init_on_free(s);
2218 
2219 	do {
2220 		object = next;
2221 		next = get_freepointer(s, object);
2222 
2223 		/* If object's reuse doesn't have to be delayed */
2224 		if (likely(slab_free_hook(s, object, init))) {
2225 			/* Move object to the new freelist */
2226 			set_freepointer(s, object, *head);
2227 			*head = object;
2228 			if (!*tail)
2229 				*tail = object;
2230 		} else {
2231 			/*
2232 			 * Adjust the reconstructed freelist depth
2233 			 * accordingly if object's reuse is delayed.
2234 			 */
2235 			--(*cnt);
2236 		}
2237 	} while (object != old_tail);
2238 
2239 	return *head != NULL;
2240 }
2241 
2242 static void *setup_object(struct kmem_cache *s, void *object)
2243 {
2244 	setup_object_debug(s, object);
2245 	object = kasan_init_slab_obj(s, object);
2246 	if (unlikely(s->ctor)) {
2247 		kasan_unpoison_new_object(s, object);
2248 		s->ctor(object);
2249 		kasan_poison_new_object(s, object);
2250 	}
2251 	return object;
2252 }
2253 
2254 /*
2255  * Slab allocation and freeing
2256  */
2257 static inline struct slab *alloc_slab_page(gfp_t flags, int node,
2258 		struct kmem_cache_order_objects oo)
2259 {
2260 	struct folio *folio;
2261 	struct slab *slab;
2262 	unsigned int order = oo_order(oo);
2263 
2264 	folio = (struct folio *)alloc_pages_node(node, flags, order);
2265 	if (!folio)
2266 		return NULL;
2267 
2268 	slab = folio_slab(folio);
2269 	__folio_set_slab(folio);
2270 	/* Make the flag visible before any changes to folio->mapping */
2271 	smp_wmb();
2272 	if (folio_is_pfmemalloc(folio))
2273 		slab_set_pfmemalloc(slab);
2274 
2275 	return slab;
2276 }
2277 
2278 #ifdef CONFIG_SLAB_FREELIST_RANDOM
2279 /* Pre-initialize the random sequence cache */
2280 static int init_cache_random_seq(struct kmem_cache *s)
2281 {
2282 	unsigned int count = oo_objects(s->oo);
2283 	int err;
2284 
2285 	/* Bailout if already initialised */
2286 	if (s->random_seq)
2287 		return 0;
2288 
2289 	err = cache_random_seq_create(s, count, GFP_KERNEL);
2290 	if (err) {
2291 		pr_err("SLUB: Unable to initialize free list for %s\n",
2292 			s->name);
2293 		return err;
2294 	}
2295 
2296 	/* Transform to an offset on the set of pages */
2297 	if (s->random_seq) {
2298 		unsigned int i;
2299 
2300 		for (i = 0; i < count; i++)
2301 			s->random_seq[i] *= s->size;
2302 	}
2303 	return 0;
2304 }
2305 
2306 /* Initialize each random sequence freelist per cache */
2307 static void __init init_freelist_randomization(void)
2308 {
2309 	struct kmem_cache *s;
2310 
2311 	mutex_lock(&slab_mutex);
2312 
2313 	list_for_each_entry(s, &slab_caches, list)
2314 		init_cache_random_seq(s);
2315 
2316 	mutex_unlock(&slab_mutex);
2317 }
2318 
2319 /* Get the next entry on the pre-computed freelist randomized */
2320 static void *next_freelist_entry(struct kmem_cache *s,
2321 				unsigned long *pos, void *start,
2322 				unsigned long page_limit,
2323 				unsigned long freelist_count)
2324 {
2325 	unsigned int idx;
2326 
2327 	/*
2328 	 * If the target page allocation failed, the number of objects on the
2329 	 * page might be smaller than the usual size defined by the cache.
2330 	 */
2331 	do {
2332 		idx = s->random_seq[*pos];
2333 		*pos += 1;
2334 		if (*pos >= freelist_count)
2335 			*pos = 0;
2336 	} while (unlikely(idx >= page_limit));
2337 
2338 	return (char *)start + idx;
2339 }
2340 
2341 /* Shuffle the single linked freelist based on a random pre-computed sequence */
2342 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2343 {
2344 	void *start;
2345 	void *cur;
2346 	void *next;
2347 	unsigned long idx, pos, page_limit, freelist_count;
2348 
2349 	if (slab->objects < 2 || !s->random_seq)
2350 		return false;
2351 
2352 	freelist_count = oo_objects(s->oo);
2353 	pos = get_random_u32_below(freelist_count);
2354 
2355 	page_limit = slab->objects * s->size;
2356 	start = fixup_red_left(s, slab_address(slab));
2357 
2358 	/* First entry is used as the base of the freelist */
2359 	cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count);
2360 	cur = setup_object(s, cur);
2361 	slab->freelist = cur;
2362 
2363 	for (idx = 1; idx < slab->objects; idx++) {
2364 		next = next_freelist_entry(s, &pos, start, page_limit,
2365 			freelist_count);
2366 		next = setup_object(s, next);
2367 		set_freepointer(s, cur, next);
2368 		cur = next;
2369 	}
2370 	set_freepointer(s, cur, NULL);
2371 
2372 	return true;
2373 }
2374 #else
2375 static inline int init_cache_random_seq(struct kmem_cache *s)
2376 {
2377 	return 0;
2378 }
2379 static inline void init_freelist_randomization(void) { }
2380 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2381 {
2382 	return false;
2383 }
2384 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
2385 
2386 static __always_inline void account_slab(struct slab *slab, int order,
2387 					 struct kmem_cache *s, gfp_t gfp)
2388 {
2389 	if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
2390 		alloc_slab_obj_exts(slab, s, gfp, true);
2391 
2392 	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
2393 			    PAGE_SIZE << order);
2394 }
2395 
2396 static __always_inline void unaccount_slab(struct slab *slab, int order,
2397 					   struct kmem_cache *s)
2398 {
2399 	if (memcg_kmem_online() || need_slab_obj_ext())
2400 		free_slab_obj_exts(slab);
2401 
2402 	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
2403 			    -(PAGE_SIZE << order));
2404 }
2405 
2406 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
2407 {
2408 	struct slab *slab;
2409 	struct kmem_cache_order_objects oo = s->oo;
2410 	gfp_t alloc_gfp;
2411 	void *start, *p, *next;
2412 	int idx;
2413 	bool shuffle;
2414 
2415 	flags &= gfp_allowed_mask;
2416 
2417 	flags |= s->allocflags;
2418 
2419 	/*
2420 	 * Let the initial higher-order allocation fail under memory pressure
2421 	 * so we fall-back to the minimum order allocation.
2422 	 */
2423 	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
2424 	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
2425 		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
2426 
2427 	slab = alloc_slab_page(alloc_gfp, node, oo);
2428 	if (unlikely(!slab)) {
2429 		oo = s->min;
2430 		alloc_gfp = flags;
2431 		/*
2432 		 * Allocation may have failed due to fragmentation.
2433 		 * Try a lower order alloc if possible
2434 		 */
2435 		slab = alloc_slab_page(alloc_gfp, node, oo);
2436 		if (unlikely(!slab))
2437 			return NULL;
2438 		stat(s, ORDER_FALLBACK);
2439 	}
2440 
2441 	slab->objects = oo_objects(oo);
2442 	slab->inuse = 0;
2443 	slab->frozen = 0;
2444 
2445 	account_slab(slab, oo_order(oo), s, flags);
2446 
2447 	slab->slab_cache = s;
2448 
2449 	kasan_poison_slab(slab);
2450 
2451 	start = slab_address(slab);
2452 
2453 	setup_slab_debug(s, slab, start);
2454 
2455 	shuffle = shuffle_freelist(s, slab);
2456 
2457 	if (!shuffle) {
2458 		start = fixup_red_left(s, start);
2459 		start = setup_object(s, start);
2460 		slab->freelist = start;
2461 		for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
2462 			next = p + s->size;
2463 			next = setup_object(s, next);
2464 			set_freepointer(s, p, next);
2465 			p = next;
2466 		}
2467 		set_freepointer(s, p, NULL);
2468 	}
2469 
2470 	return slab;
2471 }
2472 
2473 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
2474 {
2475 	if (unlikely(flags & GFP_SLAB_BUG_MASK))
2476 		flags = kmalloc_fix_flags(flags);
2477 
2478 	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
2479 
2480 	return allocate_slab(s,
2481 		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
2482 }
2483 
2484 static void __free_slab(struct kmem_cache *s, struct slab *slab)
2485 {
2486 	struct folio *folio = slab_folio(slab);
2487 	int order = folio_order(folio);
2488 	int pages = 1 << order;
2489 
2490 	__slab_clear_pfmemalloc(slab);
2491 	folio->mapping = NULL;
2492 	/* Make the mapping reset visible before clearing the flag */
2493 	smp_wmb();
2494 	__folio_clear_slab(folio);
2495 	mm_account_reclaimed_pages(pages);
2496 	unaccount_slab(slab, order, s);
2497 	__free_pages(&folio->page, order);
2498 }
2499 
2500 static void rcu_free_slab(struct rcu_head *h)
2501 {
2502 	struct slab *slab = container_of(h, struct slab, rcu_head);
2503 
2504 	__free_slab(slab->slab_cache, slab);
2505 }
2506 
2507 static void free_slab(struct kmem_cache *s, struct slab *slab)
2508 {
2509 	if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
2510 		void *p;
2511 
2512 		slab_pad_check(s, slab);
2513 		for_each_object(p, s, slab_address(slab), slab->objects)
2514 			check_object(s, slab, p, SLUB_RED_INACTIVE);
2515 	}
2516 
2517 	if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
2518 		call_rcu(&slab->rcu_head, rcu_free_slab);
2519 	else
2520 		__free_slab(s, slab);
2521 }
2522 
2523 static void discard_slab(struct kmem_cache *s, struct slab *slab)
2524 {
2525 	dec_slabs_node(s, slab_nid(slab), slab->objects);
2526 	free_slab(s, slab);
2527 }
2528 
2529 /*
2530  * SLUB reuses PG_workingset bit to keep track of whether it's on
2531  * the per-node partial list.
2532  */
2533 static inline bool slab_test_node_partial(const struct slab *slab)
2534 {
2535 	return folio_test_workingset((struct folio *)slab_folio(slab));
2536 }
2537 
2538 static inline void slab_set_node_partial(struct slab *slab)
2539 {
2540 	set_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
2541 }
2542 
2543 static inline void slab_clear_node_partial(struct slab *slab)
2544 {
2545 	clear_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
2546 }
2547 
2548 /*
2549  * Management of partially allocated slabs.
2550  */
2551 static inline void
2552 __add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
2553 {
2554 	n->nr_partial++;
2555 	if (tail == DEACTIVATE_TO_TAIL)
2556 		list_add_tail(&slab->slab_list, &n->partial);
2557 	else
2558 		list_add(&slab->slab_list, &n->partial);
2559 	slab_set_node_partial(slab);
2560 }
2561 
2562 static inline void add_partial(struct kmem_cache_node *n,
2563 				struct slab *slab, int tail)
2564 {
2565 	lockdep_assert_held(&n->list_lock);
2566 	__add_partial(n, slab, tail);
2567 }
2568 
2569 static inline void remove_partial(struct kmem_cache_node *n,
2570 					struct slab *slab)
2571 {
2572 	lockdep_assert_held(&n->list_lock);
2573 	list_del(&slab->slab_list);
2574 	slab_clear_node_partial(slab);
2575 	n->nr_partial--;
2576 }
2577 
2578 /*
2579  * Called only for kmem_cache_debug() caches instead of remove_partial(), with a
2580  * slab from the n->partial list. Remove only a single object from the slab, do
2581  * the alloc_debug_processing() checks and leave the slab on the list, or move
2582  * it to full list if it was the last free object.
2583  */
2584 static void *alloc_single_from_partial(struct kmem_cache *s,
2585 		struct kmem_cache_node *n, struct slab *slab, int orig_size)
2586 {
2587 	void *object;
2588 
2589 	lockdep_assert_held(&n->list_lock);
2590 
2591 	object = slab->freelist;
2592 	slab->freelist = get_freepointer(s, object);
2593 	slab->inuse++;
2594 
2595 	if (!alloc_debug_processing(s, slab, object, orig_size)) {
2596 		remove_partial(n, slab);
2597 		return NULL;
2598 	}
2599 
2600 	if (slab->inuse == slab->objects) {
2601 		remove_partial(n, slab);
2602 		add_full(s, n, slab);
2603 	}
2604 
2605 	return object;
2606 }
2607 
2608 /*
2609  * Called only for kmem_cache_debug() caches to allocate from a freshly
2610  * allocated slab. Allocate a single object instead of whole freelist
2611  * and put the slab to the partial (or full) list.
2612  */
2613 static void *alloc_single_from_new_slab(struct kmem_cache *s,
2614 					struct slab *slab, int orig_size)
2615 {
2616 	int nid = slab_nid(slab);
2617 	struct kmem_cache_node *n = get_node(s, nid);
2618 	unsigned long flags;
2619 	void *object;
2620 
2621 
2622 	object = slab->freelist;
2623 	slab->freelist = get_freepointer(s, object);
2624 	slab->inuse = 1;
2625 
2626 	if (!alloc_debug_processing(s, slab, object, orig_size))
2627 		/*
2628 		 * It's not really expected that this would fail on a
2629 		 * freshly allocated slab, but a concurrent memory
2630 		 * corruption in theory could cause that.
2631 		 */
2632 		return NULL;
2633 
2634 	spin_lock_irqsave(&n->list_lock, flags);
2635 
2636 	if (slab->inuse == slab->objects)
2637 		add_full(s, n, slab);
2638 	else
2639 		add_partial(n, slab, DEACTIVATE_TO_HEAD);
2640 
2641 	inc_slabs_node(s, nid, slab->objects);
2642 	spin_unlock_irqrestore(&n->list_lock, flags);
2643 
2644 	return object;
2645 }
2646 
2647 #ifdef CONFIG_SLUB_CPU_PARTIAL
2648 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
2649 #else
2650 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
2651 				   int drain) { }
2652 #endif
2653 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
2654 
2655 /*
2656  * Try to allocate a partial slab from a specific node.
2657  */
2658 static struct slab *get_partial_node(struct kmem_cache *s,
2659 				     struct kmem_cache_node *n,
2660 				     struct partial_context *pc)
2661 {
2662 	struct slab *slab, *slab2, *partial = NULL;
2663 	unsigned long flags;
2664 	unsigned int partial_slabs = 0;
2665 
2666 	/*
2667 	 * Racy check. If we mistakenly see no partial slabs then we
2668 	 * just allocate an empty slab. If we mistakenly try to get a
2669 	 * partial slab and there is none available then get_partial()
2670 	 * will return NULL.
2671 	 */
2672 	if (!n || !n->nr_partial)
2673 		return NULL;
2674 
2675 	spin_lock_irqsave(&n->list_lock, flags);
2676 	list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
2677 		if (!pfmemalloc_match(slab, pc->flags))
2678 			continue;
2679 
2680 		if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
2681 			void *object = alloc_single_from_partial(s, n, slab,
2682 							pc->orig_size);
2683 			if (object) {
2684 				partial = slab;
2685 				pc->object = object;
2686 				break;
2687 			}
2688 			continue;
2689 		}
2690 
2691 		remove_partial(n, slab);
2692 
2693 		if (!partial) {
2694 			partial = slab;
2695 			stat(s, ALLOC_FROM_PARTIAL);
2696 
2697 			if ((slub_get_cpu_partial(s) == 0)) {
2698 				break;
2699 			}
2700 		} else {
2701 			put_cpu_partial(s, slab, 0);
2702 			stat(s, CPU_PARTIAL_NODE);
2703 
2704 			if (++partial_slabs > slub_get_cpu_partial(s) / 2) {
2705 				break;
2706 			}
2707 		}
2708 	}
2709 	spin_unlock_irqrestore(&n->list_lock, flags);
2710 	return partial;
2711 }
2712 
2713 /*
2714  * Get a slab from somewhere. Search in increasing NUMA distances.
2715  */
2716 static struct slab *get_any_partial(struct kmem_cache *s,
2717 				    struct partial_context *pc)
2718 {
2719 #ifdef CONFIG_NUMA
2720 	struct zonelist *zonelist;
2721 	struct zoneref *z;
2722 	struct zone *zone;
2723 	enum zone_type highest_zoneidx = gfp_zone(pc->flags);
2724 	struct slab *slab;
2725 	unsigned int cpuset_mems_cookie;
2726 
2727 	/*
2728 	 * The defrag ratio allows a configuration of the tradeoffs between
2729 	 * inter node defragmentation and node local allocations. A lower
2730 	 * defrag_ratio increases the tendency to do local allocations
2731 	 * instead of attempting to obtain partial slabs from other nodes.
2732 	 *
2733 	 * If the defrag_ratio is set to 0 then kmalloc() always
2734 	 * returns node local objects. If the ratio is higher then kmalloc()
2735 	 * may return off node objects because partial slabs are obtained
2736 	 * from other nodes and filled up.
2737 	 *
2738 	 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
2739 	 * (which makes defrag_ratio = 1000) then every (well almost)
2740 	 * allocation will first attempt to defrag slab caches on other nodes.
2741 	 * This means scanning over all nodes to look for partial slabs which
2742 	 * may be expensive if we do it every time we are trying to find a slab
2743 	 * with available objects.
2744 	 */
2745 	if (!s->remote_node_defrag_ratio ||
2746 			get_cycles() % 1024 > s->remote_node_defrag_ratio)
2747 		return NULL;
2748 
2749 	do {
2750 		cpuset_mems_cookie = read_mems_allowed_begin();
2751 		zonelist = node_zonelist(mempolicy_slab_node(), pc->flags);
2752 		for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
2753 			struct kmem_cache_node *n;
2754 
2755 			n = get_node(s, zone_to_nid(zone));
2756 
2757 			if (n && cpuset_zone_allowed(zone, pc->flags) &&
2758 					n->nr_partial > s->min_partial) {
2759 				slab = get_partial_node(s, n, pc);
2760 				if (slab) {
2761 					/*
2762 					 * Don't check read_mems_allowed_retry()
2763 					 * here - if mems_allowed was updated in
2764 					 * parallel, that was a harmless race
2765 					 * between allocation and the cpuset
2766 					 * update
2767 					 */
2768 					return slab;
2769 				}
2770 			}
2771 		}
2772 	} while (read_mems_allowed_retry(cpuset_mems_cookie));
2773 #endif	/* CONFIG_NUMA */
2774 	return NULL;
2775 }
2776 
2777 /*
2778  * Get a partial slab, lock it and return it.
2779  */
2780 static struct slab *get_partial(struct kmem_cache *s, int node,
2781 				struct partial_context *pc)
2782 {
2783 	struct slab *slab;
2784 	int searchnode = node;
2785 
2786 	if (node == NUMA_NO_NODE)
2787 		searchnode = numa_mem_id();
2788 
2789 	slab = get_partial_node(s, get_node(s, searchnode), pc);
2790 	if (slab || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE)))
2791 		return slab;
2792 
2793 	return get_any_partial(s, pc);
2794 }
2795 
2796 #ifndef CONFIG_SLUB_TINY
2797 
2798 #ifdef CONFIG_PREEMPTION
2799 /*
2800  * Calculate the next globally unique transaction for disambiguation
2801  * during cmpxchg. The transactions start with the cpu number and are then
2802  * incremented by CONFIG_NR_CPUS.
2803  */
2804 #define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
2805 #else
2806 /*
2807  * No preemption supported therefore also no need to check for
2808  * different cpus.
2809  */
2810 #define TID_STEP 1
2811 #endif /* CONFIG_PREEMPTION */
2812 
2813 static inline unsigned long next_tid(unsigned long tid)
2814 {
2815 	return tid + TID_STEP;
2816 }
2817 
2818 #ifdef SLUB_DEBUG_CMPXCHG
2819 static inline unsigned int tid_to_cpu(unsigned long tid)
2820 {
2821 	return tid % TID_STEP;
2822 }
2823 
2824 static inline unsigned long tid_to_event(unsigned long tid)
2825 {
2826 	return tid / TID_STEP;
2827 }
2828 #endif
2829 
2830 static inline unsigned int init_tid(int cpu)
2831 {
2832 	return cpu;
2833 }
2834 
2835 static inline void note_cmpxchg_failure(const char *n,
2836 		const struct kmem_cache *s, unsigned long tid)
2837 {
2838 #ifdef SLUB_DEBUG_CMPXCHG
2839 	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
2840 
2841 	pr_info("%s %s: cmpxchg redo ", n, s->name);
2842 
2843 #ifdef CONFIG_PREEMPTION
2844 	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
2845 		pr_warn("due to cpu change %d -> %d\n",
2846 			tid_to_cpu(tid), tid_to_cpu(actual_tid));
2847 	else
2848 #endif
2849 	if (tid_to_event(tid) != tid_to_event(actual_tid))
2850 		pr_warn("due to cpu running other code. Event %ld->%ld\n",
2851 			tid_to_event(tid), tid_to_event(actual_tid));
2852 	else
2853 		pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
2854 			actual_tid, tid, next_tid(tid));
2855 #endif
2856 	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
2857 }
2858 
2859 static void init_kmem_cache_cpus(struct kmem_cache *s)
2860 {
2861 	int cpu;
2862 	struct kmem_cache_cpu *c;
2863 
2864 	for_each_possible_cpu(cpu) {
2865 		c = per_cpu_ptr(s->cpu_slab, cpu);
2866 		local_lock_init(&c->lock);
2867 		c->tid = init_tid(cpu);
2868 	}
2869 }
2870 
2871 /*
2872  * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
2873  * unfreezes the slabs and puts it on the proper list.
2874  * Assumes the slab has been already safely taken away from kmem_cache_cpu
2875  * by the caller.
2876  */
2877 static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
2878 			    void *freelist)
2879 {
2880 	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
2881 	int free_delta = 0;
2882 	void *nextfree, *freelist_iter, *freelist_tail;
2883 	int tail = DEACTIVATE_TO_HEAD;
2884 	unsigned long flags = 0;
2885 	struct slab new;
2886 	struct slab old;
2887 
2888 	if (READ_ONCE(slab->freelist)) {
2889 		stat(s, DEACTIVATE_REMOTE_FREES);
2890 		tail = DEACTIVATE_TO_TAIL;
2891 	}
2892 
2893 	/*
2894 	 * Stage one: Count the objects on cpu's freelist as free_delta and
2895 	 * remember the last object in freelist_tail for later splicing.
2896 	 */
2897 	freelist_tail = NULL;
2898 	freelist_iter = freelist;
2899 	while (freelist_iter) {
2900 		nextfree = get_freepointer(s, freelist_iter);
2901 
2902 		/*
2903 		 * If 'nextfree' is invalid, it is possible that the object at
2904 		 * 'freelist_iter' is already corrupted.  So isolate all objects
2905 		 * starting at 'freelist_iter' by skipping them.
2906 		 */
2907 		if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
2908 			break;
2909 
2910 		freelist_tail = freelist_iter;
2911 		free_delta++;
2912 
2913 		freelist_iter = nextfree;
2914 	}
2915 
2916 	/*
2917 	 * Stage two: Unfreeze the slab while splicing the per-cpu
2918 	 * freelist to the head of slab's freelist.
2919 	 */
2920 	do {
2921 		old.freelist = READ_ONCE(slab->freelist);
2922 		old.counters = READ_ONCE(slab->counters);
2923 		VM_BUG_ON(!old.frozen);
2924 
2925 		/* Determine target state of the slab */
2926 		new.counters = old.counters;
2927 		new.frozen = 0;
2928 		if (freelist_tail) {
2929 			new.inuse -= free_delta;
2930 			set_freepointer(s, freelist_tail, old.freelist);
2931 			new.freelist = freelist;
2932 		} else {
2933 			new.freelist = old.freelist;
2934 		}
2935 	} while (!slab_update_freelist(s, slab,
2936 		old.freelist, old.counters,
2937 		new.freelist, new.counters,
2938 		"unfreezing slab"));
2939 
2940 	/*
2941 	 * Stage three: Manipulate the slab list based on the updated state.
2942 	 */
2943 	if (!new.inuse && n->nr_partial >= s->min_partial) {
2944 		stat(s, DEACTIVATE_EMPTY);
2945 		discard_slab(s, slab);
2946 		stat(s, FREE_SLAB);
2947 	} else if (new.freelist) {
2948 		spin_lock_irqsave(&n->list_lock, flags);
2949 		add_partial(n, slab, tail);
2950 		spin_unlock_irqrestore(&n->list_lock, flags);
2951 		stat(s, tail);
2952 	} else {
2953 		stat(s, DEACTIVATE_FULL);
2954 	}
2955 }
2956 
2957 #ifdef CONFIG_SLUB_CPU_PARTIAL
2958 static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
2959 {
2960 	struct kmem_cache_node *n = NULL, *n2 = NULL;
2961 	struct slab *slab, *slab_to_discard = NULL;
2962 	unsigned long flags = 0;
2963 
2964 	while (partial_slab) {
2965 		slab = partial_slab;
2966 		partial_slab = slab->next;
2967 
2968 		n2 = get_node(s, slab_nid(slab));
2969 		if (n != n2) {
2970 			if (n)
2971 				spin_unlock_irqrestore(&n->list_lock, flags);
2972 
2973 			n = n2;
2974 			spin_lock_irqsave(&n->list_lock, flags);
2975 		}
2976 
2977 		if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
2978 			slab->next = slab_to_discard;
2979 			slab_to_discard = slab;
2980 		} else {
2981 			add_partial(n, slab, DEACTIVATE_TO_TAIL);
2982 			stat(s, FREE_ADD_PARTIAL);
2983 		}
2984 	}
2985 
2986 	if (n)
2987 		spin_unlock_irqrestore(&n->list_lock, flags);
2988 
2989 	while (slab_to_discard) {
2990 		slab = slab_to_discard;
2991 		slab_to_discard = slab_to_discard->next;
2992 
2993 		stat(s, DEACTIVATE_EMPTY);
2994 		discard_slab(s, slab);
2995 		stat(s, FREE_SLAB);
2996 	}
2997 }
2998 
2999 /*
3000  * Put all the cpu partial slabs to the node partial list.
3001  */
3002 static void put_partials(struct kmem_cache *s)
3003 {
3004 	struct slab *partial_slab;
3005 	unsigned long flags;
3006 
3007 	local_lock_irqsave(&s->cpu_slab->lock, flags);
3008 	partial_slab = this_cpu_read(s->cpu_slab->partial);
3009 	this_cpu_write(s->cpu_slab->partial, NULL);
3010 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3011 
3012 	if (partial_slab)
3013 		__put_partials(s, partial_slab);
3014 }
3015 
3016 static void put_partials_cpu(struct kmem_cache *s,
3017 			     struct kmem_cache_cpu *c)
3018 {
3019 	struct slab *partial_slab;
3020 
3021 	partial_slab = slub_percpu_partial(c);
3022 	c->partial = NULL;
3023 
3024 	if (partial_slab)
3025 		__put_partials(s, partial_slab);
3026 }
3027 
3028 /*
3029  * Put a slab into a partial slab slot if available.
3030  *
3031  * If we did not find a slot then simply move all the partials to the
3032  * per node partial list.
3033  */
3034 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
3035 {
3036 	struct slab *oldslab;
3037 	struct slab *slab_to_put = NULL;
3038 	unsigned long flags;
3039 	int slabs = 0;
3040 
3041 	local_lock_irqsave(&s->cpu_slab->lock, flags);
3042 
3043 	oldslab = this_cpu_read(s->cpu_slab->partial);
3044 
3045 	if (oldslab) {
3046 		if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
3047 			/*
3048 			 * Partial array is full. Move the existing set to the
3049 			 * per node partial list. Postpone the actual unfreezing
3050 			 * outside of the critical section.
3051 			 */
3052 			slab_to_put = oldslab;
3053 			oldslab = NULL;
3054 		} else {
3055 			slabs = oldslab->slabs;
3056 		}
3057 	}
3058 
3059 	slabs++;
3060 
3061 	slab->slabs = slabs;
3062 	slab->next = oldslab;
3063 
3064 	this_cpu_write(s->cpu_slab->partial, slab);
3065 
3066 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3067 
3068 	if (slab_to_put) {
3069 		__put_partials(s, slab_to_put);
3070 		stat(s, CPU_PARTIAL_DRAIN);
3071 	}
3072 }
3073 
3074 #else	/* CONFIG_SLUB_CPU_PARTIAL */
3075 
3076 static inline void put_partials(struct kmem_cache *s) { }
3077 static inline void put_partials_cpu(struct kmem_cache *s,
3078 				    struct kmem_cache_cpu *c) { }
3079 
3080 #endif	/* CONFIG_SLUB_CPU_PARTIAL */
3081 
3082 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
3083 {
3084 	unsigned long flags;
3085 	struct slab *slab;
3086 	void *freelist;
3087 
3088 	local_lock_irqsave(&s->cpu_slab->lock, flags);
3089 
3090 	slab = c->slab;
3091 	freelist = c->freelist;
3092 
3093 	c->slab = NULL;
3094 	c->freelist = NULL;
3095 	c->tid = next_tid(c->tid);
3096 
3097 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3098 
3099 	if (slab) {
3100 		deactivate_slab(s, slab, freelist);
3101 		stat(s, CPUSLAB_FLUSH);
3102 	}
3103 }
3104 
3105 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
3106 {
3107 	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
3108 	void *freelist = c->freelist;
3109 	struct slab *slab = c->slab;
3110 
3111 	c->slab = NULL;
3112 	c->freelist = NULL;
3113 	c->tid = next_tid(c->tid);
3114 
3115 	if (slab) {
3116 		deactivate_slab(s, slab, freelist);
3117 		stat(s, CPUSLAB_FLUSH);
3118 	}
3119 
3120 	put_partials_cpu(s, c);
3121 }
3122 
3123 struct slub_flush_work {
3124 	struct work_struct work;
3125 	struct kmem_cache *s;
3126 	bool skip;
3127 };
3128 
3129 /*
3130  * Flush cpu slab.
3131  *
3132  * Called from CPU work handler with migration disabled.
3133  */
3134 static void flush_cpu_slab(struct work_struct *w)
3135 {
3136 	struct kmem_cache *s;
3137 	struct kmem_cache_cpu *c;
3138 	struct slub_flush_work *sfw;
3139 
3140 	sfw = container_of(w, struct slub_flush_work, work);
3141 
3142 	s = sfw->s;
3143 	c = this_cpu_ptr(s->cpu_slab);
3144 
3145 	if (c->slab)
3146 		flush_slab(s, c);
3147 
3148 	put_partials(s);
3149 }
3150 
3151 static bool has_cpu_slab(int cpu, struct kmem_cache *s)
3152 {
3153 	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
3154 
3155 	return c->slab || slub_percpu_partial(c);
3156 }
3157 
3158 static DEFINE_MUTEX(flush_lock);
3159 static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);
3160 
3161 static void flush_all_cpus_locked(struct kmem_cache *s)
3162 {
3163 	struct slub_flush_work *sfw;
3164 	unsigned int cpu;
3165 
3166 	lockdep_assert_cpus_held();
3167 	mutex_lock(&flush_lock);
3168 
3169 	for_each_online_cpu(cpu) {
3170 		sfw = &per_cpu(slub_flush, cpu);
3171 		if (!has_cpu_slab(cpu, s)) {
3172 			sfw->skip = true;
3173 			continue;
3174 		}
3175 		INIT_WORK(&sfw->work, flush_cpu_slab);
3176 		sfw->skip = false;
3177 		sfw->s = s;
3178 		queue_work_on(cpu, flushwq, &sfw->work);
3179 	}
3180 
3181 	for_each_online_cpu(cpu) {
3182 		sfw = &per_cpu(slub_flush, cpu);
3183 		if (sfw->skip)
3184 			continue;
3185 		flush_work(&sfw->work);
3186 	}
3187 
3188 	mutex_unlock(&flush_lock);
3189 }
3190 
3191 static void flush_all(struct kmem_cache *s)
3192 {
3193 	cpus_read_lock();
3194 	flush_all_cpus_locked(s);
3195 	cpus_read_unlock();
3196 }
3197 
3198 /*
3199  * Use the cpu notifier to insure that the cpu slabs are flushed when
3200  * necessary.
3201  */
3202 static int slub_cpu_dead(unsigned int cpu)
3203 {
3204 	struct kmem_cache *s;
3205 
3206 	mutex_lock(&slab_mutex);
3207 	list_for_each_entry(s, &slab_caches, list)
3208 		__flush_cpu_slab(s, cpu);
3209 	mutex_unlock(&slab_mutex);
3210 	return 0;
3211 }
3212 
3213 #else /* CONFIG_SLUB_TINY */
3214 static inline void flush_all_cpus_locked(struct kmem_cache *s) { }
3215 static inline void flush_all(struct kmem_cache *s) { }
3216 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { }
3217 static inline int slub_cpu_dead(unsigned int cpu) { return 0; }
3218 #endif /* CONFIG_SLUB_TINY */
3219 
3220 /*
3221  * Check if the objects in a per cpu structure fit numa
3222  * locality expectations.
3223  */
3224 static inline int node_match(struct slab *slab, int node)
3225 {
3226 #ifdef CONFIG_NUMA
3227 	if (node != NUMA_NO_NODE && slab_nid(slab) != node)
3228 		return 0;
3229 #endif
3230 	return 1;
3231 }
3232 
3233 #ifdef CONFIG_SLUB_DEBUG
3234 static int count_free(struct slab *slab)
3235 {
3236 	return slab->objects - slab->inuse;
3237 }
3238 
3239 static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
3240 {
3241 	return atomic_long_read(&n->total_objects);
3242 }
3243 
3244 /* Supports checking bulk free of a constructed freelist */
3245 static inline bool free_debug_processing(struct kmem_cache *s,
3246 	struct slab *slab, void *head, void *tail, int *bulk_cnt,
3247 	unsigned long addr, depot_stack_handle_t handle)
3248 {
3249 	bool checks_ok = false;
3250 	void *object = head;
3251 	int cnt = 0;
3252 
3253 	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
3254 		if (!check_slab(s, slab))
3255 			goto out;
3256 	}
3257 
3258 	if (slab->inuse < *bulk_cnt) {
3259 		slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",
3260 			 slab->inuse, *bulk_cnt);
3261 		goto out;
3262 	}
3263 
3264 next_object:
3265 
3266 	if (++cnt > *bulk_cnt)
3267 		goto out_cnt;
3268 
3269 	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
3270 		if (!free_consistency_checks(s, slab, object, addr))
3271 			goto out;
3272 	}
3273 
3274 	if (s->flags & SLAB_STORE_USER)
3275 		set_track_update(s, object, TRACK_FREE, addr, handle);
3276 	trace(s, slab, object, 0);
3277 	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
3278 	init_object(s, object, SLUB_RED_INACTIVE);
3279 
3280 	/* Reached end of constructed freelist yet? */
3281 	if (object != tail) {
3282 		object = get_freepointer(s, object);
3283 		goto next_object;
3284 	}
3285 	checks_ok = true;
3286 
3287 out_cnt:
3288 	if (cnt != *bulk_cnt) {
3289 		slab_err(s, slab, "Bulk free expected %d objects but found %d\n",
3290 			 *bulk_cnt, cnt);
3291 		*bulk_cnt = cnt;
3292 	}
3293 
3294 out:
3295 
3296 	if (!checks_ok)
3297 		slab_fix(s, "Object at 0x%p not freed", object);
3298 
3299 	return checks_ok;
3300 }
3301 #endif /* CONFIG_SLUB_DEBUG */
3302 
3303 #if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS)
3304 static unsigned long count_partial(struct kmem_cache_node *n,
3305 					int (*get_count)(struct slab *))
3306 {
3307 	unsigned long flags;
3308 	unsigned long x = 0;
3309 	struct slab *slab;
3310 
3311 	spin_lock_irqsave(&n->list_lock, flags);
3312 	list_for_each_entry(slab, &n->partial, slab_list)
3313 		x += get_count(slab);
3314 	spin_unlock_irqrestore(&n->list_lock, flags);
3315 	return x;
3316 }
3317 #endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */
3318 
3319 #ifdef CONFIG_SLUB_DEBUG
3320 #define MAX_PARTIAL_TO_SCAN 10000
3321 
3322 static unsigned long count_partial_free_approx(struct kmem_cache_node *n)
3323 {
3324 	unsigned long flags;
3325 	unsigned long x = 0;
3326 	struct slab *slab;
3327 
3328 	spin_lock_irqsave(&n->list_lock, flags);
3329 	if (n->nr_partial <= MAX_PARTIAL_TO_SCAN) {
3330 		list_for_each_entry(slab, &n->partial, slab_list)
3331 			x += slab->objects - slab->inuse;
3332 	} else {
3333 		/*
3334 		 * For a long list, approximate the total count of objects in
3335 		 * it to meet the limit on the number of slabs to scan.
3336 		 * Scan from both the list's head and tail for better accuracy.
3337 		 */
3338 		unsigned long scanned = 0;
3339 
3340 		list_for_each_entry(slab, &n->partial, slab_list) {
3341 			x += slab->objects - slab->inuse;
3342 			if (++scanned == MAX_PARTIAL_TO_SCAN / 2)
3343 				break;
3344 		}
3345 		list_for_each_entry_reverse(slab, &n->partial, slab_list) {
3346 			x += slab->objects - slab->inuse;
3347 			if (++scanned == MAX_PARTIAL_TO_SCAN)
3348 				break;
3349 		}
3350 		x = mult_frac(x, n->nr_partial, scanned);
3351 		x = min(x, node_nr_objs(n));
3352 	}
3353 	spin_unlock_irqrestore(&n->list_lock, flags);
3354 	return x;
3355 }
3356 
3357 static noinline void
3358 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
3359 {
3360 	static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
3361 				      DEFAULT_RATELIMIT_BURST);
3362 	int node;
3363 	struct kmem_cache_node *n;
3364 
3365 	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
3366 		return;
3367 
3368 	pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
3369 		nid, gfpflags, &gfpflags);
3370 	pr_warn("  cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
3371 		s->name, s->object_size, s->size, oo_order(s->oo),
3372 		oo_order(s->min));
3373 
3374 	if (oo_order(s->min) > get_order(s->object_size))
3375 		pr_warn("  %s debugging increased min order, use slab_debug=O to disable.\n",
3376 			s->name);
3377 
3378 	for_each_kmem_cache_node(s, node, n) {
3379 		unsigned long nr_slabs;
3380 		unsigned long nr_objs;
3381 		unsigned long nr_free;
3382 
3383 		nr_free  = count_partial_free_approx(n);
3384 		nr_slabs = node_nr_slabs(n);
3385 		nr_objs  = node_nr_objs(n);
3386 
3387 		pr_warn("  node %d: slabs: %ld, objs: %ld, free: %ld\n",
3388 			node, nr_slabs, nr_objs, nr_free);
3389 	}
3390 }
3391 #else /* CONFIG_SLUB_DEBUG */
3392 static inline void
3393 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { }
3394 #endif
3395 
3396 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
3397 {
3398 	if (unlikely(slab_test_pfmemalloc(slab)))
3399 		return gfp_pfmemalloc_allowed(gfpflags);
3400 
3401 	return true;
3402 }
3403 
3404 #ifndef CONFIG_SLUB_TINY
3405 static inline bool
3406 __update_cpu_freelist_fast(struct kmem_cache *s,
3407 			   void *freelist_old, void *freelist_new,
3408 			   unsigned long tid)
3409 {
3410 	freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
3411 	freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };
3412 
3413 	return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
3414 					     &old.full, new.full);
3415 }
3416 
3417 /*
3418  * Check the slab->freelist and either transfer the freelist to the
3419  * per cpu freelist or deactivate the slab.
3420  *
3421  * The slab is still frozen if the return value is not NULL.
3422  *
3423  * If this function returns NULL then the slab has been unfrozen.
3424  */
3425 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
3426 {
3427 	struct slab new;
3428 	unsigned long counters;
3429 	void *freelist;
3430 
3431 	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
3432 
3433 	do {
3434 		freelist = slab->freelist;
3435 		counters = slab->counters;
3436 
3437 		new.counters = counters;
3438 
3439 		new.inuse = slab->objects;
3440 		new.frozen = freelist != NULL;
3441 
3442 	} while (!__slab_update_freelist(s, slab,
3443 		freelist, counters,
3444 		NULL, new.counters,
3445 		"get_freelist"));
3446 
3447 	return freelist;
3448 }
3449 
3450 /*
3451  * Freeze the partial slab and return the pointer to the freelist.
3452  */
3453 static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab)
3454 {
3455 	struct slab new;
3456 	unsigned long counters;
3457 	void *freelist;
3458 
3459 	do {
3460 		freelist = slab->freelist;
3461 		counters = slab->counters;
3462 
3463 		new.counters = counters;
3464 		VM_BUG_ON(new.frozen);
3465 
3466 		new.inuse = slab->objects;
3467 		new.frozen = 1;
3468 
3469 	} while (!slab_update_freelist(s, slab,
3470 		freelist, counters,
3471 		NULL, new.counters,
3472 		"freeze_slab"));
3473 
3474 	return freelist;
3475 }
3476 
3477 /*
3478  * Slow path. The lockless freelist is empty or we need to perform
3479  * debugging duties.
3480  *
3481  * Processing is still very fast if new objects have been freed to the
3482  * regular freelist. In that case we simply take over the regular freelist
3483  * as the lockless freelist and zap the regular freelist.
3484  *
3485  * If that is not working then we fall back to the partial lists. We take the
3486  * first element of the freelist as the object to allocate now and move the
3487  * rest of the freelist to the lockless freelist.
3488  *
3489  * And if we were unable to get a new slab from the partial slab lists then
3490  * we need to allocate a new slab. This is the slowest path since it involves
3491  * a call to the page allocator and the setup of a new slab.
3492  *
3493  * Version of __slab_alloc to use when we know that preemption is
3494  * already disabled (which is the case for bulk allocation).
3495  */
3496 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3497 			  unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size)
3498 {
3499 	void *freelist;
3500 	struct slab *slab;
3501 	unsigned long flags;
3502 	struct partial_context pc;
3503 	bool try_thisnode = true;
3504 
3505 	stat(s, ALLOC_SLOWPATH);
3506 
3507 reread_slab:
3508 
3509 	slab = READ_ONCE(c->slab);
3510 	if (!slab) {
3511 		/*
3512 		 * if the node is not online or has no normal memory, just
3513 		 * ignore the node constraint
3514 		 */
3515 		if (unlikely(node != NUMA_NO_NODE &&
3516 			     !node_isset(node, slab_nodes)))
3517 			node = NUMA_NO_NODE;
3518 		goto new_slab;
3519 	}
3520 
3521 	if (unlikely(!node_match(slab, node))) {
3522 		/*
3523 		 * same as above but node_match() being false already
3524 		 * implies node != NUMA_NO_NODE
3525 		 */
3526 		if (!node_isset(node, slab_nodes)) {
3527 			node = NUMA_NO_NODE;
3528 		} else {
3529 			stat(s, ALLOC_NODE_MISMATCH);
3530 			goto deactivate_slab;
3531 		}
3532 	}
3533 
3534 	/*
3535 	 * By rights, we should be searching for a slab page that was
3536 	 * PFMEMALLOC but right now, we are losing the pfmemalloc
3537 	 * information when the page leaves the per-cpu allocator
3538 	 */
3539 	if (unlikely(!pfmemalloc_match(slab, gfpflags)))
3540 		goto deactivate_slab;
3541 
3542 	/* must check again c->slab in case we got preempted and it changed */
3543 	local_lock_irqsave(&s->cpu_slab->lock, flags);
3544 	if (unlikely(slab != c->slab)) {
3545 		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3546 		goto reread_slab;
3547 	}
3548 	freelist = c->freelist;
3549 	if (freelist)
3550 		goto load_freelist;
3551 
3552 	freelist = get_freelist(s, slab);
3553 
3554 	if (!freelist) {
3555 		c->slab = NULL;
3556 		c->tid = next_tid(c->tid);
3557 		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3558 		stat(s, DEACTIVATE_BYPASS);
3559 		goto new_slab;
3560 	}
3561 
3562 	stat(s, ALLOC_REFILL);
3563 
3564 load_freelist:
3565 
3566 	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
3567 
3568 	/*
3569 	 * freelist is pointing to the list of objects to be used.
3570 	 * slab is pointing to the slab from which the objects are obtained.
3571 	 * That slab must be frozen for per cpu allocations to work.
3572 	 */
3573 	VM_BUG_ON(!c->slab->frozen);
3574 	c->freelist = get_freepointer(s, freelist);
3575 	c->tid = next_tid(c->tid);
3576 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3577 	return freelist;
3578 
3579 deactivate_slab:
3580 
3581 	local_lock_irqsave(&s->cpu_slab->lock, flags);
3582 	if (slab != c->slab) {
3583 		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3584 		goto reread_slab;
3585 	}
3586 	freelist = c->freelist;
3587 	c->slab = NULL;
3588 	c->freelist = NULL;
3589 	c->tid = next_tid(c->tid);
3590 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3591 	deactivate_slab(s, slab, freelist);
3592 
3593 new_slab:
3594 
3595 #ifdef CONFIG_SLUB_CPU_PARTIAL
3596 	while (slub_percpu_partial(c)) {
3597 		local_lock_irqsave(&s->cpu_slab->lock, flags);
3598 		if (unlikely(c->slab)) {
3599 			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3600 			goto reread_slab;
3601 		}
3602 		if (unlikely(!slub_percpu_partial(c))) {
3603 			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3604 			/* we were preempted and partial list got empty */
3605 			goto new_objects;
3606 		}
3607 
3608 		slab = slub_percpu_partial(c);
3609 		slub_set_percpu_partial(c, slab);
3610 
3611 		if (likely(node_match(slab, node) &&
3612 			   pfmemalloc_match(slab, gfpflags))) {
3613 			c->slab = slab;
3614 			freelist = get_freelist(s, slab);
3615 			VM_BUG_ON(!freelist);
3616 			stat(s, CPU_PARTIAL_ALLOC);
3617 			goto load_freelist;
3618 		}
3619 
3620 		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3621 
3622 		slab->next = NULL;
3623 		__put_partials(s, slab);
3624 	}
3625 #endif
3626 
3627 new_objects:
3628 
3629 	pc.flags = gfpflags;
3630 	/*
3631 	 * When a preferred node is indicated but no __GFP_THISNODE
3632 	 *
3633 	 * 1) try to get a partial slab from target node only by having
3634 	 *    __GFP_THISNODE in pc.flags for get_partial()
3635 	 * 2) if 1) failed, try to allocate a new slab from target node with
3636 	 *    GPF_NOWAIT | __GFP_THISNODE opportunistically
3637 	 * 3) if 2) failed, retry with original gfpflags which will allow
3638 	 *    get_partial() try partial lists of other nodes before potentially
3639 	 *    allocating new page from other nodes
3640 	 */
3641 	if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
3642 		     && try_thisnode))
3643 		pc.flags = GFP_NOWAIT | __GFP_THISNODE;
3644 
3645 	pc.orig_size = orig_size;
3646 	slab = get_partial(s, node, &pc);
3647 	if (slab) {
3648 		if (kmem_cache_debug(s)) {
3649 			freelist = pc.object;
3650 			/*
3651 			 * For debug caches here we had to go through
3652 			 * alloc_single_from_partial() so just store the
3653 			 * tracking info and return the object.
3654 			 */
3655 			if (s->flags & SLAB_STORE_USER)
3656 				set_track(s, freelist, TRACK_ALLOC, addr);
3657 
3658 			return freelist;
3659 		}
3660 
3661 		freelist = freeze_slab(s, slab);
3662 		goto retry_load_slab;
3663 	}
3664 
3665 	slub_put_cpu_ptr(s->cpu_slab);
3666 	slab = new_slab(s, pc.flags, node);
3667 	c = slub_get_cpu_ptr(s->cpu_slab);
3668 
3669 	if (unlikely(!slab)) {
3670 		if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
3671 		    && try_thisnode) {
3672 			try_thisnode = false;
3673 			goto new_objects;
3674 		}
3675 		slab_out_of_memory(s, gfpflags, node);
3676 		return NULL;
3677 	}
3678 
3679 	stat(s, ALLOC_SLAB);
3680 
3681 	if (kmem_cache_debug(s)) {
3682 		freelist = alloc_single_from_new_slab(s, slab, orig_size);
3683 
3684 		if (unlikely(!freelist))
3685 			goto new_objects;
3686 
3687 		if (s->flags & SLAB_STORE_USER)
3688 			set_track(s, freelist, TRACK_ALLOC, addr);
3689 
3690 		return freelist;
3691 	}
3692 
3693 	/*
3694 	 * No other reference to the slab yet so we can
3695 	 * muck around with it freely without cmpxchg
3696 	 */
3697 	freelist = slab->freelist;
3698 	slab->freelist = NULL;
3699 	slab->inuse = slab->objects;
3700 	slab->frozen = 1;
3701 
3702 	inc_slabs_node(s, slab_nid(slab), slab->objects);
3703 
3704 	if (unlikely(!pfmemalloc_match(slab, gfpflags))) {
3705 		/*
3706 		 * For !pfmemalloc_match() case we don't load freelist so that
3707 		 * we don't make further mismatched allocations easier.
3708 		 */
3709 		deactivate_slab(s, slab, get_freepointer(s, freelist));
3710 		return freelist;
3711 	}
3712 
3713 retry_load_slab:
3714 
3715 	local_lock_irqsave(&s->cpu_slab->lock, flags);
3716 	if (unlikely(c->slab)) {
3717 		void *flush_freelist = c->freelist;
3718 		struct slab *flush_slab = c->slab;
3719 
3720 		c->slab = NULL;
3721 		c->freelist = NULL;
3722 		c->tid = next_tid(c->tid);
3723 
3724 		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3725 
3726 		deactivate_slab(s, flush_slab, flush_freelist);
3727 
3728 		stat(s, CPUSLAB_FLUSH);
3729 
3730 		goto retry_load_slab;
3731 	}
3732 	c->slab = slab;
3733 
3734 	goto load_freelist;
3735 }
3736 
3737 /*
3738  * A wrapper for ___slab_alloc() for contexts where preemption is not yet
3739  * disabled. Compensates for possible cpu changes by refetching the per cpu area
3740  * pointer.
3741  */
3742 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3743 			  unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size)
3744 {
3745 	void *p;
3746 
3747 #ifdef CONFIG_PREEMPT_COUNT
3748 	/*
3749 	 * We may have been preempted and rescheduled on a different
3750 	 * cpu before disabling preemption. Need to reload cpu area
3751 	 * pointer.
3752 	 */
3753 	c = slub_get_cpu_ptr(s->cpu_slab);
3754 #endif
3755 
3756 	p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size);
3757 #ifdef CONFIG_PREEMPT_COUNT
3758 	slub_put_cpu_ptr(s->cpu_slab);
3759 #endif
3760 	return p;
3761 }
3762 
3763 static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
3764 		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3765 {
3766 	struct kmem_cache_cpu *c;
3767 	struct slab *slab;
3768 	unsigned long tid;
3769 	void *object;
3770 
3771 redo:
3772 	/*
3773 	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
3774 	 * enabled. We may switch back and forth between cpus while
3775 	 * reading from one cpu area. That does not matter as long
3776 	 * as we end up on the original cpu again when doing the cmpxchg.
3777 	 *
3778 	 * We must guarantee that tid and kmem_cache_cpu are retrieved on the
3779 	 * same cpu. We read first the kmem_cache_cpu pointer and use it to read
3780 	 * the tid. If we are preempted and switched to another cpu between the
3781 	 * two reads, it's OK as the two are still associated with the same cpu
3782 	 * and cmpxchg later will validate the cpu.
3783 	 */
3784 	c = raw_cpu_ptr(s->cpu_slab);
3785 	tid = READ_ONCE(c->tid);
3786 
3787 	/*
3788 	 * Irqless object alloc/free algorithm used here depends on sequence
3789 	 * of fetching cpu_slab's data. tid should be fetched before anything
3790 	 * on c to guarantee that object and slab associated with previous tid
3791 	 * won't be used with current tid. If we fetch tid first, object and
3792 	 * slab could be one associated with next tid and our alloc/free
3793 	 * request will be failed. In this case, we will retry. So, no problem.
3794 	 */
3795 	barrier();
3796 
3797 	/*
3798 	 * The transaction ids are globally unique per cpu and per operation on
3799 	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
3800 	 * occurs on the right processor and that there was no operation on the
3801 	 * linked list in between.
3802 	 */
3803 
3804 	object = c->freelist;
3805 	slab = c->slab;
3806 
3807 	if (!USE_LOCKLESS_FAST_PATH() ||
3808 	    unlikely(!object || !slab || !node_match(slab, node))) {
3809 		object = __slab_alloc(s, gfpflags, node, addr, c, orig_size);
3810 	} else {
3811 		void *next_object = get_freepointer_safe(s, object);
3812 
3813 		/*
3814 		 * The cmpxchg will only match if there was no additional
3815 		 * operation and if we are on the right processor.
3816 		 *
3817 		 * The cmpxchg does the following atomically (without lock
3818 		 * semantics!)
3819 		 * 1. Relocate first pointer to the current per cpu area.
3820 		 * 2. Verify that tid and freelist have not been changed
3821 		 * 3. If they were not changed replace tid and freelist
3822 		 *
3823 		 * Since this is without lock semantics the protection is only
3824 		 * against code executing on this cpu *not* from access by
3825 		 * other cpus.
3826 		 */
3827 		if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {
3828 			note_cmpxchg_failure("slab_alloc", s, tid);
3829 			goto redo;
3830 		}
3831 		prefetch_freepointer(s, next_object);
3832 		stat(s, ALLOC_FASTPATH);
3833 	}
3834 
3835 	return object;
3836 }
3837 #else /* CONFIG_SLUB_TINY */
3838 static void *__slab_alloc_node(struct kmem_cache *s,
3839 		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3840 {
3841 	struct partial_context pc;
3842 	struct slab *slab;
3843 	void *object;
3844 
3845 	pc.flags = gfpflags;
3846 	pc.orig_size = orig_size;
3847 	slab = get_partial(s, node, &pc);
3848 
3849 	if (slab)
3850 		return pc.object;
3851 
3852 	slab = new_slab(s, gfpflags, node);
3853 	if (unlikely(!slab)) {
3854 		slab_out_of_memory(s, gfpflags, node);
3855 		return NULL;
3856 	}
3857 
3858 	object = alloc_single_from_new_slab(s, slab, orig_size);
3859 
3860 	return object;
3861 }
3862 #endif /* CONFIG_SLUB_TINY */
3863 
3864 /*
3865  * If the object has been wiped upon free, make sure it's fully initialized by
3866  * zeroing out freelist pointer.
3867  */
3868 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
3869 						   void *obj)
3870 {
3871 	if (unlikely(slab_want_init_on_free(s)) && obj &&
3872 	    !freeptr_outside_object(s))
3873 		memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
3874 			0, sizeof(void *));
3875 }
3876 
3877 noinline int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
3878 {
3879 	if (__should_failslab(s, gfpflags))
3880 		return -ENOMEM;
3881 	return 0;
3882 }
3883 ALLOW_ERROR_INJECTION(should_failslab, ERRNO);
3884 
3885 static __fastpath_inline
3886 struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
3887 {
3888 	flags &= gfp_allowed_mask;
3889 
3890 	might_alloc(flags);
3891 
3892 	if (unlikely(should_failslab(s, flags)))
3893 		return NULL;
3894 
3895 	return s;
3896 }
3897 
3898 static __fastpath_inline
3899 bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
3900 			  gfp_t flags, size_t size, void **p, bool init,
3901 			  unsigned int orig_size)
3902 {
3903 	unsigned int zero_size = s->object_size;
3904 	struct slabobj_ext *obj_exts;
3905 	bool kasan_init = init;
3906 	size_t i;
3907 	gfp_t init_flags = flags & gfp_allowed_mask;
3908 
3909 	/*
3910 	 * For kmalloc object, the allocated memory size(object_size) is likely
3911 	 * larger than the requested size(orig_size). If redzone check is
3912 	 * enabled for the extra space, don't zero it, as it will be redzoned
3913 	 * soon. The redzone operation for this extra space could be seen as a
3914 	 * replacement of current poisoning under certain debug option, and
3915 	 * won't break other sanity checks.
3916 	 */
3917 	if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) &&
3918 	    (s->flags & SLAB_KMALLOC))
3919 		zero_size = orig_size;
3920 
3921 	/*
3922 	 * When slab_debug is enabled, avoid memory initialization integrated
3923 	 * into KASAN and instead zero out the memory via the memset below with
3924 	 * the proper size. Otherwise, KASAN might overwrite SLUB redzones and
3925 	 * cause false-positive reports. This does not lead to a performance
3926 	 * penalty on production builds, as slab_debug is not intended to be
3927 	 * enabled there.
3928 	 */
3929 	if (__slub_debug_enabled())
3930 		kasan_init = false;
3931 
3932 	/*
3933 	 * As memory initialization might be integrated into KASAN,
3934 	 * kasan_slab_alloc and initialization memset must be
3935 	 * kept together to avoid discrepancies in behavior.
3936 	 *
3937 	 * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
3938 	 */
3939 	for (i = 0; i < size; i++) {
3940 		p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init);
3941 		if (p[i] && init && (!kasan_init ||
3942 				     !kasan_has_integrated_init()))
3943 			memset(p[i], 0, zero_size);
3944 		kmemleak_alloc_recursive(p[i], s->object_size, 1,
3945 					 s->flags, init_flags);
3946 		kmsan_slab_alloc(s, p[i], init_flags);
3947 		if (need_slab_obj_ext()) {
3948 			obj_exts = prepare_slab_obj_exts_hook(s, flags, p[i]);
3949 #ifdef CONFIG_MEM_ALLOC_PROFILING
3950 			/*
3951 			 * Currently obj_exts is used only for allocation profiling.
3952 			 * If other users appear then mem_alloc_profiling_enabled()
3953 			 * check should be added before alloc_tag_add().
3954 			 */
3955 			if (likely(obj_exts))
3956 				alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);
3957 #endif
3958 		}
3959 	}
3960 
3961 	return memcg_slab_post_alloc_hook(s, lru, flags, size, p);
3962 }
3963 
3964 /*
3965  * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
3966  * have the fastpath folded into their functions. So no function call
3967  * overhead for requests that can be satisfied on the fastpath.
3968  *
3969  * The fastpath works by first checking if the lockless freelist can be used.
3970  * If not then __slab_alloc is called for slow processing.
3971  *
3972  * Otherwise we can simply pick the next object from the lockless free list.
3973  */
3974 static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
3975 		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3976 {
3977 	void *object;
3978 	bool init = false;
3979 
3980 	s = slab_pre_alloc_hook(s, gfpflags);
3981 	if (unlikely(!s))
3982 		return NULL;
3983 
3984 	object = kfence_alloc(s, orig_size, gfpflags);
3985 	if (unlikely(object))
3986 		goto out;
3987 
3988 	object = __slab_alloc_node(s, gfpflags, node, addr, orig_size);
3989 
3990 	maybe_wipe_obj_freeptr(s, object);
3991 	init = slab_want_init_on_alloc(gfpflags, s);
3992 
3993 out:
3994 	/*
3995 	 * When init equals 'true', like for kzalloc() family, only
3996 	 * @orig_size bytes might be zeroed instead of s->object_size
3997 	 * In case this fails due to memcg_slab_post_alloc_hook(),
3998 	 * object is set to NULL
3999 	 */
4000 	slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size);
4001 
4002 	return object;
4003 }
4004 
4005 void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)
4006 {
4007 	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,
4008 				    s->object_size);
4009 
4010 	trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
4011 
4012 	return ret;
4013 }
4014 EXPORT_SYMBOL(kmem_cache_alloc_noprof);
4015 
4016 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
4017 			   gfp_t gfpflags)
4018 {
4019 	void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
4020 				    s->object_size);
4021 
4022 	trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
4023 
4024 	return ret;
4025 }
4026 EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof);
4027 
4028 /**
4029  * kmem_cache_alloc_node - Allocate an object on the specified node
4030  * @s: The cache to allocate from.
4031  * @gfpflags: See kmalloc().
4032  * @node: node number of the target node.
4033  *
4034  * Identical to kmem_cache_alloc but it will allocate memory on the given
4035  * node, which can improve the performance for cpu bound structures.
4036  *
4037  * Fallback to other node is possible if __GFP_THISNODE is not set.
4038  *
4039  * Return: pointer to the new object or %NULL in case of error
4040  */
4041 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)
4042 {
4043 	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
4044 
4045 	trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node);
4046 
4047 	return ret;
4048 }
4049 EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
4050 
4051 /*
4052  * To avoid unnecessary overhead, we pass through large allocation requests
4053  * directly to the page allocator. We use __GFP_COMP, because we will need to
4054  * know the allocation order to free the pages properly in kfree.
4055  */
4056 static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
4057 {
4058 	struct folio *folio;
4059 	void *ptr = NULL;
4060 	unsigned int order = get_order(size);
4061 
4062 	if (unlikely(flags & GFP_SLAB_BUG_MASK))
4063 		flags = kmalloc_fix_flags(flags);
4064 
4065 	flags |= __GFP_COMP;
4066 	folio = (struct folio *)alloc_pages_node_noprof(node, flags, order);
4067 	if (folio) {
4068 		ptr = folio_address(folio);
4069 		lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
4070 				      PAGE_SIZE << order);
4071 	}
4072 
4073 	ptr = kasan_kmalloc_large(ptr, size, flags);
4074 	/* As ptr might get tagged, call kmemleak hook after KASAN. */
4075 	kmemleak_alloc(ptr, size, 1, flags);
4076 	kmsan_kmalloc_large(ptr, size, flags);
4077 
4078 	return ptr;
4079 }
4080 
4081 void *kmalloc_large_noprof(size_t size, gfp_t flags)
4082 {
4083 	void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
4084 
4085 	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
4086 		      flags, NUMA_NO_NODE);
4087 	return ret;
4088 }
4089 EXPORT_SYMBOL(kmalloc_large_noprof);
4090 
4091 void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
4092 {
4093 	void *ret = __kmalloc_large_node(size, flags, node);
4094 
4095 	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
4096 		      flags, node);
4097 	return ret;
4098 }
4099 EXPORT_SYMBOL(kmalloc_large_node_noprof);
4100 
4101 static __always_inline
4102 void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
4103 			unsigned long caller)
4104 {
4105 	struct kmem_cache *s;
4106 	void *ret;
4107 
4108 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
4109 		ret = __kmalloc_large_node(size, flags, node);
4110 		trace_kmalloc(caller, ret, size,
4111 			      PAGE_SIZE << get_order(size), flags, node);
4112 		return ret;
4113 	}
4114 
4115 	if (unlikely(!size))
4116 		return ZERO_SIZE_PTR;
4117 
4118 	s = kmalloc_slab(size, flags, caller);
4119 
4120 	ret = slab_alloc_node(s, NULL, flags, node, caller, size);
4121 	ret = kasan_kmalloc(s, ret, size, flags);
4122 	trace_kmalloc(caller, ret, size, s->size, flags, node);
4123 	return ret;
4124 }
4125 
4126 void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node)
4127 {
4128 	return __do_kmalloc_node(size, flags, node, _RET_IP_);
4129 }
4130 EXPORT_SYMBOL(__kmalloc_node_noprof);
4131 
4132 void *__kmalloc_noprof(size_t size, gfp_t flags)
4133 {
4134 	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
4135 }
4136 EXPORT_SYMBOL(__kmalloc_noprof);
4137 
4138 void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags,
4139 				       int node, unsigned long caller)
4140 {
4141 	return __do_kmalloc_node(size, flags, node, caller);
4142 }
4143 EXPORT_SYMBOL(kmalloc_node_track_caller_noprof);
4144 
4145 void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
4146 {
4147 	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
4148 					    _RET_IP_, size);
4149 
4150 	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
4151 
4152 	ret = kasan_kmalloc(s, ret, size, gfpflags);
4153 	return ret;
4154 }
4155 EXPORT_SYMBOL(kmalloc_trace_noprof);
4156 
4157 void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags,
4158 			 int node, size_t size)
4159 {
4160 	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
4161 
4162 	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
4163 
4164 	ret = kasan_kmalloc(s, ret, size, gfpflags);
4165 	return ret;
4166 }
4167 EXPORT_SYMBOL(kmalloc_node_trace_noprof);
4168 
4169 static noinline void free_to_partial_list(
4170 	struct kmem_cache *s, struct slab *slab,
4171 	void *head, void *tail, int bulk_cnt,
4172 	unsigned long addr)
4173 {
4174 	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
4175 	struct slab *slab_free = NULL;
4176 	int cnt = bulk_cnt;
4177 	unsigned long flags;
4178 	depot_stack_handle_t handle = 0;
4179 
4180 	if (s->flags & SLAB_STORE_USER)
4181 		handle = set_track_prepare();
4182 
4183 	spin_lock_irqsave(&n->list_lock, flags);
4184 
4185 	if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) {
4186 		void *prior = slab->freelist;
4187 
4188 		/* Perform the actual freeing while we still hold the locks */
4189 		slab->inuse -= cnt;
4190 		set_freepointer(s, tail, prior);
4191 		slab->freelist = head;
4192 
4193 		/*
4194 		 * If the slab is empty, and node's partial list is full,
4195 		 * it should be discarded anyway no matter it's on full or
4196 		 * partial list.
4197 		 */
4198 		if (slab->inuse == 0 && n->nr_partial >= s->min_partial)
4199 			slab_free = slab;
4200 
4201 		if (!prior) {
4202 			/* was on full list */
4203 			remove_full(s, n, slab);
4204 			if (!slab_free) {
4205 				add_partial(n, slab, DEACTIVATE_TO_TAIL);
4206 				stat(s, FREE_ADD_PARTIAL);
4207 			}
4208 		} else if (slab_free) {
4209 			remove_partial(n, slab);
4210 			stat(s, FREE_REMOVE_PARTIAL);
4211 		}
4212 	}
4213 
4214 	if (slab_free) {
4215 		/*
4216 		 * Update the counters while still holding n->list_lock to
4217 		 * prevent spurious validation warnings
4218 		 */
4219 		dec_slabs_node(s, slab_nid(slab_free), slab_free->objects);
4220 	}
4221 
4222 	spin_unlock_irqrestore(&n->list_lock, flags);
4223 
4224 	if (slab_free) {
4225 		stat(s, FREE_SLAB);
4226 		free_slab(s, slab_free);
4227 	}
4228 }
4229 
4230 /*
4231  * Slow path handling. This may still be called frequently since objects
4232  * have a longer lifetime than the cpu slabs in most processing loads.
4233  *
4234  * So we still attempt to reduce cache line usage. Just take the slab
4235  * lock and free the item. If there is no additional partial slab
4236  * handling required then we can return immediately.
4237  */
4238 static void __slab_free(struct kmem_cache *s, struct slab *slab,
4239 			void *head, void *tail, int cnt,
4240 			unsigned long addr)
4241 
4242 {
4243 	void *prior;
4244 	int was_frozen;
4245 	struct slab new;
4246 	unsigned long counters;
4247 	struct kmem_cache_node *n = NULL;
4248 	unsigned long flags;
4249 	bool on_node_partial;
4250 
4251 	stat(s, FREE_SLOWPATH);
4252 
4253 	if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
4254 		free_to_partial_list(s, slab, head, tail, cnt, addr);
4255 		return;
4256 	}
4257 
4258 	do {
4259 		if (unlikely(n)) {
4260 			spin_unlock_irqrestore(&n->list_lock, flags);
4261 			n = NULL;
4262 		}
4263 		prior = slab->freelist;
4264 		counters = slab->counters;
4265 		set_freepointer(s, tail, prior);
4266 		new.counters = counters;
4267 		was_frozen = new.frozen;
4268 		new.inuse -= cnt;
4269 		if ((!new.inuse || !prior) && !was_frozen) {
4270 			/* Needs to be taken off a list */
4271 			if (!kmem_cache_has_cpu_partial(s) || prior) {
4272 
4273 				n = get_node(s, slab_nid(slab));
4274 				/*
4275 				 * Speculatively acquire the list_lock.
4276 				 * If the cmpxchg does not succeed then we may
4277 				 * drop the list_lock without any processing.
4278 				 *
4279 				 * Otherwise the list_lock will synchronize with
4280 				 * other processors updating the list of slabs.
4281 				 */
4282 				spin_lock_irqsave(&n->list_lock, flags);
4283 
4284 				on_node_partial = slab_test_node_partial(slab);
4285 			}
4286 		}
4287 
4288 	} while (!slab_update_freelist(s, slab,
4289 		prior, counters,
4290 		head, new.counters,
4291 		"__slab_free"));
4292 
4293 	if (likely(!n)) {
4294 
4295 		if (likely(was_frozen)) {
4296 			/*
4297 			 * The list lock was not taken therefore no list
4298 			 * activity can be necessary.
4299 			 */
4300 			stat(s, FREE_FROZEN);
4301 		} else if (kmem_cache_has_cpu_partial(s) && !prior) {
4302 			/*
4303 			 * If we started with a full slab then put it onto the
4304 			 * per cpu partial list.
4305 			 */
4306 			put_cpu_partial(s, slab, 1);
4307 			stat(s, CPU_PARTIAL_FREE);
4308 		}
4309 
4310 		return;
4311 	}
4312 
4313 	/*
4314 	 * This slab was partially empty but not on the per-node partial list,
4315 	 * in which case we shouldn't manipulate its list, just return.
4316 	 */
4317 	if (prior && !on_node_partial) {
4318 		spin_unlock_irqrestore(&n->list_lock, flags);
4319 		return;
4320 	}
4321 
4322 	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
4323 		goto slab_empty;
4324 
4325 	/*
4326 	 * Objects left in the slab. If it was not on the partial list before
4327 	 * then add it.
4328 	 */
4329 	if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
4330 		add_partial(n, slab, DEACTIVATE_TO_TAIL);
4331 		stat(s, FREE_ADD_PARTIAL);
4332 	}
4333 	spin_unlock_irqrestore(&n->list_lock, flags);
4334 	return;
4335 
4336 slab_empty:
4337 	if (prior) {
4338 		/*
4339 		 * Slab on the partial list.
4340 		 */
4341 		remove_partial(n, slab);
4342 		stat(s, FREE_REMOVE_PARTIAL);
4343 	}
4344 
4345 	spin_unlock_irqrestore(&n->list_lock, flags);
4346 	stat(s, FREE_SLAB);
4347 	discard_slab(s, slab);
4348 }
4349 
4350 #ifndef CONFIG_SLUB_TINY
4351 /*
4352  * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
4353  * can perform fastpath freeing without additional function calls.
4354  *
4355  * The fastpath is only possible if we are freeing to the current cpu slab
4356  * of this processor. This typically the case if we have just allocated
4357  * the item before.
4358  *
4359  * If fastpath is not possible then fall back to __slab_free where we deal
4360  * with all sorts of special processing.
4361  *
4362  * Bulk free of a freelist with several objects (all pointing to the
4363  * same slab) possible by specifying head and tail ptr, plus objects
4364  * count (cnt). Bulk free indicated by tail pointer being set.
4365  */
4366 static __always_inline void do_slab_free(struct kmem_cache *s,
4367 				struct slab *slab, void *head, void *tail,
4368 				int cnt, unsigned long addr)
4369 {
4370 	struct kmem_cache_cpu *c;
4371 	unsigned long tid;
4372 	void **freelist;
4373 
4374 redo:
4375 	/*
4376 	 * Determine the currently cpus per cpu slab.
4377 	 * The cpu may change afterward. However that does not matter since
4378 	 * data is retrieved via this pointer. If we are on the same cpu
4379 	 * during the cmpxchg then the free will succeed.
4380 	 */
4381 	c = raw_cpu_ptr(s->cpu_slab);
4382 	tid = READ_ONCE(c->tid);
4383 
4384 	/* Same with comment on barrier() in __slab_alloc_node() */
4385 	barrier();
4386 
4387 	if (unlikely(slab != c->slab)) {
4388 		__slab_free(s, slab, head, tail, cnt, addr);
4389 		return;
4390 	}
4391 
4392 	if (USE_LOCKLESS_FAST_PATH()) {
4393 		freelist = READ_ONCE(c->freelist);
4394 
4395 		set_freepointer(s, tail, freelist);
4396 
4397 		if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
4398 			note_cmpxchg_failure("slab_free", s, tid);
4399 			goto redo;
4400 		}
4401 	} else {
4402 		/* Update the free list under the local lock */
4403 		local_lock(&s->cpu_slab->lock);
4404 		c = this_cpu_ptr(s->cpu_slab);
4405 		if (unlikely(slab != c->slab)) {
4406 			local_unlock(&s->cpu_slab->lock);
4407 			goto redo;
4408 		}
4409 		tid = c->tid;
4410 		freelist = c->freelist;
4411 
4412 		set_freepointer(s, tail, freelist);
4413 		c->freelist = head;
4414 		c->tid = next_tid(tid);
4415 
4416 		local_unlock(&s->cpu_slab->lock);
4417 	}
4418 	stat_add(s, FREE_FASTPATH, cnt);
4419 }
4420 #else /* CONFIG_SLUB_TINY */
4421 static void do_slab_free(struct kmem_cache *s,
4422 				struct slab *slab, void *head, void *tail,
4423 				int cnt, unsigned long addr)
4424 {
4425 	__slab_free(s, slab, head, tail, cnt, addr);
4426 }
4427 #endif /* CONFIG_SLUB_TINY */
4428 
4429 static __fastpath_inline
4430 void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
4431 	       unsigned long addr)
4432 {
4433 	memcg_slab_free_hook(s, slab, &object, 1);
4434 	alloc_tagging_slab_free_hook(s, slab, &object, 1);
4435 
4436 	if (likely(slab_free_hook(s, object, slab_want_init_on_free(s))))
4437 		do_slab_free(s, slab, object, object, 1, addr);
4438 }
4439 
4440 #ifdef CONFIG_MEMCG_KMEM
4441 /* Do not inline the rare memcg charging failed path into the allocation path */
4442 static noinline
4443 void memcg_alloc_abort_single(struct kmem_cache *s, void *object)
4444 {
4445 	if (likely(slab_free_hook(s, object, slab_want_init_on_free(s))))
4446 		do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_);
4447 }
4448 #endif
4449 
4450 static __fastpath_inline
4451 void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
4452 		    void *tail, void **p, int cnt, unsigned long addr)
4453 {
4454 	memcg_slab_free_hook(s, slab, p, cnt);
4455 	alloc_tagging_slab_free_hook(s, slab, p, cnt);
4456 	/*
4457 	 * With KASAN enabled slab_free_freelist_hook modifies the freelist
4458 	 * to remove objects, whose reuse must be delayed.
4459 	 */
4460 	if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt)))
4461 		do_slab_free(s, slab, head, tail, cnt, addr);
4462 }
4463 
4464 #ifdef CONFIG_KASAN_GENERIC
4465 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
4466 {
4467 	do_slab_free(cache, virt_to_slab(x), x, x, 1, addr);
4468 }
4469 #endif
4470 
4471 static inline struct kmem_cache *virt_to_cache(const void *obj)
4472 {
4473 	struct slab *slab;
4474 
4475 	slab = virt_to_slab(obj);
4476 	if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__))
4477 		return NULL;
4478 	return slab->slab_cache;
4479 }
4480 
4481 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
4482 {
4483 	struct kmem_cache *cachep;
4484 
4485 	if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
4486 	    !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
4487 		return s;
4488 
4489 	cachep = virt_to_cache(x);
4490 	if (WARN(cachep && cachep != s,
4491 		 "%s: Wrong slab cache. %s but object is from %s\n",
4492 		 __func__, s->name, cachep->name))
4493 		print_tracking(cachep, x);
4494 	return cachep;
4495 }
4496 
4497 /**
4498  * kmem_cache_free - Deallocate an object
4499  * @s: The cache the allocation was from.
4500  * @x: The previously allocated object.
4501  *
4502  * Free an object which was previously allocated from this
4503  * cache.
4504  */
4505 void kmem_cache_free(struct kmem_cache *s, void *x)
4506 {
4507 	s = cache_from_obj(s, x);
4508 	if (!s)
4509 		return;
4510 	trace_kmem_cache_free(_RET_IP_, x, s);
4511 	slab_free(s, virt_to_slab(x), x, _RET_IP_);
4512 }
4513 EXPORT_SYMBOL(kmem_cache_free);
4514 
4515 static void free_large_kmalloc(struct folio *folio, void *object)
4516 {
4517 	unsigned int order = folio_order(folio);
4518 
4519 	if (WARN_ON_ONCE(order == 0))
4520 		pr_warn_once("object pointer: 0x%p\n", object);
4521 
4522 	kmemleak_free(object);
4523 	kasan_kfree_large(object);
4524 	kmsan_kfree_large(object);
4525 
4526 	lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
4527 			      -(PAGE_SIZE << order));
4528 	folio_put(folio);
4529 }
4530 
4531 /**
4532  * kfree - free previously allocated memory
4533  * @object: pointer returned by kmalloc() or kmem_cache_alloc()
4534  *
4535  * If @object is NULL, no operation is performed.
4536  */
4537 void kfree(const void *object)
4538 {
4539 	struct folio *folio;
4540 	struct slab *slab;
4541 	struct kmem_cache *s;
4542 	void *x = (void *)object;
4543 
4544 	trace_kfree(_RET_IP_, object);
4545 
4546 	if (unlikely(ZERO_OR_NULL_PTR(object)))
4547 		return;
4548 
4549 	folio = virt_to_folio(object);
4550 	if (unlikely(!folio_test_slab(folio))) {
4551 		free_large_kmalloc(folio, (void *)object);
4552 		return;
4553 	}
4554 
4555 	slab = folio_slab(folio);
4556 	s = slab->slab_cache;
4557 	slab_free(s, slab, x, _RET_IP_);
4558 }
4559 EXPORT_SYMBOL(kfree);
4560 
4561 struct detached_freelist {
4562 	struct slab *slab;
4563 	void *tail;
4564 	void *freelist;
4565 	int cnt;
4566 	struct kmem_cache *s;
4567 };
4568 
4569 /*
4570  * This function progressively scans the array with free objects (with
4571  * a limited look ahead) and extract objects belonging to the same
4572  * slab.  It builds a detached freelist directly within the given
4573  * slab/objects.  This can happen without any need for
4574  * synchronization, because the objects are owned by running process.
4575  * The freelist is build up as a single linked list in the objects.
4576  * The idea is, that this detached freelist can then be bulk
4577  * transferred to the real freelist(s), but only requiring a single
4578  * synchronization primitive.  Look ahead in the array is limited due
4579  * to performance reasons.
4580  */
4581 static inline
4582 int build_detached_freelist(struct kmem_cache *s, size_t size,
4583 			    void **p, struct detached_freelist *df)
4584 {
4585 	int lookahead = 3;
4586 	void *object;
4587 	struct folio *folio;
4588 	size_t same;
4589 
4590 	object = p[--size];
4591 	folio = virt_to_folio(object);
4592 	if (!s) {
4593 		/* Handle kalloc'ed objects */
4594 		if (unlikely(!folio_test_slab(folio))) {
4595 			free_large_kmalloc(folio, object);
4596 			df->slab = NULL;
4597 			return size;
4598 		}
4599 		/* Derive kmem_cache from object */
4600 		df->slab = folio_slab(folio);
4601 		df->s = df->slab->slab_cache;
4602 	} else {
4603 		df->slab = folio_slab(folio);
4604 		df->s = cache_from_obj(s, object); /* Support for memcg */
4605 	}
4606 
4607 	/* Start new detached freelist */
4608 	df->tail = object;
4609 	df->freelist = object;
4610 	df->cnt = 1;
4611 
4612 	if (is_kfence_address(object))
4613 		return size;
4614 
4615 	set_freepointer(df->s, object, NULL);
4616 
4617 	same = size;
4618 	while (size) {
4619 		object = p[--size];
4620 		/* df->slab is always set at this point */
4621 		if (df->slab == virt_to_slab(object)) {
4622 			/* Opportunity build freelist */
4623 			set_freepointer(df->s, object, df->freelist);
4624 			df->freelist = object;
4625 			df->cnt++;
4626 			same--;
4627 			if (size != same)
4628 				swap(p[size], p[same]);
4629 			continue;
4630 		}
4631 
4632 		/* Limit look ahead search */
4633 		if (!--lookahead)
4634 			break;
4635 	}
4636 
4637 	return same;
4638 }
4639 
4640 /*
4641  * Internal bulk free of objects that were not initialised by the post alloc
4642  * hooks and thus should not be processed by the free hooks
4643  */
4644 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
4645 {
4646 	if (!size)
4647 		return;
4648 
4649 	do {
4650 		struct detached_freelist df;
4651 
4652 		size = build_detached_freelist(s, size, p, &df);
4653 		if (!df.slab)
4654 			continue;
4655 
4656 		do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
4657 			     _RET_IP_);
4658 	} while (likely(size));
4659 }
4660 
4661 /* Note that interrupts must be enabled when calling this function. */
4662 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
4663 {
4664 	if (!size)
4665 		return;
4666 
4667 	do {
4668 		struct detached_freelist df;
4669 
4670 		size = build_detached_freelist(s, size, p, &df);
4671 		if (!df.slab)
4672 			continue;
4673 
4674 		slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size],
4675 			       df.cnt, _RET_IP_);
4676 	} while (likely(size));
4677 }
4678 EXPORT_SYMBOL(kmem_cache_free_bulk);
4679 
4680 #ifndef CONFIG_SLUB_TINY
4681 static inline
4682 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
4683 			    void **p)
4684 {
4685 	struct kmem_cache_cpu *c;
4686 	unsigned long irqflags;
4687 	int i;
4688 
4689 	/*
4690 	 * Drain objects in the per cpu slab, while disabling local
4691 	 * IRQs, which protects against PREEMPT and interrupts
4692 	 * handlers invoking normal fastpath.
4693 	 */
4694 	c = slub_get_cpu_ptr(s->cpu_slab);
4695 	local_lock_irqsave(&s->cpu_slab->lock, irqflags);
4696 
4697 	for (i = 0; i < size; i++) {
4698 		void *object = kfence_alloc(s, s->object_size, flags);
4699 
4700 		if (unlikely(object)) {
4701 			p[i] = object;
4702 			continue;
4703 		}
4704 
4705 		object = c->freelist;
4706 		if (unlikely(!object)) {
4707 			/*
4708 			 * We may have removed an object from c->freelist using
4709 			 * the fastpath in the previous iteration; in that case,
4710 			 * c->tid has not been bumped yet.
4711 			 * Since ___slab_alloc() may reenable interrupts while
4712 			 * allocating memory, we should bump c->tid now.
4713 			 */
4714 			c->tid = next_tid(c->tid);
4715 
4716 			local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
4717 
4718 			/*
4719 			 * Invoking slow path likely have side-effect
4720 			 * of re-populating per CPU c->freelist
4721 			 */
4722 			p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
4723 					    _RET_IP_, c, s->object_size);
4724 			if (unlikely(!p[i]))
4725 				goto error;
4726 
4727 			c = this_cpu_ptr(s->cpu_slab);
4728 			maybe_wipe_obj_freeptr(s, p[i]);
4729 
4730 			local_lock_irqsave(&s->cpu_slab->lock, irqflags);
4731 
4732 			continue; /* goto for-loop */
4733 		}
4734 		c->freelist = get_freepointer(s, object);
4735 		p[i] = object;
4736 		maybe_wipe_obj_freeptr(s, p[i]);
4737 		stat(s, ALLOC_FASTPATH);
4738 	}
4739 	c->tid = next_tid(c->tid);
4740 	local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
4741 	slub_put_cpu_ptr(s->cpu_slab);
4742 
4743 	return i;
4744 
4745 error:
4746 	slub_put_cpu_ptr(s->cpu_slab);
4747 	__kmem_cache_free_bulk(s, i, p);
4748 	return 0;
4749 
4750 }
4751 #else /* CONFIG_SLUB_TINY */
4752 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
4753 				   size_t size, void **p)
4754 {
4755 	int i;
4756 
4757 	for (i = 0; i < size; i++) {
4758 		void *object = kfence_alloc(s, s->object_size, flags);
4759 
4760 		if (unlikely(object)) {
4761 			p[i] = object;
4762 			continue;
4763 		}
4764 
4765 		p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE,
4766 					 _RET_IP_, s->object_size);
4767 		if (unlikely(!p[i]))
4768 			goto error;
4769 
4770 		maybe_wipe_obj_freeptr(s, p[i]);
4771 	}
4772 
4773 	return i;
4774 
4775 error:
4776 	__kmem_cache_free_bulk(s, i, p);
4777 	return 0;
4778 }
4779 #endif /* CONFIG_SLUB_TINY */
4780 
4781 /* Note that interrupts must be enabled when calling this function. */
4782 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
4783 				 void **p)
4784 {
4785 	int i;
4786 
4787 	if (!size)
4788 		return 0;
4789 
4790 	s = slab_pre_alloc_hook(s, flags);
4791 	if (unlikely(!s))
4792 		return 0;
4793 
4794 	i = __kmem_cache_alloc_bulk(s, flags, size, p);
4795 	if (unlikely(i == 0))
4796 		return 0;
4797 
4798 	/*
4799 	 * memcg and kmem_cache debug support and memory initialization.
4800 	 * Done outside of the IRQ disabled fastpath loop.
4801 	 */
4802 	if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p,
4803 		    slab_want_init_on_alloc(flags, s), s->object_size))) {
4804 		return 0;
4805 	}
4806 	return i;
4807 }
4808 EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);
4809 
4810 
4811 /*
4812  * Object placement in a slab is made very easy because we always start at
4813  * offset 0. If we tune the size of the object to the alignment then we can
4814  * get the required alignment by putting one properly sized object after
4815  * another.
4816  *
4817  * Notice that the allocation order determines the sizes of the per cpu
4818  * caches. Each processor has always one slab available for allocations.
4819  * Increasing the allocation order reduces the number of times that slabs
4820  * must be moved on and off the partial lists and is therefore a factor in
4821  * locking overhead.
4822  */
4823 
4824 /*
4825  * Minimum / Maximum order of slab pages. This influences locking overhead
4826  * and slab fragmentation. A higher order reduces the number of partial slabs
4827  * and increases the number of allocations possible without having to
4828  * take the list_lock.
4829  */
4830 static unsigned int slub_min_order;
4831 static unsigned int slub_max_order =
4832 	IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER;
4833 static unsigned int slub_min_objects;
4834 
4835 /*
4836  * Calculate the order of allocation given an slab object size.
4837  *
4838  * The order of allocation has significant impact on performance and other
4839  * system components. Generally order 0 allocations should be preferred since
4840  * order 0 does not cause fragmentation in the page allocator. Larger objects
4841  * be problematic to put into order 0 slabs because there may be too much
4842  * unused space left. We go to a higher order if more than 1/16th of the slab
4843  * would be wasted.
4844  *
4845  * In order to reach satisfactory performance we must ensure that a minimum
4846  * number of objects is in one slab. Otherwise we may generate too much
4847  * activity on the partial lists which requires taking the list_lock. This is
4848  * less a concern for large slabs though which are rarely used.
4849  *
4850  * slab_max_order specifies the order where we begin to stop considering the
4851  * number of objects in a slab as critical. If we reach slab_max_order then
4852  * we try to keep the page order as low as possible. So we accept more waste
4853  * of space in favor of a small page order.
4854  *
4855  * Higher order allocations also allow the placement of more objects in a
4856  * slab and thereby reduce object handling overhead. If the user has
4857  * requested a higher minimum order then we start with that one instead of
4858  * the smallest order which will fit the object.
4859  */
4860 static inline unsigned int calc_slab_order(unsigned int size,
4861 		unsigned int min_order, unsigned int max_order,
4862 		unsigned int fract_leftover)
4863 {
4864 	unsigned int order;
4865 
4866 	for (order = min_order; order <= max_order; order++) {
4867 
4868 		unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
4869 		unsigned int rem;
4870 
4871 		rem = slab_size % size;
4872 
4873 		if (rem <= slab_size / fract_leftover)
4874 			break;
4875 	}
4876 
4877 	return order;
4878 }
4879 
4880 static inline int calculate_order(unsigned int size)
4881 {
4882 	unsigned int order;
4883 	unsigned int min_objects;
4884 	unsigned int max_objects;
4885 	unsigned int min_order;
4886 
4887 	min_objects = slub_min_objects;
4888 	if (!min_objects) {
4889 		/*
4890 		 * Some architectures will only update present cpus when
4891 		 * onlining them, so don't trust the number if it's just 1. But
4892 		 * we also don't want to use nr_cpu_ids always, as on some other
4893 		 * architectures, there can be many possible cpus, but never
4894 		 * onlined. Here we compromise between trying to avoid too high
4895 		 * order on systems that appear larger than they are, and too
4896 		 * low order on systems that appear smaller than they are.
4897 		 */
4898 		unsigned int nr_cpus = num_present_cpus();
4899 		if (nr_cpus <= 1)
4900 			nr_cpus = nr_cpu_ids;
4901 		min_objects = 4 * (fls(nr_cpus) + 1);
4902 	}
4903 	/* min_objects can't be 0 because get_order(0) is undefined */
4904 	max_objects = max(order_objects(slub_max_order, size), 1U);
4905 	min_objects = min(min_objects, max_objects);
4906 
4907 	min_order = max_t(unsigned int, slub_min_order,
4908 			  get_order(min_objects * size));
4909 	if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
4910 		return get_order(size * MAX_OBJS_PER_PAGE) - 1;
4911 
4912 	/*
4913 	 * Attempt to find best configuration for a slab. This works by first
4914 	 * attempting to generate a layout with the best possible configuration
4915 	 * and backing off gradually.
4916 	 *
4917 	 * We start with accepting at most 1/16 waste and try to find the
4918 	 * smallest order from min_objects-derived/slab_min_order up to
4919 	 * slab_max_order that will satisfy the constraint. Note that increasing
4920 	 * the order can only result in same or less fractional waste, not more.
4921 	 *
4922 	 * If that fails, we increase the acceptable fraction of waste and try
4923 	 * again. The last iteration with fraction of 1/2 would effectively
4924 	 * accept any waste and give us the order determined by min_objects, as
4925 	 * long as at least single object fits within slab_max_order.
4926 	 */
4927 	for (unsigned int fraction = 16; fraction > 1; fraction /= 2) {
4928 		order = calc_slab_order(size, min_order, slub_max_order,
4929 					fraction);
4930 		if (order <= slub_max_order)
4931 			return order;
4932 	}
4933 
4934 	/*
4935 	 * Doh this slab cannot be placed using slab_max_order.
4936 	 */
4937 	order = get_order(size);
4938 	if (order <= MAX_PAGE_ORDER)
4939 		return order;
4940 	return -ENOSYS;
4941 }
4942 
4943 static void
4944 init_kmem_cache_node(struct kmem_cache_node *n)
4945 {
4946 	n->nr_partial = 0;
4947 	spin_lock_init(&n->list_lock);
4948 	INIT_LIST_HEAD(&n->partial);
4949 #ifdef CONFIG_SLUB_DEBUG
4950 	atomic_long_set(&n->nr_slabs, 0);
4951 	atomic_long_set(&n->total_objects, 0);
4952 	INIT_LIST_HEAD(&n->full);
4953 #endif
4954 }
4955 
4956 #ifndef CONFIG_SLUB_TINY
4957 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4958 {
4959 	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
4960 			NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH *
4961 			sizeof(struct kmem_cache_cpu));
4962 
4963 	/*
4964 	 * Must align to double word boundary for the double cmpxchg
4965 	 * instructions to work; see __pcpu_double_call_return_bool().
4966 	 */
4967 	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
4968 				     2 * sizeof(void *));
4969 
4970 	if (!s->cpu_slab)
4971 		return 0;
4972 
4973 	init_kmem_cache_cpus(s);
4974 
4975 	return 1;
4976 }
4977 #else
4978 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4979 {
4980 	return 1;
4981 }
4982 #endif /* CONFIG_SLUB_TINY */
4983 
4984 static struct kmem_cache *kmem_cache_node;
4985 
4986 /*
4987  * No kmalloc_node yet so do it by hand. We know that this is the first
4988  * slab on the node for this slabcache. There are no concurrent accesses
4989  * possible.
4990  *
4991  * Note that this function only works on the kmem_cache_node
4992  * when allocating for the kmem_cache_node. This is used for bootstrapping
4993  * memory on a fresh node that has no slab structures yet.
4994  */
4995 static void early_kmem_cache_node_alloc(int node)
4996 {
4997 	struct slab *slab;
4998 	struct kmem_cache_node *n;
4999 
5000 	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
5001 
5002 	slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
5003 
5004 	BUG_ON(!slab);
5005 	if (slab_nid(slab) != node) {
5006 		pr_err("SLUB: Unable to allocate memory from node %d\n", node);
5007 		pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
5008 	}
5009 
5010 	n = slab->freelist;
5011 	BUG_ON(!n);
5012 #ifdef CONFIG_SLUB_DEBUG
5013 	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
5014 #endif
5015 	n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
5016 	slab->freelist = get_freepointer(kmem_cache_node, n);
5017 	slab->inuse = 1;
5018 	kmem_cache_node->node[node] = n;
5019 	init_kmem_cache_node(n);
5020 	inc_slabs_node(kmem_cache_node, node, slab->objects);
5021 
5022 	/*
5023 	 * No locks need to be taken here as it has just been
5024 	 * initialized and there is no concurrent access.
5025 	 */
5026 	__add_partial(n, slab, DEACTIVATE_TO_HEAD);
5027 }
5028 
5029 static void free_kmem_cache_nodes(struct kmem_cache *s)
5030 {
5031 	int node;
5032 	struct kmem_cache_node *n;
5033 
5034 	for_each_kmem_cache_node(s, node, n) {
5035 		s->node[node] = NULL;
5036 		kmem_cache_free(kmem_cache_node, n);
5037 	}
5038 }
5039 
5040 void __kmem_cache_release(struct kmem_cache *s)
5041 {
5042 	cache_random_seq_destroy(s);
5043 #ifndef CONFIG_SLUB_TINY
5044 	free_percpu(s->cpu_slab);
5045 #endif
5046 	free_kmem_cache_nodes(s);
5047 }
5048 
5049 static int init_kmem_cache_nodes(struct kmem_cache *s)
5050 {
5051 	int node;
5052 
5053 	for_each_node_mask(node, slab_nodes) {
5054 		struct kmem_cache_node *n;
5055 
5056 		if (slab_state == DOWN) {
5057 			early_kmem_cache_node_alloc(node);
5058 			continue;
5059 		}
5060 		n = kmem_cache_alloc_node(kmem_cache_node,
5061 						GFP_KERNEL, node);
5062 
5063 		if (!n) {
5064 			free_kmem_cache_nodes(s);
5065 			return 0;
5066 		}
5067 
5068 		init_kmem_cache_node(n);
5069 		s->node[node] = n;
5070 	}
5071 	return 1;
5072 }
5073 
5074 static void set_cpu_partial(struct kmem_cache *s)
5075 {
5076 #ifdef CONFIG_SLUB_CPU_PARTIAL
5077 	unsigned int nr_objects;
5078 
5079 	/*
5080 	 * cpu_partial determined the maximum number of objects kept in the
5081 	 * per cpu partial lists of a processor.
5082 	 *
5083 	 * Per cpu partial lists mainly contain slabs that just have one
5084 	 * object freed. If they are used for allocation then they can be
5085 	 * filled up again with minimal effort. The slab will never hit the
5086 	 * per node partial lists and therefore no locking will be required.
5087 	 *
5088 	 * For backwards compatibility reasons, this is determined as number
5089 	 * of objects, even though we now limit maximum number of pages, see
5090 	 * slub_set_cpu_partial()
5091 	 */
5092 	if (!kmem_cache_has_cpu_partial(s))
5093 		nr_objects = 0;
5094 	else if (s->size >= PAGE_SIZE)
5095 		nr_objects = 6;
5096 	else if (s->size >= 1024)
5097 		nr_objects = 24;
5098 	else if (s->size >= 256)
5099 		nr_objects = 52;
5100 	else
5101 		nr_objects = 120;
5102 
5103 	slub_set_cpu_partial(s, nr_objects);
5104 #endif
5105 }
5106 
5107 /*
5108  * calculate_sizes() determines the order and the distribution of data within
5109  * a slab object.
5110  */
5111 static int calculate_sizes(struct kmem_cache *s)
5112 {
5113 	slab_flags_t flags = s->flags;
5114 	unsigned int size = s->object_size;
5115 	unsigned int order;
5116 
5117 	/*
5118 	 * Round up object size to the next word boundary. We can only
5119 	 * place the free pointer at word boundaries and this determines
5120 	 * the possible location of the free pointer.
5121 	 */
5122 	size = ALIGN(size, sizeof(void *));
5123 
5124 #ifdef CONFIG_SLUB_DEBUG
5125 	/*
5126 	 * Determine if we can poison the object itself. If the user of
5127 	 * the slab may touch the object after free or before allocation
5128 	 * then we should never poison the object itself.
5129 	 */
5130 	if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
5131 			!s->ctor)
5132 		s->flags |= __OBJECT_POISON;
5133 	else
5134 		s->flags &= ~__OBJECT_POISON;
5135 
5136 
5137 	/*
5138 	 * If we are Redzoning then check if there is some space between the
5139 	 * end of the object and the free pointer. If not then add an
5140 	 * additional word to have some bytes to store Redzone information.
5141 	 */
5142 	if ((flags & SLAB_RED_ZONE) && size == s->object_size)
5143 		size += sizeof(void *);
5144 #endif
5145 
5146 	/*
5147 	 * With that we have determined the number of bytes in actual use
5148 	 * by the object and redzoning.
5149 	 */
5150 	s->inuse = size;
5151 
5152 	if (slub_debug_orig_size(s) ||
5153 	    (flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
5154 	    ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
5155 	    s->ctor) {
5156 		/*
5157 		 * Relocate free pointer after the object if it is not
5158 		 * permitted to overwrite the first word of the object on
5159 		 * kmem_cache_free.
5160 		 *
5161 		 * This is the case if we do RCU, have a constructor or
5162 		 * destructor, are poisoning the objects, or are
5163 		 * redzoning an object smaller than sizeof(void *).
5164 		 *
5165 		 * The assumption that s->offset >= s->inuse means free
5166 		 * pointer is outside of the object is used in the
5167 		 * freeptr_outside_object() function. If that is no
5168 		 * longer true, the function needs to be modified.
5169 		 */
5170 		s->offset = size;
5171 		size += sizeof(void *);
5172 	} else {
5173 		/*
5174 		 * Store freelist pointer near middle of object to keep
5175 		 * it away from the edges of the object to avoid small
5176 		 * sized over/underflows from neighboring allocations.
5177 		 */
5178 		s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
5179 	}
5180 
5181 #ifdef CONFIG_SLUB_DEBUG
5182 	if (flags & SLAB_STORE_USER) {
5183 		/*
5184 		 * Need to store information about allocs and frees after
5185 		 * the object.
5186 		 */
5187 		size += 2 * sizeof(struct track);
5188 
5189 		/* Save the original kmalloc request size */
5190 		if (flags & SLAB_KMALLOC)
5191 			size += sizeof(unsigned int);
5192 	}
5193 #endif
5194 
5195 	kasan_cache_create(s, &size, &s->flags);
5196 #ifdef CONFIG_SLUB_DEBUG
5197 	if (flags & SLAB_RED_ZONE) {
5198 		/*
5199 		 * Add some empty padding so that we can catch
5200 		 * overwrites from earlier objects rather than let
5201 		 * tracking information or the free pointer be
5202 		 * corrupted if a user writes before the start
5203 		 * of the object.
5204 		 */
5205 		size += sizeof(void *);
5206 
5207 		s->red_left_pad = sizeof(void *);
5208 		s->red_left_pad = ALIGN(s->red_left_pad, s->align);
5209 		size += s->red_left_pad;
5210 	}
5211 #endif
5212 
5213 	/*
5214 	 * SLUB stores one object immediately after another beginning from
5215 	 * offset 0. In order to align the objects we have to simply size
5216 	 * each object to conform to the alignment.
5217 	 */
5218 	size = ALIGN(size, s->align);
5219 	s->size = size;
5220 	s->reciprocal_size = reciprocal_value(size);
5221 	order = calculate_order(size);
5222 
5223 	if ((int)order < 0)
5224 		return 0;
5225 
5226 	s->allocflags = __GFP_COMP;
5227 
5228 	if (s->flags & SLAB_CACHE_DMA)
5229 		s->allocflags |= GFP_DMA;
5230 
5231 	if (s->flags & SLAB_CACHE_DMA32)
5232 		s->allocflags |= GFP_DMA32;
5233 
5234 	if (s->flags & SLAB_RECLAIM_ACCOUNT)
5235 		s->allocflags |= __GFP_RECLAIMABLE;
5236 
5237 	/*
5238 	 * Determine the number of objects per slab
5239 	 */
5240 	s->oo = oo_make(order, size);
5241 	s->min = oo_make(get_order(size), size);
5242 
5243 	return !!oo_objects(s->oo);
5244 }
5245 
5246 static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
5247 {
5248 	s->flags = kmem_cache_flags(flags, s->name);
5249 #ifdef CONFIG_SLAB_FREELIST_HARDENED
5250 	s->random = get_random_long();
5251 #endif
5252 
5253 	if (!calculate_sizes(s))
5254 		goto error;
5255 	if (disable_higher_order_debug) {
5256 		/*
5257 		 * Disable debugging flags that store metadata if the min slab
5258 		 * order increased.
5259 		 */
5260 		if (get_order(s->size) > get_order(s->object_size)) {
5261 			s->flags &= ~DEBUG_METADATA_FLAGS;
5262 			s->offset = 0;
5263 			if (!calculate_sizes(s))
5264 				goto error;
5265 		}
5266 	}
5267 
5268 #ifdef system_has_freelist_aba
5269 	if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
5270 		/* Enable fast mode */
5271 		s->flags |= __CMPXCHG_DOUBLE;
5272 	}
5273 #endif
5274 
5275 	/*
5276 	 * The larger the object size is, the more slabs we want on the partial
5277 	 * list to avoid pounding the page allocator excessively.
5278 	 */
5279 	s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
5280 	s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
5281 
5282 	set_cpu_partial(s);
5283 
5284 #ifdef CONFIG_NUMA
5285 	s->remote_node_defrag_ratio = 1000;
5286 #endif
5287 
5288 	/* Initialize the pre-computed randomized freelist if slab is up */
5289 	if (slab_state >= UP) {
5290 		if (init_cache_random_seq(s))
5291 			goto error;
5292 	}
5293 
5294 	if (!init_kmem_cache_nodes(s))
5295 		goto error;
5296 
5297 	if (alloc_kmem_cache_cpus(s))
5298 		return 0;
5299 
5300 error:
5301 	__kmem_cache_release(s);
5302 	return -EINVAL;
5303 }
5304 
5305 static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
5306 			      const char *text)
5307 {
5308 #ifdef CONFIG_SLUB_DEBUG
5309 	void *addr = slab_address(slab);
5310 	void *p;
5311 
5312 	slab_err(s, slab, text, s->name);
5313 
5314 	spin_lock(&object_map_lock);
5315 	__fill_map(object_map, s, slab);
5316 
5317 	for_each_object(p, s, addr, slab->objects) {
5318 
5319 		if (!test_bit(__obj_to_index(s, addr, p), object_map)) {
5320 			pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
5321 			print_tracking(s, p);
5322 		}
5323 	}
5324 	spin_unlock(&object_map_lock);
5325 #endif
5326 }
5327 
5328 /*
5329  * Attempt to free all partial slabs on a node.
5330  * This is called from __kmem_cache_shutdown(). We must take list_lock
5331  * because sysfs file might still access partial list after the shutdowning.
5332  */
5333 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
5334 {
5335 	LIST_HEAD(discard);
5336 	struct slab *slab, *h;
5337 
5338 	BUG_ON(irqs_disabled());
5339 	spin_lock_irq(&n->list_lock);
5340 	list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
5341 		if (!slab->inuse) {
5342 			remove_partial(n, slab);
5343 			list_add(&slab->slab_list, &discard);
5344 		} else {
5345 			list_slab_objects(s, slab,
5346 			  "Objects remaining in %s on __kmem_cache_shutdown()");
5347 		}
5348 	}
5349 	spin_unlock_irq(&n->list_lock);
5350 
5351 	list_for_each_entry_safe(slab, h, &discard, slab_list)
5352 		discard_slab(s, slab);
5353 }
5354 
5355 bool __kmem_cache_empty(struct kmem_cache *s)
5356 {
5357 	int node;
5358 	struct kmem_cache_node *n;
5359 
5360 	for_each_kmem_cache_node(s, node, n)
5361 		if (n->nr_partial || node_nr_slabs(n))
5362 			return false;
5363 	return true;
5364 }
5365 
5366 /*
5367  * Release all resources used by a slab cache.
5368  */
5369 int __kmem_cache_shutdown(struct kmem_cache *s)
5370 {
5371 	int node;
5372 	struct kmem_cache_node *n;
5373 
5374 	flush_all_cpus_locked(s);
5375 	/* Attempt to free all objects */
5376 	for_each_kmem_cache_node(s, node, n) {
5377 		free_partial(s, n);
5378 		if (n->nr_partial || node_nr_slabs(n))
5379 			return 1;
5380 	}
5381 	return 0;
5382 }
5383 
5384 #ifdef CONFIG_PRINTK
5385 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
5386 {
5387 	void *base;
5388 	int __maybe_unused i;
5389 	unsigned int objnr;
5390 	void *objp;
5391 	void *objp0;
5392 	struct kmem_cache *s = slab->slab_cache;
5393 	struct track __maybe_unused *trackp;
5394 
5395 	kpp->kp_ptr = object;
5396 	kpp->kp_slab = slab;
5397 	kpp->kp_slab_cache = s;
5398 	base = slab_address(slab);
5399 	objp0 = kasan_reset_tag(object);
5400 #ifdef CONFIG_SLUB_DEBUG
5401 	objp = restore_red_left(s, objp0);
5402 #else
5403 	objp = objp0;
5404 #endif
5405 	objnr = obj_to_index(s, slab, objp);
5406 	kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
5407 	objp = base + s->size * objnr;
5408 	kpp->kp_objp = objp;
5409 	if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
5410 			 || (objp - base) % s->size) ||
5411 	    !(s->flags & SLAB_STORE_USER))
5412 		return;
5413 #ifdef CONFIG_SLUB_DEBUG
5414 	objp = fixup_red_left(s, objp);
5415 	trackp = get_track(s, objp, TRACK_ALLOC);
5416 	kpp->kp_ret = (void *)trackp->addr;
5417 #ifdef CONFIG_STACKDEPOT
5418 	{
5419 		depot_stack_handle_t handle;
5420 		unsigned long *entries;
5421 		unsigned int nr_entries;
5422 
5423 		handle = READ_ONCE(trackp->handle);
5424 		if (handle) {
5425 			nr_entries = stack_depot_fetch(handle, &entries);
5426 			for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
5427 				kpp->kp_stack[i] = (void *)entries[i];
5428 		}
5429 
5430 		trackp = get_track(s, objp, TRACK_FREE);
5431 		handle = READ_ONCE(trackp->handle);
5432 		if (handle) {
5433 			nr_entries = stack_depot_fetch(handle, &entries);
5434 			for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
5435 				kpp->kp_free_stack[i] = (void *)entries[i];
5436 		}
5437 	}
5438 #endif
5439 #endif
5440 }
5441 #endif
5442 
5443 /********************************************************************
5444  *		Kmalloc subsystem
5445  *******************************************************************/
5446 
5447 static int __init setup_slub_min_order(char *str)
5448 {
5449 	get_option(&str, (int *)&slub_min_order);
5450 
5451 	if (slub_min_order > slub_max_order)
5452 		slub_max_order = slub_min_order;
5453 
5454 	return 1;
5455 }
5456 
5457 __setup("slab_min_order=", setup_slub_min_order);
5458 __setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0);
5459 
5460 
5461 static int __init setup_slub_max_order(char *str)
5462 {
5463 	get_option(&str, (int *)&slub_max_order);
5464 	slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER);
5465 
5466 	if (slub_min_order > slub_max_order)
5467 		slub_min_order = slub_max_order;
5468 
5469 	return 1;
5470 }
5471 
5472 __setup("slab_max_order=", setup_slub_max_order);
5473 __setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0);
5474 
5475 static int __init setup_slub_min_objects(char *str)
5476 {
5477 	get_option(&str, (int *)&slub_min_objects);
5478 
5479 	return 1;
5480 }
5481 
5482 __setup("slab_min_objects=", setup_slub_min_objects);
5483 __setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0);
5484 
5485 #ifdef CONFIG_HARDENED_USERCOPY
5486 /*
5487  * Rejects incorrectly sized objects and objects that are to be copied
5488  * to/from userspace but do not fall entirely within the containing slab
5489  * cache's usercopy region.
5490  *
5491  * Returns NULL if check passes, otherwise const char * to name of cache
5492  * to indicate an error.
5493  */
5494 void __check_heap_object(const void *ptr, unsigned long n,
5495 			 const struct slab *slab, bool to_user)
5496 {
5497 	struct kmem_cache *s;
5498 	unsigned int offset;
5499 	bool is_kfence = is_kfence_address(ptr);
5500 
5501 	ptr = kasan_reset_tag(ptr);
5502 
5503 	/* Find object and usable object size. */
5504 	s = slab->slab_cache;
5505 
5506 	/* Reject impossible pointers. */
5507 	if (ptr < slab_address(slab))
5508 		usercopy_abort("SLUB object not in SLUB page?!", NULL,
5509 			       to_user, 0, n);
5510 
5511 	/* Find offset within object. */
5512 	if (is_kfence)
5513 		offset = ptr - kfence_object_start(ptr);
5514 	else
5515 		offset = (ptr - slab_address(slab)) % s->size;
5516 
5517 	/* Adjust for redzone and reject if within the redzone. */
5518 	if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
5519 		if (offset < s->red_left_pad)
5520 			usercopy_abort("SLUB object in left red zone",
5521 				       s->name, to_user, offset, n);
5522 		offset -= s->red_left_pad;
5523 	}
5524 
5525 	/* Allow address range falling entirely within usercopy region. */
5526 	if (offset >= s->useroffset &&
5527 	    offset - s->useroffset <= s->usersize &&
5528 	    n <= s->useroffset - offset + s->usersize)
5529 		return;
5530 
5531 	usercopy_abort("SLUB object", s->name, to_user, offset, n);
5532 }
5533 #endif /* CONFIG_HARDENED_USERCOPY */
5534 
5535 #define SHRINK_PROMOTE_MAX 32
5536 
5537 /*
5538  * kmem_cache_shrink discards empty slabs and promotes the slabs filled
5539  * up most to the head of the partial lists. New allocations will then
5540  * fill those up and thus they can be removed from the partial lists.
5541  *
5542  * The slabs with the least items are placed last. This results in them
5543  * being allocated from last increasing the chance that the last objects
5544  * are freed in them.
5545  */
5546 static int __kmem_cache_do_shrink(struct kmem_cache *s)
5547 {
5548 	int node;
5549 	int i;
5550 	struct kmem_cache_node *n;
5551 	struct slab *slab;
5552 	struct slab *t;
5553 	struct list_head discard;
5554 	struct list_head promote[SHRINK_PROMOTE_MAX];
5555 	unsigned long flags;
5556 	int ret = 0;
5557 
5558 	for_each_kmem_cache_node(s, node, n) {
5559 		INIT_LIST_HEAD(&discard);
5560 		for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
5561 			INIT_LIST_HEAD(promote + i);
5562 
5563 		spin_lock_irqsave(&n->list_lock, flags);
5564 
5565 		/*
5566 		 * Build lists of slabs to discard or promote.
5567 		 *
5568 		 * Note that concurrent frees may occur while we hold the
5569 		 * list_lock. slab->inuse here is the upper limit.
5570 		 */
5571 		list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
5572 			int free = slab->objects - slab->inuse;
5573 
5574 			/* Do not reread slab->inuse */
5575 			barrier();
5576 
5577 			/* We do not keep full slabs on the list */
5578 			BUG_ON(free <= 0);
5579 
5580 			if (free == slab->objects) {
5581 				list_move(&slab->slab_list, &discard);
5582 				slab_clear_node_partial(slab);
5583 				n->nr_partial--;
5584 				dec_slabs_node(s, node, slab->objects);
5585 			} else if (free <= SHRINK_PROMOTE_MAX)
5586 				list_move(&slab->slab_list, promote + free - 1);
5587 		}
5588 
5589 		/*
5590 		 * Promote the slabs filled up most to the head of the
5591 		 * partial list.
5592 		 */
5593 		for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
5594 			list_splice(promote + i, &n->partial);
5595 
5596 		spin_unlock_irqrestore(&n->list_lock, flags);
5597 
5598 		/* Release empty slabs */
5599 		list_for_each_entry_safe(slab, t, &discard, slab_list)
5600 			free_slab(s, slab);
5601 
5602 		if (node_nr_slabs(n))
5603 			ret = 1;
5604 	}
5605 
5606 	return ret;
5607 }
5608 
5609 int __kmem_cache_shrink(struct kmem_cache *s)
5610 {
5611 	flush_all(s);
5612 	return __kmem_cache_do_shrink(s);
5613 }
5614 
5615 static int slab_mem_going_offline_callback(void *arg)
5616 {
5617 	struct kmem_cache *s;
5618 
5619 	mutex_lock(&slab_mutex);
5620 	list_for_each_entry(s, &slab_caches, list) {
5621 		flush_all_cpus_locked(s);
5622 		__kmem_cache_do_shrink(s);
5623 	}
5624 	mutex_unlock(&slab_mutex);
5625 
5626 	return 0;
5627 }
5628 
5629 static void slab_mem_offline_callback(void *arg)
5630 {
5631 	struct memory_notify *marg = arg;
5632 	int offline_node;
5633 
5634 	offline_node = marg->status_change_nid_normal;
5635 
5636 	/*
5637 	 * If the node still has available memory. we need kmem_cache_node
5638 	 * for it yet.
5639 	 */
5640 	if (offline_node < 0)
5641 		return;
5642 
5643 	mutex_lock(&slab_mutex);
5644 	node_clear(offline_node, slab_nodes);
5645 	/*
5646 	 * We no longer free kmem_cache_node structures here, as it would be
5647 	 * racy with all get_node() users, and infeasible to protect them with
5648 	 * slab_mutex.
5649 	 */
5650 	mutex_unlock(&slab_mutex);
5651 }
5652 
5653 static int slab_mem_going_online_callback(void *arg)
5654 {
5655 	struct kmem_cache_node *n;
5656 	struct kmem_cache *s;
5657 	struct memory_notify *marg = arg;
5658 	int nid = marg->status_change_nid_normal;
5659 	int ret = 0;
5660 
5661 	/*
5662 	 * If the node's memory is already available, then kmem_cache_node is
5663 	 * already created. Nothing to do.
5664 	 */
5665 	if (nid < 0)
5666 		return 0;
5667 
5668 	/*
5669 	 * We are bringing a node online. No memory is available yet. We must
5670 	 * allocate a kmem_cache_node structure in order to bring the node
5671 	 * online.
5672 	 */
5673 	mutex_lock(&slab_mutex);
5674 	list_for_each_entry(s, &slab_caches, list) {
5675 		/*
5676 		 * The structure may already exist if the node was previously
5677 		 * onlined and offlined.
5678 		 */
5679 		if (get_node(s, nid))
5680 			continue;
5681 		/*
5682 		 * XXX: kmem_cache_alloc_node will fallback to other nodes
5683 		 *      since memory is not yet available from the node that
5684 		 *      is brought up.
5685 		 */
5686 		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
5687 		if (!n) {
5688 			ret = -ENOMEM;
5689 			goto out;
5690 		}
5691 		init_kmem_cache_node(n);
5692 		s->node[nid] = n;
5693 	}
5694 	/*
5695 	 * Any cache created after this point will also have kmem_cache_node
5696 	 * initialized for the new node.
5697 	 */
5698 	node_set(nid, slab_nodes);
5699 out:
5700 	mutex_unlock(&slab_mutex);
5701 	return ret;
5702 }
5703 
5704 static int slab_memory_callback(struct notifier_block *self,
5705 				unsigned long action, void *arg)
5706 {
5707 	int ret = 0;
5708 
5709 	switch (action) {
5710 	case MEM_GOING_ONLINE:
5711 		ret = slab_mem_going_online_callback(arg);
5712 		break;
5713 	case MEM_GOING_OFFLINE:
5714 		ret = slab_mem_going_offline_callback(arg);
5715 		break;
5716 	case MEM_OFFLINE:
5717 	case MEM_CANCEL_ONLINE:
5718 		slab_mem_offline_callback(arg);
5719 		break;
5720 	case MEM_ONLINE:
5721 	case MEM_CANCEL_OFFLINE:
5722 		break;
5723 	}
5724 	if (ret)
5725 		ret = notifier_from_errno(ret);
5726 	else
5727 		ret = NOTIFY_OK;
5728 	return ret;
5729 }
5730 
5731 /********************************************************************
5732  *			Basic setup of slabs
5733  *******************************************************************/
5734 
5735 /*
5736  * Used for early kmem_cache structures that were allocated using
5737  * the page allocator. Allocate them properly then fix up the pointers
5738  * that may be pointing to the wrong kmem_cache structure.
5739  */
5740 
5741 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
5742 {
5743 	int node;
5744 	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
5745 	struct kmem_cache_node *n;
5746 
5747 	memcpy(s, static_cache, kmem_cache->object_size);
5748 
5749 	/*
5750 	 * This runs very early, and only the boot processor is supposed to be
5751 	 * up.  Even if it weren't true, IRQs are not up so we couldn't fire
5752 	 * IPIs around.
5753 	 */
5754 	__flush_cpu_slab(s, smp_processor_id());
5755 	for_each_kmem_cache_node(s, node, n) {
5756 		struct slab *p;
5757 
5758 		list_for_each_entry(p, &n->partial, slab_list)
5759 			p->slab_cache = s;
5760 
5761 #ifdef CONFIG_SLUB_DEBUG
5762 		list_for_each_entry(p, &n->full, slab_list)
5763 			p->slab_cache = s;
5764 #endif
5765 	}
5766 	list_add(&s->list, &slab_caches);
5767 	return s;
5768 }
5769 
5770 void __init kmem_cache_init(void)
5771 {
5772 	static __initdata struct kmem_cache boot_kmem_cache,
5773 		boot_kmem_cache_node;
5774 	int node;
5775 
5776 	if (debug_guardpage_minorder())
5777 		slub_max_order = 0;
5778 
5779 	/* Print slub debugging pointers without hashing */
5780 	if (__slub_debug_enabled())
5781 		no_hash_pointers_enable(NULL);
5782 
5783 	kmem_cache_node = &boot_kmem_cache_node;
5784 	kmem_cache = &boot_kmem_cache;
5785 
5786 	/*
5787 	 * Initialize the nodemask for which we will allocate per node
5788 	 * structures. Here we don't need taking slab_mutex yet.
5789 	 */
5790 	for_each_node_state(node, N_NORMAL_MEMORY)
5791 		node_set(node, slab_nodes);
5792 
5793 	create_boot_cache(kmem_cache_node, "kmem_cache_node",
5794 			sizeof(struct kmem_cache_node),
5795 			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
5796 
5797 	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
5798 
5799 	/* Able to allocate the per node structures */
5800 	slab_state = PARTIAL;
5801 
5802 	create_boot_cache(kmem_cache, "kmem_cache",
5803 			offsetof(struct kmem_cache, node) +
5804 				nr_node_ids * sizeof(struct kmem_cache_node *),
5805 			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
5806 
5807 	kmem_cache = bootstrap(&boot_kmem_cache);
5808 	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
5809 
5810 	/* Now we can use the kmem_cache to allocate kmalloc slabs */
5811 	setup_kmalloc_cache_index_table();
5812 	create_kmalloc_caches();
5813 
5814 	/* Setup random freelists for each cache */
5815 	init_freelist_randomization();
5816 
5817 	cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
5818 				  slub_cpu_dead);
5819 
5820 	pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
5821 		cache_line_size(),
5822 		slub_min_order, slub_max_order, slub_min_objects,
5823 		nr_cpu_ids, nr_node_ids);
5824 }
5825 
5826 void __init kmem_cache_init_late(void)
5827 {
5828 #ifndef CONFIG_SLUB_TINY
5829 	flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0);
5830 	WARN_ON(!flushwq);
5831 #endif
5832 }
5833 
5834 struct kmem_cache *
5835 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
5836 		   slab_flags_t flags, void (*ctor)(void *))
5837 {
5838 	struct kmem_cache *s;
5839 
5840 	s = find_mergeable(size, align, flags, name, ctor);
5841 	if (s) {
5842 		if (sysfs_slab_alias(s, name))
5843 			return NULL;
5844 
5845 		s->refcount++;
5846 
5847 		/*
5848 		 * Adjust the object sizes so that we clear
5849 		 * the complete object on kzalloc.
5850 		 */
5851 		s->object_size = max(s->object_size, size);
5852 		s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
5853 	}
5854 
5855 	return s;
5856 }
5857 
5858 int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
5859 {
5860 	int err;
5861 
5862 	err = kmem_cache_open(s, flags);
5863 	if (err)
5864 		return err;
5865 
5866 	/* Mutex is not taken during early boot */
5867 	if (slab_state <= UP)
5868 		return 0;
5869 
5870 	err = sysfs_slab_add(s);
5871 	if (err) {
5872 		__kmem_cache_release(s);
5873 		return err;
5874 	}
5875 
5876 	if (s->flags & SLAB_STORE_USER)
5877 		debugfs_slab_add(s);
5878 
5879 	return 0;
5880 }
5881 
5882 #ifdef SLAB_SUPPORTS_SYSFS
5883 static int count_inuse(struct slab *slab)
5884 {
5885 	return slab->inuse;
5886 }
5887 
5888 static int count_total(struct slab *slab)
5889 {
5890 	return slab->objects;
5891 }
5892 #endif
5893 
5894 #ifdef CONFIG_SLUB_DEBUG
5895 static void validate_slab(struct kmem_cache *s, struct slab *slab,
5896 			  unsigned long *obj_map)
5897 {
5898 	void *p;
5899 	void *addr = slab_address(slab);
5900 
5901 	if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
5902 		return;
5903 
5904 	/* Now we know that a valid freelist exists */
5905 	__fill_map(obj_map, s, slab);
5906 	for_each_object(p, s, addr, slab->objects) {
5907 		u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
5908 			 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
5909 
5910 		if (!check_object(s, slab, p, val))
5911 			break;
5912 	}
5913 }
5914 
5915 static int validate_slab_node(struct kmem_cache *s,
5916 		struct kmem_cache_node *n, unsigned long *obj_map)
5917 {
5918 	unsigned long count = 0;
5919 	struct slab *slab;
5920 	unsigned long flags;
5921 
5922 	spin_lock_irqsave(&n->list_lock, flags);
5923 
5924 	list_for_each_entry(slab, &n->partial, slab_list) {
5925 		validate_slab(s, slab, obj_map);
5926 		count++;
5927 	}
5928 	if (count != n->nr_partial) {
5929 		pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
5930 		       s->name, count, n->nr_partial);
5931 		slab_add_kunit_errors();
5932 	}
5933 
5934 	if (!(s->flags & SLAB_STORE_USER))
5935 		goto out;
5936 
5937 	list_for_each_entry(slab, &n->full, slab_list) {
5938 		validate_slab(s, slab, obj_map);
5939 		count++;
5940 	}
5941 	if (count != node_nr_slabs(n)) {
5942 		pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
5943 		       s->name, count, node_nr_slabs(n));
5944 		slab_add_kunit_errors();
5945 	}
5946 
5947 out:
5948 	spin_unlock_irqrestore(&n->list_lock, flags);
5949 	return count;
5950 }
5951 
5952 long validate_slab_cache(struct kmem_cache *s)
5953 {
5954 	int node;
5955 	unsigned long count = 0;
5956 	struct kmem_cache_node *n;
5957 	unsigned long *obj_map;
5958 
5959 	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
5960 	if (!obj_map)
5961 		return -ENOMEM;
5962 
5963 	flush_all(s);
5964 	for_each_kmem_cache_node(s, node, n)
5965 		count += validate_slab_node(s, n, obj_map);
5966 
5967 	bitmap_free(obj_map);
5968 
5969 	return count;
5970 }
5971 EXPORT_SYMBOL(validate_slab_cache);
5972 
5973 #ifdef CONFIG_DEBUG_FS
5974 /*
5975  * Generate lists of code addresses where slabcache objects are allocated
5976  * and freed.
5977  */
5978 
5979 struct location {
5980 	depot_stack_handle_t handle;
5981 	unsigned long count;
5982 	unsigned long addr;
5983 	unsigned long waste;
5984 	long long sum_time;
5985 	long min_time;
5986 	long max_time;
5987 	long min_pid;
5988 	long max_pid;
5989 	DECLARE_BITMAP(cpus, NR_CPUS);
5990 	nodemask_t nodes;
5991 };
5992 
5993 struct loc_track {
5994 	unsigned long max;
5995 	unsigned long count;
5996 	struct location *loc;
5997 	loff_t idx;
5998 };
5999 
6000 static struct dentry *slab_debugfs_root;
6001 
6002 static void free_loc_track(struct loc_track *t)
6003 {
6004 	if (t->max)
6005 		free_pages((unsigned long)t->loc,
6006 			get_order(sizeof(struct location) * t->max));
6007 }
6008 
6009 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
6010 {
6011 	struct location *l;
6012 	int order;
6013 
6014 	order = get_order(sizeof(struct location) * max);
6015 
6016 	l = (void *)__get_free_pages(flags, order);
6017 	if (!l)
6018 		return 0;
6019 
6020 	if (t->count) {
6021 		memcpy(l, t->loc, sizeof(struct location) * t->count);
6022 		free_loc_track(t);
6023 	}
6024 	t->max = max;
6025 	t->loc = l;
6026 	return 1;
6027 }
6028 
6029 static int add_location(struct loc_track *t, struct kmem_cache *s,
6030 				const struct track *track,
6031 				unsigned int orig_size)
6032 {
6033 	long start, end, pos;
6034 	struct location *l;
6035 	unsigned long caddr, chandle, cwaste;
6036 	unsigned long age = jiffies - track->when;
6037 	depot_stack_handle_t handle = 0;
6038 	unsigned int waste = s->object_size - orig_size;
6039 
6040 #ifdef CONFIG_STACKDEPOT
6041 	handle = READ_ONCE(track->handle);
6042 #endif
6043 	start = -1;
6044 	end = t->count;
6045 
6046 	for ( ; ; ) {
6047 		pos = start + (end - start + 1) / 2;
6048 
6049 		/*
6050 		 * There is nothing at "end". If we end up there
6051 		 * we need to add something to before end.
6052 		 */
6053 		if (pos == end)
6054 			break;
6055 
6056 		l = &t->loc[pos];
6057 		caddr = l->addr;
6058 		chandle = l->handle;
6059 		cwaste = l->waste;
6060 		if ((track->addr == caddr) && (handle == chandle) &&
6061 			(waste == cwaste)) {
6062 
6063 			l->count++;
6064 			if (track->when) {
6065 				l->sum_time += age;
6066 				if (age < l->min_time)
6067 					l->min_time = age;
6068 				if (age > l->max_time)
6069 					l->max_time = age;
6070 
6071 				if (track->pid < l->min_pid)
6072 					l->min_pid = track->pid;
6073 				if (track->pid > l->max_pid)
6074 					l->max_pid = track->pid;
6075 
6076 				cpumask_set_cpu(track->cpu,
6077 						to_cpumask(l->cpus));
6078 			}
6079 			node_set(page_to_nid(virt_to_page(track)), l->nodes);
6080 			return 1;
6081 		}
6082 
6083 		if (track->addr < caddr)
6084 			end = pos;
6085 		else if (track->addr == caddr && handle < chandle)
6086 			end = pos;
6087 		else if (track->addr == caddr && handle == chandle &&
6088 				waste < cwaste)
6089 			end = pos;
6090 		else
6091 			start = pos;
6092 	}
6093 
6094 	/*
6095 	 * Not found. Insert new tracking element.
6096 	 */
6097 	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
6098 		return 0;
6099 
6100 	l = t->loc + pos;
6101 	if (pos < t->count)
6102 		memmove(l + 1, l,
6103 			(t->count - pos) * sizeof(struct location));
6104 	t->count++;
6105 	l->count = 1;
6106 	l->addr = track->addr;
6107 	l->sum_time = age;
6108 	l->min_time = age;
6109 	l->max_time = age;
6110 	l->min_pid = track->pid;
6111 	l->max_pid = track->pid;
6112 	l->handle = handle;
6113 	l->waste = waste;
6114 	cpumask_clear(to_cpumask(l->cpus));
6115 	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
6116 	nodes_clear(l->nodes);
6117 	node_set(page_to_nid(virt_to_page(track)), l->nodes);
6118 	return 1;
6119 }
6120 
6121 static void process_slab(struct loc_track *t, struct kmem_cache *s,
6122 		struct slab *slab, enum track_item alloc,
6123 		unsigned long *obj_map)
6124 {
6125 	void *addr = slab_address(slab);
6126 	bool is_alloc = (alloc == TRACK_ALLOC);
6127 	void *p;
6128 
6129 	__fill_map(obj_map, s, slab);
6130 
6131 	for_each_object(p, s, addr, slab->objects)
6132 		if (!test_bit(__obj_to_index(s, addr, p), obj_map))
6133 			add_location(t, s, get_track(s, p, alloc),
6134 				     is_alloc ? get_orig_size(s, p) :
6135 						s->object_size);
6136 }
6137 #endif  /* CONFIG_DEBUG_FS   */
6138 #endif	/* CONFIG_SLUB_DEBUG */
6139 
6140 #ifdef SLAB_SUPPORTS_SYSFS
6141 enum slab_stat_type {
6142 	SL_ALL,			/* All slabs */
6143 	SL_PARTIAL,		/* Only partially allocated slabs */
6144 	SL_CPU,			/* Only slabs used for cpu caches */
6145 	SL_OBJECTS,		/* Determine allocated objects not slabs */
6146 	SL_TOTAL		/* Determine object capacity not slabs */
6147 };
6148 
6149 #define SO_ALL		(1 << SL_ALL)
6150 #define SO_PARTIAL	(1 << SL_PARTIAL)
6151 #define SO_CPU		(1 << SL_CPU)
6152 #define SO_OBJECTS	(1 << SL_OBJECTS)
6153 #define SO_TOTAL	(1 << SL_TOTAL)
6154 
6155 static ssize_t show_slab_objects(struct kmem_cache *s,
6156 				 char *buf, unsigned long flags)
6157 {
6158 	unsigned long total = 0;
6159 	int node;
6160 	int x;
6161 	unsigned long *nodes;
6162 	int len = 0;
6163 
6164 	nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
6165 	if (!nodes)
6166 		return -ENOMEM;
6167 
6168 	if (flags & SO_CPU) {
6169 		int cpu;
6170 
6171 		for_each_possible_cpu(cpu) {
6172 			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
6173 							       cpu);
6174 			int node;
6175 			struct slab *slab;
6176 
6177 			slab = READ_ONCE(c->slab);
6178 			if (!slab)
6179 				continue;
6180 
6181 			node = slab_nid(slab);
6182 			if (flags & SO_TOTAL)
6183 				x = slab->objects;
6184 			else if (flags & SO_OBJECTS)
6185 				x = slab->inuse;
6186 			else
6187 				x = 1;
6188 
6189 			total += x;
6190 			nodes[node] += x;
6191 
6192 #ifdef CONFIG_SLUB_CPU_PARTIAL
6193 			slab = slub_percpu_partial_read_once(c);
6194 			if (slab) {
6195 				node = slab_nid(slab);
6196 				if (flags & SO_TOTAL)
6197 					WARN_ON_ONCE(1);
6198 				else if (flags & SO_OBJECTS)
6199 					WARN_ON_ONCE(1);
6200 				else
6201 					x = data_race(slab->slabs);
6202 				total += x;
6203 				nodes[node] += x;
6204 			}
6205 #endif
6206 		}
6207 	}
6208 
6209 	/*
6210 	 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
6211 	 * already held which will conflict with an existing lock order:
6212 	 *
6213 	 * mem_hotplug_lock->slab_mutex->kernfs_mutex
6214 	 *
6215 	 * We don't really need mem_hotplug_lock (to hold off
6216 	 * slab_mem_going_offline_callback) here because slab's memory hot
6217 	 * unplug code doesn't destroy the kmem_cache->node[] data.
6218 	 */
6219 
6220 #ifdef CONFIG_SLUB_DEBUG
6221 	if (flags & SO_ALL) {
6222 		struct kmem_cache_node *n;
6223 
6224 		for_each_kmem_cache_node(s, node, n) {
6225 
6226 			if (flags & SO_TOTAL)
6227 				x = node_nr_objs(n);
6228 			else if (flags & SO_OBJECTS)
6229 				x = node_nr_objs(n) - count_partial(n, count_free);
6230 			else
6231 				x = node_nr_slabs(n);
6232 			total += x;
6233 			nodes[node] += x;
6234 		}
6235 
6236 	} else
6237 #endif
6238 	if (flags & SO_PARTIAL) {
6239 		struct kmem_cache_node *n;
6240 
6241 		for_each_kmem_cache_node(s, node, n) {
6242 			if (flags & SO_TOTAL)
6243 				x = count_partial(n, count_total);
6244 			else if (flags & SO_OBJECTS)
6245 				x = count_partial(n, count_inuse);
6246 			else
6247 				x = n->nr_partial;
6248 			total += x;
6249 			nodes[node] += x;
6250 		}
6251 	}
6252 
6253 	len += sysfs_emit_at(buf, len, "%lu", total);
6254 #ifdef CONFIG_NUMA
6255 	for (node = 0; node < nr_node_ids; node++) {
6256 		if (nodes[node])
6257 			len += sysfs_emit_at(buf, len, " N%d=%lu",
6258 					     node, nodes[node]);
6259 	}
6260 #endif
6261 	len += sysfs_emit_at(buf, len, "\n");
6262 	kfree(nodes);
6263 
6264 	return len;
6265 }
6266 
6267 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
6268 #define to_slab(n) container_of(n, struct kmem_cache, kobj)
6269 
6270 struct slab_attribute {
6271 	struct attribute attr;
6272 	ssize_t (*show)(struct kmem_cache *s, char *buf);
6273 	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
6274 };
6275 
6276 #define SLAB_ATTR_RO(_name) \
6277 	static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400)
6278 
6279 #define SLAB_ATTR(_name) \
6280 	static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600)
6281 
6282 static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
6283 {
6284 	return sysfs_emit(buf, "%u\n", s->size);
6285 }
6286 SLAB_ATTR_RO(slab_size);
6287 
6288 static ssize_t align_show(struct kmem_cache *s, char *buf)
6289 {
6290 	return sysfs_emit(buf, "%u\n", s->align);
6291 }
6292 SLAB_ATTR_RO(align);
6293 
6294 static ssize_t object_size_show(struct kmem_cache *s, char *buf)
6295 {
6296 	return sysfs_emit(buf, "%u\n", s->object_size);
6297 }
6298 SLAB_ATTR_RO(object_size);
6299 
6300 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
6301 {
6302 	return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
6303 }
6304 SLAB_ATTR_RO(objs_per_slab);
6305 
6306 static ssize_t order_show(struct kmem_cache *s, char *buf)
6307 {
6308 	return sysfs_emit(buf, "%u\n", oo_order(s->oo));
6309 }
6310 SLAB_ATTR_RO(order);
6311 
6312 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
6313 {
6314 	return sysfs_emit(buf, "%lu\n", s->min_partial);
6315 }
6316 
6317 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
6318 				 size_t length)
6319 {
6320 	unsigned long min;
6321 	int err;
6322 
6323 	err = kstrtoul(buf, 10, &min);
6324 	if (err)
6325 		return err;
6326 
6327 	s->min_partial = min;
6328 	return length;
6329 }
6330 SLAB_ATTR(min_partial);
6331 
6332 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
6333 {
6334 	unsigned int nr_partial = 0;
6335 #ifdef CONFIG_SLUB_CPU_PARTIAL
6336 	nr_partial = s->cpu_partial;
6337 #endif
6338 
6339 	return sysfs_emit(buf, "%u\n", nr_partial);
6340 }
6341 
6342 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
6343 				 size_t length)
6344 {
6345 	unsigned int objects;
6346 	int err;
6347 
6348 	err = kstrtouint(buf, 10, &objects);
6349 	if (err)
6350 		return err;
6351 	if (objects && !kmem_cache_has_cpu_partial(s))
6352 		return -EINVAL;
6353 
6354 	slub_set_cpu_partial(s, objects);
6355 	flush_all(s);
6356 	return length;
6357 }
6358 SLAB_ATTR(cpu_partial);
6359 
6360 static ssize_t ctor_show(struct kmem_cache *s, char *buf)
6361 {
6362 	if (!s->ctor)
6363 		return 0;
6364 	return sysfs_emit(buf, "%pS\n", s->ctor);
6365 }
6366 SLAB_ATTR_RO(ctor);
6367 
6368 static ssize_t aliases_show(struct kmem_cache *s, char *buf)
6369 {
6370 	return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
6371 }
6372 SLAB_ATTR_RO(aliases);
6373 
6374 static ssize_t partial_show(struct kmem_cache *s, char *buf)
6375 {
6376 	return show_slab_objects(s, buf, SO_PARTIAL);
6377 }
6378 SLAB_ATTR_RO(partial);
6379 
6380 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
6381 {
6382 	return show_slab_objects(s, buf, SO_CPU);
6383 }
6384 SLAB_ATTR_RO(cpu_slabs);
6385 
6386 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
6387 {
6388 	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
6389 }
6390 SLAB_ATTR_RO(objects_partial);
6391 
6392 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
6393 {
6394 	int objects = 0;
6395 	int slabs = 0;
6396 	int cpu __maybe_unused;
6397 	int len = 0;
6398 
6399 #ifdef CONFIG_SLUB_CPU_PARTIAL
6400 	for_each_online_cpu(cpu) {
6401 		struct slab *slab;
6402 
6403 		slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6404 
6405 		if (slab)
6406 			slabs += data_race(slab->slabs);
6407 	}
6408 #endif
6409 
6410 	/* Approximate half-full slabs, see slub_set_cpu_partial() */
6411 	objects = (slabs * oo_objects(s->oo)) / 2;
6412 	len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
6413 
6414 #ifdef CONFIG_SLUB_CPU_PARTIAL
6415 	for_each_online_cpu(cpu) {
6416 		struct slab *slab;
6417 
6418 		slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6419 		if (slab) {
6420 			slabs = data_race(slab->slabs);
6421 			objects = (slabs * oo_objects(s->oo)) / 2;
6422 			len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
6423 					     cpu, objects, slabs);
6424 		}
6425 	}
6426 #endif
6427 	len += sysfs_emit_at(buf, len, "\n");
6428 
6429 	return len;
6430 }
6431 SLAB_ATTR_RO(slabs_cpu_partial);
6432 
6433 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
6434 {
6435 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
6436 }
6437 SLAB_ATTR_RO(reclaim_account);
6438 
6439 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
6440 {
6441 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
6442 }
6443 SLAB_ATTR_RO(hwcache_align);
6444 
6445 #ifdef CONFIG_ZONE_DMA
6446 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
6447 {
6448 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
6449 }
6450 SLAB_ATTR_RO(cache_dma);
6451 #endif
6452 
6453 #ifdef CONFIG_HARDENED_USERCOPY
6454 static ssize_t usersize_show(struct kmem_cache *s, char *buf)
6455 {
6456 	return sysfs_emit(buf, "%u\n", s->usersize);
6457 }
6458 SLAB_ATTR_RO(usersize);
6459 #endif
6460 
6461 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
6462 {
6463 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
6464 }
6465 SLAB_ATTR_RO(destroy_by_rcu);
6466 
6467 #ifdef CONFIG_SLUB_DEBUG
6468 static ssize_t slabs_show(struct kmem_cache *s, char *buf)
6469 {
6470 	return show_slab_objects(s, buf, SO_ALL);
6471 }
6472 SLAB_ATTR_RO(slabs);
6473 
6474 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
6475 {
6476 	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
6477 }
6478 SLAB_ATTR_RO(total_objects);
6479 
6480 static ssize_t objects_show(struct kmem_cache *s, char *buf)
6481 {
6482 	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
6483 }
6484 SLAB_ATTR_RO(objects);
6485 
6486 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
6487 {
6488 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
6489 }
6490 SLAB_ATTR_RO(sanity_checks);
6491 
6492 static ssize_t trace_show(struct kmem_cache *s, char *buf)
6493 {
6494 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
6495 }
6496 SLAB_ATTR_RO(trace);
6497 
6498 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
6499 {
6500 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
6501 }
6502 
6503 SLAB_ATTR_RO(red_zone);
6504 
6505 static ssize_t poison_show(struct kmem_cache *s, char *buf)
6506 {
6507 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
6508 }
6509 
6510 SLAB_ATTR_RO(poison);
6511 
6512 static ssize_t store_user_show(struct kmem_cache *s, char *buf)
6513 {
6514 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
6515 }
6516 
6517 SLAB_ATTR_RO(store_user);
6518 
6519 static ssize_t validate_show(struct kmem_cache *s, char *buf)
6520 {
6521 	return 0;
6522 }
6523 
6524 static ssize_t validate_store(struct kmem_cache *s,
6525 			const char *buf, size_t length)
6526 {
6527 	int ret = -EINVAL;
6528 
6529 	if (buf[0] == '1' && kmem_cache_debug(s)) {
6530 		ret = validate_slab_cache(s);
6531 		if (ret >= 0)
6532 			ret = length;
6533 	}
6534 	return ret;
6535 }
6536 SLAB_ATTR(validate);
6537 
6538 #endif /* CONFIG_SLUB_DEBUG */
6539 
6540 #ifdef CONFIG_FAILSLAB
6541 static ssize_t failslab_show(struct kmem_cache *s, char *buf)
6542 {
6543 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
6544 }
6545 
6546 static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
6547 				size_t length)
6548 {
6549 	if (s->refcount > 1)
6550 		return -EINVAL;
6551 
6552 	if (buf[0] == '1')
6553 		WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB);
6554 	else
6555 		WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB);
6556 
6557 	return length;
6558 }
6559 SLAB_ATTR(failslab);
6560 #endif
6561 
6562 static ssize_t shrink_show(struct kmem_cache *s, char *buf)
6563 {
6564 	return 0;
6565 }
6566 
6567 static ssize_t shrink_store(struct kmem_cache *s,
6568 			const char *buf, size_t length)
6569 {
6570 	if (buf[0] == '1')
6571 		kmem_cache_shrink(s);
6572 	else
6573 		return -EINVAL;
6574 	return length;
6575 }
6576 SLAB_ATTR(shrink);
6577 
6578 #ifdef CONFIG_NUMA
6579 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
6580 {
6581 	return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
6582 }
6583 
6584 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
6585 				const char *buf, size_t length)
6586 {
6587 	unsigned int ratio;
6588 	int err;
6589 
6590 	err = kstrtouint(buf, 10, &ratio);
6591 	if (err)
6592 		return err;
6593 	if (ratio > 100)
6594 		return -ERANGE;
6595 
6596 	s->remote_node_defrag_ratio = ratio * 10;
6597 
6598 	return length;
6599 }
6600 SLAB_ATTR(remote_node_defrag_ratio);
6601 #endif
6602 
6603 #ifdef CONFIG_SLUB_STATS
6604 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
6605 {
6606 	unsigned long sum  = 0;
6607 	int cpu;
6608 	int len = 0;
6609 	int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
6610 
6611 	if (!data)
6612 		return -ENOMEM;
6613 
6614 	for_each_online_cpu(cpu) {
6615 		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
6616 
6617 		data[cpu] = x;
6618 		sum += x;
6619 	}
6620 
6621 	len += sysfs_emit_at(buf, len, "%lu", sum);
6622 
6623 #ifdef CONFIG_SMP
6624 	for_each_online_cpu(cpu) {
6625 		if (data[cpu])
6626 			len += sysfs_emit_at(buf, len, " C%d=%u",
6627 					     cpu, data[cpu]);
6628 	}
6629 #endif
6630 	kfree(data);
6631 	len += sysfs_emit_at(buf, len, "\n");
6632 
6633 	return len;
6634 }
6635 
6636 static void clear_stat(struct kmem_cache *s, enum stat_item si)
6637 {
6638 	int cpu;
6639 
6640 	for_each_online_cpu(cpu)
6641 		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
6642 }
6643 
6644 #define STAT_ATTR(si, text) 					\
6645 static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
6646 {								\
6647 	return show_stat(s, buf, si);				\
6648 }								\
6649 static ssize_t text##_store(struct kmem_cache *s,		\
6650 				const char *buf, size_t length)	\
6651 {								\
6652 	if (buf[0] != '0')					\
6653 		return -EINVAL;					\
6654 	clear_stat(s, si);					\
6655 	return length;						\
6656 }								\
6657 SLAB_ATTR(text);						\
6658 
6659 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
6660 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
6661 STAT_ATTR(FREE_FASTPATH, free_fastpath);
6662 STAT_ATTR(FREE_SLOWPATH, free_slowpath);
6663 STAT_ATTR(FREE_FROZEN, free_frozen);
6664 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
6665 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
6666 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
6667 STAT_ATTR(ALLOC_SLAB, alloc_slab);
6668 STAT_ATTR(ALLOC_REFILL, alloc_refill);
6669 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
6670 STAT_ATTR(FREE_SLAB, free_slab);
6671 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
6672 STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
6673 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
6674 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
6675 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
6676 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
6677 STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
6678 STAT_ATTR(ORDER_FALLBACK, order_fallback);
6679 STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
6680 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
6681 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
6682 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
6683 STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
6684 STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
6685 #endif	/* CONFIG_SLUB_STATS */
6686 
6687 #ifdef CONFIG_KFENCE
6688 static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)
6689 {
6690 	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE));
6691 }
6692 
6693 static ssize_t skip_kfence_store(struct kmem_cache *s,
6694 			const char *buf, size_t length)
6695 {
6696 	int ret = length;
6697 
6698 	if (buf[0] == '0')
6699 		s->flags &= ~SLAB_SKIP_KFENCE;
6700 	else if (buf[0] == '1')
6701 		s->flags |= SLAB_SKIP_KFENCE;
6702 	else
6703 		ret = -EINVAL;
6704 
6705 	return ret;
6706 }
6707 SLAB_ATTR(skip_kfence);
6708 #endif
6709 
6710 static struct attribute *slab_attrs[] = {
6711 	&slab_size_attr.attr,
6712 	&object_size_attr.attr,
6713 	&objs_per_slab_attr.attr,
6714 	&order_attr.attr,
6715 	&min_partial_attr.attr,
6716 	&cpu_partial_attr.attr,
6717 	&objects_partial_attr.attr,
6718 	&partial_attr.attr,
6719 	&cpu_slabs_attr.attr,
6720 	&ctor_attr.attr,
6721 	&aliases_attr.attr,
6722 	&align_attr.attr,
6723 	&hwcache_align_attr.attr,
6724 	&reclaim_account_attr.attr,
6725 	&destroy_by_rcu_attr.attr,
6726 	&shrink_attr.attr,
6727 	&slabs_cpu_partial_attr.attr,
6728 #ifdef CONFIG_SLUB_DEBUG
6729 	&total_objects_attr.attr,
6730 	&objects_attr.attr,
6731 	&slabs_attr.attr,
6732 	&sanity_checks_attr.attr,
6733 	&trace_attr.attr,
6734 	&red_zone_attr.attr,
6735 	&poison_attr.attr,
6736 	&store_user_attr.attr,
6737 	&validate_attr.attr,
6738 #endif
6739 #ifdef CONFIG_ZONE_DMA
6740 	&cache_dma_attr.attr,
6741 #endif
6742 #ifdef CONFIG_NUMA
6743 	&remote_node_defrag_ratio_attr.attr,
6744 #endif
6745 #ifdef CONFIG_SLUB_STATS
6746 	&alloc_fastpath_attr.attr,
6747 	&alloc_slowpath_attr.attr,
6748 	&free_fastpath_attr.attr,
6749 	&free_slowpath_attr.attr,
6750 	&free_frozen_attr.attr,
6751 	&free_add_partial_attr.attr,
6752 	&free_remove_partial_attr.attr,
6753 	&alloc_from_partial_attr.attr,
6754 	&alloc_slab_attr.attr,
6755 	&alloc_refill_attr.attr,
6756 	&alloc_node_mismatch_attr.attr,
6757 	&free_slab_attr.attr,
6758 	&cpuslab_flush_attr.attr,
6759 	&deactivate_full_attr.attr,
6760 	&deactivate_empty_attr.attr,
6761 	&deactivate_to_head_attr.attr,
6762 	&deactivate_to_tail_attr.attr,
6763 	&deactivate_remote_frees_attr.attr,
6764 	&deactivate_bypass_attr.attr,
6765 	&order_fallback_attr.attr,
6766 	&cmpxchg_double_fail_attr.attr,
6767 	&cmpxchg_double_cpu_fail_attr.attr,
6768 	&cpu_partial_alloc_attr.attr,
6769 	&cpu_partial_free_attr.attr,
6770 	&cpu_partial_node_attr.attr,
6771 	&cpu_partial_drain_attr.attr,
6772 #endif
6773 #ifdef CONFIG_FAILSLAB
6774 	&failslab_attr.attr,
6775 #endif
6776 #ifdef CONFIG_HARDENED_USERCOPY
6777 	&usersize_attr.attr,
6778 #endif
6779 #ifdef CONFIG_KFENCE
6780 	&skip_kfence_attr.attr,
6781 #endif
6782 
6783 	NULL
6784 };
6785 
6786 static const struct attribute_group slab_attr_group = {
6787 	.attrs = slab_attrs,
6788 };
6789 
6790 static ssize_t slab_attr_show(struct kobject *kobj,
6791 				struct attribute *attr,
6792 				char *buf)
6793 {
6794 	struct slab_attribute *attribute;
6795 	struct kmem_cache *s;
6796 
6797 	attribute = to_slab_attr(attr);
6798 	s = to_slab(kobj);
6799 
6800 	if (!attribute->show)
6801 		return -EIO;
6802 
6803 	return attribute->show(s, buf);
6804 }
6805 
6806 static ssize_t slab_attr_store(struct kobject *kobj,
6807 				struct attribute *attr,
6808 				const char *buf, size_t len)
6809 {
6810 	struct slab_attribute *attribute;
6811 	struct kmem_cache *s;
6812 
6813 	attribute = to_slab_attr(attr);
6814 	s = to_slab(kobj);
6815 
6816 	if (!attribute->store)
6817 		return -EIO;
6818 
6819 	return attribute->store(s, buf, len);
6820 }
6821 
6822 static void kmem_cache_release(struct kobject *k)
6823 {
6824 	slab_kmem_cache_release(to_slab(k));
6825 }
6826 
6827 static const struct sysfs_ops slab_sysfs_ops = {
6828 	.show = slab_attr_show,
6829 	.store = slab_attr_store,
6830 };
6831 
6832 static const struct kobj_type slab_ktype = {
6833 	.sysfs_ops = &slab_sysfs_ops,
6834 	.release = kmem_cache_release,
6835 };
6836 
6837 static struct kset *slab_kset;
6838 
6839 static inline struct kset *cache_kset(struct kmem_cache *s)
6840 {
6841 	return slab_kset;
6842 }
6843 
6844 #define ID_STR_LENGTH 32
6845 
6846 /* Create a unique string id for a slab cache:
6847  *
6848  * Format	:[flags-]size
6849  */
6850 static char *create_unique_id(struct kmem_cache *s)
6851 {
6852 	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
6853 	char *p = name;
6854 
6855 	if (!name)
6856 		return ERR_PTR(-ENOMEM);
6857 
6858 	*p++ = ':';
6859 	/*
6860 	 * First flags affecting slabcache operations. We will only
6861 	 * get here for aliasable slabs so we do not need to support
6862 	 * too many flags. The flags here must cover all flags that
6863 	 * are matched during merging to guarantee that the id is
6864 	 * unique.
6865 	 */
6866 	if (s->flags & SLAB_CACHE_DMA)
6867 		*p++ = 'd';
6868 	if (s->flags & SLAB_CACHE_DMA32)
6869 		*p++ = 'D';
6870 	if (s->flags & SLAB_RECLAIM_ACCOUNT)
6871 		*p++ = 'a';
6872 	if (s->flags & SLAB_CONSISTENCY_CHECKS)
6873 		*p++ = 'F';
6874 	if (s->flags & SLAB_ACCOUNT)
6875 		*p++ = 'A';
6876 	if (p != name + 1)
6877 		*p++ = '-';
6878 	p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size);
6879 
6880 	if (WARN_ON(p > name + ID_STR_LENGTH - 1)) {
6881 		kfree(name);
6882 		return ERR_PTR(-EINVAL);
6883 	}
6884 	kmsan_unpoison_memory(name, p - name);
6885 	return name;
6886 }
6887 
6888 static int sysfs_slab_add(struct kmem_cache *s)
6889 {
6890 	int err;
6891 	const char *name;
6892 	struct kset *kset = cache_kset(s);
6893 	int unmergeable = slab_unmergeable(s);
6894 
6895 	if (!unmergeable && disable_higher_order_debug &&
6896 			(slub_debug & DEBUG_METADATA_FLAGS))
6897 		unmergeable = 1;
6898 
6899 	if (unmergeable) {
6900 		/*
6901 		 * Slabcache can never be merged so we can use the name proper.
6902 		 * This is typically the case for debug situations. In that
6903 		 * case we can catch duplicate names easily.
6904 		 */
6905 		sysfs_remove_link(&slab_kset->kobj, s->name);
6906 		name = s->name;
6907 	} else {
6908 		/*
6909 		 * Create a unique name for the slab as a target
6910 		 * for the symlinks.
6911 		 */
6912 		name = create_unique_id(s);
6913 		if (IS_ERR(name))
6914 			return PTR_ERR(name);
6915 	}
6916 
6917 	s->kobj.kset = kset;
6918 	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6921 
6922 	err = sysfs_create_group(&s->kobj, &slab_attr_group);
6923 	if (err)
6924 		goto out_del_kobj;
6925 
6926 	if (!unmergeable) {
6927 		/* Setup first alias */
6928 		sysfs_slab_alias(s, s->name);
6929 	}
6930 out:
6931 	if (!unmergeable)
6932 		kfree(name);
6933 	return err;
6934 out_del_kobj:
6935 	kobject_del(&s->kobj);
6936 	goto out;
6937 }
6938 
6939 void sysfs_slab_unlink(struct kmem_cache *s)
6940 {
6941 	kobject_del(&s->kobj);
6942 }
6943 
6944 void sysfs_slab_release(struct kmem_cache *s)
6945 {
6946 	kobject_put(&s->kobj);
6947 }
6948 
6949 /*
6950  * Need to buffer aliases during bootup until sysfs becomes
6951  * available lest we lose that information.
6952  */
6953 struct saved_alias {
6954 	struct kmem_cache *s;
6955 	const char *name;
6956 	struct saved_alias *next;
6957 };
6958 
6959 static struct saved_alias *alias_list;
6960 
6961 static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
6962 {
6963 	struct saved_alias *al;
6964 
6965 	if (slab_state == FULL) {
6966 		/*
6967 		 * If we have a leftover link then remove it.
6968 		 */
6969 		sysfs_remove_link(&slab_kset->kobj, name);
6970 		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
6971 	}
6972 
6973 	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
6974 	if (!al)
6975 		return -ENOMEM;
6976 
6977 	al->s = s;
6978 	al->name = name;
6979 	al->next = alias_list;
6980 	alias_list = al;
6981 	kmsan_unpoison_memory(al, sizeof(*al));
6982 	return 0;
6983 }
6984 
6985 static int __init slab_sysfs_init(void)
6986 {
6987 	struct kmem_cache *s;
6988 	int err;
6989 
6990 	mutex_lock(&slab_mutex);
6991 
6992 	slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
6993 	if (!slab_kset) {
6994 		mutex_unlock(&slab_mutex);
6995 		pr_err("Cannot register slab subsystem.\n");
6996 		return -ENOMEM;
6997 	}
6998 
6999 	slab_state = FULL;
7000 
7001 	list_for_each_entry(s, &slab_caches, list) {
7002 		err = sysfs_slab_add(s);
7003 		if (err)
7004 			pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
7005 			       s->name);
7006 	}
7007 
7008 	while (alias_list) {
7009 		struct saved_alias *al = alias_list;
7010 
7011 		alias_list = alias_list->next;
7012 		err = sysfs_slab_alias(al->s, al->name);
7013 		if (err)
7014 			pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
7015 			       al->name);
7016 		kfree(al);
7017 	}
7018 
7019 	mutex_unlock(&slab_mutex);
7020 	return 0;
7021 }
7022 late_initcall(slab_sysfs_init);
7023 #endif /* SLAB_SUPPORTS_SYSFS */
7024 
7025 #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
7026 static int slab_debugfs_show(struct seq_file *seq, void *v)
7027 {
7028 	struct loc_track *t = seq->private;
7029 	struct location *l;
7030 	unsigned long idx;
7031 
7032 	idx = (unsigned long) t->idx;
7033 	if (idx < t->count) {
7034 		l = &t->loc[idx];
7035 
7036 		seq_printf(seq, "%7ld ", l->count);
7037 
7038 		if (l->addr)
7039 			seq_printf(seq, "%pS", (void *)l->addr);
7040 		else
7041 			seq_puts(seq, "<not-available>");
7042 
7043 		if (l->waste)
7044 			seq_printf(seq, " waste=%lu/%lu",
7045 				l->count * l->waste, l->waste);
7046 
7047 		if (l->sum_time != l->min_time) {
7048 			seq_printf(seq, " age=%ld/%llu/%ld",
7049 				l->min_time, div_u64(l->sum_time, l->count),
7050 				l->max_time);
7051 		} else
7052 			seq_printf(seq, " age=%ld", l->min_time);
7053 
7054 		if (l->min_pid != l->max_pid)
7055 			seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
7056 		else
7057 			seq_printf(seq, " pid=%ld",
7058 				l->min_pid);
7059 
7060 		if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
7061 			seq_printf(seq, " cpus=%*pbl",
7062 				 cpumask_pr_args(to_cpumask(l->cpus)));
7063 
7064 		if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
7065 			seq_printf(seq, " nodes=%*pbl",
7066 				 nodemask_pr_args(&l->nodes));
7067 
7068 #ifdef CONFIG_STACKDEPOT
7069 		{
7070 			depot_stack_handle_t handle;
7071 			unsigned long *entries;
7072 			unsigned int nr_entries, j;
7073 
7074 			handle = READ_ONCE(l->handle);
7075 			if (handle) {
7076 				nr_entries = stack_depot_fetch(handle, &entries);
7077 				seq_puts(seq, "\n");
7078 				for (j = 0; j < nr_entries; j++)
7079 					seq_printf(seq, "        %pS\n", (void *)entries[j]);
7080 			}
7081 		}
7082 #endif
7083 		seq_puts(seq, "\n");
7084 	}
7085 
7086 	if (!idx && !t->count)
7087 		seq_puts(seq, "No data\n");
7088 
7089 	return 0;
7090 }
7091 
7092 static void slab_debugfs_stop(struct seq_file *seq, void *v)
7093 {
7094 }
7095 
7096 static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
7097 {
7098 	struct loc_track *t = seq->private;
7099 
7100 	t->idx = ++(*ppos);
7101 	if (*ppos <= t->count)
7102 		return ppos;
7103 
7104 	return NULL;
7105 }
7106 
7107 static int cmp_loc_by_count(const void *a, const void *b, const void *data)
7108 {
7109 	struct location *loc1 = (struct location *)a;
7110 	struct location *loc2 = (struct location *)b;
7111 
7112 	if (loc1->count > loc2->count)
7113 		return -1;
7114 	else
7115 		return 1;
7116 }
7117 
7118 static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
7119 {
7120 	struct loc_track *t = seq->private;
7121 
7122 	t->idx = *ppos;
7123 	return ppos;
7124 }
7125 
7126 static const struct seq_operations slab_debugfs_sops = {
7127 	.start  = slab_debugfs_start,
7128 	.next   = slab_debugfs_next,
7129 	.stop   = slab_debugfs_stop,
7130 	.show   = slab_debugfs_show,
7131 };
7132 
7133 static int slab_debug_trace_open(struct inode *inode, struct file *filep)
7134 {
7135 
7136 	struct kmem_cache_node *n;
7137 	enum track_item alloc;
7138 	int node;
7139 	struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
7140 						sizeof(struct loc_track));
7141 	struct kmem_cache *s = file_inode(filep)->i_private;
7142 	unsigned long *obj_map;
7143 
7144 	if (!t)
7145 		return -ENOMEM;
7146 
7147 	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
7148 	if (!obj_map) {
7149 		seq_release_private(inode, filep);
7150 		return -ENOMEM;
7151 	}
7152 
7153 	if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
7154 		alloc = TRACK_ALLOC;
7155 	else
7156 		alloc = TRACK_FREE;
7157 
7158 	if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
7159 		bitmap_free(obj_map);
7160 		seq_release_private(inode, filep);
7161 		return -ENOMEM;
7162 	}
7163 
7164 	for_each_kmem_cache_node(s, node, n) {
7165 		unsigned long flags;
7166 		struct slab *slab;
7167 
7168 		if (!node_nr_slabs(n))
7169 			continue;
7170 
7171 		spin_lock_irqsave(&n->list_lock, flags);
7172 		list_for_each_entry(slab, &n->partial, slab_list)
7173 			process_slab(t, s, slab, alloc, obj_map);
7174 		list_for_each_entry(slab, &n->full, slab_list)
7175 			process_slab(t, s, slab, alloc, obj_map);
7176 		spin_unlock_irqrestore(&n->list_lock, flags);
7177 	}
7178 
7179 	/* Sort locations by count */
7180 	sort_r(t->loc, t->count, sizeof(struct location),
7181 		cmp_loc_by_count, NULL, NULL);
7182 
7183 	bitmap_free(obj_map);
7184 	return 0;
7185 }
7186 
7187 static int slab_debug_trace_release(struct inode *inode, struct file *file)
7188 {
7189 	struct seq_file *seq = file->private_data;
7190 	struct loc_track *t = seq->private;
7191 
7192 	free_loc_track(t);
7193 	return seq_release_private(inode, file);
7194 }
7195 
7196 static const struct file_operations slab_debugfs_fops = {
7197 	.open    = slab_debug_trace_open,
7198 	.read    = seq_read,
7199 	.llseek  = seq_lseek,
7200 	.release = slab_debug_trace_release,
7201 };
7202 
7203 static void debugfs_slab_add(struct kmem_cache *s)
7204 {
7205 	struct dentry *slab_cache_dir;
7206 
7207 	if (unlikely(!slab_debugfs_root))
7208 		return;
7209 
7210 	slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
7211 
7212 	debugfs_create_file("alloc_traces", 0400,
7213 		slab_cache_dir, s, &slab_debugfs_fops);
7214 
7215 	debugfs_create_file("free_traces", 0400,
7216 		slab_cache_dir, s, &slab_debugfs_fops);
7217 }
7218 
7219 void debugfs_slab_release(struct kmem_cache *s)
7220 {
7221 	debugfs_lookup_and_remove(s->name, slab_debugfs_root);
7222 }
7223 
7224 static int __init slab_debugfs_init(void)
7225 {
7226 	struct kmem_cache *s;
7227 
7228 	slab_debugfs_root = debugfs_create_dir("slab", NULL);
7229 
7230 	list_for_each_entry(s, &slab_caches, list)
7231 		if (s->flags & SLAB_STORE_USER)
7232 			debugfs_slab_add(s);
7233 
7234 	return 0;
7235 
7236 }
7237 __initcall(slab_debugfs_init);
7238 #endif
7239 /*
7240  * The /proc/slabinfo ABI
7241  */
7242 #ifdef CONFIG_SLUB_DEBUG
7243 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
7244 {
7245 	unsigned long nr_slabs = 0;
7246 	unsigned long nr_objs = 0;
7247 	unsigned long nr_free = 0;
7248 	int node;
7249 	struct kmem_cache_node *n;
7250 
7251 	for_each_kmem_cache_node(s, node, n) {
7252 		nr_slabs += node_nr_slabs(n);
7253 		nr_objs += node_nr_objs(n);
7254 		nr_free += count_partial_free_approx(n);
7255 	}
7256 
7257 	sinfo->active_objs = nr_objs - nr_free;
7258 	sinfo->num_objs = nr_objs;
7259 	sinfo->active_slabs = nr_slabs;
7260 	sinfo->num_slabs = nr_slabs;
7261 	sinfo->objects_per_slab = oo_objects(s->oo);
7262 	sinfo->cache_order = oo_order(s->oo);
7263 }
7264 #endif /* CONFIG_SLUB_DEBUG */
7265