1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk).
4 *
5 * (C) SGI 2006, Christoph Lameter
6 * Cleaned up and restructured to ease the addition of alternative
7 * implementations of SLAB allocators.
8 * (C) Linux Foundation 2008-2013
9 * Unified interface for all slab allocators
10 */
11
12 #ifndef _LINUX_SLAB_H
13 #define _LINUX_SLAB_H
14
15 #include <linux/cache.h>
16 #include <linux/gfp.h>
17 #include <linux/overflow.h>
18 #include <linux/types.h>
19 #include <linux/rcupdate.h>
20 #include <linux/workqueue.h>
21 #include <linux/percpu-refcount.h>
22 #include <linux/cleanup.h>
23 #include <linux/hash.h>
24
25 enum _slab_flag_bits {
26 _SLAB_CONSISTENCY_CHECKS,
27 _SLAB_RED_ZONE,
28 _SLAB_POISON,
29 _SLAB_KMALLOC,
30 _SLAB_HWCACHE_ALIGN,
31 _SLAB_CACHE_DMA,
32 _SLAB_CACHE_DMA32,
33 _SLAB_STORE_USER,
34 _SLAB_PANIC,
35 _SLAB_TYPESAFE_BY_RCU,
36 _SLAB_TRACE,
37 #ifdef CONFIG_DEBUG_OBJECTS
38 _SLAB_DEBUG_OBJECTS,
39 #endif
40 _SLAB_NOLEAKTRACE,
41 _SLAB_NO_MERGE,
42 #ifdef CONFIG_FAILSLAB
43 _SLAB_FAILSLAB,
44 #endif
45 #ifdef CONFIG_MEMCG
46 _SLAB_ACCOUNT,
47 #endif
48 #ifdef CONFIG_KASAN_GENERIC
49 _SLAB_KASAN,
50 #endif
51 _SLAB_NO_USER_FLAGS,
52 #ifdef CONFIG_KFENCE
53 _SLAB_SKIP_KFENCE,
54 #endif
55 #ifndef CONFIG_SLUB_TINY
56 _SLAB_RECLAIM_ACCOUNT,
57 #endif
58 _SLAB_OBJECT_POISON,
59 _SLAB_CMPXCHG_DOUBLE,
60 #ifdef CONFIG_SLAB_OBJ_EXT
61 _SLAB_NO_OBJ_EXT,
62 #endif
63 _SLAB_FLAGS_LAST_BIT
64 };
65
66 #define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr)))
67 #define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U))
68
69 /*
70 * Flags to pass to kmem_cache_create().
71 * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op
72 */
73 /* DEBUG: Perform (expensive) checks on alloc/free */
74 #define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS)
75 /* DEBUG: Red zone objs in a cache */
76 #define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE)
77 /* DEBUG: Poison objects */
78 #define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON)
79 /* Indicate a kmalloc slab */
80 #define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC)
81 /**
82 * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries.
83 *
84 * Sufficiently large objects are aligned on cache line boundary. For object
85 * size smaller than a half of cache line size, the alignment is on the half of
86 * cache line size. In general, if object size is smaller than 1/2^n of cache
87 * line size, the alignment is adjusted to 1/2^n.
88 *
89 * If explicit alignment is also requested by the respective
90 * &struct kmem_cache_args field, the greater of both is alignments is applied.
91 */
92 #define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN)
93 /* Use GFP_DMA memory */
94 #define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA)
95 /* Use GFP_DMA32 memory */
96 #define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32)
97 /* DEBUG: Store the last owner for bug hunting */
98 #define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER)
99 /* Panic if kmem_cache_create() fails */
100 #define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC)
101 /**
102 * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
103 *
104 * This delays freeing the SLAB page by a grace period, it does _NOT_
105 * delay object freeing. This means that if you do kmem_cache_free()
106 * that memory location is free to be reused at any time. Thus it may
107 * be possible to see another object there in the same RCU grace period.
108 *
109 * This feature only ensures the memory location backing the object
110 * stays valid, the trick to using this is relying on an independent
111 * object validation pass. Something like:
112 *
113 * ::
114 *
115 * begin:
116 * rcu_read_lock();
117 * obj = lockless_lookup(key);
118 * if (obj) {
119 * if (!try_get_ref(obj)) // might fail for free objects
120 * rcu_read_unlock();
121 * goto begin;
122 *
123 * if (obj->key != key) { // not the object we expected
124 * put_ref(obj);
125 * rcu_read_unlock();
126 * goto begin;
127 * }
128 * }
129 * rcu_read_unlock();
130 *
131 * This is useful if we need to approach a kernel structure obliquely,
132 * from its address obtained without the usual locking. We can lock
133 * the structure to stabilize it and check it's still at the given address,
134 * only if we can be sure that the memory has not been meanwhile reused
135 * for some other kind of object (which our subsystem's lock might corrupt).
136 *
137 * rcu_read_lock before reading the address, then rcu_read_unlock after
138 * taking the spinlock within the structure expected at that address.
139 *
140 * Note that it is not possible to acquire a lock within a structure
141 * allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference
142 * as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages
143 * are not zeroed before being given to the slab, which means that any
144 * locks must be initialized after each and every kmem_struct_alloc().
145 * Alternatively, make the ctor passed to kmem_cache_create() initialize
146 * the locks at page-allocation time, as is done in __i915_request_ctor(),
147 * sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers
148 * to safely acquire those ctor-initialized locks under rcu_read_lock()
149 * protection.
150 *
151 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
152 */
153 #define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU)
154 /* Trace allocations and frees */
155 #define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE)
156
157 /* Flag to prevent checks on free */
158 #ifdef CONFIG_DEBUG_OBJECTS
159 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS)
160 #else
161 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED
162 #endif
163
164 /* Avoid kmemleak tracing */
165 #define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE)
166
167 /*
168 * Prevent merging with compatible kmem caches. This flag should be used
169 * cautiously. Valid use cases:
170 *
171 * - caches created for self-tests (e.g. kunit)
172 * - general caches created and used by a subsystem, only when a
173 * (subsystem-specific) debug option is enabled
174 * - performance critical caches, should be very rare and consulted with slab
175 * maintainers, and not used together with CONFIG_SLUB_TINY
176 */
177 #define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE)
178
179 /* Fault injection mark */
180 #ifdef CONFIG_FAILSLAB
181 # define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB)
182 #else
183 # define SLAB_FAILSLAB __SLAB_FLAG_UNUSED
184 #endif
185 /**
186 * define SLAB_ACCOUNT - Account allocations to memcg.
187 *
188 * All object allocations from this cache will be memcg accounted, regardless of
189 * __GFP_ACCOUNT being or not being passed to individual allocations.
190 */
191 #ifdef CONFIG_MEMCG
192 # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
193 #else
194 # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
195 #endif
196
197 #ifdef CONFIG_KASAN_GENERIC
198 #define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN)
199 #else
200 #define SLAB_KASAN __SLAB_FLAG_UNUSED
201 #endif
202
203 /*
204 * Ignore user specified debugging flags.
205 * Intended for caches created for self-tests so they have only flags
206 * specified in the code and other flags are ignored.
207 */
208 #define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS)
209
210 #ifdef CONFIG_KFENCE
211 #define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE)
212 #else
213 #define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED
214 #endif
215
216 /* The following flags affect the page allocator grouping pages by mobility */
217 /**
218 * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable.
219 *
220 * Use this flag for caches that have an associated shrinker. As a result, slab
221 * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by
222 * mobility, and are accounted in SReclaimable counter in /proc/meminfo
223 */
224 #ifndef CONFIG_SLUB_TINY
225 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT)
226 #else
227 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED
228 #endif
229 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
230
231 /* Slab created using create_boot_cache */
232 #ifdef CONFIG_SLAB_OBJ_EXT
233 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT)
234 #else
235 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED
236 #endif
237
238 /*
239 * freeptr_t represents a SLUB freelist pointer, which might be encoded
240 * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
241 */
242 typedef struct { unsigned long v; } freeptr_t;
243
244 /*
245 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
246 *
247 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
248 *
249 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
250 * Both make kfree a no-op.
251 */
252 #define ZERO_SIZE_PTR ((void *)16)
253
254 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
255 (unsigned long)ZERO_SIZE_PTR)
256
257 #include <linux/kasan.h>
258
259 struct list_lru;
260 struct mem_cgroup;
261 /*
262 * struct kmem_cache related prototypes
263 */
264 bool slab_is_available(void);
265
266 /**
267 * struct kmem_cache_args - Less common arguments for kmem_cache_create()
268 *
269 * Any uninitialized fields of the structure are interpreted as unused. The
270 * exception is @freeptr_offset where %0 is a valid value, so
271 * @use_freeptr_offset must be also set to %true in order to interpret the field
272 * as used. For @useroffset %0 is also valid, but only with non-%0
273 * @usersize.
274 *
275 * When %NULL args is passed to kmem_cache_create(), it is equivalent to all
276 * fields unused.
277 */
278 struct kmem_cache_args {
279 /**
280 * @align: The required alignment for the objects.
281 *
282 * %0 means no specific alignment is requested.
283 */
284 unsigned int align;
285 /**
286 * @useroffset: Usercopy region offset.
287 *
288 * %0 is a valid offset, when @usersize is non-%0
289 */
290 unsigned int useroffset;
291 /**
292 * @usersize: Usercopy region size.
293 *
294 * %0 means no usercopy region is specified.
295 */
296 unsigned int usersize;
297 /**
298 * @freeptr_offset: Custom offset for the free pointer
299 * in &SLAB_TYPESAFE_BY_RCU caches
300 *
301 * By default &SLAB_TYPESAFE_BY_RCU caches place the free pointer
302 * outside of the object. This might cause the object to grow in size.
303 * Cache creators that have a reason to avoid this can specify a custom
304 * free pointer offset in their struct where the free pointer will be
305 * placed.
306 *
307 * Note that placing the free pointer inside the object requires the
308 * caller to ensure that no fields are invalidated that are required to
309 * guard against object recycling (See &SLAB_TYPESAFE_BY_RCU for
310 * details).
311 *
312 * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
313 * is specified, %use_freeptr_offset must be set %true.
314 *
315 * Note that @ctor currently isn't supported with custom free pointers
316 * as a @ctor requires an external free pointer.
317 */
318 unsigned int freeptr_offset;
319 /**
320 * @use_freeptr_offset: Whether a @freeptr_offset is used.
321 */
322 bool use_freeptr_offset;
323 /**
324 * @ctor: A constructor for the objects.
325 *
326 * The constructor is invoked for each object in a newly allocated slab
327 * page. It is the cache user's responsibility to free object in the
328 * same state as after calling the constructor, or deal appropriately
329 * with any differences between a freshly constructed and a reallocated
330 * object.
331 *
332 * %NULL means no constructor.
333 */
334 void (*ctor)(void *);
335 };
336
337 struct kmem_cache *__kmem_cache_create_args(const char *name,
338 unsigned int object_size,
339 struct kmem_cache_args *args,
340 slab_flags_t flags);
341 static inline struct kmem_cache *
__kmem_cache_create(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,void (* ctor)(void *))342 __kmem_cache_create(const char *name, unsigned int size, unsigned int align,
343 slab_flags_t flags, void (*ctor)(void *))
344 {
345 struct kmem_cache_args kmem_args = {
346 .align = align,
347 .ctor = ctor,
348 };
349
350 return __kmem_cache_create_args(name, size, &kmem_args, flags);
351 }
352
353 /**
354 * kmem_cache_create_usercopy - Create a kmem cache with a region suitable
355 * for copying to userspace.
356 * @name: A string which is used in /proc/slabinfo to identify this cache.
357 * @size: The size of objects to be created in this cache.
358 * @align: The required alignment for the objects.
359 * @flags: SLAB flags
360 * @useroffset: Usercopy region offset
361 * @usersize: Usercopy region size
362 * @ctor: A constructor for the objects, or %NULL.
363 *
364 * This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY()
365 * if whitelisting a single field is sufficient, or kmem_cache_create() with
366 * the necessary parameters passed via the args parameter (see
367 * &struct kmem_cache_args)
368 *
369 * Return: a pointer to the cache on success, NULL on failure.
370 */
371 static inline struct kmem_cache *
kmem_cache_create_usercopy(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,unsigned int useroffset,unsigned int usersize,void (* ctor)(void *))372 kmem_cache_create_usercopy(const char *name, unsigned int size,
373 unsigned int align, slab_flags_t flags,
374 unsigned int useroffset, unsigned int usersize,
375 void (*ctor)(void *))
376 {
377 struct kmem_cache_args kmem_args = {
378 .align = align,
379 .ctor = ctor,
380 .useroffset = useroffset,
381 .usersize = usersize,
382 };
383
384 return __kmem_cache_create_args(name, size, &kmem_args, flags);
385 }
386
387 /* If NULL is passed for @args, use this variant with default arguments. */
388 static inline struct kmem_cache *
__kmem_cache_default_args(const char * name,unsigned int size,struct kmem_cache_args * args,slab_flags_t flags)389 __kmem_cache_default_args(const char *name, unsigned int size,
390 struct kmem_cache_args *args,
391 slab_flags_t flags)
392 {
393 struct kmem_cache_args kmem_default_args = {};
394
395 /* Make sure we don't get passed garbage. */
396 if (WARN_ON_ONCE(args))
397 return ERR_PTR(-EINVAL);
398
399 return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
400 }
401
402 /**
403 * kmem_cache_create - Create a kmem cache.
404 * @__name: A string which is used in /proc/slabinfo to identify this cache.
405 * @__object_size: The size of objects to be created in this cache.
406 * @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL
407 * means defaults will be used for all the arguments.
408 *
409 * This is currently implemented as a macro using ``_Generic()`` to call
410 * either the new variant of the function, or a legacy one.
411 *
412 * The new variant has 4 parameters:
413 * ``kmem_cache_create(name, object_size, args, flags)``
414 *
415 * See __kmem_cache_create_args() which implements this.
416 *
417 * The legacy variant has 5 parameters:
418 * ``kmem_cache_create(name, object_size, align, flags, ctor)``
419 *
420 * The align and ctor parameters map to the respective fields of
421 * &struct kmem_cache_args
422 *
423 * Context: Cannot be called within a interrupt, but can be interrupted.
424 *
425 * Return: a pointer to the cache on success, NULL on failure.
426 */
427 #define kmem_cache_create(__name, __object_size, __args, ...) \
428 _Generic((__args), \
429 struct kmem_cache_args *: __kmem_cache_create_args, \
430 void *: __kmem_cache_default_args, \
431 default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
432
433 void kmem_cache_destroy(struct kmem_cache *s);
434 int kmem_cache_shrink(struct kmem_cache *s);
435
436 /*
437 * Please use this macro to create slab caches. Simply specify the
438 * name of the structure and maybe some flags that are listed above.
439 *
440 * The alignment of the struct determines object alignment. If you
441 * f.e. add ____cacheline_aligned_in_smp to the struct declaration
442 * then the objects will be properly aligned in SMP configurations.
443 */
444 #define KMEM_CACHE(__struct, __flags) \
445 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
446 &(struct kmem_cache_args) { \
447 .align = __alignof__(struct __struct), \
448 }, (__flags))
449
450 /*
451 * To whitelist a single field for copying to/from usercopy, use this
452 * macro instead for KMEM_CACHE() above.
453 */
454 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \
455 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
456 &(struct kmem_cache_args) { \
457 .align = __alignof__(struct __struct), \
458 .useroffset = offsetof(struct __struct, __field), \
459 .usersize = sizeof_field(struct __struct, __field), \
460 }, (__flags))
461
462 /*
463 * Common kmalloc functions provided by all allocators
464 */
465 void * __must_check krealloc_noprof(const void *objp, size_t new_size,
466 gfp_t flags) __realloc_size(2);
467 #define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__))
468
469 void kfree(const void *objp);
470 void kfree_sensitive(const void *objp);
471 size_t __ksize(const void *objp);
472
473 DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T))
474 DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T))
475
476 /**
477 * ksize - Report actual allocation size of associated object
478 *
479 * @objp: Pointer returned from a prior kmalloc()-family allocation.
480 *
481 * This should not be used for writing beyond the originally requested
482 * allocation size. Either use krealloc() or round up the allocation size
483 * with kmalloc_size_roundup() prior to allocation. If this is used to
484 * access beyond the originally requested allocation size, UBSAN_BOUNDS
485 * and/or FORTIFY_SOURCE may trip, since they only know about the
486 * originally allocated size via the __alloc_size attribute.
487 */
488 size_t ksize(const void *objp);
489
490 #ifdef CONFIG_PRINTK
491 bool kmem_dump_obj(void *object);
492 #else
kmem_dump_obj(void * object)493 static inline bool kmem_dump_obj(void *object) { return false; }
494 #endif
495
496 /*
497 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
498 * alignment larger than the alignment of a 64-bit integer.
499 * Setting ARCH_DMA_MINALIGN in arch headers allows that.
500 */
501 #ifdef ARCH_HAS_DMA_MINALIGN
502 #if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN)
503 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
504 #endif
505 #endif
506
507 #ifndef ARCH_KMALLOC_MINALIGN
508 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
509 #elif ARCH_KMALLOC_MINALIGN > 8
510 #define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
511 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
512 #endif
513
514 /*
515 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
516 * Intended for arches that get misalignment faults even for 64 bit integer
517 * aligned buffers.
518 */
519 #ifndef ARCH_SLAB_MINALIGN
520 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
521 #endif
522
523 /*
524 * Arches can define this function if they want to decide the minimum slab
525 * alignment at runtime. The value returned by the function must be a power
526 * of two and >= ARCH_SLAB_MINALIGN.
527 */
528 #ifndef arch_slab_minalign
arch_slab_minalign(void)529 static inline unsigned int arch_slab_minalign(void)
530 {
531 return ARCH_SLAB_MINALIGN;
532 }
533 #endif
534
535 /*
536 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN.
537 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN
538 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment.
539 */
540 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
541 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
542 #define __assume_page_alignment __assume_aligned(PAGE_SIZE)
543
544 /*
545 * Kmalloc array related definitions
546 */
547
548 /*
549 * SLUB directly allocates requests fitting in to an order-1 page
550 * (PAGE_SIZE*2). Larger requests are passed to the page allocator.
551 */
552 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
553 #define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT)
554 #ifndef KMALLOC_SHIFT_LOW
555 #define KMALLOC_SHIFT_LOW 3
556 #endif
557
558 /* Maximum allocatable size */
559 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
560 /* Maximum size for which we actually use a slab cache */
561 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
562 /* Maximum order allocatable via the slab allocator */
563 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
564
565 /*
566 * Kmalloc subsystem.
567 */
568 #ifndef KMALLOC_MIN_SIZE
569 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
570 #endif
571
572 /*
573 * This restriction comes from byte sized index implementation.
574 * Page size is normally 2^12 bytes and, in this case, if we want to use
575 * byte sized index which can represent 2^8 entries, the size of the object
576 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
577 * If minimum size of kmalloc is less than 16, we use it as minimum object
578 * size and give up to use byte sized index.
579 */
580 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \
581 (KMALLOC_MIN_SIZE) : 16)
582
583 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
584 #define RANDOM_KMALLOC_CACHES_NR 15 // # of cache copies
585 #else
586 #define RANDOM_KMALLOC_CACHES_NR 0
587 #endif
588
589 /*
590 * Whenever changing this, take care of that kmalloc_type() and
591 * create_kmalloc_caches() still work as intended.
592 *
593 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
594 * is for accounted but unreclaimable and non-dma objects. All the other
595 * kmem caches can have both accounted and unaccounted objects.
596 */
597 enum kmalloc_cache_type {
598 KMALLOC_NORMAL = 0,
599 #ifndef CONFIG_ZONE_DMA
600 KMALLOC_DMA = KMALLOC_NORMAL,
601 #endif
602 #ifndef CONFIG_MEMCG
603 KMALLOC_CGROUP = KMALLOC_NORMAL,
604 #endif
605 KMALLOC_RANDOM_START = KMALLOC_NORMAL,
606 KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
607 #ifdef CONFIG_SLUB_TINY
608 KMALLOC_RECLAIM = KMALLOC_NORMAL,
609 #else
610 KMALLOC_RECLAIM,
611 #endif
612 #ifdef CONFIG_ZONE_DMA
613 KMALLOC_DMA,
614 #endif
615 #ifdef CONFIG_MEMCG
616 KMALLOC_CGROUP,
617 #endif
618 NR_KMALLOC_TYPES
619 };
620
621 typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1];
622
623 extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES];
624
625 /*
626 * Define gfp bits that should not be set for KMALLOC_NORMAL.
627 */
628 #define KMALLOC_NOT_NORMAL_BITS \
629 (__GFP_RECLAIMABLE | \
630 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \
631 (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0))
632
633 extern unsigned long random_kmalloc_seed;
634
kmalloc_type(gfp_t flags,unsigned long caller)635 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, unsigned long caller)
636 {
637 /*
638 * The most common case is KMALLOC_NORMAL, so test for it
639 * with a single branch for all the relevant flags.
640 */
641 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
642 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
643 /* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
644 return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed,
645 ilog2(RANDOM_KMALLOC_CACHES_NR + 1));
646 #else
647 return KMALLOC_NORMAL;
648 #endif
649
650 /*
651 * At least one of the flags has to be set. Their priorities in
652 * decreasing order are:
653 * 1) __GFP_DMA
654 * 2) __GFP_RECLAIMABLE
655 * 3) __GFP_ACCOUNT
656 */
657 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA))
658 return KMALLOC_DMA;
659 if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE))
660 return KMALLOC_RECLAIM;
661 else
662 return KMALLOC_CGROUP;
663 }
664
665 /*
666 * Figure out which kmalloc slab an allocation of a certain size
667 * belongs to.
668 * 0 = zero alloc
669 * 1 = 65 .. 96 bytes
670 * 2 = 129 .. 192 bytes
671 * n = 2^(n-1)+1 .. 2^n
672 *
673 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
674 * typical usage is via kmalloc_index() and therefore evaluated at compile-time.
675 * Callers where !size_is_constant should only be test modules, where runtime
676 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab().
677 */
__kmalloc_index(size_t size,bool size_is_constant)678 static __always_inline unsigned int __kmalloc_index(size_t size,
679 bool size_is_constant)
680 {
681 if (!size)
682 return 0;
683
684 if (size <= KMALLOC_MIN_SIZE)
685 return KMALLOC_SHIFT_LOW;
686
687 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
688 return 1;
689 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
690 return 2;
691 if (size <= 8) return 3;
692 if (size <= 16) return 4;
693 if (size <= 32) return 5;
694 if (size <= 64) return 6;
695 if (size <= 128) return 7;
696 if (size <= 256) return 8;
697 if (size <= 512) return 9;
698 if (size <= 1024) return 10;
699 if (size <= 2 * 1024) return 11;
700 if (size <= 4 * 1024) return 12;
701 if (size <= 8 * 1024) return 13;
702 if (size <= 16 * 1024) return 14;
703 if (size <= 32 * 1024) return 15;
704 if (size <= 64 * 1024) return 16;
705 if (size <= 128 * 1024) return 17;
706 if (size <= 256 * 1024) return 18;
707 if (size <= 512 * 1024) return 19;
708 if (size <= 1024 * 1024) return 20;
709 if (size <= 2 * 1024 * 1024) return 21;
710
711 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
712 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
713 else
714 BUG();
715
716 /* Will never be reached. Needed because the compiler may complain */
717 return -1;
718 }
719 static_assert(PAGE_SHIFT <= 20);
720 #define kmalloc_index(s) __kmalloc_index(s, true)
721
722 #include <linux/alloc_tag.h>
723
724 /**
725 * kmem_cache_alloc - Allocate an object
726 * @cachep: The cache to allocate from.
727 * @flags: See kmalloc().
728 *
729 * Allocate an object from this cache.
730 * See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags.
731 *
732 * Return: pointer to the new object or %NULL in case of error
733 */
734 void *kmem_cache_alloc_noprof(struct kmem_cache *cachep,
735 gfp_t flags) __assume_slab_alignment __malloc;
736 #define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__))
737
738 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
739 gfp_t gfpflags) __assume_slab_alignment __malloc;
740 #define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__))
741
742 /**
743 * kmem_cache_charge - memcg charge an already allocated slab memory
744 * @objp: address of the slab object to memcg charge
745 * @gfpflags: describe the allocation context
746 *
747 * kmem_cache_charge allows charging a slab object to the current memcg,
748 * primarily in cases where charging at allocation time might not be possible
749 * because the target memcg is not known (i.e. softirq context)
750 *
751 * The objp should be pointer returned by the slab allocator functions like
752 * kmalloc (with __GFP_ACCOUNT in flags) or kmem_cache_alloc. The memcg charge
753 * behavior can be controlled through gfpflags parameter, which affects how the
754 * necessary internal metadata can be allocated. Including __GFP_NOFAIL denotes
755 * that overcharging is requested instead of failure, but is not applied for the
756 * internal metadata allocation.
757 *
758 * There are several cases where it will return true even if the charging was
759 * not done:
760 * More specifically:
761 *
762 * 1. For !CONFIG_MEMCG or cgroup_disable=memory systems.
763 * 2. Already charged slab objects.
764 * 3. For slab objects from KMALLOC_NORMAL caches - allocated by kmalloc()
765 * without __GFP_ACCOUNT
766 * 4. Allocating internal metadata has failed
767 *
768 * Return: true if charge was successful otherwise false.
769 */
770 bool kmem_cache_charge(void *objp, gfp_t gfpflags);
771 void kmem_cache_free(struct kmem_cache *s, void *objp);
772
773 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
774 unsigned int useroffset, unsigned int usersize,
775 void (*ctor)(void *));
776
777 /*
778 * Bulk allocation and freeing operations. These are accelerated in an
779 * allocator specific way to avoid taking locks repeatedly or building
780 * metadata structures unnecessarily.
781 *
782 * Note that interrupts must be enabled when calling these functions.
783 */
784 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
785
786 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
787 #define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
788
kfree_bulk(size_t size,void ** p)789 static __always_inline void kfree_bulk(size_t size, void **p)
790 {
791 kmem_cache_free_bulk(NULL, size, p);
792 }
793
794 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags,
795 int node) __assume_slab_alignment __malloc;
796 #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))
797
798 /*
799 * These macros allow declaring a kmem_buckets * parameter alongside size, which
800 * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call
801 * sites don't have to pass NULL.
802 */
803 #ifdef CONFIG_SLAB_BUCKETS
804 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b)
805 #define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b)
806 #define PASS_BUCKET_PARAM(_b) (_b)
807 #else
808 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size)
809 #define PASS_BUCKET_PARAMS(_size, _b) (_size)
810 #define PASS_BUCKET_PARAM(_b) NULL
811 #endif
812
813 /*
814 * The following functions are not to be used directly and are intended only
815 * for internal use from kmalloc() and kmalloc_node()
816 * with the exception of kunit tests
817 */
818
819 void *__kmalloc_noprof(size_t size, gfp_t flags)
820 __assume_kmalloc_alignment __alloc_size(1);
821
822 void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
823 __assume_kmalloc_alignment __alloc_size(1);
824
825 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
826 __assume_kmalloc_alignment __alloc_size(3);
827
828 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
829 int node, size_t size)
830 __assume_kmalloc_alignment __alloc_size(4);
831
832 void *__kmalloc_large_noprof(size_t size, gfp_t flags)
833 __assume_page_alignment __alloc_size(1);
834
835 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
836 __assume_page_alignment __alloc_size(1);
837
838 /**
839 * kmalloc - allocate kernel memory
840 * @size: how many bytes of memory are required.
841 * @flags: describe the allocation context
842 *
843 * kmalloc is the normal method of allocating memory
844 * for objects smaller than page size in the kernel.
845 *
846 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
847 * bytes. For @size of power of two bytes, the alignment is also guaranteed
848 * to be at least to the size. For other sizes, the alignment is guaranteed to
849 * be at least the largest power-of-two divisor of @size.
850 *
851 * The @flags argument may be one of the GFP flags defined at
852 * include/linux/gfp_types.h and described at
853 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
854 *
855 * The recommended usage of the @flags is described at
856 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
857 *
858 * Below is a brief outline of the most useful GFP flags
859 *
860 * %GFP_KERNEL
861 * Allocate normal kernel ram. May sleep.
862 *
863 * %GFP_NOWAIT
864 * Allocation will not sleep.
865 *
866 * %GFP_ATOMIC
867 * Allocation will not sleep. May use emergency pools.
868 *
869 * Also it is possible to set different flags by OR'ing
870 * in one or more of the following additional @flags:
871 *
872 * %__GFP_ZERO
873 * Zero the allocated memory before returning. Also see kzalloc().
874 *
875 * %__GFP_HIGH
876 * This allocation has high priority and may use emergency pools.
877 *
878 * %__GFP_NOFAIL
879 * Indicate that this allocation is in no way allowed to fail
880 * (think twice before using).
881 *
882 * %__GFP_NORETRY
883 * If memory is not immediately available,
884 * then give up at once.
885 *
886 * %__GFP_NOWARN
887 * If allocation fails, don't issue any warnings.
888 *
889 * %__GFP_RETRY_MAYFAIL
890 * Try really hard to succeed the allocation but fail
891 * eventually.
892 */
kmalloc_noprof(size_t size,gfp_t flags)893 static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags)
894 {
895 if (__builtin_constant_p(size) && size) {
896 unsigned int index;
897
898 if (size > KMALLOC_MAX_CACHE_SIZE)
899 return __kmalloc_large_noprof(size, flags);
900
901 index = kmalloc_index(size);
902 return __kmalloc_cache_noprof(
903 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
904 flags, size);
905 }
906 return __kmalloc_noprof(size, flags);
907 }
908 #define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__))
909
910 #define kmem_buckets_alloc(_b, _size, _flags) \
911 alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))
912
913 #define kmem_buckets_alloc_track_caller(_b, _size, _flags) \
914 alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_))
915
kmalloc_node_noprof(size_t size,gfp_t flags,int node)916 static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)
917 {
918 if (__builtin_constant_p(size) && size) {
919 unsigned int index;
920
921 if (size > KMALLOC_MAX_CACHE_SIZE)
922 return __kmalloc_large_node_noprof(size, flags, node);
923
924 index = kmalloc_index(size);
925 return __kmalloc_cache_node_noprof(
926 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
927 flags, node, size);
928 }
929 return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node);
930 }
931 #define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
932
933 /**
934 * kmalloc_array - allocate memory for an array.
935 * @n: number of elements.
936 * @size: element size.
937 * @flags: the type of memory to allocate (see kmalloc).
938 */
kmalloc_array_noprof(size_t n,size_t size,gfp_t flags)939 static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
940 {
941 size_t bytes;
942
943 if (unlikely(check_mul_overflow(n, size, &bytes)))
944 return NULL;
945 return kmalloc_noprof(bytes, flags);
946 }
947 #define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
948
949 /**
950 * krealloc_array - reallocate memory for an array.
951 * @p: pointer to the memory chunk to reallocate
952 * @new_n: new number of elements to alloc
953 * @new_size: new size of a single member of the array
954 * @flags: the type of memory to allocate (see kmalloc)
955 *
956 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
957 * initial memory allocation, every subsequent call to this API for the same
958 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
959 * __GFP_ZERO is not fully honored by this API.
960 *
961 * See krealloc_noprof() for further details.
962 *
963 * In any case, the contents of the object pointed to are preserved up to the
964 * lesser of the new and old sizes.
965 */
krealloc_array_noprof(void * p,size_t new_n,size_t new_size,gfp_t flags)966 static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p,
967 size_t new_n,
968 size_t new_size,
969 gfp_t flags)
970 {
971 size_t bytes;
972
973 if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
974 return NULL;
975
976 return krealloc_noprof(p, bytes, flags);
977 }
978 #define krealloc_array(...) alloc_hooks(krealloc_array_noprof(__VA_ARGS__))
979
980 /**
981 * kcalloc - allocate memory for an array. The memory is set to zero.
982 * @n: number of elements.
983 * @size: element size.
984 * @flags: the type of memory to allocate (see kmalloc).
985 */
986 #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
987
988 void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node,
989 unsigned long caller) __alloc_size(1);
990 #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
991 __kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller)
992 #define kmalloc_node_track_caller(...) \
993 alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
994
995 /*
996 * kmalloc_track_caller is a special version of kmalloc that records the
997 * calling function of the routine calling it for slab leak tracking instead
998 * of just the calling function (confusing, eh?).
999 * It's useful when the call to kmalloc comes from a widely-used standard
1000 * allocator where we care about the real place the memory allocation
1001 * request comes from.
1002 */
1003 #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE)
1004
1005 #define kmalloc_track_caller_noprof(...) \
1006 kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_)
1007
kmalloc_array_node_noprof(size_t n,size_t size,gfp_t flags,int node)1008 static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
1009 int node)
1010 {
1011 size_t bytes;
1012
1013 if (unlikely(check_mul_overflow(n, size, &bytes)))
1014 return NULL;
1015 if (__builtin_constant_p(n) && __builtin_constant_p(size))
1016 return kmalloc_node_noprof(bytes, flags, node);
1017 return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node);
1018 }
1019 #define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
1020
1021 #define kcalloc_node(_n, _size, _flags, _node) \
1022 kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node)
1023
1024 /*
1025 * Shortcuts
1026 */
1027 #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)
1028
1029 /**
1030 * kzalloc - allocate memory. The memory is set to zero.
1031 * @size: how many bytes of memory are required.
1032 * @flags: the type of memory to allocate (see kmalloc).
1033 */
kzalloc_noprof(size_t size,gfp_t flags)1034 static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
1035 {
1036 return kmalloc_noprof(size, flags | __GFP_ZERO);
1037 }
1038 #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
1039 #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1040
1041 void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) __alloc_size(1);
1042 #define kvmalloc_node_noprof(size, flags, node) \
1043 __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node)
1044 #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))
1045
1046 #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE)
1047 #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE)
1048 #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO)
1049
1050 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1051 #define kmem_buckets_valloc(_b, _size, _flags) \
1052 alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))
1053
1054 static inline __alloc_size(1, 2) void *
kvmalloc_array_node_noprof(size_t n,size_t size,gfp_t flags,int node)1055 kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
1056 {
1057 size_t bytes;
1058
1059 if (unlikely(check_mul_overflow(n, size, &bytes)))
1060 return NULL;
1061
1062 return kvmalloc_node_noprof(bytes, flags, node);
1063 }
1064
1065 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1066 #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node)
1067 #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1068
1069 #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))
1070 #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))
1071 #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__))
1072
1073 void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)
1074 __realloc_size(2);
1075 #define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__))
1076
1077 extern void kvfree(const void *addr);
1078 DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
1079
1080 extern void kvfree_sensitive(const void *addr, size_t len);
1081
1082 unsigned int kmem_cache_size(struct kmem_cache *s);
1083
1084 #ifndef CONFIG_KVFREE_RCU_BATCHED
kvfree_rcu_barrier(void)1085 static inline void kvfree_rcu_barrier(void)
1086 {
1087 rcu_barrier();
1088 }
1089
kfree_rcu_scheduler_running(void)1090 static inline void kfree_rcu_scheduler_running(void) { }
1091 #else
1092 void kvfree_rcu_barrier(void);
1093
1094 void kfree_rcu_scheduler_running(void);
1095 #endif
1096
1097 /**
1098 * kmalloc_size_roundup - Report allocation bucket size for the given size
1099 *
1100 * @size: Number of bytes to round up from.
1101 *
1102 * This returns the number of bytes that would be available in a kmalloc()
1103 * allocation of @size bytes. For example, a 126 byte request would be
1104 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly
1105 * for the general-purpose kmalloc()-based allocations, and is not for the
1106 * pre-sized kmem_cache_alloc()-based allocations.)
1107 *
1108 * Use this to kmalloc() the full bucket size ahead of time instead of using
1109 * ksize() to query the size after an allocation.
1110 */
1111 size_t kmalloc_size_roundup(size_t size);
1112
1113 void __init kmem_cache_init_late(void);
1114 void __init kvfree_rcu_init(void);
1115
1116 #endif /* _LINUX_SLAB_H */
1117