1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk).
4 *
5 * (C) SGI 2006, Christoph Lameter
6 * Cleaned up and restructured to ease the addition of alternative
7 * implementations of SLAB allocators.
8 * (C) Linux Foundation 2008-2013
9 * Unified interface for all slab allocators
10 */
11
12 #ifndef _LINUX_SLAB_H
13 #define _LINUX_SLAB_H
14
15 #include <linux/bug.h>
16 #include <linux/cache.h>
17 #include <linux/gfp.h>
18 #include <linux/overflow.h>
19 #include <linux/types.h>
20 #include <linux/rcupdate.h>
21 #include <linux/workqueue.h>
22 #include <linux/percpu-refcount.h>
23 #include <linux/cleanup.h>
24 #include <linux/hash.h>
25
26 enum _slab_flag_bits {
27 _SLAB_CONSISTENCY_CHECKS,
28 _SLAB_RED_ZONE,
29 _SLAB_POISON,
30 _SLAB_KMALLOC,
31 _SLAB_HWCACHE_ALIGN,
32 _SLAB_CACHE_DMA,
33 _SLAB_CACHE_DMA32,
34 _SLAB_STORE_USER,
35 _SLAB_PANIC,
36 _SLAB_TYPESAFE_BY_RCU,
37 _SLAB_TRACE,
38 #ifdef CONFIG_DEBUG_OBJECTS
39 _SLAB_DEBUG_OBJECTS,
40 #endif
41 _SLAB_NOLEAKTRACE,
42 _SLAB_NO_MERGE,
43 #ifdef CONFIG_FAILSLAB
44 _SLAB_FAILSLAB,
45 #endif
46 #ifdef CONFIG_MEMCG
47 _SLAB_ACCOUNT,
48 _SLAB_MAY_ACCOUNT,
49 #endif
50 #ifdef CONFIG_KASAN_GENERIC
51 _SLAB_KASAN,
52 #endif
53 _SLAB_NO_USER_FLAGS,
54 #ifdef CONFIG_KFENCE
55 _SLAB_SKIP_KFENCE,
56 #endif
57 #ifndef CONFIG_SLUB_TINY
58 _SLAB_RECLAIM_ACCOUNT,
59 #endif
60 _SLAB_OBJECT_POISON,
61 _SLAB_CMPXCHG_DOUBLE,
62 #ifdef CONFIG_SLAB_OBJ_EXT
63 _SLAB_NO_OBJ_EXT,
64 #ifdef CONFIG_64BIT
65 _SLAB_OBJ_EXT_IN_OBJ,
66 #endif
67 #endif
68 _SLAB_NO_SHEAVES,
69 _SLAB_FLAGS_LAST_BIT
70 };
71
72 #define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr)))
73 #define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U))
74
75 /*
76 * Flags to pass to kmem_cache_create().
77 * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op
78 */
79 /* DEBUG: Perform (expensive) checks on alloc/free */
80 #define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS)
81 /* DEBUG: Red zone objs in a cache */
82 #define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE)
83 /* DEBUG: Poison objects */
84 #define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON)
85 /* Indicate a kmalloc slab */
86 #define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC)
87 /**
88 * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries.
89 *
90 * Sufficiently large objects are aligned on cache line boundary. For object
91 * size smaller than a half of cache line size, the alignment is on the half of
92 * cache line size. In general, if object size is smaller than 1/2^n of cache
93 * line size, the alignment is adjusted to 1/2^n.
94 *
95 * If explicit alignment is also requested by the respective
96 * &struct kmem_cache_args field, the greater of both is alignments is applied.
97 */
98 #define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN)
99 /* Use GFP_DMA memory */
100 #define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA)
101 /* Use GFP_DMA32 memory */
102 #define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32)
103 /* DEBUG: Store the last owner for bug hunting */
104 #define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER)
105 /* Panic if kmem_cache_create() fails */
106 #define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC)
107 /**
108 * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
109 *
110 * This delays freeing the SLAB page by a grace period, it does _NOT_
111 * delay object freeing. This means that if you do kmem_cache_free()
112 * that memory location is free to be reused at any time. Thus it may
113 * be possible to see another object there in the same RCU grace period.
114 *
115 * This feature only ensures the memory location backing the object
116 * stays valid, the trick to using this is relying on an independent
117 * object validation pass. Something like:
118 *
119 * ::
120 *
121 * begin:
122 * rcu_read_lock();
123 * obj = lockless_lookup(key);
124 * if (obj) {
125 * if (!try_get_ref(obj)) // might fail for free objects
126 * rcu_read_unlock();
127 * goto begin;
128 *
129 * if (obj->key != key) { // not the object we expected
130 * put_ref(obj);
131 * rcu_read_unlock();
132 * goto begin;
133 * }
134 * }
135 * rcu_read_unlock();
136 *
137 * This is useful if we need to approach a kernel structure obliquely,
138 * from its address obtained without the usual locking. We can lock
139 * the structure to stabilize it and check it's still at the given address,
140 * only if we can be sure that the memory has not been meanwhile reused
141 * for some other kind of object (which our subsystem's lock might corrupt).
142 *
143 * rcu_read_lock before reading the address, then rcu_read_unlock after
144 * taking the spinlock within the structure expected at that address.
145 *
146 * Note that object identity check has to be done *after* acquiring a
147 * reference, therefore user has to ensure proper ordering for loads.
148 * Similarly, when initializing objects allocated with SLAB_TYPESAFE_BY_RCU,
149 * the newly allocated object has to be fully initialized *before* its
150 * refcount gets initialized and proper ordering for stores is required.
151 * refcount_{add|inc}_not_zero_acquire() and refcount_set_release() are
152 * designed with the proper fences required for reference counting objects
153 * allocated with SLAB_TYPESAFE_BY_RCU.
154 *
155 * Note that it is not possible to acquire a lock within a structure
156 * allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference
157 * as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages
158 * are not zeroed before being given to the slab, which means that any
159 * locks must be initialized after each and every kmem_struct_alloc().
160 * Alternatively, make the ctor passed to kmem_cache_create() initialize
161 * the locks at page-allocation time, as is done in __i915_request_ctor(),
162 * sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers
163 * to safely acquire those ctor-initialized locks under rcu_read_lock()
164 * protection.
165 *
166 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
167 */
168 #define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU)
169 /* Trace allocations and frees */
170 #define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE)
171
172 /* Flag to prevent checks on free */
173 #ifdef CONFIG_DEBUG_OBJECTS
174 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS)
175 #else
176 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED
177 #endif
178
179 /* Avoid kmemleak tracing */
180 #define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE)
181
182 /*
183 * Prevent merging with compatible kmem caches. This flag should be used
184 * cautiously. Valid use cases:
185 *
186 * - caches created for self-tests (e.g. kunit)
187 * - general caches created and used by a subsystem, only when a
188 * (subsystem-specific) debug option is enabled
189 * - performance critical caches, should be very rare and consulted with slab
190 * maintainers, and not used together with CONFIG_SLUB_TINY
191 */
192 #define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE)
193
194 /* Fault injection mark */
195 #ifdef CONFIG_FAILSLAB
196 # define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB)
197 #else
198 # define SLAB_FAILSLAB __SLAB_FLAG_UNUSED
199 #endif
200 /**
201 * define SLAB_ACCOUNT - Account allocations to memcg.
202 *
203 * All object allocations from this cache will be memcg accounted, regardless of
204 * __GFP_ACCOUNT being or not being passed to individual allocations.
205 */
206 #ifdef CONFIG_MEMCG
207 # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
208 # define SLAB_MAY_ACCOUNT __SLAB_FLAG_BIT(_SLAB_MAY_ACCOUNT)
209 #else
210 # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
211 # define SLAB_MAY_ACCOUNT __SLAB_FLAG_UNUSED
212 #endif
213
214 #ifdef CONFIG_KASAN_GENERIC
215 #define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN)
216 #else
217 #define SLAB_KASAN __SLAB_FLAG_UNUSED
218 #endif
219
220 /*
221 * Ignore user specified debugging flags.
222 * Intended for caches created for self-tests so they have only flags
223 * specified in the code and other flags are ignored.
224 */
225 #define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS)
226
227 #ifdef CONFIG_KFENCE
228 #define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE)
229 #else
230 #define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED
231 #endif
232
233 /* The following flags affect the page allocator grouping pages by mobility */
234 /**
235 * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable.
236 *
237 * Use this flag for caches that have an associated shrinker. As a result, slab
238 * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by
239 * mobility, and are accounted in SReclaimable counter in /proc/meminfo
240 */
241 #ifndef CONFIG_SLUB_TINY
242 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT)
243 #else
244 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED
245 #endif
246 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
247
248 /* Slab caches without obj_exts array */
249 #ifdef CONFIG_SLAB_OBJ_EXT
250 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT)
251 #else
252 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED
253 #endif
254
255 #define SLAB_NO_SHEAVES __SLAB_FLAG_BIT(_SLAB_NO_SHEAVES)
256
257 #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
258 #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ)
259 #else
260 #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_UNUSED
261 #endif
262
263 /*
264 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
265 *
266 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
267 *
268 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
269 * Both make kfree a no-op.
270 */
271 #define ZERO_SIZE_PTR ((void *)16)
272
273 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
274 (unsigned long)ZERO_SIZE_PTR)
275
276 #include <linux/kasan.h>
277
278 struct list_lru;
279 struct mem_cgroup;
280 /*
281 * struct kmem_cache related prototypes
282 */
283 bool slab_is_available(void);
284
285 /**
286 * struct kmem_cache_args - Less common arguments for kmem_cache_create()
287 *
288 * Any uninitialized fields of the structure are interpreted as unused. The
289 * exception is @freeptr_offset where %0 is a valid value, so
290 * @use_freeptr_offset must be also set to %true in order to interpret the field
291 * as used. For @useroffset %0 is also valid, but only with non-%0
292 * @usersize.
293 *
294 * When %NULL args is passed to kmem_cache_create(), it is equivalent to all
295 * fields unused.
296 */
297 struct kmem_cache_args {
298 /**
299 * @align: The required alignment for the objects.
300 *
301 * %0 means no specific alignment is requested.
302 */
303 unsigned int align;
304 /**
305 * @useroffset: Usercopy region offset.
306 *
307 * %0 is a valid offset, when @usersize is non-%0
308 */
309 unsigned int useroffset;
310 /**
311 * @usersize: Usercopy region size.
312 *
313 * %0 means no usercopy region is specified.
314 */
315 unsigned int usersize;
316 /**
317 * @freeptr_offset: Custom offset for the free pointer
318 * in caches with &SLAB_TYPESAFE_BY_RCU or @ctor
319 *
320 * By default, &SLAB_TYPESAFE_BY_RCU and @ctor caches place the free
321 * pointer outside of the object. This might cause the object to grow
322 * in size. Cache creators that have a reason to avoid this can specify
323 * a custom free pointer offset in their data structure where the free
324 * pointer will be placed.
325 *
326 * For caches with &SLAB_TYPESAFE_BY_RCU, the caller must ensure that
327 * the free pointer does not overlay fields required to guard against
328 * object recycling (See &SLAB_TYPESAFE_BY_RCU for details).
329 *
330 * For caches with @ctor, the caller must ensure that the free pointer
331 * does not overlay fields initialized by the constructor.
332 *
333 * Currently, only caches with &SLAB_TYPESAFE_BY_RCU or @ctor
334 * may specify @freeptr_offset.
335 *
336 * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
337 * is specified, @use_freeptr_offset must be set %true.
338 */
339 unsigned int freeptr_offset;
340 /**
341 * @use_freeptr_offset: Whether a @freeptr_offset is used.
342 */
343 bool use_freeptr_offset;
344 /**
345 * @ctor: A constructor for the objects.
346 *
347 * The constructor is invoked for each object in a newly allocated slab
348 * page. It is the cache user's responsibility to free object in the
349 * same state as after calling the constructor, or deal appropriately
350 * with any differences between a freshly constructed and a reallocated
351 * object.
352 *
353 * %NULL means no constructor.
354 */
355 void (*ctor)(void *);
356 /**
357 * @sheaf_capacity: Enable sheaves of given capacity for the cache.
358 *
359 * With a non-zero value, allocations from the cache go through caching
360 * arrays called sheaves. Each cpu has a main sheaf that's always
361 * present, and a spare sheaf that may be not present. When both become
362 * empty, there's an attempt to replace an empty sheaf with a full sheaf
363 * from the per-node barn.
364 *
365 * When no full sheaf is available, and gfp flags allow blocking, a
366 * sheaf is allocated and filled from slab(s) using bulk allocation.
367 * Otherwise the allocation falls back to the normal operation
368 * allocating a single object from a slab.
369 *
370 * Analogically when freeing and both percpu sheaves are full, the barn
371 * may replace it with an empty sheaf, unless it's over capacity. In
372 * that case a sheaf is bulk freed to slab pages.
373 *
374 * The sheaves do not enforce NUMA placement of objects, so allocations
375 * via kmem_cache_alloc_node() with a node specified other than
376 * NUMA_NO_NODE will bypass them.
377 *
378 * Bulk allocation and free operations also try to use the cpu sheaves
379 * and barn, but fallback to using slab pages directly.
380 *
381 * When slub_debug is enabled for the cache, the sheaf_capacity argument
382 * is ignored.
383 *
384 * %0 means no sheaves will be created.
385 */
386 unsigned int sheaf_capacity;
387 };
388
389 struct kmem_cache *__kmem_cache_create_args(const char *name,
390 unsigned int object_size,
391 struct kmem_cache_args *args,
392 slab_flags_t flags);
393 static inline struct kmem_cache *
__kmem_cache_create(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,void (* ctor)(void *))394 __kmem_cache_create(const char *name, unsigned int size, unsigned int align,
395 slab_flags_t flags, void (*ctor)(void *))
396 {
397 struct kmem_cache_args kmem_args = {
398 .align = align,
399 .ctor = ctor,
400 };
401
402 return __kmem_cache_create_args(name, size, &kmem_args, flags);
403 }
404
405 /**
406 * kmem_cache_create_usercopy - Create a kmem cache with a region suitable
407 * for copying to userspace.
408 * @name: A string which is used in /proc/slabinfo to identify this cache.
409 * @size: The size of objects to be created in this cache.
410 * @align: The required alignment for the objects.
411 * @flags: SLAB flags
412 * @useroffset: Usercopy region offset
413 * @usersize: Usercopy region size
414 * @ctor: A constructor for the objects, or %NULL.
415 *
416 * This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY()
417 * if whitelisting a single field is sufficient, or kmem_cache_create() with
418 * the necessary parameters passed via the args parameter (see
419 * &struct kmem_cache_args)
420 *
421 * Return: a pointer to the cache on success, NULL on failure.
422 */
423 static inline struct kmem_cache *
kmem_cache_create_usercopy(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,unsigned int useroffset,unsigned int usersize,void (* ctor)(void *))424 kmem_cache_create_usercopy(const char *name, unsigned int size,
425 unsigned int align, slab_flags_t flags,
426 unsigned int useroffset, unsigned int usersize,
427 void (*ctor)(void *))
428 {
429 struct kmem_cache_args kmem_args = {
430 .align = align,
431 .ctor = ctor,
432 .useroffset = useroffset,
433 .usersize = usersize,
434 };
435
436 return __kmem_cache_create_args(name, size, &kmem_args, flags);
437 }
438
439 /* If NULL is passed for @args, use this variant with default arguments. */
440 static inline struct kmem_cache *
__kmem_cache_default_args(const char * name,unsigned int size,struct kmem_cache_args * args,slab_flags_t flags)441 __kmem_cache_default_args(const char *name, unsigned int size,
442 struct kmem_cache_args *args,
443 slab_flags_t flags)
444 {
445 struct kmem_cache_args kmem_default_args = {};
446
447 /* Make sure we don't get passed garbage. */
448 if (WARN_ON_ONCE(args))
449 return ERR_PTR(-EINVAL);
450
451 return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
452 }
453
454 /**
455 * kmem_cache_create - Create a kmem cache.
456 * @__name: A string which is used in /proc/slabinfo to identify this cache.
457 * @__object_size: The size of objects to be created in this cache.
458 * @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL
459 * means defaults will be used for all the arguments.
460 *
461 * This is currently implemented as a macro using ``_Generic()`` to call
462 * either the new variant of the function, or a legacy one.
463 *
464 * The new variant has 4 parameters:
465 * ``kmem_cache_create(name, object_size, args, flags)``
466 *
467 * See __kmem_cache_create_args() which implements this.
468 *
469 * The legacy variant has 5 parameters:
470 * ``kmem_cache_create(name, object_size, align, flags, ctor)``
471 *
472 * The align and ctor parameters map to the respective fields of
473 * &struct kmem_cache_args
474 *
475 * Context: Cannot be called within a interrupt, but can be interrupted.
476 *
477 * Return: a pointer to the cache on success, NULL on failure.
478 */
479 #define kmem_cache_create(__name, __object_size, __args, ...) \
480 _Generic((__args), \
481 struct kmem_cache_args *: __kmem_cache_create_args, \
482 void *: __kmem_cache_default_args, \
483 default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
484
485 void kmem_cache_destroy(struct kmem_cache *s);
486 int kmem_cache_shrink(struct kmem_cache *s);
487
488 /*
489 * Please use this macro to create slab caches. Simply specify the
490 * name of the structure and maybe some flags that are listed above.
491 *
492 * The alignment of the struct determines object alignment. If you
493 * f.e. add ____cacheline_aligned_in_smp to the struct declaration
494 * then the objects will be properly aligned in SMP configurations.
495 */
496 #define KMEM_CACHE(__struct, __flags) \
497 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
498 &(struct kmem_cache_args) { \
499 .align = __alignof__(struct __struct), \
500 }, (__flags))
501
502 /*
503 * To whitelist a single field for copying to/from usercopy, use this
504 * macro instead for KMEM_CACHE() above.
505 */
506 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \
507 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
508 &(struct kmem_cache_args) { \
509 .align = __alignof__(struct __struct), \
510 .useroffset = offsetof(struct __struct, __field), \
511 .usersize = sizeof_field(struct __struct, __field), \
512 }, (__flags))
513
514 #ifdef CONFIG_KMALLOC_PARTITION_CACHES
515 typedef struct { unsigned long v; } kmalloc_token_t;
516 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM
517 extern unsigned long random_kmalloc_seed;
518 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = _CODE_LOCATION_ })
519 #elif defined(CONFIG_KMALLOC_PARTITION_TYPED)
520 #ifdef __CHECKER__
521 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = 0 })
522 #else /* !__CHECKER__ */
523 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = __builtin_infer_alloc_token(__VA_ARGS__) })
524 #endif /* __CHECKER__ */
525 #endif /* CONFIG_KMALLOC_PARTITION_TYPED */
526 #define DECL_TOKEN_PARAM(_token) , kmalloc_token_t (_token)
527 #define _PASS_TOKEN_PARAM(_token) , (_token)
528 #define PASS_TOKEN_PARAM(_token) (_token)
529 #define DECL_TOKEN_PARAMS(_size, _token) size_t (_size), kmalloc_token_t (_token)
530 #define PASS_TOKEN_PARAMS(_size, _token) (_size), (_token)
531 #else /* !CONFIG_KMALLOC_PARTITION_CACHES */
532 typedef struct {} kmalloc_token_t;
533 #define __kmalloc_token(...) ((kmalloc_token_t){}) /* no-op */
534 #define DECL_TOKEN_PARAM(_token)
535 #define _PASS_TOKEN_PARAM(_token)
536 #define PASS_TOKEN_PARAM(_token) ((kmalloc_token_t){})
537 #define DECL_TOKEN_PARAMS(_size, _token) size_t (_size)
538 #define PASS_TOKEN_PARAMS(_size, _token) (_size)
539 #endif /* CONFIG_KMALLOC_PARTITION_CACHES */
540
541 /*
542 * Common kmalloc functions provided by all allocators
543 */
544 void * __must_check krealloc_node_align_noprof(const void *objp,
545 DECL_TOKEN_PARAMS(new_size, token),
546 unsigned long align,
547 gfp_t flags, int nid) __realloc_size(2);
548 #define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), 1, _f, NUMA_NO_NODE)
549 #if 0 /* kernel-doc */
550 /**
551 * krealloc_node_align - reallocate memory. The contents will remain unchanged.
552 * @p: object to reallocate memory for.
553 * @new_size: how many bytes of memory are required.
554 * @align: desired alignment.
555 * @flags: the type of memory to allocate.
556 * @nid: NUMA node or NUMA_NO_NODE
557 *
558 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size
559 * is 0 and @p is not a %NULL pointer, the object pointed to is freed.
560 *
561 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
562 * Documentation/core-api/memory-allocation.rst for more details.
563 *
564 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
565 * initial memory allocation, every subsequent call to this API for the same
566 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
567 * __GFP_ZERO is not fully honored by this API.
568 *
569 * When slub_debug_orig_size() is off, krealloc() only knows about the bucket
570 * size of an allocation (but not the exact size it was allocated with) and
571 * hence implements the following semantics for shrinking and growing buffers
572 * with __GFP_ZERO::
573 *
574 * new bucket
575 * 0 size size
576 * |--------|----------------|
577 * | keep | zero |
578 *
579 * Otherwise, the original allocation size 'orig_size' could be used to
580 * precisely clear the requested size, and the new size will also be stored
581 * as the new 'orig_size'.
582 *
583 * In any case, the contents of the object pointed to are preserved up to the
584 * lesser of the new and old sizes.
585 *
586 * Return: pointer to the allocated memory or %NULL in case of error
587 */
588 void *krealloc_node_align(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid);
589 #endif
590 #define krealloc_node_align(p, new_size, align, flags, nid) \
591 alloc_hooks(krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(new_size, __kmalloc_token(new_size)), align, flags, nid))
592 #define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n)
593 #define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE)
594
595 void kfree(const void *objp);
596 void kfree_nolock(const void *objp);
597 void kfree_sensitive(const void *objp);
598
599 DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T))
600 DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T))
601
602 size_t ksize(const void *objp);
603
604 #ifdef CONFIG_PRINTK
605 bool kmem_dump_obj(void *object);
606 #else
kmem_dump_obj(void * object)607 static inline bool kmem_dump_obj(void *object) { return false; }
608 #endif
609
610 /*
611 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
612 * alignment larger than the alignment of a 64-bit integer.
613 * Setting ARCH_DMA_MINALIGN in arch headers allows that.
614 */
615 #ifdef ARCH_HAS_DMA_MINALIGN
616 #if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN)
617 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
618 #endif
619 #endif
620
621 #ifndef ARCH_KMALLOC_MINALIGN
622 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
623 #elif ARCH_KMALLOC_MINALIGN > 8
624 #define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
625 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
626 #endif
627
628 /*
629 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
630 * Intended for arches that get misalignment faults even for 64 bit integer
631 * aligned buffers.
632 */
633 #ifndef ARCH_SLAB_MINALIGN
634 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
635 #endif
636
637 /*
638 * Arches can define this function if they want to decide the minimum slab
639 * alignment at runtime. The value returned by the function must be a power
640 * of two and >= ARCH_SLAB_MINALIGN.
641 */
642 #ifndef arch_slab_minalign
arch_slab_minalign(void)643 static inline unsigned int arch_slab_minalign(void)
644 {
645 return ARCH_SLAB_MINALIGN;
646 }
647 #endif
648
649 /*
650 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN.
651 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN
652 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment.
653 */
654 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
655 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
656 #define __assume_page_alignment __assume_aligned(PAGE_SIZE)
657
658 /*
659 * Kmalloc array related definitions
660 */
661
662 /*
663 * SLUB directly allocates requests fitting in to an order-1 page
664 * (PAGE_SIZE*2). Larger requests are passed to the page allocator.
665 */
666 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
667 #define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT)
668 #ifndef KMALLOC_SHIFT_LOW
669 #define KMALLOC_SHIFT_LOW 3
670 #endif
671
672 /* Maximum allocatable size */
673 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
674 /* Maximum size for which we actually use a slab cache */
675 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
676 /* Maximum order allocatable via the slab allocator */
677 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
678
679 /*
680 * Kmalloc subsystem.
681 */
682 #ifndef KMALLOC_MIN_SIZE
683 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
684 #endif
685
686 /*
687 * This restriction comes from byte sized index implementation.
688 * Page size is normally 2^12 bytes and, in this case, if we want to use
689 * byte sized index which can represent 2^8 entries, the size of the object
690 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
691 * If minimum size of kmalloc is less than 16, we use it as minimum object
692 * size and give up to use byte sized index.
693 */
694 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \
695 (KMALLOC_MIN_SIZE) : 16)
696
697 #ifdef CONFIG_KMALLOC_PARTITION_CACHES
698 #define KMALLOC_PARTITION_CACHES_NR 15 // # of cache copies
699 #else
700 #define KMALLOC_PARTITION_CACHES_NR 0
701 #endif
702
703 /*
704 * Whenever changing this, take care of that kmalloc_type() and
705 * create_kmalloc_caches() still work as intended.
706 *
707 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
708 * is for accounted but unreclaimable and non-dma objects. All the other
709 * kmem caches can have both accounted and unaccounted objects.
710 */
711 enum kmalloc_cache_type {
712 KMALLOC_NORMAL = 0,
713 #ifndef CONFIG_ZONE_DMA
714 KMALLOC_DMA = KMALLOC_NORMAL,
715 #endif
716 #ifndef CONFIG_MEMCG
717 KMALLOC_CGROUP = KMALLOC_NORMAL,
718 #endif
719 #ifndef CONFIG_SLAB_OBJ_EXT
720 KMALLOC_NO_OBJ_EXT = KMALLOC_NORMAL,
721 #endif
722 KMALLOC_PARTITION_START = KMALLOC_NORMAL,
723 KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR,
724 #ifdef CONFIG_SLUB_TINY
725 KMALLOC_RECLAIM = KMALLOC_NORMAL,
726 #else
727 KMALLOC_RECLAIM,
728 #endif
729 #ifdef CONFIG_ZONE_DMA
730 KMALLOC_DMA,
731 #endif
732 #ifdef CONFIG_MEMCG
733 KMALLOC_CGROUP,
734 #endif
735 #ifdef CONFIG_SLAB_OBJ_EXT
736 KMALLOC_NO_OBJ_EXT,
737 #endif
738 NR_KMALLOC_TYPES
739 };
740
741 typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1];
742
743 extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES];
744
745 /*
746 * Define gfp bits that should not be set for KMALLOC_NORMAL.
747 */
748 #define KMALLOC_NOT_NORMAL_BITS \
749 (__GFP_RECLAIMABLE | \
750 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \
751 (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0))
752
kmalloc_type(gfp_t flags,kmalloc_token_t token)753 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, kmalloc_token_t token)
754 {
755 /*
756 * The most common case is KMALLOC_NORMAL, so test for it
757 * with a single branch for all the relevant flags.
758 */
759 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
760 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM
761 /* KMALLOC_PARTITION_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
762 return KMALLOC_PARTITION_START + hash_64(token.v ^ random_kmalloc_seed,
763 ilog2(KMALLOC_PARTITION_CACHES_NR + 1));
764 #elif defined(CONFIG_KMALLOC_PARTITION_TYPED)
765 return KMALLOC_PARTITION_START + token.v;
766 #else
767 return KMALLOC_NORMAL;
768 #endif
769
770 /*
771 * At least one of the flags has to be set. Their priorities in
772 * decreasing order are:
773 * 1) __GFP_DMA
774 * 2) __GFP_RECLAIMABLE
775 * 3) __GFP_ACCOUNT
776 */
777 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA))
778 return KMALLOC_DMA;
779 if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE))
780 return KMALLOC_RECLAIM;
781 else
782 return KMALLOC_CGROUP;
783 }
784
785 /*
786 * Figure out which kmalloc slab an allocation of a certain size
787 * belongs to.
788 * 0 = zero alloc
789 * 1 = 65 .. 96 bytes
790 * 2 = 129 .. 192 bytes
791 * n = 2^(n-1)+1 .. 2^n
792 *
793 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
794 * typical usage is via kmalloc_index() and therefore evaluated at compile-time.
795 * Callers where !size_is_constant should only be test modules, where runtime
796 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab().
797 */
__kmalloc_index(size_t size,bool size_is_constant)798 static __always_inline unsigned int __kmalloc_index(size_t size,
799 bool size_is_constant)
800 {
801 if (!size)
802 return 0;
803
804 if (size <= KMALLOC_MIN_SIZE)
805 return KMALLOC_SHIFT_LOW;
806
807 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
808 return 1;
809 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
810 return 2;
811 if (size <= 8) return 3;
812 if (size <= 16) return 4;
813 if (size <= 32) return 5;
814 if (size <= 64) return 6;
815 if (size <= 128) return 7;
816 if (size <= 256) return 8;
817 if (size <= 512) return 9;
818 if (size <= 1024) return 10;
819 if (size <= 2 * 1024) return 11;
820 if (size <= 4 * 1024) return 12;
821 if (size <= 8 * 1024) return 13;
822 if (size <= 16 * 1024) return 14;
823 if (size <= 32 * 1024) return 15;
824 if (size <= 64 * 1024) return 16;
825 if (size <= 128 * 1024) return 17;
826 if (size <= 256 * 1024) return 18;
827 if (size <= 512 * 1024) return 19;
828 if (size <= 1024 * 1024) return 20;
829 if (size <= 2 * 1024 * 1024) return 21;
830
831 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
832 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
833 else
834 BUG();
835
836 /* Will never be reached. Needed because the compiler may complain */
837 return -1;
838 }
839 static_assert(PAGE_SHIFT <= 20);
840 #define kmalloc_index(s) __kmalloc_index(s, true)
841
842 #include <linux/alloc_tag.h>
843
844 /**
845 * kmem_cache_alloc - Allocate an object
846 * @cachep: The cache to allocate from.
847 * @flags: See kmalloc().
848 *
849 * Allocate an object from this cache.
850 * See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags.
851 *
852 * Return: pointer to the new object or %NULL in case of error
853 */
854 void *kmem_cache_alloc_noprof(struct kmem_cache *cachep,
855 gfp_t flags) __assume_slab_alignment __malloc;
856 #define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__))
857
858 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
859 gfp_t gfpflags) __assume_slab_alignment __malloc;
860 #define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__))
861
862 /**
863 * kmem_cache_charge - memcg charge an already allocated slab memory
864 * @objp: address of the slab object to memcg charge
865 * @gfpflags: describe the allocation context
866 *
867 * kmem_cache_charge allows charging a slab object to the current memcg,
868 * primarily in cases where charging at allocation time might not be possible
869 * because the target memcg is not known (i.e. softirq context)
870 *
871 * The objp should be pointer returned by the slab allocator functions like
872 * kmalloc (with __GFP_ACCOUNT in flags) or kmem_cache_alloc. The memcg charge
873 * behavior can be controlled through gfpflags parameter, which affects how the
874 * necessary internal metadata can be allocated. Including __GFP_NOFAIL denotes
875 * that overcharging is requested instead of failure, but is not applied for the
876 * internal metadata allocation.
877 *
878 * There are several cases where it will return true even if the charging was
879 * not done:
880 * More specifically:
881 *
882 * 1. For !CONFIG_MEMCG or cgroup_disable=memory systems.
883 * 2. Already charged slab objects.
884 * 3. For slab objects from KMALLOC_NORMAL caches - allocated by kmalloc()
885 * without __GFP_ACCOUNT
886 * 4. Allocating internal metadata has failed
887 *
888 * Return: true if charge was successful otherwise false.
889 */
890 bool kmem_cache_charge(void *objp, gfp_t gfpflags);
891 void kmem_cache_free(struct kmem_cache *s, void *objp);
892
893 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
894 unsigned int useroffset, unsigned int usersize,
895 void (*ctor)(void *));
896
897 /*
898 * Bulk allocation and freeing operations. These are accelerated in an
899 * allocator specific way to avoid taking locks repeatedly or building
900 * metadata structures unnecessarily.
901 *
902 * Note that interrupts must be enabled when calling these functions.
903 */
904 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
905
906 bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
907 size_t size, void **p);
908 #define kmem_cache_alloc_bulk(...) \
909 alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
910
kfree_bulk(size_t size,void ** p)911 static __always_inline void kfree_bulk(size_t size, void **p)
912 {
913 kmem_cache_free_bulk(NULL, size, p);
914 }
915
916 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags,
917 int node) __assume_slab_alignment __malloc;
918 #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))
919
920 struct slab_sheaf *
921 kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size);
922
923 int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
924 struct slab_sheaf **sheafp, unsigned int size);
925
926 void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
927 struct slab_sheaf *sheaf);
928
929 void *kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *cachep, gfp_t gfp,
930 struct slab_sheaf *sheaf) __assume_slab_alignment __malloc;
931 #define kmem_cache_alloc_from_sheaf(...) \
932 alloc_hooks(kmem_cache_alloc_from_sheaf_noprof(__VA_ARGS__))
933
934 unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf);
935
936 /*
937 * These macros allow declaring a kmem_buckets * parameter alongside size, which
938 * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call
939 * sites don't have to pass NULL.
940 */
941 #ifdef CONFIG_SLAB_BUCKETS
942 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b)
943 #define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b)
944 #define PASS_BUCKET_PARAM(_b) (_b)
945 #else
946 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size)
947 #define PASS_BUCKET_PARAMS(_size, _b) (_size)
948 #define PASS_BUCKET_PARAM(_b) NULL
949 #endif
950
951 #define DECL_KMALLOC_PARAMS(_size, _b, _token) DECL_BUCKET_PARAMS(_size, _b) \
952 DECL_TOKEN_PARAM(_token)
953
954 #define PASS_KMALLOC_PARAMS(_size, _b, _token) PASS_BUCKET_PARAMS(_size, _b) \
955 _PASS_TOKEN_PARAM(_token)
956
957 /*
958 * The following functions are not to be used directly and are intended only
959 * for internal use from kmalloc() and kmalloc_node()
960 * with the exception of kunit tests
961 */
962
963 void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags)
964 __assume_kmalloc_alignment __alloc_size(1);
965
966 void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
967 __assume_kmalloc_alignment __alloc_size(1);
968
969 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
970 __assume_kmalloc_alignment __alloc_size(3);
971
972 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
973 int node, size_t size)
974 __assume_kmalloc_alignment __alloc_size(4);
975
976 void *__kmalloc_large_noprof(size_t size, gfp_t flags)
977 __assume_page_alignment __alloc_size(1);
978
979 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
980 __assume_page_alignment __alloc_size(1);
981
_kmalloc_noprof(size_t size,gfp_t flags,kmalloc_token_t token)982 static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
983 {
984 if (__builtin_constant_p(size) && size) {
985 unsigned int index;
986
987 if (size > KMALLOC_MAX_CACHE_SIZE)
988 return __kmalloc_large_noprof(size, flags);
989
990 index = kmalloc_index(size);
991 return __kmalloc_cache_noprof(
992 kmalloc_caches[kmalloc_type(flags, token)][index],
993 flags, size);
994 }
995 return __kmalloc_noprof(PASS_TOKEN_PARAMS(size, token), flags);
996 }
997 #define kmalloc_noprof(...) _kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
998 #if 0 /* kernel-doc */
999 /**
1000 * kmalloc - allocate kernel memory
1001 * @size: how many bytes of memory are required.
1002 * @flags: describe the allocation context
1003 *
1004 * kmalloc is the normal method of allocating memory
1005 * for objects smaller than page size in the kernel.
1006 *
1007 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
1008 * bytes. For @size of power of two bytes, the alignment is also guaranteed
1009 * to be at least to the size. For other sizes, the alignment is guaranteed to
1010 * be at least the largest power-of-two divisor of @size.
1011 *
1012 * The @flags argument may be one of the GFP flags defined at
1013 * include/linux/gfp_types.h and described at
1014 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
1015 *
1016 * The recommended usage of the @flags is described at
1017 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
1018 *
1019 * Below is a brief outline of the most useful GFP flags
1020 *
1021 * %GFP_KERNEL
1022 * Allocate normal kernel ram. May sleep.
1023 *
1024 * %GFP_NOWAIT
1025 * Allocation will not sleep.
1026 *
1027 * %GFP_ATOMIC
1028 * Allocation will not sleep. May use emergency pools.
1029 *
1030 * Also it is possible to set different flags by OR'ing
1031 * in one or more of the following additional @flags:
1032 *
1033 * %__GFP_ZERO
1034 * Zero the allocated memory before returning. Also see kzalloc().
1035 *
1036 * %__GFP_HIGH
1037 * This allocation has high priority and may use emergency pools.
1038 *
1039 * %__GFP_NOFAIL
1040 * Indicate that this allocation is in no way allowed to fail
1041 * (think twice before using).
1042 *
1043 * %__GFP_NORETRY
1044 * If memory is not immediately available,
1045 * then give up at once.
1046 *
1047 * %__GFP_NOWARN
1048 * If allocation fails, don't issue any warnings.
1049 *
1050 * %__GFP_RETRY_MAYFAIL
1051 * Try really hard to succeed the allocation but fail
1052 * eventually.
1053 */
1054 void *kmalloc(size_t size, gfp_t flags);
1055 #endif
1056 #define kmalloc(size, flags) alloc_hooks(kmalloc_noprof(size, flags))
1057
1058 void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node);
1059 #define kmalloc_nolock_noprof(_s, _f, _n) _kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), _f, _n)
1060 #if 0 /* kernel-doc */
1061 /**
1062 * kmalloc_nolock - Allocate an object of given size from any context.
1063 * @size: size to allocate
1064 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT and __GFP_ZERO allowed. Also
1065 * __GFP_NOWARN and __GFP_NOMEMALLOC are allowed but added internally thus not
1066 * necessary.
1067 * @node: node number of the target node.
1068 *
1069 * Return: pointer to the new object or NULL in case of error.
1070 * NULL does not mean EBUSY or EAGAIN. It means ENOMEM.
1071 * There is no reason to call it again and expect !NULL.
1072 */
1073 void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node);
1074 #endif
1075 #define kmalloc_nolock(size, gfp_flags, node) alloc_hooks(kmalloc_nolock_noprof(size, gfp_flags, node))
1076
1077 /**
1078 * __alloc_objs - Allocate objects of a given type using
1079 * @KMALLOC: which size-based kmalloc wrapper to allocate with.
1080 * @GFP: GFP flags for the allocation.
1081 * @TYPE: type to allocate space for.
1082 * @COUNT: how many @TYPE objects to allocate.
1083 *
1084 * Returns: Newly allocated pointer to (first) @TYPE of @COUNT-many
1085 * allocated @TYPE objects, or NULL on failure.
1086 */
1087 #define __alloc_objs(KMALLOC, GFP, TYPE, COUNT) \
1088 ({ \
1089 const size_t __obj_size = size_mul(sizeof(TYPE), COUNT); \
1090 (TYPE *)KMALLOC(__obj_size, GFP); \
1091 })
1092
1093 /**
1094 * __alloc_flex - Allocate an object that has a trailing flexible array
1095 * @KMALLOC: kmalloc wrapper function to use for allocation.
1096 * @GFP: GFP flags for the allocation.
1097 * @TYPE: type of structure to allocate space for.
1098 * @FAM: The name of the flexible array member of @TYPE structure.
1099 * @COUNT: how many @FAM elements to allocate space for.
1100 *
1101 * Returns: Newly allocated pointer to @TYPE with @COUNT-many trailing
1102 * @FAM elements, or NULL on failure or if @COUNT cannot be represented
1103 * by the member of @TYPE that counts the @FAM elements (annotated via
1104 * __counted_by()).
1105 */
1106 #define __alloc_flex(KMALLOC, GFP, TYPE, FAM, COUNT) \
1107 ({ \
1108 const size_t __count = (COUNT); \
1109 const size_t __obj_size = struct_size_t(TYPE, FAM, __count); \
1110 TYPE *__obj_ptr = KMALLOC(__obj_size, GFP); \
1111 if (__obj_ptr) \
1112 __set_flex_counter(__obj_ptr->FAM, __count); \
1113 __obj_ptr; \
1114 })
1115
1116 /**
1117 * kmalloc_obj - Allocate a single instance of the given type
1118 * @VAR_OR_TYPE: Variable or type to allocate.
1119 * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified).
1120 *
1121 * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL
1122 * on failure.
1123 */
1124 #define kmalloc_obj(VAR_OR_TYPE, ...) \
1125 __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), 1)
1126
1127 /**
1128 * kmalloc_objs - Allocate an array of the given type
1129 * @VAR_OR_TYPE: Variable or type to allocate an array of.
1130 * @COUNT: How many elements in the array.
1131 * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified).
1132 *
1133 * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success,
1134 * or NULL on failure.
1135 */
1136 #define kmalloc_objs(VAR_OR_TYPE, COUNT, ...) \
1137 __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), COUNT)
1138
1139 /**
1140 * kmalloc_flex - Allocate a single instance of the given flexible structure
1141 * @VAR_OR_TYPE: Variable or type to allocate (with its flex array).
1142 * @FAM: The name of the flexible array member of the structure.
1143 * @COUNT: How many flexible array member elements are desired.
1144 * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified).
1145 *
1146 * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on
1147 * failure. If @FAM has been annotated with __counted_by(), the allocation
1148 * will immediately fail if @COUNT is larger than what the type of the
1149 * struct's counter variable can represent.
1150 */
1151 #define kmalloc_flex(VAR_OR_TYPE, FAM, COUNT, ...) \
1152 __alloc_flex(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), FAM, COUNT)
1153
1154 /* All kzalloc aliases for kmalloc_(obj|objs|flex). */
1155 #define kzalloc_obj(P, ...) \
1156 __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), 1)
1157 #define kzalloc_objs(P, COUNT, ...) \
1158 __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT)
1159 #define kzalloc_flex(P, FAM, COUNT, ...) \
1160 __alloc_flex(kzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT)
1161
1162 /* All kvmalloc aliases for kmalloc_(obj|objs|flex). */
1163 #define kvmalloc_obj(P, ...) \
1164 __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), 1)
1165 #define kvmalloc_objs(P, COUNT, ...) \
1166 __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT)
1167 #define kvmalloc_flex(P, FAM, COUNT, ...) \
1168 __alloc_flex(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT)
1169
1170 /* All kvzalloc aliases for kmalloc_(obj|objs|flex). */
1171 #define kvzalloc_obj(P, ...) \
1172 __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), 1)
1173 #define kvzalloc_objs(P, COUNT, ...) \
1174 __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT)
1175 #define kvzalloc_flex(P, FAM, COUNT, ...) \
1176 __alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT)
1177
1178 #define kmem_buckets_alloc(_b, _size, _flags) \
1179 alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE))
1180
1181 #define kmem_buckets_alloc_node_track_caller(_b, _size, _flags, _node) \
1182 alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, _node, _RET_IP_))
1183
1184 #define kmem_buckets_alloc_track_caller(_b, _size, _flags) \
1185 kmem_buckets_alloc_node_track_caller(_b, _size, _flags, NUMA_NO_NODE)
1186
_kmalloc_node_noprof(size_t size,gfp_t flags,int node,kmalloc_token_t token)1187 static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token)
1188 {
1189 if (__builtin_constant_p(size) && size) {
1190 unsigned int index;
1191
1192 if (size > KMALLOC_MAX_CACHE_SIZE)
1193 return __kmalloc_large_node_noprof(size, flags, node);
1194
1195 index = kmalloc_index(size);
1196 return __kmalloc_cache_node_noprof(
1197 kmalloc_caches[kmalloc_type(flags, token)][index],
1198 flags, node, size);
1199 }
1200 return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags, node);
1201 }
1202 #define kmalloc_node_noprof(...) _kmalloc_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
1203 #define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
1204
_kmalloc_array_noprof(size_t n,size_t size,gfp_t flags,kmalloc_token_t token)1205 static inline __alloc_size(1, 2) void *_kmalloc_array_noprof(size_t n, size_t size, gfp_t flags, kmalloc_token_t token)
1206 {
1207 size_t bytes;
1208
1209 if (unlikely(check_mul_overflow(n, size, &bytes)))
1210 return NULL;
1211 return _kmalloc_noprof(bytes, flags, token);
1212 }
1213 #define kmalloc_array_noprof(...) _kmalloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
1214 #if 0 /* kernel-doc */
1215 /**
1216 * kmalloc_array - allocate memory for an array.
1217 * @n: number of elements.
1218 * @size: element size.
1219 * @flags: the type of memory to allocate (see kmalloc).
1220 */
1221 void *kmalloc_array(size_t n, size_t size, gfp_t flags);
1222 #endif
1223 #define kmalloc_array(n, size, flags) alloc_hooks(kmalloc_array_noprof(n, size, flags))
1224
_krealloc_array_noprof(void * p,size_t new_n,size_t new_size,gfp_t flags,kmalloc_token_t token)1225 static inline __realloc_size(2, 3) void * __must_check _krealloc_array_noprof(void *p,
1226 size_t new_n,
1227 size_t new_size,
1228 gfp_t flags, kmalloc_token_t token)
1229 {
1230 size_t bytes;
1231
1232 if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
1233 return NULL;
1234
1235 return krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(bytes, token), 1, flags, NUMA_NO_NODE);
1236 }
1237 #define krealloc_array_noprof(...) _krealloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
1238 #if 0 /* kernel-doc */
1239 /**
1240 * krealloc_array - reallocate memory for an array.
1241 * @p: pointer to the memory chunk to reallocate
1242 * @new_n: new number of elements to alloc
1243 * @new_size: new size of a single member of the array
1244 * @flags: the type of memory to allocate (see kmalloc)
1245 *
1246 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
1247 * initial memory allocation, every subsequent call to this API for the same
1248 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
1249 * __GFP_ZERO is not fully honored by this API.
1250 *
1251 * See krealloc_noprof() for further details.
1252 *
1253 * In any case, the contents of the object pointed to are preserved up to the
1254 * lesser of the new and old sizes.
1255 */
1256 void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t flags);
1257 #endif
1258 #define krealloc_array(p, new_n, new_size, flags) alloc_hooks(krealloc_array_noprof(p, new_n, new_size, flags))
1259
1260 /**
1261 * kcalloc - allocate memory for an array. The memory is set to zero.
1262 * @n: number of elements.
1263 * @size: element size.
1264 * @flags: the type of memory to allocate (see kmalloc).
1265 */
1266 #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
1267
1268 void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node,
1269 unsigned long caller) __alloc_size(1);
1270 #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
1271 __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(size, NULL, __kmalloc_token(size)), flags, node, caller)
1272 #define kmalloc_node_track_caller(...) \
1273 alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
1274
1275 /*
1276 * kmalloc_track_caller is a special version of kmalloc that records the
1277 * calling function of the routine calling it for slab leak tracking instead
1278 * of just the calling function (confusing, eh?).
1279 * It's useful when the call to kmalloc comes from a widely-used standard
1280 * allocator where we care about the real place the memory allocation
1281 * request comes from.
1282 */
1283 #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE)
1284
1285 #define kmalloc_track_caller_noprof(...) \
1286 kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_)
1287
_kmalloc_array_node_noprof(size_t n,size_t size,gfp_t flags,int node,kmalloc_token_t token)1288 static inline __alloc_size(1, 2) void *_kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
1289 int node, kmalloc_token_t token)
1290 {
1291 size_t bytes;
1292
1293 if (unlikely(check_mul_overflow(n, size, &bytes)))
1294 return NULL;
1295 if (__builtin_constant_p(n) && __builtin_constant_p(size))
1296 return _kmalloc_node_noprof(bytes, flags, node, token);
1297 return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), flags, node);
1298 }
1299 #define kmalloc_array_node_noprof(...) _kmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
1300 #define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
1301
1302 #define kcalloc_node(_n, _size, _flags, _node) \
1303 kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node)
1304
1305 /*
1306 * Shortcuts
1307 */
1308 #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)
1309
_kzalloc_noprof(size_t size,gfp_t flags,kmalloc_token_t token)1310 static inline __alloc_size(1) void *_kzalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
1311 {
1312 return _kmalloc_noprof(size, flags | __GFP_ZERO, token);
1313 }
1314 #define kzalloc_noprof(...) _kzalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
1315 #if 0 /* kernel-doc */
1316 /**
1317 * kzalloc - allocate memory. The memory is set to zero.
1318 * @size: how many bytes of memory are required.
1319 * @flags: the type of memory to allocate (see kmalloc).
1320 */
1321 void *kzalloc(size_t size, gfp_t flags);
1322 #endif
1323 #define kzalloc(size, flags) alloc_hooks(kzalloc_noprof(size, flags))
1324 #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1325
1326 void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align,
1327 gfp_t flags, int node) __alloc_size(1);
1328 #define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \
1329 __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, NULL, __kmalloc_token(_size)), _align, _flags, _node)
1330 #define kvmalloc_node_align(...) \
1331 alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__))
1332 #if 0 /* kernel-doc */
1333 /**
1334 * kvmalloc_node - attempt to allocate physically contiguous memory, but upon
1335 * failure, fall back to non-contiguous (vmalloc) allocation.
1336 * @size: size of the request.
1337 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
1338 * @node: numa node to allocate from
1339 *
1340 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
1341 * Documentation/core-api/memory-allocation.rst for more details.
1342 *
1343 * Uses kmalloc to get the memory but if the allocation fails then falls back
1344 * to the vmalloc allocator. Use kvfree for freeing the memory.
1345 *
1346 * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not.
1347 * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
1348 * preferable to the vmalloc fallback, due to visible performance drawbacks.
1349 *
1350 * Return: pointer to the allocated memory of %NULL in case of failure
1351 */
1352 void *kvmalloc_node(size_t size, gfp_t flags, int node);
1353 #endif
1354 #define kvmalloc_node(size, flags, node) kvmalloc_node_align(size, 1, flags, node)
1355 #define kvmalloc_node_noprof(size, flags, node) \
1356 kvmalloc_node_align_noprof(size, 1, flags, node)
1357 #define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE)
1358 #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE)
1359 #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO)
1360
1361 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1362
1363 #define kmem_buckets_valloc(_b, _size, _flags) \
1364 alloc_hooks(__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), 1, _flags, NUMA_NO_NODE))
1365
1366 static inline __alloc_size(1, 2) void *
_kvmalloc_array_node_noprof(size_t n,size_t size,gfp_t flags,int node,kmalloc_token_t token)1367 _kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmalloc_token_t token)
1368 {
1369 size_t bytes;
1370
1371 if (unlikely(check_mul_overflow(n, size, &bytes)))
1372 return NULL;
1373
1374 return __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), 1, flags, node);
1375 }
1376 #define kvmalloc_array_node_noprof(...) _kvmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
1377 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1378 #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node)
1379 #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1380
1381 #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))
1382 #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))
1383 #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__))
1384
1385 void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align,
1386 gfp_t flags, int nid) __realloc_size(2);
1387 #if 0 /* kernel-doc */
1388 /**
1389 * kvrealloc_node_align - reallocate memory; contents remain unchanged
1390 * @p: object to reallocate memory for
1391 * @size: the size to reallocate
1392 * @align: desired alignment
1393 * @flags: the flags for the page level allocator
1394 * @nid: NUMA node id
1395 *
1396 * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
1397 * and @p is not a %NULL pointer, the object pointed to is freed.
1398 *
1399 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
1400 * Documentation/core-api/memory-allocation.rst for more details.
1401 *
1402 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
1403 * initial memory allocation, every subsequent call to this API for the same
1404 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
1405 * __GFP_ZERO is not fully honored by this API.
1406 *
1407 * In any case, the contents of the object pointed to are preserved up to the
1408 * lesser of the new and old sizes.
1409 *
1410 * This function must not be called concurrently with itself or kvfree() for the
1411 * same memory allocation.
1412 *
1413 * Return: pointer to the allocated memory or %NULL in case of error
1414 */
1415 void *kvrealloc_node_align(const void *p, size_t size, unsigned long align, gfp_t flags, int nid);
1416 #endif
1417 #define kvrealloc_node_align(p, size, align, flags, nid) \
1418 alloc_hooks(kvrealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, __kmalloc_token(size)), align, flags, nid))
1419 #define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n)
1420 #define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE)
1421
1422 extern void kvfree(const void *addr);
1423 DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
1424
1425 extern void kvfree_atomic(const void *addr);
1426 DEFINE_FREE(kvfree_atomic, void *, if (!IS_ERR_OR_NULL(_T)) kvfree_atomic(_T))
1427
1428 extern void kvfree_sensitive(const void *addr, size_t len);
1429
1430 unsigned int kmem_cache_size(struct kmem_cache *s);
1431
1432 #ifndef CONFIG_KVFREE_RCU_BATCHED
kfree_rcu_scheduler_running(void)1433 static inline void kfree_rcu_scheduler_running(void) { }
1434 #else
1435 void kfree_rcu_scheduler_running(void);
1436 #endif
1437
1438 void kvfree_rcu_barrier(void);
1439
1440 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);
1441
1442 /**
1443 * kmalloc_size_roundup - Report allocation bucket size for the given size
1444 *
1445 * @size: Number of bytes to round up from.
1446 *
1447 * This returns the number of bytes that would be available in a kmalloc()
1448 * allocation of @size bytes. For example, a 126 byte request would be
1449 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly
1450 * for the general-purpose kmalloc()-based allocations, and is not for the
1451 * pre-sized kmem_cache_alloc()-based allocations.)
1452 *
1453 * Use this to kmalloc() the full bucket size ahead of time instead of using
1454 * ksize() to query the size after an allocation.
1455 */
1456 size_t kmalloc_size_roundup(size_t size);
1457
1458 void __init kmem_cache_init_late(void);
1459 void __init kvfree_rcu_init(void);
1460
1461 #endif /* _LINUX_SLAB_H */
1462