1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk). 4 * 5 * (C) SGI 2006, Christoph Lameter 6 * Cleaned up and restructured to ease the addition of alternative 7 * implementations of SLAB allocators. 8 * (C) Linux Foundation 2008-2013 9 * Unified interface for all slab allocators 10 */ 11 12 #ifndef _LINUX_SLAB_H 13 #define _LINUX_SLAB_H 14 15 #include <linux/bug.h> 16 #include <linux/cache.h> 17 #include <linux/gfp.h> 18 #include <linux/overflow.h> 19 #include <linux/types.h> 20 #include <linux/rcupdate.h> 21 #include <linux/workqueue.h> 22 #include <linux/percpu-refcount.h> 23 #include <linux/cleanup.h> 24 #include <linux/hash.h> 25 26 enum _slab_flag_bits { 27 _SLAB_CONSISTENCY_CHECKS, 28 _SLAB_RED_ZONE, 29 _SLAB_POISON, 30 _SLAB_KMALLOC, 31 _SLAB_HWCACHE_ALIGN, 32 _SLAB_CACHE_DMA, 33 _SLAB_CACHE_DMA32, 34 _SLAB_STORE_USER, 35 _SLAB_PANIC, 36 _SLAB_TYPESAFE_BY_RCU, 37 _SLAB_TRACE, 38 #ifdef CONFIG_DEBUG_OBJECTS 39 _SLAB_DEBUG_OBJECTS, 40 #endif 41 _SLAB_NOLEAKTRACE, 42 _SLAB_NO_MERGE, 43 #ifdef CONFIG_FAILSLAB 44 _SLAB_FAILSLAB, 45 #endif 46 #ifdef CONFIG_MEMCG 47 _SLAB_ACCOUNT, 48 #endif 49 #ifdef CONFIG_KASAN_GENERIC 50 _SLAB_KASAN, 51 #endif 52 _SLAB_NO_USER_FLAGS, 53 #ifdef CONFIG_KFENCE 54 _SLAB_SKIP_KFENCE, 55 #endif 56 #ifndef CONFIG_SLUB_TINY 57 _SLAB_RECLAIM_ACCOUNT, 58 #endif 59 _SLAB_OBJECT_POISON, 60 _SLAB_CMPXCHG_DOUBLE, 61 _SLAB_NO_OBJ_EXT, 62 #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) 63 _SLAB_OBJ_EXT_IN_OBJ, 64 #endif 65 _SLAB_FLAGS_LAST_BIT 66 }; 67 68 #define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr))) 69 #define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U)) 70 71 /* 72 * Flags to pass to kmem_cache_create(). 73 * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op 74 */ 75 /* DEBUG: Perform (expensive) checks on alloc/free */ 76 #define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS) 77 /* DEBUG: Red zone objs in a cache */ 78 #define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE) 79 /* DEBUG: Poison objects */ 80 #define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON) 81 /* Indicate a kmalloc slab */ 82 #define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC) 83 /** 84 * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries. 85 * 86 * Sufficiently large objects are aligned on cache line boundary. For object 87 * size smaller than a half of cache line size, the alignment is on the half of 88 * cache line size. In general, if object size is smaller than 1/2^n of cache 89 * line size, the alignment is adjusted to 1/2^n. 90 * 91 * If explicit alignment is also requested by the respective 92 * &struct kmem_cache_args field, the greater of both is alignments is applied. 93 */ 94 #define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN) 95 /* Use GFP_DMA memory */ 96 #define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA) 97 /* Use GFP_DMA32 memory */ 98 #define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32) 99 /* DEBUG: Store the last owner for bug hunting */ 100 #define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER) 101 /* Panic if kmem_cache_create() fails */ 102 #define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC) 103 /** 104 * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 105 * 106 * This delays freeing the SLAB page by a grace period, it does _NOT_ 107 * delay object freeing. This means that if you do kmem_cache_free() 108 * that memory location is free to be reused at any time. Thus it may 109 * be possible to see another object there in the same RCU grace period. 110 * 111 * This feature only ensures the memory location backing the object 112 * stays valid, the trick to using this is relying on an independent 113 * object validation pass. Something like: 114 * 115 * :: 116 * 117 * begin: 118 * rcu_read_lock(); 119 * obj = lockless_lookup(key); 120 * if (obj) { 121 * if (!try_get_ref(obj)) // might fail for free objects 122 * rcu_read_unlock(); 123 * goto begin; 124 * 125 * if (obj->key != key) { // not the object we expected 126 * put_ref(obj); 127 * rcu_read_unlock(); 128 * goto begin; 129 * } 130 * } 131 * rcu_read_unlock(); 132 * 133 * This is useful if we need to approach a kernel structure obliquely, 134 * from its address obtained without the usual locking. We can lock 135 * the structure to stabilize it and check it's still at the given address, 136 * only if we can be sure that the memory has not been meanwhile reused 137 * for some other kind of object (which our subsystem's lock might corrupt). 138 * 139 * rcu_read_lock before reading the address, then rcu_read_unlock after 140 * taking the spinlock within the structure expected at that address. 141 * 142 * Note that object identity check has to be done *after* acquiring a 143 * reference, therefore user has to ensure proper ordering for loads. 144 * Similarly, when initializing objects allocated with SLAB_TYPESAFE_BY_RCU, 145 * the newly allocated object has to be fully initialized *before* its 146 * refcount gets initialized and proper ordering for stores is required. 147 * refcount_{add|inc}_not_zero_acquire() and refcount_set_release() are 148 * designed with the proper fences required for reference counting objects 149 * allocated with SLAB_TYPESAFE_BY_RCU. 150 * 151 * Note that it is not possible to acquire a lock within a structure 152 * allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference 153 * as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages 154 * are not zeroed before being given to the slab, which means that any 155 * locks must be initialized after each and every kmem_struct_alloc(). 156 * Alternatively, make the ctor passed to kmem_cache_create() initialize 157 * the locks at page-allocation time, as is done in __i915_request_ctor(), 158 * sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers 159 * to safely acquire those ctor-initialized locks under rcu_read_lock() 160 * protection. 161 * 162 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. 163 */ 164 #define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU) 165 /* Trace allocations and frees */ 166 #define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE) 167 168 /* Flag to prevent checks on free */ 169 #ifdef CONFIG_DEBUG_OBJECTS 170 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS) 171 #else 172 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED 173 #endif 174 175 /* Avoid kmemleak tracing */ 176 #define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE) 177 178 /* 179 * Prevent merging with compatible kmem caches. This flag should be used 180 * cautiously. Valid use cases: 181 * 182 * - caches created for self-tests (e.g. kunit) 183 * - general caches created and used by a subsystem, only when a 184 * (subsystem-specific) debug option is enabled 185 * - performance critical caches, should be very rare and consulted with slab 186 * maintainers, and not used together with CONFIG_SLUB_TINY 187 */ 188 #define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE) 189 190 /* Fault injection mark */ 191 #ifdef CONFIG_FAILSLAB 192 # define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB) 193 #else 194 # define SLAB_FAILSLAB __SLAB_FLAG_UNUSED 195 #endif 196 /** 197 * define SLAB_ACCOUNT - Account allocations to memcg. 198 * 199 * All object allocations from this cache will be memcg accounted, regardless of 200 * __GFP_ACCOUNT being or not being passed to individual allocations. 201 */ 202 #ifdef CONFIG_MEMCG 203 # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) 204 #else 205 # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED 206 #endif 207 208 #ifdef CONFIG_KASAN_GENERIC 209 #define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN) 210 #else 211 #define SLAB_KASAN __SLAB_FLAG_UNUSED 212 #endif 213 214 /* 215 * Ignore user specified debugging flags. 216 * Intended for caches created for self-tests so they have only flags 217 * specified in the code and other flags are ignored. 218 */ 219 #define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS) 220 221 #ifdef CONFIG_KFENCE 222 #define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE) 223 #else 224 #define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED 225 #endif 226 227 /* The following flags affect the page allocator grouping pages by mobility */ 228 /** 229 * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable. 230 * 231 * Use this flag for caches that have an associated shrinker. As a result, slab 232 * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by 233 * mobility, and are accounted in SReclaimable counter in /proc/meminfo 234 */ 235 #ifndef CONFIG_SLUB_TINY 236 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT) 237 #else 238 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED 239 #endif 240 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ 241 242 /* Slab created using create_boot_cache */ 243 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT) 244 245 #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) 246 #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ) 247 #else 248 #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_UNUSED 249 #endif 250 251 /* 252 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. 253 * 254 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault. 255 * 256 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can. 257 * Both make kfree a no-op. 258 */ 259 #define ZERO_SIZE_PTR ((void *)16) 260 261 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ 262 (unsigned long)ZERO_SIZE_PTR) 263 264 #include <linux/kasan.h> 265 266 struct list_lru; 267 struct mem_cgroup; 268 /* 269 * struct kmem_cache related prototypes 270 */ 271 bool slab_is_available(void); 272 273 /** 274 * struct kmem_cache_args - Less common arguments for kmem_cache_create() 275 * 276 * Any uninitialized fields of the structure are interpreted as unused. The 277 * exception is @freeptr_offset where %0 is a valid value, so 278 * @use_freeptr_offset must be also set to %true in order to interpret the field 279 * as used. For @useroffset %0 is also valid, but only with non-%0 280 * @usersize. 281 * 282 * When %NULL args is passed to kmem_cache_create(), it is equivalent to all 283 * fields unused. 284 */ 285 struct kmem_cache_args { 286 /** 287 * @align: The required alignment for the objects. 288 * 289 * %0 means no specific alignment is requested. 290 */ 291 unsigned int align; 292 /** 293 * @useroffset: Usercopy region offset. 294 * 295 * %0 is a valid offset, when @usersize is non-%0 296 */ 297 unsigned int useroffset; 298 /** 299 * @usersize: Usercopy region size. 300 * 301 * %0 means no usercopy region is specified. 302 */ 303 unsigned int usersize; 304 /** 305 * @freeptr_offset: Custom offset for the free pointer 306 * in caches with &SLAB_TYPESAFE_BY_RCU or @ctor 307 * 308 * By default, &SLAB_TYPESAFE_BY_RCU and @ctor caches place the free 309 * pointer outside of the object. This might cause the object to grow 310 * in size. Cache creators that have a reason to avoid this can specify 311 * a custom free pointer offset in their data structure where the free 312 * pointer will be placed. 313 * 314 * For caches with &SLAB_TYPESAFE_BY_RCU, the caller must ensure that 315 * the free pointer does not overlay fields required to guard against 316 * object recycling (See &SLAB_TYPESAFE_BY_RCU for details). 317 * 318 * For caches with @ctor, the caller must ensure that the free pointer 319 * does not overlay fields initialized by the constructor. 320 * 321 * Currently, only caches with &SLAB_TYPESAFE_BY_RCU or @ctor 322 * may specify @freeptr_offset. 323 * 324 * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset 325 * is specified, @use_freeptr_offset must be set %true. 326 */ 327 unsigned int freeptr_offset; 328 /** 329 * @use_freeptr_offset: Whether a @freeptr_offset is used. 330 */ 331 bool use_freeptr_offset; 332 /** 333 * @ctor: A constructor for the objects. 334 * 335 * The constructor is invoked for each object in a newly allocated slab 336 * page. It is the cache user's responsibility to free object in the 337 * same state as after calling the constructor, or deal appropriately 338 * with any differences between a freshly constructed and a reallocated 339 * object. 340 * 341 * %NULL means no constructor. 342 */ 343 void (*ctor)(void *); 344 /** 345 * @sheaf_capacity: Enable sheaves of given capacity for the cache. 346 * 347 * With a non-zero value, allocations from the cache go through caching 348 * arrays called sheaves. Each cpu has a main sheaf that's always 349 * present, and a spare sheaf that may be not present. When both become 350 * empty, there's an attempt to replace an empty sheaf with a full sheaf 351 * from the per-node barn. 352 * 353 * When no full sheaf is available, and gfp flags allow blocking, a 354 * sheaf is allocated and filled from slab(s) using bulk allocation. 355 * Otherwise the allocation falls back to the normal operation 356 * allocating a single object from a slab. 357 * 358 * Analogically when freeing and both percpu sheaves are full, the barn 359 * may replace it with an empty sheaf, unless it's over capacity. In 360 * that case a sheaf is bulk freed to slab pages. 361 * 362 * The sheaves do not enforce NUMA placement of objects, so allocations 363 * via kmem_cache_alloc_node() with a node specified other than 364 * NUMA_NO_NODE will bypass them. 365 * 366 * Bulk allocation and free operations also try to use the cpu sheaves 367 * and barn, but fallback to using slab pages directly. 368 * 369 * When slub_debug is enabled for the cache, the sheaf_capacity argument 370 * is ignored. 371 * 372 * %0 means no sheaves will be created. 373 */ 374 unsigned int sheaf_capacity; 375 }; 376 377 struct kmem_cache *__kmem_cache_create_args(const char *name, 378 unsigned int object_size, 379 struct kmem_cache_args *args, 380 slab_flags_t flags); 381 static inline struct kmem_cache * 382 __kmem_cache_create(const char *name, unsigned int size, unsigned int align, 383 slab_flags_t flags, void (*ctor)(void *)) 384 { 385 struct kmem_cache_args kmem_args = { 386 .align = align, 387 .ctor = ctor, 388 }; 389 390 return __kmem_cache_create_args(name, size, &kmem_args, flags); 391 } 392 393 /** 394 * kmem_cache_create_usercopy - Create a kmem cache with a region suitable 395 * for copying to userspace. 396 * @name: A string which is used in /proc/slabinfo to identify this cache. 397 * @size: The size of objects to be created in this cache. 398 * @align: The required alignment for the objects. 399 * @flags: SLAB flags 400 * @useroffset: Usercopy region offset 401 * @usersize: Usercopy region size 402 * @ctor: A constructor for the objects, or %NULL. 403 * 404 * This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY() 405 * if whitelisting a single field is sufficient, or kmem_cache_create() with 406 * the necessary parameters passed via the args parameter (see 407 * &struct kmem_cache_args) 408 * 409 * Return: a pointer to the cache on success, NULL on failure. 410 */ 411 static inline struct kmem_cache * 412 kmem_cache_create_usercopy(const char *name, unsigned int size, 413 unsigned int align, slab_flags_t flags, 414 unsigned int useroffset, unsigned int usersize, 415 void (*ctor)(void *)) 416 { 417 struct kmem_cache_args kmem_args = { 418 .align = align, 419 .ctor = ctor, 420 .useroffset = useroffset, 421 .usersize = usersize, 422 }; 423 424 return __kmem_cache_create_args(name, size, &kmem_args, flags); 425 } 426 427 /* If NULL is passed for @args, use this variant with default arguments. */ 428 static inline struct kmem_cache * 429 __kmem_cache_default_args(const char *name, unsigned int size, 430 struct kmem_cache_args *args, 431 slab_flags_t flags) 432 { 433 struct kmem_cache_args kmem_default_args = {}; 434 435 /* Make sure we don't get passed garbage. */ 436 if (WARN_ON_ONCE(args)) 437 return ERR_PTR(-EINVAL); 438 439 return __kmem_cache_create_args(name, size, &kmem_default_args, flags); 440 } 441 442 /** 443 * kmem_cache_create - Create a kmem cache. 444 * @__name: A string which is used in /proc/slabinfo to identify this cache. 445 * @__object_size: The size of objects to be created in this cache. 446 * @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL 447 * means defaults will be used for all the arguments. 448 * 449 * This is currently implemented as a macro using ``_Generic()`` to call 450 * either the new variant of the function, or a legacy one. 451 * 452 * The new variant has 4 parameters: 453 * ``kmem_cache_create(name, object_size, args, flags)`` 454 * 455 * See __kmem_cache_create_args() which implements this. 456 * 457 * The legacy variant has 5 parameters: 458 * ``kmem_cache_create(name, object_size, align, flags, ctor)`` 459 * 460 * The align and ctor parameters map to the respective fields of 461 * &struct kmem_cache_args 462 * 463 * Context: Cannot be called within a interrupt, but can be interrupted. 464 * 465 * Return: a pointer to the cache on success, NULL on failure. 466 */ 467 #define kmem_cache_create(__name, __object_size, __args, ...) \ 468 _Generic((__args), \ 469 struct kmem_cache_args *: __kmem_cache_create_args, \ 470 void *: __kmem_cache_default_args, \ 471 default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__) 472 473 void kmem_cache_destroy(struct kmem_cache *s); 474 int kmem_cache_shrink(struct kmem_cache *s); 475 476 /* 477 * Please use this macro to create slab caches. Simply specify the 478 * name of the structure and maybe some flags that are listed above. 479 * 480 * The alignment of the struct determines object alignment. If you 481 * f.e. add ____cacheline_aligned_in_smp to the struct declaration 482 * then the objects will be properly aligned in SMP configurations. 483 */ 484 #define KMEM_CACHE(__struct, __flags) \ 485 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \ 486 &(struct kmem_cache_args) { \ 487 .align = __alignof__(struct __struct), \ 488 }, (__flags)) 489 490 /* 491 * To whitelist a single field for copying to/from usercopy, use this 492 * macro instead for KMEM_CACHE() above. 493 */ 494 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \ 495 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \ 496 &(struct kmem_cache_args) { \ 497 .align = __alignof__(struct __struct), \ 498 .useroffset = offsetof(struct __struct, __field), \ 499 .usersize = sizeof_field(struct __struct, __field), \ 500 }, (__flags)) 501 502 #ifdef CONFIG_KMALLOC_PARTITION_CACHES 503 typedef struct { unsigned long v; } kmalloc_token_t; 504 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM 505 extern unsigned long random_kmalloc_seed; 506 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = _CODE_LOCATION_ }) 507 #elif defined(CONFIG_KMALLOC_PARTITION_TYPED) 508 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = __builtin_infer_alloc_token(__VA_ARGS__) }) 509 #endif 510 #define DECL_TOKEN_PARAM(_token) , kmalloc_token_t (_token) 511 #define _PASS_TOKEN_PARAM(_token) , (_token) 512 #define PASS_TOKEN_PARAM(_token) (_token) 513 #define DECL_TOKEN_PARAMS(_size, _token) size_t (_size), kmalloc_token_t (_token) 514 #define PASS_TOKEN_PARAMS(_size, _token) (_size), (_token) 515 #else /* !CONFIG_KMALLOC_PARTITION_CACHES */ 516 typedef struct {} kmalloc_token_t; 517 #define __kmalloc_token(...) ((kmalloc_token_t){}) /* no-op */ 518 #define DECL_TOKEN_PARAM(_token) 519 #define _PASS_TOKEN_PARAM(_token) 520 #define PASS_TOKEN_PARAM(_token) ((kmalloc_token_t){}) 521 #define DECL_TOKEN_PARAMS(_size, _token) size_t (_size) 522 #define PASS_TOKEN_PARAMS(_size, _token) (_size) 523 #endif /* CONFIG_KMALLOC_PARTITION_CACHES */ 524 525 /* 526 * Common kmalloc functions provided by all allocators 527 */ 528 void * __must_check krealloc_node_align_noprof(const void *objp, 529 DECL_TOKEN_PARAMS(new_size, token), 530 unsigned long align, 531 gfp_t flags, int nid) __realloc_size(2); 532 #define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), 1, _f, NUMA_NO_NODE) 533 #if 0 /* kernel-doc */ 534 /** 535 * krealloc_node_align - reallocate memory. The contents will remain unchanged. 536 * @p: object to reallocate memory for. 537 * @new_size: how many bytes of memory are required. 538 * @align: desired alignment. 539 * @flags: the type of memory to allocate. 540 * @nid: NUMA node or NUMA_NO_NODE 541 * 542 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 543 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 544 * 545 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 546 * Documentation/core-api/memory-allocation.rst for more details. 547 * 548 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 549 * initial memory allocation, every subsequent call to this API for the same 550 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 551 * __GFP_ZERO is not fully honored by this API. 552 * 553 * When slub_debug_orig_size() is off, krealloc() only knows about the bucket 554 * size of an allocation (but not the exact size it was allocated with) and 555 * hence implements the following semantics for shrinking and growing buffers 556 * with __GFP_ZERO:: 557 * 558 * new bucket 559 * 0 size size 560 * |--------|----------------| 561 * | keep | zero | 562 * 563 * Otherwise, the original allocation size 'orig_size' could be used to 564 * precisely clear the requested size, and the new size will also be stored 565 * as the new 'orig_size'. 566 * 567 * In any case, the contents of the object pointed to are preserved up to the 568 * lesser of the new and old sizes. 569 * 570 * Return: pointer to the allocated memory or %NULL in case of error 571 */ 572 void *krealloc_node_align(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid); 573 #endif 574 #define krealloc_node_align(p, new_size, align, flags, nid) \ 575 alloc_hooks(krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(new_size, __kmalloc_token(new_size)), align, flags, nid)) 576 #define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n) 577 #define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE) 578 579 void kfree(const void *objp); 580 void kfree_nolock(const void *objp); 581 void kfree_sensitive(const void *objp); 582 583 DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T)) 584 DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T)) 585 586 size_t ksize(const void *objp); 587 588 #ifdef CONFIG_PRINTK 589 bool kmem_dump_obj(void *object); 590 #else 591 static inline bool kmem_dump_obj(void *object) { return false; } 592 #endif 593 594 /* 595 * Some archs want to perform DMA into kmalloc caches and need a guaranteed 596 * alignment larger than the alignment of a 64-bit integer. 597 * Setting ARCH_DMA_MINALIGN in arch headers allows that. 598 */ 599 #ifdef ARCH_HAS_DMA_MINALIGN 600 #if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN) 601 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN 602 #endif 603 #endif 604 605 #ifndef ARCH_KMALLOC_MINALIGN 606 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) 607 #elif ARCH_KMALLOC_MINALIGN > 8 608 #define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN 609 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) 610 #endif 611 612 /* 613 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. 614 * Intended for arches that get misalignment faults even for 64 bit integer 615 * aligned buffers. 616 */ 617 #ifndef ARCH_SLAB_MINALIGN 618 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 619 #endif 620 621 /* 622 * Arches can define this function if they want to decide the minimum slab 623 * alignment at runtime. The value returned by the function must be a power 624 * of two and >= ARCH_SLAB_MINALIGN. 625 */ 626 #ifndef arch_slab_minalign 627 static inline unsigned int arch_slab_minalign(void) 628 { 629 return ARCH_SLAB_MINALIGN; 630 } 631 #endif 632 633 /* 634 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN. 635 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN 636 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment. 637 */ 638 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN) 639 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN) 640 #define __assume_page_alignment __assume_aligned(PAGE_SIZE) 641 642 /* 643 * Kmalloc array related definitions 644 */ 645 646 /* 647 * SLUB directly allocates requests fitting in to an order-1 page 648 * (PAGE_SIZE*2). Larger requests are passed to the page allocator. 649 */ 650 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 651 #define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT) 652 #ifndef KMALLOC_SHIFT_LOW 653 #define KMALLOC_SHIFT_LOW 3 654 #endif 655 656 /* Maximum allocatable size */ 657 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) 658 /* Maximum size for which we actually use a slab cache */ 659 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) 660 /* Maximum order allocatable via the slab allocator */ 661 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) 662 663 /* 664 * Kmalloc subsystem. 665 */ 666 #ifndef KMALLOC_MIN_SIZE 667 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) 668 #endif 669 670 /* 671 * This restriction comes from byte sized index implementation. 672 * Page size is normally 2^12 bytes and, in this case, if we want to use 673 * byte sized index which can represent 2^8 entries, the size of the object 674 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16. 675 * If minimum size of kmalloc is less than 16, we use it as minimum object 676 * size and give up to use byte sized index. 677 */ 678 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ 679 (KMALLOC_MIN_SIZE) : 16) 680 681 #ifdef CONFIG_KMALLOC_PARTITION_CACHES 682 #define KMALLOC_PARTITION_CACHES_NR 15 // # of cache copies 683 #else 684 #define KMALLOC_PARTITION_CACHES_NR 0 685 #endif 686 687 /* 688 * Whenever changing this, take care of that kmalloc_type() and 689 * create_kmalloc_caches() still work as intended. 690 * 691 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP 692 * is for accounted but unreclaimable and non-dma objects. All the other 693 * kmem caches can have both accounted and unaccounted objects. 694 */ 695 enum kmalloc_cache_type { 696 KMALLOC_NORMAL = 0, 697 #ifndef CONFIG_ZONE_DMA 698 KMALLOC_DMA = KMALLOC_NORMAL, 699 #endif 700 #ifndef CONFIG_MEMCG 701 KMALLOC_CGROUP = KMALLOC_NORMAL, 702 #endif 703 KMALLOC_PARTITION_START = KMALLOC_NORMAL, 704 KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR, 705 #ifdef CONFIG_SLUB_TINY 706 KMALLOC_RECLAIM = KMALLOC_NORMAL, 707 #else 708 KMALLOC_RECLAIM, 709 #endif 710 #ifdef CONFIG_ZONE_DMA 711 KMALLOC_DMA, 712 #endif 713 #ifdef CONFIG_MEMCG 714 KMALLOC_CGROUP, 715 #endif 716 NR_KMALLOC_TYPES 717 }; 718 719 typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1]; 720 721 extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES]; 722 723 /* 724 * Define gfp bits that should not be set for KMALLOC_NORMAL. 725 */ 726 #define KMALLOC_NOT_NORMAL_BITS \ 727 (__GFP_RECLAIMABLE | \ 728 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \ 729 (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0)) 730 731 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, kmalloc_token_t token) 732 { 733 /* 734 * The most common case is KMALLOC_NORMAL, so test for it 735 * with a single branch for all the relevant flags. 736 */ 737 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0)) 738 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM 739 /* KMALLOC_PARTITION_CACHES_NR (=15) copies + the KMALLOC_NORMAL */ 740 return KMALLOC_PARTITION_START + hash_64(token.v ^ random_kmalloc_seed, 741 ilog2(KMALLOC_PARTITION_CACHES_NR + 1)); 742 #elif defined(CONFIG_KMALLOC_PARTITION_TYPED) 743 return KMALLOC_PARTITION_START + token.v; 744 #else 745 return KMALLOC_NORMAL; 746 #endif 747 748 /* 749 * At least one of the flags has to be set. Their priorities in 750 * decreasing order are: 751 * 1) __GFP_DMA 752 * 2) __GFP_RECLAIMABLE 753 * 3) __GFP_ACCOUNT 754 */ 755 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA)) 756 return KMALLOC_DMA; 757 if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE)) 758 return KMALLOC_RECLAIM; 759 else 760 return KMALLOC_CGROUP; 761 } 762 763 /* 764 * Figure out which kmalloc slab an allocation of a certain size 765 * belongs to. 766 * 0 = zero alloc 767 * 1 = 65 .. 96 bytes 768 * 2 = 129 .. 192 bytes 769 * n = 2^(n-1)+1 .. 2^n 770 * 771 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized; 772 * typical usage is via kmalloc_index() and therefore evaluated at compile-time. 773 * Callers where !size_is_constant should only be test modules, where runtime 774 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab(). 775 */ 776 static __always_inline unsigned int __kmalloc_index(size_t size, 777 bool size_is_constant) 778 { 779 if (!size) 780 return 0; 781 782 if (size <= KMALLOC_MIN_SIZE) 783 return KMALLOC_SHIFT_LOW; 784 785 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) 786 return 1; 787 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) 788 return 2; 789 if (size <= 8) return 3; 790 if (size <= 16) return 4; 791 if (size <= 32) return 5; 792 if (size <= 64) return 6; 793 if (size <= 128) return 7; 794 if (size <= 256) return 8; 795 if (size <= 512) return 9; 796 if (size <= 1024) return 10; 797 if (size <= 2 * 1024) return 11; 798 if (size <= 4 * 1024) return 12; 799 if (size <= 8 * 1024) return 13; 800 if (size <= 16 * 1024) return 14; 801 if (size <= 32 * 1024) return 15; 802 if (size <= 64 * 1024) return 16; 803 if (size <= 128 * 1024) return 17; 804 if (size <= 256 * 1024) return 18; 805 if (size <= 512 * 1024) return 19; 806 if (size <= 1024 * 1024) return 20; 807 if (size <= 2 * 1024 * 1024) return 21; 808 809 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) 810 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()"); 811 else 812 BUG(); 813 814 /* Will never be reached. Needed because the compiler may complain */ 815 return -1; 816 } 817 static_assert(PAGE_SHIFT <= 20); 818 #define kmalloc_index(s) __kmalloc_index(s, true) 819 820 #include <linux/alloc_tag.h> 821 822 /** 823 * kmem_cache_alloc - Allocate an object 824 * @cachep: The cache to allocate from. 825 * @flags: See kmalloc(). 826 * 827 * Allocate an object from this cache. 828 * See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags. 829 * 830 * Return: pointer to the new object or %NULL in case of error 831 */ 832 void *kmem_cache_alloc_noprof(struct kmem_cache *cachep, 833 gfp_t flags) __assume_slab_alignment __malloc; 834 #define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__)) 835 836 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, 837 gfp_t gfpflags) __assume_slab_alignment __malloc; 838 #define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__)) 839 840 /** 841 * kmem_cache_charge - memcg charge an already allocated slab memory 842 * @objp: address of the slab object to memcg charge 843 * @gfpflags: describe the allocation context 844 * 845 * kmem_cache_charge allows charging a slab object to the current memcg, 846 * primarily in cases where charging at allocation time might not be possible 847 * because the target memcg is not known (i.e. softirq context) 848 * 849 * The objp should be pointer returned by the slab allocator functions like 850 * kmalloc (with __GFP_ACCOUNT in flags) or kmem_cache_alloc. The memcg charge 851 * behavior can be controlled through gfpflags parameter, which affects how the 852 * necessary internal metadata can be allocated. Including __GFP_NOFAIL denotes 853 * that overcharging is requested instead of failure, but is not applied for the 854 * internal metadata allocation. 855 * 856 * There are several cases where it will return true even if the charging was 857 * not done: 858 * More specifically: 859 * 860 * 1. For !CONFIG_MEMCG or cgroup_disable=memory systems. 861 * 2. Already charged slab objects. 862 * 3. For slab objects from KMALLOC_NORMAL caches - allocated by kmalloc() 863 * without __GFP_ACCOUNT 864 * 4. Allocating internal metadata has failed 865 * 866 * Return: true if charge was successful otherwise false. 867 */ 868 bool kmem_cache_charge(void *objp, gfp_t gfpflags); 869 void kmem_cache_free(struct kmem_cache *s, void *objp); 870 871 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags, 872 unsigned int useroffset, unsigned int usersize, 873 void (*ctor)(void *)); 874 875 /* 876 * Bulk allocation and freeing operations. These are accelerated in an 877 * allocator specific way to avoid taking locks repeatedly or building 878 * metadata structures unnecessarily. 879 * 880 * Note that interrupts must be enabled when calling these functions. 881 */ 882 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); 883 884 bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, 885 size_t size, void **p); 886 #define kmem_cache_alloc_bulk(...) \ 887 alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) 888 889 static __always_inline void kfree_bulk(size_t size, void **p) 890 { 891 kmem_cache_free_bulk(NULL, size, p); 892 } 893 894 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags, 895 int node) __assume_slab_alignment __malloc; 896 #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__)) 897 898 struct slab_sheaf * 899 kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size); 900 901 int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, 902 struct slab_sheaf **sheafp, unsigned int size); 903 904 void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, 905 struct slab_sheaf *sheaf); 906 907 void *kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *cachep, gfp_t gfp, 908 struct slab_sheaf *sheaf) __assume_slab_alignment __malloc; 909 #define kmem_cache_alloc_from_sheaf(...) \ 910 alloc_hooks(kmem_cache_alloc_from_sheaf_noprof(__VA_ARGS__)) 911 912 unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf); 913 914 /* 915 * These macros allow declaring a kmem_buckets * parameter alongside size, which 916 * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call 917 * sites don't have to pass NULL. 918 */ 919 #ifdef CONFIG_SLAB_BUCKETS 920 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b) 921 #define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b) 922 #define PASS_BUCKET_PARAM(_b) (_b) 923 #else 924 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size) 925 #define PASS_BUCKET_PARAMS(_size, _b) (_size) 926 #define PASS_BUCKET_PARAM(_b) NULL 927 #endif 928 929 #define DECL_KMALLOC_PARAMS(_size, _b, _token) DECL_BUCKET_PARAMS(_size, _b) \ 930 DECL_TOKEN_PARAM(_token) 931 932 #define PASS_KMALLOC_PARAMS(_size, _b, _token) PASS_BUCKET_PARAMS(_size, _b) \ 933 _PASS_TOKEN_PARAM(_token) 934 935 /* 936 * The following functions are not to be used directly and are intended only 937 * for internal use from kmalloc() and kmalloc_node() 938 * with the exception of kunit tests 939 */ 940 941 void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags) 942 __assume_kmalloc_alignment __alloc_size(1); 943 944 void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node) 945 __assume_kmalloc_alignment __alloc_size(1); 946 947 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size) 948 __assume_kmalloc_alignment __alloc_size(3); 949 950 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 951 int node, size_t size) 952 __assume_kmalloc_alignment __alloc_size(4); 953 954 void *__kmalloc_large_noprof(size_t size, gfp_t flags) 955 __assume_page_alignment __alloc_size(1); 956 957 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 958 __assume_page_alignment __alloc_size(1); 959 960 static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token) 961 { 962 if (__builtin_constant_p(size) && size) { 963 unsigned int index; 964 965 if (size > KMALLOC_MAX_CACHE_SIZE) 966 return __kmalloc_large_noprof(size, flags); 967 968 index = kmalloc_index(size); 969 return __kmalloc_cache_noprof( 970 kmalloc_caches[kmalloc_type(flags, token)][index], 971 flags, size); 972 } 973 return __kmalloc_noprof(PASS_TOKEN_PARAMS(size, token), flags); 974 } 975 #define kmalloc_noprof(...) _kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 976 #if 0 /* kernel-doc */ 977 /** 978 * kmalloc - allocate kernel memory 979 * @size: how many bytes of memory are required. 980 * @flags: describe the allocation context 981 * 982 * kmalloc is the normal method of allocating memory 983 * for objects smaller than page size in the kernel. 984 * 985 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN 986 * bytes. For @size of power of two bytes, the alignment is also guaranteed 987 * to be at least to the size. For other sizes, the alignment is guaranteed to 988 * be at least the largest power-of-two divisor of @size. 989 * 990 * The @flags argument may be one of the GFP flags defined at 991 * include/linux/gfp_types.h and described at 992 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>` 993 * 994 * The recommended usage of the @flags is described at 995 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>` 996 * 997 * Below is a brief outline of the most useful GFP flags 998 * 999 * %GFP_KERNEL 1000 * Allocate normal kernel ram. May sleep. 1001 * 1002 * %GFP_NOWAIT 1003 * Allocation will not sleep. 1004 * 1005 * %GFP_ATOMIC 1006 * Allocation will not sleep. May use emergency pools. 1007 * 1008 * Also it is possible to set different flags by OR'ing 1009 * in one or more of the following additional @flags: 1010 * 1011 * %__GFP_ZERO 1012 * Zero the allocated memory before returning. Also see kzalloc(). 1013 * 1014 * %__GFP_HIGH 1015 * This allocation has high priority and may use emergency pools. 1016 * 1017 * %__GFP_NOFAIL 1018 * Indicate that this allocation is in no way allowed to fail 1019 * (think twice before using). 1020 * 1021 * %__GFP_NORETRY 1022 * If memory is not immediately available, 1023 * then give up at once. 1024 * 1025 * %__GFP_NOWARN 1026 * If allocation fails, don't issue any warnings. 1027 * 1028 * %__GFP_RETRY_MAYFAIL 1029 * Try really hard to succeed the allocation but fail 1030 * eventually. 1031 */ 1032 void *kmalloc(size_t size, gfp_t flags); 1033 #endif 1034 #define kmalloc(size, flags) alloc_hooks(kmalloc_noprof(size, flags)) 1035 1036 void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node); 1037 #define kmalloc_nolock_noprof(_s, _f, _n) _kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), _f, _n) 1038 #if 0 /* kernel-doc */ 1039 /** 1040 * kmalloc_nolock - Allocate an object of given size from any context. 1041 * @size: size to allocate 1042 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT and __GFP_ZERO allowed. Also 1043 * __GFP_NOWARN and __GFP_NOMEMALLOC are allowed but added internally thus not 1044 * necessary. 1045 * @node: node number of the target node. 1046 * 1047 * Return: pointer to the new object or NULL in case of error. 1048 * NULL does not mean EBUSY or EAGAIN. It means ENOMEM. 1049 * There is no reason to call it again and expect !NULL. 1050 */ 1051 void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); 1052 #endif 1053 #define kmalloc_nolock(size, gfp_flags, node) alloc_hooks(kmalloc_nolock_noprof(size, gfp_flags, node)) 1054 1055 /** 1056 * __alloc_objs - Allocate objects of a given type using 1057 * @KMALLOC: which size-based kmalloc wrapper to allocate with. 1058 * @GFP: GFP flags for the allocation. 1059 * @TYPE: type to allocate space for. 1060 * @COUNT: how many @TYPE objects to allocate. 1061 * 1062 * Returns: Newly allocated pointer to (first) @TYPE of @COUNT-many 1063 * allocated @TYPE objects, or NULL on failure. 1064 */ 1065 #define __alloc_objs(KMALLOC, GFP, TYPE, COUNT) \ 1066 ({ \ 1067 const size_t __obj_size = size_mul(sizeof(TYPE), COUNT); \ 1068 (TYPE *)KMALLOC(__obj_size, GFP); \ 1069 }) 1070 1071 /** 1072 * __alloc_flex - Allocate an object that has a trailing flexible array 1073 * @KMALLOC: kmalloc wrapper function to use for allocation. 1074 * @GFP: GFP flags for the allocation. 1075 * @TYPE: type of structure to allocate space for. 1076 * @FAM: The name of the flexible array member of @TYPE structure. 1077 * @COUNT: how many @FAM elements to allocate space for. 1078 * 1079 * Returns: Newly allocated pointer to @TYPE with @COUNT-many trailing 1080 * @FAM elements, or NULL on failure or if @COUNT cannot be represented 1081 * by the member of @TYPE that counts the @FAM elements (annotated via 1082 * __counted_by()). 1083 */ 1084 #define __alloc_flex(KMALLOC, GFP, TYPE, FAM, COUNT) \ 1085 ({ \ 1086 const size_t __count = (COUNT); \ 1087 const size_t __obj_size = struct_size_t(TYPE, FAM, __count); \ 1088 TYPE *__obj_ptr = KMALLOC(__obj_size, GFP); \ 1089 if (__obj_ptr) \ 1090 __set_flex_counter(__obj_ptr->FAM, __count); \ 1091 __obj_ptr; \ 1092 }) 1093 1094 /** 1095 * kmalloc_obj - Allocate a single instance of the given type 1096 * @VAR_OR_TYPE: Variable or type to allocate. 1097 * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). 1098 * 1099 * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL 1100 * on failure. 1101 */ 1102 #define kmalloc_obj(VAR_OR_TYPE, ...) \ 1103 __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), 1) 1104 1105 /** 1106 * kmalloc_objs - Allocate an array of the given type 1107 * @VAR_OR_TYPE: Variable or type to allocate an array of. 1108 * @COUNT: How many elements in the array. 1109 * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). 1110 * 1111 * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success, 1112 * or NULL on failure. 1113 */ 1114 #define kmalloc_objs(VAR_OR_TYPE, COUNT, ...) \ 1115 __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), COUNT) 1116 1117 /** 1118 * kmalloc_flex - Allocate a single instance of the given flexible structure 1119 * @VAR_OR_TYPE: Variable or type to allocate (with its flex array). 1120 * @FAM: The name of the flexible array member of the structure. 1121 * @COUNT: How many flexible array member elements are desired. 1122 * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). 1123 * 1124 * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on 1125 * failure. If @FAM has been annotated with __counted_by(), the allocation 1126 * will immediately fail if @COUNT is larger than what the type of the 1127 * struct's counter variable can represent. 1128 */ 1129 #define kmalloc_flex(VAR_OR_TYPE, FAM, COUNT, ...) \ 1130 __alloc_flex(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), FAM, COUNT) 1131 1132 /* All kzalloc aliases for kmalloc_(obj|objs|flex). */ 1133 #define kzalloc_obj(P, ...) \ 1134 __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), 1) 1135 #define kzalloc_objs(P, COUNT, ...) \ 1136 __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) 1137 #define kzalloc_flex(P, FAM, COUNT, ...) \ 1138 __alloc_flex(kzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) 1139 1140 /* All kvmalloc aliases for kmalloc_(obj|objs|flex). */ 1141 #define kvmalloc_obj(P, ...) \ 1142 __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), 1) 1143 #define kvmalloc_objs(P, COUNT, ...) \ 1144 __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) 1145 #define kvmalloc_flex(P, FAM, COUNT, ...) \ 1146 __alloc_flex(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) 1147 1148 /* All kvzalloc aliases for kmalloc_(obj|objs|flex). */ 1149 #define kvzalloc_obj(P, ...) \ 1150 __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), 1) 1151 #define kvzalloc_objs(P, COUNT, ...) \ 1152 __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) 1153 #define kvzalloc_flex(P, FAM, COUNT, ...) \ 1154 __alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) 1155 1156 #define kmem_buckets_alloc(_b, _size, _flags) \ 1157 alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE)) 1158 1159 #define kmem_buckets_alloc_node_track_caller(_b, _size, _flags, _node) \ 1160 alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, _node, _RET_IP_)) 1161 1162 #define kmem_buckets_alloc_track_caller(_b, _size, _flags) \ 1163 kmem_buckets_alloc_node_track_caller(_b, _size, _flags, NUMA_NO_NODE) 1164 1165 static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token) 1166 { 1167 if (__builtin_constant_p(size) && size) { 1168 unsigned int index; 1169 1170 if (size > KMALLOC_MAX_CACHE_SIZE) 1171 return __kmalloc_large_node_noprof(size, flags, node); 1172 1173 index = kmalloc_index(size); 1174 return __kmalloc_cache_node_noprof( 1175 kmalloc_caches[kmalloc_type(flags, token)][index], 1176 flags, node, size); 1177 } 1178 return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags, node); 1179 } 1180 #define kmalloc_node_noprof(...) _kmalloc_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1181 #define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__)) 1182 1183 static inline __alloc_size(1, 2) void *_kmalloc_array_noprof(size_t n, size_t size, gfp_t flags, kmalloc_token_t token) 1184 { 1185 size_t bytes; 1186 1187 if (unlikely(check_mul_overflow(n, size, &bytes))) 1188 return NULL; 1189 return _kmalloc_noprof(bytes, flags, token); 1190 } 1191 #define kmalloc_array_noprof(...) _kmalloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1192 #if 0 /* kernel-doc */ 1193 /** 1194 * kmalloc_array - allocate memory for an array. 1195 * @n: number of elements. 1196 * @size: element size. 1197 * @flags: the type of memory to allocate (see kmalloc). 1198 */ 1199 void *kmalloc_array(size_t n, size_t size, gfp_t flags); 1200 #endif 1201 #define kmalloc_array(n, size, flags) alloc_hooks(kmalloc_array_noprof(n, size, flags)) 1202 1203 static inline __realloc_size(2, 3) void * __must_check _krealloc_array_noprof(void *p, 1204 size_t new_n, 1205 size_t new_size, 1206 gfp_t flags, kmalloc_token_t token) 1207 { 1208 size_t bytes; 1209 1210 if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) 1211 return NULL; 1212 1213 return krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(bytes, token), 1, flags, NUMA_NO_NODE); 1214 } 1215 #define krealloc_array_noprof(...) _krealloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1216 #if 0 /* kernel-doc */ 1217 /** 1218 * krealloc_array - reallocate memory for an array. 1219 * @p: pointer to the memory chunk to reallocate 1220 * @new_n: new number of elements to alloc 1221 * @new_size: new size of a single member of the array 1222 * @flags: the type of memory to allocate (see kmalloc) 1223 * 1224 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 1225 * initial memory allocation, every subsequent call to this API for the same 1226 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 1227 * __GFP_ZERO is not fully honored by this API. 1228 * 1229 * See krealloc_noprof() for further details. 1230 * 1231 * In any case, the contents of the object pointed to are preserved up to the 1232 * lesser of the new and old sizes. 1233 */ 1234 void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t flags); 1235 #endif 1236 #define krealloc_array(p, new_n, new_size, flags) alloc_hooks(krealloc_array_noprof(p, new_n, new_size, flags)) 1237 1238 /** 1239 * kcalloc - allocate memory for an array. The memory is set to zero. 1240 * @n: number of elements. 1241 * @size: element size. 1242 * @flags: the type of memory to allocate (see kmalloc). 1243 */ 1244 #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO) 1245 1246 void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node, 1247 unsigned long caller) __alloc_size(1); 1248 #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \ 1249 __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(size, NULL, __kmalloc_token(size)), flags, node, caller) 1250 #define kmalloc_node_track_caller(...) \ 1251 alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_)) 1252 1253 /* 1254 * kmalloc_track_caller is a special version of kmalloc that records the 1255 * calling function of the routine calling it for slab leak tracking instead 1256 * of just the calling function (confusing, eh?). 1257 * It's useful when the call to kmalloc comes from a widely-used standard 1258 * allocator where we care about the real place the memory allocation 1259 * request comes from. 1260 */ 1261 #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE) 1262 1263 #define kmalloc_track_caller_noprof(...) \ 1264 kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_) 1265 1266 static inline __alloc_size(1, 2) void *_kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, 1267 int node, kmalloc_token_t token) 1268 { 1269 size_t bytes; 1270 1271 if (unlikely(check_mul_overflow(n, size, &bytes))) 1272 return NULL; 1273 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 1274 return _kmalloc_node_noprof(bytes, flags, node, token); 1275 return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), flags, node); 1276 } 1277 #define kmalloc_array_node_noprof(...) _kmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1278 #define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__)) 1279 1280 #define kcalloc_node(_n, _size, _flags, _node) \ 1281 kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node) 1282 1283 /* 1284 * Shortcuts 1285 */ 1286 #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO) 1287 1288 static inline __alloc_size(1) void *_kzalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token) 1289 { 1290 return _kmalloc_noprof(size, flags | __GFP_ZERO, token); 1291 } 1292 #define kzalloc_noprof(...) _kzalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1293 #if 0 /* kernel-doc */ 1294 /** 1295 * kzalloc - allocate memory. The memory is set to zero. 1296 * @size: how many bytes of memory are required. 1297 * @flags: the type of memory to allocate (see kmalloc). 1298 */ 1299 void *kzalloc(size_t size, gfp_t flags); 1300 #endif 1301 #define kzalloc(size, flags) alloc_hooks(kzalloc_noprof(size, flags)) 1302 #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) 1303 1304 void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align, 1305 gfp_t flags, int node) __alloc_size(1); 1306 #define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ 1307 __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, NULL, __kmalloc_token(_size)), _align, _flags, _node) 1308 #define kvmalloc_node_align(...) \ 1309 alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) 1310 #if 0 /* kernel-doc */ 1311 /** 1312 * kvmalloc_node - attempt to allocate physically contiguous memory, but upon 1313 * failure, fall back to non-contiguous (vmalloc) allocation. 1314 * @size: size of the request. 1315 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 1316 * @node: numa node to allocate from 1317 * 1318 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 1319 * Documentation/core-api/memory-allocation.rst for more details. 1320 * 1321 * Uses kmalloc to get the memory but if the allocation fails then falls back 1322 * to the vmalloc allocator. Use kvfree for freeing the memory. 1323 * 1324 * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not. 1325 * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is 1326 * preferable to the vmalloc fallback, due to visible performance drawbacks. 1327 * 1328 * Return: pointer to the allocated memory of %NULL in case of failure 1329 */ 1330 void *kvmalloc_node(size_t size, gfp_t flags, int node); 1331 #endif 1332 #define kvmalloc_node(size, flags, node) kvmalloc_node_align(size, 1, flags, node) 1333 #define kvmalloc_node_noprof(size, flags, node) \ 1334 kvmalloc_node_align_noprof(size, 1, flags, node) 1335 #define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE) 1336 #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) 1337 #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) 1338 1339 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node) 1340 1341 #define kmem_buckets_valloc(_b, _size, _flags) \ 1342 alloc_hooks(__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), 1, _flags, NUMA_NO_NODE)) 1343 1344 static inline __alloc_size(1, 2) void * 1345 _kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmalloc_token_t token) 1346 { 1347 size_t bytes; 1348 1349 if (unlikely(check_mul_overflow(n, size, &bytes))) 1350 return NULL; 1351 1352 return __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), 1, flags, node); 1353 } 1354 #define kvmalloc_array_node_noprof(...) _kvmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1355 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE) 1356 #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node) 1357 #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE) 1358 1359 #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__)) 1360 #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) 1361 #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) 1362 1363 void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align, 1364 gfp_t flags, int nid) __realloc_size(2); 1365 #if 0 /* kernel-doc */ 1366 /** 1367 * kvrealloc_node_align - reallocate memory; contents remain unchanged 1368 * @p: object to reallocate memory for 1369 * @size: the size to reallocate 1370 * @align: desired alignment 1371 * @flags: the flags for the page level allocator 1372 * @nid: NUMA node id 1373 * 1374 * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 1375 * and @p is not a %NULL pointer, the object pointed to is freed. 1376 * 1377 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 1378 * Documentation/core-api/memory-allocation.rst for more details. 1379 * 1380 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 1381 * initial memory allocation, every subsequent call to this API for the same 1382 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 1383 * __GFP_ZERO is not fully honored by this API. 1384 * 1385 * In any case, the contents of the object pointed to are preserved up to the 1386 * lesser of the new and old sizes. 1387 * 1388 * This function must not be called concurrently with itself or kvfree() for the 1389 * same memory allocation. 1390 * 1391 * Return: pointer to the allocated memory or %NULL in case of error 1392 */ 1393 void *kvrealloc_node_align(const void *p, size_t size, unsigned long align, gfp_t flags, int nid); 1394 #endif 1395 #define kvrealloc_node_align(p, size, align, flags, nid) \ 1396 alloc_hooks(kvrealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, __kmalloc_token(size)), align, flags, nid)) 1397 #define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n) 1398 #define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE) 1399 1400 extern void kvfree(const void *addr); 1401 DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T)) 1402 1403 extern void kvfree_atomic(const void *addr); 1404 DEFINE_FREE(kvfree_atomic, void *, if (!IS_ERR_OR_NULL(_T)) kvfree_atomic(_T)) 1405 1406 extern void kvfree_sensitive(const void *addr, size_t len); 1407 1408 unsigned int kmem_cache_size(struct kmem_cache *s); 1409 1410 #ifndef CONFIG_KVFREE_RCU_BATCHED 1411 static inline void kvfree_rcu_barrier(void) 1412 { 1413 rcu_barrier(); 1414 } 1415 1416 static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) 1417 { 1418 rcu_barrier(); 1419 } 1420 1421 static inline void kfree_rcu_scheduler_running(void) { } 1422 #else 1423 void kvfree_rcu_barrier(void); 1424 1425 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); 1426 1427 void kfree_rcu_scheduler_running(void); 1428 #endif 1429 1430 /** 1431 * kmalloc_size_roundup - Report allocation bucket size for the given size 1432 * 1433 * @size: Number of bytes to round up from. 1434 * 1435 * This returns the number of bytes that would be available in a kmalloc() 1436 * allocation of @size bytes. For example, a 126 byte request would be 1437 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly 1438 * for the general-purpose kmalloc()-based allocations, and is not for the 1439 * pre-sized kmem_cache_alloc()-based allocations.) 1440 * 1441 * Use this to kmalloc() the full bucket size ahead of time instead of using 1442 * ksize() to query the size after an allocation. 1443 */ 1444 size_t kmalloc_size_roundup(size_t size); 1445 1446 void __init kmem_cache_init_late(void); 1447 void __init kvfree_rcu_init(void); 1448 1449 #endif /* _LINUX_SLAB_H */ 1450