1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk). 4 * 5 * (C) SGI 2006, Christoph Lameter 6 * Cleaned up and restructured to ease the addition of alternative 7 * implementations of SLAB allocators. 8 * (C) Linux Foundation 2008-2013 9 * Unified interface for all slab allocators 10 */ 11 12 #ifndef _LINUX_SLAB_H 13 #define _LINUX_SLAB_H 14 15 #include <linux/bug.h> 16 #include <linux/cache.h> 17 #include <linux/gfp.h> 18 #include <linux/overflow.h> 19 #include <linux/types.h> 20 #include <linux/rcupdate.h> 21 #include <linux/workqueue.h> 22 #include <linux/percpu-refcount.h> 23 #include <linux/cleanup.h> 24 #include <linux/hash.h> 25 26 enum _slab_flag_bits { 27 _SLAB_CONSISTENCY_CHECKS, 28 _SLAB_RED_ZONE, 29 _SLAB_POISON, 30 _SLAB_KMALLOC, 31 _SLAB_HWCACHE_ALIGN, 32 _SLAB_CACHE_DMA, 33 _SLAB_CACHE_DMA32, 34 _SLAB_STORE_USER, 35 _SLAB_PANIC, 36 _SLAB_TYPESAFE_BY_RCU, 37 _SLAB_TRACE, 38 #ifdef CONFIG_DEBUG_OBJECTS 39 _SLAB_DEBUG_OBJECTS, 40 #endif 41 _SLAB_NOLEAKTRACE, 42 _SLAB_NO_MERGE, 43 #ifdef CONFIG_FAILSLAB 44 _SLAB_FAILSLAB, 45 #endif 46 #ifdef CONFIG_MEMCG 47 _SLAB_ACCOUNT, 48 #endif 49 #ifdef CONFIG_KASAN_GENERIC 50 _SLAB_KASAN, 51 #endif 52 _SLAB_NO_USER_FLAGS, 53 #ifdef CONFIG_KFENCE 54 _SLAB_SKIP_KFENCE, 55 #endif 56 #ifndef CONFIG_SLUB_TINY 57 _SLAB_RECLAIM_ACCOUNT, 58 #endif 59 _SLAB_OBJECT_POISON, 60 _SLAB_CMPXCHG_DOUBLE, 61 _SLAB_NO_OBJ_EXT, 62 #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) 63 _SLAB_OBJ_EXT_IN_OBJ, 64 #endif 65 _SLAB_FLAGS_LAST_BIT 66 }; 67 68 #define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr))) 69 #define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U)) 70 71 /* 72 * Flags to pass to kmem_cache_create(). 73 * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op 74 */ 75 /* DEBUG: Perform (expensive) checks on alloc/free */ 76 #define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS) 77 /* DEBUG: Red zone objs in a cache */ 78 #define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE) 79 /* DEBUG: Poison objects */ 80 #define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON) 81 /* Indicate a kmalloc slab */ 82 #define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC) 83 /** 84 * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries. 85 * 86 * Sufficiently large objects are aligned on cache line boundary. For object 87 * size smaller than a half of cache line size, the alignment is on the half of 88 * cache line size. In general, if object size is smaller than 1/2^n of cache 89 * line size, the alignment is adjusted to 1/2^n. 90 * 91 * If explicit alignment is also requested by the respective 92 * &struct kmem_cache_args field, the greater of both is alignments is applied. 93 */ 94 #define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN) 95 /* Use GFP_DMA memory */ 96 #define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA) 97 /* Use GFP_DMA32 memory */ 98 #define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32) 99 /* DEBUG: Store the last owner for bug hunting */ 100 #define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER) 101 /* Panic if kmem_cache_create() fails */ 102 #define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC) 103 /** 104 * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 105 * 106 * This delays freeing the SLAB page by a grace period, it does _NOT_ 107 * delay object freeing. This means that if you do kmem_cache_free() 108 * that memory location is free to be reused at any time. Thus it may 109 * be possible to see another object there in the same RCU grace period. 110 * 111 * This feature only ensures the memory location backing the object 112 * stays valid, the trick to using this is relying on an independent 113 * object validation pass. Something like: 114 * 115 * :: 116 * 117 * begin: 118 * rcu_read_lock(); 119 * obj = lockless_lookup(key); 120 * if (obj) { 121 * if (!try_get_ref(obj)) // might fail for free objects 122 * rcu_read_unlock(); 123 * goto begin; 124 * 125 * if (obj->key != key) { // not the object we expected 126 * put_ref(obj); 127 * rcu_read_unlock(); 128 * goto begin; 129 * } 130 * } 131 * rcu_read_unlock(); 132 * 133 * This is useful if we need to approach a kernel structure obliquely, 134 * from its address obtained without the usual locking. We can lock 135 * the structure to stabilize it and check it's still at the given address, 136 * only if we can be sure that the memory has not been meanwhile reused 137 * for some other kind of object (which our subsystem's lock might corrupt). 138 * 139 * rcu_read_lock before reading the address, then rcu_read_unlock after 140 * taking the spinlock within the structure expected at that address. 141 * 142 * Note that object identity check has to be done *after* acquiring a 143 * reference, therefore user has to ensure proper ordering for loads. 144 * Similarly, when initializing objects allocated with SLAB_TYPESAFE_BY_RCU, 145 * the newly allocated object has to be fully initialized *before* its 146 * refcount gets initialized and proper ordering for stores is required. 147 * refcount_{add|inc}_not_zero_acquire() and refcount_set_release() are 148 * designed with the proper fences required for reference counting objects 149 * allocated with SLAB_TYPESAFE_BY_RCU. 150 * 151 * Note that it is not possible to acquire a lock within a structure 152 * allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference 153 * as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages 154 * are not zeroed before being given to the slab, which means that any 155 * locks must be initialized after each and every kmem_struct_alloc(). 156 * Alternatively, make the ctor passed to kmem_cache_create() initialize 157 * the locks at page-allocation time, as is done in __i915_request_ctor(), 158 * sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers 159 * to safely acquire those ctor-initialized locks under rcu_read_lock() 160 * protection. 161 * 162 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. 163 */ 164 #define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU) 165 /* Trace allocations and frees */ 166 #define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE) 167 168 /* Flag to prevent checks on free */ 169 #ifdef CONFIG_DEBUG_OBJECTS 170 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS) 171 #else 172 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED 173 #endif 174 175 /* Avoid kmemleak tracing */ 176 #define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE) 177 178 /* 179 * Prevent merging with compatible kmem caches. This flag should be used 180 * cautiously. Valid use cases: 181 * 182 * - caches created for self-tests (e.g. kunit) 183 * - general caches created and used by a subsystem, only when a 184 * (subsystem-specific) debug option is enabled 185 * - performance critical caches, should be very rare and consulted with slab 186 * maintainers, and not used together with CONFIG_SLUB_TINY 187 */ 188 #define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE) 189 190 /* Fault injection mark */ 191 #ifdef CONFIG_FAILSLAB 192 # define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB) 193 #else 194 # define SLAB_FAILSLAB __SLAB_FLAG_UNUSED 195 #endif 196 /** 197 * define SLAB_ACCOUNT - Account allocations to memcg. 198 * 199 * All object allocations from this cache will be memcg accounted, regardless of 200 * __GFP_ACCOUNT being or not being passed to individual allocations. 201 */ 202 #ifdef CONFIG_MEMCG 203 # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) 204 #else 205 # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED 206 #endif 207 208 #ifdef CONFIG_KASAN_GENERIC 209 #define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN) 210 #else 211 #define SLAB_KASAN __SLAB_FLAG_UNUSED 212 #endif 213 214 /* 215 * Ignore user specified debugging flags. 216 * Intended for caches created for self-tests so they have only flags 217 * specified in the code and other flags are ignored. 218 */ 219 #define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS) 220 221 #ifdef CONFIG_KFENCE 222 #define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE) 223 #else 224 #define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED 225 #endif 226 227 /* The following flags affect the page allocator grouping pages by mobility */ 228 /** 229 * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable. 230 * 231 * Use this flag for caches that have an associated shrinker. As a result, slab 232 * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by 233 * mobility, and are accounted in SReclaimable counter in /proc/meminfo 234 */ 235 #ifndef CONFIG_SLUB_TINY 236 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT) 237 #else 238 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED 239 #endif 240 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ 241 242 /* Slab created using create_boot_cache */ 243 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT) 244 245 #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) 246 #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ) 247 #else 248 #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_UNUSED 249 #endif 250 251 /* 252 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. 253 * 254 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault. 255 * 256 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can. 257 * Both make kfree a no-op. 258 */ 259 #define ZERO_SIZE_PTR ((void *)16) 260 261 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ 262 (unsigned long)ZERO_SIZE_PTR) 263 264 #include <linux/kasan.h> 265 266 struct list_lru; 267 struct mem_cgroup; 268 /* 269 * struct kmem_cache related prototypes 270 */ 271 bool slab_is_available(void); 272 273 /** 274 * struct kmem_cache_args - Less common arguments for kmem_cache_create() 275 * 276 * Any uninitialized fields of the structure are interpreted as unused. The 277 * exception is @freeptr_offset where %0 is a valid value, so 278 * @use_freeptr_offset must be also set to %true in order to interpret the field 279 * as used. For @useroffset %0 is also valid, but only with non-%0 280 * @usersize. 281 * 282 * When %NULL args is passed to kmem_cache_create(), it is equivalent to all 283 * fields unused. 284 */ 285 struct kmem_cache_args { 286 /** 287 * @align: The required alignment for the objects. 288 * 289 * %0 means no specific alignment is requested. 290 */ 291 unsigned int align; 292 /** 293 * @useroffset: Usercopy region offset. 294 * 295 * %0 is a valid offset, when @usersize is non-%0 296 */ 297 unsigned int useroffset; 298 /** 299 * @usersize: Usercopy region size. 300 * 301 * %0 means no usercopy region is specified. 302 */ 303 unsigned int usersize; 304 /** 305 * @freeptr_offset: Custom offset for the free pointer 306 * in caches with &SLAB_TYPESAFE_BY_RCU or @ctor 307 * 308 * By default, &SLAB_TYPESAFE_BY_RCU and @ctor caches place the free 309 * pointer outside of the object. This might cause the object to grow 310 * in size. Cache creators that have a reason to avoid this can specify 311 * a custom free pointer offset in their data structure where the free 312 * pointer will be placed. 313 * 314 * For caches with &SLAB_TYPESAFE_BY_RCU, the caller must ensure that 315 * the free pointer does not overlay fields required to guard against 316 * object recycling (See &SLAB_TYPESAFE_BY_RCU for details). 317 * 318 * For caches with @ctor, the caller must ensure that the free pointer 319 * does not overlay fields initialized by the constructor. 320 * 321 * Currently, only caches with &SLAB_TYPESAFE_BY_RCU or @ctor 322 * may specify @freeptr_offset. 323 * 324 * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset 325 * is specified, @use_freeptr_offset must be set %true. 326 */ 327 unsigned int freeptr_offset; 328 /** 329 * @use_freeptr_offset: Whether a @freeptr_offset is used. 330 */ 331 bool use_freeptr_offset; 332 /** 333 * @ctor: A constructor for the objects. 334 * 335 * The constructor is invoked for each object in a newly allocated slab 336 * page. It is the cache user's responsibility to free object in the 337 * same state as after calling the constructor, or deal appropriately 338 * with any differences between a freshly constructed and a reallocated 339 * object. 340 * 341 * %NULL means no constructor. 342 */ 343 void (*ctor)(void *); 344 /** 345 * @sheaf_capacity: Enable sheaves of given capacity for the cache. 346 * 347 * With a non-zero value, allocations from the cache go through caching 348 * arrays called sheaves. Each cpu has a main sheaf that's always 349 * present, and a spare sheaf that may be not present. When both become 350 * empty, there's an attempt to replace an empty sheaf with a full sheaf 351 * from the per-node barn. 352 * 353 * When no full sheaf is available, and gfp flags allow blocking, a 354 * sheaf is allocated and filled from slab(s) using bulk allocation. 355 * Otherwise the allocation falls back to the normal operation 356 * allocating a single object from a slab. 357 * 358 * Analogically when freeing and both percpu sheaves are full, the barn 359 * may replace it with an empty sheaf, unless it's over capacity. In 360 * that case a sheaf is bulk freed to slab pages. 361 * 362 * The sheaves do not enforce NUMA placement of objects, so allocations 363 * via kmem_cache_alloc_node() with a node specified other than 364 * NUMA_NO_NODE will bypass them. 365 * 366 * Bulk allocation and free operations also try to use the cpu sheaves 367 * and barn, but fallback to using slab pages directly. 368 * 369 * When slub_debug is enabled for the cache, the sheaf_capacity argument 370 * is ignored. 371 * 372 * %0 means no sheaves will be created. 373 */ 374 unsigned int sheaf_capacity; 375 }; 376 377 struct kmem_cache *__kmem_cache_create_args(const char *name, 378 unsigned int object_size, 379 struct kmem_cache_args *args, 380 slab_flags_t flags); 381 static inline struct kmem_cache * 382 __kmem_cache_create(const char *name, unsigned int size, unsigned int align, 383 slab_flags_t flags, void (*ctor)(void *)) 384 { 385 struct kmem_cache_args kmem_args = { 386 .align = align, 387 .ctor = ctor, 388 }; 389 390 return __kmem_cache_create_args(name, size, &kmem_args, flags); 391 } 392 393 /** 394 * kmem_cache_create_usercopy - Create a kmem cache with a region suitable 395 * for copying to userspace. 396 * @name: A string which is used in /proc/slabinfo to identify this cache. 397 * @size: The size of objects to be created in this cache. 398 * @align: The required alignment for the objects. 399 * @flags: SLAB flags 400 * @useroffset: Usercopy region offset 401 * @usersize: Usercopy region size 402 * @ctor: A constructor for the objects, or %NULL. 403 * 404 * This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY() 405 * if whitelisting a single field is sufficient, or kmem_cache_create() with 406 * the necessary parameters passed via the args parameter (see 407 * &struct kmem_cache_args) 408 * 409 * Return: a pointer to the cache on success, NULL on failure. 410 */ 411 static inline struct kmem_cache * 412 kmem_cache_create_usercopy(const char *name, unsigned int size, 413 unsigned int align, slab_flags_t flags, 414 unsigned int useroffset, unsigned int usersize, 415 void (*ctor)(void *)) 416 { 417 struct kmem_cache_args kmem_args = { 418 .align = align, 419 .ctor = ctor, 420 .useroffset = useroffset, 421 .usersize = usersize, 422 }; 423 424 return __kmem_cache_create_args(name, size, &kmem_args, flags); 425 } 426 427 /* If NULL is passed for @args, use this variant with default arguments. */ 428 static inline struct kmem_cache * 429 __kmem_cache_default_args(const char *name, unsigned int size, 430 struct kmem_cache_args *args, 431 slab_flags_t flags) 432 { 433 struct kmem_cache_args kmem_default_args = {}; 434 435 /* Make sure we don't get passed garbage. */ 436 if (WARN_ON_ONCE(args)) 437 return ERR_PTR(-EINVAL); 438 439 return __kmem_cache_create_args(name, size, &kmem_default_args, flags); 440 } 441 442 /** 443 * kmem_cache_create - Create a kmem cache. 444 * @__name: A string which is used in /proc/slabinfo to identify this cache. 445 * @__object_size: The size of objects to be created in this cache. 446 * @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL 447 * means defaults will be used for all the arguments. 448 * 449 * This is currently implemented as a macro using ``_Generic()`` to call 450 * either the new variant of the function, or a legacy one. 451 * 452 * The new variant has 4 parameters: 453 * ``kmem_cache_create(name, object_size, args, flags)`` 454 * 455 * See __kmem_cache_create_args() which implements this. 456 * 457 * The legacy variant has 5 parameters: 458 * ``kmem_cache_create(name, object_size, align, flags, ctor)`` 459 * 460 * The align and ctor parameters map to the respective fields of 461 * &struct kmem_cache_args 462 * 463 * Context: Cannot be called within a interrupt, but can be interrupted. 464 * 465 * Return: a pointer to the cache on success, NULL on failure. 466 */ 467 #define kmem_cache_create(__name, __object_size, __args, ...) \ 468 _Generic((__args), \ 469 struct kmem_cache_args *: __kmem_cache_create_args, \ 470 void *: __kmem_cache_default_args, \ 471 default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__) 472 473 void kmem_cache_destroy(struct kmem_cache *s); 474 int kmem_cache_shrink(struct kmem_cache *s); 475 476 /* 477 * Please use this macro to create slab caches. Simply specify the 478 * name of the structure and maybe some flags that are listed above. 479 * 480 * The alignment of the struct determines object alignment. If you 481 * f.e. add ____cacheline_aligned_in_smp to the struct declaration 482 * then the objects will be properly aligned in SMP configurations. 483 */ 484 #define KMEM_CACHE(__struct, __flags) \ 485 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \ 486 &(struct kmem_cache_args) { \ 487 .align = __alignof__(struct __struct), \ 488 }, (__flags)) 489 490 /* 491 * To whitelist a single field for copying to/from usercopy, use this 492 * macro instead for KMEM_CACHE() above. 493 */ 494 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \ 495 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \ 496 &(struct kmem_cache_args) { \ 497 .align = __alignof__(struct __struct), \ 498 .useroffset = offsetof(struct __struct, __field), \ 499 .usersize = sizeof_field(struct __struct, __field), \ 500 }, (__flags)) 501 502 #ifdef CONFIG_KMALLOC_PARTITION_CACHES 503 typedef struct { unsigned long v; } kmalloc_token_t; 504 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM 505 extern unsigned long random_kmalloc_seed; 506 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = _CODE_LOCATION_ }) 507 #elif defined(CONFIG_KMALLOC_PARTITION_TYPED) 508 #define __kmalloc_token(...) ((kmalloc_token_t){ .v = __builtin_infer_alloc_token(__VA_ARGS__) }) 509 #endif 510 #define DECL_TOKEN_PARAM(_token) , kmalloc_token_t (_token) 511 #define _PASS_TOKEN_PARAM(_token) , (_token) 512 #define PASS_TOKEN_PARAM(_token) (_token) 513 #define DECL_TOKEN_PARAMS(_size, _token) size_t (_size), kmalloc_token_t (_token) 514 #define PASS_TOKEN_PARAMS(_size, _token) (_size), (_token) 515 #else /* !CONFIG_KMALLOC_PARTITION_CACHES */ 516 typedef struct {} kmalloc_token_t; 517 #define __kmalloc_token(...) ((kmalloc_token_t){}) /* no-op */ 518 #define DECL_TOKEN_PARAM(_token) 519 #define _PASS_TOKEN_PARAM(_token) 520 #define PASS_TOKEN_PARAM(_token) ((kmalloc_token_t){}) 521 #define DECL_TOKEN_PARAMS(_size, _token) size_t (_size) 522 #define PASS_TOKEN_PARAMS(_size, _token) (_size) 523 #endif /* CONFIG_KMALLOC_PARTITION_CACHES */ 524 525 /* 526 * Common kmalloc functions provided by all allocators 527 */ 528 void * __must_check krealloc_node_align_noprof(const void *objp, 529 DECL_TOKEN_PARAMS(new_size, token), 530 unsigned long align, 531 gfp_t flags, int nid) __realloc_size(2); 532 #define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), 1, _f, NUMA_NO_NODE) 533 #if 0 /* kernel-doc */ 534 /** 535 * krealloc_node_align - reallocate memory. The contents will remain unchanged. 536 * @p: object to reallocate memory for. 537 * @new_size: how many bytes of memory are required. 538 * @align: desired alignment. 539 * @flags: the type of memory to allocate. 540 * @nid: NUMA node or NUMA_NO_NODE 541 * 542 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 543 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 544 * 545 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 546 * Documentation/core-api/memory-allocation.rst for more details. 547 * 548 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 549 * initial memory allocation, every subsequent call to this API for the same 550 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 551 * __GFP_ZERO is not fully honored by this API. 552 * 553 * When slub_debug_orig_size() is off, krealloc() only knows about the bucket 554 * size of an allocation (but not the exact size it was allocated with) and 555 * hence implements the following semantics for shrinking and growing buffers 556 * with __GFP_ZERO:: 557 * 558 * new bucket 559 * 0 size size 560 * |--------|----------------| 561 * | keep | zero | 562 * 563 * Otherwise, the original allocation size 'orig_size' could be used to 564 * precisely clear the requested size, and the new size will also be stored 565 * as the new 'orig_size'. 566 * 567 * In any case, the contents of the object pointed to are preserved up to the 568 * lesser of the new and old sizes. 569 * 570 * Return: pointer to the allocated memory or %NULL in case of error 571 */ 572 void *krealloc_node_align(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid); 573 #endif 574 #define krealloc_node_align(p, new_size, align, flags, nid) \ 575 alloc_hooks(krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(new_size, __kmalloc_token(new_size)), align, flags, nid)) 576 #define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n) 577 #define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE) 578 579 void kfree(const void *objp); 580 void kfree_nolock(const void *objp); 581 void kfree_sensitive(const void *objp); 582 583 DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T)) 584 DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T)) 585 586 size_t ksize(const void *objp); 587 588 #ifdef CONFIG_PRINTK 589 bool kmem_dump_obj(void *object); 590 #else 591 static inline bool kmem_dump_obj(void *object) { return false; } 592 #endif 593 594 /* 595 * Some archs want to perform DMA into kmalloc caches and need a guaranteed 596 * alignment larger than the alignment of a 64-bit integer. 597 * Setting ARCH_DMA_MINALIGN in arch headers allows that. 598 */ 599 #ifdef ARCH_HAS_DMA_MINALIGN 600 #if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN) 601 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN 602 #endif 603 #endif 604 605 #ifndef ARCH_KMALLOC_MINALIGN 606 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) 607 #elif ARCH_KMALLOC_MINALIGN > 8 608 #define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN 609 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) 610 #endif 611 612 /* 613 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. 614 * Intended for arches that get misalignment faults even for 64 bit integer 615 * aligned buffers. 616 */ 617 #ifndef ARCH_SLAB_MINALIGN 618 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 619 #endif 620 621 /* 622 * Arches can define this function if they want to decide the minimum slab 623 * alignment at runtime. The value returned by the function must be a power 624 * of two and >= ARCH_SLAB_MINALIGN. 625 */ 626 #ifndef arch_slab_minalign 627 static inline unsigned int arch_slab_minalign(void) 628 { 629 return ARCH_SLAB_MINALIGN; 630 } 631 #endif 632 633 /* 634 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN. 635 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN 636 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment. 637 */ 638 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN) 639 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN) 640 #define __assume_page_alignment __assume_aligned(PAGE_SIZE) 641 642 /* 643 * Kmalloc array related definitions 644 */ 645 646 /* 647 * SLUB directly allocates requests fitting in to an order-1 page 648 * (PAGE_SIZE*2). Larger requests are passed to the page allocator. 649 */ 650 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 651 #define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT) 652 #ifndef KMALLOC_SHIFT_LOW 653 #define KMALLOC_SHIFT_LOW 3 654 #endif 655 656 /* Maximum allocatable size */ 657 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) 658 /* Maximum size for which we actually use a slab cache */ 659 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) 660 /* Maximum order allocatable via the slab allocator */ 661 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) 662 663 /* 664 * Kmalloc subsystem. 665 */ 666 #ifndef KMALLOC_MIN_SIZE 667 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) 668 #endif 669 670 /* 671 * This restriction comes from byte sized index implementation. 672 * Page size is normally 2^12 bytes and, in this case, if we want to use 673 * byte sized index which can represent 2^8 entries, the size of the object 674 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16. 675 * If minimum size of kmalloc is less than 16, we use it as minimum object 676 * size and give up to use byte sized index. 677 */ 678 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ 679 (KMALLOC_MIN_SIZE) : 16) 680 681 #ifdef CONFIG_KMALLOC_PARTITION_CACHES 682 #define KMALLOC_PARTITION_CACHES_NR 15 // # of cache copies 683 #else 684 #define KMALLOC_PARTITION_CACHES_NR 0 685 #endif 686 687 /* 688 * Whenever changing this, take care of that kmalloc_type() and 689 * create_kmalloc_caches() still work as intended. 690 * 691 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP 692 * is for accounted but unreclaimable and non-dma objects. All the other 693 * kmem caches can have both accounted and unaccounted objects. 694 */ 695 enum kmalloc_cache_type { 696 KMALLOC_NORMAL = 0, 697 #ifndef CONFIG_ZONE_DMA 698 KMALLOC_DMA = KMALLOC_NORMAL, 699 #endif 700 #ifndef CONFIG_MEMCG 701 KMALLOC_CGROUP = KMALLOC_NORMAL, 702 #endif 703 KMALLOC_PARTITION_START = KMALLOC_NORMAL, 704 KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR, 705 #ifdef CONFIG_SLUB_TINY 706 KMALLOC_RECLAIM = KMALLOC_NORMAL, 707 #else 708 KMALLOC_RECLAIM, 709 #endif 710 #ifdef CONFIG_ZONE_DMA 711 KMALLOC_DMA, 712 #endif 713 #ifdef CONFIG_MEMCG 714 KMALLOC_CGROUP, 715 #endif 716 NR_KMALLOC_TYPES 717 }; 718 719 typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1]; 720 721 extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES]; 722 723 /* 724 * Define gfp bits that should not be set for KMALLOC_NORMAL. 725 */ 726 #define KMALLOC_NOT_NORMAL_BITS \ 727 (__GFP_RECLAIMABLE | \ 728 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \ 729 (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0)) 730 731 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, kmalloc_token_t token) 732 { 733 /* 734 * The most common case is KMALLOC_NORMAL, so test for it 735 * with a single branch for all the relevant flags. 736 */ 737 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0)) 738 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM 739 /* KMALLOC_PARTITION_CACHES_NR (=15) copies + the KMALLOC_NORMAL */ 740 return KMALLOC_PARTITION_START + hash_64(token.v ^ random_kmalloc_seed, 741 ilog2(KMALLOC_PARTITION_CACHES_NR + 1)); 742 #elif defined(CONFIG_KMALLOC_PARTITION_TYPED) 743 return KMALLOC_PARTITION_START + token.v; 744 #else 745 return KMALLOC_NORMAL; 746 #endif 747 748 /* 749 * At least one of the flags has to be set. Their priorities in 750 * decreasing order are: 751 * 1) __GFP_DMA 752 * 2) __GFP_RECLAIMABLE 753 * 3) __GFP_ACCOUNT 754 */ 755 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA)) 756 return KMALLOC_DMA; 757 if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE)) 758 return KMALLOC_RECLAIM; 759 else 760 return KMALLOC_CGROUP; 761 } 762 763 /* 764 * Figure out which kmalloc slab an allocation of a certain size 765 * belongs to. 766 * 0 = zero alloc 767 * 1 = 65 .. 96 bytes 768 * 2 = 129 .. 192 bytes 769 * n = 2^(n-1)+1 .. 2^n 770 * 771 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized; 772 * typical usage is via kmalloc_index() and therefore evaluated at compile-time. 773 * Callers where !size_is_constant should only be test modules, where runtime 774 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab(). 775 */ 776 static __always_inline unsigned int __kmalloc_index(size_t size, 777 bool size_is_constant) 778 { 779 if (!size) 780 return 0; 781 782 if (size <= KMALLOC_MIN_SIZE) 783 return KMALLOC_SHIFT_LOW; 784 785 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) 786 return 1; 787 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) 788 return 2; 789 if (size <= 8) return 3; 790 if (size <= 16) return 4; 791 if (size <= 32) return 5; 792 if (size <= 64) return 6; 793 if (size <= 128) return 7; 794 if (size <= 256) return 8; 795 if (size <= 512) return 9; 796 if (size <= 1024) return 10; 797 if (size <= 2 * 1024) return 11; 798 if (size <= 4 * 1024) return 12; 799 if (size <= 8 * 1024) return 13; 800 if (size <= 16 * 1024) return 14; 801 if (size <= 32 * 1024) return 15; 802 if (size <= 64 * 1024) return 16; 803 if (size <= 128 * 1024) return 17; 804 if (size <= 256 * 1024) return 18; 805 if (size <= 512 * 1024) return 19; 806 if (size <= 1024 * 1024) return 20; 807 if (size <= 2 * 1024 * 1024) return 21; 808 809 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) 810 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()"); 811 else 812 BUG(); 813 814 /* Will never be reached. Needed because the compiler may complain */ 815 return -1; 816 } 817 static_assert(PAGE_SHIFT <= 20); 818 #define kmalloc_index(s) __kmalloc_index(s, true) 819 820 #include <linux/alloc_tag.h> 821 822 /** 823 * kmem_cache_alloc - Allocate an object 824 * @cachep: The cache to allocate from. 825 * @flags: See kmalloc(). 826 * 827 * Allocate an object from this cache. 828 * See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags. 829 * 830 * Return: pointer to the new object or %NULL in case of error 831 */ 832 void *kmem_cache_alloc_noprof(struct kmem_cache *cachep, 833 gfp_t flags) __assume_slab_alignment __malloc; 834 #define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__)) 835 836 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, 837 gfp_t gfpflags) __assume_slab_alignment __malloc; 838 #define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__)) 839 840 /** 841 * kmem_cache_charge - memcg charge an already allocated slab memory 842 * @objp: address of the slab object to memcg charge 843 * @gfpflags: describe the allocation context 844 * 845 * kmem_cache_charge allows charging a slab object to the current memcg, 846 * primarily in cases where charging at allocation time might not be possible 847 * because the target memcg is not known (i.e. softirq context) 848 * 849 * The objp should be pointer returned by the slab allocator functions like 850 * kmalloc (with __GFP_ACCOUNT in flags) or kmem_cache_alloc. The memcg charge 851 * behavior can be controlled through gfpflags parameter, which affects how the 852 * necessary internal metadata can be allocated. Including __GFP_NOFAIL denotes 853 * that overcharging is requested instead of failure, but is not applied for the 854 * internal metadata allocation. 855 * 856 * There are several cases where it will return true even if the charging was 857 * not done: 858 * More specifically: 859 * 860 * 1. For !CONFIG_MEMCG or cgroup_disable=memory systems. 861 * 2. Already charged slab objects. 862 * 3. For slab objects from KMALLOC_NORMAL caches - allocated by kmalloc() 863 * without __GFP_ACCOUNT 864 * 4. Allocating internal metadata has failed 865 * 866 * Return: true if charge was successful otherwise false. 867 */ 868 bool kmem_cache_charge(void *objp, gfp_t gfpflags); 869 void kmem_cache_free(struct kmem_cache *s, void *objp); 870 871 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags, 872 unsigned int useroffset, unsigned int usersize, 873 void (*ctor)(void *)); 874 875 /* 876 * Bulk allocation and freeing operations. These are accelerated in an 877 * allocator specific way to avoid taking locks repeatedly or building 878 * metadata structures unnecessarily. 879 * 880 * Note that interrupts must be enabled when calling these functions. 881 */ 882 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); 883 884 bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, 885 size_t size, void **p); 886 #define kmem_cache_alloc_bulk(...) \ 887 alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) 888 889 static __always_inline void kfree_bulk(size_t size, void **p) 890 { 891 kmem_cache_free_bulk(NULL, size, p); 892 } 893 894 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags, 895 int node) __assume_slab_alignment __malloc; 896 #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__)) 897 898 struct slab_sheaf * 899 kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size); 900 901 int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, 902 struct slab_sheaf **sheafp, unsigned int size); 903 904 void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, 905 struct slab_sheaf *sheaf); 906 907 void *kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *cachep, gfp_t gfp, 908 struct slab_sheaf *sheaf) __assume_slab_alignment __malloc; 909 #define kmem_cache_alloc_from_sheaf(...) \ 910 alloc_hooks(kmem_cache_alloc_from_sheaf_noprof(__VA_ARGS__)) 911 912 unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf); 913 914 /* 915 * These macros allow declaring a kmem_buckets * parameter alongside size, which 916 * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call 917 * sites don't have to pass NULL. 918 */ 919 #ifdef CONFIG_SLAB_BUCKETS 920 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b) 921 #define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b) 922 #define PASS_BUCKET_PARAM(_b) (_b) 923 #else 924 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size) 925 #define PASS_BUCKET_PARAMS(_size, _b) (_size) 926 #define PASS_BUCKET_PARAM(_b) NULL 927 #endif 928 929 #define DECL_KMALLOC_PARAMS(_size, _b, _token) DECL_BUCKET_PARAMS(_size, _b) \ 930 DECL_TOKEN_PARAM(_token) 931 932 #define PASS_KMALLOC_PARAMS(_size, _b, _token) PASS_BUCKET_PARAMS(_size, _b) \ 933 _PASS_TOKEN_PARAM(_token) 934 935 /* 936 * The following functions are not to be used directly and are intended only 937 * for internal use from kmalloc() and kmalloc_node() 938 * with the exception of kunit tests 939 */ 940 941 void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags) 942 __assume_kmalloc_alignment __alloc_size(1); 943 944 void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node) 945 __assume_kmalloc_alignment __alloc_size(1); 946 947 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size) 948 __assume_kmalloc_alignment __alloc_size(3); 949 950 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 951 int node, size_t size) 952 __assume_kmalloc_alignment __alloc_size(4); 953 954 void *__kmalloc_large_noprof(size_t size, gfp_t flags) 955 __assume_page_alignment __alloc_size(1); 956 957 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 958 __assume_page_alignment __alloc_size(1); 959 960 static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token) 961 { 962 if (__builtin_constant_p(size) && size) { 963 unsigned int index; 964 965 if (size > KMALLOC_MAX_CACHE_SIZE) 966 return __kmalloc_large_noprof(size, flags); 967 968 index = kmalloc_index(size); 969 return __kmalloc_cache_noprof( 970 kmalloc_caches[kmalloc_type(flags, token)][index], 971 flags, size); 972 } 973 return __kmalloc_noprof(PASS_TOKEN_PARAMS(size, token), flags); 974 } 975 #define kmalloc_noprof(...) _kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 976 #if 0 /* kernel-doc */ 977 /** 978 * kmalloc - allocate kernel memory 979 * @size: how many bytes of memory are required. 980 * @flags: describe the allocation context 981 * 982 * kmalloc is the normal method of allocating memory 983 * for objects smaller than page size in the kernel. 984 * 985 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN 986 * bytes. For @size of power of two bytes, the alignment is also guaranteed 987 * to be at least to the size. For other sizes, the alignment is guaranteed to 988 * be at least the largest power-of-two divisor of @size. 989 * 990 * The @flags argument may be one of the GFP flags defined at 991 * include/linux/gfp_types.h and described at 992 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>` 993 * 994 * The recommended usage of the @flags is described at 995 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>` 996 * 997 * Below is a brief outline of the most useful GFP flags 998 * 999 * %GFP_KERNEL 1000 * Allocate normal kernel ram. May sleep. 1001 * 1002 * %GFP_NOWAIT 1003 * Allocation will not sleep. 1004 * 1005 * %GFP_ATOMIC 1006 * Allocation will not sleep. May use emergency pools. 1007 * 1008 * Also it is possible to set different flags by OR'ing 1009 * in one or more of the following additional @flags: 1010 * 1011 * %__GFP_ZERO 1012 * Zero the allocated memory before returning. Also see kzalloc(). 1013 * 1014 * %__GFP_HIGH 1015 * This allocation has high priority and may use emergency pools. 1016 * 1017 * %__GFP_NOFAIL 1018 * Indicate that this allocation is in no way allowed to fail 1019 * (think twice before using). 1020 * 1021 * %__GFP_NORETRY 1022 * If memory is not immediately available, 1023 * then give up at once. 1024 * 1025 * %__GFP_NOWARN 1026 * If allocation fails, don't issue any warnings. 1027 * 1028 * %__GFP_RETRY_MAYFAIL 1029 * Try really hard to succeed the allocation but fail 1030 * eventually. 1031 */ 1032 void *kmalloc(size_t size, gfp_t flags); 1033 #endif 1034 #define kmalloc(size, flags) alloc_hooks(kmalloc_noprof(size, flags)) 1035 1036 void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node); 1037 #define kmalloc_nolock_noprof(_s, _f, _n) _kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), _f, _n) 1038 #if 0 /* kernel-doc */ 1039 /** 1040 * kmalloc_nolock - Allocate an object of given size from any context. 1041 * @size: size to allocate 1042 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, __GFP_ZERO, __GFP_NO_OBJ_EXT 1043 * allowed. 1044 * @node: node number of the target node. 1045 * 1046 * Return: pointer to the new object or NULL in case of error. 1047 * NULL does not mean EBUSY or EAGAIN. It means ENOMEM. 1048 * There is no reason to call it again and expect !NULL. 1049 */ 1050 void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); 1051 #endif 1052 #define kmalloc_nolock(size, gfp_flags, node) alloc_hooks(kmalloc_nolock_noprof(size, gfp_flags, node)) 1053 1054 /** 1055 * __alloc_objs - Allocate objects of a given type using 1056 * @KMALLOC: which size-based kmalloc wrapper to allocate with. 1057 * @GFP: GFP flags for the allocation. 1058 * @TYPE: type to allocate space for. 1059 * @COUNT: how many @TYPE objects to allocate. 1060 * 1061 * Returns: Newly allocated pointer to (first) @TYPE of @COUNT-many 1062 * allocated @TYPE objects, or NULL on failure. 1063 */ 1064 #define __alloc_objs(KMALLOC, GFP, TYPE, COUNT) \ 1065 ({ \ 1066 const size_t __obj_size = size_mul(sizeof(TYPE), COUNT); \ 1067 (TYPE *)KMALLOC(__obj_size, GFP); \ 1068 }) 1069 1070 /** 1071 * __alloc_flex - Allocate an object that has a trailing flexible array 1072 * @KMALLOC: kmalloc wrapper function to use for allocation. 1073 * @GFP: GFP flags for the allocation. 1074 * @TYPE: type of structure to allocate space for. 1075 * @FAM: The name of the flexible array member of @TYPE structure. 1076 * @COUNT: how many @FAM elements to allocate space for. 1077 * 1078 * Returns: Newly allocated pointer to @TYPE with @COUNT-many trailing 1079 * @FAM elements, or NULL on failure or if @COUNT cannot be represented 1080 * by the member of @TYPE that counts the @FAM elements (annotated via 1081 * __counted_by()). 1082 */ 1083 #define __alloc_flex(KMALLOC, GFP, TYPE, FAM, COUNT) \ 1084 ({ \ 1085 const size_t __count = (COUNT); \ 1086 const size_t __obj_size = struct_size_t(TYPE, FAM, __count); \ 1087 TYPE *__obj_ptr = KMALLOC(__obj_size, GFP); \ 1088 if (__obj_ptr) \ 1089 __set_flex_counter(__obj_ptr->FAM, __count); \ 1090 __obj_ptr; \ 1091 }) 1092 1093 /** 1094 * kmalloc_obj - Allocate a single instance of the given type 1095 * @VAR_OR_TYPE: Variable or type to allocate. 1096 * @GFP: GFP flags for the allocation. 1097 * 1098 * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL 1099 * on failure. 1100 */ 1101 #define kmalloc_obj(VAR_OR_TYPE, ...) \ 1102 __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), 1) 1103 1104 /** 1105 * kmalloc_objs - Allocate an array of the given type 1106 * @VAR_OR_TYPE: Variable or type to allocate an array of. 1107 * @COUNT: How many elements in the array. 1108 * @GFP: GFP flags for the allocation. 1109 * 1110 * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success, 1111 * or NULL on failure. 1112 */ 1113 #define kmalloc_objs(VAR_OR_TYPE, COUNT, ...) \ 1114 __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), COUNT) 1115 1116 /** 1117 * kmalloc_flex - Allocate a single instance of the given flexible structure 1118 * @VAR_OR_TYPE: Variable or type to allocate (with its flex array). 1119 * @FAM: The name of the flexible array member of the structure. 1120 * @COUNT: How many flexible array member elements are desired. 1121 * @GFP: GFP flags for the allocation. 1122 * 1123 * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on 1124 * failure. If @FAM has been annotated with __counted_by(), the allocation 1125 * will immediately fail if @COUNT is larger than what the type of the 1126 * struct's counter variable can represent. 1127 */ 1128 #define kmalloc_flex(VAR_OR_TYPE, FAM, COUNT, ...) \ 1129 __alloc_flex(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), FAM, COUNT) 1130 1131 /* All kzalloc aliases for kmalloc_(obj|objs|flex). */ 1132 #define kzalloc_obj(P, ...) \ 1133 __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), 1) 1134 #define kzalloc_objs(P, COUNT, ...) \ 1135 __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) 1136 #define kzalloc_flex(P, FAM, COUNT, ...) \ 1137 __alloc_flex(kzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) 1138 1139 /* All kvmalloc aliases for kmalloc_(obj|objs|flex). */ 1140 #define kvmalloc_obj(P, ...) \ 1141 __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), 1) 1142 #define kvmalloc_objs(P, COUNT, ...) \ 1143 __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) 1144 #define kvmalloc_flex(P, FAM, COUNT, ...) \ 1145 __alloc_flex(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) 1146 1147 /* All kvzalloc aliases for kmalloc_(obj|objs|flex). */ 1148 #define kvzalloc_obj(P, ...) \ 1149 __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), 1) 1150 #define kvzalloc_objs(P, COUNT, ...) \ 1151 __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) 1152 #define kvzalloc_flex(P, FAM, COUNT, ...) \ 1153 __alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) 1154 1155 #define kmem_buckets_alloc(_b, _size, _flags) \ 1156 alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE)) 1157 1158 #define kmem_buckets_alloc_track_caller(_b, _size, _flags) \ 1159 alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE, _RET_IP_)) 1160 1161 static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token) 1162 { 1163 if (__builtin_constant_p(size) && size) { 1164 unsigned int index; 1165 1166 if (size > KMALLOC_MAX_CACHE_SIZE) 1167 return __kmalloc_large_node_noprof(size, flags, node); 1168 1169 index = kmalloc_index(size); 1170 return __kmalloc_cache_node_noprof( 1171 kmalloc_caches[kmalloc_type(flags, token)][index], 1172 flags, node, size); 1173 } 1174 return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags, node); 1175 } 1176 #define kmalloc_node_noprof(...) _kmalloc_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1177 #define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__)) 1178 1179 static inline __alloc_size(1, 2) void *_kmalloc_array_noprof(size_t n, size_t size, gfp_t flags, kmalloc_token_t token) 1180 { 1181 size_t bytes; 1182 1183 if (unlikely(check_mul_overflow(n, size, &bytes))) 1184 return NULL; 1185 return _kmalloc_noprof(bytes, flags, token); 1186 } 1187 #define kmalloc_array_noprof(...) _kmalloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1188 #if 0 /* kernel-doc */ 1189 /** 1190 * kmalloc_array - allocate memory for an array. 1191 * @n: number of elements. 1192 * @size: element size. 1193 * @flags: the type of memory to allocate (see kmalloc). 1194 */ 1195 void *kmalloc_array(size_t n, size_t size, gfp_t flags); 1196 #endif 1197 #define kmalloc_array(n, size, flags) alloc_hooks(kmalloc_array_noprof(n, size, flags)) 1198 1199 static inline __realloc_size(2, 3) void * __must_check _krealloc_array_noprof(void *p, 1200 size_t new_n, 1201 size_t new_size, 1202 gfp_t flags, kmalloc_token_t token) 1203 { 1204 size_t bytes; 1205 1206 if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) 1207 return NULL; 1208 1209 return krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(bytes, token), 1, flags, NUMA_NO_NODE); 1210 } 1211 #define krealloc_array_noprof(...) _krealloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1212 #if 0 /* kernel-doc */ 1213 /** 1214 * krealloc_array - reallocate memory for an array. 1215 * @p: pointer to the memory chunk to reallocate 1216 * @new_n: new number of elements to alloc 1217 * @new_size: new size of a single member of the array 1218 * @flags: the type of memory to allocate (see kmalloc) 1219 * 1220 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 1221 * initial memory allocation, every subsequent call to this API for the same 1222 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 1223 * __GFP_ZERO is not fully honored by this API. 1224 * 1225 * See krealloc_noprof() for further details. 1226 * 1227 * In any case, the contents of the object pointed to are preserved up to the 1228 * lesser of the new and old sizes. 1229 */ 1230 void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t flags); 1231 #endif 1232 #define krealloc_array(p, new_n, new_size, flags) alloc_hooks(krealloc_array_noprof(p, new_n, new_size, flags)) 1233 1234 /** 1235 * kcalloc - allocate memory for an array. The memory is set to zero. 1236 * @n: number of elements. 1237 * @size: element size. 1238 * @flags: the type of memory to allocate (see kmalloc). 1239 */ 1240 #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO) 1241 1242 void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node, 1243 unsigned long caller) __alloc_size(1); 1244 #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \ 1245 __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(size, NULL, __kmalloc_token(size)), flags, node, caller) 1246 #define kmalloc_node_track_caller(...) \ 1247 alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_)) 1248 1249 /* 1250 * kmalloc_track_caller is a special version of kmalloc that records the 1251 * calling function of the routine calling it for slab leak tracking instead 1252 * of just the calling function (confusing, eh?). 1253 * It's useful when the call to kmalloc comes from a widely-used standard 1254 * allocator where we care about the real place the memory allocation 1255 * request comes from. 1256 */ 1257 #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE) 1258 1259 #define kmalloc_track_caller_noprof(...) \ 1260 kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_) 1261 1262 static inline __alloc_size(1, 2) void *_kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, 1263 int node, kmalloc_token_t token) 1264 { 1265 size_t bytes; 1266 1267 if (unlikely(check_mul_overflow(n, size, &bytes))) 1268 return NULL; 1269 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 1270 return _kmalloc_node_noprof(bytes, flags, node, token); 1271 return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), flags, node); 1272 } 1273 #define kmalloc_array_node_noprof(...) _kmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1274 #define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__)) 1275 1276 #define kcalloc_node(_n, _size, _flags, _node) \ 1277 kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node) 1278 1279 /* 1280 * Shortcuts 1281 */ 1282 #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO) 1283 1284 static inline __alloc_size(1) void *_kzalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token) 1285 { 1286 return _kmalloc_noprof(size, flags | __GFP_ZERO, token); 1287 } 1288 #define kzalloc_noprof(...) _kzalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1289 #if 0 /* kernel-doc */ 1290 /** 1291 * kzalloc - allocate memory. The memory is set to zero. 1292 * @size: how many bytes of memory are required. 1293 * @flags: the type of memory to allocate (see kmalloc). 1294 */ 1295 void *kzalloc(size_t size, gfp_t flags); 1296 #endif 1297 #define kzalloc(size, flags) alloc_hooks(kzalloc_noprof(size, flags)) 1298 #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) 1299 1300 void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align, 1301 gfp_t flags, int node) __alloc_size(1); 1302 #define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ 1303 __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, NULL, __kmalloc_token(_size)), _align, _flags, _node) 1304 #define kvmalloc_node_align(...) \ 1305 alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) 1306 #if 0 /* kernel-doc */ 1307 /** 1308 * kvmalloc_node - attempt to allocate physically contiguous memory, but upon 1309 * failure, fall back to non-contiguous (vmalloc) allocation. 1310 * @size: size of the request. 1311 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 1312 * @node: numa node to allocate from 1313 * 1314 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 1315 * Documentation/core-api/memory-allocation.rst for more details. 1316 * 1317 * Uses kmalloc to get the memory but if the allocation fails then falls back 1318 * to the vmalloc allocator. Use kvfree for freeing the memory. 1319 * 1320 * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not. 1321 * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is 1322 * preferable to the vmalloc fallback, due to visible performance drawbacks. 1323 * 1324 * Return: pointer to the allocated memory of %NULL in case of failure 1325 */ 1326 void *kvmalloc_node(size_t size, gfp_t flags, int node); 1327 #endif 1328 #define kvmalloc_node(size, flags, node) kvmalloc_node_align(size, 1, flags, node) 1329 #define kvmalloc_node_noprof(size, flags, node) \ 1330 kvmalloc_node_align_noprof(size, 1, flags, node) 1331 #define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE) 1332 #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) 1333 #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) 1334 1335 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node) 1336 1337 #define kmem_buckets_valloc(_b, _size, _flags) \ 1338 alloc_hooks(__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), 1, _flags, NUMA_NO_NODE)) 1339 1340 static inline __alloc_size(1, 2) void * 1341 _kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmalloc_token_t token) 1342 { 1343 size_t bytes; 1344 1345 if (unlikely(check_mul_overflow(n, size, &bytes))) 1346 return NULL; 1347 1348 return __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), 1, flags, node); 1349 } 1350 #define kvmalloc_array_node_noprof(...) _kvmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 1351 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE) 1352 #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node) 1353 #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE) 1354 1355 #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__)) 1356 #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) 1357 #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) 1358 1359 void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align, 1360 gfp_t flags, int nid) __realloc_size(2); 1361 #if 0 /* kernel-doc */ 1362 /** 1363 * kvrealloc_node_align - reallocate memory; contents remain unchanged 1364 * @p: object to reallocate memory for 1365 * @size: the size to reallocate 1366 * @align: desired alignment 1367 * @flags: the flags for the page level allocator 1368 * @nid: NUMA node id 1369 * 1370 * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 1371 * and @p is not a %NULL pointer, the object pointed to is freed. 1372 * 1373 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 1374 * Documentation/core-api/memory-allocation.rst for more details. 1375 * 1376 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 1377 * initial memory allocation, every subsequent call to this API for the same 1378 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 1379 * __GFP_ZERO is not fully honored by this API. 1380 * 1381 * In any case, the contents of the object pointed to are preserved up to the 1382 * lesser of the new and old sizes. 1383 * 1384 * This function must not be called concurrently with itself or kvfree() for the 1385 * same memory allocation. 1386 * 1387 * Return: pointer to the allocated memory or %NULL in case of error 1388 */ 1389 void *kvrealloc_node_align(const void *p, size_t size, unsigned long align, gfp_t flags, int nid); 1390 #endif 1391 #define kvrealloc_node_align(p, size, align, flags, nid) \ 1392 alloc_hooks(kvrealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, __kmalloc_token(size)), align, flags, nid)) 1393 #define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n) 1394 #define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE) 1395 1396 extern void kvfree(const void *addr); 1397 DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T)) 1398 1399 extern void kvfree_atomic(const void *addr); 1400 DEFINE_FREE(kvfree_atomic, void *, if (!IS_ERR_OR_NULL(_T)) kvfree_atomic(_T)) 1401 1402 extern void kvfree_sensitive(const void *addr, size_t len); 1403 1404 unsigned int kmem_cache_size(struct kmem_cache *s); 1405 1406 #ifndef CONFIG_KVFREE_RCU_BATCHED 1407 static inline void kvfree_rcu_barrier(void) 1408 { 1409 rcu_barrier(); 1410 } 1411 1412 static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) 1413 { 1414 rcu_barrier(); 1415 } 1416 1417 static inline void kfree_rcu_scheduler_running(void) { } 1418 #else 1419 void kvfree_rcu_barrier(void); 1420 1421 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); 1422 1423 void kfree_rcu_scheduler_running(void); 1424 #endif 1425 1426 /** 1427 * kmalloc_size_roundup - Report allocation bucket size for the given size 1428 * 1429 * @size: Number of bytes to round up from. 1430 * 1431 * This returns the number of bytes that would be available in a kmalloc() 1432 * allocation of @size bytes. For example, a 126 byte request would be 1433 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly 1434 * for the general-purpose kmalloc()-based allocations, and is not for the 1435 * pre-sized kmem_cache_alloc()-based allocations.) 1436 * 1437 * Use this to kmalloc() the full bucket size ahead of time instead of using 1438 * ksize() to query the size after an allocation. 1439 */ 1440 size_t kmalloc_size_roundup(size_t size); 1441 1442 void __init kmem_cache_init_late(void); 1443 void __init kvfree_rcu_init(void); 1444 1445 #endif /* _LINUX_SLAB_H */ 1446