1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef MM_SLAB_H 3 #define MM_SLAB_H 4 5 #include <linux/reciprocal_div.h> 6 #include <linux/list_lru.h> 7 #include <linux/local_lock.h> 8 #include <linux/random.h> 9 #include <linux/kobject.h> 10 #include <linux/sched/mm.h> 11 #include <linux/memcontrol.h> 12 #include <linux/kfence.h> 13 #include <linux/kasan.h> 14 #include <linux/slab.h> 15 16 /* 17 * Internal slab definitions 18 */ 19 20 /* slab's alloc_flags definitions */ 21 #define SLAB_ALLOC_DEFAULT 0x00 /* no flags */ 22 #define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */ 23 #define SLAB_ALLOC_NEW_SLAB 0x02 /* a flag for alloc_slab_obj_exts() */ 24 #define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */ 25 #define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */ 26 27 static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags) 28 { 29 return !(alloc_flags & SLAB_ALLOC_NOLOCK); 30 } 31 32 void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags, 33 unsigned int alloc_flags, int node) 34 __assume_kmalloc_alignment __alloc_size(1); 35 36 static __always_inline __alloc_size(1) void *_kmalloc_flags_noprof(size_t size, 37 gfp_t flags, unsigned int alloc_flags, int node, kmalloc_token_t token) 38 { 39 return __kmalloc_flags_noprof(PASS_TOKEN_PARAMS(size, token), flags, alloc_flags, node); 40 } 41 #define kmalloc_flags_noprof(...) _kmalloc_flags_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) 42 #define kmalloc_flags(...) alloc_hooks(kmalloc_flags_noprof(__VA_ARGS__)) 43 44 #ifdef CONFIG_64BIT 45 # ifdef system_has_cmpxchg128 46 # define system_has_freelist_aba() system_has_cmpxchg128() 47 # define try_cmpxchg_freelist try_cmpxchg128 48 # endif 49 typedef u128 freelist_full_t; 50 #else /* CONFIG_64BIT */ 51 # ifdef system_has_cmpxchg64 52 # define system_has_freelist_aba() system_has_cmpxchg64() 53 # define try_cmpxchg_freelist try_cmpxchg64 54 # endif 55 typedef u64 freelist_full_t; 56 #endif /* CONFIG_64BIT */ 57 58 #if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 59 #undef system_has_freelist_aba 60 #endif 61 62 /* 63 * Freelist pointer and counter to cmpxchg together, avoids the typical ABA 64 * problems with cmpxchg of just a pointer. 65 */ 66 struct freelist_counters { 67 union { 68 struct { 69 void *freelist; 70 union { 71 unsigned long counters; 72 struct { 73 unsigned inuse:16; 74 unsigned objects:15; 75 /* 76 * If slab debugging is enabled then the 77 * frozen bit can be reused to indicate 78 * that the slab was corrupted 79 */ 80 unsigned frozen:1; 81 #ifdef CONFIG_64BIT 82 /* 83 * Some optimizations use free bits in 'counters' field 84 * to save memory. In case ->stride field is not available, 85 * such optimizations are disabled. 86 */ 87 unsigned int stride; 88 #endif 89 }; 90 }; 91 }; 92 #ifdef system_has_freelist_aba 93 freelist_full_t freelist_counters; 94 #endif 95 }; 96 }; 97 98 /* Reuses the bits in struct page */ 99 struct slab { 100 memdesc_flags_t flags; 101 102 struct kmem_cache *slab_cache; 103 union { 104 struct { 105 struct list_head slab_list; 106 /* Double-word boundary */ 107 struct freelist_counters; 108 }; 109 struct rcu_head rcu_head; 110 }; 111 112 unsigned int __page_type; 113 atomic_t __page_refcount; 114 #ifdef CONFIG_SLAB_OBJ_EXT 115 unsigned long obj_exts; 116 #endif 117 }; 118 119 #define SLAB_MATCH(pg, sl) \ 120 static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) 121 SLAB_MATCH(flags, flags); 122 SLAB_MATCH(compound_info, slab_cache); /* Ensure bit 0 is clear */ 123 SLAB_MATCH(_refcount, __page_refcount); 124 #ifdef CONFIG_MEMCG 125 SLAB_MATCH(memcg_data, obj_exts); 126 #elif defined(CONFIG_SLAB_OBJ_EXT) 127 SLAB_MATCH(_unused_slab_obj_exts, obj_exts); 128 #endif 129 #undef SLAB_MATCH 130 static_assert(sizeof(struct slab) <= sizeof(struct page)); 131 #if defined(system_has_freelist_aba) 132 static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist_counters))); 133 #endif 134 135 /** 136 * slab_folio - The folio allocated for a slab 137 * @s: The slab. 138 * 139 * Slabs are allocated as folios that contain the individual objects and are 140 * using some fields in the first struct page of the folio - those fields are 141 * now accessed by struct slab. It is occasionally necessary to convert back to 142 * a folio in order to communicate with the rest of the mm. Please use this 143 * helper function instead of casting yourself, as the implementation may change 144 * in the future. 145 */ 146 #define slab_folio(s) (_Generic((s), \ 147 const struct slab *: (const struct folio *)s, \ 148 struct slab *: (struct folio *)s)) 149 150 /** 151 * page_slab - Converts from struct page to its slab. 152 * @page: A page which may or may not belong to a slab. 153 * 154 * Return: The slab which contains this page or NULL if the page does 155 * not belong to a slab. This includes pages returned from large kmalloc. 156 */ 157 static inline struct slab *page_slab(const struct page *page) 158 { 159 page = compound_head(page); 160 if (data_race(page->page_type >> 24) != PGTY_slab) 161 page = NULL; 162 163 return (struct slab *)page; 164 } 165 166 /** 167 * slab_page - The first struct page allocated for a slab 168 * @s: The slab. 169 * 170 * A convenience wrapper for converting slab to the first struct page of the 171 * underlying folio, to communicate with code not yet converted to folio or 172 * struct slab. 173 */ 174 #define slab_page(s) folio_page(slab_folio(s), 0) 175 176 static inline void *slab_address(const struct slab *slab) 177 { 178 return folio_address(slab_folio(slab)); 179 } 180 181 static inline int slab_nid(const struct slab *slab) 182 { 183 return memdesc_nid(slab->flags); 184 } 185 186 static inline pg_data_t *slab_pgdat(const struct slab *slab) 187 { 188 return NODE_DATA(slab_nid(slab)); 189 } 190 191 static inline struct slab *virt_to_slab(const void *addr) 192 { 193 return page_slab(virt_to_page(addr)); 194 } 195 196 static inline int slab_order(const struct slab *slab) 197 { 198 return folio_order(slab_folio(slab)); 199 } 200 201 static inline size_t slab_size(const struct slab *slab) 202 { 203 return PAGE_SIZE << slab_order(slab); 204 } 205 206 /* 207 * Word size structure that can be atomically updated or read and that 208 * contains both the order and the number of objects that a slab of the 209 * given order would contain. 210 */ 211 struct kmem_cache_order_objects { 212 unsigned int x; 213 }; 214 215 struct kmem_cache_per_node_ptrs { 216 struct node_barn *barn; 217 struct kmem_cache_node *node; 218 }; 219 220 /* 221 * Slab cache management. 222 */ 223 struct kmem_cache { 224 struct slub_percpu_sheaves __percpu *cpu_sheaves; 225 /* Used for retrieving partial slabs, etc. */ 226 slab_flags_t flags; 227 unsigned long min_partial; 228 unsigned int size; /* Object size including metadata */ 229 unsigned int object_size; /* Object size without metadata */ 230 struct reciprocal_value reciprocal_size; 231 unsigned int offset; /* Free pointer offset */ 232 unsigned int sheaf_capacity; 233 struct kmem_cache_order_objects oo; 234 235 /* Allocation and freeing of slabs */ 236 struct kmem_cache_order_objects min; 237 gfp_t allocflags; /* gfp flags to use on each alloc */ 238 int refcount; /* Refcount for slab cache destroy */ 239 void (*ctor)(void *object); /* Object constructor */ 240 unsigned int inuse; /* Offset to metadata */ 241 unsigned int align; /* Alignment */ 242 unsigned int red_left_pad; /* Left redzone padding size */ 243 const char *name; /* Name (only for display!) */ 244 struct list_head list; /* List of slab caches */ 245 #ifdef CONFIG_SYSFS 246 struct kobject kobj; /* For sysfs */ 247 #endif 248 #ifdef CONFIG_SLAB_FREELIST_HARDENED 249 unsigned long random; 250 #endif 251 252 #ifdef CONFIG_NUMA 253 /* 254 * Defragmentation by allocating from a remote node. 255 */ 256 unsigned int remote_node_defrag_ratio; 257 #endif 258 259 #ifdef CONFIG_SLAB_FREELIST_RANDOM 260 unsigned int *random_seq; 261 #endif 262 263 #ifdef CONFIG_KASAN_GENERIC 264 struct kasan_cache kasan_info; 265 #endif 266 267 #ifdef CONFIG_HARDENED_USERCOPY 268 unsigned int useroffset; /* Usercopy region offset */ 269 unsigned int usersize; /* Usercopy region size */ 270 #endif 271 272 #ifdef CONFIG_SLUB_STATS 273 struct kmem_cache_stats __percpu *cpu_stats; 274 #endif 275 276 struct kmem_cache_per_node_ptrs per_node[MAX_NUMNODES]; 277 }; 278 279 /* 280 * Every cache has !NULL s->cpu_sheaves but they may point to the 281 * bootstrap_sheaf temporarily during init, or permanently for the boot caches 282 * and caches with debugging enabled, or all caches with CONFIG_SLUB_TINY. This 283 * helper distinguishes whether cache has real non-bootstrap sheaves. 284 */ 285 static inline bool cache_has_sheaves(struct kmem_cache *s) 286 { 287 /* Test CONFIG_SLUB_TINY for code elimination purposes */ 288 return !IS_ENABLED(CONFIG_SLUB_TINY) && s->sheaf_capacity; 289 } 290 291 #if defined(CONFIG_SYSFS) && !defined(CONFIG_SLUB_TINY) 292 #define SLAB_SUPPORTS_SYSFS 1 293 void sysfs_slab_unlink(struct kmem_cache *s); 294 void sysfs_slab_release(struct kmem_cache *s); 295 int sysfs_slab_alias(struct kmem_cache *s, const char *name); 296 #else 297 static inline void sysfs_slab_unlink(struct kmem_cache *s) { } 298 static inline void sysfs_slab_release(struct kmem_cache *s) { } 299 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *name) 300 { return 0; } 301 #endif 302 303 void *fixup_red_left(struct kmem_cache *s, void *p); 304 305 static inline void *nearest_obj(struct kmem_cache *cache, 306 const struct slab *slab, void *x) 307 { 308 void *object = x - (x - slab_address(slab)) % cache->size; 309 void *last_object = slab_address(slab) + 310 (slab->objects - 1) * cache->size; 311 void *result = (unlikely(object > last_object)) ? last_object : object; 312 313 result = fixup_red_left(cache, result); 314 return result; 315 } 316 317 /* Determine object index from a given position */ 318 static inline unsigned int __obj_to_index(const struct kmem_cache *cache, 319 void *addr, const void *obj) 320 { 321 return reciprocal_divide(kasan_reset_tag(obj) - addr, 322 cache->reciprocal_size); 323 } 324 325 static inline unsigned int obj_to_index(const struct kmem_cache *cache, 326 const struct slab *slab, const void *obj) 327 { 328 if (is_kfence_address(obj)) 329 return 0; 330 return __obj_to_index(cache, slab_address(slab), obj); 331 } 332 333 static inline int objs_per_slab(const struct kmem_cache *cache, 334 const struct slab *slab) 335 { 336 return slab->objects; 337 } 338 339 /* 340 * State of the slab allocator. 341 * 342 * This is used to describe the states of the allocator during bootup. 343 * Allocators use this to gradually bootstrap themselves. Most allocators 344 * have the problem that the structures used for managing slab caches are 345 * allocated from slab caches themselves. 346 */ 347 enum slab_state { 348 DOWN, /* No slab functionality yet */ 349 PARTIAL, /* SLUB: kmem_cache_node available */ 350 UP, /* Slab caches usable but not all extras yet */ 351 FULL /* Everything is working */ 352 }; 353 354 extern enum slab_state slab_state; 355 356 /* The slab cache mutex protects the management structures during changes */ 357 extern struct mutex slab_mutex; 358 359 /* The list of all slab caches on the system */ 360 extern struct list_head slab_caches; 361 362 /* The slab cache that manages slab cache information */ 363 extern struct kmem_cache *kmem_cache; 364 365 /* A table of kmalloc cache names and sizes */ 366 extern const struct kmalloc_info_struct { 367 const char *name[NR_KMALLOC_TYPES]; 368 unsigned int size; 369 } kmalloc_info[]; 370 371 /* Kmalloc array related functions */ 372 void setup_kmalloc_cache_index_table(void); 373 void create_kmalloc_caches(void); 374 375 extern u8 kmalloc_size_index[24]; 376 377 static inline unsigned int size_index_elem(unsigned int bytes) 378 { 379 return (bytes - 1) / 8; 380 } 381 382 /* 383 * Find the kmem_cache structure that serves a given size of 384 * allocation 385 * 386 * This assumes size is larger than zero and not larger than 387 * KMALLOC_MAX_CACHE_SIZE and the caller must check that. 388 */ 389 static inline struct kmem_cache * 390 kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token, 391 unsigned int alloc_flags) 392 { 393 unsigned int index; 394 enum kmalloc_cache_type type = kmalloc_type(flags, token); 395 396 if (alloc_flags & SLAB_ALLOC_NO_OBJ_EXT) 397 type = KMALLOC_NO_OBJ_EXT; 398 399 if (!b) 400 b = &kmalloc_caches[type]; 401 if (size <= 192) 402 index = kmalloc_size_index[size_index_elem(size)]; 403 else 404 index = fls(size - 1); 405 406 return (*b)[index]; 407 } 408 409 gfp_t kmalloc_fix_flags(gfp_t flags); 410 411 /* Functions provided by the slab allocators */ 412 int do_kmem_cache_create(struct kmem_cache *s, const char *name, 413 unsigned int size, struct kmem_cache_args *args, 414 slab_flags_t flags); 415 416 void __init kmem_cache_init(void); 417 extern void create_boot_cache(struct kmem_cache *, const char *name, 418 unsigned int size, slab_flags_t flags, 419 unsigned int useroffset, unsigned int usersize); 420 421 int slab_unmergeable(struct kmem_cache *s); 422 bool slab_args_unmergeable(struct kmem_cache_args *args, slab_flags_t flags); 423 424 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name); 425 426 static inline bool is_kmalloc_cache(struct kmem_cache *s) 427 { 428 return (s->flags & SLAB_KMALLOC); 429 } 430 431 static inline bool is_kmalloc_normal(struct kmem_cache *s) 432 { 433 if (!is_kmalloc_cache(s)) 434 return false; 435 436 return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT)); 437 } 438 439 bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); 440 void flush_all_rcu_sheaves(void); 441 void flush_rcu_sheaves_on_cache(struct kmem_cache *s); 442 443 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ 444 SLAB_CACHE_DMA32 | SLAB_PANIC | \ 445 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS | \ 446 SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 447 SLAB_TEMPORARY | SLAB_ACCOUNT | \ 448 SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE) 449 450 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 451 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) 452 453 #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS) 454 455 bool __kmem_cache_empty(struct kmem_cache *); 456 int __kmem_cache_shutdown(struct kmem_cache *); 457 void __kmem_cache_release(struct kmem_cache *); 458 int __kmem_cache_shrink(struct kmem_cache *); 459 void slab_kmem_cache_release(struct kmem_cache *); 460 461 struct seq_file; 462 struct file; 463 464 struct slabinfo { 465 unsigned long active_objs; 466 unsigned long num_objs; 467 unsigned long active_slabs; 468 unsigned long num_slabs; 469 unsigned long shared_avail; 470 unsigned int limit; 471 unsigned int batchcount; 472 unsigned int shared; 473 unsigned int objects_per_slab; 474 unsigned int cache_order; 475 }; 476 477 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); 478 479 #ifdef CONFIG_SLUB_DEBUG 480 #ifdef CONFIG_SLUB_DEBUG_ON 481 DECLARE_STATIC_KEY_TRUE(slub_debug_enabled); 482 #else 483 DECLARE_STATIC_KEY_FALSE(slub_debug_enabled); 484 #endif 485 extern void print_tracking(struct kmem_cache *s, void *object); 486 long validate_slab_cache(struct kmem_cache *s); 487 static inline bool __slub_debug_enabled(void) 488 { 489 return static_branch_unlikely(&slub_debug_enabled); 490 } 491 #else 492 static inline void print_tracking(struct kmem_cache *s, void *object) 493 { 494 } 495 static inline bool __slub_debug_enabled(void) 496 { 497 return false; 498 } 499 #endif 500 501 /* 502 * Returns true if any of the specified slab_debug flags is enabled for the 503 * cache. Use only for flags parsed by setup_slub_debug() as it also enables 504 * the static key. 505 */ 506 static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags) 507 { 508 if (IS_ENABLED(CONFIG_SLUB_DEBUG)) 509 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS)); 510 if (__slub_debug_enabled()) 511 return s->flags & flags; 512 return false; 513 } 514 515 #if IS_ENABLED(CONFIG_SLUB_DEBUG) && IS_ENABLED(CONFIG_KUNIT) 516 bool slab_in_kunit_test(void); 517 #else 518 static inline bool slab_in_kunit_test(void) { return false; } 519 #endif 520 521 /* 522 * slub is about to manipulate internal object metadata. This memory lies 523 * outside the range of the allocated object, so accessing it would normally 524 * be reported by kasan as a bounds error. metadata_access_enable() is used 525 * to tell kasan that these accesses are OK. 526 */ 527 static inline void metadata_access_enable(void) 528 { 529 kasan_disable_current(); 530 kmsan_disable_current(); 531 } 532 533 static inline void metadata_access_disable(void) 534 { 535 kmsan_enable_current(); 536 kasan_enable_current(); 537 } 538 539 /* 540 * Return true if KMALLOC_NORMAL caches may need obj_exts arrays. 541 * 542 * Memory allocation profiling requires obj_exts for all caches. 543 * Memcg usually doesn't need them for normal kmalloc caches, but kmalloc types 544 * with a priority higher than KMALLOC_CGROUP can be aliased with KMALLOC_NORMAL. 545 */ 546 static inline bool need_kmalloc_no_objext(void) 547 { 548 if (!mem_alloc_profiling_permanently_disabled()) 549 return true; 550 551 if (!mem_cgroup_kmem_disabled() && 552 (KMALLOC_NORMAL == KMALLOC_RECLAIM)) 553 return true; 554 555 return false; 556 } 557 558 #ifdef CONFIG_SLAB_OBJ_EXT 559 560 /* 561 * slab_obj_exts - get the pointer to the slab object extension vector 562 * associated with a slab. 563 * @slab: a pointer to the slab struct 564 * 565 * Returns the address of the object extension vector associated with the slab, 566 * or zero if no such vector has been associated yet. 567 * Do not dereference the return value directly; use get/put_slab_obj_exts() 568 * pair and slab_obj_ext() to access individual elements. 569 * 570 * Example usage: 571 * 572 * obj_exts = slab_obj_exts(slab); 573 * if (obj_exts) { 574 * get_slab_obj_exts(obj_exts); 575 * obj_ext = slab_obj_ext(slab, obj_exts, obj_to_index(s, slab, obj)); 576 * // do something with obj_ext 577 * put_slab_obj_exts(obj_exts); 578 * } 579 * 580 * Note that the get/put semantics does not involve reference counting. 581 * Instead, it updates kasan/kmsan depth so that accesses to slabobj_ext 582 * won't be reported as access violations. 583 */ 584 static inline unsigned long slab_obj_exts(struct slab *slab) 585 { 586 unsigned long obj_exts = READ_ONCE(slab->obj_exts); 587 588 #ifdef CONFIG_MEMCG 589 /* 590 * obj_exts should be either NULL, a valid pointer with 591 * MEMCG_DATA_OBJEXTS bit set or be equal to OBJEXTS_ALLOC_FAIL. 592 */ 593 VM_BUG_ON_PAGE(obj_exts && !(obj_exts & MEMCG_DATA_OBJEXTS) && 594 obj_exts != OBJEXTS_ALLOC_FAIL, slab_page(slab)); 595 VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab)); 596 #endif 597 598 return obj_exts & ~OBJEXTS_FLAGS_MASK; 599 } 600 601 static inline void get_slab_obj_exts(unsigned long obj_exts) 602 { 603 VM_WARN_ON_ONCE(!obj_exts); 604 metadata_access_enable(); 605 } 606 607 static inline void put_slab_obj_exts(unsigned long obj_exts) 608 { 609 metadata_access_disable(); 610 } 611 612 #ifdef CONFIG_64BIT 613 static inline void slab_set_stride(struct slab *slab, unsigned int stride) 614 { 615 slab->stride = stride; 616 } 617 static inline unsigned int slab_get_stride(struct slab *slab) 618 { 619 return slab->stride; 620 } 621 #else 622 static inline void slab_set_stride(struct slab *slab, unsigned int stride) 623 { 624 VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext)); 625 } 626 static inline unsigned int slab_get_stride(struct slab *slab) 627 { 628 return sizeof(struct slabobj_ext); 629 } 630 #endif 631 632 /* 633 * slab_obj_ext - get the pointer to the slab object extension metadata 634 * associated with an object in a slab. 635 * @slab: a pointer to the slab struct 636 * @obj_exts: a pointer to the object extension vector 637 * @index: an index of the object 638 * 639 * Returns a pointer to the object extension associated with the object. 640 * Must be called within a section covered by get/put_slab_obj_exts(). 641 */ 642 static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, 643 unsigned long obj_exts, 644 unsigned int index) 645 { 646 struct slabobj_ext *obj_ext; 647 648 VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); 649 650 obj_ext = (struct slabobj_ext *)(obj_exts + 651 slab_get_stride(slab) * index); 652 return kasan_reset_tag(obj_ext); 653 } 654 655 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 656 gfp_t gfp, unsigned int alloc_flags); 657 658 #else /* CONFIG_SLAB_OBJ_EXT */ 659 660 static inline unsigned long slab_obj_exts(struct slab *slab) 661 { 662 return 0; 663 } 664 665 static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, 666 unsigned long obj_exts, 667 unsigned int index) 668 { 669 return NULL; 670 } 671 672 static inline void slab_set_stride(struct slab *slab, unsigned int stride) { } 673 static inline unsigned int slab_get_stride(struct slab *slab) { return 0; } 674 675 676 #endif /* CONFIG_SLAB_OBJ_EXT */ 677 678 static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) 679 { 680 return (s->flags & SLAB_RECLAIM_ACCOUNT) ? 681 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; 682 } 683 684 #ifdef CONFIG_MEMCG 685 bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 686 gfp_t flags, unsigned int slab_alloc_flags, 687 size_t size, void **p); 688 void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 689 void **p, int objects, unsigned long obj_exts); 690 #endif 691 692 void kvfree_rcu_cb(struct rcu_head *head); 693 694 static inline unsigned int large_kmalloc_order(const struct page *page) 695 { 696 return page[1].flags.f & 0xff; 697 } 698 699 static inline size_t large_kmalloc_size(const struct page *page) 700 { 701 return PAGE_SIZE << large_kmalloc_order(page); 702 } 703 704 #ifdef CONFIG_SLUB_DEBUG 705 void dump_unreclaimable_slab(void); 706 #else 707 static inline void dump_unreclaimable_slab(void) 708 { 709 } 710 #endif 711 712 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr); 713 714 #ifdef CONFIG_SLAB_FREELIST_RANDOM 715 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 716 gfp_t gfp); 717 void cache_random_seq_destroy(struct kmem_cache *cachep); 718 #else 719 static inline int cache_random_seq_create(struct kmem_cache *cachep, 720 unsigned int count, gfp_t gfp) 721 { 722 return 0; 723 } 724 static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } 725 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 726 727 static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c) 728 { 729 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 730 &init_on_alloc)) { 731 if (c->ctor) 732 return false; 733 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) 734 return flags & __GFP_ZERO; 735 return true; 736 } 737 return flags & __GFP_ZERO; 738 } 739 740 static inline bool slab_want_init_on_free(struct kmem_cache *c) 741 { 742 if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 743 &init_on_free)) 744 return !(c->ctor || 745 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 746 return false; 747 } 748 749 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 750 void debugfs_slab_release(struct kmem_cache *); 751 #else 752 static inline void debugfs_slab_release(struct kmem_cache *s) { } 753 #endif 754 755 #ifdef CONFIG_PRINTK 756 #define KS_ADDRS_COUNT 16 757 struct kmem_obj_info { 758 void *kp_ptr; 759 struct slab *kp_slab; 760 void *kp_objp; 761 unsigned long kp_data_offset; 762 struct kmem_cache *kp_slab_cache; 763 void *kp_ret; 764 void *kp_stack[KS_ADDRS_COUNT]; 765 void *kp_free_stack[KS_ADDRS_COUNT]; 766 }; 767 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab); 768 #endif 769 770 void __check_heap_object(const void *ptr, unsigned long n, 771 const struct slab *slab, bool to_user); 772 773 void defer_free_barrier(void); 774 775 static inline bool slub_debug_orig_size(struct kmem_cache *s) 776 { 777 return (kmem_cache_debug_flags(s, SLAB_STORE_USER) && 778 (s->flags & SLAB_KMALLOC)); 779 } 780 781 #ifdef CONFIG_SLUB_DEBUG 782 void skip_orig_size_check(struct kmem_cache *s, const void *object); 783 #endif 784 785 #endif /* MM_SLAB_H */ 786