1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef MM_SLAB_H 3 #define MM_SLAB_H 4 5 #include <linux/reciprocal_div.h> 6 #include <linux/list_lru.h> 7 #include <linux/local_lock.h> 8 #include <linux/random.h> 9 #include <linux/kobject.h> 10 #include <linux/sched/mm.h> 11 #include <linux/memcontrol.h> 12 #include <linux/kfence.h> 13 #include <linux/kasan.h> 14 15 /* 16 * Internal slab definitions 17 */ 18 19 #ifdef CONFIG_64BIT 20 # ifdef system_has_cmpxchg128 21 # define system_has_freelist_aba() system_has_cmpxchg128() 22 # define try_cmpxchg_freelist try_cmpxchg128 23 # endif 24 typedef u128 freelist_full_t; 25 #else /* CONFIG_64BIT */ 26 # ifdef system_has_cmpxchg64 27 # define system_has_freelist_aba() system_has_cmpxchg64() 28 # define try_cmpxchg_freelist try_cmpxchg64 29 # endif 30 typedef u64 freelist_full_t; 31 #endif /* CONFIG_64BIT */ 32 33 #if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 34 #undef system_has_freelist_aba 35 #endif 36 37 /* 38 * Freelist pointer and counter to cmpxchg together, avoids the typical ABA 39 * problems with cmpxchg of just a pointer. 40 */ 41 struct freelist_counters { 42 union { 43 struct { 44 void *freelist; 45 union { 46 unsigned long counters; 47 struct { 48 unsigned inuse:16; 49 unsigned objects:15; 50 /* 51 * If slab debugging is enabled then the 52 * frozen bit can be reused to indicate 53 * that the slab was corrupted 54 */ 55 unsigned frozen:1; 56 #ifdef CONFIG_64BIT 57 /* 58 * Some optimizations use free bits in 'counters' field 59 * to save memory. In case ->stride field is not available, 60 * such optimizations are disabled. 61 */ 62 unsigned short stride; 63 #endif 64 }; 65 }; 66 }; 67 #ifdef system_has_freelist_aba 68 freelist_full_t freelist_counters; 69 #endif 70 }; 71 }; 72 73 /* Reuses the bits in struct page */ 74 struct slab { 75 memdesc_flags_t flags; 76 77 struct kmem_cache *slab_cache; 78 union { 79 struct { 80 struct list_head slab_list; 81 /* Double-word boundary */ 82 struct freelist_counters; 83 }; 84 struct rcu_head rcu_head; 85 }; 86 87 unsigned int __page_type; 88 atomic_t __page_refcount; 89 #ifdef CONFIG_SLAB_OBJ_EXT 90 unsigned long obj_exts; 91 #endif 92 }; 93 94 #define SLAB_MATCH(pg, sl) \ 95 static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) 96 SLAB_MATCH(flags, flags); 97 SLAB_MATCH(compound_head, slab_cache); /* Ensure bit 0 is clear */ 98 SLAB_MATCH(_refcount, __page_refcount); 99 #ifdef CONFIG_MEMCG 100 SLAB_MATCH(memcg_data, obj_exts); 101 #elif defined(CONFIG_SLAB_OBJ_EXT) 102 SLAB_MATCH(_unused_slab_obj_exts, obj_exts); 103 #endif 104 #undef SLAB_MATCH 105 static_assert(sizeof(struct slab) <= sizeof(struct page)); 106 #if defined(system_has_freelist_aba) 107 static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist_counters))); 108 #endif 109 110 /** 111 * slab_folio - The folio allocated for a slab 112 * @s: The slab. 113 * 114 * Slabs are allocated as folios that contain the individual objects and are 115 * using some fields in the first struct page of the folio - those fields are 116 * now accessed by struct slab. It is occasionally necessary to convert back to 117 * a folio in order to communicate with the rest of the mm. Please use this 118 * helper function instead of casting yourself, as the implementation may change 119 * in the future. 120 */ 121 #define slab_folio(s) (_Generic((s), \ 122 const struct slab *: (const struct folio *)s, \ 123 struct slab *: (struct folio *)s)) 124 125 /** 126 * page_slab - Converts from struct page to its slab. 127 * @page: A page which may or may not belong to a slab. 128 * 129 * Return: The slab which contains this page or NULL if the page does 130 * not belong to a slab. This includes pages returned from large kmalloc. 131 */ 132 static inline struct slab *page_slab(const struct page *page) 133 { 134 unsigned long head; 135 136 head = READ_ONCE(page->compound_head); 137 if (head & 1) 138 page = (struct page *)(head - 1); 139 if (data_race(page->page_type >> 24) != PGTY_slab) 140 page = NULL; 141 142 return (struct slab *)page; 143 } 144 145 /** 146 * slab_page - The first struct page allocated for a slab 147 * @s: The slab. 148 * 149 * A convenience wrapper for converting slab to the first struct page of the 150 * underlying folio, to communicate with code not yet converted to folio or 151 * struct slab. 152 */ 153 #define slab_page(s) folio_page(slab_folio(s), 0) 154 155 static inline void *slab_address(const struct slab *slab) 156 { 157 return folio_address(slab_folio(slab)); 158 } 159 160 static inline int slab_nid(const struct slab *slab) 161 { 162 return memdesc_nid(slab->flags); 163 } 164 165 static inline pg_data_t *slab_pgdat(const struct slab *slab) 166 { 167 return NODE_DATA(slab_nid(slab)); 168 } 169 170 static inline struct slab *virt_to_slab(const void *addr) 171 { 172 return page_slab(virt_to_page(addr)); 173 } 174 175 static inline int slab_order(const struct slab *slab) 176 { 177 return folio_order(slab_folio(slab)); 178 } 179 180 static inline size_t slab_size(const struct slab *slab) 181 { 182 return PAGE_SIZE << slab_order(slab); 183 } 184 185 /* 186 * Word size structure that can be atomically updated or read and that 187 * contains both the order and the number of objects that a slab of the 188 * given order would contain. 189 */ 190 struct kmem_cache_order_objects { 191 unsigned int x; 192 }; 193 194 /* 195 * Slab cache management. 196 */ 197 struct kmem_cache { 198 struct slub_percpu_sheaves __percpu *cpu_sheaves; 199 /* Used for retrieving partial slabs, etc. */ 200 slab_flags_t flags; 201 unsigned long min_partial; 202 unsigned int size; /* Object size including metadata */ 203 unsigned int object_size; /* Object size without metadata */ 204 struct reciprocal_value reciprocal_size; 205 unsigned int offset; /* Free pointer offset */ 206 unsigned int sheaf_capacity; 207 struct kmem_cache_order_objects oo; 208 209 /* Allocation and freeing of slabs */ 210 struct kmem_cache_order_objects min; 211 gfp_t allocflags; /* gfp flags to use on each alloc */ 212 int refcount; /* Refcount for slab cache destroy */ 213 void (*ctor)(void *object); /* Object constructor */ 214 unsigned int inuse; /* Offset to metadata */ 215 unsigned int align; /* Alignment */ 216 unsigned int red_left_pad; /* Left redzone padding size */ 217 const char *name; /* Name (only for display!) */ 218 struct list_head list; /* List of slab caches */ 219 #ifdef CONFIG_SYSFS 220 struct kobject kobj; /* For sysfs */ 221 #endif 222 #ifdef CONFIG_SLAB_FREELIST_HARDENED 223 unsigned long random; 224 #endif 225 226 #ifdef CONFIG_NUMA 227 /* 228 * Defragmentation by allocating from a remote node. 229 */ 230 unsigned int remote_node_defrag_ratio; 231 #endif 232 233 #ifdef CONFIG_SLAB_FREELIST_RANDOM 234 unsigned int *random_seq; 235 #endif 236 237 #ifdef CONFIG_KASAN_GENERIC 238 struct kasan_cache kasan_info; 239 #endif 240 241 #ifdef CONFIG_HARDENED_USERCOPY 242 unsigned int useroffset; /* Usercopy region offset */ 243 unsigned int usersize; /* Usercopy region size */ 244 #endif 245 246 #ifdef CONFIG_SLUB_STATS 247 struct kmem_cache_stats __percpu *cpu_stats; 248 #endif 249 250 struct kmem_cache_node *node[MAX_NUMNODES]; 251 }; 252 253 /* 254 * Every cache has !NULL s->cpu_sheaves but they may point to the 255 * bootstrap_sheaf temporarily during init, or permanently for the boot caches 256 * and caches with debugging enabled, or all caches with CONFIG_SLUB_TINY. This 257 * helper distinguishes whether cache has real non-bootstrap sheaves. 258 */ 259 static inline bool cache_has_sheaves(struct kmem_cache *s) 260 { 261 /* Test CONFIG_SLUB_TINY for code elimination purposes */ 262 return !IS_ENABLED(CONFIG_SLUB_TINY) && s->sheaf_capacity; 263 } 264 265 #if defined(CONFIG_SYSFS) && !defined(CONFIG_SLUB_TINY) 266 #define SLAB_SUPPORTS_SYSFS 1 267 void sysfs_slab_unlink(struct kmem_cache *s); 268 void sysfs_slab_release(struct kmem_cache *s); 269 int sysfs_slab_alias(struct kmem_cache *s, const char *name); 270 #else 271 static inline void sysfs_slab_unlink(struct kmem_cache *s) { } 272 static inline void sysfs_slab_release(struct kmem_cache *s) { } 273 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *name) 274 { return 0; } 275 #endif 276 277 void *fixup_red_left(struct kmem_cache *s, void *p); 278 279 static inline void *nearest_obj(struct kmem_cache *cache, 280 const struct slab *slab, void *x) 281 { 282 void *object = x - (x - slab_address(slab)) % cache->size; 283 void *last_object = slab_address(slab) + 284 (slab->objects - 1) * cache->size; 285 void *result = (unlikely(object > last_object)) ? last_object : object; 286 287 result = fixup_red_left(cache, result); 288 return result; 289 } 290 291 /* Determine object index from a given position */ 292 static inline unsigned int __obj_to_index(const struct kmem_cache *cache, 293 void *addr, void *obj) 294 { 295 return reciprocal_divide(kasan_reset_tag(obj) - addr, 296 cache->reciprocal_size); 297 } 298 299 static inline unsigned int obj_to_index(const struct kmem_cache *cache, 300 const struct slab *slab, void *obj) 301 { 302 if (is_kfence_address(obj)) 303 return 0; 304 return __obj_to_index(cache, slab_address(slab), obj); 305 } 306 307 static inline int objs_per_slab(const struct kmem_cache *cache, 308 const struct slab *slab) 309 { 310 return slab->objects; 311 } 312 313 /* 314 * State of the slab allocator. 315 * 316 * This is used to describe the states of the allocator during bootup. 317 * Allocators use this to gradually bootstrap themselves. Most allocators 318 * have the problem that the structures used for managing slab caches are 319 * allocated from slab caches themselves. 320 */ 321 enum slab_state { 322 DOWN, /* No slab functionality yet */ 323 PARTIAL, /* SLUB: kmem_cache_node available */ 324 UP, /* Slab caches usable but not all extras yet */ 325 FULL /* Everything is working */ 326 }; 327 328 extern enum slab_state slab_state; 329 330 /* The slab cache mutex protects the management structures during changes */ 331 extern struct mutex slab_mutex; 332 333 /* The list of all slab caches on the system */ 334 extern struct list_head slab_caches; 335 336 /* The slab cache that manages slab cache information */ 337 extern struct kmem_cache *kmem_cache; 338 339 /* A table of kmalloc cache names and sizes */ 340 extern const struct kmalloc_info_struct { 341 const char *name[NR_KMALLOC_TYPES]; 342 unsigned int size; 343 } kmalloc_info[]; 344 345 /* Kmalloc array related functions */ 346 void setup_kmalloc_cache_index_table(void); 347 void create_kmalloc_caches(void); 348 349 extern u8 kmalloc_size_index[24]; 350 351 static inline unsigned int size_index_elem(unsigned int bytes) 352 { 353 return (bytes - 1) / 8; 354 } 355 356 /* 357 * Find the kmem_cache structure that serves a given size of 358 * allocation 359 * 360 * This assumes size is larger than zero and not larger than 361 * KMALLOC_MAX_CACHE_SIZE and the caller must check that. 362 */ 363 static inline struct kmem_cache * 364 kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, unsigned long caller) 365 { 366 unsigned int index; 367 368 if (!b) 369 b = &kmalloc_caches[kmalloc_type(flags, caller)]; 370 if (size <= 192) 371 index = kmalloc_size_index[size_index_elem(size)]; 372 else 373 index = fls(size - 1); 374 375 return (*b)[index]; 376 } 377 378 gfp_t kmalloc_fix_flags(gfp_t flags); 379 380 /* Functions provided by the slab allocators */ 381 int do_kmem_cache_create(struct kmem_cache *s, const char *name, 382 unsigned int size, struct kmem_cache_args *args, 383 slab_flags_t flags); 384 385 void __init kmem_cache_init(void); 386 extern void create_boot_cache(struct kmem_cache *, const char *name, 387 unsigned int size, slab_flags_t flags, 388 unsigned int useroffset, unsigned int usersize); 389 390 int slab_unmergeable(struct kmem_cache *s); 391 bool slab_args_unmergeable(struct kmem_cache_args *args, slab_flags_t flags); 392 393 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name); 394 395 static inline bool is_kmalloc_cache(struct kmem_cache *s) 396 { 397 return (s->flags & SLAB_KMALLOC); 398 } 399 400 static inline bool is_kmalloc_normal(struct kmem_cache *s) 401 { 402 if (!is_kmalloc_cache(s)) 403 return false; 404 return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT)); 405 } 406 407 bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); 408 void flush_all_rcu_sheaves(void); 409 void flush_rcu_sheaves_on_cache(struct kmem_cache *s); 410 411 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ 412 SLAB_CACHE_DMA32 | SLAB_PANIC | \ 413 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS | \ 414 SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 415 SLAB_TEMPORARY | SLAB_ACCOUNT | \ 416 SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE) 417 418 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 419 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) 420 421 #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS) 422 423 bool __kmem_cache_empty(struct kmem_cache *); 424 int __kmem_cache_shutdown(struct kmem_cache *); 425 void __kmem_cache_release(struct kmem_cache *); 426 int __kmem_cache_shrink(struct kmem_cache *); 427 void slab_kmem_cache_release(struct kmem_cache *); 428 429 struct seq_file; 430 struct file; 431 432 struct slabinfo { 433 unsigned long active_objs; 434 unsigned long num_objs; 435 unsigned long active_slabs; 436 unsigned long num_slabs; 437 unsigned long shared_avail; 438 unsigned int limit; 439 unsigned int batchcount; 440 unsigned int shared; 441 unsigned int objects_per_slab; 442 unsigned int cache_order; 443 }; 444 445 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); 446 447 #ifdef CONFIG_SLUB_DEBUG 448 #ifdef CONFIG_SLUB_DEBUG_ON 449 DECLARE_STATIC_KEY_TRUE(slub_debug_enabled); 450 #else 451 DECLARE_STATIC_KEY_FALSE(slub_debug_enabled); 452 #endif 453 extern void print_tracking(struct kmem_cache *s, void *object); 454 long validate_slab_cache(struct kmem_cache *s); 455 static inline bool __slub_debug_enabled(void) 456 { 457 return static_branch_unlikely(&slub_debug_enabled); 458 } 459 #else 460 static inline void print_tracking(struct kmem_cache *s, void *object) 461 { 462 } 463 static inline bool __slub_debug_enabled(void) 464 { 465 return false; 466 } 467 #endif 468 469 /* 470 * Returns true if any of the specified slab_debug flags is enabled for the 471 * cache. Use only for flags parsed by setup_slub_debug() as it also enables 472 * the static key. 473 */ 474 static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags) 475 { 476 if (IS_ENABLED(CONFIG_SLUB_DEBUG)) 477 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS)); 478 if (__slub_debug_enabled()) 479 return s->flags & flags; 480 return false; 481 } 482 483 #if IS_ENABLED(CONFIG_SLUB_DEBUG) && IS_ENABLED(CONFIG_KUNIT) 484 bool slab_in_kunit_test(void); 485 #else 486 static inline bool slab_in_kunit_test(void) { return false; } 487 #endif 488 489 /* 490 * slub is about to manipulate internal object metadata. This memory lies 491 * outside the range of the allocated object, so accessing it would normally 492 * be reported by kasan as a bounds error. metadata_access_enable() is used 493 * to tell kasan that these accesses are OK. 494 */ 495 static inline void metadata_access_enable(void) 496 { 497 kasan_disable_current(); 498 kmsan_disable_current(); 499 } 500 501 static inline void metadata_access_disable(void) 502 { 503 kmsan_enable_current(); 504 kasan_enable_current(); 505 } 506 507 #ifdef CONFIG_SLAB_OBJ_EXT 508 509 /* 510 * slab_obj_exts - get the pointer to the slab object extension vector 511 * associated with a slab. 512 * @slab: a pointer to the slab struct 513 * 514 * Returns the address of the object extension vector associated with the slab, 515 * or zero if no such vector has been associated yet. 516 * Do not dereference the return value directly; use get/put_slab_obj_exts() 517 * pair and slab_obj_ext() to access individual elements. 518 * 519 * Example usage: 520 * 521 * obj_exts = slab_obj_exts(slab); 522 * if (obj_exts) { 523 * get_slab_obj_exts(obj_exts); 524 * obj_ext = slab_obj_ext(slab, obj_exts, obj_to_index(s, slab, obj)); 525 * // do something with obj_ext 526 * put_slab_obj_exts(obj_exts); 527 * } 528 * 529 * Note that the get/put semantics does not involve reference counting. 530 * Instead, it updates kasan/kmsan depth so that accesses to slabobj_ext 531 * won't be reported as access violations. 532 */ 533 static inline unsigned long slab_obj_exts(struct slab *slab) 534 { 535 unsigned long obj_exts = READ_ONCE(slab->obj_exts); 536 537 #ifdef CONFIG_MEMCG 538 /* 539 * obj_exts should be either NULL, a valid pointer with 540 * MEMCG_DATA_OBJEXTS bit set or be equal to OBJEXTS_ALLOC_FAIL. 541 */ 542 VM_BUG_ON_PAGE(obj_exts && !(obj_exts & MEMCG_DATA_OBJEXTS) && 543 obj_exts != OBJEXTS_ALLOC_FAIL, slab_page(slab)); 544 VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab)); 545 #endif 546 547 return obj_exts & ~OBJEXTS_FLAGS_MASK; 548 } 549 550 static inline void get_slab_obj_exts(unsigned long obj_exts) 551 { 552 VM_WARN_ON_ONCE(!obj_exts); 553 metadata_access_enable(); 554 } 555 556 static inline void put_slab_obj_exts(unsigned long obj_exts) 557 { 558 metadata_access_disable(); 559 } 560 561 #ifdef CONFIG_64BIT 562 static inline void slab_set_stride(struct slab *slab, unsigned short stride) 563 { 564 slab->stride = stride; 565 } 566 static inline unsigned short slab_get_stride(struct slab *slab) 567 { 568 return slab->stride; 569 } 570 #else 571 static inline void slab_set_stride(struct slab *slab, unsigned short stride) 572 { 573 VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext)); 574 } 575 static inline unsigned short slab_get_stride(struct slab *slab) 576 { 577 return sizeof(struct slabobj_ext); 578 } 579 #endif 580 581 /* 582 * slab_obj_ext - get the pointer to the slab object extension metadata 583 * associated with an object in a slab. 584 * @slab: a pointer to the slab struct 585 * @obj_exts: a pointer to the object extension vector 586 * @index: an index of the object 587 * 588 * Returns a pointer to the object extension associated with the object. 589 * Must be called within a section covered by get/put_slab_obj_exts(). 590 */ 591 static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, 592 unsigned long obj_exts, 593 unsigned int index) 594 { 595 struct slabobj_ext *obj_ext; 596 597 VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); 598 599 obj_ext = (struct slabobj_ext *)(obj_exts + 600 slab_get_stride(slab) * index); 601 return kasan_reset_tag(obj_ext); 602 } 603 604 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 605 gfp_t gfp, bool new_slab); 606 607 #else /* CONFIG_SLAB_OBJ_EXT */ 608 609 static inline unsigned long slab_obj_exts(struct slab *slab) 610 { 611 return 0; 612 } 613 614 static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, 615 unsigned long obj_exts, 616 unsigned int index) 617 { 618 return NULL; 619 } 620 621 static inline void slab_set_stride(struct slab *slab, unsigned int stride) { } 622 static inline unsigned int slab_get_stride(struct slab *slab) { return 0; } 623 624 625 #endif /* CONFIG_SLAB_OBJ_EXT */ 626 627 static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) 628 { 629 return (s->flags & SLAB_RECLAIM_ACCOUNT) ? 630 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; 631 } 632 633 #ifdef CONFIG_MEMCG 634 bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 635 gfp_t flags, size_t size, void **p); 636 void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 637 void **p, int objects, unsigned long obj_exts); 638 #endif 639 640 void kvfree_rcu_cb(struct rcu_head *head); 641 642 static inline unsigned int large_kmalloc_order(const struct page *page) 643 { 644 return page[1].flags.f & 0xff; 645 } 646 647 static inline size_t large_kmalloc_size(const struct page *page) 648 { 649 return PAGE_SIZE << large_kmalloc_order(page); 650 } 651 652 #ifdef CONFIG_SLUB_DEBUG 653 void dump_unreclaimable_slab(void); 654 #else 655 static inline void dump_unreclaimable_slab(void) 656 { 657 } 658 #endif 659 660 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr); 661 662 #ifdef CONFIG_SLAB_FREELIST_RANDOM 663 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 664 gfp_t gfp); 665 void cache_random_seq_destroy(struct kmem_cache *cachep); 666 #else 667 static inline int cache_random_seq_create(struct kmem_cache *cachep, 668 unsigned int count, gfp_t gfp) 669 { 670 return 0; 671 } 672 static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } 673 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 674 675 static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c) 676 { 677 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 678 &init_on_alloc)) { 679 if (c->ctor) 680 return false; 681 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) 682 return flags & __GFP_ZERO; 683 return true; 684 } 685 return flags & __GFP_ZERO; 686 } 687 688 static inline bool slab_want_init_on_free(struct kmem_cache *c) 689 { 690 if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 691 &init_on_free)) 692 return !(c->ctor || 693 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 694 return false; 695 } 696 697 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 698 void debugfs_slab_release(struct kmem_cache *); 699 #else 700 static inline void debugfs_slab_release(struct kmem_cache *s) { } 701 #endif 702 703 #ifdef CONFIG_PRINTK 704 #define KS_ADDRS_COUNT 16 705 struct kmem_obj_info { 706 void *kp_ptr; 707 struct slab *kp_slab; 708 void *kp_objp; 709 unsigned long kp_data_offset; 710 struct kmem_cache *kp_slab_cache; 711 void *kp_ret; 712 void *kp_stack[KS_ADDRS_COUNT]; 713 void *kp_free_stack[KS_ADDRS_COUNT]; 714 }; 715 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab); 716 #endif 717 718 void __check_heap_object(const void *ptr, unsigned long n, 719 const struct slab *slab, bool to_user); 720 721 void defer_free_barrier(void); 722 723 static inline bool slub_debug_orig_size(struct kmem_cache *s) 724 { 725 return (kmem_cache_debug_flags(s, SLAB_STORE_USER) && 726 (s->flags & SLAB_KMALLOC)); 727 } 728 729 #ifdef CONFIG_SLUB_DEBUG 730 void skip_orig_size_check(struct kmem_cache *s, const void *object); 731 #endif 732 733 #endif /* MM_SLAB_H */ 734