1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 297d06609SChristoph Lameter #ifndef MM_SLAB_H 397d06609SChristoph Lameter #define MM_SLAB_H 489c2d061SVlastimil Babka 589c2d061SVlastimil Babka #include <linux/reciprocal_div.h> 689c2d061SVlastimil Babka #include <linux/list_lru.h> 789c2d061SVlastimil Babka #include <linux/local_lock.h> 889c2d061SVlastimil Babka #include <linux/random.h> 989c2d061SVlastimil Babka #include <linux/kobject.h> 1089c2d061SVlastimil Babka #include <linux/sched/mm.h> 1189c2d061SVlastimil Babka #include <linux/memcontrol.h> 1289c2d061SVlastimil Babka #include <linux/kfence.h> 1389c2d061SVlastimil Babka #include <linux/kasan.h> 1489c2d061SVlastimil Babka 1597d06609SChristoph Lameter /* 1697d06609SChristoph Lameter * Internal slab definitions 1797d06609SChristoph Lameter */ 1897d06609SChristoph Lameter 196801be4fSPeter Zijlstra #ifdef CONFIG_64BIT 206801be4fSPeter Zijlstra # ifdef system_has_cmpxchg128 216801be4fSPeter Zijlstra # define system_has_freelist_aba() system_has_cmpxchg128() 226801be4fSPeter Zijlstra # define try_cmpxchg_freelist try_cmpxchg128 236801be4fSPeter Zijlstra # endif 246801be4fSPeter Zijlstra #define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg128 256801be4fSPeter Zijlstra typedef u128 freelist_full_t; 266801be4fSPeter Zijlstra #else /* CONFIG_64BIT */ 276801be4fSPeter Zijlstra # ifdef system_has_cmpxchg64 286801be4fSPeter Zijlstra # define system_has_freelist_aba() system_has_cmpxchg64() 296801be4fSPeter Zijlstra # define try_cmpxchg_freelist try_cmpxchg64 306801be4fSPeter Zijlstra # endif 316801be4fSPeter Zijlstra #define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg64 326801be4fSPeter Zijlstra typedef u64 freelist_full_t; 336801be4fSPeter Zijlstra #endif /* CONFIG_64BIT */ 346801be4fSPeter Zijlstra 356801be4fSPeter Zijlstra #if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 366801be4fSPeter Zijlstra #undef system_has_freelist_aba 376801be4fSPeter Zijlstra #endif 386801be4fSPeter Zijlstra 396801be4fSPeter Zijlstra /* 406801be4fSPeter Zijlstra * Freelist pointer and counter to cmpxchg together, avoids the typical ABA 416801be4fSPeter Zijlstra * problems with cmpxchg of just a pointer. 426801be4fSPeter Zijlstra */ 436801be4fSPeter Zijlstra typedef union { 446801be4fSPeter Zijlstra struct { 456801be4fSPeter Zijlstra void *freelist; 466801be4fSPeter Zijlstra unsigned long counter; 476801be4fSPeter Zijlstra }; 486801be4fSPeter Zijlstra freelist_full_t full; 496801be4fSPeter Zijlstra } freelist_aba_t; 506801be4fSPeter Zijlstra 51d122019bSMatthew Wilcox (Oracle) /* Reuses the bits in struct page */ 52d122019bSMatthew Wilcox (Oracle) struct slab { 53d122019bSMatthew Wilcox (Oracle) unsigned long __page_flags; 54401fb12cSVlastimil Babka 55130d4df5SVlastimil Babka struct kmem_cache *slab_cache; 56130d4df5SVlastimil Babka union { 57130d4df5SVlastimil Babka struct { 58401fb12cSVlastimil Babka union { 59401fb12cSVlastimil Babka struct list_head slab_list; 609c01e9afSVlastimil Babka #ifdef CONFIG_SLUB_CPU_PARTIAL 61401fb12cSVlastimil Babka struct { 62d122019bSMatthew Wilcox (Oracle) struct slab *next; 63d122019bSMatthew Wilcox (Oracle) int slabs; /* Nr of slabs left */ 64d122019bSMatthew Wilcox (Oracle) }; 659c01e9afSVlastimil Babka #endif 66d122019bSMatthew Wilcox (Oracle) }; 67d122019bSMatthew Wilcox (Oracle) /* Double-word boundary */ 686801be4fSPeter Zijlstra union { 696801be4fSPeter Zijlstra struct { 70d122019bSMatthew Wilcox (Oracle) void *freelist; /* first free object */ 71d122019bSMatthew Wilcox (Oracle) union { 72401fb12cSVlastimil Babka unsigned long counters; 73401fb12cSVlastimil Babka struct { 74d122019bSMatthew Wilcox (Oracle) unsigned inuse:16; 75d122019bSMatthew Wilcox (Oracle) unsigned objects:15; 76d122019bSMatthew Wilcox (Oracle) unsigned frozen:1; 77d122019bSMatthew Wilcox (Oracle) }; 78d122019bSMatthew Wilcox (Oracle) }; 79130d4df5SVlastimil Babka }; 806801be4fSPeter Zijlstra #ifdef system_has_freelist_aba 816801be4fSPeter Zijlstra freelist_aba_t freelist_counter; 826801be4fSPeter Zijlstra #endif 836801be4fSPeter Zijlstra }; 846801be4fSPeter Zijlstra }; 85130d4df5SVlastimil Babka struct rcu_head rcu_head; 86130d4df5SVlastimil Babka }; 87401fb12cSVlastimil Babka unsigned int __unused; 88d122019bSMatthew Wilcox (Oracle) 89d122019bSMatthew Wilcox (Oracle) atomic_t __page_refcount; 90d122019bSMatthew Wilcox (Oracle) #ifdef CONFIG_MEMCG 91d122019bSMatthew Wilcox (Oracle) unsigned long memcg_data; 92d122019bSMatthew Wilcox (Oracle) #endif 93d122019bSMatthew Wilcox (Oracle) }; 94d122019bSMatthew Wilcox (Oracle) 95d122019bSMatthew Wilcox (Oracle) #define SLAB_MATCH(pg, sl) \ 96d122019bSMatthew Wilcox (Oracle) static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) 97d122019bSMatthew Wilcox (Oracle) SLAB_MATCH(flags, __page_flags); 98130d4df5SVlastimil Babka SLAB_MATCH(compound_head, slab_cache); /* Ensure bit 0 is clear */ 99d122019bSMatthew Wilcox (Oracle) SLAB_MATCH(_refcount, __page_refcount); 100d122019bSMatthew Wilcox (Oracle) #ifdef CONFIG_MEMCG 101d122019bSMatthew Wilcox (Oracle) SLAB_MATCH(memcg_data, memcg_data); 102d122019bSMatthew Wilcox (Oracle) #endif 103d122019bSMatthew Wilcox (Oracle) #undef SLAB_MATCH 104d122019bSMatthew Wilcox (Oracle) static_assert(sizeof(struct slab) <= sizeof(struct page)); 105a9e0b9f2SVlastimil Babka #if defined(system_has_freelist_aba) 1066801be4fSPeter Zijlstra static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t))); 107130d4df5SVlastimil Babka #endif 108d122019bSMatthew Wilcox (Oracle) 109d122019bSMatthew Wilcox (Oracle) /** 110d122019bSMatthew Wilcox (Oracle) * folio_slab - Converts from folio to slab. 111d122019bSMatthew Wilcox (Oracle) * @folio: The folio. 112d122019bSMatthew Wilcox (Oracle) * 113d122019bSMatthew Wilcox (Oracle) * Currently struct slab is a different representation of a folio where 114d122019bSMatthew Wilcox (Oracle) * folio_test_slab() is true. 115d122019bSMatthew Wilcox (Oracle) * 116d122019bSMatthew Wilcox (Oracle) * Return: The slab which contains this folio. 117d122019bSMatthew Wilcox (Oracle) */ 118d122019bSMatthew Wilcox (Oracle) #define folio_slab(folio) (_Generic((folio), \ 119d122019bSMatthew Wilcox (Oracle) const struct folio *: (const struct slab *)(folio), \ 120d122019bSMatthew Wilcox (Oracle) struct folio *: (struct slab *)(folio))) 121d122019bSMatthew Wilcox (Oracle) 122d122019bSMatthew Wilcox (Oracle) /** 123d122019bSMatthew Wilcox (Oracle) * slab_folio - The folio allocated for a slab 124d122019bSMatthew Wilcox (Oracle) * @slab: The slab. 125d122019bSMatthew Wilcox (Oracle) * 126d122019bSMatthew Wilcox (Oracle) * Slabs are allocated as folios that contain the individual objects and are 127d122019bSMatthew Wilcox (Oracle) * using some fields in the first struct page of the folio - those fields are 128d122019bSMatthew Wilcox (Oracle) * now accessed by struct slab. It is occasionally necessary to convert back to 129d122019bSMatthew Wilcox (Oracle) * a folio in order to communicate with the rest of the mm. Please use this 130d122019bSMatthew Wilcox (Oracle) * helper function instead of casting yourself, as the implementation may change 131d122019bSMatthew Wilcox (Oracle) * in the future. 132d122019bSMatthew Wilcox (Oracle) */ 133d122019bSMatthew Wilcox (Oracle) #define slab_folio(s) (_Generic((s), \ 134d122019bSMatthew Wilcox (Oracle) const struct slab *: (const struct folio *)s, \ 135d122019bSMatthew Wilcox (Oracle) struct slab *: (struct folio *)s)) 136d122019bSMatthew Wilcox (Oracle) 137d122019bSMatthew Wilcox (Oracle) /** 138d122019bSMatthew Wilcox (Oracle) * page_slab - Converts from first struct page to slab. 139d122019bSMatthew Wilcox (Oracle) * @p: The first (either head of compound or single) page of slab. 140d122019bSMatthew Wilcox (Oracle) * 141d122019bSMatthew Wilcox (Oracle) * A temporary wrapper to convert struct page to struct slab in situations where 142d122019bSMatthew Wilcox (Oracle) * we know the page is the compound head, or single order-0 page. 143d122019bSMatthew Wilcox (Oracle) * 144d122019bSMatthew Wilcox (Oracle) * Long-term ideally everything would work with struct slab directly or go 145d122019bSMatthew Wilcox (Oracle) * through folio to struct slab. 146d122019bSMatthew Wilcox (Oracle) * 147d122019bSMatthew Wilcox (Oracle) * Return: The slab which contains this page 148d122019bSMatthew Wilcox (Oracle) */ 149d122019bSMatthew Wilcox (Oracle) #define page_slab(p) (_Generic((p), \ 150d122019bSMatthew Wilcox (Oracle) const struct page *: (const struct slab *)(p), \ 151d122019bSMatthew Wilcox (Oracle) struct page *: (struct slab *)(p))) 152d122019bSMatthew Wilcox (Oracle) 153d122019bSMatthew Wilcox (Oracle) /** 154d122019bSMatthew Wilcox (Oracle) * slab_page - The first struct page allocated for a slab 155d122019bSMatthew Wilcox (Oracle) * @slab: The slab. 156d122019bSMatthew Wilcox (Oracle) * 157d122019bSMatthew Wilcox (Oracle) * A convenience wrapper for converting slab to the first struct page of the 158d122019bSMatthew Wilcox (Oracle) * underlying folio, to communicate with code not yet converted to folio or 159d122019bSMatthew Wilcox (Oracle) * struct slab. 160d122019bSMatthew Wilcox (Oracle) */ 161d122019bSMatthew Wilcox (Oracle) #define slab_page(s) folio_page(slab_folio(s), 0) 162d122019bSMatthew Wilcox (Oracle) 163d122019bSMatthew Wilcox (Oracle) /* 164d122019bSMatthew Wilcox (Oracle) * If network-based swap is enabled, sl*b must keep track of whether pages 165d122019bSMatthew Wilcox (Oracle) * were allocated from pfmemalloc reserves. 166d122019bSMatthew Wilcox (Oracle) */ 167d122019bSMatthew Wilcox (Oracle) static inline bool slab_test_pfmemalloc(const struct slab *slab) 168d122019bSMatthew Wilcox (Oracle) { 169d122019bSMatthew Wilcox (Oracle) return folio_test_active((struct folio *)slab_folio(slab)); 170d122019bSMatthew Wilcox (Oracle) } 171d122019bSMatthew Wilcox (Oracle) 172d122019bSMatthew Wilcox (Oracle) static inline void slab_set_pfmemalloc(struct slab *slab) 173d122019bSMatthew Wilcox (Oracle) { 174d122019bSMatthew Wilcox (Oracle) folio_set_active(slab_folio(slab)); 175d122019bSMatthew Wilcox (Oracle) } 176d122019bSMatthew Wilcox (Oracle) 177d122019bSMatthew Wilcox (Oracle) static inline void slab_clear_pfmemalloc(struct slab *slab) 178d122019bSMatthew Wilcox (Oracle) { 179d122019bSMatthew Wilcox (Oracle) folio_clear_active(slab_folio(slab)); 180d122019bSMatthew Wilcox (Oracle) } 181d122019bSMatthew Wilcox (Oracle) 182d122019bSMatthew Wilcox (Oracle) static inline void __slab_clear_pfmemalloc(struct slab *slab) 183d122019bSMatthew Wilcox (Oracle) { 184d122019bSMatthew Wilcox (Oracle) __folio_clear_active(slab_folio(slab)); 185d122019bSMatthew Wilcox (Oracle) } 186d122019bSMatthew Wilcox (Oracle) 187d122019bSMatthew Wilcox (Oracle) static inline void *slab_address(const struct slab *slab) 188d122019bSMatthew Wilcox (Oracle) { 189d122019bSMatthew Wilcox (Oracle) return folio_address(slab_folio(slab)); 190d122019bSMatthew Wilcox (Oracle) } 191d122019bSMatthew Wilcox (Oracle) 192d122019bSMatthew Wilcox (Oracle) static inline int slab_nid(const struct slab *slab) 193d122019bSMatthew Wilcox (Oracle) { 194d122019bSMatthew Wilcox (Oracle) return folio_nid(slab_folio(slab)); 195d122019bSMatthew Wilcox (Oracle) } 196d122019bSMatthew Wilcox (Oracle) 197d122019bSMatthew Wilcox (Oracle) static inline pg_data_t *slab_pgdat(const struct slab *slab) 198d122019bSMatthew Wilcox (Oracle) { 199d122019bSMatthew Wilcox (Oracle) return folio_pgdat(slab_folio(slab)); 200d122019bSMatthew Wilcox (Oracle) } 201d122019bSMatthew Wilcox (Oracle) 202d122019bSMatthew Wilcox (Oracle) static inline struct slab *virt_to_slab(const void *addr) 203d122019bSMatthew Wilcox (Oracle) { 204d122019bSMatthew Wilcox (Oracle) struct folio *folio = virt_to_folio(addr); 205d122019bSMatthew Wilcox (Oracle) 206d122019bSMatthew Wilcox (Oracle) if (!folio_test_slab(folio)) 207d122019bSMatthew Wilcox (Oracle) return NULL; 208d122019bSMatthew Wilcox (Oracle) 209d122019bSMatthew Wilcox (Oracle) return folio_slab(folio); 210d122019bSMatthew Wilcox (Oracle) } 211d122019bSMatthew Wilcox (Oracle) 212d122019bSMatthew Wilcox (Oracle) static inline int slab_order(const struct slab *slab) 213d122019bSMatthew Wilcox (Oracle) { 214d122019bSMatthew Wilcox (Oracle) return folio_order((struct folio *)slab_folio(slab)); 215d122019bSMatthew Wilcox (Oracle) } 216d122019bSMatthew Wilcox (Oracle) 217d122019bSMatthew Wilcox (Oracle) static inline size_t slab_size(const struct slab *slab) 218d122019bSMatthew Wilcox (Oracle) { 219d122019bSMatthew Wilcox (Oracle) return PAGE_SIZE << slab_order(slab); 220d122019bSMatthew Wilcox (Oracle) } 221d122019bSMatthew Wilcox (Oracle) 22219975f83SVlastimil Babka #ifdef CONFIG_SLUB_CPU_PARTIAL 22319975f83SVlastimil Babka #define slub_percpu_partial(c) ((c)->partial) 22419975f83SVlastimil Babka 22519975f83SVlastimil Babka #define slub_set_percpu_partial(c, p) \ 22619975f83SVlastimil Babka ({ \ 22719975f83SVlastimil Babka slub_percpu_partial(c) = (p)->next; \ 22819975f83SVlastimil Babka }) 22919975f83SVlastimil Babka 23019975f83SVlastimil Babka #define slub_percpu_partial_read_once(c) READ_ONCE(slub_percpu_partial(c)) 23119975f83SVlastimil Babka #else 23219975f83SVlastimil Babka #define slub_percpu_partial(c) NULL 23319975f83SVlastimil Babka 23419975f83SVlastimil Babka #define slub_set_percpu_partial(c, p) 23519975f83SVlastimil Babka 23619975f83SVlastimil Babka #define slub_percpu_partial_read_once(c) NULL 23719975f83SVlastimil Babka #endif // CONFIG_SLUB_CPU_PARTIAL 23819975f83SVlastimil Babka 23919975f83SVlastimil Babka /* 24019975f83SVlastimil Babka * Word size structure that can be atomically updated or read and that 24119975f83SVlastimil Babka * contains both the order and the number of objects that a slab of the 24219975f83SVlastimil Babka * given order would contain. 24319975f83SVlastimil Babka */ 24419975f83SVlastimil Babka struct kmem_cache_order_objects { 24519975f83SVlastimil Babka unsigned int x; 24619975f83SVlastimil Babka }; 24719975f83SVlastimil Babka 24819975f83SVlastimil Babka /* 24919975f83SVlastimil Babka * Slab cache management. 25019975f83SVlastimil Babka */ 25119975f83SVlastimil Babka struct kmem_cache { 25219975f83SVlastimil Babka #ifndef CONFIG_SLUB_TINY 25319975f83SVlastimil Babka struct kmem_cache_cpu __percpu *cpu_slab; 25419975f83SVlastimil Babka #endif 25519975f83SVlastimil Babka /* Used for retrieving partial slabs, etc. */ 25619975f83SVlastimil Babka slab_flags_t flags; 25719975f83SVlastimil Babka unsigned long min_partial; 25819975f83SVlastimil Babka unsigned int size; /* Object size including metadata */ 25919975f83SVlastimil Babka unsigned int object_size; /* Object size without metadata */ 26019975f83SVlastimil Babka struct reciprocal_value reciprocal_size; 26119975f83SVlastimil Babka unsigned int offset; /* Free pointer offset */ 26219975f83SVlastimil Babka #ifdef CONFIG_SLUB_CPU_PARTIAL 26319975f83SVlastimil Babka /* Number of per cpu partial objects to keep around */ 26419975f83SVlastimil Babka unsigned int cpu_partial; 26519975f83SVlastimil Babka /* Number of per cpu partial slabs to keep around */ 26619975f83SVlastimil Babka unsigned int cpu_partial_slabs; 26719975f83SVlastimil Babka #endif 26819975f83SVlastimil Babka struct kmem_cache_order_objects oo; 26919975f83SVlastimil Babka 27019975f83SVlastimil Babka /* Allocation and freeing of slabs */ 27119975f83SVlastimil Babka struct kmem_cache_order_objects min; 27219975f83SVlastimil Babka gfp_t allocflags; /* gfp flags to use on each alloc */ 27319975f83SVlastimil Babka int refcount; /* Refcount for slab cache destroy */ 27419975f83SVlastimil Babka void (*ctor)(void *object); /* Object constructor */ 27519975f83SVlastimil Babka unsigned int inuse; /* Offset to metadata */ 27619975f83SVlastimil Babka unsigned int align; /* Alignment */ 27719975f83SVlastimil Babka unsigned int red_left_pad; /* Left redzone padding size */ 27819975f83SVlastimil Babka const char *name; /* Name (only for display!) */ 27919975f83SVlastimil Babka struct list_head list; /* List of slab caches */ 28019975f83SVlastimil Babka #ifdef CONFIG_SYSFS 28119975f83SVlastimil Babka struct kobject kobj; /* For sysfs */ 28219975f83SVlastimil Babka #endif 28319975f83SVlastimil Babka #ifdef CONFIG_SLAB_FREELIST_HARDENED 28419975f83SVlastimil Babka unsigned long random; 28519975f83SVlastimil Babka #endif 28619975f83SVlastimil Babka 28719975f83SVlastimil Babka #ifdef CONFIG_NUMA 28819975f83SVlastimil Babka /* 28919975f83SVlastimil Babka * Defragmentation by allocating from a remote node. 29019975f83SVlastimil Babka */ 29119975f83SVlastimil Babka unsigned int remote_node_defrag_ratio; 29219975f83SVlastimil Babka #endif 29319975f83SVlastimil Babka 29419975f83SVlastimil Babka #ifdef CONFIG_SLAB_FREELIST_RANDOM 29519975f83SVlastimil Babka unsigned int *random_seq; 29619975f83SVlastimil Babka #endif 29719975f83SVlastimil Babka 29819975f83SVlastimil Babka #ifdef CONFIG_KASAN_GENERIC 29919975f83SVlastimil Babka struct kasan_cache kasan_info; 30019975f83SVlastimil Babka #endif 30119975f83SVlastimil Babka 30219975f83SVlastimil Babka #ifdef CONFIG_HARDENED_USERCOPY 30319975f83SVlastimil Babka unsigned int useroffset; /* Usercopy region offset */ 30419975f83SVlastimil Babka unsigned int usersize; /* Usercopy region size */ 30519975f83SVlastimil Babka #endif 30619975f83SVlastimil Babka 30719975f83SVlastimil Babka struct kmem_cache_node *node[MAX_NUMNODES]; 30819975f83SVlastimil Babka }; 30919975f83SVlastimil Babka 31019975f83SVlastimil Babka #if defined(CONFIG_SYSFS) && !defined(CONFIG_SLUB_TINY) 31119975f83SVlastimil Babka #define SLAB_SUPPORTS_SYSFS 31219975f83SVlastimil Babka void sysfs_slab_unlink(struct kmem_cache *s); 31319975f83SVlastimil Babka void sysfs_slab_release(struct kmem_cache *s); 31419975f83SVlastimil Babka #else 31519975f83SVlastimil Babka static inline void sysfs_slab_unlink(struct kmem_cache *s) { } 31619975f83SVlastimil Babka static inline void sysfs_slab_release(struct kmem_cache *s) { } 31719975f83SVlastimil Babka #endif 31819975f83SVlastimil Babka 31919975f83SVlastimil Babka void *fixup_red_left(struct kmem_cache *s, void *p); 32019975f83SVlastimil Babka 32119975f83SVlastimil Babka static inline void *nearest_obj(struct kmem_cache *cache, 32219975f83SVlastimil Babka const struct slab *slab, void *x) 32319975f83SVlastimil Babka { 32419975f83SVlastimil Babka void *object = x - (x - slab_address(slab)) % cache->size; 32519975f83SVlastimil Babka void *last_object = slab_address(slab) + 32619975f83SVlastimil Babka (slab->objects - 1) * cache->size; 32719975f83SVlastimil Babka void *result = (unlikely(object > last_object)) ? last_object : object; 32819975f83SVlastimil Babka 32919975f83SVlastimil Babka result = fixup_red_left(cache, result); 33019975f83SVlastimil Babka return result; 33119975f83SVlastimil Babka } 33219975f83SVlastimil Babka 33319975f83SVlastimil Babka /* Determine object index from a given position */ 33419975f83SVlastimil Babka static inline unsigned int __obj_to_index(const struct kmem_cache *cache, 33519975f83SVlastimil Babka void *addr, void *obj) 33619975f83SVlastimil Babka { 33719975f83SVlastimil Babka return reciprocal_divide(kasan_reset_tag(obj) - addr, 33819975f83SVlastimil Babka cache->reciprocal_size); 33919975f83SVlastimil Babka } 34019975f83SVlastimil Babka 34119975f83SVlastimil Babka static inline unsigned int obj_to_index(const struct kmem_cache *cache, 34219975f83SVlastimil Babka const struct slab *slab, void *obj) 34319975f83SVlastimil Babka { 34419975f83SVlastimil Babka if (is_kfence_address(obj)) 34519975f83SVlastimil Babka return 0; 34619975f83SVlastimil Babka return __obj_to_index(cache, slab_address(slab), obj); 34719975f83SVlastimil Babka } 34819975f83SVlastimil Babka 34919975f83SVlastimil Babka static inline int objs_per_slab(const struct kmem_cache *cache, 35019975f83SVlastimil Babka const struct slab *slab) 35119975f83SVlastimil Babka { 35219975f83SVlastimil Babka return slab->objects; 35319975f83SVlastimil Babka } 35407f361b2SJoonsoo Kim 35597d06609SChristoph Lameter /* 35697d06609SChristoph Lameter * State of the slab allocator. 35797d06609SChristoph Lameter * 35897d06609SChristoph Lameter * This is used to describe the states of the allocator during bootup. 35997d06609SChristoph Lameter * Allocators use this to gradually bootstrap themselves. Most allocators 36097d06609SChristoph Lameter * have the problem that the structures used for managing slab caches are 36197d06609SChristoph Lameter * allocated from slab caches themselves. 36297d06609SChristoph Lameter */ 36397d06609SChristoph Lameter enum slab_state { 36497d06609SChristoph Lameter DOWN, /* No slab functionality yet */ 36597d06609SChristoph Lameter PARTIAL, /* SLUB: kmem_cache_node available */ 366ce8eb6c4SChristoph Lameter PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */ 36797d06609SChristoph Lameter UP, /* Slab caches usable but not all extras yet */ 36897d06609SChristoph Lameter FULL /* Everything is working */ 36997d06609SChristoph Lameter }; 37097d06609SChristoph Lameter 37197d06609SChristoph Lameter extern enum slab_state slab_state; 37297d06609SChristoph Lameter 37318004c5dSChristoph Lameter /* The slab cache mutex protects the management structures during changes */ 37418004c5dSChristoph Lameter extern struct mutex slab_mutex; 3759b030cb8SChristoph Lameter 3769b030cb8SChristoph Lameter /* The list of all slab caches on the system */ 37718004c5dSChristoph Lameter extern struct list_head slab_caches; 37818004c5dSChristoph Lameter 3799b030cb8SChristoph Lameter /* The slab cache that manages slab cache information */ 3809b030cb8SChristoph Lameter extern struct kmem_cache *kmem_cache; 3819b030cb8SChristoph Lameter 382af3b5f87SVlastimil Babka /* A table of kmalloc cache names and sizes */ 383af3b5f87SVlastimil Babka extern const struct kmalloc_info_struct { 384cb5d9fb3SPengfei Li const char *name[NR_KMALLOC_TYPES]; 38555de8b9cSAlexey Dobriyan unsigned int size; 386af3b5f87SVlastimil Babka } kmalloc_info[]; 387af3b5f87SVlastimil Babka 388f97d5f63SChristoph Lameter /* Kmalloc array related functions */ 38934cc6990SDaniel Sanders void setup_kmalloc_cache_index_table(void); 390d50112edSAlexey Dobriyan void create_kmalloc_caches(slab_flags_t); 3912c59dd65SChristoph Lameter 3925a9d31d9SVlastimil Babka extern u8 kmalloc_size_index[24]; 3935a9d31d9SVlastimil Babka 3945a9d31d9SVlastimil Babka static inline unsigned int size_index_elem(unsigned int bytes) 3955a9d31d9SVlastimil Babka { 3965a9d31d9SVlastimil Babka return (bytes - 1) / 8; 3975a9d31d9SVlastimil Babka } 3985a9d31d9SVlastimil Babka 3995a9d31d9SVlastimil Babka /* 4005a9d31d9SVlastimil Babka * Find the kmem_cache structure that serves a given size of 4015a9d31d9SVlastimil Babka * allocation 4025a9d31d9SVlastimil Babka * 4035a9d31d9SVlastimil Babka * This assumes size is larger than zero and not larger than 4045a9d31d9SVlastimil Babka * KMALLOC_MAX_CACHE_SIZE and the caller must check that. 4055a9d31d9SVlastimil Babka */ 4065a9d31d9SVlastimil Babka static inline struct kmem_cache * 4075a9d31d9SVlastimil Babka kmalloc_slab(size_t size, gfp_t flags, unsigned long caller) 4085a9d31d9SVlastimil Babka { 4095a9d31d9SVlastimil Babka unsigned int index; 4105a9d31d9SVlastimil Babka 4115a9d31d9SVlastimil Babka if (size <= 192) 4125a9d31d9SVlastimil Babka index = kmalloc_size_index[size_index_elem(size)]; 4135a9d31d9SVlastimil Babka else 4145a9d31d9SVlastimil Babka index = fls(size - 1); 4155a9d31d9SVlastimil Babka 4165a9d31d9SVlastimil Babka return kmalloc_caches[kmalloc_type(flags, caller)][index]; 4175a9d31d9SVlastimil Babka } 418ed4cd17eSHyeonggon Yoo 41944405099SLong Li gfp_t kmalloc_fix_flags(gfp_t flags); 420f97d5f63SChristoph Lameter 4219b030cb8SChristoph Lameter /* Functions provided by the slab allocators */ 422d50112edSAlexey Dobriyan int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags); 42397d06609SChristoph Lameter 42489c2d061SVlastimil Babka void __init kmem_cache_init(void); 4250c474d31SCatalin Marinas void __init new_kmalloc_cache(int idx, enum kmalloc_cache_type type, 4260c474d31SCatalin Marinas slab_flags_t flags); 42745530c44SChristoph Lameter extern void create_boot_cache(struct kmem_cache *, const char *name, 428361d575eSAlexey Dobriyan unsigned int size, slab_flags_t flags, 429361d575eSAlexey Dobriyan unsigned int useroffset, unsigned int usersize); 43045530c44SChristoph Lameter 431423c929cSJoonsoo Kim int slab_unmergeable(struct kmem_cache *s); 432f4957d5bSAlexey Dobriyan struct kmem_cache *find_mergeable(unsigned size, unsigned align, 433d50112edSAlexey Dobriyan slab_flags_t flags, const char *name, void (*ctor)(void *)); 4342633d7a0SGlauber Costa struct kmem_cache * 435f4957d5bSAlexey Dobriyan __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, 436d50112edSAlexey Dobriyan slab_flags_t flags, void (*ctor)(void *)); 437423c929cSJoonsoo Kim 4380293d1fdSAlexey Dobriyan slab_flags_t kmem_cache_flags(unsigned int object_size, 43937540008SNikolay Borisov slab_flags_t flags, const char *name); 440cbb79694SChristoph Lameter 441bb944290SFeng Tang static inline bool is_kmalloc_cache(struct kmem_cache *s) 442bb944290SFeng Tang { 443bb944290SFeng Tang return (s->flags & SLAB_KMALLOC); 444bb944290SFeng Tang } 445cbb79694SChristoph Lameter 446d8843922SGlauber Costa /* Legal flag mask for kmem_cache_create(), for various configurations */ 4476d6ea1e9SNicolas Boichat #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ 4486d6ea1e9SNicolas Boichat SLAB_CACHE_DMA32 | SLAB_PANIC | \ 4495f0d5a3aSPaul E. McKenney SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS ) 450d8843922SGlauber Costa 451a9e0b9f2SVlastimil Babka #ifdef CONFIG_SLUB_DEBUG 452d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 453becfda68SLaura Abbott SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) 454d8843922SGlauber Costa #else 455d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (0) 456d8843922SGlauber Costa #endif 457d8843922SGlauber Costa 458d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 4596cd6d33cSFeng Tang SLAB_TEMPORARY | SLAB_ACCOUNT | \ 460d0bf7d57SJesper Dangaard Brouer SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE) 461d8843922SGlauber Costa 462e70954fdSThomas Garnier /* Common flags available with current configuration */ 463d8843922SGlauber Costa #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS) 464d8843922SGlauber Costa 465e70954fdSThomas Garnier /* Common flags permitted for kmem_cache_create */ 466e70954fdSThomas Garnier #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \ 467e70954fdSThomas Garnier SLAB_RED_ZONE | \ 468e70954fdSThomas Garnier SLAB_POISON | \ 469e70954fdSThomas Garnier SLAB_STORE_USER | \ 470e70954fdSThomas Garnier SLAB_TRACE | \ 471e70954fdSThomas Garnier SLAB_CONSISTENCY_CHECKS | \ 472e70954fdSThomas Garnier SLAB_MEM_SPREAD | \ 473e70954fdSThomas Garnier SLAB_NOLEAKTRACE | \ 474e70954fdSThomas Garnier SLAB_RECLAIM_ACCOUNT | \ 475e70954fdSThomas Garnier SLAB_TEMPORARY | \ 476a285909fSHyeonggon Yoo SLAB_ACCOUNT | \ 4776cd6d33cSFeng Tang SLAB_KMALLOC | \ 478d0bf7d57SJesper Dangaard Brouer SLAB_NO_MERGE | \ 479a285909fSHyeonggon Yoo SLAB_NO_USER_FLAGS) 480e70954fdSThomas Garnier 481f9e13c0aSShakeel Butt bool __kmem_cache_empty(struct kmem_cache *); 482945cf2b6SChristoph Lameter int __kmem_cache_shutdown(struct kmem_cache *); 48352b4b950SDmitry Safonov void __kmem_cache_release(struct kmem_cache *); 484c9fc5864STejun Heo int __kmem_cache_shrink(struct kmem_cache *); 48541a21285SChristoph Lameter void slab_kmem_cache_release(struct kmem_cache *); 486945cf2b6SChristoph Lameter 487b7454ad3SGlauber Costa struct seq_file; 488b7454ad3SGlauber Costa struct file; 489b7454ad3SGlauber Costa 4900d7561c6SGlauber Costa struct slabinfo { 4910d7561c6SGlauber Costa unsigned long active_objs; 4920d7561c6SGlauber Costa unsigned long num_objs; 4930d7561c6SGlauber Costa unsigned long active_slabs; 4940d7561c6SGlauber Costa unsigned long num_slabs; 4950d7561c6SGlauber Costa unsigned long shared_avail; 4960d7561c6SGlauber Costa unsigned int limit; 4970d7561c6SGlauber Costa unsigned int batchcount; 4980d7561c6SGlauber Costa unsigned int shared; 4990d7561c6SGlauber Costa unsigned int objects_per_slab; 5000d7561c6SGlauber Costa unsigned int cache_order; 5010d7561c6SGlauber Costa }; 5020d7561c6SGlauber Costa 5030d7561c6SGlauber Costa void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); 5040d7561c6SGlauber Costa void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); 505b7454ad3SGlauber Costa ssize_t slabinfo_write(struct file *file, const char __user *buffer, 506b7454ad3SGlauber Costa size_t count, loff_t *ppos); 507ba6c496eSGlauber Costa 508e42f174eSVlastimil Babka #ifdef CONFIG_SLUB_DEBUG 509e42f174eSVlastimil Babka #ifdef CONFIG_SLUB_DEBUG_ON 510e42f174eSVlastimil Babka DECLARE_STATIC_KEY_TRUE(slub_debug_enabled); 511e42f174eSVlastimil Babka #else 512e42f174eSVlastimil Babka DECLARE_STATIC_KEY_FALSE(slub_debug_enabled); 513e42f174eSVlastimil Babka #endif 514e42f174eSVlastimil Babka extern void print_tracking(struct kmem_cache *s, void *object); 5151f9f78b1SOliver Glitta long validate_slab_cache(struct kmem_cache *s); 5160d4a062aSMarco Elver static inline bool __slub_debug_enabled(void) 5170d4a062aSMarco Elver { 5180d4a062aSMarco Elver return static_branch_unlikely(&slub_debug_enabled); 5190d4a062aSMarco Elver } 520e42f174eSVlastimil Babka #else 521e42f174eSVlastimil Babka static inline void print_tracking(struct kmem_cache *s, void *object) 522e42f174eSVlastimil Babka { 523e42f174eSVlastimil Babka } 5240d4a062aSMarco Elver static inline bool __slub_debug_enabled(void) 5250d4a062aSMarco Elver { 5260d4a062aSMarco Elver return false; 5270d4a062aSMarco Elver } 528e42f174eSVlastimil Babka #endif 529e42f174eSVlastimil Babka 530e42f174eSVlastimil Babka /* 531*671776b3SXiongwei Song * Returns true if any of the specified slab_debug flags is enabled for the 532e42f174eSVlastimil Babka * cache. Use only for flags parsed by setup_slub_debug() as it also enables 533e42f174eSVlastimil Babka * the static key. 534e42f174eSVlastimil Babka */ 535e42f174eSVlastimil Babka static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags) 536e42f174eSVlastimil Babka { 5370d4a062aSMarco Elver if (IS_ENABLED(CONFIG_SLUB_DEBUG)) 538e42f174eSVlastimil Babka VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS)); 5390d4a062aSMarco Elver if (__slub_debug_enabled()) 540e42f174eSVlastimil Babka return s->flags & flags; 541e42f174eSVlastimil Babka return false; 542e42f174eSVlastimil Babka } 543e42f174eSVlastimil Babka 54484c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM 5454b5f8d9aSVlastimil Babka /* 5464b5f8d9aSVlastimil Babka * slab_objcgs - get the object cgroups vector associated with a slab 5474b5f8d9aSVlastimil Babka * @slab: a pointer to the slab struct 5484b5f8d9aSVlastimil Babka * 5494b5f8d9aSVlastimil Babka * Returns a pointer to the object cgroups vector associated with the slab, 5504b5f8d9aSVlastimil Babka * or NULL if no such vector has been associated yet. 5514b5f8d9aSVlastimil Babka */ 5524b5f8d9aSVlastimil Babka static inline struct obj_cgroup **slab_objcgs(struct slab *slab) 5534b5f8d9aSVlastimil Babka { 5544b5f8d9aSVlastimil Babka unsigned long memcg_data = READ_ONCE(slab->memcg_data); 5554b5f8d9aSVlastimil Babka 5564b5f8d9aSVlastimil Babka VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), 5574b5f8d9aSVlastimil Babka slab_page(slab)); 5584b5f8d9aSVlastimil Babka VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab)); 5594b5f8d9aSVlastimil Babka 5604b5f8d9aSVlastimil Babka return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 5614b5f8d9aSVlastimil Babka } 5624b5f8d9aSVlastimil Babka 5634b5f8d9aSVlastimil Babka int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, 5644b5f8d9aSVlastimil Babka gfp_t gfp, bool new_slab); 565fdbcb2a6SWaiman Long void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, 566fdbcb2a6SWaiman Long enum node_stat_item idx, int nr); 56784c07d11SKirill Tkhai #else /* CONFIG_MEMCG_KMEM */ 5684b5f8d9aSVlastimil Babka static inline struct obj_cgroup **slab_objcgs(struct slab *slab) 5694b5f8d9aSVlastimil Babka { 5704b5f8d9aSVlastimil Babka return NULL; 5714b5f8d9aSVlastimil Babka } 5724b5f8d9aSVlastimil Babka 5734b5f8d9aSVlastimil Babka static inline int memcg_alloc_slab_cgroups(struct slab *slab, 5742e9bd483SRoman Gushchin struct kmem_cache *s, gfp_t gfp, 5754b5f8d9aSVlastimil Babka bool new_slab) 576286e04b8SRoman Gushchin { 577286e04b8SRoman Gushchin return 0; 578286e04b8SRoman Gushchin } 57984c07d11SKirill Tkhai #endif /* CONFIG_MEMCG_KMEM */ 580b9ce5ef4SGlauber Costa 5818dfa9d55SHyeonggon Yoo size_t __ksize(const void *objp); 5828dfa9d55SHyeonggon Yoo 58311c7aec2SJesper Dangaard Brouer static inline size_t slab_ksize(const struct kmem_cache *s) 58411c7aec2SJesper Dangaard Brouer { 58511c7aec2SJesper Dangaard Brouer #ifdef CONFIG_SLUB_DEBUG 58611c7aec2SJesper Dangaard Brouer /* 58711c7aec2SJesper Dangaard Brouer * Debugging requires use of the padding between object 58811c7aec2SJesper Dangaard Brouer * and whatever may come after it. 58911c7aec2SJesper Dangaard Brouer */ 59011c7aec2SJesper Dangaard Brouer if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) 59111c7aec2SJesper Dangaard Brouer return s->object_size; 59211c7aec2SJesper Dangaard Brouer #endif 59380a9201aSAlexander Potapenko if (s->flags & SLAB_KASAN) 59480a9201aSAlexander Potapenko return s->object_size; 59511c7aec2SJesper Dangaard Brouer /* 59611c7aec2SJesper Dangaard Brouer * If we have the need to store the freelist pointer 59711c7aec2SJesper Dangaard Brouer * back there or track user information then we can 59811c7aec2SJesper Dangaard Brouer * only use the space before that information. 59911c7aec2SJesper Dangaard Brouer */ 6005f0d5a3aSPaul E. McKenney if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER)) 60111c7aec2SJesper Dangaard Brouer return s->inuse; 60211c7aec2SJesper Dangaard Brouer /* 60311c7aec2SJesper Dangaard Brouer * Else we can use all the padding etc for the allocation 60411c7aec2SJesper Dangaard Brouer */ 60511c7aec2SJesper Dangaard Brouer return s->size; 60611c7aec2SJesper Dangaard Brouer } 60711c7aec2SJesper Dangaard Brouer 608a9e0b9f2SVlastimil Babka #ifdef CONFIG_SLUB_DEBUG 609852d8be0SYang Shi void dump_unreclaimable_slab(void); 610852d8be0SYang Shi #else 611852d8be0SYang Shi static inline void dump_unreclaimable_slab(void) 612852d8be0SYang Shi { 613852d8be0SYang Shi } 614852d8be0SYang Shi #endif 615852d8be0SYang Shi 61655834c59SAlexander Potapenko void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr); 61755834c59SAlexander Potapenko 6187c00fce9SThomas Garnier #ifdef CONFIG_SLAB_FREELIST_RANDOM 6197c00fce9SThomas Garnier int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 6207c00fce9SThomas Garnier gfp_t gfp); 6217c00fce9SThomas Garnier void cache_random_seq_destroy(struct kmem_cache *cachep); 6227c00fce9SThomas Garnier #else 6237c00fce9SThomas Garnier static inline int cache_random_seq_create(struct kmem_cache *cachep, 6247c00fce9SThomas Garnier unsigned int count, gfp_t gfp) 6257c00fce9SThomas Garnier { 6267c00fce9SThomas Garnier return 0; 6277c00fce9SThomas Garnier } 6287c00fce9SThomas Garnier static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } 6297c00fce9SThomas Garnier #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 6307c00fce9SThomas Garnier 6316471384aSAlexander Potapenko static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c) 6326471384aSAlexander Potapenko { 63351cba1ebSKees Cook if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 63451cba1ebSKees Cook &init_on_alloc)) { 6356471384aSAlexander Potapenko if (c->ctor) 6366471384aSAlexander Potapenko return false; 6376471384aSAlexander Potapenko if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) 6386471384aSAlexander Potapenko return flags & __GFP_ZERO; 6396471384aSAlexander Potapenko return true; 6406471384aSAlexander Potapenko } 6416471384aSAlexander Potapenko return flags & __GFP_ZERO; 6426471384aSAlexander Potapenko } 6436471384aSAlexander Potapenko 6446471384aSAlexander Potapenko static inline bool slab_want_init_on_free(struct kmem_cache *c) 6456471384aSAlexander Potapenko { 64651cba1ebSKees Cook if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 64751cba1ebSKees Cook &init_on_free)) 6486471384aSAlexander Potapenko return !(c->ctor || 6496471384aSAlexander Potapenko (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 6506471384aSAlexander Potapenko return false; 6516471384aSAlexander Potapenko } 6526471384aSAlexander Potapenko 65364dd6849SFaiyaz Mohammed #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 65464dd6849SFaiyaz Mohammed void debugfs_slab_release(struct kmem_cache *); 65564dd6849SFaiyaz Mohammed #else 65664dd6849SFaiyaz Mohammed static inline void debugfs_slab_release(struct kmem_cache *s) { } 65764dd6849SFaiyaz Mohammed #endif 65864dd6849SFaiyaz Mohammed 6595bb1bb35SPaul E. McKenney #ifdef CONFIG_PRINTK 6608e7f37f2SPaul E. McKenney #define KS_ADDRS_COUNT 16 6618e7f37f2SPaul E. McKenney struct kmem_obj_info { 6628e7f37f2SPaul E. McKenney void *kp_ptr; 6637213230aSMatthew Wilcox (Oracle) struct slab *kp_slab; 6648e7f37f2SPaul E. McKenney void *kp_objp; 6658e7f37f2SPaul E. McKenney unsigned long kp_data_offset; 6668e7f37f2SPaul E. McKenney struct kmem_cache *kp_slab_cache; 6678e7f37f2SPaul E. McKenney void *kp_ret; 6688e7f37f2SPaul E. McKenney void *kp_stack[KS_ADDRS_COUNT]; 669e548eaa1SManinder Singh void *kp_free_stack[KS_ADDRS_COUNT]; 6708e7f37f2SPaul E. McKenney }; 6712dfe63e6SMarco Elver void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab); 6725bb1bb35SPaul E. McKenney #endif 6738e7f37f2SPaul E. McKenney 6740b3eb091SMatthew Wilcox (Oracle) void __check_heap_object(const void *ptr, unsigned long n, 6750b3eb091SMatthew Wilcox (Oracle) const struct slab *slab, bool to_user); 6760b3eb091SMatthew Wilcox (Oracle) 677946fa0dbSFeng Tang #ifdef CONFIG_SLUB_DEBUG 678946fa0dbSFeng Tang void skip_orig_size_check(struct kmem_cache *s, const void *object); 679946fa0dbSFeng Tang #endif 680946fa0dbSFeng Tang 6815240ab40SAndrey Ryabinin #endif /* MM_SLAB_H */ 682