1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 297d06609SChristoph Lameter #ifndef MM_SLAB_H 397d06609SChristoph Lameter #define MM_SLAB_H 4*89c2d061SVlastimil Babka 5*89c2d061SVlastimil Babka #include <linux/reciprocal_div.h> 6*89c2d061SVlastimil Babka #include <linux/list_lru.h> 7*89c2d061SVlastimil Babka #include <linux/local_lock.h> 8*89c2d061SVlastimil Babka #include <linux/random.h> 9*89c2d061SVlastimil Babka #include <linux/kobject.h> 10*89c2d061SVlastimil Babka #include <linux/sched/mm.h> 11*89c2d061SVlastimil Babka #include <linux/memcontrol.h> 12*89c2d061SVlastimil Babka #include <linux/fault-inject.h> 13*89c2d061SVlastimil Babka #include <linux/kmemleak.h> 14*89c2d061SVlastimil Babka #include <linux/kfence.h> 15*89c2d061SVlastimil Babka #include <linux/kasan.h> 16*89c2d061SVlastimil Babka 1797d06609SChristoph Lameter /* 1897d06609SChristoph Lameter * Internal slab definitions 1997d06609SChristoph Lameter */ 2097d06609SChristoph Lameter 216801be4fSPeter Zijlstra #ifdef CONFIG_64BIT 226801be4fSPeter Zijlstra # ifdef system_has_cmpxchg128 236801be4fSPeter Zijlstra # define system_has_freelist_aba() system_has_cmpxchg128() 246801be4fSPeter Zijlstra # define try_cmpxchg_freelist try_cmpxchg128 256801be4fSPeter Zijlstra # endif 266801be4fSPeter Zijlstra #define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg128 276801be4fSPeter Zijlstra typedef u128 freelist_full_t; 286801be4fSPeter Zijlstra #else /* CONFIG_64BIT */ 296801be4fSPeter Zijlstra # ifdef system_has_cmpxchg64 306801be4fSPeter Zijlstra # define system_has_freelist_aba() system_has_cmpxchg64() 316801be4fSPeter Zijlstra # define try_cmpxchg_freelist try_cmpxchg64 326801be4fSPeter Zijlstra # endif 336801be4fSPeter Zijlstra #define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg64 346801be4fSPeter Zijlstra typedef u64 freelist_full_t; 356801be4fSPeter Zijlstra #endif /* CONFIG_64BIT */ 366801be4fSPeter Zijlstra 376801be4fSPeter Zijlstra #if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 386801be4fSPeter Zijlstra #undef system_has_freelist_aba 396801be4fSPeter Zijlstra #endif 406801be4fSPeter Zijlstra 416801be4fSPeter Zijlstra /* 426801be4fSPeter Zijlstra * Freelist pointer and counter to cmpxchg together, avoids the typical ABA 436801be4fSPeter Zijlstra * problems with cmpxchg of just a pointer. 446801be4fSPeter Zijlstra */ 456801be4fSPeter Zijlstra typedef union { 466801be4fSPeter Zijlstra struct { 476801be4fSPeter Zijlstra void *freelist; 486801be4fSPeter Zijlstra unsigned long counter; 496801be4fSPeter Zijlstra }; 506801be4fSPeter Zijlstra freelist_full_t full; 516801be4fSPeter Zijlstra } freelist_aba_t; 526801be4fSPeter Zijlstra 53d122019bSMatthew Wilcox (Oracle) /* Reuses the bits in struct page */ 54d122019bSMatthew Wilcox (Oracle) struct slab { 55d122019bSMatthew Wilcox (Oracle) unsigned long __page_flags; 56401fb12cSVlastimil Babka 57130d4df5SVlastimil Babka struct kmem_cache *slab_cache; 58130d4df5SVlastimil Babka union { 59130d4df5SVlastimil Babka struct { 60401fb12cSVlastimil Babka union { 61401fb12cSVlastimil Babka struct list_head slab_list; 629c01e9afSVlastimil Babka #ifdef CONFIG_SLUB_CPU_PARTIAL 63401fb12cSVlastimil Babka struct { 64d122019bSMatthew Wilcox (Oracle) struct slab *next; 65d122019bSMatthew Wilcox (Oracle) int slabs; /* Nr of slabs left */ 66d122019bSMatthew Wilcox (Oracle) }; 679c01e9afSVlastimil Babka #endif 68d122019bSMatthew Wilcox (Oracle) }; 69d122019bSMatthew Wilcox (Oracle) /* Double-word boundary */ 706801be4fSPeter Zijlstra union { 716801be4fSPeter Zijlstra struct { 72d122019bSMatthew Wilcox (Oracle) void *freelist; /* first free object */ 73d122019bSMatthew Wilcox (Oracle) union { 74401fb12cSVlastimil Babka unsigned long counters; 75401fb12cSVlastimil Babka struct { 76d122019bSMatthew Wilcox (Oracle) unsigned inuse:16; 77d122019bSMatthew Wilcox (Oracle) unsigned objects:15; 78d122019bSMatthew Wilcox (Oracle) unsigned frozen:1; 79d122019bSMatthew Wilcox (Oracle) }; 80d122019bSMatthew Wilcox (Oracle) }; 81130d4df5SVlastimil Babka }; 826801be4fSPeter Zijlstra #ifdef system_has_freelist_aba 836801be4fSPeter Zijlstra freelist_aba_t freelist_counter; 846801be4fSPeter Zijlstra #endif 856801be4fSPeter Zijlstra }; 866801be4fSPeter Zijlstra }; 87130d4df5SVlastimil Babka struct rcu_head rcu_head; 88130d4df5SVlastimil Babka }; 89401fb12cSVlastimil Babka unsigned int __unused; 90d122019bSMatthew Wilcox (Oracle) 91d122019bSMatthew Wilcox (Oracle) atomic_t __page_refcount; 92d122019bSMatthew Wilcox (Oracle) #ifdef CONFIG_MEMCG 93d122019bSMatthew Wilcox (Oracle) unsigned long memcg_data; 94d122019bSMatthew Wilcox (Oracle) #endif 95d122019bSMatthew Wilcox (Oracle) }; 96d122019bSMatthew Wilcox (Oracle) 97d122019bSMatthew Wilcox (Oracle) #define SLAB_MATCH(pg, sl) \ 98d122019bSMatthew Wilcox (Oracle) static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) 99d122019bSMatthew Wilcox (Oracle) SLAB_MATCH(flags, __page_flags); 100130d4df5SVlastimil Babka SLAB_MATCH(compound_head, slab_cache); /* Ensure bit 0 is clear */ 101d122019bSMatthew Wilcox (Oracle) SLAB_MATCH(_refcount, __page_refcount); 102d122019bSMatthew Wilcox (Oracle) #ifdef CONFIG_MEMCG 103d122019bSMatthew Wilcox (Oracle) SLAB_MATCH(memcg_data, memcg_data); 104d122019bSMatthew Wilcox (Oracle) #endif 105d122019bSMatthew Wilcox (Oracle) #undef SLAB_MATCH 106d122019bSMatthew Wilcox (Oracle) static_assert(sizeof(struct slab) <= sizeof(struct page)); 107a9e0b9f2SVlastimil Babka #if defined(system_has_freelist_aba) 1086801be4fSPeter Zijlstra static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t))); 109130d4df5SVlastimil Babka #endif 110d122019bSMatthew Wilcox (Oracle) 111d122019bSMatthew Wilcox (Oracle) /** 112d122019bSMatthew Wilcox (Oracle) * folio_slab - Converts from folio to slab. 113d122019bSMatthew Wilcox (Oracle) * @folio: The folio. 114d122019bSMatthew Wilcox (Oracle) * 115d122019bSMatthew Wilcox (Oracle) * Currently struct slab is a different representation of a folio where 116d122019bSMatthew Wilcox (Oracle) * folio_test_slab() is true. 117d122019bSMatthew Wilcox (Oracle) * 118d122019bSMatthew Wilcox (Oracle) * Return: The slab which contains this folio. 119d122019bSMatthew Wilcox (Oracle) */ 120d122019bSMatthew Wilcox (Oracle) #define folio_slab(folio) (_Generic((folio), \ 121d122019bSMatthew Wilcox (Oracle) const struct folio *: (const struct slab *)(folio), \ 122d122019bSMatthew Wilcox (Oracle) struct folio *: (struct slab *)(folio))) 123d122019bSMatthew Wilcox (Oracle) 124d122019bSMatthew Wilcox (Oracle) /** 125d122019bSMatthew Wilcox (Oracle) * slab_folio - The folio allocated for a slab 126d122019bSMatthew Wilcox (Oracle) * @slab: The slab. 127d122019bSMatthew Wilcox (Oracle) * 128d122019bSMatthew Wilcox (Oracle) * Slabs are allocated as folios that contain the individual objects and are 129d122019bSMatthew Wilcox (Oracle) * using some fields in the first struct page of the folio - those fields are 130d122019bSMatthew Wilcox (Oracle) * now accessed by struct slab. It is occasionally necessary to convert back to 131d122019bSMatthew Wilcox (Oracle) * a folio in order to communicate with the rest of the mm. Please use this 132d122019bSMatthew Wilcox (Oracle) * helper function instead of casting yourself, as the implementation may change 133d122019bSMatthew Wilcox (Oracle) * in the future. 134d122019bSMatthew Wilcox (Oracle) */ 135d122019bSMatthew Wilcox (Oracle) #define slab_folio(s) (_Generic((s), \ 136d122019bSMatthew Wilcox (Oracle) const struct slab *: (const struct folio *)s, \ 137d122019bSMatthew Wilcox (Oracle) struct slab *: (struct folio *)s)) 138d122019bSMatthew Wilcox (Oracle) 139d122019bSMatthew Wilcox (Oracle) /** 140d122019bSMatthew Wilcox (Oracle) * page_slab - Converts from first struct page to slab. 141d122019bSMatthew Wilcox (Oracle) * @p: The first (either head of compound or single) page of slab. 142d122019bSMatthew Wilcox (Oracle) * 143d122019bSMatthew Wilcox (Oracle) * A temporary wrapper to convert struct page to struct slab in situations where 144d122019bSMatthew Wilcox (Oracle) * we know the page is the compound head, or single order-0 page. 145d122019bSMatthew Wilcox (Oracle) * 146d122019bSMatthew Wilcox (Oracle) * Long-term ideally everything would work with struct slab directly or go 147d122019bSMatthew Wilcox (Oracle) * through folio to struct slab. 148d122019bSMatthew Wilcox (Oracle) * 149d122019bSMatthew Wilcox (Oracle) * Return: The slab which contains this page 150d122019bSMatthew Wilcox (Oracle) */ 151d122019bSMatthew Wilcox (Oracle) #define page_slab(p) (_Generic((p), \ 152d122019bSMatthew Wilcox (Oracle) const struct page *: (const struct slab *)(p), \ 153d122019bSMatthew Wilcox (Oracle) struct page *: (struct slab *)(p))) 154d122019bSMatthew Wilcox (Oracle) 155d122019bSMatthew Wilcox (Oracle) /** 156d122019bSMatthew Wilcox (Oracle) * slab_page - The first struct page allocated for a slab 157d122019bSMatthew Wilcox (Oracle) * @slab: The slab. 158d122019bSMatthew Wilcox (Oracle) * 159d122019bSMatthew Wilcox (Oracle) * A convenience wrapper for converting slab to the first struct page of the 160d122019bSMatthew Wilcox (Oracle) * underlying folio, to communicate with code not yet converted to folio or 161d122019bSMatthew Wilcox (Oracle) * struct slab. 162d122019bSMatthew Wilcox (Oracle) */ 163d122019bSMatthew Wilcox (Oracle) #define slab_page(s) folio_page(slab_folio(s), 0) 164d122019bSMatthew Wilcox (Oracle) 165d122019bSMatthew Wilcox (Oracle) /* 166d122019bSMatthew Wilcox (Oracle) * If network-based swap is enabled, sl*b must keep track of whether pages 167d122019bSMatthew Wilcox (Oracle) * were allocated from pfmemalloc reserves. 168d122019bSMatthew Wilcox (Oracle) */ 169d122019bSMatthew Wilcox (Oracle) static inline bool slab_test_pfmemalloc(const struct slab *slab) 170d122019bSMatthew Wilcox (Oracle) { 171d122019bSMatthew Wilcox (Oracle) return folio_test_active((struct folio *)slab_folio(slab)); 172d122019bSMatthew Wilcox (Oracle) } 173d122019bSMatthew Wilcox (Oracle) 174d122019bSMatthew Wilcox (Oracle) static inline void slab_set_pfmemalloc(struct slab *slab) 175d122019bSMatthew Wilcox (Oracle) { 176d122019bSMatthew Wilcox (Oracle) folio_set_active(slab_folio(slab)); 177d122019bSMatthew Wilcox (Oracle) } 178d122019bSMatthew Wilcox (Oracle) 179d122019bSMatthew Wilcox (Oracle) static inline void slab_clear_pfmemalloc(struct slab *slab) 180d122019bSMatthew Wilcox (Oracle) { 181d122019bSMatthew Wilcox (Oracle) folio_clear_active(slab_folio(slab)); 182d122019bSMatthew Wilcox (Oracle) } 183d122019bSMatthew Wilcox (Oracle) 184d122019bSMatthew Wilcox (Oracle) static inline void __slab_clear_pfmemalloc(struct slab *slab) 185d122019bSMatthew Wilcox (Oracle) { 186d122019bSMatthew Wilcox (Oracle) __folio_clear_active(slab_folio(slab)); 187d122019bSMatthew Wilcox (Oracle) } 188d122019bSMatthew Wilcox (Oracle) 189d122019bSMatthew Wilcox (Oracle) static inline void *slab_address(const struct slab *slab) 190d122019bSMatthew Wilcox (Oracle) { 191d122019bSMatthew Wilcox (Oracle) return folio_address(slab_folio(slab)); 192d122019bSMatthew Wilcox (Oracle) } 193d122019bSMatthew Wilcox (Oracle) 194d122019bSMatthew Wilcox (Oracle) static inline int slab_nid(const struct slab *slab) 195d122019bSMatthew Wilcox (Oracle) { 196d122019bSMatthew Wilcox (Oracle) return folio_nid(slab_folio(slab)); 197d122019bSMatthew Wilcox (Oracle) } 198d122019bSMatthew Wilcox (Oracle) 199d122019bSMatthew Wilcox (Oracle) static inline pg_data_t *slab_pgdat(const struct slab *slab) 200d122019bSMatthew Wilcox (Oracle) { 201d122019bSMatthew Wilcox (Oracle) return folio_pgdat(slab_folio(slab)); 202d122019bSMatthew Wilcox (Oracle) } 203d122019bSMatthew Wilcox (Oracle) 204d122019bSMatthew Wilcox (Oracle) static inline struct slab *virt_to_slab(const void *addr) 205d122019bSMatthew Wilcox (Oracle) { 206d122019bSMatthew Wilcox (Oracle) struct folio *folio = virt_to_folio(addr); 207d122019bSMatthew Wilcox (Oracle) 208d122019bSMatthew Wilcox (Oracle) if (!folio_test_slab(folio)) 209d122019bSMatthew Wilcox (Oracle) return NULL; 210d122019bSMatthew Wilcox (Oracle) 211d122019bSMatthew Wilcox (Oracle) return folio_slab(folio); 212d122019bSMatthew Wilcox (Oracle) } 213d122019bSMatthew Wilcox (Oracle) 214d122019bSMatthew Wilcox (Oracle) static inline int slab_order(const struct slab *slab) 215d122019bSMatthew Wilcox (Oracle) { 216d122019bSMatthew Wilcox (Oracle) return folio_order((struct folio *)slab_folio(slab)); 217d122019bSMatthew Wilcox (Oracle) } 218d122019bSMatthew Wilcox (Oracle) 219d122019bSMatthew Wilcox (Oracle) static inline size_t slab_size(const struct slab *slab) 220d122019bSMatthew Wilcox (Oracle) { 221d122019bSMatthew Wilcox (Oracle) return PAGE_SIZE << slab_order(slab); 222d122019bSMatthew Wilcox (Oracle) } 223d122019bSMatthew Wilcox (Oracle) 22419975f83SVlastimil Babka #ifdef CONFIG_SLUB_CPU_PARTIAL 22519975f83SVlastimil Babka #define slub_percpu_partial(c) ((c)->partial) 22619975f83SVlastimil Babka 22719975f83SVlastimil Babka #define slub_set_percpu_partial(c, p) \ 22819975f83SVlastimil Babka ({ \ 22919975f83SVlastimil Babka slub_percpu_partial(c) = (p)->next; \ 23019975f83SVlastimil Babka }) 23119975f83SVlastimil Babka 23219975f83SVlastimil Babka #define slub_percpu_partial_read_once(c) READ_ONCE(slub_percpu_partial(c)) 23319975f83SVlastimil Babka #else 23419975f83SVlastimil Babka #define slub_percpu_partial(c) NULL 23519975f83SVlastimil Babka 23619975f83SVlastimil Babka #define slub_set_percpu_partial(c, p) 23719975f83SVlastimil Babka 23819975f83SVlastimil Babka #define slub_percpu_partial_read_once(c) NULL 23919975f83SVlastimil Babka #endif // CONFIG_SLUB_CPU_PARTIAL 24019975f83SVlastimil Babka 24119975f83SVlastimil Babka /* 24219975f83SVlastimil Babka * Word size structure that can be atomically updated or read and that 24319975f83SVlastimil Babka * contains both the order and the number of objects that a slab of the 24419975f83SVlastimil Babka * given order would contain. 24519975f83SVlastimil Babka */ 24619975f83SVlastimil Babka struct kmem_cache_order_objects { 24719975f83SVlastimil Babka unsigned int x; 24819975f83SVlastimil Babka }; 24919975f83SVlastimil Babka 25019975f83SVlastimil Babka /* 25119975f83SVlastimil Babka * Slab cache management. 25219975f83SVlastimil Babka */ 25319975f83SVlastimil Babka struct kmem_cache { 25419975f83SVlastimil Babka #ifndef CONFIG_SLUB_TINY 25519975f83SVlastimil Babka struct kmem_cache_cpu __percpu *cpu_slab; 25619975f83SVlastimil Babka #endif 25719975f83SVlastimil Babka /* Used for retrieving partial slabs, etc. */ 25819975f83SVlastimil Babka slab_flags_t flags; 25919975f83SVlastimil Babka unsigned long min_partial; 26019975f83SVlastimil Babka unsigned int size; /* Object size including metadata */ 26119975f83SVlastimil Babka unsigned int object_size; /* Object size without metadata */ 26219975f83SVlastimil Babka struct reciprocal_value reciprocal_size; 26319975f83SVlastimil Babka unsigned int offset; /* Free pointer offset */ 26419975f83SVlastimil Babka #ifdef CONFIG_SLUB_CPU_PARTIAL 26519975f83SVlastimil Babka /* Number of per cpu partial objects to keep around */ 26619975f83SVlastimil Babka unsigned int cpu_partial; 26719975f83SVlastimil Babka /* Number of per cpu partial slabs to keep around */ 26819975f83SVlastimil Babka unsigned int cpu_partial_slabs; 26919975f83SVlastimil Babka #endif 27019975f83SVlastimil Babka struct kmem_cache_order_objects oo; 27119975f83SVlastimil Babka 27219975f83SVlastimil Babka /* Allocation and freeing of slabs */ 27319975f83SVlastimil Babka struct kmem_cache_order_objects min; 27419975f83SVlastimil Babka gfp_t allocflags; /* gfp flags to use on each alloc */ 27519975f83SVlastimil Babka int refcount; /* Refcount for slab cache destroy */ 27619975f83SVlastimil Babka void (*ctor)(void *object); /* Object constructor */ 27719975f83SVlastimil Babka unsigned int inuse; /* Offset to metadata */ 27819975f83SVlastimil Babka unsigned int align; /* Alignment */ 27919975f83SVlastimil Babka unsigned int red_left_pad; /* Left redzone padding size */ 28019975f83SVlastimil Babka const char *name; /* Name (only for display!) */ 28119975f83SVlastimil Babka struct list_head list; /* List of slab caches */ 28219975f83SVlastimil Babka #ifdef CONFIG_SYSFS 28319975f83SVlastimil Babka struct kobject kobj; /* For sysfs */ 28419975f83SVlastimil Babka #endif 28519975f83SVlastimil Babka #ifdef CONFIG_SLAB_FREELIST_HARDENED 28619975f83SVlastimil Babka unsigned long random; 28719975f83SVlastimil Babka #endif 28819975f83SVlastimil Babka 28919975f83SVlastimil Babka #ifdef CONFIG_NUMA 29019975f83SVlastimil Babka /* 29119975f83SVlastimil Babka * Defragmentation by allocating from a remote node. 29219975f83SVlastimil Babka */ 29319975f83SVlastimil Babka unsigned int remote_node_defrag_ratio; 29419975f83SVlastimil Babka #endif 29519975f83SVlastimil Babka 29619975f83SVlastimil Babka #ifdef CONFIG_SLAB_FREELIST_RANDOM 29719975f83SVlastimil Babka unsigned int *random_seq; 29819975f83SVlastimil Babka #endif 29919975f83SVlastimil Babka 30019975f83SVlastimil Babka #ifdef CONFIG_KASAN_GENERIC 30119975f83SVlastimil Babka struct kasan_cache kasan_info; 30219975f83SVlastimil Babka #endif 30319975f83SVlastimil Babka 30419975f83SVlastimil Babka #ifdef CONFIG_HARDENED_USERCOPY 30519975f83SVlastimil Babka unsigned int useroffset; /* Usercopy region offset */ 30619975f83SVlastimil Babka unsigned int usersize; /* Usercopy region size */ 30719975f83SVlastimil Babka #endif 30819975f83SVlastimil Babka 30919975f83SVlastimil Babka struct kmem_cache_node *node[MAX_NUMNODES]; 31019975f83SVlastimil Babka }; 31119975f83SVlastimil Babka 31219975f83SVlastimil Babka #if defined(CONFIG_SYSFS) && !defined(CONFIG_SLUB_TINY) 31319975f83SVlastimil Babka #define SLAB_SUPPORTS_SYSFS 31419975f83SVlastimil Babka void sysfs_slab_unlink(struct kmem_cache *s); 31519975f83SVlastimil Babka void sysfs_slab_release(struct kmem_cache *s); 31619975f83SVlastimil Babka #else 31719975f83SVlastimil Babka static inline void sysfs_slab_unlink(struct kmem_cache *s) { } 31819975f83SVlastimil Babka static inline void sysfs_slab_release(struct kmem_cache *s) { } 31919975f83SVlastimil Babka #endif 32019975f83SVlastimil Babka 32119975f83SVlastimil Babka void *fixup_red_left(struct kmem_cache *s, void *p); 32219975f83SVlastimil Babka 32319975f83SVlastimil Babka static inline void *nearest_obj(struct kmem_cache *cache, 32419975f83SVlastimil Babka const struct slab *slab, void *x) 32519975f83SVlastimil Babka { 32619975f83SVlastimil Babka void *object = x - (x - slab_address(slab)) % cache->size; 32719975f83SVlastimil Babka void *last_object = slab_address(slab) + 32819975f83SVlastimil Babka (slab->objects - 1) * cache->size; 32919975f83SVlastimil Babka void *result = (unlikely(object > last_object)) ? last_object : object; 33019975f83SVlastimil Babka 33119975f83SVlastimil Babka result = fixup_red_left(cache, result); 33219975f83SVlastimil Babka return result; 33319975f83SVlastimil Babka } 33419975f83SVlastimil Babka 33519975f83SVlastimil Babka /* Determine object index from a given position */ 33619975f83SVlastimil Babka static inline unsigned int __obj_to_index(const struct kmem_cache *cache, 33719975f83SVlastimil Babka void *addr, void *obj) 33819975f83SVlastimil Babka { 33919975f83SVlastimil Babka return reciprocal_divide(kasan_reset_tag(obj) - addr, 34019975f83SVlastimil Babka cache->reciprocal_size); 34119975f83SVlastimil Babka } 34219975f83SVlastimil Babka 34319975f83SVlastimil Babka static inline unsigned int obj_to_index(const struct kmem_cache *cache, 34419975f83SVlastimil Babka const struct slab *slab, void *obj) 34519975f83SVlastimil Babka { 34619975f83SVlastimil Babka if (is_kfence_address(obj)) 34719975f83SVlastimil Babka return 0; 34819975f83SVlastimil Babka return __obj_to_index(cache, slab_address(slab), obj); 34919975f83SVlastimil Babka } 35019975f83SVlastimil Babka 35119975f83SVlastimil Babka static inline int objs_per_slab(const struct kmem_cache *cache, 35219975f83SVlastimil Babka const struct slab *slab) 35319975f83SVlastimil Babka { 35419975f83SVlastimil Babka return slab->objects; 35519975f83SVlastimil Babka } 35607f361b2SJoonsoo Kim 35797d06609SChristoph Lameter /* 35897d06609SChristoph Lameter * State of the slab allocator. 35997d06609SChristoph Lameter * 36097d06609SChristoph Lameter * This is used to describe the states of the allocator during bootup. 36197d06609SChristoph Lameter * Allocators use this to gradually bootstrap themselves. Most allocators 36297d06609SChristoph Lameter * have the problem that the structures used for managing slab caches are 36397d06609SChristoph Lameter * allocated from slab caches themselves. 36497d06609SChristoph Lameter */ 36597d06609SChristoph Lameter enum slab_state { 36697d06609SChristoph Lameter DOWN, /* No slab functionality yet */ 36797d06609SChristoph Lameter PARTIAL, /* SLUB: kmem_cache_node available */ 368ce8eb6c4SChristoph Lameter PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */ 36997d06609SChristoph Lameter UP, /* Slab caches usable but not all extras yet */ 37097d06609SChristoph Lameter FULL /* Everything is working */ 37197d06609SChristoph Lameter }; 37297d06609SChristoph Lameter 37397d06609SChristoph Lameter extern enum slab_state slab_state; 37497d06609SChristoph Lameter 37518004c5dSChristoph Lameter /* The slab cache mutex protects the management structures during changes */ 37618004c5dSChristoph Lameter extern struct mutex slab_mutex; 3779b030cb8SChristoph Lameter 3789b030cb8SChristoph Lameter /* The list of all slab caches on the system */ 37918004c5dSChristoph Lameter extern struct list_head slab_caches; 38018004c5dSChristoph Lameter 3819b030cb8SChristoph Lameter /* The slab cache that manages slab cache information */ 3829b030cb8SChristoph Lameter extern struct kmem_cache *kmem_cache; 3839b030cb8SChristoph Lameter 384af3b5f87SVlastimil Babka /* A table of kmalloc cache names and sizes */ 385af3b5f87SVlastimil Babka extern const struct kmalloc_info_struct { 386cb5d9fb3SPengfei Li const char *name[NR_KMALLOC_TYPES]; 38755de8b9cSAlexey Dobriyan unsigned int size; 388af3b5f87SVlastimil Babka } kmalloc_info[]; 389af3b5f87SVlastimil Babka 390f97d5f63SChristoph Lameter /* Kmalloc array related functions */ 39134cc6990SDaniel Sanders void setup_kmalloc_cache_index_table(void); 392d50112edSAlexey Dobriyan void create_kmalloc_caches(slab_flags_t); 3932c59dd65SChristoph Lameter 3942c59dd65SChristoph Lameter /* Find the kmalloc slab corresponding for a certain size */ 3953c615294SGONG, Ruiqi struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags, unsigned long caller); 396ed4cd17eSHyeonggon Yoo 397ed4cd17eSHyeonggon Yoo void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, 398ed4cd17eSHyeonggon Yoo int node, size_t orig_size, 399ed4cd17eSHyeonggon Yoo unsigned long caller); 400ed4cd17eSHyeonggon Yoo void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller); 401f97d5f63SChristoph Lameter 40244405099SLong Li gfp_t kmalloc_fix_flags(gfp_t flags); 403f97d5f63SChristoph Lameter 4049b030cb8SChristoph Lameter /* Functions provided by the slab allocators */ 405d50112edSAlexey Dobriyan int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags); 40697d06609SChristoph Lameter 407*89c2d061SVlastimil Babka void __init kmem_cache_init(void); 4080c474d31SCatalin Marinas void __init new_kmalloc_cache(int idx, enum kmalloc_cache_type type, 4090c474d31SCatalin Marinas slab_flags_t flags); 41045530c44SChristoph Lameter extern void create_boot_cache(struct kmem_cache *, const char *name, 411361d575eSAlexey Dobriyan unsigned int size, slab_flags_t flags, 412361d575eSAlexey Dobriyan unsigned int useroffset, unsigned int usersize); 41345530c44SChristoph Lameter 414423c929cSJoonsoo Kim int slab_unmergeable(struct kmem_cache *s); 415f4957d5bSAlexey Dobriyan struct kmem_cache *find_mergeable(unsigned size, unsigned align, 416d50112edSAlexey Dobriyan slab_flags_t flags, const char *name, void (*ctor)(void *)); 4172633d7a0SGlauber Costa struct kmem_cache * 418f4957d5bSAlexey Dobriyan __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, 419d50112edSAlexey Dobriyan slab_flags_t flags, void (*ctor)(void *)); 420423c929cSJoonsoo Kim 4210293d1fdSAlexey Dobriyan slab_flags_t kmem_cache_flags(unsigned int object_size, 42237540008SNikolay Borisov slab_flags_t flags, const char *name); 423cbb79694SChristoph Lameter 424bb944290SFeng Tang static inline bool is_kmalloc_cache(struct kmem_cache *s) 425bb944290SFeng Tang { 426bb944290SFeng Tang return (s->flags & SLAB_KMALLOC); 427bb944290SFeng Tang } 428cbb79694SChristoph Lameter 429d8843922SGlauber Costa /* Legal flag mask for kmem_cache_create(), for various configurations */ 4306d6ea1e9SNicolas Boichat #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ 4316d6ea1e9SNicolas Boichat SLAB_CACHE_DMA32 | SLAB_PANIC | \ 4325f0d5a3aSPaul E. McKenney SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS ) 433d8843922SGlauber Costa 434a9e0b9f2SVlastimil Babka #ifdef CONFIG_SLUB_DEBUG 435d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 436becfda68SLaura Abbott SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) 437d8843922SGlauber Costa #else 438d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (0) 439d8843922SGlauber Costa #endif 440d8843922SGlauber Costa 441d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 4426cd6d33cSFeng Tang SLAB_TEMPORARY | SLAB_ACCOUNT | \ 443d0bf7d57SJesper Dangaard Brouer SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE) 444d8843922SGlauber Costa 445e70954fdSThomas Garnier /* Common flags available with current configuration */ 446d8843922SGlauber Costa #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS) 447d8843922SGlauber Costa 448e70954fdSThomas Garnier /* Common flags permitted for kmem_cache_create */ 449e70954fdSThomas Garnier #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \ 450e70954fdSThomas Garnier SLAB_RED_ZONE | \ 451e70954fdSThomas Garnier SLAB_POISON | \ 452e70954fdSThomas Garnier SLAB_STORE_USER | \ 453e70954fdSThomas Garnier SLAB_TRACE | \ 454e70954fdSThomas Garnier SLAB_CONSISTENCY_CHECKS | \ 455e70954fdSThomas Garnier SLAB_MEM_SPREAD | \ 456e70954fdSThomas Garnier SLAB_NOLEAKTRACE | \ 457e70954fdSThomas Garnier SLAB_RECLAIM_ACCOUNT | \ 458e70954fdSThomas Garnier SLAB_TEMPORARY | \ 459a285909fSHyeonggon Yoo SLAB_ACCOUNT | \ 4606cd6d33cSFeng Tang SLAB_KMALLOC | \ 461d0bf7d57SJesper Dangaard Brouer SLAB_NO_MERGE | \ 462a285909fSHyeonggon Yoo SLAB_NO_USER_FLAGS) 463e70954fdSThomas Garnier 464f9e13c0aSShakeel Butt bool __kmem_cache_empty(struct kmem_cache *); 465945cf2b6SChristoph Lameter int __kmem_cache_shutdown(struct kmem_cache *); 46652b4b950SDmitry Safonov void __kmem_cache_release(struct kmem_cache *); 467c9fc5864STejun Heo int __kmem_cache_shrink(struct kmem_cache *); 46841a21285SChristoph Lameter void slab_kmem_cache_release(struct kmem_cache *); 469945cf2b6SChristoph Lameter 470b7454ad3SGlauber Costa struct seq_file; 471b7454ad3SGlauber Costa struct file; 472b7454ad3SGlauber Costa 4730d7561c6SGlauber Costa struct slabinfo { 4740d7561c6SGlauber Costa unsigned long active_objs; 4750d7561c6SGlauber Costa unsigned long num_objs; 4760d7561c6SGlauber Costa unsigned long active_slabs; 4770d7561c6SGlauber Costa unsigned long num_slabs; 4780d7561c6SGlauber Costa unsigned long shared_avail; 4790d7561c6SGlauber Costa unsigned int limit; 4800d7561c6SGlauber Costa unsigned int batchcount; 4810d7561c6SGlauber Costa unsigned int shared; 4820d7561c6SGlauber Costa unsigned int objects_per_slab; 4830d7561c6SGlauber Costa unsigned int cache_order; 4840d7561c6SGlauber Costa }; 4850d7561c6SGlauber Costa 4860d7561c6SGlauber Costa void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); 4870d7561c6SGlauber Costa void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); 488b7454ad3SGlauber Costa ssize_t slabinfo_write(struct file *file, const char __user *buffer, 489b7454ad3SGlauber Costa size_t count, loff_t *ppos); 490ba6c496eSGlauber Costa 4911a984c4eSMuchun Song static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) 4926cea1d56SRoman Gushchin { 4936cea1d56SRoman Gushchin return (s->flags & SLAB_RECLAIM_ACCOUNT) ? 494d42f3245SRoman Gushchin NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; 4956cea1d56SRoman Gushchin } 4966cea1d56SRoman Gushchin 497e42f174eSVlastimil Babka #ifdef CONFIG_SLUB_DEBUG 498e42f174eSVlastimil Babka #ifdef CONFIG_SLUB_DEBUG_ON 499e42f174eSVlastimil Babka DECLARE_STATIC_KEY_TRUE(slub_debug_enabled); 500e42f174eSVlastimil Babka #else 501e42f174eSVlastimil Babka DECLARE_STATIC_KEY_FALSE(slub_debug_enabled); 502e42f174eSVlastimil Babka #endif 503e42f174eSVlastimil Babka extern void print_tracking(struct kmem_cache *s, void *object); 5041f9f78b1SOliver Glitta long validate_slab_cache(struct kmem_cache *s); 5050d4a062aSMarco Elver static inline bool __slub_debug_enabled(void) 5060d4a062aSMarco Elver { 5070d4a062aSMarco Elver return static_branch_unlikely(&slub_debug_enabled); 5080d4a062aSMarco Elver } 509e42f174eSVlastimil Babka #else 510e42f174eSVlastimil Babka static inline void print_tracking(struct kmem_cache *s, void *object) 511e42f174eSVlastimil Babka { 512e42f174eSVlastimil Babka } 5130d4a062aSMarco Elver static inline bool __slub_debug_enabled(void) 5140d4a062aSMarco Elver { 5150d4a062aSMarco Elver return false; 5160d4a062aSMarco Elver } 517e42f174eSVlastimil Babka #endif 518e42f174eSVlastimil Babka 519e42f174eSVlastimil Babka /* 520e42f174eSVlastimil Babka * Returns true if any of the specified slub_debug flags is enabled for the 521e42f174eSVlastimil Babka * cache. Use only for flags parsed by setup_slub_debug() as it also enables 522e42f174eSVlastimil Babka * the static key. 523e42f174eSVlastimil Babka */ 524e42f174eSVlastimil Babka static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags) 525e42f174eSVlastimil Babka { 5260d4a062aSMarco Elver if (IS_ENABLED(CONFIG_SLUB_DEBUG)) 527e42f174eSVlastimil Babka VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS)); 5280d4a062aSMarco Elver if (__slub_debug_enabled()) 529e42f174eSVlastimil Babka return s->flags & flags; 530e42f174eSVlastimil Babka return false; 531e42f174eSVlastimil Babka } 532e42f174eSVlastimil Babka 53384c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM 5344b5f8d9aSVlastimil Babka /* 5354b5f8d9aSVlastimil Babka * slab_objcgs - get the object cgroups vector associated with a slab 5364b5f8d9aSVlastimil Babka * @slab: a pointer to the slab struct 5374b5f8d9aSVlastimil Babka * 5384b5f8d9aSVlastimil Babka * Returns a pointer to the object cgroups vector associated with the slab, 5394b5f8d9aSVlastimil Babka * or NULL if no such vector has been associated yet. 5404b5f8d9aSVlastimil Babka */ 5414b5f8d9aSVlastimil Babka static inline struct obj_cgroup **slab_objcgs(struct slab *slab) 5424b5f8d9aSVlastimil Babka { 5434b5f8d9aSVlastimil Babka unsigned long memcg_data = READ_ONCE(slab->memcg_data); 5444b5f8d9aSVlastimil Babka 5454b5f8d9aSVlastimil Babka VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), 5464b5f8d9aSVlastimil Babka slab_page(slab)); 5474b5f8d9aSVlastimil Babka VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab)); 5484b5f8d9aSVlastimil Babka 5494b5f8d9aSVlastimil Babka return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); 5504b5f8d9aSVlastimil Babka } 5514b5f8d9aSVlastimil Babka 5524b5f8d9aSVlastimil Babka int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, 5534b5f8d9aSVlastimil Babka gfp_t gfp, bool new_slab); 554fdbcb2a6SWaiman Long void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, 555fdbcb2a6SWaiman Long enum node_stat_item idx, int nr); 556286e04b8SRoman Gushchin 5574b5f8d9aSVlastimil Babka static inline void memcg_free_slab_cgroups(struct slab *slab) 558286e04b8SRoman Gushchin { 5594b5f8d9aSVlastimil Babka kfree(slab_objcgs(slab)); 5604b5f8d9aSVlastimil Babka slab->memcg_data = 0; 561286e04b8SRoman Gushchin } 562286e04b8SRoman Gushchin 563f2fe7b09SRoman Gushchin static inline size_t obj_full_size(struct kmem_cache *s) 564f2fe7b09SRoman Gushchin { 565f2fe7b09SRoman Gushchin /* 566f2fe7b09SRoman Gushchin * For each accounted object there is an extra space which is used 567f2fe7b09SRoman Gushchin * to store obj_cgroup membership. Charge it too. 568f2fe7b09SRoman Gushchin */ 569f2fe7b09SRoman Gushchin return s->size + sizeof(struct obj_cgroup *); 570f2fe7b09SRoman Gushchin } 571f2fe7b09SRoman Gushchin 572becaba65SRoman Gushchin /* 573becaba65SRoman Gushchin * Returns false if the allocation should fail. 574becaba65SRoman Gushchin */ 575becaba65SRoman Gushchin static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, 57688f2ef73SMuchun Song struct list_lru *lru, 577becaba65SRoman Gushchin struct obj_cgroup **objcgp, 578becaba65SRoman Gushchin size_t objects, gfp_t flags) 579f2fe7b09SRoman Gushchin { 5809855609bSRoman Gushchin struct obj_cgroup *objcg; 581f2fe7b09SRoman Gushchin 582f7a449f7SRoman Gushchin if (!memcg_kmem_online()) 583becaba65SRoman Gushchin return true; 584becaba65SRoman Gushchin 585becaba65SRoman Gushchin if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)) 586becaba65SRoman Gushchin return true; 587becaba65SRoman Gushchin 588e86828e5SRoman Gushchin /* 589e86828e5SRoman Gushchin * The obtained objcg pointer is safe to use within the current scope, 590e86828e5SRoman Gushchin * defined by current task or set_active_memcg() pair. 591e86828e5SRoman Gushchin * obj_cgroup_get() is used to get a permanent reference. 592e86828e5SRoman Gushchin */ 593e86828e5SRoman Gushchin objcg = current_obj_cgroup(); 5949855609bSRoman Gushchin if (!objcg) 595becaba65SRoman Gushchin return true; 5969855609bSRoman Gushchin 59788f2ef73SMuchun Song if (lru) { 59888f2ef73SMuchun Song int ret; 59988f2ef73SMuchun Song struct mem_cgroup *memcg; 60088f2ef73SMuchun Song 60188f2ef73SMuchun Song memcg = get_mem_cgroup_from_objcg(objcg); 60288f2ef73SMuchun Song ret = memcg_list_lru_alloc(memcg, lru, flags); 60388f2ef73SMuchun Song css_put(&memcg->css); 60488f2ef73SMuchun Song 60588f2ef73SMuchun Song if (ret) 606e86828e5SRoman Gushchin return false; 607f2fe7b09SRoman Gushchin } 608f2fe7b09SRoman Gushchin 60988f2ef73SMuchun Song if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) 610e86828e5SRoman Gushchin return false; 61188f2ef73SMuchun Song 612becaba65SRoman Gushchin *objcgp = objcg; 613becaba65SRoman Gushchin return true; 614f2fe7b09SRoman Gushchin } 615f2fe7b09SRoman Gushchin 616964d4bd3SRoman Gushchin static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, 617964d4bd3SRoman Gushchin struct obj_cgroup *objcg, 61810befea9SRoman Gushchin gfp_t flags, size_t size, 61910befea9SRoman Gushchin void **p) 620964d4bd3SRoman Gushchin { 6214b5f8d9aSVlastimil Babka struct slab *slab; 622964d4bd3SRoman Gushchin unsigned long off; 623964d4bd3SRoman Gushchin size_t i; 624964d4bd3SRoman Gushchin 625f7a449f7SRoman Gushchin if (!memcg_kmem_online() || !objcg) 62610befea9SRoman Gushchin return; 62710befea9SRoman Gushchin 628964d4bd3SRoman Gushchin for (i = 0; i < size; i++) { 629964d4bd3SRoman Gushchin if (likely(p[i])) { 6304b5f8d9aSVlastimil Babka slab = virt_to_slab(p[i]); 63110befea9SRoman Gushchin 6324b5f8d9aSVlastimil Babka if (!slab_objcgs(slab) && 6334b5f8d9aSVlastimil Babka memcg_alloc_slab_cgroups(slab, s, flags, 6342e9bd483SRoman Gushchin false)) { 63510befea9SRoman Gushchin obj_cgroup_uncharge(objcg, obj_full_size(s)); 63610befea9SRoman Gushchin continue; 63710befea9SRoman Gushchin } 63810befea9SRoman Gushchin 6394b5f8d9aSVlastimil Babka off = obj_to_index(s, slab, p[i]); 640964d4bd3SRoman Gushchin obj_cgroup_get(objcg); 6414b5f8d9aSVlastimil Babka slab_objcgs(slab)[off] = objcg; 6424b5f8d9aSVlastimil Babka mod_objcg_state(objcg, slab_pgdat(slab), 643f2fe7b09SRoman Gushchin cache_vmstat_idx(s), obj_full_size(s)); 644f2fe7b09SRoman Gushchin } else { 645f2fe7b09SRoman Gushchin obj_cgroup_uncharge(objcg, obj_full_size(s)); 646964d4bd3SRoman Gushchin } 647964d4bd3SRoman Gushchin } 648964d4bd3SRoman Gushchin } 649964d4bd3SRoman Gushchin 650b77d5b1bSMuchun Song static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 651d1b2cf6cSBharata B Rao void **p, int objects) 652964d4bd3SRoman Gushchin { 653270c6a71SRoman Gushchin struct obj_cgroup **objcgs; 654d1b2cf6cSBharata B Rao int i; 655964d4bd3SRoman Gushchin 656f7a449f7SRoman Gushchin if (!memcg_kmem_online()) 65710befea9SRoman Gushchin return; 65810befea9SRoman Gushchin 6594b5f8d9aSVlastimil Babka objcgs = slab_objcgs(slab); 660270c6a71SRoman Gushchin if (!objcgs) 661b77d5b1bSMuchun Song return; 662964d4bd3SRoman Gushchin 663b77d5b1bSMuchun Song for (i = 0; i < objects; i++) { 664b77d5b1bSMuchun Song struct obj_cgroup *objcg; 665b77d5b1bSMuchun Song unsigned int off; 666d1b2cf6cSBharata B Rao 6674b5f8d9aSVlastimil Babka off = obj_to_index(s, slab, p[i]); 668270c6a71SRoman Gushchin objcg = objcgs[off]; 66910befea9SRoman Gushchin if (!objcg) 670d1b2cf6cSBharata B Rao continue; 67110befea9SRoman Gushchin 672270c6a71SRoman Gushchin objcgs[off] = NULL; 673f2fe7b09SRoman Gushchin obj_cgroup_uncharge(objcg, obj_full_size(s)); 6744b5f8d9aSVlastimil Babka mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), 675f2fe7b09SRoman Gushchin -obj_full_size(s)); 676964d4bd3SRoman Gushchin obj_cgroup_put(objcg); 677964d4bd3SRoman Gushchin } 678d1b2cf6cSBharata B Rao } 679964d4bd3SRoman Gushchin 68084c07d11SKirill Tkhai #else /* CONFIG_MEMCG_KMEM */ 6814b5f8d9aSVlastimil Babka static inline struct obj_cgroup **slab_objcgs(struct slab *slab) 6824b5f8d9aSVlastimil Babka { 6834b5f8d9aSVlastimil Babka return NULL; 6844b5f8d9aSVlastimil Babka } 6854b5f8d9aSVlastimil Babka 6869855609bSRoman Gushchin static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) 6874d96ba35SRoman Gushchin { 6884d96ba35SRoman Gushchin return NULL; 6894d96ba35SRoman Gushchin } 6904d96ba35SRoman Gushchin 6914b5f8d9aSVlastimil Babka static inline int memcg_alloc_slab_cgroups(struct slab *slab, 6922e9bd483SRoman Gushchin struct kmem_cache *s, gfp_t gfp, 6934b5f8d9aSVlastimil Babka bool new_slab) 694286e04b8SRoman Gushchin { 695286e04b8SRoman Gushchin return 0; 696286e04b8SRoman Gushchin } 697286e04b8SRoman Gushchin 6984b5f8d9aSVlastimil Babka static inline void memcg_free_slab_cgroups(struct slab *slab) 699286e04b8SRoman Gushchin { 700286e04b8SRoman Gushchin } 701286e04b8SRoman Gushchin 702becaba65SRoman Gushchin static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, 70388f2ef73SMuchun Song struct list_lru *lru, 704becaba65SRoman Gushchin struct obj_cgroup **objcgp, 705becaba65SRoman Gushchin size_t objects, gfp_t flags) 706f2fe7b09SRoman Gushchin { 707becaba65SRoman Gushchin return true; 708f2fe7b09SRoman Gushchin } 709f2fe7b09SRoman Gushchin 710964d4bd3SRoman Gushchin static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, 711964d4bd3SRoman Gushchin struct obj_cgroup *objcg, 71210befea9SRoman Gushchin gfp_t flags, size_t size, 71310befea9SRoman Gushchin void **p) 714964d4bd3SRoman Gushchin { 715964d4bd3SRoman Gushchin } 716964d4bd3SRoman Gushchin 717b77d5b1bSMuchun Song static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 718d1b2cf6cSBharata B Rao void **p, int objects) 719964d4bd3SRoman Gushchin { 720964d4bd3SRoman Gushchin } 72184c07d11SKirill Tkhai #endif /* CONFIG_MEMCG_KMEM */ 722b9ce5ef4SGlauber Costa 723a64b5378SKees Cook static inline struct kmem_cache *virt_to_cache(const void *obj) 724a64b5378SKees Cook { 72582c1775dSMatthew Wilcox (Oracle) struct slab *slab; 726a64b5378SKees Cook 72782c1775dSMatthew Wilcox (Oracle) slab = virt_to_slab(obj); 72882c1775dSMatthew Wilcox (Oracle) if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", 729a64b5378SKees Cook __func__)) 730a64b5378SKees Cook return NULL; 73182c1775dSMatthew Wilcox (Oracle) return slab->slab_cache; 732a64b5378SKees Cook } 733a64b5378SKees Cook 734b918653bSMatthew Wilcox (Oracle) static __always_inline void account_slab(struct slab *slab, int order, 735b918653bSMatthew Wilcox (Oracle) struct kmem_cache *s, gfp_t gfp) 7366cea1d56SRoman Gushchin { 737f7a449f7SRoman Gushchin if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) 7384b5f8d9aSVlastimil Babka memcg_alloc_slab_cgroups(slab, s, gfp, true); 7392e9bd483SRoman Gushchin 740b918653bSMatthew Wilcox (Oracle) mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 741f2fe7b09SRoman Gushchin PAGE_SIZE << order); 7426cea1d56SRoman Gushchin } 7436cea1d56SRoman Gushchin 744b918653bSMatthew Wilcox (Oracle) static __always_inline void unaccount_slab(struct slab *slab, int order, 7456cea1d56SRoman Gushchin struct kmem_cache *s) 7466cea1d56SRoman Gushchin { 747f7a449f7SRoman Gushchin if (memcg_kmem_online()) 7484b5f8d9aSVlastimil Babka memcg_free_slab_cgroups(slab); 7499855609bSRoman Gushchin 750b918653bSMatthew Wilcox (Oracle) mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 751d42f3245SRoman Gushchin -(PAGE_SIZE << order)); 7526cea1d56SRoman Gushchin } 7536cea1d56SRoman Gushchin 754e42f174eSVlastimil Babka static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 755e42f174eSVlastimil Babka { 756e42f174eSVlastimil Babka struct kmem_cache *cachep; 757e42f174eSVlastimil Babka 758e42f174eSVlastimil Babka if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && 759e42f174eSVlastimil Babka !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) 760e42f174eSVlastimil Babka return s; 761e42f174eSVlastimil Babka 762e42f174eSVlastimil Babka cachep = virt_to_cache(x); 76310befea9SRoman Gushchin if (WARN(cachep && cachep != s, 764e42f174eSVlastimil Babka "%s: Wrong slab cache. %s but object is from %s\n", 765e42f174eSVlastimil Babka __func__, s->name, cachep->name)) 766e42f174eSVlastimil Babka print_tracking(cachep, x); 767e42f174eSVlastimil Babka return cachep; 768e42f174eSVlastimil Babka } 769d6a71648SHyeonggon Yoo 770d6a71648SHyeonggon Yoo void free_large_kmalloc(struct folio *folio, void *object); 771d6a71648SHyeonggon Yoo 7728dfa9d55SHyeonggon Yoo size_t __ksize(const void *objp); 7738dfa9d55SHyeonggon Yoo 77411c7aec2SJesper Dangaard Brouer static inline size_t slab_ksize(const struct kmem_cache *s) 77511c7aec2SJesper Dangaard Brouer { 77611c7aec2SJesper Dangaard Brouer #ifdef CONFIG_SLUB_DEBUG 77711c7aec2SJesper Dangaard Brouer /* 77811c7aec2SJesper Dangaard Brouer * Debugging requires use of the padding between object 77911c7aec2SJesper Dangaard Brouer * and whatever may come after it. 78011c7aec2SJesper Dangaard Brouer */ 78111c7aec2SJesper Dangaard Brouer if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) 78211c7aec2SJesper Dangaard Brouer return s->object_size; 78311c7aec2SJesper Dangaard Brouer #endif 78480a9201aSAlexander Potapenko if (s->flags & SLAB_KASAN) 78580a9201aSAlexander Potapenko return s->object_size; 78611c7aec2SJesper Dangaard Brouer /* 78711c7aec2SJesper Dangaard Brouer * If we have the need to store the freelist pointer 78811c7aec2SJesper Dangaard Brouer * back there or track user information then we can 78911c7aec2SJesper Dangaard Brouer * only use the space before that information. 79011c7aec2SJesper Dangaard Brouer */ 7915f0d5a3aSPaul E. McKenney if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER)) 79211c7aec2SJesper Dangaard Brouer return s->inuse; 79311c7aec2SJesper Dangaard Brouer /* 79411c7aec2SJesper Dangaard Brouer * Else we can use all the padding etc for the allocation 79511c7aec2SJesper Dangaard Brouer */ 79611c7aec2SJesper Dangaard Brouer return s->size; 79711c7aec2SJesper Dangaard Brouer } 79811c7aec2SJesper Dangaard Brouer 79911c7aec2SJesper Dangaard Brouer static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, 80088f2ef73SMuchun Song struct list_lru *lru, 801964d4bd3SRoman Gushchin struct obj_cgroup **objcgp, 802964d4bd3SRoman Gushchin size_t size, gfp_t flags) 80311c7aec2SJesper Dangaard Brouer { 80411c7aec2SJesper Dangaard Brouer flags &= gfp_allowed_mask; 805d92a8cfcSPeter Zijlstra 80695d6c701SDaniel Vetter might_alloc(flags); 80711c7aec2SJesper Dangaard Brouer 808fab9963aSJesper Dangaard Brouer if (should_failslab(s, flags)) 80911c7aec2SJesper Dangaard Brouer return NULL; 81011c7aec2SJesper Dangaard Brouer 81188f2ef73SMuchun Song if (!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags)) 812becaba65SRoman Gushchin return NULL; 81345264778SVladimir Davydov 81445264778SVladimir Davydov return s; 81511c7aec2SJesper Dangaard Brouer } 81611c7aec2SJesper Dangaard Brouer 817964d4bd3SRoman Gushchin static inline void slab_post_alloc_hook(struct kmem_cache *s, 818da844b78SAndrey Konovalov struct obj_cgroup *objcg, gfp_t flags, 8199ce67395SFeng Tang size_t size, void **p, bool init, 8209ce67395SFeng Tang unsigned int orig_size) 82111c7aec2SJesper Dangaard Brouer { 8229ce67395SFeng Tang unsigned int zero_size = s->object_size; 823fdb54d96SAndrey Konovalov bool kasan_init = init; 82411c7aec2SJesper Dangaard Brouer size_t i; 82511c7aec2SJesper Dangaard Brouer 82611c7aec2SJesper Dangaard Brouer flags &= gfp_allowed_mask; 827da844b78SAndrey Konovalov 828da844b78SAndrey Konovalov /* 8299ce67395SFeng Tang * For kmalloc object, the allocated memory size(object_size) is likely 8309ce67395SFeng Tang * larger than the requested size(orig_size). If redzone check is 8319ce67395SFeng Tang * enabled for the extra space, don't zero it, as it will be redzoned 8329ce67395SFeng Tang * soon. The redzone operation for this extra space could be seen as a 8339ce67395SFeng Tang * replacement of current poisoning under certain debug option, and 8349ce67395SFeng Tang * won't break other sanity checks. 8359ce67395SFeng Tang */ 8369ce67395SFeng Tang if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) && 8379ce67395SFeng Tang (s->flags & SLAB_KMALLOC)) 8389ce67395SFeng Tang zero_size = orig_size; 8399ce67395SFeng Tang 8409ce67395SFeng Tang /* 841fdb54d96SAndrey Konovalov * When slub_debug is enabled, avoid memory initialization integrated 842fdb54d96SAndrey Konovalov * into KASAN and instead zero out the memory via the memset below with 843fdb54d96SAndrey Konovalov * the proper size. Otherwise, KASAN might overwrite SLUB redzones and 844fdb54d96SAndrey Konovalov * cause false-positive reports. This does not lead to a performance 845fdb54d96SAndrey Konovalov * penalty on production builds, as slub_debug is not intended to be 846fdb54d96SAndrey Konovalov * enabled there. 847fdb54d96SAndrey Konovalov */ 848fdb54d96SAndrey Konovalov if (__slub_debug_enabled()) 849fdb54d96SAndrey Konovalov kasan_init = false; 850fdb54d96SAndrey Konovalov 851fdb54d96SAndrey Konovalov /* 852da844b78SAndrey Konovalov * As memory initialization might be integrated into KASAN, 853da844b78SAndrey Konovalov * kasan_slab_alloc and initialization memset must be 854da844b78SAndrey Konovalov * kept together to avoid discrepancies in behavior. 855da844b78SAndrey Konovalov * 856da844b78SAndrey Konovalov * As p[i] might get tagged, memset and kmemleak hook come after KASAN. 857da844b78SAndrey Konovalov */ 85811c7aec2SJesper Dangaard Brouer for (i = 0; i < size; i++) { 859fdb54d96SAndrey Konovalov p[i] = kasan_slab_alloc(s, p[i], flags, kasan_init); 860fdb54d96SAndrey Konovalov if (p[i] && init && (!kasan_init || !kasan_has_integrated_init())) 8619ce67395SFeng Tang memset(p[i], 0, zero_size); 86253128245SAndrey Konovalov kmemleak_alloc_recursive(p[i], s->object_size, 1, 86311c7aec2SJesper Dangaard Brouer s->flags, flags); 86468ef169aSAlexander Potapenko kmsan_slab_alloc(s, p[i], flags); 86511c7aec2SJesper Dangaard Brouer } 86645264778SVladimir Davydov 86710befea9SRoman Gushchin memcg_slab_post_alloc_hook(s, objcg, flags, size, p); 86811c7aec2SJesper Dangaard Brouer } 86911c7aec2SJesper Dangaard Brouer 870ca34956bSChristoph Lameter /* 871ca34956bSChristoph Lameter * The slab lists for all objects. 872ca34956bSChristoph Lameter */ 873ca34956bSChristoph Lameter struct kmem_cache_node { 874b539ce9fSJiri Kosina spinlock_t list_lock; 875ca34956bSChristoph Lameter unsigned long nr_partial; 876ca34956bSChristoph Lameter struct list_head partial; 877ca34956bSChristoph Lameter #ifdef CONFIG_SLUB_DEBUG 878ca34956bSChristoph Lameter atomic_long_t nr_slabs; 879ca34956bSChristoph Lameter atomic_long_t total_objects; 880ca34956bSChristoph Lameter struct list_head full; 881ca34956bSChristoph Lameter #endif 882ca34956bSChristoph Lameter }; 883e25839f6SWanpeng Li 88444c5356fSChristoph Lameter static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 88544c5356fSChristoph Lameter { 88644c5356fSChristoph Lameter return s->node[node]; 88744c5356fSChristoph Lameter } 88844c5356fSChristoph Lameter 88944c5356fSChristoph Lameter /* 89044c5356fSChristoph Lameter * Iterator over all nodes. The body will be executed for each node that has 89144c5356fSChristoph Lameter * a kmem_cache_node structure allocated (which is true for all online nodes) 89244c5356fSChristoph Lameter */ 89344c5356fSChristoph Lameter #define for_each_kmem_cache_node(__s, __node, __n) \ 8949163582cSMikulas Patocka for (__node = 0; __node < nr_node_ids; __node++) \ 8959163582cSMikulas Patocka if ((__n = get_node(__s, __node))) 89644c5356fSChristoph Lameter 89744c5356fSChristoph Lameter 898a9e0b9f2SVlastimil Babka #ifdef CONFIG_SLUB_DEBUG 899852d8be0SYang Shi void dump_unreclaimable_slab(void); 900852d8be0SYang Shi #else 901852d8be0SYang Shi static inline void dump_unreclaimable_slab(void) 902852d8be0SYang Shi { 903852d8be0SYang Shi } 904852d8be0SYang Shi #endif 905852d8be0SYang Shi 90655834c59SAlexander Potapenko void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr); 90755834c59SAlexander Potapenko 9087c00fce9SThomas Garnier #ifdef CONFIG_SLAB_FREELIST_RANDOM 9097c00fce9SThomas Garnier int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 9107c00fce9SThomas Garnier gfp_t gfp); 9117c00fce9SThomas Garnier void cache_random_seq_destroy(struct kmem_cache *cachep); 9127c00fce9SThomas Garnier #else 9137c00fce9SThomas Garnier static inline int cache_random_seq_create(struct kmem_cache *cachep, 9147c00fce9SThomas Garnier unsigned int count, gfp_t gfp) 9157c00fce9SThomas Garnier { 9167c00fce9SThomas Garnier return 0; 9177c00fce9SThomas Garnier } 9187c00fce9SThomas Garnier static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } 9197c00fce9SThomas Garnier #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 9207c00fce9SThomas Garnier 9216471384aSAlexander Potapenko static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c) 9226471384aSAlexander Potapenko { 92351cba1ebSKees Cook if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 92451cba1ebSKees Cook &init_on_alloc)) { 9256471384aSAlexander Potapenko if (c->ctor) 9266471384aSAlexander Potapenko return false; 9276471384aSAlexander Potapenko if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) 9286471384aSAlexander Potapenko return flags & __GFP_ZERO; 9296471384aSAlexander Potapenko return true; 9306471384aSAlexander Potapenko } 9316471384aSAlexander Potapenko return flags & __GFP_ZERO; 9326471384aSAlexander Potapenko } 9336471384aSAlexander Potapenko 9346471384aSAlexander Potapenko static inline bool slab_want_init_on_free(struct kmem_cache *c) 9356471384aSAlexander Potapenko { 93651cba1ebSKees Cook if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 93751cba1ebSKees Cook &init_on_free)) 9386471384aSAlexander Potapenko return !(c->ctor || 9396471384aSAlexander Potapenko (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 9406471384aSAlexander Potapenko return false; 9416471384aSAlexander Potapenko } 9426471384aSAlexander Potapenko 94364dd6849SFaiyaz Mohammed #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 94464dd6849SFaiyaz Mohammed void debugfs_slab_release(struct kmem_cache *); 94564dd6849SFaiyaz Mohammed #else 94664dd6849SFaiyaz Mohammed static inline void debugfs_slab_release(struct kmem_cache *s) { } 94764dd6849SFaiyaz Mohammed #endif 94864dd6849SFaiyaz Mohammed 9495bb1bb35SPaul E. McKenney #ifdef CONFIG_PRINTK 9508e7f37f2SPaul E. McKenney #define KS_ADDRS_COUNT 16 9518e7f37f2SPaul E. McKenney struct kmem_obj_info { 9528e7f37f2SPaul E. McKenney void *kp_ptr; 9537213230aSMatthew Wilcox (Oracle) struct slab *kp_slab; 9548e7f37f2SPaul E. McKenney void *kp_objp; 9558e7f37f2SPaul E. McKenney unsigned long kp_data_offset; 9568e7f37f2SPaul E. McKenney struct kmem_cache *kp_slab_cache; 9578e7f37f2SPaul E. McKenney void *kp_ret; 9588e7f37f2SPaul E. McKenney void *kp_stack[KS_ADDRS_COUNT]; 959e548eaa1SManinder Singh void *kp_free_stack[KS_ADDRS_COUNT]; 9608e7f37f2SPaul E. McKenney }; 9612dfe63e6SMarco Elver void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab); 9625bb1bb35SPaul E. McKenney #endif 9638e7f37f2SPaul E. McKenney 9640b3eb091SMatthew Wilcox (Oracle) void __check_heap_object(const void *ptr, unsigned long n, 9650b3eb091SMatthew Wilcox (Oracle) const struct slab *slab, bool to_user); 9660b3eb091SMatthew Wilcox (Oracle) 967946fa0dbSFeng Tang #ifdef CONFIG_SLUB_DEBUG 968946fa0dbSFeng Tang void skip_orig_size_check(struct kmem_cache *s, const void *object); 969946fa0dbSFeng Tang #endif 970946fa0dbSFeng Tang 9715240ab40SAndrey Ryabinin #endif /* MM_SLAB_H */ 972