1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 297d06609SChristoph Lameter #ifndef MM_SLAB_H 397d06609SChristoph Lameter #define MM_SLAB_H 497d06609SChristoph Lameter /* 597d06609SChristoph Lameter * Internal slab definitions 697d06609SChristoph Lameter */ 797d06609SChristoph Lameter 807f361b2SJoonsoo Kim #ifdef CONFIG_SLOB 907f361b2SJoonsoo Kim /* 1007f361b2SJoonsoo Kim * Common fields provided in kmem_cache by all slab allocators 1107f361b2SJoonsoo Kim * This struct is either used directly by the allocator (SLOB) 1207f361b2SJoonsoo Kim * or the allocator must include definitions for all fields 1307f361b2SJoonsoo Kim * provided in kmem_cache_common in their definition of kmem_cache. 1407f361b2SJoonsoo Kim * 1507f361b2SJoonsoo Kim * Once we can do anonymous structs (C11 standard) we could put a 1607f361b2SJoonsoo Kim * anonymous struct definition in these allocators so that the 1707f361b2SJoonsoo Kim * separate allocations in the kmem_cache structure of SLAB and 1807f361b2SJoonsoo Kim * SLUB is no longer needed. 1907f361b2SJoonsoo Kim */ 2007f361b2SJoonsoo Kim struct kmem_cache { 2107f361b2SJoonsoo Kim unsigned int object_size;/* The original size of the object */ 2207f361b2SJoonsoo Kim unsigned int size; /* The aligned/padded/added on size */ 2307f361b2SJoonsoo Kim unsigned int align; /* Alignment as calculated */ 24*d50112edSAlexey Dobriyan slab_flags_t flags; /* Active flags on the slab */ 2507f361b2SJoonsoo Kim const char *name; /* Slab name for sysfs */ 2607f361b2SJoonsoo Kim int refcount; /* Use counter */ 2707f361b2SJoonsoo Kim void (*ctor)(void *); /* Called on object slot creation */ 2807f361b2SJoonsoo Kim struct list_head list; /* List of all slab caches on the system */ 2907f361b2SJoonsoo Kim }; 3007f361b2SJoonsoo Kim 3107f361b2SJoonsoo Kim #endif /* CONFIG_SLOB */ 3207f361b2SJoonsoo Kim 3307f361b2SJoonsoo Kim #ifdef CONFIG_SLAB 3407f361b2SJoonsoo Kim #include <linux/slab_def.h> 3507f361b2SJoonsoo Kim #endif 3607f361b2SJoonsoo Kim 3707f361b2SJoonsoo Kim #ifdef CONFIG_SLUB 3807f361b2SJoonsoo Kim #include <linux/slub_def.h> 3907f361b2SJoonsoo Kim #endif 4007f361b2SJoonsoo Kim 4107f361b2SJoonsoo Kim #include <linux/memcontrol.h> 4211c7aec2SJesper Dangaard Brouer #include <linux/fault-inject.h> 4311c7aec2SJesper Dangaard Brouer #include <linux/kmemcheck.h> 4411c7aec2SJesper Dangaard Brouer #include <linux/kasan.h> 4511c7aec2SJesper Dangaard Brouer #include <linux/kmemleak.h> 467c00fce9SThomas Garnier #include <linux/random.h> 47d92a8cfcSPeter Zijlstra #include <linux/sched/mm.h> 4807f361b2SJoonsoo Kim 4997d06609SChristoph Lameter /* 5097d06609SChristoph Lameter * State of the slab allocator. 5197d06609SChristoph Lameter * 5297d06609SChristoph Lameter * This is used to describe the states of the allocator during bootup. 5397d06609SChristoph Lameter * Allocators use this to gradually bootstrap themselves. Most allocators 5497d06609SChristoph Lameter * have the problem that the structures used for managing slab caches are 5597d06609SChristoph Lameter * allocated from slab caches themselves. 5697d06609SChristoph Lameter */ 5797d06609SChristoph Lameter enum slab_state { 5897d06609SChristoph Lameter DOWN, /* No slab functionality yet */ 5997d06609SChristoph Lameter PARTIAL, /* SLUB: kmem_cache_node available */ 60ce8eb6c4SChristoph Lameter PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */ 6197d06609SChristoph Lameter UP, /* Slab caches usable but not all extras yet */ 6297d06609SChristoph Lameter FULL /* Everything is working */ 6397d06609SChristoph Lameter }; 6497d06609SChristoph Lameter 6597d06609SChristoph Lameter extern enum slab_state slab_state; 6697d06609SChristoph Lameter 6718004c5dSChristoph Lameter /* The slab cache mutex protects the management structures during changes */ 6818004c5dSChristoph Lameter extern struct mutex slab_mutex; 699b030cb8SChristoph Lameter 709b030cb8SChristoph Lameter /* The list of all slab caches on the system */ 7118004c5dSChristoph Lameter extern struct list_head slab_caches; 7218004c5dSChristoph Lameter 739b030cb8SChristoph Lameter /* The slab cache that manages slab cache information */ 749b030cb8SChristoph Lameter extern struct kmem_cache *kmem_cache; 759b030cb8SChristoph Lameter 76af3b5f87SVlastimil Babka /* A table of kmalloc cache names and sizes */ 77af3b5f87SVlastimil Babka extern const struct kmalloc_info_struct { 78af3b5f87SVlastimil Babka const char *name; 79af3b5f87SVlastimil Babka unsigned long size; 80af3b5f87SVlastimil Babka } kmalloc_info[]; 81af3b5f87SVlastimil Babka 82*d50112edSAlexey Dobriyan unsigned long calculate_alignment(slab_flags_t flags, 8345906855SChristoph Lameter unsigned long align, unsigned long size); 8445906855SChristoph Lameter 85f97d5f63SChristoph Lameter #ifndef CONFIG_SLOB 86f97d5f63SChristoph Lameter /* Kmalloc array related functions */ 8734cc6990SDaniel Sanders void setup_kmalloc_cache_index_table(void); 88*d50112edSAlexey Dobriyan void create_kmalloc_caches(slab_flags_t); 892c59dd65SChristoph Lameter 902c59dd65SChristoph Lameter /* Find the kmalloc slab corresponding for a certain size */ 912c59dd65SChristoph Lameter struct kmem_cache *kmalloc_slab(size_t, gfp_t); 92f97d5f63SChristoph Lameter #endif 93f97d5f63SChristoph Lameter 94f97d5f63SChristoph Lameter 959b030cb8SChristoph Lameter /* Functions provided by the slab allocators */ 96*d50112edSAlexey Dobriyan int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags); 9797d06609SChristoph Lameter 9845530c44SChristoph Lameter extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size, 99*d50112edSAlexey Dobriyan slab_flags_t flags); 10045530c44SChristoph Lameter extern void create_boot_cache(struct kmem_cache *, const char *name, 101*d50112edSAlexey Dobriyan size_t size, slab_flags_t flags); 10245530c44SChristoph Lameter 103423c929cSJoonsoo Kim int slab_unmergeable(struct kmem_cache *s); 104423c929cSJoonsoo Kim struct kmem_cache *find_mergeable(size_t size, size_t align, 105*d50112edSAlexey Dobriyan slab_flags_t flags, const char *name, void (*ctor)(void *)); 10612220deaSJoonsoo Kim #ifndef CONFIG_SLOB 1072633d7a0SGlauber Costa struct kmem_cache * 108a44cb944SVladimir Davydov __kmem_cache_alias(const char *name, size_t size, size_t align, 109*d50112edSAlexey Dobriyan slab_flags_t flags, void (*ctor)(void *)); 110423c929cSJoonsoo Kim 111*d50112edSAlexey Dobriyan slab_flags_t kmem_cache_flags(unsigned long object_size, 112*d50112edSAlexey Dobriyan slab_flags_t flags, const char *name, 113423c929cSJoonsoo Kim void (*ctor)(void *)); 114cbb79694SChristoph Lameter #else 1152633d7a0SGlauber Costa static inline struct kmem_cache * 116a44cb944SVladimir Davydov __kmem_cache_alias(const char *name, size_t size, size_t align, 117*d50112edSAlexey Dobriyan slab_flags_t flags, void (*ctor)(void *)) 118cbb79694SChristoph Lameter { return NULL; } 119423c929cSJoonsoo Kim 120*d50112edSAlexey Dobriyan static inline slab_flags_t kmem_cache_flags(unsigned long object_size, 121*d50112edSAlexey Dobriyan slab_flags_t flags, const char *name, 122423c929cSJoonsoo Kim void (*ctor)(void *)) 123423c929cSJoonsoo Kim { 124423c929cSJoonsoo Kim return flags; 125423c929cSJoonsoo Kim } 126cbb79694SChristoph Lameter #endif 127cbb79694SChristoph Lameter 128cbb79694SChristoph Lameter 129d8843922SGlauber Costa /* Legal flag mask for kmem_cache_create(), for various configurations */ 130d8843922SGlauber Costa #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \ 1315f0d5a3aSPaul E. McKenney SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS ) 132d8843922SGlauber Costa 133d8843922SGlauber Costa #if defined(CONFIG_DEBUG_SLAB) 134d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 135d8843922SGlauber Costa #elif defined(CONFIG_SLUB_DEBUG) 136d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 137becfda68SLaura Abbott SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) 138d8843922SGlauber Costa #else 139d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (0) 140d8843922SGlauber Costa #endif 141d8843922SGlauber Costa 142d8843922SGlauber Costa #if defined(CONFIG_SLAB) 143d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \ 144230e9fc2SVladimir Davydov SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \ 145230e9fc2SVladimir Davydov SLAB_NOTRACK | SLAB_ACCOUNT) 146d8843922SGlauber Costa #elif defined(CONFIG_SLUB) 147d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 148230e9fc2SVladimir Davydov SLAB_TEMPORARY | SLAB_NOTRACK | SLAB_ACCOUNT) 149d8843922SGlauber Costa #else 150d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (0) 151d8843922SGlauber Costa #endif 152d8843922SGlauber Costa 153e70954fdSThomas Garnier /* Common flags available with current configuration */ 154d8843922SGlauber Costa #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS) 155d8843922SGlauber Costa 156e70954fdSThomas Garnier /* Common flags permitted for kmem_cache_create */ 157e70954fdSThomas Garnier #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \ 158e70954fdSThomas Garnier SLAB_RED_ZONE | \ 159e70954fdSThomas Garnier SLAB_POISON | \ 160e70954fdSThomas Garnier SLAB_STORE_USER | \ 161e70954fdSThomas Garnier SLAB_TRACE | \ 162e70954fdSThomas Garnier SLAB_CONSISTENCY_CHECKS | \ 163e70954fdSThomas Garnier SLAB_MEM_SPREAD | \ 164e70954fdSThomas Garnier SLAB_NOLEAKTRACE | \ 165e70954fdSThomas Garnier SLAB_RECLAIM_ACCOUNT | \ 166e70954fdSThomas Garnier SLAB_TEMPORARY | \ 167e70954fdSThomas Garnier SLAB_NOTRACK | \ 168e70954fdSThomas Garnier SLAB_ACCOUNT) 169e70954fdSThomas Garnier 170945cf2b6SChristoph Lameter int __kmem_cache_shutdown(struct kmem_cache *); 17152b4b950SDmitry Safonov void __kmem_cache_release(struct kmem_cache *); 172c9fc5864STejun Heo int __kmem_cache_shrink(struct kmem_cache *); 173c9fc5864STejun Heo void __kmemcg_cache_deactivate(struct kmem_cache *s); 17441a21285SChristoph Lameter void slab_kmem_cache_release(struct kmem_cache *); 175945cf2b6SChristoph Lameter 176b7454ad3SGlauber Costa struct seq_file; 177b7454ad3SGlauber Costa struct file; 178b7454ad3SGlauber Costa 1790d7561c6SGlauber Costa struct slabinfo { 1800d7561c6SGlauber Costa unsigned long active_objs; 1810d7561c6SGlauber Costa unsigned long num_objs; 1820d7561c6SGlauber Costa unsigned long active_slabs; 1830d7561c6SGlauber Costa unsigned long num_slabs; 1840d7561c6SGlauber Costa unsigned long shared_avail; 1850d7561c6SGlauber Costa unsigned int limit; 1860d7561c6SGlauber Costa unsigned int batchcount; 1870d7561c6SGlauber Costa unsigned int shared; 1880d7561c6SGlauber Costa unsigned int objects_per_slab; 1890d7561c6SGlauber Costa unsigned int cache_order; 1900d7561c6SGlauber Costa }; 1910d7561c6SGlauber Costa 1920d7561c6SGlauber Costa void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); 1930d7561c6SGlauber Costa void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); 194b7454ad3SGlauber Costa ssize_t slabinfo_write(struct file *file, const char __user *buffer, 195b7454ad3SGlauber Costa size_t count, loff_t *ppos); 196ba6c496eSGlauber Costa 197484748f0SChristoph Lameter /* 198484748f0SChristoph Lameter * Generic implementation of bulk operations 199484748f0SChristoph Lameter * These are useful for situations in which the allocator cannot 2009f706d68SJesper Dangaard Brouer * perform optimizations. In that case segments of the object listed 201484748f0SChristoph Lameter * may be allocated or freed using these operations. 202484748f0SChristoph Lameter */ 203484748f0SChristoph Lameter void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **); 204865762a8SJesper Dangaard Brouer int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **); 205484748f0SChristoph Lameter 206127424c8SJohannes Weiner #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) 207510ded33STejun Heo 208510ded33STejun Heo /* List of all root caches. */ 209510ded33STejun Heo extern struct list_head slab_root_caches; 210510ded33STejun Heo #define root_caches_node memcg_params.__root_caches_node 211510ded33STejun Heo 212426589f5SVladimir Davydov /* 213426589f5SVladimir Davydov * Iterate over all memcg caches of the given root cache. The caller must hold 214426589f5SVladimir Davydov * slab_mutex. 215426589f5SVladimir Davydov */ 216426589f5SVladimir Davydov #define for_each_memcg_cache(iter, root) \ 2179eeadc8bSTejun Heo list_for_each_entry(iter, &(root)->memcg_params.children, \ 2189eeadc8bSTejun Heo memcg_params.children_node) 219426589f5SVladimir Davydov 220ba6c496eSGlauber Costa static inline bool is_root_cache(struct kmem_cache *s) 221ba6c496eSGlauber Costa { 2229eeadc8bSTejun Heo return !s->memcg_params.root_cache; 223ba6c496eSGlauber Costa } 2242633d7a0SGlauber Costa 225b9ce5ef4SGlauber Costa static inline bool slab_equal_or_root(struct kmem_cache *s, 226b9ce5ef4SGlauber Costa struct kmem_cache *p) 227b9ce5ef4SGlauber Costa { 228f7ce3190SVladimir Davydov return p == s || p == s->memcg_params.root_cache; 229b9ce5ef4SGlauber Costa } 230749c5415SGlauber Costa 231749c5415SGlauber Costa /* 232749c5415SGlauber Costa * We use suffixes to the name in memcg because we can't have caches 233749c5415SGlauber Costa * created in the system with the same name. But when we print them 234749c5415SGlauber Costa * locally, better refer to them with the base name 235749c5415SGlauber Costa */ 236749c5415SGlauber Costa static inline const char *cache_name(struct kmem_cache *s) 237749c5415SGlauber Costa { 238749c5415SGlauber Costa if (!is_root_cache(s)) 239f7ce3190SVladimir Davydov s = s->memcg_params.root_cache; 240749c5415SGlauber Costa return s->name; 241749c5415SGlauber Costa } 242749c5415SGlauber Costa 243f8570263SVladimir Davydov /* 244f8570263SVladimir Davydov * Note, we protect with RCU only the memcg_caches array, not per-memcg caches. 245f7ce3190SVladimir Davydov * That said the caller must assure the memcg's cache won't go away by either 246f7ce3190SVladimir Davydov * taking a css reference to the owner cgroup, or holding the slab_mutex. 247f8570263SVladimir Davydov */ 2482ade4de8SQiang Huang static inline struct kmem_cache * 2492ade4de8SQiang Huang cache_from_memcg_idx(struct kmem_cache *s, int idx) 250749c5415SGlauber Costa { 251959c8963SVladimir Davydov struct kmem_cache *cachep; 252f7ce3190SVladimir Davydov struct memcg_cache_array *arr; 253f8570263SVladimir Davydov 254f8570263SVladimir Davydov rcu_read_lock(); 255f7ce3190SVladimir Davydov arr = rcu_dereference(s->memcg_params.memcg_caches); 256959c8963SVladimir Davydov 257959c8963SVladimir Davydov /* 258959c8963SVladimir Davydov * Make sure we will access the up-to-date value. The code updating 259959c8963SVladimir Davydov * memcg_caches issues a write barrier to match this (see 260f7ce3190SVladimir Davydov * memcg_create_kmem_cache()). 261959c8963SVladimir Davydov */ 262506458efSWill Deacon cachep = READ_ONCE(arr->entries[idx]); 2638df0c2dcSPranith Kumar rcu_read_unlock(); 2648df0c2dcSPranith Kumar 265959c8963SVladimir Davydov return cachep; 266749c5415SGlauber Costa } 267943a451aSGlauber Costa 268943a451aSGlauber Costa static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s) 269943a451aSGlauber Costa { 270943a451aSGlauber Costa if (is_root_cache(s)) 271943a451aSGlauber Costa return s; 272f7ce3190SVladimir Davydov return s->memcg_params.root_cache; 273943a451aSGlauber Costa } 2745dfb4175SVladimir Davydov 275f3ccb2c4SVladimir Davydov static __always_inline int memcg_charge_slab(struct page *page, 276f3ccb2c4SVladimir Davydov gfp_t gfp, int order, 277f3ccb2c4SVladimir Davydov struct kmem_cache *s) 2785dfb4175SVladimir Davydov { 2795dfb4175SVladimir Davydov if (!memcg_kmem_enabled()) 2805dfb4175SVladimir Davydov return 0; 2815dfb4175SVladimir Davydov if (is_root_cache(s)) 2825dfb4175SVladimir Davydov return 0; 2837779f212SJohannes Weiner return memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg); 28427ee57c9SVladimir Davydov } 28527ee57c9SVladimir Davydov 28627ee57c9SVladimir Davydov static __always_inline void memcg_uncharge_slab(struct page *page, int order, 28727ee57c9SVladimir Davydov struct kmem_cache *s) 28827ee57c9SVladimir Davydov { 28945264778SVladimir Davydov if (!memcg_kmem_enabled()) 29045264778SVladimir Davydov return; 29127ee57c9SVladimir Davydov memcg_kmem_uncharge(page, order); 2925dfb4175SVladimir Davydov } 293f7ce3190SVladimir Davydov 294f7ce3190SVladimir Davydov extern void slab_init_memcg_params(struct kmem_cache *); 295510ded33STejun Heo extern void memcg_link_cache(struct kmem_cache *s); 29601fb58bcSTejun Heo extern void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s, 29701fb58bcSTejun Heo void (*deact_fn)(struct kmem_cache *)); 298f7ce3190SVladimir Davydov 299127424c8SJohannes Weiner #else /* CONFIG_MEMCG && !CONFIG_SLOB */ 300f7ce3190SVladimir Davydov 301510ded33STejun Heo /* If !memcg, all caches are root. */ 302510ded33STejun Heo #define slab_root_caches slab_caches 303510ded33STejun Heo #define root_caches_node list 304510ded33STejun Heo 305426589f5SVladimir Davydov #define for_each_memcg_cache(iter, root) \ 306426589f5SVladimir Davydov for ((void)(iter), (void)(root); 0; ) 307426589f5SVladimir Davydov 308ba6c496eSGlauber Costa static inline bool is_root_cache(struct kmem_cache *s) 309ba6c496eSGlauber Costa { 310ba6c496eSGlauber Costa return true; 311ba6c496eSGlauber Costa } 312ba6c496eSGlauber Costa 313b9ce5ef4SGlauber Costa static inline bool slab_equal_or_root(struct kmem_cache *s, 314b9ce5ef4SGlauber Costa struct kmem_cache *p) 315b9ce5ef4SGlauber Costa { 316b9ce5ef4SGlauber Costa return true; 317b9ce5ef4SGlauber Costa } 318749c5415SGlauber Costa 319749c5415SGlauber Costa static inline const char *cache_name(struct kmem_cache *s) 320749c5415SGlauber Costa { 321749c5415SGlauber Costa return s->name; 322749c5415SGlauber Costa } 323749c5415SGlauber Costa 3242ade4de8SQiang Huang static inline struct kmem_cache * 3252ade4de8SQiang Huang cache_from_memcg_idx(struct kmem_cache *s, int idx) 326749c5415SGlauber Costa { 327749c5415SGlauber Costa return NULL; 328749c5415SGlauber Costa } 329943a451aSGlauber Costa 330943a451aSGlauber Costa static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s) 331943a451aSGlauber Costa { 332943a451aSGlauber Costa return s; 333943a451aSGlauber Costa } 3345dfb4175SVladimir Davydov 335f3ccb2c4SVladimir Davydov static inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order, 336f3ccb2c4SVladimir Davydov struct kmem_cache *s) 3375dfb4175SVladimir Davydov { 3385dfb4175SVladimir Davydov return 0; 3395dfb4175SVladimir Davydov } 3405dfb4175SVladimir Davydov 34127ee57c9SVladimir Davydov static inline void memcg_uncharge_slab(struct page *page, int order, 34227ee57c9SVladimir Davydov struct kmem_cache *s) 34327ee57c9SVladimir Davydov { 34427ee57c9SVladimir Davydov } 34527ee57c9SVladimir Davydov 346f7ce3190SVladimir Davydov static inline void slab_init_memcg_params(struct kmem_cache *s) 347f7ce3190SVladimir Davydov { 348f7ce3190SVladimir Davydov } 349510ded33STejun Heo 350510ded33STejun Heo static inline void memcg_link_cache(struct kmem_cache *s) 351510ded33STejun Heo { 352510ded33STejun Heo } 353510ded33STejun Heo 354127424c8SJohannes Weiner #endif /* CONFIG_MEMCG && !CONFIG_SLOB */ 355b9ce5ef4SGlauber Costa 356b9ce5ef4SGlauber Costa static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 357b9ce5ef4SGlauber Costa { 358b9ce5ef4SGlauber Costa struct kmem_cache *cachep; 359b9ce5ef4SGlauber Costa struct page *page; 360b9ce5ef4SGlauber Costa 361b9ce5ef4SGlauber Costa /* 362b9ce5ef4SGlauber Costa * When kmemcg is not being used, both assignments should return the 363b9ce5ef4SGlauber Costa * same value. but we don't want to pay the assignment price in that 364b9ce5ef4SGlauber Costa * case. If it is not compiled in, the compiler should be smart enough 365b9ce5ef4SGlauber Costa * to not do even the assignment. In that case, slab_equal_or_root 366b9ce5ef4SGlauber Costa * will also be a constant. 367b9ce5ef4SGlauber Costa */ 368becfda68SLaura Abbott if (!memcg_kmem_enabled() && 369becfda68SLaura Abbott !unlikely(s->flags & SLAB_CONSISTENCY_CHECKS)) 370b9ce5ef4SGlauber Costa return s; 371b9ce5ef4SGlauber Costa 372b9ce5ef4SGlauber Costa page = virt_to_head_page(x); 373b9ce5ef4SGlauber Costa cachep = page->slab_cache; 374b9ce5ef4SGlauber Costa if (slab_equal_or_root(cachep, s)) 375b9ce5ef4SGlauber Costa return cachep; 376b9ce5ef4SGlauber Costa 377b9ce5ef4SGlauber Costa pr_err("%s: Wrong slab cache. %s but object is from %s\n", 3782d16e0fdSDaniel Borkmann __func__, s->name, cachep->name); 379b9ce5ef4SGlauber Costa WARN_ON_ONCE(1); 380b9ce5ef4SGlauber Costa return s; 381b9ce5ef4SGlauber Costa } 382ca34956bSChristoph Lameter 38311c7aec2SJesper Dangaard Brouer static inline size_t slab_ksize(const struct kmem_cache *s) 38411c7aec2SJesper Dangaard Brouer { 38511c7aec2SJesper Dangaard Brouer #ifndef CONFIG_SLUB 38611c7aec2SJesper Dangaard Brouer return s->object_size; 38711c7aec2SJesper Dangaard Brouer 38811c7aec2SJesper Dangaard Brouer #else /* CONFIG_SLUB */ 38911c7aec2SJesper Dangaard Brouer # ifdef CONFIG_SLUB_DEBUG 39011c7aec2SJesper Dangaard Brouer /* 39111c7aec2SJesper Dangaard Brouer * Debugging requires use of the padding between object 39211c7aec2SJesper Dangaard Brouer * and whatever may come after it. 39311c7aec2SJesper Dangaard Brouer */ 39411c7aec2SJesper Dangaard Brouer if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) 39511c7aec2SJesper Dangaard Brouer return s->object_size; 39611c7aec2SJesper Dangaard Brouer # endif 39780a9201aSAlexander Potapenko if (s->flags & SLAB_KASAN) 39880a9201aSAlexander Potapenko return s->object_size; 39911c7aec2SJesper Dangaard Brouer /* 40011c7aec2SJesper Dangaard Brouer * If we have the need to store the freelist pointer 40111c7aec2SJesper Dangaard Brouer * back there or track user information then we can 40211c7aec2SJesper Dangaard Brouer * only use the space before that information. 40311c7aec2SJesper Dangaard Brouer */ 4045f0d5a3aSPaul E. McKenney if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER)) 40511c7aec2SJesper Dangaard Brouer return s->inuse; 40611c7aec2SJesper Dangaard Brouer /* 40711c7aec2SJesper Dangaard Brouer * Else we can use all the padding etc for the allocation 40811c7aec2SJesper Dangaard Brouer */ 40911c7aec2SJesper Dangaard Brouer return s->size; 41011c7aec2SJesper Dangaard Brouer #endif 41111c7aec2SJesper Dangaard Brouer } 41211c7aec2SJesper Dangaard Brouer 41311c7aec2SJesper Dangaard Brouer static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, 41411c7aec2SJesper Dangaard Brouer gfp_t flags) 41511c7aec2SJesper Dangaard Brouer { 41611c7aec2SJesper Dangaard Brouer flags &= gfp_allowed_mask; 417d92a8cfcSPeter Zijlstra 418d92a8cfcSPeter Zijlstra fs_reclaim_acquire(flags); 419d92a8cfcSPeter Zijlstra fs_reclaim_release(flags); 420d92a8cfcSPeter Zijlstra 42111c7aec2SJesper Dangaard Brouer might_sleep_if(gfpflags_allow_blocking(flags)); 42211c7aec2SJesper Dangaard Brouer 423fab9963aSJesper Dangaard Brouer if (should_failslab(s, flags)) 42411c7aec2SJesper Dangaard Brouer return NULL; 42511c7aec2SJesper Dangaard Brouer 42645264778SVladimir Davydov if (memcg_kmem_enabled() && 42745264778SVladimir Davydov ((flags & __GFP_ACCOUNT) || (s->flags & SLAB_ACCOUNT))) 42845264778SVladimir Davydov return memcg_kmem_get_cache(s); 42945264778SVladimir Davydov 43045264778SVladimir Davydov return s; 43111c7aec2SJesper Dangaard Brouer } 43211c7aec2SJesper Dangaard Brouer 43311c7aec2SJesper Dangaard Brouer static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, 43411c7aec2SJesper Dangaard Brouer size_t size, void **p) 43511c7aec2SJesper Dangaard Brouer { 43611c7aec2SJesper Dangaard Brouer size_t i; 43711c7aec2SJesper Dangaard Brouer 43811c7aec2SJesper Dangaard Brouer flags &= gfp_allowed_mask; 43911c7aec2SJesper Dangaard Brouer for (i = 0; i < size; i++) { 44011c7aec2SJesper Dangaard Brouer void *object = p[i]; 44111c7aec2SJesper Dangaard Brouer 44211c7aec2SJesper Dangaard Brouer kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); 44311c7aec2SJesper Dangaard Brouer kmemleak_alloc_recursive(object, s->object_size, 1, 44411c7aec2SJesper Dangaard Brouer s->flags, flags); 445505f5dcbSAlexander Potapenko kasan_slab_alloc(s, object, flags); 44611c7aec2SJesper Dangaard Brouer } 44745264778SVladimir Davydov 44845264778SVladimir Davydov if (memcg_kmem_enabled()) 44911c7aec2SJesper Dangaard Brouer memcg_kmem_put_cache(s); 45011c7aec2SJesper Dangaard Brouer } 45111c7aec2SJesper Dangaard Brouer 45244c5356fSChristoph Lameter #ifndef CONFIG_SLOB 453ca34956bSChristoph Lameter /* 454ca34956bSChristoph Lameter * The slab lists for all objects. 455ca34956bSChristoph Lameter */ 456ca34956bSChristoph Lameter struct kmem_cache_node { 457ca34956bSChristoph Lameter spinlock_t list_lock; 458ca34956bSChristoph Lameter 459ca34956bSChristoph Lameter #ifdef CONFIG_SLAB 460ca34956bSChristoph Lameter struct list_head slabs_partial; /* partial list first, better asm code */ 461ca34956bSChristoph Lameter struct list_head slabs_full; 462ca34956bSChristoph Lameter struct list_head slabs_free; 463bf00bd34SDavid Rientjes unsigned long total_slabs; /* length of all slab lists */ 464bf00bd34SDavid Rientjes unsigned long free_slabs; /* length of free slab list only */ 465ca34956bSChristoph Lameter unsigned long free_objects; 466ca34956bSChristoph Lameter unsigned int free_limit; 467ca34956bSChristoph Lameter unsigned int colour_next; /* Per-node cache coloring */ 468ca34956bSChristoph Lameter struct array_cache *shared; /* shared per node */ 469c8522a3aSJoonsoo Kim struct alien_cache **alien; /* on other nodes */ 470ca34956bSChristoph Lameter unsigned long next_reap; /* updated without locking */ 471ca34956bSChristoph Lameter int free_touched; /* updated without locking */ 472ca34956bSChristoph Lameter #endif 473ca34956bSChristoph Lameter 474ca34956bSChristoph Lameter #ifdef CONFIG_SLUB 475ca34956bSChristoph Lameter unsigned long nr_partial; 476ca34956bSChristoph Lameter struct list_head partial; 477ca34956bSChristoph Lameter #ifdef CONFIG_SLUB_DEBUG 478ca34956bSChristoph Lameter atomic_long_t nr_slabs; 479ca34956bSChristoph Lameter atomic_long_t total_objects; 480ca34956bSChristoph Lameter struct list_head full; 481ca34956bSChristoph Lameter #endif 482ca34956bSChristoph Lameter #endif 483ca34956bSChristoph Lameter 484ca34956bSChristoph Lameter }; 485e25839f6SWanpeng Li 48644c5356fSChristoph Lameter static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 48744c5356fSChristoph Lameter { 48844c5356fSChristoph Lameter return s->node[node]; 48944c5356fSChristoph Lameter } 49044c5356fSChristoph Lameter 49144c5356fSChristoph Lameter /* 49244c5356fSChristoph Lameter * Iterator over all nodes. The body will be executed for each node that has 49344c5356fSChristoph Lameter * a kmem_cache_node structure allocated (which is true for all online nodes) 49444c5356fSChristoph Lameter */ 49544c5356fSChristoph Lameter #define for_each_kmem_cache_node(__s, __node, __n) \ 4969163582cSMikulas Patocka for (__node = 0; __node < nr_node_ids; __node++) \ 4979163582cSMikulas Patocka if ((__n = get_node(__s, __node))) 49844c5356fSChristoph Lameter 49944c5356fSChristoph Lameter #endif 50044c5356fSChristoph Lameter 5011df3b26fSVladimir Davydov void *slab_start(struct seq_file *m, loff_t *pos); 502276a2439SWanpeng Li void *slab_next(struct seq_file *m, void *p, loff_t *pos); 503276a2439SWanpeng Li void slab_stop(struct seq_file *m, void *p); 504bc2791f8STejun Heo void *memcg_slab_start(struct seq_file *m, loff_t *pos); 505bc2791f8STejun Heo void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos); 506bc2791f8STejun Heo void memcg_slab_stop(struct seq_file *m, void *p); 507b047501cSVladimir Davydov int memcg_slab_show(struct seq_file *m, void *p); 5085240ab40SAndrey Ryabinin 509852d8be0SYang Shi #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG) 510852d8be0SYang Shi void dump_unreclaimable_slab(void); 511852d8be0SYang Shi #else 512852d8be0SYang Shi static inline void dump_unreclaimable_slab(void) 513852d8be0SYang Shi { 514852d8be0SYang Shi } 515852d8be0SYang Shi #endif 516852d8be0SYang Shi 51755834c59SAlexander Potapenko void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr); 51855834c59SAlexander Potapenko 5197c00fce9SThomas Garnier #ifdef CONFIG_SLAB_FREELIST_RANDOM 5207c00fce9SThomas Garnier int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 5217c00fce9SThomas Garnier gfp_t gfp); 5227c00fce9SThomas Garnier void cache_random_seq_destroy(struct kmem_cache *cachep); 5237c00fce9SThomas Garnier #else 5247c00fce9SThomas Garnier static inline int cache_random_seq_create(struct kmem_cache *cachep, 5257c00fce9SThomas Garnier unsigned int count, gfp_t gfp) 5267c00fce9SThomas Garnier { 5277c00fce9SThomas Garnier return 0; 5287c00fce9SThomas Garnier } 5297c00fce9SThomas Garnier static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } 5307c00fce9SThomas Garnier #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 5317c00fce9SThomas Garnier 5325240ab40SAndrey Ryabinin #endif /* MM_SLAB_H */ 533