197d06609SChristoph Lameter #ifndef MM_SLAB_H 297d06609SChristoph Lameter #define MM_SLAB_H 397d06609SChristoph Lameter /* 497d06609SChristoph Lameter * Internal slab definitions 597d06609SChristoph Lameter */ 697d06609SChristoph Lameter 797d06609SChristoph Lameter /* 897d06609SChristoph Lameter * State of the slab allocator. 997d06609SChristoph Lameter * 1097d06609SChristoph Lameter * This is used to describe the states of the allocator during bootup. 1197d06609SChristoph Lameter * Allocators use this to gradually bootstrap themselves. Most allocators 1297d06609SChristoph Lameter * have the problem that the structures used for managing slab caches are 1397d06609SChristoph Lameter * allocated from slab caches themselves. 1497d06609SChristoph Lameter */ 1597d06609SChristoph Lameter enum slab_state { 1697d06609SChristoph Lameter DOWN, /* No slab functionality yet */ 1797d06609SChristoph Lameter PARTIAL, /* SLUB: kmem_cache_node available */ 1897d06609SChristoph Lameter PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */ 1997d06609SChristoph Lameter PARTIAL_L3, /* SLAB: kmalloc size for l3 struct available */ 2097d06609SChristoph Lameter UP, /* Slab caches usable but not all extras yet */ 2197d06609SChristoph Lameter FULL /* Everything is working */ 2297d06609SChristoph Lameter }; 2397d06609SChristoph Lameter 2497d06609SChristoph Lameter extern enum slab_state slab_state; 2597d06609SChristoph Lameter 2618004c5dSChristoph Lameter /* The slab cache mutex protects the management structures during changes */ 2718004c5dSChristoph Lameter extern struct mutex slab_mutex; 289b030cb8SChristoph Lameter 299b030cb8SChristoph Lameter /* The list of all slab caches on the system */ 3018004c5dSChristoph Lameter extern struct list_head slab_caches; 3118004c5dSChristoph Lameter 329b030cb8SChristoph Lameter /* The slab cache that manages slab cache information */ 339b030cb8SChristoph Lameter extern struct kmem_cache *kmem_cache; 349b030cb8SChristoph Lameter 3545906855SChristoph Lameter unsigned long calculate_alignment(unsigned long flags, 3645906855SChristoph Lameter unsigned long align, unsigned long size); 3745906855SChristoph Lameter 389b030cb8SChristoph Lameter /* Functions provided by the slab allocators */ 398a13a4ccSChristoph Lameter extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags); 4097d06609SChristoph Lameter 4145530c44SChristoph Lameter extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size, 4245530c44SChristoph Lameter unsigned long flags); 4345530c44SChristoph Lameter extern void create_boot_cache(struct kmem_cache *, const char *name, 4445530c44SChristoph Lameter size_t size, unsigned long flags); 4545530c44SChristoph Lameter 462633d7a0SGlauber Costa struct mem_cgroup; 47cbb79694SChristoph Lameter #ifdef CONFIG_SLUB 482633d7a0SGlauber Costa struct kmem_cache * 492633d7a0SGlauber Costa __kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size, 50cbb79694SChristoph Lameter size_t align, unsigned long flags, void (*ctor)(void *)); 51cbb79694SChristoph Lameter #else 522633d7a0SGlauber Costa static inline struct kmem_cache * 532633d7a0SGlauber Costa __kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size, 54cbb79694SChristoph Lameter size_t align, unsigned long flags, void (*ctor)(void *)) 55cbb79694SChristoph Lameter { return NULL; } 56cbb79694SChristoph Lameter #endif 57cbb79694SChristoph Lameter 58cbb79694SChristoph Lameter 59d8843922SGlauber Costa /* Legal flag mask for kmem_cache_create(), for various configurations */ 60d8843922SGlauber Costa #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \ 61d8843922SGlauber Costa SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS ) 62d8843922SGlauber Costa 63d8843922SGlauber Costa #if defined(CONFIG_DEBUG_SLAB) 64d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 65d8843922SGlauber Costa #elif defined(CONFIG_SLUB_DEBUG) 66d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 67d8843922SGlauber Costa SLAB_TRACE | SLAB_DEBUG_FREE) 68d8843922SGlauber Costa #else 69d8843922SGlauber Costa #define SLAB_DEBUG_FLAGS (0) 70d8843922SGlauber Costa #endif 71d8843922SGlauber Costa 72d8843922SGlauber Costa #if defined(CONFIG_SLAB) 73d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \ 74d8843922SGlauber Costa SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK) 75d8843922SGlauber Costa #elif defined(CONFIG_SLUB) 76d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 77d8843922SGlauber Costa SLAB_TEMPORARY | SLAB_NOTRACK) 78d8843922SGlauber Costa #else 79d8843922SGlauber Costa #define SLAB_CACHE_FLAGS (0) 80d8843922SGlauber Costa #endif 81d8843922SGlauber Costa 82d8843922SGlauber Costa #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS) 83d8843922SGlauber Costa 84945cf2b6SChristoph Lameter int __kmem_cache_shutdown(struct kmem_cache *); 85945cf2b6SChristoph Lameter 86b7454ad3SGlauber Costa struct seq_file; 87b7454ad3SGlauber Costa struct file; 88b7454ad3SGlauber Costa 890d7561c6SGlauber Costa struct slabinfo { 900d7561c6SGlauber Costa unsigned long active_objs; 910d7561c6SGlauber Costa unsigned long num_objs; 920d7561c6SGlauber Costa unsigned long active_slabs; 930d7561c6SGlauber Costa unsigned long num_slabs; 940d7561c6SGlauber Costa unsigned long shared_avail; 950d7561c6SGlauber Costa unsigned int limit; 960d7561c6SGlauber Costa unsigned int batchcount; 970d7561c6SGlauber Costa unsigned int shared; 980d7561c6SGlauber Costa unsigned int objects_per_slab; 990d7561c6SGlauber Costa unsigned int cache_order; 1000d7561c6SGlauber Costa }; 1010d7561c6SGlauber Costa 1020d7561c6SGlauber Costa void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); 1030d7561c6SGlauber Costa void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); 104b7454ad3SGlauber Costa ssize_t slabinfo_write(struct file *file, const char __user *buffer, 105b7454ad3SGlauber Costa size_t count, loff_t *ppos); 106ba6c496eSGlauber Costa 107ba6c496eSGlauber Costa #ifdef CONFIG_MEMCG_KMEM 108ba6c496eSGlauber Costa static inline bool is_root_cache(struct kmem_cache *s) 109ba6c496eSGlauber Costa { 110ba6c496eSGlauber Costa return !s->memcg_params || s->memcg_params->is_root_cache; 111ba6c496eSGlauber Costa } 1122633d7a0SGlauber Costa 1132633d7a0SGlauber Costa static inline bool cache_match_memcg(struct kmem_cache *cachep, 1142633d7a0SGlauber Costa struct mem_cgroup *memcg) 1152633d7a0SGlauber Costa { 1162633d7a0SGlauber Costa return (is_root_cache(cachep) && !memcg) || 1172633d7a0SGlauber Costa (cachep->memcg_params->memcg == memcg); 1182633d7a0SGlauber Costa } 119b9ce5ef4SGlauber Costa 1201f458cbfSGlauber Costa static inline void memcg_bind_pages(struct kmem_cache *s, int order) 1211f458cbfSGlauber Costa { 1221f458cbfSGlauber Costa if (!is_root_cache(s)) 1231f458cbfSGlauber Costa atomic_add(1 << order, &s->memcg_params->nr_pages); 1241f458cbfSGlauber Costa } 1251f458cbfSGlauber Costa 1261f458cbfSGlauber Costa static inline void memcg_release_pages(struct kmem_cache *s, int order) 1271f458cbfSGlauber Costa { 1281f458cbfSGlauber Costa if (is_root_cache(s)) 1291f458cbfSGlauber Costa return; 1301f458cbfSGlauber Costa 1311f458cbfSGlauber Costa if (atomic_sub_and_test((1 << order), &s->memcg_params->nr_pages)) 1321f458cbfSGlauber Costa mem_cgroup_destroy_cache(s); 1331f458cbfSGlauber Costa } 1341f458cbfSGlauber Costa 135b9ce5ef4SGlauber Costa static inline bool slab_equal_or_root(struct kmem_cache *s, 136b9ce5ef4SGlauber Costa struct kmem_cache *p) 137b9ce5ef4SGlauber Costa { 138b9ce5ef4SGlauber Costa return (p == s) || 139b9ce5ef4SGlauber Costa (s->memcg_params && (p == s->memcg_params->root_cache)); 140b9ce5ef4SGlauber Costa } 141*749c5415SGlauber Costa 142*749c5415SGlauber Costa /* 143*749c5415SGlauber Costa * We use suffixes to the name in memcg because we can't have caches 144*749c5415SGlauber Costa * created in the system with the same name. But when we print them 145*749c5415SGlauber Costa * locally, better refer to them with the base name 146*749c5415SGlauber Costa */ 147*749c5415SGlauber Costa static inline const char *cache_name(struct kmem_cache *s) 148*749c5415SGlauber Costa { 149*749c5415SGlauber Costa if (!is_root_cache(s)) 150*749c5415SGlauber Costa return s->memcg_params->root_cache->name; 151*749c5415SGlauber Costa return s->name; 152*749c5415SGlauber Costa } 153*749c5415SGlauber Costa 154*749c5415SGlauber Costa static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx) 155*749c5415SGlauber Costa { 156*749c5415SGlauber Costa return s->memcg_params->memcg_caches[idx]; 157*749c5415SGlauber Costa } 158ba6c496eSGlauber Costa #else 159ba6c496eSGlauber Costa static inline bool is_root_cache(struct kmem_cache *s) 160ba6c496eSGlauber Costa { 161ba6c496eSGlauber Costa return true; 162ba6c496eSGlauber Costa } 163ba6c496eSGlauber Costa 1642633d7a0SGlauber Costa static inline bool cache_match_memcg(struct kmem_cache *cachep, 1652633d7a0SGlauber Costa struct mem_cgroup *memcg) 1662633d7a0SGlauber Costa { 1672633d7a0SGlauber Costa return true; 1682633d7a0SGlauber Costa } 169b9ce5ef4SGlauber Costa 1701f458cbfSGlauber Costa static inline void memcg_bind_pages(struct kmem_cache *s, int order) 1711f458cbfSGlauber Costa { 1721f458cbfSGlauber Costa } 1731f458cbfSGlauber Costa 1741f458cbfSGlauber Costa static inline void memcg_release_pages(struct kmem_cache *s, int order) 1751f458cbfSGlauber Costa { 1761f458cbfSGlauber Costa } 1771f458cbfSGlauber Costa 178b9ce5ef4SGlauber Costa static inline bool slab_equal_or_root(struct kmem_cache *s, 179b9ce5ef4SGlauber Costa struct kmem_cache *p) 180b9ce5ef4SGlauber Costa { 181b9ce5ef4SGlauber Costa return true; 182b9ce5ef4SGlauber Costa } 183*749c5415SGlauber Costa 184*749c5415SGlauber Costa static inline const char *cache_name(struct kmem_cache *s) 185*749c5415SGlauber Costa { 186*749c5415SGlauber Costa return s->name; 187*749c5415SGlauber Costa } 188*749c5415SGlauber Costa 189*749c5415SGlauber Costa static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx) 190*749c5415SGlauber Costa { 191*749c5415SGlauber Costa return NULL; 192*749c5415SGlauber Costa } 193ba6c496eSGlauber Costa #endif 194b9ce5ef4SGlauber Costa 195b9ce5ef4SGlauber Costa static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 196b9ce5ef4SGlauber Costa { 197b9ce5ef4SGlauber Costa struct kmem_cache *cachep; 198b9ce5ef4SGlauber Costa struct page *page; 199b9ce5ef4SGlauber Costa 200b9ce5ef4SGlauber Costa /* 201b9ce5ef4SGlauber Costa * When kmemcg is not being used, both assignments should return the 202b9ce5ef4SGlauber Costa * same value. but we don't want to pay the assignment price in that 203b9ce5ef4SGlauber Costa * case. If it is not compiled in, the compiler should be smart enough 204b9ce5ef4SGlauber Costa * to not do even the assignment. In that case, slab_equal_or_root 205b9ce5ef4SGlauber Costa * will also be a constant. 206b9ce5ef4SGlauber Costa */ 207b9ce5ef4SGlauber Costa if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE)) 208b9ce5ef4SGlauber Costa return s; 209b9ce5ef4SGlauber Costa 210b9ce5ef4SGlauber Costa page = virt_to_head_page(x); 211b9ce5ef4SGlauber Costa cachep = page->slab_cache; 212b9ce5ef4SGlauber Costa if (slab_equal_or_root(cachep, s)) 213b9ce5ef4SGlauber Costa return cachep; 214b9ce5ef4SGlauber Costa 215b9ce5ef4SGlauber Costa pr_err("%s: Wrong slab cache. %s but object is from %s\n", 216b9ce5ef4SGlauber Costa __FUNCTION__, cachep->name, s->name); 217b9ce5ef4SGlauber Costa WARN_ON_ONCE(1); 218b9ce5ef4SGlauber Costa return s; 219b9ce5ef4SGlauber Costa } 22097d06609SChristoph Lameter #endif 221