1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 4 * Copyright (C) 2007 The Regents of the University of California. 5 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 6 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 7 * UCRL-CODE-235197 8 * 9 * This file is part of the SPL, Solaris Porting Layer. 10 * 11 * The SPL is free software; you can redistribute it and/or modify it 12 * under the terms of the GNU General Public License as published by the 13 * Free Software Foundation; either version 2 of the License, or (at your 14 * option) any later version. 15 * 16 * The SPL is distributed in the hope that it will be useful, but WITHOUT 17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 19 * for more details. 20 * 21 * You should have received a copy of the GNU General Public License along 22 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 23 */ 24 25 #ifndef _SPL_KMEM_CACHE_H 26 #define _SPL_KMEM_CACHE_H 27 28 #include <sys/taskq.h> 29 30 /* 31 * Slab allocation interfaces. The SPL slab differs from the standard 32 * Linux SLAB or SLUB primarily in that each cache may be backed by slabs 33 * allocated from the physical or virtual memory address space. The virtual 34 * slabs allow for good behavior when allocation large objects of identical 35 * size. This slab implementation also supports both constructors and 36 * destructors which the Linux slab does not. 37 */ 38 typedef enum kmc_bit { 39 KMC_BIT_NODEBUG = 1, /* Default behavior */ 40 KMC_BIT_KVMEM = 7, /* Use kvmalloc linux allocator */ 41 KMC_BIT_SLAB = 8, /* Use Linux slab cache */ 42 KMC_BIT_DEADLOCKED = 14, /* Deadlock detected */ 43 KMC_BIT_GROWING = 15, /* Growing in progress */ 44 KMC_BIT_REAPING = 16, /* Reaping in progress */ 45 KMC_BIT_DESTROY = 17, /* Destroy in progress */ 46 KMC_BIT_TOTAL = 18, /* Proc handler helper bit */ 47 KMC_BIT_ALLOC = 19, /* Proc handler helper bit */ 48 KMC_BIT_MAX = 20, /* Proc handler helper bit */ 49 KMC_BIT_RECLAIMABLE = 21, /* Can be freed by shrinker */ 50 } kmc_bit_t; 51 52 /* kmem move callback return values */ 53 typedef enum kmem_cbrc { 54 KMEM_CBRC_YES = 0, /* Object moved */ 55 KMEM_CBRC_NO = 1, /* Object not moved */ 56 KMEM_CBRC_LATER = 2, /* Object not moved, try again later */ 57 KMEM_CBRC_DONT_NEED = 3, /* Neither object is needed */ 58 KMEM_CBRC_DONT_KNOW = 4, /* Object unknown */ 59 } kmem_cbrc_t; 60 61 #define KMC_NODEBUG (1 << KMC_BIT_NODEBUG) 62 #define KMC_KVMEM (1 << KMC_BIT_KVMEM) 63 #define KMC_SLAB (1 << KMC_BIT_SLAB) 64 #define KMC_DEADLOCKED (1 << KMC_BIT_DEADLOCKED) 65 #define KMC_GROWING (1 << KMC_BIT_GROWING) 66 #define KMC_REAPING (1 << KMC_BIT_REAPING) 67 #define KMC_DESTROY (1 << KMC_BIT_DESTROY) 68 #define KMC_TOTAL (1 << KMC_BIT_TOTAL) 69 #define KMC_ALLOC (1 << KMC_BIT_ALLOC) 70 #define KMC_MAX (1 << KMC_BIT_MAX) 71 #define KMC_RECLAIMABLE (1 << KMC_BIT_RECLAIMABLE) 72 73 extern struct list_head spl_kmem_cache_list; 74 extern struct rw_semaphore spl_kmem_cache_sem; 75 76 #define SKM_MAGIC 0x2e2e2e2e 77 #define SKO_MAGIC 0x20202020 78 #define SKS_MAGIC 0x22222222 79 #define SKC_MAGIC 0x2c2c2c2c 80 81 #define SPL_KMEM_CACHE_OBJ_PER_SLAB 8 /* Target objects per slab */ 82 #define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */ 83 #ifdef _LP64 84 #define SPL_KMEM_CACHE_MAX_SIZE 32 /* Max slab size in MB */ 85 #else 86 #define SPL_KMEM_CACHE_MAX_SIZE 4 /* Max slab size in MB */ 87 #endif 88 89 #define SPL_MAX_ORDER (MAX_ORDER - 3) 90 #define SPL_MAX_ORDER_NR_PAGES (1 << (SPL_MAX_ORDER - 1)) 91 92 #ifdef CONFIG_SLUB 93 #define SPL_MAX_KMEM_CACHE_ORDER PAGE_ALLOC_COSTLY_ORDER 94 #define SPL_MAX_KMEM_ORDER_NR_PAGES (1 << (SPL_MAX_KMEM_CACHE_ORDER - 1)) 95 #else 96 #define SPL_MAX_KMEM_ORDER_NR_PAGES (KMALLOC_MAX_SIZE >> PAGE_SHIFT) 97 #endif 98 99 typedef int (*spl_kmem_ctor_t)(void *, void *, int); 100 typedef void (*spl_kmem_dtor_t)(void *, void *); 101 102 typedef struct spl_kmem_magazine { 103 uint32_t skm_magic; /* Sanity magic */ 104 uint32_t skm_avail; /* Available objects */ 105 uint32_t skm_size; /* Magazine size */ 106 uint32_t skm_refill; /* Batch refill size */ 107 struct spl_kmem_cache *skm_cache; /* Owned by cache */ 108 unsigned int skm_cpu; /* Owned by cpu */ 109 void *skm_objs[]; /* Object pointers */ 110 } spl_kmem_magazine_t; 111 112 typedef struct spl_kmem_obj { 113 uint32_t sko_magic; /* Sanity magic */ 114 void *sko_addr; /* Buffer address */ 115 struct spl_kmem_slab *sko_slab; /* Owned by slab */ 116 struct list_head sko_list; /* Free object list linkage */ 117 } spl_kmem_obj_t; 118 119 typedef struct spl_kmem_slab { 120 uint32_t sks_magic; /* Sanity magic */ 121 uint32_t sks_objs; /* Objects per slab */ 122 struct spl_kmem_cache *sks_cache; /* Owned by cache */ 123 struct list_head sks_list; /* Slab list linkage */ 124 struct list_head sks_free_list; /* Free object list */ 125 unsigned long sks_age; /* Last modify jiffie */ 126 uint32_t sks_ref; /* Ref count used objects */ 127 } spl_kmem_slab_t; 128 129 typedef struct spl_kmem_alloc { 130 struct spl_kmem_cache *ska_cache; /* Owned by cache */ 131 int ska_flags; /* Allocation flags */ 132 taskq_ent_t ska_tqe; /* Task queue entry */ 133 } spl_kmem_alloc_t; 134 135 typedef struct spl_kmem_emergency { 136 struct rb_node ske_node; /* Emergency tree linkage */ 137 unsigned long ske_obj; /* Buffer address */ 138 } spl_kmem_emergency_t; 139 140 typedef struct spl_kmem_cache { 141 uint32_t skc_magic; /* Sanity magic */ 142 uint32_t skc_name_size; /* Name length */ 143 char *skc_name; /* Name string */ 144 spl_kmem_magazine_t **skc_mag; /* Per-CPU warm cache */ 145 uint32_t skc_mag_size; /* Magazine size */ 146 uint32_t skc_mag_refill; /* Magazine refill count */ 147 spl_kmem_ctor_t skc_ctor; /* Constructor */ 148 spl_kmem_dtor_t skc_dtor; /* Destructor */ 149 void *skc_private; /* Private data */ 150 void *skc_vmp; /* Unused */ 151 struct kmem_cache *skc_linux_cache; /* Linux slab cache if used */ 152 unsigned long skc_flags; /* Flags */ 153 uint32_t skc_obj_size; /* Object size */ 154 uint32_t skc_obj_align; /* Object alignment */ 155 uint32_t skc_slab_objs; /* Objects per slab */ 156 uint32_t skc_slab_size; /* Slab size */ 157 atomic_t skc_ref; /* Ref count callers */ 158 taskqid_t skc_taskqid; /* Slab reclaim task */ 159 struct list_head skc_list; /* List of caches linkage */ 160 struct list_head skc_complete_list; /* Completely alloc'ed */ 161 struct list_head skc_partial_list; /* Partially alloc'ed */ 162 struct rb_root skc_emergency_tree; /* Min sized objects */ 163 spinlock_t skc_lock; /* Cache lock */ 164 wait_queue_head_t skc_waitq; /* Allocation waiters */ 165 uint64_t skc_slab_fail; /* Slab alloc failures */ 166 uint64_t skc_slab_create; /* Slab creates */ 167 uint64_t skc_slab_destroy; /* Slab destroys */ 168 uint64_t skc_slab_total; /* Slab total current */ 169 uint64_t skc_slab_alloc; /* Slab alloc current */ 170 uint64_t skc_slab_max; /* Slab max historic */ 171 uint64_t skc_obj_total; /* Obj total current */ 172 uint64_t skc_obj_alloc; /* Obj alloc current */ 173 struct percpu_counter skc_linux_alloc; /* Linux-backed Obj alloc */ 174 uint64_t skc_obj_max; /* Obj max historic */ 175 uint64_t skc_obj_deadlock; /* Obj emergency deadlocks */ 176 uint64_t skc_obj_emergency; /* Obj emergency current */ 177 uint64_t skc_obj_emergency_max; /* Obj emergency max */ 178 } spl_kmem_cache_t; 179 #define kmem_cache_t spl_kmem_cache_t 180 181 extern spl_kmem_cache_t *spl_kmem_cache_create(const char *name, size_t size, 182 size_t align, spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, 183 void *reclaim, void *priv, void *vmp, int flags); 184 extern void spl_kmem_cache_set_move(spl_kmem_cache_t *, 185 kmem_cbrc_t (*)(void *, void *, size_t, void *)); 186 extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc); 187 extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags); 188 extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj); 189 extern void spl_kmem_cache_set_allocflags(spl_kmem_cache_t *skc, gfp_t flags); 190 extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc); 191 extern void spl_kmem_reap(void); 192 extern uint64_t spl_kmem_cache_inuse(kmem_cache_t *cache); 193 extern uint64_t spl_kmem_cache_entry_size(kmem_cache_t *cache); 194 195 #ifndef SPL_KMEM_CACHE_IMPLEMENTING 196 /* 197 * Macros for the kmem_cache_* API expected by ZFS and SPL clients. We don't 198 * define them inside spl-kmem-cache.c, as that uses the kernel's incompatible 199 * kmem_cache_* facilities to implement ours. 200 */ 201 202 /* Avoid conflicts with kernel names that might be implemented as macros. */ 203 #undef kmem_cache_alloc 204 #undef kmem_cache_create 205 206 #define kmem_cache_create(name, size, align, ctor, dtor, rclm, priv, vmp, fl) \ 207 spl_kmem_cache_create(name, size, align, ctor, dtor, rclm, priv, vmp, fl) 208 #define kmem_cache_set_move(skc, move) spl_kmem_cache_set_move(skc, move) 209 #define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc) 210 #define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags) 211 #define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj) 212 #define kmem_cache_reap_now(skc) spl_kmem_cache_reap_now(skc) 213 #define kmem_reap() spl_kmem_reap() 214 #endif 215 216 /* 217 * The following functions are only available for internal use. 218 */ 219 extern int spl_kmem_cache_init(void); 220 extern void spl_kmem_cache_fini(void); 221 222 #endif /* _SPL_KMEM_CACHE_H */ 223