1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2021 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * Copyright (c) 2024-2025 The FreeBSD Foundation 8 * 9 * Portions of this software were developed by Björn Zeeb 10 * under sponsorship from the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice unmodified, this list of conditions, and the following 17 * disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 #ifndef _LINUXKPI_LINUX_SLAB_H_ 34 #define _LINUXKPI_LINUX_SLAB_H_ 35 36 #include <sys/types.h> 37 #include <sys/malloc.h> 38 #include <sys/limits.h> 39 40 #include <linux/compat.h> 41 #include <linux/types.h> 42 #include <linux/gfp.h> 43 #include <linux/llist.h> 44 #include <linux/overflow.h> 45 46 MALLOC_DECLARE(M_KMALLOC); 47 48 #define kvzalloc(size, flags) kvmalloc(size, (flags) | __GFP_ZERO) 49 #define kvcalloc(n, size, flags) kvmalloc_array(n, size, (flags) | __GFP_ZERO) 50 #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) 51 #define kzalloc_node(size, flags, node) kmalloc_node(size, (flags) | __GFP_ZERO, node) 52 #define kfree_const(ptr) kfree(ptr) 53 #define kfree_async(ptr) kfree(ptr) /* drm-kmod 5.4 compat */ 54 #define vzalloc(size) __vmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0) 55 #define vfree(arg) kfree(arg) 56 #define kvfree(arg) kfree(arg) 57 #define vmalloc_node(size, node) __vmalloc_node(size, GFP_KERNEL, node) 58 #define vmalloc_user(size) __vmalloc(size, GFP_KERNEL | __GFP_ZERO, 0) 59 #define vmalloc(size) __vmalloc(size, GFP_KERNEL, 0) 60 61 /* 62 * Prefix some functions with linux_ to avoid namespace conflict 63 * with the OpenSolaris code in the kernel. 64 */ 65 #define kmem_cache linux_kmem_cache 66 #define kmem_cache_create(...) linux_kmem_cache_create(__VA_ARGS__) 67 #define kmem_cache_alloc(...) lkpi_kmem_cache_alloc(__VA_ARGS__) 68 #define kmem_cache_zalloc(...) lkpi_kmem_cache_zalloc(__VA_ARGS__) 69 #define kmem_cache_free(...) lkpi_kmem_cache_free(__VA_ARGS__) 70 #define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__) 71 #define kmem_cache_shrink(x) (0) 72 73 #define KMEM_CACHE(__struct, flags) \ 74 linux_kmem_cache_create(#__struct, sizeof(struct __struct), \ 75 __alignof(struct __struct), (flags), NULL) 76 77 typedef void linux_kmem_ctor_t (void *); 78 79 struct linux_kmem_cache; 80 81 #define SLAB_HWCACHE_ALIGN (1 << 0) 82 #define SLAB_TYPESAFE_BY_RCU (1 << 1) 83 #define SLAB_RECLAIM_ACCOUNT (1 << 2) 84 85 #define SLAB_DESTROY_BY_RCU \ 86 SLAB_TYPESAFE_BY_RCU 87 88 #define ARCH_KMALLOC_MINALIGN \ 89 __alignof(unsigned long long) 90 91 #define ZERO_SIZE_PTR ((void *)16) 92 #define ZERO_OR_NULL_PTR(x) ((x) == NULL || (x) == ZERO_SIZE_PTR) 93 94 struct linux_kmem_cache *linux_kmem_cache_create(const char *name, 95 size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor); 96 void *lkpi_kmem_cache_alloc(struct linux_kmem_cache *, gfp_t); 97 void *lkpi_kmem_cache_zalloc(struct linux_kmem_cache *, gfp_t); 98 void lkpi_kmem_cache_free(struct linux_kmem_cache *, void *); 99 void linux_kmem_cache_destroy(struct linux_kmem_cache *); 100 101 void *lkpi_kmalloc(size_t, gfp_t); 102 void *lkpi_kvmalloc(size_t, gfp_t); 103 void *lkpi___kmalloc(size_t, gfp_t); 104 void *lkpi___kmalloc_node(size_t, gfp_t, int); 105 void *lkpi_krealloc(void *, size_t, gfp_t); 106 void lkpi_kfree(const void *); 107 108 static inline gfp_t 109 linux_check_m_flags(gfp_t flags) 110 { 111 const gfp_t m = M_NOWAIT | M_WAITOK; 112 113 /* make sure either M_NOWAIT or M_WAITOK is set */ 114 if ((flags & m) == 0) 115 flags |= M_NOWAIT; 116 else if ((flags & m) == m) 117 flags &= ~M_WAITOK; 118 119 /* mask away LinuxKPI specific flags */ 120 return (flags & GFP_NATIVE_MASK); 121 } 122 123 /* 124 * Base functions with a native implementation. 125 */ 126 static inline void * 127 kmalloc(size_t size, gfp_t flags) 128 { 129 return (lkpi_kmalloc(size, flags)); 130 } 131 132 static inline void * 133 __kmalloc(size_t size, gfp_t flags) 134 { 135 return (lkpi___kmalloc(size, flags)); 136 } 137 138 static inline void * 139 kmalloc_node(size_t size, gfp_t flags, int node) 140 { 141 return (lkpi___kmalloc_node(size, flags, node)); 142 } 143 144 static inline void * 145 krealloc(void *ptr, size_t size, gfp_t flags) 146 { 147 return (lkpi_krealloc(ptr, size, flags)); 148 } 149 150 static inline void 151 kfree(const void *ptr) 152 { 153 lkpi_kfree(ptr); 154 } 155 156 /* 157 * Other k*alloc() funtions using the above as underlying allocator. 158 */ 159 /* kmalloc */ 160 static inline void * 161 kmalloc_array(size_t n, size_t size, gfp_t flags) 162 { 163 if (WOULD_OVERFLOW(n, size)) 164 panic("%s: %zu * %zu overflowed", __func__, n, size); 165 166 return (kmalloc(size * n, flags)); 167 } 168 169 static inline void * 170 kcalloc(size_t n, size_t size, gfp_t flags) 171 { 172 flags |= __GFP_ZERO; 173 return (kmalloc_array(n, size, flags)); 174 } 175 176 /* kmalloc_node */ 177 static inline void * 178 kmalloc_array_node(size_t n, size_t size, gfp_t flags, int node) 179 { 180 if (WOULD_OVERFLOW(n, size)) 181 panic("%s: %zu * %zu overflowed", __func__, n, size); 182 183 return (kmalloc_node(size * n, flags, node)); 184 } 185 186 static inline void * 187 kcalloc_node(size_t n, size_t size, gfp_t flags, int node) 188 { 189 flags |= __GFP_ZERO; 190 return (kmalloc_array_node(n, size, flags, node)); 191 } 192 193 /* krealloc */ 194 static inline void * 195 krealloc_array(void *ptr, size_t n, size_t size, gfp_t flags) 196 { 197 if (WOULD_OVERFLOW(n, size)) 198 return NULL; 199 200 return (krealloc(ptr, n * size, flags)); 201 } 202 203 /* 204 * vmalloc/kvalloc functions. 205 */ 206 static inline void * 207 __vmalloc(size_t size, gfp_t flags, int other) 208 { 209 return (malloc(size, M_KMALLOC, linux_check_m_flags(flags))); 210 } 211 212 static inline void * 213 __vmalloc_node(size_t size, gfp_t flags, int node) 214 { 215 return (malloc_domainset(size, M_KMALLOC, 216 linux_get_vm_domain_set(node), linux_check_m_flags(flags))); 217 } 218 219 static inline void * 220 vmalloc_32(size_t size) 221 { 222 return (contigmalloc(size, M_KMALLOC, M_WAITOK, 0, UINT_MAX, 1, 1)); 223 } 224 225 /* May return non-contiguous memory. */ 226 static inline void * 227 kvmalloc(size_t size, gfp_t flags) 228 { 229 return (lkpi_kvmalloc(size, flags)); 230 } 231 232 static inline void * 233 kvmalloc_array(size_t n, size_t size, gfp_t flags) 234 { 235 if (WOULD_OVERFLOW(n, size)) 236 panic("%s: %zu * %zu overflowed", __func__, n, size); 237 238 return (kvmalloc(size * n, flags)); 239 } 240 241 static inline void * 242 kvrealloc(const void *ptr, size_t oldsize, size_t newsize, gfp_t flags) 243 { 244 void *newptr; 245 246 if (newsize <= oldsize) 247 return (__DECONST(void *, ptr)); 248 249 newptr = kvmalloc(newsize, flags); 250 if (newptr != NULL) { 251 memcpy(newptr, ptr, oldsize); 252 kvfree(ptr); 253 } 254 255 return (newptr); 256 } 257 258 /* 259 * Misc. 260 */ 261 262 static __inline void 263 kfree_sensitive(const void *ptr) 264 { 265 if (ZERO_OR_NULL_PTR(ptr)) 266 return; 267 268 zfree(__DECONST(void *, ptr), M_KMALLOC); 269 } 270 271 static inline size_t 272 ksize(const void *ptr) 273 { 274 return (malloc_usable_size(ptr)); 275 } 276 277 static inline size_t 278 kmalloc_size_roundup(size_t size) 279 { 280 if (unlikely(size == 0 || size == SIZE_MAX)) 281 return (size); 282 return (malloc_size(size)); 283 } 284 285 #endif /* _LINUXKPI_LINUX_SLAB_H_ */ 286