1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
4 * Copyright (C) 2007 The Regents of the University of California.
5 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
6 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
7 * UCRL-CODE-235197
8 *
9 * This file is part of the SPL, Solaris Porting Layer.
10 *
11 * The SPL is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by the
13 * Free Software Foundation; either version 2 of the License, or (at your
14 * option) any later version.
15 *
16 * The SPL is distributed in the hope that it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 * for more details.
20 *
21 * You should have received a copy of the GNU General Public License along
22 * with the SPL. If not, see <http://www.gnu.org/licenses/>.
23 */
24
25 #ifndef _SPL_KMEM_H
26 #define _SPL_KMEM_H
27
28 #include <sys/debug.h>
29 #include <linux/slab.h>
30 #include <linux/sched.h>
31 #include <linux/mm.h>
32 #include <linux/vmalloc.h>
33
34 extern int kmem_debugging(void);
35 __attribute__((format(printf, 1, 0)))
36 extern char *kmem_vasprintf(const char *fmt, va_list ap);
37 __attribute__((format(printf, 1, 2)))
38 extern char *kmem_asprintf(const char *fmt, ...);
39 extern char *kmem_strdup(const char *str);
40 extern void kmem_strfree(char *str);
41
42 #define kmem_scnprintf scnprintf
43
44 #define POINTER_IS_VALID(p) (!((uintptr_t)(p) & 0x3))
45 #define POINTER_INVALIDATE(pp) (*(pp) = (void *)((uintptr_t)(*(pp)) | 0x1))
46
47 /*
48 * Memory allocation interfaces
49 */
50 #define KM_SLEEP 0x0000 /* can block for memory; success guaranteed */
51 #define KM_NOSLEEP 0x0001 /* cannot block for memory; may fail */
52 #define KM_PUSHPAGE 0x0004 /* can block for memory; may use reserve */
53 #define KM_ZERO 0x1000 /* zero the allocation */
54 #define KM_VMEM 0x2000 /* caller is vmem_* wrapper */
55
56 #define KM_PUBLIC_MASK (KM_SLEEP | KM_NOSLEEP | KM_PUSHPAGE)
57
58 static int spl_fstrans_check(void);
59 void *spl_kvmalloc(size_t size, gfp_t flags);
60
61 /*
62 * Convert a KM_* flags mask to its Linux GFP_* counterpart. The conversion
63 * function is context aware which means that KM_SLEEP allocations can be
64 * safely used in syncing contexts which have set PF_FSTRANS.
65 */
66 static inline gfp_t
kmem_flags_convert(int flags)67 kmem_flags_convert(int flags)
68 {
69 gfp_t lflags = __GFP_NOWARN | __GFP_COMP;
70
71 if (flags & KM_NOSLEEP) {
72 lflags |= GFP_ATOMIC | __GFP_NORETRY;
73 } else {
74 lflags |= GFP_KERNEL;
75 if (spl_fstrans_check())
76 lflags &= ~(__GFP_IO|__GFP_FS);
77 }
78
79 if (flags & KM_PUSHPAGE)
80 lflags |= __GFP_HIGH;
81
82 if (flags & KM_ZERO)
83 lflags |= __GFP_ZERO;
84
85 return (lflags);
86 }
87
88 typedef struct {
89 struct task_struct *fstrans_thread;
90 unsigned int saved_flags;
91 } fstrans_cookie_t;
92
93 /*
94 * Introduced in Linux 3.9, however this cannot be solely relied on before
95 * Linux 3.18 as it doesn't turn off __GFP_FS as it should.
96 */
97 #ifdef PF_MEMALLOC_NOIO
98 #define __SPL_PF_MEMALLOC_NOIO (PF_MEMALLOC_NOIO)
99 #else
100 #define __SPL_PF_MEMALLOC_NOIO (0)
101 #endif
102
103 /*
104 * PF_FSTRANS is removed from Linux 4.12
105 */
106 #ifdef PF_FSTRANS
107 #define __SPL_PF_FSTRANS (PF_FSTRANS)
108 #else
109 #define __SPL_PF_FSTRANS (0)
110 #endif
111
112 #define SPL_FSTRANS (__SPL_PF_FSTRANS|__SPL_PF_MEMALLOC_NOIO)
113
114 static inline fstrans_cookie_t
spl_fstrans_mark(void)115 spl_fstrans_mark(void)
116 {
117 fstrans_cookie_t cookie;
118
119 BUILD_BUG_ON(SPL_FSTRANS == 0);
120
121 cookie.fstrans_thread = current;
122 cookie.saved_flags = current->flags & SPL_FSTRANS;
123 current->flags |= SPL_FSTRANS;
124
125 return (cookie);
126 }
127
128 static inline void
spl_fstrans_unmark(fstrans_cookie_t cookie)129 spl_fstrans_unmark(fstrans_cookie_t cookie)
130 {
131 ASSERT3P(cookie.fstrans_thread, ==, current);
132 ASSERT((current->flags & SPL_FSTRANS) == SPL_FSTRANS);
133
134 current->flags &= ~SPL_FSTRANS;
135 current->flags |= cookie.saved_flags;
136 }
137
138 static inline int
spl_fstrans_check(void)139 spl_fstrans_check(void)
140 {
141 return (current->flags & SPL_FSTRANS);
142 }
143
144 /*
145 * specifically used to check PF_FSTRANS flag, cannot be relied on for
146 * checking spl_fstrans_mark().
147 */
148 static inline int
__spl_pf_fstrans_check(void)149 __spl_pf_fstrans_check(void)
150 {
151 return (current->flags & __SPL_PF_FSTRANS);
152 }
153
154 /*
155 * Kernel compatibility for GFP flags
156 */
157 /* < 4.13 */
158 #ifndef __GFP_RETRY_MAYFAIL
159 #define __GFP_RETRY_MAYFAIL __GFP_REPEAT
160 #endif
161 /* < 4.4 */
162 #ifndef __GFP_RECLAIM
163 #define __GFP_RECLAIM __GFP_WAIT
164 #endif
165
166 #ifdef HAVE_ATOMIC64_T
167 #define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used)
168 #define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used)
169 #define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used)
170 #define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size)
171 extern atomic64_t kmem_alloc_used;
172 extern unsigned long long kmem_alloc_max;
173 #else /* HAVE_ATOMIC64_T */
174 #define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used)
175 #define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used)
176 #define kmem_alloc_used_read() atomic_read(&kmem_alloc_used)
177 #define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size)
178 extern atomic_t kmem_alloc_used;
179 extern unsigned long long kmem_alloc_max;
180 #endif /* HAVE_ATOMIC64_T */
181
182 extern unsigned int spl_kmem_alloc_warn;
183 extern unsigned int spl_kmem_alloc_max;
184
185 #define kmem_alloc(sz, fl) spl_kmem_alloc((sz), (fl), __func__, __LINE__)
186 #define kmem_zalloc(sz, fl) spl_kmem_zalloc((sz), (fl), __func__, __LINE__)
187 #define kmem_free(ptr, sz) spl_kmem_free((ptr), (sz))
188 #define kmem_cache_reap_active spl_kmem_cache_reap_active
189
190 __attribute__((malloc, alloc_size(1)))
191 extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line);
192 __attribute__((malloc, alloc_size(1)))
193 extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line);
194 extern void spl_kmem_free(const void *ptr, size_t sz);
195
196 /*
197 * 5.8 API change, pgprot_t argument removed.
198 */
199 #ifdef HAVE_VMALLOC_PAGE_KERNEL
200 #define spl_vmalloc(size, flags) __vmalloc(size, flags, PAGE_KERNEL)
201 #else
202 #define spl_vmalloc(size, flags) __vmalloc(size, flags)
203 #endif
204
205 /*
206 * The following functions are only available for internal use.
207 */
208 extern void *spl_kmem_alloc_impl(size_t size, int flags, int node);
209 extern void *spl_kmem_alloc_debug(size_t size, int flags, int node);
210 extern void *spl_kmem_alloc_track(size_t size, int flags,
211 const char *func, int line, int node);
212 extern void spl_kmem_free_impl(const void *buf, size_t size);
213 extern void spl_kmem_free_debug(const void *buf, size_t size);
214 extern void spl_kmem_free_track(const void *buf, size_t size);
215
216 extern int spl_kmem_init(void);
217 extern void spl_kmem_fini(void);
218 extern int spl_kmem_cache_reap_active(void);
219
220 #endif /* _SPL_KMEM_H */
221