1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KMSAN compiler API.
4 *
5 * This file implements __msan_XXX hooks that Clang inserts into the code
6 * compiled with -fsanitize=kernel-memory.
7 * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN
8 * instrumentation works.
9 *
10 * Copyright (C) 2017-2022 Google LLC
11 * Author: Alexander Potapenko <glider@google.com>
12 *
13 */
14
15 #include "kmsan.h"
16 #include <linux/gfp.h>
17 #include <linux/kmsan.h>
18 #include <linux/kmsan_string.h>
19 #include <linux/mm.h>
20 #include <linux/uaccess.h>
21
is_bad_asm_addr(void * addr,uintptr_t size,bool is_store)22 static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store)
23 {
24 if (IS_ENABLED(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) &&
25 (u64)addr < TASK_SIZE)
26 return true;
27 if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW))
28 return true;
29 return false;
30 }
31
32 static inline struct shadow_origin_ptr
get_shadow_origin_ptr(void * addr,u64 size,bool store)33 get_shadow_origin_ptr(void *addr, u64 size, bool store)
34 {
35 unsigned long ua_flags = user_access_save();
36 struct shadow_origin_ptr ret;
37
38 ret = kmsan_get_shadow_origin_ptr(addr, size, store);
39 user_access_restore(ua_flags);
40 return ret;
41 }
42
43 /*
44 * KMSAN instrumentation functions follow. They are not declared elsewhere in
45 * the kernel code, so they are preceded by prototypes, to silence
46 * -Wmissing-prototypes warnings.
47 */
48
49 /* Get shadow and origin pointers for a memory load with non-standard size. */
50 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
51 uintptr_t size);
__msan_metadata_ptr_for_load_n(void * addr,uintptr_t size)52 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
53 uintptr_t size)
54 {
55 return get_shadow_origin_ptr(addr, size, /*store*/ false);
56 }
57 EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);
58
59 /* Get shadow and origin pointers for a memory store with non-standard size. */
60 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
61 uintptr_t size);
__msan_metadata_ptr_for_store_n(void * addr,uintptr_t size)62 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
63 uintptr_t size)
64 {
65 return get_shadow_origin_ptr(addr, size, /*store*/ true);
66 }
67 EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);
68
69 /*
70 * Declare functions that obtain shadow/origin pointers for loads and stores
71 * with fixed size.
72 */
73 #define DECLARE_METADATA_PTR_GETTER(size) \
74 struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \
75 void *addr); \
76 struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \
77 void *addr) \
78 { \
79 return get_shadow_origin_ptr(addr, size, /*store*/ false); \
80 } \
81 EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size); \
82 struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \
83 void *addr); \
84 struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \
85 void *addr) \
86 { \
87 return get_shadow_origin_ptr(addr, size, /*store*/ true); \
88 } \
89 EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)
90
91 DECLARE_METADATA_PTR_GETTER(1);
92 DECLARE_METADATA_PTR_GETTER(2);
93 DECLARE_METADATA_PTR_GETTER(4);
94 DECLARE_METADATA_PTR_GETTER(8);
95
96 /*
97 * Handle a memory store performed by inline assembly. KMSAN conservatively
98 * attempts to unpoison the outputs of asm() directives to prevent false
99 * positives caused by missed stores.
100 *
101 * __msan_instrument_asm_store() may be called for inline assembly code when
102 * entering or leaving IRQ. We omit the check for kmsan_in_runtime() to ensure
103 * the memory written to in these cases is also marked as initialized.
104 */
105 void __msan_instrument_asm_store(void *addr, uintptr_t size);
__msan_instrument_asm_store(void * addr,uintptr_t size)106 void __msan_instrument_asm_store(void *addr, uintptr_t size)
107 {
108 unsigned long ua_flags;
109
110 if (!kmsan_enabled)
111 return;
112
113 ua_flags = user_access_save();
114 /*
115 * Most of the accesses are below 32 bytes. The exceptions so far are
116 * clwb() (64 bytes), FPU state (512 bytes) and chsc() (4096 bytes).
117 */
118 if (size > 4096) {
119 WARN_ONCE(1, "assembly store size too big: %ld\n", size);
120 size = 8;
121 }
122 if (is_bad_asm_addr(addr, size, /*is_store*/ true)) {
123 user_access_restore(ua_flags);
124 return;
125 }
126 /* Unpoisoning the memory on best effort. */
127 kmsan_internal_unpoison_memory(addr, size, /*checked*/ false);
128 user_access_restore(ua_flags);
129 }
130 EXPORT_SYMBOL(__msan_instrument_asm_store);
131
132 /*
133 * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset
134 * intrinsics with calls to respective __msan_ functions. We use
135 * get_param0_metadata() and set_retval_metadata() to store the shadow/origin
136 * values for the destination argument of these functions and use them for the
137 * functions' return values.
138 */
get_param0_metadata(u64 * shadow,depot_stack_handle_t * origin)139 static inline void get_param0_metadata(u64 *shadow,
140 depot_stack_handle_t *origin)
141 {
142 struct kmsan_ctx *ctx = kmsan_get_context();
143
144 *shadow = *(u64 *)(ctx->cstate.param_tls);
145 *origin = ctx->cstate.param_origin_tls[0];
146 }
147
set_retval_metadata(u64 shadow,depot_stack_handle_t origin)148 static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin)
149 {
150 struct kmsan_ctx *ctx = kmsan_get_context();
151
152 *(u64 *)(ctx->cstate.retval_tls) = shadow;
153 ctx->cstate.retval_origin_tls = origin;
154 }
155
156 /* Handle llvm.memmove intrinsic. */
157 void *__msan_memmove(void *dst, const void *src, uintptr_t n);
__msan_memmove(void * dst,const void * src,uintptr_t n)158 void *__msan_memmove(void *dst, const void *src, uintptr_t n)
159 {
160 depot_stack_handle_t origin;
161 void *result;
162 u64 shadow;
163
164 get_param0_metadata(&shadow, &origin);
165 result = __memmove(dst, src, n);
166 if (!n)
167 /* Some people call memmove() with zero length. */
168 return result;
169 if (!kmsan_enabled || kmsan_in_runtime())
170 return result;
171
172 kmsan_enter_runtime();
173 kmsan_internal_memmove_metadata(dst, (void *)src, n);
174 kmsan_leave_runtime();
175
176 set_retval_metadata(shadow, origin);
177 return result;
178 }
179 EXPORT_SYMBOL(__msan_memmove);
180
181 /* Handle llvm.memcpy intrinsic. */
182 void *__msan_memcpy(void *dst, const void *src, uintptr_t n);
__msan_memcpy(void * dst,const void * src,uintptr_t n)183 void *__msan_memcpy(void *dst, const void *src, uintptr_t n)
184 {
185 depot_stack_handle_t origin;
186 void *result;
187 u64 shadow;
188
189 get_param0_metadata(&shadow, &origin);
190 result = __memcpy(dst, src, n);
191 if (!n)
192 /* Some people call memcpy() with zero length. */
193 return result;
194
195 if (!kmsan_enabled || kmsan_in_runtime())
196 return result;
197
198 kmsan_enter_runtime();
199 /* Using memmove instead of memcpy doesn't affect correctness. */
200 kmsan_internal_memmove_metadata(dst, (void *)src, n);
201 kmsan_leave_runtime();
202
203 set_retval_metadata(shadow, origin);
204 return result;
205 }
206 EXPORT_SYMBOL(__msan_memcpy);
207
208 /* Handle llvm.memset intrinsic. */
209 void *__msan_memset(void *dst, int c, uintptr_t n);
__msan_memset(void * dst,int c,uintptr_t n)210 void *__msan_memset(void *dst, int c, uintptr_t n)
211 {
212 depot_stack_handle_t origin;
213 void *result;
214 u64 shadow;
215
216 get_param0_metadata(&shadow, &origin);
217 result = __memset(dst, c, n);
218 if (!kmsan_enabled || kmsan_in_runtime())
219 return result;
220
221 kmsan_enter_runtime();
222 /*
223 * Clang doesn't pass parameter metadata here, so it is impossible to
224 * use shadow of @c to set up the shadow for @dst.
225 */
226 kmsan_internal_unpoison_memory(dst, n, /*checked*/ false);
227 kmsan_leave_runtime();
228
229 set_retval_metadata(shadow, origin);
230 return result;
231 }
232 EXPORT_SYMBOL(__msan_memset);
233
234 /*
235 * Create a new origin from an old one. This is done when storing an
236 * uninitialized value to memory. When reporting an error, KMSAN unrolls and
237 * prints the whole chain of stores that preceded the use of this value.
238 */
239 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin);
__msan_chain_origin(depot_stack_handle_t origin)240 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)
241 {
242 depot_stack_handle_t ret = 0;
243 unsigned long ua_flags;
244
245 if (!kmsan_enabled || kmsan_in_runtime())
246 return ret;
247
248 ua_flags = user_access_save();
249
250 /* Creating new origins may allocate memory. */
251 kmsan_enter_runtime();
252 ret = kmsan_internal_chain_origin(origin);
253 kmsan_leave_runtime();
254 user_access_restore(ua_flags);
255 return ret;
256 }
257 EXPORT_SYMBOL(__msan_chain_origin);
258
259 /* Poison a local variable when entering a function. */
260 void __msan_poison_alloca(void *address, uintptr_t size, char *descr);
__msan_poison_alloca(void * address,uintptr_t size,char * descr)261 void __msan_poison_alloca(void *address, uintptr_t size, char *descr)
262 {
263 depot_stack_handle_t handle;
264 unsigned long entries[4];
265 unsigned long ua_flags;
266
267 if (!kmsan_enabled || kmsan_in_runtime())
268 return;
269
270 ua_flags = user_access_save();
271 entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;
272 entries[1] = (u64)descr;
273 entries[2] = (u64)__builtin_return_address(0);
274 /*
275 * With frame pointers enabled, it is possible to quickly fetch the
276 * second frame of the caller stack without calling the unwinder.
277 * Without them, simply do not bother.
278 */
279 if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER))
280 entries[3] = (u64)__builtin_return_address(1);
281 else
282 entries[3] = 0;
283
284 /* stack_depot_save() may allocate memory. */
285 kmsan_enter_runtime();
286 handle = stack_depot_save(entries, ARRAY_SIZE(entries), __GFP_HIGH);
287 kmsan_leave_runtime();
288
289 kmsan_internal_set_shadow_origin(address, size, -1, handle,
290 /*checked*/ true);
291 user_access_restore(ua_flags);
292 }
293 EXPORT_SYMBOL(__msan_poison_alloca);
294
295 /* Unpoison a local variable. */
296 void __msan_unpoison_alloca(void *address, uintptr_t size);
__msan_unpoison_alloca(void * address,uintptr_t size)297 void __msan_unpoison_alloca(void *address, uintptr_t size)
298 {
299 if (!kmsan_enabled || kmsan_in_runtime())
300 return;
301
302 kmsan_enter_runtime();
303 kmsan_internal_unpoison_memory(address, size, /*checked*/ true);
304 kmsan_leave_runtime();
305 }
306 EXPORT_SYMBOL(__msan_unpoison_alloca);
307
308 /*
309 * Report that an uninitialized value with the given origin was used in a way
310 * that constituted undefined behavior.
311 */
312 void __msan_warning(u32 origin);
__msan_warning(u32 origin)313 void __msan_warning(u32 origin)
314 {
315 if (!kmsan_enabled || kmsan_in_runtime())
316 return;
317 kmsan_enter_runtime();
318 kmsan_report(origin, /*address*/ NULL, /*size*/ 0,
319 /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ NULL,
320 REASON_ANY);
321 kmsan_leave_runtime();
322 }
323 EXPORT_SYMBOL(__msan_warning);
324
325 /*
326 * At the beginning of an instrumented function, obtain the pointer to
327 * `struct kmsan_context_state` holding the metadata for function parameters.
328 */
329 struct kmsan_context_state *__msan_get_context_state(void);
__msan_get_context_state(void)330 struct kmsan_context_state *__msan_get_context_state(void)
331 {
332 return &kmsan_get_context()->cstate;
333 }
334 EXPORT_SYMBOL(__msan_get_context_state);
335