1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 * Copyright (c) 2021 The FreeBSD Foundation
7 *
8 * Portions of this software were developed by Mark Johnston under sponsorship
9 * from the FreeBSD Foundation.
10 *
11 * This code is part of the KMSAN subsystem of the NetBSD kernel.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #define SAN_RUNTIME
36
37 #include <sys/cdefs.h>
38 #if 0
39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
40 #endif
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/bio.h>
45 #include <sys/buf.h>
46 #include <sys/conf.h>
47 #include <sys/kdb.h>
48 #include <sys/kernel.h>
49 #include <sys/linker.h>
50 #include <sys/malloc.h>
51 #include <sys/mbuf.h>
52 #include <sys/memdesc.h>
53 #include <sys/msan.h>
54 #include <sys/proc.h>
55 #include <sys/stack.h>
56 #include <sys/sysctl.h>
57 #include <sys/uio.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <machine/msan.h>
63 #include <machine/stdarg.h>
64
65 void kmsan_init_arg(size_t);
66 void kmsan_init_ret(size_t);
67
68 /* -------------------------------------------------------------------------- */
69
70 /*
71 * Part of the compiler ABI.
72 */
73
74 typedef struct {
75 uint8_t *shad;
76 msan_orig_t *orig;
77 } msan_meta_t;
78
79 #define MSAN_PARAM_SIZE 800
80 #define MSAN_RETVAL_SIZE 800
81 typedef struct {
82 uint8_t param_shadow[MSAN_PARAM_SIZE];
83 uint8_t retval_shadow[MSAN_RETVAL_SIZE];
84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
85 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
86 uint64_t va_arg_overflow_size;
87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
88 msan_orig_t retval_origin;
89 } msan_tls_t;
90
91 /* -------------------------------------------------------------------------- */
92
93 #define MSAN_NCONTEXT 4
94 #define MSAN_ORIG_MASK (~0x3)
95
96 typedef struct kmsan_td {
97 size_t ctx;
98 msan_tls_t tls[MSAN_NCONTEXT];
99 } msan_td_t;
100
101 static msan_tls_t dummy_tls;
102
103 /*
104 * Use separate dummy regions for loads and stores: stores may mark the region
105 * as uninitialized, and that can trigger false positives.
106 */
107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
110 static msan_td_t msan_thread0;
111 static bool kmsan_reporting = false;
112
113 /*
114 * Avoid clobbering any thread-local state before we panic.
115 */
116 #define kmsan_panic(f, ...) do { \
117 kmsan_disabled = true; \
118 panic(f, __VA_ARGS__); \
119 } while (0)
120
121 #define REPORT(f, ...) do { \
122 if (panic_on_violation) { \
123 kmsan_panic(f, __VA_ARGS__); \
124 } else { \
125 struct stack st; \
126 \
127 stack_save(&st); \
128 printf(f "\n", __VA_ARGS__); \
129 stack_print_ddb(&st); \
130 } \
131 } while (0)
132
133 FEATURE(kmsan, "Kernel memory sanitizer");
134
135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
136 "KMSAN options");
137
138 static bool panic_on_violation = 1;
139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
140 &panic_on_violation, 0,
141 "Panic if an invalid access is detected");
142
143 static bool kmsan_disabled __read_mostly = true;
144 #define kmsan_enabled (!kmsan_disabled)
145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
146 &kmsan_disabled, 0, "KMSAN is disabled");
147
148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
149
150 /* -------------------------------------------------------------------------- */
151
152 static inline const char *
kmsan_orig_name(int type)153 kmsan_orig_name(int type)
154 {
155 switch (type) {
156 case KMSAN_TYPE_STACK:
157 return ("stack");
158 case KMSAN_TYPE_KMEM:
159 return ("kmem");
160 case KMSAN_TYPE_MALLOC:
161 return ("malloc");
162 case KMSAN_TYPE_UMA:
163 return ("UMA");
164 default:
165 return ("unknown");
166 }
167 }
168
169 static void
kmsan_report_hook(const void * addr,msan_orig_t * orig,size_t size,size_t off,const char * hook)170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
171 const char *hook)
172 {
173 const char *typename;
174 char *var, *fn;
175 uintptr_t ptr;
176 long foff;
177 char buf[128];
178 int type;
179
180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
181 return;
182
183 kmsan_reporting = true;
184 __compiler_membar();
185
186 if (*orig == 0) {
187 REPORT("MSan: Uninitialized memory in %s, offset %zu",
188 hook, off);
189 goto out;
190 }
191
192 kmsan_md_orig_decode(*orig, &type, &ptr);
193 typename = kmsan_orig_name(type);
194
195 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
196 sizeof(buf), &foff) == 0) {
197 REPORT("MSan: Uninitialized %s memory in %s, "
198 "offset %zu/%zu, addr %p, from %s+%#lx",
199 typename, hook, off, size, addr, buf, foff);
200 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
201 /*
202 * The format of the string is: "----var@function". Parse it to
203 * display a nice warning.
204 */
205 var = (char *)ptr + 4;
206 strlcpy(buf, var, sizeof(buf));
207 var = buf;
208 fn = strchr(buf, '@');
209 *fn++ = '\0';
210 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
211 "variable '%s' from %s", typename, hook, off, var, fn);
212 } else {
213 REPORT("MSan: Uninitialized %s memory in %s, "
214 "offset %zu/%zu, addr %p, PC %p",
215 typename, hook, off, size, addr, (void *)ptr);
216 }
217
218 out:
219 __compiler_membar();
220 kmsan_reporting = false;
221 }
222
223 static void
kmsan_report_inline(msan_orig_t orig,unsigned long pc)224 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
225 {
226 const char *typename;
227 char *var, *fn;
228 uintptr_t ptr;
229 char buf[128];
230 long foff;
231 int type;
232
233 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
234 return;
235
236 kmsan_reporting = true;
237 __compiler_membar();
238
239 if (orig == 0) {
240 REPORT("MSan: uninitialized variable in %p", (void *)pc);
241 goto out;
242 }
243
244 kmsan_md_orig_decode(orig, &type, &ptr);
245 typename = kmsan_orig_name(type);
246
247 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
248 sizeof(buf), &foff) == 0) {
249 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
250 typename, buf, foff);
251 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
252 /*
253 * The format of the string is: "----var@function". Parse it to
254 * display a nice warning.
255 */
256 var = (char *)ptr + 4;
257 strlcpy(buf, var, sizeof(buf));
258 var = buf;
259 fn = strchr(buf, '@');
260 *fn++ = '\0';
261 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
262 } else {
263 REPORT("MSan: Uninitialized %s memory, origin %x",
264 typename, orig);
265 }
266
267 out:
268 __compiler_membar();
269 kmsan_reporting = false;
270 }
271
272 /* -------------------------------------------------------------------------- */
273
274 static inline msan_meta_t
kmsan_meta_get(const void * addr,size_t size,const bool write)275 kmsan_meta_get(const void *addr, size_t size, const bool write)
276 {
277 msan_meta_t ret;
278
279 if (__predict_false(!kmsan_enabled)) {
280 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
281 ret.orig = (msan_orig_t *)msan_dummy_orig;
282 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
283 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
284 ret.orig = (msan_orig_t *)msan_dummy_orig;
285 } else {
286 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
287 ret.orig =
288 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
289 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
290 MSAN_ORIG_MASK);
291 }
292
293 return (ret);
294 }
295
296 static inline void
kmsan_origin_fill(const void * addr,msan_orig_t o,size_t size)297 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
298 {
299 msan_orig_t *orig;
300 size_t i;
301
302 if (__predict_false(!kmsan_enabled))
303 return;
304 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
305 return;
306
307 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
308 size += ((uintptr_t)orig & (sizeof(*orig) - 1));
309 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
310
311 for (i = 0; i < size; i += 4) {
312 orig[i / 4] = o;
313 }
314 }
315
316 static inline void
kmsan_shadow_fill(uintptr_t addr,uint8_t c,size_t size)317 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
318 {
319 uint8_t *shad;
320
321 if (__predict_false(!kmsan_enabled))
322 return;
323 if (__predict_false(kmsan_md_unsupported(addr)))
324 return;
325
326 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
327 __builtin_memset(shad, c, size);
328 }
329
330 static inline void
kmsan_meta_copy(void * dst,const void * src,size_t size)331 kmsan_meta_copy(void *dst, const void *src, size_t size)
332 {
333 uint8_t *orig_src, *orig_dst;
334 uint8_t *shad_src, *shad_dst;
335 msan_orig_t *_src, *_dst;
336 size_t i;
337
338 if (__predict_false(!kmsan_enabled))
339 return;
340 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
341 return;
342 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
343 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
344 return;
345 }
346
347 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
348 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
349 __builtin_memmove(shad_dst, shad_src, size);
350
351 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
352 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
353 for (i = 0; i < size; i++) {
354 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
355 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
356 *_dst = *_src;
357 orig_src++;
358 orig_dst++;
359 }
360 }
361
362 static inline void
kmsan_shadow_check(uintptr_t addr,size_t size,const char * hook)363 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
364 {
365 msan_orig_t *orig;
366 uint8_t *shad;
367 size_t i;
368
369 if (__predict_false(!kmsan_enabled))
370 return;
371 if (__predict_false(kmsan_md_unsupported(addr)))
372 return;
373
374 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
375 for (i = 0; i < size; i++) {
376 if (__predict_true(shad[i] == 0))
377 continue;
378 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i);
379 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
380 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
381 break;
382 }
383 }
384
385 void
kmsan_init_arg(size_t n)386 kmsan_init_arg(size_t n)
387 {
388 msan_td_t *mtd;
389 uint8_t *arg;
390
391 if (__predict_false(!kmsan_enabled))
392 return;
393 if (__predict_false(curthread == NULL))
394 return;
395 mtd = curthread->td_kmsan;
396 arg = mtd->tls[mtd->ctx].param_shadow;
397 __builtin_memset(arg, 0, n);
398 }
399
400 void
kmsan_init_ret(size_t n)401 kmsan_init_ret(size_t n)
402 {
403 msan_td_t *mtd;
404 uint8_t *arg;
405
406 if (__predict_false(!kmsan_enabled))
407 return;
408 if (__predict_false(curthread == NULL))
409 return;
410 mtd = curthread->td_kmsan;
411 arg = mtd->tls[mtd->ctx].retval_shadow;
412 __builtin_memset(arg, 0, n);
413 }
414
415 static void
kmsan_check_arg(size_t size,const char * hook)416 kmsan_check_arg(size_t size, const char *hook)
417 {
418 msan_orig_t *orig;
419 msan_td_t *mtd;
420 uint8_t *arg;
421 size_t ctx, i;
422
423 if (__predict_false(!kmsan_enabled))
424 return;
425 if (__predict_false(curthread == NULL))
426 return;
427 mtd = curthread->td_kmsan;
428 ctx = mtd->ctx;
429 arg = mtd->tls[ctx].param_shadow;
430
431 for (i = 0; i < size; i++) {
432 if (__predict_true(arg[i] == 0))
433 continue;
434 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
435 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
436 break;
437 }
438 }
439
440 void
kmsan_thread_alloc(struct thread * td)441 kmsan_thread_alloc(struct thread *td)
442 {
443 msan_td_t *mtd;
444
445 if (!kmsan_enabled)
446 return;
447
448 mtd = td->td_kmsan;
449 if (mtd == NULL) {
450 /* We might be recycling a thread. */
451 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
452 sizeof(int));
453 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
454 }
455 __builtin_memset(mtd, 0, sizeof(*mtd));
456 mtd->ctx = 0;
457
458 if (td->td_kstack != 0)
459 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
460 KMSAN_STATE_UNINIT);
461
462 td->td_kmsan = mtd;
463 }
464
465 void
kmsan_thread_free(struct thread * td)466 kmsan_thread_free(struct thread *td)
467 {
468 msan_td_t *mtd;
469
470 if (!kmsan_enabled)
471 return;
472 if (__predict_false(td == curthread))
473 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
474
475 mtd = td->td_kmsan;
476 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
477 free(mtd, M_KMSAN);
478 td->td_kmsan = NULL;
479 }
480
481 void kmsan_intr_enter(void);
482 void kmsan_intr_leave(void);
483
484 void
kmsan_intr_enter(void)485 kmsan_intr_enter(void)
486 {
487 msan_td_t *mtd;
488
489 if (__predict_false(!kmsan_enabled))
490 return;
491
492 mtd = curthread->td_kmsan;
493 mtd->ctx++;
494 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
495 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
496 }
497
498 void
kmsan_intr_leave(void)499 kmsan_intr_leave(void)
500 {
501 msan_td_t *mtd;
502
503 if (__predict_false(!kmsan_enabled))
504 return;
505
506 mtd = curthread->td_kmsan;
507 if (__predict_false(mtd->ctx == 0))
508 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
509 mtd->ctx--;
510 }
511
512 /* -------------------------------------------------------------------------- */
513
514 void
kmsan_shadow_map(vm_offset_t addr,size_t size)515 kmsan_shadow_map(vm_offset_t addr, size_t size)
516 {
517 size_t npages, i;
518 vm_offset_t va;
519
520 MPASS(addr % PAGE_SIZE == 0);
521 MPASS(size % PAGE_SIZE == 0);
522
523 if (!kmsan_enabled)
524 return;
525
526 npages = atop(size);
527
528 va = kmsan_md_addr_to_shad(addr);
529 for (i = 0; i < npages; i++) {
530 pmap_san_enter(va + ptoa(i));
531 }
532
533 va = kmsan_md_addr_to_orig(addr);
534 for (i = 0; i < npages; i++) {
535 pmap_san_enter(va + ptoa(i));
536 }
537 }
538
539 void
kmsan_orig(const void * addr,size_t size,int type,uintptr_t pc)540 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
541 {
542 msan_orig_t orig;
543
544 orig = kmsan_md_orig_encode(type, pc);
545 kmsan_origin_fill(addr, orig, size);
546 }
547
548 void
kmsan_mark(const void * addr,size_t size,uint8_t c)549 kmsan_mark(const void *addr, size_t size, uint8_t c)
550 {
551 kmsan_shadow_fill((uintptr_t)addr, c, size);
552 }
553
554 void
kmsan_mark_bio(const struct bio * bp,uint8_t c)555 kmsan_mark_bio(const struct bio *bp, uint8_t c)
556 {
557 kmsan_mark(bp->bio_data, bp->bio_length, c);
558 }
559
560 void
kmsan_mark_mbuf(const struct mbuf * m,uint8_t c)561 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
562 {
563 do {
564 if ((m->m_flags & M_EXTPG) == 0)
565 kmsan_mark(m->m_data, m->m_len, c);
566 m = m->m_next;
567 } while (m != NULL);
568 }
569
570 void
kmsan_check(const void * p,size_t sz,const char * descr)571 kmsan_check(const void *p, size_t sz, const char *descr)
572 {
573 kmsan_shadow_check((uintptr_t)p, sz, descr);
574 }
575
576 void
kmsan_check_bio(const struct bio * bp,const char * descr)577 kmsan_check_bio(const struct bio *bp, const char *descr)
578 {
579 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
580 }
581
582 void
kmsan_check_mbuf(const struct mbuf * m,const char * descr)583 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
584 {
585 do {
586 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
587 } while ((m = m->m_next) != NULL);
588 }
589
590 void
kmsan_check_uio(const struct uio * uio,const char * descr)591 kmsan_check_uio(const struct uio *uio, const char *descr)
592 {
593 for (int i = 0; i < uio->uio_iovcnt; i++) {
594 kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len,
595 descr);
596 }
597 }
598
599 void
kmsan_init(void)600 kmsan_init(void)
601 {
602 int disabled;
603
604 disabled = 0;
605 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
606 if (disabled)
607 return;
608
609 /* Initialize the TLS for curthread. */
610 msan_thread0.ctx = 0;
611 thread0.td_kmsan = &msan_thread0;
612
613 /* Now officially enabled. */
614 kmsan_disabled = false;
615 }
616
617 /* -------------------------------------------------------------------------- */
618
619 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
620 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
621
622 msan_meta_t
__msan_metadata_ptr_for_load_n(void * addr,size_t size)623 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
624 {
625 return (kmsan_meta_get(addr, size, false));
626 }
627
628 msan_meta_t
__msan_metadata_ptr_for_store_n(void * addr,size_t size)629 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
630 {
631 return (kmsan_meta_get(addr, size, true));
632 }
633
634 #define MSAN_META_FUNC(size) \
635 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
636 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
637 { \
638 return (kmsan_meta_get(addr, size, false)); \
639 } \
640 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
641 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
642 { \
643 return (kmsan_meta_get(addr, size, true)); \
644 }
645
646 MSAN_META_FUNC(1)
647 MSAN_META_FUNC(2)
648 MSAN_META_FUNC(4)
649 MSAN_META_FUNC(8)
650
651 void __msan_instrument_asm_store(const void *, size_t);
652 msan_orig_t __msan_chain_origin(msan_orig_t);
653 void __msan_poison(const void *, size_t);
654 void __msan_unpoison(const void *, size_t);
655 void __msan_poison_alloca(const void *, uint64_t, const char *);
656 void __msan_unpoison_alloca(const void *, uint64_t);
657 void __msan_warning(msan_orig_t);
658 msan_tls_t *__msan_get_context_state(void);
659
660 void
__msan_instrument_asm_store(const void * addr,size_t size)661 __msan_instrument_asm_store(const void *addr, size_t size)
662 {
663 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
664 }
665
666 msan_orig_t
__msan_chain_origin(msan_orig_t origin)667 __msan_chain_origin(msan_orig_t origin)
668 {
669 return (origin);
670 }
671
672 void
__msan_poison(const void * addr,size_t size)673 __msan_poison(const void *addr, size_t size)
674 {
675 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
676 }
677
678 void
__msan_unpoison(const void * addr,size_t size)679 __msan_unpoison(const void *addr, size_t size)
680 {
681 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
682 }
683
684 void
__msan_poison_alloca(const void * addr,uint64_t size,const char * descr)685 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
686 {
687 msan_orig_t orig;
688
689 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
690 kmsan_origin_fill(addr, orig, size);
691 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
692 }
693
694 void
__msan_unpoison_alloca(const void * addr,uint64_t size)695 __msan_unpoison_alloca(const void *addr, uint64_t size)
696 {
697 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
698 }
699
700 void
__msan_warning(msan_orig_t origin)701 __msan_warning(msan_orig_t origin)
702 {
703 if (__predict_false(!kmsan_enabled))
704 return;
705 kmsan_report_inline(origin, KMSAN_RET_ADDR);
706 }
707
708 msan_tls_t *
__msan_get_context_state(void)709 __msan_get_context_state(void)
710 {
711 msan_td_t *mtd;
712
713 /*
714 * When APs are started, they execute some C code before curthread is
715 * set. We have to handle that here.
716 */
717 if (__predict_false(!kmsan_enabled || curthread == NULL))
718 return (&dummy_tls);
719 mtd = curthread->td_kmsan;
720 return (&mtd->tls[mtd->ctx]);
721 }
722
723 /* -------------------------------------------------------------------------- */
724
725 /*
726 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
727 * initialized areas properly.
728 */
729
730 void *
kmsan_memcpy(void * dst,const void * src,size_t len)731 kmsan_memcpy(void *dst, const void *src, size_t len)
732 {
733 /* No kmsan_check_arg, because inlined. */
734 kmsan_init_ret(sizeof(void *));
735 if (__predict_true(len != 0)) {
736 kmsan_meta_copy(dst, src, len);
737 }
738 return (__builtin_memcpy(dst, src, len));
739 }
740
741 int
kmsan_memcmp(const void * b1,const void * b2,size_t len)742 kmsan_memcmp(const void *b1, const void *b2, size_t len)
743 {
744 const uint8_t *_b1 = b1, *_b2 = b2;
745 size_t i;
746
747 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
748 "memcmp():args");
749 kmsan_init_ret(sizeof(int));
750
751 for (i = 0; i < len; i++) {
752 if (*_b1 != *_b2) {
753 kmsan_shadow_check((uintptr_t)b1, i + 1,
754 "memcmp():arg1");
755 kmsan_shadow_check((uintptr_t)b2, i + 1,
756 "memcmp():arg2");
757 return (*_b1 - *_b2);
758 }
759 _b1++, _b2++;
760 }
761
762 return (0);
763 }
764
765 void *
kmsan_memset(void * dst,int c,size_t len)766 kmsan_memset(void *dst, int c, size_t len)
767 {
768 /* No kmsan_check_arg, because inlined. */
769 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
770 kmsan_init_ret(sizeof(void *));
771 return (__builtin_memset(dst, c, len));
772 }
773
774 void *
kmsan_memmove(void * dst,const void * src,size_t len)775 kmsan_memmove(void *dst, const void *src, size_t len)
776 {
777 /* No kmsan_check_arg, because inlined. */
778 if (__predict_true(len != 0)) {
779 kmsan_meta_copy(dst, src, len);
780 }
781 kmsan_init_ret(sizeof(void *));
782 return (__builtin_memmove(dst, src, len));
783 }
784
785 __strong_reference(kmsan_memcpy, __msan_memcpy);
786 __strong_reference(kmsan_memset, __msan_memset);
787 __strong_reference(kmsan_memmove, __msan_memmove);
788
789 char *
kmsan_strcpy(char * dst,const char * src)790 kmsan_strcpy(char *dst, const char *src)
791 {
792 const char *_src = src;
793 char *_dst = dst;
794 size_t len = 0;
795
796 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
797
798 while (1) {
799 len++;
800 *dst = *src;
801 if (*src == '\0')
802 break;
803 src++, dst++;
804 }
805
806 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
807 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
808 kmsan_init_ret(sizeof(char *));
809 return (_dst);
810 }
811
812 int
kmsan_strcmp(const char * s1,const char * s2)813 kmsan_strcmp(const char *s1, const char *s2)
814 {
815 const char *_s1 = s1, *_s2 = s2;
816 size_t len = 0;
817
818 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
819 kmsan_init_ret(sizeof(int));
820
821 while (1) {
822 len++;
823 if (*s1 != *s2)
824 break;
825 if (*s1 == '\0') {
826 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
827 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
828 return (0);
829 }
830 s1++, s2++;
831 }
832
833 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
834 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
835
836 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
837 }
838
839 size_t
kmsan_strlen(const char * str)840 kmsan_strlen(const char *str)
841 {
842 const char *s;
843
844 kmsan_check_arg(sizeof(str), "strlen():args");
845
846 s = str;
847 while (1) {
848 if (*s == '\0')
849 break;
850 s++;
851 }
852
853 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
854 kmsan_init_ret(sizeof(size_t));
855 return (s - str);
856 }
857
858 int kmsan_copyin(const void *, void *, size_t);
859 int kmsan_copyout(const void *, void *, size_t);
860 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
861
862 int
kmsan_copyin(const void * uaddr,void * kaddr,size_t len)863 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
864 {
865 int ret;
866
867 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
868 "copyin():args");
869 ret = copyin(uaddr, kaddr, len);
870 if (ret == 0)
871 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
872 kmsan_init_ret(sizeof(int));
873 return (ret);
874 }
875
876 int
kmsan_copyout(const void * kaddr,void * uaddr,size_t len)877 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
878 {
879 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
880 "copyout():args");
881 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
882 kmsan_init_ret(sizeof(int));
883 return (copyout(kaddr, uaddr, len));
884 }
885
886 int
kmsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)887 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
888 {
889 size_t _done;
890 int ret;
891
892 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
893 sizeof(len) + sizeof(done), "copyinstr():args");
894 ret = copyinstr(uaddr, kaddr, len, &_done);
895 if (ret == 0)
896 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
897 if (done != NULL) {
898 *done = _done;
899 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
900 }
901 kmsan_init_ret(sizeof(int));
902 return (ret);
903 }
904
905 /* -------------------------------------------------------------------------- */
906
907 int
kmsan_fubyte(volatile const void * base)908 kmsan_fubyte(volatile const void *base)
909 {
910 int ret;
911
912 kmsan_check_arg(sizeof(base), "fubyte(): args");
913 ret = fubyte(base);
914 kmsan_init_ret(sizeof(int));
915 return (ret);
916 }
917
918 int
kmsan_fuword16(volatile const void * base)919 kmsan_fuword16(volatile const void *base)
920 {
921 int ret;
922
923 kmsan_check_arg(sizeof(base), "fuword16(): args");
924 ret = fuword16(base);
925 kmsan_init_ret(sizeof(int));
926 return (ret);
927 }
928
929 int
kmsan_fueword(volatile const void * base,long * val)930 kmsan_fueword(volatile const void *base, long *val)
931 {
932 int ret;
933
934 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
935 ret = fueword(base, val);
936 if (ret == 0)
937 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
938 sizeof(*val));
939 kmsan_init_ret(sizeof(int));
940 return (ret);
941 }
942
943 int
kmsan_fueword32(volatile const void * base,int32_t * val)944 kmsan_fueword32(volatile const void *base, int32_t *val)
945 {
946 int ret;
947
948 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
949 ret = fueword32(base, val);
950 if (ret == 0)
951 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
952 sizeof(*val));
953 kmsan_init_ret(sizeof(int));
954 return (ret);
955 }
956
957 int
kmsan_fueword64(volatile const void * base,int64_t * val)958 kmsan_fueword64(volatile const void *base, int64_t *val)
959 {
960 int ret;
961
962 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
963 ret = fueword64(base, val);
964 if (ret == 0)
965 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
966 sizeof(*val));
967 kmsan_init_ret(sizeof(int));
968 return (ret);
969 }
970
971 int
kmsan_subyte(volatile void * base,int byte)972 kmsan_subyte(volatile void *base, int byte)
973 {
974 int ret;
975
976 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
977 ret = subyte(base, byte);
978 kmsan_init_ret(sizeof(int));
979 return (ret);
980 }
981
982 int
kmsan_suword(volatile void * base,long word)983 kmsan_suword(volatile void *base, long word)
984 {
985 int ret;
986
987 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
988 ret = suword(base, word);
989 kmsan_init_ret(sizeof(int));
990 return (ret);
991 }
992
993 int
kmsan_suword16(volatile void * base,int word)994 kmsan_suword16(volatile void *base, int word)
995 {
996 int ret;
997
998 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
999 ret = suword16(base, word);
1000 kmsan_init_ret(sizeof(int));
1001 return (ret);
1002 }
1003
1004 int
kmsan_suword32(volatile void * base,int32_t word)1005 kmsan_suword32(volatile void *base, int32_t word)
1006 {
1007 int ret;
1008
1009 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1010 ret = suword32(base, word);
1011 kmsan_init_ret(sizeof(int));
1012 return (ret);
1013 }
1014
1015 int
kmsan_suword64(volatile void * base,int64_t word)1016 kmsan_suword64(volatile void *base, int64_t word)
1017 {
1018 int ret;
1019
1020 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1021 ret = suword64(base, word);
1022 kmsan_init_ret(sizeof(int));
1023 return (ret);
1024 }
1025
1026 int
kmsan_casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)1027 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1028 uint32_t newval)
1029 {
1030 int ret;
1031
1032 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1033 sizeof(newval), "casueword32(): args");
1034 ret = casueword32(base, oldval, oldvalp, newval);
1035 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1036 sizeof(*oldvalp));
1037 kmsan_init_ret(sizeof(int));
1038 return (ret);
1039 }
1040
1041 int
kmsan_casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)1042 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1043 u_long newval)
1044 {
1045 int ret;
1046
1047 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1048 sizeof(newval), "casueword32(): args");
1049 ret = casueword(base, oldval, oldvalp, newval);
1050 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1051 sizeof(*oldvalp));
1052 kmsan_init_ret(sizeof(int));
1053 return (ret);
1054 }
1055
1056 /* -------------------------------------------------------------------------- */
1057
1058 #include <machine/atomic.h>
1059 #include <sys/atomic_san.h>
1060
1061 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \
1062 void kmsan_atomic_add_##name(volatile type *ptr, type val) \
1063 { \
1064 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1065 "atomic_add_" #name "():args"); \
1066 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1067 "atomic_add_" #name "():ptr"); \
1068 atomic_add_##name(ptr, val); \
1069 }
1070
1071 #define MSAN_ATOMIC_FUNC_ADD(name, type) \
1072 _MSAN_ATOMIC_FUNC_ADD(name, type) \
1073 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
1074 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1075
1076 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1077 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1078 { \
1079 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1080 "atomic_subtract_" #name "():args"); \
1081 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1082 "atomic_subtract_" #name "():ptr"); \
1083 atomic_subtract_##name(ptr, val); \
1084 }
1085
1086 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1087 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1088 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
1089 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1090
1091 #define _MSAN_ATOMIC_FUNC_SET(name, type) \
1092 void kmsan_atomic_set_##name(volatile type *ptr, type val) \
1093 { \
1094 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1095 "atomic_set_" #name "():args"); \
1096 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1097 "atomic_set_" #name "():ptr"); \
1098 atomic_set_##name(ptr, val); \
1099 }
1100
1101 #define MSAN_ATOMIC_FUNC_SET(name, type) \
1102 _MSAN_ATOMIC_FUNC_SET(name, type) \
1103 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \
1104 _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1105
1106 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1107 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \
1108 { \
1109 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1110 "atomic_clear_" #name "():args"); \
1111 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1112 "atomic_clear_" #name "():ptr"); \
1113 atomic_clear_##name(ptr, val); \
1114 }
1115
1116 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1117 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1118 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
1119 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1120
1121 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \
1122 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1123 { \
1124 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1125 "atomic_fetchadd_" #name "():args"); \
1126 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1127 "atomic_fetchadd_" #name "():ptr"); \
1128 kmsan_init_ret(sizeof(type)); \
1129 return (atomic_fetchadd_##name(ptr, val)); \
1130 }
1131
1132 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
1133 type kmsan_atomic_readandclear_##name(volatile type *ptr) \
1134 { \
1135 kmsan_check_arg(sizeof(ptr), \
1136 "atomic_readandclear_" #name "():args"); \
1137 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1138 "atomic_readandclear_" #name "():ptr"); \
1139 kmsan_init_ret(sizeof(type)); \
1140 return (atomic_readandclear_##name(ptr)); \
1141 }
1142
1143 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
1144 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1145 { \
1146 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1147 "atomic_testandclear_" #name "():args"); \
1148 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1149 "atomic_testandclear_" #name "():ptr"); \
1150 kmsan_init_ret(sizeof(int)); \
1151 return (atomic_testandclear_##name(ptr, v)); \
1152 }
1153
1154 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
1155 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1156 { \
1157 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1158 "atomic_testandset_" #name "():args"); \
1159 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1160 "atomic_testandset_" #name "():ptr"); \
1161 kmsan_init_ret(sizeof(int)); \
1162 return (atomic_testandset_##name(ptr, v)); \
1163 }
1164
1165 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \
1166 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \
1167 { \
1168 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1169 "atomic_swap_" #name "():args"); \
1170 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1171 "atomic_swap_" #name "():ptr"); \
1172 kmsan_init_ret(sizeof(type)); \
1173 return (atomic_swap_##name(ptr, val)); \
1174 }
1175
1176 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1177 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \
1178 type nval) \
1179 { \
1180 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1181 sizeof(nval), "atomic_cmpset_" #name "():args"); \
1182 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1183 "atomic_cmpset_" #name "():ptr"); \
1184 kmsan_init_ret(sizeof(int)); \
1185 return (atomic_cmpset_##name(ptr, oval, nval)); \
1186 }
1187
1188 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1189 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1190 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
1191 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1192
1193 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1194 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1195 type nval) \
1196 { \
1197 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1198 sizeof(nval), "atomic_fcmpset_" #name "():args"); \
1199 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1200 "atomic_fcmpset_" #name "():ptr"); \
1201 kmsan_init_ret(sizeof(int)); \
1202 return (atomic_fcmpset_##name(ptr, oval, nval)); \
1203 }
1204
1205 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1206 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1207 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
1208 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1209
1210 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
1211 void kmsan_atomic_thread_fence_##name(void) \
1212 { \
1213 atomic_thread_fence_##name(); \
1214 }
1215
1216 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1217 type kmsan_atomic_load_##name(volatile type *ptr) \
1218 { \
1219 kmsan_check_arg(sizeof(ptr), \
1220 "atomic_load_" #name "():args"); \
1221 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1222 "atomic_load_" #name "():ptr"); \
1223 kmsan_init_ret(sizeof(type)); \
1224 return (atomic_load_##name(ptr)); \
1225 }
1226
1227 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \
1228 _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1229 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1230
1231 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \
1232 void kmsan_atomic_store_##name(volatile type *ptr, type val) \
1233 { \
1234 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1235 "atomic_store_" #name "():args"); \
1236 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \
1237 sizeof(type)); \
1238 atomic_store_##name(ptr, val); \
1239 }
1240
1241 #define MSAN_ATOMIC_FUNC_STORE(name, type) \
1242 _MSAN_ATOMIC_FUNC_STORE(name, type) \
1243 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1244
1245 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1246 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1247 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1248 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1249 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1250 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1251 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1252
1253 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1254 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1255 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1256 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1257 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1258 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1259 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1260
1261 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1262 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1263 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1264 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1265 MSAN_ATOMIC_FUNC_SET(int, u_int);
1266 MSAN_ATOMIC_FUNC_SET(long, u_long);
1267 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1268
1269 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1270 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1271 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1272 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1273 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1274 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1275 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1276
1277 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1278 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1279 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1280 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1281
1282 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1283 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1284 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1285 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1286 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1287
1288 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1289 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1290 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1291 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1292 MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t);
1293
1294 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1295 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1296 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1297 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1298 MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t);
1299
1300 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1301 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1302 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1303 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1304 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1305
1306 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1307 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1308 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1309 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1310 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1311 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1312 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1313
1314 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1315 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1316 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1317 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1318 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1319 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1320 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1321
1322 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1323 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1324 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1325 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1326 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1327 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1328 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1329 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1330 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1331 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1332
1333 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1334 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1335 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1336 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1337 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1338 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1339 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1340 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1341 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1342 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1343
1344 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1345 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1346 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1347 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1348
1349 void
kmsan_atomic_interrupt_fence(void)1350 kmsan_atomic_interrupt_fence(void)
1351 {
1352 atomic_interrupt_fence();
1353 }
1354
1355 /* -------------------------------------------------------------------------- */
1356
1357 #include <sys/bus.h>
1358 #include <machine/bus.h>
1359 #include <sys/bus_san.h>
1360
1361 int
kmsan_bus_space_map(bus_space_tag_t tag,bus_addr_t hnd,bus_size_t size,int flags,bus_space_handle_t * handlep)1362 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1363 int flags, bus_space_handle_t *handlep)
1364 {
1365 return (bus_space_map(tag, hnd, size, flags, handlep));
1366 }
1367
1368 void
kmsan_bus_space_unmap(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1369 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1370 bus_size_t size)
1371 {
1372 bus_space_unmap(tag, hnd, size);
1373 }
1374
1375 int
kmsan_bus_space_subregion(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,bus_space_handle_t * handlep)1376 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1377 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1378 {
1379 return (bus_space_subregion(tag, hnd, offset, size, handlep));
1380 }
1381
1382 void
kmsan_bus_space_free(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1383 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1384 bus_size_t size)
1385 {
1386 bus_space_free(tag, hnd, size);
1387 }
1388
1389 void
kmsan_bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,int flags)1390 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1391 bus_size_t offset, bus_size_t size, int flags)
1392 {
1393 bus_space_barrier(tag, hnd, offset, size, flags);
1394 }
1395
1396 #if defined(__amd64__)
1397 #define BUS_SPACE_IO(tag) ((tag) == X86_BUS_SPACE_IO)
1398 #else
1399 #define BUS_SPACE_IO(tag) (false)
1400 #endif
1401
1402 #define MSAN_BUS_READ_FUNC(func, width, type) \
1403 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
1404 bus_space_handle_t hnd, bus_size_t offset) \
1405 { \
1406 type ret; \
1407 if (!BUS_SPACE_IO(tag)) \
1408 kmsan_shadow_fill((uintptr_t)(hnd + offset), \
1409 KMSAN_STATE_INITED, (width)); \
1410 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1411 kmsan_init_ret(sizeof(type)); \
1412 return (ret); \
1413 } \
1414
1415 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \
1416 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1417 bus_space_handle_t hnd, bus_size_t size, type *buf, \
1418 bus_size_t count) \
1419 { \
1420 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \
1421 (width) * count); \
1422 bus_space_read_##func##_##width(tag, hnd, size, buf, \
1423 count); \
1424 }
1425
1426 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1427 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1428 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1429 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1430 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1431 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1432
1433 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1434 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1435 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1436 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1437 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1438 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1439
1440 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1441 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1442 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1443 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1444 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1445 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1446
1447 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1448 #ifndef __amd64__
1449 MSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
1450 MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
1451 MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
1452 MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
1453 MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
1454 #endif
1455
1456 #define MSAN_BUS_WRITE_FUNC(func, width, type) \
1457 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1458 bus_space_handle_t hnd, bus_size_t offset, type value) \
1459 { \
1460 bus_space_write##func##_##width(tag, hnd, offset, value);\
1461 } \
1462
1463 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
1464 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1465 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
1466 bus_size_t count) \
1467 { \
1468 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1469 "bus_space_write()"); \
1470 bus_space_write_##func##_##width(tag, hnd, size, buf, \
1471 count); \
1472 }
1473
1474 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1475 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1476 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1477 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1478 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1479 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1480
1481 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1482 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1483 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1484 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1485 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1486 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1487
1488 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1489 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1490 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1491 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1492 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1493 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1494
1495 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1496
1497 #define MSAN_BUS_SET_FUNC(func, width, type) \
1498 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1499 bus_space_handle_t hnd, bus_size_t offset, type value, \
1500 bus_size_t count) \
1501 { \
1502 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1503 count); \
1504 }
1505
1506 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1507 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1508 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1509 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1510
1511 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1512 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1513 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1514 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1515
1516 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1517 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1518 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1519 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1520
1521 #define MSAN_BUS_PEEK_FUNC(width, type) \
1522 int kmsan_bus_space_peek_##width(bus_space_tag_t tag, \
1523 bus_space_handle_t hnd, bus_size_t offset, type *value) \
1524 { \
1525 return (bus_space_peek_##width(tag, hnd, offset, value)); \
1526 }
1527
1528 MSAN_BUS_PEEK_FUNC(1, uint8_t)
1529 MSAN_BUS_PEEK_FUNC(2, uint16_t)
1530 MSAN_BUS_PEEK_FUNC(4, uint32_t)
1531
1532 #define MSAN_BUS_POKE_FUNC(width, type) \
1533 int kmsan_bus_space_poke_##width(bus_space_tag_t tag, \
1534 bus_space_handle_t hnd, bus_size_t offset, type value) \
1535 { \
1536 return (bus_space_poke_##width(tag, hnd, offset, value)); \
1537 }
1538
1539 MSAN_BUS_POKE_FUNC(1, uint8_t)
1540 MSAN_BUS_POKE_FUNC(2, uint16_t)
1541 MSAN_BUS_POKE_FUNC(4, uint32_t)
1542
1543 /* -------------------------------------------------------------------------- */
1544
1545 void
kmsan_bus_dmamap_sync(struct memdesc * desc,bus_dmasync_op_t op)1546 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1547 {
1548 /*
1549 * Some drivers, e.g., nvme, use the same code path for loading device
1550 * read and write requests, and will thus specify both flags. In this
1551 * case we should not do any checking since it will generally lead to
1552 * false positives.
1553 */
1554 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1555 BUS_DMASYNC_PREWRITE) {
1556 switch (desc->md_type) {
1557 case MEMDESC_VADDR:
1558 kmsan_check(desc->u.md_vaddr, desc->md_len,
1559 "dmasync");
1560 break;
1561 case MEMDESC_MBUF:
1562 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1563 break;
1564 case 0:
1565 break;
1566 default:
1567 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1568 desc->md_type);
1569 }
1570 }
1571 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1572 switch (desc->md_type) {
1573 case MEMDESC_VADDR:
1574 kmsan_mark(desc->u.md_vaddr, desc->md_len,
1575 KMSAN_STATE_INITED);
1576 break;
1577 case MEMDESC_MBUF:
1578 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1579 break;
1580 case 0:
1581 break;
1582 default:
1583 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1584 desc->md_type);
1585 }
1586 }
1587 }
1588