1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 * Copyright (c) 2021 The FreeBSD Foundation
7 *
8 * Portions of this software were developed by Mark Johnston under sponsorship
9 * from the FreeBSD Foundation.
10 *
11 * This code is part of the KMSAN subsystem of the NetBSD kernel.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #define SAN_RUNTIME
36
37 #include <sys/cdefs.h>
38 #if 0
39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
40 #endif
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/bio.h>
45 #include <sys/buf.h>
46 #include <sys/conf.h>
47 #include <sys/kdb.h>
48 #include <sys/kernel.h>
49 #include <sys/linker.h>
50 #include <sys/malloc.h>
51 #include <sys/mbuf.h>
52 #include <sys/memdesc.h>
53 #include <sys/msan.h>
54 #include <sys/proc.h>
55 #include <sys/stack.h>
56 #include <sys/sysctl.h>
57 #include <sys/uio.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <machine/msan.h>
63 #include <machine/stdarg.h>
64
65 void kmsan_init_arg(size_t);
66 void kmsan_init_ret(size_t);
67
68 /* -------------------------------------------------------------------------- */
69
70 /*
71 * Part of the compiler ABI.
72 */
73
74 typedef struct {
75 uint8_t *shad;
76 msan_orig_t *orig;
77 } msan_meta_t;
78
79 #define MSAN_PARAM_SIZE 800
80 #define MSAN_RETVAL_SIZE 800
81 typedef struct {
82 uint8_t param_shadow[MSAN_PARAM_SIZE];
83 uint8_t retval_shadow[MSAN_RETVAL_SIZE];
84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
85 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
86 uint64_t va_arg_overflow_size;
87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
88 msan_orig_t retval_origin;
89 } msan_tls_t;
90
91 /* -------------------------------------------------------------------------- */
92
93 #define MSAN_NCONTEXT 4
94 #define MSAN_ORIG_MASK (~0x3)
95
96 typedef struct kmsan_td {
97 size_t ctx;
98 msan_tls_t tls[MSAN_NCONTEXT];
99 } msan_td_t;
100
101 static msan_tls_t dummy_tls;
102
103 /*
104 * Use separate dummy regions for loads and stores: stores may mark the region
105 * as uninitialized, and that can trigger false positives.
106 */
107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
110 static msan_td_t msan_thread0;
111 static bool kmsan_reporting = false;
112
113 /*
114 * Avoid clobbering any thread-local state before we panic.
115 */
116 #define kmsan_panic(f, ...) do { \
117 kmsan_disabled = true; \
118 panic(f, __VA_ARGS__); \
119 } while (0)
120
121 #define REPORT(f, ...) do { \
122 if (panic_on_violation) { \
123 kmsan_panic(f, __VA_ARGS__); \
124 } else { \
125 struct stack st; \
126 \
127 stack_save(&st); \
128 printf(f "\n", __VA_ARGS__); \
129 stack_print_ddb(&st); \
130 } \
131 } while (0)
132
133 FEATURE(kmsan, "Kernel memory sanitizer");
134
135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
136 "KMSAN options");
137
138 static bool panic_on_violation = 1;
139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
140 &panic_on_violation, 0,
141 "Panic if an invalid access is detected");
142
143 static bool kmsan_disabled __read_mostly = true;
144 #define kmsan_enabled (!kmsan_disabled)
145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
146 &kmsan_disabled, 0, "KMSAN is disabled");
147
148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
149
150 /* -------------------------------------------------------------------------- */
151
152 static inline const char *
kmsan_orig_name(int type)153 kmsan_orig_name(int type)
154 {
155 switch (type) {
156 case KMSAN_TYPE_STACK:
157 return ("stack");
158 case KMSAN_TYPE_KMEM:
159 return ("kmem");
160 case KMSAN_TYPE_MALLOC:
161 return ("malloc");
162 case KMSAN_TYPE_UMA:
163 return ("UMA");
164 default:
165 return ("unknown");
166 }
167 }
168
169 static void
kmsan_report_hook(const void * addr,msan_orig_t * orig,size_t size,size_t off,const char * hook)170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
171 const char *hook)
172 {
173 const char *typename;
174 char *var, *fn;
175 uintptr_t ptr;
176 long foff;
177 char buf[128];
178 int type;
179
180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
181 return;
182 if (__predict_false(curthread != NULL &&
183 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0))
184 return;
185
186 kmsan_reporting = true;
187 __compiler_membar();
188
189 if (*orig == 0) {
190 REPORT("MSan: Uninitialized memory in %s, offset %zu",
191 hook, off);
192 goto out;
193 }
194
195 kmsan_md_orig_decode(*orig, &type, &ptr);
196 typename = kmsan_orig_name(type);
197
198 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
199 sizeof(buf), &foff) == 0) {
200 REPORT("MSan: Uninitialized %s memory in %s, "
201 "offset %zu/%zu, addr %p, from %s+%#lx",
202 typename, hook, off, size, addr, buf, foff);
203 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
204 /*
205 * The format of the string is: "----var@function". Parse it to
206 * display a nice warning.
207 */
208 var = (char *)ptr + 4;
209 strlcpy(buf, var, sizeof(buf));
210 var = buf;
211 fn = strchr(buf, '@');
212 *fn++ = '\0';
213 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
214 "variable '%s' from %s", typename, hook, off, var, fn);
215 } else {
216 REPORT("MSan: Uninitialized %s memory in %s, "
217 "offset %zu/%zu, addr %p, PC %p",
218 typename, hook, off, size, addr, (void *)ptr);
219 }
220
221 out:
222 __compiler_membar();
223 kmsan_reporting = false;
224 }
225
226 static void
kmsan_report_inline(msan_orig_t orig,unsigned long pc)227 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
228 {
229 const char *typename;
230 char *var, *fn;
231 uintptr_t ptr;
232 char buf[128];
233 long foff;
234 int type;
235
236 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
237 return;
238 if (__predict_false(curthread != NULL &&
239 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0))
240 return;
241
242 kmsan_reporting = true;
243 __compiler_membar();
244
245 if (orig == 0) {
246 REPORT("MSan: uninitialized variable in %p", (void *)pc);
247 goto out;
248 }
249
250 kmsan_md_orig_decode(orig, &type, &ptr);
251 typename = kmsan_orig_name(type);
252
253 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
254 sizeof(buf), &foff) == 0) {
255 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
256 typename, buf, foff);
257 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
258 /*
259 * The format of the string is: "----var@function". Parse it to
260 * display a nice warning.
261 */
262 var = (char *)ptr + 4;
263 strlcpy(buf, var, sizeof(buf));
264 var = buf;
265 fn = strchr(buf, '@');
266 *fn++ = '\0';
267 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
268 } else {
269 REPORT("MSan: Uninitialized %s memory, origin %x",
270 typename, orig);
271 }
272
273 out:
274 __compiler_membar();
275 kmsan_reporting = false;
276 }
277
278 /* -------------------------------------------------------------------------- */
279
280 static inline msan_meta_t
kmsan_meta_get(const void * addr,size_t size,const bool write)281 kmsan_meta_get(const void *addr, size_t size, const bool write)
282 {
283 msan_meta_t ret;
284
285 if (__predict_false(!kmsan_enabled)) {
286 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
287 ret.orig = (msan_orig_t *)msan_dummy_orig;
288 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
289 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
290 ret.orig = (msan_orig_t *)msan_dummy_orig;
291 } else {
292 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
293 ret.orig =
294 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
295 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
296 MSAN_ORIG_MASK);
297 }
298
299 return (ret);
300 }
301
302 static inline void
kmsan_origin_fill(const void * addr,msan_orig_t o,size_t size)303 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
304 {
305 msan_orig_t *orig;
306 size_t i;
307
308 if (__predict_false(!kmsan_enabled))
309 return;
310 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
311 return;
312
313 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
314 size += ((uintptr_t)orig & (sizeof(*orig) - 1));
315 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
316
317 for (i = 0; i < size; i += 4) {
318 orig[i / 4] = o;
319 }
320 }
321
322 static inline void
kmsan_shadow_fill(uintptr_t addr,uint8_t c,size_t size)323 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
324 {
325 uint8_t *shad;
326
327 if (__predict_false(!kmsan_enabled))
328 return;
329 if (__predict_false(kmsan_md_unsupported(addr)))
330 return;
331
332 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
333 __builtin_memset(shad, c, size);
334 }
335
336 static inline void
kmsan_meta_copy(void * dst,const void * src,size_t size)337 kmsan_meta_copy(void *dst, const void *src, size_t size)
338 {
339 uint8_t *orig_src, *orig_dst;
340 uint8_t *shad_src, *shad_dst;
341 msan_orig_t *_src, *_dst;
342 size_t i;
343
344 if (__predict_false(!kmsan_enabled))
345 return;
346 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
347 return;
348 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
349 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
350 return;
351 }
352
353 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
354 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
355 __builtin_memmove(shad_dst, shad_src, size);
356
357 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
358 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
359 for (i = 0; i < size; i++) {
360 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
361 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
362 *_dst = *_src;
363 orig_src++;
364 orig_dst++;
365 }
366 }
367
368 static inline void
kmsan_shadow_check(uintptr_t addr,size_t size,const char * hook)369 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
370 {
371 msan_orig_t *orig;
372 uint8_t *shad;
373 size_t i;
374
375 if (__predict_false(!kmsan_enabled))
376 return;
377 if (__predict_false(kmsan_md_unsupported(addr)))
378 return;
379
380 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
381 for (i = 0; i < size; i++) {
382 if (__predict_true(shad[i] == 0))
383 continue;
384 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i);
385 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
386 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
387 break;
388 }
389 }
390
391 void
kmsan_init_arg(size_t n)392 kmsan_init_arg(size_t n)
393 {
394 msan_td_t *mtd;
395 uint8_t *arg;
396
397 if (__predict_false(!kmsan_enabled))
398 return;
399 if (__predict_false(curthread == NULL))
400 return;
401 mtd = curthread->td_kmsan;
402 arg = mtd->tls[mtd->ctx].param_shadow;
403 __builtin_memset(arg, 0, n);
404 }
405
406 void
kmsan_init_ret(size_t n)407 kmsan_init_ret(size_t n)
408 {
409 msan_td_t *mtd;
410 uint8_t *arg;
411
412 if (__predict_false(!kmsan_enabled))
413 return;
414 if (__predict_false(curthread == NULL))
415 return;
416 mtd = curthread->td_kmsan;
417 arg = mtd->tls[mtd->ctx].retval_shadow;
418 __builtin_memset(arg, 0, n);
419 }
420
421 static void
kmsan_check_arg(size_t size,const char * hook)422 kmsan_check_arg(size_t size, const char *hook)
423 {
424 msan_orig_t *orig;
425 msan_td_t *mtd;
426 uint8_t *arg;
427 size_t ctx, i;
428
429 if (__predict_false(!kmsan_enabled))
430 return;
431 if (__predict_false(curthread == NULL))
432 return;
433 mtd = curthread->td_kmsan;
434 ctx = mtd->ctx;
435 arg = mtd->tls[ctx].param_shadow;
436
437 for (i = 0; i < size; i++) {
438 if (__predict_true(arg[i] == 0))
439 continue;
440 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
441 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
442 break;
443 }
444 }
445
446 void
kmsan_thread_alloc(struct thread * td)447 kmsan_thread_alloc(struct thread *td)
448 {
449 msan_td_t *mtd;
450
451 if (!kmsan_enabled)
452 return;
453
454 mtd = td->td_kmsan;
455 if (mtd == NULL) {
456 /* We might be recycling a thread. */
457 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
458 sizeof(int));
459 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
460 }
461 __builtin_memset(mtd, 0, sizeof(*mtd));
462 mtd->ctx = 0;
463
464 if (td->td_kstack != 0)
465 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
466 KMSAN_STATE_UNINIT);
467
468 td->td_kmsan = mtd;
469 }
470
471 void
kmsan_thread_free(struct thread * td)472 kmsan_thread_free(struct thread *td)
473 {
474 msan_td_t *mtd;
475
476 if (!kmsan_enabled)
477 return;
478 if (__predict_false(td == curthread))
479 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
480
481 mtd = td->td_kmsan;
482 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
483 free(mtd, M_KMSAN);
484 td->td_kmsan = NULL;
485 }
486
487 void kmsan_intr_enter(void);
488 void kmsan_intr_leave(void);
489
490 void
kmsan_intr_enter(void)491 kmsan_intr_enter(void)
492 {
493 msan_td_t *mtd;
494
495 if (__predict_false(!kmsan_enabled))
496 return;
497
498 mtd = curthread->td_kmsan;
499 mtd->ctx++;
500 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
501 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
502 }
503
504 void
kmsan_intr_leave(void)505 kmsan_intr_leave(void)
506 {
507 msan_td_t *mtd;
508
509 if (__predict_false(!kmsan_enabled))
510 return;
511
512 mtd = curthread->td_kmsan;
513 if (__predict_false(mtd->ctx == 0))
514 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
515 mtd->ctx--;
516 }
517
518 /* -------------------------------------------------------------------------- */
519
520 void
kmsan_shadow_map(vm_offset_t addr,size_t size)521 kmsan_shadow_map(vm_offset_t addr, size_t size)
522 {
523 size_t npages, i;
524 vm_offset_t va;
525
526 MPASS(addr % PAGE_SIZE == 0);
527 MPASS(size % PAGE_SIZE == 0);
528
529 if (!kmsan_enabled)
530 return;
531
532 npages = atop(size);
533
534 va = kmsan_md_addr_to_shad(addr);
535 for (i = 0; i < npages; i++) {
536 pmap_san_enter(va + ptoa(i));
537 }
538
539 va = kmsan_md_addr_to_orig(addr);
540 for (i = 0; i < npages; i++) {
541 pmap_san_enter(va + ptoa(i));
542 }
543 }
544
545 void
kmsan_orig(const void * addr,size_t size,int type,uintptr_t pc)546 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
547 {
548 msan_orig_t orig;
549
550 orig = kmsan_md_orig_encode(type, pc);
551 kmsan_origin_fill(addr, orig, size);
552 }
553
554 void
kmsan_mark(const void * addr,size_t size,uint8_t c)555 kmsan_mark(const void *addr, size_t size, uint8_t c)
556 {
557 kmsan_shadow_fill((uintptr_t)addr, c, size);
558 }
559
560 void
kmsan_mark_bio(const struct bio * bp,uint8_t c)561 kmsan_mark_bio(const struct bio *bp, uint8_t c)
562 {
563 kmsan_mark(bp->bio_data, bp->bio_length, c);
564 }
565
566 void
kmsan_mark_mbuf(const struct mbuf * m,uint8_t c)567 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
568 {
569 do {
570 if ((m->m_flags & M_EXTPG) == 0)
571 kmsan_mark(m->m_data, m->m_len, c);
572 m = m->m_next;
573 } while (m != NULL);
574 }
575
576 void
kmsan_check(const void * p,size_t sz,const char * descr)577 kmsan_check(const void *p, size_t sz, const char *descr)
578 {
579 kmsan_shadow_check((uintptr_t)p, sz, descr);
580 }
581
582 void
kmsan_check_bio(const struct bio * bp,const char * descr)583 kmsan_check_bio(const struct bio *bp, const char *descr)
584 {
585 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
586 }
587
588 void
kmsan_check_mbuf(const struct mbuf * m,const char * descr)589 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
590 {
591 do {
592 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
593 } while ((m = m->m_next) != NULL);
594 }
595
596 void
kmsan_check_uio(const struct uio * uio,const char * descr)597 kmsan_check_uio(const struct uio *uio, const char *descr)
598 {
599 for (int i = 0; i < uio->uio_iovcnt; i++) {
600 kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len,
601 descr);
602 }
603 }
604
605 void
kmsan_init(void)606 kmsan_init(void)
607 {
608 int disabled;
609
610 disabled = 0;
611 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
612 if (disabled)
613 return;
614
615 /* Initialize the TLS for curthread. */
616 msan_thread0.ctx = 0;
617 thread0.td_kmsan = &msan_thread0;
618
619 /* Now officially enabled. */
620 kmsan_disabled = false;
621 }
622
623 /* -------------------------------------------------------------------------- */
624
625 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
626 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
627
628 msan_meta_t
__msan_metadata_ptr_for_load_n(void * addr,size_t size)629 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
630 {
631 return (kmsan_meta_get(addr, size, false));
632 }
633
634 msan_meta_t
__msan_metadata_ptr_for_store_n(void * addr,size_t size)635 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
636 {
637 return (kmsan_meta_get(addr, size, true));
638 }
639
640 #define MSAN_META_FUNC(size) \
641 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
642 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
643 { \
644 return (kmsan_meta_get(addr, size, false)); \
645 } \
646 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
647 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
648 { \
649 return (kmsan_meta_get(addr, size, true)); \
650 }
651
652 MSAN_META_FUNC(1)
653 MSAN_META_FUNC(2)
654 MSAN_META_FUNC(4)
655 MSAN_META_FUNC(8)
656
657 void __msan_instrument_asm_store(const void *, size_t);
658 msan_orig_t __msan_chain_origin(msan_orig_t);
659 void __msan_poison(const void *, size_t);
660 void __msan_unpoison(const void *, size_t);
661 void __msan_poison_alloca(const void *, uint64_t, const char *);
662 void __msan_unpoison_alloca(const void *, uint64_t);
663 void __msan_warning(msan_orig_t);
664 msan_tls_t *__msan_get_context_state(void);
665
666 void
__msan_instrument_asm_store(const void * addr,size_t size)667 __msan_instrument_asm_store(const void *addr, size_t size)
668 {
669 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
670 }
671
672 msan_orig_t
__msan_chain_origin(msan_orig_t origin)673 __msan_chain_origin(msan_orig_t origin)
674 {
675 return (origin);
676 }
677
678 void
__msan_poison(const void * addr,size_t size)679 __msan_poison(const void *addr, size_t size)
680 {
681 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
682 }
683
684 void
__msan_unpoison(const void * addr,size_t size)685 __msan_unpoison(const void *addr, size_t size)
686 {
687 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
688 }
689
690 void
__msan_poison_alloca(const void * addr,uint64_t size,const char * descr)691 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
692 {
693 msan_orig_t orig;
694
695 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
696 kmsan_origin_fill(addr, orig, size);
697 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
698 }
699
700 void
__msan_unpoison_alloca(const void * addr,uint64_t size)701 __msan_unpoison_alloca(const void *addr, uint64_t size)
702 {
703 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
704 }
705
706 void
__msan_warning(msan_orig_t origin)707 __msan_warning(msan_orig_t origin)
708 {
709 if (__predict_false(!kmsan_enabled))
710 return;
711 kmsan_report_inline(origin, KMSAN_RET_ADDR);
712 }
713
714 msan_tls_t *
__msan_get_context_state(void)715 __msan_get_context_state(void)
716 {
717 msan_td_t *mtd;
718
719 /*
720 * When APs are started, they execute some C code before curthread is
721 * set. We have to handle that here.
722 */
723 if (__predict_false(!kmsan_enabled || curthread == NULL))
724 return (&dummy_tls);
725 mtd = curthread->td_kmsan;
726 return (&mtd->tls[mtd->ctx]);
727 }
728
729 /* -------------------------------------------------------------------------- */
730
731 /*
732 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
733 * initialized areas properly.
734 */
735
736 void *
kmsan_memcpy(void * dst,const void * src,size_t len)737 kmsan_memcpy(void *dst, const void *src, size_t len)
738 {
739 /* No kmsan_check_arg, because inlined. */
740 kmsan_init_ret(sizeof(void *));
741 if (__predict_true(len != 0)) {
742 kmsan_meta_copy(dst, src, len);
743 }
744 return (__builtin_memcpy(dst, src, len));
745 }
746
747 int
kmsan_memcmp(const void * b1,const void * b2,size_t len)748 kmsan_memcmp(const void *b1, const void *b2, size_t len)
749 {
750 const uint8_t *_b1 = b1, *_b2 = b2;
751 size_t i;
752
753 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
754 "memcmp():args");
755 kmsan_init_ret(sizeof(int));
756
757 for (i = 0; i < len; i++) {
758 if (*_b1 != *_b2) {
759 kmsan_shadow_check((uintptr_t)b1, i + 1,
760 "memcmp():arg1");
761 kmsan_shadow_check((uintptr_t)b2, i + 1,
762 "memcmp():arg2");
763 return (*_b1 - *_b2);
764 }
765 _b1++, _b2++;
766 }
767
768 return (0);
769 }
770
771 void *
kmsan_memset(void * dst,int c,size_t len)772 kmsan_memset(void *dst, int c, size_t len)
773 {
774 /* No kmsan_check_arg, because inlined. */
775 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
776 kmsan_init_ret(sizeof(void *));
777 return (__builtin_memset(dst, c, len));
778 }
779
780 void *
kmsan_memmove(void * dst,const void * src,size_t len)781 kmsan_memmove(void *dst, const void *src, size_t len)
782 {
783 /* No kmsan_check_arg, because inlined. */
784 if (__predict_true(len != 0)) {
785 kmsan_meta_copy(dst, src, len);
786 }
787 kmsan_init_ret(sizeof(void *));
788 return (__builtin_memmove(dst, src, len));
789 }
790
791 __strong_reference(kmsan_memcpy, __msan_memcpy);
792 __strong_reference(kmsan_memset, __msan_memset);
793 __strong_reference(kmsan_memmove, __msan_memmove);
794
795 char *
kmsan_strcpy(char * dst,const char * src)796 kmsan_strcpy(char *dst, const char *src)
797 {
798 const char *_src = src;
799 char *_dst = dst;
800 size_t len = 0;
801
802 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
803
804 while (1) {
805 len++;
806 *dst = *src;
807 if (*src == '\0')
808 break;
809 src++, dst++;
810 }
811
812 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
813 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
814 kmsan_init_ret(sizeof(char *));
815 return (_dst);
816 }
817
818 int
kmsan_strcmp(const char * s1,const char * s2)819 kmsan_strcmp(const char *s1, const char *s2)
820 {
821 const char *_s1 = s1, *_s2 = s2;
822 size_t len = 0;
823
824 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
825 kmsan_init_ret(sizeof(int));
826
827 while (1) {
828 len++;
829 if (*s1 != *s2)
830 break;
831 if (*s1 == '\0') {
832 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
833 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
834 return (0);
835 }
836 s1++, s2++;
837 }
838
839 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
840 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
841
842 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
843 }
844
845 size_t
kmsan_strlen(const char * str)846 kmsan_strlen(const char *str)
847 {
848 const char *s;
849
850 kmsan_check_arg(sizeof(str), "strlen():args");
851
852 s = str;
853 while (1) {
854 if (*s == '\0')
855 break;
856 s++;
857 }
858
859 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
860 kmsan_init_ret(sizeof(size_t));
861 return (s - str);
862 }
863
864 int kmsan_copyin(const void *, void *, size_t);
865 int kmsan_copyout(const void *, void *, size_t);
866 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
867
868 int
kmsan_copyin(const void * uaddr,void * kaddr,size_t len)869 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
870 {
871 int ret;
872
873 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
874 "copyin():args");
875 ret = copyin(uaddr, kaddr, len);
876 if (ret == 0)
877 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
878 kmsan_init_ret(sizeof(int));
879 return (ret);
880 }
881
882 int
kmsan_copyout(const void * kaddr,void * uaddr,size_t len)883 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
884 {
885 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
886 "copyout():args");
887 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
888 kmsan_init_ret(sizeof(int));
889 return (copyout(kaddr, uaddr, len));
890 }
891
892 int
kmsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)893 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
894 {
895 size_t _done;
896 int ret;
897
898 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
899 sizeof(len) + sizeof(done), "copyinstr():args");
900 ret = copyinstr(uaddr, kaddr, len, &_done);
901 if (ret == 0)
902 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
903 if (done != NULL) {
904 *done = _done;
905 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
906 }
907 kmsan_init_ret(sizeof(int));
908 return (ret);
909 }
910
911 /* -------------------------------------------------------------------------- */
912
913 int
kmsan_fubyte(volatile const void * base)914 kmsan_fubyte(volatile const void *base)
915 {
916 int ret;
917
918 kmsan_check_arg(sizeof(base), "fubyte(): args");
919 ret = fubyte(base);
920 kmsan_init_ret(sizeof(int));
921 return (ret);
922 }
923
924 int
kmsan_fuword16(volatile const void * base)925 kmsan_fuword16(volatile const void *base)
926 {
927 int ret;
928
929 kmsan_check_arg(sizeof(base), "fuword16(): args");
930 ret = fuword16(base);
931 kmsan_init_ret(sizeof(int));
932 return (ret);
933 }
934
935 int
kmsan_fueword(volatile const void * base,long * val)936 kmsan_fueword(volatile const void *base, long *val)
937 {
938 int ret;
939
940 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
941 ret = fueword(base, val);
942 if (ret == 0)
943 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
944 sizeof(*val));
945 kmsan_init_ret(sizeof(int));
946 return (ret);
947 }
948
949 int
kmsan_fueword32(volatile const void * base,int32_t * val)950 kmsan_fueword32(volatile const void *base, int32_t *val)
951 {
952 int ret;
953
954 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
955 ret = fueword32(base, val);
956 if (ret == 0)
957 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
958 sizeof(*val));
959 kmsan_init_ret(sizeof(int));
960 return (ret);
961 }
962
963 int
kmsan_fueword64(volatile const void * base,int64_t * val)964 kmsan_fueword64(volatile const void *base, int64_t *val)
965 {
966 int ret;
967
968 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
969 ret = fueword64(base, val);
970 if (ret == 0)
971 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
972 sizeof(*val));
973 kmsan_init_ret(sizeof(int));
974 return (ret);
975 }
976
977 int
kmsan_subyte(volatile void * base,int byte)978 kmsan_subyte(volatile void *base, int byte)
979 {
980 int ret;
981
982 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
983 ret = subyte(base, byte);
984 kmsan_init_ret(sizeof(int));
985 return (ret);
986 }
987
988 int
kmsan_suword(volatile void * base,long word)989 kmsan_suword(volatile void *base, long word)
990 {
991 int ret;
992
993 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
994 ret = suword(base, word);
995 kmsan_init_ret(sizeof(int));
996 return (ret);
997 }
998
999 int
kmsan_suword16(volatile void * base,int word)1000 kmsan_suword16(volatile void *base, int word)
1001 {
1002 int ret;
1003
1004 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
1005 ret = suword16(base, word);
1006 kmsan_init_ret(sizeof(int));
1007 return (ret);
1008 }
1009
1010 int
kmsan_suword32(volatile void * base,int32_t word)1011 kmsan_suword32(volatile void *base, int32_t word)
1012 {
1013 int ret;
1014
1015 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1016 ret = suword32(base, word);
1017 kmsan_init_ret(sizeof(int));
1018 return (ret);
1019 }
1020
1021 int
kmsan_suword64(volatile void * base,int64_t word)1022 kmsan_suword64(volatile void *base, int64_t word)
1023 {
1024 int ret;
1025
1026 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1027 ret = suword64(base, word);
1028 kmsan_init_ret(sizeof(int));
1029 return (ret);
1030 }
1031
1032 int
kmsan_casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)1033 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1034 uint32_t newval)
1035 {
1036 int ret;
1037
1038 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1039 sizeof(newval), "casueword32(): args");
1040 ret = casueword32(base, oldval, oldvalp, newval);
1041 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1042 sizeof(*oldvalp));
1043 kmsan_init_ret(sizeof(int));
1044 return (ret);
1045 }
1046
1047 int
kmsan_casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)1048 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1049 u_long newval)
1050 {
1051 int ret;
1052
1053 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1054 sizeof(newval), "casueword32(): args");
1055 ret = casueword(base, oldval, oldvalp, newval);
1056 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1057 sizeof(*oldvalp));
1058 kmsan_init_ret(sizeof(int));
1059 return (ret);
1060 }
1061
1062 /* -------------------------------------------------------------------------- */
1063
1064 #include <machine/atomic.h>
1065 #include <sys/atomic_san.h>
1066
1067 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \
1068 void kmsan_atomic_add_##name(volatile type *ptr, type val) \
1069 { \
1070 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1071 "atomic_add_" #name "():args"); \
1072 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1073 "atomic_add_" #name "():ptr"); \
1074 atomic_add_##name(ptr, val); \
1075 }
1076
1077 #define MSAN_ATOMIC_FUNC_ADD(name, type) \
1078 _MSAN_ATOMIC_FUNC_ADD(name, type) \
1079 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
1080 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1081
1082 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1083 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1084 { \
1085 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1086 "atomic_subtract_" #name "():args"); \
1087 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1088 "atomic_subtract_" #name "():ptr"); \
1089 atomic_subtract_##name(ptr, val); \
1090 }
1091
1092 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1093 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1094 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
1095 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1096
1097 #define _MSAN_ATOMIC_FUNC_SET(name, type) \
1098 void kmsan_atomic_set_##name(volatile type *ptr, type val) \
1099 { \
1100 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1101 "atomic_set_" #name "():args"); \
1102 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1103 "atomic_set_" #name "():ptr"); \
1104 atomic_set_##name(ptr, val); \
1105 }
1106
1107 #define MSAN_ATOMIC_FUNC_SET(name, type) \
1108 _MSAN_ATOMIC_FUNC_SET(name, type) \
1109 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \
1110 _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1111
1112 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1113 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \
1114 { \
1115 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1116 "atomic_clear_" #name "():args"); \
1117 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1118 "atomic_clear_" #name "():ptr"); \
1119 atomic_clear_##name(ptr, val); \
1120 }
1121
1122 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1123 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1124 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
1125 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1126
1127 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \
1128 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1129 { \
1130 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1131 "atomic_fetchadd_" #name "():args"); \
1132 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1133 "atomic_fetchadd_" #name "():ptr"); \
1134 kmsan_init_ret(sizeof(type)); \
1135 return (atomic_fetchadd_##name(ptr, val)); \
1136 }
1137
1138 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
1139 type kmsan_atomic_readandclear_##name(volatile type *ptr) \
1140 { \
1141 kmsan_check_arg(sizeof(ptr), \
1142 "atomic_readandclear_" #name "():args"); \
1143 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1144 "atomic_readandclear_" #name "():ptr"); \
1145 kmsan_init_ret(sizeof(type)); \
1146 return (atomic_readandclear_##name(ptr)); \
1147 }
1148
1149 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
1150 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1151 { \
1152 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1153 "atomic_testandclear_" #name "():args"); \
1154 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1155 "atomic_testandclear_" #name "():ptr"); \
1156 kmsan_init_ret(sizeof(int)); \
1157 return (atomic_testandclear_##name(ptr, v)); \
1158 }
1159
1160 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
1161 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1162 { \
1163 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1164 "atomic_testandset_" #name "():args"); \
1165 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1166 "atomic_testandset_" #name "():ptr"); \
1167 kmsan_init_ret(sizeof(int)); \
1168 return (atomic_testandset_##name(ptr, v)); \
1169 }
1170
1171 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \
1172 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \
1173 { \
1174 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1175 "atomic_swap_" #name "():args"); \
1176 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1177 "atomic_swap_" #name "():ptr"); \
1178 kmsan_init_ret(sizeof(type)); \
1179 return (atomic_swap_##name(ptr, val)); \
1180 }
1181
1182 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1183 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \
1184 type nval) \
1185 { \
1186 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1187 sizeof(nval), "atomic_cmpset_" #name "():args"); \
1188 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1189 "atomic_cmpset_" #name "():ptr"); \
1190 kmsan_init_ret(sizeof(int)); \
1191 return (atomic_cmpset_##name(ptr, oval, nval)); \
1192 }
1193
1194 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1195 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1196 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
1197 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1198
1199 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1200 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1201 type nval) \
1202 { \
1203 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1204 sizeof(nval), "atomic_fcmpset_" #name "():args"); \
1205 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1206 "atomic_fcmpset_" #name "():ptr"); \
1207 kmsan_init_ret(sizeof(int)); \
1208 return (atomic_fcmpset_##name(ptr, oval, nval)); \
1209 }
1210
1211 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1212 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1213 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
1214 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1215
1216 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
1217 void kmsan_atomic_thread_fence_##name(void) \
1218 { \
1219 atomic_thread_fence_##name(); \
1220 }
1221
1222 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1223 type kmsan_atomic_load_##name(const volatile type *ptr) \
1224 { \
1225 kmsan_check_arg(sizeof(ptr), \
1226 "atomic_load_" #name "():args"); \
1227 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1228 "atomic_load_" #name "():ptr"); \
1229 kmsan_init_ret(sizeof(type)); \
1230 return (atomic_load_##name(ptr)); \
1231 }
1232
1233 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \
1234 _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1235 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1236
1237 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \
1238 void kmsan_atomic_store_##name(volatile type *ptr, type val) \
1239 { \
1240 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1241 "atomic_store_" #name "():args"); \
1242 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \
1243 sizeof(type)); \
1244 atomic_store_##name(ptr, val); \
1245 }
1246
1247 #define MSAN_ATOMIC_FUNC_STORE(name, type) \
1248 _MSAN_ATOMIC_FUNC_STORE(name, type) \
1249 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1250
1251 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1252 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1253 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1254 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1255 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1256 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1257 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1258
1259 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1260 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1261 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1262 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1263 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1264 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1265 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1266
1267 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1268 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1269 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1270 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1271 MSAN_ATOMIC_FUNC_SET(int, u_int);
1272 MSAN_ATOMIC_FUNC_SET(long, u_long);
1273 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1274
1275 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1276 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1277 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1278 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1279 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1280 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1281 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1282
1283 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1284 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1285 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1286 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1287
1288 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1289 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1290 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1291 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1292 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1293
1294 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1295 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1296 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1297 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1298 MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t);
1299
1300 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1301 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1302 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1303 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1304 MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t);
1305
1306 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1307 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1308 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1309 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1310 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1311
1312 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1313 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1314 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1315 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1316 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1317 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1318 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1319
1320 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1321 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1322 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1323 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1324 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1325 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1326 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1327
1328 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1329 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1330 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1331 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1332 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1333 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1334 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1335 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1336 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1337 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1338
1339 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1340 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1341 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1342 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1343 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1344 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1345 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1346 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1347 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1348 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1349
1350 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1351 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1352 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1353 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1354
1355 void
kmsan_atomic_interrupt_fence(void)1356 kmsan_atomic_interrupt_fence(void)
1357 {
1358 atomic_interrupt_fence();
1359 }
1360
1361 /* -------------------------------------------------------------------------- */
1362
1363 #include <sys/bus.h>
1364 #include <machine/bus.h>
1365 #include <sys/bus_san.h>
1366
1367 int
kmsan_bus_space_map(bus_space_tag_t tag,bus_addr_t hnd,bus_size_t size,int flags,bus_space_handle_t * handlep)1368 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1369 int flags, bus_space_handle_t *handlep)
1370 {
1371 return (bus_space_map(tag, hnd, size, flags, handlep));
1372 }
1373
1374 void
kmsan_bus_space_unmap(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1375 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1376 bus_size_t size)
1377 {
1378 bus_space_unmap(tag, hnd, size);
1379 }
1380
1381 int
kmsan_bus_space_subregion(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,bus_space_handle_t * handlep)1382 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1383 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1384 {
1385 return (bus_space_subregion(tag, hnd, offset, size, handlep));
1386 }
1387
1388 void
kmsan_bus_space_free(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1389 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1390 bus_size_t size)
1391 {
1392 bus_space_free(tag, hnd, size);
1393 }
1394
1395 void
kmsan_bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,int flags)1396 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1397 bus_size_t offset, bus_size_t size, int flags)
1398 {
1399 bus_space_barrier(tag, hnd, offset, size, flags);
1400 }
1401
1402 #if defined(__amd64__)
1403 #define BUS_SPACE_IO(tag) ((tag) == X86_BUS_SPACE_IO)
1404 #else
1405 #define BUS_SPACE_IO(tag) (false)
1406 #endif
1407
1408 #define MSAN_BUS_READ_FUNC(func, width, type) \
1409 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
1410 bus_space_handle_t hnd, bus_size_t offset) \
1411 { \
1412 type ret; \
1413 if (!BUS_SPACE_IO(tag)) \
1414 kmsan_shadow_fill((uintptr_t)(hnd + offset), \
1415 KMSAN_STATE_INITED, (width)); \
1416 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1417 kmsan_init_ret(sizeof(type)); \
1418 return (ret); \
1419 } \
1420
1421 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \
1422 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1423 bus_space_handle_t hnd, bus_size_t size, type *buf, \
1424 bus_size_t count) \
1425 { \
1426 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \
1427 (width) * count); \
1428 bus_space_read_##func##_##width(tag, hnd, size, buf, \
1429 count); \
1430 }
1431
1432 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1433 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1434 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1435 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1436 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1437 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1438
1439 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1440 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1441 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1442 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1443 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1444 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1445
1446 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1447 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1448 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1449 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1450 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1451 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1452
1453 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1454 #ifndef __amd64__
1455 MSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
1456 MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
1457 MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
1458 MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
1459 MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
1460 #endif
1461
1462 #define MSAN_BUS_WRITE_FUNC(func, width, type) \
1463 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1464 bus_space_handle_t hnd, bus_size_t offset, type value) \
1465 { \
1466 bus_space_write##func##_##width(tag, hnd, offset, value);\
1467 } \
1468
1469 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
1470 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1471 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
1472 bus_size_t count) \
1473 { \
1474 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1475 "bus_space_write()"); \
1476 bus_space_write_##func##_##width(tag, hnd, size, buf, \
1477 count); \
1478 }
1479
1480 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1481 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1482 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1483 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1484 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1485 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1486
1487 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1488 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1489 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1490 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1491 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1492 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1493
1494 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1495 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1496 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1497 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1498 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1499 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1500
1501 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1502
1503 #define MSAN_BUS_SET_FUNC(func, width, type) \
1504 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1505 bus_space_handle_t hnd, bus_size_t offset, type value, \
1506 bus_size_t count) \
1507 { \
1508 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1509 count); \
1510 }
1511
1512 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1513 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1514 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1515 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1516
1517 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1518 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1519 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1520 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1521
1522 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1523 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1524 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1525 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1526
1527 #define MSAN_BUS_PEEK_FUNC(width, type) \
1528 int kmsan_bus_space_peek_##width(bus_space_tag_t tag, \
1529 bus_space_handle_t hnd, bus_size_t offset, type *value) \
1530 { \
1531 return (bus_space_peek_##width(tag, hnd, offset, value)); \
1532 }
1533
1534 MSAN_BUS_PEEK_FUNC(1, uint8_t)
1535 MSAN_BUS_PEEK_FUNC(2, uint16_t)
1536 MSAN_BUS_PEEK_FUNC(4, uint32_t)
1537
1538 #define MSAN_BUS_POKE_FUNC(width, type) \
1539 int kmsan_bus_space_poke_##width(bus_space_tag_t tag, \
1540 bus_space_handle_t hnd, bus_size_t offset, type value) \
1541 { \
1542 return (bus_space_poke_##width(tag, hnd, offset, value)); \
1543 }
1544
1545 MSAN_BUS_POKE_FUNC(1, uint8_t)
1546 MSAN_BUS_POKE_FUNC(2, uint16_t)
1547 MSAN_BUS_POKE_FUNC(4, uint32_t)
1548
1549 /* -------------------------------------------------------------------------- */
1550
1551 void
kmsan_bus_dmamap_sync(struct memdesc * desc,bus_dmasync_op_t op)1552 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1553 {
1554 /*
1555 * Some drivers, e.g., nvme, use the same code path for loading device
1556 * read and write requests, and will thus specify both flags. In this
1557 * case we should not do any checking since it will generally lead to
1558 * false positives.
1559 */
1560 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1561 BUS_DMASYNC_PREWRITE) {
1562 switch (desc->md_type) {
1563 case MEMDESC_VADDR:
1564 kmsan_check(desc->u.md_vaddr, desc->md_len,
1565 "dmasync");
1566 break;
1567 case MEMDESC_MBUF:
1568 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1569 break;
1570 case 0:
1571 break;
1572 default:
1573 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1574 desc->md_type);
1575 }
1576 }
1577 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1578 switch (desc->md_type) {
1579 case MEMDESC_VADDR:
1580 kmsan_mark(desc->u.md_vaddr, desc->md_len,
1581 KMSAN_STATE_INITED);
1582 break;
1583 case MEMDESC_MBUF:
1584 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1585 break;
1586 case 0:
1587 break;
1588 default:
1589 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1590 desc->md_type);
1591 }
1592 }
1593 }
1594