xref: /freebsd/sys/kern/subr_msan.c (revision a80b9ee15aa0f2da9f9f0c48e13324e0e5f2e840)
1 /*	$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $	*/
2 
3 /*
4  * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5  * All rights reserved.
6  * Copyright (c) 2021 The FreeBSD Foundation
7  *
8  * Portions of this software were developed by Mark Johnston under sponsorship
9  * from the FreeBSD Foundation.
10  *
11  * This code is part of the KMSAN subsystem of the NetBSD kernel.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 
35 #define	SAN_RUNTIME
36 
37 #include <sys/cdefs.h>
38 #if 0
39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
40 #endif
41 
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/bio.h>
45 #include <sys/buf.h>
46 #include <sys/conf.h>
47 #include <sys/kdb.h>
48 #include <sys/kernel.h>
49 #include <sys/linker.h>
50 #include <sys/malloc.h>
51 #include <sys/mbuf.h>
52 #include <sys/memdesc.h>
53 #include <sys/msan.h>
54 #include <sys/proc.h>
55 #include <sys/stack.h>
56 #include <sys/sysctl.h>
57 #include <sys/uio.h>
58 
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 
62 #include <machine/msan.h>
63 #include <machine/stdarg.h>
64 
65 void kmsan_init_arg(size_t);
66 void kmsan_init_ret(size_t);
67 
68 /* -------------------------------------------------------------------------- */
69 
70 /*
71  * Part of the compiler ABI.
72  */
73 
74 typedef struct {
75 	uint8_t *shad;
76 	msan_orig_t *orig;
77 } msan_meta_t;
78 
79 #define MSAN_PARAM_SIZE		800
80 #define MSAN_RETVAL_SIZE	800
81 typedef struct {
82 	uint8_t param_shadow[MSAN_PARAM_SIZE];
83 	uint8_t retval_shadow[MSAN_RETVAL_SIZE];
84 	uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
85 	uint8_t va_arg_origin[MSAN_PARAM_SIZE];
86 	uint64_t va_arg_overflow_size;
87 	msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
88 	msan_orig_t retval_origin;
89 } msan_tls_t;
90 
91 /* -------------------------------------------------------------------------- */
92 
93 #define MSAN_NCONTEXT	4
94 #define MSAN_ORIG_MASK	(~0x3)
95 
96 typedef struct kmsan_td {
97 	size_t ctx;
98 	msan_tls_t tls[MSAN_NCONTEXT];
99 } msan_td_t;
100 
101 static msan_tls_t dummy_tls;
102 
103 /*
104  * Use separate dummy regions for loads and stores: stores may mark the region
105  * as uninitialized, and that can trigger false positives.
106  */
107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
110 static msan_td_t msan_thread0;
111 static bool kmsan_reporting = false;
112 
113 /*
114  * Avoid clobbering any thread-local state before we panic.
115  */
116 #define	kmsan_panic(f, ...) do {			\
117 	kmsan_disabled = true;				\
118 	panic(f, __VA_ARGS__);				\
119 } while (0)
120 
121 #define	REPORT(f, ...) do {				\
122 	if (panic_on_violation) {			\
123 		kmsan_panic(f, __VA_ARGS__);		\
124 	} else {					\
125 		struct stack st;			\
126 							\
127 		stack_save(&st);			\
128 		printf(f "\n", __VA_ARGS__);		\
129 		stack_print_ddb(&st);			\
130 	}						\
131 } while (0)
132 
133 FEATURE(kmsan, "Kernel memory sanitizer");
134 
135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
136     "KMSAN options");
137 
138 static bool panic_on_violation = 1;
139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
140     &panic_on_violation, 0,
141     "Panic if an invalid access is detected");
142 
143 static bool kmsan_disabled __read_mostly = true;
144 #define kmsan_enabled (!kmsan_disabled)
145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
146     &kmsan_disabled, 0, "KMSAN is disabled");
147 
148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
149 
150 /* -------------------------------------------------------------------------- */
151 
152 static inline const char *
kmsan_orig_name(int type)153 kmsan_orig_name(int type)
154 {
155 	switch (type) {
156 	case KMSAN_TYPE_STACK:
157 		return ("stack");
158 	case KMSAN_TYPE_KMEM:
159 		return ("kmem");
160 	case KMSAN_TYPE_MALLOC:
161 		return ("malloc");
162 	case KMSAN_TYPE_UMA:
163 		return ("UMA");
164 	default:
165 		return ("unknown");
166 	}
167 }
168 
169 static void
kmsan_report_hook(const void * addr,msan_orig_t * orig,size_t size,size_t off,const char * hook)170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
171     const char *hook)
172 {
173 	const char *typename;
174 	char *var, *fn;
175 	uintptr_t ptr;
176 	long foff;
177 	char buf[128];
178 	int type;
179 
180 	if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
181 		return;
182 
183 	kmsan_reporting = true;
184 	__compiler_membar();
185 
186 	if (*orig == 0) {
187 		REPORT("MSan: Uninitialized memory in %s, offset %zu",
188 		    hook, off);
189 		goto out;
190 	}
191 
192 	kmsan_md_orig_decode(*orig, &type, &ptr);
193 	typename = kmsan_orig_name(type);
194 
195 	if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
196 	    sizeof(buf), &foff) == 0) {
197 		REPORT("MSan: Uninitialized %s memory in %s, "
198 		    "offset %zu/%zu, addr %p, from %s+%#lx",
199 		    typename, hook, off, size, addr, buf, foff);
200 	} else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
201 		/*
202 		 * The format of the string is: "----var@function". Parse it to
203 		 * display a nice warning.
204 		 */
205 		var = (char *)ptr + 4;
206 		strlcpy(buf, var, sizeof(buf));
207 		var = buf;
208 		fn = strchr(buf, '@');
209 		*fn++ = '\0';
210 		REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
211 		    "variable '%s' from %s", typename, hook, off, var, fn);
212 	} else {
213 		REPORT("MSan: Uninitialized %s memory in %s, "
214 		    "offset %zu/%zu, addr %p, PC %p",
215 		    typename, hook, off, size, addr, (void *)ptr);
216 	}
217 
218 out:
219 	__compiler_membar();
220 	kmsan_reporting = false;
221 }
222 
223 static void
kmsan_report_inline(msan_orig_t orig,unsigned long pc)224 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
225 {
226 	const char *typename;
227 	char *var, *fn;
228 	uintptr_t ptr;
229 	char buf[128];
230 	long foff;
231 	int type;
232 
233 	if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
234 		return;
235 
236 	kmsan_reporting = true;
237 	__compiler_membar();
238 
239 	if (orig == 0) {
240 		REPORT("MSan: uninitialized variable in %p", (void *)pc);
241 		goto out;
242 	}
243 
244 	kmsan_md_orig_decode(orig, &type, &ptr);
245 	typename = kmsan_orig_name(type);
246 
247 	if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
248 	    sizeof(buf), &foff) == 0) {
249 		REPORT("MSan: Uninitialized %s memory from %s+%#lx",
250 		    typename, buf, foff);
251 	} else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
252 		/*
253 		 * The format of the string is: "----var@function". Parse it to
254 		 * display a nice warning.
255 		 */
256 		var = (char *)ptr + 4;
257 		strlcpy(buf, var, sizeof(buf));
258 		var = buf;
259 		fn = strchr(buf, '@');
260 		*fn++ = '\0';
261 		REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
262 	} else {
263 		REPORT("MSan: Uninitialized %s memory, origin %x",
264 		    typename, orig);
265 	}
266 
267 out:
268 	__compiler_membar();
269 	kmsan_reporting = false;
270 }
271 
272 /* -------------------------------------------------------------------------- */
273 
274 static inline msan_meta_t
kmsan_meta_get(const void * addr,size_t size,const bool write)275 kmsan_meta_get(const void *addr, size_t size, const bool write)
276 {
277 	msan_meta_t ret;
278 
279 	if (__predict_false(!kmsan_enabled)) {
280 		ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
281 		ret.orig = (msan_orig_t *)msan_dummy_orig;
282 	} else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
283 		ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
284 		ret.orig = (msan_orig_t *)msan_dummy_orig;
285 	} else {
286 		ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
287 		ret.orig =
288 		    (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
289 		ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
290 		    MSAN_ORIG_MASK);
291 	}
292 
293 	return (ret);
294 }
295 
296 static inline void
kmsan_origin_fill(const void * addr,msan_orig_t o,size_t size)297 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
298 {
299 	msan_orig_t *orig;
300 	size_t i;
301 
302 	if (__predict_false(!kmsan_enabled))
303 		return;
304 	if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
305 		return;
306 
307 	orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
308 	size += ((uintptr_t)orig & (sizeof(*orig) - 1));
309 	orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
310 
311 	for (i = 0; i < size; i += 4) {
312 		orig[i / 4] = o;
313 	}
314 }
315 
316 static inline void
kmsan_shadow_fill(uintptr_t addr,uint8_t c,size_t size)317 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
318 {
319 	uint8_t *shad;
320 
321 	if (__predict_false(!kmsan_enabled))
322 		return;
323 	if (__predict_false(kmsan_md_unsupported(addr)))
324 		return;
325 
326 	shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
327 	__builtin_memset(shad, c, size);
328 }
329 
330 static inline void
kmsan_meta_copy(void * dst,const void * src,size_t size)331 kmsan_meta_copy(void *dst, const void *src, size_t size)
332 {
333 	uint8_t *orig_src, *orig_dst;
334 	uint8_t *shad_src, *shad_dst;
335 	msan_orig_t *_src, *_dst;
336 	size_t i;
337 
338 	if (__predict_false(!kmsan_enabled))
339 		return;
340 	if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
341 		return;
342 	if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
343 		kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
344 		return;
345 	}
346 
347 	shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
348 	shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
349 	__builtin_memmove(shad_dst, shad_src, size);
350 
351 	orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
352 	orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
353 	for (i = 0; i < size; i++) {
354 		_src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
355 		_dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
356 		*_dst = *_src;
357 		orig_src++;
358 		orig_dst++;
359 	}
360 }
361 
362 static inline void
kmsan_shadow_check(uintptr_t addr,size_t size,const char * hook)363 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
364 {
365 	msan_orig_t *orig;
366 	uint8_t *shad;
367 	size_t i;
368 
369 	if (__predict_false(!kmsan_enabled))
370 		return;
371 	if (__predict_false(kmsan_md_unsupported(addr)))
372 		return;
373 
374 	shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
375 	for (i = 0; i < size; i++) {
376 		if (__predict_true(shad[i] == 0))
377 			continue;
378 		orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i);
379 		orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
380 		kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
381 		break;
382 	}
383 }
384 
385 void
kmsan_init_arg(size_t n)386 kmsan_init_arg(size_t n)
387 {
388 	msan_td_t *mtd;
389 	uint8_t *arg;
390 
391 	if (__predict_false(!kmsan_enabled))
392 		return;
393 	if (__predict_false(curthread == NULL))
394 		return;
395 	mtd = curthread->td_kmsan;
396 	arg = mtd->tls[mtd->ctx].param_shadow;
397 	__builtin_memset(arg, 0, n);
398 }
399 
400 void
kmsan_init_ret(size_t n)401 kmsan_init_ret(size_t n)
402 {
403 	msan_td_t *mtd;
404 	uint8_t *arg;
405 
406 	if (__predict_false(!kmsan_enabled))
407 		return;
408 	if (__predict_false(curthread == NULL))
409 		return;
410 	mtd = curthread->td_kmsan;
411 	arg = mtd->tls[mtd->ctx].retval_shadow;
412 	__builtin_memset(arg, 0, n);
413 }
414 
415 static void
kmsan_check_arg(size_t size,const char * hook)416 kmsan_check_arg(size_t size, const char *hook)
417 {
418 	msan_orig_t *orig;
419 	msan_td_t *mtd;
420 	uint8_t *arg;
421 	size_t ctx, i;
422 
423 	if (__predict_false(!kmsan_enabled))
424 		return;
425 	if (__predict_false(curthread == NULL))
426 		return;
427 	mtd = curthread->td_kmsan;
428 	ctx = mtd->ctx;
429 	arg = mtd->tls[ctx].param_shadow;
430 
431 	for (i = 0; i < size; i++) {
432 		if (__predict_true(arg[i] == 0))
433 			continue;
434 		orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
435 		kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
436 		break;
437 	}
438 }
439 
440 void
kmsan_thread_alloc(struct thread * td)441 kmsan_thread_alloc(struct thread *td)
442 {
443 	msan_td_t *mtd;
444 
445 	if (!kmsan_enabled)
446 		return;
447 
448 	mtd = td->td_kmsan;
449 	if (mtd == NULL) {
450 		/* We might be recycling a thread. */
451 		kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
452 		    sizeof(int));
453 		mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
454 	}
455 	__builtin_memset(mtd, 0, sizeof(*mtd));
456 	mtd->ctx = 0;
457 
458 	if (td->td_kstack != 0)
459 		kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
460 		    KMSAN_STATE_UNINIT);
461 
462 	td->td_kmsan = mtd;
463 }
464 
465 void
kmsan_thread_free(struct thread * td)466 kmsan_thread_free(struct thread *td)
467 {
468 	msan_td_t *mtd;
469 
470 	if (!kmsan_enabled)
471 		return;
472 	if (__predict_false(td == curthread))
473 		kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
474 
475 	mtd = td->td_kmsan;
476 	kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
477 	free(mtd, M_KMSAN);
478 	td->td_kmsan = NULL;
479 }
480 
481 void kmsan_intr_enter(void);
482 void kmsan_intr_leave(void);
483 
484 void
kmsan_intr_enter(void)485 kmsan_intr_enter(void)
486 {
487 	msan_td_t *mtd;
488 
489 	if (__predict_false(!kmsan_enabled))
490 		return;
491 
492 	mtd = curthread->td_kmsan;
493 	mtd->ctx++;
494 	if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
495 		kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
496 }
497 
498 void
kmsan_intr_leave(void)499 kmsan_intr_leave(void)
500 {
501 	msan_td_t *mtd;
502 
503 	if (__predict_false(!kmsan_enabled))
504 		return;
505 
506 	mtd = curthread->td_kmsan;
507 	if (__predict_false(mtd->ctx == 0))
508 		kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
509 	mtd->ctx--;
510 }
511 
512 /* -------------------------------------------------------------------------- */
513 
514 void
kmsan_shadow_map(vm_offset_t addr,size_t size)515 kmsan_shadow_map(vm_offset_t addr, size_t size)
516 {
517 	size_t npages, i;
518 	vm_offset_t va;
519 
520 	MPASS(addr % PAGE_SIZE == 0);
521 	MPASS(size % PAGE_SIZE == 0);
522 
523 	if (!kmsan_enabled)
524 		return;
525 
526 	npages = atop(size);
527 
528 	va = kmsan_md_addr_to_shad(addr);
529 	for (i = 0; i < npages; i++) {
530 		pmap_san_enter(va + ptoa(i));
531 	}
532 
533 	va = kmsan_md_addr_to_orig(addr);
534 	for (i = 0; i < npages; i++) {
535 		pmap_san_enter(va + ptoa(i));
536 	}
537 }
538 
539 void
kmsan_orig(const void * addr,size_t size,int type,uintptr_t pc)540 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
541 {
542 	msan_orig_t orig;
543 
544 	orig = kmsan_md_orig_encode(type, pc);
545 	kmsan_origin_fill(addr, orig, size);
546 }
547 
548 void
kmsan_mark(const void * addr,size_t size,uint8_t c)549 kmsan_mark(const void *addr, size_t size, uint8_t c)
550 {
551 	kmsan_shadow_fill((uintptr_t)addr, c, size);
552 }
553 
554 void
kmsan_mark_bio(const struct bio * bp,uint8_t c)555 kmsan_mark_bio(const struct bio *bp, uint8_t c)
556 {
557 	kmsan_mark(bp->bio_data, bp->bio_length, c);
558 }
559 
560 void
kmsan_mark_mbuf(const struct mbuf * m,uint8_t c)561 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
562 {
563 	do {
564 		if ((m->m_flags & M_EXTPG) == 0)
565 			kmsan_mark(m->m_data, m->m_len, c);
566 		m = m->m_next;
567 	} while (m != NULL);
568 }
569 
570 void
kmsan_check(const void * p,size_t sz,const char * descr)571 kmsan_check(const void *p, size_t sz, const char *descr)
572 {
573 	kmsan_shadow_check((uintptr_t)p, sz, descr);
574 }
575 
576 void
kmsan_check_bio(const struct bio * bp,const char * descr)577 kmsan_check_bio(const struct bio *bp, const char *descr)
578 {
579 	kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
580 }
581 
582 void
kmsan_check_mbuf(const struct mbuf * m,const char * descr)583 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
584 {
585 	do {
586 		kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
587 	} while ((m = m->m_next) != NULL);
588 }
589 
590 void
kmsan_check_uio(const struct uio * uio,const char * descr)591 kmsan_check_uio(const struct uio *uio, const char *descr)
592 {
593 	for (int i = 0; i < uio->uio_iovcnt; i++) {
594 		kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len,
595 		    descr);
596 	}
597 }
598 
599 void
kmsan_init(void)600 kmsan_init(void)
601 {
602 	int disabled;
603 
604 	disabled = 0;
605 	TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
606 	if (disabled)
607 		return;
608 
609 	/* Initialize the TLS for curthread. */
610 	msan_thread0.ctx = 0;
611 	thread0.td_kmsan = &msan_thread0;
612 
613 	/* Now officially enabled. */
614 	kmsan_disabled = false;
615 }
616 
617 /* -------------------------------------------------------------------------- */
618 
619 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
620 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
621 
622 msan_meta_t
__msan_metadata_ptr_for_load_n(void * addr,size_t size)623 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
624 {
625 	return (kmsan_meta_get(addr, size, false));
626 }
627 
628 msan_meta_t
__msan_metadata_ptr_for_store_n(void * addr,size_t size)629 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
630 {
631 	return (kmsan_meta_get(addr, size, true));
632 }
633 
634 #define MSAN_META_FUNC(size)						\
635 	msan_meta_t __msan_metadata_ptr_for_load_##size(void *);	\
636 	msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr)	\
637 	{								\
638 		return (kmsan_meta_get(addr, size, false));		\
639 	}								\
640 	msan_meta_t __msan_metadata_ptr_for_store_##size(void *);	\
641 	msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr)	\
642 	{								\
643 		return (kmsan_meta_get(addr, size, true));		\
644 	}
645 
646 MSAN_META_FUNC(1)
647 MSAN_META_FUNC(2)
648 MSAN_META_FUNC(4)
649 MSAN_META_FUNC(8)
650 
651 void __msan_instrument_asm_store(const void *, size_t);
652 msan_orig_t __msan_chain_origin(msan_orig_t);
653 void __msan_poison(const void *, size_t);
654 void __msan_unpoison(const void *, size_t);
655 void __msan_poison_alloca(const void *, uint64_t, const char *);
656 void __msan_unpoison_alloca(const void *, uint64_t);
657 void __msan_warning(msan_orig_t);
658 msan_tls_t *__msan_get_context_state(void);
659 
660 void
__msan_instrument_asm_store(const void * addr,size_t size)661 __msan_instrument_asm_store(const void *addr, size_t size)
662 {
663 	kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
664 }
665 
666 msan_orig_t
__msan_chain_origin(msan_orig_t origin)667 __msan_chain_origin(msan_orig_t origin)
668 {
669 	return (origin);
670 }
671 
672 void
__msan_poison(const void * addr,size_t size)673 __msan_poison(const void *addr, size_t size)
674 {
675 	kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
676 }
677 
678 void
__msan_unpoison(const void * addr,size_t size)679 __msan_unpoison(const void *addr, size_t size)
680 {
681 	kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
682 }
683 
684 void
__msan_poison_alloca(const void * addr,uint64_t size,const char * descr)685 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
686 {
687 	msan_orig_t orig;
688 
689 	orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
690 	kmsan_origin_fill(addr, orig, size);
691 	kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
692 }
693 
694 void
__msan_unpoison_alloca(const void * addr,uint64_t size)695 __msan_unpoison_alloca(const void *addr, uint64_t size)
696 {
697 	kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
698 }
699 
700 void
__msan_warning(msan_orig_t origin)701 __msan_warning(msan_orig_t origin)
702 {
703 	if (__predict_false(!kmsan_enabled))
704 		return;
705 	kmsan_report_inline(origin, KMSAN_RET_ADDR);
706 }
707 
708 msan_tls_t *
__msan_get_context_state(void)709 __msan_get_context_state(void)
710 {
711 	msan_td_t *mtd;
712 
713 	/*
714 	 * When APs are started, they execute some C code before curthread is
715 	 * set.  We have to handle that here.
716 	 */
717 	if (__predict_false(!kmsan_enabled || curthread == NULL))
718 		return (&dummy_tls);
719 	mtd = curthread->td_kmsan;
720 	return (&mtd->tls[mtd->ctx]);
721 }
722 
723 /* -------------------------------------------------------------------------- */
724 
725 /*
726  * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
727  * initialized areas properly.
728  */
729 
730 void *
kmsan_memcpy(void * dst,const void * src,size_t len)731 kmsan_memcpy(void *dst, const void *src, size_t len)
732 {
733 	/* No kmsan_check_arg, because inlined. */
734 	kmsan_init_ret(sizeof(void *));
735 	if (__predict_true(len != 0)) {
736 		kmsan_meta_copy(dst, src, len);
737 	}
738 	return (__builtin_memcpy(dst, src, len));
739 }
740 
741 int
kmsan_memcmp(const void * b1,const void * b2,size_t len)742 kmsan_memcmp(const void *b1, const void *b2, size_t len)
743 {
744 	const uint8_t *_b1 = b1, *_b2 = b2;
745 	size_t i;
746 
747 	kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
748 	    "memcmp():args");
749 	kmsan_init_ret(sizeof(int));
750 
751 	for (i = 0; i < len; i++) {
752 		if (*_b1 != *_b2) {
753 			kmsan_shadow_check((uintptr_t)b1, i + 1,
754 			    "memcmp():arg1");
755 			kmsan_shadow_check((uintptr_t)b2, i + 1,
756 			    "memcmp():arg2");
757 			return (*_b1 - *_b2);
758 		}
759 		_b1++, _b2++;
760 	}
761 
762 	return (0);
763 }
764 
765 void *
kmsan_memset(void * dst,int c,size_t len)766 kmsan_memset(void *dst, int c, size_t len)
767 {
768 	/* No kmsan_check_arg, because inlined. */
769 	kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
770 	kmsan_init_ret(sizeof(void *));
771 	return (__builtin_memset(dst, c, len));
772 }
773 
774 void *
kmsan_memmove(void * dst,const void * src,size_t len)775 kmsan_memmove(void *dst, const void *src, size_t len)
776 {
777 	/* No kmsan_check_arg, because inlined. */
778 	if (__predict_true(len != 0)) {
779 		kmsan_meta_copy(dst, src, len);
780 	}
781 	kmsan_init_ret(sizeof(void *));
782 	return (__builtin_memmove(dst, src, len));
783 }
784 
785 __strong_reference(kmsan_memcpy, __msan_memcpy);
786 __strong_reference(kmsan_memset, __msan_memset);
787 __strong_reference(kmsan_memmove, __msan_memmove);
788 
789 char *
kmsan_strcpy(char * dst,const char * src)790 kmsan_strcpy(char *dst, const char *src)
791 {
792 	const char *_src = src;
793 	char *_dst = dst;
794 	size_t len = 0;
795 
796 	kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
797 
798 	while (1) {
799 		len++;
800 		*dst = *src;
801 		if (*src == '\0')
802 			break;
803 		src++, dst++;
804 	}
805 
806 	kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
807 	kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
808 	kmsan_init_ret(sizeof(char *));
809 	return (_dst);
810 }
811 
812 int
kmsan_strcmp(const char * s1,const char * s2)813 kmsan_strcmp(const char *s1, const char *s2)
814 {
815 	const char *_s1 = s1, *_s2 = s2;
816 	size_t len = 0;
817 
818 	kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
819 	kmsan_init_ret(sizeof(int));
820 
821 	while (1) {
822 		len++;
823 		if (*s1 != *s2)
824 			break;
825 		if (*s1 == '\0') {
826 			kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
827 			kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
828 			return (0);
829 		}
830 		s1++, s2++;
831 	}
832 
833 	kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
834 	kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
835 
836 	return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
837 }
838 
839 size_t
kmsan_strlen(const char * str)840 kmsan_strlen(const char *str)
841 {
842 	const char *s;
843 
844 	kmsan_check_arg(sizeof(str), "strlen():args");
845 
846 	s = str;
847 	while (1) {
848 		if (*s == '\0')
849 			break;
850 		s++;
851 	}
852 
853 	kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
854 	kmsan_init_ret(sizeof(size_t));
855 	return (s - str);
856 }
857 
858 int	kmsan_copyin(const void *, void *, size_t);
859 int	kmsan_copyout(const void *, void *, size_t);
860 int	kmsan_copyinstr(const void *, void *, size_t, size_t *);
861 
862 int
kmsan_copyin(const void * uaddr,void * kaddr,size_t len)863 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
864 {
865 	int ret;
866 
867 	kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
868 	    "copyin():args");
869 	ret = copyin(uaddr, kaddr, len);
870 	if (ret == 0)
871 		kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
872 	kmsan_init_ret(sizeof(int));
873 	return (ret);
874 }
875 
876 int
kmsan_copyout(const void * kaddr,void * uaddr,size_t len)877 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
878 {
879 	kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
880 	    "copyout():args");
881 	kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
882 	kmsan_init_ret(sizeof(int));
883 	return (copyout(kaddr, uaddr, len));
884 }
885 
886 int
kmsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)887 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
888 {
889 	size_t _done;
890 	int ret;
891 
892 	kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
893 	    sizeof(len) + sizeof(done), "copyinstr():args");
894 	ret = copyinstr(uaddr, kaddr, len, &_done);
895 	if (ret == 0)
896 		kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
897 	if (done != NULL) {
898 		*done = _done;
899 		kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
900 	}
901 	kmsan_init_ret(sizeof(int));
902 	return (ret);
903 }
904 
905 /* -------------------------------------------------------------------------- */
906 
907 int
kmsan_fubyte(volatile const void * base)908 kmsan_fubyte(volatile const void *base)
909 {
910 	int ret;
911 
912 	kmsan_check_arg(sizeof(base), "fubyte(): args");
913 	ret = fubyte(base);
914 	kmsan_init_ret(sizeof(int));
915 	return (ret);
916 }
917 
918 int
kmsan_fuword16(volatile const void * base)919 kmsan_fuword16(volatile const void *base)
920 {
921 	int ret;
922 
923 	kmsan_check_arg(sizeof(base), "fuword16(): args");
924 	ret = fuword16(base);
925 	kmsan_init_ret(sizeof(int));
926 	return (ret);
927 }
928 
929 int
kmsan_fueword(volatile const void * base,long * val)930 kmsan_fueword(volatile const void *base, long *val)
931 {
932 	int ret;
933 
934 	kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
935 	ret = fueword(base, val);
936 	if (ret == 0)
937 		kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
938 		    sizeof(*val));
939 	kmsan_init_ret(sizeof(int));
940 	return (ret);
941 }
942 
943 int
kmsan_fueword32(volatile const void * base,int32_t * val)944 kmsan_fueword32(volatile const void *base, int32_t *val)
945 {
946 	int ret;
947 
948 	kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
949 	ret = fueword32(base, val);
950 	if (ret == 0)
951 		kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
952 		    sizeof(*val));
953 	kmsan_init_ret(sizeof(int));
954 	return (ret);
955 }
956 
957 int
kmsan_fueword64(volatile const void * base,int64_t * val)958 kmsan_fueword64(volatile const void *base, int64_t *val)
959 {
960 	int ret;
961 
962 	kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
963 	ret = fueword64(base, val);
964 	if (ret == 0)
965 		kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
966 		    sizeof(*val));
967 	kmsan_init_ret(sizeof(int));
968 	return (ret);
969 }
970 
971 int
kmsan_subyte(volatile void * base,int byte)972 kmsan_subyte(volatile void *base, int byte)
973 {
974 	int ret;
975 
976 	kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
977 	ret = subyte(base, byte);
978 	kmsan_init_ret(sizeof(int));
979 	return (ret);
980 }
981 
982 int
kmsan_suword(volatile void * base,long word)983 kmsan_suword(volatile void *base, long word)
984 {
985 	int ret;
986 
987 	kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
988 	ret = suword(base, word);
989 	kmsan_init_ret(sizeof(int));
990 	return (ret);
991 }
992 
993 int
kmsan_suword16(volatile void * base,int word)994 kmsan_suword16(volatile void *base, int word)
995 {
996 	int ret;
997 
998 	kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
999 	ret = suword16(base, word);
1000 	kmsan_init_ret(sizeof(int));
1001 	return (ret);
1002 }
1003 
1004 int
kmsan_suword32(volatile void * base,int32_t word)1005 kmsan_suword32(volatile void *base, int32_t word)
1006 {
1007 	int ret;
1008 
1009 	kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1010 	ret = suword32(base, word);
1011 	kmsan_init_ret(sizeof(int));
1012 	return (ret);
1013 }
1014 
1015 int
kmsan_suword64(volatile void * base,int64_t word)1016 kmsan_suword64(volatile void *base, int64_t word)
1017 {
1018 	int ret;
1019 
1020 	kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1021 	ret = suword64(base, word);
1022 	kmsan_init_ret(sizeof(int));
1023 	return (ret);
1024 }
1025 
1026 int
kmsan_casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)1027 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1028     uint32_t newval)
1029 {
1030 	int ret;
1031 
1032 	kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1033 	    sizeof(newval), "casueword32(): args");
1034 	ret = casueword32(base, oldval, oldvalp, newval);
1035 	kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1036 	    sizeof(*oldvalp));
1037 	kmsan_init_ret(sizeof(int));
1038 	return (ret);
1039 }
1040 
1041 int
kmsan_casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)1042 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1043     u_long newval)
1044 {
1045 	int ret;
1046 
1047 	kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1048 	    sizeof(newval), "casueword32(): args");
1049 	ret = casueword(base, oldval, oldvalp, newval);
1050 	kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1051 	    sizeof(*oldvalp));
1052 	kmsan_init_ret(sizeof(int));
1053 	return (ret);
1054 }
1055 
1056 /* -------------------------------------------------------------------------- */
1057 
1058 #include <machine/atomic.h>
1059 #include <sys/atomic_san.h>
1060 
1061 #define _MSAN_ATOMIC_FUNC_ADD(name, type)				\
1062 	void kmsan_atomic_add_##name(volatile type *ptr, type val)	\
1063 	{								\
1064 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1065 		    "atomic_add_" #name "():args");			\
1066 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1067 		    "atomic_add_" #name "():ptr");			\
1068 		atomic_add_##name(ptr, val);				\
1069 	}
1070 
1071 #define	MSAN_ATOMIC_FUNC_ADD(name, type)				\
1072 	_MSAN_ATOMIC_FUNC_ADD(name, type)				\
1073 	_MSAN_ATOMIC_FUNC_ADD(acq_##name, type)				\
1074 	_MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1075 
1076 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)				\
1077 	void kmsan_atomic_subtract_##name(volatile type *ptr, type val)	\
1078 	{								\
1079 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1080 		    "atomic_subtract_" #name "():args");		\
1081 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1082 		    "atomic_subtract_" #name "():ptr");			\
1083 		atomic_subtract_##name(ptr, val);			\
1084 	}
1085 
1086 #define	MSAN_ATOMIC_FUNC_SUBTRACT(name, type)				\
1087 	_MSAN_ATOMIC_FUNC_SUBTRACT(name, type)				\
1088 	_MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)			\
1089 	_MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1090 
1091 #define _MSAN_ATOMIC_FUNC_SET(name, type)				\
1092 	void kmsan_atomic_set_##name(volatile type *ptr, type val)	\
1093 	{								\
1094 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1095 		    "atomic_set_" #name "():args");			\
1096 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1097 		    "atomic_set_" #name "():ptr");			\
1098 		atomic_set_##name(ptr, val);				\
1099 	}
1100 
1101 #define	MSAN_ATOMIC_FUNC_SET(name, type)				\
1102 	_MSAN_ATOMIC_FUNC_SET(name, type)				\
1103 	_MSAN_ATOMIC_FUNC_SET(acq_##name, type)				\
1104 	_MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1105 
1106 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type)				\
1107 	void kmsan_atomic_clear_##name(volatile type *ptr, type val)	\
1108 	{								\
1109 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1110 		    "atomic_clear_" #name "():args");			\
1111 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1112 		    "atomic_clear_" #name "():ptr");			\
1113 		atomic_clear_##name(ptr, val);				\
1114 	}
1115 
1116 #define	MSAN_ATOMIC_FUNC_CLEAR(name, type)				\
1117 	_MSAN_ATOMIC_FUNC_CLEAR(name, type)				\
1118 	_MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)			\
1119 	_MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1120 
1121 #define	MSAN_ATOMIC_FUNC_FETCHADD(name, type)				\
1122 	type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val)	\
1123 	{								\
1124 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1125 		    "atomic_fetchadd_" #name "():args");		\
1126 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1127 		    "atomic_fetchadd_" #name "():ptr");			\
1128 		kmsan_init_ret(sizeof(type));				\
1129 		return (atomic_fetchadd_##name(ptr, val));		\
1130 	}
1131 
1132 #define	MSAN_ATOMIC_FUNC_READANDCLEAR(name, type)			\
1133 	type kmsan_atomic_readandclear_##name(volatile type *ptr)	\
1134 	{								\
1135 		kmsan_check_arg(sizeof(ptr),				\
1136 		    "atomic_readandclear_" #name "():args");		\
1137 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1138 		    "atomic_readandclear_" #name "():ptr");		\
1139 		kmsan_init_ret(sizeof(type));				\
1140 		return (atomic_readandclear_##name(ptr));		\
1141 	}
1142 
1143 #define	MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)			\
1144 	int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1145 	{								\
1146 		kmsan_check_arg(sizeof(ptr) + sizeof(v),		\
1147 		    "atomic_testandclear_" #name "():args");		\
1148 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1149 		    "atomic_testandclear_" #name "():ptr");		\
1150 		kmsan_init_ret(sizeof(int));				\
1151 		return (atomic_testandclear_##name(ptr, v));		\
1152 	}
1153 
1154 #define	MSAN_ATOMIC_FUNC_TESTANDSET(name, type)				\
1155 	int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1156 	{								\
1157 		kmsan_check_arg(sizeof(ptr) + sizeof(v),		\
1158 		    "atomic_testandset_" #name "():args");		\
1159 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1160 		    "atomic_testandset_" #name "():ptr");		\
1161 		kmsan_init_ret(sizeof(int));				\
1162 		return (atomic_testandset_##name(ptr, v));		\
1163 	}
1164 
1165 #define	MSAN_ATOMIC_FUNC_SWAP(name, type)				\
1166 	type kmsan_atomic_swap_##name(volatile type *ptr, type val)	\
1167 	{								\
1168 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1169 		    "atomic_swap_" #name "():args");			\
1170 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1171 		    "atomic_swap_" #name "():ptr");			\
1172 		kmsan_init_ret(sizeof(type));				\
1173 		return (atomic_swap_##name(ptr, val));			\
1174 	}
1175 
1176 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type)				\
1177 	int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval,	\
1178 	    type nval)							\
1179 	{								\
1180 		kmsan_check_arg(sizeof(ptr) + sizeof(oval) +		\
1181 		    sizeof(nval), "atomic_cmpset_" #name "():args");	\
1182 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1183 		    "atomic_cmpset_" #name "():ptr");			\
1184 		kmsan_init_ret(sizeof(int));				\
1185 		return (atomic_cmpset_##name(ptr, oval, nval));		\
1186 	}
1187 
1188 #define	MSAN_ATOMIC_FUNC_CMPSET(name, type)				\
1189 	_MSAN_ATOMIC_FUNC_CMPSET(name, type)				\
1190 	_MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)			\
1191 	_MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1192 
1193 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type)				\
1194 	int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval,	\
1195 	    type nval)							\
1196 	{								\
1197 		kmsan_check_arg(sizeof(ptr) + sizeof(oval) +		\
1198 		    sizeof(nval), "atomic_fcmpset_" #name "():args");	\
1199 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1200 		    "atomic_fcmpset_" #name "():ptr");			\
1201 		kmsan_init_ret(sizeof(int));				\
1202 		return (atomic_fcmpset_##name(ptr, oval, nval));	\
1203 	}
1204 
1205 #define	MSAN_ATOMIC_FUNC_FCMPSET(name, type)				\
1206 	_MSAN_ATOMIC_FUNC_FCMPSET(name, type)				\
1207 	_MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)			\
1208 	_MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1209 
1210 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name)				\
1211 	void kmsan_atomic_thread_fence_##name(void)			\
1212 	{								\
1213 		atomic_thread_fence_##name();				\
1214 	}
1215 
1216 #define	_MSAN_ATOMIC_FUNC_LOAD(name, type)				\
1217 	type kmsan_atomic_load_##name(volatile type *ptr)		\
1218 	{								\
1219 		kmsan_check_arg(sizeof(ptr),				\
1220 		    "atomic_load_" #name "():args");			\
1221 		kmsan_shadow_check((uintptr_t)ptr, sizeof(type),	\
1222 		    "atomic_load_" #name "():ptr");			\
1223 		kmsan_init_ret(sizeof(type));				\
1224 		return (atomic_load_##name(ptr));			\
1225 	}
1226 
1227 #define	MSAN_ATOMIC_FUNC_LOAD(name, type)				\
1228 	_MSAN_ATOMIC_FUNC_LOAD(name, type)				\
1229 	_MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1230 
1231 #define	_MSAN_ATOMIC_FUNC_STORE(name, type)				\
1232 	void kmsan_atomic_store_##name(volatile type *ptr, type val)	\
1233 	{								\
1234 		kmsan_check_arg(sizeof(ptr) + sizeof(val),		\
1235 		    "atomic_store_" #name "():args");			\
1236 		kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED,	\
1237 		    sizeof(type));					\
1238 		atomic_store_##name(ptr, val);				\
1239 	}
1240 
1241 #define	MSAN_ATOMIC_FUNC_STORE(name, type)				\
1242 	_MSAN_ATOMIC_FUNC_STORE(name, type)				\
1243 	_MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1244 
1245 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1246 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1247 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1248 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1249 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1250 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1251 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1252 
1253 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1254 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1255 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1256 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1257 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1258 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1259 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1260 
1261 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1262 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1263 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1264 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1265 MSAN_ATOMIC_FUNC_SET(int, u_int);
1266 MSAN_ATOMIC_FUNC_SET(long, u_long);
1267 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1268 
1269 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1270 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1271 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1272 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1273 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1274 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1275 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1276 
1277 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1278 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1279 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1280 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1281 
1282 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1283 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1284 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1285 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1286 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1287 
1288 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1289 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1290 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1291 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1292 MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t);
1293 
1294 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1295 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1296 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1297 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1298 MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t);
1299 
1300 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1301 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1302 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1303 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1304 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1305 
1306 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1307 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1308 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1309 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1310 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1311 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1312 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1313 
1314 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1315 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1316 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1317 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1318 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1319 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1320 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1321 
1322 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1323 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1324 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1325 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1326 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1327 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1328 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1329 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1330 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1331 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1332 
1333 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1334 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1335 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1336 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1337 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1338 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1339 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1340 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1341 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1342 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1343 
1344 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1345 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1346 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1347 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1348 
1349 void
kmsan_atomic_interrupt_fence(void)1350 kmsan_atomic_interrupt_fence(void)
1351 {
1352 	atomic_interrupt_fence();
1353 }
1354 
1355 /* -------------------------------------------------------------------------- */
1356 
1357 #include <sys/bus.h>
1358 #include <machine/bus.h>
1359 #include <sys/bus_san.h>
1360 
1361 int
kmsan_bus_space_map(bus_space_tag_t tag,bus_addr_t hnd,bus_size_t size,int flags,bus_space_handle_t * handlep)1362 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1363     int flags, bus_space_handle_t *handlep)
1364 {
1365 	return (bus_space_map(tag, hnd, size, flags, handlep));
1366 }
1367 
1368 void
kmsan_bus_space_unmap(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1369 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1370     bus_size_t size)
1371 {
1372 	bus_space_unmap(tag, hnd, size);
1373 }
1374 
1375 int
kmsan_bus_space_subregion(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,bus_space_handle_t * handlep)1376 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1377     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1378 {
1379 	return (bus_space_subregion(tag, hnd, offset, size, handlep));
1380 }
1381 
1382 void
kmsan_bus_space_free(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1383 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1384     bus_size_t size)
1385 {
1386 	bus_space_free(tag, hnd, size);
1387 }
1388 
1389 void
kmsan_bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,int flags)1390 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1391     bus_size_t offset, bus_size_t size, int flags)
1392 {
1393 	bus_space_barrier(tag, hnd, offset, size, flags);
1394 }
1395 
1396 #if defined(__amd64__)
1397 #define	BUS_SPACE_IO(tag)	((tag) == X86_BUS_SPACE_IO)
1398 #else
1399 #define	BUS_SPACE_IO(tag)	(false)
1400 #endif
1401 
1402 #define MSAN_BUS_READ_FUNC(func, width, type)				\
1403 	type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag,	\
1404 	    bus_space_handle_t hnd, bus_size_t offset)			\
1405 	{								\
1406 		type ret;						\
1407 		if (!BUS_SPACE_IO(tag))					\
1408 			kmsan_shadow_fill((uintptr_t)(hnd + offset),	\
1409 			    KMSAN_STATE_INITED, (width));		\
1410 		ret = bus_space_read##func##_##width(tag, hnd, offset);	\
1411 		kmsan_init_ret(sizeof(type));				\
1412 		return (ret);						\
1413 	}								\
1414 
1415 #define MSAN_BUS_READ_PTR_FUNC(func, width, type)			\
1416 	void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag,	\
1417 	    bus_space_handle_t hnd, bus_size_t size, type *buf,		\
1418 	    bus_size_t count)						\
1419 	{								\
1420 		kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED,	\
1421 		    (width) * count);					\
1422 		bus_space_read_##func##_##width(tag, hnd, size, buf, 	\
1423 		    count);						\
1424 	}
1425 
1426 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1427 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1428 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1429 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1430 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1431 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1432 
1433 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1434 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1435 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1436 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1437 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1438 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1439 
1440 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1441 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1442 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1443 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1444 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1445 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1446 
1447 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1448 #ifndef __amd64__
1449 MSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
1450 MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
1451 MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
1452 MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
1453 MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
1454 #endif
1455 
1456 #define	MSAN_BUS_WRITE_FUNC(func, width, type)				\
1457 	void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag,	\
1458 	    bus_space_handle_t hnd, bus_size_t offset, type value)	\
1459 	{								\
1460 		bus_space_write##func##_##width(tag, hnd, offset, value);\
1461 	}								\
1462 
1463 #define	MSAN_BUS_WRITE_PTR_FUNC(func, width, type)			\
1464 	void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1465 	    bus_space_handle_t hnd, bus_size_t size, const type *buf,	\
1466 	    bus_size_t count)						\
1467 	{								\
1468 		kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1469 		    "bus_space_write()");				\
1470 		bus_space_write_##func##_##width(tag, hnd, size, buf, 	\
1471 		    count);						\
1472 	}
1473 
1474 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1475 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1476 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1477 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1478 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1479 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1480 
1481 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1482 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1483 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1484 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1485 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1486 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1487 
1488 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1489 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1490 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1491 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1492 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1493 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1494 
1495 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1496 
1497 #define	MSAN_BUS_SET_FUNC(func, width, type)				\
1498 	void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag,	\
1499 	    bus_space_handle_t hnd, bus_size_t offset, type value,	\
1500 	    bus_size_t count)						\
1501 	{								\
1502 		bus_space_set_##func##_##width(tag, hnd, offset, value,	\
1503 		    count);						\
1504 	}
1505 
1506 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1507 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1508 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1509 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1510 
1511 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1512 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1513 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1514 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1515 
1516 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1517 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1518 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1519 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1520 
1521 #define	MSAN_BUS_PEEK_FUNC(width, type)					\
1522 	int kmsan_bus_space_peek_##width(bus_space_tag_t tag,		\
1523 	    bus_space_handle_t hnd, bus_size_t offset, type *value)	\
1524 	{								\
1525 		return (bus_space_peek_##width(tag, hnd, offset, value)); \
1526 	}
1527 
1528 MSAN_BUS_PEEK_FUNC(1, uint8_t)
1529 MSAN_BUS_PEEK_FUNC(2, uint16_t)
1530 MSAN_BUS_PEEK_FUNC(4, uint32_t)
1531 
1532 #define	MSAN_BUS_POKE_FUNC(width, type)					\
1533 	int kmsan_bus_space_poke_##width(bus_space_tag_t tag,		\
1534 	    bus_space_handle_t hnd, bus_size_t offset, type value)	\
1535 	{								\
1536 		return (bus_space_poke_##width(tag, hnd, offset, value)); \
1537 	}
1538 
1539 MSAN_BUS_POKE_FUNC(1, uint8_t)
1540 MSAN_BUS_POKE_FUNC(2, uint16_t)
1541 MSAN_BUS_POKE_FUNC(4, uint32_t)
1542 
1543 /* -------------------------------------------------------------------------- */
1544 
1545 void
kmsan_bus_dmamap_sync(struct memdesc * desc,bus_dmasync_op_t op)1546 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1547 {
1548 	/*
1549 	 * Some drivers, e.g., nvme, use the same code path for loading device
1550 	 * read and write requests, and will thus specify both flags.  In this
1551 	 * case we should not do any checking since it will generally lead to
1552 	 * false positives.
1553 	 */
1554 	if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1555 	    BUS_DMASYNC_PREWRITE) {
1556 		switch (desc->md_type) {
1557 		case MEMDESC_VADDR:
1558 			kmsan_check(desc->u.md_vaddr, desc->md_len,
1559 			    "dmasync");
1560 			break;
1561 		case MEMDESC_MBUF:
1562 			kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1563 			break;
1564 		case 0:
1565 			break;
1566 		default:
1567 			kmsan_panic("%s: unhandled memdesc type %d", __func__,
1568 			    desc->md_type);
1569 		}
1570 	}
1571 	if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1572 		switch (desc->md_type) {
1573 		case MEMDESC_VADDR:
1574 			kmsan_mark(desc->u.md_vaddr, desc->md_len,
1575 			    KMSAN_STATE_INITED);
1576 			break;
1577 		case MEMDESC_MBUF:
1578 			kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1579 			break;
1580 		case 0:
1581 			break;
1582 		default:
1583 			kmsan_panic("%s: unhandled memdesc type %d", __func__,
1584 			    desc->md_type);
1585 		}
1586 	}
1587 }
1588