1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define SAN_RUNTIME 36 37 #include <sys/cdefs.h> 38 #if 0 39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bio.h> 45 #include <sys/buf.h> 46 #include <sys/conf.h> 47 #include <sys/kdb.h> 48 #include <sys/kernel.h> 49 #include <sys/linker.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/memdesc.h> 53 #include <sys/msan.h> 54 #include <sys/proc.h> 55 #include <sys/stack.h> 56 #include <sys/stdarg.h> 57 #include <sys/sysctl.h> 58 #include <sys/uio.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 63 #include <machine/msan.h> 64 65 void kmsan_init_arg(size_t); 66 void kmsan_init_ret(size_t); 67 68 /* -------------------------------------------------------------------------- */ 69 70 /* 71 * Part of the compiler ABI. 72 */ 73 74 typedef struct { 75 uint8_t *shad; 76 msan_orig_t *orig; 77 } msan_meta_t; 78 79 #define MSAN_PARAM_SIZE 800 80 #define MSAN_RETVAL_SIZE 800 81 typedef struct { 82 uint8_t param_shadow[MSAN_PARAM_SIZE]; 83 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 85 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 86 uint64_t va_arg_overflow_size; 87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 88 msan_orig_t retval_origin; 89 } msan_tls_t; 90 91 /* -------------------------------------------------------------------------- */ 92 93 #define MSAN_NCONTEXT 4 94 #define MSAN_ORIG_MASK (~0x3) 95 96 typedef struct kmsan_td { 97 size_t ctx; 98 msan_tls_t tls[MSAN_NCONTEXT]; 99 } msan_td_t; 100 101 static msan_tls_t dummy_tls; 102 103 /* 104 * Use separate dummy regions for loads and stores: stores may mark the region 105 * as uninitialized, and that can trigger false positives. 106 */ 107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 110 static msan_td_t msan_thread0; 111 static bool kmsan_reporting = false; 112 113 /* 114 * Avoid clobbering any thread-local state before we panic. 115 */ 116 #define kmsan_panic(f, ...) do { \ 117 kmsan_disabled = true; \ 118 panic(f, __VA_ARGS__); \ 119 } while (0) 120 121 #define REPORT(f, ...) do { \ 122 if (panic_on_violation) { \ 123 kmsan_panic(f, __VA_ARGS__); \ 124 } else { \ 125 struct stack st; \ 126 \ 127 stack_save(&st); \ 128 printf(f "\n", __VA_ARGS__); \ 129 stack_print_ddb(&st); \ 130 } \ 131 } while (0) 132 133 FEATURE(kmsan, "Kernel memory sanitizer"); 134 135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 136 "KMSAN options"); 137 138 static bool panic_on_violation = 1; 139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 140 &panic_on_violation, 0, 141 "Panic if an invalid access is detected"); 142 143 static bool kmsan_disabled __read_mostly = true; 144 #define kmsan_enabled (!kmsan_disabled) 145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 146 &kmsan_disabled, 0, "KMSAN is disabled"); 147 148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 149 150 /* -------------------------------------------------------------------------- */ 151 152 static inline const char * 153 kmsan_orig_name(int type) 154 { 155 switch (type) { 156 case KMSAN_TYPE_STACK: 157 return ("stack"); 158 case KMSAN_TYPE_KMEM: 159 return ("kmem"); 160 case KMSAN_TYPE_MALLOC: 161 return ("malloc"); 162 case KMSAN_TYPE_UMA: 163 return ("UMA"); 164 default: 165 return ("unknown"); 166 } 167 } 168 169 static void 170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, 171 const char *hook) 172 { 173 const char *typename; 174 char *var, *fn; 175 uintptr_t ptr; 176 long foff; 177 char buf[128]; 178 int type; 179 180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 181 return; 182 if (__predict_false(curthread != NULL && 183 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0)) 184 return; 185 186 kmsan_reporting = true; 187 __compiler_membar(); 188 189 if (*orig == 0) { 190 REPORT("MSan: Uninitialized memory in %s, offset %zu", 191 hook, off); 192 goto out; 193 } 194 195 kmsan_md_orig_decode(*orig, &type, &ptr); 196 typename = kmsan_orig_name(type); 197 198 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 199 sizeof(buf), &foff) == 0) { 200 REPORT("MSan: Uninitialized %s memory in %s, " 201 "offset %zu/%zu, addr %p, from %s+%#lx", 202 typename, hook, off, size, addr, buf, foff); 203 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 204 /* 205 * The format of the string is: "----var@function". Parse it to 206 * display a nice warning. 207 */ 208 var = (char *)ptr + 4; 209 strlcpy(buf, var, sizeof(buf)); 210 var = buf; 211 fn = strchr(buf, '@'); 212 *fn++ = '\0'; 213 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 214 "variable '%s' from %s", typename, hook, off, var, fn); 215 } else { 216 REPORT("MSan: Uninitialized %s memory in %s, " 217 "offset %zu/%zu, addr %p, PC %p", 218 typename, hook, off, size, addr, (void *)ptr); 219 } 220 221 out: 222 __compiler_membar(); 223 kmsan_reporting = false; 224 } 225 226 static void 227 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 228 { 229 const char *typename; 230 char *var, *fn; 231 uintptr_t ptr; 232 char buf[128]; 233 long foff; 234 int type; 235 236 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 237 return; 238 if (__predict_false(curthread != NULL && 239 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0)) 240 return; 241 242 kmsan_reporting = true; 243 __compiler_membar(); 244 245 if (orig == 0) { 246 REPORT("MSan: uninitialized variable in %p", (void *)pc); 247 goto out; 248 } 249 250 kmsan_md_orig_decode(orig, &type, &ptr); 251 typename = kmsan_orig_name(type); 252 253 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 254 sizeof(buf), &foff) == 0) { 255 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 256 typename, buf, foff); 257 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 258 /* 259 * The format of the string is: "----var@function". Parse it to 260 * display a nice warning. 261 */ 262 var = (char *)ptr + 4; 263 strlcpy(buf, var, sizeof(buf)); 264 var = buf; 265 fn = strchr(buf, '@'); 266 *fn++ = '\0'; 267 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 268 } else { 269 REPORT("MSan: Uninitialized %s memory, origin %x", 270 typename, orig); 271 } 272 273 out: 274 __compiler_membar(); 275 kmsan_reporting = false; 276 } 277 278 /* -------------------------------------------------------------------------- */ 279 280 static inline msan_meta_t 281 kmsan_meta_get(const void *addr, size_t size, const bool write) 282 { 283 msan_meta_t ret; 284 285 if (__predict_false(!kmsan_enabled)) { 286 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 287 ret.orig = (msan_orig_t *)msan_dummy_orig; 288 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 289 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 290 ret.orig = (msan_orig_t *)msan_dummy_orig; 291 } else { 292 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 293 ret.orig = 294 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 295 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 296 MSAN_ORIG_MASK); 297 } 298 299 return (ret); 300 } 301 302 static inline void 303 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 304 { 305 msan_orig_t *orig; 306 size_t i; 307 308 if (__predict_false(!kmsan_enabled)) 309 return; 310 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 311 return; 312 313 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 314 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 315 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 316 317 for (i = 0; i < size; i += 4) { 318 orig[i / 4] = o; 319 } 320 } 321 322 static inline void 323 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 324 { 325 uint8_t *shad; 326 327 if (__predict_false(!kmsan_enabled)) 328 return; 329 if (__predict_false(kmsan_md_unsupported(addr))) 330 return; 331 332 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 333 __builtin_memset(shad, c, size); 334 } 335 336 static inline void 337 kmsan_meta_copy(void *dst, const void *src, size_t size) 338 { 339 uint8_t *orig_src, *orig_dst; 340 uint8_t *shad_src, *shad_dst; 341 msan_orig_t *_src, *_dst; 342 size_t i; 343 344 if (__predict_false(!kmsan_enabled)) 345 return; 346 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 347 return; 348 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 349 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 350 return; 351 } 352 353 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 354 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 355 __builtin_memmove(shad_dst, shad_src, size); 356 357 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 358 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 359 for (i = 0; i < size; i++) { 360 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 361 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 362 *_dst = *_src; 363 orig_src++; 364 orig_dst++; 365 } 366 } 367 368 static inline void 369 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 370 { 371 msan_orig_t *orig; 372 uint8_t *shad; 373 size_t i; 374 375 if (__predict_false(!kmsan_enabled)) 376 return; 377 if (__predict_false(kmsan_md_unsupported(addr))) 378 return; 379 380 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 381 for (i = 0; i < size; i++) { 382 if (__predict_true(shad[i] == 0)) 383 continue; 384 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i); 385 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 386 kmsan_report_hook((const char *)addr + i, orig, size, i, hook); 387 break; 388 } 389 } 390 391 void 392 kmsan_init_arg(size_t n) 393 { 394 msan_td_t *mtd; 395 uint8_t *arg; 396 397 if (__predict_false(!kmsan_enabled)) 398 return; 399 if (__predict_false(curthread == NULL)) 400 return; 401 mtd = curthread->td_kmsan; 402 arg = mtd->tls[mtd->ctx].param_shadow; 403 __builtin_memset(arg, 0, n); 404 } 405 406 void 407 kmsan_init_ret(size_t n) 408 { 409 msan_td_t *mtd; 410 uint8_t *arg; 411 412 if (__predict_false(!kmsan_enabled)) 413 return; 414 if (__predict_false(curthread == NULL)) 415 return; 416 mtd = curthread->td_kmsan; 417 arg = mtd->tls[mtd->ctx].retval_shadow; 418 __builtin_memset(arg, 0, n); 419 } 420 421 static void 422 kmsan_check_arg(size_t size, const char *hook) 423 { 424 msan_orig_t *orig; 425 msan_td_t *mtd; 426 uint8_t *arg; 427 size_t ctx, i; 428 429 if (__predict_false(!kmsan_enabled)) 430 return; 431 if (__predict_false(curthread == NULL)) 432 return; 433 mtd = curthread->td_kmsan; 434 ctx = mtd->ctx; 435 arg = mtd->tls[ctx].param_shadow; 436 437 for (i = 0; i < size; i++) { 438 if (__predict_true(arg[i] == 0)) 439 continue; 440 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; 441 kmsan_report_hook((const char *)arg + i, orig, size, i, hook); 442 break; 443 } 444 } 445 446 void 447 kmsan_thread_alloc(struct thread *td) 448 { 449 msan_td_t *mtd; 450 451 if (!kmsan_enabled) 452 return; 453 454 mtd = td->td_kmsan; 455 if (mtd == NULL) { 456 /* We might be recycling a thread. */ 457 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 458 sizeof(int)); 459 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 460 } 461 __builtin_memset(mtd, 0, sizeof(*mtd)); 462 mtd->ctx = 0; 463 464 if (td->td_kstack != 0) 465 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 466 KMSAN_STATE_UNINIT); 467 468 td->td_kmsan = mtd; 469 } 470 471 void 472 kmsan_thread_free(struct thread *td) 473 { 474 msan_td_t *mtd; 475 476 if (!kmsan_enabled) 477 return; 478 if (__predict_false(td == curthread)) 479 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 480 481 mtd = td->td_kmsan; 482 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 483 free(mtd, M_KMSAN); 484 td->td_kmsan = NULL; 485 } 486 487 void kmsan_intr_enter(void); 488 void kmsan_intr_leave(void); 489 490 void 491 kmsan_intr_enter(void) 492 { 493 msan_td_t *mtd; 494 495 if (__predict_false(!kmsan_enabled)) 496 return; 497 498 mtd = curthread->td_kmsan; 499 mtd->ctx++; 500 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 501 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 502 } 503 504 void 505 kmsan_intr_leave(void) 506 { 507 msan_td_t *mtd; 508 509 if (__predict_false(!kmsan_enabled)) 510 return; 511 512 mtd = curthread->td_kmsan; 513 if (__predict_false(mtd->ctx == 0)) 514 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 515 mtd->ctx--; 516 } 517 518 /* -------------------------------------------------------------------------- */ 519 520 void 521 kmsan_shadow_map(vm_offset_t addr, size_t size) 522 { 523 size_t npages, i; 524 vm_offset_t va; 525 526 MPASS(addr % PAGE_SIZE == 0); 527 MPASS(size % PAGE_SIZE == 0); 528 529 if (!kmsan_enabled) 530 return; 531 532 npages = atop(size); 533 534 va = kmsan_md_addr_to_shad(addr); 535 for (i = 0; i < npages; i++) { 536 pmap_san_enter(va + ptoa(i)); 537 } 538 539 va = kmsan_md_addr_to_orig(addr); 540 for (i = 0; i < npages; i++) { 541 pmap_san_enter(va + ptoa(i)); 542 } 543 } 544 545 void 546 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 547 { 548 msan_orig_t orig; 549 550 orig = kmsan_md_orig_encode(type, pc); 551 kmsan_origin_fill(addr, orig, size); 552 } 553 554 void 555 kmsan_mark(const void *addr, size_t size, uint8_t c) 556 { 557 kmsan_shadow_fill((uintptr_t)addr, c, size); 558 } 559 560 void 561 kmsan_mark_bio(const struct bio *bp, uint8_t c) 562 { 563 kmsan_mark(bp->bio_data, bp->bio_length, c); 564 } 565 566 void 567 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 568 { 569 do { 570 if ((m->m_flags & M_EXTPG) == 0) 571 kmsan_mark(m->m_data, m->m_len, c); 572 m = m->m_next; 573 } while (m != NULL); 574 } 575 576 void 577 kmsan_check(const void *p, size_t sz, const char *descr) 578 { 579 kmsan_shadow_check((uintptr_t)p, sz, descr); 580 } 581 582 void 583 kmsan_check_bio(const struct bio *bp, const char *descr) 584 { 585 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 586 } 587 588 void 589 kmsan_check_mbuf(const struct mbuf *m, const char *descr) 590 { 591 do { 592 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 593 } while ((m = m->m_next) != NULL); 594 } 595 596 void 597 kmsan_check_uio(const struct uio *uio, const char *descr) 598 { 599 for (int i = 0; i < uio->uio_iovcnt; i++) { 600 kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len, 601 descr); 602 } 603 } 604 605 void 606 kmsan_init(void) 607 { 608 int disabled; 609 610 disabled = 0; 611 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 612 if (disabled) 613 return; 614 615 /* Initialize the TLS for curthread. */ 616 msan_thread0.ctx = 0; 617 thread0.td_kmsan = &msan_thread0; 618 619 /* Now officially enabled. */ 620 kmsan_disabled = false; 621 } 622 623 /* -------------------------------------------------------------------------- */ 624 625 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 626 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 627 628 msan_meta_t 629 __msan_metadata_ptr_for_load_n(void *addr, size_t size) 630 { 631 return (kmsan_meta_get(addr, size, false)); 632 } 633 634 msan_meta_t 635 __msan_metadata_ptr_for_store_n(void *addr, size_t size) 636 { 637 return (kmsan_meta_get(addr, size, true)); 638 } 639 640 #define MSAN_META_FUNC(size) \ 641 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 642 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 643 { \ 644 return (kmsan_meta_get(addr, size, false)); \ 645 } \ 646 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 647 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 648 { \ 649 return (kmsan_meta_get(addr, size, true)); \ 650 } 651 652 MSAN_META_FUNC(1) 653 MSAN_META_FUNC(2) 654 MSAN_META_FUNC(4) 655 MSAN_META_FUNC(8) 656 657 void __msan_instrument_asm_store(const void *, size_t); 658 msan_orig_t __msan_chain_origin(msan_orig_t); 659 void __msan_poison(const void *, size_t); 660 void __msan_unpoison(const void *, size_t); 661 void __msan_poison_alloca(const void *, uint64_t, const char *); 662 void __msan_unpoison_alloca(const void *, uint64_t); 663 void __msan_warning(msan_orig_t); 664 msan_tls_t *__msan_get_context_state(void); 665 intptr_t __msan_test_shadow(const void *, size_t); 666 667 void 668 __msan_instrument_asm_store(const void *addr, size_t size) 669 { 670 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 671 } 672 673 msan_orig_t 674 __msan_chain_origin(msan_orig_t origin) 675 { 676 return (origin); 677 } 678 679 void 680 __msan_poison(const void *addr, size_t size) 681 { 682 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 683 } 684 685 void 686 __msan_unpoison(const void *addr, size_t size) 687 { 688 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 689 } 690 691 void 692 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 693 { 694 msan_orig_t orig; 695 696 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 697 kmsan_origin_fill(addr, orig, size); 698 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 699 } 700 701 void 702 __msan_unpoison_alloca(const void *addr, uint64_t size) 703 { 704 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 705 } 706 707 void 708 __msan_warning(msan_orig_t origin) 709 { 710 if (__predict_false(!kmsan_enabled)) 711 return; 712 kmsan_report_inline(origin, KMSAN_RET_ADDR); 713 } 714 715 msan_tls_t * 716 __msan_get_context_state(void) 717 { 718 msan_td_t *mtd; 719 720 /* 721 * When APs are started, they execute some C code before curthread is 722 * set. We have to handle that here. 723 */ 724 if (__predict_false(!kmsan_enabled || curthread == NULL)) 725 return (&dummy_tls); 726 mtd = curthread->td_kmsan; 727 return (&mtd->tls[mtd->ctx]); 728 } 729 730 intptr_t 731 __msan_test_shadow(const void *addr, size_t len) 732 { 733 uint8_t *shad; 734 735 if (__predict_false(!kmsan_enabled)) 736 return (-1); 737 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 738 return (-1); 739 740 shad = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)addr); 741 for (size_t i = 0; i < len; i++) 742 if (shad[i] != 0) 743 return (i); 744 return (-1); 745 } 746 747 /* -------------------------------------------------------------------------- */ 748 749 /* 750 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 751 * initialized areas properly. 752 */ 753 754 void * 755 kmsan_memcpy(void *dst, const void *src, size_t len) 756 { 757 /* No kmsan_check_arg, because inlined. */ 758 kmsan_init_ret(sizeof(void *)); 759 if (__predict_true(len != 0)) { 760 kmsan_meta_copy(dst, src, len); 761 } 762 return (__builtin_memcpy(dst, src, len)); 763 } 764 765 int 766 kmsan_memcmp(const void *b1, const void *b2, size_t len) 767 { 768 const uint8_t *_b1 = b1, *_b2 = b2; 769 size_t i; 770 771 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 772 "memcmp():args"); 773 kmsan_init_ret(sizeof(int)); 774 775 for (i = 0; i < len; i++) { 776 if (*_b1 != *_b2) { 777 kmsan_shadow_check((uintptr_t)b1, i + 1, 778 "memcmp():arg1"); 779 kmsan_shadow_check((uintptr_t)b2, i + 1, 780 "memcmp():arg2"); 781 return (*_b1 - *_b2); 782 } 783 _b1++, _b2++; 784 } 785 786 return (0); 787 } 788 789 void * 790 kmsan_memset(void *dst, int c, size_t len) 791 { 792 /* No kmsan_check_arg, because inlined. */ 793 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 794 kmsan_init_ret(sizeof(void *)); 795 return (__builtin_memset(dst, c, len)); 796 } 797 798 void * 799 kmsan_memmove(void *dst, const void *src, size_t len) 800 { 801 /* No kmsan_check_arg, because inlined. */ 802 if (__predict_true(len != 0)) { 803 kmsan_meta_copy(dst, src, len); 804 } 805 kmsan_init_ret(sizeof(void *)); 806 return (__builtin_memmove(dst, src, len)); 807 } 808 809 __strong_reference(kmsan_memcpy, __msan_memcpy); 810 __strong_reference(kmsan_memset, __msan_memset); 811 __strong_reference(kmsan_memmove, __msan_memmove); 812 813 char * 814 kmsan_strcpy(char *dst, const char *src) 815 { 816 const char *_src = src; 817 char *_dst = dst; 818 size_t len = 0; 819 820 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 821 822 while (1) { 823 len++; 824 *dst = *src; 825 if (*src == '\0') 826 break; 827 src++, dst++; 828 } 829 830 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 831 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 832 kmsan_init_ret(sizeof(char *)); 833 return (_dst); 834 } 835 836 int 837 kmsan_strcmp(const char *s1, const char *s2) 838 { 839 const char *_s1 = s1, *_s2 = s2; 840 size_t len = 0; 841 842 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 843 kmsan_init_ret(sizeof(int)); 844 845 while (1) { 846 len++; 847 if (*s1 != *s2) 848 break; 849 if (*s1 == '\0') { 850 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 851 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 852 return (0); 853 } 854 s1++, s2++; 855 } 856 857 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 858 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 859 860 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 861 } 862 863 size_t 864 kmsan_strlen(const char *str) 865 { 866 const char *s; 867 868 kmsan_check_arg(sizeof(str), "strlen():args"); 869 870 s = str; 871 while (1) { 872 if (*s == '\0') 873 break; 874 s++; 875 } 876 877 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 878 kmsan_init_ret(sizeof(size_t)); 879 return (s - str); 880 } 881 882 int kmsan_copyin(const void *, void *, size_t); 883 int kmsan_copyout(const void *, void *, size_t); 884 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 885 886 int 887 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 888 { 889 int ret; 890 891 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 892 "copyin():args"); 893 ret = copyin(uaddr, kaddr, len); 894 if (ret == 0) 895 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 896 kmsan_init_ret(sizeof(int)); 897 return (ret); 898 } 899 900 int 901 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 902 { 903 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 904 "copyout():args"); 905 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 906 kmsan_init_ret(sizeof(int)); 907 return (copyout(kaddr, uaddr, len)); 908 } 909 910 int 911 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 912 { 913 size_t _done; 914 int ret; 915 916 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 917 sizeof(len) + sizeof(done), "copyinstr():args"); 918 ret = copyinstr(uaddr, kaddr, len, &_done); 919 if (ret == 0) 920 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 921 if (done != NULL) { 922 *done = _done; 923 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 924 } 925 kmsan_init_ret(sizeof(int)); 926 return (ret); 927 } 928 929 /* -------------------------------------------------------------------------- */ 930 931 int 932 kmsan_fubyte(volatile const void *base) 933 { 934 int ret; 935 936 kmsan_check_arg(sizeof(base), "fubyte(): args"); 937 ret = fubyte(base); 938 kmsan_init_ret(sizeof(int)); 939 return (ret); 940 } 941 942 int 943 kmsan_fuword16(volatile const void *base) 944 { 945 int ret; 946 947 kmsan_check_arg(sizeof(base), "fuword16(): args"); 948 ret = fuword16(base); 949 kmsan_init_ret(sizeof(int)); 950 return (ret); 951 } 952 953 int 954 kmsan_fueword(volatile const void *base, long *val) 955 { 956 int ret; 957 958 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 959 ret = fueword(base, val); 960 if (ret == 0) 961 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 962 sizeof(*val)); 963 kmsan_init_ret(sizeof(int)); 964 return (ret); 965 } 966 967 int 968 kmsan_fueword32(volatile const void *base, int32_t *val) 969 { 970 int ret; 971 972 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 973 ret = fueword32(base, val); 974 if (ret == 0) 975 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 976 sizeof(*val)); 977 kmsan_init_ret(sizeof(int)); 978 return (ret); 979 } 980 981 int 982 kmsan_fueword64(volatile const void *base, int64_t *val) 983 { 984 int ret; 985 986 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 987 ret = fueword64(base, val); 988 if (ret == 0) 989 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 990 sizeof(*val)); 991 kmsan_init_ret(sizeof(int)); 992 return (ret); 993 } 994 995 int 996 kmsan_subyte(volatile void *base, int byte) 997 { 998 int ret; 999 1000 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 1001 ret = subyte(base, byte); 1002 kmsan_init_ret(sizeof(int)); 1003 return (ret); 1004 } 1005 1006 int 1007 kmsan_suword(volatile void *base, long word) 1008 { 1009 int ret; 1010 1011 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 1012 ret = suword(base, word); 1013 kmsan_init_ret(sizeof(int)); 1014 return (ret); 1015 } 1016 1017 int 1018 kmsan_suword16(volatile void *base, int word) 1019 { 1020 int ret; 1021 1022 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 1023 ret = suword16(base, word); 1024 kmsan_init_ret(sizeof(int)); 1025 return (ret); 1026 } 1027 1028 int 1029 kmsan_suword32(volatile void *base, int32_t word) 1030 { 1031 int ret; 1032 1033 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 1034 ret = suword32(base, word); 1035 kmsan_init_ret(sizeof(int)); 1036 return (ret); 1037 } 1038 1039 int 1040 kmsan_suword64(volatile void *base, int64_t word) 1041 { 1042 int ret; 1043 1044 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1045 ret = suword64(base, word); 1046 kmsan_init_ret(sizeof(int)); 1047 return (ret); 1048 } 1049 1050 int 1051 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1052 uint32_t newval) 1053 { 1054 int ret; 1055 1056 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1057 sizeof(newval), "casueword32(): args"); 1058 ret = casueword32(base, oldval, oldvalp, newval); 1059 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1060 sizeof(*oldvalp)); 1061 kmsan_init_ret(sizeof(int)); 1062 return (ret); 1063 } 1064 1065 int 1066 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1067 u_long newval) 1068 { 1069 int ret; 1070 1071 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1072 sizeof(newval), "casueword32(): args"); 1073 ret = casueword(base, oldval, oldvalp, newval); 1074 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1075 sizeof(*oldvalp)); 1076 kmsan_init_ret(sizeof(int)); 1077 return (ret); 1078 } 1079 1080 /* -------------------------------------------------------------------------- */ 1081 1082 #include <machine/atomic.h> 1083 #include <sys/atomic_san.h> 1084 1085 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1086 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1087 { \ 1088 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1089 "atomic_add_" #name "():args"); \ 1090 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1091 "atomic_add_" #name "():ptr"); \ 1092 atomic_add_##name(ptr, val); \ 1093 } 1094 1095 #define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1096 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1097 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1098 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1099 1100 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1101 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1102 { \ 1103 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1104 "atomic_subtract_" #name "():args"); \ 1105 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1106 "atomic_subtract_" #name "():ptr"); \ 1107 atomic_subtract_##name(ptr, val); \ 1108 } 1109 1110 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1111 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1112 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1113 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1114 1115 #define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1116 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1117 { \ 1118 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1119 "atomic_set_" #name "():args"); \ 1120 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1121 "atomic_set_" #name "():ptr"); \ 1122 atomic_set_##name(ptr, val); \ 1123 } 1124 1125 #define MSAN_ATOMIC_FUNC_SET(name, type) \ 1126 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1127 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1128 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1129 1130 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1131 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1132 { \ 1133 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1134 "atomic_clear_" #name "():args"); \ 1135 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1136 "atomic_clear_" #name "():ptr"); \ 1137 atomic_clear_##name(ptr, val); \ 1138 } 1139 1140 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1141 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1142 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1143 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1144 1145 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1146 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1147 { \ 1148 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1149 "atomic_fetchadd_" #name "():args"); \ 1150 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1151 "atomic_fetchadd_" #name "():ptr"); \ 1152 kmsan_init_ret(sizeof(type)); \ 1153 return (atomic_fetchadd_##name(ptr, val)); \ 1154 } 1155 1156 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1157 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1158 { \ 1159 kmsan_check_arg(sizeof(ptr), \ 1160 "atomic_readandclear_" #name "():args"); \ 1161 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1162 "atomic_readandclear_" #name "():ptr"); \ 1163 kmsan_init_ret(sizeof(type)); \ 1164 return (atomic_readandclear_##name(ptr)); \ 1165 } 1166 1167 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1168 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1169 { \ 1170 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1171 "atomic_testandclear_" #name "():args"); \ 1172 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1173 "atomic_testandclear_" #name "():ptr"); \ 1174 kmsan_init_ret(sizeof(int)); \ 1175 return (atomic_testandclear_##name(ptr, v)); \ 1176 } 1177 1178 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1179 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1180 { \ 1181 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1182 "atomic_testandset_" #name "():args"); \ 1183 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1184 "atomic_testandset_" #name "():ptr"); \ 1185 kmsan_init_ret(sizeof(int)); \ 1186 return (atomic_testandset_##name(ptr, v)); \ 1187 } 1188 1189 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1190 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1191 { \ 1192 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1193 "atomic_swap_" #name "():args"); \ 1194 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1195 "atomic_swap_" #name "():ptr"); \ 1196 kmsan_init_ret(sizeof(type)); \ 1197 return (atomic_swap_##name(ptr, val)); \ 1198 } 1199 1200 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1201 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1202 type nval) \ 1203 { \ 1204 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1205 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1206 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1207 "atomic_cmpset_" #name "():ptr"); \ 1208 kmsan_init_ret(sizeof(int)); \ 1209 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1210 } 1211 1212 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1213 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1214 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1215 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1216 1217 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1218 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1219 type nval) \ 1220 { \ 1221 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1222 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1223 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1224 "atomic_fcmpset_" #name "():ptr"); \ 1225 kmsan_init_ret(sizeof(int)); \ 1226 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1227 } 1228 1229 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1230 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1231 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1232 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1233 1234 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1235 void kmsan_atomic_thread_fence_##name(void) \ 1236 { \ 1237 atomic_thread_fence_##name(); \ 1238 } 1239 1240 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1241 type kmsan_atomic_load_##name(const volatile type *ptr) \ 1242 { \ 1243 kmsan_check_arg(sizeof(ptr), \ 1244 "atomic_load_" #name "():args"); \ 1245 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1246 "atomic_load_" #name "():ptr"); \ 1247 kmsan_init_ret(sizeof(type)); \ 1248 return (atomic_load_##name(ptr)); \ 1249 } 1250 1251 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1252 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1253 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1254 1255 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1256 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1257 { \ 1258 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1259 "atomic_store_" #name "():args"); \ 1260 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1261 sizeof(type)); \ 1262 atomic_store_##name(ptr, val); \ 1263 } 1264 1265 #define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1266 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1267 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1268 1269 MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1270 MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1271 MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1272 MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1273 MSAN_ATOMIC_FUNC_ADD(int, u_int); 1274 MSAN_ATOMIC_FUNC_ADD(long, u_long); 1275 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1276 1277 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1278 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1279 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1280 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1281 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1282 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1283 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1284 1285 MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1286 MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1287 MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1288 MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1289 MSAN_ATOMIC_FUNC_SET(int, u_int); 1290 MSAN_ATOMIC_FUNC_SET(long, u_long); 1291 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1292 1293 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1294 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1295 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1296 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1297 MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1298 MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1299 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1300 1301 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1302 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1303 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1304 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1305 1306 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1307 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1308 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1309 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1310 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1311 1312 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1313 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1314 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1315 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1316 MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t); 1317 1318 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1319 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1320 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1321 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1322 MSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long); 1323 MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t); 1324 1325 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1326 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1327 MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1328 MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1329 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1330 1331 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1332 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1333 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1334 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1335 MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1336 MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1337 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1338 1339 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1340 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1341 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1342 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1343 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1344 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1345 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1346 1347 _MSAN_ATOMIC_FUNC_LOAD(bool, bool); 1348 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1349 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1350 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1351 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1352 MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1353 MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1354 MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1355 MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1356 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1357 1358 _MSAN_ATOMIC_FUNC_STORE(bool, bool); 1359 MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1360 MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1361 MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1362 MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1363 MSAN_ATOMIC_FUNC_STORE(char, u_char); 1364 MSAN_ATOMIC_FUNC_STORE(short, u_short); 1365 MSAN_ATOMIC_FUNC_STORE(int, u_int); 1366 MSAN_ATOMIC_FUNC_STORE(long, u_long); 1367 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1368 1369 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1370 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1371 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1372 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1373 1374 void 1375 kmsan_atomic_interrupt_fence(void) 1376 { 1377 atomic_interrupt_fence(); 1378 } 1379 1380 /* -------------------------------------------------------------------------- */ 1381 1382 #include <sys/bus.h> 1383 #include <machine/bus.h> 1384 #include <sys/bus_san.h> 1385 1386 int 1387 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1388 int flags, bus_space_handle_t *handlep) 1389 { 1390 return (bus_space_map(tag, hnd, size, flags, handlep)); 1391 } 1392 1393 void 1394 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1395 bus_size_t size) 1396 { 1397 bus_space_unmap(tag, hnd, size); 1398 } 1399 1400 int 1401 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1402 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1403 { 1404 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1405 } 1406 1407 void 1408 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1409 bus_size_t size) 1410 { 1411 bus_space_free(tag, hnd, size); 1412 } 1413 1414 void 1415 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1416 bus_size_t offset, bus_size_t size, int flags) 1417 { 1418 bus_space_barrier(tag, hnd, offset, size, flags); 1419 } 1420 1421 #if defined(__amd64__) 1422 #define BUS_SPACE_IO(tag) ((tag) == X86_BUS_SPACE_IO) 1423 #else 1424 #define BUS_SPACE_IO(tag) (false) 1425 #endif 1426 1427 #define MSAN_BUS_READ_FUNC(func, width, type) \ 1428 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1429 bus_space_handle_t hnd, bus_size_t offset) \ 1430 { \ 1431 type ret; \ 1432 if (!BUS_SPACE_IO(tag)) \ 1433 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1434 KMSAN_STATE_INITED, (width)); \ 1435 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1436 kmsan_init_ret(sizeof(type)); \ 1437 return (ret); \ 1438 } \ 1439 1440 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1441 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1442 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1443 bus_size_t count) \ 1444 { \ 1445 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1446 (width) * count); \ 1447 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1448 count); \ 1449 } 1450 1451 MSAN_BUS_READ_FUNC(, 1, uint8_t) 1452 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1453 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1454 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1455 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1456 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1457 1458 MSAN_BUS_READ_FUNC(, 2, uint16_t) 1459 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1460 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1461 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1462 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1463 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1464 1465 MSAN_BUS_READ_FUNC(, 4, uint32_t) 1466 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1467 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1468 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1469 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1470 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1471 1472 MSAN_BUS_READ_FUNC(, 8, uint64_t) 1473 #ifndef __amd64__ 1474 MSAN_BUS_READ_FUNC(_stream, 8, uint64_t) 1475 MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) 1476 MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) 1477 MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) 1478 MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) 1479 #endif 1480 1481 #define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1482 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1483 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1484 { \ 1485 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1486 } \ 1487 1488 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1489 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1490 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1491 bus_size_t count) \ 1492 { \ 1493 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1494 "bus_space_write()"); \ 1495 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1496 count); \ 1497 } 1498 1499 MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1500 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1501 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1502 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1503 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1504 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1505 1506 MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1507 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1508 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1509 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1510 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1511 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1512 1513 MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1514 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1515 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1516 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1517 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1518 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1519 1520 MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1521 1522 #define MSAN_BUS_SET_FUNC(func, width, type) \ 1523 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1524 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1525 bus_size_t count) \ 1526 { \ 1527 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1528 count); \ 1529 } 1530 1531 MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1532 MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1533 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1534 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1535 1536 MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1537 MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1538 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1539 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1540 1541 MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1542 MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1543 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1544 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1545 1546 #define MSAN_BUS_PEEK_FUNC(width, type) \ 1547 int kmsan_bus_space_peek_##width(bus_space_tag_t tag, \ 1548 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 1549 { \ 1550 return (bus_space_peek_##width(tag, hnd, offset, value)); \ 1551 } 1552 1553 MSAN_BUS_PEEK_FUNC(1, uint8_t) 1554 MSAN_BUS_PEEK_FUNC(2, uint16_t) 1555 MSAN_BUS_PEEK_FUNC(4, uint32_t) 1556 1557 #define MSAN_BUS_POKE_FUNC(width, type) \ 1558 int kmsan_bus_space_poke_##width(bus_space_tag_t tag, \ 1559 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1560 { \ 1561 return (bus_space_poke_##width(tag, hnd, offset, value)); \ 1562 } 1563 1564 MSAN_BUS_POKE_FUNC(1, uint8_t) 1565 MSAN_BUS_POKE_FUNC(2, uint16_t) 1566 MSAN_BUS_POKE_FUNC(4, uint32_t) 1567 1568 /* -------------------------------------------------------------------------- */ 1569 1570 void 1571 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1572 { 1573 /* 1574 * Some drivers, e.g., nvme, use the same code path for loading device 1575 * read and write requests, and will thus specify both flags. In this 1576 * case we should not do any checking since it will generally lead to 1577 * false positives. 1578 */ 1579 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1580 BUS_DMASYNC_PREWRITE) { 1581 switch (desc->md_type) { 1582 case MEMDESC_VADDR: 1583 kmsan_check(desc->u.md_vaddr, desc->md_len, 1584 "dmasync"); 1585 break; 1586 case MEMDESC_MBUF: 1587 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1588 break; 1589 case 0: 1590 break; 1591 default: 1592 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1593 desc->md_type); 1594 } 1595 } 1596 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1597 switch (desc->md_type) { 1598 case MEMDESC_VADDR: 1599 kmsan_mark(desc->u.md_vaddr, desc->md_len, 1600 KMSAN_STATE_INITED); 1601 break; 1602 case MEMDESC_MBUF: 1603 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1604 break; 1605 case 0: 1606 break; 1607 default: 1608 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1609 desc->md_type); 1610 } 1611 } 1612 } 1613