1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define SAN_RUNTIME 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 #if 0 40 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bio.h> 46 #include <sys/buf.h> 47 #include <sys/conf.h> 48 #include <sys/kdb.h> 49 #include <sys/kernel.h> 50 #include <sys/linker.h> 51 #include <sys/malloc.h> 52 #include <sys/mbuf.h> 53 #include <sys/memdesc.h> 54 #include <sys/msan.h> 55 #include <sys/proc.h> 56 #include <sys/stack.h> 57 #include <sys/sysctl.h> 58 #include <sys/uio.h> 59 60 #include <cam/cam.h> 61 #include <cam/cam_ccb.h> 62 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 66 #include <machine/msan.h> 67 #include <machine/stdarg.h> 68 69 void kmsan_init_arg(size_t); 70 void kmsan_init_ret(size_t); 71 72 /* -------------------------------------------------------------------------- */ 73 74 /* 75 * Part of the compiler ABI. 76 */ 77 78 typedef struct { 79 uint8_t *shad; 80 msan_orig_t *orig; 81 } msan_meta_t; 82 83 #define MSAN_PARAM_SIZE 800 84 #define MSAN_RETVAL_SIZE 800 85 typedef struct { 86 uint8_t param_shadow[MSAN_PARAM_SIZE]; 87 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 88 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 89 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 90 uint64_t va_arg_overflow_size; 91 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 92 msan_orig_t retval_origin; 93 } msan_tls_t; 94 95 /* -------------------------------------------------------------------------- */ 96 97 #define MSAN_NCONTEXT 4 98 #define MSAN_ORIG_MASK (~0x3) 99 100 typedef struct kmsan_td { 101 size_t ctx; 102 msan_tls_t tls[MSAN_NCONTEXT]; 103 } msan_td_t; 104 105 static msan_tls_t dummy_tls; 106 107 /* 108 * Use separate dummy regions for loads and stores: stores may mark the region 109 * as uninitialized, and that can trigger false positives. 110 */ 111 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 112 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 113 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 114 static msan_td_t msan_thread0; 115 static bool kmsan_enabled __read_mostly; 116 117 static bool kmsan_reporting = false; 118 119 /* 120 * Avoid clobbering any thread-local state before we panic. 121 */ 122 #define kmsan_panic(f, ...) do { \ 123 kmsan_enabled = false; \ 124 panic(f, __VA_ARGS__); \ 125 } while (0) 126 127 #define REPORT(f, ...) do { \ 128 if (panic_on_violation) { \ 129 kmsan_panic(f, __VA_ARGS__); \ 130 } else { \ 131 struct stack st; \ 132 \ 133 stack_save(&st); \ 134 printf(f "\n", __VA_ARGS__); \ 135 stack_print_ddb(&st); \ 136 } \ 137 } while (0) 138 139 FEATURE(kmsan, "Kernel memory sanitizer"); 140 141 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 142 "KMSAN options"); 143 144 static bool panic_on_violation = 1; 145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 146 &panic_on_violation, 0, 147 "Panic if an invalid access is detected"); 148 149 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 150 151 /* -------------------------------------------------------------------------- */ 152 153 static inline const char * 154 kmsan_orig_name(int type) 155 { 156 switch (type) { 157 case KMSAN_TYPE_STACK: 158 return ("stack"); 159 case KMSAN_TYPE_KMEM: 160 return ("kmem"); 161 case KMSAN_TYPE_MALLOC: 162 return ("malloc"); 163 case KMSAN_TYPE_UMA: 164 return ("UMA"); 165 default: 166 return ("unknown"); 167 } 168 } 169 170 static void 171 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 172 { 173 msan_orig_t *orig; 174 const char *typename; 175 char *var, *fn; 176 uintptr_t ptr; 177 long foff; 178 char buf[128]; 179 int type; 180 181 if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting)) 182 return; 183 184 kmsan_reporting = true; 185 __compiler_membar(); 186 187 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 188 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 189 190 if (*orig == 0) { 191 REPORT("MSan: Uninitialized memory in %s, offset %zu", 192 hook, off); 193 goto out; 194 } 195 196 kmsan_md_orig_decode(*orig, &type, &ptr); 197 typename = kmsan_orig_name(type); 198 199 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 200 sizeof(buf), &foff) == 0) { 201 REPORT("MSan: Uninitialized %s memory in %s, " 202 "offset %zu/%zu, addr %p, from %s+%#lx", 203 typename, hook, off, size, addr, buf, foff); 204 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 205 /* 206 * The format of the string is: "----var@function". Parse it to 207 * display a nice warning. 208 */ 209 var = (char *)ptr + 4; 210 strlcpy(buf, var, sizeof(buf)); 211 var = buf; 212 fn = strchr(buf, '@'); 213 *fn++ = '\0'; 214 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 215 "variable '%s' from %s", typename, hook, off, var, fn); 216 } else { 217 REPORT("MSan: Uninitialized %s memory in %s, " 218 "offset %zu/%zu, addr %p, PC %p", 219 typename, hook, off, size, addr, (void *)ptr); 220 } 221 222 out: 223 __compiler_membar(); 224 kmsan_reporting = false; 225 } 226 227 static void 228 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 229 { 230 const char *typename; 231 char *var, *fn; 232 uintptr_t ptr; 233 char buf[128]; 234 long foff; 235 int type; 236 237 if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting)) 238 return; 239 240 kmsan_reporting = true; 241 __compiler_membar(); 242 243 if (orig == 0) { 244 REPORT("MSan: uninitialized variable in %p", (void *)pc); 245 goto out; 246 } 247 248 kmsan_md_orig_decode(orig, &type, &ptr); 249 typename = kmsan_orig_name(type); 250 251 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 252 sizeof(buf), &foff) == 0) { 253 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 254 typename, buf, foff); 255 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 256 /* 257 * The format of the string is: "----var@function". Parse it to 258 * display a nice warning. 259 */ 260 var = (char *)ptr + 4; 261 strlcpy(buf, var, sizeof(buf)); 262 var = buf; 263 fn = strchr(buf, '@'); 264 *fn++ = '\0'; 265 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 266 } else { 267 REPORT("MSan: Uninitialized %s memory, origin %x", 268 typename, orig); 269 } 270 271 out: 272 __compiler_membar(); 273 kmsan_reporting = false; 274 } 275 276 /* -------------------------------------------------------------------------- */ 277 278 static inline msan_meta_t 279 kmsan_meta_get(const void *addr, size_t size, const bool write) 280 { 281 msan_meta_t ret; 282 283 if (__predict_false(!kmsan_enabled)) { 284 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 285 ret.orig = (msan_orig_t *)msan_dummy_orig; 286 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 287 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 288 ret.orig = (msan_orig_t *)msan_dummy_orig; 289 } else { 290 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 291 ret.orig = 292 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 293 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 294 MSAN_ORIG_MASK); 295 } 296 297 return (ret); 298 } 299 300 static inline void 301 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 302 { 303 msan_orig_t *orig; 304 size_t i; 305 306 if (__predict_false(!kmsan_enabled)) 307 return; 308 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 309 return; 310 311 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 312 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 313 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 314 315 for (i = 0; i < size; i += 4) { 316 orig[i / 4] = o; 317 } 318 } 319 320 static inline void 321 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 322 { 323 uint8_t *shad; 324 325 if (__predict_false(!kmsan_enabled)) 326 return; 327 if (__predict_false(kmsan_md_unsupported(addr))) 328 return; 329 330 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 331 __builtin_memset(shad, c, size); 332 } 333 334 static inline void 335 kmsan_meta_copy(void *dst, const void *src, size_t size) 336 { 337 uint8_t *orig_src, *orig_dst; 338 uint8_t *shad_src, *shad_dst; 339 msan_orig_t *_src, *_dst; 340 size_t i; 341 342 if (__predict_false(!kmsan_enabled)) 343 return; 344 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 345 return; 346 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 347 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 348 return; 349 } 350 351 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 352 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 353 __builtin_memmove(shad_dst, shad_src, size); 354 355 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 356 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 357 for (i = 0; i < size; i++) { 358 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 359 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 360 *_dst = *_src; 361 orig_src++; 362 orig_dst++; 363 } 364 } 365 366 static inline void 367 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 368 { 369 uint8_t *shad; 370 size_t i; 371 372 if (__predict_false(!kmsan_enabled)) 373 return; 374 if (__predict_false(kmsan_md_unsupported(addr))) 375 return; 376 377 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 378 for (i = 0; i < size; i++) { 379 if (__predict_true(shad[i] == 0)) 380 continue; 381 kmsan_report_hook((const char *)addr + i, size, i, hook); 382 break; 383 } 384 } 385 386 void 387 kmsan_init_arg(size_t n) 388 { 389 msan_td_t *mtd; 390 uint8_t *arg; 391 392 if (__predict_false(!kmsan_enabled)) 393 return; 394 if (__predict_false(curthread == NULL)) 395 return; 396 mtd = curthread->td_kmsan; 397 arg = mtd->tls[mtd->ctx].param_shadow; 398 __builtin_memset(arg, 0, n); 399 } 400 401 void 402 kmsan_init_ret(size_t n) 403 { 404 msan_td_t *mtd; 405 uint8_t *arg; 406 407 if (__predict_false(!kmsan_enabled)) 408 return; 409 if (__predict_false(curthread == NULL)) 410 return; 411 mtd = curthread->td_kmsan; 412 arg = mtd->tls[mtd->ctx].retval_shadow; 413 __builtin_memset(arg, 0, n); 414 } 415 416 static void 417 kmsan_check_arg(size_t size, const char *hook) 418 { 419 msan_td_t *mtd; 420 uint8_t *arg; 421 size_t i; 422 423 if (__predict_false(!kmsan_enabled)) 424 return; 425 if (__predict_false(curthread == NULL)) 426 return; 427 mtd = curthread->td_kmsan; 428 arg = mtd->tls[mtd->ctx].param_shadow; 429 430 for (i = 0; i < size; i++) { 431 if (__predict_true(arg[i] == 0)) 432 continue; 433 kmsan_report_hook((const char *)arg + i, size, i, hook); 434 break; 435 } 436 } 437 438 void 439 kmsan_thread_alloc(struct thread *td) 440 { 441 msan_td_t *mtd; 442 443 if (!kmsan_enabled) 444 return; 445 446 mtd = td->td_kmsan; 447 if (mtd == NULL) { 448 /* We might be recycling a thread. */ 449 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 450 sizeof(int)); 451 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 452 } 453 kmsan_memset(mtd, 0, sizeof(*mtd)); 454 mtd->ctx = 0; 455 456 if (td->td_kstack != 0) 457 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 458 KMSAN_STATE_UNINIT); 459 460 td->td_kmsan = mtd; 461 } 462 463 void 464 kmsan_thread_free(struct thread *td) 465 { 466 msan_td_t *mtd; 467 468 if (!kmsan_enabled) 469 return; 470 if (__predict_false(td == curthread)) 471 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 472 473 mtd = td->td_kmsan; 474 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 475 free(mtd, M_KMSAN); 476 td->td_kmsan = NULL; 477 } 478 479 void kmsan_intr_enter(void); 480 void kmsan_intr_leave(void); 481 482 void 483 kmsan_intr_enter(void) 484 { 485 msan_td_t *mtd; 486 487 if (__predict_false(!kmsan_enabled)) 488 return; 489 490 mtd = curthread->td_kmsan; 491 mtd->ctx++; 492 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 493 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 494 } 495 496 void 497 kmsan_intr_leave(void) 498 { 499 msan_td_t *mtd; 500 501 if (__predict_false(!kmsan_enabled)) 502 return; 503 504 mtd = curthread->td_kmsan; 505 if (__predict_false(mtd->ctx == 0)) 506 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 507 mtd->ctx--; 508 } 509 510 /* -------------------------------------------------------------------------- */ 511 512 void 513 kmsan_shadow_map(vm_offset_t addr, size_t size) 514 { 515 size_t npages, i; 516 vm_offset_t va; 517 518 MPASS(addr % PAGE_SIZE == 0); 519 MPASS(size % PAGE_SIZE == 0); 520 521 if (!kmsan_enabled) 522 return; 523 524 npages = atop(size); 525 526 va = kmsan_md_addr_to_shad(addr); 527 for (i = 0; i < npages; i++) { 528 pmap_san_enter(va + ptoa(i)); 529 } 530 531 va = kmsan_md_addr_to_orig(addr); 532 for (i = 0; i < npages; i++) { 533 pmap_san_enter(va + ptoa(i)); 534 } 535 } 536 537 void 538 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 539 { 540 msan_orig_t orig; 541 542 orig = kmsan_md_orig_encode(type, pc); 543 kmsan_origin_fill(addr, orig, size); 544 } 545 546 void 547 kmsan_mark(const void *addr, size_t size, uint8_t c) 548 { 549 kmsan_shadow_fill((uintptr_t)addr, c, size); 550 } 551 552 void 553 kmsan_mark_bio(const struct bio *bp, uint8_t c) 554 { 555 kmsan_mark(bp->bio_data, bp->bio_length, c); 556 } 557 558 static void 559 kmsan_mark_ccb(const union ccb *ccb, uint8_t c) 560 { 561 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN) 562 return; 563 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 564 return; 565 566 switch (ccb->ccb_h.func_code) { 567 case XPT_SCSI_IO: { 568 const struct ccb_scsiio *scsiio; 569 570 scsiio = &ccb->ctio; 571 kmsan_mark(scsiio->data_ptr, scsiio->dxfer_len, c); 572 break; 573 } 574 case XPT_ATA_IO: { 575 const struct ccb_ataio *ataio; 576 577 ataio = &ccb->ataio; 578 kmsan_mark(ataio->data_ptr, ataio->dxfer_len, c); 579 break; 580 } 581 case XPT_NVME_IO: { 582 const struct ccb_nvmeio *nvmeio; 583 584 nvmeio = &ccb->nvmeio; 585 kmsan_mark(nvmeio->data_ptr, nvmeio->dxfer_len, c); 586 break; 587 } 588 default: 589 kmsan_panic("%s: unhandled CCB type %d", __func__, 590 ccb->ccb_h.func_code); 591 } 592 } 593 594 void 595 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 596 { 597 do { 598 if ((m->m_flags & M_EXTPG) == 0) 599 kmsan_mark(m->m_data, m->m_len, c); 600 m = m->m_next; 601 } while (m != NULL); 602 } 603 604 void 605 kmsan_check(const void *p, size_t sz, const char *descr) 606 { 607 kmsan_shadow_check((uintptr_t)p, sz, descr); 608 } 609 610 void 611 kmsan_check_bio(const struct bio *bp, const char *descr) 612 { 613 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 614 } 615 616 void 617 kmsan_check_ccb(const union ccb *ccb, const char *descr) 618 { 619 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) 620 return; 621 switch (ccb->ccb_h.func_code) { 622 case XPT_SCSI_IO: { 623 const struct ccb_scsiio *scsiio; 624 625 scsiio = &ccb->ctio; 626 kmsan_check(scsiio->data_ptr, scsiio->dxfer_len, descr); 627 break; 628 } 629 case XPT_ATA_IO: { 630 const struct ccb_ataio *ataio; 631 632 ataio = &ccb->ataio; 633 kmsan_check(ataio->data_ptr, ataio->dxfer_len, descr); 634 break; 635 } 636 case XPT_NVME_IO: { 637 const struct ccb_nvmeio *nvmeio; 638 639 nvmeio = &ccb->nvmeio; 640 kmsan_check(nvmeio->data_ptr, nvmeio->dxfer_len, descr); 641 break; 642 } 643 default: 644 kmsan_panic("%s: unhandled CCB type %d", __func__, 645 ccb->ccb_h.func_code); 646 } 647 } 648 649 void 650 kmsan_check_mbuf(const struct mbuf *m, const char *descr) 651 { 652 do { 653 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 654 } while ((m = m->m_next) != NULL); 655 } 656 657 void 658 kmsan_init(void) 659 { 660 int disabled; 661 662 disabled = 0; 663 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 664 if (disabled) 665 return; 666 667 /* Initialize the TLS for curthread. */ 668 msan_thread0.ctx = 0; 669 thread0.td_kmsan = &msan_thread0; 670 671 /* Now officially enabled. */ 672 kmsan_enabled = true; 673 } 674 675 /* -------------------------------------------------------------------------- */ 676 677 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 678 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 679 680 msan_meta_t 681 __msan_metadata_ptr_for_load_n(void *addr, size_t size) 682 { 683 return (kmsan_meta_get(addr, size, false)); 684 } 685 686 msan_meta_t 687 __msan_metadata_ptr_for_store_n(void *addr, size_t size) 688 { 689 return (kmsan_meta_get(addr, size, true)); 690 } 691 692 #define MSAN_META_FUNC(size) \ 693 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 694 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 695 { \ 696 return (kmsan_meta_get(addr, size, false)); \ 697 } \ 698 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 699 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 700 { \ 701 return (kmsan_meta_get(addr, size, true)); \ 702 } 703 704 MSAN_META_FUNC(1) 705 MSAN_META_FUNC(2) 706 MSAN_META_FUNC(4) 707 MSAN_META_FUNC(8) 708 709 void __msan_instrument_asm_store(const void *, size_t); 710 msan_orig_t __msan_chain_origin(msan_orig_t); 711 void __msan_poison(const void *, size_t); 712 void __msan_unpoison(const void *, size_t); 713 void __msan_poison_alloca(const void *, uint64_t, const char *); 714 void __msan_unpoison_alloca(const void *, uint64_t); 715 void __msan_warning(msan_orig_t); 716 msan_tls_t *__msan_get_context_state(void); 717 718 void 719 __msan_instrument_asm_store(const void *addr, size_t size) 720 { 721 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 722 } 723 724 msan_orig_t 725 __msan_chain_origin(msan_orig_t origin) 726 { 727 return (origin); 728 } 729 730 void 731 __msan_poison(const void *addr, size_t size) 732 { 733 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 734 } 735 736 void 737 __msan_unpoison(const void *addr, size_t size) 738 { 739 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 740 } 741 742 void 743 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 744 { 745 msan_orig_t orig; 746 747 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 748 kmsan_origin_fill(addr, orig, size); 749 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 750 } 751 752 void 753 __msan_unpoison_alloca(const void *addr, uint64_t size) 754 { 755 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 756 } 757 758 void 759 __msan_warning(msan_orig_t origin) 760 { 761 if (__predict_false(!kmsan_enabled)) 762 return; 763 kmsan_report_inline(origin, KMSAN_RET_ADDR); 764 } 765 766 msan_tls_t * 767 __msan_get_context_state(void) 768 { 769 msan_td_t *mtd; 770 771 /* 772 * When APs are started, they execute some C code before curthread is 773 * set. We have to handle that here. 774 */ 775 if (__predict_false(!kmsan_enabled || curthread == NULL)) 776 return (&dummy_tls); 777 mtd = curthread->td_kmsan; 778 return (&mtd->tls[mtd->ctx]); 779 } 780 781 /* -------------------------------------------------------------------------- */ 782 783 /* 784 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 785 * initialized areas properly. 786 */ 787 788 void * 789 kmsan_memcpy(void *dst, const void *src, size_t len) 790 { 791 /* No kmsan_check_arg, because inlined. */ 792 kmsan_init_ret(sizeof(void *)); 793 if (__predict_true(len != 0)) { 794 kmsan_meta_copy(dst, src, len); 795 } 796 return (__builtin_memcpy(dst, src, len)); 797 } 798 799 int 800 kmsan_memcmp(const void *b1, const void *b2, size_t len) 801 { 802 const uint8_t *_b1 = b1, *_b2 = b2; 803 size_t i; 804 805 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 806 "memcmp():args"); 807 kmsan_init_ret(sizeof(int)); 808 809 for (i = 0; i < len; i++) { 810 if (*_b1 != *_b2) { 811 kmsan_shadow_check((uintptr_t)b1, i + 1, 812 "memcmp():arg1"); 813 kmsan_shadow_check((uintptr_t)b2, i + 1, 814 "memcmp():arg2"); 815 return (*_b1 - *_b2); 816 } 817 _b1++, _b2++; 818 } 819 820 return (0); 821 } 822 823 void * 824 kmsan_memset(void *dst, int c, size_t len) 825 { 826 /* No kmsan_check_arg, because inlined. */ 827 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 828 kmsan_init_ret(sizeof(void *)); 829 return (__builtin_memset(dst, c, len)); 830 } 831 832 void * 833 kmsan_memmove(void *dst, const void *src, size_t len) 834 { 835 /* No kmsan_check_arg, because inlined. */ 836 if (__predict_true(len != 0)) { 837 kmsan_meta_copy(dst, src, len); 838 } 839 kmsan_init_ret(sizeof(void *)); 840 return (__builtin_memmove(dst, src, len)); 841 } 842 843 __strong_reference(kmsan_memcpy, __msan_memcpy); 844 __strong_reference(kmsan_memset, __msan_memset); 845 __strong_reference(kmsan_memmove, __msan_memmove); 846 847 char * 848 kmsan_strcpy(char *dst, const char *src) 849 { 850 const char *_src = src; 851 char *_dst = dst; 852 size_t len = 0; 853 854 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 855 856 while (1) { 857 len++; 858 *dst = *src; 859 if (*src == '\0') 860 break; 861 src++, dst++; 862 } 863 864 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 865 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 866 kmsan_init_ret(sizeof(char *)); 867 return (_dst); 868 } 869 870 int 871 kmsan_strcmp(const char *s1, const char *s2) 872 { 873 const char *_s1 = s1, *_s2 = s2; 874 size_t len = 0; 875 876 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 877 kmsan_init_ret(sizeof(int)); 878 879 while (1) { 880 len++; 881 if (*s1 != *s2) 882 break; 883 if (*s1 == '\0') { 884 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 885 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 886 return (0); 887 } 888 s1++, s2++; 889 } 890 891 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 892 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 893 894 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 895 } 896 897 size_t 898 kmsan_strlen(const char *str) 899 { 900 const char *s; 901 902 kmsan_check_arg(sizeof(str), "strlen():args"); 903 904 s = str; 905 while (1) { 906 if (*s == '\0') 907 break; 908 s++; 909 } 910 911 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 912 kmsan_init_ret(sizeof(size_t)); 913 return (s - str); 914 } 915 916 int kmsan_copyin(const void *, void *, size_t); 917 int kmsan_copyout(const void *, void *, size_t); 918 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 919 920 int 921 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 922 { 923 int ret; 924 925 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 926 "copyin():args"); 927 ret = copyin(uaddr, kaddr, len); 928 if (ret == 0) 929 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 930 kmsan_init_ret(sizeof(int)); 931 return (ret); 932 } 933 934 int 935 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 936 { 937 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 938 "copyout():args"); 939 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 940 kmsan_init_ret(sizeof(int)); 941 return (copyout(kaddr, uaddr, len)); 942 } 943 944 int 945 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 946 { 947 size_t _done; 948 int ret; 949 950 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 951 sizeof(len) + sizeof(done), "copyinstr():args"); 952 ret = copyinstr(uaddr, kaddr, len, &_done); 953 if (ret == 0) 954 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 955 if (done != NULL) { 956 *done = _done; 957 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 958 } 959 kmsan_init_ret(sizeof(int)); 960 return (ret); 961 } 962 963 /* -------------------------------------------------------------------------- */ 964 965 int 966 kmsan_fubyte(volatile const void *base) 967 { 968 int ret; 969 970 kmsan_check_arg(sizeof(base), "fubyte(): args"); 971 ret = fubyte(base); 972 kmsan_init_ret(sizeof(int)); 973 return (ret); 974 } 975 976 int 977 kmsan_fuword16(volatile const void *base) 978 { 979 int ret; 980 981 kmsan_check_arg(sizeof(base), "fuword16(): args"); 982 ret = fuword16(base); 983 kmsan_init_ret(sizeof(int)); 984 return (ret); 985 } 986 987 int 988 kmsan_fueword(volatile const void *base, long *val) 989 { 990 int ret; 991 992 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 993 ret = fueword(base, val); 994 if (ret == 0) 995 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 996 sizeof(*val)); 997 kmsan_init_ret(sizeof(int)); 998 return (ret); 999 } 1000 1001 int 1002 kmsan_fueword32(volatile const void *base, int32_t *val) 1003 { 1004 int ret; 1005 1006 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 1007 ret = fueword32(base, val); 1008 if (ret == 0) 1009 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 1010 sizeof(*val)); 1011 kmsan_init_ret(sizeof(int)); 1012 return (ret); 1013 } 1014 1015 int 1016 kmsan_fueword64(volatile const void *base, int64_t *val) 1017 { 1018 int ret; 1019 1020 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 1021 ret = fueword64(base, val); 1022 if (ret == 0) 1023 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 1024 sizeof(*val)); 1025 kmsan_init_ret(sizeof(int)); 1026 return (ret); 1027 } 1028 1029 int 1030 kmsan_subyte(volatile void *base, int byte) 1031 { 1032 int ret; 1033 1034 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 1035 ret = subyte(base, byte); 1036 kmsan_init_ret(sizeof(int)); 1037 return (ret); 1038 } 1039 1040 int 1041 kmsan_suword(volatile void *base, long word) 1042 { 1043 int ret; 1044 1045 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 1046 ret = suword(base, word); 1047 kmsan_init_ret(sizeof(int)); 1048 return (ret); 1049 } 1050 1051 int 1052 kmsan_suword16(volatile void *base, int word) 1053 { 1054 int ret; 1055 1056 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 1057 ret = suword16(base, word); 1058 kmsan_init_ret(sizeof(int)); 1059 return (ret); 1060 } 1061 1062 int 1063 kmsan_suword32(volatile void *base, int32_t word) 1064 { 1065 int ret; 1066 1067 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 1068 ret = suword32(base, word); 1069 kmsan_init_ret(sizeof(int)); 1070 return (ret); 1071 } 1072 1073 int 1074 kmsan_suword64(volatile void *base, int64_t word) 1075 { 1076 int ret; 1077 1078 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1079 ret = suword64(base, word); 1080 kmsan_init_ret(sizeof(int)); 1081 return (ret); 1082 } 1083 1084 int 1085 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1086 uint32_t newval) 1087 { 1088 int ret; 1089 1090 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1091 sizeof(newval), "casueword32(): args"); 1092 ret = casueword32(base, oldval, oldvalp, newval); 1093 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1094 sizeof(*oldvalp)); 1095 kmsan_init_ret(sizeof(int)); 1096 return (ret); 1097 } 1098 1099 int 1100 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1101 u_long newval) 1102 { 1103 int ret; 1104 1105 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1106 sizeof(newval), "casueword32(): args"); 1107 ret = casueword(base, oldval, oldvalp, newval); 1108 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1109 sizeof(*oldvalp)); 1110 kmsan_init_ret(sizeof(int)); 1111 return (ret); 1112 } 1113 1114 /* -------------------------------------------------------------------------- */ 1115 1116 #include <machine/atomic.h> 1117 #include <sys/atomic_san.h> 1118 1119 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1120 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1121 { \ 1122 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1123 "atomic_add_" #name "():args"); \ 1124 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1125 "atomic_add_" #name "():ptr"); \ 1126 atomic_add_##name(ptr, val); \ 1127 } 1128 1129 #define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1130 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1131 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1132 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1133 1134 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1135 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1136 { \ 1137 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1138 "atomic_subtract_" #name "():args"); \ 1139 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1140 "atomic_subtract_" #name "():ptr"); \ 1141 atomic_subtract_##name(ptr, val); \ 1142 } 1143 1144 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1145 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1146 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1147 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1148 1149 #define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1150 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1151 { \ 1152 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1153 "atomic_set_" #name "():args"); \ 1154 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1155 "atomic_set_" #name "():ptr"); \ 1156 atomic_set_##name(ptr, val); \ 1157 } 1158 1159 #define MSAN_ATOMIC_FUNC_SET(name, type) \ 1160 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1161 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1162 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1163 1164 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1165 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1166 { \ 1167 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1168 "atomic_clear_" #name "():args"); \ 1169 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1170 "atomic_clear_" #name "():ptr"); \ 1171 atomic_clear_##name(ptr, val); \ 1172 } 1173 1174 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1175 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1176 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1177 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1178 1179 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1180 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1181 { \ 1182 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1183 "atomic_fetchadd_" #name "():args"); \ 1184 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1185 "atomic_fetchadd_" #name "():ptr"); \ 1186 kmsan_init_ret(sizeof(type)); \ 1187 return (atomic_fetchadd_##name(ptr, val)); \ 1188 } 1189 1190 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1191 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1192 { \ 1193 kmsan_check_arg(sizeof(ptr), \ 1194 "atomic_readandclear_" #name "():args"); \ 1195 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1196 "atomic_readandclear_" #name "():ptr"); \ 1197 kmsan_init_ret(sizeof(type)); \ 1198 return (atomic_readandclear_##name(ptr)); \ 1199 } 1200 1201 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1202 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1203 { \ 1204 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1205 "atomic_testandclear_" #name "():args"); \ 1206 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1207 "atomic_testandclear_" #name "():ptr"); \ 1208 kmsan_init_ret(sizeof(int)); \ 1209 return (atomic_testandclear_##name(ptr, v)); \ 1210 } 1211 1212 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1213 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1214 { \ 1215 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1216 "atomic_testandset_" #name "():args"); \ 1217 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1218 "atomic_testandset_" #name "():ptr"); \ 1219 kmsan_init_ret(sizeof(int)); \ 1220 return (atomic_testandset_##name(ptr, v)); \ 1221 } 1222 1223 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1224 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1225 { \ 1226 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1227 "atomic_swap_" #name "():args"); \ 1228 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1229 "atomic_swap_" #name "():ptr"); \ 1230 kmsan_init_ret(sizeof(type)); \ 1231 return (atomic_swap_##name(ptr, val)); \ 1232 } 1233 1234 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1235 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1236 type nval) \ 1237 { \ 1238 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1239 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1240 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1241 "atomic_cmpset_" #name "():ptr"); \ 1242 kmsan_init_ret(sizeof(int)); \ 1243 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1244 } 1245 1246 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1247 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1248 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1249 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1250 1251 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1252 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1253 type nval) \ 1254 { \ 1255 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1256 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1257 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1258 "atomic_fcmpset_" #name "():ptr"); \ 1259 kmsan_init_ret(sizeof(int)); \ 1260 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1261 } 1262 1263 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1264 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1265 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1266 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1267 1268 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1269 void kmsan_atomic_thread_fence_##name(void) \ 1270 { \ 1271 atomic_thread_fence_##name(); \ 1272 } 1273 1274 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1275 type kmsan_atomic_load_##name(volatile type *ptr) \ 1276 { \ 1277 kmsan_check_arg(sizeof(ptr), \ 1278 "atomic_load_" #name "():args"); \ 1279 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1280 "atomic_load_" #name "():ptr"); \ 1281 kmsan_init_ret(sizeof(type)); \ 1282 return (atomic_load_##name(ptr)); \ 1283 } 1284 1285 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1286 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1287 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1288 1289 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1290 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1291 { \ 1292 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1293 "atomic_store_" #name "():args"); \ 1294 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1295 sizeof(type)); \ 1296 atomic_store_##name(ptr, val); \ 1297 } 1298 1299 #define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1300 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1301 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1302 1303 MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1304 MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1305 MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1306 MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1307 MSAN_ATOMIC_FUNC_ADD(int, u_int); 1308 MSAN_ATOMIC_FUNC_ADD(long, u_long); 1309 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1310 1311 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1312 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1313 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1314 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1315 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1316 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1317 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1318 1319 MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1320 MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1321 MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1322 MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1323 MSAN_ATOMIC_FUNC_SET(int, u_int); 1324 MSAN_ATOMIC_FUNC_SET(long, u_long); 1325 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1326 1327 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1328 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1329 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1330 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1331 MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1332 MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1333 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1334 1335 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1336 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1337 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1338 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1339 1340 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1341 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1342 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1343 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1344 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1345 1346 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1347 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1348 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1349 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1350 1351 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1352 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1353 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1354 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1355 1356 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1357 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1358 MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1359 MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1360 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1361 1362 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1363 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1364 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1365 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1366 MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1367 MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1368 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1369 1370 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1371 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1372 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1373 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1374 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1375 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1376 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1377 1378 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1379 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1380 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1381 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1382 MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1383 MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1384 MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1385 MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1386 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1387 1388 MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1389 MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1390 MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1391 MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1392 MSAN_ATOMIC_FUNC_STORE(char, u_char); 1393 MSAN_ATOMIC_FUNC_STORE(short, u_short); 1394 MSAN_ATOMIC_FUNC_STORE(int, u_int); 1395 MSAN_ATOMIC_FUNC_STORE(long, u_long); 1396 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1397 1398 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1399 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1400 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1401 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1402 1403 void 1404 kmsan_atomic_interrupt_fence(void) 1405 { 1406 atomic_interrupt_fence(); 1407 } 1408 1409 /* -------------------------------------------------------------------------- */ 1410 1411 #include <sys/bus.h> 1412 #include <machine/bus.h> 1413 #include <sys/bus_san.h> 1414 1415 int 1416 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1417 int flags, bus_space_handle_t *handlep) 1418 { 1419 return (bus_space_map(tag, hnd, size, flags, handlep)); 1420 } 1421 1422 void 1423 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1424 bus_size_t size) 1425 { 1426 bus_space_unmap(tag, hnd, size); 1427 } 1428 1429 int 1430 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1431 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1432 { 1433 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1434 } 1435 1436 void 1437 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1438 bus_size_t size) 1439 { 1440 bus_space_free(tag, hnd, size); 1441 } 1442 1443 void 1444 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1445 bus_size_t offset, bus_size_t size, int flags) 1446 { 1447 bus_space_barrier(tag, hnd, offset, size, flags); 1448 } 1449 1450 /* XXXMJ x86-specific */ 1451 #define MSAN_BUS_READ_FUNC(func, width, type) \ 1452 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1453 bus_space_handle_t hnd, bus_size_t offset) \ 1454 { \ 1455 type ret; \ 1456 if ((tag) != X86_BUS_SPACE_IO) \ 1457 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1458 KMSAN_STATE_INITED, (width)); \ 1459 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1460 kmsan_init_ret(sizeof(type)); \ 1461 return (ret); \ 1462 } \ 1463 1464 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1465 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1466 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1467 bus_size_t count) \ 1468 { \ 1469 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1470 (width) * count); \ 1471 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1472 count); \ 1473 } 1474 1475 MSAN_BUS_READ_FUNC(, 1, uint8_t) 1476 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1477 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1478 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1479 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1480 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1481 1482 MSAN_BUS_READ_FUNC(, 2, uint16_t) 1483 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1484 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1485 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1486 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1487 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1488 1489 MSAN_BUS_READ_FUNC(, 4, uint32_t) 1490 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1491 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1492 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1493 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1494 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1495 1496 MSAN_BUS_READ_FUNC(, 8, uint64_t) 1497 1498 #define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1499 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1500 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1501 { \ 1502 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1503 } \ 1504 1505 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1506 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1507 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1508 bus_size_t count) \ 1509 { \ 1510 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1511 "bus_space_write()"); \ 1512 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1513 count); \ 1514 } 1515 1516 MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1517 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1518 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1519 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1520 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1521 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1522 1523 MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1524 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1525 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1526 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1527 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1528 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1529 1530 MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1531 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1532 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1533 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1534 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1535 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1536 1537 MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1538 1539 #define MSAN_BUS_SET_FUNC(func, width, type) \ 1540 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1541 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1542 bus_size_t count) \ 1543 { \ 1544 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1545 count); \ 1546 } 1547 1548 MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1549 MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1550 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1551 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1552 1553 MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1554 MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1555 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1556 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1557 1558 MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1559 MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1560 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1561 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1562 1563 /* -------------------------------------------------------------------------- */ 1564 1565 void 1566 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1567 { 1568 /* 1569 * Some drivers, e.g., nvme, use the same code path for loading device 1570 * read and write requests, and will thus specify both flags. In this 1571 * case we should not do any checking since it will generally lead to 1572 * false positives. 1573 */ 1574 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1575 BUS_DMASYNC_PREWRITE) { 1576 switch (desc->md_type) { 1577 case MEMDESC_VADDR: 1578 kmsan_check(desc->u.md_vaddr, desc->md_opaque, 1579 "dmasync"); 1580 break; 1581 case MEMDESC_BIO: 1582 kmsan_check_bio(desc->u.md_bio, "dmasync"); 1583 break; 1584 case MEMDESC_MBUF: 1585 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1586 break; 1587 case MEMDESC_CCB: 1588 kmsan_check_ccb(desc->u.md_ccb, "dmasync"); 1589 break; 1590 case 0: 1591 break; 1592 default: 1593 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1594 desc->md_type); 1595 } 1596 } 1597 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1598 switch (desc->md_type) { 1599 case MEMDESC_VADDR: 1600 kmsan_mark(desc->u.md_vaddr, desc->md_opaque, 1601 KMSAN_STATE_INITED); 1602 break; 1603 case MEMDESC_BIO: 1604 kmsan_mark_bio(desc->u.md_bio, KMSAN_STATE_INITED); 1605 break; 1606 case MEMDESC_MBUF: 1607 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1608 break; 1609 case MEMDESC_CCB: 1610 kmsan_mark_ccb(desc->u.md_ccb, KMSAN_STATE_INITED); 1611 break; 1612 case 0: 1613 break; 1614 default: 1615 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1616 desc->md_type); 1617 } 1618 } 1619 } 1620