1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define SAN_RUNTIME 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 #if 0 40 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bio.h> 46 #include <sys/buf.h> 47 #include <sys/conf.h> 48 #include <sys/kdb.h> 49 #include <sys/kernel.h> 50 #include <sys/linker.h> 51 #include <sys/malloc.h> 52 #include <sys/mbuf.h> 53 #include <sys/memdesc.h> 54 #include <sys/msan.h> 55 #include <sys/proc.h> 56 #include <sys/stack.h> 57 #include <sys/sysctl.h> 58 #include <sys/uio.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 63 #include <machine/msan.h> 64 #include <machine/stdarg.h> 65 66 void kmsan_init_arg(size_t); 67 void kmsan_init_ret(size_t); 68 69 /* -------------------------------------------------------------------------- */ 70 71 /* 72 * Part of the compiler ABI. 73 */ 74 75 typedef struct { 76 uint8_t *shad; 77 msan_orig_t *orig; 78 } msan_meta_t; 79 80 #define MSAN_PARAM_SIZE 800 81 #define MSAN_RETVAL_SIZE 800 82 typedef struct { 83 uint8_t param_shadow[MSAN_PARAM_SIZE]; 84 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 85 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 86 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 87 uint64_t va_arg_overflow_size; 88 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 89 msan_orig_t retval_origin; 90 } msan_tls_t; 91 92 /* -------------------------------------------------------------------------- */ 93 94 #define MSAN_NCONTEXT 4 95 #define MSAN_ORIG_MASK (~0x3) 96 97 typedef struct kmsan_td { 98 size_t ctx; 99 msan_tls_t tls[MSAN_NCONTEXT]; 100 } msan_td_t; 101 102 static msan_tls_t dummy_tls; 103 104 /* 105 * Use separate dummy regions for loads and stores: stores may mark the region 106 * as uninitialized, and that can trigger false positives. 107 */ 108 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 109 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 110 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 111 static msan_td_t msan_thread0; 112 static bool kmsan_enabled __read_mostly; 113 114 static bool kmsan_reporting = false; 115 116 /* 117 * Avoid clobbering any thread-local state before we panic. 118 */ 119 #define kmsan_panic(f, ...) do { \ 120 kmsan_enabled = false; \ 121 panic(f, __VA_ARGS__); \ 122 } while (0) 123 124 #define REPORT(f, ...) do { \ 125 if (panic_on_violation) { \ 126 kmsan_panic(f, __VA_ARGS__); \ 127 } else { \ 128 struct stack st; \ 129 \ 130 stack_save(&st); \ 131 printf(f "\n", __VA_ARGS__); \ 132 stack_print_ddb(&st); \ 133 } \ 134 } while (0) 135 136 FEATURE(kmsan, "Kernel memory sanitizer"); 137 138 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 139 "KMSAN options"); 140 141 static bool panic_on_violation = 1; 142 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 143 &panic_on_violation, 0, 144 "Panic if an invalid access is detected"); 145 146 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 147 148 /* -------------------------------------------------------------------------- */ 149 150 static inline const char * 151 kmsan_orig_name(int type) 152 { 153 switch (type) { 154 case KMSAN_TYPE_STACK: 155 return ("stack"); 156 case KMSAN_TYPE_KMEM: 157 return ("kmem"); 158 case KMSAN_TYPE_MALLOC: 159 return ("malloc"); 160 case KMSAN_TYPE_UMA: 161 return ("UMA"); 162 default: 163 return ("unknown"); 164 } 165 } 166 167 static void 168 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, 169 const char *hook) 170 { 171 const char *typename; 172 char *var, *fn; 173 uintptr_t ptr; 174 long foff; 175 char buf[128]; 176 int type; 177 178 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 179 return; 180 181 kmsan_reporting = true; 182 __compiler_membar(); 183 184 if (*orig == 0) { 185 REPORT("MSan: Uninitialized memory in %s, offset %zu", 186 hook, off); 187 goto out; 188 } 189 190 kmsan_md_orig_decode(*orig, &type, &ptr); 191 typename = kmsan_orig_name(type); 192 193 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 194 sizeof(buf), &foff) == 0) { 195 REPORT("MSan: Uninitialized %s memory in %s, " 196 "offset %zu/%zu, addr %p, from %s+%#lx", 197 typename, hook, off, size, addr, buf, foff); 198 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 199 /* 200 * The format of the string is: "----var@function". Parse it to 201 * display a nice warning. 202 */ 203 var = (char *)ptr + 4; 204 strlcpy(buf, var, sizeof(buf)); 205 var = buf; 206 fn = strchr(buf, '@'); 207 *fn++ = '\0'; 208 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 209 "variable '%s' from %s", typename, hook, off, var, fn); 210 } else { 211 REPORT("MSan: Uninitialized %s memory in %s, " 212 "offset %zu/%zu, addr %p, PC %p", 213 typename, hook, off, size, addr, (void *)ptr); 214 } 215 216 out: 217 __compiler_membar(); 218 kmsan_reporting = false; 219 } 220 221 static void 222 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 223 { 224 const char *typename; 225 char *var, *fn; 226 uintptr_t ptr; 227 char buf[128]; 228 long foff; 229 int type; 230 231 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 232 return; 233 234 kmsan_reporting = true; 235 __compiler_membar(); 236 237 if (orig == 0) { 238 REPORT("MSan: uninitialized variable in %p", (void *)pc); 239 goto out; 240 } 241 242 kmsan_md_orig_decode(orig, &type, &ptr); 243 typename = kmsan_orig_name(type); 244 245 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 246 sizeof(buf), &foff) == 0) { 247 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 248 typename, buf, foff); 249 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 250 /* 251 * The format of the string is: "----var@function". Parse it to 252 * display a nice warning. 253 */ 254 var = (char *)ptr + 4; 255 strlcpy(buf, var, sizeof(buf)); 256 var = buf; 257 fn = strchr(buf, '@'); 258 *fn++ = '\0'; 259 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 260 } else { 261 REPORT("MSan: Uninitialized %s memory, origin %x", 262 typename, orig); 263 } 264 265 out: 266 __compiler_membar(); 267 kmsan_reporting = false; 268 } 269 270 /* -------------------------------------------------------------------------- */ 271 272 static inline msan_meta_t 273 kmsan_meta_get(const void *addr, size_t size, const bool write) 274 { 275 msan_meta_t ret; 276 277 if (__predict_false(!kmsan_enabled)) { 278 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 279 ret.orig = (msan_orig_t *)msan_dummy_orig; 280 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 281 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 282 ret.orig = (msan_orig_t *)msan_dummy_orig; 283 } else { 284 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 285 ret.orig = 286 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 287 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 288 MSAN_ORIG_MASK); 289 } 290 291 return (ret); 292 } 293 294 static inline void 295 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 296 { 297 msan_orig_t *orig; 298 size_t i; 299 300 if (__predict_false(!kmsan_enabled)) 301 return; 302 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 303 return; 304 305 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 306 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 307 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 308 309 for (i = 0; i < size; i += 4) { 310 orig[i / 4] = o; 311 } 312 } 313 314 static inline void 315 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 316 { 317 uint8_t *shad; 318 319 if (__predict_false(!kmsan_enabled)) 320 return; 321 if (__predict_false(kmsan_md_unsupported(addr))) 322 return; 323 324 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 325 __builtin_memset(shad, c, size); 326 } 327 328 static inline void 329 kmsan_meta_copy(void *dst, const void *src, size_t size) 330 { 331 uint8_t *orig_src, *orig_dst; 332 uint8_t *shad_src, *shad_dst; 333 msan_orig_t *_src, *_dst; 334 size_t i; 335 336 if (__predict_false(!kmsan_enabled)) 337 return; 338 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 339 return; 340 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 341 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 342 return; 343 } 344 345 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 346 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 347 __builtin_memmove(shad_dst, shad_src, size); 348 349 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 350 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 351 for (i = 0; i < size; i++) { 352 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 353 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 354 *_dst = *_src; 355 orig_src++; 356 orig_dst++; 357 } 358 } 359 360 static inline void 361 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 362 { 363 msan_orig_t *orig; 364 uint8_t *shad; 365 size_t i; 366 367 if (__predict_false(!kmsan_enabled)) 368 return; 369 if (__predict_false(kmsan_md_unsupported(addr))) 370 return; 371 372 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 373 for (i = 0; i < size; i++) { 374 if (__predict_true(shad[i] == 0)) 375 continue; 376 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)&shad[i]); 377 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 378 kmsan_report_hook((const char *)addr + i, orig, size, i, hook); 379 break; 380 } 381 } 382 383 void 384 kmsan_init_arg(size_t n) 385 { 386 msan_td_t *mtd; 387 uint8_t *arg; 388 389 if (__predict_false(!kmsan_enabled)) 390 return; 391 if (__predict_false(curthread == NULL)) 392 return; 393 mtd = curthread->td_kmsan; 394 arg = mtd->tls[mtd->ctx].param_shadow; 395 __builtin_memset(arg, 0, n); 396 } 397 398 void 399 kmsan_init_ret(size_t n) 400 { 401 msan_td_t *mtd; 402 uint8_t *arg; 403 404 if (__predict_false(!kmsan_enabled)) 405 return; 406 if (__predict_false(curthread == NULL)) 407 return; 408 mtd = curthread->td_kmsan; 409 arg = mtd->tls[mtd->ctx].retval_shadow; 410 __builtin_memset(arg, 0, n); 411 } 412 413 static void 414 kmsan_check_arg(size_t size, const char *hook) 415 { 416 msan_orig_t *orig; 417 msan_td_t *mtd; 418 uint8_t *arg; 419 size_t ctx, i; 420 421 if (__predict_false(!kmsan_enabled)) 422 return; 423 if (__predict_false(curthread == NULL)) 424 return; 425 mtd = curthread->td_kmsan; 426 ctx = mtd->ctx; 427 arg = mtd->tls[ctx].param_shadow; 428 429 for (i = 0; i < size; i++) { 430 if (__predict_true(arg[i] == 0)) 431 continue; 432 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; 433 kmsan_report_hook((const char *)arg + i, orig, size, i, hook); 434 break; 435 } 436 } 437 438 void 439 kmsan_thread_alloc(struct thread *td) 440 { 441 msan_td_t *mtd; 442 443 if (!kmsan_enabled) 444 return; 445 446 mtd = td->td_kmsan; 447 if (mtd == NULL) { 448 /* We might be recycling a thread. */ 449 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 450 sizeof(int)); 451 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 452 } 453 kmsan_memset(mtd, 0, sizeof(*mtd)); 454 mtd->ctx = 0; 455 456 if (td->td_kstack != 0) 457 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 458 KMSAN_STATE_UNINIT); 459 460 td->td_kmsan = mtd; 461 } 462 463 void 464 kmsan_thread_free(struct thread *td) 465 { 466 msan_td_t *mtd; 467 468 if (!kmsan_enabled) 469 return; 470 if (__predict_false(td == curthread)) 471 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 472 473 mtd = td->td_kmsan; 474 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 475 free(mtd, M_KMSAN); 476 td->td_kmsan = NULL; 477 } 478 479 void kmsan_intr_enter(void); 480 void kmsan_intr_leave(void); 481 482 void 483 kmsan_intr_enter(void) 484 { 485 msan_td_t *mtd; 486 487 if (__predict_false(!kmsan_enabled)) 488 return; 489 490 mtd = curthread->td_kmsan; 491 mtd->ctx++; 492 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 493 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 494 } 495 496 void 497 kmsan_intr_leave(void) 498 { 499 msan_td_t *mtd; 500 501 if (__predict_false(!kmsan_enabled)) 502 return; 503 504 mtd = curthread->td_kmsan; 505 if (__predict_false(mtd->ctx == 0)) 506 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 507 mtd->ctx--; 508 } 509 510 /* -------------------------------------------------------------------------- */ 511 512 void 513 kmsan_shadow_map(vm_offset_t addr, size_t size) 514 { 515 size_t npages, i; 516 vm_offset_t va; 517 518 MPASS(addr % PAGE_SIZE == 0); 519 MPASS(size % PAGE_SIZE == 0); 520 521 if (!kmsan_enabled) 522 return; 523 524 npages = atop(size); 525 526 va = kmsan_md_addr_to_shad(addr); 527 for (i = 0; i < npages; i++) { 528 pmap_san_enter(va + ptoa(i)); 529 } 530 531 va = kmsan_md_addr_to_orig(addr); 532 for (i = 0; i < npages; i++) { 533 pmap_san_enter(va + ptoa(i)); 534 } 535 } 536 537 void 538 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 539 { 540 msan_orig_t orig; 541 542 orig = kmsan_md_orig_encode(type, pc); 543 kmsan_origin_fill(addr, orig, size); 544 } 545 546 void 547 kmsan_mark(const void *addr, size_t size, uint8_t c) 548 { 549 kmsan_shadow_fill((uintptr_t)addr, c, size); 550 } 551 552 void 553 kmsan_mark_bio(const struct bio *bp, uint8_t c) 554 { 555 kmsan_mark(bp->bio_data, bp->bio_length, c); 556 } 557 558 void 559 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 560 { 561 do { 562 if ((m->m_flags & M_EXTPG) == 0) 563 kmsan_mark(m->m_data, m->m_len, c); 564 m = m->m_next; 565 } while (m != NULL); 566 } 567 568 void 569 kmsan_check(const void *p, size_t sz, const char *descr) 570 { 571 kmsan_shadow_check((uintptr_t)p, sz, descr); 572 } 573 574 void 575 kmsan_check_bio(const struct bio *bp, const char *descr) 576 { 577 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 578 } 579 580 void 581 kmsan_check_mbuf(const struct mbuf *m, const char *descr) 582 { 583 do { 584 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 585 } while ((m = m->m_next) != NULL); 586 } 587 588 void 589 kmsan_init(void) 590 { 591 int disabled; 592 593 disabled = 0; 594 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 595 if (disabled) 596 return; 597 598 /* Initialize the TLS for curthread. */ 599 msan_thread0.ctx = 0; 600 thread0.td_kmsan = &msan_thread0; 601 602 /* Now officially enabled. */ 603 kmsan_enabled = true; 604 } 605 606 /* -------------------------------------------------------------------------- */ 607 608 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 609 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 610 611 msan_meta_t 612 __msan_metadata_ptr_for_load_n(void *addr, size_t size) 613 { 614 return (kmsan_meta_get(addr, size, false)); 615 } 616 617 msan_meta_t 618 __msan_metadata_ptr_for_store_n(void *addr, size_t size) 619 { 620 return (kmsan_meta_get(addr, size, true)); 621 } 622 623 #define MSAN_META_FUNC(size) \ 624 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 625 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 626 { \ 627 return (kmsan_meta_get(addr, size, false)); \ 628 } \ 629 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 630 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 631 { \ 632 return (kmsan_meta_get(addr, size, true)); \ 633 } 634 635 MSAN_META_FUNC(1) 636 MSAN_META_FUNC(2) 637 MSAN_META_FUNC(4) 638 MSAN_META_FUNC(8) 639 640 void __msan_instrument_asm_store(const void *, size_t); 641 msan_orig_t __msan_chain_origin(msan_orig_t); 642 void __msan_poison(const void *, size_t); 643 void __msan_unpoison(const void *, size_t); 644 void __msan_poison_alloca(const void *, uint64_t, const char *); 645 void __msan_unpoison_alloca(const void *, uint64_t); 646 void __msan_warning(msan_orig_t); 647 msan_tls_t *__msan_get_context_state(void); 648 649 void 650 __msan_instrument_asm_store(const void *addr, size_t size) 651 { 652 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 653 } 654 655 msan_orig_t 656 __msan_chain_origin(msan_orig_t origin) 657 { 658 return (origin); 659 } 660 661 void 662 __msan_poison(const void *addr, size_t size) 663 { 664 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 665 } 666 667 void 668 __msan_unpoison(const void *addr, size_t size) 669 { 670 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 671 } 672 673 void 674 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 675 { 676 msan_orig_t orig; 677 678 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 679 kmsan_origin_fill(addr, orig, size); 680 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 681 } 682 683 void 684 __msan_unpoison_alloca(const void *addr, uint64_t size) 685 { 686 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 687 } 688 689 void 690 __msan_warning(msan_orig_t origin) 691 { 692 if (__predict_false(!kmsan_enabled)) 693 return; 694 kmsan_report_inline(origin, KMSAN_RET_ADDR); 695 } 696 697 msan_tls_t * 698 __msan_get_context_state(void) 699 { 700 msan_td_t *mtd; 701 702 /* 703 * When APs are started, they execute some C code before curthread is 704 * set. We have to handle that here. 705 */ 706 if (__predict_false(!kmsan_enabled || curthread == NULL)) 707 return (&dummy_tls); 708 mtd = curthread->td_kmsan; 709 return (&mtd->tls[mtd->ctx]); 710 } 711 712 /* -------------------------------------------------------------------------- */ 713 714 /* 715 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 716 * initialized areas properly. 717 */ 718 719 void * 720 kmsan_memcpy(void *dst, const void *src, size_t len) 721 { 722 /* No kmsan_check_arg, because inlined. */ 723 kmsan_init_ret(sizeof(void *)); 724 if (__predict_true(len != 0)) { 725 kmsan_meta_copy(dst, src, len); 726 } 727 return (__builtin_memcpy(dst, src, len)); 728 } 729 730 int 731 kmsan_memcmp(const void *b1, const void *b2, size_t len) 732 { 733 const uint8_t *_b1 = b1, *_b2 = b2; 734 size_t i; 735 736 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 737 "memcmp():args"); 738 kmsan_init_ret(sizeof(int)); 739 740 for (i = 0; i < len; i++) { 741 if (*_b1 != *_b2) { 742 kmsan_shadow_check((uintptr_t)b1, i + 1, 743 "memcmp():arg1"); 744 kmsan_shadow_check((uintptr_t)b2, i + 1, 745 "memcmp():arg2"); 746 return (*_b1 - *_b2); 747 } 748 _b1++, _b2++; 749 } 750 751 return (0); 752 } 753 754 void * 755 kmsan_memset(void *dst, int c, size_t len) 756 { 757 /* No kmsan_check_arg, because inlined. */ 758 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 759 kmsan_init_ret(sizeof(void *)); 760 return (__builtin_memset(dst, c, len)); 761 } 762 763 void * 764 kmsan_memmove(void *dst, const void *src, size_t len) 765 { 766 /* No kmsan_check_arg, because inlined. */ 767 if (__predict_true(len != 0)) { 768 kmsan_meta_copy(dst, src, len); 769 } 770 kmsan_init_ret(sizeof(void *)); 771 return (__builtin_memmove(dst, src, len)); 772 } 773 774 __strong_reference(kmsan_memcpy, __msan_memcpy); 775 __strong_reference(kmsan_memset, __msan_memset); 776 __strong_reference(kmsan_memmove, __msan_memmove); 777 778 char * 779 kmsan_strcpy(char *dst, const char *src) 780 { 781 const char *_src = src; 782 char *_dst = dst; 783 size_t len = 0; 784 785 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 786 787 while (1) { 788 len++; 789 *dst = *src; 790 if (*src == '\0') 791 break; 792 src++, dst++; 793 } 794 795 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 796 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 797 kmsan_init_ret(sizeof(char *)); 798 return (_dst); 799 } 800 801 int 802 kmsan_strcmp(const char *s1, const char *s2) 803 { 804 const char *_s1 = s1, *_s2 = s2; 805 size_t len = 0; 806 807 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 808 kmsan_init_ret(sizeof(int)); 809 810 while (1) { 811 len++; 812 if (*s1 != *s2) 813 break; 814 if (*s1 == '\0') { 815 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 816 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 817 return (0); 818 } 819 s1++, s2++; 820 } 821 822 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 823 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 824 825 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 826 } 827 828 size_t 829 kmsan_strlen(const char *str) 830 { 831 const char *s; 832 833 kmsan_check_arg(sizeof(str), "strlen():args"); 834 835 s = str; 836 while (1) { 837 if (*s == '\0') 838 break; 839 s++; 840 } 841 842 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 843 kmsan_init_ret(sizeof(size_t)); 844 return (s - str); 845 } 846 847 int kmsan_copyin(const void *, void *, size_t); 848 int kmsan_copyout(const void *, void *, size_t); 849 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 850 851 int 852 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 853 { 854 int ret; 855 856 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 857 "copyin():args"); 858 ret = copyin(uaddr, kaddr, len); 859 if (ret == 0) 860 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 861 kmsan_init_ret(sizeof(int)); 862 return (ret); 863 } 864 865 int 866 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 867 { 868 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 869 "copyout():args"); 870 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 871 kmsan_init_ret(sizeof(int)); 872 return (copyout(kaddr, uaddr, len)); 873 } 874 875 int 876 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 877 { 878 size_t _done; 879 int ret; 880 881 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 882 sizeof(len) + sizeof(done), "copyinstr():args"); 883 ret = copyinstr(uaddr, kaddr, len, &_done); 884 if (ret == 0) 885 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 886 if (done != NULL) { 887 *done = _done; 888 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 889 } 890 kmsan_init_ret(sizeof(int)); 891 return (ret); 892 } 893 894 /* -------------------------------------------------------------------------- */ 895 896 int 897 kmsan_fubyte(volatile const void *base) 898 { 899 int ret; 900 901 kmsan_check_arg(sizeof(base), "fubyte(): args"); 902 ret = fubyte(base); 903 kmsan_init_ret(sizeof(int)); 904 return (ret); 905 } 906 907 int 908 kmsan_fuword16(volatile const void *base) 909 { 910 int ret; 911 912 kmsan_check_arg(sizeof(base), "fuword16(): args"); 913 ret = fuword16(base); 914 kmsan_init_ret(sizeof(int)); 915 return (ret); 916 } 917 918 int 919 kmsan_fueword(volatile const void *base, long *val) 920 { 921 int ret; 922 923 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 924 ret = fueword(base, val); 925 if (ret == 0) 926 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 927 sizeof(*val)); 928 kmsan_init_ret(sizeof(int)); 929 return (ret); 930 } 931 932 int 933 kmsan_fueword32(volatile const void *base, int32_t *val) 934 { 935 int ret; 936 937 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 938 ret = fueword32(base, val); 939 if (ret == 0) 940 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 941 sizeof(*val)); 942 kmsan_init_ret(sizeof(int)); 943 return (ret); 944 } 945 946 int 947 kmsan_fueword64(volatile const void *base, int64_t *val) 948 { 949 int ret; 950 951 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 952 ret = fueword64(base, val); 953 if (ret == 0) 954 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 955 sizeof(*val)); 956 kmsan_init_ret(sizeof(int)); 957 return (ret); 958 } 959 960 int 961 kmsan_subyte(volatile void *base, int byte) 962 { 963 int ret; 964 965 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 966 ret = subyte(base, byte); 967 kmsan_init_ret(sizeof(int)); 968 return (ret); 969 } 970 971 int 972 kmsan_suword(volatile void *base, long word) 973 { 974 int ret; 975 976 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 977 ret = suword(base, word); 978 kmsan_init_ret(sizeof(int)); 979 return (ret); 980 } 981 982 int 983 kmsan_suword16(volatile void *base, int word) 984 { 985 int ret; 986 987 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 988 ret = suword16(base, word); 989 kmsan_init_ret(sizeof(int)); 990 return (ret); 991 } 992 993 int 994 kmsan_suword32(volatile void *base, int32_t word) 995 { 996 int ret; 997 998 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 999 ret = suword32(base, word); 1000 kmsan_init_ret(sizeof(int)); 1001 return (ret); 1002 } 1003 1004 int 1005 kmsan_suword64(volatile void *base, int64_t word) 1006 { 1007 int ret; 1008 1009 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1010 ret = suword64(base, word); 1011 kmsan_init_ret(sizeof(int)); 1012 return (ret); 1013 } 1014 1015 int 1016 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1017 uint32_t newval) 1018 { 1019 int ret; 1020 1021 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1022 sizeof(newval), "casueword32(): args"); 1023 ret = casueword32(base, oldval, oldvalp, newval); 1024 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1025 sizeof(*oldvalp)); 1026 kmsan_init_ret(sizeof(int)); 1027 return (ret); 1028 } 1029 1030 int 1031 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1032 u_long newval) 1033 { 1034 int ret; 1035 1036 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1037 sizeof(newval), "casueword32(): args"); 1038 ret = casueword(base, oldval, oldvalp, newval); 1039 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1040 sizeof(*oldvalp)); 1041 kmsan_init_ret(sizeof(int)); 1042 return (ret); 1043 } 1044 1045 /* -------------------------------------------------------------------------- */ 1046 1047 #include <machine/atomic.h> 1048 #include <sys/atomic_san.h> 1049 1050 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1051 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1052 { \ 1053 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1054 "atomic_add_" #name "():args"); \ 1055 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1056 "atomic_add_" #name "():ptr"); \ 1057 atomic_add_##name(ptr, val); \ 1058 } 1059 1060 #define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1061 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1062 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1063 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1064 1065 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1066 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1067 { \ 1068 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1069 "atomic_subtract_" #name "():args"); \ 1070 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1071 "atomic_subtract_" #name "():ptr"); \ 1072 atomic_subtract_##name(ptr, val); \ 1073 } 1074 1075 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1076 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1077 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1078 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1079 1080 #define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1081 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1082 { \ 1083 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1084 "atomic_set_" #name "():args"); \ 1085 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1086 "atomic_set_" #name "():ptr"); \ 1087 atomic_set_##name(ptr, val); \ 1088 } 1089 1090 #define MSAN_ATOMIC_FUNC_SET(name, type) \ 1091 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1092 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1093 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1094 1095 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1096 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1097 { \ 1098 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1099 "atomic_clear_" #name "():args"); \ 1100 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1101 "atomic_clear_" #name "():ptr"); \ 1102 atomic_clear_##name(ptr, val); \ 1103 } 1104 1105 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1106 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1107 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1108 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1109 1110 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1111 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1112 { \ 1113 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1114 "atomic_fetchadd_" #name "():args"); \ 1115 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1116 "atomic_fetchadd_" #name "():ptr"); \ 1117 kmsan_init_ret(sizeof(type)); \ 1118 return (atomic_fetchadd_##name(ptr, val)); \ 1119 } 1120 1121 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1122 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1123 { \ 1124 kmsan_check_arg(sizeof(ptr), \ 1125 "atomic_readandclear_" #name "():args"); \ 1126 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1127 "atomic_readandclear_" #name "():ptr"); \ 1128 kmsan_init_ret(sizeof(type)); \ 1129 return (atomic_readandclear_##name(ptr)); \ 1130 } 1131 1132 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1133 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1134 { \ 1135 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1136 "atomic_testandclear_" #name "():args"); \ 1137 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1138 "atomic_testandclear_" #name "():ptr"); \ 1139 kmsan_init_ret(sizeof(int)); \ 1140 return (atomic_testandclear_##name(ptr, v)); \ 1141 } 1142 1143 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1144 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1145 { \ 1146 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1147 "atomic_testandset_" #name "():args"); \ 1148 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1149 "atomic_testandset_" #name "():ptr"); \ 1150 kmsan_init_ret(sizeof(int)); \ 1151 return (atomic_testandset_##name(ptr, v)); \ 1152 } 1153 1154 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1155 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1156 { \ 1157 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1158 "atomic_swap_" #name "():args"); \ 1159 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1160 "atomic_swap_" #name "():ptr"); \ 1161 kmsan_init_ret(sizeof(type)); \ 1162 return (atomic_swap_##name(ptr, val)); \ 1163 } 1164 1165 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1166 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1167 type nval) \ 1168 { \ 1169 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1170 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1171 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1172 "atomic_cmpset_" #name "():ptr"); \ 1173 kmsan_init_ret(sizeof(int)); \ 1174 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1175 } 1176 1177 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1178 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1179 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1180 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1181 1182 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1183 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1184 type nval) \ 1185 { \ 1186 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1187 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1188 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1189 "atomic_fcmpset_" #name "():ptr"); \ 1190 kmsan_init_ret(sizeof(int)); \ 1191 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1192 } 1193 1194 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1195 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1196 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1197 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1198 1199 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1200 void kmsan_atomic_thread_fence_##name(void) \ 1201 { \ 1202 atomic_thread_fence_##name(); \ 1203 } 1204 1205 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1206 type kmsan_atomic_load_##name(volatile type *ptr) \ 1207 { \ 1208 kmsan_check_arg(sizeof(ptr), \ 1209 "atomic_load_" #name "():args"); \ 1210 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1211 "atomic_load_" #name "():ptr"); \ 1212 kmsan_init_ret(sizeof(type)); \ 1213 return (atomic_load_##name(ptr)); \ 1214 } 1215 1216 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1217 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1218 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1219 1220 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1221 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1222 { \ 1223 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1224 "atomic_store_" #name "():args"); \ 1225 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1226 sizeof(type)); \ 1227 atomic_store_##name(ptr, val); \ 1228 } 1229 1230 #define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1231 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1232 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1233 1234 MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1235 MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1236 MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1237 MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1238 MSAN_ATOMIC_FUNC_ADD(int, u_int); 1239 MSAN_ATOMIC_FUNC_ADD(long, u_long); 1240 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1241 1242 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1243 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1244 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1245 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1246 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1247 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1248 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1249 1250 MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1251 MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1252 MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1253 MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1254 MSAN_ATOMIC_FUNC_SET(int, u_int); 1255 MSAN_ATOMIC_FUNC_SET(long, u_long); 1256 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1257 1258 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1259 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1260 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1261 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1262 MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1263 MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1264 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1265 1266 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1267 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1268 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1269 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1270 1271 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1272 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1273 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1274 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1275 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1276 1277 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1278 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1279 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1280 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1281 1282 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1283 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1284 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1285 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1286 1287 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1288 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1289 MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1290 MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1291 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1292 1293 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1294 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1295 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1296 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1297 MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1298 MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1299 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1300 1301 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1302 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1303 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1304 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1305 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1306 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1307 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1308 1309 _MSAN_ATOMIC_FUNC_LOAD(bool, bool); 1310 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1311 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1312 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1313 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1314 MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1315 MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1316 MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1317 MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1318 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1319 1320 _MSAN_ATOMIC_FUNC_STORE(bool, bool); 1321 MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1322 MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1323 MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1324 MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1325 MSAN_ATOMIC_FUNC_STORE(char, u_char); 1326 MSAN_ATOMIC_FUNC_STORE(short, u_short); 1327 MSAN_ATOMIC_FUNC_STORE(int, u_int); 1328 MSAN_ATOMIC_FUNC_STORE(long, u_long); 1329 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1330 1331 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1332 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1333 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1334 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1335 1336 void 1337 kmsan_atomic_interrupt_fence(void) 1338 { 1339 atomic_interrupt_fence(); 1340 } 1341 1342 /* -------------------------------------------------------------------------- */ 1343 1344 #include <sys/bus.h> 1345 #include <machine/bus.h> 1346 #include <sys/bus_san.h> 1347 1348 int 1349 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1350 int flags, bus_space_handle_t *handlep) 1351 { 1352 return (bus_space_map(tag, hnd, size, flags, handlep)); 1353 } 1354 1355 void 1356 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1357 bus_size_t size) 1358 { 1359 bus_space_unmap(tag, hnd, size); 1360 } 1361 1362 int 1363 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1364 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1365 { 1366 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1367 } 1368 1369 void 1370 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1371 bus_size_t size) 1372 { 1373 bus_space_free(tag, hnd, size); 1374 } 1375 1376 void 1377 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1378 bus_size_t offset, bus_size_t size, int flags) 1379 { 1380 bus_space_barrier(tag, hnd, offset, size, flags); 1381 } 1382 1383 /* XXXMJ x86-specific */ 1384 #define MSAN_BUS_READ_FUNC(func, width, type) \ 1385 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1386 bus_space_handle_t hnd, bus_size_t offset) \ 1387 { \ 1388 type ret; \ 1389 if ((tag) != X86_BUS_SPACE_IO) \ 1390 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1391 KMSAN_STATE_INITED, (width)); \ 1392 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1393 kmsan_init_ret(sizeof(type)); \ 1394 return (ret); \ 1395 } \ 1396 1397 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1398 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1399 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1400 bus_size_t count) \ 1401 { \ 1402 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1403 (width) * count); \ 1404 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1405 count); \ 1406 } 1407 1408 MSAN_BUS_READ_FUNC(, 1, uint8_t) 1409 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1410 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1411 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1412 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1413 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1414 1415 MSAN_BUS_READ_FUNC(, 2, uint16_t) 1416 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1417 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1418 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1419 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1420 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1421 1422 MSAN_BUS_READ_FUNC(, 4, uint32_t) 1423 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1424 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1425 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1426 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1427 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1428 1429 MSAN_BUS_READ_FUNC(, 8, uint64_t) 1430 1431 #define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1432 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1433 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1434 { \ 1435 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1436 } \ 1437 1438 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1439 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1440 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1441 bus_size_t count) \ 1442 { \ 1443 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1444 "bus_space_write()"); \ 1445 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1446 count); \ 1447 } 1448 1449 MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1450 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1451 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1452 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1453 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1454 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1455 1456 MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1457 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1458 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1459 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1460 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1461 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1462 1463 MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1464 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1465 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1466 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1467 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1468 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1469 1470 MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1471 1472 #define MSAN_BUS_SET_FUNC(func, width, type) \ 1473 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1474 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1475 bus_size_t count) \ 1476 { \ 1477 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1478 count); \ 1479 } 1480 1481 MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1482 MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1483 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1484 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1485 1486 MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1487 MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1488 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1489 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1490 1491 MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1492 MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1493 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1494 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1495 1496 /* -------------------------------------------------------------------------- */ 1497 1498 void 1499 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1500 { 1501 /* 1502 * Some drivers, e.g., nvme, use the same code path for loading device 1503 * read and write requests, and will thus specify both flags. In this 1504 * case we should not do any checking since it will generally lead to 1505 * false positives. 1506 */ 1507 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1508 BUS_DMASYNC_PREWRITE) { 1509 switch (desc->md_type) { 1510 case MEMDESC_VADDR: 1511 kmsan_check(desc->u.md_vaddr, desc->md_len, 1512 "dmasync"); 1513 break; 1514 case MEMDESC_MBUF: 1515 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1516 break; 1517 case 0: 1518 break; 1519 default: 1520 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1521 desc->md_type); 1522 } 1523 } 1524 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1525 switch (desc->md_type) { 1526 case MEMDESC_VADDR: 1527 kmsan_mark(desc->u.md_vaddr, desc->md_len, 1528 KMSAN_STATE_INITED); 1529 break; 1530 case MEMDESC_MBUF: 1531 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1532 break; 1533 case 0: 1534 break; 1535 default: 1536 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1537 desc->md_type); 1538 } 1539 } 1540 } 1541