1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define SAN_RUNTIME 36 37 #include <sys/cdefs.h> 38 #if 0 39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bio.h> 45 #include <sys/buf.h> 46 #include <sys/conf.h> 47 #include <sys/kdb.h> 48 #include <sys/kernel.h> 49 #include <sys/linker.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/memdesc.h> 53 #include <sys/msan.h> 54 #include <sys/proc.h> 55 #include <sys/stack.h> 56 #include <sys/sysctl.h> 57 #include <sys/uio.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 62 #include <machine/msan.h> 63 #include <machine/stdarg.h> 64 65 void kmsan_init_arg(size_t); 66 void kmsan_init_ret(size_t); 67 68 /* -------------------------------------------------------------------------- */ 69 70 /* 71 * Part of the compiler ABI. 72 */ 73 74 typedef struct { 75 uint8_t *shad; 76 msan_orig_t *orig; 77 } msan_meta_t; 78 79 #define MSAN_PARAM_SIZE 800 80 #define MSAN_RETVAL_SIZE 800 81 typedef struct { 82 uint8_t param_shadow[MSAN_PARAM_SIZE]; 83 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 85 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 86 uint64_t va_arg_overflow_size; 87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 88 msan_orig_t retval_origin; 89 } msan_tls_t; 90 91 /* -------------------------------------------------------------------------- */ 92 93 #define MSAN_NCONTEXT 4 94 #define MSAN_ORIG_MASK (~0x3) 95 96 typedef struct kmsan_td { 97 size_t ctx; 98 msan_tls_t tls[MSAN_NCONTEXT]; 99 } msan_td_t; 100 101 static msan_tls_t dummy_tls; 102 103 /* 104 * Use separate dummy regions for loads and stores: stores may mark the region 105 * as uninitialized, and that can trigger false positives. 106 */ 107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 110 static msan_td_t msan_thread0; 111 static bool kmsan_enabled __read_mostly; 112 113 static bool kmsan_reporting = false; 114 115 /* 116 * Avoid clobbering any thread-local state before we panic. 117 */ 118 #define kmsan_panic(f, ...) do { \ 119 kmsan_enabled = false; \ 120 panic(f, __VA_ARGS__); \ 121 } while (0) 122 123 #define REPORT(f, ...) do { \ 124 if (panic_on_violation) { \ 125 kmsan_panic(f, __VA_ARGS__); \ 126 } else { \ 127 struct stack st; \ 128 \ 129 stack_save(&st); \ 130 printf(f "\n", __VA_ARGS__); \ 131 stack_print_ddb(&st); \ 132 } \ 133 } while (0) 134 135 FEATURE(kmsan, "Kernel memory sanitizer"); 136 137 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 138 "KMSAN options"); 139 140 static bool panic_on_violation = 1; 141 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 142 &panic_on_violation, 0, 143 "Panic if an invalid access is detected"); 144 145 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 146 147 /* -------------------------------------------------------------------------- */ 148 149 static inline const char * 150 kmsan_orig_name(int type) 151 { 152 switch (type) { 153 case KMSAN_TYPE_STACK: 154 return ("stack"); 155 case KMSAN_TYPE_KMEM: 156 return ("kmem"); 157 case KMSAN_TYPE_MALLOC: 158 return ("malloc"); 159 case KMSAN_TYPE_UMA: 160 return ("UMA"); 161 default: 162 return ("unknown"); 163 } 164 } 165 166 static void 167 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, 168 const char *hook) 169 { 170 const char *typename; 171 char *var, *fn; 172 uintptr_t ptr; 173 long foff; 174 char buf[128]; 175 int type; 176 177 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 178 return; 179 180 kmsan_reporting = true; 181 __compiler_membar(); 182 183 if (*orig == 0) { 184 REPORT("MSan: Uninitialized memory in %s, offset %zu", 185 hook, off); 186 goto out; 187 } 188 189 kmsan_md_orig_decode(*orig, &type, &ptr); 190 typename = kmsan_orig_name(type); 191 192 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 193 sizeof(buf), &foff) == 0) { 194 REPORT("MSan: Uninitialized %s memory in %s, " 195 "offset %zu/%zu, addr %p, from %s+%#lx", 196 typename, hook, off, size, addr, buf, foff); 197 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 198 /* 199 * The format of the string is: "----var@function". Parse it to 200 * display a nice warning. 201 */ 202 var = (char *)ptr + 4; 203 strlcpy(buf, var, sizeof(buf)); 204 var = buf; 205 fn = strchr(buf, '@'); 206 *fn++ = '\0'; 207 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 208 "variable '%s' from %s", typename, hook, off, var, fn); 209 } else { 210 REPORT("MSan: Uninitialized %s memory in %s, " 211 "offset %zu/%zu, addr %p, PC %p", 212 typename, hook, off, size, addr, (void *)ptr); 213 } 214 215 out: 216 __compiler_membar(); 217 kmsan_reporting = false; 218 } 219 220 static void 221 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 222 { 223 const char *typename; 224 char *var, *fn; 225 uintptr_t ptr; 226 char buf[128]; 227 long foff; 228 int type; 229 230 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 231 return; 232 233 kmsan_reporting = true; 234 __compiler_membar(); 235 236 if (orig == 0) { 237 REPORT("MSan: uninitialized variable in %p", (void *)pc); 238 goto out; 239 } 240 241 kmsan_md_orig_decode(orig, &type, &ptr); 242 typename = kmsan_orig_name(type); 243 244 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 245 sizeof(buf), &foff) == 0) { 246 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 247 typename, buf, foff); 248 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 249 /* 250 * The format of the string is: "----var@function". Parse it to 251 * display a nice warning. 252 */ 253 var = (char *)ptr + 4; 254 strlcpy(buf, var, sizeof(buf)); 255 var = buf; 256 fn = strchr(buf, '@'); 257 *fn++ = '\0'; 258 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 259 } else { 260 REPORT("MSan: Uninitialized %s memory, origin %x", 261 typename, orig); 262 } 263 264 out: 265 __compiler_membar(); 266 kmsan_reporting = false; 267 } 268 269 /* -------------------------------------------------------------------------- */ 270 271 static inline msan_meta_t 272 kmsan_meta_get(const void *addr, size_t size, const bool write) 273 { 274 msan_meta_t ret; 275 276 if (__predict_false(!kmsan_enabled)) { 277 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 278 ret.orig = (msan_orig_t *)msan_dummy_orig; 279 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 280 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 281 ret.orig = (msan_orig_t *)msan_dummy_orig; 282 } else { 283 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 284 ret.orig = 285 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 286 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 287 MSAN_ORIG_MASK); 288 } 289 290 return (ret); 291 } 292 293 static inline void 294 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 295 { 296 msan_orig_t *orig; 297 size_t i; 298 299 if (__predict_false(!kmsan_enabled)) 300 return; 301 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 302 return; 303 304 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 305 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 306 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 307 308 for (i = 0; i < size; i += 4) { 309 orig[i / 4] = o; 310 } 311 } 312 313 static inline void 314 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 315 { 316 uint8_t *shad; 317 318 if (__predict_false(!kmsan_enabled)) 319 return; 320 if (__predict_false(kmsan_md_unsupported(addr))) 321 return; 322 323 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 324 __builtin_memset(shad, c, size); 325 } 326 327 static inline void 328 kmsan_meta_copy(void *dst, const void *src, size_t size) 329 { 330 uint8_t *orig_src, *orig_dst; 331 uint8_t *shad_src, *shad_dst; 332 msan_orig_t *_src, *_dst; 333 size_t i; 334 335 if (__predict_false(!kmsan_enabled)) 336 return; 337 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 338 return; 339 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 340 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 341 return; 342 } 343 344 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 345 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 346 __builtin_memmove(shad_dst, shad_src, size); 347 348 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 349 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 350 for (i = 0; i < size; i++) { 351 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 352 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 353 *_dst = *_src; 354 orig_src++; 355 orig_dst++; 356 } 357 } 358 359 static inline void 360 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 361 { 362 msan_orig_t *orig; 363 uint8_t *shad; 364 size_t i; 365 366 if (__predict_false(!kmsan_enabled)) 367 return; 368 if (__predict_false(kmsan_md_unsupported(addr))) 369 return; 370 371 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 372 for (i = 0; i < size; i++) { 373 if (__predict_true(shad[i] == 0)) 374 continue; 375 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)&shad[i]); 376 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 377 kmsan_report_hook((const char *)addr + i, orig, size, i, hook); 378 break; 379 } 380 } 381 382 void 383 kmsan_init_arg(size_t n) 384 { 385 msan_td_t *mtd; 386 uint8_t *arg; 387 388 if (__predict_false(!kmsan_enabled)) 389 return; 390 if (__predict_false(curthread == NULL)) 391 return; 392 mtd = curthread->td_kmsan; 393 arg = mtd->tls[mtd->ctx].param_shadow; 394 __builtin_memset(arg, 0, n); 395 } 396 397 void 398 kmsan_init_ret(size_t n) 399 { 400 msan_td_t *mtd; 401 uint8_t *arg; 402 403 if (__predict_false(!kmsan_enabled)) 404 return; 405 if (__predict_false(curthread == NULL)) 406 return; 407 mtd = curthread->td_kmsan; 408 arg = mtd->tls[mtd->ctx].retval_shadow; 409 __builtin_memset(arg, 0, n); 410 } 411 412 static void 413 kmsan_check_arg(size_t size, const char *hook) 414 { 415 msan_orig_t *orig; 416 msan_td_t *mtd; 417 uint8_t *arg; 418 size_t ctx, i; 419 420 if (__predict_false(!kmsan_enabled)) 421 return; 422 if (__predict_false(curthread == NULL)) 423 return; 424 mtd = curthread->td_kmsan; 425 ctx = mtd->ctx; 426 arg = mtd->tls[ctx].param_shadow; 427 428 for (i = 0; i < size; i++) { 429 if (__predict_true(arg[i] == 0)) 430 continue; 431 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; 432 kmsan_report_hook((const char *)arg + i, orig, size, i, hook); 433 break; 434 } 435 } 436 437 void 438 kmsan_thread_alloc(struct thread *td) 439 { 440 msan_td_t *mtd; 441 442 if (!kmsan_enabled) 443 return; 444 445 mtd = td->td_kmsan; 446 if (mtd == NULL) { 447 /* We might be recycling a thread. */ 448 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 449 sizeof(int)); 450 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 451 } 452 kmsan_memset(mtd, 0, sizeof(*mtd)); 453 mtd->ctx = 0; 454 455 if (td->td_kstack != 0) 456 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 457 KMSAN_STATE_UNINIT); 458 459 td->td_kmsan = mtd; 460 } 461 462 void 463 kmsan_thread_free(struct thread *td) 464 { 465 msan_td_t *mtd; 466 467 if (!kmsan_enabled) 468 return; 469 if (__predict_false(td == curthread)) 470 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 471 472 mtd = td->td_kmsan; 473 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 474 free(mtd, M_KMSAN); 475 td->td_kmsan = NULL; 476 } 477 478 void kmsan_intr_enter(void); 479 void kmsan_intr_leave(void); 480 481 void 482 kmsan_intr_enter(void) 483 { 484 msan_td_t *mtd; 485 486 if (__predict_false(!kmsan_enabled)) 487 return; 488 489 mtd = curthread->td_kmsan; 490 mtd->ctx++; 491 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 492 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 493 } 494 495 void 496 kmsan_intr_leave(void) 497 { 498 msan_td_t *mtd; 499 500 if (__predict_false(!kmsan_enabled)) 501 return; 502 503 mtd = curthread->td_kmsan; 504 if (__predict_false(mtd->ctx == 0)) 505 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 506 mtd->ctx--; 507 } 508 509 /* -------------------------------------------------------------------------- */ 510 511 void 512 kmsan_shadow_map(vm_offset_t addr, size_t size) 513 { 514 size_t npages, i; 515 vm_offset_t va; 516 517 MPASS(addr % PAGE_SIZE == 0); 518 MPASS(size % PAGE_SIZE == 0); 519 520 if (!kmsan_enabled) 521 return; 522 523 npages = atop(size); 524 525 va = kmsan_md_addr_to_shad(addr); 526 for (i = 0; i < npages; i++) { 527 pmap_san_enter(va + ptoa(i)); 528 } 529 530 va = kmsan_md_addr_to_orig(addr); 531 for (i = 0; i < npages; i++) { 532 pmap_san_enter(va + ptoa(i)); 533 } 534 } 535 536 void 537 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 538 { 539 msan_orig_t orig; 540 541 orig = kmsan_md_orig_encode(type, pc); 542 kmsan_origin_fill(addr, orig, size); 543 } 544 545 void 546 kmsan_mark(const void *addr, size_t size, uint8_t c) 547 { 548 kmsan_shadow_fill((uintptr_t)addr, c, size); 549 } 550 551 void 552 kmsan_mark_bio(const struct bio *bp, uint8_t c) 553 { 554 kmsan_mark(bp->bio_data, bp->bio_length, c); 555 } 556 557 void 558 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 559 { 560 do { 561 if ((m->m_flags & M_EXTPG) == 0) 562 kmsan_mark(m->m_data, m->m_len, c); 563 m = m->m_next; 564 } while (m != NULL); 565 } 566 567 void 568 kmsan_check(const void *p, size_t sz, const char *descr) 569 { 570 kmsan_shadow_check((uintptr_t)p, sz, descr); 571 } 572 573 void 574 kmsan_check_bio(const struct bio *bp, const char *descr) 575 { 576 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 577 } 578 579 void 580 kmsan_check_mbuf(const struct mbuf *m, const char *descr) 581 { 582 do { 583 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 584 } while ((m = m->m_next) != NULL); 585 } 586 587 void 588 kmsan_init(void) 589 { 590 int disabled; 591 592 disabled = 0; 593 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 594 if (disabled) 595 return; 596 597 /* Initialize the TLS for curthread. */ 598 msan_thread0.ctx = 0; 599 thread0.td_kmsan = &msan_thread0; 600 601 /* Now officially enabled. */ 602 kmsan_enabled = true; 603 } 604 605 /* -------------------------------------------------------------------------- */ 606 607 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 608 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 609 610 msan_meta_t 611 __msan_metadata_ptr_for_load_n(void *addr, size_t size) 612 { 613 return (kmsan_meta_get(addr, size, false)); 614 } 615 616 msan_meta_t 617 __msan_metadata_ptr_for_store_n(void *addr, size_t size) 618 { 619 return (kmsan_meta_get(addr, size, true)); 620 } 621 622 #define MSAN_META_FUNC(size) \ 623 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 624 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 625 { \ 626 return (kmsan_meta_get(addr, size, false)); \ 627 } \ 628 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 629 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 630 { \ 631 return (kmsan_meta_get(addr, size, true)); \ 632 } 633 634 MSAN_META_FUNC(1) 635 MSAN_META_FUNC(2) 636 MSAN_META_FUNC(4) 637 MSAN_META_FUNC(8) 638 639 void __msan_instrument_asm_store(const void *, size_t); 640 msan_orig_t __msan_chain_origin(msan_orig_t); 641 void __msan_poison(const void *, size_t); 642 void __msan_unpoison(const void *, size_t); 643 void __msan_poison_alloca(const void *, uint64_t, const char *); 644 void __msan_unpoison_alloca(const void *, uint64_t); 645 void __msan_warning(msan_orig_t); 646 msan_tls_t *__msan_get_context_state(void); 647 648 void 649 __msan_instrument_asm_store(const void *addr, size_t size) 650 { 651 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 652 } 653 654 msan_orig_t 655 __msan_chain_origin(msan_orig_t origin) 656 { 657 return (origin); 658 } 659 660 void 661 __msan_poison(const void *addr, size_t size) 662 { 663 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 664 } 665 666 void 667 __msan_unpoison(const void *addr, size_t size) 668 { 669 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 670 } 671 672 void 673 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 674 { 675 msan_orig_t orig; 676 677 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 678 kmsan_origin_fill(addr, orig, size); 679 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 680 } 681 682 void 683 __msan_unpoison_alloca(const void *addr, uint64_t size) 684 { 685 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 686 } 687 688 void 689 __msan_warning(msan_orig_t origin) 690 { 691 if (__predict_false(!kmsan_enabled)) 692 return; 693 kmsan_report_inline(origin, KMSAN_RET_ADDR); 694 } 695 696 msan_tls_t * 697 __msan_get_context_state(void) 698 { 699 msan_td_t *mtd; 700 701 /* 702 * When APs are started, they execute some C code before curthread is 703 * set. We have to handle that here. 704 */ 705 if (__predict_false(!kmsan_enabled || curthread == NULL)) 706 return (&dummy_tls); 707 mtd = curthread->td_kmsan; 708 return (&mtd->tls[mtd->ctx]); 709 } 710 711 /* -------------------------------------------------------------------------- */ 712 713 /* 714 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 715 * initialized areas properly. 716 */ 717 718 void * 719 kmsan_memcpy(void *dst, const void *src, size_t len) 720 { 721 /* No kmsan_check_arg, because inlined. */ 722 kmsan_init_ret(sizeof(void *)); 723 if (__predict_true(len != 0)) { 724 kmsan_meta_copy(dst, src, len); 725 } 726 return (__builtin_memcpy(dst, src, len)); 727 } 728 729 int 730 kmsan_memcmp(const void *b1, const void *b2, size_t len) 731 { 732 const uint8_t *_b1 = b1, *_b2 = b2; 733 size_t i; 734 735 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 736 "memcmp():args"); 737 kmsan_init_ret(sizeof(int)); 738 739 for (i = 0; i < len; i++) { 740 if (*_b1 != *_b2) { 741 kmsan_shadow_check((uintptr_t)b1, i + 1, 742 "memcmp():arg1"); 743 kmsan_shadow_check((uintptr_t)b2, i + 1, 744 "memcmp():arg2"); 745 return (*_b1 - *_b2); 746 } 747 _b1++, _b2++; 748 } 749 750 return (0); 751 } 752 753 void * 754 kmsan_memset(void *dst, int c, size_t len) 755 { 756 /* No kmsan_check_arg, because inlined. */ 757 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 758 kmsan_init_ret(sizeof(void *)); 759 return (__builtin_memset(dst, c, len)); 760 } 761 762 void * 763 kmsan_memmove(void *dst, const void *src, size_t len) 764 { 765 /* No kmsan_check_arg, because inlined. */ 766 if (__predict_true(len != 0)) { 767 kmsan_meta_copy(dst, src, len); 768 } 769 kmsan_init_ret(sizeof(void *)); 770 return (__builtin_memmove(dst, src, len)); 771 } 772 773 __strong_reference(kmsan_memcpy, __msan_memcpy); 774 __strong_reference(kmsan_memset, __msan_memset); 775 __strong_reference(kmsan_memmove, __msan_memmove); 776 777 char * 778 kmsan_strcpy(char *dst, const char *src) 779 { 780 const char *_src = src; 781 char *_dst = dst; 782 size_t len = 0; 783 784 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 785 786 while (1) { 787 len++; 788 *dst = *src; 789 if (*src == '\0') 790 break; 791 src++, dst++; 792 } 793 794 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 795 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 796 kmsan_init_ret(sizeof(char *)); 797 return (_dst); 798 } 799 800 int 801 kmsan_strcmp(const char *s1, const char *s2) 802 { 803 const char *_s1 = s1, *_s2 = s2; 804 size_t len = 0; 805 806 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 807 kmsan_init_ret(sizeof(int)); 808 809 while (1) { 810 len++; 811 if (*s1 != *s2) 812 break; 813 if (*s1 == '\0') { 814 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 815 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 816 return (0); 817 } 818 s1++, s2++; 819 } 820 821 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 822 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 823 824 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 825 } 826 827 size_t 828 kmsan_strlen(const char *str) 829 { 830 const char *s; 831 832 kmsan_check_arg(sizeof(str), "strlen():args"); 833 834 s = str; 835 while (1) { 836 if (*s == '\0') 837 break; 838 s++; 839 } 840 841 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 842 kmsan_init_ret(sizeof(size_t)); 843 return (s - str); 844 } 845 846 int kmsan_copyin(const void *, void *, size_t); 847 int kmsan_copyout(const void *, void *, size_t); 848 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 849 850 int 851 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 852 { 853 int ret; 854 855 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 856 "copyin():args"); 857 ret = copyin(uaddr, kaddr, len); 858 if (ret == 0) 859 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 860 kmsan_init_ret(sizeof(int)); 861 return (ret); 862 } 863 864 int 865 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 866 { 867 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 868 "copyout():args"); 869 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 870 kmsan_init_ret(sizeof(int)); 871 return (copyout(kaddr, uaddr, len)); 872 } 873 874 int 875 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 876 { 877 size_t _done; 878 int ret; 879 880 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 881 sizeof(len) + sizeof(done), "copyinstr():args"); 882 ret = copyinstr(uaddr, kaddr, len, &_done); 883 if (ret == 0) 884 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 885 if (done != NULL) { 886 *done = _done; 887 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 888 } 889 kmsan_init_ret(sizeof(int)); 890 return (ret); 891 } 892 893 /* -------------------------------------------------------------------------- */ 894 895 int 896 kmsan_fubyte(volatile const void *base) 897 { 898 int ret; 899 900 kmsan_check_arg(sizeof(base), "fubyte(): args"); 901 ret = fubyte(base); 902 kmsan_init_ret(sizeof(int)); 903 return (ret); 904 } 905 906 int 907 kmsan_fuword16(volatile const void *base) 908 { 909 int ret; 910 911 kmsan_check_arg(sizeof(base), "fuword16(): args"); 912 ret = fuword16(base); 913 kmsan_init_ret(sizeof(int)); 914 return (ret); 915 } 916 917 int 918 kmsan_fueword(volatile const void *base, long *val) 919 { 920 int ret; 921 922 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 923 ret = fueword(base, val); 924 if (ret == 0) 925 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 926 sizeof(*val)); 927 kmsan_init_ret(sizeof(int)); 928 return (ret); 929 } 930 931 int 932 kmsan_fueword32(volatile const void *base, int32_t *val) 933 { 934 int ret; 935 936 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 937 ret = fueword32(base, val); 938 if (ret == 0) 939 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 940 sizeof(*val)); 941 kmsan_init_ret(sizeof(int)); 942 return (ret); 943 } 944 945 int 946 kmsan_fueword64(volatile const void *base, int64_t *val) 947 { 948 int ret; 949 950 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 951 ret = fueword64(base, val); 952 if (ret == 0) 953 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 954 sizeof(*val)); 955 kmsan_init_ret(sizeof(int)); 956 return (ret); 957 } 958 959 int 960 kmsan_subyte(volatile void *base, int byte) 961 { 962 int ret; 963 964 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 965 ret = subyte(base, byte); 966 kmsan_init_ret(sizeof(int)); 967 return (ret); 968 } 969 970 int 971 kmsan_suword(volatile void *base, long word) 972 { 973 int ret; 974 975 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 976 ret = suword(base, word); 977 kmsan_init_ret(sizeof(int)); 978 return (ret); 979 } 980 981 int 982 kmsan_suword16(volatile void *base, int word) 983 { 984 int ret; 985 986 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 987 ret = suword16(base, word); 988 kmsan_init_ret(sizeof(int)); 989 return (ret); 990 } 991 992 int 993 kmsan_suword32(volatile void *base, int32_t word) 994 { 995 int ret; 996 997 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 998 ret = suword32(base, word); 999 kmsan_init_ret(sizeof(int)); 1000 return (ret); 1001 } 1002 1003 int 1004 kmsan_suword64(volatile void *base, int64_t word) 1005 { 1006 int ret; 1007 1008 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1009 ret = suword64(base, word); 1010 kmsan_init_ret(sizeof(int)); 1011 return (ret); 1012 } 1013 1014 int 1015 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1016 uint32_t newval) 1017 { 1018 int ret; 1019 1020 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1021 sizeof(newval), "casueword32(): args"); 1022 ret = casueword32(base, oldval, oldvalp, newval); 1023 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1024 sizeof(*oldvalp)); 1025 kmsan_init_ret(sizeof(int)); 1026 return (ret); 1027 } 1028 1029 int 1030 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1031 u_long newval) 1032 { 1033 int ret; 1034 1035 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1036 sizeof(newval), "casueword32(): args"); 1037 ret = casueword(base, oldval, oldvalp, newval); 1038 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1039 sizeof(*oldvalp)); 1040 kmsan_init_ret(sizeof(int)); 1041 return (ret); 1042 } 1043 1044 /* -------------------------------------------------------------------------- */ 1045 1046 #include <machine/atomic.h> 1047 #include <sys/atomic_san.h> 1048 1049 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1050 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1051 { \ 1052 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1053 "atomic_add_" #name "():args"); \ 1054 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1055 "atomic_add_" #name "():ptr"); \ 1056 atomic_add_##name(ptr, val); \ 1057 } 1058 1059 #define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1060 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1061 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1062 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1063 1064 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1065 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1066 { \ 1067 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1068 "atomic_subtract_" #name "():args"); \ 1069 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1070 "atomic_subtract_" #name "():ptr"); \ 1071 atomic_subtract_##name(ptr, val); \ 1072 } 1073 1074 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1075 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1076 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1077 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1078 1079 #define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1080 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1081 { \ 1082 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1083 "atomic_set_" #name "():args"); \ 1084 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1085 "atomic_set_" #name "():ptr"); \ 1086 atomic_set_##name(ptr, val); \ 1087 } 1088 1089 #define MSAN_ATOMIC_FUNC_SET(name, type) \ 1090 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1091 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1092 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1093 1094 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1095 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1096 { \ 1097 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1098 "atomic_clear_" #name "():args"); \ 1099 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1100 "atomic_clear_" #name "():ptr"); \ 1101 atomic_clear_##name(ptr, val); \ 1102 } 1103 1104 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1105 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1106 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1107 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1108 1109 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1110 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1111 { \ 1112 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1113 "atomic_fetchadd_" #name "():args"); \ 1114 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1115 "atomic_fetchadd_" #name "():ptr"); \ 1116 kmsan_init_ret(sizeof(type)); \ 1117 return (atomic_fetchadd_##name(ptr, val)); \ 1118 } 1119 1120 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1121 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1122 { \ 1123 kmsan_check_arg(sizeof(ptr), \ 1124 "atomic_readandclear_" #name "():args"); \ 1125 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1126 "atomic_readandclear_" #name "():ptr"); \ 1127 kmsan_init_ret(sizeof(type)); \ 1128 return (atomic_readandclear_##name(ptr)); \ 1129 } 1130 1131 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1132 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1133 { \ 1134 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1135 "atomic_testandclear_" #name "():args"); \ 1136 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1137 "atomic_testandclear_" #name "():ptr"); \ 1138 kmsan_init_ret(sizeof(int)); \ 1139 return (atomic_testandclear_##name(ptr, v)); \ 1140 } 1141 1142 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1143 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1144 { \ 1145 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1146 "atomic_testandset_" #name "():args"); \ 1147 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1148 "atomic_testandset_" #name "():ptr"); \ 1149 kmsan_init_ret(sizeof(int)); \ 1150 return (atomic_testandset_##name(ptr, v)); \ 1151 } 1152 1153 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1154 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1155 { \ 1156 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1157 "atomic_swap_" #name "():args"); \ 1158 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1159 "atomic_swap_" #name "():ptr"); \ 1160 kmsan_init_ret(sizeof(type)); \ 1161 return (atomic_swap_##name(ptr, val)); \ 1162 } 1163 1164 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1165 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1166 type nval) \ 1167 { \ 1168 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1169 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1170 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1171 "atomic_cmpset_" #name "():ptr"); \ 1172 kmsan_init_ret(sizeof(int)); \ 1173 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1174 } 1175 1176 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1177 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1178 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1179 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1180 1181 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1182 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1183 type nval) \ 1184 { \ 1185 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1186 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1187 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1188 "atomic_fcmpset_" #name "():ptr"); \ 1189 kmsan_init_ret(sizeof(int)); \ 1190 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1191 } 1192 1193 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1194 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1195 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1196 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1197 1198 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1199 void kmsan_atomic_thread_fence_##name(void) \ 1200 { \ 1201 atomic_thread_fence_##name(); \ 1202 } 1203 1204 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1205 type kmsan_atomic_load_##name(volatile type *ptr) \ 1206 { \ 1207 kmsan_check_arg(sizeof(ptr), \ 1208 "atomic_load_" #name "():args"); \ 1209 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1210 "atomic_load_" #name "():ptr"); \ 1211 kmsan_init_ret(sizeof(type)); \ 1212 return (atomic_load_##name(ptr)); \ 1213 } 1214 1215 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1216 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1217 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1218 1219 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1220 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1221 { \ 1222 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1223 "atomic_store_" #name "():args"); \ 1224 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1225 sizeof(type)); \ 1226 atomic_store_##name(ptr, val); \ 1227 } 1228 1229 #define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1230 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1231 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1232 1233 MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1234 MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1235 MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1236 MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1237 MSAN_ATOMIC_FUNC_ADD(int, u_int); 1238 MSAN_ATOMIC_FUNC_ADD(long, u_long); 1239 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1240 1241 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1242 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1243 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1244 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1245 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1246 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1247 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1248 1249 MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1250 MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1251 MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1252 MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1253 MSAN_ATOMIC_FUNC_SET(int, u_int); 1254 MSAN_ATOMIC_FUNC_SET(long, u_long); 1255 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1256 1257 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1258 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1259 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1260 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1261 MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1262 MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1263 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1264 1265 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1266 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1267 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1268 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1269 1270 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1271 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1272 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1273 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1274 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1275 1276 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1277 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1278 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1279 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1280 1281 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1282 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1283 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1284 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1285 1286 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1287 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1288 MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1289 MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1290 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1291 1292 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1293 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1294 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1295 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1296 MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1297 MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1298 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1299 1300 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1301 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1302 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1303 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1304 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1305 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1306 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1307 1308 _MSAN_ATOMIC_FUNC_LOAD(bool, bool); 1309 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1310 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1311 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1312 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1313 MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1314 MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1315 MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1316 MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1317 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1318 1319 _MSAN_ATOMIC_FUNC_STORE(bool, bool); 1320 MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1321 MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1322 MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1323 MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1324 MSAN_ATOMIC_FUNC_STORE(char, u_char); 1325 MSAN_ATOMIC_FUNC_STORE(short, u_short); 1326 MSAN_ATOMIC_FUNC_STORE(int, u_int); 1327 MSAN_ATOMIC_FUNC_STORE(long, u_long); 1328 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1329 1330 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1331 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1332 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1333 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1334 1335 void 1336 kmsan_atomic_interrupt_fence(void) 1337 { 1338 atomic_interrupt_fence(); 1339 } 1340 1341 /* -------------------------------------------------------------------------- */ 1342 1343 #include <sys/bus.h> 1344 #include <machine/bus.h> 1345 #include <sys/bus_san.h> 1346 1347 int 1348 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1349 int flags, bus_space_handle_t *handlep) 1350 { 1351 return (bus_space_map(tag, hnd, size, flags, handlep)); 1352 } 1353 1354 void 1355 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1356 bus_size_t size) 1357 { 1358 bus_space_unmap(tag, hnd, size); 1359 } 1360 1361 int 1362 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1363 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1364 { 1365 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1366 } 1367 1368 void 1369 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1370 bus_size_t size) 1371 { 1372 bus_space_free(tag, hnd, size); 1373 } 1374 1375 void 1376 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1377 bus_size_t offset, bus_size_t size, int flags) 1378 { 1379 bus_space_barrier(tag, hnd, offset, size, flags); 1380 } 1381 1382 /* XXXMJ x86-specific */ 1383 #define MSAN_BUS_READ_FUNC(func, width, type) \ 1384 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1385 bus_space_handle_t hnd, bus_size_t offset) \ 1386 { \ 1387 type ret; \ 1388 if ((tag) != X86_BUS_SPACE_IO) \ 1389 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1390 KMSAN_STATE_INITED, (width)); \ 1391 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1392 kmsan_init_ret(sizeof(type)); \ 1393 return (ret); \ 1394 } \ 1395 1396 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1397 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1398 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1399 bus_size_t count) \ 1400 { \ 1401 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1402 (width) * count); \ 1403 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1404 count); \ 1405 } 1406 1407 MSAN_BUS_READ_FUNC(, 1, uint8_t) 1408 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1409 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1410 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1411 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1412 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1413 1414 MSAN_BUS_READ_FUNC(, 2, uint16_t) 1415 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1416 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1417 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1418 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1419 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1420 1421 MSAN_BUS_READ_FUNC(, 4, uint32_t) 1422 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1423 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1424 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1425 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1426 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1427 1428 MSAN_BUS_READ_FUNC(, 8, uint64_t) 1429 1430 #define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1431 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1432 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1433 { \ 1434 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1435 } \ 1436 1437 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1438 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1439 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1440 bus_size_t count) \ 1441 { \ 1442 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1443 "bus_space_write()"); \ 1444 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1445 count); \ 1446 } 1447 1448 MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1449 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1450 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1451 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1452 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1453 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1454 1455 MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1456 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1457 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1458 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1459 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1460 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1461 1462 MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1463 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1464 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1465 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1466 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1467 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1468 1469 MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1470 1471 #define MSAN_BUS_SET_FUNC(func, width, type) \ 1472 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1473 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1474 bus_size_t count) \ 1475 { \ 1476 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1477 count); \ 1478 } 1479 1480 MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1481 MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1482 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1483 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1484 1485 MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1486 MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1487 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1488 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1489 1490 MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1491 MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1492 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1493 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1494 1495 /* -------------------------------------------------------------------------- */ 1496 1497 void 1498 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1499 { 1500 /* 1501 * Some drivers, e.g., nvme, use the same code path for loading device 1502 * read and write requests, and will thus specify both flags. In this 1503 * case we should not do any checking since it will generally lead to 1504 * false positives. 1505 */ 1506 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1507 BUS_DMASYNC_PREWRITE) { 1508 switch (desc->md_type) { 1509 case MEMDESC_VADDR: 1510 kmsan_check(desc->u.md_vaddr, desc->md_len, 1511 "dmasync"); 1512 break; 1513 case MEMDESC_MBUF: 1514 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1515 break; 1516 case 0: 1517 break; 1518 default: 1519 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1520 desc->md_type); 1521 } 1522 } 1523 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1524 switch (desc->md_type) { 1525 case MEMDESC_VADDR: 1526 kmsan_mark(desc->u.md_vaddr, desc->md_len, 1527 KMSAN_STATE_INITED); 1528 break; 1529 case MEMDESC_MBUF: 1530 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1531 break; 1532 case 0: 1533 break; 1534 default: 1535 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1536 desc->md_type); 1537 } 1538 } 1539 } 1540