1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define SAN_RUNTIME 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 #if 0 40 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bio.h> 46 #include <sys/buf.h> 47 #include <sys/conf.h> 48 #include <sys/kdb.h> 49 #include <sys/kernel.h> 50 #include <sys/linker.h> 51 #include <sys/malloc.h> 52 #include <sys/mbuf.h> 53 #include <sys/memdesc.h> 54 #include <sys/msan.h> 55 #include <sys/proc.h> 56 #include <sys/stack.h> 57 #include <sys/sysctl.h> 58 #include <sys/uio.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 63 #include <machine/msan.h> 64 #include <machine/stdarg.h> 65 66 void kmsan_init_arg(size_t); 67 void kmsan_init_ret(size_t); 68 69 /* -------------------------------------------------------------------------- */ 70 71 /* 72 * Part of the compiler ABI. 73 */ 74 75 typedef struct { 76 uint8_t *shad; 77 msan_orig_t *orig; 78 } msan_meta_t; 79 80 #define MSAN_PARAM_SIZE 800 81 #define MSAN_RETVAL_SIZE 800 82 typedef struct { 83 uint8_t param_shadow[MSAN_PARAM_SIZE]; 84 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 85 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 86 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 87 uint64_t va_arg_overflow_size; 88 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 89 msan_orig_t retval_origin; 90 } msan_tls_t; 91 92 /* -------------------------------------------------------------------------- */ 93 94 #define MSAN_NCONTEXT 4 95 #define MSAN_ORIG_MASK (~0x3) 96 97 typedef struct kmsan_td { 98 size_t ctx; 99 msan_tls_t tls[MSAN_NCONTEXT]; 100 } msan_td_t; 101 102 static msan_tls_t dummy_tls; 103 104 /* 105 * Use separate dummy regions for loads and stores: stores may mark the region 106 * as uninitialized, and that can trigger false positives. 107 */ 108 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 109 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 110 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 111 static msan_td_t msan_thread0; 112 static bool kmsan_enabled __read_mostly; 113 114 static bool kmsan_reporting = false; 115 116 /* 117 * Avoid clobbering any thread-local state before we panic. 118 */ 119 #define kmsan_panic(f, ...) do { \ 120 kmsan_enabled = false; \ 121 panic(f, __VA_ARGS__); \ 122 } while (0) 123 124 #define REPORT(f, ...) do { \ 125 if (panic_on_violation) { \ 126 kmsan_panic(f, __VA_ARGS__); \ 127 } else { \ 128 struct stack st; \ 129 \ 130 stack_save(&st); \ 131 printf(f "\n", __VA_ARGS__); \ 132 stack_print_ddb(&st); \ 133 } \ 134 } while (0) 135 136 FEATURE(kmsan, "Kernel memory sanitizer"); 137 138 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 139 "KMSAN options"); 140 141 static bool panic_on_violation = 1; 142 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 143 &panic_on_violation, 0, 144 "Panic if an invalid access is detected"); 145 146 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 147 148 /* -------------------------------------------------------------------------- */ 149 150 static inline const char * 151 kmsan_orig_name(int type) 152 { 153 switch (type) { 154 case KMSAN_TYPE_STACK: 155 return ("stack"); 156 case KMSAN_TYPE_KMEM: 157 return ("kmem"); 158 case KMSAN_TYPE_MALLOC: 159 return ("malloc"); 160 case KMSAN_TYPE_UMA: 161 return ("UMA"); 162 default: 163 return ("unknown"); 164 } 165 } 166 167 static void 168 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 169 { 170 msan_orig_t *orig; 171 const char *typename; 172 char *var, *fn; 173 uintptr_t ptr; 174 long foff; 175 char buf[128]; 176 int type; 177 178 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 179 return; 180 181 kmsan_reporting = true; 182 __compiler_membar(); 183 184 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 185 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 186 187 if (*orig == 0) { 188 REPORT("MSan: Uninitialized memory in %s, offset %zu", 189 hook, off); 190 goto out; 191 } 192 193 kmsan_md_orig_decode(*orig, &type, &ptr); 194 typename = kmsan_orig_name(type); 195 196 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 197 sizeof(buf), &foff) == 0) { 198 REPORT("MSan: Uninitialized %s memory in %s, " 199 "offset %zu/%zu, addr %p, from %s+%#lx", 200 typename, hook, off, size, addr, buf, foff); 201 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 202 /* 203 * The format of the string is: "----var@function". Parse it to 204 * display a nice warning. 205 */ 206 var = (char *)ptr + 4; 207 strlcpy(buf, var, sizeof(buf)); 208 var = buf; 209 fn = strchr(buf, '@'); 210 *fn++ = '\0'; 211 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 212 "variable '%s' from %s", typename, hook, off, var, fn); 213 } else { 214 REPORT("MSan: Uninitialized %s memory in %s, " 215 "offset %zu/%zu, addr %p, PC %p", 216 typename, hook, off, size, addr, (void *)ptr); 217 } 218 219 out: 220 __compiler_membar(); 221 kmsan_reporting = false; 222 } 223 224 static void 225 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 226 { 227 const char *typename; 228 char *var, *fn; 229 uintptr_t ptr; 230 char buf[128]; 231 long foff; 232 int type; 233 234 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 235 return; 236 237 kmsan_reporting = true; 238 __compiler_membar(); 239 240 if (orig == 0) { 241 REPORT("MSan: uninitialized variable in %p", (void *)pc); 242 goto out; 243 } 244 245 kmsan_md_orig_decode(orig, &type, &ptr); 246 typename = kmsan_orig_name(type); 247 248 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 249 sizeof(buf), &foff) == 0) { 250 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 251 typename, buf, foff); 252 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 253 /* 254 * The format of the string is: "----var@function". Parse it to 255 * display a nice warning. 256 */ 257 var = (char *)ptr + 4; 258 strlcpy(buf, var, sizeof(buf)); 259 var = buf; 260 fn = strchr(buf, '@'); 261 *fn++ = '\0'; 262 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 263 } else { 264 REPORT("MSan: Uninitialized %s memory, origin %x", 265 typename, orig); 266 } 267 268 out: 269 __compiler_membar(); 270 kmsan_reporting = false; 271 } 272 273 /* -------------------------------------------------------------------------- */ 274 275 static inline msan_meta_t 276 kmsan_meta_get(const void *addr, size_t size, const bool write) 277 { 278 msan_meta_t ret; 279 280 if (__predict_false(!kmsan_enabled)) { 281 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 282 ret.orig = (msan_orig_t *)msan_dummy_orig; 283 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 284 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 285 ret.orig = (msan_orig_t *)msan_dummy_orig; 286 } else { 287 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 288 ret.orig = 289 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 290 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 291 MSAN_ORIG_MASK); 292 } 293 294 return (ret); 295 } 296 297 static inline void 298 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 299 { 300 msan_orig_t *orig; 301 size_t i; 302 303 if (__predict_false(!kmsan_enabled)) 304 return; 305 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 306 return; 307 308 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 309 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 310 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 311 312 for (i = 0; i < size; i += 4) { 313 orig[i / 4] = o; 314 } 315 } 316 317 static inline void 318 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 319 { 320 uint8_t *shad; 321 322 if (__predict_false(!kmsan_enabled)) 323 return; 324 if (__predict_false(kmsan_md_unsupported(addr))) 325 return; 326 327 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 328 __builtin_memset(shad, c, size); 329 } 330 331 static inline void 332 kmsan_meta_copy(void *dst, const void *src, size_t size) 333 { 334 uint8_t *orig_src, *orig_dst; 335 uint8_t *shad_src, *shad_dst; 336 msan_orig_t *_src, *_dst; 337 size_t i; 338 339 if (__predict_false(!kmsan_enabled)) 340 return; 341 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 342 return; 343 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 344 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 345 return; 346 } 347 348 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 349 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 350 __builtin_memmove(shad_dst, shad_src, size); 351 352 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 353 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 354 for (i = 0; i < size; i++) { 355 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 356 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 357 *_dst = *_src; 358 orig_src++; 359 orig_dst++; 360 } 361 } 362 363 static inline void 364 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 365 { 366 uint8_t *shad; 367 size_t i; 368 369 if (__predict_false(!kmsan_enabled)) 370 return; 371 if (__predict_false(kmsan_md_unsupported(addr))) 372 return; 373 374 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 375 for (i = 0; i < size; i++) { 376 if (__predict_true(shad[i] == 0)) 377 continue; 378 kmsan_report_hook((const char *)addr + i, size, i, hook); 379 break; 380 } 381 } 382 383 void 384 kmsan_init_arg(size_t n) 385 { 386 msan_td_t *mtd; 387 uint8_t *arg; 388 389 if (__predict_false(!kmsan_enabled)) 390 return; 391 if (__predict_false(curthread == NULL)) 392 return; 393 mtd = curthread->td_kmsan; 394 arg = mtd->tls[mtd->ctx].param_shadow; 395 __builtin_memset(arg, 0, n); 396 } 397 398 void 399 kmsan_init_ret(size_t n) 400 { 401 msan_td_t *mtd; 402 uint8_t *arg; 403 404 if (__predict_false(!kmsan_enabled)) 405 return; 406 if (__predict_false(curthread == NULL)) 407 return; 408 mtd = curthread->td_kmsan; 409 arg = mtd->tls[mtd->ctx].retval_shadow; 410 __builtin_memset(arg, 0, n); 411 } 412 413 static void 414 kmsan_check_arg(size_t size, const char *hook) 415 { 416 msan_td_t *mtd; 417 uint8_t *arg; 418 size_t i; 419 420 if (__predict_false(!kmsan_enabled)) 421 return; 422 if (__predict_false(curthread == NULL)) 423 return; 424 mtd = curthread->td_kmsan; 425 arg = mtd->tls[mtd->ctx].param_shadow; 426 427 for (i = 0; i < size; i++) { 428 if (__predict_true(arg[i] == 0)) 429 continue; 430 kmsan_report_hook((const char *)arg + i, size, i, hook); 431 break; 432 } 433 } 434 435 void 436 kmsan_thread_alloc(struct thread *td) 437 { 438 msan_td_t *mtd; 439 440 if (!kmsan_enabled) 441 return; 442 443 mtd = td->td_kmsan; 444 if (mtd == NULL) { 445 /* We might be recycling a thread. */ 446 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 447 sizeof(int)); 448 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 449 } 450 kmsan_memset(mtd, 0, sizeof(*mtd)); 451 mtd->ctx = 0; 452 453 if (td->td_kstack != 0) 454 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 455 KMSAN_STATE_UNINIT); 456 457 td->td_kmsan = mtd; 458 } 459 460 void 461 kmsan_thread_free(struct thread *td) 462 { 463 msan_td_t *mtd; 464 465 if (!kmsan_enabled) 466 return; 467 if (__predict_false(td == curthread)) 468 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 469 470 mtd = td->td_kmsan; 471 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 472 free(mtd, M_KMSAN); 473 td->td_kmsan = NULL; 474 } 475 476 void kmsan_intr_enter(void); 477 void kmsan_intr_leave(void); 478 479 void 480 kmsan_intr_enter(void) 481 { 482 msan_td_t *mtd; 483 484 if (__predict_false(!kmsan_enabled)) 485 return; 486 487 mtd = curthread->td_kmsan; 488 mtd->ctx++; 489 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 490 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 491 } 492 493 void 494 kmsan_intr_leave(void) 495 { 496 msan_td_t *mtd; 497 498 if (__predict_false(!kmsan_enabled)) 499 return; 500 501 mtd = curthread->td_kmsan; 502 if (__predict_false(mtd->ctx == 0)) 503 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 504 mtd->ctx--; 505 } 506 507 /* -------------------------------------------------------------------------- */ 508 509 void 510 kmsan_shadow_map(vm_offset_t addr, size_t size) 511 { 512 size_t npages, i; 513 vm_offset_t va; 514 515 MPASS(addr % PAGE_SIZE == 0); 516 MPASS(size % PAGE_SIZE == 0); 517 518 if (!kmsan_enabled) 519 return; 520 521 npages = atop(size); 522 523 va = kmsan_md_addr_to_shad(addr); 524 for (i = 0; i < npages; i++) { 525 pmap_san_enter(va + ptoa(i)); 526 } 527 528 va = kmsan_md_addr_to_orig(addr); 529 for (i = 0; i < npages; i++) { 530 pmap_san_enter(va + ptoa(i)); 531 } 532 } 533 534 void 535 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 536 { 537 msan_orig_t orig; 538 539 orig = kmsan_md_orig_encode(type, pc); 540 kmsan_origin_fill(addr, orig, size); 541 } 542 543 void 544 kmsan_mark(const void *addr, size_t size, uint8_t c) 545 { 546 kmsan_shadow_fill((uintptr_t)addr, c, size); 547 } 548 549 void 550 kmsan_mark_bio(const struct bio *bp, uint8_t c) 551 { 552 kmsan_mark(bp->bio_data, bp->bio_length, c); 553 } 554 555 void 556 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 557 { 558 do { 559 if ((m->m_flags & M_EXTPG) == 0) 560 kmsan_mark(m->m_data, m->m_len, c); 561 m = m->m_next; 562 } while (m != NULL); 563 } 564 565 void 566 kmsan_check(const void *p, size_t sz, const char *descr) 567 { 568 kmsan_shadow_check((uintptr_t)p, sz, descr); 569 } 570 571 void 572 kmsan_check_bio(const struct bio *bp, const char *descr) 573 { 574 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 575 } 576 577 void 578 kmsan_check_mbuf(const struct mbuf *m, const char *descr) 579 { 580 do { 581 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 582 } while ((m = m->m_next) != NULL); 583 } 584 585 void 586 kmsan_init(void) 587 { 588 int disabled; 589 590 disabled = 0; 591 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 592 if (disabled) 593 return; 594 595 /* Initialize the TLS for curthread. */ 596 msan_thread0.ctx = 0; 597 thread0.td_kmsan = &msan_thread0; 598 599 /* Now officially enabled. */ 600 kmsan_enabled = true; 601 } 602 603 /* -------------------------------------------------------------------------- */ 604 605 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 606 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 607 608 msan_meta_t 609 __msan_metadata_ptr_for_load_n(void *addr, size_t size) 610 { 611 return (kmsan_meta_get(addr, size, false)); 612 } 613 614 msan_meta_t 615 __msan_metadata_ptr_for_store_n(void *addr, size_t size) 616 { 617 return (kmsan_meta_get(addr, size, true)); 618 } 619 620 #define MSAN_META_FUNC(size) \ 621 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 622 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 623 { \ 624 return (kmsan_meta_get(addr, size, false)); \ 625 } \ 626 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 627 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 628 { \ 629 return (kmsan_meta_get(addr, size, true)); \ 630 } 631 632 MSAN_META_FUNC(1) 633 MSAN_META_FUNC(2) 634 MSAN_META_FUNC(4) 635 MSAN_META_FUNC(8) 636 637 void __msan_instrument_asm_store(const void *, size_t); 638 msan_orig_t __msan_chain_origin(msan_orig_t); 639 void __msan_poison(const void *, size_t); 640 void __msan_unpoison(const void *, size_t); 641 void __msan_poison_alloca(const void *, uint64_t, const char *); 642 void __msan_unpoison_alloca(const void *, uint64_t); 643 void __msan_warning(msan_orig_t); 644 msan_tls_t *__msan_get_context_state(void); 645 646 void 647 __msan_instrument_asm_store(const void *addr, size_t size) 648 { 649 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 650 } 651 652 msan_orig_t 653 __msan_chain_origin(msan_orig_t origin) 654 { 655 return (origin); 656 } 657 658 void 659 __msan_poison(const void *addr, size_t size) 660 { 661 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 662 } 663 664 void 665 __msan_unpoison(const void *addr, size_t size) 666 { 667 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 668 } 669 670 void 671 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 672 { 673 msan_orig_t orig; 674 675 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 676 kmsan_origin_fill(addr, orig, size); 677 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 678 } 679 680 void 681 __msan_unpoison_alloca(const void *addr, uint64_t size) 682 { 683 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 684 } 685 686 void 687 __msan_warning(msan_orig_t origin) 688 { 689 if (__predict_false(!kmsan_enabled)) 690 return; 691 kmsan_report_inline(origin, KMSAN_RET_ADDR); 692 } 693 694 msan_tls_t * 695 __msan_get_context_state(void) 696 { 697 msan_td_t *mtd; 698 699 /* 700 * When APs are started, they execute some C code before curthread is 701 * set. We have to handle that here. 702 */ 703 if (__predict_false(!kmsan_enabled || curthread == NULL)) 704 return (&dummy_tls); 705 mtd = curthread->td_kmsan; 706 return (&mtd->tls[mtd->ctx]); 707 } 708 709 /* -------------------------------------------------------------------------- */ 710 711 /* 712 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 713 * initialized areas properly. 714 */ 715 716 void * 717 kmsan_memcpy(void *dst, const void *src, size_t len) 718 { 719 /* No kmsan_check_arg, because inlined. */ 720 kmsan_init_ret(sizeof(void *)); 721 if (__predict_true(len != 0)) { 722 kmsan_meta_copy(dst, src, len); 723 } 724 return (__builtin_memcpy(dst, src, len)); 725 } 726 727 int 728 kmsan_memcmp(const void *b1, const void *b2, size_t len) 729 { 730 const uint8_t *_b1 = b1, *_b2 = b2; 731 size_t i; 732 733 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 734 "memcmp():args"); 735 kmsan_init_ret(sizeof(int)); 736 737 for (i = 0; i < len; i++) { 738 if (*_b1 != *_b2) { 739 kmsan_shadow_check((uintptr_t)b1, i + 1, 740 "memcmp():arg1"); 741 kmsan_shadow_check((uintptr_t)b2, i + 1, 742 "memcmp():arg2"); 743 return (*_b1 - *_b2); 744 } 745 _b1++, _b2++; 746 } 747 748 return (0); 749 } 750 751 void * 752 kmsan_memset(void *dst, int c, size_t len) 753 { 754 /* No kmsan_check_arg, because inlined. */ 755 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 756 kmsan_init_ret(sizeof(void *)); 757 return (__builtin_memset(dst, c, len)); 758 } 759 760 void * 761 kmsan_memmove(void *dst, const void *src, size_t len) 762 { 763 /* No kmsan_check_arg, because inlined. */ 764 if (__predict_true(len != 0)) { 765 kmsan_meta_copy(dst, src, len); 766 } 767 kmsan_init_ret(sizeof(void *)); 768 return (__builtin_memmove(dst, src, len)); 769 } 770 771 __strong_reference(kmsan_memcpy, __msan_memcpy); 772 __strong_reference(kmsan_memset, __msan_memset); 773 __strong_reference(kmsan_memmove, __msan_memmove); 774 775 char * 776 kmsan_strcpy(char *dst, const char *src) 777 { 778 const char *_src = src; 779 char *_dst = dst; 780 size_t len = 0; 781 782 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 783 784 while (1) { 785 len++; 786 *dst = *src; 787 if (*src == '\0') 788 break; 789 src++, dst++; 790 } 791 792 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 793 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 794 kmsan_init_ret(sizeof(char *)); 795 return (_dst); 796 } 797 798 int 799 kmsan_strcmp(const char *s1, const char *s2) 800 { 801 const char *_s1 = s1, *_s2 = s2; 802 size_t len = 0; 803 804 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 805 kmsan_init_ret(sizeof(int)); 806 807 while (1) { 808 len++; 809 if (*s1 != *s2) 810 break; 811 if (*s1 == '\0') { 812 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 813 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 814 return (0); 815 } 816 s1++, s2++; 817 } 818 819 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 820 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 821 822 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 823 } 824 825 size_t 826 kmsan_strlen(const char *str) 827 { 828 const char *s; 829 830 kmsan_check_arg(sizeof(str), "strlen():args"); 831 832 s = str; 833 while (1) { 834 if (*s == '\0') 835 break; 836 s++; 837 } 838 839 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 840 kmsan_init_ret(sizeof(size_t)); 841 return (s - str); 842 } 843 844 int kmsan_copyin(const void *, void *, size_t); 845 int kmsan_copyout(const void *, void *, size_t); 846 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 847 848 int 849 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 850 { 851 int ret; 852 853 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 854 "copyin():args"); 855 ret = copyin(uaddr, kaddr, len); 856 if (ret == 0) 857 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 858 kmsan_init_ret(sizeof(int)); 859 return (ret); 860 } 861 862 int 863 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 864 { 865 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 866 "copyout():args"); 867 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 868 kmsan_init_ret(sizeof(int)); 869 return (copyout(kaddr, uaddr, len)); 870 } 871 872 int 873 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 874 { 875 size_t _done; 876 int ret; 877 878 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 879 sizeof(len) + sizeof(done), "copyinstr():args"); 880 ret = copyinstr(uaddr, kaddr, len, &_done); 881 if (ret == 0) 882 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 883 if (done != NULL) { 884 *done = _done; 885 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 886 } 887 kmsan_init_ret(sizeof(int)); 888 return (ret); 889 } 890 891 /* -------------------------------------------------------------------------- */ 892 893 int 894 kmsan_fubyte(volatile const void *base) 895 { 896 int ret; 897 898 kmsan_check_arg(sizeof(base), "fubyte(): args"); 899 ret = fubyte(base); 900 kmsan_init_ret(sizeof(int)); 901 return (ret); 902 } 903 904 int 905 kmsan_fuword16(volatile const void *base) 906 { 907 int ret; 908 909 kmsan_check_arg(sizeof(base), "fuword16(): args"); 910 ret = fuword16(base); 911 kmsan_init_ret(sizeof(int)); 912 return (ret); 913 } 914 915 int 916 kmsan_fueword(volatile const void *base, long *val) 917 { 918 int ret; 919 920 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 921 ret = fueword(base, val); 922 if (ret == 0) 923 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 924 sizeof(*val)); 925 kmsan_init_ret(sizeof(int)); 926 return (ret); 927 } 928 929 int 930 kmsan_fueword32(volatile const void *base, int32_t *val) 931 { 932 int ret; 933 934 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 935 ret = fueword32(base, val); 936 if (ret == 0) 937 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 938 sizeof(*val)); 939 kmsan_init_ret(sizeof(int)); 940 return (ret); 941 } 942 943 int 944 kmsan_fueword64(volatile const void *base, int64_t *val) 945 { 946 int ret; 947 948 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 949 ret = fueword64(base, val); 950 if (ret == 0) 951 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 952 sizeof(*val)); 953 kmsan_init_ret(sizeof(int)); 954 return (ret); 955 } 956 957 int 958 kmsan_subyte(volatile void *base, int byte) 959 { 960 int ret; 961 962 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 963 ret = subyte(base, byte); 964 kmsan_init_ret(sizeof(int)); 965 return (ret); 966 } 967 968 int 969 kmsan_suword(volatile void *base, long word) 970 { 971 int ret; 972 973 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 974 ret = suword(base, word); 975 kmsan_init_ret(sizeof(int)); 976 return (ret); 977 } 978 979 int 980 kmsan_suword16(volatile void *base, int word) 981 { 982 int ret; 983 984 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 985 ret = suword16(base, word); 986 kmsan_init_ret(sizeof(int)); 987 return (ret); 988 } 989 990 int 991 kmsan_suword32(volatile void *base, int32_t word) 992 { 993 int ret; 994 995 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 996 ret = suword32(base, word); 997 kmsan_init_ret(sizeof(int)); 998 return (ret); 999 } 1000 1001 int 1002 kmsan_suword64(volatile void *base, int64_t word) 1003 { 1004 int ret; 1005 1006 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1007 ret = suword64(base, word); 1008 kmsan_init_ret(sizeof(int)); 1009 return (ret); 1010 } 1011 1012 int 1013 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1014 uint32_t newval) 1015 { 1016 int ret; 1017 1018 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1019 sizeof(newval), "casueword32(): args"); 1020 ret = casueword32(base, oldval, oldvalp, newval); 1021 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1022 sizeof(*oldvalp)); 1023 kmsan_init_ret(sizeof(int)); 1024 return (ret); 1025 } 1026 1027 int 1028 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1029 u_long newval) 1030 { 1031 int ret; 1032 1033 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1034 sizeof(newval), "casueword32(): args"); 1035 ret = casueword(base, oldval, oldvalp, newval); 1036 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1037 sizeof(*oldvalp)); 1038 kmsan_init_ret(sizeof(int)); 1039 return (ret); 1040 } 1041 1042 /* -------------------------------------------------------------------------- */ 1043 1044 #include <machine/atomic.h> 1045 #include <sys/atomic_san.h> 1046 1047 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1048 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1049 { \ 1050 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1051 "atomic_add_" #name "():args"); \ 1052 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1053 "atomic_add_" #name "():ptr"); \ 1054 atomic_add_##name(ptr, val); \ 1055 } 1056 1057 #define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1058 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1059 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1060 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1061 1062 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1063 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1064 { \ 1065 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1066 "atomic_subtract_" #name "():args"); \ 1067 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1068 "atomic_subtract_" #name "():ptr"); \ 1069 atomic_subtract_##name(ptr, val); \ 1070 } 1071 1072 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1073 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1074 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1075 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1076 1077 #define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1078 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1079 { \ 1080 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1081 "atomic_set_" #name "():args"); \ 1082 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1083 "atomic_set_" #name "():ptr"); \ 1084 atomic_set_##name(ptr, val); \ 1085 } 1086 1087 #define MSAN_ATOMIC_FUNC_SET(name, type) \ 1088 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1089 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1090 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1091 1092 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1093 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1094 { \ 1095 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1096 "atomic_clear_" #name "():args"); \ 1097 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1098 "atomic_clear_" #name "():ptr"); \ 1099 atomic_clear_##name(ptr, val); \ 1100 } 1101 1102 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1103 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1104 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1105 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1106 1107 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1108 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1109 { \ 1110 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1111 "atomic_fetchadd_" #name "():args"); \ 1112 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1113 "atomic_fetchadd_" #name "():ptr"); \ 1114 kmsan_init_ret(sizeof(type)); \ 1115 return (atomic_fetchadd_##name(ptr, val)); \ 1116 } 1117 1118 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1119 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1120 { \ 1121 kmsan_check_arg(sizeof(ptr), \ 1122 "atomic_readandclear_" #name "():args"); \ 1123 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1124 "atomic_readandclear_" #name "():ptr"); \ 1125 kmsan_init_ret(sizeof(type)); \ 1126 return (atomic_readandclear_##name(ptr)); \ 1127 } 1128 1129 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1130 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1131 { \ 1132 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1133 "atomic_testandclear_" #name "():args"); \ 1134 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1135 "atomic_testandclear_" #name "():ptr"); \ 1136 kmsan_init_ret(sizeof(int)); \ 1137 return (atomic_testandclear_##name(ptr, v)); \ 1138 } 1139 1140 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1141 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1142 { \ 1143 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1144 "atomic_testandset_" #name "():args"); \ 1145 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1146 "atomic_testandset_" #name "():ptr"); \ 1147 kmsan_init_ret(sizeof(int)); \ 1148 return (atomic_testandset_##name(ptr, v)); \ 1149 } 1150 1151 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1152 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1153 { \ 1154 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1155 "atomic_swap_" #name "():args"); \ 1156 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1157 "atomic_swap_" #name "():ptr"); \ 1158 kmsan_init_ret(sizeof(type)); \ 1159 return (atomic_swap_##name(ptr, val)); \ 1160 } 1161 1162 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1163 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1164 type nval) \ 1165 { \ 1166 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1167 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1168 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1169 "atomic_cmpset_" #name "():ptr"); \ 1170 kmsan_init_ret(sizeof(int)); \ 1171 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1172 } 1173 1174 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1175 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1176 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1177 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1178 1179 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1180 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1181 type nval) \ 1182 { \ 1183 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1184 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1185 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1186 "atomic_fcmpset_" #name "():ptr"); \ 1187 kmsan_init_ret(sizeof(int)); \ 1188 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1189 } 1190 1191 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1192 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1193 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1194 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1195 1196 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1197 void kmsan_atomic_thread_fence_##name(void) \ 1198 { \ 1199 atomic_thread_fence_##name(); \ 1200 } 1201 1202 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1203 type kmsan_atomic_load_##name(volatile type *ptr) \ 1204 { \ 1205 kmsan_check_arg(sizeof(ptr), \ 1206 "atomic_load_" #name "():args"); \ 1207 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1208 "atomic_load_" #name "():ptr"); \ 1209 kmsan_init_ret(sizeof(type)); \ 1210 return (atomic_load_##name(ptr)); \ 1211 } 1212 1213 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1214 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1215 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1216 1217 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1218 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1219 { \ 1220 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1221 "atomic_store_" #name "():args"); \ 1222 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1223 sizeof(type)); \ 1224 atomic_store_##name(ptr, val); \ 1225 } 1226 1227 #define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1228 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1229 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1230 1231 MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1232 MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1233 MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1234 MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1235 MSAN_ATOMIC_FUNC_ADD(int, u_int); 1236 MSAN_ATOMIC_FUNC_ADD(long, u_long); 1237 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1238 1239 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1240 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1241 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1242 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1243 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1244 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1245 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1246 1247 MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1248 MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1249 MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1250 MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1251 MSAN_ATOMIC_FUNC_SET(int, u_int); 1252 MSAN_ATOMIC_FUNC_SET(long, u_long); 1253 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1254 1255 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1256 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1257 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1258 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1259 MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1260 MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1261 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1262 1263 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1264 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1265 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1266 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1267 1268 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1269 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1270 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1271 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1272 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1273 1274 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1275 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1276 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1277 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1278 1279 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1280 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1281 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1282 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1283 1284 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1285 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1286 MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1287 MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1288 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1289 1290 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1291 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1292 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1293 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1294 MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1295 MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1296 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1297 1298 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1299 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1300 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1301 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1302 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1303 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1304 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1305 1306 _MSAN_ATOMIC_FUNC_LOAD(bool, bool); 1307 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1308 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1309 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1310 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1311 MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1312 MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1313 MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1314 MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1315 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1316 1317 _MSAN_ATOMIC_FUNC_STORE(bool, bool); 1318 MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1319 MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1320 MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1321 MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1322 MSAN_ATOMIC_FUNC_STORE(char, u_char); 1323 MSAN_ATOMIC_FUNC_STORE(short, u_short); 1324 MSAN_ATOMIC_FUNC_STORE(int, u_int); 1325 MSAN_ATOMIC_FUNC_STORE(long, u_long); 1326 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1327 1328 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1329 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1330 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1331 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1332 1333 void 1334 kmsan_atomic_interrupt_fence(void) 1335 { 1336 atomic_interrupt_fence(); 1337 } 1338 1339 /* -------------------------------------------------------------------------- */ 1340 1341 #include <sys/bus.h> 1342 #include <machine/bus.h> 1343 #include <sys/bus_san.h> 1344 1345 int 1346 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1347 int flags, bus_space_handle_t *handlep) 1348 { 1349 return (bus_space_map(tag, hnd, size, flags, handlep)); 1350 } 1351 1352 void 1353 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1354 bus_size_t size) 1355 { 1356 bus_space_unmap(tag, hnd, size); 1357 } 1358 1359 int 1360 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1361 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1362 { 1363 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1364 } 1365 1366 void 1367 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1368 bus_size_t size) 1369 { 1370 bus_space_free(tag, hnd, size); 1371 } 1372 1373 void 1374 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1375 bus_size_t offset, bus_size_t size, int flags) 1376 { 1377 bus_space_barrier(tag, hnd, offset, size, flags); 1378 } 1379 1380 /* XXXMJ x86-specific */ 1381 #define MSAN_BUS_READ_FUNC(func, width, type) \ 1382 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1383 bus_space_handle_t hnd, bus_size_t offset) \ 1384 { \ 1385 type ret; \ 1386 if ((tag) != X86_BUS_SPACE_IO) \ 1387 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1388 KMSAN_STATE_INITED, (width)); \ 1389 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1390 kmsan_init_ret(sizeof(type)); \ 1391 return (ret); \ 1392 } \ 1393 1394 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1395 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1396 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1397 bus_size_t count) \ 1398 { \ 1399 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1400 (width) * count); \ 1401 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1402 count); \ 1403 } 1404 1405 MSAN_BUS_READ_FUNC(, 1, uint8_t) 1406 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1407 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1408 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1409 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1410 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1411 1412 MSAN_BUS_READ_FUNC(, 2, uint16_t) 1413 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1414 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1415 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1416 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1417 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1418 1419 MSAN_BUS_READ_FUNC(, 4, uint32_t) 1420 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1421 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1422 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1423 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1424 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1425 1426 MSAN_BUS_READ_FUNC(, 8, uint64_t) 1427 1428 #define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1429 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1430 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1431 { \ 1432 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1433 } \ 1434 1435 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1436 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1437 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1438 bus_size_t count) \ 1439 { \ 1440 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1441 "bus_space_write()"); \ 1442 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1443 count); \ 1444 } 1445 1446 MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1447 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1448 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1449 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1450 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1451 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1452 1453 MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1454 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1455 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1456 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1457 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1458 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1459 1460 MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1461 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1462 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1463 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1464 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1465 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1466 1467 MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1468 1469 #define MSAN_BUS_SET_FUNC(func, width, type) \ 1470 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1471 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1472 bus_size_t count) \ 1473 { \ 1474 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1475 count); \ 1476 } 1477 1478 MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1479 MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1480 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1481 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1482 1483 MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1484 MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1485 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1486 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1487 1488 MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1489 MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1490 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1491 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1492 1493 /* -------------------------------------------------------------------------- */ 1494 1495 void 1496 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1497 { 1498 /* 1499 * Some drivers, e.g., nvme, use the same code path for loading device 1500 * read and write requests, and will thus specify both flags. In this 1501 * case we should not do any checking since it will generally lead to 1502 * false positives. 1503 */ 1504 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1505 BUS_DMASYNC_PREWRITE) { 1506 switch (desc->md_type) { 1507 case MEMDESC_VADDR: 1508 kmsan_check(desc->u.md_vaddr, desc->md_len, 1509 "dmasync"); 1510 break; 1511 case MEMDESC_MBUF: 1512 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1513 break; 1514 case 0: 1515 break; 1516 default: 1517 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1518 desc->md_type); 1519 } 1520 } 1521 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1522 switch (desc->md_type) { 1523 case MEMDESC_VADDR: 1524 kmsan_mark(desc->u.md_vaddr, desc->md_len, 1525 KMSAN_STATE_INITED); 1526 break; 1527 case MEMDESC_MBUF: 1528 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1529 break; 1530 case 0: 1531 break; 1532 default: 1533 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1534 desc->md_type); 1535 } 1536 } 1537 } 1538